0x2620/padma_migration
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

import/export

Browse source
This commit is contained in:
j 2011-12-05 14:49:34 +01:00
parent f435f2b14b
commit d3725eb30f
2 changed files with 113 additions and 80 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

183
import_padma.py
View file

@ -2,6 +2,7 @@
from __future__ import division from __future__ import division
import os import os
import sys import sys
import hashlib
import_dir = os.path.normpath(os.path.abspath(os.path.dirname(__file__))) import_dir = os.path.normpath(os.path.abspath(os.path.dirname(__file__)))
root_dir = os.path.normpath(os.path.abspath(sys.argv[1])) root_dir = os.path.normpath(os.path.abspath(sys.argv[1]))
@ -28,26 +29,29 @@ from django.contrib.auth.models import User, Group
from datetime import datetime from datetime import datetime
from ox.utils import json from ox.utils import json
import ox import ox
from item.models import Item import monkey_patch.models
from item.models import Item, get_item
from annotation.models import Annotation from annotation.models import Annotation
from archive.models import File from archive.models import File
from urlalias.models import IDAlias, LayerAlias, ListAlias from urlalias.models import IDAlias, LayerAlias, ListAlias
from place.models import Place from place.models import Place
from itemlist.models import List from itemlist.models import List
from django.db import connection, transaction from django.db import connection, transaction
from user.models import SessionData
os.chdir(import_dir) os.chdir(import_dir)
with open('users.json') as f: users = json.load(f) with open('padma/users.json') as f: users = json.load(f)
with open('padma_files.json') as f: padma = json.load(f) with open('padma/files.json') as f: padma = json.load(f)
with open('padma_locations.json') as f: locations = json.load(f) with open('padma/locations.json') as f: locations = json.load(f)
with open('padma_lists.json') as f: lists = json.load(f) with open('padma/lists.json') as f: lists = json.load(f)
with open('padma_data.json') as f: padma_data = json.load(f) with open('padma/data.json') as f: padma_data = json.load(f)
longest_username = max([len(u['username'].strip()) for u in users]) + 1 longest_username = max([len(u['username'].strip()) for u in users]) + 1
if longest_username > 255: if longest_username > 255:
@ -56,20 +60,27 @@ if longest_username > 255:
cursor.execute('ALTER TABLE auth_user ALTER COLUMN username TYPE varchar(%d);'%longest_username) cursor.execute('ALTER TABLE auth_user ALTER COLUMN username TYPE varchar(%d);'%longest_username)
transaction.commit_unless_managed() transaction.commit_unless_managed()
print "now users" print "import users"
for u in users: for u in users:
username = u['username'].strip() username = u['username'].strip().lower()
user, created = User.objects.get_or_create(username=username) user, created = User.objects.get_or_create(username=username)
user.email = u['email'] user.email = u['email']
user.password = u['password'] user.password = u['password']
user.date_joined = datetime.strptime(u['created'], '%Y-%m-%dT%H:%M:%SZ') user.date_joined = datetime.strptime(u['created'], '%Y-%m-%dT%H:%M:%SZ')
user.save() user.save()
profile = user.get_profile() profile = user.get_profile()
if 'admin' in user['groups']: if 'admin' in u['groups']:
profile.set_level('admin') profile.set_level('admin')
else: else:
profile.set_level('member') profile.set_level('member')
profile.save() profile.save()
if SessionData.objects.filter(user=user).count() == 0:
s = SessionData()
s.user = user
s.session_key = hashlib.sha1(user.username).hexdigest()
s.firstseen = s.lastseen = user.date_joined
s.timesseen = 1
s.save()
for g in u['groups']: for g in u['groups']:
if g and g.strip() and g != 'admin': if g and g.strip() and g != 'admin':
group, created = Group.objects.get_or_create(name=g) group, created = Group.objects.get_or_create(name=g)
@ -97,81 +108,94 @@ def item_data(data):
def import_layers(item, layers): def import_layers(item, layers):
Annotation.objects.filter(item=item).delete() Annotation.objects.filter(item=item).delete()
print "importing %d annotations" % len(layers) print "importing %d annotations" % len(layers)
for layer in layers: with transaction.commit_on_success():
oldLayerId = layer['id'] for layer in layers:
annotation = Annotation(item=item, layer=layer['track']) oldLayerId = layer['id']
annotation.start = float(layer['time_in'])/1000 layer_name = '%ss'%layer['track']
annotation.end = float(layer['time_out'])/1000 annotation = Annotation(item=item, layer=layer_name)
username = layer['creator'].strip() annotation.start = float(layer['time_in'])/1000
annotation.user = User.objects.get(username=username) annotation.end = float(layer['time_out'])/1000
annotation.value = layer['value'] username = layer['creator'].strip().lower()
annotation.created = datetime.fromtimestamp(int(layer['created'])) annotation.user = User.objects.get(username=username)
annotation.modified = datetime.fromtimestamp(int(layer['modified'])) annotation.value = layer['value']
annotation.save() annotation.created = datetime.fromtimestamp(int(layer['created']))
#migration alias annotation.modified = datetime.fromtimestamp(int(layer['modified']))
alias, created = LayerAlias.objects.get_or_create(old=oldLayerId) annotation.save()
alias.new = annotation.public_id #migration alias
alias.save() alias, created = LayerAlias.objects.get_or_create(old=oldLayerId)
alias.new = annotation.public_id
alias.save()
i=1
for oldId in sorted(padma, key=lambda x: padma[x]['created']): for oldId in sorted(padma, key=lambda x: padma[x]['created']):
itemId = ox.to26(i) item = get_item({
print '\n', itemId, oldId 'title': padma_data[oldId]['title']
qs = Item.objects.filter(itemId=itemId) })
if qs.count() == 0: print '\n', oldId, item.itemId
item = Item(itemId=itemId) #if True:
else: data = padma_data[oldId]
item = qs[0] _data = item_data(data)
username = _data.pop('creator').strip().lower()
item.user = User.objects.get(username=username)
for key in _data:
item.data[key] = _data[key]
if 'poster_frame' in item.data:
item.poster_frame = float(item.data.pop('poster_frame')) / 1000
if 'published' in item.data:
item.published = datetime.fromtimestamp(int(item.data.pop('published')))
if 'created' in item.data:
item.created = datetime.fromtimestamp(int(item.data.pop('created')))
if 'modified' in item.data:
item.modified = datetime.fromtimestamp(int(item.data.pop('modified')))
item.level = not data.get('public', False) and 2 or 0
item.save()
item.make_poster(True)
import_layers(item, data['layers'])
#link file
if oldId in padma:
if padma[oldId]['oshash']:
print 'add file', padma[oldId]['oshash']
oshash = padma[oldId]['oshash']
qs = File.objects.filter(oshash=oshash)
if qs.count() == 0:
f = File()
f.oshash = oshash
else:
f = qs[0]
f.item = item
f.path = padma[oldId].get('file', '')
f.save()
if 'ogg_oshash' in padma[oldId]:
print 'add file', padma[oldId]['ogg_oshash']
oshash = padma[oldId]['ogg_oshash']
qs = File.objects.filter(oshash=oshash)
if qs.count() == 0:
f = File()
f.oshash = oshash
else:
f = qs[0]
f.item = item
f.path = padma[oldId].get('ogg', '')
f.save()
alias, created = IDAlias.objects.get_or_create(old=oldId) alias, created = IDAlias.objects.get_or_create(old=oldId)
alias.new = itemId alias.new = item.itemId
alias.save() alias.save()
if True or not item.data: print item, item.itemId
data = padma_data[oldId]
_data = item_data(data)
for key in _data:
item.data[key] = _data[key]
if 'poster_frame' in data:
item.poster_frame = float(item.data.pop('poster_frame')) / 1000
if 'published' in data:
item.published = datetime.fromtimestamp(int(item.data.pop('published')))
if 'created' in data:
item.created = datetime.fromtimestamp(int(item.data.pop('created')))
if 'modified' in data:
item.modified = datetime.fromtimestamp(int(item.data.pop('modified')))
item.level = data.get('public', False) and 0 or 2
username = item.data.pop('creator').strip()
item.user = User.objects.get(username=username)
item.save()
import_layers(item, data['layers'])
#link file
if oldId in padma:
if padma[oldId]['oshash']:
print 'add file', padma[oldId]['oshash']
f, created = File.objects.get_or_create(oshash=padma[oldId]['oshash'],
item=item)
f.path = f.get('file', '')
f.save()
if 'ogg_oshash' in padma[oldId]:
print 'add file', padma[oldId]['ogg_oshash']
f, created = File.objects.get_or_create(oshash=padma[oldId]['ogg_oshash'],
item=item)
f.path = f.get('ogg', '')
f.save()
i += 1
print item, item.available
#lists #lists
for l in lists: for l in lists:
l['user'] = User.objects.get(username=l['user']) l['user'] = User.objects.get(username=l['user'].strip().lower())
p = List(name=l['name'], user=l['user']) p,c = List.objects.get_or_create(name=l['title'], user=l['user'])
p.type = l['type'] == 'static' and 'static' or 'smart' p.type = l['type'] == 'static' and 'static' or 'smart'
p.status = l['public'] and 'featured' or 'private' p.status = l['public'] and 'featured' or 'private'
p.description = l['description'] p.description = l['description']
p.save() p.save()
if l['type'] == 'static': if l['type'] == 'static':
for v in l['videos']: for v in l['items']:
i = Item.objects.get(data__contains=v) try:
p.add(i) itemId = IDAlias.objects.get(old=v).new
i = Item.objects.get(itemId=itemId)
p.add(i)
except Item.DoesNotExist:
print p.name, v
else: else:
key = l['query']['key'] key = l['query']['key']
value= l['query']['value'] value= l['query']['value']
@ -185,10 +209,19 @@ for l in lists:
#Places #Places
for l in locations: for l in locations:
oldId = l.pop('id') oldId = l.pop('id')
l['user'] = User.objects.get(username=l['user']) if 'user' in l:
l['user'] = User.objects.get(username=l['user'].strip().lower())
else:
l['user'] = User.objects.all().order_by('id')[0]
l['created'] = datetime.fromtimestamp(int(l['created'])) l['created'] = datetime.fromtimestamp(int(l['created']))
l['modified'] = datetime.fromtimestamp(int(l['modified'])) l['modified'] = datetime.fromtimestamp(int(l['modified']))
p = Place(**l) l['alternativeNames'] = tuple(l['alternativeNames'])
l['geoname'] = l['name']
p, c = Place.objects.get_or_create(name=l['name'])
for key in l:
if key != 'annotations':
setattr(p, key, l[key])
p.save() p.save()
#FIXME matches #FIXME matches

10
padma_dump.py
View file

@ -38,7 +38,7 @@ data = {}
for v in Video.select(): for v in Video.select():
data[v.hid] = v.jsondump() data[v.hid] = v.jsondump()
with open(os.path.join(prefix, 'padma_data.json'), 'w') as f: with open(os.path.join(prefix, 'data.json'), 'w') as f:
json.dump(data, f, indent=2) json.dump(data, f, indent=2)
users = [] users = []
@ -68,7 +68,7 @@ for v in Video.select().orderBy('id'):
f['ogg_oshash'] = info['oshash'] f['ogg_oshash'] = info['oshash']
files[v.hid] = f files[v.hid] = f
with open(os.path.join(prefix, 'padma_files.json'), 'w') as f: with open(os.path.join(prefix, 'files.json'), 'w') as f:
json.dump(files, f, indent=2) json.dump(files, f, indent=2)
lists = [] lists = []
@ -88,7 +88,7 @@ for l in List.select().orderBy('id'):
else: else:
data['items'] = [v.hid for v in l.videos] data['items'] = [v.hid for v in l.videos]
lists.append(data) lists.append(data)
with open(os.path.join(prefix, 'padma_lists.json'), 'w') as f: with open(os.path.join(prefix, 'lists.json'), 'w') as f:
json.dump(lists, f, indent=2) json.dump(lists, f, indent=2)
locations = [] locations = []
@ -116,7 +116,7 @@ for l in Location.select().orderBy('id'):
data['annotations'].append(a.hid) data['annotations'].append(a.hid)
locations.append(data) locations.append(data)
with open(os.path.join(prefix, 'padma_locations.json'), 'w') as f: with open(os.path.join(prefix, 'locations.json'), 'w') as f:
json.dump(locations, f, indent=2) json.dump(locations, f, indent=2)
notes = [] notes = []
@ -127,5 +127,5 @@ for n in Notes.select(Notes.q.notes!=''):
'note': n.notes 'note': n.notes
}) })
with open(os.path.join(prefix, 'padma_notes.json'), 'w') as f: with open(os.path.join(prefix, 'notes.json'), 'w') as f:
json.dump(notes, f, indent=2) json.dump(notes, f, indent=2)