diff --git a/import_padma.py b/import_padma.py index e468c5f..4ac8a9e 100755 --- a/import_padma.py +++ b/import_padma.py @@ -2,6 +2,7 @@ from __future__ import division import os import sys +import hashlib import_dir = os.path.normpath(os.path.abspath(os.path.dirname(__file__))) root_dir = os.path.normpath(os.path.abspath(sys.argv[1])) @@ -28,26 +29,29 @@ from django.contrib.auth.models import User, Group from datetime import datetime from ox.utils import json import ox -from item.models import Item +import monkey_patch.models + +from item.models import Item, get_item from annotation.models import Annotation from archive.models import File from urlalias.models import IDAlias, LayerAlias, ListAlias from place.models import Place from itemlist.models import List from django.db import connection, transaction +from user.models import SessionData os.chdir(import_dir) -with open('users.json') as f: users = json.load(f) +with open('padma/users.json') as f: users = json.load(f) -with open('padma_files.json') as f: padma = json.load(f) +with open('padma/files.json') as f: padma = json.load(f) -with open('padma_locations.json') as f: locations = json.load(f) +with open('padma/locations.json') as f: locations = json.load(f) -with open('padma_lists.json') as f: lists = json.load(f) +with open('padma/lists.json') as f: lists = json.load(f) -with open('padma_data.json') as f: padma_data = json.load(f) +with open('padma/data.json') as f: padma_data = json.load(f) longest_username = max([len(u['username'].strip()) for u in users]) + 1 if longest_username > 255: @@ -56,20 +60,27 @@ if longest_username > 255: cursor.execute('ALTER TABLE auth_user ALTER COLUMN username TYPE varchar(%d);'%longest_username) transaction.commit_unless_managed() -print "now users" +print "import users" for u in users: - username = u['username'].strip() + username = u['username'].strip().lower() user, created = User.objects.get_or_create(username=username) user.email = u['email'] user.password = u['password'] user.date_joined = datetime.strptime(u['created'], '%Y-%m-%dT%H:%M:%SZ') user.save() profile = user.get_profile() - if 'admin' in user['groups']: + if 'admin' in u['groups']: profile.set_level('admin') else: profile.set_level('member') profile.save() + if SessionData.objects.filter(user=user).count() == 0: + s = SessionData() + s.user = user + s.session_key = hashlib.sha1(user.username).hexdigest() + s.firstseen = s.lastseen = user.date_joined + s.timesseen = 1 + s.save() for g in u['groups']: if g and g.strip() and g != 'admin': group, created = Group.objects.get_or_create(name=g) @@ -97,81 +108,94 @@ def item_data(data): def import_layers(item, layers): Annotation.objects.filter(item=item).delete() print "importing %d annotations" % len(layers) - for layer in layers: - oldLayerId = layer['id'] - annotation = Annotation(item=item, layer=layer['track']) - annotation.start = float(layer['time_in'])/1000 - annotation.end = float(layer['time_out'])/1000 - username = layer['creator'].strip() - annotation.user = User.objects.get(username=username) - annotation.value = layer['value'] - annotation.created = datetime.fromtimestamp(int(layer['created'])) - annotation.modified = datetime.fromtimestamp(int(layer['modified'])) - annotation.save() - #migration alias - alias, created = LayerAlias.objects.get_or_create(old=oldLayerId) - alias.new = annotation.public_id - alias.save() + with transaction.commit_on_success(): + for layer in layers: + oldLayerId = layer['id'] + layer_name = '%ss'%layer['track'] + annotation = Annotation(item=item, layer=layer_name) + annotation.start = float(layer['time_in'])/1000 + annotation.end = float(layer['time_out'])/1000 + username = layer['creator'].strip().lower() + annotation.user = User.objects.get(username=username) + annotation.value = layer['value'] + annotation.created = datetime.fromtimestamp(int(layer['created'])) + annotation.modified = datetime.fromtimestamp(int(layer['modified'])) + annotation.save() + #migration alias + alias, created = LayerAlias.objects.get_or_create(old=oldLayerId) + alias.new = annotation.public_id + alias.save() -i=1 for oldId in sorted(padma, key=lambda x: padma[x]['created']): - itemId = ox.to26(i) - print '\n', itemId, oldId - qs = Item.objects.filter(itemId=itemId) - if qs.count() == 0: - item = Item(itemId=itemId) - else: - item = qs[0] + item = get_item({ + 'title': padma_data[oldId]['title'] + }) + print '\n', oldId, item.itemId + #if True: + data = padma_data[oldId] + _data = item_data(data) + username = _data.pop('creator').strip().lower() + item.user = User.objects.get(username=username) + for key in _data: + item.data[key] = _data[key] + if 'poster_frame' in item.data: + item.poster_frame = float(item.data.pop('poster_frame')) / 1000 + if 'published' in item.data: + item.published = datetime.fromtimestamp(int(item.data.pop('published'))) + if 'created' in item.data: + item.created = datetime.fromtimestamp(int(item.data.pop('created'))) + if 'modified' in item.data: + item.modified = datetime.fromtimestamp(int(item.data.pop('modified'))) + item.level = not data.get('public', False) and 2 or 0 + item.save() + item.make_poster(True) + import_layers(item, data['layers']) + #link file + if oldId in padma: + if padma[oldId]['oshash']: + print 'add file', padma[oldId]['oshash'] + oshash = padma[oldId]['oshash'] + qs = File.objects.filter(oshash=oshash) + if qs.count() == 0: + f = File() + f.oshash = oshash + else: + f = qs[0] + f.item = item + f.path = padma[oldId].get('file', '') + f.save() + if 'ogg_oshash' in padma[oldId]: + print 'add file', padma[oldId]['ogg_oshash'] + oshash = padma[oldId]['ogg_oshash'] + qs = File.objects.filter(oshash=oshash) + if qs.count() == 0: + f = File() + f.oshash = oshash + else: + f = qs[0] + f.item = item + f.path = padma[oldId].get('ogg', '') + f.save() alias, created = IDAlias.objects.get_or_create(old=oldId) - alias.new = itemId + alias.new = item.itemId alias.save() - if True or not item.data: - data = padma_data[oldId] - _data = item_data(data) - for key in _data: - item.data[key] = _data[key] - if 'poster_frame' in data: - item.poster_frame = float(item.data.pop('poster_frame')) / 1000 - if 'published' in data: - item.published = datetime.fromtimestamp(int(item.data.pop('published'))) - if 'created' in data: - item.created = datetime.fromtimestamp(int(item.data.pop('created'))) - if 'modified' in data: - item.modified = datetime.fromtimestamp(int(item.data.pop('modified'))) - item.level = data.get('public', False) and 0 or 2 - username = item.data.pop('creator').strip() - item.user = User.objects.get(username=username) - item.save() - import_layers(item, data['layers']) - #link file - if oldId in padma: - if padma[oldId]['oshash']: - print 'add file', padma[oldId]['oshash'] - f, created = File.objects.get_or_create(oshash=padma[oldId]['oshash'], - item=item) - f.path = f.get('file', '') - f.save() - if 'ogg_oshash' in padma[oldId]: - print 'add file', padma[oldId]['ogg_oshash'] - f, created = File.objects.get_or_create(oshash=padma[oldId]['ogg_oshash'], - item=item) - f.path = f.get('ogg', '') - f.save() - i += 1 - print item, item.available - + print item, item.itemId #lists for l in lists: - l['user'] = User.objects.get(username=l['user']) - p = List(name=l['name'], user=l['user']) + l['user'] = User.objects.get(username=l['user'].strip().lower()) + p,c = List.objects.get_or_create(name=l['title'], user=l['user']) p.type = l['type'] == 'static' and 'static' or 'smart' p.status = l['public'] and 'featured' or 'private' p.description = l['description'] p.save() if l['type'] == 'static': - for v in l['videos']: - i = Item.objects.get(data__contains=v) - p.add(i) + for v in l['items']: + try: + itemId = IDAlias.objects.get(old=v).new + i = Item.objects.get(itemId=itemId) + p.add(i) + except Item.DoesNotExist: + print p.name, v else: key = l['query']['key'] value= l['query']['value'] @@ -185,10 +209,19 @@ for l in lists: #Places for l in locations: oldId = l.pop('id') - l['user'] = User.objects.get(username=l['user']) + if 'user' in l: + l['user'] = User.objects.get(username=l['user'].strip().lower()) + else: + l['user'] = User.objects.all().order_by('id')[0] l['created'] = datetime.fromtimestamp(int(l['created'])) l['modified'] = datetime.fromtimestamp(int(l['modified'])) - p = Place(**l) + l['alternativeNames'] = tuple(l['alternativeNames']) + l['geoname'] = l['name'] + p, c = Place.objects.get_or_create(name=l['name']) + for key in l: + if key != 'annotations': + setattr(p, key, l[key]) p.save() #FIXME matches + diff --git a/padma_dump.py b/padma_dump.py index 43af961..8a42c05 100755 --- a/padma_dump.py +++ b/padma_dump.py @@ -38,7 +38,7 @@ data = {} for v in Video.select(): data[v.hid] = v.jsondump() -with open(os.path.join(prefix, 'padma_data.json'), 'w') as f: +with open(os.path.join(prefix, 'data.json'), 'w') as f: json.dump(data, f, indent=2) users = [] @@ -68,7 +68,7 @@ for v in Video.select().orderBy('id'): f['ogg_oshash'] = info['oshash'] files[v.hid] = f -with open(os.path.join(prefix, 'padma_files.json'), 'w') as f: +with open(os.path.join(prefix, 'files.json'), 'w') as f: json.dump(files, f, indent=2) lists = [] @@ -88,7 +88,7 @@ for l in List.select().orderBy('id'): else: data['items'] = [v.hid for v in l.videos] lists.append(data) -with open(os.path.join(prefix, 'padma_lists.json'), 'w') as f: +with open(os.path.join(prefix, 'lists.json'), 'w') as f: json.dump(lists, f, indent=2) locations = [] @@ -116,7 +116,7 @@ for l in Location.select().orderBy('id'): data['annotations'].append(a.hid) locations.append(data) -with open(os.path.join(prefix, 'padma_locations.json'), 'w') as f: +with open(os.path.join(prefix, 'locations.json'), 'w') as f: json.dump(locations, f, indent=2) notes = [] @@ -127,5 +127,5 @@ for n in Notes.select(Notes.q.notes!=''): 'note': n.notes }) -with open(os.path.join(prefix, 'padma_notes.json'), 'w') as f: +with open(os.path.join(prefix, 'notes.json'), 'w') as f: json.dump(notes, f, indent=2)