import/export

import_padma.py

@@ -2,6 +2,7 @@
 from __future__ import division
 import os
 import sys
+import hashlib
 
 import_dir = os.path.normpath(os.path.abspath(os.path.dirname(__file__)))
 root_dir = os.path.normpath(os.path.abspath(sys.argv[1]))
@@ -28,26 +29,29 @@ from django.contrib.auth.models import User, Group
 from datetime import datetime
 from ox.utils import json
 import ox
-from item.models import Item
+import monkey_patch.models
+
+from item.models import Item, get_item
 from annotation.models import Annotation
 from archive.models import File
 from urlalias.models import IDAlias, LayerAlias, ListAlias
 from place.models import Place
 from itemlist.models import List
 from django.db import connection, transaction
+from user.models import SessionData
 
 
 os.chdir(import_dir)
 
-with open('users.json') as f: users = json.load(f)
+with open('padma/users.json') as f: users = json.load(f)
 
-with open('padma_files.json') as f: padma  = json.load(f)
+with open('padma/files.json') as f: padma  = json.load(f)
 
-with open('padma_locations.json') as f: locations = json.load(f)
+with open('padma/locations.json') as f: locations = json.load(f)
 
-with open('padma_lists.json') as f: lists = json.load(f)
+with open('padma/lists.json') as f: lists = json.load(f)
 
-with open('padma_data.json') as f: padma_data  = json.load(f)
+with open('padma/data.json') as f: padma_data  = json.load(f)
 
 longest_username = max([len(u['username'].strip()) for u in users]) + 1
 if longest_username > 255:
@@ -56,20 +60,27 @@ if longest_username > 255:
     cursor.execute('ALTER TABLE auth_user ALTER COLUMN username TYPE varchar(%d);'%longest_username)
     transaction.commit_unless_managed()
 
-print "now users"
+print "import users"
 for u in users:
-    username = u['username'].strip()
+    username = u['username'].strip().lower()
     user, created = User.objects.get_or_create(username=username)
     user.email = u['email']
     user.password = u['password']
     user.date_joined = datetime.strptime(u['created'], '%Y-%m-%dT%H:%M:%SZ')
     user.save()
     profile = user.get_profile()
-    if 'admin' in user['groups']:
+    if 'admin' in u['groups']:
         profile.set_level('admin')
     else:
         profile.set_level('member')
     profile.save()
+    if SessionData.objects.filter(user=user).count() == 0:
+        s = SessionData()
+        s.user = user
+        s.session_key = hashlib.sha1(user.username).hexdigest()
+        s.firstseen = s.lastseen = user.date_joined
+        s.timesseen = 1
+        s.save()
     for g in u['groups']:
         if g and g.strip() and g != 'admin':
             group, created = Group.objects.get_or_create(name=g)
@@ -97,12 +108,14 @@ def item_data(data):
 def import_layers(item, layers):
     Annotation.objects.filter(item=item).delete()
     print "importing %d annotations" % len(layers)
+    with transaction.commit_on_success():
         for layer in layers:
             oldLayerId = layer['id']
-        annotation = Annotation(item=item, layer=layer['track'])
+            layer_name = '%ss'%layer['track']
+            annotation = Annotation(item=item, layer=layer_name)
             annotation.start = float(layer['time_in'])/1000
             annotation.end = float(layer['time_out'])/1000
-        username = layer['creator'].strip()
+            username = layer['creator'].strip().lower()
             annotation.user = User.objects.get(username=username)
             annotation.value = layer['value']
             annotation.created = datetime.fromtimestamp(int(layer['created']))
@@ -113,65 +126,76 @@ def import_layers(item, layers):
             alias.new = annotation.public_id
             alias.save()
 
-i=1
 for oldId in sorted(padma, key=lambda x: padma[x]['created']):
-    itemId = ox.to26(i)
+    item = get_item({
-    print '\n', itemId, oldId
+        'title': padma_data[oldId]['title']
-    qs = Item.objects.filter(itemId=itemId) 
+    })
-    if qs.count() == 0:
+    print '\n', oldId, item.itemId
-        item = Item(itemId=itemId)
+    #if True:
-    else:
-        item = qs[0]
-    alias, created = IDAlias.objects.get_or_create(old=oldId)
-    alias.new = itemId
-    alias.save()
-    if True or not item.data:
     data = padma_data[oldId]
     _data = item_data(data)
+    username = _data.pop('creator').strip().lower()
+    item.user = User.objects.get(username=username)
     for key in _data:
         item.data[key] = _data[key]
-        if 'poster_frame' in data:
+    if 'poster_frame' in item.data:
         item.poster_frame = float(item.data.pop('poster_frame')) / 1000
-        if 'published' in data:
+    if 'published' in item.data:
         item.published = datetime.fromtimestamp(int(item.data.pop('published')))
-        if 'created' in data:
+    if 'created' in item.data:
         item.created = datetime.fromtimestamp(int(item.data.pop('created')))
-        if 'modified' in data:
+    if 'modified' in item.data:
         item.modified = datetime.fromtimestamp(int(item.data.pop('modified')))
-        item.level = data.get('public', False) and 0 or 2
+    item.level = not data.get('public', False) and 2 or 0
-        username = item.data.pop('creator').strip()
-        item.user = User.objects.get(username=username)
     item.save()
+    item.make_poster(True)
     import_layers(item, data['layers'])
     #link file
     if oldId in padma:
         if padma[oldId]['oshash']:
             print 'add file', padma[oldId]['oshash']
-                f, created = File.objects.get_or_create(oshash=padma[oldId]['oshash'],
+            oshash = padma[oldId]['oshash']
-                                                        item=item)
+            qs = File.objects.filter(oshash=oshash)
-                f.path = f.get('file', '')
+            if qs.count() == 0:
+                f = File()
+                f.oshash = oshash
+            else:
+                f = qs[0]
+            f.item = item
+            f.path = padma[oldId].get('file', '')
             f.save()
         if 'ogg_oshash' in padma[oldId]:
             print 'add file', padma[oldId]['ogg_oshash']
-                f, created = File.objects.get_or_create(oshash=padma[oldId]['ogg_oshash'],
+            oshash = padma[oldId]['ogg_oshash']
-                                                        item=item)
+            qs = File.objects.filter(oshash=oshash)
-                f.path = f.get('ogg', '')
+            if qs.count() == 0:
+                f = File()
+                f.oshash = oshash
+            else:
+                f = qs[0]
+            f.item = item
+            f.path = padma[oldId].get('ogg', '')
             f.save()
-    i += 1
+    alias, created = IDAlias.objects.get_or_create(old=oldId)
-    print item, item.available
+    alias.new = item.itemId
-
+    alias.save()
+    print item, item.itemId
 #lists
 for l in lists:
-    l['user'] = User.objects.get(username=l['user'])
+    l['user'] = User.objects.get(username=l['user'].strip().lower())
-    p = List(name=l['name'], user=l['user'])
+    p,c = List.objects.get_or_create(name=l['title'], user=l['user'])
     p.type = l['type'] == 'static' and 'static' or 'smart'
     p.status = l['public'] and 'featured' or 'private'
     p.description = l['description']
     p.save()
     if l['type'] == 'static':
-        for v in l['videos']:
+        for v in l['items']:
-            i = Item.objects.get(data__contains=v)
+            try:
+                itemId = IDAlias.objects.get(old=v).new
+                i = Item.objects.get(itemId=itemId)
                 p.add(i)
+            except Item.DoesNotExist:
+                print p.name, v
     else:
         key = l['query']['key']
         value= l['query']['value']
@@ -185,10 +209,19 @@ for l in lists:
 #Places
 for l in locations:
     oldId = l.pop('id')
-    l['user'] = User.objects.get(username=l['user'])
+    if 'user' in l:
+        l['user'] = User.objects.get(username=l['user'].strip().lower())
+    else:
+        l['user'] = User.objects.all().order_by('id')[0]
     l['created'] = datetime.fromtimestamp(int(l['created']))
     l['modified'] = datetime.fromtimestamp(int(l['modified']))
-    p = Place(**l)
+    l['alternativeNames'] = tuple(l['alternativeNames'])
+    l['geoname'] = l['name']
+    p, c = Place.objects.get_or_create(name=l['name'])
+    for key in l:
+        if key != 'annotations':
+            setattr(p, key, l[key])
     p.save()
     #FIXME matches
 
+

padma_dump.py

@@ -38,7 +38,7 @@ data = {}
 for v in Video.select():
       data[v.hid] = v.jsondump()
 
-with open(os.path.join(prefix, 'padma_data.json'), 'w') as f:
+with open(os.path.join(prefix, 'data.json'), 'w') as f:
   json.dump(data, f, indent=2)
 
 users = []
@@ -68,7 +68,7 @@ for v in Video.select().orderBy('id'):
     f['ogg_oshash'] = info['oshash']
     files[v.hid] = f
 
-with open(os.path.join(prefix, 'padma_files.json'), 'w') as f:
+with open(os.path.join(prefix, 'files.json'), 'w') as f:
     json.dump(files, f, indent=2)
 
 lists = []
@@ -88,7 +88,7 @@ for l in List.select().orderBy('id'):
     else:
       data['items'] = [v.hid for v in l.videos]
     lists.append(data)
-with open(os.path.join(prefix, 'padma_lists.json'), 'w') as f:
+with open(os.path.join(prefix, 'lists.json'), 'w') as f:
     json.dump(lists, f, indent=2)
 
 locations = []
@@ -116,7 +116,7 @@ for l in Location.select().orderBy('id'):
       data['annotations'].append(a.hid)
     locations.append(data)
 
-with open(os.path.join(prefix, 'padma_locations.json'), 'w') as f:
+with open(os.path.join(prefix, 'locations.json'), 'w') as f:
     json.dump(locations, f, indent=2)
 
 notes = []
@@ -127,5 +127,5 @@ for n in Notes.select(Notes.q.notes!=''):
     'note': n.notes
   })
 
-with open(os.path.join(prefix, 'padma_notes.json'), 'w') as f:
+with open(os.path.join(prefix, 'notes.json'), 'w') as f:
     json.dump(notes, f, indent=2)