import/export
This commit is contained in:
j
parent
f435f2b14b
commit
d3725eb30f
183
import_padma.py
183
import_padma.py
|
|
@ -2,6 +2,7 @@
|
|||
from __future__ import division
|
||||
import os
|
||||
import sys
|
||||
import hashlib
|
||||
|
||||
import_dir = os.path.normpath(os.path.abspath(os.path.dirname(__file__)))
|
||||
root_dir = os.path.normpath(os.path.abspath(sys.argv[1]))
|
||||
|
|
@ -28,26 +29,29 @@ from django.contrib.auth.models import User, Group
|
|||
from datetime import datetime
|
||||
from ox.utils import json
|
||||
import ox
|
||||
from item.models import Item
|
||||
import monkey_patch.models
|
||||
|
||||
from item.models import Item, get_item
|
||||
from annotation.models import Annotation
|
||||
from archive.models import File
|
||||
from urlalias.models import IDAlias, LayerAlias, ListAlias
|
||||
from place.models import Place
|
||||
from itemlist.models import List
|
||||
from django.db import connection, transaction
|
||||
from user.models import SessionData
|
||||
|
||||
|
||||
os.chdir(import_dir)
|
||||
|
||||
with open('users.json') as f: users = json.load(f)
|
||||
with open('padma/users.json') as f: users = json.load(f)
|
||||
|
||||
with open('padma_files.json') as f: padma = json.load(f)
|
||||
with open('padma/files.json') as f: padma = json.load(f)
|
||||
|
||||
with open('padma_locations.json') as f: locations = json.load(f)
|
||||
with open('padma/locations.json') as f: locations = json.load(f)
|
||||
|
||||
with open('padma_lists.json') as f: lists = json.load(f)
|
||||
with open('padma/lists.json') as f: lists = json.load(f)
|
||||
|
||||
with open('padma_data.json') as f: padma_data = json.load(f)
|
||||
with open('padma/data.json') as f: padma_data = json.load(f)
|
||||
|
||||
longest_username = max([len(u['username'].strip()) for u in users]) + 1
|
||||
if longest_username > 255:
|
||||
|
|
@ -56,20 +60,27 @@ if longest_username > 255:
|
|||
cursor.execute('ALTER TABLE auth_user ALTER COLUMN username TYPE varchar(%d);'%longest_username)
|
||||
transaction.commit_unless_managed()
|
||||
|
||||
print "now users"
|
||||
print "import users"
|
||||
for u in users:
|
||||
username = u['username'].strip()
|
||||
username = u['username'].strip().lower()
|
||||
user, created = User.objects.get_or_create(username=username)
|
||||
user.email = u['email']
|
||||
user.password = u['password']
|
||||
user.date_joined = datetime.strptime(u['created'], '%Y-%m-%dT%H:%M:%SZ')
|
||||
user.save()
|
||||
profile = user.get_profile()
|
||||
if 'admin' in user['groups']:
|
||||
if 'admin' in u['groups']:
|
||||
profile.set_level('admin')
|
||||
else:
|
||||
profile.set_level('member')
|
||||
profile.save()
|
||||
if SessionData.objects.filter(user=user).count() == 0:
|
||||
s = SessionData()
|
||||
s.user = user
|
||||
s.session_key = hashlib.sha1(user.username).hexdigest()
|
||||
s.firstseen = s.lastseen = user.date_joined
|
||||
s.timesseen = 1
|
||||
s.save()
|
||||
for g in u['groups']:
|
||||
if g and g.strip() and g != 'admin':
|
||||
group, created = Group.objects.get_or_create(name=g)
|
||||
|
|
@ -97,81 +108,94 @@ def item_data(data):
|
|||
def import_layers(item, layers):
|
||||
Annotation.objects.filter(item=item).delete()
|
||||
print "importing %d annotations" % len(layers)
|
||||
for layer in layers:
|
||||
oldLayerId = layer['id']
|
||||
annotation = Annotation(item=item, layer=layer['track'])
|
||||
annotation.start = float(layer['time_in'])/1000
|
||||
annotation.end = float(layer['time_out'])/1000
|
||||
username = layer['creator'].strip()
|
||||
annotation.user = User.objects.get(username=username)
|
||||
annotation.value = layer['value']
|
||||
annotation.created = datetime.fromtimestamp(int(layer['created']))
|
||||
annotation.modified = datetime.fromtimestamp(int(layer['modified']))
|
||||
annotation.save()
|
||||
#migration alias
|
||||
alias, created = LayerAlias.objects.get_or_create(old=oldLayerId)
|
||||
alias.new = annotation.public_id
|
||||
alias.save()
|
||||
with transaction.commit_on_success():
|
||||
for layer in layers:
|
||||
oldLayerId = layer['id']
|
||||
layer_name = '%ss'%layer['track']
|
||||
annotation = Annotation(item=item, layer=layer_name)
|
||||
annotation.start = float(layer['time_in'])/1000
|
||||
annotation.end = float(layer['time_out'])/1000
|
||||
username = layer['creator'].strip().lower()
|
||||
annotation.user = User.objects.get(username=username)
|
||||
annotation.value = layer['value']
|
||||
annotation.created = datetime.fromtimestamp(int(layer['created']))
|
||||
annotation.modified = datetime.fromtimestamp(int(layer['modified']))
|
||||
annotation.save()
|
||||
#migration alias
|
||||
alias, created = LayerAlias.objects.get_or_create(old=oldLayerId)
|
||||
alias.new = annotation.public_id
|
||||
alias.save()
|
||||
|
||||
i=1
|
||||
for oldId in sorted(padma, key=lambda x: padma[x]['created']):
|
||||
itemId = ox.to26(i)
|
||||
print '\n', itemId, oldId
|
||||
qs = Item.objects.filter(itemId=itemId)
|
||||
if qs.count() == 0:
|
||||
item = Item(itemId=itemId)
|
||||
else:
|
||||
item = qs[0]
|
||||
item = get_item({
|
||||
'title': padma_data[oldId]['title']
|
||||
})
|
||||
print '\n', oldId, item.itemId
|
||||
#if True:
|
||||
data = padma_data[oldId]
|
||||
_data = item_data(data)
|
||||
username = _data.pop('creator').strip().lower()
|
||||
item.user = User.objects.get(username=username)
|
||||
for key in _data:
|
||||
item.data[key] = _data[key]
|
||||
if 'poster_frame' in item.data:
|
||||
item.poster_frame = float(item.data.pop('poster_frame')) / 1000
|
||||
if 'published' in item.data:
|
||||
item.published = datetime.fromtimestamp(int(item.data.pop('published')))
|
||||
if 'created' in item.data:
|
||||
item.created = datetime.fromtimestamp(int(item.data.pop('created')))
|
||||
if 'modified' in item.data:
|
||||
item.modified = datetime.fromtimestamp(int(item.data.pop('modified')))
|
||||
item.level = not data.get('public', False) and 2 or 0
|
||||
item.save()
|
||||
item.make_poster(True)
|
||||
import_layers(item, data['layers'])
|
||||
#link file
|
||||
if oldId in padma:
|
||||
if padma[oldId]['oshash']:
|
||||
print 'add file', padma[oldId]['oshash']
|
||||
oshash = padma[oldId]['oshash']
|
||||
qs = File.objects.filter(oshash=oshash)
|
||||
if qs.count() == 0:
|
||||
f = File()
|
||||
f.oshash = oshash
|
||||
else:
|
||||
f = qs[0]
|
||||
f.item = item
|
||||
f.path = padma[oldId].get('file', '')
|
||||
f.save()
|
||||
if 'ogg_oshash' in padma[oldId]:
|
||||
print 'add file', padma[oldId]['ogg_oshash']
|
||||
oshash = padma[oldId]['ogg_oshash']
|
||||
qs = File.objects.filter(oshash=oshash)
|
||||
if qs.count() == 0:
|
||||
f = File()
|
||||
f.oshash = oshash
|
||||
else:
|
||||
f = qs[0]
|
||||
f.item = item
|
||||
f.path = padma[oldId].get('ogg', '')
|
||||
f.save()
|
||||
alias, created = IDAlias.objects.get_or_create(old=oldId)
|
||||
alias.new = itemId
|
||||
alias.new = item.itemId
|
||||
alias.save()
|
||||
if True or not item.data:
|
||||
data = padma_data[oldId]
|
||||
_data = item_data(data)
|
||||
for key in _data:
|
||||
item.data[key] = _data[key]
|
||||
if 'poster_frame' in data:
|
||||
item.poster_frame = float(item.data.pop('poster_frame')) / 1000
|
||||
if 'published' in data:
|
||||
item.published = datetime.fromtimestamp(int(item.data.pop('published')))
|
||||
if 'created' in data:
|
||||
item.created = datetime.fromtimestamp(int(item.data.pop('created')))
|
||||
if 'modified' in data:
|
||||
item.modified = datetime.fromtimestamp(int(item.data.pop('modified')))
|
||||
item.level = data.get('public', False) and 0 or 2
|
||||
username = item.data.pop('creator').strip()
|
||||
item.user = User.objects.get(username=username)
|
||||
item.save()
|
||||
import_layers(item, data['layers'])
|
||||
#link file
|
||||
if oldId in padma:
|
||||
if padma[oldId]['oshash']:
|
||||
print 'add file', padma[oldId]['oshash']
|
||||
f, created = File.objects.get_or_create(oshash=padma[oldId]['oshash'],
|
||||
item=item)
|
||||
f.path = f.get('file', '')
|
||||
f.save()
|
||||
if 'ogg_oshash' in padma[oldId]:
|
||||
print 'add file', padma[oldId]['ogg_oshash']
|
||||
f, created = File.objects.get_or_create(oshash=padma[oldId]['ogg_oshash'],
|
||||
item=item)
|
||||
f.path = f.get('ogg', '')
|
||||
f.save()
|
||||
i += 1
|
||||
print item, item.available
|
||||
|
||||
print item, item.itemId
|
||||
#lists
|
||||
for l in lists:
|
||||
l['user'] = User.objects.get(username=l['user'])
|
||||
p = List(name=l['name'], user=l['user'])
|
||||
l['user'] = User.objects.get(username=l['user'].strip().lower())
|
||||
p,c = List.objects.get_or_create(name=l['title'], user=l['user'])
|
||||
p.type = l['type'] == 'static' and 'static' or 'smart'
|
||||
p.status = l['public'] and 'featured' or 'private'
|
||||
p.description = l['description']
|
||||
p.save()
|
||||
if l['type'] == 'static':
|
||||
for v in l['videos']:
|
||||
i = Item.objects.get(data__contains=v)
|
||||
p.add(i)
|
||||
for v in l['items']:
|
||||
try:
|
||||
itemId = IDAlias.objects.get(old=v).new
|
||||
i = Item.objects.get(itemId=itemId)
|
||||
p.add(i)
|
||||
except Item.DoesNotExist:
|
||||
print p.name, v
|
||||
else:
|
||||
key = l['query']['key']
|
||||
value= l['query']['value']
|
||||
|
|
@ -185,10 +209,19 @@ for l in lists:
|
|||
#Places
|
||||
for l in locations:
|
||||
oldId = l.pop('id')
|
||||
l['user'] = User.objects.get(username=l['user'])
|
||||
if 'user' in l:
|
||||
l['user'] = User.objects.get(username=l['user'].strip().lower())
|
||||
else:
|
||||
l['user'] = User.objects.all().order_by('id')[0]
|
||||
l['created'] = datetime.fromtimestamp(int(l['created']))
|
||||
l['modified'] = datetime.fromtimestamp(int(l['modified']))
|
||||
p = Place(**l)
|
||||
l['alternativeNames'] = tuple(l['alternativeNames'])
|
||||
l['geoname'] = l['name']
|
||||
p, c = Place.objects.get_or_create(name=l['name'])
|
||||
for key in l:
|
||||
if key != 'annotations':
|
||||
setattr(p, key, l[key])
|
||||
p.save()
|
||||
#FIXME matches
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ data = {}
|
|||
for v in Video.select():
|
||||
data[v.hid] = v.jsondump()
|
||||
|
||||
with open(os.path.join(prefix, 'padma_data.json'), 'w') as f:
|
||||
with open(os.path.join(prefix, 'data.json'), 'w') as f:
|
||||
json.dump(data, f, indent=2)
|
||||
|
||||
users = []
|
||||
|
|
@ -68,7 +68,7 @@ for v in Video.select().orderBy('id'):
|
|||
f['ogg_oshash'] = info['oshash']
|
||||
files[v.hid] = f
|
||||
|
||||
with open(os.path.join(prefix, 'padma_files.json'), 'w') as f:
|
||||
with open(os.path.join(prefix, 'files.json'), 'w') as f:
|
||||
json.dump(files, f, indent=2)
|
||||
|
||||
lists = []
|
||||
|
|
@ -88,7 +88,7 @@ for l in List.select().orderBy('id'):
|
|||
else:
|
||||
data['items'] = [v.hid for v in l.videos]
|
||||
lists.append(data)
|
||||
with open(os.path.join(prefix, 'padma_lists.json'), 'w') as f:
|
||||
with open(os.path.join(prefix, 'lists.json'), 'w') as f:
|
||||
json.dump(lists, f, indent=2)
|
||||
|
||||
locations = []
|
||||
|
|
@ -116,7 +116,7 @@ for l in Location.select().orderBy('id'):
|
|||
data['annotations'].append(a.hid)
|
||||
locations.append(data)
|
||||
|
||||
with open(os.path.join(prefix, 'padma_locations.json'), 'w') as f:
|
||||
with open(os.path.join(prefix, 'locations.json'), 'w') as f:
|
||||
json.dump(locations, f, indent=2)
|
||||
|
||||
notes = []
|
||||
|
|
@ -127,5 +127,5 @@ for n in Notes.select(Notes.q.notes!=''):
|
|||
'note': n.notes
|
||||
})
|
||||
|
||||
with open(os.path.join(prefix, 'padma_notes.json'), 'w') as f:
|
||||
with open(os.path.join(prefix, 'notes.json'), 'w') as f:
|
||||
json.dump(notes, f, indent=2)
|
||||
|
|
|
|||
Loading…
Reference in New Issue