padma migration
This commit is contained in:
j
commit
272c228326
|
|
@ -0,0 +1,155 @@
|
|||
#!/usr/bin/env python
|
||||
from __future__ import division
|
||||
import os
|
||||
import sys
|
||||
|
||||
import_dir = os.path.normpath(os.path.abspath(os.path.dirname(__file__)))
|
||||
root_dir = os.path.normpath(os.path.abspath(sys.argv[1]))
|
||||
os.chdir(root_dir)
|
||||
|
||||
#using virtualenv's activate_this.py to reorder sys.path
|
||||
activate_this = os.path.join(root_dir, '..', 'bin', 'activate_this.py')
|
||||
execfile(activate_this, dict(__file__=activate_this))
|
||||
|
||||
sys.path.insert(0, root_dir)
|
||||
|
||||
from django.core.management import setup_environ
|
||||
try:
|
||||
import settings # Assumed to be in the same directory.
|
||||
except ImportError:
|
||||
import sys
|
||||
sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n(If the file settings.py does indeed exist, it's causing an ImportError somehow.)\n" % __file__)
|
||||
sys.exit(1)
|
||||
settings.DEBUG=False
|
||||
|
||||
setup_environ(settings)
|
||||
|
||||
from django.contrib.auth.models import User, Group
|
||||
from datetime import datetime
|
||||
from ox.utils import json
|
||||
import ox
|
||||
from item.models import Item
|
||||
from annotation.models import Annotation, Layer
|
||||
from archive.models import File
|
||||
from urlalias.models import IDAlias, LayerAlias
|
||||
from django.db import connection, transaction
|
||||
|
||||
|
||||
os.chdir(import_dir)
|
||||
|
||||
with open('users.json') as f: users = json.load(f)
|
||||
|
||||
with open('padma_files.json') as f: padma = json.load(f)
|
||||
|
||||
with open('padma_data.json') as f: padma_data = json.load(f)
|
||||
|
||||
longest_username = max([len(u['username'].strip()) for u in users]) + 1
|
||||
if longest_username > 255:
|
||||
print "longer usernames, extending table to", longest_username, 'fix in monkey_patch/models.py'
|
||||
cursor = connection.cursor()
|
||||
cursor.execute('ALTER TABLE auth_user ALTER COLUMN username TYPE varchar(%d);'%longest_username)
|
||||
transaction.commit_unless_managed()
|
||||
|
||||
print "now users"
|
||||
for u in users:
|
||||
username = u['username'].strip()
|
||||
user, created = User.objects.get_or_create(username=username)
|
||||
user.email = u['email']
|
||||
user.password = u['password']
|
||||
user.date_joined = datetime.strptime(u['created'], '%Y-%m-%dT%H:%M:%SZ')
|
||||
user.save()
|
||||
profile = user.get_profile()
|
||||
if 'admin' in user['groups']:
|
||||
profile.set_level('admin')
|
||||
else:
|
||||
profile.set_level('member')
|
||||
profile.save()
|
||||
for g in u['groups']:
|
||||
if g and g.strip() and g not 'admin':
|
||||
group, created = Group.objects.get_or_create(name=g)
|
||||
user.groups.add(group)
|
||||
|
||||
def item_data(data):
|
||||
d = {}
|
||||
for key in data:
|
||||
if key in data:
|
||||
value = data[key]
|
||||
if isinstance(value, basestring):
|
||||
value = value.replace('\r\n', '\n').strip()
|
||||
d[{
|
||||
u'id': u'oldId',
|
||||
u'categories': u'category',
|
||||
u'languages': u'language',
|
||||
}.get(key, key)] = value
|
||||
if 'director' in d:
|
||||
d['director'] = unicode(d['director']).strip().split(', ')
|
||||
for key in ('layers', 'duration', 'size', 'public'):
|
||||
if key in d:
|
||||
del d[key]
|
||||
return d
|
||||
|
||||
def import_layers(item, layers):
|
||||
Annotation.objects.filter(item=item).delete()
|
||||
print "importing %d annotations" % len(layers)
|
||||
for layer in layers:
|
||||
oldLayerId = layer['id']
|
||||
annotation = Annotation(item=item, layer=layer['track'])
|
||||
annotation.start = float(layer['time_in'])/1000
|
||||
annotation.end = float(layer['time_out'])/1000
|
||||
username = layer['creator'].strip()
|
||||
annotation.user = User.objects.get(username=username)
|
||||
annotation.value = layer['value']
|
||||
annotation.created = datetime.fromtimestamp(int(layer['created']))
|
||||
annotation.modified = datetime.fromtimestamp(int(layer['modified']))
|
||||
annotation.save()
|
||||
#migration alias
|
||||
alias, created = LayerAlias.objects.get_or_create(old=oldLayerId)
|
||||
alias.new = annotation.public_id
|
||||
alias.save()
|
||||
|
||||
i=1
|
||||
for oldId in sorted(padma, key=lambda x: padma[x]['created']):
|
||||
itemId = ox.to26(i)
|
||||
print '\n', itemId, oldId
|
||||
qs = Item.objects.filter(itemId=itemId)
|
||||
if qs.count() == 0:
|
||||
item = Item(itemId=itemId)
|
||||
else:
|
||||
item = qs[0]
|
||||
alias, created = IDAlias.objects.get_or_create(old=oldId)
|
||||
alias.new = itemId
|
||||
alias.save()
|
||||
if True or not item.data:
|
||||
data = padma_data[oldId]
|
||||
_data = item_data(data)
|
||||
for key in _data:
|
||||
item.data[key] = _data[key]
|
||||
if 'poster_frame' in data:
|
||||
item.poster_frame = float(item.data.pop('poster_frame')) / 1000
|
||||
if 'published' in data:
|
||||
item.published = datetime.fromtimestamp(int(item.data.pop('published')))
|
||||
if 'created' in data:
|
||||
item.created = datetime.fromtimestamp(int(item.data.pop('created')))
|
||||
if 'modified' in data:
|
||||
item.modified = datetime.fromtimestamp(int(item.data.pop('modified')))
|
||||
item.level = data.get('public', False) and 0 or 2
|
||||
username = item.data.pop('creator').strip()
|
||||
item.user = User.objects.get(username=username)
|
||||
item.save()
|
||||
import_layers(item, data['layers'])
|
||||
#link file
|
||||
if oldId in padma:
|
||||
if padma[oldId]['oshash']:
|
||||
print 'add file', padma[oldId]['oshash']
|
||||
f, created = File.objects.get_or_create(oshash=padma[oldId]['oshash'],
|
||||
item=item)
|
||||
f.path = f.get('file', '')
|
||||
f.save()
|
||||
if 'ogg_oshash' in padma[oldId]:
|
||||
print 'add file', padma[oldId]['ogg_oshash']
|
||||
f, created = File.objects.get_or_create(oshash=padma[oldId]['ogg_oshash'],
|
||||
item=item)
|
||||
f.path = f.get('ogg', '')
|
||||
f.save()
|
||||
i += 1
|
||||
print item, item.available
|
||||
|
|
@ -0,0 +1,74 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
# vi:si:et:sw=2:sts=2:ts=2
|
||||
# GPL written 2008 by j@pad.ma
|
||||
|
||||
import pkg_resources
|
||||
pkg_resources.require("TurboGears")
|
||||
|
||||
from turbogears import config, update_config, start_server
|
||||
import cherrypy
|
||||
cherrypy.lowercase_api = True
|
||||
from os.path import *
|
||||
import sys
|
||||
|
||||
# first look on the command line for a desired config file,
|
||||
# if it's not on the command line, then
|
||||
# look for setup.py in this directory. If it's not there, this script is
|
||||
# probably installed
|
||||
if len(sys.argv) > 1:
|
||||
update_config(configfile=sys.argv[1],
|
||||
modulename="padma.config")
|
||||
elif exists(join(dirname(__file__), "setup.py")):
|
||||
update_config(configfile="dev.cfg",modulename="padma.config")
|
||||
else:
|
||||
update_config(configfile="prod.cfg",modulename="padma.config")
|
||||
config.update(dict(package="padma"))
|
||||
|
||||
from padma.model import *
|
||||
|
||||
import os
|
||||
import ox
|
||||
import simplejson as json
|
||||
|
||||
prefix = '/tmp/padma'
|
||||
os.makedirs(prefix)
|
||||
|
||||
data = {}
|
||||
for v in Video.select():
|
||||
data[v.hid] = v.jsondump()
|
||||
|
||||
with open(os.path.join(prefix, 'padma_data.json'), 'w') as f:
|
||||
json.dump(data, f)
|
||||
|
||||
users = []
|
||||
for u in User.select().orderBy('id'):
|
||||
users.append({
|
||||
'id': u.id,
|
||||
'username': u.user_name.strip(),
|
||||
'email': u.email_address,
|
||||
'password': 'sha1$$' + u.password,
|
||||
'created': u.created.strftime('%Y-%m-%dT%H:%M:%SZ'),
|
||||
'groups': [g.name for g in u.groups],
|
||||
})
|
||||
|
||||
with open(os.path.join(prefix, 'users.json'), 'w') as f:
|
||||
json.dump(users, f, indent=2)
|
||||
|
||||
|
||||
files = []
|
||||
for v in Video.select().orderBy('id'):
|
||||
f = {
|
||||
'sha1sum': v.source_hash,
|
||||
'ogg': v.filename,
|
||||
'id': v.hid,
|
||||
'created': int(v.created.strftime('%s'))
|
||||
}
|
||||
info = ox.avinfo(v.filename)
|
||||
f['oshash'] = info.get('metadata', {}).get('SOURCE_OSHASH', '')
|
||||
f['ogg_oshash'] = info['oshash']
|
||||
files.append(f)
|
||||
|
||||
with open(os.path.join(prefix, 'padma_files.json'), 'w') as f:
|
||||
json.dump(files, f, indent=2)
|
||||
|
||||
Loading…
Reference in New Issue