0x2620/openmedialibrary
1
0
Fork 0
Code Issues 131 Pull requests Releases Wiki Activity

Open Media Library

Browse source
This commit is contained in:
j 2014-05-04 19:26:43 +02:00
commit 2ee2bc178a
228 changed files with 85988 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

0
oml/__init__.py Normal file
View file

16
oml/__main__.py Normal file
View file

@ -0,0 +1,16 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
import os
import sys
import app
import server
if len(sys.argv) > 1 and sys.argv[1] == 'server':
import server
server.run()
else:
app.manager.run()

6
oml/api.py Normal file
View file

@ -0,0 +1,6 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
import item.api
import user.api

43
oml/app.py Normal file
View file

@ -0,0 +1,43 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from flask import Flask
from flask.ext.script import Manager
from flask.ext.migrate import Migrate, MigrateCommand
import oxflask.api
import settings
from settings import db
import changelog
import item.models
import user.models
import item.person
import item.api
import user.api
import item.views
import commands
app = Flask('openmedialibrary', static_folder=settings.static_path)
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:////%s' % settings.db_path
app.register_blueprint(oxflask.api.app)
app.register_blueprint(item.views.app)
db.init_app(app)
migrate = Migrate(app, db)
manager = Manager(app)
manager.add_command('db', MigrateCommand)
manager.add_command('setup', commands.Setup)
manager.add_command('update_static', commands.UpdateStatic)
manager.add_command('release', commands.Release)
@app.route('/')
@app.route('/<path:path>')
def main(path=None):
return app.send_static_file('html/oml.html')

230
oml/changelog.py Normal file
View file

@ -0,0 +1,230 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
import json
from datetime import datetime
from ed25519_utils import valid
import settings
from settings import db
import state
from websocket import trigger_event
class Changelog(db.Model):
'''
additem itemid metadata from file (info) + OLID
edititem itemid name->id (i.e. olid-> OL...M)
removeitem itemid
addlist name
editlist name {name: newname}
orderlists [name, name, name]
removelist name
additemtolist listname itemid
removeitemfromlist listname itemid
editusername username
editcontact string
addpeer peerid peername
removepeer peerid peername
'''
id = db.Column(db.Integer(), primary_key=True)
created = db.Column(db.DateTime())
user_id = db.Column(db.String(43))
revision = db.Column(db.BigInteger())
data = db.Column(db.Text())
sig = db.Column(db.String(96))
@classmethod
def record(cls, user, action, *args):
c = cls()
c.created = datetime.now()
c.user_id = user.id
c.revision = cls.query.filter_by(user_id=user.id).count()
c.data = json.dumps([action, args])
timestamp = c.timestamp
_data = str(c.revision) + str(timestamp) + c.data
c.sig = settings.sk.sign(_data, encoding='base64')
db.session.add(c)
db.session.commit()
if state.online:
state.nodes.queue('online', 'pushChanges', [c.json()])
@property
def timestamp(self):
return self.created.strftime('%s')
@classmethod
def apply_change(cls, user, change, rebuild=False):
revision, timestamp, sig, data = change
last = Changelog.query.filter_by(user_id=user.id).order_by('-revision').first()
next_revision = last.revision + 1 if last else 0
if revision == next_revision:
_data = str(revision) + str(timestamp) + data
if rebuild:
sig = settings.sk.sign(_data, encoding='base64')
if valid(user.id, _data, sig):
c = cls()
c.created = datetime.now()
c.user_id = user.id
c.revision = revision
c.data = data
c.sig = sig
action, args = json.loads(data)
print 'apply change', action
if getattr(c, 'action_' + action)(user, timestamp, *args):
print 'change applied'
db.session.add(c)
db.session.commit()
return True
else:
print 'INVLAID SIGNATURE ON CHANGE', change
raise Exception, 'invalid signature'
else:
print 'revsion does not match! got', revision, 'expecting', next_revision
return False
def __repr__(self):
return self.data
def verify(self):
_data = str(self.revision) + str(self.timestamp) + self.data
return valid(self.user_id, _data, self.sig)
def json(self):
return [self.revision, self.timestamp, self.sig, self.data]
@classmethod
def restore(cls, user_id, path=None):
from user.models import User
user = User.get_or_create(user_id)
if not path:
path = '/tmp/oml_changelog_%s.json' % user_id
with open(path, 'r') as fd:
for change in fd:
change = json.loads(change)
cls.apply_change(user, change, user_id == settings.USER_ID)
@classmethod
def export(cls, user_id, path=None):
if not path:
path = '/tmp/oml_changelog_%s.json' % user_id
with open(path, 'w') as fd:
for c in cls.query.filter_by(user_id=user_id).order_by('revision'):
fd.write(json.dumps(c.json()) + '\n')
def action_additem(self, user, timestamp, itemid, info):
from item.models import Item
i = Item.get(itemid)
if i and i.timestamp > timestamp:
return True
if not i:
i = Item.get_or_create(itemid, info)
i.users.append(user)
i.update()
return True
def action_edititem(self, user, timestamp, itemid, meta):
from item.models import Item
i = Item.get(itemid)
if i.timestamp > timestamp:
return True
key = meta.keys()[0]
if not meta[key] and i.meta.get('mainid') == key:
print 'remove id mapping', key, meta[key], 'currently', i.meta[key]
i.update_mainid(key, meta[key])
elif meta[key] and (i.meta.get('mainid') != key or meta[key] != i.meta.get(key)):
print 'new mapping', key, meta[key], 'currently', i.meta.get('mainid'), i.meta.get(i.meta.get('mainid'))
i.update_mainid(key, meta[key])
return True
def action_removeitem(self, user, timestamp, itemid):
from item.models import Item
i = Item.get(itemid)
if not i or i.timestamp > timestamp:
return True
i.users.remove(user)
if i.users:
i.update()
else:
db.session.delete(i)
db.session.commit()
return True
def action_addlist(self, user, timestamp, name, query=None):
from user.models import List
l = List.create(user.id, name)
return True
def action_editlist(self, user, timestamp, name, new):
from user.models import List
l = List.get_or_create(user.id, name)
if 'name' in new:
l.name = new['name']
l.save()
return True
def action_orderlists(self, user, timestamp, lists):
from user.models import List
position = 0
for name in lists:
l = List.get_or_create(user.id, name)
l.position = position
l.save()
position += 1
return True
def action_removelist(self, user, timestamp, name):
from user.models import List
l = List.get(user.id, name)
if l:
l.remove()
return True
def action_addlistitem(self, user, timestamp, name, itemid):
from item.models import Item
from user.models import List
l = List.get(user.id, name)
i = Item.get(itemid)
if l and i:
i.lists.append(l)
i.update()
return True
def action_removelistitem(self, user, timestamp, name, itemid):
from item.models import Item
from user.models import List
l = List.get(user.id, name)
i = Item.get(itemid)
if l and i:
i.lists.remove(l)
i.update()
return True
def action_editusername(self, user, timestamp, username):
user.info['username'] = username
user.save()
return True
def action_editcontact(self, user, timestamp, contact):
user.info['contact'] = contact
user.save()
return True
def action_adduser(self, user, timestamp, peerid, username):
from user.models import User
if not 'users' in user.info:
user.info['users'] = {}
user.info['users'][peerid] = username
user.save()
User.get_or_create(peerid)
#fixme, add username to user?
return True
def action_removeuser(self, user, timestamp, peerid):
if 'users' in user.info and peerid in user.info['users']:
del user.info['users'][peerid]
user.save()
#fixme, remove from User table if no other connection exists
return True

115
oml/commands.py Normal file
View file

@ -0,0 +1,115 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from flask.ext.script import Command
class Setup(Command):
"""
setup new node
"""
def run(self):
import setup
setup.create_default_lists()
class UpdateStatic(Command):
"""
setup new node
"""
def run(self):
import subprocess
import os
import settings
def r(*cmd):
print ' '.join(cmd)
return subprocess.call(cmd)
oxjs = os.path.join(settings.static_path, 'oxjs')
if not os.path.exists(oxjs):
r('git', 'clone', 'https://git.0x2620.org/oxjs.git', oxjs)
r('python', os.path.join(oxjs, 'tools', 'build', 'build.py'))
r('python', os.path.join(settings.static_path, 'py', 'build.py'))
class Release(Command):
"""
release new version
"""
def run(self):
print 'checking...'
import settings
import os
import subprocess
import json
import hashlib
import ed25519
from os.path import join, exists, dirname
root_dir = dirname(settings.base_dir)
os.chdir(root_dir)
def run(*cmd):
p = subprocess.Popen(cmd)
p.wait()
return p.returncode
def get(*cmd):
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, error = p.communicate()
return stdout
def version(module):
os.chdir(join(root_dir, module))
version = get('git', 'log', '-1', '--format=%cd', '--date=iso').split(' ')[0].replace('-', '')
version += '-' + get('git', 'rev-list', 'HEAD', '--count').strip()
version += '-' + get('git', 'describe', '--always').strip()
os.chdir(root_dir)
return version
with open(os.path.expanduser('~/Private/openmedialibrary_release.key')) as fd:
SIG_KEY=ed25519.SigningKey(fd.read())
SIG_ENCODING='base64'
def sign(release):
value = []
for module in sorted(release['modules']):
value += ['%s/%s' % (release['modules'][module]['version'], release['modules'][module]['sha1'])]
value = '\n'.join(value)
sig = SIG_KEY.sign(value, encoding=SIG_ENCODING)
release['signature'] = sig
def sha1sum(path):
h = hashlib.sha1()
with open(path) as fd:
for chunk in iter(lambda: fd.read(128*h.block_size), ''):
h.update(chunk)
return h.hexdigest()
MODULES = ['platform', 'openmedialibrary']
VERSIONS = {module:version(module) for module in MODULES}
EXCLUDE=[
'--exclude', '.git', '--exclude', '.bzr',
'--exclude', '.*.swp', '--exclude', '._*', '--exclude', '.DS_Store'
]
#run('./ctl', 'update_static')
for module in MODULES:
tar = join('updates', '%s-%s.tar.bz2' % (module, VERSIONS[module]))
if not exists(tar):
cmd = ['tar', 'cvjf', tar, '%s/' % module] + EXCLUDE
if module in ('openmedialibrary', ):
cmd += ['--exclude', '*.pyc']
if module == 'openmedialibrary':
cmd += ['--exclude', 'oxjs/examples', '--exclude', 'gunicorn.pid']
run(*cmd)
release = {}
release['modules'] = {module: {
'name': '%s-%s.tar.bz2' % (module, VERSIONS[module]),
'version': VERSIONS[module],
'sha1': sha1sum(join('updates', '%s-%s.tar.bz2' % (module, VERSIONS[module])))
} for module in MODULES}
sign(release)
with open('updates/release.json', 'w') as fd:
json.dump(release, fd, indent=2)
print 'signed latest release in updates/release.json'

44
oml/directory.py Normal file
View file

@ -0,0 +1,44 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
# DHT placeholder
import requests
import ed25519
import json
import settings
base = settings.server['directory_service']
def get(vk):
id = vk.to_ascii(encoding='base64')
url ='%s/%s' % (base, id)
r = requests.get(url)
sig = r.headers.get('X-Ed25519-Signature')
data = r.content
if sig and data:
vk = ed25519.VerifyingKey(id, encoding='base64')
try:
vk.verify(sig, data, encoding='base64')
data = json.loads(data)
except ed25519.BadSignatureError:
print 'invalid signature'
data = None
return data
def put(sk, data):
id = sk.get_verifying_key().to_ascii(encoding='base64')
data = json.dumps(data)
sig = sk.sign(data, encoding='base64')
url ='%s/%s' % (base, id)
headers = {
'X-Ed25519-Signature': sig
}
try:
r = requests.put(url, data, headers=headers)
except:
import traceback
print 'directory.put failed:', data
traceback.print_exc()
return False
return r.status_code == 200

46
oml/downloads.py Normal file
View file

@ -0,0 +1,46 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from __future__ import division
from threading import Thread
import time
import state
class Downloads(Thread):
def __init__(self, app):
self._app = app
self._running = True
Thread.__init__(self)
self.daemon = True
self.start()
def download_next(self):
import item.models
for i in item.models.Item.query.filter(
item.models.Item.transferadded!=None).filter(
item.models.Item.transferprogress<1):
print 'DOWNLOAD', i, i.users
for p in i.users:
if state.nodes.check_online(p.id):
r = state.nodes.download(p.id, i)
print 'download ok?', r
return True
return False
def run(self):
time.sleep(2)
with self._app.app_context():
while self._running:
if state.online:
self.download_next()
time.sleep(10)
else:
time.sleep(20)
def join(self):
self._running = False
self._q.put(None)
return Thread.join(self)

14
oml/ed25519_utils.py Normal file
View file

@ -0,0 +1,14 @@
import ed25519
ENCODING='base64'
def valid(key, value, sig):
'''
validate that value was signed by key
'''
vk = ed25519.VerifyingKey(str(key), encoding=ENCODING)
try:
vk.verify(str(sig), str(value), encoding=ENCODING)
#except ed25519.BadSignatureError:
except:
return False
return True

0
oml/item/__init__.py Normal file
View file

19
oml/item/add.py Normal file
View file

@ -0,0 +1,19 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
import base64
import models
import ox
import scan
def add(path):
info = scan.get_metadata(path)
id = info.pop('id')
item = models.Item.get_or_create(id)
item.path = path
item.info = info
models.db.session.add(item)
models.db.session.commit()

210
oml/item/api.py Normal file
View file

@ -0,0 +1,210 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from datetime import datetime
from flask import json
from oxflask.api import actions
from oxflask.shortcuts import returns_json
from oml import utils
import query
import models
import settings
from changelog import Changelog
import re
import state
import utils
@returns_json
def find(request):
'''
find items
'''
response = {}
data = json.loads(request.form['data']) if 'data' in request.form else {}
q = query.parse(data)
if 'group' in q:
response['items'] = []
'''
items = 'items'
item_qs = q['qs']
order_by = query.order_by_group(q)
qs = models.Facet.objects.filter(key=q['group']).filter(item__id__in=item_qs)
qs = qs.values('value').annotate(items=Count('id')).order_by(*order_by)
if 'positions' in q:
response['positions'] = {}
ids = [j['value'] for j in qs]
response['positions'] = utils.get_positions(ids, q['positions'])
elif 'range' in data:
qs = qs[q['range'][0]:q['range'][1]]
response['items'] = [{'name': i['value'], 'items': i[items]} for i in qs]
else:
response['items'] = qs.count()
'''
_g = {}
key = utils.get_by_id(settings.config['itemKeys'], q['group'])
for item in q['qs']:
i = item.json()
if q['group'] in i:
values = i[q['group']]
if isinstance(values, basestring):
values = [values]
for value in values:
if key.get('filterMap') and value:
value = re.compile(key.get('filterMap')).findall(value)
if value:
value = value[0]
else:
continue
if value not in _g:
_g[value] = 0
_g[value] += 1
g = [{'name': k, 'items': _g[k]} for k in _g]
if 'sort' in data: # parse adds default sort to q!
g.sort(key=lambda k: k[q['sort'][0]['key']])
if q['sort'][0]['operator'] == '-':
g.reverse()
if 'positions' in data:
response['positions'] = {}
ids = [k['name'] for k in g]
response['positions'] = utils.get_positions(ids, data['positions'])
elif 'range' in data:
response['items'] = g[q['range'][0]:q['range'][1]]
else:
response['items'] = len(g)
elif 'position' in data:
ids = [i.id for i in q['qs']]
response['position'] = utils.get_positions(ids, [data['qs'][0].id])[0]
elif 'positions' in data:
ids = [i.id for i in q['qs']]
response['positions'] = utils.get_positions(ids, data['positions'])
elif 'keys' in data:
'''
qs = qs[q['range'][0]:q['range'][1]]
response['items'] = [p.json(data['keys']) for p in qs]
'''
response['items'] = []
for i in q['qs'][q['range'][0]:q['range'][1]]:
j = i.json()
response['items'].append({k:j[k] for k in j if not data['keys'] or k in data['keys']})
else:
items = [i.json() for i in q['qs']]
response['items'] = len(items)
response['size'] = sum([i.get('size',0) for i in items])
return response
actions.register(find)
@returns_json
def get(request):
response = {}
data = json.loads(request.form['data']) if 'data' in request.form else {}
item = models.Item.get(data['id'])
if item:
response = item.json(data['keys'] if 'keys' in data else None)
return response
actions.register(get)
@returns_json
def edit(request):
response = {}
data = json.loads(request.form['data']) if 'data' in request.form else {}
print 'edit', data
item = models.Item.get(data['id'])
keys = filter(lambda k: k in models.Item.id_keys, data.keys())
print item, keys
if item and keys and item.json()['mediastate'] == 'available':
key = keys[0]
print 'update mainid', key, data[key]
item.update_mainid(key, data[key])
response = item.json()
else:
print 'can only edit available items'
response = item.json()
return response
actions.register(edit, cache=False)
@returns_json
def identify(request):
'''
takes {
title: string,
author: [string],
publisher: string,
date: string
}
returns {
title: string,
autor: [string],
date: string,
}
'''
response = {}
data = json.loads(request.form['data']) if 'data' in request.form else {}
response = {
'items': [
{
u'title': u'Cinema',
u'author': [u'Gilles Deleuze'],
u'date': u'1986-10',
u'publisher': u'University of Minnesota Press',
u'isbn10': u'0816613990',
},
{
u'title': u'How to Change the World: Reflections on Marx and Marxism',
u'author': [u'Eric Hobsbawm'],
u'date': u'2011-09-06',
u'publisher': u'Yale University Press',
u'isbn13': u'9780300176162',
}
]
}
return response
actions.register(identify)
@returns_json
def download(request):
response = {}
data = json.loads(request.form['data']) if 'data' in request.form else {}
item = models.Item.get(data['id'])
if item:
item.transferprogress = 0
item.transferadded = datetime.now()
p = models.User.get(settings.USER_ID)
if p not in item.users:
item.users.append(p)
item.update()
response = {'status': 'queued'}
return response
actions.register(download, cache=False)
@returns_json
def cancelDownload(request):
response = {}
data = json.loads(request.form['data']) if 'data' in request.form else {}
item = models.Item.get(data['id'])
if item:
item.transferprogress = None
item.transferadded = None
p = models.User.get(settings.USER_ID)
if p in item.users:
item.users.remove(p)
item.update()
response = {'status': 'cancelled'}
return response
actions.register(cancelDownload, cache=False)
@returns_json
def scan(request):
state.main.add_callback(state.websockets[0].put, json.dumps(['scan', {}]))
return {}
actions.register(scan, cache=False)
@returns_json
def _import(request):
state.main.add_callback(state.websockets[0].put, json.dumps(['import', {}]))
return {}
actions.register(_import, 'import', cache=False)

74
oml/item/covers.py Normal file
View file

@ -0,0 +1,74 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
import sqlite3
import Image
from StringIO import StringIO
from settings import covers_db_path
class Covers(dict):
def __init__(self, db):
self._db = db
def connect(self):
self.conn = sqlite3.connect(self._db, timeout=10)
self.create()
def create(self):
c = self.conn.cursor()
c.execute(u'CREATE TABLE IF NOT EXISTS cover (id varchar(64) unique, data blob)')
c.execute(u'CREATE TABLE IF NOT EXISTS setting (key varchar(256) unique, value text)')
if int(self.get_setting(c, 'version', 0)) < 1:
self.set_setting(c, 'version', 1)
def get_setting(self, c, key, default=None):
c.execute(u'SELECT value FROM setting WHERE key = ?', (key, ))
for row in c:
return row[0]
return default
def set_setting(self, c, key, value):
c.execute(u'INSERT OR REPLACE INTO setting values (?, ?)', (key, str(value)))
def black(self):
img = Image.new('RGB', (80, 128))
o = StringIO()
img.save(o, format='jpeg')
data = o.getvalue()
o.close()
return data
def __getitem__(self, id, default=None):
sql = u'SELECT data FROM cover WHERE id=?'
self.connect()
c = self.conn.cursor()
c.execute(sql, (id, ))
data = default
for row in c:
data = row[0]
break
c.close()
self.conn.close()
return data
def __setitem__(self, id, data):
sql = u'INSERT OR REPLACE INTO cover values (?, ?)'
self.connect()
c = self.conn.cursor()
data = sqlite3.Binary(data)
c.execute(sql, (id, data))
self.conn.commit()
c.close()
self.conn.close()
def __delitem__(self, id):
sql = u'DELETE FROM cover WHERE id = ?'
self.connect()
c = self.conn.cursor()
c.execute(sql, (id, ))
self.conn.commit()
c.close()
self.conn.close()
covers = Covers(covers_db_path)

13
oml/item/migrate.py Normal file
View file

@ -0,0 +1,13 @@
import models
from copy import deepcopy
def import_all():
for i in models.items:
item = models.Item.get_or_create(i['id'])
item.path = i['path']
item.info = deepcopy(i)
del item.info['path']
del item.info['id']
item.meta = item.info.pop('meta', {})
models.db.session.add(item)
models.db.session.commit()

427
oml/item/models.py Normal file
View file

@ -0,0 +1,427 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
import os
import re
import base64
import json
import hashlib
from datetime import datetime
from StringIO import StringIO
import Image
import ox
import settings
from settings import db, config
from user.models import User
from person import get_sort_name
import media
from meta import scraper
import utils
from oxflask.db import MutableDict
from covers import covers
from changelog import Changelog
from websocket import trigger_event
class Work(db.Model):
created = db.Column(db.DateTime())
modified = db.Column(db.DateTime())
id = db.Column(db.String(32), primary_key=True)
meta = db.Column(MutableDict.as_mutable(db.PickleType(pickler=json)))
def __repr__(self):
return self.id
def __init__(self, id):
self.id = id
self.created = datetime.now()
self.modified = datetime.now()
class Edition(db.Model):
created = db.Column(db.DateTime())
modified = db.Column(db.DateTime())
id = db.Column(db.String(32), primary_key=True)
meta = db.Column(MutableDict.as_mutable(db.PickleType(pickler=json)))
work_id = db.Column(db.String(32), db.ForeignKey('work.id'))
work = db.relationship('Work', backref=db.backref('editions', lazy='dynamic'))
def __repr__(self):
return self.id
def __init__(self, id):
self.id = id
self.created = datetime.now()
self.modified = datetime.now()
user_items = db.Table('useritem',
db.Column('user_id', db.String(43), db.ForeignKey('user.id')),
db.Column('item_id', db.String(32), db.ForeignKey('item.id'))
)
class Item(db.Model):
created = db.Column(db.DateTime())
modified = db.Column(db.DateTime())
id = db.Column(db.String(32), primary_key=True)
info = db.Column(MutableDict.as_mutable(db.PickleType(pickler=json)))
meta = db.Column(MutableDict.as_mutable(db.PickleType(pickler=json)))
added = db.Column(db.DateTime()) # added to local library
accessed = db.Column(db.DateTime())
timesaccessed = db.Column(db.Integer())
transferadded = db.Column(db.DateTime())
transferprogress = db.Column(db.Float())
users = db.relationship('User', secondary=user_items,
backref=db.backref('items', lazy='dynamic'))
edition_id = db.Column(db.String(32), db.ForeignKey('edition.id'))
edition = db.relationship('Edition', backref=db.backref('items', lazy='dynamic'))
work_id = db.Column(db.String(32), db.ForeignKey('work.id'))
work = db.relationship('Work', backref=db.backref('items', lazy='dynamic'))
@property
def timestamp(self):
return self.modified.strftime('%s')
def __repr__(self):
return self.id
def __init__(self, id):
if isinstance(id, list):
id = base64.b32encode(hashlib.sha1(''.join(id)).digest())
self.id = id
self.created = datetime.now()
self.modified = datetime.now()
self.info = {}
self.meta = {}
@classmethod
def get(cls, id):
if isinstance(id, list):
id = base64.b32encode(hashlib.sha1(''.join(id)).digest())
return cls.query.filter_by(id=id).first()
@classmethod
def get_or_create(cls, id, info=None):
if isinstance(id, list):
id = base64.b32encode(hashlib.sha1(''.join(id)).digest())
item = cls.query.filter_by(id=id).first()
if not item:
item = cls(id=id)
if info:
item.info = info
db.session.add(item)
db.session.commit()
return item
def json(self, keys=None):
j = {}
j['id'] = self.id
j['created'] = self.created
j['modified'] = self.modified
j['timesaccessed'] = self.timesaccessed
j['accessed'] = self.accessed
j['added'] = self.added
j['transferadded'] = self.transferadded
j['transferprogress'] = self.transferprogress
j['users'] = map(str, list(self.users))
if self.info:
j.update(self.info)
if self.meta:
j.update(self.meta)
for key in self.id_keys + ['mainid']:
if key not in self.meta and key in j:
del j[key]
'''
if self.work_id:
j['work'] = {
'olid': self.work_id
}
j['work'].update(self.work.meta)
'''
if keys:
for k in j.keys():
if k not in keys:
del j[k]
return j
def get_path(self):
f = self.files.first()
prefs = settings.preferences
prefix = os.path.join(os.path.expanduser(prefs['libraryPath']), 'Books/')
return os.path.join(prefix, f.path) if f else None
def update_sort(self):
for key in config['itemKeys']:
if key.get('sort'):
value = self.json().get(key['id'], None)
sort_type = key.get('sortType', key['type'])
if value:
if sort_type == 'integer':
value = int(value)
elif sort_type == 'float':
value = float(value)
elif sort_type == 'date':
pass
elif sort_type == 'name':
if not isinstance(value, list):
value = [value]
value = map(get_sort_name, value)
value = ox.sort_string(u'\n'.join(value))
elif sort_type == 'title':
value = utils.sort_title(value).lower()
else:
if isinstance(value, list):
value = u'\n'.join(value)
if value:
value = unicode(value)
value = ox.sort_string(value).lower()
setattr(self, 'sort_%s' % key['id'], value)
def update_find(self):
for key in config['itemKeys']:
if key.get('find') or key.get('filter'):
value = self.json().get(key['id'], None)
if key.get('filterMap') and value:
value = re.compile(key.get('filterMap')).findall(value)[0]
print key['id'], value
if value:
if isinstance(value, list):
Find.query.filter_by(item_id=self.id, key=key['id']).delete()
for v in value:
f = Find(item_id=self.id, key=key['id'])
f.value = v.lower()
db.session.add(f)
else:
f = Find.get_or_create(self.id, key['id'])
f.value = value.lower()
db.session.add(f)
else:
f = Find.get(self.id, key['id'])
if f:
db.session.delete(f)
def update_lists(self):
Find.query.filter_by(item_id=self.id, key='list').delete()
for p in self.users:
f = Find()
f.item_id = self.id
f.key = 'list'
if p.id == settings.USER_ID:
f.value = ':'
else:
f.value = '%s:' % p.id
db.session.add(f)
def update(self):
users = map(str, list(self.users))
self.meta['mediastate'] = 'available' # available, unavailable, transferring
if self.transferadded and self.transferprogress < 1:
self.meta['mediastate'] = 'transferring'
else:
self.meta['mediastate'] = 'available' if settings.USER_ID in users else 'unavailable'
self.update_sort()
self.update_find()
self.update_lists()
self.modified = datetime.now()
self.save()
def save(self):
db.session.add(self)
db.session.commit()
def update_mainid(self, key, id):
record = {}
if id:
self.meta[key] = id
self.meta['mainid'] = key
record[key] = id
else:
if key in self.meta:
del self.meta[key]
if 'mainid' in self.meta:
del self.meta['mainid']
record[key] = ''
for k in self.id_keys:
if k != key:
if k in self.meta:
del self.meta[k]
print 'mainid', 'mainid' in self.meta, self.meta.get('mainid')
print 'key', key, self.meta.get(key)
# get metadata from external resources
self.scrape()
self.update()
self.update_cover()
db.session.add(self)
db.session.commit()
user = User.get_or_create(settings.USER_ID)
if user in self.users:
Changelog.record(user, 'edititem', self.id, record)
def extract_cover(self):
path = self.get_path()
if not path:
return getattr(media, self.meta['extensions']).cover(path)
def update_cover(self):
cover = None
if 'cover' in self.meta:
cover = ox.cache.read_url(self.meta['cover'])
#covers[self.id] = requests.get(self.meta['cover']).content
if cover:
covers[self.id] = cover
path = self.get_path()
if not cover and path:
cover = self.extract_cover()
if cover:
covers[self.id] = cover
if cover:
img = Image.open(StringIO(cover))
self.meta['coverRatio'] = img.size[0]/img.size[1]
for p in (':128', ':256'):
del covers['%s%s' % (self.id, p)]
return cover
def scrape(self):
mainid = self.meta.get('mainid')
print 'scrape', mainid, self.meta.get(mainid)
if mainid == 'olid':
scraper.update_ol(self)
scraper.add_lookupbyisbn(self)
elif mainid in ('isbn10', 'isbn13'):
scraper.add_lookupbyisbn(self)
elif mainid == 'lccn':
import meta.lccn
info = meta.lccn.info(self.meta[mainid])
for key in info:
self.meta[key] = info[key]
else:
print 'FIX UPDATE', mainid
self.update()
def save_file(self, content):
p = User.get(settings.USER_ID)
f = File.get(self.id)
if not f:
path = 'Downloads/%s.%s' % (self.id, self.info['extension'])
f = File.get_or_create(self.id, self.info, path=path)
path = self.get_path()
if not os.path.exists(path):
ox.makedirs(os.path.dirname(path))
with open(path, 'wb') as fd:
fd.write(content)
if p not in self.users:
self.users.append(p)
self.transferprogress = 1
self.added = datetime.now()
Changelog.record(p, 'additem', self.id, self.info)
self.update()
trigger_event('transfer', {
'id': self.id, 'progress': 1
})
return True
else:
print 'TRIED TO SAVE EXISTING FILE!!!'
self.transferprogress = 1
self.update()
return False
for key in config['itemKeys']:
if key.get('sort'):
sort_type = key.get('sortType', key['type'])
if sort_type == 'integer':
col = db.Column(db.BigInteger(), index=True)
elif sort_type == 'float':
col = db.Column(db.Float(), index=True)
elif sort_type == 'date':
col = db.Column(db.DateTime(), index=True)
else:
col = db.Column(db.String(1000), index=True)
setattr(Item, 'sort_%s' % key['id'], col)
Item.id_keys = ['isbn10', 'isbn13', 'lccn', 'olid', 'oclc']
Item.item_keys = config['itemKeys']
Item.filter_keys = []
class Find(db.Model):
id = db.Column(db.Integer(), primary_key=True)
item_id = db.Column(db.String(32), db.ForeignKey('item.id'))
item = db.relationship('Item', backref=db.backref('find', lazy='dynamic'))
key = db.Column(db.String(200), index=True)
value = db.Column(db.Text())
def __repr__(self):
return (u'%s=%s' % (self.key, self.value)).encode('utf-8')
@classmethod
def get(cls, item, key):
return cls.query.filter_by(item_id=item, key=key).first()
@classmethod
def get_or_create(cls, item, key):
f = cls.get(item, key)
if not f:
f = cls(item_id=item, key=key)
db.session.add(f)
db.session.commit()
return f
class File(db.Model):
created = db.Column(db.DateTime())
modified = db.Column(db.DateTime())
sha1 = db.Column(db.String(32), primary_key=True)
path = db.Column(db.String(2048))
info = db.Column(MutableDict.as_mutable(db.PickleType(pickler=json)))
item_id = db.Column(db.String(32), db.ForeignKey('item.id'))
item = db.relationship('Item', backref=db.backref('files', lazy='dynamic'))
@classmethod
def get(cls, sha1):
return cls.query.filter_by(sha1=sha1).first()
@classmethod
def get_or_create(cls, sha1, info=None, path=None):
f = cls.get(sha1)
if not f:
f = cls(sha1=sha1)
if info:
f.info = info
if path:
f.path = path
f.item_id = Item.get_or_create(id=sha1, info=info).id
db.session.add(f)
db.session.commit()
return f
def __repr__(self):
return self.sha1
def __init__(self, sha1):
self.sha1 = sha1
self.created = datetime.now()
self.modified = datetime.now()

42
oml/item/person.py Normal file
View file

@ -0,0 +1,42 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
import unicodedata
import ox
from settings import db
def get_sort_name(name, sortname=None):
name = unicodedata.normalize('NFKD', name).strip()
if name:
person = Person.get(name)
if not person:
person = Person(name=name, sortname=sortname)
person.save()
sortname = unicodedata.normalize('NFKD', person.sortname)
else:
sortname = u''
return sortname
class Person(db.Model):
name = db.Column(db.String(1024), primary_key=True)
sortname = db.Column(db.String())
numberofnames = db.Column(db.Integer())
def __repr__(self):
return self.name
@classmethod
def get(cls, name):
return cls.query.filter_by(name=name).first()
def save(self):
if not self.sortname:
self.sortname = ox.get_sort_name(self.name)
self.sortname = unicodedata.normalize('NFKD', self.sortname)
self.sortsortname = ox.sort_string(self.sortname)
self.numberofnames = len(self.name.split(' '))
db.session.add(self)
db.session.commit()

83
oml/item/query.py Normal file
View file

@ -0,0 +1,83 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
import settings
import models
import utils
import oxflask.query
from sqlalchemy.sql.expression import nullslast
def parse(data):
query = {}
query['range'] = [0, 100]
query['sort'] = [{'key':'title', 'operator':'+'}]
for key in ('keys', 'group', 'list', 'range', 'sort', 'query'):
if key in data:
query[key] = data[key]
print data
query['qs'] = oxflask.query.Parser(models.Item).find(data)
if 'query' in query and 'conditions' in query['query'] and query['query']['conditions']:
conditions = query['query']['conditions']
condition = conditions[0]
if condition['key'] == '*':
value = condition['value'].lower()
query['qs'] = models.Item.query.join(
models.Find, models.Find.item_id==models.Item.id).filter(
models.Find.value.contains(value))
if 'group' in query:
query['qs'] = order_by_group(query['qs'], query['sort'])
else:
query['qs'] = order(query['qs'], query['sort'])
return query
def order(qs, sort, prefix='sort_'):
order_by = []
if len(sort) == 1:
additional_sort = settings.config['user']['ui']['listSort']
key = utils.get_by_id(models.Item.item_keys, sort[0]['key'])
for s in key.get('additionalSort', additional_sort):
if s['key'] not in [e['key'] for e in sort]:
sort.append(s)
for e in sort:
operator = e['operator']
if operator != '-':
operator = ''
else:
operator = ' DESC'
key = {}.get(e['key'], e['key'])
if key not in ('fixme', ):
key = "%s%s" % (prefix, key)
order = '%s%s' % (key, operator)
order_by.append(order)
if order_by:
#nulllast not supported in sqlite, use IS NULL hack instead
#order_by = map(nullslast, order_by)
_order_by = []
for order in order_by:
nulls = "%s IS NULL" % order.split(' ')[0]
_order_by.append(nulls)
_order_by.append(order)
order_by = _order_by
qs = qs.order_by(*order_by)
return qs
def order_by_group(qs, sort):
return qs
if 'sort' in query:
if len(query['sort']) == 1 and query['sort'][0]['key'] == 'items':
order_by = query['sort'][0]['operator'] == '-' and '-items' or 'items'
if query['group'] == "year":
secondary = query['sort'][0]['operator'] == '-' and '-sortvalue' or 'sortvalue'
order_by = (order_by, secondary)
elif query['group'] != "keyword":
order_by = (order_by, 'sortvalue')
else:
order_by = (order_by, 'value')
else:
order_by = query['sort'][0]['operator'] == '-' and '-sortvalue' or 'sortvalue'
order_by = (order_by, 'items')
else:
order_by = ('-sortvalue', 'items')
return order_by

182
oml/item/scan.py Normal file
View file

@ -0,0 +1,182 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from __future__ import division
import os
import shutil
from datetime import datetime
import ox
from app import app
import settings
from settings import db
from item.models import File
from user.models import User
from changelog import Changelog
import media
from websocket import trigger_event
def remove_missing():
dirty = False
with app.app_context():
user = User.get_or_create(settings.USER_ID)
prefs = settings.preferences
prefix = os.path.join(os.path.expanduser(prefs['libraryPath']), 'Books/')
for f in File.query:
if not os.path.exists(f.item.get_path()):
dirty = True
print 'file gone', f, f.item.get_path()
f.item.users.remove(user)
if not f.item.users:
print 'last user, remove'
db.session.delete(f.item)
else:
f.item.update_lists()
Changelog.record(user, 'removeitem', f.item.id)
db.session.delete(f)
if dirty:
db.session.commit()
def run_scan():
remove_missing()
with app.app_context():
prefs = settings.preferences
prefix = os.path.join(os.path.expanduser(prefs['libraryPath']), 'Books/')
if not prefix[-1] == '/':
prefix += '/'
user = User.get_or_create(settings.USER_ID)
assert isinstance(prefix, unicode)
extensions = ['pdf', 'epub', 'txt']
books = []
for root, folders, files in os.walk(prefix):
for f in files:
#if f.startswith('._') or f == '.DS_Store':
if f.startswith('.'):
continue
f = os.path.join(root, f)
ext = f.split('.')[-1]
if ext in extensions:
books.append(f)
trigger_event('scan', {
'path': prefix,
'files': len(books)
})
position = 0
added = 0
for f in ox.sorted_strings(books):
position += 1
id = media.get_id(f)
file = File.get(id)
path = f[len(prefix):]
if not file:
data = media.metadata(f)
ext = f.split('.')[-1]
data['extension'] = ext
data['size'] = os.stat(f).st_size
file = File.get_or_create(id, data, path)
item = file.item
if 'mainid' in file.info:
del file.info['mainid']
db.session.add(file)
if 'mainid' in item.info:
item.meta['mainid'] = item.info.pop('mainid')
item.meta[item.meta['mainid']] = item.info[item.meta['mainid']]
db.session.add(item)
item.users.append(user)
Changelog.record(user, 'additem', item.id, item.info)
if item.meta.get('mainid'):
Changelog.record(user, 'edititem', item.id, {
item.meta['mainid']: item.meta[item.meta['mainid']]
})
item.added = datetime.now()
item.scrape()
added += 1
trigger_event('scan', {
'position': position,
'length': len(books),
'path': path,
'progress': position/len(books),
'added': added,
})
trigger_event('scan', {
'progress': 1,
'added': added,
'done': True
})
def run_import():
with app.app_context():
prefs = settings.preferences
prefix = os.path.expanduser(prefs['importPath'])
prefix_books = os.path.join(os.path.expanduser(prefs['libraryPath']), 'Books/')
prefix_imported = os.path.join(prefix_books, 'Imported/')
if not prefix[-1] == '/':
prefix += '/'
user = User.get_or_create(settings.USER_ID)
assert isinstance(prefix, unicode)
extensions = ['pdf', 'epub', 'txt']
books = []
for root, folders, files in os.walk(prefix):
for f in files:
#if f.startswith('._') or f == '.DS_Store':
if f.startswith('.'):
continue
f = os.path.join(root, f)
ext = f.split('.')[-1]
if ext in extensions:
books.append(f)
trigger_event('import', {
'path': prefix,
'files': len(books)
})
position = 0
added = 0
for f in ox.sorted_strings(books):
position += 1
id = media.get_id(f)
file = File.get(id)
path = f[len(prefix):]
if not file:
f_import = f
f = f.replace(prefix, prefix_imported)
ox.makedirs(os.path.dirname(f))
shutil.move(f_import, f)
path = f[len(prefix_books):]
data = media.metadata(f)
ext = f.split('.')[-1]
data['extension'] = ext
data['size'] = os.stat(f).st_size
file = File.get_or_create(id, data, path)
item = file.item
if 'mainid' in file.info:
del file.info['mainid']
db.session.add(file)
if 'mainid' in item.info:
item.meta['mainid'] = item.info.pop('mainid')
item.meta[item.meta['mainid']] = item.info[item.meta['mainid']]
db.session.add(item)
item.users.append(user)
Changelog.record(user, 'additem', item.id, item.info)
if item.meta.get('mainid'):
Changelog.record(user, 'edititem', item.id, {
item.meta['mainid']: item.meta[item.meta['mainid']]
})
item.scrape()
added += 1
trigger_event('import', {
'position': position,
'length': len(books),
'path': path,
'progress': position/len(books),
'added': added,
})
trigger_event('import', {
'progress': 1,
'added': added,
'done': True
})

101
oml/item/views.py Normal file
View file

@ -0,0 +1,101 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
import os
from datetime import datetime
import zipfile
import mimetypes
from StringIO import StringIO
import Image
from flask import Blueprint
from flask import json, request, make_response, abort, send_file
from covers import covers
import settings
from models import Item, db
from utils import resize_image
app = Blueprint('item', __name__, static_folder=settings.static_path)
@app.route('/<string:id>/epub/')
@app.route('/<string:id>/epub/<path:filename>')
def epub(id, filename=''):
item = Item.get(id)
if not item or item.info['extension'] != 'epub':
abort(404)
path = item.get_path()
z = zipfile.ZipFile(path)
if filename == '':
return '<br>\n'.join([f.filename for f in z.filelist])
if filename not in [f.filename for f in z.filelist]:
abort(404)
resp = make_response(z.read(filename))
resp.content_type = {
'xpgt': 'application/vnd.adobe-page-template+xml'
}.get(filename.split('.')[0], mimetypes.guess_type(filename)[0]) or 'text/plain'
return resp
@app.route('/<string:id>/get')
@app.route('/<string:id>/txt/')
@app.route('/<string:id>/pdf')
def get(id):
item = Item.get(id)
if not item:
abort(404)
path = item.get_path()
mimetype={
'epub': 'application/epub+zip',
'pdf': 'application/pdf',
}.get(path.split('.')[-1], None)
return send_file(path, mimetype=mimetype)
@app.route('/<string:id>/cover.jpg')
@app.route('/<string:id>/cover<int:size>.jpg')
def cover(id, size=None):
item = Item.get(id)
if not item:
abort(404)
data = None
if size:
data = covers['%s:%s' % (id, size)]
if data:
size = None
if not data:
data = covers[id]
if not data:
print 'check for cover', id
data = item.update_cover()
if not data:
data = covers.black()
if size:
data = covers['%s:%s' % (id, size)] = resize_image(data, size=size)
data = str(data)
if not 'coverRatio' in item.meta:
#img = Image.open(StringIO(str(covers[id])))
img = Image.open(StringIO(data))
item.meta['coverRatio'] = float(img.size[0])/img.size[1]
db.session.add(item)
db.session.commit()
resp = make_response(data)
resp.content_type = "image/jpeg"
return resp
@app.route('/<string:id>/reader/')
def reader(id, filename=''):
item = Item.get(id)
if item.info['extension'] == 'epub':
html = 'html/epub.html'
elif item.info['extension'] == 'pdf':
html = 'html/pdf.html'
elif item.info['extension'] == 'txt':
html = 'html/txt.html'
else:
abort(404)
item.sort_accessed = item.accessed = datetime.now()
item.sort_timesaccessed = item.timesaccessed = (item.timesaccessed or 0) + 1
item.save()
return app.send_static_file(html)

45
oml/media/__init__.py Normal file
View file

@ -0,0 +1,45 @@
import pdf
import epub
import txt
import os
import base64
import ox
def get_id(f):
return base64.b32encode(ox.sha1sum(f).decode('hex'))
def metadata(f):
ext = f.split('.')[-1]
data = {}
if ext == 'pdf':
info = pdf.info(f)
elif ext == 'epub':
info = epub.info(f)
elif ext == 'txt':
info = txt.info(f)
for key in ('title', 'author', 'date', 'publisher', 'isbn'):
if key in info:
value = info[key]
if isinstance(value, str):
try:
value = value.decode('utf-8')
except:
value = None
if value:
data[key] = info[key]
if 'isbn' in data:
value = data.pop('isbn')
if len(value) == 10:
data['isbn10'] = value
data['mainid'] = 'isbn10'
else:
data['isbn13'] = value
data['mainid'] = 'isbn13'
if not 'title' in data:
data['title'] = os.path.splitext(os.path.basename(f))[0]
if 'author' in data and isinstance(data['author'], basestring):
data['author'] = [data['author']]
return data

63
oml/media/epub.py Normal file
View file

@ -0,0 +1,63 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from __future__ import division
import sys
import xml.etree.ElementTree as ET
import zipfile
from StringIO import StringIO
import Image
import stdnum.isbn
from utils import normalize_isbn, find_isbns
def cover(path):
img = Image.new('RGB', (80, 128))
o = StringIO()
img.save(o, format='jpeg')
data = o.getvalue()
o.close()
return data
def info(epub):
data = {}
z = zipfile.ZipFile(epub)
opf = [f.filename for f in z.filelist if f.filename.endswith('opf')]
if opf:
info = ET.fromstring(z.read(opf[0]))
metadata = info.findall('{http://www.idpf.org/2007/opf}metadata')[0]
for e in metadata.getchildren():
if e.text:
key = e.tag.split('}')[-1]
key = {
'creator': 'author',
}.get(key, key)
value = e.text
if key == 'identifier':
value = normalize_isbn(value)
if stdnum.isbn.is_valid(value):
data['isbn'] = value
else:
data[key] = e.text
text = extract_text(epub)
data['textsize'] = len(text)
if not 'isbn' in data:
isbn = extract_isbn(text)
if isbn:
data['isbn'] = isbn
return data
def extract_text(path):
data = ''
z = zipfile.ZipFile(path)
for f in z.filelist:
if f.filename.endswith('html'):
data += z.read(f.filename)
return data
def extract_isbn(data):
isbns = find_isbns(data)
if isbns:
return isbns[0]

140
oml/media/pdf.py Normal file
View file

@ -0,0 +1,140 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from __future__ import division
import sys
import tempfile
import subprocess
import os
import shutil
from glob import glob
from pyPdf import PdfFileReader
import stdnum.isbn
import settings
from utils import normalize_isbn, find_isbns
def cover(pdf):
if sys.platform == 'darwin':
return ql_cover(pdf)
else:
return page(pdf, 1)
def ql_cover(pdf):
tmp = tempfile.mkdtemp()
cmd = [
'qlmanage',
'-t',
'-s',
'1024',
'-o',
tmp,
pdf
]
p = subprocess.Popen(cmd)
p.wait()
image = glob('%s/*' % tmp)[0]
with open(image, 'rb') as fd:
data = fd.read()
shutil.rmtree(tmp)
return data
def page(pdf, page):
image = tempfile.mkstemp('.jpg')[1]
cmd = [
'gs', '-q',
'-dBATCH', '-dSAFER', '-dNOPAUSE', '-dNOPROMPT',
'-dMaxBitmap=500000000',
'-dAlignToPixels=0', '-dGridFitTT=2',
'-sDEVICE=jpeg', '-dTextAlphaBits=4', '-dGraphicsAlphaBits=4',
'-r72',
'-dUseCropBox',
'-dFirstPage=%d' % page,
'-dLastPage=%d' % page,
'-sOutputFile=%s' % image,
pdf
]
p = subprocess.Popen(cmd)
p.wait()
with open(image, 'rb') as fd:
data = fd.read()
os.unlink(image)
return data
def info(pdf):
data = {}
with open(pdf, 'rb') as fd:
try:
pdfreader = PdfFileReader(fd)
info = pdfreader.getDocumentInfo()
if info:
for key in info:
if info[key]:
data[key[1:].lower()] = info[key]
xmp =pdfreader.getXmpMetadata()
if xmp:
for key in dir(xmp):
if key.startswith('dc_'):
value = getattr(xmp, key)
if isinstance(value, dict) and 'x-default' in value:
value = value['x-default']
elif isinstance(value, list):
value = [v.strip() for v in value if v.strip()]
_key = key[3:]
if value and _key not in data:
data[_key] = value
except:
print 'FAILED TO PARSE', pdf
import traceback
print traceback.print_exc()
if 'identifier' in data:
value = normalize_isbn(data['identifier'])
if stdnum.isbn.is_valid(value):
data['isbn'] = value
del data['identifier']
'''
cmd = ['pdfinfo', pdf]
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
for line in stdout.strip().split('\n'):
parts = line.split(':')
key = parts[0].lower().strip()
if key:
data[key] = ':'.join(parts[1:]).strip()
for key in data.keys():
if not data[key]:
del data[key]
'''
text = extract_text(pdf)
data['textsize'] = len(text)
if settings.server['extract_text']:
if not 'isbn' in data:
isbn = extract_isbn(text)
if isbn:
data['isbn'] = isbn
return data
'''
#possbile alternative with gs
tmp = tempfile.mkstemp('.txt')[1]
cmd = ['gs', '-dBATCH', '-dNOPAUSE', '-sDEVICE=txtwrite', '-dFirstPage=3', '-dLastPage=5', '-sOutputFile=%s'%tmp, pdf]
'''
def extract_text(pdf):
if sys.platform == 'darwin':
cmd = ['/usr/bin/mdimport' '-d2', pdf]
else:
cmd = ['pdftotext', pdf, '-']
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
if sys.platform == 'darwin':
stdout = stderr.split('kMDItemTextContent = "')[-1].split('\n')[0][:-2]
return stdout.strip()
def extract_isbn(text):
isbns = find_isbns(text)
if isbns:
return isbns[0]

41
oml/media/txt.py Normal file
View file

@ -0,0 +1,41 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from __future__ import division
import sys
import os
from utils import find_isbns
from StringIO import StringIO
import Image
from pdf import ql_cover
def cover(path):
if sys.platform == 'darwin':
return ql_cover(path)
img = Image.new('RGB', (80, 128))
o = StringIO()
img.save(o, format='jpeg')
data = o.getvalue()
o.close()
return data
def info(path):
data = {}
data['title'] = os.path.splitext(os.path.basename(path))[0]
text = extract_text(path)
isbn = extract_isbn(text)
if isbn:
data['isbn'] = isbn
data['textsize'] = len(text)
return data
def extract_text(path):
with open(path) as fd:
data = fd.read()
return data
def extract_isbn(text):
isbns = find_isbns(text)
if isbns:
return isbns[0]

0
oml/meta/__init__.py Normal file
View file

52
oml/meta/lccn.py Normal file
View file

@ -0,0 +1,52 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from __future__ import division
import ox
from ox.cache import read_url
import xml.etree.ElementTree as ET
from utils import normalize_isbn
from marc_countries import COUNTRIES
def info(id):
ns = '{http://www.loc.gov/mods/v3}'
url = 'http://lccn.loc.gov/%s/mods' % id
data = read_url(url)
mods = ET.fromstring(data)
info = {}
info['title'] = ''.join([e.text for e in mods.findall(ns + 'titleInfo')[0]])
origin = mods.findall(ns + 'originInfo')
if origin:
info['place'] = []
for place in origin[0].findall(ns + 'place'):
terms = place.findall(ns + 'placeTerm')
if terms and terms[0].attrib['type'] == 'text':
e = terms[0]
info['place'].append(e.text)
elif terms and terms[0].attrib['type'] == 'code':
e = terms[0]
info['country'] = COUNTRIES.get(e.text, e.text)
info['publisher'] = ''.join([e.text for e in origin[0].findall(ns + 'publisher')])
info['date'] = ''.join([e.text for e in origin[0].findall(ns + 'dateIssued')])
for i in mods.findall(ns + 'identifier'):
if i.attrib['type'] == 'oclc':
info['oclc'] = i.text.replace('ocn', '')
if i.attrib['type'] == 'lccn':
info['lccn'] = i.text
if i.attrib['type'] == 'isbn':
isbn = normalize_isbn(i.text)
info['isbn%s'%len(isbn)] = isbn
for i in mods.findall(ns + 'classification'):
if i.attrib['authority'] == 'ddc':
info['classification'] = i.text
info['author'] = []
for a in mods.findall(ns + 'name'):
if a.attrib['usage'] == 'primary':
info['author'].append(''.join([e.text for e in a.findall(ns + 'namePart')]))
info['author'] = [ox.normalize_name(a[:-1]) for a in info['author']]
for key in info.keys():
if not info[key]:
del info[key]
return info

409
oml/meta/marc_countries.py Normal file
View file

@ -0,0 +1,409 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
COUNTRIES = {
"gw": "Germany",
"gv": "Guinea",
"gu": "Guam",
"gt": "Guatemala",
"gs": "Georgia (Republic)",
"gr": "Greece",
"-ge": "Germany (East)",
"gp": "Guadeloupe",
"mnu": "Minnesota",
"gy": "Guyana",
"gd": "Grenada",
"gb": "Kiribati",
"go": "Gabon",
"gm": "Gambia",
"alu": "Alabama",
"gi": "Gibraltar",
"gh": "Ghana",
"tz": "Tanzania",
"tv": "Tuvalu",
"tu": "Turkey",
"tr": "Trinidad and Tobago",
"ts": "United Arab Emirates",
"to": "Tonga",
"tl": "Tokelau",
"tk": "Turkmenistan",
"th": "Thailand",
"ti": "Tunisia",
"tg": "Togo",
"tc": "Turks and Caicos Islands",
"ta": "Tajikistan",
"-gn": "Gilbert and Ellice Islands",
"-us": "United States",
"-ajr": "Azerbaijan S.S.R.",
"-iu": "Israel-Syria Demilitarized Zones",
"-iw": "Israel-Jordan Demilitarized Zones",
"za": "Zambia",
"nbu": "Nebraska",
"scu": "South Carolina",
"bg": "Bangladesh",
"cau": "California",
"abc": "Alberta",
"xoa": "Northern Territory",
"meu": "Maine",
"ctu": "Connecticut",
"my": "Malaysia",
"aku": "Alaska",
"gl": "Greenland",
"-cn": "Canada",
"wiu": "Wisconsin",
"-cz": "Canal Zone",
"txu": "Texas",
"-cs": "Czechoslovakia",
"-cp": "Canton and Enderbury Islands",
"msu": "Mississippi",
"-ln": "Central and Southern Line Islands",
"nkc": "New Brunswick",
"it": "Italy",
"tnu": "Tennessee",
"vp": "Various places",
"mg": "Madagascar",
"mf": "Mauritius",
"mc": "Monaco",
"-ur": "Soviet Union",
"mm": "Malta",
"ml": "Mali",
"mo": "Montenegro",
"flu": "Florida",
"deu": "Delaware",
"mk": "Oman",
"mj": "Montserrat",
"mu": "Mauritania",
"mw": "Malawi",
"mv": "Moldova",
"mq": "Martinique",
"mp": "Mongolia",
"mr": "Morocco",
"-ui": "United Kingdom Misc. Islands",
"mx": "Mexico",
"-uk": "United Kingdom",
"mz": "Mozambique",
"kyu": "Kentucky",
"hiu": "Hawaii",
"enk": "England",
"nyu": "New York (State)",
"fp": "French Polynesia",
"fr": "France",
"fs": "Terres australes et antarctiques fran&ccedil;aises",
"mau": "Massachusetts",
"snc": "Saskatchewan",
"fa": "Faroe Islands",
"fg": "French Guiana",
"lau": "Louisiana",
"fj": "Fiji",
"fk": "Falkland Islands",
"fm": "Micronesia (Federated States)",
"sz": "Switzerland",
"sy": "Syria",
"sx": "Namibia",
"ss": "Western Sahara",
"sr": "Surinam",
"sq": "Swaziland",
"sp": "Spain",
"sw": "Sweden",
"su": "Saudi Arabia",
"st": "Saint-Martin",
"sj": "Sudan",
"si": "Singapore",
"sh": "Spanish North Africa",
"so": "Somalia",
"sn": "Sint Maarten",
"sm": "San Marino",
"sl": "Sierra Leone",
"sc": "Saint-Barth&eacute;lemy",
"sa": "South Africa",
"sg": "Senegal",
"sf": "Sao Tome and Principe",
"se": "Seychelles",
"sd": "South Sudan",
"-unr": "Ukraine",
"-kgr": "Kirghiz S.S.R.",
"le": "Lebanon",
"lb": "Liberia",
"-hk": "Hong Kong",
"lo": "Lesotho",
"lh": "Liechtenstein",
"li": "Lithuania",
"lv": "Latvia",
"lu": "Luxembourg",
"vtu": "Vermont",
"ls": "Laos",
"xc": "Maldives",
"ly": "Libya",
"oku": "Oklahoma",
"ye": "Yemen",
"-tkr": "Turkmen S.S.R.",
"nfc": "Newfoundland and Labrador",
"ft": "Djibouti",
"em": "Timor-Leste",
"eg": "Equatorial Guinea",
"ea": "Eritrea",
"ec": "Ecuador",
"-gsr": "Georgian S.S.R.",
"et": "Ethiopia",
"es": "El Salvador",
"er": "Estonia",
"ru": "Russia (Federation)",
"rw": "Rwanda",
"re": "R&eacute;union",
"rb": "Serbia",
"rm": "Romania",
"rh": "Zimbabwe",
"-err": "Estonia",
"oru": "Oregon",
"quc": "Qu&eacute;bec (Province)",
"ntc": "Northwest Territories",
"wlk": "Wales",
"xj": "Saint Helena",
"xk": "Saint Lucia",
"xh": "Niue",
"xn": "Macedonia",
"xo": "Slovakia",
"xl": "Saint Pierre and Miquelon",
"xm": "Saint Vincent and the Grenadines",
"xb": "Cocos (Keeling) Islands",
"onc": "Ontario",
"xa": "Christmas Island (Indian Ocean)",
"xf": "Midway Islands",
"xd": "Saint Kitts-Nevis",
"xe": "Marshall Islands",
"nhu": "New Hampshire",
"xx": "No place, unknown, or undetermined",
"fi": "Finland",
"xr": "Czech Republic",
"xs": "South Georgia and the South Sandwich Islands",
"xp": "Spratly Island",
"xv": "Slovenia",
"-tt": "Trust Territory of the Pacific Islands",
"iau": "Iowa",
"ncu": "North Carolina",
"stk": "Scotland",
"xra": "South Australia",
"miu": "Michigan",
"kg": "Kyrgyzstan",
"ke": "Kenya",
"ko": "Korea (South)",
"kn": "Korea (North)",
"kv": "Kosovo",
"ku": "Kuwait",
"kz": "Kazakhstan",
"-pt": "Portuguese Timor",
"ksu": "Kansas",
"dm": "Benin",
"dk": "Denmark",
"-ys": "Yemen (People's Democratic Republic)",
"-yu": "Serbia and Montenegro",
"-bwr": "Byelorussian S.S.R.",
"dr": "Dominican Republic",
"dq": "Dominica",
"qa": "Qatar",
"aru": "Arkansas",
"nuc": "Nunavut",
"wf": "Wallis and Futuna",
"wk": "Wake Island",
"wj": "West Bank of the Jordan River",
"jm": "Jamaica",
"vra": "Victoria",
"jo": "Jordan",
"ws": "Samoa",
"ji": "Johnston Atoll",
"-na": "Netherlands Antilles",
"ja": "Japan",
"cou": "Colorado",
"-wb": "West Berlin",
"ilu": "Illinois",
"-nm": "Northern Mariana Islands",
"ck": "Colombia",
"cj": "Cayman Islands",
"ci": "Croatia",
"ch": "China (Republic : 1949- )",
"co": "Cura&ccedil;ao",
"cm": "Cameroon",
"cl": "Chile",
"-rur": "Russian S.F.S.R.",
"cb": "Cambodia",
"ca": "Caribbean Netherlands",
"cg": "Congo (Democratic Republic)",
"cf": "Congo (Brazzaville)",
"-lir": "Lithuania",
"cd": "Chad",
"cy": "Cyprus",
"cx": "Central African Republic",
"cr": "Costa Rica",
"cq": "Comoros",
"cw": "Cook Islands",
"cv": "Cape Verde",
"cu": "Cuba",
"pr": "Puerto Rico",
"pp": "Papua New Guinea",
"pw": "Palau",
"py": "Paraguay",
"pc": "Pitcairn Island",
"pf": "Paracel Islands",
"pg": "Guinea-Bissau",
"pe": "Peru",
"pk": "Pakistan",
"ph": "Philippines",
"pn": "Panama",
"po": "Portugal",
"pl": "Poland",
"pic": "Prince Edward Island",
"xxu": "United States",
"gau": "Georgia",
"xxc": "Canada",
"xxk": "United Kingdom",
"iy": "Iraq-Saudi Arabia Neutral Zone",
"vb": "British Virgin Islands",
"vc": "Vatican City",
"ve": "Venezuela",
"iq": "Iraq",
"vi": "Virgin Islands of the United States",
"is": "Israel",
"ir": "Iran",
"vm": "Vietnam",
"iv": "C&ocirc;te d'Ivoire",
"ii": "India",
"-ac": "Ashmore and Cartier Islands",
"io": "Indonesia",
"-ai": "Anguilla",
"ic": "Iceland",
"ie": "Ireland",
"pau": "Pennsylvania",
"-jn": "Jan Mayen",
"nik": "Northern Ireland",
"wyu": "Wyoming",
"-air": "Armenian S.S.R.",
"-sv": "Swan Islands",
"-mvr": "Moldavian S.S.R.",
"-sk": "Sikkim",
"riu": "Rhode Island",
"-sb": "Svalbard",
"-xi": "Saint Kitts-Nevis-Anguilla",
"wea": "Western Australia",
"cc": "China",
"nvu": "Nevada",
"mou": "Missouri",
"ce": "Sri Lanka",
"qea": "Queensland",
"-mh": "Macao",
"nju": "New Jersey",
"ykc": "Yukon Territory",
"-vs": "Vietnam, South",
"tma": "Tasmania",
"-vn": "Vietnam, North",
"bd": "Burundi",
"be": "Belgium",
"bf": "Bahamas",
"nmu": "New Mexico",
"ba": "Bahrain",
"bb": "Barbados",
"bl": "Brazil",
"bm": "Bermuda Islands",
"bn": "Bosnia and Hercegovina",
"bo": "Bolivia",
"bh": "Belize",
"bi": "British Indian Ocean Territory",
"bt": "Bhutan",
"bu": "Bulgaria",
"bv": "Bouvet Island",
"bw": "Belarus",
"bp": "Solomon Islands",
"br": "Burma",
"bs": "Botswana",
"dcu": "District of Columbia",
"bx": "Brunei",
"aca": "Australian Capital Territory",
"idu": "Idaho",
"xna": "New South Wales",
"ot": "Mayotte",
"ndu": "North Dakota",
"nsc": "Nova Scotia",
"-kzr": "Kazakh S.S.R.",
"mbc": "Manitoba",
"-lvr": "Latvia",
"-uzr": "Uzbek S.S.R.",
"wau": "Washington (State)",
"vau": "Virginia",
"sdu": "South Dakota",
"gz": "Gaza Strip",
"ht": "Haiti",
"hu": "Hungary",
"ho": "Honduras",
"hm": "Heard and McDonald Islands",
"xga": "Coral Sea Islands Territory",
"uy": "Uruguay",
"uz": "Uzbekistan",
"uv": "Burkina Faso",
"up": "United States Misc. Pacific Islands",
"mtu": "Montana",
"un": "Ukraine",
"utu": "Utah",
"ug": "Uganda",
"ua": "Egypt",
"azu": "Arizona",
"uc": "United States Misc. Caribbean Islands",
"aa": "Albania",
"ae": "Algeria",
"ag": "Argentina",
"af": "Afghanistan",
"ai": "Armenia (Republic)",
"inu": "Indiana",
"uik": "United Kingdom Misc. Islands",
"aj": "Azerbaijan",
"am": "Anguilla",
"ao": "Angola",
"an": "Andorra",
"aq": "Antigua and Barbuda",
"as": "American Samoa",
"au": "Austria",
"at": "Australia",
"aw": "Aruba",
"ay": "Antarctica",
"ohu": "Ohio",
"nl": "New Caledonia",
"-ry": "Ryukyu Islands, Southern",
"nn": "Vanuatu",
"no": "Norway",
"ne": "Netherlands",
"ng": "Niger",
"nx": "Norfolk Island",
"nz": "New Zealand",
"np": "Nepal",
"nq": "Nicaragua",
"nr": "Nigeria",
"mdu": "Maryland",
"nu": "Nauru",
"nw": "Northern Mariana Islands",
"wvu": "West Virginia",
"-xxr": "Soviet Union",
"-tar": "Tajik S.S.R.",
"bcc": "British Columbia"
}
if __name__ == '__main__':
import json
import re
import ox
from ox.cache import read_url
url = "http://www.loc.gov/marc/countries/countries_code.html"
data = read_url(url)
countries = dict([
[ox.strip_tags(c) for c in r]
for r in re.compile('<tr>.*?class="code">(.*?)</td>.*?<td>(.*?)</td>', re.DOTALL).findall(data)
])
data = json.dumps(countries, indent=4, ensure_ascii=False).encode('utf-8')
with open(__file__) as f:
pydata = f.read()
pydata = re.sub(
re.compile('\nCOUNTRIES = {.*?}\n\n', re.DOTALL),
'\nCOUNTRIES = %s\n\n' % data, pydata)
with open(__file__, 'w') as f:
f.write(pydata)

67
oml/meta/ol.py Normal file
View file

@ -0,0 +1,67 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from __future__ import division
from ox.cache import read_url
import json
from utils import normalize_isbn
from marc_countries import COUNTRIES
def find(query):
url = 'https://openlibrary.org/search.json?q=%s' % query
data = json.loads(read_url(url))
return data
def authors(authors):
return resolve_names(authors)
def resolve_names(objects, key='name'):
r = []
for o in objects:
url = 'https://openlibrary.org%s.json' % o['key']
data = json.loads(read_url(url))
r.append(data[key])
return r
def languages(languages):
return resolve_names(languages)
def info(id):
data = {}
url = 'https://openlibrary.org/books/%s.json' % id
info = json.loads(read_url(url))
keys = {
'title': 'title',
'authors': 'author',
'publishers': 'publisher',
'languages': 'language',
'publish_places': 'place',
'publish_country': 'country',
'covers': 'cover',
'isbn_10': 'isbn10',
'isbn_13': 'isbn13',
'lccn': 'lccn',
'oclc_numbers': 'oclc',
'dewey_decimal_class': 'classification',
'number_of_pages': 'pages',
}
for key in keys:
if key in info:
value = info[key]
if key == 'authors':
value = authors(value)
elif key == 'publish_country':
value = COUNTRIES.get(value, value)
elif key == 'covers':
value = 'https://covers.openlibrary.org/b/id/%s.jpg' % value[0]
value = COUNTRIES.get(value, value)
elif key == 'languages':
value = languages(value)
elif isinstance(value, list) and key not in ('publish_places'):
value = value[0]
if key in ('isbn_10', 'isbn_13'):
value = normalize_isbn(value)
data[keys[key]] = value
return data

32
oml/meta/scraper.py Normal file
View file

@ -0,0 +1,32 @@
import json
from ox.cache import read_url
import ox.web.lookupbyisbn
from utils import normalize_isbn
import ol
def add_lookupbyisbn(item):
isbn = item.meta.get('isbn10', item.meta.get('isbn13'))
if isbn:
more = ox.web.lookupbyisbn.get_data(isbn)
if more:
for key in more:
if more[key]:
value = more[key]
if isinstance(value, basestring):
value = ox.strip_tags(ox.decode_html(value))
elif isinstance(value, list):
value = [ox.strip_tags(ox.decode_html(v)) for v in value]
item.meta[key] = value
if 'author' in item.meta and isinstance(item.meta['author'], basestring):
item.meta['author'] = [item.meta['author']]
if 'isbn' in item.meta:
del item.meta['isbn']
def update_ol(item):
info = ol.info(item.meta['olid'])
for key in info:
item.meta[key] = info[key]

0
oml/node/__init__.py Normal file
View file

87
oml/node/api.py Normal file
View file

@ -0,0 +1,87 @@
import settings
from changelog import Changelog
from user.models import User
import state
from websocket import trigger_event
def api_pullChanges(app, remote_id, user_id=None, from_=None, to=None):
if user_id and not from_ and not to:
from_ = user_id
user_id = None
if user_id and from_ and not to:
if isinstance(user_id, int):
to = from_
from_ = user_id
user_id = None
from_ = from_ or 0
if user_id:
return []
if not user_id:
user_id = settings.USER_ID
qs = Changelog.query.filter_by(user_id=user_id)
if from_:
qs = qs.filter(Changelog.revision>=from_)
if to:
qs = qs.filter(Changelog.revision<to)
state.nodes.queue('add', remote_id)
return [c.json() for c in qs]
def api_pushChanges(app, user_id, changes):
user = User.get(user_id)
for change in changes:
if not Changelog.apply_change(user, change):
print 'FAILED TO APPLY CHANGE', change
state.nodes.queue(user_id, 'pullChanges')
return False
return True
def api_requestPeering(app, user_id, username, message):
user = User.get_or_create(user_id)
if not user.info:
user.info = {}
if not user.peered:
if user.pending == 'sent':
user.info['message'] = message
user.update_peering(True, username)
else:
user.pending = 'received'
user.info['username'] = username
user.info['message'] = message
user.save()
trigger_event('peering', user.json())
return True
return False
def api_acceptPeering(app, user_id, username, message):
user = User.get(user_id)
if user and user.pending == 'sent':
if not user.info:
user.info = {}
user.info['username'] = username
user.info['message'] = message
user.update_peering(True, username)
trigger_event('peering', user.json())
return True
return False
def api_rejectPeering(app, user_id, message):
user = User.get(user_id)
if user:
if not user.info:
user.info = {}
user.info['message'] = message
user.update_peering(False)
trigger_event('peering', user.json())
return True
return False
def api_removePeering(app, user_id, message):
user = User.get(user_id)
if user:
user.peered = False
user.info['message'] = message
user.save()
trigger_event('peering', {'id': user.id, 'peered': user.peered})
return True
return False

25
oml/node/gencert.py Normal file
View file

@ -0,0 +1,25 @@
import OpenSSL
key = OpenSSL.crypto.PKey()
key.generate_key(OpenSSL.crypto.TYPE_RSA, 2048)
ca = OpenSSL.crypto.X509()
ca.set_version(2)
ca.set_serial_number(1)
ca.get_subject().CN = "put_ed25519_key_here"
ca.gmtime_adj_notBefore(0)
ca.gmtime_adj_notAfter(24 * 60 * 60)
ca.set_issuer(ca.get_subject())
ca.set_pubkey(key)
ca.add_extensions([
OpenSSL.crypto.X509Extension("basicConstraints", True,
"CA:TRUE, pathlen:0"),
OpenSSL.crypto.X509Extension("keyUsage", True,
"keyCertSign, cRLSign"),
OpenSSL.crypto.X509Extension("subjectKeyIdentifier", False, "hash",
subject=ca),
OpenSSL.crypto.X509Extension("authorityKeyIdentifier", False, "keyid:always",issuer=ca)
])
ca.sign(key, "sha1")
open("MyCertificate.crt.bin", "wb").write(OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_ASN1, ca))

119
oml/node/server.py Normal file
View file

@ -0,0 +1,119 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
import os
import sys
import tornado
from tornado.web import StaticFileHandler, Application, FallbackHandler
from tornado.wsgi import WSGIContainer
from tornado.httpserver import HTTPServer
from tornado.ioloop import IOLoop, PeriodicCallback
import settings
import directory
import utils
import state
import user
import json
from ed25519_utils import valid
import api
class NodeHandler(tornado.web.RequestHandler):
def initialize(self, app):
self.app = app
def post(self):
request = self.request
if request.method == 'POST':
'''
API
pullChanges [userid] from [to]
pushChanges [index, change]
requestPeering username message
acceptPeering username message
rejectPeering message
removePeering message
ping responds public ip
'''
key = str(request.headers['X-Ed25519-Key'])
sig = str(request.headers['X-Ed25519-Signature'])
data = request.body
content = {}
if valid(key, data, sig):
action, args = json.loads(data)
print 'action', action, args
if action == 'ping':
content = {
'ip': request.remote_addr
}
else:
with self.app.app_context():
if action in (
'requestPeering', 'acceptPeering', 'rejectPeering', 'removePeering'
) or user.models.User.get(key):
content = getattr(api, 'api_' + action)(self.app, key, *args)
else:
print 'PEER', key, 'IS UNKNOWN SEND 403'
self.set_status(403)
content = {
'status': 'not peered'
}
content = json.dumps(content)
sig = settings.sk.sign(content, encoding='base64')
self.set_header('X-Ed25519-Signature', sig)
self.write(content)
self.finish()
def get(self):
self.write('Open Media Library')
self.finish()
class ShareHandler(tornado.web.RequestHandler):
def initialize(self, app):
self.app = app
def get(self, id):
with self.app.app_context():
import item.models
i = item.models.Item.get(id)
if not i:
self.set_status(404)
self.finish()
path = i.get_path()
mimetype = {
'epub': 'application/epub+zip',
'pdf': 'application/pdf',
'txt': 'text/plain',
}.get(path.split('.')[-1], None)
self.set_header('Content-Type', mimetype)
print 'GET file', id
with open(path, 'rb') as f:
while 1:
data = f.read(16384)
if not data:
break
self.write(data)
self.finish()
def start(app):
http_server = tornado.web.Application([
(r"/get/(.*)", ShareHandler, dict(app=app)),
(r".*", NodeHandler, dict(app=app)),
])
#tr = WSGIContainer(node_app)
#http_server= HTTPServer(tr)
http_server.listen(settings.server['node_port'], settings.server['node_address'])
host = utils.get_public_ipv4()
state.online = directory.put(settings.sk, {
'host': host,
'port': settings.server['node_port']
})
return http_server

19
oml/node/utils.py Normal file
View file

@ -0,0 +1,19 @@
import socket
import requests
from urlparse import urlparse
def get_public_ipv6():
host = ('2a01:4f8:120:3201::3', 25519)
s = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
s.connect(host)
ip = s.getsockname()[0]
s.close()
return ip
def get_public_ipv4():
host = ('10.0.3.1', 25519)
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.connect(host)
ip = s.getsockname()[0]
s.close()
return ip

263
oml/nodes.py Normal file
View file

@ -0,0 +1,263 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from __future__ import division
from Queue import Queue
from threading import Thread
import json
from datetime import datetime
import os
import ox
import ed25519
import requests
import settings
import user.models
from changelog import Changelog
import directory
from websocket import trigger_event
ENCODING='base64'
class Node(object):
online = False
download_speed = 0
def __init__(self, app, user):
self._app = app
self.user_id = user.id
key = str(user.id)
self.vk = ed25519.VerifyingKey(key, encoding=ENCODING)
self.go_online()
@property
def url(self):
if ':' in self.host:
url = 'http://[%s]:%s' % (self.host, self.port)
else:
url = 'http://%s:%s' % (self.host, self.port)
return url
def resolve_host(self):
r = directory.get(self.vk)
if r:
self.host = r['host']
if 'port' in r:
self.port = r['port']
else:
self.host = None
self.port = 9851
def request(self, action, *args):
if not self.host:
self.resolve_host()
if not self.host:
return None
content = json.dumps([action, args])
sig = settings.sk.sign(content, encoding=ENCODING)
headers = {
'User-Agent': settings.USER_AGENT,
'Accept': 'text/plain',
'Accept-Encoding': 'gzip',
'Content-Type': 'application/json',
'X-Ed25519-Key': settings.USER_ID,
'X-Ed25519-Signature': sig,
}
r = requests.post(self.url, data=content, headers=headers)
if r.status_code == 403:
print 'REMOTE ENDED PEERING'
if self.user.peered:
self.user.update_peering(False)
data = r.content
sig = r.headers.get('X-Ed25519-Signature')
if sig and self._valid(data, sig):
response = json.loads(data)
else:
response = None
return response
def _valid(self, data, sig):
try:
self.vk.verify(sig, data, encoding=ENCODING)
#except ed25519.BadSignatureError:
except:
return False
return True
@property
def user(self):
return user.models.User.get_or_create(self.user_id)
def go_online(self):
self.resolve_host()
if self.user.peered:
try:
self.online = False
print 'type to connect to', self.user_id
self.pullChanges()
print 'connected to', self.user_id
self.online = True
except:
import traceback
traceback.print_exc()
print 'failed to connect to', self.user_id
self.online = False
else:
self.online = False
trigger_event('status', {
'id': self.user_id,
'status': 'online' if self.online else 'offline'
})
def pullChanges(self):
with self._app.app_context():
last = Changelog.query.filter_by(user_id=self.user_id).order_by('-revision').first()
from_revision = last.revision + 1 if last else 0
changes = self.request('pullChanges', from_revision)
if not changes:
return False
for change in changes:
if not Changelog.apply_change(self.user, change):
print 'FAIL', change
break
return False
return True
def pushChanges(self, changes):
print 'pushing changes to', self.user_id, changes
try:
r = self.request('pushChanges', changes)
except:
self.online = False
trigger_event('status', {
'id': self.user_id,
'status': 'offline'
})
r = False
print r
def requestPeering(self, message):
p = self.user
p.pending = 'sent'
p.save()
r = self.request('requestPeering', settings.preferences['username'], message)
return True
def acceptPeering(self, message):
r = self.request('acceptPeering', settings.preferences['username'], message)
p = self.user
p.update_peering(True)
self.go_online()
return True
def rejectPeering(self, message):
r = self.request('rejectPeering', message)
p = self.user
p.update_peering(False)
return True
def removePeering(self, message):
r = self.request('removePeering', message)
p = self.user
p.update_peering(False)
return True
def download(self, item):
url = '%s/get/%s' % (self.url, item.id)
headers = {
'User-Agent': settings.USER_AGENT,
}
t1 = datetime.now()
print 'GET', url
r = requests.get(url, headers=headers)
if r.status_code == 200:
t2 = datetime.now()
duration = (t2-t1).total_seconds()
if duration:
self.download_speed = len(r.content) / duration
print 'SPEED', ox.format_bits(self.download_speed)
return item.save_file(r.content)
else:
print 'FAILED', url
return False
def download_upgrade(self):
for module in settings.release['modules']:
path = os.path.join(settings.update_path, settings.release['modules'][module]['name'])
if not os.path.exists(path):
url = '%s/oml/%s' % (self.url, settings.release['modules'][module]['name'])
sha1 = settings.release['modules'][module]['sha1']
headers = {
'User-Agent': settings.USER_AGENT,
}
r = requests.get(url, headers=headers)
if r.status_code == 200:
with open(path, 'w') as fd:
fd.write(r.content)
if (ox.sha1sum(path) != sha1):
print 'invalid update!'
os.unlink(path)
return False
else:
return False
class Nodes(Thread):
_nodes = {}
def __init__(self, app):
self._app = app
self._q = Queue()
self._running = True
Thread.__init__(self)
self.daemon = True
self.start()
def queue(self, *args):
self._q.put(list(args))
def check_online(self, id):
return id in self._nodes and self._nodes[id].online
def download(self, id, item):
return id in self._nodes and self._nodes[id].download(item)
def _call(self, target, action, *args):
print 'call', target, action, args
if target == 'all':
nodes = self._nodes.values()
elif target == 'online':
nodes = [n for n in self._nodes.values() if n.online]
else:
nodes = [self._nodes[target]]
for node in nodes:
getattr(node, action)(*args)
def _add_node(self, user_id):
if user_id not in self._nodes:
from user.models import User
self._nodes[user_id] = Node(self._app, User.get_or_create(user_id))
else:
self._nodes[user_id].online = True
trigger_event('status', {
'id': user_id,
'status': 'online'
})
def run(self):
with self._app.app_context():
while self._running:
args = self._q.get()
if args:
if args[0] == 'add':
self._add_node(args[1])
else:
print 'next', args
self._call(*args)
def join(self):
self._running = False
self._q.put(None)
return Thread.join(self)

0
oml/oxflask/__init__.py Normal file
View file

154
oml/oxflask/api.py Normal file
View file

@ -0,0 +1,154 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from __future__ import division, with_statement
import inspect
import sys
import json
from flask import request, Blueprint
from .shortcuts import render_to_json_response, json_response
app = Blueprint('oxflask', __name__)
@app.route('/api/', methods=['POST', 'OPTIONS'])
def api():
if request.method == "OPTIONS":
response = render_to_json_response({'status': {'code': 200, 'text': 'use POST'}})
response.headers['Access-Control-Allow-Origin'] = '*'
return response
if not 'action' in request.form:
methods = actions.keys()
api = []
for f in sorted(methods):
api.append({'name': f,
'doc': actions.doc(f).replace('\n', '<br>\n')})
return render_to_json_response(api)
action = request.form['action']
f = actions.get(action)
if f:
response = f(request)
else:
response = render_to_json_response(json_response(status=400,
text='Unknown action %s' % action))
response.headers['Access-Control-Allow-Origin'] = '*'
return response
def trim(docstring):
if not docstring:
return ''
# Convert tabs to spaces (following the normal Python rules)
# and split into a list of lines:
lines = docstring.expandtabs().splitlines()
# Determine minimum indentation (first line doesn't count):
indent = sys.maxint
for line in lines[1:]:
stripped = line.lstrip()
if stripped:
indent = min(indent, len(line) - len(stripped))
# Remove indentation (first line is special):
trimmed = [lines[0].strip()]
if indent < sys.maxint:
for line in lines[1:]:
trimmed.append(line[indent:].rstrip())
# Strip off trailing and leading blank lines:
while trimmed and not trimmed[-1]:
trimmed.pop()
while trimmed and not trimmed[0]:
trimmed.pop(0)
# Return a single string:
return '\n'.join(trimmed)
class ApiActions(dict):
properties = {}
versions = {}
def __init__(self):
def api(request):
'''
returns list of all known api actions
param data {
docs: bool
}
if docs is true, action properties contain docstrings
return {
status: {'code': int, 'text': string},
data: {
actions: {
'api': {
cache: true,
doc: 'recursion'
},
'hello': {
cache: true,
..
}
...
}
}
}
'''
data = json.loads(request.form.get('data', '{}'))
docs = data.get('docs', False)
code = data.get('code', False)
version = getattr(request, 'version', None)
if version:
_actions = self.versions.get(version, {}).keys()
_actions = list(set(_actions + self.keys()))
else:
_actions = self.keys()
_actions.sort()
actions = {}
for a in _actions:
actions[a] = self.properties[a]
if docs:
actions[a]['doc'] = self.doc(a, version)
if code:
actions[a]['code'] = self.code(a, version)
response = json_response({'actions': actions})
return render_to_json_response(response)
self.register(api)
def doc(self, name, version=None):
if version:
f = self.versions[version].get(name, self.get(name))
else:
f = self[name]
return trim(f.__doc__)
def code(self, name, version=None):
if version:
f = self.versions[version].get(name, self.get(name))
else:
f = self[name]
if name != 'api' and hasattr(f, 'func_closure') and f.func_closure:
fc = filter(lambda c: hasattr(c.cell_contents, '__call__'), f.func_closure)
f = fc[len(fc)-1].cell_contents
info = f.func_code.co_filename
info = u'%s:%s' % (info, f.func_code.co_firstlineno)
return info, trim(inspect.getsource(f))
def register(self, method, action=None, cache=True, version=None):
if not action:
action = method.func_name
if version:
if not version in self.versions:
self.versions[version] = {}
self.versions[version][action] = method
else:
self[action] = method
self.properties[action] = {'cache': cache}
def unregister(self, action):
if action in self:
del self[action]
actions = ApiActions()
def error(request):
'''
this action is used to test api error codes, it should return a 503 error
'''
success = error_is_success
return render_to_json_response({})
actions.register(error)

27
oml/oxflask/db.py Normal file
View file

@ -0,0 +1,27 @@
from sqlalchemy.ext.mutable import Mutable
class MutableDict(Mutable, dict):
@classmethod
def coerce(cls, key, value):
"Convert plain dictionaries to MutableDict."
if not isinstance(value, MutableDict):
if isinstance(value, dict):
return MutableDict(value)
# this call will raise ValueError
return Mutable.coerce(key, value)
else:
return value
def __setitem__(self, key, value):
"Detect dictionary set events and emit change events."
dict.__setitem__(self, key, value)
self.changed()
def __delitem__(self, key):
"Detect dictionary del events and emit change events."
dict.__delitem__(self, key)
self.changed()

245
oml/oxflask/query.py Normal file
View file

@ -0,0 +1,245 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from sqlalchemy.sql.expression import and_, not_, or_, ClauseElement
from datetime import datetime
import unicodedata
from sqlalchemy.sql import operators, extract
import utils
import settings
def get_operator(op, type='str'):
return {
'str': {
'==': operators.ilike_op,
'>': operators.gt,
'>=': operators.ge,
'<': operators.lt,
'<=': operators.le,
'^': operators.startswith_op,
'$': operators.endswith_op,
},
'int': {
'==': operators.eq,
'>': operators.gt,
'>=': operators.ge,
'<': operators.lt,
'<=': operators.le,
}
}[type].get(op, {
'str': operators.contains_op,
'int': operators.eq
}[type])
class Parser(object):
def __init__(self, model):
self._model = model
self._find = model.find.mapper.class_
self._user = model.users.mapper.class_
self._list = model.lists.mapper.class_
self.item_keys = model.item_keys
self.filter_keys = model.filter_keys
def parse_condition(self, condition):
'''
condition: {
value: "war"
}
or
condition: {
key: "year",
value: [1970, 1980],
operator: "="
}
...
'''
k = condition.get('key', '*')
if not k:
k = '*'
v = condition['value']
op = condition.get('operator')
if not op:
op = '='
if op.startswith('!'):
op = op[1:]
exclude = True
else:
exclude = False
key_type = (utils.get_by_id(self.item_keys, k) or {'type': 'string'}).get('type')
if isinstance(key_type, list):
key_type = key_type[0]
if k == 'list':
key_type = ''
if (not exclude and op == '=' or op in ('$', '^')) and v == '':
return None
elif k == 'resolution':
q = self.parse_condition({'key': 'width', 'value': v[0], 'operator': op}) \
& self.parse_condition({'key': 'height', 'value': v[1], 'operator': op})
if exclude:
q = ~q
return q
elif isinstance(v, list) and len(v) == 2 and op == '=':
q = self.parse_condition({'key': k, 'value': v[0], 'operator': '>='}) \
& self.parse_condition({'key': k, 'value': v[1], 'operator': '<'})
if exclude:
q = ~q
return q
elif key_type == 'boolean':
q = getattr(self._model, 'find_%s' % k) == v
if exclude:
q = ~q
return q
elif key_type in ("string", "text"):
if isinstance(v, unicode):
v = unicodedata.normalize('NFKD', v).lower()
q = get_operator(op)(self._find.value, v.lower())
if k != '*':
q &= (self._find.key == k)
if exclude:
q = ~q
return q
elif k == 'list':
'''
q = Q(id=0)
l = v.split(":")
if len(l) == 1:
vqs = Volume.objects.filter(name=v, user=user)
if vqs.count() == 1:
v = vqs[0]
q = Q(files__instances__volume__id=v.id)
elif len(l) >= 2:
l = (l[0], ":".join(l[1:]))
lqs = list(List.objects.filter(name=l[1], user__username=l[0]))
if len(lqs) == 1 and lqs[0].accessible(user):
l = lqs[0]
if l.query.get('static', False) == False:
data = l.query
q = self.parse_conditions(data.get('conditions', []),
data.get('operator', '&'),
user, l.user)
else:
q = Q(id__in=l.items.all())
if exclude:
q = ~q
else:
q = Q(id=0)
'''
l = v.split(":")
nickname = l[0]
name = ':'.join(l[1:])
if nickname:
p = self._user.query.filter_by(nickname=nickname).first()
v = '%s:%s' % (p.id, name)
else:
p = self._user.query.filter_by(id=settings.USER_ID).first()
v = ':%s' % name
#print 'get list:', p.id, name, l, v
if name:
l = self._list.query.filter_by(user_id=p.id, name=name).first()
else:
l = None
if l and l._query:
data = l._query
q = self.parse_conditions(data.get('conditions', []),
data.get('operator', '&'))
else:
q = (self._find.key == 'list') & (self._find.value == v)
return q
elif key_type == 'date':
def parse_date(d):
while len(d) < 3:
d.append(1)
return datetime(*[int(i) for i in d])
#using sort here since find only contains strings
v = parse_date(v.split('-'))
vk = getattr(self._model, 'sort_%s' % k)
q = get_operator(op, 'int')(vk, v)
if exclude:
q = ~q
return q
else: #integer, float, time
q = get_operator(op, 'int')(getattr(self._model, 'sort_%s'%k), v)
if exclude:
q = ~q
return q
def parse_conditions(self, conditions, operator):
'''
conditions: [
{
value: "war"
}
{
key: "year",
value: "1970-1980,
operator: "!="
},
{
key: "country",
value: "f",
operator: "^"
}
],
operator: "&"
'''
conn = []
for condition in conditions:
if 'conditions' in condition:
q = self.parse_conditions(condition['conditions'],
condition.get('operator', '&'))
else:
q = self.parse_condition(condition)
if isinstance(q, list):
conn += q
else:
conn.append(q)
conn = [q for q in conn if not isinstance(q, None.__class__)]
if conn:
if operator == '|':
q = conn[0]
for c in conn[1:]:
q = q | c
q = [q]
else:
q = conn
return q
return []
def find(self, data):
'''
query: {
conditions: [
{
value: "war"
}
{
key: "year",
value: "1970-1980,
operator: "!="
},
{
key: "country",
value: "f",
operator: "^"
}
],
operator: "&"
}
'''
#join query with operator
qs = self._model.query
#only include items that have hard metadata
conditions = self.parse_conditions(data.get('query', {}).get('conditions', []),
data.get('query', {}).get('operator', '&'))
for c in conditions:
qs = qs.join(self._find).filter(c)
qs = qs.group_by(self._model.id)
return qs

34
oml/oxflask/shortcuts.py Normal file
View file

@ -0,0 +1,34 @@
from functools import wraps
import datetime
import json
from flask import Response
def json_response(data=None, status=200, text='ok'):
if not data:
data = {}
return {'status': {'code': status, 'text': text}, 'data': data}
def _to_json(python_object):
if isinstance(python_object, datetime.datetime):
if python_object.year < 1900:
tt = python_object.timetuple()
return '%d-%02d-%02dT%02d:%02d%02dZ' % tuple(list(tt)[:6])
return python_object.strftime('%Y-%m-%dT%H:%M:%SZ')
raise TypeError(u'%s %s is not JSON serializable' % (repr(python_object), type(python_object)))
def json_dumps(obj):
indent = 2
return json.dumps(obj, indent=indent, default=_to_json, ensure_ascii=False).encode('utf-8')
def render_to_json_response(obj, content_type="text/json", status=200):
resp = Response(json_dumps(obj), status=status, content_type=content_type)
return resp
def returns_json(f):
@wraps(f)
def decorated_function(*args, **kwargs):
r = f(*args, **kwargs)
return render_to_json_response(json_response(r))
return decorated_function

8
oml/oxflask/utils.py Normal file
View file

@ -0,0 +1,8 @@
def get_by_key(objects, key, value):
obj = filter(lambda o: o.get(key) == value, objects)
return obj and obj[0] or None
def get_by_id(objects, id):
return get_by_key(objects, 'id', id)

37
oml/pdict.py Normal file
View file

@ -0,0 +1,37 @@
import os
import json
class pdict(dict):
def __init__(self, path, defaults=None):
self._path = None
self._defaults = defaults
if os.path.exists(path):
with open(path) as fd:
_data = json.load(fd)
for key in _data:
self[key] = _data[key]
self._path = path
def _save(self):
if self._path:
with open(self._path, 'w') as fd:
json.dump(self, fd, indent=1)
def get(self, key, default=None):
if default == None and self._defaults:
default = self._defaults.get(key)
return dict.get(self, key, default)
def __getitem__(self, key):
if key not in self and self._defaults and key in self._defaults:
return self._defaults[key]
return dict.__getitem__(self, key)
def __setitem__(self, key, value):
dict.__setitem__(self, key, value)
self._save()
def __delitem__(self, key):
dict.__delitem__(self, key)
self._save()

59
oml/server.py Normal file
View file

@ -0,0 +1,59 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
import os
import sys
from tornado.web import StaticFileHandler, Application, FallbackHandler
from tornado.wsgi import WSGIContainer
from tornado.httpserver import HTTPServer
from tornado.ioloop import IOLoop
from app import app
import settings
import websocket
import state
import node.server
def run():
root_dir = os.path.normpath(os.path.join(os.path.abspath(os.path.dirname(__file__)), '..'))
PID = sys.argv[2] if len(sys.argv) > 2 else None
state.main = IOLoop.instance()
static_path = os.path.join(root_dir, 'static')
options = {
'debug': not PID
}
tr = WSGIContainer(app)
handlers = [
(r'/(favicon.ico)', StaticFileHandler, {'path': static_path}),
(r'/static/(.*)', StaticFileHandler, {'path': static_path}),
(r'/ws', websocket.Handler),
(r".*", FallbackHandler, dict(fallback=tr)),
]
http_server = HTTPServer(Application(handlers, **options))
http_server.listen(settings.server['port'], settings.server['address'])
if PID:
with open(PID, 'w') as pid:
pid.write('%s' % os.getpid())
def start_node():
import user
import downloads
import nodes
state.node = node.server.start(app)
state.nodes = nodes.Nodes(app)
state.downloads = downloads.Downloads(app)
def add_users(app):
with app.app_context():
for p in user.models.User.query.filter_by(peered=True):
state.nodes.queue('add', p.id)
state.main.add_callback(add_users, app)
state.main.add_callback(start_node)
state.main.start()

71
oml/settings.py Normal file
View file

@ -0,0 +1,71 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from flask.ext.sqlalchemy import SQLAlchemy
import json
import os
import ed25519
from pdict import pdict
base_dir = os.path.normpath(os.path.join(os.path.abspath(os.path.dirname(__file__)), '..'))
static_path = os.path.join(base_dir, 'static')
updates_path = os.path.join(base_dir, 'updates')
oml_config_path = os.path.join(base_dir, 'config.json')
config_dir = os.path.normpath(os.path.join(base_dir, '..', 'config'))
if not os.path.exists(config_dir):
os.makedirs(config_dir)
db_path = os.path.join(config_dir, 'openmedialibrary.db')
covers_db_path = os.path.join(config_dir, 'covers.db')
key_path = os.path.join(config_dir, 'node.key')
db = SQLAlchemy()
if os.path.exists(oml_config_path):
with open(oml_config_path) as fd:
config = json.load(fd)
else:
config = {}
preferences = pdict(os.path.join(config_dir, 'preferences.json'), config['user']['preferences'])
ui = pdict(os.path.join(config_dir, 'ui.json'), config['user']['ui'])
server = pdict(os.path.join(config_dir, 'server.json'))
server_defaults = {
'port': 9842,
'address': '127.0.0.1',
'node_port': 9851,
'node_address': '::',
'extract_text': True,
'directory_service': 'http://[2a01:4f8:120:3201::3]:25519',
'lookup_service': 'http://data.openmedialibrary.com',
}
for key in server_defaults:
if key not in server:
server[key] = server_defaults[key]
release = pdict(os.path.join(config_dir, 'release.json'))
if os.path.exists(key_path):
with open(key_path) as fd:
sk = ed25519.SigningKey(fd.read())
vk = sk.get_verifying_key()
else:
sk, vk = ed25519.create_keypair()
with open(key_path, 'w') as fd:
os.chmod(key_path, 0600)
fd.write(sk.to_bytes())
os.chmod(key_path, 0400)
USER_ID = vk.to_ascii(encoding='base64')
if 'modules' in release and 'openmedialibrary' in release['modules']:
VERSION = release['modules']['openmedialibrary']['version']
else:
VERSION = 'git'
USER_AGENT = 'OpenMediaLibrary/%s' % VERSION

14
oml/setup.py Normal file
View file

@ -0,0 +1,14 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
import settings
from user.models import List, User
def create_default_lists(user_id=None):
user_id = user_id or settings.USER_ID
user = User.get_or_create(user_id)
for list in settings.config['lists']:
l = List.get(user_id, list['title'])
if not l:
l = List.create(user_id, list['title'], list.get('query'))

9
oml/state.py Normal file
View file

@ -0,0 +1,9 @@
websockets = []
nodes = False
main = None
online = False
def user():
import settings
import user.models
return user.models.User.get_or_create(settings.USER_ID)

97
oml/update.py Normal file
View file

@ -0,0 +1,97 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from __future__ import division
#https://github.com/hiroakis/tornado-websocket-example/blob/master/app.py
#http://stackoverflow.com/questions/5892895/tornado-websocket-question
#possibly get https://github.com/methane/wsaccel
#possibly run the full django app throw tornado instead of gunicorn
#https://github.com/bdarnell/django-tornado-demo/blob/master/testsite/tornado_main.py
#http://stackoverflow.com/questions/7190431/tornado-with-django
#http://www.tornadoweb.org/en/stable/wsgi.html
from tornado.httpserver import HTTPServer
from tornado.ioloop import IOLoop
from tornado.web import Application
from tornado.websocket import WebSocketHandler
from Queue import Queue
import urllib2
import os
from contextlib import closing
import json
from threading import Thread
class Background:
def __init__(self, handler):
self.handler = handler
self.q = Queue()
def worker(self):
while True:
message = self.q.get()
action, data = json.loads(message)
if action == 'get':
if 'url' in data and data['url'].startswith('http'):
self.download(data['url'], '/tmp/test.data')
elif action == 'update':
self.post({'error': 'not implemented'})
else:
self.post({'error': 'unknown action'})
self.q.task_done()
def join(self):
self.q.join()
def put(self, data):
self.q.put(data)
def post(self, data):
if not isinstance(data, basestring):
data = json.dumps(data)
main.add_callback(lambda: self.handler.write_message(data))
def download(self, url, filename):
dirname = os.path.dirname(filename)
if dirname and not os.path.exists(dirname):
os.makedirs(dirname)
with open(filename, 'w') as f:
with closing(urllib2.urlopen(url)) as u:
size = int(u.headers.get('content-length', 0))
done = 0
chunk_size = max(min(1024*1024, int(size/100)), 4096)
print 'chunksize', chunk_size
for data in iter(lambda: u.read(chunk_size), ''):
f.write(data)
done += len(data)
if size:
percent = done/size
self.post({'url': url, 'size': size, 'done': done, 'percent': percent})
class Handler(WebSocketHandler):
def open(self):
print "New connection opened."
self.background = Background(self)
self.t = Thread(target=self.background.worker)
self.t.daemon = True
self.t.start()
#websocket calls
def on_message(self, message):
self.background.put(message)
def on_close(self):
print "Connection closed."
self.background.join()
print "Server started."
HTTPServer(Application([("/", Handler)])).listen(28161)
main = IOLoop.instance()
main.start()

0
oml/user/__init__.py Normal file
View file

216
oml/user/api.py Normal file
View file

@ -0,0 +1,216 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
import os
from copy import deepcopy
from flask import json
from oxflask.api import actions
from oxflask.shortcuts import returns_json
import models
from item.models import Item
from utils import get_position_by_id
import settings
import state
from changelog import Changelog
@returns_json
def init(request):
'''
this is an init request to test stuff
'''
#print 'init', request
response = {}
if os.path.exists(settings.oml_config_path):
with open(settings.oml_config_path) as fd:
config = json.load(fd)
else:
config = {}
response['config'] = config
response['user'] = deepcopy(config['user'])
if settings.preferences:
response['user']['preferences'] = settings.preferences
response['user']['id'] = settings.USER_ID
response['user']['online'] = state.online
if settings.ui:
response['user']['ui'] = settings.ui
return response
actions.register(init)
def update_dict(root, data):
for key in data:
keys = map(lambda p: p.replace('\0', '\\.'), key.replace('\\.', '\0').split('.'))
value = data[key]
p = root
while len(keys)>1:
key = keys.pop(0)
if isinstance(p, list):
p = p[get_position_by_id(p, key)]
else:
if key not in p:
p[key] = {}
p = p[key]
if value == None and keys[0] in p:
del p[keys[0]]
else:
p[keys[0]] = value
@returns_json
def setPreferences(request):
data = json.loads(request.form['data']) if 'data' in request.form else {}
update_dict(settings.preferences, data)
return settings.preferences
actions.register(setPreferences)
@returns_json
def setUI(request):
data = json.loads(request.form['data']) if 'data' in request.form else {}
update_dict(settings.ui, data)
return settings.ui
actions.register(setUI)
@returns_json
def getUsers(request):
users = []
for u in models.User.query.filter(models.User.id!=settings.USER_ID).all():
users.append(u.json())
return {
"users": users
}
actions.register(getUsers)
@returns_json
def getLists(request):
lists = {}
for u in models.User.query.filter((models.User.peered==True)|(models.User.id==settings.USER_ID)):
lists[u.id] = u.lists_json()
return {
'lists': lists
}
actions.register(getLists)
@returns_json
def addList(request):
data = json.loads(request.form['data']) if 'data' in request.form else {}
user_id = settings.USER_ID
l = models.List.get(user_id, data['name'])
if not l:
l = models.List.create(user_id, data['name'], data.get('query'))
if 'items' in data:
l.add_items(data['items'])
return l.json()
return {}
actions.register(addList, cache=False)
@returns_json
def removeList(request):
data = json.loads(request.form['data']) if 'data' in request.form else {}
l = models.List.get(data['id'])
if l:
l.remove()
return {}
actions.register(removeList, cache=False)
@returns_json
def editList(request):
data = json.loads(request.form['data']) if 'data' in request.form else {}
l = models.List.get_or_create(data['id'])
name = l.name
if 'name' in data:
l.name = data['name']
l.type = 'static'
if 'query' in data:
l._query = data['query']
l.type = 'smart'
if l.type == 'static' and name != l.name:
Changelog.record(state.user(), 'editlist', name, {'name': l.name})
l.save()
return {}
actions.register(editList, cache=False)
@returns_json
def addListItem(request):
data = json.loads(request.form['data']) if 'data' in request.form else {}
l = models.List.get_or_create(data['id'])
i = Item.get(data['item'])
if l and i:
l.items.append(i)
models.db.session.add(l)
i.update()
return {}
actions.register(addListItem, cache=False)
@returns_json
def removeListItem(request):
data = json.loads(request.form['data']) if 'data' in request.form else {}
l = models.List.get(data['id'])
i = Item.get(data['item'])
if l and i:
l.items.remove(i)
models.db.session.add(l)
i.update()
return {}
actions.register(removeListItem, cache=False)
@returns_json
def sortLists(request):
data = json.loads(request.form['data']) if 'data' in request.form else {}
n = 0
print 'sortLists', data
for id in data['ids']:
l = models.List.get(id)
l.position = n
n += 1
models.db.session.add(l)
models.db.session.commit()
return {}
actions.register(sortLists, cache=False)
@returns_json
def editUser(request):
data = json.loads(request.form['data']) if 'data' in request.form else {}
if 'nickname' in data:
p = models.User.get_or_create(data['id'])
p.set_nickname(data['nickname'])
p.save()
return {}
actions.register(editUser, cache=False)
@returns_json
def requestPeering(request):
data = json.loads(request.form['data']) if 'data' in request.form else {}
p = models.User.get_or_create(data['id'])
state.nodes.queue('add', p.id)
state.nodes.queue(p.id, 'requestPeering', data.get('message', ''))
return {}
actions.register(requestPeering, cache=False)
@returns_json
def acceptPeering(request):
data = json.loads(request.form['data']) if 'data' in request.form else {}
p = models.User.get_or_create(data['id'])
state.nodes.queue('add', p.id)
state.nodes.queue(p.id, 'acceptPeering', data.get('message', ''))
return {}
actions.register(acceptPeering, cache=False)
@returns_json
def rejectPeering(request):
data = json.loads(request.form['data']) if 'data' in request.form else {}
p = models.User.get_or_create(data['id'])
state.nodes.queue('add', p.id)
state.nodes.queue(p.id, 'rejectPeering', data.get('message', ''))
return {}
actions.register(rejectPeering, cache=False)
@returns_json
def removePeering(request):
data = json.loads(request.form['data']) if 'data' in request.form else {}
p = models.User.get_or_create(data['id'])
state.nodes.queue('add', p.id)
state.nodes.queue(p.id, 'removePeering', data.get('message', ''))
return {}
actions.register(removePeering, cache=False)

213
oml/user/models.py Normal file
View file

@ -0,0 +1,213 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
import json
from oxflask.db import MutableDict
import oxflask.query
from changelog import Changelog
import settings
from settings import db
import state
class User(db.Model):
created = db.Column(db.DateTime())
modified = db.Column(db.DateTime())
id = db.Column(db.String(43), primary_key=True)
info = db.Column(MutableDict.as_mutable(db.PickleType(pickler=json)))
#nickname = db.Column(db.String(256), unique=True)
nickname = db.Column(db.String(256))
pending = db.Column(db.String(64)) # sent|received
peered = db.Column(db.Boolean())
online = db.Column(db.Boolean())
def __repr__(self):
return self.id
@classmethod
def get(cls, id):
return cls.query.filter_by(id=id).first()
@classmethod
def get_or_create(cls, id):
user = cls.get(id)
if not user:
user = cls(id=id, peered=False, online=False)
user.info = {}
user.save()
return user
def save(self):
db.session.add(self)
db.session.commit()
def json(self):
j = {}
if self.info:
j.update(self.info)
j['id'] = self.id
if self.pending:
j['pending'] = self.pending
j['peered'] = self.peered
j['online'] = self.check_online()
j['nickname'] = self.nickname
return j
def check_online(self):
return state.nodes.check_online(self.id)
def lists_json(self):
return [l.json() for l in self.lists.order_by('position')]
def update_peering(self, peered, username=None):
if peered:
self.pending = ''
self.peered = True
if username:
self.info['username'] = username
self.set_nickname(self.info.get('username', 'anonymous'))
else:
self.peered = False
self.nickname = None
for i in self.items:
i.users.remove(self)
if not i.users:
print 'last user, remove'
db.session.delete(i)
else:
i.update_lists()
self.save()
def set_nickname(self, nickname):
username = nickname
n = 2
while self.query.filter_by(nickname=nickname).filter(User.id!=self.id).first():
nickname = '%s [%d]' % (username, n)
n += 1
self.nickname = nickname
list_items = db.Table('listitem',
db.Column('list_id', db.Integer(), db.ForeignKey('list.id')),
db.Column('item_id', db.String(32), db.ForeignKey('item.id'))
)
class List(db.Model):
id = db.Column(db.Integer(), primary_key=True)
name = db.Column(db.String())
position = db.Column(db.Integer())
type = db.Column(db.String(64))
_query = db.Column('query', MutableDict.as_mutable(db.PickleType(pickler=json)))
user_id = db.Column(db.String(43), db.ForeignKey('user.id'))
user = db.relationship('User', backref=db.backref('lists', lazy='dynamic'))
items = db.relationship('Item', secondary=list_items,
backref=db.backref('lists', lazy='dynamic'))
@classmethod
def get(cls, user_id, name=None):
if not name:
user_id, name = cls.get_user_name(user_id)
return cls.query.filter_by(user_id=user_id, name=name).first()
@classmethod
def get_user_name(cls, user_id):
l = user_id.split(':')
nickname = l[0]
name = ':'.join(l[1:])
if nickname:
user = User.query.filter_by(nickname=nickname).first()
user_id = user.id
else:
user_id = settings.USER_ID
return user_id, name
@classmethod
def get_or_create(cls, user_id, name=None):
if not name:
user_id, name = cls.get_user_name(user_id)
l = cls.get(user_id, name)
if not l:
l = cls(name=name, user_id=user_id)
db.session.add(l)
db.session.commit()
return l
@classmethod
def create(cls, user_id, name, query=None):
l = cls(name=name, user_id=user_id)
l._query = query
l.type = 'smart' if l._query else 'static'
l.position = cls.query.filter_by(user_id=user_id).count()
if user_id == settings.USER_ID:
p = User.get(settings.USER_ID)
if not l._query:
Changelog.record(p, 'addlist', l.name)
db.session.add(l)
db.session.commit()
return l
def add_items(self, items):
from item.models import Item
for item_id in items:
i = Item.get(item_id)
self.items.add(i)
db.session.add(self)
db.session.commit()
def remove_items(self, items):
from item.models import Item
for item_id in items:
i = Item.get(item_id)
self.items.remove(i)
db.session.add(self)
db.session.commit()
def remove(self):
if not self._query:
for i in self.items:
self.items.remove(i)
if not self._query:
print 'record change: removelist', self.user, self.name
Changelog.record(self.user, 'removelist', self.name)
db.session.delete(self)
db.session.commit()
@property
def public_id(self):
id = ''
if self.user_id != settings.USER_ID:
id += self.user_id
id = '%s:%s' % (id, self.name)
return id
def items_count(self):
from item.models import Item
if self._query:
data = self._query
return oxflask.query.Parser(Item).find({'query': data}).count()
else:
return len(self.items)
def json(self):
r = {
'id': self.public_id,
'name': self.name,
'index': self.position,
'items': self.items_count(),
'type': self.type
}
if self.type == 'smart':
r['query'] = self._query
return r
def save(self):
db.session.add(self)
db.session.commit()

95
oml/utils.py Normal file
View file

@ -0,0 +1,95 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
import Image
from StringIO import StringIO
import re
import stdnum.isbn
import ox
def valid_olid(id):
return id.startswith('OL') and id.endswith('M')
def get_positions(ids, pos):
'''
>>> get_positions([1,2,3,4], [2,4])
{2: 1, 4: 3}
'''
positions = {}
for i in pos:
try:
positions[i] = ids.index(i)
except:
pass
return positions
def get_by_key(objects, key, value):
obj = filter(lambda o: o.get(key) == value, objects)
return obj and obj[0] or None
def get_by_id(objects, id):
return get_by_key(objects, 'id', id)
def resize_image(data, width=None, size=None):
source = Image.open(StringIO(data)).convert('RGB')
source_width = source.size[0]
source_height = source.size[1]
if size:
if source_width > source_height:
width = size
height = int(width / (float(source_width) / source_height))
height = height - height % 2
else:
height = size
width = int(height * (float(source_width) / source_height))
width = width - width % 2
else:
height = int(width / (float(source_width) / source_height))
height = height - height % 2
width = max(width, 1)
height = max(height, 1)
if width < source_width:
resize_method = Image.ANTIALIAS
else:
resize_method = Image.BICUBIC
output = source.resize((width, height), resize_method)
o = StringIO()
output.save(o, format='jpeg')
data = o.getvalue()
o.close()
return data
def sort_title(title):
title = title.replace(u'Æ', 'Ae')
if isinstance(title, str):
title = unicode(title)
title = ox.sort_string(title)
#title
title = re.sub(u'[\'!¿¡,\.;\-"\:\*\[\]]', '', title)
return title.strip()
def normalize_isbn(value):
return ''.join([s for s in value if s.isdigit() or s == 'X'])
def find_isbns(text):
matches = re.compile('\d[\d\-X\ ]+').findall(text)
matches = [normalize_isbn(value) for value in matches]
return [isbn for isbn in matches if stdnum.isbn.is_valid(isbn)
and len(isbn) in (10, 13)
and isbn not in (
'0' * 10,
'0' * 13,
)]
def get_position_by_id(list, key):
for i in range(0, len(list)):
if list[i]['id'] == key:
return i
return -1

86
oml/websocket.py Normal file
View file

@ -0,0 +1,86 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from __future__ import division
from tornado.websocket import WebSocketHandler
from tornado.ioloop import IOLoop
from Queue import Queue
import urllib2
import os
from contextlib import closing
import json
from threading import Thread
from oxflask.shortcuts import json_dumps
import state
class Background:
def __init__(self, handler):
self.handler = handler
self.main = IOLoop.instance()
self.q = Queue()
self.connected = True
def worker(self):
while self.connected:
message = self.q.get()
action, data = json.loads(message)
print action
print data
import item.scan
if action == 'ping':
self.post(['pong', data])
elif action == 'import':
item.scan.run_import()
elif action == 'scan':
item.scan.run_scan()
elif action == 'update':
self.post(['error', {'error': 'not implemented'}])
else:
self.post(['error', {'error': 'unknown action'}])
self.q.task_done()
def join(self):
self.q.join()
def put(self, data):
self.q.put(data)
def post(self, data):
if not isinstance(data, basestring):
data = json_dumps(data)
self.main.add_callback(lambda: self.handler.write_message(data))
class Handler(WebSocketHandler):
def open(self):
print "New connection opened."
self.background = Background(self)
state.websockets.append(self.background)
self.t = Thread(target=self.background.worker)
self.t.daemon = True
self.t.start()
#websocket calls
def on_message(self, message):
self.background.put(message)
def on_close(self):
print "Connection closed."
state.websockets.remove(self.background)
self.background.connected = False
def trigger_event(event, data):
if len(state.websockets):
print 'trigger event', event, data, len(state.websockets)
for ws in state.websockets:
try:
ws.post([event, data])
except:
import traceback
traceback.print_exc()
print 'failed to send to ws', ws, event, data