diff --git a/oml/app.py b/oml/app.py index becdce7..fc1a3a9 100644 --- a/oml/app.py +++ b/oml/app.py @@ -10,9 +10,10 @@ import logging import settings -from settings import db import changelog + +from db import session import item.models import user.models import item.person @@ -22,13 +23,9 @@ import api import commands app = Flask('openmedialibrary', static_folder=settings.static_path) -app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:////%s' % settings.db_path -db.init_app(app) -migrate = Migrate(app, db) manager = Manager(app, with_default_commands=False) -manager.add_command('db', MigrateCommand) manager.add_command('release', commands.Release) manager.add_command('debug', commands.Debug) manager.add_command('update', commands.Update) diff --git a/oml/changelog.py b/oml/changelog.py index 51d0d43..a85f6fc 100644 --- a/oml/changelog.py +++ b/oml/changelog.py @@ -10,7 +10,8 @@ from datetime import datetime from utils import valid, datetime2ts, ts2datetime import settings -from settings import db +import db +import sqlalchemy as sa import state from websocket import trigger_event @@ -35,15 +36,16 @@ class Changelog(db.Model): editmeta key, value data (i.e. 'isbn', '0000000000', {title: 'Example'}) resetmeta key, value ''' - id = db.Column(db.Integer(), primary_key=True) + __tablename__ = 'changelog' + id = sa.Column(sa.Integer(), primary_key=True) - created = db.Column(db.DateTime()) - timestamp = db.Column(db.BigInteger()) + created = sa.Column(sa.DateTime()) + timestamp = sa.Column(sa.BigInteger()) - user_id = db.Column(db.String(43)) - revision = db.Column(db.BigInteger()) - data = db.Column(db.Text()) - sig = db.Column(db.String(96)) + user_id = sa.Column(sa.String(43)) + revision = sa.Column(sa.BigInteger()) + data = sa.Column(sa.Text()) + sig = sa.Column(sa.String(96)) @classmethod def record(cls, user, action, *args): diff --git a/oml/commands.py b/oml/commands.py index e06954e..a6a073b 100644 --- a/oml/commands.py +++ b/oml/commands.py @@ -95,7 +95,7 @@ class PostUpdate(Command): Option('-n', '--new', dest='new'), ] - def run(selfi, old, new): + def run(self, old, new): if old <= '20140521-65-e14c686' and new > '20140521-65-e14c686': if not os.path.exists(settings.db_path): r('./ctl', 'setup') @@ -122,11 +122,9 @@ class Setup(Command): Setup new node """ def run(self): - r('./ctl', 'db', 'upgrade') import setup + setup.create_db() setup.create_default_lists() - settings.db.session.connection().execute("PRAGMA journal_mode=WAL") - settings.db.session.commit() class UpdateStatic(Command): """ diff --git a/oml/db.py b/oml/db.py index 8e6f126..45943cd 100644 --- a/oml/db.py +++ b/oml/db.py @@ -1,4 +1,19 @@ +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker from sqlalchemy.ext.mutable import Mutable +from sqlalchemy.ext.declarative import declarative_base +import settings + + +engine = create_engine('sqlite:////%s' % settings.db_path) +Session = sessionmaker(bind=engine) + +# create a Session +session = Session() + + +Model = declarative_base() + class MutableDict(Mutable, dict): @classmethod diff --git a/oml/item/models.py b/oml/item/models.py index 77e6374..3318d13 100644 --- a/oml/item/models.py +++ b/oml/item/models.py @@ -16,10 +16,12 @@ import unicodedata import Image import ox +import db from db import MutableDict +import sqlalchemy as sa import settings -from settings import db, config +from settings import config from person import get_sort_name @@ -39,29 +41,30 @@ from utils import remove_empty_folders logger = logging.getLogger('oml.item.model') - -user_items = db.Table('useritem', - db.Column('user_id', db.String(43), db.ForeignKey('user.id')), - db.Column('item_id', db.String(32), db.ForeignKey('item.id')) +metadata = sa.MetaData() +user_items = sa.Table('useritem', metadata, + sa.Column('user_id', sa.String(43), sa.ForeignKey('user.id')), + sa.Column('item_id', sa.String(32), sa.ForeignKey('item.id')) ) class Item(db.Model): + __tablename__ = 'item' - created = db.Column(db.DateTime()) - modified = db.Column(db.DateTime()) + created = sa.Column(sa.DateTime()) + modified = sa.Column(sa.DateTime()) - id = db.Column(db.String(32), primary_key=True) + id = sa.Column(sa.String(32), primary_key=True) - info = db.Column(MutableDict.as_mutable(db.PickleType(pickler=json))) - meta = db.Column(MutableDict.as_mutable(db.PickleType(pickler=json))) + info = sa.Column(MutableDict.as_mutable(sa.PickleType(pickler=json))) + meta = sa.Column(MutableDict.as_mutable(sa.PickleType(pickler=json))) # why is this in db and not in i.e. info? - added = db.Column(db.DateTime()) # added to local library - accessed = db.Column(db.DateTime()) - timesaccessed = db.Column(db.Integer()) + added = sa.Column(sa.DateTime()) # added to local library + accessed = sa.Column(sa.DateTime()) + timesaccessed = sa.Column(sa.Integer()) - users = db.relationship('User', secondary=user_items, - backref=db.backref('items', lazy='dynamic')) + users = sa.orm.relationship('User', secondary=user_items, + backref=sa.orm.backref('items', lazy='dynamic')) @property def timestamp(self): @@ -404,8 +407,10 @@ class Item(db.Model): Changelog.record(user, 'removeitem', self.id) class Sort(db.Model): - item_id = db.Column(db.String(32), db.ForeignKey('item.id'), primary_key=True) - item = db.relationship('Item', backref=db.backref('sort', lazy='dynamic')) + __tablename__ = 'sort' + + item_id = sa.Column(sa.String(32), sa.ForeignKey('item.id'), primary_key=True) + item = sa.orm.relationship('Item', backref=sa.orm.backref('sort', lazy='dynamic')) def __repr__(self): return '%s_sort' % self.item_id @@ -427,13 +432,13 @@ for key in config['itemKeys']: if key.get('sort'): sort_type = key.get('sortType', key['type']) if sort_type == 'integer': - col = db.Column(db.BigInteger(), index=True) + col = sa.Column(sa.BigInteger(), index=True) elif sort_type == 'float': - col = db.Column(db.Float(), index=True) + col = sa.Column(sa.Float(), index=True) elif sort_type == 'date': - col = db.Column(db.DateTime(), index=True) + col = sa.Column(sa.DateTime(), index=True) else: - col = db.Column(db.String(1000), index=True) + col = sa.Column(sa.String(1000), index=True) setattr(Sort, '%s' % key['id'], col) Item.id_keys = ['isbn', 'lccn', 'olid', 'oclc', 'asin'] @@ -441,12 +446,14 @@ Item.item_keys = config['itemKeys'] Item.filter_keys = [k['id'] for k in config['itemKeys'] if k.get('filter')] class Find(db.Model): - id = db.Column(db.Integer(), primary_key=True) - item_id = db.Column(db.String(32), db.ForeignKey('item.id')) - item = db.relationship('Item', backref=db.backref('find', lazy='dynamic')) - key = db.Column(db.String(200), index=True) - value = db.Column(db.Text()) - findvalue = db.Column(db.Text(), index=True) + __tablename__ = 'find' + + id = sa.Column(sa.Integer(), primary_key=True) + item_id = sa.Column(sa.String(32), sa.ForeignKey('item.id')) + item = sa.orm.relationship('Item', backref=sa.orm.backref('find', lazy='dynamic')) + key = sa.Column(sa.String(200), index=True) + value = sa.Column(sa.Text()) + findvalue = sa.Column(sa.Text(), index=True) def __repr__(self): return (u'%s=%s' % (self.key, self.findvalue)).encode('utf-8') @@ -465,17 +472,18 @@ class Find(db.Model): return f class File(db.Model): + __tablename__ = 'file' - created = db.Column(db.DateTime()) - modified = db.Column(db.DateTime()) + created = sa.Column(sa.DateTime()) + modified = sa.Column(sa.DateTime()) - sha1 = db.Column(db.String(32), primary_key=True) - path = db.Column(db.String(2048)) + sha1 = sa.Column(sa.String(32), primary_key=True) + path = sa.Column(sa.String(2048)) - info = db.Column(MutableDict.as_mutable(db.PickleType(pickler=json))) + info = sa.Column(MutableDict.as_mutable(sa.PickleType(pickler=json))) - item_id = db.Column(db.String(32), db.ForeignKey('item.id')) - item = db.relationship('Item', backref=db.backref('files', lazy='dynamic')) + item_id = sa.Column(sa.String(32), sa.ForeignKey('item.id')) + item = sa.orm.relationship('Item', backref=sa.orm.backref('files', lazy='dynamic')) @classmethod def get(cls, sha1): @@ -553,12 +561,13 @@ class File(db.Model): class Transfer(db.Model): + __tablename__ = 'transfer' - item_id = db.Column(db.String(32), db.ForeignKey('item.id'), primary_key=True) - item = db.relationship('Item', backref=db.backref('transfer', lazy='dynamic')) + item_id = sa.Column(sa.String(32), sa.ForeignKey('item.id'), primary_key=True) + item = sa.orm.relationship('Item', backref=sa.orm.backref('transfer', lazy='dynamic')) - added = db.Column(db.DateTime()) - progress = db.Column(db.Float()) + added = sa.Column(sa.DateTime()) + progress = sa.Column(sa.Float()) def __repr__(self): return '='.join(map(str, [self.item_id, self.progress])) @@ -582,16 +591,17 @@ class Transfer(db.Model): db.session.commit() class Metadata(db.Model): + __tablename__ = 'metadata' - created = db.Column(db.DateTime()) - modified = db.Column(db.DateTime()) + created = sa.Column(sa.DateTime()) + modified = sa.Column(sa.DateTime()) - id = db.Column(db.Integer(), primary_key=True) + id = sa.Column(sa.Integer(), primary_key=True) - key = db.Column(db.String(256)) - value = db.Column(db.String(256)) + key = sa.Column(sa.String(256)) + value = sa.Column(sa.String(256)) - data = db.Column(MutableDict.as_mutable(db.PickleType(pickler=json))) + data = sa.Column(MutableDict.as_mutable(sa.PickleType(pickler=json))) def __repr__(self): return '='.join([self.key, self.value]) diff --git a/oml/item/person.py b/oml/item/person.py index 602d642..c472487 100644 --- a/oml/item/person.py +++ b/oml/item/person.py @@ -6,7 +6,8 @@ import unicodedata import ox -from settings import db +import db +import sqlalchemy as sa def get_sort_name(name, sortname=None): name = unicodedata.normalize('NFKD', name).strip() @@ -21,9 +22,11 @@ def get_sort_name(name, sortname=None): return sortname class Person(db.Model): - name = db.Column(db.String(1024), primary_key=True) - sortname = db.Column(db.String()) - numberofnames = db.Column(db.Integer()) + __tablename__ = 'person' + + name = sa.Column(sa.String(1024), primary_key=True) + sortname = sa.Column(sa.String()) + numberofnames = sa.Column(sa.Integer()) def __repr__(self): return self.name diff --git a/oml/settings.py b/oml/settings.py index f2a55e5..dca5ab3 100644 --- a/oml/settings.py +++ b/oml/settings.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- # vi:si:et:sw=4:sts=4:ts=4 -from flask.ext.sqlalchemy import SQLAlchemy import json import os import ed25519 @@ -24,7 +23,6 @@ key_path = os.path.join(config_path, 'node.key') ssl_cert_path = os.path.join(config_path, 'node.ssl.crt') ssl_key_path = os.path.join(config_path, 'node.ssl.key') -db = SQLAlchemy() if os.path.exists(oml_config_path): with open(oml_config_path) as fd: diff --git a/oml/setup.py b/oml/setup.py index 6965293..7d5308e 100644 --- a/oml/setup.py +++ b/oml/setup.py @@ -1,9 +1,25 @@ # -*- coding: utf-8 -*- # vi:si:et:sw=4:sts=4:ts=4 +import os + import settings +from db import session + from user.models import List, User + +def create_db(): + if not os.path.exists(settings.db_path): + print 'create db' + session.connection().execute("PRAGMA journal_mode=WAL") + session.commit() + upgrade_db('0') + +def upgrade_db(old): + if old <= '20140527-120-3cb9819': + create_index('ix_find_findvalue', 'find', ['findvalue'], unique=False) + def create_default_lists(user_id=None): user_id = user_id or settings.USER_ID user = User.get_or_create(user_id) diff --git a/oml/user/models.py b/oml/user/models.py index ac7e4f4..51046de 100644 --- a/oml/user/models.py +++ b/oml/user/models.py @@ -3,12 +3,13 @@ import json +import db from db import MutableDict +import sqlalchemy as sa from queryparser import Parser from changelog import Changelog import settings -from settings import db import state @@ -16,19 +17,20 @@ import logging logger = logging.getLogger('oml.user.models') class User(db.Model): + __tablename__ = 'user' - created = db.Column(db.DateTime()) - modified = db.Column(db.DateTime()) + created = sa.Column(sa.DateTime()) + modified = sa.Column(sa.DateTime()) - id = db.Column(db.String(43), primary_key=True) - info = db.Column(MutableDict.as_mutable(db.PickleType(pickler=json))) + id = sa.Column(sa.String(43), primary_key=True) + info = sa.Column(MutableDict.as_mutable(sa.PickleType(pickler=json))) - nickname = db.Column(db.String(256), unique=True) + nickname = sa.Column(sa.String(256), unique=True) - pending = db.Column(db.String(64)) # sent|received - queued = db.Column(db.Boolean()) - peered = db.Column(db.Boolean()) - online = db.Column(db.Boolean()) + pending = sa.Column(sa.String(64)) # sent|received + queued = sa.Column(sa.Boolean()) + peered = sa.Column(sa.Boolean()) + online = sa.Column(sa.Boolean()) def __repr__(self): return self.id @@ -126,24 +128,27 @@ class User(db.Model): n += 1 self.nickname = nickname -list_items = db.Table('listitem', - db.Column('list_id', db.Integer(), db.ForeignKey('list.id')), - db.Column('item_id', db.String(32), db.ForeignKey('item.id')) +metadata = sa.MetaData() +list_items = sa.Table('listitem', metadata, + sa.Column('list_id', sa.Integer(), sa.ForeignKey('list.id')), + sa.Column('item_id', sa.String(32), sa.ForeignKey('item.id')) ) class List(db.Model): - id = db.Column(db.Integer(), primary_key=True) - name = db.Column(db.String()) - index_ = db.Column(db.Integer()) + __tablename__ = 'list' - type = db.Column(db.String(64)) - _query = db.Column('query', MutableDict.as_mutable(db.PickleType(pickler=json))) + id = sa.Column(sa.Integer(), primary_key=True) + name = sa.Column(sa.String()) + index_ = sa.Column(sa.Integer()) - user_id = db.Column(db.String(43), db.ForeignKey('user.id')) - user = db.relationship('User', backref=db.backref('lists', lazy='dynamic')) + type = sa.Column(sa.String(64)) + _query = sa.Column('query', MutableDict.as_mutable(sa.PickleType(pickler=json))) - items = db.relationship('Item', secondary=list_items, - backref=db.backref('lists', lazy='dynamic')) + user_id = sa.Column(sa.String(43), sa.ForeignKey('user.id')) + user = sa.orm.relationship('User', backref=sa.orm.backref('lists', lazy='dynamic')) + + items = sa.orm.relationship('Item', secondary=list_items, + backref=sa.orm.backref('lists', lazy='dynamic')) @classmethod def get(cls, user_id, name=None):