sync sort table to config

oml/db.py

@@ -1,3 +1,4 @@
+import re
 from contextlib import contextmanager
 from sqlalchemy import create_engine, MetaData
 from sqlalchemy import orm
@@ -77,3 +78,46 @@ class MutableDict(Mutable, dict):
 
         dict.__delitem__(self, key)
         self.changed()
+
+def run_sql(sql):
+    with session() as s:
+        s.connection().execute(sql)
+        s.commit()
+
+def table_exists(table):
+    return get_create_table(table) is not None
+
+def get_create_table(table):
+    with session() as s:
+        sql = "SELECT sql FROM sqlite_master WHERE type='table' AND name = ?"
+        row = s.connection().execute(sql, (table, )).fetchone()
+        return row[0] if row else None
+
+def get_table_columms(table):
+    create_table = get_create_table(table)
+    return [r.strip().split()[0] for r in re.compile('(.*?),').findall(create_table)][:-1]
+
+def drop_columns(table, columns):
+    if isinstance(columns, str):
+        columns = [columns]
+    new_columns = [c for c in get_table_columms(table) if c not in columns]
+    info = {
+        'table': table,
+        'columns': ','.join(new_columns),
+    }
+    create_table = get_create_table(table)
+    for column in columns:
+        create_table = re.sub('( %s .*?,)'%column, '', create_table)
+    create_table = create_table.replace('\n', '').replace(',', ',\n')
+    create_table = re.sub('\n *', '\n    ', create_table).replace('(    ', '(\n    ')
+    sql = [
+        'ALTER TABLE {table} RENAME TO {table}_old',
+        create_table,
+        'INSERT INTO {table} ({columns}) SELECT {columns} FROM {table}_old',
+        'DROP TABLE {table}_old'
+    ]
+    with session() as s:
+        for q in sql:
+            q = q.format(**info)
+            s.connection().execute(q)
+        s.commit()

oml/item/models.py

@@ -12,6 +12,7 @@ import unicodedata
 
 from PIL import Image
 import ox
+from sqlalchemy.schema import CreateTable
 import sqlalchemy as sa
 
 from changelog import Changelog
@@ -471,6 +472,7 @@ class Sort(db.Model):
             state.db.session.commit()
         return f
 
+Item.sort_keys = []
 for key in config['itemKeys']:
     if key.get('sort'):
         sort_type = key.get('sortType', key['type'])
@@ -483,6 +485,7 @@ for key in config['itemKeys']:
         else:
             col = sa.Column(sa.String(1000), index=True)
         setattr(Sort, '%s' % key['id'], col)
+        Item.sort_keys.append(key['id'])
 
 Item.id_keys = ['isbn', 'lccn', 'olid', 'oclc', 'asin']
 Item.item_keys = config['itemKeys'] 
@@ -780,3 +783,21 @@ def remove_unused_names():
     for p in Person.query.filter(Person.sortname.notin_(used)):
         state.db.session.delete(p)
     state.db.session.commit()
+
+def update_sort_table():
+    current = db.get_table_columms('sort')
+    drop_columns = list(set(current) - set(Item.sort_keys+['item_id']))
+    add_columns = list(set(Item.sort_keys+['item_id'])-set(current))
+    if drop_columns:
+        db.drop_columns('sort', drop_columns)
+    if add_columns:
+        create_table = str(CreateTable(Sort.__table__).compile(db.engine)).split('\n')
+        sql = []
+        for col in add_columns:
+            add = [r for r in create_table if '\t%s ' % col in r][0].strip()[:-1]
+            sql.append('ALTER TABLE sort ADD '+add)
+            sql.append('CREATE INDEX ix_sort_{col} ON sort ({col})'.format(col=col))
+        with db.session() as s:
+            for q in sql:
+                s.connection().execute(q)
+            s.commit()

oml/setup.py

@@ -5,14 +5,10 @@ import os
 
 import settings
 import db
+from db import run_sql
 
 from user.models import List, User
 
-def run_sql(sql):
-    with db.session() as session:
-        session.connection().execute(sql)
-        session.commit()
-
 def create_db():
     if not os.path.exists(settings.db_path):
         sql = '''
@@ -123,17 +119,12 @@ CREATE TABLE sort (
     transferprogress FLOAT, 
     id VARCHAR(1000), 
     isbn VARCHAR(1000), 
-    asin VARCHAR(1000), 
-    lccn VARCHAR(1000), 
-    olid VARCHAR(1000), 
-    oclc VARCHAR(1000), 
     random BIGINT, 
     PRIMARY KEY (item_id), 
     FOREIGN KEY(item_id) REFERENCES item (id)
 );
 CREATE INDEX ix_sort_accessed ON sort (accessed);
 CREATE INDEX ix_sort_added ON sort (added);
-CREATE INDEX ix_sort_asin ON sort (asin);
 CREATE INDEX ix_sort_author ON sort (author);
 CREATE INDEX ix_sort_country ON sort (country);
 CREATE INDEX ix_sort_created ON sort (created);
@@ -142,11 +133,8 @@ CREATE INDEX ix_sort_extension ON sort (extension);
 CREATE INDEX ix_sort_id ON sort (id);
 CREATE INDEX ix_sort_isbn ON sort (isbn);
 CREATE INDEX ix_sort_language ON sort (language);
-CREATE INDEX ix_sort_lccn ON sort (lccn);
 CREATE INDEX ix_sort_mediastate ON sort (mediastate);
 CREATE INDEX ix_sort_modified ON sort (modified);
-CREATE INDEX ix_sort_oclc ON sort (oclc);
-CREATE INDEX ix_sort_olid ON sort (olid);
 CREATE INDEX ix_sort_pages ON sort (pages);
 CREATE INDEX ix_sort_place ON sort (place);
 CREATE INDEX ix_sort_publisher ON sort (publisher);
@@ -281,9 +269,7 @@ def upgrade_db(old, new=None):
                 if f['id'] == 'classification':
                     f['id'] = 'categories'
                     settings.ui._save()
-            run_sql('ALTER TABLE sort ADD categories VARCHAR(1000)')
+            if not db.table_exists('user_metadata'):
-            run_sql('ALTER TABLE sort ADD series VARCHAR(1000)')
-            run_sql('CREATE INDEX ix_sort_categories ON sort (categories)')
                 run_sql('''CREATE TABLE user_metadata (
                                 created DATETIME, 
                                 modified DATETIME, 
@@ -367,6 +353,7 @@ def upgrade_db(old, new=None):
         run_sql('CREATE INDEX ix_find_findvalue ON find (findvalue)')
 
     if old <= '20150307-272-557f4d3':
+        if not db.table_exists('scrape'):
             run_sql('''CREATE TABLE scrape (
     item_id VARCHAR(32) NOT NULL, 
     added DATETIME, 
@@ -456,9 +443,7 @@ def upgrade_db(old, new=None):
             session.execute(sql)
             session.commit()
     if old <= '20160111-603-90648f9' and not new:
-        run_sql('ALTER TABLE sort ADD categories VARCHAR(1000)')
+        if not db.table_exists('user_metadata'):
-        run_sql('ALTER TABLE sort ADD series VARCHAR(1000)')
-        run_sql('CREATE INDEX ix_sort_categories ON sort (categories)')
             run_sql('''CREATE TABLE user_metadata (
                             created DATETIME, 
                             modified DATETIME, 
@@ -472,6 +457,8 @@ def upgrade_db(old, new=None):
                         )''')
             run_sql('CREATE UNIQUE INDEX IF NOT EXISTS user_metadata_index ON user_metadata(id, user_id)')
             run_sql('CREATE INDEX ix_user_metadata_data_hash ON user_metadata (data_hash)')
+    import item.models
+    item.models.update_sort_table()
 
 def create_default_lists(user_id=None):
     with db.session():