sync sort table to config

oml/db.py

@@ -1,3 +1,4 @@
+import re
 from contextlib import contextmanager
 from sqlalchemy import create_engine, MetaData
 from sqlalchemy import orm
@@ -77,3 +78,46 @@ class MutableDict(Mutable, dict):
 
         dict.__delitem__(self, key)
         self.changed()
+
+def run_sql(sql):
+    with session() as s:
+        s.connection().execute(sql)
+        s.commit()
+
+def table_exists(table):
+    return get_create_table(table) is not None
+
+def get_create_table(table):
+    with session() as s:
+        sql = "SELECT sql FROM sqlite_master WHERE type='table' AND name = ?"
+        row = s.connection().execute(sql, (table, )).fetchone()
+        return row[0] if row else None
+
+def get_table_columms(table):
+    create_table = get_create_table(table)
+    return [r.strip().split()[0] for r in re.compile('(.*?),').findall(create_table)][:-1]
+
+def drop_columns(table, columns):
+    if isinstance(columns, str):
+        columns = [columns]
+    new_columns = [c for c in get_table_columms(table) if c not in columns]
+    info = {
+        'table': table,
+        'columns': ','.join(new_columns),
+    }
+    create_table = get_create_table(table)
+    for column in columns:
+        create_table = re.sub('( %s .*?,)'%column, '', create_table)
+    create_table = create_table.replace('\n', '').replace(',', ',\n')
+    create_table = re.sub('\n *', '\n    ', create_table).replace('(    ', '(\n    ')
+    sql = [
+        'ALTER TABLE {table} RENAME TO {table}_old',
+        create_table,
+        'INSERT INTO {table} ({columns}) SELECT {columns} FROM {table}_old',
+        'DROP TABLE {table}_old'
+    ]
+    with session() as s:
+        for q in sql:
+            q = q.format(**info)
+            s.connection().execute(q)
+        s.commit()

oml/item/models.py

@@ -12,6 +12,7 @@ import unicodedata
 
 from PIL import Image
 import ox
+from sqlalchemy.schema import CreateTable
 import sqlalchemy as sa
 
 from changelog import Changelog
@@ -471,6 +472,7 @@ class Sort(db.Model):
             state.db.session.commit()
         return f
 
+Item.sort_keys = []
 for key in config['itemKeys']:
     if key.get('sort'):
         sort_type = key.get('sortType', key['type'])
@@ -483,6 +485,7 @@ for key in config['itemKeys']:
         else:
             col = sa.Column(sa.String(1000), index=True)
         setattr(Sort, '%s' % key['id'], col)
+        Item.sort_keys.append(key['id'])
 
 Item.id_keys = ['isbn', 'lccn', 'olid', 'oclc', 'asin']
 Item.item_keys = config['itemKeys'] 
@@ -780,3 +783,21 @@ def remove_unused_names():
     for p in Person.query.filter(Person.sortname.notin_(used)):
         state.db.session.delete(p)
     state.db.session.commit()
+
+def update_sort_table():
+    current = db.get_table_columms('sort')
+    drop_columns = list(set(current) - set(Item.sort_keys+['item_id']))
+    add_columns = list(set(Item.sort_keys+['item_id'])-set(current))
+    if drop_columns:
+        db.drop_columns('sort', drop_columns)
+    if add_columns:
+        create_table = str(CreateTable(Sort.__table__).compile(db.engine)).split('\n')
+        sql = []
+        for col in add_columns:
+            add = [r for r in create_table if '\t%s ' % col in r][0].strip()[:-1]
+            sql.append('ALTER TABLE sort ADD '+add)
+            sql.append('CREATE INDEX ix_sort_{col} ON sort ({col})'.format(col=col))
+        with db.session() as s:
+            for q in sql:
+                s.connection().execute(q)
+            s.commit()

oml/setup.py

@@ -5,14 +5,10 @@ import os
 
 import settings
 import db
+from db import run_sql
 
 from user.models import List, User
 
-def run_sql(sql):
-    with db.session() as session:
-        session.connection().execute(sql)
-        session.commit()
-
 def create_db():
     if not os.path.exists(settings.db_path):
         sql = '''
@@ -123,17 +119,12 @@ CREATE TABLE sort (
     transferprogress FLOAT, 
     id VARCHAR(1000), 
     isbn VARCHAR(1000), 
-    asin VARCHAR(1000), 
-    lccn VARCHAR(1000), 
-    olid VARCHAR(1000), 
-    oclc VARCHAR(1000), 
     random BIGINT, 
     PRIMARY KEY (item_id), 
     FOREIGN KEY(item_id) REFERENCES item (id)
 );
 CREATE INDEX ix_sort_accessed ON sort (accessed);
 CREATE INDEX ix_sort_added ON sort (added);
-CREATE INDEX ix_sort_asin ON sort (asin);
 CREATE INDEX ix_sort_author ON sort (author);
 CREATE INDEX ix_sort_country ON sort (country);
 CREATE INDEX ix_sort_created ON sort (created);
@@ -142,11 +133,8 @@ CREATE INDEX ix_sort_extension ON sort (extension);
 CREATE INDEX ix_sort_id ON sort (id);
 CREATE INDEX ix_sort_isbn ON sort (isbn);
 CREATE INDEX ix_sort_language ON sort (language);
-CREATE INDEX ix_sort_lccn ON sort (lccn);
 CREATE INDEX ix_sort_mediastate ON sort (mediastate);
 CREATE INDEX ix_sort_modified ON sort (modified);
-CREATE INDEX ix_sort_oclc ON sort (oclc);
-CREATE INDEX ix_sort_olid ON sort (olid);
 CREATE INDEX ix_sort_pages ON sort (pages);
 CREATE INDEX ix_sort_place ON sort (place);
 CREATE INDEX ix_sort_publisher ON sort (publisher);
@@ -281,22 +269,20 @@ def upgrade_db(old, new=None):
                 if f['id'] == 'classification':
                     f['id'] = 'categories'
                     settings.ui._save()
-            run_sql('ALTER TABLE sort ADD categories VARCHAR(1000)')
+            if not db.table_exists('user_metadata'):
-            run_sql('ALTER TABLE sort ADD series VARCHAR(1000)')
+                run_sql('''CREATE TABLE user_metadata (
-            run_sql('CREATE INDEX ix_sort_categories ON sort (categories)')
+                                created DATETIME, 
-            run_sql('''CREATE TABLE user_metadata (
+                                modified DATETIME, 
-                            created DATETIME, 
+                                id INTEGER NOT NULL, 
-                            modified DATETIME, 
+                                item_id VARCHAR(32), 
-                            id INTEGER NOT NULL, 
+                                user_id VARCHAR(43), 
-                            item_id VARCHAR(32), 
+                                data_hash VARCHAR(40), 
-                            user_id VARCHAR(43), 
+                                data BLOB, 
-                            data_hash VARCHAR(40), 
+                                PRIMARY KEY (id), 
-                            data BLOB, 
+                                FOREIGN KEY(user_id) REFERENCES user (id)
-                            PRIMARY KEY (id), 
+                            )''')
-                            FOREIGN KEY(user_id) REFERENCES user (id)
+                run_sql('CREATE UNIQUE INDEX IF NOT EXISTS user_metadata_index ON user_metadata(id, user_id)')
-                        )''')
+                run_sql('CREATE INDEX ix_user_metadata_data_hash ON user_metadata (data_hash)')
-            run_sql('CREATE UNIQUE INDEX IF NOT EXISTS user_metadata_index ON user_metadata(id, user_id)')
-            run_sql('CREATE INDEX ix_user_metadata_data_hash ON user_metadata (data_hash)')
             from meta.utils import to_isbn13
             from item.models import Item
             from user.models import Metadata
@@ -367,13 +353,14 @@ def upgrade_db(old, new=None):
         run_sql('CREATE INDEX ix_find_findvalue ON find (findvalue)')
 
     if old <= '20150307-272-557f4d3':
-        run_sql('''CREATE TABLE scrape (
+        if not db.table_exists('scrape'):
+            run_sql('''CREATE TABLE scrape (
     item_id VARCHAR(32) NOT NULL, 
     added DATETIME, 
     PRIMARY KEY (item_id), 
     FOREIGN KEY(item_id) REFERENCES item (id)
 )''')
-        run_sql('CREATE INDEX idx_scrape_added ON scrape (added)')
+            run_sql('CREATE INDEX idx_scrape_added ON scrape (added)')
     if old <= '20151118-346-7e86e68':
         old_key = os.path.join(settings.config_path, 'node.ssl.key')
         if os.path.exists(old_key):
@@ -456,22 +443,22 @@ def upgrade_db(old, new=None):
             session.execute(sql)
             session.commit()
     if old <= '20160111-603-90648f9' and not new:
-        run_sql('ALTER TABLE sort ADD categories VARCHAR(1000)')
+        if not db.table_exists('user_metadata'):
-        run_sql('ALTER TABLE sort ADD series VARCHAR(1000)')
+            run_sql('''CREATE TABLE user_metadata (
-        run_sql('CREATE INDEX ix_sort_categories ON sort (categories)')
+                            created DATETIME, 
-        run_sql('''CREATE TABLE user_metadata (
+                            modified DATETIME, 
-                        created DATETIME, 
+                            id INTEGER NOT NULL, 
-                        modified DATETIME, 
+                            item_id VARCHAR(32), 
-                        id INTEGER NOT NULL, 
+                            user_id VARCHAR(43), 
-                        item_id VARCHAR(32), 
+                            data_hash VARCHAR(40), 
-                        user_id VARCHAR(43), 
+                            data BLOB, 
-                        data_hash VARCHAR(40), 
+                            PRIMARY KEY (id), 
-                        data BLOB, 
+                            FOREIGN KEY(user_id) REFERENCES user (id)
-                        PRIMARY KEY (id), 
+                        )''')
-                        FOREIGN KEY(user_id) REFERENCES user (id)
+            run_sql('CREATE UNIQUE INDEX IF NOT EXISTS user_metadata_index ON user_metadata(id, user_id)')
-                    )''')
+            run_sql('CREATE INDEX ix_user_metadata_data_hash ON user_metadata (data_hash)')
-        run_sql('CREATE UNIQUE INDEX IF NOT EXISTS user_metadata_index ON user_metadata(id, user_id)')
+    import item.models
-        run_sql('CREATE INDEX ix_user_metadata_data_hash ON user_metadata (data_hash)')
+    item.models.update_sort_table()
 
 def create_default_lists(user_id=None):
     with db.session():