update database on startup

oml/db.py

@@ -121,3 +121,14 @@ def drop_columns(table, columns):
             q = q.format(**info)
             s.connection().execute(q)
         s.commit()
+
+def get_layout():
+    layout = {
+        'tables': []
+    }
+    with session() as s:
+        sql = "SELECT name FROM sqlite_master WHERE type='table'"
+        layout['tables'] = sorted([r[0] for r in s.connection().execute(sql).fetchall() if not r[0].startswith('sqlite_')])
+        sql = "SELECT name FROM sqlite_master WHERE type='index'"
+        layout['indexes'] = sorted([r[0] for r in s.connection().execute(sql).fetchall() if not r[0].startswith('sqlite_')])
+    return layout

oml/item/models.py

@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 # vi:si:et:sw=4:sts=4:ts=4
 from datetime import datetime
-from io import BytesIO
 import base64
 import hashlib
 import os
@@ -10,7 +9,6 @@ import shutil
 import stat
 import unicodedata
 
-from PIL import Image
 import ox
 from sqlalchemy.schema import CreateTable
 import sqlalchemy as sa
@@ -801,3 +799,13 @@ def update_sort_table():
             for q in sql:
                 s.connection().execute(q)
             s.commit()
+    layout = db.get_layout()
+    sort_indexes = [i[len('ix_sort_'):] for i in layout['indexes'] if i.startswith('ix_sort_')]
+    sql = []
+    for col in set(Item.sort_keys)-set(sort_indexes):
+        sql.append('CREATE INDEX ix_sort_{col} ON sort ({col})'.format(col=col))
+    if sql:
+        with db.session() as s:
+            for q in sql:
+                s.connection().execute(q)
+            s.commit()

oml/setup.py

@@ -103,7 +103,6 @@ CREATE TABLE sort (
     author VARCHAR(1000), 
     publisher VARCHAR(1000), 
     place VARCHAR(1000), 
-    country VARCHAR(1000), 
     date VARCHAR(1000), 
     language VARCHAR(1000), 
     pages BIGINT, 
@@ -126,7 +125,6 @@ CREATE TABLE sort (
 CREATE INDEX ix_sort_accessed ON sort (accessed);
 CREATE INDEX ix_sort_added ON sort (added);
 CREATE INDEX ix_sort_author ON sort (author);
-CREATE INDEX ix_sort_country ON sort (country);
 CREATE INDEX ix_sort_created ON sort (created);
 CREATE INDEX ix_sort_date ON sort (date);
 CREATE INDEX ix_sort_extension ON sort (extension);
@@ -164,8 +162,10 @@ PRAGMA journal_mode=WAL
 '''
         for statement in sql.split(';'):
             run_sql(statement)
-        upgrade_db('0')
         create_default_lists()
+        settings.server['db_version'] = settings.DB_VERSION
+    update_database()
+
 
 def upgrade_db(old, new=None):
 
@@ -457,8 +457,6 @@ def upgrade_db(old, new=None):
                         )''')
             run_sql('CREATE UNIQUE INDEX IF NOT EXISTS user_metadata_index ON user_metadata(id, user_id)')
             run_sql('CREATE INDEX ix_user_metadata_data_hash ON user_metadata (data_hash)')
-    import item.models
-    item.models.update_sort_table()
 
 def create_default_lists(user_id=None):
     with db.session():
@@ -470,3 +468,66 @@ def create_default_lists(user_id=None):
             if not l:
                 l = List.create(user_id, list['title'], list.get('query'))
 
+def update_database():
+    import sqlalchemy as sa
+    from sqlalchemy.schema import CreateTable
+
+    import changelog
+    import item.models
+    import item.person
+    import user.models
+    tables = [
+        changelog.Changelog,
+        item.models.File,
+        item.models.Find,
+        item.models.Item,
+        item.models.Metadata,
+        item.models.Scrape,
+        item.models.Sort,
+        item.models.Transfer,
+        item.models.user_items,
+        item.person.Person,
+        user.models.List,
+        user.models.List,
+        user.models.list_items,
+        user.models.Metadata,
+        user.models.User,
+    ]
+    indexes = [
+        'CREATE INDEX idx_scrape_added ON scrape (added)',
+        'CREATE INDEX ix_find_findvalue ON find (findvalue)',
+        'CREATE INDEX ix_find_key ON find ("key")',
+        'CREATE INDEX ix_useritem_user ON useritem ("user_id")',
+        'CREATE INDEX ix_user_metadata_data_hash ON user_metadata (data_hash)',
+        'CREATE UNIQUE INDEX listitem_index on listitem(list_id,item_id)',
+        'CREATE UNIQUE INDEX useritem_index on useritem(user_id,item_id)',
+        'CREATE UNIQUE INDEX user_metadata_index ON user_metadata(id, user_id)',
+    ]
+    layout = db.get_layout()
+    sql = []
+    for t in tables:
+        if isinstance(t, sa.Table):
+            table = t
+        else:
+            table = t.__table__
+        name = table.name
+        if name not in layout['tables']:
+            create_table = str(CreateTable(table).compile(db.engine))
+            sql.append(create_table)
+    index_names = set()
+    for index in indexes:
+        name = index.split('INDEX ')[1].split()[0]
+        if name not in layout['indexes']:
+            sql.append(index)
+        index_names.add(name)
+    for index in set(name for name in layout['indexes'] if not name.startswith('ix_sort_')) - index_names:
+        sql.append('DROP INDEX ' + index)
+    if sql:
+        with db.session() as s:
+            for q in sql:
+                s.connection().execute(q)
+            s.commit()
+
+    import item.models
+    item.models.update_sort_table()
+

oml/utils.py

@@ -9,8 +9,6 @@ from io import StringIO, BytesIO
 import re
 import stdnum.isbn
 import socket
-import io
-import gzip
 import time
 import hashlib
 from datetime import datetime