diff --git a/oml/db.py b/oml/db.py index 727d409..06bcecf 100644 --- a/oml/db.py +++ b/oml/db.py @@ -121,3 +121,14 @@ def drop_columns(table, columns): q = q.format(**info) s.connection().execute(q) s.commit() + +def get_layout(): + layout = { + 'tables': [] + } + with session() as s: + sql = "SELECT name FROM sqlite_master WHERE type='table'" + layout['tables'] = sorted([r[0] for r in s.connection().execute(sql).fetchall() if not r[0].startswith('sqlite_')]) + sql = "SELECT name FROM sqlite_master WHERE type='index'" + layout['indexes'] = sorted([r[0] for r in s.connection().execute(sql).fetchall() if not r[0].startswith('sqlite_')]) + return layout diff --git a/oml/item/models.py b/oml/item/models.py index 09d975b..b621412 100644 --- a/oml/item/models.py +++ b/oml/item/models.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- # vi:si:et:sw=4:sts=4:ts=4 from datetime import datetime -from io import BytesIO import base64 import hashlib import os @@ -10,7 +9,6 @@ import shutil import stat import unicodedata -from PIL import Image import ox from sqlalchemy.schema import CreateTable import sqlalchemy as sa @@ -801,3 +799,13 @@ def update_sort_table(): for q in sql: s.connection().execute(q) s.commit() + layout = db.get_layout() + sort_indexes = [i[len('ix_sort_'):] for i in layout['indexes'] if i.startswith('ix_sort_')] + sql = [] + for col in set(Item.sort_keys)-set(sort_indexes): + sql.append('CREATE INDEX ix_sort_{col} ON sort ({col})'.format(col=col)) + if sql: + with db.session() as s: + for q in sql: + s.connection().execute(q) + s.commit() diff --git a/oml/setup.py b/oml/setup.py index fbb22f6..2e9318c 100644 --- a/oml/setup.py +++ b/oml/setup.py @@ -103,7 +103,6 @@ CREATE TABLE sort ( author VARCHAR(1000), publisher VARCHAR(1000), place VARCHAR(1000), - country VARCHAR(1000), date VARCHAR(1000), language VARCHAR(1000), pages BIGINT, @@ -126,7 +125,6 @@ CREATE TABLE sort ( CREATE INDEX ix_sort_accessed ON sort (accessed); CREATE INDEX ix_sort_added ON sort (added); CREATE INDEX ix_sort_author ON sort (author); -CREATE INDEX ix_sort_country ON sort (country); CREATE INDEX ix_sort_created ON sort (created); CREATE INDEX ix_sort_date ON sort (date); CREATE INDEX ix_sort_extension ON sort (extension); @@ -164,8 +162,10 @@ PRAGMA journal_mode=WAL ''' for statement in sql.split(';'): run_sql(statement) - upgrade_db('0') create_default_lists() + settings.server['db_version'] = settings.DB_VERSION + update_database() + def upgrade_db(old, new=None): @@ -457,8 +457,6 @@ def upgrade_db(old, new=None): )''') run_sql('CREATE UNIQUE INDEX IF NOT EXISTS user_metadata_index ON user_metadata(id, user_id)') run_sql('CREATE INDEX ix_user_metadata_data_hash ON user_metadata (data_hash)') - import item.models - item.models.update_sort_table() def create_default_lists(user_id=None): with db.session(): @@ -470,3 +468,66 @@ def create_default_lists(user_id=None): if not l: l = List.create(user_id, list['title'], list.get('query')) +def update_database(): + import sqlalchemy as sa + from sqlalchemy.schema import CreateTable + + import changelog + import item.models + import item.person + import user.models + tables = [ + changelog.Changelog, + item.models.File, + item.models.Find, + item.models.Item, + item.models.Metadata, + item.models.Scrape, + item.models.Sort, + item.models.Transfer, + item.models.user_items, + item.person.Person, + user.models.List, + user.models.List, + user.models.list_items, + user.models.Metadata, + user.models.User, + ] + indexes = [ + 'CREATE INDEX idx_scrape_added ON scrape (added)', + 'CREATE INDEX ix_find_findvalue ON find (findvalue)', + 'CREATE INDEX ix_find_key ON find ("key")', + 'CREATE INDEX ix_useritem_user ON useritem ("user_id")', + 'CREATE INDEX ix_user_metadata_data_hash ON user_metadata (data_hash)', + 'CREATE UNIQUE INDEX listitem_index on listitem(list_id,item_id)', + 'CREATE UNIQUE INDEX useritem_index on useritem(user_id,item_id)', + 'CREATE UNIQUE INDEX user_metadata_index ON user_metadata(id, user_id)', + ] + layout = db.get_layout() + sql = [] + for t in tables: + if isinstance(t, sa.Table): + table = t + else: + table = t.__table__ + name = table.name + if name not in layout['tables']: + create_table = str(CreateTable(table).compile(db.engine)) + sql.append(create_table) + index_names = set() + for index in indexes: + name = index.split('INDEX ')[1].split()[0] + if name not in layout['indexes']: + sql.append(index) + index_names.add(name) + for index in set(name for name in layout['indexes'] if not name.startswith('ix_sort_')) - index_names: + sql.append('DROP INDEX ' + index) + if sql: + with db.session() as s: + for q in sql: + s.connection().execute(q) + s.commit() + + import item.models + item.models.update_sort_table() + diff --git a/oml/utils.py b/oml/utils.py index d2322a4..b827e33 100644 --- a/oml/utils.py +++ b/oml/utils.py @@ -9,8 +9,6 @@ from io import StringIO, BytesIO import re import stdnum.isbn import socket -import io -import gzip import time import hashlib from datetime import datetime