0x2620
/
openmedialibrary
1
0
Fork 0
Code Issues 131 Pull Requests Releases Wiki Activity

sync sort table to config

This commit is contained in:
j 2016-01-16 21:27:15 +05:30
parent 601c23f985
commit 69710090e0
3 changed files with 99 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
oml/db.py
View File

@ -1,3 +1,4 @@
import re
from contextlib import contextmanager
from sqlalchemy import create_engine, MetaData
from sqlalchemy import orm
@ -77,3 +78,46 @@ class MutableDict(Mutable, dict):
dict.__delitem__(self, key)
self.changed()
def run_sql(sql):
with session() as s:
s.connection().execute(sql)
s.commit()
def table_exists(table):
return get_create_table(table) is not None
def get_create_table(table):
with session() as s:
sql = "SELECT sql FROM sqlite_master WHERE type='table' AND name = ?"
row = s.connection().execute(sql, (table, )).fetchone()
return row[0] if row else None
def get_table_columms(table):
create_table = get_create_table(table)
return [r.strip().split()[0] for r in re.compile('(.*?),').findall(create_table)][:-1]
def drop_columns(table, columns):
if isinstance(columns, str):
columns = [columns]
new_columns = [c for c in get_table_columms(table) if c not in columns]
info = {
'table': table,
'columns': ','.join(new_columns),
}
create_table = get_create_table(table)
for column in columns:
create_table = re.sub('( %s .*?,)'%column, '', create_table)
create_table = create_table.replace('\n', '').replace(',', ',\n')
create_table = re.sub('\n *', '\n ', create_table).replace('( ', '(\n ')
sql = [
'ALTER TABLE {table} RENAME TO {table}_old',
create_table,
'INSERT INTO {table} ({columns}) SELECT {columns} FROM {table}_old',
'DROP TABLE {table}_old'
]
with session() as s:
for q in sql:
q = q.format(**info)
s.connection().execute(q)
s.commit()

21
oml/item/models.py
View File

@ -12,6 +12,7 @@ import unicodedata
from PIL import Image
import ox
from sqlalchemy.schema import CreateTable
import sqlalchemy as sa
from changelog import Changelog
@ -471,6 +472,7 @@ class Sort(db.Model):
state.db.session.commit()
return f
Item.sort_keys = []
for key in config['itemKeys']:
if key.get('sort'):
sort_type = key.get('sortType', key['type'])
@ -483,6 +485,7 @@ for key in config['itemKeys']:
else:
col = sa.Column(sa.String(1000), index=True)
setattr(Sort, '%s' % key['id'], col)
Item.sort_keys.append(key['id'])
Item.id_keys = ['isbn', 'lccn', 'olid', 'oclc', 'asin']
Item.item_keys = config['itemKeys']
@ -780,3 +783,21 @@ def remove_unused_names():
for p in Person.query.filter(Person.sortname.notin_(used)):
state.db.session.delete(p)
state.db.session.commit()
def update_sort_table():
current = db.get_table_columms('sort')
drop_columns = list(set(current) - set(Item.sort_keys+['item_id']))
add_columns = list(set(Item.sort_keys+['item_id'])-set(current))
if drop_columns:
db.drop_columns('sort', drop_columns)
if add_columns:
create_table = str(CreateTable(Sort.__table__).compile(db.engine)).split('\n')
sql = []
for col in add_columns:
add = [r for r in create_table if '\t%s ' % col in r][0].strip()[:-1]
sql.append('ALTER TABLE sort ADD '+add)
sql.append('CREATE INDEX ix_sort_{col} ON sort ({col})'.format(col=col))
with db.session() as s:
for q in sql:
s.connection().execute(q)
s.commit()

81
oml/setup.py
View File

@ -5,14 +5,10 @@ import os
import settings
import db
from db import run_sql
from user.models import List, User
def run_sql(sql):
with db.session() as session:
session.connection().execute(sql)
session.commit()
def create_db():
if not os.path.exists(settings.db_path):
sql = '''
@ -123,17 +119,12 @@ CREATE TABLE sort (
transferprogress FLOAT,
id VARCHAR(1000),
isbn VARCHAR(1000),
asin VARCHAR(1000),
lccn VARCHAR(1000),
olid VARCHAR(1000),
oclc VARCHAR(1000),
random BIGINT,
PRIMARY KEY (item_id),
FOREIGN KEY(item_id) REFERENCES item (id)
);
CREATE INDEX ix_sort_accessed ON sort (accessed);
CREATE INDEX ix_sort_added ON sort (added);
CREATE INDEX ix_sort_asin ON sort (asin);
CREATE INDEX ix_sort_author ON sort (author);
CREATE INDEX ix_sort_country ON sort (country);
CREATE INDEX ix_sort_created ON sort (created);
@ -142,11 +133,8 @@ CREATE INDEX ix_sort_extension ON sort (extension);
CREATE INDEX ix_sort_id ON sort (id);
CREATE INDEX ix_sort_isbn ON sort (isbn);
CREATE INDEX ix_sort_language ON sort (language);
CREATE INDEX ix_sort_lccn ON sort (lccn);
CREATE INDEX ix_sort_mediastate ON sort (mediastate);
CREATE INDEX ix_sort_modified ON sort (modified);
CREATE INDEX ix_sort_oclc ON sort (oclc);
CREATE INDEX ix_sort_olid ON sort (olid);
CREATE INDEX ix_sort_pages ON sort (pages);
CREATE INDEX ix_sort_place ON sort (place);
CREATE INDEX ix_sort_publisher ON sort (publisher);
@ -281,22 +269,20 @@ def upgrade_db(old, new=None):
if f['id'] == 'classification':
f['id'] = 'categories'
settings.ui._save()
run_sql('ALTER TABLE sort ADD categories VARCHAR(1000)')
run_sql('ALTER TABLE sort ADD series VARCHAR(1000)')
run_sql('CREATE INDEX ix_sort_categories ON sort (categories)')
run_sql('''CREATE TABLE user_metadata (
created DATETIME,
modified DATETIME,
id INTEGER NOT NULL,
item_id VARCHAR(32),
user_id VARCHAR(43),
data_hash VARCHAR(40),
data BLOB,
PRIMARY KEY (id),
FOREIGN KEY(user_id) REFERENCES user (id)
)''')
run_sql('CREATE UNIQUE INDEX IF NOT EXISTS user_metadata_index ON user_metadata(id, user_id)')
run_sql('CREATE INDEX ix_user_metadata_data_hash ON user_metadata (data_hash)')
if not db.table_exists('user_metadata'):
run_sql('''CREATE TABLE user_metadata (
created DATETIME,
modified DATETIME,
id INTEGER NOT NULL,
item_id VARCHAR(32),
user_id VARCHAR(43),
data_hash VARCHAR(40),
data BLOB,
PRIMARY KEY (id),
FOREIGN KEY(user_id) REFERENCES user (id)
)''')
run_sql('CREATE UNIQUE INDEX IF NOT EXISTS user_metadata_index ON user_metadata(id, user_id)')
run_sql('CREATE INDEX ix_user_metadata_data_hash ON user_metadata (data_hash)')
from meta.utils import to_isbn13
from item.models import Item
from user.models import Metadata
@ -367,13 +353,14 @@ def upgrade_db(old, new=None):
run_sql('CREATE INDEX ix_find_findvalue ON find (findvalue)')
if old <= '20150307-272-557f4d3':
run_sql('''CREATE TABLE scrape (
if not db.table_exists('scrape'):
run_sql('''CREATE TABLE scrape (
item_id VARCHAR(32) NOT NULL,
added DATETIME,
PRIMARY KEY (item_id),
FOREIGN KEY(item_id) REFERENCES item (id)
)''')
run_sql('CREATE INDEX idx_scrape_added ON scrape (added)')
run_sql('CREATE INDEX idx_scrape_added ON scrape (added)')
if old <= '20151118-346-7e86e68':
old_key = os.path.join(settings.config_path, 'node.ssl.key')
if os.path.exists(old_key):
@ -456,22 +443,22 @@ def upgrade_db(old, new=None):
session.execute(sql)
session.commit()
if old <= '20160111-603-90648f9' and not new:
run_sql('ALTER TABLE sort ADD categories VARCHAR(1000)')
run_sql('ALTER TABLE sort ADD series VARCHAR(1000)')
run_sql('CREATE INDEX ix_sort_categories ON sort (categories)')
run_sql('''CREATE TABLE user_metadata (
created DATETIME,
modified DATETIME,
id INTEGER NOT NULL,
item_id VARCHAR(32),
user_id VARCHAR(43),
data_hash VARCHAR(40),
data BLOB,
PRIMARY KEY (id),
FOREIGN KEY(user_id) REFERENCES user (id)
)''')
run_sql('CREATE UNIQUE INDEX IF NOT EXISTS user_metadata_index ON user_metadata(id, user_id)')
run_sql('CREATE INDEX ix_user_metadata_data_hash ON user_metadata (data_hash)')
if not db.table_exists('user_metadata'):
run_sql('''CREATE TABLE user_metadata (
created DATETIME,
modified DATETIME,
id INTEGER NOT NULL,
item_id VARCHAR(32),
user_id VARCHAR(43),
data_hash VARCHAR(40),
data BLOB,
PRIMARY KEY (id),
FOREIGN KEY(user_id) REFERENCES user (id)
)''')
run_sql('CREATE UNIQUE INDEX IF NOT EXISTS user_metadata_index ON user_metadata(id, user_id)')
run_sql('CREATE INDEX ix_user_metadata_data_hash ON user_metadata (data_hash)')
import item.models
item.models.update_sort_table()
def create_default_lists(user_id=None):
with db.session():