openmedialibrary/oml/setup.py

543 lines
20 KiB
Python

# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
import os
import settings
import db
from db import run_sql
from user.models import List, User
def create_db():
if not os.path.exists(settings.db_path):
sql = '''
CREATE TABLE item (
created DATETIME,
modified DATETIME,
id VARCHAR(32) NOT NULL,
info BLOB,
meta BLOB,
added DATETIME,
accessed DATETIME,
timesaccessed INTEGER,
PRIMARY KEY (id)
);
CREATE TABLE changelog (
id INTEGER NOT NULL,
created DATETIME,
timestamp BIGINT,
user_id VARCHAR(43),
revision BIGINT,
data TEXT,
sig VARCHAR(96),
PRIMARY KEY (id)
);
CREATE TABLE user (
created DATETIME,
modified DATETIME,
id VARCHAR(43) NOT NULL,
info BLOB,
nickname VARCHAR(256),
pending VARCHAR(64),
queued BOOLEAN,
peered BOOLEAN,
online BOOLEAN,
PRIMARY KEY (id),
UNIQUE (nickname),
CHECK (queued IN (0, 1)),
CHECK (peered IN (0, 1)),
CHECK (online IN (0, 1))
);
CREATE TABLE metadata (
created DATETIME,
modified DATETIME,
id INTEGER NOT NULL,
"key" VARCHAR(256),
value VARCHAR(256),
data BLOB,
PRIMARY KEY (id)
);
CREATE TABLE person (
name VARCHAR(1024) NOT NULL,
sortname VARCHAR,
numberofnames INTEGER,
PRIMARY KEY (name)
);
CREATE TABLE transfer (
item_id VARCHAR(32) NOT NULL,
added DATETIME,
progress FLOAT,
PRIMARY KEY (item_id),
FOREIGN KEY(item_id) REFERENCES item (id)
);
CREATE TABLE find (
id INTEGER NOT NULL,
item_id VARCHAR(32),
"key" VARCHAR(200),
value TEXT,
findvalue TEXT,
PRIMARY KEY (id),
FOREIGN KEY(item_id) REFERENCES item (id)
);
CREATE INDEX ix_find_key ON find ("key");
CREATE TABLE list (
id INTEGER NOT NULL,
name VARCHAR,
index_ INTEGER,
type VARCHAR(64),
"query" BLOB,
user_id VARCHAR(43),
PRIMARY KEY (id),
FOREIGN KEY(user_id) REFERENCES user (id)
);
CREATE TABLE useritem (
user_id VARCHAR(43),
item_id VARCHAR(32),
FOREIGN KEY(item_id) REFERENCES item (id),
FOREIGN KEY(user_id) REFERENCES user (id)
);
CREATE TABLE sort (
item_id VARCHAR(32) NOT NULL,
title VARCHAR(1000),
author VARCHAR(1000),
publisher VARCHAR(1000),
place VARCHAR(1000),
date VARCHAR(1000),
language VARCHAR(1000),
pages BIGINT,
extension VARCHAR(1000),
size BIGINT,
created DATETIME,
added DATETIME,
modified DATETIME,
accessed DATETIME,
timesaccessed BIGINT,
mediastate VARCHAR(1000),
transferadded DATETIME,
transferprogress FLOAT,
id VARCHAR(1000),
isbn VARCHAR(1000),
random BIGINT,
PRIMARY KEY (item_id),
FOREIGN KEY(item_id) REFERENCES item (id)
);
CREATE INDEX ix_sort_accessed ON sort (accessed);
CREATE INDEX ix_sort_added ON sort (added);
CREATE INDEX ix_sort_author ON sort (author);
CREATE INDEX ix_sort_created ON sort (created);
CREATE INDEX ix_sort_date ON sort (date);
CREATE INDEX ix_sort_extension ON sort (extension);
CREATE INDEX ix_sort_id ON sort (id);
CREATE INDEX ix_sort_isbn ON sort (isbn);
CREATE INDEX ix_sort_language ON sort (language);
CREATE INDEX ix_sort_mediastate ON sort (mediastate);
CREATE INDEX ix_sort_modified ON sort (modified);
CREATE INDEX ix_sort_pages ON sort (pages);
CREATE INDEX ix_sort_place ON sort (place);
CREATE INDEX ix_sort_publisher ON sort (publisher);
CREATE INDEX ix_sort_random ON sort (random);
CREATE INDEX ix_sort_size ON sort (size);
CREATE INDEX ix_sort_timesaccessed ON sort (timesaccessed);
CREATE INDEX ix_sort_title ON sort (title);
CREATE INDEX ix_sort_transferadded ON sort (transferadded);
CREATE INDEX ix_sort_transferprogress ON sort (transferprogress);
CREATE TABLE file (
created DATETIME,
modified DATETIME,
sha1 VARCHAR(32) NOT NULL,
path VARCHAR(2048),
info BLOB,
item_id VARCHAR(32),
PRIMARY KEY (sha1),
FOREIGN KEY(item_id) REFERENCES item (id)
);
CREATE TABLE listitem (
list_id INTEGER,
item_id VARCHAR(32),
FOREIGN KEY(item_id) REFERENCES item (id),
FOREIGN KEY(list_id) REFERENCES list (id)
);
PRAGMA journal_mode=WAL
'''
for statement in sql.split(';'):
run_sql(statement)
create_default_lists()
settings.server['db_version'] = settings.DB_VERSION
update_database()
def upgrade_db(old, new=None):
def run_after(v):
return old <= v and new > v
if new:
if old <= '20140525-92-eac91e7' and new > '20140525-92-eac91e7':
with db.session():
import user.models
for u in user.models.User.query:
u.update_name()
u.save()
import item.models
for f in item.models.File.query:
changed = False
for key in ('mediastate', 'coverRatio', 'previewRatio'):
if key in f.info:
del f.info[key]
changed = True
if changed:
f.save()
if old <= '20140526-118-d451eb3' and new > '20140526-118-d451eb3':
with db.session():
import item.models
item.models.Find.query.filter_by(key='list').delete()
if old <= '20160109-573-094097b' and new > '20160109-573-094097b':
import sys
if sys.platform.startswith('linux'):
import integration
integration.install_xdg()
if old <= '20160110-581-e08780a' and new > '20160110-581-e08780a':
with db.session():
import item.models
from item.icons import icons
for f in item.models.File.query:
if f.info['extension'] == 'epub':
i = f.item
key = 'cover:%s'%i.id
cover = icons[key]
key = 'preview:%s'%i.id
preview = i.extract_preview()
update_item = False
if not preview:
del icons[key]
for resolution in (128, 256, 512):
del icons['%s:%s' % (key, resolution)]
if 'previewRatio' in i.info:
del i.info['previewRatio']
update_item = True
if not cover and 'coverRatio' in i.info:
del i.info['coverRatio']
update_item = True
if update_item:
i.save()
if old <= '20160110-583-59a3709' and new > '20160110-583-59a3709':
with db.session() as session:
import ox
from item.models import Item, Find, Metadata
from meta.utils import decode_html_data
def cleanup_description(data):
if 'description' in data:
description = data['description']
description = description.replace('<br>', '\n').replace('</p><p>', '\n\n')
data['description'] = ox.strip_tags(description).strip()
for m in Metadata.query:
data = decode_html_data(m.data.copy())
cleanup_description(data)
if data != m.data:
#print(m, m.data, '\n',data, '\n\n')
m.data = data
m.save()
for i in Item.query:
changed = False
meta = decode_html_data(i.meta.copy())
cleanup_description(meta)
if meta != i.meta:
#print(i, i.meta, '\n', meta, '\n\n')
i.meta = meta
changed = True
info = decode_html_data(i.info.copy())
cleanup_description(info)
if info != i.info:
#print(i, i.info, '\n', info, '\n\n')
i.info = info
changed = True
if changed:
i.update_sort()
i.update_find()
i.save()
for tag in ('&apos;', '&quot;', '&#'):
items = set([f.item_id for f in Find.query.filter(Find.value.ilike('%'+tag+'%'))])
if items:
for i in Item.query.filter(Item.id.in_(items)):
#print(tag, i)
i.update_sort()
i.update_find()
session.commit()
if old <= '20160111-603-90648f9' and new > '20160111-603-90648f9':
for f in settings.ui['filters']:
if f['id'] == 'classification':
f['id'] = 'categories'
settings.ui._save()
if not db.table_exists('user_metadata'):
run_sql('''CREATE TABLE user_metadata (
created DATETIME,
modified DATETIME,
id INTEGER NOT NULL,
item_id VARCHAR(32),
user_id VARCHAR(43),
data_hash VARCHAR(40),
data BLOB,
PRIMARY KEY (id),
FOREIGN KEY(user_id) REFERENCES user (id)
)''')
run_sql('CREATE UNIQUE INDEX IF NOT EXISTS user_metadata_index ON user_metadata(id, user_id)')
run_sql('CREATE INDEX ix_user_metadata_data_hash ON user_metadata (data_hash)')
from meta.utils import to_isbn13
from item.models import Item
from user.models import Metadata
with db.session() as session:
for i in Item.query:
update = False
if 'primaryid' in i.meta:
del i.meta['primaryid']
update = True
if 'primaryid' in i.info:
del i.info['primaryid']
update = True
for key in i.meta_keys:
if key not in i.meta and key in i.info:
i.meta[key] = i.info[key]
update = True
if 'isbn' in i.meta and isinstance(i.meta['isbn'], list):
isbns = [to_isbn13(isbn) for isbn in i.meta['isbn']]
isbns = [isbn for isbn in isbns if isbn]
if isbns:
i.meta['isbn'] = isbns[0]
if 'isbn' in i.info:
i.info['isbn'] = i.meta['isbn']
else:
del i.meta['isbn']
if 'isbn' in i.info:
del i.info['isbn']
update = True
if 'isbn' in i.meta and not i.meta['isbn']:
del i.meta['isbn']
update = True
if update:
session.add(i)
for u in i.users:
if u.id != settings.USER_ID:
Metadata.get_or_create(u.id, i.id, i.meta, False)
session.commit()
if run_after('20160111-617-206e39c'):
from item.models import File
import media
with db.session() as session:
for f in File.query:
if f.info.get('extension') == 'epub':
if not 'tableofcontents' in f.item.meta:
f.info = media.metadata(f.fullpath())
if 'tableofcontents' in f.info:
f.item.meta['tableofcontents'] = f.info['tableofcontents']
f.item.update()
session.add(f.item)
session.add(f)
session.commit()
if run_after('20160112-651-de984a3'):
from item.models import File
import media
with db.session() as session:
for f in File.query:
if f.info.get('extension') == 'pdf':
if not 'tableofcontents' in f.item.meta:
f.info = media.metadata(f.fullpath())
if 'tableofcontents' in f.info:
f.item.meta['tableofcontents'] = f.info['tableofcontents']
f.item.update()
session.add(f.item)
session.add(f)
session.commit()
if old <= '20140527-120-3cb9819':
run_sql('CREATE INDEX ix_find_findvalue ON find (findvalue)')
if old <= '20150307-272-557f4d3':
if not db.table_exists('scrape'):
run_sql('''CREATE TABLE scrape (
item_id VARCHAR(32) NOT NULL,
added DATETIME,
PRIMARY KEY (item_id),
FOREIGN KEY(item_id) REFERENCES item (id)
)''')
run_sql('CREATE INDEX idx_scrape_added ON scrape (added)')
if old <= '20151118-346-7e86e68':
old_key = os.path.join(settings.config_path, 'node.ssl.key')
if os.path.exists(old_key):
os.unlink(old_key)
if settings.OLD_USER_ID:
statements = [
"UPDATE user SET id = '{nid}' WHERE id = '{oid}'",
"UPDATE list SET user_id = '{nid}' WHERE user_id = '{oid}'",
"UPDATE useritem SET user_id = '{nid}' WHERE user_id = '{oid}'",
"UPDATE changelog SET user_id = '{nid}' WHERE user_id = '{oid}'",
]
for sql in statements:
run_sql(sql.format(oid=settings.OLD_USER_ID, nid=settings.USER_ID))
if old <= '20151201-384-03c2439':
with db.session():
import item.models
for i in item.models.Item.query:
for f in i.files.all():
f.move()
if old <= '20160103-423-05ca6c9':
with db.session():
import item.models
for i in item.models.Item.query:
if 'id' in i.meta:
del i.meta['id']
i.save()
for m in item.models.Metadata.query:
if 'id' in m.data:
del m.data['id']
m.save()
if old <= '20160106-495-d1b9e96':
run_sql('CREATE INDEX ix_useritem_user ON useritem ("user_id")')
if old <= '20160106-497-c86ba8a':
with db.session() as session:
u = User.get(settings.USER_ID)
if u:
l = u.library
for i in u.items.all():
if not i in l.items:
l.items.append(i)
session.add(l)
for u in User.query.filter_by(peered=True):
l = u.library
for i in u.items.all():
if not i in l.items:
l.items.append(i)
session.add(l)
l.items_count()
session.commit()
if old <= '20160107-508-a0c1970':
with db.session() as session:
for l in List.query.filter_by(name=' [2]'):
if not len(l.items):
l.remove()
if old <= '20160107-509-e0857fc':
add_useritem_index()
add_listitem_index()
if old <= '20160111-603-90648f9' and not new:
if not db.table_exists('user_metadata'):
run_sql('''CREATE TABLE user_metadata (
created DATETIME,
modified DATETIME,
id INTEGER NOT NULL,
item_id VARCHAR(32),
user_id VARCHAR(43),
data_hash VARCHAR(40),
data BLOB,
PRIMARY KEY (id),
FOREIGN KEY(user_id) REFERENCES user (id)
)''')
run_sql('CREATE UNIQUE INDEX IF NOT EXISTS user_metadata_index ON user_metadata(id, user_id)')
run_sql('CREATE INDEX ix_user_metadata_data_hash ON user_metadata (data_hash)')
def create_default_lists(user_id=None):
with db.session():
user_id = user_id or settings.USER_ID
user = User.get_or_create(user_id)
user.update_name()
for list in settings.config['lists']:
l = List.get(user_id, list['title'])
if not l:
l = List.create(user_id, list['title'], list.get('query'))
def update_database():
import sqlalchemy as sa
from sqlalchemy.schema import CreateTable
import changelog
import item.models
import item.person
import user.models
tables = [
changelog.Changelog,
item.models.File,
item.models.Find,
item.models.Item,
item.models.Metadata,
item.models.Scrape,
item.models.Sort,
item.models.Transfer,
item.models.user_items,
item.person.Person,
user.models.List,
user.models.List,
user.models.list_items,
user.models.Metadata,
user.models.User,
]
indexes = [
'CREATE INDEX idx_scrape_added ON scrape (added)',
'CREATE INDEX ix_find_findvalue ON find (findvalue)',
'CREATE INDEX ix_find_key ON find ("key")',
'CREATE INDEX ix_useritem_user ON useritem ("user_id")',
'CREATE INDEX ix_user_metadata_data_hash ON user_metadata (data_hash)',
'CREATE UNIQUE INDEX listitem_index on listitem(list_id, item_id)',
'CREATE UNIQUE INDEX useritem_index on useritem(user_id, item_id)',
'CREATE UNIQUE INDEX user_metadata_index ON user_metadata(id, user_id)',
]
layout = db.get_layout()
sql = []
for t in tables:
if isinstance(t, sa.Table):
table = t
else:
table = t.__table__
name = table.name
if name not in layout['tables']:
create_table = str(CreateTable(table).compile(db.engine))
sql.append(create_table)
index_names = set()
for index in indexes:
name = index.split('INDEX ')[1].split()[0]
if name not in layout['indexes']:
if name == 'useritem_index':
add_useritem_index()
elif name == 'listitem_index':
add_listitem_index()
else:
sql.append(index)
index_names.add(name)
for index in set(name for name in layout['indexes'] if not name.startswith('ix_sort_')) - index_names:
sql.append('DROP INDEX ' + index)
if sql:
with db.session() as s:
for q in sql:
s.connection().execute(q)
s.commit()
import item.models
item.models.update_sort_table()
def add_listitem_index():
with db.session() as session:
sql = "SELECT COUNT(*) AS c,list_id,item_id FROM listitem GROUP BY list_id, item_id HAVING c>1"
doubles = [r for r in session.execute(sql)]
for r in doubles:
params = {'list_id': r[1], 'item_id': r[2]}
sql = "DELETE FROM listitem WHERE list_id = :list_id AND item_id = :item_id"
session.execute(sql, params)
sql = "INSERT INTO listitem (list_id, item_id) VALUES (:list_id, :item_id)"
session.execute(sql, params)
session.commit()
sql = 'CREATE UNIQUE INDEX IF NOT EXISTS listitem_index on listitem(list_id,item_id)'
session.execute(sql)
session.commit()
def add_useritem_index():
with db.session() as session:
sql = "SELECT COUNT(*) AS c,user_id,item_id FROM useritem GROUP BY user_id, item_id HAVING c>1"
doubles = [r for r in session.execute(sql)]
for r in doubles:
params = {'user_id': r[1], 'item_id': r[2]}
sql = "DELETE FROM useritem WHERE user_id = :user_id AND item_id = :item_id"
session.execute(sql, params)
sql = "INSERT INTO useritem (user_id, item_id) VALUES (:user_id, :item_id)"
session.execute(sql, params)
session.commit()
sql = 'CREATE UNIQUE INDEX IF NOT EXISTS useritem_index on useritem(user_id,item_id)'
session.execute(sql)
session.commit()