# -*- coding: utf-8 -*- from contextlib import closing import base64 import json import os import tarfile from threading import Thread import urllib.request import urllib.error import urllib.parse import shutil import subprocess import sys import time import OpenSSL.crypto import ox from oxtornado import actions from sqlalchemy.sql.expression import text import settings import utils import db from integration import get_trayicon_version import logging logger = logging.getLogger(__name__) ENCODING = 'base64' def verify(release): verified = False value = [] for module in sorted(release['modules']): value += [str('%s/%s' % (release['modules'][module]['version'], release['modules'][module]['sha1']))] value = '\n'.join(value) value = value.encode() for digest in ('sha512', 'sha256', 'sha1'): if 'signature_%s' % digest in release: tls_sig = base64.b64decode(release['signature_%s' % digest].encode()) cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, settings.OML_UPDATE_CERT) try: OpenSSL.crypto.verify(cert, tls_sig, value, digest) verified = True except OpenSSL.crypto.Error: logger.debug('invalid tls signature') verified = False break if 'signature' in release and not verified: import ed25519 vk = ed25519.VerifyingKey(settings.OML_UPDATE_KEY, encoding=ENCODING) sig = release['signature'].encode() try: vk.verify(sig, value, encoding=ENCODING) verified = True except ed25519.BadSignatureError: verified = False return verified def get(url, filename=None): request = urllib.request.Request(url, headers={ 'User-Agent': settings.USER_AGENT }) with closing(urllib.request.urlopen(request)) as u: if not filename: data = u.read() return data else: dirname = os.path.dirname(filename) if dirname and not os.path.exists(dirname): os.makedirs(dirname) with open(filename, 'wb') as fd: data = u.read(4096) while data: fd.write(data) data = u.read(4096) def check(): if settings.release: release_data = get(settings.server.get('release_url', 'https://downloads.openmedialibrary.com/release.json')) release = json.loads(release_data.decode('utf-8')) old = current_version('openmedialibrary') new = release['modules']['openmedialibrary']['version'] return verify(release) and old < new return False def current_version(module): if 'modules' in settings.release \ and module in settings.release['modules'] \ and 'version' in settings.release['modules'][module]: version = settings.release['modules'][module]['version'] else: version = '' return version def get_latest_release(): try: release_data = get(settings.server.get('release_url')) release = json.loads(release_data.decode('utf-8')) if verify(release): ox.makedirs(settings.updates_path) with open(os.path.join(settings.updates_path, 'release.json'), 'wb') as fd: fd.write(release_data) return release except: logger.debug('failed to get latest release') def get_platform(): name = sys.platform if name.startswith('darwin'): name = 'darwin64' elif name.startswith('linux'): import platform machine = platform.machine() if machine == 'armv7l': name = 'linux_armv7l' elif machine == 'aarch64': name = 'linux_aarch64' elif machine == 'x86_64': name = 'linux64' else: name = 'linux32' return name def download(): if not os.path.exists(os.path.join(settings.data_path, 'release.json')): return True release = get_latest_release() platform = get_platform() if release: ox.makedirs(settings.updates_path) os.chdir(os.path.dirname(settings.base_dir)) current_files = {'release.json'} for module in release['modules']: module_tar = os.path.join(settings.updates_path, release['modules'][module]['name']) if release['modules'][module].get('platform', platform) == platform and \ release['modules'][module]['version'] > current_version(module): base_url = settings.server.get('release_url').rsplit('/', 1)[0] url = '/'.join([base_url, release['modules'][module]['name']]) if not os.path.exists(module_tar): logger.debug('download %s', os.path.basename(module_tar)) get(url, module_tar) if ox.sha1sum(module_tar) != release['modules'][module]['sha1']: logger.debug('invalid checksum %s', os.path.basename(module_tar)) os.unlink(module_tar) return False current_files.add(os.path.basename(module_tar)) for f in set(next(os.walk(settings.updates_path))[2])-current_files: os.unlink(os.path.join(settings.updates_path, f)) return True return True def install(): if not os.path.exists(os.path.join(settings.updates_path, 'release.json')): return True if not os.path.exists(os.path.join(settings.data_path, 'release.json')): return True with open(os.path.join(settings.updates_path, 'release.json')) as fd: release = json.load(fd) old_version = current_version('openmedialibrary') new_version = release['modules']['openmedialibrary']['version'] if verify(release) and old_version < new_version: base = os.path.dirname(settings.base_dir) os.chdir(base) platform = get_platform() for module in release['modules']: if release['modules'][module].get('platform', platform) == platform and \ release['modules'][module]['version'] > current_version(module): module_tar = os.path.join(settings.updates_path, release['modules'][module]['name']) if os.path.exists(module_tar) and ox.sha1sum(module_tar) == release['modules'][module]['sha1']: #tar fails if old platform is moved before extract new = '%s_new' % module ox.makedirs(new) os.chdir(new) tar = tarfile.open(module_tar) tar.extractall() tar.close() os.chdir(base) module_old = '%s_old' % module if os.path.exists(module_old): rmtree(module_old) if os.path.exists(module): move(module, module_old) move(os.path.join(new, module), module) if platform != 'win32' and os.path.exists(module_old): rmtree(module_old) rmtree(new) else: if os.path.exists(module_tar): os.unlink(module_tar) return False shutil.copy(os.path.join(settings.updates_path, 'release.json'), os.path.join(settings.data_path, 'release.json')) upgrade_app() # FIXME: still needed? if old_version < '20160112-651-de984a3' and platform != 'win32': subprocess.call(['./ctl', 'postupdate', '-o', old_version, '-n', new_version]) return True return True def move(src, dst): try: shutil.move(src, dst) except: logger.debug('failed to move %s to %s', src, dst) raise def rmtree(path): try: shutil.rmtree(path) except: logger.debug('failed to remove %s', path) raise def update_available(): db_version = settings.server.get('db_version', 0) if db_version < settings.DB_VERSION: return True if not os.path.exists(os.path.join(settings.updates_path, 'release.json')): return False if not os.path.exists(os.path.join(settings.data_path, 'release.json')): return False with open(os.path.join(settings.updates_path, 'release.json')) as fd: release = json.load(fd) old_version = current_version('openmedialibrary') new_version = release['modules']['openmedialibrary']['version'] return verify(release) and old_version < new_version def restart_oml(update=False): if update: get_latest_release() utils.ctl('restart') def get_app_version(app): plist = app + '/Contents/Info.plist' if os.path.exists(plist): cmd = ['defaults', 'read', plist, 'CFBundleShortVersionString'] return subprocess.check_output(cmd).strip() def upgrade_app(): base = os.path.dirname(settings.base_dir) if sys.platform == 'darwin': bundled_app = os.path.join(base, 'platform_darwin64/Applications/Open Media Library.app') app = '/Applications/Open Media Library.app' version = get_app_version(app) current_version = get_app_version(bundled_app) if version and current_version and version != current_version: try: shutil.rmtree(app) shutil.copytree(bundled_app, app) except: logger.debug('Failed to update Application', exc_info=True) ''' elif sys.platform == 'win32': current_version = get_trayicon_version() if current_version != '0.2.0.0': msi = os.path.normpath(os.path.join(base, 'platform_win32', 'Open Media Library.msi')) cmd = ['msiexec.exe', '/qb', '/I', msi] startupinfo = subprocess.STARTUPINFO() startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW startupinfo.wShowWindow = subprocess.SW_HIDE subprocess.Popen(cmd, cwd=settings.base_dir, start_new_session=True, startupinfo=startupinfo) ''' def getVersion(data): ''' check if new version is available ''' response = { 'current': settings.MINOR_VERSION, 'version': settings.MINOR_VERSION, 'upgrade': False, } if settings.MINOR_VERSION == 'git': ''' cmd = ['git', 'rev-parse', '@'] p = subprocess.Popen(cmd, stdout=subprocess.PIPE, close_fds=True) stdout, stderr = p.communicate() current = stdout.strip() cmd = ['git', 'ls-remote', 'origin', '-h', 'refs/heads/master'] p = subprocess.Popen(cmd, stdout=subprocess.PIPE, close_fds=True) stdout, stderr = p.communicate() new = stdout.strip()[:40] response['update'] = len(new) == 40 and current != new ''' response['update'] = False else: get_latest_release() if not os.path.exists(os.path.join(settings.updates_path, 'release.json')): return response if not os.path.exists(os.path.join(settings.data_path, 'release.json')): return response with open(os.path.join(settings.updates_path, 'release.json')) as fd: release = json.load(fd) current = current_version('openmedialibrary') response['current'] = current new = release['modules']['openmedialibrary']['version'] response['version'] = new response['update'] = current < new return response actions.register(getVersion, cache=False) actions.register(getVersion, cache=False, version='public') def restart(data): ''' restart (and upgrade if upgrades are available) ''' restart_oml(data.get('update')) return {} actions.register(restart, cache=False) class Update(Thread): _status = { 'reload': False, 'status': 'Updating Open Media Library...' } def __init__(self): Thread.__init__(self) self.daemon = True self.start() def status(self, status, reload=False): from websocket import trigger_event self._status = { 'reload': reload, 'status': status, } trigger_event('updatestatus', self._status) def install(self): while update_available(): self.status('Downloading...') max_retry = 5 while max_retry > 0 and not download(): max_retry -= 1 self.status('Download failed, retrying...') time.sleep(5) self.status('Downloading...') self.status('Installing...') if not install(): self.status('Installation failed.') return True return False def update_database(self): db_version = settings.server.get('db_version', 0) if db_version < settings.DB_VERSION: self.status('Updating...') if db_version < 3: db_version = migrate_3() if db_version < 4: db_version = migrate_4() if db_version < 5: db_version = migrate_5() if db_version < 6: db_version = migrate_6() if db_version < 7: db_version = migrate_7() if db_version < 9: db_version = migrate_8() if db_version < 10: db_version = migrate_10() if db_version < 11: db_version = migrate_11() if db_version < 12: db_version = migrate_12() if db_version < 13: db_version = migrate_13() if db_version < 15: db_version = migrate_15() if db_version < 16: db_version = migrate_16() if db_version < 17: db_version = migrate_17() if db_version < 18: db_version = migrate_18() settings.server['db_version'] = db_version def run(self): self.status('Checking for updates...') self.update_database() if self.install(): restart_oml() return self.status('Relaunching...', True) restart_oml() def migrate_3(): with db.session(): import item.models for i in item.models.Item.find({ 'query': { 'conditions': [{ 'key':'mediastate', 'value':'available', 'operator': '==' }] } }): if not i.files.all(): i.remove_file() else: f = i.files.all()[0] if not 'pages' in i.info and 'pages' in f.info: i.info['pages'] = f.info['pages'] i.save() return 3 def migrate_4(): with db.session() as session: import item.models from meta.utils import to_isbn13 for i in item.models.Item.query: update = False if 'isbn' in i.meta and isinstance(i.meta['isbn'], list): isbns = [to_isbn13(isbn) for isbn in i.meta['isbn']] isbns = [isbn for isbn in isbns if isbn] if isbns: i.meta['isbn'] = isbns[0] if 'isbn' in i.info: i.info['isbn'] = i.meta['isbn'] else: del i.meta['isbn'] if 'isbn' in i.info: del i.info['isbn'] update = True if 'isbn' in i.meta and not i.meta['isbn']: del i.meta['isbn'] update = True if update: session.add(i) session.commit() return 4 def migrate_5(): db.run_sql([ 'DROP INDEX IF EXISTS user_metadata_index', 'CREATE UNIQUE INDEX user_metadata_index ON user_metadata(item_id, user_id)', 'UPDATE sort SET sharemetadata = 0', ]) with db.session() as session: import user.models for m in user.models.Metadata.query: data_hash = m.get_hash() if m.data_hash != data_hash: m.data_hash = data_hash session.add(m) session.commit() import item.models for i in item.models.Item.query: update = False if '_from' in i.info: del i.info['_from'] update = True if update: session.add(i) session.commit() for i in item.models.Item.query: i.sync_metadata() return 5 def migrate_6(): with db.session() as session: import item.models for s in item.models.Sort.query.filter_by(author=''): s.item.update_sort() for s in item.models.Sort.query.filter_by(publisher=''): s.item.update_sort() for s in item.models.Sort.query.filter_by(language=''): s.item.update_sort() for s in item.models.Sort.query.filter_by(place=''): s.item.update_sort() for s in item.models.Sort.query.filter_by(isbn=''): s.item.update_sort() for s in item.models.Sort.query.filter_by(date=''): s.item.update_sort() session.commit() return 6 def migrate_7(): with db.session() as session: import changelog for c in changelog.Changelog.query: if 'editmeta' in c.data or 'resetmeta' in c.data: session.delete(c) session.commit() db.run_sql('DROP TABLE IF EXISTS metadata') db.run_sql('DROP TABLE IF EXISTS scrape') db.run_sql('VACUUM') return 7 def migrate_8(): for key in ('directory_service', 'meta_service', 'local_lookup', 'cert'): if key in settings.server: del settings.server[key] list_cache = os.path.join(settings.data_path, 'list_cache.json') if os.path.exists(list_cache): os.unlink(list_cache) with db.session() as session: import item.models for i in item.models.Item.query: delta = set(i.meta)-set(i.meta_keys) if delta: for key in delta: del i.meta[key] session.add(i) session.commit() import changelog import user.models changelog.Changelog.query.delete() u = user.models.User.get(settings.USER_ID) u.rebuild_changelog() for peer in user.models.User.query: if peer.id != u.id: if len(peer.id) != 16: session.delete(peer) session.commit() return 8 def migrate_10(): with db.session() as session: from item.models import Item, Find from utils import get_by_id from item.person import get_sort_name import unicodedata sort_names = {} updates = {} for f in Find.query.filter(Find.key.in_(Item.filter_keys)): sort_type = get_by_id(settings.config['itemKeys'], f.key).get('sortType') if sort_type == 'person': if f.value in sort_names: sortvalue = sort_names[f.value] else: sortvalue = sort_names[f.value] = get_sort_name(f.value) else: sortvalue = f.value if sortvalue: sortvalue = ox.sort_string(unicodedata.normalize('NFKD', sortvalue)).lower() if not f.key in updates: updates[f.key] = {} updates[f.key][f.value] = sortvalue for key in updates: for value in updates[key]: Find.query.filter_by(key=key, value=value).update({'sortvalue': updates[key][value]}) session.commit() return 10 def migrate_11(): with db.session() as session: from user.models import User, Metadata, List from changelog import Changelog import utils for u in User.query.filter_by(peered=True): peer = utils.get_peer(u.id) last = Changelog.query.filter_by(user_id=u.id).order_by(text('-revision')).first() if last: peer.info['revision'] = last.revision listorder = [] for l in List.query.filter_by(user_id=u.id).order_by('index_'): if l.name: peer.info['lists'][l.name] = [i.id for i in l.get_items()] listorder.append(l.name) if 'listorder' not in peer.info: peer.info['listorder'] = listorder for m in Metadata.query.filter_by(user_id=u.id): peer.library[m.item_id] = { 'meta': dict(m.data), 'meta_hash': m.data_hash, 'modified': m.modified, } peer.library.commit() peer.sync_info() peer.sync_db() Changelog.query.filter_by(user_id=u.id).delete() Metadata.query.filter_by(user_id=u.id).delete() session.commit() if db.table_exists('transfer'): import state import downloads state.online = False state.downloads = downloads.Downloads() r = state.db.session.execute('SELECT item_id, added, progress FROM transfer') for t in r: item_id, added, progress = t if added: state.downloads.transfers[item_id] = { 'added': added, 'progress': progress } state.db.session.commit() state.downloads.transfers.commit() state.downloads = None return 11 def migrate_12(): db.run_sql([ 'DROP TABLE IF EXISTS transfer' ]) return 12 def migrate_13(): import settings import changelog import os import json path = os.path.join(settings.data_path, 'peers', '%s.log' % settings.USER_ID) if not os.path.exists(path): folder = os.path.dirname(path) if not os.path.exists(folder): os.makedirs(folder) with db.session() as session: revision = -1 qs = changelog.Changelog.query.filter_by(user_id=settings.USER_ID) with open(path, 'w') as fd: for c in qs.order_by('timestamp'): data = json.dumps([c.revision, c.timestamp, json.loads(c.data)], ensure_ascii=False) fd.write(data + '\n') revision = c.revision if revision > -1: settings.server['revision'] = revision return 13 def migrate_15(): from user.models import List, User with db.session(): l = List.get(':Public') if l and not len(l.items): l.remove() for u in User.query: if 'local' in u.info: del u.info['local'] u.save() return 15 def migrate_16(): db.run_sql([ '''CREATE TABLE user2 ( created DATETIME, modified DATETIME, id VARCHAR(43) NOT NULL, info BLOB, nickname VARCHAR(256), pending VARCHAR(64), queued BOOLEAN, peered BOOLEAN, online BOOLEAN, PRIMARY KEY (id), CHECK (queued IN (0, 1)), CHECK (peered IN (0, 1)), CHECK (online IN (0, 1)) )''', '''INSERT INTO user2 (created, modified, id, info, nickname, pending, queued, peered, online) SELECT created, modified, id, info, nickname, pending, queued, peered, online FROM user''', 'DROP TABLE user', 'ALTER TABLE user2 RENAME TO user', 'CREATE INDEX IF NOT EXISTS ix_user_nickname ON user (nickname)' ]) return 16 def migrate_17(): from user.models import List, User from changelog import add_record with db.session(): l = List.get(':Public') if not l: add_record('removelist', 'Public') lists = [] for l in List.query.filter_by(user_id=settings.USER_ID).order_by('index_'): if l.type == 'static' and l.name not in ('', 'Inbox'): lists.append(l.name) add_record('orderlists', lists) return 17 def migrate_18(): db.run_sql([ '''CREATE TABLE annotation ( _id INTEGER NOT NULL, id VARCHAR(43), created DATETIME, modified DATETIME, user_id VARCHAR(43), item_id VARCHAR(43), data BLOB, findquotes TEXT, findnotes TEXT, PRIMARY KEY (_id), FOREIGN KEY(user_id) REFERENCES user (id), FOREIGN KEY(item_id) REFERENCES item (id) )''']) db.run_sql([ 'CREATE INDEX ix_annotation_findquotes ON annotation (findquotes)', 'CREATE INDEX ix_annotation_findnotes ON annotation (findnotes)' ]) return 18