openmedialibrary/oml/update.py

700 lines
24 KiB
Python

# -*- coding: utf-8 -*-
from contextlib import closing
import base64
import json
import os
import tarfile
from threading import Thread
import urllib.request
import urllib.error
import urllib.parse
import shutil
import subprocess
import sys
import time
import OpenSSL.crypto
import ox
from oxtornado import actions
from sqlalchemy.sql.expression import text
import settings
import utils
import db
from integration import get_trayicon_version
import logging
logger = logging.getLogger(__name__)
ENCODING = 'base64'
def verify(release):
verified = False
value = []
for module in sorted(release['modules']):
value += [str('%s/%s' % (release['modules'][module]['version'], release['modules'][module]['sha1']))]
value = '\n'.join(value)
value = value.encode()
for digest in ('sha512', 'sha256', 'sha1'):
if 'signature_%s' % digest in release:
tls_sig = base64.b64decode(release['signature_%s' % digest].encode())
cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, settings.OML_UPDATE_CERT)
try:
OpenSSL.crypto.verify(cert, tls_sig, value, digest)
verified = True
except OpenSSL.crypto.Error:
logger.debug('invalid tls signature')
verified = False
break
if 'signature' in release and not verified:
import ed25519
vk = ed25519.VerifyingKey(settings.OML_UPDATE_KEY, encoding=ENCODING)
sig = release['signature'].encode()
try:
vk.verify(sig, value, encoding=ENCODING)
verified = True
except ed25519.BadSignatureError:
verified = False
return verified
def get(url, filename=None):
request = urllib.request.Request(url, headers={
'User-Agent': settings.USER_AGENT
})
with closing(urllib.request.urlopen(request)) as u:
if not filename:
data = u.read()
return data
else:
dirname = os.path.dirname(filename)
if dirname and not os.path.exists(dirname):
os.makedirs(dirname)
with open(filename, 'wb') as fd:
data = u.read(4096)
while data:
fd.write(data)
data = u.read(4096)
def check():
if settings.release:
release_data = get(settings.server.get('release_url',
'https://downloads.openmedialibrary.com/release.json'))
release = json.loads(release_data.decode('utf-8'))
old = current_version('openmedialibrary')
new = release['modules']['openmedialibrary']['version']
return verify(release) and old < new
return False
def current_version(module):
if 'modules' in settings.release \
and module in settings.release['modules'] \
and 'version' in settings.release['modules'][module]:
version = settings.release['modules'][module]['version']
else:
version = ''
return version
def get_latest_release():
try:
release_data = get(settings.server.get('release_url'))
release = json.loads(release_data.decode('utf-8'))
if verify(release):
ox.makedirs(settings.updates_path)
with open(os.path.join(settings.updates_path, 'release.json'), 'wb') as fd:
fd.write(release_data)
return release
except:
logger.debug('failed to get latest release')
def get_platform():
name = sys.platform
if name.startswith('darwin'):
name = 'darwin64'
elif name.startswith('linux'):
import platform
machine = platform.machine()
if machine == 'armv7l':
name = 'linux_armv7l'
elif machine == 'aarch64':
name = 'linux_aarch64'
elif machine == 'x86_64':
name = 'linux64'
else:
name = 'linux32'
return name
def download():
if not os.path.exists(os.path.join(settings.data_path, 'release.json')):
return True
release = get_latest_release()
platform = get_platform()
if release:
ox.makedirs(settings.updates_path)
os.chdir(os.path.dirname(settings.base_dir))
current_files = {'release.json'}
for module in release['modules']:
module_tar = os.path.join(settings.updates_path, release['modules'][module]['name'])
if release['modules'][module].get('platform', platform) == platform and \
release['modules'][module]['version'] > current_version(module):
base_url = settings.server.get('release_url').rsplit('/', 1)[0]
url = '/'.join([base_url, release['modules'][module]['name']])
if not os.path.exists(module_tar):
logger.debug('download %s', os.path.basename(module_tar))
get(url, module_tar)
if ox.sha1sum(module_tar) != release['modules'][module]['sha1']:
logger.debug('invalid checksum %s', os.path.basename(module_tar))
os.unlink(module_tar)
return False
current_files.add(os.path.basename(module_tar))
for f in set(next(os.walk(settings.updates_path))[2])-current_files:
os.unlink(os.path.join(settings.updates_path, f))
return True
return True
def install():
if not os.path.exists(os.path.join(settings.updates_path, 'release.json')):
return True
if not os.path.exists(os.path.join(settings.data_path, 'release.json')):
return True
with open(os.path.join(settings.updates_path, 'release.json')) as fd:
release = json.load(fd)
old_version = current_version('openmedialibrary')
new_version = release['modules']['openmedialibrary']['version']
if verify(release) and old_version < new_version:
base = os.path.dirname(settings.base_dir)
os.chdir(base)
platform = get_platform()
for module in release['modules']:
if release['modules'][module].get('platform', platform) == platform and \
release['modules'][module]['version'] > current_version(module):
module_tar = os.path.join(settings.updates_path, release['modules'][module]['name'])
if os.path.exists(module_tar) and ox.sha1sum(module_tar) == release['modules'][module]['sha1']:
#tar fails if old platform is moved before extract
new = '%s_new' % module
ox.makedirs(new)
os.chdir(new)
tar = tarfile.open(module_tar)
tar.extractall()
tar.close()
os.chdir(base)
module_old = '%s_old' % module
if os.path.exists(module_old):
rmtree(module_old)
if os.path.exists(module):
move(module, module_old)
move(os.path.join(new, module), module)
if platform != 'win32' and os.path.exists(module_old):
rmtree(module_old)
rmtree(new)
else:
if os.path.exists(module_tar):
os.unlink(module_tar)
return False
shutil.copy(os.path.join(settings.updates_path, 'release.json'), os.path.join(settings.data_path, 'release.json'))
upgrade_app()
# FIXME: still needed?
if old_version < '20160112-651-de984a3' and platform != 'win32':
subprocess.call(['./ctl', 'postupdate', '-o', old_version, '-n', new_version])
return True
return True
def move(src, dst):
try:
shutil.move(src, dst)
except:
logger.debug('failed to move %s to %s', src, dst)
raise
def rmtree(path):
try:
shutil.rmtree(path)
except:
logger.debug('failed to remove %s', path)
raise
def update_available():
db_version = settings.server.get('db_version', 0)
if db_version < settings.DB_VERSION:
return True
if not os.path.exists(os.path.join(settings.updates_path, 'release.json')):
return False
if not os.path.exists(os.path.join(settings.data_path, 'release.json')):
return False
with open(os.path.join(settings.updates_path, 'release.json')) as fd:
release = json.load(fd)
old_version = current_version('openmedialibrary')
new_version = release['modules']['openmedialibrary']['version']
return verify(release) and old_version < new_version
def restart_oml(update=False):
if update:
get_latest_release()
utils.ctl('restart')
def get_app_version(app):
plist = app + '/Contents/Info.plist'
if os.path.exists(plist):
cmd = ['defaults', 'read', plist, 'CFBundleShortVersionString']
return subprocess.check_output(cmd).strip()
def upgrade_app():
base = os.path.dirname(settings.base_dir)
if sys.platform == 'darwin':
bundled_app = os.path.join(base, 'platform_darwin64/Applications/Open Media Library.app')
app = '/Applications/Open Media Library.app'
version = get_app_version(app)
current_version = get_app_version(bundled_app)
if version and current_version and version != current_version:
try:
shutil.rmtree(app)
shutil.copytree(bundled_app, app)
except:
logger.debug('Failed to update Application', exc_info=True)
'''
elif sys.platform == 'win32':
current_version = get_trayicon_version()
if current_version != '0.2.0.0':
msi = os.path.normpath(os.path.join(base, 'platform_win32', 'Open Media Library.msi'))
cmd = ['msiexec.exe', '/qb', '/I', msi]
startupinfo = subprocess.STARTUPINFO()
startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
startupinfo.wShowWindow = subprocess.SW_HIDE
subprocess.Popen(cmd, cwd=settings.base_dir, start_new_session=True, startupinfo=startupinfo)
'''
def getVersion(data):
'''
check if new version is available
'''
response = {
'current': settings.MINOR_VERSION,
'version': settings.MINOR_VERSION,
'upgrade': False,
}
if settings.MINOR_VERSION == 'git':
'''
cmd = ['git', 'rev-parse', '@']
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, close_fds=True)
stdout, stderr = p.communicate()
current = stdout.strip()
cmd = ['git', 'ls-remote', 'origin', '-h', 'refs/heads/master']
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, close_fds=True)
stdout, stderr = p.communicate()
new = stdout.strip()[:40]
response['update'] = len(new) == 40 and current != new
'''
response['update'] = False
else:
get_latest_release()
if not os.path.exists(os.path.join(settings.updates_path, 'release.json')):
return response
if not os.path.exists(os.path.join(settings.data_path, 'release.json')):
return response
with open(os.path.join(settings.updates_path, 'release.json')) as fd:
release = json.load(fd)
current = current_version('openmedialibrary')
response['current'] = current
new = release['modules']['openmedialibrary']['version']
response['version'] = new
response['update'] = current < new
return response
actions.register(getVersion, cache=False)
actions.register(getVersion, cache=False, version='public')
def restart(data):
'''
restart (and upgrade if upgrades are available)
'''
restart_oml(data.get('update'))
return {}
actions.register(restart, cache=False)
class Update(Thread):
_status = {
'reload': False,
'status': 'Updating Open Media Library...'
}
def __init__(self):
Thread.__init__(self)
self.daemon = True
self.start()
def status(self, status, reload=False):
from websocket import trigger_event
self._status = {
'reload': reload,
'status': status,
}
trigger_event('updatestatus', self._status)
def install(self):
while update_available():
self.status('Downloading...')
max_retry = 5
while max_retry > 0 and not download():
max_retry -= 1
self.status('Download failed, retrying...')
time.sleep(5)
self.status('Downloading...')
self.status('Installing...')
if not install():
self.status('Installation failed.')
return True
return False
def update_database(self):
db_version = settings.server.get('db_version', 0)
if db_version < settings.DB_VERSION:
self.status('Updating...')
if db_version < 3:
db_version = migrate_3()
if db_version < 4:
db_version = migrate_4()
if db_version < 5:
db_version = migrate_5()
if db_version < 6:
db_version = migrate_6()
if db_version < 7:
db_version = migrate_7()
if db_version < 9:
db_version = migrate_8()
if db_version < 10:
db_version = migrate_10()
if db_version < 11:
db_version = migrate_11()
if db_version < 12:
db_version = migrate_12()
if db_version < 13:
db_version = migrate_13()
if db_version < 15:
db_version = migrate_15()
if db_version < 16:
db_version = migrate_16()
if db_version < 17:
db_version = migrate_17()
if db_version < 18:
db_version = migrate_18()
settings.server['db_version'] = db_version
def run(self):
self.status('Checking for updates...')
self.update_database()
if self.install():
restart_oml()
return
self.status('Relaunching...', True)
restart_oml()
def migrate_3():
with db.session():
import item.models
for i in item.models.Item.find({
'query': {
'conditions': [{
'key':'mediastate',
'value':'available',
'operator': '=='
}]
}
}):
if not i.files.all():
i.remove_file()
else:
f = i.files.all()[0]
if not 'pages' in i.info and 'pages' in f.info:
i.info['pages'] = f.info['pages']
i.save()
return 3
def migrate_4():
with db.session() as session:
import item.models
from meta.utils import to_isbn13
for i in item.models.Item.query:
update = False
if 'isbn' in i.meta and isinstance(i.meta['isbn'], list):
isbns = [to_isbn13(isbn) for isbn in i.meta['isbn']]
isbns = [isbn for isbn in isbns if isbn]
if isbns:
i.meta['isbn'] = isbns[0]
if 'isbn' in i.info:
i.info['isbn'] = i.meta['isbn']
else:
del i.meta['isbn']
if 'isbn' in i.info:
del i.info['isbn']
update = True
if 'isbn' in i.meta and not i.meta['isbn']:
del i.meta['isbn']
update = True
if update:
session.add(i)
session.commit()
return 4
def migrate_5():
db.run_sql([
'DROP INDEX IF EXISTS user_metadata_index',
'CREATE UNIQUE INDEX user_metadata_index ON user_metadata(item_id, user_id)',
'UPDATE sort SET sharemetadata = 0',
])
with db.session() as session:
import user.models
for m in user.models.Metadata.query:
data_hash = m.get_hash()
if m.data_hash != data_hash:
m.data_hash = data_hash
session.add(m)
session.commit()
import item.models
for i in item.models.Item.query:
update = False
if '_from' in i.info:
del i.info['_from']
update = True
if update:
session.add(i)
session.commit()
for i in item.models.Item.query:
i.sync_metadata()
return 5
def migrate_6():
with db.session() as session:
import item.models
for s in item.models.Sort.query.filter_by(author=''):
s.item.update_sort()
for s in item.models.Sort.query.filter_by(publisher=''):
s.item.update_sort()
for s in item.models.Sort.query.filter_by(language=''):
s.item.update_sort()
for s in item.models.Sort.query.filter_by(place=''):
s.item.update_sort()
for s in item.models.Sort.query.filter_by(isbn=''):
s.item.update_sort()
for s in item.models.Sort.query.filter_by(date=''):
s.item.update_sort()
session.commit()
return 6
def migrate_7():
with db.session() as session:
import changelog
for c in changelog.Changelog.query:
if 'editmeta' in c.data or 'resetmeta' in c.data:
session.delete(c)
session.commit()
db.run_sql('DROP TABLE IF EXISTS metadata')
db.run_sql('DROP TABLE IF EXISTS scrape')
db.run_sql('VACUUM')
return 7
def migrate_8():
for key in ('directory_service', 'meta_service', 'local_lookup', 'cert'):
if key in settings.server:
del settings.server[key]
list_cache = os.path.join(settings.data_path, 'list_cache.json')
if os.path.exists(list_cache):
os.unlink(list_cache)
with db.session() as session:
import item.models
for i in item.models.Item.query:
delta = set(i.meta)-set(i.meta_keys)
if delta:
for key in delta:
del i.meta[key]
session.add(i)
session.commit()
import changelog
import user.models
changelog.Changelog.query.delete()
u = user.models.User.get(settings.USER_ID)
u.rebuild_changelog()
for peer in user.models.User.query:
if peer.id != u.id:
if len(peer.id) != 16:
session.delete(peer)
session.commit()
return 8
def migrate_10():
with db.session() as session:
from item.models import Item, Find
from utils import get_by_id
from item.person import get_sort_name
import unicodedata
sort_names = {}
updates = {}
for f in Find.query.filter(Find.key.in_(Item.filter_keys)):
sort_type = get_by_id(settings.config['itemKeys'], f.key).get('sortType')
if sort_type == 'person':
if f.value in sort_names:
sortvalue = sort_names[f.value]
else:
sortvalue = sort_names[f.value] = get_sort_name(f.value)
else:
sortvalue = f.value
if sortvalue:
sortvalue = ox.sort_string(unicodedata.normalize('NFKD', sortvalue)).lower()
if not f.key in updates:
updates[f.key] = {}
updates[f.key][f.value] = sortvalue
for key in updates:
for value in updates[key]:
Find.query.filter_by(key=key, value=value).update({'sortvalue': updates[key][value]})
session.commit()
return 10
def migrate_11():
with db.session() as session:
from user.models import User, Metadata, List
from changelog import Changelog
import utils
for u in User.query.filter_by(peered=True):
peer = utils.get_peer(u.id)
last = Changelog.query.filter_by(user_id=u.id).order_by(text('-revision')).first()
if last:
peer.info['revision'] = last.revision
listorder = []
for l in List.query.filter_by(user_id=u.id).order_by('index_'):
if l.name:
peer.info['lists'][l.name] = [i.id for i in l.get_items()]
listorder.append(l.name)
if 'listorder' not in peer.info:
peer.info['listorder'] = listorder
for m in Metadata.query.filter_by(user_id=u.id):
peer.library[m.item_id] = {
'meta': dict(m.data),
'meta_hash': m.data_hash,
'modified': m.modified,
}
peer.library.commit()
peer.sync_info()
peer.sync_db()
Changelog.query.filter_by(user_id=u.id).delete()
Metadata.query.filter_by(user_id=u.id).delete()
session.commit()
if db.table_exists('transfer'):
import state
import downloads
state.online = False
state.downloads = downloads.Downloads()
r = state.db.session.execute('SELECT item_id, added, progress FROM transfer')
for t in r:
item_id, added, progress = t
if added:
state.downloads.transfers[item_id] = {
'added': added,
'progress': progress
}
state.db.session.commit()
state.downloads.transfers.commit()
state.downloads = None
return 11
def migrate_12():
db.run_sql([
'DROP TABLE IF EXISTS transfer'
])
return 12
def migrate_13():
import settings
import changelog
import os
import json
path = os.path.join(settings.data_path, 'peers', '%s.log' % settings.USER_ID)
if not os.path.exists(path):
folder = os.path.dirname(path)
if not os.path.exists(folder):
os.makedirs(folder)
with db.session() as session:
revision = -1
qs = changelog.Changelog.query.filter_by(user_id=settings.USER_ID)
with open(path, 'w') as fd:
for c in qs.order_by('timestamp'):
data = json.dumps([c.revision, c.timestamp, json.loads(c.data)], ensure_ascii=False)
fd.write(data + '\n')
revision = c.revision
if revision > -1:
settings.server['revision'] = revision
return 13
def migrate_15():
from user.models import List, User
with db.session():
l = List.get(':Public')
if l and not len(l.items):
l.remove()
for u in User.query:
if 'local' in u.info:
del u.info['local']
u.save()
return 15
def migrate_16():
db.run_sql([
'''CREATE TABLE user2 (
created DATETIME,
modified DATETIME,
id VARCHAR(43) NOT NULL,
info BLOB,
nickname VARCHAR(256),
pending VARCHAR(64),
queued BOOLEAN,
peered BOOLEAN,
online BOOLEAN,
PRIMARY KEY (id),
CHECK (queued IN (0, 1)),
CHECK (peered IN (0, 1)),
CHECK (online IN (0, 1))
)''',
'''INSERT INTO user2 (created, modified, id, info, nickname, pending, queued, peered, online)
SELECT created, modified, id, info, nickname, pending, queued, peered, online FROM user''',
'DROP TABLE user',
'ALTER TABLE user2 RENAME TO user',
'CREATE INDEX IF NOT EXISTS ix_user_nickname ON user (nickname)'
])
return 16
def migrate_17():
from user.models import List, User
from changelog import add_record
with db.session():
l = List.get(':Public')
if not l:
add_record('removelist', 'Public')
lists = []
for l in List.query.filter_by(user_id=settings.USER_ID).order_by('index_'):
if l.type == 'static' and l.name not in ('', 'Inbox'):
lists.append(l.name)
add_record('orderlists', lists)
return 17
def migrate_18():
db.run_sql([
'''CREATE TABLE annotation (
_id INTEGER NOT NULL,
id VARCHAR(43),
created DATETIME,
modified DATETIME,
user_id VARCHAR(43),
item_id VARCHAR(43),
data BLOB,
findquotes TEXT,
findnotes TEXT,
PRIMARY KEY (_id),
FOREIGN KEY(user_id) REFERENCES user (id),
FOREIGN KEY(item_id) REFERENCES item (id)
)'''])
db.run_sql([
'CREATE INDEX ix_annotation_findquotes ON annotation (findquotes)',
'CREATE INDEX ix_annotation_findnotes ON annotation (findnotes)'
])
return 18