From 739cfd8c9d8967ca95e7635b7a3282906f0ab54d Mon Sep 17 00:00:00 2001 From: j Date: Sat, 23 Feb 2019 12:01:10 +0530 Subject: [PATCH 1/6] changelog uses \n newlines --- oml/library.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/oml/library.py b/oml/library.py index 09cc474..90798aa 100644 --- a/oml/library.py +++ b/oml/library.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -import codecs import json import os import time @@ -53,7 +52,7 @@ class Peer(object): def apply_log(self): changes = [] if os.path.exists(self._logpath): - with codecs.open(self._logpath, 'r', encoding='utf-8') as fd: + with open(self._logpath, 'r', encoding='utf-8', newline='\n') as fd: for line in fd: if line: try: From 08a3477e88b631091dab1fe941f4fa031cbf636c Mon Sep 17 00:00:00 2001 From: j Date: Sat, 23 Feb 2019 12:32:20 +0530 Subject: [PATCH 2/6] cleanup users --- oml/library.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/oml/library.py b/oml/library.py index 90798aa..eee921e 100644 --- a/oml/library.py +++ b/oml/library.py @@ -446,6 +446,9 @@ def cleanup_peers(): other_peers[id] = peer.info['peers'][id] known_peers.add(id) for u in user.models.User.query.filter(user.models.User.id.notin_(list(known_peers))): - if not 'local' in u.info and not u.pending: + if state.nodes and u.id in state.nodes.local: + continue + if not u.pending: state.db.session.delete(u) + state.db.session.commit() From b3246f05db11e75478b407df54b9b196a125e5b6 Mon Sep 17 00:00:00 2001 From: j Date: Sat, 23 Feb 2019 12:50:35 +0530 Subject: [PATCH 3/6] fix broken logs --- oml/settings.py | 2 +- oml/update.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/oml/settings.py b/oml/settings.py index 39df3a2..86888ee 100644 --- a/oml/settings.py +++ b/oml/settings.py @@ -95,4 +95,4 @@ FULLTEXT_SUPPORT = fulltext.platform_supported() if not FULLTEXT_SUPPORT: config['itemKeys'] = [k for k in config['itemKeys'] if k['id'] != 'fulltext'] -DB_VERSION = 18 +DB_VERSION = 19 diff --git a/oml/update.py b/oml/update.py index 3ae467a..861a819 100644 --- a/oml/update.py +++ b/oml/update.py @@ -379,6 +379,8 @@ class Update(Thread): db_version = migrate_17() if db_version < 18: db_version = migrate_18() + if db_version < 19: + db_version = migrate_19() settings.server['db_version'] = db_version def run(self): @@ -698,3 +700,15 @@ def migrate_18(): 'CREATE INDEX ix_annotation_findnotes ON annotation (findnotes)' ]) return 18 + +def migrate_19(): + from user.models import User + with db.session(): + peers = [u for u in User.query.filter_by(peered=True)] + peers.sort(key=lambda u: utils.user_sort_key(u.json())) + for u in peers: + peer = utils.get_peer(u.id) + if not peer.info.get('revision') and os.path.exists(peer._logpath) and os.path.getsize(peer._logpath): + logger.debug('try to apply pending logs for %s', u.id) + peer.apply_log() + return 19 From 7ee959cd118d52d3b1e9405ee00225d368f69d47 Mon Sep 17 00:00:00 2001 From: j Date: Sat, 23 Feb 2019 17:11:50 +0530 Subject: [PATCH 4/6] all changelogs should be utf-8 --- oml/changelog.py | 2 +- oml/settings.py | 2 +- oml/update.py | 26 +++++++++++++++++++++++++- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/oml/changelog.py b/oml/changelog.py index 05296e0..f641fd4 100644 --- a/oml/changelog.py +++ b/oml/changelog.py @@ -446,7 +446,7 @@ class Changelog(db.Model): listitems = data[2] #remove from additemlists removed = [] - if list_id in changes.get('addlistitems',{}): + if list_id in changes.get('addlistitems', {}): removed = [ i for i in changes['addlistitems'][list_id] if i in listitems ] diff --git a/oml/settings.py b/oml/settings.py index 86888ee..5c06f74 100644 --- a/oml/settings.py +++ b/oml/settings.py @@ -95,4 +95,4 @@ FULLTEXT_SUPPORT = fulltext.platform_supported() if not FULLTEXT_SUPPORT: config['itemKeys'] = [k for k in config['itemKeys'] if k['id'] != 'fulltext'] -DB_VERSION = 19 +DB_VERSION = 20 diff --git a/oml/update.py b/oml/update.py index 861a819..5f7f7fe 100644 --- a/oml/update.py +++ b/oml/update.py @@ -381,6 +381,8 @@ class Update(Thread): db_version = migrate_18() if db_version < 19: db_version = migrate_19() + if db_version < 20: + db_version = migrate_20() settings.server['db_version'] = db_version def run(self): @@ -710,5 +712,27 @@ def migrate_19(): peer = utils.get_peer(u.id) if not peer.info.get('revision') and os.path.exists(peer._logpath) and os.path.getsize(peer._logpath): logger.debug('try to apply pending logs for %s', u.id) - peer.apply_log() + try: + peer.apply_log() + except: + logger.error('failed to apply log for %s', u.id) return 19 + +def migrate_20(): + from glob import glob + changed = False + for log in glob(os.path.join(settings.data_path, 'peers', '*.log')): + with open(log, 'rb') as fd: + data = fd.read() + try: + data.decode('utf-8') + except UnicodeDecodeError: + data = data.decode('Windows-1252') + logger.error('convert %s to utf-8', log) + with open(log, 'wb') as fd: + fd.write(data.encode('utf-8')) + changed = True + + if changed: + migrate_19() + return 20 From 349383ff39d2080d39561b9d82ef3c0cd31bd858 Mon Sep 17 00:00:00 2001 From: j Date: Tue, 5 Mar 2019 18:23:28 +0100 Subject: [PATCH 5/6] write utf-8 --- oml/update.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/oml/update.py b/oml/update.py index 5f7f7fe..10d24e2 100644 --- a/oml/update.py +++ b/oml/update.py @@ -621,10 +621,10 @@ def migrate_13(): with db.session() as session: revision = -1 qs = changelog.Changelog.query.filter_by(user_id=settings.USER_ID) - with open(path, 'w') as fd: + with open(path, 'wb') as fd: for c in qs.order_by('timestamp'): - data = json.dumps([c.revision, c.timestamp, json.loads(c.data)], ensure_ascii=False) - fd.write(data + '\n') + data = json.dumps([c.revision, c.timestamp, json.loads(c.data)], ensure_ascii=False).encode('utf-8') + fd.write(data + b'\n') revision = c.revision if revision > -1: settings.server['revision'] = revision From 111ea307a99c220bae0e03fc901dbf0ac441f9ea Mon Sep 17 00:00:00 2001 From: j Date: Tue, 5 Mar 2019 18:31:21 +0100 Subject: [PATCH 6/6] fix annotations sort --- static/js/annotationPanel.js | 21 ++++++++------------- static/js/viewer.js | 4 +--- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/static/js/annotationPanel.js b/static/js/annotationPanel.js index f0cf0e5..e9fee31 100644 --- a/static/js/annotationPanel.js +++ b/static/js/annotationPanel.js @@ -64,17 +64,6 @@ oml.ui.annotationPanel = function(options, self) { // borderColor: 'transparent', float: 'right' }).bindEvent({ - change: function(data) { - if (data.id == 'showAnnotationUsers') { - var value = data.checked[0].id; - oml.UI.set('showAnnotationUsers', value); - oml.$ui.viewer.renderAnnotations(); - } else if (data.id == 'sortAnnotations') { - var value = data.checked[0].id; - oml.UI.set('sortAnnotations', value); - oml.$ui.viewer.renderAnnotations(); - } - }, click: function(data) { var id = data.id; if (id == 'exportAnnotations') { @@ -98,8 +87,14 @@ oml.ui.annotationPanel = function(options, self) { }, change: function(data) { var id = data.id; - console.log('change', data) - if (id == 'show') { + if (data.id == 'showAnnotationUsers') { + var value = data.checked[0].id; + oml.UI.set('showAnnotationUsers', value); + oml.$ui.viewer.renderAnnotations(); + } else if (data.id == 'sortAnnotations') { + var value = data.checked[0].id; + oml.UI.set('sortAnnotations', value); + } else if (id == 'show') { console.log('show', data) oml.UI.set({annotationsShow: data.checked[0].id}); } else if (id == 'sort') { diff --git a/static/js/viewer.js b/static/js/viewer.js index 24c48a2..616aacf 100644 --- a/static/js/viewer.js +++ b/static/js/viewer.js @@ -206,9 +206,6 @@ oml.ui.viewer = function() { if (sortKey == 'date') { sortKey = 'created' } - if (sortKey == 'date') { - sortKey = 'created' - } if (sortKey == 'quote') { sortKey = 'text' } @@ -216,6 +213,7 @@ oml.ui.viewer = function() { loadAnnotations(function() { that.renderAnnotations() }) + return } annotations = Ox.sortBy(annotations, sortKey) oml.$ui.annotationFolder.empty();