From 8169d8e7204505c1679e20d0a00cf6628e4cf553 Mon Sep 17 00:00:00 2001 From: j Date: Sat, 16 Feb 2019 13:26:31 +0530 Subject: [PATCH 1/5] avoid tor deadlock, clenaup annotations on delete --- oml/item/models.py | 17 ++++++++++++++--- oml/tor.py | 1 + 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/oml/item/models.py b/oml/item/models.py index c9523fb..508f46e 100644 --- a/oml/item/models.py +++ b/oml/item/models.py @@ -311,13 +311,20 @@ class Item(db.Model): def delete(self, commit=True): Sort.query.filter_by(item_id=self.id).delete() + Find.query.filter_by(item_id=self.id).delete() if state.downloads and self.id in state.downloads.transfers: del state.downloads.transfers[self.id] state.db.session.delete(self) - icons.clear('cover:%s' % self.id) - icons.clear('preview:%s' % self.id) if commit: state.db.session.commit() + icons.clear('cover:%s' % self.id) + icons.clear('preview:%s' % self.id) + + def remove_annotations(self): + from annotation.models import Annotation + for a in Annotation.query.filter_by(item_id=self.id, user_id=state.user()): + a.add_record('removeannotation') + a.delete() meta_keys = ( 'author', @@ -562,6 +569,7 @@ class Item(db.Model): state.db.session.add(self) def remove_file(self): + folders = set() for f in self.files.all(): path = f.fullpath() if os.path.exists(path): @@ -569,10 +577,12 @@ class Item(db.Model): try: os.chmod(path, mode) os.unlink(path) - remove_empty_folders(os.path.dirname(path)) + folders.add(os.path.dirname(path)) except: pass state.db.session.delete(f) + for folder in folders: + remove_empty_folders(folder) user = state.user() if user in self.users: self.users.remove(user) @@ -589,6 +599,7 @@ class Item(db.Model): if self.id in state.downloads.transfers: del state.downloads.transfers[self.id] add_record('removeitem', self.id) + self.remove_annotations() class Sort(db.Model): __tablename__ = 'sort' diff --git a/oml/tor.py b/oml/tor.py index 18180c1..306a539 100644 --- a/oml/tor.py +++ b/oml/tor.py @@ -103,6 +103,7 @@ DirReqStatistics 0 for line in self.p.stdout: self._status.append(line) logger.debug(line) + self.p.communicate() time.sleep(0.5) self.p = None From b4b66f9bd552f2b1da8e870a4cdbcbc637bfe5a6 Mon Sep 17 00:00:00 2001 From: j Date: Sat, 16 Feb 2019 13:28:18 +0530 Subject: [PATCH 2/5] remove annotations and other annotation menu options --- static/js/annotationPanel.js | 32 ++++++++++++++++------- static/js/removeAnnotations.js | 48 ++++++++++++++++++++++++++++++++++ static/js/viewer.js | 18 ++++++++++++- static/json/js.json | 1 + 4 files changed, 88 insertions(+), 11 deletions(-) create mode 100644 static/js/removeAnnotations.js diff --git a/static/js/annotationPanel.js b/static/js/annotationPanel.js index c189abc..f0cf0e5 100644 --- a/static/js/annotationPanel.js +++ b/static/js/annotationPanel.js @@ -1,6 +1,7 @@ 'use strict'; -oml.ui.annotationPanel = function() { +oml.ui.annotationPanel = function(options, self) { + self = self || {}; var ui = oml.user.ui; var ui = oml.user.ui; @@ -29,14 +30,15 @@ oml.ui.annotationPanel = function() { click: function() { var $annotation = oml.$ui.annotationFolder.find('.OMLAnnotation.selected') $annotation.length && $annotation.delete() - $deleteQuote.options({disabled: true}) + that.updateSelection() } }).appendTo($bar); var $menuButton = Ox.MenuButton({ items: [ - {id: 'addAnnotation', title: 'Add Annotation', disabled: true}, - {id: 'removeAnnotation', title: 'Remove Annotation', disabled: true}, + {id: 'addAnnotation', title: 'Add Annotation', disabled: true, keyboard: 'return'}, + {id: 'removeAnnotation', title: 'Remove Annotation', disabled: true, keyboard: 'delete'}, + {id: 'removeAnnotations', title: 'Remove All Annotation', disabled: true}, {}, {id: 'show', title: Ox._('Show Annotations'), disabled: true}, {group: 'showAnnotationUsers', min: 1, max: 1, items: [ @@ -82,6 +84,14 @@ oml.ui.annotationPanel = function() { }, function(result) { oml.ui.exportAnnotationsDialog(result.data).open() }) + } else if (id =='addAnnotation') { + oml.$ui.viewer.postMessage('addAnnotation', {}) + } else if (id =='removeAnnotation') { + var $annotation = oml.$ui.annotationFolder.find('.OMLAnnotation.selected') + $annotation.length && $annotation.delete() + that.updateSelection() + } else if (id =='removeAnnotations') { + oml.ui.removeAnnotationsDialog().open() } else { console.log('click', id, data) } @@ -113,16 +123,18 @@ oml.ui.annotationPanel = function() { } ], orientation: 'vertical' + }, self).update({ + hasAnnotations: function() { + $menuButton[self.options.hasAnnotations ? 'enableItem' : 'disableItem']('removeAnnotations') + } }); that.updateSelection = function(selection) { - $addQuote.options({ - disabled: !selection - }) var $annotation = oml.$ui.annotationFolder.find('.OMLAnnotation.selected') - $deleteQuote.options({ - disabled: !$annotation.length - }) + $addQuote.options({disabled: !selection}) + $deleteQuote.options({disabled: !$annotation.length}) + $menuButton[selection ? 'enableItem' : 'disableItem']('addAnnotation') + $menuButton[$annotation.length ? 'enableItem' : 'disableItem']('removeAnnotation') } return that; diff --git a/static/js/removeAnnotations.js b/static/js/removeAnnotations.js new file mode 100644 index 0000000..d9e169a --- /dev/null +++ b/static/js/removeAnnotations.js @@ -0,0 +1,48 @@ +'use strict'; + +oml.ui.removeAnnotationsDialog = function() { + + var ui = oml.user.ui, + + annotations = oml.$ui.viewer.getAnnotations().filter(function(a) { + return a.user == oml.user.id + }), + annotationsName = Ox._(annotations.length == 1 ? 'Annotation' : 'Annotations'), + theseAnnotationsName = annotations.length == 1 + ? Ox._('this annotation') + : Ox._('these {0} annotations', [Ox.formatNumber(annotations.length)]), + + that = oml.ui.confirmDialog({ + buttons: [ + Ox.Button({ + style: 'squared', + title: Ox._('No, Keep {0}', [annotationsName]) + }), + Ox.Button({ + style: 'squared', + title: Ox._('Yes, Delete {0}', [annotationsName]) + }) + ], + content: Ox._( + 'Are you sure that you want to permanently delete {0}?', + [theseAnnotationsName] + ), + title: Ox._('Delete {0}', [annotationsName]) + }, function() { + Ox.serialForEach(annotations, function(a, index, annotations, next) { + oml.api.removeAnnotation({ + item: ui.item, + annotation: a.id + }, function(result) { + next() + }) + }, function() { + Ox.Request.clearCache(); + oml.$ui.viewer.renderAnnotations(true); + }) + }); + + return that; + +}; + diff --git a/static/js/viewer.js b/static/js/viewer.js index eb4d902..24c48a2 100644 --- a/static/js/viewer.js +++ b/static/js/viewer.js @@ -111,6 +111,7 @@ oml.ui.viewer = function() { if (save !== false) { oml.api.addAnnotation(a) } + data.user = a.user || oml.user.id data.notes = data.notes || []; annotations.push(data); } @@ -157,9 +158,13 @@ oml.ui.viewer = function() { oml.$ui.annotationFolder.append($annotation); $annotation.annotate(); oml.$ui.annotationPanel.updateSelection(false) + oml.$ui.annotationPanel.options({hasAnnotations: true}) } else if (event == 'removeAnnotation') { oml.$ui.annotationFolder.find('#a-' + data.id).remove() data.id && removeAnnotation(data.id) + oml.$ui.annotationPanel.options({hasAnnotations: annotations.filter(function(a) { + return a.user == oml.user.id + }).length > 0}) } else if (event == 'selectAnnotation') { if (data.id) { var $annotation = oml.$ui.annotationFolder.find('#a-' + data.id) @@ -196,7 +201,7 @@ oml.ui.viewer = function() { that.getAnnotations = function() { return annotations; } - that.renderAnnotations = function() { + that.renderAnnotations = function(load=false) { var sortKey = ui.sortAnnotations if (sortKey == 'date') { sortKey = 'created' @@ -207,9 +212,15 @@ oml.ui.viewer = function() { if (sortKey == 'quote') { sortKey = 'text' } + if (load) { + loadAnnotations(function() { + that.renderAnnotations() + }) + } annotations = Ox.sortBy(annotations, sortKey) oml.$ui.annotationFolder.empty(); var visibleAnnotations = []; + var hasAnnotations = false; annotations.forEach(function(data) { //that.postMessage('removeAnnotation', {id: data.id}) if (ui.showAnnotationUsers == 'all' || data.user == oml.user.id) { @@ -217,7 +228,11 @@ oml.ui.viewer = function() { oml.$ui.annotationFolder.append($annotation); visibleAnnotations.push(data) } + if (data.user == oml.user.id) { + hasAnnotations = true + } }) + oml.$ui.annotationPanel.options({hasAnnotations: hasAnnotations}) // fixme: trigger loaded event from reader instead? setTimeout(function() { that.postMessage('addAnnotations', { @@ -225,6 +240,7 @@ oml.ui.viewer = function() { replace: true }) }, 500) + } return that.updateElement(); }; diff --git a/static/json/js.json b/static/json/js.json index f283185..e493cf5 100644 --- a/static/json/js.json +++ b/static/json/js.json @@ -58,6 +58,7 @@ "preferencesPanel.js", "previewButton.js", "previewDialog.js", + "removeAnnotations.js", "resetUIDialog.js", "rightPanel.js", "sectionButtons.js", From 41d90e1736a84c2866040148018c17e3f9444f03 Mon Sep 17 00:00:00 2001 From: j Date: Wed, 20 Feb 2019 18:22:20 +0530 Subject: [PATCH 3/5] windows fails to open utf-8 files as utf-8 --- oml/changelog.py | 6 +++--- oml/library.py | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/oml/changelog.py b/oml/changelog.py index 3f89f47..55dc853 100644 --- a/oml/changelog.py +++ b/oml/changelog.py @@ -36,14 +36,14 @@ def add_record(action, *args, **kwargs): revision = next_revision() data = [revision, timestamp, [action] + list(args)] - data = json.dumps(data, ensure_ascii=False) + data = json.dumps(data, ensure_ascii=False).encode('utf-8') path = changelog_path() if os.path.exists(path): - mode = 'a' + mode = 'ab' state.changelog_size = os.path.getsize(path) else: - mode = 'w' + mode = 'wb' state.changelog_size = 0 makefolder(path) with open(path, mode) as fd: diff --git a/oml/library.py b/oml/library.py index 3c17ed0..09cc474 100644 --- a/oml/library.py +++ b/oml/library.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +import codecs import json import os import time @@ -52,7 +53,7 @@ class Peer(object): def apply_log(self): changes = [] if os.path.exists(self._logpath): - with open(self._logpath) as fd: + with codecs.open(self._logpath, 'r', encoding='utf-8') as fd: for line in fd: if line: try: From fa9482de48d5cbd1b723d806c92d0c1f210784d8 Mon Sep 17 00:00:00 2001 From: j Date: Wed, 20 Feb 2019 18:28:25 +0530 Subject: [PATCH 4/5] less debug output --- oml/nodes.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/oml/nodes.py b/oml/nodes.py index 3e4c575..27d7658 100644 --- a/oml/nodes.py +++ b/oml/nodes.py @@ -70,6 +70,8 @@ class Node(Thread): else: if not self._q.qsize(): time.sleep(5) + else: + time.sleep(0.1) self.queue(action[0], *action[1]) else: logger.debug('unknown action %s', action) @@ -83,7 +85,8 @@ class Node(Thread): self._q.put('ping') def queue(self, action, *args): - logger.debug('queue node action %s->%s%s', self.user_id, action, args) + if DEBUG_NODES: + logger.debug('queue node action %s->%s%s', self.user_id, action, args) self._q.put([action, args]) def _call(self, action, *args): From a45445d1149076d1c4e3b27d33bec917dc080027 Mon Sep 17 00:00:00 2001 From: j Date: Wed, 20 Feb 2019 18:35:23 +0530 Subject: [PATCH 5/5] disable authentication to make running multiple nodes easier --- oml/tor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oml/tor.py b/oml/tor.py index 306a539..17056ca 100644 --- a/oml/tor.py +++ b/oml/tor.py @@ -45,7 +45,7 @@ AvoidDiskWrites 1 Log notice stdout SocksPort 9830 ControlPort 9831 -CookieAuthentication 1 +#CookieAuthentication 1 '''.strip()) tor_data = os.path.join(settings.data_path, 'TorData') if sys.platform == 'win32':