0x2620
/
openmedialibrary
1
0
Fork 0
Code Issues 131 Pull Requests Releases Wiki Activity

store annotations in db and sync with peers

This commit is contained in:
j 2019-02-10 17:46:35 +05:30
parent 131a6a3215
commit e0cba14d6a
21 changed files with 385 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
config.json
View File

@ -241,6 +241,18 @@
"format": {"type": "boolean", "args": []},
"sort": true
},
{
"id": "quotes",
"title": "Quotes",
"find": true,
"type": "text"
},
{
"id": "notes",
"title": "Notes",
"find": true,
"type": "text"
},
{
"id": "fulltext",
"title": "Full Text",

48
oml/annotation/api.py Normal file
View File

@ -0,0 +1,48 @@
# -*- coding: utf-8 -*-
from oxtornado import actions
from . import models
import settings
import state
from changelog import add_record
import logging
logger = logging.getLogger(__name__)
def getAnnotations(data):
response = {}
response['annotations'] = models.Annotation.get_by_item(data['id'])
return response
actions.register(getAnnotations)
def addAnnotation(data):
item_id = data.pop('item')
a = models.Annotation.create(item_id, settings.USER_ID, data)
a.add_record('addannotation')
response = a.json()
return response
actions.register(addAnnotation)
def editNote(data):
a = models.Annotation.get(state.user(), data['item'], data['annotation'])
if a:
a.data['notes'] = data['notes']
a.add_record('editannotation')
a.save()
response = a.json()
else:
response = {}
return response
actions.register(editNote)
def removeAnnotation(data):
a = models.Annotation.get(state.user(), data['item'], data['annotation'])
if a:
a.add_record('removeannotation')
a.delete()
response = {}
return response
actions.register(removeAnnotation)

68
oml/annotation/models.py
View File

@ -4,6 +4,7 @@ import json
import logging
import os
import shutil
import unicodedata
from sqlalchemy.orm import load_only
import ox
@ -31,10 +32,15 @@ class Annotation(db.Model):
created = sa.Column(sa.DateTime())
modified = sa.Column(sa.DateTime())
user_id = sa.Column(sa.String(43), sa.ForeignKey('user.id'))
user = sa.orm.relationship('User', backref=sa.orm.backref('annotations', lazy='dynamic'))
item_id = sa.Column(sa.String(43), sa.ForeignKey('item.id'))
item = sa.orm.relationship('Item', backref=sa.orm.backref('items', lazy='dynamic'))
data = sa.Column(MutableDict.as_mutable(sa.PickleType(pickler=json_pickler)))
findquotes = sa.Column(sa.Text(), index=True)
findnotes = sa.Column(sa.Text(), index=True)
def __init__(self, item_id, user_id, data):
self.created = datetime.utcnow()
self.modified = datetime.utcnow()
@ -43,28 +49,44 @@ class Annotation(db.Model):
self.data = data
@classmethod
def create(cls, **kwargs):
a = cls(**kwargs)
state.db.session.add(a)
def create(cls, item_id, user_id, data):
a = cls(item_id, user_id, data)
a.save()
return a
def delete(self):
state.db.session.delete(self)
state.db.session.commit()
@classmethod
def get(cls, user, item_id, annotation_id):
for a in cls.query.filter_by(item_id=item_id, user=user, _id=annotation_id):
if a.data.get('id') == annotation_id:
return a
if isinstance(user, str):
qs = cls.query.filter_by(item_id=item_id, user_id=user, id=annotation_id)
else:
qs = cls.query.filter_by(item_id=item_id, user=user, id=annotation_id)
for a in qs:
return a
@classmethod
def get_by_item(cls, user, item_id):
def get_by_item(cls, item_id):
annotations = []
for a in cls.query.filter_by(item_id=item_id):
annotations.append(a.json())
return annotations
def save(self):
_id = self.data.get('id')
if _id:
self._id = _id
id = self.data.get('id')
if id:
self.id = id
self.findquotes = unicodedata.normalize('NFKD', self.data.get('text', '')).lower()
note = self.data.get('notes', {})
if isinstance(note, list) and note:
note = note[0]
if isinstance(note, dict):
note = note.get('value', '')
else:
note = ''
self.findnotes = unicodedata.normalize('NFKD', note).lower()
state.db.session.add(self)
state.db.session.commit()
@ -73,6 +95,30 @@ class Annotation(db.Model):
data['created'] = self.created
data['modified'] = self.modified
data['user'] = self.user_id
data['_id'] = ox.toAZ(self.id)
data['_id'] = ox.toAZ(self._id)
if isinstance(data.get('notes'), dict):
note = data['notes']
if self.user_id != settings.USER_ID:
note['user'] = self.user_id
if not note.get('id'):
note['id'] = 'A'
data['notes'] = [data['notes']]
if 'notes' not in data:
data['notes'] = []
return data
def add_record(self, action):
args = [self.item_id]
if action == 'addannotation':
args.append(self.data)
elif action == 'editannotation':
args.append(self.id)
args.append({
'notes': self.data['notes']
})
elif action == 'removeannotation':
args.append(self.id)
else:
raise Exception('unknown action')
add_record(action, *args)

1
oml/api.py
View File

@ -12,6 +12,7 @@ from oxtornado import actions
import item.api
import user.api
import annotation.api
import update
import utils

27
oml/changelog.py
View File

@ -1,8 +1,8 @@
# -*- coding: utf-8 -*-
import os
from datetime import datetime
import json
import os
import sqlalchemy as sa
from sqlalchemy.sql.expression import text
@ -76,6 +76,10 @@ class Changelog(db.Model):
addpeer peerid peername
removepeer peerid peername
editpeer peerid {username: string, contact: string}
addannotation itemid data
editannotation itemid annotationid data
removeannotation itemid annotationid
'''
__tablename__ = 'changelog'
id = sa.Column(sa.Integer(), primary_key=True)
@ -327,6 +331,27 @@ class Changelog(db.Model):
peer.save()
return True
def action_addannotation(self, user, timestamp, itemid, data):
from annotation.models import Annotation
Annotation.create(item_id=itemid, user_id=user.id, data=data)
return True
def action_editannotation(self, user, timestamp, itemid, annotationid, data):
from annotation.models import Annotation
a = Annotation.get(user, itemid, annotationid)
if a:
for key in data:
a.data[key] = data[key]
a.save()
return True
def action_removeannotation(self, user, timestamp, itemid, annotationid):
from annotation.models import Annotation
a = Annotation.get(user, itemid, annotationid)
if a:
a.delete()
return True
@classmethod
def aggregated_changes(cls, since=None, user_id=None):
from item.models import Item

27
oml/library.py
View File

@ -190,6 +190,33 @@ class Peer(object):
self.info['username'] = args[0]
elif action == 'editcontact':
self.info['contact'] = args[0]
elif action == 'addannotation':
from annotation.models import Annotation
if len(args) == 2:
itemid, data = args
Annotation.create(item_id=itemid, user_id=self.id, data=data)
else:
logger.error('invalid entry %s %s', action, args)
elif action == 'editannotation':
from annotation.models import Annotation
if len(args) == 3:
itemid, annotationid, data = args
a = Annotation.get(self.id, itemid, annotationid)
if a:
for key in data:
a.data[key] = data[key]
a.save()
else:
logger.error('invalid entry %s %s', action, args)
elif action == 'removeannotation':
from annotation.models import Annotation
if len(args) == 2:
itemid, annotationid = args
a = Annotation.get(self.id, itemid, annotationid)
if a:
a.delete()
else:
logger.error('invalid entry %s %s', action, args)
else:
logger.debug('UNKNOWN ACTION:', action)
self.info['revision'] = revision

4
oml/localnodes.py
View File

@ -152,8 +152,8 @@ class LocalNodes(dict):
if state.tasks:
state.tasks.queue('removelocalinfo', id)
def get(self, user_id):
data = super().get(user_id)
def get_data(self, user_id):
data = self.get(user_id)
if data and can_connect(data):
return data
return None

7
oml/nodes.py
View File

@ -124,9 +124,12 @@ class Node(Thread):
self.local = None
self.port = 9851
def is_local(self):
return self._nodes and self.user_id in self._nodes.local
def get_local(self):
if self._nodes and self._nodes.local:
return self._nodes.local.get(self.user_id)
return self._nodes.local.get_data(self.user_id)
return None
def request(self, action, *args):
@ -216,7 +219,7 @@ class Node(Thread):
return False
def is_online(self):
return self.online or self.get_local() is not None
return self.online or self.is_local()
def send_response(self):
self._q.put('send_response')

13
oml/queryparser.py
View File

@ -116,7 +116,18 @@ class Parser(object):
elif k == 'fulltext':
ids = find_fulltext(v)
return self.in_ids(ids, exclude)
elif k in ('notes', 'quotes'):
from annotation.models import Annotation
if isinstance(v, str):
v = unicodedata.normalize('NFKD', v).lower()
ids = set()
if k == 'notes':
qs = Annotation.query.filter(get_operator('=')(Annotation.findnotes, v))
elif k == 'quotes':
qs = Annotation.query.filter(get_operator('=')(Annotation.findquotes, v))
for a in qs:
ids.add(a.item_id)
return self.in_ids(ids, exclude)
elif key_type in ("string", "text"):
if isinstance(v, str):
v = unicodedata.normalize('NFKD', v).lower()

2
oml/server.py
View File

@ -170,8 +170,8 @@ def run():
import bandwidth
state.bandwidth = bandwidth.Bandwidth()
state.tor = tor.Tor()
state.node = node.server.start()
state.nodes = nodes.Nodes()
state.node = node.server.start()
def publish():
if not state.tor.is_online():

4
oml/settings.py
View File

@ -82,7 +82,7 @@ if 'modules' in release and 'openmedialibrary' in release['modules']:
else:
MINOR_VERSION = 'git'
NODE_PROTOCOL = "0.8"
NODE_PROTOCOL = "0.9"
VERSION = "%s.%s" % (NODE_PROTOCOL, MINOR_VERSION)
USER_AGENT = 'OpenMediaLibrary/%s' % VERSION
@ -95,4 +95,4 @@ FULLTEXT_SUPPORT = fulltext.platform_supported()
if not FULLTEXT_SUPPORT:
config['itemKeys'] = [k for k in config['itemKeys'] if k['id'] != 'fulltext']
DB_VERSION = 17
DB_VERSION = 18

16
oml/setup.py
View File

@ -151,6 +151,22 @@ CREATE TABLE listitem (
FOREIGN KEY(item_id) REFERENCES item (id),
FOREIGN KEY(list_id) REFERENCES list (id)
);
CREATE TABLE annotation (
_id INTEGER NOT NULL,
id VARCHAR(43),
created DATETIME,
modified DATETIME,
user_id VARCHAR(43),
item_id VARCHAR(43),
data BLOB,
findquotes TEXT,
findnotes TEXT,
PRIMARY KEY (_id),
FOREIGN KEY(user_id) REFERENCES user (id),
FOREIGN KEY(item_id) REFERENCES item (id)
);
CREATE INDEX ix_annotation_findquotes ON annotation (findquotes);
CREATE INDEX ix_annotation_findnotes ON annotation (findnotes);
PRAGMA journal_mode=WAL
'''
for statement in sql.split(';'):

24
oml/update.py
View File

@ -377,6 +377,8 @@ class Update(Thread):
db_version = migrate_16()
if db_version < 17:
db_version = migrate_17()
if db_version < 18:
db_version = migrate_18()
settings.server['db_version'] = db_version
def run(self):
@ -674,3 +676,25 @@ def migrate_17():
lists.append(l.name)
add_record('orderlists', lists)
return 17
def migrate_18():
db.run_sql([
'''CREATE TABLE annotation (
_id INTEGER NOT NULL,
id VARCHAR(43),
created DATETIME,
modified DATETIME,
user_id VARCHAR(43),
item_id VARCHAR(43),
data BLOB,
findquotes TEXT,
findnotes TEXT,
PRIMARY KEY (_id),
FOREIGN KEY(user_id) REFERENCES user (id),
FOREIGN KEY(item_id) REFERENCES item (id)
)'''])
db.run_sql([
'CREATE INDEX ix_annotation_findquotes ON annotation (findquotes)',
'CREATE INDEX ix_annotation_findnotes ON annotation (findnotes)'
])
return 18

2
oml/user/api.py
View File

@ -474,7 +474,7 @@ def removePeering(data):
if len(data.get('id', '')) not in (16, 43):
logger.debug('invalid user id')
return {}
u = models.User.get(data['id'], for_udpate=True)
u = models.User.get(data['id'], for_update=True)
if u:
u.info['message'] = data.get('message', '')
u.update_peering(False)

3
oml/user/models.py
View File

@ -187,7 +187,9 @@ class User(db.Model):
def cleanup(self):
from item.models import user_items, Item
from annotation.models import Annotation
List.query.filter_by(user_id=self.id).delete()
Annotation.query.filter_by(user_id=self.id).delete()
c_user_id = user_items.columns['user_id']
q = user_items.delete().where(c_user_id.is_(self.id))
state.db.session.execute(q)
@ -197,6 +199,7 @@ class User(db.Model):
state.peers[self.id].remove()
del state.peers[self.id]
def update_name(self):
if self.id == settings.USER_ID:
name = settings.preferences.get('username', 'anonymous')

12
static/js/annotation.js
View File

@ -46,14 +46,22 @@ oml.ui.annotation = function(annotation, $iframe) {
note.value = data.value
note.modified = (new Date).toISOString()
} else {
annotation.notes.push({
annotation.notes.push(note = {
created: data.created || (new Date).toISOString(),
modified: (new Date).toISOString(),
id: data.id,
user: '',
value: data.value
})
}
oml.api.editNote({
item: oml.user.ui.item,
annotation: annotation.id,
notes: {
created: note.created,
modified: note.modified,
value: note.value
}
})
that.triggerEvent('change')
}
});

17
static/js/annotationPanel.js
View File

@ -1,6 +1,7 @@
'use strict';
oml.ui.annotationPanel = function() {
var ui = oml.user.ui;
var ui = oml.user.ui;
@ -28,6 +29,7 @@ oml.ui.annotationPanel = function() {
click: function() {
var $annotation = oml.$ui.annotationFolder.find('.OMLAnnotation.selected')
$annotation.length && $annotation.delete()
$deleteQuote.options({disabled: true})
}
}).appendTo($bar);
@ -80,6 +82,21 @@ oml.ui.annotationPanel = function() {
}, function(result) {
oml.ui.exportAnnotationsDialog(result.data).open()
})
} else {
console.log('click', id, data)
}
},
change: function(data) {
var id = data.id;
console.log('change', data)
if (id == 'show') {
console.log('show', data)
oml.UI.set({annotationsShow: data.checked[0].id});
} else if (id == 'sort') {
console.log('sort', data)
oml.UI.set({annotationsSort: data.checked[0].id});
} else {
console.log('change', id, data)
}
}
}).appendTo($bar);

3
static/js/exportAnnotationsDialog.js
View File

@ -47,7 +47,8 @@ oml.ui.exportAnnotationsDialog = function(data) {
var annotations = oml.$ui.viewer.getAnnotations()
var text = 'Annotations for ' + data.title + ' (' + data.author.join(', ') + ')\n\n\n\n'
text += annotations.map(function(annotation) {
var text = 'Quote:\n\n' + annotation.text
var page = annotation.pageLabel || annotation.page
var text = 'Quote' + (page ? ' Page ' + page : '' )+ ':\n\n' + annotation.text
if (annotation.notes.length) {
text += '\n\nNotes:\n' + annotation.notes.map(function(note) {
return note.value

138
static/js/viewer.js
View File

@ -15,6 +15,12 @@ oml.ui.viewer = function() {
},
oml_showannotations: function() {
panel.toggleElement(1);
},
oml_sortannotations: function(data) {
that.renderAnnotations()
},
oml_annotationusers: function(data) {
that.renderAnnotations()
}
}),
panel = Ox.SplitPanel({
@ -40,38 +46,81 @@ oml.ui.viewer = function() {
$iframe, item;
function loadAnnotations(callback) {
annotations = JSON.parse(localStorage[item + '.annotations'] || '[]')
annotations.forEach(function(data) {
if (data.comments && !data.notes) {
data.notes = data.comments
delete data.comments
}
data.notes = data.notes || [];
})
callback && callback(annotations)
}
function saveAnnotations(data) {
if (data) {
data.created = data.created || (new Date).toISOString();
if (data.comments && !data.notes) {
data.notes = data.comments
delete data.comments
}
data.notes = data.notes || [];
annotations.push(data);
if (localStorage[item + '.annotations']) {
annotations = JSON.parse(localStorage[item + '.annotations'] || '[]')
var ids = []
annotations.forEach(function(data) {
if (data.comments && !data.notes) {
data.notes = data.comments
delete data.comments
}
if (!Ox.contains(ids, data.id)) {
ids.push(data.id)
var note
if (data.notes && data.notes.length) {
note = data.notes[0]
delete data.notes
}
addAnnotation(data, false)
if (note) {
data.notes = [note]
} else {
data.notes = []
}
if (data.notes.length) {
var note = data.notes[0]
oml.api.editNote({
item: ui.item,
annotation: data.id,
notes: {
created: note.created,
modified: note.modified,
value: note.value
}
})
}
} else {
console.log('ignore second time', data)
}
})
localStorage[oml.user.ui.item + '.annotations_'] = localStorage[oml.user.ui.item + '.annotations']
delete localStorage[oml.user.ui.item + '.annotations']
callback && callback(annotations)
} else {
oml.api.getAnnotations({
id: ui.item
}, function(result) {
console.log(result)
annotations = result.data.annotations
callback && callback(annotations)
})
}
localStorage[item + '.annotations'] = JSON.stringify(annotations)
}
function addAnnotation(data, save) {
var a = Ox.extend({}, data)
a.created = a.created || (new Date).toISOString();
a.item = ui.item
if (save !== false) {
oml.api.addAnnotation(a)
}
data.notes = data.notes || [];
annotations.push(data);
}
function removeAnnotation(id) {
annotations = annotations.filter(function(annotation) {
return annotation.id != id
oml.api.removeAnnotation({
item: ui.item,
annotation: id
}, function(result) {
annotations = annotations.filter(function(annotation) {
return annotation.id != id
})
})
saveAnnotations()
}
var annotationEvents = {
change: function() {
saveAnnotations()
change: function(data) {
// console.log('change', data)
},
'delete': function(data) {
oml.$ui.annotationFolder.find('#a-' + data.id).remove()
@ -95,14 +144,14 @@ oml.ui.viewer = function() {
}).onMessage(function(data, event) {
console.log('got', event, data)
if (event == 'addAnnotation') {
saveAnnotations(data);
addAnnotation(data);
var $annotation = oml.ui.annotation(data, $iframe).bindEvent(annotationEvents)
oml.$ui.annotationFolder.append($annotation);
$annotation.annotate();
oml.$ui.annotationPanel.updateSelection(false)
} else if (event == 'removeAnnotation') {
oml.$ui.annotationFolder.find('#a-' + data.id).remove()
removeAnnotation(data.id)
data.id && removeAnnotation(data.id)
} else if (event == 'selectAnnotation') {
if (data.id) {
var $annotation = oml.$ui.annotationFolder.find('#a-' + data.id)
@ -115,18 +164,13 @@ oml.ui.viewer = function() {
} else if (event == 'selectText') {
oml.$ui.annotationPanel.updateSelection(data)
} else {
console.log('trigger unknwon event', event, data)
that.triggerEvent(event, data);
}
}).bindEvent({
init: function() {
loadAnnotations(function(annotations) {
that.renderAnnotations()
// fixme: trigger loaded event from reader instead?
setTimeout(function() {
that.postMessage('addAnnotations', {
annotations: annotations
})
}, 500)
})
}
}).appendTo(frame);
@ -145,12 +189,34 @@ oml.ui.viewer = function() {
return annotations;
}
that.renderAnnotations = function() {
var sortKey = ui.sortAnnotations
if (sortKey == 'date') {
sortKey = 'created'
}
if (sortKey == 'date') {
sortKey = 'created'
}
if (sortKey == 'quote') {
sortKey = 'text'
}
annotations = Ox.sortBy(annotations, sortKey)
oml.$ui.annotationFolder.empty();
Ox.sortBy(annotations, ui.sortAnnotations);
var visibleAnnotations = [];
annotations.forEach(function(data) {
var $annotation = oml.ui.annotation(data, $iframe).bindEvent(annotationEvents)
oml.$ui.annotationFolder.append($annotation);
//that.postMessage('removeAnnotation', {id: data.id})
if (ui.showAnnotationUsers == 'all' || data.user == oml.user.id) {
var $annotation = oml.ui.annotation(data, $iframe).bindEvent(annotationEvents)
oml.$ui.annotationFolder.append($annotation);
visibleAnnotations.push(data)
}
})
// fixme: trigger loaded event from reader instead?
setTimeout(function() {
that.postMessage('addAnnotations', {
annotations: visibleAnnotations,
replace: true
})
}, 500)
}
return that.updateElement();
};

6
static/reader/epub.js
View File

@ -22,6 +22,12 @@ Ox.load({
} else if (event == 'addAnnotation') {
createAnnotation()
} else if (event == 'addAnnotations') {
if (data.replace) {
annotations.forEach(function(a) {
reader.rendition.annotations.remove(a.cfiRange)
})
annotations = []
}
data.annotations.forEach(function(annotation) {
annotations.push(annotation)
renderAnnotation(annotation)

14
static/reader/pdf.js
View File

@ -8,7 +8,6 @@ Ox.load({
}
}, function() {
Ox.$parent.bindMessage(function(data, event) {
console.log('got', event, 'data', data)
if (event == 'selectAnnotation') {
var annotation = annotations.filter(function(a) { return a.id == data.id })[0]
var delay = 0
@ -20,7 +19,7 @@ Ox.load({
PDFViewerApplication.pdfViewer.currentPageNumber = annotation.page;
delay = 250
}
setTimeout(function() {
annotation && setTimeout(function() {
var el = document.querySelector('.a' + annotation.id);
if (el && !isInView(el)) {
document.querySelector('#viewerContainer').scrollTop = el.offsetTop + el.parentElement.offsetTop - 64;
@ -30,12 +29,20 @@ Ox.load({
} else if (event == 'addAnnotation') {
createAnnotation()
} else if (event == 'addAnnotations') {
if (data.replace) {
document.querySelectorAll('.oml-annotation').forEach(function(a) {
a.remove()
})
annotations = []
}
data.annotations.forEach(function(annotation) {
annotations.push(annotation)
renderAnnotation(annotation)
})
} else if (event == 'removeAnnotation') {
removeAnnotation(data.id)
} else {
console.log('got', event, 'data', data)
}
})
})
@ -96,6 +103,7 @@ function getHighlight() {
var position = [pageNumber].concat(Ox.sort(selected.map(function(c) { return [c[1], c[0]]}))[0]);
return {
page: pageNumber,
pageLabel: PDFViewerApplication.pdfViewer.currentPageLabel,
position: position,
coords: selected,
text: text,
@ -176,7 +184,7 @@ function deselectAllAnnotations() {
g.classList.remove('selected')
g.style.backgroundColor = 'yellow'
var id = $(g).parents('.oml-annotation').data('id')
console.log('deselect', g, id)
//console.log('deselect', g, id)
if (!Ox.contains(ids, id)) {
ids.push(id)
Ox.$parent.postMessage('selectAnnotation', {id: null})