0x2620/openmedialibrary
1
0
Fork 0
Code Issues 131 Pull requests Releases Wiki Activity

store annotations in db and sync with peers

Browse source
This commit is contained in:
j 2019-02-10 17:46:35 +05:30
commit e0cba14d6a
21 changed files with 385 additions and 63 deletions
Download patch file Download diff file Expand all files Collapse all files

48
oml/annotation/api.py Normal file
View file

@ -0,0 +1,48 @@
# -*- coding: utf-8 -*-
from oxtornado import actions
from . import models
import settings
import state
from changelog import add_record
import logging
logger = logging.getLogger(__name__)
def getAnnotations(data):
response = {}
response['annotations'] = models.Annotation.get_by_item(data['id'])
return response
actions.register(getAnnotations)
def addAnnotation(data):
item_id = data.pop('item')
a = models.Annotation.create(item_id, settings.USER_ID, data)
a.add_record('addannotation')
response = a.json()
return response
actions.register(addAnnotation)
def editNote(data):
a = models.Annotation.get(state.user(), data['item'], data['annotation'])
if a:
a.data['notes'] = data['notes']
a.add_record('editannotation')
a.save()
response = a.json()
else:
response = {}
return response
actions.register(editNote)
def removeAnnotation(data):
a = models.Annotation.get(state.user(), data['item'], data['annotation'])
if a:
a.add_record('removeannotation')
a.delete()
response = {}
return response
actions.register(removeAnnotation)

68
oml/annotation/models.py
View file

@ -4,6 +4,7 @@ import json
import logging
import os
import shutil
import unicodedata
from sqlalchemy.orm import load_only
import ox
@ -31,10 +32,15 @@ class Annotation(db.Model):
created = sa.Column(sa.DateTime())
modified = sa.Column(sa.DateTime())
user_id = sa.Column(sa.String(43), sa.ForeignKey('user.id'))
user = sa.orm.relationship('User', backref=sa.orm.backref('annotations', lazy='dynamic'))
item_id = sa.Column(sa.String(43), sa.ForeignKey('item.id'))
item = sa.orm.relationship('Item', backref=sa.orm.backref('items', lazy='dynamic'))
data = sa.Column(MutableDict.as_mutable(sa.PickleType(pickler=json_pickler)))
findquotes = sa.Column(sa.Text(), index=True)
findnotes = sa.Column(sa.Text(), index=True)
def __init__(self, item_id, user_id, data):
self.created = datetime.utcnow()
self.modified = datetime.utcnow()
@ -43,28 +49,44 @@ class Annotation(db.Model):
self.data = data
@classmethod
def create(cls, **kwargs):
a = cls(**kwargs)
state.db.session.add(a)
def create(cls, item_id, user_id, data):
a = cls(item_id, user_id, data)
a.save()
return a
def delete(self):
state.db.session.delete(self)
state.db.session.commit()
@classmethod
def get(cls, user, item_id, annotation_id):
for a in cls.query.filter_by(item_id=item_id, user=user, _id=annotation_id):
if a.data.get('id') == annotation_id:
return a
if isinstance(user, str):
qs = cls.query.filter_by(item_id=item_id, user_id=user, id=annotation_id)
else:
qs = cls.query.filter_by(item_id=item_id, user=user, id=annotation_id)
for a in qs:
return a
@classmethod
def get_by_item(cls, user, item_id):
def get_by_item(cls, item_id):
annotations = []
for a in cls.query.filter_by(item_id=item_id):
annotations.append(a.json())
return annotations
def save(self):
_id = self.data.get('id')
if _id:
self._id = _id
id = self.data.get('id')
if id:
self.id = id
self.findquotes = unicodedata.normalize('NFKD', self.data.get('text', '')).lower()
note = self.data.get('notes', {})
if isinstance(note, list) and note:
note = note[0]
if isinstance(note, dict):
note = note.get('value', '')
else:
note = ''
self.findnotes = unicodedata.normalize('NFKD', note).lower()
state.db.session.add(self)
state.db.session.commit()
@ -73,6 +95,30 @@ class Annotation(db.Model):
data['created'] = self.created
data['modified'] = self.modified
data['user'] = self.user_id
data['_id'] = ox.toAZ(self.id)
data['_id'] = ox.toAZ(self._id)
if isinstance(data.get('notes'), dict):
note = data['notes']
if self.user_id != settings.USER_ID:
note['user'] = self.user_id
if not note.get('id'):
note['id'] = 'A'
data['notes'] = [data['notes']]
if 'notes' not in data:
data['notes'] = []
return data
def add_record(self, action):
args = [self.item_id]
if action == 'addannotation':
args.append(self.data)
elif action == 'editannotation':
args.append(self.id)
args.append({
'notes': self.data['notes']
})
elif action == 'removeannotation':
args.append(self.id)
else:
raise Exception('unknown action')
add_record(action, *args)

1
oml/api.py
View file

@ -12,6 +12,7 @@ from oxtornado import actions
import item.api
import user.api
import annotation.api
import update
import utils

27
oml/changelog.py
View file

@ -1,8 +1,8 @@
# -*- coding: utf-8 -*-
import os
from datetime import datetime
import json
import os
import sqlalchemy as sa
from sqlalchemy.sql.expression import text
@ -76,6 +76,10 @@ class Changelog(db.Model):
addpeer peerid peername
removepeer peerid peername
editpeer peerid {username: string, contact: string}
addannotation itemid data
editannotation itemid annotationid data
removeannotation itemid annotationid
'''
__tablename__ = 'changelog'
id = sa.Column(sa.Integer(), primary_key=True)
@ -327,6 +331,27 @@ class Changelog(db.Model):
peer.save()
return True
def action_addannotation(self, user, timestamp, itemid, data):
from annotation.models import Annotation
Annotation.create(item_id=itemid, user_id=user.id, data=data)
return True
def action_editannotation(self, user, timestamp, itemid, annotationid, data):
from annotation.models import Annotation
a = Annotation.get(user, itemid, annotationid)
if a:
for key in data:
a.data[key] = data[key]
a.save()
return True
def action_removeannotation(self, user, timestamp, itemid, annotationid):
from annotation.models import Annotation
a = Annotation.get(user, itemid, annotationid)
if a:
a.delete()
return True
@classmethod
def aggregated_changes(cls, since=None, user_id=None):
from item.models import Item

27
oml/library.py
View file

@ -190,6 +190,33 @@ class Peer(object):
self.info['username'] = args[0]
elif action == 'editcontact':
self.info['contact'] = args[0]
elif action == 'addannotation':
from annotation.models import Annotation
if len(args) == 2:
itemid, data = args
Annotation.create(item_id=itemid, user_id=self.id, data=data)
else:
logger.error('invalid entry %s %s', action, args)
elif action == 'editannotation':
from annotation.models import Annotation
if len(args) == 3:
itemid, annotationid, data = args
a = Annotation.get(self.id, itemid, annotationid)
if a:
for key in data:
a.data[key] = data[key]
a.save()
else:
logger.error('invalid entry %s %s', action, args)
elif action == 'removeannotation':
from annotation.models import Annotation
if len(args) == 2:
itemid, annotationid = args
a = Annotation.get(self.id, itemid, annotationid)
if a:
a.delete()
else:
logger.error('invalid entry %s %s', action, args)
else:
logger.debug('UNKNOWN ACTION:', action)
self.info['revision'] = revision

4
oml/localnodes.py
View file

@ -152,8 +152,8 @@ class LocalNodes(dict):
if state.tasks:
state.tasks.queue('removelocalinfo', id)
def get(self, user_id):
data = super().get(user_id)
def get_data(self, user_id):
data = self.get(user_id)
if data and can_connect(data):
return data
return None

7
oml/nodes.py
View file

@ -124,9 +124,12 @@ class Node(Thread):
self.local = None
self.port = 9851
def is_local(self):
return self._nodes and self.user_id in self._nodes.local
def get_local(self):
if self._nodes and self._nodes.local:
return self._nodes.local.get(self.user_id)
return self._nodes.local.get_data(self.user_id)
return None
def request(self, action, *args):
@ -216,7 +219,7 @@ class Node(Thread):
return False
def is_online(self):
return self.online or self.get_local() is not None
return self.online or self.is_local()
def send_response(self):
self._q.put('send_response')

13
oml/queryparser.py
View file

@ -116,7 +116,18 @@ class Parser(object):
elif k == 'fulltext':
ids = find_fulltext(v)
return self.in_ids(ids, exclude)
elif k in ('notes', 'quotes'):
from annotation.models import Annotation
if isinstance(v, str):
v = unicodedata.normalize('NFKD', v).lower()
ids = set()
if k == 'notes':
qs = Annotation.query.filter(get_operator('=')(Annotation.findnotes, v))
elif k == 'quotes':
qs = Annotation.query.filter(get_operator('=')(Annotation.findquotes, v))
for a in qs:
ids.add(a.item_id)
return self.in_ids(ids, exclude)
elif key_type in ("string", "text"):
if isinstance(v, str):
v = unicodedata.normalize('NFKD', v).lower()

2
oml/server.py
View file

@ -170,8 +170,8 @@ def run():
import bandwidth
state.bandwidth = bandwidth.Bandwidth()
state.tor = tor.Tor()
state.node = node.server.start()
state.nodes = nodes.Nodes()
state.node = node.server.start()
def publish():
if not state.tor.is_online():

4
oml/settings.py
View file

@ -82,7 +82,7 @@ if 'modules' in release and 'openmedialibrary' in release['modules']:
else:
MINOR_VERSION = 'git'
NODE_PROTOCOL = "0.8"
NODE_PROTOCOL = "0.9"
VERSION = "%s.%s" % (NODE_PROTOCOL, MINOR_VERSION)
USER_AGENT = 'OpenMediaLibrary/%s' % VERSION
@ -95,4 +95,4 @@ FULLTEXT_SUPPORT = fulltext.platform_supported()
if not FULLTEXT_SUPPORT:
config['itemKeys'] = [k for k in config['itemKeys'] if k['id'] != 'fulltext']
DB_VERSION = 17
DB_VERSION = 18

16
oml/setup.py
View file

@ -151,6 +151,22 @@ CREATE TABLE listitem (
FOREIGN KEY(item_id) REFERENCES item (id),
FOREIGN KEY(list_id) REFERENCES list (id)
);
CREATE TABLE annotation (
_id INTEGER NOT NULL,
id VARCHAR(43),
created DATETIME,
modified DATETIME,
user_id VARCHAR(43),
item_id VARCHAR(43),
data BLOB,
findquotes TEXT,
findnotes TEXT,
PRIMARY KEY (_id),
FOREIGN KEY(user_id) REFERENCES user (id),
FOREIGN KEY(item_id) REFERENCES item (id)
);
CREATE INDEX ix_annotation_findquotes ON annotation (findquotes);
CREATE INDEX ix_annotation_findnotes ON annotation (findnotes);
PRAGMA journal_mode=WAL
'''
for statement in sql.split(';'):

24
oml/update.py
View file

@ -377,6 +377,8 @@ class Update(Thread):
db_version = migrate_16()
if db_version < 17:
db_version = migrate_17()
if db_version < 18:
db_version = migrate_18()
settings.server['db_version'] = db_version
def run(self):
@ -674,3 +676,25 @@ def migrate_17():
lists.append(l.name)
add_record('orderlists', lists)
return 17
def migrate_18():
db.run_sql([
'''CREATE TABLE annotation (
_id INTEGER NOT NULL,
id VARCHAR(43),
created DATETIME,
modified DATETIME,
user_id VARCHAR(43),
item_id VARCHAR(43),
data BLOB,
findquotes TEXT,
findnotes TEXT,
PRIMARY KEY (_id),
FOREIGN KEY(user_id) REFERENCES user (id),
FOREIGN KEY(item_id) REFERENCES item (id)
)'''])
db.run_sql([
'CREATE INDEX ix_annotation_findquotes ON annotation (findquotes)',
'CREATE INDEX ix_annotation_findnotes ON annotation (findnotes)'
])
return 18

2
oml/user/api.py
View file

@ -474,7 +474,7 @@ def removePeering(data):
if len(data.get('id', '')) not in (16, 43):
logger.debug('invalid user id')
return {}
u = models.User.get(data['id'], for_udpate=True)
u = models.User.get(data['id'], for_update=True)
if u:
u.info['message'] = data.get('message', '')
u.update_peering(False)

3
oml/user/models.py
View file

@ -187,7 +187,9 @@ class User(db.Model):
def cleanup(self):
from item.models import user_items, Item
from annotation.models import Annotation
List.query.filter_by(user_id=self.id).delete()
Annotation.query.filter_by(user_id=self.id).delete()
c_user_id = user_items.columns['user_id']
q = user_items.delete().where(c_user_id.is_(self.id))
state.db.session.execute(q)
@ -197,6 +199,7 @@ class User(db.Model):
state.peers[self.id].remove()
del state.peers[self.id]
def update_name(self):
if self.id == settings.USER_ID:
name = settings.preferences.get('username', 'anonymous')