openmedialibrary/oml/changelog.py

458 lines
16 KiB
Python
Raw Normal View History

2014-05-04 17:26:43 +00:00
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
2014-09-02 22:32:44 +00:00
2014-05-04 17:26:43 +00:00
from datetime import datetime
2014-08-12 08:16:57 +00:00
import json
2014-05-04 17:26:43 +00:00
2014-08-12 08:16:57 +00:00
import sqlalchemy as sa
2014-05-04 17:26:43 +00:00
from utils import datetime2ts, ts2datetime
2014-08-12 08:16:57 +00:00
from websocket import trigger_event
2014-08-09 15:03:16 +00:00
import db
2014-08-12 08:16:57 +00:00
import settings
2014-05-04 17:26:43 +00:00
import state
2014-08-12 08:16:57 +00:00
import logging
2015-11-29 14:56:25 +00:00
logger = logging.getLogger(__name__)
2014-05-17 14:26:59 +00:00
2014-05-04 17:26:43 +00:00
class Changelog(db.Model):
'''
additem itemid metadata from file (info) + OLID
edititem itemid name->id (i.e. olid-> OL...M)
removeitem itemid
addlist name
editlist name {name: newname}
orderlists [name, name, name]
removelist name
2014-05-12 12:57:47 +00:00
addlistitems listname [ids]
removelistitems listname [ids]
2014-05-04 17:26:43 +00:00
editusername username
editcontact string
addpeer peerid peername
removepeer peerid peername
2014-05-21 00:02:21 +00:00
editmeta key, value data (i.e. 'isbn', '0000000000', {title: 'Example'})
resetmeta key, value
2014-05-04 17:26:43 +00:00
'''
2014-08-09 15:03:16 +00:00
__tablename__ = 'changelog'
id = sa.Column(sa.Integer(), primary_key=True)
2014-05-04 17:26:43 +00:00
2014-08-09 15:03:16 +00:00
created = sa.Column(sa.DateTime())
timestamp = sa.Column(sa.BigInteger())
2014-05-04 17:26:43 +00:00
2014-08-09 15:03:16 +00:00
user_id = sa.Column(sa.String(43))
revision = sa.Column(sa.BigInteger())
data = sa.Column(sa.Text())
sig = sa.Column(sa.String(96))
2014-05-04 17:26:43 +00:00
@classmethod
def record(cls, user, action, *args):
c = cls()
2014-05-20 00:43:54 +00:00
c.created = datetime.utcnow()
2014-05-20 17:03:31 +00:00
c.timestamp = datetime2ts(c.created)
2014-05-04 17:26:43 +00:00
c.user_id = user.id
c.revision = cls.query.filter_by(user_id=user.id).count()
2014-10-31 11:45:54 +00:00
c.data = json.dumps([action] + list(args), ensure_ascii=False)
2014-05-20 00:43:54 +00:00
_data = str(c.revision) + str(c.timestamp) + c.data
2014-09-09 10:08:04 +00:00
_data = _data.encode()
2014-08-09 16:14:14 +00:00
state.db.session.add(c)
state.db.session.commit()
2015-11-30 23:26:35 +00:00
#if state.nodes:
# state.nodes.queue('peered', 'pushChanges', [c.json()])
logger.debug('record change: %s', c.json())
2014-05-04 17:26:43 +00:00
2016-01-08 17:24:46 +00:00
@classmethod
def record_ts(cls, user, ts, action, *args):
# fixme remove
c = cls()
c.created = datetime.utcnow()
2016-01-11 18:52:27 +00:00
if ts:
c.timestamp = datetime2ts(ts)
else:
c.timestamp = datetime2ts(c.created)
2016-01-08 17:24:46 +00:00
c.user_id = user.id
c.revision = cls.query.filter_by(user_id=user.id).count()
c.data = json.dumps([action] + list(args), ensure_ascii=False)
_data = str(c.revision) + str(c.timestamp) + c.data
_data = _data.encode()
state.db.session.add(c)
state.db.session.commit()
#if state.nodes:
# state.nodes.queue('peered', 'pushChanges', [c.json()])
logger.debug('record change: %s', c.json())
2016-01-08 17:24:46 +00:00
2014-05-16 14:30:16 +00:00
@classmethod
def apply_changes(cls, user, changes):
trigger = changes
2014-05-16 14:30:16 +00:00
for change in changes:
if not cls.apply_change(user, change, trigger=False):
logger.debug('FAIL %s', change)
trigger = False
break
return False
if trigger:
2014-05-17 00:14:15 +00:00
trigger_event('change', {});
2014-05-16 14:30:16 +00:00
return True
2014-05-04 17:26:43 +00:00
@classmethod
def apply_change(cls, user, change, trigger=True):
revision, timestamp, data = change
last = cls.query.filter_by(user_id=user.id).order_by('-revision').first()
2014-05-04 17:26:43 +00:00
next_revision = last.revision + 1 if last else 0
2015-12-01 14:15:18 +00:00
if revision >= next_revision:
c = cls()
c.created = datetime.utcnow()
c.timestamp = timestamp
c.user_id = user.id
c.revision = revision
c.data = data
args = json.loads(data)
2016-01-13 09:58:06 +00:00
logger.debug('apply change from %s: %s(%s)', user.name, args[0], args[1:])
if getattr(c, 'action_' + args[0])(user, timestamp, *args[1:]):
logger.debug('change applied')
state.db.session.add(c)
state.db.session.commit()
if trigger:
trigger_event('change', {});
return True
else:
logger.debug('could not apply change')
2014-05-04 17:26:43 +00:00
else:
2014-05-17 14:26:59 +00:00
logger.debug('revsion does not match! got %s expecting %s', revision, next_revision)
2014-05-04 17:26:43 +00:00
return False
def __repr__(self):
return self.data
def json(self):
2014-05-20 17:03:31 +00:00
timestamp = self.timestamp or datetime2ts(self.created)
return [self.revision, timestamp, self.data]
2014-05-04 17:26:43 +00:00
@classmethod
def restore(cls, user_id, path=None):
from user.models import User
user = User.get_or_create(user_id)
if not path:
path = '/tmp/oml_changelog_%s.json' % user_id
with open(path, 'r') as fd:
for change in fd:
change = json.loads(change)
cls.apply_change(user, change, user_id == settings.USER_ID)
@classmethod
def export(cls, user_id, path=None):
if not path:
path = '/tmp/oml_changelog_%s.json' % user_id
with open(path, 'w') as fd:
for c in cls.query.filter_by(user_id=user_id).order_by('revision'):
2014-10-31 11:45:54 +00:00
fd.write(json.dumps(c.json(), ensure_ascii=False) + '\n')
2014-05-04 17:26:43 +00:00
def action_additem(self, user, timestamp, itemid, info):
from item.models import Item
i = Item.get(itemid)
if i:
if user not in i.users:
i.add_user(user)
i.update()
else:
2014-05-04 17:26:43 +00:00
i = Item.get_or_create(itemid, info)
2014-05-20 17:03:31 +00:00
i.modified = ts2datetime(timestamp)
if user not in i.users:
i.add_user(user)
2016-01-13 09:58:06 +00:00
i.info['_from'] = user.id
i.update()
2014-05-04 17:26:43 +00:00
return True
def action_edititem(self, user, timestamp, itemid, meta):
from user.models import Metadata
m = Metadata.get_or_create(user.id, itemid)
m.edit(meta)
2016-01-13 09:58:06 +00:00
#FIXME: "sometimes" update item too...
2014-05-04 17:26:43 +00:00
from item.models import Item
i = Item.get(itemid)
2016-01-13 09:58:06 +00:00
if i:
update = False
if len(i.users) == 1 and user in i.users:
update = True
if i.info.get('_from') == user.id:
update = True
if update:
i.edit(meta, ts2datetime(timestamp))
2014-05-04 17:26:43 +00:00
return True
def action_removeitem(self, user, timestamp, itemid):
from item.models import Item
from user.models import Metadata
2014-05-04 17:26:43 +00:00
i = Item.get(itemid)
if i:
if user in i.users:
i.users.remove(user)
if i.users:
i.update()
else:
i.delete()
2016-01-12 05:44:16 +00:00
Metadata.query.filter_by(user_id=user.id, item_id=itemid).delete()
2014-05-04 17:26:43 +00:00
return True
def action_addlist(self, user, timestamp, name, query=None):
from user.models import List
2016-01-07 11:09:52 +00:00
if name == '':
return True
2014-05-04 17:26:43 +00:00
l = List.create(user.id, name)
2016-01-13 16:16:31 +00:00
trigger_event('addlist', {'id': l.public_id, 'user': user.id})
2014-05-04 17:26:43 +00:00
return True
def action_editlist(self, user, timestamp, name, new):
from user.models import List
l = List.get_or_create(user.id, name)
if 'name' in new:
l.name = new['name']
l.save()
2016-01-13 16:16:31 +00:00
trigger_event('editlist', {'id': l.public_id, 'user': user.id})
2014-05-04 17:26:43 +00:00
return True
def action_orderlists(self, user, timestamp, lists):
from user.models import List
2014-05-25 18:06:12 +00:00
idx = 0
2014-05-04 17:26:43 +00:00
for name in lists:
l = List.get_or_create(user.id, name)
2014-05-25 18:06:12 +00:00
l.index_ = idx
2014-05-04 17:26:43 +00:00
l.save()
2014-05-25 18:06:12 +00:00
idx += 1
2016-01-13 16:16:31 +00:00
trigger_event('orderlists', {'user': user.id})
2014-05-04 17:26:43 +00:00
return True
def action_removelist(self, user, timestamp, name):
from user.models import List
l = List.get(user.id, name)
if l:
l.remove()
2016-01-13 09:58:06 +00:00
trigger_event('removelist', {'id': l.public_id})
2014-05-04 17:26:43 +00:00
return True
2014-05-12 12:57:47 +00:00
def action_addlistitems(self, user, timestamp, name, ids):
2014-05-04 17:26:43 +00:00
from user.models import List
2014-05-12 12:57:47 +00:00
l = List.get_or_create(user.id, name)
l.add_items(ids)
2014-05-04 17:26:43 +00:00
return True
2014-05-12 23:43:27 +00:00
def action_removelistitems(self, user, timestamp, name, ids):
2014-05-04 17:26:43 +00:00
from user.models import List
l = List.get(user.id, name)
2014-05-12 12:57:47 +00:00
if l:
l.remove_items(ids)
2014-05-04 17:26:43 +00:00
return True
def action_editusername(self, user, timestamp, username):
2014-05-25 18:06:12 +00:00
from user.models import List
old = user.nickname
2014-05-04 17:26:43 +00:00
user.info['username'] = username
2014-05-25 12:16:04 +00:00
user.update_name()
2014-05-25 18:06:12 +00:00
if old != user.nickname:
List.rename_user(old, user.nickname)
2014-05-04 17:26:43 +00:00
user.save()
return True
def action_editcontact(self, user, timestamp, contact):
user.info['contact'] = contact
user.save()
return True
2014-05-12 23:43:27 +00:00
def action_addpeer(self, user, timestamp, peerid, username):
2015-11-29 14:56:25 +00:00
if len(peerid) == 16:
from user.models import User
if not 'users' in user.info:
user.info['users'] = {}
user.info['users'][peerid] = username
user.save()
peer = User.get_or_create(peerid)
if not 'username' in peer.info:
peer.info['username'] = username
peer.update_name()
peer.save()
2014-05-04 17:26:43 +00:00
return True
2014-05-12 23:43:27 +00:00
def action_removepeer(self, user, timestamp, peerid):
2014-05-04 17:26:43 +00:00
if 'users' in user.info and peerid in user.info['users']:
del user.info['users'][peerid]
user.save()
#fixme, remove from User table if no other connection exists
return True
2014-05-21 00:02:21 +00:00
def action_editmeta(self, user, timestamp, key, value, data):
return True
'''>> Metadata no longer tracked per isbn'''
2014-05-21 00:02:21 +00:00
from item.models import Metadata
m = Metadata.get(key, value)
if not m or m.timestamp < timestamp:
if not m:
m = Metadata.get_or_create(key, value)
if m.edit(data, False):
2014-05-21 00:02:21 +00:00
m.update_items()
return True
def action_resetmeta(self, user, timestamp, key, value):
return True
2014-05-21 00:02:21 +00:00
from item.models import Metadata
m = Metadata.get(key, value)
if m and m.timestamp < timestamp:
m.reset()
return True
2016-01-13 09:58:06 +00:00
@classmethod
def aggregated_changes(cls, since=None, user_id=None):
2016-01-13 15:19:48 +00:00
from item.models import Item
2016-01-13 09:58:06 +00:00
from user.models import List
if not user_id:
user_id = settings.USER_ID
qs = cls.query.filter_by(user_id=user_id)
qs = Changelog.query.filter_by(user_id=user_id)
if since:
qs = qs.filter(Changelog.revision>=since)
changes = {}
orderlists = False
for c in qs.order_by('timestamp'):
revision = c.revision
timestamp = c.timestamp
data = json.loads(c.data)
op = data[0]
if op in ('editmeta', 'resetmeta'):
continue
action = changes.setdefault(op, {})
if op == 'additem':
item_id = data[1]
info = data[2]
action[item_id] = [revision, timestamp, info]
if item_id in changes.get('removeitem', []):
del changes['removeitem'][item_id]
2016-01-13 15:19:48 +00:00
i = Item.get(item_id)
if i:
changes.setdefault('edititem', {})[item_id] = [revision+1, timestamp, i.meta]
2016-01-13 09:58:06 +00:00
elif op == 'edititem':
item_id = data[1]
meta = data[2]
if not item_id in action:
action[item_id] = [revision, timestamp, meta]
else:
action[item_id][0] = revision
action[item_id][1] = timestamp
action[item_id][2].update(meta)
elif op == 'removeitem':
item_id = data[1]
if item_id in changes.get('additem', []):
del changes['additem'][item_id]
else:
action[item_id] = [revision, timestamp]
if item_id in changes.get('edititem', []):
del changes['edititem'][item_id]
elif op == 'addlist':
list_id = data[1]
if list_id:
ids = data[2] if len(data) > 2 else []
action[list_id] = [revision, timestamp, ids]
elif op == 'editlist':
old_id = data[1]
new_id = data[2]['name']
r = revision
if old_id not in changes.get('addlist', []):
2016-01-13 16:16:31 +00:00
action[old_id] = [revision, timestamp, {'name': new_id}]
2016-01-13 09:58:06 +00:00
r += 1
for a in ('addlist', 'addlistitems', 'removelistitems'):
if a in changes and old_id in changes[a]:
changes[a][new_id] = changes[a].pop(old_id)
changes[a][new_id][0] = r
elif op == 'orderlists':
orderlists = True
elif op == 'removelist':
list_id = data[1]
if list_id not in changes.get('addlist', []):
action[list_id] = [revision, timestamp]
for a in ('addlist', 'addlistitems', 'removelistitems'):
if a in changes and list_id in changes[a]:
del changes[a][list_id]
elif op == 'addlistitems':
list_id = data[1]
if not list_id:
continue
listitems = data[2]
if list_id not in action:
action[list_id] = [revision, timestamp, []]
action[list_id][0] = revision
action[list_id][1] = timestamp
action[list_id][2] += listitems
#remove from removelistitems!
if list_id in changes.get('remvelistitems', {}):
changes['remvelistitems'][list_id] = [
i for i in changes['remvelistitems'][list_id] if i not in listitems
]
elif op == 'removelistitems':
list_id = data[1]
listitems = data[2]
#remove from additemlists
removed = []
if list_id in changes.get('addlistitems',{}):
removed = [
i for i in changes['addlistitems'][list_id] if i in listitems
]
changes['addlistitems'][list_id] = [
i for i in changes['addlistitems'][list_id] if i not in listitems
]
#remove remaining items
listitems = [
i for i in listitems if i not in removed
]
if listitems:
action[list_id] = [revision, timestamp, listitems]
elif op == 'editusername':
old_name = data[1]
new_name = data[2]
#fixme merge multiple edits
action[old_name] = [revision, timestamp, new_name]
elif op == 'editcontact':
old_contact = data[1]
new_contact = data[2]
#fixme merge multiple edits
action[old_contact] = [revision, timestamp, new_contact]
pass
elif op == 'addpeer':
peer = data[1]
2016-01-13 15:19:48 +00:00
username = data[2]
action[peer] = [revision, timestamp, username]
2016-01-13 09:58:06 +00:00
if peer in changes.get('removepeer', []):
del changes['removepeer'][peer]
elif op == 'removepeer':
peer = data[1]
if peer in changes.get('addpeer', []):
del changes['addpeer'][peer]
else:
action[peer] = [revision, timestamp, []]
elif op == 'editmeta':
pass
elif op == 'resetmeta':
pass
else:
print('unknonw action', data)
_changes = []
for op in list(changes):
if not changes[op]:
del changes[op]
else:
for id in changes[op]:
data = changes[op][id]
rv = data[0]
ts = data[1]
data = [op, id] + data[2:]
_changes.append([rv, ts, json.dumps(data)])
_changes.sort(key=lambda change: (change[0], change[1]))
if orderlists:
ids = [l.name for l in List.query.filter_by(user_id=user_id,type='static').order_by('index_') if l.name]
if len(ids) > 1:
2016-01-13 16:16:31 +00:00
_changes.append([-1, timestamp, json.dumps(['orderlists', ids])])
2016-01-13 09:58:06 +00:00
if _changes:
2016-01-13 16:16:31 +00:00
r = revision
for c in reversed(_changes):
2016-01-13 09:58:06 +00:00
c[0] = r
2016-01-13 16:16:31 +00:00
r -= 1
2016-01-13 09:58:06 +00:00
return _changes