From c95e22869cf52121e2815870d12dcb6e588bf25b Mon Sep 17 00:00:00 2001 From: j Date: Sat, 2 Feb 2019 12:43:37 +0530 Subject: [PATCH 01/14] queue peering requests per node --- oml/localnodes.py | 6 ++++-- oml/nodes.py | 50 ++++++++++++++++++++++++++++++++++++---------- oml/queryparser.py | 6 +++++- oml/user/api.py | 10 +++++----- 4 files changed, 53 insertions(+), 19 deletions(-) diff --git a/oml/localnodes.py b/oml/localnodes.py index f3fa7ec..20562e7 100644 --- a/oml/localnodes.py +++ b/oml/localnodes.py @@ -153,5 +153,7 @@ class LocalNodes(dict): state.tasks.queue('removelocalinfo', id) def get(self, user_id): - if user_id in self and can_connect(self[user_id]): - return self[user_id] if user_id in self else None + data = super().get(user_id) + if data and can_connect(data): + return data + return None diff --git a/oml/nodes.py b/oml/nodes.py index e3621d9..a9b9096 100644 --- a/oml/nodes.py +++ b/oml/nodes.py @@ -37,10 +37,9 @@ class Node(Thread): _online = None TIMEOUT = 5 - def __init__(self, nodes, user): + def __init__(self, nodes, user_id): self._nodes = nodes - self.user = user - self.user_id = user.id + self.user_id = user_id self._opener = get_opener(self.user_id) self._q = Queue() Thread.__init__(self) @@ -57,6 +56,13 @@ class Node(Thread): self._send_response() elif action == 'ping': self.online = self.can_connect() + elif isinstance(action, list) and len(action) == 2: + if self.online: + self._call(action[0], *action[1]) + else: + if not self._q.qsize(): + time.sleep(5) + self.queue(action[0], *action[1]) else: logger.debug('unknown action %s', action) @@ -68,6 +74,14 @@ class Node(Thread): if state.online or self.get_local(): self._q.put('ping') + def queue(self, action, *args): + logger.debug('queue node action %s->%s%s', self.user_id, action, args) + self._q.put([action, args]) + + def _call(self, action, *args): + r = getattr(self, action)(*args) + logger.debug('call node api %s->%s%s = %s', self.user_id, action, args, r) + @property def url(self): if self.local: @@ -321,6 +335,7 @@ class Node(Thread): return False def peering(self, action): + pull_changes = False with db.session(): u = user.models.User.get_or_create(self.user_id) user_info = u.info @@ -335,6 +350,8 @@ class Node(Thread): if 'message' in u.info: del u.info['message'] u.save() + if action == 'acceptPeering': + pull_changes = True else: logger.debug('peering failed? %s %s', action, r) if action in ('cancelPeering', 'rejectPeering', 'removePeering'): @@ -342,6 +359,8 @@ class Node(Thread): with db.session(): u = user.models.User.get(self.user_id) trigger_event('peering.%s' % action.replace('Peering', ''), u.json()) + if pull_changes: + self.pullChanges() return True headers = { @@ -514,9 +533,15 @@ class Nodes(Thread): def queue(self, *args): if args: - logger.debug('queue "%s", %s entries in queue', args[0], self._q.qsize()) + logger.debug('queue "%s", %s entries in queue', args, self._q.qsize()) self._q.put(list(args)) + def peer_queue(self, peer, action, *args): + if peer not in self._nodes: + self._add(peer) + self._nodes[peer].queue(action, *args) + + def is_online(self, id): return id in self._nodes and self._nodes[id].is_online() @@ -536,18 +561,18 @@ class Nodes(Thread): elif target == 'online': nodes = [n for n in list(self._nodes.values()) if n.online] else: - if not target in self._nodes: + if target not in self._nodes: self._add(target) nodes = [self._nodes[target]] for node in nodes: - r = getattr(node, action)(*args) - logger.debug('call node api %s->%s%s = %s', node.user_id, action, args, r) + node._call(action, *args) def _add(self, user_id, send_response=False): if user_id not in self._nodes: from user.models import User with db.session(): - self._nodes[user_id] = Node(self, User.get_or_create(user_id)) + User.get_or_create(user_id) + self._nodes[user_id] = Node(self, user_id) else: self._nodes[user_id].ping() if send_response: @@ -608,8 +633,11 @@ def update_online(): def check_nodes(): if state.online: + ids = [] with db.session(): for u in user.models.User.query.filter_by(queued=True): - if not state.nodes.is_online(u.id): - logger.debug('queued peering message for %s trying to connect...', u.id) - state.nodes.queue('add', u.id, True) + ids.append(u.id) + for id in ids: + if not state.nodes.is_online(id): + logger.debug('queued peering message for %s trying to connect...', id) + state.nodes.queue('add', id, True) diff --git a/oml/queryparser.py b/oml/queryparser.py index 033e068..2810729 100644 --- a/oml/queryparser.py +++ b/oml/queryparser.py @@ -128,7 +128,11 @@ class Parser(object): elif k == 'list': nickname, name = v.split(':', 1) if nickname: - u = self._user.query.filter_by(nickname=nickname).one() + try: + u = self._user.query.filter_by(nickname=nickname).one() + except sqlalchemy.orm.exc.NoResultFound: + ids = [] + return self.in_ids(ids, exclude) else: u = self._user.query.filter_by(id=settings.USER_ID).one() if not name: diff --git a/oml/user/api.py b/oml/user/api.py index be190ad..f63b8a0 100644 --- a/oml/user/api.py +++ b/oml/user/api.py @@ -422,7 +422,7 @@ def requestPeering(data): u.info['nickname'] = data.get('nickname', '') u.update_name() u.save() - state.nodes.queue(u.id, 'peering', 'requestPeering') + state.nodes.peer_queue(u.id, 'peering', 'requestPeering') return {} actions.register(requestPeering, cache=False) @@ -441,7 +441,7 @@ def acceptPeering(data): u = models.User.get_or_create(data['id']) u.info['message'] = data.get('message', '') u.update_peering(True) - state.nodes.queue(u.id, 'peering', 'acceptPeering') + state.nodes.peer_queue(u.id, 'peering', 'acceptPeering') return {} actions.register(acceptPeering, cache=False) @@ -459,7 +459,7 @@ def rejectPeering(data): u = models.User.get_or_create(data['id']) u.info['message'] = data.get('message', '') u.update_peering(False) - state.nodes.queue(u.id, 'peering', 'rejectPeering') + state.nodes.peer_queue(u.id, 'peering', 'rejectPeering') return {} actions.register(rejectPeering, cache=False) @@ -477,7 +477,7 @@ def removePeering(data): u = models.User.get_or_create(data['id']) u.info['message'] = data.get('message', '') u.update_peering(False) - state.nodes.queue(u.id, 'peering', 'removePeering') + state.nodes.peer_queue(u.id, 'peering', 'removePeering') return {} actions.register(removePeering, cache=False) @@ -493,7 +493,7 @@ def cancelPeering(data): u = models.User.get_or_create(data['id']) u.info['message'] = data.get('message', '') u.update_peering(False) - state.nodes.queue(u.id, 'peering', 'cancelPeering') + state.nodes.peer_queue(u.id, 'peering', 'cancelPeering') return {} actions.register(cancelPeering, cache=False) From ad15b8a25a25ae3c7e3d5ae4927fed2c61561699 Mon Sep 17 00:00:00 2001 From: j Date: Sat, 2 Feb 2019 12:57:46 +0530 Subject: [PATCH 02/14] watch websocket errors --- oml/websocket.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/oml/websocket.py b/oml/websocket.py index 27a3d4e..12795ce 100644 --- a/oml/websocket.py +++ b/oml/websocket.py @@ -49,7 +49,11 @@ class Handler(WebSocketHandler): #websocket calls def on_message(self, message): - action, data = json.loads(message) + try: + action, data = json.loads(message) + except json.decoder.JSONDecodeError: + logger.debug('invalid websocket message: %s', message) + return if state.tasks: state.tasks.queue(action, data) @@ -62,7 +66,17 @@ class Handler(WebSocketHandler): if self.ws_connection is None: self.on_close() else: - state.main.add_callback(lambda: self.write_message(message)) + state.main.add_callback(lambda: self._write_message(message)) + + async def _write_message(self, message): + try: + task = self.write_message(message) + await task + except tornado.iostream.StreamClosedError as e: + self.on_close() + except tornado.websocket.WebSocketClosedError as e: + self.on_close() + def trigger_event(event, data): #if len(state.websockets): From 265a4791fe19297a18ce8da44e66f661f18e9e71 Mon Sep 17 00:00:00 2001 From: j Date: Sat, 2 Feb 2019 14:04:03 +0530 Subject: [PATCH 03/14] log updated --- oml/library.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/oml/library.py b/oml/library.py index 11f9103..274f00c 100644 --- a/oml/library.py +++ b/oml/library.py @@ -327,6 +327,8 @@ class Peer(object): if state.shutdown: break state.db.session.commit() + if update_items: + logger.debug('updated %s items', len(update_items)) ids = set(self.library.keys()) changed = False for name, l in self.info.get('lists', {}).items(): From 611fc2b3737f2d92d8e24846eed7339424f6e138 Mon Sep 17 00:00:00 2001 From: j Date: Sat, 2 Feb 2019 14:21:25 +0530 Subject: [PATCH 04/14] ping nodes to update online status --- oml/nodes.py | 73 +++++++++++++++++++++------------------------------- 1 file changed, 30 insertions(+), 43 deletions(-) diff --git a/oml/nodes.py b/oml/nodes.py index a9b9096..5ba54ee 100644 --- a/oml/nodes.py +++ b/oml/nodes.py @@ -1,16 +1,17 @@ # -*- coding: utf-8 -*- - +from io import BytesIO from queue import Queue from threading import Thread -import json -from io import BytesIO import gzip -import urllib.request, urllib.error, urllib.parse +import json import os -import time import socket import socks +import time +import urllib.error +import urllib.parse +import urllib.request import ox from tornado.ioloop import PeriodicCallback @@ -42,6 +43,8 @@ class Node(Thread): self.user_id = user_id self._opener = get_opener(self.user_id) self._q = Queue() + self._pingcb = PeriodicCallback(self.ping, 10 * settings.server['pull_interval']) + self._pingcb.start() Thread.__init__(self) self.daemon = True self.start() @@ -52,10 +55,15 @@ class Node(Thread): action = self._q.get() if state.shutdown: break - if action == 'send_response': - self._send_response() - elif action == 'ping': + if action == 'ping': self.online = self.can_connect() + elif action == 'send_response': + if self.online: + self._send_response() + else: + if not self._q.qsize(): + time.sleep(5) + self.send_response() elif isinstance(action, list) and len(action) == 2: if self.online: self._call(action[0], *action[1]) @@ -216,34 +224,26 @@ class Node(Thread): def _send_response(self): with db.session(): u = user.models.User.get(self.user_id) - send_response = u and u.peered or u.queued if u: user_pending = u.pending user_peered = u.peered user_queued = u.queued + else: + user_queued = False if DEBUG_NODES: logger.debug('go online peered=%s queued=%s %s (%s)', u.peered, u.queued, u.id, u.nickname) - if send_response: - try: - self.online = self.can_connect() - except: - if DEBUG_NODES: - logger.debug('failed to connect to %s', self.user_id) - self.online = False - if self.online: - if DEBUG_NODES: - logger.debug('connected to %s', self.url) - if user_queued: - if DEBUG_NODES: - logger.debug('queued peering event pending=%s peered=%s', user_pending, user_peered) - if user_pending == 'sent': - self.peering('requestPeering') - elif user_pending == '' and user_peered: - self.peering('acceptPeering') - else: - #fixme, what about cancel/reject peering here? - self.peering('removePeering') + if user_queued: + if DEBUG_NODES: + logger.debug('connected to %s', self.url) + logger.debug('queued peering event pending=%s peered=%s', user_pending, user_peered) + if user_pending == 'sent': + self.peering('requestPeering') + elif user_pending == '' and user_peered: + self.peering('acceptPeering') + else: + #fixme, what about cancel/reject peering here? + self.peering('removePeering') def trigger_status(self): if self.online is not None: @@ -600,7 +600,7 @@ class Nodes(Thread): if state.shutdown: break node = self._nodes.get(u['id']) - if node: + if node and node.is_online(): node.pullChanges() self._pulling = False @@ -614,8 +614,6 @@ class Nodes(Thread): def publish_node(): update_online() - state.check_nodes = PeriodicCallback(check_nodes, 120000) - state.check_nodes.start() state._online = PeriodicCallback(update_online, 60000) state._online.start() @@ -630,14 +628,3 @@ def update_online(): if state.online: for node in list(state.nodes._nodes.values()): node.trigger_status() - -def check_nodes(): - if state.online: - ids = [] - with db.session(): - for u in user.models.User.query.filter_by(queued=True): - ids.append(u.id) - for id in ids: - if not state.nodes.is_online(id): - logger.debug('queued peering message for %s trying to connect...', id) - state.nodes.queue('add', id, True) From a111aaac463926dcfc46821a0f3b64bb08db3711 Mon Sep 17 00:00:00 2001 From: j Date: Sat, 2 Feb 2019 14:29:05 +0530 Subject: [PATCH 05/14] unlink log on remove --- oml/library.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/oml/library.py b/oml/library.py index 274f00c..94e30e8 100644 --- a/oml/library.py +++ b/oml/library.py @@ -208,8 +208,9 @@ class Peer(object): def remove(self): self.join() - os.unlink(self._dbpath) - os.unlink(self._infopath) + for path in (self._dbpath, self._logpath, self._infopath): + if os.path.exists(path): + os.unlink(path) def sync_db(self): import item.models From c7e316ab22546d5535731027cdc3ad209b22ad8b Mon Sep 17 00:00:00 2001 From: j Date: Sat, 2 Feb 2019 14:29:12 +0530 Subject: [PATCH 06/14] select for update --- oml/user/api.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/oml/user/api.py b/oml/user/api.py index f63b8a0..b3aff56 100644 --- a/oml/user/api.py +++ b/oml/user/api.py @@ -474,10 +474,11 @@ def removePeering(data): if len(data.get('id', '')) not in (16, 43): logger.debug('invalid user id') return {} - u = models.User.get_or_create(data['id']) - u.info['message'] = data.get('message', '') - u.update_peering(False) - state.nodes.peer_queue(u.id, 'peering', 'removePeering') + u = models.User.get(data['id'], for_udpate=True) + if u: + u.info['message'] = data.get('message', '') + u.update_peering(False) + state.nodes.peer_queue(u.id, 'peering', 'removePeering') return {} actions.register(removePeering, cache=False) From de9a172f39ad2c6e0189807ee1c66e855ad8c3f0 Mon Sep 17 00:00:00 2001 From: j Date: Sat, 2 Feb 2019 14:42:26 +0530 Subject: [PATCH 07/14] start callbacks on main thread --- oml/nodes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/oml/nodes.py b/oml/nodes.py index 5ba54ee..fd7a6ea 100644 --- a/oml/nodes.py +++ b/oml/nodes.py @@ -44,7 +44,7 @@ class Node(Thread): self._opener = get_opener(self.user_id) self._q = Queue() self._pingcb = PeriodicCallback(self.ping, 10 * settings.server['pull_interval']) - self._pingcb.start() + state.main.add_callback(self._pingcb.start) Thread.__init__(self) self.daemon = True self.start() @@ -512,7 +512,7 @@ class Nodes(Thread): self.queue('add', u.id, True) self.local = LocalNodes() self._pullcb = PeriodicCallback(self.pull, settings.server['pull_interval']) - self._pullcb.start() + state.main.add_callback(self._pingcb.start) Thread.__init__(self) self.daemon = True self.start() From d84e950ca81435935c62d5612f60a0112ddbb7e1 Mon Sep 17 00:00:00 2001 From: j Date: Sat, 2 Feb 2019 14:57:04 +0530 Subject: [PATCH 08/14] typo --- oml/nodes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oml/nodes.py b/oml/nodes.py index fd7a6ea..47a0f7c 100644 --- a/oml/nodes.py +++ b/oml/nodes.py @@ -512,7 +512,7 @@ class Nodes(Thread): self.queue('add', u.id, True) self.local = LocalNodes() self._pullcb = PeriodicCallback(self.pull, settings.server['pull_interval']) - state.main.add_callback(self._pingcb.start) + state.main.add_callback(self.pullcb.start) Thread.__init__(self) self.daemon = True self.start() From bcfc8ba9b2c4b7833d20d5173cbb470a9f81f5fa Mon Sep 17 00:00:00 2001 From: j Date: Sat, 2 Feb 2019 14:58:10 +0530 Subject: [PATCH 09/14] typo(again) --- oml/nodes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oml/nodes.py b/oml/nodes.py index 47a0f7c..09809fc 100644 --- a/oml/nodes.py +++ b/oml/nodes.py @@ -512,7 +512,7 @@ class Nodes(Thread): self.queue('add', u.id, True) self.local = LocalNodes() self._pullcb = PeriodicCallback(self.pull, settings.server['pull_interval']) - state.main.add_callback(self.pullcb.start) + state.main.add_callback(self._pullcb.start) Thread.__init__(self) self.daemon = True self.start() From 4033958341ad8ca6fb2c765701c18391fabed250 Mon Sep 17 00:00:00 2001 From: j Date: Sat, 2 Feb 2019 15:53:53 +0530 Subject: [PATCH 10/14] remove unique contraint for user.nickname --- oml/queryparser.py | 2 +- oml/settings.py | 2 +- oml/setup.py | 2 +- oml/update.py | 29 ++++++++++++++++++++++++++++- oml/user/models.py | 2 +- oml/websocket.py | 8 ++++---- 6 files changed, 36 insertions(+), 9 deletions(-) diff --git a/oml/queryparser.py b/oml/queryparser.py index 2810729..939611b 100644 --- a/oml/queryparser.py +++ b/oml/queryparser.py @@ -129,7 +129,7 @@ class Parser(object): nickname, name = v.split(':', 1) if nickname: try: - u = self._user.query.filter_by(nickname=nickname).one() + u = self._user.query.filter_by(nickname=nickname, peered=True).one() except sqlalchemy.orm.exc.NoResultFound: ids = [] return self.in_ids(ids, exclude) diff --git a/oml/settings.py b/oml/settings.py index 4ef0843..bdc6c31 100644 --- a/oml/settings.py +++ b/oml/settings.py @@ -95,4 +95,4 @@ FULLTEXT_SUPPORT = fulltext.platform_supported() if not FULLTEXT_SUPPORT: config['itemKeys'] = [k for k in config['itemKeys'] if k['id'] != 'fulltext'] -DB_VERSION = 15 +DB_VERSION = 16 diff --git a/oml/setup.py b/oml/setup.py index b243a00..69024d0 100644 --- a/oml/setup.py +++ b/oml/setup.py @@ -43,11 +43,11 @@ CREATE TABLE user ( peered BOOLEAN, online BOOLEAN, PRIMARY KEY (id), - UNIQUE (nickname), CHECK (queued IN (0, 1)), CHECK (peered IN (0, 1)), CHECK (online IN (0, 1)) ); +CREATE INDEX ix_user_nichname ON user (nichname); CREATE TABLE metadata ( created DATETIME, modified DATETIME, diff --git a/oml/update.py b/oml/update.py index 07cddb4..4177800 100644 --- a/oml/update.py +++ b/oml/update.py @@ -373,6 +373,8 @@ class Update(Thread): db_version = migrate_13() if db_version < 15: db_version = migrate_15() + if db_version < 16: + db_version = migrate_16() settings.server['db_version'] = db_version def run(self): @@ -437,7 +439,7 @@ def migrate_5(): 'DROP INDEX IF EXISTS user_metadata_index', 'CREATE UNIQUE INDEX user_metadata_index ON user_metadata(item_id, user_id)', 'UPDATE sort SET sharemetadata = 0', - ]), + ]) with db.session() as session: import user.models for m in user.models.Metadata.query: @@ -631,3 +633,28 @@ def migrate_15(): del u.info['local'] u.save() return 15 + +def migrate_16(): + db.run_sql([ + '''CREATE TABLE user2 ( + created DATETIME, + modified DATETIME, + id VARCHAR(43) NOT NULL, + info BLOB, + nickname VARCHAR(256), + pending VARCHAR(64), + queued BOOLEAN, + peered BOOLEAN, + online BOOLEAN, + PRIMARY KEY (id), + CHECK (queued IN (0, 1)), + CHECK (peered IN (0, 1)), + CHECK (online IN (0, 1)) + )''', + '''INSERT INTO user2 (created, modified, id, info, nickname, pending, queued, peered, online) + SELECT created, modified, id, info, nickname, pending, queued, peered, online FROM user''', + 'DROP TABLE user', + 'ALTER TABLE user2 RENAME TO user', + 'CREATE INDEX IF NOT EXISTS ix_user_nickname ON user (nickname)' + ]) + return 16 diff --git a/oml/user/models.py b/oml/user/models.py index 3d7284a..8e7164d 100644 --- a/oml/user/models.py +++ b/oml/user/models.py @@ -30,7 +30,7 @@ class User(db.Model): id = sa.Column(sa.String(43), primary_key=True) info = sa.Column(MutableDict.as_mutable(sa.PickleType(pickler=json_pickler))) - nickname = sa.Column(sa.String(256), unique=True) + nickname = sa.Column(sa.String(256), index=True) pending = sa.Column(sa.String(64)) # sent|received queued = sa.Column(sa.Boolean()) diff --git a/oml/websocket.py b/oml/websocket.py index 12795ce..9178bb8 100644 --- a/oml/websocket.py +++ b/oml/websocket.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- - -from tornado.websocket import WebSocketHandler +from tornado.websocket import WebSocketHandler, WebSocketClosedError from tornado.ioloop import IOLoop +from tornado.iostream import StreamClosedError import json from oxtornado import json_dumps @@ -72,9 +72,9 @@ class Handler(WebSocketHandler): try: task = self.write_message(message) await task - except tornado.iostream.StreamClosedError as e: + except StreamClosedError as e: self.on_close() - except tornado.websocket.WebSocketClosedError as e: + except WebSocketClosedError as e: self.on_close() From 431dc8e194f3b4c95ddc5398cca2bd7590ae82bd Mon Sep 17 00:00:00 2001 From: j Date: Sat, 2 Feb 2019 17:36:11 +0530 Subject: [PATCH 11/14] ping on action --- oml/nodes.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/oml/nodes.py b/oml/nodes.py index 09809fc..c5cfbfe 100644 --- a/oml/nodes.py +++ b/oml/nodes.py @@ -539,9 +539,10 @@ class Nodes(Thread): def peer_queue(self, peer, action, *args): if peer not in self._nodes: self._add(peer) + elif not self._nodes[peer].is_online(): + self._nodes[peer].ping() self._nodes[peer].queue(action, *args) - def is_online(self, id): return id in self._nodes and self._nodes[id].is_online() @@ -557,7 +558,12 @@ class Nodes(Thread): if target == 'all': nodes = list(self._nodes.values()) elif target == 'peered': - nodes = [n for n in list(self._nodes.values()) if n.user.peered] + ids = [] + with db.session(): + from user.models import User + for u in User.query.filter(User.id != settings.USER_ID).filter_by(peered=True).all(): + ids.append(u.id) + nodes = [n for n in list(self._nodes.values()) if n.user_id in ids] elif target == 'online': nodes = [n for n in list(self._nodes.values()) if n.online] else: From 9d11bbba531dd8cd6623cfeb66c47f0ccf678173 Mon Sep 17 00:00:00 2001 From: j Date: Sat, 2 Feb 2019 17:36:28 +0530 Subject: [PATCH 12/14] don't record addlistitems for Inbox --- oml/item/models.py | 2 +- oml/user/models.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/oml/item/models.py b/oml/item/models.py index 149a678..3922b4f 100644 --- a/oml/item/models.py +++ b/oml/item/models.py @@ -538,7 +538,7 @@ class Item(db.Model): add_record('additem', self.id, f.info) add_record('edititem', self.id, self.meta) for l in self.lists.filter_by(user_id=settings.USER_ID): - if l.name != '': + if l.name != '' and l.name != 'Inbox': add_record('addlistitems', l.name, [self.id]) self.update() f.move() diff --git a/oml/user/models.py b/oml/user/models.py index 8e7164d..a7344f0 100644 --- a/oml/user/models.py +++ b/oml/user/models.py @@ -339,7 +339,7 @@ class List(db.Model): state.db.session.add(self) if commit: state.db.session.commit() - if self.user_id == settings.USER_ID and self.name != '' and available_items: + if self.user_id == settings.USER_ID and self.name not in ('', 'Inbox') and available_items: add_record('addlistitems', self.name, available_items) def get_items(self): From 5f36b2eab47648dfffbdb66a43d07ac6ca78a916 Mon Sep 17 00:00:00 2001 From: j Date: Sat, 2 Feb 2019 17:53:45 +0530 Subject: [PATCH 13/14] get rid of Public lists --- oml/settings.py | 2 +- oml/update.py | 16 ++++++++++++++++ oml/user/models.py | 2 +- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/oml/settings.py b/oml/settings.py index bdc6c31..1d5d9ca 100644 --- a/oml/settings.py +++ b/oml/settings.py @@ -95,4 +95,4 @@ FULLTEXT_SUPPORT = fulltext.platform_supported() if not FULLTEXT_SUPPORT: config['itemKeys'] = [k for k in config['itemKeys'] if k['id'] != 'fulltext'] -DB_VERSION = 16 +DB_VERSION = 17 diff --git a/oml/update.py b/oml/update.py index 4177800..e51ed08 100644 --- a/oml/update.py +++ b/oml/update.py @@ -375,6 +375,8 @@ class Update(Thread): db_version = migrate_15() if db_version < 16: db_version = migrate_16() + if db_version < 17: + db_version = migrate_17() settings.server['db_version'] = db_version def run(self): @@ -658,3 +660,17 @@ def migrate_16(): 'CREATE INDEX IF NOT EXISTS ix_user_nickname ON user (nickname)' ]) return 16 + +def migrate_17(): + from user.models import List, User + from changelog import add_record + with db.session(): + l = List.get(':Public') + if not l: + add_record('removelist', 'Public') + lists = [] + for l in List.query.filter_by(user_id=settings.USER_ID).order_by('index_'): + if l.type == 'static' and l.name not in ('', 'Inbox'): + lists.append(l.name) + add_record('orderlists', lists) + return 17 diff --git a/oml/user/models.py b/oml/user/models.py index a7344f0..77ff81e 100644 --- a/oml/user/models.py +++ b/oml/user/models.py @@ -368,7 +368,7 @@ class List(db.Model): q = list_items.delete().where(list_items.columns['list_id'].is_(self.id)) state.db.session.execute(q) if not self._query: - if self.user_id == settings.USER_ID and self.name != '': + if self.user_id == settings.USER_ID and self.name not in ('', 'Inbox'): add_record('removelist', self.name) state.db.session.delete(self) if commit: From 103aab3f4a09207e25e2e86daee6d912e27adedd Mon Sep 17 00:00:00 2001 From: j Date: Sat, 2 Feb 2019 18:04:06 +0530 Subject: [PATCH 14/14] keep listorder in sync --- oml/library.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/oml/library.py b/oml/library.py index 94e30e8..362826d 100644 --- a/oml/library.py +++ b/oml/library.py @@ -130,16 +130,22 @@ class Peer(object): self.info['lists'][new['name']] = self.info['lists'].pop(name) else: self.info['lists'][new['name']] = [] + if name in self.info['listorder']: + self.info['listorder'] = [new['name'] if n == name else n for n in self.info['listorder']] elif action == 'orderlists': self.info['listorder'] = args[0] elif action == 'removelist': name = args[0] if name in self.info['lists']: del self.info['lists'][name] + if name in self.info['listorder']: + self.info['listorder'] = [n for n in self.info['listorder'] if n != name] elif action == 'addlistitems': name, ids = args if name not in self.info['lists']: self.info['lists'][name] = [] + if name not in self.info['listorder']: + self.info['listorder'].append(name) self.info['lists'][name] = list(set(self.info['lists'][name]) | set(ids)) elif action == 'removelistitems': name, ids = args