diff --git a/oml/changelog.py b/oml/changelog.py index ab7cd2b..6a13f9b 100644 --- a/oml/changelog.py +++ b/oml/changelog.py @@ -20,8 +20,8 @@ class Changelog(db.Model): editlist name {name: newname} orderlists [name, name, name] removelist name - additemtolist listname itemid - removeitemfromlist listname itemid + addlistitems listname [ids] + removelistitems listname [ids] editusername username editcontact string addpeer peerid peername @@ -182,24 +182,17 @@ class Changelog(db.Model): l.remove() return True - def action_addlistitem(self, user, timestamp, name, itemid): - from item.models import Item + def action_addlistitems(self, user, timestamp, name, ids): from user.models import List - l = List.get(user.id, name) - i = Item.get(itemid) - if l and i: - i.lists.append(l) - i.update() + l = List.get_or_create(user.id, name) + l.add_items(ids) return True - def action_removelistitem(self, user, timestamp, name, itemid): - from item.models import Item + def action_removelistitem(self, user, timestamp, name, ids): from user.models import List l = List.get(user.id, name) - i = Item.get(itemid) - if l and i: - i.lists.remove(l) - i.update() + if l: + l.remove_items(ids) return True def action_editusername(self, user, timestamp, username): diff --git a/oml/item/api.py b/oml/item/api.py index 5bc498d..5e42e20 100644 --- a/oml/item/api.py +++ b/oml/item/api.py @@ -6,7 +6,6 @@ from flask import json from oxflask.api import actions from oxflask.shortcuts import returns_json -from oml import utils import query import models @@ -118,6 +117,8 @@ def edit(request): if item and keys and item.json()['mediastate'] == 'available': key = keys[0] print 'update mainid', key, data[key] + if key in ('isbn10', 'isbn13'): + data[key] = utils.normalize_isbn(data[key]) item.update_mainid(key, data[key]) response = item.json() else: diff --git a/oml/item/models.py b/oml/item/models.py index ce5e689..a53931f 100644 --- a/oml/item/models.py +++ b/oml/item/models.py @@ -233,6 +233,12 @@ class Item(db.Model): else: f.value = '%s:' % p.id db.session.add(f) + for l in self.lists: + f = Find() + f.item_id = self.id + f.key = 'list' + f.value = l.find_id + db.session.add(f) def update(self): users = map(str, list(self.users)) diff --git a/oml/localnodes.py b/oml/localnodes.py new file mode 100644 index 0000000..4d7553e --- /dev/null +++ b/oml/localnodes.py @@ -0,0 +1,104 @@ +# -*- coding: utf-8 -*- +# vi:si:et:sw=4:sts=4:ts=4 + +import socket +import thread +import json +import struct +from threading import Thread + +from settings import preferences, server, USER_ID +from node.utils import get_public_ipv6 + +def can_connect(data): + try: + s = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) + s.settimeout(1) + s.connect((data['host'], data['port'])) + s.close() + return True + except: + pass + return False + +class LocalNodes(Thread): + _active = True + _nodes = {} + + _BROADCAST = "ff02::1" + _PORT = 9851 + TTL = 2 + + def __init__(self, app): + self._app = app + Thread.__init__(self) + self.daemon = True + self.start() + self.host = get_public_ipv6() + self.send() + + def send(self): + s = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) + + ttl = struct.pack('@i', self.TTL) + s.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_HOPS, ttl) + message = json.dumps({ + 'id': USER_ID, + 'username': preferences.get('username', 'anonymous'), + 'host': self.host, + 'port': server['node_port'], + }) + s.sendto(message + '\0', (self._BROADCAST, self._PORT)) + + def receive(self): + s = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.bind(('', self._PORT)) + group_bin = socket.inet_pton(socket.AF_INET6, self._BROADCAST) + '\0'*4 + s.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_JOIN_GROUP, group_bin) + + while self._active: + data, addr = s.recvfrom(1024) + while data[-1] == '\0': + data = data[:-1] # Strip trailing \0's + data = self.validate(data) + if data: + if data['id'] not in self._nodes: + thread.start_new_thread(self.new_node, (data, )) + else: + print 'UPDATE NODE', data + self._nodes[data['id']] = data + + def validate(self, data): + try: + data = json.loads(data) + except: + return None + for key in ['id', 'username', 'host', 'port']: + if key not in data: + return None + return data + + def get(self, user_id): + if user_id in self._nodes: + if can_connect(self._nodes[user_id]): + return self._nodes[user_id] + + def new_node(self, data): + print 'NEW NODE', data + if can_connect(data): + self._nodes[data['id']] = data + with self._app.app_context(): + import user.models + u = user.models.User.get_or_create(data['id']) + u.info['username'] = data['username'] + u.info['local'] = data + u.save() + self.send() + + def run(self): + self.receive() + + def join(self): + self._active = False + return Thread.join(self) diff --git a/oml/node/server.py b/oml/node/server.py index 05e8296..5dc09a5 100644 --- a/oml/node/server.py +++ b/oml/node/server.py @@ -107,11 +107,8 @@ def start(app): (r"/get/(.*)", ShareHandler, dict(app=app)), (r".*", NodeHandler, dict(app=app)), ]) - - #tr = WSGIContainer(node_app) - #http_server= HTTPServer(tr) http_server.listen(settings.server['node_port'], settings.server['node_address']) - host = utils.get_public_ipv4() + host = utils.get_public_ipv6() state.online = directory.put(settings.sk, { 'host': host, 'port': settings.server['node_port'] diff --git a/oml/node/utils.py b/oml/node/utils.py index 712f401..c9d5e1b 100644 --- a/oml/node/utils.py +++ b/oml/node/utils.py @@ -1,6 +1,7 @@ +# -*- coding: utf-8 -*- +# vi:si:et:sw=4:sts=4:ts=4 + import socket -import requests -from urlparse import urlparse def get_public_ipv6(): host = ('2a01:4f8:120:3201::3', 25519) diff --git a/oml/nodes.py b/oml/nodes.py index 07ff7f4..b92d1d6 100644 --- a/oml/nodes.py +++ b/oml/nodes.py @@ -19,6 +19,7 @@ from changelog import Changelog import directory from websocket import trigger_event +from localnodes import LocalNodes ENCODING='base64' @@ -26,8 +27,9 @@ class Node(object): online = False download_speed = 0 - def __init__(self, app, user): - self._app = app + def __init__(self, nodes, user): + self._nodes = nodes + self._app = nodes._app self.user_id = user.id key = str(user.id) self.vk = ed25519.VerifyingKey(key, encoding=ENCODING) @@ -35,10 +37,15 @@ class Node(object): @property def url(self): - if ':' in self.host: - url = 'http://[%s]:%s' % (self.host, self.port) + local = self.get_local() + if local: + url = 'http://[%s]:%s' % (local['host'], local['port']) + print 'using local peer discovery to access node', url else: - url = 'http://%s:%s' % (self.host, self.port) + if ':' in self.host: + url = 'http://[%s]:%s' % (self.host, self.port) + else: + url = 'http://%s:%s' % (self.host, self.port) return url def resolve_host(self): @@ -51,6 +58,11 @@ class Node(object): self.host = None self.port = 9851 + def get_local(self): + if self._nodes and self._nodes._local: + return self._nodes._local.get(self.user_id) + return None + def request(self, action, *args): if not self.host: self.resolve_host() @@ -211,6 +223,7 @@ class Nodes(Thread): self._app = app self._q = Queue() self._running = True + self._local = LocalNodes(app) Thread.__init__(self) self.daemon = True self.start() @@ -238,7 +251,7 @@ class Nodes(Thread): def _add_node(self, user_id): if user_id not in self._nodes: from user.models import User - self._nodes[user_id] = Node(self._app, User.get_or_create(user_id)) + self._nodes[user_id] = Node(self, User.get_or_create(user_id)) else: self._nodes[user_id].online = True trigger_event('status', { diff --git a/oml/oxflask/query.py b/oml/oxflask/query.py index 77bdec2..97341a4 100644 --- a/oml/oxflask/query.py +++ b/oml/oxflask/query.py @@ -144,6 +144,8 @@ class Parser(object): l = self._list.query.filter_by(user_id=p.id, name=name).first() else: l = None + if l: + v = l.find_id if l and l._query: data = l._query q = self.parse_conditions(data.get('conditions', []), diff --git a/oml/settings.py b/oml/settings.py index d4b5305..944f75e 100644 --- a/oml/settings.py +++ b/oml/settings.py @@ -38,7 +38,7 @@ server_defaults = { 'port': 9842, 'address': '127.0.0.1', 'node_port': 9851, - 'node_address': '::', + 'node_address': '', 'extract_text': True, 'directory_service': 'http://[2a01:4f8:120:3201::3]:25519', 'lookup_service': 'http://data.openmedialibrary.com', diff --git a/oml/user/api.py b/oml/user/api.py index c702eaa..39049f4 100644 --- a/oml/user/api.py +++ b/oml/user/api.py @@ -84,9 +84,9 @@ actions.register(getUsers) @returns_json def getLists(request): - lists = {} + lists = [] for u in models.User.query.filter((models.User.peered==True)|(models.User.id==settings.USER_ID)): - lists[u.id] = u.lists_json() + lists += u.lists_json() return { 'lists': lists } @@ -132,28 +132,24 @@ def editList(request): actions.register(editList, cache=False) @returns_json -def addListItem(request): +def addListItems(request): data = json.loads(request.form['data']) if 'data' in request.form else {} - l = models.List.get_or_create(data['id']) - i = Item.get(data['item']) - if l and i: - l.items.append(i) - models.db.session.add(l) - i.update() + l = models.List.get_or_create(data['list']) + if l: + l.add_items(data['items']) + return l.json() return {} -actions.register(addListItem, cache=False) +actions.register(addListItems, cache=False) @returns_json -def removeListItem(request): +def removeListItems(request): data = json.loads(request.form['data']) if 'data' in request.form else {} - l = models.List.get(data['id']) - i = Item.get(data['item']) - if l and i: - l.items.remove(i) - models.db.session.add(l) - i.update() + l = models.List.get(data['list']) + if l: + l.remove_items(data['items']) + return l.json() return {} -actions.register(removeListItem, cache=False) +actions.register(removeListItems, cache=False) @returns_json def sortLists(request): diff --git a/oml/user/models.py b/oml/user/models.py index 9dafcf1..9ce7379 100644 --- a/oml/user/models.py +++ b/oml/user/models.py @@ -59,7 +59,7 @@ class User(db.Model): return j def check_online(self): - return state.nodes.check_online(self.id) + return state.nodes and state.nodes.check_online(self.id) def lists_json(self): return [l.json() for l in self.lists.order_by('position')] @@ -158,25 +158,39 @@ class List(db.Model): from item.models import Item for item_id in items: i = Item.get(item_id) - self.items.add(i) + self.items.append(i) + i.update() db.session.add(self) db.session.commit() + for item_id in items: + i = Item.get(item_id) + i.update_lists() + db.session.commit() + if self.user_id == settings.USER_ID: + Changelog.record(self.user, 'addlistitems', self.name, items) def remove_items(self, items): from item.models import Item for item_id in items: i = Item.get(item_id) self.items.remove(i) + i.update() db.session.add(self) db.session.commit() + for item_id in items: + i = Item.get(item_id) + i.update_lists() + db.session.commit() + if self.user_id == settings.USER_ID: + Changelog.record(self.user, 'removelistitems', self.name, items) def remove(self): if not self._query: for i in self.items: self.items.remove(i) if not self._query: - print 'record change: removelist', self.user, self.name - Changelog.record(self.user, 'removelist', self.name) + if self.user_id == settings.USER_ID: + Changelog.record(self.user, 'removelist', self.name) db.session.delete(self) db.session.commit() @@ -184,10 +198,21 @@ class List(db.Model): def public_id(self): id = '' if self.user_id != settings.USER_ID: - id += self.user_id - id = '%s:%s' % (id, self.name) + id += self.user.nickname + id = u'%s:%s' % (id, self.name) return id + @property + def find_id(self): + id = '' + if self.user_id != settings.USER_ID: + id += self.user_id + id = u'%s:%s' % (id, self.id) + return id + + def __repr__(self): + return self.public_id.encode('utf-8') + def items_count(self): from item.models import Item if self._query: @@ -199,6 +224,7 @@ class List(db.Model): def json(self): r = { 'id': self.public_id, + 'user': self.user.nickname if self.user_id != settings.USER_ID else settings.preferences['username'], 'name': self.name, 'index': self.position, 'items': self.items_count(), diff --git a/static/js/browser.js b/static/js/browser.js index 1074929..d012e4c 100644 --- a/static/js/browser.js +++ b/static/js/browser.js @@ -9,8 +9,8 @@ oml.ui.browser = function() { defaultRatio: oml.config.coverRatio, draggable: true, item: function(data, sort, size) { - var color = oml.getFileTypeColor(data).map(function(rgb) { - return rgb.concat(0.8) + var color = oml.getFileInfoColor(ui.fileInfo, data).map(function(rgb) { + return rgb.concat(0.8); }), ratio = data.coverRatio || oml.config.coverRatio, width = Math.round(ratio >= 1 ? size : size * ratio), @@ -28,12 +28,14 @@ oml.ui.browser = function() { height: Math.round(size / 12.8) + 'px', borderWidth: Math.round(size / 64) + 'px 0', borderStyle: 'solid', - borderColor: 'rgba(' + color[1].join(', ') + ')', + borderColor: 'rgba(' + color[2].join(', ') + ')', margin: Math.round(size / 18) + 'px ' + Math.round(width / 3) + 'px', fontSize: Math.round(size / 16) + 'px', textAlign: 'center', - color: 'rgba(' + color[1].join(', ') + ')', - backgroundColor: 'rgba(' + color[0].join(', ') + ')', + color: 'rgba(' + color[2].join(', ') + ')', + backgroundImage: '-webkit-linear-gradient(top, ' + color.slice(0, 2).map(function(rgba) { + return 'rgba(' + rgba.join(', ') + ')'; + }).join(', ') + ')', WebkitTransform: 'rotate(45deg)' }) .html( @@ -55,8 +57,8 @@ oml.ui.browser = function() { }), callback); }, keys: [ - 'author', 'coverRatio', 'extension', - 'id', 'size', 'textsize', 'title' + 'author', 'coverRatio', 'extension', 'id', + 'mediastate', 'size', 'textsize', 'title' ], max: 1, min: 1, @@ -101,6 +103,8 @@ oml.ui.browser = function() { } }); + oml.enableDragAndDrop(that); + return that; }; \ No newline at end of file diff --git a/static/js/folders.js b/static/js/folders.js index 5c4467a..00957c6 100644 --- a/static/js/folders.js +++ b/static/js/folders.js @@ -58,200 +58,187 @@ oml.ui.folders = function() { oml.$ui.libraryList = []; oml.$ui.folderList = []; - oml.api.getUsers(function(result) { + oml.getUsersAndLists(function(users, lists) { - var peers = result.data.users.filter(function(user) { - return user.peered; - }); + Ox.print('GOT USERS AND LISTS', users, lists); - oml.api.getLists(function(result) { + users.forEach(function(user, index) { - Ox.print('GOT LISTS', result.data); + var $content, + items = lists.filter(function(list) { + return list.user == user.nickname + && list.type != 'library'; + }), + libraryId = (!index ? '' : user.nickname) + ':' - var users = [ - { - id: oml.user.id, - nickname: oml.user.preferences.username, - online: oml.user.online - } - ].concat(peers), + userIndex[user.nickname] = index; - lists = result.data.lists; - - users.forEach(function(user, index) { - - var $content, - libraryId = (!index ? '' : user.nickname) + ':'; - - userIndex[user.nickname] = index; - - oml.$ui.folder[index] = Ox.CollapsePanel({ - collapsed: false, - extras: [ - oml.ui.statusIcon( - !oml.user.online ? 'unknown' - : user.online ? 'connected' - : 'disconnected' - ), - {}, - Ox.Button({ - style: 'symbol', - title: 'info', - tooltip: Ox._(!index ? 'Preferences' : 'Profile'), - type: 'image' - }) - .bindEvent({ - click: function() { - if (!index) { - oml.UI.set({ - page: 'preferences', - 'part.preferences': 'account' - }); - } else { - oml.UI.set({page: 'users'}) - } + oml.$ui.folder[index] = Ox.CollapsePanel({ + collapsed: false, + extras: [ + oml.ui.statusIcon( + !oml.user.online ? 'unknown' + : user.online ? 'connected' + : 'disconnected' + ), + {}, + Ox.Button({ + style: 'symbol', + title: 'info', + tooltip: Ox._(!index ? 'Preferences' : 'Profile'), + type: 'image' + }) + .bindEvent({ + click: function() { + if (!index) { + oml.UI.set({ + page: 'preferences', + 'part.preferences': 'account' + }); + } else { + oml.UI.set({page: 'users'}) } - }) - ], - title: Ox.encodeHTMLEntities(user.nickname) - }) - .css({ - width: ui.sidebarSize + } + }) + ], + title: Ox.encodeHTMLEntities(user.nickname) + }) + .css({ + width: ui.sidebarSize + }) + .bindEvent({ + toggle: function(data) { + oml.UI.set('showFolder.' + user.nickname, !data.collapsed); + } + }) + .bindEvent( + 'oml_showfolder.' + user.nickname.toLowerCase(), + function(data) { + oml.$ui.folder[index].options({collapsed: !data.value}); + } + ) + .appendTo(that); + + $content = oml.$ui.folder[index].$content + .css({ + height: (1 + items.length) * 16 + 'px' + }); + + $lists.push( + oml.$ui.libraryList[index] = oml.ui.folderList({ + items: [ + { + id: libraryId, + name: Ox._('Library'), + type: 'library', + items: -1 + } + ] }) .bindEvent({ - toggle: function(data) { - oml.UI.set('showFolder.' + user.nickname, !data.collapsed); + add: function() { + !index && oml.addList(); + }, + load: function() { + oml.api.find({ + query: getFind(libraryId) + }, function(result) { + oml.$ui.libraryList[index].value( + libraryId, 'items', result.data.items + ); + }); + }, + open: function() { + oml.$ui.listDialog = oml.ui.listDialog().open(); + }, + select: function(data) { + oml.UI.set({find: getFind(data.ids[0])}); + }, + selectnext: function() { + oml.UI.set({find: getFind(lists[user.id][0].id)}); + }, + selectprevious: function() { + var userId = !index ? null : users[index - 1].id, + set = { + find: getFind( + !index + ? '' + : Ox.last(lists[userId]).id + ) + }; + if (userId) { + Ox.extend(set, 'showFolder.' + userId, true); + } + oml.UI.set(set); } }) - .bindEvent( - 'oml_showfolder.' + user.nickname.toLowerCase(), - function(data) { - oml.$ui.folder[index].options({collapsed: !data.value}); - } - ) - .appendTo(that); + .appendTo($content) + ); - $content = oml.$ui.folder[index].$content - .css({ - height: (1 + lists[user.id].length) * 16 + 'px' - }); - - $lists.push( - oml.$ui.libraryList[index] = oml.ui.folderList({ - items: [ - { - id: libraryId, - name: Ox._('Library'), - type: 'library', - items: -1 - } - ] - }) - .bindEvent({ - add: function() { - !index && oml.addList(); - }, - load: function() { - oml.api.find({ - query: getFind(libraryId) - }, function(result) { - oml.$ui.libraryList[index].value( - libraryId, 'items', result.data.items - ); - }); - }, - open: function() { - oml.$ui.listDialog = oml.ui.listDialog().open(); - }, - select: function(data) { - oml.UI.set({find: getFind(data.ids[0])}); - }, - selectnext: function() { - oml.UI.set({find: getFind(lists[user.id][0].id)}); - }, - selectprevious: function() { - var userId = !index ? null : users[index - 1].id, - set = { - find: getFind( - !index - ? '' - : Ox.last(lists[userId]).id - ) - }; - if (userId) { - Ox.extend(set, 'showFolder.' + userId, true); - } - oml.UI.set(set); - } - }) - .appendTo($content) - ); - - $lists.push( - oml.$ui.folderList[index] = oml.ui.folderList({ - draggable: !!index, - items: lists[user.id], - sortable: true - }) - .bindEvent({ - add: function() { - !index && oml.addList(); - }, - 'delete': function() { - !index && oml.deleteList(); - }, - key_control_d: function() { - oml.addList(ui._list); - }, - load: function() { + $lists.push( + oml.$ui.folderList[index] = oml.ui.folderList({ + draggable: !!index, + items: items, + sortable: true + }) + .bindEvent({ + add: function() { + !index && oml.addList(); + }, + 'delete': function() { + !index && oml.deleteList(); + }, + key_control_d: function() { + oml.addList(ui._list); + }, + load: function() { + // ... + }, + move: function(data) { + lists[user.id] = data.ids.map(function(listId) { + return Ox.getObjectById(lists[user.id], listId); + }); + oml.api.sortLists({ + ids: data.ids, + user: user.id + }, function(result) { // ... - }, - move: function(data) { - lists[user.id] = data.ids.map(function(listId) { - return Ox.getObjectById(lists[user.id], listId); - }); - oml.api.sortLists({ - ids: data.ids, - user: user.id - }, function(result) { - // ... - }); - }, - open: function() { - oml.ui.listDialog().open(); - }, - select: function(data) { - oml.UI.set({find: getFind(data.ids[0])}); - }, - selectnext: function() { - if (index < users.length - 1) { - oml.UI.set(Ox.extend( - {find: getFind(users[index + 1].nickname + ':')}, - 'showFolder.' + users[index + 1].nickname, - true - )); - } - }, - selectprevious: function() { - oml.UI.set({find: getFind(libraryId)}); + }); + }, + open: function() { + oml.ui.listDialog().open(); + }, + select: function(data) { + oml.UI.set({find: getFind(data.ids[0])}); + }, + selectnext: function() { + if (index < users.length - 1) { + oml.UI.set(Ox.extend( + {find: getFind(users[index + 1].nickname + ':')}, + 'showFolder.' + users[index + 1].nickname, + true + )); } - }) - .bindEvent(function(data, event) { - if (!index) { - Ox.print('LIST EVENT', event, data); - } - }) - .css({height: lists[user.id].length * 16 + 'px'}) - .appendTo($content) - ); + }, + selectprevious: function() { + oml.UI.set({find: getFind(libraryId)}); + } + }) + .bindEvent(function(data, event) { + if (!index) { + Ox.print('LIST EVENT', event, data); + } + }) + .css({height: items.length * 16 + 'px'}) + .appendTo($content) + ); - oml.$ui.folderList[index].$body.css({top: '16px'}); - - }); - - selectList(); + oml.$ui.folderList[index].$body.css({top: '16px'}); }); + + selectList(); + }); function getFind(list) { diff --git a/static/js/gridView.js b/static/js/gridView.js index e0ca368..f0fce27 100644 --- a/static/js/gridView.js +++ b/static/js/gridView.js @@ -8,8 +8,8 @@ oml.ui.gridView = function() { defaultRatio: oml.config.coverRatio, draggable: true, item: function(data, sort, size) { - var color = oml.getFileTypeColor(data).map(function(rgb) { - return rgb.concat(0.8) + var color = oml.getFileInfoColor(ui.fileInfo, data).map(function(rgb) { + return rgb.concat(0.8); }), ratio = data.coverRatio || oml.config.coverRatio, width = Math.round(ratio >= 1 ? size : size * ratio), @@ -20,6 +20,9 @@ oml.ui.gridView = function() { ? (data.author || '') : data[sortKey] ); size = size || 128; + Ox.print('WTF', '-webkit-linear-gradient(top, ' + color.slice(2).map(function(rgba) { + return 'rgba(' + rgba.join(', ') + ')'; + }).join(', ') + ')'); return { extra: ui.showFileInfo ? $('