# -*- coding: utf-8 -*- from datetime import datetime import json import os import shutil import ox from sqlalchemy.orm import load_only import sqlalchemy as sa from changelog import add_record from db import MutableDict import db import json_pickler import settings import state import utils import media from websocket import trigger_event import logging logger = logging.getLogger(__name__) class User(db.Model): __tablename__ = 'user' created = sa.Column(sa.DateTime()) modified = sa.Column(sa.DateTime()) id = sa.Column(sa.String(128), primary_key=True) info = sa.Column(MutableDict.as_mutable(sa.PickleType(pickler=json_pickler))) nickname = sa.Column(sa.String(256), index=True) pending = sa.Column(sa.String(64)) # sent|received queued = sa.Column(sa.Boolean()) peered = sa.Column(sa.Boolean()) online = sa.Column(sa.Boolean()) def __repr__(self): return self.id @classmethod def get(cls, id, for_update=False): qs = cls.query.filter_by(id=id) if for_update: qs = qs.with_for_update() user = qs.first() if user and not user.info: user.info = {} return user @classmethod def get_or_create(cls, id): user = cls.get(id) if not user: user = cls(id=id, peered=False, online=False) user.info = {} if state.nodes and state.nodes.local and id in state.nodes.local: user.info['username'] = state.nodes.local[id]['username'] user.update_name() user.save() return user def save(self): state.db.session.add(self) state.db.session.commit() @property def name(self): name = self.nickname if self.id != settings.USER_ID else '' return name @property def library(self): l = List.get_or_create(self.id, '') if l.index_ != -1: l.index_ = -1 l.save() return l def json(self, keys=None): j = {} if self.info: j.update(self.info) if state.nodes and self.id in state.nodes.local: j['local'] = state.nodes.local[self.id].copy() j['id'] = self.id if self.pending: j['pending'] = self.pending j['peered'] = self.peered if not keys or 'online' in keys: j['online'] = self.is_online() j['name'] = self.name if not keys or 'username' in keys or 'contact' in keys: if self.id == settings.USER_ID: j['username'] = settings.preferences['username'] j['contact'] = settings.preferences['contact'] elif self.id in state.peers: peer = state.peers[self.id] for key in ('username', 'contact'): if key in peer.info: j[key] = peer.info[key] if keys: for k in set(j) - set(keys): del j[k] return j def export_library(self): old_path = os.path.join(os.path.expanduser(settings.preferences['libraryPath']), 'Books', 'library.json') if os.path.exists(old_path): os.unlink(old_path) path = os.path.join(settings.data_path, 'library.json') self.library.export_json(path) def is_online(self): return state.nodes and state.nodes.is_online(self.id) def trigger_status(self): trigger_event('status', { 'id': self.id, 'online': self.is_online() }) def lists_json(self): self.library if self.id != settings.USER_ID: peer = utils.get_peer(self.id) lists = [] lists.append({ 'id': self.nickname + ':', 'user': self.name, 'items': len(peer.library), 'name': 'Library', 'type': 'library' }) index = 0 for name in peer.info.get('listorder', peer.info.get('lists', []).keys()): lists.append({ 'id': '%s:%s' % (self.nickname, name), 'user': self.name, 'name': name, 'index': index, 'items': len(peer.info['lists'].get(name, [])), 'type': 'static' }) index += 1 return lists return [l.json() for l in self.lists.order_by('index_')] def update_peering(self, peered, username=None): was_peering = self.peered if peered: logging.debug('update_peering, pending: %s queued: %s', self.pending, self.queued) self.queued = self.pending != 'sent' self.pending = '' if username: self.info['username'] = username self.update_name() if not was_peering: add_record('addpeer', self.id, self.nickname) if 'index' not in self.info: self.info['index'] = max([ u.info.get('index', -1) for u in User.query.filter_by(peered=True) if u.id != self.id ] + [0]) + 1 self.peered = True self.save() if self.id in state.removepeer: del state.removepeer[self.id] else: self.pending = '' self.peered = False self.queued = False if 'index' in self.info: del self.info['index'] self.update_name() self.save() if self.name in settings.ui['showFolder']: del settings.ui['showFolder'][self.name] settings.ui._save() state.removepeer[self.id] = True self.cleanup() if was_peering: add_record('removepeer', self.id) self.save() def cleanup(self): from item.models import user_items, Item from annotation.models import Annotation List.query.filter_by(user_id=self.id).delete() Annotation.query.filter_by(user_id=self.id).delete() c_user_id = user_items.columns['user_id'] q = user_items.delete().where(c_user_id.is_(self.id)) state.db.session.execute(q) Item.remove_without_user() self.save() if self.id in state.peers: state.peers[self.id].remove() del state.peers[self.id] def update_name(self): if self.id == settings.USER_ID: name = settings.preferences.get('username', 'anonymous') else: name = self.info.get('nickname') or self.info.get('username') or 'anonymous' nickname = name n = 2 while self.query.filter_by(nickname=nickname).filter(User.id != self.id).first(): nickname = '%s [%d]' % (name, n) n += 1 self.nickname = nickname def rebuild_changelog(self): logger.error('no longer used') return Changelog.query.filter_by(user_id=self.id).delete() for item in self.library.get_items().order_by('created'): Changelog.record(self, 'additem', item.id, item.info, _commit=False) Changelog.record(self, 'edititem', item.id, item.meta, _commit=False) lists = [] for l in List.query.filter_by(user_id=self.id, type='static').order_by('index_'): if l.name and l.name != 'Inbox': lists.append(l.name) Changelog.record(self, 'addlist', l.name, _commit=False) items = [i.id for i in l.get_items().options(load_only('id'))] if items: Changelog.record(self, 'addlistitems', l.name, items, _commit=False) if len(lists) > 1: Changelog.record(self, 'orderlists', lists, _commit=False) for peer in User.query.filter_by(peered=True): Changelog.record(self, 'addpeer', peer.id, self.nickname, _commit=False) if peer.info.get('contact'): Changelog.record(self, 'editpeer', peer.id, { 'contact': peer.info.get('contact') }, _commit=False) if settings.preferences.get('contact'): Changelog.record(self, 'editcontact', settings.preferences.get('contact'), _commit=False) state.db.session.commit() list_items = sa.Table('listitem', db.metadata, sa.Column('list_id', sa.Integer(), sa.ForeignKey('list.id')), sa.Column('item_id', sa.String(32), sa.ForeignKey('item.id'))) class List(db.Model): __tablename__ = 'list' id = sa.Column(sa.Integer(), primary_key=True) name = sa.Column(sa.String()) index_ = sa.Column(sa.Integer()) type = sa.Column(sa.String(64)) _query = sa.Column('query', MutableDict.as_mutable(sa.PickleType(pickler=json_pickler))) user_id = sa.Column(sa.String(128), sa.ForeignKey('user.id')) user = sa.orm.relationship('User', backref=sa.orm.backref('lists', lazy='dynamic')) items = sa.orm.relationship('Item', secondary=list_items, backref=sa.orm.backref('lists', lazy='dynamic')) @classmethod def get(cls, user_id, name=None): if name is None: user_id, name = cls.get_user_name(user_id) return cls.query.filter_by(user_id=user_id, name=name).first() @classmethod def get_user_name(cls, user_id): nickname, name = user_id.split(':', 1) if nickname: user = User.query.filter_by(nickname=nickname).first() user_id = user.id else: user_id = settings.USER_ID return user_id, name @classmethod def get_or_create(cls, user_id, name=None, query=None): if name is None: user_id, name = cls.get_user_name(user_id) l = cls.get(user_id, name) if not l: l = cls.create(user_id, name, query) return l @classmethod def create(cls, user_id, name, query=None): prefix = name n = 2 while cls.get(user_id, name): name = '%s [%s]' % (prefix, n) n += 1 l = cls(user_id=user_id, name=name) if query: l._query = query l.type = 'smart' else: l.type = 'static' l.index_ = cls.query.filter_by(user_id=user_id).count() state.db.session.add(l) state.db.session.commit() if user_id == settings.USER_ID: if l.type == 'static' and name != '' and name != 'Inbox': add_record('addlist', l.name) return l @classmethod def rename_user(cls, old, new): for l in cls.query.filter(cls._query != None): def update_conditions(conditions): changed = False for c in conditions: if 'conditions' in c: changed = update_conditions(c['conditions']) else: if c.get('key') == 'list' and c.get('value', '').startswith('%s:' % old): c['value'] = '%s:%s' % new, c['value'].split(':', 1)[1] changed = True return changed if l._query and update_conditions(l._query.get('conditions', [])): l.save() def add_items(self, items, commit=True): from item.models import Item available_items = [] for item_id in items: i = Item.get(item_id) if i: if self.user_id == settings.USER_ID and i.info.get('mediastate') != 'available': i.queue_download() if i not in self.items: self.items.append(i) i.update(commit=False) if i.info['mediastate'] == 'available': available_items.append(item_id) state.db.session.add(self) if commit: state.db.session.commit() if self.user_id == settings.USER_ID and self.name not in ('', 'Inbox') and available_items: add_record('addlistitems', self.name, available_items) def get_items(self): from item.models import Item if self.type == 'smart': return Item.find({'query': self._query}) else: return self.user.items.join(Item.lists, aliased=True).filter(List.id == self.id) def remove_items(self, items, commit=True): from item.models import Item for item_id in items: i = Item.get(item_id) if i: if i in self.items: self.items.remove(i) i.update(commit=commit) state.db.session.add(self) if commit: state.db.session.commit() if self.user_id == settings.USER_ID and self.name != '': add_record('removelistitems', self.name, items) def remove(self, commit=True): if not self._query: q = list_items.delete().where(list_items.columns['list_id'].is_(self.id)) state.db.session.execute(q) if not self._query: if self.user_id == settings.USER_ID and self.name not in ('', 'Inbox'): add_record('removelist', self.name) state.db.session.delete(self) if commit: state.db.session.commit() @property def public_id(self): id = '' if self.user_id != settings.USER_ID: id += self.user.nickname id = '%s:%s' % (id, self.name) return id @property def find_id(self): id = '' if self.user_id != settings.USER_ID: id += self.user_id id = '%s:%s' % (id, self.id) return id def __repr__(self): return self.public_id def items_count(self): if self.user_id != settings.USER_ID: peer = utils.get_peer(self.user_id) if self.name: return len(peer.info['lists'].get(self.name, [])) else: return len(peer.library) return self.get_items().count() def json(self): r = { 'id': self.public_id, 'user': self.user.name, 'name': self.name, 'index': self.index_, 'items': self.items_count(), 'type': self.type } if self.name == '': r['name'] = 'Library' r['type'] = 'library' del r['index'] if self.type == 'smart': r['query'] = self._query return r def save(self): state.db.session.add(self) state.db.session.commit() def create_symlinks(self): pass def export_json(self, path=None): from utils import _to_json if not path: if self.name: name = os.path.join('Lists', self.name) else: name = 'Books' path = os.path.join(os.path.expanduser(settings.preferences['libraryPath']), name, 'library.json') ox.makedirs(os.path.dirname(path)) items = [] for i in self.get_items(): j = i.json() for f in i.files: j['path'] = f.path break items.append(j) with open(path, 'w', encoding='utf-8') as f: json.dump(items, f, indent=4, default=_to_json, ensure_ascii=False, sort_keys=True) class Metadata(db.Model): __tablename__ = 'user_metadata' created = sa.Column(sa.DateTime()) modified = sa.Column(sa.DateTime()) id = sa.Column(sa.Integer(), primary_key=True) item_id = sa.Column(sa.String(32)) user_id = sa.Column(sa.String(128), sa.ForeignKey('user.id')) data_hash = sa.Column(sa.String(40), index=True) data = sa.Column(MutableDict.as_mutable(sa.PickleType(pickler=json_pickler))) def __repr__(self): return '{item}/{user}'.format(item=self.item_id, user=self.user_id) @property def timestamp(self): return utils.datetime2ts(self.modified) @classmethod def get(cls, user_id, item_id): return cls.query.filter_by(item_id=item_id, user_id=user_id).first() @classmethod def get_or_create(cls, user_id, item_id, data=None, commit=True): m = cls.get(user_id=user_id, item_id=item_id) if not m: m = cls(user_id=user_id, item_id=item_id) m.created = datetime.utcnow() if data: m.data = data else: m.data = {} m.save(commit=commit) elif data: m.edit(data, commit=commit) return m def get_hash(self): return utils.get_meta_hash(self.data) def save(self, commit=True, modified=None): if modified is None: self.modified = datetime.utcnow() else: self.modified = modified state.db.session.add(self) if commit: state.db.session.commit() def edit(self, data, commit=True, modified=None): changes = {} if 'isbn' in data and isinstance(data['isbn'], list): isbns = [utils.to_isbn13(isbn) for isbn in data['isbn']] isbns = [isbn for isbn in isbns if isbn] if isbns: data['isbn'] = isbns[0] else: del data['isbn'] for key in data: if key == 'id': continue if data[key] != self.data.get(key): self.data[key] = data[key] changes[key] = data[key] if changes: self.data_hash = self.get_hash() self.save(commit=commit, modified=modified) return changes def delete(self): state.db.session.delete(self) state.db.session.commit() def export_list(data): with db.session(): self = List.get(data['list']) if not self: return mode = data.get('mode') prefix = data.get('path') if mode not in ('add', 'replace'): logger.debug('invalid mode %s', mode) return if not prefix or prefix == '/': logger.debug('invalid export path %s', prefix) trigger_event('activity', { 'activity': 'export', 'path': prefix, 'progress': [0, 0], 'status': {'code': 404, 'text': 'invalid export path'} }) return root = prefix while not os.path.exists(root) and root != '/': root = os.path.dirname(root) if not os.access(root, os.W_OK): logger.debug('can not write to %s', root) trigger_event('activity', { 'activity': 'export', 'path': prefix, 'progress': [0, 0], 'path': prefix, 'status': {'code': 404, 'text': 'permission denied'} }) return if os.path.exists(prefix): existing_files = set( os.path.join(root, f) for root, _, files in os.walk(prefix) for f in files ) else: existing_files = set() new_files = set() count = self.get_items().count() n = 1 for i in self.get_items(): if i.files.all(): f = i.files.all()[0] source = f.fullpath() target = os.path.join(prefix, f.path) if mode == 'add': p = 1 parts = target.rsplit('.', 1) while os.path.exists(target) and media.get_id(target) != f.sha1: target = '.'.join([parts[0], f.sha1[:p], parts[1]]) p += 1 ox.makedirs(os.path.dirname(target)) if os.path.exists(target): if mode == 'replace' and media.get_id(target) != f.sha1: os.unlink(target) shutil.copy2(source, target) else: shutil.copy2(source, target) new_files.add(target) trigger_event('activity', { 'activity': 'export', 'path': prefix, 'progress': [n, count] }) n += 1 if mode == 'replace': for f in list(existing_files - new_files): os.unlink(f) utils.remove_empty_folders(prefix) self.export_json(os.path.join(prefix, 'library.json')) trigger_event('activity', { 'activity': 'export', 'progress': [count, count], 'path': prefix, 'status': {'code': 200, 'text': ''}, }) def update_user_peering(user_id, peered, username=None): with db.session(): u = User.get(user_id, for_update=True) if u: u.update_peering(peered, username) def remove_local_info(id): if state.nodes: trigger_event('status', { 'id': id, 'online': state.nodes.is_online(id) }) def add_local_info(data): if state.nodes: state.nodes.queue('add', data['id']) def upload(data): delay = 60 with db.session(): u = User.get(data['user']) if u: if u.is_online() and state.nodes._nodes[u.id].upload(data['items']): pass else: state.main.call_later(delay, lambda: state.tasks.queue('upload', data))