# -*- coding: utf-8 -*- # vi:si:et:sw=4:sts=4:ts=4 from datetime import datetime import base64 import hashlib import os import re import shutil import stat import unicodedata import ox from sqlalchemy.schema import CreateTable import sqlalchemy as sa from changelog import Changelog from db import MutableDict import json_pickler from .icons import icons from .person import get_sort_name, Person from queryparser import Parser from settings import config from utils import remove_empty_folders, get_ratio from websocket import trigger_event import db import media #import metaremote as meta import meta import settings import state import utils import logging logger = logging.getLogger(__name__) user_items = sa.Table('useritem', db.metadata, sa.Column('user_id', sa.String(43), sa.ForeignKey('user.id')), sa.Column('item_id', sa.String(32), sa.ForeignKey('item.id')) ) class Item(db.Model): __tablename__ = 'item' created = sa.Column(sa.DateTime()) modified = sa.Column(sa.DateTime()) id = sa.Column(sa.String(32), primary_key=True) info = sa.Column(MutableDict.as_mutable(sa.PickleType(pickler=json_pickler))) meta = sa.Column(MutableDict.as_mutable(sa.PickleType(pickler=json_pickler))) # why is this in db and not in i.e. info? added = sa.Column(sa.DateTime()) # added to local library accessed = sa.Column(sa.DateTime()) timesaccessed = sa.Column(sa.Integer()) users = sa.orm.relationship('User', secondary=user_items, backref=sa.orm.backref('items', lazy='dynamic')) @property def timestamp(self): return utils.datetime2ts(self.modified) def __repr__(self): return self.id def __init__(self, id): if isinstance(id, list): id = base64.b32encode(hashlib.sha1(''.join(id)).digest()) self.id = id self.created = datetime.utcnow() self.modified = datetime.utcnow() self.info = {} self.meta = {} @classmethod def get(cls, id): if isinstance(id, list): id = base64.b32encode(hashlib.sha1(''.join(id)).digest()) return cls.query.filter_by(id=id).first() @classmethod def get_or_create(cls, id, info=None): if isinstance(id, list): id = base64.b32encode(hashlib.sha1(''.join(id)).digest()) item = cls.query.filter_by(id=id).first() if not item: item = cls(id=id) if info: item.info = info state.db.session.add(item) state.db.session.commit() return item @classmethod def find(cls, data): return Parser(cls, user_items, Find, Sort).find(data) def add_user(self, user): if not user in self.users: self.users.append(user) l = user.library if not self in l.items: l.items.append(self) state.db.session.add(l) def json(self, keys=None): j = {} j['id'] = self.id j['created'] = self.created j['modified'] = self.modified j['timesaccessed'] = self.timesaccessed j['accessed'] = self.accessed j['added'] = self.added t = Transfer.get(self.id) if t: j['transferadded'] = t.added j['transferprogress'] = t.progress # unused and slow #j['users'] = list(map(str, list(self.users))) if self.info: meta_keys = [k for k in self.meta_keys if k != 'pages'] for key in self.info: if (not keys or key in keys) and key not in meta_keys: j[key] = self.info[key] if self.meta: j.update(self.meta) for key in self.id_keys: if key not in self.meta and key in j: del j[key] if keys: for k in list(j): if k not in keys: del j[k] for key in [k['id'] for k in settings.config['itemKeys'] if isinstance(k['type'], list)]: if key in j and not isinstance(j[key], list): j[key] = [j[key]] return j def get_path(self): f = self.files.first() return f.fullpath() if f else None def update_sort(self): update = False s = Sort.get_or_create(self.id) for key in config['itemKeys']: if key.get('sort'): value = self.json().get(key['id'], None) sort_type = key.get('sortType', key['type']) if value: if sort_type == 'integer': if isinstance(value, str): value = int(re.sub('[^0-9]', '', value)) else: value = int(value) elif sort_type == 'float': value = float(value) elif sort_type == 'date': pass elif sort_type == 'person': if not isinstance(value, list): value = [value] value = list(map(get_sort_name, value)) value = ox.sort_string('\n'.join(value)).lower() elif sort_type == 'title': if isinstance(value, dict): value = list(value.values()) if isinstance(value, list): value = ''.join(value) value = ox.get_sort_title(value) value = utils.sort_title(value).lower() else: if isinstance(value, list): value = '\n'.join(value) if value: value = str(value) value = ox.sort_string(value).lower() elif isinstance(value, list): #empty list value = None if getattr(s, key['id']) != value: setattr(s, key['id'], value) update = True if update: state.db.session.add(s) def update_find(self): def add(k, v): f = Find.query.filter_by(item_id=self.id, key=k, value=v).first() if not f: f = Find(item_id=self.id, key=k) if f.value != v: f.findvalue = unicodedata.normalize('NFKD', v).lower() f.value = v state.db.session.add(f) keys = [] for key in config['itemKeys']: if key.get('find') or key.get('filter') or key.get('type') in [['string'], 'string']: value = self.json().get(key['id'], None) if key.get('filterMap') and value: value = re.compile(key.get('filterMap')).findall(value) if value: value = value[0] if value: keys.append(key['id']) if isinstance(value, dict): value = ' '.join(list(value.values())) if not isinstance(value, list): value = [value] value = [ v.decode('utf-8') if isinstance(v, bytes) else v for v in value ] for v in value: add(key['id'], v) for f in Find.query.filter_by(item_id=self.id, key=key['id']).filter(Find.value.notin_(value)): state.db.session.delete(f) for f in Find.query.filter_by(item_id=self.id).filter(Find.key.notin_(keys)): state.db.session.delete(f) def update(self, modified=None): for key in ('mediastate', 'coverRatio', 'previewRatio'): if key in self.meta: if key not in self.info: self.info[key] = self.meta[key] del self.meta[key] users = list(map(str, list(self.users))) self.info['mediastate'] = 'available' # available, unavailable, transferring t = Transfer.get(self.id) if t and t.added and t.progress < 1: self.info['mediastate'] = 'transferring' else: self.info['mediastate'] = 'available' if settings.USER_ID in users else 'unavailable' if modified: self.modified = modified else: self.modified = datetime.utcnow() self.update_sort() self.update_find() self.save() def save(self): state.db.session.add(self) state.db.session.commit() def delete(self, commit=True): Sort.query.filter_by(item_id=self.id).delete() Transfer.query.filter_by(item_id=self.id).delete() Scrape.query.filter_by(item_id=self.id).delete() state.db.session.delete(self) icons.clear('cover:%s' % self.id) icons.clear('preview:%s' % self.id) if commit: state.db.session.commit() meta_keys = ( 'author', 'categories', 'cover', 'date', 'description', 'edition', 'isbn', 'language', 'pages', 'place', 'publisher', 'series', 'tableofcontents', 'title' ) def update_meta(self, data, modified=None, reset_from=False): update = False record = {} for key in self.meta_keys: if key in data: if self.meta.get(key) != data[key]: record[key] = data[key] self.meta[key] = data[key] update = True for key in list(self.meta): if key not in self.meta_keys: del self.meta[key] update = True if reset_from and '_from' in self.info: del self.info['_from'] update = True if update: self.update(modified) self.save() if 'cover' in record: self.update_icons() user = state.user() if record and user in self.users: Changelog.record_ts(user, modified, 'edititem', self.id, record) def edit(self, data, modified=None, reset_from=False): Scrape.query.filter_by(item_id=self.id).delete() self.update_meta(data, modified, reset_from=reset_from) for f in self.files.all(): f.move() def extract_preview(self): path = self.get_path() if path: return getattr(media, self.info['extension']).cover(path) def update_cover(self): key = 'cover:%s'%self.id cover = None if 'cover' in self.meta and self.meta['cover']: try: cover = ox.cache.read_url(self.meta['cover']) except: logger.debug('unable to read cover url %s', self.meta['cover']) cover = None if cover: icons[key] = cover self.info['coverRatio'] = get_ratio(cover) else: del icons[key] if not cover: if 'previewRatio' in self.info: self.info['coverRatio'] = self.info['previewRatio'] elif 'coverRatio' in self.info: del self.info['coverRatio'] icons.clear('cover:%s:' % self.id) def update_preview(self): key = 'preview:%s'%self.id preview = self.extract_preview() if preview: icons[key] = preview self.info['previewRatio'] = get_ratio(preview) if not 'coverRatio' in self.info: self.info['coverRatio'] = self.info['previewRatio'] else: del icons[key] if 'previewRatio' in self.info: del self.info['previewRatio'] if not preview: if 'coverRatio' in self.info: self.info['previewRatio'] = self.info['coverRatio'] elif 'previewRatio' in self.info: del self.info['previewRatio'] icons.clear('preview:%s:' % self.id) def update_icons(self): self.update_cover() self.update_preview() def load_metadata(self): ''' load metadata from user_metadata or get via isbn? ''' for key in self.meta_keys: if key in self.info: if key not in self.meta: self.meta[key] = self.info[key] if key != 'pages': del self.info[key] #FIXME get from user_meta if state.online: if 'isbn' in self.meta: data = meta.lookup('isbn', self.meta['isbn']) if data: self.meta.update(data) def queue_download(self): u = state.user() if not u in self.users: t = Transfer.get_or_create(self.id) logger.debug('queue %s for download', self.id) self.add_user(u) def save_file(self, content): u = state.user() f = File.get(self.id) content_id = media.get_id(data=content) if content_id != self.id: logger.debug('INVALID CONTENT %s vs %s', self.id, content_id) return False if not f: path = 'Downloads/%s.%s' % (self.id, self.info['extension']) info = self.info.copy() for key in ('mediastate', 'coverRatio', 'previewRatio'): if key in info: del info[key] f = File.get_or_create(self.id, info, path=path) path = self.get_path() if not os.path.exists(path): ox.makedirs(os.path.dirname(path)) with open(path, 'wb') as fd: fd.write(content) f.info = media.metadata(path) f.save() for key in ('tableofcontents', ): if key not in self.meta and key in f.info: self.meta[key] = f.info[key] if u not in self.users: self.add_user(u) t = Transfer.get_or_create(self.id) t.progress = 1 t.save() self.added = datetime.utcnow() Changelog.record(u, 'additem', self.id, f.info) Changelog.record(u, 'edititem', self.id, self.meta) self.update() f.move() self.update_icons() self.save() trigger_event('transfer', { 'id': self.id, 'progress': 1 }) return True else: logger.debug('TRIED TO SAVE EXISTING FILE!!!') t = Transfer.get_or_create(self.id) t.progress = 1 t.save() self.update() return False def remove_file(self): for f in self.files.all(): path = f.fullpath() if os.path.exists(path): os.unlink(path) remove_empty_folders(os.path.dirname(path)) state.db.session.delete(f) user = state.user() if user in self.users: self.users.remove(user) for l in self.lists.filter_by(user_id=user.id): l.items.remove(self) if not self.users: self.delete() else: self.update() Transfer.query.filter_by(item_id=self.id).delete() Changelog.record(user, 'removeitem', self.id) class Sort(db.Model): __tablename__ = 'sort' item_id = sa.Column(sa.String(32), sa.ForeignKey('item.id'), primary_key=True) item = sa.orm.relationship('Item', backref=sa.orm.backref('sort', lazy='dynamic')) def __repr__(self): return '%s_sort' % self.item_id @classmethod def get(cls, item_id): return cls.query.filter_by(item_id=item_id).first() @classmethod def get_or_create(cls, item_id): f = cls.get(item_id) if not f: f = cls(item_id=item_id) state.db.session.add(f) state.db.session.commit() return f Item.sort_keys = [] for key in config['itemKeys']: if key.get('sort'): sort_type = key.get('sortType', key['type']) if sort_type == 'integer': col = sa.Column(sa.BigInteger(), index=True) elif sort_type == 'float': col = sa.Column(sa.Float(), index=True) elif sort_type == 'date': col = sa.Column(sa.DateTime(), index=True) else: col = sa.Column(sa.String(1000), index=True) setattr(Sort, '%s' % key['id'], col) Item.sort_keys.append(key['id']) Item.id_keys = ['isbn', 'lccn', 'olid', 'oclc', 'asin'] Item.item_keys = config['itemKeys'] Item.filter_keys = [k['id'] for k in config['itemKeys'] if k.get('filter')] class Find(db.Model): __tablename__ = 'find' id = sa.Column(sa.Integer(), primary_key=True) item_id = sa.Column(sa.String(32), sa.ForeignKey('item.id')) item = sa.orm.relationship('Item', backref=sa.orm.backref('find_', lazy='dynamic')) key = sa.Column(sa.String(200), index=True) value = sa.Column(sa.Text()) findvalue = sa.Column(sa.Text(), index=True) def __repr__(self): return '%s=%s' % (self.key, self.findvalue) @classmethod def get(cls, item, key): return cls.query.filter_by(item_id=item, key=key).first() @classmethod def get_or_create(cls, item, key): f = cls.get(item, key) if not f: f = cls(item_id=item, key=key) state.db.session.add(f) state.db.session.commit() return f class File(db.Model): __tablename__ = 'file' created = sa.Column(sa.DateTime()) modified = sa.Column(sa.DateTime()) sha1 = sa.Column(sa.String(32), primary_key=True) path = sa.Column(sa.String(2048)) info = sa.Column(MutableDict.as_mutable(sa.PickleType(pickler=json_pickler))) item_id = sa.Column(sa.String(32), sa.ForeignKey('item.id')) item = sa.orm.relationship('Item', backref=sa.orm.backref('files', lazy='dynamic')) @classmethod def get(cls, sha1): return cls.query.filter_by(sha1=sha1).first() @classmethod def get_or_create(cls, sha1, info=None, path=None): f = cls.get(sha1) if not f: f = cls(sha1=sha1) if info: f.info = info if path: f.path = path f.item_id = Item.get_or_create(id=sha1, info=info).id state.db.session.add(f) state.db.session.commit() return f def __repr__(self): return self.sha1 def __init__(self, sha1): self.sha1 = sha1 self.created = datetime.utcnow() self.modified = datetime.utcnow() def fullpath(self): prefs = settings.preferences prefix = os.path.join(os.path.expanduser(prefs['libraryPath']), 'Books/') return os.path.join(prefix, self.path) def make_readonly(self): current_path = self.fullpath() if os.path.exists(current_path): mode = os.stat(current_path)[stat.ST_MODE] readonly = mode & ~stat.S_IWUSR & ~stat.S_IWGRP & ~stat.S_IWOTH if mode != readonly: os.chmod(current_path, readonly) def move(self): def format_underscores(string): return re.sub('^\.|\.$|:|/|\?|<|>', '_', string) prefs = settings.preferences prefix = os.path.join(os.path.expanduser(prefs['libraryPath']), 'Books/') j = self.item.json() current_path = self.fullpath() if not os.path.exists(current_path): logger.debug('file is missing. %s', current_path) return self.make_readonly() author = '; '.join([get_sort_name(a) for a in j.get('author', [])]) if not author: author = 'Unknown Author' if ' (Ed.)' in author: author = author.replace(' (Ed.)', '') + ' (Ed.)' title = j.get('title', 'Untitled') extension = j['extension'] if len(title) > 100: title = title[:100] title = format_underscores(title) author = format_underscores(author) publisher = j.get('publisher') if publisher: extra = ', '.join(publisher) else: extra = '' date = j.get('date') if date and len(date) >= 4: extra += ' ' + date[:4] if extra: extra = format_underscores(extra) title = '%s (%s)' % (title, extra.strip()) filename = '%s.%s' % (title, extension) first = unicodedata.normalize('NFD', author[0].upper())[0].upper() new_path = os.path.join(first, author, filename) new_path = new_path.replace('\x00', '') new_path = ox.decode_html(new_path) if self.path == new_path: return h = '' while os.path.exists(os.path.join(prefix, new_path)): h = self.sha1[:len(h)+1] filename = '%s.%s.%s' % (title, h, extension) first = unicodedata.normalize('NFD', author[0].upper())[0].upper() new_path = os.path.join(first, author, filename) if current_path == os.path.join(prefix, new_path): break if self.path != new_path: path = os.path.join(prefix, new_path) ox.makedirs(os.path.dirname(path)) shutil.move(current_path, path) self.path = new_path self.save() for folder in set(os.path.dirname(p) for p in [current_path, path]): remove_empty_folders(folder) def save(self): state.db.session.add(self) state.db.session.commit() class Scrape(db.Model): __tablename__ = 'scrape' item_id = sa.Column(sa.String(32), sa.ForeignKey('item.id'), primary_key=True) item = sa.orm.relationship('Item', backref=sa.orm.backref('scraping', lazy='dynamic')) added = sa.Column(sa.DateTime()) def __repr__(self): return '='.join(map(str, [self.item_id, self.added])) @classmethod def get(cls, item_id): return cls.query.filter_by(item_id=item_id).first() @classmethod def get_or_create(cls, item_id): t = cls.get(item_id) if not t: t = cls(item_id=item_id) t.added = datetime.utcnow() t.save() return t def save(self): state.db.session.add(self) state.db.session.commit() def remove(self): state.db.session.delete(self) state.db.session.commit() class Transfer(db.Model): __tablename__ = 'transfer' item_id = sa.Column(sa.String(32), sa.ForeignKey('item.id'), primary_key=True) item = sa.orm.relationship('Item', backref=sa.orm.backref('transfer', lazy='dynamic')) added = sa.Column(sa.DateTime()) progress = sa.Column(sa.Float()) def __repr__(self): return '='.join(map(str, [self.item_id, self.progress])) @classmethod def get(cls, item_id): return cls.query.filter_by(item_id=item_id).first() @classmethod def get_or_create(cls, item_id): t = cls.get(item_id) if not t: t = cls(item_id=item_id) t.added = datetime.utcnow() t.progress = 0 t.save() return t def save(self): state.db.session.add(self) state.db.session.commit() class Metadata(db.Model): __tablename__ = 'metadata' created = sa.Column(sa.DateTime()) modified = sa.Column(sa.DateTime()) id = sa.Column(sa.Integer(), primary_key=True) key = sa.Column(sa.String(256)) value = sa.Column(sa.String(256)) data = sa.Column(MutableDict.as_mutable(sa.PickleType(pickler=json_pickler))) def __repr__(self): return '='.join([self.key, self.value]) @property def timestamp(self): return utils.datetime2ts(self.modified) @classmethod def get(cls, key, value): return cls.query.filter_by(key=key, value=value).first() @classmethod def get_or_create(cls, key, value): m = cls.get(key, value) if not m: m = cls(key=key, value=value) m.created = datetime.utcnow() m.data = {} m.save() return m def save(self): self.modified = datetime.utcnow() state.db.session.add(self) state.db.session.commit() def reset(self): user = state.user() Changelog.record(user, 'resetmeta', self.key, self.value) state.db.session.delete(self) state.db.session.commit() self.update_items() def edit(self, data, record=True): changed = {} for key in data: if key == 'id': continue if data[key] != self.data.get(key): self.data[key] = data[key] changed[key] = data[key] if changed: self.save() if record: user = state.user() Changelog.record(user, 'editmeta', self.key, self.value, changed) return changed def update_items(self): for f in Find.query.filter_by(key=self.key, value=self.value): if f.item: f.item.update() @classmethod def load(self, key, value): m = self.get(key, value) if m: if 'id' in m.data: del m.data['id'] return m.data return {} def remove_unused_names(): used = list(set( get_sort_name(a) for i in Item.query for a in i.meta.get('author', []) )) for p in Person.query.filter(Person.sortname.notin_(used)): state.db.session.delete(p) state.db.session.commit() def update_sort_table(): current = db.get_table_columms('sort') drop_columns = list(set(current) - set(Item.sort_keys+['item_id'])) add_columns = list(set(Item.sort_keys+['item_id'])-set(current)) if drop_columns: db.drop_columns('sort', drop_columns) if add_columns: create_table = str(CreateTable(Sort.__table__).compile(db.engine)).split('\n') sql = [] for col in add_columns: add = [r for r in create_table if '\t%s ' % col in r][0].strip()[:-1] sql.append('ALTER TABLE sort ADD '+add) sql.append('CREATE INDEX ix_sort_{col} ON sort ({col})'.format(col=col)) with db.session() as s: for q in sql: s.connection().execute(q) s.commit() layout = db.get_layout() sort_indexes = [i[len('ix_sort_'):] for i in layout['indexes'] if i.startswith('ix_sort_')] sql = [] for col in set(Item.sort_keys)-set(sort_indexes): sql.append('CREATE INDEX ix_sort_{col} ON sort ({col})'.format(col=col)) if sql: with db.session() as s: for q in sql: s.connection().execute(q) s.commit()