openmedialibrary/oml/item/models.py

# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from __future__ import division

import os
import re
import base64
import json
import hashlib
from datetime import datetime
from StringIO import StringIO
import shutil
import logging

import Image
import ox

import settings
from settings import db, config

from person import get_sort_name

import media
import meta

import state
import utils

from oxflask.db import MutableDict

from icons import icons
from changelog import Changelog
from websocket import trigger_event
from utils import remove_empty_folders

logger = logging.getLogger('oml.item.model')

class Work(db.Model):

    created = db.Column(db.DateTime())
    modified = db.Column(db.DateTime())

    id = db.Column(db.String(32), primary_key=True)

    meta = db.Column(MutableDict.as_mutable(db.PickleType(pickler=json)))

    def __repr__(self):
        return self.id

    def __init__(self, id):
        self.id = id
        self.created = datetime.utcnow()
        self.modified = datetime.utcnow()

class Edition(db.Model):

    created = db.Column(db.DateTime())
    modified = db.Column(db.DateTime())

    id = db.Column(db.String(32), primary_key=True)

    meta = db.Column(MutableDict.as_mutable(db.PickleType(pickler=json)))

    work_id = db.Column(db.String(32), db.ForeignKey('work.id'))
    work = db.relationship('Work', backref=db.backref('editions', lazy='dynamic'))

    def __repr__(self):
        return self.id

    def __init__(self, id):
        self.id = id
        self.created = datetime.utcnow()
        self.modified = datetime.utcnow()

user_items = db.Table('useritem',
    db.Column('user_id', db.String(43), db.ForeignKey('user.id')),
    db.Column('item_id', db.String(32), db.ForeignKey('item.id'))
)

class Item(db.Model):

    created = db.Column(db.DateTime())
    modified = db.Column(db.DateTime())

    id = db.Column(db.String(32), primary_key=True)

    info = db.Column(MutableDict.as_mutable(db.PickleType(pickler=json)))
    meta = db.Column(MutableDict.as_mutable(db.PickleType(pickler=json)))

    added = db.Column(db.DateTime()) # added to local library
    accessed = db.Column(db.DateTime())
    timesaccessed = db.Column(db.Integer())

    transferadded = db.Column(db.DateTime())
    transferprogress = db.Column(db.Float())

    users = db.relationship('User', secondary=user_items,
        backref=db.backref('items', lazy='dynamic'))

    edition_id = db.Column(db.String(32), db.ForeignKey('edition.id'))
    edition = db.relationship('Edition', backref=db.backref('items', lazy='dynamic'))

    work_id = db.Column(db.String(32), db.ForeignKey('work.id'))
    work = db.relationship('Work', backref=db.backref('items', lazy='dynamic'))

    @property
    def timestamp(self):
        return utils.datetime2ts(self.modified)

    def __repr__(self):
        return self.id

    def __init__(self, id):
        if isinstance(id, list):
            id = base64.b32encode(hashlib.sha1(''.join(id)).digest())
        self.id = id
        self.created = datetime.utcnow()
        self.modified = datetime.utcnow()
        self.info = {}
        self.meta = {}

    @classmethod
    def get(cls, id):
        if isinstance(id, list):
            id = base64.b32encode(hashlib.sha1(''.join(id)).digest())
        return cls.query.filter_by(id=id).first()

    @classmethod
    def get_or_create(cls, id, info=None):
        if isinstance(id, list):
            id = base64.b32encode(hashlib.sha1(''.join(id)).digest())
        item = cls.query.filter_by(id=id).first()
        if not item:
            item = cls(id=id)
            if info:
                item.info = info
            db.session.add(item)
            db.session.commit()
        return item

    def json(self, keys=None):
        j  = {}
        j['id'] = self.id
        j['created'] = self.created
        j['modified'] = self.modified
        j['timesaccessed'] = self.timesaccessed
        j['accessed'] = self.accessed
        j['added'] = self.added
        j['transferadded'] = self.transferadded
        j['transferprogress'] = self.transferprogress
        j['users'] = map(str, list(self.users))

        if self.info:
            j.update(self.info)
        if self.meta:
            j.update(self.meta)

        for key in self.id_keys + ['primaryid']:
            if key not in self.meta and key in j:
                del j[key]
        '''
        if self.work_id:
            j['work'] = {
                'olid': self.work_id
            }
            j['work'].update(self.work.meta)
        '''
        if keys:
            for k in j.keys():
                if k not in keys:
                    del j[k]
        return j

    def get_path(self):
        f = self.files.first()
        return f.fullpath() if f else None

    def update_sort(self):
        for key in config['itemKeys']:
            if key.get('sort'):
                value = self.json().get(key['id'], None)
                sort_type = key.get('sortType', key['type'])
                if value:
                    if sort_type == 'integer':
                        value = int(value)
                    elif sort_type == 'float':
                        value = float(value)
                    elif sort_type == 'date':
                        pass
                    elif sort_type == 'name':
                        if not isinstance(value, list):
                            value = [value]
                        value = map(get_sort_name, value)
                        value = ox.sort_string(u'\n'.join(value))
                    elif sort_type == 'title':
                        value = utils.sort_title(value).lower()
                    else:
                        if isinstance(value, list):
                            value = u'\n'.join(value)
                        if value:
                            value = unicode(value)
                            value = ox.sort_string(value).lower()
                elif isinstance(value, list): #empty list
                    value = ''
                setattr(self, 'sort_%s' % key['id'], value)

    def update_find(self):

        def add(k, v):
            f = Find(item_id=self.id, key=k)
            f.findvalue = v.lower().strip()
            f.value = v
            db.session.add(f)

        for key in config['itemKeys']:
            if key.get('find') or key.get('filter') or key.get('type') in [['string'], 'string']:
                value = self.json().get(key['id'], None)
                if key.get('filterMap') and value:
                    value = re.compile(key.get('filterMap')).findall(value)
                    if value: value = value[0]
                if value:
                    Find.query.filter_by(item_id=self.id, key=key['id']).delete()
                    if not isinstance(value, list):
                        value = [value]
                    for v in value:
                        add(key['id'], v)
                else:
                    f = Find.get(self.id, key['id'])
                    if f:
                        db.session.delete(f)

    def update_lists(self):
        Find.query.filter_by(item_id=self.id, key='list').delete()
        for p in self.users:
            f = Find()
            f.item_id = self.id
            f.key = 'list'
            if p.id == settings.USER_ID:
                f.value = ':'
            else:
                f.value = '%s:' % p.id
            db.session.add(f)
        for l in self.lists:
            f = Find()
            f.item_id = self.id
            f.key = 'list'
            f.value = l.find_id
            db.session.add(f)

    def update(self):
        for key in ('mediastate', 'coverRatio', 'previewRatio'):
            if key in self.meta:
                if key not in self.info:
                    self.info[key] = self.meta[key]
                del self.meta[key]
        users = map(str, list(self.users))
        self.info['mediastate'] = 'available' # available, unavailable, transferring
        if self.transferadded and self.transferprogress < 1:
            self.info['mediastate'] = 'transferring'
        else:
            self.info['mediastate'] = 'available' if settings.USER_ID in users else 'unavailable'
        #fixme: also load metadata for other ids?
        if 'primaryid' in self.meta:
            self.meta.update(Metadata.load(*self.meta['primaryid']))
        self.update_sort()
        self.update_find()
        self.update_lists()
        #self.modified = datetime.utcnow()
        self.save()

    def save(self):
        db.session.add(self)
        db.session.commit()

    meta_keys = ('title', 'author', 'date', 'publisher', 'edition', 'language')

    def update_meta(self, data):
        update = False
        record = {}
        for key in self.meta_keys:
            if key in data:
                self.meta[key] = data[key]
                record[key] = data[key]
                update = True
        for key in self.meta.keys():
            if key not in self.meta_keys:
                del self.meta[key]
                update = True
        if update:
            self.update()
            self.modified = datetime.utcnow()
            self.save()
            user = state.user()
            if user in self.users:
                Changelog.record(user, 'edititem', self.id, record)

    def update_primaryid(self, key=None, id=None):
        if key is None and id is None:
            if 'primaryid' not in self.meta:
                return
            else:
                key = self.meta['primaryid'][0]
        record = {}
        if id:
            self.meta[key] = id
            self.meta['primaryid'] = [key, id]
            record[key] = id
        else:
            if key in self.meta:
                del self.meta[key]
            if 'primaryid' in self.meta:
                del self.meta['primaryid']
            record[key] = ''
        for k in self.id_keys:
            if k != key:
                if k in self.meta:
                    del self.meta[k]
        logger.debug('set primaryid %s %s', key, id)

        # get metadata from external resources
        self.scrape()
        self.update()
        self.update_icons()
        self.modified = datetime.utcnow()
        self.save()
        user = state.user()
        if user in self.users:
            Changelog.record(user, 'edititem', self.id, record)

    def edit_metadata(self, data):
        if 'primaryid' in self.meta:
            m = Metadata.get_or_create(*self.meta['primaryid'])
            m.edit(data)
            m.update_items()
        else:
            self.update_meta(data)

    def extract_preview(self):
        path = self.get_path()
        if path:
            return getattr(media, self.info['extension']).cover(path)

    def update_icons(self):
        def get_ratio(data):
            img = Image.open(StringIO(data))
            return img.size[0]/img.size[1]
        key = 'cover:%s'%self.id
        cover = None
        if 'cover' in self.meta and self.meta['cover']:
            cover = ox.cache.read_url(self.meta['cover'])
            #covers[self.id] = requests.get(self.meta['cover']).content
            if cover:
                icons[key] = cover
                self.info['coverRatio'] = get_ratio(cover)
        else:
            if icons[key]:
                del icons[key]
        path = self.get_path()
        key = 'preview:%s'%self.id
        if path:
            preview = self.extract_preview()
            if preview:
                icons[key] = preview
                self.info['previewRatio'] = get_ratio(preview)
                if not cover:
                    self.info['coverRatio'] = self.info['previewRatio']
        elif cover:
            self.info['previewRatio'] = self.info['coverRatio']
        for key in ('cover', 'preview'):
            key = '%s:%s' % (key, self.id)
            for resolution in (128, 256, 512):
                del icons['%s:%s' % (key, resolution)]

    def scrape(self):
        primaryid = self.meta.get('primaryid')
        logger.debug('scrape %s', primaryid)
        if primaryid:
            m = meta.lookup(*primaryid)
            m['primaryid'] = primaryid
            self.meta = m
        self.update()

    def queue_download(self):
        u = state.user()
        if not u in self.users:
            logger.debug('queue %s for download', self.id)
            self.transferprogress = 0
            self.transferadded = datetime.utcnow()
            self.users.append(u)
        else:
            logger.debug('%s already queued for download? %s %s', self.id, self.transferprogress, self.transferadded)

    def save_file(self, content):
        u = state.user()
        f = File.get(self.id)
        content_id = media.get_id(data=content)
        if content_id != self.id:
            logger.debug('INVALID CONTENT %s vs %s', self.id, content_id)
            return False
        if not f:
            path = 'Downloads/%s.%s' % (self.id, self.info['extension'])
            f = File.get_or_create(self.id, self.info, path=path)
            path = self.get_path()
            if not os.path.exists(path):
                ox.makedirs(os.path.dirname(path))
                with open(path, 'wb') as fd:
                    fd.write(content)
                if u not in self.users:
                    self.users.append(u)
                self.transferprogress = 1
                self.added = datetime.utcnow()
                Changelog.record(u, 'additem', self.id, self.info)
                self.update()
                f.move()
                self.update_icons()
                trigger_event('transfer', {
                    'id': self.id, 'progress': 1
                })
                return True
        else:
            logger.debug('TRIED TO SAVE EXISTING FILE!!!')
            self.transferprogress = 1
            self.update()
        return False

    def remove_file(self):
        for f in self.files.all():
            path = f.fullpath()
            logger.debug('remove file %s', path)
            if os.path.exists(path):
                os.unlink(path)
                remove_empty_folders(os.path.dirname(path))
            db.session.delete(f)
        user = state.user()
        if user in self.users:
            self.users.remove(user)
        for l in self.lists.filter_by(user_id=user.id):
            l.items.remove(self)
        db.session.commit()
        if not self.users:
            db.session.delete(self)
        else:
            self.update()
        Changelog.record(user, 'removeitem', self.id)

for key in config['itemKeys']:
    if key.get('sort'):
        sort_type = key.get('sortType', key['type'])
        if sort_type == 'integer':
            col = db.Column(db.BigInteger(), index=True)
        elif sort_type == 'float':
            col = db.Column(db.Float(), index=True)
        elif sort_type == 'date':
            col = db.Column(db.DateTime(), index=True)
        else:
            col = db.Column(db.String(1000), index=True)
        setattr(Item, 'sort_%s' % key['id'], col)

Item.id_keys = ['isbn', 'lccn', 'olid', 'oclc', 'asin']
Item.item_keys = config['itemKeys'] 
Item.filter_keys = [k['id'] for k in config['itemKeys'] if k.get('filter')]

class Find(db.Model):
    id = db.Column(db.Integer(), primary_key=True)
    item_id = db.Column(db.String(32), db.ForeignKey('item.id'))
    item = db.relationship('Item', backref=db.backref('find', lazy='dynamic'))
    key = db.Column(db.String(200), index=True)
    value = db.Column(db.Text())
    findvalue = db.Column(db.Text())

    def __repr__(self):
        return (u'%s=%s' % (self.key, self.findvalue)).encode('utf-8')

    @classmethod
    def get(cls, item, key):
        return cls.query.filter_by(item_id=item, key=key).first()

    @classmethod
    def get_or_create(cls, item, key):
        f = cls.get(item, key)
        if not f:
            f = cls(item_id=item, key=key)
            db.session.add(f)
            db.session.commit()
        return f

class File(db.Model):

    created = db.Column(db.DateTime())
    modified = db.Column(db.DateTime())

    sha1 = db.Column(db.String(32), primary_key=True)
    path = db.Column(db.String(2048))

    info = db.Column(MutableDict.as_mutable(db.PickleType(pickler=json)))

    item_id = db.Column(db.String(32), db.ForeignKey('item.id'))
    item = db.relationship('Item', backref=db.backref('files', lazy='dynamic'))

    @classmethod
    def get(cls, sha1):
        return cls.query.filter_by(sha1=sha1).first()

    @classmethod
    def get_or_create(cls, sha1, info=None, path=None):
        f = cls.get(sha1)
        if not f:
            f = cls(sha1=sha1)
            if info:
                f.info = info
            if path:
                f.path = path
            f.item_id = Item.get_or_create(id=sha1, info=info).id
            db.session.add(f)
            db.session.commit()
        return f

    def __repr__(self):
        return self.sha1

    def __init__(self, sha1):
        self.sha1 = sha1
        self.created = datetime.utcnow()
        self.modified = datetime.utcnow()

    def fullpath(self):
        prefs = settings.preferences
        prefix = os.path.join(os.path.expanduser(prefs['libraryPath']), 'Books/')
        return os.path.join(prefix, self.path)

    def move(self):
        def format_underscores(string):
            return re.sub('^\.|\.$|:|/|\?|<|>', '_', string)
        prefs = settings.preferences
        prefix = os.path.join(os.path.expanduser(prefs['libraryPath']), 'Books/')
        j = self.item.json()

        current_path = self.fullpath()
        author = '; '.join([ox.canonical_name(a) for a in j.get('author', [])])
        if not author:
            author = 'Unknown Author'
        title = j.get('title', 'Untitled')
        extension = j['extension']

        if len(title) > 100:
            title = title[:100]

        title = format_underscores(title)
        author = format_underscores(author)

        filename = '%s.%s' % (title, extension)
        new_path = os.path.join(author[0].upper(), author, filename)
        if self.path == new_path:
            return
        h = ''
        while os.path.exists(os.path.join(prefix, new_path)):
            h = self.sha1[:len(h)+1]
            filename = '%s.%s.%s' % (title, h, extension)
            new_path = os.path.join(author[0].upper(), author, filename)
            if current_path == os.path.join(prefix, new_path):
                break
        if self.path != new_path:
            path = os.path.join(prefix, new_path)
            ox.makedirs(os.path.dirname(path))
            shutil.move(current_path, path)
            self.path = new_path
            self.save()

    def save(self):
        db.session.add(self)
        db.session.commit()

class Metadata(db.Model):

    created = db.Column(db.DateTime())
    modified = db.Column(db.DateTime())

    id = db.Column(db.Integer(), primary_key=True)

    key = db.Column(db.String(256))
    value = db.Column(db.String(256))

    data = db.Column(MutableDict.as_mutable(db.PickleType(pickler=json)))

    def __repr__(self):
        return '='.join([self.key, self.value])

    @property
    def timestamp(self):
        return utils.datetime2ts(self.modified)

    @classmethod
    def get(cls, key, value):
        return cls.query.filter_by(key=key, value=value).first()

    @classmethod
    def get_or_create(cls, key, value):
        m = cls.get(key, value)
        if not m:
            m = cls(key=key, value=value)
            m.created = datetime.utcnow()
            m.data = {}
            m.save()
        return m

    def save(self):
        self.modified = datetime.utcnow()
        db.session.add(self)
        db.session.commit()

    def reset(self):
        user = state.user()
        Changelog.record(user, 'resetmeta', self.key, self.value)
        db.session.delete(self)
        db.session.commit()
        self.update_items()

    def edit(self, data):
        changed = {}
        for key in data:
            if key not in data or data[key] != self.data.get(key):
                self.data[key] = data[key]
                changed[key] = data[key]
        if changed:
            self.save()
            user = state.user()
            Changelog.record(user, 'editmeta', self.key, self.value, changed)
        return changed

    def update_items(self):
        for f in Find.query.filter_by(key=self.key, value=self.value):
            f.item.scrape()

    @classmethod
    def load(self, key, value):
        m = self.get(key, value)
        if m:
            return m.data
        return {}