openmedialibrary/oml/item/scan.py

# -*- coding: utf-8 -*-


from datetime import datetime
import os
import shutil
import stat
import time
import unicodedata

import ox

from changelog import add_record
from item.models import File, Item
from user.models import List
from utils import remove_empty_folders, same_path
from websocket import trigger_event
import db
import media
import settings
import state

import logging
logger = logging.getLogger(__name__)

extensions = ['epub', 'pdf', 'txt', 'cbr', 'cbz']

def remove_missing(books=None):
    dirty = False
    logger.debug('remove missing')
    prefix = get_prefix()
    oml_prefix = os.path.dirname(prefix)
    if books is None:
        books = collect_books(prefix)
    with db.session():
        if os.path.exists(prefix) and os.path.exists(oml_prefix):
            logger.debug('scan for removed files')
            db_paths = []
            items = {}
            for f in File.query:
                if state.shutdown:
                    return
                path = f.fullpath()
                db_paths.append(path)
                if f.item:
                    items[path] = f.sha1
                else:
                    logger.debug('remove orphaned file %s', f)
                    state.db.session.delete(f)
                    dirty = True
            if dirty:
                state.db.session.commit()
                dirty = False
            nfc_books = {unicodedata.normalize('NFC', path) for path in books}
            removed = [
                path for path in db_paths
                if unicodedata.normalize('NFC', path) not in nfc_books
            ]
            if removed and os.path.exists(prefix) and os.path.exists(oml_prefix):
                logger.debug('%s files removed', len(removed))
                ids = [items[path] for path in removed]
                if ids:
                    orphaned = set(ids)
                    for i in Item.query.filter(Item.id.in_(ids)):
                        if state.shutdown:
                            continue
                        i.missing_file()
                        orphaned.remove(i.id)
                        dirty = True
                    if orphaned:
                        logger.debug('%s files orphaned', len(orphaned))
                        for f in File.query.filter(File.sha1.in_(orphaned)):
                            if state.shutdown:
                                continue
                            state.db.session.delete(f)
                        dirty = True
            if dirty:
                state.db.session.commit()
                state.cache.clear('group:')
            logger.debug('update filenames')
            for f in File.query:
                if state.shutdown:
                    return
                f.move()
            logger.debug('remove empty folders')
            remove_empty_folders(prefix, True)
    logger.debug('remove missing done')

def add_file(id, f, prefix, from_=None, commit=True):
    user = state.user()
    path = f[len(prefix):]
    logger.debug('%s extract metadata %s', id, path)
    data = media.metadata(f, from_)
    logger.debug('%s create file %s', id, path)
    file = File.get_or_create(id, data, path)
    item = file.item
    item.add_user(user)
    item.added = datetime.utcnow()
    logger.debug('%s load metadata %s', id, path)
    item.load_metadata()
    add_record('additem', item.id, file.info)
    add_record('edititem', item.id, item.meta)
    logger.debug('%s extract icons %s', id, path)
    item.update_icons()
    item.modified = datetime.utcnow()
    logger.debug('%s save item', id)
    item.update(commit=commit)
    logger.debug('%s added', id)
    return file

def get_prefix():
    prefs = settings.preferences
    prefix = os.path.join(os.path.expanduser(prefs['libraryPath']), 'Books' + os.sep)
    if not prefix[-1] == os.sep:
        prefix += os.sep
    assert isinstance(prefix, str)
    return prefix

def collect_books(prefix, status=None):
    logger.debug('collect books')
    books = []
    count = 0
    for root, folders, files in os.walk(prefix):
        for f in files:
            if state.shutdown:
                return []
            if f.startswith('.'):
                continue
            f = os.path.join(root, f)
            ext = f.split('.')[-1].lower()
            if ext == 'kepub':
                ext = 'epub'
            if ext in extensions:
                books.append(os.path.normpath(f))
                count += 1
            if status and not status(count):
                return None
    logger.debug('found %s books', len(books))
    return books


def run_scan():
    logger.debug('run_scan')
    prefix = get_prefix()
    books = collect_books(prefix)
    remove_missing(books)
    ids = set()

    added = 0

    with db.session():
        user = state.user()
        for f in ox.sorted_strings(books):
            if state.shutdown:
                break
            if os.path.exists(f):
                id = media.get_id(f)
                file = File.get(id)
                if file:
                    f1 = file.fullpath()
                    f2 = os.path.join(prefix, f)
                    if not same_path(f1, f2) and os.path.exists(f1) and os.path.exists(f2):
                        logger.debug('file exists in multiple locations %s', id)
                        logger.debug('"%s" vs "%s"', f1, f2)
                        os.chmod(f2, stat.S_IWRITE)
                        os.unlink(f2)
                        continue
                if id in ids:
                    logger.debug('file exists in multiple locations %s', id)
                    if file:
                        f1 = file.fullpath()
                        f2 = os.path.join(prefix, f)
                        if not same_path(f1, f2) and os.path.exists(f1) and os.path.exists(f2):
                            logger.debug('"%s" vs "%s"', f1, f2)
                            os.chmod(f2, stat.S_IWRITE)
                            os.unlink(f2)
                            continue
                else:
                    ids.add(id)

                if not file:
                    file = add_file(id, f, prefix, f)
                    added += 1
                elif user not in file.item.users:
                    item = file.item
                    item.add_user(user)
                    logger.debug('add %s to local user', id)
                    add_record('additem', item.id, file.info)
                    add_record('edititem', item.id, item.meta)
                    item.update()
                    added += 1
                if file and file.item.info.get('missing'):
                    logger.debug('missing file showed up again %s: %s', id, file.fullpath())
                    del file.item.info['missing']
                    file.item.save()
                if file and not file.item.added:
                    file.item.added = datetime.utcnow()
                    if file.item.accessed:
                        file.item.added = file.item.accessed
                    file.item.save()
        library_items = len(user.library.items)
    if state.shutdown:
        return
    if added:
        trigger_event('change', {})
        logger.debug('imported %s unknown books', added)
    if len(ids) != len(books):
        logger.debug('number of books %s vs number of ids %s', len(books), len(ids))
    if library_items != len(books):
        library_items = set([str(i) for i in user.library.items])
        gone = library_items - ids
        first = True
        if gone:
            for id in gone:
                i = Item.get(id)
                if i.info.get('mediastate') == 'transferring':
                    continue
                path = i.get_path()
                if not path or not os.path.exists(path):
                    if first:
                        logger.debug('number of books %s vs number of items in library %s', len(books), library_items)
                        first = False
                    logger.debug('cleaning orphaned record %s %s', i, path)
                    i.remove_file()
        missing = ids - library_items
        if missing:
            logger.debug('%s items in library without a record', len(missing))
    settings.server['last_scan'] = time.mktime(time.gmtime())

def change_path(old, new):
    old_icons = os.path.join(old, 'Metadata', 'icons.db')
    new_icons = os.path.join(new, 'Metadata', 'icons.db')
    if os.path.exists(old_icons) and not os.path.exists(new_icons):
        ox.makedirs(os.path.dirname(new_icons))
        shutil.move(old_icons, new_icons)
        import item.icons
        item.icons.icons = item.icons.Icons(new_icons)

    new_books = os.path.join(new, 'Books')
    if not os.path.exists(new_books):
        ox.makedirs(new)
        shutil.move(os.path.join(old, 'Books'), new_books)
        remove_empty_folders(old)
    else:
        ox.makedirs(new_books)
    run_scan()
    trigger_event('change', {})

def run_import(options=None):
    options = options or {}
    logger.debug('run_import')
    if state.activity.get('cancel'):
        logger.debug('import canceled')
        state.activity = {}
        return
    state.activity = {}
    prefs = settings.preferences
    prefix = os.path.expanduser(options.get('path', prefs['importPath']))
    if os.path.islink(prefix):
        prefix = os.path.realpath(prefix)
    if not prefix[-1] == os.sep:
        prefix += os.sep
    prefix_books = get_prefix()
    prefix_imported = os.path.join(prefix_books, '.import' + os.sep)
    if prefix_books.startswith(prefix) or prefix.startswith(prefix_books):
        error = 'invalid path'
    elif not os.path.exists(prefix):
        error = 'path not found'
    elif not os.path.isdir(prefix):
        error = 'path must be a folder'
    else:
        error = None
    if error:
        trigger_event('activity', {
            'activity': 'import',
            'progress': [0, 0],
            'status': {'code': 404, 'text': error}
        })
        state.activity = {}
        return
    listname = options.get('list')
    if listname:
        listitems = []
    assert isinstance(prefix, str)
    books = []

    def activity(count):
        if count % 100 == 0:
            state.activity = {
                'activity': 'import',
                'path': prefix,
                'progress': [0, count],
            }
            trigger_event('activity', state.activity)
        if state.activity.get('cancel'):
            logger.debug('active import canceled')
            state.activity = {}
            return False
        return True
    books = collect_books(prefix, status=activity)
    if books is None:
        return
    state.activity = {
        'activity': 'import',
        'path': prefix,
        'progress': [0, len(books)],
    }
    trigger_event('activity', state.activity)
    position = 0
    added = 0
    last = 0
    for f in ox.sorted_strings(books):
        position += 1
        if not os.path.exists(f):
            continue
        with db.session():
            id = media.get_id(f)
            file = File.get(id)
            f_import = f
            if not file:
                f = f.replace(prefix, prefix_imported)
                ox.makedirs(os.path.dirname(f))
                if options.get('mode') == 'move':
                    try:
                        shutil.move(f_import, f)
                    except:
                        shutil.copy2(f_import, f)
                else:
                    shutil.copy2(f_import, f)
                file = add_file(id, f, prefix_books, f_import)
                file.move()
                added += 1
            elif options.get('mode') == 'move':
                try:
                    os.chmod(f_import, stat.S_IWRITE)
                    os.unlink(f_import)
                except:
                    pass
            if listname:
                listitems.append(file.item.id)
        if state.activity.get('cancel'):
            state.activity = {}
            return
        if state.shutdown:
            return
        if time.time() - last > 5:
            last = time.time()
            state.activity = {
                'activity': 'import',
                'progress': [position, len(books)],
                'path': prefix,
                'added': added,
            }
            trigger_event('activity', state.activity)

    if listname and listitems:
        with db.session():
            l = List.get(settings.USER_ID, listname)
            if l:
                l.add_items(listitems)
    trigger_event('activity', {
        'activity': 'import',
        'progress': [position, len(books)],
        'path': prefix,
        'status': {'code': 200, 'text': ''},
        'added': added,
    })
    state.activity = {}
    remove_empty_folders(prefix_books)
    if options.get('mode') == 'move':
        remove_empty_folders(prefix, True)

def import_folder():
    if not (state.activity and state.activity.get('activity') == 'import'):
        import_path = settings.preferences['importPath']
        import_path = os.path.normpath(os.path.expanduser(import_path))
        import_path_base = os.path.dirname(import_path)
        if not os.path.exists(import_path) and os.path.exists(import_path_base):
            os.makedirs(import_path)
        logger.debug('scan importPath %s', import_path)
        if os.path.exists(import_path):
            run_import({
                'path': import_path,
                'mode': 'move'
            })
            remove_empty_folders(import_path, True)
    if state.main:
        state.main.call_later(10*60, lambda: state.tasks.queue('scanimport'))