openmedialibrary/oml/item/scan.py

# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from __future__ import division

import os
import shutil
from datetime import datetime
import time

import ox

from app import app
import settings
from settings import db
from item.models import File
from user.models import User, List

from changelog import Changelog

import media
from websocket import trigger_event
import state
from utils import remove_empty_folders

import logging
logger = logging.getLogger('oml.item.scan')

extensions = ['epub', 'pdf', 'txt']

def remove_missing():
    dirty = False
    with app.app_context():
        prefs = settings.preferences
        prefix = os.path.join(os.path.expanduser(prefs['libraryPath']), 'Books/')
        if os.path.exists(prefix):
            for f in File.query:
                path = f.item.get_path()
                if not os.path.exists(path):
                    dirty = True
                    f.item.remove_file()
            if dirty:
                db.session.commit()

def add_file(id, f, prefix, from_=None):
    user = state.user()
    path = f[len(prefix):]
    data = media.metadata(f, from_)
    file = File.get_or_create(id, data, path)
    item = file.item
    if 'primaryid' in file.info:
        del file.info['primaryid']
        db.session.add(file)
    if 'primaryid' in item.info:
        item.meta['primaryid'] = item.info.pop('primaryid')
        db.session.add(item)
    item.users.append(user)
    Changelog.record(user, 'additem', item.id, file.info)
    if item.meta.get('primaryid'):
        Changelog.record(user, 'edititem', item.id, dict([item.meta['primaryid']]))
    item.added = datetime.utcnow()
    item.scrape()
    item.update_icons()
    item.save()
    return file

def run_scan():
    remove_missing()
    with app.app_context():
        prefs = settings.preferences
        prefix = os.path.join(os.path.expanduser(prefs['libraryPath']), 'Books/')
        if not prefix[-1] == '/':
            prefix += '/'
        assert isinstance(prefix, unicode)
        books = []
        for root, folders, files in os.walk(prefix):
            for f in files:
                #if f.startswith('._') or f == '.DS_Store':
                if f.startswith('.'):
                    continue
                f = os.path.join(root, f)
                ext = f.split('.')[-1]
                if ext in extensions:
                    books.append(f)

        position = 0
        added = 0
        for f in ox.sorted_strings(books):
            position += 1
            id = media.get_id(f)
            file = File.get(id)
            if not file:
                file = add_file(id, f, prefix, f)
                added += 1
                trigger_event('change', {})

def run_import(options=None):
    options = options or {}

    with app.app_context():
        prefs = settings.preferences
        prefix = os.path.expanduser(options.get('path', prefs['importPath']))
        if os.path.islink(prefix):
            prefix = os.path.realpath(prefix)
        if not prefix[-1] == '/':
            prefix += '/'
        prefix_books = os.path.join(os.path.expanduser(prefs['libraryPath']), 'Books/')
        prefix_imported = os.path.join(prefix_books, 'Imported/')
        if prefix_books.startswith(prefix) or prefix.startswith(prefix_books):
            error = 'invalid path'
        elif not os.path.exists(prefix):
            error = 'path not found'
        elif not os.path.isdir(prefix):
            error = 'path must be a folder'
        else:
            error = None
        if error:
            trigger_event('activity', {
                'activity': 'import',
                'progress': [0, 0],
                'status': {'code': 404, 'text': error}
            })
            state.activity = {}
            return
        listname = options.get('list')
        if listname:
            listitems = []
        assert isinstance(prefix, unicode)
        books = []
        count = 0
        for root, folders, files in os.walk(prefix):
            for f in files:
                #if f.startswith('._') or f == '.DS_Store':
                if f.startswith('.'):
                    continue
                f = os.path.join(root, f)
                ext = f.split('.')[-1]
                if ext in extensions:
                    books.append(f)
                    count += 1
                    if state.activity.get('cancel'):
                        state.activity = {}
                        return
                    if count % 1000 == 0:
                        state.activity = {
                            'activity': 'import',
                            'path': prefix,
                            'progress': [0, count],
                        }
                        trigger_event('activity', state.activity)
        state.activity = {
            'activity': 'import',
            'path': prefix,
            'progress': [0, len(books)],
        }
        trigger_event('activity', state.activity)
        position = 0
        added = 0
        last = 0
        for f in ox.sorted_strings(books):
            position += 1
            if not os.path.exists(f):
                continue
            id = media.get_id(f)
            file = File.get(id)
            if not file:
                f_import = f
                f = f.replace(prefix, prefix_imported)
                ox.makedirs(os.path.dirname(f))
                if options.get('mode') == 'move':
                    shutil.move(f_import, f)
                else:
                    shutil.copy(f_import, f)
                file = add_file(id, f, prefix_books, f_import)
                file.move()
                added += 1
            if listname:
                listitems.append(file.item.id)
            if time.time() - last > 5:
                last = time.time()
                state.activity = {
                    'activity': 'import',
                    'progress': [position, len(books)],
                    'path': prefix,
                    'added': added,
                }
                trigger_event('activity', state.activity)

            if state.activity.get('cancel'):
                state.activity = {}
                return
        if listname and listitems:
            l = List.get(settings.USER_ID, listname)
            if l:
                l.add_items(listitems)
        trigger_event('activity', {
            'activity': 'import',
            'progress': [position, len(books)],
            'path': prefix,
            'status': {'code': 200, 'text': ''},
            'added': added,
        })
        state.activity = {}
        remove_empty_folders(prefix_books)
        if options.get('mode') == 'move':
            remove_empty_folders(prefix)