# -*- coding: utf-8 -*- # vi:si:et:sw=4:sts=4:ts=4 from datetime import datetime import os import shutil import stat import time import ox from changelog import Changelog from item.models import File, Item from user.models import List from utils import remove_empty_folders from websocket import trigger_event import db import media import settings import state import logging logger = logging.getLogger(__name__) extensions = ['epub', 'pdf', 'txt', 'cbr', 'cbz'] def remove_missing(books=None): dirty = False logger.debug('remove missing') prefix = get_prefix() if books is None: books = collect_books(prefix) with db.session(): if os.path.exists(prefix): logger.debug('scan for removed files') db_paths = [] items = {} for f in File.query: if state.shutdown: return path = f.fullpath() db_paths.append(path) if f.item: items[path] = f.sha1 else: state.db.session.delete(f) dirty = True if dirty: state.db.session.commit() dirty = False removed = set(db_paths) - set(books) if removed: logger.debug('%s files removed', len(removed)) ids = [items[path] for path in removed] orphaned = set(ids) for i in Item.query.filter(Item.id.in_(ids)): i.remove_file() orphaned.remove(i.id) dirty = True if orphaned: logger.debug('%s files orphaned', len(orphaned)) for f in File.query.filter(File.sha1.in_(orphaned)): state.db.session.delete(f) dirty = True if dirty: state.db.session.commit() state.cache.clear('group:') logger.debug('update filenames') for f in File.query: if state.shutdown: return f.move() logger.debug('remove empty folders') remove_empty_folders(prefix, True) logger.debug('remove missing done') def add_file(id, f, prefix, from_=None, commit=True): user = state.user() path = f[len(prefix):] logger.debug('%s extract metadata %s', id, path) data = media.metadata(f, from_) logger.debug('%s create file %s', id, path) file = File.get_or_create(id, data, path) item = file.item item.add_user(user) item.added = datetime.utcnow() logger.debug('%s load metadata %s', id, path) item.load_metadata() Changelog.record(user, 'additem', item.id, file.info) Changelog.record(user, 'edititem', item.id, item.meta) logger.debug('%s extract icons %s', id, path) item.update_icons() item.modified = datetime.utcnow() logger.debug('%s save item', id) item.update(commit=commit) logger.debug('%s added', id) return file def get_prefix(): prefs = settings.preferences prefix = os.path.join(os.path.expanduser(prefs['libraryPath']), 'Books' + os.sep) if not prefix[-1] == os.sep: prefix += os.sep assert isinstance(prefix, str) return prefix def collect_books(prefix, status=None): logger.debug('collect books') books = [] count = 0 for root, folders, files in os.walk(prefix): for f in files: if state.shutdown: return [] if f.startswith('.'): continue f = os.path.join(root, f) ext = f.split('.')[-1].lower() if ext == 'kepub': ext = 'epub' if ext in extensions: books.append(os.path.normpath(f)) count += 1 if status and not status(count): return None logger.debug('found %s books', len(books)) return books def run_scan(): logger.debug('run_scan') prefix = get_prefix() books = collect_books(prefix) remove_missing(books) added = 0 with db.session(): for f in ox.sorted_strings(books): if state.shutdown: break if os.path.exists(f): id = media.get_id(f) file = File.get(id) if not file: file = add_file(id, f, prefix, f) added += 1 if added: trigger_event('change', {}) logger.debug('imported %s unknown books', added) def change_path(old, new): new_books = os.path.join(new, 'Books') if not os.path.exists(new_books): ox.makedirs(new) shutil.move(os.path.join(old, 'Books'), new_books) remove_empty_folders(old) else: ox.makedirs(new_books) run_scan() trigger_event('change', {}) def run_import(options=None): options = options or {} logger.debug('run_import') if state.activity.get('cancel'): logger.debug('import canceled') state.activity = {} return state.activity = {} prefs = settings.preferences prefix = os.path.expanduser(options.get('path', prefs['importPath'])) if os.path.islink(prefix): prefix = os.path.realpath(prefix) if not prefix[-1] == os.sep: prefix += os.sep prefix_books = get_prefix() prefix_imported = os.path.join(prefix_books, '.import' + os.sep) if prefix_books.startswith(prefix) or prefix.startswith(prefix_books): error = 'invalid path' elif not os.path.exists(prefix): error = 'path not found' elif not os.path.isdir(prefix): error = 'path must be a folder' else: error = None if error: trigger_event('activity', { 'activity': 'import', 'progress': [0, 0], 'status': {'code': 404, 'text': error} }) state.activity = {} return listname = options.get('list') if listname: listitems = [] assert isinstance(prefix, str) books = [] def activity(count): if count % 100 == 0: state.activity = { 'activity': 'import', 'path': prefix, 'progress': [0, count], } trigger_event('activity', state.activity) if state.activity.get('cancel'): logger.debug('active import canceled') state.activity = {} return False return True books = collect_books(prefix, status=activity) if books is None: return state.activity = { 'activity': 'import', 'path': prefix, 'progress': [0, len(books)], } trigger_event('activity', state.activity) position = 0 added = 0 last = 0 for f in ox.sorted_strings(books): position += 1 if not os.path.exists(f): continue with db.session(): id = media.get_id(f) file = File.get(id) f_import = f if not file: f = f.replace(prefix, prefix_imported) ox.makedirs(os.path.dirname(f)) if options.get('mode') == 'move': try: shutil.move(f_import, f) except: shutil.copy2(f_import, f) else: shutil.copy2(f_import, f) file = add_file(id, f, prefix_books, f_import) file.move() added += 1 elif options.get('mode') == 'move': try: os.chmod(f_import, stat.S_IWRITE) os.unlink(f_import) except: pass if listname: listitems.append(file.item.id) if state.activity.get('cancel'): state.activity = {} return if state.shutdown: return if time.time() - last > 5: last = time.time() state.activity = { 'activity': 'import', 'progress': [position, len(books)], 'path': prefix, 'added': added, } trigger_event('activity', state.activity) if listname and listitems: with db.session(): l = List.get(settings.USER_ID, listname) if l: l.add_items(listitems) trigger_event('activity', { 'activity': 'import', 'progress': [position, len(books)], 'path': prefix, 'status': {'code': 200, 'text': ''}, 'added': added, }) state.activity = {} remove_empty_folders(prefix_books) if options.get('mode') == 'move': remove_empty_folders(prefix, True) def import_folder(): if not (state.activity and state.activity.get('activity') == 'import'): import_path = settings.preferences['importPath'] import_path = os.path.normpath(os.path.expanduser(import_path)) import_path_base = os.path.dirname(import_path) if not os.path.exists(import_path) and os.path.exists(import_path_base): os.makedirs(import_path) logger.debug('scan importPath %s', import_path) if os.path.exists(import_path): run_import({ 'path': import_path, 'mode': 'move' }) remove_empty_folders(import_path, True) if state.main: state.main.call_later(10*60, lambda: state.tasks.queue('scanimport'))