388 lines
13 KiB
Python
388 lines
13 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
|
|
from datetime import datetime
|
|
import os
|
|
import shutil
|
|
import stat
|
|
import time
|
|
import unicodedata
|
|
|
|
import ox
|
|
|
|
from changelog import add_record
|
|
from item.models import File, Item
|
|
from user.models import List
|
|
from utils import remove_empty_folders, same_path
|
|
from websocket import trigger_event
|
|
import db
|
|
import media
|
|
import settings
|
|
import state
|
|
|
|
import logging
|
|
logger = logging.getLogger(__name__)
|
|
|
|
extensions = ['epub', 'pdf', 'txt', 'cbr', 'cbz']
|
|
|
|
def remove_missing(books=None):
|
|
dirty = False
|
|
logger.debug('remove missing')
|
|
prefix = get_prefix()
|
|
oml_prefix = os.path.dirname(prefix)
|
|
if books is None:
|
|
books = collect_books(prefix)
|
|
with db.session():
|
|
if os.path.exists(prefix) and os.path.exists(oml_prefix):
|
|
logger.debug('scan for removed files')
|
|
db_paths = []
|
|
items = {}
|
|
for f in File.query:
|
|
if state.shutdown:
|
|
return
|
|
path = f.fullpath()
|
|
db_paths.append(path)
|
|
if f.item:
|
|
items[path] = f.sha1
|
|
else:
|
|
logger.debug('remove orphaned file %s', f)
|
|
state.db.session.delete(f)
|
|
dirty = True
|
|
if dirty:
|
|
state.db.session.commit()
|
|
dirty = False
|
|
nfc_books = {unicodedata.normalize('NFC', path) for path in books}
|
|
removed = [
|
|
path for path in db_paths
|
|
if unicodedata.normalize('NFC', path) not in nfc_books
|
|
]
|
|
if removed and os.path.exists(prefix) and os.path.exists(oml_prefix):
|
|
logger.debug('%s files removed', len(removed))
|
|
ids = [items[path] for path in removed]
|
|
if ids:
|
|
orphaned = set(ids)
|
|
for i in Item.query.filter(Item.id.in_(ids)):
|
|
if state.shutdown:
|
|
continue
|
|
i.missing_file()
|
|
orphaned.remove(i.id)
|
|
dirty = True
|
|
if orphaned:
|
|
logger.debug('%s files orphaned', len(orphaned))
|
|
for f in File.query.filter(File.sha1.in_(orphaned)):
|
|
if state.shutdown:
|
|
continue
|
|
state.db.session.delete(f)
|
|
dirty = True
|
|
if dirty:
|
|
state.db.session.commit()
|
|
state.cache.clear('group:')
|
|
logger.debug('update filenames')
|
|
for f in File.query:
|
|
if state.shutdown:
|
|
return
|
|
f.move()
|
|
logger.debug('remove empty folders')
|
|
remove_empty_folders(prefix, True)
|
|
logger.debug('remove missing done')
|
|
|
|
def add_file(id, f, prefix, from_=None, commit=True):
|
|
user = state.user()
|
|
path = f[len(prefix):]
|
|
logger.debug('%s extract metadata %s', id, path)
|
|
data = media.metadata(f, from_)
|
|
logger.debug('%s create file %s', id, path)
|
|
file = File.get_or_create(id, data, path)
|
|
item = file.item
|
|
item.add_user(user)
|
|
item.added = datetime.utcnow()
|
|
logger.debug('%s load metadata %s', id, path)
|
|
item.load_metadata()
|
|
add_record('additem', item.id, file.info)
|
|
add_record('edititem', item.id, item.meta)
|
|
logger.debug('%s extract icons %s', id, path)
|
|
item.update_icons()
|
|
item.modified = datetime.utcnow()
|
|
logger.debug('%s save item', id)
|
|
item.update(commit=commit)
|
|
logger.debug('%s added', id)
|
|
return file
|
|
|
|
def get_prefix():
|
|
prefs = settings.preferences
|
|
prefix = os.path.join(os.path.expanduser(prefs['libraryPath']), 'Books' + os.sep)
|
|
if not prefix[-1] == os.sep:
|
|
prefix += os.sep
|
|
assert isinstance(prefix, str)
|
|
return prefix
|
|
|
|
def collect_books(prefix, status=None):
|
|
logger.debug('collect books')
|
|
books = []
|
|
count = 0
|
|
for root, folders, files in os.walk(prefix):
|
|
for f in files:
|
|
if state.shutdown:
|
|
return []
|
|
if f.startswith('.'):
|
|
continue
|
|
f = os.path.join(root, f)
|
|
ext = f.split('.')[-1].lower()
|
|
if ext == 'kepub':
|
|
ext = 'epub'
|
|
if ext in extensions:
|
|
books.append(os.path.normpath(f))
|
|
count += 1
|
|
if status and not status(count):
|
|
return None
|
|
logger.debug('found %s books', len(books))
|
|
return books
|
|
|
|
|
|
def run_scan():
|
|
logger.debug('run_scan')
|
|
prefix = get_prefix()
|
|
books = collect_books(prefix)
|
|
remove_missing(books)
|
|
ids = set()
|
|
|
|
added = 0
|
|
|
|
with db.session():
|
|
user = state.user()
|
|
for f in ox.sorted_strings(books):
|
|
if state.shutdown:
|
|
break
|
|
if os.path.exists(f):
|
|
id = media.get_id(f)
|
|
file = File.get(id)
|
|
if file:
|
|
f1 = file.fullpath()
|
|
f2 = os.path.join(prefix, f)
|
|
if not same_path(f1, f2) and os.path.exists(f1) and os.path.exists(f2):
|
|
logger.debug('file exists in multiple locations %s', id)
|
|
logger.debug('"%s" vs "%s"', f1, f2)
|
|
os.chmod(f2, stat.S_IWRITE)
|
|
os.unlink(f2)
|
|
continue
|
|
if id in ids:
|
|
logger.debug('file exists in multiple locations %s', id)
|
|
if file:
|
|
f1 = file.fullpath()
|
|
f2 = os.path.join(prefix, f)
|
|
if not same_path(f1, f2) and os.path.exists(f1) and os.path.exists(f2):
|
|
logger.debug('"%s" vs "%s"', f1, f2)
|
|
os.chmod(f2, stat.S_IWRITE)
|
|
os.unlink(f2)
|
|
continue
|
|
else:
|
|
ids.add(id)
|
|
|
|
if not file:
|
|
file = add_file(id, f, prefix, f)
|
|
added += 1
|
|
elif user not in file.item.users:
|
|
item = file.item
|
|
item.add_user(user)
|
|
logger.debug('add %s to local user', id)
|
|
add_record('additem', item.id, file.info)
|
|
add_record('edititem', item.id, item.meta)
|
|
item.update()
|
|
added += 1
|
|
if file and file.item.info.get('missing'):
|
|
logger.debug('missing file showed up again %s: %s', id, file.fullpath())
|
|
del file.item.info['missing']
|
|
file.item.save()
|
|
if file and not file.item.added:
|
|
file.item.added = datetime.utcnow()
|
|
if file.item.accessed:
|
|
file.item.added = file.item.accessed
|
|
file.item.save()
|
|
library_items = len(user.library.items)
|
|
if state.shutdown:
|
|
return
|
|
if added:
|
|
trigger_event('change', {})
|
|
logger.debug('imported %s unknown books', added)
|
|
if len(ids) != len(books):
|
|
logger.debug('number of books %s vs number of ids %s', len(books), len(ids))
|
|
if library_items != len(books):
|
|
library_items = set([str(i) for i in user.library.items])
|
|
gone = library_items - ids
|
|
first = True
|
|
if gone:
|
|
for id in gone:
|
|
i = Item.get(id)
|
|
if i.info.get('mediastate') == 'transferring':
|
|
continue
|
|
path = i.get_path()
|
|
if not path or not os.path.exists(path):
|
|
if first:
|
|
logger.debug('number of books %s vs number of items in library %s', len(books), library_items)
|
|
first = False
|
|
logger.debug('cleaning orphaned record %s %s', i, path)
|
|
i.remove_file()
|
|
missing = ids - library_items
|
|
if missing:
|
|
logger.debug('%s items in library without a record', len(missing))
|
|
settings.server['last_scan'] = time.mktime(time.gmtime())
|
|
|
|
def change_path(old, new):
|
|
old_icons = os.path.join(old, 'Metadata', 'icons.db')
|
|
new_icons = os.path.join(new, 'Metadata', 'icons.db')
|
|
if os.path.exists(old_icons) and not os.path.exists(new_icons):
|
|
ox.makedirs(os.path.dirname(new_icons))
|
|
shutil.move(old_icons, new_icons)
|
|
import item.icons
|
|
item.icons.icons = item.icons.Icons(new_icons)
|
|
|
|
new_books = os.path.join(new, 'Books')
|
|
if not os.path.exists(new_books):
|
|
ox.makedirs(new)
|
|
shutil.move(os.path.join(old, 'Books'), new_books)
|
|
remove_empty_folders(old)
|
|
else:
|
|
ox.makedirs(new_books)
|
|
run_scan()
|
|
trigger_event('change', {})
|
|
|
|
def run_import(options=None):
|
|
options = options or {}
|
|
logger.debug('run_import')
|
|
if state.activity.get('cancel'):
|
|
logger.debug('import canceled')
|
|
state.activity = {}
|
|
return
|
|
state.activity = {}
|
|
prefs = settings.preferences
|
|
prefix = os.path.expanduser(options.get('path', prefs['importPath']))
|
|
if os.path.islink(prefix):
|
|
prefix = os.path.realpath(prefix)
|
|
if not prefix[-1] == os.sep:
|
|
prefix += os.sep
|
|
prefix_books = get_prefix()
|
|
prefix_imported = os.path.join(prefix_books, '.import' + os.sep)
|
|
if prefix_books.startswith(prefix) or prefix.startswith(prefix_books):
|
|
error = 'invalid path'
|
|
elif not os.path.exists(prefix):
|
|
error = 'path not found'
|
|
elif not os.path.isdir(prefix):
|
|
error = 'path must be a folder'
|
|
else:
|
|
error = None
|
|
if error:
|
|
trigger_event('activity', {
|
|
'activity': 'import',
|
|
'progress': [0, 0],
|
|
'status': {'code': 404, 'text': error}
|
|
})
|
|
state.activity = {}
|
|
return
|
|
listname = options.get('list')
|
|
if listname:
|
|
listitems = []
|
|
assert isinstance(prefix, str)
|
|
books = []
|
|
|
|
def activity(count):
|
|
if count % 100 == 0:
|
|
state.activity = {
|
|
'activity': 'import',
|
|
'path': prefix,
|
|
'progress': [0, count],
|
|
}
|
|
trigger_event('activity', state.activity)
|
|
if state.activity.get('cancel'):
|
|
logger.debug('active import canceled')
|
|
state.activity = {}
|
|
return False
|
|
return True
|
|
books = collect_books(prefix, status=activity)
|
|
if books is None:
|
|
return
|
|
state.activity = {
|
|
'activity': 'import',
|
|
'path': prefix,
|
|
'progress': [0, len(books)],
|
|
}
|
|
trigger_event('activity', state.activity)
|
|
position = 0
|
|
added = 0
|
|
last = 0
|
|
for f in ox.sorted_strings(books):
|
|
position += 1
|
|
if not os.path.exists(f):
|
|
continue
|
|
with db.session():
|
|
id = media.get_id(f)
|
|
file = File.get(id)
|
|
f_import = f
|
|
if not file:
|
|
f = f.replace(prefix, prefix_imported)
|
|
ox.makedirs(os.path.dirname(f))
|
|
if options.get('mode') == 'move':
|
|
try:
|
|
shutil.move(f_import, f)
|
|
except:
|
|
shutil.copy2(f_import, f)
|
|
else:
|
|
shutil.copy2(f_import, f)
|
|
file = add_file(id, f, prefix_books, f_import)
|
|
file.move()
|
|
added += 1
|
|
elif options.get('mode') == 'move':
|
|
try:
|
|
os.chmod(f_import, stat.S_IWRITE)
|
|
os.unlink(f_import)
|
|
except:
|
|
pass
|
|
if listname:
|
|
listitems.append(file.item.id)
|
|
if state.activity.get('cancel'):
|
|
state.activity = {}
|
|
return
|
|
if state.shutdown:
|
|
return
|
|
if time.time() - last > 5:
|
|
last = time.time()
|
|
state.activity = {
|
|
'activity': 'import',
|
|
'progress': [position, len(books)],
|
|
'path': prefix,
|
|
'added': added,
|
|
}
|
|
trigger_event('activity', state.activity)
|
|
|
|
if listname and listitems:
|
|
with db.session():
|
|
l = List.get(settings.USER_ID, listname)
|
|
if l:
|
|
l.add_items(listitems)
|
|
trigger_event('activity', {
|
|
'activity': 'import',
|
|
'progress': [position, len(books)],
|
|
'path': prefix,
|
|
'status': {'code': 200, 'text': ''},
|
|
'added': added,
|
|
})
|
|
state.activity = {}
|
|
remove_empty_folders(prefix_books)
|
|
if options.get('mode') == 'move':
|
|
remove_empty_folders(prefix, True)
|
|
|
|
def import_folder():
|
|
if not (state.activity and state.activity.get('activity') == 'import'):
|
|
import_path = settings.preferences['importPath']
|
|
import_path = os.path.normpath(os.path.expanduser(import_path))
|
|
import_path_base = os.path.dirname(import_path)
|
|
if not os.path.exists(import_path) and os.path.exists(import_path_base):
|
|
os.makedirs(import_path)
|
|
logger.debug('scan importPath %s', import_path)
|
|
if os.path.exists(import_path):
|
|
run_import({
|
|
'path': import_path,
|
|
'mode': 'move'
|
|
})
|
|
remove_empty_folders(import_path, True)
|
|
if state.main:
|
|
state.main.call_later(10*60, lambda: state.tasks.queue('scanimport'))
|