openmedialibrary/oml/item/scan.py

205 lines
6.6 KiB
Python

# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from __future__ import division
import os
import shutil
from datetime import datetime
import time
import ox
from app import app
import settings
from settings import db
from item.models import File
from user.models import User, List
from changelog import Changelog
import media
from websocket import trigger_event
import state
from utils import remove_empty_folders
import logging
logger = logging.getLogger('oml.item.scan')
extensions = ['epub', 'pdf', 'txt']
def remove_missing():
dirty = False
with app.app_context():
prefs = settings.preferences
prefix = os.path.join(os.path.expanduser(prefs['libraryPath']), 'Books/')
if os.path.exists(prefix):
for f in File.query:
path = f.item.get_path()
if not os.path.exists(path):
dirty = True
f.item.remove_file()
if dirty:
db.session.commit()
def add_file(id, f, prefix, from_=None):
user = state.user()
path = f[len(prefix):]
data = media.metadata(f, from_)
file = File.get_or_create(id, data, path)
item = file.item
if 'primaryid' in file.info:
del file.info['primaryid']
db.session.add(file)
if 'primaryid' in item.info:
item.meta['primaryid'] = item.info.pop('primaryid')
db.session.add(item)
item.users.append(user)
Changelog.record(user, 'additem', item.id, file.info)
if item.meta.get('primaryid'):
Changelog.record(user, 'edititem', item.id, dict([item.meta['primaryid']]))
item.added = datetime.utcnow()
item.scrape()
item.update_icons()
item.save()
return file
def run_scan():
remove_missing()
with app.app_context():
prefs = settings.preferences
prefix = os.path.join(os.path.expanduser(prefs['libraryPath']), 'Books/')
if not prefix[-1] == '/':
prefix += '/'
assert isinstance(prefix, unicode)
books = []
for root, folders, files in os.walk(prefix):
for f in files:
#if f.startswith('._') or f == '.DS_Store':
if f.startswith('.'):
continue
f = os.path.join(root, f)
ext = f.split('.')[-1]
if ext in extensions:
books.append(f)
position = 0
added = 0
for f in ox.sorted_strings(books):
position += 1
id = media.get_id(f)
file = File.get(id)
if not file:
file = add_file(id, f, prefix, f)
added += 1
trigger_event('change', {})
def run_import(options=None):
options = options or {}
with app.app_context():
prefs = settings.preferences
prefix = os.path.expanduser(options.get('path', prefs['importPath']))
if os.path.islink(prefix):
prefix = os.path.realpath(prefix)
if not prefix[-1] == '/':
prefix += '/'
prefix_books = os.path.join(os.path.expanduser(prefs['libraryPath']), 'Books/')
prefix_imported = os.path.join(prefix_books, 'Imported/')
if prefix_books.startswith(prefix) or prefix.startswith(prefix_books):
error = 'invalid path'
elif not os.path.exists(prefix):
error = 'path not found'
elif not os.path.isdir(prefix):
error = 'path must be a folder'
else:
error = None
if error:
trigger_event('activity', {
'activity': 'import',
'progress': [0, 0],
'status': {'code': 404, 'text': error}
})
state.activity = {}
return
listname = options.get('list')
if listname:
listitems = []
assert isinstance(prefix, unicode)
books = []
count = 0
for root, folders, files in os.walk(prefix):
for f in files:
#if f.startswith('._') or f == '.DS_Store':
if f.startswith('.'):
continue
f = os.path.join(root, f)
ext = f.split('.')[-1]
if ext in extensions:
books.append(f)
count += 1
if state.activity.get('cancel'):
state.activity = {}
return
if count % 1000 == 0:
state.activity = {
'activity': 'import',
'path': prefix,
'progress': [0, count],
}
trigger_event('activity', state.activity)
state.activity = {
'activity': 'import',
'path': prefix,
'progress': [0, len(books)],
}
trigger_event('activity', state.activity)
position = 0
added = 0
last = 0
for f in ox.sorted_strings(books):
position += 1
if not os.path.exists(f):
continue
id = media.get_id(f)
file = File.get(id)
if not file:
f_import = f
f = f.replace(prefix, prefix_imported)
ox.makedirs(os.path.dirname(f))
if options.get('mode') == 'move':
shutil.move(f_import, f)
else:
shutil.copy(f_import, f)
file = add_file(id, f, prefix_books, f_import)
file.move()
added += 1
if listname:
listitems.append(file.item.id)
if time.time() - last > 5:
last = time.time()
state.activity = {
'activity': 'import',
'progress': [position, len(books)],
'path': prefix,
'added': added,
}
trigger_event('activity', state.activity)
if state.activity.get('cancel'):
state.activity = {}
return
if listname and listitems:
l = List.get(settings.USER_ID, listname)
if l:
l.add_items(listitems)
trigger_event('activity', {
'activity': 'import',
'progress': [position, len(books)],
'path': prefix,
'status': {'code': 200, 'text': ''},
'added': added,
})
state.activity = {}
remove_empty_folders(prefix_books)
if options.get('mode') == 'move':
remove_empty_folders(prefix)