0x2620/openmedialibrary
1
0
Fork 0
Code Issues 131 Pull requests Releases Wiki Activity

shorter db sessions

Browse source
This commit is contained in:
j 2015-12-24 20:41:47 +05:30
parent ebc0b95022
commit 935f8d7f2b
4 changed files with 141 additions and 136 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
oml/downloads.py
View file

@ -47,10 +47,10 @@ class Downloads(Thread):
def run(self): def run(self):
time.sleep(2) time.sleep(2)
with db.session(): while self._running:
while self._running: with db.session():
self.download_next() self.download_next()
time.sleep(0.5) time.sleep(0.5)
def join(self): def join(self):
self._running = False self._running = False
@ -87,10 +87,13 @@ class ScrapeThread(Thread):
def run(self): def run(self):
time.sleep(2) time.sleep(2)
with db.session(): while self._running:
while self._running: wait = False
with db.session():
if not self.scrape_queue(): if not self.scrape_queue():
time.sleep(1) wait = True
if wait:
time.sleep(1)
def join(self): def join(self):
self._running = False self._running = False

31
oml/item/handlers.py
View file

@ -37,22 +37,22 @@ class EpubHandler(OMLHandler):
with db.session(): with db.session():
item = Item.get(id) item = Item.get(id)
path = item.get_path() path = item.get_path()
if not item or item.info['extension'] != 'epub' or not path: if not item or item.info['extension'] != 'epub' or not path:
self.set_status(404)
self.write('')
else:
z = zipfile.ZipFile(path)
if filename == '':
self.write('<br>\n'.join([f.filename for f in z.filelist]))
elif filename not in [f.filename for f in z.filelist]:
self.set_status(404) self.set_status(404)
self.write('') self.write('')
else: else:
z = zipfile.ZipFile(path) content_type = {
if filename == '': 'xpgt': 'application/vnd.adobe-page-template+xml'
self.write('<br>\n'.join([f.filename for f in z.filelist])) }.get(filename.split('.')[0], mimetypes.guess_type(filename)[0]) or 'text/plain'
elif filename not in [f.filename for f in z.filelist]: self.set_header('Content-Type', content_type)
self.set_status(404) self.write(z.read(filename))
self.write('')
else:
content_type = {
'xpgt': 'application/vnd.adobe-page-template+xml'
}.get(filename.split('.')[0], mimetypes.guess_type(filename)[0]) or 'text/plain'
self.set_header('Content-Type', content_type)
self.write(z.read(filename))
def serve_static(handler, path, mimetype, include_body=True, disposition=None): def serve_static(handler, path, mimetype, include_body=True, disposition=None):
handler.set_header('Content-Type', mimetype) handler.set_header('Content-Type', mimetype)
@ -112,7 +112,7 @@ class FileHandler(OMLHandler):
disposition = os.path.basename(path) disposition = os.path.basename(path)
else: else:
disposition = None disposition = None
return serve_static(self, path, mimetype, include_body, disposition=disposition) return serve_static(self, path, mimetype, include_body, disposition=disposition)
class ReaderHandler(OMLHandler): class ReaderHandler(OMLHandler):
@ -137,7 +137,8 @@ class ReaderHandler(OMLHandler):
item.timesaccessed = (item.timesaccessed or 0) + 1 item.timesaccessed = (item.timesaccessed or 0) + 1
item.update_sort() item.update_sort()
item.save() item.save()
return serve_static(self, os.path.join(settings.static_path, html), 'text/html') path = os.path.join(settings.static_path, html)
return serve_static(self, path, 'text/html')
class UploadHandler(tornado.web.RequestHandler): class UploadHandler(tornado.web.RequestHandler):

4
oml/item/icons.py
View file

@ -123,7 +123,7 @@ def get_icon_app(id, type_, size, callback):
from item.models import Item from item.models import Item
item = Item.get(id) item = Item.get(id)
if not item: if not item:
callback('') data = ''
else: else:
if type_ == 'cover' and not item.meta.get('cover'): if type_ == 'cover' and not item.meta.get('cover'):
type_ = 'preview' type_ = 'preview'
@ -145,7 +145,7 @@ def get_icon_app(id, type_, size, callback):
if size: if size:
data = icons[skey] = resize_image(data, size=size) data = icons[skey] = resize_image(data, size=size)
data = bytes(data) or '' data = bytes(data) or ''
callback(data) callback(data)
class IconHandler(tornado.web.RequestHandler): class IconHandler(tornado.web.RequestHandler):

227
oml/item/scan.py
View file

@ -65,31 +65,31 @@ def add_file(id, f, prefix, from_=None):
def run_scan(): def run_scan():
remove_missing() remove_missing()
with db.session(): prefs = settings.preferences
prefs = settings.preferences prefix = os.path.join(os.path.expanduser(prefs['libraryPath']), 'Books/')
prefix = os.path.join(os.path.expanduser(prefs['libraryPath']), 'Books/') if not prefix[-1] == '/':
if not prefix[-1] == '/': prefix += '/'
prefix += '/' assert isinstance(prefix, str)
assert isinstance(prefix, str) books = []
books = [] for root, folders, files in os.walk(prefix):
for root, folders, files in os.walk(prefix): for f in files:
for f in files:
if not state.tasks.connected:
return
#if f.startswith('._') or f == '.DS_Store':
if f.startswith('.'):
continue
f = os.path.join(root, f)
ext = f.split('.')[-1]
if ext in extensions:
books.append(f)
position = 0
added = 0
for f in ox.sorted_strings(books):
if not state.tasks.connected: if not state.tasks.connected:
return return
position += 1 #if f.startswith('._') or f == '.DS_Store':
if f.startswith('.'):
continue
f = os.path.join(root, f)
ext = f.split('.')[-1]
if ext in extensions:
books.append(f)
position = 0
added = 0
for f in ox.sorted_strings(books):
if not state.tasks.connected:
return
position += 1
with db.session():
id = media.get_id(f) id = media.get_id(f)
file = File.get(id) file = File.get(id)
if not file: if not file:
@ -100,73 +100,73 @@ def run_scan():
def run_import(options=None): def run_import(options=None):
options = options or {} options = options or {}
with db.session(): logger.debug('run_import')
logger.debug('run_import') prefs = settings.preferences
prefs = settings.preferences prefix = os.path.expanduser(options.get('path', prefs['importPath']))
prefix = os.path.expanduser(options.get('path', prefs['importPath'])) if os.path.islink(prefix):
if os.path.islink(prefix): prefix = os.path.realpath(prefix)
prefix = os.path.realpath(prefix) if not prefix[-1] == '/':
if not prefix[-1] == '/': prefix += '/'
prefix += '/' prefix_books = os.path.join(os.path.expanduser(prefs['libraryPath']), 'Books/')
prefix_books = os.path.join(os.path.expanduser(prefs['libraryPath']), 'Books/') prefix_imported = os.path.join(prefix_books, 'Imported/')
prefix_imported = os.path.join(prefix_books, 'Imported/') if prefix_books.startswith(prefix) or prefix.startswith(prefix_books):
if prefix_books.startswith(prefix) or prefix.startswith(prefix_books): error = 'invalid path'
error = 'invalid path' elif not os.path.exists(prefix):
elif not os.path.exists(prefix): error = 'path not found'
error = 'path not found' elif not os.path.isdir(prefix):
elif not os.path.isdir(prefix): error = 'path must be a folder'
error = 'path must be a folder' else:
else: error = None
error = None if error:
if error: trigger_event('activity', {
trigger_event('activity', {
'activity': 'import',
'progress': [0, 0],
'status': {'code': 404, 'text': error}
})
state.activity = {}
return
listname = options.get('list')
if listname:
listitems = []
assert isinstance(prefix, str)
books = []
count = 0
for root, folders, files in os.walk(prefix):
for f in files:
if not state.tasks.connected:
return
#if f.startswith('._') or f == '.DS_Store':
if f.startswith('.'):
continue
f = os.path.join(root, f)
ext = f.split('.')[-1]
if ext in extensions:
books.append(f)
count += 1
if state.activity.get('cancel'):
state.activity = {}
return
if count % 1000 == 0:
state.activity = {
'activity': 'import',
'path': prefix,
'progress': [0, count],
}
trigger_event('activity', state.activity)
state.activity = {
'activity': 'import', 'activity': 'import',
'path': prefix, 'progress': [0, 0],
'progress': [0, len(books)], 'status': {'code': 404, 'text': error}
} })
trigger_event('activity', state.activity) state.activity = {}
position = 0 return
added = 0 listname = options.get('list')
last = 0 if listname:
for f in ox.sorted_strings(books): listitems = []
position += 1 assert isinstance(prefix, str)
if not os.path.exists(f): books = []
count = 0
for root, folders, files in os.walk(prefix):
for f in files:
if not state.tasks.connected:
return
#if f.startswith('._') or f == '.DS_Store':
if f.startswith('.'):
continue continue
f = os.path.join(root, f)
ext = f.split('.')[-1]
if ext in extensions:
books.append(f)
count += 1
if state.activity.get('cancel'):
state.activity = {}
return
if count % 1000 == 0:
state.activity = {
'activity': 'import',
'path': prefix,
'progress': [0, count],
}
trigger_event('activity', state.activity)
state.activity = {
'activity': 'import',
'path': prefix,
'progress': [0, len(books)],
}
trigger_event('activity', state.activity)
position = 0
added = 0
last = 0
for f in ox.sorted_strings(books):
position += 1
if not os.path.exists(f):
continue
with db.session():
id = media.get_id(f) id = media.get_id(f)
file = File.get(id) file = File.get(id)
if not file: if not file:
@ -180,33 +180,34 @@ def run_import(options=None):
file = add_file(id, f, prefix_books, f_import) file = add_file(id, f, prefix_books, f_import)
file.move() file.move()
added += 1 added += 1
if listname: if listname:
listitems.append(file.item.id) listitems.append(file.item.id)
if time.time() - last > 5: if time.time() - last > 5:
last = time.time() last = time.time()
state.activity = { state.activity = {
'activity': 'import', 'activity': 'import',
'progress': [position, len(books)], 'progress': [position, len(books)],
'path': prefix, 'path': prefix,
'added': added, 'added': added,
} }
trigger_event('activity', state.activity) trigger_event('activity', state.activity)
if state.activity.get('cancel'): if state.activity.get('cancel'):
state.activity = {} state.activity = {}
return return
with db.session():
if listname and listitems: if listname and listitems:
l = List.get(settings.USER_ID, listname) l = List.get(settings.USER_ID, listname)
if l: if l:
l.add_items(listitems) l.add_items(listitems)
trigger_event('activity', { trigger_event('activity', {
'activity': 'import', 'activity': 'import',
'progress': [position, len(books)], 'progress': [position, len(books)],
'path': prefix, 'path': prefix,
'status': {'code': 200, 'text': ''}, 'status': {'code': 200, 'text': ''},
'added': added, 'added': added,
}) })
state.activity = {} state.activity = {}
remove_empty_folders(prefix_books) remove_empty_folders(prefix_books)
if options.get('mode') == 'move': if options.get('mode') == 'move':
remove_empty_folders(prefix) remove_empty_folders(prefix)