From 935f8d7f2b9ce0ec9ad29c43c101237cd49f1a22 Mon Sep 17 00:00:00 2001 From: j Date: Thu, 24 Dec 2015 20:41:47 +0530 Subject: [PATCH] shorter db sessions --- oml/downloads.py | 15 +-- oml/item/handlers.py | 31 +++--- oml/item/icons.py | 4 +- oml/item/scan.py | 227 ++++++++++++++++++++++--------------------- 4 files changed, 141 insertions(+), 136 deletions(-) diff --git a/oml/downloads.py b/oml/downloads.py index 5169985..a44b18a 100644 --- a/oml/downloads.py +++ b/oml/downloads.py @@ -47,10 +47,10 @@ class Downloads(Thread): def run(self): time.sleep(2) - with db.session(): - while self._running: + while self._running: + with db.session(): self.download_next() - time.sleep(0.5) + time.sleep(0.5) def join(self): self._running = False @@ -87,10 +87,13 @@ class ScrapeThread(Thread): def run(self): time.sleep(2) - with db.session(): - while self._running: + while self._running: + wait = False + with db.session(): if not self.scrape_queue(): - time.sleep(1) + wait = True + if wait: + time.sleep(1) def join(self): self._running = False diff --git a/oml/item/handlers.py b/oml/item/handlers.py index a2f3dcd..ca3733a 100644 --- a/oml/item/handlers.py +++ b/oml/item/handlers.py @@ -37,22 +37,22 @@ class EpubHandler(OMLHandler): with db.session(): item = Item.get(id) path = item.get_path() - if not item or item.info['extension'] != 'epub' or not path: + if not item or item.info['extension'] != 'epub' or not path: + self.set_status(404) + self.write('') + else: + z = zipfile.ZipFile(path) + if filename == '': + self.write('
\n'.join([f.filename for f in z.filelist])) + elif filename not in [f.filename for f in z.filelist]: self.set_status(404) self.write('') else: - z = zipfile.ZipFile(path) - if filename == '': - self.write('
\n'.join([f.filename for f in z.filelist])) - elif filename not in [f.filename for f in z.filelist]: - self.set_status(404) - self.write('') - else: - content_type = { - 'xpgt': 'application/vnd.adobe-page-template+xml' - }.get(filename.split('.')[0], mimetypes.guess_type(filename)[0]) or 'text/plain' - self.set_header('Content-Type', content_type) - self.write(z.read(filename)) + content_type = { + 'xpgt': 'application/vnd.adobe-page-template+xml' + }.get(filename.split('.')[0], mimetypes.guess_type(filename)[0]) or 'text/plain' + self.set_header('Content-Type', content_type) + self.write(z.read(filename)) def serve_static(handler, path, mimetype, include_body=True, disposition=None): handler.set_header('Content-Type', mimetype) @@ -112,7 +112,7 @@ class FileHandler(OMLHandler): disposition = os.path.basename(path) else: disposition = None - return serve_static(self, path, mimetype, include_body, disposition=disposition) + return serve_static(self, path, mimetype, include_body, disposition=disposition) class ReaderHandler(OMLHandler): @@ -137,7 +137,8 @@ class ReaderHandler(OMLHandler): item.timesaccessed = (item.timesaccessed or 0) + 1 item.update_sort() item.save() - return serve_static(self, os.path.join(settings.static_path, html), 'text/html') + path = os.path.join(settings.static_path, html) + return serve_static(self, path, 'text/html') class UploadHandler(tornado.web.RequestHandler): diff --git a/oml/item/icons.py b/oml/item/icons.py index 8413121..ede367f 100644 --- a/oml/item/icons.py +++ b/oml/item/icons.py @@ -123,7 +123,7 @@ def get_icon_app(id, type_, size, callback): from item.models import Item item = Item.get(id) if not item: - callback('') + data = '' else: if type_ == 'cover' and not item.meta.get('cover'): type_ = 'preview' @@ -145,7 +145,7 @@ def get_icon_app(id, type_, size, callback): if size: data = icons[skey] = resize_image(data, size=size) data = bytes(data) or '' - callback(data) + callback(data) class IconHandler(tornado.web.RequestHandler): diff --git a/oml/item/scan.py b/oml/item/scan.py index 8068a5a..ae9665d 100644 --- a/oml/item/scan.py +++ b/oml/item/scan.py @@ -65,31 +65,31 @@ def add_file(id, f, prefix, from_=None): def run_scan(): remove_missing() - with db.session(): - prefs = settings.preferences - prefix = os.path.join(os.path.expanduser(prefs['libraryPath']), 'Books/') - if not prefix[-1] == '/': - prefix += '/' - assert isinstance(prefix, str) - books = [] - for root, folders, files in os.walk(prefix): - for f in files: - if not state.tasks.connected: - return - #if f.startswith('._') or f == '.DS_Store': - if f.startswith('.'): - continue - f = os.path.join(root, f) - ext = f.split('.')[-1] - if ext in extensions: - books.append(f) - - position = 0 - added = 0 - for f in ox.sorted_strings(books): + prefs = settings.preferences + prefix = os.path.join(os.path.expanduser(prefs['libraryPath']), 'Books/') + if not prefix[-1] == '/': + prefix += '/' + assert isinstance(prefix, str) + books = [] + for root, folders, files in os.walk(prefix): + for f in files: if not state.tasks.connected: return - position += 1 + #if f.startswith('._') or f == '.DS_Store': + if f.startswith('.'): + continue + f = os.path.join(root, f) + ext = f.split('.')[-1] + if ext in extensions: + books.append(f) + + position = 0 + added = 0 + for f in ox.sorted_strings(books): + if not state.tasks.connected: + return + position += 1 + with db.session(): id = media.get_id(f) file = File.get(id) if not file: @@ -100,73 +100,73 @@ def run_scan(): def run_import(options=None): options = options or {} - with db.session(): - logger.debug('run_import') - prefs = settings.preferences - prefix = os.path.expanduser(options.get('path', prefs['importPath'])) - if os.path.islink(prefix): - prefix = os.path.realpath(prefix) - if not prefix[-1] == '/': - prefix += '/' - prefix_books = os.path.join(os.path.expanduser(prefs['libraryPath']), 'Books/') - prefix_imported = os.path.join(prefix_books, 'Imported/') - if prefix_books.startswith(prefix) or prefix.startswith(prefix_books): - error = 'invalid path' - elif not os.path.exists(prefix): - error = 'path not found' - elif not os.path.isdir(prefix): - error = 'path must be a folder' - else: - error = None - if error: - trigger_event('activity', { - 'activity': 'import', - 'progress': [0, 0], - 'status': {'code': 404, 'text': error} - }) - state.activity = {} - return - listname = options.get('list') - if listname: - listitems = [] - assert isinstance(prefix, str) - books = [] - count = 0 - for root, folders, files in os.walk(prefix): - for f in files: - if not state.tasks.connected: - return - #if f.startswith('._') or f == '.DS_Store': - if f.startswith('.'): - continue - f = os.path.join(root, f) - ext = f.split('.')[-1] - if ext in extensions: - books.append(f) - count += 1 - if state.activity.get('cancel'): - state.activity = {} - return - if count % 1000 == 0: - state.activity = { - 'activity': 'import', - 'path': prefix, - 'progress': [0, count], - } - trigger_event('activity', state.activity) - state.activity = { + logger.debug('run_import') + prefs = settings.preferences + prefix = os.path.expanduser(options.get('path', prefs['importPath'])) + if os.path.islink(prefix): + prefix = os.path.realpath(prefix) + if not prefix[-1] == '/': + prefix += '/' + prefix_books = os.path.join(os.path.expanduser(prefs['libraryPath']), 'Books/') + prefix_imported = os.path.join(prefix_books, 'Imported/') + if prefix_books.startswith(prefix) or prefix.startswith(prefix_books): + error = 'invalid path' + elif not os.path.exists(prefix): + error = 'path not found' + elif not os.path.isdir(prefix): + error = 'path must be a folder' + else: + error = None + if error: + trigger_event('activity', { 'activity': 'import', - 'path': prefix, - 'progress': [0, len(books)], - } - trigger_event('activity', state.activity) - position = 0 - added = 0 - last = 0 - for f in ox.sorted_strings(books): - position += 1 - if not os.path.exists(f): + 'progress': [0, 0], + 'status': {'code': 404, 'text': error} + }) + state.activity = {} + return + listname = options.get('list') + if listname: + listitems = [] + assert isinstance(prefix, str) + books = [] + count = 0 + for root, folders, files in os.walk(prefix): + for f in files: + if not state.tasks.connected: + return + #if f.startswith('._') or f == '.DS_Store': + if f.startswith('.'): continue + f = os.path.join(root, f) + ext = f.split('.')[-1] + if ext in extensions: + books.append(f) + count += 1 + if state.activity.get('cancel'): + state.activity = {} + return + if count % 1000 == 0: + state.activity = { + 'activity': 'import', + 'path': prefix, + 'progress': [0, count], + } + trigger_event('activity', state.activity) + state.activity = { + 'activity': 'import', + 'path': prefix, + 'progress': [0, len(books)], + } + trigger_event('activity', state.activity) + position = 0 + added = 0 + last = 0 + for f in ox.sorted_strings(books): + position += 1 + if not os.path.exists(f): + continue + with db.session(): id = media.get_id(f) file = File.get(id) if not file: @@ -180,33 +180,34 @@ def run_import(options=None): file = add_file(id, f, prefix_books, f_import) file.move() added += 1 - if listname: - listitems.append(file.item.id) - if time.time() - last > 5: - last = time.time() - state.activity = { - 'activity': 'import', - 'progress': [position, len(books)], - 'path': prefix, - 'added': added, - } - trigger_event('activity', state.activity) + if listname: + listitems.append(file.item.id) + if time.time() - last > 5: + last = time.time() + state.activity = { + 'activity': 'import', + 'progress': [position, len(books)], + 'path': prefix, + 'added': added, + } + trigger_event('activity', state.activity) - if state.activity.get('cancel'): - state.activity = {} - return + if state.activity.get('cancel'): + state.activity = {} + return + with db.session(): if listname and listitems: l = List.get(settings.USER_ID, listname) if l: l.add_items(listitems) - trigger_event('activity', { - 'activity': 'import', - 'progress': [position, len(books)], - 'path': prefix, - 'status': {'code': 200, 'text': ''}, - 'added': added, - }) - state.activity = {} - remove_empty_folders(prefix_books) - if options.get('mode') == 'move': - remove_empty_folders(prefix) + trigger_event('activity', { + 'activity': 'import', + 'progress': [position, len(books)], + 'path': prefix, + 'status': {'code': 200, 'text': ''}, + 'added': added, + }) + state.activity = {} + remove_empty_folders(prefix_books) + if options.get('mode') == 'move': + remove_empty_folders(prefix)