From 1e39fc48b287dd49d2bfcdcc4da29e59e5a18aa8 Mon Sep 17 00:00:00 2001 From: j Date: Sun, 13 Oct 2019 11:16:45 +0100 Subject: [PATCH] use ThreadPoolExecutor --- oml/item/handlers.py | 116 ++++++++++++++++++++++--------------------- 1 file changed, 60 insertions(+), 56 deletions(-) diff --git a/oml/item/handlers.py b/oml/item/handlers.py index 19d6460..b20001a 100644 --- a/oml/item/handlers.py +++ b/oml/item/handlers.py @@ -17,7 +17,8 @@ import db import settings import tornado.web import tornado.gen -import tornado.concurrent +from concurrent.futures import ThreadPoolExecutor +from tornado.concurrent import run_on_executor from oxtornado import json_dumps, json_response @@ -27,6 +28,8 @@ import state import logging logger = logging.getLogger(__name__) +MAX_WORKERS = 4 + class OptionalBasicAuthMixin(object): class SendChallenge(Exception): @@ -177,6 +180,7 @@ class ReaderHandler(OMLHandler): return serve_static(self, path, 'text/html') class UploadHandler(OMLHandler): + executor = ThreadPoolExecutor(max_workers=MAX_WORKERS) def initialize(self, context=None): self._context = context @@ -184,7 +188,60 @@ class UploadHandler(OMLHandler): def get(self): self.write('use POST') - @tornado.web.asynchronous + @run_on_executor + def save_files(self, request): + listname = request.arguments.get('list', None) + if listname: + listname = listname[0] + if isinstance(listname, bytes): + listname = listname.decode('utf-8') + with self._context(): + prefs = settings.preferences + ids = [] + for upload in request.files.get('files', []): + filename = upload.filename + id = get_id(data=upload.body) + ids.append(id) + file = File.get(id) + if not file or not os.path.exists(file.fullpath()): + logger.debug('add %s to library', id) + prefix_books = os.path.join(os.path.expanduser(prefs['libraryPath']), 'Books' + os.sep) + prefix_imported = os.path.join(prefix_books, '.import' + os.sep) + ox.makedirs(prefix_imported) + import_name = os.path.join(prefix_imported, filename) + n = 1 + while os.path.exists(import_name): + n += 1 + name, extension = filename.rsplit('.', 1) + if extension == 'kepub': + extension = 'epub' + import_name = os.path.join(prefix_imported, '%s [%d].%s' % (name, n, extension)) + with open(import_name, 'wb') as fd: + fd.write(upload.body) + file = add_file(id, import_name, prefix_books) + file.move() + else: + user = state.user() + if not file.item: + item = Item.get_or_create(id=file.sha1, info=file.info) + file.item_id = item.id + state.db.session.add(file) + state.db.session.commit() + else: + item = file.item + if user not in item.users: + logger.debug('add %s to local user', id) + item.add_user(user) + add_record('additem', item.id, file.info) + add_record('edititem', item.id, item.meta) + item.update() + if listname and ids: + list_ = List.get(settings.USER_ID, listname) + if list_: + list_.add_items(ids) + response = json_response({'ids': ids}) + return response + @tornado.gen.coroutine def post(self): if 'origin' in self.request.headers and self.request.host not in self.request.headers['origin']: @@ -193,60 +250,7 @@ class UploadHandler(OMLHandler): self.write('') return - def save_files(context, request, callback): - listname = request.arguments.get('list', None) - if listname: - listname = listname[0] - if isinstance(listname, bytes): - listname = listname.decode('utf-8') - with context(): - prefs = settings.preferences - ids = [] - for upload in request.files.get('files', []): - filename = upload.filename - id = get_id(data=upload.body) - ids.append(id) - file = File.get(id) - if not file or not os.path.exists(file.fullpath()): - logger.debug('add %s to library', id) - prefix_books = os.path.join(os.path.expanduser(prefs['libraryPath']), 'Books' + os.sep) - prefix_imported = os.path.join(prefix_books, '.import' + os.sep) - ox.makedirs(prefix_imported) - import_name = os.path.join(prefix_imported, filename) - n = 1 - while os.path.exists(import_name): - n += 1 - name, extension = filename.rsplit('.', 1) - if extension == 'kepub': - extension = 'epub' - import_name = os.path.join(prefix_imported, '%s [%d].%s' % (name, n, extension)) - with open(import_name, 'wb') as fd: - fd.write(upload.body) - file = add_file(id, import_name, prefix_books) - file.move() - else: - user = state.user() - if not file.item: - item = Item.get_or_create(id=file.sha1, info=file.info) - file.item_id = item.id - state.db.session.add(file) - state.db.session.commit() - else: - item = file.item - if user not in item.users: - logger.debug('add %s to local user', id) - item.add_user(user) - add_record('additem', item.id, file.info) - add_record('edititem', item.id, item.meta) - item.update() - if listname and ids: - l = List.get(settings.USER_ID, listname) - if l: - l.add_items(ids) - response = json_response({'ids': ids}) - callback(response) - - response = yield tornado.gen.Task(save_files, self._context, self.request) + response = yield self.save_files(self.request) if 'status' not in response: response = json_response(response) response = json_dumps(response)