shorter db sessions
This commit is contained in:
parent
ebc0b95022
commit
935f8d7f2b
4 changed files with 141 additions and 136 deletions
|
|
@ -47,10 +47,10 @@ class Downloads(Thread):
|
|||
|
||||
def run(self):
|
||||
time.sleep(2)
|
||||
with db.session():
|
||||
while self._running:
|
||||
while self._running:
|
||||
with db.session():
|
||||
self.download_next()
|
||||
time.sleep(0.5)
|
||||
time.sleep(0.5)
|
||||
|
||||
def join(self):
|
||||
self._running = False
|
||||
|
|
@ -87,10 +87,13 @@ class ScrapeThread(Thread):
|
|||
|
||||
def run(self):
|
||||
time.sleep(2)
|
||||
with db.session():
|
||||
while self._running:
|
||||
while self._running:
|
||||
wait = False
|
||||
with db.session():
|
||||
if not self.scrape_queue():
|
||||
time.sleep(1)
|
||||
wait = True
|
||||
if wait:
|
||||
time.sleep(1)
|
||||
|
||||
def join(self):
|
||||
self._running = False
|
||||
|
|
|
|||
|
|
@ -37,22 +37,22 @@ class EpubHandler(OMLHandler):
|
|||
with db.session():
|
||||
item = Item.get(id)
|
||||
path = item.get_path()
|
||||
if not item or item.info['extension'] != 'epub' or not path:
|
||||
if not item or item.info['extension'] != 'epub' or not path:
|
||||
self.set_status(404)
|
||||
self.write('')
|
||||
else:
|
||||
z = zipfile.ZipFile(path)
|
||||
if filename == '':
|
||||
self.write('<br>\n'.join([f.filename for f in z.filelist]))
|
||||
elif filename not in [f.filename for f in z.filelist]:
|
||||
self.set_status(404)
|
||||
self.write('')
|
||||
else:
|
||||
z = zipfile.ZipFile(path)
|
||||
if filename == '':
|
||||
self.write('<br>\n'.join([f.filename for f in z.filelist]))
|
||||
elif filename not in [f.filename for f in z.filelist]:
|
||||
self.set_status(404)
|
||||
self.write('')
|
||||
else:
|
||||
content_type = {
|
||||
'xpgt': 'application/vnd.adobe-page-template+xml'
|
||||
}.get(filename.split('.')[0], mimetypes.guess_type(filename)[0]) or 'text/plain'
|
||||
self.set_header('Content-Type', content_type)
|
||||
self.write(z.read(filename))
|
||||
content_type = {
|
||||
'xpgt': 'application/vnd.adobe-page-template+xml'
|
||||
}.get(filename.split('.')[0], mimetypes.guess_type(filename)[0]) or 'text/plain'
|
||||
self.set_header('Content-Type', content_type)
|
||||
self.write(z.read(filename))
|
||||
|
||||
def serve_static(handler, path, mimetype, include_body=True, disposition=None):
|
||||
handler.set_header('Content-Type', mimetype)
|
||||
|
|
@ -112,7 +112,7 @@ class FileHandler(OMLHandler):
|
|||
disposition = os.path.basename(path)
|
||||
else:
|
||||
disposition = None
|
||||
return serve_static(self, path, mimetype, include_body, disposition=disposition)
|
||||
return serve_static(self, path, mimetype, include_body, disposition=disposition)
|
||||
|
||||
class ReaderHandler(OMLHandler):
|
||||
|
||||
|
|
@ -137,7 +137,8 @@ class ReaderHandler(OMLHandler):
|
|||
item.timesaccessed = (item.timesaccessed or 0) + 1
|
||||
item.update_sort()
|
||||
item.save()
|
||||
return serve_static(self, os.path.join(settings.static_path, html), 'text/html')
|
||||
path = os.path.join(settings.static_path, html)
|
||||
return serve_static(self, path, 'text/html')
|
||||
|
||||
class UploadHandler(tornado.web.RequestHandler):
|
||||
|
||||
|
|
|
|||
|
|
@ -123,7 +123,7 @@ def get_icon_app(id, type_, size, callback):
|
|||
from item.models import Item
|
||||
item = Item.get(id)
|
||||
if not item:
|
||||
callback('')
|
||||
data = ''
|
||||
else:
|
||||
if type_ == 'cover' and not item.meta.get('cover'):
|
||||
type_ = 'preview'
|
||||
|
|
@ -145,7 +145,7 @@ def get_icon_app(id, type_, size, callback):
|
|||
if size:
|
||||
data = icons[skey] = resize_image(data, size=size)
|
||||
data = bytes(data) or ''
|
||||
callback(data)
|
||||
callback(data)
|
||||
|
||||
class IconHandler(tornado.web.RequestHandler):
|
||||
|
||||
|
|
|
|||
227
oml/item/scan.py
227
oml/item/scan.py
|
|
@ -65,31 +65,31 @@ def add_file(id, f, prefix, from_=None):
|
|||
|
||||
def run_scan():
|
||||
remove_missing()
|
||||
with db.session():
|
||||
prefs = settings.preferences
|
||||
prefix = os.path.join(os.path.expanduser(prefs['libraryPath']), 'Books/')
|
||||
if not prefix[-1] == '/':
|
||||
prefix += '/'
|
||||
assert isinstance(prefix, str)
|
||||
books = []
|
||||
for root, folders, files in os.walk(prefix):
|
||||
for f in files:
|
||||
if not state.tasks.connected:
|
||||
return
|
||||
#if f.startswith('._') or f == '.DS_Store':
|
||||
if f.startswith('.'):
|
||||
continue
|
||||
f = os.path.join(root, f)
|
||||
ext = f.split('.')[-1]
|
||||
if ext in extensions:
|
||||
books.append(f)
|
||||
|
||||
position = 0
|
||||
added = 0
|
||||
for f in ox.sorted_strings(books):
|
||||
prefs = settings.preferences
|
||||
prefix = os.path.join(os.path.expanduser(prefs['libraryPath']), 'Books/')
|
||||
if not prefix[-1] == '/':
|
||||
prefix += '/'
|
||||
assert isinstance(prefix, str)
|
||||
books = []
|
||||
for root, folders, files in os.walk(prefix):
|
||||
for f in files:
|
||||
if not state.tasks.connected:
|
||||
return
|
||||
position += 1
|
||||
#if f.startswith('._') or f == '.DS_Store':
|
||||
if f.startswith('.'):
|
||||
continue
|
||||
f = os.path.join(root, f)
|
||||
ext = f.split('.')[-1]
|
||||
if ext in extensions:
|
||||
books.append(f)
|
||||
|
||||
position = 0
|
||||
added = 0
|
||||
for f in ox.sorted_strings(books):
|
||||
if not state.tasks.connected:
|
||||
return
|
||||
position += 1
|
||||
with db.session():
|
||||
id = media.get_id(f)
|
||||
file = File.get(id)
|
||||
if not file:
|
||||
|
|
@ -100,73 +100,73 @@ def run_scan():
|
|||
def run_import(options=None):
|
||||
options = options or {}
|
||||
|
||||
with db.session():
|
||||
logger.debug('run_import')
|
||||
prefs = settings.preferences
|
||||
prefix = os.path.expanduser(options.get('path', prefs['importPath']))
|
||||
if os.path.islink(prefix):
|
||||
prefix = os.path.realpath(prefix)
|
||||
if not prefix[-1] == '/':
|
||||
prefix += '/'
|
||||
prefix_books = os.path.join(os.path.expanduser(prefs['libraryPath']), 'Books/')
|
||||
prefix_imported = os.path.join(prefix_books, 'Imported/')
|
||||
if prefix_books.startswith(prefix) or prefix.startswith(prefix_books):
|
||||
error = 'invalid path'
|
||||
elif not os.path.exists(prefix):
|
||||
error = 'path not found'
|
||||
elif not os.path.isdir(prefix):
|
||||
error = 'path must be a folder'
|
||||
else:
|
||||
error = None
|
||||
if error:
|
||||
trigger_event('activity', {
|
||||
'activity': 'import',
|
||||
'progress': [0, 0],
|
||||
'status': {'code': 404, 'text': error}
|
||||
})
|
||||
state.activity = {}
|
||||
return
|
||||
listname = options.get('list')
|
||||
if listname:
|
||||
listitems = []
|
||||
assert isinstance(prefix, str)
|
||||
books = []
|
||||
count = 0
|
||||
for root, folders, files in os.walk(prefix):
|
||||
for f in files:
|
||||
if not state.tasks.connected:
|
||||
return
|
||||
#if f.startswith('._') or f == '.DS_Store':
|
||||
if f.startswith('.'):
|
||||
continue
|
||||
f = os.path.join(root, f)
|
||||
ext = f.split('.')[-1]
|
||||
if ext in extensions:
|
||||
books.append(f)
|
||||
count += 1
|
||||
if state.activity.get('cancel'):
|
||||
state.activity = {}
|
||||
return
|
||||
if count % 1000 == 0:
|
||||
state.activity = {
|
||||
'activity': 'import',
|
||||
'path': prefix,
|
||||
'progress': [0, count],
|
||||
}
|
||||
trigger_event('activity', state.activity)
|
||||
state.activity = {
|
||||
logger.debug('run_import')
|
||||
prefs = settings.preferences
|
||||
prefix = os.path.expanduser(options.get('path', prefs['importPath']))
|
||||
if os.path.islink(prefix):
|
||||
prefix = os.path.realpath(prefix)
|
||||
if not prefix[-1] == '/':
|
||||
prefix += '/'
|
||||
prefix_books = os.path.join(os.path.expanduser(prefs['libraryPath']), 'Books/')
|
||||
prefix_imported = os.path.join(prefix_books, 'Imported/')
|
||||
if prefix_books.startswith(prefix) or prefix.startswith(prefix_books):
|
||||
error = 'invalid path'
|
||||
elif not os.path.exists(prefix):
|
||||
error = 'path not found'
|
||||
elif not os.path.isdir(prefix):
|
||||
error = 'path must be a folder'
|
||||
else:
|
||||
error = None
|
||||
if error:
|
||||
trigger_event('activity', {
|
||||
'activity': 'import',
|
||||
'path': prefix,
|
||||
'progress': [0, len(books)],
|
||||
}
|
||||
trigger_event('activity', state.activity)
|
||||
position = 0
|
||||
added = 0
|
||||
last = 0
|
||||
for f in ox.sorted_strings(books):
|
||||
position += 1
|
||||
if not os.path.exists(f):
|
||||
'progress': [0, 0],
|
||||
'status': {'code': 404, 'text': error}
|
||||
})
|
||||
state.activity = {}
|
||||
return
|
||||
listname = options.get('list')
|
||||
if listname:
|
||||
listitems = []
|
||||
assert isinstance(prefix, str)
|
||||
books = []
|
||||
count = 0
|
||||
for root, folders, files in os.walk(prefix):
|
||||
for f in files:
|
||||
if not state.tasks.connected:
|
||||
return
|
||||
#if f.startswith('._') or f == '.DS_Store':
|
||||
if f.startswith('.'):
|
||||
continue
|
||||
f = os.path.join(root, f)
|
||||
ext = f.split('.')[-1]
|
||||
if ext in extensions:
|
||||
books.append(f)
|
||||
count += 1
|
||||
if state.activity.get('cancel'):
|
||||
state.activity = {}
|
||||
return
|
||||
if count % 1000 == 0:
|
||||
state.activity = {
|
||||
'activity': 'import',
|
||||
'path': prefix,
|
||||
'progress': [0, count],
|
||||
}
|
||||
trigger_event('activity', state.activity)
|
||||
state.activity = {
|
||||
'activity': 'import',
|
||||
'path': prefix,
|
||||
'progress': [0, len(books)],
|
||||
}
|
||||
trigger_event('activity', state.activity)
|
||||
position = 0
|
||||
added = 0
|
||||
last = 0
|
||||
for f in ox.sorted_strings(books):
|
||||
position += 1
|
||||
if not os.path.exists(f):
|
||||
continue
|
||||
with db.session():
|
||||
id = media.get_id(f)
|
||||
file = File.get(id)
|
||||
if not file:
|
||||
|
|
@ -180,33 +180,34 @@ def run_import(options=None):
|
|||
file = add_file(id, f, prefix_books, f_import)
|
||||
file.move()
|
||||
added += 1
|
||||
if listname:
|
||||
listitems.append(file.item.id)
|
||||
if time.time() - last > 5:
|
||||
last = time.time()
|
||||
state.activity = {
|
||||
'activity': 'import',
|
||||
'progress': [position, len(books)],
|
||||
'path': prefix,
|
||||
'added': added,
|
||||
}
|
||||
trigger_event('activity', state.activity)
|
||||
if listname:
|
||||
listitems.append(file.item.id)
|
||||
if time.time() - last > 5:
|
||||
last = time.time()
|
||||
state.activity = {
|
||||
'activity': 'import',
|
||||
'progress': [position, len(books)],
|
||||
'path': prefix,
|
||||
'added': added,
|
||||
}
|
||||
trigger_event('activity', state.activity)
|
||||
|
||||
if state.activity.get('cancel'):
|
||||
state.activity = {}
|
||||
return
|
||||
if state.activity.get('cancel'):
|
||||
state.activity = {}
|
||||
return
|
||||
with db.session():
|
||||
if listname and listitems:
|
||||
l = List.get(settings.USER_ID, listname)
|
||||
if l:
|
||||
l.add_items(listitems)
|
||||
trigger_event('activity', {
|
||||
'activity': 'import',
|
||||
'progress': [position, len(books)],
|
||||
'path': prefix,
|
||||
'status': {'code': 200, 'text': ''},
|
||||
'added': added,
|
||||
})
|
||||
state.activity = {}
|
||||
remove_empty_folders(prefix_books)
|
||||
if options.get('mode') == 'move':
|
||||
remove_empty_folders(prefix)
|
||||
trigger_event('activity', {
|
||||
'activity': 'import',
|
||||
'progress': [position, len(books)],
|
||||
'path': prefix,
|
||||
'status': {'code': 200, 'text': ''},
|
||||
'added': added,
|
||||
})
|
||||
state.activity = {}
|
||||
remove_empty_folders(prefix_books)
|
||||
if options.get('mode') == 'move':
|
||||
remove_empty_folders(prefix)
|
||||
|
|
|
|||
Loading…
Reference in a new issue