faster startup check

oml/item/models.py

@@ -724,8 +724,6 @@ class File(db.Model):
             logger.debug('file is missing. %s', current_path)
             return
 
-        self.make_readonly()
-
         author = '; '.join([get_sort_name(a) for a in j.get('author', [])])
         if not author:
             author = 'Unknown Author'
@@ -775,6 +773,7 @@ class File(db.Model):
             self.save()
             for folder in set(os.path.dirname(p) for p in [current_path, path]):
                 remove_empty_folders(folder)
+            self.make_readonly()
 
     def save(self):
         state.db.session.add(self)

oml/item/scan.py

@@ -10,7 +10,7 @@ import time
 import ox
 
 from changelog import Changelog
-from item.models import File
+from item.models import File, Item
 from user.models import List
 from utils import remove_empty_folders
 from websocket import trigger_event
@@ -24,31 +24,49 @@ logger = logging.getLogger(__name__)
 
 extensions = ['epub', 'pdf', 'txt', 'cbr', 'cbz']
 
-def remove_missing():
+def remove_missing(books=None):
     dirty = False
+    logger.debug('remove missing')
+    if books is None:
+        books = collect_books()
     with db.session():
         prefs = settings.preferences
         prefix = os.path.join(os.path.expanduser(prefs['libraryPath']), 'Books' + os.sep)
         if os.path.exists(prefix):
+            logger.debug('scan for removed files')
+            db_paths = []
+            items = {}
             for f in File.query:
                 if state.shutdown:
                     return
-                if f.item:
+                path = f.fullpath()
-                    path = f.item.get_path()
+                db_paths.append(path)
-                    if not os.path.exists(path):
+                items[path] = f.sha1
+            removed = set(db_paths) - set(books)
+            if removed:
+                logger.debug('%s files removed', len(removed))
+                ids = [items[path] for path in removed]
+                orphaned = set(ids)
+                for i in Item.query.filter(Item.id.in_(ids)):
+                    i.remove_file()
+                    orphaned.remove(i.id)
                     dirty = True
-                        f.item.remove_file()
+                if orphaned:
-                else:
+                    logger.debug('%s files orphaned', len(orphaned))
+                    for f in File.query.filter(File.sha1.in_(orphaned)):
                         state.db.session.delete(f)
                     dirty = True
             if dirty:
                 state.db.session.commit()
                 state.cache.clear('group:')
+            logger.debug('update filenames')
             for f in File.query:
                 if state.shutdown:
                     return
                 f.move()
+            logger.debug('remove empty folders')
             remove_empty_folders(prefix, True)
+    logger.debug('remove missing done')
 
 def add_file(id, f, prefix, from_=None, commit=True):
     user = state.user()
@@ -72,19 +90,18 @@ def add_file(id, f, prefix, from_=None, commit=True):
     logger.debug('%s added', id)
     return file
 
-def run_scan():
+def collect_books():
-    remove_missing()
     prefs = settings.preferences
     prefix = os.path.join(os.path.expanduser(prefs['libraryPath']), 'Books' + os.sep)
     if not prefix[-1] == os.sep:
         prefix += os.sep
     assert isinstance(prefix, str)
+    logger.debug('collect books')
     books = []
     for root, folders, files in os.walk(prefix):
         for f in files:
             if state.shutdown:
                 return
-            #if f.startswith('._') or f == '.DS_Store':
             if f.startswith('.'):
                 continue
             f = os.path.join(root, f)
@@ -93,6 +110,13 @@ def run_scan():
                 ext = 'epub'
             if ext in extensions:
                 books.append(f)
+    logger.debug('found %s books', len(books))
+    return books
+
+def run_scan():
+    logger.debug('run_scan')
+    books = collect_books()
+    remove_missing(books)
 
     position = 0
     added = 0
@@ -100,14 +124,15 @@ def run_scan():
         if state.shutdown:
             return
         position += 1
-        with db.session():
         if os.path.exists(f):
             id = media.get_id(f)
+            with db.session():
                 file = File.get(id)
                 if not file:
                     file = add_file(id, f, prefix, f)
                     added += 1
                     trigger_event('change', {})
+    logger.debug('imported unknown books')
 
 def change_path(old, new):
     new_books = os.path.join(new, 'Books')