0x2620/openmedialibrary
1
0
Fork 0
Code Issues 131 Pull requests Releases Wiki Activity

double check that volume is still around, only flag files as missing

Browse source
This commit is contained in:
j 2019-02-13 12:36:25 +05:30
commit 8928b5447b
3 changed files with 66 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

5
oml/item/models.py
View file

@ -556,6 +556,11 @@ class Item(db.Model):
return True
return False
def missing_file(self):
logger.debug('file is missing! %s: %s', self.id, self.get_path())
self.info['missing'] = True
state.db.session.add(self)
def remove_file(self):
for f in self.files.all():
path = f.fullpath()

39
oml/item/scan.py
View file

@ -29,10 +29,11 @@ def remove_missing(books=None):
dirty = False
logger.debug('remove missing')
prefix = get_prefix()
oml_prefix = os.path.dirname(prefix)
if books is None:
books = collect_books(prefix)
with db.session():
if os.path.exists(prefix):
if os.path.exists(prefix) and os.path.exists(oml_prefix):
logger.debug('scan for removed files')
db_paths = []
items = {}
@ -44,29 +45,35 @@ def remove_missing(books=None):
if f.item:
items[path] = f.sha1
else:
logger.debug('remove orphaned file %s', f)
state.db.session.delete(f)
dirty = True
if dirty:
state.db.session.commit()
dirty = False
nfd_books = {unicodedata.normalize('NFD', path) for path in books}
nfc_books = {unicodedata.normalize('NFC', path) for path in books}
removed = [
path for path in db_paths
if unicodedata.normalize('NFD', path) not in nfd_books
if unicodedata.normalize('NFC', path) not in nfc_books
]
if removed:
if removed and os.path.exists(prefix) and os.path.exists(oml_prefix):
logger.debug('%s files removed', len(removed))
ids = [items[path] for path in removed]
orphaned = set(ids)
for i in Item.query.filter(Item.id.in_(ids)):
i.remove_file()
orphaned.remove(i.id)
dirty = True
if orphaned:
logger.debug('%s files orphaned', len(orphaned))
for f in File.query.filter(File.sha1.in_(orphaned)):
state.db.session.delete(f)
dirty = True
if ids:
orphaned = set(ids)
for i in Item.query.filter(Item.id.in_(ids)):
if state.shutdown:
continue
i.missing_file()
orphaned.remove(i.id)
dirty = True
if orphaned:
logger.debug('%s files orphaned', len(orphaned))
for f in File.query.filter(File.sha1.in_(orphaned)):
if state.shutdown:
continue
state.db.session.delete(f)
dirty = True
if dirty:
state.db.session.commit()
state.cache.clear('group:')
@ -182,6 +189,10 @@ def run_scan():
add_record('edititem', item.id, item.meta)
item.update()
added += 1
if file and file.item.info.get('missing'):
logger.debug('missing file showed up again %s: %s', id, file.fullpath())
del file.item.info['missing']
file.item.save()
if file and not file.item.added:
file.item.added = datetime.utcnow()
if file.item.accessed: