0x2620/pandora_client
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

only look for images in available volumes

Browse source
This commit is contained in:
j 2016-10-27 14:15:07 +02:00
parent 86cccee265
commit 662c8569fb
1 changed files with 42 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

80
pandora_client/__init__.py
View file

@ -667,6 +667,7 @@ class Client(object):
if not self.user: if not self.user:
print("you need to login") print("you need to login")
return return
documents = [] documents = []
if args: if args:
data = [] data = []
@ -694,6 +695,9 @@ class Client(object):
files = [] files = []
info = [] info = []
else: else:
if not self.active_volumes():
print("no volumes found, mount volumes and run again")
return
# send empty list to get updated list of requested info/files/data # send empty list to get updated list of requested info/files/data
post = {'info': {}} post = {'info': {}}
r = self.api.update(post) r = self.api.update(post)
@ -784,48 +788,47 @@ class Client(object):
print(r) print(r)
def _get_documents(self): def _get_documents(self):
query = { files = []
'conditions': [ for volume in self.active_volumes():
{'key': 'filename', 'operator': '', 'value': value} query = {
for value in DOCUMENT_FORMATS 'conditions': [
], {'key': 'list', 'value': volume, 'operator': '=='},
'operator': '|' {
}
n = self.api.findMedia({'query': query})['data']['items']
if n:
o = 0
chunk = 5000
files = []
while o < n:
files += self.api.findMedia({
'query': {
'conditions': [ 'conditions': [
{'key': 'filename', 'operator': '', 'value': value} {'key': 'filename', 'operator': '', 'value': value}
for value in DOCUMENT_FORMATS for value in DOCUMENT_FORMATS
], ],
'operator': '|' 'operator': '|'
}, }
'keys': ['item', 'id', 'extension'], ],
'range': [o, o+chunk] 'operator': '&'
})['data']['items'] }
o += chunk n = self.api.findMedia({'query': query})['data']['items']
d = [] if n:
o = 0 o = 0
while o < len(files): chunk = 5000
d += self.api.findDocuments({ while o < n:
'query': { files += self.api.findMedia({
'conditions': [ 'query': query,
{'key': 'oshash', 'operator': '==', 'value': f['id']} 'keys': ['item', 'id', 'extension'],
for f in files[o:o+chunk] 'range': [o, o+chunk]
], })['data']['items']
'operator': '|' o += chunk
}, d = []
'keys': ['id', 'oshash', 'extension'], o = 0
'range': [0, chunk] while o < len(files):
})['data']['items'] d += self.api.findDocuments({
o += chunk 'query': {
else: 'conditions': [
d = [] {'key': 'oshash', 'operator': '==', 'value': f['id']}
for f in files[o:o+chunk]
],
'operator': '|'
},
'keys': ['id', 'oshash', 'extension'],
'range': [0, chunk]
})['data']['items']
o += chunk
available = set(f['oshash'] available = set(f['oshash']
for f in d if f['extension'] in DOCUMENT_FORMATS) for f in d if f['extension'] in DOCUMENT_FORMATS)
missing = [(f['id'], f['item']) for f in files missing = [(f['id'], f['item']) for f in files
@ -847,7 +850,8 @@ class Client(object):
return None return None
def _add_document(self, f, item=None): def _add_document(self, f, item=None):
if f.split('.')[-1] not in DOCUMENT_FORMATS: if f.split('.')[-1].lower() not in DOCUMENT_FORMATS:
print('skip, not a document', f)
return False return False
oshash = ox.oshash(f) oshash = ox.oshash(f)
did = self.find_document(oshash) did = self.find_document(oshash)