0x2620/pandora_client
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

only look for images in available volumes

Browse source
This commit is contained in:
j 2016-10-27 14:15:07 +02:00
parent 86cccee265
commit 662c8569fb
1 changed files with 42 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

80
pandora_client/__init__.py
View file

@ -667,6 +667,7 @@ class Client(object):
if not self.user:
print("you need to login")
return
documents = []
if args:
data = []
@ -694,6 +695,9 @@ class Client(object):
files = []
info = []
else:
if not self.active_volumes():
print("no volumes found, mount volumes and run again")
return
# send empty list to get updated list of requested info/files/data
post = {'info': {}}
r = self.api.update(post)
@ -784,48 +788,47 @@ class Client(object):
print(r)
def _get_documents(self):
query = {
'conditions': [
{'key': 'filename', 'operator': '', 'value': value}
for value in DOCUMENT_FORMATS
],
'operator': '|'
}
n = self.api.findMedia({'query': query})['data']['items']
if n:
o = 0
chunk = 5000
files = []
while o < n:
files += self.api.findMedia({
'query': {
files = []
for volume in self.active_volumes():
query = {
'conditions': [
{'key': 'list', 'value': volume, 'operator': '=='},
{
'conditions': [
{'key': 'filename', 'operator': '', 'value': value}
for value in DOCUMENT_FORMATS
],
'operator': '|'
},
'keys': ['item', 'id', 'extension'],
'range': [o, o+chunk]
})['data']['items']
o += chunk
d = []
o = 0
while o < len(files):
d += self.api.findDocuments({
'query': {
'conditions': [
{'key': 'oshash', 'operator': '==', 'value': f['id']}
for f in files[o:o+chunk]
],
'operator': '|'
},
'keys': ['id', 'oshash', 'extension'],
'range': [0, chunk]
})['data']['items']
o += chunk
else:
d = []
}
],
'operator': '&'
}
n = self.api.findMedia({'query': query})['data']['items']
if n:
o = 0
chunk = 5000
while o < n:
files += self.api.findMedia({
'query': query,
'keys': ['item', 'id', 'extension'],
'range': [o, o+chunk]
})['data']['items']
o += chunk
d = []
o = 0
while o < len(files):
d += self.api.findDocuments({
'query': {
'conditions': [
{'key': 'oshash', 'operator': '==', 'value': f['id']}
for f in files[o:o+chunk]
],
'operator': '|'
},
'keys': ['id', 'oshash', 'extension'],
'range': [0, chunk]
})['data']['items']
o += chunk
available = set(f['oshash']
for f in d if f['extension'] in DOCUMENT_FORMATS)
missing = [(f['id'], f['item']) for f in files
@ -847,7 +850,8 @@ class Client(object):
return None
def _add_document(self, f, item=None):
if f.split('.')[-1] not in DOCUMENT_FORMATS:
if f.split('.')[-1].lower() not in DOCUMENT_FORMATS:
print('skip, not a document', f)
return False
oshash = ox.oshash(f)
did = self.find_document(oshash)