0x2620/pandora_client
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

only upload documents once

Browse source
This commit is contained in:
j 2016-10-29 16:17:06 +02:00
parent 662c8569fb
commit 9e40462d78
1 changed files with 12 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
pandora_client/__init__.py
View file

@ -808,20 +808,21 @@ class Client(object):
o = 0
chunk = 5000
while o < n:
files += self.api.findMedia({
files += [f for f in self.api.findMedia({
'query': query,
'keys': ['item', 'id', 'extension'],
'range': [o, o+chunk]
})['data']['items']
})['data']['items'] if f['extension'] in DOCUMENT_FORMATS]
o += chunk
d = []
documents = []
o = 0
while o < len(files):
d += self.api.findDocuments({
ids = list(set(f['id'] for f in files))
while o < len(ids):
documents += self.api.findDocuments({
'query': {
'conditions': [
{'key': 'oshash', 'operator': '==', 'value': f['id']}
for f in files[o:o+chunk]
{'key': 'oshash', 'operator': '==', 'value': id}
for id in ids[o:o+chunk]
],
'operator': '|'
},
@ -829,10 +830,9 @@ class Client(object):
'range': [0, chunk]
})['data']['items']
o += chunk
available = set(f['oshash']
for f in d if f['extension'] in DOCUMENT_FORMATS)
missing = [(f['id'], f['item']) for f in files
if f['id'] not in available and f['extension'] in DOCUMENT_FORMATS]
available = set(f['oshash'] for f in documents)
missing = [(f['id'], f['item']) for f in files if f['id'] not in available]
missing = list(set(missing))
return missing
def find_document(self, oshash):
@ -867,7 +867,7 @@ class Client(object):
'id': did,
'item': item
})
return True
return did
def upload_document(self, args):
if not self.user: