54 lines
1.4 KiB
Python
54 lines
1.4 KiB
Python
|
#!/usr/bin/python
|
||
|
|
#
|
||
|
|
# upload documents
|
||
|
|
#
|
||
|
|
from __future__ import print_function
|
||
|
|
|
||
|
|
import os
|
||
|
|
import subprocess
|
||
|
|
from glob import glob
|
||
|
|
import json
|
||
|
|
|
||
|
|
import ox
|
||
|
|
import pandora_client
|
||
|
|
from pandora_client.utils import hash_prefix
|
||
|
|
|
||
|
|
MISSING = os.path.expanduser('~/.ox/missing_documents.json')
|
||
|
|
DONE = os.path.expanduser('~/.ox/documents_uploaded.txt')
|
||
|
|
|
||
|
|
def get_documents(client):
|
||
|
|
if os.path.exists(MISSING):
|
||
|
|
with open(MISSING) as fd:
|
||
|
|
documents = json.load(fd)
|
||
|
|
else:
|
||
|
|
documents = client._get_documents()
|
||
|
|
with open(MISSING, 'w') as fd:
|
||
|
|
json.dump(documents, fd, indent=4)
|
||
|
|
return documents
|
||
|
|
|
||
|
|
|
||
|
|
def done(oshash):
|
||
|
|
with open(DONE, 'a') as fd:
|
||
|
|
fd.write(oshash + '\n')
|
||
|
|
|
||
|
|
if __name__ == '__main__':
|
||
|
|
uploaded = []
|
||
|
|
if os.path.exists(DONE):
|
||
|
|
with open(DONE) as fd:
|
||
|
|
uploaded = fd.read().strip().split('\n')
|
||
|
|
|
||
|
|
client = pandora_client.Client(os.path.expanduser('~/.ox/client.json'), False)
|
||
|
|
documents = get_documents(client)
|
||
|
|
_documents = []
|
||
|
|
for oshash, item in documents:
|
||
|
|
if oshash in uploaded:
|
||
|
|
continue
|
||
|
|
for path in client.path(oshash):
|
||
|
|
if os.path.exists(path):
|
||
|
|
_documents.append([path, oshash, item])
|
||
|
|
break
|
||
|
|
print('uploading %s documents' % len(_documents))
|
||
|
|
for path, oshash, item in _documents:
|
||
|
|
client._add_document(path, item)
|
||
|
|
done(oshash)
|