diff --git a/pandora_client/__init__.py b/pandora_client/__init__.py index 8eb4716..60225ac 100755 --- a/pandora_client/__init__.py +++ b/pandora_client/__init__.py @@ -32,6 +32,8 @@ socket.setdefaulttimeout(300) CHUNK_SIZE = 1024*1024 default_media_cache = os.environ.get('oxMEDIA', '~/.ox/media') +DOCUMENT_FORMATS = ('jpg', 'pdf', 'png') + def get_frames(filename, prefix, info, force=False): oshash = info['oshash'] cache = os.path.join(prefix, os.path.join(*utils.hash_prefix(oshash))) @@ -646,6 +648,7 @@ class Client(object): print "you need to login" return conn, c = self._conn() + documents = [] if args: data = [] for arg in args: @@ -678,7 +681,8 @@ class Client(object): data = r['data']['data'] files = r['data']['file'] info = r['data']['info'] - + documents = self._get_documents() + if info: r = self.update_info(info) data = r['data']['data'] @@ -691,6 +695,12 @@ class Client(object): if os.path.exists(path): self.api.uploadData(path, oshash) break + if documents: + print 'uploading %s documents' % len(documents) + for oshash, item in documents: + for path in self.path(oshash): + if os.path.exists(path): + self._add_document(path, item) if data: print 'encoding and uploading %s videos' % len(data) @@ -751,41 +761,66 @@ class Client(object): if r.get('status', {}).get('code') != 200: print r + def _get_documents(self): + files = self.api.findMedia({ + 'query': { + 'conditions': [ + {'key': 'filename', 'operator': '', 'value': value} + for value in DOCUMENT_FORMATS + ], + 'operator': '|' + }, + 'keys': ['item', 'id'], + 'range': [0, 5000] + })['data']['items'] + d = self.api.findDocuments({ + 'query': { + 'conditions': [ + {'key': 'oshash', 'operator': '==', 'value': f['id']} + for f in files + ], + 'operator': '|' + }, + 'keys': ['id', 'oshash'], + 'range': [0, len(files)] + })['data']['items'] + available = set(f['oshash'] for f in d) + missing = [(f['id'], f['item']) for f in files if f['id'] not in available] + return missing + + def _add_document(self, f, item=None): + if f.split('.')[-1] not in DOCUMENT_FORMATS: + return False + url = '%supload/document/' % self._config['url'] + r = self.api.upload_chunks(url, f, { + 'filename': os.path.basename(f) + }) + if r and item: + oshash = ox.oshash(f) + r = self.api.findDocuments({ + "keys": ['id'], + "query": { + "conditions": [{"key": 'oshash', "value": oshash, "operator": '=='}], + "operator": '&' + } + }) + did = r['data']['items'][0]['id'] + r = self.api.addDocument({ + 'id': did, + 'item': item + }) + return True + def upload_document(self, args): if not self.user: print "you need to login" return conn, c = self._conn() for f in args: - if f.split('.')[-1] not in ('jpg', 'pdf', 'png'): + r = self._add_document(f) + if not r: print 'unsupported format', f continue - url = '%supload/document/' % self._config['url'] - r = self.api.upload_chunks(url, f, { - 'filename': os.path.basename(f) - }) - ''' - if r: - oshash = ox.oshash(f) - r = self.api.findDocuments({ - "keys": ['id'], - "query": { - "conditions": [{"key": 'oshash', "value": oshash, "operator": '=='}], - "operator": '&' - } - }) - did = r['data']['items'][0]['id'] - if description: - r = api.editDocument({ - 'id': did, - 'description': description - }) - if item: - r = api.addDocument({ - 'id': did, - 'item': item - }) - ''' def files(self, prefix): if not prefix.endswith('/'):