include upload_chunks

ox/api.py

@@ -5,10 +5,12 @@ from types import MethodType
 import gzip
 import os
 import shutil
+import sys
 
 from six.moves import http_cookiejar as cookielib
 from six import BytesIO, PY2
 from six.moves import urllib
+from six.moves.urllib.parse import urlparse
 
 from . import __version__
 from .utils import json
@@ -16,6 +18,8 @@ from .form import MultiPartForm
 
 __all__ = ['getAPI', 'API']
 
+CHUNK_SIZE = 1024*1024*5
+
 def getAPI(url, cj=None):
     return API(url, cj)
 
@@ -138,6 +142,72 @@ class API(object):
                     fd.write(chunk)
             shutil.move(tmpname, filename)
 
+    def upload_chunks(self, url, filename, data=None):
+        form = MultiPartForm()
+        if data:
+            for key in data:
+                form.add_field(key, data[key])
+        data = self._json_request(url, form)
+
+        def full_url(path):
+            if path.startswith('/'):
+                u = urlparse(url)
+                path = '%s://%s%s' % (u.scheme, u.netloc, path)
+            return path
+
+        if 'uploadUrl' in data:
+            uploadUrl = full_url(data['uploadUrl'])
+            f = open(filename, 'rb')
+            fsize = os.stat(filename).st_size
+            done = 0
+            start = time.mktime(time.localtime())
+            if 'offset' in data and data['offset'] < fsize:
+                done = data['offset']
+                f.seek(done)
+                resume_offset = done
+            else:
+                resume_offset = 0
+            chunk = f.read(CHUNK_SIZE)
+            fname = os.path.basename(filename)
+            if not isinstance(fname, bytes):
+                fname = fname.encode('utf-8')
+            while chunk:
+                form = MultiPartForm()
+                form.add_file('chunk', fname, chunk)
+                if len(chunk) < CHUNK_SIZE or f.tell() == fsize:
+                    form.add_field('done', '1')
+                form.add_field('offset', str(done))
+                try:
+                    data = self._json_request(uploadUrl, form)
+                except KeyboardInterrupt:
+                    print("\ninterrupted by user.")
+                    sys.exit(1)
+                except:
+                    print("uploading chunk failed, will try again in 5 seconds\r", end='')
+                    sys.stdout.flush()
+                    data = {'result': -1}
+                    time.sleep(5)
+                if data and 'status' in data:
+                    if data['status']['code'] == 403:
+                        print("login required")
+                        return False
+                    if data['status']['code'] != 200:
+                        print("request returned error, will try again in 5 seconds")
+                        if DEBUG:
+                            print(data)
+                        time.sleep(5)
+                if data and data.get('result') == 1:
+                    done += len(chunk)
+                    if data.get('offset') not in (None, done):
+                        print('server offset out of sync, continue from', data['offset'])
+                        done = data['offset']
+                        f.seek(done)
+                    chunk = f.read(CHUNK_SIZE)
+            if data and 'result' in data and data.get('result') == 1:
+                return data.get('id', True)
+            else:
+                return False
+        return False
 
 def signin(url):
     import sys