diff --git a/ox/api.py b/ox/api.py index 0594fe2..c6ec525 100644 --- a/ox/api.py +++ b/ox/api.py @@ -5,10 +5,12 @@ from types import MethodType import gzip import os import shutil +import sys from six.moves import http_cookiejar as cookielib from six import BytesIO, PY2 from six.moves import urllib +from six.moves.urllib.parse import urlparse from . import __version__ from .utils import json @@ -16,6 +18,8 @@ from .form import MultiPartForm __all__ = ['getAPI', 'API'] +CHUNK_SIZE = 1024*1024*5 + def getAPI(url, cj=None): return API(url, cj) @@ -138,6 +142,72 @@ class API(object): fd.write(chunk) shutil.move(tmpname, filename) + def upload_chunks(self, url, filename, data=None): + form = MultiPartForm() + if data: + for key in data: + form.add_field(key, data[key]) + data = self._json_request(url, form) + + def full_url(path): + if path.startswith('/'): + u = urlparse(url) + path = '%s://%s%s' % (u.scheme, u.netloc, path) + return path + + if 'uploadUrl' in data: + uploadUrl = full_url(data['uploadUrl']) + f = open(filename, 'rb') + fsize = os.stat(filename).st_size + done = 0 + start = time.mktime(time.localtime()) + if 'offset' in data and data['offset'] < fsize: + done = data['offset'] + f.seek(done) + resume_offset = done + else: + resume_offset = 0 + chunk = f.read(CHUNK_SIZE) + fname = os.path.basename(filename) + if not isinstance(fname, bytes): + fname = fname.encode('utf-8') + while chunk: + form = MultiPartForm() + form.add_file('chunk', fname, chunk) + if len(chunk) < CHUNK_SIZE or f.tell() == fsize: + form.add_field('done', '1') + form.add_field('offset', str(done)) + try: + data = self._json_request(uploadUrl, form) + except KeyboardInterrupt: + print("\ninterrupted by user.") + sys.exit(1) + except: + print("uploading chunk failed, will try again in 5 seconds\r", end='') + sys.stdout.flush() + data = {'result': -1} + time.sleep(5) + if data and 'status' in data: + if data['status']['code'] == 403: + print("login required") + return False + if data['status']['code'] != 200: + print("request returned error, will try again in 5 seconds") + if DEBUG: + print(data) + time.sleep(5) + if data and data.get('result') == 1: + done += len(chunk) + if data.get('offset') not in (None, done): + print('server offset out of sync, continue from', data['offset']) + done = data['offset'] + f.seek(done) + chunk = f.read(CHUNK_SIZE) + if data and 'result' in data and data.get('result') == 1: + return data.get('id', True) + else: + return False + return False def signin(url): import sys