include upload_chunks

This commit is contained in:
j 2018-08-05 20:45:49 +01:00
parent 704a0c224b
commit b7a18203ac

View file

@ -5,10 +5,12 @@ from types import MethodType
import gzip import gzip
import os import os
import shutil import shutil
import sys
from six.moves import http_cookiejar as cookielib from six.moves import http_cookiejar as cookielib
from six import BytesIO, PY2 from six import BytesIO, PY2
from six.moves import urllib from six.moves import urllib
from six.moves.urllib.parse import urlparse
from . import __version__ from . import __version__
from .utils import json from .utils import json
@ -16,6 +18,8 @@ from .form import MultiPartForm
__all__ = ['getAPI', 'API'] __all__ = ['getAPI', 'API']
CHUNK_SIZE = 1024*1024*5
def getAPI(url, cj=None): def getAPI(url, cj=None):
return API(url, cj) return API(url, cj)
@ -138,6 +142,72 @@ class API(object):
fd.write(chunk) fd.write(chunk)
shutil.move(tmpname, filename) shutil.move(tmpname, filename)
def upload_chunks(self, url, filename, data=None):
form = MultiPartForm()
if data:
for key in data:
form.add_field(key, data[key])
data = self._json_request(url, form)
def full_url(path):
if path.startswith('/'):
u = urlparse(url)
path = '%s://%s%s' % (u.scheme, u.netloc, path)
return path
if 'uploadUrl' in data:
uploadUrl = full_url(data['uploadUrl'])
f = open(filename, 'rb')
fsize = os.stat(filename).st_size
done = 0
start = time.mktime(time.localtime())
if 'offset' in data and data['offset'] < fsize:
done = data['offset']
f.seek(done)
resume_offset = done
else:
resume_offset = 0
chunk = f.read(CHUNK_SIZE)
fname = os.path.basename(filename)
if not isinstance(fname, bytes):
fname = fname.encode('utf-8')
while chunk:
form = MultiPartForm()
form.add_file('chunk', fname, chunk)
if len(chunk) < CHUNK_SIZE or f.tell() == fsize:
form.add_field('done', '1')
form.add_field('offset', str(done))
try:
data = self._json_request(uploadUrl, form)
except KeyboardInterrupt:
print("\ninterrupted by user.")
sys.exit(1)
except:
print("uploading chunk failed, will try again in 5 seconds\r", end='')
sys.stdout.flush()
data = {'result': -1}
time.sleep(5)
if data and 'status' in data:
if data['status']['code'] == 403:
print("login required")
return False
if data['status']['code'] != 200:
print("request returned error, will try again in 5 seconds")
if DEBUG:
print(data)
time.sleep(5)
if data and data.get('result') == 1:
done += len(chunk)
if data.get('offset') not in (None, done):
print('server offset out of sync, continue from', data['offset'])
done = data['offset']
f.seek(done)
chunk = f.read(CHUNK_SIZE)
if data and 'result' in data and data.get('result') == 1:
return data.get('id', True)
else:
return False
return False
def signin(url): def signin(url):
import sys import sys