show progress during upload

This commit is contained in:
j 2011-12-30 18:47:28 +05:30
parent 070ab9c187
commit 67af01db75
3 changed files with 82 additions and 24 deletions

View file

@ -1,28 +1,28 @@
#!/usr/bin/python #!/usr/bin/python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4 # vi:si:et:sw=4:sts=4:ts=4
# GPL 2010 # GPL 2012
from __future__ import division, with_statement from __future__ import division, with_statement
import os import os
import urllib2
import cookielib
import json import json
import sqlite3 import sqlite3
import time import time
import shutil import shutil
import webbrowser import sys
from firefogg import Firefogg
import ox import ox
import extract import extract
import utils import utils
__version__ = '0.2'
DEBUG = False DEBUG = False
__version__ = '0.2'
CHUNK_SIZE = 1024*1024
default_media_cache = os.environ.get('oxMEDIA', os.path.expanduser('~/.ox/media')) default_media_cache = os.environ.get('oxMEDIA', os.path.expanduser('~/.ox/media'))
def encode(filename, prefix, profile, info=None): def encode(filename, prefix, profile, info=None):
if not info: if not info:
info = utils.avinfo(filename) info = utils.avinfo(filename)
@ -41,6 +41,7 @@ def encode(filename, prefix, profile, info=None):
frames.append(frame_f) frames.append(frame_f)
video_f = os.path.join(cache, profile) video_f = os.path.join(cache, profile)
if not os.path.exists(video_f): if not os.path.exists(video_f):
return False
extract.video(filename, video_f, profile, info) extract.video(filename, video_f, profile, info)
return { return {
'info': info, 'info': info,
@ -171,7 +172,7 @@ class Client(object):
conn.commit() conn.commit()
def scan(self): def scan(self):
print "Checking for new files ..." print "checking for new files ..."
for name in self._config['volumes']: for name in self._config['volumes']:
path = self._config['volumes'][name] path = self._config['volumes'][name]
path = os.path.normpath(path) path = os.path.normpath(path)
@ -200,7 +201,7 @@ class Client(object):
c.execute('UPDATE file SET deleted=? WHERE path=?', (deleted, f)) c.execute('UPDATE file SET deleted=? WHERE path=?', (deleted, f))
conn.commit() conn.commit()
print "Scanned volume %s: %s files, %s new, %s deleted" % ( print "scanned volume %s: %s files, %s new, %s deleted" % (
name, len(files), len(files) - len(known_files), len(deleted_files)) name, len(files), len(files) - len(known_files), len(deleted_files))
def extract(self): def extract(self):
@ -257,13 +258,13 @@ class Client(object):
post = {} post = {}
post['files'] = files['files'] post['files'] = files['files']
post['volume'] = name post['volume'] = name
print 'Sending list of files in %s (%s total)' % (name, len(post['files'])) print 'sending list of files in %s (%s total)' % (name, len(post['files']))
r = self.api.update(post) r = self.api.update(post)
if r['status']['code'] == 200: if r['status']['code'] == 200:
#backend works on update request asyncronously, wait for it to finish #backend works on update request asyncronously, wait for it to finish
if 'taskId' in r['data']: if 'taskId' in r['data']:
t = self.api.taskStatus(task_id=r['data']['taskId']) t = self.api.taskStatus(task_id=r['data']['taskId'])
print 'Waiting for server ...' print 'waiting for server ...'
while t['data']['status'] == 'PENDING': while t['data']['status'] == 'PENDING':
time.sleep(5) time.sleep(5)
t = self.api.taskStatus(task_id=r['data']['taskId']) t = self.api.taskStatus(task_id=r['data']['taskId'])
@ -275,7 +276,7 @@ class Client(object):
info = r['data']['info'] info = r['data']['info']
max_info = 100 max_info = 100
total = len(info) total = len(info)
print 'Sending info for %s files' % total print 'sending info for %s files' % total
for offset in range(0, total, max_info): for offset in range(0, total, max_info):
post = {'info': {}, 'upload': True} post = {'info': {}, 'upload': True}
for oshash in info[offset:offset+max_info]: for oshash in info[offset:offset+max_info]:
@ -291,7 +292,7 @@ class Client(object):
files.append(path) files.append(path)
break break
if files: if files:
print '\nCould encoded and upload %s videos:\n' % len(files) print '\ncould encoded and upload %s videos:\n' % len(files)
print '\n'.join(files) print '\n'.join(files)
if r['data']['file']: if r['data']['file']:
files = [] files = []
@ -301,7 +302,7 @@ class Client(object):
files.append(path) files.append(path)
break break
if files: if files:
print '\nCould upload %s subtitles:\n' % len(files) print '\ncould upload %s subtitles:\n' % len(files)
print '\n'.join(files) print '\n'.join(files)
def upload(self): def upload(self):
@ -335,14 +336,14 @@ class Client(object):
r = self.api.update(post) r = self.api.update(post)
if r['data']['file']: if r['data']['file']:
print 'Uploading %s files' % len(r['data']['file']) print 'uploading %s files' % len(r['data']['file'])
for oshash in r['data']['file']: for oshash in r['data']['file']:
if oshash in filenames: if oshash in filenames:
filename = filenames[oshash] filename = filenames[oshash]
self.api.uploadData(os.path.join(prefix, filename), oshash) self.api.uploadData(os.path.join(prefix, filename), oshash)
if r['data']['data']: if r['data']['data']:
print 'Encoding and uploading %s videos' % len(r['data']['data']) print 'encoding and uploading %s videos' % len(r['data']['data'])
for oshash in r['data']['data']: for oshash in r['data']['data']:
data = {} data = {}
if oshash in filenames: if oshash in filenames:
@ -396,8 +397,6 @@ class API(ox.API):
self.media_cache = default_media_cache self.media_cache = default_media_cache
def uploadVideo(self, filename, data, profile, info=None): def uploadVideo(self, filename, data, profile, info=None):
if DEBUG:
print filename.encode('utf-8')
i = encode(filename, self.media_cache, profile, info) i = encode(filename, self.media_cache, profile, info)
if not i: if not i:
print "failed" print "failed"
@ -419,10 +418,9 @@ class API(ox.API):
#upload video #upload video
if os.path.exists(i['video']): if os.path.exists(i['video']):
size = ox.formatBytes(os.path.getsize(i['video'])) size = ox.formatBytes(os.path.getsize(i['video']))
print "Uploading %s (%s) of %s" % (profile, size, filename) print "uploading %s of %s (%s)" % (profile, os.path.basename(filename), size)
url = self.url + 'upload/' + '?profile=' + str(profile) + '&id=' + i['oshash'] url = self.url + 'upload/' + '?profile=' + str(profile) + '&id=' + i['oshash']
ogg = Firefogg(cj=self._cj, debug=DEBUG) if not self.upload_chunks(url, i['video'], data):
if not ogg.upload(url, i['video'], data):
if DEBUG: if DEBUG:
print "failed" print "failed"
return False return False
@ -443,3 +441,62 @@ class API(ox.API):
r = self._json_request(self.url, form) r = self._json_request(self.url, form)
return r return r
def upload_chunks(self, url, filename, data=None):
form = ox.MultiPartForm()
if not data:
for key in data:
form.add_field(key, data[key])
data = self._json_request(url, form)
if 'uploadUrl' in data:
uploadUrl = data['uploadUrl']
f = open(filename)
fsize = os.stat(filename).st_size
done = 0
chunk = f.read(CHUNK_SIZE)
fname = os.path.basename(filename)
if isinstance(fname, unicode):
fname = fname.encode('utf-8')
while chunk:
print '%0.2f%% %s of %s uploaded \r' % (
100 * done/fsize, ox.formatBytes(done), ox.formatBytes(fsize)),
sys.stdout.flush()
form = ox.MultiPartForm()
form.add_file('chunk', fname, chunk)
if len(chunk) < CHUNK_SIZE or f.tell() == fsize:
form.add_field('done', '1')
try:
data = self._json_request(uploadUrl, form)
except KeyboardInterrupt:
print "\ninterrupted by user."
sys.exit(1)
except:
print "uploading chunk failed, will try again in 5 seconds"
if DEBUG:
print uploadUrl
import traceback
traceback.print_exc()
data = {'result': -1}
time.sleep(5)
if data and 'status' in data:
if data['status']['code'] == 403:
print "login required"
return False
if data['status']['code'] != 200:
print "request returned error, will try again in 5 seconds"
if DEBUG:
print data
time.sleep(5)
if data and data['result'] == 1:
done += len(chunk)
chunk = f.read(CHUNK_SIZE)
print ' '
return data and 'result' in data and data['result'] == 1
else:
if DEBUG:
if 'status' in data and data['status']['code'] == 401:
print "login required"
else:
print "failed to upload file to", url
print data
return False

View file

@ -164,8 +164,9 @@ def video(video, target, profile, info):
print 'Output:\t', target print 'Output:\t', target
except KeyboardInterrupt: except KeyboardInterrupt:
r = 1 r = 1
print "\n\ncleanup unfinished encoding:\nremoving", target if os.path.exists(target):
print "\n" print "\n\ncleanup unfinished encoding:\nremoving", target
os.unlink(target) print "\n"
os.unlink(target)
sys.exit(1) sys.exit(1)
return r == 0 return r == 0

View file

@ -36,7 +36,7 @@ or instead of OxFF to keep archive and pan.do/ra instance in sync.
packages=[ packages=[
'pandora_client' 'pandora_client'
], ],
install_requires=['python-firefogg', 'ox'], install_requires=['ox'],
keywords = [ keywords = [
], ],
classifiers = [ classifiers = [