From 67af01db756799113dec9d74c133765e19a35145 Mon Sep 17 00:00:00 2001 From: j <0x006A@0x2620.org> Date: Fri, 30 Dec 2011 18:47:28 +0530 Subject: [PATCH] show progress during upload --- pandora_client/__init__.py | 97 ++++++++++++++++++++++++++++++-------- pandora_client/extract.py | 7 +-- setup.py | 2 +- 3 files changed, 82 insertions(+), 24 deletions(-) diff --git a/pandora_client/__init__.py b/pandora_client/__init__.py index a2cb02d..cd160eb 100644 --- a/pandora_client/__init__.py +++ b/pandora_client/__init__.py @@ -1,28 +1,28 @@ #!/usr/bin/python # -*- coding: utf-8 -*- # vi:si:et:sw=4:sts=4:ts=4 -# GPL 2010 +# GPL 2012 from __future__ import division, with_statement import os -import urllib2 -import cookielib import json import sqlite3 import time import shutil -import webbrowser +import sys -from firefogg import Firefogg import ox import extract import utils -__version__ = '0.2' DEBUG = False + +__version__ = '0.2' +CHUNK_SIZE = 1024*1024 default_media_cache = os.environ.get('oxMEDIA', os.path.expanduser('~/.ox/media')) + def encode(filename, prefix, profile, info=None): if not info: info = utils.avinfo(filename) @@ -41,6 +41,7 @@ def encode(filename, prefix, profile, info=None): frames.append(frame_f) video_f = os.path.join(cache, profile) if not os.path.exists(video_f): + return False extract.video(filename, video_f, profile, info) return { 'info': info, @@ -171,7 +172,7 @@ class Client(object): conn.commit() def scan(self): - print "Checking for new files ..." + print "checking for new files ..." for name in self._config['volumes']: path = self._config['volumes'][name] path = os.path.normpath(path) @@ -200,7 +201,7 @@ class Client(object): c.execute('UPDATE file SET deleted=? WHERE path=?', (deleted, f)) conn.commit() - print "Scanned volume %s: %s files, %s new, %s deleted" % ( + print "scanned volume %s: %s files, %s new, %s deleted" % ( name, len(files), len(files) - len(known_files), len(deleted_files)) def extract(self): @@ -257,13 +258,13 @@ class Client(object): post = {} post['files'] = files['files'] post['volume'] = name - print 'Sending list of files in %s (%s total)' % (name, len(post['files'])) + print 'sending list of files in %s (%s total)' % (name, len(post['files'])) r = self.api.update(post) if r['status']['code'] == 200: #backend works on update request asyncronously, wait for it to finish if 'taskId' in r['data']: t = self.api.taskStatus(task_id=r['data']['taskId']) - print 'Waiting for server ...' + print 'waiting for server ...' while t['data']['status'] == 'PENDING': time.sleep(5) t = self.api.taskStatus(task_id=r['data']['taskId']) @@ -275,7 +276,7 @@ class Client(object): info = r['data']['info'] max_info = 100 total = len(info) - print 'Sending info for %s files' % total + print 'sending info for %s files' % total for offset in range(0, total, max_info): post = {'info': {}, 'upload': True} for oshash in info[offset:offset+max_info]: @@ -291,7 +292,7 @@ class Client(object): files.append(path) break if files: - print '\nCould encoded and upload %s videos:\n' % len(files) + print '\ncould encoded and upload %s videos:\n' % len(files) print '\n'.join(files) if r['data']['file']: files = [] @@ -301,7 +302,7 @@ class Client(object): files.append(path) break if files: - print '\nCould upload %s subtitles:\n' % len(files) + print '\ncould upload %s subtitles:\n' % len(files) print '\n'.join(files) def upload(self): @@ -335,14 +336,14 @@ class Client(object): r = self.api.update(post) if r['data']['file']: - print 'Uploading %s files' % len(r['data']['file']) + print 'uploading %s files' % len(r['data']['file']) for oshash in r['data']['file']: if oshash in filenames: filename = filenames[oshash] self.api.uploadData(os.path.join(prefix, filename), oshash) if r['data']['data']: - print 'Encoding and uploading %s videos' % len(r['data']['data']) + print 'encoding and uploading %s videos' % len(r['data']['data']) for oshash in r['data']['data']: data = {} if oshash in filenames: @@ -396,8 +397,6 @@ class API(ox.API): self.media_cache = default_media_cache def uploadVideo(self, filename, data, profile, info=None): - if DEBUG: - print filename.encode('utf-8') i = encode(filename, self.media_cache, profile, info) if not i: print "failed" @@ -419,10 +418,9 @@ class API(ox.API): #upload video if os.path.exists(i['video']): size = ox.formatBytes(os.path.getsize(i['video'])) - print "Uploading %s (%s) of %s" % (profile, size, filename) + print "uploading %s of %s (%s)" % (profile, os.path.basename(filename), size) url = self.url + 'upload/' + '?profile=' + str(profile) + '&id=' + i['oshash'] - ogg = Firefogg(cj=self._cj, debug=DEBUG) - if not ogg.upload(url, i['video'], data): + if not self.upload_chunks(url, i['video'], data): if DEBUG: print "failed" return False @@ -443,3 +441,62 @@ class API(ox.API): r = self._json_request(self.url, form) return r + def upload_chunks(self, url, filename, data=None): + form = ox.MultiPartForm() + if not data: + for key in data: + form.add_field(key, data[key]) + data = self._json_request(url, form) + if 'uploadUrl' in data: + uploadUrl = data['uploadUrl'] + f = open(filename) + fsize = os.stat(filename).st_size + done = 0 + chunk = f.read(CHUNK_SIZE) + fname = os.path.basename(filename) + if isinstance(fname, unicode): + fname = fname.encode('utf-8') + while chunk: + print '%0.2f%% %s of %s uploaded \r' % ( + 100 * done/fsize, ox.formatBytes(done), ox.formatBytes(fsize)), + sys.stdout.flush() + form = ox.MultiPartForm() + form.add_file('chunk', fname, chunk) + if len(chunk) < CHUNK_SIZE or f.tell() == fsize: + form.add_field('done', '1') + try: + data = self._json_request(uploadUrl, form) + except KeyboardInterrupt: + print "\ninterrupted by user." + sys.exit(1) + except: + print "uploading chunk failed, will try again in 5 seconds" + if DEBUG: + print uploadUrl + import traceback + traceback.print_exc() + data = {'result': -1} + time.sleep(5) + if data and 'status' in data: + if data['status']['code'] == 403: + print "login required" + return False + if data['status']['code'] != 200: + print "request returned error, will try again in 5 seconds" + if DEBUG: + print data + time.sleep(5) + if data and data['result'] == 1: + done += len(chunk) + chunk = f.read(CHUNK_SIZE) + print ' ' + return data and 'result' in data and data['result'] == 1 + else: + if DEBUG: + if 'status' in data and data['status']['code'] == 401: + print "login required" + else: + print "failed to upload file to", url + print data + return False + diff --git a/pandora_client/extract.py b/pandora_client/extract.py index 0493c8f..c50e6f4 100644 --- a/pandora_client/extract.py +++ b/pandora_client/extract.py @@ -164,8 +164,9 @@ def video(video, target, profile, info): print 'Output:\t', target except KeyboardInterrupt: r = 1 - print "\n\ncleanup unfinished encoding:\nremoving", target - print "\n" - os.unlink(target) + if os.path.exists(target): + print "\n\ncleanup unfinished encoding:\nremoving", target + print "\n" + os.unlink(target) sys.exit(1) return r == 0 diff --git a/setup.py b/setup.py index 7762977..d189ccc 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ or instead of OxFF to keep archive and pan.do/ra instance in sync. packages=[ 'pandora_client' ], - install_requires=['python-firefogg', 'ox'], + install_requires=['ox'], keywords = [ ], classifiers = [