cleanup, use Firefogg module

OxFF/bin/oxd.py

@@ -18,16 +18,12 @@ import tempfile
 import time
 from threading import Thread
 
-from twisted.cred.portal import IRealm, Portal
-from twisted.cred.checkers import InMemoryUsernamePasswordDatabaseDontUse
 from twisted.internet import task, reactor, threads
 from twisted.web import server
-from twisted.web.guard import HTTPAuthSessionWrapper, DigestCredentialFactory
+from twisted.web.resource import Resource
-from twisted.web.resource import Resource, IResource
 from twisted.web.static import File
 from twisted.web.server import NOT_DONE_YET
 
-from zope.interface import implements
 
 FFMPEG2THEORA = 'ffmpeg2theora'
 
@@ -93,21 +89,6 @@ def avinfo(filename):
 def hash_prefix(h):
     return [h[:2], h[2:4], h[4:6], h[6:]]
 
-def extract_all_stills():
-    db = Database('dev.sqlite')
-    conn = db.conn()
-    c = conn.cursor()
-    sql = 'SELECT path, oshash, info FROM file'
-    c.execute(sql)
-    for row in c:
-        video = row[0]
-        oshash = row[1]
-        info = json.loads(row[2])
-        if not 'Extras/' in video and 'video' in info and info['video']:
-            prefix = os.path.join('media', os.path.join(*hash_prefix(oshash)))
-            print video
-            extract_stills(video, prefix, info)
-
 def run_command(cmd, timeout=25):
     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
     while timeout > 0:
@@ -120,7 +101,7 @@ def run_command(cmd, timeout=25):
         killedpid, stat = os.waitpid(p.pid, os.WNOHANG)
     return p.returncode
     
-def extract_still(video, target, position):
+def extract_frame(video, target, position):
     fdir = os.path.dirname(target)
     if fdir and not os.path.exists(fdir):
         os.makedirs(fdir)
@@ -153,13 +134,13 @@ def extract_still(video, target, position):
     cmd = [
         vlc_path, '--vout=dummy', video, '--start-time=%s'%position, '--stop-time=%s'%out,
         '-I', 'dummy', '--video-filter=scene', '--scene-path=%s'%framedir,
-        '--scene-format=png', '--scene-ratio=25', '--scene-prefix=still', '--swscale-mode=2',
+        '--scene-format=png', '--scene-ratio=25', '--scene-prefix=frame', '--swscale-mode=2',
         '--sout-transcode-vcodec=avcodec', '--noaudio', 'vlc://quit',
     ]
     #print cmd
     run_command(cmd)
     
-    images = glob('%s/still*.png' % framedir)
+    images = glob('%s/frame*.png' % framedir)
     if images:
         shutil.move(images[0], target)
     shutil.rmtree(framedir)
@@ -175,6 +156,9 @@ def extract_still(video, target, position):
     images = glob('%s/*.png' % framedir)
     if images:
         shutil.move(images[-1], target)
+        r = 0
+    else:
+        r = 1
     os.chdir(cwd)
     shutil.rmtree(framedir)
     return r == 0
@@ -241,6 +225,7 @@ def extract_video(video, target, profile, info):
     fps = AspectRatio(info['video'][0]['framerate'])
 
     width = int(dar * height)
+    width += width % 2 
 
     bitrate = height*width*fps*bpp/1000
     aspect = dar.ratio
@@ -276,7 +261,10 @@ def extract_video(video, target, profile, info):
     while line:
         if line.startswith('frame='):
             frames = line.split('=')[1].strip().split(' ')[0]
-            enc_status[info['oshash']] = (float(frames) / fps) / info['duration']
+            if enc_status[info['oshash']] == 'cancel':
+                p.kill()
+            else:
+                enc_status[info['oshash']] = (float(frames) / fps) / info['duration']
         line = p.stderr.readline()
 
     p.wait()
@@ -461,14 +449,14 @@ class Database(object):
         if derivative['status'] == STATUS_NEW:
             if name.endswith('.png'):
                 for pos in video_frame_positions(f['info']['duration']):
-                    still_name = '%s.png' % pos
+                    frame_name = '%s.png' % pos
-                    still_d = self.derivative(oshash, still_name)
+                    frame_d = self.derivative(oshash, frame_name)
-                    if still_d['status'] == STATUS_NEW:
+                    if frame_d['status'] == STATUS_NEW:
-                        self.derivative(oshash, still_name, STATUS_EXTRACTING)
+                        self.derivative(oshash, frame_name, STATUS_EXTRACTING)
-                        if extract_still(f['path'], still_d['path'], pos):
+                        if extract_frame(f['path'], frame_d['path'], pos):
-                            self.derivative(oshash, still_name, STATUS_AVAILABLE)
+                            self.derivative(oshash, frame_name, STATUS_AVAILABLE)
                         else:
-                            self.derivative(oshash, still_name, STATUS_FAILED)
+                            self.derivative(oshash, frame_name, STATUS_FAILED)
             elif name.endswith('.webm'):
                 profile = name[:-5]
                 print 'now lets go, are we having fun?'
@@ -476,7 +464,10 @@ class Database(object):
                 if extract_video(f['path'], derivative['path'], profile, f['info']):
                     self.derivative(oshash, name, STATUS_AVAILABLE)
                 else:
-                    self.derivative(oshash, name, STATUS_FAILED)
+                    if enc_status[oshash] == 'cancel':
+                        self.derivative(oshash, name, STATUS_NEW)
+                    else:
+                        self.derivative(oshash, name, STATUS_FAILED)
 
     #volumes
     def update(self, path):
@@ -553,7 +544,7 @@ class Database(object):
         conn, c = self.conn()
         c.execute('DELETE FROM volume WHERE site=? AND user=? AND name=?', [site, user, name])
         conn.commit()
-        #fixme, files could be still used by sub volumes
+        #fixme, files could be frame used by sub volumes
         #c.execute('DELETE FROM file WHERE path LIKE ?', ["%s%%"%path])
 
     def rename_volume(self, site, user, name, new_name):
@@ -637,7 +628,7 @@ class OxControl(Resource):
                 args[arg] = request.args.get(arg)[0]
             return args
 
-        if request.args.get('key', '') != self.authkey: 
+        if request.args.get('key', [''])[0] != self.authkey: 
             response = {'status': '403', 'text': 'wrong authentication key provided'}
             return json_response(request, response)
 
@@ -703,6 +694,15 @@ class OxControl(Resource):
             threads.deferToThread(render, request)
             return NOT_DONE_YET
 
+        if request.path == '/cancel':
+            """
+            extract derivatives from videos
+            """
+            global enc_status
+            oshash = request.args.get("oshash", [None])[0]
+            if oshash and oshash in enc_status:
+                enc_status[oshash] = 'cancel'
+
         if request.path == '/extract':
             """
             extract derivatives from videos
@@ -719,7 +719,7 @@ class OxControl(Resource):
             elif not 'duration' in f['info']:
                 response = {'status': 'unkown format, can not extract data'}
             else:
-                if media == 'stills':
+                if media == 'frames':
                     name = '%s.png'%video_frame_positions(f['info']['duration'])[0]
                 elif media.endswith('.webm'):
                     profile = media[:-5]
@@ -745,8 +745,8 @@ class OxControl(Resource):
                         response['progress'] = enc_status.get(oshash, 0)
 
                 files = [f['path'] for f in self.db.derivatives(oshash)]
-                if media == 'stills':
+                if media == 'frames':
-                    response['stills'] = filter(lambda f: f.endswith('.png'), files)
+                    response['frames'] = filter(lambda f: f.endswith('.png'), files)
                 else:
                     response['video'] = filter(lambda f: f.endswith(media), files)
                     if response['video']: response['video'] = response['video'][0]
@@ -767,7 +767,7 @@ class OxControl(Resource):
                 response['info'] = f['info']
                 files = [f['location'] for f in self.db.derivatives(oshash)]
                 response['video'] = filter(lambda f: f.endswith('.webm'), files)
-                response['stills'] = filter(lambda f: f.endswith('.png'), files)
+                response['frames'] = filter(lambda f: f.endswith('.png'), files)
             return json_response(request, response)
 
         if request.path == '/stop':
@@ -778,7 +778,7 @@ class OxControl(Resource):
             response = {'status': 'ok'}
             return json_response(request, response)
 
-        return "<!DOCTYPE html><html>this is not for humans</html>"
+        return "<!DOCTYPE html><html><head></head><body>this is not for humans</body></html>"
 
 if __name__ == '__main__':
     db = 'dev.sqlite'
@@ -789,35 +789,11 @@ if __name__ == '__main__':
 
     port = 2620
     interface = '127.0.0.1'
-    interface = '10.26.20.10'
-    interface = '0.0.0.0'
 
     print 'http://%s:%d/' % (interface, port)
 
     root = OxControl(db)
 
-    """
-    username = root.db.get('username', 'fix')
-    password = root.db.get('password', 'me')
-
-    checker = InMemoryUsernamePasswordDatabaseDontUse()
-    checker.addUser(username, password)
-
-    class PublicHTMLRealm(object):
-        implements(IRealm)
-
-        def requestAvatar(self, avatarId, mind, *interfaces):
-            if IResource in interfaces:
-                return (IResource, root, lambda: None)
-            raise NotImplementedError()
-
-    portal = Portal(PublicHTMLRealm(), [checker])
-
-    credentialFactory = DigestCredentialFactory("md5", "oxbackend")
-    resource = HTTPAuthSessionWrapper(portal, [credentialFactory])
-
-    site = server.Site(resource)
-    """
     site = server.Site(root)
     reactor.listenTCP(port, site, interface=interface)
     reactor.run()

OxFF/components/OxFF.js

@@ -12,9 +12,10 @@ const Ci = Components.interfaces;
 
 Components.utils.import("resource://gre/modules/XPCOMUtils.jsm");
 
+Components.utils.import("resource://firefogg/uploader.jsm");
+
 Components.utils.import("resource://ox/utils.jsm");
 Components.utils.import("resource://ox/oxff.jsm");
-Components.utils.import("resource://ox/firefogg.jsm");
 
 var OxFFFactory =
 {
@@ -37,12 +38,12 @@ function OxFF() {
   this.app = Cc["@mozilla.org/fuel/application;1"].getService(Ci.fuelIApplication);
 
   this._site = this._window.document.location.hostname;
-  if(!this._site) {
+  if (!this._site) {
       this._site = 'localhost';
   }
   this.access();
 
-  if(!oxff.get('username')) {
+  if (!oxff.get('username')) {
     oxff.set('username', ox.makeRandomString(8));
     oxff.set('password', ox.makeRandomString(8));
   }
@@ -133,7 +134,7 @@ OxFF.prototype = {
     var msg = this.extensionID + ": ";
     for(var i=0;i<arguments.length;i++) {
       msg += arguments[i];
-      if(i+1<arguments.length)
+      if (i+1<arguments.length)
         msg += ', ';
     }
     this.app.console.log(msg);
@@ -153,13 +154,13 @@ OxFF.prototype = {
     q.params.site = this._site;
     this._access = false;  
     while(q.executeStep()) {
-      if(q.row.access == 1) {
+      if (q.row.access == 1) {
         this._access = true;
       }
     }
     q.finalize();
 
-    if(request && !this._access) {
+    if (request && !this._access) {
       var windowMediator = Cc["@mozilla.org/appshell/window-mediator;1"].getService(Ci.nsIWindowMediator);
       var nsWindow = windowMediator.getMostRecentWindow("navigator:browser");
       var box  = nsWindow.gBrowser.getNotificationBox();
@@ -178,86 +179,50 @@ OxFF.prototype = {
     }
     return this._access;
   },
-  
   api: function(action, data, callback) {
     var _this = this;
     if (typeof(data) == 'function') {
         callback = data;
         data = {};
     }
-    if(!this.canAccess())
+    if (!this.canAccess())
       return false;
 
     data["key"] = this.authkey;
     data["site"] = this._site;
     data["user"] = this._user;
-
+    
-    var req = Cc["@mozilla.org/xmlextras/xmlhttprequest;1"]
-                        .createInstance(Ci.nsIXMLHttpRequest);
-
-    req.addEventListener("error", function(e) {
-      _this.startDaemon();
-      ox.setTimeout(function() { _this.api(action, data, callback); }, 1000);
-    }, false);
-    req.addEventListener("load", function(e) {
-        //links should have base prefixed or whatever proxy is used to access them, i.e. some api call
-        //var data = JSON.parse(e.target.responseText);
-        //callback(data);
-        callback(e.target.responseText);
-    }, false);
-
     var base = this.base; //later base can not be a public property
     var url = base + action;
 
-    //req.open("POST", url, true, this.authkey, this.password);
+    ox.request({
-    req.open("POST", url, true);
+      url: url,
-
+      data: data,
-    try {
+      load: function(e) {
-    var formData = Cc["@mozilla.org/files/formdata;1"].createInstance(Ci.nsIDOMFormData);    
+          //links should have base prefixed or whatever proxy is used to access them, i.e. some api call
-    if (data) {
+          //var data = JSON.parse(e.target.responseText);
-        for(key in data) {
+          //callback(data);
-            formData.append(key, data[key]);
+          callback(e.target.responseText);
-        }
+      },
-    }
+      error: function(e) {
-    } catch(e) {
+        _this.startDaemon();
-      //FF3.6 fallback
+        ox.setTimeout(function() { _this.api(action, data, callback); }, 1000);
-      var converter = Cc["@mozilla.org/intl/scriptableunicodeconverter"].createInstance(Ci.nsIScriptableUnicodeConverter);
-      converter.charset = "UTF-8";
-      var boundary = "--------XX" + Math.random();
-      var formData='';
-      if (data) {
-          for(key in data) {
-            if (data[key]) {
-              formData +="--" + boundary + "\r\n" +
-                         "Content-Disposition: form-data; name=\""+key+"\"\r\n\r\n" +
-                         data[key] + "\r\n";
-            }
-          }
       }
-      formData += "--" + boundary + "--\r\n";
+    });
-      req.setRequestHeader("Content-type", "multipart/form-data; boundary=" + boundary);
-      req.setRequestHeader("Content-length", formData.length);
-      var formData = converter.convertToInputStream(formData);
-    }
-
-    req.send(formData);
     return true;
   },
-  extract: function(oshash, media, callback) {
+  chunk_upload: function(url, data, file, callback, progress) {
-    return this.api('extract', {'oshash': oshash, 'media': media}, callback);
+    if (!this._chunk_upload) {
-  },
-  upload: function(url, file, callback, progress) {
-    if(!this.chunk_upload) {
       var _this = this;
-      this.chunk_upload = new FirefoggUploader(this, file, url, {});
+      this._chunk_upload = new FirefoggUploader(this, file, url, data);
-      this.chunk_upload.onProgress(function(upload_progress) {
+      this._chunk_upload.onProgress(function(upload_progress) {
         var info = {'status': 'uploading', 'progress': upload_progress};
         progress(JSON.stringify(info));
       });
-      this.chunk_upload.ready = true;
+      this._chunk_upload.ready = true;
-      this.chunk_upload.start();
+      this._chunk_upload.start();
-      this.chunk_upload.done(function() {
+      this._chunk_upload.done(function() {
-        _this.chunk_upload = null;
+        _this._chunk_upload = null;
         var info = {'status': 'done', 'progress': 1};
         callback(JSON.stringify(info));
       });
@@ -265,33 +230,106 @@ OxFF.prototype = {
     }
     return false;
   },
-  uploadVideo: function(oshash, url, profile, callback, progress) {
+  upload: function(options, callback, progress) {
     var _this = this;
 
-    if(!this.canAccess())
+    if (!this.canAccess())
       return false;
 
-    if(progress)
+    var options = JSON.parse(options);
+    if (options.data)
+      var formData = options.data;
+    else
+      var formData = {};
+    var url = options.url;
+
+    if (callback)
+      callback = callback.callback;
+    if (progress)
       progress = progress.callback;
 
-    var timer = ox.setInterval(function() {
+    if (options.action == 'frames') {
-      _this.api('extract', {'oshash': oshash, 'media': profile}, function(result) {
+      var timer = ox.setInterval(function() {
-        var data = JSON.parse(result);
+        _this.api('extract', {oshash: options.oshash, media: 'frames'}, function(result) {
-        if (data.status == 'extracting') {
+          var data = JSON.parse(result);
-          if(progress) {
+          if (data.status == 'extracting') {
-            var info = {'status': data.status, 'progress': data.progress};
+            if (progress && data.progress >= 0) {
-            progress(JSON.stringify(info));
+              var info = {'status': data.status, 'progress': data.progress};
-          }
+              progress(JSON.stringify(info));
-        } else {
+            }
-          timer.cancel();
+          } else {
-          if(data.status == 'available') {
+            timer.cancel();
-            _this.upload(url, data.video, callback.callback, progress);
+            if (data.status == 'available') {
-          } else if(data.status == 'failed') {
+              formData.frame = new Array();
-            _this.debug('failed');
+              for(i in data.frames) {
+                var f  = Cc["@mozilla.org/file/local;1"].createInstance(Ci.nsILocalFile);
+                f.initWithPath(data.frames[i]);
+                formData.frame.push(f);
+              }
+              ox.request({
+                url: url,
+                data: formData,
+                load: function(e) {
+                    if (callback)
+                      callback(e.target.responseText);
+                },
+                error: function(e) {
+                  _this.debug('uploading frames failed');
+                }
+              });
+            } else if (data.status == 'failed') {
+              _this.debug('uploading frames failed');
+            }
           }
+        });
+      }, 2000);
+    } else if (options.action == 'file') { //FIXME: should this use Firefogg style upload for lager files?
+      var conn = oxff.getDB();
+      var q = conn.createStatement("SELECT path FROM file WHERE oshash = :oshash LIMIT 1");
+      q.params.oshash = options.oshash;
+      if (q.executeStep())
+        var path = q.row.path;
+      q.finalize();
+      if (path) {
+        formData.file = Cc["@mozilla.org/file/local;1"].createInstance(Ci.nsILocalFile)
+        formData.file.initWithPath(path);
+        if (formData.file.exists()) { //FIXME: should check if more than one file with oshash is known
+          ox.request({
+            url: url,
+            data: formData,
+            load: function(e) {
+                if (callback)
+                  callback(e.target.responseText);
+            },
+            error: function(e) {
+              _this.debug('uploading file failed');
+            }
+          });
+          return true;
         }
-      });
+      }
-    }, 2000);
+      return false;
+    } else if (options.action == 'video') {
+      var timer = ox.setInterval(function() {
+        _this.api('extract', {oshash: options.oshash, media: options.profile}, function(result) {
+          var data = JSON.parse(result);
+          if (data.status == 'extracting') {
+            if (progress && data.progress >= 0) {
+              var info = {'status': data.status, 'progress': data.progress};
+              progress(JSON.stringify(info));
+            }
+          } else {
+            timer.cancel();
+            if (data.status == 'available') {
+              _this.chunk_upload(url, formData, data.video, callback, progress);
+            } else if (data.status == 'failed') {
+              _this.debug('uploading video failed');
+            }
+          }
+        });
+      }, 2000);
+
+    }
     return true;
   },
   files: function(volume, callback) {
@@ -301,7 +339,7 @@ OxFF.prototype = {
     return this.api('get', {'oshash': oshash, 'media': media}, callback.callback);
   },
   login: function(user) {
-    if(this._user==null) {
+    if (this._user==null) {
       this._user = user;
       return true;
     }
@@ -313,9 +351,9 @@ OxFF.prototype = {
       _this._user = null;
       _this._daemon = null;
     });
-    if (this.chunk_upload) {
+    if (this._chunk_upload) {
-      this.chunk_upload.cancel();
+      this._chunk_upload.cancel();
-      this.chunk_upload = null;
+      this._chunk_upload = null;
     }
     return true;
   },
@@ -324,7 +362,7 @@ OxFF.prototype = {
     return true;
   },
   setLocation: function(name) {
-    if(!this._access) {
+    if (!this._access) {
         return false;
     }
     const nsIFilePicker = Ci.nsIFilePicker;
@@ -358,11 +396,11 @@ OxFF.prototype = {
     oxff.access(this._site, this._access);
   },
   canAccess: function() {
-      if(!this._access) {
+      if (!this._access) {
         this.debug('permission deinied');
         return false;
     }
-    if(!this._user) {
+    if (!this._user) {
         this.debug('login first');
         return false;
     }
@@ -371,7 +409,7 @@ OxFF.prototype = {
   startDaemon: function() {
     var _this = this;
     //if daemon is already running to not start it again
-    if(this._daemon && this._daemon.isRunning) {
+    if (this._daemon && this._daemon.isRunning) {
       this.debug('daemon is still starting up, not starting again');
       return false;
     }

OxFF/install.rdf

@@ -10,16 +10,16 @@
     <em:targetApplication>
       <Description>
        	<em:id>{ec8030f7-c20a-464f-9b0e-13a3a9e97384}</em:id>
-      	<em:minVersion>3.6</em:minVersion>
+      	<em:minVersion>4.0b1</em:minVersion>
-      	<em:maxVersion>4.0b9</em:maxVersion>
+      	<em:maxVersion>4.0</em:maxVersion>
       </Description>
     </em:targetApplication>		
 
     <em:requires>
       <Description>
         <em:id>firefogg@firefogg.org</em:id>
-        <em:minVersion>1.2.10</em:minVersion>
+        <em:minVersion>1.9.00</em:minVersion>
-        <em:maxVersion>1.2.*</em:maxVersion>
+        <em:maxVersion>2.0.*</em:maxVersion>
       </Description>
     </em:requires>
 

OxFF/modules/firefogg.jsm

@@ -1,299 +0,0 @@
-// -*- coding: utf-8 -*-
-// vi:si:et:sw=2:sts=2:ts=2:ft=js
-
-let EXPORTED_SYMBOLS = [ "FirefoggUploader" ];
-
-const Cc = Components.classes;
-const Ci = Components.interfaces;
-
-function FirefoggUploader(parent, path, url, data) {
-  this._parent = parent;
-  this._path = path;
-  this._url = url;
-  this._chunkUrl = false;
-  this._data = data;
-  this._status = 'initiated';
-  this.encodingStatus = 'encoding';
-  this.progress = 0.0;
-  this.chunkSize = 1024*1024; //1MB, is there a need to make this variable?
-  this.chunksUploaded = 0;
-  this.chunkStatus = {};
-  this._done_cb = function(upload) {};
-  this._onProgress = function(progress) {};
-  this.filename = 'video.ogv';
-  this.canceled = false;
-  this._retries = 0;
-  this._maxRetry = -1;
-  this.ready = false;
-}
-
-/*
- Internal helper object dealing with chunked upload as defined at
- http://firefogg.org/dev/chunk_post.html
-*/
-FirefoggUploader.prototype = {
-  start: function() {
-    var boundary = "--------XX" + Math.random();
-
-    this._status = 'requesting chunk upload';
-
-    var req = Cc["@mozilla.org/xmlextras/xmlhttprequest;1"].createInstance(Ci.nsIXMLHttpRequest);
-
-    //XMLHttpRequest status callbacks
-    var upload = this;
-    function transferComplete(evt) {
-      var response = {};
-      upload.responseText = evt.target.responseText;
-      try {
-        response = JSON.parse(evt.target.responseText);
-        //dump(evt.target.responseText);
-      } catch(e) {
-        dump('FAILED to parse response:\n\n');
-        dump(evt.target.responseText);
-        response = {};
-      }
-      if (response.maxRetry) {
-        upload._maxRetry = response.maxRetry;
-      }
-      upload._chunkUrl = response.uploadUrl;
-      if (upload._chunkUrl) {
-        //dump('now tracking chunk uploads to ' + upload._chunkUrl + '\n');
-        upload._status = 'uploading';
-        upload._progress = 0.0;
-        upload._uploadChunkId = 0;
-        upload.uploadChunk(0, false);
-      } else {
-        upload._status = 'upload failed, no upload url provided';
-        upload._progress = -1;
-        upload._parent.cancel();
-        return;
-      }
-    }
-    function transferFailed(evt) {
-      upload._status = 'uplaod failed';
-      upload._progress = -1;
-      upload.responseText = evt.target.responseText;
-      //dump(evt.target.responseText);
-    }
-    req.addEventListener("load", transferComplete, false);
-    req.addEventListener("error", transferFailed, false);
-
-    req.open("POST", this._url);
-
-    var formData = "";
-    for (key in this._data) {
-      formData += "--" + boundary + "\r\n" +
-                  "Content-Disposition: form-data; name=\""+key+"\"\r\n\r\n" + this._data[key] + "\r\n";
-    }
-    formData += "--" + boundary + "--\r\n";
-
-    var converter = Cc["@mozilla.org/intl/scriptableunicodeconverter"].createInstance(Ci.nsIScriptableUnicodeConverter);
-    converter.charset = "UTF-8";
-    var formStream = converter.convertToInputStream(formData);
-
-    req.setRequestHeader("Content-type", "multipart/form-data; boundary=" + boundary);
-    req.setRequestHeader("Content-length", formStream.available());
-    req.send(formStream);
-  },
-  onProgress: function(cb) {
-    this._onProgress = cb;
-  },
-  done: function(done_cb) {
-    //used by Firefogg to indicate that encoding is done
-    //provies callback that is called once upload is done too
-    this.encodingStatus = 'done';
-    this._done_cb = done_cb;
-  },
-  uploadChunk: function(chunkId, last) {
-    /*
-      upload chunk with given chunkId, last indicated if this is the last chunk.
-      once upload is done next chunk is queued
-    */
-    var file = Cc["@mozilla.org/file/local;1"].createInstance(Ci.nsILocalFile);
-    file.initWithPath(this._path);
-    this.filename = file.leafName;
-    this.filename = this.filename.replace(/\(\.\d+\).ogv/, '.ogv');
-    this.filename = this.filename.replace(/\(\.\d+\).webm/, '.webm');
-
-    var fileStream = Cc["@mozilla.org/network/file-input-stream;1"].createInstance(Ci.nsIFileInputStream);
-    fileStream.init(file, -1, -1, false);
-    fileStream.QueryInterface(Ci.nsISeekableStream);
-    var f = Cc["@mozilla.org/binaryinputstream;1"].createInstance(Ci.nsIBinaryInputStream);
-    f.setInputStream(fileStream);
-
-    var bytesAvailable = fileStream.available();
-    var chunk = false;
-    var chunkOffset = chunkId * this.chunkSize;
-    this.progress = parseFloat(chunkOffset)/bytesAvailable;
-
-    if (last) {
-      //last chunk, read add remaining data
-      fileStream.seek(fileStream.NS_SEEK_SET, chunkOffset);
-      chunk = f.readBytes(bytesAvailable-chunkOffset);
-    }
-    else if (this.ready && bytesAvailable >= chunkOffset + this.chunkSize) {
-      //read next chunk
-      fileStream.seek(fileStream.NS_SEEK_SET, chunkOffset);
-      chunk = f.readBytes(this.chunkSize);
-    } else {
-      if (this.encodingStatus == 'done' && this._status == 'uploading') {
-          //uploading is done and last chunk is < chunkSize, upload remaining data
-          this._status = 'success';
-          this.uploadChunk(chunkId, true);
-      } else {
-        //encoding not ready. wait for 2 seconds and try again
-        var timer = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer);
-        timer.initWithCallback(
-          function(obj, chunkId, last) {
-            return function() { obj.uploadChunk(chunkId, last) }
-          }(this, chunkId, last),
-          2000, Ci.nsITimer.TYPE_ONE_SHOT);
-      }
-      f.close();
-      fileStream.close();
-      return;
-    }
-    f.close();
-    fileStream.close();
-
-    //FIXME: should this be checked so it does not get called again?
-    this.chunkStatus[chunkId] = {};
-
-    //dump('POST ' + this._chunkUrl + ' uploading chunk ' + chunkId +' ('+chunk.length+' bytes)\n');
-
-    var boundary = "--------XX" + Math.random();
-
-    var req = Cc["@mozilla.org/xmlextras/xmlhttprequest;1"].createInstance(Ci.nsIXMLHttpRequest);
-
-    var upload = this;
-    function transferComplete(evt) {
-      //if server resonds with valid {result=1} and status was is still 'uploading' go on to next chunk
-      upload.responseText = evt.target.responseText;
-      try {
-        var response = JSON.parse(evt.target.responseText);
-      } catch(e) {
-        dump('FAILED to parse response:\n\n');
-        dump(evt.target.responseText);
-        var response = {};
-      }
-      /*
-      if (upload.canceled) {
-        //FIXME: should we let the server know that upload was canceled?
-      }
-      else */
-
-      /*
-      dump('\n\n');
-      dump(evt.target.responseText);
-      dump('\n************\n\n');
-      */
-      if (response.done == 1) {
-        //upload finished, update state and expose result
-        upload.resultUrl = response.resultUrl;
-        upload.progress = 1;
-        if (upload._done_cb)
-          upload._done_cb(upload);
-        //reset retry counter
-        upload._retries = 0;
-      }
-      else if (response.result == 1) {
-        //start uploading next chunk
-        upload._uploadChunkId = chunkId + 1;
-        upload.uploadChunk(upload._uploadChunkId, false);
-        //upload status
-        upload.chunkStatus[chunkId].progress = 1;
-        upload.chunkStatus[chunkId].loaded = evt.loaded;
-        upload.chunksUploaded++;
-        //reset retry counter
-        upload._retries = 0;
-      } else {
-        //failed to upload, try again in 3 second
-        //dump('could not parse response, failed to upload, will try again in 3 second\n');
-        //dump(evt.target.responseText);
-        if (upload.max_retry > 0 && upload._retries > upload.max_retry) {
-          upload.cancel();
-          upload._status = 'uplaod failed';
-          upload._progress = -1;
-        } else {
-          upload._retries++;
-          var timer = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer);
-          timer.initWithCallback(
-            function(obj, chunkId, last) {
-              return function() { obj.uploadChunk(chunkId, last); };
-            }(upload, chunkId, last), 3000, Ci.nsITimer.TYPE_ONE_SHOT);
-        }
-      }
-    }
-    function transferFailed(evt) {
-      //failed to upload, try again in 3 second
-      //dump('transferFailed, will try again in 3 second\n');
-      if (upload.max_retry > 0 && upload._retries > upload.max_retry) {
-        upload.cancel();
-        upload._status = 'uplaod failed';
-        upload._progress = -1;
-      } else {
-        upload._retries++;
-        var timer = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer);
-        timer.initWithCallback(
-          function(obj, chunkId, last) {
-            return function() { obj.uploadChunk(chunkId, last); };
-          }(upload, chunkId, last), 3000, Ci.nsITimer.TYPE_ONE_SHOT);
-      }
-    }
-    function updateProgress(evt) {
-      if (evt.lengthComputable) {
-        upload.progress = parseFloat(chunkOffset + evt.loaded)/bytesAvailable;
-        upload.chunkStatus[chunkId].loaded = evt.loaded;
-        upload.chunkStatus[chunkId].loaded = evt.total;
-        //dump('progress: chunk ' + chunkId + ' uploaded ' + evt.loaded + '\n');
-        upload._onProgress(upload.progress);
-      }
-    }
-    req.upload.addEventListener("progress", updateProgress, false);
-
-    req.addEventListener("load", transferComplete, false);
-    req.addEventListener("error", transferFailed, false);
-
-    req.open("POST", this._chunkUrl);
-
-    var chunk = "--" + boundary + "\r\n" +
-                "Content-Disposition: form-data; name=\"chunk\"; filename=\"" + this.filename + "\"\r\n" +
-                "Content-type: video/ogg\r\n\r\n" + chunk;
-    var chunkStream = Cc["@mozilla.org/io/string-input-stream;1"].createInstance(Ci.nsIStringInputStream);
-    chunkStream.setData(chunk, chunk.length);
-
-    // write  done flag into stream
-    var formData = "\r\n";
-    var _data = {
-      chunkId: chunkId
-    };
-    if (last) {
-      _data['done'] = 1;
-    }
-    for (key in _data) {
-        formData += "--" + boundary + "\r\n" +
-                    "Content-Disposition: form-data; name=\""+key+"\"\r\n\r\n" + _data[key] + "\r\n";
-    }
-    formData += "--" + boundary + "--\r\n";
-    var converter = Cc["@mozilla.org/intl/scriptableunicodeconverter"].createInstance(Ci.nsIScriptableUnicodeConverter);
-    converter.charset = "UTF-8";
-    var formStream = converter.convertToInputStream(formData);
-
-    var multiStream = Cc["@mozilla.org/io/multiplex-input-stream;1"].createInstance(Ci.nsIMultiplexInputStream);
-    multiStream.appendStream(chunkStream);
-    multiStream.appendStream(formStream);
-
-    //send mupltiplrex stream
-    req.setRequestHeader("Content-type", "multipart/form-data; boundary=" + boundary);
-    req.setRequestHeader("Content-length", multiStream.available());
-    //this._current_req = req;
-    req.send(multiStream);
-  },
-  cancel: function() {
-    this.canceled = true;
-    if (this._current_req) {
-      this._current_req.abort();
-      this._current_req = null;
-    }
-  },
-}

OxFF/modules/utils.jsm

@@ -51,6 +51,85 @@ let ox = {
         }
         return s;
     },
+    request: function(options) {
+        var url = options.url,
+            data = options.data,
+            boundary = "--------XX" + Math.random();
+
+        var converter = Cc["@mozilla.org/intl/scriptableunicodeconverter"]
+                          .createInstance(Ci.nsIScriptableUnicodeConverter);
+        converter.charset = "UTF-8";
+
+        var req = Cc["@mozilla.org/xmlextras/xmlhttprequest;1"]
+                    .createInstance(Ci.nsIXMLHttpRequest);
+
+        var multiStream = Cc["@mozilla.org/io/multiplex-input-stream;1"]
+                            .createInstance(Ci.nsIMultiplexInputStream);
+
+        if(options.progress)
+            req.upload.addEventListener("progress", options.progress, false);
+        if(options.load)
+            req.addEventListener("load", options.load, false);
+        if(options.error)
+            req.addEventListener("error", options.error, false);
+        if(options.abort)
+            req.addEventListener("abort", options.abort, false);
+
+        function appendData(key, value) {
+            if(value.leafName) {
+              try {
+                var mimeService = Cc["@mozilla.org/mime;1"].createInstance(Ci.nsIMIMEService);
+                var mimeType = mimeService.getTypeFromFile(value);
+              }
+              catch(e) {
+                var mimeType = "application/octet-stream";
+              }
+
+              var filename = value.leafName;
+              var formData = "--" + boundary + "\r\n" +
+                             "Content-Disposition: form-data; name=\"" + key +
+                             "\"; filename=\"" + filename + "\"\r\n" +
+                             "Content-type: " + mimeType + "\r\n\r\n";
+              var formData = converter.convertToInputStream(formData);
+              multiStream.appendStream(formData);
+
+              var fileStream = Cc["@mozilla.org/network/file-input-stream;1"]
+                                 .createInstance(Ci.nsIFileInputStream);
+              fileStream.init(value, 0x01, 0644, 0x04); // file is an nsIFile instance
+              multiStream.appendStream(fileStream);
+
+              formData = "\r\n";
+              formData = converter.convertToInputStream(formData);
+              multiStream.appendStream(formData);
+
+            } else {
+              var formData = "--" + boundary + "\r\n" +
+                          "Content-Disposition: form-data; name=\""+key+"\"\r\n\r\n" +
+                          value + "\r\n";
+              formData = converter.convertToInputStream(formData);
+              multiStream.appendStream(formData);
+            }
+        }
+        if (data) {
+            for(key in data) {
+                if (typeof(data[key]) == 'object' && data[key].length>0) {
+                    for(i in data[key])
+                        appendData(key, data[key][i]);
+                } else if (data[key]) {
+                    appendData(key, data[key]);
+                }
+            }
+        }
+        var formData = "--" + boundary + "--\r\n";
+        formData = converter.convertToInputStream(formData);
+        multiStream.appendStream(formData);
+
+        req.open("POST", url);
+        req.setRequestHeader("Content-type", "multipart/form-data; boundary=" + boundary);
+        req.setRequestHeader("Content-length", multiStream.available());
+        req.send(multiStream);
+        return req;
+    },
     subprocess: function(command, options, callback) {
         if(!this.ipcService) {
             this.ipcService = Cc["@mozilla.org/process/ipc-service;1"]

src/nsIOxFF.idl

@@ -27,7 +27,6 @@ interface nsIOxFF : nsISupports
   boolean update(in oxICallback callback);
   boolean files(in AString volume, in oxICallback callback);
   boolean get(in AString oshash, in AString media, in oxICallback callback);
-  boolean extract(in AString oshash, in AString media, in oxICallback callback);
+  boolean upload(in AString options, [optional] in oxICallback callback, [optional] in oxICallback progress);
-  boolean uploadVideo(in AString oshash, in AString url, in AString profile, in oxICallback callback, [optional] in oxICallback progress);
 };
 

test/import.html

@@ -4,6 +4,20 @@
 <script src="http://oxjs.org/js/jquery-1.4.2.min.js"></script>
 <script type="text/javascript" src="/static/oxjs/build/js/ox.js"></script>
 <script>
+function absolute_url(url) {
+    var base = document.location.href;
+    if (url.substring(0, 1) == '/') {
+        url = document.location.href.substring(0, document.location.href.length-document.location.pathname.length) + url;
+    }
+    else {
+        if(base.substring(base.length-1) == '/')
+            url = base + url;
+        else
+            url = base + '/' + url;
+    }
+    return url;
+}
+
 pandora = {};
 pandora.request_url = '/api/';
 
@@ -102,31 +116,63 @@ function for_each_sorted(elements, callback) {
     });
 }
 
+function uploadFile(oshash) {
+    var url = absolute_url('/api/');
+    ox.upload(JSON.stringify({
+        url: url,
+        data: {action: 'upload', oshash: oshash},
+        oshash: oshash,
+        action: 'file'
+        }),
+        function(result) {
+            $('#' + oshash).css('background', '#fff');
+            console.log(result);
+        }
+    );
+}
+
 function extract(oshash) {
-    console.log(oshash);
+    /*
-   /*
     ox.extract(oshash, 'stills', function(result) {
         console.log(result);
     });
-*/
+    */
-    profile = '96p.webm';
+    var url = absolute_url('/api/');
-    var url = 'http://127.0.0.1:8000/api/upload/?profile='+profile+'&oshash='+oshash;
+    ox.upload(JSON.stringify({
-    
+        url: url,
-    $('<div>').attr('id', 'progress_'+oshash)
+        data: {action: 'upload', oshash: oshash},
-              .appendTo('#' + oshash);
+        oshash: oshash,
-    ox.uploadVideo(oshash, url, profile, function(result) {
+        action: 'frames'
-        console.log(result);
+        }),
-        $('#' + oshash).css('background', '#fff');
+        function(result) {
-        $('#' + oshash).parent().css('background', '#fff');
+            //FIXME: check result before posting video
-        $('#progress_'+oshash).remove();
+            profile = '96p.webm';
-    }, function(result) {
+            var url = absolute_url('/api/upload/') + '?profile=' + profile + '&oshash=' + oshash;
-        try {
+
-            var data = JSON.parse(result);
+            $('<div>').attr('id', 'progress_'+oshash)
-            $('#progress_'+oshash).html(data.status +': '+ data.progress);
+                      .appendTo('#' + oshash);
-        } catch(e) {
+
-            console.log('progress failed', e);
+            ox.upload(JSON.stringify({
+                oshash: oshash,
+                action: 'video',
+                profile: profile,
+                url: url
+                }),
+                function(result) {
+                    console.log(result);
+                    $('#' + oshash).css('background', '#fff');
+                    $('#' + oshash).parent().css('background', '#fff');
+                    $('#progress_'+oshash).remove();
+                }, function(result) {
+                    try {
+                        var data = JSON.parse(result);
+                        $('#progress_'+oshash).html(data.status +': '+ data.progress);
+                    } catch(e) {
+                        console.log('progress failed', e);
+                    }
+            });
         }
-    });
+    );
 }
 
 function update() {
@@ -160,6 +206,12 @@ function update() {
                             });
                             $.each(result.data.file, function(i, oshash) {
                                 $('#' + oshash).css('background', 'blue');
+                                $('#' + oshash).unbind('click').click(function() {
+                                    $(this).unbind('click');
+                                    uploadFile(this.id);
+                                    return true;
+                                });
+
                             });
                         }
                         if (result.data.info.length>0) {