update blocking in oxd + callback in oxff, split modules in ox.jsm and oxff.jsm

OxFF/bin/oxd.py

@@ -16,16 +16,17 @@ import sys
 import shutil
 import tempfile
 import time
-import thread
 from threading import Thread
 
 from twisted.cred.portal import IRealm, Portal
 from twisted.cred.checkers import InMemoryUsernamePasswordDatabaseDontUse
-from twisted.internet import task, reactor
+from twisted.internet import task, reactor, threads
 from twisted.web import server
 from twisted.web.guard import HTTPAuthSessionWrapper, DigestCredentialFactory
 from twisted.web.resource import Resource, IResource
 from twisted.web.static import File
+from twisted.web.server import NOT_DONE_YET
+
 from zope.interface import implements
 
 FFMPEG2THEORA = 'ffmpeg2theora'
@@ -355,9 +356,9 @@ class Database(object):
             break
         return f
 
-    def files(self, site, user, since=None):
+    def files(self, site, user, volume, since=None):
         conn, c = self.conn()
-        c.execute('SELECT path from volume where site=? AND user=?', (site, user))
+        c.execute('SELECT path from volume where site=? AND user=? AND path=?', (site, user, volume))
         prefix = None
         for row in c:
             prefix = row[0]
@@ -366,7 +367,6 @@ class Database(object):
         #since 2 volumes can have the same file/folder, needs some check for that or other structure
         def get_files(files, key, sql, t=()):
             t = list(t) + [u"%s%%"%prefix]
-
             c.execute(sql, t)
             for row in c:
                 folder = row[0]
@@ -486,7 +486,7 @@ class Database(object):
                 dirpath = dirpath.decode('utf-8')
             if filenames:
                 prefix = dirpath[len(path)+1:]
-                for filename in filenames:
+                for filename in sorted(filenames):
                     if isinstance(filename, str):
                         filename = filename.decode('utf-8')
                     if not filename.startswith('._') and not filename in ('.DS_Store', ):
@@ -523,11 +523,16 @@ class Database(object):
 
     def volumes(self, site, user):
         conn, c = self.conn()
-        sql = 'SELECT name, path FROM volumes WHERE site=? AND user=? ORDER BY name';
+        sql = 'SELECT path FROM volume WHERE site=? AND user=? ORDER BY path';
-        c.execute(sql, [site])
+        c.execute(sql, [site, user])
         volumes = {}
         for row in c:
-            volumes[row[0]] = row[1]
+            path = row[0]
+            volumes[path] = {}
+            if os.path.exists(path):
+                volumes[path]['available'] = True
+            else:
+                volumes[path]['available'] = False
         return volumes
 
     def update_volumes(self, site, user):
@@ -543,19 +548,20 @@ class Database(object):
         paths = filter(not_subpath, paths)
         for path in paths:
             if os.path.exists(path): #might need another check to make sure its not empty
-                c.execute(u'UPDATE volume SET updating=1 WHERE path LIKE ?', ['%s%%'%path])
+                c.execute(u'UPDATE volume SET updating=1 WHERE site=? AND user=? AND path LIKE ?', [site, user, '%s%%'%path])
                 conn.commit()
                 self.spider(path)
                 updated = time.mktime(time.localtime())
-                c.execute(u'UPDATE volume SET updated=?, updating=0 WHERE path LIKE ?', (updated, '%s%%'%path))
+                c.execute(u'UPDATE volume SET updated=?, updating=0 WHERE site=? AND user=? AND  path LIKE ?',
+                           (updated, site, user, '%s%%'%path))
                 conn.commit()
 
     def remove_volume(self, site, user, name):
         conn, c = self.conn()
         c.execute('DELETE FROM volume WHERE site=? AND user=? AND name=?', [site, user, name])
+        conn.commit()
         #fixme, files could be still used by sub volumes
         #c.execute('DELETE FROM file WHERE path LIKE ?', ["%s%%"%path])
-        conn.commit()
 
 #web
 def json_response(request, data):
@@ -593,28 +599,27 @@ class OxControl(Resource):
 
     def render_POST(self, request):
         print request.path, request.args
-        if request.path == '/add_volume':
+        def required_args(*required_args):
             args = {}
-            for arg in ('site', 'user', 'path'):
+            for arg in required_args:
                 args[arg] = request.args.get(arg)[0]
+            return args
+
+        if request.path == '/add_volume':
+            args = required_args('site', 'user', 'path')
             self.db.add_volume(**args)
             response = {'status': 'ok'}
             return json_response(request, response)
 
         if request.path == '/remove_volume':
-            args = {}
+            args = required_args('site', 'user', 'path')
-            for arg in ('site', 'user', 'path'):
-                args[arg] = request.args.get(arg)[0]
             self.db.remove_volume(**args)
             response = {'status': 'ok'}
             return json_response(request, response)
 
         if request.path == '/volumes':
-            args = {}
+            args = required_args('site', 'user')
-            for arg in ['site']:
+            response = self.db.volumes(**args)
-                args[arg] = request.args.get(arg)[0]
-            response = {}
-            response['volumes'] = self.db.volumes(**args)
             return json_response(request, response)
 
         if request.path == '/files':
@@ -625,9 +630,7 @@ class OxControl(Resource):
                 since (optional) timestamp, return changes since
            files for user
             """
-            args = {}
+            args = required_args('site', 'user', 'volume')
-            for arg in ['site', 'user']:
-                args[arg] = request.args[arg][0]
             since = request.args.get("since", [None])[0]
             if since:
                 args['since'] = float(since)
@@ -639,14 +642,15 @@ class OxControl(Resource):
             """
             checks for new files in all known volumes for that user
             """
-            args = {}
+            args = required_args('site', 'user')
-            for arg in ['site', 'user']:
-                args[arg] = request.args[arg][0]
 
-            #update in another thread, this otherwise blocks web server
+            def render(request):
-            thread.start_new_thread(self.db.update_volumes, (args['site'], args['user']))
+                self.db.update_volumes(args['site'], args['user'])
-            response = {'status': 'ok'}
+                response = {'status': 'ok'}
-            return json_response(request, response)
+                request.write(json_response(request, response))
+                request.finish()
+            threads.deferToThread(render, request)
+            return NOT_DONE_YET
 
         if request.path == '/extract':
             """

OxFF/components/OxFF.js

@@ -13,6 +13,7 @@ Components.utils.import("resource://gre/modules/AddonManager.jsm");
 Components.utils.import("resource://gre/modules/XPCOMUtils.jsm");
 
 Components.utils.import("resource://ox/utils.jsm");
+Components.utils.import("resource://ox/oxff.jsm");
 
 var OxFFFactory =
 {
@@ -40,12 +41,12 @@ function OxFF() {
   }
   this.access();
 
-  if(!ox.get('username')) {
+  if(!oxff.get('username')) {
-    ox.set('username', ox.makeRandomString(8));
+    oxff.set('username', ox.makeRandomString(8));
-    ox.set('password', ox.makeRandomString(8));
+    oxff.set('password', ox.makeRandomString(8));
   }
-  this.username = ox.get('username');
+  this.username = oxff.get('username');
-  this.password = ox.get('password');
+  this.password = oxff.get('password');
 
   AddonManager.getAddonByID('firefogg@firefogg.org', function(addon) {
     if (addon.hasResource('bin')) {
@@ -137,7 +138,7 @@ OxFF.prototype = {
     if (typeof(request) == 'undefined') request = false;
     var _this = this;
 
-    var conn = ox.getDB();
+    var conn = oxff.getDB();
     var q = conn.createStatement("SELECT access FROM site WHERE site = :site");
     q.params.site = this._site;
     this._access = false;  
@@ -146,6 +147,7 @@ OxFF.prototype = {
         this._access = true;
       }
     }
+    q.finalize();
 
     if(request && !this._access) {
       var windowMediator = Cc["@mozilla.org/appshell/window-mediator;1"].getService(Ci.nsIWindowMediator);
@@ -217,8 +219,8 @@ OxFF.prototype = {
   extract: function(oshash, media, callback) {
     return this.api('extract', {'oshash': oshash, 'media': media}, callback);
   },
-  files: function(callback) {
+  files: function(volume, callback) {
-    return this.api('files', callback.callback);
+    return this.api('files', {'volume': volume}, callback.callback);
   },
   get: function(oshash, media, callback) {
     return this.api('get', {'oshash': oshash, 'media': media}, callback.callback);
@@ -234,11 +236,12 @@ OxFF.prototype = {
     var _this = this;
     this.api('stop', function() {
         _this._user = null;
+        _this._daemon = null;
     });
     return true;
   },
-  update: function() {
+  update: function(callback) {
-    this.api('update', function() {});
+    this.api('update', callback.callback);
     return true;
   },
 
@@ -259,47 +262,25 @@ OxFF.prototype = {
     return false;
   },
   volumes: function(callback) {
-    return this.api('volumes', callback);
+    return this.api('volumes', callback.callback);
   },
 
   //private functions
-  /*
-  _internal_update: function() {
-    var new_files = [];
-    var volumes = this.getVolumes(this._site);
-
-    for (i in volumes) {
-        var volume = volumes[i];
-        var files = ox.glob(volume);
-        //filter files, only include folders and files with the right extensions,
-        //where to store or get the list of valid extensions? would be nice if set by domain...
-        
-        for (f in files) {
-            var file = files[f];
-            //check if file is in known files
-            var info = this.getFormatInfo(file, false);
-            //strip volume path from info,
-            //+1 since volume paths are wihtout trailing folder seperator
-            info.path = file.substr(volume.length+1, file.length)
-            new_files.push(info);
-            //add to Files table
-        }
-    }
-    return JSON.stringify(new_files);
-  }, */
   permitAccess: function() {
     this._access = true;
-    var conn = ox.getDB();
+    var conn = oxff.getDB();
     var q = conn.createStatement("INSERT OR REPLACE INTO site values (:site, 1)");
     q.params.site = this._site;
     q.executeStep();
+    q.finalize();
   },
   denyAccess: function() {
     this._access = false;
-    var conn = ox.getDB();
+    var conn = oxff.getDB();
     var q = conn.createStatement("INSERT OR REPLACE INTO site values (:site, 0)");
     q.params.site = this._site;
     q.executeStep();
+    q.finalize();
   },
   startDaemon: function() {
     var _this = this;
@@ -321,7 +302,7 @@ OxFF.prototype = {
       } catch (e) {}
       _this._daemon = Cc["@mozilla.org/process/util;1"].createInstance(Ci.nsIProcess);
       _this._daemon.init(file);
-      var args = [ox.getDBFile().path, _this._ffmpeg2theora];
+      var args = [oxff.getDBFile().path, _this._ffmpeg2theora];
       _this._daemon.runw(false, args, args.length);
     }
     AddonManager.getAddonByID(this.extensionID, function(addon) {
@@ -333,22 +314,6 @@ OxFF.prototype = {
       }
     });
   },
-  getVolumes: function(site) {
-    var conn = ox.getDB();
-    var q = conn.createStatement("SELECT path FROM volume WHERE domain = :domain");
-    q.params.domain = site;
-    var volumes = [];
-    try {
-        while (q.executeStep()) {
-            var path = q.row.path;
-            volumes.push(path);
-        }
-    }
-    finally {
-        q.reset();  
-    }
-    return volumes;
-  },
   getFormatInfo: function(filename, callback) {
     var that = this;
     var get_json = function(data) {

OxFF/modules/oxff.jsm

@@ -0,0 +1,44 @@
+// -*- coding: utf-8 -*-
+// vi:si:et:sw=4:sts=4:ts=4
+
+let EXPORTED_SYMBOLS = [ "oxff" ];
+
+const Cc = Components.classes;
+const Ci = Components.interfaces;
+
+Components.utils.import("resource://ox/utils.jsm");
+
+let oxff = {
+    getDBFile: function() {
+        var file = Cc["@mozilla.org/file/directory_service;1"].getService(Ci.nsIProperties)  
+                     .get("ProfD", Ci.nsIFile);  
+        file.append("OxFF.sqlite");
+        return file;
+    },
+    getDB: function() {
+        var file = this.getDBFile();
+        var storageService = Cc["@mozilla.org/storage/service;1"].getService(Ci.mozIStorageService);
+        var conn = storageService.openDatabase(file);
+        conn.executeSimpleSQL("CREATE TABLE IF NOT EXISTS site (site varchar(1024) unique, access INT)");
+        conn.executeSimpleSQL("CREATE TABLE IF NOT EXISTS setting (key varchar(1024) unique, value text)");
+        return conn;
+    },
+    get: function(key, value) {
+        var conn = this.getDB();
+        var q = conn.createStatement("SELECT value FROM setting WHERE key = :key");
+        q.params.key = key;
+        if (q.executeStep())
+          value = q.row.value;
+        q.finalize();
+        return value;
+    },
+    set: function(key, value) {
+        var conn = this.getDB();
+        var q = conn.createStatement("INSERT OR REPLACE INTO setting values (:key, :value)");
+        q.params.key = key;
+        q.params.value = value;
+        q.executeStep();
+        q.finalize();
+    },
+};
+

OxFF/modules/utils.jsm

@@ -7,35 +7,6 @@ const Cc = Components.classes;
 const Ci = Components.interfaces;
 
 let ox = {
-    getDBFile: function() {
-        var file = Cc["@mozilla.org/file/directory_service;1"].getService(Ci.nsIProperties)  
-                     .get("ProfD", Ci.nsIFile);  
-        file.append("OxFF.sqlite");
-        return file;
-    },
-    getDB: function() {
-        var file = this.getDBFile();
-        var storageService = Cc["@mozilla.org/storage/service;1"].getService(Ci.mozIStorageService);
-        var conn = storageService.openDatabase(file);
-        conn.executeSimpleSQL("CREATE TABLE IF NOT EXISTS site (site varchar(1024) unique, access INT)");
-        conn.executeSimpleSQL("CREATE TABLE IF NOT EXISTS setting (key varchar(1024) unique, value text)");
-        return conn;
-    },
-    get: function(key, defaultValue) {
-        var conn = this.getDB();
-        var q = conn.createStatement("SELECT value FROM setting WHERE key = :key");
-        q.params.key = key;
-        if (q.executeStep())
-          return q.row.value;
-        return defaultValue;
-    },
-    set: function(key, value) {
-        var conn = this.getDB();
-        var q = conn.createStatement("INSERT OR REPLACE INTO setting values (:key, :value)");
-        q.params.key = key;
-        q.params.value = value;
-        q.executeStep();
-    },
     setTimeout: function(callback, timeout) {
         var timer = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer);
         timer.initWithCallback(callback, timeout, Ci.nsITimer.TYPE_ONE_SHOT);

src/nsIOxFF.idl

@@ -20,9 +20,9 @@ interface nsIOxFF : nsISupports
   string import();
   float progress(in AString oshash);
   boolean access([optional] in boolean request);
-  boolean update();
+  boolean update(in oxICallback callback);
-  string volumes();
+  string volumes(in oxICallback callback);
-  boolean files(in oxICallback callback);
+  boolean files(in AString volume, in oxICallback callback);
   boolean get(in AString oshash, in AString media, in oxICallback callback);
   boolean extract(in AString oshash, in AString media, in oxICallback callback);
   boolean logout();

test/import.html

@@ -1,10 +1,59 @@
+<!doctype html>
+<html>
+<head>
+<script src="http://oxjs.org/js/jquery-1.4.2.min.js"></script>
 <script>
+
 var ox = new OxFF();
 ox.access(true);
-ox.get('b2c8f0aa3a447d09', 'stills', function(result) { console.log(result);});
+ox.login('username');
-ox.login('j');
 
+//ox.get('b2c8f0aa3a447d09', 'stills', function(result) { console.log(result);});
 //ox.files(function(result) { console.log(result);});
 
+function update() {
+    var updating = true;
+    ox.update(function(result) {
+        console.log('updating done', result);
+        updating = false });
+    var getFiles = function() {
+        ox.volumes(function(result) {
+            var volumes = JSON.parse(result);
+            for(volume in volumes) {
+                (function(volume) {
+                    var volumeId = volume.replace(/[ \/]/g, '_');
+                    var $volume = $('#'+volumeId);
+                    if($volume.length==0)
+                        $volume = $('<div>').attr('id', volumeId);
+                    $volume.html('<h1>'+volume+'</h1>');
+                    $('#files').append($volume);
+                    ox.files(volume, function(result) {
+                        var files = JSON.parse(result);
+                        $.each(files, function(folder, f) {
+                            $file = $('<div>').html(folder);
+                            $volume.append($file);
+                        });
+                    });
+                }(volume));
+            }
+            if(updating) {
+                setTimeout(getFiles, 2000);
+            }
+        });
+    }
+    getFiles();
+}
 
+function addVolume() {
+    console.log(ox.addVolume())
+}
 </script>
+</head>
+<body>
+    <input type="button" onClick="addVolume()" value="Add Volume">
+    <input type="button" onClick="update()" value="Update">
+    <div id="files">
+        
+    </div>
+</body>
+</html>