diff --git a/OxFF/bin/oxd.py b/OxFF/bin/oxd.py index 80b53d9..cc3d1ad 100755 --- a/OxFF/bin/oxd.py +++ b/OxFF/bin/oxd.py @@ -16,16 +16,17 @@ import sys import shutil import tempfile import time -import thread from threading import Thread from twisted.cred.portal import IRealm, Portal from twisted.cred.checkers import InMemoryUsernamePasswordDatabaseDontUse -from twisted.internet import task, reactor +from twisted.internet import task, reactor, threads from twisted.web import server from twisted.web.guard import HTTPAuthSessionWrapper, DigestCredentialFactory from twisted.web.resource import Resource, IResource from twisted.web.static import File +from twisted.web.server import NOT_DONE_YET + from zope.interface import implements FFMPEG2THEORA = 'ffmpeg2theora' @@ -355,9 +356,9 @@ class Database(object): break return f - def files(self, site, user, since=None): + def files(self, site, user, volume, since=None): conn, c = self.conn() - c.execute('SELECT path from volume where site=? AND user=?', (site, user)) + c.execute('SELECT path from volume where site=? AND user=? AND path=?', (site, user, volume)) prefix = None for row in c: prefix = row[0] @@ -366,7 +367,6 @@ class Database(object): #since 2 volumes can have the same file/folder, needs some check for that or other structure def get_files(files, key, sql, t=()): t = list(t) + [u"%s%%"%prefix] - c.execute(sql, t) for row in c: folder = row[0] @@ -486,7 +486,7 @@ class Database(object): dirpath = dirpath.decode('utf-8') if filenames: prefix = dirpath[len(path)+1:] - for filename in filenames: + for filename in sorted(filenames): if isinstance(filename, str): filename = filename.decode('utf-8') if not filename.startswith('._') and not filename in ('.DS_Store', ): @@ -523,11 +523,16 @@ class Database(object): def volumes(self, site, user): conn, c = self.conn() - sql = 'SELECT name, path FROM volumes WHERE site=? AND user=? ORDER BY name'; - c.execute(sql, [site]) + sql = 'SELECT path FROM volume WHERE site=? AND user=? ORDER BY path'; + c.execute(sql, [site, user]) volumes = {} for row in c: - volumes[row[0]] = row[1] + path = row[0] + volumes[path] = {} + if os.path.exists(path): + volumes[path]['available'] = True + else: + volumes[path]['available'] = False return volumes def update_volumes(self, site, user): @@ -543,19 +548,20 @@ class Database(object): paths = filter(not_subpath, paths) for path in paths: if os.path.exists(path): #might need another check to make sure its not empty - c.execute(u'UPDATE volume SET updating=1 WHERE path LIKE ?', ['%s%%'%path]) + c.execute(u'UPDATE volume SET updating=1 WHERE site=? AND user=? AND path LIKE ?', [site, user, '%s%%'%path]) conn.commit() self.spider(path) updated = time.mktime(time.localtime()) - c.execute(u'UPDATE volume SET updated=?, updating=0 WHERE path LIKE ?', (updated, '%s%%'%path)) + c.execute(u'UPDATE volume SET updated=?, updating=0 WHERE site=? AND user=? AND path LIKE ?', + (updated, site, user, '%s%%'%path)) conn.commit() def remove_volume(self, site, user, name): conn, c = self.conn() c.execute('DELETE FROM volume WHERE site=? AND user=? AND name=?', [site, user, name]) + conn.commit() #fixme, files could be still used by sub volumes #c.execute('DELETE FROM file WHERE path LIKE ?', ["%s%%"%path]) - conn.commit() #web def json_response(request, data): @@ -593,28 +599,27 @@ class OxControl(Resource): def render_POST(self, request): print request.path, request.args - if request.path == '/add_volume': + def required_args(*required_args): args = {} - for arg in ('site', 'user', 'path'): + for arg in required_args: args[arg] = request.args.get(arg)[0] + return args + + if request.path == '/add_volume': + args = required_args('site', 'user', 'path') self.db.add_volume(**args) response = {'status': 'ok'} return json_response(request, response) if request.path == '/remove_volume': - args = {} - for arg in ('site', 'user', 'path'): - args[arg] = request.args.get(arg)[0] + args = required_args('site', 'user', 'path') self.db.remove_volume(**args) response = {'status': 'ok'} return json_response(request, response) if request.path == '/volumes': - args = {} - for arg in ['site']: - args[arg] = request.args.get(arg)[0] - response = {} - response['volumes'] = self.db.volumes(**args) + args = required_args('site', 'user') + response = self.db.volumes(**args) return json_response(request, response) if request.path == '/files': @@ -625,9 +630,7 @@ class OxControl(Resource): since (optional) timestamp, return changes since files for user """ - args = {} - for arg in ['site', 'user']: - args[arg] = request.args[arg][0] + args = required_args('site', 'user', 'volume') since = request.args.get("since", [None])[0] if since: args['since'] = float(since) @@ -639,14 +642,15 @@ class OxControl(Resource): """ checks for new files in all known volumes for that user """ - args = {} - for arg in ['site', 'user']: - args[arg] = request.args[arg][0] + args = required_args('site', 'user') - #update in another thread, this otherwise blocks web server - thread.start_new_thread(self.db.update_volumes, (args['site'], args['user'])) - response = {'status': 'ok'} - return json_response(request, response) + def render(request): + self.db.update_volumes(args['site'], args['user']) + response = {'status': 'ok'} + request.write(json_response(request, response)) + request.finish() + threads.deferToThread(render, request) + return NOT_DONE_YET if request.path == '/extract': """ diff --git a/OxFF/components/OxFF.js b/OxFF/components/OxFF.js index 233ddac..496f5fc 100644 --- a/OxFF/components/OxFF.js +++ b/OxFF/components/OxFF.js @@ -13,6 +13,7 @@ Components.utils.import("resource://gre/modules/AddonManager.jsm"); Components.utils.import("resource://gre/modules/XPCOMUtils.jsm"); Components.utils.import("resource://ox/utils.jsm"); +Components.utils.import("resource://ox/oxff.jsm"); var OxFFFactory = { @@ -40,12 +41,12 @@ function OxFF() { } this.access(); - if(!ox.get('username')) { - ox.set('username', ox.makeRandomString(8)); - ox.set('password', ox.makeRandomString(8)); + if(!oxff.get('username')) { + oxff.set('username', ox.makeRandomString(8)); + oxff.set('password', ox.makeRandomString(8)); } - this.username = ox.get('username'); - this.password = ox.get('password'); + this.username = oxff.get('username'); + this.password = oxff.get('password'); AddonManager.getAddonByID('firefogg@firefogg.org', function(addon) { if (addon.hasResource('bin')) { @@ -137,7 +138,7 @@ OxFF.prototype = { if (typeof(request) == 'undefined') request = false; var _this = this; - var conn = ox.getDB(); + var conn = oxff.getDB(); var q = conn.createStatement("SELECT access FROM site WHERE site = :site"); q.params.site = this._site; this._access = false; @@ -146,6 +147,7 @@ OxFF.prototype = { this._access = true; } } + q.finalize(); if(request && !this._access) { var windowMediator = Cc["@mozilla.org/appshell/window-mediator;1"].getService(Ci.nsIWindowMediator); @@ -217,8 +219,8 @@ OxFF.prototype = { extract: function(oshash, media, callback) { return this.api('extract', {'oshash': oshash, 'media': media}, callback); }, - files: function(callback) { - return this.api('files', callback.callback); + files: function(volume, callback) { + return this.api('files', {'volume': volume}, callback.callback); }, get: function(oshash, media, callback) { return this.api('get', {'oshash': oshash, 'media': media}, callback.callback); @@ -234,11 +236,12 @@ OxFF.prototype = { var _this = this; this.api('stop', function() { _this._user = null; + _this._daemon = null; }); return true; }, - update: function() { - this.api('update', function() {}); + update: function(callback) { + this.api('update', callback.callback); return true; }, @@ -259,47 +262,25 @@ OxFF.prototype = { return false; }, volumes: function(callback) { - return this.api('volumes', callback); + return this.api('volumes', callback.callback); }, //private functions - /* - _internal_update: function() { - var new_files = []; - var volumes = this.getVolumes(this._site); - - for (i in volumes) { - var volume = volumes[i]; - var files = ox.glob(volume); - //filter files, only include folders and files with the right extensions, - //where to store or get the list of valid extensions? would be nice if set by domain... - - for (f in files) { - var file = files[f]; - //check if file is in known files - var info = this.getFormatInfo(file, false); - //strip volume path from info, - //+1 since volume paths are wihtout trailing folder seperator - info.path = file.substr(volume.length+1, file.length) - new_files.push(info); - //add to Files table - } - } - return JSON.stringify(new_files); - }, */ permitAccess: function() { this._access = true; - var conn = ox.getDB(); + var conn = oxff.getDB(); var q = conn.createStatement("INSERT OR REPLACE INTO site values (:site, 1)"); q.params.site = this._site; q.executeStep(); + q.finalize(); }, denyAccess: function() { this._access = false; - var conn = ox.getDB(); + var conn = oxff.getDB(); var q = conn.createStatement("INSERT OR REPLACE INTO site values (:site, 0)"); q.params.site = this._site; q.executeStep(); + q.finalize(); }, startDaemon: function() { var _this = this; @@ -321,7 +302,7 @@ OxFF.prototype = { } catch (e) {} _this._daemon = Cc["@mozilla.org/process/util;1"].createInstance(Ci.nsIProcess); _this._daemon.init(file); - var args = [ox.getDBFile().path, _this._ffmpeg2theora]; + var args = [oxff.getDBFile().path, _this._ffmpeg2theora]; _this._daemon.runw(false, args, args.length); } AddonManager.getAddonByID(this.extensionID, function(addon) { @@ -333,22 +314,6 @@ OxFF.prototype = { } }); }, - getVolumes: function(site) { - var conn = ox.getDB(); - var q = conn.createStatement("SELECT path FROM volume WHERE domain = :domain"); - q.params.domain = site; - var volumes = []; - try { - while (q.executeStep()) { - var path = q.row.path; - volumes.push(path); - } - } - finally { - q.reset(); - } - return volumes; - }, getFormatInfo: function(filename, callback) { var that = this; var get_json = function(data) { diff --git a/OxFF/components/nsIOxFF.xpt b/OxFF/components/nsIOxFF.xpt index fdb5276..6e995e9 100644 Binary files a/OxFF/components/nsIOxFF.xpt and b/OxFF/components/nsIOxFF.xpt differ diff --git a/OxFF/modules/oxff.jsm b/OxFF/modules/oxff.jsm new file mode 100644 index 0000000..18adc37 --- /dev/null +++ b/OxFF/modules/oxff.jsm @@ -0,0 +1,44 @@ +// -*- coding: utf-8 -*- +// vi:si:et:sw=4:sts=4:ts=4 + +let EXPORTED_SYMBOLS = [ "oxff" ]; + +const Cc = Components.classes; +const Ci = Components.interfaces; + +Components.utils.import("resource://ox/utils.jsm"); + +let oxff = { + getDBFile: function() { + var file = Cc["@mozilla.org/file/directory_service;1"].getService(Ci.nsIProperties) + .get("ProfD", Ci.nsIFile); + file.append("OxFF.sqlite"); + return file; + }, + getDB: function() { + var file = this.getDBFile(); + var storageService = Cc["@mozilla.org/storage/service;1"].getService(Ci.mozIStorageService); + var conn = storageService.openDatabase(file); + conn.executeSimpleSQL("CREATE TABLE IF NOT EXISTS site (site varchar(1024) unique, access INT)"); + conn.executeSimpleSQL("CREATE TABLE IF NOT EXISTS setting (key varchar(1024) unique, value text)"); + return conn; + }, + get: function(key, value) { + var conn = this.getDB(); + var q = conn.createStatement("SELECT value FROM setting WHERE key = :key"); + q.params.key = key; + if (q.executeStep()) + value = q.row.value; + q.finalize(); + return value; + }, + set: function(key, value) { + var conn = this.getDB(); + var q = conn.createStatement("INSERT OR REPLACE INTO setting values (:key, :value)"); + q.params.key = key; + q.params.value = value; + q.executeStep(); + q.finalize(); + }, +}; + diff --git a/OxFF/modules/utils.jsm b/OxFF/modules/utils.jsm index b77fbb5..a07e846 100644 --- a/OxFF/modules/utils.jsm +++ b/OxFF/modules/utils.jsm @@ -7,35 +7,6 @@ const Cc = Components.classes; const Ci = Components.interfaces; let ox = { - getDBFile: function() { - var file = Cc["@mozilla.org/file/directory_service;1"].getService(Ci.nsIProperties) - .get("ProfD", Ci.nsIFile); - file.append("OxFF.sqlite"); - return file; - }, - getDB: function() { - var file = this.getDBFile(); - var storageService = Cc["@mozilla.org/storage/service;1"].getService(Ci.mozIStorageService); - var conn = storageService.openDatabase(file); - conn.executeSimpleSQL("CREATE TABLE IF NOT EXISTS site (site varchar(1024) unique, access INT)"); - conn.executeSimpleSQL("CREATE TABLE IF NOT EXISTS setting (key varchar(1024) unique, value text)"); - return conn; - }, - get: function(key, defaultValue) { - var conn = this.getDB(); - var q = conn.createStatement("SELECT value FROM setting WHERE key = :key"); - q.params.key = key; - if (q.executeStep()) - return q.row.value; - return defaultValue; - }, - set: function(key, value) { - var conn = this.getDB(); - var q = conn.createStatement("INSERT OR REPLACE INTO setting values (:key, :value)"); - q.params.key = key; - q.params.value = value; - q.executeStep(); - }, setTimeout: function(callback, timeout) { var timer = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer); timer.initWithCallback(callback, timeout, Ci.nsITimer.TYPE_ONE_SHOT); diff --git a/src/nsIOxFF.idl b/src/nsIOxFF.idl index 59933dd..f612702 100644 --- a/src/nsIOxFF.idl +++ b/src/nsIOxFF.idl @@ -20,9 +20,9 @@ interface nsIOxFF : nsISupports string import(); float progress(in AString oshash); boolean access([optional] in boolean request); - boolean update(); - string volumes(); - boolean files(in oxICallback callback); + boolean update(in oxICallback callback); + string volumes(in oxICallback callback); + boolean files(in AString volume, in oxICallback callback); boolean get(in AString oshash, in AString media, in oxICallback callback); boolean extract(in AString oshash, in AString media, in oxICallback callback); boolean logout(); diff --git a/test/import.html b/test/import.html index 133962a..83199e5 100644 --- a/test/import.html +++ b/test/import.html @@ -1,10 +1,59 @@ + + + + + + + + +
+ +
+ +