update blocking in oxd + callback in oxff, split modules in ox.jsm and oxff.jsm
This commit is contained in:
j
parent
9cf7c84558
commit
1be85a89d8
7 changed files with 153 additions and 120 deletions
|
|
@ -16,16 +16,17 @@ import sys
|
|||
import shutil
|
||||
import tempfile
|
||||
import time
|
||||
import thread
|
||||
from threading import Thread
|
||||
|
||||
from twisted.cred.portal import IRealm, Portal
|
||||
from twisted.cred.checkers import InMemoryUsernamePasswordDatabaseDontUse
|
||||
from twisted.internet import task, reactor
|
||||
from twisted.internet import task, reactor, threads
|
||||
from twisted.web import server
|
||||
from twisted.web.guard import HTTPAuthSessionWrapper, DigestCredentialFactory
|
||||
from twisted.web.resource import Resource, IResource
|
||||
from twisted.web.static import File
|
||||
from twisted.web.server import NOT_DONE_YET
|
||||
|
||||
from zope.interface import implements
|
||||
|
||||
FFMPEG2THEORA = 'ffmpeg2theora'
|
||||
|
|
@ -355,9 +356,9 @@ class Database(object):
|
|||
break
|
||||
return f
|
||||
|
||||
def files(self, site, user, since=None):
|
||||
def files(self, site, user, volume, since=None):
|
||||
conn, c = self.conn()
|
||||
c.execute('SELECT path from volume where site=? AND user=?', (site, user))
|
||||
c.execute('SELECT path from volume where site=? AND user=? AND path=?', (site, user, volume))
|
||||
prefix = None
|
||||
for row in c:
|
||||
prefix = row[0]
|
||||
|
|
@ -366,7 +367,6 @@ class Database(object):
|
|||
#since 2 volumes can have the same file/folder, needs some check for that or other structure
|
||||
def get_files(files, key, sql, t=()):
|
||||
t = list(t) + [u"%s%%"%prefix]
|
||||
|
||||
c.execute(sql, t)
|
||||
for row in c:
|
||||
folder = row[0]
|
||||
|
|
@ -486,7 +486,7 @@ class Database(object):
|
|||
dirpath = dirpath.decode('utf-8')
|
||||
if filenames:
|
||||
prefix = dirpath[len(path)+1:]
|
||||
for filename in filenames:
|
||||
for filename in sorted(filenames):
|
||||
if isinstance(filename, str):
|
||||
filename = filename.decode('utf-8')
|
||||
if not filename.startswith('._') and not filename in ('.DS_Store', ):
|
||||
|
|
@ -523,11 +523,16 @@ class Database(object):
|
|||
|
||||
def volumes(self, site, user):
|
||||
conn, c = self.conn()
|
||||
sql = 'SELECT name, path FROM volumes WHERE site=? AND user=? ORDER BY name';
|
||||
c.execute(sql, [site])
|
||||
sql = 'SELECT path FROM volume WHERE site=? AND user=? ORDER BY path';
|
||||
c.execute(sql, [site, user])
|
||||
volumes = {}
|
||||
for row in c:
|
||||
volumes[row[0]] = row[1]
|
||||
path = row[0]
|
||||
volumes[path] = {}
|
||||
if os.path.exists(path):
|
||||
volumes[path]['available'] = True
|
||||
else:
|
||||
volumes[path]['available'] = False
|
||||
return volumes
|
||||
|
||||
def update_volumes(self, site, user):
|
||||
|
|
@ -543,19 +548,20 @@ class Database(object):
|
|||
paths = filter(not_subpath, paths)
|
||||
for path in paths:
|
||||
if os.path.exists(path): #might need another check to make sure its not empty
|
||||
c.execute(u'UPDATE volume SET updating=1 WHERE path LIKE ?', ['%s%%'%path])
|
||||
c.execute(u'UPDATE volume SET updating=1 WHERE site=? AND user=? AND path LIKE ?', [site, user, '%s%%'%path])
|
||||
conn.commit()
|
||||
self.spider(path)
|
||||
updated = time.mktime(time.localtime())
|
||||
c.execute(u'UPDATE volume SET updated=?, updating=0 WHERE path LIKE ?', (updated, '%s%%'%path))
|
||||
c.execute(u'UPDATE volume SET updated=?, updating=0 WHERE site=? AND user=? AND path LIKE ?',
|
||||
(updated, site, user, '%s%%'%path))
|
||||
conn.commit()
|
||||
|
||||
def remove_volume(self, site, user, name):
|
||||
conn, c = self.conn()
|
||||
c.execute('DELETE FROM volume WHERE site=? AND user=? AND name=?', [site, user, name])
|
||||
conn.commit()
|
||||
#fixme, files could be still used by sub volumes
|
||||
#c.execute('DELETE FROM file WHERE path LIKE ?', ["%s%%"%path])
|
||||
conn.commit()
|
||||
|
||||
#web
|
||||
def json_response(request, data):
|
||||
|
|
@ -593,28 +599,27 @@ class OxControl(Resource):
|
|||
|
||||
def render_POST(self, request):
|
||||
print request.path, request.args
|
||||
if request.path == '/add_volume':
|
||||
def required_args(*required_args):
|
||||
args = {}
|
||||
for arg in ('site', 'user', 'path'):
|
||||
for arg in required_args:
|
||||
args[arg] = request.args.get(arg)[0]
|
||||
return args
|
||||
|
||||
if request.path == '/add_volume':
|
||||
args = required_args('site', 'user', 'path')
|
||||
self.db.add_volume(**args)
|
||||
response = {'status': 'ok'}
|
||||
return json_response(request, response)
|
||||
|
||||
if request.path == '/remove_volume':
|
||||
args = {}
|
||||
for arg in ('site', 'user', 'path'):
|
||||
args[arg] = request.args.get(arg)[0]
|
||||
args = required_args('site', 'user', 'path')
|
||||
self.db.remove_volume(**args)
|
||||
response = {'status': 'ok'}
|
||||
return json_response(request, response)
|
||||
|
||||
if request.path == '/volumes':
|
||||
args = {}
|
||||
for arg in ['site']:
|
||||
args[arg] = request.args.get(arg)[0]
|
||||
response = {}
|
||||
response['volumes'] = self.db.volumes(**args)
|
||||
args = required_args('site', 'user')
|
||||
response = self.db.volumes(**args)
|
||||
return json_response(request, response)
|
||||
|
||||
if request.path == '/files':
|
||||
|
|
@ -625,9 +630,7 @@ class OxControl(Resource):
|
|||
since (optional) timestamp, return changes since
|
||||
files for user
|
||||
"""
|
||||
args = {}
|
||||
for arg in ['site', 'user']:
|
||||
args[arg] = request.args[arg][0]
|
||||
args = required_args('site', 'user', 'volume')
|
||||
since = request.args.get("since", [None])[0]
|
||||
if since:
|
||||
args['since'] = float(since)
|
||||
|
|
@ -639,14 +642,15 @@ class OxControl(Resource):
|
|||
"""
|
||||
checks for new files in all known volumes for that user
|
||||
"""
|
||||
args = {}
|
||||
for arg in ['site', 'user']:
|
||||
args[arg] = request.args[arg][0]
|
||||
args = required_args('site', 'user')
|
||||
|
||||
#update in another thread, this otherwise blocks web server
|
||||
thread.start_new_thread(self.db.update_volumes, (args['site'], args['user']))
|
||||
response = {'status': 'ok'}
|
||||
return json_response(request, response)
|
||||
def render(request):
|
||||
self.db.update_volumes(args['site'], args['user'])
|
||||
response = {'status': 'ok'}
|
||||
request.write(json_response(request, response))
|
||||
request.finish()
|
||||
threads.deferToThread(render, request)
|
||||
return NOT_DONE_YET
|
||||
|
||||
if request.path == '/extract':
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ Components.utils.import("resource://gre/modules/AddonManager.jsm");
|
|||
Components.utils.import("resource://gre/modules/XPCOMUtils.jsm");
|
||||
|
||||
Components.utils.import("resource://ox/utils.jsm");
|
||||
Components.utils.import("resource://ox/oxff.jsm");
|
||||
|
||||
var OxFFFactory =
|
||||
{
|
||||
|
|
@ -40,12 +41,12 @@ function OxFF() {
|
|||
}
|
||||
this.access();
|
||||
|
||||
if(!ox.get('username')) {
|
||||
ox.set('username', ox.makeRandomString(8));
|
||||
ox.set('password', ox.makeRandomString(8));
|
||||
if(!oxff.get('username')) {
|
||||
oxff.set('username', ox.makeRandomString(8));
|
||||
oxff.set('password', ox.makeRandomString(8));
|
||||
}
|
||||
this.username = ox.get('username');
|
||||
this.password = ox.get('password');
|
||||
this.username = oxff.get('username');
|
||||
this.password = oxff.get('password');
|
||||
|
||||
AddonManager.getAddonByID('firefogg@firefogg.org', function(addon) {
|
||||
if (addon.hasResource('bin')) {
|
||||
|
|
@ -137,7 +138,7 @@ OxFF.prototype = {
|
|||
if (typeof(request) == 'undefined') request = false;
|
||||
var _this = this;
|
||||
|
||||
var conn = ox.getDB();
|
||||
var conn = oxff.getDB();
|
||||
var q = conn.createStatement("SELECT access FROM site WHERE site = :site");
|
||||
q.params.site = this._site;
|
||||
this._access = false;
|
||||
|
|
@ -146,6 +147,7 @@ OxFF.prototype = {
|
|||
this._access = true;
|
||||
}
|
||||
}
|
||||
q.finalize();
|
||||
|
||||
if(request && !this._access) {
|
||||
var windowMediator = Cc["@mozilla.org/appshell/window-mediator;1"].getService(Ci.nsIWindowMediator);
|
||||
|
|
@ -217,8 +219,8 @@ OxFF.prototype = {
|
|||
extract: function(oshash, media, callback) {
|
||||
return this.api('extract', {'oshash': oshash, 'media': media}, callback);
|
||||
},
|
||||
files: function(callback) {
|
||||
return this.api('files', callback.callback);
|
||||
files: function(volume, callback) {
|
||||
return this.api('files', {'volume': volume}, callback.callback);
|
||||
},
|
||||
get: function(oshash, media, callback) {
|
||||
return this.api('get', {'oshash': oshash, 'media': media}, callback.callback);
|
||||
|
|
@ -234,11 +236,12 @@ OxFF.prototype = {
|
|||
var _this = this;
|
||||
this.api('stop', function() {
|
||||
_this._user = null;
|
||||
_this._daemon = null;
|
||||
});
|
||||
return true;
|
||||
},
|
||||
update: function() {
|
||||
this.api('update', function() {});
|
||||
update: function(callback) {
|
||||
this.api('update', callback.callback);
|
||||
return true;
|
||||
},
|
||||
|
||||
|
|
@ -259,47 +262,25 @@ OxFF.prototype = {
|
|||
return false;
|
||||
},
|
||||
volumes: function(callback) {
|
||||
return this.api('volumes', callback);
|
||||
return this.api('volumes', callback.callback);
|
||||
},
|
||||
|
||||
//private functions
|
||||
/*
|
||||
_internal_update: function() {
|
||||
var new_files = [];
|
||||
var volumes = this.getVolumes(this._site);
|
||||
|
||||
for (i in volumes) {
|
||||
var volume = volumes[i];
|
||||
var files = ox.glob(volume);
|
||||
//filter files, only include folders and files with the right extensions,
|
||||
//where to store or get the list of valid extensions? would be nice if set by domain...
|
||||
|
||||
for (f in files) {
|
||||
var file = files[f];
|
||||
//check if file is in known files
|
||||
var info = this.getFormatInfo(file, false);
|
||||
//strip volume path from info,
|
||||
//+1 since volume paths are wihtout trailing folder seperator
|
||||
info.path = file.substr(volume.length+1, file.length)
|
||||
new_files.push(info);
|
||||
//add to Files table
|
||||
}
|
||||
}
|
||||
return JSON.stringify(new_files);
|
||||
}, */
|
||||
permitAccess: function() {
|
||||
this._access = true;
|
||||
var conn = ox.getDB();
|
||||
var conn = oxff.getDB();
|
||||
var q = conn.createStatement("INSERT OR REPLACE INTO site values (:site, 1)");
|
||||
q.params.site = this._site;
|
||||
q.executeStep();
|
||||
q.finalize();
|
||||
},
|
||||
denyAccess: function() {
|
||||
this._access = false;
|
||||
var conn = ox.getDB();
|
||||
var conn = oxff.getDB();
|
||||
var q = conn.createStatement("INSERT OR REPLACE INTO site values (:site, 0)");
|
||||
q.params.site = this._site;
|
||||
q.executeStep();
|
||||
q.finalize();
|
||||
},
|
||||
startDaemon: function() {
|
||||
var _this = this;
|
||||
|
|
@ -321,7 +302,7 @@ OxFF.prototype = {
|
|||
} catch (e) {}
|
||||
_this._daemon = Cc["@mozilla.org/process/util;1"].createInstance(Ci.nsIProcess);
|
||||
_this._daemon.init(file);
|
||||
var args = [ox.getDBFile().path, _this._ffmpeg2theora];
|
||||
var args = [oxff.getDBFile().path, _this._ffmpeg2theora];
|
||||
_this._daemon.runw(false, args, args.length);
|
||||
}
|
||||
AddonManager.getAddonByID(this.extensionID, function(addon) {
|
||||
|
|
@ -333,22 +314,6 @@ OxFF.prototype = {
|
|||
}
|
||||
});
|
||||
},
|
||||
getVolumes: function(site) {
|
||||
var conn = ox.getDB();
|
||||
var q = conn.createStatement("SELECT path FROM volume WHERE domain = :domain");
|
||||
q.params.domain = site;
|
||||
var volumes = [];
|
||||
try {
|
||||
while (q.executeStep()) {
|
||||
var path = q.row.path;
|
||||
volumes.push(path);
|
||||
}
|
||||
}
|
||||
finally {
|
||||
q.reset();
|
||||
}
|
||||
return volumes;
|
||||
},
|
||||
getFormatInfo: function(filename, callback) {
|
||||
var that = this;
|
||||
var get_json = function(data) {
|
||||
|
|
|
|||
Binary file not shown.
44
OxFF/modules/oxff.jsm
Normal file
44
OxFF/modules/oxff.jsm
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
// -*- coding: utf-8 -*-
|
||||
// vi:si:et:sw=4:sts=4:ts=4
|
||||
|
||||
let EXPORTED_SYMBOLS = [ "oxff" ];
|
||||
|
||||
const Cc = Components.classes;
|
||||
const Ci = Components.interfaces;
|
||||
|
||||
Components.utils.import("resource://ox/utils.jsm");
|
||||
|
||||
let oxff = {
|
||||
getDBFile: function() {
|
||||
var file = Cc["@mozilla.org/file/directory_service;1"].getService(Ci.nsIProperties)
|
||||
.get("ProfD", Ci.nsIFile);
|
||||
file.append("OxFF.sqlite");
|
||||
return file;
|
||||
},
|
||||
getDB: function() {
|
||||
var file = this.getDBFile();
|
||||
var storageService = Cc["@mozilla.org/storage/service;1"].getService(Ci.mozIStorageService);
|
||||
var conn = storageService.openDatabase(file);
|
||||
conn.executeSimpleSQL("CREATE TABLE IF NOT EXISTS site (site varchar(1024) unique, access INT)");
|
||||
conn.executeSimpleSQL("CREATE TABLE IF NOT EXISTS setting (key varchar(1024) unique, value text)");
|
||||
return conn;
|
||||
},
|
||||
get: function(key, value) {
|
||||
var conn = this.getDB();
|
||||
var q = conn.createStatement("SELECT value FROM setting WHERE key = :key");
|
||||
q.params.key = key;
|
||||
if (q.executeStep())
|
||||
value = q.row.value;
|
||||
q.finalize();
|
||||
return value;
|
||||
},
|
||||
set: function(key, value) {
|
||||
var conn = this.getDB();
|
||||
var q = conn.createStatement("INSERT OR REPLACE INTO setting values (:key, :value)");
|
||||
q.params.key = key;
|
||||
q.params.value = value;
|
||||
q.executeStep();
|
||||
q.finalize();
|
||||
},
|
||||
};
|
||||
|
||||
|
|
@ -7,35 +7,6 @@ const Cc = Components.classes;
|
|||
const Ci = Components.interfaces;
|
||||
|
||||
let ox = {
|
||||
getDBFile: function() {
|
||||
var file = Cc["@mozilla.org/file/directory_service;1"].getService(Ci.nsIProperties)
|
||||
.get("ProfD", Ci.nsIFile);
|
||||
file.append("OxFF.sqlite");
|
||||
return file;
|
||||
},
|
||||
getDB: function() {
|
||||
var file = this.getDBFile();
|
||||
var storageService = Cc["@mozilla.org/storage/service;1"].getService(Ci.mozIStorageService);
|
||||
var conn = storageService.openDatabase(file);
|
||||
conn.executeSimpleSQL("CREATE TABLE IF NOT EXISTS site (site varchar(1024) unique, access INT)");
|
||||
conn.executeSimpleSQL("CREATE TABLE IF NOT EXISTS setting (key varchar(1024) unique, value text)");
|
||||
return conn;
|
||||
},
|
||||
get: function(key, defaultValue) {
|
||||
var conn = this.getDB();
|
||||
var q = conn.createStatement("SELECT value FROM setting WHERE key = :key");
|
||||
q.params.key = key;
|
||||
if (q.executeStep())
|
||||
return q.row.value;
|
||||
return defaultValue;
|
||||
},
|
||||
set: function(key, value) {
|
||||
var conn = this.getDB();
|
||||
var q = conn.createStatement("INSERT OR REPLACE INTO setting values (:key, :value)");
|
||||
q.params.key = key;
|
||||
q.params.value = value;
|
||||
q.executeStep();
|
||||
},
|
||||
setTimeout: function(callback, timeout) {
|
||||
var timer = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer);
|
||||
timer.initWithCallback(callback, timeout, Ci.nsITimer.TYPE_ONE_SHOT);
|
||||
|
|
|
|||
|
|
@ -20,9 +20,9 @@ interface nsIOxFF : nsISupports
|
|||
string import();
|
||||
float progress(in AString oshash);
|
||||
boolean access([optional] in boolean request);
|
||||
boolean update();
|
||||
string volumes();
|
||||
boolean files(in oxICallback callback);
|
||||
boolean update(in oxICallback callback);
|
||||
string volumes(in oxICallback callback);
|
||||
boolean files(in AString volume, in oxICallback callback);
|
||||
boolean get(in AString oshash, in AString media, in oxICallback callback);
|
||||
boolean extract(in AString oshash, in AString media, in oxICallback callback);
|
||||
boolean logout();
|
||||
|
|
|
|||
|
|
@ -1,10 +1,59 @@
|
|||
<!doctype html>
|
||||
<html>
|
||||
<head>
|
||||
<script src="http://oxjs.org/js/jquery-1.4.2.min.js"></script>
|
||||
<script>
|
||||
|
||||
var ox = new OxFF();
|
||||
ox.access(true);
|
||||
ox.get('b2c8f0aa3a447d09', 'stills', function(result) { console.log(result);});
|
||||
ox.login('j');
|
||||
ox.login('username');
|
||||
|
||||
//ox.get('b2c8f0aa3a447d09', 'stills', function(result) { console.log(result);});
|
||||
//ox.files(function(result) { console.log(result);});
|
||||
|
||||
function update() {
|
||||
var updating = true;
|
||||
ox.update(function(result) {
|
||||
console.log('updating done', result);
|
||||
updating = false });
|
||||
var getFiles = function() {
|
||||
ox.volumes(function(result) {
|
||||
var volumes = JSON.parse(result);
|
||||
for(volume in volumes) {
|
||||
(function(volume) {
|
||||
var volumeId = volume.replace(/[ \/]/g, '_');
|
||||
var $volume = $('#'+volumeId);
|
||||
if($volume.length==0)
|
||||
$volume = $('<div>').attr('id', volumeId);
|
||||
$volume.html('<h1>'+volume+'</h1>');
|
||||
$('#files').append($volume);
|
||||
ox.files(volume, function(result) {
|
||||
var files = JSON.parse(result);
|
||||
$.each(files, function(folder, f) {
|
||||
$file = $('<div>').html(folder);
|
||||
$volume.append($file);
|
||||
});
|
||||
});
|
||||
}(volume));
|
||||
}
|
||||
if(updating) {
|
||||
setTimeout(getFiles, 2000);
|
||||
}
|
||||
});
|
||||
}
|
||||
getFiles();
|
||||
}
|
||||
|
||||
function addVolume() {
|
||||
console.log(ox.addVolume())
|
||||
}
|
||||
</script>
|
||||
</head>
|
||||
<body>
|
||||
<input type="button" onClick="addVolume()" value="Add Volume">
|
||||
<input type="button" onClick="update()" value="Update">
|
||||
<div id="files">
|
||||
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
|
|
|
|||
Loading…
Reference in a new issue