0x2620/OxFF
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

less magic in oxd, files returns files with path,stats,oshash and info a dict with oshash keys and related info

Browse source
This commit is contained in:
j 2010-08-11 00:10:26 +02:00
parent 05a922defb
commit edc69304e0
3 changed files with 67 additions and 81 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

67
OxFF/bin/oxd.py
View file

@ -285,8 +285,6 @@ class Database(object):
db = [ db = [
'''CREATE TABLE IF NOT EXISTS file ( '''CREATE TABLE IF NOT EXISTS file (
path varchar(1024) unique, path varchar(1024) unique,
folder varchar(1024),
filename varchar(1024),
oshash varchar(16), oshash varchar(16),
atime FLOAT, atime FLOAT,
ctime FLOAT, ctime FLOAT,
@ -347,17 +345,15 @@ class Database(object):
def file(self, oshash): def file(self, oshash):
conn, c = self.conn() conn, c = self.conn()
f = {} f = {}
sql = 'SELECT path, folder, filename, info FROM file WHERE oshash=?' sql = 'SELECT path, info FROM file WHERE oshash=?'
c.execute(sql, (oshash, )) c.execute(sql, (oshash, ))
for row in c: for row in c:
f['path'] = row[0] f['path'] = row[0]
f['folder'] = row[1] f['info'] = json.loads(row[2])
f['filename'] = row[2]
f['info'] = json.loads(row[3])
break break
return f return f
def files(self, site, user, volume, since=None): def files(self, site, user, volume):
conn, c = self.conn() conn, c = self.conn()
c.execute('SELECT path from volume where site=? AND user=? AND name=?', (site, user, volume)) c.execute('SELECT path from volume where site=? AND user=? AND name=?', (site, user, volume))
prefix = None prefix = None
@ -365,32 +361,28 @@ class Database(object):
prefix = row[0] prefix = row[0]
if not prefix: if not prefix:
return {} return {}
#since 2 volumes can have the same file/folder, needs some check for that or other structure
def get_files(files, key, sql, t=()):
t = list(t) + [u"%s%%"%prefix]
c.execute(sql, t)
for row in c:
folder = row[0]
filename = row[1]
info = json.loads(row[2])
if key:
if not key in files: files[key]={}
if not folder in files[key]: files[key][folder]={}
files[key][folder][filename] = info
else:
if not folder in files: files[folder]={}
files[folder][filename] = info
files = {} files = {}
sql_prefix = 'SELECT folder, filename, info FROM file WHERE ' files['info'] = {}
sql_postfix = ' deleted < 0 AND path LIKE ? ORDER BY path' files['files'] = []
if since: sql = 'SELECT path, oshash, info, atime, ctime, mtime FROM file WHERE deleted < 0 AND path LIKE ? ORDER BY path'
get_files(files, 'deleted', sql_prefix + 'deleted >= ? ORDER BY path' , (since, )) t = [u"%s%%"%prefix]
get_files(files, 'modified', c.execute(sql, t)
sql_prefix + 'created < ? AND modified >= ? AND'+sql_postfix, for row in c:
(since, since)) path = row[0]
get_files(files, 'new', sql_prefix + 'created >= ? AND'+sql_postfix, (since, )) oshash = row[1]
else: info = json.loads(row[2])
get_files(files, None, sql_prefix + sql_postfix) for key in ('atime', 'ctime', 'mtime', 'path'):
if key in info:
del info[key]
files['info'][oshash] = info
files['files'].append({
'oshash': oshash,
'path': path[len(prefix)+1:],
'atime': row[3],
'ctime': row[4],
'mtime': row[5],
})
return files return files
#derivative #derivative
@ -453,7 +445,7 @@ class Database(object):
self.derivative(oshash, name, STATUS_FAILED) self.derivative(oshash, name, STATUS_FAILED)
#volumes #volumes
def update(self, path, folder, filename): def update(self, path):
conn, c = self.conn() conn, c = self.conn()
update = True update = True
@ -470,13 +462,11 @@ class Database(object):
break break
if update: if update:
info = avinfo(path) info = avinfo(path)
for key in ('atime', 'ctime', 'mtime'):
info[key] = getattr(stat, 'st_'+key)
oshash = info['oshash'] oshash = info['oshash']
deleted = -1 deleted = -1
t = (path, folder, filename, oshash, stat.st_atime, stat.st_ctime, stat.st_mtime, t = (path, oshash, stat.st_atime, stat.st_ctime, stat.st_mtime,
stat.st_size, json.dumps(info), created, modified, deleted) stat.st_size, json.dumps(info), created, modified, deleted)
c.execute(u'INSERT OR REPLACE INTO file values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', t) c.execute(u'INSERT OR REPLACE INTO file values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', t)
conn.commit() conn.commit()
def spider(self, path): def spider(self, path):
@ -486,14 +476,13 @@ class Database(object):
if isinstance(dirpath, str): if isinstance(dirpath, str):
dirpath = dirpath.decode('utf-8') dirpath = dirpath.decode('utf-8')
if filenames: if filenames:
prefix = dirpath[len(path)+1:]
for filename in sorted(filenames): for filename in sorted(filenames):
if isinstance(filename, str): if isinstance(filename, str):
filename = filename.decode('utf-8') filename = filename.decode('utf-8')
if not filename.startswith('._') and not filename in ('.DS_Store', ): if not filename.startswith('._') and not filename in ('.DS_Store', ):
file_path = os.path.join(dirpath, filename) file_path = os.path.join(dirpath, filename)
files.append(file_path) files.append(file_path)
self.update(file_path, prefix, filename) self.update(file_path)
conn, c = self.conn() conn, c = self.conn()
c.execute('SELECT path FROM file WHERE path LIKE ? AND deleted < 0', ["%s%%"%path]) c.execute('SELECT path FROM file WHERE path LIKE ? AND deleted < 0', ["%s%%"%path])

2
OxFF/components/OxFF.js
View file

@ -192,7 +192,7 @@ OxFF.prototype = {
req.addEventListener("error", function(e) { req.addEventListener("error", function(e) {
_this.startDaemon(); _this.startDaemon();
ox.setTimeout(function() { _this.api(action, data, callback); }, 500); ox.setTimeout(function() { _this.api(action, data, callback); }, 1000);
}, false); }, false);
req.addEventListener("load", function(e) { req.addEventListener("load", function(e) {
//links should have base prefixed or whatever proxy is used to access them, i.e. some api call //links should have base prefixed or whatever proxy is used to access them, i.e. some api call

79
test/import.html
View file

@ -17,7 +17,7 @@ pandora.request = function(fn, data, callback) {
success: function(response) { success: function(response) {
if (typeof callback != "undefined") if (typeof callback != "undefined")
callback(response); callback(response);
console.log(response); //console.log(response);
}, },
error: function (xhr, ajaxOptions, thrownError){ error: function (xhr, ajaxOptions, thrownError){
var response = {}; var response = {};
@ -112,42 +112,32 @@ function update() {
var volumes = JSON.parse(result); var volumes = JSON.parse(result);
for(volume in volumes) { for(volume in volumes) {
ox.files(volume, function(result) { ox.files(volume, function(result) {
var _files = [];
var _info = {};
var data = JSON.parse(result); var data = JSON.parse(result);
$.each(data, function(folder, files) {
$.each(files, function(f, info) {
var f = {
oshash: info.oshash,
name: f,
folder: folder,
ctime: info.ctime, atime: info.atime, mtime: info.mtime
};
_files.push(f);
_info[info.oshash] = info;
});
});
pandora.request('update', { pandora.request('update', {
'volume': volume, 'files': _files 'volume': volume, 'files': data.files
}, function(result) { }, function(result) {
var data = {'info': {}}; var post = {'info': {}};
if (result.data.info.length>0) { function highlight_resulsts(result) {
$.each(_info, function(oshash, info) { $.each(result.data.data, function(i, oshash) {
if($.inArray(oshash, result.data.info) >= 0) { $('#' + oshash).css('background', 'red');
data.info[oshash] = info; $('#' + oshash).parent().css('background', 'orange');
}
}); });
pandora.request('update', data, function(result) { $.each(result.data.file, function(i, oshash) {
console.log(result); $('#' + oshash).css('background', 'blue');
}); });
} }
$.each(result.data.data, function(i, oshash) { if (result.data.info.length>0) {
$('#' + oshash).css('background', 'red'); $.each(data.info, function(oshash, info) {
$('#' + oshash).parent().css('background', 'orange'); if($.inArray(oshash, result.data.info) >= 0) {
}); post.info[oshash] = info;
$.each(result.data.file, function(i, oshash) { }
$('#' + oshash).css('background', 'blue'); });
}); pandora.request('update', post, function(result) {
highlight_resulsts(result);
});
} else {
highlight_resulsts(result);
}
}); });
}); });
} }
@ -168,8 +158,17 @@ function update() {
ox.files(volume, function(result) { ox.files(volume, function(result) {
var data = JSON.parse(result); var data = JSON.parse(result);
var _files = []; var _files = [];
$.each(data, function(folder, files) { $.each(data.files, function(i, file) {
if(!folder) folder = "rootfolder"; var folder = file.path.split('/');
folder.pop();
if(folder.length==0) {
folder.push("rootfolder");
}
//FIXME: this is also done on the backend but might require more sub options
if (folder[folder.length-1] == "Extras" || folder[folder.length-1] == "Versions")
folder.pop();
folder = folder.join('/');
var folderId = 'folder_'+safe_id(folder); var folderId = 'folder_'+safe_id(folder);
var $folder = $('#'+folderId); var $folder = $('#'+folderId);
if($folder.length==0) { if($folder.length==0) {
@ -177,14 +176,12 @@ function update() {
$folder.find('h3').click(function() { $(this).parent().find('div').toggle();}); $folder.find('h3').click(function() { $(this).parent().find('div').toggle();});
$volume.append($folder); $volume.append($folder);
} }
for_each_sorted(files, function(f, info) { var fileId = file.oshash;
var fileId = info.oshash; var $file = $('#'+fileId);
var $file = $('#'+fileId); if($file.length==0) {
if($file.length==0) { $file = $('<div>').attr('id', fileId).html(file.path).hide();
$file = $('<div>').attr('id', fileId).html(f).hide(); $folder.append($file);
$folder.append($file); }
}
});
}); });
}); });
}(volume)); }(volume));