From b92f0c1bd0f66b23a932d54ae834b59647e4c082 Mon Sep 17 00:00:00 2001 From: j <0x006A@0x2620.org> Date: Sun, 12 Apr 2015 21:23:04 +0200 Subject: [PATCH] chrome only show the first 100 enties in a folder, use more nested folders and move existing files into folders --- static/js/fs.js | 326 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 219 insertions(+), 107 deletions(-) diff --git a/static/js/fs.js b/static/js/fs.js index 6b00f500..696a051c 100644 --- a/static/js/fs.js +++ b/static/js/fs.js @@ -16,8 +16,39 @@ pandora.fs = (function() { that.fs = fs; that.fs.root.createReader().readEntries(function(results) { results.forEach(function(entry) { - if (entry.isFile && !Ox.startsWith(entry.name, 'partial::')) { - that.local[entry.name] = entry.toURL(); + if (entry.isFile) { + if (Ox.startsWith(entry.name, 'partial::')) { + fileEntry.remove(function() {}); + } else { + var name = entry.name.split('::'), + filename = Ox.last(name), + foldername = name[0] + "::" + filename.split('p')[0]; + var key = foldername[0] + '/' + foldername; + createTree(key, function(folder) { + entry.moveTo(folder, filename, function(e) { + var name = folder.name + '/' + e.name; + that.local[name] = e.toURL(); + that.storeBlob(new Blob(['ok']), key + '/done', function() {}); + }, function() { + Ox.print('error moving', filename); + }); + }); + } + } else if (entry.isDirectory) { + entry.createReader().readEntries(function(prefixes) { + prefixes.forEach(function(prefix) { + prefix.isDirectory && prefix.createReader().readEntries(function(contents) { + if (contents.filter(function(e) { return e.name == 'done'}).length) { + contents.forEach(function(e) { + if (e.name != 'done') { + var name = prefix.name + '/' + e.name; + that.local[name] = e.toURL(); + } + }); + } + }); + }); + }); } }); }); @@ -26,71 +57,80 @@ pandora.fs = (function() { }); } - function cacheVideo(id, callback) { - active = true; - pandora.api.get({id: id, keys: ['parts']}, function(result) { - var parts = result.data.parts, sizes = []; - downloadPart(0); + function createTree(folder, callback, root) { + var parts = folder.split('/'); + root = root || that.fs.root; + root.getDirectory(parts.shift(), {create: true}, function(folder) { + if (parts.length) { + createTree(parts.join('/'), callback, folder); + } else { + callback(folder); + } + }, function(error) { + Ox.Log('FS', 'error', error); + callback(); + }); + } - function downloadPart(part) { - if (that.getVideoURL(id, pandora.user.ui.videoResolution, part + 1)) { - done(); - } else { - that.downloadVideoURL(id, pandora.user.ui.videoResolution, part + 1, false, function(result) { - result.progress = 1/parts * (part + result.progress); - if (!that.downloads[id]) { - result.cancel && result.cancel(); - return; - } - that.downloads[id].progress = result.progress; - if (result.cancel) { - that.downloads[id].cancel = result.cancel; - } - if (result.total) { - sizes[part] = result.total; - that.downloads[id].size = Ox.sum(sizes); - } - if (result.url) { - done(); - } - callback && callback(result); - }); - } - function done() { - if (part + 1 == parts) { - renamePart(0); + function cacheVideo(id, callback) { + var key = id[0] + '/' + id + '::' + pandora.user.ui.videoResolution; + active = true; + createTree(key, function(folder) { + pandora.api.get({id: id, keys: ['parts']}, function(result) { + var parts = result.data.parts, sizes = []; + downloadPart(0); + + function downloadPart(part) { + var partName = key + '/' + pandora.user.ui.videoResolution + + 'p' + (part + 1) + '.' + pandora.user.videoFormat, + url = that.getVideoURL(id, pandora.user.ui.videoResolution, part + 1); + if (url) { + done({url: url}); } else { - setTimeout(function() { - downloadPart(part + 1); + that.downloadVideoURL(id, pandora.user.ui.videoResolution, part + 1, false, function(result) { + result.progress = 1/parts * (part + result.progress); + if (!that.downloads[id]) { + result.cancel && result.cancel(); + return; + } + that.downloads[id].progress = result.progress; + if (result.cancel) { + that.downloads[id].cancel = result.cancel; + } + if (result.total) { + sizes[part] = result.total; + that.downloads[id].size = Ox.sum(sizes); + } + if (result.url) { + done(result); + } else { + callback && callback(result); + } }); } - } - function renamePart(i) { - var name = that.getVideoName(id, pandora.user.ui.videoResolution, i + 1), - partialName = 'partial::' + name; - renameFile(partialName, name, function(fileEntry) { - if (fileEntry) { - Ox.Log('FS', 'renamed file', partialName, 'to', name); - that.local[name] = fileEntry.toURL(); + function done(result) { + that.local[partName] = result.url; + if (part + 1 == parts) { + //fixme + that.storeBlob(new Blob(['ok']), key + '/done', function() { + delete that.downloads[id]; + active = false; + callback && callback(result); + if (queue.length) { + var next = queue.shift(); + setTimeout(function() { + cacheVideo(next[0], next[1]); + }, 50); + } + }); } else { - Ox.print('rename failed', name); - callback && callback({progress: -1, error: 'rename failed'}); + setTimeout(function() { + downloadPart(part + 1); + }); } - if (i + 1 == parts) { - delete that.downloads[id]; - active = false; - if (queue.length) { - var next = queue.shift(); - setTimeout(function() { - cacheVideo(next[0], next[1]); - }, 50); - } - } else { - renamePart(i + 1); - } - }); + } } - } + }); }); } @@ -108,7 +148,7 @@ pandora.fs = (function() { that.cacheVideo = function(id, callback) { if (that.getVideoURL(id, pandora.user.ui.videoResolution, 1) || that.downloads[id]) { - callback({progress: 1}); + callback && callback({progress: 1}); return; } else { that.downloads = that.downloads || {}; @@ -131,38 +171,53 @@ pandora.fs = (function() { }; that.getVideoName = function(id, resolution, part, track) { - return pandora.getVideoURLName(id, resolution, part, track).replace('/', '::'); + return pandora.getVideoURLName(id, resolution, part, track).replace(id + '\/', id + '::' + resolution + '/'); }; that.removeVideo = function(id, callback) { if (that.downloads && that.downloads[id] && that.downloads[id].cancel) { that.downloads[id].cancel(); delete that.downloads[id]; - } else { - pandora.api.get({id: id, keys: ['parts']}, function(result) { - var count = result.data.parts * pandora.site.video.resolutions.length, done = 0; - Ox.range(result.data.parts).forEach(function(part) { - pandora.site.video.resolutions.forEach(function(resolution) { - var name = that.getVideoName(id, resolution, part + 1); - that.fs.root.getFile(name, {create: false}, function(fileEntry) { - // remove existing file - fileEntry.remove(function(e) { - if (that.local[name]) { - delete that.local[name]; - } - }); - ++done == count && callback(); - }, function() { // file not found - ++done == count && callback(); - }); - // remove partial file too - that.fs.root.getFile('partial::' + name, {create: false}, function(fileEntry) { - fileEntry.remove(function(e) {}); + } + // remove legacy files too + pandora.api.get({id: id, keys: ['parts']}, function(result) { + var count = result.data.parts * pandora.site.video.resolutions.length, done = 0; + Ox.range(result.data.parts).forEach(function(part) { + pandora.site.video.resolutions.forEach(function(resolution) { + var name = pandora.getVideoURLName(id, resolution, part + 1).replace('/', '::'); + that.fs.root.getFile(name, {create: false}, function(fileEntry) { + // remove existing file + fileEntry.remove(function(e) { + if (that.local[name]) { + delete that.local[name]; + } }); + }, function() { // file not found + }); + // remove partial file too + that.fs.root.getFile('partial::' + name, {create: false}, function(fileEntry) { + fileEntry.remove(function(e) {}); }); }); }); - } + }); + Ox.parallelForEach(pandora.site.video.resolutions, + function(resolution, i, resolutions, cb) { + var key = id + '::' + resolution; + that.fs.root.getDirectory(key[0] + '/' + key, {create: false}, function(dir) { + dir.removeRecursively(cb, cb); + Object.keys(that.local).forEach(function(name) { + if (Ox.startsWith(name, key)) { + delete that.local[name]; + } + }); + cb(); + }, function() { + cb(); + }); + }, function() { + callback(); + }); }; that.storeBlob = function(blob, name, callback, append) { @@ -208,14 +263,13 @@ pandora.fs = (function() { //fixme: would be nice to download videos from subdomains, // currently throws a cross domain error var name = that.getVideoName(id, resolution, part, track), - partialName = 'partial::' + name, url = '/' + pandora.getVideoURLName(id, resolution, part, track), blobSize = 5*1024*1024, total; Ox.Log('FS', 'start downloading', url); getSize(url, function(size) { Ox.Log('FS', 'HEAD', url, size); total = size; - that.fs.root.getFile(partialName, {create: false}, function(fileEntry) { + that.fs.root.getFile(name, {create: false}, function(fileEntry) { fileEntry.getMetadata(function(meta) { if (meta.size >= total) { Ox.Log('FS', 'file exists, done', meta.size, total); @@ -277,7 +331,6 @@ pandora.fs = (function() { var progress = (event.loaded + offset) / total; callback({ progress: progress, - request: xhr, total: total, cancel: function() { xhr.abort(); @@ -295,7 +348,7 @@ pandora.fs = (function() { xhr.addEventListener('load', function() { var blob = xhr.response; setTimeout(function() { - that.storeBlob(blob, partialName, function(response) { + that.storeBlob(blob, name[0] + '/' + name, function(response) { if (offset + blob.size < total) { partialDownload(offset + blob.size); } else { @@ -327,30 +380,76 @@ pandora.fs = (function() { that.getVideos = function(callback) { var files = {}; that.fs.root.createReader().readEntries(function(results) { - var n = 0; if (results.length) { - results.forEach(function(fileEntry) { - if (Ox.startsWith(fileEntry.name, 'partial::')) { - ++n == results.length && callback(Ox.values(files).concat(Ox.values(that.downloads))); - } else { - fileEntry.getMetadata(function(meta) { - var item = fileEntry.name.split('::')[0], - resolution = parseInt(fileEntry.name.split('::')[1].split('p')[0]), - part = parseInt(fileEntry.name.split('::')[1].split('p')[1].split('.')[0]), - key = item + '::' + resolution; - if (!(that.downloads && that.downloads[item])) { - files[key] = Ox.extend(files[key] || {}, { - added: meta.modificationTime, - id: item + '::' + resolution, - item: item, - progress: 1, - resolution: resolution, - size: files[key] ? files[key].size + meta.size: meta.size - }); + Ox.parallelForEach(results, function(fileEntry, i, entries, next_entry) { + if (fileEntry.isFile) { + if (Ox.startsWith(fileEntry.name, 'partial::')) { + next_entry(); + } else { + fileEntry.getMetadata(function(meta) { + var item = fileEntry.name.split('::')[0], + resolution = parseInt(fileEntry.name.split('::')[1].split('p')[0]), + part = parseInt(fileEntry.name.split('::')[1].split('p')[1].split('.')[0]), + key = item + '::' + resolution; + if (!(that.downloads && that.downloads[item])) { + files[key] = Ox.extend(files[key] || {}, { + added: meta.modificationTime, + id: item + '::' + resolution, + item: item, + progress: 1, + resolution: resolution, + size: files[key] ? files[key].size + meta.size: meta.size + }); + } + next_entry(); + }); + } + } else if (fileEntry.isDirectory) { + fileEntry.createReader().readEntries(function(prefixes) { + if (prefixes.length) { + Ox.parallelForEach(prefixes, function(prefix, i, prefixes, callback) { + prefix.createReader().readEntries(function(contents) { + if (contents.filter(function(f) { return f.name == 'done' }).length) { + Ox.parallelMap(contents, + function(e, i, col, callback) { + e.getMetadata(function(meta) { + callback(meta); + }); + }, function(meta) { + var item = prefix.name.split('::')[0], + resolution = parseInt(prefix.name.split('::')[1]), + key = item + '::' + resolution; + if (!(that.downloads && that.downloads[item])) { + files[key] = Ox.extend(files[key] || {}, { + added: Ox.min(meta.map(function(m) { + return m.modificationTime; + })), + id: item + '::' + resolution, + item: item, + progress: 1, + resolution: resolution, + size: Ox.sum(meta.map(function(m) { + return m.size; + })) + }); + } + callback(); + } + ); + } else { + callback(); + } + }); + }, next_entry); + } else { + next_entry(); } - ++n == results.length && callback(Ox.values(files).concat(Ox.values(that.downloads))); }); + } else { + next_entry(); } + }, function() { + callback(Ox.values(files).concat(Ox.values(that.downloads))); }); } else { callback(Ox.values(that.downloads)); @@ -363,5 +462,18 @@ pandora.fs = (function() { return that.local[name]; }; + that._tree = function(root) { + root = root || that.fs.root; + root.createReader().readEntries(function(results) { + results.forEach(function(entry) { + if (entry.isFile) { + Ox.print(entry.fullPath); + } else { + that._tree(entry); + } + }); + }); + }; + return that; }());