From 5e9816e693cfa4d5c1e97edc3c289f7a30ac9059 Mon Sep 17 00:00:00 2001 From: j <0x006A@0x2620.org> Date: Fri, 20 Mar 2015 13:03:05 +0000 Subject: [PATCH] Add option to use Chrome's FileSystem API to cache videos explicitly --- static/js/cacheDialog.js | 304 +++++++++++++++++++++++++++++++++ static/js/fs.js | 234 +++++++++++++++++++++++++ static/js/preferencesDialog.js | 12 ++ static/js/utils.js | 18 +- 4 files changed, 561 insertions(+), 7 deletions(-) create mode 100644 static/js/cacheDialog.js create mode 100644 static/js/fs.js diff --git a/static/js/cacheDialog.js b/static/js/cacheDialog.js new file mode 100644 index 00000000..f0fae4b4 --- /dev/null +++ b/static/js/cacheDialog.js @@ -0,0 +1,304 @@ +pandora.ui.cacheDialog = function() { + + var ui = pandora.user.ui, + cachedVideos, + + $list = Ox.TableList({ + columns: [ + { + id: 'id', + title: Ox._('ID'), + visible: false, + width: 8 + }, + { + id: 'item', + title: Ox._('Item'), + visible: true, + width: 48 + }, + { + id: 'title', + operator: '+', + removable: false, + title: Ox._('Title'), + visible: true, + width: 192 + }, + { + id: 'resolution', + align: 'right', + operator: '+', + title: Ox._('Resolution'), + visible: true, + width: 72 + }, + { + id: 'size', + align: 'right', + operator: '-', + title: Ox._('Size'), + format: {type: 'value', args: ['B']}, + visible: true, + width: 64 + }, + { + id: 'added', + operator: '+', + title: Ox._('Added'), + format: {"type": "date", "args": ["%Y-%m-%d %H:%M:%S"]}, + visible: true, + width: 128 + }, + { + id: 'progress', + align: 'right', + operator: '+', + title: Ox._('Progress'), + format: function(data) { + return (data == 1 ? '100' : Ox.formatNumber(data * 100, 2)) + ' %'; + }, + visible: true, + width: 96 + } + ], + columnsVisible: true, + items: function(data, callback) { + cachedVideos + ? cachedVideos(data, callback) + : getCachedVideos(function(files) { + cachedVideos = Ox.api(files); + cachedVideos(data, callback); + }); + }, + keys: ['author'], + scrollbarVisible: true, + sort: [{key: 'progress', operator: '+'}], + unique: 'id' + }).bindEvent({ + 'delete': function(data) { + Ox.print('remove items', data.ids, data); + removeVideos(data.ids); + }, + select: function(data) { + $cancelButton.options({ + disabled: !data.ids.length + }); + } + }), + + $statusbar = Ox.Bar({size: 16}), + + $panel = Ox.SplitPanel({ + elements: [ + {element: $list}, + {element: $statusbar, size: 16} + ], + orientation: 'vertical' + }), + + $item = Ox.Element(), + + $cacheButton = Ox.Button({ + title: 'Cache Video...', + width: 128, + disabled: pandora.user.ui.section != 'items' + || !!pandora.fs.getVideoURL(pandora.user.ui.item, pandora.user.ui.videoResolution, 1) + }) + .css({ + margin: '8px' + }) + .bindEvent({ + click: function() { + $cacheButton.options({disabled: true}); + pandora.fs.cacheVideo(pandora.user.ui.item); + setTimeout(function() { + getCachedVideos(function(files) { + cachedVideos = Ox.api(files); + $list.reloadList(true); + }); + }, 50); + } + }) + .appendTo($item), + + $cacheListButton = Ox.Button({ + title: 'Cache List...', + width: 128, + disabled: !pandora.user.ui._list || pandora.user.ui.section != 'items' + }) + .css({ + margin: '8px' + }) + .bindEvent({ + click: function() { + $cacheListButton.options({disabled: true}); + pandora.api.find({ + query: { + conditions: [ + {'key': 'list', 'value': pandora.getListData().id} + ], + operator: '&' + }, + range: [0, pandora.getListData().items], + keys: ['id'] + }, function(result) { + result.data.items.forEach(function(item) { + pandora.fs.cacheVideo(item.id); + }); + setTimeout(function() { + getCachedVideos(function(files) { + cachedVideos = Ox.api(files); + $list.reloadList(true); + }); + }, 50); + }) + } + }) + .appendTo($item), + + $cacheEditButton = Ox.Button({ + title: 'Cache Edit...', + width: 128, + disabled: !pandora.user.ui.edit || pandora.user.ui.section != 'edits' + }) + .css({ + margin: '8px' + }) + .bindEvent({ + click: function() { + $cacheEditButton.options({disabled: true}); + pandora.api.getEdit({ + id: pandora.user.ui.edit, + keys: ['id', 'clips'] + }, function(result) { + Ox.unique(result.data.clips.map(function(clip) { + return clip.item + })).forEach(function(item) { + var update = false; + pandora.fs.cacheVideo(item, function(data) { + if (!update) { + update = true; + getCachedVideos(function(files) { + Ox.print(files, item, files.filter(function(f) { return f.item == item})); + Ox.print('downloads', pandora.fs.downloads); + cachedVideos = Ox.api(files); + $list.reloadList(true); + }); + } else { + Ox.print('download', data); + } + }); + }); + }) + } + }) + .appendTo($item), + + $cancelButton = Ox.Button({ + title: 'Remove...', + width: 128, + disabled: !($list.options('selected') || []).length + }) + .css({ + margin: '8px' + }) + .bindEvent({ + click: function() { + removeVideos($list.options('selected') || []); + } + }) + .appendTo($item), + + $content = Ox.SplitPanel({ + elements: [ + {element: $panel}, + {element: $item, size: 160} + ], + orientation: 'horizontal' + }), + + that = Ox.Dialog({ + buttons: [ + Ox.Button({ + id: 'done', + title: Ox._('Done') + }) + .bindEvent({ + click: function() { + that.close(); + } + }) + ], + closeButton: true, + content: $content, + height: 384, + title: Ox._('Manage Cached Videos'), + width: 768 + }) + .bindEvent({ + close: function() { + clearInterval(self.update); + } + }); + + self.update = setInterval(updateActiveDownloads, 1000); + + function getCachedVideos(callback) { + pandora.fs.getVideos(function(files) { + var items = Ox.unique(files.map(function(file) { + return file.item; + })); + pandora.api.find({ + query: { + conditions: items.map(function(item) { + return { + key: 'id', + operator: '==', + value: item + } + }), + operator: '|' + }, + keys: ['title', 'id'], + range: [0, items.length] + }, function(result) { + + files.forEach(function(file) { + file.title = result.data.items.filter(function(item) { + return item.id == file.item; + })[0].title; + }); + callback(files); + }); + }); + } + + function removeVideos(items) { + var ids = Ox.unique(items).map(function(id) { + return id.split('::')[0]; + }), done = 0; + ids.forEach(function(id) { + pandora.fs.removeVideo(id, function() { + ++done == ids.length && getCachedVideos(function(files) { + cachedVideos = Ox.api(files); + $list.reloadList(true); + }); + }); + }); + } + + function updateActiveDownloads() { + pandora.fs.getVideos(function(files) { + files.forEach(function(file) { + var current = $list.value(file.id); + if (!Ox.isEmpty(current) && current.progress != file.progress) { + $list.value(file.id, 'progress', file.progress); + } + }); + }); + } + + return that; + +}; diff --git a/static/js/fs.js b/static/js/fs.js new file mode 100644 index 00000000..032cde1e --- /dev/null +++ b/static/js/fs.js @@ -0,0 +1,234 @@ +'use strict'; + +pandora.fs = (function() { + var that = { + local: {}, + downloads: {}, + enabled: false, + }, + requestedBytes = 100*1024*1024*1024; // 100GB + + if(window.webkitRequestFileSystem) { + window.webkitRequestFileSystem(window.PERSISTENT, requestedBytes, function(fs) { + that.fs = fs; + that.fs.root.createReader().readEntries(function(results) { + results.forEach(function(entry) { + if (entry.isFile) { + that.local[entry.name] = entry.toURL(); + } + }); + }); + that.enabled = true; + }, function(e) { + Ox.Log('FS', 'Error:', e); + }); + } + + function getVideoName(id, resolution, part, track) { + return pandora.getVideoURLName(id, resolution, part, track).replace('/', '::'); + } + + that.cacheVideo = function(id, callback) { + that.downloads = that.downloads || {}; + that.downloads[id] = { + added: new Date(), + cancel: function() { + }, + id: id + '::' + pandora.user.ui.videoResolution, + item: id, + progress: 0, + resolution: pandora.user.ui.videoResolution, + size: 0 + }; + pandora.api.get({id: id, keys: ['parts']}, function(result) { + var parts = result.data.parts, sizes = []; + downloadPart(0); + + function downloadPart(part) { + that.downloadVideoURL(id, pandora.user.ui.videoResolution, part + 1, false, function(result) { + result.progress = 1/parts * (part + result.progress); + that.downloads[id].progress = result.progress; + if (result.cancel) { + that.downloads[id].cancel = result.cancel; + } + if (result.total) { + sizes[part] = result.total; + that.downloads[id].size = Ox.sum(sizes); + } + if (result.url) { + if (part + 1 == parts) { + delete that.downloads[id]; + } else { + downloadPart(part + 1); + } + } + callback && callback(result); + }); + } + }); + }; + + that.removeVideo = function(id, callback) { + if (that.downloads && that.downloads[id] && that.downloads[id].cancel) { + that.downloads[id].cancel(); + delete that.downloads[id]; + } else { + pandora.api.get({id: id, keys: ['parts']}, function(result) { + var count = result.data.parts * pandora.site.video.resolutions.length, done = 0; + Ox.range(result.data.parts).forEach(function(part) { + pandora.site.video.resolutions.forEach(function(resolution) { + var name = getVideoName(id, resolution, part + 1); + that.fs.root.getFile(name, {create: false}, function(fileEntry) { + // remove existing file + fileEntry.remove(function(e) { + if (that.local[name]) { + delete that.local[name]; + } + }); + ++done == count && callback(); + }, function() { // file not found + ++done == count && callback(); + }); + }); + }); + }); + } + }; + + that.storeBlob = function(blob, name, callback) { + requestQuota(blob.size, function() { + that.fs.root.getFile(name, {create: true}, function(fileEntry) { + fileEntry.createWriter(function(fileWriter) { + fileWriter.onwriteend = function(e) { + that.local[name] = fileEntry.toURL(); + callback({progress: 1, url: that.local[name]}); + }; + fileWriter.onerror = function(event) { + Ox.Log('FS', 'Write failed: ' + event.toString()); + callback({progress: -1, event: event}); + }; + fileWriter.write(blob); + }, function(event) { + callback({progress: -1, event: event}); + }); + }, function(event) { + callback({progress: -1, event: event}); + }); + }, function(event) { + callback({progress: -1, event: event}); + }); + + function requestQuota(size, callback, error) { + navigator.webkitPersistentStorage.queryUsageAndQuota(function(usage, quota) { + if (quota - usage < size) { + navigator.webkitPersistentStorage.requestQuota(quota + requestedBytes, function(grantedBytes) { + callback(); + }, error); + } else { + callback(); + } + }); + } + }; + + that.downloadVideoURL = function(id, resolution, part, track, callback) { + //fixme: would be nice to download videos from subdomains, + // currently throws a cross domain error + var name = getVideoName(id, resolution, part, track), + url = '/' + pandora.getVideoURLName(id, resolution, part, track), + blobs = [], blobSize = 5*1024*1024, total; + Ox.Log('FS', 'start downloading', url); + partialDownload(0); + function partialDownload(offset) { + var end = offset + blobSize; + if (total) { + end = Math.min(end, total); + } + Ox.Log('FS', 'download part', url, offset, end); + var xhr = new XMLHttpRequest(); + xhr.open('GET', url, true); + xhr.setRequestHeader('Range', 'bytes=' + offset + '-' + end); + xhr.withCredentials = true; + xhr.responseType = 'blob'; + xhr.addEventListener('progress', function(event) { + if (event.lengthComputable) { + if (!total) { + var range = xhr.getResponseHeader('Content-Range'); + total = Math.round(Ox.last(range.split('/'))); + } + var progress = (event.loaded + offset) / total; + callback({ + progress: progress, + request: xhr, + total: total, + cancel: function() { + xhr.abort(); + } + }); + } + }); + xhr.addEventListener('load', function() { + blobs.push(xhr.response); + if (offset + blobSize < total) { + partialDownload(offset + blobSize + 1); + } else { + that.storeBlob(new Blob(blobs), name, callback); + } + }); + xhr.addEventListener('error', function (event) { + Ox.print('partial download failed. retrying in 1 second'); + //fixme. make blobSize smaller if this fails? + setTimeout(function() { + partialDownload(offset); + }, 1000); + }); + xhr.addEventListener('abort', function (event) { + callback({progress: -1, event: event}); + }); + xhr.addEventListener('timeout', function (event) { + Ox.print('partial download, timeout'); + setTimeout(function() { + partialDownload(offset); + }, 1000); + }); + xhr.send(); + } + }; + + that.getVideos = function(callback) { + var files = {}; + that.fs.root.createReader().readEntries(function(results) { + var n = 0; + if (results.length) { + results.forEach(function(fileEntry) { + fileEntry.getMetadata(function(meta) { + var item = fileEntry.name.split('::')[0], + resolution = parseInt(fileEntry.name.split('::')[1].split('p')[0]), + part = parseInt(fileEntry.name.split('::')[1].split('p')[1].split('.')[0]), + key = item + '::' + resolution; + if (!(that.downloads && that.downloads[item])) { + files[key] = Ox.extend(files[key] || {}, { + added: meta.modificationTime, + id: item + '::' + resolution, + item: item, + progress: 1, + resolution: resolution, + size: files[key] ? files[key].size + meta.size: meta.size + }); + } + ++n == results.length && callback(Ox.values(files).concat(Ox.values(that.downloads))); + }); + }); + } else { + callback(Ox.values(that.downloads)); + } + }); + }; + + that.getVideoURL = function(id, resolution, part, track) { + var name = getVideoName(id, resolution, part, track); + return that.local[name]; + }; + + return that; +}()); diff --git a/static/js/preferencesDialog.js b/static/js/preferencesDialog.js index cb096962..12010e11 100644 --- a/static/js/preferencesDialog.js +++ b/static/js/preferencesDialog.js @@ -178,6 +178,18 @@ pandora.ui.preferencesDialog = function() { } }) .css({position: 'absolute', left: '96px', top: '40px'}) + ).append( + Ox.Button({ + title: Ox._('Manage Cache...'), + disabled: !pandora.fs.enabled + width: 160 + }) + .bindEvent({ + click: function() { + pandora.$ui.cacheDialog = pandora.ui.cacheDialog().open(); + } + }) + .css({position: 'absolute', left: '96px', top: '64px'}) ); } return $content; diff --git a/static/js/utils.js b/static/js/utils.js index 035a4a7d..9f706c5e 100644 --- a/static/js/utils.js +++ b/static/js/utils.js @@ -1791,15 +1791,19 @@ pandora.getMediaURL = function(url) { return pandora.site.site.mediaprefix + url; }; +pandora.getVideoURLName = function(id, resolution, part, track) { + return id + '/' + resolution + 'p' + part + (track ? '.' + track : '') + '.' + pandora.user.videoFormat; +}; + pandora.getVideoURL = function(id, resolution, part, track) { var prefix = pandora.site.site.videoprefix - .replace('{id}', id) - .replace('{part}', part) - .replace('{resolution}', resolution) - .replace('{uid}', Ox.uid()); - return prefix + '/' + id + '/' + resolution + 'p' + part - + (track ? '.' + track : '') - + '.' + pandora.user.videoFormat; + .replace('{id}', id) + .replace('{part}', part) + .replace('{resolution}', resolution) + .replace('{uid}', Ox.uid()), + local = pandora.fs && pandora.fs.getVideoURL(id, resolution, part, track); + return local || prefix + '/' + + pandora.getVideoURLName(id, resolution, part, track); }; pandora.getVideoOptions = function(data) {