From 8788dd9fe8b43e2ff5409e7a513e24ba64578ed5 Mon Sep 17 00:00:00 2001 From: j Date: Sun, 7 Jun 2020 17:19:46 +0200 Subject: [PATCH 01/46] pandoractl installs itself --- vm/pandora_install.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/vm/pandora_install.sh b/vm/pandora_install.sh index ec77d3b0..6f9d66d6 100755 --- a/vm/pandora_install.sh +++ b/vm/pandora_install.sh @@ -199,7 +199,6 @@ if [ "$NGINX" == "local" ]; then cp "/srv/pandora/etc/nginx/pandora" "/etc/nginx/sites-available/pandora" rm -f /etc/nginx/sites-enabled/default /etc/nginx/sites-enabled/pandora ln -s ../sites-available/pandora /etc/nginx/sites-enabled/pandora -ln -s /srv/pandora/ctl /usr/local/bin/pandoractl read -r -d '' GZIP < Date: Tue, 21 Jul 2020 13:12:23 +0200 Subject: [PATCH 02/46] delete items from archive while in list --- static/js/mainMenu.js | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/static/js/mainMenu.js b/static/js/mainMenu.js index e1a80b5e..9ff8a94e 100644 --- a/static/js/mainMenu.js +++ b/static/js/mainMenu.js @@ -442,7 +442,37 @@ pandora.ui.mainMenu = function() { }); } } else if (data.id == 'deletefromarchive') { - if (ui.section == 'documents') { + if (ui.section == 'items') { + var ids; + if (ui.item) { + ids = [ui.item] + } else { + ids = ui.listSelection + } + pandora.api.find({ + query: { + conditions: [{ + key: 'id', + operator: '&', + value: ids + }], + operator: '&' + }, + keys: ['id', 'title'], + range: [0, ui.listSelection.length] + }, function(result) { + pandora.$ui.deleteItemsDialog = pandora.ui.deleteItemsDialog({ + items: result.data.items + }, function() { + Ox.Request.clearCache(); + if (ui.item) { + pandora.UI.set({item: ''}); + } else { + pandora.$ui.list.reloadList() + } + }).open(); + }); + } else if (ui.section == 'documents') { var files; if (ui.document) { files = [pandora.$ui.document.info()]; @@ -1364,6 +1394,9 @@ pandora.ui.mainMenu = function() { { id: 'clearclipboard', title: Ox._('Clear Clipboard'), disabled: !clipboardItems}, {}, { id: 'delete', title: Ox._('{0} {1} {2}', [deleteVerb, selectionItemName, listName]), disabled: !canDelete, keyboard: 'delete' }, + ui._list ? [ + { id: 'deletefromarchive', title: Ox._('{0} {1} {2}', [Ox._('Delete'), selectionItemName, Ox._('from Archive')]), disabled: !canDelete } + ] : [], {}, { id: 'undo', title: undoText ? Ox._('Undo {0}', [undoText]) : Ox._('Undo'), disabled: !undoText, keyboard: 'control z' }, { id: 'redo', title: redoText ? Ox._('Redo {0}', [redoText]) : Ox._('Redo'), disabled: !redoText, keyboard: 'shift control z' }, From 953cb93745c25d888950a6674c7455835246dc9a Mon Sep 17 00:00:00 2001 From: j Date: Tue, 21 Jul 2020 14:23:08 +0200 Subject: [PATCH 03/46] load editable in batch dialog --- static/js/editDialog.js | 40 +++++++++++++++++++++++++------- static/js/editDocumentsDialog.js | 35 ++++++++++++++++++++++------ static/js/utils.js | 4 ++-- 3 files changed, 61 insertions(+), 18 deletions(-) diff --git a/static/js/editDialog.js b/static/js/editDialog.js index f35747c0..458cb987 100644 --- a/static/js/editDialog.js +++ b/static/js/editDialog.js @@ -4,14 +4,12 @@ pandora.ui.editDialog = function() { var ui = pandora.user.ui, hasChanged = false, - ids = ui.listSelection.filter(function(id) { - return pandora.$ui.list.value(id, 'editable'); - }), - keys = pandora.site.itemKeys.filter(function(key) { + ids = ui.listSelection, + keys = ['editable'].concat(pandora.site.itemKeys.filter(function(key) { return key.id != '*' }).map(function(key) { return key.id - }), + })), listKeys = pandora.site.itemKeys.filter(function(key) { return Ox.isArray(key.type); }).map(function(key){ @@ -90,22 +88,46 @@ pandora.ui.editDialog = function() { }, function(result) { var data = {}, isMixed = {}, - items = result.data.items; - keys.forEach(function(key) { + updateTitle = false, + items = result.data.items.filter(function(item) { + if (!item.editable) { + updateTitle = true + } + return item.editable; + }); + + if (updateTitle) { + that.options({ + title: Ox._('Edit Metadata for {0}', [ + Ox.formatNumber(items.length) + ' ' + Ox._( + items.length == 1 ? pandora.site.itemName.singular : pandora.site.itemName.plural + ) + ]) + }) + // no editable items + if (!items.length) { + that.close() + return + } + } + keys.filter(function(key) { + return key != 'editable' + }).forEach(function(key) { var isArray = Ox.contains(listKeys, key), values = items.map(function(item) { return item[key]; }); if (isArray) { values = values.map(function(value) { - return (value || []).join(separator); + value = value || [] + return value.join ? value.join(separator) : value; }); } if (Ox.unique(values).length > 1) { isMixed[key] = true; } data[key] = isMixed[key] ? null - : isArray ? values[0].split(separator) + : isArray && values.length ? values[0].split(separator) : values[0]; }); that.options({ diff --git a/static/js/editDocumentsDialog.js b/static/js/editDocumentsDialog.js index 620cc153..c5cdec12 100644 --- a/static/js/editDocumentsDialog.js +++ b/static/js/editDocumentsDialog.js @@ -3,14 +3,12 @@ pandora.ui.editDocumentsDialog = function() { var ui = pandora.user.ui, hasChanged = false, - ids = ui.collectionSelection.filter(function(id) { - return pandora.$ui.list.value(id, 'editable'); - }), - keys = pandora.site.documentKeys.filter(function(key) { + ids = ui.collectionSelection, + keys = ['editable'].concat(pandora.site.documentKeys.filter(function(key) { return key.id != '*' }).map(function(key) { return key.id - }), + })), listKeys = pandora.site.documentKeys.filter(function(key) { return Ox.isArray(key.type); }).map(function(key){ @@ -89,8 +87,31 @@ pandora.ui.editDocumentsDialog = function() { }, function(result) { var data = {}, isMixed = {}, - items = result.data.items; - keys.forEach(function(key) { + updateTitle = false, + items = result.data.items.filter(function(item) { + if (!item.editable) { + updateTitle = true + } + return item.editable; + }); + if (updateTitle) { + that.options({ + title: Ox._('Edit Metadata for {0}', [ + Ox.formatNumber(items.length) + ' ' + Ox._( + items.length == 1 ? 'Document' : 'Documents' + ) + ]) + }) + // no editable items + if (!items.length) { + that.close() + return + } + } + + keys.filter(function(key) { + return key != 'editable' + }).forEach(function(key) { var isArray = Ox.contains(listKeys, key), values = items.map(function(item) { return item[key]; diff --git a/static/js/utils.js b/static/js/utils.js index d4d0f610..f5d9590f 100644 --- a/static/js/utils.js +++ b/static/js/utils.js @@ -725,11 +725,11 @@ pandora.uploadDroppedFiles = function(files) { pandora.enableBatchEdit = function(section) { var ui = pandora.user.ui; if (section == 'documents') { - return !ui.document && ui.collectionSelection.length > 1 && ui.collectionSelection.every(function(item) { + return !ui.document && ui.collectionSelection.length > 1 && ui.collectionSelection.some(function(item) { return pandora.$ui.list && pandora.$ui.list.value(item, 'editable'); }) } else { - return !ui.item && ui.listSelection.length > 1 && ui.listSelection.every(function(item) { + return !ui.item && ui.listSelection.length > 1 && ui.listSelection.some(function(item) { return pandora.$ui.list && pandora.$ui.list.value(item, 'editable'); }) } From 4bde77abccd15c0abe2df6ecd2779b4db80456fd Mon Sep 17 00:00:00 2001 From: j Date: Tue, 21 Jul 2020 14:49:34 +0200 Subject: [PATCH 04/46] rebuild posters --- .../management/commands/rebuild_posters.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 pandora/item/management/commands/rebuild_posters.py diff --git a/pandora/item/management/commands/rebuild_posters.py b/pandora/item/management/commands/rebuild_posters.py new file mode 100644 index 00000000..d22e4882 --- /dev/null +++ b/pandora/item/management/commands/rebuild_posters.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- + +import os +from glob import glob + +from django.core.management.base import BaseCommand + +import app.monkey_patch +from ... import models +from ... import tasks + +class Command(BaseCommand): + """ + rebuild posters for all items. + """ + help = 'rebuild all posters for all items.' + args = '' + + def handle(self, **options): + offset = 0 + chunk = 100 + count = models.Item.objects.count() + while offset <= count: + for i in models.Item.objects.all().order_by('id')[offset:offset+chunk]: + print(i) + if i.poster: + i.poster.delete() + i.make_poster() + offset += chunk From e4815f091d2d52c2e4a5e4f3625f7bd922e368bf Mon Sep 17 00:00:00 2001 From: j Date: Tue, 21 Jul 2020 14:51:14 +0200 Subject: [PATCH 05/46] move rights level code into infoViewUtils --- static/js/infoView.js | 137 +---------------------------------- static/js/infoView.padma.js | 141 +----------------------------------- static/js/infoViewUtils.js | 141 ++++++++++++++++++++++++++++++++++++ 3 files changed, 145 insertions(+), 274 deletions(-) diff --git a/static/js/infoView.js b/static/js/infoView.js index 6e68a95a..5bd588ed 100644 --- a/static/js/infoView.js +++ b/static/js/infoView.js @@ -198,9 +198,7 @@ pandora.ui.infoView = function(data, isMixed) { top: margin + 'px', right: margin + 'px' }) - .appendTo($info), - - $capabilities; + .appendTo($info); [$options, $edit].forEach(function($element) { $element.find('input').css({ @@ -346,7 +344,7 @@ pandora.ui.infoView = function(data, isMixed) { .append(formatKey('Rights Level', 'statistics')) .append($rightsLevel) .appendTo($statistics); - renderRightsLevel(); + pandora.renderRightsLevel(that, $rightsLevel, data, isMixed, isMultiple, canEdit); // Notes -------------------------------------------------------------------- @@ -496,15 +494,6 @@ pandora.ui.infoView = function(data, isMixed) { return ret; } - function getRightsLevelElement(rightsLevel) { - return Ox.Theme.formatColorLevel( - rightsLevel, - pandora.site.rightsLevels.map(function(rightsLevel) { - return rightsLevel.name; - }) - ); - } - function getValue(key, value) { return !value ? '' : Ox.contains(specialListKeys, key) ? value.join('; ') @@ -512,81 +501,6 @@ pandora.ui.infoView = function(data, isMixed) { : value; } - function renderCapabilities(rightsLevel) { - var capabilities = [].concat( - canEdit ? [{name: 'canSeeItem', symbol: 'Find'}] : [], - [ - {name: 'canPlayClips', symbol: 'PlayInToOut'}, - {name: 'canPlayVideo', symbol: 'Play'}, - {name: 'canDownloadVideo', symbol: 'Download'} - ] - ), - userLevels = canEdit ? pandora.site.userLevels : [pandora.user.level]; - $capabilities.empty(); - userLevels.forEach(function(userLevel, i) { - var $element, - $line = $('
') - .css({ - height: '16px', - marginBottom: '4px' - }) - .appendTo($capabilities); - if (canEdit) { - $element = Ox.Theme.formatColorLevel(i, userLevels.map(function(userLevel) { - return Ox.toTitleCase(userLevel); - }), [0, 240]); - Ox.Label({ - textAlign: 'center', - title: Ox.toTitleCase(userLevel), - width: 60 - }) - .addClass('OxColor OxColorGradient') - .css({ - float: 'left', - height: '12px', - paddingTop: '2px', - background: $element.css('background'), - fontSize: '8px', - color: $element.css('color') - }) - .data({OxColor: $element.data('OxColor')}) - .appendTo($line); - } - capabilities.forEach(function(capability) { - var hasCapability = pandora.hasCapability(capability.name, userLevel) >= rightsLevel, - $element = Ox.Theme.formatColorLevel(hasCapability, ['', '']); - Ox.Button({ - tooltip: (canEdit ? Ox.toTitleCase(userLevel) : 'You') + ' ' - + (hasCapability ? 'can' : 'can\'t') + ' ' - + Ox.toSlashes(capability.name) - .split('/').slice(1).join(' ') - .toLowerCase(), - title: capability.symbol, - type: 'image' - }) - .addClass('OxColor OxColorGradient') - .css({background: $element.css('background')}) - .css('margin' + (canEdit ? 'Left' : 'Right'), '4px') - .data({OxColor: $element.data('OxColor')}) - .appendTo($line); - }); - if (!canEdit) { - Ox.Button({ - title: Ox._('Help'), - tooltip: Ox._('About Rights'), - type: 'image' - }) - .css({marginLeft: '52px'}) - .bindEvent({ - click: function() { - pandora.UI.set({page: 'rights'}); - } - }) - .appendTo($line); - } - }); - } - function renderGroup(keys) { var $element; keys.forEach(function(key) { displayedKeys.push(key) }); @@ -632,53 +546,6 @@ pandora.ui.infoView = function(data, isMixed) { } } - function renderRightsLevel() { - var $rightsLevelElement = getRightsLevelElement(data.rightslevel), - $rightsLevelSelect; - $rightsLevel.empty(); - if (canEdit) { - $rightsLevelSelect = Ox.Select({ - items: pandora.site.rightsLevels.map(function(rightsLevel, i) { - return {id: i, title: rightsLevel.name}; - }), - width: 128, - value: data.rightslevel - }) - .addClass('OxColor OxColorGradient') - .css({ - marginBottom: '4px', - background: $rightsLevelElement.css('background') - }) - .data({OxColor: $rightsLevelElement.data('OxColor')}) - .bindEvent({ - change: function(event) { - var rightsLevel = event.value; - $rightsLevelElement = getRightsLevelElement(rightsLevel); - $rightsLevelSelect - .css({background: $rightsLevelElement.css('background')}) - .data({OxColor: $rightsLevelElement.data('OxColor')}) - renderCapabilities(rightsLevel); - var edit = { - id: isMultiple ? ui.listSelection : data.id, - rightslevel: rightsLevel - }; - pandora.api.edit(edit, function(result) { - that.triggerEvent('change', Ox.extend({}, 'rightslevel', rightsLevel)); - }); - } - }) - .appendTo($rightsLevel); - } else { - $rightsLevelElement - .css({ - marginBottom: '4px' - }) - .appendTo($rightsLevel); - } - $capabilities = $('
').appendTo($rightsLevel); - renderCapabilities(data.rightslevel); - } - function toggleIconSize() { iconSize = iconSize == 256 ? 512 : 256; iconWidth = iconRatio > 1 ? iconSize : Math.round(iconSize * iconRatio); diff --git a/static/js/infoView.padma.js b/static/js/infoView.padma.js index 93125b48..b6d44118 100644 --- a/static/js/infoView.padma.js +++ b/static/js/infoView.padma.js @@ -204,9 +204,7 @@ pandora.ui.infoView = function(data, isMixed) { top: margin + 'px', right: margin + 'px' }) - .appendTo($info), - - $capabilities; + .appendTo($info); [$options, $edit].forEach(function($element) { $element.find('input').css({ @@ -437,7 +435,7 @@ pandora.ui.infoView = function(data, isMixed) { .append(formatKey(Ox._('Rights Level'), 'statistics')) .append($rightsLevel) .appendTo($statistics); - renderRightsLevel(); + pandora.renderRightsLevel(that, $rightsLevel, data, isMixed, isMultiple, canEdit); // User and Groups --------------------------------------------------------- if (!isMultiple) { @@ -628,100 +626,12 @@ pandora.ui.infoView = function(data, isMixed) { return formatLink(key, ret, key == 'date' && value); } - function getRightsLevelElement(rightsLevel) { - return Ox.Theme.formatColorLevel( - rightsLevel, - pandora.site.rightsLevels.map(function(rightsLevel) { - return rightsLevel.name; - }) - ); - } - function getValue(key, value) { return !value ? '' : Ox.contains(listKeys, key) ? value.join(', ') : value; } - function renderCapabilities(rightsLevel) { - var capabilities = [].concat( - canEdit ? [{name: 'canSeeItem', symbol: 'Find'}] : [], - [ - {name: 'canPlayClips', symbol: 'PlayInToOut'}, - {name: 'canPlayVideo', symbol: 'Play'}, - {name: 'canDownloadVideo', symbol: 'Download'} - ] - ), - userLevels = canEdit ? pandora.site.userLevels : [pandora.user.level]; - $capabilities.empty(); - userLevels.forEach(function(userLevel, i) { - var $element, - $line = $('
') - .css({ - height: '16px', - marginBottom: '4px' - }) - .appendTo($capabilities); - if (canEdit) { - $element = Ox.Theme.formatColorLevel(i, userLevels.map(function(userLevel) { - return Ox.toTitleCase(userLevel); - }), [0, 240]); - Ox.Label({ - textAlign: 'center', - title: Ox._(Ox.toTitleCase(userLevel)), - width: 60 - }) - .addClass('OxColor OxColorGradient') - .css({ - float: 'left', - height: '12px', - paddingTop: '2px', - background: $element.css('background'), - fontSize: '8px', - color: $element.css('color') - }) - .data({OxColor: $element.data('OxColor')}) - .appendTo($line); - } - capabilities.forEach(function(capability) { - var hasCapability = pandora.hasCapability(capability.name, userLevel) >= rightsLevel, - $element = Ox.Theme.formatColorLevel(hasCapability, ['', '']); - Ox.Button({ - tooltip: Ox._('{0} ' - + (hasCapability ? 'can' : 'can\'t') + ' ' - + Ox.toSlashes(capability.name) - .split('/').slice(1).join(' ') - .toLowerCase() - .replace('see item', 'see the item') - .replace('play video', 'play the full video') - .replace('download video', 'download the video'), - [canEdit ? Ox._(Ox.toTitleCase(userLevel)) : Ox._('You')]), - title: capability.symbol, - type: 'image' - }) - .addClass('OxColor OxColorGradient') - .css({background: $element.css('background')}) - .css('margin' + (canEdit ? 'Left' : 'Right'), '4px') - .data({OxColor: $element.data('OxColor')}) - .appendTo($line); - }); - if (!canEdit) { - Ox.Button({ - title: Ox._('Help'), - tooltip: Ox._('About Rights'), - type: 'image' - }) - .css({marginLeft: '52px'}) - .bindEvent({ - click: function() { - pandora.UI.set({page: 'rights'}); - } - }) - .appendTo($line); - } - }); - } - function renderGroup(keys) { var $element; if (canEdit || keys.filter(function(key) { @@ -757,53 +667,6 @@ pandora.ui.infoView = function(data, isMixed) { } } - function renderRightsLevel() { - var $rightsLevelElement = getRightsLevelElement(data.rightslevel), - $rightsLevelSelect; - $rightsLevel.empty(); - if (canEdit) { - $rightsLevelSelect = Ox.Select({ - items: pandora.site.rightsLevels.map(function(rightsLevel, i) { - return {id: i, title: Ox._(rightsLevel.name)}; - }), - width: 128, - value: data.rightslevel - }) - .addClass('OxColor OxColorGradient') - .css({ - marginBottom: '4px', - background: $rightsLevelElement.css('background') - }) - .data({OxColor: $rightsLevelElement.data('OxColor')}) - .bindEvent({ - change: function(event) { - var rightsLevel = event.value; - $rightsLevelElement = getRightsLevelElement(rightsLevel); - $rightsLevelSelect - .css({background: $rightsLevelElement.css('background')}) - .data({OxColor: $rightsLevelElement.data('OxColor')}) - renderCapabilities(rightsLevel); - var edit = { - id: isMultiple ? ui.listSelection : data.id, - rightslevel: rightsLevel - }; - pandora.api.edit(edit, function(result) { - that.triggerEvent('change', Ox.extend({}, 'rightslevel', rightsLevel)); - }); - } - }) - .appendTo($rightsLevel); - } else { - $rightsLevelElement - .css({ - marginBottom: '4px' - }) - .appendTo($rightsLevel); - } - $capabilities = $('
').appendTo($rightsLevel); - renderCapabilities(data.rightslevel); - } - function toggleIconSize() { iconSize = iconSize == 256 ? 512 : 256; iconWidth = iconRatio > 1 ? iconSize : Math.round(iconSize * iconRatio); diff --git a/static/js/infoViewUtils.js b/static/js/infoViewUtils.js index 9ab90665..9d9de29a 100644 --- a/static/js/infoViewUtils.js +++ b/static/js/infoViewUtils.js @@ -14,3 +14,144 @@ pandora.cleanupDate = function(value) { return value }; +pandora.renderRightsLevel = function(that, $rightsLevel, data, isMixed, isMultiple, canEdit) { + var rightsLevels = pandora.site.rightsLevels.map(function(rightsLevel) { + return rightsLevel.name; + }).concat(isMultiple ? ['Mixed'] : []), + rightsLevel = isMixed.rightslevel ? rightsLevels.length - 1 : data.rightslevel, + $capabilities, + $rightsLevelElement = getRightsLevelElement(rightsLevel), + $rightsLevelSelect; + $rightsLevel.empty(); + if (canEdit) { + $rightsLevelSelect = Ox.Select({ + items: pandora.site.rightsLevels.map(function(rightsLevel, i) { + return {id: i, title: Ox._(rightsLevel.name)}; + }).concat(isMultiple ? [ + {id: rightsLevels.length - 1, title: Ox._('Mixed'), disabled: true} + ] : []), + width: 128, + value: rightsLevel + }) + .addClass('OxColor OxColorGradient') + .css({ + marginBottom: '4px', + background: $rightsLevelElement.css('background') + }) + .data({OxColor: $rightsLevelElement.data('OxColor')}) + .bindEvent({ + change: function(event) { + var rightsLevel = event.value; + $rightsLevelElement = getRightsLevelElement(rightsLevel); + $rightsLevelSelect + .css({background: $rightsLevelElement.css('background')}) + .data({OxColor: $rightsLevelElement.data('OxColor')}) + if (rightsLevel < pandora.site.rightsLevels.length) { + renderCapabilities(rightsLevel); + var edit = { + id: isMultiple ? ui.listSelection : data.id, + rightslevel: rightsLevel + }; + pandora.api.edit(edit, function(result) { + that.triggerEvent('change', Ox.extend({}, 'rightslevel', rightsLevel)); + }); + } + } + }) + .appendTo($rightsLevel); + } else { + $rightsLevelElement + .css({ + marginBottom: '4px' + }) + .appendTo($rightsLevel); + } + $capabilities = $('
').appendTo($rightsLevel); + !isMixed.rightslevel && renderCapabilities(data.rightslevel); + + + function getRightsLevelElement(rightsLevel) { + return Ox.Theme.formatColorLevel(rightsLevel, rightsLevels) + } + + function renderCapabilities(rightsLevel) { + var capabilities = [].concat( + canEdit ? [{name: 'canSeeItem', symbol: 'Find'}] : [], + [ + {name: 'canPlayClips', symbol: 'PlayInToOut'}, + {name: 'canPlayVideo', symbol: 'Play'}, + {name: 'canDownloadVideo', symbol: 'Download'} + ] + ), + userLevels = canEdit ? pandora.site.userLevels : [pandora.user.level]; + $capabilities.empty(); + userLevels.forEach(function(userLevel, i) { + var $element, + $line = $('
') + .css({ + height: '16px', + marginBottom: '4px' + }) + .appendTo($capabilities); + if (canEdit) { + $element = Ox.Theme.formatColorLevel(i, userLevels.map(function(userLevel) { + return Ox.toTitleCase(userLevel); + }), [0, 240]); + Ox.Label({ + textAlign: 'center', + title: Ox._(Ox.toTitleCase(userLevel)), + width: 60 + }) + .addClass('OxColor OxColorGradient') + .css({ + float: 'left', + height: '12px', + paddingTop: '2px', + background: $element.css('background'), + fontSize: '8px', + color: $element.css('color') + }) + .data({OxColor: $element.data('OxColor')}) + .appendTo($line); + } + capabilities.forEach(function(capability) { + var hasCapability = pandora.hasCapability(capability.name, userLevel) >= rightsLevel, + $element = Ox.Theme.formatColorLevel(hasCapability, ['', '']); + Ox.Button({ + tooltip: Ox._('{0} ' + + (hasCapability ? 'can' : 'can\'t') + ' ' + + Ox.toSlashes(capability.name) + .split('/').slice(1).join(' ') + .toLowerCase() + .replace('see item', 'see the item') + .replace('play video', 'play the full video') + .replace('download video', 'download the video'), + [canEdit ? Ox._(Ox.toTitleCase(userLevel)) : Ox._('You')]), + title: capability.symbol, + type: 'image' + }) + .addClass('OxColor OxColorGradient') + .css({background: $element.css('background')}) + .css('margin' + (canEdit ? 'Left' : 'Right'), '4px') + .data({OxColor: $element.data('OxColor')}) + .appendTo($line); + }); + if (!canEdit) { + Ox.Button({ + title: Ox._('Help'), + tooltip: Ox._('About Rights'), + type: 'image' + }) + .css({marginLeft: '52px'}) + .bindEvent({ + click: function() { + pandora.UI.set({page: 'rights'}); + } + }) + .appendTo($line); + } + }); + } + +} + From 638dfc8bb38402e973e4df220915e8aebc99f9a6 Mon Sep 17 00:00:00 2001 From: j Date: Tue, 21 Jul 2020 15:26:55 +0200 Subject: [PATCH 06/46] get all ids --- static/js/editDialog.js | 3 ++- static/js/editDocumentsDialog.js | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/static/js/editDialog.js b/static/js/editDialog.js index 458cb987..8e646d97 100644 --- a/static/js/editDialog.js +++ b/static/js/editDialog.js @@ -84,7 +84,8 @@ pandora.ui.editDialog = function() { } ], operator: '&' - } + }, + range: [0, ids.length] }, function(result) { var data = {}, isMixed = {}, diff --git a/static/js/editDocumentsDialog.js b/static/js/editDocumentsDialog.js index c5cdec12..97b329ff 100644 --- a/static/js/editDocumentsDialog.js +++ b/static/js/editDocumentsDialog.js @@ -83,7 +83,8 @@ pandora.ui.editDocumentsDialog = function() { } ], operator: '&' - } + }, + range: [0, ids.length] }, function(result) { var data = {}, isMixed = {}, From 80597790f91d6ab831c0ac9b7467cc732c77a895 Mon Sep 17 00:00:00 2001 From: j Date: Tue, 18 Aug 2020 10:17:14 +0200 Subject: [PATCH 07/46] no title placeholder --- static/js/documentInfoView.js | 1 + 1 file changed, 1 insertion(+) diff --git a/static/js/documentInfoView.js b/static/js/documentInfoView.js index 4e40a84e..1ba47cf5 100644 --- a/static/js/documentInfoView.js +++ b/static/js/documentInfoView.js @@ -218,6 +218,7 @@ pandora.ui.documentInfoView = function(data, isMixed) { .append( Ox.EditableContent({ editable: canEdit, + placeholder: formatLight(Ox._('No Title')), tooltip: canEdit ? pandora.getEditTooltip() : '', value: data.title || '' }) From 800725d54c8896100a8793357664c7f7ccb9f4ba Mon Sep 17 00:00:00 2001 From: j Date: Mon, 24 Aug 2020 01:05:42 +0200 Subject: [PATCH 08/46] add more collumn state --- static/js/collection.js | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/static/js/collection.js b/static/js/collection.js index 51017b50..a6f6f70a 100644 --- a/static/js/collection.js +++ b/static/js/collection.js @@ -13,20 +13,6 @@ pandora.ui.collection = function() { if (view == 'list') { that = Ox.TableList({ - draggable: true, - keys: keys, - items: function(data, callback) { - pandora.api.findDocuments(Ox.extend(data, { - query: ui.findDocuments - }), callback); - return Ox.clone(data, true); - }, - selected: ui.collectionSelection, - sort: ui.collectionSort.concat([ - {key: 'extension', operator: '+'}, - {key: 'title', operator: '+'} - ]), - unique: 'id', columns: pandora.site.documentSortKeys.filter(function(key) { return !key.capability || pandora.hasCapability(key.capability); @@ -54,7 +40,24 @@ pandora.ui.collection = function() { }; }), columnsVisible: true, + columnsMovable: true, + columnsRemovable: true, + columnsResizable: true, + columnsVisible: true, + draggable: true, + items: function(data, callback) { + pandora.api.findDocuments(Ox.extend(data, { + query: ui.findDocuments + }), callback); + return Ox.clone(data, true); + }, scrollbarVisible: true, + selected: ui.collectionSelection, + sort: ui.collectionSort.concat([ + {key: 'extension', operator: '+'}, + {key: 'title', operator: '+'} + ]), + unique: 'id', }) .bindEvent({ columnchange: function(data) { From ffb512a3046945d6af3a2e2f0d9f5bbdfc8d5a8c Mon Sep 17 00:00:00 2001 From: j Date: Sat, 29 Aug 2020 23:32:57 +0200 Subject: [PATCH 09/46] add oshash to changelog, keep upload filename, show direct upload instance --- pandora/archive/models.py | 11 ++++++++++- pandora/archive/views.py | 10 ++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/pandora/archive/models.py b/pandora/archive/models.py index 05882c40..620494e9 100644 --- a/pandora/archive/models.py +++ b/pandora/archive/models.py @@ -336,7 +336,9 @@ class File(models.Model): def done_cb(): if done: - self.info.update(ox.avinfo(self.data.path)) + info = ox.avinfo(self.data.path) + del info['path'] + self.info.update(info) self.parse_info() # reject invalid uploads if self.info.get('oshash') != self.oshash: @@ -486,6 +488,13 @@ class File(models.Model): data['instances'] = [] if 'path' in data: data['path'] = os.path.basename(data['path']) + if 'instances' in data and 'filename' in self.info and self.data: + data['instances'].append({ + 'ignore': False, + 'path': self.info['filename'], + 'user': f.item.user.username, + 'volume': 'Direct Upload' + }) return data def all_paths(self): diff --git a/pandora/archive/views.py b/pandora/archive/views.py index 848bfc7e..fd026c1f 100644 --- a/pandora/archive/views.py +++ b/pandora/archive/views.py @@ -195,7 +195,9 @@ def addMedia(request, data): response['data']['item'] = f.item.public_id response['data']['itemUrl'] = request.build_absolute_uri('/%s' % f.item.public_id) if not f.available: - add_changelog(request, data, f.item.public_id) + changelog_data = data.copy() + changelog_data['oshash'] = oshash + add_changelog(request, changelog_data, f.item.public_id) else: if 'item' in data: i = Item.objects.get(public_id=data['item']) @@ -220,11 +222,15 @@ def addMedia(request, data): if 'info' in data and data['info'] and isinstance(data['info'], dict): f.info = data['info'] f.info['extension'] = extension + if 'filename' in data: + f.info['filename'] = data['filename'] f.parse_info() f.save() response['data']['item'] = i.public_id response['data']['itemUrl'] = request.build_absolute_uri('/%s' % i.public_id) - add_changelog(request, data, i.public_id) + changelog_data = data.copy() + changelog_data['oshash'] = oshash + add_changelog(request, changelog_data, i.public_id) return render_to_json_response(response) actions.register(addMedia, cache=False) From d795e403441e198eadae744c88bcc95501b1f733 Mon Sep 17 00:00:00 2001 From: j Date: Sat, 29 Aug 2020 23:37:59 +0200 Subject: [PATCH 10/46] fix username --- pandora/archive/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandora/archive/models.py b/pandora/archive/models.py index 620494e9..4f9ead25 100644 --- a/pandora/archive/models.py +++ b/pandora/archive/models.py @@ -492,7 +492,7 @@ class File(models.Model): data['instances'].append({ 'ignore': False, 'path': self.info['filename'], - 'user': f.item.user.username, + 'user': self.item.user.username if self.item and self.item.user else 'system', 'volume': 'Direct Upload' }) return data From 2d8a3f24dcbd78878cd503d35885d69c41f56d37 Mon Sep 17 00:00:00 2001 From: j Date: Sat, 29 Aug 2020 23:54:19 +0200 Subject: [PATCH 11/46] can see > direct upload --- pandora/archive/models.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pandora/archive/models.py b/pandora/archive/models.py index 4f9ead25..56080815 100644 --- a/pandora/archive/models.py +++ b/pandora/archive/models.py @@ -483,11 +483,6 @@ class File(models.Model): user.is_staff or \ self.item.user == user or \ self.item.groups.filter(id__in=user.groups.all()).count() > 0 - if not can_see_media: - if 'instances' in data: - data['instances'] = [] - if 'path' in data: - data['path'] = os.path.basename(data['path']) if 'instances' in data and 'filename' in self.info and self.data: data['instances'].append({ 'ignore': False, @@ -495,6 +490,11 @@ class File(models.Model): 'user': self.item.user.username if self.item and self.item.user else 'system', 'volume': 'Direct Upload' }) + if not can_see_media: + if 'instances' in data: + data['instances'] = [] + if 'path' in data: + data['path'] = os.path.basename(data['path']) return data def all_paths(self): From ca7741f92c65a98db1aff54e01f66555215efc2e Mon Sep 17 00:00:00 2001 From: j Date: Fri, 11 Sep 2020 13:55:01 +0200 Subject: [PATCH 12/46] run bulk update as task --- pandora/document/tasks.py | 19 ++++++++++++++++++- pandora/document/views.py | 11 ++++++----- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/pandora/document/tasks.py b/pandora/document/tasks.py index f08983bb..b75fbefe 100644 --- a/pandora/document/tasks.py +++ b/pandora/document/tasks.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +import ox from celery.task import task @task(queue="encoding") @@ -6,3 +6,20 @@ def extract_fulltext(id): from . import models d = models.Document.objects.get(id=id) d.update_fulltext() + + +@task(queue='default') +def builk_edit(data, username): + from django.db import transaction + from . import models + from item.models import Item + user = models.User.objects.get(username=username) + item = 'item' in data and Item.objects.get(public_id=data['item']) or None + documents = models.Document.objects.filter(pk__in=map(ox.fromAZ, data['id'])) + for document in documents: + if document.editable(user, item): + with transaction.atomic(): + document.refresh_from_db() + document.edit(data, user, item) + document.save() + return {} diff --git a/pandora/document/views.py b/pandora/document/views.py index 84279ac9..f22bed81 100644 --- a/pandora/document/views.py +++ b/pandora/document/views.py @@ -23,6 +23,7 @@ from archive.chunk import process_chunk from changelog.models import add_changelog from . import models +from . import tasks def get_document_or_404_json(request, id): response = {'status': {'code': 404, @@ -131,13 +132,13 @@ def editDocument(request, data): item = 'item' in data and Item.objects.get(public_id=data['item']) or None if data['id']: if isinstance(data['id'], list): - documents = models.Document.objects.filter(pk__in=map(ox.fromAZ, data['id'])) + add_changelog(request, data) + t = tasks.builk_edit.delay(data, request.user.username) + response['data']['taskId'] = t.task_id else: - documents = [models.Document.get(data['id'])] - for document in documents: + document = models.Document.get(data['id']) if document.editable(request.user, item): - if document == documents[0]: - add_changelog(request, data) + add_changelog(request, data) document.edit(data, request.user, item) document.save() response['data'] = document.json(user=request.user, item=item) From 40edf9dd4af47ae249648b343555c2b8e46e8d17 Mon Sep 17 00:00:00 2001 From: j Date: Fri, 11 Sep 2020 13:57:54 +0200 Subject: [PATCH 13/46] typo --- pandora/document/tasks.py | 2 +- pandora/document/views.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pandora/document/tasks.py b/pandora/document/tasks.py index b75fbefe..7bede1a9 100644 --- a/pandora/document/tasks.py +++ b/pandora/document/tasks.py @@ -9,7 +9,7 @@ def extract_fulltext(id): @task(queue='default') -def builk_edit(data, username): +def bulk_edit(data, username): from django.db import transaction from . import models from item.models import Item diff --git a/pandora/document/views.py b/pandora/document/views.py index f22bed81..5fc47466 100644 --- a/pandora/document/views.py +++ b/pandora/document/views.py @@ -133,7 +133,7 @@ def editDocument(request, data): if data['id']: if isinstance(data['id'], list): add_changelog(request, data) - t = tasks.builk_edit.delay(data, request.user.username) + t = tasks.bulk_edit.delay(data, request.user.username) response['data']['taskId'] = t.task_id else: document = models.Document.get(data['id']) From 1c0462393c19eb66828a6a5082fed1a9b019f09e Mon Sep 17 00:00:00 2001 From: j Date: Fri, 11 Sep 2020 14:07:08 +0200 Subject: [PATCH 14/46] add to current item --- static/js/addFilesDialog.js | 34 +++++++++++++--------------------- static/js/addItemDialog.js | 14 ++++++++++---- static/js/mediaExistsDialog.js | 14 ++++++++++---- 3 files changed, 33 insertions(+), 29 deletions(-) diff --git a/static/js/addFilesDialog.js b/static/js/addFilesDialog.js index 2a4e023c..0f75c07e 100644 --- a/static/js/addFilesDialog.js +++ b/static/js/addFilesDialog.js @@ -106,7 +106,7 @@ pandora.ui.addFilesDialog = function(options) { }); var selectItems = []; - if (!pandora.site.itemRequiresVideo && pandora.user.ui.item) { + if (pandora.user.ui.item && options.editable) { selectItems.push({ id: 'add', title: Ox._( @@ -114,31 +114,23 @@ pandora.ui.addFilesDialog = function(options) { [pandora.site.itemName.singular.toLowerCase()] ) }); + } + if (options.items.length > 1) { selectItems.push({ - id: 'one', + id: 'multiple', title: Ox._( - options.items.length > 1 ? 'Create new {0} with multiple parts' : 'Create new {0}', - [pandora.site.itemName.singular.toLowerCase()] - ) - }); - } else { - if (options.items.length > 1) { - selectItems.push({ - id: 'multiple', - title: Ox._( - 'Create multiple {0}', - [pandora.site.itemName.plural.toLowerCase()] - ) - }); - } - selectItems.push({ - id: 'one', - title: Ox._( - 'Create one {0} with multiple parts', - [pandora.site.itemName.singular.toLowerCase()] + 'Create multiple {0}', + [pandora.site.itemName.plural.toLowerCase()] ) }); } + selectItems.push({ + id: 'one', + title: Ox._( + options.items.length > 1 ? 'Create new {0} with multiple parts' : 'Create new {0}', + [pandora.site.itemName.singular.toLowerCase()] + ) + }); var $select = Ox.Select({ items: selectItems, width: 256 diff --git a/static/js/addItemDialog.js b/static/js/addItemDialog.js index d41a9ed3..ab1213f2 100644 --- a/static/js/addItemDialog.js +++ b/static/js/addItemDialog.js @@ -270,10 +270,16 @@ pandora.ui.addItemDialog = function(options) { } else { $screen.stop(); that.close(); - pandora.ui.addFilesDialog({ - action: selected, - items: items - }).open(); + (pandora.user.ui.item ? pandora.api.get : Ox.noop)({ + id: pandora.user.ui.item, + keys: ['editable'] + }, function(result) { + pandora.ui.addFilesDialog({ + action: selected, + items: items, + editable: pandora.user.ui.item && result.data.editable + }).open(); + }) } }) } diff --git a/static/js/mediaExistsDialog.js b/static/js/mediaExistsDialog.js index a6bcad98..4388b652 100644 --- a/static/js/mediaExistsDialog.js +++ b/static/js/mediaExistsDialog.js @@ -58,10 +58,16 @@ pandora.ui.mediaExistsDialog = function(options) { return existing.indexOf(item.oshash) == -1; }); that.close(); - pandora.ui.addFilesDialog({ - action: options.action, - items: items - }).open(); + (pandora.user.ui.item ? pandora.api.get : Ox.noop)({ + id: pandora.user.ui.item, + keys: ['editable'] + }, function(result) { + pandora.ui.addFilesDialog({ + action: options.action, + items: items, + editable: pandora.user.ui.item && result.data.editable + }).open(); + }) } }) ]; From 940632369a2a9ffbded282b22162a68d62019213 Mon Sep 17 00:00:00 2001 From: j Date: Fri, 11 Sep 2020 14:16:43 +0200 Subject: [PATCH 15/46] list view, load keys --- static/js/collection.js | 1 + 1 file changed, 1 insertion(+) diff --git a/static/js/collection.js b/static/js/collection.js index a6f6f70a..eb397602 100644 --- a/static/js/collection.js +++ b/static/js/collection.js @@ -51,6 +51,7 @@ pandora.ui.collection = function() { }), callback); return Ox.clone(data, true); }, + keys: keys, scrollbarVisible: true, selected: ui.collectionSelection, sort: ui.collectionSort.concat([ From 5024a2ba0c0ebeefb8103ef23907cc3e88b80757 Mon Sep 17 00:00:00 2001 From: j Date: Fri, 11 Sep 2020 14:31:50 +0200 Subject: [PATCH 16/46] display lists in collection list view --- static/js/collection.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/static/js/collection.js b/static/js/collection.js index eb397602..5b188f7f 100644 --- a/static/js/collection.js +++ b/static/js/collection.js @@ -26,7 +26,11 @@ pandora.ui.collection = function() { defaultWidth: key.columnWidth, format: (function() { return function(value, data) { - return pandora.formatDocumentKey(key, data); + var value = pandora.formatDocumentKey(key, data); + if (Ox.isArray(value)) { + value = value.join(', '); + } + return value; } })(), id: key.id, From 945ac98dad8575eb7c014fd4753dcc9dc493df9b Mon Sep 17 00:00:00 2001 From: j Date: Sun, 13 Sep 2020 11:06:22 +0200 Subject: [PATCH 17/46] fix info in item documents panel --- static/js/documentsPanel.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/static/js/documentsPanel.js b/static/js/documentsPanel.js index e1663aba..2337352b 100644 --- a/static/js/documentsPanel.js +++ b/static/js/documentsPanel.js @@ -810,9 +810,8 @@ pandora.ui.documentsPanel = function(options) { item: function(data, sort, size) { var sortKey = sort[0].key, infoKey = sortKey == 'title' ? 'extension' : sortKey, - info = ( - Ox.getObjectById(pandora.site.documentKeys, infoKey).format || Ox.identity - )(data[infoKey]), + key = Ox.getObjectById(pandora.site.documentKeys, infoKey), + info = pandora.formatDocumentKey(key, data, size), size = size || 128; return { height: Math.round(data.ratio > 1 ? size / data.ratio : size), From d00cf08638e666a111138bb051ac3021757bb419 Mon Sep 17 00:00:00 2001 From: j Date: Sun, 13 Sep 2020 16:41:34 +0200 Subject: [PATCH 18/46] move buld item metadata edits to tasks too --- pandora/item/tasks.py | 19 +++++++++++++++++++ pandora/item/views.py | 27 ++++++++++++++------------- static/js/infoViewUtils.js | 3 ++- 3 files changed, 35 insertions(+), 14 deletions(-) diff --git a/pandora/item/tasks.py b/pandora/item/tasks.py index 27511acc..a7c0c68a 100644 --- a/pandora/item/tasks.py +++ b/pandora/item/tasks.py @@ -5,6 +5,7 @@ from urllib.parse import quote import gzip import os import random +import logging from celery.task import task, periodic_task from django.conf import settings @@ -16,6 +17,9 @@ from app.utils import limit_rate from taskqueue.models import Task +logger = logging.getLogger(__name__) + + @periodic_task(run_every=timedelta(days=1), queue='encoding') def cronjob(**kwargs): if limit_rate('item.tasks.cronjob', 8 * 60 * 60): @@ -350,3 +354,18 @@ def update_sitemap(base_url): f.write(data) with gzip.open(sitemap, 'wb') as f: f.write(data) + + +@task(queue='default') +def bulk_edit(data, username): + from django.db import transaction + from . import models + from .views import edit_item + user = models.User.objects.get(username=username) + items = models.Item.objects.filter(public_id__in=data['id']) + for item in items: + if item.editable(user): + with transaction.atomic(): + item.refresh_from_db() + response = edit_item(user, item, data) + return {} diff --git a/pandora/item/views.py b/pandora/item/views.py index 23fadc29..7ad11572 100644 --- a/pandora/item/views.py +++ b/pandora/item/views.py @@ -533,17 +533,18 @@ def get(request, data): return render_to_json_response(response) actions.register(get) -def edit_item(request, item, data): +def edit_item(user, item, data): + data = data.copy() update_clips = False response = json_response(status=200, text='ok') if 'rightslevel' in data: - if request.user.profile.capability('canEditRightsLevel'): + if user.profile.capability('canEditRightsLevel'): item.level = int(data['rightslevel']) else: response = json_response(status=403, text='permission denied') del data['rightslevel'] if 'user' in data: - if request.user.profile.get_level() in ('admin', 'staff') and \ + if user.profile.get_level() in ('admin', 'staff') and \ models.User.objects.filter(username=data['user']).exists(): new_user = models.User.objects.get(username=data['user']) if new_user != item.user: @@ -551,10 +552,10 @@ def edit_item(request, item, data): update_clips = True del data['user'] if 'groups' in data: - if not request.user.profile.capability('canManageUsers'): + if not user.profile.capability('canManageUsers'): # Users wihtout canManageUsers can only add/remove groups they are not in groups = set([g.name for g in item.groups.all()]) - user_groups = set([g.name for g in request.user.groups.all()]) + user_groups = set([g.name for g in user.groups.all()]) other_groups = list(groups - user_groups) data['groups'] = [g for g in data['groups'] if g in user_groups] + other_groups r = item.edit(data) @@ -597,7 +598,7 @@ def add(request, data): i.make_poster() del data['title'] if data: - response = edit_item(request, item, data) + response = edit_item(request.user, item, data) response['data'] = item.json() add_changelog(request, request_data, item.public_id) return render_to_json_response(response) @@ -619,16 +620,16 @@ def edit(request, data): see: add, find, get, lookup, remove, upload ''' if isinstance(data['id'], list): - items = models.Item.objects.filter(public_id__in=data['id']) + add_changelog(request, data) + t = tasks.bulk_edit.delay(data, request.user.username) + response = json_response(status=200, text='ok') + response['data']['taskId'] = t.task_id else: - items = [get_object_or_404_json(models.Item, public_id=data['id'])] - for item in items: + item = get_object_or_404_json(models.Item, public_id=data['id']) if item.editable(request.user): - request_data = data.copy() - response = edit_item(request, item, data) + add_changelog(request, data) + response = edit_item(request.user, item, data) response['data'] = item.json() - if item == items[0]: - add_changelog(request, request_data) else: response = json_response(status=403, text='permission denied') return render_to_json_response(response) diff --git a/static/js/infoViewUtils.js b/static/js/infoViewUtils.js index 9d9de29a..6c415c1b 100644 --- a/static/js/infoViewUtils.js +++ b/static/js/infoViewUtils.js @@ -15,7 +15,8 @@ pandora.cleanupDate = function(value) { }; pandora.renderRightsLevel = function(that, $rightsLevel, data, isMixed, isMultiple, canEdit) { - var rightsLevels = pandora.site.rightsLevels.map(function(rightsLevel) { + var ui = pandora.user.ui, + rightsLevels = pandora.site.rightsLevels.map(function(rightsLevel) { return rightsLevel.name; }).concat(isMultiple ? ['Mixed'] : []), rightsLevel = isMixed.rightslevel ? rightsLevels.length - 1 : data.rightslevel, From 2779d8d099c2af4e3ece27c7c713ef5163078c3a Mon Sep 17 00:00:00 2001 From: j Date: Sat, 19 Sep 2020 14:49:44 +0200 Subject: [PATCH 19/46] change password in case user already existed --- vm/pandora_install.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/vm/pandora_install.sh b/vm/pandora_install.sh index 6f9d66d6..dba5cb98 100755 --- a/vm/pandora_install.sh +++ b/vm/pandora_install.sh @@ -118,6 +118,7 @@ fi if [ "$RABBITMQ" == "local" ]; then RABBITPWD=$(pwgen -n 16 -1) rabbitmqctl add_user pandora $RABBITPWD + rabbitmqctl change_password pandora $RABBITPWD rabbitmqctl add_vhost /pandora rabbitmqctl set_permissions -p /pandora pandora ".*" ".*" ".*" CELERY_BROKER_URL="amqp://pandora:$RABBITPWD@localhost:5672//pandora" From 5bbb6ca195f9abcfeeafd309fa163630a4043f4e Mon Sep 17 00:00:00 2001 From: j Date: Mon, 21 Sep 2020 15:03:27 +0200 Subject: [PATCH 20/46] support ipv6 --- vm/pandora_install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/pandora_install.sh b/vm/pandora_install.sh index dba5cb98..62fac308 100755 --- a/vm/pandora_install.sh +++ b/vm/pandora_install.sh @@ -186,7 +186,7 @@ fi # if pandora is running inside a container, expose backend at port 2620 if [ "$LXC" == "yes" ]; then - sed -i s/127.0.0.1/0.0.0.0/g /srv/pandora/pandora/gunicorn_config.py + sed -i "s/127.0.0.1/[::]/g" /srv/pandora/pandora/gunicorn_config.py echo "WEBSOCKET_ADDRESS = \"0.0.0.0\"" >> /srv/pandora/pandora/local_settings.py fi /srv/pandora/ctl start From 2dfe0f3ff2cf3b36a3826ff0de14ede0594a5af5 Mon Sep 17 00:00:00 2001 From: j Date: Tue, 22 Sep 2020 12:49:20 +0200 Subject: [PATCH 21/46] queue [add|edit]Annotation calls addAnnotation was called multiple times creating multiple annotations editAnnotation calls might overwrite later calls depending on response time. --- static/js/editor.js | 68 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 52 insertions(+), 16 deletions(-) diff --git a/static/js/editor.js b/static/js/editor.js index 3d53d89a..16da2d01 100644 --- a/static/js/editor.js +++ b/static/js/editor.js @@ -205,9 +205,9 @@ pandora.ui.editor = function(data) { }).open(); }, editannotation: function(data) { - Ox.Log('', 'editAnnotation', data); + Ox.Log('', 'editAnnotation', data.id, data); function callback(result) { - Ox.Log('', 'editAnnotation result', result); + Ox.Log('', 'editAnnotation result', result.data.id, result); if (!Ox.isEmpty(result.data)) { result.data.date = Ox.formatDate( result.data.modified.slice(0, 10), '%B %e, %Y' @@ -222,21 +222,56 @@ pandora.ui.editor = function(data) { pandora.UI.set('videoPoints.' + ui.item + '.annotation', result.data.id.split('/')[1] || ''); Ox.Request.clearCache(); }; + var edit = { + 'in': data['in'], + out: data.out, + value: data.value + } if (data.id[0] == '_') { - pandora.api.addAnnotation({ - 'in': data['in'], - item: ui.item, - layer: data.layer, - out: data.out, - value: data.value - }, callback); + edit.item = ui.item; + edit.layer = data.layer; + + if (queue[data.id]) { + queue[data.id].push(edit); + } else { + queue[data.id] = []; + pandora.api.addAnnotation(edit, function(result) { + callback(result); + var id = data.id, + pending = queue[id]; + delete queue[id]; + pending.length && Ox.serialForEach(pending, function(edit, index, array, callback) { + edit.id = id + Ox.Log('', 'process pending editAnnotation request', id, edit); + pandora.api.editAnnotation(edit, function(result) { + callback(); + }) + }, function() { + Ox.Request.clearCache(); + }); + }); + } } else { - pandora.api.editAnnotation({ - id: data.id, - 'in': data['in'], - out: data.out, - value: data.value - }, callback); + edit.id = data.id; + if (queue[data.id]) { + queue[data.id].push(edit); + } else { + queue[data.id] = []; + pandora.api.editAnnotation(edit, function(result) { + callback(result); + var pending = queue[edit.id]; + delete queue[edit.id]; + pending.length && Ox.serialForEach(pending, function(edit, index, array, cb) { + Ox.Log('', 'process pending editAnnotation request', edit.id, edit); + pandora.api.editAnnotation(edit, function(result) { + callback(result); + cb(); + }) + }, function() { + Ox.Request.clearCache(); + }); + }); + } } }, embedselection: function() { @@ -387,7 +422,8 @@ pandora.ui.editor = function(data) { pandora_videotimeline: function(data) { that.options({timeline: data.value}); } - }); + }), + queue = []; pandora._dontSelectResult = false; From 09f9580e1eeac9b93c836ba3f5f7bdde74f17ce1 Mon Sep 17 00:00:00 2001 From: j Date: Tue, 22 Sep 2020 14:49:44 +0200 Subject: [PATCH 22/46] use new id after create --- static/js/editor.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/static/js/editor.js b/static/js/editor.js index 16da2d01..8cd460a5 100644 --- a/static/js/editor.js +++ b/static/js/editor.js @@ -237,7 +237,7 @@ pandora.ui.editor = function(data) { queue[data.id] = []; pandora.api.addAnnotation(edit, function(result) { callback(result); - var id = data.id, + var id = result.data.id, pending = queue[id]; delete queue[id]; pending.length && Ox.serialForEach(pending, function(edit, index, array, callback) { From 03e85c6eeff64a2de37ce7fffe87c8697999709d Mon Sep 17 00:00:00 2001 From: j Date: Fri, 25 Sep 2020 12:49:26 +0200 Subject: [PATCH 23/46] double check --- static/js/editor.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/static/js/editor.js b/static/js/editor.js index 8cd460a5..28d3b5ba 100644 --- a/static/js/editor.js +++ b/static/js/editor.js @@ -240,7 +240,7 @@ pandora.ui.editor = function(data) { var id = result.data.id, pending = queue[id]; delete queue[id]; - pending.length && Ox.serialForEach(pending, function(edit, index, array, callback) { + pending && pending.length && Ox.serialForEach(pending, function(edit, index, array, callback) { edit.id = id Ox.Log('', 'process pending editAnnotation request', id, edit); pandora.api.editAnnotation(edit, function(result) { @@ -261,7 +261,7 @@ pandora.ui.editor = function(data) { callback(result); var pending = queue[edit.id]; delete queue[edit.id]; - pending.length && Ox.serialForEach(pending, function(edit, index, array, cb) { + pending && pending.length && Ox.serialForEach(pending, function(edit, index, array, cb) { Ox.Log('', 'process pending editAnnotation request', edit.id, edit); pandora.api.editAnnotation(edit, function(result) { callback(result); From fdac35c00db489d6e8c683a2ecfb790e226845e1 Mon Sep 17 00:00:00 2001 From: j Date: Sun, 27 Sep 2020 13:03:45 +0200 Subject: [PATCH 24/46] update django, sync docker install --- docker/base/install.sh | 6 +++++- docker/publish.sh | 12 ++++++++++++ requirements.txt | 2 +- vm/pandora_install.sh | 1 + 4 files changed, 19 insertions(+), 2 deletions(-) create mode 100644 docker/publish.sh diff --git a/docker/base/install.sh b/docker/base/install.sh index 5c209f07..1bfa1bda 100755 --- a/docker/base/install.sh +++ b/docker/base/install.sh @@ -30,6 +30,7 @@ apt-get update -qq apt-get install -y \ netcat-openbsd \ sudo \ + iproute2 \ vim \ wget \ pwgen \ @@ -47,17 +48,20 @@ apt-get install -y \ python3-cssselect \ python3-html5lib \ python3-ox \ + python3-elasticsearch \ oxframe \ ffmpeg \ mkvtoolnix \ gpac \ imagemagick \ poppler-utils \ - youtube-dl \ ipython3 \ + tesseract-ocr \ + tesseract-ocr-eng \ postfix \ postgresql-client +apt-get install -y --no-install-recommends youtube-dl rtmpdump apt-get clean rm -f /install.sh diff --git a/docker/publish.sh b/docker/publish.sh new file mode 100644 index 00000000..f4ef0193 --- /dev/null +++ b/docker/publish.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# push new version of pan.do/ra to docker hub +set -e + +cd /tmp +git clone https://code.0x2620.org/0x2620/pandora +cd pandora +./docker/build.sh + +docker push 0x2620/pandora-base:latest +docker push 0x2620/pandora-nginx:latest +docker push 0x2620/pandora:latest diff --git a/requirements.txt b/requirements.txt index c3abad48..f45609b4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Django==3.0.6 +Django==3.0.10 simplejson chardet celery>4.3.0 diff --git a/vm/pandora_install.sh b/vm/pandora_install.sh index 62fac308..75adf3d0 100755 --- a/vm/pandora_install.sh +++ b/vm/pandora_install.sh @@ -89,6 +89,7 @@ apt-get install -y \ python3-pyinotify \ python3-simplejson \ python3-lxml \ + python3-cssselect \ python3-html5lib \ python3-ox \ python3-elasticsearch \ From 213adcaaaabd69e28a3357a72ef814ce36140fe0 Mon Sep 17 00:00:00 2001 From: j Date: Sun, 27 Sep 2020 17:19:02 +0200 Subject: [PATCH 25/46] update requirements --- requirements.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index f45609b4..e2f2c2eb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1,15 @@ Django==3.0.10 simplejson chardet -celery>4.3.0 +celery<5.0,>4.3 django-celery-results django-extensions==2.2.9 gunicorn==20.0.4 html5lib -requests==2.23.0 +requests<3.0.0,>=2.24.0 +urllib3<2.0.0,>=1.25.2 tornado<5 -geoip2==3.0.0 +geoip2==4.1.0 youtube-dl>=2020.5.8 python-memcached elasticsearch From 7ab789f80a48cd1dcfdeeaf04c1880e61b7f5945 Mon Sep 17 00:00:00 2001 From: j Date: Sun, 27 Sep 2020 17:53:37 +0200 Subject: [PATCH 26/46] docker fixes --- ctl | 7 ++-- docker/base/install.sh | 1 + docker/entrypoint.sh | 49 +++++++++++----------- docker/install.sh | 10 ++--- docker/setup-docker-compose.sh | 2 +- docker/wait-for-file | 2 +- pandora/app/management/commands/init_db.py | 2 + pandora/user/utils.py | 2 +- 8 files changed, 37 insertions(+), 38 deletions(-) diff --git a/ctl b/ctl index 28c74e12..b78183ba 100755 --- a/ctl +++ b/ctl @@ -17,7 +17,7 @@ if [ "$action" = "init" ]; then SUDO="" PANDORA_USER=`ls -l update.py | cut -f3 -d" "` if [ `whoami` != $PANDORA_USER ]; then - SUDO="sudo -H -u $PANDORA_USER" + SUDO="sudo -E -H -u $PANDORA_USER" fi $SUDO python3 -m venv --system-site-packages . branch=`cat .git/HEAD | sed 's@/@\n@g' | tail -n1` @@ -62,11 +62,10 @@ if [ ! -z $cmd ]; then SUDO="" PANDORA_USER=`ls -l update.py | cut -f3 -d" "` if [ `whoami` != $PANDORA_USER ]; then - SUDO="sudo -H -u $PANDORA_USER" + SUDO="sudo -E -H -u $PANDORA_USER" fi shift - $SUDO "$BASE/$cmd" $@ - exit $? + exec $SUDO "$BASE/$cmd" $@ fi if [ `whoami` != 'root' ]; then diff --git a/docker/base/install.sh b/docker/base/install.sh index 1bfa1bda..2d016eba 100755 --- a/docker/base/install.sh +++ b/docker/base/install.sh @@ -30,6 +30,7 @@ apt-get update -qq apt-get install -y \ netcat-openbsd \ sudo \ + rsync \ iproute2 \ vim \ wget \ diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index cd78c6f4..123883f8 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -8,6 +8,8 @@ export LANG=en_US.UTF-8 mkdir -p /run/pandora chown -R ${user}.${user} /run/pandora +update="/usr/bin/sudo -u $user -E -H /srv/pandora/update.py" + # pan.do/ra services if [ "$action" = "pandora" ]; then if [ ! -e /srv/pandora/initialized ]; then @@ -26,12 +28,12 @@ if [ "$action" = "pandora" ]; then /overlay/install.py echo "Initializing database..." - echo "CREATE EXTENSION pg_trgm;" | /srv/pandora/pandora/manage.py dbshell + echo "CREATE EXTENSION pg_trgm;" | /srv/pandora/pandora/manage.py dbshell || true /srv/pandora/pandora/manage.py init_db - /srv/pandora/update.py db + $update db echo "Generating static files..." - /srv/pandora/update.py static chown -R ${user}.${user} /srv/pandora/ + $update static touch /srv/pandora/initialized fi /srv/pandora_base/docker/wait-for db 5432 @@ -44,54 +46,53 @@ if [ "$action" = "encoding" ]; then /srv/pandora_base/docker/wait-for-file /srv/pandora/initialized /srv/pandora_base/docker/wait-for rabbitmq 5672 name=pandora-encoding-$(hostname) + cd /srv/pandora/pandora exec /usr/bin/sudo -u $user -E -H \ - /srv/pandora/bin/python \ - /srv/pandora/pandora/manage.py \ - celery worker \ - -c 1 \ - -Q encoding -n $name \ - -l INFO + /srv/pandora/bin/celery \ + -A app worker \ + -Q encoding -n ${name} \ + --pidfile /run/pandora/encoding.pid \ + --maxtasksperchild 500 \ + -c 1 \ + -l INFO fi if [ "$action" = "tasks" ]; then /srv/pandora_base/docker/wait-for-file /srv/pandora/initialized /srv/pandora_base/docker/wait-for rabbitmq 5672 name=pandora-default-$(hostname) + cd /srv/pandora/pandora exec /usr/bin/sudo -u $user -E -H \ - /srv/pandora/bin/python \ - /srv/pandora/pandora/manage.py \ - celery worker \ - -Q default,celery -n $name \ + /srv/pandora/bin/celery \ + -A app worker \ + -Q default,celery -n ${name} \ + --pidfile /run/pandora/tasks.pid \ --maxtasksperchild 1000 \ -l INFO fi if [ "$action" = "cron" ]; then /srv/pandora_base/docker/wait-for-file /srv/pandora/initialized /srv/pandora_base/docker/wait-for rabbitmq 5672 + cd /srv/pandora/pandora exec /usr/bin/sudo -u $user -E -H \ - /srv/pandora/bin/python \ - /srv/pandora/pandora/manage.py \ - celerybeat -s /run/pandora/celerybeat-schedule \ + /srv/pandora/bin/celery \ + -A app beat \ + -s /run/pandora/celerybeat-schedule \ --pidfile /run/pandora/cron.pid \ -l INFO fi if [ "$action" = "websocketd" ]; then /srv/pandora_base/docker/wait-for-file /srv/pandora/initialized /srv/pandora_base/docker/wait-for rabbitmq 5672 + cd /srv/pandora/pandora exec /usr/bin/sudo -u $user -E -H \ /srv/pandora/bin/python \ /srv/pandora/pandora/manage.py websocketd fi # pan.do/ra management and update -if [ "$action" = "manage.py" ]; then +if [ "$action" = "ctl" ]; then shift - exec /usr/bin/sudo -u $user -E -H \ - /srv/pandora/pandora/manage.py "$@" -fi -if [ "$action" = "update.py" ]; then - shift - exec /usr/bin/sudo -u $user -E -H \ - /srv/pandora/update.py "$@" + exec /srv/pandora/ctl "$@" fi if [ "$action" = "bash" ]; then shift diff --git a/docker/install.sh b/docker/install.sh index e1a87181..081a18af 100755 --- a/docker/install.sh +++ b/docker/install.sh @@ -56,13 +56,9 @@ cp /srv/pandora/docker/entrypoint.sh /entrypoint.sh mv /srv/pandora/ /srv/pandora_base/ mkdir /pandora ln -s /pandora /srv/pandora -cat > /usr/local/bin/update.py << EOF -#!/bin/sh -exec /srv/pandora/update.py \$@ -EOF -cat > /usr/local/bin/manage.py << EOF +cat > /usr/local/bin/pandoractl << EOF #!/bin/sh -exec /srv/pandora/pandora/manage.py \$@ +exec /srv/pandora/ctl \$@ EOF -chmod +x /usr/local/bin/manage.py /usr/local/bin/update.py +chmod +x /usr/local/bin/pandoractl diff --git a/docker/setup-docker-compose.sh b/docker/setup-docker-compose.sh index 7665e7b8..2f48179f 100755 --- a/docker/setup-docker-compose.sh +++ b/docker/setup-docker-compose.sh @@ -26,7 +26,7 @@ and to get started run this: To update pan.do/ra run: - docker-compose run pandora update.py + docker-compose run pandora ctl update EOF touch __init__.py diff --git a/docker/wait-for-file b/docker/wait-for-file index 83b043e4..d4fe19d6 100755 --- a/docker/wait-for-file +++ b/docker/wait-for-file @@ -1,5 +1,5 @@ #!/bin/sh -TIMEOUT=60 +TIMEOUT=180 TARGET="$1" for i in `seq $TIMEOUT` ; do diff --git a/pandora/app/management/commands/init_db.py b/pandora/app/management/commands/init_db.py index 92ac750f..a7957395 100644 --- a/pandora/app/management/commands/init_db.py +++ b/pandora/app/management/commands/init_db.py @@ -11,6 +11,8 @@ def run(cmd): stdout, stderr = p.communicate() if p.returncode != 0: + print('failed to run:', cmd) + print(stdout) print(stderr) sys.exit(1) diff --git a/pandora/user/utils.py b/pandora/user/utils.py index 49fa7884..5328a464 100644 --- a/pandora/user/utils.py +++ b/pandora/user/utils.py @@ -22,7 +22,7 @@ def get_location(ip): else: try: location = g.city(ip) - except GeoIP2Exception: + except: try: location = g.country(s.ip) except: From 63d82ad7e7ee0b4e9e38dd8b36cfd9880208306b Mon Sep 17 00:00:00 2001 From: j Date: Sun, 27 Sep 2020 17:58:35 +0200 Subject: [PATCH 27/46] remove config reloader --- pandora/app/config.py | 57 ------------------- pandora/app/monkey_patch.py | 2 - .../commands/rebuild_documentfind.py | 1 - .../management/commands/sync_documentsort.py | 1 - pandora/item/management/commands/get_frame.py | 1 - .../management/commands/rebuild_filter.py | 1 - .../item/management/commands/rebuild_find.py | 1 - .../management/commands/rebuild_indexes.py | 1 - .../item/management/commands/rebuild_sort.py | 1 - .../item/management/commands/sqlfindindex.py | 1 - .../item/management/commands/sync_itemsort.py | 1 - pandora/manage.py | 2 - pandora/settings.py | 2 - 13 files changed, 72 deletions(-) diff --git a/pandora/app/config.py b/pandora/app/config.py index e950e5e9..631204f1 100644 --- a/pandora/app/config.py +++ b/pandora/app/config.py @@ -24,9 +24,6 @@ User = get_user_model() _win = (sys.platform == "win32") -RUN_RELOADER = True -NOTIFIER = None - def get_version(): git_dir = join(dirname(dirname(dirname(__file__))), '.git') if exists(git_dir): @@ -258,45 +255,6 @@ check the README for further details. pass - -def reloader_thread(): - global NOTIFIER - settings.RELOADER_RUNNING=True - _config_mtime = 0 - try: - import pyinotify - INOTIFY = True - except: - INOTIFY = False - if INOTIFY: - def add_watch(): - name = os.path.realpath(settings.SITE_CONFIG) - wm.add_watch(name, pyinotify.IN_CLOSE_WRITE, reload_config) - - def reload_config(event): - load_config() - add_watch() - - wm = pyinotify.WatchManager() - add_watch() - notifier = pyinotify.Notifier(wm) - NOTIFIER = notifier - notifier.loop() - else: - while RUN_RELOADER: - try: - stat = os.stat(settings.SITE_CONFIG) - mtime = stat.st_mtime - if _win: - mtime -= stat.st_ctime - if mtime > _config_mtime: - load_config() - _config_mtime = mtime - time.sleep(10) - except: - #sys.stderr.write("reloading config failed\n") - pass - def update_static(): oxjs_build = os.path.join(settings.STATIC_ROOT, 'oxjs/tools/build/build.py') if os.path.exists(oxjs_build): @@ -406,18 +364,3 @@ def update_geoip(force=False): else: print('failed to download GeoLite2-City.mmdb') -def init(): - if not settings.RELOADER_RUNNING: - load_config(True) - if settings.RELOAD_CONFIG: - thread.start_new_thread(reloader_thread, ()) - -def shutdown(): - if settings.RELOADER_RUNNING: - RUN_RELOADER = False - settings.RELOADER_RUNNING = False - if NOTIFIER: - NOTIFIER.stop() - - - diff --git a/pandora/app/monkey_patch.py b/pandora/app/monkey_patch.py index f4579d56..0bf0212d 100644 --- a/pandora/app/monkey_patch.py +++ b/pandora/app/monkey_patch.py @@ -7,9 +7,7 @@ from django.core.validators import MaxLengthValidator User = get_user_model() -# load config from json from . import config -config.init() NEW_LENGTH = { 'username': 255, diff --git a/pandora/document/management/commands/rebuild_documentfind.py b/pandora/document/management/commands/rebuild_documentfind.py index af56b748..157072c3 100644 --- a/pandora/document/management/commands/rebuild_documentfind.py +++ b/pandora/document/management/commands/rebuild_documentfind.py @@ -5,7 +5,6 @@ from django.db import connection, transaction from django.db.models import fields from django.conf import settings -settings.RELOAD_CONFIG = False import app.monkey_patch from ... import models diff --git a/pandora/document/management/commands/sync_documentsort.py b/pandora/document/management/commands/sync_documentsort.py index fa6cb5d8..ed3b13e2 100644 --- a/pandora/document/management/commands/sync_documentsort.py +++ b/pandora/document/management/commands/sync_documentsort.py @@ -5,7 +5,6 @@ from django.db import connection, transaction from django.db.models import fields from django.conf import settings -settings.RELOAD_CONFIG = False import app.monkey_patch diff --git a/pandora/item/management/commands/get_frame.py b/pandora/item/management/commands/get_frame.py index 1aaba624..fe96f9b1 100644 --- a/pandora/item/management/commands/get_frame.py +++ b/pandora/item/management/commands/get_frame.py @@ -4,7 +4,6 @@ from django.core.management.base import BaseCommand from django.conf import settings from django.db import transaction -settings.RELOAD_CONFIG = False import app.monkey_patch from ... import models diff --git a/pandora/item/management/commands/rebuild_filter.py b/pandora/item/management/commands/rebuild_filter.py index eb52b13b..8beaa7c9 100644 --- a/pandora/item/management/commands/rebuild_filter.py +++ b/pandora/item/management/commands/rebuild_filter.py @@ -6,7 +6,6 @@ from django.db import connection, transaction from django.db.models import fields from django.conf import settings -settings.RELOAD_CONFIG = False import app.monkey_patch from ... import models import clip.models diff --git a/pandora/item/management/commands/rebuild_find.py b/pandora/item/management/commands/rebuild_find.py index 8418fc2f..7681682d 100644 --- a/pandora/item/management/commands/rebuild_find.py +++ b/pandora/item/management/commands/rebuild_find.py @@ -5,7 +5,6 @@ from django.db import connection, transaction from django.db.models import fields from django.conf import settings -settings.RELOAD_CONFIG = False import app.monkey_patch from ... import models import clip.models diff --git a/pandora/item/management/commands/rebuild_indexes.py b/pandora/item/management/commands/rebuild_indexes.py index 5ad61971..fb197f4c 100644 --- a/pandora/item/management/commands/rebuild_indexes.py +++ b/pandora/item/management/commands/rebuild_indexes.py @@ -5,7 +5,6 @@ from django.db import connection, transaction from django.db.models import fields from django.conf import settings -settings.RELOAD_CONFIG = False import app.monkey_patch from ... import models import clip.models diff --git a/pandora/item/management/commands/rebuild_sort.py b/pandora/item/management/commands/rebuild_sort.py index 5e29ade8..fbcc7d96 100644 --- a/pandora/item/management/commands/rebuild_sort.py +++ b/pandora/item/management/commands/rebuild_sort.py @@ -6,7 +6,6 @@ from django.db import connection, transaction from django.db.models import fields from django.conf import settings -settings.RELOAD_CONFIG = False import app.monkey_patch from ... import models diff --git a/pandora/item/management/commands/sqlfindindex.py b/pandora/item/management/commands/sqlfindindex.py index 81c39488..4348f6cb 100644 --- a/pandora/item/management/commands/sqlfindindex.py +++ b/pandora/item/management/commands/sqlfindindex.py @@ -5,7 +5,6 @@ from django.core.management.base import BaseCommand from django.db import connection, transaction from django.conf import settings -settings.RELOAD_CONFIG = False import app.monkey_patch from ... import models diff --git a/pandora/item/management/commands/sync_itemsort.py b/pandora/item/management/commands/sync_itemsort.py index 4640229d..bc34e949 100644 --- a/pandora/item/management/commands/sync_itemsort.py +++ b/pandora/item/management/commands/sync_itemsort.py @@ -5,7 +5,6 @@ from django.db import connection, transaction from django.db.models import fields from django.conf import settings -settings.RELOAD_CONFIG = False import app.monkey_patch from ... import models import clip.models diff --git a/pandora/manage.py b/pandora/manage.py index 700a1802..4800a366 100755 --- a/pandora/manage.py +++ b/pandora/manage.py @@ -73,5 +73,3 @@ if __name__ == "__main__": sys.stderr.write("Error: Can't find '%s'.\nBefore you run pan.do/ra you must create it\n" % settings.SITE_CONFIG) sys.exit(1) execute_from_command_line(sys.argv) - import app.config - app.config.shutdown() diff --git a/pandora/settings.py b/pandora/settings.py index 99eda0ba..b2596015 100644 --- a/pandora/settings.py +++ b/pandora/settings.py @@ -220,7 +220,6 @@ XACCELREDIRECT = False SITE_CONFIG = join(PROJECT_ROOT, 'config.jsonc') DEFAULT_CONFIG = join(PROJECT_ROOT, 'config.pandora.jsonc') -RELOAD_CONFIG = False #used if CONFIG['canDownloadVideo'] is set TRACKER_URL = "udp://tracker.openbittorrent.com:80" @@ -275,7 +274,6 @@ SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') DATA_UPLOAD_MAX_MEMORY_SIZE = 32 * 1024 * 1024 -RELOADER_RUNNING = False #you can ignore things below this line #========================================================================= LOCAL_APPS = [] From f13f7d238ccb6d1936ce98b86153abb535cc2675 Mon Sep 17 00:00:00 2001 From: j Date: Sun, 27 Sep 2020 18:02:23 +0200 Subject: [PATCH 28/46] still load config --- pandora/app/config.py | 13 ++++++++++++- pandora/app/monkey_patch.py | 2 ++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/pandora/app/config.py b/pandora/app/config.py index 631204f1..3e96ac27 100644 --- a/pandora/app/config.py +++ b/pandora/app/config.py @@ -254,7 +254,6 @@ check the README for further details. except: pass - def update_static(): oxjs_build = os.path.join(settings.STATIC_ROOT, 'oxjs/tools/build/build.py') if os.path.exists(oxjs_build): @@ -364,3 +363,15 @@ def update_geoip(force=False): else: print('failed to download GeoLite2-City.mmdb') +def init(): + load_config(True) + +def shutdown(): + if settings.RELOADER_RUNNING: + RUN_RELOADER = False + settings.RELOADER_RUNNING = False + if NOTIFIER: + NOTIFIER.stop() + + + diff --git a/pandora/app/monkey_patch.py b/pandora/app/monkey_patch.py index 0bf0212d..f4579d56 100644 --- a/pandora/app/monkey_patch.py +++ b/pandora/app/monkey_patch.py @@ -7,7 +7,9 @@ from django.core.validators import MaxLengthValidator User = get_user_model() +# load config from json from . import config +config.init() NEW_LENGTH = { 'username': 255, From ac8c01ee98fc1dfbcc020da5e61e36ba0bbbea86 Mon Sep 17 00:00:00 2001 From: j Date: Tue, 29 Sep 2020 11:03:39 +0200 Subject: [PATCH 29/46] add menu item to toggle icon size --- static/js/infoView.padma.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/static/js/infoView.padma.js b/static/js/infoView.padma.js index b6d44118..e5e894d0 100644 --- a/static/js/infoView.padma.js +++ b/static/js/infoView.padma.js @@ -44,6 +44,12 @@ pandora.ui.infoView = function(data, isMixed) { $options = Ox.MenuButton({ items: [ + { + id: 'toggle', + title: Ox._('Toggle {0} size...', [ + Ox._(ui.icons == 'posters' ? 'poster' : 'icon') + ]), + }, { id: 'delete', title: Ox._('Delete {0}...', [pandora.site.itemName.singular]), @@ -62,7 +68,9 @@ pandora.ui.infoView = function(data, isMixed) { }) .bindEvent({ click: function(data_) { - if (data_.id == 'delete') { + if (data_.id == 'toggle') { + toggleIconSize() + } else if (data_.id == 'delete') { pandora.$ui.deleteItemsDialog = pandora.ui.deleteItemsDialog({ items: [data] }).open(); From f157ebf2c4866de64ec8c301ff6c177d68e548e1 Mon Sep 17 00:00:00 2001 From: j Date: Thu, 15 Oct 2020 11:46:16 +0200 Subject: [PATCH 30/46] honor _blank --- static/js/utils.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/static/js/utils.js b/static/js/utils.js index f5d9590f..a94bed6c 100644 --- a/static/js/utils.js +++ b/static/js/utils.js @@ -375,7 +375,11 @@ pandora.clickLink = function(e, selectEmbed) { if (match) { (selectEmbed || pandora.$ui.textPanel.selectEmbed)(parseInt(match[1])); } else { - pandora.openURL(e.target.href); + if (e.target.target == '_blank') { + pandora.openLink(e.target.href); + } else { + pandora.openURL(e.target.href); + } } }; From 5d6e753b055f7ff05f9f2045dd4583fd38952c6c Mon Sep 17 00:00:00 2001 From: j Date: Sat, 17 Oct 2020 09:47:07 +0200 Subject: [PATCH 31/46] DEA.L. and DEAIL. work for aac --- pandora/archive/extract.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pandora/archive/extract.py b/pandora/archive/extract.py index dcb9bdcf..70f8e254 100644 --- a/pandora/archive/extract.py +++ b/pandora/archive/extract.py @@ -2,13 +2,13 @@ import os from os.path import exists - import fractions +import math +import re +import shutil import subprocess import tempfile import time -import math -import shutil from distutils.spawn import find_executable from glob import glob @@ -63,8 +63,8 @@ def supported_formats(): 'webm': 'libvpx' in stdout and 'libvorbis' in stdout, 'vp8': 'libvpx' in stdout and 'libvorbis' in stdout, 'vp9': 'libvpx-vp9' in stdout and 'libopus' in stdout, - 'mp4': 'libx264' in stdout and 'DEA.L. aac' in stdout, - 'h264': 'libx264' in stdout and 'DEA.L. aac' in stdout, + 'mp4': 'libx264' in stdout and bool(re.compile('DEA.L. aac').findall(stdout)), + 'h264': 'libx264' in stdout and bool(re.compile('DEA.L. aac').findall(stdout)), } def stream(video, target, profile, info, audio_track=0, flags={}): From 4cf28434eb96eaad5c02cfa174fd6373a29c4723 Mon Sep 17 00:00:00 2001 From: j Date: Sun, 8 Nov 2020 22:29:24 +0100 Subject: [PATCH 32/46] pass referer to video download --- pandora/archive/external.py | 9 ++++++++- pandora/archive/tasks.py | 4 ++-- pandora/archive/views.py | 3 ++- static/js/importMediaDialog.js | 1 + 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/pandora/archive/external.py b/pandora/archive/external.py index 510d3f70..1fbe33f3 100644 --- a/pandora/archive/external.py +++ b/pandora/archive/external.py @@ -57,6 +57,8 @@ def get_info(url): info[-1]['tags'] = [] if 'upload_date' in i and i['upload_date']: info[-1]['date'] = '-'.join([i['upload_date'][:4], i['upload_date'][4:6], i['upload_date'][6:]]) + if 'referer' not in info[-1]: + info[-1]['referer'] = url return info def add_subtitles(item, media, tmp): @@ -84,7 +86,7 @@ def add_subtitles(item, media, tmp): sub.selected = True sub.save() -def download(item_id, url): +def download(item_id, url, referer=None): item = Item.objects.get(public_id=item_id) info = get_info(url) if not len(info): @@ -96,6 +98,11 @@ def download(item_id, url): tmp = tmp.decode('utf-8') os.chdir(tmp) cmd = ['youtube-dl', '-q', media['url']] + if referer: + cmd += ['--referer', referer] + elif 'referer' in media: + cmd += ['--referer', media['referer']] + if settings.CONFIG['video'].get('reuseUload', False): max_resolution = max(settings.CONFIG['video']['resolutions']) format = settings.CONFIG['video']['formats'][0] diff --git a/pandora/archive/tasks.py b/pandora/archive/tasks.py index 022cd377..a2aa3d8e 100644 --- a/pandora/archive/tasks.py +++ b/pandora/archive/tasks.py @@ -200,8 +200,8 @@ def update_stream(id): c.save() @task(queue="encoding") -def download_media(item_id, url): - return external.download(item_id, url) +def download_media(item_id, url, referer=None): + return external.download(item_id, url, referer) @task(queue='default') def move_media(data, user): diff --git a/pandora/archive/views.py b/pandora/archive/views.py index fd026c1f..308e7c10 100644 --- a/pandora/archive/views.py +++ b/pandora/archive/views.py @@ -745,6 +745,7 @@ def addMediaUrl(request, data): takes { url: string, // url + referer: string // optional referer url item: string // item } returns { @@ -757,7 +758,7 @@ def addMediaUrl(request, data): response = json_response() i = Item.objects.get(public_id=data['item']) Task.start(i, request.user) - t = tasks.download_media.delay(data['item'], data['url']) + t = tasks.download_media.delay(data['item'], data['url'], data.get('referer')) response['data']['taskId'] = t.task_id add_changelog(request, data, data['item']) return render_to_json_response(response) diff --git a/static/js/importMediaDialog.js b/static/js/importMediaDialog.js index 05bfff24..419aeb4e 100644 --- a/static/js/importMediaDialog.js +++ b/static/js/importMediaDialog.js @@ -132,6 +132,7 @@ pandora.ui.importMediaDialog = function(options) { pandora.api.edit(edit, function(result) { pandora.api.addMediaUrl({ url: info.url, + referer: info.referer, item: edit.id }, function(result) { if (result.data.taskId) { From 41fce072a37d684876badc6f7ce23aa6b9b70b1e Mon Sep 17 00:00:00 2001 From: j Date: Sun, 8 Nov 2020 22:43:47 +0100 Subject: [PATCH 33/46] pass referer in more places --- static/js/addFilesDialog.js | 1 + static/js/addItemDialog.js | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/static/js/addFilesDialog.js b/static/js/addFilesDialog.js index 0f75c07e..4bfa0a71 100644 --- a/static/js/addFilesDialog.js +++ b/static/js/addFilesDialog.js @@ -216,6 +216,7 @@ pandora.ui.addFilesDialog = function(options) { ), function(result) { pandora.api.addMediaUrl({ url: item.url, + referer: item.referer, item: id }, callback); }); diff --git a/static/js/addItemDialog.js b/static/js/addItemDialog.js index ab1213f2..238a7d6e 100644 --- a/static/js/addItemDialog.js +++ b/static/js/addItemDialog.js @@ -222,7 +222,8 @@ pandora.ui.addItemDialog = function(options) { } return value; }); - values.url= info.url; + values.url = info.url; + values.referer = info.referer; return Ox.extend(info, values); } From 1bba157f7fb2c5a45d1068a95d7e147c24f5c17d Mon Sep 17 00:00:00 2001 From: j Date: Sun, 8 Nov 2020 22:54:49 +0100 Subject: [PATCH 34/46] pass referer to get_info --- pandora/archive/external.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pandora/archive/external.py b/pandora/archive/external.py index 1fbe33f3..8eeb0e24 100644 --- a/pandora/archive/external.py +++ b/pandora/archive/external.py @@ -36,8 +36,10 @@ info_key_map = { 'display_id': 'id', } -def get_info(url): +def get_info(url, referer=None): cmd = ['youtube-dl', '-j', '--all-subs', url] + if referer: + cmd += ['--referer', referer] p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) @@ -88,7 +90,7 @@ def add_subtitles(item, media, tmp): def download(item_id, url, referer=None): item = Item.objects.get(public_id=item_id) - info = get_info(url) + info = get_info(url, referer) if not len(info): return '%s contains no videos' % url media = info[0] From e7df18f72780dfbf01dc6f5225a81c0ded6e1952 Mon Sep 17 00:00:00 2001 From: j Date: Sun, 8 Nov 2020 23:26:16 +0100 Subject: [PATCH 35/46] typo --- pandora/archive/external.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandora/archive/external.py b/pandora/archive/external.py index 8eeb0e24..cb0e7ef1 100644 --- a/pandora/archive/external.py +++ b/pandora/archive/external.py @@ -105,7 +105,7 @@ def download(item_id, url, referer=None): elif 'referer' in media: cmd += ['--referer', media['referer']] - if settings.CONFIG['video'].get('reuseUload', False): + if settings.CONFIG['video'].get('reuseUpload', False): max_resolution = max(settings.CONFIG['video']['resolutions']) format = settings.CONFIG['video']['formats'][0] if format == 'mp4': From b377956a13e330a98edf3db982b33ff7c614adb9 Mon Sep 17 00:00:00 2001 From: j Date: Thu, 26 Nov 2020 20:22:59 +0100 Subject: [PATCH 36/46] add option to add/remove values to listkeys in batchedit dialog --- static/js/addRemoveKeyDialog.js | 124 ++++++++++++++++++++++++++++++++ static/js/documentInfoView.js | 7 ++ static/js/infoView.js | 7 ++ 3 files changed, 138 insertions(+) create mode 100644 static/js/addRemoveKeyDialog.js diff --git a/static/js/addRemoveKeyDialog.js b/static/js/addRemoveKeyDialog.js new file mode 100644 index 00000000..107d55bc --- /dev/null +++ b/static/js/addRemoveKeyDialog.js @@ -0,0 +1,124 @@ + +pandora.ui.addRemoveKeyDialog = function(options) { + var item = Ox.getObjectById(pandora.site.documentKeys, options.key), + itemTitle = Ox._(item ? item.title : options.key), + dialogTitle = Ox._('Add/Remove {0}', [itemTitle]); + + return Ox.Element('') + .addClass('OxLight') + .html(' [+]') + .css({ + cursor: 'pointer', + }) + .options({ + tooltip: dialogTitle + }) + .on('click', function(event) { + var add, remove, + dialog = Ox.Dialog({ + buttons: [ + Ox.Button({ + title: Ox._('Cancel') + }) + .css({margin: '4px 4px 4px 0'}) + .bindEvent({ + click: function() { + dialog.close() + } + }), + Ox.Button({ + title: Ox._('Update'), + width: 52 + }).bindEvent({ + click: function() { + var addValue = add.value() + var removeValue = remove.value() + if (!addValue.length && !removeValue.length) { + dialog.close() + return + } + var $loading = Ox.LoadingScreen({ + width: 256, + height: 64 + }) + dialog.options({ + content: $loading.start() + }) + var ids = options.ids; + pandora.api[{ + 'items': 'find', + 'documents': 'findDocuments', + }[options.section]]({ + query: { + conditions: [{key: 'id', operator: '&', value: ids}] + }, + range: [0, ids.length], + keys: ['id', options.key] + + }, function(result) { + Ox.serialForEach(result.data.items, + function(item, index, items, callback) { + var changed = false + var value = item[options.key] || [] + if (addValue.length && !Ox.contains(value, addValue)) { + value.push(addValue) + changed = true + } + if (removeValue.length && Ox.contains(value, removeValue)) { + value = value.filter(function(v) { return v != removeValue; } ) + changed = true + } + if (changed) { + var edit = {id: item.id} + edit[options.key] = value + pandora.api[{ + 'items': 'edit', + 'documents': 'editDocument', + }[options.section]](edit, function(response) { + callback() + }) + } else { + callback() + } + }, + function() { + $loading.stop() + dialog.close() + } + ) + }) + } + }) + ], + closeButton: true, + content: Ox.Element() + .css({ + padding: '8px' + }) + .append( + add = Ox.Input({ + width: 240, + type: 'input', + label: 'Add' + }) + ) + .append( + Ox.Element().html('').css({ + height: '8px' + }) + ) + .append( + remove = Ox.Input({ + width: 240, + type: 'input', + label: 'Remove' + }) + ), + height: 64, + removeOnClose: true, + title: dialogTitle, + width: 256 + }).open(); + }) + +} diff --git a/static/js/documentInfoView.js b/static/js/documentInfoView.js index 1ba47cf5..7dc7e86d 100644 --- a/static/js/documentInfoView.js +++ b/static/js/documentInfoView.js @@ -572,6 +572,13 @@ pandora.ui.documentInfoView = function(data, isMixed) { } }) .appendTo($element); + if (isMixed[key] && Ox.contains(listKeys, key)) { + pandora.ui.addRemoveKeyDialog({ + ids: ui.collectionSelection, + key: key, + section: ui.section + }).appendTo($element) + } } }); $element.appendTo($text); diff --git a/static/js/infoView.js b/static/js/infoView.js index 5bd588ed..17d88645 100644 --- a/static/js/infoView.js +++ b/static/js/infoView.js @@ -529,6 +529,13 @@ pandora.ui.infoView = function(data, isMixed) { } }) .appendTo($element); + if (isMixed[key] && Ox.contains(listKeys, key)) { + pandora.ui.addRemoveKeyDialog({ + ids: ui.listSelection, + key: key, + section: ui.section + }).appendTo($element) + } } }); $element.appendTo($text); From 7d8b38e627aa689457179a72b3a9c498d43128c7 Mon Sep 17 00:00:00 2001 From: j Date: Fri, 5 Feb 2021 10:07:14 +0100 Subject: [PATCH 37/46] cast list values to str, just in case --- pandora/item/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandora/item/models.py b/pandora/item/models.py index 1733e896..5172990c 100644 --- a/pandora/item/models.py +++ b/pandora/item/models.py @@ -1022,7 +1022,7 @@ class Item(models.Model): elif sort_type == 'string': value = self.get(source, '') if isinstance(value, list): - value = ','.join(value) + value = ','.join([str(v) for v in value]) value = utils.sort_string(value)[:955] set_value(s, name, value) elif sort_type == 'words': From 7ea1f38a0fb002a0a7ae82b1f37e30b96ba3503b Mon Sep 17 00:00:00 2001 From: j Date: Fri, 19 Mar 2021 10:19:19 +0100 Subject: [PATCH 38/46] check body --- pandora/websocket/worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandora/websocket/worker.py b/pandora/websocket/worker.py index 1a0e633d..9a3d7bd4 100644 --- a/pandora/websocket/worker.py +++ b/pandora/websocket/worker.py @@ -24,7 +24,7 @@ class Worker(ConsumerMixin): def process_task(self, body, message): try: - if body['task'] == 'trigger_event': + if isinstance(body, dict) and body.get('task') == 'trigger_event': daemon.trigger_event(*body['args']) except: logger.error('faild to trigger event %s', body, exc_info=True) From 4689af20505946b1325231b3a9412ac868871277 Mon Sep 17 00:00:00 2001 From: j Date: Fri, 19 Mar 2021 10:19:29 +0100 Subject: [PATCH 39/46] replace all not just first --- static/js/utils.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/static/js/utils.js b/static/js/utils.js index a94bed6c..574e4a87 100644 --- a/static/js/utils.js +++ b/static/js/utils.js @@ -2659,8 +2659,9 @@ pandora.openURL = function(url) { }; pandora.safeDocumentName = function(name) { - ['?', '#', '%'].forEach(function(c) { - name = name.replace(c, '_'); + ['\\?', '#', '%', '/'].forEach(function(c) { + var r = new RegExp(c, 'g') + name = name.replace(r, '_'); }) return name; }; From 20b3831862a459a00a2fe1b1302b4ec760916d49 Mon Sep 17 00:00:00 2001 From: j Date: Mon, 29 Mar 2021 11:03:12 +0200 Subject: [PATCH 40/46] don't fail if getsize fails on tmp file --- pandora/item/views.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pandora/item/views.py b/pandora/item/views.py index 7ad11572..62c6b21a 100644 --- a/pandora/item/views.py +++ b/pandora/item/views.py @@ -1030,7 +1030,10 @@ def download(request, id, resolution=None, format='webm', part=None): return HttpResponseForbidden() elif r is True: response = HttpResponse(FileWrapper(video), content_type=content_type) - response['Content-Length'] = os.path.getsize(video.name) + try: + response['Content-Length'] = os.path.getsize(video.name) + except: + pass else: response = HttpFileResponse(r, content_type=content_type) else: From 7f319982546bfe44987bbb1f3b9d7d83a276b437 Mon Sep 17 00:00:00 2001 From: j Date: Thu, 1 Apr 2021 11:30:28 +0200 Subject: [PATCH 41/46] add 0p for audio only profile --- pandora/archive/extract.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pandora/archive/extract.py b/pandora/archive/extract.py index 70f8e254..ebfd2ae7 100644 --- a/pandora/archive/extract.py +++ b/pandora/archive/extract.py @@ -145,6 +145,13 @@ def stream(video, target, profile, info, audio_track=0, flags={}): audioquality = -1 audiobitrate = '22k' audiochannels = 1 + elif profile == '0p': + info['video'] = [] + audiorate = 48000 + audioquality = 6 + audiobitrate = None + audiochannels = None + audio_codec = 'libopus' else: height = 96 From 7fa3f7d8fe25cc1fab3016ce1c9a6e80bd0877f7 Mon Sep 17 00:00:00 2001 From: j Date: Thu, 29 Apr 2021 21:57:52 +0200 Subject: [PATCH 42/46] no title fallback --- static/js/documentInfoView.js | 2 +- static/js/infoView.js | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/static/js/documentInfoView.js b/static/js/documentInfoView.js index 7dc7e86d..bb69563e 100644 --- a/static/js/documentInfoView.js +++ b/static/js/documentInfoView.js @@ -218,7 +218,7 @@ pandora.ui.documentInfoView = function(data, isMixed) { .append( Ox.EditableContent({ editable: canEdit, - placeholder: formatLight(Ox._('No Title')), + placeholder: formatLight(Ox._( isMixed.title ? 'Mixed title' : 'Untitled')), tooltip: canEdit ? pandora.getEditTooltip() : '', value: data.title || '' }) diff --git a/static/js/infoView.js b/static/js/infoView.js index 17d88645..f094f947 100644 --- a/static/js/infoView.js +++ b/static/js/infoView.js @@ -228,6 +228,7 @@ pandora.ui.infoView = function(data, isMixed) { Ox.EditableContent({ editable: canEdit, tooltip: canEdit ? pandora.getEditTooltip() : '', + placeholder: formatLight(Ox._( isMixed.title ? 'Mixed title' : 'Untitled')), value: data.title || '' }) .css({ From 814f98a571326a43a8417b3dc2a6bb14cc02d159 Mon Sep 17 00:00:00 2001 From: j Date: Wed, 5 May 2021 20:54:25 +0200 Subject: [PATCH 43/46] only switch to subtitles if no layer was selected --- static/js/importAnnotationsDialog.js | 29 ++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/static/js/importAnnotationsDialog.js b/static/js/importAnnotationsDialog.js index 559a8fa7..13e83118 100644 --- a/static/js/importAnnotationsDialog.js +++ b/static/js/importAnnotationsDialog.js @@ -5,7 +5,6 @@ pandora.ui.importAnnotationsDialog = function(options) { var layers = pandora.site.layers.filter(function(layer) { return layer.canAddAnnotations[pandora.user.level]; }), - languages = Ox.sortBy(Ox.LANGUAGES.map(function(language) { return {id: language.code, title: language.name}; }), 'title'), @@ -13,7 +12,12 @@ pandora.ui.importAnnotationsDialog = function(options) { $content = Ox.Element().css({margin: '16px'}), $layerSelect = Ox.Select({ - items: layers, + items: [{ + id: '', + type: '', + title: Ox._('Select Layer...'), + disabled: true, + }].concat(layers), label: Ox._('Layer'), labelWidth: 128, width: 384 @@ -22,7 +26,12 @@ pandora.ui.importAnnotationsDialog = function(options) { marginTop: '16px' }) .bindEvent({ - change: updateLanguageSelect + change: function(data) { + updateLanguageSelect() + that[ + (data.value.length && $fileInput.value().length) ? 'enableButton' : 'disableButton' + ]('import'); + } }) .appendTo($content), @@ -72,13 +81,16 @@ pandora.ui.importAnnotationsDialog = function(options) { var format = Ox.last(data.value[0].name.split('.')); $formatSelect.options({value: format}); var subtitlesLayer = pandora.getSubtitlesLayer() - if (subtitlesLayer && format == 'srt' && Ox.getObjectById(layers, subtitlesLayer)) { + if ( + subtitlesLayer && !$layerSelect.value().length && + format == 'srt' && Ox.getObjectById(layers, subtitlesLayer) + ) { $layerSelect.options({value: subtitlesLayer}) } updateLanguageSelect(); } that[ - data.value.length ? 'enableButton' : 'disableButton' + (data.value.length && $layerSelect.value().length) ? 'enableButton' : 'disableButton' ]('import'); } }) @@ -226,9 +238,10 @@ pandora.ui.importAnnotationsDialog = function(options) { } function updateLanguageSelect() { - var layerType = Ox.getObjectById( - pandora.site.layers, $layerSelect.value() - ).type; + var layer = $layerSelect.value(), + layerType = layer.length ? Ox.getObjectById( + pandora.site.layers, layer, + ).type : ''; if (layerType != 'text') { $languageSelect.value(pandora.site.language); } From 4e1e6e3b17556db62b46d6f5291ef12ea21d5e92 Mon Sep 17 00:00:00 2001 From: j Date: Sat, 15 May 2021 14:22:24 +0200 Subject: [PATCH 44/46] new youtube-dl version --- requirements.txt | 2 +- update.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e2f2c2eb..b03274a2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ requests<3.0.0,>=2.24.0 urllib3<2.0.0,>=1.25.2 tornado<5 geoip2==4.1.0 -youtube-dl>=2020.5.8 +youtube-dl>=2021.4.26 python-memcached elasticsearch future diff --git a/update.py b/update.py index 3a2c7a40..685c1dc9 100755 --- a/update.py +++ b/update.py @@ -297,6 +297,8 @@ if __name__ == "__main__": run_sql(sql) run(join(base, 'pandora/manage.py'), 'migrate', 'system') run(join(base, 'pandora/manage.py'), 'update_geoip') + if old <= 6383: + run('./bin/pip', 'install', '-r', 'requirements.txt') else: if len(sys.argv) == 1: branch = get_branch() From 54362e8b41dede09c6568b81722793a087eb887f Mon Sep 17 00:00:00 2001 From: j Date: Sat, 15 May 2021 14:38:58 +0200 Subject: [PATCH 45/46] reference pandoractl instead of update.py --- README.md | 5 +++++ update.py | 2 +- vm/README.md | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 044a3cb8..7801e180 100644 --- a/README.md +++ b/README.md @@ -50,4 +50,9 @@ export BRANCH=stable # change to 'master' to get current developement version More info at https://code.0x2620.org/0x2620/pandora/wiki/Customization +## Update + + To update your existing instlalation run + + pandoractl update diff --git a/update.py b/update.py index 685c1dc9..2d52da38 100755 --- a/update.py +++ b/update.py @@ -80,7 +80,7 @@ def get_release(): def reload_notice(base): - print('\nPlease restart pan.do/ra to finish the update:\n\tsudo %s/ctl reload\n' % base) + print('\nPlease restart pan.do/ra to finish the update:\n\tsudo pandoractl reload\n') def check_services(base): diff --git a/vm/README.md b/vm/README.md index ff055e0c..6f935dd1 100644 --- a/vm/README.md +++ b/vm/README.md @@ -42,5 +42,5 @@ Pan.do/ra is installed in /srv/pandora and is served with nginx on http://pandor to get the latest version of pan.do/ra cd /srv/pandora - ./update.py + pandoractl update From c17cc6d5cfdbe9425357890e08ce6eec54a654c1 Mon Sep 17 00:00:00 2001 From: j Date: Wed, 19 May 2021 16:02:18 +0100 Subject: [PATCH 46/46] don't fail on invalid arrays --- pandora/item/models.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pandora/item/models.py b/pandora/item/models.py index 5172990c..04ec81dd 100644 --- a/pandora/item/models.py +++ b/pandora/item/models.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import json +import logging import os import re import shutil @@ -42,6 +43,7 @@ from user.utils import update_groups from user.models import Group import archive.models +logger = logging.getLogger(__name__) User = get_user_model() @@ -855,7 +857,7 @@ class Item(models.Model): values = list(set(values)) else: values = self.get(key, '') - if isinstance(values, list): + if values and isinstance(values, list) and isinstance(values[0], str): save(key, '\n'.join(values)) else: save(key, values) @@ -1099,7 +1101,11 @@ class Item(models.Model): _current_values.append(value[0]) current_values = _current_values - current_values = list(set(current_values)) + try: + current_values = list(set(current_values)) + except: + logger.error('invalid facet data for %s: %s', key, current_values) + current_values = [] current_values = [ox.decode_html(ox.strip_tags(v)) for v in current_values] current_values = [unicodedata.normalize('NFKD', v) for v in current_values] self.update_facet_values(key, current_values)