From 9c82a18585ed0ac6483a8f0d02a900a9d49215fe Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Thu, 18 Jul 2019 11:24:04 +0200
Subject: [PATCH 001/139] use lower value

---
 etc/nginx/pandora | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/etc/nginx/pandora b/etc/nginx/pandora
index 24125480..8ef1f433 100644
--- a/etc/nginx/pandora
+++ b/etc/nginx/pandora
@@ -42,7 +42,7 @@ server {
             proxy_set_header Proxy "";
             proxy_redirect off;
             proxy_buffering off;
-            proxy_read_timeout 999999999;
+            proxy_read_timeout 99999;
             proxy_pass http://127.0.0.1:2622/;
     }
 

From 97b5c0538a2c7d21709aa28fb9896ae1505c5170 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Fri, 19 Jul 2019 12:21:45 +0200
Subject: [PATCH 002/139] use bionic for VM

---
 vm/build.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/vm/build.sh b/vm/build.sh
index 0f2f86bc..c1940ba9 100755
--- a/vm/build.sh
+++ b/vm/build.sh
@@ -5,11 +5,11 @@ BASE=`pwd`
 VERSION=`cd ..;git rev-list HEAD --count`
 TARGET=${BASE}/pandora-r${VERSION}.vdi
 
-img=xenial-server-cloudimg-amd64-disk1.img
+img=bionic-server-cloudimg-amd64.img
 
 if [ ! -e $img ]; then
     echo downloading $img
-    curl -s -O https://cloud-images.ubuntu.com/xenial/current/$img
+    curl -s -O https://cloud-images.ubuntu.com/bionic/current/$img
 fi
 echo preparing ${TARGET}.img
 cp -a $img ${TARGET}.img

From 2650e5b50b8b667f6231b254fda90afd8198aaa8 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Fri, 19 Jul 2019 12:55:05 +0200
Subject: [PATCH 003/139] 2 cores are enough to build

---
 vm/build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/vm/build.sh b/vm/build.sh
index c1940ba9..06ee5817 100755
--- a/vm/build.sh
+++ b/vm/build.sh
@@ -19,7 +19,7 @@ qemu-img resize ${TARGET}.img +998G
 
 echo boot image and install pandora
 kvm -m 1024 \
-    -smp 4 \
+    -smp 2 \
     -cdrom seed.img \
     -device e1000,netdev=user.0 \
     -netdev user,id=user.0,hostfwd=tcp::5555-:22,hostfwd=tcp::2620-:80 \

From 0076aae8968c937017cabceb7d748f13c13681a5 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Fri, 19 Jul 2019 19:04:12 +0200
Subject: [PATCH 004/139] import drag and drop files

---
 static/js/addItemDialog.js |  9 +++++++
 static/js/importScreen.js  | 54 ++++++++++++++++++++++++++++++++++++++
 static/js/pandora.js       | 31 ++++++++++++++++++++++
 static/js/utils.js         | 25 ++++++++++++++++++
 4 files changed, 119 insertions(+)
 create mode 100644 static/js/importScreen.js

diff --git a/static/js/addItemDialog.js b/static/js/addItemDialog.js
index 465a8da7..71361fda 100644
--- a/static/js/addItemDialog.js
+++ b/static/js/addItemDialog.js
@@ -69,6 +69,15 @@ pandora.ui.addItemDialog = function(options) {
         title: Ox._('Add {0}', [pandora.site.itemName.singular]),
         width: 544
     });
+    if (options.files) {
+        that.options({content: $screen.start()});
+        $button.options({disabled: true});
+        Ox.serialMap(options.files, function(file, index, files, callback) {
+            getFileInfo(file, function(info) {
+                callback(Ox.extend(info, {file: file}));
+            });
+        }, onInfo);
+    }
 
     function createButton() {
         $button = Ox[selected == 'upload' ? 'FileButton' : 'Button']({
diff --git a/static/js/importScreen.js b/static/js/importScreen.js
new file mode 100644
index 00000000..8749647c
--- /dev/null
+++ b/static/js/importScreen.js
@@ -0,0 +1,54 @@
+'use strict';
+
+pandora.ui.importScreen = function() {
+
+    var that = Ox.Element()
+        .attr({id: 'importScreen'})
+        .css({
+            position: 'absolute',
+            left: 0,
+            top: 0,
+            right: 0,
+            bottom: 0,
+            backgroundColor: 'rgba(0, 0, 0, 0.5)',
+            zIndex: 1000
+        })
+        .on({
+            click: function() {
+                that.remove();
+            },
+            dragleave: function() {
+                that.remove();
+            }
+        });
+
+    Ox.Element()
+        .css({
+            position: 'absolute',
+            left: 0,
+            top: 0,
+            right: 0,
+            bottom: 0,
+            width: pandora.hasCapability('canAddItems') ? 192 : 256,
+            height: 16,
+            padding: '8px 0',
+            borderRadius: 8,
+            margin: 'auto',
+            background: 'rgba(255, 255, 255, 0.9)',
+            fontSize: 13,
+            color: 'rgb(0, 0, 0)',
+            textAlign: 'center'
+        })
+        .text(
+            Ox._(pandora.hasCapability('canAddItems') ? (
+                'Import {0}'
+            ) : (
+                'You are not allowed to import {0}'
+            ),
+            [pandora.user.ui.section == 'documents' ? 'Documents' : pandora.site.itemName.plural])
+        )
+        .appendTo(that);
+
+    return that;
+
+};
diff --git a/static/js/pandora.js b/static/js/pandora.js
index 0962c235..0c0f76ba 100644
--- a/static/js/pandora.js
+++ b/static/js/pandora.js
@@ -281,6 +281,37 @@ appPanel
             resize: pandora.resizeWindow,
             unload: pandora.unloadWindow
         })
+        Ox.$document.on({
+            dragenter: function(event) {
+                if (Ox.contains(event.originalEvent.dataTransfer.types, 'Files')) {
+                    event.originalEvent.preventDefault();
+                    event.originalEvent.stopPropagation();
+                    if (!$('#importScreen').length) {
+                        pandora.ui.importScreen().appendTo(Ox.$body);
+                    }
+                } else {
+                    console.log(event.originalEvent.dataTransfer);
+                }
+            },
+            dragover: function(event) {
+                event.originalEvent.preventDefault();
+                event.originalEvent.stopPropagation();
+            },
+            dragstart: function(event) {
+                event.originalEvent.preventDefault();
+                event.originalEvent.stopPropagation();
+            },
+            drop: function(event) {
+                $('#importScreen').remove();
+                if (pandora.hasCapability('canAddItems')) {
+                    if (event.originalEvent.dataTransfer.files.length) {
+                        event.originalEvent.preventDefault();
+                        event.originalEvent.stopPropagation();
+                        pandora.uploadDroppedFiles(event.originalEvent.dataTransfer.files)
+                    }
+                }
+            }
+        });
         Ox.extend(pandora, {
             $ui: {},
             site: data.site,
diff --git a/static/js/utils.js b/static/js/utils.js
index 5825e724..8bb3a9dc 100644
--- a/static/js/utils.js
+++ b/static/js/utils.js
@@ -404,6 +404,31 @@ pandora.createLinks = function($element) {
     });
 };
 
+pandora.uploadDroppedFiles = function(files) {
+    var documentExtensions = ['pdf', /* 'epub', 'txt', */ 'png', 'gif', 'jpg'];
+    files = Ox.map(files, function(file) { return file});
+
+    if (files.every(function(file) {
+        var extension = file.name.split('.').pop().toLowerCase()
+        return Ox.contains(documentExtensions, extension)
+    })) {
+        pandora.ui.uploadDocumentDialog({
+            files: files
+        }, function(files) {
+            if (files) {
+                Ox.Request.clearCache('findDocuments');
+                if (pandora.user.ui.document || pandora.user.ui.section != 'documents') {
+                    pandora.UI.set({section: 'documents', document: ''});
+                } else {
+                    pandora.$ui.list && pandora.$ui.list.reloadList();
+                }
+            }
+        }).open();
+    } else {
+        pandora.ui.addItemDialog({files: files}).open()
+    }
+};
+
 (function() {
 
     pandora.doHistory = function(action, items, targets, index, callback) {

From f838a6be6070e872a8e960f0b6f12c8078edd1de Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Mon, 22 Jul 2019 11:11:04 +0200
Subject: [PATCH 005/139] default to multiple videos

---
 static/js/addFilesDialog.js | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/static/js/addFilesDialog.js b/static/js/addFilesDialog.js
index 75bdca18..e718b218 100644
--- a/static/js/addFilesDialog.js
+++ b/static/js/addFilesDialog.js
@@ -117,6 +117,15 @@ pandora.ui.addFilesDialog = function(options) {
             )
         });
     } else {
+        if (options.items.length > 1) {
+            selectItems.push({
+                id: 'multiple',
+                title: Ox._(
+                    'Create multiple {0}',
+                    [pandora.site.itemName.plural.toLowerCase()]
+                )
+            });
+        }
         selectItems.push({
             id: 'one',
             title: Ox._(
@@ -125,15 +134,6 @@ pandora.ui.addFilesDialog = function(options) {
             )
         });
     }
-    if (options.items.length > 1) {
-        selectItems.push({
-            id: 'multiple',
-            title: Ox._(
-                'Create multiple {0}',
-                [pandora.site.itemName.plural.toLowerCase()]
-            )
-        });
-    }
     var $select = Ox.Select({
         items: selectItems,
         width: 256

From 33b105b9b80ede56c8c93261934d86758d9fef54 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Mon, 22 Jul 2019 11:11:25 +0200
Subject: [PATCH 006/139] avoid password autocompltee

---
 static/js/preferencesDialog.js | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/static/js/preferencesDialog.js b/static/js/preferencesDialog.js
index aa74d003..70af78f2 100644
--- a/static/js/preferencesDialog.js
+++ b/static/js/preferencesDialog.js
@@ -47,6 +47,9 @@ pandora.ui.preferencesDialog = function() {
                                         validate: pandora.validateNewPassword,
                                         width: 320
                                     })
+                                    .attr({
+                                        autocomplete: 'new-password'
+                                    })
                                     .bindEvent({
                                         validate: function(data) {
                                             data.valid && pandora.api.editPreferences({password: data.value});

From 115f05625c3dc9872a4250251ebfb887825723d5 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Mon, 22 Jul 2019 11:11:34 +0200
Subject: [PATCH 007/139] ignore old migrations

---
 update.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/update.py b/update.py
index 41283454..119f9adb 100755
--- a/update.py
+++ b/update.py
@@ -332,6 +332,7 @@ if __name__ == "__main__":
             '-- Model missing for table: djcelery_intervalschedule\n',
             '-- Model missing for table: djcelery_workerstate\n',
             '-- Model missing for table: djcelery_taskstate\n',
+            '-- Model missing for table: south_migrationhistory\n',
             '-- Model missing for table: cache\n',
         ]:
             if row in diff:

From 0b8ea2d517e694dd7fce9aa8ca973daac37bc7a9 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Mon, 22 Jul 2019 11:11:42 +0200
Subject: [PATCH 008/139] space

---
 static/js/utils.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/static/js/utils.js b/static/js/utils.js
index 8bb3a9dc..79ab43d8 100644
--- a/static/js/utils.js
+++ b/static/js/utils.js
@@ -406,7 +406,7 @@ pandora.createLinks = function($element) {
 
 pandora.uploadDroppedFiles = function(files) {
     var documentExtensions = ['pdf', /* 'epub', 'txt', */ 'png', 'gif', 'jpg'];
-    files = Ox.map(files, function(file) { return file});
+    files = Ox.map(files, function(file) { return file });
 
     if (files.every(function(file) {
         var extension = file.name.split('.').pop().toLowerCase()

From a786f5d9bcb1d1a90faff2934611b3daaf4f50d2 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Tue, 23 Jul 2019 11:26:13 +0200
Subject: [PATCH 009/139] remove unused code

---
 static/js/uploadVideoDialog.js | 599 ---------------------------------
 static/js/utils.js             |   5 +-
 2 files changed, 2 insertions(+), 602 deletions(-)
 delete mode 100644 static/js/uploadVideoDialog.js

diff --git a/static/js/uploadVideoDialog.js b/static/js/uploadVideoDialog.js
deleted file mode 100644
index cb455a67..00000000
--- a/static/js/uploadVideoDialog.js
+++ /dev/null
@@ -1,599 +0,0 @@
-'use strict';
-
-pandora.ui.uploadVideoDialog = function(data) {
-
-    var cancelled = false,
-        file,
-        hasFirefogg = !(Ox.isUndefined(window.Firefogg)) && (
-            $.browser.version < '35' || Firefogg().version >= 334
-        ),
-        infoContent = Ox._('Please select the video file that you want to upload.'),
-        itemView = pandora.hasCapability('canSeeExtraItemViews') ? 'media' : 'info',
-        selectFile,
-        $actionButton,
-        $closeButton,
-        $content = Ox.Element().css({margin: '16px'}),
-        $info = $('<div>')
-            .css({padding: '4px'})
-            .html(infoContent),
-        $progress,
-        $status = $('<div>').css({padding: '4px', paddingTop: '8px'}),
-        that = Ox.Dialog({
-            buttons: [
-                $closeButton = Ox.Button({
-                    id: 'close',
-                    title: Ox._('Close')
-                }).css({
-                    float: 'left'
-                }).bindEvent({
-                    click: function() {
-                        if ($closeButton.options('title') == Ox._('Cancel')) {
-                            cancelled = true;
-                            $info.html(infoContent);
-                            $status.html('');
-                            pandora.firefogg && pandora.firefogg.cancel();
-                            pandora.$ui.upload && pandora.$ui.upload.abort();
-                            $closeButton.options('title', Ox._('Close'));
-                            if ($actionButton.options('title') == Ox._('Upload')) {
-                                $closeButton.options('title', Ox._('Close'));
-                                $actionButton.replaceWith($actionButton = hasFirefogg
-                                    ? getFirefoggButton()
-                                    : getSelectVideoButton()
-                                );
-                            }
-                            $actionButton.show();
-                        } else {
-                            that.triggerEvent('close');
-                        }
-                    }
-                }),
-                $actionButton = hasFirefogg ? getFirefoggButton() : getSelectVideoButton()
-            ],
-            content: $content,
-            height: 128,
-            removeOnClose: true,
-            width: 368,
-            title: Ox._('Upload Video'),
-        })
-        .bindEvent({
-            close: function(data) {
-                if (pandora.firefogg) {
-                    pandora.firefogg.cancel();
-                    delete pandora.firefogg;
-                }
-                that.close();
-            }
-        });
-
-    if (!pandora.site.itemRequiresVideo && !pandora.user.ui.item) {
-        $info.html(Ox._(
-            'You can only upload a video to an existing {0}.'
-            + ' Please check if an entry for the {0}'
-            + ' you want to upload exists, and create one otherwise.',
-            [pandora.site.itemName.singular.toLowerCase()]
-        ));
-        $actionButton.hide();
-    }
-    $content.append($info);
-    $content.append($status);
-
-    function aspectratio(ratio) {
-        var denominator, numerator;
-        ratio = ratio.split(':');
-        numerator = ratio[0];
-        if (ratio.length == 2) {
-            denominator = ratio[1];
-        }
-        if (Math.abs(numerator / denominator - 4/3) < 0.03) {
-            numerator = 4;
-            denominator = 3;
-        } else if (Math.abs(numerator / denominator - 16/9) < 0.02) {
-            numerator = 16;
-            denominator = 9;
-        }
-        return {
-            denominator: denominator,
-            'float': numerator / denominator,
-            numerator: numerator,
-            ratio: numerator + ':' + denominator
-        };
-    }
-
-    function resetProgress(status) {
-        $progress = Ox.Progressbar({
-            progress: 0,
-            showPercent: true,
-            showTime: true,
-            width: 304
-        });
-        $status.html(status || '').append($progress);
-    }
-
-    function directUpload(file, info) {
-        resetProgress();
-        pandora.api.addMedia({
-            filename: info.name,
-            id: info.oshash,
-            item: pandora.site.itemRequiresVideo
-                ? undefined
-                : pandora.user.ui.item
-        }, function(result) {
-            uploadStream(result.data.item, info, file);
-        });
-    }
-
-    function encode() {
-        var filename = pandora.firefogg.sourceFilename,
-            info = JSON.parse(pandora.firefogg.sourceInfo),
-            item,
-            oshash = info.oshash;
-        $info.html('<b>' + filename + '</b><br>' + Ox._('encoding...'));
-        resetProgress();
-        pandora.api.addMedia({
-            filename: filename,
-            id: oshash,
-            info: info,
-            item: pandora.site.itemRequiresVideo
-                ? undefined
-                : pandora.user.ui.item
-        }, function(result) {
-            item = result.data.item;
-            pandora.firefogg.encode(
-                getEncodingOptions(info),
-                function(result, file) {
-                    result = JSON.parse(result);
-                    if (result.progress != 1) {
-                        $status.html(
-                            cancelled
-                            ? Ox._('Encoding cancelled.')
-                            : Ox._('Encoding failed.')
-                        );
-                        delete pandora.firefogg;
-                        return;
-                    }
-                    setTimeout(function() {
-                        $info.html(
-                            '<b>' + filename + '</b><br>'
-                            + Ox._('uploading...')
-                        );
-                        uploadStream(item, info, file);
-                    });
-                },
-                Ox.throttle(function(progress) {
-                    progress = JSON.parse(progress).progress || 0;
-                    $progress.options({progress: progress});
-                }, 1000)
-            );
-        });
-    }
-
-    function getInfo(file, callback) {
-        Ox.oshash(file, function(oshash) {
-            var $video = $('<video>'),
-                url = URL.createObjectURL(file),
-                info = {
-                    audio: [],
-                    direct: false,
-                    oshash: oshash,
-                    name: file.name,
-                    size: file.size,
-                    video: []
-                };
-            $video.one('error', function(event) {
-                callback(info);
-            });
-            $video.one('loadedmetadata', function(event) {
-                info.duration = $video[0].duration;
-                if ($video[0].videoHeight > 0) {
-                    info.video.push({
-                        height: $video[0].videoHeight,
-                        width: $video[0].videoWidth
-                    });
-                }
-                if (info.duration) {
-                    info.bitrate = info.size * 8 / info.duration / 1000;
-                }
-                var format = pandora.site.video.formats[0],
-                    resolution = getResolution(info);
-                info.direct = Ox.endsWith(info.name, format)
-                    && info.video.length > 0
-                    && info.video[0].height <= resolution;
-                callback(info);
-            });
-            $video[0].src = url;
-        });
-    }
-
-    function getResolution(info) {
-        var height = info.video && info.video.length
-                ? info.video[0].height
-                : Ox.max(pandora.site.video.resolutions),
-            resolution = Ox.sort(pandora.site.video.resolutions)
-                .filter(function(resolution) {
-                    return height <= resolution;
-                })[0] || Ox.max(pandora.site.video.resolutions);
-        return resolution;
-    }
-
-    function getFirefoggButton() {
-        return Ox.Button({
-            id: 'action',
-            title: Ox._('Select Video')
-        }).bindEvent({
-            click: function() {
-                if ($actionButton.options('title') == Ox._('Select Video')) {
-                    if (selectVideo()) {
-                        $closeButton.options('title', Ox._('Cancel'));
-                        $actionButton.options('title', Ox._('Upload'));
-                    }
-                } else if ($actionButton.options('title') == Ox._('Cancel')) {
-                    cancelled = true;
-                    pandora.firefogg && pandora.firefogg.cancel();
-                    pandora.$ui.upload && pandora.$ui.upload.abort();
-                    $actionButton.options('title', Ox._('Select Video'));
-                    $closeButton.show();
-                } else {
-                    $closeButton.options('title', Ox._('Cancel'));
-                    $actionButton.hide().options('title', Ox._('Select Video'));
-                    encode();
-                }
-            }
-        })
-    }
-
-    function getSelectVideoButton() {
-        return Ox.FileButton({
-            id: 'action',
-            title: Ox._('Select Video'),
-            maxFiles: 1,
-            width: 96
-        }).css({
-            float: 'left'
-        }).bindEvent({
-            click: function(data) {
-                if (data.files.length) {
-                    cancelled = false;
-                    $actionButton.replaceWith($actionButton = Ox.Button({
-                        id: 'action',
-                        title: 'Upload',
-                        disabled: true
-                    }).css({
-                        float: 'left'
-                    }));
-                    getInfo(data.files[0], function(info) {
-                        $actionButton.options({
-                            disabled: false
-                        }).bindEvent({
-                            click: function() {
-                                $actionButton.replaceWith($actionButton = getSelectVideoButton().hide());
-                                info.direct
-                                    ? directUpload(data.files[0], info)
-                                    : upload(data.files[0]);
-                            }
-                        });
-                        $info.html(formatVideoInfo(info));
-                        $status.html(
-                            info.direct
-                            ? Ox._(
-                                'Your video will be used directly.'
-                            )
-                            : Ox._(
-                                'Your video will be transcoded on the server.'
-                            )
-                        );
-                    });
-                    $closeButton.options('title', Ox._('Cancel'));
-                }
-            }
-        });
-    }
-
-    function uploadStream(item, info, file) {
-        var oshash = info.oshash,
-            format = pandora.site.video.formats[0],
-            resolution = getResolution(info);
-        pandora.$ui.upload = pandora.chunkupload({
-            file: file,
-            url: '/api/upload/?profile=' + resolution + 'p.'
-                + format + '&id=' + oshash,
-            data: {}
-        }).bindEvent({
-            done: function(data) {
-                if (data.progress == 1) {
-                    Ox.Request.clearCache();
-                    if (
-                        pandora.user.ui.item == item
-                        && pandora.user.ui.itemView == itemView
-                    ) {
-                        pandora.$ui.item.reload();
-                    } else {
-                        pandora.UI.set({
-                            item: item,
-                            itemView: itemView
-                        });
-                    }
-                    delete pandora.firefogg;
-                    that.close();
-                } else {
-                    $status.html(Ox._('Upload failed.'));
-                    pandora.api.logError({
-                        text: data.responseText,
-                        url: '/' + item,
-                        line: 1
-                    });
-                }
-            },
-            progress: function(data) {
-                $progress.options({progress: data.progress || 0});
-            },
-        });
-    }
-
-    function upload(file) {
-        resetProgress();
-        $info.html(Ox._('Uploading {0}', [file.name]));
-        Ox.oshash(file, function(oshash) {
-            pandora.api.findMedia({
-                query: {
-                    conditions: [{key: 'oshash', value: oshash}]
-                },
-                keys: ['id', 'item', 'available']
-            }, function(result) {
-                if (
-                    result.data.items.length === 0
-                    || !result.data.items[0].available
-                ) {
-                    pandora.api.addMedia({
-                        filename: file.name,
-                        id: oshash,
-                        item: pandora.site.itemRequiresVideo
-                            ? undefined
-                            : pandora.user.ui.item
-                    }, function(result) {
-                        var item = result.data.item;
-                        pandora.$ui.upload = pandora.chunkupload({
-                            file: file,
-                            url: '/api/upload/direct/',
-                            data: {
-                                id: oshash
-                            }
-                        }).bindEvent({
-                            done: function(data) {
-                                if (data.progress == 1) {
-                                    Ox.Request.clearCache();
-                                    if (
-                                        pandora.user.ui.item == item
-                                        && pandora.user.ui.itemView == itemView
-                                    ) {
-                                        pandora.$ui.item.reload();
-                                    } else {
-                                        pandora.UI.set({
-                                            item: item,
-                                            itemView: itemView
-                                        });
-                                    }
-                                    that.close();
-                                } else {
-                                    $status.html(
-                                        cancelled
-                                        ? Ox._('Upload cancelled.')
-                                        : Ox._('Upload failed.')
-                                    );
-                                    !cancelled && pandora.api.logError({
-                                        text: data.responseText,
-                                        url: '/' + item,
-                                        line: 1
-                                    });
-                                }
-                            },
-                            progress: function(data) {
-                                $progress.options({
-                                    progress: data.progress || 0
-                                });
-                            }
-                        });
-                    });
-                } else {
-                    pandora.UI.set({
-                        item: result.data.items[0].item,
-                        itemView: itemView
-                    });
-                    that.close();
-                }
-            });
-        });
-    }
-
-    function getEncodingOptions(info) {
-        var bpp = 0.17,
-            dar,
-            format = pandora.site.video.formats[0],
-            fps,
-            options = {},
-            resolution = getResolution(info);
-        if (format == 'webm') {
-            options.videoCodec = 'vp8';
-            options.audioCodec = 'vorbis';
-        } else if (format == 'ogv') {
-            options.videoCodec = 'theora';
-            options.audioCodec = 'vorbis';
-        }
-        if (resolution == 720) {
-            options.height = 720;
-            options.samplerate = 48000;
-            options.audioQuality = 5;
-        } else if (resolution == 480) {
-            options.height = 480;
-            options.samplerate = 44100;
-            options.audioQuality = 3;
-            options.channels = 2;
-        } else if (resolution == 432) {
-            options.height = 432;
-            options.samplerate = 44100;
-            options.audioQuality = 3;
-            options.channels = 2;
-        } else if (resolution == 360) {
-            options.height = 320;
-            options.samplerate = 44100;
-            options.audioQuality = 1;
-            options.channels = 1;
-        } else if (resolution == 288) {
-            options.height = 288;
-            options.samplerate = 44100;
-            options.audioQuality = 0;
-            options.channels = 1;
-        } else if (resolution == 240) {
-            options.height = 240;
-            options.samplerate = 44100;
-            options.audioQuality = 0;
-            options.channels = 1;
-        } else if (resolution == 144) {
-            options.height = 144;
-            options.samplerate = 22050;
-            options.audioQuality = -1;
-            options.audioBitrate = 22;
-            options.channels = 1;
-        } else if (resolution == 96) {
-            options.height = 96;
-            options.samplerate = 22050;
-            options.audioQuality = -1;
-            options.audioBitrate = 22;
-            options.channels = 1;
-        }
-        if (info.video && info.video.length) {
-            info.video.forEach(function(video) {
-                if (!video.display_aspect_ratio) {
-                    video.display_aspect_ratio = video.width + ':' + video.height;
-                    video.pixel_aspect_ratio = '1:1';
-                }
-            });
-            dar = aspectratio(info.video[0].display_aspect_ratio);
-            fps = aspectratio(info.video[0].framerate).float;
-            options.width = parseInt(dar.float * options.height, 10);
-            options.width += options.width % 2;
-            // interlaced hdv material is detected with double framerates
-            if (fps == 50) {
-                fps = options.framerate = 25;
-            } else if (fps == 60) {
-                fps = options.framerate = 30;
-            }
-            if (Math.abs(options.width/options.height - dar.float) < 0.02) {
-                options.aspect = options.width + ':' + options.height;
-            } else {
-                options.aspect = dar.ratio;
-            }
-            options.videoBitrate = Math.round(
-                options.height * options.width * fps * bpp / 1000
-            );
-            options.denoise = true; 
-            options.deinterlace = true; 
-        } else {
-            options.noVideo = true;
-        }
-        if (info.audio) {
-            if (options.cannels && info.audio[0].channels < options.channels) {
-                delete options.channels;
-            }
-        } else {
-            options.noAudio = true;
-            delete options.samplerate;
-            delete options.audioQuality;
-            delete options.channels;
-        }
-        options.noUpscaling = true;
-        if (
-            (!info.video.length || (
-                info.video[0].codec == options.videoCodec
-                && info.video[0].height <= options.height
-            ))
-            && (!info.audio.length || info.audio[0].codec == options.audioCodec)
-        ) {
-            options = {passthrough: true};
-        }
-        return JSON.stringify(options);
-    }
-
-    function formatInfo(info) {
-        var html = '<b>' + info.path + '</b><br>';
-        if (info.video && info.video.length > 0) {
-            var video = info.video[0];
-            html += video.width + '×' + video.height  + ' (' + video.codec + ')';
-        }
-        if (
-            info.video && info.video.length > 0
-            && info.audio && info.audio.length > 0
-        ) {
-            html += ' / ';
-        }
-        if (info.audio && info.audio.length > 0) {
-            var audio = info.audio[0];
-            html += {1: 'mono', 2: 'stereo', 6: '5.1'}[audio.channels]
-                + ' ' + audio.samplerate / 1000 + ' kHz (' + audio.codec + ')';
-        }
-        html += '<br>' + Ox.formatValue(info.size, 'B')
-            + ' / ' + Ox.formatDuration(info.duration);
-        return html; 
-    }
-
-    function formatVideoInfo(info) {
-        var html = '<b>' + info.name + '</b><br>';
-        if (info.video && info.video.length > 0) {
-            html += info.video[0].width + '×' + info.video[0].height;
-        }
-        html += '<br>' + Ox.formatValue(info.size, 'B');
-        if(info.duration) {
-            html += ' / ' + Ox.formatDuration(info.duration);
-        }
-        return html; 
-    }
-
-    function selectVideo() {
-        cancelled = false;
-        pandora.firefogg = new Firefogg();
-        pandora.firefogg.setFormat(pandora.site.video.formats[0]);
-        if (pandora.firefogg.selectVideo()) {
-            var info = JSON.parse(pandora.firefogg.sourceInfo),
-                options = JSON.parse(getEncodingOptions(info)),
-                oshash = info.oshash,
-                filename = pandora.firefogg.sourceFilename,
-                item;
-            pandora.api.findMedia({
-                query: {
-                    conditions: [{key: 'oshash', value: oshash}]
-                },
-                keys: ['id', 'available']
-            }, function(result) {
-                if (
-                    result.data.items.length === 0
-                    || !result.data.items[0].available
-                ) {
-                    $info.html(formatInfo(info));
-                    $status.html(
-                        options.passthrough
-                        ? Ox._('Your video will be uploaded directly.')
-                        : Ox._('Your video will be transcoded before upload.')
-                    );
-                } else {
-                    pandora.api.find({
-                        query: {
-                            conditions: [{key: 'oshash', value: oshash}]
-                        },
-                        keys: ['id']
-                    }, function(result) {
-                        pandora.UI.set({
-                            item: result.data.items[0].id,
-                            itemView: itemView
-                        });
-                        delete pandora.firefogg;
-                        that.close();
-                    });
-                }
-            });
-            return true;
-        }
-        return false;
-    }
-
-    return that;
-
-};
diff --git a/static/js/utils.js b/static/js/utils.js
index 79ab43d8..b111f0e4 100644
--- a/static/js/utils.js
+++ b/static/js/utils.js
@@ -3218,9 +3218,8 @@ pandora.updateStatus = function(item) {
         return ui.item == item && [
             'info', 'player', 'editor', 'timeline'
         ].indexOf(ui.itemView) > -1 && !(
-            // fixme: still wrong
-            pandora.$ui.uploadVideoDialog
-            && pandora.$ui.uploadVideoDialog.is('::visible')
+            pandora.$ui.addItemDialog
+            && pandora.$ui.addItemDialog.is('::visible')
         );
     }
 };

From daed8b816c649cb01a3808d7fa3a284966212977 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Tue, 23 Jul 2019 11:36:31 +0200
Subject: [PATCH 010/139] add loading screen while preparing files for upload

---
 static/js/addFilesDialog.js | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/static/js/addFilesDialog.js b/static/js/addFilesDialog.js
index e718b218..6a31b26f 100644
--- a/static/js/addFilesDialog.js
+++ b/static/js/addFilesDialog.js
@@ -8,6 +8,11 @@ pandora.ui.addFilesDialog = function(options) {
     }).bindEvent({
         click: function() {
             $button.options({disabled: true});
+            that.disableCloseButton()
+            var $screen = Ox.LoadingScreen({
+                size: 16
+            });
+            that.options({content: $screen.start()});
             (options.action == 'upload' ? uploadVideos : importVideos)(function() {
                 that.close();
                 pandora.ui.tasksDialog({

From 0e21f9ebc1a99d321d81e8af05cfbae245e889b9 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Tue, 23 Jul 2019 16:16:15 +0200
Subject: [PATCH 011/139] imdb can also be 8 digits

---
 static/js/infoView.indiancinema.js | 2 +-
 static/js/mediaView.js             | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/static/js/infoView.indiancinema.js b/static/js/infoView.indiancinema.js
index 3353e3fe..ed56ecaa 100644
--- a/static/js/infoView.indiancinema.js
+++ b/static/js/infoView.indiancinema.js
@@ -656,7 +656,7 @@ pandora.ui.infoView = function(data, isMixed) {
                     ? Ox.decodeHTMLEntities(value).split('; ').map(Ox.encodeHTMLEntities)
                     : [];
             } else if (key == 'imdbId') {
-                edit[key] = value ? value.match(/\d{7}/)[0] : value;
+                edit[key] = value ? value.match(/\d+?/)[0] : value;
             } else {
                 edit[key] = value;
             }
diff --git a/static/js/mediaView.js b/static/js/mediaView.js
index 1c425f76..f7ad8e4e 100644
--- a/static/js/mediaView.js
+++ b/static/js/mediaView.js
@@ -406,7 +406,7 @@ pandora.ui.mediaView = function(options) {
                 var conditions, matches;
                 if (key == 'id' && data.value.substr(0, 2) != '0x') {
                     if (pandora.site.site.id == '0xdb') {
-                        matches = data.value.match(/\d{7}/);
+                        matches = data.value.match(/\d+?/);
                     } else {
                         matches = data.value.match(/[A-Z]+/);
                     }

From b1e6e37b7e3d7992858d6fe7147ebe877a6c4329 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Tue, 23 Jul 2019 16:39:41 +0200
Subject: [PATCH 012/139] as many digits as possible

---
 static/js/infoView.indiancinema.js | 2 +-
 static/js/mediaView.js             | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/static/js/infoView.indiancinema.js b/static/js/infoView.indiancinema.js
index ed56ecaa..9776dd42 100644
--- a/static/js/infoView.indiancinema.js
+++ b/static/js/infoView.indiancinema.js
@@ -656,7 +656,7 @@ pandora.ui.infoView = function(data, isMixed) {
                     ? Ox.decodeHTMLEntities(value).split('; ').map(Ox.encodeHTMLEntities)
                     : [];
             } else if (key == 'imdbId') {
-                edit[key] = value ? value.match(/\d+?/)[0] : value;
+                edit[key] = value ? value.match(/\d+/)[0] : value;
             } else {
                 edit[key] = value;
             }
diff --git a/static/js/mediaView.js b/static/js/mediaView.js
index f7ad8e4e..f7773148 100644
--- a/static/js/mediaView.js
+++ b/static/js/mediaView.js
@@ -406,7 +406,7 @@ pandora.ui.mediaView = function(options) {
                 var conditions, matches;
                 if (key == 'id' && data.value.substr(0, 2) != '0x') {
                     if (pandora.site.site.id == '0xdb') {
-                        matches = data.value.match(/\d+?/);
+                        matches = data.value.match(/\d+/);
                     } else {
                         matches = data.value.match(/[A-Z]+/);
                     }

From 7a539b37b81bec550ee416af27f94772c0219605 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Tue, 23 Jul 2019 16:47:22 +0200
Subject: [PATCH 013/139] imdb ids can be longer

---
 .../migrations/0002_auto_20190723_1446.py     | 20 +++++++++++++++++++
 pandora/person/models.py                      |  2 +-
 .../migrations/0002_auto_20190723_1446.py     | 20 +++++++++++++++++++
 pandora/title/models.py                       |  2 +-
 4 files changed, 42 insertions(+), 2 deletions(-)
 create mode 100644 pandora/person/migrations/0002_auto_20190723_1446.py
 create mode 100644 pandora/title/migrations/0002_auto_20190723_1446.py

diff --git a/pandora/person/migrations/0002_auto_20190723_1446.py b/pandora/person/migrations/0002_auto_20190723_1446.py
new file mode 100644
index 00000000..acb168e1
--- /dev/null
+++ b/pandora/person/migrations/0002_auto_20190723_1446.py
@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.22 on 2019-07-23 14:46
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('person', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='person',
+            name='imdbId',
+            field=models.CharField(blank=True, max_length=16),
+        ),
+    ]
diff --git a/pandora/person/models.py b/pandora/person/models.py
index 20c637c4..84fe86d6 100644
--- a/pandora/person/models.py
+++ b/pandora/person/models.py
@@ -38,7 +38,7 @@ class Person(models.Model):
     #FIXME: how to deal with aliases
     aliases = fields.TupleField(default=[])
 
-    imdbId = models.CharField(max_length=7, blank=True)
+    imdbId = models.CharField(max_length=16, blank=True)
     wikipediaId = models.CharField(max_length=1000, blank=True)
 
     objects = managers.PersonManager()
diff --git a/pandora/title/migrations/0002_auto_20190723_1446.py b/pandora/title/migrations/0002_auto_20190723_1446.py
new file mode 100644
index 00000000..c1a36a2b
--- /dev/null
+++ b/pandora/title/migrations/0002_auto_20190723_1446.py
@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.22 on 2019-07-23 14:46
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('title', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='title',
+            name='imdbId',
+            field=models.CharField(blank=True, max_length=16),
+        ),
+    ]
diff --git a/pandora/title/models.py b/pandora/title/models.py
index 11c12339..fceb00c6 100644
--- a/pandora/title/models.py
+++ b/pandora/title/models.py
@@ -32,7 +32,7 @@ class Title(models.Model):
     sortsorttitle = models.CharField(max_length=1000)
     edited = models.BooleanField(default=False)
 
-    imdbId = models.CharField(max_length=7, blank=True)
+    imdbId = models.CharField(max_length=16, blank=True)
 
     objects = managers.TitleManager()
 

From ec3175d8854fb0a74fa831e4c374372856d9f209 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Tue, 23 Jul 2019 17:42:56 +0200
Subject: [PATCH 014/139] imdb check

---
 pandora/item/models.py | 10 +++++-----
 pandora/item/utils.py  |  4 ++++
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/pandora/item/models.py b/pandora/item/models.py
index 5dae6d91..5c5de0af 100644
--- a/pandora/item/models.py
+++ b/pandora/item/models.py
@@ -387,7 +387,7 @@ class Item(models.Model):
             if self.oxdbId != oxdbId:
                 q = Item.objects.filter(oxdbId=oxdbId).exclude(id=self.id)
                 if q.count() != 0:
-                    if len(self.public_id) == 7:
+                    if utils.is_imdb_id(self.public_id):
                         self.oxdbId = None
                         q[0].merge_with(self, save=False)
                     else:
@@ -401,14 +401,14 @@ class Item(models.Model):
                             q = Item.objects.filter(oxdbId=oxdbId).exclude(id=self.id)
                 self.oxdbId = oxdbId
                 update_poster = True
-                if len(self.public_id) != 7:
+                if not utils.is_imdb_id(self.public_id):
                     update_ids = True
 
         # id changed, what about existing item with new id?
-        if settings.USE_IMDB and len(self.public_id) != 7 and self.oxdbId != self.public_id:
+        if settings.USE_IMDB and not utils.is_imdb_id(self.public_id) and self.oxdbId != self.public_id:
             self.public_id = self.oxdbId
             # FIXME: move files to new id here
-        if settings.USE_IMDB and len(self.public_id) == 7:
+        if settings.USE_IMDB and utils.is_imdb_id(self.public_id):
             for key in ('title', 'year', 'director', 'season', 'episode',
                         'seriesTitle', 'episodeTitle'):
                 if key in self.data:
@@ -418,7 +418,7 @@ class Item(models.Model):
         if settings.USE_IMDB:
             defaults = list(filter(lambda k: 'default' in k, settings.CONFIG['itemKeys']))
             for k in defaults:
-                if len(self.public_id) == 7:
+                if utils.is_imdb_id(self.public_id):
                     if k['id'] in self.data and self.data[k['id']] == k['default']:
                         del self.data[k['id']]
                 else:
diff --git a/pandora/item/utils.py b/pandora/item/utils.py
index 38ca83ff..9d0beb66 100644
--- a/pandora/item/utils.py
+++ b/pandora/item/utils.py
@@ -103,3 +103,7 @@ def normalize_dict(encoding, data):
     elif isinstance(data, list):
         return [normalize_dict(encoding, value) for value in data]
     return data
+
+
+def is_imdb_id(id):
+    return bool(len(id) >= 7 and str(id).isdigit())

From 87de67be98e603ba3f85ae803e9b587023f75062 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Tue, 23 Jul 2019 17:45:30 +0200
Subject: [PATCH 015/139] one more imdb id check

---
 pandora/archive/tasks.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/pandora/archive/tasks.py b/pandora/archive/tasks.py
index e9cc8712..6bb34bb6 100644
--- a/pandora/archive/tasks.py
+++ b/pandora/archive/tasks.py
@@ -210,6 +210,7 @@ def download_media(item_id, url):
 def move_media(data, user):
     from changelog.models import add_changelog
     from item.models import get_item, Item
+    from item.utils import is_imdb_id
     from annotation.models import Annotation
 
     user = models.User.objects.get(username=user)
@@ -218,7 +219,7 @@ def move_media(data, user):
         i = Item.objects.get(public_id=data['item'])
     else:
         data['public_id'] = data.pop('item').strip()
-        if len(data['public_id']) != 7:
+        if not is_imdb_id(data['public_id']):
             del data['public_id']
             if 'director' in data and isinstance(data['director'], string_types):
                 if data['director'] == '':

From e8a13b4e4de05c8275f87c83a3f1380f3316ee1b Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Tue, 23 Jul 2019 17:52:23 +0200
Subject: [PATCH 016/139] fix merge with

---
 pandora/item/models.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pandora/item/models.py b/pandora/item/models.py
index 5c5de0af..de129388 100644
--- a/pandora/item/models.py
+++ b/pandora/item/models.py
@@ -389,6 +389,7 @@ class Item(models.Model):
                 if q.count() != 0:
                     if utils.is_imdb_id(self.public_id):
                         self.oxdbId = None
+                        self.update_sort()
                         q[0].merge_with(self, save=False)
                     else:
                         n = 1

From 91a7c0cedea79071e5720b091471ff5dc4b11e16 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Tue, 23 Jul 2019 17:55:08 +0200
Subject: [PATCH 017/139] another place sort is required now

---
 pandora/archive/tasks.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/pandora/archive/tasks.py b/pandora/archive/tasks.py
index 6bb34bb6..2443149a 100644
--- a/pandora/archive/tasks.py
+++ b/pandora/archive/tasks.py
@@ -209,7 +209,7 @@ def download_media(item_id, url):
 @task(queue='default')
 def move_media(data, user):
     from changelog.models import add_changelog
-    from item.models import get_item, Item
+    from item.models import get_item, Item, ItemSort
     from item.utils import is_imdb_id
     from annotation.models import Annotation
 
@@ -229,6 +229,11 @@ def move_media(data, user):
             i = get_item(data, user=user)
         else:
             i = get_item({'imdbId': data['public_id']}, user=user)
+    try:
+        i.sort
+    except ItemSort.DoesNotExist:
+        i.update_sort()
+
     changed = [i.public_id]
     old_item = None
     for f in models.File.objects.filter(oshash__in=data['ids']):

From c6ba56188ddb1f245f29273dfdacd0f3ea7a4da5 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Sun, 28 Jul 2019 21:31:49 +0200
Subject: [PATCH 018/139] don't fail if item is gone

---
 pandora/taskqueue/models.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/pandora/taskqueue/models.py b/pandora/taskqueue/models.py
index 6a36e3bf..9dda9da1 100644
--- a/pandora/taskqueue/models.py
+++ b/pandora/taskqueue/models.py
@@ -183,13 +183,17 @@ class Task(models.Model):
     def json(self):
         if self.status != 'canceled':
             self.update()
-        return {
+        data = {
             'started': self.started,
             'ended': self.ended,
             'status': self.status,
-            'title': self.item.get('title'),
-            'item': self.item.public_id,
-            'user': self.user and self.user.username or '',
             'id': self.public_id,
+            'user': self.user and self.user.username or '',
         }
+        try:
+            data['title'] = self.item.get('title')
+            data['item'] = self.item.public_id
+        except:
+            pass
+        return data
 

From f2e7cdbea8cc37428e2429a880afec5b57ac2423 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Mon, 29 Jul 2019 11:57:20 +0200
Subject: [PATCH 019/139] reuse empty items

---
 static/js/addFilesDialog.js | 47 +++++++++++++++++++++++++++++++++++--
 1 file changed, 45 insertions(+), 2 deletions(-)

diff --git a/static/js/addFilesDialog.js b/static/js/addFilesDialog.js
index 6a31b26f..2a4e023c 100644
--- a/static/js/addFilesDialog.js
+++ b/static/js/addFilesDialog.js
@@ -149,6 +149,49 @@ pandora.ui.addFilesDialog = function(options) {
     $($select.find('.OxButton')[0]).css({margin: '-1px'});
     $button.parent().parent().append($select);
 
+    function getNewOrEmptyItem(data, callback) {
+        pandora.api.find({
+            query: {
+                conditions: [
+                    {key: 'title', value: data.title, operator: '=='}
+                ]
+            },
+            keys: ['id']
+        }, function(result) {
+            if (!result.data.items.length) {
+                pandora.api.add(data, callback)
+            } else {
+                var isNew = true
+                Ox.serialForEach(result.data.items, function(item, index, items, next) {
+                    isNew && pandora.api.findMedia({
+                        query: {
+                            conditions: [
+                                {key: 'id', value: item.id, operator: '=='}
+                            ]
+                        },
+                        keys: ['id']
+                    }, function(result) {
+                        if (!result.data.items.length) {
+                            isNew = false
+                            callback({
+                                data: {
+                                    title: data.title,
+                                    id: item.id
+                                }
+                            })
+                        }
+                        next()
+                    })
+                }, function() {
+                    if (isNew) {
+                        pandora.api.add(data, callback)
+                    }
+                })
+
+            }
+        })
+    }
+
     function importVideos(callback) {
         var id, title;
         ($select.value() == 'add' ? pandora.api.get : Ox.noop)({
@@ -166,7 +209,7 @@ pandora.ui.addFilesDialog = function(options) {
                 } else {
                     title = items[$select.value() == 'one' ? 0 : index].title;
                 }
-                (isNewItem ? pandora.api.add : Ox.noop)({
+                (isNewItem ? getNewOrEmptyItem : Ox.noop)({
                     title: title
                 }, function(result) {
                     if (isNewItem) {
@@ -206,7 +249,7 @@ pandora.ui.addFilesDialog = function(options) {
                 } else {
                     title = items[$select.value() == 'one' ? 0 : index].title;
                 }
-                (isNewItem ? pandora.api.add : Ox.noop)({
+                (isNewItem ? getNewOrEmptyItem : Ox.noop)({
                     title: title
                 }, function(result) {
                     if (isNewItem) {

From 0147b56d6955d5f32e1dfce9f45f66ce3e3961c9 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Mon, 29 Jul 2019 12:27:32 +0200
Subject: [PATCH 020/139] get all matches

---
 static/js/addItemDialog.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/static/js/addItemDialog.js b/static/js/addItemDialog.js
index 71361fda..d41a9ed3 100644
--- a/static/js/addItemDialog.js
+++ b/static/js/addItemDialog.js
@@ -230,6 +230,7 @@ pandora.ui.addItemDialog = function(options) {
         // FIXME: what about pending/aborted uploads
         pandora.api.findMedia({
             keys: ['id', 'item', 'url'],
+            range: [0, items.length],
             query: {
                 conditions: selected == 'upload' ? items.map(function(item) {
                     return {key: 'oshash', operator: '==', value: item.oshash};

From 0dc1f22abfda08a1963dfadf3d6d9218234a97af Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Tue, 30 Jul 2019 13:10:21 +0200
Subject: [PATCH 021/139] work around chrome autocomplete for passwords

---
 static/js/preferencesDialog.js | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/static/js/preferencesDialog.js b/static/js/preferencesDialog.js
index 70af78f2..25faaf54 100644
--- a/static/js/preferencesDialog.js
+++ b/static/js/preferencesDialog.js
@@ -43,7 +43,7 @@ pandora.ui.preferencesDialog = function() {
                                         id: 'password',
                                         label: Ox._('New Password'),
                                         labelWidth: 120,
-                                        type: 'password',
+                                        type: 'text',
                                         validate: pandora.validateNewPassword,
                                         width: 320
                                     })
@@ -51,6 +51,11 @@ pandora.ui.preferencesDialog = function() {
                                         autocomplete: 'new-password'
                                     })
                                     .bindEvent({
+                                        focus: function(data) {
+                                            this.options({
+                                                type: 'password'
+                                            })
+                                        },
                                         validate: function(data) {
                                             data.valid && pandora.api.editPreferences({password: data.value});
                                         }

From 794b28f833634e00f8f573043cc3fea9791c057d Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Tue, 30 Jul 2019 13:24:07 +0200
Subject: [PATCH 022/139] stay logged in after changing password

---
 pandora/user/views.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/pandora/user/views.py b/pandora/user/views.py
index d9639d70..88dc2276 100644
--- a/pandora/user/views.py
+++ b/pandora/user/views.py
@@ -6,7 +6,7 @@ random.seed()
 import re
 import json
 
-from django.contrib.auth import authenticate, login, logout
+from django.contrib.auth import authenticate, login, logout, update_session_auth_hash
 from django.template import loader
 from django.conf import settings
 from django.core.mail import send_mail, BadHeaderError, EmailMessage
@@ -719,7 +719,9 @@ def editPreferences(request, data):
         profile.save()
     if 'password' in data:
         change = True
-        request.user.set_password(data['password'])
+        user = request.user
+        user.set_password(data['password'])
+        update_session_auth_hash(request, user)
     if 'script' in data:
         profile = request.user.profile
         profile.preferences['script'] = data['script']

From 9596248dc4733dabe6a54e22fcec67bd343c2019 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Wed, 31 Jul 2019 11:53:30 +0200
Subject: [PATCH 023/139] use refresh_from_db

---
 pandora/archive/models.py | 5 ++---
 pandora/archive/tasks.py  | 7 +++----
 2 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/pandora/archive/models.py b/pandora/archive/models.py
index 306590b2..3b437fed 100644
--- a/pandora/archive/models.py
+++ b/pandora/archive/models.py
@@ -756,9 +756,8 @@ class Stream(models.Model):
         ok, error = extract.stream(media, target, self.name(), info, flags=self.flags)
         # file could have been moved while encoding
         # get current version from db and update
-        _self = Stream.objects.get(id=self.id)
-        _self.update_status(ok, error)
-        return _self
+        self.refresh_from_db()
+        self.update_status(ok, error)
 
     def get_index(self):
         index = 1
diff --git a/pandora/archive/tasks.py b/pandora/archive/tasks.py
index 2443149a..92446d58 100644
--- a/pandora/archive/tasks.py
+++ b/pandora/archive/tasks.py
@@ -128,7 +128,7 @@ def process_stream(fileId):
         stream = streams[0]
         stream.make_timeline()
         stream.extract_derivatives()
-        file = models.File.objects.get(id=fileId)
+        file.refresh_from_db()
         file.encoding = False
         file.save()
     file.item.update_selected()
@@ -158,13 +158,12 @@ def extract_stream(fileId):
         if created:
             file.extract_frames()
             stream.media.name = stream.path(stream.name())
-            stream = stream.encode()
+            stream.encode()
             if stream.available:
                 stream.make_timeline()
                 stream.extract_derivatives()
                 file.extract_tracks()
-                # get current version from db
-                file = models.File.objects.get(id=fileId)
+                file.refresh_from_db()
                 if not file.item.rendered \
                         and not file.item.files.exclude(id=fileId).filter(Q(queued=True) | Q(encoding=True)).count():
                     file.item.update_timeline()

From 02a2d00f9dcc8ebf629e839230cc0136eb6ae946 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Thu, 1 Aug 2019 15:36:06 +0200
Subject: [PATCH 024/139] fix findPlace performance

---
 pandora/place/managers.py | 1 +
 pandora/place/views.py    | 1 -
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/pandora/place/managers.py b/pandora/place/managers.py
index ad824591..75d59b1d 100644
--- a/pandora/place/managers.py
+++ b/pandora/place/managers.py
@@ -141,4 +141,5 @@ class PlaceManager(Manager):
                                      user)
         if conditions:
             qs = qs.filter(conditions)
+        qs = qs.distinct()
         return qs
diff --git a/pandora/place/views.py b/pandora/place/views.py
index 9b9ba72f..0b4c105a 100644
--- a/pandora/place/views.py
+++ b/pandora/place/views.py
@@ -239,7 +239,6 @@ def findPlaces(request, data):
     qs = order_query(query['qs'], query['sort'])
     qs = qs.distinct()
     if 'keys' in data:
-        qs = qs.select_related('user__profile')
         qs = qs[query['range'][0]:query['range'][1]]
         response['data']['items'] = [p.json(data['keys'], request.user) for p in qs]
     elif 'position' in query:

From 8b30add49e027b9609dcf82fe5c12ae11994f7c8 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Thu, 1 Aug 2019 15:39:02 +0200
Subject: [PATCH 025/139] readd selected_related but without arguments

---
 pandora/place/views.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pandora/place/views.py b/pandora/place/views.py
index 0b4c105a..01533b52 100644
--- a/pandora/place/views.py
+++ b/pandora/place/views.py
@@ -240,6 +240,7 @@ def findPlaces(request, data):
     qs = qs.distinct()
     if 'keys' in data:
         qs = qs[query['range'][0]:query['range'][1]]
+        qs = qs.select_related()
         response['data']['items'] = [p.json(data['keys'], request.user) for p in qs]
     elif 'position' in query:
         ids = [i.get_id() for i in qs]

From a149c7d7a07ae13bb69f91d29234115a8ea80215 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Thu, 1 Aug 2019 15:47:20 +0200
Subject: [PATCH 026/139] item sort is directly attached to clip

---
 pandora/clip/views.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/pandora/clip/views.py b/pandora/clip/views.py
index df2c9103..700a95a1 100644
--- a/pandora/clip/views.py
+++ b/pandora/clip/views.py
@@ -102,8 +102,6 @@ def findClips(request, data):
         subtitles = utils.get_by_key(layers, 'isSubtitles', True)
         layer_ids = [k['id'] for k in layers]
         keys = list(filter(lambda k: k not in layer_ids + ['annotations'], data['keys']))
-        if list(filter(lambda k: k not in models.Clip.clip_keys, keys)):
-            qs = qs.select_related('item__sort')
 
         clips = {}
         response['data']['items'] = clip_jsons = []

From 6706b936144376c4e9ffaaebbf38825a8a1cade6 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Thu, 1 Aug 2019 17:00:02 +0200
Subject: [PATCH 027/139] minor performance improvements

---
 pandora/edit/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pandora/edit/models.py b/pandora/edit/models.py
index 2dfe2ee0..4ce6ab29 100644
--- a/pandora/edit/models.py
+++ b/pandora/edit/models.py
@@ -248,7 +248,7 @@ class Edit(models.Model):
             clips_query = self.clip_query()
             if clips_query['conditions']:
                 clips = clip.models.Clip.objects.find({'query': clips_query}, user)
-                items = [i['id'] for i in self.get_items(user).values('id')]
+                items = self.get_items(user).values('id')
                 clips = clips.filter(item__in=items)
             else:
                 clips = clip.models.Clip.objects.filter(id=None)

From 9a6b9a0bbb40bfe4f6663206e7fdcf2641641a44 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Fri, 2 Aug 2019 11:44:31 +0200
Subject: [PATCH 028/139] don't reload filters, fixes #3225

---
 static/js/mainPanel.js | 35 +++++++++++++++++++++++++++++++++--
 1 file changed, 33 insertions(+), 2 deletions(-)

diff --git a/static/js/mainPanel.js b/static/js/mainPanel.js
index e94f700c..7460a3c8 100644
--- a/static/js/mainPanel.js
+++ b/static/js/mainPanel.js
@@ -23,8 +23,39 @@ pandora.ui.mainPanel = function() {
         .bindEvent({
             pandora_finddocuments: function() {
                 var previousUI = pandora.UI.getPrevious();
-                if (!previousUI.document && ui._list == previousUI._list) {
-                    that.replaceElement(1, pandora.$ui.documentPanel = pandora.ui.documentPanel());
+                Ox.Log('FIND', 'handled in mainPanel', previousUI.item, previousUI._list)
+                if (!previousUI.document && ui._collection == previousUI._collection) {
+                    pandora.$ui.list.reloadList();
+
+                    // FIXME: why is this being handled _here_?
+                    ui._documentFilterState.forEach(function(data, i) {
+                        if (!Ox.isEqual(data.selected, previousUI._documentFilterState[i].selected)) {
+                            pandora.$ui.documentFilters[i].options(
+                                ui.showFilters ? {
+                                    selected: data.selected
+                                } : {
+                                    _selected: data.selected,
+                                    selected: []
+                                }
+                            );
+                        }
+                        if (!Ox.isEqual(data.find, previousUI._documentFilterState[i].find)) {
+                            if (!ui.showFilters) {
+                                pandora.$ui.documentFilters[i].options({
+                                    _selected: data.selected
+                                });
+                            }
+                            // we can call reloadList here, since the items function
+                            // handles the hidden filters case without making requests
+                            pandora.$ui.documentFilters[i].reloadList();
+                        }
+                    });
+                } else {
+                    if (pandora.stayInItemView) {
+                        pandora.stayInItemView = false;
+                    } else {
+                        that.replaceElement(1, pandora.$ui.rightPanel = pandora.ui.rightPanel());
+                    }
                 }
             },
             pandora_document: function(data) {

From df5dabbec534487ea26e1a92d557918c92af5ab6 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Fri, 2 Aug 2019 13:15:51 +0200
Subject: [PATCH 029/139] imdb id sort

---
 pandora/item/models.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/pandora/item/models.py b/pandora/item/models.py
index de129388..f3a1aa25 100644
--- a/pandora/item/models.py
+++ b/pandora/item/models.py
@@ -939,6 +939,8 @@ class Item(models.Model):
         s.oxdbId = self.oxdbId
         if not settings.USE_IMDB and s.public_id.isupper() and s.public_id.isalpha():
             s.public_id = ox.sort_string(str(ox.fromAZ(s.public_id)))
+        else:
+            s.public_id = ox.sort_string(str(ox.fromAZ(s.public_id)))
         s.modified = self.modified or datetime.now()
         s.created = self.created or datetime.now()
         s.rightslevel = self.level

From 9686aab8603399fe2c37562eadc0a234d54c10d2 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Fri, 2 Aug 2019 13:40:02 +0200
Subject: [PATCH 030/139] remove fromAZ

---
 pandora/item/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pandora/item/models.py b/pandora/item/models.py
index f3a1aa25..16c4b2ca 100644
--- a/pandora/item/models.py
+++ b/pandora/item/models.py
@@ -940,7 +940,7 @@ class Item(models.Model):
         if not settings.USE_IMDB and s.public_id.isupper() and s.public_id.isalpha():
             s.public_id = ox.sort_string(str(ox.fromAZ(s.public_id)))
         else:
-            s.public_id = ox.sort_string(str(ox.fromAZ(s.public_id)))
+            s.public_id = ox.sort_string(s.public_id)
         s.modified = self.modified or datetime.now()
         s.created = self.created or datetime.now()
         s.rightslevel = self.level

From ac7ffb184585632b63d85d06e4547b383244d071 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Wed, 7 Aug 2019 10:55:48 +0200
Subject: [PATCH 031/139] clenaup

---
 pandora/item/management/commands/update_external.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pandora/item/management/commands/update_external.py b/pandora/item/management/commands/update_external.py
index aea8c530..4dbd1cbd 100644
--- a/pandora/item/management/commands/update_external.py
+++ b/pandora/item/management/commands/update_external.py
@@ -17,9 +17,9 @@ class Command(BaseCommand):
 
     def add_arguments(self, parser):
         parser.add_argument('--all', action='store_true', dest='all',
-            default=False, help='update all items, otherwise oldes N'),
+            default=False, help='update all items, otherwise oldes N')
         parser.add_argument('-n', '--items', action='store', dest='items', type=int,
-            default=30, help='number of items ot update'),
+            default=30, help='number of items ot update')
 
     def handle(self, **options):
         offset = 0

From 72272dfde45b9b4d61b5bd65563eeae499179044 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Wed, 7 Aug 2019 10:55:54 +0200
Subject: [PATCH 032/139] avoid double titles

---
 static/js/idDialog.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/static/js/idDialog.js b/static/js/idDialog.js
index aab1adc6..ba82d19f 100644
--- a/static/js/idDialog.js
+++ b/static/js/idDialog.js
@@ -201,7 +201,7 @@ pandora.ui.idDialog = function(data) {
         }
         return Ox.filter([
             item.id,
-            item.title + (item.originalTitle ? ' (' + item.originalTitle + ')' : ''),
+            item.title + ((item.originalTitle && item.title != item.originalTitle) ? ' (' + item.originalTitle + ')' : ''),
             item.director ? item.director.join(', ') : '',
             item.year
         ]).join(' &mdash; ');

From e4a54b3cd419a3288b5815fbc339a614feb39bd8 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Thu, 15 Aug 2019 13:48:23 +0200
Subject: [PATCH 033/139] fix sortKey

---
 static/js/utils.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/static/js/utils.js b/static/js/utils.js
index b111f0e4..1661b418 100644
--- a/static/js/utils.js
+++ b/static/js/utils.js
@@ -145,7 +145,7 @@ pandora.addFolderItem = function(section) {
     }
     function getPosterFrames(newList) {
         var query,
-            sortKey = Ox.getObjectById(pandora.site.itemKeys, 'votes')
+            sortKey = section == 'items' && Ox.getObjectById(pandora.site.itemKeys, 'votes')
                 ? 'votes' : 'timesaccessed';
         if (!isDuplicate) {
             ({

From a309a9c3f5d95554beb51db2d447105e180099fc Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Thu, 15 Aug 2019 14:10:53 +0200
Subject: [PATCH 034/139] add document collection from results

---
 static/js/utils.js | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/static/js/utils.js b/static/js/utils.js
index 1661b418..f5e44f2d 100644
--- a/static/js/utils.js
+++ b/static/js/utils.js
@@ -39,7 +39,6 @@ pandora.addFolderItem = function(section) {
                 if (isItems) {
                     data.items = ui.listSelection;
                 } else if (section == 'documents') {
-                    //fixme
                     data.items = ui.collectionSelection;
                 } else {
                     data.clips = pandora.getClipData(
@@ -49,7 +48,7 @@ pandora.addFolderItem = function(section) {
                     );
                 }
             } else {
-                data.query = ui.find;
+                data.query = isItems ? ui.find : ui.findDocuments;
             }
         }
         if (ui.section == 'items' && section == 'edits') {
@@ -140,7 +139,7 @@ pandora.addFolderItem = function(section) {
             documents: 'addCollection',
             edits: 'addEdit'
         }[section]](data, function(result) {
-            getPosterFrames(result.data.id);
+            section == 'documents' ? reloadFolder(newList) : getPosterFrames(result.data.id);
         });
     }
     function getPosterFrames(newList) {

From aab6dcc31a227e3a0869c71c915cee07b86675c4 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Thu, 15 Aug 2019 14:12:35 +0200
Subject: [PATCH 035/139] add document collection from results (fixup)

---
 static/js/utils.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/static/js/utils.js b/static/js/utils.js
index f5e44f2d..467d7818 100644
--- a/static/js/utils.js
+++ b/static/js/utils.js
@@ -139,7 +139,7 @@ pandora.addFolderItem = function(section) {
             documents: 'addCollection',
             edits: 'addEdit'
         }[section]](data, function(result) {
-            section == 'documents' ? reloadFolder(newList) : getPosterFrames(result.data.id);
+            section == 'documents' ? reloadFolder(result.data.id) : getPosterFrames(result.data.id);
         });
     }
     function getPosterFrames(newList) {

From 932a12b66ed6d9be5e5f724d4a40f0879fe1451e Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Thu, 15 Aug 2019 14:45:49 +0200
Subject: [PATCH 036/139] update document panel in pandora_finddocuments

---
 static/js/folderList.js | 4 +++-
 static/js/mainPanel.js  | 6 +++---
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/static/js/folderList.js b/static/js/folderList.js
index c97083bf..03cccd74 100644
--- a/static/js/folderList.js
+++ b/static/js/folderList.js
@@ -287,7 +287,9 @@ pandora.ui.folderList = function(id, section) {
         // works when switching back from browser, but won't work on load, since
         // getListData relies on $folderList, so selectList is called in init handler
         selected: pandora.getListData().folder == id
-            ? [ui[ui.section == 'items' ? '_list' : ui.section.slice(0, -1)]]
+            ? [ui[ui.section == 'items' ? '_list'
+                  : ui.section == 'documents' ? '_collection'
+                  : ui.section.slice(0, -1)]]
             : [],
         sort: [{key: 'position', operator: '+'}],
         sortable: id != 'featured' || canEditFeatured,
diff --git a/static/js/mainPanel.js b/static/js/mainPanel.js
index 7460a3c8..7213ae59 100644
--- a/static/js/mainPanel.js
+++ b/static/js/mainPanel.js
@@ -23,7 +23,7 @@ pandora.ui.mainPanel = function() {
         .bindEvent({
             pandora_finddocuments: function() {
                 var previousUI = pandora.UI.getPrevious();
-                Ox.Log('FIND', 'handled in mainPanel', previousUI.item, previousUI._list)
+                Ox.Log('FIND', 'finddocuments handled in mainPanel', previousUI.document, previousUI._collection)
                 if (!previousUI.document && ui._collection == previousUI._collection) {
                     pandora.$ui.list.reloadList();
 
@@ -54,7 +54,7 @@ pandora.ui.mainPanel = function() {
                     if (pandora.stayInItemView) {
                         pandora.stayInItemView = false;
                     } else {
-                        that.replaceElement(1, pandora.$ui.rightPanel = pandora.ui.rightPanel());
+                        that.replaceElement(1, pandora.$ui.documentPanel = pandora.ui.documentPanel());
                     }
                 }
             },
@@ -68,7 +68,7 @@ pandora.ui.mainPanel = function() {
             },
             pandora_find: function() {
                 var previousUI = pandora.UI.getPrevious();
-                Ox.Log('FIND', 'handled in mainPanel', previousUI.item, previousUI._list)
+                Ox.Log('FIND', 'find handled in mainPanel', previousUI.item, previousUI._list)
                 if (!previousUI.item && ui._list == previousUI._list) {
                     if (['map', 'calendar'].indexOf(ui.listView) > -1) {
                         pandora.$ui.contentPanel.replaceElement(1,

From e4f8ffbf86a915ee59aec14da5b93ccc7944a23a Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Thu, 15 Aug 2019 14:50:16 +0200
Subject: [PATCH 037/139] getPosterFrames for collections

---
 static/js/utils.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/static/js/utils.js b/static/js/utils.js
index 467d7818..7f80064d 100644
--- a/static/js/utils.js
+++ b/static/js/utils.js
@@ -139,7 +139,7 @@ pandora.addFolderItem = function(section) {
             documents: 'addCollection',
             edits: 'addEdit'
         }[section]](data, function(result) {
-            section == 'documents' ? reloadFolder(result.data.id) : getPosterFrames(result.data.id);
+            getPosterFrames(result.data.id);
         });
     }
     function getPosterFrames(newList) {

From 65012d34ee94cf840f73951aa1ec8a193800e306 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Mon, 19 Aug 2019 11:27:11 +0200
Subject: [PATCH 038/139] use python3

---
 pandora/manage.py | 2 +-
 update.py         | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pandora/manage.py b/pandora/manage.py
index d33da6f9..700a1802 100755
--- a/pandora/manage.py
+++ b/pandora/manage.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 import os
 import signal
 import sys
diff --git a/update.py b/update.py
index 119f9adb..0e6c04de 100755
--- a/update.py
+++ b/update.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 from __future__ import print_function
 import json
 import os

From c5d481ee78161e6951d1741e2eea3b0fb9aef258 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Mon, 19 Aug 2019 22:28:35 +0200
Subject: [PATCH 039/139] move client_max_body_size

---
 etc/nginx/pandora | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/etc/nginx/pandora b/etc/nginx/pandora
index 8ef1f433..419120aa 100644
--- a/etc/nginx/pandora
+++ b/etc/nginx/pandora
@@ -55,11 +55,11 @@ server {
             proxy_buffering off;
             proxy_read_timeout 90;    #should be in sync with gunicorn timeout
             proxy_connect_timeout 90; #should be in sync with gunicorn timeout
+            client_max_body_size 32m;
             if (!-f $request_filename) {
                 proxy_pass http://127.0.0.1:2620;
                 break;
             }
-            client_max_body_size 32m;
     }
 
     error_page  400  /;

From 1d5fdf35ec828a775221f2bd93519307607b9fb9 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Mon, 19 Aug 2019 22:28:52 +0200
Subject: [PATCH 040/139] # in pdf name not working

---
 static/js/document.js       | 2 +-
 static/js/documentDialog.js | 2 +-
 static/js/utils.js          | 5 +++++
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/static/js/document.js b/static/js/document.js
index 6d1b2567..a13332d2 100644
--- a/static/js/document.js
+++ b/static/js/document.js
@@ -66,7 +66,7 @@ pandora.ui.document = function() {
                         ? pandora.user.ui.documents[item.id].position
                         : 1,
                     url: '/documents/' + item.id + '/'
-                        + item.title.replace('?', '_') + '.' + item.extension,
+                        + pandora.safePDFName(item.title + '.' + item.extension),
                     width: that.width(),
                     zoom: 'fit'
                 })
diff --git a/static/js/documentDialog.js b/static/js/documentDialog.js
index a8d1d581..b2c17308 100644
--- a/static/js/documentDialog.js
+++ b/static/js/documentDialog.js
@@ -196,7 +196,7 @@ pandora.ui.documentDialog = function(options) {
                         ? pandora.user.ui.documents[item.id].position
                         : 1,
                     url: '/documents/' + item.id + '/'
-                        + item.title.replace('?', '_') + '.' + item.extension,
+                        + pandora.safePDFName(item.title + '.' + item.extension),
                     width: dialogWidth,
                     zoom: 'fit'
                 })
diff --git a/static/js/utils.js b/static/js/utils.js
index 7f80064d..3ce86c15 100644
--- a/static/js/utils.js
+++ b/static/js/utils.js
@@ -2625,6 +2625,11 @@ pandora.openURL = function(url) {
     }
 };
 
+pandora.safePDFName = function(name) {
+    name = name.replace('?', '_').replace('#', '_');
+    return name;
+};
+
 pandora.saveURL = function(url, name) {
     var link = document.createElement('a');
     if (typeof link.download === 'string') {

From ce423303d55a1dadb9e7c010eda0cc898160dde3 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Wed, 21 Aug 2019 16:23:34 +0200
Subject: [PATCH 041/139] encode clips

---
 pandora/archive/extract.py       | 67 +++++++++++++++++++++++++++-----
 pandora/archive/views.py         |  2 +-
 pandora/item/models.py           | 40 +++++++++++++++++--
 pandora/item/tasks.py            | 28 +++++++++++++
 pandora/item/views.py            | 54 ++++++++++++++++++++-----
 static/js/downloadVideoDialog.js | 49 ++++++++++++++++++++---
 6 files changed, 210 insertions(+), 30 deletions(-)

diff --git a/pandora/archive/extract.py b/pandora/archive/extract.py
index f038d474..28b19f6b 100644
--- a/pandora/archive/extract.py
+++ b/pandora/archive/extract.py
@@ -609,11 +609,14 @@ def timeline_strip(item, cuts, info, prefix):
             timeline_image.save(timeline_file)
 
 
-def chop(video, start, end, subtitles=None):
+def chop(video, start, end, subtitles=None, dest=None, encode=False):
     t = end - start
-    tmp = tempfile.mkdtemp()
     ext = os.path.splitext(video)[1]
-    choped_video = '%s/tmp%s' % (tmp, ext)
+    if dest is None:
+        tmp = tempfile.mkdtemp()
+        choped_video = '%s/tmp%s' % (tmp, ext)
+    else:
+        choped_video = dest
     if subtitles and ext == '.mp4':
         subtitles_f = choped_video + '.full.srt'
         with open(subtitles_f, 'wb') as fd:
@@ -625,25 +628,71 @@ def chop(video, start, end, subtitles=None):
         if subtitles_f:
             os.unlink(subtitles_f)
     else:
+        if encode:
+            bpp = 0.17
+            if ext == '.mp4':
+                vcodec = [
+                    '-c:v', 'libx264',
+                    '-preset:v', 'medium',
+                    '-profile:v', 'high',
+                    '-level', '4.0',
+                ]
+                acodec = [
+                    '-c:a', 'aac',
+                    '-aq', '6',
+                    '-strict', '-2'
+                ]
+            else:
+                vcodec = [
+                    '-c:v', 'libvpx',
+                    '-deadline', 'good',
+                    '-cpu-used', '0',
+                    '-lag-in-frames', '25',
+                    '-auto-alt-ref', '1',
+                ]
+                acodec = [
+                    '-c:a', 'libvorbis',
+                    '-aq', '6',
+                ]
+            info = ox.avinfo(video)
+            if not info['audio']:
+                acodec = []
+            if not info['video']:
+                vcodec = []
+            else:
+                height = info['video'][0]['height']
+                width = info['video'][0]['width']
+                fps = 30
+                bitrate = height*width*fps*bpp/1000
+                vcodec += ['-vb', '%dk' % bitrate]
+            encoding = vcodec + acodec
+        else:
+            encoding = [
+                '-c:v', 'copy',
+                '-c:a', 'copy',
+            ]
         cmd = [
             settings.FFMPEG,
             '-y',
             '-i', video,
             '-ss', '%.3f' % start,
             '-t', '%.3f' % t,
-            '-c:v', 'copy',
-            '-c:a', 'copy',
+        ] + encoding + [
             '-f', ext[1:],
             choped_video
         ]
+        print(cmd)
         p = subprocess.Popen(cmd, stdin=subprocess.PIPE,
                              stdout=open('/dev/null', 'w'),
                              stderr=open('/dev/null', 'w'),
                              close_fds=True)
         p.wait()
-    f = open(choped_video, 'rb')
-    os.unlink(choped_video)
     if subtitles_f and os.path.exists(subtitles_f):
         os.unlink(subtitles_f)
-    os.rmdir(tmp)
-    return f
+    if dest is None:
+        f = open(choped_video, 'rb')
+        os.unlink(choped_video)
+        os.rmdir(tmp)
+        return f
+    else:
+        return None
diff --git a/pandora/archive/views.py b/pandora/archive/views.py
index 62731fc8..0b8abddd 100644
--- a/pandora/archive/views.py
+++ b/pandora/archive/views.py
@@ -368,7 +368,7 @@ def direct_upload(request):
     return render_to_json_response(response)
 
 
-@login_required_json
+#@login_required_json
 def getTaskStatus(request, data):
     '''
     Gets the status for a given task
diff --git a/pandora/item/models.py b/pandora/item/models.py
index 16c4b2ca..ba9f7f61 100644
--- a/pandora/item/models.py
+++ b/pandora/item/models.py
@@ -14,14 +14,15 @@ from glob import glob
 
 from six import PY2, string_types
 from six.moves.urllib.parse import quote
-from django.db import models, transaction, connection
-from django.db.models import Q, Sum, Max
+
 from django.conf import settings
 from django.contrib.auth import get_user_model
-
+from django.core.files.temp import NamedTemporaryFile
+from django.db import models, transaction, connection
+from django.db.models import Q, Sum, Max
 from django.db.models.signals import pre_delete
-from django.utils import datetime_safe
 from django.utils.encoding import python_2_unicode_compatible
+from django.utils import datetime_safe
 
 import ox
 from oxdjango.fields import JSONField, to_json
@@ -1182,6 +1183,37 @@ class Item(models.Model):
                     return None
                 return path
 
+    def extract_clip(self, in_, out, resolution, format, track=None, force=False):
+        streams = self.streams(track)
+        stream = streams[0].get(resolution, format)
+        if streams.count() > 1 and stream.info['duration'] < out:
+            video = NamedTemporaryFile(suffix='.%s' % format)
+            r = self.merge_streams(video.name, resolution, format)
+            if not r:
+                return False
+            path = video.name
+            duration = sum(item.cache['durations'])
+        else:
+            path = stream.media.path
+            duration = stream.info['duration']
+
+        cache_name = '%s_%sp_%s.%s' % (self.public_id, resolution, '%s,%s' % (in_, out), format)
+        cache_path = os.path.join(settings.MEDIA_ROOT, self.path('cache/%s' % cache_name))
+        if os.path.exists(cache_path) and not force:
+            return cache_path
+        if duration >= out:
+            subtitles = utils.get_by_key(settings.CONFIG['layers'], 'isSubtitles', True)
+            if subtitles:
+                srt = self.srt(subtitles['id'], encoder=ox.srt)
+                if len(srt) < 4:
+                    srt = None
+            else:
+                srt = None
+            ox.makedirs(os.path.dirname(cache_path))
+            extract.chop(path, in_, out, subtitles=srt, dest=cache_path, encode=True)
+            return cache_path
+        return False
+
     @property
     def timeline_prefix(self):
         videos = self.streams()
diff --git a/pandora/item/tasks.py b/pandora/item/tasks.py
index a0c9cac5..d43db126 100644
--- a/pandora/item/tasks.py
+++ b/pandora/item/tasks.py
@@ -22,6 +22,7 @@ def cronjob(**kwargs):
     if limit_rate('item.tasks.cronjob', 8 * 60 * 60):
         update_random_sort()
         update_random_clip_sort()
+        clear_cache.delay()
 
 def update_random_sort():
     from . import models
@@ -125,6 +126,33 @@ def load_subtitles(public_id):
         item.update_sort()
         item.update_facets()
 
+
+@task(queue="encoding")
+def extract_clip(public_id, in_, out, resolution, format, track=None):
+    from . import models
+    try:
+        item = models.Item.objects.get(public_id=public_id)
+    except models.Item.DoesNotExist:
+        return False
+    if item.extract_clip(in_, out, resolution, format, track):
+        return True
+    return False
+
+
+@task(queue="encoding")
+def clear_cache(days=60):
+    import subprocess
+    path = os.path.join(settings.MEDIA_ROOT, 'media')
+    cmd = ['find', path, '-iregex', '.*/frames/.*', '-atime', '+%s' % days, '-type', 'f', '-exec', 'rm', '{}', ';']
+    subprocess.check_output(cmd)
+    path = os.path.join(settings.MEDIA_ROOT, 'items')
+    cmd = ['find', path, '-iregex', '.*/cache/.*', '-atime', '+%s' % days, '-type', 'f', '-exec', 'rm', '{}', ';']
+    subprocess.check_output(cmd)
+    path = settings.MEDIA_ROOT
+    cmd = ['find', path, '-type', 'd', '-size', '0', '-prune', '-exec', 'rmdir', '{}', ';']
+    subprocess.check_output(cmd)
+
+
 @task(ignore_results=True, queue='default')
 def update_sitemap(base_url):
     from . import models
diff --git a/pandora/item/views.py b/pandora/item/views.py
index 02d8f924..182f7068 100644
--- a/pandora/item/views.py
+++ b/pandora/item/views.py
@@ -638,6 +638,32 @@ def edit(request, data):
     return render_to_json_response(response)
 actions.register(edit, cache=False)
 
+
+def extractClip(request, data):
+    '''
+    Extract and cache clip
+
+    takes {
+        item: string
+        resolution: int
+        format: string
+        in: float
+        out: float
+    }
+    returns {
+        taskId: string, // taskId
+    }
+    '''
+    item = get_object_or_404_json(models.Item, public_id=data['item'])
+    if not item.access(request.user):
+        return HttpResponseForbidden()
+
+    response = json_response()
+    t = tasks.extract_clip.delay(data['item'], data['in'], data['out'], data['resolution'], data['format'])
+    response['data']['taskId'] = t.task_id
+    return render_to_json_response(response)
+actions.register(extractClip, cache=False)
+
 @login_required_json
 def remove(request, data):
     '''
@@ -1056,6 +1082,23 @@ def video(request, id, resolution, format, index=None, track=None):
         ext = '.%s' % format
         duration = stream.info['duration']
 
+        filename = u"Clip of %s - %s-%s - %s %s%s" % (
+            item.get('title'),
+            ox.format_duration(t[0] * 1000).replace(':', '.')[:-4],
+            ox.format_duration(t[1] * 1000).replace(':', '.')[:-4],
+            settings.SITENAME.replace('/', '-'),
+            item.public_id,
+            ext
+        )
+        content_type = mimetypes.guess_type(path)[0]
+
+        cache_name = '%s_%sp_%s.%s' % (item.public_id, resolution, '%s,%s' % (t[0], t[1]), format)
+        cache_path = os.path.join(settings.MEDIA_ROOT, item.path('cache/%s' % cache_name))
+        if os.path.exists(cache_path):
+            response = HttpFileResponse(cache_path, content_type=content_type)
+            response['Content-Disposition'] = "attachment; filename*=UTF-8''%s" % quote(filename.encode('utf-8'))
+            return response
+
         # multipart request beyond first part, merge parts and chop that
         if not index and streams.count() > 1 and stream.info['duration'] < t[1]:
             video = NamedTemporaryFile(suffix=ext)
@@ -1065,7 +1108,6 @@ def video(request, id, resolution, format, index=None, track=None):
             path = video.name
             duration = sum(item.cache['durations'])
 
-        content_type = mimetypes.guess_type(path)[0]
         if len(t) == 2 and t[1] > t[0] and duration >= t[1]:
             # FIXME: could be multilingual here
             subtitles = utils.get_by_key(settings.CONFIG['layers'], 'isSubtitles', True)
@@ -1076,20 +1118,12 @@ def video(request, id, resolution, format, index=None, track=None):
             else:
                 srt = None
             response = HttpResponse(extract.chop(path, t[0], t[1], subtitles=srt), content_type=content_type)
-            filename = u"Clip of %s - %s-%s - %s %s%s" % (
-                item.get('title'),
-                ox.format_duration(t[0] * 1000).replace(':', '.')[:-4],
-                ox.format_duration(t[1] * 1000).replace(':', '.')[:-4],
-                settings.SITENAME,
-                item.public_id,
-                ext
-            )
             response['Content-Disposition'] = "attachment; filename*=UTF-8''%s" % quote(filename.encode('utf-8'))
             return response
         else:
             filename = "%s - %s %s%s" % (
                 item.get('title'),
-                settings.SITENAME,
+                settings.SITENAME.replace('/', '-'),
                 item.public_id,
                 ext
             )
diff --git a/static/js/downloadVideoDialog.js b/static/js/downloadVideoDialog.js
index b632351d..214bcd21 100644
--- a/static/js/downloadVideoDialog.js
+++ b/static/js/downloadVideoDialog.js
@@ -66,6 +66,8 @@ pandora.ui.downloadVideoDialog = function(options) {
             ]
         }).appendTo($content),
 
+        failed = false,
+
         that = Ox.Dialog({
             buttons: [
                 Ox.Button({
@@ -73,20 +75,55 @@ pandora.ui.downloadVideoDialog = function(options) {
                     title: Ox._('Download')
                 }).bindEvent({
                     click: function() {
-                        that.close();
+                        if (failed) {
+                            that.close();
+                            return
+                        }
                         var values = $form.values(),
                             url
                         if (options.out) {
-                            url = '/' + options.item
-                                + '/' + values.resolution
-                                + 'p.' + values.format
-                                + '?t=' + options['in'] + ',' + options.out;
+                            var $screen = Ox.LoadingScreen({
+                                size: 16
+                            })
+                            that.options({content: $screen.start()});
+                            pandora.api.extractClip({
+                                item: options.item,
+                                resolution: values.resolution,
+                                format: values.format,
+                                'in': options['in'],
+                                out: options.out
+                            }, function(result) {
+                                if (result.data.taskId) {
+                                    pandora.wait(result.data.taskId, function(result) {
+                                        console.log('wait -> ', result)
+                                        if (result.data.result) {
+                                            url = '/' + options.item
+                                                + '/' + values.resolution
+                                                + 'p.' + values.format
+                                                + '?t=' + options['in'] + ',' + options.out;
+                                            that.close();
+                                            document.location.href = url
+                                        } else {
+                                        }
+                                    }, 1000)
+                                } else {
+                                    that.options({content: 'Failed to extract clip.'});
+                                    that.options('buttons')[0].options({
+                                        title: Ox._('Close')
+                                    });
+                                    failed = true;
+                                }
+                            })
+
                         } else {
                             url = '/' + options.item
                                 + '/download/' + values.resolution
                                 + 'p.' + values.format
                         }
-                        document.location.href = url
+                        if (url) {
+                            that.close();
+                            document.location.href = url
+                        }
                     }
                 })
             ],

From 413c98b0399500ec2f735a74a2a68374e8205f32 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Mon, 14 Oct 2019 12:16:23 +0100
Subject: [PATCH 042/139] accept jpeg

---
 static/js/utils.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/static/js/utils.js b/static/js/utils.js
index 3ce86c15..ebdc5cd0 100644
--- a/static/js/utils.js
+++ b/static/js/utils.js
@@ -404,7 +404,7 @@ pandora.createLinks = function($element) {
 };
 
 pandora.uploadDroppedFiles = function(files) {
-    var documentExtensions = ['pdf', /* 'epub', 'txt', */ 'png', 'gif', 'jpg'];
+    var documentExtensions = ['pdf', /* 'epub', 'txt', */ 'png', 'gif', 'jpg', 'jpeg'];
     files = Ox.map(files, function(file) { return file });
 
     if (files.every(function(file) {

From 7fd0daf8d32578d2d3ac75cb1c61f9b2bb8af1c1 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Mon, 14 Oct 2019 12:16:38 +0100
Subject: [PATCH 043/139] default to at position in icma

---
 pandora/config.indiancinema.jsonc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pandora/config.indiancinema.jsonc b/pandora/config.indiancinema.jsonc
index be2a79db..3ba5b93b 100644
--- a/pandora/config.indiancinema.jsonc
+++ b/pandora/config.indiancinema.jsonc
@@ -1695,7 +1695,7 @@
             "annotationsCalendarSize": 128,
             "annotationsHighlight": "none",
             "annotationsMapSize": 128,
-            "annotationsRange": "all",
+            "annotationsRange": "position",
             "annotationsSize": 256,
             "annotationsSort": "position",
             "calendarFind": "",

From b49a97209c0c577f952f1c558818f7bd808ed819 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Sat, 19 Oct 2019 12:44:57 +0100
Subject: [PATCH 044/139] disable debug

---
 pandora/document/views.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pandora/document/views.py b/pandora/document/views.py
index a95ca06e..083090ee 100644
--- a/pandora/document/views.py
+++ b/pandora/document/views.py
@@ -210,7 +210,7 @@ def parse_query(data, user):
     for key in ('keys', 'group', 'file', 'range', 'position', 'positions', 'sort'):
         if key in data:
             query[key] = data[key]
-    print(query.get('sort'), data.get('sort'))
+    #print(query.get('sort'), data.get('sort'))
     query['qs'] = models.Document.objects.find(data, user)
     query['item'] = get_item(data.get('query', {}))
     return query

From 401347e29b242570bb023068b96d76071af5c528 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Sat, 19 Oct 2019 12:45:05 +0100
Subject: [PATCH 045/139] longer queue

---
 pandora/archive/extract.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pandora/archive/extract.py b/pandora/archive/extract.py
index 28b19f6b..618bad85 100644
--- a/pandora/archive/extract.py
+++ b/pandora/archive/extract.py
@@ -216,6 +216,7 @@ def stream(video, target, profile, info, audio_track=0, flags={}):
             '-vb', '%dk' % bitrate,
             '-aspect', aspect,
             # '-vf', 'yadif',
+            '-max_muxing_queue_size', '512',
             '-vf', 'hqdn3d%s,scale=%s:%s' % (crop, width, height),
             '-g', '%d' % int(fps*5),
         ]

From 3df8380099765df9a65d13c554684b73ee9fdac4 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Sat, 19 Oct 2019 12:45:11 +0100
Subject: [PATCH 046/139] icma defaults

---
 pandora/config.indiancinema.jsonc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pandora/config.indiancinema.jsonc b/pandora/config.indiancinema.jsonc
index 3ba5b93b..77f5b855 100644
--- a/pandora/config.indiancinema.jsonc
+++ b/pandora/config.indiancinema.jsonc
@@ -1695,7 +1695,7 @@
             "annotationsCalendarSize": 128,
             "annotationsHighlight": "none",
             "annotationsMapSize": 128,
-            "annotationsRange": "position",
+            "annotationsRange": "selection",
             "annotationsSize": 256,
             "annotationsSort": "position",
             "calendarFind": "",
@@ -1852,7 +1852,7 @@
             "videoSize": "small",
             "videoSubtitles": true,
             "videoSubtitlesOffset": 0,
-            "videoTimeline": "slitscan",
+            "videoTimeline": "keyframes",
             "videoView": "player",
             "videoVolume": 1
         },

From b5bb6bd6224b32377f0f3a03d5f70cce432da2b9 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Sat, 19 Oct 2019 12:48:34 +0100
Subject: [PATCH 047/139] seasonal timeline changes

---
 pandora/config.padma.jsonc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pandora/config.padma.jsonc b/pandora/config.padma.jsonc
index e797cd15..b406bb87 100644
--- a/pandora/config.padma.jsonc
+++ b/pandora/config.padma.jsonc
@@ -1160,7 +1160,7 @@
             "annotationsHighlight": "none",
             "annotationsHighlight": false,
             "annotationsMapSize": 128,
-            "annotationsRange": "position",
+            "annotationsRange": "selection",
             "annotationsSize": 256,
             "annotationsSort": "position",
             "calendarFind": "",
@@ -1310,7 +1310,7 @@
             "videoSize": "large",
             "videoSubtitles": false,
             "videoSubtitlesOffset": 0,
-            "videoTimeline": "antialias",
+            "videoTimeline": "keyframes",
             "videoView": "player",
             "videoVolume": 1
         },

From 014df3766ce530a748dce806160aa589a173071c Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Tue, 22 Oct 2019 16:15:49 +0200
Subject: [PATCH 048/139] only use sendReferrer in site

---
 pandora/config.pandora.jsonc | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/pandora/config.pandora.jsonc b/pandora/config.pandora.jsonc
index 9658b265..f7fdd8f2 100644
--- a/pandora/config.pandora.jsonc
+++ b/pandora/config.pandora.jsonc
@@ -990,11 +990,6 @@ examples (config.SITENAME.jsonc) that are part of this pan.do/ra distribution.
         {"name": "Private", "color": [255, 128, 128]}
     ],
     /*
-    "sendReferrer", if set to false, will cause all outgoing links to originate
-    from one single URL
-    */
-    "sendReferrer": false,
-    /*
     "site" contains various settings for this instance. In "email", "contact"
     if the address in the contact form (to), "system" is the address used by
     the system (from).

From 53c9d13aa7df850931cfb28e087ffc24d771ec52 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Thu, 31 Oct 2019 11:10:14 +0100
Subject: [PATCH 049/139] list bionic

---
 vm/LXC_README.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/vm/LXC_README.md b/vm/LXC_README.md
index 44af135e..92cd6262 100644
--- a/vm/LXC_README.md
+++ b/vm/LXC_README.md
@@ -4,7 +4,7 @@
 
 # Installing pan.do/ra inside LXC
 
-1) Install lxc on the host (Ubuntu 16.04 or later):
+1) Install lxc on the host (Ubuntu 18.04 or later):
 
     sudo apt-get install lxc
 
@@ -15,7 +15,7 @@
 
 2) Create a new container, use different names if installing multiple instances:
 
-    sudo lxc-create -n pandora -t ubuntu -- -r xenial
+    sudo lxc-create -n pandora -t ubuntu-cloud -- -r bionic
 
     or
 
@@ -28,12 +28,12 @@
 4) Attach to container and install pan.do/ra
 
     sudo lxc-attach -n pandora --clear-env
-    apt-get update -qq && apt-get upgrade -y
-    apt-get -y install curl ca-certificates
     sed -i s/ubuntu/pandora/g /etc/passwd /etc/shadow /etc/group
     mv /home/ubuntu /home/pandora
     echo "pandora:pandora" | chpasswd
     echo PasswordAuthentication no >> /etc/ssh/sshd_config
+    apt-get update -qq && apt-get upgrade -y
+    apt-get -y install curl ca-certificates
     locale-gen en_US.UTF-8
     update-locale LANG=en_US.UTF-8
     export LANG=en_US.UTF-8

From 6af2a1cbe65475b7ed32eba1cb9703f1310ab2ed Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Thu, 31 Oct 2019 11:10:29 +0100
Subject: [PATCH 050/139] default to list of countries

---
 pandora/config.pandora.jsonc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pandora/config.pandora.jsonc b/pandora/config.pandora.jsonc
index f7fdd8f2..afec959e 100644
--- a/pandora/config.pandora.jsonc
+++ b/pandora/config.pandora.jsonc
@@ -550,7 +550,7 @@ examples (config.SITENAME.jsonc) that are part of this pan.do/ra distribution.
         {
             "id": "country",
             "title": "Country",
-            "type": "string",
+            "type": ["string"],
             "autocomplete": true,
             "columnWidth": 180,
             "filter": true,

From ae2e3e45c6d26feb129917c71bddbb3af1884954 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Thu, 31 Oct 2019 11:16:16 +0100
Subject: [PATCH 051/139] download parts and source

---
 pandora/config.0xdb.jsonc         |  1 +
 pandora/config.indiancinema.jsonc |  1 +
 pandora/config.padma.jsonc        |  1 +
 pandora/config.pandora.jsonc      |  1 +
 pandora/item/models.py            |  3 ++
 pandora/item/urls.py              |  1 +
 pandora/item/views.py             | 31 +++++++++++++-----
 static/js/downloadVideoDialog.js  | 53 +++++++++++++++++++++++++++----
 static/js/editor.js               |  4 ++-
 static/js/utils.js                |  1 +
 10 files changed, 82 insertions(+), 15 deletions(-)

diff --git a/pandora/config.0xdb.jsonc b/pandora/config.0xdb.jsonc
index 7b300891..bcae7125 100644
--- a/pandora/config.0xdb.jsonc
+++ b/pandora/config.0xdb.jsonc
@@ -38,6 +38,7 @@
         "canAddItems": {"staff": true, "admin": true},
         "canAddDocuments": {"staff": true, "admin": true},
         "canDownloadVideo": {"guest": -1, "member": -1, "friend": -1, "staff": -1, "admin": -1},
+        "canDownloadSource": {"guest": -1, "member": -1, "friend": -1, "staff": -1, "admin": -1},
         "canEditAnnotations": {"staff": true, "admin": true},
         "canEditEntities": {"staff": true, "admin": true},
         "canEditDocuments": {"staff": true, "admin": true},
diff --git a/pandora/config.indiancinema.jsonc b/pandora/config.indiancinema.jsonc
index 77f5b855..40876054 100644
--- a/pandora/config.indiancinema.jsonc
+++ b/pandora/config.indiancinema.jsonc
@@ -39,6 +39,7 @@
         "canAddItems": {"researcher": true, "staff": true, "admin": true},
         "canAddDocuments": {"researcher": true, "staff": true, "admin": true},
         "canDownloadVideo": {"guest": -1, "member": -1, "researcher": 3, "staff": 3, "admin": 3},
+        "canDownloadSource": {"guest": -1, "member": -1, "researcher": -1, "staff": -1, "admin": -1},
         "canEditAnnotations": {"staff": true, "admin": true},
         "canEditDocuments": {"researcher": true, "staff": true, "admin": true},
         "canEditEntities": {"staff": true, "admin": true},
diff --git a/pandora/config.padma.jsonc b/pandora/config.padma.jsonc
index b406bb87..c78740a9 100644
--- a/pandora/config.padma.jsonc
+++ b/pandora/config.padma.jsonc
@@ -38,6 +38,7 @@
         "canAddItems": {"member": true, "staff": true, "admin": true},
         "canAddDocuments": {"member": true, "staff": true, "admin": true},
         "canDownloadVideo": {"guest": 0, "member": 0, "staff": 4, "admin": 4},
+        "canDownloadSource": {"guest": -1, "member": -1, "staff": 4, "admin": 4},
         "canEditAnnotations": {"staff": true, "admin": true},
         "canEditEntities": {"staff": true, "admin": true},
         "canEditDocuments": {"staff": true, "admin": true},
diff --git a/pandora/config.pandora.jsonc b/pandora/config.pandora.jsonc
index afec959e..c418566f 100644
--- a/pandora/config.pandora.jsonc
+++ b/pandora/config.pandora.jsonc
@@ -45,6 +45,7 @@ examples (config.SITENAME.jsonc) that are part of this pan.do/ra distribution.
         "canAddItems": {"member": true, "staff": true, "admin": true},
         "canAddDocuments": {"member": true, "staff": true, "admin": true},
         "canDownloadVideo": {"guest": 1, "member": 1, "staff": 4, "admin": 4},
+        "canDownloadSource": {"member": 1, "staff": 4, "admin": 4},
         "canEditAnnotations": {"staff": true, "admin": true},
         "canEditDocuments": {"staff": true, "admin": true},
         "canEditEntities": {"staff": true, "admin": true},
diff --git a/pandora/item/models.py b/pandora/item/models.py
index ba9f7f61..dfd593df 100644
--- a/pandora/item/models.py
+++ b/pandora/item/models.py
@@ -639,6 +639,9 @@ class Item(models.Model):
             if self.poster_height:
                 i['posterRatio'] = self.poster_width / self.poster_height
 
+        if 'source' in keys:
+            i['source'] = self.files.filter(selected=True).exclude(data='').exists()
+
         streams = self.streams()
         i['durations'] = [s.duration for s in streams]
         i['duration'] = sum(i['durations'])
diff --git a/pandora/item/urls.py b/pandora/item/urls.py
index 948a1878..332da86d 100644
--- a/pandora/item/urls.py
+++ b/pandora/item/urls.py
@@ -16,6 +16,7 @@ urlpatterns = [
     url(r'^(?P<id>[A-Z0-9].*)/download$', views.download),
     url(r'^(?P<id>[A-Z0-9].*)/download/$', views.download),
     url(r'^(?P<id>[A-Z0-9].*)/download/source/(?P<part>\d+)?$', views.download_source),
+    url(r'^(?P<id>[A-Z0-9].*)/download/(?P<resolution>\d+)p(?P<part>\d+)\.(?P<format>webm|ogv|mp4)$', views.download),
     url(r'^(?P<id>[A-Z0-9].*)/download/(?P<resolution>\d+)p\.(?P<format>webm|ogv|mp4)$', views.download),
 
     #video
diff --git a/pandora/item/views.py b/pandora/item/views.py
index 182f7068..f5539b61 100644
--- a/pandora/item/views.py
+++ b/pandora/item/views.py
@@ -992,6 +992,8 @@ def download_source(request, id, part=None):
         raise Http404
 
     parts = ['%s - %s ' % (item.get('title'), settings.SITENAME), item.public_id]
+    if len(streams) > 1:
+        parts.append('.Part %d' % (part + 1))
     parts.append('.')
     parts.append(f.extension)
     filename = ''.join(parts)
@@ -1002,7 +1004,7 @@ def download_source(request, id, part=None):
     response['Content-Disposition'] = "attachment; filename*=UTF-8''%s" % quote(filename.encode('utf-8'))
     return response
 
-def download(request, id, resolution=None, format='webm'):
+def download(request, id, resolution=None, format='webm', part=None):
     item = get_object_or_404(models.Item, public_id=id)
     if not resolution or int(resolution) not in settings.CONFIG['video']['resolutions']:
         resolution = max(settings.CONFIG['video']['resolutions'])
@@ -1010,22 +1012,35 @@ def download(request, id, resolution=None, format='webm'):
         resolution = int(resolution)
     if not item.access(request.user) or not item.rendered:
         return HttpResponseForbidden()
+    if part is not None:
+        part = int(part) - 1
+        streams = item.streams()
+        if part > len(streams):
+            raise Http404
     ext = '.%s' % format
     parts = ['%s - %s ' % (item.get('title'), settings.SITENAME), item.public_id]
     if resolution != max(settings.CONFIG['video']['resolutions']):
         parts.append('.%dp' % resolution)
+    if part is not None:
+        parts.append('.Part %d' % (part + 1))
     parts.append(ext)
     filename = ''.join(parts)
     video = NamedTemporaryFile(suffix=ext)
     content_type = mimetypes.guess_type(video.name)[0]
-    r = item.merge_streams(video.name, resolution, format)
-    if not r:
-        return HttpResponseForbidden()
-    elif r is True:
-        response = HttpResponse(FileWrapper(video), content_type=content_type)
-        response['Content-Length'] = os.path.getsize(video.name)
+    if part is None:
+        r = item.merge_streams(video.name, resolution, format)
+        if not r:
+            return HttpResponseForbidden()
+        elif r is True:
+            response = HttpResponse(FileWrapper(video), content_type=content_type)
+            response['Content-Length'] = os.path.getsize(video.name)
+        else:
+            response = HttpFileResponse(r, content_type=content_type)
     else:
-        response = HttpFileResponse(r, content_type=content_type)
+        stream = streams[part].get(resolution, format)
+        path = stream.media.path
+        content_type = mimetypes.guess_type(path)[0]
+        response = HttpFileResponse(path, content_type=content_type)
     response['Content-Disposition'] = "attachment; filename*=UTF-8''%s" % quote(filename.encode('utf-8'))
     return response
 
diff --git a/static/js/downloadVideoDialog.js b/static/js/downloadVideoDialog.js
index 214bcd21..88d58b7e 100644
--- a/static/js/downloadVideoDialog.js
+++ b/static/js/downloadVideoDialog.js
@@ -9,6 +9,10 @@ pandora.ui.downloadVideoDialog = function(options) {
             'mp4': 'MP4',
         },
 
+        parts = Ox.max(options.video.map(function(video) {
+            return video.index
+        })),
+
         $content = Ox.Element()
             .css({margin: '16px'}),
 
@@ -27,6 +31,9 @@ pandora.ui.downloadVideoDialog = function(options) {
             .css({marginBottom: '16px'})
             .appendTo($content),
 
+        $format,
+        $resolution,
+
         $form = window.$form = Ox.Form({
             items: [
                 Ox.Select({
@@ -36,7 +43,10 @@ pandora.ui.downloadVideoDialog = function(options) {
                                 id: format,
                                 title: formats[format]
                             };
-                        }),
+                        }).concat(!options.out && options.source ? [{
+                            id: 'source',
+                            title: Ox._('Source')
+                        }] : []),
                         label: Ox._('Format'),
                         labelWidth: 120,
                         value: pandora.site.video.downloadFormat,
@@ -44,9 +54,14 @@ pandora.ui.downloadVideoDialog = function(options) {
                     })
                     .bindEvent({
                         change: function(data) {
+                            if (data.value == 'source') {
+                                $resolution.hide()
+                            } else {
+                                $resolution.show()
+                            }
                         }
                     }),
-                Ox.Select({
+                $resolution = Ox.Select({
                         id: 'resolution',
                         items: pandora.site.video.resolutions.map(function(resolution) {
                             return {
@@ -63,7 +78,25 @@ pandora.ui.downloadVideoDialog = function(options) {
                         change: function(data) {
                         }
                     })
-            ]
+            ].concat(parts ? [
+                Ox.Select({
+                        id: 'part',
+                        items: Ox.range(parts + 1).map(function(resolution, idx) {
+                            return {
+                                id: idx + 1,
+                                title: 'Part ' + (idx+1)
+                            };
+                        }),
+                        label: Ox._('Part'),
+                        labelWidth: 120,
+                        value: 1,
+                        width: 240
+                    })
+                    .bindEvent({
+                        change: function(data) {
+                        }
+                    })
+            ] : [])
         }).appendTo($content),
 
         failed = false,
@@ -116,9 +149,17 @@ pandora.ui.downloadVideoDialog = function(options) {
                             })
 
                         } else {
-                            url = '/' + options.item
-                                + '/download/' + values.resolution
-                                + 'p.' + values.format
+                            if (values.format == 'source') {
+                                url = '/' + options.item
+                                    + '/download/source/'
+                                    + (values.part ? values.part : '')
+                            } else {
+                                url = '/' + options.item
+                                    + '/download/' + values.resolution
+                                    + 'p'
+                                    + (values.part ? values.part : '')
+                                    + '.' + values.format
+                            }
                         }
                         if (url) {
                             that.close();
diff --git a/static/js/editor.js b/static/js/editor.js
index 0997ac84..562c3bc0 100644
--- a/static/js/editor.js
+++ b/static/js/editor.js
@@ -189,7 +189,9 @@ pandora.ui.editor = function(data) {
                 pandora.ui.downloadVideoDialog({
                     item: ui.item,
                     rightsLevel: rightsLevel,
-                    title: data.title
+                    source: data.source && pandora.hasCapability('canDownloadSource'),
+                    title: data.title,
+                    video: data.video
                 }).open();
             },
             downloadselection: function(selection) {
diff --git a/static/js/utils.js b/static/js/utils.js
index ebdc5cd0..d2667546 100644
--- a/static/js/utils.js
+++ b/static/js/utils.js
@@ -2359,6 +2359,7 @@ pandora.VIDEO_OPTIONS_KEYS = [
     'rendered',
     'rightslevel',
     'size',
+    'source',
     'streams',
     'title',
     'videoRatio'

From 85095eda44c13399c8a7ac180bbc47c1f4b219aa Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Fri, 1 Nov 2019 15:22:53 +0100
Subject: [PATCH 052/139] only use current search if its inside a layer

---
 static/js/editor.js | 2 +-
 static/js/utils.js  | 8 ++++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/static/js/editor.js b/static/js/editor.js
index 562c3bc0..6c3106b9 100644
--- a/static/js/editor.js
+++ b/static/js/editor.js
@@ -28,7 +28,7 @@ pandora.ui.editor = function(data) {
             enableSetPosterFrame: !pandora.site.media.importFrames && data.editable,
             enableSubtitles: ui.videoSubtitles,
             find: ui.itemFind,
-            findLayer: ui._findState.key,
+            findLayer: pandora.getFindLayer(),
             getFrameURL: function(position) {
                 return pandora.getMediaURL('/' + ui.item + '/' + ui.videoResolution + 'p' + position + '.jpg?' + data.modified);
             },
diff --git a/static/js/utils.js b/static/js/utils.js
index d2667546..977c95e7 100644
--- a/static/js/utils.js
+++ b/static/js/utils.js
@@ -1439,6 +1439,14 @@ pandora.getFoldersWidth = function(section) {
     return width;
 };
 
+pandora.getFindLayer = function() {
+    var key = pandora.user.ui._findState.key
+    if (!!Ox.getObjectById(pandora.site.layers, key)) {
+        key = 'all'
+    }
+    return key
+};
+
 pandora.getHash = function(state, callback) {
     // FIXME: remove this
     var embedKeys = [

From b0a8f889103cfd1ffc2e906fdbf889c1411e411e Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Fri, 1 Nov 2019 15:27:16 +0100
Subject: [PATCH 053/139] fix context

---
 static/js/utils.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/static/js/utils.js b/static/js/utils.js
index 977c95e7..20b59253 100644
--- a/static/js/utils.js
+++ b/static/js/utils.js
@@ -1441,7 +1441,7 @@ pandora.getFoldersWidth = function(section) {
 
 pandora.getFindLayer = function() {
     var key = pandora.user.ui._findState.key
-    if (!!Ox.getObjectById(pandora.site.layers, key)) {
+    if (!Ox.getObjectById(pandora.site.layers, key)) {
         key = 'all'
     }
     return key

From 86cc4b3f31fb845308e6ac2e8719748dcba481f9 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Fri, 1 Nov 2019 15:32:34 +0100
Subject: [PATCH 054/139] all key is *

---
 static/js/utils.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/static/js/utils.js b/static/js/utils.js
index 20b59253..32c40c08 100644
--- a/static/js/utils.js
+++ b/static/js/utils.js
@@ -1442,7 +1442,7 @@ pandora.getFoldersWidth = function(section) {
 pandora.getFindLayer = function() {
     var key = pandora.user.ui._findState.key
     if (!Ox.getObjectById(pandora.site.layers, key)) {
-        key = 'all'
+        key = '*'
     }
     return key
 };

From ce8b693922711110a4b9f5c48437ac3e7d3d42fa Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Fri, 1 Nov 2019 16:50:40 +0100
Subject: [PATCH 055/139] keys might be none

---
 pandora/item/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pandora/item/models.py b/pandora/item/models.py
index dfd593df..a5bf7064 100644
--- a/pandora/item/models.py
+++ b/pandora/item/models.py
@@ -639,7 +639,7 @@ class Item(models.Model):
             if self.poster_height:
                 i['posterRatio'] = self.poster_width / self.poster_height
 
-        if 'source' in keys:
+        if keys and 'source' in keys:
             i['source'] = self.files.filter(selected=True).exclude(data='').exists()
 
         streams = self.streams()

From 551ffc5c86dde51f12d374860d6af6fa1e0b877e Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Fri, 1 Nov 2019 19:48:35 +0100
Subject: [PATCH 056/139] make sure video is 420p

---
 pandora/archive/extract.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pandora/archive/extract.py b/pandora/archive/extract.py
index 618bad85..f85c464e 100644
--- a/pandora/archive/extract.py
+++ b/pandora/archive/extract.py
@@ -239,6 +239,7 @@ def stream(video, target, profile, info, audio_track=0, flags={}):
                 '-preset:v', 'medium',
                 '-profile:v', 'high',
                 '-level', '4.0',
+                '-pix_fmt', 'yuv420p',
             ]
         video_settings += ['-map', '0:%s,0:0' % info['video'][0]['id']]
         audio_only = False

From 05b7fc1f7e8e0bc56d5dfde05201740607f6e81b Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Sat, 2 Nov 2019 08:32:49 +0100
Subject: [PATCH 057/139] default flags

---
 pandora/archive/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pandora/archive/models.py b/pandora/archive/models.py
index 3b437fed..199c90b1 100644
--- a/pandora/archive/models.py
+++ b/pandora/archive/models.py
@@ -523,7 +523,7 @@ class File(models.Model):
                     n += 1
                 profile = '%sp.%s' % (resolution, config['formats'][0])
                 target = os.path.join(tmp, language + '_' + profile)
-                ok, error = extract.stream(media, target, profile, info, audio_track=i+1, flags=self.flags)
+                ok, error = extract.stream(media, target, profile, info, audio_track=i+1, flags={})
                 if ok:
                     tinfo = ox.avinfo(target)
                     del tinfo['path']

From 9881bfa62a6fd9f14816de33e578554750516494 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Sat, 2 Nov 2019 15:55:02 +0100
Subject: [PATCH 058/139] don't count srt files

---
 pandora/item/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pandora/item/models.py b/pandora/item/models.py
index a5bf7064..f625fb80 100644
--- a/pandora/item/models.py
+++ b/pandora/item/models.py
@@ -640,7 +640,7 @@ class Item(models.Model):
                 i['posterRatio'] = self.poster_width / self.poster_height
 
         if keys and 'source' in keys:
-            i['source'] = self.files.filter(selected=True).exclude(data='').exists()
+            i['source'] = self.streams().exclude(file__data='').exists()
 
         streams = self.streams()
         i['durations'] = [s.duration for s in streams]

From 05aa23166ea9a28657b1988b681ec8d7937e0412 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Sat, 2 Nov 2019 17:14:35 +0100
Subject: [PATCH 059/139] ignore broken poster frames

---
 pandora/itemlist/models.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/pandora/itemlist/models.py b/pandora/itemlist/models.py
index e9e877bb..7d696381 100644
--- a/pandora/itemlist/models.py
+++ b/pandora/itemlist/models.py
@@ -271,12 +271,13 @@ class List(models.Model):
                 self.save()
         for i in self.poster_frames:
             from item.models import Item
-            qs = Item.objects.filter(public_id=i['item'])
-            if qs.count() > 0:
-                if i.get('position'):
-                    frame = qs[0].frame(i['position'])
-                    if frame:
-                        frames.append(frame)
+            if 'item' in i:
+                qs = Item.objects.filter(public_id=i['item'])
+                if qs.count() > 0:
+                    if i.get('position'):
+                        frame = qs[0].frame(i['position'])
+                        if frame:
+                            frames.append(frame)
         self.icon.name = self.path('icon.jpg')
         icon = self.icon.path
         if frames:

From faeb2ddd2d625d52c58f973bde781ef6dcc7e34e Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Tue, 12 Nov 2019 11:40:14 +0000
Subject: [PATCH 060/139] document folded annotations, include clip_id in
 annots

---
 pandora/annotation/models.py | 2 ++
 pandora/clip/views.py        | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/pandora/annotation/models.py b/pandora/annotation/models.py
index 470af358..0326e37d 100644
--- a/pandora/annotation/models.py
+++ b/pandora/annotation/models.py
@@ -378,6 +378,8 @@ class Annotation(models.Model):
                 streams = self.item.streams()
                 if streams:
                     j['videoRatio'] = streams[0].aspect_ratio
+            if 'clip' in keys:
+                j[key] = self.clip.public_id
             for key in keys:
                 if key not in j:
                     if key in self._clip_keys:
diff --git a/pandora/clip/views.py b/pandora/clip/views.py
index 700a95a1..7c2fcd23 100644
--- a/pandora/clip/views.py
+++ b/pandora/clip/views.py
@@ -78,7 +78,7 @@ def findClips(request, data):
     takes {
         query: object, // find clips, query object, see `find`
         itemsQuery: object, // limit to matching items, query object, see `find`
-        keys: [string], // list of properties to return
+        keys: [string], // list of properties to return, include 'annotations' to get all annotations for a clip
         positions: [int], // list of positions
         range: [int, int], // range of results to return
         sort: [object] // list of sort objects, see `find`

From aaf444ae96af97ec4e9dd08173970e19bbd6eb0e Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Tue, 12 Nov 2019 17:48:10 +0000
Subject: [PATCH 061/139] hide unused dialogs from menu

---
 static/js/mainMenu.js | 6 ++++--
 static/js/utils.js    | 3 +++
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/static/js/mainMenu.js b/static/js/mainMenu.js
index b797b4bd..87bfbc52 100644
--- a/static/js/mainMenu.js
+++ b/static/js/mainMenu.js
@@ -229,8 +229,10 @@ pandora.ui.mainMenu = function() {
                         { id: 'names', title: Ox._('Manage Names...'), disabled: !pandora.hasCapability('canManageTitlesAndNames') },
                         { id: 'translations', title: Ox._('Manage Translations...'), disabled: !pandora.hasCapability('canManageTranslations') },
                         {},
-                        { id: 'places', title: Ox._('Manage Places...'), disabled: !pandora.hasCapability('canManagePlacesAndEvents') },
-                        { id: 'events', title: Ox._('Manage Events...'), disabled: !pandora.hasCapability('canManagePlacesAndEvents') },
+                        pandora.hasView('map')
+                        ? [{ id: 'places', title: Ox._('Manage Places...'), disabled: !pandora.hasCapability('canManagePlacesAndEvents') }] : [],
+                        pandora.hasView('calendar')
+                        ? [{ id: 'events', title: Ox._('Manage Events...'), disabled: !pandora.hasCapability('canManagePlacesAndEvents') }] : [],
                         {},
                         { id: 'users', title: Ox._('Manage Users...'), disabled: !pandora.hasCapability('canManageUsers') },
                         { id: 'statistics', title: Ox._('Statistics...'), disabled: !pandora.hasCapability('canManageUsers') },
diff --git a/static/js/utils.js b/static/js/utils.js
index 32c40c08..ae2758cc 100644
--- a/static/js/utils.js
+++ b/static/js/utils.js
@@ -2489,6 +2489,9 @@ pandora.hasPlacesLayer = function() {
     });
 };
 
+pandora.hasView = function(id) {
+    return !!(Ox.getObjectById(pandora.site.itemViews, id) || Ox.getObjectById(pandora.site.listViews, id))
+};
 
 pandora.isClipView = function(view, item) {
     if (pandora.user.ui.section == 'items') {

From f60222c9b5b266af6f75d1f2a4d69e1a70748f4d Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Tue, 12 Nov 2019 17:50:07 +0000
Subject: [PATCH 062/139] processing note

---
 static/js/item.js | 27 ++++++++++++++++++++++-----
 1 file changed, 22 insertions(+), 5 deletions(-)

diff --git a/static/js/item.js b/static/js/item.js
index 91c9b8f4..c5045d1f 100644
--- a/static/js/item.js
+++ b/static/js/item.js
@@ -41,11 +41,28 @@ pandora.ui.item = function() {
                     pandora.user.ui.itemView.slice(0, 1)
                 ) > -1 ? 'an': 'a') + ' '
                 +'{1} view.', [result.data.title, Ox._(pandora.user.ui.itemView)]);
-            pandora.$ui.contentPanel.replaceElement(1,
-                Ox.Element()
-                    .css({marginTop: '32px', fontSize: '12px', textAlign: 'center'})
-                    .html(html)
-            );
+
+            var note = Ox.Element()
+                .css({marginTop: '32px', fontSize: '12px', textAlign: 'center'})
+            pandora.$ui.contentPanel.replaceElement(1, note);
+            if (pandora.user.username == item.user || pandora.hasCapability('canSeeAllTasks')) {
+                pandora.api.getTasks({
+                    user: pandora.hasCapability('canSeeAllTasks') ? '' : pandora.user.username
+                }, function(result_) {
+                    var tasks = result_.data.items.filter(function(task) { return task.item == item})
+                    if (tasks.length == 0) {
+                        html = Ox._(
+                            '<i>{0}</i> is currently processed. '
+                            + '{1} view will be available in a moment.',
+                            [result.data.title, Ox._(pandora.user.ui.itemView)]
+                        )
+                    }
+                    note.html(html)
+                })
+            } else {
+                note.html(html)
+            }
+
             pandora.site.itemViews.filter(function(view) {
                 return view.id == 'documents';
             }).length && pandora.api.get({

From f012d999425f9fc6b8eaae7c4ca4f9fe7b98715c Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Tue, 12 Nov 2019 17:50:24 +0000
Subject: [PATCH 063/139] shared itemView utils

---
 static/js/infoView.indiancinema.js | 19 +++----------------
 static/js/infoView.js              | 17 ++---------------
 static/js/infoViewUtils.js         | 16 ++++++++++++++++
 3 files changed, 21 insertions(+), 31 deletions(-)
 create mode 100644 static/js/infoViewUtils.js

diff --git a/static/js/infoView.indiancinema.js b/static/js/infoView.indiancinema.js
index 9776dd42..277c706d 100644
--- a/static/js/infoView.indiancinema.js
+++ b/static/js/infoView.indiancinema.js
@@ -621,19 +621,6 @@ pandora.ui.infoView = function(data, isMixed) {
     $('<div>').css({height: '16px'}).appendTo($statistics);
 
 
-    function cleanupDate(value) {
-        if (/\d{2}-\d{2}-\d{4}/.test(value)) {
-            value = Ox.reverse(value.split('-')).join('-')
-        }
-        if (/\d{4}i\/\d{2}\/\d{d}/.test(value)) {
-            value = value.split('/').join('-')
-        }
-        if (/\d{2}\/\d{2}\/\d{4}/.test(value)) {
-            value = Ox.reverse(value.split('/')).join('-')
-        }
-        return value
-    }
-
     function editMetadata(key, value) {
         if (value != data[key]) {
             var itemKey = Ox.getObjectById(pandora.site.itemKeys, key);
@@ -661,7 +648,7 @@ pandora.ui.infoView = function(data, isMixed) {
                 edit[key] = value;
             }
             if (itemKey && itemKey.type && itemKey.type[0] == 'date') {
-                edit[key] = edit[key].map(cleanupDate);
+                edit[key] = edit[key].map(pandora.cleanupDate);
             }
             pandora.api.edit(edit, function(result) {
                 if (!isMultiple) {
@@ -763,14 +750,14 @@ pandora.ui.infoView = function(data, isMixed) {
             specialListKeys.indexOf(key) > -1 && itemKey && itemKey.type[0] == 'date'
         ) {
             ret = value.split('; ').map(function(date) {
-                date = cleanupDate(date)
+                date = pandora.cleanupDate(date)
                 return date ? formatLink(Ox.formatDate(date,
                     ['', '%Y', '%B %Y', '%B %e, %Y'][date.split('-').length],
                     true
                 ), key, date) : '';
             }).join('; ');
         } else if (['releasedate'].indexOf(key) > -1) {
-            value = cleanupDate(value);
+            value = pandora.cleanupDate(value);
             ret = value ? Ox.formatDate(value,
                 ['', '%Y', '%B %Y', '%B %e, %Y'][value.split('-').length],
                 true
diff --git a/static/js/infoView.js b/static/js/infoView.js
index 9c18ff69..a484ac27 100644
--- a/static/js/infoView.js
+++ b/static/js/infoView.js
@@ -371,19 +371,6 @@ pandora.ui.infoView = function(data, isMixed) {
 
     $('<div>').css({height: '16px'}).appendTo($statistics);
 
-    function cleanupDate(value) {
-        if (/\d{2}-\d{2}-\d{4}/.test(value)) {
-            value = Ox.reverse(value.split('-')).join('-')
-        }
-        if (/\d{4}i\/\d{2}\/\d{d}/.test(value)) {
-            value = value.split('/').join('-')
-        }
-        if (/\d{2}\/\d{2}\/\d{4}/.test(value)) {
-            value = Ox.reverse(value.split('/')).join('-')
-        }
-        return value
-    }
-
     function editMetadata(key, value) {
         if (value != data[key]) {
             var itemKey = Ox.getObjectById(pandora.site.itemKeys, key);
@@ -400,7 +387,7 @@ pandora.ui.infoView = function(data, isMixed) {
                 edit[key] = value ? value : null;
             }
             if (itemKey && itemKey.type && itemKey.type[0] == 'date') {
-                edit[key] = edit[key].map(cleanupDate);
+                edit[key] = edit[key].map(pandora.cleanupDate);
             }
             pandora.api.edit(edit, function(result) {
                 if (!isMultiple) {
@@ -474,7 +461,7 @@ pandora.ui.infoView = function(data, isMixed) {
             listKeys.indexOf(key) > -1 && Ox.getObjectById(pandora.site.itemKeys, key).type[0] == 'date'
         ) {
             ret = value.split('; ').map(function(date) {
-                date = cleanupDate(date)
+                date = pandora.cleanupDate(date)
                 return date ? formatLink(Ox.formatDate(date,
                     ['', '%Y', '%B %Y', '%B %e, %Y'][date.split('-').length],
                     true
diff --git a/static/js/infoViewUtils.js b/static/js/infoViewUtils.js
new file mode 100644
index 00000000..9ab90665
--- /dev/null
+++ b/static/js/infoViewUtils.js
@@ -0,0 +1,16 @@
+'use strict';
+
+
+pandora.cleanupDate = function(value) {
+    if (/\d{2}-\d{2}-\d{4}/.test(value)) {
+        value = Ox.reverse(value.split('-')).join('-')
+    }
+    if (/\d{4}i\/\d{2}\/\d{d}/.test(value)) {
+        value = value.split('/').join('-')
+    }
+    if (/\d{2}\/\d{2}\/\d{4}/.test(value)) {
+        value = Ox.reverse(value.split('/')).join('-')
+    }
+    return value
+};
+

From 6e10bb17d2b5bc5392fd65973174595852dd9a0b Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Tue, 12 Nov 2019 18:18:12 +0000
Subject: [PATCH 064/139] display any additional fields specified in config on
 info page

---
 static/js/infoView.js | 30 ++++++++++++++++++++++++++++--
 1 file changed, 28 insertions(+), 2 deletions(-)

diff --git a/static/js/infoView.js b/static/js/infoView.js
index a484ac27..abdc4180 100644
--- a/static/js/infoView.js
+++ b/static/js/infoView.js
@@ -38,6 +38,15 @@ pandora.ui.infoView = function(data, isMixed) {
             })
         ),
         posterKeys = nameKeys.concat(['title', 'year']),
+        displayedKeys = [ // FIXME: can tis be a flag in the config?
+            'title', 'notes', 'name', 'summary', 'id',
+            'hue', 'saturation', 'lightness', 'cutsperminute', 'volume',
+            'user', 'rightslevel', 'bitrate', 'timesaccessed',
+            'numberoffiles', 'numberofannotations', 'numberofcuts', 'words', 'wordsperminute',
+            'duration', 'aspectratio', 'pixels', 'size', 'resolution',
+            'created', 'modified', 'accessed',
+            'random'
+        ],
         statisticsWidth = 128,
 
         $bar = Ox.Bar({size: 16})
@@ -236,14 +245,18 @@ pandora.ui.infoView = function(data, isMixed) {
         )
         .appendTo($text);
 
-    // Director, Year and Country ----------------------------------------------
+    // Director, Year and Country, Language --------------------------------
 
-    renderGroup(['director', 'year', 'country']);
+    renderGroup(['director', 'year', 'country', 'language']);
 
     // Featuring ----------------------------------------------
 
     renderGroup(['featuring']);
 
+    // Render any remaing keys defined in config
+
+    renderRemainingKeys();
+
     // Summary -----------------------------------------------------------------
 
     if (canEdit || data.summary) {
@@ -278,6 +291,7 @@ pandora.ui.infoView = function(data, isMixed) {
             .appendTo($text);
     }
 
+
     // Duration, Aspect Ratio --------------------------------------------------
     if (!isMultiple) {
         ['duration', 'aspectratio'].forEach(function(key) {
@@ -575,6 +589,7 @@ pandora.ui.infoView = function(data, isMixed) {
 
     function renderGroup(keys) {
         var $element;
+        keys.forEach(function(key) { displayedKeys.push(key) });
         if (canEdit || keys.filter(function(key) {
             return data[key];
         }).length) {
@@ -606,6 +621,17 @@ pandora.ui.infoView = function(data, isMixed) {
         }
     }
 
+    function renderRemainingKeys() {
+        var keys = pandora.site.itemKeys.filter(function(item) {
+            return item.id != '*' && item.type != 'layer' && !Ox.contains(displayedKeys, item.id);
+        }).map(function(item) {
+            return item.id;
+        });
+        if (keys.length) {
+            renderGroup(keys)
+        }
+    }
+
     function renderRightsLevel() {
         var $rightsLevelElement = getRightsLevelElement(data.rightslevel),
             $rightsLevelSelect;

From 69083d15211a0fb86af8ec79a6aa386221e28e38 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Tue, 12 Nov 2019 18:33:24 +0000
Subject: [PATCH 065/139] don't return empty annotations

---
 pandora/clip/models.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/pandora/clip/models.py b/pandora/clip/models.py
index f4f55651..37641c15 100644
--- a/pandora/clip/models.py
+++ b/pandora/clip/models.py
@@ -121,7 +121,9 @@ class MetaClip(object):
                         annotations = annotations.filter(q)
                 entity_cache = {}
                 j['annotations'] = [
-                    a.json(keys=['value', 'id', 'layer'], entity_cache=entity_cache) for a in annotations
+                    a.json(keys=['value', 'id', 'layer'], entity_cache=entity_cache)
+                    for a in annotations
+                    if a.value
                 ]
             if 'layers' in keys:
                 j['layers'] = self.get_layers()

From dd7e1d0ba0bebb28e34fd65127d21d3d7c423d29 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Wed, 13 Nov 2019 12:41:38 +0000
Subject: [PATCH 066/139] select subtitles layer by default if selecting an srt
 file

---
 static/js/importAnnotationsDialog.js | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/static/js/importAnnotationsDialog.js b/static/js/importAnnotationsDialog.js
index 7f2f6174..87593f15 100644
--- a/static/js/importAnnotationsDialog.js
+++ b/static/js/importAnnotationsDialog.js
@@ -68,7 +68,14 @@ pandora.ui.importAnnotationsDialog = function(options) {
         .bindEvent({
             change: function(data) {
                 $status.empty();
-                data.value.length && $formatSelect.options({value: Ox.last(data.value[0].name.split('.'))});
+                if (data.value.length) {
+                    var format = Ox.last(data.value[0].name.split('.'));
+                    $formatSelect.options({value: format});
+                    var subtitlesLayer = pandora.getSubtitlesLayer()
+                    if (subtitlesLayer && format == 'srt' && Ox.getObjectById(layers, subtitlesLayer))  {
+                        $layerSelect.options({value: subtitlesLayer})
+                    }
+                }
                 that[
                     data.value.length ? 'enableButton' : 'disableButton'
                 ]('import');

From ef4921b16dacee5aedc0f35c2dac2659b01e3ef8 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Wed, 13 Nov 2019 18:09:12 +0000
Subject: [PATCH 067/139] add keywords layer

---
 pandora/config.0xdb.jsonc | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/pandora/config.0xdb.jsonc b/pandora/config.0xdb.jsonc
index bcae7125..343c44b0 100644
--- a/pandora/config.0xdb.jsonc
+++ b/pandora/config.0xdb.jsonc
@@ -98,7 +98,7 @@
     text of clips (in grid view, below the icon). Excluding a layer from this
     list means it will not be included in find annotations.
     */
-    "clipLayers": ["subtitles"],
+    "clipLayers": ["subtitles", "keywords"],
     "documentKeys": [
         {
             "id": "*",
@@ -710,6 +710,14 @@
             "advanced": true,
             "find": true
         },
+        {
+            "id": "keywords",
+            "title": "Keywords",
+            "type": "layer",
+            "autocomplete": true,
+            "filter": true,
+            "find": true
+        },
         {
             "id": "subtitles",
             "title": "Subtitles",
@@ -998,6 +1006,15 @@
             tooltip that appears on mouseover.
     */
     "layers": [
+        {
+            "id": "keywords",
+            "title": "Keywords",
+            "canAddAnnotations": {"member": true, "staff": true, "admin": true},
+            "item": "Keyword",
+            "autocomplete": true,
+            "overlap": true,
+            "type": "string"
+        },
         {
             "id": "privatenotes",
             "title": "Private Notes",

From 95c08e929e5d29aed2a53da28cc923b5bc0a3a77 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Fri, 15 Nov 2019 15:21:29 +0100
Subject: [PATCH 068/139] show remaining document keys

---
 static/js/documentInfoView.js | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/static/js/documentInfoView.js b/static/js/documentInfoView.js
index 9b3cd79e..c9d65fd2 100644
--- a/static/js/documentInfoView.js
+++ b/static/js/documentInfoView.js
@@ -28,6 +28,13 @@ pandora.ui.documentInfoView = function(data, isMixed) {
         }).map(function(key){
             return key.id;
         }),
+        displayedKeys = [ // FIXME: can tis be a flag in the config?
+            'title', 'notes', 'name', 'description', 'id',
+            'user', 'rightslevel', 'timesaccessed',
+            'extension', 'dimensions', 'size', 'matches',
+            'created', 'modified', 'accessed',
+            'random', 'entity'
+        ],
         statisticsWidth = 128,
 
         $bar = Ox.Bar({size: 16})
@@ -234,6 +241,10 @@ pandora.ui.documentInfoView = function(data, isMixed) {
 
     Ox.getObjectById(pandora.site.documentKeys, 'keywords') && renderGroup(['keywords'])
 
+    // Render any remaing keys defined in config
+
+    renderRemainingKeys();
+
 
     // Description -------------------------------------------------------------
 
@@ -321,6 +332,7 @@ pandora.ui.documentInfoView = function(data, isMixed) {
 
     }
 
+
     // Extension, Dimensions, Size ---------------------------------------------
 
     ['extension', 'dimensions', 'size'].forEach(function(key) {
@@ -533,6 +545,7 @@ pandora.ui.documentInfoView = function(data, isMixed) {
 
     function renderGroup(keys) {
         var $element;
+        keys.forEach(function(key) { displayedKeys.push(key) });
         if (canEdit || keys.filter(function(key) {
             return data[key];
         }).length) {
@@ -565,6 +578,17 @@ pandora.ui.documentInfoView = function(data, isMixed) {
         return $element;
     }
 
+    function renderRemainingKeys() {
+        var keys = pandora.site.documentKeys.filter(function(item) {
+            return item.id != '*' && !Ox.contains(displayedKeys, item.id);
+        }).map(function(item) {
+            return item.id;
+        });
+        if (keys.length) {
+            renderGroup(keys)
+        }
+    }
+
     function renderRightsLevel() {
         var $rightsLevelElement = getRightsLevelElement(data.rightslevel),
             $rightsLevelSelect;

From e06b263237b6b6c1e635f79963b25e87bb4c32b6 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Fri, 15 Nov 2019 16:44:43 +0100
Subject: [PATCH 069/139] always enable default filters

---
 pandora/app/config.py | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/pandora/app/config.py b/pandora/app/config.py
index 3a9dd36d..bbe15994 100644
--- a/pandora/app/config.py
+++ b/pandora/app/config.py
@@ -71,7 +71,7 @@ def load_config(init=False):
         if getattr(settings, 'SITEURL', False):
             config['site']['url'] = settings.SITEURL
         settings.URL = config['site']['url']
-        settings.EMAIL_SUBJECT_PREFIX = '[%s]'%settings.SITENAME
+        settings.EMAIL_SUBJECT_PREFIX = '[%s]' % settings.SITENAME
         settings.DEFAULT_FROM_EMAIL = config['site']['email']['system']
         settings.SERVER_EMAIL = config['site']['email']['system']
         config['site']['videoprefix'] = settings.VIDEO_PREFIX
@@ -79,9 +79,9 @@ def load_config(init=False):
         config['site']['googleapikey'] = getattr(settings, 'GOOGLE_API_KEY')
         config['site']['version'] = get_version()
         config['site']['dontValidateUser'] = not settings.AUTH_CHECK_USERNAME
-        if not 'folderdepth' in config['site']:
+        if 'folderdepth' not in config['site']:
             config['site']['folderdepth'] = settings.USE_IMDB and 4 or 3
-        if 'sendReferrer' in config and not 'sendReferrer' in config['site']:
+        if 'sendReferrer' in config and 'sendReferrer' not in config['site']:
             config['site']['sendReferrer'] = config.pop('sendReferrer')
 
         # enable default filters if needed
@@ -91,6 +91,13 @@ def load_config(init=False):
                 key['filter'] = True
                 sys.stderr.write('enabled filter for "%s" since its used as default filter.\n' % (key['id']))
 
+        # enable default document filters if needed
+        default_filters = [f['id'] for f in config['user']['ui']['documentFilters']]
+        for key in config['documentKeys']:
+            if key['id'] in default_filters and not key.get('filter'):
+                key['filter'] = True
+                sys.stderr.write('enabled filter for documeny key "%s" since its used as default filter.\n' % (key['id']))
+
         config['keys'] = {}
         for key in config['itemKeys']:
             config['keys'][key['id']] = key

From f8c1c3e328e69f478618c24080877ea3c6b84cec Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Sat, 16 Nov 2019 00:18:06 +0100
Subject: [PATCH 070/139] reuse lookup

---
 static/js/documentFilter.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/static/js/documentFilter.js b/static/js/documentFilter.js
index 960eca09..ab7f39fe 100644
--- a/static/js/documentFilter.js
+++ b/static/js/documentFilter.js
@@ -4,7 +4,7 @@ pandora.ui.documentFilter = function(id) {
     var i = Ox.getIndexById(pandora.user.ui.documentFilters, id),
         filter = Ox.getObjectById(pandora.site.documentFilters, id),
         panelWidth = Ox.$document.width() - (pandora.user.ui.showSidebar * pandora.user.ui.sidebarSize) - 1,
-        title = Ox._(Ox.getObjectById(pandora.site.documentFilters, id).title),
+        title = Ox._(filter.title),
         //width = pandora.getFilterWidth(i, panelWidth),
         that = Ox.TableList({
             _selected: !pandora.user.ui.showFilters

From fe023c2f974631fae41eee2ae8ec3efd83930971 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Sun, 17 Nov 2019 13:02:12 +0100
Subject: [PATCH 071/139] fulltext search for documents

optional fulltext search for documents using elasticsearch
text is extracted from pdfs and via ocr from images
---
 pandora/document/fulltext.py | 85 ++++++++++++++++++++++++++++++++++++
 pandora/document/managers.py |  6 +++
 pandora/document/models.py   |  6 ++-
 pandora/settings.py          |  3 ++
 requirements.txt             |  1 +
 vm/pandora_install.sh        |  3 ++
 6 files changed, 102 insertions(+), 2 deletions(-)
 create mode 100644 pandora/document/fulltext.py

diff --git a/pandora/document/fulltext.py b/pandora/document/fulltext.py
new file mode 100644
index 00000000..3990b63d
--- /dev/null
+++ b/pandora/document/fulltext.py
@@ -0,0 +1,85 @@
+import subprocess
+
+from django.conf import settings
+
+
+def extract_text(pdf):
+    cmd = ['pdftotext', pdf, '-']
+    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    stdout, stderr = p.communicate()
+    stdout = stdout.decode()
+    return stdout.strip()
+
+def ocr_image(path):
+    cmd = ['tesseract', path, '-', 'txt']
+    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    stdout, stderr = p.communicate()
+    stdout = stdout.decode()
+    return stdout.strip()
+
+class FulltextMixin:
+    _ES_INDEX = "document-index"
+
+    @classmethod
+    def elasticsearch(cls):
+        from elasticsearch import Elasticsearch
+        es = Elasticsearch(settings.ELASTICSEARCH_HOST)
+        return es
+
+    def extract_fulltext(self):
+        if self.extension == 'pdf':
+            return extract_text(self.file.path)
+        elif self.extension in ('png', 'jpg'):
+            return ocr_image(self.file.path)
+        elif self.extension == 'html':
+            return self.data.get('text', '')
+        return ''
+
+    def delete_fulltext(self):
+        res = self.elasticsearch().delete(index=self._ES_INDEX, doc_type='document', id=self.id)
+
+    def update_fulltext(self):
+        text = self.extract_fulltext()
+        if text:
+            doc = {
+                'text': text.lower()
+            }
+            res = self.elasticsearch().index(index=self._ES_INDEX, doc_type='document', id=self.id, body=doc)
+
+    @classmethod
+    def find_fulltext(cls, query):
+        ids = cls.find_fulltext_ids(query)
+        return cls.objects.filter(id__in=ids)
+
+    @classmethod
+    def find_fulltext_ids(cls, query):
+        if query[0] == '"' and query[-1] == '"':
+            query = {
+                "match_phrase": {
+                    "text": query.lower()[1:-1]
+                },
+            }
+        else:
+            query = {
+                "match": {
+                    "text": {
+                        "query": query.lower(),
+                        "operator": "and"
+                    }
+                }
+            }
+        ids = []
+        res = None
+        from_ = 0
+        es = cls.elasticsearch()
+        while not res or len(ids) < res['hits']['total']['value']:
+            res = es.search(index=cls._ES_INDEX, body={
+                "from": from_,
+                "_source": False,
+                "query": query
+            })
+            if not res['hits']['hits']:
+                break
+            ids += [int(r['_id']) for r in res['hits']['hits']]
+            from_ += len(res['hits']['hits'])
+        return ids
diff --git a/pandora/document/managers.py b/pandora/document/managers.py
index 7c97b0f1..b55afeea 100644
--- a/pandora/document/managers.py
+++ b/pandora/document/managers.py
@@ -128,6 +128,12 @@ def buildCondition(k, op, v, user, exclude=False, owner=None):
             else:
                 q = Q(id=0)
         return q
+    elif key_type == 'fulltext':
+        qs = models.Document.find_fulltext_ids(v)
+        q = Q(id__in=qs)
+        if exclude:
+            q = ~Q(id__in=qs)
+        return q
     elif key_type == 'boolean':
         q = Q(**{'find__key': k, 'find__value': v})
         if exclude:
diff --git a/pandora/document/models.py b/pandora/document/models.py
index 81fd4dd1..a436360a 100644
--- a/pandora/document/models.py
+++ b/pandora/document/models.py
@@ -30,6 +30,7 @@ from user.models import Group
 
 from . import managers
 from . import utils
+from .fulltext import FulltextMixin
 
 User = get_user_model()
 
@@ -40,7 +41,7 @@ def get_path(f, x):
     return f.path(x)
 
 @python_2_unicode_compatible
-class Document(models.Model):
+class Document(models.Model, FulltextMixin):
 
     created = models.DateTimeField(auto_now_add=True)
     modified = models.DateTimeField(auto_now=True)
@@ -153,7 +154,7 @@ class Document(models.Model):
                 i = key['id']
                 if i == 'rightslevel':
                     save(i, self.rightslevel)
-                elif i not in ('*', 'dimensions') and i not in self.facet_keys:
+                elif i not in ('*', 'dimensions', 'fulltext') and i not in self.facet_keys:
                     value = data.get(i)
                     if isinstance(value, list):
                         value = u'\n'.join(value)
@@ -277,6 +278,7 @@ class Document(models.Model):
                 self.update_sort()
                 self.update_find()
                 self.update_facets()
+                self.update_fulltext()
             new = False
         else:
             new = True
diff --git a/pandora/settings.py b/pandora/settings.py
index 1e8f1d56..e4f46698 100644
--- a/pandora/settings.py
+++ b/pandora/settings.py
@@ -204,6 +204,9 @@ CELERY_BROKER_URL = 'amqp://pandora:box@localhost:5672//pandora'
 
 SEND_CELERY_ERROR_EMAILS = False
 
+# Elasticsearch
+ELASTICSEARCH_HOST = None
+
 #with apache x-sendfile or lighttpd set this to True
 XSENDFILE = False
 
diff --git a/requirements.txt b/requirements.txt
index d26e9bd9..ba40eaae 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -11,3 +11,4 @@ tornado<5
 geoip2==2.9.0
 youtube-dl>=2019.4.30
 python-memcached
+elasticsearch
diff --git a/vm/pandora_install.sh b/vm/pandora_install.sh
index 3a62e90d..1c635773 100755
--- a/vm/pandora_install.sh
+++ b/vm/pandora_install.sh
@@ -91,6 +91,7 @@ apt-get install -y \
     python3-lxml \
     python3-html5lib \
     python3-ox \
+    python3-elasticsearch \
     oxframe \
     ffmpeg \
     mkvtoolnix \
@@ -98,6 +99,8 @@ apt-get install -y \
     imagemagick \
     poppler-utils \
     ipython3 \
+    tesseract-ocr \
+    tesseract-ocr-eng \
     postfix \
     postgresql-client $EXTRA
 

From f0a46db39f8b99ce2f3a549e50a7e2e3e7d7c055 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Sun, 17 Nov 2019 16:34:26 +0100
Subject: [PATCH 072/139] script to install elasticsearch

---
 vm/install_elasticsearch.sh | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100755 vm/install_elasticsearch.sh

diff --git a/vm/install_elasticsearch.sh b/vm/install_elasticsearch.sh
new file mode 100755
index 00000000..87e7a0ba
--- /dev/null
+++ b/vm/install_elasticsearch.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+curl -sL https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
+echo "deb https://artifacts.elastic.co/packages/7.x/apt stable main" > /etc/apt/sources.list.d/elasticsearch.list
+apt-get update -qq
+apt-get -y install elasticsearch
+systemctl enable elasticsearch.service
+systemctl start elasticsearch.service
+#curl -X GET "http://localhost:9200/?pretty"

From 923ed5bf83f505a2c4adcbff10991425678c5566 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Mon, 18 Nov 2019 18:08:33 +0100
Subject: [PATCH 073/139] keywords already used, use tag instead

---
 pandora/config.0xdb.jsonc | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/pandora/config.0xdb.jsonc b/pandora/config.0xdb.jsonc
index 343c44b0..8440de9a 100644
--- a/pandora/config.0xdb.jsonc
+++ b/pandora/config.0xdb.jsonc
@@ -711,8 +711,8 @@
             "find": true
         },
         {
-            "id": "keywords",
-            "title": "Keywords",
+            "id": "tags",
+            "title": "Tags",
             "type": "layer",
             "autocomplete": true,
             "filter": true,
@@ -1007,10 +1007,10 @@
     */
     "layers": [
         {
-            "id": "keywords",
-            "title": "Keywords",
+            "id": "tags",
+            "title": "Tags",
             "canAddAnnotations": {"member": true, "staff": true, "admin": true},
-            "item": "Keyword",
+            "item": "Tag",
             "autocomplete": true,
             "overlap": true,
             "type": "string"

From e859e29f9a5b742cf7872546e416b2ad0009e881 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Tue, 19 Nov 2019 16:45:18 +0100
Subject: [PATCH 074/139] only render if key is defined

---
 static/js/infoView.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/static/js/infoView.js b/static/js/infoView.js
index abdc4180..e95d2682 100644
--- a/static/js/infoView.js
+++ b/static/js/infoView.js
@@ -251,7 +251,7 @@ pandora.ui.infoView = function(data, isMixed) {
 
     // Featuring ----------------------------------------------
 
-    renderGroup(['featuring']);
+    Ox.getObjectById(pandora.site.itemKeys, 'featuring') && renderGroup(['featuring']);
 
     // Render any remaing keys defined in config
 

From a1725a5f9257ca8d473f169832eb3cdd751e99fa Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Tue, 19 Nov 2019 17:23:23 +0100
Subject: [PATCH 075/139] only map keys if not already defined

---
 static/js/metadataDialog.js | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/static/js/metadataDialog.js b/static/js/metadataDialog.js
index 06ee9d7d..c9e5c3ea 100644
--- a/static/js/metadataDialog.js
+++ b/static/js/metadataDialog.js
@@ -181,6 +181,9 @@ pandora.ui.metadataDialog = function(data) {
     }
 
     function getKey(key) {
+        if (Ox.getObjectById(pandora.site.itemKeys, key) && mapKeys[key]) {
+            return key
+        }
         return mapKeys[key] || key;
     }
 

From 8240a9ede678a80739ca83c7cd0acf1f52ecbf6d Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Wed, 20 Nov 2019 13:28:46 +0100
Subject: [PATCH 076/139] enable languages

---
 static/js/importAnnotationsDialog.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/static/js/importAnnotationsDialog.js b/static/js/importAnnotationsDialog.js
index 87593f15..559a8fa7 100644
--- a/static/js/importAnnotationsDialog.js
+++ b/static/js/importAnnotationsDialog.js
@@ -75,6 +75,7 @@ pandora.ui.importAnnotationsDialog = function(options) {
                     if (subtitlesLayer && format == 'srt' && Ox.getObjectById(layers, subtitlesLayer))  {
                         $layerSelect.options({value: subtitlesLayer})
                     }
+                    updateLanguageSelect();
                 }
                 that[
                     data.value.length ? 'enableButton' : 'disableButton'

From 5863e7fb319a7634c16fb2df6b1dba84345b31a6 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Wed, 20 Nov 2019 13:37:16 +0100
Subject: [PATCH 077/139] fix mixed notes

---
 static/js/infoView.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/static/js/infoView.js b/static/js/infoView.js
index e95d2682..6e68a95a 100644
--- a/static/js/infoView.js
+++ b/static/js/infoView.js
@@ -368,7 +368,7 @@ pandora.ui.infoView = function(data, isMixed) {
             .append(
                 Ox.EditableContent({
                         height: 128,
-                        placeholder: formatLight(Ox._(isMixed ? 'Mixed notes' : 'No notes')),
+                        placeholder: formatLight(Ox._(isMixed.notes ? 'Mixed notes' : 'No notes')),
                         tooltip: pandora.getEditTooltip(),
                         type: 'textarea',
                         value: data.notes || '',

From 6c7c08448dfb65953f481cbf40f95f10c51968e2 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Wed, 20 Nov 2019 15:29:41 +0100
Subject: [PATCH 078/139] don't list undefined as year

---
 static/js/idDialog.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/static/js/idDialog.js b/static/js/idDialog.js
index ba82d19f..e06f5bdd 100644
--- a/static/js/idDialog.js
+++ b/static/js/idDialog.js
@@ -97,7 +97,7 @@ pandora.ui.idDialog = function(data) {
                         labelWidth: 128,
                         value: Ox.decodeHTMLEntities(key == 'director' && data[key]
                             ? data[key].join(', ')
-                            : ('' + data[key])),
+                            : ('' + (data[key] || ''))),
                         width: formWidth
                     })
                     .css({display: 'inline-block', margin: '3px'})

From a5e231f43cff8f583436cb16841e055f3e5fb1ef Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Wed, 20 Nov 2019 16:34:17 +0100
Subject: [PATCH 079/139] fix drag and drop of videos

---
 static/js/utils.js | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/static/js/utils.js b/static/js/utils.js
index ae2758cc..c3978345 100644
--- a/static/js/utils.js
+++ b/static/js/utils.js
@@ -424,7 +424,10 @@ pandora.uploadDroppedFiles = function(files) {
             }
         }).open();
     } else {
-        pandora.ui.addItemDialog({files: files}).open()
+        pandora.ui.addItemDialog({
+            files: files,
+            selected: 'upload'
+        }).open()
     }
 };
 

From 007b9394c742511c8105398b730bc2cbe90fff42 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Fri, 29 Nov 2019 20:45:10 +0100
Subject: [PATCH 080/139] fix image urls

---
 static/js/document.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/static/js/document.js b/static/js/document.js
index a13332d2..1f517fab 100644
--- a/static/js/document.js
+++ b/static/js/document.js
@@ -80,7 +80,7 @@ pandora.ui.document = function() {
                     imageHeight: item.dimensions[1],
                     imagePreviewURL: pandora.getMediaURL('/documents/' + item.id + '/256p.jpg?' + item.modified),
                     imageURL: pandora.getMediaURL('/documents/' + item.id + '/'
-                        + item.title + '.' + item.extension + '?' + item.modified),
+                        + pandora.safePDFName(item.title) + '.' + item.extension + '?' + item.modified),
                     imageWidth: item.dimensions[0],
                     width: that.width()
                 }).css({

From f311062b5a31057a24d337112499fe999b124980 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Fri, 29 Nov 2019 20:48:40 +0100
Subject: [PATCH 081/139] better function name

---
 static/js/document.js | 4 ++--
 static/js/utils.js    | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/static/js/document.js b/static/js/document.js
index 1f517fab..22fee0a0 100644
--- a/static/js/document.js
+++ b/static/js/document.js
@@ -66,7 +66,7 @@ pandora.ui.document = function() {
                         ? pandora.user.ui.documents[item.id].position
                         : 1,
                     url: '/documents/' + item.id + '/'
-                        + pandora.safePDFName(item.title + '.' + item.extension),
+                        + pandora.safeDocumentName(item.title) + '.' + item.extension,
                     width: that.width(),
                     zoom: 'fit'
                 })
@@ -80,7 +80,7 @@ pandora.ui.document = function() {
                     imageHeight: item.dimensions[1],
                     imagePreviewURL: pandora.getMediaURL('/documents/' + item.id + '/256p.jpg?' + item.modified),
                     imageURL: pandora.getMediaURL('/documents/' + item.id + '/'
-                        + pandora.safePDFName(item.title) + '.' + item.extension + '?' + item.modified),
+                        + pandora.safeDocumentName(item.title) + '.' + item.extension + '?' + item.modified),
                     imageWidth: item.dimensions[0],
                     width: that.width()
                 }).css({
diff --git a/static/js/utils.js b/static/js/utils.js
index c3978345..a4de127d 100644
--- a/static/js/utils.js
+++ b/static/js/utils.js
@@ -2640,7 +2640,7 @@ pandora.openURL = function(url) {
     }
 };
 
-pandora.safePDFName = function(name) {
+pandora.safeDocumentName = function(name) {
     name = name.replace('?', '_').replace('#', '_');
     return name;
 };

From 2f7e8a078038e672e480c25463cde7bbd0ce47b5 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Sun, 1 Dec 2019 01:20:58 +0100
Subject: [PATCH 082/139] year db is only 4 chars long

---
 pandora/item/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pandora/item/models.py b/pandora/item/models.py
index f625fb80..ba98cb1e 100644
--- a/pandora/item/models.py
+++ b/pandora/item/models.py
@@ -1047,7 +1047,7 @@ class Item(models.Model):
                         value = len(value)
                     set_value(s, name, value)
                 elif sort_type == 'year':
-                    value = self.get(source)
+                    value = self.get(source)[:4]
                     set_value(s, name, value)
                 elif sort_type == 'date':
                     value = value_ = self.get(source)

From 0172a152d33d6a9e9577b2812b568c9fb8e4517c Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Sun, 1 Dec 2019 15:25:45 +0100
Subject: [PATCH 083/139] dont use 'fulltext' type, add fulltext flag

---
 pandora/document/managers.py | 7 ++++++-
 pandora/document/models.py   | 4 +++-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/pandora/document/managers.py b/pandora/document/managers.py
index b55afeea..7f045390 100644
--- a/pandora/document/managers.py
+++ b/pandora/document/managers.py
@@ -36,6 +36,8 @@ def get_key_type(k):
     }.get(key_type, key_type)
     return key_type
 
+
+
 def parseCondition(condition, user, item=None, owner=None):
     '''
     '''
@@ -68,6 +70,9 @@ def buildCondition(k, op, v, user, exclude=False, owner=None):
         k = 'collection'
 
     key_type = get_key_type(k)
+
+    key_config = (utils.get_by_id(settings.CONFIG['documentKeys'], k) or {'type': 'string'}).get('type')
+
     facet_keys = models.Document.facet_keys
     if k == 'id':
         if op == '&' and isinstance(v, list):
@@ -128,7 +133,7 @@ def buildCondition(k, op, v, user, exclude=False, owner=None):
             else:
                 q = Q(id=0)
         return q
-    elif key_type == 'fulltext':
+    elif key_config.fulltext:
         qs = models.Document.find_fulltext_ids(v)
         q = Q(id__in=qs)
         if exclude:
diff --git a/pandora/document/models.py b/pandora/document/models.py
index a436360a..0c61f019 100644
--- a/pandora/document/models.py
+++ b/pandora/document/models.py
@@ -154,7 +154,9 @@ class Document(models.Model, FulltextMixin):
                 i = key['id']
                 if i == 'rightslevel':
                     save(i, self.rightslevel)
-                elif i not in ('*', 'dimensions', 'fulltext') and i not in self.facet_keys:
+                if key.get('fulltext'):
+                    continue
+                elif i not in ('*', 'dimensions') and i not in self.facet_keys:
                     value = data.get(i)
                     if isinstance(value, list):
                         value = u'\n'.join(value)

From ff4e32d769eeb51502ef3003ea80b3c57d80610c Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Sun, 1 Dec 2019 15:28:04 +0100
Subject: [PATCH 084/139] typo

---
 pandora/document/managers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pandora/document/managers.py b/pandora/document/managers.py
index 7f045390..8bbb6c1f 100644
--- a/pandora/document/managers.py
+++ b/pandora/document/managers.py
@@ -133,7 +133,7 @@ def buildCondition(k, op, v, user, exclude=False, owner=None):
             else:
                 q = Q(id=0)
         return q
-    elif key_config.fulltext:
+    elif key_config.get('fulltext'):
         qs = models.Document.find_fulltext_ids(v)
         q = Q(id__in=qs)
         if exclude:

From 697fee8c60a9e98886b0ab0d9ede111e28f310a9 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Sun, 1 Dec 2019 15:31:05 +0100
Subject: [PATCH 085/139] fix key_config

---
 pandora/document/managers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pandora/document/managers.py b/pandora/document/managers.py
index 8bbb6c1f..fa0cd64a 100644
--- a/pandora/document/managers.py
+++ b/pandora/document/managers.py
@@ -71,7 +71,7 @@ def buildCondition(k, op, v, user, exclude=False, owner=None):
 
     key_type = get_key_type(k)
 
-    key_config = (utils.get_by_id(settings.CONFIG['documentKeys'], k) or {'type': 'string'}).get('type')
+    key_config = (utils.get_by_id(settings.CONFIG['documentKeys'], k) or {'type': 'string'})
 
     facet_keys = models.Document.facet_keys
     if k == 'id':

From 96912f14b0ef45b596ac06aaa56b9ac6044ad5c7 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Sun, 1 Dec 2019 16:10:08 +0100
Subject: [PATCH 086/139] use pandora.safeDocumentName

---
 static/js/documentDialog.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/static/js/documentDialog.js b/static/js/documentDialog.js
index b2c17308..752bca35 100644
--- a/static/js/documentDialog.js
+++ b/static/js/documentDialog.js
@@ -196,7 +196,7 @@ pandora.ui.documentDialog = function(options) {
                         ? pandora.user.ui.documents[item.id].position
                         : 1,
                     url: '/documents/' + item.id + '/'
-                        + pandora.safePDFName(item.title + '.' + item.extension),
+                        + pandora.safeDocumentName(item.title) + '.' + item.extension,
                     width: dialogWidth,
                     zoom: 'fit'
                 })
@@ -211,7 +211,7 @@ pandora.ui.documentDialog = function(options) {
                     imageHeight: item.dimensions[1],
                     imagePreviewURL: pandora.getMediaURL('/documents/' + item.id + '/256p.jpg?' + item.modified),
                     imageURL: pandora.getMediaURL('/documents/' + item.id + '/'
-                        + item.title + '.' + item.extension + '?' + item.modified),
+                        + pandora.safeDocumentName(item.title) + '.' + item.extension + '?' + item.modified),
                     imageWidth: item.dimensions[0],
                     width: dialogWidth
                 })

From 44b4f092d1059e3f5c97b14f39c10a619b1a4e81 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Sun, 1 Dec 2019 16:17:58 +0100
Subject: [PATCH 087/139] only extract fulltext if fulltext key is defined

---
 pandora/document/fulltext.py | 25 +++++++++++++++----------
 1 file changed, 15 insertions(+), 10 deletions(-)

diff --git a/pandora/document/fulltext.py b/pandora/document/fulltext.py
index 3990b63d..d64cd258 100644
--- a/pandora/document/fulltext.py
+++ b/pandora/document/fulltext.py
@@ -27,10 +27,11 @@ class FulltextMixin:
         return es
 
     def extract_fulltext(self):
-        if self.extension == 'pdf':
-            return extract_text(self.file.path)
-        elif self.extension in ('png', 'jpg'):
-            return ocr_image(self.file.path)
+        if self.file:
+            if self.extension == 'pdf':
+                return extract_text(self.file.path)
+            elif self.extension in ('png', 'jpg'):
+                return ocr_image(self.file.path)
         elif self.extension == 'html':
             return self.data.get('text', '')
         return ''
@@ -38,13 +39,17 @@ class FulltextMixin:
     def delete_fulltext(self):
         res = self.elasticsearch().delete(index=self._ES_INDEX, doc_type='document', id=self.id)
 
+    def has_fulltext_key():
+        return bool([k for k in settings.CONFIG['documentKeys'] if k.get('fulltext')])
+
     def update_fulltext(self):
-        text = self.extract_fulltext()
-        if text:
-            doc = {
-                'text': text.lower()
-            }
-            res = self.elasticsearch().index(index=self._ES_INDEX, doc_type='document', id=self.id, body=doc)
+        if self.has_fulltext_key():
+            text = self.extract_fulltext()
+            if text:
+                doc = {
+                    'text': text.lower()
+                }
+                res = self.elasticsearch().index(index=self._ES_INDEX, doc_type='document', id=self.id, body=doc)
 
     @classmethod
     def find_fulltext(cls, query):

From 823b8f2136d69f861ac102614308f950a9a67c32 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Sun, 1 Dec 2019 16:20:20 +0100
Subject: [PATCH 088/139] foundations to directly use upoaded videos if format
 is usable

---
 pandora/archive/external.py | 11 +++++
 pandora/archive/extract.py  | 96 +++++++++++++++++++++++++++++++++++++
 pandora/archive/models.py   | 62 +++++++++++++++++++++++-
 pandora/settings.py         |  1 +
 4 files changed, 168 insertions(+), 2 deletions(-)

diff --git a/pandora/archive/external.py b/pandora/archive/external.py
index 4555f53f..69179e4f 100644
--- a/pandora/archive/external.py
+++ b/pandora/archive/external.py
@@ -97,6 +97,17 @@ def download(item_id, url):
         tmp = tmp.decode('utf-8')
     os.chdir(tmp)
     cmd = ['youtube-dl', '-q', media['url']]
+    if settings.CONFIG['video'].get('reuseUload', False):
+        max_resolution = max(settings.CONFIG['video']['resolutions'])
+        format = settings.CONFIG['video']['formats'][0]
+        if format == 'mp4':
+            cmd += [
+                '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/bestvideo+bestaudio',
+                '--merge-output-format', 'mp4'
+            ]
+        elif format == 'webm':
+            cmd += ['--merge-output-format', 'webm']
+
     p = subprocess.Popen(cmd,
                          stdout=subprocess.PIPE,
                          stderr=subprocess.PIPE, close_fds=True)
diff --git a/pandora/archive/extract.py b/pandora/archive/extract.py
index f85c464e..872ee7ea 100644
--- a/pandora/archive/extract.py
+++ b/pandora/archive/extract.py
@@ -698,3 +698,99 @@ def chop(video, start, end, subtitles=None, dest=None, encode=False):
         return f
     else:
         return None
+
+def has_faststart(path):
+    cmd = [settings.FFPROBE, '-v', 'trace', '-i', path]
+    p = subprocess.Popen(cmd, stdin=subprocess.PIPE,
+                         stdout=subprocess.PIPE,
+                         stderr=subprocess.STDOUT,
+                         close_fds=True)
+    stdout, stderr = p.communicate()
+    moov = "type:'moov'"
+    mdat = "type:'mdat'"
+    blocks = [b for b in stdout.decode().split('\n') if moov in b or mdat in b]
+    if blocks and moov in blocks[0]:
+        return True
+    return False
+
+def remux_stream(src, dst):
+    info = ox.avinfo(src)
+    if info.get('audio'):
+        audio = ['-c:a', 'copy']
+    else:
+        audio = []
+    if info.get('video'):
+        video = ['-c:v', 'copy']
+    else:
+        video = []
+    cmd = [
+        settings.FFMPEG,
+        '-nostats', '-loglevel', 'error',
+        '-map_metadata', '-1', '-sn',
+        '-i', src,
+    ] + video + [
+    ] + audio + [
+        '-movflags', '+faststart',
+        dst
+    ]
+    p = subprocess.Popen(cmd, stdin=subprocess.PIPE,
+                         stdout=open('/dev/null', 'w'),
+                         stderr=open('/dev/null', 'w'),
+                         close_fds=True)
+    p.wait()
+    return True, None
+
+
+def ffprobe(path, *args):
+    cmd = [settings.FFPROBE, '-loglevel', 'error', '-print_format', 'json', '-i', path] + list(args)
+    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
+    stdout, stderr = p.communicate()
+    return json.loads(stdout.decode())
+
+
+def get_chapters(path):
+    info = ffprobe(path, '-show_chapters')
+    chapters = []
+    n = 0
+    for chapter in info.get('chapters', []):
+        n += 1
+        chapters.append({
+            'in': chapter['start_time'],
+            'out': chapter['end_time'],
+            'value': chapter.get('tags', {}).get('title', 'Chapter %s' % n)
+        })
+    return chapters
+
+def get_text_subtitles(path):
+    subtitles = []
+    for stream in ffprobe(path, '-show_streams')['streams']:
+        if stream.get('codec_name') in ('subrip', 'aas', 'text'):
+            subtitles.append({
+                'index': stream['index'],
+                'language': stream['tags']['language'],
+            })
+    return subtitles
+
+def has_img_subtitles(path):
+    subtitles = []
+    for stream in ffprobe(path, '-show_streams')['streams']:
+        if stream.get('codec_type') == 'subtitle' and stream.get('codec_name') in ('dvbsub', 'pgssub'):
+            subtitles.append({
+                'index': stream['index'],
+                'language': stream['tags']['language'],
+            })
+    return subtitles
+
+def extract_subtitles(path, language=None):
+    extra = []
+    if language:
+        tracks = get_text_subtitles(path)
+        track = [t for t in tracks if t['language'] == language]
+        if track:
+            extra = ['-map', '0:%s' % track[0]['index']]
+        else:
+            raise Exception("unknown language: %s" % language)
+    cmd = ['ffmpeg', '-loglevel', 'error', '-i', path] + extra + ['-f', 'srt', '-']
+    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
+    stdout, stderr = p.communicate()
+    return ox.srt.loads(stdout.decode())
diff --git a/pandora/archive/models.py b/pandora/archive/models.py
index 199c90b1..75d25de7 100644
--- a/pandora/archive/models.py
+++ b/pandora/archive/models.py
@@ -376,6 +376,50 @@ class File(models.Model):
             return save_chunk(stream, stream.media, chunk, offset, name, done_cb)
         return False, 0
 
+    def extract_text_data(self):
+        if self.data:
+            for sub in extract.get_text_subtitles(self.data.path):
+                srt = extract.extract_subtitles(self.data.path, sub['language'])
+                # fixme add subtitles, possibly with language!
+            chapters = extract.get_chapters(self.data.path)
+            if chapters:
+                # fixme add chapters as notes
+                pass
+
+    def get_codec(self, type):
+        track = self.info.get(type)
+        if track:
+            return track[0].get('codec')
+
+    MP4_VCODECS = ['h264']
+    MP4_ACODECS = ['aac', None]
+    WEBM_VCODECS = ['vp8', 'vp9']
+    WEBM_ACODECS = ['vorbis', 'opus', None]
+
+    def can_remux(self):
+        config = settings.CONFIG['video']
+        height = self.info['video'][0]['height'] if self.info.get('video') else None
+        max_resolution = max(config['resolutions'])
+        if height <= max_resolution and self.extension in ('mov', 'mkv', 'mp4', 'm4v'):
+            vcodec = self.get_codec('video')
+            acodec = self.get_codec('audio')
+            if vcodec in self.MP4_VCODECS and acodec in self.MP4_ACODECS:
+                return True
+        return False
+
+    def can_stream(self):
+        config = settings.CONFIG['video']
+        height = self.info['video'][0]['height'] if self.info.get('video') else None
+        max_resolution = max(config['resolutions'])
+        if height <= max_resolution and config['formats'][0] == self.extension:
+            vcodec = self.get_codec('video')
+            acodec = self.get_codec('audio')
+            if self.extension in ['mp4', 'm4v'] and vcodec in self.MP4_VCODECS and acodec in self.MP4_ACODECS:
+                return extract.has_faststart(self.data.path)
+            elif self.extension == 'webm' and vcodec in self.WEBM_VCODECS and acodec in self.WEBM_ACODECS:
+                return True
+        return False
+
     def stream_resolution(self):
         config = settings.CONFIG['video']
         height = self.info['video'][0]['height'] if self.info.get('video') else None
@@ -747,13 +791,27 @@ class Stream(models.Model):
                         derivative.encode()
 
     def encode(self):
+        reuse = settings.CONFIG['video'].get('reuseUpload', False)
         media = self.source.media.path if self.source else self.file.data.path
-
         if not self.media:
             self.media.name = self.path(self.name())
         target = self.media.path
         info = ox.avinfo(media)
-        ok, error = extract.stream(media, target, self.name(), info, flags=self.flags)
+
+        done = False
+        if reuse and not self.source:
+            if self.file.can_stream():
+                ok, error = True, None
+                ox.makedirs(os.path.dirname(target))
+                shutil.move(self.file.data.path, target)
+                self.file.data.name = ''
+                self.file.save()
+            elif self.file.can_remux():
+                ok, error = extract.remux_stream(media, target)
+                done = True
+        if not done:
+            ok, error = extract.stream(media, target, self.name(), info, flags=self.flags)
+
         # file could have been moved while encoding
         # get current version from db and update
         self.refresh_from_db()
diff --git a/pandora/settings.py b/pandora/settings.py
index e4f46698..83dfb602 100644
--- a/pandora/settings.py
+++ b/pandora/settings.py
@@ -178,6 +178,7 @@ CACHES = {
 AUTH_PROFILE_MODULE = 'user.UserProfile'
 AUTH_CHECK_USERNAME = True
 FFMPEG = 'ffmpeg'
+FFPROBE = 'ffprobe'
 FFMPEG_SUPPORTS_VP9 = True
 FFMPEG_DEBUG = False
 

From 2889c15dc8c392f12a67e303544eb0a8fde5bd9c Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Sun, 1 Dec 2019 17:35:54 +0100
Subject: [PATCH 089/139] remove documents from index

---
 pandora/document/fulltext.py | 7 ++++---
 pandora/document/models.py   | 1 +
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/pandora/document/fulltext.py b/pandora/document/fulltext.py
index d64cd258..6779113c 100644
--- a/pandora/document/fulltext.py
+++ b/pandora/document/fulltext.py
@@ -36,12 +36,13 @@ class FulltextMixin:
             return self.data.get('text', '')
         return ''
 
-    def delete_fulltext(self):
-        res = self.elasticsearch().delete(index=self._ES_INDEX, doc_type='document', id=self.id)
-
     def has_fulltext_key():
         return bool([k for k in settings.CONFIG['documentKeys'] if k.get('fulltext')])
 
+    def delete_fulltext(self):
+        if self.has_fulltext_key():
+            res = self.elasticsearch().delete(index=self._ES_INDEX, doc_type='document', id=self.id)
+
     def update_fulltext(self):
         if self.has_fulltext_key():
             text = self.extract_fulltext()
diff --git a/pandora/document/models.py b/pandora/document/models.py
index 0c61f019..c938c260 100644
--- a/pandora/document/models.py
+++ b/pandora/document/models.py
@@ -653,6 +653,7 @@ def delete_document(sender, **kwargs):
     if t.file:
         t.delete_cache()
         t.file.delete(save=False)
+    t.delete_fulltext()
 pre_delete.connect(delete_document, sender=Document)
 
 class ItemProperties(models.Model):

From 14e197819d6a6ad7f2950afeaa4895c7a3f4e045 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Sun, 1 Dec 2019 17:45:28 +0100
Subject: [PATCH 090/139] expose findDocumentsSelect and ...Input

---
 static/js/findDocumentsElement.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/static/js/findDocumentsElement.js b/static/js/findDocumentsElement.js
index ad7efcb1..4f5dd464 100644
--- a/static/js/findDocumentsElement.js
+++ b/static/js/findDocumentsElement.js
@@ -33,7 +33,7 @@ pandora.ui.findDocumentsElement = function() {
                             }
                         }),
                 ] : [], [
-                    $findSelect = Ox.Select({
+                    pandora.$ui.findDocumentsSelect = $findSelect = Ox.Select({
                             id: 'select',
                             items: [].concat(
                                 pandora.site.documentKeys.filter(function(key) {
@@ -70,7 +70,7 @@ pandora.ui.findDocumentsElement = function() {
                                 }
                             }
                         }),
-                    $findInput = Ox.Input({
+                    pandora.$ui.findDocumentsInput = $findInput = Ox.Input({
                         autocomplete: autocompleteFunction(),
                         autocompleteSelect: true,
                         autocompleteSelectHighlight: true,

From eaad9badbb8e51938f5fc7353a38b7bc13c98140 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Mon, 2 Dec 2019 13:38:56 +0100
Subject: [PATCH 091/139] crop pdf pages

---
 pandora/document/models.py | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/pandora/document/models.py b/pandora/document/models.py
index c938c260..838168ee 100644
--- a/pandora/document/models.py
+++ b/pandora/document/models.py
@@ -522,8 +522,14 @@ class Document(models.Model, FulltextMixin):
         else:
             path = src
         if self.extension == 'pdf':
+            crop = []
             if page:
-                page = int(page)
+                if ',' in page:
+                    crop = list(map(int, page.split(',')))
+                    page = crop[0]
+                    crop = crop[1:]
+                else:
+                    page = int(page)
             if page and page > 1 and page <= self.pages:
                 src = os.path.join(folder, '1024p%d.jpg' % page)
             else:
@@ -533,6 +539,18 @@ class Document(models.Model, FulltextMixin):
                 self.extract_page(page)
             if size:
                 path = os.path.join(folder, '%dp%d.jpg' % (size, page))
+            if len(crop) == 4:
+                path = os.path.join(folder, '%dp%d,%s.jpg' % (1024, page, ','.join(map(str, crop)))
+                if not os.path.exists(path):
+                    img = Image.open(src).crop(crop)
+                    img.save(path)
+                else:
+                    img = Image.open(path)
+                src = path
+                if size < max(img.size):
+                    path = os.path.join(folder, '%dp%d,%s.jpg' % (size, page, ','.join(map(str, crop)))
+                    if not os.path.exists(path):
+                        resize_image(src, path, size=size)
         elif self.extension in ('jpg', 'png', 'gif'):
             if os.path.exists(src):
                 if size and page:

From a0b3a144c6b30122e92eb5c3ca6d3311612f86e5 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Mon, 2 Dec 2019 13:40:52 +0100
Subject: [PATCH 092/139] fix typo

---
 pandora/document/models.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pandora/document/models.py b/pandora/document/models.py
index 838168ee..0c653fa5 100644
--- a/pandora/document/models.py
+++ b/pandora/document/models.py
@@ -540,7 +540,7 @@ class Document(models.Model, FulltextMixin):
             if size:
                 path = os.path.join(folder, '%dp%d.jpg' % (size, page))
             if len(crop) == 4:
-                path = os.path.join(folder, '%dp%d,%s.jpg' % (1024, page, ','.join(map(str, crop)))
+                path = os.path.join(folder, '%dp%d,%s.jpg' % (1024, page, ','.join(map(str, crop))))
                 if not os.path.exists(path):
                     img = Image.open(src).crop(crop)
                     img.save(path)
@@ -548,7 +548,7 @@ class Document(models.Model, FulltextMixin):
                     img = Image.open(path)
                 src = path
                 if size < max(img.size):
-                    path = os.path.join(folder, '%dp%d,%s.jpg' % (size, page, ','.join(map(str, crop)))
+                    path = os.path.join(folder, '%dp%d,%s.jpg' % (size, page, ','.join(map(str, crop))))
                     if not os.path.exists(path):
                         resize_image(src, path, size=size)
         elif self.extension in ('jpg', 'png', 'gif'):

From 5efd7eadc67ba46bdd917f0c409b9b059d2149c9 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Mon, 2 Dec 2019 19:11:40 +0100
Subject: [PATCH 093/139] fix has_fulltext_key check

---
 pandora/document/fulltext.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pandora/document/fulltext.py b/pandora/document/fulltext.py
index 6779113c..e3e665cc 100644
--- a/pandora/document/fulltext.py
+++ b/pandora/document/fulltext.py
@@ -36,7 +36,7 @@ class FulltextMixin:
             return self.data.get('text', '')
         return ''
 
-    def has_fulltext_key():
+    def has_fulltext_key(self):
         return bool([k for k in settings.CONFIG['documentKeys'] if k.get('fulltext')])
 
     def delete_fulltext(self):

From 8b6607a74e039a2d1da9a2df3e2b0ead79d2a779 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Mon, 2 Dec 2019 19:16:13 +0100
Subject: [PATCH 094/139] only extract fulltext once

---
 pandora/document/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pandora/document/models.py b/pandora/document/models.py
index 0c653fa5..56f46600 100644
--- a/pandora/document/models.py
+++ b/pandora/document/models.py
@@ -280,7 +280,6 @@ class Document(models.Model, FulltextMixin):
                 self.update_sort()
                 self.update_find()
                 self.update_facets()
-                self.update_fulltext()
             new = False
         else:
             new = True
@@ -506,6 +505,7 @@ class Document(models.Model, FulltextMixin):
                     self.oshash = ox.oshash(self.file.path)
                     self.save()
                     self.delete_cache()
+                    self.update_fulltext()
                 return True, self.file.size
 
             return save_chunk(self, self.file, chunk, offset, name, done_cb)

From 565f0bb5fa96cd703e6a02b3a4c19bb53fe560ba Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Mon, 2 Dec 2019 20:09:49 +0100
Subject: [PATCH 095/139] refactor view menu, customize for documents view

---
 static/js/mainMenu.js | 391 ++++++++++++++++++++++++------------------
 1 file changed, 223 insertions(+), 168 deletions(-)

diff --git a/static/js/mainMenu.js b/static/js/mainMenu.js
index 87bfbc52..abb64847 100644
--- a/static/js/mainMenu.js
+++ b/static/js/mainMenu.js
@@ -52,170 +52,7 @@ pandora.ui.mainMenu = function() {
                     ] },
                     getListMenu(),
                     getItemMenu(),
-                    { id: 'viewMenu', title: Ox._('View'), items: [
-                        { id: 'section', title: Ox._('Section'), items: [
-                            { group: 'viewsection', min: 1, max: 1, items: Object.keys(pandora.site.sectionFolders).map(function(section) {
-                                return {
-                                    id: section,
-                                    title: section == 'items' ? Ox._(pandora.site.itemName.plural) : Ox._(Ox.toTitleCase(section)),
-                                    checked: ui.section == section
-                                };
-                            }) }
-                        ] },
-                        {},
-                        { id: 'movies', title: Ox._('View {0}', [Ox._(pandora.site.itemName.plural)]), items: [
-                            { group: 'listview', min: 1, max: 1, items: pandora.site.listViews.map(function(view) {
-                                return Ox.extend({
-                                    checked: ui.listView == view.id
-                                }, view, {
-                                    keyboard: listViewKey <= 10
-                                        ? 'shift ' + (listViewKey++%10)
-                                        : void 0,
-                                    title: Ox._(view.title)
-                                });
-                            }) },
-                        ]},
-                        { id: 'icons', title: Ox._('Icons'), items: [].concat([
-                                { group: 'viewicons', min: 1, max: 1, items: ['posters', 'frames'].map(function(icons) {
-                                    return {id: icons, title: Ox._(Ox.toTitleCase(icons)), checked: ui.icons == icons};
-                                }) },
-                                {},
-                            ], pandora.site.media.importPosters ? [
-                                { id: 'showsiteposters', title: Ox._('Always Show {0} Poster', [pandora.site.site.name]), checked: ui.showSitePosters },
-                                {}
-                            ] : [], [
-                                { id: 'showreflections', title: Ox._('Show Reflections'), checked: true, disabled: true }
-                            ]
-                        ) },
-                        { id: 'timelines', title: Ox._('Timelines'), items: [
-                            { group: 'viewtimelines', min: 1, max: 1, items: pandora.site.timelines.map(function(mode) {
-                                return {id: mode.id, title: Ox._(mode.title), checked: ui.videoTimeline == mode.id};
-                            }) }
-                        ]},
-                        { id: 'columns', title: Ox._('Columns'), items: [
-                            { id: 'loadcolumns', title: Ox._('Load Layout...'), disabled: true },
-                            { id: 'savecolumns', title: Ox._('Save Layout...'), disabled: true },
-                            {},
-                            { id: 'resetcolumns', title: Ox._('Reset Layout'), disabled: true }
-                        ] },
-                        { id: 'filters', title: Ox._('Filters'), disabled: ui.section != 'items', items: [
-                            { group: 'filters', min: 5, max: 5, items: pandora.site.filters.map(function(filter) {
-                                return Ox.extend({
-                                    checked: Ox.getIndexById(ui.filters, filter.id) > -1
-                                }, filter, {
-                                    title: Ox._(filter.title)
-                                });
-                            }) },
-                            {},
-                            { id: 'resetfilters', title: Ox._('Reset Filters') }
-                        ] },
-                        {},
-                        { id: 'item', title: [
-                                Ox._('Open {0}', [Ox._(pandora.site.itemName.singular)]),
-                                Ox._('Open {0}', [Ox._(pandora.site.itemName.plural)])
-                            ], items: [
-                            { group: 'itemview', min: 1, max: 1, items: pandora.site.itemViews.filter(function(view) {
-                                return view.id != 'data' && view.id != 'media' ||
-                                    pandora.hasCapability('canSeeExtraItemViews');
-                            }).map(function(view) {
-                                return Ox.extend({
-                                    checked: ui.itemView == view.id
-                                }, view, {
-                                    keyboard: itemViewKey <= 10
-                                        ? 'shift ' + (itemViewKey++%10)
-                                        : void 0,
-                                    title: Ox._(view.title)
-                                });
-                            }) },
-                        ] },
-                        { id: 'clips', title: Ox._('Open Clips'), items: [
-                            { group: 'videoview', min: 1, max: 1, items: ['player', 'editor', 'timeline'].map(function(view) {
-                                return {id: view, title: Ox._(Ox.toTitleCase(view)), checked: ui.videoView == view};
-                            }) }
-                        ] },
-                        { id: 'documents', title: Ox._('Open Documents'), items: [
-                            { group: 'documentview', min: 1, max: 1, items: ['info', 'view'].map(function(id) {
-                                return {
-                                    id: id,
-                                    checked: ui.documentView == id,
-                                    keyboard: documentViewKey <= 10
-                                        ? 'shift ' + (documentViewKey++%10)
-                                        : void 0,
-                                    title: Ox._(Ox.toTitleCase(id))
-                                }
-                            }) }
-                        ] },
-                        {},
-                        {
-                            id: 'showsidebar',
-                            title: Ox._((ui.showSidebar ? 'Hide' : 'Show') + ' Sidebar'),
-                            keyboard: 'shift s'
-                        },
-                        {
-                            id: 'showinfo',
-                            title: Ox._((ui.showInfo ? 'Hide' : 'Show') + ' Info'),
-                            disabled: !ui.showSidebar, keyboard: 'shift i'
-                        },
-                        {
-                            id: 'showfilters',
-                            title: Ox._((ui.showFilters ? 'Hide' : 'Show') + ' Filters'),
-                            disabled: ui.section != 'items' || !!ui.item, keyboard: 'shift f'
-                        },
-                        {
-                            id: 'showbrowser',
-                            title: Ox._((ui.showBrowser ? 'Hide': 'Show') + ' {0} Browser', [Ox._(pandora.site.itemName.singular)]),
-                            disabled: !ui.item, keyboard: 'shift b'
-                        },
-                        {
-                            id: 'showdocument',
-                            title: Ox._((ui.showDocument ? 'Hide' : 'Show') + ' Document'),
-                            disabled: !hasDocument(), keyboard: 'shift d'
-                        },
-                        {
-                            id: 'showtimeline',
-                            title: Ox._((ui.showTimeline ? 'Hide' : 'Show') + ' Timeline'),
-                            disabled: !hasTimeline(), keyboard: 'shift t'
-                        },
-                        {
-                            id: 'showannotations',
-                            title: Ox._((ui.showAnnotations ? 'Hide' : 'Show') + ' Annotations'),
-                            disabled: !hasAnnotations(), keyboard: 'shift a'
-                        },
-                        {
-                            id: 'showclips',
-                            title: Ox._((ui.showClips ? 'Hide' : 'Show') + ' Clips'),
-                            disabled: !hasClips(), keyboard: 'shift c'
-                        },
-                        {},
-                        {
-                            id: 'togglefullscreen',
-                            title: Ox._((fullscreenState ? 'Exit' : 'Enter') + ' Fullscreen'),
-                            disabled: fullscreenState === void 0,
-                            keyboard: /^Mac/.test(window.navigator.platform)
-                                ? 'shift alt f'
-                                : 'F11'
-
-                        },
-                        {
-                            id: 'entervideofullscreen',
-                            title: Ox._('Enter Video Fullscreen'),
-                            disabled: !ui.item || ui.itemView != 'player'
-                        },
-                        {},
-                        { id: 'theme', title: Ox._('Theme'), items: [
-                            { group: 'settheme', min: 1, max: 1, items: pandora.site.themes.map(function(theme) {
-                                return {id: theme, title: Ox.Theme.getThemeData(theme).themeName, checked: ui.theme == theme}
-                            }) }
-                        ] },
-                        { id: 'locale',
-                          title: Ox._('Language'), items: [
-                            { group: 'setlocale', min: 1, max: 1, items: pandora.site.languages.map(function(locale) {
-                                return {id: locale, title: Ox.LOCALE_NAMES[locale], checked: ui.locale == locale}
-                            }) }
-                        ] },
-                        {},
-                        { id: 'embed', title: Ox._('Embed...') }
-                    ]},
+                    getViewMenu(),
                     getSortMenu(),
                     getFindMenu(),
                     { id: 'dataMenu', title: Ox._('Data'), items: [
@@ -314,6 +151,9 @@ pandora.ui.mainMenu = function() {
                     pandora.UI.set({listSort: [{key: value, operator: pandora.getSortOperator(value)}]});
                 } else if (data.id == 'itemview') {
                     pandora.UI.set({itemView: value});
+                } else if (data.id == 'collectionview') {
+                    var set = {collectionView: value};
+                    pandora.UI.set(set);
                 } else if (data.id == 'listview') {
                     var set = {listView: value};
                     if (
@@ -597,10 +437,17 @@ pandora.ui.mainMenu = function() {
                     fromMenu = true;
                     pandora.history.clear();
                 } else if (data.id == 'resetfilters') {
-                    pandora.UI.set({
-                        filters: pandora.site.user.ui.filters
-                    });
-                    pandora.$ui.contentPanel.replaceElement(0, pandora.$ui.browser = pandora.ui.browser());
+                    if (ui.section == 'documents') {
+                        pandora.UI.set({
+                            documentFilters: pandora.site.user.ui.documentFilters
+                        });
+                        pandora.$ui.documentContentPanel.replaceElement(0, pandora.$ui.documentBrowser = pandora.ui.documentBrowser());
+                    } else {
+                        pandora.UI.set({
+                            filters: pandora.site.user.ui.filters
+                        });
+                        pandora.$ui.contentPanel.replaceElement(0, pandora.$ui.browser = pandora.ui.browser());
+                    }
                 } else if (data.id == 'showsidebar') {
                     pandora.UI.set({showSidebar: !ui.showSidebar});
                 } else if (data.id == 'showinfo') {
@@ -840,6 +687,9 @@ pandora.ui.mainMenu = function() {
                     );
                 }
             },
+            pandora_collectionview: function(data) {
+                that.checkItem('viewMenu_documents_' + data.value);
+            },
             pandora_listview: function(data) {
                 that.checkItem('viewMenu_movies_' + data.value);
                 if (
@@ -857,6 +707,7 @@ pandora.ui.mainMenu = function() {
             },
             pandora_section: function(data) {
                 lists = {};
+                that.replaceMenu('viewMenu', getViewMenu());
                 that.checkItem('viewMenu_section_' + data.value);
                 that.replaceMenu('listMenu', getListMenu());
                 that.replaceMenu('itemMenu', getItemMenu());
@@ -1557,6 +1408,124 @@ pandora.ui.mainMenu = function() {
         ] };
     }
 
+    function getSectionViews() {
+        if (ui.section == 'documents') {
+            console.log('now documents')
+            return [
+                { id: 'documents', title: Ox._('View Documents'), items: [
+                    { group: 'collectionview', min: 1, max: 1, items: pandora.site.listViews.filter(function(view) {
+                        return Ox.contains(['list', 'grid'], view.id)
+                    }).map(function(view) {
+                        return Ox.extend({
+                            checked: ui.collectionView == view.id
+                        }, view, {
+                            keyboard: listViewKey <= 10
+                                ? 'shift ' + (listViewKey++%10)
+                                : void 0,
+                            title: Ox._(view.title)
+                        });
+                    }) },
+                ]},
+                { id: 'filters', title: Ox._('Filters'), items: [
+                    { group: 'filters', min: 5, max: 5, items: pandora.site.documentFilters.map(function(filter) {
+                        return Ox.extend({
+                            checked: Ox.getIndexById(ui.documentFilters, filter.id) > -1
+                        }, filter, {
+                            title: Ox._(filter.title)
+                        });
+                    }) },
+                    {},
+                    { id: 'resetfilters', title: Ox._('Reset Filters') }
+                ] },
+            ]
+        } else {
+            return [
+                { id: 'movies', title: Ox._('View {0}', [Ox._(pandora.site.itemName.plural)]), items: [
+                    { group: 'listview', min: 1, max: 1, items: pandora.site.listViews.map(function(view) {
+                        return Ox.extend({
+                            checked: ui.listView == view.id
+                        }, view, {
+                            keyboard: listViewKey <= 10
+                                ? 'shift ' + (listViewKey++%10)
+                                : void 0,
+                            title: Ox._(view.title)
+                        });
+                    }) },
+                ]},
+                { id: 'icons', title: Ox._('Icons'), items: [].concat([
+                        { group: 'viewicons', min: 1, max: 1, items: ['posters', 'frames'].map(function(icons) {
+                            return {id: icons, title: Ox._(Ox.toTitleCase(icons)), checked: ui.icons == icons};
+                        }) },
+                        {},
+                    ], pandora.site.media.importPosters ? [
+                        { id: 'showsiteposters', title: Ox._('Always Show {0} Poster', [pandora.site.site.name]), checked: ui.showSitePosters },
+                        {}
+                    ] : [], [
+                        { id: 'showreflections', title: Ox._('Show Reflections'), checked: true, disabled: true }
+                    ]
+                ) },
+                { id: 'timelines', title: Ox._('Timelines'), items: [
+                    { group: 'viewtimelines', min: 1, max: 1, items: pandora.site.timelines.map(function(mode) {
+                        return {id: mode.id, title: Ox._(mode.title), checked: ui.videoTimeline == mode.id};
+                    }) }
+                ]},
+                { id: 'columns', title: Ox._('Columns'), items: [
+                    { id: 'loadcolumns', title: Ox._('Load Layout...'), disabled: true },
+                    { id: 'savecolumns', title: Ox._('Save Layout...'), disabled: true },
+                    {},
+                    { id: 'resetcolumns', title: Ox._('Reset Layout'), disabled: true }
+                ] },
+                { id: 'filters', title: Ox._('Filters'), disabled: ui.section != 'items', items: [
+                    { group: 'filters', min: 5, max: 5, items: pandora.site.filters.map(function(filter) {
+                        return Ox.extend({
+                            checked: Ox.getIndexById(ui.filters, filter.id) > -1
+                        }, filter, {
+                            title: Ox._(filter.title)
+                        });
+                    }) },
+                    {},
+                    { id: 'resetfilters', title: Ox._('Reset Filters') }
+                ] },
+                {},
+                { id: 'item', title: [
+                        Ox._('Open {0}', [Ox._(pandora.site.itemName.singular)]),
+                        Ox._('Open {0}', [Ox._(pandora.site.itemName.plural)])
+                    ], items: [
+                    { group: 'itemview', min: 1, max: 1, items: pandora.site.itemViews.filter(function(view) {
+                        return view.id != 'data' && view.id != 'media' ||
+                            pandora.hasCapability('canSeeExtraItemViews');
+                    }).map(function(view) {
+                        return Ox.extend({
+                            checked: ui.itemView == view.id
+                        }, view, {
+                            keyboard: itemViewKey <= 10
+                                ? 'shift ' + (itemViewKey++%10)
+                                : void 0,
+                            title: Ox._(view.title)
+                        });
+                    }) },
+                ] },
+                { id: 'clips', title: Ox._('Open Clips'), items: [
+                    { group: 'videoview', min: 1, max: 1, items: ['player', 'editor', 'timeline'].map(function(view) {
+                        return {id: view, title: Ox._(Ox.toTitleCase(view)), checked: ui.videoView == view};
+                    }) }
+                ] },
+                { id: 'documents', title: Ox._('Open Documents'), items: [
+                    { group: 'documentview', min: 1, max: 1, items: ['info', 'view'].map(function(id) {
+                        return {
+                            id: id,
+                            checked: ui.documentView == id,
+                            keyboard: documentViewKey <= 10
+                                ? 'shift ' + (documentViewKey++%10)
+                                : void 0,
+                            title: Ox._(Ox.toTitleCase(id))
+                        }
+                    }) }
+                ] }
+            ]
+        }
+    }
+
     function getSortMenu() {
 
         if (ui.section == 'documents') {
@@ -1636,6 +1605,92 @@ pandora.ui.mainMenu = function() {
         ] };
     }
 
+    function getViewMenu() {
+        return { id: 'viewMenu', title: Ox._('View'), items: [
+            { id: 'section', title: Ox._('Section'), items: [
+                { group: 'viewsection', min: 1, max: 1, items: Object.keys(pandora.site.sectionFolders).map(function(section) {
+                    return {
+                        id: section,
+                        title: section == 'items' ? Ox._(pandora.site.itemName.plural) : Ox._(Ox.toTitleCase(section)),
+                        checked: ui.section == section
+                    };
+                }) }
+            ] },
+            {},
+            getSectionViews(),
+            {},
+            {
+                id: 'showsidebar',
+                title: Ox._((ui.showSidebar ? 'Hide' : 'Show') + ' Sidebar'),
+                keyboard: 'shift s'
+            },
+            {
+                id: 'showinfo',
+                title: Ox._((ui.showInfo ? 'Hide' : 'Show') + ' Info'),
+                disabled: !ui.showSidebar, keyboard: 'shift i'
+            },
+            {
+                id: 'showfilters',
+                title: Ox._((ui.showFilters ? 'Hide' : 'Show') + ' Filters'),
+                disabled: ui.section != 'items' || !!ui.item, keyboard: 'shift f'
+            },
+            {
+                id: 'showbrowser',
+                title: Ox._((ui.showBrowser ? 'Hide': 'Show') + ' {0} Browser', [Ox._(pandora.site.itemName.singular)]),
+                disabled: !ui.item, keyboard: 'shift b'
+            },
+            {
+                id: 'showdocument',
+                title: Ox._((ui.showDocument ? 'Hide' : 'Show') + ' Document'),
+                disabled: !hasDocument(), keyboard: 'shift d'
+            },
+            {
+                id: 'showtimeline',
+                title: Ox._((ui.showTimeline ? 'Hide' : 'Show') + ' Timeline'),
+                disabled: !hasTimeline(), keyboard: 'shift t'
+            },
+            {
+                id: 'showannotations',
+                title: Ox._((ui.showAnnotations ? 'Hide' : 'Show') + ' Annotations'),
+                disabled: !hasAnnotations(), keyboard: 'shift a'
+            },
+            {
+                id: 'showclips',
+                title: Ox._((ui.showClips ? 'Hide' : 'Show') + ' Clips'),
+                disabled: !hasClips(), keyboard: 'shift c'
+            },
+            {},
+            {
+                id: 'togglefullscreen',
+                title: Ox._((fullscreenState ? 'Exit' : 'Enter') + ' Fullscreen'),
+                disabled: fullscreenState === void 0,
+                keyboard: /^Mac/.test(window.navigator.platform)
+                    ? 'shift alt f'
+                    : 'F11'
+
+            },
+            {
+                id: 'entervideofullscreen',
+                title: Ox._('Enter Video Fullscreen'),
+                disabled: !ui.item || ui.itemView != 'player'
+            },
+            {},
+            { id: 'theme', title: Ox._('Theme'), items: [
+                { group: 'settheme', min: 1, max: 1, items: pandora.site.themes.map(function(theme) {
+                    return {id: theme, title: Ox.Theme.getThemeData(theme).themeName, checked: ui.theme == theme}
+                }) }
+            ] },
+            { id: 'locale',
+              title: Ox._('Language'), items: [
+                { group: 'setlocale', min: 1, max: 1, items: pandora.site.languages.map(function(locale) {
+                    return {id: locale, title: Ox.LOCALE_NAMES[locale], checked: ui.locale == locale}
+                }) }
+            ] },
+            {},
+            { id: 'embed', title: Ox._('Embed...') }
+        ]}
+    }
+
     function hasAnnotations() {
         return ui.section == 'items' && ui.item && pandora.isVideoView();
     }

From 1d9e43b551c73614fb87a07dc1ea4a35bf70f1c4 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Mon, 2 Dec 2019 21:01:14 +0100
Subject: [PATCH 096/139] % is also not ok in pdf names

---
 static/js/utils.js | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/static/js/utils.js b/static/js/utils.js
index a4de127d..52cef4e8 100644
--- a/static/js/utils.js
+++ b/static/js/utils.js
@@ -2641,7 +2641,9 @@ pandora.openURL = function(url) {
 };
 
 pandora.safeDocumentName = function(name) {
-    name = name.replace('?', '_').replace('#', '_');
+    ['?', '#', '%'].forEach(function(c) {
+        name = name.replace(c, '_');
+    })
     return name;
 };
 

From ead29304bcd50128ffdece2075c27fbb54709a65 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Mon, 2 Dec 2019 21:29:14 +0100
Subject: [PATCH 097/139] document might already be gone

---
 pandora/document/fulltext.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/pandora/document/fulltext.py b/pandora/document/fulltext.py
index e3e665cc..b383e8df 100644
--- a/pandora/document/fulltext.py
+++ b/pandora/document/fulltext.py
@@ -41,7 +41,11 @@ class FulltextMixin:
 
     def delete_fulltext(self):
         if self.has_fulltext_key():
-            res = self.elasticsearch().delete(index=self._ES_INDEX, doc_type='document', id=self.id)
+            from elasticsearch.exceptions import NotFoundError
+            try:
+                res = self.elasticsearch().delete(index=self._ES_INDEX, doc_type='document', id=self.id)
+            except NotFoundError:
+                pass
 
     def update_fulltext(self):
         if self.has_fulltext_key():

From a57f68f3c2a5aa83acf3371fd62d2e5444637646 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Wed, 4 Dec 2019 13:12:27 +0000
Subject: [PATCH 098/139] remove debug

---
 static/js/mainMenu.js | 1 -
 1 file changed, 1 deletion(-)

diff --git a/static/js/mainMenu.js b/static/js/mainMenu.js
index abb64847..19a40ed9 100644
--- a/static/js/mainMenu.js
+++ b/static/js/mainMenu.js
@@ -1410,7 +1410,6 @@ pandora.ui.mainMenu = function() {
 
     function getSectionViews() {
         if (ui.section == 'documents') {
-            console.log('now documents')
             return [
                 { id: 'documents', title: Ox._('View Documents'), items: [
                     { group: 'collectionview', min: 1, max: 1, items: pandora.site.listViews.filter(function(view) {

From 955d0f1f2a6bec33b3e22fbf8281a3883322159a Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Thu, 5 Dec 2019 00:10:34 +0000
Subject: [PATCH 099/139] fix document folder reload

---
 static/js/utils.js | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/static/js/utils.js b/static/js/utils.js
index 52cef4e8..3d7e4d9e 100644
--- a/static/js/utils.js
+++ b/static/js/utils.js
@@ -2706,7 +2706,11 @@ pandora.reloadList = function() {
     Ox.Log('', 'reloadList')
     var listData = pandora.getListData();
     Ox.Request.clearCache(); // fixme: remove
-    if (pandora.$ui.filters) {
+    if (pandora.user.ui.section == 'documents' && pandora.$ui.documentFilters) {
+        pandora.$ui.documentFilters.forEach(function($filter) {
+            $filter.reloadList();
+        });
+    } else if (pandora.$ui.filters) {
         pandora.$ui.filters.forEach(function($filter) {
             $filter.reloadList();
         });

From 16b900c5419f5557ca3c2911df5352c8aada0efd Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Thu, 5 Dec 2019 00:29:11 +0000
Subject: [PATCH 100/139] fix autocomplte and filter update in documents filter
 form

---
 static/js/documentFilterForm.js | 24 ++++++++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/static/js/documentFilterForm.js b/static/js/documentFilterForm.js
index 577375c0..957bb512 100644
--- a/static/js/documentFilterForm.js
+++ b/static/js/documentFilterForm.js
@@ -27,7 +27,7 @@ pandora.ui.documentFilterForm = function(options) {
                     if (key.format && key.format.type == 'ColorPercent') {
                         key.format.type = 'percent';
                     }
-                    Ox.print(key);
+                    key.autocomplete = autocompleteFunction(key)
                     return key;
                 }).concat([{
                     id: 'collection',
@@ -69,6 +69,24 @@ pandora.ui.documentFilterForm = function(options) {
         that.getList = that.$filter.getList;
         that.value = that.$filter.value;
     });
+    function autocompleteFunction(key) {
+        return key.autocomplete ? function(value, callback) {
+            pandora.api.autocomplete({
+                key: key.id,
+                query: {
+                    conditions: [],
+                    operator: '&'
+                },
+                range: [0, 100],
+                sort: key.autocompleteSort,
+                value: value
+            }, function(result) {
+                callback(result.data.items.map(function(item) {
+                    return Ox.decodeHTMLEntities(item);
+                }));
+            });
+        } : null;
+    }
     that.updateResults = function() {
         if (mode == 'collection') {
             Ox.Request.clearCache(collection.id);
@@ -81,11 +99,9 @@ pandora.ui.documentFilterForm = function(options) {
                     }
                 })
                 .reloadList();
-            /*
-            pandora.$ui.filters && pandora.$ui.filters.forEach(function($filter) {
+            pandora.$ui.documentFilters && pandora.$ui.documentFilters.forEach(function($filter) {
                 $filter.reloadList();
             });
-            */
         } else {
             pandora.UI.set({findDocuments: Ox.clone(that.$filter.options('value'), true)});
             pandora.$ui.findElement.updateElement();

From 05dcc997b652d918e3512a895b3bad2d208cf765 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Thu, 5 Dec 2019 12:18:57 +0000
Subject: [PATCH 101/139] don't fail on duplicate documents, only upload new
 documents if possible

---
 static/js/uploadDocumentDialog.js | 95 ++++++++++++++++++-------------
 1 file changed, 57 insertions(+), 38 deletions(-)

diff --git a/static/js/uploadDocumentDialog.js b/static/js/uploadDocumentDialog.js
index b4da9c92..4672e0af 100644
--- a/static/js/uploadDocumentDialog.js
+++ b/static/js/uploadDocumentDialog.js
@@ -5,6 +5,8 @@ pandora.ui.uploadDocumentDialog = function(options, callback) {
         extensions = files.map(function(file) {
             return file.name.split('.').pop().toLowerCase()
         }),
+        existingFiles = [],
+        uploadFiles = [],
 
         supportedExtensions = ['gif', 'jpg', 'jpeg', 'pdf', 'png'],
 
@@ -56,9 +58,9 @@ pandora.ui.uploadDocumentDialog = function(options, callback) {
                 height: 112,
                 keys: {escape: 'close'},
                 width: 288,
-                title: files.length == 1
+                title: uploadFiles.length == 1
                     ? Ox._('Upload Document')
-                    : Ox._('Upload {0} Documents', [files.length])
+                    : Ox._('Upload {0} Documents', [uploadFiles.length])
             })
             .bindEvent({
                 open: function() {
@@ -73,46 +75,63 @@ pandora.ui.uploadDocumentDialog = function(options, callback) {
             Ox._('Supported file types are GIF, JPG, PNG and PDF.')
         );
     } else {
-        var valid = true;
+        var oshashes = [];
         Ox.parallelForEach(files, function(file, index, array, callback) {
             var extension = file.name.split('.').pop().toLowerCase(),
                 filename = file.name.split('.').slice(0, -1).join('.') + '.'
                     + (extension == 'jpeg' ? 'jpg' : extension);
-            valid && Ox.oshash(file, function(oshash) {
-                pandora.api.findDocuments({
-                    keys: ['id', 'user', 'title', 'extension'],
-                    query: {
-                        conditions: [{
-                            key: 'oshash',
-                            operator: '==',
-                            value: oshash
-                        }],
-                        operator: '&'
-                    },
-                    range: [0, 1],
-                    sort: [{key: 'title', operator: '+'}]
-                }, function(result) {
-                    if (result.data.items.length) {
-                        var id = result.data.items[0].title + '.'
-                            + result.data.items[0].extension;
-                        valid && errorDialog(
-                            filename == id
-                            ? Ox._(
-                                'The file "{0}" already exists.',
-                                [filename]
-                            )
-                            : Ox._(
-                                'The file "{0}" already exists as "{1}".',
-                                [filename, id]
-                            )
-                        ).open();
-                        valid = false;
-                    }
+            Ox.oshash(file, function(oshash) {
+                // skip duplicate files
+                if (Ox.contains(oshashes, oshash)) {
                     callback();
-                })
+                } else {
+                    oshashes.push(oshash)
+                    pandora.api.findDocuments({
+                        keys: ['id', 'user', 'title', 'extension'],
+                        query: {
+                            conditions: [{
+                                key: 'oshash',
+                                operator: '==',
+                                value: oshash
+                            }],
+                            operator: '&'
+                        },
+                        range: [0, 1],
+                        sort: [{key: 'title', operator: '+'}]
+                    }, function(result) {
+                        if (result.data.items.length) {
+                            var id = result.data.items[0].title + '.'
+                                + result.data.items[0].extension;
+                            existingFiles.push({
+                                id: id,
+                                filename: filename
+
+                            })
+                        } else {
+                            uploadFiles.push(file)
+                        }
+                        callback();
+                    })
+                }
             });
         } ,function() {
-            valid && $uploadDialog.open();
+            if (uploadFiles.length) {
+                $uploadDialog.open();
+            } else if (existingFiles.length) {
+                var filename = existingFiles[0].filename
+                var id = existingFiles[0].id
+                errorDialog(
+                    filename == id
+                    ? Ox._(
+                        'The file "{0}" already exists.',
+                        [filename]
+                    )
+                    : Ox._(
+                        'The file "{0}" already exists as "{1}".',
+                        [filename, id]
+                    )
+                ).open();
+            }
         });
         return {open: Ox.noop};
     }
@@ -138,7 +157,7 @@ pandora.ui.uploadDocumentDialog = function(options, callback) {
     function uploadFile(part) {
         var data = {
             },
-            file = files[part],
+            file = uploadFiles[part],
             extension = file.name.split('.').pop().toLowerCase(),
             filename = file.name.split('.').slice(0, -1).join('.') + '.'
                 + (extension == 'jpeg' ? 'jpg' : extension);
@@ -158,7 +177,7 @@ pandora.ui.uploadDocumentDialog = function(options, callback) {
                     if (data.progress == 1) {
                         part++;
                         ids.push(data.response.id);
-                        if (part == files.length) {
+                        if (part == uploadFiles.length) {
                             $progress.options({progress: data.progress});
                             callback && callback({ids: ids});
                             $uploadDialog.close();
@@ -174,7 +193,7 @@ pandora.ui.uploadDocumentDialog = function(options, callback) {
                 },
                 progress: function(data) {
                     var progress = data.progress || 0;
-                    progress = part / files.length + 1 / files.length * progress;
+                    progress = part / uploadFiles.length + 1 / uploadFiles.length * progress;
                     $progress.options({progress: progress});
                 }
             });

From 20e43ff0c7036b894835bba61f5b81a6682aeb9e Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Thu, 5 Dec 2019 15:52:54 +0000
Subject: [PATCH 102/139] add toggleIconSize to documents view

---
 static/js/documentInfoView.js | 32 +++++++++++++++++++++++++++++++-
 1 file changed, 31 insertions(+), 1 deletion(-)

diff --git a/static/js/documentInfoView.js b/static/js/documentInfoView.js
index c9d65fd2..4e40a84e 100644
--- a/static/js/documentInfoView.js
+++ b/static/js/documentInfoView.js
@@ -133,7 +133,7 @@ pandora.ui.documentInfoView = function(data, isMixed) {
                     height: iconHeight + 'px'
                 })
                 .bindEvent({
-                    // singleclick: toggleIconSize
+                    singleclick: toggleIconSize
                 })
                 .appendTo($info),
 
@@ -636,6 +636,36 @@ pandora.ui.documentInfoView = function(data, isMixed) {
         //renderCapabilities(data.rightslevel);
     }
 
+    function toggleIconSize() {
+        iconSize = iconSize == 256 ? 512 : 256;
+        iconWidth = iconRatio > 1 ? iconSize : Math.round(iconSize * iconRatio);
+        iconHeight = iconRatio < 1 ? iconSize : Math.round(iconSize / iconRatio);
+        iconLeft = iconSize == 256 ? Math.floor((iconSize - iconWidth) / 2) : 0,
+        $icon.animate({
+            left: margin + iconLeft + 'px',
+            width: iconWidth + 'px',
+            height: iconHeight + 'px',
+        }, 250);
+        $reflection.animate({
+            top: margin + iconHeight + 'px',
+            width: iconSize + 'px',
+            height: iconSize / 2 + 'px'
+        }, 250);
+        $reflectionIcon.animate({
+            left: iconLeft + 'px',
+            width: iconWidth + 'px',
+            height: iconHeight + 'px',
+        }, 250);
+        $reflectionGradient.animate({
+            width: iconSize + 'px',
+            height: iconSize / 2 + 'px'
+        }, 250);
+        $text.animate({
+            left: margin + (iconSize == 256 ? 256 : iconWidth) + margin + 'px'
+        }, 250);
+        pandora.UI.set({infoIconSize: iconSize});
+    }
+
     that.reload = function() {
         /*
         var src = '/documents/' + data.id + '/512p.jpg?' + data.modified;

From 484e052627d192d8a6257353ef96e5ffc7ab68dc Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Thu, 5 Dec 2019 16:27:15 +0000
Subject: [PATCH 103/139] resize filters

---
 static/js/utils.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/static/js/utils.js b/static/js/utils.js
index 3d7e4d9e..6cfefc2d 100644
--- a/static/js/utils.js
+++ b/static/js/utils.js
@@ -3026,6 +3026,7 @@ pandora.resizeWindow = function() {
             }
         }
     } else if (pandora.user.ui.section == 'documents') {
+        pandora.resizeFilters(pandora.$ui.documentPanel.width());
         if (pandora.user.ui.document) {
             pandora.$ui.document && pandora.$ui.document.update();
         } else {

From e260950c88c0bdeb260bf55a2811d006352d20ae Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Thu, 5 Dec 2019 16:41:03 +0000
Subject: [PATCH 104/139] add format option to map

---
 pandora/document/models.py | 2 ++
 pandora/item/models.py     | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/pandora/document/models.py b/pandora/document/models.py
index 56f46600..6f6466f0 100644
--- a/pandora/document/models.py
+++ b/pandora/document/models.py
@@ -412,6 +412,8 @@ class Document(models.Model, FulltextMixin):
                 and document_key['value'].get('type') == 'map' \
                 and self.get_value(document_key['value']['key']):
             value = re.compile(document_key['value']['map']).findall(self.get_value(document_key['value']['key']))
+            if value and document_key['value'].get('format'):
+                value = [document_key['value']['format'].format(value[0])]
             return value[0] if value else default
         elif key == 'user':
             return self.user.username
diff --git a/pandora/item/models.py b/pandora/item/models.py
index ba98cb1e..695558c3 100644
--- a/pandora/item/models.py
+++ b/pandora/item/models.py
@@ -215,6 +215,8 @@ class Item(models.Model):
                 and item_key['value'].get('type') == 'map' \
                 and self.get(item_key['value']['key']):
             value = re.compile(item_key['value']['map']).findall(self.get(item_key['value']['key']))
+            if value and item_key['value'].get('format'):
+                value = [item_key['value']['format'].format(value[0])]
             return value[0] if value else default
         return default
 

From 24a6bb7c03fc65a7588e74782c741e0f484cd368 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Fri, 6 Dec 2019 13:07:53 +0000
Subject: [PATCH 105/139] ignore non utf-8 output

---
 pandora/document/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pandora/document/utils.py b/pandora/document/utils.py
index c579f0dc..233276b5 100644
--- a/pandora/document/utils.py
+++ b/pandora/document/utils.py
@@ -15,7 +15,7 @@ def pdfinfo(pdf):
     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
     stdout, stderr = p.communicate()
     data = {}
-    for line in stdout.decode('utf-8').strip().split('\n'):
+    for line in stdout.decode('utf-8', 'replace').strip().split('\n'):
         parts = line.split(':')
         key = parts[0].lower().strip()
         if key:

From 211bde468d48251360778fb90aac5068a110a982 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Fri, 6 Dec 2019 13:13:09 +0000
Subject: [PATCH 106/139] add delete from collection, delete from archive
 option to menu

---
 static/js/mainMenu.js | 52 ++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 46 insertions(+), 6 deletions(-)

diff --git a/static/js/mainMenu.js b/static/js/mainMenu.js
index 19a40ed9..b53bec0b 100644
--- a/static/js/mainMenu.js
+++ b/static/js/mainMenu.js
@@ -402,6 +402,47 @@ pandora.ui.mainMenu = function() {
                             });
                         }
                     } else if (ui.section == 'documents') {
+                        var files, ids = [];
+                        if (ui.document) {
+                            files = [pandora.$ui.document.info()];
+                            ids = [files[0].id];
+                        } else {
+                            files = pandora.$ui.list.options('selected').map(function(id) {
+                                ids.push(id)
+                                return pandora.$ui.list.value(id);
+                            });
+                        }
+                        if (ui._collection) {
+                            //fixme use history
+                            //pandora.doHistory('delete', files, ui._collection, function() {
+                            pandora.api.removeCollectionItems({
+                                collection: ui._collection,
+                                items: ids
+                            }, function() {
+                                pandora.UI.set({collectionSelection: []});
+                                pandora.reloadList();
+                            });
+                        } else {
+                            pandora.ui.deleteDocumentDialog(
+                                files,
+                                function() {
+                                    Ox.Request.clearCache();
+                                    if (ui.document) {
+                                        pandora.UI.set({document: ''});
+                                    } else {
+                                        pandora.$ui.list.reloadList()
+                                    }
+                                }
+                            ).open();
+                        }
+                    } else if (ui.section == 'edits') {
+                        var clips = pandora.$ui.editPanel.getSelectedClips();
+                        pandora.doHistory('delete', clips, ui.edit, function(result) {
+                            pandora.$ui.editPanel.updatePanel(function() {});
+                        });
+                    }
+                } else if (data.id == 'deletefromarchive') {
+                    if (ui.section == 'documents') {
                         var files;
                         if (ui.document) {
                             files = [pandora.$ui.document.info()];
@@ -421,11 +462,6 @@ pandora.ui.mainMenu = function() {
                                 }
                             }
                         ).open();
-                    } else if (ui.section == 'edits') {
-                        var clips = pandora.$ui.editPanel.getSelectedClips();
-                        pandora.doHistory('delete', clips, ui.edit, function(result) {
-                            pandora.$ui.editPanel.updatePanel(function() {});
-                        });
                     }
                 } else if (data.id == 'undo') {
                     fromMenu = true;
@@ -1045,7 +1081,11 @@ pandora.ui.mainMenu = function() {
             { id: 'paste', title: clipboardItems == 0 ? Ox._('Paste') : Ox._('Paste {0}', [clipboardItemName]), disabled: !canPaste, keyboard: 'control v' },
             { id: 'clearclipboard', title: Ox._('Clear Clipboard'), disabled: !clipboardItems},
             {},
-            { id: 'delete', title: Ox._('{0} {1} {2}', [deleteVerb, selectionItemName, listName]), disabled: !canDelete, keyboard: 'delete' },
+            [
+                { id: 'delete', title: Ox._('{0} {1} {2}', [deleteVerb, selectionItemName, listName]), disabled: !canDelete, keyboard: 'delete' }
+            ].concat(ui._collection ? [
+                { id: 'deletefromarchive', title: Ox._('{0} {1} {2}', [Ox._('Delete'), selectionItemName, Ox._('from Archive')]), disabled: !canDelete }
+            ] : []),
             {},
             { id: 'undo', title: undoText ? Ox._('Undo {0}', [undoText]) : Ox._('Undo'), disabled: !undoText, keyboard: 'control z' },
             { id: 'redo', title: redoText ? Ox._('Redo {0}', [redoText]) : Ox._('Redo'), disabled: !redoText, keyboard: 'shift control z' },

From 390a5bc9aac45ea890022c617605cb0c0a935820 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Sun, 8 Dec 2019 15:46:08 +0000
Subject: [PATCH 107/139] typo

---
 pandora/app/config.py     | 2 +-
 pandora/document/views.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pandora/app/config.py b/pandora/app/config.py
index bbe15994..b20686c3 100644
--- a/pandora/app/config.py
+++ b/pandora/app/config.py
@@ -96,7 +96,7 @@ def load_config(init=False):
         for key in config['documentKeys']:
             if key['id'] in default_filters and not key.get('filter'):
                 key['filter'] = True
-                sys.stderr.write('enabled filter for documeny key "%s" since its used as default filter.\n' % (key['id']))
+                sys.stderr.write('enabled filter for document key "%s" since its used as default filter.\n' % (key['id']))
 
         config['keys'] = {}
         for key in config['itemKeys']:
diff --git a/pandora/document/views.py b/pandora/document/views.py
index 083090ee..2cc080e6 100644
--- a/pandora/document/views.py
+++ b/pandora/document/views.py
@@ -439,7 +439,7 @@ def upload(request):
 
 def autocompleteDocuments(request, data):
     '''
-    Returns autocomplete strings for a given documeny key and search string
+    Returns autocomplete strings for a given document key and search string
     takes {
         key: string, // document key
         value: string, // search string

From 234a5b67b979bd65fe00f34dcd8622941fb0de8f Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Sun, 8 Dec 2019 16:00:15 +0000
Subject: [PATCH 108/139] extract fulltext in encoding queue

---
 pandora/document/models.py | 3 ++-
 pandora/document/tasks.py  | 8 ++++++++
 2 files changed, 10 insertions(+), 1 deletion(-)
 create mode 100644 pandora/document/tasks.py

diff --git a/pandora/document/models.py b/pandora/document/models.py
index 6f6466f0..4d85f785 100644
--- a/pandora/document/models.py
+++ b/pandora/document/models.py
@@ -30,6 +30,7 @@ from user.models import Group
 
 from . import managers
 from . import utils
+from . import tasks
 from .fulltext import FulltextMixin
 
 User = get_user_model()
@@ -507,7 +508,7 @@ class Document(models.Model, FulltextMixin):
                     self.oshash = ox.oshash(self.file.path)
                     self.save()
                     self.delete_cache()
-                    self.update_fulltext()
+                    tasks.extract_fulltext.delay(self.id)
                 return True, self.file.size
 
             return save_chunk(self, self.file, chunk, offset, name, done_cb)
diff --git a/pandora/document/tasks.py b/pandora/document/tasks.py
new file mode 100644
index 00000000..f08983bb
--- /dev/null
+++ b/pandora/document/tasks.py
@@ -0,0 +1,8 @@
+# -*- coding: utf-8 -*-
+from celery.task import task
+
+@task(queue="encoding")
+def extract_fulltext(id):
+    from . import models
+    d = models.Document.objects.get(id=id)
+    d.update_fulltext()

From 2093922208d06e90c4bd9af89b93834fd245d917 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Sun, 8 Dec 2019 21:19:23 +0000
Subject: [PATCH 109/139] year might be none or int

---
 pandora/item/models.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/pandora/item/models.py b/pandora/item/models.py
index 695558c3..b9dd78ee 100644
--- a/pandora/item/models.py
+++ b/pandora/item/models.py
@@ -1049,7 +1049,9 @@ class Item(models.Model):
                         value = len(value)
                     set_value(s, name, value)
                 elif sort_type == 'year':
-                    value = self.get(source)[:4]
+                    value = self.get(source)
+                    if isinstance(value, str):
+                        value = value[:4]
                     set_value(s, name, value)
                 elif sort_type == 'date':
                     value = value_ = self.get(source)

From 5168ed4a33b38d5fc1441a02831d870ca8e37ade Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Mon, 9 Dec 2019 13:12:44 +0000
Subject: [PATCH 110/139] define sections in one location

---
 static/js/mainMenu.js       | 10 ++++------
 static/js/pandora.js        |  5 +++++
 static/js/sectionButtons.js |  6 +-----
 static/js/sectionSelect.js  |  6 +-----
 4 files changed, 11 insertions(+), 16 deletions(-)

diff --git a/static/js/mainMenu.js b/static/js/mainMenu.js
index b53bec0b..58282ecf 100644
--- a/static/js/mainMenu.js
+++ b/static/js/mainMenu.js
@@ -1647,12 +1647,10 @@ pandora.ui.mainMenu = function() {
     function getViewMenu() {
         return { id: 'viewMenu', title: Ox._('View'), items: [
             { id: 'section', title: Ox._('Section'), items: [
-                { group: 'viewsection', min: 1, max: 1, items: Object.keys(pandora.site.sectionFolders).map(function(section) {
-                    return {
-                        id: section,
-                        title: section == 'items' ? Ox._(pandora.site.itemName.plural) : Ox._(Ox.toTitleCase(section)),
-                        checked: ui.section == section
-                    };
+                { group: 'viewsection', min: 1, max: 1, items: pandora.site.sections.map(function(section) {
+                    section = Ox.extend({}, section)
+                    section.checked = section.id == ui.section;
+                    return section;
                 }) }
             ] },
             {},
diff --git a/static/js/pandora.js b/static/js/pandora.js
index 0c0f76ba..92f34709 100644
--- a/static/js/pandora.js
+++ b/static/js/pandora.js
@@ -395,6 +395,11 @@ appPanel
             }) ? 'manual' : data.site.layers.some(function(layer) {
                 return layer.hasPlaces;
             }) ? 'auto' : 'none',
+            sections: [
+                {id: 'items', title: Ox._(pandora.site.itemName.plural)},
+                {id: 'edits', title: Ox._('Edits')},
+                {id: 'documents', title: Ox._('Documents')}
+            ],
             sectionFolders: {
                 items: [
                     {id: 'personal', title: 'Personal Lists'},
diff --git a/static/js/sectionButtons.js b/static/js/sectionButtons.js
index cb0f6093..79d1a1fa 100644
--- a/static/js/sectionButtons.js
+++ b/static/js/sectionButtons.js
@@ -2,11 +2,7 @@
 
 pandora.ui.sectionButtons = function(section) {
     var that = Ox.ButtonGroup({
-            buttons: [
-                {id: 'items', title: Ox._(pandora.site.itemName.plural)},
-                {id: 'edits', title: Ox._('Edits')},
-                {id: 'documents', title: Ox._('Documents')}
-            ],
+            buttons: pandora.site.sections,
             id: 'sectionButtons',
             selectable: true,
             value: section || pandora.user.ui.section
diff --git a/static/js/sectionSelect.js b/static/js/sectionSelect.js
index af05755f..1abdf825 100644
--- a/static/js/sectionSelect.js
+++ b/static/js/sectionSelect.js
@@ -4,11 +4,7 @@ pandora.ui.sectionSelect = function(section) {
     // fixme: duplicated
     var that = Ox.Select({
             id: 'sectionSelect',
-            items: [
-                {id: 'items', title: Ox._(pandora.site.itemName.plural)},
-                {id: 'edits', title: Ox._('Edits')},
-                {id: 'documents', title: Ox._('Documents')}
-            ],
+            items: pandora.site.sections,
             value: section || pandora.user.ui.section
         }).css({
             float: 'left',

From 30ceb79f464bc91567bbb949dd4d9c6e48d712ba Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Tue, 17 Dec 2019 19:37:44 +0200
Subject: [PATCH 111/139] no torrents by default

---
 pandora/config.pandora.jsonc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pandora/config.pandora.jsonc b/pandora/config.pandora.jsonc
index c418566f..bd76faa2 100644
--- a/pandora/config.pandora.jsonc
+++ b/pandora/config.pandora.jsonc
@@ -1274,6 +1274,6 @@ examples (config.SITENAME.jsonc) that are part of this pan.do/ra distribution.
         "formats": ["webm", "mp4"],
         "previewRatio": 1.3333333333,
         "resolutions": [240, 480],
-        "torrent": true
+        "torrent": false
     }
 }

From 4d2dff3afc5cb7d1fdb3b607fa873cff1cfc4bc2 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Wed, 18 Dec 2019 16:44:09 +0200
Subject: [PATCH 112/139] fix not in smart list queries

---
 pandora/item/managers.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/pandora/item/managers.py b/pandora/item/managers.py
index cd185441..be409e5f 100644
--- a/pandora/item/managers.py
+++ b/pandora/item/managers.py
@@ -165,7 +165,10 @@ def parseCondition(condition, user, owner=None):
                     else:
                         q = Q(id__in=l.items.all())
                     if exclude:
-                        q = ~q
+                        if isinstance(q, list):
+                            q = [~x for x in q]
+                        else:
+                            q = ~q
             else:
                 q = Q(id=0)
         return q

From 24211e47db8535d056d9421eefe63419954e4aa2 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Thu, 19 Dec 2019 14:22:14 +0200
Subject: [PATCH 113/139] split sitemap, fixes #3231

---
 pandora/item/tasks.py | 72 ++++++++++++++++++++++++++++++++++---------
 pandora/item/views.py |  9 ++++++
 pandora/urls.py       |  1 +
 3 files changed, 68 insertions(+), 14 deletions(-)

diff --git a/pandora/item/tasks.py b/pandora/item/tasks.py
index d43db126..eca80083 100644
--- a/pandora/item/tasks.py
+++ b/pandora/item/tasks.py
@@ -161,13 +161,47 @@ def update_sitemap(base_url):
     def absolute_url(url):
         return base_url + url
 
-    urlset = ET.Element('urlset')
-    urlset.attrib['xmlns'] = "http://www.sitemaps.org/schemas/sitemap/0.9"
-    urlset.attrib['xmlns:xsi'] = "http://www.w3.org/2001/XMLSchema-instance"
-    urlset.attrib['xsi:schemaLocation'] = "http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd"
-    urlset.attrib['xmlns:video'] = "http://www.google.com/schemas/sitemap-video/1.1"
+    state = {}
+    state['part'] = 1
+    state['count'] = 0
 
-    url = ET.SubElement(urlset, "url")
+    def new_urlset():
+        urlset = ET.Element('urlset')
+        urlset.attrib['xmlns'] = "http://www.sitemaps.org/schemas/sitemap/0.9"
+        urlset.attrib['xmlns:xsi'] = "http://www.w3.org/2001/XMLSchema-instance"
+        urlset.attrib['xsi:schemaLocation'] = "http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd"
+        urlset.attrib['xmlns:video'] = "http://www.google.com/schemas/sitemap-video/1.1"
+        return urlset
+
+    def save_urlset():
+        s = ET.SubElement(sitemap_index, "sitemap")
+        loc = ET.SubElement(s, "loc")
+        loc.text = absolute_url("sitemap%06d.xml" % state['part'])
+        lastmod = ET.SubElement(s, "lastmod")
+        lastmod.text = datetime.now().strftime("%Y-%m-%d")
+        data = b'<?xml version="1.0" encoding="UTF-8"?>\n' + ET.tostring(state['urlset'])
+        path = os.path.abspath(os.path.join(settings.MEDIA_ROOT, 'sitemap%06d.xml.gz' % state['part']))
+        with open(path[:-3], 'wb') as f:
+            f.write(data)
+        with gzip.open(path, 'wb') as f:
+            f.write(data)
+        state['part'] += 1
+        state['count'] = 0
+        state['urlset'] = new_urlset()
+
+    def tick():
+        state['count'] += 1
+        if state['count'] > 40000:
+            save_urlset()
+
+    sitemap_index = ET.Element('sitemapindex')
+    sitemap_index.attrib['xmlns'] = "http://www.sitemaps.org/schemas/sitemap/0.9"
+    sitemap_index.attrib['xmlns:xsi'] = "http://www.w3.org/2001/XMLSchema-instance"
+    sitemap_index.attrib['xsi:schemaLocation'] = "http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd"
+
+    state['urlset'] = new_urlset()
+
+    url = ET.SubElement(state['urlset'], "url")
     loc = ET.SubElement(url, "loc")
     loc.text = absolute_url('') 
     # always, hourly, daily, weekly, monthly, yearly, never
@@ -179,9 +213,10 @@ def update_sitemap(base_url):
     # priority of page on site values 0.1 - 1.0
     priority = ET.SubElement(url, "priority")
     priority.text = '1.0'
+    tick()
 
     for page in [s['id'] for s in settings.CONFIG['sitePages']]:
-        url = ET.SubElement(urlset, "url")
+        url = ET.SubElement(state['urlset'], "url")
         loc = ET.SubElement(url, "loc")
         loc.text = absolute_url(page)
         # always, hourly, daily, weekly, monthly, yearly, never
@@ -190,11 +225,12 @@ def update_sitemap(base_url):
         # priority of page on site values 0.1 - 1.0
         priority = ET.SubElement(url, "priority")
         priority.text = '1.0'
+        tick()
 
     allowed_level = settings.CONFIG['capabilities']['canSeeItem']['guest']
     can_play = settings.CONFIG['capabilities']['canPlayVideo']['guest']
     for i in models.Item.objects.filter(level__lte=allowed_level):
-        url = ET.SubElement(urlset, "url")
+        url = ET.SubElement(state['urlset'], "url")
         # URL of the page. This URL must begin with the protocol (such as http)
         loc = ET.SubElement(url, "loc")
         loc.text = absolute_url("%s/info" % i.public_id)
@@ -230,11 +266,12 @@ def update_sitemap(base_url):
                 el.text = "%s" % int(duration)
             el = ET.SubElement(video, "video:live")
             el.text = "no"
+        tick()
 
     # Featured Lists
     from itemlist.models import List
     for l in List.objects.filter(Q(status='featured') | Q(status='public')):
-        url = ET.SubElement(urlset, "url")
+        url = ET.SubElement(state['urlset'], "url")
         # URL of the page. This URL must begin with the protocol (such as http)
         loc = ET.SubElement(url, "loc")
         loc.text = absolute_url("list==%s" % quote(l.get_id()))
@@ -248,10 +285,12 @@ def update_sitemap(base_url):
         # priority of page on site values 0.1 - 1.0
         priority = ET.SubElement(url, "priority")
         priority.text = '1.0' if l.status == 'featured' else '0.75'
+        tick()
+
     # Featured Edits
     from edit.models import Edit
     for l in Edit.objects.filter(Q(status='featured') | Q(status='public')):
-        url = ET.SubElement(urlset, "url")
+        url = ET.SubElement(state['urlset'], "url")
         # URL of the page. This URL must begin with the protocol (such as http)
         loc = ET.SubElement(url, "loc")
         loc.text = absolute_url(l.get_absolute_url()[1:])
@@ -265,10 +304,12 @@ def update_sitemap(base_url):
         # priority of page on site values 0.1 - 1.0
         priority = ET.SubElement(url, "priority")
         priority.text = '1.0' if l.status == 'featured' else '0.75'
+        tick()
+
     # Featured Collections
     from documentcollection.models import Collection
     for l in Collection.objects.filter(Q(status='featured') | Q(status='public')):
-        url = ET.SubElement(urlset, "url")
+        url = ET.SubElement(state['urlset'], "url")
         # URL of the page. This URL must begin with the protocol (such as http)
         loc = ET.SubElement(url, "loc")
         loc.text = absolute_url("documents/collection==%s" % quote(l.get_id()))
@@ -282,10 +323,11 @@ def update_sitemap(base_url):
         # priority of page on site values 0.1 - 1.0
         priority = ET.SubElement(url, "priority")
         priority.text = '1.0' if l.status == 'featured' else '0.75'
+        tick()
 
     from document.models import Document
     for d in Document.objects.filter(rightslevel=0).filter(Q(extension='html') | Q(extension='pdf')):
-        url = ET.SubElement(urlset, "url")
+        url = ET.SubElement(state['urlset'], "url")
         # URL of the page. This URL must begin with the protocol (such as http)
         loc = ET.SubElement(url, "loc")
         loc.text = absolute_url(d.get_id())
@@ -301,8 +343,10 @@ def update_sitemap(base_url):
         priority.text = '0.75'
         if d.collections.filter(Q(status='featured') | Q(status='public')).count():
             priority.text = '1.0'
-
-    data = b'<?xml version="1.0" encoding="UTF-8"?>\n' + ET.tostring(urlset)
+        tick()
+    if state['count']:
+        save_urlset()
+    data = b'<?xml version="1.0" encoding="UTF-8"?>\n' + ET.tostring(sitemap_index)
     with open(sitemap[:-3], 'wb') as f:
         f.write(data)
     with gzip.open(sitemap, 'wb') as f:
diff --git a/pandora/item/views.py b/pandora/item/views.py
index f5539b61..97f8aa23 100644
--- a/pandora/item/views.py
+++ b/pandora/item/views.py
@@ -1375,6 +1375,15 @@ def sitemap_xml(request):
     response['Content-Type'] = 'application/xml'
     return response
 
+def sitemap_part_xml(request, part):
+    part = int(part)
+    sitemap = os.path.abspath(os.path.join(settings.MEDIA_ROOT, 'sitemap%06d.xml' % part))
+    if not os.path.exists(sitemap):
+        raise Http404
+    response = HttpFileResponse(sitemap)
+    response['Content-Type'] = 'application/xml'
+    return response
+
 def item_json(request, id):
     level = settings.CONFIG['capabilities']['canSeeItem']['guest']
     if not request.user.is_anonymous():
diff --git a/pandora/urls.py b/pandora/urls.py
index 169b3d47..12328f5d 100644
--- a/pandora/urls.py
+++ b/pandora/urls.py
@@ -64,6 +64,7 @@ urlpatterns = [
     url(r'^atom.xml$', item.views.atom_xml),
     url(r'^robots.txt$', app.views.robots_txt),
     url(r'^sitemap.xml$', item.views.sitemap_xml),
+    url(r'^sitemap(?P<part>\d+).xml$', item.views.sitemap_part_xml),
     url(r'', include(item.urls)),
 ]
 #sould this not be enabled by default? nginx should handle those

From 3e46dc78cb53134cbfd215333c41becc0f025426 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Thu, 19 Dec 2019 20:54:37 +0200
Subject: [PATCH 114/139] update.py should only be used as pandora user

---
 update.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/update.py b/update.py
index 0e6c04de..6b32fc67 100755
--- a/update.py
+++ b/update.py
@@ -127,6 +127,9 @@ def get_branch(path=None):
 
 
 if __name__ == "__main__":
+    if os.stat(__file__).st_uid != os.getuid() or os.getuid() == 0:
+        print('you must run update.py as the pandora user')
+        sys.exit(1)
     base = os.path.normpath(os.path.abspath(os.path.dirname(__file__)))
     os.chdir(base)
     activate_venv(base)

From 246413027c818bb453603c8c03740e191a122751 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Thu, 19 Dec 2019 21:14:46 +0200
Subject: [PATCH 115/139] fix items without sort

---
 pandora/item/management/commands/sync_itemsort.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/pandora/item/management/commands/sync_itemsort.py b/pandora/item/management/commands/sync_itemsort.py
index d70889ac..8af5e14a 100644
--- a/pandora/item/management/commands/sync_itemsort.py
+++ b/pandora/item/management/commands/sync_itemsort.py
@@ -107,6 +107,8 @@ class Command(BaseCommand):
                     print(sql)
                 cursor.execute(sql)
             transaction.commit()
+            for i in models.Item.objects.filter(sort=None):
+                i.save()
             if rebuild:
                 print("Updating sort values...")
                 ids = [i['id'] for i in models.Item.objects.all().values('id')]
@@ -115,3 +117,5 @@ class Command(BaseCommand):
                     if options['debug']:
                         print(i)
                     i.update_sort()
+        for i in models.Item.objects.filter(sort=None):
+            i.save()

From 2b1e33df489c87c015719d20406c309814045b9f Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Fri, 20 Dec 2019 16:07:42 +0200
Subject: [PATCH 116/139] try to work around filter configuration issues

---
 pandora/app/config.py | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/pandora/app/config.py b/pandora/app/config.py
index b20686c3..8b5b98f7 100644
--- a/pandora/app/config.py
+++ b/pandora/app/config.py
@@ -86,6 +86,21 @@ def load_config(init=False):
 
         # enable default filters if needed
         default_filters = [f['id'] for f in config['user']['ui']['filters']]
+        available_filters = [key['id'] for key in config['itemKeys'] if key.get('filter')]
+        unknown_ids = set(default_filters) - set(available_filters)
+        if unknown_ids:
+            sys.stderr.write('WARNING: unknown item keys in default filters: %s.\n' % list(unknown_ids))
+            unused_filters = [key for key in available_filters if key not in default_filters]
+            if len(unused_filters) < len(unknown_ids):
+                sys.stderr.write('you need at least 5 item filters')
+            else:
+                auto_filters = unused_filters[:len(unknown_ids)]
+                default_filters += auto_filters
+                for key in auto_filters:
+                    config['user']['ui']['filters'].append({
+                        "id": key, "sort": [{"key": "items", "operator": "-"}]
+                    })
+                sys.stderr.write('         using the following document filters instead: %s.\n' % auto_filters)
         for key in config['itemKeys']:
             if key['id'] in default_filters and not key.get('filter'):
                 key['filter'] = True
@@ -93,6 +108,22 @@ def load_config(init=False):
 
         # enable default document filters if needed
         default_filters = [f['id'] for f in config['user']['ui']['documentFilters']]
+        available_filters = [key['id'] for key in config['documentKeys'] if key.get('filter')]
+        unknown_ids = set(default_filters) - set(available_filters)
+        if unknown_ids:
+            sys.stderr.write('WARNING: unknown document keys in default filters: %s.\n' % list(unknown_ids))
+            unused_filters = [key for key in available_filters if key not in default_filters]
+            if len(unused_filters) < len(unknown_ids):
+                sys.stderr.write('you need at least 5 item filters')
+            else:
+                auto_filters = unused_filters[:len(unknown_ids)]
+                default_filters += auto_filters
+                for key in auto_filters:
+                    config['user']['ui']['documentFilters'].append({
+                        "id": key, "sort": [{"key": "items", "operator": "-"}]
+                    })
+                sys.stderr.write('         using the following document filters instead: %s.\n' % auto_filters)
+
         for key in config['documentKeys']:
             if key['id'] in default_filters and not key.get('filter'):
                 key['filter'] = True

From b711d65bea1390db14ca55c16fb560c36f828b6a Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Fri, 20 Dec 2019 16:08:22 +0200
Subject: [PATCH 117/139] run ctl init as pandora user

---
 ctl | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/ctl b/ctl
index da1dcd63..1b93c214 100755
--- a/ctl
+++ b/ctl
@@ -9,33 +9,37 @@ fi
 if [ "$action" = "init" ]; then
     cd "`dirname "$0"`"
     BASE=`pwd`
-    python3 -m venv --system-site-packages .
-
+    SUDO=""
+    PANDORA_USER=`ls -l update.py | cut -f3 -d" "`
+    if [ `whoami` != $PANDORA_USER ]; then
+        SUDO="sudo -H -u $PANDORA_USER"
+    fi
+    $SUDO python3 -m venv --system-site-packages .
     branch=`cat .git/HEAD  | sed 's@/@\n@g' | tail -n1`
 
     # Work around broken venv module in Ubuntu 16.04 / Debian 9
     if [ ! -e bin/pip ]; then
-        bin/python3 -m pip install -U --ignore-installed "pip<9"
+        $SUDO bin/python3 -m pip install -U --ignore-installed "pip<9"
     fi
     if [ ! -d static/oxjs ]; then
-        git clone --depth 1 -b $branch https://git.0x2620.org/oxjs.git static/oxjs
+        $SUDO git clone --depth 1 -b $branch https://git.0x2620.org/oxjs.git static/oxjs
     fi
-    mkdir -p src
+    $SUDO mkdir -p src
     if [ ! -d src/oxtimelines ]; then
-        git clone --depth 1 -b $branch https://git.0x2620.org/oxtimelines.git src/oxtimelines
+        $SUDO git clone --depth 1 -b $branch https://git.0x2620.org/oxtimelines.git src/oxtimelines
     fi
     for package in oxtimelines python-ox; do
         cd ${BASE}
         if [ ! -d src/${package} ]; then
-            git clone --depth 1 -b $branch https://git.0x2620.org/${package}.git src/${package}
+            $SUDO git clone --depth 1 -b $branch https://git.0x2620.org/${package}.git src/${package}
         fi
         cd ${BASE}/src/${package}
-        ${BASE}/bin/python setup.py develop
+        $SUDO ${BASE}/bin/python setup.py develop
     done
     cd ${BASE}
-    ./bin/pip install -r requirements.txt
+    $SUDO ./bin/pip install -r requirements.txt
     if [ ! -e pandora/gunicorn_config.py ]; then
-        cp pandora/gunicorn_config.py.in pandora/gunicorn_config.py
+        $SUDO cp pandora/gunicorn_config.py.in pandora/gunicorn_config.py
     fi
     exit 0
 fi

From ae211e0e95e295bd7ff3faa67709116dc1fc8023 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Fri, 20 Dec 2019 16:59:42 +0200
Subject: [PATCH 118/139] fix permissions

---
 vm/pandora_install.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/vm/pandora_install.sh b/vm/pandora_install.sh
index 1c635773..5f957bb2 100755
--- a/vm/pandora_install.sh
+++ b/vm/pandora_install.sh
@@ -129,6 +129,7 @@ fi
 git clone https://git.0x2620.org/pandora.git /srv/pandora
 cd /srv/pandora
 git checkout $BRANCH
+chown -R $PANDORA:$PANDORA /srv/pandora
 ./ctl init
 
 # create config.jsonc from templates in git

From d3f2cf770fe5374b09624010bdbc3131c33bc14b Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Fri, 27 Dec 2019 11:19:43 +0100
Subject: [PATCH 119/139] select frame in show site poster mode or if no other
 posters exist

---
 static/js/infoView.0xdb.js | 158 +++++++++++++++++++++++++++++--------
 1 file changed, 126 insertions(+), 32 deletions(-)

diff --git a/static/js/infoView.0xdb.js b/static/js/infoView.0xdb.js
index 273acb62..f2324286 100644
--- a/static/js/infoView.0xdb.js
+++ b/static/js/infoView.0xdb.js
@@ -919,12 +919,12 @@ pandora.ui.infoView = function(data, isMixed) {
         });
     }
 
-    function renderList() {
+    function renderFrames() {
         pandora.api.get({
             id: data.id,
-            keys: [ui.icons == 'posters' ? 'posters' : 'frames']
+            keys: ['frames']
         }, 0, function(result) {
-            var images = result.data[ui.icons == 'posters' ? 'posters' : 'frames'].map(function(image) {
+            var images = result.data.frames.map(function(image) {
                     return Ox.extend(image, {index: image.index.toString()});
                 }),
                 selectedImage = images.filter(function(image) {
@@ -932,8 +932,8 @@ pandora.ui.infoView = function(data, isMixed) {
                 })[0],
                 modified = data.modified;
             $list = Ox.IconList({
-                    defaultRatio: ui.icons == 'posters' || !data.stream ? pandora.site.posters.ratio : data.stream.aspectratio,
-                    fixedRatio: ui.icons == 'posters' || !data.stream ? false : data.stream.aspectratio,
+                    defaultRatio: !data.stream ? pandora.site.posters.ratio : data.stream.aspectratio,
+                    fixedRatio: !data.stream ? false : data.stream.aspectratio,
                     item: function(data, sort, size) {
                         var ratio = data.width / data.height;
                         size = size || 128;
@@ -941,17 +941,124 @@ pandora.ui.infoView = function(data, isMixed) {
                             height: ratio <= 1 ? size : size / ratio,
                             id: data.id,
                             info: data.width + ' × ' + data.height + ' px',
-                            title: ui.icons == 'posters' ? data.source : Ox.formatDuration(data.position),
+                            title: Ox.formatDuration(data.position),
+                            url: data.url,
+                            width: ratio >= 1 ? size : size * ratio
+                        }
+                    },
+                    items: images,
+                    keys: ['index', 'position', 'width', 'height', 'url'],
+                    max: 1,
+                    min: 1,
+                    orientation: 'both',
+                    // fixme: should never be undefined
+                    selected: selectedImage ? [selectedImage['index']] : [],
+                    size: 128,
+                    sort: [{key: 'index', operator: '+'}],
+                    unique: 'index'
+                })
+                .addClass('OxMedia')
+                .css({
+                    display: 'block',
+                    position: 'absolute',
+                    left: 0,
+                    top: 0,
+                    width: listWidth + 'px',
+                    height: getHeight() + 'px'
+                })
+                .bindEvent({
+                    select: function(event) {
+                        var index = event.ids[0];
+                        selectedImage = images.filter(function(image) {
+                            return image.index == index;
+                        })[0];
+                        var imageRatio = selectedImage.width / selectedImage.height,
+                            src = selectedImage.url
+                        if ($browserImages.length == 0) {
+                            $browserImages = pandora.$ui.browser.find('img[src*="/' + data.id + '/"]');
+                        }
+                        pandora.api.setPosterFrame({
+                            id: data.id,
+                            // fixme: api slightly inconsistent, this shouldn't be "position"
+                            position: selectedImage.index
+                        }, function() {
+                            var src;
+                            Ox.Request.clearCache();
+                            if (ui.icons == 'frames') {
+                                src = pandora.getMediaURL('/' + data.id + '/icon512.jpg?' + Ox.uid());
+                                $icon.attr({src: src});
+                                $reflectionIcon.attr({src: src});
+                                if (pandora.$ui.videoPreview) {
+                                    pandora.$ui.videoPreview.options({
+                                        position: $list.value(selectedImage.index, 'position')
+                                    });
+                                }
+                            } else if (ui.icons == 'posters' && ui.showSitePosters) {
+                                src = pandora.getMediaURL('/' + data.id + '/siteposter512.jpg?' + Ox.uid());
+                                $icon.attr({src: src});
+                                $reflectionIcon.attr({src: src});
+                            }
+                            $browserImages.each(function() {
+                                $(this).attr({src: pandora.getMediaURL('/' + data.id + '/' + (
+                                    ui.icons == 'posters'
+                                        ? ui.showSitePosters ? 'siteposter' : 'poster'
+                                        : 'icon'
+                                ) + '128.jpg?' + Ox.uid())});
+                            });
+                            if (ui.listSort[0].key == 'modified') {
+                                pandora.$ui.browser.reloadList();
+                            }
+                        });
+                    }
+                })
+                .appendTo($info);
+            $list.size();
+        });
+    }
+
+    function renderList() {
+        if (ui.icons == 'posters' && !ui.showSitePosters) {
+            renderPosters()
+        } else {
+            renderFrames()
+        }
+    }
+
+    function renderPosters() {
+        pandora.api.get({
+            id: data.id,
+            keys: ['posters']
+        }, 0, function(result) {
+            var images = result.data.posters.map(function(image) {
+                    return Ox.extend(image, {index: image.index.toString()});
+                }),
+                selectedImage = images.filter(function(image) {
+                    return image.selected;
+                })[0],
+                modified = data.modified;
+            if (images.length == 1) {
+                renderFrames()
+                return
+            }
+            $list = Ox.IconList({
+                    defaultRatio: pandora.site.posters.ratio,
+                    fixedRatio: false,
+                    item: function(data, sort, size) {
+                        var ratio = data.width / data.height;
+                        size = size || 128;
+                        return {
+                            height: ratio <= 1 ? size : size / ratio,
+                            id: data.id,
+                            info: data.width + ' × ' + data.height + ' px',
+                            title: data.source,
                             url: data.url.replace('http://', '//') + (
-                                ui.icons == 'posters' && data.source == pandora.site.site.url ? '?' + modified : ''
+                                data.source == pandora.site.site.url ? '?' + modified : ''
                             ),
                             width: ratio >= 1 ? size : size * ratio
                         }
                     },
                     items: images,
-                    keys: ui.icons == 'posters'
-                        ? ['index', 'source', 'width', 'height', 'url']
-                        : ['index', 'position', 'width', 'height', 'url'],
+                    keys: ['index', 'source', 'width', 'height', 'url'],
                     max: 1,
                     min: 1,
                     orientation: 'both',
@@ -981,12 +1088,12 @@ pandora.ui.infoView = function(data, isMixed) {
                         if ($browserImages.length == 0) {
                             $browserImages = pandora.$ui.browser.find('img[src*="/' + data.id + '/"]');
                         }
-                        if (ui.icons == 'posters' && !ui.showSitePosters) {
+                        if (!ui.showSitePosters) {
                             $browserImages.each(function() {
                                 var $this = $(this),
                                     size = Math.max($this.width(), $this.height());
                                 $this.attr({src: src});
-                                ui.icons == 'posters' && $this.css(imageRatio < 1 ? {
+                                $this.css(imageRatio < 1 ? {
                                     width: Math.round(size * imageRatio) + 'px',
                                     height: size + 'px'
                                 } : {
@@ -1000,31 +1107,17 @@ pandora.ui.infoView = function(data, isMixed) {
                             iconSize = iconSize == 256 ? 512 : 256;
                             toggleIconSize();
                         }
-                        pandora.api[ui.icons == 'posters' ? 'setPoster' : 'setPosterFrame'](Ox.extend({
-                            id: data.id
-                        }, ui.icons == 'posters' ? {
+                        pandora.api.setPoster({
+                            id: data.id,
                             source: selectedImage.source
-                        } : {
-                             // fixme: api slightly inconsistent, this shouldn't be "position"
-                            position: selectedImage.index
-                        }), function() {
+                        }, function() {
                             var src;
                             Ox.Request.clearCache();
-                            if (ui.icons == 'frames') {
-                                src = pandora.getMediaURL('/' + data.id + '/icon512.jpg?' + Ox.uid());
-                                $icon.attr({src: src});
-                                $reflectionIcon.attr({src: src});
-                                if (pandora.$ui.videoPreview) {
-                                    pandora.$ui.videoPreview.options({
-                                        position: $list.value(selectedImage.index, 'position')
-                                    });
-                                }
-                            }
                             if (!ui.showSitePosters) {
                                 $browserImages.each(function() {
-                                    $(this).attr({src: pandora.getMediaURL('/' + data.id + '/' + (
-                                        ui.icons == 'posters' ? 'poster' : 'icon'
-                                    ) + '128.jpg?' + Ox.uid())});
+                                    $(this).attr({
+                                        src: pandora.getMediaURL('/' + data.id + '/poster128.jpg?' + Ox.uid())
+                                    });
                                 });
                             }
                             if (ui.listSort[0].key == 'modified') {
@@ -1038,6 +1131,7 @@ pandora.ui.infoView = function(data, isMixed) {
         });
     }
 
+
     function renderRightsLevel() {
         var $rightsLevelElement = getRightsLevelElement(data.rightslevel),
             $rightsLevelSelect;

From 7fa2a8a33803ec632d1471933a5cf2c65291b5ba Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Fri, 17 Jan 2020 23:34:07 +0100
Subject: [PATCH 120/139] fix save as collection in documents section

---
 static/js/documentFilterForm.js |  1 +
 static/js/filterDialog.js       | 25 ++++++++++++++++++-------
 2 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/static/js/documentFilterForm.js b/static/js/documentFilterForm.js
index 957bb512..a1fb25c9 100644
--- a/static/js/documentFilterForm.js
+++ b/static/js/documentFilterForm.js
@@ -37,6 +37,7 @@ pandora.ui.documentFilterForm = function(options) {
                         return item.id;
                     })
                 }]),
+                listName: Ox._('Collection'),
                 list: mode == 'find' ? {
                     sort: pandora.user.ui.collectionSort,
                     view: pandora.user.ui.collectionView
diff --git a/static/js/filterDialog.js b/static/js/filterDialog.js
index 3e5b7b46..6ad2efed 100644
--- a/static/js/filterDialog.js
+++ b/static/js/filterDialog.js
@@ -12,7 +12,9 @@ pandora.ui.filterDialog = function() {
                         click: function() {
                             var list = pandora.$ui.filterForm.getList();
                             if (list.save) {
-                                pandora.api.addList({
+                                pandora.api[
+                                    pandora.user.ui.section == 'documents' ? 'addCollection' : 'addList'
+                                ]({
                                     name: list.name,
                                     query: list.query,
                                     status: 'private',
@@ -20,12 +22,21 @@ pandora.ui.filterDialog = function() {
                                 }, function(result) {
                                     var $list = pandora.$ui.folderList.personal,
                                         id = result.data.id;
-                                    pandora.UI.set({
-                                        find: {
-                                            conditions: [{key: 'list', value: id, operator: '=='}],
-                                            operator: '&'
-                                        }
-                                    });
+                                    if (pandora.user.ui.section) {
+                                        pandora.UI.set({
+                                            findDocuments: {
+                                                conditions: [{key: 'collection', value: id, operator: '=='}],
+                                                operator: '&'
+                                            }
+                                        });
+                                    } else {
+                                        pandora.UI.set({
+                                            find: {
+                                                conditions: [{key: 'list', value: id, operator: '=='}],
+                                                operator: '&'
+                                            }
+                                        });
+                                    }
                                     Ox.Request.clearCache(); // fixme: remove
                                     $list.bindEventOnce({
                                             load: function(data) {

From addce753236ef4ecd285a21a5696c1834d3b8e76 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Sat, 18 Jan 2020 00:08:20 +0100
Subject: [PATCH 121/139] fix collection dubplication

---
 static/js/folderList.js |  2 +-
 static/js/folders.js    | 25 ++++++++++++++++++++++---
 static/js/utils.js      | 21 ++++++++++++++-------
 3 files changed, 37 insertions(+), 11 deletions(-)

diff --git a/static/js/folderList.js b/static/js/folderList.js
index 03cccd74..d9d6bca7 100644
--- a/static/js/folderList.js
+++ b/static/js/folderList.js
@@ -391,7 +391,7 @@ pandora.ui.folderList = function(id, section) {
         },
         key_control_d: function() {
             if (that.options('selected').length) {
-                pandora.addFolderItem(ui.section, ui._list);
+                pandora.addFolderItem(ui.section, ui.section == 'documents' ? ui._collection : ui._list);
             }
         },
         key_control_e: function() {
diff --git a/static/js/folders.js b/static/js/folders.js
index 21903e2b..20d4f613 100644
--- a/static/js/folders.js
+++ b/static/js/folders.js
@@ -52,13 +52,13 @@ pandora.ui.folders = function(section) {
                                     [Ox._(folderItem)]),
                                     keyboard: 'shift control n',
                                     disabled:  ui.section == 'documents'
-                                        ? ui.collectionSelection == 0
+                                        ? ui.collectionSelection.length == 0
                                         : ui.listSelection.length == 0
                                 },
                                 { id: 'newsmartlist', title: Ox._('New Smart {0}', [Ox._(folderItem)]), keyboard: 'alt control n' },
                                 { id: 'newsmartlistfromresults', title: Ox._('New Smart {0} from Results', [Ox._(folderItem)]), keyboard: 'shift alt control n' },
                                 {},
-                                { id: 'duplicatelist', title: Ox._('Duplicate Selected {0}', [Ox._(folderItem)]), keyboard: 'control d', disabled: !ui._list },
+                                { id: 'duplicatelist', title: Ox._('Duplicate Selected {0}', [Ox._(folderItem)]), keyboard: 'control d', disabled: ui.section == 'documents' ? !ui._collection : !ui._list },
                                 { id: 'editlist', title: Ox._('Edit Selected {0}...', [Ox._(folderItem)]), keyboard: 'control e', disabled: !editable },
                                 { id: 'deletelist', title: Ox._('Delete Selected {0}...', [Ox._(folderItem)]), keyboard: 'delete', disabled: !editable }
                             ],
@@ -75,13 +75,18 @@ pandora.ui.folders = function(section) {
                                 ], data.id)) {
                                     pandora.addList(data.id.indexOf('smart') > -1, data.id.indexOf('from') > -1);
                                 } else if (data.id == 'duplicatelist') {
-                                    pandora.addList(ui._list);
+                                    pandora.addList(ui.section == 'documents' ? ui._collection : ui._list);
                                 } else if (data.id == 'editlist') {
                                     pandora.ui.listDialog().open();
                                 } else if (data.id == 'deletelist') {
                                     pandora.ui.deleteListDialog().open();
                                 }
                             },
+                            pandora_collectionselection: function(data) {
+                                pandora.$ui.personalListsMenu[
+                                    data.value.length ? 'enableItem' : 'disableItem'
+                                ]('newlistfromselection');
+                            },
                             pandora_find: function() {
                                 // fixme: duplicated
                                 var action = ui._list
@@ -96,6 +101,20 @@ pandora.ui.folders = function(section) {
                                     ui.listSelection.length ? 'enableItem' : 'disableItem'
                                 ]('newlistfromselection');
                             },
+                            pandora_finddocuments: function() {
+                                // fixme: duplicated
+                                var action = ui._collection
+                                    && pandora.getListData(ui._collection).user == pandora.user.username
+                                    ? 'enableItem' : 'disableItem'
+                                pandora.$ui.personalListsMenu[
+                                    ui._collection ? 'enableItem' : 'disableItem'
+                                ]('duplicatelist');
+                                pandora.$ui.personalListsMenu[action]('editlist');
+                                pandora.$ui.personalListsMenu[action]('deletelist');
+                                pandora.$ui.personalListsMenu[
+                                    ui.collectionSelection.length ? 'enableItem' : 'disableItem'
+                                ]('newlistfromselection');
+                            },
                             pandora_listselection: function(data) {
                                 pandora.$ui.personalListsMenu[
                                     data.value.length ? 'enableItem' : 'disableItem'
diff --git a/static/js/utils.js b/static/js/utils.js
index 6cfefc2d..425a877a 100644
--- a/static/js/utils.js
+++ b/static/js/utils.js
@@ -81,18 +81,28 @@ pandora.addFolderItem = function(section) {
             data.description = result.data.items[0].description;
             if (data.type == 'static') {
                 var query;
-                if (isItems) {
+                if (Ox.contains(['items', 'documents'], section)) {
                     query = {
                         conditions: [{
-                            key: 'list',
+                            key: {
+                                items: 'list',
+                                documents: 'collection'
+                            }[section],
                             operator: '==',
                             value: list
                         }],
                         operator: '&'
                     };
-                    pandora.api.find({query: query}, function(result) {
+
+                    pandora.api[{
+                        items: 'find',
+                        documents: 'findDocuments'
+                    }[section]]({query: query}, function(result) {
                         if (result.data.items) {
-                            pandora.api.find({
+                            pandora.api[{
+                                items: 'find',
+                                documents: 'findDocuments'
+                            }[section]]({
                                 query: query,
                                 keys: ['id'],
                                 sort: [{key: 'id', operator: ''}],
@@ -107,9 +117,6 @@ pandora.addFolderItem = function(section) {
                             addList();
                         }
                     });
-                } else if(section == 'documents') {
-                    //fixme
-                    addList();
                 } else {
                     pandora.api.getEdit({
                         id: list,

From 230275e4d84619bbf3fe207bbc46a34d4ea0ba04 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Sun, 19 Jan 2020 11:58:39 +0100
Subject: [PATCH 122/139] use pandora.site.documentKeys for item documents view

---
 pandora/app/config.py       |  11 +++
 static/js/documentsPanel.js | 162 ++++++++++--------------------------
 2 files changed, 56 insertions(+), 117 deletions(-)

diff --git a/pandora/app/config.py b/pandora/app/config.py
index 8b5b98f7..2ee803eb 100644
--- a/pandora/app/config.py
+++ b/pandora/app/config.py
@@ -186,6 +186,17 @@ def load_config(init=False):
             if level not in config[key]:
                 config[key] = default.get(key, 0)
 
+        config['user']['ui']['documentsSort'] = [
+            s for s in config['user']['ui']['documentsSort']
+            if get_by_id(config['documentKeys'], s['key'])
+        ]
+        if not config['user']['ui']['documentsSort']:
+            sort_key = [k for k in config['documentKeys'] if k['id'] != '*'][0]
+            config['user']['ui']['documentsSort'] = [{
+                "key": sort_key['id'],
+                "operator": sort_key.get('operator', '+')
+            }]
+
         for key in ('language', 'importMetadata'):
             if key not in config:
                 sys.stderr.write("adding default value:\n\t\"%s\": %s,\n\n" % (key, json.dumps(default[key])))
diff --git a/static/js/documentsPanel.js b/static/js/documentsPanel.js
index f0f03a9a..e1663aba 100644
--- a/static/js/documentsPanel.js
+++ b/static/js/documentsPanel.js
@@ -1,107 +1,8 @@
 'use strict';
 
-pandora.documentColumns = [
-    {
-        id: 'title',
-        operator: '+',
-        title: Ox._('Title'),
-        find: true,
-        visible: true,
-        width: 256
-    },
-    {
-        id: 'id',
-        operator: '+',
-        title: Ox._('ID'),
-        visible: true,
-        width: 64
-    },
-    {
-        format: function(value) {
-            return value.toUpperCase();
-        },
-        id: 'extension',
-        operator: '+',
-        title: Ox._('Extension'),
-        find: true,
-        visible: true,
-        width: 64
-    },
-    {
-        align: 'right',
-        format: function(value, data) {
-            return Ox.isArray(value)
-                ? Ox.formatDimensions(value, 'px')
-                : Ox.formatCount(value, (data && data.extension == 'html') ? 'word' : 'page');
-        },
-        id: 'dimensions',
-        operator: '-',
-        title: Ox._('Dimensions'),
-        visible: true,
-        width: 128
-    },
-    {
-        align: 'right',
-        format: function(value) {
-            return Ox.formatValue(value, 'B');
-        },
-        id: 'size',
-        operator: '-',
-        title: Ox._('Size'),
-        visible: true,
-        width: 64
-    },
-    {
-        id: 'description',
-        operator: '+',
-        title: Ox._('Description'),
-        find: true,
-        visible: true,
-        width: 256
-    },
-    {
-        align: 'right',
-        id: 'matches',
-        operator: '-',
-        title: Ox._('Matches'),
-        visible: true,
-        width: 64
-    },
-    {
-        id: 'user',
-        operator: '+',
-        title: Ox._('User'),
-        find: true,
-        visible: true,
-        width: 128
-    },
-    {
-        align: 'right',
-        format: function(value) {
-            return Ox.formatDate(value, '%F %T');
-        },
-        id: 'created',
-        operator: '-',
-        title: Ox._('Created'),
-        visible: true,
-        width: 144
-    },
-    {
-        align: 'right',
-        format: function(value) {
-            return Ox.formatDate(value, '%F %T');
-        },
-        id: 'modified',
-        operator: '-',
-        title: Ox._('Modified'),
-        visible: true,
-        width: 144
-    }
-];
-
 pandora.ui.documentSortSelect = function() {
     var ui = pandora.user.ui,
-            $orderButton = Ox.Button({
+        $orderButton = Ox.Button({
             overlap: 'left',
             title: getOrderButtonTitle(),
             tooltip: getOrderButtonTooltip(),
@@ -116,7 +17,9 @@ pandora.ui.documentSortSelect = function() {
             }
         }),
         $sortSelect = Ox.Select({
-            items: pandora.documentColumns.map(function(column) {
+            items: pandora.site.documentKeys.filter(function(key) {
+                return key.sort;
+            }).map(function(column) {
                 return {
                     id: column.id,
                     title: Ox._('Sort by {0}', [column.title])
@@ -130,7 +33,7 @@ pandora.ui.documentSortSelect = function() {
                 var key = data.value;
                 pandora.UI.set({documentsSort: [{
                     key: key,
-                    operator: Ox.getObjectById(pandora.documentColumns, key).operator
+                    operator: Ox.getObjectById(pandora.site.documentKeys, key).operator
                 }]});
             }
         }),
@@ -861,21 +764,46 @@ pandora.ui.documentsPanel = function(options) {
 
     function renderList() {
         var options = {
-            items: pandora.api.findDocuments,
-            keys: ['description', 'dimensions', 'extension', 'id', 'title', 'ratio', 'size', 'user', 'entities', 'modified'],
-            query: {
-                conditions: isItemView ? [{ key: 'item', value: ui.item, operator: '==' }] : [],
-                operator: '&'
-            },
-            selected: ui.documentsSelection[isItemView ? ui.item : ''] || [],
-            sort: ui.documentsSort.concat([
-                {key: 'extension', operator: '+'},
-                {key: 'title', operator: '+'}
-            ]),
-            unique: 'id'
-        };
+                items: pandora.api.findDocuments,
+                keys: ['description', 'dimensions', 'extension', 'id', 'title', 'ratio', 'size', 'user', 'entities', 'modified'],
+                query: {
+                    conditions: isItemView ? [{ key: 'item', value: ui.item, operator: '==' }] : [],
+                    operator: '&'
+                },
+                selected: ui.documentsSelection[isItemView ? ui.item : ''] || [],
+                sort: ui.documentsSort.concat([
+                    {key: 'extension', operator: '+'},
+                    {key: 'title', operator: '+'}
+                ]),
+                unique: 'id'
+            };
         return (ui.documentsView == 'list' ? Ox.TableList(Ox.extend(options, {
-            columns: pandora.documentColumns,
+            columns: pandora.site.documentSortKeys.filter(function(key) {
+                return (!key.capability
+                    || pandora.hasCapability(key.capability)) && key.columnWidth;
+            }).map(function(key) {
+                var position = ui.collectionColumns.indexOf(key.id);
+                return {
+                    addable: key.id != 'random',
+                    align: ['string', 'text'].indexOf(
+                        Ox.isArray(key.type) ? key.type[0]: key.type
+                    ) > -1 ? 'left' : key.type == 'list' ? 'center' : 'right',
+                    defaultWidth: key.columnWidth,
+                    format: (function() {
+                        return function(value, data) {
+                            return pandora.formatDocumentKey(key, data);
+                        }
+                    })(),
+                    id: key.id,
+                    operator: pandora.getDocumentSortOperator(key.id),
+                    position: position,
+                    removable: !key.columnRequired,
+                    title: Ox._(key.title),
+                    type: key.type,
+                    visible: position > -1,
+                    width: ui.collectionColumnWidth[key.id] || key.columnWidth
+                };
+            }),
             columnsVisible: true,
             scrollbarVisible: true,
         })) : Ox.IconList(Ox.extend(options, {
@@ -883,7 +811,7 @@ pandora.ui.documentsPanel = function(options) {
                 var sortKey = sort[0].key,
                     infoKey = sortKey == 'title' ? 'extension' : sortKey,
                     info = (
-                        Ox.getObjectById(pandora.documentColumns, infoKey).format || Ox.identity
+                        Ox.getObjectById(pandora.site.documentKeys, infoKey).format || Ox.identity
                     )(data[infoKey]),
                     size = size || 128;
                 return {

From c64b13537b395d917c69aef99d7632c165d8cf90 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Sun, 19 Jan 2020 12:48:03 +0100
Subject: [PATCH 123/139] fix html documents preview in dialog

---
 static/js/documentDialog.js |  5 ++---
 static/js/textPanel.js      | 21 +++++++++++++++++----
 2 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/static/js/documentDialog.js b/static/js/documentDialog.js
index 752bca35..89bc072b 100644
--- a/static/js/documentDialog.js
+++ b/static/js/documentDialog.js
@@ -201,8 +201,7 @@ pandora.ui.documentDialog = function(options) {
                     zoom: 'fit'
                 })
                 : item.extension == 'html'
-                ? pandora.ui.textPanel(item).css({
-                })
+                ? pandora.$ui.textPanel = pandora.ui.textPanel(item)
                 : Ox.ImageViewer({
                     area: pandora.user.ui.documents[item.id]
                         ? pandora.user.ui.documents[item.id].position
@@ -243,7 +242,7 @@ pandora.ui.documentDialog = function(options) {
     }
 
     function setTitle() {
-        that.options({title: item.title + '.' + item.extension});
+        that.options({title: item.title + (item.extension == 'html' ? '' : '.' + item.extension)});
     }
 
     that.getItems = function() {
diff --git a/static/js/textPanel.js b/static/js/textPanel.js
index a7aa4129..c62a435e 100644
--- a/static/js/textPanel.js
+++ b/static/js/textPanel.js
@@ -1,16 +1,30 @@
 'use strict';
 
 pandora.ui.textPanel = function(text, $toolbar) {
+    if (Ox.isUndefined(text.text)) {
+        var that = Ox.Element().append(Ox.LoadingScreen().start())
+        pandora.api.getDocument({
+            id: text.id,
+            keys: ['text']
+        }, function(result) {
+            text.text = result.data.text
+            if (text.text) {
+                pandora.$ui.textPanel.replaceWith(pandora.$ui.textPanel = pandora.ui.textPanel(text, $toolbar))
+            }
+        })
+        return that;
+    }
+
     var textElement,
         textEmbed,
         embedURLs = getEmbedURLs(text.text),
         that = Ox.SplitPanel({
             elements: [
                 {
-                    element: textElement = pandora.$ui.textElement = pandora.ui.textHTML(text)
+                    element: textElement = pandora.ui.textHTML(text)
                 },
                 {
-                    element: textEmbed = pandora.ui.textEmbed(textElement),
+                    element: textEmbed = pandora.ui.textEmbed(),
                     collapsed: !embedURLs.length,
                     size: pandora.user.ui.embedSize,
                     resizable: true,
@@ -124,7 +138,6 @@ pandora.ui.textPanel = function(text, $toolbar) {
             0),
             position = 100 * scrollTop / Math.max(1, textElement[0].scrollHeight);
         textElement.scrollTo(position);
-        window.text = textElement;
     }
 
     that.selectEmbed = function(index, scroll) {
@@ -431,7 +444,7 @@ pandora.ui.textHTML = function(text) {
 
 };
 
-pandora.ui.textEmbed = function(textElement) {
+pandora.ui.textEmbed = function(textEmbed) {
 
     var that = Ox.Element()
             .bindEvent({

From 53ed483f9c329fadd13c367bf38be395f382120a Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Tue, 21 Jan 2020 14:09:01 +0100
Subject: [PATCH 124/139] don't fail with empty fulltext query

---
 pandora/document/fulltext.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pandora/document/fulltext.py b/pandora/document/fulltext.py
index b383e8df..5ecdab05 100644
--- a/pandora/document/fulltext.py
+++ b/pandora/document/fulltext.py
@@ -63,7 +63,7 @@ class FulltextMixin:
 
     @classmethod
     def find_fulltext_ids(cls, query):
-        if query[0] == '"' and query[-1] == '"':
+        if query and query[0] == '"' and query[-1] == '"':
             query = {
                 "match_phrase": {
                     "text": query.lower()[1:-1]

From a2ff9480cf14728d28f808ee54743f7b8b177fc6 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Tue, 21 Jan 2020 14:09:38 +0100
Subject: [PATCH 125/139] no results for empty query

---
 pandora/document/fulltext.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/pandora/document/fulltext.py b/pandora/document/fulltext.py
index 5ecdab05..040658ce 100644
--- a/pandora/document/fulltext.py
+++ b/pandora/document/fulltext.py
@@ -63,7 +63,9 @@ class FulltextMixin:
 
     @classmethod
     def find_fulltext_ids(cls, query):
-        if query and query[0] == '"' and query[-1] == '"':
+        if not query:
+            return []
+        elif query[0] == '"' and query[-1] == '"':
             query = {
                 "match_phrase": {
                     "text": query.lower()[1:-1]

From 0f889f927d569326b573c73eb8b36c12cc8fc94e Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Tue, 21 Jan 2020 15:18:11 +0100
Subject: [PATCH 126/139] don't load debugger.js

---
 pandora/text/templates/pdf/viewer.html | 5 +----
 static/pdf.js/index.html               | 4 +---
 2 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/pandora/text/templates/pdf/viewer.html b/pandora/text/templates/pdf/viewer.html
index 1cdd5c19..d5bcc6f5 100644
--- a/pandora/text/templates/pdf/viewer.html
+++ b/pandora/text/templates/pdf/viewer.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html>
+<!DOCTYPE html>
 <!--
 Copyright 2012 Mozilla Foundation
 
@@ -41,9 +41,6 @@ See https://github.com/adobe-type-tools/cmap-resources
 <script src="/static/pdf.js/l10n.js"></script>
 <script src="/static/pdf.js/pdf.js"></script>
 
-
-
-    <script src="/static/pdf.js/debugger.js"></script>
     <script type="text/javascript">
         var DEFAULT_URL = '{{url}}',
             embeds = {{embeds|safe}},
diff --git a/static/pdf.js/index.html b/static/pdf.js/index.html
index ea0f40b0..5b103831 100644
--- a/static/pdf.js/index.html
+++ b/static/pdf.js/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html>
+<!DOCTYPE html>
 <!--
 Copyright 2012 Mozilla Foundation
 
@@ -43,8 +43,6 @@ See https://github.com/adobe-type-tools/cmap-resources
 
 
 
-    <script src="/static/pdf.js/debugger.js"></script>
-    
     <script src="/static/pdf.js/embeds.js"></script>
     <script src="/static/pdf.js/viewer.js"></script>
     <link rel="stylesheet" type="text/css" href="/static/pdf.js/css/videopdf.css"/>

From 66f9a517690149991f3dab5999b7a75d0341ec60 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Thu, 30 Jan 2020 12:33:02 +0100
Subject: [PATCH 127/139] fix new edit form query

---
 static/js/utils.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/static/js/utils.js b/static/js/utils.js
index 425a877a..a08f49a0 100644
--- a/static/js/utils.js
+++ b/static/js/utils.js
@@ -48,7 +48,7 @@ pandora.addFolderItem = function(section) {
                     );
                 }
             } else {
-                data.query = isItems ? ui.find : ui.findDocuments;
+                data.query = section == 'documents' ? ui.findDocuments : ui.find;
             }
         }
         if (ui.section == 'items' && section == 'edits') {

From 6b82d7bbf7886c5d05cfad7c3adb0f662a7451f3 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Sun, 2 Feb 2020 15:31:32 +0100
Subject: [PATCH 128/139] use db-ip mmdb

---
 pandora/app/config.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/pandora/app/config.py b/pandora/app/config.py
index 2ee803eb..d9a459cc 100644
--- a/pandora/app/config.py
+++ b/pandora/app/config.py
@@ -397,8 +397,9 @@ def update_geoip(force=False):
     path = os.path.join(settings.GEOIP_PATH, 'GeoLite2-City.mmdb')
     if not os.path.exists(path) or force:
         url = 'http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz'
+        url = 'https://download.db-ip.com/free/dbip-city-lite-2020-01.mmdb.gz'
         print('download', url)
-        ox.net.save_url(url, "%s.gz"%path)
+        ox.net.save_url(url, "%s.gz" % path)
         if os.path.exists(path):
             os.unlink(path)
         os.system('gunzip "%s.gz"' % path)

From 9e0df969e952cac738b9842153ddf5958d23506c Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Sun, 2 Feb 2020 15:33:56 +0100
Subject: [PATCH 129/139] pass video source to download selection

---
 static/js/editor.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/static/js/editor.js b/static/js/editor.js
index 6c3106b9..3d53d89a 100644
--- a/static/js/editor.js
+++ b/static/js/editor.js
@@ -200,7 +200,8 @@ pandora.ui.editor = function(data) {
                     'in': selection['in'],
                     out: selection.out,
                     rightsLevel: rightsLevel,
-                    title: data.title
+                    title: data.title,
+                    video: data.video
                 }).open();
             },
             editannotation: function(data) {

From 6e9b9715fb7e4b82f0cfda99f25dd83cbc65b891 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Sun, 2 Feb 2020 15:42:14 +0100
Subject: [PATCH 130/139] no parts selection if in/out is set

---
 static/js/downloadVideoDialog.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/static/js/downloadVideoDialog.js b/static/js/downloadVideoDialog.js
index 88d58b7e..9e32cad9 100644
--- a/static/js/downloadVideoDialog.js
+++ b/static/js/downloadVideoDialog.js
@@ -9,7 +9,7 @@ pandora.ui.downloadVideoDialog = function(options) {
             'mp4': 'MP4',
         },
 
-        parts = Ox.max(options.video.map(function(video) {
+        parts = options.out ? null : Ox.max(options.video.map(function(video) {
             return video.index
         })),
 

From ecafcbdb2633e60241949a6eb8918349c2ff29f4 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Wed, 5 Feb 2020 16:55:56 +0100
Subject: [PATCH 131/139] update djang

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index ba40eaae..426726e2 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,4 @@
-Django==1.11.22
+Django==1.11.28
 simplejson
 chardet
 celery>4

From c4ff34e9c7f142432addd9024a2fff7fd4794553 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Wed, 5 Feb 2020 16:56:14 +0100
Subject: [PATCH 132/139] make sure title and original title are not the same

---
 static/js/metadataDialog.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/static/js/metadataDialog.js b/static/js/metadataDialog.js
index c9e5c3ea..3c0a718f 100644
--- a/static/js/metadataDialog.js
+++ b/static/js/metadataDialog.js
@@ -225,7 +225,7 @@ pandora.ui.metadataDialog = function(data) {
             if (result.data) {
                 imdb = Ox.clone(result.data, true);
                 if (!Ox.contains(keys, 'originalTitle')) {
-                    if (imdb.originalTitle) {
+                    if (imdb.originalTitle && imdb.originalTitle != imdb.title) {
                         imdb.alternativeTitles = [[imdb.title, []]].concat(imdb.alternativeTitles || []);
                         imdb.title = imdb.originalTitle;
                     }

From 6d1f46a96dd1a52c316a0e7d3f8a36cc765d9b03 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Wed, 5 Feb 2020 17:40:22 +0100
Subject: [PATCH 133/139] fix filters after adding defaults

---
 pandora/app/config.py | 50 ++++++++++++++++++++++---------------------
 1 file changed, 26 insertions(+), 24 deletions(-)

diff --git a/pandora/app/config.py b/pandora/app/config.py
index d9a459cc..db5a1bac 100644
--- a/pandora/app/config.py
+++ b/pandora/app/config.py
@@ -105,30 +105,6 @@ def load_config(init=False):
             if key['id'] in default_filters and not key.get('filter'):
                 key['filter'] = True
                 sys.stderr.write('enabled filter for "%s" since its used as default filter.\n' % (key['id']))
-
-        # enable default document filters if needed
-        default_filters = [f['id'] for f in config['user']['ui']['documentFilters']]
-        available_filters = [key['id'] for key in config['documentKeys'] if key.get('filter')]
-        unknown_ids = set(default_filters) - set(available_filters)
-        if unknown_ids:
-            sys.stderr.write('WARNING: unknown document keys in default filters: %s.\n' % list(unknown_ids))
-            unused_filters = [key for key in available_filters if key not in default_filters]
-            if len(unused_filters) < len(unknown_ids):
-                sys.stderr.write('you need at least 5 item filters')
-            else:
-                auto_filters = unused_filters[:len(unknown_ids)]
-                default_filters += auto_filters
-                for key in auto_filters:
-                    config['user']['ui']['documentFilters'].append({
-                        "id": key, "sort": [{"key": "items", "operator": "-"}]
-                    })
-                sys.stderr.write('         using the following document filters instead: %s.\n' % auto_filters)
-
-        for key in config['documentKeys']:
-            if key['id'] in default_filters and not key.get('filter'):
-                key['filter'] = True
-                sys.stderr.write('enabled filter for document key "%s" since its used as default filter.\n' % (key['id']))
-
         config['keys'] = {}
         for key in config['itemKeys']:
             config['keys'][key['id']] = key
@@ -210,6 +186,32 @@ def load_config(init=False):
         if 'downloadFormat' not in config['video']:
             config['video']['downloadFormat'] = default['video']['downloadFormat']
 
+
+        # enable default document filters if needed
+        default_filters = [f['id'] for f in config['user']['ui']['documentFilters']]
+        available_filters = [key['id'] for key in config['documentKeys'] if key.get('filter')]
+        unknown_ids = set(default_filters) - set(available_filters)
+        if unknown_ids:
+            sys.stderr.write('WARNING: unknown document keys in default filters: %s.\n' % list(unknown_ids))
+            unused_filters = [key for key in available_filters if key not in default_filters]
+            if len(unused_filters) < len(unknown_ids):
+                sys.stderr.write('you need at least 5 item filters')
+            else:
+                auto_filters = unused_filters[:len(unknown_ids)]
+                default_filters += auto_filters
+                for key in auto_filters:
+                    config['user']['ui']['documentFilters'].append({
+                        "id": key, "sort": [{"key": "items", "operator": "-"}]
+                    })
+                sys.stderr.write('         using the following document filters instead: %s.\n' % auto_filters)
+
+        for key in config['documentKeys']:
+            if key['id'] in default_filters and not key.get('filter'):
+                key['filter'] = True
+                sys.stderr.write('enabled filter for document key "%s" since its used as default filter.\n' % (key['id']))
+
+
+
         old_formats = getattr(settings, 'CONFIG', {}).get('video', {}).get('formats', [])
         formats = config.get('video', {}).get('formats')
         if set(old_formats) != set(formats):

From 8dd47e770f921173e3f4af4265ca902d223f7e09 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Thu, 6 Feb 2020 11:25:02 +0100
Subject: [PATCH 134/139] fix background task check

---
 static/js/item.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/static/js/item.js b/static/js/item.js
index c5045d1f..8de8740f 100644
--- a/static/js/item.js
+++ b/static/js/item.js
@@ -45,12 +45,12 @@ pandora.ui.item = function() {
             var note = Ox.Element()
                 .css({marginTop: '32px', fontSize: '12px', textAlign: 'center'})
             pandora.$ui.contentPanel.replaceElement(1, note);
-            if (pandora.user.username == item.user || pandora.hasCapability('canSeeAllTasks')) {
+            if (pandora.user.username == result.data.user || pandora.hasCapability('canSeeAllTasks')) {
                 pandora.api.getTasks({
                     user: pandora.hasCapability('canSeeAllTasks') ? '' : pandora.user.username
                 }, function(result_) {
                     var tasks = result_.data.items.filter(function(task) { return task.item == item})
-                    if (tasks.length == 0) {
+                    if (tasks.length > 0) {
                         html = Ox._(
                             '<i>{0}</i> is currently processed. '
                             + '{1} view will be available in a moment.',

From 966587112cbe7a231cfc793a1422baf8f15a0fb7 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Thu, 13 Feb 2020 21:35:42 +0100
Subject: [PATCH 135/139] fix show/hide filter menu entry for documents

---
 static/js/mainMenu.js | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/static/js/mainMenu.js b/static/js/mainMenu.js
index 58282ecf..e1a80b5e 100644
--- a/static/js/mainMenu.js
+++ b/static/js/mainMenu.js
@@ -1669,7 +1669,11 @@ pandora.ui.mainMenu = function() {
             {
                 id: 'showfilters',
                 title: Ox._((ui.showFilters ? 'Hide' : 'Show') + ' Filters'),
-                disabled: ui.section != 'items' || !!ui.item, keyboard: 'shift f'
+                disabled: (
+                    !Ox.contains(['items', 'documents'], ui.section) ||
+                    (ui.section == 'items' && !!ui.item) ||
+                    (ui.section == 'documents' && !!ui.document)
+                ), keyboard: 'shift f'
             },
             {
                 id: 'showbrowser',

From 6f4bdebd883221968d81a6c9968c7ecd49ca0bc6 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Sun, 22 Mar 2020 16:45:29 +0100
Subject: [PATCH 136/139] format crop string

---
 pandora/archive/extract.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pandora/archive/extract.py b/pandora/archive/extract.py
index 872ee7ea..90422d47 100644
--- a/pandora/archive/extract.py
+++ b/pandora/archive/extract.py
@@ -191,7 +191,7 @@ def stream(video, target, profile, info, audio_track=0, flags={}):
             w = info['video'][0]['width'] - flags['crop']['left'] - flags['crop']['right']
             x = flags['crop']['left']
             y = flags['crop']['top']
-            crop = ',crop=w=%s:h=%s:x=%s:y=%s' (w, h, x, y)
+            crop = ',crop=w=%s:h=%s:x=%s:y=%s' % (w, h, x, y)
             aspect = dar * (info['video'][0]['width'] / info['video'][0]['height']) * (w/h)
             if abs(w/h - aspect) < 0.02:
                 aspect = '%s:%s' % (w, h)

From 4cce468daea267e192bbd3cbfc97b26a0f4e7b1c Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Sun, 22 Mar 2020 17:21:57 +0100
Subject: [PATCH 137/139] dbip-city-lite url changes, get latest url from
 download page

---
 pandora/app/config.py | 23 +++++++++++++++--------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/pandora/app/config.py b/pandora/app/config.py
index db5a1bac..db407178 100644
--- a/pandora/app/config.py
+++ b/pandora/app/config.py
@@ -1,12 +1,13 @@
 # -*- coding: utf-8 -*-
 from __future__ import division, print_function, absolute_import
 
+import codecs
 import os
-import sys
+import re
 import shutil
 import subprocess
+import sys
 import time
-import codecs
 from os.path import dirname, exists, join
 from glob import glob
 
@@ -399,12 +400,18 @@ def update_geoip(force=False):
     path = os.path.join(settings.GEOIP_PATH, 'GeoLite2-City.mmdb')
     if not os.path.exists(path) or force:
         url = 'http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz'
-        url = 'https://download.db-ip.com/free/dbip-city-lite-2020-01.mmdb.gz'
-        print('download', url)
-        ox.net.save_url(url, "%s.gz" % path)
-        if os.path.exists(path):
-            os.unlink(path)
-        os.system('gunzip "%s.gz"' % path)
+        index = ox.net.read_url('https://db-ip.com/db/download/ip-to-country-lite').decode()
+        match = re.compile('href=[\'"](http.*.mmdb.gz)').findall(index)
+        if match:
+            url = match[0]
+            print(url)
+            print('download', url)
+            ox.net.save_url(url, "%s.gz" % path)
+            if os.path.exists(path):
+                os.unlink(path)
+            os.system('gunzip "%s.gz"' % path)
+        else:
+            print('failed to download dbip-country-lite-2020-03.mmdb.gz')
 
 def init():
     if not settings.RELOADER_RUNNING:

From 9e272791bce2e23b1c69961a2527023dbb5d9260 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Sun, 22 Mar 2020 17:45:35 +0100
Subject: [PATCH 138/139] remove debug

---
 pandora/app/config.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/pandora/app/config.py b/pandora/app/config.py
index db407178..36dfdfe3 100644
--- a/pandora/app/config.py
+++ b/pandora/app/config.py
@@ -404,7 +404,6 @@ def update_geoip(force=False):
         match = re.compile('href=[\'"](http.*.mmdb.gz)').findall(index)
         if match:
             url = match[0]
-            print(url)
             print('download', url)
             ox.net.save_url(url, "%s.gz" % path)
             if os.path.exists(path):

From 2d25c60606004a02297e521211f3d08e823cd8df Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Fri, 17 Apr 2020 15:33:30 +0200
Subject: [PATCH 139/139] make private items private

---
 pandora/config.indiancinema.jsonc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pandora/config.indiancinema.jsonc b/pandora/config.indiancinema.jsonc
index 40876054..786a53d4 100644
--- a/pandora/config.indiancinema.jsonc
+++ b/pandora/config.indiancinema.jsonc
@@ -76,7 +76,7 @@
         "canSeeExtraItemViews": {"researcher": true, "staff": true, "admin": true},
         "canSeeMedia": {"researcher": true, "staff": true, "admin": true},
         "canSeeDocument": {"guest": 1, "member": 1, "researcher": 2, "staff": 3, "admin": 3},
-        "canSeeItem": {"guest": 3, "member": 3, "researcher": 3, "staff": 3, "admin": 3},
+        "canSeeItem": {"guest": 2, "member": 2, "researcher": 2, "staff": 3, "admin": 3},
         "canSeeSize": {"researcher": true, "staff": true, "admin": true},
         "canSeeSoftwareVersion": {"researcher": true, "staff": true, "admin": true},
         "canSendMail": {"staff": true, "admin": true}