add 0p for audio only profile

addAnnotation was called multiple times creating multiple annotations
editAnnotation calls might overwrite later calls depending on response
time.

ctl

@@ -17,7 +17,7 @@ if [ "$action" = "init" ]; then
     SUDO=""
     PANDORA_USER=`ls -l update.py | cut -f3 -d" "`
     if [ `whoami` != $PANDORA_USER ]; then
-        SUDO="sudo -H -u $PANDORA_USER"
+        SUDO="sudo -E -H -u $PANDORA_USER"
     fi
     $SUDO python3 -m venv --system-site-packages .
     branch=`cat .git/HEAD  | sed 's@/@\n@g' | tail -n1`
@@ -62,11 +62,10 @@ if [ ! -z $cmd ]; then
     SUDO=""
     PANDORA_USER=`ls -l update.py | cut -f3 -d" "`
     if [ `whoami` != $PANDORA_USER ]; then
-        SUDO="sudo -H -u $PANDORA_USER"
+        SUDO="sudo -E -H -u $PANDORA_USER"
     fi
     shift
-    $SUDO "$BASE/$cmd" $@
+    exec $SUDO "$BASE/$cmd" $@
-    exit $?
 fi
 
 if [ `whoami` != 'root' ]; then

docker/base/install.sh

@@ -30,6 +30,8 @@ apt-get update -qq
 apt-get install -y \
     netcat-openbsd \
     sudo \
+    rsync \
+    iproute2 \
     vim \
     wget \
     pwgen \
@@ -47,17 +49,20 @@ apt-get install -y \
     python3-cssselect \
     python3-html5lib \
     python3-ox \
+    python3-elasticsearch \
     oxframe \
     ffmpeg \
     mkvtoolnix \
     gpac \
     imagemagick \
     poppler-utils \
-    youtube-dl \
     ipython3 \
+    tesseract-ocr \
+    tesseract-ocr-eng \
     postfix \
     postgresql-client
 
+apt-get install -y --no-install-recommends youtube-dl rtmpdump
 apt-get clean
 
 rm -f /install.sh

docker/entrypoint.sh

@@ -8,6 +8,8 @@ export LANG=en_US.UTF-8
 mkdir -p /run/pandora
 chown -R ${user}.${user} /run/pandora
 
+update="/usr/bin/sudo -u $user -E -H /srv/pandora/update.py"
+
 # pan.do/ra services
 if [ "$action" = "pandora" ]; then
     if [ ! -e /srv/pandora/initialized ]; then
@@ -26,12 +28,12 @@ if [ "$action" = "pandora" ]; then
         /overlay/install.py
 
         echo "Initializing database..."
-        echo "CREATE EXTENSION pg_trgm;" | /srv/pandora/pandora/manage.py dbshell
+        echo "CREATE EXTENSION pg_trgm;" | /srv/pandora/pandora/manage.py dbshell || true
         /srv/pandora/pandora/manage.py init_db
-        /srv/pandora/update.py db
+        $update db
         echo "Generating static files..."
-        /srv/pandora/update.py static
         chown -R ${user}.${user} /srv/pandora/
+        $update static
         touch /srv/pandora/initialized
     fi
     /srv/pandora_base/docker/wait-for db 5432
@@ -44,54 +46,53 @@ if [ "$action" = "encoding" ]; then
     /srv/pandora_base/docker/wait-for-file /srv/pandora/initialized
     /srv/pandora_base/docker/wait-for rabbitmq 5672
     name=pandora-encoding-$(hostname)
+    cd /srv/pandora/pandora
     exec /usr/bin/sudo -u $user -E -H \
-        /srv/pandora/bin/python \
+        /srv/pandora/bin/celery \
-        /srv/pandora/pandora/manage.py \
+            -A app worker \
-        celery worker \
+            -Q encoding -n ${name} \
-        -c 1 \
+            --pidfile /run/pandora/encoding.pid \
-        -Q encoding -n $name \
+            --maxtasksperchild 500 \
-        -l INFO
+            -c 1 \
+            -l INFO
 fi
 if [ "$action" = "tasks" ]; then
     /srv/pandora_base/docker/wait-for-file /srv/pandora/initialized
     /srv/pandora_base/docker/wait-for rabbitmq 5672
     name=pandora-default-$(hostname)
+    cd /srv/pandora/pandora
     exec /usr/bin/sudo -u $user -E -H \
-        /srv/pandora/bin/python \
+        /srv/pandora/bin/celery \
-        /srv/pandora/pandora/manage.py \
+        -A app worker \
-        celery worker \
+        -Q default,celery -n ${name} \
-        -Q default,celery -n $name \
+        --pidfile /run/pandora/tasks.pid \
         --maxtasksperchild 1000 \
         -l INFO
 fi
 if [ "$action" = "cron" ]; then
     /srv/pandora_base/docker/wait-for-file /srv/pandora/initialized
     /srv/pandora_base/docker/wait-for rabbitmq 5672
+    cd /srv/pandora/pandora
     exec /usr/bin/sudo -u $user -E -H \
-        /srv/pandora/bin/python \
+        /srv/pandora/bin/celery \
-        /srv/pandora/pandora/manage.py \
+        -A app beat \
-        celerybeat -s /run/pandora/celerybeat-schedule \
+        -s /run/pandora/celerybeat-schedule \
         --pidfile /run/pandora/cron.pid \
         -l INFO
 fi
 if [ "$action" = "websocketd" ]; then
     /srv/pandora_base/docker/wait-for-file /srv/pandora/initialized
     /srv/pandora_base/docker/wait-for rabbitmq 5672
+    cd /srv/pandora/pandora
     exec /usr/bin/sudo -u $user -E -H \
         /srv/pandora/bin/python \
         /srv/pandora/pandora/manage.py websocketd
 fi
 
 # pan.do/ra management and update
-if [ "$action" = "manage.py" ]; then
+if [ "$action" = "ctl" ]; then
     shift
-    exec /usr/bin/sudo -u $user -E -H \
+    exec /srv/pandora/ctl "$@"
-        /srv/pandora/pandora/manage.py "$@"
-fi
-if [ "$action" = "update.py" ]; then
-    shift
-    exec /usr/bin/sudo -u $user -E -H \
-        /srv/pandora/update.py "$@"
 fi
 if [ "$action" = "bash" ]; then
     shift

docker/install.sh

@@ -56,13 +56,9 @@ cp /srv/pandora/docker/entrypoint.sh /entrypoint.sh
 mv /srv/pandora/ /srv/pandora_base/
 mkdir /pandora
 ln -s /pandora /srv/pandora
-cat > /usr/local/bin/update.py << EOF
-#!/bin/sh
-exec /srv/pandora/update.py \$@
-EOF
 
-cat > /usr/local/bin/manage.py << EOF
+cat > /usr/local/bin/pandoractl << EOF
 #!/bin/sh
-exec /srv/pandora/pandora/manage.py \$@
+exec /srv/pandora/ctl \$@
 EOF
-chmod +x /usr/local/bin/manage.py /usr/local/bin/update.py
+chmod +x /usr/local/bin/pandoractl

docker/publish.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+# push new version of pan.do/ra to docker hub
+set -e
+
+cd /tmp
+git clone https://code.0x2620.org/0x2620/pandora
+cd pandora
+./docker/build.sh
+
+docker push 0x2620/pandora-base:latest
+docker push 0x2620/pandora-nginx:latest
+docker push 0x2620/pandora:latest

docker/setup-docker-compose.sh

@@ -26,7 +26,7 @@ and to get started run this:
 
 To update pan.do/ra run:
 
-    docker-compose run pandora update.py
+    docker-compose run pandora ctl update
 
 EOF
 touch __init__.py

docker/wait-for-file

@@ -1,5 +1,5 @@
 #!/bin/sh
-TIMEOUT=60
+TIMEOUT=180
 TARGET="$1"
 
 for i in `seq $TIMEOUT` ; do

pandora/app/config.py

@@ -24,9 +24,6 @@ User = get_user_model()
 
 _win = (sys.platform == "win32")
 
-RUN_RELOADER = True
-NOTIFIER = None
-
 def get_version():
     git_dir = join(dirname(dirname(dirname(__file__))), '.git')
     if exists(git_dir):
@@ -257,46 +254,6 @@ check the README for further details.
             except:
                 pass
 
-
-
-def reloader_thread():
-    global NOTIFIER
-    settings.RELOADER_RUNNING=True
-    _config_mtime = 0
-    try:
-        import pyinotify
-        INOTIFY = True
-    except:
-        INOTIFY = False
-    if INOTIFY:
-        def add_watch():
-            name = os.path.realpath(settings.SITE_CONFIG)
-            wm.add_watch(name, pyinotify.IN_CLOSE_WRITE, reload_config)
-
-        def reload_config(event):
-            load_config()
-            add_watch()
-
-        wm = pyinotify.WatchManager()
-        add_watch()
-        notifier = pyinotify.Notifier(wm)
-        NOTIFIER = notifier
-        notifier.loop()
-    else:
-        while RUN_RELOADER:
-            try:
-                stat = os.stat(settings.SITE_CONFIG)
-                mtime = stat.st_mtime
-                if _win:
-                    mtime -= stat.st_ctime
-                if mtime > _config_mtime:
-                    load_config()
-                    _config_mtime = mtime
-                time.sleep(10)
-            except:
-                #sys.stderr.write("reloading config failed\n")
-                pass
-
 def update_static():
     oxjs_build = os.path.join(settings.STATIC_ROOT, 'oxjs/tools/build/build.py')
     if os.path.exists(oxjs_build):
@@ -407,10 +364,7 @@ def update_geoip(force=False):
             print('failed to download GeoLite2-City.mmdb')
 
 def init():
-    if not settings.RELOADER_RUNNING:
+    load_config(True)
-        load_config(True)
-        if settings.RELOAD_CONFIG:
-            thread.start_new_thread(reloader_thread, ())
 
 def shutdown():
     if settings.RELOADER_RUNNING:

pandora/app/management/commands/init_db.py

@@ -11,6 +11,8 @@ def run(cmd):
     stdout, stderr = p.communicate()
 
     if p.returncode != 0:
+        print('failed to run:', cmd)
+        print(stdout)
         print(stderr)
         sys.exit(1)
 

pandora/archive/external.py

@@ -36,8 +36,10 @@ info_key_map = {
     'display_id': 'id',
 }
 
-def get_info(url):
+def get_info(url, referer=None):
     cmd = ['youtube-dl', '-j', '--all-subs', url]
+    if referer:
+        cmd += ['--referer', referer]
     p = subprocess.Popen(cmd,
                          stdout=subprocess.PIPE,
                          stderr=subprocess.PIPE, close_fds=True)
@@ -57,6 +59,8 @@ def get_info(url):
                     info[-1]['tags'] = []
                 if 'upload_date' in i and i['upload_date']:
                     info[-1]['date'] = '-'.join([i['upload_date'][:4], i['upload_date'][4:6], i['upload_date'][6:]])
+                if 'referer' not in info[-1]:
+                    info[-1]['referer'] = url
     return info
 
 def add_subtitles(item, media, tmp):
@@ -84,9 +88,9 @@ def add_subtitles(item, media, tmp):
                     sub.selected = True
                     sub.save()
 
-def download(item_id, url):
+def download(item_id, url, referer=None):
     item = Item.objects.get(public_id=item_id)
-    info = get_info(url)
+    info = get_info(url, referer)
     if not len(info):
         return '%s contains no videos' % url
     media = info[0]
@@ -96,7 +100,12 @@ def download(item_id, url):
         tmp = tmp.decode('utf-8')
     os.chdir(tmp)
     cmd = ['youtube-dl', '-q', media['url']]
-    if settings.CONFIG['video'].get('reuseUload', False):
+    if referer:
+        cmd += ['--referer', referer]
+    elif 'referer' in media:
+        cmd += ['--referer', media['referer']]
+
+    if settings.CONFIG['video'].get('reuseUpload', False):
         max_resolution = max(settings.CONFIG['video']['resolutions'])
         format = settings.CONFIG['video']['formats'][0]
         if format == 'mp4':

pandora/archive/extract.py

@@ -2,13 +2,13 @@
 
 import os
 from os.path import exists
-
 import fractions
+import math
+import re
+import shutil
 import subprocess
 import tempfile
 import time
-import math
-import shutil
 from distutils.spawn import find_executable
 from glob import glob
 
@@ -63,8 +63,8 @@ def supported_formats():
         'webm': 'libvpx' in stdout and 'libvorbis' in stdout,
         'vp8': 'libvpx' in stdout and 'libvorbis' in stdout,
         'vp9': 'libvpx-vp9' in stdout and 'libopus' in stdout,
-        'mp4': 'libx264' in stdout and 'DEA.L. aac' in stdout,
+        'mp4': 'libx264' in stdout and bool(re.compile('DEA.L. aac').findall(stdout)),
-        'h264': 'libx264' in stdout and 'DEA.L. aac' in stdout,
+        'h264': 'libx264' in stdout and bool(re.compile('DEA.L. aac').findall(stdout)),
     }
 
 def stream(video, target, profile, info, audio_track=0, flags={}):
@@ -145,6 +145,13 @@ def stream(video, target, profile, info, audio_track=0, flags={}):
         audioquality = -1
         audiobitrate = '22k'
         audiochannels = 1
+    elif profile == '0p':
+        info['video'] = []
+        audiorate = 48000
+        audioquality = 6
+        audiobitrate = None
+        audiochannels = None
+        audio_codec = 'libopus'
     else:
         height = 96
 

pandora/archive/models.py

@@ -336,7 +336,9 @@ class File(models.Model):
 
             def done_cb():
                 if done:
-                    self.info.update(ox.avinfo(self.data.path))
+                    info = ox.avinfo(self.data.path)
+                    del info['path']
+                    self.info.update(info)
                     self.parse_info()
                     # reject invalid uploads
                     if self.info.get('oshash') != self.oshash:
@@ -481,6 +483,13 @@ class File(models.Model):
                 user.is_staff or \
                 self.item.user == user or \
                 self.item.groups.filter(id__in=user.groups.all()).count() > 0
+        if 'instances' in data and 'filename' in self.info and self.data:
+            data['instances'].append({
+                'ignore': False,
+                'path': self.info['filename'],
+                'user': self.item.user.username if self.item and self.item.user else 'system',
+                'volume': 'Direct Upload'
+            })
         if not can_see_media:
             if 'instances' in data:
                 data['instances'] = []

pandora/archive/tasks.py

@@ -200,8 +200,8 @@ def update_stream(id):
         c.save()
 
 @task(queue="encoding")
-def download_media(item_id, url):
+def download_media(item_id, url, referer=None):
-    return external.download(item_id, url)
+    return external.download(item_id, url, referer)
 
 @task(queue='default')
 def move_media(data, user):

pandora/archive/views.py

@@ -195,7 +195,9 @@ def addMedia(request, data):
         response['data']['item'] = f.item.public_id
         response['data']['itemUrl'] = request.build_absolute_uri('/%s' % f.item.public_id)
         if not f.available:
-            add_changelog(request, data, f.item.public_id)
+            changelog_data = data.copy()
+            changelog_data['oshash'] = oshash
+            add_changelog(request, changelog_data, f.item.public_id)
     else:
         if 'item' in data:
             i = Item.objects.get(public_id=data['item'])
@@ -220,11 +222,15 @@ def addMedia(request, data):
         if 'info' in data and data['info'] and isinstance(data['info'], dict):
             f.info = data['info']
         f.info['extension'] = extension
+        if 'filename' in data:
+            f.info['filename'] = data['filename']
         f.parse_info()
         f.save()
         response['data']['item'] = i.public_id
         response['data']['itemUrl'] = request.build_absolute_uri('/%s' % i.public_id)
-        add_changelog(request, data, i.public_id)
+        changelog_data = data.copy()
+        changelog_data['oshash'] = oshash
+        add_changelog(request, changelog_data, i.public_id)
     return render_to_json_response(response)
 actions.register(addMedia, cache=False)
 
@@ -739,6 +745,7 @@ def addMediaUrl(request, data):
 
     takes {
         url: string, // url
+        referer: string // optional referer url
         item: string // item
     }
     returns {
@@ -751,7 +758,7 @@ def addMediaUrl(request, data):
         response = json_response()
         i = Item.objects.get(public_id=data['item'])
         Task.start(i, request.user)
-        t = tasks.download_media.delay(data['item'], data['url'])
+        t = tasks.download_media.delay(data['item'], data['url'], data.get('referer'))
         response['data']['taskId'] = t.task_id
         add_changelog(request, data, data['item'])
     return render_to_json_response(response)

pandora/document/management/commands/rebuild_documentfind.py

@@ -5,7 +5,6 @@ from django.db import connection, transaction
 from django.db.models import fields
 from django.conf import settings
 
-settings.RELOAD_CONFIG = False
 import app.monkey_patch
 from ... import models
 

pandora/document/management/commands/sync_documentsort.py

@@ -5,7 +5,6 @@ from django.db import connection, transaction
 from django.db.models import fields
 from django.conf import settings
 
-settings.RELOAD_CONFIG = False
 import app.monkey_patch
 
 

pandora/document/tasks.py

@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+import ox
 from celery.task import task
 
 @task(queue="encoding")
@@ -6,3 +6,20 @@ def extract_fulltext(id):
     from . import models
     d = models.Document.objects.get(id=id)
     d.update_fulltext()
+
+
+@task(queue='default')
+def bulk_edit(data, username):
+    from django.db import transaction
+    from . import models
+    from item.models import Item
+    user = models.User.objects.get(username=username)
+    item = 'item' in data and Item.objects.get(public_id=data['item']) or None
+    documents = models.Document.objects.filter(pk__in=map(ox.fromAZ, data['id']))
+    for document in documents:
+        if document.editable(user, item):
+            with transaction.atomic():
+                document.refresh_from_db()
+                document.edit(data, user, item)
+                document.save()
+    return {}

pandora/document/views.py

@@ -23,6 +23,7 @@ from archive.chunk import process_chunk
 from changelog.models import add_changelog
 
 from . import models
+from . import tasks
 
 def get_document_or_404_json(request, id):
     response = {'status': {'code': 404,
@@ -131,13 +132,13 @@ def editDocument(request, data):
     item = 'item' in data and Item.objects.get(public_id=data['item']) or None
     if data['id']:
         if isinstance(data['id'], list):
-            documents = models.Document.objects.filter(pk__in=map(ox.fromAZ, data['id']))
+            add_changelog(request, data)
+            t = tasks.bulk_edit.delay(data, request.user.username)
+            response['data']['taskId'] = t.task_id
         else:
-            documents = [models.Document.get(data['id'])]
+            document = models.Document.get(data['id'])
-        for document in documents:
             if document.editable(request.user, item):
-                if document == documents[0]:
+                add_changelog(request, data)
-                    add_changelog(request, data)
                 document.edit(data, request.user, item)
                 document.save()
                 response['data'] = document.json(user=request.user, item=item)

pandora/item/management/commands/get_frame.py

@@ -4,7 +4,6 @@ from django.core.management.base import BaseCommand
 from django.conf import settings
 from django.db import transaction
 
-settings.RELOAD_CONFIG = False
 import app.monkey_patch
 from ... import models
 

pandora/item/management/commands/rebuild_filter.py

@@ -6,7 +6,6 @@ from django.db import connection, transaction
 from django.db.models import fields
 from django.conf import settings
 
-settings.RELOAD_CONFIG = False
 import app.monkey_patch
 from ... import models
 import clip.models

pandora/item/management/commands/rebuild_find.py

@@ -5,7 +5,6 @@ from django.db import connection, transaction
 from django.db.models import fields
 from django.conf import settings
 
-settings.RELOAD_CONFIG = False
 import app.monkey_patch
 from ... import models
 import clip.models

pandora/item/management/commands/rebuild_indexes.py

@@ -5,7 +5,6 @@ from django.db import connection, transaction
 from django.db.models import fields
 from django.conf import settings
 
-settings.RELOAD_CONFIG = False
 import app.monkey_patch
 from ... import models
 import clip.models

pandora/item/management/commands/rebuild_posters.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+
+import os
+from glob import glob
+
+from django.core.management.base import BaseCommand
+
+import app.monkey_patch
+from ... import models
+from ... import tasks
+
+class Command(BaseCommand):
+    """
+    rebuild posters for all items.
+    """
+    help = 'rebuild all posters for all items.'
+    args = ''
+
+    def handle(self, **options):
+        offset = 0
+        chunk = 100
+        count = models.Item.objects.count()
+        while offset <= count:
+            for i in models.Item.objects.all().order_by('id')[offset:offset+chunk]:
+                print(i)
+                if i.poster:
+                    i.poster.delete()
+                i.make_poster()
+            offset += chunk

pandora/item/management/commands/rebuild_sort.py

@@ -6,7 +6,6 @@ from django.db import connection, transaction
 from django.db.models import fields
 from django.conf import settings
 
-settings.RELOAD_CONFIG = False
 import app.monkey_patch
 from ... import models
 

pandora/item/management/commands/sqlfindindex.py

@@ -5,7 +5,6 @@ from django.core.management.base import BaseCommand
 from django.db import connection, transaction
 from django.conf import settings
 
-settings.RELOAD_CONFIG = False
 import app.monkey_patch
 from ... import models
 

pandora/item/management/commands/sync_itemsort.py

@@ -5,7 +5,6 @@ from django.db import connection, transaction
 from django.db.models import fields
 from django.conf import settings
 
-settings.RELOAD_CONFIG = False
 import app.monkey_patch
 from ... import models
 import clip.models

pandora/item/models.py

@@ -1022,7 +1022,7 @@ class Item(models.Model):
                 elif sort_type == 'string':
                     value = self.get(source, '')
                     if isinstance(value, list):
-                        value = ','.join(value)
+                        value = ','.join([str(v) for v in value])
                     value = utils.sort_string(value)[:955]
                     set_value(s, name, value)
                 elif sort_type == 'words':

pandora/item/tasks.py

@@ -5,6 +5,7 @@ from urllib.parse import quote
 import gzip
 import os
 import random
+import logging
 
 from celery.task import task, periodic_task
 from django.conf import settings
@@ -16,6 +17,9 @@ from app.utils import limit_rate
 from taskqueue.models import Task
 
 
+logger = logging.getLogger(__name__)
+
+
 @periodic_task(run_every=timedelta(days=1), queue='encoding')
 def cronjob(**kwargs):
     if limit_rate('item.tasks.cronjob', 8 * 60 * 60):
@@ -350,3 +354,18 @@ def update_sitemap(base_url):
         f.write(data)
     with gzip.open(sitemap, 'wb') as f:
         f.write(data)
+
+
+@task(queue='default')
+def bulk_edit(data, username):
+    from django.db import transaction
+    from . import models
+    from .views import edit_item
+    user = models.User.objects.get(username=username)
+    items = models.Item.objects.filter(public_id__in=data['id'])
+    for item in items:
+        if item.editable(user):
+            with transaction.atomic():
+                item.refresh_from_db()
+                response = edit_item(user, item, data)
+    return {}

pandora/item/views.py

@@ -533,17 +533,18 @@ def get(request, data):
     return render_to_json_response(response)
 actions.register(get)
 
-def edit_item(request, item, data):
+def edit_item(user, item, data):
+    data = data.copy()
     update_clips = False
     response = json_response(status=200, text='ok')
     if 'rightslevel' in data:
-        if request.user.profile.capability('canEditRightsLevel'):
+        if user.profile.capability('canEditRightsLevel'):
             item.level = int(data['rightslevel'])
         else:
             response = json_response(status=403, text='permission denied')
         del data['rightslevel']
     if 'user' in data:
-        if request.user.profile.get_level() in ('admin', 'staff') and \
+        if user.profile.get_level() in ('admin', 'staff') and \
                 models.User.objects.filter(username=data['user']).exists():
             new_user = models.User.objects.get(username=data['user'])
             if new_user != item.user:
@@ -551,10 +552,10 @@ def edit_item(request, item, data):
                 update_clips = True
         del data['user']
     if 'groups' in data:
-        if not request.user.profile.capability('canManageUsers'):
+        if not user.profile.capability('canManageUsers'):
             # Users wihtout canManageUsers can only add/remove groups they are not in
             groups = set([g.name for g in item.groups.all()])
-            user_groups = set([g.name for g in request.user.groups.all()])
+            user_groups = set([g.name for g in user.groups.all()])
             other_groups = list(groups - user_groups)
             data['groups'] = [g for g in data['groups'] if g in user_groups] + other_groups
     r = item.edit(data)
@@ -597,7 +598,7 @@ def add(request, data):
             i.make_poster()
         del data['title']
         if data:
-            response = edit_item(request, item, data)
+            response = edit_item(request.user, item, data)
         response['data'] = item.json()
         add_changelog(request, request_data, item.public_id)
     return render_to_json_response(response)
@@ -619,16 +620,16 @@ def edit(request, data):
     see: add, find, get, lookup, remove, upload
     '''
     if isinstance(data['id'], list):
-        items = models.Item.objects.filter(public_id__in=data['id'])
+        add_changelog(request, data)
+        t = tasks.bulk_edit.delay(data, request.user.username)
+        response = json_response(status=200, text='ok')
+        response['data']['taskId'] = t.task_id
     else:
-        items = [get_object_or_404_json(models.Item, public_id=data['id'])]
+        item = get_object_or_404_json(models.Item, public_id=data['id'])
-    for item in items:
         if item.editable(request.user):
-            request_data = data.copy()
+            add_changelog(request, data)
-            response = edit_item(request, item, data)
+            response = edit_item(request.user, item, data)
             response['data'] = item.json()
-            if item == items[0]:
-                add_changelog(request, request_data)
         else:
             response = json_response(status=403, text='permission denied')
     return render_to_json_response(response)
@@ -1029,7 +1030,10 @@ def download(request, id, resolution=None, format='webm', part=None):
             return HttpResponseForbidden()
         elif r is True:
             response = HttpResponse(FileWrapper(video), content_type=content_type)
-            response['Content-Length'] = os.path.getsize(video.name)
+            try:
+                response['Content-Length'] = os.path.getsize(video.name)
+            except:
+                pass
         else:
             response = HttpFileResponse(r, content_type=content_type)
     else:

pandora/manage.py

@@ -73,5 +73,3 @@ if __name__ == "__main__":
         sys.stderr.write("Error: Can't find '%s'.\nBefore you run pan.do/ra you must create it\n" % settings.SITE_CONFIG)
         sys.exit(1)
     execute_from_command_line(sys.argv)
-    import app.config
-    app.config.shutdown()

pandora/settings.py

@@ -220,7 +220,6 @@ XACCELREDIRECT = False
 
 SITE_CONFIG = join(PROJECT_ROOT, 'config.jsonc')
 DEFAULT_CONFIG = join(PROJECT_ROOT, 'config.pandora.jsonc')
-RELOAD_CONFIG = False
 
 #used if CONFIG['canDownloadVideo'] is set
 TRACKER_URL = "udp://tracker.openbittorrent.com:80"
@@ -275,7 +274,6 @@ SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
 
 DATA_UPLOAD_MAX_MEMORY_SIZE = 32 * 1024 * 1024
 
-RELOADER_RUNNING = False
 #you can ignore things below this line
 #=========================================================================
 LOCAL_APPS = []

pandora/user/utils.py

@@ -22,7 +22,7 @@ def get_location(ip):
     else:
         try:
             location = g.city(ip)
-        except GeoIP2Exception:
+        except:
             try:
                 location = g.country(s.ip)
             except:

pandora/websocket/worker.py

@@ -24,7 +24,7 @@ class Worker(ConsumerMixin):
 
     def process_task(self, body, message):
         try:
-            if body['task'] == 'trigger_event':
+            if isinstance(body, dict) and body.get('task') == 'trigger_event':
                 daemon.trigger_event(*body['args'])
         except:
             logger.error('faild to trigger event %s', body, exc_info=True)

requirements.txt

@@ -1,14 +1,15 @@
-Django==3.0.6
+Django==3.0.10
 simplejson
 chardet
-celery>4.3.0
+celery<5.0,>4.3
 django-celery-results
 django-extensions==2.2.9
 gunicorn==20.0.4
 html5lib
-requests==2.23.0
+requests<3.0.0,>=2.24.0
+urllib3<2.0.0,>=1.25.2
 tornado<5
-geoip2==3.0.0
+geoip2==4.1.0
 youtube-dl>=2020.5.8
 python-memcached
 elasticsearch

static/js/addFilesDialog.js

@@ -106,7 +106,7 @@ pandora.ui.addFilesDialog = function(options) {
     });
 
     var selectItems = [];
-    if (!pandora.site.itemRequiresVideo && pandora.user.ui.item) {
+    if (pandora.user.ui.item && options.editable) {
         selectItems.push({
             id: 'add',
             title: Ox._(
@@ -114,31 +114,23 @@ pandora.ui.addFilesDialog = function(options) {
                 [pandora.site.itemName.singular.toLowerCase()]
             )
         });
+    }
+    if (options.items.length > 1) {
         selectItems.push({
-            id: 'one',
+            id: 'multiple',
             title: Ox._(
-                options.items.length > 1 ? 'Create new {0} with multiple parts' : 'Create new {0}',
+                'Create multiple {0}',
-                [pandora.site.itemName.singular.toLowerCase()]
+                [pandora.site.itemName.plural.toLowerCase()]
-            )
-        });
-    } else {
-        if (options.items.length > 1) {
-            selectItems.push({
-                id: 'multiple',
-                title: Ox._(
-                    'Create multiple {0}',
-                    [pandora.site.itemName.plural.toLowerCase()]
-                )
-            });
-        }
-        selectItems.push({
-            id: 'one',
-            title: Ox._(
-                'Create one {0} with multiple parts',
-                [pandora.site.itemName.singular.toLowerCase()]
             )
         });
     }
+    selectItems.push({
+        id: 'one',
+        title: Ox._(
+            options.items.length > 1 ? 'Create new {0} with multiple parts' : 'Create new {0}',
+            [pandora.site.itemName.singular.toLowerCase()]
+        )
+    });
     var $select = Ox.Select({
         items: selectItems,
         width: 256
@@ -224,6 +216,7 @@ pandora.ui.addFilesDialog = function(options) {
                     ), function(result) {
                         pandora.api.addMediaUrl({
                             url: item.url,
+                            referer: item.referer,
                             item: id
                         }, callback);
                     });

static/js/addItemDialog.js

@@ -222,7 +222,8 @@ pandora.ui.addItemDialog = function(options) {
             }
             return value;
         });
-        values.url= info.url;
+        values.url = info.url;
+        values.referer = info.referer;
         return Ox.extend(info, values);
     }
 
@@ -270,10 +271,16 @@ pandora.ui.addItemDialog = function(options) {
             } else {
                 $screen.stop();
                 that.close();
-                pandora.ui.addFilesDialog({
+                (pandora.user.ui.item ? pandora.api.get : Ox.noop)({
-                    action: selected,
+                    id: pandora.user.ui.item,
-                    items: items
+                    keys: ['editable']
-                }).open();
+                }, function(result) {
+                    pandora.ui.addFilesDialog({
+                        action: selected,
+                        items: items,
+                        editable: pandora.user.ui.item && result.data.editable
+                    }).open();
+                })
             }
         })
     }

static/js/addRemoveKeyDialog.js

@@ -0,0 +1,124 @@
+
+pandora.ui.addRemoveKeyDialog = function(options) {
+    var item = Ox.getObjectById(pandora.site.documentKeys, options.key),
+        itemTitle = Ox._(item ? item.title : options.key),
+        dialogTitle = Ox._('Add/Remove {0}', [itemTitle]);
+
+    return Ox.Element('<span>')
+        .addClass('OxLight')
+        .html(' [+]')
+        .css({
+            cursor: 'pointer',
+        })
+        .options({
+            tooltip: dialogTitle
+        })
+        .on('click', function(event) {
+            var add, remove,
+                dialog = Ox.Dialog({
+                buttons: [
+                    Ox.Button({
+                            title: Ox._('Cancel')
+                        })
+                        .css({margin: '4px 4px 4px 0'})
+                        .bindEvent({
+                            click: function() {
+                                dialog.close()
+                            }
+                        }),
+                    Ox.Button({
+                            title: Ox._('Update'),
+                            width: 52
+                        }).bindEvent({
+                            click: function() {
+                                var addValue = add.value()
+                                var removeValue = remove.value()
+                                if (!addValue.length && !removeValue.length) {
+                                    dialog.close()
+                                    return
+                                }
+                                var $loading = Ox.LoadingScreen({
+                                    width: 256,
+                                    height: 64
+                                })
+                                dialog.options({
+                                    content: $loading.start()
+                                })
+                                var ids = options.ids;
+                                pandora.api[{
+                                    'items': 'find',
+                                    'documents': 'findDocuments',
+                                }[options.section]]({
+                                    query: {
+                                        conditions: [{key: 'id', operator: '&', value: ids}]
+                                    },
+                                    range: [0, ids.length],
+                                    keys: ['id', options.key]
+
+                                }, function(result) {
+                                    Ox.serialForEach(result.data.items,
+                                        function(item, index, items, callback) {
+                                            var changed = false
+                                            var value = item[options.key] || []
+                                            if (addValue.length && !Ox.contains(value, addValue)) {
+                                                value.push(addValue)
+                                                changed = true
+                                            }
+                                            if (removeValue.length && Ox.contains(value, removeValue)) {
+                                                value = value.filter(function(v) { return v != removeValue; } )
+                                                changed = true
+                                            }
+                                            if (changed) {
+                                                var edit = {id: item.id}
+                                                edit[options.key] = value
+                                                pandora.api[{
+                                                    'items': 'edit',
+                                                    'documents': 'editDocument',
+                                                }[options.section]](edit, function(response) {
+                                                    callback()
+                                                })
+                                            } else {
+                                                callback()
+                                            }
+                                        },
+                                        function() {
+                                            $loading.stop()
+                                            dialog.close()
+                                        }
+                                    )
+                                })
+                            }
+                        })
+                ],
+                closeButton: true,
+                content: Ox.Element()
+                    .css({
+                        padding: '8px'
+                    })
+                    .append(
+                        add = Ox.Input({
+                            width: 240,
+                            type: 'input',
+                            label: 'Add'
+                        })
+                    )
+                    .append(
+                        Ox.Element().html('').css({
+                            height: '8px'
+                        })
+                    )
+                    .append(
+                        remove = Ox.Input({
+                            width: 240,
+                            type: 'input',
+                            label: 'Remove'
+                        })
+                    ),
+                height: 64,
+                removeOnClose: true,
+                title: dialogTitle,
+                width: 256
+            }).open();
+        })
+
+}

static/js/collection.js

@@ -13,20 +13,6 @@ pandora.ui.collection = function() {
 
     if (view == 'list') {
         that = Ox.TableList({
-            draggable: true,
-            keys: keys,
-            items: function(data, callback) {
-                pandora.api.findDocuments(Ox.extend(data, {
-                    query: ui.findDocuments
-                }), callback);
-                return Ox.clone(data, true);
-            },
-            selected: ui.collectionSelection,
-            sort: ui.collectionSort.concat([
-                {key: 'extension', operator: '+'},
-                {key: 'title', operator: '+'}
-            ]),
-            unique: 'id',
             columns: pandora.site.documentSortKeys.filter(function(key) {
                 return !key.capability
                     || pandora.hasCapability(key.capability);
@@ -40,7 +26,11 @@ pandora.ui.collection = function() {
                     defaultWidth: key.columnWidth,
                     format: (function() {
                         return function(value, data) {
-                            return pandora.formatDocumentKey(key, data);
+                            var value = pandora.formatDocumentKey(key, data);
+                            if (Ox.isArray(value)) {
+                                value = value.join(', ');
+                            }
+                            return value;
                         }
                     })(),
                     id: key.id,
@@ -54,7 +44,25 @@ pandora.ui.collection = function() {
                 };
             }),
             columnsVisible: true,
+            columnsMovable: true,
+            columnsRemovable: true,
+            columnsResizable: true,
+            columnsVisible: true,
+            draggable: true,
+            items: function(data, callback) {
+                pandora.api.findDocuments(Ox.extend(data, {
+                    query: ui.findDocuments
+                }), callback);
+                return Ox.clone(data, true);
+            },
+            keys: keys,
             scrollbarVisible: true,
+            selected: ui.collectionSelection,
+            sort: ui.collectionSort.concat([
+                {key: 'extension', operator: '+'},
+                {key: 'title', operator: '+'}
+            ]),
+            unique: 'id',
         })
         .bindEvent({
             columnchange: function(data) {

static/js/documentInfoView.js

@@ -218,6 +218,7 @@ pandora.ui.documentInfoView = function(data, isMixed) {
         .append(
             Ox.EditableContent({
                     editable: canEdit,
+                    placeholder: formatLight(Ox._('No Title')),
                     tooltip: canEdit ? pandora.getEditTooltip() : '',
                     value: data.title || ''
                 })
@@ -571,6 +572,13 @@ pandora.ui.documentInfoView = function(data, isMixed) {
                             }
                         })
                         .appendTo($element);
+                    if (isMixed[key] && Ox.contains(listKeys, key)) {
+                        pandora.ui.addRemoveKeyDialog({
+                            ids: ui.collectionSelection,
+                            key: key,
+                            section: ui.section
+                        }).appendTo($element)
+                    }
                 }
             });
             $element.appendTo($text);

static/js/documentsPanel.js

@@ -810,9 +810,8 @@ pandora.ui.documentsPanel = function(options) {
             item: function(data, sort, size) {
                 var sortKey = sort[0].key,
                     infoKey = sortKey == 'title' ? 'extension' : sortKey,
-                    info = (
+                    key = Ox.getObjectById(pandora.site.documentKeys, infoKey),
-                        Ox.getObjectById(pandora.site.documentKeys, infoKey).format || Ox.identity
+                    info = pandora.formatDocumentKey(key, data, size),
-                    )(data[infoKey]),
                     size = size || 128;
                 return {
                     height: Math.round(data.ratio > 1 ? size / data.ratio : size),

static/js/editDialog.js

@@ -4,14 +4,12 @@ pandora.ui.editDialog = function() {
 
     var ui = pandora.user.ui,
         hasChanged = false,
-        ids = ui.listSelection.filter(function(id) {
+        ids = ui.listSelection,
-            return pandora.$ui.list.value(id, 'editable');
+        keys = ['editable'].concat(pandora.site.itemKeys.filter(function(key) {
-        }),
-        keys = pandora.site.itemKeys.filter(function(key) {
             return key.id != '*'
         }).map(function(key) {
             return key.id
-        }),
+        })),
         listKeys = pandora.site.itemKeys.filter(function(key) {
             return Ox.isArray(key.type);
         }).map(function(key){
@@ -86,26 +84,51 @@ pandora.ui.editDialog = function() {
                     }
                 ],
                 operator: '&'
-            }
+            },
+            range: [0, ids.length]
         }, function(result) {
             var data = {},
                 isMixed = {},
-                items = result.data.items;
+                updateTitle = false,
-            keys.forEach(function(key) {
+                items = result.data.items.filter(function(item) {
+                    if (!item.editable) {
+                        updateTitle = true
+                    }
+                    return item.editable;
+                });
+
+            if (updateTitle) {
+                that.options({
+                    title: Ox._('Edit Metadata for {0}', [
+                        Ox.formatNumber(items.length) + ' ' + Ox._(
+                            items.length == 1 ? pandora.site.itemName.singular : pandora.site.itemName.plural
+                        )
+                    ])
+                })
+                // no editable items
+                if (!items.length) {
+                    that.close()
+                    return
+                }
+            }
+            keys.filter(function(key) {
+                return key != 'editable'
+            }).forEach(function(key) {
                 var isArray = Ox.contains(listKeys, key),
                     values = items.map(function(item) {
                         return item[key];
                     });
                 if (isArray) {
                     values = values.map(function(value) {
-                        return (value || []).join(separator);
+                        value = value || []
+                        return value.join ? value.join(separator) : value;
                     });
                 }
                 if (Ox.unique(values).length > 1) {
                     isMixed[key] = true;
                 }
                 data[key] = isMixed[key] ? null
-                    : isArray ? values[0].split(separator)
+                    : isArray && values.length ? values[0].split(separator)
                     : values[0];
             });
             that.options({

static/js/editDocumentsDialog.js

@@ -3,14 +3,12 @@
 pandora.ui.editDocumentsDialog = function() {
     var ui = pandora.user.ui,
         hasChanged = false,
-        ids = ui.collectionSelection.filter(function(id) {
+        ids = ui.collectionSelection,
-            return pandora.$ui.list.value(id, 'editable');
+        keys = ['editable'].concat(pandora.site.documentKeys.filter(function(key) {
-        }),
-        keys = pandora.site.documentKeys.filter(function(key) {
             return key.id != '*'
         }).map(function(key) {
             return key.id
-        }),
+        })),
         listKeys = pandora.site.documentKeys.filter(function(key) {
             return Ox.isArray(key.type);
         }).map(function(key){
@@ -85,12 +83,36 @@ pandora.ui.editDocumentsDialog = function() {
                     }
                 ],
                 operator: '&'
-            }
+            },
+            range: [0, ids.length]
         }, function(result) {
             var data = {},
                 isMixed = {},
-                items = result.data.items;
+                updateTitle = false,
-            keys.forEach(function(key) {
+                items = result.data.items.filter(function(item) {
+                    if (!item.editable) {
+                        updateTitle = true
+                    }
+                    return item.editable;
+                });
+            if (updateTitle) {
+                that.options({
+                    title: Ox._('Edit Metadata for {0}', [
+                        Ox.formatNumber(items.length) + ' ' + Ox._(
+                            items.length == 1 ? 'Document' : 'Documents'
+                        )
+                    ])
+                })
+                // no editable items
+                if (!items.length) {
+                    that.close()
+                    return
+                }
+            }
+
+            keys.filter(function(key) {
+                return key != 'editable'
+            }).forEach(function(key) {
                 var isArray = Ox.contains(listKeys, key),
                     values = items.map(function(item) {
                         return item[key];

static/js/editor.js

@@ -205,9 +205,9 @@ pandora.ui.editor = function(data) {
                 }).open();
             },
             editannotation: function(data) {
-                Ox.Log('', 'editAnnotation', data);
+                Ox.Log('', 'editAnnotation', data.id, data);
                 function callback(result) {
-                    Ox.Log('', 'editAnnotation result', result);
+                    Ox.Log('', 'editAnnotation result', result.data.id, result);
                     if (!Ox.isEmpty(result.data)) {
                         result.data.date = Ox.formatDate(
                             result.data.modified.slice(0, 10), '%B %e, %Y'
@@ -222,21 +222,56 @@ pandora.ui.editor = function(data) {
                     pandora.UI.set('videoPoints.' + ui.item + '.annotation', result.data.id.split('/')[1] || '');
                     Ox.Request.clearCache();
                 };
+                var edit = {
+                    'in': data['in'],
+                    out: data.out,
+                    value: data.value
+                }
                 if (data.id[0] == '_') {
-                    pandora.api.addAnnotation({
+                    edit.item = ui.item;
-                        'in': data['in'],
+                    edit.layer = data.layer;
-                        item: ui.item,
+
-                        layer: data.layer,
+                    if (queue[data.id]) {
-                        out: data.out,
+                        queue[data.id].push(edit);
-                        value: data.value
+                    } else {
-                    }, callback);
+                        queue[data.id] = [];
+                        pandora.api.addAnnotation(edit, function(result) {
+                            callback(result);
+                            var id = result.data.id,
+                                pending = queue[id];
+                            delete queue[id];
+                            pending && pending.length && Ox.serialForEach(pending, function(edit, index, array, callback) {
+                                edit.id = id
+                                Ox.Log('', 'process pending editAnnotation request', id, edit);
+                                pandora.api.editAnnotation(edit, function(result) {
+                                    callback();
+                                })
+                            }, function() {
+                                Ox.Request.clearCache();
+                            });
+                        });
+                    }
                 } else {
-                    pandora.api.editAnnotation({
+                    edit.id = data.id;
-                        id: data.id,
+                    if (queue[data.id]) {
-                        'in': data['in'],
+                        queue[data.id].push(edit);
-                        out: data.out,
+                    } else {
-                        value: data.value
+                        queue[data.id] = [];
-                    }, callback);
+                        pandora.api.editAnnotation(edit, function(result) {
+                            callback(result);
+                            var pending = queue[edit.id];
+                            delete queue[edit.id];
+                            pending && pending.length && Ox.serialForEach(pending, function(edit, index, array, cb) {
+                                Ox.Log('', 'process pending editAnnotation request', edit.id, edit);
+                                pandora.api.editAnnotation(edit, function(result) {
+                                    callback(result);
+                                    cb();
+                                })
+                            }, function() {
+                                Ox.Request.clearCache();
+                            });
+                        });
+                    }
                 }
             },
             embedselection: function() {
@@ -387,7 +422,8 @@ pandora.ui.editor = function(data) {
             pandora_videotimeline: function(data) {
                 that.options({timeline: data.value});
             }
-        });
+        }),
+        queue = [];
 
     pandora._dontSelectResult = false;
 

static/js/importMediaDialog.js

@@ -132,6 +132,7 @@ pandora.ui.importMediaDialog = function(options) {
                 pandora.api.edit(edit, function(result) {
                     pandora.api.addMediaUrl({
                         url: info.url,
+                        referer: info.referer,
                         item: edit.id
                     }, function(result) {
                         if (result.data.taskId) {

static/js/infoView.js

@@ -198,9 +198,7 @@ pandora.ui.infoView = function(data, isMixed) {
                 top: margin + 'px',
                 right: margin + 'px'
             })
-            .appendTo($info),
+            .appendTo($info);
-
-        $capabilities;
 
     [$options, $edit].forEach(function($element) {
         $element.find('input').css({
@@ -346,7 +344,7 @@ pandora.ui.infoView = function(data, isMixed) {
         .append(formatKey('Rights Level', 'statistics'))
         .append($rightsLevel)
         .appendTo($statistics);
-    renderRightsLevel();
+    pandora.renderRightsLevel(that, $rightsLevel, data, isMixed, isMultiple, canEdit);
 
     // Notes --------------------------------------------------------------------
 
@@ -496,15 +494,6 @@ pandora.ui.infoView = function(data, isMixed) {
         return ret;
     }
 
-    function getRightsLevelElement(rightsLevel) {
-        return Ox.Theme.formatColorLevel(
-            rightsLevel,
-            pandora.site.rightsLevels.map(function(rightsLevel) {
-                return rightsLevel.name;
-            })
-        );
-    }
-
     function getValue(key, value) {
         return !value ? ''
             : Ox.contains(specialListKeys, key) ? value.join('; ')
@@ -512,81 +501,6 @@ pandora.ui.infoView = function(data, isMixed) {
             : value;
     }
 
-    function renderCapabilities(rightsLevel) {
-        var capabilities = [].concat(
-                canEdit ? [{name: 'canSeeItem', symbol: 'Find'}] : [],
-                [
-                    {name: 'canPlayClips', symbol: 'PlayInToOut'},
-                    {name: 'canPlayVideo', symbol: 'Play'},
-                    {name: 'canDownloadVideo', symbol: 'Download'}
-                ]
-            ),
-            userLevels = canEdit ? pandora.site.userLevels : [pandora.user.level];
-        $capabilities.empty();
-        userLevels.forEach(function(userLevel, i) {
-            var $element,
-                $line = $('<div>')
-                    .css({
-                        height: '16px',
-                        marginBottom: '4px'
-                    })
-                    .appendTo($capabilities);
-            if (canEdit) {
-                $element = Ox.Theme.formatColorLevel(i, userLevels.map(function(userLevel) {
-                    return Ox.toTitleCase(userLevel);
-                }), [0, 240]);
-                Ox.Label({
-                        textAlign: 'center',
-                        title: Ox.toTitleCase(userLevel),
-                        width: 60
-                    })
-                    .addClass('OxColor OxColorGradient')
-                    .css({
-                        float: 'left',
-                        height: '12px',
-                        paddingTop: '2px',
-                        background: $element.css('background'),
-                        fontSize: '8px',
-                        color: $element.css('color')
-                    })
-                    .data({OxColor: $element.data('OxColor')})
-                    .appendTo($line);
-            }
-            capabilities.forEach(function(capability) {
-                var hasCapability = pandora.hasCapability(capability.name, userLevel) >= rightsLevel,
-                    $element = Ox.Theme.formatColorLevel(hasCapability, ['', '']);
-                Ox.Button({
-                        tooltip: (canEdit ? Ox.toTitleCase(userLevel) : 'You') + ' '
-                            + (hasCapability ? 'can' : 'can\'t') + ' '
-                            + Ox.toSlashes(capability.name)
-                                .split('/').slice(1).join(' ')
-                                .toLowerCase(),
-                        title: capability.symbol,
-                        type: 'image'
-                    })
-                    .addClass('OxColor OxColorGradient')
-                    .css({background: $element.css('background')})
-                    .css('margin' + (canEdit ? 'Left' : 'Right'), '4px')
-                    .data({OxColor: $element.data('OxColor')})
-                    .appendTo($line);
-            });
-            if (!canEdit) {
-                Ox.Button({
-                    title: Ox._('Help'),
-                    tooltip: Ox._('About Rights'),
-                    type: 'image'
-                })
-                .css({marginLeft: '52px'})
-                .bindEvent({
-                    click: function() {
-                        pandora.UI.set({page: 'rights'});
-                    }
-                })
-                .appendTo($line);
-            }
-        });
-    }
-
     function renderGroup(keys) {
         var $element;
         keys.forEach(function(key) { displayedKeys.push(key) });
@@ -615,6 +529,13 @@ pandora.ui.infoView = function(data, isMixed) {
                             }
                         })
                         .appendTo($element);
+                    if (isMixed[key] && Ox.contains(listKeys, key)) {
+                        pandora.ui.addRemoveKeyDialog({
+                            ids: ui.listSelection,
+                            key: key,
+                            section: ui.section
+                        }).appendTo($element)
+                    }
                 }
             });
             $element.appendTo($text);
@@ -632,53 +553,6 @@ pandora.ui.infoView = function(data, isMixed) {
         }
     }
 
-    function renderRightsLevel() {
-        var $rightsLevelElement = getRightsLevelElement(data.rightslevel),
-            $rightsLevelSelect;
-        $rightsLevel.empty();
-        if (canEdit) {
-            $rightsLevelSelect = Ox.Select({
-                    items: pandora.site.rightsLevels.map(function(rightsLevel, i) {
-                        return {id: i, title: rightsLevel.name};
-                    }),
-                    width: 128,
-                    value: data.rightslevel
-                })
-                .addClass('OxColor OxColorGradient')
-                .css({
-                    marginBottom: '4px',
-                    background: $rightsLevelElement.css('background')
-                })
-                .data({OxColor: $rightsLevelElement.data('OxColor')})
-                .bindEvent({
-                    change: function(event) {
-                        var rightsLevel = event.value;
-                        $rightsLevelElement = getRightsLevelElement(rightsLevel);
-                        $rightsLevelSelect
-                            .css({background: $rightsLevelElement.css('background')})
-                            .data({OxColor: $rightsLevelElement.data('OxColor')})
-                        renderCapabilities(rightsLevel);
-                        var edit = {
-                            id: isMultiple ? ui.listSelection : data.id,
-                            rightslevel: rightsLevel
-                        };
-                        pandora.api.edit(edit, function(result) {
-                            that.triggerEvent('change', Ox.extend({}, 'rightslevel', rightsLevel));
-                        });
-                    }
-                })
-                .appendTo($rightsLevel);
-        } else {
-            $rightsLevelElement
-                .css({
-                    marginBottom: '4px'
-                })
-                .appendTo($rightsLevel);
-        }
-        $capabilities = $('<div>').appendTo($rightsLevel);
-        renderCapabilities(data.rightslevel);
-    }
-
     function toggleIconSize() {
         iconSize = iconSize == 256 ? 512 : 256;
         iconWidth = iconRatio > 1 ? iconSize : Math.round(iconSize * iconRatio);

static/js/infoView.padma.js

@@ -44,6 +44,12 @@ pandora.ui.infoView = function(data, isMixed) {
 
         $options = Ox.MenuButton({
                 items: [
+                    {
+                        id: 'toggle',
+                        title: Ox._('Toggle {0} size...', [
+                            Ox._(ui.icons == 'posters' ? 'poster' : 'icon')
+                        ]),
+                    },
                     {
                         id: 'delete',
                         title: Ox._('Delete {0}...', [pandora.site.itemName.singular]),
@@ -62,7 +68,9 @@ pandora.ui.infoView = function(data, isMixed) {
             })
             .bindEvent({
                 click: function(data_) {
-                    if (data_.id == 'delete') {
+                    if (data_.id == 'toggle') {
+                        toggleIconSize()
+                    } else if (data_.id == 'delete') {
                         pandora.$ui.deleteItemsDialog = pandora.ui.deleteItemsDialog({
                             items: [data]
                         }).open();
@@ -204,9 +212,7 @@ pandora.ui.infoView = function(data, isMixed) {
                 top: margin + 'px',
                 right: margin + 'px'
             })
-            .appendTo($info),
+            .appendTo($info);
-
-        $capabilities;
 
     [$options, $edit].forEach(function($element) {
         $element.find('input').css({
@@ -437,7 +443,7 @@ pandora.ui.infoView = function(data, isMixed) {
         .append(formatKey(Ox._('Rights Level'), 'statistics'))
         .append($rightsLevel)
         .appendTo($statistics);
-    renderRightsLevel();
+    pandora.renderRightsLevel(that, $rightsLevel, data, isMixed, isMultiple, canEdit);
 
     // User and Groups ---------------------------------------------------------
     if (!isMultiple) {
@@ -628,100 +634,12 @@ pandora.ui.infoView = function(data, isMixed) {
         return formatLink(key, ret, key == 'date' && value);
     }
 
-    function getRightsLevelElement(rightsLevel) {
-        return Ox.Theme.formatColorLevel(
-            rightsLevel,
-            pandora.site.rightsLevels.map(function(rightsLevel) {
-                return rightsLevel.name;
-            })
-        );
-    }
-
     function getValue(key, value) {
         return !value ? ''
             : Ox.contains(listKeys, key) ? value.join(', ')
             : value;
     }
 
-    function renderCapabilities(rightsLevel) {
-        var capabilities = [].concat(
-                canEdit ? [{name: 'canSeeItem', symbol: 'Find'}] : [],
-                [
-                    {name: 'canPlayClips', symbol: 'PlayInToOut'},
-                    {name: 'canPlayVideo', symbol: 'Play'},
-                    {name: 'canDownloadVideo', symbol: 'Download'}
-                ]
-            ),
-            userLevels = canEdit ? pandora.site.userLevels : [pandora.user.level];
-        $capabilities.empty();
-        userLevels.forEach(function(userLevel, i) {
-            var $element,
-                $line = $('<div>')
-                    .css({
-                        height: '16px',
-                        marginBottom: '4px'
-                    })
-                    .appendTo($capabilities);
-            if (canEdit) {
-                $element = Ox.Theme.formatColorLevel(i, userLevels.map(function(userLevel) {
-                    return Ox.toTitleCase(userLevel);
-                }), [0, 240]);
-                Ox.Label({
-                        textAlign: 'center',
-                        title: Ox._(Ox.toTitleCase(userLevel)),
-                        width: 60
-                    })
-                    .addClass('OxColor OxColorGradient')
-                    .css({
-                        float: 'left',
-                        height: '12px',
-                        paddingTop: '2px',
-                        background: $element.css('background'),
-                        fontSize: '8px',
-                        color: $element.css('color')
-                    })
-                    .data({OxColor: $element.data('OxColor')})
-                    .appendTo($line);
-            }
-            capabilities.forEach(function(capability) {
-                var hasCapability = pandora.hasCapability(capability.name, userLevel) >= rightsLevel,
-                    $element = Ox.Theme.formatColorLevel(hasCapability, ['', '']);
-                Ox.Button({
-                        tooltip: Ox._('{0} '
-                            + (hasCapability ? 'can' : 'can\'t') + ' '
-                            + Ox.toSlashes(capability.name)
-                                .split('/').slice(1).join(' ')
-                                .toLowerCase()
-                                .replace('see item', 'see the item')
-                                .replace('play video', 'play the full video')
-                                .replace('download video', 'download the video'),
-                            [canEdit ? Ox._(Ox.toTitleCase(userLevel)) : Ox._('You')]),
-                        title: capability.symbol,
-                        type: 'image'
-                    })
-                    .addClass('OxColor OxColorGradient')
-                    .css({background: $element.css('background')})
-                    .css('margin' + (canEdit ? 'Left' : 'Right'), '4px')
-                    .data({OxColor: $element.data('OxColor')})
-                    .appendTo($line);
-            });
-            if (!canEdit) {
-                Ox.Button({
-                    title: Ox._('Help'),
-                    tooltip: Ox._('About Rights'),
-                    type: 'image'
-                })
-                .css({marginLeft: '52px'})
-                .bindEvent({
-                    click: function() {
-                        pandora.UI.set({page: 'rights'});
-                    }
-                })
-                .appendTo($line);
-            }
-        });
-    }
-
     function renderGroup(keys) {
         var $element;
         if (canEdit || keys.filter(function(key) {
@@ -757,53 +675,6 @@ pandora.ui.infoView = function(data, isMixed) {
         }
     }
 
-    function renderRightsLevel() {
-        var $rightsLevelElement = getRightsLevelElement(data.rightslevel),
-            $rightsLevelSelect;
-        $rightsLevel.empty();
-        if (canEdit) {
-            $rightsLevelSelect = Ox.Select({
-                    items: pandora.site.rightsLevels.map(function(rightsLevel, i) {
-                        return {id: i, title: Ox._(rightsLevel.name)};
-                    }),
-                    width: 128,
-                    value: data.rightslevel
-                })
-                .addClass('OxColor OxColorGradient')
-                .css({
-                    marginBottom: '4px',
-                    background: $rightsLevelElement.css('background')
-                })
-                .data({OxColor: $rightsLevelElement.data('OxColor')})
-                .bindEvent({
-                    change: function(event) {
-                        var rightsLevel = event.value;
-                        $rightsLevelElement = getRightsLevelElement(rightsLevel);
-                        $rightsLevelSelect
-                            .css({background: $rightsLevelElement.css('background')})
-                            .data({OxColor: $rightsLevelElement.data('OxColor')})
-                        renderCapabilities(rightsLevel);
-                        var edit = {
-                            id: isMultiple ? ui.listSelection : data.id,
-                            rightslevel: rightsLevel
-                        };
-                        pandora.api.edit(edit, function(result) {
-                            that.triggerEvent('change', Ox.extend({}, 'rightslevel', rightsLevel));
-                        });
-                    }
-                })
-                .appendTo($rightsLevel);
-        } else {
-            $rightsLevelElement
-                .css({
-                    marginBottom: '4px'
-                })
-                .appendTo($rightsLevel);
-        }
-        $capabilities = $('<div>').appendTo($rightsLevel);
-        renderCapabilities(data.rightslevel);
-    }
-
     function toggleIconSize() {
         iconSize = iconSize == 256 ? 512 : 256;
         iconWidth = iconRatio > 1 ? iconSize : Math.round(iconSize * iconRatio);

static/js/infoViewUtils.js

@@ -14,3 +14,145 @@ pandora.cleanupDate = function(value) {
     return value
 };
 
+pandora.renderRightsLevel = function(that, $rightsLevel, data, isMixed, isMultiple, canEdit) {
+    var ui = pandora.user.ui,
+        rightsLevels = pandora.site.rightsLevels.map(function(rightsLevel) {
+                return rightsLevel.name;
+            }).concat(isMultiple ? ['Mixed'] : []),
+        rightsLevel = isMixed.rightslevel ? rightsLevels.length - 1 : data.rightslevel,
+        $capabilities,
+        $rightsLevelElement = getRightsLevelElement(rightsLevel),
+        $rightsLevelSelect;
+    $rightsLevel.empty();
+    if (canEdit) {
+        $rightsLevelSelect = Ox.Select({
+                items: pandora.site.rightsLevels.map(function(rightsLevel, i) {
+                    return {id: i, title: Ox._(rightsLevel.name)};
+                }).concat(isMultiple ? [
+                    {id: rightsLevels.length - 1, title: Ox._('Mixed'), disabled: true}
+                ] : []),
+                width: 128,
+                value: rightsLevel
+            })
+            .addClass('OxColor OxColorGradient')
+            .css({
+                marginBottom: '4px',
+                background: $rightsLevelElement.css('background')
+            })
+            .data({OxColor: $rightsLevelElement.data('OxColor')})
+            .bindEvent({
+                change: function(event) {
+                    var rightsLevel = event.value;
+                    $rightsLevelElement = getRightsLevelElement(rightsLevel);
+                    $rightsLevelSelect
+                        .css({background: $rightsLevelElement.css('background')})
+                        .data({OxColor: $rightsLevelElement.data('OxColor')})
+                    if (rightsLevel < pandora.site.rightsLevels.length) {
+                        renderCapabilities(rightsLevel);
+                        var edit = {
+                            id: isMultiple ? ui.listSelection : data.id,
+                            rightslevel: rightsLevel
+                        };
+                        pandora.api.edit(edit, function(result) {
+                            that.triggerEvent('change', Ox.extend({}, 'rightslevel', rightsLevel));
+                        });
+                    }
+                }
+            })
+            .appendTo($rightsLevel);
+    } else {
+        $rightsLevelElement
+            .css({
+                marginBottom: '4px'
+            })
+            .appendTo($rightsLevel);
+    }
+    $capabilities = $('<div>').appendTo($rightsLevel);
+    !isMixed.rightslevel && renderCapabilities(data.rightslevel);
+
+
+    function getRightsLevelElement(rightsLevel) {
+        return Ox.Theme.formatColorLevel(rightsLevel, rightsLevels)
+    }
+
+    function renderCapabilities(rightsLevel) {
+        var capabilities = [].concat(
+                canEdit ? [{name: 'canSeeItem', symbol: 'Find'}] : [],
+                [
+                    {name: 'canPlayClips', symbol: 'PlayInToOut'},
+                    {name: 'canPlayVideo', symbol: 'Play'},
+                    {name: 'canDownloadVideo', symbol: 'Download'}
+                ]
+            ),
+            userLevels = canEdit ? pandora.site.userLevels : [pandora.user.level];
+        $capabilities.empty();
+        userLevels.forEach(function(userLevel, i) {
+            var $element,
+                $line = $('<div>')
+                    .css({
+                        height: '16px',
+                        marginBottom: '4px'
+                    })
+                    .appendTo($capabilities);
+            if (canEdit) {
+                $element = Ox.Theme.formatColorLevel(i, userLevels.map(function(userLevel) {
+                    return Ox.toTitleCase(userLevel);
+                }), [0, 240]);
+                Ox.Label({
+                        textAlign: 'center',
+                        title: Ox._(Ox.toTitleCase(userLevel)),
+                        width: 60
+                    })
+                    .addClass('OxColor OxColorGradient')
+                    .css({
+                        float: 'left',
+                        height: '12px',
+                        paddingTop: '2px',
+                        background: $element.css('background'),
+                        fontSize: '8px',
+                        color: $element.css('color')
+                    })
+                    .data({OxColor: $element.data('OxColor')})
+                    .appendTo($line);
+            }
+            capabilities.forEach(function(capability) {
+                var hasCapability = pandora.hasCapability(capability.name, userLevel) >= rightsLevel,
+                    $element = Ox.Theme.formatColorLevel(hasCapability, ['', '']);
+                Ox.Button({
+                        tooltip: Ox._('{0} '
+                            + (hasCapability ? 'can' : 'can\'t') + ' '
+                            + Ox.toSlashes(capability.name)
+                                .split('/').slice(1).join(' ')
+                                .toLowerCase()
+                                .replace('see item', 'see the item')
+                                .replace('play video', 'play the full video')
+                                .replace('download video', 'download the video'),
+                            [canEdit ? Ox._(Ox.toTitleCase(userLevel)) : Ox._('You')]),
+                        title: capability.symbol,
+                        type: 'image'
+                    })
+                    .addClass('OxColor OxColorGradient')
+                    .css({background: $element.css('background')})
+                    .css('margin' + (canEdit ? 'Left' : 'Right'), '4px')
+                    .data({OxColor: $element.data('OxColor')})
+                    .appendTo($line);
+            });
+            if (!canEdit) {
+                Ox.Button({
+                    title: Ox._('Help'),
+                    tooltip: Ox._('About Rights'),
+                    type: 'image'
+                })
+                .css({marginLeft: '52px'})
+                .bindEvent({
+                    click: function() {
+                        pandora.UI.set({page: 'rights'});
+                    }
+                })
+                .appendTo($line);
+            }
+        });
+    }
+
+}
+

static/js/mainMenu.js

@@ -442,7 +442,37 @@ pandora.ui.mainMenu = function() {
                         });
                     }
                 } else if (data.id == 'deletefromarchive') {
-                    if (ui.section == 'documents') {
+                    if (ui.section == 'items') {
+                        var ids;
+                        if (ui.item) {
+                            ids = [ui.item]
+                        } else {
+                            ids = ui.listSelection
+                        }
+                        pandora.api.find({
+                            query: {
+                                conditions: [{
+                                    key: 'id',
+                                    operator: '&',
+                                    value: ids
+                                }],
+                                operator: '&'
+                            },
+                            keys: ['id', 'title'],
+                            range: [0, ui.listSelection.length]
+                        }, function(result) {
+                            pandora.$ui.deleteItemsDialog = pandora.ui.deleteItemsDialog({
+                                items: result.data.items
+                            }, function() {
+                                Ox.Request.clearCache();
+                                if (ui.item) {
+                                    pandora.UI.set({item: ''});
+                                } else {
+                                    pandora.$ui.list.reloadList()
+                                }
+                            }).open();
+                        });
+                    } else if (ui.section == 'documents') {
                         var files;
                         if (ui.document) {
                             files = [pandora.$ui.document.info()];
@@ -1364,6 +1394,9 @@ pandora.ui.mainMenu = function() {
             { id: 'clearclipboard', title: Ox._('Clear Clipboard'), disabled: !clipboardItems},
             {},
             { id: 'delete', title: Ox._('{0} {1} {2}', [deleteVerb, selectionItemName, listName]), disabled: !canDelete, keyboard: 'delete' },
+            ui._list ? [
+                { id: 'deletefromarchive', title: Ox._('{0} {1} {2}', [Ox._('Delete'), selectionItemName, Ox._('from Archive')]), disabled: !canDelete }
+            ] : [],
             {},
             { id: 'undo', title: undoText ? Ox._('Undo {0}', [undoText]) : Ox._('Undo'), disabled: !undoText, keyboard: 'control z' },
             { id: 'redo', title: redoText ? Ox._('Redo {0}', [redoText]) : Ox._('Redo'), disabled: !redoText, keyboard: 'shift control z' },

static/js/mediaExistsDialog.js

@@ -58,10 +58,16 @@ pandora.ui.mediaExistsDialog = function(options) {
                         return existing.indexOf(item.oshash) == -1;
                     });
                 that.close();
-                pandora.ui.addFilesDialog({
+                (pandora.user.ui.item ? pandora.api.get : Ox.noop)({
-                    action: options.action,
+                    id: pandora.user.ui.item,
-                    items: items
+                    keys: ['editable']
-                }).open();
+                }, function(result) {
+                    pandora.ui.addFilesDialog({
+                        action: options.action,
+                        items: items,
+                        editable: pandora.user.ui.item && result.data.editable
+                    }).open();
+                })
             }
         })
     ];

static/js/utils.js

@@ -375,7 +375,11 @@ pandora.clickLink = function(e, selectEmbed) {
     if (match) {
         (selectEmbed || pandora.$ui.textPanel.selectEmbed)(parseInt(match[1]));
     } else {
-        pandora.openURL(e.target.href);
+        if (e.target.target == '_blank') {
+            pandora.openLink(e.target.href);
+        } else {
+            pandora.openURL(e.target.href);
+        }
     }
 };
 
@@ -725,11 +729,11 @@ pandora.uploadDroppedFiles = function(files) {
 pandora.enableBatchEdit = function(section) {
     var ui = pandora.user.ui;
     if (section == 'documents') {
-        return !ui.document && ui.collectionSelection.length > 1 && ui.collectionSelection.every(function(item) {
+        return !ui.document && ui.collectionSelection.length > 1 && ui.collectionSelection.some(function(item) {
             return pandora.$ui.list && pandora.$ui.list.value(item, 'editable');
         })
     } else {
-        return !ui.item && ui.listSelection.length > 1 && ui.listSelection.every(function(item) {
+        return !ui.item && ui.listSelection.length > 1 && ui.listSelection.some(function(item) {
             return pandora.$ui.list && pandora.$ui.list.value(item, 'editable');
         })
     }
@@ -2655,8 +2659,9 @@ pandora.openURL = function(url) {
 };
 
 pandora.safeDocumentName = function(name) {
-    ['?', '#', '%'].forEach(function(c) {
+    ['\\?', '#', '%', '/'].forEach(function(c) {
-        name = name.replace(c, '_');
+        var r = new RegExp(c, 'g')
+        name = name.replace(r, '_');
     })
     return name;
 };

vm/pandora_install.sh

@@ -89,6 +89,7 @@ apt-get install -y \
     python3-pyinotify \
     python3-simplejson \
     python3-lxml \
+    python3-cssselect \
     python3-html5lib \
     python3-ox \
     python3-elasticsearch \
@@ -118,6 +119,7 @@ fi
 if [ "$RABBITMQ" == "local" ]; then
     RABBITPWD=$(pwgen -n 16 -1)
     rabbitmqctl add_user pandora $RABBITPWD
+    rabbitmqctl change_password pandora $RABBITPWD
     rabbitmqctl add_vhost /pandora
     rabbitmqctl set_permissions -p /pandora pandora ".*" ".*" ".*"
     CELERY_BROKER_URL="amqp://pandora:$RABBITPWD@localhost:5672//pandora"
@@ -185,7 +187,7 @@ fi
 
 # if pandora is running inside a container, expose backend at port 2620
 if [ "$LXC" == "yes" ]; then
-    sed -i s/127.0.0.1/0.0.0.0/g /srv/pandora/pandora/gunicorn_config.py
+    sed -i "s/127.0.0.1/[::]/g" /srv/pandora/pandora/gunicorn_config.py
     echo "WEBSOCKET_ADDRESS = \"0.0.0.0\"" >> /srv/pandora/pandora/local_settings.py
 fi
 /srv/pandora/ctl start
@@ -199,7 +201,6 @@ if [ "$NGINX" == "local" ]; then
 cp "/srv/pandora/etc/nginx/pandora" "/etc/nginx/sites-available/pandora"
 rm -f /etc/nginx/sites-enabled/default /etc/nginx/sites-enabled/pandora
 ln -s ../sites-available/pandora /etc/nginx/sites-enabled/pandora
-ln -s /srv/pandora/ctl /usr/local/bin/pandoractl
 
 read -r -d '' GZIP <<EOI
 gzip_static  on;\\