avoid async, fails in python3.7

.dockerignore

@@ -0,0 +1,3 @@
+.env
+data/
+overlay/

.gitignore

@@ -29,3 +29,5 @@ static/django_extensions
 *.swp
 pandora/gunicorn_config.py
 .DS_Store
+.env
+overlay/

Dockerfile

@@ -0,0 +1,11 @@
+FROM 0x2620/pandora-base:latest
+
+LABEL maintainer="0x2620@0x2620.org"
+
+ENV LANG en_US.UTF-8
+
+#VOLUME /pandora
+COPY . /srv/pandora
+RUN /srv/pandora/docker/install.sh
+
+ENTRYPOINT [ "/entrypoint.sh" ]

README.md

@@ -2,177 +2,51 @@
 
   for more information on pan.do/ra visit our website at https://pan.do/ra
 
-## SETUP
+## Installing pan.do/ra
 
-  pan.do/ra is known to work with Ubuntu 16.04,
-  but other distributions might also work.
+  we recommend to run pan.do/ra inside of LXD or LXC or dedicated VM or server.
+  You will need at least 2GB of free disk space
 
-  The instructions below are for Ubuntu 16.04.
-  All command given expect that you are root.
+  pan.do/ra is known to work with Ubuntu 18.04 and Debian/10 (buster),
+  other distributions might also work, let us know if it works for you.
 
-  To run pan.do/ra you need to install and setup:
+  Use the following commands as root to install pan.do/ra and all dependencies:
 
-    python 3.5
-    postgres
-    nginx (or apache2)
-    additional video packages
+```
+cd /root
+curl -sL https://pan.do/ra-install > pandora_install.sh
+chmod +x pandora_install.sh
+./pandora_install.sh 2>&1 | tee pandora_install.log
+```
+
+ For step by step installation, look at [pandora_install.sh](vm/pandora_install.sh)
 
 
-## Installing required packages
+## Configuration
 
-1) add pandora ppa to get all packages in the required version
+  pan.do/ra is mostly configured in two places:
 
-    apt-get install software-properties-common
-    add-apt-repository ppa:j/pandora
-    apt-get update
+### /srv/pandora/pandora/local_settings.py
 
-2) install all required packages
-
-    apt-get install git \
-            python3-setuptools python3-pip python3-venv ipython3 \
-            python3-dev python3-pil python3-numpy python3-psycopg2 \
-            python3-pyinotify python3-simplejson \
-            python3-geoip python3-html5lib python3-lxml \
-            postgresql postgresql-contrib rabbitmq-server \
-            poppler-utils mkvtoolnix gpac imagemagick \
-            youtube-dl python3-ox oxframe ffmpeg
+  this file contains local Django configuration overwrites,
+  like database configuration, email backend and more.
 
 
-## Prepare Environment
+### /srv/pandora/pandora/config.jsonc
 
-1) add pandora user and set permissions
+  config.jsonc can be used in configure the pan.do/ra related
+  settings. From title to item keys to video resolutions.
 
-    adduser pandora --disabled-login --disabled-password
-
-2) Setup Database
-
-    su postgres
-    createuser pandora
-    createdb  -T template0 --locale=C --encoding=UTF8 -O pandora pandora
-    echo "CREATE EXTENSION pg_trgm;" | psql pandora
-    exit
-
-3) Setup RabbitMQ
-
-  You have to use the same password here and in BROKER_URL in local_settings.py
-
-    rabbitmqctl add_user pandora PASSWORD
-    rabbitmqctl add_vhost /pandora
-    rabbitmqctl set_permissions -p /pandora pandora ".*" ".*" ".*"
+  More info at
+  https://code.0x2620.org/0x2620/pandora/wiki/Configuration
 
 
-## Install Pan.do/ra
+## Customization
 
-1) Get code from git
+  pan.do/ra can be customized, this is mostly done by adding
+  JavaScript files that replace or enhance parts of pan.do/ra
 
-    cd /srv/
-    git clone https://git.0x2620.org/pandora.git pandora
-    cd pandora
-    ./ctl init
+  More info at
+  https://code.0x2620.org/0x2620/pandora/wiki/Customization
 
-    cd /srv
-    chown -R pandora.pandora pandora
 
-2) create local_settings.py and config.jsonc
-
-2.1) create file /srv/pandora/pandora/local_settings.py with the following content:
-
-    DATABASES = {
-        'default': {
-            'NAME': 'pandora',
-            'ENGINE': 'django.db.backends.postgresql_psycopg2',
-            'USER': 'pandora',
-            'PASSWORD': '',
-        }
-    }
-    DB_GIN_TRGM = True
-    BROKER_URL = 'amqp://pandora:PASSWORD@localhost:5672//pandora'
-
-    #with apache x-sendfile or lighttpd set this to True
-    XSENDFILE = False
-
-    #with nginx X-Accel-Redirect set this to True
-    XACCELREDIRECT = True
-
-2.2) create config.jsonc
-
-  config.jsonc holds the configuration for your site.
-  To start you can copy /srv/pandora/pandora/config.pandora.jsonc
-  to /srv/pandora/pandora/config.jsonc but have a look at 
-  https://wiki.0x2620.org/wiki/pandora/configuration and
-  config.0xdb.jsonc config.padma.jsonc for configuration options.
-
-3) initialize database
-
-    su pandora
-    cd /srv/pandora/pandora
-    ./manage.py init_db
-
-4) install init scripts and start daemons
-
-    /srv/pandora/ctl install
-    /srv/pandora/ctl start
-
-5) Setup Webserver
-a) nginx (recommended)
-
-    apt-get install nginx
-    cp /srv/pandora/etc/nginx/pandora /etc/nginx/sites-available/pandora
-    cd /etc/nginx/sites-enabled
-    ln -s ../sites-available/pandora
-
-    #read comments in /etc/nginx/sites-available/pandora for setting
-    #your hostname and other required settings
-    #make sure XACCELREDIRECT = True in /srv/pandora/pandora/local_settings.py
-    
-    service nginx reload
-
-b) apache2 (if you need it for other sites on the same server)
-
-    apt-get install apache2-mpm-prefork libapache2-mod-xsendfile
-    a2enmod xsendfile
-    a2enmod proxy_http
-    a2enmod proxy_wstunnel
-    cp /srv/pandora/etc/apache2/pandora.conf /etc/apache2/sites-available/pandora.conf
-    a2ensite pandora
-
-    #read comments in /etc/apache2/sites-available/pandora.conf for setting
-    #your hostname and other required settings
-    #make sure XSENDFILE = True in /srv/pandora/pandora/local_settings.py
-    
-    service apache2 reload
-
-  Now you can open pandora in your browser, the first user to sign up will become admin.
-
-##  Updating
-
-  To update pandora to the latest development version run this:
-
-    su pandora
-    cd /srv/pandora
-    ./update.py dev
-
-  this will update pandora/oxjs/python-ox and list possible upgrades to the db
-
-  to update your database run:
-
-    su pandora
-    cd /srv/pandora
-    ./update.py db
-
-## Development
-
-  in one terminal:
-
-    cd /srv/pandora/pandora
-    ./manage.py runserver 2620
-
-  and background task in another:
-
-    cd /srv/pandora/pandora
-    ./manage.py celeryd -B -Q celery,default,encoding -l INFO
-
-  now you can access your local pandora instace at http://127.0.0.1:8000/
-
-  we use virtual machines/lxc for development and deployment,
-  more info on that in vm/LXC_README.md

docker-compose.yml

@@ -0,0 +1,91 @@
+version: '3'
+
+volumes:
+    pandora:
+    postgres:
+    rabbitmq:
+
+networks:
+    backend:
+
+services:
+    proxy:
+        build: docker/nginx
+        ports:
+            - "127.0.0.1:2620:80"
+        networks:
+            - backend
+            - default
+        links:
+            - pandora
+            - websocketd
+        depends_on:
+            - pandora
+            - websocketd
+        volumes:
+            - pandora:/pandora
+            - ./overlay:/overlay
+        restart: unless-stopped
+
+    db:
+        image: postgres:latest
+        networks:
+            - backend
+        env_file: .env
+        volumes:
+            - postgres:/var/lib/postgresql/data/
+        restart: unless-stopped
+
+    rabbitmq:
+        hostname: rabbitmq
+        image: rabbitmq:latest
+        env_file: .env
+        networks:
+            - backend
+        volumes:
+            - rabbitmq:/var/lib/rabbitmq
+        restart: unless-stopped
+
+    pandora:
+        hostname: pandora
+        build: .
+        command: pandora
+        volumes:
+            - pandora:/pandora
+            - ./overlay:/overlay
+        networks:
+            - backend
+        env_file: .env
+        links:
+            - db
+            - rabbitmq
+        depends_on:
+            - db
+            - rabbitmq
+        restart: unless-stopped
+
+    encoding: &app_base
+        build: .
+        command: encoding
+        env_file: .env
+        networks:
+            - backend
+        volumes:
+            - pandora:/pandora
+            - ./overlay:/overlay
+        restart: unless-stopped
+
+    tasks:
+        <<: *app_base
+        command: tasks
+
+    cron:
+        <<: *app_base
+        command: cron
+
+    websocketd:
+        <<: *app_base
+        command: websocketd
+        networks:
+            - backend
+

docker/base/Dockerfile

@@ -0,0 +1,14 @@
+FROM debian:buster
+
+LABEL maintainer="0x2620@0x2620.org"
+
+ENV LANG en_US.UTF-8
+
+RUN apt-get update && \
+    LANG=C.UTF-8 DEBIAN_FRONTEND=noninteractive apt-get install -y apt-utils locales && \
+    echo en_US.UTF-8 UTF-8 > /etc/locale.gen && \
+    locale-gen en_US.UTF-8 && \
+    update-locale LANG=en_US.UTF-8
+
+COPY ./install.sh /install.sh
+RUN /install.sh

docker/base/install.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+
+UBUNTU_CODENAME=bionic
+if [ -e /etc/os-release ]; then
+    . /etc/os-release
+fi
+
+export DEBIAN_FRONTEND=noninteractive
+echo 'Acquire::Languages "none";' > /etc/apt/apt.conf.d/99languages
+apt-get update -qq
+echo "deb http://ppa.launchpad.net/j/pandora/ubuntu ${UBUNTU_CODENAME} main" > /etc/apt/sources.list.d/j-pandora.list
+apt-get install -y gnupg2
+apt-key add - <<EOF
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v1
+
+mI0ESXYhEgEEALl9jDTdmgpApPbjN+7b85dC92HisPUp56ifEkKJOBj0X5HhRqxs
+Wjx/zlP4/XJGrHnxJyrdPxjSwAXz7bNdeggkN4JWdusTkr5GOXvggQnng0X7f/rX
+oJwoEGtYOCODLPs6PC0qjh5yPzJVeiRsKUOZ7YVNnwNwdfS4D8RZvtCrABEBAAG0
+FExhdW5jaHBhZCBQUEEgZm9yIGpeiLYEEwECACAFAkl2IRICGwMGCwkIBwMCBBUC
+CAMEFgIDAQIeAQIXgAAKCRAohRM8AZde82FfA/9OB/64/YLaCpizHZ8f6DK3rGgF
+e6mX3rFK8yOKGGL06316VhDzfzMiZSauUZ0t+lKHR/KZYeSaFwEoUoblTG/s4IIo
+9aBMHWhVXJW6eifKUmTGqEn2/0UxoWQq2C3F6njMkCaP+ALOD5uzaSYGdjqAUAwS
+pAAGSEQ4uz6bYSeM4Q==
+=SM2a
+-----END PGP PUBLIC KEY BLOCK-----
+EOF
+
+apt-get update -qq
+apt-get install -y \
+    netcat-openbsd \
+    sudo \
+    vim \
+    wget \
+    pwgen \
+    git \
+    python3-setuptools \
+    python3-pip \
+    python3-venv \
+    python3-dev \
+    python3-pil \
+    python3-numpy \
+    python3-psycopg2 \
+    python3-pyinotify \
+    python3-simplejson \
+    python3-lxml \
+    python3-cssselect \
+    python3-html5lib \
+    python3-ox \
+    oxframe \
+    ffmpeg \
+    mkvtoolnix \
+    gpac \
+    imagemagick \
+    poppler-utils \
+    youtube-dl \
+    ipython3 \
+    postfix \
+    postgresql-client
+
+apt-get clean
+
+rm -f /install.sh

docker/build.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+cd `dirname $0`
+
+HOST=`/sbin/ip route | grep docker0 | awk '{ print $9 }'`
+PORT=3142
+nc -z "$HOST" "$PORT" > /dev/null 2>&1
+result=$?
+if [ $result -eq 0 ] ; then
+    proxy="--build-arg http_proxy=http://$HOST:$PORT"
+else
+    proxy=
+fi
+
+docker build $proxy -t 0x2620/pandora-base base
+docker build -t 0x2620/pandora-nginx nginx
+cd ..
+docker build -t 0x2620/pandora .

docker/dot.env.sample.py

@@ -0,0 +1,30 @@
+#!/usr/bin/python3
+import os
+from binascii import hexlify
+import string
+
+chars = string.ascii_letters + string.digits
+
+def pwgen(length=16):
+    return ''.join(chars[c % len(chars)] for c in os.urandom(length))
+
+
+print('''SECRET_KEY={SECRET_KEY}
+
+POSTGRES_USER=pandora
+POSTGRES_PASSWORD={POSTGRES_PASSWORD}
+
+RABBITMQ_DEFAULT_USER=pandora
+RABBITMQ_DEFAULT_PASS={RABBITMQ_PASSWORD}
+
+# required to send out password reset emails
+#EMAIL_HOST='example.com'
+#EMAIL_USER='mail@example.com'
+#EMAIL_PASSWORD='fixme'
+#EMAIL_PORT=587
+#EMAIL_TLS=true
+'''.format(
+    SECRET_KEY=hexlify(os.urandom(64)).decode(),
+    POSTGRES_PASSWORD=pwgen(),
+    RABBITMQ_PASSWORD=pwgen()
+))

docker/entrypoint.sh

@@ -0,0 +1,134 @@
+#!/bin/bash
+set -e
+
+action="$1"
+user=pandora
+
+export LANG=en_US.UTF-8
+mkdir -p /run/pandora
+chown -R ${user}.${user} /run/pandora
+
+# pan.do/ra services
+if [ "$action" = "pandora" ]; then
+    if [ ! -e /srv/pandora/initialized ]; then
+        echo "Setting up Pan.do/ra:"
+        echo "Waiting for database connection..."
+        /srv/pandora_base/docker/wait-for db 5432
+        echo "Installing pan.do/ra..."
+        rsync -a /srv/pandora_base/ /srv/pandora/
+
+        if [ ! -e /overlay/install.py ]; then
+            rsync -a /srv/pandora_base/docker/overlay/ /overlay/
+            if [ ! -e /overlay/config.jsonc ]; then
+                mv /srv/pandora/pandora/config.jsonc /overlay/config.jsonc
+            fi
+        fi
+        /overlay/install.py
+
+        echo "Initializing database..."
+        echo "CREATE EXTENSION pg_trgm;" | /srv/pandora/pandora/manage.py dbshell
+        /srv/pandora/pandora/manage.py init_db
+        /srv/pandora/update.py db
+        echo "Generating static files..."
+        /srv/pandora/update.py static
+        chown -R ${user}.${user} /srv/pandora/
+        touch /srv/pandora/initialized
+    fi
+    /srv/pandora_base/docker/wait-for db 5432
+    /srv/pandora_base/docker/wait-for rabbitmq 5672
+    cd /srv/pandora/pandora
+    exec /usr/bin/sudo -u $user -E -H \
+        /srv/pandora/bin/gunicorn wsgi:application -c gunicorn_config.py
+fi
+if [ "$action" = "encoding" ]; then
+    /srv/pandora_base/docker/wait-for-file /srv/pandora/initialized
+    /srv/pandora_base/docker/wait-for rabbitmq 5672
+    name=pandora-encoding-$(hostname)
+    exec /usr/bin/sudo -u $user -E -H \
+        /srv/pandora/bin/python \
+        /srv/pandora/pandora/manage.py \
+        celery worker \
+        -c 1 \
+        -Q encoding -n $name \
+        -l INFO
+fi
+if [ "$action" = "tasks" ]; then
+    /srv/pandora_base/docker/wait-for-file /srv/pandora/initialized
+    /srv/pandora_base/docker/wait-for rabbitmq 5672
+    name=pandora-default-$(hostname)
+    exec /usr/bin/sudo -u $user -E -H \
+        /srv/pandora/bin/python \
+        /srv/pandora/pandora/manage.py \
+        celery worker \
+        -Q default,celery -n $name \
+        --maxtasksperchild 1000 \
+        -l INFO
+fi
+if [ "$action" = "cron" ]; then
+    /srv/pandora_base/docker/wait-for-file /srv/pandora/initialized
+    /srv/pandora_base/docker/wait-for rabbitmq 5672
+    exec /usr/bin/sudo -u $user -E -H \
+        /srv/pandora/bin/python \
+        /srv/pandora/pandora/manage.py \
+        celerybeat -s /run/pandora/celerybeat-schedule \
+        --pidfile /run/pandora/cron.pid \
+        -l INFO
+fi
+if [ "$action" = "websocketd" ]; then
+    /srv/pandora_base/docker/wait-for-file /srv/pandora/initialized
+    /srv/pandora_base/docker/wait-for rabbitmq 5672
+    exec /usr/bin/sudo -u $user -E -H \
+        /srv/pandora/bin/python \
+        /srv/pandora/pandora/manage.py websocketd
+fi
+
+# pan.do/ra management and update
+if [ "$action" = "manage.py" ]; then
+    shift
+    exec /usr/bin/sudo -u $user -E -H \
+        /srv/pandora/pandora/manage.py "$@"
+fi
+if [ "$action" = "update.py" ]; then
+    shift
+    exec /usr/bin/sudo -u $user -E -H \
+        /srv/pandora/update.py "$@"
+fi
+if [ "$action" = "bash" ]; then
+    shift
+    cd /
+    exec /bin/bash "$@"
+fi
+
+# pan.do/ra setup hooks
+if [ "$action" = "docker-compose.yml" ]; then
+    cat /srv/pandora_base/docker-compose.yml | \
+        sed "s#build: \.#image: 0x2620/pandora:latest#g" | \
+        sed "s#\./overlay:#.:#g" | \
+        sed "s#build: docker/nginx#image: 0x2620/pandora-nginx:latest#g"
+    exit
+fi
+if [ "$action" = ".env" ]; then
+    exec /srv/pandora_base/docker/dot.env.sample.py
+fi
+if [ "$action" = "config.jsonc" ]; then
+    cat /srv/pandora_base/pandora/config.pandora.jsonc
+    exit
+fi
+if [ "$action" = "setup" ]; then
+    cat /srv/pandora_base/docker/setup-docker-compose.sh
+    exit
+fi
+
+# pan.do/ra info
+echo pan.do/ra docker container - https://pan.do/ra
+echo
+echo use this container with docker-compose,
+echo to setup a new docker-compose envrionment run:
+echo 
+echo "  mkdir <sitename> && cd <sitename>"
+echo "  docker run 0x2620/pandora setup | sh"
+echo
+echo adjust created files to match your needs and run:
+echo
+echo "  docker-compose up"
+echo

docker/install.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+export LANG=en_US.UTF-8
+PANDORA=${PANDORA-pandora}
+echo Installing pandora with user: $PANDORA
+getent passwd $PANDORA > /dev/null 2>&1 || adduser --disabled-password --gecos "" $PANDORA
+
+HOST=$(hostname -s)
+HOST_CONFIG="/srv/pandora/pandora/config.$HOST.jsonc"
+SITE_CONFIG="/srv/pandora/pandora/config.jsonc"
+test -e $HOST_CONFIG && cp $HOST_CONFIG $SITE_CONFIG
+test -e $SITE_CONFIG || cp /srv/pandora/pandora/config.pandora.jsonc $SITE_CONFIG
+
+cat > /srv/pandora/pandora/local_settings.py <<EOF
+import os
+
+SECRET_KEY = os.environ.get('SECRET_KEY')
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.postgresql_psycopg2',
+        'NAME': 'postgres',
+        'USER': os.environ.get('POSTGRES_USER'),
+        'PASSWORD': os.environ.get('POSTGRES_PASSWORD'),
+        'HOST': 'db',
+        'PORT': 5432,
+    }
+}
+BROKER_URL = "amqp://{0}:{1}@rabbitmq:5672//".format(os.environ.get('RABBITMQ_DEFAULT_USER'), os.environ.get('RABBITMQ_DEFAULT_PASS'))
+XACCELREDIRECT = True
+
+DEBUG = False
+TEMPLATE_DEBUG = DEBUG
+JSON_DEBUG = False
+DB_GIN_TRGM = True
+WEBSOCKET_ADDRESS = "0.0.0.0"
+
+EMAIL_HOST=os.environ.get('EMAIL_HOST')
+EMAIL_HOST_USER=os.environ.get('EMAIL_USER')
+EMAIL_HOST_PASSWORD=os.environ.get('EMAIL_PASSWORD')
+EMAIL_PORT=int(os.environ.get('EMAIL_PORT', 587))
+EMAIL_USE_TLS=os.environ.get('EMAIL_TLS', 'true').lower() == 'true'
+
+overlay_settings = 'overlay_settings.py'
+if os.path.exists(overlay_settings):
+    from overlay_settings import *
+EOF
+
+cp /srv/pandora/pandora/gunicorn_config.py.in /srv/pandora/pandora/gunicorn_config.py
+sed -i s/127.0.0.1/0.0.0.0/g /srv/pandora/pandora/gunicorn_config.py
+
+chown -R $PANDORA:$PANDORA /srv/pandora
+
+cd /srv/pandora
+./ctl init
+cp /srv/pandora/docker/entrypoint.sh /entrypoint.sh
+
+mv /srv/pandora/ /srv/pandora_base/
+mkdir /pandora
+ln -s /pandora /srv/pandora
+cat > /usr/local/bin/update.py << EOF
+#!/bin/sh
+exec /srv/pandora/update.py \$@
+EOF
+
+cat > /usr/local/bin/manage.py << EOF
+#!/bin/sh
+exec /srv/pandora/pandora/manage.py \$@
+EOF
+chmod +x /usr/local/bin/manage.py /usr/local/bin/update.py

docker/nginx/Dockerfile

@@ -0,0 +1,4 @@
+FROM nginx
+LABEL maintainer="0x2620@0x2620.org"
+ENV LANG en_US.UTF-8
+COPY nginx.conf /etc/nginx/nginx.conf

docker/nginx/nginx.conf

@@ -0,0 +1,96 @@
+worker_processes 4;
+
+events { worker_connections 1024; }
+
+http {
+    include mime.types;
+
+    sendfile on;
+    tcp_nopush on;
+    tcp_nodelay on;
+    keepalive_timeout 65;
+    types_hash_max_size 2048;
+    server_tokens off;
+
+    charset utf-8;
+    charset_types text/plain text/css application/json text/json application/x-javascript text/xml application/xml application/xml+rss text/javascript application/javascript text/x-js;
+
+    gzip  on;
+    gzip_static  on;
+    gzip_http_version 1.1;
+    gzip_vary on;
+    gzip_comp_level 6;
+    gzip_proxied any;
+    gzip_types text/plain text/css application/json text/json application/x-javascript text/xml application/xml application/xml+rss text/javascript application/javascript text/x-js;
+    gzip_buffers 16 8k;
+    gzip_disable "MSIE [1-6]\.(?!.*SV1)";
+
+    upstream pandora-web {
+        server pandora:2620;
+    }
+
+    upstream pandora-websocket {
+        server websocketd:2622;
+    }
+
+    server {
+
+        listen   80 default;
+
+        access_log  off;
+        error_log   /var/log/nginx/error.log;
+
+        location /favicon.ico {
+            root   /pandora/static;
+        }
+
+        location /static/ {
+            root   /pandora;
+            autoindex off;
+        }
+        location /data/ {
+            internal;
+            root   /pandora;
+        }
+
+        location /api/ws/ {
+                proxy_http_version 1.1;
+                proxy_set_header Host $http_host;
+                proxy_set_header Upgrade $http_upgrade;
+                proxy_set_header Connection "upgrade";
+                proxy_set_header Proxy "";
+                proxy_redirect off;
+                proxy_buffering off;
+                proxy_read_timeout 999999999;
+                proxy_pass http://pandora-websocket/;
+        }
+
+        location / {
+                proxy_set_header X-Forwarded-For $remote_addr;
+                proxy_set_header X-Forwarded-Proto http;
+                proxy_set_header Host $http_host;
+                proxy_set_header Proxy "";
+                proxy_redirect off;
+                proxy_buffering off;
+                proxy_read_timeout 90;    #should be in sync with gunicorn timeout
+                proxy_connect_timeout 90; #should be in sync with gunicorn timeout
+                if (!-f $request_filename) {
+                    proxy_pass http://pandora-web;
+                    break;
+                }
+                client_max_body_size 32m;
+        }
+
+        error_page  400  /;
+        error_page  404  /404.html;
+        location  /404.html {
+            root   /pandora/static/html;
+        }
+
+        # redirect server error pages to the static page /50x.html
+        error_page   500 502 503 504  /50x.html;
+        location  /50x.html {
+            root   /pandora/static/html;
+        }
+    }
+}

docker/overlay/install.py

@@ -0,0 +1,91 @@
+#!/usr/bin/python3
+
+import os
+from os.path import join, abspath, basename, dirname
+
+# change this
+name = 'placeholder'
+overwrite = (
+    #('home', 'indiancinema'),
+    #('infoView', 'indiancinema'),
+)
+
+base = abspath(dirname(__file__))
+os.chdir(base)
+
+for root, folders, files in os.walk(join(base, 'static')):
+    for f in files:
+        src = join(root, f)
+        target = src.replace(base, '/srv/pandora')
+        rel_src = os.path.relpath(src, dirname(target))
+        if os.path.exists(target):
+            os.unlink(target)
+        os.symlink(rel_src, target)
+
+if overwrite:
+    os.chdir('/srv/pandora/static/js')
+    for filename, sitename in overwrite:
+        src = '%s.%s.js' % (filename, sitename)
+        target = '%s.%s.js' % (filename, name)
+        if os.path.exists(target):
+            os.unlink(target)
+        os.symlink(src, target)
+
+os.chdir(base)
+src = join(base, 'config.jsonc')
+target = '/srv/pandora/pandora/config.%s.jsonc' % name
+rel_src = os.path.relpath(src, dirname(target))
+if os.path.exists(target):
+    os.unlink(target)
+os.symlink(rel_src, target)
+t = '/srv/pandora/pandora/config.jsonc'
+if os.path.exists(t):
+    os.unlink(t)
+os.symlink(basename(target), t)
+
+for root, folders, files in os.walk(join(base, 'scripts')):
+    for f in files:
+        src = join(root, f)
+        target = src.replace(base, '/srv/pandora')
+        rel_src = os.path.relpath(src, dirname(target))
+        if os.path.exists(target):
+            os.unlink(target)
+        os.symlink(rel_src, target)
+        if f == 'poster.%s.py' % name:
+            t = os.path.join(dirname(target), 'poster.py')
+            if os.path.exists(t):
+                os.unlink(t)
+            os.symlink(f, os.path.join(dirname(target), t))
+
+if os.path.exists('settings.py'):
+    target = os.path.join('/srv/pandora/pandora/overlay_settings.py',)
+    rel_src = os.path.relpath(os.path.join(base, 'settings.py'), dirname(target))
+    if os.path.exists(target):
+        os.unlink(target)
+    os.symlink(rel_src, target)
+
+if os.path.exists('__init__.py'):
+    # make module available to pandora
+    target = os.path.join('/srv/pandora/pandora/', name)
+    rel_src = os.path.relpath(base, dirname(target))
+    if os.path.exists(target):
+        os.unlink(target)
+    os.symlink(rel_src, target)
+
+    # include module in local settings
+    local_settings_py = '/srv/pandora/pandora/local_settings.py'
+    with open(local_settings_py) as fd:
+        local_settings_changed = False
+        local_settings = fd.read()
+        if 'LOCAL_APPS' not in local_settings:
+            local_settings += '\nLOCAL_APPS = ["%s"]\n' % name
+            local_settings_changed = True
+        else:
+            apps = re.compile('(LOCAL_APPS.*?)\]', re.DOTALL).findall(local_settings)[0]
+            if name not in apps:
+                new_apps = apps.strip() + ',\n"%s"\n' % name
+                local_settings = local_settings.replace(apps, new_apps)
+                local_settings_changed = True
+    if local_settings_changed:
+        with open(local_settings_py, 'w') as fd:
+            fd.write(local_settings)

docker/overlay/settings.py

@@ -0,0 +1 @@
+# local settings go here

docker/setup-docker-compose.sh

@@ -0,0 +1,32 @@
+#!/bin/sh
+docker run 0x2620/pandora docker-compose.yml > docker-compose.yml
+if [ ! -e .env ]; then
+    docker run 0x2620/pandora .env > .env
+    echo .env >> .gitignore
+fi
+if [ ! -e config.jsonc ]; then
+    docker run 0x2620/pandora config.jsonc > config.jsonc
+fi
+cat > README.md << EOF
+pan.do/ra docker instance
+
+this folder was created with
+
+    docker run 0x2620/pandora setup | sh
+
+To start pan.do/ra adjust the files in this folder:
+
+ - add email configuration to .env
+ - adjust config.jsonc to customize pan.do/ra
+ - add local django settings to settings.py
+
+and to get started run this:
+
+    docker-compose up -d
+
+To update pan.do/ra run:
+
+    docker-compose run pandora update.py
+
+EOF
+touch __init__.py

docker/wait-for

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+TIMEOUT=15
+
+HOST="$1"
+PORT="$2"
+
+for i in `seq $TIMEOUT` ; do
+    nc -z "$HOST" "$PORT" > /dev/null 2>&1
+    result=$?
+    if [ $result -eq 0 ] ; then
+      exit 0
+    fi
+    sleep 1
+done
+echo "Failed to connect to database at $HOST:$PORT" >&2
+exit 1

docker/wait-for-file

@@ -0,0 +1,12 @@
+#!/bin/sh
+TIMEOUT=60
+TARGET="$1"
+
+for i in `seq $TIMEOUT` ; do
+    if [ -e "$TARGET" ]; then
+        exit 0
+    fi
+    sleep 1
+done
+echo "Giving up waiting for file $TARGET" >&2
+exit 1

pandora/archive/tasks.py

@@ -210,6 +210,7 @@ def download_media(item_id, url):
 def move_media(data, user):
     from changelog.models import add_changelog
     from item.models import get_item, Item
+    from annotation.models import Annotation
 
     user = models.User.objects.get(username=user)
 
@@ -228,12 +229,26 @@ def move_media(data, user):
         else:
             i = get_item({'imdbId': data['public_id']}, user=user)
     changed = [i.public_id]
+    old_item = None
     for f in models.File.objects.filter(oshash__in=data['ids']):
-        if f.item.id != i.public_id and f.editable(user):
+        if f.item.public_id != i.public_id and f.editable(user):
             if f.item.public_id not in changed:
                 changed.append(f.item.public_id)
+            old_item = f.item
             f.item = i
             f.save()
+
+    if old_item:
+        data['from'] = old_item.public_id
+
+    # If all files are moved to a new item, keep annotations
+    if old_item and old_item.files.count() == 0 and i.files.count() == len(data['ids']):
+        for a in old_item.annotations.all().order_by('id'):
+            a.item = i
+            a.set_public_id()
+            Annotation.objects.filter(id=a.id).update(item=i, public_id=a.public_id)
+        old_item.clips.all().update(item=i, sort=i.sort)
+
     for public_id in changed:
         c = Item.objects.get(public_id=public_id)
         if c.files.count() == 0 and settings.CONFIG['itemRequiresVideo']:

pandora/config.0xdb.jsonc

@@ -165,6 +165,7 @@
             "type": "string",
             "columnWidth": 120,
             //"format": {"type": "date", "args": ["%a, %b %e, %Y"]},
+            "filter": true,
             "sort": true
         },
         {
@@ -220,6 +221,7 @@
         },
         {
             "id": "size",
+            "filter": true,
             "operator": "-",
             "title": "Size",
             "type": "integer",
@@ -1230,11 +1232,19 @@
             },
             "document": "",
             "documents": {},
-            "documentView": "view",
+            "documentFiltersSize": 176,
             "documentSize": 256,
             "documentsSelection": {},
             "documentsSort": [{"key": "name", "operator": "+"}],
             "documentsView": "grid",
+            "documentView": "view",
+            "documentFilters": [
+                {"id": "author", "sort": [{"key": "items", "operator": "-"}]},
+                {"id": "place", "sort": [{"key": "items", "operator": "-"}]},
+                {"id": "date", "sort": [{"key": "name", "operator": "-"}]},
+                {"id": "publisher", "sort": [{"key": "items", "operator": "-"}]},
+                {"id": "language", "sort": [{"key": "items", "operator": "-"}]}
+            ],
             "edit": "",
             "edits": {},
             "editSelection": [],
@@ -1303,6 +1313,7 @@
             "showCalendarControls": true, // fixme: should be false
             "showClips": true,
             "showDocument": true,
+            "showDocumentFilters": false,
             "showFilters": true,
             "showIconBrowser": false,
             "showInfo": true,

pandora/config.indiancinema.jsonc

@@ -212,6 +212,7 @@
             "type": "string",
             "columnWidth": 120,
             //"format": {"type": "date", "args": ["%a, %b %e, %Y"]},
+            "filter": true,
             "sort": true
         },
         {
@@ -238,6 +239,14 @@
             "find": true,
             "sort": true
         },
+        {
+            "id": "keywords",
+            "title": "Keywords",
+            "type": ["string"],
+            "columnWidth": 128,
+            "filter": true,
+            "find": true
+        },
         {
             "id": "id",
             "operator": "+",
@@ -1709,7 +1718,15 @@
                 }
             },
             "documentView": "view",
+            "documentFilters": [
+                {"id": "author", "sort": [{"key": "items", "operator": "-"}]},
+                {"id": "place", "sort": [{"key": "items", "operator": "-"}]},
+                {"id": "date", "sort": [{"key": "name", "operator": "-"}]},
+                {"id": "publisher", "sort": [{"key": "items", "operator": "-"}]},
+                {"id": "language", "sort": [{"key": "items", "operator": "-"}]}
+            ],
             "documents": {},
+            "documentFiltersSize": 176,
             "documentSize": 256,
             "documentsSelection": {},
             "documentsSort": [{"key": "title", "operator": "+"}],
@@ -1782,6 +1799,7 @@
             "showCalendarControls": true, // fixme: should be false
             "showClips": true,
             "showDocument": true,
+            "showDocumentFilters": false,
             "showFilters": true,
             "showIconBrowser": false,
             "showInfo": true,

pandora/config.padma.jsonc

@@ -210,6 +210,7 @@
             "type": "string",
             "columnWidth": 120,
             //"format": {"type": "date", "args": ["%a, %b %e, %Y"]},
+            "filter": true,
             "sort": true
         },
         {
@@ -236,6 +237,14 @@
             "find": true,
             "sort": true
         },
+        {
+            "id": "keywords",
+            "title": "Keywords",
+            "type": ["string"],
+            "columnWidth": 128,
+            "filter": true,
+            "find": true
+        },
         {
             "id": "id",
             "operator": "+",
@@ -1175,11 +1184,19 @@
             },
             "document": "",
             "documents": {},
+            "documentFiltersSize": 176,
             "documentSize": 256,
             "documentView": "view",
             "documentsSelection": {},
             "documentsSort": [{"key": "title", "operator": "+"}],
             "documentsView": "grid",
+            "documentFilters": [
+                {"id": "author", "sort": [{"key": "items", "operator": "-"}]},
+                {"id": "place", "sort": [{"key": "items", "operator": "-"}]},
+                {"id": "date", "sort": [{"key": "name", "operator": "-"}]},
+                {"id": "publisher", "sort": [{"key": "items", "operator": "-"}]},
+                {"id": "language", "sort": [{"key": "items", "operator": "-"}]}
+            ],
             "edit": "",
             "edits": {},
             "editSelection": [],
@@ -1244,6 +1261,7 @@
             "showCalendarControls": true, // fixme: should be false
             "showClips": true,
             "showDocument": true,
+            "showDocumentFilters": false,
             "showFilters": true,
             "showIconBrowser": false,
             "showInfo": true,

pandora/config.pandora.jsonc

@@ -217,6 +217,7 @@ examples (config.SITENAME.jsonc) that are part of this pan.do/ra distribution.
             "type": "string",
             "columnWidth": 120,
             //"format": {"type": "date", "args": ["%a, %b %e, %Y"]},
+            "filter": true,
             "sort": true
         },
         {
@@ -1122,11 +1123,19 @@ examples (config.SITENAME.jsonc) that are part of this pan.do/ra distribution.
             },
             "document": "",
             "documents": {},
+            "documentFiltersSize": 176,
             "documentSize": 256,
             "documentView": "view",
             "documentsSelection": {},
             "documentsSort": [{"key": "title", "operator": "+"}],
             "documentsView": "grid",
+            "documentFilters": [
+                {"id": "author", "sort": [{"key": "items", "operator": "-"}]},
+                {"id": "place", "sort": [{"key": "items", "operator": "-"}]},
+                {"id": "date", "sort": [{"key": "name", "operator": "-"}]},
+                {"id": "publisher", "sort": [{"key": "items", "operator": "-"}]},
+                {"id": "language", "sort": [{"key": "items", "operator": "-"}]}
+            ],
             "edit": "",
             "edits": {},
             "editSelection": [],
@@ -1191,6 +1200,7 @@ examples (config.SITENAME.jsonc) that are part of this pan.do/ra distribution.
             "showCalendarControls": false,
             "showClips": true,
             "showDocument": true,
+            "showDocumentFilters": false,
             "showFilters": true,
             "showIconBrowser": false,
             "showInfo": true,

pandora/document/models.py

@@ -192,7 +192,7 @@ class Document(models.Model):
                 sort_value = u'; '.join([get_name_sort(name) for name in values])
             if not sort_value:
                 sort_value = u''
-            return sort_value
+            return sort_value.lower()
 
         def set_value(s, name, value):
             if isinstance(value, string_types):
@@ -403,6 +403,13 @@ class Document(models.Model):
             'rightslevel',
         ):
             return getattr(self, key)
+        document_key = utils.get_by_id(settings.CONFIG['documentKeys'], key)
+        if document_key and 'value' in document_key \
+                and isinstance(document_key['value'], dict) \
+                and document_key['value'].get('type') == 'map' \
+                and self.get_value(document_key['value']['key']):
+            value = re.compile(document_key['value']['map']).findall(self.get_value(document_key['value']['key']))
+            return value[0] if value else default
         elif key == 'user':
             return self.user.username
         else:

pandora/document/utils.py

@@ -26,6 +26,7 @@ def extract_pdfpage(pdf, image, page):
     page = str(page)
     cmd = [
         'pdftocairo',
+        '-cropbox',
         '-jpeg',
         '-f', page, '-l', page,
         '-singlefile',

pandora/documentcollection/migrations/0004_jsonfield.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.4 on 2018-06-19 17:23
+from __future__ import unicode_literals
+
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations, models
+
+def remove_posterframes(apps, schema_editor):
+    Collection = apps.get_model("documentcollection", "Collection")
+    for c in Collection.objects.exclude(poster_frames=None):
+        c.poster_frames = []
+        c.save()
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('documentcollection', '0003_jsonfield'),
+    ]
+
+    operations = [
+        migrations.RunPython(remove_posterframes),
+    ]

pandora/documentcollection/models.py

@@ -265,38 +265,23 @@ class Collection(models.Model):
 
     def update_icon(self):
         frames = []
-        #fixme
-        '''
         if not self.poster_frames:
-            documents = self.get_documents(self.user)
+            documents = self.get_documents(self.user).all()
             if documents.count():
                 poster_frames = []
                 for i in range(0, documents.count(), max(1, int(documents.count()/4))):
                     poster_frames.append({
-                        'document': documents[int(i)].id,
-                        'position': documents[int(i)].poster_frame
+                        'document': documents[int(i)].get_id(),
                     })
                 self.poster_frames = tuple(poster_frames)
                 self.save()
         for i in self.poster_frames:
             from document.models import Document
-            qs = Document.objects.filter(id=i['document'])
-            if qs.count() > 0:
-                if i.get('position'):
-                    frame = qs[0].frame(i['position'])
-                    if frame:
-                        frames.append(frame)
-        '''
-        from item.models import Item
-        for i in self.poster_frames:
-            try:
-                qs = Item.objects.filter(public_id=i['item'])
+            if 'document' in i:
+                qs = Document.objects.filter(id=ox.fromAZ(i['document']))
                 if qs.count() > 0:
-                    frame = qs[0].frame(i['position'])
-                    if frame:
-                        frames.append(frame)
-            except:
-                pass
+                    frame = qs[0].thumbnail(size=1024, page=i.get('page'))
+                    frames.append(frame)
         self.icon.name = self.path('icon.jpg')
         icon = self.icon.path
         if frames:

pandora/event/models.py

@@ -129,7 +129,7 @@ class Event(models.Model):
                 value = get_name_sort(value)
             else:
                 value = get_title_sort(value)
-        self.name_sort = utils.sort_string(value)
+        self.name_sort = utils.sort_string(value).lower()
 
     def save(self, *args, **kwargs):
         if not self.name_sort:

pandora/item/models.py

@@ -478,6 +478,11 @@ class Item(models.Model):
             a.item = other
             a.set_public_id()
             Annotation.objects.filter(id=a.id).update(item=other, public_id=a.public_id)
+        try:
+            other_sort = other.sort
+        except:
+            other_sort = None
+        self.clips.all().update(item=other, sort=other_sort)
 
         if hasattr(self, 'files'):
             for f in self.files.all():
@@ -487,6 +492,8 @@ class Item(models.Model):
         if save:
             other.save()
             # FIXME: update poster, stills and streams after this
+            if other_sort is None:
+                other.clips.all().update(sort=other.sort)
 
     def merge_streams(self, output, resolution=None, format="webm"):
         streams = [s.get(resolution, format).media.path for s in self.streams()]
@@ -902,7 +909,7 @@ class Item(models.Model):
                 sort_value = u'; '.join([get_name_sort(name) for name in values])
             if not sort_value:
                 sort_value = u''
-            return sort_value
+            return sort_value.lower()
 
         def set_value(s, name, value):
             if isinstance(value, string_types):
@@ -1046,8 +1053,7 @@ class Item(models.Model):
                                 pass
                             else:
                                 continue
-                    if value_ is not None:
-                        set_value(s, name, value_)
+                    set_value(s, name, value_)
         s.save()
 
     def update_facet(self, key):
@@ -1355,7 +1361,7 @@ class Item(models.Model):
         qs = qs.order_by('file__part', 'file__sort_path')
         return qs
 
-    def update_timeline(self, async=True):
+    def update_timeline(self, async_=True):
         streams = self.streams()
         self.make_timeline()
         if streams.count() == 1:
@@ -1385,7 +1391,7 @@ class Item(models.Model):
         self.rendered = streams.count() > 0
         self.save()
         if self.rendered:
-            if async:
+            if async_:
                 get_sequences.delay(self.public_id)
             else:
                 get_sequences(self.public_id)

pandora/item/tasks.py

@@ -102,7 +102,7 @@ def update_timeline(public_id):
         item = models.Item.objects.get(public_id=public_id)
     except models.Item.DoesNotExist:
         return
-    item.update_timeline(async=False)
+    item.update_timeline(async_=False)
     Task.finish(item)
 
 @task(queue="encoding")
@@ -111,7 +111,7 @@ def rebuild_timeline(public_id):
     i = models.Item.objects.get(public_id=public_id)
     for s in i.streams():
         s.make_timeline()
-    i.update_timeline(async=False)
+    i.update_timeline(async_=False)
 
 @task(queue="encoding")
 def load_subtitles(public_id):

pandora/person/models.py

@@ -50,18 +50,19 @@ class Person(models.Model):
         if not self.sortname:
             self.sortname = ox.get_sort_name(self.name)
             self.sortname = unicodedata.normalize('NFKD', self.sortname)
-        self.sortsortname = utils.sort_string(self.sortname)
+        self.sortsortname = utils.sort_string(self.sortname).lower()
         self.numberofnames = len(self.name.split(' '))
         super(Person, self).save(*args, **kwargs)
 
     def update_itemsort(self):
+        sortname = self.sortname.lower()
         item.models.Facet.objects.filter(
             key__in=item.models.Item.person_keys + ['name'],
             value=self.name
         ).exclude(
-            sortvalue=self.sortname
+            sortvalue=sortname
         ).update(
-            sortvalue=self.sortname
+            sortvalue=sortname
         )
         for i in item.models.Item.objects.filter(facets__in=item.models.Facet.objects.filter(
             key__in=item.models.Item.person_keys + ['name'],

requirements.txt

@@ -1,4 +1,4 @@
-Django==1.11.13
+Django==1.11.22
 simplejson
 chardet
 celery==3.1.26.post2
@@ -6,8 +6,8 @@ django-celery==3.2.2
 django-extensions==2.0.7
 gunicorn==19.8.1
 html5lib
-requests==2.19.1
+requests==2.22.0
 tornado<5
 geoip2==2.9.0
-youtube-dl
+youtube-dl>=2019.4.30
 python-memcached

static/js/UI.js

@@ -24,6 +24,9 @@ pandora.UI = (function() {
             pandora.user.ui._filterState = pandora.getFilterState(
                 pandora.user.ui.find
             );
+            pandora.user.ui._documentFilterState = pandora.getDocumentFilterState(
+                pandora.user.ui.findDocuments
+            );
             pandora.user.ui._findState = pandora.getFindState(
                 pandora.user.ui.find
             );
@@ -85,6 +88,7 @@ pandora.UI = (function() {
                 // (values we put in add will be changed, but won't trigger)
                 collection = pandora.getCollectionState(args.findDocuments);
                 pandora.user.ui._collection = collection;
+                pandora.user.ui._documentFilterState = pandora.getDocumentFilterState(args.findDocuments);
                 pandora.user.ui._findDocumentsState = pandora.getFindDocumentsState(args.findDocuments);
                 if (pandora.$ui.appPanel && !pandora.stayInItemView) {
                     // if we're not on page load, and if find isn't a context change

static/js/URL.js

@@ -144,6 +144,7 @@ pandora.URL = (function() {
 
         pandora.user.ui._list = pandora.getListState(pandora.user.ui.find);
         pandora.user.ui._filterState = pandora.getFilterState(pandora.user.ui.find);
+        pandora.user.ui._documentFilterState = pandora.getDocumentFilterState(pandora.user.ui.findDocuments);
         pandora.user.ui._findState = pandora.getFindState(pandora.user.ui.find);
         pandora.user.ui._collection = pandora.getCollectionState(pandora.user.ui.findDocuments);
         pandora.user.ui._findDocumentsState = pandora.getFindDocumentsState(pandora.user.ui.findDocuments);

static/js/addItemDialog.js

@@ -107,7 +107,7 @@ pandora.ui.addItemDialog = function(options) {
                     pandora.api.getMediaUrlInfo({
                         url: input
                     }, function(result) {
-                        if (requslt.data.items.length) {
+                        if (result.data.items.length) {
                             onInfo(result.data.items.map(getVideoInfo));
                         } else {
                             $screen.stop();

static/js/browser.js

@@ -52,7 +52,7 @@ pandora.ui.browser = function() {
                                         selected: selected
                                     });
                                 }
-                            }).reloadList();                            
+                            }).reloadList();
                         }
                     });
                     pandora.$ui.filters.updateMenus();

static/js/collection.js

@@ -13,6 +13,7 @@ pandora.ui.collection = function() {
 
     if (view == 'list') {
         that = Ox.TableList({
+            draggable: true,
             keys: keys,
             items: function(data, callback) {
                 pandora.api.findDocuments(Ox.extend(data, {

static/js/collectionIconPanel.js

@@ -0,0 +1,249 @@
+'use strict';
+
+pandora.ui.collectionIconPanel = function(listData) {
+
+    var quarter = 0,
+        quarters = ['top-left', 'top-right', 'bottom-left', 'bottom-right'],
+
+        ui = pandora.user.ui,
+        folderItems = pandora.getFolderItems(ui.section),
+        folderItem = folderItems.slice(0, -1),
+
+
+        $iconPanel = Ox.Element(),
+
+        $icon = $('<img>')
+            .attr({
+                src: pandora.getListIcon(ui.section, listData.id, 256)
+            })
+            .css({position: 'absolute', borderRadius: '64px', margin: '16px'})
+            .appendTo($iconPanel),
+
+        $previewPanel = Ox.Element(),
+
+        $preview,
+
+        $list = Ox.Element(),
+
+        ui = pandora.user.ui,
+
+        that = Ox.SplitPanel({
+            elements: [
+                {
+                    element: $iconPanel,
+                    size: 280
+                },
+                {
+                    element: $previewPanel
+                },
+                {
+                    element: $list,
+                    size: 144 + Ox.UI.SCROLLBAR_SIZE
+                }
+            ],
+            orientation: 'horizontal'
+        });
+
+    pandora.api['find' + folderItems]({
+        query: {
+            conditions: [{key: 'id', value: listData.id, operator: '=='}],
+            operator: '&'
+        },
+        keys: ['posterFrames']
+    }, function(result) {
+
+        var posterFrames = result.data.items[0].posterFrames,
+            posterFrame = posterFrames[quarter],
+
+            $interface = Ox.Element({
+                    tooltip: function(e) {
+                        var quarterName = ($(e.target).attr('id') || '').replace('-', ' ');
+                        return quarterName ? Ox._('Edit ' + quarterName + ' image') : null;
+                    }
+                })
+                .css({
+                    position: 'absolute',
+                    width: '256px',
+                    height: '256px',
+                    marginLeft: '16px',
+                    marginTop: '16px',
+                    cursor: 'pointer'
+                })
+                .on({
+                    click: function(e) {
+                        clickIcon(e);
+                    },
+                    dblclick: function(e) {
+                        clickIcon(e, true);
+                    }
+                })
+                .appendTo($iconPanel);
+
+        renderQuarters();
+
+        $list = Ox.IconList({
+            borderRadius: 16,
+            item: function(data, sort) {
+                var infoKey = ['title', 'author'].indexOf(sort[0].key) > -1
+                        ? pandora.site.documentKeys.filter(function(key) {
+                            return ['year', 'date'].indexOf(key.id) > -1
+                        }).map(function(key) {
+                            return key.id;
+                        })[0] : sort[0],key,
+                    size = 128;
+                return {
+                    height: size,
+                    id: data.id,
+                    info: data[infoKey] || '',
+                    title: data.title,
+                    url: pandora.getMediaURL('/documents/' + data.id + '/' + size + 'p.jpg?' + data.modified),
+                    width: size
+                };
+            },
+            items: function(data, callback) {
+                var listData = pandora.getListData();
+                pandora.api.findDocuments(Ox.extend(data, {
+                    query: {
+                        conditions: [{key: 'collection', value: listData.id, operator: '=='}],
+                        operator: '&'
+                    }
+                }), callback);
+            },
+            keys: ['duration', 'id', 'modified', 'title'],
+            max: 1,
+            min: 1,
+            //orientation: 'vertical',
+            selected: posterFrame ? [posterFrame.document] : [],
+            size: 128,
+            sort: ui.collectionSort,
+            unique: 'id'
+        })
+        //.css({width: '144px'})
+        .bindEvent({
+            open: function(data) {
+                setPosterFrame(data.ids[0], $list.value(data.ids[0], 'posterFrame'))
+            },
+            select: function(data) {
+                renderPreview($list.value(data.ids[0]));
+            }
+        })
+        .bindEventOnce({
+            load: function() {
+                var itemData;
+                if (!posterFrame) {
+                    itemData = $list.value(0);
+                    $list.options({selected: [itemData.id]});
+                } else {
+                    itemData = $list.value(posterFrame.item);
+                }
+                itemData && renderPreview(itemData);
+            }
+        })
+        .gainFocus();
+
+        that.replaceElement(2, $list);
+
+        function clickIcon(e, isDoubleClick) {
+            quarter = quarters.indexOf($(e.target).attr('id'));
+            renderQuarters();
+            if (isDoubleClick && posterFrames.length) {
+                var item = posterFrames[quarter].item;
+                $list.options({selected: [item]});
+                renderPreview($list.value(item), posterFrames[quarter].page);
+            }
+        }
+
+        function renderPreview(itemData, page) {
+            var size = 256;
+            if (itemData.id) {
+                $preview = Ox.Element('<img>').attr({
+                    src: pandora.getMediaURL('/documents/' + itemData.id + '/' + size + 'p.jpg?' + itemData.modified),
+                })
+                .css({
+                    width: size + 'px',
+                    height: size + 'px',
+                    marginLeft: '8px', marginTop: '16px', overflow: 'hidden'
+                })
+                .on({
+                    click: function(d) {
+                        setPosterFrame(itemData.id);
+                    }
+                });
+            } else {
+                $preview = Ox.Element()
+            }
+            $previewPanel.empty().append($preview);
+        }
+
+        function renderQuarters() {
+            $interface.empty();
+            quarters.forEach(function(q, i) {
+                $interface.append(
+                    $('<div>')
+                        .attr({id: q})
+                        .css({
+                            float: 'left',
+                            width: '126px',
+                            height: '126px',
+                            border: '1px solid rgba(255, 255, 255, ' + (i == quarter ? 0.75 : 0) + ')',
+                            background: 'rgba(0, 0, 0, ' + (i == quarter ? 0 : 0.75) + ')'
+                        })
+                        .css('border-' + q + '-radius', '64px')
+                );
+            });
+        }
+
+        function setPosterFrame(document, page) {
+            var posterFrame = {document: document, page: page};
+            if (posterFrames.length) {
+                posterFrames[quarter] = posterFrame;
+            } else {
+                posterFrames = Ox.range(4).map(function() { return Ox.clone(posterFrame); } );
+            }
+            pandora.api['edit' + folderItem]({
+                id: listData.id,
+                posterFrames: posterFrames
+            }, function() {
+                $icon.attr({
+                    src: pandora.getListIcon(ui.section, listData.id, 256)
+                });
+                pandora.$ui.folderList[listData.folder].$element
+                    .find('img[src*="'
+                        + pandora.getMediaURL('/' + encodeURIComponent(listData.id))
+                        + '/"]'
+                    )
+                    .attr({
+                        src: pandora.getListIcon(ui.section, listData.id, 256)
+                    });
+                pandora.$ui.info.updateListInfo();
+            });
+            $preview.options({page: page});
+        }
+
+    });
+
+    function renderFrame() {
+        $frame.css({borderRadius: 0});
+        $frame.css('border-' + quarters[quarter] + '-radius', '128px');
+    }
+
+    that.updateQuery = function(key, value) {
+        $list.options({
+            items: function(data, callback) {
+                pandora.api.findDocuments(Ox.extend(data, {
+                    query: {
+                        conditions: [{key: 'collection', value: listData.id, operator: '=='}].concat(
+                        value !== ''
+                            ? [{key: key, value: value, operator: '='}]
+                            : []
+                        ),
+                        operator: '&'
+                    }
+                }), callback);
+            }
+        });
+    };
+
+    return that;
+
+}

static/js/documentBrowser.js

@@ -3,7 +3,55 @@
 pandora.ui.documentBrowser = function() {
     var that;
     if (!pandora.user.ui.document) {
-        that = Ox.Element().html('fixme');
+        pandora.user.ui.filterSizes = pandora.getFilterSizes();
+        pandora.$ui.documentFilters = pandora.ui.documentFilters();
+        that = Ox.SplitPanel({
+            elements: [
+                {
+                    element: pandora.$ui.documentFilters[0],
+                    size: pandora.user.ui.filterSizes[0]
+                },
+                {
+                    element: pandora.$ui.documentFiltersInnerPanel = pandora.ui.documentFiltersInnerPanel()
+                },
+                {
+                    element: pandora.$ui.documentFilters[4],
+                    size: pandora.user.ui.filterSizes[4]
+                },
+            ],
+            id: 'browser',
+            orientation: 'horizontal'
+        })
+        .bindEvent({
+            resize: function(data) {
+                pandora.$ui.documentFilters.forEach(function(list) {
+                    list.size();
+                });
+            },
+            resizeend: function(data) {
+                pandora.UI.set({documentFiltersSize: data.size});
+            },
+            toggle: function(data) {
+                data.collapsed && pandora.$ui.list.gainFocus();
+                pandora.UI.set({showDocumentFilters: !data.collapsed});
+                if (!data.collapsed) {
+                    pandora.$ui.documentFilters.forEach(function($documentFilter) {
+                        var selected = $documentFilter.options('_selected');
+                        if (selected) {
+                            $documentFilter.bindEventOnce({
+                                load: function() {
+                                    $documentFilter.options({
+                                        _selected: false,
+                                        selected: selected
+                                    });
+                                }
+                            }).reloadList();
+                        }
+                    });
+                    pandora.$ui.documentFilters.updateMenus();
+                }
+            }
+        });
     } else {
         var that = Ox.IconList({
             borderRadius: 0,

static/js/documentContentPanel.js

@@ -4,16 +4,14 @@ pandora.ui.documentContentPanel = function() {
     var that = Ox.SplitPanel({
             elements: !pandora.user.ui.document ? [
                 {
-                    collapsed: true,
-                    collapsible: false, //fixme
+                    collapsed: !pandora.user.ui.showDocumentFilters,
+                    collapsible: true,
                     element: pandora.$ui.documentBrowser = pandora.ui.documentBrowser(),
-                    resizable: false, //fixme
+                    resizable: true,
                     resize: [96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256],
-                    size: 96,
-                    tooltip: '' /* fixme:
-                        Ox._('filters') + ' <span class="OxBright">'
-                            + Ox.SYMBOLS.shift + 'F</span>'
-                    */
+                    size: pandora.user.ui.documentFiltersSize,
+                    tooltip: Ox._('filters') + ' <span class="OxBright">'
+                        + Ox.SYMBOLS.shift + 'F</span>'
                 },
                 {
                     element: pandora.$ui.list = pandora.ui.collection()

static/js/documentDialog.js

@@ -196,7 +196,7 @@ pandora.ui.documentDialog = function(options) {
                         ? pandora.user.ui.documents[item.id].position
                         : 1,
                     url: '/documents/' + item.id + '/'
-                        + item.title + '.' + item.extension,
+                        + item.title.replace('?', '_') + '.' + item.extension,
                     width: dialogWidth,
                     zoom: 'fit'
                 })

static/js/documentFilter.js

@@ -0,0 +1,337 @@
+'use strict';
+
+pandora.ui.documentFilter = function(id) {
+    var i = Ox.getIndexById(pandora.user.ui.documentFilters, id),
+        filter = Ox.getObjectById(pandora.site.documentFilters, id),
+        panelWidth = Ox.$document.width() - (pandora.user.ui.showSidebar * pandora.user.ui.sidebarSize) - 1,
+        title = Ox._(Ox.getObjectById(pandora.site.documentFilters, id).title),
+        //width = pandora.getFilterWidth(i, panelWidth),
+        that = Ox.TableList({
+            _selected: !pandora.user.ui.showFilters
+                ? pandora.user.ui._documentFilterState[i].selected
+                : false,
+            columns: [
+                {
+                    align: 'left',
+                    id: 'name',
+                    format: function(value) {
+                        var layer = Ox.getObjectById(pandora.site.layers, filter.id),
+                            key = Ox.getObjectById(pandora.site.itemKeys, filter.id);
+                        if ((layer && layer.translate) || (key && key.translate)) {
+                            value = Ox._(value)
+                        }
+                        return filter.flag
+                            ? $('<div>')
+                                .append(
+                                    $('<img>')
+                                        .attr({src: Ox[
+                                            filter.flag == 'country'
+                                                ? 'getFlagByGeoname'
+                                                : 'getFlagByLanguage'
+                                        ](value, 16)})
+                                        .css({
+                                            float: 'left',
+                                            width: '14px',
+                                            height: '14px',
+                                            margin: '0 3px 0 -2px',
+                                            borderRadius: '4px'
+                                        })
+                                )
+                                .append(
+                                    $('<div>')
+                                        .addClass('flagname')
+                                        .css({
+                                            float: 'left',
+                                            width: pandora.user.ui.filterSizes[i]
+                                                - 68 - Ox.UI.SCROLLBAR_SIZE,
+                                            textOverflow: 'ellipsis',
+                                            overflowX: 'hidden'
+                                        })
+                                        .html(value)
+                                )
+                            : value
+                    },
+                    operator: filter.type == 'string' || filter.type == 'layer' ? '+' : '-',
+                    title: title,
+                    visible: true,
+                    width: pandora.user.ui.filterSizes[i] - 44 - Ox.UI.SCROLLBAR_SIZE
+                },
+                {
+                    align: 'right',
+                    format: function(value) {
+                        return Ox.formatNumber(value);
+                    },
+                    id: 'items',
+                    operator: '-',
+                    title: '#',
+                    visible: true,
+                    width: 44
+                }
+            ],
+            columnsVisible: true,
+            id: 'filter_' + id,
+            items: function(data, callback) {
+                if (pandora.user.ui.showFilters) {
+                    delete data.keys;
+                    return pandora.api.findDocuments(Ox.extend(data, {
+                        group: id,
+                        query: pandora.user.ui._documentFilterState[i].find
+                    }), callback);
+                } else {
+                    callback({data: {items: data.keys ? [] : 0}});
+                }
+            },
+            scrollbarVisible: true,
+            selected: pandora.user.ui.showFilters
+                ? pandora.user.ui._documentFilterState[i].selected
+                : [],
+            sort: [{
+                key: pandora.user.ui.documentFilters[i].sort[0].key,
+                operator: pandora.user.ui.documentFilters[i].sort[0].operator
+            }],
+            unique: 'name'
+        })
+        .bindEvent({
+            init: function(data) {
+                that.setColumnTitle(
+                    'name',
+                    Ox._(Ox.getObjectById(pandora.site.documentFilters, id).title)
+                    + '<div class="OxColumnStatus OxLight">'
+                    + Ox.formatNumber(data.items) + '</div>'
+                );
+            },
+            paste: function(data) {
+                pandora.$ui.list.triggerEvent('paste', data);
+            },
+            select: function(data) {
+                // fixme: cant index be an empty array, instead of -1?
+                // FIXME: this is still incorrect when deselecting a filter item
+                // makes a selected item in another filter disappear
+                var conditions = data.ids.map(function(value) {
+                        return {
+                            key: id,
+                            value: value,
+                            operator: '=='
+                        };
+                    }),
+                    index = pandora.user.ui._documentFilterState[i].index,
+                    find = Ox.clone(pandora.user.ui.findDocuments, true);
+                if (Ox.isArray(index)) {
+                    // this filter had multiple selections and the | query
+                    // was on the top level, i.e. not bracketed
+                    find = {
+                        conditions: conditions,
+                        operator: conditions.length > 1 ? '|' : '&'
+                    }
+                } else {
+                    if (index == -1) {
+                        // this filter had no selection, i.e. no query
+                        index = find.conditions.length;
+                        if (find.operator == '|') {
+                            find = {
+                                conditions: [find],
+                                operator: '&'
+                            };
+                            index = 1;
+                        } else {
+                            find.operator = '&';
+                        }
+                    }
+                    if (conditions.length == 0) {
+                        // nothing selected
+                        find.conditions.splice(index, 1);
+                        if (find.conditions.length == 1) {
+                            if (find.conditions[0].conditions) {
+                                // unwrap single remaining bracketed query
+                                find = {
+                                    conditions: find.conditions[0].conditions,
+                                    operator: '|'
+                                };
+                            } else {
+                                find.operator = '&';
+                            }
+                        }
+                    } else if (conditions.length == 1) {
+                        // one item selected
+                        find.conditions[index] = conditions[0];
+                    } else {
+                        // multiple items selected
+                        if (pandora.user.ui.findDocuments.conditions.length == 1) {
+                            find = {
+                                conditions: conditions,
+                                operator: '|'
+                            };
+                        } else {
+                            find.conditions[index] = {
+                                conditions: conditions,
+                                operator: '|'
+                            };
+                        }
+                    }
+                }
+                pandora.UI.set({findDocuments: find});
+                pandora.$ui.documentFilters.updateMenus();
+            },
+            sort: function(data) {
+                Ox.Log('', 'SORT', data)
+                var filters = Ox.clone(pandora.user.ui.documentFilters, true);
+                /*
+                pandora.$ui.mainMenu.checkItem('sortMenu_sortfilters_sortfilter' + id + '_' + data.key);
+                pandora.$ui.mainMenu.checkItem('sortMenu_orderfilters_orderfilter' + id + '_' + (data.operator == '+' ? 'ascending' : 'descending'));
+                */
+                filters[i].sort = [{key: data.key, operator: data.operator}];
+                pandora.UI.set({documentFilters: filters});
+            }
+        }),
+        $menu = Ox.MenuButton({
+                items: [
+                    {id: 'clearFilter', title: Ox._('Clear Filter'), keyboard: 'shift control a'},
+                    {id: 'clearFilters', title: Ox._('Clear All Filters'), keyboard: 'shift alt control a'},
+                    {},
+                    {group: 'filter', max: 1, min: 1, items: pandora.site.documentFilters.map(function(filter) {
+                        return Ox.extend({checked: filter.id == id}, filter);
+                    })}
+                ],
+                type: 'image',
+            })
+            .css(Ox.UI.SCROLLBAR_SIZE == 8 ? {
+                right: '-1px',
+                width: '8px',
+            } : {
+                right: '2px',
+                width: (Ox.UI.SCROLLBAR_SIZE - 2) + 'px'
+            })
+            .bindEvent({
+                change: function(data) {
+                    var filters = Ox.clone(pandora.user.ui.documentFilters, true),
+                        find,
+                        id_ = data.checked[0].id,
+                        i_ = Ox.getIndexById(pandora.user.ui.documentFilters, id_);
+                    if (i_ == -1) {
+                        // new filter was not part of old filter set
+                        if (pandora.user.ui._documentFilterState[i].selected.length) {
+                            // if filter with selection gets replaced, update find
+                            find = Ox.clone(pandora.user.ui.findDocuments, true);
+                            find.conditions.splice(pandora.user.ui._documentFilterState[i].index, 1);
+                        }
+                        filters[i] = makeFilter(id_);
+                        pandora.UI.set(Ox.extend({
+                            documentFilters: filters
+                        }, find ? {
+                            findDocuments: find
+                        } : {}));
+                        replaceFilter(i, id_);
+                        // fixme: there is an obscure special case not yet covered:
+                        // switching to a new filter may change find from advanced to not advanced
+                        // if part of the existing query works as a filter selection in the new filter
+                    } else {
+                        // swap two existing filters
+                        var filterData = Ox.clone(pandora.user.ui._documentFilterState[i]);
+                        pandora.user.ui._documentFilterState[i] = pandora.user.ui._documentFilterState[i_];
+                        pandora.user.ui._documentFilterState[i_] = filterData;
+                        filters[i] = makeFilter(id_, pandora.user.ui.documentFilters[i_].sort);
+                        filters[i_] = makeFilter(id, pandora.user.ui.documentFilters[i].sort);
+                        pandora.UI.set({documentFilters: filters});
+                        replaceFilter(i, id_);
+                        replaceFilter(i_, id);
+                    }
+                    pandora.$ui.documentFilters.updateMenus();
+                    function makeFilter(id, sort) {
+                        // makes user.ui._documentFilterState object from site.documentFilters object
+                        var filter = Ox.getObjectById(pandora.site.documentFilters, id);
+                        return {
+                            id: filter.id,
+                            sort: sort || [{key: filter.type == 'integer' ? 'name' : 'items', operator: '-'}]
+                        };
+                    }
+                    function replaceFilter(i, id) {
+                        var isOuter = i % 4 == 0;
+                        pandora.$ui[isOuter ? 'documentBrowser' : 'documentFiltersInnerPanel'].replaceElement(
+                            isOuter ? i / 2 : i - 1,
+                            pandora.$ui.documentFilters[i] = pandora.ui.documentFilter(id)
+                        );
+                    }
+                },
+                click: function(data) {
+                    if (data.id == 'clearFilter') {
+                        // FIXME: List should trigger event on options change
+                        if (!Ox.isEmpty(that.options('selected'))) {
+                            that.options({selected: []}).triggerEvent('select', {ids: []});
+                        }
+                    } else if (data.id == 'clearFilters') {
+                        pandora.$ui.documentFilters.clearFilters();
+                    }
+                }
+            })
+            .appendTo(that.$bar);
+    Ox.UI.SCROLLBAR_SIZE < 16 && $($menu.find('input')[0]).css({
+        marginRight: '-3px',
+        marginTop: '1px',
+        width: '8px',
+        height: '8px'
+    });
+    that.disableMenuItem = function(id) {
+        $menu.disableItem(id);
+    };
+    that.enableMenuItem = function(id) {
+        $menu.enableItem(id);
+    };
+    return that;
+};
+
+pandora.ui.documentFilters = function() {
+    var $filters = [];
+    pandora.user.ui.documentFilters.forEach(function(filter, i) {
+        $filters[i] = pandora.ui.documentFilter(filter.id);
+    });
+    $filters.clearFilters = function() {
+        var find = Ox.clone(pandora.user.ui.findDocuments, true),
+            indices = pandora.user.ui._documentFilterState.map(function(filterState) {
+                return filterState.index;
+            }).filter(function(index) {
+                return index > -1;
+            });
+        find.conditions = find.conditions.filter(function(condition, index) {
+            return !Ox.contains(indices, index);
+        });
+        pandora.UI.set({findDocuments: find})
+    };
+    $filters.updateMenus = function() {
+        var selected = $filters.map(function($filter) {
+                return !Ox.isEmpty($filter.options('selected'));
+            }),
+            filtersHaveSelection = !!Ox.sum(selected);
+        $filters.forEach(function($filter, i) {
+            $filter[
+                 selected[i] ? 'enableMenuItem' : 'disableMenuItem'
+            ]('clearFilter');
+            $filter[
+                filtersHaveSelection ? 'enableMenuItem' : 'disableMenuItem'
+            ]('clearFilters');
+        });
+        return $filters;
+    };
+    return $filters.updateMenus();
+};
+
+pandora.ui.documentFiltersInnerPanel = function() {
+    var that = Ox.SplitPanel({
+        elements: [
+            {
+                element: pandora.$ui.documentFilters[1],
+                size: pandora.user.ui.filterSizes[1]
+            },
+            {
+                element: pandora.$ui.documentFilters[2]
+            },
+            {
+                element: pandora.$ui.documentFilters[3],
+                size: pandora.user.ui.filterSizes[3]
+            }
+        ],
+        orientation: 'horizontal'
+    });
+    return that;
+};
+
+

static/js/documentFilterForm.js

@@ -42,7 +42,7 @@ pandora.ui.documentFilterForm = function(options) {
                     view: pandora.user.ui.collectionView
                 } : null,
                 sortKeys: pandora.site.documentSortKeys,
-                value: Ox.clone(mode == 'collection' ? collection.query : pandora.user.ui.documentFind, true),
+                value: Ox.clone(mode == 'collection' ? collection.query : pandora.user.ui.findDocuments, true),
                 viewKeys: pandora.site.collectionViews
             })
             .css(mode == 'embed' ? {} : {padding: '16px'})
@@ -87,7 +87,7 @@ pandora.ui.documentFilterForm = function(options) {
             });
             */
         } else {
-            pandora.UI.set({find: Ox.clone(that.$filter.options('value'), true)});
+            pandora.UI.set({findDocuments: Ox.clone(that.$filter.options('value'), true)});
             pandora.$ui.findElement.updateElement();
         }
     };

static/js/documentInfoView.js

@@ -232,6 +232,9 @@ pandora.ui.documentInfoView = function(data, isMixed) {
     renderGroup(['author', 'date', 'type']);
     renderGroup(['publisher', 'place', 'series', 'edition', 'language']);
 
+    Ox.getObjectById(pandora.site.documentKeys, 'keywords') && renderGroup(['keywords'])
+
+
     // Description -------------------------------------------------------------
 
     if (canEdit || data.description) {

static/js/documentToolbar.js

@@ -64,7 +64,7 @@ pandora.ui.documentToolbar = function() {
         }
     })
     that.append(
-        pandora.$ui.findDocumentsElement = pandora.ui.findDocumentsElement(function(data) {
+        pandora.$ui.findElement= pandora.ui.findDocumentsElement(function(data) {
             var key = data.key,
                 value = data.value,
                 conditions;

static/js/editDocumentsDialog.js

@@ -97,7 +97,8 @@ pandora.ui.editDocumentsDialog = function() {
                     });
                 if (isArray) {
                     values = values.map(function(value) {
-                        return (value || []).join(separator);
+                        value = value || []
+                        return value.join ? value.join(separator) : value;
                     });
                 }
                 if (Ox.unique(values).length > 1) {

static/js/filter.js

@@ -174,7 +174,7 @@ pandora.ui.filter = function(id) {
             },
             sort: function(data) {
                 Ox.Log('', 'SORT', data)
-                var filters = Ox.clone(pandora.user.ui.filters);
+                var filters = Ox.clone(pandora.user.ui.filters, true);
                 pandora.$ui.mainMenu.checkItem('sortMenu_sortfilters_sortfilter' + id + '_' + data.key);
                 pandora.$ui.mainMenu.checkItem('sortMenu_orderfilters_orderfilter' + id + '_' + (data.operator == '+' ? 'ascending' : 'descending'));
                 filters[i].sort = [{key: data.key, operator: data.operator}];
@@ -201,7 +201,7 @@ pandora.ui.filter = function(id) {
             })
             .bindEvent({
                 change: function(data) {
-                    var filters = Ox.clone(pandora.user.ui.filters),
+                    var filters = Ox.clone(pandora.user.ui.filters, true),
                         find,
                         id_ = data.checked[0].id,
                         i_ = Ox.getIndexById(pandora.user.ui.filters, id_);

static/js/filterDialog.js

@@ -42,7 +42,12 @@ pandora.ui.filterDialog = function() {
                         }
                     })
             ],
-            content: pandora.$ui.filterForm = pandora.ui.filterForm({mode: 'find'}),
+            content: pandora.$ui.filterForm = (pandora.user.ui.section == 'documents'
+                ? pandora.ui.documentFilterForm
+                : pandora.ui.filterForm
+            )({
+                mode: 'find'
+            }),
             maxWidth: 648 + Ox.UI.SCROLLBAR_SIZE,
             minHeight: 264,
             minWidth: 648 + Ox.UI.SCROLLBAR_SIZE,

static/js/findDocumentsElement.js

@@ -35,26 +35,39 @@ pandora.ui.findDocumentsElement = function() {
                 ] : [], [
                     $findSelect = Ox.Select({
                             id: 'select',
-                            items: pandora.site.documentKeys.filter(function(key) {
-                                return key.find;
-                            }).map(function(key) {
+                            items: [].concat(
+                                pandora.site.documentKeys.filter(function(key) {
+                                    return key.find;
+                                }).map(function(key) {
                                     return {
                                         id: key.id,
                                         title: Ox._('Find: {0}', [Ox._(key.title)])
                                     };
                                 }),
+                                [{}, {
+                                    id: 'advanced',
+                                    title: Ox._('Find: Advanced...')
+                                }]
+                            ),
+
                             overlap: 'right',
                             value: findKey,
                             width: 128
                         })
                         .bindEvent({
                             change: function(data) {
-                                //pandora.$ui.mainMenu.checkItem('findMenu_find_' + data.value);
-                                $findInput.options({
-                                    autocomplete: autocompleteFunction(),
-                                    placeholder: ''
-                                }).focusInput(true);
-                                previousFindKey = data.value;
+                                if (data.value == 'advanced') {
+                                    that.updateElement();
+                                    pandora.$ui.mainMenu.checkItem('findMenu_find_' + previousFindKey);
+                                    pandora.$ui.filterDialog = pandora.ui.filterDialog().open();
+                                } else {
+                                    //pandora.$ui.mainMenu.checkItem('findMenu_find_' + data.value);
+                                    $findInput.options({
+                                        autocomplete: autocompleteFunction(),
+                                        placeholder: ''
+                                    }).focusInput(true);
+                                    previousFindKey = data.value;
+                                }
                             }
                         }),
                     $findInput = Ox.Input({
@@ -77,7 +90,7 @@ pandora.ui.findDocumentsElement = function() {
                         focus: function(data) {
                             if ($findSelect.value() == 'advanced') {
                                 if (hasPressedClear) {
-                                    pandora.UI.set({find: pandora.site.user.ui.find});
+                                    pandora.UI.set({findDocuments: pandora.site.user.ui.findDocuments});
                                     that.updateElement();
                                     hasPressedClear = false;
                                 }

static/js/folderBrowserList.js

@@ -235,7 +235,7 @@ pandora.ui.folderBrowserList = function(id, section) {
                     pandora.UI.set({
                         findDocuments: {
                             conditions: list ? [
-                                {key: 'list', value: data.ids[0], operator: '=='}
+                                {key: 'collection', value: data.ids[0], operator: '=='}
                             ] : [],
                             operator: '&'
                         }

static/js/folders.js

@@ -46,7 +46,15 @@ pandora.ui.folders = function(section) {
                         pandora.$ui.personalListsMenu = Ox.MenuButton({
                             items: [
                                 { id: 'newlist', title: Ox._('New {0}', [Ox._(folderItem)]), keyboard: 'control n' },
-                                { id: 'newlistfromselection', title: Ox._('New {0} from Selection', [Ox._(folderItem)]), keyboard: 'shift control n', disabled: ui.listSelection.length == 0 },
+                                {
+                                    id: 'newlistfromselection',
+                                    title: Ox._('New {0} from Selection',
+                                    [Ox._(folderItem)]),
+                                    keyboard: 'shift control n',
+                                    disabled:  ui.section == 'documents'
+                                        ? ui.collectionSelection == 0
+                                        : ui.listSelection.length == 0
+                                },
                                 { id: 'newsmartlist', title: Ox._('New Smart {0}', [Ox._(folderItem)]), keyboard: 'alt control n' },
                                 { id: 'newsmartlistfromresults', title: Ox._('New Smart {0} from Results', [Ox._(folderItem)]), keyboard: 'shift alt control n' },
                                 {},

static/js/infoView.indiancinema.js

@@ -24,7 +24,7 @@ pandora.ui.infoView = function(data, isMixed) {
         listWidth = 0,
         margin = 16,
         // these may contain commas, and are thus separated by semicolons
-        specialListKeys = ['alternativeTitles', 'productionCompany'].concat(
+        specialListKeys = ['alternativeTitles', 'productionCompany', 'laboratory'].concat(
             pandora.site.itemKeys.filter(function(key) {
                 return key.type[0] == 'date'
             }).map(function(key) {
@@ -439,6 +439,12 @@ pandora.ui.infoView = function(data, isMixed) {
 
     // Songs
     if (data.songs || canEdit) {
+        $('<div>')
+            .css({
+                marginTop: '12px',
+            })
+            .html(formatKey('songs'))
+            .appendTo($text);
         Ox.EditableContent({
                 clickLink: pandora.clickLink,
                 collapseToEnd: false,
@@ -449,7 +455,7 @@ pandora.ui.infoView = function(data, isMixed) {
                         '<img style="float: left; max-width: 256px; max-height: 256px; margin: 0 16px 16px 0" src='
                     );
                 },
-                placeholder: formatLight(Ox._(isMixed.songs ? 'Mixed Songs' : 'No Songs')),
+                placeholder: formatLight(Ox._(isMixed.songs ? 'Mixed Songs' : 'unknown')),
                 tooltip: canEdit ? pandora.getEditTooltip() : '',
                 type: 'textarea',
                 value: data.songs || ''
@@ -457,7 +463,6 @@ pandora.ui.infoView = function(data, isMixed) {
             })
             .css(css)
             .css({
-                marginTop: '12px',
                 overflow: 'hidden'
             })
             .bindEvent({
@@ -651,7 +656,7 @@ pandora.ui.infoView = function(data, isMixed) {
                     ? Ox.decodeHTMLEntities(value).split('; ').map(Ox.encodeHTMLEntities)
                     : [];
             } else if (key == 'imdbId') {
-                edit[key] = value.match(/\d{7}/)[0];
+                edit[key] = value ? value.match(/\d{7}/)[0] : value;
             } else {
                 edit[key] = value;
             }
@@ -751,10 +756,11 @@ pandora.ui.infoView = function(data, isMixed) {
 
     function formatValue(key, value) {
         var ret;
+        var itemKey = Ox.getObjectById(pandora.site.itemKeys, key)
         if (key == 'year') {
             ret = formatLink(value, 'year');
         } else if (
-            listKeys.indexOf(key) > -1 && Ox.getObjectById(pandora.site.itemKeys, key).type[0] == 'date'
+            specialListKeys.indexOf(key) > -1 && itemKey && itemKey.type[0] == 'date'
         ) {
             ret = value.split('; ').map(function(date) {
                 date = cleanupDate(date)
@@ -906,8 +912,8 @@ pandora.ui.infoView = function(data, isMixed) {
                 return value[0];
             }).join('; ')
             : key == 'runtime' ? Math.round(value / 60)
-            : Ox.contains(listKeys, key) ? value.join(', ')
-            : Ox.contains(specialListKeys, key) ? value.join('; ')
+            : Ox.contains(listKeys, key) && value.join ? value.join(', ')
+            : Ox.contains(specialListKeys, key) && value.join ? value.join('; ')
             : value;
     }
 

static/js/listDialog.js

@@ -22,7 +22,12 @@ pandora.ui.listDialog = function(section) {
                 if (id == 'general') {
                     return pandora.ui.listGeneralPanel(listData);
                 } else if (id == 'icon') {
-                    return pandora.$ui.listIconPanel = pandora.ui.listIconPanel(listData);
+                    if (pandora.user.ui.section == 'documents') {
+                        pandora.$ui.listIconPanel = pandora.ui.collectionIconPanel(listData);
+                    } else {
+                        pandora.$ui.listIconPanel = pandora.ui.listIconPanel(listData);
+                    }
+                    return pandora.$ui.listIconPanel
                 } else if (id == 'query') {
                     return pandora.$ui.filterForm = (pandora.user.ui.section == 'documents'
                         ? pandora.ui.documentFilterForm
@@ -64,7 +69,11 @@ pandora.ui.listDialog = function(section) {
     var $findElement = Ox.FormElementGroup({
             elements: [
                 pandora.$ui.findIconItemSelect = Ox.Select({
-                    items: pandora.site.findKeys.map(function(findKey) {
+                    items: (
+                        pandora.user.ui.section == 'items'
+                        ? pandora.site.findKeys
+                        : pandora.site.documentKeys
+                    ).map(function(findKey) {
                         return {id: findKey.id, title: Ox._('Find: {0}', [Ox._(findKey.title)])};
                     }),
                     overlap: 'right',

static/js/pandora.js

@@ -342,6 +342,15 @@ appPanel
                     type: Ox.isArray(key.type) ? key.type[0] : key.type
                 };
             }),
+            documentFilters: data.site.documentKeys.filter(function(key) {
+                return key.filter;
+            }).map(function(key) {
+                return {
+                    id: key.id,
+                    title: key.title,
+                    type: Ox.isArray(key.type) ? key.type[0] : key.type
+                };
+            }),
             findKeys: data.site.itemKeys.filter(function(key) {
                 return key.find;
             }),

static/js/utils.js

@@ -192,7 +192,9 @@ pandora.addFolderItem = function(section) {
                 }, function(result) {
                     var posterFrames = result
                         ? result.data.items.map(function(item) {
-                            return {item: item.id, position: item.posterFrame};
+                            return section == 'documents'
+                                ? {document: item.id}
+                                : {item: item.id, position: item.posterFrame};
                         }) : [];
                     posterFrames = posterFrames.length == 1
                         ? Ox.repeat([posterFrames[0]], 4)
@@ -860,7 +862,7 @@ pandora.enableDragAndDrop = function($list, canMove, section, getItems) {
                             items: drag.ids
 
                         }, function() {
-                            Ox.Request.clearCache('find');
+                            Ox.Request.clearCache('findDocuments');
                             pandora.api.findDocuments({
                                 query: {
                                     conditions: [{
@@ -2626,6 +2628,7 @@ pandora.signin = function(data) {
     });
     pandora.user.ui._list = pandora.getListState(pandora.user.ui.find);
     pandora.user.ui._filterState = pandora.getFilterState(pandora.user.ui.find);
+    pandora.user.ui._documentFilterState = pandora.getDocumentFilterState(pandora.user.ui.findDocuments);
     pandora.user.ui._findState = pandora.getFindState(pandora.user.ui.find);
     pandora.user.ui._collection = pandora.getCollectionState(pandora.user.ui.findDocuments);
     pandora.user.ui._findDocumentsState = pandora.getFindDocumentsState(pandora.user.ui.findDocuments);
@@ -2642,6 +2645,7 @@ pandora.signout = function(data) {
     pandora.user = data.user;
     pandora.user.ui._list = pandora.getListState(pandora.user.ui.find);
     pandora.user.ui._filterState = pandora.getFilterState(pandora.user.ui.find);
+    pandora.user.ui._documentFilterState = pandora.getDocumentFilterState(pandora.user.ui.findDocuments);
     pandora.user.ui._findState = pandora.getFindState(pandora.user.ui.find);
     pandora.user.ui._collection = pandora.getCollectionState(pandora.user.ui.findDocuments);
     pandora.user.ui._findDocumentsState = pandora.getFindDocumentsState(pandora.user.ui.findDocuments);
@@ -2825,9 +2829,15 @@ pandora.resizeFilters = function(width) {
     pandora.$ui.browser && pandora.$ui.browser
         .size(0, pandora.user.ui.filterSizes[0])
         .size(2, pandora.user.ui.filterSizes[4]);
+    pandora.$ui.documentBrowser && pandora.$ui.documentBrowser
+        .size(0, pandora.user.ui.filterSizes[0])
+        .size(2, pandora.user.ui.filterSizes[4]);
     pandora.$ui.filtersInnerPanel && pandora.$ui.filtersInnerPanel
         .size(0, pandora.user.ui.filterSizes[1])
         .size(2, pandora.user.ui.filterSizes[3]);
+    pandora.$ui.documentFiltersInnerPanel && pandora.$ui.documentFiltersInnerPanel
+        .size(0, pandora.user.ui.filterSizes[1])
+        .size(2, pandora.user.ui.filterSizes[3]);
     pandora.$ui.filters && pandora.$ui.filters.forEach(function($list, i) {
         $list.resizeColumn(
             'name',
@@ -2839,6 +2849,17 @@ pandora.resizeFilters = function(width) {
             });
         }
     });
+    pandora.$ui.documentFilters && pandora.$ui.documentFilters.forEach(function($list, i) {
+        $list.resizeColumn(
+            'name',
+            pandora.user.ui.filterSizes[i] - 44 - Ox.UI.SCROLLBAR_SIZE
+        );
+        if (pandora.site.flags) {
+            $list.find('.flagname').css({
+                width: pandora.user.ui.filterSizes[i] - 68 - Ox.UI.SCROLLBAR_SIZE
+            });
+        }
+    });
 };
 
 pandora.resizeFolders = function(section) {
@@ -3396,6 +3417,7 @@ pandora.wait = function(id, callback, timeout) {
         }
         return state;
     };
+
     function getState(find, key) {
         var index, state = '';
         if (find.operator == '&') {
@@ -3429,12 +3451,10 @@ pandora.wait = function(id, callback, timeout) {
         if (find.operator == '&') {
             // number of conditions that are not list or filters
             conditions = find.conditions.length
-                - !!pandora.user.ui._collection;
-            /*
-                - pandora.user.ui._filterState.filter(function(filter) {
+                - !!pandora.user.ui._collection
+                - pandora.user.ui._documentFilterState.filter(function(filter) {
                     return filter.index > -1;
                 }).length;
-            */
             // indices of non-advanced find queries
             indices = pandora.site.documentKeys.map(function(findKey) {
                 return oneCondition(find.conditions, findKey.id, '=');
@@ -3456,16 +3476,62 @@ pandora.wait = function(id, callback, timeout) {
                 key: 'advanced',
                 value: ''
             };
-            /*
             Ox.forEach(pandora.user.ui.documentFilters, function(key) {
                 if (everyCondition(find.conditions, key, '==')) {
                     state.key = '*';
                     return false;
                 }
             });
-            */
         }
         return state;
     };
 
+    pandora.getDocumentFilterState = function(find) {
+        // A filter is selected if exactly one condition in an & query or every
+        // condition in an | query has the filter id as key and "==" as operator
+        return pandora.user.ui.documentFilters.map(function(filter) {
+            // FIXME: cant index be an empty array, instead of -1?
+            var key = filter.id,
+                state = {index: -1, find: Ox.clone(find, true), selected: []};
+            if (find.operator == '&') {
+                // include conditions where all subconditions match
+                state.index = oneCondition(find.conditions, key, '==', true);
+                if (state.index > -1) {
+                    state.selected = find.conditions[state.index].conditions
+                        ? find.conditions[state.index].conditions.map(function(condition) {
+                            return condition.value;
+                        })
+                        : [find.conditions[state.index].value];
+                }
+            } else {
+                if (everyCondition(find.conditions, key, '==')) {
+                    state.index = Ox.range(find.conditions.length);
+                    state.selected = find.conditions.map(function(condition) {
+                        return condition.value;
+                    });
+                }
+            }
+            if (state.selected.length) {
+                if (Ox.isArray(state.index)) {
+                    // every condition in an | query matches this filter
+                    state.find = {conditions: [], operator: ''};
+                } else {
+                    // one condition in an & query matches this filter
+                    state.find.conditions.splice(state.index, 1);
+                    if (
+                        state.find.conditions.length == 1
+                        && state.find.conditions[0].conditions
+                    ) {
+                        // unwrap single remaining bracketed query
+                        state.find = {
+                            conditions: state.find.conditions[0].conditions,
+                            operator: state.find.conditions[0].operator
+                        };
+                    }
+                }
+            }
+            return state;
+        });
+    };
+
 }());

update.py

@@ -58,7 +58,6 @@ def run(*cmd):
     p.wait()
     return p.returncode
 
-
 def get(*cmd):
     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     stdout, error = p.communicate()
@@ -135,7 +134,10 @@ if __name__ == "__main__":
     if len(sys.argv) == 2 and sys.argv[1] in ('database', 'db'):
         os.chdir(join(base, 'pandora'))
         print('\nRunning "./manage.py migrate"\n')
-        run('./manage.py', 'migrate', '--noinput')
+        r = get('./manage.py', 'migrate', '--noinput')
+        r = r.replace("Your models have changes that are not yet reflected in a migration, and so won't be applied.", '')
+        r = r.replace("Run 'manage.py makemigrations' to make new migrations, and then re-run 'manage.py migrate' to apply them.", '')
+        print(r)
         run('./manage.py', 'sqlfindindex')
         run('./manage.py', 'sync_itemsort')
         run('./manage.py', 'sync_documentsort')
@@ -257,6 +259,8 @@ if __name__ == "__main__":
         if old <= 6064:
             run('./bin/pip', 'install', '-r', 'requirements.txt')
             run('./pandora/manage.py', 'createcachetable')
+        if old <= 6108:
+            run('./bin/pip', 'install', '-r', 'requirements.txt')
     else:
         if len(sys.argv) == 1:
             branch = get_branch()

vm/LXC_README.md

@@ -19,7 +19,7 @@
 
     or
 
-    sudo lxc-create -n pandora -t debian -- -r stretch
+    sudo lxc-create -n pandora -t debian -- -r buster
 
 3) Install pan.do/ra in container:
 

vm/LXD_README.md

@@ -0,0 +1,38 @@
+# Preparations
+
+    you will need at least 2GB of free disk space to install pan.do/ra
+
+# Installing pan.do/ra inside LXD
+
+1) Install lxd on the host (Ubuntu 16.04 or later):
+
+    sudo apt-get install lxd
+
+[on debian you can use snap install lxd]
+
+2) Create a new container, use different names if installing multiple instances:
+
+    sudo lxc launch ubuntu:18.04 pandora
+
+    or
+
+    sudo lxc launch images:debian/10 pandora
+
+3) Attach to container and install pan.do/ra
+
+    sudo lxc exec pandora bash
+    apt-get update -qq && apt-get upgrade -y
+    apt-get -y install curl ca-certificates
+    sed -i s/ubuntu/pandora/g /etc/passwd /etc/shadow /etc/group
+    mv /home/ubuntu /home/pandora
+    echo "pandora:pandora" | chpasswd
+    echo PasswordAuthentication no >> /etc/ssh/sshd_config
+    locale-gen en_US.UTF-8
+    update-locale LANG=en_US.UTF-8
+    export LANG=en_US.UTF-8
+
+    cd /root
+    curl -sL https://pan.do/ra-install > pandora_install.sh
+    chmod +x pandora_install.sh
+    ./pandora_install.sh 2>&1 | tee pandora_install.log
+

vm/pandora_install.sh

@@ -1,4 +1,9 @@
 #!/bin/bash
+#
+# pan.do/ra installer
+# ===================
+#
+
 PANDORA=${PANDORA-pandora}
 
 POSTGRES=${POSTGRES-local}
@@ -6,20 +11,22 @@ RABBITMQ=${RABBITMQ-local}
 NGINX=${NGINX-local}
 BRANCH=${BRANCH-stable}
 
+# add a pandora user
 echo Installing pandora with user: $PANDORA
 getent passwd $PANDORA > /dev/null 2>&1 || adduser --disabled-password --gecos "" $PANDORA
 
+#
+# install pan.do/ra ppa
+# 
+# apt-get install software-properties-common
+# add-apt-repository ppa:j/pandora
+#
 LXC=`grep -q lxc /proc/1/environ && echo 'yes' || echo 'no'`
 if [ -e /etc/os-release ]; then
     . /etc/os-release
 fi
-if [ -d "/run/systemd/system/" ]; then
-    SYSTEMD="yes"
-else
-    SYSTEMD="no"
-fi
 if [ -z "$UBUNTU_CODENAME" ]; then
-    UBUNTU_CODENAME=zesty
+    UBUNTU_CODENAME=bionic
 fi
 export DEBIAN_FRONTEND=noninteractive
 echo "deb http://ppa.launchpad.net/j/pandora/ubuntu ${UBUNTU_CODENAME} main" > /etc/apt/sources.list.d/j-pandora.list
@@ -46,10 +53,12 @@ echo 'Acquire::Languages "none";' > /etc/apt/apt.conf.d/99languages
 apt-get update -qq
 
 if [ "$LXC" == "no" ]; then
-apt-get install -y \
-    acpid \
-    ntp
+apt-get install -y acpid
+systemctl enable systemd-timesyncd.service
 fi
+
+# add postgres, rabbitmq and nginx
+# unless they are running on another host
 EXTRA=""
 if [ "$POSTGRES" == "local" ]; then
     EXTRA="$EXTRA postgresql postgresql-contrib"
@@ -61,6 +70,7 @@ if [ "$NGINX" == "local" ]; then
     EXTRA="$EXTRA nginx"
 fi
 
+# install all required packages
 apt-get install -y \
     sudo \
     openssh-server \
@@ -87,18 +97,21 @@ apt-get install -y \
     gpac \
     imagemagick \
     poppler-utils \
-    youtube-dl \
     ipython3 \
     postfix \
     postgresql-client $EXTRA
 
+apt-get install -y --no-install-recommends youtube-dl rtmpdump
+
+# setup database
+
 if [ "$POSTGRES" == "local" ]; then
     sudo -u postgres createuser -S -D -R $PANDORA
     sudo -u postgres createdb  -T template0 --locale=C --encoding=UTF8 -O $PANDORA pandora
     echo "CREATE EXTENSION pg_trgm;" | sudo -u postgres psql pandora
 fi
 
-#rabbitmq
+# setup rabbitmq
 if [ "$RABBITMQ" == "local" ]; then
     RABBITPWD=$(pwgen -n 16 -1)
     rabbitmqctl add_user pandora $RABBITPWD
@@ -109,18 +122,20 @@ else
     BROKER_URL="$RABBITMQ"
 fi
 
-#pandora
+# checkout pandora from git
 git clone https://git.0x2620.org/pandora.git /srv/pandora
 cd /srv/pandora
 git checkout $BRANCH
 ./ctl init
 
+# create config.jsonc from templates in git
 HOST=$(hostname -s)
 HOST_CONFIG="/srv/pandora/pandora/config.$HOST.jsonc"
 SITE_CONFIG="/srv/pandora/pandora/config.jsonc"
 test -e $HOST_CONFIG && cp $HOST_CONFIG $SITE_CONFIG
 test -e $SITE_CONFIG || cp /srv/pandora/pandora/config.pandora.jsonc $SITE_CONFIG
 
+# create local_settings.py
 cat > /srv/pandora/pandora/local_settings.py <<EOF
 DATABASES = {
     'default': {
@@ -141,15 +156,18 @@ EOF
 
 MANAGE="sudo -H -u $PANDORA /srv/pandora/pandora/manage.py"
 
+# more sure all files are owned by the pandora user
 mkdir /srv/pandora/data
 chown -R $PANDORA:$PANDORA /srv/pandora
 
+# initialize the database
 echo "Initialize database..."
 cd /srv/pandora/pandora
 $MANAGE init_db
 $MANAGE createcachetable
 echo "UPDATE django_site SET domain = '$HOST.local', name = '$HOST.local' WHERE 1=1;" | $MANAGE dbshell
 
+# install pandora systemd services
 /srv/pandora/ctl install
 if [ "$PANDORA" != "pandora" ]; then
     sed -i \
@@ -161,6 +179,7 @@ if [ "$PANDORA" != "pandora" ]; then
     systemctl daemon-reload
 fi
 
+# if pandora is running inside a container, expose backend at port 2620
 if [ "$LXC" == "yes" ]; then
     sed -i s/127.0.0.1/0.0.0.0/g /srv/pandora/pandora/gunicorn_config.py
     echo "WEBSOCKET_ADDRESS = \"0.0.0.0\"" >> /srv/pandora/pandora/local_settings.py
@@ -170,7 +189,7 @@ fi
 #logrotate
 #cp "/srv/pandora/etc/logrotate.d/pandora" "/etc/logrotate.d/pandora"
 
-#nginx
+# configure nginx
 if [ "$NGINX" == "local" ]; then
 
 cp "/srv/pandora/etc/nginx/pandora" "/etc/nginx/sites-available/default"
@@ -191,21 +210,9 @@ service nginx restart
 
 fi
 
-if [ "$LXC" == "yes" ]; then
-    test -e /etc/init/avahi-daemon.conf && sed -i "s/-D/--no-rlimits -D/g" /etc/init/avahi-daemon.conf
-fi
-
+# additional configurations if installed outside of LXD/LXC
 if [ "$LXC" == "no" ]; then
-    if [ "$SYSTEMD" == "yes" ]; then
-        echo Servers=pool.ntp.org >> /etc/systemd/timesyncd.conf
-    else
-cat > /etc/cron.d/ntp_fixtime <<EOF
-# /etc/cron.d/ntp_fixtime: vms can go out of sync, run ntpdate to sync time
-
-*/10 * * * *   root /usr/sbin/ntpdate pool.ntp.org >/dev/null
-EOF
-    fi
-
+echo Servers=pool.ntp.org >> /etc/systemd/timesyncd.conf
 cat > /usr/local/bin/genissue <<EOF
 #!/bin/sh
 HOST=\$(ps ax | grep avahi-daemon | grep local | sed "s/.*\[\(.*\)\].*/\1/g" | sed 's/\.$//')
@@ -224,7 +231,7 @@ chmod +x /usr/local/bin/genissue
 
 cat > /etc/rc.local <<EOF
 #!/bin/sh -e
-#vm has one network interface and that might change, make sure its not persistent
+# vm has one network interface and that might change, make sure its not persistent
 rm -f /etc/udev/rules.d/70-persistent-net.rules
 
 #update issue
@@ -279,6 +286,10 @@ if has('mouse')
   set mouse=
 endif
 EOF
-if [ -e /usr/share/vim/vim80/defaults.vim ]; then
-    sed -i 's/ set mouse=a/" set mouse=a/g' /usr/share/vim/vim80/defaults.vim
-fi
+
+cat > /etc/vim/vimrc.local <<EOF
+runtime! defaults.vim
+let g:skip_defaults_vim = 1
+
+set mouse=
+EOF