diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..a1b0dfbb --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +.env +data/ +overlay/ diff --git a/.gitignore b/.gitignore index bc18996b..da0f04d5 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,5 @@ static/django_extensions *.swp pandora/gunicorn_config.py .DS_Store +.env +overlay/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..a1b287ee --- /dev/null +++ b/Dockerfile @@ -0,0 +1,11 @@ +FROM 0x2620/pandora-base:latest + +LABEL maintainer="0x2620@0x2620.org" + +ENV LANG en_US.UTF-8 + +#VOLUME /pandora +COPY . /srv/pandora +RUN /srv/pandora/docker/install.sh + +ENTRYPOINT [ "/entrypoint.sh" ] diff --git a/README.md b/README.md index c8b7ba2d..902b3f46 100644 --- a/README.md +++ b/README.md @@ -2,177 +2,51 @@ for more information on pan.do/ra visit our website at https://pan.do/ra -## SETUP +## Installing pan.do/ra - pan.do/ra is known to work with Ubuntu 16.04, - but other distributions might also work. + we recommend to run pan.do/ra inside of LXD or LXC or dedicated VM or server. + You will need at least 2GB of free disk space - The instructions below are for Ubuntu 16.04. - All command given expect that you are root. + pan.do/ra is known to work with Ubuntu 18.04 and Debian/10 (buster), + other distributions might also work, let us know if it works for you. - To run pan.do/ra you need to install and setup: + Use the following commands as root to install pan.do/ra and all dependencies: - python 3.5 - postgres - nginx (or apache2) - additional video packages +``` +cd /root +curl -sL https://pan.do/ra-install > pandora_install.sh +chmod +x pandora_install.sh +./pandora_install.sh 2>&1 | tee pandora_install.log +``` + + For step by step installation, look at [pandora_install.sh](vm/pandora_install.sh) -## Installing required packages +## Configuration -1) add pandora ppa to get all packages in the required version + pan.do/ra is mostly configured in two places: - apt-get install software-properties-common - add-apt-repository ppa:j/pandora - apt-get update +### /srv/pandora/pandora/local_settings.py -2) install all required packages - - apt-get install git \ - python3-setuptools python3-pip python3-venv ipython3 \ - python3-dev python3-pil python3-numpy python3-psycopg2 \ - python3-pyinotify python3-simplejson \ - python3-geoip python3-html5lib python3-lxml \ - postgresql postgresql-contrib rabbitmq-server \ - poppler-utils mkvtoolnix gpac imagemagick \ - youtube-dl python3-ox oxframe ffmpeg + this file contains local Django configuration overwrites, + like database configuration, email backend and more. -## Prepare Environment +### /srv/pandora/pandora/config.jsonc -1) add pandora user and set permissions + config.jsonc can be used in configure the pan.do/ra related + settings. From title to item keys to video resolutions. - adduser pandora --disabled-login --disabled-password - -2) Setup Database - - su postgres - createuser pandora - createdb -T template0 --locale=C --encoding=UTF8 -O pandora pandora - echo "CREATE EXTENSION pg_trgm;" | psql pandora - exit - -3) Setup RabbitMQ - - You have to use the same password here and in BROKER_URL in local_settings.py - - rabbitmqctl add_user pandora PASSWORD - rabbitmqctl add_vhost /pandora - rabbitmqctl set_permissions -p /pandora pandora ".*" ".*" ".*" + More info at + https://code.0x2620.org/0x2620/pandora/wiki/Configuration -## Install Pan.do/ra +## Customization -1) Get code from git + pan.do/ra can be customized, this is mostly done by adding + JavaScript files that replace or enhance parts of pan.do/ra - cd /srv/ - git clone https://git.0x2620.org/pandora.git pandora - cd pandora - ./ctl init + More info at + https://code.0x2620.org/0x2620/pandora/wiki/Customization - cd /srv - chown -R pandora.pandora pandora -2) create local_settings.py and config.jsonc - -2.1) create file /srv/pandora/pandora/local_settings.py with the following content: - - DATABASES = { - 'default': { - 'NAME': 'pandora', - 'ENGINE': 'django.db.backends.postgresql_psycopg2', - 'USER': 'pandora', - 'PASSWORD': '', - } - } - DB_GIN_TRGM = True - BROKER_URL = 'amqp://pandora:PASSWORD@localhost:5672//pandora' - - #with apache x-sendfile or lighttpd set this to True - XSENDFILE = False - - #with nginx X-Accel-Redirect set this to True - XACCELREDIRECT = True - -2.2) create config.jsonc - - config.jsonc holds the configuration for your site. - To start you can copy /srv/pandora/pandora/config.pandora.jsonc - to /srv/pandora/pandora/config.jsonc but have a look at - https://wiki.0x2620.org/wiki/pandora/configuration and - config.0xdb.jsonc config.padma.jsonc for configuration options. - -3) initialize database - - su pandora - cd /srv/pandora/pandora - ./manage.py init_db - -4) install init scripts and start daemons - - /srv/pandora/ctl install - /srv/pandora/ctl start - -5) Setup Webserver -a) nginx (recommended) - - apt-get install nginx - cp /srv/pandora/etc/nginx/pandora /etc/nginx/sites-available/pandora - cd /etc/nginx/sites-enabled - ln -s ../sites-available/pandora - - #read comments in /etc/nginx/sites-available/pandora for setting - #your hostname and other required settings - #make sure XACCELREDIRECT = True in /srv/pandora/pandora/local_settings.py - - service nginx reload - -b) apache2 (if you need it for other sites on the same server) - - apt-get install apache2-mpm-prefork libapache2-mod-xsendfile - a2enmod xsendfile - a2enmod proxy_http - a2enmod proxy_wstunnel - cp /srv/pandora/etc/apache2/pandora.conf /etc/apache2/sites-available/pandora.conf - a2ensite pandora - - #read comments in /etc/apache2/sites-available/pandora.conf for setting - #your hostname and other required settings - #make sure XSENDFILE = True in /srv/pandora/pandora/local_settings.py - - service apache2 reload - - Now you can open pandora in your browser, the first user to sign up will become admin. - -## Updating - - To update pandora to the latest development version run this: - - su pandora - cd /srv/pandora - ./update.py dev - - this will update pandora/oxjs/python-ox and list possible upgrades to the db - - to update your database run: - - su pandora - cd /srv/pandora - ./update.py db - -## Development - - in one terminal: - - cd /srv/pandora/pandora - ./manage.py runserver 2620 - - and background task in another: - - cd /srv/pandora/pandora - ./manage.py celeryd -B -Q celery,default,encoding -l INFO - - now you can access your local pandora instace at http://127.0.0.1:8000/ - - we use virtual machines/lxc for development and deployment, - more info on that in vm/LXC_README.md diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..1360247c --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,91 @@ +version: '3' + +volumes: + pandora: + postgres: + rabbitmq: + +networks: + backend: + +services: + proxy: + build: docker/nginx + ports: + - "127.0.0.1:2620:80" + networks: + - backend + - default + links: + - pandora + - websocketd + depends_on: + - pandora + - websocketd + volumes: + - pandora:/pandora + - ./overlay:/overlay + restart: unless-stopped + + db: + image: postgres:latest + networks: + - backend + env_file: .env + volumes: + - postgres:/var/lib/postgresql/data/ + restart: unless-stopped + + rabbitmq: + hostname: rabbitmq + image: rabbitmq:latest + env_file: .env + networks: + - backend + volumes: + - rabbitmq:/var/lib/rabbitmq + restart: unless-stopped + + pandora: + hostname: pandora + build: . + command: pandora + volumes: + - pandora:/pandora + - ./overlay:/overlay + networks: + - backend + env_file: .env + links: + - db + - rabbitmq + depends_on: + - db + - rabbitmq + restart: unless-stopped + + encoding: &app_base + build: . + command: encoding + env_file: .env + networks: + - backend + volumes: + - pandora:/pandora + - ./overlay:/overlay + restart: unless-stopped + + tasks: + <<: *app_base + command: tasks + + cron: + <<: *app_base + command: cron + + websocketd: + <<: *app_base + command: websocketd + networks: + - backend + diff --git a/docker/base/Dockerfile b/docker/base/Dockerfile new file mode 100644 index 00000000..727891e8 --- /dev/null +++ b/docker/base/Dockerfile @@ -0,0 +1,14 @@ +FROM debian:buster + +LABEL maintainer="0x2620@0x2620.org" + +ENV LANG en_US.UTF-8 + +RUN apt-get update && \ + LANG=C.UTF-8 DEBIAN_FRONTEND=noninteractive apt-get install -y apt-utils locales && \ + echo en_US.UTF-8 UTF-8 > /etc/locale.gen && \ + locale-gen en_US.UTF-8 && \ + update-locale LANG=en_US.UTF-8 + +COPY ./install.sh /install.sh +RUN /install.sh diff --git a/docker/base/install.sh b/docker/base/install.sh new file mode 100755 index 00000000..5c209f07 --- /dev/null +++ b/docker/base/install.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +UBUNTU_CODENAME=bionic +if [ -e /etc/os-release ]; then + . /etc/os-release +fi + +export DEBIAN_FRONTEND=noninteractive +echo 'Acquire::Languages "none";' > /etc/apt/apt.conf.d/99languages +apt-get update -qq +echo "deb http://ppa.launchpad.net/j/pandora/ubuntu ${UBUNTU_CODENAME} main" > /etc/apt/sources.list.d/j-pandora.list +apt-get install -y gnupg2 +apt-key add - < /dev/null 2>&1 +result=$? +if [ $result -eq 0 ] ; then + proxy="--build-arg http_proxy=http://$HOST:$PORT" +else + proxy= +fi + +docker build $proxy -t 0x2620/pandora-base base +docker build -t 0x2620/pandora-nginx nginx +cd .. +docker build -t 0x2620/pandora . diff --git a/docker/dot.env.sample.py b/docker/dot.env.sample.py new file mode 100755 index 00000000..c6f5c2f4 --- /dev/null +++ b/docker/dot.env.sample.py @@ -0,0 +1,30 @@ +#!/usr/bin/python3 +import os +from binascii import hexlify +import string + +chars = string.ascii_letters + string.digits + +def pwgen(length=16): + return ''.join(chars[c % len(chars)] for c in os.urandom(length)) + + +print('''SECRET_KEY={SECRET_KEY} + +POSTGRES_USER=pandora +POSTGRES_PASSWORD={POSTGRES_PASSWORD} + +RABBITMQ_DEFAULT_USER=pandora +RABBITMQ_DEFAULT_PASS={RABBITMQ_PASSWORD} + +# required to send out password reset emails +#EMAIL_HOST='example.com' +#EMAIL_USER='mail@example.com' +#EMAIL_PASSWORD='fixme' +#EMAIL_PORT=587 +#EMAIL_TLS=true +'''.format( + SECRET_KEY=hexlify(os.urandom(64)).decode(), + POSTGRES_PASSWORD=pwgen(), + RABBITMQ_PASSWORD=pwgen() +)) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100755 index 00000000..cd78c6f4 --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,134 @@ +#!/bin/bash +set -e + +action="$1" +user=pandora + +export LANG=en_US.UTF-8 +mkdir -p /run/pandora +chown -R ${user}.${user} /run/pandora + +# pan.do/ra services +if [ "$action" = "pandora" ]; then + if [ ! -e /srv/pandora/initialized ]; then + echo "Setting up Pan.do/ra:" + echo "Waiting for database connection..." + /srv/pandora_base/docker/wait-for db 5432 + echo "Installing pan.do/ra..." + rsync -a /srv/pandora_base/ /srv/pandora/ + + if [ ! -e /overlay/install.py ]; then + rsync -a /srv/pandora_base/docker/overlay/ /overlay/ + if [ ! -e /overlay/config.jsonc ]; then + mv /srv/pandora/pandora/config.jsonc /overlay/config.jsonc + fi + fi + /overlay/install.py + + echo "Initializing database..." + echo "CREATE EXTENSION pg_trgm;" | /srv/pandora/pandora/manage.py dbshell + /srv/pandora/pandora/manage.py init_db + /srv/pandora/update.py db + echo "Generating static files..." + /srv/pandora/update.py static + chown -R ${user}.${user} /srv/pandora/ + touch /srv/pandora/initialized + fi + /srv/pandora_base/docker/wait-for db 5432 + /srv/pandora_base/docker/wait-for rabbitmq 5672 + cd /srv/pandora/pandora + exec /usr/bin/sudo -u $user -E -H \ + /srv/pandora/bin/gunicorn wsgi:application -c gunicorn_config.py +fi +if [ "$action" = "encoding" ]; then + /srv/pandora_base/docker/wait-for-file /srv/pandora/initialized + /srv/pandora_base/docker/wait-for rabbitmq 5672 + name=pandora-encoding-$(hostname) + exec /usr/bin/sudo -u $user -E -H \ + /srv/pandora/bin/python \ + /srv/pandora/pandora/manage.py \ + celery worker \ + -c 1 \ + -Q encoding -n $name \ + -l INFO +fi +if [ "$action" = "tasks" ]; then + /srv/pandora_base/docker/wait-for-file /srv/pandora/initialized + /srv/pandora_base/docker/wait-for rabbitmq 5672 + name=pandora-default-$(hostname) + exec /usr/bin/sudo -u $user -E -H \ + /srv/pandora/bin/python \ + /srv/pandora/pandora/manage.py \ + celery worker \ + -Q default,celery -n $name \ + --maxtasksperchild 1000 \ + -l INFO +fi +if [ "$action" = "cron" ]; then + /srv/pandora_base/docker/wait-for-file /srv/pandora/initialized + /srv/pandora_base/docker/wait-for rabbitmq 5672 + exec /usr/bin/sudo -u $user -E -H \ + /srv/pandora/bin/python \ + /srv/pandora/pandora/manage.py \ + celerybeat -s /run/pandora/celerybeat-schedule \ + --pidfile /run/pandora/cron.pid \ + -l INFO +fi +if [ "$action" = "websocketd" ]; then + /srv/pandora_base/docker/wait-for-file /srv/pandora/initialized + /srv/pandora_base/docker/wait-for rabbitmq 5672 + exec /usr/bin/sudo -u $user -E -H \ + /srv/pandora/bin/python \ + /srv/pandora/pandora/manage.py websocketd +fi + +# pan.do/ra management and update +if [ "$action" = "manage.py" ]; then + shift + exec /usr/bin/sudo -u $user -E -H \ + /srv/pandora/pandora/manage.py "$@" +fi +if [ "$action" = "update.py" ]; then + shift + exec /usr/bin/sudo -u $user -E -H \ + /srv/pandora/update.py "$@" +fi +if [ "$action" = "bash" ]; then + shift + cd / + exec /bin/bash "$@" +fi + +# pan.do/ra setup hooks +if [ "$action" = "docker-compose.yml" ]; then + cat /srv/pandora_base/docker-compose.yml | \ + sed "s#build: \.#image: 0x2620/pandora:latest#g" | \ + sed "s#\./overlay:#.:#g" | \ + sed "s#build: docker/nginx#image: 0x2620/pandora-nginx:latest#g" + exit +fi +if [ "$action" = ".env" ]; then + exec /srv/pandora_base/docker/dot.env.sample.py +fi +if [ "$action" = "config.jsonc" ]; then + cat /srv/pandora_base/pandora/config.pandora.jsonc + exit +fi +if [ "$action" = "setup" ]; then + cat /srv/pandora_base/docker/setup-docker-compose.sh + exit +fi + +# pan.do/ra info +echo pan.do/ra docker container - https://pan.do/ra +echo +echo use this container with docker-compose, +echo to setup a new docker-compose envrionment run: +echo +echo " mkdir && cd " +echo " docker run 0x2620/pandora setup | sh" +echo +echo adjust created files to match your needs and run: +echo +echo " docker-compose up" +echo diff --git a/docker/install.sh b/docker/install.sh new file mode 100755 index 00000000..d1be570f --- /dev/null +++ b/docker/install.sh @@ -0,0 +1,68 @@ +#!/bin/bash +export LANG=en_US.UTF-8 +PANDORA=${PANDORA-pandora} +echo Installing pandora with user: $PANDORA +getent passwd $PANDORA > /dev/null 2>&1 || adduser --disabled-password --gecos "" $PANDORA + +HOST=$(hostname -s) +HOST_CONFIG="/srv/pandora/pandora/config.$HOST.jsonc" +SITE_CONFIG="/srv/pandora/pandora/config.jsonc" +test -e $HOST_CONFIG && cp $HOST_CONFIG $SITE_CONFIG +test -e $SITE_CONFIG || cp /srv/pandora/pandora/config.pandora.jsonc $SITE_CONFIG + +cat > /srv/pandora/pandora/local_settings.py < /usr/local/bin/update.py << EOF +#!/bin/sh +exec /srv/pandora/update.py \$@ +EOF + +cat > /usr/local/bin/manage.py << EOF +#!/bin/sh +exec /srv/pandora/pandora/manage.py \$@ +EOF +chmod +x /usr/local/bin/manage.py /usr/local/bin/update.py diff --git a/docker/nginx/Dockerfile b/docker/nginx/Dockerfile new file mode 100644 index 00000000..c2c9ef58 --- /dev/null +++ b/docker/nginx/Dockerfile @@ -0,0 +1,4 @@ +FROM nginx +LABEL maintainer="0x2620@0x2620.org" +ENV LANG en_US.UTF-8 +COPY nginx.conf /etc/nginx/nginx.conf diff --git a/docker/nginx/nginx.conf b/docker/nginx/nginx.conf new file mode 100644 index 00000000..bb8b6998 --- /dev/null +++ b/docker/nginx/nginx.conf @@ -0,0 +1,96 @@ +worker_processes 4; + +events { worker_connections 1024; } + +http { + include mime.types; + + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 2048; + server_tokens off; + + charset utf-8; + charset_types text/plain text/css application/json text/json application/x-javascript text/xml application/xml application/xml+rss text/javascript application/javascript text/x-js; + + gzip on; + gzip_static on; + gzip_http_version 1.1; + gzip_vary on; + gzip_comp_level 6; + gzip_proxied any; + gzip_types text/plain text/css application/json text/json application/x-javascript text/xml application/xml application/xml+rss text/javascript application/javascript text/x-js; + gzip_buffers 16 8k; + gzip_disable "MSIE [1-6]\.(?!.*SV1)"; + + upstream pandora-web { + server pandora:2620; + } + + upstream pandora-websocket { + server websocketd:2622; + } + + server { + + listen 80 default; + + access_log off; + error_log /var/log/nginx/error.log; + + location /favicon.ico { + root /pandora/static; + } + + location /static/ { + root /pandora; + autoindex off; + } + location /data/ { + internal; + root /pandora; + } + + location /api/ws/ { + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Proxy ""; + proxy_redirect off; + proxy_buffering off; + proxy_read_timeout 999999999; + proxy_pass http://pandora-websocket/; + } + + location / { + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-Proto http; + proxy_set_header Host $http_host; + proxy_set_header Proxy ""; + proxy_redirect off; + proxy_buffering off; + proxy_read_timeout 90; #should be in sync with gunicorn timeout + proxy_connect_timeout 90; #should be in sync with gunicorn timeout + if (!-f $request_filename) { + proxy_pass http://pandora-web; + break; + } + client_max_body_size 32m; + } + + error_page 400 /; + error_page 404 /404.html; + location /404.html { + root /pandora/static/html; + } + + # redirect server error pages to the static page /50x.html + error_page 500 502 503 504 /50x.html; + location /50x.html { + root /pandora/static/html; + } + } +} diff --git a/docker/overlay/__init__.py b/docker/overlay/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/docker/overlay/install.py b/docker/overlay/install.py new file mode 100755 index 00000000..8bf0ed50 --- /dev/null +++ b/docker/overlay/install.py @@ -0,0 +1,91 @@ +#!/usr/bin/python3 + +import os +from os.path import join, abspath, basename, dirname + +# change this +name = 'placeholder' +overwrite = ( + #('home', 'indiancinema'), + #('infoView', 'indiancinema'), +) + +base = abspath(dirname(__file__)) +os.chdir(base) + +for root, folders, files in os.walk(join(base, 'static')): + for f in files: + src = join(root, f) + target = src.replace(base, '/srv/pandora') + rel_src = os.path.relpath(src, dirname(target)) + if os.path.exists(target): + os.unlink(target) + os.symlink(rel_src, target) + +if overwrite: + os.chdir('/srv/pandora/static/js') + for filename, sitename in overwrite: + src = '%s.%s.js' % (filename, sitename) + target = '%s.%s.js' % (filename, name) + if os.path.exists(target): + os.unlink(target) + os.symlink(src, target) + +os.chdir(base) +src = join(base, 'config.jsonc') +target = '/srv/pandora/pandora/config.%s.jsonc' % name +rel_src = os.path.relpath(src, dirname(target)) +if os.path.exists(target): + os.unlink(target) +os.symlink(rel_src, target) +t = '/srv/pandora/pandora/config.jsonc' +if os.path.exists(t): + os.unlink(t) +os.symlink(basename(target), t) + +for root, folders, files in os.walk(join(base, 'scripts')): + for f in files: + src = join(root, f) + target = src.replace(base, '/srv/pandora') + rel_src = os.path.relpath(src, dirname(target)) + if os.path.exists(target): + os.unlink(target) + os.symlink(rel_src, target) + if f == 'poster.%s.py' % name: + t = os.path.join(dirname(target), 'poster.py') + if os.path.exists(t): + os.unlink(t) + os.symlink(f, os.path.join(dirname(target), t)) + +if os.path.exists('settings.py'): + target = os.path.join('/srv/pandora/pandora/overlay_settings.py',) + rel_src = os.path.relpath(os.path.join(base, 'settings.py'), dirname(target)) + if os.path.exists(target): + os.unlink(target) + os.symlink(rel_src, target) + +if os.path.exists('__init__.py'): + # make module available to pandora + target = os.path.join('/srv/pandora/pandora/', name) + rel_src = os.path.relpath(base, dirname(target)) + if os.path.exists(target): + os.unlink(target) + os.symlink(rel_src, target) + + # include module in local settings + local_settings_py = '/srv/pandora/pandora/local_settings.py' + with open(local_settings_py) as fd: + local_settings_changed = False + local_settings = fd.read() + if 'LOCAL_APPS' not in local_settings: + local_settings += '\nLOCAL_APPS = ["%s"]\n' % name + local_settings_changed = True + else: + apps = re.compile('(LOCAL_APPS.*?)\]', re.DOTALL).findall(local_settings)[0] + if name not in apps: + new_apps = apps.strip() + ',\n"%s"\n' % name + local_settings = local_settings.replace(apps, new_apps) + local_settings_changed = True + if local_settings_changed: + with open(local_settings_py, 'w') as fd: + fd.write(local_settings) diff --git a/docker/overlay/settings.py b/docker/overlay/settings.py new file mode 100644 index 00000000..436f6acf --- /dev/null +++ b/docker/overlay/settings.py @@ -0,0 +1 @@ +# local settings go here diff --git a/docker/setup-docker-compose.sh b/docker/setup-docker-compose.sh new file mode 100755 index 00000000..7665e7b8 --- /dev/null +++ b/docker/setup-docker-compose.sh @@ -0,0 +1,32 @@ +#!/bin/sh +docker run 0x2620/pandora docker-compose.yml > docker-compose.yml +if [ ! -e .env ]; then + docker run 0x2620/pandora .env > .env + echo .env >> .gitignore +fi +if [ ! -e config.jsonc ]; then + docker run 0x2620/pandora config.jsonc > config.jsonc +fi +cat > README.md << EOF +pan.do/ra docker instance + +this folder was created with + + docker run 0x2620/pandora setup | sh + +To start pan.do/ra adjust the files in this folder: + + - add email configuration to .env + - adjust config.jsonc to customize pan.do/ra + - add local django settings to settings.py + +and to get started run this: + + docker-compose up -d + +To update pan.do/ra run: + + docker-compose run pandora update.py + +EOF +touch __init__.py diff --git a/docker/wait-for b/docker/wait-for new file mode 100755 index 00000000..d531df08 --- /dev/null +++ b/docker/wait-for @@ -0,0 +1,17 @@ +#!/bin/sh + +TIMEOUT=15 + +HOST="$1" +PORT="$2" + +for i in `seq $TIMEOUT` ; do + nc -z "$HOST" "$PORT" > /dev/null 2>&1 + result=$? + if [ $result -eq 0 ] ; then + exit 0 + fi + sleep 1 +done +echo "Failed to connect to database at $HOST:$PORT" >&2 +exit 1 diff --git a/docker/wait-for-file b/docker/wait-for-file new file mode 100755 index 00000000..83b043e4 --- /dev/null +++ b/docker/wait-for-file @@ -0,0 +1,12 @@ +#!/bin/sh +TIMEOUT=60 +TARGET="$1" + +for i in `seq $TIMEOUT` ; do + if [ -e "$TARGET" ]; then + exit 0 + fi + sleep 1 +done +echo "Giving up waiting for file $TARGET" >&2 +exit 1 diff --git a/pandora/archive/tasks.py b/pandora/archive/tasks.py index b490ee18..e9cc8712 100644 --- a/pandora/archive/tasks.py +++ b/pandora/archive/tasks.py @@ -210,6 +210,7 @@ def download_media(item_id, url): def move_media(data, user): from changelog.models import add_changelog from item.models import get_item, Item + from annotation.models import Annotation user = models.User.objects.get(username=user) @@ -228,12 +229,26 @@ def move_media(data, user): else: i = get_item({'imdbId': data['public_id']}, user=user) changed = [i.public_id] + old_item = None for f in models.File.objects.filter(oshash__in=data['ids']): - if f.item.id != i.public_id and f.editable(user): + if f.item.public_id != i.public_id and f.editable(user): if f.item.public_id not in changed: changed.append(f.item.public_id) + old_item = f.item f.item = i f.save() + + if old_item: + data['from'] = old_item.public_id + + # If all files are moved to a new item, keep annotations + if old_item and old_item.files.count() == 0 and i.files.count() == len(data['ids']): + for a in old_item.annotations.all().order_by('id'): + a.item = i + a.set_public_id() + Annotation.objects.filter(id=a.id).update(item=i, public_id=a.public_id) + old_item.clips.all().update(item=i, sort=i.sort) + for public_id in changed: c = Item.objects.get(public_id=public_id) if c.files.count() == 0 and settings.CONFIG['itemRequiresVideo']: diff --git a/pandora/config.0xdb.jsonc b/pandora/config.0xdb.jsonc index f7f91cf9..7b300891 100644 --- a/pandora/config.0xdb.jsonc +++ b/pandora/config.0xdb.jsonc @@ -165,6 +165,7 @@ "type": "string", "columnWidth": 120, //"format": {"type": "date", "args": ["%a, %b %e, %Y"]}, + "filter": true, "sort": true }, { @@ -220,6 +221,7 @@ }, { "id": "size", + "filter": true, "operator": "-", "title": "Size", "type": "integer", @@ -1230,11 +1232,19 @@ }, "document": "", "documents": {}, - "documentView": "view", + "documentFiltersSize": 176, "documentSize": 256, "documentsSelection": {}, "documentsSort": [{"key": "name", "operator": "+"}], "documentsView": "grid", + "documentView": "view", + "documentFilters": [ + {"id": "author", "sort": [{"key": "items", "operator": "-"}]}, + {"id": "place", "sort": [{"key": "items", "operator": "-"}]}, + {"id": "date", "sort": [{"key": "name", "operator": "-"}]}, + {"id": "publisher", "sort": [{"key": "items", "operator": "-"}]}, + {"id": "language", "sort": [{"key": "items", "operator": "-"}]} + ], "edit": "", "edits": {}, "editSelection": [], @@ -1303,6 +1313,7 @@ "showCalendarControls": true, // fixme: should be false "showClips": true, "showDocument": true, + "showDocumentFilters": false, "showFilters": true, "showIconBrowser": false, "showInfo": true, diff --git a/pandora/config.indiancinema.jsonc b/pandora/config.indiancinema.jsonc index e8673197..be2a79db 100644 --- a/pandora/config.indiancinema.jsonc +++ b/pandora/config.indiancinema.jsonc @@ -212,6 +212,7 @@ "type": "string", "columnWidth": 120, //"format": {"type": "date", "args": ["%a, %b %e, %Y"]}, + "filter": true, "sort": true }, { @@ -238,6 +239,14 @@ "find": true, "sort": true }, + { + "id": "keywords", + "title": "Keywords", + "type": ["string"], + "columnWidth": 128, + "filter": true, + "find": true + }, { "id": "id", "operator": "+", @@ -1709,7 +1718,15 @@ } }, "documentView": "view", + "documentFilters": [ + {"id": "author", "sort": [{"key": "items", "operator": "-"}]}, + {"id": "place", "sort": [{"key": "items", "operator": "-"}]}, + {"id": "date", "sort": [{"key": "name", "operator": "-"}]}, + {"id": "publisher", "sort": [{"key": "items", "operator": "-"}]}, + {"id": "language", "sort": [{"key": "items", "operator": "-"}]} + ], "documents": {}, + "documentFiltersSize": 176, "documentSize": 256, "documentsSelection": {}, "documentsSort": [{"key": "title", "operator": "+"}], @@ -1782,6 +1799,7 @@ "showCalendarControls": true, // fixme: should be false "showClips": true, "showDocument": true, + "showDocumentFilters": false, "showFilters": true, "showIconBrowser": false, "showInfo": true, diff --git a/pandora/config.padma.jsonc b/pandora/config.padma.jsonc index e2f75a3a..e797cd15 100644 --- a/pandora/config.padma.jsonc +++ b/pandora/config.padma.jsonc @@ -210,6 +210,7 @@ "type": "string", "columnWidth": 120, //"format": {"type": "date", "args": ["%a, %b %e, %Y"]}, + "filter": true, "sort": true }, { @@ -236,6 +237,14 @@ "find": true, "sort": true }, + { + "id": "keywords", + "title": "Keywords", + "type": ["string"], + "columnWidth": 128, + "filter": true, + "find": true + }, { "id": "id", "operator": "+", @@ -1175,11 +1184,19 @@ }, "document": "", "documents": {}, + "documentFiltersSize": 176, "documentSize": 256, "documentView": "view", "documentsSelection": {}, "documentsSort": [{"key": "title", "operator": "+"}], "documentsView": "grid", + "documentFilters": [ + {"id": "author", "sort": [{"key": "items", "operator": "-"}]}, + {"id": "place", "sort": [{"key": "items", "operator": "-"}]}, + {"id": "date", "sort": [{"key": "name", "operator": "-"}]}, + {"id": "publisher", "sort": [{"key": "items", "operator": "-"}]}, + {"id": "language", "sort": [{"key": "items", "operator": "-"}]} + ], "edit": "", "edits": {}, "editSelection": [], @@ -1244,6 +1261,7 @@ "showCalendarControls": true, // fixme: should be false "showClips": true, "showDocument": true, + "showDocumentFilters": false, "showFilters": true, "showIconBrowser": false, "showInfo": true, diff --git a/pandora/config.pandora.jsonc b/pandora/config.pandora.jsonc index 90e97086..9658b265 100644 --- a/pandora/config.pandora.jsonc +++ b/pandora/config.pandora.jsonc @@ -217,6 +217,7 @@ examples (config.SITENAME.jsonc) that are part of this pan.do/ra distribution. "type": "string", "columnWidth": 120, //"format": {"type": "date", "args": ["%a, %b %e, %Y"]}, + "filter": true, "sort": true }, { @@ -1122,11 +1123,19 @@ examples (config.SITENAME.jsonc) that are part of this pan.do/ra distribution. }, "document": "", "documents": {}, + "documentFiltersSize": 176, "documentSize": 256, "documentView": "view", "documentsSelection": {}, "documentsSort": [{"key": "title", "operator": "+"}], "documentsView": "grid", + "documentFilters": [ + {"id": "author", "sort": [{"key": "items", "operator": "-"}]}, + {"id": "place", "sort": [{"key": "items", "operator": "-"}]}, + {"id": "date", "sort": [{"key": "name", "operator": "-"}]}, + {"id": "publisher", "sort": [{"key": "items", "operator": "-"}]}, + {"id": "language", "sort": [{"key": "items", "operator": "-"}]} + ], "edit": "", "edits": {}, "editSelection": [], @@ -1191,6 +1200,7 @@ examples (config.SITENAME.jsonc) that are part of this pan.do/ra distribution. "showCalendarControls": false, "showClips": true, "showDocument": true, + "showDocumentFilters": false, "showFilters": true, "showIconBrowser": false, "showInfo": true, diff --git a/pandora/document/models.py b/pandora/document/models.py index 268302b5..81fd4dd1 100644 --- a/pandora/document/models.py +++ b/pandora/document/models.py @@ -192,7 +192,7 @@ class Document(models.Model): sort_value = u'; '.join([get_name_sort(name) for name in values]) if not sort_value: sort_value = u'' - return sort_value + return sort_value.lower() def set_value(s, name, value): if isinstance(value, string_types): @@ -403,6 +403,13 @@ class Document(models.Model): 'rightslevel', ): return getattr(self, key) + document_key = utils.get_by_id(settings.CONFIG['documentKeys'], key) + if document_key and 'value' in document_key \ + and isinstance(document_key['value'], dict) \ + and document_key['value'].get('type') == 'map' \ + and self.get_value(document_key['value']['key']): + value = re.compile(document_key['value']['map']).findall(self.get_value(document_key['value']['key'])) + return value[0] if value else default elif key == 'user': return self.user.username else: diff --git a/pandora/document/utils.py b/pandora/document/utils.py index 3df53197..c579f0dc 100644 --- a/pandora/document/utils.py +++ b/pandora/document/utils.py @@ -26,6 +26,7 @@ def extract_pdfpage(pdf, image, page): page = str(page) cmd = [ 'pdftocairo', + '-cropbox', '-jpeg', '-f', page, '-l', page, '-singlefile', diff --git a/pandora/documentcollection/migrations/0004_jsonfield.py b/pandora/documentcollection/migrations/0004_jsonfield.py new file mode 100644 index 00000000..d76aa109 --- /dev/null +++ b/pandora/documentcollection/migrations/0004_jsonfield.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.4 on 2018-06-19 17:23 +from __future__ import unicode_literals + +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models + +def remove_posterframes(apps, schema_editor): + Collection = apps.get_model("documentcollection", "Collection") + for c in Collection.objects.exclude(poster_frames=None): + c.poster_frames = [] + c.save() + +class Migration(migrations.Migration): + + dependencies = [ + ('documentcollection', '0003_jsonfield'), + ] + + operations = [ + migrations.RunPython(remove_posterframes), + ] diff --git a/pandora/documentcollection/models.py b/pandora/documentcollection/models.py index 014f6bec..e0295c47 100644 --- a/pandora/documentcollection/models.py +++ b/pandora/documentcollection/models.py @@ -265,38 +265,23 @@ class Collection(models.Model): def update_icon(self): frames = [] - #fixme - ''' if not self.poster_frames: - documents = self.get_documents(self.user) + documents = self.get_documents(self.user).all() if documents.count(): poster_frames = [] for i in range(0, documents.count(), max(1, int(documents.count()/4))): poster_frames.append({ - 'document': documents[int(i)].id, - 'position': documents[int(i)].poster_frame + 'document': documents[int(i)].get_id(), }) self.poster_frames = tuple(poster_frames) self.save() for i in self.poster_frames: from document.models import Document - qs = Document.objects.filter(id=i['document']) - if qs.count() > 0: - if i.get('position'): - frame = qs[0].frame(i['position']) - if frame: - frames.append(frame) - ''' - from item.models import Item - for i in self.poster_frames: - try: - qs = Item.objects.filter(public_id=i['item']) + if 'document' in i: + qs = Document.objects.filter(id=ox.fromAZ(i['document'])) if qs.count() > 0: - frame = qs[0].frame(i['position']) - if frame: - frames.append(frame) - except: - pass + frame = qs[0].thumbnail(size=1024, page=i.get('page')) + frames.append(frame) self.icon.name = self.path('icon.jpg') icon = self.icon.path if frames: diff --git a/pandora/event/models.py b/pandora/event/models.py index 1e992557..e66c86fe 100644 --- a/pandora/event/models.py +++ b/pandora/event/models.py @@ -129,7 +129,7 @@ class Event(models.Model): value = get_name_sort(value) else: value = get_title_sort(value) - self.name_sort = utils.sort_string(value) + self.name_sort = utils.sort_string(value).lower() def save(self, *args, **kwargs): if not self.name_sort: diff --git a/pandora/item/models.py b/pandora/item/models.py index d15b30f6..5dae6d91 100644 --- a/pandora/item/models.py +++ b/pandora/item/models.py @@ -478,6 +478,11 @@ class Item(models.Model): a.item = other a.set_public_id() Annotation.objects.filter(id=a.id).update(item=other, public_id=a.public_id) + try: + other_sort = other.sort + except: + other_sort = None + self.clips.all().update(item=other, sort=other_sort) if hasattr(self, 'files'): for f in self.files.all(): @@ -487,6 +492,8 @@ class Item(models.Model): if save: other.save() # FIXME: update poster, stills and streams after this + if other_sort is None: + other.clips.all().update(sort=other.sort) def merge_streams(self, output, resolution=None, format="webm"): streams = [s.get(resolution, format).media.path for s in self.streams()] @@ -902,7 +909,7 @@ class Item(models.Model): sort_value = u'; '.join([get_name_sort(name) for name in values]) if not sort_value: sort_value = u'' - return sort_value + return sort_value.lower() def set_value(s, name, value): if isinstance(value, string_types): @@ -1046,8 +1053,7 @@ class Item(models.Model): pass else: continue - if value_ is not None: - set_value(s, name, value_) + set_value(s, name, value_) s.save() def update_facet(self, key): @@ -1355,7 +1361,7 @@ class Item(models.Model): qs = qs.order_by('file__part', 'file__sort_path') return qs - def update_timeline(self, async=True): + def update_timeline(self, async_=True): streams = self.streams() self.make_timeline() if streams.count() == 1: @@ -1385,7 +1391,7 @@ class Item(models.Model): self.rendered = streams.count() > 0 self.save() if self.rendered: - if async: + if async_: get_sequences.delay(self.public_id) else: get_sequences(self.public_id) diff --git a/pandora/item/tasks.py b/pandora/item/tasks.py index 0bda0651..a0c9cac5 100644 --- a/pandora/item/tasks.py +++ b/pandora/item/tasks.py @@ -102,7 +102,7 @@ def update_timeline(public_id): item = models.Item.objects.get(public_id=public_id) except models.Item.DoesNotExist: return - item.update_timeline(async=False) + item.update_timeline(async_=False) Task.finish(item) @task(queue="encoding") @@ -111,7 +111,7 @@ def rebuild_timeline(public_id): i = models.Item.objects.get(public_id=public_id) for s in i.streams(): s.make_timeline() - i.update_timeline(async=False) + i.update_timeline(async_=False) @task(queue="encoding") def load_subtitles(public_id): diff --git a/pandora/person/models.py b/pandora/person/models.py index a98bd68b..20c637c4 100644 --- a/pandora/person/models.py +++ b/pandora/person/models.py @@ -50,18 +50,19 @@ class Person(models.Model): if not self.sortname: self.sortname = ox.get_sort_name(self.name) self.sortname = unicodedata.normalize('NFKD', self.sortname) - self.sortsortname = utils.sort_string(self.sortname) + self.sortsortname = utils.sort_string(self.sortname).lower() self.numberofnames = len(self.name.split(' ')) super(Person, self).save(*args, **kwargs) def update_itemsort(self): + sortname = self.sortname.lower() item.models.Facet.objects.filter( key__in=item.models.Item.person_keys + ['name'], value=self.name ).exclude( - sortvalue=self.sortname + sortvalue=sortname ).update( - sortvalue=self.sortname + sortvalue=sortname ) for i in item.models.Item.objects.filter(facets__in=item.models.Facet.objects.filter( key__in=item.models.Item.person_keys + ['name'], diff --git a/requirements.txt b/requirements.txt index a3b5c98f..1881524a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Django==1.11.13 +Django==1.11.22 simplejson chardet celery==3.1.26.post2 @@ -6,8 +6,8 @@ django-celery==3.2.2 django-extensions==2.0.7 gunicorn==19.8.1 html5lib -requests==2.19.1 +requests==2.22.0 tornado<5 geoip2==2.9.0 -youtube-dl +youtube-dl>=2019.4.30 python-memcached diff --git a/static/js/UI.js b/static/js/UI.js index 149a0e81..94979e69 100644 --- a/static/js/UI.js +++ b/static/js/UI.js @@ -24,6 +24,9 @@ pandora.UI = (function() { pandora.user.ui._filterState = pandora.getFilterState( pandora.user.ui.find ); + pandora.user.ui._documentFilterState = pandora.getDocumentFilterState( + pandora.user.ui.findDocuments + ); pandora.user.ui._findState = pandora.getFindState( pandora.user.ui.find ); @@ -85,6 +88,7 @@ pandora.UI = (function() { // (values we put in add will be changed, but won't trigger) collection = pandora.getCollectionState(args.findDocuments); pandora.user.ui._collection = collection; + pandora.user.ui._documentFilterState = pandora.getDocumentFilterState(args.findDocuments); pandora.user.ui._findDocumentsState = pandora.getFindDocumentsState(args.findDocuments); if (pandora.$ui.appPanel && !pandora.stayInItemView) { // if we're not on page load, and if find isn't a context change diff --git a/static/js/URL.js b/static/js/URL.js index f31f6f15..4a08db11 100644 --- a/static/js/URL.js +++ b/static/js/URL.js @@ -144,6 +144,7 @@ pandora.URL = (function() { pandora.user.ui._list = pandora.getListState(pandora.user.ui.find); pandora.user.ui._filterState = pandora.getFilterState(pandora.user.ui.find); + pandora.user.ui._documentFilterState = pandora.getDocumentFilterState(pandora.user.ui.findDocuments); pandora.user.ui._findState = pandora.getFindState(pandora.user.ui.find); pandora.user.ui._collection = pandora.getCollectionState(pandora.user.ui.findDocuments); pandora.user.ui._findDocumentsState = pandora.getFindDocumentsState(pandora.user.ui.findDocuments); diff --git a/static/js/addItemDialog.js b/static/js/addItemDialog.js index 4bdef958..465a8da7 100644 --- a/static/js/addItemDialog.js +++ b/static/js/addItemDialog.js @@ -107,7 +107,7 @@ pandora.ui.addItemDialog = function(options) { pandora.api.getMediaUrlInfo({ url: input }, function(result) { - if (requslt.data.items.length) { + if (result.data.items.length) { onInfo(result.data.items.map(getVideoInfo)); } else { $screen.stop(); diff --git a/static/js/browser.js b/static/js/browser.js index 323011d0..4f86c114 100644 --- a/static/js/browser.js +++ b/static/js/browser.js @@ -52,7 +52,7 @@ pandora.ui.browser = function() { selected: selected }); } - }).reloadList(); + }).reloadList(); } }); pandora.$ui.filters.updateMenus(); diff --git a/static/js/collection.js b/static/js/collection.js index d8296de4..51017b50 100644 --- a/static/js/collection.js +++ b/static/js/collection.js @@ -13,6 +13,7 @@ pandora.ui.collection = function() { if (view == 'list') { that = Ox.TableList({ + draggable: true, keys: keys, items: function(data, callback) { pandora.api.findDocuments(Ox.extend(data, { diff --git a/static/js/collectionIconPanel.js b/static/js/collectionIconPanel.js new file mode 100644 index 00000000..35007d84 --- /dev/null +++ b/static/js/collectionIconPanel.js @@ -0,0 +1,249 @@ +'use strict'; + +pandora.ui.collectionIconPanel = function(listData) { + + var quarter = 0, + quarters = ['top-left', 'top-right', 'bottom-left', 'bottom-right'], + + ui = pandora.user.ui, + folderItems = pandora.getFolderItems(ui.section), + folderItem = folderItems.slice(0, -1), + + + $iconPanel = Ox.Element(), + + $icon = $('') + .attr({ + src: pandora.getListIcon(ui.section, listData.id, 256) + }) + .css({position: 'absolute', borderRadius: '64px', margin: '16px'}) + .appendTo($iconPanel), + + $previewPanel = Ox.Element(), + + $preview, + + $list = Ox.Element(), + + ui = pandora.user.ui, + + that = Ox.SplitPanel({ + elements: [ + { + element: $iconPanel, + size: 280 + }, + { + element: $previewPanel + }, + { + element: $list, + size: 144 + Ox.UI.SCROLLBAR_SIZE + } + ], + orientation: 'horizontal' + }); + + pandora.api['find' + folderItems]({ + query: { + conditions: [{key: 'id', value: listData.id, operator: '=='}], + operator: '&' + }, + keys: ['posterFrames'] + }, function(result) { + + var posterFrames = result.data.items[0].posterFrames, + posterFrame = posterFrames[quarter], + + $interface = Ox.Element({ + tooltip: function(e) { + var quarterName = ($(e.target).attr('id') || '').replace('-', ' '); + return quarterName ? Ox._('Edit ' + quarterName + ' image') : null; + } + }) + .css({ + position: 'absolute', + width: '256px', + height: '256px', + marginLeft: '16px', + marginTop: '16px', + cursor: 'pointer' + }) + .on({ + click: function(e) { + clickIcon(e); + }, + dblclick: function(e) { + clickIcon(e, true); + } + }) + .appendTo($iconPanel); + + renderQuarters(); + + $list = Ox.IconList({ + borderRadius: 16, + item: function(data, sort) { + var infoKey = ['title', 'author'].indexOf(sort[0].key) > -1 + ? pandora.site.documentKeys.filter(function(key) { + return ['year', 'date'].indexOf(key.id) > -1 + }).map(function(key) { + return key.id; + })[0] : sort[0],key, + size = 128; + return { + height: size, + id: data.id, + info: data[infoKey] || '', + title: data.title, + url: pandora.getMediaURL('/documents/' + data.id + '/' + size + 'p.jpg?' + data.modified), + width: size + }; + }, + items: function(data, callback) { + var listData = pandora.getListData(); + pandora.api.findDocuments(Ox.extend(data, { + query: { + conditions: [{key: 'collection', value: listData.id, operator: '=='}], + operator: '&' + } + }), callback); + }, + keys: ['duration', 'id', 'modified', 'title'], + max: 1, + min: 1, + //orientation: 'vertical', + selected: posterFrame ? [posterFrame.document] : [], + size: 128, + sort: ui.collectionSort, + unique: 'id' + }) + //.css({width: '144px'}) + .bindEvent({ + open: function(data) { + setPosterFrame(data.ids[0], $list.value(data.ids[0], 'posterFrame')) + }, + select: function(data) { + renderPreview($list.value(data.ids[0])); + } + }) + .bindEventOnce({ + load: function() { + var itemData; + if (!posterFrame) { + itemData = $list.value(0); + $list.options({selected: [itemData.id]}); + } else { + itemData = $list.value(posterFrame.item); + } + itemData && renderPreview(itemData); + } + }) + .gainFocus(); + + that.replaceElement(2, $list); + + function clickIcon(e, isDoubleClick) { + quarter = quarters.indexOf($(e.target).attr('id')); + renderQuarters(); + if (isDoubleClick && posterFrames.length) { + var item = posterFrames[quarter].item; + $list.options({selected: [item]}); + renderPreview($list.value(item), posterFrames[quarter].page); + } + } + + function renderPreview(itemData, page) { + var size = 256; + if (itemData.id) { + $preview = Ox.Element('').attr({ + src: pandora.getMediaURL('/documents/' + itemData.id + '/' + size + 'p.jpg?' + itemData.modified), + }) + .css({ + width: size + 'px', + height: size + 'px', + marginLeft: '8px', marginTop: '16px', overflow: 'hidden' + }) + .on({ + click: function(d) { + setPosterFrame(itemData.id); + } + }); + } else { + $preview = Ox.Element() + } + $previewPanel.empty().append($preview); + } + + function renderQuarters() { + $interface.empty(); + quarters.forEach(function(q, i) { + $interface.append( + $('
') + .attr({id: q}) + .css({ + float: 'left', + width: '126px', + height: '126px', + border: '1px solid rgba(255, 255, 255, ' + (i == quarter ? 0.75 : 0) + ')', + background: 'rgba(0, 0, 0, ' + (i == quarter ? 0 : 0.75) + ')' + }) + .css('border-' + q + '-radius', '64px') + ); + }); + } + + function setPosterFrame(document, page) { + var posterFrame = {document: document, page: page}; + if (posterFrames.length) { + posterFrames[quarter] = posterFrame; + } else { + posterFrames = Ox.range(4).map(function() { return Ox.clone(posterFrame); } ); + } + pandora.api['edit' + folderItem]({ + id: listData.id, + posterFrames: posterFrames + }, function() { + $icon.attr({ + src: pandora.getListIcon(ui.section, listData.id, 256) + }); + pandora.$ui.folderList[listData.folder].$element + .find('img[src*="' + + pandora.getMediaURL('/' + encodeURIComponent(listData.id)) + + '/"]' + ) + .attr({ + src: pandora.getListIcon(ui.section, listData.id, 256) + }); + pandora.$ui.info.updateListInfo(); + }); + $preview.options({page: page}); + } + + }); + + function renderFrame() { + $frame.css({borderRadius: 0}); + $frame.css('border-' + quarters[quarter] + '-radius', '128px'); + } + + that.updateQuery = function(key, value) { + $list.options({ + items: function(data, callback) { + pandora.api.findDocuments(Ox.extend(data, { + query: { + conditions: [{key: 'collection', value: listData.id, operator: '=='}].concat( + value !== '' + ? [{key: key, value: value, operator: '='}] + : [] + ), + operator: '&' + } + }), callback); + } + }); + }; + + return that; + +} diff --git a/static/js/documentBrowser.js b/static/js/documentBrowser.js index 3bcb2699..0895b8fc 100644 --- a/static/js/documentBrowser.js +++ b/static/js/documentBrowser.js @@ -3,7 +3,55 @@ pandora.ui.documentBrowser = function() { var that; if (!pandora.user.ui.document) { - that = Ox.Element().html('fixme'); + pandora.user.ui.filterSizes = pandora.getFilterSizes(); + pandora.$ui.documentFilters = pandora.ui.documentFilters(); + that = Ox.SplitPanel({ + elements: [ + { + element: pandora.$ui.documentFilters[0], + size: pandora.user.ui.filterSizes[0] + }, + { + element: pandora.$ui.documentFiltersInnerPanel = pandora.ui.documentFiltersInnerPanel() + }, + { + element: pandora.$ui.documentFilters[4], + size: pandora.user.ui.filterSizes[4] + }, + ], + id: 'browser', + orientation: 'horizontal' + }) + .bindEvent({ + resize: function(data) { + pandora.$ui.documentFilters.forEach(function(list) { + list.size(); + }); + }, + resizeend: function(data) { + pandora.UI.set({documentFiltersSize: data.size}); + }, + toggle: function(data) { + data.collapsed && pandora.$ui.list.gainFocus(); + pandora.UI.set({showDocumentFilters: !data.collapsed}); + if (!data.collapsed) { + pandora.$ui.documentFilters.forEach(function($documentFilter) { + var selected = $documentFilter.options('_selected'); + if (selected) { + $documentFilter.bindEventOnce({ + load: function() { + $documentFilter.options({ + _selected: false, + selected: selected + }); + } + }).reloadList(); + } + }); + pandora.$ui.documentFilters.updateMenus(); + } + } + }); } else { var that = Ox.IconList({ borderRadius: 0, diff --git a/static/js/documentContentPanel.js b/static/js/documentContentPanel.js index 6e5703de..b1bbc5d0 100644 --- a/static/js/documentContentPanel.js +++ b/static/js/documentContentPanel.js @@ -4,16 +4,14 @@ pandora.ui.documentContentPanel = function() { var that = Ox.SplitPanel({ elements: !pandora.user.ui.document ? [ { - collapsed: true, - collapsible: false, //fixme + collapsed: !pandora.user.ui.showDocumentFilters, + collapsible: true, element: pandora.$ui.documentBrowser = pandora.ui.documentBrowser(), - resizable: false, //fixme + resizable: true, resize: [96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256], - size: 96, - tooltip: '' /* fixme: - Ox._('filters') + ' ' - + Ox.SYMBOLS.shift + 'F' - */ + size: pandora.user.ui.documentFiltersSize, + tooltip: Ox._('filters') + ' ' + + Ox.SYMBOLS.shift + 'F' }, { element: pandora.$ui.list = pandora.ui.collection() diff --git a/static/js/documentDialog.js b/static/js/documentDialog.js index 8afacc37..a8d1d581 100644 --- a/static/js/documentDialog.js +++ b/static/js/documentDialog.js @@ -196,7 +196,7 @@ pandora.ui.documentDialog = function(options) { ? pandora.user.ui.documents[item.id].position : 1, url: '/documents/' + item.id + '/' - + item.title + '.' + item.extension, + + item.title.replace('?', '_') + '.' + item.extension, width: dialogWidth, zoom: 'fit' }) diff --git a/static/js/documentFilter.js b/static/js/documentFilter.js new file mode 100644 index 00000000..960eca09 --- /dev/null +++ b/static/js/documentFilter.js @@ -0,0 +1,337 @@ +'use strict'; + +pandora.ui.documentFilter = function(id) { + var i = Ox.getIndexById(pandora.user.ui.documentFilters, id), + filter = Ox.getObjectById(pandora.site.documentFilters, id), + panelWidth = Ox.$document.width() - (pandora.user.ui.showSidebar * pandora.user.ui.sidebarSize) - 1, + title = Ox._(Ox.getObjectById(pandora.site.documentFilters, id).title), + //width = pandora.getFilterWidth(i, panelWidth), + that = Ox.TableList({ + _selected: !pandora.user.ui.showFilters + ? pandora.user.ui._documentFilterState[i].selected + : false, + columns: [ + { + align: 'left', + id: 'name', + format: function(value) { + var layer = Ox.getObjectById(pandora.site.layers, filter.id), + key = Ox.getObjectById(pandora.site.itemKeys, filter.id); + if ((layer && layer.translate) || (key && key.translate)) { + value = Ox._(value) + } + return filter.flag + ? $('
') + .append( + $('') + .attr({src: Ox[ + filter.flag == 'country' + ? 'getFlagByGeoname' + : 'getFlagByLanguage' + ](value, 16)}) + .css({ + float: 'left', + width: '14px', + height: '14px', + margin: '0 3px 0 -2px', + borderRadius: '4px' + }) + ) + .append( + $('
') + .addClass('flagname') + .css({ + float: 'left', + width: pandora.user.ui.filterSizes[i] + - 68 - Ox.UI.SCROLLBAR_SIZE, + textOverflow: 'ellipsis', + overflowX: 'hidden' + }) + .html(value) + ) + : value + }, + operator: filter.type == 'string' || filter.type == 'layer' ? '+' : '-', + title: title, + visible: true, + width: pandora.user.ui.filterSizes[i] - 44 - Ox.UI.SCROLLBAR_SIZE + }, + { + align: 'right', + format: function(value) { + return Ox.formatNumber(value); + }, + id: 'items', + operator: '-', + title: '#', + visible: true, + width: 44 + } + ], + columnsVisible: true, + id: 'filter_' + id, + items: function(data, callback) { + if (pandora.user.ui.showFilters) { + delete data.keys; + return pandora.api.findDocuments(Ox.extend(data, { + group: id, + query: pandora.user.ui._documentFilterState[i].find + }), callback); + } else { + callback({data: {items: data.keys ? [] : 0}}); + } + }, + scrollbarVisible: true, + selected: pandora.user.ui.showFilters + ? pandora.user.ui._documentFilterState[i].selected + : [], + sort: [{ + key: pandora.user.ui.documentFilters[i].sort[0].key, + operator: pandora.user.ui.documentFilters[i].sort[0].operator + }], + unique: 'name' + }) + .bindEvent({ + init: function(data) { + that.setColumnTitle( + 'name', + Ox._(Ox.getObjectById(pandora.site.documentFilters, id).title) + + '
' + + Ox.formatNumber(data.items) + '
' + ); + }, + paste: function(data) { + pandora.$ui.list.triggerEvent('paste', data); + }, + select: function(data) { + // fixme: cant index be an empty array, instead of -1? + // FIXME: this is still incorrect when deselecting a filter item + // makes a selected item in another filter disappear + var conditions = data.ids.map(function(value) { + return { + key: id, + value: value, + operator: '==' + }; + }), + index = pandora.user.ui._documentFilterState[i].index, + find = Ox.clone(pandora.user.ui.findDocuments, true); + if (Ox.isArray(index)) { + // this filter had multiple selections and the | query + // was on the top level, i.e. not bracketed + find = { + conditions: conditions, + operator: conditions.length > 1 ? '|' : '&' + } + } else { + if (index == -1) { + // this filter had no selection, i.e. no query + index = find.conditions.length; + if (find.operator == '|') { + find = { + conditions: [find], + operator: '&' + }; + index = 1; + } else { + find.operator = '&'; + } + } + if (conditions.length == 0) { + // nothing selected + find.conditions.splice(index, 1); + if (find.conditions.length == 1) { + if (find.conditions[0].conditions) { + // unwrap single remaining bracketed query + find = { + conditions: find.conditions[0].conditions, + operator: '|' + }; + } else { + find.operator = '&'; + } + } + } else if (conditions.length == 1) { + // one item selected + find.conditions[index] = conditions[0]; + } else { + // multiple items selected + if (pandora.user.ui.findDocuments.conditions.length == 1) { + find = { + conditions: conditions, + operator: '|' + }; + } else { + find.conditions[index] = { + conditions: conditions, + operator: '|' + }; + } + } + } + pandora.UI.set({findDocuments: find}); + pandora.$ui.documentFilters.updateMenus(); + }, + sort: function(data) { + Ox.Log('', 'SORT', data) + var filters = Ox.clone(pandora.user.ui.documentFilters, true); + /* + pandora.$ui.mainMenu.checkItem('sortMenu_sortfilters_sortfilter' + id + '_' + data.key); + pandora.$ui.mainMenu.checkItem('sortMenu_orderfilters_orderfilter' + id + '_' + (data.operator == '+' ? 'ascending' : 'descending')); + */ + filters[i].sort = [{key: data.key, operator: data.operator}]; + pandora.UI.set({documentFilters: filters}); + } + }), + $menu = Ox.MenuButton({ + items: [ + {id: 'clearFilter', title: Ox._('Clear Filter'), keyboard: 'shift control a'}, + {id: 'clearFilters', title: Ox._('Clear All Filters'), keyboard: 'shift alt control a'}, + {}, + {group: 'filter', max: 1, min: 1, items: pandora.site.documentFilters.map(function(filter) { + return Ox.extend({checked: filter.id == id}, filter); + })} + ], + type: 'image', + }) + .css(Ox.UI.SCROLLBAR_SIZE == 8 ? { + right: '-1px', + width: '8px', + } : { + right: '2px', + width: (Ox.UI.SCROLLBAR_SIZE - 2) + 'px' + }) + .bindEvent({ + change: function(data) { + var filters = Ox.clone(pandora.user.ui.documentFilters, true), + find, + id_ = data.checked[0].id, + i_ = Ox.getIndexById(pandora.user.ui.documentFilters, id_); + if (i_ == -1) { + // new filter was not part of old filter set + if (pandora.user.ui._documentFilterState[i].selected.length) { + // if filter with selection gets replaced, update find + find = Ox.clone(pandora.user.ui.findDocuments, true); + find.conditions.splice(pandora.user.ui._documentFilterState[i].index, 1); + } + filters[i] = makeFilter(id_); + pandora.UI.set(Ox.extend({ + documentFilters: filters + }, find ? { + findDocuments: find + } : {})); + replaceFilter(i, id_); + // fixme: there is an obscure special case not yet covered: + // switching to a new filter may change find from advanced to not advanced + // if part of the existing query works as a filter selection in the new filter + } else { + // swap two existing filters + var filterData = Ox.clone(pandora.user.ui._documentFilterState[i]); + pandora.user.ui._documentFilterState[i] = pandora.user.ui._documentFilterState[i_]; + pandora.user.ui._documentFilterState[i_] = filterData; + filters[i] = makeFilter(id_, pandora.user.ui.documentFilters[i_].sort); + filters[i_] = makeFilter(id, pandora.user.ui.documentFilters[i].sort); + pandora.UI.set({documentFilters: filters}); + replaceFilter(i, id_); + replaceFilter(i_, id); + } + pandora.$ui.documentFilters.updateMenus(); + function makeFilter(id, sort) { + // makes user.ui._documentFilterState object from site.documentFilters object + var filter = Ox.getObjectById(pandora.site.documentFilters, id); + return { + id: filter.id, + sort: sort || [{key: filter.type == 'integer' ? 'name' : 'items', operator: '-'}] + }; + } + function replaceFilter(i, id) { + var isOuter = i % 4 == 0; + pandora.$ui[isOuter ? 'documentBrowser' : 'documentFiltersInnerPanel'].replaceElement( + isOuter ? i / 2 : i - 1, + pandora.$ui.documentFilters[i] = pandora.ui.documentFilter(id) + ); + } + }, + click: function(data) { + if (data.id == 'clearFilter') { + // FIXME: List should trigger event on options change + if (!Ox.isEmpty(that.options('selected'))) { + that.options({selected: []}).triggerEvent('select', {ids: []}); + } + } else if (data.id == 'clearFilters') { + pandora.$ui.documentFilters.clearFilters(); + } + } + }) + .appendTo(that.$bar); + Ox.UI.SCROLLBAR_SIZE < 16 && $($menu.find('input')[0]).css({ + marginRight: '-3px', + marginTop: '1px', + width: '8px', + height: '8px' + }); + that.disableMenuItem = function(id) { + $menu.disableItem(id); + }; + that.enableMenuItem = function(id) { + $menu.enableItem(id); + }; + return that; +}; + +pandora.ui.documentFilters = function() { + var $filters = []; + pandora.user.ui.documentFilters.forEach(function(filter, i) { + $filters[i] = pandora.ui.documentFilter(filter.id); + }); + $filters.clearFilters = function() { + var find = Ox.clone(pandora.user.ui.findDocuments, true), + indices = pandora.user.ui._documentFilterState.map(function(filterState) { + return filterState.index; + }).filter(function(index) { + return index > -1; + }); + find.conditions = find.conditions.filter(function(condition, index) { + return !Ox.contains(indices, index); + }); + pandora.UI.set({findDocuments: find}) + }; + $filters.updateMenus = function() { + var selected = $filters.map(function($filter) { + return !Ox.isEmpty($filter.options('selected')); + }), + filtersHaveSelection = !!Ox.sum(selected); + $filters.forEach(function($filter, i) { + $filter[ + selected[i] ? 'enableMenuItem' : 'disableMenuItem' + ]('clearFilter'); + $filter[ + filtersHaveSelection ? 'enableMenuItem' : 'disableMenuItem' + ]('clearFilters'); + }); + return $filters; + }; + return $filters.updateMenus(); +}; + +pandora.ui.documentFiltersInnerPanel = function() { + var that = Ox.SplitPanel({ + elements: [ + { + element: pandora.$ui.documentFilters[1], + size: pandora.user.ui.filterSizes[1] + }, + { + element: pandora.$ui.documentFilters[2] + }, + { + element: pandora.$ui.documentFilters[3], + size: pandora.user.ui.filterSizes[3] + } + ], + orientation: 'horizontal' + }); + return that; +}; + + diff --git a/static/js/documentFilterForm.js b/static/js/documentFilterForm.js index 2d0992a6..577375c0 100644 --- a/static/js/documentFilterForm.js +++ b/static/js/documentFilterForm.js @@ -42,7 +42,7 @@ pandora.ui.documentFilterForm = function(options) { view: pandora.user.ui.collectionView } : null, sortKeys: pandora.site.documentSortKeys, - value: Ox.clone(mode == 'collection' ? collection.query : pandora.user.ui.documentFind, true), + value: Ox.clone(mode == 'collection' ? collection.query : pandora.user.ui.findDocuments, true), viewKeys: pandora.site.collectionViews }) .css(mode == 'embed' ? {} : {padding: '16px'}) @@ -87,7 +87,7 @@ pandora.ui.documentFilterForm = function(options) { }); */ } else { - pandora.UI.set({find: Ox.clone(that.$filter.options('value'), true)}); + pandora.UI.set({findDocuments: Ox.clone(that.$filter.options('value'), true)}); pandora.$ui.findElement.updateElement(); } }; diff --git a/static/js/documentInfoView.js b/static/js/documentInfoView.js index 592afd45..9b3cd79e 100644 --- a/static/js/documentInfoView.js +++ b/static/js/documentInfoView.js @@ -232,6 +232,9 @@ pandora.ui.documentInfoView = function(data, isMixed) { renderGroup(['author', 'date', 'type']); renderGroup(['publisher', 'place', 'series', 'edition', 'language']); + Ox.getObjectById(pandora.site.documentKeys, 'keywords') && renderGroup(['keywords']) + + // Description ------------------------------------------------------------- if (canEdit || data.description) { diff --git a/static/js/documentToolbar.js b/static/js/documentToolbar.js index 347fc2df..ef12a856 100644 --- a/static/js/documentToolbar.js +++ b/static/js/documentToolbar.js @@ -64,7 +64,7 @@ pandora.ui.documentToolbar = function() { } }) that.append( - pandora.$ui.findDocumentsElement = pandora.ui.findDocumentsElement(function(data) { + pandora.$ui.findElement= pandora.ui.findDocumentsElement(function(data) { var key = data.key, value = data.value, conditions; diff --git a/static/js/editDocumentsDialog.js b/static/js/editDocumentsDialog.js index 4d6b345f..620cc153 100644 --- a/static/js/editDocumentsDialog.js +++ b/static/js/editDocumentsDialog.js @@ -97,7 +97,8 @@ pandora.ui.editDocumentsDialog = function() { }); if (isArray) { values = values.map(function(value) { - return (value || []).join(separator); + value = value || [] + return value.join ? value.join(separator) : value; }); } if (Ox.unique(values).length > 1) { diff --git a/static/js/filter.js b/static/js/filter.js index 57c27864..c52fdcda 100644 --- a/static/js/filter.js +++ b/static/js/filter.js @@ -174,7 +174,7 @@ pandora.ui.filter = function(id) { }, sort: function(data) { Ox.Log('', 'SORT', data) - var filters = Ox.clone(pandora.user.ui.filters); + var filters = Ox.clone(pandora.user.ui.filters, true); pandora.$ui.mainMenu.checkItem('sortMenu_sortfilters_sortfilter' + id + '_' + data.key); pandora.$ui.mainMenu.checkItem('sortMenu_orderfilters_orderfilter' + id + '_' + (data.operator == '+' ? 'ascending' : 'descending')); filters[i].sort = [{key: data.key, operator: data.operator}]; @@ -201,7 +201,7 @@ pandora.ui.filter = function(id) { }) .bindEvent({ change: function(data) { - var filters = Ox.clone(pandora.user.ui.filters), + var filters = Ox.clone(pandora.user.ui.filters, true), find, id_ = data.checked[0].id, i_ = Ox.getIndexById(pandora.user.ui.filters, id_); diff --git a/static/js/filterDialog.js b/static/js/filterDialog.js index 88be1d55..3e5b7b46 100644 --- a/static/js/filterDialog.js +++ b/static/js/filterDialog.js @@ -42,7 +42,12 @@ pandora.ui.filterDialog = function() { } }) ], - content: pandora.$ui.filterForm = pandora.ui.filterForm({mode: 'find'}), + content: pandora.$ui.filterForm = (pandora.user.ui.section == 'documents' + ? pandora.ui.documentFilterForm + : pandora.ui.filterForm + )({ + mode: 'find' + }), maxWidth: 648 + Ox.UI.SCROLLBAR_SIZE, minHeight: 264, minWidth: 648 + Ox.UI.SCROLLBAR_SIZE, diff --git a/static/js/findDocumentsElement.js b/static/js/findDocumentsElement.js index e8fcfea7..ad7efcb1 100644 --- a/static/js/findDocumentsElement.js +++ b/static/js/findDocumentsElement.js @@ -35,26 +35,39 @@ pandora.ui.findDocumentsElement = function() { ] : [], [ $findSelect = Ox.Select({ id: 'select', - items: pandora.site.documentKeys.filter(function(key) { - return key.find; - }).map(function(key) { + items: [].concat( + pandora.site.documentKeys.filter(function(key) { + return key.find; + }).map(function(key) { return { id: key.id, title: Ox._('Find: {0}', [Ox._(key.title)]) }; }), + [{}, { + id: 'advanced', + title: Ox._('Find: Advanced...') + }] + ), + overlap: 'right', value: findKey, width: 128 }) .bindEvent({ change: function(data) { - //pandora.$ui.mainMenu.checkItem('findMenu_find_' + data.value); - $findInput.options({ - autocomplete: autocompleteFunction(), - placeholder: '' - }).focusInput(true); - previousFindKey = data.value; + if (data.value == 'advanced') { + that.updateElement(); + pandora.$ui.mainMenu.checkItem('findMenu_find_' + previousFindKey); + pandora.$ui.filterDialog = pandora.ui.filterDialog().open(); + } else { + //pandora.$ui.mainMenu.checkItem('findMenu_find_' + data.value); + $findInput.options({ + autocomplete: autocompleteFunction(), + placeholder: '' + }).focusInput(true); + previousFindKey = data.value; + } } }), $findInput = Ox.Input({ @@ -77,7 +90,7 @@ pandora.ui.findDocumentsElement = function() { focus: function(data) { if ($findSelect.value() == 'advanced') { if (hasPressedClear) { - pandora.UI.set({find: pandora.site.user.ui.find}); + pandora.UI.set({findDocuments: pandora.site.user.ui.findDocuments}); that.updateElement(); hasPressedClear = false; } diff --git a/static/js/folderBrowserList.js b/static/js/folderBrowserList.js index c08c9355..3d9ed81e 100644 --- a/static/js/folderBrowserList.js +++ b/static/js/folderBrowserList.js @@ -235,7 +235,7 @@ pandora.ui.folderBrowserList = function(id, section) { pandora.UI.set({ findDocuments: { conditions: list ? [ - {key: 'list', value: data.ids[0], operator: '=='} + {key: 'collection', value: data.ids[0], operator: '=='} ] : [], operator: '&' } diff --git a/static/js/folders.js b/static/js/folders.js index 188f163e..21903e2b 100644 --- a/static/js/folders.js +++ b/static/js/folders.js @@ -46,7 +46,15 @@ pandora.ui.folders = function(section) { pandora.$ui.personalListsMenu = Ox.MenuButton({ items: [ { id: 'newlist', title: Ox._('New {0}', [Ox._(folderItem)]), keyboard: 'control n' }, - { id: 'newlistfromselection', title: Ox._('New {0} from Selection', [Ox._(folderItem)]), keyboard: 'shift control n', disabled: ui.listSelection.length == 0 }, + { + id: 'newlistfromselection', + title: Ox._('New {0} from Selection', + [Ox._(folderItem)]), + keyboard: 'shift control n', + disabled: ui.section == 'documents' + ? ui.collectionSelection == 0 + : ui.listSelection.length == 0 + }, { id: 'newsmartlist', title: Ox._('New Smart {0}', [Ox._(folderItem)]), keyboard: 'alt control n' }, { id: 'newsmartlistfromresults', title: Ox._('New Smart {0} from Results', [Ox._(folderItem)]), keyboard: 'shift alt control n' }, {}, diff --git a/static/js/infoView.indiancinema.js b/static/js/infoView.indiancinema.js index cb34ec87..3353e3fe 100644 --- a/static/js/infoView.indiancinema.js +++ b/static/js/infoView.indiancinema.js @@ -24,7 +24,7 @@ pandora.ui.infoView = function(data, isMixed) { listWidth = 0, margin = 16, // these may contain commas, and are thus separated by semicolons - specialListKeys = ['alternativeTitles', 'productionCompany'].concat( + specialListKeys = ['alternativeTitles', 'productionCompany', 'laboratory'].concat( pandora.site.itemKeys.filter(function(key) { return key.type[0] == 'date' }).map(function(key) { @@ -439,6 +439,12 @@ pandora.ui.infoView = function(data, isMixed) { // Songs if (data.songs || canEdit) { + $('
') + .css({ + marginTop: '12px', + }) + .html(formatKey('songs')) + .appendTo($text); Ox.EditableContent({ clickLink: pandora.clickLink, collapseToEnd: false, @@ -449,7 +455,7 @@ pandora.ui.infoView = function(data, isMixed) { ' -1 && Ox.getObjectById(pandora.site.itemKeys, key).type[0] == 'date' + specialListKeys.indexOf(key) > -1 && itemKey && itemKey.type[0] == 'date' ) { ret = value.split('; ').map(function(date) { date = cleanupDate(date) @@ -906,8 +912,8 @@ pandora.ui.infoView = function(data, isMixed) { return value[0]; }).join('; ') : key == 'runtime' ? Math.round(value / 60) - : Ox.contains(listKeys, key) ? value.join(', ') - : Ox.contains(specialListKeys, key) ? value.join('; ') + : Ox.contains(listKeys, key) && value.join ? value.join(', ') + : Ox.contains(specialListKeys, key) && value.join ? value.join('; ') : value; } diff --git a/static/js/listDialog.js b/static/js/listDialog.js index 41ba382d..35051d32 100644 --- a/static/js/listDialog.js +++ b/static/js/listDialog.js @@ -22,7 +22,12 @@ pandora.ui.listDialog = function(section) { if (id == 'general') { return pandora.ui.listGeneralPanel(listData); } else if (id == 'icon') { - return pandora.$ui.listIconPanel = pandora.ui.listIconPanel(listData); + if (pandora.user.ui.section == 'documents') { + pandora.$ui.listIconPanel = pandora.ui.collectionIconPanel(listData); + } else { + pandora.$ui.listIconPanel = pandora.ui.listIconPanel(listData); + } + return pandora.$ui.listIconPanel } else if (id == 'query') { return pandora.$ui.filterForm = (pandora.user.ui.section == 'documents' ? pandora.ui.documentFilterForm @@ -64,7 +69,11 @@ pandora.ui.listDialog = function(section) { var $findElement = Ox.FormElementGroup({ elements: [ pandora.$ui.findIconItemSelect = Ox.Select({ - items: pandora.site.findKeys.map(function(findKey) { + items: ( + pandora.user.ui.section == 'items' + ? pandora.site.findKeys + : pandora.site.documentKeys + ).map(function(findKey) { return {id: findKey.id, title: Ox._('Find: {0}', [Ox._(findKey.title)])}; }), overlap: 'right', diff --git a/static/js/pandora.js b/static/js/pandora.js index 571e58a1..0962c235 100644 --- a/static/js/pandora.js +++ b/static/js/pandora.js @@ -342,6 +342,15 @@ appPanel type: Ox.isArray(key.type) ? key.type[0] : key.type }; }), + documentFilters: data.site.documentKeys.filter(function(key) { + return key.filter; + }).map(function(key) { + return { + id: key.id, + title: key.title, + type: Ox.isArray(key.type) ? key.type[0] : key.type + }; + }), findKeys: data.site.itemKeys.filter(function(key) { return key.find; }), diff --git a/static/js/utils.js b/static/js/utils.js index 9b774d63..5825e724 100644 --- a/static/js/utils.js +++ b/static/js/utils.js @@ -192,7 +192,9 @@ pandora.addFolderItem = function(section) { }, function(result) { var posterFrames = result ? result.data.items.map(function(item) { - return {item: item.id, position: item.posterFrame}; + return section == 'documents' + ? {document: item.id} + : {item: item.id, position: item.posterFrame}; }) : []; posterFrames = posterFrames.length == 1 ? Ox.repeat([posterFrames[0]], 4) @@ -860,7 +862,7 @@ pandora.enableDragAndDrop = function($list, canMove, section, getItems) { items: drag.ids }, function() { - Ox.Request.clearCache('find'); + Ox.Request.clearCache('findDocuments'); pandora.api.findDocuments({ query: { conditions: [{ @@ -2626,6 +2628,7 @@ pandora.signin = function(data) { }); pandora.user.ui._list = pandora.getListState(pandora.user.ui.find); pandora.user.ui._filterState = pandora.getFilterState(pandora.user.ui.find); + pandora.user.ui._documentFilterState = pandora.getDocumentFilterState(pandora.user.ui.findDocuments); pandora.user.ui._findState = pandora.getFindState(pandora.user.ui.find); pandora.user.ui._collection = pandora.getCollectionState(pandora.user.ui.findDocuments); pandora.user.ui._findDocumentsState = pandora.getFindDocumentsState(pandora.user.ui.findDocuments); @@ -2642,6 +2645,7 @@ pandora.signout = function(data) { pandora.user = data.user; pandora.user.ui._list = pandora.getListState(pandora.user.ui.find); pandora.user.ui._filterState = pandora.getFilterState(pandora.user.ui.find); + pandora.user.ui._documentFilterState = pandora.getDocumentFilterState(pandora.user.ui.findDocuments); pandora.user.ui._findState = pandora.getFindState(pandora.user.ui.find); pandora.user.ui._collection = pandora.getCollectionState(pandora.user.ui.findDocuments); pandora.user.ui._findDocumentsState = pandora.getFindDocumentsState(pandora.user.ui.findDocuments); @@ -2825,9 +2829,15 @@ pandora.resizeFilters = function(width) { pandora.$ui.browser && pandora.$ui.browser .size(0, pandora.user.ui.filterSizes[0]) .size(2, pandora.user.ui.filterSizes[4]); + pandora.$ui.documentBrowser && pandora.$ui.documentBrowser + .size(0, pandora.user.ui.filterSizes[0]) + .size(2, pandora.user.ui.filterSizes[4]); pandora.$ui.filtersInnerPanel && pandora.$ui.filtersInnerPanel .size(0, pandora.user.ui.filterSizes[1]) .size(2, pandora.user.ui.filterSizes[3]); + pandora.$ui.documentFiltersInnerPanel && pandora.$ui.documentFiltersInnerPanel + .size(0, pandora.user.ui.filterSizes[1]) + .size(2, pandora.user.ui.filterSizes[3]); pandora.$ui.filters && pandora.$ui.filters.forEach(function($list, i) { $list.resizeColumn( 'name', @@ -2839,6 +2849,17 @@ pandora.resizeFilters = function(width) { }); } }); + pandora.$ui.documentFilters && pandora.$ui.documentFilters.forEach(function($list, i) { + $list.resizeColumn( + 'name', + pandora.user.ui.filterSizes[i] - 44 - Ox.UI.SCROLLBAR_SIZE + ); + if (pandora.site.flags) { + $list.find('.flagname').css({ + width: pandora.user.ui.filterSizes[i] - 68 - Ox.UI.SCROLLBAR_SIZE + }); + } + }); }; pandora.resizeFolders = function(section) { @@ -3396,6 +3417,7 @@ pandora.wait = function(id, callback, timeout) { } return state; }; + function getState(find, key) { var index, state = ''; if (find.operator == '&') { @@ -3429,12 +3451,10 @@ pandora.wait = function(id, callback, timeout) { if (find.operator == '&') { // number of conditions that are not list or filters conditions = find.conditions.length - - !!pandora.user.ui._collection; - /* - - pandora.user.ui._filterState.filter(function(filter) { + - !!pandora.user.ui._collection + - pandora.user.ui._documentFilterState.filter(function(filter) { return filter.index > -1; }).length; - */ // indices of non-advanced find queries indices = pandora.site.documentKeys.map(function(findKey) { return oneCondition(find.conditions, findKey.id, '='); @@ -3456,16 +3476,62 @@ pandora.wait = function(id, callback, timeout) { key: 'advanced', value: '' }; - /* Ox.forEach(pandora.user.ui.documentFilters, function(key) { if (everyCondition(find.conditions, key, '==')) { state.key = '*'; return false; } }); - */ } return state; }; + pandora.getDocumentFilterState = function(find) { + // A filter is selected if exactly one condition in an & query or every + // condition in an | query has the filter id as key and "==" as operator + return pandora.user.ui.documentFilters.map(function(filter) { + // FIXME: cant index be an empty array, instead of -1? + var key = filter.id, + state = {index: -1, find: Ox.clone(find, true), selected: []}; + if (find.operator == '&') { + // include conditions where all subconditions match + state.index = oneCondition(find.conditions, key, '==', true); + if (state.index > -1) { + state.selected = find.conditions[state.index].conditions + ? find.conditions[state.index].conditions.map(function(condition) { + return condition.value; + }) + : [find.conditions[state.index].value]; + } + } else { + if (everyCondition(find.conditions, key, '==')) { + state.index = Ox.range(find.conditions.length); + state.selected = find.conditions.map(function(condition) { + return condition.value; + }); + } + } + if (state.selected.length) { + if (Ox.isArray(state.index)) { + // every condition in an | query matches this filter + state.find = {conditions: [], operator: ''}; + } else { + // one condition in an & query matches this filter + state.find.conditions.splice(state.index, 1); + if ( + state.find.conditions.length == 1 + && state.find.conditions[0].conditions + ) { + // unwrap single remaining bracketed query + state.find = { + conditions: state.find.conditions[0].conditions, + operator: state.find.conditions[0].operator + }; + } + } + } + return state; + }); + }; + }()); diff --git a/update.py b/update.py index 40c2532b..8639b8b8 100755 --- a/update.py +++ b/update.py @@ -58,7 +58,6 @@ def run(*cmd): p.wait() return p.returncode - def get(*cmd): p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, error = p.communicate() @@ -135,7 +134,10 @@ if __name__ == "__main__": if len(sys.argv) == 2 and sys.argv[1] in ('database', 'db'): os.chdir(join(base, 'pandora')) print('\nRunning "./manage.py migrate"\n') - run('./manage.py', 'migrate', '--noinput') + r = get('./manage.py', 'migrate', '--noinput') + r = r.replace("Your models have changes that are not yet reflected in a migration, and so won't be applied.", '') + r = r.replace("Run 'manage.py makemigrations' to make new migrations, and then re-run 'manage.py migrate' to apply them.", '') + print(r) run('./manage.py', 'sqlfindindex') run('./manage.py', 'sync_itemsort') run('./manage.py', 'sync_documentsort') @@ -257,6 +259,8 @@ if __name__ == "__main__": if old <= 6064: run('./bin/pip', 'install', '-r', 'requirements.txt') run('./pandora/manage.py', 'createcachetable') + if old <= 6108: + run('./bin/pip', 'install', '-r', 'requirements.txt') else: if len(sys.argv) == 1: branch = get_branch() diff --git a/vm/LXC_README.md b/vm/LXC_README.md index 092304d3..44af135e 100644 --- a/vm/LXC_README.md +++ b/vm/LXC_README.md @@ -19,7 +19,7 @@ or - sudo lxc-create -n pandora -t debian -- -r stretch + sudo lxc-create -n pandora -t debian -- -r buster 3) Install pan.do/ra in container: diff --git a/vm/LXD_README.md b/vm/LXD_README.md new file mode 100644 index 00000000..3b003deb --- /dev/null +++ b/vm/LXD_README.md @@ -0,0 +1,38 @@ +# Preparations + + you will need at least 2GB of free disk space to install pan.do/ra + +# Installing pan.do/ra inside LXD + +1) Install lxd on the host (Ubuntu 16.04 or later): + + sudo apt-get install lxd + +[on debian you can use snap install lxd] + +2) Create a new container, use different names if installing multiple instances: + + sudo lxc launch ubuntu:18.04 pandora + + or + + sudo lxc launch images:debian/10 pandora + +3) Attach to container and install pan.do/ra + + sudo lxc exec pandora bash + apt-get update -qq && apt-get upgrade -y + apt-get -y install curl ca-certificates + sed -i s/ubuntu/pandora/g /etc/passwd /etc/shadow /etc/group + mv /home/ubuntu /home/pandora + echo "pandora:pandora" | chpasswd + echo PasswordAuthentication no >> /etc/ssh/sshd_config + locale-gen en_US.UTF-8 + update-locale LANG=en_US.UTF-8 + export LANG=en_US.UTF-8 + + cd /root + curl -sL https://pan.do/ra-install > pandora_install.sh + chmod +x pandora_install.sh + ./pandora_install.sh 2>&1 | tee pandora_install.log + diff --git a/vm/pandora_install.sh b/vm/pandora_install.sh index 342010cd..834060c5 100755 --- a/vm/pandora_install.sh +++ b/vm/pandora_install.sh @@ -1,4 +1,9 @@ #!/bin/bash +# +# pan.do/ra installer +# =================== +# + PANDORA=${PANDORA-pandora} POSTGRES=${POSTGRES-local} @@ -6,20 +11,22 @@ RABBITMQ=${RABBITMQ-local} NGINX=${NGINX-local} BRANCH=${BRANCH-stable} +# add a pandora user echo Installing pandora with user: $PANDORA getent passwd $PANDORA > /dev/null 2>&1 || adduser --disabled-password --gecos "" $PANDORA +# +# install pan.do/ra ppa +# +# apt-get install software-properties-common +# add-apt-repository ppa:j/pandora +# LXC=`grep -q lxc /proc/1/environ && echo 'yes' || echo 'no'` if [ -e /etc/os-release ]; then . /etc/os-release fi -if [ -d "/run/systemd/system/" ]; then - SYSTEMD="yes" -else - SYSTEMD="no" -fi if [ -z "$UBUNTU_CODENAME" ]; then - UBUNTU_CODENAME=zesty + UBUNTU_CODENAME=bionic fi export DEBIAN_FRONTEND=noninteractive echo "deb http://ppa.launchpad.net/j/pandora/ubuntu ${UBUNTU_CODENAME} main" > /etc/apt/sources.list.d/j-pandora.list @@ -46,10 +53,12 @@ echo 'Acquire::Languages "none";' > /etc/apt/apt.conf.d/99languages apt-get update -qq if [ "$LXC" == "no" ]; then -apt-get install -y \ - acpid \ - ntp +apt-get install -y acpid +systemctl enable systemd-timesyncd.service fi + +# add postgres, rabbitmq and nginx +# unless they are running on another host EXTRA="" if [ "$POSTGRES" == "local" ]; then EXTRA="$EXTRA postgresql postgresql-contrib" @@ -61,6 +70,7 @@ if [ "$NGINX" == "local" ]; then EXTRA="$EXTRA nginx" fi +# install all required packages apt-get install -y \ sudo \ openssh-server \ @@ -87,18 +97,21 @@ apt-get install -y \ gpac \ imagemagick \ poppler-utils \ - youtube-dl \ ipython3 \ postfix \ postgresql-client $EXTRA +apt-get install -y --no-install-recommends youtube-dl rtmpdump + +# setup database + if [ "$POSTGRES" == "local" ]; then sudo -u postgres createuser -S -D -R $PANDORA sudo -u postgres createdb -T template0 --locale=C --encoding=UTF8 -O $PANDORA pandora echo "CREATE EXTENSION pg_trgm;" | sudo -u postgres psql pandora fi -#rabbitmq +# setup rabbitmq if [ "$RABBITMQ" == "local" ]; then RABBITPWD=$(pwgen -n 16 -1) rabbitmqctl add_user pandora $RABBITPWD @@ -109,18 +122,20 @@ else BROKER_URL="$RABBITMQ" fi -#pandora +# checkout pandora from git git clone https://git.0x2620.org/pandora.git /srv/pandora cd /srv/pandora git checkout $BRANCH ./ctl init +# create config.jsonc from templates in git HOST=$(hostname -s) HOST_CONFIG="/srv/pandora/pandora/config.$HOST.jsonc" SITE_CONFIG="/srv/pandora/pandora/config.jsonc" test -e $HOST_CONFIG && cp $HOST_CONFIG $SITE_CONFIG test -e $SITE_CONFIG || cp /srv/pandora/pandora/config.pandora.jsonc $SITE_CONFIG +# create local_settings.py cat > /srv/pandora/pandora/local_settings.py <> /srv/pandora/pandora/local_settings.py @@ -170,7 +189,7 @@ fi #logrotate #cp "/srv/pandora/etc/logrotate.d/pandora" "/etc/logrotate.d/pandora" -#nginx +# configure nginx if [ "$NGINX" == "local" ]; then cp "/srv/pandora/etc/nginx/pandora" "/etc/nginx/sites-available/default" @@ -191,21 +210,9 @@ service nginx restart fi -if [ "$LXC" == "yes" ]; then - test -e /etc/init/avahi-daemon.conf && sed -i "s/-D/--no-rlimits -D/g" /etc/init/avahi-daemon.conf -fi - +# additional configurations if installed outside of LXD/LXC if [ "$LXC" == "no" ]; then - if [ "$SYSTEMD" == "yes" ]; then - echo Servers=pool.ntp.org >> /etc/systemd/timesyncd.conf - else -cat > /etc/cron.d/ntp_fixtime </dev/null -EOF - fi - +echo Servers=pool.ntp.org >> /etc/systemd/timesyncd.conf cat > /usr/local/bin/genissue < /etc/rc.local < /etc/vim/vimrc.local <