From fa1b98365b380647f229e2018bcce05f01c17dbf Mon Sep 17 00:00:00 2001 From: j Date: Sat, 6 Jul 2019 18:12:32 +0200 Subject: [PATCH] add docker build --- .dockerignore | 3 + .gitignore | 2 + Dockerfile | 11 +++ docker-compose.yml | 91 ++++++++++++++++++++++ docker/base/Dockerfile | 14 ++++ docker/base/install.sh | 63 ++++++++++++++++ docker/build.sh | 17 +++++ docker/dot.env.sample.py | 30 ++++++++ docker/entrypoint.sh | 134 +++++++++++++++++++++++++++++++++ docker/install.sh | 68 +++++++++++++++++ docker/nginx/Dockerfile | 4 + docker/nginx/nginx.conf | 96 +++++++++++++++++++++++ docker/overlay/__init__.py | 0 docker/overlay/install.py | 91 ++++++++++++++++++++++ docker/overlay/settings.py | 1 + docker/setup-docker-compose.sh | 32 ++++++++ docker/wait-for | 17 +++++ docker/wait-for-file | 12 +++ 18 files changed, 686 insertions(+) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 docker-compose.yml create mode 100644 docker/base/Dockerfile create mode 100755 docker/base/install.sh create mode 100755 docker/build.sh create mode 100755 docker/dot.env.sample.py create mode 100755 docker/entrypoint.sh create mode 100755 docker/install.sh create mode 100644 docker/nginx/Dockerfile create mode 100644 docker/nginx/nginx.conf create mode 100644 docker/overlay/__init__.py create mode 100755 docker/overlay/install.py create mode 100644 docker/overlay/settings.py create mode 100755 docker/setup-docker-compose.sh create mode 100755 docker/wait-for create mode 100755 docker/wait-for-file diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..a1b0dfbb --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +.env +data/ +overlay/ diff --git a/.gitignore b/.gitignore index bc18996b..da0f04d5 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,5 @@ static/django_extensions *.swp pandora/gunicorn_config.py .DS_Store +.env +overlay/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..a1b287ee --- /dev/null +++ b/Dockerfile @@ -0,0 +1,11 @@ +FROM 0x2620/pandora-base:latest + +LABEL maintainer="0x2620@0x2620.org" + +ENV LANG en_US.UTF-8 + +#VOLUME /pandora +COPY . /srv/pandora +RUN /srv/pandora/docker/install.sh + +ENTRYPOINT [ "/entrypoint.sh" ] diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..1360247c --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,91 @@ +version: '3' + +volumes: + pandora: + postgres: + rabbitmq: + +networks: + backend: + +services: + proxy: + build: docker/nginx + ports: + - "127.0.0.1:2620:80" + networks: + - backend + - default + links: + - pandora + - websocketd + depends_on: + - pandora + - websocketd + volumes: + - pandora:/pandora + - ./overlay:/overlay + restart: unless-stopped + + db: + image: postgres:latest + networks: + - backend + env_file: .env + volumes: + - postgres:/var/lib/postgresql/data/ + restart: unless-stopped + + rabbitmq: + hostname: rabbitmq + image: rabbitmq:latest + env_file: .env + networks: + - backend + volumes: + - rabbitmq:/var/lib/rabbitmq + restart: unless-stopped + + pandora: + hostname: pandora + build: . + command: pandora + volumes: + - pandora:/pandora + - ./overlay:/overlay + networks: + - backend + env_file: .env + links: + - db + - rabbitmq + depends_on: + - db + - rabbitmq + restart: unless-stopped + + encoding: &app_base + build: . + command: encoding + env_file: .env + networks: + - backend + volumes: + - pandora:/pandora + - ./overlay:/overlay + restart: unless-stopped + + tasks: + <<: *app_base + command: tasks + + cron: + <<: *app_base + command: cron + + websocketd: + <<: *app_base + command: websocketd + networks: + - backend + diff --git a/docker/base/Dockerfile b/docker/base/Dockerfile new file mode 100644 index 00000000..727891e8 --- /dev/null +++ b/docker/base/Dockerfile @@ -0,0 +1,14 @@ +FROM debian:buster + +LABEL maintainer="0x2620@0x2620.org" + +ENV LANG en_US.UTF-8 + +RUN apt-get update && \ + LANG=C.UTF-8 DEBIAN_FRONTEND=noninteractive apt-get install -y apt-utils locales && \ + echo en_US.UTF-8 UTF-8 > /etc/locale.gen && \ + locale-gen en_US.UTF-8 && \ + update-locale LANG=en_US.UTF-8 + +COPY ./install.sh /install.sh +RUN /install.sh diff --git a/docker/base/install.sh b/docker/base/install.sh new file mode 100755 index 00000000..5c209f07 --- /dev/null +++ b/docker/base/install.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +UBUNTU_CODENAME=bionic +if [ -e /etc/os-release ]; then + . /etc/os-release +fi + +export DEBIAN_FRONTEND=noninteractive +echo 'Acquire::Languages "none";' > /etc/apt/apt.conf.d/99languages +apt-get update -qq +echo "deb http://ppa.launchpad.net/j/pandora/ubuntu ${UBUNTU_CODENAME} main" > /etc/apt/sources.list.d/j-pandora.list +apt-get install -y gnupg2 +apt-key add - < /dev/null 2>&1 +result=$? +if [ $result -eq 0 ] ; then + proxy="--build-arg http_proxy=http://$HOST:$PORT" +else + proxy= +fi + +docker build $proxy -t 0x2620/pandora-base base +docker build -t 0x2620/pandora-nginx nginx +cd .. +docker build -t 0x2620/pandora . diff --git a/docker/dot.env.sample.py b/docker/dot.env.sample.py new file mode 100755 index 00000000..c6f5c2f4 --- /dev/null +++ b/docker/dot.env.sample.py @@ -0,0 +1,30 @@ +#!/usr/bin/python3 +import os +from binascii import hexlify +import string + +chars = string.ascii_letters + string.digits + +def pwgen(length=16): + return ''.join(chars[c % len(chars)] for c in os.urandom(length)) + + +print('''SECRET_KEY={SECRET_KEY} + +POSTGRES_USER=pandora +POSTGRES_PASSWORD={POSTGRES_PASSWORD} + +RABBITMQ_DEFAULT_USER=pandora +RABBITMQ_DEFAULT_PASS={RABBITMQ_PASSWORD} + +# required to send out password reset emails +#EMAIL_HOST='example.com' +#EMAIL_USER='mail@example.com' +#EMAIL_PASSWORD='fixme' +#EMAIL_PORT=587 +#EMAIL_TLS=true +'''.format( + SECRET_KEY=hexlify(os.urandom(64)).decode(), + POSTGRES_PASSWORD=pwgen(), + RABBITMQ_PASSWORD=pwgen() +)) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100755 index 00000000..cd78c6f4 --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,134 @@ +#!/bin/bash +set -e + +action="$1" +user=pandora + +export LANG=en_US.UTF-8 +mkdir -p /run/pandora +chown -R ${user}.${user} /run/pandora + +# pan.do/ra services +if [ "$action" = "pandora" ]; then + if [ ! -e /srv/pandora/initialized ]; then + echo "Setting up Pan.do/ra:" + echo "Waiting for database connection..." + /srv/pandora_base/docker/wait-for db 5432 + echo "Installing pan.do/ra..." + rsync -a /srv/pandora_base/ /srv/pandora/ + + if [ ! -e /overlay/install.py ]; then + rsync -a /srv/pandora_base/docker/overlay/ /overlay/ + if [ ! -e /overlay/config.jsonc ]; then + mv /srv/pandora/pandora/config.jsonc /overlay/config.jsonc + fi + fi + /overlay/install.py + + echo "Initializing database..." + echo "CREATE EXTENSION pg_trgm;" | /srv/pandora/pandora/manage.py dbshell + /srv/pandora/pandora/manage.py init_db + /srv/pandora/update.py db + echo "Generating static files..." + /srv/pandora/update.py static + chown -R ${user}.${user} /srv/pandora/ + touch /srv/pandora/initialized + fi + /srv/pandora_base/docker/wait-for db 5432 + /srv/pandora_base/docker/wait-for rabbitmq 5672 + cd /srv/pandora/pandora + exec /usr/bin/sudo -u $user -E -H \ + /srv/pandora/bin/gunicorn wsgi:application -c gunicorn_config.py +fi +if [ "$action" = "encoding" ]; then + /srv/pandora_base/docker/wait-for-file /srv/pandora/initialized + /srv/pandora_base/docker/wait-for rabbitmq 5672 + name=pandora-encoding-$(hostname) + exec /usr/bin/sudo -u $user -E -H \ + /srv/pandora/bin/python \ + /srv/pandora/pandora/manage.py \ + celery worker \ + -c 1 \ + -Q encoding -n $name \ + -l INFO +fi +if [ "$action" = "tasks" ]; then + /srv/pandora_base/docker/wait-for-file /srv/pandora/initialized + /srv/pandora_base/docker/wait-for rabbitmq 5672 + name=pandora-default-$(hostname) + exec /usr/bin/sudo -u $user -E -H \ + /srv/pandora/bin/python \ + /srv/pandora/pandora/manage.py \ + celery worker \ + -Q default,celery -n $name \ + --maxtasksperchild 1000 \ + -l INFO +fi +if [ "$action" = "cron" ]; then + /srv/pandora_base/docker/wait-for-file /srv/pandora/initialized + /srv/pandora_base/docker/wait-for rabbitmq 5672 + exec /usr/bin/sudo -u $user -E -H \ + /srv/pandora/bin/python \ + /srv/pandora/pandora/manage.py \ + celerybeat -s /run/pandora/celerybeat-schedule \ + --pidfile /run/pandora/cron.pid \ + -l INFO +fi +if [ "$action" = "websocketd" ]; then + /srv/pandora_base/docker/wait-for-file /srv/pandora/initialized + /srv/pandora_base/docker/wait-for rabbitmq 5672 + exec /usr/bin/sudo -u $user -E -H \ + /srv/pandora/bin/python \ + /srv/pandora/pandora/manage.py websocketd +fi + +# pan.do/ra management and update +if [ "$action" = "manage.py" ]; then + shift + exec /usr/bin/sudo -u $user -E -H \ + /srv/pandora/pandora/manage.py "$@" +fi +if [ "$action" = "update.py" ]; then + shift + exec /usr/bin/sudo -u $user -E -H \ + /srv/pandora/update.py "$@" +fi +if [ "$action" = "bash" ]; then + shift + cd / + exec /bin/bash "$@" +fi + +# pan.do/ra setup hooks +if [ "$action" = "docker-compose.yml" ]; then + cat /srv/pandora_base/docker-compose.yml | \ + sed "s#build: \.#image: 0x2620/pandora:latest#g" | \ + sed "s#\./overlay:#.:#g" | \ + sed "s#build: docker/nginx#image: 0x2620/pandora-nginx:latest#g" + exit +fi +if [ "$action" = ".env" ]; then + exec /srv/pandora_base/docker/dot.env.sample.py +fi +if [ "$action" = "config.jsonc" ]; then + cat /srv/pandora_base/pandora/config.pandora.jsonc + exit +fi +if [ "$action" = "setup" ]; then + cat /srv/pandora_base/docker/setup-docker-compose.sh + exit +fi + +# pan.do/ra info +echo pan.do/ra docker container - https://pan.do/ra +echo +echo use this container with docker-compose, +echo to setup a new docker-compose envrionment run: +echo +echo " mkdir && cd " +echo " docker run 0x2620/pandora setup | sh" +echo +echo adjust created files to match your needs and run: +echo +echo " docker-compose up" +echo diff --git a/docker/install.sh b/docker/install.sh new file mode 100755 index 00000000..d1be570f --- /dev/null +++ b/docker/install.sh @@ -0,0 +1,68 @@ +#!/bin/bash +export LANG=en_US.UTF-8 +PANDORA=${PANDORA-pandora} +echo Installing pandora with user: $PANDORA +getent passwd $PANDORA > /dev/null 2>&1 || adduser --disabled-password --gecos "" $PANDORA + +HOST=$(hostname -s) +HOST_CONFIG="/srv/pandora/pandora/config.$HOST.jsonc" +SITE_CONFIG="/srv/pandora/pandora/config.jsonc" +test -e $HOST_CONFIG && cp $HOST_CONFIG $SITE_CONFIG +test -e $SITE_CONFIG || cp /srv/pandora/pandora/config.pandora.jsonc $SITE_CONFIG + +cat > /srv/pandora/pandora/local_settings.py < /usr/local/bin/update.py << EOF +#!/bin/sh +exec /srv/pandora/update.py \$@ +EOF + +cat > /usr/local/bin/manage.py << EOF +#!/bin/sh +exec /srv/pandora/pandora/manage.py \$@ +EOF +chmod +x /usr/local/bin/manage.py /usr/local/bin/update.py diff --git a/docker/nginx/Dockerfile b/docker/nginx/Dockerfile new file mode 100644 index 00000000..c2c9ef58 --- /dev/null +++ b/docker/nginx/Dockerfile @@ -0,0 +1,4 @@ +FROM nginx +LABEL maintainer="0x2620@0x2620.org" +ENV LANG en_US.UTF-8 +COPY nginx.conf /etc/nginx/nginx.conf diff --git a/docker/nginx/nginx.conf b/docker/nginx/nginx.conf new file mode 100644 index 00000000..bb8b6998 --- /dev/null +++ b/docker/nginx/nginx.conf @@ -0,0 +1,96 @@ +worker_processes 4; + +events { worker_connections 1024; } + +http { + include mime.types; + + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 2048; + server_tokens off; + + charset utf-8; + charset_types text/plain text/css application/json text/json application/x-javascript text/xml application/xml application/xml+rss text/javascript application/javascript text/x-js; + + gzip on; + gzip_static on; + gzip_http_version 1.1; + gzip_vary on; + gzip_comp_level 6; + gzip_proxied any; + gzip_types text/plain text/css application/json text/json application/x-javascript text/xml application/xml application/xml+rss text/javascript application/javascript text/x-js; + gzip_buffers 16 8k; + gzip_disable "MSIE [1-6]\.(?!.*SV1)"; + + upstream pandora-web { + server pandora:2620; + } + + upstream pandora-websocket { + server websocketd:2622; + } + + server { + + listen 80 default; + + access_log off; + error_log /var/log/nginx/error.log; + + location /favicon.ico { + root /pandora/static; + } + + location /static/ { + root /pandora; + autoindex off; + } + location /data/ { + internal; + root /pandora; + } + + location /api/ws/ { + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Proxy ""; + proxy_redirect off; + proxy_buffering off; + proxy_read_timeout 999999999; + proxy_pass http://pandora-websocket/; + } + + location / { + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-Proto http; + proxy_set_header Host $http_host; + proxy_set_header Proxy ""; + proxy_redirect off; + proxy_buffering off; + proxy_read_timeout 90; #should be in sync with gunicorn timeout + proxy_connect_timeout 90; #should be in sync with gunicorn timeout + if (!-f $request_filename) { + proxy_pass http://pandora-web; + break; + } + client_max_body_size 32m; + } + + error_page 400 /; + error_page 404 /404.html; + location /404.html { + root /pandora/static/html; + } + + # redirect server error pages to the static page /50x.html + error_page 500 502 503 504 /50x.html; + location /50x.html { + root /pandora/static/html; + } + } +} diff --git a/docker/overlay/__init__.py b/docker/overlay/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/docker/overlay/install.py b/docker/overlay/install.py new file mode 100755 index 00000000..8bf0ed50 --- /dev/null +++ b/docker/overlay/install.py @@ -0,0 +1,91 @@ +#!/usr/bin/python3 + +import os +from os.path import join, abspath, basename, dirname + +# change this +name = 'placeholder' +overwrite = ( + #('home', 'indiancinema'), + #('infoView', 'indiancinema'), +) + +base = abspath(dirname(__file__)) +os.chdir(base) + +for root, folders, files in os.walk(join(base, 'static')): + for f in files: + src = join(root, f) + target = src.replace(base, '/srv/pandora') + rel_src = os.path.relpath(src, dirname(target)) + if os.path.exists(target): + os.unlink(target) + os.symlink(rel_src, target) + +if overwrite: + os.chdir('/srv/pandora/static/js') + for filename, sitename in overwrite: + src = '%s.%s.js' % (filename, sitename) + target = '%s.%s.js' % (filename, name) + if os.path.exists(target): + os.unlink(target) + os.symlink(src, target) + +os.chdir(base) +src = join(base, 'config.jsonc') +target = '/srv/pandora/pandora/config.%s.jsonc' % name +rel_src = os.path.relpath(src, dirname(target)) +if os.path.exists(target): + os.unlink(target) +os.symlink(rel_src, target) +t = '/srv/pandora/pandora/config.jsonc' +if os.path.exists(t): + os.unlink(t) +os.symlink(basename(target), t) + +for root, folders, files in os.walk(join(base, 'scripts')): + for f in files: + src = join(root, f) + target = src.replace(base, '/srv/pandora') + rel_src = os.path.relpath(src, dirname(target)) + if os.path.exists(target): + os.unlink(target) + os.symlink(rel_src, target) + if f == 'poster.%s.py' % name: + t = os.path.join(dirname(target), 'poster.py') + if os.path.exists(t): + os.unlink(t) + os.symlink(f, os.path.join(dirname(target), t)) + +if os.path.exists('settings.py'): + target = os.path.join('/srv/pandora/pandora/overlay_settings.py',) + rel_src = os.path.relpath(os.path.join(base, 'settings.py'), dirname(target)) + if os.path.exists(target): + os.unlink(target) + os.symlink(rel_src, target) + +if os.path.exists('__init__.py'): + # make module available to pandora + target = os.path.join('/srv/pandora/pandora/', name) + rel_src = os.path.relpath(base, dirname(target)) + if os.path.exists(target): + os.unlink(target) + os.symlink(rel_src, target) + + # include module in local settings + local_settings_py = '/srv/pandora/pandora/local_settings.py' + with open(local_settings_py) as fd: + local_settings_changed = False + local_settings = fd.read() + if 'LOCAL_APPS' not in local_settings: + local_settings += '\nLOCAL_APPS = ["%s"]\n' % name + local_settings_changed = True + else: + apps = re.compile('(LOCAL_APPS.*?)\]', re.DOTALL).findall(local_settings)[0] + if name not in apps: + new_apps = apps.strip() + ',\n"%s"\n' % name + local_settings = local_settings.replace(apps, new_apps) + local_settings_changed = True + if local_settings_changed: + with open(local_settings_py, 'w') as fd: + fd.write(local_settings) diff --git a/docker/overlay/settings.py b/docker/overlay/settings.py new file mode 100644 index 00000000..436f6acf --- /dev/null +++ b/docker/overlay/settings.py @@ -0,0 +1 @@ +# local settings go here diff --git a/docker/setup-docker-compose.sh b/docker/setup-docker-compose.sh new file mode 100755 index 00000000..7665e7b8 --- /dev/null +++ b/docker/setup-docker-compose.sh @@ -0,0 +1,32 @@ +#!/bin/sh +docker run 0x2620/pandora docker-compose.yml > docker-compose.yml +if [ ! -e .env ]; then + docker run 0x2620/pandora .env > .env + echo .env >> .gitignore +fi +if [ ! -e config.jsonc ]; then + docker run 0x2620/pandora config.jsonc > config.jsonc +fi +cat > README.md << EOF +pan.do/ra docker instance + +this folder was created with + + docker run 0x2620/pandora setup | sh + +To start pan.do/ra adjust the files in this folder: + + - add email configuration to .env + - adjust config.jsonc to customize pan.do/ra + - add local django settings to settings.py + +and to get started run this: + + docker-compose up -d + +To update pan.do/ra run: + + docker-compose run pandora update.py + +EOF +touch __init__.py diff --git a/docker/wait-for b/docker/wait-for new file mode 100755 index 00000000..d531df08 --- /dev/null +++ b/docker/wait-for @@ -0,0 +1,17 @@ +#!/bin/sh + +TIMEOUT=15 + +HOST="$1" +PORT="$2" + +for i in `seq $TIMEOUT` ; do + nc -z "$HOST" "$PORT" > /dev/null 2>&1 + result=$? + if [ $result -eq 0 ] ; then + exit 0 + fi + sleep 1 +done +echo "Failed to connect to database at $HOST:$PORT" >&2 +exit 1 diff --git a/docker/wait-for-file b/docker/wait-for-file new file mode 100755 index 00000000..83b043e4 --- /dev/null +++ b/docker/wait-for-file @@ -0,0 +1,12 @@ +#!/bin/sh +TIMEOUT=60 +TARGET="$1" + +for i in `seq $TIMEOUT` ; do + if [ -e "$TARGET" ]; then + exit 0 + fi + sleep 1 +done +echo "Giving up waiting for file $TARGET" >&2 +exit 1