add docker build

This commit is contained in:
j 2019-07-06 18:12:32 +02:00 committed by root
parent 5698d86756
commit fa1b98365b
18 changed files with 686 additions and 0 deletions

3
.dockerignore Normal file
View File

@ -0,0 +1,3 @@
.env
data/
overlay/

2
.gitignore vendored
View File

@ -29,3 +29,5 @@ static/django_extensions
*.swp
pandora/gunicorn_config.py
.DS_Store
.env
overlay/

11
Dockerfile Normal file
View File

@ -0,0 +1,11 @@
FROM 0x2620/pandora-base:latest
LABEL maintainer="0x2620@0x2620.org"
ENV LANG en_US.UTF-8
#VOLUME /pandora
COPY . /srv/pandora
RUN /srv/pandora/docker/install.sh
ENTRYPOINT [ "/entrypoint.sh" ]

91
docker-compose.yml Normal file
View File

@ -0,0 +1,91 @@
version: '3'
volumes:
pandora:
postgres:
rabbitmq:
networks:
backend:
services:
proxy:
build: docker/nginx
ports:
- "127.0.0.1:2620:80"
networks:
- backend
- default
links:
- pandora
- websocketd
depends_on:
- pandora
- websocketd
volumes:
- pandora:/pandora
- ./overlay:/overlay
restart: unless-stopped
db:
image: postgres:latest
networks:
- backend
env_file: .env
volumes:
- postgres:/var/lib/postgresql/data/
restart: unless-stopped
rabbitmq:
hostname: rabbitmq
image: rabbitmq:latest
env_file: .env
networks:
- backend
volumes:
- rabbitmq:/var/lib/rabbitmq
restart: unless-stopped
pandora:
hostname: pandora
build: .
command: pandora
volumes:
- pandora:/pandora
- ./overlay:/overlay
networks:
- backend
env_file: .env
links:
- db
- rabbitmq
depends_on:
- db
- rabbitmq
restart: unless-stopped
encoding: &app_base
build: .
command: encoding
env_file: .env
networks:
- backend
volumes:
- pandora:/pandora
- ./overlay:/overlay
restart: unless-stopped
tasks:
<<: *app_base
command: tasks
cron:
<<: *app_base
command: cron
websocketd:
<<: *app_base
command: websocketd
networks:
- backend

14
docker/base/Dockerfile Normal file
View File

@ -0,0 +1,14 @@
FROM debian:buster
LABEL maintainer="0x2620@0x2620.org"
ENV LANG en_US.UTF-8
RUN apt-get update && \
LANG=C.UTF-8 DEBIAN_FRONTEND=noninteractive apt-get install -y apt-utils locales && \
echo en_US.UTF-8 UTF-8 > /etc/locale.gen && \
locale-gen en_US.UTF-8 && \
update-locale LANG=en_US.UTF-8
COPY ./install.sh /install.sh
RUN /install.sh

63
docker/base/install.sh Executable file
View File

@ -0,0 +1,63 @@
#!/bin/bash
UBUNTU_CODENAME=bionic
if [ -e /etc/os-release ]; then
. /etc/os-release
fi
export DEBIAN_FRONTEND=noninteractive
echo 'Acquire::Languages "none";' > /etc/apt/apt.conf.d/99languages
apt-get update -qq
echo "deb http://ppa.launchpad.net/j/pandora/ubuntu ${UBUNTU_CODENAME} main" > /etc/apt/sources.list.d/j-pandora.list
apt-get install -y gnupg2
apt-key add - <<EOF
-----BEGIN PGP PUBLIC KEY BLOCK-----
Version: GnuPG v1
mI0ESXYhEgEEALl9jDTdmgpApPbjN+7b85dC92HisPUp56ifEkKJOBj0X5HhRqxs
Wjx/zlP4/XJGrHnxJyrdPxjSwAXz7bNdeggkN4JWdusTkr5GOXvggQnng0X7f/rX
oJwoEGtYOCODLPs6PC0qjh5yPzJVeiRsKUOZ7YVNnwNwdfS4D8RZvtCrABEBAAG0
FExhdW5jaHBhZCBQUEEgZm9yIGpeiLYEEwECACAFAkl2IRICGwMGCwkIBwMCBBUC
CAMEFgIDAQIeAQIXgAAKCRAohRM8AZde82FfA/9OB/64/YLaCpizHZ8f6DK3rGgF
e6mX3rFK8yOKGGL06316VhDzfzMiZSauUZ0t+lKHR/KZYeSaFwEoUoblTG/s4IIo
9aBMHWhVXJW6eifKUmTGqEn2/0UxoWQq2C3F6njMkCaP+ALOD5uzaSYGdjqAUAwS
pAAGSEQ4uz6bYSeM4Q==
=SM2a
-----END PGP PUBLIC KEY BLOCK-----
EOF
apt-get update -qq
apt-get install -y \
netcat-openbsd \
sudo \
vim \
wget \
pwgen \
git \
python3-setuptools \
python3-pip \
python3-venv \
python3-dev \
python3-pil \
python3-numpy \
python3-psycopg2 \
python3-pyinotify \
python3-simplejson \
python3-lxml \
python3-cssselect \
python3-html5lib \
python3-ox \
oxframe \
ffmpeg \
mkvtoolnix \
gpac \
imagemagick \
poppler-utils \
youtube-dl \
ipython3 \
postfix \
postgresql-client
apt-get clean
rm -f /install.sh

17
docker/build.sh Executable file
View File

@ -0,0 +1,17 @@
#!/bin/sh
cd `dirname $0`
HOST=`/sbin/ip route | grep docker0 | awk '{ print $9 }'`
PORT=3142
nc -z "$HOST" "$PORT" > /dev/null 2>&1
result=$?
if [ $result -eq 0 ] ; then
proxy="--build-arg http_proxy=http://$HOST:$PORT"
else
proxy=
fi
docker build $proxy -t 0x2620/pandora-base base
docker build -t 0x2620/pandora-nginx nginx
cd ..
docker build -t 0x2620/pandora .

30
docker/dot.env.sample.py Executable file
View File

@ -0,0 +1,30 @@
#!/usr/bin/python3
import os
from binascii import hexlify
import string
chars = string.ascii_letters + string.digits
def pwgen(length=16):
return ''.join(chars[c % len(chars)] for c in os.urandom(length))
print('''SECRET_KEY={SECRET_KEY}
POSTGRES_USER=pandora
POSTGRES_PASSWORD={POSTGRES_PASSWORD}
RABBITMQ_DEFAULT_USER=pandora
RABBITMQ_DEFAULT_PASS={RABBITMQ_PASSWORD}
# required to send out password reset emails
#EMAIL_HOST='example.com'
#EMAIL_USER='mail@example.com'
#EMAIL_PASSWORD='fixme'
#EMAIL_PORT=587
#EMAIL_TLS=true
'''.format(
SECRET_KEY=hexlify(os.urandom(64)).decode(),
POSTGRES_PASSWORD=pwgen(),
RABBITMQ_PASSWORD=pwgen()
))

134
docker/entrypoint.sh Executable file
View File

@ -0,0 +1,134 @@
#!/bin/bash
set -e
action="$1"
user=pandora
export LANG=en_US.UTF-8
mkdir -p /run/pandora
chown -R ${user}.${user} /run/pandora
# pan.do/ra services
if [ "$action" = "pandora" ]; then
if [ ! -e /srv/pandora/initialized ]; then
echo "Setting up Pan.do/ra:"
echo "Waiting for database connection..."
/srv/pandora_base/docker/wait-for db 5432
echo "Installing pan.do/ra..."
rsync -a /srv/pandora_base/ /srv/pandora/
if [ ! -e /overlay/install.py ]; then
rsync -a /srv/pandora_base/docker/overlay/ /overlay/
if [ ! -e /overlay/config.jsonc ]; then
mv /srv/pandora/pandora/config.jsonc /overlay/config.jsonc
fi
fi
/overlay/install.py
echo "Initializing database..."
echo "CREATE EXTENSION pg_trgm;" | /srv/pandora/pandora/manage.py dbshell
/srv/pandora/pandora/manage.py init_db
/srv/pandora/update.py db
echo "Generating static files..."
/srv/pandora/update.py static
chown -R ${user}.${user} /srv/pandora/
touch /srv/pandora/initialized
fi
/srv/pandora_base/docker/wait-for db 5432
/srv/pandora_base/docker/wait-for rabbitmq 5672
cd /srv/pandora/pandora
exec /usr/bin/sudo -u $user -E -H \
/srv/pandora/bin/gunicorn wsgi:application -c gunicorn_config.py
fi
if [ "$action" = "encoding" ]; then
/srv/pandora_base/docker/wait-for-file /srv/pandora/initialized
/srv/pandora_base/docker/wait-for rabbitmq 5672
name=pandora-encoding-$(hostname)
exec /usr/bin/sudo -u $user -E -H \
/srv/pandora/bin/python \
/srv/pandora/pandora/manage.py \
celery worker \
-c 1 \
-Q encoding -n $name \
-l INFO
fi
if [ "$action" = "tasks" ]; then
/srv/pandora_base/docker/wait-for-file /srv/pandora/initialized
/srv/pandora_base/docker/wait-for rabbitmq 5672
name=pandora-default-$(hostname)
exec /usr/bin/sudo -u $user -E -H \
/srv/pandora/bin/python \
/srv/pandora/pandora/manage.py \
celery worker \
-Q default,celery -n $name \
--maxtasksperchild 1000 \
-l INFO
fi
if [ "$action" = "cron" ]; then
/srv/pandora_base/docker/wait-for-file /srv/pandora/initialized
/srv/pandora_base/docker/wait-for rabbitmq 5672
exec /usr/bin/sudo -u $user -E -H \
/srv/pandora/bin/python \
/srv/pandora/pandora/manage.py \
celerybeat -s /run/pandora/celerybeat-schedule \
--pidfile /run/pandora/cron.pid \
-l INFO
fi
if [ "$action" = "websocketd" ]; then
/srv/pandora_base/docker/wait-for-file /srv/pandora/initialized
/srv/pandora_base/docker/wait-for rabbitmq 5672
exec /usr/bin/sudo -u $user -E -H \
/srv/pandora/bin/python \
/srv/pandora/pandora/manage.py websocketd
fi
# pan.do/ra management and update
if [ "$action" = "manage.py" ]; then
shift
exec /usr/bin/sudo -u $user -E -H \
/srv/pandora/pandora/manage.py "$@"
fi
if [ "$action" = "update.py" ]; then
shift
exec /usr/bin/sudo -u $user -E -H \
/srv/pandora/update.py "$@"
fi
if [ "$action" = "bash" ]; then
shift
cd /
exec /bin/bash "$@"
fi
# pan.do/ra setup hooks
if [ "$action" = "docker-compose.yml" ]; then
cat /srv/pandora_base/docker-compose.yml | \
sed "s#build: \.#image: 0x2620/pandora:latest#g" | \
sed "s#\./overlay:#.:#g" | \
sed "s#build: docker/nginx#image: 0x2620/pandora-nginx:latest#g"
exit
fi
if [ "$action" = ".env" ]; then
exec /srv/pandora_base/docker/dot.env.sample.py
fi
if [ "$action" = "config.jsonc" ]; then
cat /srv/pandora_base/pandora/config.pandora.jsonc
exit
fi
if [ "$action" = "setup" ]; then
cat /srv/pandora_base/docker/setup-docker-compose.sh
exit
fi
# pan.do/ra info
echo pan.do/ra docker container - https://pan.do/ra
echo
echo use this container with docker-compose,
echo to setup a new docker-compose envrionment run:
echo
echo " mkdir <sitename> && cd <sitename>"
echo " docker run 0x2620/pandora setup | sh"
echo
echo adjust created files to match your needs and run:
echo
echo " docker-compose up"
echo

68
docker/install.sh Executable file
View File

@ -0,0 +1,68 @@
#!/bin/bash
export LANG=en_US.UTF-8
PANDORA=${PANDORA-pandora}
echo Installing pandora with user: $PANDORA
getent passwd $PANDORA > /dev/null 2>&1 || adduser --disabled-password --gecos "" $PANDORA
HOST=$(hostname -s)
HOST_CONFIG="/srv/pandora/pandora/config.$HOST.jsonc"
SITE_CONFIG="/srv/pandora/pandora/config.jsonc"
test -e $HOST_CONFIG && cp $HOST_CONFIG $SITE_CONFIG
test -e $SITE_CONFIG || cp /srv/pandora/pandora/config.pandora.jsonc $SITE_CONFIG
cat > /srv/pandora/pandora/local_settings.py <<EOF
import os
SECRET_KEY = os.environ.get('SECRET_KEY')
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.postgresql_psycopg2',
'NAME': 'postgres',
'USER': os.environ.get('POSTGRES_USER'),
'PASSWORD': os.environ.get('POSTGRES_PASSWORD'),
'HOST': 'db',
'PORT': 5432,
}
}
BROKER_URL = "amqp://{0}:{1}@rabbitmq:5672//".format(os.environ.get('RABBITMQ_DEFAULT_USER'), os.environ.get('RABBITMQ_DEFAULT_PASS'))
XACCELREDIRECT = True
DEBUG = False
TEMPLATE_DEBUG = DEBUG
JSON_DEBUG = False
DB_GIN_TRGM = True
WEBSOCKET_ADDRESS = "0.0.0.0"
EMAIL_HOST=os.environ.get('EMAIL_HOST')
EMAIL_HOST_USER=os.environ.get('EMAIL_USER')
EMAIL_HOST_PASSWORD=os.environ.get('EMAIL_PASSWORD')
EMAIL_PORT=int(os.environ.get('EMAIL_PORT', 587))
EMAIL_USE_TLS=os.environ.get('EMAIL_TLS', 'true').lower() == 'true'
overlay_settings = 'overlay_settings.py'
if os.path.exists(overlay_settings):
from overlay_settings import *
EOF
cp /srv/pandora/pandora/gunicorn_config.py.in /srv/pandora/pandora/gunicorn_config.py
sed -i s/127.0.0.1/0.0.0.0/g /srv/pandora/pandora/gunicorn_config.py
chown -R $PANDORA:$PANDORA /srv/pandora
cd /srv/pandora
./ctl init
cp /srv/pandora/docker/entrypoint.sh /entrypoint.sh
mv /srv/pandora/ /srv/pandora_base/
mkdir /pandora
ln -s /pandora /srv/pandora
cat > /usr/local/bin/update.py << EOF
#!/bin/sh
exec /srv/pandora/update.py \$@
EOF
cat > /usr/local/bin/manage.py << EOF
#!/bin/sh
exec /srv/pandora/pandora/manage.py \$@
EOF
chmod +x /usr/local/bin/manage.py /usr/local/bin/update.py

4
docker/nginx/Dockerfile Normal file
View File

@ -0,0 +1,4 @@
FROM nginx
LABEL maintainer="0x2620@0x2620.org"
ENV LANG en_US.UTF-8
COPY nginx.conf /etc/nginx/nginx.conf

96
docker/nginx/nginx.conf Normal file
View File

@ -0,0 +1,96 @@
worker_processes 4;
events { worker_connections 1024; }
http {
include mime.types;
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
server_tokens off;
charset utf-8;
charset_types text/plain text/css application/json text/json application/x-javascript text/xml application/xml application/xml+rss text/javascript application/javascript text/x-js;
gzip on;
gzip_static on;
gzip_http_version 1.1;
gzip_vary on;
gzip_comp_level 6;
gzip_proxied any;
gzip_types text/plain text/css application/json text/json application/x-javascript text/xml application/xml application/xml+rss text/javascript application/javascript text/x-js;
gzip_buffers 16 8k;
gzip_disable "MSIE [1-6]\.(?!.*SV1)";
upstream pandora-web {
server pandora:2620;
}
upstream pandora-websocket {
server websocketd:2622;
}
server {
listen 80 default;
access_log off;
error_log /var/log/nginx/error.log;
location /favicon.ico {
root /pandora/static;
}
location /static/ {
root /pandora;
autoindex off;
}
location /data/ {
internal;
root /pandora;
}
location /api/ws/ {
proxy_http_version 1.1;
proxy_set_header Host $http_host;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_set_header Proxy "";
proxy_redirect off;
proxy_buffering off;
proxy_read_timeout 999999999;
proxy_pass http://pandora-websocket/;
}
location / {
proxy_set_header X-Forwarded-For $remote_addr;
proxy_set_header X-Forwarded-Proto http;
proxy_set_header Host $http_host;
proxy_set_header Proxy "";
proxy_redirect off;
proxy_buffering off;
proxy_read_timeout 90; #should be in sync with gunicorn timeout
proxy_connect_timeout 90; #should be in sync with gunicorn timeout
if (!-f $request_filename) {
proxy_pass http://pandora-web;
break;
}
client_max_body_size 32m;
}
error_page 400 /;
error_page 404 /404.html;
location /404.html {
root /pandora/static/html;
}
# redirect server error pages to the static page /50x.html
error_page 500 502 503 504 /50x.html;
location /50x.html {
root /pandora/static/html;
}
}
}

View File

91
docker/overlay/install.py Executable file
View File

@ -0,0 +1,91 @@
#!/usr/bin/python3
import os
from os.path import join, abspath, basename, dirname
# change this
name = 'placeholder'
overwrite = (
#('home', 'indiancinema'),
#('infoView', 'indiancinema'),
)
base = abspath(dirname(__file__))
os.chdir(base)
for root, folders, files in os.walk(join(base, 'static')):
for f in files:
src = join(root, f)
target = src.replace(base, '/srv/pandora')
rel_src = os.path.relpath(src, dirname(target))
if os.path.exists(target):
os.unlink(target)
os.symlink(rel_src, target)
if overwrite:
os.chdir('/srv/pandora/static/js')
for filename, sitename in overwrite:
src = '%s.%s.js' % (filename, sitename)
target = '%s.%s.js' % (filename, name)
if os.path.exists(target):
os.unlink(target)
os.symlink(src, target)
os.chdir(base)
src = join(base, 'config.jsonc')
target = '/srv/pandora/pandora/config.%s.jsonc' % name
rel_src = os.path.relpath(src, dirname(target))
if os.path.exists(target):
os.unlink(target)
os.symlink(rel_src, target)
t = '/srv/pandora/pandora/config.jsonc'
if os.path.exists(t):
os.unlink(t)
os.symlink(basename(target), t)
for root, folders, files in os.walk(join(base, 'scripts')):
for f in files:
src = join(root, f)
target = src.replace(base, '/srv/pandora')
rel_src = os.path.relpath(src, dirname(target))
if os.path.exists(target):
os.unlink(target)
os.symlink(rel_src, target)
if f == 'poster.%s.py' % name:
t = os.path.join(dirname(target), 'poster.py')
if os.path.exists(t):
os.unlink(t)
os.symlink(f, os.path.join(dirname(target), t))
if os.path.exists('settings.py'):
target = os.path.join('/srv/pandora/pandora/overlay_settings.py',)
rel_src = os.path.relpath(os.path.join(base, 'settings.py'), dirname(target))
if os.path.exists(target):
os.unlink(target)
os.symlink(rel_src, target)
if os.path.exists('__init__.py'):
# make module available to pandora
target = os.path.join('/srv/pandora/pandora/', name)
rel_src = os.path.relpath(base, dirname(target))
if os.path.exists(target):
os.unlink(target)
os.symlink(rel_src, target)
# include module in local settings
local_settings_py = '/srv/pandora/pandora/local_settings.py'
with open(local_settings_py) as fd:
local_settings_changed = False
local_settings = fd.read()
if 'LOCAL_APPS' not in local_settings:
local_settings += '\nLOCAL_APPS = ["%s"]\n' % name
local_settings_changed = True
else:
apps = re.compile('(LOCAL_APPS.*?)\]', re.DOTALL).findall(local_settings)[0]
if name not in apps:
new_apps = apps.strip() + ',\n"%s"\n' % name
local_settings = local_settings.replace(apps, new_apps)
local_settings_changed = True
if local_settings_changed:
with open(local_settings_py, 'w') as fd:
fd.write(local_settings)

View File

@ -0,0 +1 @@
# local settings go here

32
docker/setup-docker-compose.sh Executable file
View File

@ -0,0 +1,32 @@
#!/bin/sh
docker run 0x2620/pandora docker-compose.yml > docker-compose.yml
if [ ! -e .env ]; then
docker run 0x2620/pandora .env > .env
echo .env >> .gitignore
fi
if [ ! -e config.jsonc ]; then
docker run 0x2620/pandora config.jsonc > config.jsonc
fi
cat > README.md << EOF
pan.do/ra docker instance
this folder was created with
docker run 0x2620/pandora setup | sh
To start pan.do/ra adjust the files in this folder:
- add email configuration to .env
- adjust config.jsonc to customize pan.do/ra
- add local django settings to settings.py
and to get started run this:
docker-compose up -d
To update pan.do/ra run:
docker-compose run pandora update.py
EOF
touch __init__.py

17
docker/wait-for Executable file
View File

@ -0,0 +1,17 @@
#!/bin/sh
TIMEOUT=15
HOST="$1"
PORT="$2"
for i in `seq $TIMEOUT` ; do
nc -z "$HOST" "$PORT" > /dev/null 2>&1
result=$?
if [ $result -eq 0 ] ; then
exit 0
fi
sleep 1
done
echo "Failed to connect to database at $HOST:$PORT" >&2
exit 1

12
docker/wait-for-file Executable file
View File

@ -0,0 +1,12 @@
#!/bin/sh
TIMEOUT=60
TARGET="$1"
for i in `seq $TIMEOUT` ; do
if [ -e "$TARGET" ]; then
exit 0
fi
sleep 1
done
echo "Giving up waiting for file $TARGET" >&2
exit 1