forked from 0x2620/pandora
Compare commits
321 commits
Author | SHA1 | Date | |
---|---|---|---|
ff236e8828 | |||
34af2b1fab | |||
d83309d4cd | |||
59c2045ac6 | |||
a24d96b098 | |||
03daede441 | |||
9e6ecb5459 | |||
e7ede6ade0 | |||
f4bfe9294b | |||
c69ca372ee | |||
9e00dd09e3 | |||
7cc1504069 | |||
d5ace7aeca | |||
1b1442e664 | |||
18cbf0ec9c | |||
a8aa619217 | |||
ff1c929d4d | |||
e8c28c16c3 | |||
caae7d84bc | |||
a218b906cc | |||
446a748a79 | |||
1ccc8df75c | |||
f245824448 | |||
12e0181157 | |||
4748930460 | |||
b8de041316 | |||
140b7454e0 | |||
9ed5a4abd9 | |||
d7b2cb18f7 | |||
48f5bcfd11 | |||
b482600179 | |||
ca701b9749 | |||
3579a4de45 | |||
df1a96f97a | |||
8b60075f39 | |||
9e464a1d63 | |||
d98177e686 | |||
27d64f2ad5 | |||
cf0ad718f7 | |||
8e32737776 | |||
3b56274713 | |||
774450d263 | |||
e1a28c941a | |||
c55c733bcd | |||
853efbba9a | |||
5ade53a0b5 | |||
5af5812179 | |||
a25509e1a4 | |||
8fa00f31a8 | |||
82615cfb18 | |||
5bf71ead09 | |||
55f07ba4b7 | |||
7ad121d912 | |||
1b7ee7b275 | |||
03bd598785 | |||
40c8c52180 | |||
9b2fbe2e3f | |||
ecfdee6298 | |||
5ffcdda762 | |||
037be04549 | |||
d02055d85e | |||
3683e60497 | |||
876044ce74 | |||
74fca45a9f | |||
bc3208a846 | |||
540ee0f4c0 | |||
35852c36cf | |||
1c8ea51d9b | |||
ba5fba5a10 | |||
69cd34489a | |||
5fa3b66a43 | |||
f9edbbe1c6 | |||
4794c0f68a | |||
de818c4204 | |||
bcece622e3 | |||
7c96d9e994 | |||
2cd000698e | |||
a32ede6b2f | |||
a890b11790 | |||
dca95f9d95 | |||
667ad0c9d5 | |||
4ef8773562 | |||
384d92214d | |||
19bb7e3b83 | |||
95485f3e4b | |||
4c415c6b5d | |||
a537963b9f | |||
ee11e6b7e1 | |||
b82440df24 | |||
4fed1112a6 | |||
f671971b35 | |||
c0420cabcb | |||
06dbc277c7 | |||
a2b7cc9744 | |||
615860a7e8 | |||
a99a3eee7a | |||
32a73f402b | |||
47b348f724 | |||
5bc0b70d73 | |||
b0356a63bd | |||
6cf62ed1cd | |||
39015c0148 | |||
900f9bb44d | |||
628bc728ed | |||
869d154d5b | |||
2d30059cee | |||
cbaeffde68 | |||
eee51e672c | |||
49187c8faf | |||
2905774b10 | |||
d7f087125e | |||
61dd667a71 | |||
ce27979259 | |||
09e3f2bc7d | |||
d44e5e2b25 | |||
56f248894e | |||
a8d906743a | |||
f2ad63143a | |||
48aba7bde8 | |||
a7e660a8da | |||
a5c35d0f41 | |||
9d5222f0cc | |||
a25ad7c947 | |||
4073942ec2 | |||
8c977ba11b | |||
5c951e2559 | |||
094fb06b92 | |||
fdc7279836 | |||
f561f1b5e2 | |||
27e5449e59 | |||
4fb8879ba0 | |||
543d01696e | |||
a035d37229 | |||
ff0c9b8221 | |||
13d6f7e316 | |||
262e3f29e0 | |||
00194f13ba | |||
27c2e9c849 | |||
c80f16c77a | |||
d4df903f82 | |||
7cc5ae8d65 | |||
4913bf8d38 | |||
7369954c5f | |||
30af377ecb | |||
6e283ad870 | |||
b13b621fd0 | |||
84c2a3ac3c | |||
cf6374e8a6 | |||
d4e89db4c1 | |||
a8cc838d7b | |||
cff1c06546 | |||
9b1f482a21 | |||
bea0d301a4 | |||
17801df8de | |||
5466247848 | |||
e7292a5989 | |||
b0ece6a566 | |||
83df2c0011 | |||
9355ae691d | |||
5225026d66 | |||
dcdbfee72d | |||
e41b7e19d4 | |||
1175e5d5bf | |||
6e8a338f8a | |||
6fb635ebf8 | |||
11c8bb9076 | |||
fd294baec9 | |||
51038b7c59 | |||
adaeb16c69 | |||
409c5a1fc2 | |||
512f07400d | |||
034b448846 | |||
bd9d2ecd7e | |||
028def73d9 | |||
a667fc5b5b | |||
f5becafad4 | |||
92d5c6e763 | |||
962be98592 | |||
eb15170059 | |||
a2c2e32bfe | |||
d3b9d87c43 | |||
8c3d089bd8 | |||
4ddb170bc9 | |||
f65336c28d | |||
345195e959 | |||
69fae04bb8 | |||
94fc4fbe7a | |||
64b8967ded | |||
068293050e | |||
dffb47ffef | |||
9c3edac263 | |||
9340c0e578 | |||
f71434a1ff | |||
3c73ef8999 | |||
f664927b5f | |||
21db208556 | |||
55e47c36bd | |||
48e6d4af6f | |||
57d3fc0d32 | |||
0d70326aa8 | |||
614beae48e | |||
19bd818461 | |||
448a4ff5e1 | |||
42bfbb0808 | |||
7216e255de | |||
a3880f3b38 | |||
97c8cf67b9 | |||
873ec27803 | |||
623bbd472c | |||
99a135c7d3 | |||
a80af18400 | |||
ffc2504c0f | |||
729b2ea771 | |||
ccbc966282 | |||
171c0b6095 | |||
1c8b5b4b48 | |||
1b0de9ade4 | |||
65710e5ef1 | |||
40e2481f61 | |||
d191f24d03 | |||
86de7f6269 | |||
9d124d65af | |||
4be61a3982 | |||
e3e5e3b9e5 | |||
5505d495cf | |||
af65750363 | |||
5cfe392e22 | |||
79c5d948b5 | |||
e4133ffa1d | |||
723c4e3f42 | |||
85fd930360 | |||
e237d08120 | |||
5c89662f14 | |||
fa42aa4810 | |||
d1c157ffb4 | |||
950bec609d | |||
cc2b60453b | |||
2fec9590f7 | |||
0cfb499594 | |||
491d8c4629 | |||
476fa9f0a3 | |||
65fc082b1b | |||
49356d2f7a | |||
4dbb5e3c51 | |||
ae8363d4e2 | |||
10dd0768d1 | |||
bc123a885e | |||
ed31259fdd | |||
26708eff1f | |||
fd318ef706 | |||
8a16c7e37f | |||
6118bf3c3a | |||
cabcbeb35d | |||
098d953bbc | |||
0ba80ada1f | |||
0a7b6c0452 | |||
b1834520bd | |||
066d273e10 | |||
cd8cd40626 | |||
cf95aeff7a | |||
feb10ececb | |||
47e4f1cd84 | |||
098811d383 | |||
41d417e2a1 | |||
a4d0830e4b | |||
5583fad801 | |||
ffd735e622 | |||
894e5c5064 | |||
166b566fde | |||
157cc7478b | |||
02b89c06dd | |||
49ef611bdb | |||
86bb19b9b6 | |||
a9f883859a | |||
08dc8f2bea | |||
c17cc6d5cf | |||
54362e8b41 | |||
4e1e6e3b17 | |||
814f98a571 | |||
7fa3f7d8fe | |||
7f31998254 | |||
20b3831862 | |||
4689af2050 | |||
7ea1f38a0f | |||
7d8b38e627 | |||
b377956a13 | |||
e7df18f727 | |||
1bba157f7f | |||
41fce072a3 | |||
4cf28434eb | |||
5d6e753b05 | |||
f157ebf2c4 | |||
ac8c01ee98 | |||
f13f7d238c | |||
63d82ad7e7 | |||
7ab789f80a | |||
213adcaaaa | |||
fdac35c00d | |||
03e85c6eef | |||
09f9580e1e | |||
2dfe0f3ff2 | |||
5bbb6ca195 | |||
2779d8d099 | |||
d00cf08638 | |||
945ac98dad | |||
5024a2ba0c | |||
940632369a | |||
1c0462393c | |||
40edf9dd4a | |||
ca7741f92c | |||
2d8a3f24dc | |||
d795e40344 | |||
ffb512a304 | |||
800725d54c | |||
80597790f9 | |||
638dfc8bb3 | |||
e4815f091d | |||
4bde77abcc | |||
953cb93745 | |||
19da7ca26d | |||
8788dd9fe8 |
637 changed files with 158409 additions and 85258 deletions
2
.gitignore
vendored
2
.gitignore
vendored
|
@ -36,3 +36,5 @@ pandora/gunicorn_config.py
|
|||
.DS_Store
|
||||
.env
|
||||
overlay/
|
||||
pandora/encoding.conf
|
||||
pandora/tasks.conf
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
FROM 0x2620/pandora-base:latest
|
||||
FROM code.0x2620.org/0x2620/pandora-base:latest
|
||||
|
||||
LABEL maintainer="0x2620@0x2620.org"
|
||||
|
||||
|
|
|
@ -7,7 +7,7 @@
|
|||
We recommend to run pan.do/ra inside of LXD or LXC or dedicated VM or server.
|
||||
You will need at least 2GB of free disk space
|
||||
|
||||
pan.do/ra is known to work with Ubuntu 18.04, 20.04 and Debian/10 (buster),
|
||||
pan.do/ra is known to work with Debian/12 (bookworm) and Ubuntu 20.04,
|
||||
other distributions might also work, let us know if it works for you.
|
||||
|
||||
Use the following commands as root to install pan.do/ra and all dependencies:
|
||||
|
@ -16,7 +16,7 @@
|
|||
cd /root
|
||||
curl -sL https://pan.do/ra-install > pandora_install.sh
|
||||
chmod +x pandora_install.sh
|
||||
export BRANCH=stable # change to 'master' to get current developement version
|
||||
export BRANCH=master # change to 'stable' to get the latest release (sometimes outdated)
|
||||
./pandora_install.sh 2>&1 | tee pandora_install.log
|
||||
```
|
||||
|
||||
|
@ -50,4 +50,9 @@ export BRANCH=stable # change to 'master' to get current developement version
|
|||
More info at
|
||||
https://code.0x2620.org/0x2620/pandora/wiki/Customization
|
||||
|
||||
## Update
|
||||
|
||||
To update your existing instlalation run
|
||||
|
||||
pandoractl update
|
||||
|
||||
|
|
35
ctl
35
ctl
|
@ -17,7 +17,7 @@ if [ "$action" = "init" ]; then
|
|||
SUDO=""
|
||||
PANDORA_USER=`ls -l update.py | cut -f3 -d" "`
|
||||
if [ `whoami` != $PANDORA_USER ]; then
|
||||
SUDO="sudo -H -u $PANDORA_USER"
|
||||
SUDO="sudo -E -H -u $PANDORA_USER"
|
||||
fi
|
||||
$SUDO python3 -m venv --system-site-packages .
|
||||
branch=`cat .git/HEAD | sed 's@/@\n@g' | tail -n1`
|
||||
|
@ -27,25 +27,30 @@ if [ "$action" = "init" ]; then
|
|||
$SUDO bin/python3 -m pip install -U --ignore-installed "pip<9"
|
||||
fi
|
||||
if [ ! -d static/oxjs ]; then
|
||||
$SUDO git clone --depth 1 -b $branch https://git.0x2620.org/oxjs.git static/oxjs
|
||||
$SUDO git clone -b $branch https://code.0x2620.org/0x2620/oxjs.git static/oxjs
|
||||
fi
|
||||
$SUDO mkdir -p src
|
||||
if [ ! -d src/oxtimelines ]; then
|
||||
$SUDO git clone --depth 1 -b $branch https://git.0x2620.org/oxtimelines.git src/oxtimelines
|
||||
fi
|
||||
for package in oxtimelines python-ox; do
|
||||
cd ${BASE}
|
||||
if [ ! -d src/${package} ]; then
|
||||
$SUDO git clone --depth 1 -b $branch https://git.0x2620.org/${package}.git src/${package}
|
||||
$SUDO git clone -b $branch https://code.0x2620.org/0x2620/${package}.git src/${package}
|
||||
fi
|
||||
cd ${BASE}/src/${package}
|
||||
$SUDO ${BASE}/bin/python setup.py develop
|
||||
|
||||
$SUDO ${BASE}/bin/pip install -e .
|
||||
|
||||
done
|
||||
cd ${BASE}
|
||||
$SUDO ./bin/pip install -r requirements.txt
|
||||
if [ ! -e pandora/gunicorn_config.py ]; then
|
||||
$SUDO cp pandora/gunicorn_config.py.in pandora/gunicorn_config.py
|
||||
for template in gunicorn_config.py encoding.conf tasks.conf; do
|
||||
if [ ! -e pandora/$template ]; then
|
||||
$SUDO cp pandora/${template}.in pandora/$template
|
||||
fi
|
||||
done
|
||||
exit 0
|
||||
fi
|
||||
if [ "$action" = "version" ]; then
|
||||
git rev-list HEAD --count
|
||||
exit 0
|
||||
fi
|
||||
|
||||
|
@ -62,11 +67,10 @@ if [ ! -z $cmd ]; then
|
|||
SUDO=""
|
||||
PANDORA_USER=`ls -l update.py | cut -f3 -d" "`
|
||||
if [ `whoami` != $PANDORA_USER ]; then
|
||||
SUDO="sudo -H -u $PANDORA_USER"
|
||||
SUDO="sudo -E -H -u $PANDORA_USER"
|
||||
fi
|
||||
shift
|
||||
$SUDO "$BASE/$cmd" $@
|
||||
exit $?
|
||||
exec $SUDO "$BASE/$cmd" $@
|
||||
fi
|
||||
|
||||
if [ `whoami` != 'root' ]; then
|
||||
|
@ -74,10 +78,15 @@ if [ `whoami` != 'root' ]; then
|
|||
exit 1
|
||||
fi
|
||||
if [ "$action" = "install" ]; then
|
||||
cd "`dirname "$0"`"
|
||||
cd "`dirname "$self"`"
|
||||
BASE=`pwd`
|
||||
if [ -x /bin/systemctl ]; then
|
||||
if [ -d /etc/systemd/system/ ]; then
|
||||
for template in gunicorn_config.py encoding.conf tasks.conf; do
|
||||
if [ ! -e pandora/$template ]; then
|
||||
$SUDO cp pandora/${template}.in pandora/$template
|
||||
fi
|
||||
done
|
||||
for service in $SERVICES; do
|
||||
if [ -e /lib/systemd/system/${service}.service ]; then
|
||||
rm -f /lib/systemd/system/${service}.service \
|
||||
|
|
|
@ -15,7 +15,6 @@ services:
|
|||
- "127.0.0.1:2620:80"
|
||||
networks:
|
||||
- backend
|
||||
- default
|
||||
links:
|
||||
- pandora
|
||||
- websocketd
|
||||
|
@ -28,7 +27,7 @@ services:
|
|||
restart: unless-stopped
|
||||
|
||||
db:
|
||||
image: postgres:latest
|
||||
image: postgres:15
|
||||
networks:
|
||||
- backend
|
||||
env_file: .env
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
FROM debian:buster
|
||||
FROM debian:12
|
||||
|
||||
LABEL maintainer="0x2620@0x2620.org"
|
||||
|
||||
|
|
|
@ -1,9 +1,17 @@
|
|||
#!/bin/bash
|
||||
|
||||
UBUNTU_CODENAME=bionic
|
||||
if [ -e /etc/os-release ]; then
|
||||
. /etc/os-release
|
||||
fi
|
||||
if [ -z "$UBUNTU_CODENAME" ]; then
|
||||
UBUNTU_CODENAME=bionic
|
||||
fi
|
||||
if [ "$VERSION_CODENAME" = "bullseye" ]; then
|
||||
UBUNTU_CODENAME=focal
|
||||
fi
|
||||
if [ "$VERSION_CODENAME" = "bookworm" ]; then
|
||||
UBUNTU_CODENAME=lunar
|
||||
fi
|
||||
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
echo 'Acquire::Languages "none";' > /etc/apt/apt.conf.d/99languages
|
||||
|
@ -30,6 +38,8 @@ apt-get update -qq
|
|||
apt-get install -y \
|
||||
netcat-openbsd \
|
||||
sudo \
|
||||
rsync \
|
||||
iproute2 \
|
||||
vim \
|
||||
wget \
|
||||
pwgen \
|
||||
|
@ -42,22 +52,23 @@ apt-get install -y \
|
|||
python3-numpy \
|
||||
python3-psycopg2 \
|
||||
python3-pyinotify \
|
||||
python3-simplejson \
|
||||
python3-lxml \
|
||||
python3-cssselect \
|
||||
python3-html5lib \
|
||||
python3-ox \
|
||||
python3-elasticsearch \
|
||||
oxframe \
|
||||
ffmpeg \
|
||||
mkvtoolnix \
|
||||
gpac \
|
||||
imagemagick \
|
||||
poppler-utils \
|
||||
youtube-dl \
|
||||
ipython3 \
|
||||
tesseract-ocr \
|
||||
tesseract-ocr-eng \
|
||||
postfix \
|
||||
postgresql-client
|
||||
|
||||
apt-get install -y --no-install-recommends youtube-dl rtmpdump
|
||||
apt-get clean
|
||||
|
||||
rm -f /install.sh
|
||||
|
|
|
@ -11,7 +11,7 @@ else
|
|||
proxy=
|
||||
fi
|
||||
|
||||
docker build $proxy -t 0x2620/pandora-base base
|
||||
docker build -t 0x2620/pandora-nginx nginx
|
||||
docker build $proxy -t code.0x2620.org/0x2620/pandora-base base
|
||||
docker build -t code.0x2620.org/0x2620/pandora-nginx nginx
|
||||
cd ..
|
||||
docker build -t 0x2620/pandora .
|
||||
docker build -t code.0x2620.org/0x2620/pandora .
|
||||
|
|
|
@ -6,7 +6,9 @@ user=pandora
|
|||
|
||||
export LANG=en_US.UTF-8
|
||||
mkdir -p /run/pandora
|
||||
chown -R ${user}.${user} /run/pandora
|
||||
chown -R ${user}:${user} /run/pandora
|
||||
|
||||
update="/usr/bin/sudo -u $user -E -H /srv/pandora/update.py"
|
||||
|
||||
# pan.do/ra services
|
||||
if [ "$action" = "pandora" ]; then
|
||||
|
@ -26,12 +28,12 @@ if [ "$action" = "pandora" ]; then
|
|||
/overlay/install.py
|
||||
|
||||
echo "Initializing database..."
|
||||
echo "CREATE EXTENSION pg_trgm;" | /srv/pandora/pandora/manage.py dbshell
|
||||
echo "CREATE EXTENSION pg_trgm;" | /srv/pandora/pandora/manage.py dbshell || true
|
||||
/srv/pandora/pandora/manage.py init_db
|
||||
/srv/pandora/update.py db
|
||||
$update db
|
||||
echo "Generating static files..."
|
||||
/srv/pandora/update.py static
|
||||
chown -R ${user}.${user} /srv/pandora/
|
||||
chown -R ${user}:${user} /srv/pandora/
|
||||
$update static
|
||||
touch /srv/pandora/initialized
|
||||
fi
|
||||
/srv/pandora_base/docker/wait-for db 5432
|
||||
|
@ -44,54 +46,53 @@ if [ "$action" = "encoding" ]; then
|
|||
/srv/pandora_base/docker/wait-for-file /srv/pandora/initialized
|
||||
/srv/pandora_base/docker/wait-for rabbitmq 5672
|
||||
name=pandora-encoding-$(hostname)
|
||||
cd /srv/pandora/pandora
|
||||
exec /usr/bin/sudo -u $user -E -H \
|
||||
/srv/pandora/bin/python \
|
||||
/srv/pandora/pandora/manage.py \
|
||||
celery worker \
|
||||
/srv/pandora/bin/celery \
|
||||
-A app worker \
|
||||
-Q encoding -n ${name} \
|
||||
--pidfile /run/pandora/encoding.pid \
|
||||
--max-tasks-per-child 500 \
|
||||
-c 1 \
|
||||
-Q encoding -n $name \
|
||||
-l INFO
|
||||
fi
|
||||
if [ "$action" = "tasks" ]; then
|
||||
/srv/pandora_base/docker/wait-for-file /srv/pandora/initialized
|
||||
/srv/pandora_base/docker/wait-for rabbitmq 5672
|
||||
name=pandora-default-$(hostname)
|
||||
cd /srv/pandora/pandora
|
||||
exec /usr/bin/sudo -u $user -E -H \
|
||||
/srv/pandora/bin/python \
|
||||
/srv/pandora/pandora/manage.py \
|
||||
celery worker \
|
||||
-Q default,celery -n $name \
|
||||
--maxtasksperchild 1000 \
|
||||
/srv/pandora/bin/celery \
|
||||
-A app worker \
|
||||
-Q default,celery -n ${name} \
|
||||
--pidfile /run/pandora/tasks.pid \
|
||||
--max-tasks-per-child 1000 \
|
||||
-l INFO
|
||||
fi
|
||||
if [ "$action" = "cron" ]; then
|
||||
/srv/pandora_base/docker/wait-for-file /srv/pandora/initialized
|
||||
/srv/pandora_base/docker/wait-for rabbitmq 5672
|
||||
cd /srv/pandora/pandora
|
||||
exec /usr/bin/sudo -u $user -E -H \
|
||||
/srv/pandora/bin/python \
|
||||
/srv/pandora/pandora/manage.py \
|
||||
celerybeat -s /run/pandora/celerybeat-schedule \
|
||||
/srv/pandora/bin/celery \
|
||||
-A app beat \
|
||||
-s /run/pandora/celerybeat-schedule \
|
||||
--pidfile /run/pandora/cron.pid \
|
||||
-l INFO
|
||||
fi
|
||||
if [ "$action" = "websocketd" ]; then
|
||||
/srv/pandora_base/docker/wait-for-file /srv/pandora/initialized
|
||||
/srv/pandora_base/docker/wait-for rabbitmq 5672
|
||||
cd /srv/pandora/pandora
|
||||
exec /usr/bin/sudo -u $user -E -H \
|
||||
/srv/pandora/bin/python \
|
||||
/srv/pandora/pandora/manage.py websocketd
|
||||
fi
|
||||
|
||||
# pan.do/ra management and update
|
||||
if [ "$action" = "manage.py" ]; then
|
||||
if [ "$action" = "ctl" ]; then
|
||||
shift
|
||||
exec /usr/bin/sudo -u $user -E -H \
|
||||
/srv/pandora/pandora/manage.py "$@"
|
||||
fi
|
||||
if [ "$action" = "update.py" ]; then
|
||||
shift
|
||||
exec /usr/bin/sudo -u $user -E -H \
|
||||
/srv/pandora/update.py "$@"
|
||||
exec /srv/pandora/ctl "$@"
|
||||
fi
|
||||
if [ "$action" = "bash" ]; then
|
||||
shift
|
||||
|
@ -102,9 +103,9 @@ fi
|
|||
# pan.do/ra setup hooks
|
||||
if [ "$action" = "docker-compose.yml" ]; then
|
||||
cat /srv/pandora_base/docker-compose.yml | \
|
||||
sed "s#build: \.#image: 0x2620/pandora:latest#g" | \
|
||||
sed "s#build: \.#image: code.0x2620.org/0x2620/pandora:latest#g" | \
|
||||
sed "s#\./overlay:#.:#g" | \
|
||||
sed "s#build: docker/nginx#image: 0x2620/pandora-nginx:latest#g"
|
||||
sed "s#build: docker/nginx#image: code.0x2620.org/0x2620/pandora-nginx:latest#g"
|
||||
exit
|
||||
fi
|
||||
if [ "$action" = ".env" ]; then
|
||||
|
@ -130,5 +131,5 @@ echo " docker run 0x2620/pandora setup | sh"
|
|||
echo
|
||||
echo adjust created files to match your needs and run:
|
||||
echo
|
||||
echo " docker-compose up"
|
||||
echo " docker compose up"
|
||||
echo
|
||||
|
|
|
@ -56,13 +56,9 @@ cp /srv/pandora/docker/entrypoint.sh /entrypoint.sh
|
|||
mv /srv/pandora/ /srv/pandora_base/
|
||||
mkdir /pandora
|
||||
ln -s /pandora /srv/pandora
|
||||
cat > /usr/local/bin/update.py << EOF
|
||||
#!/bin/sh
|
||||
exec /srv/pandora/update.py \$@
|
||||
EOF
|
||||
|
||||
cat > /usr/local/bin/manage.py << EOF
|
||||
cat > /usr/local/bin/pandoractl << EOF
|
||||
#!/bin/sh
|
||||
exec /srv/pandora/pandora/manage.py \$@
|
||||
exec /srv/pandora/ctl \$@
|
||||
EOF
|
||||
chmod +x /usr/local/bin/manage.py /usr/local/bin/update.py
|
||||
chmod +x /usr/local/bin/pandoractl
|
||||
|
|
12
docker/publish.sh
Normal file
12
docker/publish.sh
Normal file
|
@ -0,0 +1,12 @@
|
|||
#!/bin/bash
|
||||
# push new version of pan.do/ra to code.0x2620.org
|
||||
set -e
|
||||
|
||||
cd /tmp
|
||||
git clone https://code.0x2620.org/0x2620/pandora
|
||||
cd pandora
|
||||
./docker/build.sh
|
||||
|
||||
docker push code.0x2620.org/0x2620/pandora-base:latest
|
||||
docker push code.0x2620.org/0x2620/pandora-nginx:latest
|
||||
docker push code.0x2620.org/0x2620/pandora:latest
|
|
@ -1,18 +1,18 @@
|
|||
#!/bin/sh
|
||||
docker run 0x2620/pandora docker-compose.yml > docker-compose.yml
|
||||
docker run --rm code.0x2620.org/0x2620/pandora docker-compose.yml > docker-compose.yml
|
||||
if [ ! -e .env ]; then
|
||||
docker run 0x2620/pandora .env > .env
|
||||
docker run --rm code.0x2620.org/0x2620/pandora .env > .env
|
||||
echo .env >> .gitignore
|
||||
fi
|
||||
if [ ! -e config.jsonc ]; then
|
||||
docker run 0x2620/pandora config.jsonc > config.jsonc
|
||||
docker run --rm code.0x2620.org/0x2620/pandora config.jsonc > config.jsonc
|
||||
fi
|
||||
cat > README.md << EOF
|
||||
pan.do/ra docker instance
|
||||
|
||||
this folder was created with
|
||||
|
||||
docker run 0x2620/pandora setup | sh
|
||||
docker run --rm code.0x2620.org/0x2620/pandora setup | sh
|
||||
|
||||
To start pan.do/ra adjust the files in this folder:
|
||||
|
||||
|
@ -22,11 +22,14 @@ To start pan.do/ra adjust the files in this folder:
|
|||
|
||||
and to get started run this:
|
||||
|
||||
docker-compose up -d
|
||||
docker compose up -d
|
||||
|
||||
To update pan.do/ra run:
|
||||
|
||||
docker-compose run pandora update.py
|
||||
docker compose run --rm pandora ctl update
|
||||
|
||||
To run pan.do/ra manage shell:
|
||||
|
||||
docker compose run --rm pandora ctl manage shell
|
||||
EOF
|
||||
touch __init__.py
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
#!/bin/sh
|
||||
TIMEOUT=60
|
||||
TIMEOUT=180
|
||||
TARGET="$1"
|
||||
|
||||
for i in `seq $TIMEOUT` ; do
|
||||
|
|
|
@ -17,6 +17,7 @@ server {
|
|||
#server_name pandora.YOURDOMAIN.COM;
|
||||
|
||||
listen 80 default;
|
||||
listen [::]:80 default;
|
||||
|
||||
access_log /var/log/nginx/pandora.access.log;
|
||||
error_log /var/log/nginx/pandora.error.log;
|
||||
|
|
1
etc/sudoers.d/pandora
Normal file
1
etc/sudoers.d/pandora
Normal file
|
@ -0,0 +1 @@
|
|||
pandora ALL=(ALL:ALL) NOPASSWD:/usr/local/bin/pandoractl
|
|
@ -11,7 +11,7 @@ PIDFile=/run/pandora/cron.pid
|
|||
WorkingDirectory=/srv/pandora/pandora
|
||||
ExecStart=/srv/pandora/bin/celery \
|
||||
-A app beat \
|
||||
-s /run/pandora/celerybeat-schedule \
|
||||
--scheduler django_celery_beat.schedulers:DatabaseScheduler \
|
||||
--pidfile /run/pandora/cron.pid \
|
||||
-l INFO
|
||||
ExecReload=/bin/kill -HUP $MAINPID
|
||||
|
|
|
@ -7,14 +7,16 @@ Type=simple
|
|||
Restart=always
|
||||
User=pandora
|
||||
Group=pandora
|
||||
EnvironmentFile=/srv/pandora/pandora/encoding.conf
|
||||
PIDFile=/run/pandora/encoding.pid
|
||||
WorkingDirectory=/srv/pandora/pandora
|
||||
ExecStart=/srv/pandora/bin/celery \
|
||||
-A app worker \
|
||||
-Q encoding -n pandora-encoding \
|
||||
--pidfile /run/pandora/encoding.pid \
|
||||
--maxtasksperchild 500 \
|
||||
-l INFO
|
||||
-c $CONCURRENCY \
|
||||
--max-tasks-per-child $MAX_TASKS_PER_CHILD \
|
||||
-l $LOGLEVEL
|
||||
ExecReload=/bin/kill -TERM $MAINPID
|
||||
|
||||
[Install]
|
||||
|
|
|
@ -7,14 +7,16 @@ Type=simple
|
|||
Restart=always
|
||||
User=pandora
|
||||
Group=pandora
|
||||
EnvironmentFile=/srv/pandora/pandora/tasks.conf
|
||||
PIDFile=/run/pandora/tasks.pid
|
||||
WorkingDirectory=/srv/pandora/pandora
|
||||
ExecStart=/srv/pandora/bin/celery \
|
||||
-A app worker \
|
||||
-Q default,celery -n pandora-default \
|
||||
--pidfile /run/pandora/tasks.pid \
|
||||
--maxtasksperchild 1000 \
|
||||
-l INFO
|
||||
-c $CONCURRENCY \
|
||||
--max-tasks-per-child $MAX_TASKS_PER_CHILD \
|
||||
-l $LOGLEVEL
|
||||
ExecReload=/bin/kill -TERM $MAINPID
|
||||
|
||||
[Install]
|
||||
|
|
6
pandora/annotation/apps.py
Normal file
6
pandora/annotation/apps.py
Normal file
|
@ -0,0 +1,6 @@
|
|||
from django.apps import AppConfig
|
||||
|
||||
|
||||
class AnnotationConfig(AppConfig):
|
||||
default_auto_field = "django.db.models.BigAutoField"
|
||||
name = 'annotation'
|
|
@ -27,6 +27,7 @@ class Command(BaseCommand):
|
|||
parser.add_argument('username', help='username')
|
||||
parser.add_argument('item', help='item')
|
||||
parser.add_argument('layer', help='layer')
|
||||
parser.add_argument('language', help='language', default="")
|
||||
parser.add_argument('filename', help='filename.srt')
|
||||
|
||||
def handle(self, *args, **options):
|
||||
|
@ -34,6 +35,7 @@ class Command(BaseCommand):
|
|||
public_id = options['item']
|
||||
layer_id = options['layer']
|
||||
filename = options['filename']
|
||||
language = options.get("language")
|
||||
|
||||
user = User.objects.get(username=username)
|
||||
item = Item.objects.get(public_id=public_id)
|
||||
|
@ -47,6 +49,9 @@ class Command(BaseCommand):
|
|||
for i in range(len(annotations)-1):
|
||||
if annotations[i]['out'] == annotations[i+1]['in']:
|
||||
annotations[i]['out'] = annotations[i]['out'] - 0.001
|
||||
if language:
|
||||
for annotation in annotations:
|
||||
annotation["value"] = '<span lang="%s">%s</span>' % (language, annotation["value"])
|
||||
tasks.add_annotations.delay({
|
||||
'item': item.public_id,
|
||||
'layer': layer_id,
|
||||
|
|
|
@ -0,0 +1,18 @@
|
|||
# Generated by Django 4.2.3 on 2023-07-27 21:24
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('annotation', '0003_auto_20160219_1537'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='annotation',
|
||||
name='id',
|
||||
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
|
||||
),
|
||||
]
|
|
@ -163,22 +163,19 @@ class Annotation(models.Model):
|
|||
self.sortvalue = None
|
||||
self.languages = None
|
||||
|
||||
with transaction.atomic():
|
||||
if not self.clip or self.start != self.clip.start or self.end != self.clip.end:
|
||||
self.clip, created = Clip.get_or_create(self.item, self.start, self.end)
|
||||
|
||||
with transaction.atomic():
|
||||
if set_public_id:
|
||||
self.set_public_id()
|
||||
|
||||
super(Annotation, self).save(*args, **kwargs)
|
||||
|
||||
if self.clip:
|
||||
Clip.objects.filter(**{
|
||||
'id': self.clip.id,
|
||||
self.layer: False
|
||||
}).update(**{self.layer: True})
|
||||
# update clip.findvalue
|
||||
self.clip.save()
|
||||
self.clip.update_findvalue()
|
||||
setattr(self.clip, self.layer, True)
|
||||
self.clip.save(update_fields=[self.layer, 'sortvalue', 'findvalue'])
|
||||
|
||||
# update matches in bulk if called from load_subtitles
|
||||
if not delay_matches:
|
||||
|
@ -267,6 +264,9 @@ class Annotation(models.Model):
|
|||
from translation.models import Translation
|
||||
layer = self.get_layer()
|
||||
if layer.get('translate'):
|
||||
for lang in settings.CONFIG['languages']:
|
||||
if lang == settings.CONFIG['language']:
|
||||
continue
|
||||
Translation.objects.get_or_create(lang=lang, key=self.value, defaults={'type': Translation.CONTENT})
|
||||
|
||||
def delete(self, *args, **kwargs):
|
||||
|
|
|
@ -5,12 +5,12 @@ from django.contrib.auth import get_user_model
|
|||
from django.db import transaction
|
||||
|
||||
import ox
|
||||
from celery.task import task
|
||||
from app.celery import app
|
||||
|
||||
from .models import Annotation
|
||||
|
||||
|
||||
@task(ignore_results=False, queue='default')
|
||||
@app.task(ignore_results=False, queue='default')
|
||||
def add_annotations(data):
|
||||
from item.models import Item
|
||||
from entity.models import Entity
|
||||
|
@ -51,7 +51,7 @@ def add_annotations(data):
|
|||
annotation.item.update_facets()
|
||||
return True
|
||||
|
||||
@task(ignore_results=True, queue='default')
|
||||
@app.task(ignore_results=True, queue='default')
|
||||
def update_item(id, force=False):
|
||||
from item.models import Item
|
||||
from clip.models import Clip
|
||||
|
@ -72,7 +72,7 @@ def update_item(id, force=False):
|
|||
a.item.save()
|
||||
|
||||
|
||||
@task(ignore_results=True, queue='default')
|
||||
@app.task(ignore_results=True, queue='default')
|
||||
def update_annotations(layers, value):
|
||||
items = {}
|
||||
|
||||
|
|
|
@ -183,7 +183,7 @@ def addAnnotation(request, data):
|
|||
|
||||
layer_id = data['layer']
|
||||
layer = get_by_id(settings.CONFIG['layers'], layer_id)
|
||||
if layer['canAddAnnotations'].get(request.user.profile.get_level()):
|
||||
if layer['canAddAnnotations'].get(request.user.profile.get_level()) or item.editable(request.user):
|
||||
if layer['type'] == 'entity':
|
||||
try:
|
||||
value = Entity.get_by_name(ox.decode_html(data['value']), layer['entity']).get_id()
|
||||
|
@ -241,8 +241,7 @@ def addAnnotations(request, data):
|
|||
|
||||
layer_id = data['layer']
|
||||
layer = get_by_id(settings.CONFIG['layers'], layer_id)
|
||||
if item.editable(request.user) \
|
||||
and layer['canAddAnnotations'].get(request.user.profile.get_level()):
|
||||
if item.editable(request.user):
|
||||
response = json_response()
|
||||
data['user'] = request.user.username
|
||||
t = add_annotations.delay(data)
|
||||
|
|
7
pandora/app/apps.py
Normal file
7
pandora/app/apps.py
Normal file
|
@ -0,0 +1,7 @@
|
|||
from django.apps import AppConfig
|
||||
|
||||
|
||||
class AppConfig(AppConfig):
|
||||
default_auto_field = "django.db.models.BigAutoField"
|
||||
name = 'app'
|
||||
|
|
@ -6,16 +6,8 @@ root_dir = os.path.normpath(os.path.abspath(os.path.dirname(__file__)))
|
|||
root_dir = os.path.dirname(root_dir)
|
||||
os.chdir(root_dir)
|
||||
|
||||
# set the default Django settings module for the 'celery' program.
|
||||
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'settings')
|
||||
|
||||
app = Celery('pandora')
|
||||
|
||||
# Using a string here means the worker doesn't have to serialize
|
||||
# the configuration object to child processes.
|
||||
# - namespace='CELERY' means all celery-related configuration keys
|
||||
# should have a `CELERY_` prefix.
|
||||
app = Celery('pandora', broker_connection_retry_on_startup=True)
|
||||
app.config_from_object('django.conf:settings', namespace='CELERY')
|
||||
|
||||
# Load task modules from all registered Django app configs.
|
||||
app.autodiscover_tasks()
|
||||
|
|
|
@ -24,9 +24,6 @@ User = get_user_model()
|
|||
|
||||
_win = (sys.platform == "win32")
|
||||
|
||||
RUN_RELOADER = True
|
||||
NOTIFIER = None
|
||||
|
||||
def get_version():
|
||||
git_dir = join(dirname(dirname(dirname(__file__))), '.git')
|
||||
if exists(git_dir):
|
||||
|
@ -136,6 +133,12 @@ def load_config(init=False):
|
|||
added = []
|
||||
for key in sorted(d):
|
||||
if key not in c:
|
||||
if key not in (
|
||||
'hidden',
|
||||
'find',
|
||||
'findDocuments',
|
||||
'videoPoints',
|
||||
):
|
||||
added.append("\"%s\": %s," % (key, json.dumps(d[key])))
|
||||
c[key] = d[key]
|
||||
if added:
|
||||
|
@ -257,46 +260,6 @@ check the README for further details.
|
|||
except:
|
||||
pass
|
||||
|
||||
|
||||
|
||||
def reloader_thread():
|
||||
global NOTIFIER
|
||||
settings.RELOADER_RUNNING=True
|
||||
_config_mtime = 0
|
||||
try:
|
||||
import pyinotify
|
||||
INOTIFY = True
|
||||
except:
|
||||
INOTIFY = False
|
||||
if INOTIFY:
|
||||
def add_watch():
|
||||
name = os.path.realpath(settings.SITE_CONFIG)
|
||||
wm.add_watch(name, pyinotify.IN_CLOSE_WRITE, reload_config)
|
||||
|
||||
def reload_config(event):
|
||||
load_config()
|
||||
add_watch()
|
||||
|
||||
wm = pyinotify.WatchManager()
|
||||
add_watch()
|
||||
notifier = pyinotify.Notifier(wm)
|
||||
NOTIFIER = notifier
|
||||
notifier.loop()
|
||||
else:
|
||||
while RUN_RELOADER:
|
||||
try:
|
||||
stat = os.stat(settings.SITE_CONFIG)
|
||||
mtime = stat.st_mtime
|
||||
if _win:
|
||||
mtime -= stat.st_ctime
|
||||
if mtime > _config_mtime:
|
||||
load_config()
|
||||
_config_mtime = mtime
|
||||
time.sleep(10)
|
||||
except:
|
||||
#sys.stderr.write("reloading config failed\n")
|
||||
pass
|
||||
|
||||
def update_static():
|
||||
oxjs_build = os.path.join(settings.STATIC_ROOT, 'oxjs/tools/build/build.py')
|
||||
if os.path.exists(oxjs_build):
|
||||
|
@ -364,7 +327,11 @@ def update_static():
|
|||
#locale
|
||||
for f in sorted(glob(os.path.join(settings.STATIC_ROOT, 'json/locale.pandora.*.json'))):
|
||||
with open(f) as fd:
|
||||
try:
|
||||
locale = json.load(fd)
|
||||
except:
|
||||
print("failed to parse %s" % f)
|
||||
raise
|
||||
site_locale = f.replace('locale.pandora', 'locale.' + settings.CONFIG['site']['id'])
|
||||
locale_file = f.replace('locale.pandora', 'locale')
|
||||
print('write', locale_file)
|
||||
|
@ -407,17 +374,4 @@ def update_geoip(force=False):
|
|||
print('failed to download GeoLite2-City.mmdb')
|
||||
|
||||
def init():
|
||||
if not settings.RELOADER_RUNNING:
|
||||
load_config(True)
|
||||
if settings.RELOAD_CONFIG:
|
||||
thread.start_new_thread(reloader_thread, ())
|
||||
|
||||
def shutdown():
|
||||
if settings.RELOADER_RUNNING:
|
||||
RUN_RELOADER = False
|
||||
settings.RELOADER_RUNNING = False
|
||||
if NOTIFIER:
|
||||
NOTIFIER.stop()
|
||||
|
||||
|
||||
|
||||
|
|
|
@ -11,6 +11,8 @@ def run(cmd):
|
|||
stdout, stderr = p.communicate()
|
||||
|
||||
if p.returncode != 0:
|
||||
print('failed to run:', cmd)
|
||||
print(stdout)
|
||||
print(stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
|
|
@ -0,0 +1,23 @@
|
|||
# Generated by Django 4.2.3 on 2023-07-27 21:24
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('app', '0001_initial'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='page',
|
||||
name='id',
|
||||
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='settings',
|
||||
name='id',
|
||||
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
|
||||
),
|
||||
]
|
34
pandora/app/oidc.py
Normal file
34
pandora/app/oidc.py
Normal file
|
@ -0,0 +1,34 @@
|
|||
import unicodedata
|
||||
|
||||
from django.contrib.auth import get_user_model
|
||||
|
||||
import mozilla_django_oidc.auth
|
||||
|
||||
from user.utils import prepare_user
|
||||
|
||||
User = get_user_model()
|
||||
|
||||
|
||||
class OIDCAuthenticationBackend(mozilla_django_oidc.auth.OIDCAuthenticationBackend):
|
||||
def create_user(self, claims):
|
||||
user = super(OIDCAuthenticationBackend, self).create_user(claims)
|
||||
username = claims.get("preferred_username")
|
||||
n = 1
|
||||
if username and username != user.username:
|
||||
uname = username
|
||||
while User.objects.filter(username=uname).exclude(id=user.id).exists():
|
||||
n += 1
|
||||
uname = '%s (%s)' % (username, n)
|
||||
user.username = uname
|
||||
user.save()
|
||||
prepare_user(user)
|
||||
return user
|
||||
|
||||
def update_user(self, user, claims):
|
||||
print("update user", user, claims)
|
||||
#user.save()
|
||||
return user
|
||||
|
||||
|
||||
def generate_username(email):
|
||||
return unicodedata.normalize('NFKC', email)[:150]
|
|
@ -2,13 +2,16 @@
|
|||
|
||||
import datetime
|
||||
|
||||
from celery.task import periodic_task
|
||||
from app.celery import app
|
||||
from celery.schedules import crontab
|
||||
|
||||
|
||||
@periodic_task(run_every=crontab(hour=6, minute=0), queue='encoding')
|
||||
@app.task(queue='encoding')
|
||||
def cron(**kwargs):
|
||||
from django.db import transaction
|
||||
from django.contrib.sessions.models import Session
|
||||
Session.objects.filter(expire_date__lt=datetime.datetime.now()).delete()
|
||||
transaction.commit()
|
||||
|
||||
@app.on_after_finalize.connect
|
||||
def setup_periodic_tasks(sender, **kwargs):
|
||||
sender.add_periodic_task(crontab(hour=6, minute=0), cron.s())
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
import copy
|
||||
from datetime import datetime
|
||||
import base64
|
||||
import copy
|
||||
|
||||
from django.shortcuts import render, redirect
|
||||
from django.conf import settings
|
||||
|
@ -53,9 +53,10 @@ def embed(request, id):
|
|||
})
|
||||
|
||||
def redirect_url(request, url):
|
||||
if request.META['QUERY_STRING']:
|
||||
url += "?" + request.META['QUERY_STRING']
|
||||
|
||||
try:
|
||||
url = base64.decodebytes(url.encode()).decode()
|
||||
except:
|
||||
pass
|
||||
if settings.CONFIG['site'].get('sendReferrer', False):
|
||||
return redirect(url)
|
||||
else:
|
||||
|
@ -162,7 +163,7 @@ def init(request, data):
|
|||
del config['keys']
|
||||
|
||||
if 'HTTP_ACCEPT_LANGUAGE' in request.META:
|
||||
response['data']['locale'] = request.META['HTTP_ACCEPT_LANGUAGE'].split(';')[0].split('-')[0]
|
||||
response['data']['locale'] = request.META['HTTP_ACCEPT_LANGUAGE'].split(';')[0].split('-')[0].split(',')[0]
|
||||
|
||||
if request.META.get('HTTP_X_PREFIX') == 'NO':
|
||||
config['site']['videoprefix'] = ''
|
||||
|
@ -183,6 +184,7 @@ def init(request, data):
|
|||
except:
|
||||
pass
|
||||
|
||||
config['site']['oidc'] = bool(getattr(settings, 'OIDC_RP_CLIENT_ID', False))
|
||||
response['data']['site'] = config
|
||||
response['data']['user'] = init_user(request.user, request)
|
||||
request.session['last_init'] = str(datetime.now())
|
||||
|
@ -245,7 +247,7 @@ def getEmbedDefaults(request, data):
|
|||
i = qs[0].cache
|
||||
response['data']['item'] = i['id']
|
||||
response['data']['itemDuration'] = i['duration']
|
||||
response['data']['itemRatio'] = i['videoRatio']
|
||||
response['data']['itemRatio'] = i.get('videoRatio', settings.CONFIG['video']['previewRatio'])
|
||||
qs = List.objects.exclude(status='private').order_by('name')
|
||||
if qs.exists():
|
||||
i = qs[0].json()
|
||||
|
|
7
pandora/archive/apps.py
Normal file
7
pandora/archive/apps.py
Normal file
|
@ -0,0 +1,7 @@
|
|||
from django.apps import AppConfig
|
||||
|
||||
|
||||
class ArchiveConfig(AppConfig):
|
||||
default_auto_field = "django.db.models.BigAutoField"
|
||||
name = 'archive'
|
||||
|
|
@ -1,10 +1,11 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import shutil
|
||||
import tempfile
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
import ox
|
||||
from django.conf import settings
|
||||
|
@ -14,6 +15,9 @@ from item.tasks import load_subtitles
|
|||
|
||||
from . import models
|
||||
|
||||
logger = logging.getLogger('pandora.' + __name__)
|
||||
|
||||
|
||||
info_keys = [
|
||||
'title',
|
||||
'description',
|
||||
|
@ -36,8 +40,14 @@ info_key_map = {
|
|||
'display_id': 'id',
|
||||
}
|
||||
|
||||
def get_info(url):
|
||||
cmd = ['youtube-dl', '-j', '--all-subs', url]
|
||||
YT_DLP = ['yt-dlp']
|
||||
if settings.YT_DLP_EXTRA:
|
||||
YT_DLP += settings.YT_DLP_EXTRA
|
||||
|
||||
def get_info(url, referer=None):
|
||||
cmd = YT_DLP + ['-j', '--all-subs', url]
|
||||
if referer:
|
||||
cmd += ['--referer', referer]
|
||||
p = subprocess.Popen(cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE, close_fds=True)
|
||||
|
@ -57,6 +67,8 @@ def get_info(url):
|
|||
info[-1]['tags'] = []
|
||||
if 'upload_date' in i and i['upload_date']:
|
||||
info[-1]['date'] = '-'.join([i['upload_date'][:4], i['upload_date'][4:6], i['upload_date'][6:]])
|
||||
if 'referer' not in info[-1]:
|
||||
info[-1]['referer'] = url
|
||||
return info
|
||||
|
||||
def add_subtitles(item, media, tmp):
|
||||
|
@ -84,9 +96,18 @@ def add_subtitles(item, media, tmp):
|
|||
sub.selected = True
|
||||
sub.save()
|
||||
|
||||
def download(item_id, url):
|
||||
def load_formats(url):
|
||||
cmd = YT_DLP + ['-q', url, '-j', '-F']
|
||||
p = subprocess.Popen(cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE, close_fds=True)
|
||||
stdout, stderr = p.communicate()
|
||||
formats = stdout.decode().strip().split('\n')[-1]
|
||||
return json.loads(formats)
|
||||
|
||||
def download(item_id, url, referer=None):
|
||||
item = Item.objects.get(public_id=item_id)
|
||||
info = get_info(url)
|
||||
info = get_info(url, referer)
|
||||
if not len(info):
|
||||
return '%s contains no videos' % url
|
||||
media = info[0]
|
||||
|
@ -95,13 +116,19 @@ def download(item_id, url):
|
|||
if isinstance(tmp, bytes):
|
||||
tmp = tmp.decode('utf-8')
|
||||
os.chdir(tmp)
|
||||
cmd = ['youtube-dl', '-q', media['url']]
|
||||
if settings.CONFIG['video'].get('reuseUload', False):
|
||||
cmd = YT_DLP + ['-q', media['url']]
|
||||
if referer:
|
||||
cmd += ['--referer', referer]
|
||||
elif 'referer' in media:
|
||||
cmd += ['--referer', media['referer']]
|
||||
cmd += ['-o', '%(title)80s.%(ext)s']
|
||||
|
||||
if settings.CONFIG['video'].get('reuseUpload', False):
|
||||
max_resolution = max(settings.CONFIG['video']['resolutions'])
|
||||
format = settings.CONFIG['video']['formats'][0]
|
||||
if format == 'mp4':
|
||||
cmd += [
|
||||
'-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/bestvideo+bestaudio',
|
||||
'-f', 'bestvideo[height<=%s][ext=mp4]+bestaudio[ext=m4a]' % max_resolution,
|
||||
'--merge-output-format', 'mp4'
|
||||
]
|
||||
elif format == 'webm':
|
||||
|
@ -111,6 +138,50 @@ def download(item_id, url):
|
|||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE, close_fds=True)
|
||||
stdout, stderr = p.communicate()
|
||||
if stderr and b'Requested format is not available.' in stderr:
|
||||
formats = load_formats(url)
|
||||
has_audio = bool([fmt for fmt in formats['formats'] if fmt['resolution'] == 'audio only'])
|
||||
has_video = bool([fmt for fmt in formats['formats'] if 'x' in fmt['resolution']])
|
||||
|
||||
cmd = [
|
||||
'yt-dlp', '-q', url,
|
||||
'-o', '%(title)80s.%(ext)s'
|
||||
]
|
||||
if referer:
|
||||
cmd += ['--referer', referer]
|
||||
elif 'referer' in media:
|
||||
cmd += ['--referer', media['referer']]
|
||||
if has_video and not has_audio:
|
||||
cmd += [
|
||||
'-f', 'bestvideo[height<=%s][ext=mp4]' % max_resolution,
|
||||
]
|
||||
elif not has_video and has_audio:
|
||||
cmd += [
|
||||
'bestaudio[ext=m4a]'
|
||||
]
|
||||
else:
|
||||
cmd = []
|
||||
if cmd:
|
||||
p = subprocess.Popen(cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE, close_fds=True)
|
||||
stdout, stderr = p.communicate()
|
||||
if stderr and b'Requested format is not available.' in stderr:
|
||||
cmd = [
|
||||
'yt-dlp', '-q', url,
|
||||
'-o', '%(title)80s.%(ext)s'
|
||||
]
|
||||
if referer:
|
||||
cmd += ['--referer', referer]
|
||||
elif 'referer' in media:
|
||||
cmd += ['--referer', media['referer']]
|
||||
if cmd:
|
||||
p = subprocess.Popen(cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE, close_fds=True)
|
||||
stdout, stderr = p.communicate()
|
||||
if stdout or stderr:
|
||||
logger.error("import failed:\n%s\n%s\n%s", " ".join(cmd), stdout.decode(), stderr.decode())
|
||||
parts = list(os.listdir(tmp))
|
||||
if parts:
|
||||
part = 1
|
||||
|
@ -138,6 +209,7 @@ def download(item_id, url):
|
|||
f.extract_stream()
|
||||
status = True
|
||||
else:
|
||||
logger.error("failed to import %s file already exists %s", url, oshash)
|
||||
status = 'file exists'
|
||||
if len(parts) == 1:
|
||||
add_subtitles(f.item, media, tmp)
|
||||
|
|
|
@ -1,26 +1,30 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
import os
|
||||
from distutils.spawn import find_executable
|
||||
from glob import glob
|
||||
from os.path import exists
|
||||
|
||||
import fractions
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
import math
|
||||
import shutil
|
||||
from distutils.spawn import find_executable
|
||||
from glob import glob
|
||||
|
||||
import numpy as np
|
||||
import ox
|
||||
import ox.image
|
||||
from ox.utils import json
|
||||
from django.conf import settings
|
||||
from PIL import Image
|
||||
from PIL import Image, ImageOps
|
||||
|
||||
from .chop import Chop, make_keyframe_index
|
||||
|
||||
|
||||
logger = logging.getLogger('pandora.' + __name__)
|
||||
|
||||
img_extension = 'jpg'
|
||||
|
||||
MAX_DISTANCE = math.sqrt(3 * pow(255, 2))
|
||||
|
@ -57,14 +61,15 @@ def supported_formats():
|
|||
stdout = stdout.decode('utf-8')
|
||||
stderr = stderr.decode('utf-8')
|
||||
version = stderr.split('\n')[0].split(' ')[2]
|
||||
mp4 = 'libx264' in stdout and bool(re.compile('DEA.L. aac').findall(stdout))
|
||||
return {
|
||||
'version': version.split('.'),
|
||||
'ogg': 'libtheora' in stdout and 'libvorbis' in stdout,
|
||||
'webm': 'libvpx' in stdout and 'libvorbis' in stdout,
|
||||
'vp8': 'libvpx' in stdout and 'libvorbis' in stdout,
|
||||
'vp9': 'libvpx-vp9' in stdout and 'libopus' in stdout,
|
||||
'mp4': 'libx264' in stdout and 'DEA.L. aac' in stdout,
|
||||
'h264': 'libx264' in stdout and 'DEA.L. aac' in stdout,
|
||||
'mp4': mp4,
|
||||
'h264': mp4,
|
||||
}
|
||||
|
||||
def stream(video, target, profile, info, audio_track=0, flags={}):
|
||||
|
@ -145,10 +150,17 @@ def stream(video, target, profile, info, audio_track=0, flags={}):
|
|||
audioquality = -1
|
||||
audiobitrate = '22k'
|
||||
audiochannels = 1
|
||||
elif profile == '0p':
|
||||
info['video'] = []
|
||||
audiorate = 48000
|
||||
audioquality = 6
|
||||
audiobitrate = None
|
||||
audiochannels = None
|
||||
audio_codec = 'libopus'
|
||||
else:
|
||||
height = 96
|
||||
|
||||
if settings.FFMPEG_SUPPORTS_VP9:
|
||||
if settings.USE_VP9 and settings.FFMPEG_SUPPORTS_VP9:
|
||||
audio_codec = 'libopus'
|
||||
video_codec = 'libvpx-vp9'
|
||||
|
||||
|
@ -211,7 +223,7 @@ def stream(video, target, profile, info, audio_track=0, flags={}):
|
|||
bitrate = height*width*fps*bpp/1000
|
||||
|
||||
video_settings = trim + [
|
||||
'-vb', '%dk' % bitrate,
|
||||
'-b:v', '%dk' % bitrate,
|
||||
'-aspect', aspect,
|
||||
# '-vf', 'yadif',
|
||||
'-max_muxing_queue_size', '512',
|
||||
|
@ -239,6 +251,8 @@ def stream(video, target, profile, info, audio_track=0, flags={}):
|
|||
'-level', '4.0',
|
||||
'-pix_fmt', 'yuv420p',
|
||||
]
|
||||
if info['video'][0].get("force_framerate"):
|
||||
video_settings += ['-r:v', str(fps)]
|
||||
video_settings += ['-map', '0:%s,0:0' % info['video'][0]['id']]
|
||||
audio_only = False
|
||||
else:
|
||||
|
@ -278,7 +292,7 @@ def stream(video, target, profile, info, audio_track=0, flags={}):
|
|||
ac = min(ac, audiochannels)
|
||||
audio_settings += ['-ac', str(ac)]
|
||||
if audiobitrate:
|
||||
audio_settings += ['-ab', audiobitrate]
|
||||
audio_settings += ['-b:a', audiobitrate]
|
||||
if format == 'mp4':
|
||||
audio_settings += ['-c:a', 'aac', '-strict', '-2']
|
||||
elif audio_codec == 'libopus':
|
||||
|
@ -311,14 +325,15 @@ def stream(video, target, profile, info, audio_track=0, flags={}):
|
|||
pass1_post = post[:]
|
||||
pass1_post[-1] = '/dev/null'
|
||||
if format == 'webm':
|
||||
if video_codec != 'libvpx-vp9':
|
||||
pass1_post = ['-speed', '4'] + pass1_post
|
||||
post = ['-speed', '1'] + post
|
||||
cmds.append(base + ['-an', '-pass', '1', '-passlogfile', '%s.log' % target]
|
||||
+ video_settings + pass1_post)
|
||||
cmds.append(base + ['-pass', '1', '-passlogfile', '%s.log' % target]
|
||||
+ video_settings + ['-an'] + pass1_post)
|
||||
cmds.append(base + ['-pass', '2', '-passlogfile', '%s.log' % target]
|
||||
+ audio_settings + video_settings + post)
|
||||
+ video_settings + audio_settings + post)
|
||||
else:
|
||||
cmds.append(base + audio_settings + video_settings + post)
|
||||
cmds.append(base + video_settings + audio_settings + post)
|
||||
|
||||
if settings.FFMPEG_DEBUG:
|
||||
print('\n'.join([' '.join(cmd) for cmd in cmds]))
|
||||
|
@ -426,10 +441,15 @@ def frame_direct(video, target, position):
|
|||
r = run_command(cmd)
|
||||
return r == 0
|
||||
|
||||
def open_image_rgb(image_source):
|
||||
source = Image.open(image_source)
|
||||
source = ImageOps.exif_transpose(source)
|
||||
source = source.convert('RGB')
|
||||
return source
|
||||
|
||||
def resize_image(image_source, image_output, width=None, size=None):
|
||||
if exists(image_source):
|
||||
source = Image.open(image_source).convert('RGB')
|
||||
source = open_image_rgb(image_source)
|
||||
source_width = source.size[0]
|
||||
source_height = source.size[1]
|
||||
if size:
|
||||
|
@ -450,7 +470,7 @@ def resize_image(image_source, image_output, width=None, size=None):
|
|||
height = max(height, 1)
|
||||
|
||||
if width < source_width:
|
||||
resize_method = Image.ANTIALIAS
|
||||
resize_method = Image.LANCZOS
|
||||
else:
|
||||
resize_method = Image.BICUBIC
|
||||
output = source.resize((width, height), resize_method)
|
||||
|
@ -464,7 +484,7 @@ def timeline(video, prefix, modes=None, size=None):
|
|||
size = [64, 16]
|
||||
if isinstance(video, str):
|
||||
video = [video]
|
||||
cmd = ['../bin/oxtimelines',
|
||||
cmd = [os.path.normpath(os.path.join(settings.BASE_DIR, '../bin/oxtimelines')),
|
||||
'-s', ','.join(map(str, reversed(sorted(size)))),
|
||||
'-m', ','.join(modes),
|
||||
'-o', prefix,
|
||||
|
@ -596,7 +616,7 @@ def timeline_strip(item, cuts, info, prefix):
|
|||
print(frame, 'cut', c, 'frame', s, frame, 'width', widths[s], box)
|
||||
# FIXME: why does this have to be frame+1?
|
||||
frame_image = Image.open(item.frame((frame+1)/fps))
|
||||
frame_image = frame_image.crop(box).resize((widths[s], timeline_height), Image.ANTIALIAS)
|
||||
frame_image = frame_image.crop(box).resize((widths[s], timeline_height), Image.LANCZOS)
|
||||
for x_ in range(widths[s]):
|
||||
line_image.append(frame_image.crop((x_, 0, x_ + 1, timeline_height)))
|
||||
frame += widths[s]
|
||||
|
@ -724,18 +744,23 @@ def remux_stream(src, dst):
|
|||
cmd = [
|
||||
settings.FFMPEG,
|
||||
'-nostats', '-loglevel', 'error',
|
||||
'-map_metadata', '-1', '-sn',
|
||||
'-i', src,
|
||||
'-map_metadata', '-1', '-sn',
|
||||
] + video + [
|
||||
] + audio + [
|
||||
'-movflags', '+faststart',
|
||||
dst
|
||||
]
|
||||
print(cmd)
|
||||
p = subprocess.Popen(cmd, stdin=subprocess.PIPE,
|
||||
stdout=open('/dev/null', 'w'),
|
||||
stderr=open('/dev/null', 'w'),
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
close_fds=True)
|
||||
p.wait()
|
||||
stdout, stderr = p.communicate()
|
||||
if stderr:
|
||||
logger.error("failed to remux %s %s", cmd, stderr)
|
||||
return False, stderr
|
||||
else:
|
||||
return True, None
|
||||
|
||||
|
||||
|
|
|
@ -0,0 +1,100 @@
|
|||
# Generated by Django 4.2.3 on 2023-07-27 21:24
|
||||
|
||||
import django.core.serializers.json
|
||||
from django.db import migrations, models
|
||||
import oxdjango.fields
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('archive', '0005_auto_20180804_1554'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='file',
|
||||
name='extension',
|
||||
field=models.CharField(default='', max_length=255, null=True),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='file',
|
||||
name='id',
|
||||
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='file',
|
||||
name='info',
|
||||
field=oxdjango.fields.JSONField(default=dict, editable=False, encoder=django.core.serializers.json.DjangoJSONEncoder),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='file',
|
||||
name='language',
|
||||
field=models.CharField(default='', max_length=255, null=True),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='file',
|
||||
name='part',
|
||||
field=models.CharField(default='', max_length=255, null=True),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='file',
|
||||
name='part_title',
|
||||
field=models.CharField(default='', max_length=255, null=True),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='file',
|
||||
name='path',
|
||||
field=models.CharField(default='', max_length=2048),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='file',
|
||||
name='sort_path',
|
||||
field=models.CharField(default='', max_length=2048),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='file',
|
||||
name='type',
|
||||
field=models.CharField(default='', max_length=255),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='file',
|
||||
name='version',
|
||||
field=models.CharField(default='', max_length=255, null=True),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='frame',
|
||||
name='id',
|
||||
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='instance',
|
||||
name='id',
|
||||
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='stream',
|
||||
name='error',
|
||||
field=models.TextField(blank=True, default=''),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='stream',
|
||||
name='format',
|
||||
field=models.CharField(default='webm', max_length=255),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='stream',
|
||||
name='id',
|
||||
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='stream',
|
||||
name='info',
|
||||
field=oxdjango.fields.JSONField(default=dict, editable=False, encoder=django.core.serializers.json.DjangoJSONEncoder),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='volume',
|
||||
name='id',
|
||||
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
|
||||
),
|
||||
]
|
|
@ -0,0 +1,17 @@
|
|||
# Generated by Django 4.2.3 on 2023-08-18 12:54
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('archive', '0006_alter_file_extension_alter_file_id_alter_file_info_and_more'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddIndex(
|
||||
model_name='stream',
|
||||
index=models.Index(fields=['file', 'source', 'available'], name='archive_str_file_id_69a542_idx'),
|
||||
),
|
||||
]
|
|
@ -151,8 +151,10 @@ class File(models.Model):
|
|||
self.sampleate = 0
|
||||
self.channels = 0
|
||||
|
||||
if self.framerate:
|
||||
if self.framerate and self.duration > 0:
|
||||
self.pixels = int(self.width * self.height * float(utils.parse_decimal(self.framerate)) * self.duration)
|
||||
else:
|
||||
self.pixels = 0
|
||||
|
||||
def get_path_info(self):
|
||||
data = {}
|
||||
|
@ -181,6 +183,13 @@ class File(models.Model):
|
|||
for type in ox.movie.EXTENSIONS:
|
||||
if data['extension'] in ox.movie.EXTENSIONS[type]:
|
||||
data['type'] = type
|
||||
if data['extension'] == 'ogg' and self.info.get('video'):
|
||||
data['type'] = 'video'
|
||||
if data['type'] == 'unknown':
|
||||
if self.info.get('video'):
|
||||
data['type'] = 'video'
|
||||
elif self.info.get('audio'):
|
||||
data['type'] = 'audio'
|
||||
if 'part' in data and isinstance(data['part'], int):
|
||||
data['part'] = str(data['part'])
|
||||
return data
|
||||
|
@ -268,7 +277,7 @@ class File(models.Model):
|
|||
|
||||
if self.type not in ('audio', 'video'):
|
||||
self.duration = None
|
||||
else:
|
||||
elif self.id:
|
||||
duration = sum([s.info.get('duration', 0)
|
||||
for s in self.streams.filter(source=None)])
|
||||
if duration:
|
||||
|
@ -276,7 +285,7 @@ class File(models.Model):
|
|||
|
||||
if self.is_subtitle:
|
||||
self.available = self.data and True or False
|
||||
else:
|
||||
elif self.id:
|
||||
self.available = not self.uploading and \
|
||||
self.streams.filter(source=None, available=True).count()
|
||||
super(File, self).save(*args, **kwargs)
|
||||
|
@ -336,7 +345,9 @@ class File(models.Model):
|
|||
|
||||
def done_cb():
|
||||
if done:
|
||||
self.info.update(ox.avinfo(self.data.path))
|
||||
info = ox.avinfo(self.data.path)
|
||||
del info['path']
|
||||
self.info.update(info)
|
||||
self.parse_info()
|
||||
# reject invalid uploads
|
||||
if self.info.get('oshash') != self.oshash:
|
||||
|
@ -363,8 +374,8 @@ class File(models.Model):
|
|||
self.info.update(stream.info)
|
||||
self.parse_info()
|
||||
self.save()
|
||||
if stream.info.get('video'):
|
||||
extract.make_keyframe_index(stream.media.path)
|
||||
#if stream.info.get('video'):
|
||||
# extract.make_keyframe_index(stream.media.path)
|
||||
return True, stream.media.size
|
||||
return save_chunk(stream, stream.media, chunk, offset, name, done_cb)
|
||||
return False, 0
|
||||
|
@ -393,7 +404,7 @@ class File(models.Model):
|
|||
config = settings.CONFIG['video']
|
||||
height = self.info['video'][0]['height'] if self.info.get('video') else None
|
||||
max_resolution = max(config['resolutions'])
|
||||
if height <= max_resolution and self.extension in ('mov', 'mkv', 'mp4', 'm4v'):
|
||||
if height and height <= max_resolution and self.extension in ('mov', 'mkv', 'mp4', 'm4v'):
|
||||
vcodec = self.get_codec('video')
|
||||
acodec = self.get_codec('audio')
|
||||
if vcodec in self.MP4_VCODECS and acodec in self.MP4_ACODECS:
|
||||
|
@ -404,7 +415,7 @@ class File(models.Model):
|
|||
config = settings.CONFIG['video']
|
||||
height = self.info['video'][0]['height'] if self.info.get('video') else None
|
||||
max_resolution = max(config['resolutions'])
|
||||
if height <= max_resolution and config['formats'][0] == self.extension:
|
||||
if height and height <= max_resolution and config['formats'][0] == self.extension:
|
||||
vcodec = self.get_codec('video')
|
||||
acodec = self.get_codec('audio')
|
||||
if self.extension in ['mp4', 'm4v'] and vcodec in self.MP4_VCODECS and acodec in self.MP4_ACODECS:
|
||||
|
@ -481,6 +492,13 @@ class File(models.Model):
|
|||
user.is_staff or \
|
||||
self.item.user == user or \
|
||||
self.item.groups.filter(id__in=user.groups.all()).count() > 0
|
||||
if 'instances' in data and 'filename' in self.info and self.data:
|
||||
data['instances'].append({
|
||||
'ignore': False,
|
||||
'path': self.info['filename'],
|
||||
'user': self.item.user.username if self.item and self.item.user else 'system',
|
||||
'volume': 'Direct Upload'
|
||||
})
|
||||
if not can_see_media:
|
||||
if 'instances' in data:
|
||||
data['instances'] = []
|
||||
|
@ -716,6 +734,9 @@ class Stream(models.Model):
|
|||
|
||||
class Meta:
|
||||
unique_together = ("file", "resolution", "format")
|
||||
indexes = [
|
||||
models.Index(fields=['file', 'source', 'available'])
|
||||
]
|
||||
|
||||
file = models.ForeignKey(File, related_name='streams', on_delete=models.CASCADE)
|
||||
resolution = models.IntegerField(default=96)
|
||||
|
@ -795,8 +816,14 @@ class Stream(models.Model):
|
|||
shutil.move(self.file.data.path, target)
|
||||
self.file.data.name = ''
|
||||
self.file.save()
|
||||
self.available = True
|
||||
self.save()
|
||||
done = True
|
||||
elif self.file.can_remux():
|
||||
ok, error = extract.remux_stream(media, target)
|
||||
if ok:
|
||||
self.available = True
|
||||
self.save()
|
||||
done = True
|
||||
if not done:
|
||||
ok, error = extract.stream(media, target, self.name(), info, flags=self.flags)
|
||||
|
@ -805,7 +832,7 @@ class Stream(models.Model):
|
|||
# get current version from db and update
|
||||
try:
|
||||
self.refresh_from_db()
|
||||
except archive.models.DoesNotExist:
|
||||
except Stream.DoesNotExist:
|
||||
pass
|
||||
else:
|
||||
self.update_status(ok, error)
|
||||
|
|
|
@ -1,11 +1,9 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
from datetime import datetime
|
||||
from time import time
|
||||
|
||||
import celery.task.control
|
||||
import kombu.five
|
||||
from time import time, monotonic
|
||||
|
||||
from app.celery import app
|
||||
|
||||
from .models import File
|
||||
|
||||
|
@ -18,7 +16,7 @@ def parse_job(job):
|
|||
'file': f.oshash
|
||||
}
|
||||
if job['time_start']:
|
||||
start_time = datetime.fromtimestamp(time() - (kombu.five.monotonic() - job['time_start']))
|
||||
start_time = datetime.fromtimestamp(time() - (monotonic() - job['time_start']))
|
||||
r.update({
|
||||
'started': start_time,
|
||||
'running': (datetime.now() - start_time).total_seconds()
|
||||
|
@ -30,7 +28,7 @@ def parse_job(job):
|
|||
def status():
|
||||
status = []
|
||||
encoding_jobs = ('archive.tasks.extract_stream', 'archive.tasks.process_stream')
|
||||
c = celery.task.control.inspect()
|
||||
c = app.control.inspect()
|
||||
for job in c.active(safe=True).get('celery@pandora-encoding', []):
|
||||
if job['name'] in encoding_jobs:
|
||||
status.append(parse_job(job))
|
||||
|
@ -67,7 +65,7 @@ def fill_queue():
|
|||
def get_celery_worker_status():
|
||||
ERROR_KEY = "ERROR"
|
||||
try:
|
||||
insp = celery.task.control.inspect()
|
||||
insp = app.control.inspect()
|
||||
d = insp.stats()
|
||||
if not d:
|
||||
d = {ERROR_KEY: 'No running Celery workers were found.'}
|
||||
|
|
|
@ -2,13 +2,14 @@
|
|||
|
||||
from glob import glob
|
||||
|
||||
from celery.task import task
|
||||
from django.conf import settings
|
||||
from django.db import transaction
|
||||
from django.db.models import Q
|
||||
|
||||
from item.models import Item
|
||||
from item.tasks import update_poster, update_timeline
|
||||
from taskqueue.models import Task
|
||||
from app.celery import app
|
||||
|
||||
from . import models
|
||||
from . import extract
|
||||
|
@ -68,7 +69,7 @@ def update_or_create_instance(volume, f):
|
|||
instance.file.item.update_wanted()
|
||||
return instance
|
||||
|
||||
@task(ignore_results=True, queue='default')
|
||||
@app.task(ignore_results=True, queue='default')
|
||||
def update_files(user, volume, files):
|
||||
user = models.User.objects.get(username=user)
|
||||
volume, created = models.Volume.objects.get_or_create(user=user, name=volume)
|
||||
|
@ -100,7 +101,7 @@ def update_files(user, volume, files):
|
|||
Task.start(i, user)
|
||||
update_timeline.delay(i.public_id)
|
||||
|
||||
@task(ignore_results=True, queue='default')
|
||||
@app.task(ignore_results=True, queue='default')
|
||||
def update_info(user, info):
|
||||
user = models.User.objects.get(username=user)
|
||||
files = models.File.objects.filter(oshash__in=list(info))
|
||||
|
@ -114,7 +115,7 @@ def update_info(user, info):
|
|||
Task.start(i, user)
|
||||
update_timeline.delay(i.public_id)
|
||||
|
||||
@task(queue="encoding")
|
||||
@app.task(queue="encoding")
|
||||
def process_stream(fileId):
|
||||
'''
|
||||
process uploaded stream
|
||||
|
@ -140,7 +141,7 @@ def process_stream(fileId):
|
|||
Task.finish(file.item)
|
||||
return True
|
||||
|
||||
@task(queue="encoding")
|
||||
@app.task(queue="encoding")
|
||||
def extract_stream(fileId):
|
||||
'''
|
||||
extract stream from direct upload
|
||||
|
@ -169,7 +170,7 @@ def extract_stream(fileId):
|
|||
models.File.objects.filter(id=fileId).update(encoding=False)
|
||||
Task.finish(file.item)
|
||||
|
||||
@task(queue="encoding")
|
||||
@app.task(queue="encoding")
|
||||
def extract_derivatives(fileId, rebuild=False):
|
||||
file = models.File.objects.get(id=fileId)
|
||||
streams = file.streams.filter(source=None)
|
||||
|
@ -177,7 +178,7 @@ def extract_derivatives(fileId, rebuild=False):
|
|||
streams[0].extract_derivatives(rebuild)
|
||||
return True
|
||||
|
||||
@task(queue="encoding")
|
||||
@app.task(queue="encoding")
|
||||
def update_stream(id):
|
||||
s = models.Stream.objects.get(pk=id)
|
||||
if not glob("%s*" % s.timeline_prefix):
|
||||
|
@ -199,11 +200,11 @@ def update_stream(id):
|
|||
c.update_calculated_values()
|
||||
c.save()
|
||||
|
||||
@task(queue="encoding")
|
||||
def download_media(item_id, url):
|
||||
return external.download(item_id, url)
|
||||
@app.task(queue="encoding")
|
||||
def download_media(item_id, url, referer=None):
|
||||
return external.download(item_id, url, referer)
|
||||
|
||||
@task(queue='default')
|
||||
@app.task(queue='default')
|
||||
def move_media(data, user):
|
||||
from changelog.models import add_changelog
|
||||
from item.models import get_item, Item, ItemSort
|
||||
|
@ -248,6 +249,7 @@ def move_media(data, user):
|
|||
if old_item and old_item.files.count() == 0 and i.files.count() == len(data['ids']):
|
||||
for a in old_item.annotations.all().order_by('id'):
|
||||
a.item = i
|
||||
with transaction.atomic():
|
||||
a.set_public_id()
|
||||
Annotation.objects.filter(id=a.id).update(item=i, public_id=a.public_id)
|
||||
old_item.clips.all().update(item=i, sort=i.sort)
|
||||
|
|
|
@ -103,7 +103,7 @@ def update(request, data):
|
|||
file__available=False,
|
||||
file__wanted=True)]
|
||||
|
||||
if list(filter(lambda l: l['id'] == 'subtitles', settings.CONFIG['layers'])):
|
||||
if utils.get_by_key(settings.CONFIG['layers'], 'isSubtitles', True):
|
||||
qs = files.filter(
|
||||
file__is_subtitle=True,
|
||||
file__available=False
|
||||
|
@ -195,7 +195,9 @@ def addMedia(request, data):
|
|||
response['data']['item'] = f.item.public_id
|
||||
response['data']['itemUrl'] = request.build_absolute_uri('/%s' % f.item.public_id)
|
||||
if not f.available:
|
||||
add_changelog(request, data, f.item.public_id)
|
||||
changelog_data = data.copy()
|
||||
changelog_data['oshash'] = oshash
|
||||
add_changelog(request, changelog_data, f.item.public_id)
|
||||
else:
|
||||
if 'item' in data:
|
||||
i = Item.objects.get(public_id=data['item'])
|
||||
|
@ -220,11 +222,15 @@ def addMedia(request, data):
|
|||
if 'info' in data and data['info'] and isinstance(data['info'], dict):
|
||||
f.info = data['info']
|
||||
f.info['extension'] = extension
|
||||
if 'filename' in data:
|
||||
f.info['filename'] = data['filename']
|
||||
f.parse_info()
|
||||
f.save()
|
||||
response['data']['item'] = i.public_id
|
||||
response['data']['itemUrl'] = request.build_absolute_uri('/%s' % i.public_id)
|
||||
add_changelog(request, data, i.public_id)
|
||||
changelog_data = data.copy()
|
||||
changelog_data['oshash'] = oshash
|
||||
add_changelog(request, changelog_data, i.public_id)
|
||||
return render_to_json_response(response)
|
||||
actions.register(addMedia, cache=False)
|
||||
|
||||
|
@ -739,6 +745,7 @@ def addMediaUrl(request, data):
|
|||
|
||||
takes {
|
||||
url: string, // url
|
||||
referer: string // optional referer url
|
||||
item: string // item
|
||||
}
|
||||
returns {
|
||||
|
@ -751,7 +758,7 @@ def addMediaUrl(request, data):
|
|||
response = json_response()
|
||||
i = Item.objects.get(public_id=data['item'])
|
||||
Task.start(i, request.user)
|
||||
t = tasks.download_media.delay(data['item'], data['url'])
|
||||
t = tasks.download_media.delay(data['item'], data['url'], data.get('referer'))
|
||||
response['data']['taskId'] = t.task_id
|
||||
add_changelog(request, data, data['item'])
|
||||
return render_to_json_response(response)
|
||||
|
|
7
pandora/changelog/apps.py
Normal file
7
pandora/changelog/apps.py
Normal file
|
@ -0,0 +1,7 @@
|
|||
from django.apps import AppConfig
|
||||
|
||||
|
||||
class ChangelogConfig(AppConfig):
|
||||
default_auto_field = "django.db.models.BigAutoField"
|
||||
name = 'changelog'
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
# Generated by Django 4.2.3 on 2023-07-27 21:24
|
||||
|
||||
import django.core.serializers.json
|
||||
from django.db import migrations, models
|
||||
import oxdjango.fields
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('changelog', '0002_jsonfield'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='changelog',
|
||||
name='id',
|
||||
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='changelog',
|
||||
name='value',
|
||||
field=oxdjango.fields.JSONField(default=dict, editable=False, encoder=django.core.serializers.json.DjangoJSONEncoder),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='log',
|
||||
name='data',
|
||||
field=oxdjango.fields.JSONField(default=dict, editable=False, encoder=django.core.serializers.json.DjangoJSONEncoder),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='log',
|
||||
name='id',
|
||||
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
|
||||
),
|
||||
]
|
7
pandora/clip/apps.py
Normal file
7
pandora/clip/apps.py
Normal file
|
@ -0,0 +1,7 @@
|
|||
from django.apps import AppConfig
|
||||
|
||||
|
||||
class ClipConfig(AppConfig):
|
||||
default_auto_field = "django.db.models.BigAutoField"
|
||||
name = 'clip'
|
||||
|
|
@ -17,6 +17,7 @@ keymap = {
|
|||
'place': 'annotations__places__id',
|
||||
'text': 'findvalue',
|
||||
'annotations': 'findvalue',
|
||||
'layer': 'annotations__layer',
|
||||
'user': 'annotations__user__username',
|
||||
}
|
||||
case_insensitive_keys = ('annotations__user__username', )
|
||||
|
|
18
pandora/clip/migrations/0004_id_bigint.py
Normal file
18
pandora/clip/migrations/0004_id_bigint.py
Normal file
|
@ -0,0 +1,18 @@
|
|||
# Generated by Django 4.2.3 on 2023-07-27 21:24
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('clip', '0003_auto_20160219_1805'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='clip',
|
||||
name='id',
|
||||
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
|
||||
),
|
||||
]
|
|
@ -8,6 +8,7 @@ import ox
|
|||
from archive import extract
|
||||
|
||||
from . import managers
|
||||
from .utils import add_cuts
|
||||
|
||||
|
||||
def get_layers(item, interval=None, user=None):
|
||||
|
@ -59,9 +60,7 @@ class MetaClip(object):
|
|||
self.hue = self.saturation = self.lightness = 0
|
||||
self.volume = 0
|
||||
|
||||
def save(self, *args, **kwargs):
|
||||
if self.duration != self.end - self.start:
|
||||
self.update_calculated_values()
|
||||
def update_findvalue(self):
|
||||
if not self.aspect_ratio and self.item:
|
||||
streams = self.item.streams()
|
||||
if streams:
|
||||
|
@ -89,6 +88,11 @@ class MetaClip(object):
|
|||
self.findvalue = '\n'.join(list(filter(None, [a.findvalue for a in anns])))
|
||||
for l in [k['id'] for k in settings.CONFIG['layers']]:
|
||||
setattr(self, l, l in anns_by_layer and bool(len(anns_by_layer[l])))
|
||||
|
||||
def save(self, *args, **kwargs):
|
||||
if self.duration != self.end - self.start:
|
||||
self.update_calculated_values()
|
||||
self.update_findvalue()
|
||||
models.Model.save(self, *args, **kwargs)
|
||||
|
||||
clip_keys = ('id', 'in', 'out', 'position', 'created', 'modified',
|
||||
|
@ -111,8 +115,7 @@ class MetaClip(object):
|
|||
del j[key]
|
||||
#needed here to make item find with clips work
|
||||
if 'annotations' in keys:
|
||||
#annotations = self.annotations.filter(layer__in=settings.CONFIG['clipLayers'])
|
||||
annotations = self.annotations.all()
|
||||
annotations = self.annotations.all().exclude(value='')
|
||||
if qs:
|
||||
for q in qs:
|
||||
annotations = annotations.filter(q)
|
||||
|
@ -150,12 +153,12 @@ class MetaClip(object):
|
|||
data['annotation'] = qs[0].public_id
|
||||
data['parts'] = self.item.cache['parts']
|
||||
data['durations'] = self.item.cache['durations']
|
||||
for key in ('title', 'director', 'year', 'videoRatio'):
|
||||
for key in settings.CONFIG['itemTitleKeys'] + ['videoRatio']:
|
||||
value = self.item.cache.get(key)
|
||||
if value:
|
||||
data[key] = value
|
||||
data['duration'] = data['out'] - data['in']
|
||||
data['cuts'] = tuple([c for c in self.item.get('cuts', []) if c > self.start and c < self.end])
|
||||
add_cuts(data, self.item, self.start, self.end)
|
||||
data['layers'] = self.get_layers(user)
|
||||
data['streams'] = [s.file.oshash for s in self.item.streams()]
|
||||
return data
|
||||
|
@ -186,6 +189,7 @@ class MetaClip(object):
|
|||
def __str__(self):
|
||||
return self.public_id
|
||||
|
||||
|
||||
class Meta:
|
||||
unique_together = ("item", "start", "end")
|
||||
|
||||
|
|
22
pandora/clip/utils.py
Normal file
22
pandora/clip/utils.py
Normal file
|
@ -0,0 +1,22 @@
|
|||
|
||||
|
||||
def add_cuts(data, item, start, end):
|
||||
cuts = []
|
||||
last = False
|
||||
outer = []
|
||||
first = 0
|
||||
for cut in item.get('cuts', []):
|
||||
if cut > start and cut < end:
|
||||
if not cuts:
|
||||
outer.append(first)
|
||||
cuts.append(cut)
|
||||
last = True
|
||||
elif cut <= start:
|
||||
first = cut
|
||||
elif cut >= end:
|
||||
if not len(outer):
|
||||
outer.append(first)
|
||||
if len(outer) == 1:
|
||||
outer.append(cut)
|
||||
data['cuts'] = tuple(cuts)
|
||||
data['outerCuts'] = tuple(outer)
|
|
@ -1009,7 +1009,7 @@
|
|||
{
|
||||
"id": "tags",
|
||||
"title": "Tags",
|
||||
"canAddAnnotations": {"member": true, "staff": true, "admin": true},
|
||||
"canAddAnnotations": {"member": true, "friend": true, "staff": true, "admin": true},
|
||||
"item": "Tag",
|
||||
"autocomplete": true,
|
||||
"overlap": true,
|
||||
|
@ -1399,10 +1399,8 @@
|
|||
corner of the screen
|
||||
"resolutions": List of video resolutions. Supported values are 96, 144,
|
||||
240, 288, 360, 432, 480, 720 and 1080.
|
||||
"torrent": If true, video downloads are offered via BitTorrent
|
||||
*/
|
||||
"video": {
|
||||
"torrent": false,
|
||||
"formats": ["webm", "mp4"],
|
||||
// fixme: this should be named "ratio" or "defaultRatio",
|
||||
// as it also applies to clip lists (on load)
|
||||
|
|
|
@ -73,13 +73,14 @@
|
|||
"canSeeAccessed": {"researcher": true, "staff": true, "admin": true},
|
||||
"canSeeAllTasks": {"staff": true, "admin": true},
|
||||
"canSeeDebugMenu": {"researcher": true, "staff": true, "admin": true},
|
||||
"canSeeExtraItemViews": {"researcher": true, "staff": true, "admin": true},
|
||||
"canSeeMedia": {"researcher": true, "staff": true, "admin": true},
|
||||
"canSeeDocument": {"guest": 1, "member": 1, "researcher": 2, "staff": 3, "admin": 3},
|
||||
"canSeeExtraItemViews": {"researcher": true, "staff": true, "admin": true},
|
||||
"canSeeItem": {"guest": 2, "member": 2, "researcher": 2, "staff": 3, "admin": 3},
|
||||
"canSeeMedia": {"researcher": true, "staff": true, "admin": true},
|
||||
"canSeeSize": {"researcher": true, "staff": true, "admin": true},
|
||||
"canSeeSoftwareVersion": {"researcher": true, "staff": true, "admin": true},
|
||||
"canSendMail": {"staff": true, "admin": true}
|
||||
"canSendMail": {"staff": true, "admin": true},
|
||||
"canShare": {"staff": true, "admin": true}
|
||||
},
|
||||
/*
|
||||
"clipKeys" are the properties that clips can be sorted by (the values are
|
||||
|
@ -312,6 +313,14 @@
|
|||
"autocomplete": true,
|
||||
"columnWidth": 128
|
||||
},
|
||||
{
|
||||
"id": "fulltext",
|
||||
"operator": "+",
|
||||
"title": "Fulltext",
|
||||
"type": "text",
|
||||
"fulltext": true,
|
||||
"find": true
|
||||
},
|
||||
{
|
||||
"id": "created",
|
||||
"operator": "-",
|
||||
|
@ -1494,6 +1503,7 @@
|
|||
"hasEvents": true,
|
||||
"hasPlaces": true,
|
||||
"item": "Keyword",
|
||||
"autocomplete": true,
|
||||
"overlap": true,
|
||||
"type": "string"
|
||||
},
|
||||
|
@ -1875,10 +1885,8 @@
|
|||
corner of the screen
|
||||
"resolutions": List of video resolutions. Supported values are 96, 144,
|
||||
240, 288, 360, 432, 480, 720 and 1080.
|
||||
"torrent": If true, video downloads are offered via BitTorrent
|
||||
*/
|
||||
"video": {
|
||||
"torrent": false,
|
||||
"formats": ["webm", "mp4"],
|
||||
"previewRatio": 1.375,
|
||||
"resolutions": [240, 480]
|
||||
|
|
|
@ -71,13 +71,14 @@
|
|||
"canSeeAccessed": {"staff": true, "admin": true},
|
||||
"canSeeAllTasks": {"staff": true, "admin": true},
|
||||
"canSeeDebugMenu": {"staff": true, "admin": true},
|
||||
"canSeeExtraItemViews": {"staff": true, "admin": true},
|
||||
"canSeeMedia": {"staff": true, "admin": true},
|
||||
"canSeeDocument": {"guest": 1, "member": 1, "staff": 4, "admin": 4},
|
||||
"canSeeExtraItemViews": {"staff": true, "admin": true},
|
||||
"canSeeItem": {"guest": 1, "member": 1, "staff": 4, "admin": 4},
|
||||
"canSeeMedia": {"staff": true, "admin": true},
|
||||
"canSeeSize": {"staff": true, "admin": true},
|
||||
"canSeeSoftwareVersion": {"staff": true, "admin": true},
|
||||
"canSendMail": {"staff": true, "admin": true}
|
||||
"canSendMail": {"staff": true, "admin": true},
|
||||
"canShare": {"staff": true, "admin": true}
|
||||
},
|
||||
/*
|
||||
"clipKeys" are the properties that clips can be sorted by (the values are
|
||||
|
@ -246,6 +247,28 @@
|
|||
"filter": true,
|
||||
"find": true
|
||||
},
|
||||
{
|
||||
"id": "source",
|
||||
"title": "Source",
|
||||
"type": "string",
|
||||
"autocomplete": true,
|
||||
"description": true,
|
||||
"columnWidth": 180,
|
||||
"filter": true,
|
||||
"find": true,
|
||||
"sort": true
|
||||
},
|
||||
{
|
||||
"id": "project",
|
||||
"title": "Project",
|
||||
"type": "string",
|
||||
"autocomplete": true,
|
||||
"description": true,
|
||||
"columnWidth": 120,
|
||||
"filter": true,
|
||||
"find": true,
|
||||
"sort": true
|
||||
},
|
||||
{
|
||||
"id": "id",
|
||||
"operator": "+",
|
||||
|
@ -291,6 +314,24 @@
|
|||
"sort": true,
|
||||
"columnWidth": 256
|
||||
},
|
||||
{
|
||||
"id": "content",
|
||||
"operator": "+",
|
||||
"title": "Content",
|
||||
"type": "text",
|
||||
"find": true,
|
||||
"sort": true,
|
||||
"columnWidth": 256
|
||||
},
|
||||
{
|
||||
"id": "translation",
|
||||
"operator": "+",
|
||||
"title": "Translation",
|
||||
"type": "text",
|
||||
"find": true,
|
||||
"sort": true,
|
||||
"columnWidth": 256
|
||||
},
|
||||
{
|
||||
"id": "matches",
|
||||
"operator": "-",
|
||||
|
@ -310,6 +351,20 @@
|
|||
"autocomplete": true,
|
||||
"columnWidth": 128
|
||||
},
|
||||
{
|
||||
"id": "notes",
|
||||
"title": "Notes",
|
||||
"type": "text",
|
||||
"capability": "canEditMetadata"
|
||||
},
|
||||
{
|
||||
"id": "fulltext",
|
||||
"operator": "+",
|
||||
"title": "Fulltext",
|
||||
"type": "text",
|
||||
"fulltext": true,
|
||||
"find": true
|
||||
},
|
||||
{
|
||||
"id": "created",
|
||||
"operator": "-",
|
||||
|
@ -545,7 +600,6 @@
|
|||
"title": "Director",
|
||||
"type": ["string"],
|
||||
"autocomplete": true,
|
||||
"columnRequired": true,
|
||||
"columnWidth": 180,
|
||||
"sort": true,
|
||||
"sortType": "person"
|
||||
|
@ -564,7 +618,6 @@
|
|||
"title": "Featuring",
|
||||
"type": ["string"],
|
||||
"autocomplete": true,
|
||||
"columnRequired": true,
|
||||
"columnWidth": 180,
|
||||
"filter": true,
|
||||
"sort": true,
|
||||
|
@ -620,7 +673,7 @@
|
|||
{
|
||||
"id": "annotations",
|
||||
"title": "Annotations",
|
||||
"type": "string", // fixme: not the best type for this magic key
|
||||
"type": "text", // fixme: not the best type for this magic key
|
||||
"find": true
|
||||
},
|
||||
{
|
||||
|
@ -658,7 +711,7 @@
|
|||
},
|
||||
{
|
||||
"id": "numberofannotations",
|
||||
"title": "Annotations",
|
||||
"title": "Number of Annotations",
|
||||
"type": "integer",
|
||||
"columnWidth": 60,
|
||||
"sort": true
|
||||
|
@ -794,12 +847,16 @@
|
|||
"id": "user",
|
||||
"title": "User",
|
||||
"type": "string",
|
||||
"columnWidth": 90,
|
||||
"capability": "canSeeMedia",
|
||||
"sort": true,
|
||||
"find": true
|
||||
},
|
||||
{
|
||||
"id": "groups",
|
||||
"title": "Group",
|
||||
"columnWidth": 90,
|
||||
"sort": true,
|
||||
"type": ["string"]
|
||||
},
|
||||
{
|
||||
|
@ -1332,10 +1389,8 @@
|
|||
corner of the screen
|
||||
"resolutions": List of video resolutions. Supported values are 96, 144,
|
||||
240, 288, 360, 432, 480, 720 and 1080.
|
||||
"torrent": If true, video downloads are offered via BitTorrent
|
||||
*/
|
||||
"video": {
|
||||
"torrent": true,
|
||||
"formats": ["webm", "mp4"],
|
||||
"previewRatio": 1.3333333333,
|
||||
//supported resolutions are
|
||||
|
|
|
@ -29,7 +29,7 @@ examples (config.SITENAME.jsonc) that are part of this pan.do/ra distribution.
|
|||
"text": Text shown on mouseover
|
||||
*/
|
||||
"cantPlay": {
|
||||
"icon": "noCopyright",
|
||||
"icon": "NoCopyright",
|
||||
"link": "",
|
||||
"text": ""
|
||||
},
|
||||
|
@ -67,7 +67,7 @@ examples (config.SITENAME.jsonc) that are part of this pan.do/ra distribution.
|
|||
"canManageEntities": {"member": true, "staff": true, "admin": true},
|
||||
"canManageHome": {"staff": true, "admin": true},
|
||||
"canManagePlacesAndEvents": {"member": true, "staff": true, "admin": true},
|
||||
"canManageTitlesAndNames": {"member": true, "staff": true, "admin": true},
|
||||
"canManageTitlesAndNames": {"member": false, "staff": true, "admin": true},
|
||||
"canManageTranslations": {"admin": true},
|
||||
"canManageUsers": {"staff": true, "admin": true},
|
||||
"canPlayClips": {"guest": 1, "member": 1, "staff": 4, "admin": 4},
|
||||
|
@ -102,8 +102,7 @@ examples (config.SITENAME.jsonc) that are part of this pan.do/ra distribution.
|
|||
],
|
||||
/*
|
||||
"clipLayers" is the ordered list of public layers that will appear as the
|
||||
text of clips (in grid view, below the icon). Excluding a layer from this
|
||||
list means it will not be included in find annotations.
|
||||
text of clips (in grid view, below the icon).
|
||||
*/
|
||||
"clipLayers": ["publicnotes", "keywords", "subtitles"],
|
||||
/*
|
||||
|
@ -351,11 +350,11 @@ examples (config.SITENAME.jsonc) that are part of this pan.do/ra distribution.
|
|||
"type": "enum",
|
||||
"columnWidth": 90,
|
||||
"format": {"type": "ColorLevel", "args": [
|
||||
["Public", "Out of Copyright", "Under Copyright", "Private"]
|
||||
["Public", "Restricted", "Private"]
|
||||
]},
|
||||
"sort": true,
|
||||
"sortOperator": "+",
|
||||
"values": ["Public", "Out of Copyright", "Under Copyright", "Private", "Unknown"]
|
||||
"values": ["Public", "Restricted", "Private", "Unknown"]
|
||||
}
|
||||
],
|
||||
/*
|
||||
|
@ -753,6 +752,13 @@ examples (config.SITENAME.jsonc) that are part of this pan.do/ra distribution.
|
|||
"capability": "canSeeMedia",
|
||||
"find": true
|
||||
},
|
||||
{
|
||||
"id": "filename",
|
||||
"title": "Filename",
|
||||
"type": ["string"],
|
||||
"capability": "canSeeMedia",
|
||||
"find": true
|
||||
},
|
||||
{
|
||||
"id": "created",
|
||||
"title": "Date Created",
|
||||
|
@ -1159,6 +1165,11 @@ examples (config.SITENAME.jsonc) that are part of this pan.do/ra distribution.
|
|||
"findDocuments": {"conditions": [], "operator": "&"},
|
||||
"followPlayer": true,
|
||||
"help": "",
|
||||
"hidden": {
|
||||
"collections": [],
|
||||
"edits": [],
|
||||
"lists": []
|
||||
},
|
||||
"icons": "posters",
|
||||
"infoIconSize": 256,
|
||||
"item": "",
|
||||
|
@ -1267,13 +1278,11 @@ examples (config.SITENAME.jsonc) that are part of this pan.do/ra distribution.
|
|||
corner of the screen
|
||||
"resolutions": List of video resolutions. Supported values are 96, 144,
|
||||
240, 288, 360, 432, 480, 720 and 1080.
|
||||
"torrent": If true, video downloads are offered via BitTorrent
|
||||
*/
|
||||
"video": {
|
||||
"downloadFormat": "webm",
|
||||
"formats": ["webm", "mp4"],
|
||||
"previewRatio": 1.3333333333,
|
||||
"resolutions": [240, 480],
|
||||
"torrent": false
|
||||
"resolutions": [240, 480]
|
||||
}
|
||||
}
|
||||
|
|
7
pandora/document/apps.py
Normal file
7
pandora/document/apps.py
Normal file
|
@ -0,0 +1,7 @@
|
|||
from django.apps import AppConfig
|
||||
|
||||
|
||||
class DocumentConfig(AppConfig):
|
||||
default_auto_field = "django.db.models.BigAutoField"
|
||||
name = 'document'
|
||||
|
|
@ -1,14 +1,37 @@
|
|||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
from django.conf import settings
|
||||
|
||||
|
||||
def extract_text(pdf):
|
||||
logger = logging.getLogger('pandora.' + __name__)
|
||||
|
||||
|
||||
def extract_text(pdf, page=None):
|
||||
if page is not None:
|
||||
page = str(page)
|
||||
cmd = ['pdftotext', '-f', page, '-l', page, pdf, '-']
|
||||
else:
|
||||
cmd = ['pdftotext', pdf, '-']
|
||||
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
stdout, stderr = p.communicate()
|
||||
stdout = stdout.decode()
|
||||
return stdout.strip()
|
||||
stdout = stdout.decode().strip()
|
||||
if not stdout:
|
||||
if page:
|
||||
# split page from pdf and ocr
|
||||
fd, page_pdf = tempfile.mkstemp('.pdf')
|
||||
cmd = ['pdfseparate', '-f', page, '-l', page, pdf, page_pdf]
|
||||
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
|
||||
stdout, stderr = p.communicate()
|
||||
text = ocr_image(page_pdf)
|
||||
os.unlink(page_pdf)
|
||||
os.close(fd)
|
||||
return text
|
||||
else:
|
||||
return ocr_image(pdf)
|
||||
return stdout
|
||||
|
||||
def ocr_image(path):
|
||||
cmd = ['tesseract', path, '-', 'txt']
|
||||
|
@ -43,9 +66,11 @@ class FulltextMixin:
|
|||
if self.has_fulltext_key():
|
||||
from elasticsearch.exceptions import NotFoundError
|
||||
try:
|
||||
res = self.elasticsearch().delete(index=self._ES_INDEX, doc_type='document', id=self.id)
|
||||
res = self.elasticsearch().delete(index=self._ES_INDEX, id=self.id)
|
||||
except NotFoundError:
|
||||
pass
|
||||
except:
|
||||
logger.error('failed to delete fulltext document', exc_info=True)
|
||||
|
||||
def update_fulltext(self):
|
||||
if self.has_fulltext_key():
|
||||
|
@ -54,7 +79,7 @@ class FulltextMixin:
|
|||
doc = {
|
||||
'text': text.lower()
|
||||
}
|
||||
res = self.elasticsearch().index(index=self._ES_INDEX, doc_type='document', id=self.id, body=doc)
|
||||
res = self.elasticsearch().index(index=self._ES_INDEX, id=self.id, body=doc)
|
||||
|
||||
@classmethod
|
||||
def find_fulltext(cls, query):
|
||||
|
@ -95,3 +120,69 @@ class FulltextMixin:
|
|||
ids += [int(r['_id']) for r in res['hits']['hits']]
|
||||
from_ += len(res['hits']['hits'])
|
||||
return ids
|
||||
|
||||
def highlight_page(self, page, query, size):
|
||||
import pypdfium2 as pdfium
|
||||
from PIL import Image
|
||||
from PIL import ImageDraw
|
||||
|
||||
pdfpath = self.file.path
|
||||
pagenumber = int(page) - 1
|
||||
jpg = tempfile.NamedTemporaryFile(suffix='.jpg')
|
||||
output = jpg.name
|
||||
TINT_COLOR = (255, 255, 0)
|
||||
TRANSPARENCY = .45
|
||||
OPACITY = int(255 * TRANSPARENCY)
|
||||
scale = 150/72
|
||||
|
||||
pdf = pdfium.PdfDocument(pdfpath)
|
||||
page = pdf[pagenumber]
|
||||
|
||||
bitmap = page.render(scale=scale, rotation=0)
|
||||
img = bitmap.to_pil().convert('RGBA')
|
||||
overlay = Image.new('RGBA', img.size, TINT_COLOR+(0,))
|
||||
draw = ImageDraw.Draw(overlay)
|
||||
|
||||
textpage = page.get_textpage()
|
||||
search = textpage.search(query)
|
||||
result = search.get_next()
|
||||
while result:
|
||||
pos, steps = result
|
||||
steps += 1
|
||||
while steps:
|
||||
box = textpage.get_charbox(pos)
|
||||
box = [b*scale for b in box]
|
||||
tl = (box[0], img.size[1] - box[3])
|
||||
br = (box[2], img.size[1] - box[1])
|
||||
draw.rectangle((tl, br), fill=TINT_COLOR+(OPACITY,))
|
||||
pos += 1
|
||||
steps -= 1
|
||||
result = search.get_next()
|
||||
img = Image.alpha_composite(img, overlay)
|
||||
img = img.convert("RGB")
|
||||
aspect = img.size[0] / img.size[1]
|
||||
resize_method = Image.LANCZOS
|
||||
if img.size[0] >= img.size[1]:
|
||||
width = size
|
||||
height = int(size / aspect)
|
||||
else:
|
||||
width = int(size / aspect)
|
||||
height = size
|
||||
img = img.resize((width, height), resize_method)
|
||||
img.save(output, quality=72)
|
||||
return jpg
|
||||
|
||||
|
||||
class FulltextPageMixin(FulltextMixin):
|
||||
_ES_INDEX = "document-page-index"
|
||||
|
||||
def extract_fulltext(self):
|
||||
if self.document.file:
|
||||
if self.document.extension == 'pdf':
|
||||
return extract_text(self.document.file.path, self.page)
|
||||
elif self.extension in ('png', 'jpg'):
|
||||
return ocr_image(self.document.file.path)
|
||||
elif self.extension == 'html':
|
||||
# FIXME: is there a nice way to split that into pages
|
||||
return self.data.get('text', '')
|
||||
return ''
|
||||
|
|
|
@ -5,7 +5,6 @@ from django.db import connection, transaction
|
|||
from django.db.models import fields
|
||||
from django.conf import settings
|
||||
|
||||
settings.RELOAD_CONFIG = False
|
||||
import app.monkey_patch
|
||||
from ... import models
|
||||
|
||||
|
|
|
@ -5,7 +5,6 @@ from django.db import connection, transaction
|
|||
from django.db.models import fields
|
||||
from django.conf import settings
|
||||
|
||||
settings.RELOAD_CONFIG = False
|
||||
import app.monkey_patch
|
||||
|
||||
|
||||
|
|
|
@ -1,4 +1,5 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
from datetime import datetime
|
||||
import unicodedata
|
||||
|
||||
from django.db.models import Q, Manager
|
||||
|
@ -14,6 +15,7 @@ from documentcollection.models import Collection
|
|||
from item import utils
|
||||
from user.models import Group
|
||||
|
||||
from .pages import PageManager
|
||||
|
||||
keymap = {
|
||||
'item': 'items__public_id',
|
||||
|
@ -61,7 +63,7 @@ def parseCondition(condition, user, item=None, owner=None):
|
|||
|
||||
def buildCondition(k, op, v, user, exclude=False, owner=None):
|
||||
import entity.models
|
||||
from . import models
|
||||
from .. import models
|
||||
|
||||
# fixme: frontend should never call with list
|
||||
if k == 'list':
|
||||
|
@ -297,5 +299,8 @@ class DocumentManager(Manager):
|
|||
q |= Q(groups__in=user.groups.all())
|
||||
rendered_q |= Q(groups__in=user.groups.all())
|
||||
qs = qs.filter(q)
|
||||
max_level = len(settings.CONFIG['documentRightsLevels'])
|
||||
qs = qs.filter(rightslevel__lte=max_level)
|
||||
|
||||
return qs
|
||||
|
302
pandora/document/managers/pages.py
Normal file
302
pandora/document/managers/pages.py
Normal file
|
@ -0,0 +1,302 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
from datetime import datetime
|
||||
import unicodedata
|
||||
|
||||
from six import string_types
|
||||
from django.db.models import Q, Manager
|
||||
from django.conf import settings
|
||||
|
||||
import ox
|
||||
from oxdjango.query import QuerySet
|
||||
|
||||
import entity.managers
|
||||
from oxdjango.managers import get_operator
|
||||
|
||||
from documentcollection.models import Collection
|
||||
from item import utils
|
||||
from user.models import Group
|
||||
|
||||
|
||||
keymap = {
|
||||
'item': 'items__public_id',
|
||||
}
|
||||
default_key = 'title'
|
||||
|
||||
def get_key_type(k):
|
||||
key_type = (utils.get_by_id(settings.CONFIG['documentKeys'], k) or {'type': 'string'}).get('type')
|
||||
if isinstance(key_type, list):
|
||||
key_type = key_type[0]
|
||||
key_type = {
|
||||
'title': 'string',
|
||||
'person': 'string',
|
||||
'text': 'string',
|
||||
'year': 'string',
|
||||
'length': 'string',
|
||||
'layer': 'string',
|
||||
'list': 'list',
|
||||
}.get(key_type, key_type)
|
||||
return key_type
|
||||
|
||||
|
||||
def parseCondition(condition, user, item=None, owner=None):
|
||||
'''
|
||||
'''
|
||||
k = condition.get('key', default_key)
|
||||
k = keymap.get(k, k)
|
||||
if not k:
|
||||
k = default_key
|
||||
if item and k == 'description':
|
||||
item_conditions = condition.copy()
|
||||
item_conditions['key'] = 'items__itemproperties__description'
|
||||
return parseCondition(condition, user) | parseCondition(item_conditions, user)
|
||||
|
||||
v = condition['value']
|
||||
op = condition.get('operator')
|
||||
if not op:
|
||||
op = '='
|
||||
|
||||
if op.startswith('!'):
|
||||
return buildCondition(k, op[1:], v, user, True, owner=owner)
|
||||
else:
|
||||
return buildCondition(k, op, v, user, owner=owner)
|
||||
|
||||
def buildCondition(k, op, v, user, exclude=False, owner=None):
|
||||
import entity.models
|
||||
from .. import models
|
||||
|
||||
# fixme: frontend should never call with list
|
||||
if k == 'list':
|
||||
print('fixme: frontend should never call with list', k, op, v)
|
||||
k = 'collection'
|
||||
|
||||
key_type = get_key_type(k)
|
||||
|
||||
key_config = (utils.get_by_id(settings.CONFIG['documentKeys'], k) or {'type': 'string'})
|
||||
|
||||
facet_keys = models.Document.facet_keys
|
||||
if k == 'document':
|
||||
k = 'document__id'
|
||||
if op == '&' and isinstance(v, list):
|
||||
v = [ox.fromAZ(id_) for id_ in v]
|
||||
k += get_operator(op)
|
||||
else:
|
||||
v = ox.fromAZ(v)
|
||||
q = Q(**{k: v})
|
||||
if exclude:
|
||||
q = ~Q(document__id__in=models.Document.objects.filter(q))
|
||||
return q
|
||||
elif k == 'rightslevel':
|
||||
q = Q(document__rightslevel=v)
|
||||
if exclude:
|
||||
q = ~Q(document__rightslevel=v)
|
||||
return q
|
||||
elif k == 'groups':
|
||||
if op == '==' and v == '$my':
|
||||
if not owner:
|
||||
owner = user
|
||||
groups = owner.groups.all()
|
||||
else:
|
||||
key = 'name' + get_operator(op)
|
||||
groups = Group.objects.filter(**{key: v})
|
||||
if not groups.count():
|
||||
return Q(id=0)
|
||||
q = Q(document__groups__in=groups)
|
||||
if exclude:
|
||||
q = ~q
|
||||
return q
|
||||
elif k in ('oshash', 'items__public_id'):
|
||||
q = Q(**{k: v})
|
||||
if exclude:
|
||||
q = ~Q(id__in=models.Document.objects.filter(q))
|
||||
return q
|
||||
elif isinstance(v, bool):
|
||||
key = k
|
||||
elif k == 'entity':
|
||||
entity_key, entity_v = entity.managers.namePredicate(op, v)
|
||||
key = 'id__in'
|
||||
v = entity.models.DocumentProperties.objects.filter(**{
|
||||
'entity__' + entity_key: entity_v
|
||||
}).values_list('document_id', flat=True)
|
||||
elif k == 'collection':
|
||||
q = Q(id=0)
|
||||
l = v.split(":", 1)
|
||||
if len(l) >= 2:
|
||||
lqs = list(Collection.objects.filter(name=l[1], user__username=l[0]))
|
||||
if len(lqs) == 1 and lqs[0].accessible(user):
|
||||
l = lqs[0]
|
||||
if l.query.get('static', False) is False:
|
||||
data = l.query
|
||||
q = parseConditions(data.get('conditions', []),
|
||||
data.get('operator', '&'),
|
||||
user, owner=l.user)
|
||||
else:
|
||||
q = Q(id__in=l.documents.all())
|
||||
else:
|
||||
q = Q(id=0)
|
||||
return q
|
||||
elif key_config.get('fulltext'):
|
||||
qs = models.Page.find_fulltext_ids(v)
|
||||
q = Q(id__in=qs)
|
||||
if exclude:
|
||||
q = ~Q(id__in=qs)
|
||||
return q
|
||||
elif key_type == 'boolean':
|
||||
q = Q(**{'find__key': k, 'find__value': v})
|
||||
if exclude:
|
||||
q = ~Q(id__in=models.Document.objects.filter(q))
|
||||
return q
|
||||
elif key_type == "string":
|
||||
in_find = True
|
||||
if in_find:
|
||||
value_key = 'find__value'
|
||||
else:
|
||||
value_key = k
|
||||
if isinstance(v, string_types):
|
||||
v = unicodedata.normalize('NFKD', v).lower()
|
||||
if k in facet_keys:
|
||||
in_find = False
|
||||
facet_value = 'facets__value' + get_operator(op, 'istr')
|
||||
v = models.Document.objects.filter(**{'facets__key': k, facet_value: v})
|
||||
value_key = 'id__in'
|
||||
else:
|
||||
value_key = value_key + get_operator(op)
|
||||
k = str(k)
|
||||
value_key = str(value_key)
|
||||
if k == '*':
|
||||
q = Q(**{'find__value' + get_operator(op): v}) | \
|
||||
Q(**{'facets__value' + get_operator(op, 'istr'): v})
|
||||
elif in_find:
|
||||
q = Q(**{'find__key': k, value_key: v})
|
||||
else:
|
||||
q = Q(**{value_key: v})
|
||||
if exclude:
|
||||
q = ~Q(id__in=models.Document.objects.filter(q))
|
||||
return q
|
||||
elif key_type == 'date':
|
||||
def parse_date(d):
|
||||
while len(d) < 3:
|
||||
d.append(1)
|
||||
return datetime(*[int(i) for i in d])
|
||||
|
||||
#using sort here since find only contains strings
|
||||
v = parse_date(v.split('-'))
|
||||
vk = 'sort__%s%s' % (k, get_operator(op, 'int'))
|
||||
vk = str(vk)
|
||||
q = Q(**{vk: v})
|
||||
if exclude:
|
||||
q = ~q
|
||||
return q
|
||||
else: # integer, float, list, time
|
||||
#use sort table here
|
||||
if key_type == 'time':
|
||||
v = int(utils.parse_time(v))
|
||||
|
||||
vk = 'sort__%s%s' % (k, get_operator(op, 'int'))
|
||||
vk = str(vk)
|
||||
q = Q(**{vk: v})
|
||||
if exclude:
|
||||
q = ~q
|
||||
return q
|
||||
key = str(key)
|
||||
q = Q(**{key: v})
|
||||
if exclude:
|
||||
q = ~q
|
||||
return q
|
||||
|
||||
|
||||
def parseConditions(conditions, operator, user, item=None, owner=None):
|
||||
'''
|
||||
conditions: [
|
||||
{
|
||||
value: "war"
|
||||
}
|
||||
{
|
||||
key: "year",
|
||||
value: "1970-1980,
|
||||
operator: "!="
|
||||
},
|
||||
{
|
||||
key: "country",
|
||||
value: "f",
|
||||
operator: "^"
|
||||
}
|
||||
],
|
||||
operator: "&"
|
||||
'''
|
||||
conn = []
|
||||
for condition in conditions:
|
||||
if 'conditions' in condition:
|
||||
q = parseConditions(condition['conditions'],
|
||||
condition.get('operator', '&'), user, item, owner=owner)
|
||||
if q:
|
||||
conn.append(q)
|
||||
pass
|
||||
else:
|
||||
conn.append(parseCondition(condition, user, item, owner=owner))
|
||||
if conn:
|
||||
q = conn[0]
|
||||
for c in conn[1:]:
|
||||
if operator == '|':
|
||||
q = q | c
|
||||
else:
|
||||
q = q & c
|
||||
return q
|
||||
return None
|
||||
|
||||
|
||||
class PageManager(Manager):
|
||||
|
||||
def get_query_set(self):
|
||||
return QuerySet(self.model)
|
||||
|
||||
def find(self, data, user, item=None):
|
||||
'''
|
||||
query: {
|
||||
conditions: [
|
||||
{
|
||||
value: "war"
|
||||
}
|
||||
{
|
||||
key: "year",
|
||||
value: "1970-1980,
|
||||
operator: "!="
|
||||
},
|
||||
{
|
||||
key: "country",
|
||||
value: "f",
|
||||
operator: "^"
|
||||
}
|
||||
],
|
||||
operator: "&"
|
||||
}
|
||||
'''
|
||||
|
||||
#join query with operator
|
||||
qs = self.get_query_set()
|
||||
query = data.get('query', {})
|
||||
conditions = parseConditions(query.get('conditions', []),
|
||||
query.get('operator', '&'),
|
||||
user, item)
|
||||
if conditions:
|
||||
qs = qs.filter(conditions)
|
||||
qs = qs.distinct()
|
||||
|
||||
#anonymous can only see public items
|
||||
if not user or user.is_anonymous:
|
||||
level = 'guest'
|
||||
allowed_level = settings.CONFIG['capabilities']['canSeeDocument'][level]
|
||||
qs = qs.filter(document__rightslevel__lte=allowed_level)
|
||||
rendered_q = Q(rendered=True)
|
||||
#users can see public items, there own items and items of there groups
|
||||
else:
|
||||
level = user.profile.get_level()
|
||||
allowed_level = settings.CONFIG['capabilities']['canSeeDocument'][level]
|
||||
q = Q(document__rightslevel__lte=allowed_level) | Q(document__user=user)
|
||||
rendered_q = Q(rendered=True) | Q(document__user=user)
|
||||
if user.groups.count():
|
||||
q |= Q(document__groups__in=user.groups.all())
|
||||
rendered_q |= Q(document__groups__in=user.groups.all())
|
||||
qs = qs.filter(q)
|
||||
|
||||
return qs
|
||||
|
35
pandora/document/migrations/0012_auto_20200513_0001.py
Normal file
35
pandora/document/migrations/0012_auto_20200513_0001.py
Normal file
|
@ -0,0 +1,35 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
# Generated by Django 1.11.22 on 2020-05-13 00:01
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import django.core.serializers.json
|
||||
from django.db import migrations, models
|
||||
import django.db.models.deletion
|
||||
import document.fulltext
|
||||
import oxdjango.fields
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('document', '0011_jsonfield'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name='Page',
|
||||
fields=[
|
||||
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('created', models.DateTimeField(auto_now_add=True)),
|
||||
('modified', models.DateTimeField(auto_now=True)),
|
||||
('page', models.IntegerField(default=1)),
|
||||
('data', oxdjango.fields.JSONField(default=dict, editable=False, encoder=django.core.serializers.json.DjangoJSONEncoder)),
|
||||
],
|
||||
bases=(models.Model, document.fulltext.FulltextPageMixin),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='page',
|
||||
name='document',
|
||||
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='pages_set', to='document.Document'),
|
||||
),
|
||||
]
|
55
pandora/document/migrations/0013_id_bigint.py
Normal file
55
pandora/document/migrations/0013_id_bigint.py
Normal file
|
@ -0,0 +1,55 @@
|
|||
# Generated by Django 4.2.3 on 2023-07-27 21:24
|
||||
|
||||
import django.core.serializers.json
|
||||
from django.db import migrations, models
|
||||
import oxdjango.fields
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('document', '0012_auto_20200513_0001'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='access',
|
||||
name='id',
|
||||
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='document',
|
||||
name='data',
|
||||
field=oxdjango.fields.JSONField(default=dict, editable=False, encoder=django.core.serializers.json.DjangoJSONEncoder),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='document',
|
||||
name='id',
|
||||
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='facet',
|
||||
name='id',
|
||||
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='find',
|
||||
name='id',
|
||||
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='itemproperties',
|
||||
name='description',
|
||||
field=models.TextField(default=''),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='itemproperties',
|
||||
name='id',
|
||||
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='page',
|
||||
name='id',
|
||||
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
|
||||
),
|
||||
]
|
|
@ -6,11 +6,12 @@ import os
|
|||
import re
|
||||
import unicodedata
|
||||
|
||||
from django.conf import settings
|
||||
from django.contrib.auth import get_user_model
|
||||
from django.db import models, transaction
|
||||
from django.db.models import Q, Sum, Max
|
||||
from django.contrib.auth import get_user_model
|
||||
from django.db.models.signals import pre_delete
|
||||
from django.conf import settings
|
||||
from django.utils import datetime_safe
|
||||
from oxdjango.fields import JSONField
|
||||
|
||||
from PIL import Image
|
||||
|
@ -21,7 +22,7 @@ from oxdjango.sortmodel import get_sort_field
|
|||
from person.models import get_name_sort
|
||||
from item.models import Item
|
||||
from annotation.models import Annotation
|
||||
from archive.extract import resize_image
|
||||
from archive.extract import resize_image, open_image_rgb
|
||||
from archive.chunk import save_chunk
|
||||
from user.models import Group
|
||||
from user.utils import update_groups
|
||||
|
@ -29,7 +30,7 @@ from user.utils import update_groups
|
|||
from . import managers
|
||||
from . import utils
|
||||
from . import tasks
|
||||
from .fulltext import FulltextMixin
|
||||
from .fulltext import FulltextMixin, FulltextPageMixin
|
||||
|
||||
User = get_user_model()
|
||||
|
||||
|
@ -79,7 +80,7 @@ class Document(models.Model, FulltextMixin):
|
|||
current_values = []
|
||||
for k in settings.CONFIG['documentKeys']:
|
||||
if k.get('sortType') == 'person':
|
||||
current_values += self.get(k['id'], [])
|
||||
current_values += self.get_value(k['id'], [])
|
||||
if not isinstance(current_values, list):
|
||||
if not current_values:
|
||||
current_values = []
|
||||
|
@ -327,6 +328,9 @@ class Document(models.Model, FulltextMixin):
|
|||
def editable(self, user, item=None):
|
||||
if not user or user.is_anonymous:
|
||||
return False
|
||||
max_level = len(settings.CONFIG['rightsLevels'])
|
||||
if self.rightslevel > max_level:
|
||||
return False
|
||||
if self.user == user or \
|
||||
self.groups.filter(id__in=user.groups.all()).count() > 0 or \
|
||||
user.is_staff or \
|
||||
|
@ -346,6 +350,8 @@ class Document(models.Model, FulltextMixin):
|
|||
groups = data.pop('groups')
|
||||
update_groups(self, groups)
|
||||
for key in data:
|
||||
if key == "id":
|
||||
continue
|
||||
k = list(filter(lambda i: i['id'] == key, settings.CONFIG['documentKeys']))
|
||||
ktype = k and k[0].get('type') or ''
|
||||
if key == 'text' and self.extension == 'html':
|
||||
|
@ -546,10 +552,10 @@ class Document(models.Model, FulltextMixin):
|
|||
if len(crop) == 4:
|
||||
path = os.path.join(folder, '%dp%d,%s.jpg' % (1024, page, ','.join(map(str, crop))))
|
||||
if not os.path.exists(path):
|
||||
img = Image.open(src).crop(crop)
|
||||
img = open_image_rgb(src).crop(crop)
|
||||
img.save(path)
|
||||
else:
|
||||
img = Image.open(path)
|
||||
img = open_image_rgb(path)
|
||||
src = path
|
||||
if size < max(img.size):
|
||||
path = os.path.join(folder, '%dp%d,%s.jpg' % (size, page, ','.join(map(str, crop))))
|
||||
|
@ -562,10 +568,10 @@ class Document(models.Model, FulltextMixin):
|
|||
if len(crop) == 4:
|
||||
path = os.path.join(folder, '%s.jpg' % ','.join(map(str, crop)))
|
||||
if not os.path.exists(path):
|
||||
img = Image.open(src).crop(crop)
|
||||
img = open_image_rgb(src).convert('RGB').crop(crop)
|
||||
img.save(path)
|
||||
else:
|
||||
img = Image.open(path)
|
||||
img = open_image_rgb(path)
|
||||
src = path
|
||||
if size < max(img.size):
|
||||
path = os.path.join(folder, '%sp%s.jpg' % (size, ','.join(map(str, crop))))
|
||||
|
@ -574,7 +580,7 @@ class Document(models.Model, FulltextMixin):
|
|||
if os.path.exists(src) and not os.path.exists(path):
|
||||
image_size = max(self.width, self.height)
|
||||
if image_size == -1:
|
||||
image_size = max(*Image.open(src).size)
|
||||
image_size = max(*open_image_rgb(src).size)
|
||||
if size > image_size:
|
||||
path = src
|
||||
else:
|
||||
|
@ -586,6 +592,11 @@ class Document(models.Model, FulltextMixin):
|
|||
image = os.path.join(os.path.dirname(pdf), '1024p%d.jpg' % page)
|
||||
utils.extract_pdfpage(pdf, image, page)
|
||||
|
||||
def create_pages(self):
|
||||
for page in range(self.pages):
|
||||
page += 1
|
||||
p, c = Page.objects.get_or_create(document=self, page=page)
|
||||
|
||||
def get_info(self):
|
||||
if self.extension == 'pdf':
|
||||
self.thumbnail(1024)
|
||||
|
@ -595,7 +606,7 @@ class Document(models.Model, FulltextMixin):
|
|||
self.pages = utils.pdfpages(self.file.path)
|
||||
elif self.width == -1:
|
||||
self.pages = -1
|
||||
self.width, self.height = Image.open(self.file.path).size
|
||||
self.width, self.height = open_image_rgb(self.file.path).size
|
||||
|
||||
def get_ratio(self):
|
||||
if self.extension == 'pdf':
|
||||
|
@ -702,6 +713,41 @@ class ItemProperties(models.Model):
|
|||
super(ItemProperties, self).save(*args, **kwargs)
|
||||
|
||||
|
||||
class Page(models.Model, FulltextPageMixin):
|
||||
|
||||
created = models.DateTimeField(auto_now_add=True)
|
||||
modified = models.DateTimeField(auto_now=True)
|
||||
|
||||
document = models.ForeignKey(Document, related_name='pages_set', on_delete=models.CASCADE)
|
||||
page = models.IntegerField(default=1)
|
||||
data = JSONField(default=dict, editable=False)
|
||||
|
||||
objects = managers.PageManager()
|
||||
|
||||
def __str__(self):
|
||||
return u"%s:%s" % (self.document, self.page)
|
||||
|
||||
def json(self, keys=None, user=None):
|
||||
data = {}
|
||||
data['document'] = ox.toAZ(self.document.id)
|
||||
data['page'] = self.page
|
||||
data['id'] = '{document}/{page}'.format(**data)
|
||||
document_keys = []
|
||||
if keys:
|
||||
for key in list(data):
|
||||
if key not in keys:
|
||||
del data[key]
|
||||
for key in keys:
|
||||
if 'fulltext' in key:
|
||||
data['fulltext'] = self.extract_fulltext()
|
||||
elif key in ('document', 'page', 'id'):
|
||||
pass
|
||||
else:
|
||||
document_keys.append(key)
|
||||
if document_keys:
|
||||
data.update(self.document.json(document_keys, user))
|
||||
return data
|
||||
|
||||
class Access(models.Model):
|
||||
class Meta:
|
||||
unique_together = ("document", "user")
|
||||
|
|
135
pandora/document/page_views.py
Normal file
135
pandora/document/page_views.py
Normal file
|
@ -0,0 +1,135 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
import os
|
||||
import re
|
||||
from glob import glob
|
||||
import unicodedata
|
||||
|
||||
import ox
|
||||
from ox.utils import json
|
||||
from oxdjango.api import actions
|
||||
from oxdjango.decorators import login_required_json
|
||||
from oxdjango.http import HttpFileResponse
|
||||
from oxdjango.shortcuts import render_to_json_response, get_object_or_404_json, json_response, HttpErrorJson
|
||||
from django import forms
|
||||
from django.db.models import Count, Sum
|
||||
from django.conf import settings
|
||||
|
||||
from item import utils
|
||||
from item.models import Item
|
||||
from itemlist.models import List
|
||||
from entity.models import Entity
|
||||
from archive.chunk import process_chunk
|
||||
from changelog.models import add_changelog
|
||||
|
||||
from . import models
|
||||
from . import tasks
|
||||
|
||||
def parse_query(data, user):
|
||||
query = {}
|
||||
query['range'] = [0, 100]
|
||||
query['sort'] = [{'key': 'page', 'operator': '+'}, {'key': 'title', 'operator': '+'}]
|
||||
for key in ('keys', 'group', 'file', 'range', 'position', 'positions', 'sort'):
|
||||
if key in data:
|
||||
query[key] = data[key]
|
||||
query['qs'] = models.Page.objects.find(data, user)
|
||||
return query
|
||||
|
||||
def _order_query(qs, sort):
|
||||
prefix = 'document__sort__'
|
||||
order_by = []
|
||||
for e in sort:
|
||||
operator = e['operator']
|
||||
if operator != '-':
|
||||
operator = ''
|
||||
key = {
|
||||
'index': 'document__items__itemproperties__index',
|
||||
'position': 'id',
|
||||
'name': 'title',
|
||||
}.get(e['key'], e['key'])
|
||||
if key == 'resolution':
|
||||
order_by.append('%swidth' % operator)
|
||||
order_by.append('%sheight' % operator)
|
||||
else:
|
||||
if '__' not in key and key not in ('created', 'modified', 'page'):
|
||||
key = "%s%s" % (prefix, key)
|
||||
order = '%s%s' % (operator, key)
|
||||
order_by.append(order)
|
||||
if order_by:
|
||||
qs = qs.order_by(*order_by, nulls_last=True)
|
||||
qs = qs.distinct()
|
||||
return qs
|
||||
|
||||
def _order_by_group(query):
|
||||
prefix = 'document__sort__'
|
||||
if 'sort' in query:
|
||||
op = '-' if query['sort'][0]['operator'] == '-' else ''
|
||||
if len(query['sort']) == 1 and query['sort'][0]['key'] == 'items':
|
||||
order_by = op + prefix + 'items'
|
||||
if query['group'] == "year":
|
||||
secondary = op + prefix + 'sortvalue'
|
||||
order_by = (order_by, secondary)
|
||||
elif query['group'] != "keyword":
|
||||
order_by = (order_by, prefix + 'sortvalue')
|
||||
else:
|
||||
order_by = (order_by, 'value')
|
||||
else:
|
||||
order_by = op + prefix + 'sortvalue'
|
||||
order_by = (order_by, prefix + 'items')
|
||||
else:
|
||||
order_by = ('-' + prefix + 'sortvalue', prefix + 'items')
|
||||
return order_by
|
||||
|
||||
def findPages(request, data):
|
||||
'''
|
||||
Finds documents pages for a given query
|
||||
takes {
|
||||
query: object, // query object, see `find`
|
||||
sort: [object], // list of sort objects, see `find`
|
||||
range: [int, int], // range of results, per current sort order
|
||||
keys: [string] // list of keys to return
|
||||
}
|
||||
returns {
|
||||
items: [{ // list of pages
|
||||
id: string
|
||||
page: int
|
||||
}]
|
||||
}
|
||||
'''
|
||||
query = parse_query(data, request.user)
|
||||
#order
|
||||
qs = _order_query(query['qs'], query['sort'])
|
||||
|
||||
response = json_response()
|
||||
if 'group' in query:
|
||||
response['data']['items'] = []
|
||||
items = 'items'
|
||||
document_qs = query['qs']
|
||||
order_by = _order_by_group(query)
|
||||
qs = models.Facet.objects.filter(key=query['group']).filter(document__id__in=document_qs)
|
||||
qs = qs.values('value').annotate(items=Count('id')).order_by(*order_by)
|
||||
|
||||
if 'positions' in query:
|
||||
response['data']['positions'] = {}
|
||||
ids = [j['value'] for j in qs]
|
||||
response['data']['positions'] = utils.get_positions(ids, query['positions'])
|
||||
elif 'range' in data:
|
||||
qs = qs[query['range'][0]:query['range'][1]]
|
||||
response['data']['items'] = [{'name': i['value'], 'items': i[items]} for i in qs]
|
||||
else:
|
||||
response['data']['items'] = qs.count()
|
||||
elif 'keys' in data:
|
||||
qs = qs[query['range'][0]:query['range'][1]]
|
||||
|
||||
response['data']['items'] = [l.json(data['keys'], request.user) for l in qs]
|
||||
elif 'position' in data:
|
||||
#FIXME: actually implement position requests
|
||||
response['data']['position'] = 0
|
||||
elif 'positions' in data:
|
||||
ids = list(qs.values_list('id', flat=True))
|
||||
response['data']['positions'] = utils.get_positions(ids, query['positions'], decode_id=True)
|
||||
else:
|
||||
response['data']['items'] = qs.count()
|
||||
return render_to_json_response(response)
|
||||
actions.register(findPages)
|
||||
|
|
@ -1,8 +1,30 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
from celery.task import task
|
||||
import ox
|
||||
from app.celery import app
|
||||
|
||||
@task(queue="encoding")
|
||||
@app.task(queue="encoding")
|
||||
def extract_fulltext(id):
|
||||
from . import models
|
||||
d = models.Document.objects.get(id=id)
|
||||
d.update_fulltext()
|
||||
d.create_pages()
|
||||
for page in d.pages_set.all():
|
||||
page.update_fulltext()
|
||||
|
||||
|
||||
@app.task(queue='default')
|
||||
def bulk_edit(data, username):
|
||||
from django.db import transaction
|
||||
from . import models
|
||||
from item.models import Item
|
||||
user = models.User.objects.get(username=username)
|
||||
item = 'item' in data and Item.objects.get(public_id=data['item']) or None
|
||||
ids = data['id']
|
||||
del data['id']
|
||||
documents = models.Document.objects.filter(pk__in=map(ox.fromAZ, ids))
|
||||
for document in documents:
|
||||
if document.editable(user, item):
|
||||
with transaction.atomic():
|
||||
document.refresh_from_db()
|
||||
document.edit(data, user, item)
|
||||
document.save()
|
||||
return {}
|
||||
|
|
|
@ -12,8 +12,10 @@ from oxdjango.decorators import login_required_json
|
|||
from oxdjango.http import HttpFileResponse
|
||||
from oxdjango.shortcuts import render_to_json_response, get_object_or_404_json, json_response, HttpErrorJson
|
||||
from django import forms
|
||||
from django.db.models import Count, Sum
|
||||
from django.conf import settings
|
||||
from django.db.models import Count, Sum
|
||||
from django.http import HttpResponse
|
||||
from django.shortcuts import render
|
||||
|
||||
from item import utils
|
||||
from item.models import Item
|
||||
|
@ -23,6 +25,8 @@ from archive.chunk import process_chunk
|
|||
from changelog.models import add_changelog
|
||||
|
||||
from . import models
|
||||
from . import tasks
|
||||
from . import page_views
|
||||
|
||||
def get_document_or_404_json(request, id):
|
||||
response = {'status': {'code': 404,
|
||||
|
@ -131,12 +135,12 @@ def editDocument(request, data):
|
|||
item = 'item' in data and Item.objects.get(public_id=data['item']) or None
|
||||
if data['id']:
|
||||
if isinstance(data['id'], list):
|
||||
documents = models.Document.objects.filter(pk__in=map(ox.fromAZ, data['id']))
|
||||
add_changelog(request, data)
|
||||
t = tasks.bulk_edit.delay(data, request.user.username)
|
||||
response['data']['taskId'] = t.task_id
|
||||
else:
|
||||
documents = [models.Document.get(data['id'])]
|
||||
for document in documents:
|
||||
document = models.Document.get(data['id'])
|
||||
if document.editable(request.user, item):
|
||||
if document == documents[0]:
|
||||
add_changelog(request, data)
|
||||
document.edit(data, request.user, item)
|
||||
document.save()
|
||||
|
@ -379,8 +383,12 @@ def file(request, id, name=None):
|
|||
def thumbnail(request, id, size=256, page=None):
|
||||
size = int(size)
|
||||
document = get_document_or_404_json(request, id)
|
||||
if "q" in request.GET and page:
|
||||
img = document.highlight_page(page, request.GET["q"], size)
|
||||
return HttpResponse(img, content_type="image/jpeg")
|
||||
return HttpFileResponse(document.thumbnail(size, page=page))
|
||||
|
||||
|
||||
@login_required_json
|
||||
def upload(request):
|
||||
if 'id' in request.GET:
|
||||
|
@ -505,3 +513,37 @@ def autocompleteDocuments(request, data):
|
|||
response['data']['items'] = [i['value'] for i in qs]
|
||||
return render_to_json_response(response)
|
||||
actions.register(autocompleteDocuments)
|
||||
|
||||
|
||||
def document(request, fragment):
|
||||
context = {}
|
||||
parts = fragment.split('/')
|
||||
# FIXME: parse collection urls and return the right metadata for those
|
||||
id = parts[0]
|
||||
page = None
|
||||
crop = None
|
||||
if len(parts) == 2:
|
||||
rect = parts[1].split(',')
|
||||
if len(rect) == 1:
|
||||
page = rect[0]
|
||||
else:
|
||||
crop = rect
|
||||
try:
|
||||
document = models.Document.objects.filter(id=ox.fromAZ(id)).first()
|
||||
except:
|
||||
document = None
|
||||
if document and document.access(request.user):
|
||||
context['title'] = document.data['title']
|
||||
if document.data.get('description'):
|
||||
context['description'] = document.data['description']
|
||||
link = request.build_absolute_uri(document.get_absolute_url())
|
||||
public_id = ox.toAZ(document.id)
|
||||
preview = '/documents/%s/512p.jpg' % public_id
|
||||
if page:
|
||||
preview = '/documents/%s/512p%s.jpg' % (public_id, page)
|
||||
if crop:
|
||||
preview = '/documents/%s/512p%s.jpg' % (public_id, ','.join(crop))
|
||||
context['preview'] = request.build_absolute_uri(preview)
|
||||
context['url'] = request.build_absolute_uri('/documents/' + fragment)
|
||||
context['settings'] = settings
|
||||
return render(request, "document.html", context)
|
||||
|
|
7
pandora/documentcollection/apps.py
Normal file
7
pandora/documentcollection/apps.py
Normal file
|
@ -0,0 +1,7 @@
|
|||
from django.apps import AppConfig
|
||||
|
||||
|
||||
class DocumentcollectionConfig(AppConfig):
|
||||
default_auto_field = "django.db.models.BigAutoField"
|
||||
name = 'documentcollection'
|
||||
|
|
@ -0,0 +1,61 @@
|
|||
# Generated by Django 4.2.3 on 2023-07-27 21:28
|
||||
|
||||
import django.core.serializers.json
|
||||
from django.db import migrations, models
|
||||
import documentcollection.models
|
||||
import oxdjango.fields
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('documentcollection', '0004_jsonfield'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='collection',
|
||||
name='description',
|
||||
field=models.TextField(default=''),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='collection',
|
||||
name='id',
|
||||
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='collection',
|
||||
name='poster_frames',
|
||||
field=oxdjango.fields.JSONField(default=list, editable=False, encoder=django.core.serializers.json.DjangoJSONEncoder),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='collection',
|
||||
name='query',
|
||||
field=oxdjango.fields.JSONField(default=documentcollection.models.default_query, editable=False, encoder=django.core.serializers.json.DjangoJSONEncoder),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='collection',
|
||||
name='sort',
|
||||
field=oxdjango.fields.JSONField(default=documentcollection.models.get_collectionsort, editable=False, encoder=django.core.serializers.json.DjangoJSONEncoder),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='collection',
|
||||
name='status',
|
||||
field=models.CharField(default='private', max_length=20),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='collection',
|
||||
name='type',
|
||||
field=models.CharField(default='static', max_length=255),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='collectiondocument',
|
||||
name='id',
|
||||
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='position',
|
||||
name='id',
|
||||
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
|
||||
),
|
||||
]
|
|
@ -34,6 +34,9 @@ def get_collectionview():
|
|||
def get_collectionsort():
|
||||
return tuple(settings.CONFIG['user']['ui']['collectionSort'])
|
||||
|
||||
def default_query():
|
||||
return {"static": True}
|
||||
|
||||
class Collection(models.Model):
|
||||
|
||||
class Meta:
|
||||
|
@ -46,7 +49,7 @@ class Collection(models.Model):
|
|||
name = models.CharField(max_length=255)
|
||||
status = models.CharField(max_length=20, default='private')
|
||||
_status = ['private', 'public', 'featured']
|
||||
query = JSONField(default=lambda: {"static": True}, editable=False)
|
||||
query = JSONField(default=default_query, editable=False)
|
||||
type = models.CharField(max_length=255, default='static')
|
||||
description = models.TextField(default='')
|
||||
|
||||
|
|
|
@ -86,6 +86,11 @@ def findCollections(request, data):
|
|||
for x in data.get('query', {}).get('conditions', [])
|
||||
)
|
||||
|
||||
is_personal = request.user.is_authenticated and any(
|
||||
(x['key'] == 'user' and x['value'] == request.user.username and x['operator'] == '==')
|
||||
for x in data.get('query', {}).get('conditions', [])
|
||||
)
|
||||
|
||||
if is_section_request:
|
||||
qs = query['qs']
|
||||
if not is_featured and not request.user.is_anonymous:
|
||||
|
@ -94,6 +99,9 @@ def findCollections(request, data):
|
|||
else:
|
||||
qs = _order_query(query['qs'], query['sort'])
|
||||
|
||||
if is_personal and request.user.profile.ui.get('hidden', {}).get('collections'):
|
||||
qs = qs.exclude(name__in=request.user.profile.ui['hidden']['collections'])
|
||||
|
||||
response = json_response()
|
||||
if 'keys' in data:
|
||||
qs = qs[query['range'][0]:query['range'][1]]
|
||||
|
@ -238,7 +246,7 @@ def addCollection(request, data):
|
|||
'type' and 'view'.
|
||||
see: editCollection, findCollections, getCollection, removeCollection, sortCollections
|
||||
'''
|
||||
data['name'] = re.sub(' \[\d+\]$', '', data.get('name', 'Untitled')).strip()
|
||||
data['name'] = re.sub(r' \[\d+\]$', '', data.get('name', 'Untitled')).strip()
|
||||
name = data['name']
|
||||
if not name:
|
||||
name = "Untitled"
|
||||
|
|
7
pandora/edit/apps.py
Normal file
7
pandora/edit/apps.py
Normal file
|
@ -0,0 +1,7 @@
|
|||
from django.apps import AppConfig
|
||||
|
||||
|
||||
class EditConfig(AppConfig):
|
||||
default_auto_field = "django.db.models.BigAutoField"
|
||||
name = 'edit'
|
||||
|
41
pandora/edit/migrations/0006_id_bigint_jsonfield.py
Normal file
41
pandora/edit/migrations/0006_id_bigint_jsonfield.py
Normal file
|
@ -0,0 +1,41 @@
|
|||
# Generated by Django 4.2.3 on 2023-07-27 21:28
|
||||
|
||||
import django.core.serializers.json
|
||||
from django.db import migrations, models
|
||||
import edit.models
|
||||
import oxdjango.fields
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('edit', '0005_jsonfield'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='clip',
|
||||
name='id',
|
||||
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='edit',
|
||||
name='id',
|
||||
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='edit',
|
||||
name='poster_frames',
|
||||
field=oxdjango.fields.JSONField(default=list, editable=False, encoder=django.core.serializers.json.DjangoJSONEncoder),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='edit',
|
||||
name='query',
|
||||
field=oxdjango.fields.JSONField(default=edit.models.default_query, editable=False, encoder=django.core.serializers.json.DjangoJSONEncoder),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='position',
|
||||
name='id',
|
||||
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
|
||||
),
|
||||
]
|
|
@ -13,6 +13,7 @@ from django.conf import settings
|
|||
from django.db import models, transaction
|
||||
from django.db.models import Max
|
||||
from django.contrib.auth import get_user_model
|
||||
from django.core.cache import cache
|
||||
|
||||
from oxdjango.fields import JSONField
|
||||
|
||||
|
@ -24,6 +25,7 @@ import clip.models
|
|||
from archive import extract
|
||||
from user.utils import update_groups
|
||||
from user.models import Group
|
||||
from clip.utils import add_cuts
|
||||
|
||||
from . import managers
|
||||
|
||||
|
@ -33,6 +35,9 @@ User = get_user_model()
|
|||
def get_path(f, x): return f.path(x)
|
||||
def get_icon_path(f, x): return get_path(f, 'icon.jpg')
|
||||
|
||||
def default_query():
|
||||
return {"static": True}
|
||||
|
||||
class Edit(models.Model):
|
||||
|
||||
class Meta:
|
||||
|
@ -51,7 +56,7 @@ class Edit(models.Model):
|
|||
description = models.TextField(default='')
|
||||
rightslevel = models.IntegerField(db_index=True, default=0)
|
||||
|
||||
query = JSONField(default=lambda: {"static": True}, editable=False)
|
||||
query = JSONField(default=default_query, editable=False)
|
||||
type = models.CharField(max_length=255, default='static')
|
||||
|
||||
icon = models.ImageField(default=None, blank=True, null=True, upload_to=get_icon_path)
|
||||
|
@ -93,6 +98,8 @@ class Edit(models.Model):
|
|||
# dont add clip if in/out are invalid
|
||||
if not c.annotation:
|
||||
duration = c.item.sort.duration
|
||||
if c.start is None or c.end is None:
|
||||
return False
|
||||
if c.start > c.end \
|
||||
or round(c.start, 3) >= round(duration, 3) \
|
||||
or round(c.end, 3) > round(duration, 3):
|
||||
|
@ -507,7 +514,7 @@ class Clip(models.Model):
|
|||
if value:
|
||||
data[key] = value
|
||||
data['duration'] = data['out'] - data['in']
|
||||
data['cuts'] = tuple([c for c in self.item.get('cuts', []) if c > self.start and c < self.end])
|
||||
add_cuts(data, self.item, self.start, self.end)
|
||||
data['layers'] = self.get_layers(user)
|
||||
data['streams'] = [s.file.oshash for s in self.item.streams()]
|
||||
return data
|
||||
|
|
|
@ -3,14 +3,16 @@
|
|||
import os
|
||||
import re
|
||||
|
||||
import ox
|
||||
from oxdjango.api import actions
|
||||
from oxdjango.decorators import login_required_json
|
||||
from oxdjango.http import HttpFileResponse
|
||||
from oxdjango.shortcuts import render_to_json_response, get_object_or_404_json, json_response
|
||||
import ox
|
||||
|
||||
from django.conf import settings
|
||||
from django.db import transaction
|
||||
from django.db.models import Max
|
||||
from oxdjango.http import HttpFileResponse
|
||||
from oxdjango.api import actions
|
||||
from django.conf import settings
|
||||
from django.db.models import Sum
|
||||
|
||||
from item import utils
|
||||
from changelog.models import add_changelog
|
||||
|
@ -190,7 +192,7 @@ def _order_clips(edit, sort):
|
|||
'in': 'start',
|
||||
'out': 'end',
|
||||
'text': 'sortvalue',
|
||||
'volume': 'sortvolume',
|
||||
'volume': 'volume' if edit.type == 'smart' else 'sortvolume',
|
||||
'item__sort__item': 'item__sort__public_id',
|
||||
}.get(key, key)
|
||||
order = '%s%s' % (operator, key)
|
||||
|
@ -260,7 +262,7 @@ def addEdit(request, data):
|
|||
}
|
||||
see: editEdit, findEdit, getEdit, removeEdit, sortEdits
|
||||
'''
|
||||
data['name'] = re.sub(' \[\d+\]$', '', data.get('name', 'Untitled')).strip()
|
||||
data['name'] = re.sub(r' \[\d+\]$', '', data.get('name', 'Untitled')).strip()
|
||||
name = data['name']
|
||||
if not name:
|
||||
name = "Untitled"
|
||||
|
@ -412,6 +414,11 @@ def findEdits(request, data):
|
|||
|
||||
is_featured = any(filter(is_featured_condition, data.get('query', {}).get('conditions', [])))
|
||||
|
||||
is_personal = request.user.is_authenticated and any(
|
||||
(x['key'] == 'user' and x['value'] == request.user.username and x['operator'] == '==')
|
||||
for x in data.get('query', {}).get('conditions', [])
|
||||
)
|
||||
|
||||
if is_section_request:
|
||||
qs = query['qs']
|
||||
if not is_featured and not request.user.is_anonymous:
|
||||
|
@ -420,6 +427,9 @@ def findEdits(request, data):
|
|||
else:
|
||||
qs = _order_query(query['qs'], query['sort'])
|
||||
|
||||
if is_personal and request.user.profile.ui.get('hidden', {}).get('edits'):
|
||||
qs = qs.exclude(name__in=request.user.profile.ui['hidden']['edits'])
|
||||
|
||||
response = json_response()
|
||||
if 'keys' in data:
|
||||
qs = qs[query['range'][0]:query['range'][1]]
|
||||
|
|
3
pandora/encoding.conf.in
Normal file
3
pandora/encoding.conf.in
Normal file
|
@ -0,0 +1,3 @@
|
|||
LOGLEVEL=info
|
||||
MAX_TASKS_PER_CHILD=500
|
||||
CONCURRENCY=1
|
6
pandora/entity/apps.py
Normal file
6
pandora/entity/apps.py
Normal file
|
@ -0,0 +1,6 @@
|
|||
from django.apps import AppConfig
|
||||
|
||||
|
||||
class EntityConfig(AppConfig):
|
||||
default_auto_field = "django.db.models.BigAutoField"
|
||||
name = 'entity'
|
|
@ -0,0 +1,50 @@
|
|||
# Generated by Django 4.2.3 on 2023-07-27 21:28
|
||||
|
||||
import django.core.serializers.json
|
||||
from django.db import migrations, models
|
||||
import oxdjango.fields
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('entity', '0006_auto_20180918_0903'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='documentproperties',
|
||||
name='data',
|
||||
field=oxdjango.fields.JSONField(default=dict, editable=False, encoder=django.core.serializers.json.DjangoJSONEncoder),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='documentproperties',
|
||||
name='id',
|
||||
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='entity',
|
||||
name='data',
|
||||
field=oxdjango.fields.JSONField(default=dict, editable=False, encoder=django.core.serializers.json.DjangoJSONEncoder),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='entity',
|
||||
name='id',
|
||||
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='entity',
|
||||
name='name_find',
|
||||
field=models.TextField(default=''),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='find',
|
||||
name='id',
|
||||
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='link',
|
||||
name='id',
|
||||
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
|
||||
),
|
||||
]
|
6
pandora/event/apps.py
Normal file
6
pandora/event/apps.py
Normal file
|
@ -0,0 +1,6 @@
|
|||
from django.apps import AppConfig
|
||||
|
||||
|
||||
class EventConfig(AppConfig):
|
||||
default_auto_field = "django.db.models.BigAutoField"
|
||||
name = 'event'
|
|
@ -0,0 +1,43 @@
|
|||
# Generated by Django 4.2.3 on 2023-07-27 21:28
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('event', '0003_auto_20160304_1644'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='event',
|
||||
name='duration',
|
||||
field=models.CharField(default='', max_length=255),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='event',
|
||||
name='end',
|
||||
field=models.CharField(default='', max_length=255),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='event',
|
||||
name='id',
|
||||
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='event',
|
||||
name='name_find',
|
||||
field=models.TextField(default=''),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='event',
|
||||
name='start',
|
||||
field=models.CharField(default='', max_length=255),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='event',
|
||||
name='type',
|
||||
field=models.CharField(default='', max_length=255),
|
||||
),
|
||||
]
|
|
@ -1,20 +1,26 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
from celery.task import task
|
||||
from app.celery import app
|
||||
|
||||
from .models import Event
|
||||
|
||||
|
||||
'''
|
||||
@periodic_task(run_every=crontab(hour=7, minute=30), queue='encoding')
|
||||
from celery.schedules import crontab
|
||||
|
||||
@app.task(ignore_results=True, queue='encoding')
|
||||
def update_all_matches(**kwargs):
|
||||
ids = [e['id'] for e in Event.objects.all().values('id')]
|
||||
for i in ids:
|
||||
e = Event.objects.get(pk=i)
|
||||
e.update_matches()
|
||||
|
||||
@app.on_after_finalize.connect
|
||||
def setup_periodic_tasks(sender, **kwargs):
|
||||
sender.add_periodic_task(crontab(hour=7, minute=30), update_all_matches.s())
|
||||
'''
|
||||
|
||||
@task(ignore_results=True, queue='default')
|
||||
@app.task(ignore_results=True, queue='default')
|
||||
def update_matches(eventId):
|
||||
event = Event.objects.get(pk=eventId)
|
||||
event.update_matches()
|
||||
|
|
|
@ -2,4 +2,5 @@ from django.apps import AppConfig
|
|||
|
||||
|
||||
class HomeConfig(AppConfig):
|
||||
default_auto_field = "django.db.models.BigAutoField"
|
||||
name = 'home'
|
||||
|
|
|
@ -0,0 +1,30 @@
|
|||
# Generated by Django 4.2.3 on 2023-07-27 21:28
|
||||
|
||||
import django.core.serializers.json
|
||||
from django.db import migrations, models
|
||||
import oxdjango.fields
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('home', '0002_jsonfield'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='item',
|
||||
name='data',
|
||||
field=oxdjango.fields.JSONField(default=dict, editable=False, encoder=django.core.serializers.json.DjangoJSONEncoder),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='item',
|
||||
name='id',
|
||||
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='item',
|
||||
name='index',
|
||||
field=models.IntegerField(default=-1),
|
||||
),
|
||||
]
|
6
pandora/item/apps.py
Normal file
6
pandora/item/apps.py
Normal file
|
@ -0,0 +1,6 @@
|
|||
from django.apps import AppConfig
|
||||
|
||||
|
||||
class ItemConfig(AppConfig):
|
||||
default_auto_field = "django.db.models.BigAutoField"
|
||||
name = 'item'
|
|
@ -4,7 +4,6 @@ from django.core.management.base import BaseCommand
|
|||
from django.conf import settings
|
||||
from django.db import transaction
|
||||
|
||||
settings.RELOAD_CONFIG = False
|
||||
import app.monkey_patch
|
||||
from ... import models
|
||||
|
||||
|
|
|
@ -6,7 +6,6 @@ from django.db import connection, transaction
|
|||
from django.db.models import fields
|
||||
from django.conf import settings
|
||||
|
||||
settings.RELOAD_CONFIG = False
|
||||
import app.monkey_patch
|
||||
from ... import models
|
||||
import clip.models
|
||||
|
|
|
@ -5,7 +5,6 @@ from django.db import connection, transaction
|
|||
from django.db.models import fields
|
||||
from django.conf import settings
|
||||
|
||||
settings.RELOAD_CONFIG = False
|
||||
import app.monkey_patch
|
||||
from ... import models
|
||||
import clip.models
|
||||
|
|
|
@ -5,7 +5,6 @@ from django.db import connection, transaction
|
|||
from django.db.models import fields
|
||||
from django.conf import settings
|
||||
|
||||
settings.RELOAD_CONFIG = False
|
||||
import app.monkey_patch
|
||||
from ... import models
|
||||
import clip.models
|
||||
|
|
29
pandora/item/management/commands/rebuild_posters.py
Normal file
29
pandora/item/management/commands/rebuild_posters.py
Normal file
|
@ -0,0 +1,29 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
import os
|
||||
from glob import glob
|
||||
|
||||
from django.core.management.base import BaseCommand
|
||||
|
||||
import app.monkey_patch
|
||||
from ... import models
|
||||
from ... import tasks
|
||||
|
||||
class Command(BaseCommand):
|
||||
"""
|
||||
rebuild posters for all items.
|
||||
"""
|
||||
help = 'rebuild all posters for all items.'
|
||||
args = ''
|
||||
|
||||
def handle(self, **options):
|
||||
offset = 0
|
||||
chunk = 100
|
||||
count = models.Item.objects.count()
|
||||
while offset <= count:
|
||||
for i in models.Item.objects.all().order_by('id')[offset:offset+chunk]:
|
||||
print(i)
|
||||
if i.poster:
|
||||
i.poster.delete()
|
||||
i.make_poster()
|
||||
offset += chunk
|
|
@ -6,7 +6,6 @@ from django.db import connection, transaction
|
|||
from django.db.models import fields
|
||||
from django.conf import settings
|
||||
|
||||
settings.RELOAD_CONFIG = False
|
||||
import app.monkey_patch
|
||||
from ... import models
|
||||
|
||||
|
|
|
@ -5,7 +5,6 @@ from django.core.management.base import BaseCommand
|
|||
from django.db import connection, transaction
|
||||
from django.conf import settings
|
||||
|
||||
settings.RELOAD_CONFIG = False
|
||||
import app.monkey_patch
|
||||
from ... import models
|
||||
|
||||
|
|
|
@ -5,7 +5,6 @@ from django.db import connection, transaction
|
|||
from django.db.models import fields
|
||||
from django.conf import settings
|
||||
|
||||
settings.RELOAD_CONFIG = False
|
||||
import app.monkey_patch
|
||||
from ... import models
|
||||
import clip.models
|
||||
|
|
|
@ -33,7 +33,7 @@ def parseCondition(condition, user, owner=None):
|
|||
k = {'id': 'public_id'}.get(k, k)
|
||||
if not k:
|
||||
k = '*'
|
||||
v = condition['value']
|
||||
v = condition.get('value', '')
|
||||
op = condition.get('operator')
|
||||
if not op:
|
||||
op = '='
|
||||
|
@ -62,6 +62,9 @@ def parseCondition(condition, user, owner=None):
|
|||
if k == 'list':
|
||||
key_type = ''
|
||||
|
||||
if k in ('width', 'height'):
|
||||
key_type = 'integer'
|
||||
|
||||
if k == 'groups':
|
||||
if op == '==' and v == '$my':
|
||||
if not owner:
|
||||
|
@ -86,8 +89,11 @@ def parseCondition(condition, user, owner=None):
|
|||
elif k == 'rendered':
|
||||
return Q(rendered=v)
|
||||
elif k == 'resolution':
|
||||
if isinstance(v, list) and len(v) == 2:
|
||||
q = parseCondition({'key': 'width', 'value': v[0], 'operator': op}, user) \
|
||||
& parseCondition({'key': 'height', 'value': v[1], 'operator': op}, user)
|
||||
else:
|
||||
q = Q(id=0)
|
||||
if exclude:
|
||||
q = ~q
|
||||
return q
|
||||
|
@ -318,6 +324,8 @@ class ItemManager(Manager):
|
|||
q |= Q(groups__in=user.groups.all())
|
||||
rendered_q |= Q(groups__in=user.groups.all())
|
||||
qs = qs.filter(q)
|
||||
max_level = len(settings.CONFIG['rightsLevels'])
|
||||
qs = qs.filter(level__lte=max_level)
|
||||
if settings.CONFIG.get('itemRequiresVideo') and level != 'admin':
|
||||
qs = qs.filter(rendered_q)
|
||||
return qs
|
||||
|
|
|
@ -71,7 +71,7 @@ class Migration(migrations.Migration):
|
|||
('poster_width', models.IntegerField(default=0)),
|
||||
('poster_frame', models.FloatField(default=-1)),
|
||||
('icon', models.ImageField(blank=True, default=None, upload_to=item.models.get_icon_path)),
|
||||
('torrent', models.FileField(blank=True, default=None, max_length=1000, upload_to=item.models.get_torrent_path)),
|
||||
('torrent', models.FileField(blank=True, default=None, max_length=1000)),
|
||||
('stream_info', oxdjango.fields.DictField(default={}, editable=False)),
|
||||
('stream_aspect', models.FloatField(default=1.3333333333333333)),
|
||||
],
|
||||
|
|
19
pandora/item/migrations/0005_auto_20230710_0852.py
Normal file
19
pandora/item/migrations/0005_auto_20230710_0852.py
Normal file
|
@ -0,0 +1,19 @@
|
|||
# Generated by Django 3.0.10 on 2023-07-10 08:52
|
||||
|
||||
import django.core.serializers.json
|
||||
from django.db import migrations, models
|
||||
import oxdjango.fields
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('item', '0004_json_cache'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RemoveField(
|
||||
model_name='item',
|
||||
name='torrent',
|
||||
),
|
||||
]
|
65
pandora/item/migrations/0006_id_bigint_jsonfied_and_more.py
Normal file
65
pandora/item/migrations/0006_id_bigint_jsonfied_and_more.py
Normal file
|
@ -0,0 +1,65 @@
|
|||
# Generated by Django 4.2.3 on 2023-07-27 21:28
|
||||
|
||||
import django.core.serializers.json
|
||||
from django.db import migrations, models
|
||||
import oxdjango.fields
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('item', '0005_auto_20230710_0852'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='access',
|
||||
name='id',
|
||||
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='annotationsequence',
|
||||
name='id',
|
||||
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='description',
|
||||
name='id',
|
||||
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='facet',
|
||||
name='id',
|
||||
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='item',
|
||||
name='cache',
|
||||
field=oxdjango.fields.JSONField(default=dict, editable=False, encoder=django.core.serializers.json.DjangoJSONEncoder),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='item',
|
||||
name='data',
|
||||
field=oxdjango.fields.JSONField(default=dict, editable=False, encoder=django.core.serializers.json.DjangoJSONEncoder),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='item',
|
||||
name='external_data',
|
||||
field=oxdjango.fields.JSONField(default=dict, editable=False, encoder=django.core.serializers.json.DjangoJSONEncoder),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='item',
|
||||
name='id',
|
||||
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='item',
|
||||
name='stream_info',
|
||||
field=oxdjango.fields.JSONField(default=dict, editable=False, encoder=django.core.serializers.json.DjangoJSONEncoder),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='itemfind',
|
||||
name='id',
|
||||
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
|
||||
),
|
||||
]
|
|
@ -1,6 +1,7 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
|
@ -42,6 +43,7 @@ from user.utils import update_groups
|
|||
from user.models import Group
|
||||
import archive.models
|
||||
|
||||
logger = logging.getLogger('pandora.' + __name__)
|
||||
User = get_user_model()
|
||||
|
||||
|
||||
|
@ -155,9 +157,6 @@ def get_icon_path(f, x):
|
|||
def get_poster_path(f, x):
|
||||
return get_path(f, 'poster.jpg')
|
||||
|
||||
def get_torrent_path(f, x):
|
||||
return get_path(f, 'torrent.torrent')
|
||||
|
||||
class Item(models.Model):
|
||||
created = models.DateTimeField(auto_now_add=True)
|
||||
modified = models.DateTimeField(auto_now=True)
|
||||
|
@ -183,7 +182,6 @@ class Item(models.Model):
|
|||
|
||||
icon = models.ImageField(default=None, blank=True, upload_to=get_icon_path)
|
||||
|
||||
torrent = models.FileField(default=None, blank=True, max_length=1000, upload_to=get_torrent_path)
|
||||
stream_info = JSONField(default=dict, editable=False)
|
||||
|
||||
# stream related fields
|
||||
|
@ -231,6 +229,9 @@ class Item(models.Model):
|
|||
def editable(self, user):
|
||||
if user.is_anonymous:
|
||||
return False
|
||||
max_level = len(settings.CONFIG['rightsLevels'])
|
||||
if self.level > max_level:
|
||||
return False
|
||||
if user.profile.capability('canEditMetadata') or \
|
||||
user.is_staff or \
|
||||
self.user == user or \
|
||||
|
@ -238,7 +239,7 @@ class Item(models.Model):
|
|||
return True
|
||||
return False
|
||||
|
||||
def edit(self, data):
|
||||
def edit(self, data, is_task=False):
|
||||
data = data.copy()
|
||||
# FIXME: how to map the keys to the right place to write them to?
|
||||
if 'id' in data:
|
||||
|
@ -255,11 +256,12 @@ class Item(models.Model):
|
|||
description = data.pop(key)
|
||||
if isinstance(description, dict):
|
||||
for value in description:
|
||||
value = ox.sanitize_html(value)
|
||||
d, created = Description.objects.get_or_create(key=k, value=value)
|
||||
d.description = ox.sanitize_html(description[value])
|
||||
d.save()
|
||||
else:
|
||||
value = data.get(k, self.get(k, ''))
|
||||
value = ox.sanitize_html(data.get(k, self.get(k, '')))
|
||||
if not description:
|
||||
description = ''
|
||||
d, created = Description.objects.get_or_create(key=k, value=value)
|
||||
|
@ -294,6 +296,9 @@ class Item(models.Model):
|
|||
self.data[key] = ox.escape_html(data[key])
|
||||
p = self.save()
|
||||
if not settings.USE_IMDB and list(filter(lambda k: k in self.poster_keys, data)):
|
||||
if is_task:
|
||||
tasks.update_poster(self.public_id)
|
||||
else:
|
||||
p = tasks.update_poster.delay(self.public_id)
|
||||
return p
|
||||
|
||||
|
@ -473,6 +478,7 @@ class Item(models.Model):
|
|||
|
||||
for a in self.annotations.all().order_by('id'):
|
||||
a.item = other
|
||||
with transaction.atomic():
|
||||
a.set_public_id()
|
||||
Annotation.objects.filter(id=a.id).update(item=other, public_id=a.public_id)
|
||||
try:
|
||||
|
@ -517,6 +523,7 @@ class Item(models.Model):
|
|||
cmd, stdout=open('/dev/null', 'w'), stderr=open('/dev/null', 'w'), close_fds=True)
|
||||
p.wait()
|
||||
os.unlink(tmp_output_txt)
|
||||
os.close(fd)
|
||||
return True
|
||||
else:
|
||||
return None
|
||||
|
@ -634,11 +641,11 @@ class Item(models.Model):
|
|||
if self.poster_height:
|
||||
i['posterRatio'] = self.poster_width / self.poster_height
|
||||
|
||||
if keys and 'source' in keys:
|
||||
i['source'] = self.streams().exclude(file__data='').exists()
|
||||
if keys and 'hasSource' in keys:
|
||||
i['hasSource'] = self.streams().exclude(file__data='').exists()
|
||||
|
||||
streams = self.streams()
|
||||
i['durations'] = [s.duration for s in streams]
|
||||
i['durations'] = [s[0] for s in streams.values_list('duration')]
|
||||
i['duration'] = sum(i['durations'])
|
||||
i['audioTracks'] = self.audio_tracks()
|
||||
if not i['audioTracks']:
|
||||
|
@ -694,10 +701,12 @@ class Item(models.Model):
|
|||
else:
|
||||
values = self.get(key)
|
||||
if values:
|
||||
values = [ox.sanitize_html(value) for value in values]
|
||||
for d in Description.objects.filter(key=key, value__in=values):
|
||||
i['%sdescription' % key][d.value] = d.description
|
||||
else:
|
||||
qs = Description.objects.filter(key=key, value=self.get(key, ''))
|
||||
value = ox.sanitize_html(self.get(key, ''))
|
||||
qs = Description.objects.filter(key=key, value=value)
|
||||
i['%sdescription' % key] = '' if qs.count() == 0 else qs[0].description
|
||||
if keys:
|
||||
info = {}
|
||||
|
@ -855,7 +864,7 @@ class Item(models.Model):
|
|||
values = list(set(values))
|
||||
else:
|
||||
values = self.get(key, '')
|
||||
if isinstance(values, list):
|
||||
if values and isinstance(values, list) and isinstance(values[0], str):
|
||||
save(key, '\n'.join(values))
|
||||
else:
|
||||
save(key, values)
|
||||
|
@ -1017,12 +1026,16 @@ class Item(models.Model):
|
|||
set_value(s, name, value)
|
||||
elif sort_type == 'person':
|
||||
value = sortNames(self.get(source, []))
|
||||
if value is None:
|
||||
value = ''
|
||||
value = utils.sort_string(value)[:955]
|
||||
set_value(s, name, value)
|
||||
elif sort_type == 'string':
|
||||
value = self.get(source, '')
|
||||
if value is None:
|
||||
value = ''
|
||||
if isinstance(value, list):
|
||||
value = ','.join(value)
|
||||
value = ','.join([str(v) for v in value])
|
||||
value = utils.sort_string(value)[:955]
|
||||
set_value(s, name, value)
|
||||
elif sort_type == 'words':
|
||||
|
@ -1099,7 +1112,11 @@ class Item(models.Model):
|
|||
_current_values.append(value[0])
|
||||
current_values = _current_values
|
||||
|
||||
try:
|
||||
current_values = list(set(current_values))
|
||||
except:
|
||||
logger.error('invalid facet data for %s: %s', key, current_values)
|
||||
current_values = []
|
||||
current_values = [ox.decode_html(ox.strip_tags(v)) for v in current_values]
|
||||
current_values = [unicodedata.normalize('NFKD', v) for v in current_values]
|
||||
self.update_facet_values(key, current_values)
|
||||
|
@ -1192,7 +1209,7 @@ class Item(models.Model):
|
|||
if not r:
|
||||
return False
|
||||
path = video.name
|
||||
duration = sum(item.cache['durations'])
|
||||
duration = sum(self.item.cache['durations'])
|
||||
else:
|
||||
path = stream.media.path
|
||||
duration = stream.info['duration']
|
||||
|
@ -1288,90 +1305,6 @@ class Item(models.Model):
|
|||
self.files.filter(selected=True).update(selected=False)
|
||||
self.save()
|
||||
|
||||
def get_torrent(self, request):
|
||||
if self.torrent:
|
||||
self.torrent.seek(0)
|
||||
data = ox.torrent.bdecode(self.torrent.read())
|
||||
url = request.build_absolute_uri("%s/torrent/" % self.get_absolute_url())
|
||||
if url.startswith('https://'):
|
||||
url = 'http' + url[5:]
|
||||
data['url-list'] = ['%s%s' % (url, u.split('torrent/')[1]) for u in data['url-list']]
|
||||
return ox.torrent.bencode(data)
|
||||
|
||||
def make_torrent(self):
|
||||
if not settings.CONFIG['video'].get('torrent'):
|
||||
return
|
||||
streams = self.streams()
|
||||
if streams.count() == 0:
|
||||
return
|
||||
base = self.path('torrent')
|
||||
base = os.path.abspath(os.path.join(settings.MEDIA_ROOT, base))
|
||||
if not isinstance(base, bytes):
|
||||
base = base.encode('utf-8')
|
||||
if os.path.exists(base):
|
||||
shutil.rmtree(base)
|
||||
ox.makedirs(base)
|
||||
|
||||
filename = utils.safe_filename(ox.decode_html(self.get('title')))
|
||||
base = self.path('torrent/%s' % filename)
|
||||
base = os.path.abspath(os.path.join(settings.MEDIA_ROOT, base))
|
||||
size = 0
|
||||
duration = 0.0
|
||||
if streams.count() == 1:
|
||||
v = streams[0]
|
||||
media_path = v.media.path
|
||||
extension = media_path.split('.')[-1]
|
||||
url = "%s/torrent/%s.%s" % (self.get_absolute_url(),
|
||||
quote(filename.encode('utf-8')),
|
||||
extension)
|
||||
video = "%s.%s" % (base, extension)
|
||||
if not isinstance(media_path, bytes):
|
||||
media_path = media_path.encode('utf-8')
|
||||
if not isinstance(video, bytes):
|
||||
video = video.encode('utf-8')
|
||||
media_path = os.path.relpath(media_path, os.path.dirname(video))
|
||||
os.symlink(media_path, video)
|
||||
size = v.media.size
|
||||
duration = v.duration
|
||||
else:
|
||||
url = "%s/torrent/" % self.get_absolute_url()
|
||||
part = 1
|
||||
ox.makedirs(base)
|
||||
for v in streams:
|
||||
media_path = v.media.path
|
||||
extension = media_path.split('.')[-1]
|
||||
video = "%s/%s.Part %d.%s" % (base, filename, part, extension)
|
||||
part += 1
|
||||
if not isinstance(media_path, bytes):
|
||||
media_path = media_path.encode('utf-8')
|
||||
if not isinstance(video, bytes):
|
||||
video = video.encode('utf-8')
|
||||
media_path = os.path.relpath(media_path, os.path.dirname(video))
|
||||
os.symlink(media_path, video)
|
||||
size += v.media.size
|
||||
duration += v.duration
|
||||
video = base
|
||||
|
||||
torrent = '%s.torrent' % base
|
||||
url = "http://%s%s" % (settings.CONFIG['site']['url'], url)
|
||||
meta = {
|
||||
'filesystem_encoding': 'utf-8',
|
||||
'target': torrent,
|
||||
'url-list': url,
|
||||
}
|
||||
if duration:
|
||||
meta['playtime'] = ox.format_duration(duration*1000)[:-4]
|
||||
|
||||
# slightly bigger torrent file but better for streaming
|
||||
piece_size_pow2 = 15 # 1 mbps -> 32KB pieces
|
||||
if size / duration >= 1000000:
|
||||
piece_size_pow2 = 16 # 2 mbps -> 64KB pieces
|
||||
meta['piece_size_pow2'] = piece_size_pow2
|
||||
|
||||
ox.torrent.create_torrent(video, settings.TRACKER_URL, meta)
|
||||
self.torrent.name = torrent[len(settings.MEDIA_ROOT)+1:]
|
||||
self.save()
|
||||
|
||||
def audio_tracks(self):
|
||||
tracks = [f['language']
|
||||
for f in self.files.filter(selected=True).filter(Q(is_video=True) | Q(is_audio=True)).values('language')
|
||||
|
@ -1379,11 +1312,10 @@ class Item(models.Model):
|
|||
return sorted(set(tracks))
|
||||
|
||||
def streams(self, track=None):
|
||||
files = self.files.filter(selected=True).filter(Q(is_audio=True) | Q(is_video=True))
|
||||
qs = archive.models.Stream.objects.filter(
|
||||
source=None, available=True, file__item=self, file__selected=True
|
||||
).filter(
|
||||
Q(file__is_audio=True) | Q(file__is_video=True)
|
||||
)
|
||||
file__in=files, source=None, available=True
|
||||
).select_related()
|
||||
if not track:
|
||||
tracks = self.audio_tracks()
|
||||
if len(tracks) > 1:
|
||||
|
@ -1422,7 +1354,6 @@ class Item(models.Model):
|
|||
self.select_frame()
|
||||
self.make_poster()
|
||||
self.make_icon()
|
||||
self.make_torrent()
|
||||
self.rendered = streams.count() > 0
|
||||
self.save()
|
||||
if self.rendered:
|
||||
|
@ -1608,6 +1539,13 @@ class Item(models.Model):
|
|||
cmd += ['-l', timeline]
|
||||
if frame:
|
||||
cmd += ['-f', frame]
|
||||
if settings.ITEM_ICON_DATA:
|
||||
cmd += '-d', '-'
|
||||
data = self.json()
|
||||
data = utils.normalize_dict('NFC', data)
|
||||
p = subprocess.Popen(cmd, stdin=subprocess.PIPE, close_fds=True)
|
||||
p.communicate(json.dumps(data, default=to_json).encode('utf-8'))
|
||||
else:
|
||||
p = subprocess.Popen(cmd, close_fds=True)
|
||||
p.wait()
|
||||
# remove cached versions
|
||||
|
@ -1621,11 +1559,13 @@ class Item(models.Model):
|
|||
return icon
|
||||
|
||||
def add_empty_clips(self):
|
||||
if not settings.EMPTY_CLIPS:
|
||||
return
|
||||
subtitles = utils.get_by_key(settings.CONFIG['layers'], 'isSubtitles', True)
|
||||
if not subtitles:
|
||||
return
|
||||
# otherwise add empty 5 seconds annotation every minute
|
||||
duration = sum([s.duration for s in self.streams()])
|
||||
duration = sum([s[0] for s in self.streams().values_list('duration')])
|
||||
layer = subtitles['id']
|
||||
# FIXME: allow annotations from no user instead?
|
||||
user = User.objects.all().order_by('id')[0]
|
||||
|
@ -1874,6 +1814,8 @@ class Description(models.Model):
|
|||
value = models.CharField(max_length=1000, db_index=True)
|
||||
description = models.TextField()
|
||||
|
||||
def __str__(self):
|
||||
return "%s=%s" % (self.key, self.value)
|
||||
|
||||
class AnnotationSequence(models.Model):
|
||||
item = models.OneToOneField('Item', related_name='_annotation_sequence', on_delete=models.CASCADE)
|
||||
|
@ -1889,7 +1831,6 @@ class AnnotationSequence(models.Model):
|
|||
|
||||
@classmethod
|
||||
def nextid(cls, item):
|
||||
with transaction.atomic():
|
||||
s, created = cls.objects.get_or_create(item=item)
|
||||
if created:
|
||||
nextid = s.value
|
||||
|
|
|
@ -24,10 +24,6 @@ urls = [
|
|||
re_path(r'^(?P<id>[A-Z0-9].*)/(?P<resolution>\d+)p(?P<index>\d*)\.(?P<format>webm|ogv|mp4)$', views.video),
|
||||
re_path(r'^(?P<id>[A-Z0-9].*)/(?P<resolution>\d+)p(?P<index>\d*)\.(?P<track>.+)\.(?P<format>webm|ogv|mp4)$', views.video),
|
||||
|
||||
#torrent
|
||||
re_path(r'^(?P<id>[A-Z0-9].*)/torrent$', views.torrent),
|
||||
re_path(r'^(?P<id>[A-Z0-9].*)/torrent/(?P<filename>.*?)$', views.torrent),
|
||||
|
||||
#export
|
||||
re_path(r'^(?P<id>[A-Z0-9].*)/json$', views.item_json),
|
||||
re_path(r'^(?P<id>[A-Z0-9].*)/xml$', views.item_xml),
|
||||
|
|
|
@ -2,27 +2,35 @@
|
|||
|
||||
from datetime import timedelta, datetime
|
||||
from urllib.parse import quote
|
||||
import xml.etree.ElementTree as ET
|
||||
import gzip
|
||||
import os
|
||||
import random
|
||||
import logging
|
||||
|
||||
from celery.task import task, periodic_task
|
||||
from app.celery import app
|
||||
from celery.schedules import crontab
|
||||
from django.conf import settings
|
||||
from django.db import connection, transaction
|
||||
from django.db.models import Q
|
||||
from ox.utils import ET
|
||||
|
||||
from app.utils import limit_rate
|
||||
from taskqueue.models import Task
|
||||
|
||||
|
||||
@periodic_task(run_every=timedelta(days=1), queue='encoding')
|
||||
logger = logging.getLogger('pandora.' + __name__)
|
||||
|
||||
@app.task(queue='encoding')
|
||||
def cronjob(**kwargs):
|
||||
if limit_rate('item.tasks.cronjob', 8 * 60 * 60):
|
||||
update_random_sort()
|
||||
update_random_clip_sort()
|
||||
clear_cache.delay()
|
||||
|
||||
@app.on_after_finalize.connect
|
||||
def setup_periodic_tasks(sender, **kwargs):
|
||||
sender.add_periodic_task(timedelta(days=1), cronjob.s())
|
||||
|
||||
def update_random_sort():
|
||||
from . import models
|
||||
if list(filter(lambda f: f['id'] == 'random', settings.CONFIG['itemKeys'])):
|
||||
|
@ -50,7 +58,7 @@ def update_random_clip_sort():
|
|||
cursor.execute(row)
|
||||
|
||||
|
||||
@task(ignore_results=True, queue='default')
|
||||
@app.task(ignore_results=True, queue='default')
|
||||
def update_clips(public_id):
|
||||
from . import models
|
||||
try:
|
||||
|
@ -59,7 +67,7 @@ def update_clips(public_id):
|
|||
return
|
||||
item.clips.all().update(user=item.user.id)
|
||||
|
||||
@task(ignore_results=True, queue='default')
|
||||
@app.task(ignore_results=True, queue='default')
|
||||
def update_poster(public_id):
|
||||
from . import models
|
||||
try:
|
||||
|
@ -77,7 +85,7 @@ def update_poster(public_id):
|
|||
icon=item.icon.name
|
||||
)
|
||||
|
||||
@task(ignore_results=True, queue='default')
|
||||
@app.task(ignore_results=True, queue='default')
|
||||
def update_file_paths(public_id):
|
||||
from . import models
|
||||
try:
|
||||
|
@ -86,7 +94,7 @@ def update_file_paths(public_id):
|
|||
return
|
||||
item.update_file_paths()
|
||||
|
||||
@task(ignore_results=True, queue='default')
|
||||
@app.task(ignore_results=True, queue='default')
|
||||
def update_external(public_id):
|
||||
from . import models
|
||||
try:
|
||||
|
@ -95,7 +103,7 @@ def update_external(public_id):
|
|||
return
|
||||
item.update_external()
|
||||
|
||||
@task(queue="encoding")
|
||||
@app.task(queue="encoding")
|
||||
def update_timeline(public_id):
|
||||
from . import models
|
||||
try:
|
||||
|
@ -105,7 +113,7 @@ def update_timeline(public_id):
|
|||
item.update_timeline(async_=False)
|
||||
Task.finish(item)
|
||||
|
||||
@task(queue="encoding")
|
||||
@app.task(queue="encoding")
|
||||
def rebuild_timeline(public_id):
|
||||
from . import models
|
||||
i = models.Item.objects.get(public_id=public_id)
|
||||
|
@ -113,7 +121,7 @@ def rebuild_timeline(public_id):
|
|||
s.make_timeline()
|
||||
i.update_timeline(async_=False)
|
||||
|
||||
@task(queue="encoding")
|
||||
@app.task(queue="encoding")
|
||||
def load_subtitles(public_id):
|
||||
from . import models
|
||||
try:
|
||||
|
@ -126,7 +134,7 @@ def load_subtitles(public_id):
|
|||
item.update_facets()
|
||||
|
||||
|
||||
@task(queue="encoding")
|
||||
@app.task(queue="encoding")
|
||||
def extract_clip(public_id, in_, out, resolution, format, track=None):
|
||||
from . import models
|
||||
try:
|
||||
|
@ -138,7 +146,7 @@ def extract_clip(public_id, in_, out, resolution, format, track=None):
|
|||
return False
|
||||
|
||||
|
||||
@task(queue="encoding")
|
||||
@app.task(queue="encoding")
|
||||
def clear_cache(days=60):
|
||||
import subprocess
|
||||
path = os.path.join(settings.MEDIA_ROOT, 'media')
|
||||
|
@ -152,7 +160,7 @@ def clear_cache(days=60):
|
|||
subprocess.check_output(cmd)
|
||||
|
||||
|
||||
@task(ignore_results=True, queue='default')
|
||||
@app.task(ignore_results=True, queue='default')
|
||||
def update_sitemap(base_url):
|
||||
from . import models
|
||||
sitemap = os.path.abspath(os.path.join(settings.MEDIA_ROOT, 'sitemap.xml.gz'))
|
||||
|
@ -350,3 +358,18 @@ def update_sitemap(base_url):
|
|||
f.write(data)
|
||||
with gzip.open(sitemap, 'wb') as f:
|
||||
f.write(data)
|
||||
|
||||
|
||||
@app.task(queue='default')
|
||||
def bulk_edit(data, username):
|
||||
from django.db import transaction
|
||||
from . import models
|
||||
from .views import edit_item
|
||||
user = models.User.objects.get(username=username)
|
||||
items = models.Item.objects.filter(public_id__in=data['id'])
|
||||
for item in items:
|
||||
if item.editable(user):
|
||||
with transaction.atomic():
|
||||
item.refresh_from_db()
|
||||
response = edit_item(user, item, data, is_task=True)
|
||||
return {}
|
||||
|
|
|
@ -71,7 +71,7 @@ def join_tiles(source_paths, durations, target_path):
|
|||
if not w or large_tile_i < large_tile_n - 1:
|
||||
w = 60
|
||||
data['target_images']['large'] = data['target_images']['large'].resize(
|
||||
(w, small_tile_h), Image.ANTIALIAS
|
||||
(w, small_tile_h), Image.LANCZOS
|
||||
)
|
||||
if data['target_images']['small']:
|
||||
data['target_images']['small'].paste(
|
||||
|
@ -90,7 +90,7 @@ def join_tiles(source_paths, durations, target_path):
|
|||
if data['full_tile_widths'][0]:
|
||||
resized = data['target_images']['large'].resize((
|
||||
data['full_tile_widths'][0], large_tile_h
|
||||
), Image.ANTIALIAS)
|
||||
), Image.LANCZOS)
|
||||
data['target_images']['full'].paste(resized, (data['full_tile_offset'], 0))
|
||||
data['full_tile_offset'] += data['full_tile_widths'][0]
|
||||
data['full_tile_widths'] = data['full_tile_widths'][1:]
|
||||
|
@ -196,7 +196,7 @@ def join_tiles(source_paths, durations, target_path):
|
|||
#print(image_file)
|
||||
image_file = '%stimeline%s%dp.jpg' % (target_path, full_tile_mode, small_tile_h)
|
||||
data['target_images']['full'].resize(
|
||||
(full_tile_w, small_tile_h), Image.ANTIALIAS
|
||||
(full_tile_w, small_tile_h), Image.LANCZOS
|
||||
).save(image_file)
|
||||
#print(image_file)
|
||||
|
||||
|
|
|
@ -61,7 +61,7 @@ def sort_title(title):
|
|||
title = sort_string(title)
|
||||
|
||||
#title
|
||||
title = re.sub('[\'!¿¡,\.;\-"\:\*\[\]]', '', title)
|
||||
title = re.sub(r'[\'!¿¡,\.;\-"\:\*\[\]]', '', title)
|
||||
return title.strip()
|
||||
|
||||
def get_positions(ids, pos, decode_id=False):
|
||||
|
|
|
@ -16,12 +16,14 @@ from wsgiref.util import FileWrapper
|
|||
from django.conf import settings
|
||||
|
||||
from ox.utils import json, ET
|
||||
|
||||
from oxdjango.decorators import login_required_json
|
||||
from oxdjango.shortcuts import render_to_json_response, get_object_or_404_json, json_response
|
||||
from oxdjango.http import HttpFileResponse
|
||||
import ox
|
||||
|
||||
from oxdjango.api import actions
|
||||
from oxdjango.decorators import login_required_json
|
||||
from oxdjango.http import HttpFileResponse
|
||||
from oxdjango.shortcuts import render_to_json_response, get_object_or_404_json, json_response
|
||||
import oxdjango
|
||||
|
||||
from . import models
|
||||
from . import utils
|
||||
from . import tasks
|
||||
|
@ -32,7 +34,6 @@ from clip.models import Clip
|
|||
from user.models import has_capability
|
||||
from changelog.models import add_changelog
|
||||
|
||||
from oxdjango.api import actions
|
||||
|
||||
|
||||
def _order_query(qs, sort, prefix='sort__'):
|
||||
|
@ -308,7 +309,7 @@ def find(request, data):
|
|||
responsive UI: First leave out `keys` to get totals as fast as possible,
|
||||
then pass `positions` to get the positions of previously selected items,
|
||||
finally make the query with the `keys` you need and an appropriate `range`.
|
||||
For more examples, see https://wiki.0x2620.org/wiki/pandora/QuerySyntax.
|
||||
For more examples, see https://code.0x2620.org/0x2620/pandora/wiki/QuerySyntax.
|
||||
see: add, edit, get, lookup, remove, upload
|
||||
'''
|
||||
if settings.JSON_DEBUG:
|
||||
|
@ -533,17 +534,18 @@ def get(request, data):
|
|||
return render_to_json_response(response)
|
||||
actions.register(get)
|
||||
|
||||
def edit_item(request, item, data):
|
||||
def edit_item(user, item, data, is_task=False):
|
||||
data = data.copy()
|
||||
update_clips = False
|
||||
response = json_response(status=200, text='ok')
|
||||
if 'rightslevel' in data:
|
||||
if request.user.profile.capability('canEditRightsLevel'):
|
||||
if user.profile.capability('canEditRightsLevel'):
|
||||
item.level = int(data['rightslevel'])
|
||||
else:
|
||||
response = json_response(status=403, text='permission denied')
|
||||
del data['rightslevel']
|
||||
if 'user' in data:
|
||||
if request.user.profile.get_level() in ('admin', 'staff') and \
|
||||
if user.profile.get_level() in ('admin', 'staff') and \
|
||||
models.User.objects.filter(username=data['user']).exists():
|
||||
new_user = models.User.objects.get(username=data['user'])
|
||||
if new_user != item.user:
|
||||
|
@ -551,13 +553,13 @@ def edit_item(request, item, data):
|
|||
update_clips = True
|
||||
del data['user']
|
||||
if 'groups' in data:
|
||||
if not request.user.profile.capability('canManageUsers'):
|
||||
if not user.profile.capability('canManageUsers'):
|
||||
# Users wihtout canManageUsers can only add/remove groups they are not in
|
||||
groups = set([g.name for g in item.groups.all()])
|
||||
user_groups = set([g.name for g in request.user.groups.all()])
|
||||
user_groups = set([g.name for g in user.groups.all()])
|
||||
other_groups = list(groups - user_groups)
|
||||
data['groups'] = [g for g in data['groups'] if g in user_groups] + other_groups
|
||||
r = item.edit(data)
|
||||
r = item.edit(data, is_task=is_task)
|
||||
if r:
|
||||
r.wait()
|
||||
if update_clips:
|
||||
|
@ -594,10 +596,10 @@ def add(request, data):
|
|||
if p:
|
||||
p.wait()
|
||||
else:
|
||||
i.make_poster()
|
||||
item.make_poster()
|
||||
del data['title']
|
||||
if data:
|
||||
response = edit_item(request, item, data)
|
||||
response = edit_item(request.user, item, data)
|
||||
response['data'] = item.json()
|
||||
add_changelog(request, request_data, item.public_id)
|
||||
return render_to_json_response(response)
|
||||
|
@ -619,16 +621,16 @@ def edit(request, data):
|
|||
see: add, find, get, lookup, remove, upload
|
||||
'''
|
||||
if isinstance(data['id'], list):
|
||||
items = models.Item.objects.filter(public_id__in=data['id'])
|
||||
add_changelog(request, data)
|
||||
t = tasks.bulk_edit.delay(data, request.user.username)
|
||||
response = json_response(status=200, text='ok')
|
||||
response['data']['taskId'] = t.task_id
|
||||
else:
|
||||
items = [get_object_or_404_json(models.Item, public_id=data['id'])]
|
||||
for item in items:
|
||||
item = get_object_or_404_json(models.Item, public_id=data['id'])
|
||||
if item.editable(request.user):
|
||||
request_data = data.copy()
|
||||
response = edit_item(request, item, data)
|
||||
add_changelog(request, data)
|
||||
response = edit_item(request.user, item, data)
|
||||
response['data'] = item.json()
|
||||
if item == items[0]:
|
||||
add_changelog(request, request_data)
|
||||
else:
|
||||
response = json_response(status=403, text='permission denied')
|
||||
return render_to_json_response(response)
|
||||
|
@ -947,9 +949,11 @@ def timeline(request, id, size, position=-1, format='jpg', mode=None):
|
|||
if not item.access(request.user):
|
||||
return HttpResponseForbidden()
|
||||
|
||||
modes = [t['id'] for t in settings.CONFIG['timelines']]
|
||||
if not mode:
|
||||
mode = 'antialias'
|
||||
modes = [t['id'] for t in settings.CONFIG['timelines']]
|
||||
if mode not in modes:
|
||||
mode = modes[0]
|
||||
if mode not in modes:
|
||||
raise Http404
|
||||
modes.pop(modes.index(mode))
|
||||
|
@ -1029,7 +1033,10 @@ def download(request, id, resolution=None, format='webm', part=None):
|
|||
return HttpResponseForbidden()
|
||||
elif r is True:
|
||||
response = HttpResponse(FileWrapper(video), content_type=content_type)
|
||||
try:
|
||||
response['Content-Length'] = os.path.getsize(video.name)
|
||||
except:
|
||||
pass
|
||||
else:
|
||||
response = HttpFileResponse(r, content_type=content_type)
|
||||
else:
|
||||
|
@ -1040,27 +1047,6 @@ def download(request, id, resolution=None, format='webm', part=None):
|
|||
response['Content-Disposition'] = "attachment; filename*=UTF-8''%s" % quote(filename.encode('utf-8'))
|
||||
return response
|
||||
|
||||
def torrent(request, id, filename=None):
|
||||
item = get_object_or_404(models.Item, public_id=id)
|
||||
if not item.access(request.user):
|
||||
return HttpResponseForbidden()
|
||||
if not item.torrent:
|
||||
raise Http404
|
||||
if not filename or filename.endswith('.torrent'):
|
||||
response = HttpResponse(item.get_torrent(request),
|
||||
content_type='application/x-bittorrent')
|
||||
filename = utils.safe_filename("%s.torrent" % item.get('title'))
|
||||
response['Content-Disposition'] = "attachment; filename*=UTF-8''%s" % quote(filename.encode('utf-8'))
|
||||
return response
|
||||
while filename.startswith('/'):
|
||||
filename = filename[1:]
|
||||
filename = filename.replace('/../', '/')
|
||||
filename = item.path('torrent/%s' % filename)
|
||||
filename = os.path.abspath(os.path.join(settings.MEDIA_ROOT, filename))
|
||||
response = HttpFileResponse(filename)
|
||||
response['Content-Disposition'] = "attachment; filename*=UTF-8''%s" % \
|
||||
quote(os.path.basename(filename.encode('utf-8')))
|
||||
return response
|
||||
|
||||
def video(request, id, resolution, format, index=None, track=None):
|
||||
resolution = int(resolution)
|
||||
|
@ -1282,12 +1268,6 @@ def atom_xml(request):
|
|||
el.text = "1:1"
|
||||
|
||||
if has_capability(request.user, 'canDownloadVideo'):
|
||||
if item.torrent:
|
||||
el = ET.SubElement(entry, "link")
|
||||
el.attrib['rel'] = 'enclosure'
|
||||
el.attrib['type'] = 'application/x-bittorrent'
|
||||
el.attrib['href'] = '%s/torrent/' % page_link
|
||||
el.attrib['length'] = '%s' % ox.get_torrent_size(item.torrent.path)
|
||||
# FIXME: loop over streams
|
||||
# for s in item.streams().filter(resolution=max(settings.CONFIG['video']['resolutions'])):
|
||||
for s in item.streams().filter(source=None):
|
||||
|
@ -1310,12 +1290,15 @@ def atom_xml(request):
|
|||
'application/atom+xml'
|
||||
)
|
||||
|
||||
|
||||
def oembed(request):
|
||||
format = request.GET.get('format', 'json')
|
||||
maxwidth = int(request.GET.get('maxwidth', 640))
|
||||
maxheight = int(request.GET.get('maxheight', 480))
|
||||
|
||||
url = request.GET['url']
|
||||
url = request.GET.get('url')
|
||||
if not url:
|
||||
raise Http404
|
||||
parts = urlparse(url).path.split('/')
|
||||
if len(parts) < 2:
|
||||
raise Http404
|
||||
|
|
7
pandora/itemlist/apps.py
Normal file
7
pandora/itemlist/apps.py
Normal file
|
@ -0,0 +1,7 @@
|
|||
from django.apps import AppConfig
|
||||
|
||||
|
||||
class ItemListConfig(AppConfig):
|
||||
default_auto_field = "django.db.models.BigAutoField"
|
||||
name = 'itemlist'
|
||||
|
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue