0x2620
/
pandora
2
1
Fork 1
Code Issues 1.3k Pull Requests Releases Wiki Activity

use one version of ffmpeg for all tasks, remove avconv dependency, fixes #2606

This commit is contained in:
j 2015-01-05 16:07:55 +00:00
parent e44703854e
commit e1bc418e0c
6 changed files with 61 additions and 161 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
README
View File

@ -3,9 +3,9 @@ pan.do/ra - open media archive
for more information on pan.do/ra visit our website at https://pan.do/ra
== SETUP ==
pan.do/ra is known to work with Ubuntu 12.04,
pan.do/ra is known to work with Ubuntu 14.04,
but other distributions should also work.
The instructions below are for Ubuntu 12.04.
The instructions below are for Ubuntu 14.04.
All commans given expect that you are root.
To run pan.do/ra you need to install and setup:
@ -18,7 +18,7 @@ To run pan.do/ra you need to install and setup:
* Installing required packages
1) add pandora ppa to get all packages in the required version
apt-get install python-software-properties
apt-get install software-properties-common
add-apt-repository ppa:j/pandora
apt-get update
@ -28,9 +28,8 @@ To run pan.do/ra you need to install and setup:
python-dev python-imaging python-numpy python-psycopg2 \
python-geoip python-html5lib python-lxml \
postgresql postgresql-contrib rabbitmq-server \
ffmpeg2theora libav-tools libavcodec-extra-53 melt \
python-ox oxframe imagemagick poppler-utils mkvtoolnix gpac
poppler-utils mkvtoolnix gpac imagemagick \
python-ox oxframe ffmpeg
* Prepare Environment
1) add pandora user and set permissions
@ -44,7 +43,7 @@ To run pan.do/ra you need to install and setup:
exit
3) Setup RabbitMQ
Important: "use_your_own" is a password and you have to use the same value here and below for BROKER_PASSWORD
Important: "use_your_own" is a password and you have to use the same value here and below for BROKER_URL
rabbitmqctl add_user pandora use_your_own
rabbitmqctl add_vhost /pandora
@ -77,7 +76,7 @@ Important: "use_your_own" is a password and you have to use the same value here
}
}
DB_GIN_TRGM = True
BROKER_PASSWORD = "use_your_own"
BROKER_URL = 'amqp://pandora:<use_your_own>@localhost:5672//pandora'
#with apache x-sendfile or lighttpd set this to True
XSENDFILE = False
@ -143,14 +142,10 @@ To update a pandora installation get the latest version from bzr by running
this will update pandora/oxjs/python-ox and list possible upgrades to the db
to update your database tables, use
to update your database run:
su pandora
cd /srv/pandora/pandora
./manage.py sqldiff -a
to check if there are changes and
./manage.py sqldiff -a | ./manage.py dbshell
to apply them.
cd /srv/pandora
./update.py db
=== Development ===
in one terminal:

21
pandora/app/config.py
View File

@ -16,7 +16,7 @@ from django.contrib.auth.models import User
import ox.jsonc
from ox.utils import json
from archive.extract import supported_formats, AVCONV, avconv_version
from archive.extract import supported_formats
from item.utils import get_by_id
@ -126,25 +126,24 @@ def load_config():
if f not in sformats or not sformats[f]:
sys.stderr.write('''WARNING:
Your configuration contains a video format "%s" that is
not supported by your version of avconv. Make sure you
dont have a local version of avconv in /usr/local/bin
and libavcodec-extra-53 and libav-tools are installed:
sudo apt-get install libavcodec-extra-53 libav-tools
not supported by your version of ffmpeg. Make sure you
dont have a local version of ffmpeg in /usr/local/bin
and ffmpeg is installed from ppa:j/pandora:
sudo add-apt-repository ppa:j/pandora
sudo apt-get install ffmpeg
''' % f)
else:
sys.stderr.write('''WARNING:
You dont have "%s" installed.
To fix this on Ubuntu 12.04, run:
You dont have "ffmpeg" installed. To fix this on Ubuntu 14.04, run:
sudo apt-get install libavcodec-extra-53 libav-tools
sudo add-apt-repository ppa:j/pandora
sudo apt-get install ffmpeg
check the README for further details.
''' % AVCONV)
''')
settings.AVCONV_VERSION = avconv_version()
settings.CONFIG = config
admin = len(settings.CONFIG['userLevels']) - 1
if not 'syncdb' in sys.argv \

134
pandora/archive/extract.py
View File

@ -11,6 +11,7 @@ import tempfile
import time
import math
import shutil
from distutils.spawn import find_executable
from glob import glob
import numpy as np
@ -22,8 +23,6 @@ from django.conf import settings
img_extension='jpg'
AVCONV = 'avconv'
MAX_DISTANCE = math.sqrt(3 * pow(255, 2))
@ -50,32 +49,18 @@ class AspectRatio(fractions.Fraction):
return "%d:%d" % (self.numerator, self.denominator)
def supported_formats():
p = subprocess.Popen(['which', AVCONV],
stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
stdout, stderr = p.communicate()
if not stdout.strip():
if not find_executable(settings.FFMPEG):
return None
p = subprocess.Popen([AVCONV, '-codecs'],
p = subprocess.Popen([settings.FFMPEG, '-codecs'],
stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
stdout, stderr = p.communicate()
return {
'ogg': 'libtheora' in stdout and 'libvorbis' in stdout,
'webm': 'libvpx' in stdout and 'libvorbis' in stdout,
'mp4': 'libx264' in stdout and 'libvo_aacenc' in stdout,
'mp4': 'libx264' in stdout and 'DEA.L. aac' in stdout,
}
def avconv_version():
p = subprocess.Popen([AVCONV],
stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
stdout, stderr = p.communicate()
version = stderr.split(' ')[2].split('-')[0]
try:
version = int(version.split('.')[0])
except:
pass
return version
def stream(video, target, profile, info, avconv=None, audio_track=0):
def stream(video, target, profile, info, audio_track=0):
if not os.path.exists(target):
ox.makedirs(os.path.dirname(target))
@ -189,55 +174,13 @@ def stream(video, target, profile, info, avconv=None, audio_track=0):
'-auto-alt-ref', '1',
]
if format == 'mp4':
#quicktime does not support bpyramid
'''
video_settings += [
'-vcodec', 'libx264',
'-flags', '+loop+mv4',
'-cmp', '256',
'-partitions', '+parti4x4+parti8x8+partp4x4+partp8x8+partb8x8',
'-me_method', 'hex',
'-subq', '7',
'-trellis', '1',
'-refs', '5',
'-bf', '3',
'-flags2', '+bpyramid+wpred+mixed_refs+dct8x8',
'-coder', '1',
'-me_range', '16',
'-keyint_min', '25', #FIXME: should this be related to fps?
'-sc_threshold','40',
'-i_qfactor', '0.71',
'-qmin', '10', '-qmax', '51',
'-qdiff', '4'
'-c:v', 'libx264',
'-preset:v', 'medium',
'-profile:v', 'baseline',
# does not work with avconv in Ubuntu 14.04 yet
#'-level', '3.0',
]
'''
if settings.AVCONV_VERSION >= 9:
video_settings += [
'-vcodec', 'libx264',
'-preset:v', 'medium',
'-profile:v', 'baseline',
# does not work with avconv in Ubuntu 14.04 yet
#'-level', '3.0',
]
else:
video_settings += [
'-vcodec', 'libx264',
'-flags', '+loop+mv4',
'-cmp', '256',
'-partitions', '+parti4x4+parti8x8+partp4x4+partp8x8+partb8x8',
'-me_method', 'hex',
'-subq', '7',
'-trellis', '1',
'-refs', '5',
'-bf', '0',
'-flags2', '+mixed_refs',
'-coder', '0',
'-me_range', '16',
'-sc_threshold', '40',
'-i_qfactor', '0.71',
'-qmin', '10', '-qmax', '51',
'-qdiff', '4'
]
video_settings += ['-map', '0:%s,0:0'%info['video'][0]['id']]
else:
video_settings = ['-vn']
@ -271,15 +214,13 @@ def stream(video, target, profile, info, avconv=None, audio_track=0):
if audiobitrate:
audio_settings += ['-ab', audiobitrate]
if format == 'mp4':
audio_settings += ['-acodec', 'libvo_aacenc']
audio_settings += ['-c:a', 'aac', '-strict', '-2']
else:
audio_settings += ['-acodec', 'libvorbis']
audio_settings += ['-c:a', 'libvorbis']
else:
audio_settings = ['-an']
if not avconv:
avconv = AVCONV
cmd = [avconv, '-y', '-i', video, '-threads', '4', '-map_metadata', '-1', '-sn'] \
cmd = [settings.FFMPEG, '-y', '-i', video, '-threads', '4', '-map_metadata', '-1', '-sn'] \
+ audio_settings \
+ video_settings
@ -355,52 +296,41 @@ def frame(video, frame, position, height=128, redo=False, info=None):
if redo or not exists(frame):
ox.makedirs(folder)
if video.endswith('.mp4'):
if settings.FFMPEG:
cmd = ffmpeg_frame_cmd(video, frame, position, height)
else:
cmd = avconv_frame_cmd(video, frame, position, height)
cmd = ffmpeg_frame_cmd(video, frame, position, height)
else:
cmd = ['oxframe', '-i', video, '-o', frame,
'-p', str(position), '-y', str(height)]
run_command(cmd)
def avconv_frame_cmd(video, frame, position, height=128):
cmd = [
AVCONV, '-y',
'-ss', str(position),
'-i', video,
'-an', '-vframes', '1',
'-vf', 'scale=-1:%s' % height
]
if not frame.endswith('.png'):
cmd += ['-f', 'mjpeg']
cmd += [frame]
return cmd
def ffmpeg_frame_cmd(video, frame, position, height=128):
cmd = [
settings.FFMPEG, '-y',
'-ss', str(position),
'-i', video,
'-an', '-frames:v', '1',
'-vf', 'scale=-1:%s' % height,
'-vf', 'scale=-1:%s' % height if height else 'scale=iw*sar:ih',
frame
]
return cmd
def ffmpeg_version():
p = subprocess.Popen([settings.FFMPEG],
stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
stdout, stderr = p.communicate()
version = stderr.split(' ')[2].split('-')[0]
try:
version = tuple(map(int, version.split('.')))
except:
pass
return version
def frame_direct(video, target, position):
fdir = os.path.dirname(target)
if fdir and not os.path.exists(fdir):
os.makedirs(fdir)
pre = position - 2
if pre < 0:
pre = 0
else:
position = 2
cmd = [ox.file.cmd('ffmpeg'), '-y', '-ss', str(pre), '-i', video, '-ss', str(position),
'-vf', 'scale=iw*sar:ih',
'-an', '-vframes', '1', target]
cmd = ffmpeg_frame_cmd(video, target, position, None)
r = run_command(cmd)
return r == 0
@ -595,13 +525,13 @@ def chop(video, start, end):
ext = os.path.splitext(video)[1]
choped_video = '%s/tmp%s' % (tmp, ext)
cmd = [
'ffmpeg',
settings.FFMPEG,
'-y',
'-i', video,
'-ss', '%.3f'%start,
'-t', '%.3f'%t,
'-vcodec', 'copy',
'-acodec', 'copy',
'-c:v', 'copy',
'-c:a', 'copy',
'-f', ext[1:],
choped_video
]

11
pandora/archive/models.py
View File

@ -453,9 +453,6 @@ class File(models.Model):
if self.data and len(audio) > 1:
config = settings.CONFIG['video']
resolution = self.stream_resolution()
ffmpeg = ox.file.cmd('ffmpeg')
if ffmpeg == 'ffmpeg':
ffmpeg = None
tmp = tempfile.mkdtemp()
if not self.info.get('language'):
self.info['language'] = parse_language(audio[0].get('language'))
@ -471,8 +468,7 @@ class File(models.Model):
n += 1
profile = '%sp.%s' % (resolution, config['formats'][0])
target = os.path.join(tmp, language + '_' + profile)
ok, error = extract.stream(media, target, profile, info, ffmpeg,
audio_track=i+1)
ok, error = extract.stream(media, target, profile, info, audio_track=i+1)
if ok:
tinfo = ox.avinfo(target)
del tinfo['path']
@ -676,15 +672,12 @@ class Stream(models.Model):
def encode(self):
media = self.source.media.path if self.source else self.file.data.path
ffmpeg = ox.file.cmd('ffmpeg')
if self.source or ffmpeg == 'ffmpeg' or self.format != 'webm':
ffmpeg = None
if not self.media:
self.media.name = self.path(self.name())
target = self.media.path
info = ox.avinfo(media)
ok, error = extract.stream(media, target, self.name(), info, ffmpeg)
ok, error = extract.stream(media, target, self.name(), info)
# file could have been moved while encoding
# get current version from db and update
_self = Stream.objects.get(id=self.id)

3
pandora/settings.py
View File

@ -152,8 +152,7 @@ LOGGING = {
AUTH_PROFILE_MODULE = 'user.UserProfile'
AUTH_CHECK_USERNAME = True
AVCONV_VERSION = 0
FFMPEG=False
FFMPEG='ffmpeg'
#=========================================================================
#Pan.do/ra related settings settings

28
vm/firstboot.sh
View File

@ -6,6 +6,11 @@ else
ID=unknown
fi
UBUNTU_VERSION="$VERSION_ID"
if [ "$ID" == "debian" ]; then
SYSTEMD="yes"
else
SYSTEMD="no"
fi
export DEBIAN_FRONTEND=noninteractive
echo "deb http://ppa.launchpad.net/j/pandora/ubuntu trusty main" > /etc/apt/sources.list.d/j-pandora.list
apt-key add - << EOF
@ -24,20 +29,6 @@ pAAGSEQ4uz6bYSeM4Q==
-----END PGP PUBLIC KEY BLOCK-----
EOF
if [ "$ID" == "debian" ]; then
SYSTEMD="yes"
else
SYSTEMD="no"
if [ "$UBUNTU_VERSION" == "12.04" ]; then
EXTRA=python-software-properties
else
EXTRA=""
fi
apt-get install -y \
update-manager-core \
software-properties-common \
$EXTRA
fi
apt-get update
if [ "$LXC" == "no" ]; then
@ -46,12 +37,6 @@ apt-get install -y \
ntp
fi
if [ "$UBUNTU_VERSION" == "12.04" ]; then
LIBAVCODEC_EXTRA=libavcodec-extra-53
else
LIBAVCODEC_EXTRA=libavcodec-extra
fi
apt-get install -y \
openssh-server \
vim \
@ -77,8 +62,7 @@ apt-get install -y \
python-html5lib \
python-ox \
oxframe \
$LIBAVCODEC_EXTRA \
libav-tools \
ffmpeg \
ffmpeg2theora \
mkvtoolnix \
gpac \