use one version of ffmpeg for all tasks, remove avconv dependency, fixes #2606

This commit is contained in:
j 2015-01-05 16:07:55 +00:00
parent e44703854e
commit e1bc418e0c
6 changed files with 61 additions and 161 deletions

25
README
View file

@ -3,9 +3,9 @@ pan.do/ra - open media archive
for more information on pan.do/ra visit our website at https://pan.do/ra for more information on pan.do/ra visit our website at https://pan.do/ra
== SETUP == == SETUP ==
pan.do/ra is known to work with Ubuntu 12.04, pan.do/ra is known to work with Ubuntu 14.04,
but other distributions should also work. but other distributions should also work.
The instructions below are for Ubuntu 12.04. The instructions below are for Ubuntu 14.04.
All commans given expect that you are root. All commans given expect that you are root.
To run pan.do/ra you need to install and setup: To run pan.do/ra you need to install and setup:
@ -18,7 +18,7 @@ To run pan.do/ra you need to install and setup:
* Installing required packages * Installing required packages
1) add pandora ppa to get all packages in the required version 1) add pandora ppa to get all packages in the required version
apt-get install python-software-properties apt-get install software-properties-common
add-apt-repository ppa:j/pandora add-apt-repository ppa:j/pandora
apt-get update apt-get update
@ -28,9 +28,8 @@ To run pan.do/ra you need to install and setup:
python-dev python-imaging python-numpy python-psycopg2 \ python-dev python-imaging python-numpy python-psycopg2 \
python-geoip python-html5lib python-lxml \ python-geoip python-html5lib python-lxml \
postgresql postgresql-contrib rabbitmq-server \ postgresql postgresql-contrib rabbitmq-server \
ffmpeg2theora libav-tools libavcodec-extra-53 melt \ poppler-utils mkvtoolnix gpac imagemagick \
python-ox oxframe imagemagick poppler-utils mkvtoolnix gpac python-ox oxframe ffmpeg
* Prepare Environment * Prepare Environment
1) add pandora user and set permissions 1) add pandora user and set permissions
@ -44,7 +43,7 @@ To run pan.do/ra you need to install and setup:
exit exit
3) Setup RabbitMQ 3) Setup RabbitMQ
Important: "use_your_own" is a password and you have to use the same value here and below for BROKER_PASSWORD Important: "use_your_own" is a password and you have to use the same value here and below for BROKER_URL
rabbitmqctl add_user pandora use_your_own rabbitmqctl add_user pandora use_your_own
rabbitmqctl add_vhost /pandora rabbitmqctl add_vhost /pandora
@ -77,7 +76,7 @@ Important: "use_your_own" is a password and you have to use the same value here
} }
} }
DB_GIN_TRGM = True DB_GIN_TRGM = True
BROKER_PASSWORD = "use_your_own" BROKER_URL = 'amqp://pandora:<use_your_own>@localhost:5672//pandora'
#with apache x-sendfile or lighttpd set this to True #with apache x-sendfile or lighttpd set this to True
XSENDFILE = False XSENDFILE = False
@ -143,14 +142,10 @@ To update a pandora installation get the latest version from bzr by running
this will update pandora/oxjs/python-ox and list possible upgrades to the db this will update pandora/oxjs/python-ox and list possible upgrades to the db
to update your database tables, use to update your database run:
su pandora su pandora
cd /srv/pandora/pandora cd /srv/pandora
./manage.py sqldiff -a ./update.py db
to check if there are changes and
./manage.py sqldiff -a | ./manage.py dbshell
to apply them.
=== Development === === Development ===
in one terminal: in one terminal:

View file

@ -16,7 +16,7 @@ from django.contrib.auth.models import User
import ox.jsonc import ox.jsonc
from ox.utils import json from ox.utils import json
from archive.extract import supported_formats, AVCONV, avconv_version from archive.extract import supported_formats
from item.utils import get_by_id from item.utils import get_by_id
@ -126,25 +126,24 @@ def load_config():
if f not in sformats or not sformats[f]: if f not in sformats or not sformats[f]:
sys.stderr.write('''WARNING: sys.stderr.write('''WARNING:
Your configuration contains a video format "%s" that is Your configuration contains a video format "%s" that is
not supported by your version of avconv. Make sure you not supported by your version of ffmpeg. Make sure you
dont have a local version of avconv in /usr/local/bin dont have a local version of ffmpeg in /usr/local/bin
and libavcodec-extra-53 and libav-tools are installed: and ffmpeg is installed from ppa:j/pandora:
sudo apt-get install libavcodec-extra-53 libav-tools
sudo add-apt-repository ppa:j/pandora
sudo apt-get install ffmpeg
''' % f) ''' % f)
else: else:
sys.stderr.write('''WARNING: sys.stderr.write('''WARNING:
You dont have "%s" installed. You dont have "ffmpeg" installed. To fix this on Ubuntu 14.04, run:
To fix this on Ubuntu 12.04, run:
sudo apt-get install libavcodec-extra-53 libav-tools sudo add-apt-repository ppa:j/pandora
sudo apt-get install ffmpeg
check the README for further details. check the README for further details.
''' % AVCONV) ''')
settings.AVCONV_VERSION = avconv_version()
settings.CONFIG = config settings.CONFIG = config
admin = len(settings.CONFIG['userLevels']) - 1 admin = len(settings.CONFIG['userLevels']) - 1
if not 'syncdb' in sys.argv \ if not 'syncdb' in sys.argv \

View file

@ -11,6 +11,7 @@ import tempfile
import time import time
import math import math
import shutil import shutil
from distutils.spawn import find_executable
from glob import glob from glob import glob
import numpy as np import numpy as np
@ -22,8 +23,6 @@ from django.conf import settings
img_extension='jpg' img_extension='jpg'
AVCONV = 'avconv'
MAX_DISTANCE = math.sqrt(3 * pow(255, 2)) MAX_DISTANCE = math.sqrt(3 * pow(255, 2))
@ -50,32 +49,18 @@ class AspectRatio(fractions.Fraction):
return "%d:%d" % (self.numerator, self.denominator) return "%d:%d" % (self.numerator, self.denominator)
def supported_formats(): def supported_formats():
p = subprocess.Popen(['which', AVCONV], if not find_executable(settings.FFMPEG):
stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
stdout, stderr = p.communicate()
if not stdout.strip():
return None return None
p = subprocess.Popen([AVCONV, '-codecs'], p = subprocess.Popen([settings.FFMPEG, '-codecs'],
stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
stdout, stderr = p.communicate() stdout, stderr = p.communicate()
return { return {
'ogg': 'libtheora' in stdout and 'libvorbis' in stdout, 'ogg': 'libtheora' in stdout and 'libvorbis' in stdout,
'webm': 'libvpx' in stdout and 'libvorbis' in stdout, 'webm': 'libvpx' in stdout and 'libvorbis' in stdout,
'mp4': 'libx264' in stdout and 'libvo_aacenc' in stdout, 'mp4': 'libx264' in stdout and 'DEA.L. aac' in stdout,
} }
def avconv_version(): def stream(video, target, profile, info, audio_track=0):
p = subprocess.Popen([AVCONV],
stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
stdout, stderr = p.communicate()
version = stderr.split(' ')[2].split('-')[0]
try:
version = int(version.split('.')[0])
except:
pass
return version
def stream(video, target, profile, info, avconv=None, audio_track=0):
if not os.path.exists(target): if not os.path.exists(target):
ox.makedirs(os.path.dirname(target)) ox.makedirs(os.path.dirname(target))
@ -189,55 +174,13 @@ def stream(video, target, profile, info, avconv=None, audio_track=0):
'-auto-alt-ref', '1', '-auto-alt-ref', '1',
] ]
if format == 'mp4': if format == 'mp4':
#quicktime does not support bpyramid
'''
video_settings += [ video_settings += [
'-vcodec', 'libx264', '-c:v', 'libx264',
'-flags', '+loop+mv4', '-preset:v', 'medium',
'-cmp', '256', '-profile:v', 'baseline',
'-partitions', '+parti4x4+parti8x8+partp4x4+partp8x8+partb8x8', # does not work with avconv in Ubuntu 14.04 yet
'-me_method', 'hex', #'-level', '3.0',
'-subq', '7',
'-trellis', '1',
'-refs', '5',
'-bf', '3',
'-flags2', '+bpyramid+wpred+mixed_refs+dct8x8',
'-coder', '1',
'-me_range', '16',
'-keyint_min', '25', #FIXME: should this be related to fps?
'-sc_threshold','40',
'-i_qfactor', '0.71',
'-qmin', '10', '-qmax', '51',
'-qdiff', '4'
] ]
'''
if settings.AVCONV_VERSION >= 9:
video_settings += [
'-vcodec', 'libx264',
'-preset:v', 'medium',
'-profile:v', 'baseline',
# does not work with avconv in Ubuntu 14.04 yet
#'-level', '3.0',
]
else:
video_settings += [
'-vcodec', 'libx264',
'-flags', '+loop+mv4',
'-cmp', '256',
'-partitions', '+parti4x4+parti8x8+partp4x4+partp8x8+partb8x8',
'-me_method', 'hex',
'-subq', '7',
'-trellis', '1',
'-refs', '5',
'-bf', '0',
'-flags2', '+mixed_refs',
'-coder', '0',
'-me_range', '16',
'-sc_threshold', '40',
'-i_qfactor', '0.71',
'-qmin', '10', '-qmax', '51',
'-qdiff', '4'
]
video_settings += ['-map', '0:%s,0:0'%info['video'][0]['id']] video_settings += ['-map', '0:%s,0:0'%info['video'][0]['id']]
else: else:
video_settings = ['-vn'] video_settings = ['-vn']
@ -271,15 +214,13 @@ def stream(video, target, profile, info, avconv=None, audio_track=0):
if audiobitrate: if audiobitrate:
audio_settings += ['-ab', audiobitrate] audio_settings += ['-ab', audiobitrate]
if format == 'mp4': if format == 'mp4':
audio_settings += ['-acodec', 'libvo_aacenc'] audio_settings += ['-c:a', 'aac', '-strict', '-2']
else: else:
audio_settings += ['-acodec', 'libvorbis'] audio_settings += ['-c:a', 'libvorbis']
else: else:
audio_settings = ['-an'] audio_settings = ['-an']
if not avconv: cmd = [settings.FFMPEG, '-y', '-i', video, '-threads', '4', '-map_metadata', '-1', '-sn'] \
avconv = AVCONV
cmd = [avconv, '-y', '-i', video, '-threads', '4', '-map_metadata', '-1', '-sn'] \
+ audio_settings \ + audio_settings \
+ video_settings + video_settings
@ -355,52 +296,41 @@ def frame(video, frame, position, height=128, redo=False, info=None):
if redo or not exists(frame): if redo or not exists(frame):
ox.makedirs(folder) ox.makedirs(folder)
if video.endswith('.mp4'): if video.endswith('.mp4'):
if settings.FFMPEG: cmd = ffmpeg_frame_cmd(video, frame, position, height)
cmd = ffmpeg_frame_cmd(video, frame, position, height)
else:
cmd = avconv_frame_cmd(video, frame, position, height)
else: else:
cmd = ['oxframe', '-i', video, '-o', frame, cmd = ['oxframe', '-i', video, '-o', frame,
'-p', str(position), '-y', str(height)] '-p', str(position), '-y', str(height)]
run_command(cmd) run_command(cmd)
def avconv_frame_cmd(video, frame, position, height=128):
cmd = [
AVCONV, '-y',
'-ss', str(position),
'-i', video,
'-an', '-vframes', '1',
'-vf', 'scale=-1:%s' % height
]
if not frame.endswith('.png'):
cmd += ['-f', 'mjpeg']
cmd += [frame]
return cmd
def ffmpeg_frame_cmd(video, frame, position, height=128): def ffmpeg_frame_cmd(video, frame, position, height=128):
cmd = [ cmd = [
settings.FFMPEG, '-y', settings.FFMPEG, '-y',
'-ss', str(position), '-ss', str(position),
'-i', video, '-i', video,
'-an', '-frames:v', '1', '-an', '-frames:v', '1',
'-vf', 'scale=-1:%s' % height, '-vf', 'scale=-1:%s' % height if height else 'scale=iw*sar:ih',
frame frame
] ]
return cmd return cmd
def ffmpeg_version():
p = subprocess.Popen([settings.FFMPEG],
stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
stdout, stderr = p.communicate()
version = stderr.split(' ')[2].split('-')[0]
try:
version = tuple(map(int, version.split('.')))
except:
pass
return version
def frame_direct(video, target, position): def frame_direct(video, target, position):
fdir = os.path.dirname(target) fdir = os.path.dirname(target)
if fdir and not os.path.exists(fdir): if fdir and not os.path.exists(fdir):
os.makedirs(fdir) os.makedirs(fdir)
cmd = ffmpeg_frame_cmd(video, target, position, None)
pre = position - 2
if pre < 0:
pre = 0
else:
position = 2
cmd = [ox.file.cmd('ffmpeg'), '-y', '-ss', str(pre), '-i', video, '-ss', str(position),
'-vf', 'scale=iw*sar:ih',
'-an', '-vframes', '1', target]
r = run_command(cmd) r = run_command(cmd)
return r == 0 return r == 0
@ -595,13 +525,13 @@ def chop(video, start, end):
ext = os.path.splitext(video)[1] ext = os.path.splitext(video)[1]
choped_video = '%s/tmp%s' % (tmp, ext) choped_video = '%s/tmp%s' % (tmp, ext)
cmd = [ cmd = [
'ffmpeg', settings.FFMPEG,
'-y', '-y',
'-i', video, '-i', video,
'-ss', '%.3f'%start, '-ss', '%.3f'%start,
'-t', '%.3f'%t, '-t', '%.3f'%t,
'-vcodec', 'copy', '-c:v', 'copy',
'-acodec', 'copy', '-c:a', 'copy',
'-f', ext[1:], '-f', ext[1:],
choped_video choped_video
] ]

View file

@ -453,9 +453,6 @@ class File(models.Model):
if self.data and len(audio) > 1: if self.data and len(audio) > 1:
config = settings.CONFIG['video'] config = settings.CONFIG['video']
resolution = self.stream_resolution() resolution = self.stream_resolution()
ffmpeg = ox.file.cmd('ffmpeg')
if ffmpeg == 'ffmpeg':
ffmpeg = None
tmp = tempfile.mkdtemp() tmp = tempfile.mkdtemp()
if not self.info.get('language'): if not self.info.get('language'):
self.info['language'] = parse_language(audio[0].get('language')) self.info['language'] = parse_language(audio[0].get('language'))
@ -471,8 +468,7 @@ class File(models.Model):
n += 1 n += 1
profile = '%sp.%s' % (resolution, config['formats'][0]) profile = '%sp.%s' % (resolution, config['formats'][0])
target = os.path.join(tmp, language + '_' + profile) target = os.path.join(tmp, language + '_' + profile)
ok, error = extract.stream(media, target, profile, info, ffmpeg, ok, error = extract.stream(media, target, profile, info, audio_track=i+1)
audio_track=i+1)
if ok: if ok:
tinfo = ox.avinfo(target) tinfo = ox.avinfo(target)
del tinfo['path'] del tinfo['path']
@ -676,15 +672,12 @@ class Stream(models.Model):
def encode(self): def encode(self):
media = self.source.media.path if self.source else self.file.data.path media = self.source.media.path if self.source else self.file.data.path
ffmpeg = ox.file.cmd('ffmpeg')
if self.source or ffmpeg == 'ffmpeg' or self.format != 'webm':
ffmpeg = None
if not self.media: if not self.media:
self.media.name = self.path(self.name()) self.media.name = self.path(self.name())
target = self.media.path target = self.media.path
info = ox.avinfo(media) info = ox.avinfo(media)
ok, error = extract.stream(media, target, self.name(), info, ffmpeg) ok, error = extract.stream(media, target, self.name(), info)
# file could have been moved while encoding # file could have been moved while encoding
# get current version from db and update # get current version from db and update
_self = Stream.objects.get(id=self.id) _self = Stream.objects.get(id=self.id)

View file

@ -152,8 +152,7 @@ LOGGING = {
AUTH_PROFILE_MODULE = 'user.UserProfile' AUTH_PROFILE_MODULE = 'user.UserProfile'
AUTH_CHECK_USERNAME = True AUTH_CHECK_USERNAME = True
AVCONV_VERSION = 0 FFMPEG='ffmpeg'
FFMPEG=False
#========================================================================= #=========================================================================
#Pan.do/ra related settings settings #Pan.do/ra related settings settings

View file

@ -6,6 +6,11 @@ else
ID=unknown ID=unknown
fi fi
UBUNTU_VERSION="$VERSION_ID" UBUNTU_VERSION="$VERSION_ID"
if [ "$ID" == "debian" ]; then
SYSTEMD="yes"
else
SYSTEMD="no"
fi
export DEBIAN_FRONTEND=noninteractive export DEBIAN_FRONTEND=noninteractive
echo "deb http://ppa.launchpad.net/j/pandora/ubuntu trusty main" > /etc/apt/sources.list.d/j-pandora.list echo "deb http://ppa.launchpad.net/j/pandora/ubuntu trusty main" > /etc/apt/sources.list.d/j-pandora.list
apt-key add - << EOF apt-key add - << EOF
@ -24,20 +29,6 @@ pAAGSEQ4uz6bYSeM4Q==
-----END PGP PUBLIC KEY BLOCK----- -----END PGP PUBLIC KEY BLOCK-----
EOF EOF
if [ "$ID" == "debian" ]; then
SYSTEMD="yes"
else
SYSTEMD="no"
if [ "$UBUNTU_VERSION" == "12.04" ]; then
EXTRA=python-software-properties
else
EXTRA=""
fi
apt-get install -y \
update-manager-core \
software-properties-common \
$EXTRA
fi
apt-get update apt-get update
if [ "$LXC" == "no" ]; then if [ "$LXC" == "no" ]; then
@ -46,12 +37,6 @@ apt-get install -y \
ntp ntp
fi fi
if [ "$UBUNTU_VERSION" == "12.04" ]; then
LIBAVCODEC_EXTRA=libavcodec-extra-53
else
LIBAVCODEC_EXTRA=libavcodec-extra
fi
apt-get install -y \ apt-get install -y \
openssh-server \ openssh-server \
vim \ vim \
@ -77,8 +62,7 @@ apt-get install -y \
python-html5lib \ python-html5lib \
python-ox \ python-ox \
oxframe \ oxframe \
$LIBAVCODEC_EXTRA \ ffmpeg \
libav-tools \
ffmpeg2theora \ ffmpeg2theora \
mkvtoolnix \ mkvtoolnix \
gpac \ gpac \