From e1bc418e0c19d15d01aa9decd2710c44a3c11634 Mon Sep 17 00:00:00 2001 From: j <0x006A@0x2620.org> Date: Mon, 5 Jan 2015 16:07:55 +0000 Subject: [PATCH] use one version of ffmpeg for all tasks, remove avconv dependency, fixes #2606 --- README | 25 +++---- pandora/app/config.py | 21 +++--- pandora/archive/extract.py | 134 +++++++++---------------------------- pandora/archive/models.py | 11 +-- pandora/settings.py | 3 +- vm/firstboot.sh | 28 ++------ 6 files changed, 61 insertions(+), 161 deletions(-) diff --git a/README b/README index e9c2cb02..90469ef2 100644 --- a/README +++ b/README @@ -3,9 +3,9 @@ pan.do/ra - open media archive for more information on pan.do/ra visit our website at https://pan.do/ra == SETUP == -pan.do/ra is known to work with Ubuntu 12.04, +pan.do/ra is known to work with Ubuntu 14.04, but other distributions should also work. -The instructions below are for Ubuntu 12.04. +The instructions below are for Ubuntu 14.04. All commans given expect that you are root. To run pan.do/ra you need to install and setup: @@ -18,7 +18,7 @@ To run pan.do/ra you need to install and setup: * Installing required packages 1) add pandora ppa to get all packages in the required version - apt-get install python-software-properties + apt-get install software-properties-common add-apt-repository ppa:j/pandora apt-get update @@ -28,9 +28,8 @@ To run pan.do/ra you need to install and setup: python-dev python-imaging python-numpy python-psycopg2 \ python-geoip python-html5lib python-lxml \ postgresql postgresql-contrib rabbitmq-server \ - ffmpeg2theora libav-tools libavcodec-extra-53 melt \ - python-ox oxframe imagemagick poppler-utils mkvtoolnix gpac - + poppler-utils mkvtoolnix gpac imagemagick \ + python-ox oxframe ffmpeg * Prepare Environment 1) add pandora user and set permissions @@ -44,7 +43,7 @@ To run pan.do/ra you need to install and setup: exit 3) Setup RabbitMQ -Important: "use_your_own" is a password and you have to use the same value here and below for BROKER_PASSWORD +Important: "use_your_own" is a password and you have to use the same value here and below for BROKER_URL rabbitmqctl add_user pandora use_your_own rabbitmqctl add_vhost /pandora @@ -77,7 +76,7 @@ Important: "use_your_own" is a password and you have to use the same value here } } DB_GIN_TRGM = True - BROKER_PASSWORD = "use_your_own" + BROKER_URL = 'amqp://pandora:@localhost:5672//pandora' #with apache x-sendfile or lighttpd set this to True XSENDFILE = False @@ -143,14 +142,10 @@ To update a pandora installation get the latest version from bzr by running this will update pandora/oxjs/python-ox and list possible upgrades to the db -to update your database tables, use +to update your database run: su pandora - cd /srv/pandora/pandora - ./manage.py sqldiff -a -to check if there are changes and - ./manage.py sqldiff -a | ./manage.py dbshell -to apply them. - + cd /srv/pandora + ./update.py db === Development === in one terminal: diff --git a/pandora/app/config.py b/pandora/app/config.py index 3feda184..50c7d4df 100644 --- a/pandora/app/config.py +++ b/pandora/app/config.py @@ -16,7 +16,7 @@ from django.contrib.auth.models import User import ox.jsonc from ox.utils import json -from archive.extract import supported_formats, AVCONV, avconv_version +from archive.extract import supported_formats from item.utils import get_by_id @@ -126,25 +126,24 @@ def load_config(): if f not in sformats or not sformats[f]: sys.stderr.write('''WARNING: Your configuration contains a video format "%s" that is -not supported by your version of avconv. Make sure you -dont have a local version of avconv in /usr/local/bin -and libavcodec-extra-53 and libav-tools are installed: - - sudo apt-get install libavcodec-extra-53 libav-tools +not supported by your version of ffmpeg. Make sure you +dont have a local version of ffmpeg in /usr/local/bin +and ffmpeg is installed from ppa:j/pandora: + sudo add-apt-repository ppa:j/pandora + sudo apt-get install ffmpeg ''' % f) else: sys.stderr.write('''WARNING: -You dont have "%s" installed. -To fix this on Ubuntu 12.04, run: +You dont have "ffmpeg" installed. To fix this on Ubuntu 14.04, run: - sudo apt-get install libavcodec-extra-53 libav-tools + sudo add-apt-repository ppa:j/pandora + sudo apt-get install ffmpeg check the README for further details. -''' % AVCONV) +''') - settings.AVCONV_VERSION = avconv_version() settings.CONFIG = config admin = len(settings.CONFIG['userLevels']) - 1 if not 'syncdb' in sys.argv \ diff --git a/pandora/archive/extract.py b/pandora/archive/extract.py index 88a7e832..ca27e9c2 100644 --- a/pandora/archive/extract.py +++ b/pandora/archive/extract.py @@ -11,6 +11,7 @@ import tempfile import time import math import shutil +from distutils.spawn import find_executable from glob import glob import numpy as np @@ -22,8 +23,6 @@ from django.conf import settings img_extension='jpg' -AVCONV = 'avconv' - MAX_DISTANCE = math.sqrt(3 * pow(255, 2)) @@ -50,32 +49,18 @@ class AspectRatio(fractions.Fraction): return "%d:%d" % (self.numerator, self.denominator) def supported_formats(): - p = subprocess.Popen(['which', AVCONV], - stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) - stdout, stderr = p.communicate() - if not stdout.strip(): + if not find_executable(settings.FFMPEG): return None - p = subprocess.Popen([AVCONV, '-codecs'], + p = subprocess.Popen([settings.FFMPEG, '-codecs'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) stdout, stderr = p.communicate() return { 'ogg': 'libtheora' in stdout and 'libvorbis' in stdout, 'webm': 'libvpx' in stdout and 'libvorbis' in stdout, - 'mp4': 'libx264' in stdout and 'libvo_aacenc' in stdout, + 'mp4': 'libx264' in stdout and 'DEA.L. aac' in stdout, } -def avconv_version(): - p = subprocess.Popen([AVCONV], - stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) - stdout, stderr = p.communicate() - version = stderr.split(' ')[2].split('-')[0] - try: - version = int(version.split('.')[0]) - except: - pass - return version - -def stream(video, target, profile, info, avconv=None, audio_track=0): +def stream(video, target, profile, info, audio_track=0): if not os.path.exists(target): ox.makedirs(os.path.dirname(target)) @@ -189,55 +174,13 @@ def stream(video, target, profile, info, avconv=None, audio_track=0): '-auto-alt-ref', '1', ] if format == 'mp4': - #quicktime does not support bpyramid - ''' video_settings += [ - '-vcodec', 'libx264', - '-flags', '+loop+mv4', - '-cmp', '256', - '-partitions', '+parti4x4+parti8x8+partp4x4+partp8x8+partb8x8', - '-me_method', 'hex', - '-subq', '7', - '-trellis', '1', - '-refs', '5', - '-bf', '3', - '-flags2', '+bpyramid+wpred+mixed_refs+dct8x8', - '-coder', '1', - '-me_range', '16', - '-keyint_min', '25', #FIXME: should this be related to fps? - '-sc_threshold','40', - '-i_qfactor', '0.71', - '-qmin', '10', '-qmax', '51', - '-qdiff', '4' + '-c:v', 'libx264', + '-preset:v', 'medium', + '-profile:v', 'baseline', + # does not work with avconv in Ubuntu 14.04 yet + #'-level', '3.0', ] - ''' - if settings.AVCONV_VERSION >= 9: - video_settings += [ - '-vcodec', 'libx264', - '-preset:v', 'medium', - '-profile:v', 'baseline', - # does not work with avconv in Ubuntu 14.04 yet - #'-level', '3.0', - ] - else: - video_settings += [ - '-vcodec', 'libx264', - '-flags', '+loop+mv4', - '-cmp', '256', - '-partitions', '+parti4x4+parti8x8+partp4x4+partp8x8+partb8x8', - '-me_method', 'hex', - '-subq', '7', - '-trellis', '1', - '-refs', '5', - '-bf', '0', - '-flags2', '+mixed_refs', - '-coder', '0', - '-me_range', '16', - '-sc_threshold', '40', - '-i_qfactor', '0.71', - '-qmin', '10', '-qmax', '51', - '-qdiff', '4' - ] video_settings += ['-map', '0:%s,0:0'%info['video'][0]['id']] else: video_settings = ['-vn'] @@ -271,15 +214,13 @@ def stream(video, target, profile, info, avconv=None, audio_track=0): if audiobitrate: audio_settings += ['-ab', audiobitrate] if format == 'mp4': - audio_settings += ['-acodec', 'libvo_aacenc'] + audio_settings += ['-c:a', 'aac', '-strict', '-2'] else: - audio_settings += ['-acodec', 'libvorbis'] + audio_settings += ['-c:a', 'libvorbis'] else: audio_settings = ['-an'] - if not avconv: - avconv = AVCONV - cmd = [avconv, '-y', '-i', video, '-threads', '4', '-map_metadata', '-1', '-sn'] \ + cmd = [settings.FFMPEG, '-y', '-i', video, '-threads', '4', '-map_metadata', '-1', '-sn'] \ + audio_settings \ + video_settings @@ -355,52 +296,41 @@ def frame(video, frame, position, height=128, redo=False, info=None): if redo or not exists(frame): ox.makedirs(folder) if video.endswith('.mp4'): - if settings.FFMPEG: - cmd = ffmpeg_frame_cmd(video, frame, position, height) - else: - cmd = avconv_frame_cmd(video, frame, position, height) + cmd = ffmpeg_frame_cmd(video, frame, position, height) else: cmd = ['oxframe', '-i', video, '-o', frame, '-p', str(position), '-y', str(height)] run_command(cmd) -def avconv_frame_cmd(video, frame, position, height=128): - cmd = [ - AVCONV, '-y', - '-ss', str(position), - '-i', video, - '-an', '-vframes', '1', - '-vf', 'scale=-1:%s' % height - ] - if not frame.endswith('.png'): - cmd += ['-f', 'mjpeg'] - cmd += [frame] - return cmd - def ffmpeg_frame_cmd(video, frame, position, height=128): cmd = [ settings.FFMPEG, '-y', '-ss', str(position), '-i', video, '-an', '-frames:v', '1', - '-vf', 'scale=-1:%s' % height, + '-vf', 'scale=-1:%s' % height if height else 'scale=iw*sar:ih', frame ] return cmd + +def ffmpeg_version(): + p = subprocess.Popen([settings.FFMPEG], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) + stdout, stderr = p.communicate() + version = stderr.split(' ')[2].split('-')[0] + try: + version = tuple(map(int, version.split('.'))) + except: + pass + return version + + def frame_direct(video, target, position): fdir = os.path.dirname(target) if fdir and not os.path.exists(fdir): os.makedirs(fdir) - - pre = position - 2 - if pre < 0: - pre = 0 - else: - position = 2 - cmd = [ox.file.cmd('ffmpeg'), '-y', '-ss', str(pre), '-i', video, '-ss', str(position), - '-vf', 'scale=iw*sar:ih', - '-an', '-vframes', '1', target] + cmd = ffmpeg_frame_cmd(video, target, position, None) r = run_command(cmd) return r == 0 @@ -595,13 +525,13 @@ def chop(video, start, end): ext = os.path.splitext(video)[1] choped_video = '%s/tmp%s' % (tmp, ext) cmd = [ - 'ffmpeg', + settings.FFMPEG, '-y', '-i', video, '-ss', '%.3f'%start, '-t', '%.3f'%t, - '-vcodec', 'copy', - '-acodec', 'copy', + '-c:v', 'copy', + '-c:a', 'copy', '-f', ext[1:], choped_video ] diff --git a/pandora/archive/models.py b/pandora/archive/models.py index 17d5a912..567df14e 100644 --- a/pandora/archive/models.py +++ b/pandora/archive/models.py @@ -453,9 +453,6 @@ class File(models.Model): if self.data and len(audio) > 1: config = settings.CONFIG['video'] resolution = self.stream_resolution() - ffmpeg = ox.file.cmd('ffmpeg') - if ffmpeg == 'ffmpeg': - ffmpeg = None tmp = tempfile.mkdtemp() if not self.info.get('language'): self.info['language'] = parse_language(audio[0].get('language')) @@ -471,8 +468,7 @@ class File(models.Model): n += 1 profile = '%sp.%s' % (resolution, config['formats'][0]) target = os.path.join(tmp, language + '_' + profile) - ok, error = extract.stream(media, target, profile, info, ffmpeg, - audio_track=i+1) + ok, error = extract.stream(media, target, profile, info, audio_track=i+1) if ok: tinfo = ox.avinfo(target) del tinfo['path'] @@ -676,15 +672,12 @@ class Stream(models.Model): def encode(self): media = self.source.media.path if self.source else self.file.data.path - ffmpeg = ox.file.cmd('ffmpeg') - if self.source or ffmpeg == 'ffmpeg' or self.format != 'webm': - ffmpeg = None if not self.media: self.media.name = self.path(self.name()) target = self.media.path info = ox.avinfo(media) - ok, error = extract.stream(media, target, self.name(), info, ffmpeg) + ok, error = extract.stream(media, target, self.name(), info) # file could have been moved while encoding # get current version from db and update _self = Stream.objects.get(id=self.id) diff --git a/pandora/settings.py b/pandora/settings.py index 8f87256b..6abbc25a 100644 --- a/pandora/settings.py +++ b/pandora/settings.py @@ -152,8 +152,7 @@ LOGGING = { AUTH_PROFILE_MODULE = 'user.UserProfile' AUTH_CHECK_USERNAME = True -AVCONV_VERSION = 0 -FFMPEG=False +FFMPEG='ffmpeg' #========================================================================= #Pan.do/ra related settings settings diff --git a/vm/firstboot.sh b/vm/firstboot.sh index 01c90a70..af84c01f 100755 --- a/vm/firstboot.sh +++ b/vm/firstboot.sh @@ -6,6 +6,11 @@ else ID=unknown fi UBUNTU_VERSION="$VERSION_ID" +if [ "$ID" == "debian" ]; then + SYSTEMD="yes" +else + SYSTEMD="no" +fi export DEBIAN_FRONTEND=noninteractive echo "deb http://ppa.launchpad.net/j/pandora/ubuntu trusty main" > /etc/apt/sources.list.d/j-pandora.list apt-key add - << EOF @@ -24,20 +29,6 @@ pAAGSEQ4uz6bYSeM4Q== -----END PGP PUBLIC KEY BLOCK----- EOF -if [ "$ID" == "debian" ]; then - SYSTEMD="yes" -else - SYSTEMD="no" - if [ "$UBUNTU_VERSION" == "12.04" ]; then - EXTRA=python-software-properties - else - EXTRA="" - fi - apt-get install -y \ - update-manager-core \ - software-properties-common \ - $EXTRA -fi apt-get update if [ "$LXC" == "no" ]; then @@ -46,12 +37,6 @@ apt-get install -y \ ntp fi -if [ "$UBUNTU_VERSION" == "12.04" ]; then - LIBAVCODEC_EXTRA=libavcodec-extra-53 -else - LIBAVCODEC_EXTRA=libavcodec-extra -fi - apt-get install -y \ openssh-server \ vim \ @@ -77,8 +62,7 @@ apt-get install -y \ python-html5lib \ python-ox \ oxframe \ - $LIBAVCODEC_EXTRA \ - libav-tools \ + ffmpeg \ ffmpeg2theora \ mkvtoolnix \ gpac \