use one version of ffmpeg for all tasks, remove avconv dependency, fixes #2606

README

@@ -3,9 +3,9 @@ pan.do/ra - open media archive
 for more information on pan.do/ra visit our website at https://pan.do/ra
 
 == SETUP ==
-pan.do/ra is known to work with Ubuntu 12.04,
+pan.do/ra is known to work with Ubuntu 14.04,
 but other distributions should also work.
-The instructions below are for Ubuntu 12.04.
+The instructions below are for Ubuntu 14.04.
 All commans given expect that you are root.
 
 To run pan.do/ra you need to install and setup:
@@ -18,7 +18,7 @@ To run pan.do/ra you need to install and setup:
 * Installing required packages
 1) add pandora ppa to get all packages in the required version
 
-    apt-get install python-software-properties
+    apt-get install software-properties-common
     add-apt-repository ppa:j/pandora
     apt-get update
 
@@ -28,9 +28,8 @@ To run pan.do/ra you need to install and setup:
             python-dev python-imaging python-numpy python-psycopg2 \
             python-geoip python-html5lib python-lxml \
             postgresql postgresql-contrib rabbitmq-server \
-            ffmpeg2theora libav-tools libavcodec-extra-53 melt \
+            poppler-utils mkvtoolnix gpac imagemagick \
-            python-ox oxframe imagemagick poppler-utils mkvtoolnix gpac
+            python-ox oxframe ffmpeg
-
 
 * Prepare Environment
 1) add pandora user and set permissions
@@ -44,7 +43,7 @@ To run pan.do/ra you need to install and setup:
    exit
 
 3) Setup RabbitMQ
-Important: "use_your_own" is a password and you have to use the same value here and below for BROKER_PASSWORD
+Important: "use_your_own" is a password and you have to use the same value here and below for BROKER_URL
 
     rabbitmqctl add_user pandora use_your_own
     rabbitmqctl add_vhost /pandora
@@ -77,7 +76,7 @@ Important: "use_your_own" is a password and you have to use the same value here
         }
     }
     DB_GIN_TRGM = True
-    BROKER_PASSWORD = "use_your_own"
+    BROKER_URL = 'amqp://pandora:<use_your_own>@localhost:5672//pandora'
 
     #with apache x-sendfile or lighttpd set this to True
     XSENDFILE = False
@@ -143,14 +142,10 @@ To update a pandora installation get the latest version from bzr by running
 
 this will update pandora/oxjs/python-ox and list possible upgrades to the db
 
-to update your database tables, use
+to update your database run:
   su pandora
-  cd /srv/pandora/pandora
+  cd /srv/pandora
-   ./manage.py sqldiff -a
+   ./update.py db
-to check if there are changes and
-   ./manage.py sqldiff -a | ./manage.py dbshell
-to apply them.
-
 
 === Development ===
   in one terminal:

pandora/app/config.py

@@ -16,7 +16,7 @@ from django.contrib.auth.models import User
 import ox.jsonc
 from ox.utils import json
 
-from archive.extract import supported_formats, AVCONV, avconv_version
+from archive.extract import supported_formats
 from item.utils import get_by_id
 
 
@@ -126,25 +126,24 @@ def load_config():
                     if f not in sformats or not sformats[f]:
                         sys.stderr.write('''WARNING:
 Your configuration contains a video format "%s" that is
-not supported by your version of avconv. Make sure you
+not supported by your version of ffmpeg. Make sure you
-dont have a local version of avconv in /usr/local/bin
+dont have a local version of ffmpeg in /usr/local/bin
-and libavcodec-extra-53 and libav-tools are installed:
+and ffmpeg is installed from ppa:j/pandora:
-
-    sudo apt-get install libavcodec-extra-53 libav-tools
 
+    sudo add-apt-repository ppa:j/pandora
+    sudo apt-get install ffmpeg
 ''' % f)
             else:
                 sys.stderr.write('''WARNING:
-You dont have "%s" installed.
+You dont have "ffmpeg" installed. To fix this on Ubuntu 14.04, run:
-To fix this on Ubuntu 12.04, run:
 
-    sudo apt-get install libavcodec-extra-53 libav-tools
+    sudo add-apt-repository ppa:j/pandora
+    sudo apt-get install ffmpeg
 
 check the README for further details.
 
-''' % AVCONV)
+''')
 
-        settings.AVCONV_VERSION = avconv_version()
         settings.CONFIG = config
         admin = len(settings.CONFIG['userLevels']) - 1
         if not 'syncdb' in sys.argv \

pandora/archive/extract.py

@@ -11,6 +11,7 @@ import tempfile
 import time
 import math
 import shutil
+from distutils.spawn import find_executable
 from glob import glob
 
 import numpy as np
@@ -22,8 +23,6 @@ from django.conf import settings
 
 img_extension='jpg'
 
-AVCONV = 'avconv'
-
 MAX_DISTANCE = math.sqrt(3 * pow(255, 2))
 
 
@@ -50,32 +49,18 @@ class AspectRatio(fractions.Fraction):
         return "%d:%d" % (self.numerator, self.denominator)
 
 def supported_formats():
-    p = subprocess.Popen(['which', AVCONV],
+    if not find_executable(settings.FFMPEG):
-            stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
-    stdout, stderr = p.communicate()
-    if not stdout.strip():
         return None
-    p = subprocess.Popen([AVCONV, '-codecs'],
+    p = subprocess.Popen([settings.FFMPEG, '-codecs'],
             stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
     stdout, stderr = p.communicate()
     return {
         'ogg': 'libtheora' in stdout and 'libvorbis' in stdout,
         'webm': 'libvpx' in stdout and 'libvorbis' in stdout,
-        'mp4': 'libx264' in stdout and 'libvo_aacenc' in stdout,
+        'mp4': 'libx264' in stdout and 'DEA.L. aac' in stdout,
     }
 
-def avconv_version():
+def stream(video, target, profile, info, audio_track=0):
-    p = subprocess.Popen([AVCONV],
-            stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
-    stdout, stderr = p.communicate()
-    version = stderr.split(' ')[2].split('-')[0]
-    try:
-        version = int(version.split('.')[0])
-    except:
-        pass
-    return version
-
-def stream(video, target, profile, info, avconv=None, audio_track=0):
     if not os.path.exists(target):
         ox.makedirs(os.path.dirname(target))
 
@@ -189,55 +174,13 @@ def stream(video, target, profile, info, avconv=None, audio_track=0):
                 '-auto-alt-ref', '1',
             ]
         if format == 'mp4':
-            #quicktime does not support bpyramid
-            '''
             video_settings += [
-                '-vcodec', 'libx264',
+                '-c:v', 'libx264',
-                '-flags', '+loop+mv4',
+                '-preset:v', 'medium',
-                '-cmp', '256',
+                '-profile:v', 'baseline',
-                '-partitions', '+parti4x4+parti8x8+partp4x4+partp8x8+partb8x8',
+                # does not work with avconv in Ubuntu 14.04 yet
-                '-me_method', 'hex',
+                #'-level', '3.0',
-                '-subq', '7',
-                '-trellis', '1',
-                '-refs', '5',
-                '-bf', '3',
-                '-flags2', '+bpyramid+wpred+mixed_refs+dct8x8',
-                '-coder', '1',
-                '-me_range', '16',
-                '-keyint_min', '25', #FIXME: should this be related to fps?
-                '-sc_threshold','40',
-                '-i_qfactor', '0.71',
-                '-qmin', '10', '-qmax', '51',
-                '-qdiff', '4'
             ]
-            '''
-            if settings.AVCONV_VERSION >= 9:
-                video_settings += [
-                    '-vcodec', 'libx264',
-                    '-preset:v', 'medium',
-                    '-profile:v', 'baseline',
-                    # does not work with avconv in Ubuntu 14.04 yet
-                    #'-level', '3.0',
-                ]
-            else:
-                video_settings += [
-                    '-vcodec', 'libx264',
-                    '-flags', '+loop+mv4',
-                    '-cmp', '256',
-                    '-partitions', '+parti4x4+parti8x8+partp4x4+partp8x8+partb8x8',
-                    '-me_method', 'hex',
-                    '-subq', '7',
-                    '-trellis', '1',
-                    '-refs', '5',
-                    '-bf', '0',
-                    '-flags2', '+mixed_refs',
-                    '-coder', '0',
-                    '-me_range', '16',
-                    '-sc_threshold', '40',
-                    '-i_qfactor', '0.71',
-                    '-qmin', '10', '-qmax', '51',
-                    '-qdiff', '4'
-                ]
         video_settings += ['-map', '0:%s,0:0'%info['video'][0]['id']]
     else:
         video_settings = ['-vn']
@@ -271,15 +214,13 @@ def stream(video, target, profile, info, avconv=None, audio_track=0):
         if audiobitrate:
             audio_settings += ['-ab', audiobitrate]
         if format == 'mp4':
-            audio_settings += ['-acodec', 'libvo_aacenc']
+            audio_settings += ['-c:a', 'aac', '-strict', '-2']
         else:
-            audio_settings += ['-acodec', 'libvorbis']
+            audio_settings += ['-c:a', 'libvorbis']
     else:
         audio_settings = ['-an']
 
-    if not avconv:
+    cmd = [settings.FFMPEG, '-y', '-i', video, '-threads', '4', '-map_metadata', '-1', '-sn'] \
-        avconv = AVCONV
-    cmd = [avconv, '-y', '-i', video, '-threads', '4', '-map_metadata', '-1', '-sn'] \
           + audio_settings \
           + video_settings
 
@@ -355,52 +296,41 @@ def frame(video, frame, position, height=128, redo=False, info=None):
         if redo or not exists(frame):
             ox.makedirs(folder)
             if video.endswith('.mp4'):
-                if settings.FFMPEG:
+                cmd = ffmpeg_frame_cmd(video, frame, position, height)
-                    cmd = ffmpeg_frame_cmd(video, frame, position, height)
-                else:
-                    cmd = avconv_frame_cmd(video, frame, position, height)
             else:
                 cmd = ['oxframe', '-i', video, '-o', frame,
                     '-p', str(position), '-y', str(height)]
             run_command(cmd)
 
-def avconv_frame_cmd(video, frame, position, height=128):
-    cmd = [
-        AVCONV, '-y',
-        '-ss', str(position),
-        '-i', video,
-        '-an', '-vframes', '1',
-        '-vf', 'scale=-1:%s' % height
-    ]
-    if not frame.endswith('.png'):
-        cmd += ['-f', 'mjpeg']
-    cmd += [frame]
-    return cmd
-
 def ffmpeg_frame_cmd(video, frame, position, height=128):
     cmd = [
         settings.FFMPEG, '-y',
         '-ss', str(position),
         '-i', video,
         '-an', '-frames:v', '1',
-        '-vf', 'scale=-1:%s' % height,
+        '-vf', 'scale=-1:%s' % height if height else 'scale=iw*sar:ih',
         frame
     ]
     return cmd
 
+
+def ffmpeg_version():
+    p = subprocess.Popen([settings.FFMPEG],
+            stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
+    stdout, stderr = p.communicate()
+    version = stderr.split(' ')[2].split('-')[0]
+    try:
+        version = tuple(map(int, version.split('.')))
+    except:
+        pass
+    return version
+
+
 def frame_direct(video, target, position):
     fdir = os.path.dirname(target)
     if fdir and not os.path.exists(fdir):
         os.makedirs(fdir)
-
+    cmd = ffmpeg_frame_cmd(video, target, position, None)
-    pre = position - 2
-    if pre < 0:
-        pre = 0
-    else:
-        position = 2
-    cmd = [ox.file.cmd('ffmpeg'), '-y', '-ss', str(pre), '-i', video, '-ss', str(position),
-            '-vf', 'scale=iw*sar:ih',
-            '-an', '-vframes', '1', target]
     r = run_command(cmd)
     return r == 0
 
@@ -595,13 +525,13 @@ def chop(video, start, end):
     ext = os.path.splitext(video)[1]
     choped_video = '%s/tmp%s' % (tmp, ext)
     cmd = [
-        'ffmpeg',
+        settings.FFMPEG,
         '-y',
         '-i', video,
         '-ss', '%.3f'%start,
         '-t', '%.3f'%t,
-        '-vcodec', 'copy',
+        '-c:v', 'copy',
-        '-acodec', 'copy',
+        '-c:a', 'copy',
         '-f', ext[1:],
         choped_video
     ]

pandora/archive/models.py

@@ -453,9 +453,6 @@ class File(models.Model):
         if self.data and len(audio) > 1:
             config = settings.CONFIG['video']
             resolution = self.stream_resolution()
-            ffmpeg = ox.file.cmd('ffmpeg')
-            if ffmpeg == 'ffmpeg':
-                ffmpeg = None
             tmp = tempfile.mkdtemp()
             if not self.info.get('language'):
                 self.info['language'] = parse_language(audio[0].get('language'))
@@ -471,8 +468,7 @@ class File(models.Model):
                     n += 1
                 profile = '%sp.%s' % (resolution, config['formats'][0])
                 target = os.path.join(tmp, language + '_' + profile)
-                ok, error = extract.stream(media, target, profile, info, ffmpeg,
+                ok, error = extract.stream(media, target, profile, info, audio_track=i+1)
-                        audio_track=i+1)
                 if ok:
                     tinfo = ox.avinfo(target)
                     del tinfo['path']
@@ -676,15 +672,12 @@ class Stream(models.Model):
 
     def encode(self):
         media = self.source.media.path if self.source else self.file.data.path
-        ffmpeg = ox.file.cmd('ffmpeg')
-        if self.source or ffmpeg == 'ffmpeg' or self.format != 'webm':
-            ffmpeg = None
 
         if not self.media:
             self.media.name = self.path(self.name())
         target = self.media.path
         info = ox.avinfo(media)
-        ok, error = extract.stream(media, target, self.name(), info, ffmpeg)
+        ok, error = extract.stream(media, target, self.name(), info)
         # file could have been moved while encoding
         # get current version from db and update
         _self = Stream.objects.get(id=self.id)

pandora/settings.py

@@ -152,8 +152,7 @@ LOGGING = {
 
 AUTH_PROFILE_MODULE = 'user.UserProfile'
 AUTH_CHECK_USERNAME = True
-AVCONV_VERSION = 0
+FFMPEG='ffmpeg'
-FFMPEG=False
 
 #=========================================================================
 #Pan.do/ra related settings settings

vm/firstboot.sh

@@ -6,6 +6,11 @@ else
     ID=unknown
 fi
 UBUNTU_VERSION="$VERSION_ID"
+if [ "$ID" == "debian" ]; then
+    SYSTEMD="yes"
+else
+    SYSTEMD="no"
+fi
 export DEBIAN_FRONTEND=noninteractive
 echo "deb http://ppa.launchpad.net/j/pandora/ubuntu trusty main" > /etc/apt/sources.list.d/j-pandora.list
 apt-key add - << EOF
@@ -24,20 +29,6 @@ pAAGSEQ4uz6bYSeM4Q==
 -----END PGP PUBLIC KEY BLOCK-----
 EOF
 
-if [ "$ID" == "debian" ]; then
-    SYSTEMD="yes"
-else
-    SYSTEMD="no"
-    if [ "$UBUNTU_VERSION" == "12.04" ]; then
-        EXTRA=python-software-properties
-    else
-        EXTRA=""
-    fi
-    apt-get install -y \
-        update-manager-core \
-        software-properties-common \
-        $EXTRA
-fi
 apt-get update
 
 if [ "$LXC" == "no" ]; then
@@ -46,12 +37,6 @@ apt-get install -y \
     ntp
 fi
 
-if [ "$UBUNTU_VERSION" == "12.04" ]; then
-    LIBAVCODEC_EXTRA=libavcodec-extra-53
-else
-    LIBAVCODEC_EXTRA=libavcodec-extra
-fi
-
 apt-get install -y \
     openssh-server \
     vim \
@@ -77,8 +62,7 @@ apt-get install -y \
     python-html5lib \
     python-ox \
     oxframe \
-    $LIBAVCODEC_EXTRA \
+    ffmpeg \
-    libav-tools \
     ffmpeg2theora \
     mkvtoolnix \
     gpac \