encode clips

pandora/archive/extract.py

@@ -609,11 +609,14 @@ def timeline_strip(item, cuts, info, prefix):
             timeline_image.save(timeline_file)
 
 
-def chop(video, start, end, subtitles=None):
+def chop(video, start, end, subtitles=None, dest=None, encode=False):
     t = end - start
-    tmp = tempfile.mkdtemp()
     ext = os.path.splitext(video)[1]
+    if dest is None:
+        tmp = tempfile.mkdtemp()
         choped_video = '%s/tmp%s' % (tmp, ext)
+    else:
+        choped_video = dest
     if subtitles and ext == '.mp4':
         subtitles_f = choped_video + '.full.srt'
         with open(subtitles_f, 'wb') as fd:
@@ -625,25 +628,71 @@ def chop(video, start, end, subtitles=None):
         if subtitles_f:
             os.unlink(subtitles_f)
     else:
+        if encode:
+            bpp = 0.17
+            if ext == '.mp4':
+                vcodec = [
+                    '-c:v', 'libx264',
+                    '-preset:v', 'medium',
+                    '-profile:v', 'high',
+                    '-level', '4.0',
+                ]
+                acodec = [
+                    '-c:a', 'aac',
+                    '-aq', '6',
+                    '-strict', '-2'
+                ]
+            else:
+                vcodec = [
+                    '-c:v', 'libvpx',
+                    '-deadline', 'good',
+                    '-cpu-used', '0',
+                    '-lag-in-frames', '25',
+                    '-auto-alt-ref', '1',
+                ]
+                acodec = [
+                    '-c:a', 'libvorbis',
+                    '-aq', '6',
+                ]
+            info = ox.avinfo(video)
+            if not info['audio']:
+                acodec = []
+            if not info['video']:
+                vcodec = []
+            else:
+                height = info['video'][0]['height']
+                width = info['video'][0]['width']
+                fps = 30
+                bitrate = height*width*fps*bpp/1000
+                vcodec += ['-vb', '%dk' % bitrate]
+            encoding = vcodec + acodec
+        else:
+            encoding = [
+                '-c:v', 'copy',
+                '-c:a', 'copy',
+            ]
         cmd = [
             settings.FFMPEG,
             '-y',
             '-i', video,
             '-ss', '%.3f' % start,
             '-t', '%.3f' % t,
-            '-c:v', 'copy',
+        ] + encoding + [
-            '-c:a', 'copy',
             '-f', ext[1:],
             choped_video
         ]
+        print(cmd)
         p = subprocess.Popen(cmd, stdin=subprocess.PIPE,
                              stdout=open('/dev/null', 'w'),
                              stderr=open('/dev/null', 'w'),
                              close_fds=True)
         p.wait()
-    f = open(choped_video, 'rb')
-    os.unlink(choped_video)
     if subtitles_f and os.path.exists(subtitles_f):
         os.unlink(subtitles_f)
+    if dest is None:
+        f = open(choped_video, 'rb')
+        os.unlink(choped_video)
         os.rmdir(tmp)
         return f
+    else:
+        return None

pandora/archive/views.py

@@ -368,7 +368,7 @@ def direct_upload(request):
     return render_to_json_response(response)
 
 
-@login_required_json
+#@login_required_json
 def getTaskStatus(request, data):
     '''
     Gets the status for a given task

pandora/item/models.py

@@ -14,14 +14,15 @@ from glob import glob
 
 from six import PY2, string_types
 from six.moves.urllib.parse import quote
-from django.db import models, transaction, connection
+
-from django.db.models import Q, Sum, Max
 from django.conf import settings
 from django.contrib.auth import get_user_model
-
+from django.core.files.temp import NamedTemporaryFile
+from django.db import models, transaction, connection
+from django.db.models import Q, Sum, Max
 from django.db.models.signals import pre_delete
-from django.utils import datetime_safe
 from django.utils.encoding import python_2_unicode_compatible
+from django.utils import datetime_safe
 
 import ox
 from oxdjango.fields import JSONField, to_json
@@ -1182,6 +1183,37 @@ class Item(models.Model):
                     return None
                 return path
 
+    def extract_clip(self, in_, out, resolution, format, track=None, force=False):
+        streams = self.streams(track)
+        stream = streams[0].get(resolution, format)
+        if streams.count() > 1 and stream.info['duration'] < out:
+            video = NamedTemporaryFile(suffix='.%s' % format)
+            r = self.merge_streams(video.name, resolution, format)
+            if not r:
+                return False
+            path = video.name
+            duration = sum(item.cache['durations'])
+        else:
+            path = stream.media.path
+            duration = stream.info['duration']
+
+        cache_name = '%s_%sp_%s.%s' % (self.public_id, resolution, '%s,%s' % (in_, out), format)
+        cache_path = os.path.join(settings.MEDIA_ROOT, self.path('cache/%s' % cache_name))
+        if os.path.exists(cache_path) and not force:
+            return cache_path
+        if duration >= out:
+            subtitles = utils.get_by_key(settings.CONFIG['layers'], 'isSubtitles', True)
+            if subtitles:
+                srt = self.srt(subtitles['id'], encoder=ox.srt)
+                if len(srt) < 4:
+                    srt = None
+            else:
+                srt = None
+            ox.makedirs(os.path.dirname(cache_path))
+            extract.chop(path, in_, out, subtitles=srt, dest=cache_path, encode=True)
+            return cache_path
+        return False
+
     @property
     def timeline_prefix(self):
         videos = self.streams()

pandora/item/tasks.py

@@ -22,6 +22,7 @@ def cronjob(**kwargs):
     if limit_rate('item.tasks.cronjob', 8 * 60 * 60):
         update_random_sort()
         update_random_clip_sort()
+        clear_cache.delay()
 
 def update_random_sort():
     from . import models
@@ -125,6 +126,33 @@ def load_subtitles(public_id):
         item.update_sort()
         item.update_facets()
 
+
+@task(queue="encoding")
+def extract_clip(public_id, in_, out, resolution, format, track=None):
+    from . import models
+    try:
+        item = models.Item.objects.get(public_id=public_id)
+    except models.Item.DoesNotExist:
+        return False
+    if item.extract_clip(in_, out, resolution, format, track):
+        return True
+    return False
+
+
+@task(queue="encoding")
+def clear_cache(days=60):
+    import subprocess
+    path = os.path.join(settings.MEDIA_ROOT, 'media')
+    cmd = ['find', path, '-iregex', '.*/frames/.*', '-atime', '+%s' % days, '-type', 'f', '-exec', 'rm', '{}', ';']
+    subprocess.check_output(cmd)
+    path = os.path.join(settings.MEDIA_ROOT, 'items')
+    cmd = ['find', path, '-iregex', '.*/cache/.*', '-atime', '+%s' % days, '-type', 'f', '-exec', 'rm', '{}', ';']
+    subprocess.check_output(cmd)
+    path = settings.MEDIA_ROOT
+    cmd = ['find', path, '-type', 'd', '-size', '0', '-prune', '-exec', 'rmdir', '{}', ';']
+    subprocess.check_output(cmd)
+
+
 @task(ignore_results=True, queue='default')
 def update_sitemap(base_url):
     from . import models

pandora/item/views.py

@@ -638,6 +638,32 @@ def edit(request, data):
     return render_to_json_response(response)
 actions.register(edit, cache=False)
 
+
+def extractClip(request, data):
+    '''
+    Extract and cache clip
+
+    takes {
+        item: string
+        resolution: int
+        format: string
+        in: float
+        out: float
+    }
+    returns {
+        taskId: string, // taskId
+    }
+    '''
+    item = get_object_or_404_json(models.Item, public_id=data['item'])
+    if not item.access(request.user):
+        return HttpResponseForbidden()
+
+    response = json_response()
+    t = tasks.extract_clip.delay(data['item'], data['in'], data['out'], data['resolution'], data['format'])
+    response['data']['taskId'] = t.task_id
+    return render_to_json_response(response)
+actions.register(extractClip, cache=False)
+
 @login_required_json
 def remove(request, data):
     '''
@@ -1056,6 +1082,23 @@ def video(request, id, resolution, format, index=None, track=None):
         ext = '.%s' % format
         duration = stream.info['duration']
 
+        filename = u"Clip of %s - %s-%s - %s %s%s" % (
+            item.get('title'),
+            ox.format_duration(t[0] * 1000).replace(':', '.')[:-4],
+            ox.format_duration(t[1] * 1000).replace(':', '.')[:-4],
+            settings.SITENAME.replace('/', '-'),
+            item.public_id,
+            ext
+        )
+        content_type = mimetypes.guess_type(path)[0]
+
+        cache_name = '%s_%sp_%s.%s' % (item.public_id, resolution, '%s,%s' % (t[0], t[1]), format)
+        cache_path = os.path.join(settings.MEDIA_ROOT, item.path('cache/%s' % cache_name))
+        if os.path.exists(cache_path):
+            response = HttpFileResponse(cache_path, content_type=content_type)
+            response['Content-Disposition'] = "attachment; filename*=UTF-8''%s" % quote(filename.encode('utf-8'))
+            return response
+
         # multipart request beyond first part, merge parts and chop that
         if not index and streams.count() > 1 and stream.info['duration'] < t[1]:
             video = NamedTemporaryFile(suffix=ext)
@@ -1065,7 +1108,6 @@ def video(request, id, resolution, format, index=None, track=None):
             path = video.name
             duration = sum(item.cache['durations'])
 
-        content_type = mimetypes.guess_type(path)[0]
         if len(t) == 2 and t[1] > t[0] and duration >= t[1]:
             # FIXME: could be multilingual here
             subtitles = utils.get_by_key(settings.CONFIG['layers'], 'isSubtitles', True)
@@ -1076,20 +1118,12 @@ def video(request, id, resolution, format, index=None, track=None):
             else:
                 srt = None
             response = HttpResponse(extract.chop(path, t[0], t[1], subtitles=srt), content_type=content_type)
-            filename = u"Clip of %s - %s-%s - %s %s%s" % (
-                item.get('title'),
-                ox.format_duration(t[0] * 1000).replace(':', '.')[:-4],
-                ox.format_duration(t[1] * 1000).replace(':', '.')[:-4],
-                settings.SITENAME,
-                item.public_id,
-                ext
-            )
             response['Content-Disposition'] = "attachment; filename*=UTF-8''%s" % quote(filename.encode('utf-8'))
             return response
         else:
             filename = "%s - %s %s%s" % (
                 item.get('title'),
-                settings.SITENAME,
+                settings.SITENAME.replace('/', '-'),
                 item.public_id,
                 ext
             )

static/js/downloadVideoDialog.js

@@ -66,6 +66,8 @@ pandora.ui.downloadVideoDialog = function(options) {
             ]
         }).appendTo($content),
 
+        failed = false,
+
         that = Ox.Dialog({
             buttons: [
                 Ox.Button({
@@ -73,21 +75,56 @@ pandora.ui.downloadVideoDialog = function(options) {
                     title: Ox._('Download')
                 }).bindEvent({
                     click: function() {
+                        if (failed) {
                             that.close();
+                            return
+                        }
                         var values = $form.values(),
                             url
                         if (options.out) {
+                            var $screen = Ox.LoadingScreen({
+                                size: 16
+                            })
+                            that.options({content: $screen.start()});
+                            pandora.api.extractClip({
+                                item: options.item,
+                                resolution: values.resolution,
+                                format: values.format,
+                                'in': options['in'],
+                                out: options.out
+                            }, function(result) {
+                                if (result.data.taskId) {
+                                    pandora.wait(result.data.taskId, function(result) {
+                                        console.log('wait -> ', result)
+                                        if (result.data.result) {
                                             url = '/' + options.item
                                                 + '/' + values.resolution
                                                 + 'p.' + values.format
                                                 + '?t=' + options['in'] + ',' + options.out;
+                                            that.close();
+                                            document.location.href = url
+                                        } else {
+                                        }
+                                    }, 1000)
+                                } else {
+                                    that.options({content: 'Failed to extract clip.'});
+                                    that.options('buttons')[0].options({
+                                        title: Ox._('Close')
+                                    });
+                                    failed = true;
+                                }
+                            })
+
                         } else {
                             url = '/' + options.item
                                 + '/download/' + values.resolution
                                 + 'p.' + values.format
                         }
+                        if (url) {
+                            that.close();
                             document.location.href = url
                         }
+                    }
                 })
             ],
             closeButton: true,