From ce423303d55a1dadb9e7c010eda0cc898160dde3 Mon Sep 17 00:00:00 2001 From: j Date: Wed, 21 Aug 2019 16:23:34 +0200 Subject: [PATCH] encode clips --- pandora/archive/extract.py | 67 +++++++++++++++++++++++++++----- pandora/archive/views.py | 2 +- pandora/item/models.py | 40 +++++++++++++++++-- pandora/item/tasks.py | 28 +++++++++++++ pandora/item/views.py | 54 ++++++++++++++++++++----- static/js/downloadVideoDialog.js | 49 ++++++++++++++++++++--- 6 files changed, 210 insertions(+), 30 deletions(-) diff --git a/pandora/archive/extract.py b/pandora/archive/extract.py index f038d474..28b19f6b 100644 --- a/pandora/archive/extract.py +++ b/pandora/archive/extract.py @@ -609,11 +609,14 @@ def timeline_strip(item, cuts, info, prefix): timeline_image.save(timeline_file) -def chop(video, start, end, subtitles=None): +def chop(video, start, end, subtitles=None, dest=None, encode=False): t = end - start - tmp = tempfile.mkdtemp() ext = os.path.splitext(video)[1] - choped_video = '%s/tmp%s' % (tmp, ext) + if dest is None: + tmp = tempfile.mkdtemp() + choped_video = '%s/tmp%s' % (tmp, ext) + else: + choped_video = dest if subtitles and ext == '.mp4': subtitles_f = choped_video + '.full.srt' with open(subtitles_f, 'wb') as fd: @@ -625,25 +628,71 @@ def chop(video, start, end, subtitles=None): if subtitles_f: os.unlink(subtitles_f) else: + if encode: + bpp = 0.17 + if ext == '.mp4': + vcodec = [ + '-c:v', 'libx264', + '-preset:v', 'medium', + '-profile:v', 'high', + '-level', '4.0', + ] + acodec = [ + '-c:a', 'aac', + '-aq', '6', + '-strict', '-2' + ] + else: + vcodec = [ + '-c:v', 'libvpx', + '-deadline', 'good', + '-cpu-used', '0', + '-lag-in-frames', '25', + '-auto-alt-ref', '1', + ] + acodec = [ + '-c:a', 'libvorbis', + '-aq', '6', + ] + info = ox.avinfo(video) + if not info['audio']: + acodec = [] + if not info['video']: + vcodec = [] + else: + height = info['video'][0]['height'] + width = info['video'][0]['width'] + fps = 30 + bitrate = height*width*fps*bpp/1000 + vcodec += ['-vb', '%dk' % bitrate] + encoding = vcodec + acodec + else: + encoding = [ + '-c:v', 'copy', + '-c:a', 'copy', + ] cmd = [ settings.FFMPEG, '-y', '-i', video, '-ss', '%.3f' % start, '-t', '%.3f' % t, - '-c:v', 'copy', - '-c:a', 'copy', + ] + encoding + [ '-f', ext[1:], choped_video ] + print(cmd) p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=open('/dev/null', 'w'), stderr=open('/dev/null', 'w'), close_fds=True) p.wait() - f = open(choped_video, 'rb') - os.unlink(choped_video) if subtitles_f and os.path.exists(subtitles_f): os.unlink(subtitles_f) - os.rmdir(tmp) - return f + if dest is None: + f = open(choped_video, 'rb') + os.unlink(choped_video) + os.rmdir(tmp) + return f + else: + return None diff --git a/pandora/archive/views.py b/pandora/archive/views.py index 62731fc8..0b8abddd 100644 --- a/pandora/archive/views.py +++ b/pandora/archive/views.py @@ -368,7 +368,7 @@ def direct_upload(request): return render_to_json_response(response) -@login_required_json +#@login_required_json def getTaskStatus(request, data): ''' Gets the status for a given task diff --git a/pandora/item/models.py b/pandora/item/models.py index 16c4b2ca..ba9f7f61 100644 --- a/pandora/item/models.py +++ b/pandora/item/models.py @@ -14,14 +14,15 @@ from glob import glob from six import PY2, string_types from six.moves.urllib.parse import quote -from django.db import models, transaction, connection -from django.db.models import Q, Sum, Max + from django.conf import settings from django.contrib.auth import get_user_model - +from django.core.files.temp import NamedTemporaryFile +from django.db import models, transaction, connection +from django.db.models import Q, Sum, Max from django.db.models.signals import pre_delete -from django.utils import datetime_safe from django.utils.encoding import python_2_unicode_compatible +from django.utils import datetime_safe import ox from oxdjango.fields import JSONField, to_json @@ -1182,6 +1183,37 @@ class Item(models.Model): return None return path + def extract_clip(self, in_, out, resolution, format, track=None, force=False): + streams = self.streams(track) + stream = streams[0].get(resolution, format) + if streams.count() > 1 and stream.info['duration'] < out: + video = NamedTemporaryFile(suffix='.%s' % format) + r = self.merge_streams(video.name, resolution, format) + if not r: + return False + path = video.name + duration = sum(item.cache['durations']) + else: + path = stream.media.path + duration = stream.info['duration'] + + cache_name = '%s_%sp_%s.%s' % (self.public_id, resolution, '%s,%s' % (in_, out), format) + cache_path = os.path.join(settings.MEDIA_ROOT, self.path('cache/%s' % cache_name)) + if os.path.exists(cache_path) and not force: + return cache_path + if duration >= out: + subtitles = utils.get_by_key(settings.CONFIG['layers'], 'isSubtitles', True) + if subtitles: + srt = self.srt(subtitles['id'], encoder=ox.srt) + if len(srt) < 4: + srt = None + else: + srt = None + ox.makedirs(os.path.dirname(cache_path)) + extract.chop(path, in_, out, subtitles=srt, dest=cache_path, encode=True) + return cache_path + return False + @property def timeline_prefix(self): videos = self.streams() diff --git a/pandora/item/tasks.py b/pandora/item/tasks.py index a0c9cac5..d43db126 100644 --- a/pandora/item/tasks.py +++ b/pandora/item/tasks.py @@ -22,6 +22,7 @@ def cronjob(**kwargs): if limit_rate('item.tasks.cronjob', 8 * 60 * 60): update_random_sort() update_random_clip_sort() + clear_cache.delay() def update_random_sort(): from . import models @@ -125,6 +126,33 @@ def load_subtitles(public_id): item.update_sort() item.update_facets() + +@task(queue="encoding") +def extract_clip(public_id, in_, out, resolution, format, track=None): + from . import models + try: + item = models.Item.objects.get(public_id=public_id) + except models.Item.DoesNotExist: + return False + if item.extract_clip(in_, out, resolution, format, track): + return True + return False + + +@task(queue="encoding") +def clear_cache(days=60): + import subprocess + path = os.path.join(settings.MEDIA_ROOT, 'media') + cmd = ['find', path, '-iregex', '.*/frames/.*', '-atime', '+%s' % days, '-type', 'f', '-exec', 'rm', '{}', ';'] + subprocess.check_output(cmd) + path = os.path.join(settings.MEDIA_ROOT, 'items') + cmd = ['find', path, '-iregex', '.*/cache/.*', '-atime', '+%s' % days, '-type', 'f', '-exec', 'rm', '{}', ';'] + subprocess.check_output(cmd) + path = settings.MEDIA_ROOT + cmd = ['find', path, '-type', 'd', '-size', '0', '-prune', '-exec', 'rmdir', '{}', ';'] + subprocess.check_output(cmd) + + @task(ignore_results=True, queue='default') def update_sitemap(base_url): from . import models diff --git a/pandora/item/views.py b/pandora/item/views.py index 02d8f924..182f7068 100644 --- a/pandora/item/views.py +++ b/pandora/item/views.py @@ -638,6 +638,32 @@ def edit(request, data): return render_to_json_response(response) actions.register(edit, cache=False) + +def extractClip(request, data): + ''' + Extract and cache clip + + takes { + item: string + resolution: int + format: string + in: float + out: float + } + returns { + taskId: string, // taskId + } + ''' + item = get_object_or_404_json(models.Item, public_id=data['item']) + if not item.access(request.user): + return HttpResponseForbidden() + + response = json_response() + t = tasks.extract_clip.delay(data['item'], data['in'], data['out'], data['resolution'], data['format']) + response['data']['taskId'] = t.task_id + return render_to_json_response(response) +actions.register(extractClip, cache=False) + @login_required_json def remove(request, data): ''' @@ -1056,6 +1082,23 @@ def video(request, id, resolution, format, index=None, track=None): ext = '.%s' % format duration = stream.info['duration'] + filename = u"Clip of %s - %s-%s - %s %s%s" % ( + item.get('title'), + ox.format_duration(t[0] * 1000).replace(':', '.')[:-4], + ox.format_duration(t[1] * 1000).replace(':', '.')[:-4], + settings.SITENAME.replace('/', '-'), + item.public_id, + ext + ) + content_type = mimetypes.guess_type(path)[0] + + cache_name = '%s_%sp_%s.%s' % (item.public_id, resolution, '%s,%s' % (t[0], t[1]), format) + cache_path = os.path.join(settings.MEDIA_ROOT, item.path('cache/%s' % cache_name)) + if os.path.exists(cache_path): + response = HttpFileResponse(cache_path, content_type=content_type) + response['Content-Disposition'] = "attachment; filename*=UTF-8''%s" % quote(filename.encode('utf-8')) + return response + # multipart request beyond first part, merge parts and chop that if not index and streams.count() > 1 and stream.info['duration'] < t[1]: video = NamedTemporaryFile(suffix=ext) @@ -1065,7 +1108,6 @@ def video(request, id, resolution, format, index=None, track=None): path = video.name duration = sum(item.cache['durations']) - content_type = mimetypes.guess_type(path)[0] if len(t) == 2 and t[1] > t[0] and duration >= t[1]: # FIXME: could be multilingual here subtitles = utils.get_by_key(settings.CONFIG['layers'], 'isSubtitles', True) @@ -1076,20 +1118,12 @@ def video(request, id, resolution, format, index=None, track=None): else: srt = None response = HttpResponse(extract.chop(path, t[0], t[1], subtitles=srt), content_type=content_type) - filename = u"Clip of %s - %s-%s - %s %s%s" % ( - item.get('title'), - ox.format_duration(t[0] * 1000).replace(':', '.')[:-4], - ox.format_duration(t[1] * 1000).replace(':', '.')[:-4], - settings.SITENAME, - item.public_id, - ext - ) response['Content-Disposition'] = "attachment; filename*=UTF-8''%s" % quote(filename.encode('utf-8')) return response else: filename = "%s - %s %s%s" % ( item.get('title'), - settings.SITENAME, + settings.SITENAME.replace('/', '-'), item.public_id, ext ) diff --git a/static/js/downloadVideoDialog.js b/static/js/downloadVideoDialog.js index b632351d..214bcd21 100644 --- a/static/js/downloadVideoDialog.js +++ b/static/js/downloadVideoDialog.js @@ -66,6 +66,8 @@ pandora.ui.downloadVideoDialog = function(options) { ] }).appendTo($content), + failed = false, + that = Ox.Dialog({ buttons: [ Ox.Button({ @@ -73,20 +75,55 @@ pandora.ui.downloadVideoDialog = function(options) { title: Ox._('Download') }).bindEvent({ click: function() { - that.close(); + if (failed) { + that.close(); + return + } var values = $form.values(), url if (options.out) { - url = '/' + options.item - + '/' + values.resolution - + 'p.' + values.format - + '?t=' + options['in'] + ',' + options.out; + var $screen = Ox.LoadingScreen({ + size: 16 + }) + that.options({content: $screen.start()}); + pandora.api.extractClip({ + item: options.item, + resolution: values.resolution, + format: values.format, + 'in': options['in'], + out: options.out + }, function(result) { + if (result.data.taskId) { + pandora.wait(result.data.taskId, function(result) { + console.log('wait -> ', result) + if (result.data.result) { + url = '/' + options.item + + '/' + values.resolution + + 'p.' + values.format + + '?t=' + options['in'] + ',' + options.out; + that.close(); + document.location.href = url + } else { + } + }, 1000) + } else { + that.options({content: 'Failed to extract clip.'}); + that.options('buttons')[0].options({ + title: Ox._('Close') + }); + failed = true; + } + }) + } else { url = '/' + options.item + '/download/' + values.resolution + 'p.' + values.format } - document.location.href = url + if (url) { + that.close(); + document.location.href = url + } } }) ],