From 9c778bb7de1f3c1efcab5b5e54962fab3f6b3ac9 Mon Sep 17 00:00:00 2001 From: j Date: Mon, 16 Oct 2023 23:26:09 +0100 Subject: [PATCH] add voice over, output 6 channels, add 5.1 mix --- management/commands/generate_clips.py | 21 ++++ render.py | 116 ++++++++++++++++-- render_kdenlive.py | 163 ++++++++++++++++++++++---- 3 files changed, 263 insertions(+), 37 deletions(-) diff --git a/management/commands/generate_clips.py b/management/commands/generate_clips.py index 3d0091e..477b1f0 100644 --- a/management/commands/generate_clips.py +++ b/management/commands/generate_clips.py @@ -1,5 +1,6 @@ import json import os +from collections import defaultdict from django.core.management.base import BaseCommand from django.conf import settings @@ -44,3 +45,23 @@ class Command(BaseCommand): with open(os.path.join(prefix, 'clips.json'), 'w') as fd: json.dump(clips, fd, indent=2, ensure_ascii=False) + + voice_over = defaultdict(dict) + for vo in item.models.Item.objects.filter( + data__type__contains="Voice Over", + ): + fragment_id = int(vo.get('title').split('_')[0]) + source = vo.files.all()[0] + batch = vo.get('batch')[0].replace('Text-', '') + src = source.data.path + target = os.path.join(prefix, 'voice_over', batch, '%s.wav' % fragment_id) + os.makedirs(os.path.dirname(target), exist_ok=True) + if os.path.exists(target): + os.unlink(target) + os.symlink(src, target) + voice_over[fragment_id][batch] = { + "src": target, + "duration": source.duration + } + with open(os.path.join(prefix, 'voice_over.json'), 'w') as fd: + json.dump(voice_over, fd, indent=2, ensure_ascii=False) diff --git a/render.py b/render.py index d8d1995..0fd5add 100644 --- a/render.py +++ b/render.py @@ -4,10 +4,11 @@ import os import subprocess import sys import time +from pathlib import Path import ox from .pi import random -from .render_kdenlive import KDEnliveProject +from .render_kdenlive import KDEnliveProject, _CACHE def random_choice(seq, items, pop=False): @@ -34,7 +35,7 @@ def chance(seq, chance): return (seq() / 10) >= chance -def compose(clips, target=150, base=1024): +def compose(clips, target=150, base=1024, voice_over=None): length = 0 scene = { 'front': { @@ -54,6 +55,30 @@ def compose(clips, target=150, base=1024): } all_clips = clips.copy() seq = random(base) + + voice_overs = [] + if voice_over: + vo_keys = list(voice_over) + if chance(seq, 0.5): + voice_overs.append(voice_over[vo_keys[chance(seq, len(vo_keys))]]) + elif len(vo_keys) >= 2: + vo1 = vo_keys.pop(chance(seq, len(vo_keys))) + vo2 = vo_keys.pop(chance(seq, len(vo_keys))) + voice_overs.append(voice_over[vo1]) + if voice_over[vo1]["duration"] + voice_over[vo2]["duration"] < target: + voice_overs.append(voice_over[vo2]) + vo_min = sum([vo['duration'] for vo in voice_overs]) + if vo_min > target: + target = vo_min + if vo_min < target: + offset = (target - vo_min) / 2 + scene['audio']['A3'].append({ + 'blank': True, + 'duration': offset + }) + for vo in voice_overs: + scene['audio']['A3'].append(vo) + while target - length > 0 and clips: clip = random_choice(seq, clips, True) if not clips: @@ -119,6 +144,7 @@ def compose(clips, target=150, base=1024): 'duration': clip['duration'], 'src': fg, }) + return scene def get_scene_duration(scene): @@ -145,11 +171,29 @@ def render(root, scene, prefix=''): with open(path, 'w') as fd: fd.write(project.to_xml()) files.append(path) + if timeline == "audio": + duration = project.get_duration() + for track, clips in data.items(): + project = KDEnliveProject(root) + for clip in clips: + project.append_clip(track, clip) + track_duration = project.get_duration() + delta = duration - track_duration + if delta > 0: + project.append_clip(track, {'blank': True, "duration": delta/24}) + path = os.path.join(root, prefix + "%s-%s.kdenlive" % (timeline, track)) + with open(path, 'w') as fd: + fd.write(project.to_xml()) + files.append(path) return files -def get_fragments(clips): +def get_fragments(clips, voice_over): import itemlist.models + import item.models + from collections import defaultdict + fragments = [] + for l in itemlist.models.List.objects.filter(status='featured').order_by('name'): if l.name.split(' ')[0].isdigit(): fragment = { @@ -157,10 +201,12 @@ def get_fragments(clips): 'tags': [t['value'] for t in l.query['conditions'][1]['conditions']], 'description': l.description } + fragment["id"] = int(fragment['name'].split(' ')[0]) fragment['clips'] = [] for clip in clips: if set(clip['tags']) & set(fragment['tags']): fragment['clips'].append(clip) + fragment["voice_over"] = voice_over.get(str(fragment["id"]), {}) fragments.append(fragment) fragments.sort(key=lambda f: ox.sort_string(f['name'])) return fragments @@ -171,17 +217,25 @@ def render_all(options): duration = int(options['duration']) base = int(options['offset']) + _cache = os.path.join(prefix, "cache.json") + if os.path.exists(_cache): + with open(_cache) as fd: + _CACHE.update(json.load(fd)) + with open(os.path.join(prefix, "clips.json")) as fd: clips = json.load(fd) - - fragments = get_fragments(clips) + with open(os.path.join(prefix, "voice_over.json")) as fd: + voice_over = json.load(fd) + fragments = get_fragments(clips, voice_over) + with open(os.path.join(prefix, "fragments.json"), "w") as fd: + json.dump(fragments, fd, indent=2, ensure_ascii=False) position = target_position = 0 target = fragment_target = duration / len(fragments) base_prefix = os.path.join(prefix, 'render', str(base)) for fragment in fragments: - n = int(fragment['name'].split(' ')[0]) + fragment_id = int(fragment['name'].split(' ')[0]) name = fragment['name'].replace(' ', '_') - if n < 10: + if fragment_id < 10: name = '0' + name if not fragment['clips']: print("skipping empty fragment", name) @@ -189,7 +243,7 @@ def render_all(options): fragment_prefix = os.path.join(base_prefix, name) os.makedirs(fragment_prefix, exist_ok=True) - scene = compose(fragment['clips'], target=target, base=base) + scene = compose(fragment['clips'], target=target, base=base, voice_over=fragment['voice_over']) scene_duration = get_scene_duration(scene) print("%s %s -> %s (%s)" % (name, target, scene_duration, fragment_target)) position += scene_duration @@ -207,22 +261,58 @@ def render_all(options): if not options['no_video']: for timeline in timelines: + print(timeline) ext = '.mp4' - if '-audio.kdenlive' in timeline: + if '/audio' in timeline: ext = '.wav' cmd = [ 'xvfb-run', '-a', 'melt', timeline, - '-consumer', 'avformat:%s' % timeline.replace('.kdenlive', ext) + '-consumer', 'avformat:%s' % timeline.replace('.kdenlive', ext), + '-quiet' ] subprocess.call(cmd) - if ext == '.wav': + if ext == '.wav' and timeline.endswith('audio.kdenlive'): cmd = [ - 'ffmpeg', '-i', + 'ffmpeg', '-y', + '-nostats', '-loglevel', 'error', + '-i', timeline.replace('.kdenlive', ext), timeline.replace('.kdenlive', '.mp4') ] subprocess.call(cmd) os.unlink(timeline.replace('.kdenlive', ext)) - print("Duration - Target: %s Actual: %s" % (target_position, position)) + fragment_prefix = Path(fragment_prefix) + cmds = [] + for src, out1, out2 in ( + ('audio-A1.wav', 'fl.wav', 'fr.wav'), + ('audio-A2.wav', 'fc.wav', 'lfe.wav'), + ('audio-A3.wav', 'bl.wav', 'br.wav'), + ): + cmds.append([ + 'ffmpeg', '-y', + '-nostats', '-loglevel', 'error', + '-i', fragment_prefix / src, + '-filter_complex', + "[0:0]pan=1|c0=c0[left]; [0:0]pan=1|c0=c1[right]", + "-map", "[left]", fragment_prefix / out1, + "-map", "[right]", fragment_prefix / out2, + ]) + cmds.append([ + 'ffmpeg', '-y', + '-nostats', '-loglevel', 'error', + '-i', fragment_prefix / "fl.wav", + '-i', fragment_prefix / "fr.wav", + '-i', fragment_prefix / "fc.wav", + '-i', fragment_prefix / "lfe.wav", + '-i', fragment_prefix / "bl.wav", + '-i', fragment_prefix / "br.wav", + '-filter_complex', "[0:a][1:a][2:a][3:a][4:a][5:a]amerge=inputs=6[a]", + "-map", "[a]", "-c:a", "aac", fragment_prefix / "audio-5.1.mp4" + ]) + for cmd in cmds: + subprocess.call(cmd) + print("Duration - Target: %s Actual: %s" % (target_position, position)) + with open(_cache, "w") as fd: + json.dump(_CACHE, fd) diff --git a/render_kdenlive.py b/render_kdenlive.py index 0bea58c..53c6b07 100644 --- a/render_kdenlive.py +++ b/render_kdenlive.py @@ -18,7 +18,7 @@ class KDEnliveProject: def to_xml(self): track = self._main_tractor.xpath(".//track")[0] - duration = max(self._duration.values()) + duration = self.get_duration() values = { "in": "0", "out": str(duration - 1) @@ -85,13 +85,85 @@ class KDEnliveProject: ["mlt_image_format", "rgba"], ["set.test_audio", "0"], ]), + a4 := self.get_element("playlist", children=[ + ["kdenlive:audio_track", "1"], + ]), + a4e := self.get_element("playlist", children=[ + ["kdenlive:audio_track", "1"], + ]), + t_a4 := self.get_element("tractor", children=[ + ["kdenlive:audio_track", "1"], + ["kdenlive:trackheight", "69"], + ["kdenlive:timeline_active", "1"], + ["kdenlive:collapsed", "0"], + ["kdenlive:thumbs_format", None], + ["kdenlive:audio_rec", None], + self.get_element("track", attrib={"hide": "video", "producer": a4.attrib["id"]}), + self.get_element("track", attrib={"hide": "video", "producer": a4e.attrib["id"]}), + self.get_element("filter", [ + ["window", "75"], + ["max_gain", "20dB"], + ["mlt_service", "volume"], + ["internal_added", "237"], + ["disable", "1"], + ]), + self.get_element("filter", [ + ["channel", "-1"], + ["mlt_service", "panner"], + ["internal_added", "237"], + ["start", "0.5"], + ["disable", "1"], + ]), + self.get_element("filter", [ + ["iec_scale", "0"], + ["mlt_service", "audiolevel"], + ["dbpeak", "1"], + ["disable", "1"], + ]), + ]), + a3 := self.get_element("playlist", children=[ + ["kdenlive:audio_track", "1"], + ]), + a3e := self.get_element("playlist", children=[ + ["kdenlive:audio_track", "1"], + ]), + t_a3 := self.get_element("tractor", children=[ + ["kdenlive:audio_track", "1"], + ["kdenlive:trackheight", "69"], + ["kdenlive:timeline_active", "1"], + ["kdenlive:collapsed", "0"], + ["kdenlive:thumbs_format", None], + ["kdenlive:audio_rec", None], + self.get_element("track", attrib={"hide": "video", "producer": a3.attrib["id"]}), + self.get_element("track", attrib={"hide": "video", "producer": a3e.attrib["id"]}), + self.get_element("filter", [ + ["window", "75"], + ["max_gain", "20dB"], + ["mlt_service", "volume"], + ["internal_added", "237"], + ["disable", "1"], + ]), + self.get_element("filter", [ + ["channel", "-1"], + ["mlt_service", "panner"], + ["internal_added", "237"], + ["start", "0.5"], + ["disable", "1"], + ]), + self.get_element("filter", [ + ["iec_scale", "0"], + ["mlt_service", "audiolevel"], + ["dbpeak", "1"], + ["disable", "1"], + ]), + ]), a2 := self.get_element("playlist", children=[ ["kdenlive:audio_track", "1"], ]), a2e := self.get_element("playlist", children=[ ["kdenlive:audio_track", "1"], ]), - t0 := self.get_element("tractor", children=[ + t_a2 := self.get_element("tractor", children=[ ["kdenlive:audio_track", "1"], ["kdenlive:trackheight", "69"], ["kdenlive:timeline_active", "1"], @@ -127,7 +199,7 @@ class KDEnliveProject: a1e := self.get_element("playlist", children=[ ["kdenlive:audio_track", "1"], ]), - t1 := self.get_element("tractor", children=[ + t_a1 := self.get_element("tractor", children=[ ["kdenlive:audio_track", "1"], ["kdenlive:trackheight", "69"], ["kdenlive:timeline_active", "1"], @@ -219,8 +291,10 @@ class KDEnliveProject: ["kdenlive:sequenceproperties.disablepreview", "0"], self.get_element("track", attrib={"producer": p0.attrib["id"]}), - self.get_element("track", attrib={"producer": t0.attrib["id"]}), - self.get_element("track", attrib={"producer": t1.attrib["id"]}), + self.get_element("track", attrib={"producer": t_a4.attrib["id"]}), + self.get_element("track", attrib={"producer": t_a3.attrib["id"]}), + self.get_element("track", attrib={"producer": t_a2.attrib["id"]}), + self.get_element("track", attrib={"producer": t_a1.attrib["id"]}), self.get_element("track", attrib={"producer": t2.attrib["id"]}), self.get_element("track", attrib={"producer": t3.attrib["id"]}), self.get_element("transition", [ @@ -246,6 +320,26 @@ class KDEnliveProject: self.get_element("transition", [ ["a_track", "0"], ["b_track", "3"], + ["mlt_service", "mix"], + ["kdenlive_id", "mix"], + ["internal_added", "237"], + ["always_active", "1"], + ["accepts_blanks", "1"], + ["sum", "1"], + ]), + self.get_element("transition", [ + ["a_track", "0"], + ["b_track", "4"], + ["mlt_service", "mix"], + ["kdenlive_id", "mix"], + ["internal_added", "237"], + ["always_active", "1"], + ["accepts_blanks", "1"], + ["sum", "1"], + ]), + self.get_element("transition", [ + ["a_track", "0"], + ["b_track", "5"], ["compositing", "0"], ["distort", "0"], ["rotate_center", "0"], @@ -258,7 +352,7 @@ class KDEnliveProject: ]), self.get_element("transition", [ ["a_track", "0"], - ["b_track", "4"], + ["b_track", "6"], ["compositing", "0"], ["distort", "0"], ["rotate_center", "0"], @@ -313,11 +407,13 @@ class KDEnliveProject: self._sequence = sequence self._main_bin = main_bin self._main_tractor = t4 - self._audio_tractor = t1 + self._audio_tractor = t_a1 self._v1 = v1 self._v2 = v2 self._a1 = a1 self._a2 = a2 + self._a3 = a3 + self._a4 = a4 def get_counter(self, prefix): self._counters[prefix] += 1 @@ -330,7 +426,7 @@ class KDEnliveProject: if file in _CACHE: out = _CACHE[file] else: - out = _CACHE[file] = subprocess.check_output(['melt', file, '-consumer', 'xml']) + out = _CACHE[file] = subprocess.check_output(['melt', file, '-consumer', 'xml']).decode() chain = lxml.etree.fromstring(out).xpath('producer')[0] chain.tag = 'chain' chain.attrib['id'] = self.get_id('chain') @@ -358,6 +454,11 @@ class KDEnliveProject: mlt_service.text = "avformat-novalidate" return chain + def get_duration(self): + if not self._duration: + return 0 + return max(self._duration.values()) + def get_element(self, tag, children=[], attrib={}, text=None): element = lxml.etree.Element(tag) if tag not in ( @@ -445,13 +546,6 @@ class KDEnliveProject: def append_clip(self, track_id, clip): - path = clip['src'] - filters = clip.get("filter", {}) - frames = int(self._fps * clip['duration']) - self._duration[track_id] += frames - #print(path, filters) - chain = self.get_chain(path) - id = get_propery(chain, "kdenlive:id") if track_id == "V1": track = self._v1 elif track_id == "V2": @@ -460,9 +554,30 @@ class KDEnliveProject: track = self._a1 elif track_id == "A2": track = self._a2 + elif track_id == "A3": + track = self._a3 + elif track_id == "A4": + track = self._a4 else: print('!!', track_id) + frames = int(self._fps * clip['duration']) + self._duration[track_id] += frames + + if clip.get("blank"): + track.append( + self.get_element("blank", attrib={ + "length": str(frames), + }) + ) + return + + path = clip['src'] + filters = clip.get("filter", {}) + #print(path, filters) + chain = self.get_chain(path) + id = get_propery(chain, "kdenlive:id") + if track_id[0] == 'A': has_audio = False for prop in chain.xpath('property'): @@ -504,14 +619,14 @@ class KDEnliveProject: ["kdenlive:id", id], ] + filters_), ) - chain = self.get_chain(path, id) - self._tree.append(chain) - self._main_bin.append( - self.get_element("entry", attrib={ - "producer": chain.attrib["id"], - "in": chain.attrib["in"], - "out": chain.attrib["out"], - }), - ) + chain = self.get_chain(path, id) + self._tree.append(chain) + self._main_bin.append( + self.get_element("entry", attrib={ + "producer": chain.attrib["id"], + "in": chain.attrib["in"], + "out": chain.attrib["out"], + }), + )