add voice over, output 6 channels, add 5.1 mix

management/commands/generate_clips.py

@@ -1,5 +1,6 @@
 import json
 import os
+from collections import defaultdict
 
 from django.core.management.base import BaseCommand
 from django.conf import settings
@@ -44,3 +45,23 @@ class Command(BaseCommand):
 
         with open(os.path.join(prefix, 'clips.json'), 'w') as fd:
             json.dump(clips, fd, indent=2, ensure_ascii=False)
+
+        voice_over = defaultdict(dict)
+        for vo in item.models.Item.objects.filter(
+            data__type__contains="Voice Over",
+        ):
+            fragment_id = int(vo.get('title').split('_')[0])
+            source = vo.files.all()[0]
+            batch = vo.get('batch')[0].replace('Text-', '')
+            src = source.data.path
+            target = os.path.join(prefix, 'voice_over', batch,  '%s.wav' % fragment_id)
+            os.makedirs(os.path.dirname(target), exist_ok=True)
+            if os.path.exists(target):
+                os.unlink(target)
+            os.symlink(src, target)
+            voice_over[fragment_id][batch] = {
+                "src": target,
+                "duration": source.duration
+            }
+        with open(os.path.join(prefix, 'voice_over.json'), 'w') as fd:
+            json.dump(voice_over, fd, indent=2, ensure_ascii=False)

render.py

@@ -4,10 +4,11 @@ import os
 import subprocess
 import sys
 import time
+from pathlib import Path
 
 import ox
 from .pi import random
-from .render_kdenlive import KDEnliveProject
+from .render_kdenlive import KDEnliveProject, _CACHE
 
 
 def random_choice(seq, items, pop=False):
@@ -34,7 +35,7 @@ def chance(seq, chance):
     return (seq() / 10) >= chance
 
 
-def compose(clips, target=150, base=1024):
+def compose(clips, target=150, base=1024, voice_over=None):
     length = 0
     scene = {
         'front': {
@@ -54,6 +55,30 @@ def compose(clips, target=150, base=1024):
     }
     all_clips = clips.copy()
     seq = random(base)
+
+    voice_overs = []
+    if voice_over:
+        vo_keys = list(voice_over)
+        if chance(seq, 0.5):
+            voice_overs.append(voice_over[vo_keys[chance(seq, len(vo_keys))]])
+        elif len(vo_keys) >= 2:
+            vo1 = vo_keys.pop(chance(seq, len(vo_keys)))
+            vo2 = vo_keys.pop(chance(seq, len(vo_keys)))
+            voice_overs.append(voice_over[vo1])
+            if voice_over[vo1]["duration"] + voice_over[vo2]["duration"] < target:
+                voice_overs.append(voice_over[vo2])
+        vo_min = sum([vo['duration'] for vo in voice_overs])
+        if vo_min > target:
+            target = vo_min
+        if vo_min < target:
+            offset = (target - vo_min) / 2
+            scene['audio']['A3'].append({
+                'blank': True,
+                'duration': offset
+            })
+        for vo in voice_overs:
+            scene['audio']['A3'].append(vo)
+
     while target - length > 0 and clips:
         clip = random_choice(seq, clips, True)
         if not clips:
@@ -119,6 +144,7 @@ def compose(clips, target=150, base=1024):
             'duration': clip['duration'],
             'src': fg,
         })
+
     return scene
 
 def get_scene_duration(scene):
@@ -145,11 +171,29 @@ def render(root, scene, prefix=''):
         with open(path, 'w') as fd:
             fd.write(project.to_xml())
         files.append(path)
+        if timeline == "audio":
+            duration = project.get_duration()
+            for track, clips in data.items():
+                project = KDEnliveProject(root)
+                for clip in clips:
+                    project.append_clip(track, clip)
+                track_duration = project.get_duration()
+                delta = duration - track_duration
+                if delta > 0:
+                    project.append_clip(track, {'blank': True, "duration": delta/24})
+                path = os.path.join(root, prefix + "%s-%s.kdenlive" % (timeline, track))
+                with open(path, 'w') as fd:
+                    fd.write(project.to_xml())
+                files.append(path)
     return files
 
-def get_fragments(clips):
+def get_fragments(clips, voice_over):
     import itemlist.models
+    import item.models
+    from collections import defaultdict
+
     fragments = []
+
     for l in itemlist.models.List.objects.filter(status='featured').order_by('name'):
         if l.name.split(' ')[0].isdigit():
             fragment = {
@@ -157,10 +201,12 @@ def get_fragments(clips):
                 'tags': [t['value'] for t in l.query['conditions'][1]['conditions']],
                 'description': l.description
             }
+            fragment["id"] = int(fragment['name'].split(' ')[0])
             fragment['clips'] = []
             for clip in clips:
                 if set(clip['tags']) & set(fragment['tags']):
                     fragment['clips'].append(clip)
+            fragment["voice_over"] = voice_over.get(str(fragment["id"]), {})
             fragments.append(fragment)
     fragments.sort(key=lambda f: ox.sort_string(f['name']))
     return fragments
@@ -171,17 +217,25 @@ def render_all(options):
     duration = int(options['duration'])
     base = int(options['offset'])
 
+    _cache = os.path.join(prefix, "cache.json")
+    if os.path.exists(_cache):
+        with open(_cache) as fd:
+            _CACHE.update(json.load(fd))
+
     with open(os.path.join(prefix, "clips.json")) as fd:
         clips = json.load(fd)
-
+    with open(os.path.join(prefix, "voice_over.json")) as fd:
-    fragments = get_fragments(clips)
+        voice_over = json.load(fd)
+    fragments = get_fragments(clips, voice_over)
+    with open(os.path.join(prefix, "fragments.json"), "w") as fd:
+        json.dump(fragments, fd, indent=2, ensure_ascii=False)
     position = target_position = 0
     target = fragment_target = duration / len(fragments)
     base_prefix = os.path.join(prefix, 'render', str(base))
     for fragment in fragments:
-        n = int(fragment['name'].split(' ')[0])
+        fragment_id = int(fragment['name'].split(' ')[0])
         name = fragment['name'].replace(' ', '_')
-        if n < 10:
+        if fragment_id < 10:
             name = '0' + name
         if not fragment['clips']:
             print("skipping empty fragment", name)
@@ -189,7 +243,7 @@ def render_all(options):
         fragment_prefix = os.path.join(base_prefix, name)
         os.makedirs(fragment_prefix, exist_ok=True)
 
-        scene = compose(fragment['clips'], target=target, base=base)
+        scene = compose(fragment['clips'], target=target, base=base, voice_over=fragment['voice_over'])
         scene_duration = get_scene_duration(scene)
         print("%s %s -> %s (%s)" % (name, target, scene_duration, fragment_target))
         position += scene_duration
@@ -207,22 +261,58 @@ def render_all(options):
 
         if not options['no_video']:
             for timeline in timelines:
+                print(timeline)
                 ext = '.mp4'
-                if '-audio.kdenlive' in timeline:
+                if '/audio' in timeline:
                     ext = '.wav'
                 cmd = [
                     'xvfb-run', '-a',
                     'melt', timeline,
-                    '-consumer', 'avformat:%s' % timeline.replace('.kdenlive', ext)
+                    '-consumer', 'avformat:%s' % timeline.replace('.kdenlive', ext),
+                    '-quiet'
                 ]
                 subprocess.call(cmd)
-                if ext == '.wav':
+                if ext == '.wav' and timeline.endswith('audio.kdenlive'):
                     cmd = [
-                        'ffmpeg', '-i',
+                        'ffmpeg', '-y',
+                        '-nostats', '-loglevel', 'error',
+                        '-i',
                         timeline.replace('.kdenlive', ext),
                         timeline.replace('.kdenlive', '.mp4')
                     ]
                     subprocess.call(cmd)
                     os.unlink(timeline.replace('.kdenlive', ext))
-    print("Duration - Target: %s Actual: %s" % (target_position, position))
 
+            fragment_prefix = Path(fragment_prefix)
+            cmds = []
+            for src, out1, out2 in (
+                ('audio-A1.wav', 'fl.wav', 'fr.wav'),
+                ('audio-A2.wav', 'fc.wav', 'lfe.wav'),
+                ('audio-A3.wav', 'bl.wav', 'br.wav'),
+            ):
+                cmds.append([
+                    'ffmpeg', '-y',
+                    '-nostats', '-loglevel', 'error',
+                    '-i', fragment_prefix / src,
+                    '-filter_complex',
+                    "[0:0]pan=1|c0=c0[left]; [0:0]pan=1|c0=c1[right]",
+                    "-map", "[left]", fragment_prefix / out1,
+                    "-map", "[right]", fragment_prefix / out2,
+                ])
+                cmds.append([
+                    'ffmpeg', '-y',
+                    '-nostats', '-loglevel', 'error',
+                    '-i', fragment_prefix / "fl.wav",
+                    '-i', fragment_prefix / "fr.wav",
+                    '-i', fragment_prefix / "fc.wav",
+                    '-i', fragment_prefix / "lfe.wav",
+                    '-i', fragment_prefix / "bl.wav",
+                    '-i', fragment_prefix / "br.wav",
+                    '-filter_complex', "[0:a][1:a][2:a][3:a][4:a][5:a]amerge=inputs=6[a]",
+                    "-map", "[a]", "-c:a", "aac", fragment_prefix / "audio-5.1.mp4"
+                ])
+            for cmd in cmds:
+                subprocess.call(cmd)
+    print("Duration - Target: %s Actual: %s" % (target_position, position))
+    with open(_cache, "w") as fd:
+        json.dump(_CACHE, fd)

render_kdenlive.py

@@ -18,7 +18,7 @@ class KDEnliveProject:
 
     def to_xml(self):
         track = self._main_tractor.xpath(".//track")[0]
-        duration = max(self._duration.values())
+        duration = self.get_duration()
         values = {
             "in": "0",
             "out": str(duration - 1)
@@ -85,13 +85,85 @@ class KDEnliveProject:
                 ["mlt_image_format", "rgba"],
                 ["set.test_audio", "0"],
             ]),
+            a4 := self.get_element("playlist", children=[
+                ["kdenlive:audio_track", "1"],
+            ]),
+            a4e := self.get_element("playlist", children=[
+                ["kdenlive:audio_track", "1"],
+            ]),
+            t_a4 := self.get_element("tractor", children=[
+                ["kdenlive:audio_track", "1"],
+                ["kdenlive:trackheight", "69"],
+                ["kdenlive:timeline_active", "1"],
+                ["kdenlive:collapsed", "0"],
+                ["kdenlive:thumbs_format", None],
+                ["kdenlive:audio_rec", None],
+                self.get_element("track", attrib={"hide": "video", "producer": a4.attrib["id"]}),
+                self.get_element("track", attrib={"hide": "video", "producer": a4e.attrib["id"]}),
+                self.get_element("filter", [
+                    ["window", "75"],
+                    ["max_gain", "20dB"],
+                    ["mlt_service", "volume"],
+                    ["internal_added", "237"],
+                    ["disable", "1"],
+                ]),
+                self.get_element("filter", [
+                    ["channel", "-1"],
+                    ["mlt_service", "panner"],
+                    ["internal_added", "237"],
+                    ["start", "0.5"],
+                    ["disable", "1"],
+                ]),
+                self.get_element("filter", [
+                    ["iec_scale", "0"],
+                    ["mlt_service", "audiolevel"],
+                    ["dbpeak", "1"],
+                    ["disable", "1"],
+                ]),
+            ]),
+            a3 := self.get_element("playlist", children=[
+                ["kdenlive:audio_track", "1"],
+            ]),
+            a3e := self.get_element("playlist", children=[
+                ["kdenlive:audio_track", "1"],
+            ]),
+            t_a3 := self.get_element("tractor", children=[
+                ["kdenlive:audio_track", "1"],
+                ["kdenlive:trackheight", "69"],
+                ["kdenlive:timeline_active", "1"],
+                ["kdenlive:collapsed", "0"],
+                ["kdenlive:thumbs_format", None],
+                ["kdenlive:audio_rec", None],
+                self.get_element("track", attrib={"hide": "video", "producer": a3.attrib["id"]}),
+                self.get_element("track", attrib={"hide": "video", "producer": a3e.attrib["id"]}),
+                self.get_element("filter", [
+                    ["window", "75"],
+                    ["max_gain", "20dB"],
+                    ["mlt_service", "volume"],
+                    ["internal_added", "237"],
+                    ["disable", "1"],
+                ]),
+                self.get_element("filter", [
+                    ["channel", "-1"],
+                    ["mlt_service", "panner"],
+                    ["internal_added", "237"],
+                    ["start", "0.5"],
+                    ["disable", "1"],
+                ]),
+                self.get_element("filter", [
+                    ["iec_scale", "0"],
+                    ["mlt_service", "audiolevel"],
+                    ["dbpeak", "1"],
+                    ["disable", "1"],
+                ]),
+            ]),
             a2 := self.get_element("playlist", children=[
                 ["kdenlive:audio_track", "1"],
             ]),
             a2e := self.get_element("playlist", children=[
                 ["kdenlive:audio_track", "1"],
             ]),
-            t0 := self.get_element("tractor", children=[
+            t_a2 := self.get_element("tractor", children=[
                 ["kdenlive:audio_track", "1"],
                 ["kdenlive:trackheight", "69"],
                 ["kdenlive:timeline_active", "1"],
@@ -127,7 +199,7 @@ class KDEnliveProject:
             a1e := self.get_element("playlist", children=[
                 ["kdenlive:audio_track", "1"],
             ]),
-            t1 := self.get_element("tractor", children=[
+            t_a1 := self.get_element("tractor", children=[
                 ["kdenlive:audio_track", "1"],
                 ["kdenlive:trackheight", "69"],
                 ["kdenlive:timeline_active", "1"],
@@ -219,8 +291,10 @@ class KDEnliveProject:
                 ["kdenlive:sequenceproperties.disablepreview", "0"],
 
                 self.get_element("track", attrib={"producer": p0.attrib["id"]}),
-                self.get_element("track", attrib={"producer": t0.attrib["id"]}),
+                self.get_element("track", attrib={"producer": t_a4.attrib["id"]}),
-                self.get_element("track", attrib={"producer": t1.attrib["id"]}),
+                self.get_element("track", attrib={"producer": t_a3.attrib["id"]}),
+                self.get_element("track", attrib={"producer": t_a2.attrib["id"]}),
+                self.get_element("track", attrib={"producer": t_a1.attrib["id"]}),
                 self.get_element("track", attrib={"producer": t2.attrib["id"]}),
                 self.get_element("track", attrib={"producer": t3.attrib["id"]}),
                 self.get_element("transition", [
@@ -246,6 +320,26 @@ class KDEnliveProject:
                 self.get_element("transition", [
                     ["a_track", "0"],
                     ["b_track", "3"],
+                    ["mlt_service", "mix"],
+                    ["kdenlive_id", "mix"],
+                    ["internal_added", "237"],
+                    ["always_active", "1"],
+                    ["accepts_blanks", "1"],
+                    ["sum", "1"],
+                ]),
+                self.get_element("transition", [
+                    ["a_track", "0"],
+                    ["b_track", "4"],
+                    ["mlt_service", "mix"],
+                    ["kdenlive_id", "mix"],
+                    ["internal_added", "237"],
+                    ["always_active", "1"],
+                    ["accepts_blanks", "1"],
+                    ["sum", "1"],
+                ]),
+                self.get_element("transition", [
+                    ["a_track", "0"],
+                    ["b_track", "5"],
                     ["compositing", "0"],
                     ["distort", "0"],
                     ["rotate_center", "0"],
@@ -258,7 +352,7 @@ class KDEnliveProject:
                 ]),
                 self.get_element("transition", [
                     ["a_track", "0"],
-                    ["b_track", "4"],
+                    ["b_track", "6"],
                     ["compositing", "0"],
                     ["distort", "0"],
                     ["rotate_center", "0"],
@@ -313,11 +407,13 @@ class KDEnliveProject:
         self._sequence = sequence
         self._main_bin = main_bin
         self._main_tractor = t4
-        self._audio_tractor = t1
+        self._audio_tractor = t_a1
         self._v1 = v1
         self._v2 = v2
         self._a1 = a1
         self._a2 = a2
+        self._a3 = a3
+        self._a4 = a4
 
     def get_counter(self, prefix):
         self._counters[prefix] += 1
@@ -330,7 +426,7 @@ class KDEnliveProject:
         if file in _CACHE:
             out = _CACHE[file]
         else:
-            out = _CACHE[file] = subprocess.check_output(['melt', file, '-consumer', 'xml'])
+            out = _CACHE[file] = subprocess.check_output(['melt', file, '-consumer', 'xml']).decode()
         chain = lxml.etree.fromstring(out).xpath('producer')[0]
         chain.tag = 'chain'
         chain.attrib['id'] = self.get_id('chain')
@@ -358,6 +454,11 @@ class KDEnliveProject:
         mlt_service.text = "avformat-novalidate"
         return chain
 
+    def get_duration(self):
+        if not self._duration:
+            return 0
+        return max(self._duration.values())
+
     def get_element(self, tag, children=[], attrib={}, text=None):
         element = lxml.etree.Element(tag)
         if tag not in (
@@ -445,13 +546,6 @@ class KDEnliveProject:
 
 
     def append_clip(self, track_id, clip):
-        path = clip['src']
-        filters = clip.get("filter", {})
-        frames = int(self._fps * clip['duration'])
-        self._duration[track_id] += frames
-        #print(path, filters)
-        chain = self.get_chain(path)
-        id = get_propery(chain, "kdenlive:id")
         if track_id == "V1":
             track = self._v1
         elif track_id == "V2":
@@ -460,9 +554,30 @@ class KDEnliveProject:
             track = self._a1
         elif track_id == "A2":
             track = self._a2
+        elif track_id == "A3":
+            track = self._a3
+        elif track_id == "A4":
+            track = self._a4
         else:
             print('!!', track_id)
 
+        frames = int(self._fps * clip['duration'])
+        self._duration[track_id] += frames
+
+        if clip.get("blank"):
+            track.append(
+                self.get_element("blank", attrib={
+                    "length": str(frames),
+                })
+            )
+            return
+
+        path = clip['src']
+        filters = clip.get("filter", {})
+        #print(path, filters)
+        chain = self.get_chain(path)
+        id = get_propery(chain, "kdenlive:id")
+
         if track_id[0] == 'A':
             has_audio = False
             for prop in chain.xpath('property'):