audio downmix

encode.py

@@ -4,7 +4,7 @@ import os
 import shutil
 import subprocess
 
-for xml in glob('output/*/*.xml'):
+for xml in sorted(glob('output/*/*.xml')):
     if xml.endswith('.audio.xml'):
         continue
     audio_xml = xml.replace('.xml', '.audio.xml')
@@ -16,6 +16,7 @@ for xml in glob('output/*/*.xml'):
         subprocess.call([
             'qmelt', xml, '-consumer', 'avformat:' + video, 'vcodec=libx264', 'strict=-2'
         ])
+        '''
         subprocess.call([
             'qmelt', audio_xml, '-consumer', 'avformat:' + audio,
         ])
@@ -33,3 +34,5 @@ for xml in glob('output/*/*.xml'):
         os.unlink(video)
         os.unlink(audio)
         shutil.move(pre, mp4)
+        '''
+        shutil.move(video, mp4)

render.py

@@ -241,29 +241,68 @@ def sequence(seq, letter):
         blank = {'blank': True, 'duration': duration - last_text}
         result['text'].append(blank)
 
+    def add_blank(track, d):
+        if track and track[-1].get('blank'):
+            track[-1]['duration'] += d
+        else:
+            blank = {'blank': True, 'duration': d}
+            track.append(blank)
+        return d
+        position += d
+
     # music
+    track = 'music'
     if letter in MUSIC:
-        position = last_music = 0
+        position = 0
         while position < duration:
             n = seq()
-            if n == 0:
+            if n < 5:
-                blank = {'blank': True, 'duration': position - last_music}
+                n = seq()
-                result['music'].append(blank)
+                position += add_blank(result[track], min(n, duration-position))
+            else:
                 n = seq()
-
                 clip = MUSIC[letter][n]
                 position += clip['duration']
-                if position > duration and result['music'][-1].get('blank'):
+                if position > duration \
-                    result['music'][-1]['duration'] -= (position-duration)
+                        and result[track][-1].get('blank') \
+                        and result[track][-1]['duration'] > clip['duration']:
+                    result[track][-1]['duration'] -= (position-duration)
+                    print('one last alignment')
                     position = duration
-                result['music'].append(clip)
+                if position <= duration:
-                last_music = position
+                    result[track].append(clip)
                 else:
-                position += n
+                    position -= clip['duration']
-        if last_music < duration:
+                    break
-            blank = {'blank': True, 'duration': duration - last_music}
+        if position < duration:
-            result['music'].append(blank)
+            position += add_blank(result[track], duration - position)
     # vocals
+    track = 'vocals'
+    if letter in VOCALS:
+        position = 0
+        while position < duration:
+            n = seq()
+            if n < 5:
+                n = seq()
+                position += add_blank(result[track], min(n, duration-position))
+            else:
+                n = seq()
+                clip = VOCALS[letter][n]
+                position += clip['duration']
+                if position > duration \
+                        and result[track][-1].get('blank') \
+                        and result[track][-1]['duration'] > clip['duration']:
+                    result[track][-1]['duration'] -= (position-duration)
+                    position = duration
+                if position <= duration:
+                    result[track].append(clip)
+                else:
+                    position -= clip['duration']
+                    break
+        if position < duration:
+            print('padding', track, duration, position)
+            position += add_blank(result[track], duration - position)
+    '''
     if letter in VOCALS:
         n = seq()
         clip = VOCALS[letter][n]
@@ -279,7 +318,13 @@ def sequence(seq, letter):
         if n != 1:
             blank = {'blank': True, 'duration': silence - silence_start}
             result['vocals'].append(blank)
+    '''
 
+    for track in result:
+        if result[track]:
+            tduration = sum([c['duration'] for c in result[track]])
+            if not abs(tduration - duration) < 0.000001:
+                raise Exception('invalid duration %s vs %s %s' % (tduration, duration, result[track]))
     return result
 
 
@@ -305,4 +350,4 @@ if __name__ == '__main__':
                 with open(tjson, 'w') as fd:
                     fd.write(current)
                 subprocess.call(['./render_mlt.py', tjson])
-                subprocess.call(['./render_audio.py', tjson])
+                #subprocess.call(['./render_audio.py', tjson])

render_audio.py

@@ -43,7 +43,6 @@ def add_clip(playlist, file_, duration):
 def add_blank(playlist, length):
     playlist.blank(length)
 
-
 for clip in data['music']:
     frames = int(clip['duration'] * fps)
     if clip.get('blank'):
@@ -58,9 +57,11 @@ for clip in data['vocals']:
     else:
         add_clip(vocals, clip['path'], frames)
 
-multitrack.connect(music, 0)
+multitrack.connect(vocals, 0)
-multitrack.connect(vocals, 1)
+multitrack.connect(music, 1)
 composite = mlt.Transition(profile, "mix")
+composite.set("start", 0.01)
+composite.set("end", 0.01)
 composite.set("combine", 1)
 
 tractor.plant_transition(composite)

render_mlt.py

@@ -28,6 +28,8 @@ with open(source) as fd:
 
 video = mlt.Playlist()
 overlay = mlt.Playlist()
+music = mlt.Playlist()
+vocals = mlt.Playlist()
 
 fps = 60
 profile = mlt.Profile("atsc_1080p_%d" % fps)
@@ -49,6 +51,14 @@ def add_clip(playlist, file_, in_, duration):
     clip.set_in_and_out(in_, in_+duration-1)
     playlist.append(clip)
 
+def add_audio_clip(playlist, file_, duration):
+    in_ = 0
+    if not isinstance(file_, str):
+        file_ = file_.encode('utf-8')
+    clip = mlt.Producer(profile, file_)
+    clip.set_in_and_out(in_, in_+duration-1)
+    playlist.append(clip)
+
 def add_blank(playlist, length):
     playlist.blank(length)
 
@@ -61,6 +71,7 @@ def add_text(playlist, value, length):
     text.set('length', length)
     playlist.append(text)
 
+
 for clip in data['clips']:
     if clip.get('black'):
         # fixme seconds to fps! duration fame etc!!
@@ -84,6 +95,20 @@ for clip in data['text']:
         frames = int(clip['duration'] * fps)
         add_text(overlay, clip['text'].upper(), frames)
 
+for clip in data['music']:
+    frames = int(clip['duration'] * fps)
+    if clip.get('blank'):
+        add_blank(music, frames)
+    else:
+        add_audio_clip(music, clip['path'], frames)
+
+for clip in data['vocals']:
+    frames = int(clip['duration'] * fps)
+    if clip.get('blank'):
+        add_blank(vocals, frames)
+    else:
+        add_audio_clip(vocals, clip['path'], frames)
+
 multitrack.connect(video, 0)
 multitrack.connect(overlay, 1)
 composite = mlt.Transition(profile, "composite")
@@ -91,8 +116,50 @@ composite = mlt.Transition(profile, "composite")
 
 tractor.plant_transition(composite)
 
+volume = mlt.Filter(profile, "volume")
+volume.set("gain", '0.01')
+tractor.plant_filter(volume)
+
+# mix vocals and music
+
+atractor = mlt.Tractor()
+atractor.mark_in = -1
+atractor.mark_out = -1
+
+audio = atractor.multitrack()
+audio.connect(vocals, 0)
+audio.connect(music, 1)
+mix = mlt.Transition(profile, "mix")
+mix.set("start", 0.20)
+mix.set("end", 0.20)
+atractor.plant_transition(mix)
+
+# mix video + audio
+dtractor = mlt.Tractor()
+dtractor.mark_in = -1
+dtractor.mark_out = -1
+dmix = dtractor.multitrack()
+dmix.connect(atractor, 0)
+dmix.connect(tractor, 1)
+
+mix2 = mlt.Transition(profile, "mix")
+mix2.set("start", 0.5)
+mix2.set("end", 0.5)
+dtractor.plant_transition(mix2)
+
+
+output = mlt.Tractor()
+tractor.mark_in = -1
+tractor.mark_out = -1
+output_tracks = output.multitrack()
+output_tracks.connect(dtractor, 0)
+
+norm = mlt.Filter(profile, "volume")
+norm.set("gain", "-12dB")
+output.plant_filter(norm)
+
 consumer = 'xml'
 consumer = mlt.Consumer(profile, consumer, target)
-consumer.connect(tractor)
+consumer.connect(output)
 #consumer.set("real_time", -2)
 consumer.start()