From a407dfd1b5efaebf0a48151e5095bc7e9638bc1b Mon Sep 17 00:00:00 2001 From: j Date: Tue, 16 May 2017 12:59:51 +0000 Subject: [PATCH] 5.1 --- encode.py | 88 ++++++++++++++++++----- keyword_overview.py | 13 ++-- keywords.py | 3 +- render.py | 43 ++++++++---- render_mlt.py | 166 +++++++++++++++++++++----------------------- 5 files changed, 188 insertions(+), 125 deletions(-) diff --git a/encode.py b/encode.py index 6cef8ee..7378823 100755 --- a/encode.py +++ b/encode.py @@ -24,10 +24,11 @@ def get_videoduration(video): def is_new(xml, mp4): if not os.path.exists(mp4): return True - xtime = os.path.getmtime(xml) - vtime = max( - os.path.getmtime(mp4), + vtime = os.path.getmtime(mp4) + xtime = max( + os.path.getmtime(xml), os.path.getmtime('text.html'), + os.path.getmtime('encode.py'), os.path.getmtime('DRONES.json'), os.path.getmtime('VOCALS.json'), ) @@ -35,29 +36,76 @@ def is_new(xml, mp4): def encode(xml, force=False): audio_xml = xml.replace('.xml', '.audio.xml') + vocals_xml = xml.replace('.xml', '.vocals.xml') mp4 = xml.replace('.xml', '.mp4') mp4_480p = mp4.replace('.mp4', '.480p.mp4') - pre = mp4 + '.pre.mp4' - pre_480p = mp4_480p + '.pre.mp4' - video = mp4 + '.v.mp4' + video = mp4 + '.v.mov' + amix = mp4 + '.amix.mp4' audio = mp4 + '.wav' - if force or is_new(xml, mp4): - subprocess.call([ - 'qmelt', xml, '-consumer', 'avformat:' + video, 'vcodec=libx264', 'strict=-2' - ]) + vocals = mp4 + '.vocals.wav' + silence = 'silence_mono.wav' + left = video + '_left.wav' + right = video + '_right.wav' + + public_mp4 = 'public/' + mp4.split('/')[-1][0].lower() + mp4.split('/')[-2] + '.1080p.mp4' + public_mp4_480p = public_mp4.replace('.1080p.mp4', '.480p.mp4') + + if force or is_new(xml, public_mp4): + cmd = [ + 'qmelt', xml, '-consumer', + 'avformat:' + video, + 'vcodec=libx264', + 'acodec=pcm_s16le' + ] + subprocess.call(cmd) duration = get_videoduration(video) + cmd = [ + 'ffmpeg', '-y', '-i', video, + '-map_channel', '0.1.0', left, + '-map_channel', '0.1.1', right, + ] + subprocess.call(cmd) + cmd = [ + 'qmelt', vocals_xml, '-consumer', + 'avformat:' + vocals, + 'acodec=pcm_s16le', + 'ac=1' + ] + subprocess.call(cmd) + cmd = [ + 'ffmpeg', '-y', + '-i', left, # FL + '-i', right, # FR + '-i', vocals, # FC + '-i', silence, # LFE + '-i', vocals, # BL + '-i', vocals, # BR + '-filter_complex', + '[0:0][1:0][2:0][3:0][4:0][5:0] amerge=inputs=6[aout]', + '-map', "[aout]", + '-strict', '-2', + '-c:a', 'aac', + amix + ] + subprocess.call(cmd) + os.unlink(left) + os.unlink(right) + os.unlink(vocals) + cmd = [ 'ffmpeg', '-y', '-i', video, + '-i', amix, + '-t', duration, '-c:a', 'copy', '-c:v', 'copy', - '-t', duration, + '-map', '0:v:0', '-map', '1:a:0', '-movflags', '+faststart', - pre + mp4 ] subprocess.call(cmd) os.unlink(video) - shutil.move(pre, mp4) + os.unlink(amix) cmd = [ 'ffmpeg', '-y', '-i', mp4, @@ -68,14 +116,22 @@ def encode(xml, force=False): '-b:v', '750k', '-profile:v', 'high', '-movflags', '+faststart', - pre_480p + mp4_480p ] subprocess.call(cmd) - shutil.move(pre_480p, mp4_480p) + shutil.move(mp4, public_mp4) + shutil.move(mp4_480p, public_mp4_480p) def encode_all(): for xml in sorted(glob('output/*/*.xml')): - if xml.endswith('.audio.xml') or xml.endswith('.vocals.xml'): + parts = xml.split('.') + if len(parts) > 2 and parts[-2] in ( + 'audio', + 'drones', + 'music', + 'source', + 'vocals', + ): continue encode(xml) diff --git a/keyword_overview.py b/keyword_overview.py index 15a53fa..9697425 100755 --- a/keyword_overview.py +++ b/keyword_overview.py @@ -76,14 +76,13 @@ for letter in sorted(KEYWORDS): print(size) print('\t', len(buckets[size]), 'clips', len(bucket_tags), 'tags', 'durations from %.3f to %.3f' % (dmin, dmax)) + used_tags = [ + '%s (%d)' % (t, bucket_tags[t]) + for t in sorted(bucket_tags, key=lambda t: (-bucket_tags[t],t)) + ] + print('\t', 'used tags:', ', '.join(used_tags)) if set(letter_tags) - set(bucket_tags): - used_tags = [ - '%s (%d)' % (t, bucket_tags[t]) - for t in sorted(bucket_tags, key=lambda t: (-bucket_tags[t],t)) - ] - print('\t', 'used tags:', ', '.join(used_tags)) - if set(letter_tags) - set(bucket_tags): - print('\t', 'missing tags:', ', '.join(sorted(set(letter_tags) - set(bucket_tags)))) + print('\t', 'missing tags:', ', '.join(sorted(set(letter_tags) - set(bucket_tags)))) for tag in sorted(known_tags): diff --git a/keywords.py b/keywords.py index f0dce4b..a4cdac0 100644 --- a/keywords.py +++ b/keywords.py @@ -82,7 +82,8 @@ KEYWORDS = { "geography", "ghost", "spirit", "guerillas", - "transmission" + "transmission", + "gene z hanrahan" ], "H": [ "air-conditioner", diff --git a/render.py b/render.py index ecbe766..19b05e8 100755 --- a/render.py +++ b/render.py @@ -16,9 +16,11 @@ import ox.web.auth base_url = 'http://127.0.0.1:2620' FRAME_DURATION = 1/60 +MAX_DURATION = 40 HIDDEN_TAGS = [ - "women with white males" + "women with white males", + "gene z hanrahan" ] # items to not use at all @@ -77,7 +79,7 @@ else: if not os.path.exists('DRONES.json'): DRONES = defaultdict(list) - prefix = 'drones' + prefix = 'drones_wav' for letter in os.listdir(prefix): for fn in sorted(os.listdir(os.path.join(prefix, letter))): path = os.path.join(prefix, letter, fn) @@ -146,7 +148,7 @@ def get_clips(tag): clip['out'] = int(clip['out'] / FRAME_DURATION) * FRAME_DURATION clip['duration'] = clip['out'] - clip['in'] clip['tag'] = tag - clips = [clip for clip in clips if clip['duration']] + clips = [clip for clip in clips if clip['duration'] and clip['duration'] <= MAX_DURATION] for clip in clips: fduration = ox.avinfo(clip['path'])['duration'] if clip['out'] > fduration: @@ -214,7 +216,17 @@ def filter_clips(clips, duration, max_duration=0): clips_[clip['tag']].append(clip) return clips_ +def add_blank(track, d): + if track and track[-1].get('blank'): + track[-1]['duration'] += d + else: + blank = {'blank': True, 'duration': d} + track.append(blank) + return d + def sequence(seq, letter): + + tags = KEYWORDS[letter] clips = {tag: get_clips(tag) for tag in tags} all_clips = clips.copy() @@ -229,6 +241,11 @@ def sequence(seq, letter): duration = 0 MAX_DURATION = 60 * 2 + 5 MIN_DURATION = 60 * 2 - 4 + + # add 1 black frame for sync playback + duration = 1 * FRAME_DURATION + result['clips'].append({'black': True, 'duration': duration}) + while duration < MAX_DURATION and not duration >= MIN_DURATION: # clip duration: 1-10 n = seq() @@ -264,6 +281,10 @@ def sequence(seq, letter): # text overlay position = last_text = 0 tags_text = [] + + # no overlay for the first 2 frames + position = last_text = add_blank(result['text'], 2 * FRAME_DURATION) + while position < duration: n = seq() if n == 0: @@ -293,14 +314,6 @@ def sequence(seq, letter): blank = {'blank': True, 'duration': duration - last_text} result['text'].append(blank) - def add_blank(track, d): - if track and track[-1].get('blank'): - track[-1]['duration'] += d - else: - blank = {'blank': True, 'duration': d} - track.append(blank) - return d - position += d # music track = 'music' @@ -405,12 +418,12 @@ def sequence(seq, letter): if result[track]: tduration = sum([c['duration'] for c in result[track]]) if not abs(tduration - duration) < 0.000001: - raise Exception('invalid duration %s vs %s %s' % (tduration, duration, result[track])) + raise Exception('invalid duration on track: %s %s vs %s %s' % (track, tduration, duration, result[track])) return result if __name__ == '__main__': - encode = len(sys.argv) < 2 or sys.argv[1] != 'json' + render_xml = len(sys.argv) < 2 or sys.argv[1] != 'json' for n in range(10): seq = random(n * 1000) #for letter in ('T', 'W'): @@ -431,7 +444,7 @@ if __name__ == '__main__': if current != old: with open(tjson, 'w') as fd: fd.write(current) - if encode: + if render_xml: if current != old or os.path.getmtime(tjson) < os.path.getmtime('render_mlt.py'): - subprocess.call(['./render_mlt.py', tjson, 'encode']) + subprocess.call(['./render_mlt.py', tjson]) #subprocess.call(['./render_audio.py', tjson]) diff --git a/render_mlt.py b/render_mlt.py index 2d7c747..9103491 100755 --- a/render_mlt.py +++ b/render_mlt.py @@ -15,35 +15,6 @@ app = QtWidgets.QApplication(sys.argv) #mlt.mlt_log_set_level(40) # verbose mlt.Factory.init() -fps = 60 -profile = mlt.Profile("atsc_1080p_%d" % fps) -#profile.set_explicit(1) - -tractor = mlt.Tractor(profile) -tractor.mark_in = -1 -tractor.mark_out = -1 - -multitrack = tractor.multitrack() - -source = sys.argv[1] -if len(sys.argv) > 2: - encode = sys.argv[2] == 'encode' -else: - encode = False - -target = source.replace('.json', '.xml') -target_audio = source.replace('.json', '.audio.xml') -target_vocals = source.replace('.json', '.vocals.xml') - -with open(source) as fd: - data = json.load(fd) - -video = mlt.Playlist() -overlay = mlt.Playlist() -music = mlt.Playlist() -vocals = mlt.Playlist() -drones0 = mlt.Playlist() -drones1 = mlt.Playlist() def add_color(playlist, color, duration): red = mlt.Producer(profile, 'color:' + color) @@ -99,13 +70,66 @@ def add_text(playlist, value, length): text.set('length', length) playlist.append(text) +def mix_audio_tracks(a, b, ratio, combine=False): + tractor = mlt.Tractor(profile) + + audio = tractor.multitrack() + audio.connect(a, 0) + audio.connect(b, 1) + mix = mlt.Transition(profile, "mix") + mix.set("start", ratio) + mix.set("end", ratio) + #mix.set("always_active", 1) + if combine: + mix.set("combine", 1) + tractor.plant_transition(mix) + return tractor + + +def save_xml(track, filename): + consumer = mlt.Consumer(profile, 'xml', filename) + consumer.connect(track) + consumer.start() + +# main +fps = 60 +profile = mlt.Profile("atsc_1080p_%d" % fps) +#profile.set_explicit(1) + +source = sys.argv[1] + +target = source.replace('.json', '.xml') +target_audio = source.replace('.json', '.audio.xml') +target_audio_wav = target_audio + '.wav' +target_vocals = source.replace('.json', '.vocals.xml') +target_music = source.replace('.json', '.music.xml') +target_drones = source.replace('.json', '.drones.xml') +target_source = source.replace('.json', '.source.xml') + +with open(source) as fd: + data = json.load(fd) + +video = mlt.Playlist() +overlay = mlt.Playlist() +music = mlt.Playlist() +vocals = mlt.Playlist() +drones0 = mlt.Playlist() +drones1 = mlt.Playlist() + +# hide Set to 1 to hide the video (make it an audio-only track), +# 2 to hide the audio (make it a video-only track), +# or 3 to hide audio and video (hidden track). +drones0.set("hide", 1) +drones1.set("hide", 1) +vocals.set("hide", 1) +music.set("hide", 1) + for clip in data['clips']: frames = int(clip['duration'] * fps) if not frames: continue if clip.get('black'): - # fixme seconds to fps! duration fame etc!! add_color(video, 'black', frames) else: #print(clip['duration'], clip['path']) @@ -151,95 +175,65 @@ for name, plist in ( for clip in data[name]: frames = int(clip['duration'] * fps) if clip.get('blank'): - add_blank(plist, frames) + add_silence(plist, frames) else: add_audio_clip(plist, clip['path'], frames) -multitrack.connect(video, 0) -multitrack.connect(overlay, 1) -composite = mlt.Transition(profile, "composite") -#composite.set('fill', 1) - -tractor.plant_transition(composite) - -''' -volume = mlt.Filter(profile, "volume") -volume.set("gain", '0.12') -tractor.plant_filter(volume) -''' - -def mix_audio_tracks(a, b, ratio): - tractor = mlt.Tractor(profile) - tractor.mark_in = -1 - tractor.mark_out = -1 - - audio = tractor.multitrack() - audio.connect(a, 0) - audio.connect(b, 1) - mix = mlt.Transition(profile, "mix") - mix.set("start", ratio) - mix.set("end", ratio) - #mix.set("always_active", 1) - #mix.set("combine", 1) - tractor.plant_transition(mix) - return tractor - - -consumer = 'xml' -consumer = mlt.Consumer(profile, consumer, target_vocals) -consumer.connect(vocals) -consumer.start() +save_xml(vocals, target_vocals) +video.set("hide", 1) +save_xml(video, target_source) +video.set("hide", 0) # mix drones drones = mix_audio_tracks(drones0, drones1, 0.5) +save_xml(drones, target_drones) +save_xml(music, target_music) # mix drones + music mtractor = mix_audio_tracks(drones, music, 0.3) norm = mlt.Filter(profile, "volume") +# lower volume norm.set("gain", "-12dB") mtractor.plant_filter(norm) # background and vocals -atractor = mix_audio_tracks(vocals, mtractor, 0.4) -consumer = mlt.Consumer(profile, 'xml', target_audio) -consumer.connect(atractor) -consumer.start() +# vocals are on extra track now +#atractor = mix_audio_tracks(vocals, mtractor, 0.4) +atractor = mtractor -target_audio_wav = target_audio + '.wav' +save_xml(atractor, target_audio) + +''' +''' subprocess.call([ 'qmelt', target_audio, '-consumer', 'avformat:' + target_audio_wav, ]) - audiomix = mlt.Playlist() duration = sum(clip['duration'] for clip in data['clips']) add_audio_clip(audiomix, target_audio_wav, int(duration * fps)) -# mix vocals and music -#atractor = mix_audio_tracks(vocals, mtractor, 0.20) - # mix video + audio -#dtractor = mix_audio_tracks(atractor, tractor, 0.5) -dtractor = mix_audio_tracks(audiomix, tractor, 0.29) +#tractor = mix_audio_tracks(atractor, video, 0.29) +# with vocals to background 0.4 -> 0.29 +# with vocals as extra track 0.725 +#tractor = mix_audio_tracks(audiomix, video, 0.5) +tractor = mix_audio_tracks(audiomix, video, 0.6) output = mlt.Tractor(profile) -tractor.mark_in = -1 -tractor.mark_out = -1 output_tracks = output.multitrack() -output_tracks.connect(dtractor, 0) +output_tracks.connect(tractor, 0) +output_tracks.connect(overlay, 1) norm = mlt.Filter(profile, "volume") #norm.set("gain", "-6dB") norm.set("gain", "3dB") output.plant_filter(norm) -consumer = 'xml' -consumer = mlt.Consumer(profile, consumer, target) -consumer.connect(output) -#consumer.set("real_time", -2) -consumer.start() +composite = mlt.Transition(profile, "composite") +#composite.set('fill', 1) +output.plant_transition(composite) -if encode: - subprocess.call(['./encode.py', target]) +save_xml(output, target)