diff --git a/encode.py b/encode.py index 6cef8ee..51d2a58 100755 --- a/encode.py +++ b/encode.py @@ -24,40 +24,92 @@ def get_videoduration(video): def is_new(xml, mp4): if not os.path.exists(mp4): return True - xtime = os.path.getmtime(xml) - vtime = max( - os.path.getmtime(mp4), + vtime = os.path.getmtime(mp4) + xtime = max( + os.path.getmtime(xml), os.path.getmtime('text.html'), + os.path.getmtime('encode.py'), os.path.getmtime('DRONES.json'), os.path.getmtime('VOCALS.json'), ) return vtime < xtime def encode(xml, force=False): + item_json = xml.replace('.xml', '.json') audio_xml = xml.replace('.xml', '.audio.xml') + vocals_xml = xml.replace('.xml', '.vocals.xml') mp4 = xml.replace('.xml', '.mp4') mp4_480p = mp4.replace('.mp4', '.480p.mp4') - pre = mp4 + '.pre.mp4' - pre_480p = mp4_480p + '.pre.mp4' - video = mp4 + '.v.mp4' + video = mp4 + '.v.mov' + amix = mp4 + '.amix.mp4' audio = mp4 + '.wav' - if force or is_new(xml, mp4): - subprocess.call([ - 'qmelt', xml, '-consumer', 'avformat:' + video, 'vcodec=libx264', 'strict=-2' - ]) + vocals = mp4 + '.vocals.wav' + silence = 'silence_mono.wav' + left = video + '_left.wav' + right = video + '_right.wav' + + public_mp4 = 'public/' + mp4.split('/')[-1][0].lower() + mp4.split('/')[-2] + '.1080p.mp4' + public_mp4_480p = public_mp4.replace('.1080p.mp4', '.480p.mp4') + + if force or is_new(xml, public_mp4): + cmd = [ + 'qmelt', xml, '-consumer', + 'avformat:' + video, + 'vcodec=libx264', + 'acodec=pcm_s16le' + ] + subprocess.call(cmd) duration = get_videoduration(video) + cmd = [ + 'ffmpeg', '-y', '-i', video, + '-map_channel', '0.1.0', left, + '-map_channel', '0.1.1', right, + ] + subprocess.call(cmd) + cmd = [ + 'qmelt', vocals_xml, '-consumer', + 'avformat:' + vocals, + 'acodec=pcm_s16le', + 'ac=1' + ] + subprocess.call(cmd) + #for wav in (left, right, vocals): + # cmd = ['normalize-audio', wav] + # subprocess.call(cmd) + cmd = [ + 'ffmpeg', '-y', + '-i', left, # FL + '-i', right, # FR + '-i', vocals, # FC + '-i', silence, # LFE + '-i', vocals, # BL + '-i', vocals, # BR + '-filter_complex', + '[0:0][1:0][2:0][3:0][4:0][5:0] amerge=inputs=6[aout]', + '-map', "[aout]", + '-strict', '-2', + '-c:a', 'aac', + amix + ] + subprocess.call(cmd) + os.unlink(left) + os.unlink(right) + os.unlink(vocals) + cmd = [ 'ffmpeg', '-y', '-i', video, + '-i', amix, + '-t', duration, '-c:a', 'copy', '-c:v', 'copy', - '-t', duration, + '-map', '0:v:0', '-map', '1:a:0', '-movflags', '+faststart', - pre + mp4 ] subprocess.call(cmd) os.unlink(video) - shutil.move(pre, mp4) + os.unlink(amix) cmd = [ 'ffmpeg', '-y', '-i', mp4, @@ -68,14 +120,27 @@ def encode(xml, force=False): '-b:v', '750k', '-profile:v', 'high', '-movflags', '+faststart', - pre_480p + mp4_480p + ] + subprocess.call(cmd) + shutil.move(mp4, public_mp4) + shutil.move(mp4_480p, public_mp4_480p) + cmd = [ + './subtitles.py', + item_json ] subprocess.call(cmd) - shutil.move(pre_480p, mp4_480p) def encode_all(): for xml in sorted(glob('output/*/*.xml')): - if xml.endswith('.audio.xml') or xml.endswith('.vocals.xml'): + parts = xml.split('.') + if len(parts) > 2 and parts[-2] in ( + 'audio', + 'drones', + 'music', + 'source', + 'vocals', + ): continue encode(xml) diff --git a/keyword_overview.py b/keyword_overview.py index 15a53fa..9697425 100755 --- a/keyword_overview.py +++ b/keyword_overview.py @@ -76,14 +76,13 @@ for letter in sorted(KEYWORDS): print(size) print('\t', len(buckets[size]), 'clips', len(bucket_tags), 'tags', 'durations from %.3f to %.3f' % (dmin, dmax)) + used_tags = [ + '%s (%d)' % (t, bucket_tags[t]) + for t in sorted(bucket_tags, key=lambda t: (-bucket_tags[t],t)) + ] + print('\t', 'used tags:', ', '.join(used_tags)) if set(letter_tags) - set(bucket_tags): - used_tags = [ - '%s (%d)' % (t, bucket_tags[t]) - for t in sorted(bucket_tags, key=lambda t: (-bucket_tags[t],t)) - ] - print('\t', 'used tags:', ', '.join(used_tags)) - if set(letter_tags) - set(bucket_tags): - print('\t', 'missing tags:', ', '.join(sorted(set(letter_tags) - set(bucket_tags)))) + print('\t', 'missing tags:', ', '.join(sorted(set(letter_tags) - set(bucket_tags)))) for tag in sorted(known_tags): diff --git a/keywords.py b/keywords.py index f0dce4b..9573d5d 100644 --- a/keywords.py +++ b/keywords.py @@ -82,7 +82,8 @@ KEYWORDS = { "geography", "ghost", "spirit", "guerillas", - "transmission" + "transmission", + "gene z hanrahan" ], "H": [ "air-conditioner", @@ -280,7 +281,7 @@ KEYWORDS = { "first contact", "foreigner", "intercourse", - "xenophilia/xenophopia", + "xenophilia/xenophobia", ], "Y": [ "first contact", diff --git a/playout/playsync.py b/playout/playsync.py index 9a2fc29..8eb39f4 100755 --- a/playout/playsync.py +++ b/playout/playsync.py @@ -191,7 +191,7 @@ def main(): player = mpv.MPV( log_handler=my_log, input_default_bindings=True, - input_vo_keyboard=True, sub_text_font_size=28 + input_vo_keyboard=True, sub_text_font_size=28, sub_text_font='Menlo' ) player.fullscreen = not args.window player.loop = 'inf' diff --git a/render.py b/render.py index 484adb6..19b05e8 100755 --- a/render.py +++ b/render.py @@ -16,9 +16,11 @@ import ox.web.auth base_url = 'http://127.0.0.1:2620' FRAME_DURATION = 1/60 +MAX_DURATION = 40 HIDDEN_TAGS = [ - "women with white males" + "women with white males", + "gene z hanrahan" ] # items to not use at all @@ -77,9 +79,10 @@ else: if not os.path.exists('DRONES.json'): DRONES = defaultdict(list) - for letter in os.listdir('drones'): - for fn in sorted(os.listdir(os.path.join('drones', letter))): - path = os.path.join('drones', letter, fn) + prefix = 'drones_wav' + for letter in os.listdir(prefix): + for fn in sorted(os.listdir(os.path.join(prefix, letter))): + path = os.path.join(prefix, letter, fn) DRONES[letter[0]].append({ 'path': path, 'duration': ox.avinfo(path)['duration'] @@ -145,7 +148,7 @@ def get_clips(tag): clip['out'] = int(clip['out'] / FRAME_DURATION) * FRAME_DURATION clip['duration'] = clip['out'] - clip['in'] clip['tag'] = tag - clips = [clip for clip in clips if clip['duration']] + clips = [clip for clip in clips if clip['duration'] and clip['duration'] <= MAX_DURATION] for clip in clips: fduration = ox.avinfo(clip['path'])['duration'] if clip['out'] > fduration: @@ -213,7 +216,17 @@ def filter_clips(clips, duration, max_duration=0): clips_[clip['tag']].append(clip) return clips_ +def add_blank(track, d): + if track and track[-1].get('blank'): + track[-1]['duration'] += d + else: + blank = {'blank': True, 'duration': d} + track.append(blank) + return d + def sequence(seq, letter): + + tags = KEYWORDS[letter] clips = {tag: get_clips(tag) for tag in tags} all_clips = clips.copy() @@ -228,6 +241,11 @@ def sequence(seq, letter): duration = 0 MAX_DURATION = 60 * 2 + 5 MIN_DURATION = 60 * 2 - 4 + + # add 1 black frame for sync playback + duration = 1 * FRAME_DURATION + result['clips'].append({'black': True, 'duration': duration}) + while duration < MAX_DURATION and not duration >= MIN_DURATION: # clip duration: 1-10 n = seq() @@ -263,6 +281,10 @@ def sequence(seq, letter): # text overlay position = last_text = 0 tags_text = [] + + # no overlay for the first 2 frames + position = last_text = add_blank(result['text'], 2 * FRAME_DURATION) + while position < duration: n = seq() if n == 0: @@ -292,14 +314,6 @@ def sequence(seq, letter): blank = {'blank': True, 'duration': duration - last_text} result['text'].append(blank) - def add_blank(track, d): - if track and track[-1].get('blank'): - track[-1]['duration'] += d - else: - blank = {'blank': True, 'duration': d} - track.append(blank) - return d - position += d # music track = 'music' @@ -404,12 +418,12 @@ def sequence(seq, letter): if result[track]: tduration = sum([c['duration'] for c in result[track]]) if not abs(tduration - duration) < 0.000001: - raise Exception('invalid duration %s vs %s %s' % (tduration, duration, result[track])) + raise Exception('invalid duration on track: %s %s vs %s %s' % (track, tduration, duration, result[track])) return result if __name__ == '__main__': - encode = len(sys.argv) < 2 or sys.argv[1] != 'json' + render_xml = len(sys.argv) < 2 or sys.argv[1] != 'json' for n in range(10): seq = random(n * 1000) #for letter in ('T', 'W'): @@ -430,7 +444,7 @@ if __name__ == '__main__': if current != old: with open(tjson, 'w') as fd: fd.write(current) - if encode: + if render_xml: if current != old or os.path.getmtime(tjson) < os.path.getmtime('render_mlt.py'): - subprocess.call(['./render_mlt.py', tjson, 'encode']) + subprocess.call(['./render_mlt.py', tjson]) #subprocess.call(['./render_audio.py', tjson]) diff --git a/render_mlt.py b/render_mlt.py index 471f951..aa71692 100755 --- a/render_mlt.py +++ b/render_mlt.py @@ -15,63 +15,36 @@ app = QtWidgets.QApplication(sys.argv) #mlt.mlt_log_set_level(40) # verbose mlt.Factory.init() -fps = 60 -profile = mlt.Profile("atsc_1080p_%d" % fps) -#profile.set_explicit(1) - -tractor = mlt.Tractor(profile) -tractor.mark_in = -1 -tractor.mark_out = -1 - -multitrack = tractor.multitrack() - -source = sys.argv[1] -if len(sys.argv) > 2: - encode = sys.argv[2] == 'encode' -else: - encode = False - -target = source.replace('.json', '.xml') -target_audio = source.replace('.json', '.audio.xml') -target_vocals = source.replace('.json', '.vocals.xml') - -with open(source) as fd: - data = json.load(fd) - -video = mlt.Playlist() -overlay = mlt.Playlist() -music = mlt.Playlist() -vocals = mlt.Playlist() -drones0 = mlt.Playlist() -drones1 = mlt.Playlist() def add_color(playlist, color, duration): red = mlt.Producer(profile, 'color:' + color) red.set_in_and_out(0, duration) playlist.append(red) -def add_clip(playlist, file_, in_, duration): +def add_clip(playlist, clip, in_, duration): + file_ = clip['path'] if not isinstance(file_, str): file_ = file_.encode('utf-8') - ''' info = ox.avinfo(file_) tractor = mlt.Tractor(profile) tracks = tractor.multitrack() video = mlt.Playlist() - ''' - clip = mlt.Producer(profile, file_) - clip.set_in_and_out(in_, in_+duration-1) - playlist.append(clip) - ''' - video.append(clip) + c = mlt.Producer(profile, file_) + c.set_in_and_out(in_, in_+duration-1) + video.append(c) tracks.connect(video, 0) - if not not info.get('audio'): + if not info.get('audio'): audio = mlt.Playlist() add_silence(audio, duration) tracks.connect(audio, 1) - #tracks.set_in_and_out(in_, in_+duration-1) + else: + volume = mlt.Filter(profile, "volume") + if clip.get('tag', '') == 'gong': + volume.set("gain", '0.8') + else: + volume.set("gain", '0.12') + tractor.plant_filter(volume) playlist.append(tractor) - ''' def add_audio_clip(playlist, file_, duration): in_ = 0 @@ -97,13 +70,66 @@ def add_text(playlist, value, length): text.set('length', length) playlist.append(text) +def mix_audio_tracks(a, b, ratio, combine=False): + tractor = mlt.Tractor(profile) + + audio = tractor.multitrack() + audio.connect(a, 0) + audio.connect(b, 1) + mix = mlt.Transition(profile, "mix") + mix.set("start", ratio) + mix.set("end", ratio) + #mix.set("always_active", 1) + if combine: + mix.set("combine", 1) + tractor.plant_transition(mix) + return tractor + + +def save_xml(track, filename): + consumer = mlt.Consumer(profile, 'xml', filename) + consumer.connect(track) + consumer.start() + +# main +fps = 60 +profile = mlt.Profile("atsc_1080p_%d" % fps) +#profile.set_explicit(1) + +source = sys.argv[1] + +target = source.replace('.json', '.xml') +target_audio = source.replace('.json', '.audio.xml') +target_audio_wav = target_audio + '.wav' +target_vocals = source.replace('.json', '.vocals.xml') +target_music = source.replace('.json', '.music.xml') +target_drones = source.replace('.json', '.drones.xml') +target_source = source.replace('.json', '.source.xml') + +with open(source) as fd: + data = json.load(fd) + +video = mlt.Playlist() +overlay = mlt.Playlist() +music = mlt.Playlist() +vocals = mlt.Playlist() +drones0 = mlt.Playlist() +drones1 = mlt.Playlist() + +# hide Set to 1 to hide the video (make it an audio-only track), +# 2 to hide the audio (make it a video-only track), +# or 3 to hide audio and video (hidden track). +drones0.set("hide", 1) +drones1.set("hide", 1) +vocals.set("hide", 1) +music.set("hide", 1) + for clip in data['clips']: frames = int(clip['duration'] * fps) if not frames: continue if clip.get('black'): - # fixme seconds to fps! duration fame etc!! add_color(video, 'black', frames) else: #print(clip['duration'], clip['path']) @@ -112,7 +138,7 @@ for clip in data['clips']: sys.exit(1) # fixme seconds to fps! in_ = int(clip['in'] * fps) - add_clip(video, clip['path'], in_, frames) + add_clip(video, clip, in_, frames) add_color(video, 'black', 60) for clip in data['text']: @@ -149,93 +175,65 @@ for name, plist in ( for clip in data[name]: frames = int(clip['duration'] * fps) if clip.get('blank'): - add_blank(plist, frames) + add_silence(plist, frames) else: add_audio_clip(plist, clip['path'], frames) -multitrack.connect(video, 0) -multitrack.connect(overlay, 1) -composite = mlt.Transition(profile, "composite") -#composite.set('fill', 1) - -tractor.plant_transition(composite) - -volume = mlt.Filter(profile, "volume") -volume.set("gain", '0.12') -tractor.plant_filter(volume) - -def mix_audio_tracks(a, b, ratio): - tractor = mlt.Tractor(profile) - tractor.mark_in = -1 - tractor.mark_out = -1 - - audio = tractor.multitrack() - audio.connect(a, 0) - audio.connect(b, 1) - mix = mlt.Transition(profile, "mix") - mix.set("start", ratio) - mix.set("end", ratio) - #mix.set("always_active", 1) - #mix.set("combine", 1) - tractor.plant_transition(mix) - return tractor - - -consumer = 'xml' -consumer = mlt.Consumer(profile, consumer, target_vocals) -consumer.connect(vocals) -consumer.start() +save_xml(vocals, target_vocals) +video.set("hide", 1) +save_xml(video, target_source) +video.set("hide", 0) # mix drones drones = mix_audio_tracks(drones0, drones1, 0.5) +save_xml(drones, target_drones) +save_xml(music, target_music) # mix drones + music mtractor = mix_audio_tracks(drones, music, 0.3) norm = mlt.Filter(profile, "volume") +# lower volume norm.set("gain", "-12dB") mtractor.plant_filter(norm) # background and vocals -atractor = mix_audio_tracks(vocals, mtractor, 0.4) -consumer = mlt.Consumer(profile, 'xml', target_audio) -consumer.connect(atractor) -consumer.start() +# vocals are on extra track now +#atractor = mix_audio_tracks(vocals, mtractor, 0.4) +atractor = mtractor -target_audio_wav = target_audio + '.wav' +save_xml(atractor, target_audio) + +''' +''' subprocess.call([ 'qmelt', target_audio, '-consumer', 'avformat:' + target_audio_wav, ]) - audiomix = mlt.Playlist() duration = sum(clip['duration'] for clip in data['clips']) add_audio_clip(audiomix, target_audio_wav, int(duration * fps)) -# mix vocals and music -#atractor = mix_audio_tracks(vocals, mtractor, 0.20) - # mix video + audio -#dtractor = mix_audio_tracks(atractor, tractor, 0.5) -dtractor = mix_audio_tracks(audiomix, tractor, 0.29) +#tractor = mix_audio_tracks(atractor, video, 0.29) +# with vocals to background 0.4 -> 0.29 +# with vocals as extra track 0.725 +#tractor = mix_audio_tracks(audiomix, video, 0.5) +tractor = mix_audio_tracks(audiomix, video, 0.6) output = mlt.Tractor(profile) -tractor.mark_in = -1 -tractor.mark_out = -1 output_tracks = output.multitrack() -output_tracks.connect(dtractor, 0) +output_tracks.connect(tractor, 0) +output_tracks.connect(overlay, 1) norm = mlt.Filter(profile, "volume") #norm.set("gain", "-6dB") -norm.set("gain", "3dB") +norm.set("gain", "6dB") output.plant_filter(norm) -consumer = 'xml' -consumer = mlt.Consumer(profile, consumer, target) -consumer.connect(output) -#consumer.set("real_time", -2) -consumer.start() +composite = mlt.Transition(profile, "composite") +#composite.set('fill', 1) +output.plant_transition(composite) -if encode: - subprocess.call(['./encode.py', target]) +save_xml(output, target) diff --git a/subtitles.py b/subtitles.py new file mode 100755 index 0000000..513cd6c --- /dev/null +++ b/subtitles.py @@ -0,0 +1,122 @@ +#!/usr/bin/python3 +import os +import sys +import json +import subprocess +from collections import defaultdict +import string +from glob import glob +from copy import deepcopy + +import ox +import ox.web.auth + + +base_url = 'http://127.0.0.1:2620' + +FRAME_DURATION = 1/60 +MAX_DURATION = 40 + +HIDDEN_TAGS = [ + "women with white males", + "gene z hanrahan" +] + +# items to not use at all +BLACKLIST = [ + 'XN' +] + +api = None + +def get_api(): + global api + if not api: + api = ox.API(base_url + '/api/') + api.signin(**ox.web.auth.get('cdosea')) + + +def update_subtitles(): + get_api() + items = api.find({ + 'query': { + 'conditions': [{'key': 'tags', 'value': 'Vocal', 'operator': '=='}] + }, + 'keys': ['id', 'title'], + 'range': [0, 1000]})['data']['items'] + for item in items: + ''' + info = api.findMedia({ + 'query': { + 'conditions': [ + {'key': 'id', 'operator': '==', 'value': item['id']} + ] + }, + 'keys': ['id', 'extension'], + 'range': [0, 1] + })['data']['items'][0] + ''' + item['subtitles'] = api.get({'id': item['id'], 'keys': ['layers']})['data']['layers']['subtitles'] + + return items + +def get_subtitles(items, id): + for item in items: + if item['title'].startswith(id): + return deepcopy(item['subtitles']) + +def render_subtitles(item_json, output_json, output_srt): + with open(item_json) as fd: + item = json.load(fd) + + subtitles = [] + position = 0 + subs = {} + for clip in item['vocals']: + if not clip.get('blank'): + # vocals/A/A4_chaton.wav + id = clip['path'].split('/')[-1][:2] + clip_subtitles = get_subtitles(items, id) + clip_subtitles.sort(key=lambda c: (c['in'], c['out'], c['id'])) + + for sub in clip_subtitles: + sub_in = float('%0.3f'% (sub['in'] + position)) + sub_out = float('%0.3f' % (sub['out'] + position)) + sub_id = '%0.3f-%0.3f' % (sub_in, sub_out) + if sub_id not in subs: + subs[sub_id] = { + 'in': sub_in, + 'out': sub_out, + 'value': [], + 'ids': [] + } + subs[sub_id]['value'].append(sub['value'].replace('
', '')) + subs[sub_id]['ids'].append(sub['id']) + position += clip['duration'] + subs = sorted(subs.values(), key=lambda c: (c['in'], c['out'])) + for sub in subs: + sub['value'] = '\n'.join(sub['value']) + subtitles.append(sub) + + with open(output_srt, 'wb') as fd: + fd.write(ox.srt.encode(subtitles)) + with open(output_json, 'w') as fd: + json.dump(subtitles, fd, indent=4, ensure_ascii=False) + +if __name__ == '__main__': + if os.path.exists('subtitles.json'): + items = json.load(open('subtitles.json')) + else: + items = update_subtitles() + with open('subtitles.json', 'w') as fd: + json.dump(items, fd, indent=4, ensure_ascii=False) + + if len(sys.argv) > 1: + files = sys.argv[1:] + else: + files = glob('output/*/*.json') + for item_json in files: + prefix = 'public/' + item_json.split('/')[-1][0].lower() + item_json.split('/')[-2] + '.1080p.' + output_json = prefix + 'json' + output_srt = prefix + 'srt' + render_subtitles(item_json, output_json, output_srt)