0x2620/pandora_t_for_time
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

add voice over, output 6 channels, add 5.1 mix

Browse source
This commit is contained in:
j 2023-10-16 23:26:09 +01:00
parent 033fe8b2b5
commit 9c778bb7de
3 changed files with 263 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
management/commands/generate_clips.py
View file

@ -1,5 +1,6 @@
import json
import os
from collections import defaultdict
from django.core.management.base import BaseCommand
from django.conf import settings
@ -44,3 +45,23 @@ class Command(BaseCommand):
with open(os.path.join(prefix, 'clips.json'), 'w') as fd:
json.dump(clips, fd, indent=2, ensure_ascii=False)
voice_over = defaultdict(dict)
for vo in item.models.Item.objects.filter(
data__type__contains="Voice Over",
):
fragment_id = int(vo.get('title').split('_')[0])
source = vo.files.all()[0]
batch = vo.get('batch')[0].replace('Text-', '')
src = source.data.path
target = os.path.join(prefix, 'voice_over', batch, '%s.wav' % fragment_id)
os.makedirs(os.path.dirname(target), exist_ok=True)
if os.path.exists(target):
os.unlink(target)
os.symlink(src, target)
voice_over[fragment_id][batch] = {
"src": target,
"duration": source.duration
}
with open(os.path.join(prefix, 'voice_over.json'), 'w') as fd:
json.dump(voice_over, fd, indent=2, ensure_ascii=False)

116
render.py
View file

@ -4,10 +4,11 @@ import os
import subprocess
import sys
import time
from pathlib import Path
import ox
from .pi import random
from .render_kdenlive import KDEnliveProject
from .render_kdenlive import KDEnliveProject, _CACHE
def random_choice(seq, items, pop=False):
@ -34,7 +35,7 @@ def chance(seq, chance):
return (seq() / 10) >= chance
def compose(clips, target=150, base=1024):
def compose(clips, target=150, base=1024, voice_over=None):
length = 0
scene = {
'front': {
@ -54,6 +55,30 @@ def compose(clips, target=150, base=1024):
}
all_clips = clips.copy()
seq = random(base)
voice_overs = []
if voice_over:
vo_keys = list(voice_over)
if chance(seq, 0.5):
voice_overs.append(voice_over[vo_keys[chance(seq, len(vo_keys))]])
elif len(vo_keys) >= 2:
vo1 = vo_keys.pop(chance(seq, len(vo_keys)))
vo2 = vo_keys.pop(chance(seq, len(vo_keys)))
voice_overs.append(voice_over[vo1])
if voice_over[vo1]["duration"] + voice_over[vo2]["duration"] < target:
voice_overs.append(voice_over[vo2])
vo_min = sum([vo['duration'] for vo in voice_overs])
if vo_min > target:
target = vo_min
if vo_min < target:
offset = (target - vo_min) / 2
scene['audio']['A3'].append({
'blank': True,
'duration': offset
})
for vo in voice_overs:
scene['audio']['A3'].append(vo)
while target - length > 0 and clips:
clip = random_choice(seq, clips, True)
if not clips:
@ -119,6 +144,7 @@ def compose(clips, target=150, base=1024):
'duration': clip['duration'],
'src': fg,
})
return scene
def get_scene_duration(scene):
@ -145,11 +171,29 @@ def render(root, scene, prefix=''):
with open(path, 'w') as fd:
fd.write(project.to_xml())
files.append(path)
if timeline == "audio":
duration = project.get_duration()
for track, clips in data.items():
project = KDEnliveProject(root)
for clip in clips:
project.append_clip(track, clip)
track_duration = project.get_duration()
delta = duration - track_duration
if delta > 0:
project.append_clip(track, {'blank': True, "duration": delta/24})
path = os.path.join(root, prefix + "%s-%s.kdenlive" % (timeline, track))
with open(path, 'w') as fd:
fd.write(project.to_xml())
files.append(path)
return files
def get_fragments(clips):
def get_fragments(clips, voice_over):
import itemlist.models
import item.models
from collections import defaultdict
fragments = []
for l in itemlist.models.List.objects.filter(status='featured').order_by('name'):
if l.name.split(' ')[0].isdigit():
fragment = {
@ -157,10 +201,12 @@ def get_fragments(clips):
'tags': [t['value'] for t in l.query['conditions'][1]['conditions']],
'description': l.description
}
fragment["id"] = int(fragment['name'].split(' ')[0])
fragment['clips'] = []
for clip in clips:
if set(clip['tags']) & set(fragment['tags']):
fragment['clips'].append(clip)
fragment["voice_over"] = voice_over.get(str(fragment["id"]), {})
fragments.append(fragment)
fragments.sort(key=lambda f: ox.sort_string(f['name']))
return fragments
@ -171,17 +217,25 @@ def render_all(options):
duration = int(options['duration'])
base = int(options['offset'])
_cache = os.path.join(prefix, "cache.json")
if os.path.exists(_cache):
with open(_cache) as fd:
_CACHE.update(json.load(fd))
with open(os.path.join(prefix, "clips.json")) as fd:
clips = json.load(fd)
fragments = get_fragments(clips)
with open(os.path.join(prefix, "voice_over.json")) as fd:
voice_over = json.load(fd)
fragments = get_fragments(clips, voice_over)
with open(os.path.join(prefix, "fragments.json"), "w") as fd:
json.dump(fragments, fd, indent=2, ensure_ascii=False)
position = target_position = 0
target = fragment_target = duration / len(fragments)
base_prefix = os.path.join(prefix, 'render', str(base))
for fragment in fragments:
n = int(fragment['name'].split(' ')[0])
fragment_id = int(fragment['name'].split(' ')[0])
name = fragment['name'].replace(' ', '_')
if n < 10:
if fragment_id < 10:
name = '0' + name
if not fragment['clips']:
print("skipping empty fragment", name)
@ -189,7 +243,7 @@ def render_all(options):
fragment_prefix = os.path.join(base_prefix, name)
os.makedirs(fragment_prefix, exist_ok=True)
scene = compose(fragment['clips'], target=target, base=base)
scene = compose(fragment['clips'], target=target, base=base, voice_over=fragment['voice_over'])
scene_duration = get_scene_duration(scene)
print("%s %s -> %s (%s)" % (name, target, scene_duration, fragment_target))
position += scene_duration
@ -207,22 +261,58 @@ def render_all(options):
if not options['no_video']:
for timeline in timelines:
print(timeline)
ext = '.mp4'
if '-audio.kdenlive' in timeline:
if '/audio' in timeline:
ext = '.wav'
cmd = [
'xvfb-run', '-a',
'melt', timeline,
'-consumer', 'avformat:%s' % timeline.replace('.kdenlive', ext)
'-consumer', 'avformat:%s' % timeline.replace('.kdenlive', ext),
'-quiet'
]
subprocess.call(cmd)
if ext == '.wav':
if ext == '.wav' and timeline.endswith('audio.kdenlive'):
cmd = [
'ffmpeg', '-i',
'ffmpeg', '-y',
'-nostats', '-loglevel', 'error',
'-i',
timeline.replace('.kdenlive', ext),
timeline.replace('.kdenlive', '.mp4')
]
subprocess.call(cmd)
os.unlink(timeline.replace('.kdenlive', ext))
print("Duration - Target: %s Actual: %s" % (target_position, position))
fragment_prefix = Path(fragment_prefix)
cmds = []
for src, out1, out2 in (
('audio-A1.wav', 'fl.wav', 'fr.wav'),
('audio-A2.wav', 'fc.wav', 'lfe.wav'),
('audio-A3.wav', 'bl.wav', 'br.wav'),
):
cmds.append([
'ffmpeg', '-y',
'-nostats', '-loglevel', 'error',
'-i', fragment_prefix / src,
'-filter_complex',
"[0:0]pan=1|c0=c0[left]; [0:0]pan=1|c0=c1[right]",
"-map", "[left]", fragment_prefix / out1,
"-map", "[right]", fragment_prefix / out2,
])
cmds.append([
'ffmpeg', '-y',
'-nostats', '-loglevel', 'error',
'-i', fragment_prefix / "fl.wav",
'-i', fragment_prefix / "fr.wav",
'-i', fragment_prefix / "fc.wav",
'-i', fragment_prefix / "lfe.wav",
'-i', fragment_prefix / "bl.wav",
'-i', fragment_prefix / "br.wav",
'-filter_complex', "[0:a][1:a][2:a][3:a][4:a][5:a]amerge=inputs=6[a]",
"-map", "[a]", "-c:a", "aac", fragment_prefix / "audio-5.1.mp4"
])
for cmd in cmds:
subprocess.call(cmd)
print("Duration - Target: %s Actual: %s" % (target_position, position))
with open(_cache, "w") as fd:
json.dump(_CACHE, fd)

163
render_kdenlive.py
View file

@ -18,7 +18,7 @@ class KDEnliveProject:
def to_xml(self):
track = self._main_tractor.xpath(".//track")[0]
duration = max(self._duration.values())
duration = self.get_duration()
values = {
"in": "0",
"out": str(duration - 1)
@ -85,13 +85,85 @@ class KDEnliveProject:
["mlt_image_format", "rgba"],
["set.test_audio", "0"],
]),
a4 := self.get_element("playlist", children=[
["kdenlive:audio_track", "1"],
]),
a4e := self.get_element("playlist", children=[
["kdenlive:audio_track", "1"],
]),
t_a4 := self.get_element("tractor", children=[
["kdenlive:audio_track", "1"],
["kdenlive:trackheight", "69"],
["kdenlive:timeline_active", "1"],
["kdenlive:collapsed", "0"],
["kdenlive:thumbs_format", None],
["kdenlive:audio_rec", None],
self.get_element("track", attrib={"hide": "video", "producer": a4.attrib["id"]}),
self.get_element("track", attrib={"hide": "video", "producer": a4e.attrib["id"]}),
self.get_element("filter", [
["window", "75"],
["max_gain", "20dB"],
["mlt_service", "volume"],
["internal_added", "237"],
["disable", "1"],
]),
self.get_element("filter", [
["channel", "-1"],
["mlt_service", "panner"],
["internal_added", "237"],
["start", "0.5"],
["disable", "1"],
]),
self.get_element("filter", [
["iec_scale", "0"],
["mlt_service", "audiolevel"],
["dbpeak", "1"],
["disable", "1"],
]),
]),
a3 := self.get_element("playlist", children=[
["kdenlive:audio_track", "1"],
]),
a3e := self.get_element("playlist", children=[
["kdenlive:audio_track", "1"],
]),
t_a3 := self.get_element("tractor", children=[
["kdenlive:audio_track", "1"],
["kdenlive:trackheight", "69"],
["kdenlive:timeline_active", "1"],
["kdenlive:collapsed", "0"],
["kdenlive:thumbs_format", None],
["kdenlive:audio_rec", None],
self.get_element("track", attrib={"hide": "video", "producer": a3.attrib["id"]}),
self.get_element("track", attrib={"hide": "video", "producer": a3e.attrib["id"]}),
self.get_element("filter", [
["window", "75"],
["max_gain", "20dB"],
["mlt_service", "volume"],
["internal_added", "237"],
["disable", "1"],
]),
self.get_element("filter", [
["channel", "-1"],
["mlt_service", "panner"],
["internal_added", "237"],
["start", "0.5"],
["disable", "1"],
]),
self.get_element("filter", [
["iec_scale", "0"],
["mlt_service", "audiolevel"],
["dbpeak", "1"],
["disable", "1"],
]),
]),
a2 := self.get_element("playlist", children=[
["kdenlive:audio_track", "1"],
]),
a2e := self.get_element("playlist", children=[
["kdenlive:audio_track", "1"],
]),
t0 := self.get_element("tractor", children=[
t_a2 := self.get_element("tractor", children=[
["kdenlive:audio_track", "1"],
["kdenlive:trackheight", "69"],
["kdenlive:timeline_active", "1"],
@ -127,7 +199,7 @@ class KDEnliveProject:
a1e := self.get_element("playlist", children=[
["kdenlive:audio_track", "1"],
]),
t1 := self.get_element("tractor", children=[
t_a1 := self.get_element("tractor", children=[
["kdenlive:audio_track", "1"],
["kdenlive:trackheight", "69"],
["kdenlive:timeline_active", "1"],
@ -219,8 +291,10 @@ class KDEnliveProject:
["kdenlive:sequenceproperties.disablepreview", "0"],
self.get_element("track", attrib={"producer": p0.attrib["id"]}),
self.get_element("track", attrib={"producer": t0.attrib["id"]}),
self.get_element("track", attrib={"producer": t1.attrib["id"]}),
self.get_element("track", attrib={"producer": t_a4.attrib["id"]}),
self.get_element("track", attrib={"producer": t_a3.attrib["id"]}),
self.get_element("track", attrib={"producer": t_a2.attrib["id"]}),
self.get_element("track", attrib={"producer": t_a1.attrib["id"]}),
self.get_element("track", attrib={"producer": t2.attrib["id"]}),
self.get_element("track", attrib={"producer": t3.attrib["id"]}),
self.get_element("transition", [
@ -246,6 +320,26 @@ class KDEnliveProject:
self.get_element("transition", [
["a_track", "0"],
["b_track", "3"],
["mlt_service", "mix"],
["kdenlive_id", "mix"],
["internal_added", "237"],
["always_active", "1"],
["accepts_blanks", "1"],
["sum", "1"],
]),
self.get_element("transition", [
["a_track", "0"],
["b_track", "4"],
["mlt_service", "mix"],
["kdenlive_id", "mix"],
["internal_added", "237"],
["always_active", "1"],
["accepts_blanks", "1"],
["sum", "1"],
]),
self.get_element("transition", [
["a_track", "0"],
["b_track", "5"],
["compositing", "0"],
["distort", "0"],
["rotate_center", "0"],
@ -258,7 +352,7 @@ class KDEnliveProject:
]),
self.get_element("transition", [
["a_track", "0"],
["b_track", "4"],
["b_track", "6"],
["compositing", "0"],
["distort", "0"],
["rotate_center", "0"],
@ -313,11 +407,13 @@ class KDEnliveProject:
self._sequence = sequence
self._main_bin = main_bin
self._main_tractor = t4
self._audio_tractor = t1
self._audio_tractor = t_a1
self._v1 = v1
self._v2 = v2
self._a1 = a1
self._a2 = a2
self._a3 = a3
self._a4 = a4
def get_counter(self, prefix):
self._counters[prefix] += 1
@ -330,7 +426,7 @@ class KDEnliveProject:
if file in _CACHE:
out = _CACHE[file]
else:
out = _CACHE[file] = subprocess.check_output(['melt', file, '-consumer', 'xml'])
out = _CACHE[file] = subprocess.check_output(['melt', file, '-consumer', 'xml']).decode()
chain = lxml.etree.fromstring(out).xpath('producer')[0]
chain.tag = 'chain'
chain.attrib['id'] = self.get_id('chain')
@ -358,6 +454,11 @@ class KDEnliveProject:
mlt_service.text = "avformat-novalidate"
return chain
def get_duration(self):
if not self._duration:
return 0
return max(self._duration.values())
def get_element(self, tag, children=[], attrib={}, text=None):
element = lxml.etree.Element(tag)
if tag not in (
@ -445,13 +546,6 @@ class KDEnliveProject:
def append_clip(self, track_id, clip):
path = clip['src']
filters = clip.get("filter", {})
frames = int(self._fps * clip['duration'])
self._duration[track_id] += frames
#print(path, filters)
chain = self.get_chain(path)
id = get_propery(chain, "kdenlive:id")
if track_id == "V1":
track = self._v1
elif track_id == "V2":
@ -460,9 +554,30 @@ class KDEnliveProject:
track = self._a1
elif track_id == "A2":
track = self._a2
elif track_id == "A3":
track = self._a3
elif track_id == "A4":
track = self._a4
else:
print('!!', track_id)
frames = int(self._fps * clip['duration'])
self._duration[track_id] += frames
if clip.get("blank"):
track.append(
self.get_element("blank", attrib={
"length": str(frames),
})
)
return
path = clip['src']
filters = clip.get("filter", {})
#print(path, filters)
chain = self.get_chain(path)
id = get_propery(chain, "kdenlive:id")
if track_id[0] == 'A':
has_audio = False
for prop in chain.xpath('property'):
@ -504,14 +619,14 @@ class KDEnliveProject:
["kdenlive:id", id],
] + filters_),
)
chain = self.get_chain(path, id)
self._tree.append(chain)
self._main_bin.append(
self.get_element("entry", attrib={
"producer": chain.attrib["id"],
"in": chain.attrib["in"],
"out": chain.attrib["out"],
}),
)
chain = self.get_chain(path, id)
self._tree.append(chain)
self._main_bin.append(
self.get_element("entry", attrib={
"producer": chain.attrib["id"],
"in": chain.attrib["in"],
"out": chain.attrib["out"],
}),
)