diff --git a/management/commands/export_subtitles.py b/management/commands/export_subtitles.py index 1807fbb..8782c77 100644 --- a/management/commands/export_subtitles.py +++ b/management/commands/export_subtitles.py @@ -10,15 +10,16 @@ class Command(BaseCommand): help = 'export all subtitles for translations' def add_arguments(self, parser): - pass + parser.add_argument('--lang', action='store', dest='lang', default="", help='subtitle language') def handle(self, **options): - + lang = options["lang"] import annotation.models import item.models for i in item.models.Item.objects.filter(data__type__contains='Voice Over').order_by('sort__title'): print("## %s %s" % (i.get("title"), i.public_id)) - for sub in i.annotations.all().filter(layer='subtitles').exclude(value='').order_by("start"): + + for sub in i.annotations.all().filter(layer='subtitles').exclude(value='').filter(languages=lang).order_by("start"): if not sub.languages: print(sub.value.strip() + "\n") print("\n\n\n") diff --git a/management/commands/generate_clips.py b/management/commands/generate_clips.py index 9fd1c13..d7ae9f8 100644 --- a/management/commands/generate_clips.py +++ b/management/commands/generate_clips.py @@ -28,6 +28,7 @@ class Command(BaseCommand): help = 'generate symlinks to clips and clips.json' def add_arguments(self, parser): + parser.add_argument('--lang', action='store', dest='lang', default=None, help='subtitle language') parser.add_argument('--prefix', action='store', dest='prefix', default="/srv/t_for_time", help='prefix to build clips in') def handle(self, **options): @@ -102,7 +103,7 @@ class Command(BaseCommand): os.unlink(target) os.symlink(src, target) subs = [] - for sub in vo.annotations.filter(layer="subtitles").exclude(value="").order_by("start"): + for sub in vo.annotations.filter(layer="subtitles", languages=options["lang"]).exclude(value="").order_by("start"): sdata = get_srt(sub) subs.append(sdata) voice_over[fragment_id][batch] = { diff --git a/management/commands/import_subtitles.py b/management/commands/import_subtitles.py new file mode 100644 index 0000000..c812325 --- /dev/null +++ b/management/commands/import_subtitles.py @@ -0,0 +1,106 @@ +import json +import os +import subprocess + +import ox + +from django.core.management.base import BaseCommand +from django.conf import settings + +from item.models import Item +from annotation.models import Annotation + + +class Command(BaseCommand): + help = 'export all subtitles for translations' + + def add_arguments(self, parser): + parser.add_argument('--lang', action='store', dest='lang', default=None, help='subtitle language') + parser.add_argument('--test', action='store_true', dest='test', default=False, help='test run') + parser.add_argument('args', metavar='args', type=str, nargs='*', help='file or url') + + def handle(self, filename, **options): + if not options["lang"]: + print("--lang is required") + return + lang = options["lang"] + + if filename.startswith("http"): + data = ox.net.read_url(filename).decode() + else: + with open(filename) as fd: + data = fd.read() + + data = data.strip().split('\n## ')[1:] + + invalid = [] + valid = [] + for block in data: + title, block = block.split('\n', 1) + block = block.strip() + title = title.strip() + item_id = title.split(' ')[-1] + item = Item.objects.get(public_id=item_id) + + subtitles_en = item.annotations.filter(layer="subtitles", languages=None).exclude(value='') + lines = block.split('\n\n') + if len(lines) != subtitles_en.count(): + print('%s: number of subtitles does not match, en: %s vs %s: %s' % (title, subtitles_en.count(), lang, len(lines))) + continue + + if options["test"]: + print('%s: valid %s subtitles' % (title, len(lines))) + else: + n = 0 + item.annotations.filter(layer="subtitles", languages=lang).delete() + for sub_en in subtitles_en.order_by('start'): + sub = Annotation() + sub.item = sub_en.item + sub.user = sub_en.user + sub.layer = sub_en.layer + sub.start = sub_en.start + sub.end = sub_en.end + sub.value = '%s' % (lang, lines[n]) + sub.save() + n += 1 + + ''' + srt = 'vocals_txt/%s/%s' % (title[0], title.replace('.wav', '.srt')) + filename = 'vocals_txt/%s/%s' % (title[0], title.replace('.wav', '.' + lang + '.srt')) + + folder = os.path.dirname(filename) + if not os.path.exists(folder): + os.makedirs(folder) + data = json.load(open(srt + '.json')) + subs = block.replace('\n\n', '\n').split('\n') + if len(data) != len(subs): + print('invalid', title, 'expected', len(data), 'got', len(subs)) + invalid.append('## %s\n\n%s' % (title, block)) + valid.append('## %s\n\n%s' % (title, '\n\n'.join([d['value'] for d in data]))) + continue + + for i, sub in enumerate(data): + sub['value'] = subs[i] + kodata = ox.srt.encode(data) + current = None + if os.path.exists(filename): + with open(filename, 'rb') as fd: + current = fd.read() + if current != kodata: + print('update', title, filename) + with open(filename, 'wb') as fd: + fd.write(kodata) + with open(filename + '.json', 'w') as fd: + ko = [{ + 'in': s['in'], + 'out': s['out'], + 'value': s['value'], + } for s in data] + json.dump(ko, fd, ensure_ascii=False, indent=4) + + if invalid: + with open('invalid_%s_subtitles.txt' % lang, 'w') as fd: + fd.write('\n\n\n\n'.join(invalid)) + with open('invalid_%s_subtitles_en.txt' % lang, 'w') as fd: + fd.write('\n\n\n\n'.join(valid)) + ''' diff --git a/management/commands/update_subtitles.py b/management/commands/update_subtitles.py index 2ca2511..9bfd644 100644 --- a/management/commands/update_subtitles.py +++ b/management/commands/update_subtitles.py @@ -15,6 +15,7 @@ class Command(BaseCommand): parser.add_argument('--prefix', action='store', dest='prefix', default="/srv/t_for_time", help='prefix to build clips in') parser.add_argument('--duration', action='store', dest='duration', default="3600", help='target duration of all fragments in seconds') parser.add_argument('--offset', action='store', dest='offset', default="1024", help='inital offset in pi') + parser.add_argument('--lang', action='store', dest='lang', default=None, help='subtitle language') def handle(self, **options): update_subtitles(options) diff --git a/render.py b/render.py index 21cf212..72d3cf9 100644 --- a/render.py +++ b/render.py @@ -353,6 +353,7 @@ def render(root, scene, prefix=''): files.append(path) return files + def get_fragments(clips, voice_over, prefix): import itemlist.models import item.models @@ -590,6 +591,7 @@ def update_subtitles(options): prefix = Path(options['prefix']) duration = int(options['duration']) base = int(options['offset']) + lang = options["lang"] _cache = os.path.join(prefix, "cache.json") if os.path.exists(_cache): @@ -609,7 +611,7 @@ def update_subtitles(options): vo = item.models.Item.objects.filter(data__batch__icontains=batch, data__title__startswith=fragment_id + '_').first() if vo: #print("%s => %s %s" % (clip['src'], vo, vo.get('batch'))) - for sub in vo.annotations.filter(layer="subtitles").exclude(value="").order_by("start"): + for sub in vo.annotations.filter(layer="subtitles").filter(languages=lang).exclude(value="").order_by("start"): sdata = get_srt(sub, offset) subs.append(sdata) else: