import/export subtitles

management/commands/export_subtitles.py

@@ -10,15 +10,16 @@ class Command(BaseCommand):
     help = 'export all subtitles for translations'
 
     def add_arguments(self, parser):
-        pass
+        parser.add_argument('--lang', action='store', dest='lang', default="", help='subtitle language')
 
     def handle(self, **options):
-
+        lang = options["lang"]
         import annotation.models
         import item.models
         for i in item.models.Item.objects.filter(data__type__contains='Voice Over').order_by('sort__title'):
             print("## %s %s" % (i.get("title"), i.public_id))
-            for sub in i.annotations.all().filter(layer='subtitles').exclude(value='').order_by("start"):
+
+            for sub in i.annotations.all().filter(layer='subtitles').exclude(value='').filter(languages=lang).order_by("start"):
                 if not sub.languages:
                     print(sub.value.strip() + "\n")
             print("\n\n\n")

management/commands/generate_clips.py

@@ -28,6 +28,7 @@ class Command(BaseCommand):
     help = 'generate symlinks to clips and clips.json'
 
     def add_arguments(self, parser):
+        parser.add_argument('--lang', action='store', dest='lang', default=None, help='subtitle language')
         parser.add_argument('--prefix', action='store', dest='prefix', default="/srv/t_for_time", help='prefix to build clips in')
 
     def handle(self, **options):
@@ -102,7 +103,7 @@ class Command(BaseCommand):
                 os.unlink(target)
             os.symlink(src, target)
             subs = []
-            for sub in vo.annotations.filter(layer="subtitles").exclude(value="").order_by("start"):
+            for sub in vo.annotations.filter(layer="subtitles", languages=options["lang"]).exclude(value="").order_by("start"):
                 sdata = get_srt(sub)
                 subs.append(sdata)
             voice_over[fragment_id][batch] = {

management/commands/import_subtitles.py

@@ -0,0 +1,106 @@
+import json
+import os
+import subprocess
+
+import ox
+
+from django.core.management.base import BaseCommand
+from django.conf import settings
+
+from item.models import Item
+from annotation.models import Annotation
+
+
+class Command(BaseCommand):
+    help = 'export all subtitles for translations'
+
+    def add_arguments(self, parser):
+        parser.add_argument('--lang', action='store', dest='lang', default=None, help='subtitle language')
+        parser.add_argument('--test', action='store_true', dest='test', default=False, help='test run')
+        parser.add_argument('args', metavar='args', type=str, nargs='*', help='file or url')
+
+    def handle(self, filename, **options):
+        if not options["lang"]:
+            print("--lang is required")
+            return
+        lang = options["lang"]
+
+        if filename.startswith("http"):
+            data = ox.net.read_url(filename).decode()
+        else:
+            with open(filename) as fd:
+                data = fd.read()
+
+        data = data.strip().split('\n## ')[1:]
+
+        invalid = []
+        valid = []
+        for block in data:
+            title, block = block.split('\n', 1)
+            block = block.strip()
+            title = title.strip()
+            item_id = title.split(' ')[-1]
+            item = Item.objects.get(public_id=item_id)
+
+            subtitles_en = item.annotations.filter(layer="subtitles", languages=None).exclude(value='')
+            lines = block.split('\n\n')
+            if len(lines) != subtitles_en.count():
+                print('%s: number of subtitles does not match, en: %s vs %s: %s' % (title, subtitles_en.count(), lang, len(lines)))
+                continue
+
+            if options["test"]:
+                print('%s: valid %s subtitles' % (title, len(lines)))
+            else:
+                n = 0
+                item.annotations.filter(layer="subtitles", languages=lang).delete()
+                for sub_en in subtitles_en.order_by('start'):
+                    sub = Annotation()
+                    sub.item = sub_en.item
+                    sub.user = sub_en.user
+                    sub.layer = sub_en.layer
+                    sub.start = sub_en.start
+                    sub.end = sub_en.end
+                    sub.value = '<span lang="%s">%s</span>' % (lang, lines[n])
+                    sub.save()
+                    n += 1
+
+            '''
+            srt = 'vocals_txt/%s/%s' % (title[0], title.replace('.wav', '.srt'))
+            filename = 'vocals_txt/%s/%s' % (title[0], title.replace('.wav', '.' + lang + '.srt'))
+
+            folder = os.path.dirname(filename)
+            if not os.path.exists(folder):
+                os.makedirs(folder)
+            data = json.load(open(srt + '.json'))
+            subs = block.replace('\n\n', '\n').split('\n')
+            if len(data) != len(subs):
+                print('invalid', title, 'expected', len(data), 'got', len(subs))
+                invalid.append('## %s\n\n%s' % (title, block))
+                valid.append('## %s\n\n%s' % (title, '\n\n'.join([d['value'] for d in data])))
+                continue
+
+            for i, sub in enumerate(data):
+                sub['value'] = subs[i]
+            kodata = ox.srt.encode(data)
+            current = None
+            if os.path.exists(filename):
+                with open(filename, 'rb') as fd:
+                    current = fd.read()
+            if current != kodata:
+                print('update', title, filename)
+                with open(filename, 'wb') as fd:
+                    fd.write(kodata)
+                with open(filename + '.json', 'w') as fd:
+                    ko = [{
+                        'in': s['in'],
+                        'out': s['out'],
+                        'value': s['value'],
+                    } for s in data]
+                    json.dump(ko, fd, ensure_ascii=False, indent=4)
+
+        if invalid:
+            with open('invalid_%s_subtitles.txt' % lang, 'w') as fd:
+                fd.write('\n\n\n\n'.join(invalid))
+            with open('invalid_%s_subtitles_en.txt' % lang, 'w') as fd:
+                fd.write('\n\n\n\n'.join(valid))
+                '''

management/commands/update_subtitles.py

@@ -15,6 +15,7 @@ class Command(BaseCommand):
         parser.add_argument('--prefix', action='store', dest='prefix', default="/srv/t_for_time", help='prefix to build clips in')
         parser.add_argument('--duration', action='store', dest='duration', default="3600", help='target duration of all fragments in seconds')
         parser.add_argument('--offset', action='store', dest='offset', default="1024", help='inital offset in pi')
+        parser.add_argument('--lang', action='store', dest='lang', default=None, help='subtitle language')
 
     def handle(self, **options):
         update_subtitles(options)

render.py

@@ -353,6 +353,7 @@ def render(root, scene, prefix=''):
         files.append(path)
     return files
 
+
 def get_fragments(clips, voice_over, prefix):
     import itemlist.models
     import item.models
@@ -590,6 +591,7 @@ def update_subtitles(options):
     prefix = Path(options['prefix'])
     duration = int(options['duration'])
     base = int(options['offset'])
+    lang = options["lang"]
 
     _cache = os.path.join(prefix, "cache.json")
     if os.path.exists(_cache):
@@ -609,7 +611,7 @@ def update_subtitles(options):
                 vo = item.models.Item.objects.filter(data__batch__icontains=batch, data__title__startswith=fragment_id + '_').first()
                 if vo:
                     #print("%s => %s %s" % (clip['src'], vo, vo.get('batch')))
-                    for sub in vo.annotations.filter(layer="subtitles").exclude(value="").order_by("start"):
+                    for sub in vo.annotations.filter(layer="subtitles").filter(languages=lang).exclude(value="").order_by("start"):
                         sdata = get_srt(sub, offset)
                         subs.append(sdata)
                 else: