more models

generate.py

@@ -626,12 +626,50 @@ def kling_v2v_reference(item, character, keep=False):
     img.add(ai)
     return ai
 
+def kling_v2v_edit(item, background, keep=False):
+    # https://fal.ai/models/fal-ai/kling-video/o1/video-to-video/edit
+    return
+
+    video_url = public_video_url(item)
+
+    images = []
+    elements = {
+        "reference_image_urls": [],
+        "frontal_image": ""
+    }
+    data = {
+        "prompt": prompt,
+        "video_url": video_url,
+        "image_urls": images,
+        "elements": elements,
+        "keep_audio": False,
+        "character_orientation": "video",
+    }
+    print(data)
+    handler = fal_client.submit(model, arguments=data)
+    request_id = handler.request_id
+    print(request_id)
+    result = fal_wait_for(model, request_id)
+    print(result)
+    output_url = result["video"]["url"]
+    ox.net.save_url(output_url, output, overwrite=True)
+    ai = add_ai_variant(item, output, "ai:v2v-replace")
+    ai.data["prompt"] = ox.escape_html(prompt)
+    ai.data["firstframe"] = image_url.split("?")[0]
+    ai.data["model"] = model
+    ai.save()
+    if not keep:
+        shutil.rmtree(os.path.dirname(output))
+    img.add(ai)
+    return ai
+
 def wan_reference_to_video(foreground, background, keep=False):
     foreground_url = public_video_url(foreground)
     background_url = public_video_url(background)
     prompt = "Use the character from @Video1 and use @Video2 as background"
     model = "wan/v2.6/reference-to-video"
     prompt_hash = hashlib.sha1((prompt + foreground_url + background_url).encode()).hexdigest()
+    item = background
     output = "/srv/pandora/static/power/cache/%s_%s/ai.mp4" % (
         item.public_id,
         prompt_hash,
@@ -646,12 +684,12 @@ def wan_reference_to_video(foreground, background, keep=False):
         "video_urls": [
             foreground_url,
             background_url,
-        ]
+        ],
         "aspect_ratio": "16:9",
         "resolution": "1080p",
-        "enable_prompt_expansion": false,
+        "enable_prompt_expansion": False,
-        "multi_shots": true,
+        "multi_shots": True,
-        "enable_safety_checker": false
+        "enable_safety_checker": False
     }
     print(data)
     handler = fal_client.submit(model, arguments=data)
@@ -669,6 +707,142 @@ def wan_reference_to_video(foreground, background, keep=False):
         shutil.rmtree(os.path.dirname(output))
     return ai
 
+def wan_animate_replace(item, character, keep=False):
+    item_url = public_video_url(item)
+    prompt = ""
+    model = "fal-ai/wan/v2.2-14b/animate/replace"
+    prompt_hash = hashlib.sha1((prompt + foreground_url + background_url).encode()).hexdigest()
+    output = "/srv/pandora/static/power/cache/%s_%s/ai.mp4" % (
+        item.public_id,
+        prompt_hash,
+    )
+    data = {
+        "video_url": item_url,
+        "image_url": character_url,
+        "resolution": "720p",
+        "video_quality": "high",
+        "guidance_scale": 1,
+        "enable_safety_checker": False,
+        "enable_output_safety_checker": False,
+    }
+    print(data)
+    handler = fal_client.submit(model, arguments=data)
+    request_id = handler.request_id
+    print(request_id)
+    result = fal_wait_for(model, request_id)
+    print(result)
+    output_url = result["video"]["url"]
+    ox.net.save_url(output_url, output, overwrite=True)
+    ai = add_ai_variant(item, output, "ai:foreground-background")
+    ai.data["prompt"] = ox.escape_html(prompt)
+    ai.data["model"] = model
+    ai.save()
+    if not keep:
+        shutil.rmtree(os.path.dirname(output))
+    return ai
+
+def ltx_a2v(item, character, prompt=None, keep=False):
+    video_url = public_video_url(item)
+    audio_path = item.streams()[0].file.data.path
+    character = get_character_document(character)
+
+    position = 0
+    cid = get_character_document(character).get_id()
+    first_frame = item.documents.filter(
+        data__character=cid,
+        data__position=position
+    ).order_by('-created').first()
+    if not first_frame:
+        first_frame = replace_character(item, character, position)
+    image_url = public_document_url(first_frame)
+    prefix = "/srv/pandora/static/power/cache/%s_a2v" % (item.public_id)
+    os.makedirs(prefix, exist_ok=True)
+    output = prefix + '/audio.m4a'
+    if not os.path.exists(output):
+        cmd = ['ffmpeg', '-hide_banner', '-nostats', '-i', audio_path, '-vn', '-c:a', 'copy', output]
+        subprocess.call(cmd)
+    if not os.path.exists(output):
+        raise Exception
+    audio_url = public_url(output)
+    model = "fal-ai/ltx-2-19b/audio-to-video"
+    neutral = True
+    if prompt is None:
+        prompt = describe_item(item, neutral)
+    data = {
+        "audio_url": audio_url,
+        "image_url": image_url,
+        "video_size": {"width": 1280, "height": 720},
+        "match_audio_length": True,
+        "fps": 24,
+        "prompt": prompt,
+        "enable_safety_checker": False,
+    }
+    print(data)
+    handler = fal_client.submit(model, arguments=data)
+    request_id = handler.request_id
+    print(request_id)
+    result = fal_wait_for(model, request_id)
+    print(result)
+    output_url = result["video"]["url"]
+    output = prefix + "/ai.mp4"
+    ox.net.save_url(output_url, output, overwrite=True)
+    ai = add_ai_variant(item, output, "ai:audio-to-video")
+    ai.data["model"] = model
+    ai.data["seed"] = result["seed"]
+    ai.save()
+    first_frame.add(ai)
+    ai.data["firstframe"] = first_frame.get_id()
+    if not keep:
+        shutil.rmtree(prefix)
+    return ai
+
+def ltx_v2v(item, character, prompt=None, keep=False):
+    video_url = public_video_url(item)
+    character = get_character_document(character)
+
+    position = 0
+    cid = get_character_document(character).get_id()
+    first_frame = item.documents.filter(
+        data__character=cid,
+        data__position=position
+    ).order_by('-created').first()
+    if not first_frame:
+        first_frame = replace_character(item, character, position)
+    image_url = public_document_url(first_frame)
+    prefix = "/srv/pandora/static/power/cache/%s_ltx_v2v" % (item.public_id)
+    model = "fal-ai/ltx-2-19b/video-to-video"
+    neutral = True
+    if prompt is None:
+        prompt = describe_item(item, neutral)
+    data = {
+        "prompt": prompt,
+        "video_url": video_url,
+        "image_url": image_url,
+        "video_size": {"width": 1280, "height": 720},
+        "match_video_length": True,
+        "fps": 24,
+        "generate_audio": False,
+        "enable_safety_checker": False,
+    }
+    print(data)
+    handler = fal_client.submit(model, arguments=data)
+    request_id = handler.request_id
+    print(request_id)
+    result = fal_wait_for(model, request_id)
+    print(result)
+    output_url = result["video"]["url"]
+    output = prefix + "/ai.mp4"
+    ox.net.save_url(output_url, output, overwrite=True)
+    ai = add_ai_variant(item, output, "ai:video-to-video")
+    ai.data["model"] = model
+    ai.data["seed"] = result["seed"]
+    ai.save()
+    first_frame.add(ai)
+    ai.data["firstframe"] = first_frame.get_id()
+    if not keep:
+        shutil.rmtree(prefix)
+    return ai
+
 def replace_character_motion_control(item, character, keep=False):
     if isinstance(item, str):
         item = Item.objects.get(public_id=item)
@@ -754,8 +928,15 @@ def describe_video(url, neutral=False, face=False):
 def describe_item(item, neutral=False):
     if isinstance(item, str):
         item = Item.objects.get(public_id=item)
+    if item.get("prompt") and neutral:
+        return item.get("prompt")
     video_url = public_video_url(item)
-    return describe_video(video_url, neutral)
+    prompt = describe_video(video_url, neutral)
+    if neutral:
+        item.refresh_from_db()
+        item.data["prompt"] = prompt
+        item.save()
+    return prompt
 
 
 def reshoot_item(item, extra_prompt=None, first_frame=None, keep=False, prompt=None):
@@ -839,13 +1020,25 @@ def reshoot_item_segments(item, character, keep=False):
             segment_video_url = public_url(output)
             prompt = describe_video(segment_video_url, neutral=True)
             prompts.append("Segment %s: " % (n + 1) + prompt)
-            segment_character = character
+            if character == "annotation":
+                a = item.annotations.filter(
+                    layer='prompts', start__gte=position, end__gt=position
+                ).order_by('start').first()
+                if a:
+                    segment_character = a.value
+                else:
+                    segment_character = None
+            else:
+                segment_character = character
             if position:
                 segment_character = segment_first_frame_url
-            segment_first_frame = replace_character(
+            if segment_character:
-                item, segment_character, position, seed=seed
+                segment_first_frame = replace_character(
-            )
+                    item, segment_character, position, seed=seed
-            segment_first_frame_url = public_document_url(segment_first_frame)
+                )
+                segment_first_frame_url = public_document_url(segment_first_frame)
+            else:
+                segment_first_frame_url = public_url(output.replace(".mp4", "_first.jpg"))
             status = i2v_bytedance(
                 segment_first_frame_url, prompt, segment_duration, output_ai, seed=seed
             )
@@ -1432,14 +1625,18 @@ def faceswap_item_segments(item, keep=False):
             stype, segment = segment
         else:
             stype = "n"
-        character = item.annotations.filter(
-            layer='prompts', start__gte=position, end__gt=position
-        ).order_by('start').first().value
-        character = get_character_document(character)
-        chracter_url = public_document_url(character)
-
         output = "%s/%06d.mp4" % (prefix, n)
         output_ai = "%s/%06d_ai.mp4" % (prefix, n)
+        character = item.annotations.filter(
+            layer='prompts', start__gte=position, end__gt=position
+        ).order_by('start').first()
+        if character:
+            character = character.value
+            character = get_character_document(character)
+            chracter_url = public_document_url(character)
+        else:
+            character = output.replace('.mp4', '_first.jpg')
+            chracter_url = public_url(character)
         segment_duration = segment - position
         if os.path.exists(output):
             segment_video_url = public_url(output)
@@ -1454,12 +1651,6 @@ def faceswap_item_segments(item, keep=False):
             print(result)
             output_url = result["video"]["url"]
             ox.net.save_url(output_url, output_ai, overwrite=True)
-            '''
-            trimmed = "%s/%06d_ai_trimmed.mp4" % (prefix, n)
-            frames = int(segment_duration * 24)
-            trim_video(output_ai, trimmed, frames, stype == "c")
-            processed.append(trimmed)
-            '''
             processed.append(output_ai)
         n += 1
         position = segment

render.py

@@ -1112,3 +1112,27 @@ def update_unused():
         l.add(i)
     for i in l.items.filter(public_id__in=set(used)):
         l.remove(i)
+
+def unused_tags():
+    import itemlist.models
+    import item.models
+    prefix = default_prefix
+
+    with open(os.path.join(prefix, "clips.json")) as fd:
+        clips = json.load(fd)
+    with open(os.path.join(prefix, "voice_over.json")) as fd:
+        voice_over = json.load(fd)
+    fragments = get_fragments(clips, voice_over, prefix)
+    tags = []
+
+    for fragment in fragments:
+        tags += fragment['tags']
+
+    used_tags = set(tags)
+    all_tags = {t.value for t in item.models.Facet.objects.filter(key='tags').distinct()}
+    unused_tags = all_tags - used_tags
+    unused_items = itemlist.models.List.objects.get(name='Unused Material').items.all()
+    with open("/srv/pandora/static/power/unused-tags.txt", "w") as fd:
+        for tag in sorted(unused_tags):
+            count = unused_items.filter(data__tags__contains=tag).count()
+            fd.write("%s (%d unused video clips)\n" % (tag, count))