prcoess list with extra prompt, lipsync

generate.py

@@ -4,6 +4,7 @@ import math
 import os
 import time
 import shutil
+import subprocess
 
 import cv2
 import ox
@@ -105,6 +106,7 @@ def make_single_character_image(character):
     if extension == "jpeg":
         extension = "jpg"
     file = Document(user=character.user)
+    file.rightslevel = 2
     file.data["title"] = character.data['title'].replace('Character', 'Single Character')
     file.extension = extension
     file.width = -1
@@ -131,6 +133,7 @@ def make_single_character_image_flux(character):
     if extension == "jpeg":
         extension = "jpg"
     file = Document(user=character.user)
+    file.rightslevel = 2
     file.data["title"] = character.data['title'].replace('Character', 'FLUX Single Character')
     file.extension = extension
     file.width = -1
@@ -498,10 +501,12 @@ def fal_replace_character(item, character, position=0):
     return img
 
 
-def replace_character(item, character, position=0, seed=None):
+def replace_character(item, character, position=0, seed=None, extra=None):
     prompt = REPLACE_CHARACTER_PROMPT
     if character == "P5":
         prompt = prompt.replace("child", "teenager")
+    if extra:
+        prompt += " " + extra
     character = get_character_document(character)
     if isinstance(character, Document):
         character_url = public_document_url(character)
@@ -517,9 +522,18 @@ def replace_character(item, character, position=0, seed=None):
     return frame
 
 
-def kling_lipsync(audio_item, video_item):
+def kling_lipsync(audio_item, video_item, keep=False):
     video_url = public_video_url(video_item)
-    audio_url = public_video_url(audio_item)
+    audio_path = audio_item.streams()[0].file.data.path
+    prefix = "/srv/pandora/static/power/cache/%s" % (audio_item.public_id)
+    os.makedirs(prefix, exist_ok=True)
+    output = prefix + '/audio.m4a'
+    if not os.path.exists(output):
+        cmd = ['ffmpeg', '-hide_banner', '-nostats', '-i', audio_path, '-vn', '-c:a', 'copy', output]
+        subprocess.call(cmd)
+    if not os.path.exists(output):
+        raise Exception
+    audio_url = public_url(output)
     model = "fal-ai/kling-video/lipsync/audio-to-video"
     data = {
         "video_url": video_url,
@@ -532,13 +546,13 @@ def kling_lipsync(audio_item, video_item):
     result = fal_wait_for(model, request_id)
     print(result)
     output_url = result["video"]["url"]
+    output = prefix + "/lipsync.mp4"
     ox.net.save_url(output_url, output, overwrite=True)
-    ai = add_ai_variant(item, output, "ai:lipsync")
+    ai = add_ai_variant(audio_item, output, "ai:lipsync")
     ai.data["model"] = model
     ai.save()
     if not keep:
-        shutil.rmtree(os.path.dirname(output))
-    img.add(ai)
+        shutil.rmtree(prefix)
     return ai
 
 def kling_v2v_reference(item, character, keep=False):
@@ -1177,6 +1191,7 @@ def add_ai_variant(item, video_path, type):
     file.oshash = ox.oshash(video_path)
     file.item = ai
     file.path = "%s.mp4" % type
+    file.extension = "mp4"
     file.info = ox.avinfo(video_path)
     del file.info["path"]
     file.parse_info()
@@ -1198,6 +1213,7 @@ def add_ai_image(item, position, url, extension=None):
         if extension == "jpeg":
             extension = "jpg"
     file = Document(user=item.user)
+    file.rightslevel = 2
     file.data["title"] = "%s at %s" % (item.get("title"), position)
     file.data["position"] = position
     file.extension = extension
@@ -1282,3 +1298,43 @@ def process_motion_firstframe(character="P1", keep=False):
             add_tag(item, 'ai-failed')
             print('>> failed', item)
 
+
+
+def process_list(list, process, character='P1', extra=None):
+    position = 0
+    if extra is None and list.description:
+        extra = list.description
+    for item in list.items.all():
+        if 'ai-failed' in item.data.get('tags', []):
+            print('>> skip', item)
+            continue
+        if item.sort.duration > 30:
+            pass
+            #reshoot_item_segments(item, character)
+        else:
+            cid = get_character_document(character).get_id()
+            first_frame = item.documents.filter(
+                data__character=cid, data__position=position
+            ).order_by('-created').first()
+            if not first_frame:
+                try:
+                    first_frame = replace_character(item, character, position, extra=extra)
+                except:
+                    item.refresh_from_db()
+                    add_tag(item, 'ai-failed')
+                    print('>> failed', item)
+            if first_frame.items.filter(data__type__icontains='ai:').exists():
+                continue
+            print(item, first_frame)
+            try:
+                process(item, first_frame=first_frame)
+            except:
+                item.refresh_from_db()
+                add_tag(item, 'ai-failed')
+                print('>> failed', item)
+
+def reshoot_mustache(character="P1"):
+    l = itemlist.models.List.objects.get(name='mustache')
+    extra = "The person is wearing a fake mustache in the style of the mustache from image 1"
+    return process_list(l, reshoot_item, character, extra=extra)
+