diff --git a/config.jsonc b/config.jsonc index d28251a..6c757fb 100644 --- a/config.jsonc +++ b/config.jsonc @@ -246,6 +246,15 @@ examples (config.SITENAME.jsonc) that are part of this pan.do/ra distribution. "find": true, "sort": true }, + { + "id": "tags", + "title": "Tags", + "type": ["string"], + "autocomplete": true, + "columnWidth": 180, + "filter": true, + "sort": true + }, { "id": "model", "title": "Model", diff --git a/generate.py b/generate.py index d06c6f2..dc0fce2 100644 --- a/generate.py +++ b/generate.py @@ -13,6 +13,8 @@ from byteplussdkarkruntime import Ark from django.conf import settings from item.models import Item +from document.models import Document +from archive.models import File, Stream os.environ["FAL_KEY"] = settings.FAL_KEY @@ -36,6 +38,14 @@ def public_document_url(document): ) return url +def public_video_url(item): + url = "%s%s/download/source/?token=%s" % ( + settings.PUBLIC_URL, + item.public_id, + settings.PUBLIC_TOKEN, + ) + return url + def trim_video(src, dst, frames, start0=False): cap = cv2.VideoCapture(src) @@ -377,11 +387,7 @@ def replace_character_motion_control(item, character, keep=False): else: img = character image_url = public_document_url(img) - video_url = "%s%s/download/source/?token=%s" % ( - settings.PUBLIC_URL, - item.public_id, - settings.PUBLIC_TOKEN, - ) + video_url = public_video_url(item) prompt = "" model = "fal-ai/kling-video/v2.6/pro/motion-control" prompt_hash = hashlib.sha1((prompt + image_url).encode()).hexdigest() @@ -411,6 +417,28 @@ def replace_character_motion_control(item, character, keep=False): img.add(ai) return ai +def describe_video_neutral(url): + prompt = ( + "Detect cuts or scene changes and describe each scene, use as much details as you can. " + "Describe each person incudling detalied apreance, haircut in a gender neutral way, " + "describe each objects, animal or plant, describe foreground and backgroud, " + "describe from what angle the scene is filmed, incude details about camera model, lense, depth of field used to film this scene. " + "Use the format: . CAMERA CUT TO . CAMERA CUT TO . " + "Don't mention it if you don't find a cut." + ) + data = { + "input": [ + { + "role": "user", + "content": [ + {"type": "input_video", "video_url": url, "fps": 1}, + {"type": "input_text", "text": prompt}, + ], + } + ], + } + response = bytedance_response(data) + return response["output"][1]["content"][0]["text"] def describe_video(url): prompt = ( @@ -439,11 +467,7 @@ def describe_video(url): def describe_item(item): if isinstance(item, str): item = Item.objects.get(public_id=item) - video_url = "%s%s/download/source/?token=%s" % ( - settings.PUBLIC_URL, - item.public_id, - settings.PUBLIC_TOKEN, - ) + video_url = public_video_url(item) return describe_video(video_url) @@ -452,7 +476,11 @@ def reshoot_item(item, extra_prompt=None, first_frame=None, keep=False): item = Item.objects.get(public_id=item) duration = item.sort.duration frames = int(duration * 24) - prompt = describe_item(item) + if first_frame: + prompt = describe_item_neutral(item) + else: + prompt = describe_item(item) + if extra_prompt: prompt += " " + extra_prompt prompt_hash = hashlib.sha1((prompt).encode()).hexdigest() @@ -479,7 +507,8 @@ def reshoot_item(item, extra_prompt=None, first_frame=None, keep=False): ai.data["seed"] = status["seed"] if first_frame: ai.data["firstframe"] = first_frame.split('?')[0] - + if isinstance(first_frame, Document): + first_frame.add(ai) ai.save() if not keep: shutil.rmtree(os.path.dirname(output)) @@ -583,11 +612,7 @@ def transform_remake_video(item_id, image_prompt, video_prompt): def restyle_video(item_id, prompt): item = Item.objects.get(public_id=item_id) - video_url = "%s%s/download/source/?token=%s" % ( - settings.PUBLIC_URL, - item.public_id, - settings.PUBLIC_TOKEN, - ) + video_url = public_video_url(item) model = "decart/lucy-restyle" handler = fal_client.submit( model, @@ -625,11 +650,7 @@ def fal_wait_for(model, request_id): def motion_control_preprocess_image(item_id, image_prompt, video_prompt): item = Item.objects.get(public_id=item_id) - video_url = "%s%s/download/source/?token=%s" % ( - settings.PUBLIC_URL, - item.public_id, - settings.PUBLIC_TOKEN, - ) + video_url = public_video_url(item) model = "fal-ai/kling-video/v2.6/pro/motion-control" prompt_hash = hashlib.sha1((image_prompt + video_prompt).encode()).hexdigest() output = "/srv/pandora/static/power/cache/%s_%s.mp4" % (item.public_id, prompt_hash) @@ -874,7 +895,6 @@ def luma_modify_item(item, prompt="", image_prompt=None, first_frame=None, keep= def add_ai_variant(item, video_path, type): if isinstance(item, str): item = Item.objects.get(public_id=item) - from archive.models import File, Stream ai = Item() ai.user = item.user @@ -900,7 +920,6 @@ def add_ai_variant(item, video_path, type): return ai def add_ai_image(item, position, url, extension=None): - from document.models import Document if extension is None: extension = url.split('.')[-1].split('?')[0] if extension == 'jpeg': extension = 'jpg'