From 8f2a8794c5e6971f0f48e554b03bbd69cdbca3a7 Mon Sep 17 00:00:00 2001 From: H1yori233 Date: Tue, 12 May 2026 11:04:32 +0000 Subject: [PATCH 01/11] svi inference --- examples/inference/basic/basic_svi_i2v.py | 44 +++++ fastvideo/api/sampling_param.py | 6 + .../basic/wan/wan_svi_i2v_pipeline.py | 161 ++++++++++++++++++ fastvideo/pipelines/lora_pipeline.py | 2 + fastvideo/pipelines/pipeline_batch_info.py | 8 + fastvideo/pipelines/stages/__init__.py | 2 + .../stages/svi_image_vae_encoding.py | 156 +++++++++++++++++ 7 files changed, 379 insertions(+) create mode 100644 examples/inference/basic/basic_svi_i2v.py create mode 100644 fastvideo/pipelines/basic/wan/wan_svi_i2v_pipeline.py create mode 100644 fastvideo/pipelines/stages/svi_image_vae_encoding.py diff --git a/examples/inference/basic/basic_svi_i2v.py b/examples/inference/basic/basic_svi_i2v.py new file mode 100644 index 0000000000..8f65a72c06 --- /dev/null +++ b/examples/inference/basic/basic_svi_i2v.py @@ -0,0 +1,44 @@ +from fastvideo import VideoGenerator + +OUTPUT_PATH = "video_samples_svi_shot" + + +def main(): + generator = VideoGenerator.from_pretrained( + "Wan-AI/Wan2.1-I2V-14B-480P-Diffusers", + override_pipeline_cls_name="WanSVIImageToVideoPipeline", + lora_path=("./Stable-Video-Infinity/weights/Stable-Video-Infinity/" + "version-1.0/svi-shot.safetensors"), + lora_nickname="svi-shot", + num_gpus=1, + dit_cpu_offload=False, + vae_cpu_offload=False, + text_encoder_cpu_offload=True, + pin_cpu_memory=True, # set to false if low CPU RAM or hit obscure "CUDA error: Invalid argument" + ) + + prompt = ("A sleek white motor yacht speeds across the turquoise blue sea, " + "leaving a dramatic wake of white foam behind it under a clear blue sky.") + image_path = "./Stable-Video-Infinity/data/toy_test/shot/frame.jpg" + + video = generator.generate_video( + prompt, + image_path=image_path, + output_path=OUTPUT_PATH, + save_video=True, + height=448, + width=832, + num_frames=81, + num_inference_steps=20, + guidance_scale=5.0, + seed=42, + # SVI knobs. Shot/Tom variants use num_motion_frames=1 + ref_pad_num=-1; + # Film uses num_motion_frames=5 + ref_pad_num=0. num_clips>1 enables motion-frame chaining. + svi_num_clips=1, + svi_num_motion_frames=1, + svi_ref_pad_num=-1, + ) + + +if __name__ == "__main__": + main() diff --git a/fastvideo/api/sampling_param.py b/fastvideo/api/sampling_param.py index f89d97e205..b4b490303d 100644 --- a/fastvideo/api/sampling_param.py +++ b/fastvideo/api/sampling_param.py @@ -97,6 +97,12 @@ class SamplingParam: movement_distance: float | None = None camera_rotation: str | None = None + # Stable-Video-Infinity image conditioning + svi_ref_pad_num: int | None = None # -1 = tile ref; 0 = zero pad; k>0 = ref for first k slots + svi_ref_pad_cfg: bool = False # widens y-mask to len(first_frames) instead of {first frame only} + svi_num_clips: int = 1 # >1 enables motion-frame chaining + svi_num_motion_frames: int = 1 # tail frames carried from clip K to clip K+1 (1=Shot/Tom, 5=Film) + # LTX-2 multi-modal CFG and STG. # cfg_scale defaults are 1.0 (CFG off) so ``ForwardBatch.__post_init__`` # doesn't force ``do_classifier_free_guidance`` on non-LTX-2 models that diff --git a/fastvideo/pipelines/basic/wan/wan_svi_i2v_pipeline.py b/fastvideo/pipelines/basic/wan/wan_svi_i2v_pipeline.py new file mode 100644 index 0000000000..b455425f00 --- /dev/null +++ b/fastvideo/pipelines/basic/wan/wan_svi_i2v_pipeline.py @@ -0,0 +1,161 @@ +# SPDX-License-Identifier: Apache-2.0 +""" +Wan I2V pipeline variant for Stable-Video-Infinity multi-clip inference. + +This module contains an implementation of the SVI-flavored Wan I2V +pipeline using the modular pipeline architecture. +""" + +import dataclasses + +import PIL.Image +import torch + +from fastvideo.fastvideo_args import FastVideoArgs +from fastvideo.logger import init_logger +from fastvideo.pipelines.basic.wan.wan_i2v_pipeline import WanImageToVideoPipeline +from fastvideo.pipelines.pipeline_batch_info import ForwardBatch + +# isort: off +from fastvideo.pipelines.stages import (ConditioningStage, DecodingStage, DenoisingStage, ImageEncodingStage, + InputValidationStage, LatentPreparationStage, SVIImageVAEEncodingStage, + TextEncodingStage, TimestepPreparationStage) +# isort: on + +logger = init_logger(__name__) + + +def _tensor_to_pil_list(frames: torch.Tensor) -> list[PIL.Image.Image]: + """Convert a video tensor to a list of PIL frames.""" + arr = (frames.detach().to(torch.float32).cpu().clamp(0, 1) * 255.0).round().to(torch.uint8) + arr = arr.permute(1, 2, 3, 0).numpy() + return [PIL.Image.fromarray(a) for a in arr] + + +class WanSVIImageToVideoPipeline(WanImageToVideoPipeline): + """ + Pipeline for Stable-Video-Infinity multi-clip I2V generation on Wan 2.1. + """ + + def create_pipeline_stages(self, fastvideo_args: FastVideoArgs): + """Set up pipeline stages with proper dependency injection.""" + self.add_stage(stage_name="input_validation_stage", stage=InputValidationStage()) + self.add_stage( + stage_name="prompt_encoding_stage", + stage=TextEncodingStage( + text_encoders=[self.get_module("text_encoder")], + tokenizers=[self.get_module("tokenizer")], + ), + ) + if (self.get_module("image_encoder") is not None and self.get_module("image_processor") is not None): + self.add_stage( + stage_name="image_encoding_stage", + stage=ImageEncodingStage( + image_encoder=self.get_module("image_encoder"), + image_processor=self.get_module("image_processor"), + ), + ) + self.add_stage(stage_name="conditioning_stage", stage=ConditioningStage()) + self.add_stage( + stage_name="timestep_preparation_stage", + stage=TimestepPreparationStage(scheduler=self.get_module("scheduler")), + ) + self.add_stage( + stage_name="latent_preparation_stage", + stage=LatentPreparationStage( + scheduler=self.get_module("scheduler"), + transformer=self.get_module("transformer"), + ), + ) + self.add_stage( + stage_name="image_latent_preparation_stage", + stage=SVIImageVAEEncodingStage(vae=self.get_module("vae")), + ) + self.add_stage( + stage_name="denoising_stage", + stage=DenoisingStage( + transformer=self.get_module("transformer"), + transformer_2=self.get_module("transformer_2"), + scheduler=self.get_module("scheduler"), + ), + ) + self.add_stage(stage_name="decoding_stage", stage=DecodingStage(vae=self.get_module("vae"))) + + @torch.no_grad() + def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> ForwardBatch: + if not self.post_init_called: + self.post_init() + + num_clips = max(1, int(batch.svi_num_clips or 1)) + num_motion = max(1, int(batch.svi_num_motion_frames or 1)) + + if isinstance(batch.prompt, list): + prompts: list[str | None] = list(batch.prompt) + else: + prompts = [batch.prompt] + if len(prompts) < num_clips: + prompts = prompts + [prompts[-1]] * (num_clips - len(prompts)) + elif len(prompts) > num_clips: + prompts = prompts[:num_clips] + + if num_clips == 1: + batch.prompt = prompts[0] + if batch.svi_random_ref_frame is None: + batch.svi_random_ref_frame = batch.pil_image # type: ignore[assignment] + if not batch.svi_first_frames: + assert isinstance(batch.pil_image, PIL.Image.Image) + batch.svi_first_frames = [batch.pil_image] + for stage in self.stages: + batch = stage(batch, fastvideo_args) + return batch + + assert isinstance(batch.pil_image, PIL.Image.Image) + random_ref = batch.svi_random_ref_frame or batch.pil_image + motion_frames: list[PIL.Image.Image] = batch.svi_first_frames or [batch.pil_image] + + clip_outputs: list[torch.Tensor] = [] + for clip_idx in range(num_clips): + clip_batch = dataclasses.replace( + batch, + prompt=prompts[clip_idx], + pil_image=motion_frames[0], + svi_first_frames=motion_frames, + svi_random_ref_frame=random_ref, + prompt_embeds=[], + negative_prompt_embeds=None, + prompt_attention_mask=None, + negative_attention_mask=None, + clip_embedding_pos=None, + clip_embedding_neg=None, + image_embeds=[], + preprocessed_image=None, + latents=None, + image_latent=None, + noise_pred=None, + output=None, + timesteps=None, + timestep=None, + step_index=None, + is_prompt_processed=False, + ) + for stage in self.stages: + clip_batch = stage(clip_batch, fastvideo_args) + + assert clip_batch.output is not None + clip_outputs.append(clip_batch.output) + logger.info("SVI clip %d/%d generated, frames shape=%s", clip_idx + 1, num_clips, clip_batch.output.shape) + + if clip_idx + 1 < num_clips: + tail = clip_batch.output[0, :, -num_motion:, :, :] + motion_frames = _tensor_to_pil_list(tail) + + # Drop the first num_motion frames of each follow-up clip to avoid duplicating + # the previous clip's tail in the stitched output. + concatenated = [clip_outputs[0]] + for video in clip_outputs[1:]: + concatenated.append(video[:, :, num_motion:, :, :]) + batch.output = torch.cat(concatenated, dim=2) + return batch + + +EntryClass = WanSVIImageToVideoPipeline diff --git a/fastvideo/pipelines/lora_pipeline.py b/fastvideo/pipelines/lora_pipeline.py index 3f0559fbc8..33257d94d5 100644 --- a/fastvideo/pipelines/lora_pipeline.py +++ b/fastvideo/pipelines/lora_pipeline.py @@ -317,7 +317,9 @@ def set_lora_adapter(self, lora_nickname: str, lora_path: str | None = None): # for name, weight in lora_state_dict.items(): # Extract weights (lora_A, lora_B, and lora_alpha) name = name.replace("diffusion_model.", "") + name = name.replace("pipe.dit.", "") name = name.replace(".weight", "") + name = name.replace(".default", "") if "lora_alpha" in name: # Store alpha with minimal mapping - same processing as lora_A/lora_B diff --git a/fastvideo/pipelines/pipeline_batch_info.py b/fastvideo/pipelines/pipeline_batch_info.py index 9a0b2479f6..b7a12b506e 100644 --- a/fastvideo/pipelines/pipeline_batch_info.py +++ b/fastvideo/pipelines/pipeline_batch_info.py @@ -150,6 +150,14 @@ class ForwardBatch: movement_distance: float | None = None camera_rotation: str | None = None + # Stable-Video-Infinity image conditioning + svi_first_frames: list[PIL.Image.Image] | None = None # Motion frames forwarded across clips + svi_random_ref_frame: PIL.Image.Image | None = None # Reference frame for ref-padded slots + svi_ref_pad_num: int | None = None # -1=tile ref; 0=zero pad; k>0=ref for first k, zero rest + svi_ref_pad_cfg: bool = False # widens y-mask to len(first_frames) instead of frame 0 only + svi_num_clips: int = 1 # >1 enables motion-frame chaining + svi_num_motion_frames: int = 1 # Tail frames carried clip K -> K+1 (1=Shot/Tom, 5=Film) + # Latent dimensions height_latents: list[int] | int | None = None width_latents: list[int] | int | None = None diff --git a/fastvideo/pipelines/stages/__init__.py b/fastvideo/pipelines/stages/__init__.py index f9ff472c90..a10b6e5753 100644 --- a/fastvideo/pipelines/stages/__init__.py +++ b/fastvideo/pipelines/stages/__init__.py @@ -20,6 +20,7 @@ VideoVAEEncodingStage, Hy15ImageEncodingStage, HYWorldImageEncodingStage) from fastvideo.pipelines.stages.gamecraft_image_encoding import (GameCraftImageVAEEncodingStage) +from fastvideo.pipelines.stages.svi_image_vae_encoding import SVIImageVAEEncodingStage from fastvideo.pipelines.stages.input_validation import InputValidationStage from fastvideo.pipelines.stages.latent_preparation import (Cosmos25LatentPreparationStage, CosmosLatentPreparationStage, Cosmos25AutoLatentPreparationStage, @@ -87,6 +88,7 @@ "ImageVAEEncodingStage", "VideoVAEEncodingStage", "GameCraftImageVAEEncodingStage", + "SVIImageVAEEncodingStage", "TextEncodingStage", "Cosmos25TextEncodingStage", # LongCat stages diff --git a/fastvideo/pipelines/stages/svi_image_vae_encoding.py b/fastvideo/pipelines/stages/svi_image_vae_encoding.py new file mode 100644 index 0000000000..f8aea2fdd2 --- /dev/null +++ b/fastvideo/pipelines/stages/svi_image_vae_encoding.py @@ -0,0 +1,156 @@ +# SPDX-License-Identifier: Apache-2.0 +""" +SVI Image VAE Encoding Stage for Stable-Video-Infinity I2V generation. + +This stage handles encoding a list of motion frames and reference-padded +slots to latent space with SVI-specific mask construction for I2V +conditioning. Used by the WanSVIImageToVideoPipeline. +""" + +import PIL +import torch + +from fastvideo.distributed import get_local_torch_device +from fastvideo.fastvideo_args import ExecutionMode, FastVideoArgs +from fastvideo.logger import init_logger +from fastvideo.models.vaes.common import ParallelTiledVAE +from fastvideo.pipelines.pipeline_batch_info import ForwardBatch +from fastvideo.pipelines.stages.image_encoding import ImageVAEEncodingStage +from fastvideo.pipelines.stages.validators import StageValidators as V +from fastvideo.pipelines.stages.validators import VerificationResult +from fastvideo.utils import PRECISION_TO_TYPE + +logger = init_logger(__name__) + + +class SVIImageVAEEncodingStage(ImageVAEEncodingStage): + """ + Stage for encoding motion frames and reference padding for Stable-Video-Infinity. + """ + + vae: ParallelTiledVAE + + def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> ForwardBatch: + if fastvideo_args.mode != ExecutionMode.INFERENCE: + raise NotImplementedError("SVIImageVAEEncodingStage only supports inference mode") + + assert isinstance(batch.height, int) and isinstance(batch.width, int) + assert isinstance(batch.num_frames, int) + height, width, num_frames = batch.height, batch.width, batch.num_frames + + first_frames = batch.svi_first_frames + if not first_frames: + assert isinstance(batch.pil_image, PIL.Image.Image) + first_frames = [batch.pil_image] + random_ref_frame = batch.svi_random_ref_frame + if random_ref_frame is None: + random_ref_frame = first_frames[0] + ref_pad_num = batch.svi_ref_pad_num if batch.svi_ref_pad_num is not None else 0 + ref_pad_cfg = batch.svi_ref_pad_cfg + + num_condition_frames = len(first_frames) + remaining_frames = num_frames - num_condition_frames + if remaining_frames < 0: + raise ValueError(f"num_frames={num_frames} smaller than len(first_frames)={num_condition_frames}") + + device = get_local_torch_device() + self.vae = self.vae.to(device) + vae_scale = self.vae.spatial_compression_ratio + latent_height = height // vae_scale + latent_width = width // vae_scale + temporal_compression = self.vae.temporal_compression_ratio + + # 1) Mask. (1, T, H/8, W/8) -> reshape -> (4, T_lat, H/8, W/8). + msk = torch.ones(1, num_frames, latent_height, latent_width, device=device, dtype=torch.float32) + if ref_pad_cfg: + msk[:, num_condition_frames:] = 0 + else: + msk[:, 1:] = 0 + msk = torch.concat( + [torch.repeat_interleave(msk[:, 0:1], repeats=temporal_compression, dim=1), msk[:, 1:]], + dim=1, + ) + msk = msk.view(1, msk.shape[1] // temporal_compression, temporal_compression, latent_height, latent_width) + msk = msk.transpose(1, 2)[0] # (4, T_lat, H/8, W/8) + + # 2) Condition frames as (1, 3, num_condition_frames, H, W). + condition_frames: list[torch.Tensor] = [] + for frame in first_frames: + t = self.preprocess(frame, vae_scale_factor=vae_scale, height=height, width=width).to(device=device, + dtype=torch.float32) + # preprocess returns (1, 3, H, W); make it (1, 3, 1, H, W) for the temporal cat. + condition_frames.append(t.unsqueeze(2)) + vae_input_condition = torch.cat(condition_frames, dim=2) # (1, 3, num_cond, H, W) + + # 3) Padding frames as (1, 3, remaining_frames, H, W). + if remaining_frames == 0: + vae_input_pad = torch.empty( + vae_input_condition.shape[0], + vae_input_condition.shape[1], + 0, + height, + width, + device=device, + dtype=torch.float32, + ) + elif ref_pad_num == 0: + vae_input_pad = vae_input_condition.new_zeros(vae_input_condition.shape[0], vae_input_condition.shape[1], + remaining_frames, height, width) + elif ref_pad_num == -1: + ref_t = self.preprocess(random_ref_frame, vae_scale_factor=vae_scale, height=height, + width=width).to(device=device, dtype=torch.float32).unsqueeze(2) + vae_input_pad = ref_t.repeat(1, 1, remaining_frames, 1, 1) + elif ref_pad_num > 0: + ref_t = self.preprocess(random_ref_frame, vae_scale_factor=vae_scale, height=height, + width=width).to(device=device, dtype=torch.float32).unsqueeze(2) + ref_pad = ref_t.repeat(1, 1, min(ref_pad_num, remaining_frames), 1, 1) + if remaining_frames > ref_pad_num: + zero_pad = ref_t.new_zeros(ref_t.shape[0], ref_t.shape[1], remaining_frames - ref_pad_num, height, + width) + vae_input_pad = torch.cat([ref_pad, zero_pad], dim=2) + else: + vae_input_pad = ref_pad + else: + raise ValueError(f"Unsupported ref_pad_num={ref_pad_num} (expected -1, 0, or positive int)") + + video_condition = torch.cat([vae_input_condition, vae_input_pad], dim=2) + assert video_condition.shape[2] == num_frames, video_condition.shape + + # 4) VAE encode (autocast handling mirrors parent stage). + vae_dtype = PRECISION_TO_TYPE[fastvideo_args.pipeline_config.vae_precision] + vae_autocast_enabled = (vae_dtype != torch.float32) and not fastvideo_args.disable_autocast + with torch.autocast(device_type="cuda", dtype=vae_dtype, enabled=vae_autocast_enabled): + if fastvideo_args.pipeline_config.vae_tiling: + self.vae.enable_tiling() + if not vae_autocast_enabled: + video_condition = video_condition.to(vae_dtype) + encoder_output = self.vae.encode(video_condition) + + if batch.generator is None: + raise ValueError("Generator must be provided") + latent = self.retrieve_latents(encoder_output, batch.generator) + + if hasattr(self.vae, "shift_factor") and self.vae.shift_factor is not None: + shift = self.vae.shift_factor + if isinstance(shift, torch.Tensor): + shift = shift.to(latent.device, latent.dtype) + latent = latent - shift + + scaling = self.vae.scaling_factor + if isinstance(scaling, torch.Tensor): + scaling = scaling.to(latent.device, latent.dtype) + latent = latent * scaling + + # 5) y = concat(mask, latent) along channel dim. latent is (1, 16, T_lat, H_lat, W_lat). + mask_batched = msk.unsqueeze(0).to(latent.device, latent.dtype) + batch.image_latent = torch.concat([mask_batched, latent], dim=1) # (1, 20, T_lat, H_lat, W_lat) + + if hasattr(self, "maybe_free_model_hooks"): + self.maybe_free_model_hooks() + self.vae.to("cpu") + return batch + + def verify_output(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> VerificationResult: + result = VerificationResult() + result.add_check("image_latent", batch.image_latent, [V.is_tensor, V.with_dims(5)]) + return result From 2f61f434d4a02ec2eee56636a05c09296767b0e3 Mon Sep 17 00:00:00 2001 From: H1yori233 Date: Wed, 13 May 2026 00:14:24 +0000 Subject: [PATCH 02/11] inference with model variant configuration --- examples/inference/basic/basic_svi_i2v.py | 67 ++++++++++++++----- .../basic/wan/wan_svi_i2v_pipeline.py | 8 +-- fastvideo/utils.py | 12 +++- 3 files changed, 65 insertions(+), 22 deletions(-) diff --git a/examples/inference/basic/basic_svi_i2v.py b/examples/inference/basic/basic_svi_i2v.py index 8f65a72c06..be514e452a 100644 --- a/examples/inference/basic/basic_svi_i2v.py +++ b/examples/inference/basic/basic_svi_i2v.py @@ -1,29 +1,65 @@ from fastvideo import VideoGenerator -OUTPUT_PATH = "video_samples_svi_shot" +# Available variants: "shot", "film", "tom" +# Each variant uses a different LoRA + motion-frame conditioning: +# - shot: 1 motion frame + reference-frame padding (single-prompt demo) +# - film: 5 motion frames + zero padding (multi-prompt long-story continuity) +# - tom: 1 motion frame + reference-frame padding (cartoon style) +MODEL_VARIANT = "shot" + +VARIANT_CONFIG = { + "shot": { + "lora_path": "vita-video-gen/svi-model/version-1.0/svi-shot.safetensors", + "image_url": + "https://raw.githubusercontent.com/vita-epfl/Stable-Video-Infinity/main/data/toy_test/shot/frame.jpg", + "prompt": ("A sleek white motor yacht speeds across the turquoise blue sea, " + "leaving a dramatic wake of white foam behind it under a clear blue sky."), + "num_motion_frames": 1, + "ref_pad_num": -1, + }, + "film": { + "lora_path": "vita-video-gen/svi-model/version-1.0/svi-film.safetensors", + "image_url": + "https://raw.githubusercontent.com/vita-epfl/Stable-Video-Infinity/main/data/toy_test/film/frame.jpg", + "prompt": ("A Siamese kitten rests snugly inside a straw hat, its head slightly tilted " + "as it gazes curiously to the side."), + "num_motion_frames": 5, + "ref_pad_num": 0, + }, + "tom": { + "lora_path": "vita-video-gen/svi-model/version-1.0/svi-tom.safetensors", + "image_url": + "https://raw.githubusercontent.com/vita-epfl/Stable-Video-Infinity/main/data/toy_test/tom/frame.png", + "prompt": ("A static shot of the bright 1950s kitchen, turquoise cabinets and a chrome " + "sink glinting; Tom cat hovers over the counter, yellow eyes narrowed, while " + "Jerry mouse stands defiantly in a tiny milk puddle near a stack of purple plates."), + "num_motion_frames": 1, + "ref_pad_num": -1, + }, +} + +OUTPUT_PATH = "video_samples_svi" def main(): + config = VARIANT_CONFIG[MODEL_VARIANT] + generator = VideoGenerator.from_pretrained( "Wan-AI/Wan2.1-I2V-14B-480P-Diffusers", override_pipeline_cls_name="WanSVIImageToVideoPipeline", - lora_path=("./Stable-Video-Infinity/weights/Stable-Video-Infinity/" - "version-1.0/svi-shot.safetensors"), - lora_nickname="svi-shot", + lora_path=config["lora_path"], + lora_nickname=f"svi-{MODEL_VARIANT}", num_gpus=1, dit_cpu_offload=False, vae_cpu_offload=False, text_encoder_cpu_offload=True, - pin_cpu_memory=True, # set to false if low CPU RAM or hit obscure "CUDA error: Invalid argument" + # Set pin_cpu_memory to false if CPU RAM is limited and there're no frequent CPU-GPU transfer + pin_cpu_memory=True, ) - prompt = ("A sleek white motor yacht speeds across the turquoise blue sea, " - "leaving a dramatic wake of white foam behind it under a clear blue sky.") - image_path = "./Stable-Video-Infinity/data/toy_test/shot/frame.jpg" - - video = generator.generate_video( - prompt, - image_path=image_path, + generator.generate_video( + prompt=config["prompt"], + image_path=config["image_url"], output_path=OUTPUT_PATH, save_video=True, height=448, @@ -32,11 +68,10 @@ def main(): num_inference_steps=20, guidance_scale=5.0, seed=42, - # SVI knobs. Shot/Tom variants use num_motion_frames=1 + ref_pad_num=-1; - # Film uses num_motion_frames=5 + ref_pad_num=0. num_clips>1 enables motion-frame chaining. + # SVI motion-frame knobs vary per variant; num_clips>1 enables motion-frame chaining. svi_num_clips=1, - svi_num_motion_frames=1, - svi_ref_pad_num=-1, + svi_num_motion_frames=config["num_motion_frames"], + svi_ref_pad_num=config["ref_pad_num"], ) diff --git a/fastvideo/pipelines/basic/wan/wan_svi_i2v_pipeline.py b/fastvideo/pipelines/basic/wan/wan_svi_i2v_pipeline.py index b455425f00..bf66bc5b7f 100644 --- a/fastvideo/pipelines/basic/wan/wan_svi_i2v_pipeline.py +++ b/fastvideo/pipelines/basic/wan/wan_svi_i2v_pipeline.py @@ -100,15 +100,13 @@ def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> Forward if num_clips == 1: batch.prompt = prompts[0] - if batch.svi_random_ref_frame is None: - batch.svi_random_ref_frame = batch.pil_image # type: ignore[assignment] - if not batch.svi_first_frames: - assert isinstance(batch.pil_image, PIL.Image.Image) - batch.svi_first_frames = [batch.pil_image] for stage in self.stages: batch = stage(batch, fastvideo_args) return batch + # Multi-clip needs the reference image up front to construct motion frames + # for clip 0. Run InputValidationStage now to resolve image_path -> pil_image. + self.input_validation_stage(batch, fastvideo_args) assert isinstance(batch.pil_image, PIL.Image.Image) random_ref = batch.svi_random_ref_frame or batch.pil_image motion_frames: list[PIL.Image.Image] = batch.svi_first_frames or [batch.pil_image] diff --git a/fastvideo/utils.py b/fastvideo/utils.py index 37ebc0140b..9282cc66f4 100644 --- a/fastvideo/utils.py +++ b/fastvideo/utils.py @@ -583,7 +583,7 @@ def maybe_download_lora(model_name_or_path: str, local_dir: str | None = None, d model_name_or_path: Local path or Hugging Face Hub model ID local_dir: Local directory to save the model download: Whether to download the model from Hugging Face Hub - + Returns: Local path to the model """ @@ -592,6 +592,16 @@ def maybe_download_lora(model_name_or_path: str, local_dir: str | None = None, d if os.path.isfile(model_name_or_path): return model_name_or_path + # Handle "org/repo//.safetensors" by downloading the single + # specified file, for HF repos that ship several LoRA variants side by side. + parts = model_name_or_path.split("/") + if (len(parts) >= 3 and not model_name_or_path.startswith(("/", ".")) + and model_name_or_path.endswith(".safetensors")): + from huggingface_hub import hf_hub_download + repo_id = "/".join(parts[:2]) + filename = "/".join(parts[2:]) + return hf_hub_download(repo_id=repo_id, filename=filename, local_dir=local_dir) + local_path = maybe_download_model(model_name_or_path, local_dir, download) weight_name = _best_guess_weight_name(model_name_or_path, file_extension=".safetensors") From 869bd7c17ffe060ab04a4428d323f3a7aad0f0ca Mon Sep 17 00:00:00 2001 From: H1yori233 Date: Wed, 13 May 2026 01:05:37 +0000 Subject: [PATCH 03/11] motion frame handling --- fastvideo/pipelines/basic/wan/presets.py | 23 +++++++++++++++++++ .../basic/wan/wan_svi_i2v_pipeline.py | 8 ++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/fastvideo/pipelines/basic/wan/presets.py b/fastvideo/pipelines/basic/wan/presets.py index 0c4f48165b..7cdf19819e 100644 --- a/fastvideo/pipelines/basic/wan/presets.py +++ b/fastvideo/pipelines/basic/wan/presets.py @@ -120,6 +120,28 @@ }, ) +WAN_SVI_I2V_14B_480P = InferencePreset( + name="wan_svi_i2v_14b_480p", + version=1, + model_family="wan", + description="Stable-Video-Infinity multi-clip I2V on Wan 2.1 14B at 480p", + workload_type="i2v", + stage_schemas=(_DENOISE_STAGE, ), + defaults={ + "height": 448, + "width": 832, + "num_frames": 81, + "fps": 16, + "guidance_scale": 5.0, + "num_inference_steps": 20, + "negative_prompt": _NEGATIVE_PROMPT_EN, + # SVI-Shot defaults; override per variant in user kwargs. + "svi_num_clips": 1, + "svi_num_motion_frames": 1, + "svi_ref_pad_num": -1, + }, +) + # ------------------------------------------------------------------- # Wan 2.2 presets # ------------------------------------------------------------------- @@ -334,6 +356,7 @@ WAN_T2V_14B, WAN_I2V_14B_480P, WAN_I2V_14B_720P, + WAN_SVI_I2V_14B_480P, WAN_2_2_T2V_A14B, WAN_2_2_I2V_A14B, WAN_FUN_1_3B_INP, diff --git a/fastvideo/pipelines/basic/wan/wan_svi_i2v_pipeline.py b/fastvideo/pipelines/basic/wan/wan_svi_i2v_pipeline.py index bf66bc5b7f..e95f423f81 100644 --- a/fastvideo/pipelines/basic/wan/wan_svi_i2v_pipeline.py +++ b/fastvideo/pipelines/basic/wan/wan_svi_i2v_pipeline.py @@ -109,7 +109,10 @@ def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> Forward self.input_validation_stage(batch, fastvideo_args) assert isinstance(batch.pil_image, PIL.Image.Image) random_ref = batch.svi_random_ref_frame or batch.pil_image - motion_frames: list[PIL.Image.Image] = batch.svi_first_frames or [batch.pil_image] + # Clip 0: when the caller did not pre-stage motion frames, repeat the ref + # image num_motion times. For num_motion=1 (Shot/Tom) this is just [ref]; + # for num_motion=5 (Film) this matches upstream's --repeat_first_clip path. + motion_frames: list[PIL.Image.Image] = (batch.svi_first_frames or [batch.pil_image] * num_motion) clip_outputs: list[torch.Tensor] = [] for clip_idx in range(num_clips): @@ -117,6 +120,9 @@ def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> Forward batch, prompt=prompts[clip_idx], pil_image=motion_frames[0], + # Clear image_path so per-clip InputValidationStage does not + # reload the original ref and clobber motion_frames[0]. + image_path=None, svi_first_frames=motion_frames, svi_random_ref_frame=random_ref, prompt_embeds=[], From f8f49e70165588953c1b633b3bd8dd26f2578bff Mon Sep 17 00:00:00 2001 From: H1yori233 Date: Wed, 13 May 2026 01:54:22 +0000 Subject: [PATCH 04/11] update test, merge stage --- fastvideo/pipelines/stages/__init__.py | 5 +- fastvideo/pipelines/stages/image_encoding.py | 132 ++++++++++++++ .../stages/svi_image_vae_encoding.py | 156 ---------------- .../stages/test_svi_image_vae_encoding.py | 168 ++++++++++++++++++ 4 files changed, 302 insertions(+), 159 deletions(-) delete mode 100644 fastvideo/pipelines/stages/svi_image_vae_encoding.py create mode 100644 fastvideo/tests/stages/test_svi_image_vae_encoding.py diff --git a/fastvideo/pipelines/stages/__init__.py b/fastvideo/pipelines/stages/__init__.py index a10b6e5753..047f2a72a4 100644 --- a/fastvideo/pipelines/stages/__init__.py +++ b/fastvideo/pipelines/stages/__init__.py @@ -17,10 +17,9 @@ from fastvideo.pipelines.stages.encoding import EncodingStage from fastvideo.pipelines.stages.image_encoding import (ImageEncodingStage, MatrixGameImageEncodingStage, RefImageEncodingStage, ImageVAEEncodingStage, - VideoVAEEncodingStage, Hy15ImageEncodingStage, - HYWorldImageEncodingStage) + SVIImageVAEEncodingStage, VideoVAEEncodingStage, + Hy15ImageEncodingStage, HYWorldImageEncodingStage) from fastvideo.pipelines.stages.gamecraft_image_encoding import (GameCraftImageVAEEncodingStage) -from fastvideo.pipelines.stages.svi_image_vae_encoding import SVIImageVAEEncodingStage from fastvideo.pipelines.stages.input_validation import InputValidationStage from fastvideo.pipelines.stages.latent_preparation import (Cosmos25LatentPreparationStage, CosmosLatentPreparationStage, Cosmos25AutoLatentPreparationStage, diff --git a/fastvideo/pipelines/stages/image_encoding.py b/fastvideo/pipelines/stages/image_encoding.py index dda1cf1ad6..522eb872e5 100644 --- a/fastvideo/pipelines/stages/image_encoding.py +++ b/fastvideo/pipelines/stages/image_encoding.py @@ -7,6 +7,7 @@ - RefImageEncodingStage: Encodes reference image for Wan2.1 control pipeline - ImageVAEEncodingStage: Encodes images to latent space using VAE for I2V generation - VideoVAEEncodingStage: Encodes videos to latent space using VAE for V2V and control tasks +- SVIImageVAEEncodingStage: Encodes motion frames + reference padding for Stable-Video-Infinity """ import PIL @@ -863,3 +864,134 @@ def forward( self.vae.to("cpu") return batch + + +class SVIImageVAEEncodingStage(ImageVAEEncodingStage): + """ + Stage for encoding motion frames and reference padding for Stable-Video-Infinity. + """ + + def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> ForwardBatch: + if fastvideo_args.mode != ExecutionMode.INFERENCE: + raise NotImplementedError("SVIImageVAEEncodingStage only supports inference mode") + + assert isinstance(batch.height, int) and isinstance(batch.width, int) + assert isinstance(batch.num_frames, int) + height, width, num_frames = batch.height, batch.width, batch.num_frames + + first_frames = batch.svi_first_frames + if not first_frames: + assert isinstance(batch.pil_image, PIL.Image.Image) + first_frames = [batch.pil_image] + random_ref_frame = batch.svi_random_ref_frame + if random_ref_frame is None: + random_ref_frame = first_frames[0] + ref_pad_num = batch.svi_ref_pad_num if batch.svi_ref_pad_num is not None else 0 + ref_pad_cfg = batch.svi_ref_pad_cfg + + num_condition_frames = len(first_frames) + remaining_frames = num_frames - num_condition_frames + if remaining_frames < 0: + raise ValueError(f"num_frames={num_frames} smaller than len(first_frames)={num_condition_frames}") + + device = get_local_torch_device() + self.vae = self.vae.to(device) + vae_scale = self.vae.spatial_compression_ratio + latent_height = height // vae_scale + latent_width = width // vae_scale + temporal_compression = self.vae.temporal_compression_ratio + + # 1) Mask. (1, T, H/8, W/8) -> reshape -> (4, T_lat, H/8, W/8). + msk = torch.ones(1, num_frames, latent_height, latent_width, device=device, dtype=torch.float32) + if ref_pad_cfg: + msk[:, num_condition_frames:] = 0 + else: + msk[:, 1:] = 0 + msk = torch.concat( + [torch.repeat_interleave(msk[:, 0:1], repeats=temporal_compression, dim=1), msk[:, 1:]], + dim=1, + ) + msk = msk.view(1, msk.shape[1] // temporal_compression, temporal_compression, latent_height, latent_width) + msk = msk.transpose(1, 2)[0] # (4, T_lat, H/8, W/8) + + # 2) Condition frames as (1, 3, num_condition_frames, H, W). + condition_frames: list[torch.Tensor] = [] + for frame in first_frames: + t = self.preprocess(frame, vae_scale_factor=vae_scale, height=height, width=width).to(device=device, + dtype=torch.float32) + # preprocess returns (1, 3, H, W); make it (1, 3, 1, H, W) for the temporal cat. + condition_frames.append(t.unsqueeze(2)) + vae_input_condition = torch.cat(condition_frames, dim=2) # (1, 3, num_cond, H, W) + + # 3) Padding frames as (1, 3, remaining_frames, H, W). + if remaining_frames == 0: + vae_input_pad = torch.empty( + vae_input_condition.shape[0], + vae_input_condition.shape[1], + 0, + height, + width, + device=device, + dtype=torch.float32, + ) + elif ref_pad_num == 0: + vae_input_pad = vae_input_condition.new_zeros(vae_input_condition.shape[0], vae_input_condition.shape[1], + remaining_frames, height, width) + elif ref_pad_num == -1: + ref_t = self.preprocess(random_ref_frame, vae_scale_factor=vae_scale, height=height, + width=width).to(device=device, dtype=torch.float32).unsqueeze(2) + vae_input_pad = ref_t.repeat(1, 1, remaining_frames, 1, 1) + elif ref_pad_num > 0: + ref_t = self.preprocess(random_ref_frame, vae_scale_factor=vae_scale, height=height, + width=width).to(device=device, dtype=torch.float32).unsqueeze(2) + ref_pad = ref_t.repeat(1, 1, min(ref_pad_num, remaining_frames), 1, 1) + if remaining_frames > ref_pad_num: + zero_pad = ref_t.new_zeros(ref_t.shape[0], ref_t.shape[1], remaining_frames - ref_pad_num, height, + width) + vae_input_pad = torch.cat([ref_pad, zero_pad], dim=2) + else: + vae_input_pad = ref_pad + else: + raise ValueError(f"Unsupported ref_pad_num={ref_pad_num} (expected -1, 0, or positive int)") + + video_condition = torch.cat([vae_input_condition, vae_input_pad], dim=2) + assert video_condition.shape[2] == num_frames, video_condition.shape + + # 4) VAE encode (autocast handling mirrors parent stage). + vae_dtype = PRECISION_TO_TYPE[fastvideo_args.pipeline_config.vae_precision] + vae_autocast_enabled = (vae_dtype != torch.float32) and not fastvideo_args.disable_autocast + with torch.autocast(device_type="cuda", dtype=vae_dtype, enabled=vae_autocast_enabled): + if fastvideo_args.pipeline_config.vae_tiling: + self.vae.enable_tiling() + if not vae_autocast_enabled: + video_condition = video_condition.to(vae_dtype) + encoder_output = self.vae.encode(video_condition) + + if batch.generator is None: + raise ValueError("Generator must be provided") + latent = self.retrieve_latents(encoder_output, batch.generator) + + if hasattr(self.vae, "shift_factor") and self.vae.shift_factor is not None: + shift = self.vae.shift_factor + if isinstance(shift, torch.Tensor): + shift = shift.to(latent.device, latent.dtype) + latent = latent - shift + + scaling = self.vae.scaling_factor + if isinstance(scaling, torch.Tensor): + scaling = scaling.to(latent.device, latent.dtype) + latent = latent * scaling + + # 5) y = concat(mask, latent) along channel dim. latent is (1, 16, T_lat, H_lat, W_lat). + mask_batched = msk.unsqueeze(0).to(latent.device, latent.dtype) + batch.image_latent = torch.concat([mask_batched, latent], dim=1) # (1, 20, T_lat, H_lat, W_lat) + + if hasattr(self, "maybe_free_model_hooks"): + self.maybe_free_model_hooks() + self.vae.to("cpu") + return batch + + def verify_output(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> VerificationResult: + result = VerificationResult() + result.add_check("image_latent", batch.image_latent, [V.is_tensor, V.with_dims(5)]) + return result diff --git a/fastvideo/pipelines/stages/svi_image_vae_encoding.py b/fastvideo/pipelines/stages/svi_image_vae_encoding.py deleted file mode 100644 index f8aea2fdd2..0000000000 --- a/fastvideo/pipelines/stages/svi_image_vae_encoding.py +++ /dev/null @@ -1,156 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -""" -SVI Image VAE Encoding Stage for Stable-Video-Infinity I2V generation. - -This stage handles encoding a list of motion frames and reference-padded -slots to latent space with SVI-specific mask construction for I2V -conditioning. Used by the WanSVIImageToVideoPipeline. -""" - -import PIL -import torch - -from fastvideo.distributed import get_local_torch_device -from fastvideo.fastvideo_args import ExecutionMode, FastVideoArgs -from fastvideo.logger import init_logger -from fastvideo.models.vaes.common import ParallelTiledVAE -from fastvideo.pipelines.pipeline_batch_info import ForwardBatch -from fastvideo.pipelines.stages.image_encoding import ImageVAEEncodingStage -from fastvideo.pipelines.stages.validators import StageValidators as V -from fastvideo.pipelines.stages.validators import VerificationResult -from fastvideo.utils import PRECISION_TO_TYPE - -logger = init_logger(__name__) - - -class SVIImageVAEEncodingStage(ImageVAEEncodingStage): - """ - Stage for encoding motion frames and reference padding for Stable-Video-Infinity. - """ - - vae: ParallelTiledVAE - - def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> ForwardBatch: - if fastvideo_args.mode != ExecutionMode.INFERENCE: - raise NotImplementedError("SVIImageVAEEncodingStage only supports inference mode") - - assert isinstance(batch.height, int) and isinstance(batch.width, int) - assert isinstance(batch.num_frames, int) - height, width, num_frames = batch.height, batch.width, batch.num_frames - - first_frames = batch.svi_first_frames - if not first_frames: - assert isinstance(batch.pil_image, PIL.Image.Image) - first_frames = [batch.pil_image] - random_ref_frame = batch.svi_random_ref_frame - if random_ref_frame is None: - random_ref_frame = first_frames[0] - ref_pad_num = batch.svi_ref_pad_num if batch.svi_ref_pad_num is not None else 0 - ref_pad_cfg = batch.svi_ref_pad_cfg - - num_condition_frames = len(first_frames) - remaining_frames = num_frames - num_condition_frames - if remaining_frames < 0: - raise ValueError(f"num_frames={num_frames} smaller than len(first_frames)={num_condition_frames}") - - device = get_local_torch_device() - self.vae = self.vae.to(device) - vae_scale = self.vae.spatial_compression_ratio - latent_height = height // vae_scale - latent_width = width // vae_scale - temporal_compression = self.vae.temporal_compression_ratio - - # 1) Mask. (1, T, H/8, W/8) -> reshape -> (4, T_lat, H/8, W/8). - msk = torch.ones(1, num_frames, latent_height, latent_width, device=device, dtype=torch.float32) - if ref_pad_cfg: - msk[:, num_condition_frames:] = 0 - else: - msk[:, 1:] = 0 - msk = torch.concat( - [torch.repeat_interleave(msk[:, 0:1], repeats=temporal_compression, dim=1), msk[:, 1:]], - dim=1, - ) - msk = msk.view(1, msk.shape[1] // temporal_compression, temporal_compression, latent_height, latent_width) - msk = msk.transpose(1, 2)[0] # (4, T_lat, H/8, W/8) - - # 2) Condition frames as (1, 3, num_condition_frames, H, W). - condition_frames: list[torch.Tensor] = [] - for frame in first_frames: - t = self.preprocess(frame, vae_scale_factor=vae_scale, height=height, width=width).to(device=device, - dtype=torch.float32) - # preprocess returns (1, 3, H, W); make it (1, 3, 1, H, W) for the temporal cat. - condition_frames.append(t.unsqueeze(2)) - vae_input_condition = torch.cat(condition_frames, dim=2) # (1, 3, num_cond, H, W) - - # 3) Padding frames as (1, 3, remaining_frames, H, W). - if remaining_frames == 0: - vae_input_pad = torch.empty( - vae_input_condition.shape[0], - vae_input_condition.shape[1], - 0, - height, - width, - device=device, - dtype=torch.float32, - ) - elif ref_pad_num == 0: - vae_input_pad = vae_input_condition.new_zeros(vae_input_condition.shape[0], vae_input_condition.shape[1], - remaining_frames, height, width) - elif ref_pad_num == -1: - ref_t = self.preprocess(random_ref_frame, vae_scale_factor=vae_scale, height=height, - width=width).to(device=device, dtype=torch.float32).unsqueeze(2) - vae_input_pad = ref_t.repeat(1, 1, remaining_frames, 1, 1) - elif ref_pad_num > 0: - ref_t = self.preprocess(random_ref_frame, vae_scale_factor=vae_scale, height=height, - width=width).to(device=device, dtype=torch.float32).unsqueeze(2) - ref_pad = ref_t.repeat(1, 1, min(ref_pad_num, remaining_frames), 1, 1) - if remaining_frames > ref_pad_num: - zero_pad = ref_t.new_zeros(ref_t.shape[0], ref_t.shape[1], remaining_frames - ref_pad_num, height, - width) - vae_input_pad = torch.cat([ref_pad, zero_pad], dim=2) - else: - vae_input_pad = ref_pad - else: - raise ValueError(f"Unsupported ref_pad_num={ref_pad_num} (expected -1, 0, or positive int)") - - video_condition = torch.cat([vae_input_condition, vae_input_pad], dim=2) - assert video_condition.shape[2] == num_frames, video_condition.shape - - # 4) VAE encode (autocast handling mirrors parent stage). - vae_dtype = PRECISION_TO_TYPE[fastvideo_args.pipeline_config.vae_precision] - vae_autocast_enabled = (vae_dtype != torch.float32) and not fastvideo_args.disable_autocast - with torch.autocast(device_type="cuda", dtype=vae_dtype, enabled=vae_autocast_enabled): - if fastvideo_args.pipeline_config.vae_tiling: - self.vae.enable_tiling() - if not vae_autocast_enabled: - video_condition = video_condition.to(vae_dtype) - encoder_output = self.vae.encode(video_condition) - - if batch.generator is None: - raise ValueError("Generator must be provided") - latent = self.retrieve_latents(encoder_output, batch.generator) - - if hasattr(self.vae, "shift_factor") and self.vae.shift_factor is not None: - shift = self.vae.shift_factor - if isinstance(shift, torch.Tensor): - shift = shift.to(latent.device, latent.dtype) - latent = latent - shift - - scaling = self.vae.scaling_factor - if isinstance(scaling, torch.Tensor): - scaling = scaling.to(latent.device, latent.dtype) - latent = latent * scaling - - # 5) y = concat(mask, latent) along channel dim. latent is (1, 16, T_lat, H_lat, W_lat). - mask_batched = msk.unsqueeze(0).to(latent.device, latent.dtype) - batch.image_latent = torch.concat([mask_batched, latent], dim=1) # (1, 20, T_lat, H_lat, W_lat) - - if hasattr(self, "maybe_free_model_hooks"): - self.maybe_free_model_hooks() - self.vae.to("cpu") - return batch - - def verify_output(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> VerificationResult: - result = VerificationResult() - result.add_check("image_latent", batch.image_latent, [V.is_tensor, V.with_dims(5)]) - return result diff --git a/fastvideo/tests/stages/test_svi_image_vae_encoding.py b/fastvideo/tests/stages/test_svi_image_vae_encoding.py new file mode 100644 index 0000000000..07d52ab070 --- /dev/null +++ b/fastvideo/tests/stages/test_svi_image_vae_encoding.py @@ -0,0 +1,168 @@ +from unittest.mock import MagicMock + +import numpy as np +import PIL.Image +import pytest +import torch + +from fastvideo.fastvideo_args import ExecutionMode, FastVideoArgs +from fastvideo.pipelines.pipeline_batch_info import ForwardBatch +from fastvideo.pipelines.stages.image_encoding import SVIImageVAEEncodingStage + + +def _diffsynth_preprocess(img: PIL.Image.Image) -> torch.Tensor: + """Reference (diffsynth) preprocess: uint8 * (2/255) - 1, permute to CHW.""" + arr = np.array(img, dtype=np.float32) * (2 / 255) - 1 + return torch.from_numpy(arr).permute(2, 0, 1).unsqueeze(0) + + +def _reference_pre_vae( + first_frames: list[PIL.Image.Image], + random_ref_frame: PIL.Image.Image, + num_frames: int, + height: int, + width: int, + ref_pad_num: int, + ref_pad_cfg: bool, + vae_scale: int, + temporal_compression: int, +) -> tuple[torch.Tensor, torch.Tensor]: + """Mirror of encode_images_adaptive mask + video_condition construction.""" + num_condition_frames = len(first_frames) + remaining_frames = num_frames - num_condition_frames + random_ref = _diffsynth_preprocess(random_ref_frame) + + msk = torch.ones(1, num_frames, height // vae_scale, width // vae_scale, dtype=torch.float32) + if ref_pad_cfg: + msk[:, num_condition_frames:] = 0 + else: + msk[:, 1:] = 0 + msk = torch.concat( + [torch.repeat_interleave(msk[:, 0:1], repeats=temporal_compression, dim=1), msk[:, 1:]], + dim=1, + ) + msk = msk.view(1, msk.shape[1] // temporal_compression, temporal_compression, height // vae_scale, + width // vae_scale) + msk = msk.transpose(1, 2)[0] + + if len(first_frames) > 1: + frame_tensors = [_diffsynth_preprocess(f) for f in first_frames] + vae_input_condition = torch.cat(frame_tensors, dim=0).permute(1, 0, 2, 3) + else: + vae_input_condition = _diffsynth_preprocess(first_frames[0]).transpose(0, 1) + + if ref_pad_num == 0: + vae_input_pad = torch.zeros(3, remaining_frames, height, width, dtype=torch.float32) + elif ref_pad_num > 0: + pad_imgs = [random_ref.transpose(0, 1)] * ref_pad_num + if remaining_frames > ref_pad_num: + pad_imgs += [torch.zeros(3, 1, height, width, dtype=torch.float32)] * (remaining_frames - ref_pad_num) + vae_input_pad = torch.cat(pad_imgs, dim=1) + elif ref_pad_num == -1: + vae_input_pad = random_ref.transpose(0, 1).repeat(1, remaining_frames, 1, 1) + else: + raise ValueError(ref_pad_num) + + return msk, torch.concat([vae_input_condition, vae_input_pad], dim=1) + + +class _CaptureVAE: + """Mock VAE that records its encode() input and returns a zero latent.""" + + spatial_compression_ratio = 8 + temporal_compression_ratio = 4 + scaling_factor = 1.0 + shift_factor = None + + def __init__(self): + self.captured_input: torch.Tensor | None = None + + def to(self, *_a, **_kw): + return self + + def enable_tiling(self): + pass + + def encode(self, x: torch.Tensor): + self.captured_input = x.detach().clone() + t_lat = (x.shape[2] - 1) // self.temporal_compression_ratio + 1 + h_lat = x.shape[3] // self.spatial_compression_ratio + w_lat = x.shape[4] // self.spatial_compression_ratio + latent = torch.zeros(x.shape[0], 16, t_lat, h_lat, w_lat, device=x.device, dtype=torch.float32) + out = MagicMock() + out.sample = lambda _g: latent + out.mode = lambda: latent + return out + + +def _make_pil(h: int, w: int, seed: int) -> PIL.Image.Image: + arr = np.random.default_rng(seed).integers(0, 256, size=(h, w, 3), dtype=np.uint8) + return PIL.Image.fromarray(arr) + + +def _make_fastvideo_args() -> FastVideoArgs: + args = MagicMock(spec=FastVideoArgs) + args.mode = ExecutionMode.INFERENCE + args.disable_autocast = True + args.pipeline_config = MagicMock() + args.pipeline_config.vae_precision = "fp32" + args.pipeline_config.vae_tiling = False + return args + + +@pytest.mark.parametrize( + "num_motion, ref_pad_num, ref_pad_cfg", + [ + (1, -1, False), + (5, 0, False), + (1, 3, False), + (5, -1, True), + ], +) +def test_pre_vae_parity(num_motion: int, ref_pad_num: int, ref_pad_cfg: bool): + height, width, num_frames = 48, 96, 17 + + first_frames = [_make_pil(height, width, seed=i) for i in range(num_motion)] + random_ref = _make_pil(height, width, seed=999) + + expected_msk, expected_video = _reference_pre_vae( + first_frames=first_frames, + random_ref_frame=random_ref, + num_frames=num_frames, + height=height, + width=width, + ref_pad_num=ref_pad_num, + ref_pad_cfg=ref_pad_cfg, + vae_scale=8, + temporal_compression=4, + ) + + vae = _CaptureVAE() + stage = SVIImageVAEEncodingStage(vae=vae) # type: ignore[arg-type] + + batch = ForwardBatch( + data_type="video", + generator=torch.Generator().manual_seed(0), + height=height, + width=width, + num_frames=num_frames, + pil_image=first_frames[0], + svi_first_frames=first_frames, + svi_random_ref_frame=random_ref, + svi_ref_pad_num=ref_pad_num, + svi_ref_pad_cfg=ref_pad_cfg, + ) + stage.forward(batch, _make_fastvideo_args()) + + assert vae.captured_input is not None + # Stage feeds (1, 3, T, H, W); reference produces (3, T, H, W). + # .cpu() keeps the comparison device-agnostic when get_local_torch_device() picks CUDA. + actual_video = vae.captured_input.squeeze(0).cpu() + assert actual_video.shape == expected_video.shape + # FastVideo preprocess is (uint8/255) -> 2x-1; diffsynth is uint8*(2/255) - 1. Same math, ~1 ULP apart. + torch.testing.assert_close(actual_video, expected_video, atol=2e-7, rtol=1e-5) + + assert batch.image_latent is not None + actual_msk = batch.image_latent[0, :4].cpu() + assert actual_msk.shape == expected_msk.shape + torch.testing.assert_close(actual_msk, expected_msk, atol=0.0, rtol=0.0) From 189ab4191172131b84ebec1e2ea1dc7b43172b1f Mon Sep 17 00:00:00 2001 From: H1yori233 Date: Thu, 14 May 2026 07:13:00 +0000 Subject: [PATCH 05/11] small fix --- fastvideo/pipelines/basic/wan/wan_svi_i2v_pipeline.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/fastvideo/pipelines/basic/wan/wan_svi_i2v_pipeline.py b/fastvideo/pipelines/basic/wan/wan_svi_i2v_pipeline.py index e95f423f81..90bc081a22 100644 --- a/fastvideo/pipelines/basic/wan/wan_svi_i2v_pipeline.py +++ b/fastvideo/pipelines/basic/wan/wan_svi_i2v_pipeline.py @@ -109,10 +109,7 @@ def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> Forward self.input_validation_stage(batch, fastvideo_args) assert isinstance(batch.pil_image, PIL.Image.Image) random_ref = batch.svi_random_ref_frame or batch.pil_image - # Clip 0: when the caller did not pre-stage motion frames, repeat the ref - # image num_motion times. For num_motion=1 (Shot/Tom) this is just [ref]; - # for num_motion=5 (Film) this matches upstream's --repeat_first_clip path. - motion_frames: list[PIL.Image.Image] = (batch.svi_first_frames or [batch.pil_image] * num_motion) + motion_frames: list[PIL.Image.Image] = (batch.svi_first_frames or [batch.pil_image]) clip_outputs: list[torch.Tensor] = [] for clip_idx in range(num_clips): From 332f91826d2a27086d5bbc7cb4923d0a254a5c33 Mon Sep 17 00:00:00 2001 From: H1yori233 Date: Thu, 14 May 2026 08:40:49 +0000 Subject: [PATCH 06/11] align scheduler --- fastvideo/pipelines/basic/wan/wan_svi_i2v_pipeline.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/fastvideo/pipelines/basic/wan/wan_svi_i2v_pipeline.py b/fastvideo/pipelines/basic/wan/wan_svi_i2v_pipeline.py index 90bc081a22..d24678ba23 100644 --- a/fastvideo/pipelines/basic/wan/wan_svi_i2v_pipeline.py +++ b/fastvideo/pipelines/basic/wan/wan_svi_i2v_pipeline.py @@ -13,6 +13,7 @@ from fastvideo.fastvideo_args import FastVideoArgs from fastvideo.logger import init_logger +from fastvideo.models.schedulers.scheduling_flow_match_euler_discrete import FlowMatchEulerDiscreteScheduler from fastvideo.pipelines.basic.wan.wan_i2v_pipeline import WanImageToVideoPipeline from fastvideo.pipelines.pipeline_batch_info import ForwardBatch @@ -39,6 +40,8 @@ class WanSVIImageToVideoPipeline(WanImageToVideoPipeline): def create_pipeline_stages(self, fastvideo_args: FastVideoArgs): """Set up pipeline stages with proper dependency injection.""" + self.modules["scheduler"] = FlowMatchEulerDiscreteScheduler(num_train_timesteps=1000, shift=5.0) + self.add_stage(stage_name="input_validation_stage", stage=InputValidationStage()) self.add_stage( stage_name="prompt_encoding_stage", @@ -86,6 +89,9 @@ def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> Forward if not self.post_init_called: self.post_init() + n_steps = int(batch.num_inference_steps) + batch.sigmas = torch.linspace(1.0, 0.0, n_steps + 1)[:-1].tolist() + num_clips = max(1, int(batch.svi_num_clips or 1)) num_motion = max(1, int(batch.svi_num_motion_frames or 1)) From d2c194417c219dc8e336ab6f122b91aa1dbe1e9b Mon Sep 17 00:00:00 2001 From: H1yori233 Date: Wed, 3 Jun 2026 04:41:27 +0000 Subject: [PATCH 07/11] some fix --- .../basic/wan/wan_svi_i2v_pipeline.py | 25 +++++-- fastvideo/pipelines/lora_pipeline.py | 7 +- .../lora/test_maybe_download_lora.py | 73 +++++++++++++++++++ .../pipelines/test_svi_multiclip_stitch.py | 53 ++++++++++++++ 4 files changed, 150 insertions(+), 8 deletions(-) create mode 100644 fastvideo/tests/inference/lora/test_maybe_download_lora.py create mode 100644 fastvideo/tests/pipelines/test_svi_multiclip_stitch.py diff --git a/fastvideo/pipelines/basic/wan/wan_svi_i2v_pipeline.py b/fastvideo/pipelines/basic/wan/wan_svi_i2v_pipeline.py index d24678ba23..402386947a 100644 --- a/fastvideo/pipelines/basic/wan/wan_svi_i2v_pipeline.py +++ b/fastvideo/pipelines/basic/wan/wan_svi_i2v_pipeline.py @@ -33,6 +33,21 @@ def _tensor_to_pil_list(frames: torch.Tensor) -> list[PIL.Image.Image]: return [PIL.Image.fromarray(a) for a in arr] +def _validate_multiclip_frames(num_motion: int, num_frames: int) -> None: + """num_motion must be < num_frames, else follow-up clips stitch to empty.""" + if num_motion >= num_frames: + raise ValueError(f"svi_num_motion_frames ({num_motion}) must be smaller than num_frames ({num_frames}) " + "for multi-clip generation; otherwise stitched follow-up clips would be empty.") + + +def _stitch_clip_outputs(clip_outputs: list[torch.Tensor], num_motion: int) -> torch.Tensor: + """Concat per-clip (B,C,T,H,W) videos on time, dropping each follow-up's leading num_motion frames.""" + concatenated = [clip_outputs[0]] + for video in clip_outputs[1:]: + concatenated.append(video[:, :, num_motion:, :, :]) + return torch.cat(concatenated, dim=2) + + class WanSVIImageToVideoPipeline(WanImageToVideoPipeline): """ Pipeline for Stable-Video-Infinity multi-clip I2V generation on Wan 2.1. @@ -110,6 +125,9 @@ def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> Forward batch = stage(batch, fastvideo_args) return batch + num_frames = int(batch.num_frames) if batch.num_frames is not None else 0 + _validate_multiclip_frames(num_motion, num_frames) + # Multi-clip needs the reference image up front to construct motion frames # for clip 0. Run InputValidationStage now to resolve image_path -> pil_image. self.input_validation_stage(batch, fastvideo_args) @@ -156,12 +174,7 @@ def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> Forward tail = clip_batch.output[0, :, -num_motion:, :, :] motion_frames = _tensor_to_pil_list(tail) - # Drop the first num_motion frames of each follow-up clip to avoid duplicating - # the previous clip's tail in the stitched output. - concatenated = [clip_outputs[0]] - for video in clip_outputs[1:]: - concatenated.append(video[:, :, num_motion:, :, :]) - batch.output = torch.cat(concatenated, dim=2) + batch.output = _stitch_clip_outputs(clip_outputs, num_motion) return batch diff --git a/fastvideo/pipelines/lora_pipeline.py b/fastvideo/pipelines/lora_pipeline.py index a9e9203760..c71f2db4fc 100644 --- a/fastvideo/pipelines/lora_pipeline.py +++ b/fastvideo/pipelines/lora_pipeline.py @@ -325,9 +325,12 @@ def set_lora_adapter(self, for name, weight in lora_state_dict.items(): # Extract weights (lora_A, lora_B, and lora_alpha) name = name.replace("diffusion_model.", "") - name = name.replace("pipe.dit.", "") + # Guarded so non-SVI / non-PEFT adapters are provably untouched. + if "pipe.dit." in name: + name = name.replace("pipe.dit.", "") name = name.replace(".weight", "") - name = name.replace(".default", "") + if ".default" in name: + name = name.replace(".default", "") if "lora_alpha" in name: # Store alpha with minimal mapping - same processing as lora_A/lora_B diff --git a/fastvideo/tests/inference/lora/test_maybe_download_lora.py b/fastvideo/tests/inference/lora/test_maybe_download_lora.py new file mode 100644 index 0000000000..6391207bf0 --- /dev/null +++ b/fastvideo/tests/inference/lora/test_maybe_download_lora.py @@ -0,0 +1,73 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Path-routing tests for fastvideo.utils.maybe_download_lora.""" +from __future__ import annotations + +import huggingface_hub + +import fastvideo.utils as fv_utils +from fastvideo.utils import maybe_download_lora + + +def test_triple_slash_downloads_single_file(monkeypatch): + """org/repo/sub/file.safetensors -> hf_hub_download(repo_id=org/repo, filename=sub/file).""" + calls: list[dict] = [] + + def fake_hf_hub_download(*, repo_id, filename, local_dir=None, **kwargs): + calls.append({"repo_id": repo_id, "filename": filename, "local_dir": local_dir}) + return f"/cache/{repo_id}/{filename}" + + monkeypatch.setattr(huggingface_hub, "hf_hub_download", fake_hf_hub_download) + + result = maybe_download_lora("vita-video-gen/svi-model/version-1.0/svi-shot.safetensors") + + assert len(calls) == 1 + assert calls[0]["repo_id"] == "vita-video-gen/svi-model" + assert calls[0]["filename"] == "version-1.0/svi-shot.safetensors" + assert result == "/cache/vita-video-gen/svi-model/version-1.0/svi-shot.safetensors" + + +def test_triple_slash_keeps_only_first_two_segments_as_repo(monkeypatch): + """A deeper nesting still maps repo_id to the first two segments.""" + captured: dict = {} + + def fake_hf_hub_download(*, repo_id, filename, local_dir=None, **kwargs): + captured["repo_id"] = repo_id + captured["filename"] = filename + return "ok" + + monkeypatch.setattr(huggingface_hub, "hf_hub_download", fake_hf_hub_download) + + maybe_download_lora("org/repo/a/b/c/weights.safetensors") + + assert captured["repo_id"] == "org/repo" + assert captured["filename"] == "a/b/c/weights.safetensors" + + +def test_local_file_short_circuits(tmp_path, monkeypatch): + """An existing local .safetensors file is returned verbatim, no download.""" + + def boom(*args, **kwargs): + raise AssertionError("hf_hub_download must not be called for a local file") + + monkeypatch.setattr(huggingface_hub, "hf_hub_download", boom) + + f = tmp_path / "version-1.0" / "svi-shot.safetensors" + f.parent.mkdir(parents=True) + f.write_bytes(b"\x00") + + assert maybe_download_lora(str(f)) == str(f) + + +def test_plain_repo_id_falls_through_to_repo_download(monkeypatch): + """A two-segment HF id (no .safetensors suffix) does NOT hit the single-file branch.""" + + def boom(*args, **kwargs): + raise AssertionError("two-segment repo id must not take the single-file branch") + + monkeypatch.setattr(huggingface_hub, "hf_hub_download", boom) + monkeypatch.setattr(fv_utils, "maybe_download_model", lambda *a, **k: "/cache/org/repo") + monkeypatch.setattr(fv_utils, "_best_guess_weight_name", lambda *a, **k: "adapter.safetensors") + + result = maybe_download_lora("org/repo") + + assert result == "/cache/org/repo/adapter.safetensors" diff --git a/fastvideo/tests/pipelines/test_svi_multiclip_stitch.py b/fastvideo/tests/pipelines/test_svi_multiclip_stitch.py new file mode 100644 index 0000000000..7c1eea3b21 --- /dev/null +++ b/fastvideo/tests/pipelines/test_svi_multiclip_stitch.py @@ -0,0 +1,53 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Tests for SVI multi-clip stitch index math and the frame-budget guard.""" +from __future__ import annotations + +import pytest +import torch + +from fastvideo.pipelines.basic.wan.wan_svi_i2v_pipeline import (_stitch_clip_outputs, _validate_multiclip_frames) + + +def _clip(num_frames: int, fill: float) -> torch.Tensor: + # (B=1, C=3, T=num_frames, H=2, W=2) + return torch.full((1, 3, num_frames, 2, 2), fill, dtype=torch.float32) + + +def test_stitch_drops_motion_overlap_from_followups(): + num_motion = 2 + clips = [_clip(9, 0.0), _clip(9, 1.0), _clip(9, 2.0)] + + out = _stitch_clip_outputs(clips, num_motion) + + expected_t = 9 + (9 - num_motion) + (9 - num_motion) + assert out.shape == (1, 3, expected_t, 2, 2) + + # Provenance: 9 from clip0, then 7 from clip1, then 7 from clip2. + assert torch.all(out[:, :, :9] == 0.0) + assert torch.all(out[:, :, 9:9 + 7] == 1.0) + assert torch.all(out[:, :, 9 + 7:] == 2.0) + + +def test_stitch_single_clip_is_identity(): + clip = _clip(5, 3.0) + out = _stitch_clip_outputs([clip], num_motion=2) + assert torch.equal(out, clip) + + +def test_stitch_num_motion_one_drops_single_frame(): + clips = [_clip(4, 0.0), _clip(4, 1.0)] + out = _stitch_clip_outputs(clips, num_motion=1) + assert out.shape[2] == 4 + 3 + + +def test_validate_rejects_motion_ge_frames(): + with pytest.raises(ValueError, match="must be smaller than num_frames"): + _validate_multiclip_frames(num_motion=5, num_frames=5) + with pytest.raises(ValueError, match="must be smaller than num_frames"): + _validate_multiclip_frames(num_motion=8, num_frames=5) + + +def test_validate_accepts_motion_lt_frames(): + # Should not raise. + _validate_multiclip_frames(num_motion=1, num_frames=81) + _validate_multiclip_frames(num_motion=5, num_frames=6) From 5070bb37ccc82c49b1f9371fc840c18f19654c40 Mon Sep 17 00:00:00 2001 From: H1yori233 Date: Tue, 14 Jul 2026 08:37:22 +0000 Subject: [PATCH 08/11] Fix SVI multi-clip integration and review issues --- .../inference_schema_parity_inventory.yaml | 5 ++ examples/inference/basic/basic_svi_i2v.py | 36 +++++++--- fastvideo/api/sampling_param.py | 1 + fastvideo/pipelines/basic/wan/presets.py | 1 + .../basic/wan/wan_svi_i2v_pipeline.py | 57 +++++++++++---- fastvideo/pipelines/lora_pipeline.py | 17 +++-- fastvideo/pipelines/pipeline_batch_info.py | 1 + .../lora/test_maybe_download_lora.py | 32 +++++++++ fastvideo/tests/modal/pr_test.py | 2 +- .../pipelines/test_svi_multiclip_stitch.py | 71 ++++++++++++++++++- fastvideo/utils.py | 3 +- 11 files changed, 194 insertions(+), 32 deletions(-) diff --git a/docs/design/inference_schema_parity_inventory.yaml b/docs/design/inference_schema_parity_inventory.yaml index cf4eb7386e..947a474e76 100644 --- a/docs/design/inference_schema_parity_inventory.yaml +++ b/docs/design/inference_schema_parity_inventory.yaml @@ -476,6 +476,11 @@ surfaces: trajectory_type: request.extensions.gen3c.trajectory_type movement_distance: request.extensions.gen3c.movement_distance camera_rotation: request.extensions.gen3c.camera_rotation + svi_ref_pad_num: request.extensions.svi.ref_pad_num + svi_ref_pad_cfg: request.extensions.svi.ref_pad_cfg + svi_num_clips: request.extensions.svi.num_clips + svi_num_motion_frames: request.extensions.svi.num_motion_frames + svi_clip_prompts: request.extensions.svi.clip_prompts prompt_attention_mask: request.extensions.hyworld.prompt_attention_mask negative_attention_mask: request.extensions.hyworld.negative_attention_mask camera_states: request.extensions.hunyuangamecraft.camera_states diff --git a/examples/inference/basic/basic_svi_i2v.py b/examples/inference/basic/basic_svi_i2v.py index be514e452a..3741ab9ba7 100644 --- a/examples/inference/basic/basic_svi_i2v.py +++ b/examples/inference/basic/basic_svi_i2v.py @@ -2,7 +2,7 @@ # Available variants: "shot", "film", "tom" # Each variant uses a different LoRA + motion-frame conditioning: -# - shot: 1 motion frame + reference-frame padding (single-prompt demo) +# - shot: 1 motion frame + reference-frame padding # - film: 5 motion frames + zero padding (multi-prompt long-story continuity) # - tom: 1 motion frame + reference-frame padding (cartoon style) MODEL_VARIANT = "shot" @@ -12,8 +12,13 @@ "lora_path": "vita-video-gen/svi-model/version-1.0/svi-shot.safetensors", "image_url": "https://raw.githubusercontent.com/vita-epfl/Stable-Video-Infinity/main/data/toy_test/shot/frame.jpg", - "prompt": ("A sleek white motor yacht speeds across the turquoise blue sea, " - "leaving a dramatic wake of white foam behind it under a clear blue sky."), + "prompts": [ + ("A sleek white motor yacht speeds across the turquoise blue sea, " + "leaving a dramatic wake of white foam behind it under a clear blue sky."), + ("The camera follows the same white yacht from behind as it continues across " + "the turquoise sea, its foamy wake widening under the clear sky."), + ], + "num_clips": 2, "num_motion_frames": 1, "ref_pad_num": -1, }, @@ -21,8 +26,13 @@ "lora_path": "vita-video-gen/svi-model/version-1.0/svi-film.safetensors", "image_url": "https://raw.githubusercontent.com/vita-epfl/Stable-Video-Infinity/main/data/toy_test/film/frame.jpg", - "prompt": ("A Siamese kitten rests snugly inside a straw hat, its head slightly tilted " - "as it gazes curiously to the side."), + "prompts": [ + ("A Siamese kitten rests snugly inside a straw hat, its head slightly tilted " + "as it gazes curiously to the side."), + ("The same Siamese kitten slowly lifts its head from the straw hat and looks " + "toward the camera while the warm room remains unchanged."), + ], + "num_clips": 2, "num_motion_frames": 5, "ref_pad_num": 0, }, @@ -30,9 +40,14 @@ "lora_path": "vita-video-gen/svi-model/version-1.0/svi-tom.safetensors", "image_url": "https://raw.githubusercontent.com/vita-epfl/Stable-Video-Infinity/main/data/toy_test/tom/frame.png", - "prompt": ("A static shot of the bright 1950s kitchen, turquoise cabinets and a chrome " - "sink glinting; Tom cat hovers over the counter, yellow eyes narrowed, while " - "Jerry mouse stands defiantly in a tiny milk puddle near a stack of purple plates."), + "prompts": [ + ("A static shot of the bright 1950s kitchen, turquoise cabinets and a chrome " + "sink glinting; Tom cat hovers over the counter, yellow eyes narrowed, while " + "Jerry mouse stands defiantly in a tiny milk puddle near a stack of purple plates."), + ("In the same bright 1950s kitchen, Tom leans closer across the counter while Jerry " + "holds his ground beside the purple plates, preserving the cartoon style."), + ], + "num_clips": 2, "num_motion_frames": 1, "ref_pad_num": -1, }, @@ -58,7 +73,7 @@ def main(): ) generator.generate_video( - prompt=config["prompt"], + prompt=config["prompts"][0], image_path=config["image_url"], output_path=OUTPUT_PATH, save_video=True, @@ -69,7 +84,8 @@ def main(): guidance_scale=5.0, seed=42, # SVI motion-frame knobs vary per variant; num_clips>1 enables motion-frame chaining. - svi_num_clips=1, + svi_num_clips=config["num_clips"], + svi_clip_prompts=config["prompts"], svi_num_motion_frames=config["num_motion_frames"], svi_ref_pad_num=config["ref_pad_num"], ) diff --git a/fastvideo/api/sampling_param.py b/fastvideo/api/sampling_param.py index 93c90e09a1..213407ce1c 100644 --- a/fastvideo/api/sampling_param.py +++ b/fastvideo/api/sampling_param.py @@ -111,6 +111,7 @@ class SamplingParam: svi_ref_pad_cfg: bool = False # widens y-mask to len(first_frames) instead of {first frame only} svi_num_clips: int = 1 # >1 enables motion-frame chaining svi_num_motion_frames: int = 1 # tail frames carried from clip K to clip K+1 (1=Shot/Tom, 5=Film) + svi_clip_prompts: list[str] | None = None # optional one-to-one prompt list for multi-clip generation # LTX-2 multi-modal CFG and STG. # Class-level defaults match the *distilled* LTX-2 schedule diff --git a/fastvideo/pipelines/basic/wan/presets.py b/fastvideo/pipelines/basic/wan/presets.py index 8b57c15a53..a48aacbca7 100644 --- a/fastvideo/pipelines/basic/wan/presets.py +++ b/fastvideo/pipelines/basic/wan/presets.py @@ -139,6 +139,7 @@ "svi_num_clips": 1, "svi_num_motion_frames": 1, "svi_ref_pad_num": -1, + "svi_clip_prompts": None, }, ) diff --git a/fastvideo/pipelines/basic/wan/wan_svi_i2v_pipeline.py b/fastvideo/pipelines/basic/wan/wan_svi_i2v_pipeline.py index 402386947a..6a0552a4f2 100644 --- a/fastvideo/pipelines/basic/wan/wan_svi_i2v_pipeline.py +++ b/fastvideo/pipelines/basic/wan/wan_svi_i2v_pipeline.py @@ -48,6 +48,34 @@ def _stitch_clip_outputs(clip_outputs: list[torch.Tensor], num_motion: int) -> t return torch.cat(concatenated, dim=2) +def _resolve_clip_prompts( + prompt: str | list[str] | None, + clip_prompts: list[str] | None, + num_clips: int, +) -> list[str]: + """Return exactly one non-empty prompt per generated clip.""" + if clip_prompts is None: + if not isinstance(prompt, str) or not prompt.strip(): + raise ValueError("SVI requires a non-empty primary prompt") + return [prompt] * num_clips + + if len(clip_prompts) != num_clips: + raise ValueError( + f"svi_clip_prompts must contain exactly svi_num_clips entries " + f"({num_clips}), but got {len(clip_prompts)}" + ) + if any(not isinstance(item, str) or not item.strip() for item in clip_prompts): + raise ValueError("svi_clip_prompts entries must be non-empty strings") + return list(clip_prompts) + + +def _clip_seed(seed: int | None, clip_idx: int) -> int: + """Match the SVI reference by varying diffusion noise per clip.""" + if seed is None: + raise ValueError("SVI requires a seed") + return int(seed) + clip_idx + + class WanSVIImageToVideoPipeline(WanImageToVideoPipeline): """ Pipeline for Stable-Video-Infinity multi-clip I2V generation on Wan 2.1. @@ -55,7 +83,11 @@ class WanSVIImageToVideoPipeline(WanImageToVideoPipeline): def create_pipeline_stages(self, fastvideo_args: FastVideoArgs): """Set up pipeline stages with proper dependency injection.""" - self.modules["scheduler"] = FlowMatchEulerDiscreteScheduler(num_train_timesteps=1000, shift=5.0) + flow_shift = fastvideo_args.pipeline_config.flow_shift + self.modules["scheduler"] = FlowMatchEulerDiscreteScheduler( + num_train_timesteps=1000, + shift=5.0 if flow_shift is None else flow_shift, + ) self.add_stage(stage_name="input_validation_stage", stage=InputValidationStage()) self.add_stage( @@ -105,19 +137,13 @@ def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> Forward self.post_init() n_steps = int(batch.num_inference_steps) - batch.sigmas = torch.linspace(1.0, 0.0, n_steps + 1)[:-1].tolist() + if batch.sigmas is None: + batch.sigmas = torch.linspace(1.0, 0.0, n_steps + 1)[:-1].tolist() num_clips = max(1, int(batch.svi_num_clips or 1)) num_motion = max(1, int(batch.svi_num_motion_frames or 1)) - if isinstance(batch.prompt, list): - prompts: list[str | None] = list(batch.prompt) - else: - prompts = [batch.prompt] - if len(prompts) < num_clips: - prompts = prompts + [prompts[-1]] * (num_clips - len(prompts)) - elif len(prompts) > num_clips: - prompts = prompts[:num_clips] + prompts = _resolve_clip_prompts(batch.prompt, batch.svi_clip_prompts, num_clips) if num_clips == 1: batch.prompt = prompts[0] @@ -140,7 +166,8 @@ def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> Forward clip_batch = dataclasses.replace( batch, prompt=prompts[clip_idx], - pil_image=motion_frames[0], + seed=_clip_seed(batch.seed, clip_idx), + pil_image=motion_frames[-1], # Clear image_path so per-clip InputValidationStage does not # reload the original ref and clobber motion_frames[0]. image_path=None, @@ -168,7 +195,13 @@ def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> Forward assert clip_batch.output is not None clip_outputs.append(clip_batch.output) - logger.info("SVI clip %d/%d generated, frames shape=%s", clip_idx + 1, num_clips, clip_batch.output.shape) + logger.info( + "SVI clip %d/%d generated with seed=%d, frames shape=%s", + clip_idx + 1, + num_clips, + clip_batch.seed, + clip_batch.output.shape, + ) if clip_idx + 1 < num_clips: tail = clip_batch.output[0, :, -num_motion:, :, :] diff --git a/fastvideo/pipelines/lora_pipeline.py b/fastvideo/pipelines/lora_pipeline.py index c71f2db4fc..b993772c94 100644 --- a/fastvideo/pipelines/lora_pipeline.py +++ b/fastvideo/pipelines/lora_pipeline.py @@ -2,6 +2,7 @@ from collections import defaultdict from collections.abc import Hashable from contextlib import nullcontext +import re from typing import Any from collections.abc import Generator @@ -29,6 +30,14 @@ logger = init_logger(__name__) +def _normalize_lora_key(name: str) -> str: + """Normalize known DiffSynth/PEFT wrappers without rewriting module names.""" + name = name.removeprefix("diffusion_model.") + name = name.removeprefix("pipe.dit.") + name = re.sub(r"(\.lora_[AB])\.default(?=\.weight$)", r"\1", name) + return name.removesuffix(".weight") + + def _get_hook_ctx(module: nn.Module | None): if module is None: return nullcontext() @@ -324,13 +333,7 @@ def set_lora_adapter(self, to_merge_params: defaultdict[Hashable, dict[Any, Any]] = (defaultdict(dict)) for name, weight in lora_state_dict.items(): # Extract weights (lora_A, lora_B, and lora_alpha) - name = name.replace("diffusion_model.", "") - # Guarded so non-SVI / non-PEFT adapters are provably untouched. - if "pipe.dit." in name: - name = name.replace("pipe.dit.", "") - name = name.replace(".weight", "") - if ".default" in name: - name = name.replace(".default", "") + name = _normalize_lora_key(name) if "lora_alpha" in name: # Store alpha with minimal mapping - same processing as lora_A/lora_B diff --git a/fastvideo/pipelines/pipeline_batch_info.py b/fastvideo/pipelines/pipeline_batch_info.py index b1caa02010..af859a1c89 100644 --- a/fastvideo/pipelines/pipeline_batch_info.py +++ b/fastvideo/pipelines/pipeline_batch_info.py @@ -162,6 +162,7 @@ class ForwardBatch: svi_ref_pad_cfg: bool = False # widens y-mask to len(first_frames) instead of frame 0 only svi_num_clips: int = 1 # >1 enables motion-frame chaining svi_num_motion_frames: int = 1 # Tail frames carried clip K -> K+1 (1=Shot/Tom, 5=Film) + svi_clip_prompts: list[str] | None = None # One prompt per clip; primary prompt is reused when omitted # Latent dimensions height_latents: list[int] | int | None = None diff --git a/fastvideo/tests/inference/lora/test_maybe_download_lora.py b/fastvideo/tests/inference/lora/test_maybe_download_lora.py index 6391207bf0..afc3f919e0 100644 --- a/fastvideo/tests/inference/lora/test_maybe_download_lora.py +++ b/fastvideo/tests/inference/lora/test_maybe_download_lora.py @@ -2,9 +2,12 @@ """Path-routing tests for fastvideo.utils.maybe_download_lora.""" from __future__ import annotations +from contextlib import contextmanager + import huggingface_hub import fastvideo.utils as fv_utils +from fastvideo.pipelines.lora_pipeline import _normalize_lora_key from fastvideo.utils import maybe_download_lora @@ -18,12 +21,26 @@ def fake_hf_hub_download(*, repo_id, filename, local_dir=None, **kwargs): monkeypatch.setattr(huggingface_hub, "hf_hub_download", fake_hf_hub_download) + lock_events: list[str] = [] + + @contextmanager + def fake_lock(path): + lock_events.append(f"enter:{path}") + yield + lock_events.append(f"exit:{path}") + + monkeypatch.setattr(fv_utils, "get_lock", fake_lock) + result = maybe_download_lora("vita-video-gen/svi-model/version-1.0/svi-shot.safetensors") assert len(calls) == 1 assert calls[0]["repo_id"] == "vita-video-gen/svi-model" assert calls[0]["filename"] == "version-1.0/svi-shot.safetensors" assert result == "/cache/vita-video-gen/svi-model/version-1.0/svi-shot.safetensors" + assert lock_events == [ + "enter:vita-video-gen/svi-model/version-1.0/svi-shot.safetensors", + "exit:vita-video-gen/svi-model/version-1.0/svi-shot.safetensors", + ] def test_triple_slash_keeps_only_first_two_segments_as_repo(monkeypatch): @@ -71,3 +88,18 @@ def boom(*args, **kwargs): result = maybe_download_lora("org/repo") assert result == "/cache/org/repo/adapter.safetensors" + + +def test_normalize_lora_key_only_strips_peft_default_adapter_segment(): + assert _normalize_lora_key("pipe.dit.blocks.0.attn.lora_A.default.weight") == ( + "blocks.0.attn.lora_A" + ) + assert _normalize_lora_key("blocks.default.attn.lora_A.weight") == ( + "blocks.default.attn.lora_A" + ) + + +def test_normalize_lora_key_only_strips_pipe_dit_prefix(): + assert _normalize_lora_key("module.pipe.dit.attn.lora_B.weight") == ( + "module.pipe.dit.attn.lora_B" + ) diff --git a/fastvideo/tests/modal/pr_test.py b/fastvideo/tests/modal/pr_test.py index 038fc72671..04253b7671 100644 --- a/fastvideo/tests/modal/pr_test.py +++ b/fastvideo/tests/modal/pr_test.py @@ -276,7 +276,7 @@ def run_self_forcing_tests(): @app.function(gpu="L40S:1", image=image, timeout=900, secrets=[ci_env_secret]) def run_unit_test(): run_test( - "pytest ./fastvideo/tests/api/ ./fastvideo/tests/contract/ ./fastvideo/tests/dataset/ ./fastvideo/tests/workflow/ ./fastvideo/tests/entrypoints/ ./fastvideo/tests/train/ ./fastvideo/tests/stages/ ./fastvideo/tests/ops/ ./fastvideo/tests/training/test_trackers.py ./fastvideo/tests/attention/test_sdpa_metadata_mask_contract.py --ignore=./fastvideo/tests/entrypoints/test_openai_api_integration.py --ignore=./fastvideo/tests/train/models --ignore=./fastvideo/tests/train/methods -vs" + "pytest ./fastvideo/tests/api/ ./fastvideo/tests/contract/ ./fastvideo/tests/dataset/ ./fastvideo/tests/workflow/ ./fastvideo/tests/entrypoints/ ./fastvideo/tests/train/ ./fastvideo/tests/stages/ ./fastvideo/tests/pipelines/ ./fastvideo/tests/ops/ ./fastvideo/tests/training/test_trackers.py ./fastvideo/tests/attention/test_sdpa_metadata_mask_contract.py --ignore=./fastvideo/tests/entrypoints/test_openai_api_integration.py --ignore=./fastvideo/tests/train/models --ignore=./fastvideo/tests/train/methods -vs" ) diff --git a/fastvideo/tests/pipelines/test_svi_multiclip_stitch.py b/fastvideo/tests/pipelines/test_svi_multiclip_stitch.py index 7c1eea3b21..75a4ec2e48 100644 --- a/fastvideo/tests/pipelines/test_svi_multiclip_stitch.py +++ b/fastvideo/tests/pipelines/test_svi_multiclip_stitch.py @@ -2,10 +2,20 @@ """Tests for SVI multi-clip stitch index math and the frame-budget guard.""" from __future__ import annotations +from types import SimpleNamespace + import pytest import torch +from PIL import Image -from fastvideo.pipelines.basic.wan.wan_svi_i2v_pipeline import (_stitch_clip_outputs, _validate_multiclip_frames) +from fastvideo.pipelines.basic.wan.wan_svi_i2v_pipeline import ( + WanSVIImageToVideoPipeline, + _clip_seed, + _resolve_clip_prompts, + _stitch_clip_outputs, + _validate_multiclip_frames, +) +from fastvideo.pipelines.pipeline_batch_info import ForwardBatch def _clip(num_frames: int, fill: float) -> torch.Tensor: @@ -51,3 +61,62 @@ def test_validate_accepts_motion_lt_frames(): # Should not raise. _validate_multiclip_frames(num_motion=1, num_frames=81) _validate_multiclip_frames(num_motion=5, num_frames=6) + + +def test_clip_seed_offsets_each_chunk(): + assert [_clip_seed(42, idx) for idx in range(3)] == [42, 43, 44] + + +def test_resolve_clip_prompts_reuses_primary_prompt_by_default(): + assert _resolve_clip_prompts("one prompt", None, 3) == ["one prompt"] * 3 + + +def test_resolve_clip_prompts_accepts_exact_per_clip_prompts(): + prompts = ["first", "second"] + assert _resolve_clip_prompts("fallback", prompts, 2) == prompts + + +def test_resolve_clip_prompts_rejects_wrong_length_or_empty_entries(): + with pytest.raises(ValueError, match="exactly svi_num_clips"): + _resolve_clip_prompts("fallback", ["only one"], 2) + with pytest.raises(ValueError, match="non-empty strings"): + _resolve_clip_prompts("fallback", ["first", ""], 2) + + +def test_multiclip_forward_uses_distinct_prompts_seeds_and_last_tail_frame(): + pipeline = object.__new__(WanSVIImageToVideoPipeline) + pipeline.post_init_called = True + pipeline.input_validation_stage = lambda batch, _args: batch + + observed: list[tuple[str | list[str] | None, int | None, tuple[int, int, int]]] = [] + + def fake_stage(batch, _args): + assert isinstance(batch.pil_image, Image.Image) + observed.append((batch.prompt, batch.seed, batch.pil_image.getpixel((0, 0)))) + fill = 0.5 if len(observed) == 1 else 0.75 + batch.output = torch.full((1, 3, 4, 2, 2), fill) + return batch + + pipeline._stages = [fake_stage] + batch = ForwardBatch( + data_type="i2v", + prompt="fallback", + seed=10, + num_frames=4, + num_inference_steps=2, + svi_num_clips=2, + svi_num_motion_frames=1, + svi_clip_prompts=["first", "second"], + pil_image=Image.new("RGB", (2, 2), color=(7, 7, 7)), + ) + + output = pipeline.forward(batch, SimpleNamespace()) + + assert output.output is not None + assert output.output.shape == (1, 3, 7, 2, 2) + assert [(prompt, seed) for prompt, seed, _pixel in observed] == [ + ("first", 10), + ("second", 11), + ] + assert observed[0][2] == (7, 7, 7) + assert observed[1][2] == (128, 128, 128) diff --git a/fastvideo/utils.py b/fastvideo/utils.py index 9282cc66f4..b21f367ac5 100644 --- a/fastvideo/utils.py +++ b/fastvideo/utils.py @@ -600,7 +600,8 @@ def maybe_download_lora(model_name_or_path: str, local_dir: str | None = None, d from huggingface_hub import hf_hub_download repo_id = "/".join(parts[:2]) filename = "/".join(parts[2:]) - return hf_hub_download(repo_id=repo_id, filename=filename, local_dir=local_dir) + with get_lock(model_name_or_path): + return hf_hub_download(repo_id=repo_id, filename=filename, local_dir=local_dir) local_path = maybe_download_model(model_name_or_path, local_dir, download) weight_name = _best_guess_weight_name(model_name_or_path, file_extension=".safetensors") From 083ad8ab0b78a67c77b2edbcdec5be158ae1ec97 Mon Sep 17 00:00:00 2001 From: H1yori233 Date: Tue, 14 Jul 2026 08:43:01 +0000 Subject: [PATCH 09/11] Apply project YAPF formatting --- fastvideo/pipelines/basic/wan/wan_svi_i2v_pipeline.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/fastvideo/pipelines/basic/wan/wan_svi_i2v_pipeline.py b/fastvideo/pipelines/basic/wan/wan_svi_i2v_pipeline.py index 6a0552a4f2..b7945a7264 100644 --- a/fastvideo/pipelines/basic/wan/wan_svi_i2v_pipeline.py +++ b/fastvideo/pipelines/basic/wan/wan_svi_i2v_pipeline.py @@ -60,10 +60,8 @@ def _resolve_clip_prompts( return [prompt] * num_clips if len(clip_prompts) != num_clips: - raise ValueError( - f"svi_clip_prompts must contain exactly svi_num_clips entries " - f"({num_clips}), but got {len(clip_prompts)}" - ) + raise ValueError(f"svi_clip_prompts must contain exactly svi_num_clips entries " + f"({num_clips}), but got {len(clip_prompts)}") if any(not isinstance(item, str) or not item.strip() for item in clip_prompts): raise ValueError("svi_clip_prompts entries must be non-empty strings") return list(clip_prompts) From cb3b9cf3d727b886e71e43f594435e219f4158cb Mon Sep 17 00:00:00 2001 From: H1yori233 Date: Wed, 15 Jul 2026 06:32:49 +0000 Subject: [PATCH 10/11] Align SVI inference with official recipes --- .../inference_schema_parity_inventory.yaml | 1 + examples/inference/basic/basic_svi_i2v.py | 35 ++++--- fastvideo/api/sampling_param.py | 1 + fastvideo/pipelines/basic/wan/presets.py | 5 +- .../basic/wan/wan_svi_i2v_pipeline.py | 43 ++++----- fastvideo/pipelines/pipeline_batch_info.py | 1 + fastvideo/pipelines/stages/image_encoding.py | 65 +++++-------- .../pipelines/test_svi_multiclip_stitch.py | 92 +++++++++++++++---- .../stages/test_svi_image_vae_encoding.py | 3 +- 9 files changed, 137 insertions(+), 109 deletions(-) diff --git a/docs/design/inference_schema_parity_inventory.yaml b/docs/design/inference_schema_parity_inventory.yaml index 947a474e76..03998b8211 100644 --- a/docs/design/inference_schema_parity_inventory.yaml +++ b/docs/design/inference_schema_parity_inventory.yaml @@ -480,6 +480,7 @@ surfaces: svi_ref_pad_cfg: request.extensions.svi.ref_pad_cfg svi_num_clips: request.extensions.svi.num_clips svi_num_motion_frames: request.extensions.svi.num_motion_frames + svi_seed_stride: request.extensions.svi.seed_stride svi_clip_prompts: request.extensions.svi.clip_prompts prompt_attention_mask: request.extensions.hyworld.prompt_attention_mask negative_attention_mask: request.extensions.hyworld.negative_attention_mask diff --git a/examples/inference/basic/basic_svi_i2v.py b/examples/inference/basic/basic_svi_i2v.py index 3741ab9ba7..61aa8beaf8 100644 --- a/examples/inference/basic/basic_svi_i2v.py +++ b/examples/inference/basic/basic_svi_i2v.py @@ -1,10 +1,5 @@ from fastvideo import VideoGenerator -# Available variants: "shot", "film", "tom" -# Each variant uses a different LoRA + motion-frame conditioning: -# - shot: 1 motion frame + reference-frame padding -# - film: 5 motion frames + zero padding (multi-prompt long-story continuity) -# - tom: 1 motion frame + reference-frame padding (cartoon style) MODEL_VARIANT = "shot" VARIANT_CONFIG = { @@ -15,26 +10,26 @@ "prompts": [ ("A sleek white motor yacht speeds across the turquoise blue sea, " "leaving a dramatic wake of white foam behind it under a clear blue sky."), - ("The camera follows the same white yacht from behind as it continues across " - "the turquoise sea, its foamy wake widening under the clear sky."), ], "num_clips": 2, "num_motion_frames": 1, "ref_pad_num": -1, + "height": 448, }, "film": { - "lora_path": "vita-video-gen/svi-model/version-1.0/svi-film.safetensors", + "lora_path": "vita-video-gen/svi-model/version-1.0/svi-film-opt-10212025.safetensors", "image_url": "https://raw.githubusercontent.com/vita-epfl/Stable-Video-Infinity/main/data/toy_test/film/frame.jpg", "prompts": [ ("A Siamese kitten rests snugly inside a straw hat, its head slightly tilted " "as it gazes curiously to the side."), - ("The same Siamese kitten slowly lifts its head from the straw hat and looks " - "toward the camera while the warm room remains unchanged."), + ("The Siamese kitten decides to explore the room and jumps out of the hat " + "onto the soft carpet below."), ], "num_clips": 2, "num_motion_frames": 5, "ref_pad_num": 0, + "height": 480, }, "tom": { "lora_path": "vita-video-gen/svi-model/version-1.0/svi-tom.safetensors", @@ -44,12 +39,14 @@ ("A static shot of the bright 1950s kitchen, turquoise cabinets and a chrome " "sink glinting; Tom cat hovers over the counter, yellow eyes narrowed, while " "Jerry mouse stands defiantly in a tiny milk puddle near a stack of purple plates."), - ("In the same bright 1950s kitchen, Tom leans closer across the counter while Jerry " - "holds his ground beside the purple plates, preserving the cartoon style."), + ("Close-up on Tom cat’s face: a wicked smirk creases his white muzzle; his black " + "brows angle into a sharp V as he crooks one claw toward Jerry mouse like a " + "menacing metronome."), ], "num_clips": 2, "num_motion_frames": 1, - "ref_pad_num": -1, + "ref_pad_num": 0, + "height": 560, }, } @@ -68,6 +65,7 @@ def main(): dit_cpu_offload=False, vae_cpu_offload=False, text_encoder_cpu_offload=True, + flow_shift=5.0, # Set pin_cpu_memory to false if CPU RAM is limited and there're no frequent CPU-GPU transfer pin_cpu_memory=True, ) @@ -77,16 +75,17 @@ def main(): image_path=config["image_url"], output_path=OUTPUT_PATH, save_video=True, - height=448, + height=config["height"], width=832, num_frames=81, - num_inference_steps=20, + fps=24, + num_inference_steps=50, guidance_scale=5.0, - seed=42, - # SVI motion-frame knobs vary per variant; num_clips>1 enables motion-frame chaining. + seed=0, svi_num_clips=config["num_clips"], - svi_clip_prompts=config["prompts"], + svi_clip_prompts=None if MODEL_VARIANT == "shot" else config["prompts"], svi_num_motion_frames=config["num_motion_frames"], + svi_seed_stride=42, svi_ref_pad_num=config["ref_pad_num"], ) diff --git a/fastvideo/api/sampling_param.py b/fastvideo/api/sampling_param.py index 213407ce1c..072de4ea3c 100644 --- a/fastvideo/api/sampling_param.py +++ b/fastvideo/api/sampling_param.py @@ -111,6 +111,7 @@ class SamplingParam: svi_ref_pad_cfg: bool = False # widens y-mask to len(first_frames) instead of {first frame only} svi_num_clips: int = 1 # >1 enables motion-frame chaining svi_num_motion_frames: int = 1 # tail frames carried from clip K to clip K+1 (1=Shot/Tom, 5=Film) + svi_seed_stride: int = 42 svi_clip_prompts: list[str] | None = None # optional one-to-one prompt list for multi-clip generation # LTX-2 multi-modal CFG and STG. diff --git a/fastvideo/pipelines/basic/wan/presets.py b/fastvideo/pipelines/basic/wan/presets.py index a48aacbca7..8675eb9d12 100644 --- a/fastvideo/pipelines/basic/wan/presets.py +++ b/fastvideo/pipelines/basic/wan/presets.py @@ -131,13 +131,14 @@ "height": 448, "width": 832, "num_frames": 81, - "fps": 16, + "fps": 24, "guidance_scale": 5.0, - "num_inference_steps": 20, + "num_inference_steps": 50, "negative_prompt": _NEGATIVE_PROMPT_EN, # SVI-Shot defaults; override per variant in user kwargs. "svi_num_clips": 1, "svi_num_motion_frames": 1, + "svi_seed_stride": 42, "svi_ref_pad_num": -1, "svi_clip_prompts": None, }, diff --git a/fastvideo/pipelines/basic/wan/wan_svi_i2v_pipeline.py b/fastvideo/pipelines/basic/wan/wan_svi_i2v_pipeline.py index b7945a7264..ab11873aa8 100644 --- a/fastvideo/pipelines/basic/wan/wan_svi_i2v_pipeline.py +++ b/fastvideo/pipelines/basic/wan/wan_svi_i2v_pipeline.py @@ -28,7 +28,7 @@ def _tensor_to_pil_list(frames: torch.Tensor) -> list[PIL.Image.Image]: """Convert a video tensor to a list of PIL frames.""" - arr = (frames.detach().to(torch.float32).cpu().clamp(0, 1) * 255.0).round().to(torch.uint8) + arr = (frames.detach().to(torch.float32).cpu().clamp(0, 1) * 255.0).to(torch.uint8) arr = arr.permute(1, 2, 3, 0).numpy() return [PIL.Image.fromarray(a) for a in arr] @@ -41,11 +41,9 @@ def _validate_multiclip_frames(num_motion: int, num_frames: int) -> None: def _stitch_clip_outputs(clip_outputs: list[torch.Tensor], num_motion: int) -> torch.Tensor: - """Concat per-clip (B,C,T,H,W) videos on time, dropping each follow-up's leading num_motion frames.""" - concatenated = [clip_outputs[0]] - for video in clip_outputs[1:]: - concatenated.append(video[:, :, num_motion:, :, :]) - return torch.cat(concatenated, dim=2) + """Stitch clips using the SVI overlap convention.""" + clips = [video[:, :, :-num_motion] for video in clip_outputs[:-1]] + return torch.cat([*clips, clip_outputs[-1]], dim=2) def _resolve_clip_prompts( @@ -67,11 +65,11 @@ def _resolve_clip_prompts( return list(clip_prompts) -def _clip_seed(seed: int | None, clip_idx: int) -> int: +def _clip_seed(seed: int | None, clip_idx: int, seed_stride: int) -> int: """Match the SVI reference by varying diffusion noise per clip.""" if seed is None: raise ValueError("SVI requires a seed") - return int(seed) + clip_idx + return int(seed) + clip_idx * seed_stride class WanSVIImageToVideoPipeline(WanImageToVideoPipeline): @@ -81,10 +79,9 @@ class WanSVIImageToVideoPipeline(WanImageToVideoPipeline): def create_pipeline_stages(self, fastvideo_args: FastVideoArgs): """Set up pipeline stages with proper dependency injection.""" - flow_shift = fastvideo_args.pipeline_config.flow_shift self.modules["scheduler"] = FlowMatchEulerDiscreteScheduler( num_train_timesteps=1000, - shift=5.0 if flow_shift is None else flow_shift, + shift=fastvideo_args.pipeline_config.flow_shift, ) self.add_stage(stage_name="input_validation_stage", stage=InputValidationStage()) @@ -140,34 +137,28 @@ def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> Forward num_clips = max(1, int(batch.svi_num_clips or 1)) num_motion = max(1, int(batch.svi_num_motion_frames or 1)) + seed_stride = int(batch.svi_seed_stride) prompts = _resolve_clip_prompts(batch.prompt, batch.svi_clip_prompts, num_clips) - - if num_clips == 1: - batch.prompt = prompts[0] - for stage in self.stages: - batch = stage(batch, fastvideo_args) - return batch - num_frames = int(batch.num_frames) if batch.num_frames is not None else 0 - _validate_multiclip_frames(num_motion, num_frames) + if num_clips > 1: + _validate_multiclip_frames(num_motion, num_frames) - # Multi-clip needs the reference image up front to construct motion frames - # for clip 0. Run InputValidationStage now to resolve image_path -> pil_image. + # Resolve image_path before constructing the first motion window. self.input_validation_stage(batch, fastvideo_args) assert isinstance(batch.pil_image, PIL.Image.Image) - random_ref = batch.svi_random_ref_frame or batch.pil_image - motion_frames: list[PIL.Image.Image] = (batch.svi_first_frames or [batch.pil_image]) + assert isinstance(batch.height, int) and isinstance(batch.width, int) + reference_frame = batch.pil_image.resize((batch.width, batch.height)) + random_ref = batch.svi_random_ref_frame or reference_frame + motion_frames = batch.svi_first_frames or [reference_frame] clip_outputs: list[torch.Tensor] = [] for clip_idx in range(num_clips): clip_batch = dataclasses.replace( batch, prompt=prompts[clip_idx], - seed=_clip_seed(batch.seed, clip_idx), - pil_image=motion_frames[-1], - # Clear image_path so per-clip InputValidationStage does not - # reload the original ref and clobber motion_frames[0]. + seed=_clip_seed(batch.seed, clip_idx, seed_stride), + pil_image=motion_frames[0], image_path=None, svi_first_frames=motion_frames, svi_random_ref_frame=random_ref, diff --git a/fastvideo/pipelines/pipeline_batch_info.py b/fastvideo/pipelines/pipeline_batch_info.py index af859a1c89..e4b36ea816 100644 --- a/fastvideo/pipelines/pipeline_batch_info.py +++ b/fastvideo/pipelines/pipeline_batch_info.py @@ -162,6 +162,7 @@ class ForwardBatch: svi_ref_pad_cfg: bool = False # widens y-mask to len(first_frames) instead of frame 0 only svi_num_clips: int = 1 # >1 enables motion-frame chaining svi_num_motion_frames: int = 1 # Tail frames carried clip K -> K+1 (1=Shot/Tom, 5=Film) + svi_seed_stride: int = 42 svi_clip_prompts: list[str] | None = None # One prompt per clip; primary prompt is reused when omitted # Latent dimensions diff --git a/fastvideo/pipelines/stages/image_encoding.py b/fastvideo/pipelines/stages/image_encoding.py index 5c1a80cbb6..a4965f2690 100644 --- a/fastvideo/pipelines/stages/image_encoding.py +++ b/fastvideo/pipelines/stages/image_encoding.py @@ -902,7 +902,6 @@ def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> Forward latent_width = width // vae_scale temporal_compression = self.vae.temporal_compression_ratio - # 1) Mask. (1, T, H/8, W/8) -> reshape -> (4, T_lat, H/8, W/8). msk = torch.ones(1, num_frames, latent_height, latent_width, device=device, dtype=torch.float32) if ref_pad_cfg: msk[:, num_condition_frames:] = 0 @@ -915,50 +914,31 @@ def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> Forward msk = msk.view(1, msk.shape[1] // temporal_compression, temporal_compression, latent_height, latent_width) msk = msk.transpose(1, 2)[0] # (4, T_lat, H/8, W/8) - # 2) Condition frames as (1, 3, num_condition_frames, H, W). - condition_frames: list[torch.Tensor] = [] - for frame in first_frames: - t = self.preprocess(frame, vae_scale_factor=vae_scale, height=height, width=width).to(device=device, - dtype=torch.float32) - # preprocess returns (1, 3, H, W); make it (1, 3, 1, H, W) for the temporal cat. - condition_frames.append(t.unsqueeze(2)) - vae_input_condition = torch.cat(condition_frames, dim=2) # (1, 3, num_cond, H, W) - - # 3) Padding frames as (1, 3, remaining_frames, H, W). - if remaining_frames == 0: - vae_input_pad = torch.empty( - vae_input_condition.shape[0], - vae_input_condition.shape[1], - 0, - height, - width, - device=device, - dtype=torch.float32, - ) - elif ref_pad_num == 0: - vae_input_pad = vae_input_condition.new_zeros(vae_input_condition.shape[0], vae_input_condition.shape[1], - remaining_frames, height, width) - elif ref_pad_num == -1: - ref_t = self.preprocess(random_ref_frame, vae_scale_factor=vae_scale, height=height, - width=width).to(device=device, dtype=torch.float32).unsqueeze(2) - vae_input_pad = ref_t.repeat(1, 1, remaining_frames, 1, 1) - elif ref_pad_num > 0: - ref_t = self.preprocess(random_ref_frame, vae_scale_factor=vae_scale, height=height, - width=width).to(device=device, dtype=torch.float32).unsqueeze(2) - ref_pad = ref_t.repeat(1, 1, min(ref_pad_num, remaining_frames), 1, 1) - if remaining_frames > ref_pad_num: - zero_pad = ref_t.new_zeros(ref_t.shape[0], ref_t.shape[1], remaining_frames - ref_pad_num, height, - width) - vae_input_pad = torch.cat([ref_pad, zero_pad], dim=2) - else: - vae_input_pad = ref_pad - else: + condition_frames = [ + self.preprocess(frame, vae_scale_factor=vae_scale, height=height, + width=width).to(device=device, dtype=torch.float32).unsqueeze(2) + for frame in first_frames + ] + vae_input_condition = torch.cat(condition_frames, dim=2) + + if ref_pad_num < -1: raise ValueError(f"Unsupported ref_pad_num={ref_pad_num} (expected -1, 0, or positive int)") + vae_input_pad = vae_input_condition.new_zeros( + vae_input_condition.shape[0], + vae_input_condition.shape[1], + remaining_frames, + height, + width, + ) + if ref_pad_num != 0 and remaining_frames: + pad_count = remaining_frames if ref_pad_num == -1 else min(ref_pad_num, remaining_frames) + ref = self.preprocess(random_ref_frame, vae_scale_factor=vae_scale, height=height, + width=width).to(device=device, dtype=torch.float32).unsqueeze(2) + vae_input_pad[:, :, :pad_count] = ref video_condition = torch.cat([vae_input_condition, vae_input_pad], dim=2) assert video_condition.shape[2] == num_frames, video_condition.shape - # 4) VAE encode (autocast handling mirrors parent stage). vae_dtype = PRECISION_TO_TYPE[fastvideo_args.pipeline_config.vae_precision] vae_autocast_enabled = (vae_dtype != torch.float32) and not fastvideo_args.disable_autocast with torch.autocast(device_type="cuda", dtype=vae_dtype, enabled=vae_autocast_enabled): @@ -970,7 +950,7 @@ def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> Forward if batch.generator is None: raise ValueError("Generator must be provided") - latent = self.retrieve_latents(encoder_output, batch.generator) + latent = self.retrieve_latents(encoder_output, batch.generator, sample_mode="argmax") if hasattr(self.vae, "shift_factor") and self.vae.shift_factor is not None: shift = self.vae.shift_factor @@ -983,9 +963,8 @@ def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> Forward scaling = scaling.to(latent.device, latent.dtype) latent = latent * scaling - # 5) y = concat(mask, latent) along channel dim. latent is (1, 16, T_lat, H_lat, W_lat). mask_batched = msk.unsqueeze(0).to(latent.device, latent.dtype) - batch.image_latent = torch.concat([mask_batched, latent], dim=1) # (1, 20, T_lat, H_lat, W_lat) + batch.image_latent = torch.concat([mask_batched, latent], dim=1) if hasattr(self, "maybe_free_model_hooks"): self.maybe_free_model_hooks() diff --git a/fastvideo/tests/pipelines/test_svi_multiclip_stitch.py b/fastvideo/tests/pipelines/test_svi_multiclip_stitch.py index 75a4ec2e48..9b1dd52f8f 100644 --- a/fastvideo/tests/pipelines/test_svi_multiclip_stitch.py +++ b/fastvideo/tests/pipelines/test_svi_multiclip_stitch.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -"""Tests for SVI multi-clip stitch index math and the frame-budget guard.""" +"""Tests for SVI multi-clip inference.""" from __future__ import annotations from types import SimpleNamespace @@ -8,6 +8,7 @@ import torch from PIL import Image +import fastvideo.pipelines.basic.wan.wan_svi_i2v_pipeline as svi_pipeline from fastvideo.pipelines.basic.wan.wan_svi_i2v_pipeline import ( WanSVIImageToVideoPipeline, _clip_seed, @@ -23,7 +24,7 @@ def _clip(num_frames: int, fill: float) -> torch.Tensor: return torch.full((1, 3, num_frames, 2, 2), fill, dtype=torch.float32) -def test_stitch_drops_motion_overlap_from_followups(): +def test_stitch_drops_motion_tail_from_non_final_clips(): num_motion = 2 clips = [_clip(9, 0.0), _clip(9, 1.0), _clip(9, 2.0)] @@ -32,10 +33,9 @@ def test_stitch_drops_motion_overlap_from_followups(): expected_t = 9 + (9 - num_motion) + (9 - num_motion) assert out.shape == (1, 3, expected_t, 2, 2) - # Provenance: 9 from clip0, then 7 from clip1, then 7 from clip2. - assert torch.all(out[:, :, :9] == 0.0) - assert torch.all(out[:, :, 9:9 + 7] == 1.0) - assert torch.all(out[:, :, 9 + 7:] == 2.0) + assert torch.all(out[:, :, :7] == 0.0) + assert torch.all(out[:, :, 7:7 + 7] == 1.0) + assert torch.all(out[:, :, 7 + 7:] == 2.0) def test_stitch_single_clip_is_identity(): @@ -64,7 +64,21 @@ def test_validate_accepts_motion_lt_frames(): def test_clip_seed_offsets_each_chunk(): - assert [_clip_seed(42, idx) for idx in range(3)] == [42, 43, 44] + assert [_clip_seed(0, idx, 42) for idx in range(3)] == [0, 42, 84] + assert [_clip_seed(7, idx, 0) for idx in range(3)] == [7, 7, 7] + + +def test_svi_scheduler_uses_official_shift(monkeypatch): + monkeypatch.setattr(svi_pipeline, "DenoisingStage", lambda **_kwargs: SimpleNamespace()) + pipeline = object.__new__(WanSVIImageToVideoPipeline) + pipeline.modules = {} + pipeline._stages = [] + pipeline._stage_name_mapping = {} + + args = SimpleNamespace(pipeline_config=SimpleNamespace(flow_shift=5.0)) + pipeline.create_pipeline_stages(args) + + assert pipeline.modules["scheduler"].shift == 5.0 def test_resolve_clip_prompts_reuses_primary_prompt_by_default(): @@ -83,18 +97,57 @@ def test_resolve_clip_prompts_rejects_wrong_length_or_empty_entries(): _resolve_clip_prompts("fallback", ["first", ""], 2) -def test_multiclip_forward_uses_distinct_prompts_seeds_and_last_tail_frame(): +def test_single_clip_uses_resized_svi_reference(): + pipeline = object.__new__(WanSVIImageToVideoPipeline) + pipeline.post_init_called = True + pipeline.input_validation_stage = lambda batch, _args: batch + observed = [] + + def fake_stage(batch, _args): + observed.append((batch.pil_image.size, batch.svi_first_frames[0].size, + batch.svi_random_ref_frame.size)) + batch.output = _clip(4, 0.5) + return batch + + pipeline._stages = [fake_stage] + batch = ForwardBatch( + data_type="i2v", + prompt="prompt", + seed=0, + height=4, + width=6, + num_frames=4, + num_inference_steps=2, + svi_num_clips=1, + pil_image=Image.new("RGB", (2, 2)), + ) + + output = pipeline.forward(batch, SimpleNamespace()) + + assert output.output.shape == (1, 3, 4, 2, 2) + assert observed == [((6, 4), (6, 4), (6, 4))] + + +def test_multiclip_forward_uses_motion_head_and_keeps_padding_reference(): pipeline = object.__new__(WanSVIImageToVideoPipeline) pipeline.post_init_called = True pipeline.input_validation_stage = lambda batch, _args: batch - observed: list[tuple[str | list[str] | None, int | None, tuple[int, int, int]]] = [] + observed = [] def fake_stage(batch, _args): assert isinstance(batch.pil_image, Image.Image) - observed.append((batch.prompt, batch.seed, batch.pil_image.getpixel((0, 0)))) - fill = 0.5 if len(observed) == 1 else 0.75 - batch.output = torch.full((1, 3, 4, 2, 2), fill) + assert batch.svi_first_frames + assert isinstance(batch.svi_random_ref_frame, Image.Image) + observed.append(( + batch.prompt, + batch.seed, + batch.pil_image.getpixel((0, 0)), + batch.svi_first_frames[0].getpixel((0, 0)), + batch.svi_random_ref_frame.getpixel((0, 0)), + )) + values = [0.0, 0.25, 0.5, 0.75] if len(observed) == 1 else [0.1, 0.2, 0.3, 0.4] + batch.output = torch.tensor(values).view(1, 1, 4, 1, 1).expand(1, 3, 4, 2, 2) return batch pipeline._stages = [fake_stage] @@ -102,10 +155,13 @@ def fake_stage(batch, _args): data_type="i2v", prompt="fallback", seed=10, + height=2, + width=2, num_frames=4, num_inference_steps=2, svi_num_clips=2, - svi_num_motion_frames=1, + svi_num_motion_frames=2, + svi_seed_stride=42, svi_clip_prompts=["first", "second"], pil_image=Image.new("RGB", (2, 2), color=(7, 7, 7)), ) @@ -113,10 +169,8 @@ def fake_stage(batch, _args): output = pipeline.forward(batch, SimpleNamespace()) assert output.output is not None - assert output.output.shape == (1, 3, 7, 2, 2) - assert [(prompt, seed) for prompt, seed, _pixel in observed] == [ - ("first", 10), - ("second", 11), + assert output.output.shape == (1, 3, 6, 2, 2) + assert observed == [ + ("first", 10, (7, 7, 7), (7, 7, 7), (7, 7, 7)), + ("second", 52, (127, 127, 127), (127, 127, 127), (7, 7, 7)), ] - assert observed[0][2] == (7, 7, 7) - assert observed[1][2] == (128, 128, 128) diff --git a/fastvideo/tests/stages/test_svi_image_vae_encoding.py b/fastvideo/tests/stages/test_svi_image_vae_encoding.py index 07d52ab070..2fa905bb5f 100644 --- a/fastvideo/tests/stages/test_svi_image_vae_encoding.py +++ b/fastvideo/tests/stages/test_svi_image_vae_encoding.py @@ -90,7 +90,7 @@ def encode(self, x: torch.Tensor): w_lat = x.shape[4] // self.spatial_compression_ratio latent = torch.zeros(x.shape[0], 16, t_lat, h_lat, w_lat, device=x.device, dtype=torch.float32) out = MagicMock() - out.sample = lambda _g: latent + out.sample = lambda _g: latent + 1 out.mode = lambda: latent return out @@ -166,3 +166,4 @@ def test_pre_vae_parity(num_motion: int, ref_pad_num: int, ref_pad_cfg: bool): actual_msk = batch.image_latent[0, :4].cpu() assert actual_msk.shape == expected_msk.shape torch.testing.assert_close(actual_msk, expected_msk, atol=0.0, rtol=0.0) + assert torch.count_nonzero(batch.image_latent[:, 4:]) == 0 From 60b276d39034f192832e82b0b082b8e28c133ba3 Mon Sep 17 00:00:00 2001 From: H1yori233 Date: Wed, 15 Jul 2026 06:45:56 +0000 Subject: [PATCH 11/11] Apply YAPF formatting to SVI stage --- fastvideo/pipelines/stages/image_encoding.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fastvideo/pipelines/stages/image_encoding.py b/fastvideo/pipelines/stages/image_encoding.py index a4965f2690..1431c3e4c3 100644 --- a/fastvideo/pipelines/stages/image_encoding.py +++ b/fastvideo/pipelines/stages/image_encoding.py @@ -916,8 +916,7 @@ def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> Forward condition_frames = [ self.preprocess(frame, vae_scale_factor=vae_scale, height=height, - width=width).to(device=device, dtype=torch.float32).unsqueeze(2) - for frame in first_frames + width=width).to(device=device, dtype=torch.float32).unsqueeze(2) for frame in first_frames ] vae_input_condition = torch.cat(condition_frames, dim=2)