[feat]: Stable-Video-Infinity inference on Wan 2.1 I2V 14B 480P

docs/design/inference_schema_parity_inventory.yaml

@@ -492,6 +492,12 @@ surfaces:
       trajectory_type: request.extensions.gen3c.trajectory_type
       movement_distance: request.extensions.gen3c.movement_distance
       camera_rotation: request.extensions.gen3c.camera_rotation
+      svi_ref_pad_num: request.extensions.svi.ref_pad_num
+      svi_ref_pad_cfg: request.extensions.svi.ref_pad_cfg
+      svi_num_clips: request.extensions.svi.num_clips
+      svi_num_motion_frames: request.extensions.svi.num_motion_frames
+      svi_seed_stride: request.extensions.svi.seed_stride
+      svi_clip_prompts: request.extensions.svi.clip_prompts
       prompt_attention_mask: request.extensions.hyworld.prompt_attention_mask
       negative_attention_mask: request.extensions.hyworld.negative_attention_mask
       camera_states: request.extensions.hunyuangamecraft.camera_states

examples/inference/basic/basic_svi_i2v.py

@@ -0,0 +1,94 @@
+from fastvideo import VideoGenerator
+
+MODEL_VARIANT = "shot"
+
+VARIANT_CONFIG = {
+    "shot": {
+        "lora_path": "vita-video-gen/svi-model/version-1.0/svi-shot.safetensors",
+        "image_url":
+        "https://raw.githubusercontent.com/vita-epfl/Stable-Video-Infinity/main/data/toy_test/shot/frame.jpg",
+        "prompts": [
+            ("A sleek white motor yacht speeds across the turquoise blue sea, "
+             "leaving a dramatic wake of white foam behind it under a clear blue sky."),
+        ],
+        "num_clips": 2,
+        "num_motion_frames": 1,
+        "ref_pad_num": -1,
+        "height": 448,
+    },
+    "film": {
+        "lora_path": "vita-video-gen/svi-model/version-1.0/svi-film-opt-10212025.safetensors",
+        "image_url":
+        "https://raw.githubusercontent.com/vita-epfl/Stable-Video-Infinity/main/data/toy_test/film/frame.jpg",
+        "prompts": [
+            ("A Siamese kitten rests snugly inside a straw hat, its head slightly tilted "
+             "as it gazes curiously to the side."),
+            ("The Siamese kitten decides to explore the room and jumps out of the hat "
+             "onto the soft carpet below."),
+        ],
+        "num_clips": 2,
+        "num_motion_frames": 5,
+        "ref_pad_num": 0,
+        "height": 480,
+    },
+    "tom": {
+        "lora_path": "vita-video-gen/svi-model/version-1.0/svi-tom.safetensors",
+        "image_url":
+        "https://raw.githubusercontent.com/vita-epfl/Stable-Video-Infinity/main/data/toy_test/tom/frame.png",
+        "prompts": [
+            ("A static shot of the bright 1950s kitchen, turquoise cabinets and a chrome "
+             "sink glinting; Tom cat hovers over the counter, yellow eyes narrowed, while "
+             "Jerry mouse stands defiantly in a tiny milk puddle near a stack of purple plates."),
+            ("Close-up on Tom cat’s face: a wicked smirk creases his white muzzle; his black "
+             "brows angle into a sharp V as he crooks one claw toward Jerry mouse like a "
+             "menacing metronome."),
+        ],
+        "num_clips": 2,
+        "num_motion_frames": 1,
+        "ref_pad_num": 0,
+        "height": 560,
+    },
+}
+
+OUTPUT_PATH = "video_samples_svi"
+
+
+def main():
+    config = VARIANT_CONFIG[MODEL_VARIANT]
+
+    generator = VideoGenerator.from_pretrained(
+        "Wan-AI/Wan2.1-I2V-14B-480P-Diffusers",
+        override_pipeline_cls_name="WanSVIImageToVideoPipeline",
+        lora_path=config["lora_path"],
+        lora_nickname=f"svi-{MODEL_VARIANT}",
+        num_gpus=1,
+        dit_cpu_offload=False,
+        vae_cpu_offload=False,
+        text_encoder_cpu_offload=True,
+        flow_shift=5.0,
+        # Set pin_cpu_memory to false if CPU RAM is limited and there're no frequent CPU-GPU transfer
+        pin_cpu_memory=True,
+    )
+
+    generator.generate_video(
+        prompt=config["prompts"][0],
+        image_path=config["image_url"],
+        output_path=OUTPUT_PATH,
+        save_video=True,
+        height=config["height"],
+        width=832,
+        num_frames=81,
+        fps=24,
+        num_inference_steps=50,
+        guidance_scale=5.0,
+        seed=0,
+        svi_num_clips=config["num_clips"],
+        svi_clip_prompts=None if MODEL_VARIANT == "shot" else config["prompts"],
+        svi_num_motion_frames=config["num_motion_frames"],
+        svi_seed_stride=42,
+        svi_ref_pad_num=config["ref_pad_num"],
+    )
+
+
+if __name__ == "__main__":
+    main()

fastvideo/api/sampling_param.py

@@ -112,6 +112,14 @@ class SamplingParam:
     movement_distance: float | None = None
     camera_rotation: str | None = None
 
+    # Stable-Video-Infinity image conditioning
+    svi_ref_pad_num: int | None = None  # -1 = tile ref; 0 = zero pad; k>0 = ref for first k slots
+    svi_ref_pad_cfg: bool = False  # widens y-mask to len(first_frames) instead of {first frame only}
+    svi_num_clips: int = 1  # >1 enables motion-frame chaining
+    svi_num_motion_frames: int = 1  # tail frames carried from clip K to clip K+1 (1=Shot/Tom, 5=Film)
+    svi_seed_stride: int = 42
+    svi_clip_prompts: list[str] | None = None  # optional one-to-one prompt list for multi-clip generation
+
     # LTX-2 multi-modal CFG and STG.
     # Class-level defaults match the *distilled* LTX-2 schedule
     # (mirrors ``FastVideo-internal/.../LTX2DistilledSamplingParam``):

fastvideo/pipelines/basic/wan/presets.py

@@ -120,6 +120,30 @@
     },
 )
 
+WAN_SVI_I2V_14B_480P = InferencePreset(
+    name="wan_svi_i2v_14b_480p",
+    version=1,
+    model_family="wan",
+    description="Stable-Video-Infinity multi-clip I2V on Wan 2.1 14B at 480p",
+    workload_type="i2v",
+    stage_schemas=(_DENOISE_STAGE, ),
+    defaults={
+        "height": 448,
+        "width": 832,
+        "num_frames": 81,
+        "fps": 24,
+        "guidance_scale": 5.0,
+        "num_inference_steps": 50,
+        "negative_prompt": _NEGATIVE_PROMPT_EN,
+        # SVI-Shot defaults; override per variant in user kwargs.
+        "svi_num_clips": 1,
+        "svi_num_motion_frames": 1,
+        "svi_seed_stride": 42,
+        "svi_ref_pad_num": -1,
+        "svi_clip_prompts": None,
+    },
+)
+
 # -------------------------------------------------------------------
 # Wan 2.2 presets
 # -------------------------------------------------------------------
@@ -352,6 +376,7 @@
     WAN_T2V_14B,
     WAN_I2V_14B_480P,
     WAN_I2V_14B_720P,
+    WAN_SVI_I2V_14B_480P,
     WAN_2_2_T2V_A14B,
     WAN_2_2_I2V_A14B,
     WAN_FUN_1_3B_INP,

fastvideo/pipelines/basic/wan/wan_svi_i2v_pipeline.py

@@ -0,0 +1,203 @@
+# SPDX-License-Identifier: Apache-2.0
+"""
+Wan I2V pipeline variant for Stable-Video-Infinity multi-clip inference.
+
+This module contains an implementation of the SVI-flavored Wan I2V
+pipeline using the modular pipeline architecture.
+"""
+
+import dataclasses
+
+import PIL.Image
+import torch
+
+from fastvideo.fastvideo_args import FastVideoArgs
+from fastvideo.logger import init_logger
+from fastvideo.models.schedulers.scheduling_flow_match_euler_discrete import FlowMatchEulerDiscreteScheduler
+from fastvideo.pipelines.basic.wan.wan_i2v_pipeline import WanImageToVideoPipeline
+from fastvideo.pipelines.pipeline_batch_info import ForwardBatch
+
+# isort: off
+from fastvideo.pipelines.stages import (ConditioningStage, DecodingStage, DenoisingStage, ImageEncodingStage,
+                                        InputValidationStage, LatentPreparationStage, SVIImageVAEEncodingStage,
+                                        TextEncodingStage, TimestepPreparationStage)
+# isort: on
+
+logger = init_logger(__name__)
+
+
+def _tensor_to_pil_list(frames: torch.Tensor) -> list[PIL.Image.Image]:
+    """Convert a video tensor to a list of PIL frames."""
+    arr = (frames.detach().to(torch.float32).cpu().clamp(0, 1) * 255.0).to(torch.uint8)
+    arr = arr.permute(1, 2, 3, 0).numpy()
+    return [PIL.Image.fromarray(a) for a in arr]
+
+
+def _validate_multiclip_frames(num_motion: int, num_frames: int) -> None:
+    """num_motion must be < num_frames, else follow-up clips stitch to empty."""
+    if num_motion >= num_frames:
+        raise ValueError(f"svi_num_motion_frames ({num_motion}) must be smaller than num_frames ({num_frames}) "
+                         "for multi-clip generation; otherwise stitched follow-up clips would be empty.")
+
+
+def _stitch_clip_outputs(clip_outputs: list[torch.Tensor], num_motion: int) -> torch.Tensor:
+    """Stitch clips using the SVI overlap convention."""
+    clips = [video[:, :, :-num_motion] for video in clip_outputs[:-1]]
+    return torch.cat([*clips, clip_outputs[-1]], dim=2)
+
+
+def _resolve_clip_prompts(
+    prompt: str | list[str] | None,
+    clip_prompts: list[str] | None,
+    num_clips: int,
+) -> list[str]:
+    """Return exactly one non-empty prompt per generated clip."""
+    if clip_prompts is None:
+        if not isinstance(prompt, str) or not prompt.strip():
+            raise ValueError("SVI requires a non-empty primary prompt")
+        return [prompt] * num_clips
+
+    if len(clip_prompts) != num_clips:
+        raise ValueError(f"svi_clip_prompts must contain exactly svi_num_clips entries "
+                         f"({num_clips}), but got {len(clip_prompts)}")
+    if any(not isinstance(item, str) or not item.strip() for item in clip_prompts):
+        raise ValueError("svi_clip_prompts entries must be non-empty strings")
+    return list(clip_prompts)
+
+
+def _clip_seed(seed: int | None, clip_idx: int, seed_stride: int) -> int:
+    """Match the SVI reference by varying diffusion noise per clip."""
+    if seed is None:
+        raise ValueError("SVI requires a seed")
+    return int(seed) + clip_idx * seed_stride
+
+
+class WanSVIImageToVideoPipeline(WanImageToVideoPipeline):
+    """
+    Pipeline for Stable-Video-Infinity multi-clip I2V generation on Wan 2.1.
+    """
+
+    def create_pipeline_stages(self, fastvideo_args: FastVideoArgs):
+        """Set up pipeline stages with proper dependency injection."""
+        self.modules["scheduler"] = FlowMatchEulerDiscreteScheduler(
+            num_train_timesteps=1000,
+            shift=fastvideo_args.pipeline_config.flow_shift,
+        )
+
+        self.add_stage(stage_name="input_validation_stage", stage=InputValidationStage())
+        self.add_stage(
+            stage_name="prompt_encoding_stage",
+            stage=TextEncodingStage(
+                text_encoders=[self.get_module("text_encoder")],
+                tokenizers=[self.get_module("tokenizer")],
+            ),
+        )
+        if (self.get_module("image_encoder") is not None and self.get_module("image_processor") is not None):
+            self.add_stage(
+                stage_name="image_encoding_stage",
+                stage=ImageEncodingStage(
+                    image_encoder=self.get_module("image_encoder"),
+                    image_processor=self.get_module("image_processor"),
+                ),
+            )
+        self.add_stage(stage_name="conditioning_stage", stage=ConditioningStage())
+        self.add_stage(
+            stage_name="timestep_preparation_stage",
+            stage=TimestepPreparationStage(scheduler=self.get_module("scheduler")),
+        )
+        self.add_stage(
+            stage_name="latent_preparation_stage",
+            stage=LatentPreparationStage(
+                scheduler=self.get_module("scheduler"),
+                transformer=self.get_module("transformer"),
+            ),
+        )
+        self.add_stage(
+            stage_name="image_latent_preparation_stage",
+            stage=SVIImageVAEEncodingStage(vae=self.get_module("vae")),
+        )
+        self.add_stage(
+            stage_name="denoising_stage",
+            stage=DenoisingStage(
+                transformer=self.get_module("transformer"),
+                transformer_2=self.get_module("transformer_2"),
+                scheduler=self.get_module("scheduler"),
+            ),
+        )
+        self.add_stage(stage_name="decoding_stage", stage=DecodingStage(vae=self.get_module("vae")))
+
+    @torch.no_grad()
+    def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> ForwardBatch:
+        if not self.post_init_called:
+            self.post_init()
+
+        n_steps = int(batch.num_inference_steps)
+        if batch.sigmas is None:
+            batch.sigmas = torch.linspace(1.0, 0.0, n_steps + 1)[:-1].tolist()
+
+        num_clips = max(1, int(batch.svi_num_clips or 1))
+        num_motion = max(1, int(batch.svi_num_motion_frames or 1))
+        seed_stride = int(batch.svi_seed_stride)
+
+        prompts = _resolve_clip_prompts(batch.prompt, batch.svi_clip_prompts, num_clips)
+        num_frames = int(batch.num_frames) if batch.num_frames is not None else 0
+        if num_clips > 1:
+            _validate_multiclip_frames(num_motion, num_frames)
+
+        # Resolve image_path before constructing the first motion window.
+        self.input_validation_stage(batch, fastvideo_args)
+        assert isinstance(batch.pil_image, PIL.Image.Image)
+        assert isinstance(batch.height, int) and isinstance(batch.width, int)
+        reference_frame = batch.pil_image.resize((batch.width, batch.height))
+        random_ref = batch.svi_random_ref_frame or reference_frame
+        motion_frames = batch.svi_first_frames or [reference_frame]
+
+        clip_outputs: list[torch.Tensor] = []
+        for clip_idx in range(num_clips):
+            clip_batch = dataclasses.replace(
+                batch,
+                prompt=prompts[clip_idx],
+                seed=_clip_seed(batch.seed, clip_idx, seed_stride),
+                pil_image=motion_frames[0],
+                image_path=None,
+                svi_first_frames=motion_frames,
+                svi_random_ref_frame=random_ref,
+                prompt_embeds=[],
+                negative_prompt_embeds=None,
+                prompt_attention_mask=None,
+                negative_attention_mask=None,
+                clip_embedding_pos=None,
+                clip_embedding_neg=None,
+                image_embeds=[],
+                preprocessed_image=None,
+                latents=None,
+                image_latent=None,
+                noise_pred=None,
+                output=None,
+                timesteps=None,
+                timestep=None,
+                step_index=None,
+                is_prompt_processed=False,
+            )
+            for stage in self.stages:
+                clip_batch = stage(clip_batch, fastvideo_args)
+
+            assert clip_batch.output is not None
+            clip_outputs.append(clip_batch.output)
+            logger.info(
+                "SVI clip %d/%d generated with seed=%d, frames shape=%s",
+                clip_idx + 1,
+                num_clips,
+                clip_batch.seed,
+                clip_batch.output.shape,
+            )
+
+            if clip_idx + 1 < num_clips:
+                tail = clip_batch.output[0, :, -num_motion:, :, :]
+                motion_frames = _tensor_to_pil_list(tail)
+
+        batch.output = _stitch_clip_outputs(clip_outputs, num_motion)
+        return batch
+
+
+EntryClass = WanSVIImageToVideoPipeline

fastvideo/pipelines/lora_pipeline.py

@@ -2,6 +2,7 @@
 from collections import defaultdict
 from collections.abc import Hashable
 from contextlib import nullcontext
+import re
 from typing import Any
 from collections.abc import Generator
 
@@ -29,6 +30,14 @@
 logger = init_logger(__name__)
 
 
+def _normalize_lora_key(name: str) -> str:
+    """Normalize known DiffSynth/PEFT wrappers without rewriting module names."""
+    name = name.removeprefix("diffusion_model.")
+    name = name.removeprefix("pipe.dit.")
+    name = re.sub(r"(\.lora_[AB])\.default(?=\.weight$)", r"\1", name)
+    return name.removesuffix(".weight")
+
+
 def _get_hook_ctx(module: nn.Module | None):
     if module is None:
         return nullcontext()
@@ -324,8 +333,7 @@ def set_lora_adapter(self,
             to_merge_params: defaultdict[Hashable, dict[Any, Any]] = (defaultdict(dict))
             for name, weight in lora_state_dict.items():
                 # Extract weights (lora_A, lora_B, and lora_alpha)
-                name = name.replace("diffusion_model.", "")
-                name = name.replace(".weight", "")
+                name = _normalize_lora_key(name)
 
                 if "lora_alpha" in name:
                     # Store alpha with minimal mapping - same processing as lora_A/lora_B