[feat]: Stable-Video-Infinity inference on Wan 2.1 I2V 14B 480P

examples/inference/basic/basic_svi_i2v.py

@@ -0,0 +1,79 @@
+from fastvideo import VideoGenerator
+
+# Available variants: "shot", "film", "tom"
+# Each variant uses a different LoRA + motion-frame conditioning:
+#   - shot: 1 motion frame + reference-frame padding (single-prompt demo)
+#   - film: 5 motion frames + zero padding (multi-prompt long-story continuity)
+#   - tom:  1 motion frame + reference-frame padding (cartoon style)
+MODEL_VARIANT = "shot"
+
+VARIANT_CONFIG = {
+    "shot": {
+        "lora_path": "vita-video-gen/svi-model/version-1.0/svi-shot.safetensors",
+        "image_url":
+        "https://raw.githubusercontent.com/vita-epfl/Stable-Video-Infinity/main/data/toy_test/shot/frame.jpg",
+        "prompt": ("A sleek white motor yacht speeds across the turquoise blue sea, "
+                   "leaving a dramatic wake of white foam behind it under a clear blue sky."),
+        "num_motion_frames": 1,
+        "ref_pad_num": -1,
+    },
+    "film": {
+        "lora_path": "vita-video-gen/svi-model/version-1.0/svi-film.safetensors",
+        "image_url":
+        "https://raw.githubusercontent.com/vita-epfl/Stable-Video-Infinity/main/data/toy_test/film/frame.jpg",
+        "prompt": ("A Siamese kitten rests snugly inside a straw hat, its head slightly tilted "
+                   "as it gazes curiously to the side."),
+        "num_motion_frames": 5,
+        "ref_pad_num": 0,
+    },
+    "tom": {
+        "lora_path": "vita-video-gen/svi-model/version-1.0/svi-tom.safetensors",
+        "image_url":
+        "https://raw.githubusercontent.com/vita-epfl/Stable-Video-Infinity/main/data/toy_test/tom/frame.png",
+        "prompt": ("A static shot of the bright 1950s kitchen, turquoise cabinets and a chrome "
+                   "sink glinting; Tom cat hovers over the counter, yellow eyes narrowed, while "
+                   "Jerry mouse stands defiantly in a tiny milk puddle near a stack of purple plates."),
+        "num_motion_frames": 1,
+        "ref_pad_num": -1,
+    },
+}
+
+OUTPUT_PATH = "video_samples_svi"
+
+
+def main():
+    config = VARIANT_CONFIG[MODEL_VARIANT]
+
+    generator = VideoGenerator.from_pretrained(
+        "Wan-AI/Wan2.1-I2V-14B-480P-Diffusers",
+        override_pipeline_cls_name="WanSVIImageToVideoPipeline",
+        lora_path=config["lora_path"],
+        lora_nickname=f"svi-{MODEL_VARIANT}",
+        num_gpus=1,
+        dit_cpu_offload=False,
+        vae_cpu_offload=False,
+        text_encoder_cpu_offload=True,
+        # Set pin_cpu_memory to false if CPU RAM is limited and there're no frequent CPU-GPU transfer
+        pin_cpu_memory=True,
+    )
+
+    generator.generate_video(
+        prompt=config["prompt"],
+        image_path=config["image_url"],
+        output_path=OUTPUT_PATH,
+        save_video=True,
+        height=448,
+        width=832,
+        num_frames=81,
+        num_inference_steps=20,
+        guidance_scale=5.0,
+        seed=42,
+        # SVI motion-frame knobs vary per variant; num_clips>1 enables motion-frame chaining.
+        svi_num_clips=1,
+        svi_num_motion_frames=config["num_motion_frames"],
+        svi_ref_pad_num=config["ref_pad_num"],
+    )
+
+
+if __name__ == "__main__":
+    main()

fastvideo/api/sampling_param.py

@@ -97,6 +97,12 @@ class SamplingParam:
     movement_distance: float | None = None
     camera_rotation: str | None = None
 
+    # Stable-Video-Infinity image conditioning
+    svi_ref_pad_num: int | None = None  # -1 = tile ref; 0 = zero pad; k>0 = ref for first k slots
+    svi_ref_pad_cfg: bool = False  # widens y-mask to len(first_frames) instead of {first frame only}
+    svi_num_clips: int = 1  # >1 enables motion-frame chaining
+    svi_num_motion_frames: int = 1  # tail frames carried from clip K to clip K+1 (1=Shot/Tom, 5=Film)
+
     # LTX-2 multi-modal CFG and STG.
     # cfg_scale defaults are 1.0 (CFG off) so ``ForwardBatch.__post_init__``
     # doesn't force ``do_classifier_free_guidance`` on non-LTX-2 models that

fastvideo/pipelines/basic/wan/presets.py

@@ -120,6 +120,28 @@
     },
 )
 
+WAN_SVI_I2V_14B_480P = InferencePreset(
+    name="wan_svi_i2v_14b_480p",
+    version=1,
+    model_family="wan",
+    description="Stable-Video-Infinity multi-clip I2V on Wan 2.1 14B at 480p",
+    workload_type="i2v",
+    stage_schemas=(_DENOISE_STAGE, ),
+    defaults={
+        "height": 448,
+        "width": 832,
+        "num_frames": 81,
+        "fps": 16,
+        "guidance_scale": 5.0,
+        "num_inference_steps": 20,
+        "negative_prompt": _NEGATIVE_PROMPT_EN,
+        # SVI-Shot defaults; override per variant in user kwargs.
+        "svi_num_clips": 1,
+        "svi_num_motion_frames": 1,
+        "svi_ref_pad_num": -1,
+    },
+)
+
 # -------------------------------------------------------------------
 # Wan 2.2 presets
 # -------------------------------------------------------------------
@@ -334,6 +356,7 @@
     WAN_T2V_14B,
     WAN_I2V_14B_480P,
     WAN_I2V_14B_720P,
+    WAN_SVI_I2V_14B_480P,
     WAN_2_2_T2V_A14B,
     WAN_2_2_I2V_A14B,
     WAN_FUN_1_3B_INP,

fastvideo/pipelines/basic/wan/wan_svi_i2v_pipeline.py

@@ -0,0 +1,165 @@
+# SPDX-License-Identifier: Apache-2.0
+"""
+Wan I2V pipeline variant for Stable-Video-Infinity multi-clip inference.
+
+This module contains an implementation of the SVI-flavored Wan I2V
+pipeline using the modular pipeline architecture.
+"""
+
+import dataclasses
+
+import PIL.Image
+import torch
+
+from fastvideo.fastvideo_args import FastVideoArgs
+from fastvideo.logger import init_logger
+from fastvideo.pipelines.basic.wan.wan_i2v_pipeline import WanImageToVideoPipeline
+from fastvideo.pipelines.pipeline_batch_info import ForwardBatch
+
+# isort: off
+from fastvideo.pipelines.stages import (ConditioningStage, DecodingStage, DenoisingStage, ImageEncodingStage,
+                                        InputValidationStage, LatentPreparationStage, SVIImageVAEEncodingStage,
+                                        TextEncodingStage, TimestepPreparationStage)
+# isort: on
+
+logger = init_logger(__name__)
+
+
+def _tensor_to_pil_list(frames: torch.Tensor) -> list[PIL.Image.Image]:
+    """Convert a video tensor to a list of PIL frames."""
+    arr = (frames.detach().to(torch.float32).cpu().clamp(0, 1) * 255.0).round().to(torch.uint8)
+    arr = arr.permute(1, 2, 3, 0).numpy()
+    return [PIL.Image.fromarray(a) for a in arr]
+
+
+class WanSVIImageToVideoPipeline(WanImageToVideoPipeline):
+    """
+    Pipeline for Stable-Video-Infinity multi-clip I2V generation on Wan 2.1.
+    """
+
+    def create_pipeline_stages(self, fastvideo_args: FastVideoArgs):
+        """Set up pipeline stages with proper dependency injection."""
+        self.add_stage(stage_name="input_validation_stage", stage=InputValidationStage())
+        self.add_stage(
+            stage_name="prompt_encoding_stage",
+            stage=TextEncodingStage(
+                text_encoders=[self.get_module("text_encoder")],
+                tokenizers=[self.get_module("tokenizer")],
+            ),
+        )
+        if (self.get_module("image_encoder") is not None and self.get_module("image_processor") is not None):
+            self.add_stage(
+                stage_name="image_encoding_stage",
+                stage=ImageEncodingStage(
+                    image_encoder=self.get_module("image_encoder"),
+                    image_processor=self.get_module("image_processor"),
+                ),
+            )
+        self.add_stage(stage_name="conditioning_stage", stage=ConditioningStage())
+        self.add_stage(
+            stage_name="timestep_preparation_stage",
+            stage=TimestepPreparationStage(scheduler=self.get_module("scheduler")),
+        )
+        self.add_stage(
+            stage_name="latent_preparation_stage",
+            stage=LatentPreparationStage(
+                scheduler=self.get_module("scheduler"),
+                transformer=self.get_module("transformer"),
+            ),
+        )
+        self.add_stage(
+            stage_name="image_latent_preparation_stage",
+            stage=SVIImageVAEEncodingStage(vae=self.get_module("vae")),
+        )
+        self.add_stage(
+            stage_name="denoising_stage",
+            stage=DenoisingStage(
+                transformer=self.get_module("transformer"),
+                transformer_2=self.get_module("transformer_2"),
+                scheduler=self.get_module("scheduler"),
+            ),
+        )
+        self.add_stage(stage_name="decoding_stage", stage=DecodingStage(vae=self.get_module("vae")))
+
+    @torch.no_grad()
+    def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> ForwardBatch:
+        if not self.post_init_called:
+            self.post_init()
+
+        num_clips = max(1, int(batch.svi_num_clips or 1))
+        num_motion = max(1, int(batch.svi_num_motion_frames or 1))
+
+        if isinstance(batch.prompt, list):
+            prompts: list[str | None] = list(batch.prompt)
+        else:
+            prompts = [batch.prompt]
+        if len(prompts) < num_clips:
+            prompts = prompts + [prompts[-1]] * (num_clips - len(prompts))
+        elif len(prompts) > num_clips:
+            prompts = prompts[:num_clips]
+
+        if num_clips == 1:
+            batch.prompt = prompts[0]
+            for stage in self.stages:
+                batch = stage(batch, fastvideo_args)
+            return batch
+
+        # Multi-clip needs the reference image up front to construct motion frames
+        # for clip 0. Run InputValidationStage now to resolve image_path -> pil_image.
+        self.input_validation_stage(batch, fastvideo_args)
+        assert isinstance(batch.pil_image, PIL.Image.Image)
+        random_ref = batch.svi_random_ref_frame or batch.pil_image
+        # Clip 0: when the caller did not pre-stage motion frames, repeat the ref
+        # image num_motion times. For num_motion=1 (Shot/Tom) this is just [ref];
+        # for num_motion=5 (Film) this matches upstream's --repeat_first_clip path.
+        motion_frames: list[PIL.Image.Image] = (batch.svi_first_frames or [batch.pil_image] * num_motion)
+
+        clip_outputs: list[torch.Tensor] = []
+        for clip_idx in range(num_clips):
+            clip_batch = dataclasses.replace(
+                batch,
+                prompt=prompts[clip_idx],
+                pil_image=motion_frames[0],
+                # Clear image_path so per-clip InputValidationStage does not
+                # reload the original ref and clobber motion_frames[0].
+                image_path=None,
+                svi_first_frames=motion_frames,
+                svi_random_ref_frame=random_ref,
+                prompt_embeds=[],
+                negative_prompt_embeds=None,
+                prompt_attention_mask=None,
+                negative_attention_mask=None,
+                clip_embedding_pos=None,
+                clip_embedding_neg=None,
+                image_embeds=[],
+                preprocessed_image=None,
+                latents=None,
+                image_latent=None,
+                noise_pred=None,
+                output=None,
+                timesteps=None,
+                timestep=None,
+                step_index=None,
+                is_prompt_processed=False,
+            )
+            for stage in self.stages:
+                clip_batch = stage(clip_batch, fastvideo_args)
+
+            assert clip_batch.output is not None
+            clip_outputs.append(clip_batch.output)
+            logger.info("SVI clip %d/%d generated, frames shape=%s", clip_idx + 1, num_clips, clip_batch.output.shape)
+
+            if clip_idx + 1 < num_clips:
+                tail = clip_batch.output[0, :, -num_motion:, :, :]
+                motion_frames = _tensor_to_pil_list(tail)
+
+        # Drop the first num_motion frames of each follow-up clip to avoid duplicating
+        # the previous clip's tail in the stitched output.
+        concatenated = [clip_outputs[0]]
+        for video in clip_outputs[1:]:
+            concatenated.append(video[:, :, num_motion:, :, :])
+        batch.output = torch.cat(concatenated, dim=2)
+        return batch
+
+
+EntryClass = WanSVIImageToVideoPipeline

fastvideo/pipelines/lora_pipeline.py

@@ -317,7 +317,9 @@ def set_lora_adapter(self, lora_nickname: str, lora_path: str | None = None):  #
             for name, weight in lora_state_dict.items():
                 # Extract weights (lora_A, lora_B, and lora_alpha)
                 name = name.replace("diffusion_model.", "")
+                name = name.replace("pipe.dit.", "")
                 name = name.replace(".weight", "")
+                name = name.replace(".default", "")
 
                 if "lora_alpha" in name:
                     # Store alpha with minimal mapping - same processing as lora_A/lora_B

fastvideo/pipelines/pipeline_batch_info.py

@@ -150,6 +150,14 @@ class ForwardBatch:
     movement_distance: float | None = None
     camera_rotation: str | None = None
 
+    # Stable-Video-Infinity image conditioning
+    svi_first_frames: list[PIL.Image.Image] | None = None  # Motion frames forwarded across clips
+    svi_random_ref_frame: PIL.Image.Image | None = None  # Reference frame for ref-padded slots
+    svi_ref_pad_num: int | None = None  # -1=tile ref; 0=zero pad; k>0=ref for first k, zero rest
+    svi_ref_pad_cfg: bool = False  # widens y-mask to len(first_frames) instead of frame 0 only
+    svi_num_clips: int = 1  # >1 enables motion-frame chaining
+    svi_num_motion_frames: int = 1  # Tail frames carried clip K -> K+1 (1=Shot/Tom, 5=Film)
+
     # Latent dimensions
     height_latents: list[int] | int | None = None
     width_latents: list[int] | int | None = None

fastvideo/pipelines/stages/__init__.py

@@ -17,8 +17,8 @@
 from fastvideo.pipelines.stages.encoding import EncodingStage
 from fastvideo.pipelines.stages.image_encoding import (ImageEncodingStage, MatrixGameImageEncodingStage,
                                                        RefImageEncodingStage, ImageVAEEncodingStage,
-                                                       VideoVAEEncodingStage, Hy15ImageEncodingStage,
-                                                       HYWorldImageEncodingStage)
+                                                       SVIImageVAEEncodingStage, VideoVAEEncodingStage,
+                                                       Hy15ImageEncodingStage, HYWorldImageEncodingStage)
 from fastvideo.pipelines.stages.gamecraft_image_encoding import (GameCraftImageVAEEncodingStage)
 from fastvideo.pipelines.stages.input_validation import InputValidationStage
 from fastvideo.pipelines.stages.latent_preparation import (Cosmos25LatentPreparationStage, CosmosLatentPreparationStage,
@@ -87,6 +87,7 @@
     "ImageVAEEncodingStage",
     "VideoVAEEncodingStage",
     "GameCraftImageVAEEncodingStage",
+    "SVIImageVAEEncodingStage",
     "TextEncodingStage",
     "Cosmos25TextEncodingStage",
     # LongCat stages