[bugfix]: condition HunyuanVideo 1.5 i2v on the reference image

Fixes #1692.

Passing image_path to HunyuanVideo15ImageToVideoPipeline raised
StageVerificationError before denoising. Hy15ImageEncodingStage only wrote
image_embeds on the branch where pil_image is None, so with an image the field
kept its ForwardBatch default of [] and the inherited output check rejected it.
Nothing encoded the image either way: the SiglipVisionModel and
SiglipImageProcessor the i2v checkpoints ship were not in the pipeline's
required modules, and the stage was constructed with image_encoder=None.

Declare the SigLIP pair on the two i2v pipeline configs, load it in the i2v
pipeline, and give the stage the path it lacked: SigLIP hidden states into
image_embeds, VAE latents of the reference image into the first frame of the
conditioning stream, and a mask channel set to 1 on that frame.

Channel order is the part that is easy to get wrong, because it is invisible
until an image is supplied. DenoisingStage appends video_latent and a zero pad
separately, which yields [latents 32][mask 1][conditioning 32], but the model
expects [latents 32][conditioning 32][mask 1]. For t2v both trailing blocks are
zero, so the two orders produce the same tensor and the discrepancy never
surfaces. The i2v path therefore populates image_latent as one 33 channel block
and leaves video_latent unset, so denoising takes the branch that appends it
whole. Reference for the expected layout is the diffusers
HunyuanVideo15ImageToVideoPipeline, which builds
cat([latents, cond_latents_concat, mask_concat], dim=1); HYWorldImageEncodingStage
already assembles image_latent the same way.

t2v and the two super-resolution pipelines are untouched. They never carry a
pil_image, so they keep the existing branch, including video_latent, which
SRDenoisingStage reads directly.

Adds CPU tests over both paths: the t2v zeros, the SigLIP embeddings being
non-zero so the transformer's is_t2v check turns off, the conditioning and mask
landing in that order with the mask on frame 0 only, the scaling factor being
applied, video_latent staying unset, and a clear error when a pipeline without
the encoder is handed an image.

fastvideo/tests/stages/test_hy15_image_encoding_stage.py: 7 passed

Not verified here: an end to end i2v run on a GPU. Doing that next and will
post the before and after in the issue.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: KyleNeverGivesUp

fastvideo/configs/pipelines/hunyuan15.py

@@ -8,7 +8,7 @@
 
 from fastvideo.configs.models import DiTConfig, EncoderConfig, VAEConfig
 from fastvideo.configs.models.dits import HunyuanVideo15Config
-from fastvideo.configs.models.encoders import (BaseEncoderOutput, Qwen2_5_VLConfig, T5Config)
+from fastvideo.configs.models.encoders import (BaseEncoderOutput, Qwen2_5_VLConfig, SiglipVisionConfig, T5Config)
 from fastvideo.configs.models.vaes import Hunyuan15VAEConfig
 from fastvideo.configs.models.upsamplers import SRTo720pUpsamplerConfig, SRTo1080pUpsamplerConfig
 from fastvideo.configs.pipelines.base import PipelineConfig, UpsamplerConfig
@@ -124,6 +124,10 @@ def __post_init__(self):
 class Hunyuan15I2V480PStepDistilledConfig(Hunyuan15T2V480PConfig):
     flow_shift: int = 7
 
+    # The i2v checkpoints ship a SigLIP vision tower; declaring it here is what
+    # makes the loader build one.
+    image_encoder_config: EncoderConfig = field(default_factory=SiglipVisionConfig)
+
 
 @dataclass
 class Hunyuan15T2V720PConfig(Hunyuan15T2V480PConfig):
@@ -140,6 +144,8 @@ class Hunyuan15I2V720PConfig(Hunyuan15T2V720PConfig):
     # HunyuanConfig-specific parameters with defaults
     flow_shift: int = 7
 
+    image_encoder_config: EncoderConfig = field(default_factory=SiglipVisionConfig)
+
 
 @dataclass
 class Hunyuan15SR1080PConfig(Hunyuan15T2V720PConfig):

fastvideo/pipelines/basic/hunyuan15/hunyuan15_i2v_pipeline.py

@@ -20,8 +20,12 @@
 
 class HunyuanVideo15ImageToVideoPipeline(ComposedPipelineBase):
 
+    # image_encoder and feature_extractor are the SigLIP pair the i2v
+    # checkpoints ship; without them the reference image has no route into the
+    # transformer and the run is text-to-video wearing an i2v label.
     _required_config_modules = [
-        "text_encoder", "text_encoder_2", "tokenizer", "tokenizer_2", "vae", "transformer", "scheduler"
+        "text_encoder", "text_encoder_2", "tokenizer", "tokenizer_2", "vae", "transformer", "scheduler",
+        "image_encoder", "feature_extractor"
     ]
 
     def create_pipeline_stages(self, fastvideo_args: FastVideoArgs):
@@ -47,7 +51,9 @@ def create_pipeline_stages(self, fastvideo_args: FastVideoArgs):
                                                     transformer=self.get_module("transformer")))
 
         self.add_stage(stage_name="image_encoding_stage",
-                       stage=Hy15ImageEncodingStage(image_encoder=None, image_processor=None))
+                       stage=Hy15ImageEncodingStage(image_encoder=self.get_module("image_encoder"),
+                                                    image_processor=self.get_module("feature_extractor"),
+                                                    vae=self.get_module("vae")))
 
         self.add_stage(stage_name="denoising_stage",
                        stage=DenoisingStage(transformer=self.get_module("transformer"),

fastvideo/pipelines/stages/image_encoding.py

@@ -15,6 +15,7 @@
 from fastvideo.distributed import get_local_torch_device
 from fastvideo.fastvideo_args import ExecutionMode, FastVideoArgs
 from fastvideo.forward_context import set_forward_context
+from fastvideo.image_processor import ImageProcessor
 from fastvideo.logger import init_logger
 from fastvideo.models.vaes.common import ParallelTiledVAE
 from fastvideo.models.vision_utils import (get_default_height_width, normalize, numpy_to_pt, pil_to_numpy, resize,
@@ -97,22 +98,94 @@ def verify_output(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> V
 class Hy15ImageEncodingStage(ImageEncodingStage):
     """
     Stage for encoding image prompts into embeddings for HunyuanVideo1.5 models.
+
+    Without a reference image (T2V and the super-resolution pipelines) the
+    conditioning slots are zero-filled and the transformer takes its T2V branch.
+    With one, SigLIP supplies ``image_embeds`` and the VAE supplies the first
+    frame of the conditioning latent, which is what turns that branch off.
     """
 
+    def __init__(self, image_encoder=None, image_processor=None, vae=None):
+        super().__init__(image_encoder=image_encoder, image_processor=image_processor)
+        self.vae = vae
+
     def verify_input(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> VerificationResult:
         """Verify image encoding stage inputs."""
         return VerificationResult()
 
+    @torch.no_grad()
     def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> ForwardBatch:
         """
         Encode the prompt into image encoder hidden states.
         """
+        device = get_local_torch_device()
+
         if batch.pil_image is None:
-            batch.image_embeds = [torch.zeros(1, 729, 1152, device=get_local_torch_device())]
+            batch.image_embeds = [torch.zeros(1, 729, 1152, device=device)]
+            raw_latent_shape = list(batch.raw_latent_shape)
+            raw_latent_shape[1] = 1
+            batch.video_latent = torch.zeros(tuple(raw_latent_shape), device=device)
+            return batch
+
+        if self.image_encoder is None or self.image_processor is None or self.vae is None:
+            raise ValueError("HunyuanVideo 1.5 needs an image encoder, its processor and the VAE to condition on a "
+                             "reference image, but this pipeline was built without them. Image-to-video requires "
+                             "HunyuanVideo15ImageToVideoPipeline and an i2v checkpoint.")
+
+        _, latent_channels, latent_temporal, latent_height, latent_width = batch.raw_latent_shape
+
+        # 1. SigLIP hidden states. The transformer reads all-zero image
+        # embeddings as "this is T2V", so this is what selects the I2V branch.
+        self.image_encoder = self.image_encoder.to(device)
+        encoder_dtype = next(self.image_encoder.parameters()).dtype
+        image_inputs = self.image_processor.preprocess(
+            images=batch.pil_image,
+            do_resize=True,
+            do_convert_rgb=True,
+            return_tensors="pt",
+        ).to(device=device, dtype=encoder_dtype)
+        with set_forward_context(current_timestep=0, attn_metadata=None):
+            batch.image_embeds = [self.image_encoder(**image_inputs).last_hidden_state]
+        if fastvideo_args.image_encoder_cpu_offload:
+            self.image_encoder.to("cpu")
 
-        raw_latent_shape = list(batch.raw_latent_shape)
-        raw_latent_shape[1] = 1
-        batch.video_latent = torch.zeros(tuple(raw_latent_shape), device=get_local_torch_device())
+        # 2. The reference image occupies the first latent frame of the
+        # conditioning stream; every later frame is zero and the mask channel
+        # marks which one is real.
+        vae_dtype = PRECISION_TO_TYPE[fastvideo_args.pipeline_config.vae_precision]
+        vae_autocast_enabled = (vae_dtype != torch.float32) and not fastvideo_args.disable_autocast
+
+        image_processor = ImageProcessor(vae_scale_factor=8)
+        pixels = image_processor.preprocess(batch.pil_image, batch.height, batch.width)
+        pixels = pixels.unsqueeze(2).to(device=device, dtype=vae_dtype)
+
+        self.vae = self.vae.to(device)
+        with torch.autocast(device_type="cuda", dtype=vae_dtype, enabled=vae_autocast_enabled):
+            cond_latents = self.vae.encode(pixels).mode()
+            cond_latents = cond_latents * self.vae.config.scaling_factor
+        if fastvideo_args.vae_cpu_offload:
+            self.vae.to("cpu")
+
+        cond_latents = cond_latents.repeat(1, 1, latent_temporal, 1, 1)
+        cond_latents[:, :, 1:] = 0.0
+
+        mask = torch.zeros(
+            1,
+            1,
+            latent_temporal,
+            latent_height,
+            latent_width,
+            device=device,
+            dtype=cond_latents.dtype,
+        )
+        mask[:, :, 0] = 1.0
+
+        # Latents, then conditioning, then mask: the denoising stage appends
+        # ``image_latent`` as one block, so the channel order has to be built
+        # here. Leaving ``video_latent`` unset keeps it out of the earlier
+        # branch, which would append the mask before the conditioning.
+        batch.image_latent = torch.cat([cond_latents, mask], dim=1)
+        assert batch.image_latent.shape[1] == latent_channels + 1
         return batch
 
 

fastvideo/tests/stages/test_hy15_image_encoding_stage.py

@@ -0,0 +1,168 @@
+# SPDX-License-Identifier: Apache-2.0
+"""CPU tests for ``Hy15ImageEncodingStage``.
+
+The failure this guards against is quiet: with no reference image conditioning,
+HunyuanVideo 1.5 still trains and still samples, it just ignores the image. So
+these assert the channel layout rather than only that a tensor came back.
+
+Ordering matters and is invisible in T2V. ``DenoisingStage`` appends
+``image_latent`` as one block, so the 65 channel input is
+``[latents 32][conditioning 32][mask 1]``. For T2V the last 33 are all zero and
+either order produces the same tensor, which is why a swapped layout would go
+unnoticed until an image is actually supplied.
+"""
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import PIL.Image
+import pytest
+import torch
+
+from fastvideo.pipelines.stages.image_encoding import Hy15ImageEncodingStage
+
+LATENT_SHAPE = (1, 32, 5, 8, 12)  # B, C, T, H, W
+REFERENCE_IMAGE = PIL.Image.new("RGB", (LATENT_SHAPE[4] * 8, LATENT_SHAPE[3] * 8), (120, 60, 30))
+VISION_TOKENS = 729
+VISION_DIM = 1152
+
+
+class _FakeSiglip(torch.nn.Module):
+    """Returns a constant non-zero hidden state, which is all the stage reads."""
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.marker = torch.nn.Parameter(torch.ones(1))
+
+    def forward(self, pixel_values: torch.Tensor):
+        batch = pixel_values.shape[0]
+        return SimpleNamespace(last_hidden_state=torch.full((batch, VISION_TOKENS, VISION_DIM), 0.5))
+
+
+class _FakeProcessor:
+
+    def preprocess(self, images, **kwargs):
+        return SimpleNamespace(to=lambda **_: {"pixel_values": torch.zeros(1, 3, 384, 384)})
+
+
+class _FakeVAE(torch.nn.Module):
+    """Encodes to a constant so the first frame is distinguishable from zero."""
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.config = SimpleNamespace(scaling_factor=2.0)
+
+    def encode(self, pixels: torch.Tensor):
+        _, _, _, height, width = pixels.shape
+        latent = torch.full((1, LATENT_SHAPE[1], 1, height // 8, width // 8), 3.0)
+        return SimpleNamespace(mode=lambda: latent)
+
+    def to(self, *args, **kwargs):
+        return self
+
+
+def _batch(pil_image):
+    return SimpleNamespace(
+        pil_image=pil_image,
+        raw_latent_shape=LATENT_SHAPE,
+        image_embeds=[],
+        image_latent=None,
+        video_latent=None,
+        height=LATENT_SHAPE[3] * 8,
+        width=LATENT_SHAPE[4] * 8,
+    )
+
+
+def _args():
+    return SimpleNamespace(
+        pipeline_config=SimpleNamespace(vae_precision="fp32"),
+        disable_autocast=True,
+        image_encoder_cpu_offload=False,
+        vae_cpu_offload=False,
+    )
+
+
+@pytest.fixture(autouse=True)
+def _cpu_device(monkeypatch):
+    """Pin the stage to CPU so the fakes and the real code agree on device.
+
+    ``get_local_torch_device`` resolves to MPS on macOS and CUDA elsewhere, and
+    nothing here needs an accelerator.
+    """
+    monkeypatch.setattr(
+        "fastvideo.pipelines.stages.image_encoding.get_local_torch_device",
+        lambda: torch.device("cpu"),
+    )
+
+
+def _i2v_stage():
+    return Hy15ImageEncodingStage(
+        image_encoder=_FakeSiglip(),
+        image_processor=_FakeProcessor(),
+        vae=_FakeVAE(),
+    )
+
+
+class TestTextToVideoPath:
+    """No reference image: the transformer must still see the T2V layout."""
+
+    def test_image_embeds_are_zero(self) -> None:
+        batch = Hy15ImageEncodingStage().forward(_batch(None), _args())
+
+        assert len(batch.image_embeds) == 1
+        assert batch.image_embeds[0].shape == (1, VISION_TOKENS, VISION_DIM)
+        # The transformer keys its T2V branch off this being all zero.
+        assert torch.all(batch.image_embeds[0] == 0)
+
+    def test_conditioning_stays_on_the_video_latent_slot(self) -> None:
+        batch = Hy15ImageEncodingStage().forward(_batch(None), _args())
+
+        # The super-resolution stages read video_latent directly, so T2V keeps
+        # populating it rather than switching to image_latent.
+        assert batch.video_latent is not None
+        assert batch.video_latent.shape == (1, 1) + LATENT_SHAPE[2:]
+        assert torch.all(batch.video_latent == 0)
+        assert batch.image_latent is None
+
+
+class TestImageToVideoPath:
+
+    def test_image_embeds_come_from_siglip(self) -> None:
+        batch = _i2v_stage().forward(_batch(REFERENCE_IMAGE), _args())
+
+        assert batch.image_embeds[0].shape == (1, VISION_TOKENS, VISION_DIM)
+        # Non-zero is the whole point: all-zero would re-select the T2V branch.
+        assert not torch.all(batch.image_embeds[0] == 0)
+
+    def test_channel_layout_is_conditioning_then_mask(self) -> None:
+        batch = _i2v_stage().forward(_batch(REFERENCE_IMAGE), _args())
+
+        assert batch.image_latent.shape == (1, LATENT_SHAPE[1] + 1) + LATENT_SHAPE[2:]
+
+        conditioning = batch.image_latent[:, :LATENT_SHAPE[1]]
+        mask = batch.image_latent[:, LATENT_SHAPE[1]:]
+
+        # A swapped layout would put a 1-wide block first, so check that the
+        # trailing channel is the mask and not part of the conditioning.
+        assert torch.all(mask[:, :, 0] == 1.0)
+        assert torch.all(mask[:, :, 1:] == 0.0)
+        assert torch.all(conditioning[:, :, 0] != 0)
+        assert torch.all(conditioning[:, :, 1:] == 0)
+
+    def test_scaling_factor_is_applied(self) -> None:
+        batch = _i2v_stage().forward(_batch(REFERENCE_IMAGE), _args())
+
+        # Fake VAE emits 3.0 and declares scaling_factor 2.0.
+        assert torch.allclose(batch.image_latent[:, :LATENT_SHAPE[1], 0],
+                              torch.full((1, LATENT_SHAPE[1]) + LATENT_SHAPE[3:], 6.0))
+
+    def test_video_latent_left_unset(self) -> None:
+        batch = _i2v_stage().forward(_batch(REFERENCE_IMAGE), _args())
+
+        # DenoisingStage checks video_latent first and would append the mask
+        # ahead of the conditioning, so this slot has to stay empty.
+        assert batch.video_latent is None
+
+    def test_missing_encoder_is_an_explicit_error(self) -> None:
+        with pytest.raises(ValueError, match="image encoder"):
+            Hy15ImageEncodingStage().forward(_batch(REFERENCE_IMAGE), _args())