[misc] Address review: CI registration, generation-level contracts, quality trade-off docs

Author: SuhaanCoding

.github/workflows/ci-macos-mlx.yml

@@ -86,6 +86,7 @@ jobs:
             fastvideo/tests/mlx/test_mlx_minimax_h3_vsa.py \
             fastvideo/tests/mlx/test_mlx_minimax_h3_vsa_regressions.py \
             fastvideo/tests/mlx/test_mlx_minimax_h3_fast_mode.py \
+            fastvideo/tests/mlx/test_mlx_minimax_h3_fast_spatial.py \
             fastvideo/tests/mlx/test_mlx_fastwan_benchmark.py \
             fastvideo/tests/mlx/test_taehv_decode.py \
             fastvideo/tests/mlx/test_frame_upsample.py \
@@ -149,6 +150,7 @@ jobs:
             fastvideo/tests/mlx/test_mlx_minimax_h3_vsa.py \
             fastvideo/tests/mlx/test_mlx_minimax_h3_vsa_regressions.py \
             fastvideo/tests/mlx/test_mlx_minimax_h3_fast_mode.py \
+            fastvideo/tests/mlx/test_mlx_minimax_h3_fast_spatial.py \
             fastvideo/tests/mlx/test_mlx_fastwan_benchmark.py \
             fastvideo/tests/mlx/test_taehv_decode.py \
             fastvideo/tests/mlx/test_frame_upsample.py \

docs/getting_started/installation/mps.md

@@ -196,7 +196,10 @@ Add `--fast-spatial` for spatial fast mode, `--fast`'s spatial twin. It
 denoises and decodes on the smallest 32px-aligned canvas covering the
 requested size divided by `--fast-spatial-scale` (a 480x832 request runs on a
 256x416 canvas), then resamples the decoded frames up to the requested size
-in pixel space. It composes with `--fast`:
+in pixel space. It composes with `--fast`. This is a speed/quality trade-off
+and stays off by default: the output carries the reduced canvas's detail
+budget, so it reads softer than a native-resolution render, with the unsharp
+pass countering some but not all of the difference:
 
 ```bash
 python examples/inference/basic/mlx_fasth3.py \

examples/inference/basic/mlx_fasth3.py

@@ -23,6 +23,9 @@
 denoises and decodes on the smallest 32px-aligned canvas covering
 height/width divided by ``--fast-spatial-scale``, then resamples the decoded
 frames up to the requested size in pixel space. The two modes compose.
+This trades fine detail for speed: the output carries the reduced canvas's
+detail budget and reads softer than a native-resolution render, so it stays
+off by default.
 
 This entrypoint currently supports text-to-video-with-audio only. It does not
 yet wire FL2VA, Ref2VA, or two-pass refinement.
@@ -81,7 +84,8 @@ def parse_args() -> argparse.Namespace:
         action=argparse.BooleanOptionalAction,
         default=False,
         help="denoise and decode at height/width // fast-spatial-scale on H3's 32px grid, "
-        "then resample the decoded frames up to the requested size; composes with --fast",
+        "then resample the decoded frames up to the requested size; composes with --fast. "
+        "Trades fine detail for speed",
     )
     parser.add_argument("--fast-spatial-scale", type=int, default=2,
                         help="spatial reduction factor for --fast-spatial (default: 2)")

fastvideo/tests/mlx/test_mlx_minimax_h3_fast_spatial.py

@@ -3,13 +3,22 @@
 
 from __future__ import annotations
 
+from pathlib import Path
+
 import numpy as np
 import pytest
 
 pytest.importorskip("mlx.core", reason="MLX is required for MiniMax H3 fast-spatial tests")
 
+from fastvideo.mlx_runtime import rife_interp  # noqa: E402
 from fastvideo.mlx_runtime.frame_upsample import upsample_frames  # noqa: E402
+from fastvideo.mlx_runtime.minimax_h3 import (  # noqa: E402
+    build_packed_layout,
+    video_latent_num_frames,
+)
 from fastvideo.mlx_runtime.minimax_h3_pipeline import (  # noqa: E402
+    DEFAULT_FAST_SPATIAL_SHARPEN,
+    MiniMaxH3MLXPipeline,
     _center_crop_frames,
     _preflight_media_dependencies,
     plan_fast_spatial,
@@ -71,3 +80,172 @@ def test_spatial_crop_then_upsample_restores_exact_target_size() -> None:
     upsampled = upsample_frames(cropped, width=plan.target_width, height=plan.target_height,
                                 mode="bilinear", sharpen=0.0)
     assert np.stack(upsampled).shape == (2, 480, 832, 3)
+
+
+# -- generation-level orchestration contracts (heavyweight phases mocked) ----
+
+
+def _generate_with_mocked_phases(monkeypatch, tmp_path, **generate_kwargs):
+    """Run real ``generate()`` orchestration with condition/denoise/decode/mux stubbed."""
+    events: list[str] = []
+    calls: dict = {}
+
+    pipeline = MiniMaxH3MLXPipeline.__new__(MiniMaxH3MLXPipeline)
+    pipeline.dit_checkpoint = tmp_path
+
+    monkeypatch.setattr("fastvideo.mlx_runtime.minimax_h3_pipeline._validate_checkpoint_step_ladder",
+                        lambda _checkpoint, _steps: None)
+    monkeypatch.setattr("fastvideo.mlx_runtime.minimax_h3_pipeline._preflight_media_dependencies",
+                        lambda **_kwargs: None)
+    monkeypatch.setattr("fastvideo.mlx_runtime.minimax_h3_pipeline.mlx_h3_checkpoint_vsa_capable",
+                        lambda _checkpoint: False)
+
+    def fake_encode_prompt(_prompt):
+        return np.zeros((8, 8), dtype=np.float32), np.zeros(8, dtype=np.int64)
+
+    def fake_denoise(_text_rows, _token_tags, **kwargs):
+        events.append("denoise")
+        calls["denoise"] = kwargs
+        return np.zeros((4, 4), dtype=np.float32), np.zeros((4, 4), dtype=np.float32)
+
+    def fake_decode_video(_rows, *, height, width, num_frames, tiled):
+        events.append("decode_video")
+        calls["decode_video"] = {"height": height, "width": width, "num_frames": num_frames}
+        return np.zeros((num_frames, height, width, 3), dtype=np.uint8)
+
+    def fake_decode_audio(_rows, *, num_frames):
+        events.append("decode_audio")
+        calls["decode_audio"] = {"num_frames": num_frames}
+        return np.zeros((2, 64), dtype=np.float32)
+
+    pipeline.encode_prompt = fake_encode_prompt
+    pipeline.denoise = fake_denoise
+    pipeline.decode_video = fake_decode_video
+    pipeline.decode_audio = fake_decode_audio
+    pipeline.mux = lambda _frames, _waveform, output_path: Path(output_path)
+
+    def fake_interpolate(frames, target, *, model):
+        events.append("rife")
+        calls["rife"] = {"target": target}
+        return [np.array(frames[0]) for _ in range(target)]
+
+    def fake_load_model(weights_dir=None):
+        return object()
+
+    fake_load_model.cache_clear = lambda: None
+    monkeypatch.setattr(rife_interp, "interpolate_to_frame_count", fake_interpolate)
+    monkeypatch.setattr(rife_interp, "load_model", fake_load_model)
+
+    def fake_sharpen(frames, amount):
+        events.append("sharpen")
+        calls.setdefault("sharpen", []).append(amount)
+        return list(frames)
+
+    def fake_upsample(frames, *, width, height, mode, sharpen):
+        events.append("upsample")
+        calls["upsample"] = {"width": width, "height": height, "mode": mode, "sharpen": sharpen}
+        return [np.zeros((height, width, 3), dtype=np.uint8) for _ in frames]
+
+    monkeypatch.setattr("fastvideo.mlx_runtime.minimax_h3_pipeline._sharpen_frames", fake_sharpen)
+    monkeypatch.setattr("fastvideo.mlx_runtime.minimax_h3_pipeline.upsample_frames", fake_upsample)
+
+    result = pipeline.generate(
+        "(S1) test prompt",
+        output_path=tmp_path / "out.mp4",
+        height=480,
+        width=832,
+        num_frames=124,
+        save_frames=True,
+        **generate_kwargs,
+    )
+    return events, calls, result
+
+
+def test_generate_spatial_only_denoises_reduced_canvas_with_full_audio(
+        monkeypatch: pytest.MonkeyPatch, tmp_path) -> None:
+    events, calls, result = _generate_with_mocked_phases(monkeypatch, tmp_path, fast_spatial=True)
+
+    assert calls["denoise"]["height"] == 256
+    assert calls["denoise"]["width"] == 416
+    assert calls["denoise"]["num_frames"] == 124
+    assert calls["denoise"]["audio_num_frames"] is None
+    assert calls["denoise"]["video_temporal_scale"] == 1.0
+    assert calls["decode_video"] == {"height": 256, "width": 416, "num_frames": 124}
+    assert calls["decode_audio"] == {"num_frames": 124}
+    assert calls["upsample"] == {
+        "width": 832, "height": 480, "mode": "lanczos",
+        "sharpen": pytest.approx(DEFAULT_FAST_SPATIAL_SHARPEN),
+    }
+    assert "rife" not in events
+    assert "sharpen" not in events
+    assert result.frames.shape == (124, 480, 832, 3)
+    assert "spatial_upsample_s" in result.timings
+
+
+def test_generate_temporal_only_control_keeps_full_canvas(monkeypatch: pytest.MonkeyPatch, tmp_path) -> None:
+    events, calls, result = _generate_with_mocked_phases(monkeypatch, tmp_path, fast=True)
+
+    assert calls["denoise"]["height"] == 480
+    assert calls["denoise"]["width"] == 832
+    assert calls["denoise"]["num_frames"] == 73
+    assert calls["denoise"]["audio_num_frames"] == 124
+    assert calls["denoise"]["video_temporal_scale"] > 1.0
+    assert calls["decode_video"] == {"height": 480, "width": 832, "num_frames": 73}
+    assert calls["rife"] == {"target": 124}
+    assert calls["decode_audio"] == {"num_frames": 124}
+    assert calls["sharpen"] == [pytest.approx(0.6)]
+    assert "upsample" not in events
+    assert result.frames.shape == (124, 480, 832, 3)
+
+
+def test_generate_stacked_runs_rife_before_upsample_with_one_sharpen(
+        monkeypatch: pytest.MonkeyPatch, tmp_path) -> None:
+    events, calls, result = _generate_with_mocked_phases(monkeypatch, tmp_path, fast=True, fast_spatial=True)
+
+    assert calls["denoise"]["height"] == 256
+    assert calls["denoise"]["width"] == 416
+    assert calls["denoise"]["num_frames"] == 73
+    assert calls["denoise"]["audio_num_frames"] == 124
+    assert calls["decode_video"] == {"height": 256, "width": 416, "num_frames": 73}
+    assert calls["rife"] == {"target": 124}
+    assert events.index("rife") < events.index("upsample")
+    assert "sharpen" not in events
+    assert calls["upsample"]["sharpen"] == pytest.approx(0.6)
+    assert calls["decode_audio"] == {"num_frames": 124}
+    assert result.frames.shape == (124, 480, 832, 3)
+
+
+# -- reduced VSA layout contracts --------------------------------------------
+
+
+def test_reduced_layout_has_8x13_video_grid_and_unchanged_audio_prefix() -> None:
+    reduced = build_packed_layout(8, 37, 16, 26, 207)
+    full = build_packed_layout(8, 37, 30, 52, 207)
+
+    assert reduced.video_indices.shape[0] == 37 * 8 * 13
+    assert full.video_indices.shape[0] == 37 * 15 * 26
+    np.testing.assert_array_equal(reduced.audio_indices, full.audio_indices)
+    # A/V sync rides on the audio rows' temporal positions (column 0), which
+    # must not move with the canvas. Column 2 is excluded on purpose: audio
+    # rows borrow the video width grid's edge coordinates for their spatial
+    # tag, so it tracks the canvas the same way any native resolution change
+    # does.
+    np.testing.assert_array_equal(
+        reduced.position_ids[reduced.audio_indices, 0],
+        full.position_ids[full.audio_indices, 0],
+    )
+    np.testing.assert_array_equal(reduced.position_ids[reduced.audio_indices, 1],
+                                  np.zeros(reduced.audio_indices.shape[0]))
+
+
+def test_reduced_layout_stacked_uses_22_latent_frames() -> None:
+    assert video_latent_num_frames(73) == 22
+    stacked = build_packed_layout(8, 22, 16, 26, 207, video_temporal_scale=1.7)
+    baseline = build_packed_layout(8, 22, 16, 26, 207)
+
+    assert stacked.video_indices.shape[0] == 22 * 8 * 13
+    np.testing.assert_array_equal(stacked.audio_indices, baseline.audio_indices)
+    np.testing.assert_array_equal(
+        stacked.position_ids[stacked.audio_indices],
+        baseline.position_ids[baseline.audio_indices],
+    )