Skip to content

Commit 9700656

Browse files
Merge upstream/main into report-signal-terminated-workers
Resolves the one conflict, in spark_performance.md. main rewrote the H3 entry in the Running safely list, since lazy_module_load now owns the load split, and added the TAEH3 and two-Spark bullets. Those are taken as is. This branch's earlyoom diagnostics bullet is kept and moved to the end of the list.
2 parents a1b4127 + aaef49b commit 9700656

83 files changed

Lines changed: 5149 additions & 407 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,8 @@ eggs/
5555

5656
# MkDocs documentation
5757
site/
58+
docs/assets/cookbook-serving.json
59+
examples/serving/clients/node_modules/
5860
docs/getting_started/examples/
5961
docs/examples/
6062
docs/inference/examples/

README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
**FastVideo is a unified post-training and real-time inference framework for accelerated video generation.**
1010

1111
## NEWS
12-
- `2026/08/23`: [FastH3 Preview v0.2](https://huggingface.co/FastVideo/FastVideo-Minimax-FastH3-Preview-v0.2) is a 4-step DMD2-distilled MiniMax-H3 checkpoint that generates synchronized video and audio. Run the verified [basic FastH3 example](examples/inference/basic/basic_fasth3.py), see the [inference guide](examples/inference/basic/README.md#fasth3-preview), or have a coding agent install FastVideo with the [agent setup prompt](#install-with-an-ai-coding-agent).
12+
- `2026/08/27`: [FastH3 Preview v1](https://haoailab.com/blogs/fasth3-preview/) is an open-weight 4-step sparse-distilled MiniMax-H3 model for synchronized video-and-audio generation, developed in collaboration with [Nuva Lab](https://nuvalab.ai/) and the [NVIDIA FastGen team](https://github.com/NVlabs/FastGen). Download the recommended [VSA / Data-Free weights](https://huggingface.co/FastVideo/FastVideo-FastH3-4-step-Preview-v1-VSA-DataFree), or see the [full FastH3 collection](https://huggingface.co/collections/FastVideo/fastvideo-fasth3).
1313
- `2026/08/19`: FastVideo now supports MLX on Apple Silicon with [FastMetal-QAD](https://huggingface.co/collections/FastVideo/fastmetal), a family of 1.3B, 5B, and 14B models optimized for Mac—follow the [Apple Silicon guide](https://hao-ai-lab.github.io/FastVideo/getting_started/installation/mps/) and read the [Blog](https://haoailab.com/blogs/fastmetal/).
1414
- `2026/06/23`: Release FastWan-QAD: 5s of Video generated in 1.8s E2E. See the [FastWan-QAD models](https://huggingface.co/FastVideo/FastWan-QAD-FP8-1.3B), [Attn-QAT training guide](https://haoailab.com/FastVideo/training/attn_qat/), and [blog](https://haoailab.com/blogs/fastwan-qad/).
1515
- `2026/03/17`: Release demo: Into the Dreamverse: Vibe Directing in FastVideo, check out the [Blog](https://haoailab.com/blogs/dreamverse/).

docs/assets/cookbook-recipes.json

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
2-
"version": 5,
2+
"version": 6,
33
"recipes": [
44
{
55
"id": "fastwan21-t2v",
@@ -451,6 +451,10 @@
451451
"summary": "Run the DMD2-distilled FastH3 Preview with four DiT forwards, trained H3 sparse attention, compiled decode, and synchronized audio.",
452452
"model": "FastVideo/FastVideo-Minimax-FastH3-Preview-v0.2",
453453
"source": "examples/inference/basic/basic_fasth3.py",
454+
"serving": {
455+
"source": "examples/serving/openai_fasth3.yaml",
456+
"install": "UV_TORCH_BACKEND=cu130 uv pip install -e \".[fasth3]\""
457+
},
454458
"command": "UV_TORCH_BACKEND=cu130 uv pip install -e \".[fasth3]\"\npython examples/inference/basic/basic_fasth3.py --prompt \"(S1) A presenter says <d>[English] FastVideo runs FastH3.</d>\" --profile all",
455459
"gpu_types": ["NVIDIA"],
456460
"hardware": {
@@ -467,6 +471,11 @@
467471
},
468472
{
469473
"id": "fasth3-preview-mlx",
474+
"serving": {
475+
"source": "examples/serving/mlx_fasth3.yaml",
476+
"install": "uv pip install -e \".[mlx]\"",
477+
"prepare": "hf download FastVideo/FastVideo-Minimax-FastH3-Preview-v0.2 --local-dir ./FastH3-Preview-v0.2\npython scripts/checkpoint_conversion/convert_minimax_h3_mlx.py --model-root ./FastH3-Preview-v0.2/transformer --out ./FastH3-MLX --formats \"int6\""
478+
},
470479
"group": "fasth3-preview",
471480
"group_label": "FastH3 Preview",
472481
"group_task": "4-step text to video + audio",
@@ -636,6 +645,29 @@
636645
"evidence": "Source-backed",
637646
"limitations": ["The upstream checkpoint must be converted to Diffusers layout via scripts/checkpoint_conversion/convert_mmaudio_to_diffusers.py unless loaded from the FastVideo converted repo as done here."]
638647
},
648+
{
649+
"id": "fasth3-spark-pair",
650+
"family": "minimax_h3",
651+
"stage": "inference",
652+
"task": "Few-step text to video (with audio)",
653+
"label": "FastH3 on two DGX Sparks (sequence parallel)",
654+
"summary": "Run one FastH3 clip across two GB10s with Ray sequence parallel over QSFP RoCE. Sequential load and lazy module load stay on because SP replicates the DiT on each node.",
655+
"model": "FastVideo/FastVideo-FastH3-4-step-Preview-v1-VSA-DataFree",
656+
"source": "examples/inference/basic/basic_fasth3_spark_pair.yaml",
657+
"command": "source examples/inference/optimizations/spark_pair_env.sh && FASTVIDEO_VSA_SM100A=0 FASTVIDEO_FA4=0 FASTVIDEO_ATTENTION_BACKEND=VIDEO_SPARSE_ATTN_H3 FASTVIDEO_VAE_PARALLEL_DECODE=1 fastvideo generate --config examples/inference/basic/basic_fasth3_spark_pair.yaml",
658+
"gpu_types": ["NVIDIA"],
659+
"hardware": {
660+
"platform": "cuda",
661+
"gpu_count": 2,
662+
"accelerator": "NVIDIA GB10 (DGX Spark pair)",
663+
"evidence": "validated",
664+
"evidence_url": "https://github.com/hao-ai-lab/FastVideo/pull/1803"
665+
},
666+
"evidence": "Verified",
667+
"expected_artifact": "MP4 under outputs/fasth3_spark_pair/",
668+
"modes": ["T2VA", "2-Spark SP"],
669+
"limitations": ["Requires a two-node Ray cluster on the QSFP interconnect. Height, width, frames, and steps in the YAML are examples. Edit them or pass CLI flags. See docs/getting_started/installation/spark_pair.md."]
670+
},
639671
{
640672
"id": "matrix-game-2",
641673
"family": "matrixgame",

docs/assets/cookbook.js

Lines changed: 119 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -220,6 +220,9 @@
220220
const count = root.querySelector("[data-cookbook-count]");
221221
const result = root.querySelector(".cookbook-result");
222222
const commandBlock = root.querySelector(".cookbook-command");
223+
const servingPanel = root.querySelector("[data-cookbook-serving]");
224+
const usage = root.querySelector("[data-cookbook-usage]");
225+
const servingAvailability = root.querySelector("[data-cookbook-serving-availability]");
223226

224227
modelOptions.setAttribute("aria-label", "Recipe");
225228
hardwareOptions.setAttribute("aria-label", "Runtime");
@@ -237,6 +240,18 @@
237240
const familyRecipes = recipes.filter((recipe) => recipe.family === family);
238241
if (!familyRecipes.length) return;
239242

243+
let servingProfiles = {};
244+
let servingLoadFailed = false;
245+
if (servingPanel && familyRecipes.some((recipe) => recipe.serving)) {
246+
try {
247+
const dataUrl = new URL(root.dataset.recipes, document.baseURI);
248+
servingProfiles = await loadRecipes(new URL("cookbook-serving.json", dataUrl));
249+
} catch (error) {
250+
servingLoadFailed = true;
251+
console.error("Failed to load FastVideo serving profiles", error);
252+
}
253+
}
254+
240255
const byId = new Map(familyRecipes.map((recipe) => [recipe.id, recipe]));
241256
const groups = new Map();
242257
familyRecipes.forEach((recipe) => {
@@ -263,13 +278,21 @@
263278

264279
const query = new URLSearchParams(window.location.search);
265280
const requestedRecipe = query.get("recipe");
266-
const defaultRecipeId = familyRecipes[0].id;
281+
const defaultRecipeId = byId.has(root.dataset.defaultRecipe) ? root.dataset.defaultRecipe : familyRecipes[0].id;
267282
let selectedRecipeId = requestedRecipe && byId.has(requestedRecipe) ? requestedRecipe : defaultRecipeId;
268283
let selectedGroupId = groupIdFor(byId.get(selectedRecipeId));
269284
let renderedRuntimeGroup = null;
285+
// Keep previously shared local/openai links working after renaming workflows.
286+
const workflow = (value) => ["local", "python"].includes(value) ? "python" : "server";
287+
let usagePreference = workflow(query.get("use"));
288+
let selectedClient = ["python", "javascript", "curl"].includes(query.get("client")) ? query.get("client") : "curl";
289+
const clientDetails = servingPanel?.querySelector(".cookbook-serving__code");
290+
if (clientDetails && query.has("client")) clientDetails.open = true;
270291

271292
const renderRuntimeOptions = () => {
272293
const groupRecipes = groups.get(selectedGroupId) || [];
294+
const renderedIds = [...hardwareOptions.querySelectorAll("[data-recipe-id]")].map((option) => option.dataset.recipeId);
295+
if (renderedIds.join(",") === groupRecipes.map((recipe) => recipe.id).join(",")) return;
273296
hardwareOptions.replaceChildren();
274297
groupRecipes.forEach((recipe) => {
275298
const runtime = runtimeFor(recipe);
@@ -313,6 +336,55 @@
313336
renderedRuntimeGroup = selectedGroupId;
314337
}
315338
const runtime = runtimeFor(recipe);
339+
const profile = servingPanel && servingProfiles[recipe.id];
340+
const useServer = Boolean(profile && usagePreference === "server");
341+
// The measured local profile and the server config have separate evidence.
342+
const activeRecipe = useServer ? { ...recipe, hardware: profile.hardware, evidence: "Source-backed" } : recipe;
343+
344+
if (usage) {
345+
usage.querySelectorAll("[data-cookbook-mode]").forEach((option) => {
346+
const selected = option.dataset.cookbookMode === (useServer ? "server" : "python");
347+
option.disabled = option.dataset.cookbookMode === "server" && !profile;
348+
option.classList.toggle("cookbook-option--selected", selected);
349+
option.setAttribute("aria-pressed", String(selected));
350+
});
351+
servingAvailability.textContent = profile
352+
? "The playground and API clients share one server process. Both workflows can run on your own machine."
353+
: servingLoadFailed
354+
? "Server examples could not be loaded. Open the H3 server guide below, or use Python directly."
355+
: "This recipe uses Python directly. For the playground and API clients, choose FastH3 Preview with CUDA or MLX.";
356+
servingPanel.hidden = !useServer;
357+
commandBlock.hidden = useServer;
358+
root.querySelector("[data-cookbook-python-note]").hidden = useServer;
359+
}
360+
361+
if (useServer) {
362+
const isMLX = profile.runtime === "mlx";
363+
servingPanel.querySelector("[data-cookbook-server-lifetime]").textContent = isMLX
364+
? "Start once, then change prompts in the playground or your app. MLX reuses its pipeline and prompt cache, but loads and releases model components between phases to limit unified-memory use. It does not keep all weights resident."
365+
: "Start once, then change prompts in the playground or your app. CUDA requests reuse the loaded model. The Python SDK can also reuse a generator within one process.";
366+
servingPanel.querySelector("[data-cookbook-install-guide]").href = isMLX
367+
? "../../getting_started/installation/mps/#run-fasth3-preview" : "../../getting_started/installation/gpu/";
368+
servingPanel.querySelector("[data-cookbook-prepare]").hidden = !profile.prepare;
369+
servingPanel.querySelector("[data-cookbook-server-prepare]").textContent = profile.prepare;
370+
servingPanel.querySelector("[data-cookbook-server-install]").textContent = profile.install;
371+
servingPanel.querySelector("[data-cookbook-server-command]").textContent = profile.command;
372+
servingPanel.querySelector("[data-cookbook-health-command]").textContent = profile.health_command;
373+
servingPanel.querySelector("[data-cookbook-playground]").href = profile.playground_url;
374+
const client = profile.clients[selectedClient];
375+
const filename = client.source.split("/").pop();
376+
servingPanel.querySelector("[data-cookbook-client-install]").textContent = client.install;
377+
const clientCode = servingPanel.querySelector("[data-cookbook-client-code]");
378+
clientCode.className = `language-${selectedClient === "curl" ? "bash" : selectedClient}`;
379+
clientCode.textContent = client.code;
380+
servingPanel.querySelector("[data-cookbook-client-filename]").textContent = filename;
381+
servingPanel.querySelector("[data-cookbook-client-source]").href = `https://github.com/hao-ai-lab/FastVideo/blob/main/${client.source}`;
382+
const runner = { python: "python", javascript: "node", curl: "bash" }[selectedClient];
383+
servingPanel.querySelector("[data-cookbook-client-run]").textContent = `Save as ${filename} and run ${runner} ${filename}. The MP4 is saved with the job ID as its filename.`;
384+
servingPanel.querySelectorAll("[data-cookbook-client]").forEach((option) => {
385+
option.setAttribute("aria-pressed", String(option.dataset.cookbookClient === selectedClient));
386+
});
387+
}
316388

317389
modelOptions.querySelectorAll("button").forEach((option) => {
318390
const selected = option.dataset.recipeGroup === selectedGroupId;
@@ -323,26 +395,40 @@
323395
const selected = option.dataset.recipeId === recipe.id;
324396
option.classList.toggle("cookbook-option--selected", selected);
325397
option.setAttribute("aria-pressed", String(selected));
398+
const candidate = byId.get(option.dataset.recipeId);
399+
const candidateProfile = servingProfiles[candidate.id];
400+
const candidateRuntime = runtimeFor(candidateProfile && usagePreference === "server"
401+
? { ...candidate, hardware: candidateProfile.hardware } : candidate);
402+
option.querySelector("span").textContent = candidateRuntime.hint;
326403
});
327404

328-
description.textContent = recipe.summary;
329-
label.textContent = recipe.label;
405+
description.textContent = useServer
406+
? `FastH3 Preview generates video with audio. This server profile uses the checked-in ${profile.runtime.toUpperCase()} configuration.`
407+
: recipe.summary;
408+
label.textContent = useServer ? `${recipe.group_label || recipe.label} · Server` : recipe.label;
330409
model.textContent = recipe.model;
331410
task.textContent = recipe.task;
332-
hardwareValue.textContent = runtimeSummary(recipe);
333-
if (artifact) artifact.textContent = recipe.expected_artifact || "Not yet documented for this recipe.";
411+
hardwareValue.textContent = runtimeSummary(activeRecipe);
412+
if (artifact) artifact.textContent = useServer ? "MP4 with audio" : recipe.expected_artifact || "Not yet documented for this recipe.";
334413
if (evidenceCell) {
335-
evidenceCell.textContent = recipe.evidence || "Source-backed";
336-
evidenceCell.classList.toggle("cookbook-badge--verified", recipe.evidence === "Verified");
337-
evidenceCell.classList.toggle("cookbook-badge--source-backed", recipe.evidence !== "Verified");
414+
evidenceCell.textContent = activeRecipe.evidence || "Source-backed";
415+
evidenceCell.classList.toggle("cookbook-badge--verified", activeRecipe.evidence === "Verified");
416+
evidenceCell.classList.toggle("cookbook-badge--source-backed", activeRecipe.evidence !== "Verified");
338417
}
339-
source.href = `https://github.com/hao-ai-lab/FastVideo/blob/main/${recipe.source}`;
418+
source.href = `https://github.com/hao-ai-lab/FastVideo/blob/main/${useServer ? profile.source : recipe.source}`;
419+
source.textContent = useServer ? "View server configuration" : "Open example source";
340420
modelLink.href = `https://huggingface.co/${recipe.model}`;
341421
command.textContent = recipe.command;
342422

343-
renderHardwareEvidence(hardwareState, hardwareBadge, recipe);
423+
renderHardwareEvidence(hardwareState, hardwareBadge, activeRecipe);
344424

345-
const limitations = recipe.limitations || [];
425+
const limitations = useServer ? [
426+
profile.runtime === "mlx"
427+
? "This MLX server config has no recorded hardware run. Measurements from the Python recipe are not server memory requirements. Only text-to-video/audio is wired; reference inputs and fast modes are not exposed here."
428+
: "This server config has no recorded serving benchmark. Compilation is disabled, unlike the measured Python performance profile.",
429+
`${profile.sampling.width} × ${profile.sampling.height} · ${profile.sampling.num_frames} frames · ${profile.sampling.fps} fps. The server supplies these defaults; the client sends the model and prompt.`,
430+
"Generation is serialized. Job metadata is held in memory and is lost when the server restarts.",
431+
] : recipe.limitations || [];
346432
notes.replaceChildren();
347433
notes.hidden = limitations.length === 0;
348434
if (limitations.length) {
@@ -361,10 +447,16 @@
361447
nextQuery.set("recipe", recipe.id);
362448
nextQuery.set("runtime", runtime.id);
363449
nextQuery.delete("gpus");
450+
if (usage) {
451+
nextQuery.set("use", useServer ? "server" : "python");
452+
if (useServer && clientDetails?.open) nextQuery.set("client", selectedClient);
453+
else nextQuery.delete("client");
454+
}
364455
const nextUrl = `${window.location.pathname}?${nextQuery.toString()}${window.location.hash}`;
365456
if (historyMode === "push") window.history.pushState({}, "", nextUrl);
366457
else if (historyMode === "replace") window.history.replaceState({}, "", nextUrl);
367-
status.textContent = `${recipe.label} selected for ${runtime.label}.`;
458+
const modeSummary = useServer ? " with a persistent server" : "";
459+
status.textContent = `${recipe.label} selected for ${runtime.label}${modeSummary}.`;
368460
};
369461

370462
modelOptions.addEventListener("click", (event) => {
@@ -380,6 +472,18 @@
380472
selectedGroupId = groupIdFor(byId.get(selectedRecipeId));
381473
render({ historyMode: "push" });
382474
});
475+
usage?.addEventListener("click", (event) => {
476+
const option = event.target.closest("button[data-cookbook-mode]");
477+
if (!option || option.disabled) return;
478+
usagePreference = option.dataset.cookbookMode;
479+
render({ historyMode: "push" });
480+
});
481+
servingPanel?.addEventListener("click", (event) => {
482+
const option = event.target.closest("button[data-cookbook-client]");
483+
if (!option) return;
484+
selectedClient = option.dataset.cookbookClient;
485+
render({ historyMode: "push" });
486+
});
383487

384488
render();
385489

@@ -389,6 +493,9 @@
389493
const nextRecipe = nextQuery.get("recipe");
390494
selectedRecipeId = nextRecipe && byId.has(nextRecipe) ? nextRecipe : defaultRecipeId;
391495
selectedGroupId = groupIdFor(byId.get(selectedRecipeId));
496+
usagePreference = workflow(nextQuery.get("use"));
497+
selectedClient = ["python", "javascript", "curl"].includes(nextQuery.get("client")) ? nextQuery.get("client") : "curl";
498+
if (clientDetails) clientDetails.open = nextQuery.has("client");
392499
render({ historyMode: "none" });
393500
};
394501
bindFamilyPopstate();

0 commit comments

Comments
 (0)