Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docs/design/inference_schema_parity_inventory.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,7 @@ surfaces:
vae_sp: generator.pipeline.preset_overrides.vae_sp
dmd_denoising_steps: generator.pipeline.preset_overrides.dmd_denoising_steps
ti2v_task: generator.pipeline.preset_overrides.ti2v_task
lucy_edit_task: generator.pipeline.preset_overrides.lucy_edit_task
boundary_ratio: generator.pipeline.preset_overrides.boundary_ratio
compatibility_only:
model_path: "Redundant with generator.model_path."
Expand Down
4 changes: 4 additions & 0 deletions docs/inference/support_matrix.md
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ pipeline initialization and sampling.
| FastWan2.1 T2V 1.3B | `FastVideo/FastWan2.1-T2V-1.3B-Diffusers` | 480P | ⭕ | ⭕ | ⭕ | ✅ | ⭕ |
| FastWan2.2 TI2V 5B Full Attn* | `FastVideo/FastWan2.2-TI2V-5B-FullAttn-Diffusers` | 720P | ⭕ | ⭕ | ⭕ | ✅ | ⭕ |
| Wan2.2 TI2V 5B | `Wan-AI/Wan2.2-TI2V-5B-Diffusers` | 720P | ⭕ | ⭕ | ✅ | ⭕ | ⭕ |
| Lucy Edit Dev 5B*** | `decart-ai/Lucy-Edit-Dev` | 480P | ⭕ | ⭕ | ⭕ | ⭕ | ⭕ |
| Wan2.2 T2V A14B | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | 480P<br>720P | ❌ | ❌ | ✅ | ⭕ | ⭕ |
| Wan2.2 I2V A14B | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | 480P<br>720P | ❌ | ❌ | ✅ | ⭕ | ⭕ |
| HunyuanVideo | `hunyuanvideo-community/HunyuanVideo` | 720px1280p<br>544px960p | ❌ | ✅ | ✅ | ⭕ | ⭕ |
Expand All @@ -78,6 +79,9 @@ pipeline initialization and sampling.

**Note**: Wan2.2 TI2V 5B has some quality issues when performing I2V generation. We are working on fixing this issue.

***Lucy Edit Dev uses a non-commercial model license. FastVideo support is
focused on inference integration for video editing workflows.

`Sliding Tile Attn (Legacy Branch)` entries refer to the archived
`sta_do_not_delete` branch workflow, not active `main` inference wiring.

Expand Down
38 changes: 38 additions & 0 deletions examples/inference/basic/basic_lucy_edit.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
from fastvideo import VideoGenerator

OUTPUT_PATH = "video_samples_lucy_edit"


def main():
generator = VideoGenerator.from_pretrained(
"decart-ai/Lucy-Edit-Dev",
num_gpus=1,
use_fsdp_inference=False,
dit_cpu_offload=True,
vae_cpu_offload=False,
text_encoder_cpu_offload=True,
pin_cpu_memory=True,
)

prompt = ("Change the apron and blouse to a classic clown costume: satin "
"polka-dot jumpsuit in bright primary colors, ruffled white collar, "
"oversized pom-pom buttons, white gloves, oversized red shoes, red "
"foam nose; soft window light from left, eye-level medium shot.")
video_path = "https://d2drjpuinn46lb.cloudfront.net/painter_original_edit.mp4"

generator.generate_video(
prompt,
negative_prompt="",
video_path=video_path,
output_path=OUTPUT_PATH,
save_video=True,
height=480,
width=832,
num_frames=81,
fps=24,
guidance_scale=5.0,
)


if __name__ == "__main__":
main()
8 changes: 4 additions & 4 deletions fastvideo/configs/pipelines/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,12 @@
from fastvideo.configs.pipelines.matrixgame3 import MatrixGame3I2V720PConfig
from fastvideo.pipelines.basic.ltx2.pipeline_configs import LTX2T2VConfig
from fastvideo.registry import get_pipeline_config_cls_from_name
from fastvideo.configs.pipelines.wan import (SelfForcingWanT2V480PConfig, WanI2V480PConfig, WanI2V720PConfig,
WanT2V480PConfig, WanT2V720PConfig)
from fastvideo.configs.pipelines.wan import (LucyEditDevConfig, SelfForcingWanT2V480PConfig, WanI2V480PConfig,
WanI2V720PConfig, WanT2V480PConfig, WanT2V720PConfig)

__all__ = [
"HunyuanConfig", "FastHunyuanConfig", "HunyuanGameCraftPipelineConfig", "PipelineConfig", "Hunyuan15T2V480PConfig",
"Hunyuan15T2V720PConfig", "WanT2V480PConfig", "WanI2V480PConfig", "WanT2V720PConfig", "WanI2V720PConfig",
"SelfForcingWanT2V480PConfig", "CosmosConfig", "Cosmos25Config", "LTX2T2VConfig", "HYWorldConfig",
"MatrixGame2I2V480PConfig", "MatrixGame3I2V720PConfig", "get_pipeline_config_cls_from_name"
"SelfForcingWanT2V480PConfig", "LucyEditDevConfig", "CosmosConfig", "Cosmos25Config", "LTX2T2VConfig",
"HYWorldConfig", "MatrixGame2I2V480PConfig", "MatrixGame3I2V720PConfig", "get_pipeline_config_cls_from_name"
]
3 changes: 2 additions & 1 deletion fastvideo/configs/pipelines/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,9 @@ class PipelineConfig:
# DMD parameters
dmd_denoising_steps: list[int] | None = field(default=None)

# Wan2.2 TI2V parameters
# Wan2.2 task modifiers
ti2v_task: bool = False
lucy_edit_task: bool = False
boundary_ratio: float | None = None

# Compilation
Expand Down
138 changes: 138 additions & 0 deletions fastvideo/configs/pipelines/wan.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,11 @@

from fastvideo.configs.models import DiTConfig, EncoderConfig, VAEConfig
from fastvideo.configs.models.dits import WanVideoConfig
from fastvideo.configs.models.dits.wanvideo import WanVideoArchConfig
from fastvideo.configs.models.encoders import (BaseEncoderOutput, CLIPVisionConfig, T5Config,
WAN2_1ControlCLIPVisionConfig)
from fastvideo.configs.models.vaes import WanVAEConfig
from fastvideo.configs.models.vaes.wanvae import WanVAEArchConfig
from fastvideo.configs.pipelines.base import PipelineConfig


Expand Down Expand Up @@ -120,6 +122,142 @@ class Wan2_2_TI2V_5B_Config(WanT2V480PConfig):
expand_timesteps: bool = True

def __post_init__(self) -> None:
assert not (self.ti2v_task and self.lucy_edit_task)
self.vae_config.load_encoder = True
self.vae_config.load_decoder = True
self.dit_config.expand_timesteps = self.expand_timesteps


@dataclass
class LucyEditDevConfig(Wan2_2_TI2V_5B_Config):
"""Configuration for Decart Lucy Edit Dev video editing."""

dit_config: DiTConfig = field(default_factory=lambda: WanVideoConfig(arch_config=WanVideoArchConfig(
num_attention_heads=24,
in_channels=96,
Comment thread
aryan5v marked this conversation as resolved.
out_channels=48,
ffn_dim=14336,
num_layers=30,
)))
vae_config: VAEConfig = field(default_factory=lambda: WanVAEConfig(arch_config=WanVAEArchConfig(
base_dim=160,
decoder_base_dim=256,
z_dim=48,
in_channels=12,
out_channels=12,
scale_factor_spatial=16,
patch_size=2,
is_residual=True,
clip_output=False,
latents_mean=(
-0.2289,
-0.0052,
-0.1323,
-0.2339,
-0.2799,
0.0174,
0.1838,
0.1557,
-0.1382,
0.0542,
0.2813,
0.0891,
0.1570,
-0.0098,
0.0375,
-0.1825,
-0.2246,
-0.1207,
-0.0698,
0.5109,
0.2665,
-0.2108,
-0.2158,
0.2502,
-0.2055,
-0.0322,
0.1109,
0.1567,
-0.0729,
0.0899,
-0.2799,
-0.1230,
-0.0313,
-0.1649,
0.0117,
0.0723,
-0.2839,
-0.2083,
-0.0520,
0.3748,
0.0152,
0.1957,
0.1433,
-0.2944,
0.3573,
-0.0548,
-0.1681,
-0.0667,
),
latents_std=(
0.4765,
1.0364,
0.4514,
1.1677,
0.5313,
0.4990,
0.4818,
0.5013,
0.8158,
1.0344,
0.5894,
1.0901,
0.6885,
0.6165,
0.8454,
0.4978,
0.5759,
0.3523,
0.7135,
0.6804,
0.5833,
1.4146,
0.8986,
0.5659,
0.7069,
0.5338,
0.4889,
0.4917,
0.4069,
0.4999,
0.6866,
0.4093,
0.5709,
0.6065,
0.6415,
0.4944,
0.5726,
1.2042,
0.5458,
1.6887,
0.3971,
1.0600,
0.3943,
0.5537,
0.5444,
0.4089,
0.7468,
0.7744,
),
)))
ti2v_task: bool = False
lucy_edit_task: bool = True

def __post_init__(self) -> None:
assert not (self.ti2v_task and self.lucy_edit_task)
# Lucy uses Wan2.2's enhanced 48-channel VAE latents. Denoising
# concatenates noise + video latents, matching the 96-channel
# transformer input declared above.
self.vae_config.load_encoder = True
self.vae_config.load_decoder = True
self.dit_config.expand_timesteps = self.expand_timesteps
Expand Down
80 changes: 80 additions & 0 deletions fastvideo/pipelines/basic/wan/lucy_edit_pipeline.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
# SPDX-License-Identifier: Apache-2.0
"""Lucy Edit video editing pipeline.

Lucy Edit uses a Wan2.2 5B transformer with an input video latent appended to
the noisy latent channels. The stage topology is therefore closest to Wan V2V,
but the model repo does not include CLIP image-encoder components.
"""

from fastvideo.fastvideo_args import FastVideoArgs
from fastvideo.logger import init_logger
from fastvideo.pipelines.basic.wan.wan_v2v_pipeline import WanVideoToVideoPipeline
from fastvideo.pipelines.stages import (
ConditioningStage,
DecodingStage,
DenoisingStage,
InputValidationStage,
LatentPreparationStage,
TextEncodingStage,
TimestepPreparationStage,
VideoVAEEncodingStage,
)

logger = init_logger(__name__)


class LucyEditPipeline(WanVideoToVideoPipeline):
"""FastVideo pipeline for decart-ai/Lucy-Edit-Dev."""

_required_config_modules = [
"text_encoder",
"tokenizer",
"vae",
"transformer",
"scheduler",
]

def create_pipeline_stages(self, fastvideo_args: FastVideoArgs):
self.add_stage(stage_name="input_validation_stage", stage=InputValidationStage())

self.add_stage(
stage_name="prompt_encoding_stage",
stage=TextEncodingStage(
text_encoders=[self.get_module("text_encoder")],
tokenizers=[self.get_module("tokenizer")],
),
)

self.add_stage(stage_name="conditioning_stage", stage=ConditioningStage())

self.add_stage(
stage_name="timestep_preparation_stage",
stage=TimestepPreparationStage(scheduler=self.get_module("scheduler")),
)

self.add_stage(
stage_name="latent_preparation_stage",
stage=LatentPreparationStage(
scheduler=self.get_module("scheduler"),
transformer=self.get_module("transformer"),
),
)

self.add_stage(
stage_name="video_latent_preparation_stage",
stage=VideoVAEEncodingStage(vae=self.get_module("vae")),
)

self.add_stage(
stage_name="denoising_stage",
stage=DenoisingStage(
transformer=self.get_module("transformer"),
transformer_2=self.get_module("transformer_2"),
scheduler=self.get_module("scheduler"),
),
)

self.add_stage(stage_name="decoding_stage", stage=DecodingStage(vae=self.get_module("vae")))


EntryClass = LucyEditPipeline
19 changes: 19 additions & 0 deletions fastvideo/pipelines/basic/wan/presets.py
Original file line number Diff line number Diff line change
Expand Up @@ -268,6 +268,24 @@
},
)

LUCY_EDIT_DEV = InferencePreset(
name="lucy_edit_dev",
version=1,
model_family="wan",
description="Lucy Edit Dev 5B video editing",
workload_type="t2v",
stage_schemas=(_DENOISE_STAGE, ),
defaults={
"height": 480,
"width": 832,
"num_frames": 81,
"fps": 24,
"guidance_scale": 5.0,
"num_inference_steps": 50,
"negative_prompt": "",
},
)

# -------------------------------------------------------------------
# Self-Forcing (causal) presets
# -------------------------------------------------------------------
Expand Down Expand Up @@ -341,6 +359,7 @@
FAST_WAN_T2V_480P,
WAN_2_2_TI2V_5B,
FAST_WAN_2_2_TI2V_5B,
LUCY_EDIT_DEV,
SF_WAN_T2V_1_3B,
SF_WAN_2_2_T2V_A14B,
SF_WAN_2_2_I2V_A14B,
Expand Down
Loading
Loading