[bugfix] Fix Dreamverse Modal compile warmup latency (#1394)

Author: Davids048

apps/dreamverse/dreamverse/config.py

@@ -164,6 +164,7 @@ def _optional_env(*names: str) -> str | None:
 
 DEVTOOLS_ENABLED = _env_bool("FASTVIDEO_ENABLE_DEVTOOLS", False)
 PROMPT_SAFETY_ENABLED = _env_bool("FASTVIDEO_ENABLE_PROMPT_SAFETY", False)
+DREAMVERSE_MAX_AUTOTUNE = _env_bool("DREAMVERSE_MAX_AUTOTUNE", True)
 
 
 def _resolve_devtools_paths(

apps/dreamverse/dreamverse/video_generation.py

@@ -25,6 +25,7 @@
     MODEL_CONFIG,
     NUM_FRAMES,
     NUM_INFERENCE_STEPS,
+    DREAMVERSE_MAX_AUTOTUNE,
 )
 
 # Multi-frame decoded continuation defaults from
@@ -257,6 +258,7 @@ def initialize(self, model_config: dict | None = None) -> None:
                              or self.current_model_config["model_path"])
 
         enable_compile = os.getenv("ENABLE_TORCH_COMPILE", "1") == "1"
+        compile_mode = "max-autotune-no-cudagraphs" if DREAMVERSE_MAX_AUTOTUNE else None
 
         components = ComponentConfig(
             config_root=config_model_path,
@@ -280,9 +282,10 @@ def initialize(self, model_config: dict | None = None) -> None:
                 compile=CompileConfig(
                     enabled=enable_compile,
                     text_encoder_enabled=enable_compile,
+                    vae_enabled=enable_compile,
                     backend="inductor",
                     fullgraph=True,
-                    mode="max-autotune-no-cudagraphs",
+                    mode=compile_mode,
                     dynamic=False,
                 ),
                 use_fsdp_inference=False,

apps/dreamverse/scripts/modal/README.md

@@ -97,12 +97,26 @@ knobs below live in the image or `modal_app.py` unless you intentionally change
 - `FASTVIDEO_ENABLE_DEVTOOLS`: enable Dreamverse devtools behavior.
 - `FASTVIDEO_PROMPT_PROVIDER` / `FASTVIDEO_PROMPT_*_MODEL`: try prompt-rewriter provider or model choices.
 - `FASTVIDEO_ENABLE_STARTUP_WARMUP`: trade slower startup for a warmer first request.
+- `DREAMVERSE_MAX_AUTOTUNE`: enable or disable PyTorch Inductor max-autotune for the compiled Dreamverse runtime.
+
+The Modal wrapper defaults to torch compile with Inductor max-autotune enabled
+for the fastest generation after startup warmup. If you need shorter
+compile/warmup time, disable max-autotune via:
+
+```bash
+DREAMVERSE_IMAGE=ghcr.io/<org>/<repo>/dreamverse:<tag> \
+DREAMVERSE_MAX_AUTOTUNE=0 \
+  modal deploy apps/dreamverse/scripts/modal/modal_app.py
+```
 
 ### Autoscaling and cost safety
 
-The current deployed script is capped with `max_containers=1`. If uncapped, concurrent requests could
-spawn multiple containers, while `max_containers=1` queued requests onto one
-container instead of multiplying B200 cost.
+The current deployed script keeps exactly one B200 container warm by setting
+`min_containers=1` and `max_containers=1`. `min_containers=1` prevents Modal
+from scaling the deployment down to zero after idle periods, which avoids paying
+the expensive torch-compile/startup-warmup cost again on the next request.
+`max_containers=1` caps concurrency so concurrent requests queue onto the warm
+container instead of spawning additional B200 containers and multiplying cost.
 
 ### Common gotchas
 

apps/dreamverse/scripts/modal/modal_app.py

@@ -16,11 +16,13 @@
 
 image = modal.Image.from_registry(IMAGE)
 image = image.env({
+    "DREAMVERSE_IMAGE": IMAGE,
     "HF_HOME": "/root/.cache/huggingface",
     "FASTVIDEO_DREAMVERSE_HOME": "/var/lib/dreamverse",
-    "FASTVIDEO_ENABLE_STARTUP_WARMUP": "0",
+    "FASTVIDEO_ENABLE_STARTUP_WARMUP": "1",
     "FASTVIDEO_GPU_COUNT": "1",
-    "ENABLE_TORCH_COMPILE": "0",
+    "ENABLE_TORCH_COMPILE": "1",
+    "DREAMVERSE_MAX_AUTOTUNE": os.environ.get("DREAMVERSE_MAX_AUTOTUNE", "1"),
     "STREAM_MODE": "av_fmp4",
 })
 
@@ -37,6 +39,7 @@
     memory=65536,
     timeout=7200,
     startup_timeout=4800,
+    min_containers=1,
     max_containers=1,
     secrets=[modal.Secret.from_name("dreamverse-api-keys")],
     volumes={

fastvideo/models/encoders/gemma.py

@@ -363,7 +363,32 @@ def named_parameters(self, prefix: str = "", recurse: bool = True):
 
     def prepare_for_compile(self) -> None:
         # Load Gemma outside Dynamo so torch.compile does not trace HF file-system checks.
-        _ = self.gemma_model
+        model = self.gemma_model
+        # Run one tiny eager forward before torch.compile. FastVideo calls the text encoder with
+        # Transformers' output_hidden_states path, which wraps each Gemma layer
+        # forward method and restores it by setting an instance-level forward
+        # attribute. If that attribute appears after Dynamo captures guards, the
+        # next text-encoder call recompiles; doing it here makes the first
+        # compiled call see the stable layer state.
+        token_id = getattr(model.config, "eos_token_id", None)
+        if isinstance(token_id, (list, tuple)):
+            token_id = token_id[0] if token_id else None
+        if token_id is None:
+            token_id = getattr(model.config, "pad_token_id", 0)
+        input_ids = torch.full(
+            (1, 1),
+            int(token_id or 0),
+            dtype=torch.long,
+            device=model.device,
+        )
+        attention_mask = torch.ones_like(input_ids)
+        with torch.no_grad():
+            model(
+                input_ids=input_ids,
+                attention_mask=attention_mask,
+                output_hidden_states=True,
+                return_dict=True,
+            )
 
     @property
     def gemma_model(self) -> Gemma3ForConditionalGeneration: