Speed up deploy archive creation

news/+faster-deploy-archives.performance.md

@@ -0,0 +1 @@
+Speed up deployment archive creation by reducing filesystem checks for excluded files and avoiding recompression of precompressed frontend assets.

reflex/utils/build.py

@@ -34,6 +34,41 @@ def set_env_json():
     )
 
 
+def _zip_compress_type(component_name: constants.ComponentName, file: Path) -> int:
+    """Select compression suitable for one archive entry.
+
+    Args:
+        component_name: The archive being created.
+        file: The source file being archived.
+
+    Returns:
+        The ZIP compression type for the file.
+    """
+    if component_name == constants.ComponentName.FRONTEND and file.suffix in {
+        ".gz",
+        ".br",
+        ".zst",
+    }:
+        return zipfile.ZIP_STORED
+    return zipfile.ZIP_DEFLATED
+
+
+def _is_excluded_archive_path(
+    path: Path, excluded_file_ids: set[tuple[int, int]]
+) -> bool:
+    """Check whether an archive path has an excluded file identity.
+
+    Args:
+        path: The path being considered for the archive.
+        excluded_file_ids: Device and inode pairs that must be excluded.
+
+    Returns:
+        Whether the path refers to an excluded file or directory.
+    """
+    stat = path.stat()
+    return (stat.st_dev, stat.st_ino) in excluded_file_ids
+
+
 def _zip(
     *,
     component_name: constants.ComponentName,
@@ -62,6 +97,12 @@ def _zip(
     root_directory = Path(root_directory).resolve()
     directory_names_to_exclude = directory_names_to_exclude or set()
     files_to_exclude = files_to_exclude or set()
+    excluded_file_ids = set()
+    for excluded_file in files_to_exclude:
+        if excluded_file.exists():
+            stat = excluded_file.stat()
+            excluded_file_ids.add((stat.st_dev, stat.st_ino))
+
     files_to_zip: list[Path] = []
     # Traverse the root directory in a top-down manner. In this traversal order,
     # we can modify the dirs list in-place to remove directories we don't want to include.
@@ -74,10 +115,11 @@ def _zip(
             subdirectory_name
             for subdirectory_name in subdirectories_names
             if subdirectory_name not in directory_names_to_exclude
-            and not any(
-                (directory_path / subdirectory_name).samefile(exclude)
-                for exclude in files_to_exclude
-                if exclude.exists()
+            and (
+                not excluded_file_ids
+                or not _is_excluded_archive_path(
+                    directory_path / subdirectory_name, excluded_file_ids
+                )
             )
             and not subdirectory_name.startswith(".")
             and (
@@ -95,10 +137,9 @@ def _zip(
         files_to_zip += [
             directory_path / subfile_name
             for subfile_name in subfiles_names
-            if not any(
-                (directory_path / subfile_name).samefile(excluded_file)
-                for excluded_file in files_to_exclude
-                if excluded_file.exists()
+            if not excluded_file_ids
+            or not _is_excluded_archive_path(
+                directory_path / subfile_name, excluded_file_ids
             )
         ]
     if globs_to_include:
@@ -118,7 +159,11 @@ def _zip(
         for file in files_to_zip:
             logger.debug(f"{target}: {file}", extra={"progress": progress})
             progress.advance(task)
-            zipf.write(file, Path(file).relative_to(root_directory))
+            zipf.write(
+                file,
+                file.relative_to(root_directory),
+                compress_type=_zip_compress_type(component_name, file),
+            )
 
 
 def zip_app(

tests/units/utils/test_build.py

@@ -4,16 +4,99 @@
 
 import gzip
 import json
+import zipfile
 from pathlib import Path
 
 import pytest
 import reflex_base
 from pytest_mock import MockerFixture
+from reflex_base import constants
 
 from reflex.plugins import EmbedPlugin, Plugin
 from reflex.utils import build, path_ops
 
 
+@pytest.mark.parametrize("suffix", [".gz", ".br", ".zst"])
+@pytest.mark.parametrize("component_name", list(constants.ComponentName))
+def test_zip_precompressed_sidecars(
+    tmp_path: Path, suffix: str, component_name: constants.ComponentName
+):
+    """Skip recompression for frontend sidecars and preserve backend compression."""
+    static_dir = tmp_path / "static"
+    static_dir.mkdir()
+    source = b"export const greeting = 'hello';\n" * 100
+    sidecar = gzip.compress(source)
+    (static_dir / "app.js").write_bytes(source)
+    (static_dir / f"app.js{suffix}").write_bytes(sidecar)
+    target = tmp_path / "frontend.zip"
+
+    build._zip(
+        component_name=component_name,
+        target=target,
+        root_directory=static_dir,
+        exclude_venv_directories=False,
+    )
+
+    with zipfile.ZipFile(target) as archive:
+        assert archive.read("app.js") == source
+        assert archive.read(f"app.js{suffix}") == sidecar
+        assert archive.getinfo("app.js").compress_type == zipfile.ZIP_DEFLATED
+        assert archive.getinfo(f"app.js{suffix}").compress_type == (
+            zipfile.ZIP_STORED
+            if component_name == constants.ComponentName.FRONTEND
+            else zipfile.ZIP_DEFLATED
+        )
+
+
+def test_zip_excludes_files_and_hardlink_aliases(tmp_path: Path):
+    """Archive exclusions must match file identity, including hard-linked aliases."""
+    root = tmp_path / "app"
+    root.mkdir()
+    (root / "app.py").write_text("print('app')")
+    excluded_file = root / "secret.txt"
+    excluded_file.write_text("secret")
+    (root / "alias.txt").hardlink_to(excluded_file)
+    excluded_dir = root / "excluded"
+    excluded_dir.mkdir()
+    (excluded_dir / "private.txt").write_text("private")
+    target = tmp_path / "backend.zip"
+
+    build._zip(
+        component_name=constants.ComponentName.BACKEND,
+        target=target,
+        root_directory=root,
+        exclude_venv_directories=True,
+        files_to_exclude={excluded_file, excluded_dir, root / "missing"},
+    )
+
+    with zipfile.ZipFile(target) as archive:
+        assert archive.namelist() == ["app.py"]
+        assert archive.read("app.py") == b"print('app')"
+
+
+@pytest.mark.skipif(constants.IS_WINDOWS, reason="Requires directory symlinks")
+def test_zip_excludes_directory_symlink_aliases(tmp_path: Path):
+    """An excluded directory must also be skipped through a symbolic link."""
+    root = tmp_path / "app"
+    root.mkdir()
+    excluded_dir = root / "excluded"
+    excluded_dir.mkdir()
+    (excluded_dir / "private.txt").write_text("private")
+    (root / "alias").symlink_to(excluded_dir, target_is_directory=True)
+    target = tmp_path / "backend.zip"
+
+    build._zip(
+        component_name=constants.ComponentName.BACKEND,
+        target=target,
+        root_directory=root,
+        exclude_venv_directories=True,
+        files_to_exclude={excluded_dir},
+    )
+
+    with zipfile.ZipFile(target) as archive:
+        assert archive.namelist() == []
+
+
 def test_compress_static_output_overwrites_stale_sidecars(
     tmp_path: Path, mocker: MockerFixture
 ):