diff --git a/news/+faster-deploy-archives.performance.md b/news/+faster-deploy-archives.performance.md new file mode 100644 index 00000000000..5b849a20ef0 --- /dev/null +++ b/news/+faster-deploy-archives.performance.md @@ -0,0 +1 @@ +Speed up deployment archive creation by reducing filesystem checks for excluded files and avoiding recompression of precompressed frontend assets. diff --git a/reflex/utils/build.py b/reflex/utils/build.py index 3673408051c..45bd2895a71 100644 --- a/reflex/utils/build.py +++ b/reflex/utils/build.py @@ -34,6 +34,41 @@ def set_env_json(): ) +def _zip_compress_type(component_name: constants.ComponentName, file: Path) -> int: + """Select compression suitable for one archive entry. + + Args: + component_name: The archive being created. + file: The source file being archived. + + Returns: + The ZIP compression type for the file. + """ + if component_name == constants.ComponentName.FRONTEND and file.suffix in { + ".gz", + ".br", + ".zst", + }: + return zipfile.ZIP_STORED + return zipfile.ZIP_DEFLATED + + +def _is_excluded_archive_path( + path: Path, excluded_file_ids: set[tuple[int, int]] +) -> bool: + """Check whether an archive path has an excluded file identity. + + Args: + path: The path being considered for the archive. + excluded_file_ids: Device and inode pairs that must be excluded. + + Returns: + Whether the path refers to an excluded file or directory. + """ + stat = path.stat() + return (stat.st_dev, stat.st_ino) in excluded_file_ids + + def _zip( *, component_name: constants.ComponentName, @@ -62,6 +97,12 @@ def _zip( root_directory = Path(root_directory).resolve() directory_names_to_exclude = directory_names_to_exclude or set() files_to_exclude = files_to_exclude or set() + excluded_file_ids = set() + for excluded_file in files_to_exclude: + if excluded_file.exists(): + stat = excluded_file.stat() + excluded_file_ids.add((stat.st_dev, stat.st_ino)) + files_to_zip: list[Path] = [] # Traverse the root directory in a top-down manner. In this traversal order, # we can modify the dirs list in-place to remove directories we don't want to include. @@ -74,10 +115,11 @@ def _zip( subdirectory_name for subdirectory_name in subdirectories_names if subdirectory_name not in directory_names_to_exclude - and not any( - (directory_path / subdirectory_name).samefile(exclude) - for exclude in files_to_exclude - if exclude.exists() + and ( + not excluded_file_ids + or not _is_excluded_archive_path( + directory_path / subdirectory_name, excluded_file_ids + ) ) and not subdirectory_name.startswith(".") and ( @@ -95,10 +137,9 @@ def _zip( files_to_zip += [ directory_path / subfile_name for subfile_name in subfiles_names - if not any( - (directory_path / subfile_name).samefile(excluded_file) - for excluded_file in files_to_exclude - if excluded_file.exists() + if not excluded_file_ids + or not _is_excluded_archive_path( + directory_path / subfile_name, excluded_file_ids ) ] if globs_to_include: @@ -118,7 +159,11 @@ def _zip( for file in files_to_zip: logger.debug(f"{target}: {file}", extra={"progress": progress}) progress.advance(task) - zipf.write(file, Path(file).relative_to(root_directory)) + zipf.write( + file, + file.relative_to(root_directory), + compress_type=_zip_compress_type(component_name, file), + ) def zip_app( diff --git a/tests/units/utils/test_build.py b/tests/units/utils/test_build.py index 750d8890275..9128d931ce5 100644 --- a/tests/units/utils/test_build.py +++ b/tests/units/utils/test_build.py @@ -4,16 +4,99 @@ import gzip import json +import zipfile from pathlib import Path import pytest import reflex_base from pytest_mock import MockerFixture +from reflex_base import constants from reflex.plugins import EmbedPlugin, Plugin from reflex.utils import build, path_ops +@pytest.mark.parametrize("suffix", [".gz", ".br", ".zst"]) +@pytest.mark.parametrize("component_name", list(constants.ComponentName)) +def test_zip_precompressed_sidecars( + tmp_path: Path, suffix: str, component_name: constants.ComponentName +): + """Skip recompression for frontend sidecars and preserve backend compression.""" + static_dir = tmp_path / "static" + static_dir.mkdir() + source = b"export const greeting = 'hello';\n" * 100 + sidecar = gzip.compress(source) + (static_dir / "app.js").write_bytes(source) + (static_dir / f"app.js{suffix}").write_bytes(sidecar) + target = tmp_path / "frontend.zip" + + build._zip( + component_name=component_name, + target=target, + root_directory=static_dir, + exclude_venv_directories=False, + ) + + with zipfile.ZipFile(target) as archive: + assert archive.read("app.js") == source + assert archive.read(f"app.js{suffix}") == sidecar + assert archive.getinfo("app.js").compress_type == zipfile.ZIP_DEFLATED + assert archive.getinfo(f"app.js{suffix}").compress_type == ( + zipfile.ZIP_STORED + if component_name == constants.ComponentName.FRONTEND + else zipfile.ZIP_DEFLATED + ) + + +def test_zip_excludes_files_and_hardlink_aliases(tmp_path: Path): + """Archive exclusions must match file identity, including hard-linked aliases.""" + root = tmp_path / "app" + root.mkdir() + (root / "app.py").write_text("print('app')") + excluded_file = root / "secret.txt" + excluded_file.write_text("secret") + (root / "alias.txt").hardlink_to(excluded_file) + excluded_dir = root / "excluded" + excluded_dir.mkdir() + (excluded_dir / "private.txt").write_text("private") + target = tmp_path / "backend.zip" + + build._zip( + component_name=constants.ComponentName.BACKEND, + target=target, + root_directory=root, + exclude_venv_directories=True, + files_to_exclude={excluded_file, excluded_dir, root / "missing"}, + ) + + with zipfile.ZipFile(target) as archive: + assert archive.namelist() == ["app.py"] + assert archive.read("app.py") == b"print('app')" + + +@pytest.mark.skipif(constants.IS_WINDOWS, reason="Requires directory symlinks") +def test_zip_excludes_directory_symlink_aliases(tmp_path: Path): + """An excluded directory must also be skipped through a symbolic link.""" + root = tmp_path / "app" + root.mkdir() + excluded_dir = root / "excluded" + excluded_dir.mkdir() + (excluded_dir / "private.txt").write_text("private") + (root / "alias").symlink_to(excluded_dir, target_is_directory=True) + target = tmp_path / "backend.zip" + + build._zip( + component_name=constants.ComponentName.BACKEND, + target=target, + root_directory=root, + exclude_venv_directories=True, + files_to_exclude={excluded_dir}, + ) + + with zipfile.ZipFile(target) as archive: + assert archive.namelist() == [] + + def test_compress_static_output_overwrites_stale_sidecars( tmp_path: Path, mocker: MockerFixture ):