fix: avoid vendoring adot layer dependencies (#628)

Author: zhongkechen

.github/scripts/build_lambda_layer.py

@@ -14,25 +14,19 @@
 from pathlib import Path
 
 
-ARCHITECTURE_PLATFORMS = {
-    "x86_64": "manylinux2014_x86_64",
-    "arm64": "manylinux2014_aarch64",
-}
-SUPPORTED_PYTHON_VERSIONS = ("3.11", "3.12", "3.13", "3.14")
+UNIVERSAL_WHEEL_SUFFIX = "-py3-none-any.whl"
 
 
 @dataclass(frozen=True)
 class BuildConfig:
     output: Path
-    target_python: str
-    architecture: str
     sdk_distribution: Path
     otel_distribution: Path
     build_dir: Path | None = None
 
 
 def build_layer(config: BuildConfig) -> Path:
-    """Build a Lambda layer containing the SDK and OpenTelemetry plugin."""
+    """Build a universal Lambda layer containing the SDK and OTel plugin."""
 
     _validate_config(config)
 
@@ -44,42 +38,28 @@ def build_layer(config: BuildConfig) -> Path:
         layer_python_dir = work_dir / "python"
         layer_python_dir.mkdir(parents=True)
 
-        _install_layer_dependencies(config, layer_python_dir)
+        _install_layer_distributions(config, layer_python_dir)
         _write_zip(config.output, work_dir)
 
     return config.output
 
 
 def _validate_config(config: BuildConfig) -> None:
-    if config.architecture not in ARCHITECTURE_PLATFORMS:
-        supported = ", ".join(sorted(ARCHITECTURE_PLATFORMS))
-        raise ValueError(
-            f"Unsupported architecture: {config.architecture}. "
-            f"Supported architectures: {supported}"
-        )
-
-    if config.target_python not in SUPPORTED_PYTHON_VERSIONS:
-        supported = ", ".join(SUPPORTED_PYTHON_VERSIONS)
-        raise ValueError(
-            f"Unsupported Python version: {config.target_python}. "
-            f"Supported versions: {supported}"
-        )
-
     for distribution in (config.sdk_distribution, config.otel_distribution):
         if not distribution.is_file():
             raise FileNotFoundError(distribution)
+        if not distribution.name.endswith(UNIVERSAL_WHEEL_SUFFIX):
+            raise ValueError(
+                f"Layer distributions must be universal wheels: {distribution}"
+            )
 
 
 def _validate_output_location(output: Path, build_dir: Path) -> None:
     if output.resolve().is_relative_to(build_dir.resolve()):
         raise ValueError("Layer output must be outside the build directory")
 
 
-def _install_layer_dependencies(config: BuildConfig, target_dir: Path) -> None:
-    python_version = config.target_python
-    abi = f"cp{python_version.replace('.', '')}"
-    platform = ARCHITECTURE_PLATFORMS[config.architecture]
-
+def _install_layer_distributions(config: BuildConfig, target_dir: Path) -> None:
     command = [
         sys.executable,
         "-m",
@@ -88,17 +68,10 @@ def _install_layer_dependencies(config: BuildConfig, target_dir: Path) -> None:
         "--upgrade",
         "--target",
         str(target_dir),
-        "--platform",
-        platform,
-        "--implementation",
-        "cp",
-        "--python-version",
-        python_version,
-        "--abi",
-        abi,
         "--only-binary",
         ":all:",
         "--no-compile",
+        "--no-deps",
         str(config.sdk_distribution),
         str(config.otel_distribution),
     ]
@@ -119,21 +92,9 @@ def _write_zip(output: Path, layer_root: Path) -> None:
 def main(argv: list[str] | None = None) -> int:
     parser = argparse.ArgumentParser(
         prog="build_lambda_layer.py",
-        description="Build the AWS Durable Execution SDK OTel plugin Lambda layer.",
+        description="Build the universal AWS Durable Execution SDK OTel plugin layer.",
     )
     parser.add_argument("--output", type=Path, required=True)
-    parser.add_argument(
-        "--target-python",
-        choices=SUPPORTED_PYTHON_VERSIONS,
-        required=True,
-        help="Lambda Python minor version.",
-    )
-    parser.add_argument(
-        "--architecture",
-        choices=sorted(ARCHITECTURE_PLATFORMS),
-        required=True,
-        help="Lambda instruction set architecture.",
-    )
     parser.add_argument("--sdk-distribution", type=Path, required=True)
     parser.add_argument("--otel-distribution", type=Path, required=True)
     parser.add_argument(
@@ -146,8 +107,6 @@ def main(argv: list[str] | None = None) -> int:
     output = build_layer(
         BuildConfig(
             output=args.output,
-            target_python=args.target_python,
-            architecture=args.architecture,
             sdk_distribution=args.sdk_distribution,
             otel_distribution=args.otel_distribution,
             build_dir=args.build_dir,

.github/scripts/tests/test_build_lambda_layer.py

@@ -14,7 +14,45 @@
 from build_lambda_layer import BuildConfig, build_layer
 
 
-def test_build_layer_installs_dependencies_and_zips_lambda_layout(
+def _write_test_wheel(
+    directory: Path,
+    distribution: str,
+    package_files: tuple[str, ...],
+    dependencies: tuple[str, ...] = (),
+) -> Path:
+    normalized_distribution = distribution.replace("-", "_")
+    wheel = directory / f"{normalized_distribution}-1.0.0-py3-none-any.whl"
+    dist_info = f"{normalized_distribution}-1.0.0.dist-info"
+    metadata = [
+        "Metadata-Version: 2.1",
+        f"Name: {distribution}",
+        "Version: 1.0.0",
+        *(f"Requires-Dist: {dependency}==1.0.0" for dependency in dependencies),
+        "",
+    ]
+
+    with zipfile.ZipFile(wheel, "w") as archive:
+        for package_file in package_files:
+            archive.writestr(package_file, "")
+        archive.writestr(f"{dist_info}/METADATA", "\n".join(metadata))
+        archive.writestr(
+            f"{dist_info}/WHEEL",
+            "\n".join(
+                (
+                    "Wheel-Version: 1.0",
+                    "Generator: test",
+                    "Root-Is-Purelib: true",
+                    "Tag: py3-none-any",
+                    "",
+                )
+            ),
+        )
+        archive.writestr(f"{dist_info}/RECORD", "")
+
+    return wheel
+
+
+def test_build_layer_installs_distributions_and_zips_lambda_layout(
     monkeypatch: pytest.MonkeyPatch,
     tmp_path: Path,
 ) -> None:
@@ -42,17 +80,17 @@ def fake_run(command: list[str], check: bool) -> subprocess.CompletedProcess[str
     output = build_layer(
         BuildConfig(
             output=tmp_path / "layer.zip",
-            target_python="3.12",
-            architecture="arm64",
             sdk_distribution=sdk_wheel,
             otel_distribution=otel_wheel,
         )
     )
 
     assert output == tmp_path / "layer.zip"
-    assert commands[0][commands[0].index("--platform") + 1] == "manylinux2014_aarch64"
-    assert commands[0][commands[0].index("--abi") + 1] == "cp312"
     assert "--no-compile" in commands[0]
+    assert "--no-deps" in commands[0]
+    assert "--platform" not in commands[0]
+    assert "--python-version" not in commands[0]
+    assert "--abi" not in commands[0]
     assert str(sdk_wheel) in commands[0]
     assert str(otel_wheel) in commands[0]
 
@@ -66,52 +104,84 @@ def fake_run(command: list[str], check: bool) -> subprocess.CompletedProcess[str
         )
 
 
-@pytest.mark.parametrize(
-    ("target_python", "architecture", "error"),
-    [
-        ("3.10", "x86_64", "Unsupported Python version"),
-        ("3.12", "sparc", "Unsupported architecture"),
-    ],
-)
-def test_build_layer_rejects_unsupported_targets(
-    target_python: str,
-    architecture: str,
-    error: str,
+def test_build_layer_excludes_adot_and_runtime_dependencies(
+    monkeypatch: pytest.MonkeyPatch,
     tmp_path: Path,
 ) -> None:
-    sdk_wheel = tmp_path / "sdk.whl"
-    otel_wheel = tmp_path / "otel.whl"
-    sdk_wheel.write_text("sdk")
-    otel_wheel.write_text("otel")
+    sdk_wheel = _write_test_wheel(
+        tmp_path,
+        "aws-durable-execution-sdk-python",
+        ("aws_durable_execution_sdk_python/__init__.py",),
+        ("boto3",),
+    )
+    otel_wheel = _write_test_wheel(
+        tmp_path,
+        "aws-durable-execution-sdk-python-otel",
+        ("aws_durable_execution_sdk_python_otel/__init__.py",),
+        ("aws-opentelemetry-distro", "opentelemetry-api"),
+    )
+    _write_test_wheel(tmp_path, "boto3", ("boto3/__init__.py",))
+    _write_test_wheel(
+        tmp_path,
+        "aws-opentelemetry-distro",
+        ("amazon/opentelemetry/distro/__init__.py",),
+    )
+    _write_test_wheel(
+        tmp_path,
+        "opentelemetry-api",
+        ("opentelemetry/__init__.py",),
+    )
+    monkeypatch.setenv("PIP_FIND_LINKS", str(tmp_path))
+    monkeypatch.setenv("PIP_NO_INDEX", "1")
 
-    with pytest.raises(ValueError, match=error):
-        build_layer(
-            BuildConfig(
-                output=tmp_path / "layer.zip",
-                target_python=target_python,
-                architecture=architecture,
-                sdk_distribution=sdk_wheel,
-                otel_distribution=otel_wheel,
-            )
+    output = build_layer(
+        BuildConfig(
+            output=tmp_path / "layer.zip",
+            sdk_distribution=sdk_wheel,
+            otel_distribution=otel_wheel,
         )
+    )
+
+    with zipfile.ZipFile(output) as archive:
+        names = archive.namelist()
+
+    assert "python/aws_durable_execution_sdk_python/__init__.py" in names
+    assert "python/aws_durable_execution_sdk_python_otel/__init__.py" in names
+    assert not any(name.startswith("python/amazon/") for name in names)
+    assert not any(name.startswith("python/boto3/") for name in names)
+    assert not any(name.startswith("python/opentelemetry/") for name in names)
 
 
 def test_build_layer_requires_built_distributions(tmp_path: Path) -> None:
     with pytest.raises(FileNotFoundError):
         build_layer(
             BuildConfig(
                 output=tmp_path / "layer.zip",
-                target_python="3.13",
-                architecture="x86_64",
                 sdk_distribution=tmp_path / "missing-sdk.whl",
                 otel_distribution=tmp_path / "missing-otel.whl",
             )
         )
 
 
+def test_build_layer_requires_universal_wheels(tmp_path: Path) -> None:
+    sdk_wheel = tmp_path / "sdk-1.0.0-cp311-cp311-manylinux2014_x86_64.whl"
+    otel_wheel = tmp_path / "otel-1.0.0-py3-none-any.whl"
+    sdk_wheel.write_text("sdk")
+    otel_wheel.write_text("otel")
+
+    with pytest.raises(ValueError, match="must be universal wheels"):
+        build_layer(
+            BuildConfig(
+                output=tmp_path / "layer.zip",
+                sdk_distribution=sdk_wheel,
+                otel_distribution=otel_wheel,
+            )
+        )
+
+
 def test_build_layer_rejects_output_inside_build_directory(tmp_path: Path) -> None:
-    sdk_wheel = tmp_path / "sdk.whl"
-    otel_wheel = tmp_path / "otel.whl"
+    sdk_wheel = tmp_path / "sdk-1.0.0-py3-none-any.whl"
+    otel_wheel = tmp_path / "otel-1.0.0-py3-none-any.whl"
     sdk_wheel.write_text("sdk")
     otel_wheel.write_text("otel")
     build_dir = tmp_path / "layer"
@@ -120,8 +190,6 @@ def test_build_layer_rejects_output_inside_build_directory(tmp_path: Path) -> No
         build_layer(
             BuildConfig(
                 output=build_dir / "layer.zip",
-                target_python="3.13",
-                architecture="x86_64",
                 sdk_distribution=sdk_wheel,
                 otel_distribution=otel_wheel,
                 build_dir=build_dir,