chore: support adaptor wheel overrides on Windows and Linux

src/deadline/cinema4d_submitter/adaptor_override_environment.yaml

@@ -16,69 +16,20 @@ environment:
   script:
     actions:
       onEnter:
-        command: '{{Env.File.Enter}}'
+        # The Cinema 4D Conda environment provides "python" on Windows and Linux.
+        command: python
+        args:
+        - '{{Env.File.SetupAdaptor}}'
+        - '{{Session.WorkingDirectory}}'
+        - '{{Param.OverrideAdaptorWheels}}'
+        - '{{Param.OverrideAdaptorName}}'
+        cancelation:
+          mode: NOTIFY_THEN_TERMINATE
     embeddedFiles:
-    - name: Enter
-      filename: override-adaptor-enter.sh
+    - name: SetupAdaptor
+      filename: setup-adaptor.py
       type: TEXT
       runnable: true
       data: |
-        #!/bin/env bash
-
-        set -euo pipefail
-
-        echo "The adaptor wheels that are attached to the job:"
-        ls '{{Param.OverrideAdaptorWheels}}'
-        echo ""
-
-        # Create a venv and activate it in this environment
-        echo "Creating Python venv for the {{Param.OverrideAdaptorName}} command"
-        /usr/local/bin/python3 -m venv '{{Session.WorkingDirectory}}/venv'
-        {{Env.File.InitialVars}}
-        . '{{Session.WorkingDirectory}}/venv/bin/activate'
-        {{Env.File.CaptureVars}}
-        echo ""
-
-        echo "Installing adaptor into the venv"
-        pip install '{{Param.OverrideAdaptorWheels}}'/openjd*.whl
-        pip install '{{Param.OverrideAdaptorWheels}}'/deadline*.whl
-        echo ""
-
-        if [ ! -f '{{Session.WorkingDirectory}}/venv/bin/{{Param.OverrideAdaptorName}}' ]; then
-          echo "The Override Adaptor {{Param.OverrideAdaptorName}} was not installed as expected."
-          exit 1
-        fi
-    - name: InitialVars
-      filename: initial-vars
-      type: TEXT
-      runnable: true
-      data: |
-        #!/usr/bin/env python3
-        import os, json
-        envfile = "{{Session.WorkingDirectory}}/.envInitial"
-        with open(envfile, "w", encoding="utf8") as f:
-            json.dump(dict(os.environ), f)
-    - name: CaptureVars
-      filename: capture-vars
-      type: TEXT
-      runnable: true
-      data: |
-        #!/usr/bin/env python3
-        import os, json, sys
-        envfile = "{{Session.WorkingDirectory}}/.envInitial"
-        if os.path.isfile(envfile):
-            with open(envfile, "r", encoding="utf8") as f:
-                before = json.load(f)
-        else:
-            print("No initial environment found, must run Env.File.CaptureVars script first")
-            sys.exit(1)
-        after = dict(os.environ)
-
-        put = {k: v for k, v in after.items() if v != before.get(k)}
-        delete = {k for k in before if k not in after}
-
-        for k, v in put.items():
-            print(f"updating {k}={v}")
-            print(f"openjd_env: {k}={v}")
-        for k in delete:
-            print(f"openjd_unset_env: {k}")
+        # Replaced with setup_adaptor_wheels.py while building the job template.
+        pass

src/deadline/cinema4d_submitter/cinema4d_render_submitter.py

@@ -42,6 +42,7 @@
 from .font_utils import FONTS_DIR, get_font_manager_environment, scene_has_fonts
 from .platform_utils import is_macos, is_windows
 from .scene import Animation, Scene, get_renderer_warning
+from .setup_adaptor_wheels import _find_wheels
 from .style import C4D_STYLE
 from .takes import TakeSelection
 from .template_timeout_patcher import add_timeouts_to_job_template
@@ -256,6 +257,54 @@ def _get_parameter_definition(
     return parameter
 
 
+def _get_adaptor_override_environment(wheels_path: Path) -> dict[str, Any]:
+    if not wheels_path.is_dir():
+        raise RuntimeError(
+            "The Developer Option 'Include Adaptor Wheels' is enabled, but the wheels "
+            f"directory does not exist:\n{wheels_path}"
+        )
+
+    try:
+        _find_wheels(wheels_path)
+    except RuntimeError as exc:
+        raise RuntimeError(
+            "The Developer Option 'Include Adaptor Wheels' is enabled, but the wheels "
+            f"directory contains the wrong wheels:\n{exc}"
+        ) from exc
+
+    with open(Path(__file__).parent / "adaptor_override_environment.yaml") as file:
+        override_environment = yaml.safe_load(file)
+
+    override_adaptor_wheels_param = _get_parameter_definition(
+        override_environment["parameterDefinitions"],
+        "OverrideAdaptorWheels",
+    )
+    override_adaptor_wheels_param["default"] = str(wheels_path)
+    override_adaptor_name_param = _get_parameter_definition(
+        override_environment["parameterDefinitions"],
+        "OverrideAdaptorName",
+    )
+    override_adaptor_name_param["default"] = "cinema4d-openjd"
+
+    setup_script_path = Path(__file__).parent / "setup_adaptor_wheels.py"
+    setup_script = setup_script_path.read_text(encoding="utf8")
+    setup_file = next(
+        (
+            embedded_file
+            for embedded_file in override_environment["environment"]["script"]["embeddedFiles"]
+            if embedded_file["name"] == "SetupAdaptor"
+        ),
+        None,
+    )
+    if setup_file is None:
+        raise RuntimeError(
+            "Adaptor override environment is missing the 'SetupAdaptor' embedded file"
+        )
+    setup_file["data"] = setup_script
+
+    return override_environment
+
+
 def _get_job_template(
     settings: RenderSubmitterUISettings,
     renderers: set[str],
@@ -387,40 +436,9 @@ def _get_job_template(
 
     # If this developer option is enabled, merge the adaptor_override_environment
     if settings.include_adaptor_wheels:
-        with open(Path(__file__).parent / "adaptor_override_environment.yaml") as f:
-            override_environment = yaml.safe_load(f)
-
         # Read DEVELOPMENT.md for instructions to create the wheels directory.
         wheels_path = Path(__file__).parent.parent.parent.parent / "wheels"
-        if not wheels_path.exists() and wheels_path.is_dir():
-            raise RuntimeError(
-                "The Developer Option 'Include Adaptor Wheels' is enabled, but the wheels directory does not exist:\n"
-                + str(wheels_path)
-            )
-        wheels_path_package_names = {
-            path.split("-", 1)[0] for path in os.listdir(wheels_path) if path.endswith(".whl")
-        }
-        if wheels_path_package_names != {
-            "openjd_adaptor_runtime",
-            "deadline",
-            "deadline_cloud_for_cinema4d",
-        }:
-            raise RuntimeError(
-                "The Developer Option 'Include Adaptor Wheels' is enabled, but the wheels directory contains the wrong wheels:\n"
-                + "Expected: openjd_adaptor_runtime, deadline, and deadline_cloud_for_cinema4d\n"
-                + f"Actual: {wheels_path_package_names}"
-            )
-
-        override_adaptor_wheels_param = _get_parameter_definition(
-            override_environment["parameterDefinitions"],
-            "OverrideAdaptorWheels",
-        )
-        override_adaptor_wheels_param["default"] = str(wheels_path)
-        override_adaptor_name_param = _get_parameter_definition(
-            override_environment["parameterDefinitions"],
-            "OverrideAdaptorName",
-        )
-        override_adaptor_name_param["default"] = "cinema4d-openjd"
+        override_environment = _get_adaptor_override_environment(wheels_path)
 
         # There are no parameter conflicts between these two templates, so this works
         job_template["parameterDefinitions"].extend(override_environment["parameterDefinitions"])

src/deadline/cinema4d_submitter/setup_adaptor_wheels.py

@@ -0,0 +1,178 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+"""Set up locally built adaptor wheels on a Deadline Cloud worker."""
+
+from __future__ import annotations
+
+import argparse
+import os
+import re
+import subprocess
+import sys
+from collections.abc import Sequence
+from importlib.metadata import distributions
+from pathlib import Path
+
+_EXPECTED_WHEEL_PREFIXES = (
+    "openjd_adaptor_runtime-",
+    "deadline-",
+    "deadline_cloud_for_cinema_4d-",
+)
+
+
+def _find_wheels(wheels_dir: Path) -> list[Path]:
+    wheels = sorted(wheels_dir.glob("*.whl"))
+    selected: list[Path] = []
+
+    for prefix in _EXPECTED_WHEEL_PREFIXES:
+        matches = [wheel for wheel in wheels if wheel.name.startswith(prefix)]
+        if len(matches) != 1:
+            raise RuntimeError(
+                f"Expected exactly one wheel matching '{prefix}*.whl' in {wheels_dir}, "
+                f"found {[wheel.name for wheel in matches]}"
+            )
+        selected.append(matches[0])
+
+    if len(wheels) != len(selected):
+        unexpected = sorted(set(wheels) - set(selected))
+        raise RuntimeError(
+            "The adaptor wheels directory contains unexpected wheels: "
+            + ", ".join(wheel.name for wheel in unexpected)
+        )
+
+    return selected
+
+
+def _get_venv_paths(venv_dir: Path, adaptor_name: str) -> tuple[Path, Path, Path]:
+    if os.name == "nt":
+        bin_dir = venv_dir / "Scripts"
+        return bin_dir, bin_dir / "python.exe", bin_dir / f"{adaptor_name}.exe"
+
+    bin_dir = venv_dir / "bin"
+    return bin_dir, bin_dir / "python", bin_dir / adaptor_name
+
+
+def _get_site_packages(venv_python: Path) -> Path:
+    result = subprocess.run(
+        [
+            str(venv_python),
+            "-c",
+            "import sysconfig; print(sysconfig.get_paths()['purelib'])",
+        ],
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    return Path(result.stdout.strip())
+
+
+def _get_distribution_version(site_packages: Path, distribution_name: str) -> str:
+    canonical_name = re.sub(r"[-_.]+", "-", distribution_name).lower()
+    for distribution in distributions(path=[str(site_packages)]):
+        installed_name = distribution.metadata["Name"]
+        if installed_name and re.sub(r"[-_.]+", "-", installed_name).lower() == canonical_name:
+            return distribution.version
+    raise RuntimeError(f"Could not find '{distribution_name}' in {site_packages}")
+
+
+def _install_wheels(venv_python: Path, wheels: list[Path]) -> None:
+    # The active Conda environment supplies transitive dependencies. Development
+    # wheels use generated versions that may not satisfy each other's release ranges.
+    subprocess.run(
+        [
+            str(venv_python),
+            "-m",
+            "pip",
+            "install",
+            "--disable-pip-version-check",
+            "--force-reinstall",
+            "--no-deps",
+            *[str(wheel) for wheel in wheels],
+        ],
+        check=True,
+    )
+
+
+def _prioritize_site_packages(site_packages: Path) -> None:
+    # Queue dependencies may be exposed through PYTHONPATH. Keep that environment
+    # unchanged for Cinema 4D, but make the attached wheels win module resolution.
+    (site_packages / "_deadline_adaptor_override.pth").write_text(
+        f"import sys; sys.path.insert(0, {str(site_packages)!r})\n",
+        encoding="utf8",
+    )
+
+
+def _emit_environment_changes(
+    before: dict[str, str],
+    *,
+    venv_dir: Path,
+    venv_bin: Path,
+) -> None:
+    after = dict(before)
+    after["PATH"] = os.pathsep.join(filter(None, (str(venv_bin), before.get("PATH", ""))))
+    after["VIRTUAL_ENV"] = str(venv_dir)
+    after.pop("PYTHONHOME", None)
+
+    for key, value in sorted(after.items()):
+        if value != before.get(key):
+            print(f"openjd_env: {key}={value}")
+
+    for key in sorted(before):
+        if key not in after:
+            print(f"openjd_unset_env: {key}")
+
+
+def _parse_args(argv: Sequence[str] | None = None) -> tuple[Path, Path, str]:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("working_directory")
+    parser.add_argument("wheels_directory")
+    parser.add_argument("adaptor_name")
+    args = parser.parse_args(argv)
+    return Path(args.working_directory), Path(args.wheels_directory), args.adaptor_name
+
+
+def main(argv: Sequence[str] | None = None) -> None:
+    working_dir, wheels_dir, adaptor_name = _parse_args(argv)
+    before_environment = dict(os.environ)
+
+    print(f"Setting up {adaptor_name} from attached wheels on {sys.platform}")
+    wheels = _find_wheels(wheels_dir)
+    for wheel in wheels:
+        print(f"  {wheel.name}")
+
+    venv_dir = working_dir / "adaptor-venv"
+    print(f"Creating adaptor virtual environment at {venv_dir}")
+    subprocess.run(
+        [sys.executable, "-m", "venv", "--system-site-packages", str(venv_dir)],
+        check=True,
+    )
+
+    venv_bin, venv_python, adaptor_executable = _get_venv_paths(venv_dir, adaptor_name)
+    _install_wheels(venv_python, wheels)
+
+    if not adaptor_executable.is_file():
+        raise RuntimeError(
+            f"The override adaptor '{adaptor_name}' was not installed at {adaptor_executable}"
+        )
+
+    site_packages = _get_site_packages(venv_python)
+    _prioritize_site_packages(site_packages)
+    adaptor_version = _get_distribution_version(site_packages, "deadline-cloud-for-cinema-4d")
+    print(
+        "ADAPTOR_OVERRIDE_READY "
+        f"executable={adaptor_executable} "
+        f"package=deadline-cloud-for-cinema-4d version={adaptor_version}"
+    )
+
+    _emit_environment_changes(
+        before_environment,
+        venv_dir=venv_dir,
+        venv_bin=venv_bin,
+    )
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except Exception as exc:
+        print(f"ADAPTOR_OVERRIDE_FAILED: {exc}", file=sys.stderr)
+        raise