diff --git a/src/deadline/cinema4d_submitter/adaptor_override_environment.yaml b/src/deadline/cinema4d_submitter/adaptor_override_environment.yaml index 68608484..1a13c423 100644 --- a/src/deadline/cinema4d_submitter/adaptor_override_environment.yaml +++ b/src/deadline/cinema4d_submitter/adaptor_override_environment.yaml @@ -16,69 +16,20 @@ environment: script: actions: onEnter: - command: '{{Env.File.Enter}}' + # The Cinema 4D Conda environment provides "python" on Windows and Linux. + command: python + args: + - '{{Env.File.SetupAdaptor}}' + - '{{Session.WorkingDirectory}}' + - '{{Param.OverrideAdaptorWheels}}' + - '{{Param.OverrideAdaptorName}}' + cancelation: + mode: NOTIFY_THEN_TERMINATE embeddedFiles: - - name: Enter - filename: override-adaptor-enter.sh + - name: SetupAdaptor + filename: setup-adaptor.py type: TEXT runnable: true data: | - #!/bin/env bash - - set -euo pipefail - - echo "The adaptor wheels that are attached to the job:" - ls '{{Param.OverrideAdaptorWheels}}' - echo "" - - # Create a venv and activate it in this environment - echo "Creating Python venv for the {{Param.OverrideAdaptorName}} command" - /usr/local/bin/python3 -m venv '{{Session.WorkingDirectory}}/venv' - {{Env.File.InitialVars}} - . '{{Session.WorkingDirectory}}/venv/bin/activate' - {{Env.File.CaptureVars}} - echo "" - - echo "Installing adaptor into the venv" - pip install '{{Param.OverrideAdaptorWheels}}'/openjd*.whl - pip install '{{Param.OverrideAdaptorWheels}}'/deadline*.whl - echo "" - - if [ ! -f '{{Session.WorkingDirectory}}/venv/bin/{{Param.OverrideAdaptorName}}' ]; then - echo "The Override Adaptor {{Param.OverrideAdaptorName}} was not installed as expected." - exit 1 - fi - - name: InitialVars - filename: initial-vars - type: TEXT - runnable: true - data: | - #!/usr/bin/env python3 - import os, json - envfile = "{{Session.WorkingDirectory}}/.envInitial" - with open(envfile, "w", encoding="utf8") as f: - json.dump(dict(os.environ), f) - - name: CaptureVars - filename: capture-vars - type: TEXT - runnable: true - data: | - #!/usr/bin/env python3 - import os, json, sys - envfile = "{{Session.WorkingDirectory}}/.envInitial" - if os.path.isfile(envfile): - with open(envfile, "r", encoding="utf8") as f: - before = json.load(f) - else: - print("No initial environment found, must run Env.File.CaptureVars script first") - sys.exit(1) - after = dict(os.environ) - - put = {k: v for k, v in after.items() if v != before.get(k)} - delete = {k for k in before if k not in after} - - for k, v in put.items(): - print(f"updating {k}={v}") - print(f"openjd_env: {k}={v}") - for k in delete: - print(f"openjd_unset_env: {k}") + # Replaced with setup_adaptor_wheels.py while building the job template. + pass diff --git a/src/deadline/cinema4d_submitter/cinema4d_render_submitter.py b/src/deadline/cinema4d_submitter/cinema4d_render_submitter.py index c69e0032..329d8da8 100644 --- a/src/deadline/cinema4d_submitter/cinema4d_render_submitter.py +++ b/src/deadline/cinema4d_submitter/cinema4d_render_submitter.py @@ -42,6 +42,7 @@ from .font_utils import FONTS_DIR, get_font_manager_environment, scene_has_fonts from .platform_utils import is_macos, is_windows from .scene import Animation, Scene, get_renderer_warning +from .setup_adaptor_wheels import _find_wheels from .style import C4D_STYLE from .takes import TakeSelection from .template_timeout_patcher import add_timeouts_to_job_template @@ -256,6 +257,54 @@ def _get_parameter_definition( return parameter +def _get_adaptor_override_environment(wheels_path: Path) -> dict[str, Any]: + if not wheels_path.is_dir(): + raise RuntimeError( + "The Developer Option 'Include Adaptor Wheels' is enabled, but the wheels " + f"directory does not exist:\n{wheels_path}" + ) + + try: + _find_wheels(wheels_path) + except RuntimeError as exc: + raise RuntimeError( + "The Developer Option 'Include Adaptor Wheels' is enabled, but the wheels " + f"directory contains the wrong wheels:\n{exc}" + ) from exc + + with open(Path(__file__).parent / "adaptor_override_environment.yaml") as file: + override_environment = yaml.safe_load(file) + + override_adaptor_wheels_param = _get_parameter_definition( + override_environment["parameterDefinitions"], + "OverrideAdaptorWheels", + ) + override_adaptor_wheels_param["default"] = str(wheels_path) + override_adaptor_name_param = _get_parameter_definition( + override_environment["parameterDefinitions"], + "OverrideAdaptorName", + ) + override_adaptor_name_param["default"] = "cinema4d-openjd" + + setup_script_path = Path(__file__).parent / "setup_adaptor_wheels.py" + setup_script = setup_script_path.read_text(encoding="utf8") + setup_file = next( + ( + embedded_file + for embedded_file in override_environment["environment"]["script"]["embeddedFiles"] + if embedded_file["name"] == "SetupAdaptor" + ), + None, + ) + if setup_file is None: + raise RuntimeError( + "Adaptor override environment is missing the 'SetupAdaptor' embedded file" + ) + setup_file["data"] = setup_script + + return override_environment + + def _get_job_template( settings: RenderSubmitterUISettings, renderers: set[str], @@ -387,40 +436,9 @@ def _get_job_template( # If this developer option is enabled, merge the adaptor_override_environment if settings.include_adaptor_wheels: - with open(Path(__file__).parent / "adaptor_override_environment.yaml") as f: - override_environment = yaml.safe_load(f) - # Read DEVELOPMENT.md for instructions to create the wheels directory. wheels_path = Path(__file__).parent.parent.parent.parent / "wheels" - if not wheels_path.exists() and wheels_path.is_dir(): - raise RuntimeError( - "The Developer Option 'Include Adaptor Wheels' is enabled, but the wheels directory does not exist:\n" - + str(wheels_path) - ) - wheels_path_package_names = { - path.split("-", 1)[0] for path in os.listdir(wheels_path) if path.endswith(".whl") - } - if wheels_path_package_names != { - "openjd_adaptor_runtime", - "deadline", - "deadline_cloud_for_cinema4d", - }: - raise RuntimeError( - "The Developer Option 'Include Adaptor Wheels' is enabled, but the wheels directory contains the wrong wheels:\n" - + "Expected: openjd_adaptor_runtime, deadline, and deadline_cloud_for_cinema4d\n" - + f"Actual: {wheels_path_package_names}" - ) - - override_adaptor_wheels_param = _get_parameter_definition( - override_environment["parameterDefinitions"], - "OverrideAdaptorWheels", - ) - override_adaptor_wheels_param["default"] = str(wheels_path) - override_adaptor_name_param = _get_parameter_definition( - override_environment["parameterDefinitions"], - "OverrideAdaptorName", - ) - override_adaptor_name_param["default"] = "cinema4d-openjd" + override_environment = _get_adaptor_override_environment(wheels_path) # There are no parameter conflicts between these two templates, so this works job_template["parameterDefinitions"].extend(override_environment["parameterDefinitions"]) diff --git a/src/deadline/cinema4d_submitter/setup_adaptor_wheels.py b/src/deadline/cinema4d_submitter/setup_adaptor_wheels.py new file mode 100644 index 00000000..47e75b57 --- /dev/null +++ b/src/deadline/cinema4d_submitter/setup_adaptor_wheels.py @@ -0,0 +1,178 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +"""Set up locally built adaptor wheels on a Deadline Cloud worker.""" + +from __future__ import annotations + +import argparse +import os +import re +import subprocess +import sys +from collections.abc import Sequence +from importlib.metadata import distributions +from pathlib import Path + +_EXPECTED_WHEEL_PREFIXES = ( + "openjd_adaptor_runtime-", + "deadline-", + "deadline_cloud_for_cinema_4d-", +) + + +def _find_wheels(wheels_dir: Path) -> list[Path]: + wheels = sorted(wheels_dir.glob("*.whl")) + selected: list[Path] = [] + + for prefix in _EXPECTED_WHEEL_PREFIXES: + matches = [wheel for wheel in wheels if wheel.name.startswith(prefix)] + if len(matches) != 1: + raise RuntimeError( + f"Expected exactly one wheel matching '{prefix}*.whl' in {wheels_dir}, " + f"found {[wheel.name for wheel in matches]}" + ) + selected.append(matches[0]) + + if len(wheels) != len(selected): + unexpected = sorted(set(wheels) - set(selected)) + raise RuntimeError( + "The adaptor wheels directory contains unexpected wheels: " + + ", ".join(wheel.name for wheel in unexpected) + ) + + return selected + + +def _get_venv_paths(venv_dir: Path, adaptor_name: str) -> tuple[Path, Path, Path]: + if os.name == "nt": + bin_dir = venv_dir / "Scripts" + return bin_dir, bin_dir / "python.exe", bin_dir / f"{adaptor_name}.exe" + + bin_dir = venv_dir / "bin" + return bin_dir, bin_dir / "python", bin_dir / adaptor_name + + +def _get_site_packages(venv_python: Path) -> Path: + result = subprocess.run( + [ + str(venv_python), + "-c", + "import sysconfig; print(sysconfig.get_paths()['purelib'])", + ], + check=True, + capture_output=True, + text=True, + ) + return Path(result.stdout.strip()) + + +def _get_distribution_version(site_packages: Path, distribution_name: str) -> str: + canonical_name = re.sub(r"[-_.]+", "-", distribution_name).lower() + for distribution in distributions(path=[str(site_packages)]): + installed_name = distribution.metadata["Name"] + if installed_name and re.sub(r"[-_.]+", "-", installed_name).lower() == canonical_name: + return distribution.version + raise RuntimeError(f"Could not find '{distribution_name}' in {site_packages}") + + +def _install_wheels(venv_python: Path, wheels: list[Path]) -> None: + # The active Conda environment supplies transitive dependencies. Development + # wheels use generated versions that may not satisfy each other's release ranges. + subprocess.run( + [ + str(venv_python), + "-m", + "pip", + "install", + "--disable-pip-version-check", + "--force-reinstall", + "--no-deps", + *[str(wheel) for wheel in wheels], + ], + check=True, + ) + + +def _prioritize_site_packages(site_packages: Path) -> None: + # Queue dependencies may be exposed through PYTHONPATH. Keep that environment + # unchanged for Cinema 4D, but make the attached wheels win module resolution. + (site_packages / "_deadline_adaptor_override.pth").write_text( + f"import sys; sys.path.insert(0, {str(site_packages)!r})\n", + encoding="utf8", + ) + + +def _emit_environment_changes( + before: dict[str, str], + *, + venv_dir: Path, + venv_bin: Path, +) -> None: + after = dict(before) + after["PATH"] = os.pathsep.join(filter(None, (str(venv_bin), before.get("PATH", "")))) + after["VIRTUAL_ENV"] = str(venv_dir) + after.pop("PYTHONHOME", None) + + for key, value in sorted(after.items()): + if value != before.get(key): + print(f"openjd_env: {key}={value}") + + for key in sorted(before): + if key not in after: + print(f"openjd_unset_env: {key}") + + +def _parse_args(argv: Sequence[str] | None = None) -> tuple[Path, Path, str]: + parser = argparse.ArgumentParser() + parser.add_argument("working_directory") + parser.add_argument("wheels_directory") + parser.add_argument("adaptor_name") + args = parser.parse_args(argv) + return Path(args.working_directory), Path(args.wheels_directory), args.adaptor_name + + +def main(argv: Sequence[str] | None = None) -> None: + working_dir, wheels_dir, adaptor_name = _parse_args(argv) + before_environment = dict(os.environ) + + print(f"Setting up {adaptor_name} from attached wheels on {sys.platform}") + wheels = _find_wheels(wheels_dir) + for wheel in wheels: + print(f" {wheel.name}") + + venv_dir = working_dir / "adaptor-venv" + print(f"Creating adaptor virtual environment at {venv_dir}") + subprocess.run( + [sys.executable, "-m", "venv", "--system-site-packages", str(venv_dir)], + check=True, + ) + + venv_bin, venv_python, adaptor_executable = _get_venv_paths(venv_dir, adaptor_name) + _install_wheels(venv_python, wheels) + + if not adaptor_executable.is_file(): + raise RuntimeError( + f"The override adaptor '{adaptor_name}' was not installed at {adaptor_executable}" + ) + + site_packages = _get_site_packages(venv_python) + _prioritize_site_packages(site_packages) + adaptor_version = _get_distribution_version(site_packages, "deadline-cloud-for-cinema-4d") + print( + "ADAPTOR_OVERRIDE_READY " + f"executable={adaptor_executable} " + f"package=deadline-cloud-for-cinema-4d version={adaptor_version}" + ) + + _emit_environment_changes( + before_environment, + venv_dir=venv_dir, + venv_bin=venv_bin, + ) + + +if __name__ == "__main__": + try: + main() + except Exception as exc: + print(f"ADAPTOR_OVERRIDE_FAILED: {exc}", file=sys.stderr) + raise diff --git a/test/unit/deadline_submitter_for_cinema4d/test_adaptor_override.py b/test/unit/deadline_submitter_for_cinema4d/test_adaptor_override.py new file mode 100644 index 00000000..d684395b --- /dev/null +++ b/test/unit/deadline_submitter_for_cinema4d/test_adaptor_override.py @@ -0,0 +1,196 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +import os +from pathlib import Path +from unittest import mock + +import pytest + +from deadline.cinema4d_submitter.cinema4d_render_submitter import ( + _get_adaptor_override_environment, +) +from deadline.cinema4d_submitter.setup_adaptor_wheels import ( + _emit_environment_changes, + _find_wheels, + _get_distribution_version, + _get_venv_paths, + _install_wheels, + _parse_args, + _prioritize_site_packages, +) + +_WHEEL_NAMES = ( + "openjd_adaptor_runtime-0.9.0-py3-none-any.whl", + "deadline-0.60.4-py3-none-any.whl", + "deadline_cloud_for_cinema_4d-0.12.1-py3-none-any.whl", +) + + +def _create_wheels(wheels_dir: Path) -> None: + wheels_dir.mkdir() + for wheel_name in _WHEEL_NAMES: + (wheels_dir / wheel_name).touch() + + +def test_get_adaptor_override_environment_embeds_cross_platform_setup(tmp_path): + wheels_dir = tmp_path / "wheels" + _create_wheels(wheels_dir) + + result = _get_adaptor_override_environment(wheels_dir) + + parameters = {parameter["name"]: parameter for parameter in result["parameterDefinitions"]} + assert parameters["OverrideAdaptorWheels"]["default"] == str(wheels_dir) + assert parameters["OverrideAdaptorName"]["default"] == "cinema4d-openjd" + + script = result["environment"]["script"] + assert script["actions"]["onEnter"] == { + "command": "python", + "args": [ + "{{Env.File.SetupAdaptor}}", + "{{Session.WorkingDirectory}}", + "{{Param.OverrideAdaptorWheels}}", + "{{Param.OverrideAdaptorName}}", + ], + "cancelation": {"mode": "NOTIFY_THEN_TERMINATE"}, + } + setup_script = script["embeddedFiles"][0]["data"] + assert "ADAPTOR_OVERRIDE_READY" in setup_script + assert '[sys.executable, "-m", "venv", "--system-site-packages", str(venv_dir)]' in setup_script + assert "{{Session.WorkingDirectory}}" not in setup_script + + +def test_get_adaptor_override_environment_rejects_missing_directory(tmp_path): + wheels_dir = tmp_path / "missing" + + with pytest.raises(RuntimeError, match="wheels directory does not exist"): + _get_adaptor_override_environment(wheels_dir) + + +def test_get_adaptor_override_environment_rejects_duplicate_package_wheels(tmp_path): + wheels_dir = tmp_path / "wheels" + _create_wheels(wheels_dir) + (wheels_dir / "deadline_cloud_for_cinema_4d-0.12.2-py3-none-any.whl").touch() + + with pytest.raises(RuntimeError, match="Expected exactly one wheel"): + _get_adaptor_override_environment(wheels_dir) + + +def test_find_wheels_requires_exact_expected_set(tmp_path): + wheels_dir = tmp_path / "wheels" + _create_wheels(wheels_dir) + + assert [wheel.name for wheel in _find_wheels(wheels_dir)] == list(_WHEEL_NAMES) + + (wheels_dir / "unexpected-1.0-py3-none-any.whl").touch() + with pytest.raises(RuntimeError, match="unexpected wheels"): + _find_wheels(wheels_dir) + + +@pytest.mark.parametrize( + ("os_name", "bin_directory", "python_name", "adaptor_name"), + [ + ("nt", "Scripts", "python.exe", "cinema4d-openjd.exe"), + ("posix", "bin", "python", "cinema4d-openjd"), + ], +) +def test_get_venv_paths_for_worker_os(tmp_path, os_name, bin_directory, python_name, adaptor_name): + venv_dir = tmp_path / "venv" + + with mock.patch( + "deadline.cinema4d_submitter.setup_adaptor_wheels.os.name", + os_name, + ): + result = _get_venv_paths(venv_dir, "cinema4d-openjd") + + expected_bin = venv_dir / bin_directory + assert result == ( + expected_bin, + expected_bin / python_name, + expected_bin / adaptor_name, + ) + + +def test_get_distribution_version_ignores_other_site_packages(tmp_path): + site_packages = tmp_path / "site-packages" + dist_info = site_packages / "deadline_cloud_for_cinema_4d-1.2.3.dist-info" + dist_info.mkdir(parents=True) + (dist_info / "METADATA").write_text( + "Metadata-Version: 2.1\n" "Name: deadline-cloud-for-cinema-4d\n" "Version: 1.2.3\n", + encoding="utf8", + ) + + assert _get_distribution_version(site_packages, "deadline_cloud_for_cinema_4d") == "1.2.3" + + +def test_install_wheels_forces_attached_development_versions(tmp_path): + venv_python = tmp_path / "venv" / "bin" / "python" + wheels = [tmp_path / wheel_name for wheel_name in _WHEEL_NAMES] + + with mock.patch("deadline.cinema4d_submitter.setup_adaptor_wheels.subprocess.run") as run_mock: + _install_wheels(venv_python, wheels) + + assert run_mock.call_args_list == [ + mock.call( + [ + str(venv_python), + "-m", + "pip", + "install", + "--disable-pip-version-check", + "--force-reinstall", + "--no-deps", + *[str(wheel) for wheel in wheels], + ], + check=True, + ), + ] + + +def test_parse_args_keeps_windows_paths_out_of_script_source(): + working_directory = "D:\\" + wheels_directory = "Z:\\adaptor wheels\\" + + result = _parse_args([working_directory, wheels_directory, "cinema4d-openjd"]) + + assert result == ( + Path(working_directory), + Path(wheels_directory), + "cinema4d-openjd", + ) + + +def test_prioritize_site_packages_does_not_change_pythonpath(tmp_path): + site_packages = tmp_path / "venv" / "site-packages" + site_packages.mkdir(parents=True) + + _prioritize_site_packages(site_packages) + + precedence_file = site_packages / "_deadline_adaptor_override.pth" + assert precedence_file.read_text(encoding="utf8") == ( + f"import sys; sys.path.insert(0, {str(site_packages)!r})\n" + ) + assert "PYTHONPATH" not in precedence_file.read_text(encoding="utf8") + + +def test_emit_environment_changes_uses_platform_path_separator(capsys, tmp_path): + before = { + "PATH": os.pathsep.join(("base", "bin")), + "PYTHONPATH": "base-packages", + "PYTHONHOME": "base-python", + "UNCHANGED": "value", + } + venv_dir = tmp_path / "venv" + venv_bin = venv_dir / "Scripts" + + _emit_environment_changes( + before, + venv_dir=venv_dir, + venv_bin=venv_bin, + ) + + output = capsys.readouterr().out.splitlines() + assert f"openjd_env: PATH={venv_bin}{os.pathsep}{before['PATH']}" in output + assert f"openjd_env: VIRTUAL_ENV={venv_dir}" in output + assert "openjd_unset_env: PYTHONHOME" in output + assert all("PYTHONPATH" not in line for line in output) + assert all("UNCHANGED" not in line for line in output)