Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions tools/internal/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -86,8 +86,10 @@ supported platforms, download URLs, checksums, and archive layouts.
`rules_multitool` consumes the files for Bazel. The standard-library-only
[`catalog.py`](catalog.py) reads metadata for
[`devcontainer/install.py`](devcontainer/install.py) and README generation.
The reader has no installation side effects; download, platform selection,
and installation remain in the container installer.
The reader also prepares version and checksum variables for setup scripts and
feature tests through `load_setup_metadata()`. It has no installation side
effects; shell quoting, download, platform selection and installation remain in
the container installer.
Other direct packages and special setup are defined in
[`providers.json`](../providers.json): Python package pins, APT constraints,
npm package identities, Bazel preinstallation and CodeQL metadata. The selections reference these names
Expand Down
36 changes: 36 additions & 0 deletions tools/internal/catalog.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
from __future__ import annotations

import json
import re
from collections.abc import Iterator
from pathlib import Path
from typing import NotRequired, TypedDict
Expand Down Expand Up @@ -323,3 +324,38 @@ def apt_requirements(
package += "=" + tool["version"] + "*"
requirements.append(package)
return requirements


def load_setup_metadata() -> dict[str, str]:
"""Turn definition fields into variables such as codeql_version for setup and tests."""
values: dict[str, str] = {}

def emit(name: str, value: object) -> None:
if not re.fullmatch(r"[a-zA-Z_][a-zA-Z0-9_]*", name):
raise SystemExit(f"Invalid metadata variable: {name}")
if isinstance(value, dict):
for key, child in value.items():
emit(name + "_" + key, child)
elif isinstance(value, str):
values[name] = value

for name, definition in load_providers().items():
# Use only version and checksum fields. Other data, such as platform
# names containing slashes, cannot be used as shell variable names.
for field in ("version", "compatibility_version", "amd64", "arm64"):
if field in definition:
emit(name.replace("-", "_") + "_" + field, definition[field])
# Unpinned APT tools can still have expected versions for feature tests.
# Give these expectations the same _version names as pinned tools.
if "compatibility_version" in definition:
emit(
name.replace("-", "_") + "_version", definition["compatibility_version"]
)
# Native and Python command versions use the same variable names as the
# other definitions, so feature tests need just one metadata query.
for name, version in load_catalog_versions().items():
emit(name.replace("-", "_") + "_version", version)
# Python package and entrypoint metadata is also available to setup scripts.
for name, definition in load_python_tools().items():
emit(name.replace("-", "_"), definition)
return values
42 changes: 3 additions & 39 deletions tools/internal/devcontainer/install.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
load_environments,
load_python_tools,
load_providers,
load_setup_metadata,
load_tool,
resolve_lockfile,
)
Expand Down Expand Up @@ -429,56 +430,19 @@ def _apply_environment(args, selection, definitions, setup_scripts) -> int:
# Pass variables to the setup processes. Python cannot change the
# environment of the shell that launched this installer.
if "setup" in args.providers and setup_scripts:
environment = {**os.environ, **_metadata_values()}
environment = {**os.environ, **load_setup_metadata()}
environment["SCORE_TOOL_ENVIRONMENT"] = args.environment
environment["SCORE_TOOLS_INSTALLER"] = str(Path(__file__).resolve())
for script in setup_scripts:
subprocess.run(["bash", str(script)], check=True, env=environment)
return 0


def _metadata_values() -> dict[str, str]:
"""Turn definition fields into variables such as codeql_version for setup and tests."""
import re

values: dict[str, str] = {}

def emit(name, value):
if not re.fullmatch(r"[a-zA-Z_][a-zA-Z0-9_]*", name):
raise SystemExit(f"Invalid metadata variable: {name}")
if isinstance(value, dict):
for key, child in value.items():
emit(name + "_" + key, child)
elif isinstance(value, str):
values[name] = value

for name, definition in load_providers().items():
# Use only version and checksum fields. Other data, such as platform
# names containing slashes, cannot be used as shell variable names.
for field in ("version", "compatibility_version", "amd64", "arm64"):
if field in definition:
emit(name.replace("-", "_") + "_" + field, definition[field])
# Unpinned APT tools can still have expected versions for feature tests.
# Give these expectations the same _version names as pinned tools.
if "compatibility_version" in definition:
emit(
name.replace("-", "_") + "_version", definition["compatibility_version"]
)
# Native and Python command versions use the same variable names as the
# other definitions, so feature tests need just one metadata query.
for name, version in load_catalog_versions().items():
emit(name.replace("-", "_") + "_version", version)
# Python package and entrypoint metadata is also available to setup scripts.
for name, definition in load_python_tools().items():
emit(name.replace("-", "_"), definition)
return values


def _cmd_shell_metadata(args: argparse.Namespace) -> int:
"""Print shell-quoted exports so feature tests can read versions in their shell."""
import shlex

for name, value in _metadata_values().items():
for name, value in load_setup_metadata().items():
print("export " + name + "=" + shlex.quote(value))
return 0

Expand Down
8 changes: 8 additions & 0 deletions tools/tests/catalog_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,14 @@


class CatalogTest(unittest.TestCase):
def test_setup_metadata_includes_every_native_and_python_command_version(
self,
) -> None:
# Feature tests should get all expected versions in one shell-metadata call.
metadata = catalog.load_setup_metadata()
for name, version in catalog.load_catalog_versions().items():
self.assertEqual(metadata[name.replace("-", "_") + "_version"], version)

def test_metadata_matches_definitions(self) -> None:
versions = {}
descriptions = {}
Expand Down
8 changes: 0 additions & 8 deletions tools/tests/environment_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -425,14 +425,6 @@ def test_default_lifecycle_stops_if_runtime_installation_fails(self) -> None:
[call.args[0].phase for call in apply.call_args_list], ["build", "runtime"]
)

def test_shell_metadata_includes_every_native_and_python_command_version(
self,
) -> None:
# Feature tests should get all expected versions in one shell-metadata call.
metadata = installer._metadata_values()
for name, version in catalog.load_catalog_versions().items():
self.assertEqual(metadata[name.replace("-", "_") + "_version"], version)

def test_apt_tool_setup_is_retained_when_installation_is_delegated(self) -> None:
# Java is selected through APT, but also needs java.sh to set JAVA_HOME.
# Both an image build and a normal user must execute that setup.
Expand Down