diff --git a/.flake8 b/.flake8 index 98e9a7dbc..27d94a76d 100644 --- a/.flake8 +++ b/.flake8 @@ -78,6 +78,13 @@ per-file-ignores = WPS226, # WPS115: "Require snake_case for naming class attributes" -- testing legitimate case, ignored in main code WPS115, + tests/pytest/tool_version_test.py: + # WPS226: "Forbid the overuse of string literals" -- same legitimate test-fixture rationale as _cli_test.py above + WPS226, + # WPS202: "Found too many module members" -- this suite covers one resolution branch per test, so the module is intentionally wide rather than deep + WPS202, + # WPS204: "Found overused expression" -- every test wires the same hermetic env/PATH sandbox and re-asserts "no download happened"; deduplicating that into helpers would hide what each test actually guarantees + WPS204, # We will not spend time on fixing complexity in deprecated hook src/pre_commit_terraform/terraform_docs_replace.py: WPS232 diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 1da68503e..7f1510d38 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -134,6 +134,9 @@ You can use [this PR](https://github.com/antonbabenko/pre-commit-terraform/pull/ * `docker build -t pre-commit --build-arg INSTALL_ALL=true .` * `docker build -t pre-commit --build-arg _VERSION=latest .` * `docker build -t pre-commit --build-arg _VERSION=<1.2.3> .` + + > [!NOTE] + > `tools/install/.sh` has a second call site besides the Dockerfile: if the new tool is distributed as a GitHub (or HashiCorp) release binary, `hooks/_common.sh::common::resolve_tool_path` invokes this same script at hook run-time to support [`--hook-config=--tool-version=`](../README.md#most-hooks-pin-a-specific-tool-version). Wire your new hook to it the same way the existing binary-wrapping hooks are (look at how `terraform_tflint.sh` or `terrascan.sh` call `common::resolve_tool_path`), unless the tool has a different distribution model (e.g. pip, like `checkov`) - in that case, skip this and leave a comment explaining why, as `terraform_checkov.sh` does. 2. Add Docker structure tests to [`.github/.container-structure-test-config.yaml`](.container-structure-test-config.yaml) 3. Add new hook to [`.pre-commit-hooks.yaml`](../.pre-commit-hooks.yaml) 4. Create hook file. Don't forget to make it executable via `chmod +x /path/to/hook/file`. diff --git a/.github/workflows/build-image-test.yaml b/.github/workflows/build-image-test.yaml index 9734f319e..77f4d42f3 100644 --- a/.github/workflows/build-image-test.yaml +++ b/.github/workflows/build-image-test.yaml @@ -68,13 +68,12 @@ jobs: context: . build-args: | INSTALL_ALL=true + GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} push: false load: true tags: ${{ env.IMAGE }} # Fix multi-platform: https://github.com/docker/buildx/issues/1533 provenance: false - secrets: | - "github_token=${{ secrets.GITHUB_TOKEN }}" - name: Setup Container Structure Tests if: steps.changed-files-specific.outputs.any_changed == 'true' @@ -121,10 +120,9 @@ jobs: context: . build-args: | INSTALL_ALL=true + GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} platforms: linux/amd64,linux/arm64 push: false tags: ${{ env.IMAGE }} # Fix multi-platform: https://github.com/docker/buildx/issues/1533 provenance: false - secrets: | - "github_token=${{ secrets.GITHUB_TOKEN }}" diff --git a/.github/workflows/build-image.yaml b/.github/workflows/build-image.yaml index a5ce590e2..b9cb7a5c6 100644 --- a/.github/workflows/build-image.yaml +++ b/.github/workflows/build-image.yaml @@ -58,6 +58,7 @@ jobs: context: . build-args: | INSTALL_ALL=true + GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} platforms: linux/amd64,linux/arm64 push: true tags: | @@ -65,8 +66,6 @@ jobs: ${{ env.IMAGE_REPO }}:latest # Fix multi-platform: https://github.com/docker/buildx/issues/1533 provenance: false - secrets: | - "github_token=${{ secrets.GITHUB_TOKEN }}" - name: Build and Push nightly if: github.event_name == 'schedule' @@ -75,11 +74,10 @@ jobs: context: . build-args: | INSTALL_ALL=true + GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} platforms: linux/amd64,linux/arm64 push: true tags: | ${{ env.IMAGE_REPO }}:nightly # Fix multi-platform: https://github.com/docker/buildx/issues/1533 provenance: false - secrets: | - "github_token=${{ secrets.GITHUB_TOKEN }}" diff --git a/.github/workflows/reusable-tox.yml b/.github/workflows/reusable-tox.yml index cb9c23d88..c4c4c595e 100644 --- a/.github/workflows/reusable-tox.yml +++ b/.github/workflows/reusable-tox.yml @@ -311,6 +311,13 @@ jobs: id: tox-run shell: bash env: + # Consumed by tools/install/_common.sh (via `pass_env = GITHUB_*` + # in tox.ini) to authenticate GitHub API calls made while + # resolving `--tool-version` pins - without it, requests are + # unauthenticated and share GitHub's low per-IP rate limit + # across every job in this workflow's matrix, which is exactly + # what test_real_download_on_cache_miss can hit. + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} INPUT_TOX_RUN_POSARGS: ${{ inputs.tox-run-posargs }} run: |- tox_common_args=( diff --git a/Dockerfile b/Dockerfile index c7c53cd13..291293bc9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -65,6 +65,8 @@ RUN if [ "$INSTALL_ALL" != "false" ]; then \ echo "TRIVY_VERSION=latest" >> /.env \ ; fi +# DL3064 - empty default; real value passed via --build-arg, never hardcoded +# hadolint ignore=DL3064 ARG GITHUB_TOKEN=${GITHUB_TOKEN:-""} # Docker `RUN`s shouldn't be consolidated here @@ -116,6 +118,8 @@ RUN apk add --no-cache \ git=~2 \ # All hooks deps bash=~5 \ + # needed by tools/install/.sh for --tool-version downloads + curl=~8 \ # pre-commit-hooks deps: https://github.com/pre-commit/pre-commit-hooks musl-dev=~1 \ gcc=~14 \ @@ -152,6 +156,8 @@ COPY tools/entrypoint.sh /entrypoint.sh ENV PRE_COMMIT_COLOR=${PRE_COMMIT_COLOR:-always} +# DL3064 - empty default; real value passed via `-e` at runtime, never hardcoded +# hadolint ignore=DL3064 ENV INFRACOST_API_KEY=${INFRACOST_API_KEY:-} ENV INFRACOST_SKIP_UPDATE_CHECK=${INFRACOST_SKIP_UPDATE_CHECK:-false} diff --git a/README.md b/README.md index f93a39460..5a71950ae 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,8 @@ If you want to support the development of `pre-commit-terraform` and [many other * [All hooks: Set env vars inside hook at runtime](#all-hooks-set-env-vars-inside-hook-at-runtime) * [All hooks: Disable color output](#all-hooks-disable-color-output) * [All hooks: Log levels](#all-hooks-log-levels) + * [Most hooks: Pin a specific tool version](#most-hooks-pin-a-specific-tool-version) + * [Keeping pinned versions up-to-date using Renovate](#keeping-pinned-versions-up-to-date-using-renovate) * [Many hooks: Parallelism](#many-hooks-parallelism) * [checkov (deprecated) and terraform\_checkov](#checkov-deprecated-and-terraform_checkov) * [infracost\_breakdown](#infracost_breakdown) @@ -77,6 +79,7 @@ If you want to support the development of `pre-commit-terraform` and [many other * [About Docker image security](#about-docker-image-security) * [File Permissions](#file-permissions) * [Download Terraform modules from private GitHub repositories](#download-terraform-modules-from-private-github-repositories) + * [Mount tools cache directory](#mount-tools-cache-directory) * [GitHub Actions](#github-actions) * [Authors](#authors) * [License](#license) @@ -434,6 +437,89 @@ PCT_LOG=trace pre-commit run -a Less verbose log levels will be implemented in [#562](https://github.com/antonbabenko/pre-commit-terraform/issues/562). +### Most hooks: Pin a specific tool version + +> All hooks, which wrap a tool distributed as a downloadable release asset. Not supported for `checkov`/`terraform_checkov` (distributed via PyPi) and for deprecated `terraform_docs_replace` hook. + +1. You can pin a specific version of the wrapped tool per hook, independent of whatever is on your `$PATH` or baked into the Docker image. If that version isn't already cached locally, it's downloaded from the tool's GitHub releases on first use, then reused (without re-downloading) on every subsequent run. + + Config example: + + ```yaml + - id: terraform_tflint + args: + - --hook-config=--tool-version=0.50.0 + ``` + +2. The same `--tool-version` key also works for `terraform_validate`, `terraform_fmt` and `terraform_providers_lock`, which resolve their Terraform/OpenTofu binary through [`--tf-path`](#11-custom-terraform-binaries-and-opentofu-support) - by default it downloads/uses whichever of `terraform`/`opentofu` binary the rest of that precedence chain would otherwise have picked (`terraform`, unless it's missing from `$PATH` while `tofu` is present, in which case `opentofu`). To pick explicitly instead of relying on the auto-detection mechanism, set `--tf-path` to the literal value of `terraform`, `opentofu` or `tofu`: + + ```yaml + - id: terraform_validate + args: + - --hook-config=--tf-path=opentofu + - --hook-config=--tool-version=1.12.0 + ``` + + `--tf-path` and `--tool-version` either can be set on its own, or combined: + + | `--tf-path` | `--tool-version` | Behavior | + | ------------------------------------------ | ---------------- | -------------------------------------------------------------------------------------------------------------------------------------------- | + | set to a literal path/binary name | unset | Use that local binary as-is (normal [`--tf-path`](#11-custom-terraform-binaries-and-opentofu-support) behavior); `--tool-version` is ignored | + | unset | set | Auto-detect `terraform`/`opentofu` as described above, then download/use that tool at the pinned version | + | set to `terraform`, `opentofu`, or `tofu` | set | Download/use the pinned version of whichever of the two was selected | + | set to anything else (e.g. an actual path) | set | Error - combined with `--tool-version`, `--tf-path` only accepts `terraform`, `opentofu`, or `tofu` | + | unset | unset | Falls back to the normal [`--tf-path`](#11-custom-terraform-binaries-and-opentofu-support) precedence chain (env vars, then `$PATH`) | + +> [!TIP] +> 3. Since this resolves to a version-specific cached binary rather than mutating `$PATH`, the same hook can be listed multiple times with different pinned versions, e.g. to test compatibility across tool versions in one run: +> +> ```yaml +> - id: terraform_tflint +> args: +> - --hook-config=--tool-version=0.50.0 +> - id: terraform_tflint +> args: +> - --hook-config=--tool-version=0.55.0 +> ``` + +4. By default, if a different version of the tool is already on `$PATH`, the pinned version still wins (with a warning message logged) based on `--hook-config=--tool-version-mode=strict`. Set it to `prefer-local` (as opposite to `strict`) to invert that: if the tool already resolves via `$PATH`, this local binary is used as-is and no download is attempted; the pinned version is only downloaded/used as a fallback when nothing is found locally. + + ```yaml + - id: terraform_tflint + args: + - --hook-config=--tool-version=0.50.0 + - --hook-config=--tool-version-mode=prefer-local + ``` + +5. By default, downloaded binaries are cached under `$XDG_CACHE_HOME/pre-commit-terraform/` (or `$HOME/.cache/pre-commit-terraform/` if `XDG_CACHE_HOME` is unset). Override this location with the `PCT_TOOL_CACHE_DIR` environment variable - see [Mount tools cache directory](#mount-tools-cache-directory) for the Docker case. + +6. If a `GITHUB_TOKEN` environment variable is set, it's inherited automatically to authenticate GitHub API requests made during version resolution, the same way it already is utilized for [building your own Docker image](#docker-usage). + +#### Keeping pinned versions up-to-date using Renovate + +Neither Renovate's built-in [`pre-commit` manager](https://docs.renovatebot.com/modules/manager/pre-commit/) (which only understands `repo:`/`rev:` and `additional_dependencies` for Go/Node/Python) nor its `dockerfileVersions` preset can "look" inside hook's `args:`, so the `--tool-version` pin needs its own [`customManagers`](https://docs.renovatebot.com/modules/manager/regex/) entry in your own `renovate.json5` config file, using the same `# renovate: datasource=... depName=...` annotation convention commonly used for Dockerfile `ARG *_VERSION` pins: + +```yaml +- id: terraform_tflint + args: + # renovate: datasource=github-releases depName=terraform-linters/tflint + - --hook-config=--tool-version=0.50.0 +``` + +```json5 +{ + customManagers: [ + { + customType: "regex", + managerFilePatterns: ["/\\.pre-commit-config\\.ya?ml$/"], + matchStrings: [ + "# renovate: datasource=(?\\S+) depName=(?\\S+)\\s+-\\s+--hook-config=--tool-version=(?\\S+)", + ], + }, + ], +} +``` + ### Many hooks: Parallelism > All, except deprecated hooks: `checkov`, `terraform_docs_replace` and hooks which can't be paralleled this way: `infracost_breakdown`, `terraform_wrapper_module_for_each`. @@ -1295,6 +1381,26 @@ Finally, you can execute `docker run` with an additional volume mount so that th docker run --rm -e "USERID=$(id -u):$(id -g)" -v ~/.netrc:/root/.netrc -v $(pwd):/lint -w /lint ghcr.io/antonbabenko/pre-commit-terraform:latest run -a ``` +### Mount tools cache directory + +A container's own filesystem is discarded after `docker run` exits, so a version downloaded via [`--tool-version`](#most-hooks-pin-a-specific-tool-version) would otherwise be re-downloaded on every single run. Mount the cache directory as a volume to persist it across runs, the same way you would for [`TF_PLUGIN_CACHE_DIR`](https://developer.hashicorp.com/terraform/cli/config/config-file#provider-plugin-cache): + +> [!IMPORTANT] +> With a non-root `USERID` (the [recommended](#4-run) way to run the image), do not mount the cache under `/root/...`: the container switches to that UID/GID via `su-exec`, which neither grants it permission to traverse `/root` nor changes `$HOME`, so the mount would be unreachable and pinned-tool resolution would fail instead of using it. Point `PCT_TOOL_CACHE_DIR` at a container path any UID can write to instead (e.g. under `/tmp`), and pre-create the host directory so Docker doesn't auto-create it as `root`-owned on first mount. + +```bash +TAG=latest +mkdir -p ~/.cache/pre-commit-terraform +docker run \ + -e "USERID=$(id -u):$(id -g)" \ + -e PCT_TOOL_CACHE_DIR=/tmp/pre-commit-terraform-cache \ + -v ~/.cache/pre-commit-terraform:/tmp/pre-commit-terraform-cache \ + -v $(pwd):/lint -w /lint \ + ghcr.io/antonbabenko/pre-commit-terraform:$TAG run -a +``` + +If you set `PCT_TOOL_CACHE_DIR` to a different custom location, mount the host cache directory at that same container path instead. + ## GitHub Actions You can use this hook in your GitHub Actions workflow together with [pre-commit](https://pre-commit.com). To easy up diff --git a/hatch.toml b/hatch.toml index 1cb850f63..a741fec1a 100644 --- a/hatch.toml +++ b/hatch.toml @@ -2,8 +2,11 @@ include = [ '.codecov.yml', '.coveragerc', + 'hooks/', + 'lib_getopt', 'src/', 'tests/', + 'tools/install/', 'pytest.ini', 'tox.ini', ] diff --git a/hooks/_common.sh b/hooks/_common.sh index e59a22b95..d7209731c 100644 --- a/hooks/_common.sh +++ b/hooks/_common.sh @@ -309,14 +309,21 @@ function common::get_cpu_num { # 3. Complete hook execution and return exit code # Arguments: # hook_id (string) hook ID, see `- id` for details in .pre-commit-hooks.yaml file +# tool_path (string) path to the wrapped tool's binary, already resolved +# by the calling hook via `common::resolve_tool_path` +# (empty for hooks with no resolvable binary, e.g. checkov). +# Deliberately NOT resolved here: each hook resolves once in `main`, +# because resolution logs a NOTE and downloads on a cache miss, so +# doing it again here would duplicate both # args_array_length (integer) Count of arguments in args array. # args (array) arguments that configure wrapped tool behavior # files (array) filenames to check ####################################################################### function common::per_dir_hook { local -r hook_id="$1" - local -i args_array_length=$2 - shift 2 + local -r tool_path="$2" + local -i args_array_length=$3 + shift 3 local -a args=() # Expand args to a true array. # Based on https://stackoverflow.com/a/10953834 @@ -328,13 +335,11 @@ function common::per_dir_hook { # despite there's only one positional ARG left local -a -r files=("$@") - local -r tf_path=$(common::get_tf_binary_path) - # check is (optional) function defined if [ "$(type -t run_hook_on_whole_repo)" == function ] && # check is hook run via `pre-commit run --all` common::is_hook_run_on_whole_repo "$hook_id" "${files[@]}"; then - run_hook_on_whole_repo "${args[@]}" + run_hook_on_whole_repo "$tool_path" "${args[@]}" exit 0 fi @@ -430,7 +435,7 @@ function common::per_dir_hook { pushd "$dir_path" > /dev/null fi - per_dir_hook_unique_part "$dir_path" "$change_dir_in_unique_part" "$parallelism_disabled" "$tf_path" "${args[@]}" + per_dir_hook_unique_part "$dir_path" "$change_dir_in_unique_part" "$parallelism_disabled" "$tool_path" "${args[@]}" } & pids+=("$!") @@ -492,9 +497,281 @@ function common::colorify { echo -e "${COLOR}${TEXT}${RESET}" >&2 } +####################################################################### +# Look up a single `--hook-config=--key=value` entry's value. +# Globals: +# HOOK_CONFIG (array) arguments that configure hook behavior +# Arguments: +# key (string) hook-config key to look up, including its leading `--` +# (e.g. "--tool-version") +# Outputs: +# Prints the value if the key is present in $HOOK_CONFIG, prints +# nothing otherwise +####################################################################### +function common::get_hook_config_value { + local -r key="$1" + local config value + + for config in "${HOOK_CONFIG[@]}"; do + if [[ $config == "$key"=* ]]; then + value=${config#*=} + value=${value%;} + break + fi + done + + echo "$value" +} + +####################################################################### +# Detect current OS/architecture using the same naming convention +# `tools/install/*.sh` expects (normally provided automatically by +# Docker buildx as TARGETOS/TARGETARCH build args; outside of a Docker +# build they don't exist and must be derived here instead). +# Globals (init and populate): +# TARGETOS (string) +# TARGETARCH (string) +####################################################################### +function common::detect_os_arch { + TARGETOS="$(uname -s | tr '[:upper:]' '[:lower:]')" + TARGETARCH="$(uname -m)" + + case "$TARGETARCH" in + x86_64) TARGETARCH="amd64" ;; + aarch64 | arm64) TARGETARCH="arm64" ;; + esac + + export TARGETOS TARGETARCH +} + +####################################################################### +# Download and install one tool version into a private, per-process +# temp directory, then atomically publish the resulting binary into +# the shared cache - concurrency-safe against other processes +# populating the same (tool, version) entry at the same time. +# Globals: +# GITHUB_TOKEN - forwarded automatically; read directly by the +# invoked installer script +# Arguments: +# tool_name (string) tool name, matching a `tools/install/.sh` +# file and its expected `${TOOL^^}_VERSION` environment variable +# version (string) exact version to install +# installer_script (string) absolute path to the installer to invoke +# env_var_name (string) env var the installer reads its version from +# cache_dir (string) final, shared cache dir for this (tool, version). +# Must already exist. +# cached_bin (string) expected absolute path to the resolved binary +# Outputs: +# Returns 0 once `cached_bin` exists, ours or a race winner's. +# Returns 1 with an error message if the install itself failed. +####################################################################### +function common::populate_tool_cache { + local -r tool_name="$1" + local -r version="$2" + local -r installer_script="$3" + local -r env_var_name="$4" + local -r cache_dir="$5" + local -r cached_bin="$6" + + if [[ -x $cached_bin ]]; then + return 0 + fi + + local tmp_dir + tmp_dir=$(mktemp -d "${cache_dir}.XXXXXXXXXX") || { + common::colorify "red" "ERROR: Failed to create a temp directory for '$tool_name' version '$version'." + return 1 + } + + # Redirect the installer's own stdout to stderr: this is a plain + # function call, not a "$(...)" capture, so anything printed here + # would flow straight through to `common::resolve_tool_path`'s own + # stdout - the resolved path, captured via "$(...)" by every caller + # of *that* function - and installers like terraform.sh/tflint.sh + # call bare `unzip` (no `-q`), which prints "Archive: ... inflating: + # ..." to stdout by default. + if ! ( + cd "$tmp_dir" || exit 1 + export "$env_var_name=$version" + "$installer_script" 1>&2 + ); then + common::colorify "red" "ERROR: Failed to download '$tool_name' version '$version' via '$installer_script'." + rm -rf "$tmp_dir" + return 1 + fi + + # `ln` (hard link, no `-f`) fails with EEXIST instead of silently + # replacing an existing destination - unlike `mv`, which would + # clobber a binary a sibling process may already be running. + # `tmp_dir` is a sibling of `cache_dir` (both under the same parent), + # so this is guaranteed to stay on one filesystem. + if ! ln "$tmp_dir/$(basename "$cached_bin")" "$cached_bin" 2> /dev/null; then + rm -rf "$tmp_dir" + # Lost the race - the winner's copy is equally valid. + [[ -x $cached_bin ]] && return 0 + common::colorify "red" "ERROR: Failed to update '$cached_bin' with '$tool_name' version '$version'." + return 1 + fi + + rm -rf "$tmp_dir" +} + +####################################################################### +# Resolve a specific version of a wrapped tool's binary, downloading +# and caching it on demand if it isn't already cached. +# +# Reuses the existing `tools/install/.sh` installer scripts +# instead of re-implementing per-tool download logic. +# Requires a downloadable release binary to resolve. +# +# Environment variables: +# PCT_TOOL_CACHE_DIR (string) if set, used as the complete cache +# root path as-is +# XDG_CACHE_HOME (string) if set (and PCT_TOOL_CACHE_DIR is not), +# "$XDG_CACHE_HOME/pre-commit-terraform" is used as the cache root +# GITHUB_TOKEN (string) forwarded automatically, since it's read +# directly by the invoked installer script +# Arguments: +# tool (string) tool name: +# - matching a `tools/install/.sh` file and its expected +# `${TOOL^^}_VERSION` environment variable name; +# - "tf" for Terraform/OpenTofu, resolved via `common::get_tf_binary_path` +# - empty for hooks with no resolvable binary (e.g. checkov) +# version (string) exact version requested (e.g. "1.7.5"), or empty +# if no `--tool-version` was requested +# Outputs: +# Prints the absolute path to the resolved binary, or the bare $tool +# name unchanged if no version was requested (empty string if $tool +# itself is also empty). If a download is attempted and fails - exit +# 1 with an error message. +####################################################################### +function common::resolve_tool_path { + local -r tool_name="$1" + local -r version="$2" + + # + # Check if configuration is valid + # + + # No resolvable tool name (e.g. checkov, which is pip-distributed); + # keeps "--tool-version" a documented no-op for it instead of erroring on an empty tool name. + [[ ! $tool_name ]] && return + + # "tf" is a placeholder, not a real tool. Delegate to + # `common::get_tf_binary_path`, which applies the extra precedence rules + # (--tf-path, PCT_TFPATH/TERRAGRUNT_TFPATH, terraform-vs-opentofu choice) + # then calls back here with the concrete name - which no longer matches + # "tf", so it falls through below instead of recursing. + if [[ $tool_name == "tf" ]]; then + common::get_tf_binary_path "$version" + return + fi + + if [[ ! $version ]]; then + # Check if the tool discoverable in the system's PATH + if ! command -v "$tool_name" > /dev/null; then + common::colorify "red" \ + "ERROR: '$tool_name' is required by '$HOOK_ID' pre-commit hook but it is not discoverable in the system's PATH.\n" \ + "Since '--hook-config=--tool-version=…' was not specified, no version resolution was attempted.\n\n" \ + "Please install '$tool_name' manually or specify in .pre-commit-config.yaml a version to download and cache via:\n" \ + "args:\n" \ + " - --hook-config=--tool-version=" + exit 1 + fi + + echo "$tool_name" + return + fi + + # + # Choose whether to prefer the local $PATH version of a tool over a requested version, if both exist. + # + local -r tool_version_mode=$(common::get_hook_config_value "--tool-version-mode") + + # Reject unknown values instead of silently treating them as "strict": + # a typo like "prefer_local" would otherwise do the exact opposite of + # what the user asked for, with no indication of why. + case "$tool_version_mode" in + "" | strict | prefer-local) ;; + *) + common::colorify "red" \ + "ERROR: '--tool-version-mode=$tool_version_mode' is not a valid value.\n" \ + "'--tool-version-mode=' must be either 'strict' (default) or 'prefer-local'." + exit 1 + ;; + esac + + # opentofu.sh renames "opentofu" back to "tofu" (tools/install/opentofu.sh); + # $PATH lookups below must use that real binary name, not $tool_name. + local resolved_bin_name="$tool_name" + [[ $tool_name == "opentofu" ]] && resolved_bin_name="tofu" + + if command -v "$resolved_bin_name" &> /dev/null; then + if [[ $tool_version_mode == "prefer-local" ]]; then + common::colorify "green" \ + "NOTE: version '$version' was requested for '$tool_name', but '--tool-version-mode=prefer-local' " \ + "is set and '$resolved_bin_name' is already found on \$PATH - using that instead." + command -v "$resolved_bin_name" + return + fi + + common::colorify "green" \ + "NOTE: The requested '$tool_name' version '$version' will be used instead of whatever is on \$PATH." + fi + + # + # Check if the requested version is already cached + # + + # Cache key needs OS/arch too - a cache dir shared across platforms + # (e.g. Docker volume mount) would else serve an incompatible binary. + common::detect_os_arch + + local -r cache_root="${PCT_TOOL_CACHE_DIR:-${XDG_CACHE_HOME:-$HOME/.cache}/pre-commit-terraform}" + local -r cache_dir="$cache_root/$tool_name/$version/${TARGETOS}_${TARGETARCH}" + local -r cached_bin="$cache_dir/$resolved_bin_name" + + if [[ -x $cached_bin ]]; then + echo "$cached_bin" + return + fi + + # + # Download and cache the requested version + # + + local -r script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" + local -r installer_script="$script_dir/../tools/install/${tool_name}.sh" + + if [[ ! -f $installer_script ]]; then + common::colorify "red" "ERROR: pinning a version is not supported for '$tool_name' (no installer found at '$installer_script')." + exit 1 + fi + + common::colorify "green" "Downloading '$tool_name' version '$version'..." + + local env_var_name="${tool_name//-/_}" + # `${var^^}` is bash 4+ only; macOS ships bash 3.2 by default. + env_var_name="$(tr '[:lower:]' '[:upper:]' <<< "${env_var_name}_VERSION")" + + mkdir -p "$cache_dir" + + common::populate_tool_cache "$tool_name" "$version" "$installer_script" "$env_var_name" "$cache_dir" "$cached_bin" || exit $? + + if [[ ! -x $cached_bin ]]; then + common::colorify "red" "ERROR: '$tool_name' installer completed but expected binary was not found at '$cached_bin'." + exit 1 + fi + + echo "$cached_bin" +} + ####################################################################### # Get Terraform/OpenTofu binary path # Allows user to set the path to custom Terraform or OpenTofu binary +# Arguments: +# tool_version (string) value of a requested `--tool-version` +# hook-config, or empty if none was requested # Globals (init and populate): # HOOK_CONFIG (array) arguments that configure hook behavior # PCT_TFPATH (string) user defined env var with path to Terraform/OpenTofu binary @@ -503,21 +780,45 @@ function common::colorify { # If failed - exit 1 with error message about missing Terraform/OpenTofu binary ####################################################################### function common::get_tf_binary_path { - local hook_config_tf_path + local -r tool_version="$1" - for config in "${HOOK_CONFIG[@]}"; do - if [[ $config == --tf-path=* ]]; then - hook_config_tf_path=${config#*=} - hook_config_tf_path=${hook_config_tf_path%;} - break - fi - done + local -r hook_config_tf_path=$(common::get_hook_config_value "--tf-path") - # direct hook config, has the highest precedence - if [[ $hook_config_tf_path ]]; then + # direct hook config, has the highest precedence - but only when NOT + # combined with --tool-version. When it IS also set, --tf-path is + # reinterpreted below as an explicit terraform/opentofu selector + # rather than a literal binary path. + if [[ $hook_config_tf_path && ! $tool_version ]]; then echo "$hook_config_tf_path" return + # '--hook-config=--tool-version=X.Y.Z': download/cache a pinned + # Terraform/OpenTofu version on demand. + elif [[ $tool_version ]]; then + local tf_tool + case "$hook_config_tf_path" in + terraform) + tf_tool="terraform" + ;; + opentofu | tofu) + tf_tool="opentofu" + ;; + "") + # Terraform preferred; opentofu only if terraform isn't on $PATH but tofu is). + tf_tool="terraform" + ! command -v terraform &> /dev/null && command -v tofu &> /dev/null && tf_tool="opentofu" + ;; + *) + common::colorify "red" \ + "ERROR: '--tf-path=$hook_config_tf_path' combined with '--tool-version' is not a valid value.\n" \ + "'--tf-path=' must be either 'terraform', 'opentofu'/'tofu', or unset." + exit 1 + ;; + esac + + common::resolve_tool_path "$tf_tool" "$tool_version" + return + # environment variable elif [[ $PCT_TFPATH ]]; then echo "$PCT_TFPATH" @@ -539,7 +840,12 @@ function common::get_tf_binary_path { return else - common::colorify "red" "Neither Terraform nor OpenTofu binary could be found. Please either set the \"--tf-path\" hook configuration argument, or set the \"PCT_TFPATH\" environment variable, or set the \"TERRAGRUNT_TFPATH\" environment variable, or install Terraform or OpenTofu globally." + common::colorify "red" \ + 'Neither Terraform nor OpenTofu binary could be found. Please do one of the following:\n' \ + '- set the "--tf-path" hook configuration argument, along with "--tool-version" (to download and cache) or without it (to use already installed one)\n' \ + '- set the "PCT_TFPATH" environment variable\n' \ + '- set the "TERRAGRUNT_TFPATH" environment variable\n' \ + '- install Terraform or OpenTofu yourself and run "pre-commit" again' exit 1 fi } @@ -639,11 +945,15 @@ function common::export_provided_env_vars { } ####################################################################### -# Check if the installed Terragrunt version is >=0.78.0 or not +# Check if the given Terragrunt binary's version is >=0.78.0 or not # # This function helps to determine which terragrunt subcomand to use # based on Terragrunt version # +# Arguments: +# tool_path (string) resolved path to the terragrunt binary to check +# (the actually resolved/pinned binary, NOT whatever's on $PATH - +# those can differ once --tool-version is in play) # Returns: # - 0 if version >= 0.78.0 # - 1 if version < 0.78.0 @@ -651,10 +961,11 @@ function common::export_provided_env_vars { ####################################################################### # TODO: Drop after May 2027. Two years to upgrade is more than enough. function common::terragrunt_version_ge_0.78 { + local -r tool_path="$1" local terragrunt_version # Extract version number (e.g., "terragrunt version v0.80.4" -> "0.80") - terragrunt_version=$(terragrunt --version 2> /dev/null | grep -oE '[0-9]+\.[0-9]+') + terragrunt_version=$("$tool_path" --version 2> /dev/null | grep -oE '[0-9]+\.[0-9]+') # If we can't parse version, default to newer command [[ ! $terragrunt_version ]] && return 0 diff --git a/hooks/infracost_breakdown.sh b/hooks/infracost_breakdown.sh index 14d34b82f..8c89acc6f 100755 --- a/hooks/infracost_breakdown.sh +++ b/hooks/infracost_breakdown.sh @@ -12,8 +12,11 @@ function main { common::parse_cmdline "$@" common::export_provided_env_vars "${ENV_VARS[@]}" common::parse_and_export_env_vars + + local -r tool_name="infracost" + # shellcheck disable=SC2153 # False positive - infracost_breakdown_ "${HOOK_CONFIG[*]}" "${ARGS[*]}" + infracost_breakdown_ "$tool_name" "${HOOK_CONFIG[*]}" "${ARGS[*]}" } ####################################################################### @@ -22,6 +25,7 @@ function main { # Environment variables: # PRE_COMMIT_COLOR (string) If set to `never` - do not colorize output # Arguments: +# tool_name (string) name of the wrapped tool, used to resolve its path # hook_config (string with array) arguments that configure hook behavior # args (string with array) arguments that configure wrapped tool behavior # Outputs: @@ -29,9 +33,15 @@ function main { # diff, summary about infracost check (non-supported resources etc.) ####################################################################### function infracost_breakdown_ { - local -r hook_config="$1" + local -r tool_name="$1" + local -r hook_config="$2" local args - read -r -a args <<< "$2" + read -r -a args <<< "$3" + + local -r tool_version=$(common::get_hook_config_value "--tool-version") + local tool_path + tool_path=$(common::resolve_tool_path "$tool_name" "$tool_version") || exit $? + readonly tool_path # Get hook settings IFS=";" read -r -a checks <<< "$hook_config" @@ -41,7 +51,7 @@ function infracost_breakdown_ { fi local RESULTS - RESULTS="$(infracost breakdown "${args[@]}" --format json)" + RESULTS="$("$tool_path" breakdown "${args[@]}" --format json)" local API_VERSION API_VERSION="$(jq -r .version <<< "$RESULTS")" diff --git a/hooks/terraform_checkov.sh b/hooks/terraform_checkov.sh index 39a93ef5f..368d055c7 100755 --- a/hooks/terraform_checkov.sh +++ b/hooks/terraform_checkov.sh @@ -18,8 +18,11 @@ function main { export ANSI_COLORS_DISABLED=true fi + # checkov is distributed via PyPi, not as GitHub release asset, so it has no + # resolved tool_path (see per_dir_hook_unique_part below) - pass an + # empty placeholder to satisfy common::per_dir_hook's signature. # shellcheck disable=SC2153 # ARGS is set in common::parse_cmdline - common::per_dir_hook "$HOOK_ID" "${#ARGS[@]}" "${ARGS[@]}" "${FILES[@]}" + common::per_dir_hook "$HOOK_ID" "" "${#ARGS[@]}" "${ARGS[@]}" "${FILES[@]}" } ####################################################################### @@ -33,7 +36,7 @@ function main { # Availability depends on hook. # parallelism_disabled (bool) if true - skip lock mechanism # args (array) arguments that configure wrapped tool behavior -# tf_path (string) PATH to Terraform/OpenTofu binary +# tool_path (string) unused - checkov has no resolved tool_path # Outputs: # If failed - print out hook checks status ####################################################################### @@ -45,7 +48,7 @@ function per_dir_hook_unique_part { # shellcheck disable=SC2034 # Unused var. local -r parallelism_disabled="$3" # shellcheck disable=SC2034 # Unused var. - local -r tf_path="$4" + local -r tool_path="$4" shift 4 local -a -r args=("$@") @@ -60,9 +63,13 @@ function per_dir_hook_unique_part { # Unique part of `common::per_dir_hook`. The function is executed one time # in the root git repo # Arguments: +# tool_path (string) unused - checkov has no resolved tool_path # args (array) arguments that configure wrapped tool behavior ####################################################################### function run_hook_on_whole_repo { + # shellcheck disable=SC2034 # Unused var. + local -r tool_path="$1" + shift local -a -r args=("$@") # pass the arguments to hook diff --git a/hooks/terraform_docs.sh b/hooks/terraform_docs.sh index 7644b5e4c..ff80dccae 100755 --- a/hooks/terraform_docs.sh +++ b/hooks/terraform_docs.sh @@ -24,8 +24,11 @@ function main { for i in "${!ARGS[@]}"; do ARGS[i]=${ARGS[i]/--config=/--config=$(pwd)\/} done + + local -r tool_name="terraform-docs" + # shellcheck disable=SC2153 # False positive - terraform_docs "${HOOK_CONFIG[*]}" "${ARGS[*]}" "${FILES[@]}" + terraform_docs "$tool_name" "${HOOK_CONFIG[*]}" "${ARGS[*]}" "${FILES[@]}" } ####################################################################### @@ -52,20 +55,22 @@ function replace_old_markers { # (depending on provided hook_config) terraform documentation in # Markdown # Arguments: +# tool_name (string) name of the wrapped tool, used to resolve its path # hook_config (string with array) arguments that configure hook behavior # args (string with array) arguments that configure wrapped tool behavior # files (array) filenames to check ####################################################################### function terraform_docs { - local -r hook_config="$1" - local args="$2" - shift 2 + local -r tool_name="$1" + local -r hook_config="$2" + local args="$3" + shift 3 local -a -r files=("$@") - if [[ ! $(command -v terraform-docs) ]]; then - echo "ERROR: terraform-docs is required by terraform_docs pre-commit hook but is not installed or in the system's PATH." - exit 1 - fi + local -r tool_version=$(common::get_hook_config_value "--tool-version") + local tool_path + tool_path=$(common::resolve_tool_path "$tool_name" "$tool_version") || exit $? + readonly tool_path local -a paths @@ -244,7 +249,7 @@ function terraform_docs { # shellcheck disable=SC2206 # Need to pass $tf_docs_formatter and $args as separate arguments, not as single string local tfdocs_cmd=( - terraform-docs + "$tool_path" --output-mode="$output_mode" --output-file="$output_file" $tf_docs_formatter diff --git a/hooks/terraform_fmt.sh b/hooks/terraform_fmt.sh index e8f974fd7..2ef8d2b0d 100755 --- a/hooks/terraform_fmt.sh +++ b/hooks/terraform_fmt.sh @@ -18,8 +18,15 @@ function main { ARGS+=("-no-color") fi + local -r tool_name="tf" # Will be resolved into real tool inside 'common::resolve_tool_path' + + local -r tool_version=$(common::get_hook_config_value "--tool-version") + local tool_path + tool_path=$(common::resolve_tool_path "$tool_name" "$tool_version") || exit $? + readonly tool_path + # shellcheck disable=SC2153 # False positive - common::per_dir_hook "$HOOK_ID" "${#ARGS[@]}" "${ARGS[@]}" "${FILES[@]}" + common::per_dir_hook "$HOOK_ID" "$tool_path" "${#ARGS[@]}" "${ARGS[@]}" "${FILES[@]}" } ####################################################################### @@ -33,7 +40,7 @@ function main { # Availability depends on hook. # parallelism_disabled (bool) if true - skip lock mechanism # args (array) arguments that configure wrapped tool behavior -# tf_path (string) PATH to Terraform/OpenTofu binary +# tool_path (string) PATH to Terraform/OpenTofu binary # Outputs: # If failed - print out hook checks status ####################################################################### @@ -44,12 +51,12 @@ function per_dir_hook_unique_part { local -r change_dir_in_unique_part="$2" # shellcheck disable=SC2034 # Unused var. local -r parallelism_disabled="$3" - local -r tf_path="$4" + local -r tool_path="$4" shift 4 local -a -r args=("$@") # pass the arguments to hook - "$tf_path" fmt "${args[@]}" + "$tool_path" fmt "${args[@]}" # return exit code to common::per_dir_hook local exit_code=$? diff --git a/hooks/terraform_providers_lock.sh b/hooks/terraform_providers_lock.sh index 32283cd33..372cc21e9 100755 --- a/hooks/terraform_providers_lock.sh +++ b/hooks/terraform_providers_lock.sh @@ -15,8 +15,15 @@ function main { common::parse_and_export_env_vars # JFYI: suppress color for `terraform providers lock` is N/A` + local -r tool_name="tf" # Will be resolved into real tool inside 'common::resolve_tool_path' + + local -r tool_version=$(common::get_hook_config_value "--tool-version") + local tool_path + tool_path=$(common::resolve_tool_path "$tool_name" "$tool_version") || exit $? + readonly tool_path + # shellcheck disable=SC2153 # False positive - common::per_dir_hook "$HOOK_ID" "${#ARGS[@]}" "${ARGS[@]}" "${FILES[@]}" + common::per_dir_hook "$HOOK_ID" "$tool_path" "${#ARGS[@]}" "${ARGS[@]}" "${FILES[@]}" } ####################################################################### @@ -87,7 +94,7 @@ function lockfile_contains_all_needed_sha { # Availability depends on hook. # parallelism_disabled (bool) if true - skip lock mechanism # args (array) arguments that configure wrapped tool behavior -# tf_path (string) PATH to Terraform/OpenTofu binary +# tool_path (string) PATH to Terraform/OpenTofu binary # Outputs: # If failed - print out hook checks status ####################################################################### @@ -96,7 +103,7 @@ function per_dir_hook_unique_part { # shellcheck disable=SC2034 # Unused var. local -r change_dir_in_unique_part="$2" local -r parallelism_disabled="$3" - local -r tf_path="$4" + local -r tool_path="$4" shift 4 local -a -r args=("$@") @@ -161,7 +168,7 @@ Please update your configuration." if [ ! "$mode" ]; then common::colorify "yellow" "DEPRECATION NOTICE: We introduced '--mode' flag for this hook. Check migration instructions at https://github.com/antonbabenko/pre-commit-terraform#terraform_providers_lock" - common::terraform_init "$tf_path providers lock" "$dir_path" "$parallelism_disabled" "$tf_path" || { + common::terraform_init "$tool_path providers lock" "$dir_path" "$parallelism_disabled" "$tool_path" || { exit_code=$? return $exit_code } @@ -192,7 +199,7 @@ All required platforms: ${platforms_names[*]}" #? Don't require `tf init` for providers, but required `tf init` for modules #? Mitigated by `function match_validate_errors` from terraform_validate hook # pass the arguments to hook - "$tf_path" providers lock "${args[@]}" + "$tool_path" providers lock "${args[@]}" exit_code=$? if [[ $exit_code -ne 0 ]]; then diff --git a/hooks/terraform_tflint.sh b/hooks/terraform_tflint.sh index 1c2439f0f..462e7cffd 100755 --- a/hooks/terraform_tflint.sh +++ b/hooks/terraform_tflint.sh @@ -16,11 +16,17 @@ function main { # JFYI: tflint color already suppressed via PRE_COMMIT_COLOR=never + local -r tool_name="tflint" + local -r tool_version=$(common::get_hook_config_value "--tool-version") + local tool_path + tool_path=$(common::resolve_tool_path "$tool_name" "$tool_version") || exit $? + readonly tool_path + # Run `tflint --init` for check that plugins installed. # It should run once on whole repo. { # shellcheck disable=SC2153 # ARGS is set in common::parse_cmdline - TFLINT_INIT=$(tflint --init "${ARGS[@]}" 2>&1) 2> /dev/null && + TFLINT_INIT=$("$tool_path" --init "${ARGS[@]}" 2>&1) 2> /dev/null && common::colorify "green" "Command 'tflint --init' successfully done:" && echo -e "${TFLINT_INIT}\n\n\n" } || { @@ -30,7 +36,7 @@ function main { return ${exit_code} } - common::per_dir_hook "$HOOK_ID" "${#ARGS[@]}" "${ARGS[@]}" "${FILES[@]}" + common::per_dir_hook "$HOOK_ID" "$tool_path" "${#ARGS[@]}" "${ARGS[@]}" "${FILES[@]}" } ####################################################################### @@ -44,7 +50,7 @@ function main { # Availability depends on hook. # parallelism_disabled (bool) if true - skip lock mechanism # args (array) arguments that configure wrapped tool behavior -# tf_path (string) PATH to Terraform/OpenTofu binary +# tool_path (string) resolved path to the wrapped tool's binary # Outputs: # If failed - print out hook checks status ####################################################################### @@ -53,8 +59,7 @@ function per_dir_hook_unique_part { local -r change_dir_in_unique_part="$2" # shellcheck disable=SC2034 # Unused var. local -r parallelism_disabled="$3" - # shellcheck disable=SC2034 # Unused var. - local -r tf_path="$4" + local -r tool_path="$4" shift 4 local -a -r args=("$@") @@ -63,7 +68,7 @@ function per_dir_hook_unique_part { fi # shellcheck disable=SC2086 # we need to remove the arg if its unset - TFLINT_OUTPUT=$(tflint ${dir_args:-} "${args[@]}" 2>&1) + TFLINT_OUTPUT=$("$tool_path" ${dir_args:-} "${args[@]}" 2>&1) local exit_code=$? if [ $exit_code -ne 0 ]; then diff --git a/hooks/terraform_tfsec.sh b/hooks/terraform_tfsec.sh index aa1f3fc8b..3d4e5d9ca 100755 --- a/hooks/terraform_tfsec.sh +++ b/hooks/terraform_tfsec.sh @@ -21,7 +21,14 @@ function main { common::colorify "yellow" "tfsec tool was deprecated, and replaced by trivy. You can check trivy hook here:" common::colorify "yellow" "https://github.com/antonbabenko/pre-commit-terraform/tree/master#terraform_trivy" - common::per_dir_hook "$HOOK_ID" "${#ARGS[@]}" "${ARGS[@]}" "${FILES[@]}" + local -r tool_name="tfsec" + + local -r tool_version=$(common::get_hook_config_value "--tool-version") + local tool_path + tool_path=$(common::resolve_tool_path "$tool_name" "$tool_version") || exit $? + readonly tool_path + + common::per_dir_hook "$HOOK_ID" "$tool_path" "${#ARGS[@]}" "${ARGS[@]}" "${FILES[@]}" } ####################################################################### @@ -35,7 +42,7 @@ function main { # Availability depends on hook. # parallelism_disabled (bool) if true - skip lock mechanism # args (array) arguments that configure wrapped tool behavior -# tf_path (string) PATH to Terraform/OpenTofu binary +# tool_path (string) resolved path to the wrapped tool's binary # Outputs: # If failed - print out hook checks status ####################################################################### @@ -46,13 +53,12 @@ function per_dir_hook_unique_part { local -r change_dir_in_unique_part="$2" # shellcheck disable=SC2034 # Unused var. local -r parallelism_disabled="$3" - # shellcheck disable=SC2034 # Unused var. - local -r tf_path="$4" + local -r tool_path="$4" shift 4 local -a -r args=("$@") # pass the arguments to hook - tfsec "${args[@]}" + "$tool_path" "${args[@]}" # return exit code to common::per_dir_hook local exit_code=$? @@ -63,13 +69,15 @@ function per_dir_hook_unique_part { # Unique part of `common::per_dir_hook`. The function is executed one time # in the root git repo # Arguments: +# tool_path (string) resolved path to the wrapped tool's binary # args (array) arguments that configure wrapped tool behavior ####################################################################### function run_hook_on_whole_repo { + local -r tool_path="$1" + shift local -a -r args=("$@") - # pass the arguments to hook - tfsec "$(pwd)" "${args[@]}" + "$tool_path" "$(pwd)" "${args[@]}" # return exit code to common::per_dir_hook local exit_code=$? diff --git a/hooks/terraform_trivy.sh b/hooks/terraform_trivy.sh index 86f56f013..4ee36bff3 100755 --- a/hooks/terraform_trivy.sh +++ b/hooks/terraform_trivy.sh @@ -13,8 +13,15 @@ function main { common::export_provided_env_vars "${ENV_VARS[@]}" common::parse_and_export_env_vars + local -r tool_name="trivy" + + local -r tool_version=$(common::get_hook_config_value "--tool-version") + local tool_path + tool_path=$(common::resolve_tool_path "$tool_name" "$tool_version") || exit $? + readonly tool_path + # shellcheck disable=SC2153 # ARGS is set in common::parse_cmdline - common::per_dir_hook "$HOOK_ID" "${#ARGS[@]}" "${ARGS[@]}" "${FILES[@]}" + common::per_dir_hook "$HOOK_ID" "$tool_path" "${#ARGS[@]}" "${ARGS[@]}" "${FILES[@]}" } ####################################################################### @@ -28,7 +35,7 @@ function main { # Availability depends on hook. # parallelism_disabled (bool) if true - skip lock mechanism # args (array) arguments that configure wrapped tool behavior -# tf_path (string) PATH to Terraform/OpenTofu binary +# tool_path (string) resolved path to the wrapped tool's binary # Outputs: # If failed - print out hook checks status ####################################################################### @@ -39,13 +46,12 @@ function per_dir_hook_unique_part { local -r change_dir_in_unique_part="$2" # shellcheck disable=SC2034 # Unused var. local -r parallelism_disabled="$3" - # shellcheck disable=SC2034 # Unused var. - local -r tf_path="$4" + local -r tool_path="$4" shift 4 local -a -r args=("$@") # pass the arguments to hook - trivy conf "$(pwd)" --exit-code=1 "${args[@]}" + "$tool_path" conf "$(pwd)" --exit-code=1 "${args[@]}" # return exit code to common::per_dir_hook local exit_code=$? @@ -56,13 +62,15 @@ function per_dir_hook_unique_part { # Unique part of `common::per_dir_hook`. The function is executed one time # in the root git repo # Arguments: +# tool_path (string) resolved path to the wrapped tool's binary # args (array) arguments that configure wrapped tool behavior ####################################################################### function run_hook_on_whole_repo { + local -r tool_path="$1" + shift local -a -r args=("$@") - # pass the arguments to hook - trivy conf "$(pwd)" --exit-code=1 "${args[@]}" + "$tool_path" conf "$(pwd)" --exit-code=1 "${args[@]}" # return exit code to common::per_dir_hook local exit_code=$? diff --git a/hooks/terraform_validate.sh b/hooks/terraform_validate.sh index 21b635b84..c6b36c500 100755 --- a/hooks/terraform_validate.sh +++ b/hooks/terraform_validate.sh @@ -21,8 +21,16 @@ function main { if [ "$PRE_COMMIT_COLOR" = "never" ]; then ARGS+=("-no-color") fi + + local -r tool_name="tf" # Will be resolved into real tool inside 'common::resolve_tool_path' + + local -r tool_version=$(common::get_hook_config_value "--tool-version") + local tool_path + tool_path=$(common::resolve_tool_path "$tool_name" "$tool_version") || exit $? + readonly tool_path + # shellcheck disable=SC2153 # False positive - common::per_dir_hook "$HOOK_ID" "${#ARGS[@]}" "${ARGS[@]}" "${FILES[@]}" + common::per_dir_hook "$HOOK_ID" "$tool_path" "${#ARGS[@]}" "${ARGS[@]}" "${FILES[@]}" } ####################################################################### @@ -82,7 +90,7 @@ function match_validate_errors { # Availability depends on hook. # parallelism_disabled (bool) if true - skip lock mechanism # args (array) arguments that configure wrapped tool behavior -# tf_path (string) PATH to Terraform/OpenTofu binary +# tool_path (string) PATH to Terraform/OpenTofu binary # Outputs: # If failed - print out hook checks status ####################################################################### @@ -91,7 +99,7 @@ function per_dir_hook_unique_part { # shellcheck disable=SC2034 # Unused var. local -r change_dir_in_unique_part="$2" local -r parallelism_disabled="$3" - local -r tf_path="$4" + local -r tool_path="$4" shift 4 local -a -r args=("$@") @@ -123,25 +131,25 @@ function per_dir_hook_unique_part { # First try `terraform validate` with the hope that all deps are # pre-installed. That is needed for cases when `.terraform/modules` # or `.terraform/providers` missed AND that is expected. - "$tf_path" validate "${args[@]}" &> /dev/null && { + "$tool_path" validate "${args[@]}" &> /dev/null && { exit_code=$? return $exit_code } # In case `terraform validate` failed to execute # - check is simple `terraform init` will help - common::terraform_init "$tf_path validate" "$dir_path" "$parallelism_disabled" "$tf_path" || { + common::terraform_init "$tool_path validate" "$dir_path" "$parallelism_disabled" "$tool_path" || { exit_code=$? return $exit_code } if [ "$retry_once_with_cleanup" != "true" ]; then # terraform validate only - validate_output=$("$tf_path" validate "${args[@]}" 2>&1) + validate_output=$("$tool_path" validate "${args[@]}" 2>&1) exit_code=$? else # terraform validate, plus capture possible errors - validate_output=$("$tf_path" validate -json "${args[@]}" 2>&1) + validate_output=$("$tool_path" validate -json "${args[@]}" 2>&1) exit_code=$? # Match specific validation errors @@ -159,12 +167,12 @@ function per_dir_hook_unique_part { common::colorify "yellow" "Re-validating: $dir_path" - common::terraform_init "$tf_path validate" "$dir_path" "$parallelism_disabled" "$tf_path" || { + common::terraform_init "$tool_path validate" "$dir_path" "$parallelism_disabled" "$tool_path" || { exit_code=$? return $exit_code } - validate_output=$("$tf_path" validate "${args[@]}" 2>&1) + validate_output=$("$tool_path" validate "${args[@]}" 2>&1) exit_code=$? fi fi diff --git a/hooks/terraform_wrapper_module_for_each.sh b/hooks/terraform_wrapper_module_for_each.sh index 9d47cf599..037f9a77d 100755 --- a/hooks/terraform_wrapper_module_for_each.sh +++ b/hooks/terraform_wrapper_module_for_each.sh @@ -14,10 +14,15 @@ function main { common::parse_and_export_env_vars # JFYI: suppress color for `hcledit` is N/A` - check_dependencies + local -r tool_name="hcledit" + + local -r tool_version=$(common::get_hook_config_value "--tool-version") + local tool_path + tool_path=$(common::resolve_tool_path "$tool_name" "$tool_version") || exit $? + readonly tool_path # shellcheck disable=SC2153 # False positive - terraform_module_wrapper_ "${ARGS[*]}" + terraform_module_wrapper_ "$tool_path" "${ARGS[*]}" } readonly CONTENT_MAIN_TF='module "wrapper" {}' @@ -143,8 +148,9 @@ inputs = { ```' function terraform_module_wrapper_ { + local -r tool_path="$1" local args - read -r -a args <<< "$1" + read -r -a args <<< "$2" local root_dir local module_dir="" # values: empty (default), "." (just root module), or a single module (e.g. "modules/iam-user") @@ -321,14 +327,14 @@ EOF # Get names of module variables in all terraform files # shellcheck disable=SC2207 - module_vars=($(echo "$all_tf_content" | hcledit block list | { grep "^variable\." | cut -d'.' -f 2 | sort || true; })) + module_vars=($(echo "$all_tf_content" | "$tool_path" block list | { grep "^variable\." | cut -d'.' -f 2 | sort || true; })) # Get names of module outputs in all terraform files # shellcheck disable=SC2207 - module_outputs=($(echo "$all_tf_content" | hcledit block list | { grep "^output\." | cut -d'.' -f 2 || true; })) + module_outputs=($(echo "$all_tf_content" | "$tool_path" block list | { grep "^output\." | cut -d'.' -f 2 || true; })) # Get names of module providers in all terraform files - module_providers=$(echo "$all_tf_content" | hcledit block list | { grep "^provider\." || true; }) + module_providers=$(echo "$all_tf_content" | "$tool_path" block list | { grep "^provider\." || true; }) if [[ $module_providers ]]; then common::colorify "yellow" "Skipping ${full_module_dir} because it is a legacy module which contains its own local provider configurations and so calls to it may not use the for_each argument." @@ -338,7 +344,7 @@ EOF # Looking for sensitive output local wrapper_output_sensitive="# sensitive = false # No sensitive module output found" for module_output in "${module_outputs[@]}"; do - module_output_sensitive=$(echo "$all_tf_content" | hcledit attribute get "output.${module_output}.sensitive") + module_output_sensitive=$(echo "$all_tf_content" | "$tool_path" attribute get "output.${module_output}.sensitive") # At least one output is sensitive - the wrapper's output should be sensitive, too if [[ "$module_output_sensitive" == "true" ]]; then @@ -348,15 +354,15 @@ EOF done # Create content of temporary main.tf file - hcledit attribute append module.wrapper.source "\"${relative_source_path}${module_dir}\"" --newline -f "$tmp_file_tf" -u - hcledit attribute append module.wrapper.for_each var.items --newline -f "$tmp_file_tf" -u + "$tool_path" attribute append module.wrapper.source "\"${relative_source_path}${module_dir}\"" --newline -f "$tmp_file_tf" -u + "$tool_path" attribute append module.wrapper.for_each var.items --newline -f "$tmp_file_tf" -u # Add newline before the first variable in a loop local newline="--newline" for module_var in "${module_vars[@]}"; do # Get default value for the variable - var_default=$(echo "$all_tf_content" | hcledit attribute get "variable.${module_var}.default") + var_default=$(echo "$all_tf_content" | "$tool_path" attribute get "variable.${module_var}.default") # Empty default means that the variable is required if [[ ! $var_default ]]; then @@ -377,7 +383,7 @@ EOF var_value="try(each.value.${module_var}, var.defaults.${module_var}, $var_default)" fi - hcledit attribute append "module.wrapper.${module_var}" "${var_value}" $newline -f "$tmp_file_tf" -u + "$tool_path" attribute append "module.wrapper.${module_var}" "${var_value}" $newline -f "$tmp_file_tf" -u newline="" done @@ -399,9 +405,9 @@ EOF cp "${full_module_dir}/versions.tf" "${output_dir}/versions.tf" # Don't propagate redundant `provider_meta` attributes # AWS-provider specific - hcledit attribute rm "terraform.provider_meta.${module_repo_provider}.user_agent" -f "${output_dir}/versions.tf" -u + "$tool_path" attribute rm "terraform.provider_meta.${module_repo_provider}.user_agent" -f "${output_dir}/versions.tf" -u # GCP-provider specific - hcledit attribute rm "terraform.provider_meta.${module_repo_provider}.module_name" -f "${output_dir}/versions.tf" -u + "$tool_path" attribute rm "terraform.provider_meta.${module_repo_provider}.module_name" -f "${output_dir}/versions.tf" -u else echo "$CONTENT_VERSIONS_TF" > "${output_dir}/versions.tf" fi @@ -427,14 +433,6 @@ EOF } -function check_dependencies { - if ! command -v hcledit > /dev/null; then - echo "ERROR: The binary 'hcledit' is required by this hook but is not installed or is not in the system's PATH." - echo "Check documentation: https://github.com/minamijoyo/hcledit" - exit 1 - fi -} - function create_tmp_file_tf { # Can't append extension for mktemp, so renaming instead tmp_file=$(mktemp "${TMPDIR:-/tmp}/tfwrapper-XXXXXXXXXX") diff --git a/hooks/terragrunt_fmt.sh b/hooks/terragrunt_fmt.sh index 2d6697ae3..e1a53f705 100755 --- a/hooks/terragrunt_fmt.sh +++ b/hooks/terragrunt_fmt.sh @@ -14,14 +14,20 @@ function main { common::parse_and_export_env_vars # JFYI: `terragrunt hcl format` color already suppressed via PRE_COMMIT_COLOR=never - if common::terragrunt_version_ge_0.78; then + local -r tool_name="terragrunt" + local -r tool_version=$(common::get_hook_config_value "--tool-version") + local tool_path + tool_path=$(common::resolve_tool_path "$tool_name" "$tool_version") || exit $? + readonly tool_path + + if common::terragrunt_version_ge_0.78 "$tool_path"; then local -ra SUBCOMMAND=(hcl format) else local -ra SUBCOMMAND=(hclfmt) fi # shellcheck disable=SC2153 # False positive - common::per_dir_hook "$HOOK_ID" "${#ARGS[@]}" "${ARGS[@]}" "${FILES[@]}" + common::per_dir_hook "$HOOK_ID" "$tool_path" "${#ARGS[@]}" "${ARGS[@]}" "${FILES[@]}" } ####################################################################### @@ -35,7 +41,7 @@ function main { # Availability depends on hook. # parallelism_disabled (bool) if true - skip lock mechanism # args (array) arguments that configure wrapped tool behavior -# tf_path (string) PATH to Terraform/OpenTofu binary +# tool_path (string) resolved path to the wrapped tool's binary # Outputs: # If failed - print out hook checks status ####################################################################### @@ -46,13 +52,12 @@ function per_dir_hook_unique_part { local -r change_dir_in_unique_part="$2" # shellcheck disable=SC2034 # Unused var. local -r parallelism_disabled="$3" - # shellcheck disable=SC2034 # Unused var. - local -r tf_path="$4" + local -r tool_path="$4" shift 4 local -a -r args=("$@") # pass the arguments to hook - terragrunt "${SUBCOMMAND[@]}" "${args[@]}" + "$tool_path" "${SUBCOMMAND[@]}" "${args[@]}" # return exit code to common::per_dir_hook local exit_code=$? @@ -63,13 +68,15 @@ function per_dir_hook_unique_part { # Unique part of `common::per_dir_hook`. The function is executed one time # in the root git repo # Arguments: +# tool_path (string) resolved path to the wrapped tool's binary # args (array) arguments that configure wrapped tool behavior ####################################################################### function run_hook_on_whole_repo { + local -r tool_path="$1" + shift local -a -r args=("$@") - # pass the arguments to hook - terragrunt "${SUBCOMMAND[@]}" "$(pwd)" "${args[@]}" + "$tool_path" "${SUBCOMMAND[@]}" "$(pwd)" "${args[@]}" # return exit code to common::per_dir_hook local exit_code=$? diff --git a/hooks/terragrunt_providers_lock.sh b/hooks/terragrunt_providers_lock.sh index cf6fb13ea..94e9e3c28 100755 --- a/hooks/terragrunt_providers_lock.sh +++ b/hooks/terragrunt_providers_lock.sh @@ -14,7 +14,13 @@ function main { common::parse_and_export_env_vars # JFYI: terragrunt providers lock color already suppressed via PRE_COMMIT_COLOR=never - if common::terragrunt_version_ge_0.78; then + local -r tool_name="terragrunt" + local -r tool_version=$(common::get_hook_config_value "--tool-version") + local tool_path + tool_path=$(common::resolve_tool_path "$tool_name" "$tool_version") || exit $? + readonly tool_path + + if common::terragrunt_version_ge_0.78 "$tool_path"; then local -ra SUBCOMMAND=(run -- providers lock) local -ra RUN_ALL_SUBCOMMAND=(run --all -- providers lock) else @@ -23,7 +29,7 @@ function main { fi # shellcheck disable=SC2153 # False positive - common::per_dir_hook "$HOOK_ID" "${#ARGS[@]}" "${ARGS[@]}" "${FILES[@]}" + common::per_dir_hook "$HOOK_ID" "$tool_path" "${#ARGS[@]}" "${ARGS[@]}" "${FILES[@]}" } ####################################################################### @@ -37,7 +43,7 @@ function main { # Availability depends on hook. # parallelism_disabled (bool) if true - skip lock mechanism # args (array) arguments that configure wrapped tool behavior -# tf_path (string) PATH to Terraform/OpenTofu binary +# tool_path (string) resolved path to the wrapped tool's binary # Outputs: # If failed - print out hook checks status ####################################################################### @@ -48,13 +54,12 @@ function per_dir_hook_unique_part { local -r change_dir_in_unique_part="$2" # shellcheck disable=SC2034 # Unused var. local -r parallelism_disabled="$3" - # shellcheck disable=SC2034 # Unused var. - local -r tf_path="$4" + local -r tool_path="$4" shift 4 local -a -r args=("$@") # pass the arguments to hook - terragrunt "${SUBCOMMAND[@]}" "${args[@]}" + "$tool_path" "${SUBCOMMAND[@]}" "${args[@]}" # return exit code to common::per_dir_hook local exit_code=$? @@ -65,13 +70,15 @@ function per_dir_hook_unique_part { # Unique part of `common::per_dir_hook`. The function is executed one time # in the root git repo # Arguments: +# tool_path (string) resolved path to the wrapped tool's binary # args (array) arguments that configure wrapped tool behavior ####################################################################### function run_hook_on_whole_repo { + local -r tool_path="$1" + shift local -a -r args=("$@") - # pass the arguments to hook - terragrunt "${RUN_ALL_SUBCOMMAND[@]}" "${args[@]}" + "$tool_path" "${RUN_ALL_SUBCOMMAND[@]}" "${args[@]}" # return exit code to common::per_dir_hook local exit_code=$? diff --git a/hooks/terragrunt_validate.sh b/hooks/terragrunt_validate.sh index a3f7e1d0a..7d31a3457 100755 --- a/hooks/terragrunt_validate.sh +++ b/hooks/terragrunt_validate.sh @@ -14,7 +14,13 @@ function main { common::parse_and_export_env_vars # JFYI: terragrunt validate color already suppressed via PRE_COMMIT_COLOR=never - if common::terragrunt_version_ge_0.78; then + local -r tool_name="terragrunt" + local -r tool_version=$(common::get_hook_config_value "--tool-version") + local tool_path + tool_path=$(common::resolve_tool_path "$tool_name" "$tool_version") || exit $? + readonly tool_path + + if common::terragrunt_version_ge_0.78 "$tool_path"; then local -ra SUBCOMMAND=(run -- validate) local -ra RUN_ALL_SUBCOMMAND=(run --all -- validate) else @@ -23,7 +29,7 @@ function main { fi # shellcheck disable=SC2153 # False positive - common::per_dir_hook "$HOOK_ID" "${#ARGS[@]}" "${ARGS[@]}" "${FILES[@]}" + common::per_dir_hook "$HOOK_ID" "$tool_path" "${#ARGS[@]}" "${ARGS[@]}" "${FILES[@]}" } ####################################################################### @@ -37,7 +43,7 @@ function main { # Availability depends on hook. # parallelism_disabled (bool) if true - skip lock mechanism # args (array) arguments that configure wrapped tool behavior -# tf_path (string) PATH to Terraform/OpenTofu binary +# tool_path (string) resolved path to the wrapped tool's binary # Outputs: # If failed - print out hook checks status ####################################################################### @@ -48,13 +54,12 @@ function per_dir_hook_unique_part { local -r change_dir_in_unique_part="$2" # shellcheck disable=SC2034 # Unused var. local -r parallelism_disabled="$3" - # shellcheck disable=SC2034 # Unused var. - local -r tf_path="$4" + local -r tool_path="$4" shift 4 local -a -r args=("$@") # pass the arguments to hook - terragrunt "${SUBCOMMAND[@]}" "${args[@]}" + "$tool_path" "${SUBCOMMAND[@]}" "${args[@]}" # return exit code to common::per_dir_hook local exit_code=$? @@ -65,13 +70,15 @@ function per_dir_hook_unique_part { # Unique part of `common::per_dir_hook`. The function is executed one time # in the root git repo # Arguments: +# tool_path (string) resolved path to the wrapped tool's binary # args (array) arguments that configure wrapped tool behavior ####################################################################### function run_hook_on_whole_repo { + local -r tool_path="$1" + shift local -a -r args=("$@") - # pass the arguments to hook - terragrunt "${RUN_ALL_SUBCOMMAND[@]}" "${args[@]}" + "$tool_path" "${RUN_ALL_SUBCOMMAND[@]}" "${args[@]}" # return exit code to common::per_dir_hook local exit_code=$? diff --git a/hooks/terragrunt_validate_inputs.sh b/hooks/terragrunt_validate_inputs.sh index 39e8e484e..c3a2fbc63 100755 --- a/hooks/terragrunt_validate_inputs.sh +++ b/hooks/terragrunt_validate_inputs.sh @@ -14,7 +14,13 @@ function main { common::parse_and_export_env_vars # JFYI: terragrunt validate color already suppressed via PRE_COMMIT_COLOR=never - if common::terragrunt_version_ge_0.78; then + local -r tool_name="terragrunt" + local -r tool_version=$(common::get_hook_config_value "--tool-version") + local tool_path + tool_path=$(common::resolve_tool_path "$tool_name" "$tool_version") || exit $? + readonly tool_path + + if common::terragrunt_version_ge_0.78 "$tool_path"; then local -ra SUBCOMMAND=(hcl validate --inputs) local -ra RUN_ALL_SUBCOMMAND=(run --all hcl validate --inputs) else @@ -23,7 +29,7 @@ function main { fi # shellcheck disable=SC2153 # False positive - common::per_dir_hook "$HOOK_ID" "${#ARGS[@]}" "${ARGS[@]}" "${FILES[@]}" + common::per_dir_hook "$HOOK_ID" "$tool_path" "${#ARGS[@]}" "${ARGS[@]}" "${FILES[@]}" } ####################################################################### @@ -37,7 +43,7 @@ function main { # Availability depends on hook. # parallelism_disabled (bool) if true - skip lock mechanism # args (array) arguments that configure wrapped tool behavior -# tf_path (string) PATH to Terraform/OpenTofu binary +# tool_path (string) resolved path to the wrapped tool's binary # Outputs: # If failed - print out hook checks status ####################################################################### @@ -48,13 +54,12 @@ function per_dir_hook_unique_part { local -r change_dir_in_unique_part="$2" # shellcheck disable=SC2034 # Unused var. local -r parallelism_disabled="$3" - # shellcheck disable=SC2034 # Unused var. - local -r tf_path="$4" + local -r tool_path="$4" shift 4 local -a -r args=("$@") # pass the arguments to hook - terragrunt "${SUBCOMMAND[@]}" "${args[@]}" + "$tool_path" "${SUBCOMMAND[@]}" "${args[@]}" # return exit code to common::per_dir_hook local exit_code=$? @@ -65,13 +70,15 @@ function per_dir_hook_unique_part { # Unique part of `common::per_dir_hook`. The function is executed one time # in the root git repo # Arguments: +# tool_path (string) resolved path to the wrapped tool's binary # args (array) arguments that configure wrapped tool behavior ####################################################################### function run_hook_on_whole_repo { + local -r tool_path="$1" + shift local -a -r args=("$@") - # pass the arguments to hook - terragrunt "${RUN_ALL_SUBCOMMAND[@]}" "${args[@]}" + "$tool_path" "${RUN_ALL_SUBCOMMAND[@]}" "${args[@]}" # return exit code to common::per_dir_hook local exit_code=$? diff --git a/hooks/terrascan.sh b/hooks/terrascan.sh index 277abca97..079b9390e 100755 --- a/hooks/terrascan.sh +++ b/hooks/terrascan.sh @@ -18,8 +18,15 @@ function main { done # JFYI: terrascan color already suppressed via PRE_COMMIT_COLOR=never + local -r tool_name="terrascan" + + local -r tool_version=$(common::get_hook_config_value "--tool-version") + local tool_path + tool_path=$(common::resolve_tool_path "$tool_name" "$tool_version") || exit $? + readonly tool_path + # shellcheck disable=SC2153 # False positive - common::per_dir_hook "$HOOK_ID" "${#ARGS[@]}" "${ARGS[@]}" "${FILES[@]}" + common::per_dir_hook "$HOOK_ID" "$tool_path" "${#ARGS[@]}" "${ARGS[@]}" "${FILES[@]}" } ####################################################################### @@ -33,7 +40,7 @@ function main { # Availability depends on hook. # parallelism_disabled (bool) if true - skip lock mechanism # args (array) arguments that configure wrapped tool behavior -# tf_path (string) PATH to Terraform/OpenTofu binary +# tool_path (string) resolved path to the wrapped tool's binary # Outputs: # If failed - print out hook checks status ####################################################################### @@ -44,13 +51,12 @@ function per_dir_hook_unique_part { local -r change_dir_in_unique_part="$2" # shellcheck disable=SC2034 # Unused var. local -r parallelism_disabled="$3" - # shellcheck disable=SC2034 # Unused var. - local -r tf_path="$4" + local -r tool_path="$4" shift 4 local -a -r args=("$@") # pass the arguments to hook - terrascan scan -i terraform "${args[@]}" + "$tool_path" scan -i terraform "${args[@]}" # return exit code to common::per_dir_hook local exit_code=$? @@ -61,13 +67,15 @@ function per_dir_hook_unique_part { # Unique part of `common::per_dir_hook`. The function is executed one time # in the root git repo # Arguments: +# tool_path (string) resolved path to the wrapped tool's binary # args (array) arguments that configure wrapped tool behavior ####################################################################### function run_hook_on_whole_repo { + local -r tool_path="$1" + shift local -a -r args=("$@") - # pass the arguments to hook - terrascan scan -i terraform "${args[@]}" + "$tool_path" scan -i terraform "${args[@]}" # return exit code to common::per_dir_hook local exit_code=$? diff --git a/hooks/tfupdate.sh b/hooks/tfupdate.sh index 1d474318b..b897ab103 100755 --- a/hooks/tfupdate.sh +++ b/hooks/tfupdate.sh @@ -25,8 +25,15 @@ function main { exit 1 fi + local -r tool_name="tfupdate" + + local -r tool_version=$(common::get_hook_config_value "--tool-version") + local tool_path + tool_path=$(common::resolve_tool_path "$tool_name" "$tool_version") || exit $? + readonly tool_path + # shellcheck disable=SC2153 # False positive - common::per_dir_hook "$HOOK_ID" "${#ARGS[@]}" "${ARGS[@]}" "${FILES[@]}" + common::per_dir_hook "$HOOK_ID" "$tool_path" "${#ARGS[@]}" "${ARGS[@]}" "${FILES[@]}" } ####################################################################### # Unique part of `common::per_dir_hook`. The function is executed in loop @@ -39,7 +46,7 @@ function main { # Availability depends on hook. # parallelism_disabled (bool) if true - skip lock mechanism # args (array) arguments that configure wrapped tool behavior -# tf_path (string) PATH to Terraform/OpenTofu binary +# tool_path (string) resolved path to the wrapped tool's binary # Outputs: # If failed - print out hook checks status ####################################################################### @@ -50,13 +57,12 @@ function per_dir_hook_unique_part { local -r change_dir_in_unique_part="$2" # shellcheck disable=SC2034 # Unused var. local -r parallelism_disabled="$3" - # shellcheck disable=SC2034 # Unused var. - local -r tf_path="$4" + local -r tool_path="$4" shift 4 local -a -r args=("$@") # pass the arguments to hook - tfupdate "${args[@]}" . + "$tool_path" "${args[@]}" . # return exit code to common::per_dir_hook local exit_code=$? @@ -67,13 +73,15 @@ function per_dir_hook_unique_part { # Unique part of `common::per_dir_hook`. The function is executed one time # in the root git repo # Arguments: +# tool_path (string) resolved path to the wrapped tool's binary # args (array) arguments that configure wrapped tool behavior ####################################################################### function run_hook_on_whole_repo { + local -r tool_path="$1" + shift local -a -r args=("$@") - # pass the arguments to hook - tfupdate "${args[@]}" --recursive . + "$tool_path" "${args[@]}" --recursive . # return exit code to common::per_dir_hook local exit_code=$? diff --git a/pytest.ini b/pytest.ini index b2bb788d6..f4b6dedea 100644 --- a/pytest.ini +++ b/pytest.ini @@ -57,6 +57,7 @@ junit_suite_name = awx_plugins_test_suite # A mapping of markers to their descriptions allowed in strict mode: markers = + network: test performs a real outbound network request (deselect with `-m "not network"`) minversion = 6.1.0 diff --git a/tests/pytest/tool_version_test.py b/tests/pytest/tool_version_test.py new file mode 100644 index 000000000..6d814d0e5 --- /dev/null +++ b/tests/pytest/tool_version_test.py @@ -0,0 +1,1259 @@ +"""Black-box regression tests for `--hook-config=--tool-version=` resolution. + +Every test here invokes a real hook script under `hooks/` as a subprocess +and asserts on its exit code, the cache directory's filesystem state, and +its output - never on a bash-internal function name - so these tests +require minimal changes if a hook is ever reimplemented in another +language. + +Each network-free test proves *which* binary the hook actually executed by +pre-populating the cache (or `$PATH`) with a stub that echoes a unique +marker, then asserting that marker in the hook's output. Asserting only on +a log line or on a file the test itself created cannot distinguish a +resolved stub from an unrelated real tool that happens to be installed on +the machine running the suite. + +NOTE: the module-level `pytestmark` skip leaves every function body below +unexecuted on Windows, and `covdefaults` gates coverage at 100%, so every +module-level `def` in this file needs a `# pragma: win32 no cover`. +""" + +from __future__ import annotations + +import os +import platform +import shutil +import stat +import subprocess +import sys +import tempfile +from pathlib import Path +from typing import NamedTuple + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[2] +HOOKS_DIR = REPO_ROOT / 'hooks' + + +def _current_os_arch( + *, + os_name: str | None = None, + machine: str | None = None, +) -> str: # pragma: win32 no cover + """Build the `_` cache-path segment for this host. + + Mirrors `common::detect_os_arch` (hooks/_common.sh) exactly. + + Args: + os_name: Override for `platform.system()`, for deterministic tests. + machine: Override for `platform.machine()`, for deterministic tests. + + Returns: + `_`, e.g. `linux_amd64` or `darwin_arm64`. + """ + os_name = (os_name or platform.system()).lower() + arch = machine or platform.machine() + if arch == 'x86_64': + arch = 'amd64' + elif arch in {'aarch64', 'arm64'}: + arch = 'arm64' + return f'{os_name}_{arch}' + + +# Both are resolved against *this* process' `PATH` by `execvp`, not against +# the `PATH` handed to the subprocess - which is what keeps a sandboxed +# `PATH` (see `_sandbox_path_dir`) from breaking the interpreter itself +# while still hiding wrapped CLI tools from the hook under test. +GIT = shutil.which('git') or 'git' +BASH = shutil.which('bash') or 'bash' + +# Diagnostic messages emitted by `common::colorify` in `hooks/_common.sh`. +# Named here so that a reworded message breaks in one place, and so that +# every assertion points at the implementation line it is coupled to. +# `common::colorify` writes to stderr, hence the stderr/stdout merge in +# `_run_hook` below. +DOWNLOAD_MSG = "Downloading '" # hooks/_common.sh:662 +STRICT_OVERRIDE_MSG = 'used instead of whatever is on $PATH' # :629 +PREFER_LOCAL_MSG = "'--tool-version-mode=prefer-local'" # :622 +NO_INSTALLER_MSG = 'no installer found' # :658 +MODE_INVALID_MSG = "'--tool-version-mode=prefer_local' is not a valid value" +TOOL_MISSING_MSG = "is not discoverable in the system's PATH" # :602 +TF_PATH_INVALID_MSG = 'not a valid value' # :739 +PARALLELISM_CAPPED_MSG = 'Observed Parallelism limit' # :409 + +PINNED_TFLINT_VERSION = '0.50.0' +# Old enough to sit past GitHub's unpaginated 30-release page. +OLD_TFLINT_VERSION = '0.44.1' +PINNED_TF_VERSION = '1.9.0' +# Any tool, any version: used where the cache is seeded by the test and +# the exact number is irrelevant to what is being asserted. +PINNED_ANY_VERSION = '9.9.9' + +HOOK_TIMEOUT_SECONDS = 300 +VERSION_CHECK_TIMEOUT_SECONDS = 60 + + +class _CachedTool(NamedTuple): + """Cache entry a hook is expected to resolve a pinned tool to.""" + + tool_dir: str + bin_name: str + + def stub_path( # pragma: win32 no cover + self, + cache_root: Path, + version: str, + ) -> Path: + """Build the `///_/` path. + + Args: + cache_root: Cache root the hook is pointed at. + version: Pinned version under test. + + Returns: + Path the hook must resolve the pinned tool to. + """ + return ( + cache_root + / self.tool_dir + / version + / _current_os_arch() + / self.bin_name + ) + + +class _HookWiring(NamedTuple): + """A hook plus the cached tool its `tool_name` must resolve to.""" + + hook_name: str + cached_tool: _CachedTool + extra_args: tuple[str, ...] + + +_SANDBOX_REQUIRED_TOOLS = ( + 'awk', + 'basename', + # The stubs written by `_write_stub` are `#!/usr/bin/env bash` scripts, + # so `env` must be able to find `bash` on the sandboxed `PATH` too. + 'bash', + 'cat', + 'cut', + 'dirname', + 'env', + 'grep', + 'head', + 'mkdir', + 'mktemp', + 'rm', + 'sed', + 'sort', + 'tail', + 'tr', + 'uname', + 'wc', + 'git', +) +_SANDBOX_OPTIONAL_TOOLS = ( + 'chmod', + 'cp', + 'curl', + 'date', + 'find', + 'getopt', + 'id', + 'ln', + 'ls', + 'mv', + 'nproc', + 'printf', + 'readlink', + 'realpath', + 'seq', + 'stat', + 'sysctl', + 'tar', + 'tee', + 'touch', + 'uniq', + 'unzip', + 'xargs', +) +# Probed in this order by `common::get_cpu_num` (hooks/_common.sh:235) to +# size parallelism. Each is absent on some platform this project supports, +# which is why they are optional above rather than required. +_CPU_COUNT_TOOLS = ('nproc', 'sysctl') + +pytestmark = pytest.mark.skipif( + sys.platform == 'win32', + reason=( + 'Hook-subprocess tests are skipped on Windows: this repository ' + 'does not fully support/guarantee Windows hook execution ' + '(see README.md / .github/CONTRIBUTING.md).' + ), +) + + +def _write_stub(path: Path, marker: str) -> None: # pragma: win32 no cover + """Write a fake, executable binary that prints a marker and exits 0.""" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + f'#!/usr/bin/env bash\necho "{marker}"\nexit 0\n', + encoding='utf-8', + ) + exec_bits = stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH + path.chmod(path.stat().st_mode | exec_bits) + + +def _sandbox_path_dir(base: Path) -> Path: # pragma: win32 no cover + """Build a `PATH` dir with coreutils but no wrapped CLI tool. + + None of the linked names collides with a tool any hook wraps + (`tflint`, `terraform`, `tofu`, ...), so a hook run against this + `PATH` can only resolve a wrapped tool through the cache - never + through an incidental real install on the host. + + Returns: + Path to the constructed directory, usable as a `PATH` entry. + """ + path_dir = base / 'sandbox-path' + path_dir.mkdir() + for tool in _SANDBOX_REQUIRED_TOOLS: + found = shutil.which(tool) + assert found is not None, f'{tool!r} not found on PATH' + (path_dir / tool).symlink_to(found) + for optional_tool in _SANDBOX_OPTIONAL_TOOLS: + optional_found = shutil.which(optional_tool) + # `no branch`: whether the False arc is ever taken depends on the + # host's `PATH`, not on anything a test controls - every CI runner + # resolves all of these, while e.g. a Linux box that keeps + # `/usr/sbin` off `PATH` resolves no `sysctl`. These are optional + # precisely because the hooks treat them as optional too, see + # `hooks/_common.sh:300`: `nproc || sysctl -n hw.ncpu || echo 1`. + if optional_found is not None: # pragma: no branch + (path_dir / optional_tool).symlink_to(optional_found) + return path_dir + + +def _hook_env( # pragma: win32 no cover + cache_env: dict[str, str], + path: str, +) -> dict[str, str]: + """Build a minimal, hermetic environment for a hook subprocess. + + Inheriting `os.environ` wholesale would let `PCT_TFPATH`, + `TERRAGRUNT_TFPATH`, `PRE_COMMIT_COLOR` or `TF_*` from the developer's + shell change what these tests resolve, so only an explicit allowlist + is forwarded. + + Args: + cache_env: Variables that decide the cache root - + `PCT_TOOL_CACHE_DIR`, or `XDG_CACHE_HOME`/`HOME` when + exercising the fallbacks. Merged last, so it can override + `HOME`. + path: Value for `PATH`. + + Returns: + The environment mapping to hand to `subprocess.run`. + """ + return { + 'PATH': path, + 'HOME': os.environ.get('HOME', ''), + 'TMPDIR': os.environ.get('TMPDIR', tempfile.gettempdir()), + 'LC_ALL': 'C', + # `common::colorify` wraps every message in ANSI escapes unless + # this is set; plain text keeps substring assertions honest. + 'PRE_COMMIT_COLOR': 'never', + # Read directly by `tools/install/_common.sh`. Forwarded as an + # empty string when absent, which that script treats as unset. + 'GITHUB_TOKEN': os.environ.get('GITHUB_TOKEN', ''), + **cache_env, + } + + +def _pct_cache_env( # pragma: win32 no cover + cache_dir: Path, +) -> dict[str, str]: + """Point the cache root straight at `cache_dir`. + + Returns: + A `PCT_TOOL_CACHE_DIR` mapping for `_hook_env`. + """ + return {'PCT_TOOL_CACHE_DIR': str(cache_dir)} + + +@pytest.fixture +def tmp_repo(tmp_path: Path) -> Path: # pragma: win32 no cover + """Create a minimal git repo with one tracked, provider-free `.tf` file. + + Returns: + Path to the created repo directory. + """ + repo = tmp_path / 'repo' + repo.mkdir() + # `--template=` disables Git's init templates: this project's own + # README tells users to set `init.templateDir` to a directory with + # pre-commit installed, which would otherwise install a real + # pre-commit hook into this throwaway repo and run it on commit. + subprocess.run( # noqa: S603 + (GIT, 'init', '--quiet', '--template=', '--initial-branch=main'), + cwd=repo, + check=True, + ) + subprocess.run( # noqa: S603 + (GIT, 'config', 'user.email', 't@t.com'), + cwd=repo, + check=True, + ) + subprocess.run( # noqa: S603 + (GIT, 'config', 'user.name', 't'), + cwd=repo, + check=True, + ) + (repo / 'a.tf').write_text( + 'variable "x" { default = 1 }\n', + encoding='utf-8', + ) + subprocess.run((GIT, 'add', 'a.tf'), cwd=repo, check=True) # noqa: S603 + subprocess.run( # noqa: S603 + (GIT, 'commit', '--quiet', '--no-verify', '-m', 'init'), + cwd=repo, + check=True, + ) + return repo + + +@pytest.fixture +def cache_dir(tmp_path: Path) -> Path: # pragma: win32 no cover + """Create a dedicated, empty cache root for one test. + + Returns: + Path to the created, empty cache root directory. + """ + cache = tmp_path / 'cache' + cache.mkdir() + return cache + + +def _run_hook( # pragma: win32 no cover + hook_name: str, + args: list[str], + *, + cwd: Path, + env: dict[str, str], +) -> subprocess.CompletedProcess[str]: + """Invoke a hook script on the single `a.tf` file of a temp repo. + + Returns: + The completed process, with stderr folded into `.stdout`. + """ + hook_path = HOOKS_DIR / hook_name + if not hook_path.is_file(): # pragma: no cover + # `hooks/` is not part of the wheel, only of the sdist, so a + # packaging regression must fail with a pointed message here + # instead of as a confusing assertion mismatch further down. + pytest.fail(f'Hook script not found: {hook_path}') + # `common::colorify` writes every diagnostic to stderr while a wrapped + # tool's own output goes to stdout, so the two are merged at the OS + # level to give each caller one ready-to-grep string. + return subprocess.run( # noqa: S603 + (BASH, str(hook_path), *args, '--', 'a.tf'), + cwd=cwd, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + check=False, + timeout=HOOK_TIMEOUT_SECONDS, + ) + + +def _run_concurrent_hooks( # pragma: win32 no cover + count: int, + hook_name: str, + args: list[str], + *, + cwd: Path, + env: dict[str, str], +) -> list[str]: + """Run `count` copies of a hook concurrently, on the same `a.tf`. + + Every copy is started before any of them is waited on, so all + `count` copies genuinely overlap instead of running one after + another. + + Returns: + Each process' merged stdout/stderr, in start order. + """ + hook_path = HOOKS_DIR / hook_name + if not hook_path.is_file(): # pragma: no cover + # `hooks/` is not part of the wheel, only of the sdist, so a + # packaging regression must fail with a pointed message here + # instead of as a confusing assertion mismatch further down. + pytest.fail(f'Hook script not found: {hook_path}') + processes = [ + subprocess.Popen( # noqa: S603 + (BASH, str(hook_path), *args, '--', 'a.tf'), + cwd=cwd, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + for _ in range(count) + ] + outputs = [ + proc.communicate(timeout=HOOK_TIMEOUT_SECONDS)[0] for proc in processes + ] + for output, proc in zip(outputs, processes, strict=True): + # 2 is tflint's own lint findings on the minimal fixture, not ours. + assert proc.returncode in {0, 2}, output + return outputs + + +@pytest.mark.parametrize( + ('machine', 'expected_arch'), + ( + pytest.param('x86_64', 'amd64', id='x86_64'), + pytest.param('aarch64', 'arm64', id='aarch64'), + pytest.param('arm64', 'arm64', id='arm64-native'), + pytest.param('armv7l', 'armv7l', id='unmapped-passthrough'), + ), +) +def test_current_os_arch_matches_bash_mapping( # pragma: win32 no cover + machine: str, + expected_arch: str, +) -> None: + """Check `_current_os_arch` mirrors `common::detect_os_arch`'s mapping. + + Uses an injected `machine`, not the host's real one, so every + branch runs regardless of which CI arch executes this suite. + """ + assert _current_os_arch(os_name='Linux', machine=machine) == ( + f'linux_{expected_arch}' + ) + + +def test_cache_hit_uses_cached_binary( # pragma: win32 no cover + tmp_repo: Path, + cache_dir: Path, + tmp_path: Path, +) -> None: + """Check a pre-populated cache entry is executed, with no download.""" + stub = ( + cache_dir + / 'tflint' + / PINNED_TFLINT_VERSION + / _current_os_arch() + / 'tflint' + ) + _write_stub(stub, 'CACHED_TFLINT') + + hook_run = _run_hook( + 'terraform_tflint.sh', + [f'--hook-config=--tool-version={PINNED_TFLINT_VERSION}'], + cwd=tmp_repo, + env=_hook_env( + _pct_cache_env(cache_dir), + str(_sandbox_path_dir(tmp_path)), + ), + ) + + combined = hook_run.stdout + assert 'CACHED_TFLINT' in combined, combined + assert DOWNLOAD_MSG not in combined, combined + assert hook_run.returncode == 0, combined + + +def test_strict_mode_prefers_pinned_over_path( # pragma: win32 no cover + tmp_repo: Path, + cache_dir: Path, + tmp_path: Path, +) -> None: + """Check that `strict` (default) mode executes the pinned version.""" + sandbox_path_dir = _sandbox_path_dir(tmp_path) + _write_stub(sandbox_path_dir / 'tflint', 'LOCAL_TFLINT') + + pinned = ( + cache_dir + / 'tflint' + / PINNED_TFLINT_VERSION + / _current_os_arch() + / 'tflint' + ) + _write_stub(pinned, 'PINNED_TFLINT') + + hook_run = _run_hook( + 'terraform_tflint.sh', + [f'--hook-config=--tool-version={PINNED_TFLINT_VERSION}'], + cwd=tmp_repo, + env=_hook_env(_pct_cache_env(cache_dir), str(sandbox_path_dir)), + ) + + combined = hook_run.stdout + assert 'PINNED_TFLINT' in combined, combined + assert 'LOCAL_TFLINT' not in combined, combined + assert STRICT_OVERRIDE_MSG in combined, combined + assert hook_run.returncode == 0, combined + + +def test_prefer_local_mode_uses_path_binary( # pragma: win32 no cover + tmp_repo: Path, + cache_dir: Path, + tmp_path: Path, +) -> None: + """Check `prefer-local` mode executes the `$PATH` binary instead.""" + sandbox_path_dir = _sandbox_path_dir(tmp_path) + _write_stub(sandbox_path_dir / 'tflint', 'LOCAL_TFLINT') + + pinned = ( + cache_dir + / 'tflint' + / PINNED_TFLINT_VERSION + / _current_os_arch() + / 'tflint' + ) + _write_stub(pinned, 'PINNED_TFLINT') + + hook_run = _run_hook( + 'terraform_tflint.sh', + [ + f'--hook-config=--tool-version={PINNED_TFLINT_VERSION}', + '--hook-config=--tool-version-mode=prefer-local', + ], + cwd=tmp_repo, + env=_hook_env(_pct_cache_env(cache_dir), str(sandbox_path_dir)), + ) + + combined = hook_run.stdout + assert 'LOCAL_TFLINT' in combined, combined + assert 'PINNED_TFLINT' not in combined, combined + assert PREFER_LOCAL_MSG in combined, combined + assert DOWNLOAD_MSG not in combined, combined + assert hook_run.returncode == 0, combined + + +@pytest.mark.parametrize( + ('hook_name', 'cached_tool'), + ( + pytest.param( + 'terraform_tflint.sh', + _CachedTool('tflint', 'tflint'), + id='tflint', + ), + pytest.param( + 'terragrunt_fmt.sh', + _CachedTool('terragrunt', 'terragrunt'), + id='terragrunt-fmt', + ), + ), +) +def test_tool_is_resolved_only_once( # pragma: win32 no cover + tmp_repo: Path, + cache_dir: Path, + tmp_path: Path, + hook_name: str, + cached_tool: _CachedTool, +) -> None: + """Check a hook resolves its pinned tool exactly once per invocation. + + Hooks that need the resolved path for a pre-flight command (`tflint + --init`, terragrunt's CLI-syntax version probe) used to resolve once + for that and again inside `common::per_dir_hook`, logging the NOTE + twice and entering the download path twice on a cache miss. + + Args: + tmp_repo: Temp git repo fixture. + cache_dir: Temp cache root fixture. + tmp_path: Temp dir fixture, used for the sandboxed `PATH`. + hook_name: Hook script filename under `hooks/`. + cached_tool: Cache entry the hook is expected to resolve to. + """ + sandbox_path_dir = _sandbox_path_dir(tmp_path) + _write_stub(sandbox_path_dir / cached_tool.bin_name, 'LOCAL_TOOL') + _write_stub( + cached_tool.stub_path(cache_dir, PINNED_ANY_VERSION), + 'PINNED_TOOL', + ) + + hook_run = _run_hook( + hook_name, + [f'--hook-config=--tool-version={PINNED_ANY_VERSION}'], + cwd=tmp_repo, + env=_hook_env(_pct_cache_env(cache_dir), str(sandbox_path_dir)), + ) + + combined = hook_run.stdout + assert combined.count(STRICT_OVERRIDE_MSG) == 1, combined + assert DOWNLOAD_MSG not in combined, combined + + +def test_rejects_invalid_tool_version_mode( # pragma: win32 no cover + tmp_repo: Path, + cache_dir: Path, + tmp_path: Path, +) -> None: + """Check an unrecognized `--tool-version-mode` is rejected, not ignored. + + Any value other than `strict`/`prefer-local` used to be treated as + `strict`, so a typo such as `prefer_local` silently did the opposite + of what was asked for. + """ + sandbox_path_dir = _sandbox_path_dir(tmp_path) + _write_stub(sandbox_path_dir / 'tflint', 'LOCAL_TFLINT') + pinned = ( + cache_dir + / 'tflint' + / PINNED_TFLINT_VERSION + / _current_os_arch() + / 'tflint' + ) + _write_stub(pinned, 'PINNED_TFLINT') + + hook_run = _run_hook( + 'terraform_tflint.sh', + [ + f'--hook-config=--tool-version={PINNED_TFLINT_VERSION}', + '--hook-config=--tool-version-mode=prefer_local', + ], + cwd=tmp_repo, + env=_hook_env(_pct_cache_env(cache_dir), str(sandbox_path_dir)), + ) + + combined = hook_run.stdout + assert hook_run.returncode != 0, combined + assert MODE_INVALID_MSG in combined, combined + assert 'LOCAL_TFLINT' not in combined, combined + assert 'PINNED_TFLINT' not in combined, combined + assert DOWNLOAD_MSG not in combined, combined + + +@pytest.mark.parametrize( + ('tf_path_value', 'cached_tool'), + ( + pytest.param( + 'terraform', + _CachedTool('terraform', 'terraform'), + id='terraform', + ), + pytest.param( + 'opentofu', + _CachedTool('opentofu', 'tofu'), + id='opentofu', + ), + pytest.param( + 'tofu', + _CachedTool('opentofu', 'tofu'), + id='tofu-alias', + ), + ), +) +def test_tf_path_selector( # pragma: win32 no cover + tmp_repo: Path, + cache_dir: Path, + tmp_path: Path, + tf_path_value: str, + cached_tool: _CachedTool, +) -> None: + """Check that `--tf-path=terraform|opentofu|tofu` selects the tool. + + Args: + tmp_repo: Temp git repo fixture. + cache_dir: Temp cache root fixture. + tmp_path: Temp dir fixture, used for the sandboxed `PATH`. + tf_path_value: The `--tf-path` value under test. + cached_tool: Cache entry the hook is expected to resolve to. + """ + _write_stub( + cached_tool.stub_path(cache_dir, PINNED_TF_VERSION), + 'SELECTED_TF', + ) + + hook_run = _run_hook( + 'terraform_fmt.sh', + [ + f'--hook-config=--tf-path={tf_path_value}', + f'--hook-config=--tool-version={PINNED_TF_VERSION}', + ], + cwd=tmp_repo, + env=_hook_env( + _pct_cache_env(cache_dir), + str(_sandbox_path_dir(tmp_path)), + ), + ) + + combined = hook_run.stdout + assert 'SELECTED_TF' in combined, combined + assert DOWNLOAD_MSG not in combined, combined + assert hook_run.returncode == 0, combined + + +@pytest.mark.parametrize( + ('path_bin_names', 'cached_tool'), + ( + pytest.param( + ('terraform',), + _CachedTool('terraform', 'terraform'), + id='only-terraform-on-path', + ), + pytest.param( + ('tofu',), + _CachedTool('opentofu', 'tofu'), + id='only-tofu-on-path', + ), + pytest.param( + ('terraform', 'tofu'), + _CachedTool('terraform', 'terraform'), + id='both-on-path-terraform-wins', + ), + ), +) +def test_tf_path_autodetect( # pragma: win32 no cover + tmp_repo: Path, + cache_dir: Path, + tmp_path: Path, + path_bin_names: tuple[str, ...], + cached_tool: _CachedTool, +) -> None: + """Check `--tool-version` with `--tf-path` unset auto-detects the tool. + + This is the documented default (README's `--tf-path`/`--tool-version` + matrix): Terraform wins whenever it is on `$PATH` - including when + `tofu` is there too - and OpenTofu is picked only when `terraform` is + absent while `tofu` is present. + + Args: + tmp_repo: Temp git repo fixture. + cache_dir: Temp cache root fixture. + tmp_path: Temp dir fixture, used for the sandboxed `PATH`. + path_bin_names: Terraform-ish binaries to put on `$PATH`. + cached_tool: Cache entry the hook is expected to resolve to. + """ + sandbox_path_dir = _sandbox_path_dir(tmp_path) + for path_bin_name in path_bin_names: + _write_stub(sandbox_path_dir / path_bin_name, 'ON_PATH_TF') + + _write_stub( + cached_tool.stub_path(cache_dir, PINNED_TF_VERSION), + 'AUTODETECTED_TF', + ) + + hook_run = _run_hook( + 'terraform_fmt.sh', + [f'--hook-config=--tool-version={PINNED_TF_VERSION}'], + cwd=tmp_repo, + env=_hook_env(_pct_cache_env(cache_dir), str(sandbox_path_dir)), + ) + + combined = hook_run.stdout + assert 'AUTODETECTED_TF' in combined, combined + assert 'ON_PATH_TF' not in combined, combined + assert DOWNLOAD_MSG not in combined, combined + assert hook_run.returncode == 0, combined + + +def test_tf_path_passthrough_without_tool_version( # pragma: win32 no cover + tmp_repo: Path, + cache_dir: Path, + tmp_path: Path, +) -> None: + """Check `--tf-path` alone still passes a literal path through as-is. + + Guards the pre-`--tool-version` `--tf-path` contract: without a pinned + version, an arbitrary path/binary name must be used verbatim and no + cache entry may be created. + """ + custom_binary = tmp_path / 'custom' / 'my-terraform' + _write_stub(custom_binary, 'CUSTOM_TF') + + hook_run = _run_hook( + 'terraform_fmt.sh', + [f'--hook-config=--tf-path={custom_binary}'], + cwd=tmp_repo, + env=_hook_env( + _pct_cache_env(cache_dir), + str(_sandbox_path_dir(tmp_path)), + ), + ) + + combined = hook_run.stdout + assert 'CUSTOM_TF' in combined, combined + assert DOWNLOAD_MSG not in combined, combined + assert not list(cache_dir.iterdir()), combined + assert hook_run.returncode == 0, combined + + +@pytest.mark.parametrize( + 'cache_root_var', + ( + pytest.param('XDG_CACHE_HOME', id='xdg-cache-home'), + pytest.param('HOME', id='home-dot-cache'), + ), +) +def test_cache_root_fallbacks( # pragma: win32 no cover + tmp_repo: Path, + tmp_path: Path, + cache_root_var: str, +) -> None: + """Check the cache root falls back when `PCT_TOOL_CACHE_DIR` is unset. + + `$XDG_CACHE_HOME/pre-commit-terraform` and + `$HOME/.cache/pre-commit-terraform` are both documented (README) and + are what the Docker cache-mount instructions rely on. + + Args: + tmp_repo: Temp git repo fixture. + tmp_path: Temp dir fixture, used for the sandboxed `PATH`. + cache_root_var: Env var under test, deciding the cache root. + """ + base = tmp_path / 'fallback' + # `XDG_CACHE_HOME` is itself the cache dir; `HOME` gains a `.cache`. + cache_root_suffixes = {'XDG_CACHE_HOME': (), 'HOME': ('.cache',)} + cache_root = base.joinpath(*cache_root_suffixes[cache_root_var]) + _write_stub( + _CachedTool('tflint', 'tflint').stub_path( + cache_root / 'pre-commit-terraform', + PINNED_TFLINT_VERSION, + ), + 'FALLBACK_TFLINT', + ) + + hook_run = _run_hook( + 'terraform_tflint.sh', + [f'--hook-config=--tool-version={PINNED_TFLINT_VERSION}'], + cwd=tmp_repo, + env=_hook_env( + {cache_root_var: str(base)}, + str(_sandbox_path_dir(tmp_path)), + ), + ) + + combined = hook_run.stdout + assert 'FALLBACK_TFLINT' in combined, combined + assert DOWNLOAD_MSG not in combined, combined + assert hook_run.returncode == 0, combined + + +# Every hook that wires a tool name into `common::resolve_tool_path`. The +# `tf`-sentinel hooks are pinned to `--tf-path=terraform` so the expected +# cache subdir stays independent of what the host has installed. +_TF_PATH_TERRAFORM = ('--hook-config=--tf-path=terraform',) +_WIRED_HOOKS = ( + pytest.param( + _HookWiring( + 'terraform_tflint.sh', + _CachedTool('tflint', 'tflint'), + (), + ), + id='tflint', + ), + pytest.param( + _HookWiring('terraform_trivy.sh', _CachedTool('trivy', 'trivy'), ()), + id='trivy', + ), + pytest.param( + _HookWiring('terraform_tfsec.sh', _CachedTool('tfsec', 'tfsec'), ()), + id='tfsec', + ), + pytest.param( + _HookWiring('terrascan.sh', _CachedTool('terrascan', 'terrascan'), ()), + id='terrascan', + ), + pytest.param( + _HookWiring( + 'terraform_wrapper_module_for_each.sh', + _CachedTool('hcledit', 'hcledit'), + (), + ), + id='hcledit', + ), + pytest.param( + _HookWiring( + 'terraform_docs.sh', + _CachedTool('terraform-docs', 'terraform-docs'), + (), + ), + id='terraform-docs', + ), + pytest.param( + _HookWiring('tfupdate.sh', _CachedTool('tfupdate', 'tfupdate'), ()), + id='tfupdate', + ), + pytest.param( + _HookWiring( + 'infracost_breakdown.sh', + _CachedTool('infracost', 'infracost'), + (), + ), + id='infracost', + ), + pytest.param( + _HookWiring( + 'terragrunt_fmt.sh', + _CachedTool('terragrunt', 'terragrunt'), + (), + ), + id='terragrunt-fmt', + ), + pytest.param( + _HookWiring( + 'terragrunt_validate.sh', + _CachedTool('terragrunt', 'terragrunt'), + (), + ), + id='terragrunt-validate', + ), + pytest.param( + _HookWiring( + 'terragrunt_validate_inputs.sh', + _CachedTool('terragrunt', 'terragrunt'), + (), + ), + id='terragrunt-validate-inputs', + ), + pytest.param( + _HookWiring( + 'terragrunt_providers_lock.sh', + _CachedTool('terragrunt', 'terragrunt'), + (), + ), + id='terragrunt-providers-lock', + ), + pytest.param( + _HookWiring( + 'terraform_fmt.sh', + _CachedTool('terraform', 'terraform'), + _TF_PATH_TERRAFORM, + ), + id='tf-fmt', + ), + pytest.param( + _HookWiring( + 'terraform_validate.sh', + _CachedTool('terraform', 'terraform'), + _TF_PATH_TERRAFORM, + ), + id='tf-validate', + ), + pytest.param( + _HookWiring( + 'terraform_providers_lock.sh', + _CachedTool('terraform', 'terraform'), + _TF_PATH_TERRAFORM, + ), + id='tf-providers-lock', + ), +) + + +@pytest.mark.parametrize('wiring', _WIRED_HOOKS) +def test_hook_wires_its_own_tool_name( # pragma: win32 no cover + tmp_repo: Path, + cache_dir: Path, + tmp_path: Path, + wiring: _HookWiring, +) -> None: + """Check each hook resolves `--tool-version` for the tool it wraps. + + A hook passing a wrong or misspelled tool name to + `common::resolve_tool_path` would miss the cache entry seeded at + `//_/` and try to download instead, + so a cache hit here is what proves the wiring. + + Args: + tmp_repo: Temp git repo fixture. + cache_dir: Temp cache root fixture. + tmp_path: Temp dir fixture, used for the sandboxed `PATH`. + wiring: Hook under test plus the cache entry it must resolve. + """ + _write_stub( + wiring.cached_tool.stub_path(cache_dir, PINNED_ANY_VERSION), + 'WIRED_TOOL', + ) + + hook_run = _run_hook( + wiring.hook_name, + [ + *wiring.extra_args, + f'--hook-config=--tool-version={PINNED_ANY_VERSION}', + ], + cwd=tmp_repo, + env=_hook_env( + _pct_cache_env(cache_dir), + str(_sandbox_path_dir(tmp_path)), + ), + ) + + combined = hook_run.stdout + # Exit code is deliberately not asserted: a stub that ignores its + # arguments makes several of these hooks fail for unrelated reasons. + # Only the resolution decision is under test here. + assert DOWNLOAD_MSG not in combined, combined + assert NO_INSTALLER_MSG not in combined, combined + + +def test_tf_path_rejects_invalid_value( # pragma: win32 no cover + tmp_repo: Path, + cache_dir: Path, + tmp_path: Path, +) -> None: + """Check an unrecognized `--tf-path` + `--tool-version` errors.""" + hook_run = _run_hook( + 'terraform_fmt.sh', + [ + '--hook-config=--tf-path=/some/custom/path', + f'--hook-config=--tool-version={PINNED_TF_VERSION}', + ], + cwd=tmp_repo, + env=_hook_env( + _pct_cache_env(cache_dir), + str(_sandbox_path_dir(tmp_path)), + ), + ) + + combined = hook_run.stdout + assert hook_run.returncode != 0, combined + assert TF_PATH_INVALID_MSG in combined, combined + assert DOWNLOAD_MSG not in combined, combined + assert not list(cache_dir.iterdir()), combined + + +def test_checkov_ignores_tool_version( # pragma: win32 no cover + tmp_repo: Path, + cache_dir: Path, +) -> None: + """Check checkov (pip-distributed) treats `--tool-version` as a no-op. + + `terraform_checkov.sh` passes an empty tool name, so resolution returns + early and the no-op is observable as an untouched cache root. Whether + checkov itself then succeeds is out of scope, so its exit code is not + asserted - a broken checkov install must not fail this test. + """ + hook_run = _run_hook( + 'terraform_checkov.sh', + ['--hook-config=--tool-version=1.2.3', '--args=--quiet'], + cwd=tmp_repo, + env=_hook_env(_pct_cache_env(cache_dir), os.environ['PATH']), + ) + + combined = hook_run.stdout + assert not list(cache_dir.iterdir()), combined + assert DOWNLOAD_MSG not in combined, combined + assert NO_INSTALLER_MSG not in combined, combined + assert TOOL_MISSING_MSG not in combined, combined + + +def test_actionable_error_when_tool_missing( # pragma: win32 no cover + tmp_repo: Path, + cache_dir: Path, + tmp_path: Path, +) -> None: + """Check the actionable error when unpinned and absent from PATH.""" + hook_run = _run_hook( + 'terraform_tflint.sh', + [], + cwd=tmp_repo, + env=_hook_env( + _pct_cache_env(cache_dir), + str(_sandbox_path_dir(tmp_path)), + ), + ) + + combined = hook_run.stdout + assert hook_run.returncode != 0, combined + assert TOOL_MISSING_MSG in combined, combined + assert "'tflint' is required by" in combined, combined + assert '--hook-config=--tool-version=' in combined, combined + + +def test_hook_runs_without_cpu_count_tools( # pragma: win32 no cover + tmp_repo: Path, + cache_dir: Path, + tmp_path: Path, +) -> None: + """Check a hook still runs when no CPU-count tool is on `PATH`. + + `common::get_cpu_num` ends in `nproc || sysctl -n hw.ncpu || echo 1` + (hooks/_common.sh:300, or :289 on a cgroup-v2 host), and + `common::per_dir_hook` derives `parallelism_limit` from what it + returns. With both probes hidden, that trailing fallback is the only + thing left producing a value at all, and each assertion below pins a + distinct way of losing it: drop the `|| echo 1` and the hook dies with + 127 mid-run, make it yield 0 instead of 1 and `parallelism_limit` goes + negative, trading silent serial execution for a scary warning. + + Args: + tmp_repo: Temp git repo fixture. + cache_dir: Empty cache root fixture, seeded with a stub below. + tmp_path: Temp dir fixture, used for the sandboxed `PATH`. + """ + sandbox_path_dir = _sandbox_path_dir(tmp_path) + for cpu_count_tool in _CPU_COUNT_TOOLS: + # `missing_ok`: whether either got symlinked at all is exactly the + # host-dependent thing this test exists to make irrelevant. + (sandbox_path_dir / cpu_count_tool).unlink(missing_ok=True) + + _write_stub( + _CachedTool('tflint', 'tflint').stub_path( + cache_dir, + PINNED_TFLINT_VERSION, + ), + 'CACHED_TFLINT', + ) + + hook_run = _run_hook( + 'terraform_tflint.sh', + [f'--hook-config=--tool-version={PINNED_TFLINT_VERSION}'], + cwd=tmp_repo, + env=_hook_env(_pct_cache_env(cache_dir), str(sandbox_path_dir)), + ) + + combined = hook_run.stdout + assert 'CACHED_TFLINT' in combined, combined + # `CPU` falls back to 1, which `common::per_dir_hook` treats as the + # documented single-core case and stays quiet about (:407). + assert PARALLELISM_CAPPED_MSG not in combined, combined + assert hook_run.returncode == 0, combined + + +@pytest.mark.network +def test_real_download_on_cache_miss( # pragma: win32 no cover + tmp_repo: Path, + cache_dir: Path, +) -> None: + """Check a genuine end-to-end download on a cache miss (network).""" + hook_run = _run_hook( + 'terraform_tflint.sh', + [f'--hook-config=--tool-version={PINNED_TFLINT_VERSION}'], + cwd=tmp_repo, + env=_hook_env(_pct_cache_env(cache_dir), os.environ['PATH']), + ) + + combined = hook_run.stdout + assert DOWNLOAD_MSG in combined, combined + + cached_bin = ( + cache_dir + / 'tflint' + / PINNED_TFLINT_VERSION + / _current_os_arch() + / 'tflint' + ) + assert os.access(cached_bin, os.X_OK), combined + + version_check = subprocess.run( # noqa: S603 + (str(cached_bin), '--version'), + capture_output=True, + text=True, + check=False, + timeout=VERSION_CHECK_TIMEOUT_SECONDS, + ) + assert version_check.returncode == 0, version_check.stderr + assert PINNED_TFLINT_VERSION in version_check.stdout + + # 2 is tflint's own lint findings on the minimal fixture, not ours. + assert hook_run.returncode in {0, 2}, combined + + +@pytest.mark.network +def test_real_download_beyond_release_page_one( # pragma: win32 no cover + tmp_repo: Path, + cache_dir: Path, +) -> None: + """Check pinning a version older than GitHub's first `/releases` page. + + Regression test: unpaginated `/releases` only covers the 30 newest, + so an older pin used to silently fail with an empty asset URL. + """ + hook_run = _run_hook( + 'terraform_tflint.sh', + [f'--hook-config=--tool-version={OLD_TFLINT_VERSION}'], + cwd=tmp_repo, + env=_hook_env(_pct_cache_env(cache_dir), os.environ['PATH']), + ) + + combined = hook_run.stdout + assert DOWNLOAD_MSG in combined, combined + + cached_bin = ( + cache_dir + / 'tflint' + / OLD_TFLINT_VERSION + / _current_os_arch() + / 'tflint' + ) + assert os.access(cached_bin, os.X_OK), combined + + version_check = subprocess.run( # noqa: S603 + (str(cached_bin), '--version'), + capture_output=True, + text=True, + check=False, + timeout=VERSION_CHECK_TIMEOUT_SECONDS, + ) + assert version_check.returncode == 0, version_check.stderr + assert OLD_TFLINT_VERSION in version_check.stdout + + # 2 is tflint's own lint findings on the minimal fixture, not ours. + assert hook_run.returncode in {0, 2}, combined + + +@pytest.mark.network +def test_concurrent_cache_miss_is_race_free( # pragma: win32 no cover + tmp_repo: Path, + cache_dir: Path, +) -> None: + """Check N processes racing the same cache miss don't corrupt it. + + Regression test for a race in `common::populate_tool_cache`: + before it staged each download in a private, per-process directory + and atomically published only the resulting binary, N processes + hitting the same uncached (tool, version) at once shared one + `curl`/`unzip` working directory - so one process' cleanup + (`rm "$PKG"`) could delete the archive out from under another's + still-running `unzip` ("cannot find or open ... .zip"), or `unzip` + could meet a binary a sibling had already extracted and block on + an interactive overwrite prompt, hanging (then failing) under + pre-commit's non-interactive stdin. + """ + # 2 is the minimal N that can reproduce a race at all; every + # process beyond that adds real-network exposure (see this + # function's own `@pytest.mark.network`) without proving anything + # a race between 2 doesn't already prove. + outputs = _run_concurrent_hooks( + 2, + 'terraform_tflint.sh', + [f'--hook-config=--tool-version={PINNED_TFLINT_VERSION}'], + cwd=tmp_repo, + env=_hook_env(_pct_cache_env(cache_dir), os.environ['PATH']), + ) + + cached_bin = ( + cache_dir + / 'tflint' + / PINNED_TFLINT_VERSION + / _current_os_arch() + / 'tflint' + ) + assert os.access(cached_bin, os.X_OK), outputs + + version_check = subprocess.run( # noqa: S603 + (str(cached_bin), '--version'), + capture_output=True, + text=True, + check=False, + timeout=VERSION_CHECK_TIMEOUT_SECONDS, + ) + assert version_check.returncode == 0, version_check.stderr + assert PINNED_TFLINT_VERSION in version_check.stdout + + # Every staging dir is cleaned up whether its process won the race + # or lost it - none left behind regardless of outcome. Now a + # sibling inside the version dir, not the tool dir (os_arch added). + leftovers = list( + (cache_dir / 'tflint' / PINNED_TFLINT_VERSION).glob( + f'{_current_os_arch()}.*', + ), + ) + assert not leftovers, outputs diff --git a/tools/install/_common.sh b/tools/install/_common.sh index eaaafa0cd..5648c7e91 100755 --- a/tools/install/_common.sh +++ b/tools/install/_common.sh @@ -8,10 +8,15 @@ TOOL=${0##*/} readonly TOOL=${TOOL%%.*} # Get "TOOL_VERSION" -# shellcheck disable=SC1091 # Created in Dockerfile before execution of this script -source /.env +# /.env is created in the Dockerfile before this script runs there; when +# this script is invoked directly (e.g. at hook run-time, outside a Docker +# build), it won't exist and the version env var is expected to already be +# exported by the caller instead. +# shellcheck disable=SC1091 +[[ -f /.env ]] && source /.env env_var_name="${TOOL//-/_}" -env_var_name="${env_var_name^^}_VERSION" +# `${var^^}` is bash 4+ only; macOS ships bash 3.2 by default. +env_var_name="$(tr '[:lower:]' '[:upper:]' <<< "${env_var_name}_VERSION")" # shellcheck disable=SC2034 # Used in other scripts readonly VERSION="${!env_var_name}" @@ -69,7 +74,24 @@ function common::install_from_gh_release { if [[ $VERSION == latest ]]; then "${CURL_CMD[@]}" -L "$("${CURL_CMD[@]}" -s "${RELEASES}/latest" | grep -o -E -i -m 1 "$GH_RELEASE_REGEX_LATEST")" > "$PKG" else - "${CURL_CMD[@]}" -L "$("${CURL_CMD[@]}" -s "$RELEASES" | grep -o -E -i -m 1 "$GH_RELEASE_REGEX_SPECIFIC_VERSION")" > "$PKG" + # Unpaginated $RELEASES only has the 30 newest releases; page + # through (100/page) until matched or an empty page ends it. + local page=1 + local -r max_pages=20 # 2000 releases; generous for any wrapped tool + local asset_url="" page_releases + while [[ -z $asset_url && $page -le $max_pages ]]; do + page_releases=$("${CURL_CMD[@]}" -s "${RELEASES}?per_page=100&page=${page}") + [[ $page_releases == "[]" ]] && break + asset_url=$(grep -o -E -i -m 1 "$GH_RELEASE_REGEX_SPECIFIC_VERSION" <<< "$page_releases" || true) + ((page++)) + done + + if [[ -z $asset_url ]]; then + echo "ERROR: could not find a '$TOOL' release asset matching version '$VERSION' (looked through up to $((page - 1)) page(s) of releases)." >&2 + exit 1 + fi + + "${CURL_CMD[@]}" -L "$asset_url" > "$PKG" fi # Make tool ready to use diff --git a/tools/install/terrascan.sh b/tools/install/terrascan.sh index 4393159d3..83636e190 100755 --- a/tools/install/terrascan.sh +++ b/tools/install/terrascan.sh @@ -11,7 +11,8 @@ readonly SCRIPT_DIR [[ $TARGETARCH == amd64 ]] && ARCH="x86_64" || ARCH="$TARGETARCH" readonly ARCH # Convert the first letter to Uppercase -OS="${TARGETOS^}" +# `${var^}` is bash 4+ only; macOS ships bash 3.2 by default. +OS="$(tr '[:lower:]' '[:upper:]' <<< "${TARGETOS:0:1}")${TARGETOS:1}" GH_ORG="tenable" GH_RELEASE_REGEX_SPECIFIC_VERSION="https://.+?${VERSION}_${OS}_${ARCH}.tar.gz"