release.yml

name: Release
on:
  push:
    branches: [main]
  workflow_dispatch:
permissions:
  contents: write
  packages: write
  pull-requests: write
  actions: read
  id-token: write
jobs:
  check_changesets:
    name: Check Changesets
    runs-on: ubuntu-24.04
    outputs:
      changeset_outputs_hasChangesets:
        ${{ steps.changesets.outputs.hasChangesets }}
    steps:
      - name: Checkout
        uses: actions/checkout@v6
        with:
          # Fetch existing tags so `changeset tag` (run via `yarn release`)
          # skips packages whose current version is already tagged. With the
          # default shallow, tag-less checkout it re-creates ALL package tags
          # locally, and the changesets action's `git push origin <tag>` is
          # rejected for any package that didn't bump in this release (tags
          # are annotated, so even same-commit re-pushes are rejected).
          # See the 2.30.1 release failure (run 29261242580).
          fetch-depth: 0
          fetch-tags: true
      - name: Setup node
        uses: actions/setup-node@v6
        with:
          node-version-file: '.nvmrc'
          cache-dependency-path: 'yarn.lock'
          cache: 'yarn'
          # Note: Do NOT set registry-url here — it creates a NODE_AUTH_TOKEN
          # that conflicts with npm OIDC trusted publishing
      # npm OIDC trusted publishing requires npm >= 11.5.1 client-side. Node 22
      # LTS ships npm 10.x, which has no OIDC code path and falls back to
      # NODE_AUTH_TOKEN, failing ENEEDAUTH.
      - name: Upgrade npm for OIDC trusted publishing
        # Pin to 11.x (not @latest): OIDC only needs npm >= 11.5.1, and floating
        # tags drag in majors that raise the node floor / lack a stable GA and
        # break CI. 11.x supports node 22.9+.
        #
        # NOTE: this bootstraps via the node-bundled npm, so the .nvmrc node
        # version must ship a working npm. Node 22.22.2's bundled npm 10.9.7 has
        # a broken module tree (missing promise-retry) and crashes here — see
        # actions/runner-images#13883. 22.23.1 bundles npm 10.9.8, which works.
        # Keep .nvmrc off 22.22.2.
        run: npm install -g npm@11
      - name: Configure npm registry
        run: npm config set registry https://registry.npmjs.org/
      - name: Install root dependencies
        run: yarn install
      # Build @hyperdx/common-utils before changesets publish runs.
      # @hyperdx/cli's `prepublishOnly` runs `yarn build` (tsup), which bundles
      # imports from `@hyperdx/common-utils/dist/*`. Without this step those
      # paths fail to resolve and the npm publish for @hyperdx/cli is skipped
      # while the GitHub release/tag still succeeds. See HDX-4075.
      - name: Build common-utils
        run: make ci-build
      # The changesets action force-rebuilds changeset-release/main from main
      # on every run, destroying any commits on it — including human edits to
      # the AI-generated root CHANGELOG.md. Capture the branch's current
      # CHANGELOG.md first so the release_changelog job can reuse an existing
      # (possibly human-edited) section when the changeset set is unchanged.
      - name: Capture release-notes state before branch rebuild
        # always(), matching the changesets/action step below: that step force-
        # rebuilds changeset-release/main even when an earlier step in this job
        # failed, so without this the only copy of a maintainer's edit would be
        # skipped and then destroyed.
        if: always()
        continue-on-error: true
        run: |
          mkdir -p /tmp/prev-release-notes
          if git fetch origin changeset-release/main; then
            git show FETCH_HEAD:CHANGELOG.md \
              > /tmp/prev-release-notes/CHANGELOG.md || true
          fi
      - name: Upload previous release notes
        if: always()
        # continue-on-error, like the capture step above: this artifact is only
        # an optimisation for the changelog jobs. Without it a transient
        # artifact-service failure (or an upload conflict when this job is
        # re-run, a documented recovery path here) would fail check_changesets
        # — the job every other job needs, and the one that publishes to npm.
        continue-on-error: true
        uses: actions/upload-artifact@v7
        with:
          name: prev-release-notes
          path: /tmp/prev-release-notes/
          if-no-files-found: ignore
          retention-days: 1
          # Deliberately NOT overwrite: true. Artifacts are scoped to the run,
          # not the attempt, so re-running this job after an unrelated failure
          # would re-capture an already-rebuilt branch and replace the good
          # capture — destroying the only copy of a maintainer's edit. Without
          # overwrite the re-upload conflicts, which continue-on-error above
          # turns into a warning, leaving the first attempt's capture intact.
      - name: Create Release Pull Request or Publish to npm
        if: always()
        id: changesets
        uses: changesets/action@v1
        with:
          commit: 'chore(release): bump HyperDX app/package versions'
          title: 'Release HyperDX'
          version: yarn run version
          publish: yarn release
        env:
          YARN_ENABLE_IMMUTABLE_INSTALLS: false
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          NPM_CONFIG_PROVENANCE: true

  # ---------------------------------------------------------------------------
  # Root changelog – write an AI-generated cross-package summary into the
  # "Release HyperDX" PR.
  #
  # Split into two jobs on purpose. The model runs in `release_changelog_draft`
  # with `contents: read` and no push credentials, and its only output is an
  # artifact. `release_changelog_publish` then splices and pushes from a fresh
  # checkout the model never touched. The model reads changeset bodies, commit
  # messages and PR bodies — all attacker-controlled on a public repo — so it is
  # treated as untrusted: an injected prompt must not be able to reach a
  # credential, and must not be able to alter the script that does the splicing.
  # Note that a prefix allowlist like `Bash(git log:*)` is NOT read-only
  # (`git log --output=<path> --format=format:<content>` writes any file), which
  # is why the boundary is the job, not the tool list.
  #
  # Fail-soft by design: no job that builds, publishes or notifies downstream
  # depends on either of these, so a failure here cannot block a release. They
  # are listed in slack-notify-failure's `needs` so genuine breakage is visible
  # rather than silent — routine conditions (a merged release PR, a lost branch
  # race, a superseded draft) are signalled as skips and warnings, not
  # failures, precisely so that alert stays meaningful.
  # ---------------------------------------------------------------------------
  release_changelog_draft:
    name: Draft Root Changelog
    runs-on: ubuntu-24.04
    needs: [check_changesets]
    if: needs.check_changesets.outputs.changeset_outputs_hasChangesets == 'true'
    timeout-minutes: 20
    concurrency:
      # Queue rather than cancel. A cancelled job is not a failure and nothing
      # needs this one, so under pushes arriving faster than a full generation
      # cycle every draft would be cancelled in turn and the release PR would
      # silently end up with no changelog section. Stale runs are discarded at
      # the push instead (non-fast-forward) or by the publish staleness guard.
      group: release-changelog-${{ github.workflow }}
      cancel-in-progress: false
    permissions:
      contents: read
    outputs:
      skip: ${{ steps.meta.outputs.skip }}
      version: ${{ steps.meta.outputs.version }}
      inputs: ${{ steps.meta.outputs.inputs }}
      empty: ${{ steps.changesets_export.outputs.empty }}
      branch_sha: ${{ steps.meta.outputs.branch_sha }}
    steps:
      - name: Checkout release branch
        uses: actions/checkout@v6
        with:
          ref: changeset-release/main
          fetch-depth: 0
          # The model has filesystem read access in this job; do not leave a
          # push token in .git/config for it to find. This job never pushes.
          persist-credentials: false

      # The pinned CLI declares `node >=22`, and every other step here shells
      # out to `node`. Without this the job silently inherits whatever the
      # runner image ships, so an image bump could break a release.
      - name: Setup node
        uses: actions/setup-node@v6
        with:
          node-version-file: '.nvmrc'

      - name: Compute release metadata
        id: meta
        run: |
          set -euo pipefail
          # Read the version from the fixed group's canonical package, NOT the
          # root package.json: version.sh:20 uses BSD `sed -i ''`, which is a
          # no-op under the runner's GNU sed, so the root version has been
          # frozen at 2.0.0 since Jan 2025 while app/api track releases.
          # Comparing root versions would make this job skip on every release.
          VERSION="$(node -p "require('./packages/app/package.json').version")"
          PREV_VERSION="$(git show origin/main:packages/app/package.json \
            | node -p "JSON.parse(require('fs').readFileSync(0,'utf8')).version")"
          # Only releases that bump the fixed group (app/api/otel-collector) get
          # a root changelog section. A CLI-only or common-utils-only release
          # leaves the version unchanged, and generating for it would overwrite
          # the already-published section for that version. Those packages have
          # their own changelogs (and the CLI a GitHub Release).
          if [ "$VERSION" = "$PREV_VERSION" ]; then
            echo "App version unchanged (${VERSION}); skipping root changelog"
            echo "skip=true" >> "$GITHUB_OUTPUT"
          else
            echo "skip=false" >> "$GITHUB_OUTPUT"
          fi
          # If this hash is unchanged since the last generation, the previous
          # section (and any human edits to it) is reused instead of
          # regenerated. Shared with the publish job's staleness guard.
          INPUTS="$(.github/scripts/changeset-hash.sh origin/main)"
          echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
          echo "prev_version=${PREV_VERSION}" >> "$GITHUB_OUTPUT"
          # Commit this draft was built from. The version and the changeset hash
          # are both unchanged by a rebuild that replays the same main state, and
          # a changelog blob is unchanged by a rebuild that discards a section —
          # so neither identifies "the branch I drafted against". The tip does:
          # changesets/action force-pushes a fresh commit on every rebuild, so
          # any rebuild at all moves it.
          echo "branch_sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
          echo "inputs=${INPUTS}" >> "$GITHUB_OUTPUT"

      - name: Download previous release notes
        if: steps.meta.outputs.skip == 'false'
        continue-on-error: true
        uses: actions/download-artifact@v8
        with:
          name: prev-release-notes
          path: /tmp/prev-release-notes

      - name: Reuse existing section if inputs unchanged
        if: steps.meta.outputs.skip == 'false'
        id: reuse
        env:
          VERSION: ${{ steps.meta.outputs.version }}
          INPUTS: ${{ steps.meta.outputs.inputs }}
        run: |
          set -euo pipefail
          # `extract` exits 2 for a missing file, a hash mismatch, or a section
          # a maintainer emptied out — all of which mean "regenerate". Exit 1 is
          # something else: a bad flag, an unreadable file, a parse error. Read
          # as a cache miss it would degrade into "regenerate every release"
          # with nothing in the log to say edit reuse had stopped working.
          set +e
          node .github/scripts/release-notes.mjs extract \
            --changelog /tmp/prev-release-notes/CHANGELOG.md \
            --version "$VERSION" \
            --inputs "$INPUTS" \
            > /tmp/release-notes-body.md
          STATUS=$?
          set -e
          if [ "$STATUS" -ne 0 ] && [ "$STATUS" -ne 2 ]; then
            echo "::error::release-notes extract failed with status ${STATUS}"
            exit 1
          fi
          if [ "$STATUS" -eq 0 ]; then
            # The reused body is a maintainer's edit, not model output, so it
            # can legitimately contain things the validator rejects (a `---`
            # thematic break, a link to some other host). Check it HERE and
            # treat a failure as a cache miss, so the release regenerates with
            # their text as context instead of the publish job going red with
            # the edit recoverable only from an expiring artifact.
            #
            # Strip the generated package list first: it carries HTML comment
            # markers that the validator would otherwise flag.
            node .github/scripts/release-notes.mjs strip-package-list \
              --body /tmp/release-notes-body.md
            if node .github/scripts/release-notes.mjs validate \
                --body /tmp/release-notes-body.md; then
              echo "Reusing previous section (changeset set unchanged)"
              echo "hit=true" >> "$GITHUB_OUTPUT"
            else
              echo "::warning::Previous section failed validation; regenerating with it as context"
              cp /tmp/release-notes-body.md /tmp/previous-section.md
              echo "hit=false" >> "$GITHUB_OUTPUT"
            fi
          else
            echo "hit=false" >> "$GITHUB_OUTPUT"
          fi

      - name: Extract previous section as generator context
        if:
          steps.meta.outputs.skip == 'false' && steps.reuse.outputs.hit ==
          'false'
        env:
          VERSION: ${{ steps.meta.outputs.version }}
          PREV_VERSION: ${{ steps.meta.outputs.prev_version }}
        run: |
          set -euo pipefail
          # The reuse step writes this file when a maintainer's edit failed
          # validation; that is better context than anything below, so keep it.
          if [ -f /tmp/previous-section.md ]; then
            echo "Using the section the reuse step set aside"
            exit 0
          fi
          # The changeset set changed, so the section must be regenerated — but
          # a maintainer may have edited the previous one. Hand it to the
          # generator so their phrasing is preserved for changes it covers.
          # Matched with --latest, not by version: a new changeset that raises
          # the bump level also changes the version, so a version-keyed lookup
          # would miss in exactly the case that triggers regeneration.
          node .github/scripts/release-notes.mjs extract \
            --changelog /tmp/prev-release-notes/CHANGELOG.md \
            --latest \
            > /tmp/previous-section.md \
            || rm -f /tmp/previous-section.md
          # On the first run of a cycle the newest captured section belongs to
          # the PREVIOUS release. The prompt calls this file "the previous
          # generation of this very section" and asks for its phrasing to be
          # preserved, which would carry last release's bullets into the new
          # notes — so keep it only when it really is this version's section.
          # Compare against PREV_VERSION, not VERSION. A new changeset that
          # raises the bump level changes VERSION mid-cycle, and that is exactly
          # the case --latest exists to cover — the newest section is still this
          # cycle's work, just filed under a version we have since superseded.
          # Only a section matching the last RELEASED version is stale context.
          if LATEST="$(node .github/scripts/release-notes.mjs latest-version \
              --changelog /tmp/prev-release-notes/CHANGELOG.md)"; then
            if [ -f /tmp/previous-section.md ] && [ "$LATEST" = "$PREV_VERSION" ]; then
              echo "Newest captured section is the released ${PREV_VERSION}; generating from scratch"
              rm -f /tmp/previous-section.md
            fi
          elif [ -f /tmp/previous-section.md ]; then
            # No version to compare against, so there is no way to tell this
            # cycle's section from the last release's. Handing over the wrong one
            # carries last release's bullets into these notes; handing over
            # nothing only costs some phrasing.
            echo "::warning::Cannot determine the newest section's version; discarding it as context"
            rm -f /tmp/previous-section.md
          fi

      - name: Export changesets for the generator
        id: changesets_export
        if:
          steps.meta.outputs.skip == 'false' && steps.reuse.outputs.hit ==
          'false'
        run: |
          set -euo pipefail
          mkdir -p /tmp/changesets
          # `|| true` on each filter: merging the release PR mid-run consumes
          # every changeset, and an empty grep exits 1 under pipefail.
          git ls-tree -r --name-only origin/main -- .changeset \
            | { grep '\.md$' || true; } \
            | { grep -v '^\.changeset/README\.md$' || true; } \
            | while read -r f; do
                git show "origin/main:${f}" > "/tmp/changesets/$(basename "$f")"
              done
          if [ -z "$(ls -A /tmp/changesets)" ]; then
            # Routine: the release PR was merged mid-run, which consumes every
            # changeset. Signalled as an output, not exit 1 — a red Release run
            # for an expected condition trains people to ignore red runs.
            echo "::notice::No changesets on main (release PR already merged?); skipping generation"
            echo "empty=true" >> "$GITHUB_OUTPUT"
          else
            echo "empty=false" >> "$GITHUB_OUTPUT"
            ls /tmp/changesets
          fi

      # Everything the generator needs is materialised here, by trusted shell,
      # so the model itself needs no Bash at all. That is the point: a prefix
      # allowlist over git is not read-only. `git log --output=<path>` writes an
      # arbitrary file, and a `[diff] external = <cmd>` stanza written into
      # .git/config makes a plain `git diff` execute it — which would be
      # arbitrary code execution in the one process that holds
      # ANTHROPIC_API_KEY.
      - name: Collect generator inputs
        if:
          steps.meta.outputs.skip == 'false' && steps.reuse.outputs.hit ==
          'false' && steps.changesets_export.outputs.empty != 'true'
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          REPO: ${{ github.repository }}
        run: |
          set -euo pipefail
          mkdir -p /tmp/inputs

          # Which entries landed in which package this release.
          git diff origin/main -- 'packages/*/CHANGELOG.md' \
            > /tmp/inputs/package-changelogs.diff

          # Changeset entries carry short commit ids; resolve them to PR numbers
          # and titles so the model can cite `(#NNN)` without network access.
          # The same call carries the author and their association with the repo,
          # which is what tells an outside contribution from one of ours — the
          # model has no network, so a contributor it is not handed here cannot
          # be thanked.
          : > /tmp/inputs/pr-references.txt
          : > /tmp/inputs/contributors.txt
          grep -ohE '^\+- [0-9a-f]{8}:' /tmp/inputs/package-changelogs.diff \
            | sed 's/^+- //; s/://' | sort -u \
            | while read -r sha; do
                rec="$(gh api "repos/${REPO}/commits/${sha}/pulls" \
                  --jq '.[0] | select(. != null)
                        | [.number, .title, .user.login, .user.type,
                           .author_association] | @tsv' 2>/dev/null || true)"
                if [ -z "$rec" ]; then
                  continue
                fi
                num="$(printf '%s' "$rec" | cut -f1)"
                title="$(printf '%s' "$rec" | cut -f2)"
                login="$(printf '%s' "$rec" | cut -f3)"
                kind="$(printf '%s' "$rec" | cut -f4)"
                assoc="$(printf '%s' "$rec" | cut -f5)"
                echo "${sha} #${num} ${title}" >> /tmp/inputs/pr-references.txt
                # Anyone carrying repo rights is us; everyone else contributed
                # from outside. Bots are neither, and thanking one reads as a
                # mistake in notes users see.
                case "$assoc" in
                  MEMBER | OWNER | COLLABORATOR) continue ;;
                esac
                if [ "$kind" = "Bot" ]; then
                  continue
                fi
                case "$login" in
                  *'[bot]') continue ;;
                esac
                echo "#${num} @${login}" >> /tmp/inputs/contributors.txt
              done || true
          # One person can land several PRs in a release.
          sort -u -o /tmp/inputs/contributors.txt /tmp/inputs/contributors.txt

          # The generator's own instructions and the style reference, copied out
          # of the checkout so every path the model may read lives under /tmp
          # and the allowlist below can be a single directory.
          cp .github/prompts/release-changelog.md /tmp/inputs/instructions.md
          cp CHANGELOG.md /tmp/inputs/CHANGELOG.md 2> /dev/null || true

          # Every changeset in ONE file at a known path. The generator gets Read
          # and Write and nothing else — no Glob, no LS — so it cannot discover
          # `.changeset/gentle-boats-serve.md` by name, and left to infer the
          # release from the diff alone it silently drops changes.
          : > /tmp/inputs/changesets.md
          for f in /tmp/changesets/*.md; do
            [ -e "$f" ] || continue
            printf '\n===== %s =====\n\n' "$(basename "$f")" \
              >> /tmp/inputs/changesets.md
            cat "$f" >> /tmp/inputs/changesets.md
          done

          echo "--- collected inputs ---"
          wc -l /tmp/inputs/package-changelogs.diff \
            /tmp/inputs/pr-references.txt /tmp/inputs/contributors.txt

      - name: Generate release notes with Claude
        if:
          steps.meta.outputs.skip == 'false' && steps.reuse.outputs.hit ==
          'false' && steps.changesets_export.outputs.empty != 'true'
        # The CLI directly, not anthropics/claude-code-action. That action only
        # accepts GitHub entity events — issues, pull_request*, workflow_dispatch,
        # repository_dispatch, schedule, workflow_run — and throws
        # "Unsupported event type: push" before running anything. Everything it
        # would add on top of the CLI (a GitHub token, entity context, progress
        # comments on a PR) is what this job deliberately does without, so the
        # CLI is both the fix and the smaller surface.
        #
        # Pinned, not floating: this runs on the release path. The `stable`
        # dist-tag is where to look when bumping.
        #
        # --setting-sources user matches deep-resolve.yml: without it, a
        # `.claude/settings.json` on the checked-out branch could widen the
        # allowlist or register hooks.
        #
        # Read and Write are confined to paths, not just named. This job holds
        # ANTHROPIC_API_KEY, so an unrestricted Read is a path to it —
        # /proc/self/environ carries the whole environment, and the body is
        # published to a public branch, where a credential grep on a literal
        # prefix is defeated by any encoding. Everything the model needs was
        # copied under /tmp by the trusted step above; /proc and the runner's
        # own temp are denied outright in case a later Read rule widens.
        # No Bash at all: a prefix allowlist over git is not read-only.
        env:
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          VERSION: ${{ steps.meta.outputs.version }}
          REPO: ${{ github.repository }}
        # Three separate mechanisms, because they do different jobs:
        #
        # --tools is the one that RESTRICTS. It replaces the built-in tool set,
        # so there is no Bash, Grep, Glob or Task to reach past the path rules
        # below — an allowlist alone would leave Grep unscoped and able to pull
        # matched lines out of the checkout. This job holds ANTHROPIC_API_KEY and
        # reads attacker-influenceable text, and its output is published to a
        # public branch, so a credential grep on a literal prefix (which any
        # encoding defeats) is not the thing standing in the way.
        #
        # --allowedTools only PRE-APPROVES, it does not restrict: it is what
        # keeps a `-p` run from stalling on a permission prompt it cannot
        # answer. The single write is granted by `Edit(<file>)`, not
        # `Write(<file>)` — file permissions are checked against Edit and Read
        # rules only, and a Write rule is accepted but never consulted, which is
        # why an earlier `Write(//tmp/release-notes-body.md)` granted nothing.
        # Scoping it to the one file beats `--permission-mode acceptEdits`,
        # which would auto-approve every path in the working directory.
        #
        # --disallowedTools DENIES, and deny beats allow. /proc/self/environ
        # carries the whole environment, including the API key.
        #
        # Deliberately NOT --bare. It looked like free hardening (no hooks, no
        # plugin sync, no CLAUDE.md discovery) but it left the model holding
        # `Read` alone: it never registered the `Write` from --tools, so the
        # generator printed the body to stdout, wrote no file, and exited 0.
        # See run 31149802657. On a GitHub-hosted runner it was guarding against
        # nothing anyway: there is no user settings file and no MCP server for
        # --setting-sources user to pick up. That stops being true if this job
        # ever moves to a self-hosted runner, where $HOME persists between jobs —
        # confine the settings sources then rather than reaching for --bare.
        #
        # /tmp as the working directory keeps the checkout — scripts, workflow,
        # git config — outside the workspace entirely.
        working-directory: /tmp
        run: |
          set -euo pipefail
          # Two attempts: an overloaded API or a registry blip is routine, and
          # this job is wired into slack-notify-failure, so a transient failure
          # must not page on-call.
          for attempt in 1 2; do
            # The reuse step redirects into this file, so its fall-through path
            # leaves a body behind. Clear it before each attempt, or the
            # assertion below passes on someone else's text.
            rm -f /tmp/release-notes-body.md
            # The prompt goes in on stdin: --tools and the two tool-rule flags
            # are variadic, so a positional prompt would be swallowed as another
            # tool name.
            if printf 'VERSION: %s\nREPO: %s\n\n%s\n' \
                "$VERSION" "$REPO" \
                'Read the instructions in /tmp/inputs/instructions.md and follow them exactly to write /tmp/release-notes-body.md for this release.' \
                | npx --yes @anthropic-ai/claude-code@2.1.220 \
                    --print \
                    --setting-sources user \
                    --tools "Read" "Write" \
                    --allowedTools "Read(//tmp/**)" "Edit(//tmp/release-notes-body.md)" \
                    --disallowedTools "Read(//proc/**)" "Read(//sys/**)" \
                      "Read(//home/**)" "Read(//etc/**)"; then
              break
            fi
            if [ "$attempt" = 2 ]; then
              echo "::error::The changelog generator failed twice; skipping this release's section"
              exit 1
            fi
            echo "::warning::Changelog generator attempt ${attempt} failed; retrying"
            sleep 30
          done
          # The CLI exits 0 when the model ends its turn without writing —
          # including when it spots a prompt-injection attempt and follows the
          # instructions to write nothing. Catch that here, in the job that ran
          # the model, rather than as "Body is blank." two jobs downstream where
          # the alert names the validator instead of the generator.
          test -s /tmp/release-notes-body.md || {
            echo "::error::The generator wrote no release notes body"
            exit 1
          }

      - name: Upload generated body
        # Nothing to hand over if generation was skipped for want of changesets.
        if:
          steps.meta.outputs.skip == 'false' &&
          steps.changesets_export.outputs.empty != 'true'
        uses: actions/upload-artifact@v7
        with:
          name: release-notes-body
          path: /tmp/release-notes-body.md
          retention-days: 1
          overwrite: true

  release_changelog_publish:
    name: Publish Root Changelog
    runs-on: ubuntu-24.04
    needs: [release_changelog_draft]
    # `empty` guards the routine merged-mid-run case: the draft uploads no
    # artifact there, and download-artifact has no missing-artifact escape
    # hatch, so without this the job fails on the path the draft calls routine.
    if:
      needs.release_changelog_draft.outputs.skip == 'false' &&
      needs.release_changelog_draft.outputs.empty != 'true'
    timeout-minutes: 10
    permissions:
      contents: write
    steps:
      # Merging the release PR between drafting and publishing deletes
      # changeset-release/main (head-branch auto-deletion), and actions/checkout
      # fails outright on a missing ref. That is a routine condition, so probe
      # for the branch first and skip quietly rather than redden the release run
      # and page on-call. Every later step is gated on the guard's `stale`
      # output, which stays empty when the guard itself is skipped.
      - name: Probe for the release branch
        id: probe
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          REPO: ${{ github.repository }}
        run: |
          set -euo pipefail
          # A 404 is the routine case. Anything else (rate limit, transport
          # error) must fail loudly: read as "branch gone" it would silently
          # stop publishing a changelog for every release.
          if OUT="$(gh api "repos/${REPO}/git/ref/heads/changeset-release/main" \
              --jq .ref 2>&1)"; then
            echo "exists=true" >> "$GITHUB_OUTPUT"
          elif grep -q 'Not Found' <<< "$OUT"; then
            echo "::notice::changeset-release/main is gone (release PR merged?); nothing to publish"
            echo "exists=false" >> "$GITHUB_OUTPUT"
          else
            echo "::error::Could not probe changeset-release/main: ${OUT}"
            exit 1
          fi

      # Fresh checkout: the scripts and the working tree here were never
      # visible to the model, so a compromised draft job cannot alter what runs.
      - name: Checkout release branch
        if: steps.probe.outputs.exists == 'true'
        uses: actions/checkout@v6
        with:
          ref: changeset-release/main
          fetch-depth: 0

      # A later run can rebuild changeset-release/main between this run's draft
      # and publish. Because this job re-checks-out the branch, the push would
      # fast-forward cleanly onto that newer state and commit THIS run's
      # (now stale) body — in the worst case an orphan section for a version the
      # branch no longer builds, which insertSection never cleans up because it
      # only replaces a matching version.
      #
      # The branch tip is the load-bearing key: a rebuild replaying the same
      # main state leaves the version, the changeset hash AND the changelog blob
      # all identical, so any of those alone can miss a rebuild entirely. The
      # version and hash stay as a cheap, legible signal of WHAT moved in the
      # warning. Skipping is self-correcting — the draft job queues rather than
      # cancels, so the run that rebuilt the branch publishes its own section.
      - name: Verify the release branch still matches the draft
        id: guard
        if: steps.probe.outputs.exists == 'true'
        env:
          DRAFT_VERSION: ${{ needs.release_changelog_draft.outputs.version }}
          DRAFT_INPUTS: ${{ needs.release_changelog_draft.outputs.inputs }}
          DRAFT_BRANCH_SHA:
            ${{ needs.release_changelog_draft.outputs.branch_sha }}
        run: |
          set -euo pipefail
          VERSION="$(node -p "require('./packages/app/package.json').version")"
          INPUTS="$(.github/scripts/changeset-hash.sh origin/main)"
          BRANCH_SHA="$(git rev-parse HEAD)"
          if [ "$VERSION" != "$DRAFT_VERSION" ] \
             || [ "$INPUTS" != "$DRAFT_INPUTS" ] \
             || [ "$BRANCH_SHA" != "$DRAFT_BRANCH_SHA" ]; then
            echo "::warning::Release branch moved since drafting (version ${DRAFT_VERSION}->${VERSION}, inputs ${DRAFT_INPUTS}->${INPUTS}, tip ${DRAFT_BRANCH_SHA}->${BRANCH_SHA}); skipping publish, the newer run will regenerate"
            echo "stale=true" >> "$GITHUB_OUTPUT"
          else
            echo "stale=false" >> "$GITHUB_OUTPUT"
          fi

      - name: Download generated body
        if: steps.guard.outputs.stale == 'false'
        uses: actions/download-artifact@v8
        with:
          name: release-notes-body
          path: /tmp

      - name: Validate generated notes
        if: steps.guard.outputs.stale == 'false'
        # Every check lives in release-notes.mjs with a must-accept/must-reject
        # corpus under `node --test`. It used to be inline greps here, which is
        # how several CommonMark constructs slipped past unnoticed. These checks
        # are fail-fast feedback, not the security boundary — that is
        # WhatsNewDrawer.tsx, which drops image nodes and allowlists link
        # targets on the parsed AST.
        run: |
          set -euo pipefail
          node .github/scripts/release-notes.mjs validate \
            --body /tmp/release-notes-body.md

      - name: Append package changelog list
        if: steps.guard.outputs.stale == 'false'
        env:
          # Absolute URLs: this markdown is also rendered inside the in-app
          # "What's new" modal, where relative links would resolve against the
          # app origin and break.
          REPO_URL: ${{ github.server_url }}/${{ github.repository }}
        run: |
          set -euo pipefail
          # A reuse hit hands back the previously published body, which already
          # carries a package list; without stripping it first, every republish
          # would stack another copy.
          node .github/scripts/release-notes.mjs strip-package-list \
            --body /tmp/release-notes-body.md
          # A bullet list, not a GFM pipe table: the in-app modal renders this
          # with react-markdown and no remark-gfm, where a table degrades to
          # literal `| --- |` text. Plain CommonMark renders correctly in both
          # the modal and on GitHub. (remark-gfm would also autolink bare URLs,
          # creating links the validation step above never inspects.)
          {
            echo ""
            # Markers, so a later strip removes only the generated list and not
            # anything a maintainer wrote after it.
            echo "<!-- hyperdx-package-list -->"
            echo ""
            echo "### 📦 Package changelogs"
            echo ""
            for pkg in packages/*/package.json; do
              dir="$(dirname "$pkg")"
              name="$(node -p "require('./${pkg}').name")"
              version="$(node -p "require('./${pkg}').version")"
              old="$(git show "origin/main:${pkg}" 2>/dev/null \
                | node -p "JSON.parse(require('fs').readFileSync(0,'utf8')).version" \
                || echo '')"
              if [ "$version" != "$old" ]; then
                anchor="${version//./}"
                echo "- \`${name}\` ${old:-new} → ${version} — [changelog](${REPO_URL}/blob/main/${dir}/CHANGELOG.md#${anchor})"
              fi
            done
            echo ""
            echo "<!-- /hyperdx-package-list -->"
          } >> /tmp/release-notes-body.md

      - name: Splice into CHANGELOG.md and push
        if: steps.guard.outputs.stale == 'false'
        env:
          VERSION: ${{ needs.release_changelog_draft.outputs.version }}
          INPUTS: ${{ needs.release_changelog_draft.outputs.inputs }}
        run: |
          set -euo pipefail
          node .github/scripts/release-notes.mjs insert \
            --changelog CHANGELOG.md \
            --body /tmp/release-notes-body.md \
            --version "$VERSION" \
            --inputs "$INPUTS" \
            --date "$(date -u +%F)"
          # --intent-to-add so `git status` sees the file even when insert had to
          # scaffold it: a branch without a CHANGELOG.md leaves it untracked,
          # which the porcelain check below would read as "nothing to push".
          git add --intent-to-add CHANGELOG.md
          # Belt and braces: this job must only ever touch the changelog.
          # " M" is the ordinary edit; " A"/"AM" the scaffolded-from-nothing case
          # (intent-to-add leaves the content unstaged, so the status is " A").
          case "$(git status --porcelain)" in
            '')
              echo "CHANGELOG.md unchanged; nothing to push"
              exit 0
              ;;
            ' M CHANGELOG.md' | ' A CHANGELOG.md' | 'AM CHANGELOG.md') ;;
            *)
              echo "::error::Unexpected working-tree changes:"
              git status --porcelain
              exit 1
              ;;
          esac
          git config user.name "github-actions[bot]"
          git config user.email "github-actions[bot]@users.noreply.github.com"
          git add CHANGELOG.md
          git commit -m "chore(release): update root CHANGELOG.md"
          # A push to main during generation force-rebuilds the branch, which
          # rejects this push as non-fast-forward. That case is routine and the
          # next run regenerates on top of the rebuilt branch. Anything else —
          # a revoked permission, a new ruleset on changeset-release/* — must
          # fail loudly rather than silently stop publishing every release.
          # Note neither case recovers a maintainer's edit made during that
          # window; see the caveat in AGENTS.md.
          if ! PUSH_OUT="$(git push origin HEAD:changeset-release/main 2>&1)"; then
            echo "$PUSH_OUT"
            if grep -qiE 'non-fast-forward|fetch first|rejected.*(stale info|behind)' <<< "$PUSH_OUT"; then
              echo "::warning::changeset-release/main moved during generation; the next run will regenerate"
            else
              echo "::error::Pushing the changelog failed for a reason other than losing the branch race."
              exit 1
            fi
          fi

  # ---------------------------------------------------------------------------
  # Check if version already published (skip-if-exists)
  # ---------------------------------------------------------------------------
  check_version:
    name: Check if version exists
    needs: check_changesets
    runs-on: ubuntu-24.04
    if:
      needs.check_changesets.outputs.changeset_outputs_hasChangesets == 'false'
    outputs:
      should_release: ${{ steps.check.outputs.should_release }}
    steps:
      - name: Checkout
        uses: actions/checkout@v6
      - name: Load Environment Variables from .env
        uses: xom9ikk/dotenv@v2
      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_HUB_USERNAME }}
          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
      - name: Check if app image tag already exists
        id: check
        run: |
          TAG_EXISTS=$(docker manifest inspect ${{ env.IMAGE_NAME_DOCKERHUB }}:${{ env.IMAGE_VERSION }}${{ env.IMAGE_VERSION_SUB_TAG }} > /dev/null 2>&1 && echo "true" || echo "false")
          if [ "$TAG_EXISTS" = "true" ]; then
            echo "Tag ${{ env.IMAGE_NAME_DOCKERHUB }}:${{ env.IMAGE_VERSION }}${{ env.IMAGE_VERSION_SUB_TAG }} already exists. Skipping release."
            echo "should_release=false" >> $GITHUB_OUTPUT
          else
            echo "Tag does not exist. Proceeding with release."
            echo "should_release=true" >> $GITHUB_OUTPUT
          fi

  # ---------------------------------------------------------------------------
  # OTel Collector – build each arch natively, then merge into multi-arch tag
  # ---------------------------------------------------------------------------
  build-otel-collector:
    name: Build OTel Collector (${{ matrix.arch }})
    needs: [check_changesets, check_version]
    if: needs.check_version.outputs.should_release == 'true'
    strategy:
      fail-fast: true
      matrix:
        include:
          - arch: amd64
            platform: linux/amd64
            runner: ubuntu-latest
          - arch: arm64
            platform: linux/arm64
            runner: ubuntu-latest-arm64
    runs-on: ${{ matrix.runner }}
    steps:
      - name: Checkout
        uses: actions/checkout@v6
      - name: Setup Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_HUB_USERNAME }}
          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Load Environment Variables from .env
        uses: xom9ikk/dotenv@v2
      - name: Build and Push
        uses: docker/build-push-action@v6
        with:
          context: .
          file: ./docker/otel-collector/Dockerfile
          platforms: ${{ matrix.platform }}
          target: prod
          tags: |
            ${{ env.OTEL_COLLECTOR_IMAGE_NAME_DOCKERHUB }}:${{ env.IMAGE_VERSION }}${{ env.IMAGE_VERSION_SUB_TAG }}-${{ matrix.arch }}
            ${{ env.NEXT_OTEL_COLLECTOR_IMAGE_NAME_DOCKERHUB }}:${{ env.IMAGE_VERSION }}${{ env.IMAGE_VERSION_SUB_TAG }}-${{ matrix.arch }}
          push: true
          cache-from: type=gha,scope=otel-collector-${{ matrix.arch }}
          cache-to: type=gha,mode=max,scope=otel-collector-${{ matrix.arch }}

  publish-otel-collector:
    name: Publish OTel Collector Manifest
    needs: [check_version, build-otel-collector]
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v6
      - name: Load Environment Variables from .env
        uses: xom9ikk/dotenv@v2
      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_HUB_USERNAME }}
          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
      - name: Setup Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Create multi-arch manifests
        run: |
          VERSION="${{ env.IMAGE_VERSION }}${{ env.IMAGE_VERSION_SUB_TAG }}"
          MAJOR="${{ env.IMAGE_VERSION }}"
          LATEST="${{ env.IMAGE_LATEST_TAG }}"
          for IMAGE in "${{ env.OTEL_COLLECTOR_IMAGE_NAME_DOCKERHUB }}" "${{ env.NEXT_OTEL_COLLECTOR_IMAGE_NAME_DOCKERHUB }}"; do
            docker buildx imagetools create \
              -t "${IMAGE}:${VERSION}" \
              -t "${IMAGE}:${MAJOR}" \
              -t "${IMAGE}:${LATEST}" \
              "${IMAGE}:${VERSION}-amd64" \
              "${IMAGE}:${VERSION}-arm64"
          done

  # ---------------------------------------------------------------------------
  # App (fullstack prod) – build each arch natively, then merge
  # ---------------------------------------------------------------------------
  build-app:
    name: Build App (${{ matrix.arch }})
    needs: [check_changesets, check_version]
    if: needs.check_version.outputs.should_release == 'true'
    strategy:
      fail-fast: true
      matrix:
        include:
          - arch: amd64
            platform: linux/amd64
            runner: Large-Runner-x64-32
          - arch: arm64
            platform: linux/arm64
            runner: Large-Runner-ARM64-32
    runs-on: ${{ matrix.runner }}
    steps:
      - name: Checkout
        uses: actions/checkout@v6
      - name: Setup Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_HUB_USERNAME }}
          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Load Environment Variables from .env
        uses: xom9ikk/dotenv@v2
      - name: Build and Push
        uses: docker/build-push-action@v6
        with:
          file: ./docker/hyperdx/Dockerfile
          platforms: ${{ matrix.platform }}
          target: prod
          build-contexts: |
            hyperdx=./docker/hyperdx
            api=./packages/api
            app=./packages/app
          build-args: |
            CODE_VERSION=${{ env.CODE_VERSION }}
          tags: |
            ${{ env.IMAGE_NAME_DOCKERHUB }}:${{ env.IMAGE_VERSION }}${{ env.IMAGE_VERSION_SUB_TAG }}-${{ matrix.arch }}
          push: true
          sbom: true
          provenance: true
          cache-from: type=gha,scope=app-${{ matrix.arch }}
          cache-to: type=gha,mode=max,scope=app-${{ matrix.arch }}

  publish-app:
    name: Publish App Manifest
    needs: [check_version, build-app]
    runs-on: ubuntu-latest
    outputs:
      app_was_pushed: 'true'
    steps:
      - name: Checkout
        uses: actions/checkout@v6
      - name: Load Environment Variables from .env
        uses: xom9ikk/dotenv@v2
      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_HUB_USERNAME }}
          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
      - name: Setup Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Create multi-arch manifest
        run: |
          VERSION="${{ env.IMAGE_VERSION }}${{ env.IMAGE_VERSION_SUB_TAG }}"
          MAJOR="${{ env.IMAGE_VERSION }}"
          LATEST="${{ env.IMAGE_LATEST_TAG }}"
          IMAGE="${{ env.IMAGE_NAME_DOCKERHUB }}"
          docker buildx imagetools create \
            -t "${IMAGE}:${VERSION}" \
            -t "${IMAGE}:${MAJOR}" \
            -t "${IMAGE}:${LATEST}" \
            "${IMAGE}:${VERSION}-amd64" \
            "${IMAGE}:${VERSION}-arm64"

  # ---------------------------------------------------------------------------
  # Local (all-in-one-noauth) – build each arch natively, then merge
  # ---------------------------------------------------------------------------
  build-local:
    name: Build Local (${{ matrix.arch }})
    needs: [check_changesets, check_version]
    if: needs.check_version.outputs.should_release == 'true'
    strategy:
      fail-fast: true
      matrix:
        include:
          - arch: amd64
            platform: linux/amd64
            runner: Large-Runner-x64-32
          - arch: arm64
            platform: linux/arm64
            runner: Large-Runner-ARM64-32
    runs-on: ${{ matrix.runner }}
    steps:
      - name: Checkout
        uses: actions/checkout@v6
      - name: Setup Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_HUB_USERNAME }}
          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Load Environment Variables from .env
        uses: xom9ikk/dotenv@v2
      - name: Build and Push
        uses: docker/build-push-action@v6
        with:
          file: ./docker/hyperdx/Dockerfile
          platforms: ${{ matrix.platform }}
          target: all-in-one-noauth
          build-contexts: |
            clickhouse=./docker/clickhouse
            otel-collector=./docker/otel-collector
            hyperdx=./docker/hyperdx
            api=./packages/api
            app=./packages/app
          build-args: |
            CODE_VERSION=${{ env.CODE_VERSION }}
          tags: |
            ${{ env.LOCAL_IMAGE_NAME_DOCKERHUB }}:${{ env.IMAGE_VERSION }}${{ env.IMAGE_VERSION_SUB_TAG }}-${{ matrix.arch }}
            ${{ env.NEXT_LOCAL_IMAGE_NAME_DOCKERHUB }}:${{ env.IMAGE_VERSION }}${{ env.IMAGE_VERSION_SUB_TAG }}-${{ matrix.arch }}
          push: true
          sbom: true
          provenance: true
          cache-from: type=gha,scope=local-${{ matrix.arch }}
          cache-to: type=gha,mode=max,scope=local-${{ matrix.arch }}

  # Boot the freshly built amd64 Local image and push one OTLP log end-to-end
  # into ClickHouse before we publish the multi-arch manifest. Gates publishing
  # on the artefact actually running, not just building. Reuses the per-arch tag
  # the build job already pushed (same reference the publish job merges below).
  smoke-local:
    name: Smoke Test Local Image
    needs: [check_version, build-local]
    runs-on: ubuntu-latest
    timeout-minutes: 15
    steps:
      - name: Checkout
        uses: actions/checkout@v6
      - name: Load Environment Variables from .env
        uses: xom9ikk/dotenv@v2
      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_HUB_USERNAME }}
          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
      - name: Smoke test built image
        run: |
          scripts/ci/smoke-all-in-one.sh "${{ env.LOCAL_IMAGE_NAME_DOCKERHUB }}:${{ env.IMAGE_VERSION }}${{ env.IMAGE_VERSION_SUB_TAG }}-amd64"

  publish-local:
    name: Publish Local Manifest
    needs: [check_version, build-local, smoke-local]
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v6
      - name: Load Environment Variables from .env
        uses: xom9ikk/dotenv@v2
      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_HUB_USERNAME }}
          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
      - name: Setup Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Create multi-arch manifests
        run: |
          VERSION="${{ env.IMAGE_VERSION }}${{ env.IMAGE_VERSION_SUB_TAG }}"
          MAJOR="${{ env.IMAGE_VERSION }}"
          LATEST="${{ env.IMAGE_LATEST_TAG }}"
          for IMAGE in "${{ env.LOCAL_IMAGE_NAME_DOCKERHUB }}" "${{ env.NEXT_LOCAL_IMAGE_NAME_DOCKERHUB }}"; do
            docker buildx imagetools create \
              -t "${IMAGE}:${VERSION}" \
              -t "${IMAGE}:${MAJOR}" \
              -t "${IMAGE}:${LATEST}" \
              "${IMAGE}:${VERSION}-amd64" \
              "${IMAGE}:${VERSION}-arm64"
          done

  # ---------------------------------------------------------------------------
  # All-in-One (all-in-one-auth) – build each arch natively, then merge
  # ---------------------------------------------------------------------------
  build-all-in-one:
    name: Build All-in-One (${{ matrix.arch }})
    needs: [check_changesets, check_version]
    if: needs.check_version.outputs.should_release == 'true'
    strategy:
      fail-fast: true
      matrix:
        include:
          - arch: amd64
            platform: linux/amd64
            runner: Large-Runner-x64-32
          - arch: arm64
            platform: linux/arm64
            runner: Large-Runner-ARM64-32
    runs-on: ${{ matrix.runner }}
    steps:
      - name: Checkout
        uses: actions/checkout@v6
      - name: Setup Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_HUB_USERNAME }}
          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Load Environment Variables from .env
        uses: xom9ikk/dotenv@v2
      - name: Build and Push
        uses: docker/build-push-action@v6
        with:
          file: ./docker/hyperdx/Dockerfile
          platforms: ${{ matrix.platform }}
          target: all-in-one-auth
          build-contexts: |
            clickhouse=./docker/clickhouse
            otel-collector=./docker/otel-collector
            hyperdx=./docker/hyperdx
            api=./packages/api
            app=./packages/app
          build-args: |
            CODE_VERSION=${{ env.CODE_VERSION }}
          tags: |
            ${{ env.ALL_IN_ONE_IMAGE_NAME_DOCKERHUB }}:${{ env.IMAGE_VERSION }}${{ env.IMAGE_VERSION_SUB_TAG }}-${{ matrix.arch }}
            ${{ env.NEXT_ALL_IN_ONE_IMAGE_NAME_DOCKERHUB }}:${{ env.IMAGE_VERSION }}${{ env.IMAGE_VERSION_SUB_TAG }}-${{ matrix.arch }}
          push: true
          sbom: true
          provenance: true
          cache-from: type=gha,scope=all-in-one-${{ matrix.arch }}
          cache-to: type=gha,mode=max,scope=all-in-one-${{ matrix.arch }}

  # Boot the freshly built amd64 All-in-One (auth) image and assert it comes up
  # healthy with ClickHouse answering, before we publish the multi-arch manifest.
  # Health-only (not full OTLP ingest): the auth image gates ingestion behind an
  # INGESTION_API_KEY, so its collector never opens a keyless OTLP receiver — a
  # canary push would always fail. Gates publishing on the artefact actually
  # booting, not just building. Reuses the per-arch tag the build job already
  # pushed (same reference the publish job merges below).
  smoke-all-in-one:
    name: Smoke Test All-in-One Image
    needs: [check_version, build-all-in-one]
    runs-on: ubuntu-latest
    timeout-minutes: 15
    steps:
      - name: Checkout
        uses: actions/checkout@v6
      - name: Load Environment Variables from .env
        uses: xom9ikk/dotenv@v2
      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_HUB_USERNAME }}
          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
      # The auth all-in-one image gates ingestion behind an INGESTION_API_KEY
      # bootstrap, so its bundled collector never opens a keyless OTLP receiver
      # — a canary POST can't land. Health-only mode asserts the app is healthy
      # and ClickHouse is up, which is the runnable check this artefact supports.
      - name: Smoke test built image
        run: |
          scripts/ci/smoke-all-in-one.sh "${{ env.ALL_IN_ONE_IMAGE_NAME_DOCKERHUB }}:${{ env.IMAGE_VERSION }}${{ env.IMAGE_VERSION_SUB_TAG }}-amd64" --health-only

  publish-all-in-one:
    name: Publish All-in-One Manifest
    needs: [check_version, build-all-in-one, smoke-all-in-one]
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v6
      - name: Load Environment Variables from .env
        uses: xom9ikk/dotenv@v2
      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_HUB_USERNAME }}
          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
      - name: Setup Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Create multi-arch manifests
        run: |
          VERSION="${{ env.IMAGE_VERSION }}${{ env.IMAGE_VERSION_SUB_TAG }}"
          MAJOR="${{ env.IMAGE_VERSION }}"
          LATEST="${{ env.IMAGE_LATEST_TAG }}"
          for IMAGE in "${{ env.ALL_IN_ONE_IMAGE_NAME_DOCKERHUB }}" "${{ env.NEXT_ALL_IN_ONE_IMAGE_NAME_DOCKERHUB }}"; do
            docker buildx imagetools create \
              -t "${IMAGE}:${VERSION}" \
              -t "${IMAGE}:${MAJOR}" \
              -t "${IMAGE}:${LATEST}" \
              "${IMAGE}:${VERSION}-amd64" \
              "${IMAGE}:${VERSION}-arm64"
          done

  # ---------------------------------------------------------------------------
  # CLI – compile standalone binaries and upload as GitHub Release assets
  # npm publishing is handled by changesets in the check_changesets job above.
  # This job only compiles platform-specific binaries and creates a GH Release.
  # ---------------------------------------------------------------------------
  release-cli:
    name: Release CLI Binaries
    needs: [check_changesets, check_version]
    if: needs.check_version.outputs.should_release == 'true'
    runs-on: ubuntu-24.04
    steps:
      - name: Checkout
        uses: actions/checkout@v6
      - name: Setup node
        uses: actions/setup-node@v6
        with:
          node-version-file: '.nvmrc'
          cache-dependency-path: 'yarn.lock'
          cache: 'yarn'
      - name: Setup Bun
        uses: oven-sh/setup-bun@v2
        with:
          bun-version: '1.3.11'
      - name: Install dependencies
        run: yarn install
      - name: Build common-utils
        run: make ci-build
      - name: Get CLI version
        id: cli_version
        run: |
          CLI_VERSION=$(node -p "require('./packages/cli/package.json').version")
          echo "version=${CLI_VERSION}" >> $GITHUB_OUTPUT
          echo "CLI version: ${CLI_VERSION}"
      - name: Check if CLI release already exists
        id: check_cli_release
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          if gh release view "cli-v${{ steps.cli_version.outputs.version }}" > /dev/null 2>&1; then
            echo "Release cli-v${{ steps.cli_version.outputs.version }} already exists. Skipping."
            echo "exists=true" >> $GITHUB_OUTPUT
          else
            echo "Release does not exist. Proceeding."
            echo "exists=false" >> $GITHUB_OUTPUT
          fi
      - name: Compile CLI binaries
        if: steps.check_cli_release.outputs.exists == 'false'
        working-directory: packages/cli
        run: |
          yarn compile:linux
          yarn compile:macos
          yarn compile:macos-x64
      - name: Build release notes
        if: steps.check_cli_release.outputs.exists == 'false'
        run: |
          VERSION="${{ steps.cli_version.outputs.version }}"
          CHANGELOG="packages/cli/CHANGELOG.md"

          # Extract the section for this version from CHANGELOG.md
          # Grab everything between "## $VERSION" and the next "## " heading
          NOTES=""
          if [ -f "$CHANGELOG" ]; then
            NOTES=$(awk -v ver="## ${VERSION}" '
              $0 == ver { found=1; next }
              found && /^## / { exit }
              found { print }
            ' "$CHANGELOG")
          fi

          # Write the combined release body
          {
            echo "## @hyperdx/cli v${VERSION}"
            echo ""
            if [ -n "$NOTES" ]; then
              echo "$NOTES"
              echo ""
              echo "---"
              echo ""
            fi
            cat <<'INSTALL'
          ### Installation

          **npm (recommended):**
          ```bash
          npm install -g @hyperdx/cli
          ```

          **Or run directly with npx:**
          ```bash
          npx @hyperdx/cli tui -s <your-hyperdx-api-url>
          ```

          **Manual download (standalone binary, no Node.js required):**
          ```bash
          # macOS Apple Silicon
          INSTALL
            echo "curl -L https://github.com/hyperdxio/hyperdx/releases/download/cli-v${VERSION}/hdx-darwin-arm64 -o hdx"
            echo "# macOS Intel"
            echo "curl -L https://github.com/hyperdxio/hyperdx/releases/download/cli-v${VERSION}/hdx-darwin-x64 -o hdx"
            echo "# Linux x64"
            echo "curl -L https://github.com/hyperdxio/hyperdx/releases/download/cli-v${VERSION}/hdx-linux-x64 -o hdx"
            cat <<'USAGE'

          chmod +x hdx && sudo mv hdx /usr/local/bin/
          ```

          ### Usage

          ```bash
          hdx auth login -s <your-hyperdx-api-url>
          hdx tui
          ```
          USAGE
          } > /tmp/cli-release-notes.md

          echo "--- Generated release notes ---"
          cat /tmp/cli-release-notes.md
      - name: Create GitHub Release
        if: steps.check_cli_release.outputs.exists == 'false'
        uses: softprops/action-gh-release@153bb8e04406b158c6c84fc1615b65b24149a1fe # v2.6.1
        with:
          tag_name: cli-v${{ steps.cli_version.outputs.version }}
          name: '@hyperdx/cli v${{ steps.cli_version.outputs.version }}'
          body_path: /tmp/cli-release-notes.md
          draft: false
          prerelease: false
          files: |
            packages/cli/dist/hdx-linux-x64
            packages/cli/dist/hdx-darwin-arm64
            packages/cli/dist/hdx-darwin-x64
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

  # ---------------------------------------------------------------------------
  # Downstream notifications
  # ---------------------------------------------------------------------------
  notify_helm_charts:
    name: Notify Helm-Charts Downstream
    needs:
      [
        check_changesets,
        publish-app,
        publish-otel-collector,
        publish-local,
        publish-all-in-one,
      ]
    runs-on: ubuntu-24.04
    if: |
      needs.check_changesets.outputs.changeset_outputs_hasChangesets == 'false' &&
      needs.publish-app.outputs.app_was_pushed == 'true'
    steps:
      - name: Checkout
        uses: actions/checkout@v6
      - name: Load Environment Variables from .env
        uses: xom9ikk/dotenv@v2
      - name: Notify Helm-Charts Downstream
        uses: actions/github-script@v9
        continue-on-error: true
        env:
          TAG: ${{ env.IMAGE_VERSION }}${{ env.IMAGE_VERSION_SUB_TAG }}
        with:
          github-token: ${{ secrets.CH_BOT_PAT }}
          script: |
            const { TAG } = process.env;
            const result = await github.rest.actions.createWorkflowDispatch({
              owner: 'ClickHouse',
              repo: 'ClickStack-helm-charts',
              workflow_id: '${{ secrets.DOWNSTREAM_HC_WORKFLOW_ID }}',
              ref: 'main',
              inputs: {
                tag: TAG
              }
            });

  notify_ch:
    name: Notify CH Downstream
    needs:
      [
        check_changesets,
        publish-app,
        publish-otel-collector,
        publish-local,
        publish-all-in-one,
      ]
    runs-on: ubuntu-24.04
    # Temporarily disabled:
    if: false
    # if: |
    #   needs.check_changesets.outputs.changeset_outputs_hasChangesets == 'false' &&
    #   needs.publish-app.outputs.app_was_pushed == 'true'
    steps:
      - name: Checkout
        uses: actions/checkout@v6
      - name: Load Environment Variables from .env
        uses: xom9ikk/dotenv@v2
      - name: Get Downstream App Installation Token
        id: auth
        uses: actions/create-github-app-token@v2
        with:
          app-id: ${{ secrets.DOWNSTREAM_CH_APP_ID }}
          private-key: ${{ secrets.DOWNSTREAM_CH_APP_PRIVATE_KEY }}
          owner: ${{ secrets.DOWNSTREAM_CH_OWNER }}
      - name: Notify CH Downstream
        uses: actions/github-script@v9
        continue-on-error: true
        env:
          TAG: ${{ env.IMAGE_VERSION }}${{ env.IMAGE_VERSION_SUB_TAG }}
        with:
          github-token: ${{ steps.auth.outputs.token }}
          script: |
            const { TAG } = process.env;
            const result = await github.rest.actions.createWorkflowDispatch({
              owner: '${{ secrets.DOWNSTREAM_CH_OWNER }}',
              repo: '${{ secrets.DOWNSTREAM_DP_REPO }}',
              workflow_id: '${{ secrets.DOWNSTREAM_DP_WORKFLOW_ID }}',
              ref: 'main',
              inputs: {
                tag: TAG
              }
            });

  notify_clickhouse_clickstack:
    needs:
      [
        check_changesets,
        publish-app,
        publish-otel-collector,
        publish-local,
        publish-all-in-one,
      ]
    if: needs.publish-app.outputs.app_was_pushed == 'true'
    timeout-minutes: 5
    runs-on: ubuntu-24.04
    steps:
      - name: Checkout
        uses: actions/checkout@v6
      - name: Load Environment Variables from .env
        uses: xom9ikk/dotenv@v2
      - name: Notify ClickHouse/clickhouse-clickstack Downstream
        uses: actions/github-script@v9
        continue-on-error: true
        env:
          TAG: ${{ env.IMAGE_VERSION }}${{ env.IMAGE_VERSION_SUB_TAG }}
        with:
          github-token: ${{ secrets.CH_BOT_PAT }}
          script: |
            const { TAG } = process.env;
            const result = await github.rest.actions.createWorkflowDispatch({
              owner: 'ClickHouse',
              repo: 'clickhouse-clickstack',
              workflow_id: 'sync-hyperdx.yml',
              ref: 'main',
              inputs: {
                tag: TAG,
              }
            });

  # ---------------------------------------------------------------------------
  # Failure notification
  # ---------------------------------------------------------------------------
  slack-notify-failure:
    needs:
      [
        check_changesets,
        release_changelog_draft,
        release_changelog_publish,
        publish-app,
        publish-otel-collector,
        publish-local,
        publish-all-in-one,
        release-cli,
        notify_helm_charts,
        notify_ch,
        notify_clickhouse_clickstack,
      ]
    runs-on: ubuntu-24.04
    if: failure()
    steps:
      - name: Get failed steps
        id: get_failed_steps
        uses: actions/github-script@v9
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          script: |
            const { data } = await github.rest.actions.listJobsForWorkflowRun({
              owner: context.repo.owner,
              repo: context.repo.repo,
              run_id: context.runId,
            });

            const failedSteps = data.jobs
              .filter(job => job.conclusion === 'failure')
              .flatMap(job => {
                const stepNames = (job.steps ?? [])
                  .filter(step => step.conclusion === 'failure')
                  .map(step => `${job.name} → ${step.name}`);
                return stepNames.length > 0 ? stepNames : [`${job.name} → (unknown step)`];
              })
              .join(', ');

            core.setOutput('failed_steps', failedSteps || 'unknown');
      - name: Slack Notification
        uses: 8398a7/action-slack@v3
        with:
          status: custom
          fields: repo,workflow
          custom_payload: |
            {
              "text": "Release workflow failed!",
              "attachments": [{
                "color": "danger",
                "fields": [
                  {
                    "title": "Failed step(s)",
                    "value": "${{ steps.get_failed_steps.outputs.failed_steps }}",
                    "short": false
                  },
                  {
                    "title": "Commit",
                    "value": "<https://github.com/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}>",
                    "short": false
                  },
                  {
                    "title": "Actor",
                    "value": "${{ github.actor }}",
                    "short": true
                  },
                  {
                    "title": "Run",
                    "value": "<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.run_id }}>",
                    "short": true
                  }
                ]
              }]
            }
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_ENG_NOTIFS }}
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

  otel-cicd-action:
    if: always()
    name: OpenTelemetry Export Trace
    runs-on: ubuntu-latest
    needs:
      [
        check_changesets,
        publish-app,
        publish-otel-collector,
        publish-local,
        publish-all-in-one,
        release-cli,
        notify_helm_charts,
        notify_ch,
        notify_clickhouse_clickstack,
        slack-notify-failure,
      ]
    steps:
      - name: Export workflow
        uses: corentinmusard/otel-cicd-action@v4
        with:
          otlpEndpoint: ${{ secrets.OTLP_ENDPOINT }}/v1/traces
          otlpHeaders: ${{ secrets.OTLP_HEADERS }}
          otelServiceName: 'release-hyperdx-oss-workflow'
          githubToken: ${{ secrets.GITHUB_TOKEN }}