_soak.yml

# Reusable: soak tests (quick + ASan, all platforms)
name: Soak

on:
  workflow_call:
    inputs:
      duration_minutes:
        description: 'Soak duration in minutes (per leg)'
        type: number
        default: 10
      legs:
        description: "Soak legs for the quick jobs, passed to scripts/soak-legs.sh --legs (e.g. 'quick,query-leak' or 'query-leak' for a long #581-only run)"
        type: string
        default: 'quick,query-leak'
      run_asan:
        description: 'Run ASan soak in addition to quick soak'
        type: boolean
        default: false
      version:
        description: 'Version string for build'
        type: string
        default: ''
      use_release_artifacts:
        description: 'Soak the already selected binaries-* artifacts instead of rebuilding quick-lane binaries'
        type: boolean
        default: false

permissions:
  contents: read

jobs:
  soak-quick:
    strategy:
      fail-fast: false
      matrix:
        include:
          - os: ubuntu-latest
            goos: linux
            goarch: amd64
            cc: gcc
            cxx: g++
          - os: ubuntu-24.04-arm
            goos: linux
            goarch: arm64
            cc: gcc
            cxx: g++
          - os: macos-14
            goos: darwin
            goarch: arm64
            cc: cc
            cxx: c++
          - os: macos-15-intel
            goos: darwin
            goarch: amd64
            cc: cc
            cxx: c++
    runs-on: ${{ matrix.os }}
    # Budget doctrine (this line has been silently violated TWICE): the job
    # timeout must exceed the WORST-CASE run length, and duration_minutes is
    # per LEG. Two 240-min legs (the nightly) need ~480 min + build + analysis;
    # the previous 300 silently killed the second leg at the timeout — the same
    # truncation class as the original hard-coded 30. Keep a literal (the
    # inputs context is not reliably available at job-setup time): 560.
    timeout-minutes: 560
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
      - name: Install deps (Linux)
        if: startsWith(matrix.os, 'ubuntu')
        run: sudo apt-get update && sudo apt-get install -y zlib1g-dev python3 git
      - name: Build
        if: ${{ !inputs.use_release_artifacts }}
        run: scripts/build.sh ${{ inputs.version && format('--version {0}', inputs.version) || '' }} CC=${{ matrix.cc }} CXX=${{ matrix.cxx }}
      - name: Download selected release container
        if: ${{ inputs.use_release_artifacts }}
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: binaries-${{ matrix.goos }}-${{ matrix.goarch }}
          path: ${{ runner.temp }}/cbm-selected-container
      - name: Extract selected release executable
        if: ${{ inputs.use_release_artifacts }}
        run: |
          mkdir -p "$RUNNER_TEMP/cbm-selected-artifact"
          tar -xzf "$RUNNER_TEMP/cbm-selected-container/codebase-memory-mcp-${{ matrix.goos }}-${{ matrix.goarch }}.tar.gz" \
            -C "$RUNNER_TEMP/cbm-selected-artifact" codebase-memory-mcp
          chmod +x "$RUNNER_TEMP/cbm-selected-artifact/codebase-memory-mcp"
          "$RUNNER_TEMP/cbm-selected-artifact/codebase-memory-mcp" --version
      # The soak SEQUENCE and its guards live in the ONE canonical entry
      # scripts/soak-legs.sh — never in workflow YAML. Leg selection is a
      # workflow input so a long #581-only run needs no separate workflow.
      - name: Soak legs (${{ inputs.legs }}, ${{ inputs.duration_minutes }} min each)
        env:
          SELECTED_BINARY: ${{ runner.temp }}/cbm-selected-artifact/codebase-memory-mcp
          USE_RELEASE_ARTIFACTS: ${{ inputs.use_release_artifacts }}
        run: |
          binary=build/c/codebase-memory-mcp
          test "$USE_RELEASE_ARTIFACTS" != true || binary="$SELECTED_BINARY"
          scripts/soak-legs.sh --legs "${{ inputs.legs }}" "$binary" ${{ inputs.duration_minutes }}
      - name: Upload metrics
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: soak-quick-${{ matrix.goos }}-${{ matrix.goarch }}
          path: soak-results/
          retention-days: 14
      - name: Upload query-leak metrics
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: soak-query-leak-${{ matrix.goos }}-${{ matrix.goarch }}
          path: soak-results-query-leak/
          retention-days: 14

  soak-quick-windows:
    runs-on: windows-latest
    # Budget doctrine (see soak-quick): duration_minutes is per LEG, so two
    # 240-min legs need ~480 min + build; 300 silently killed the second leg.
    timeout-minutes: 560
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - uses: msys2/setup-msys2@66cd2cce69caa17b53920067426061ca1de3a884 # v2
        with:
          msystem: CLANG64
          path-type: inherit
          install: >-
            mingw-w64-clang-x86_64-clang
            mingw-w64-clang-x86_64-zlib
            mingw-w64-clang-x86_64-python3
            make
            git
            coreutils
            unzip
      # Same protected root the test legs and the VM soak use. Without it the
      # soak ran under the runner's inherited LocalAppData\Temp ACLs while the
      # local soak ran hardened — the two venues were soaking different shapes.
      - name: Create protected per-user temp root
        shell: pwsh
        run: |
          $root = & scripts/ci/new-protected-temp-root.ps1 `
            -Prefix 'cbm-ci-tmp-' `
            -ProtectDir (Join-Path $env:GITHUB_WORKSPACE "build\c")
          "CBM_CI_TEMP_ROOT=$root" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
      - name: Build
        if: ${{ !inputs.use_release_artifacts }}
        shell: msys2 {0}
        run: scripts/build.sh ${{ inputs.version && format('--version {0}', inputs.version) || '' }} CC=clang CXX=clang++
      - name: Download selected release container
        if: ${{ inputs.use_release_artifacts }}
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: binaries-windows-amd64
          path: ${{ runner.temp }}/cbm-selected-container
      - name: Extract selected release executable
        if: ${{ inputs.use_release_artifacts }}
        shell: msys2 {0}
        run: |
          ARTIFACT_DIR="$(cygpath -u "$RUNNER_TEMP")/cbm-selected-artifact"
          mkdir -p "$ARTIFACT_DIR"
          unzip -q "$(cygpath -u "$RUNNER_TEMP")/cbm-selected-container/codebase-memory-mcp-windows-amd64.zip" \
            codebase-memory-mcp.exe -d "$ARTIFACT_DIR"
          "$ARTIFACT_DIR/codebase-memory-mcp.exe" --version
      # Both release-gating legs via the ONE canonical soak entry; only the
      # protected-TEMP provisioning is venue text. soak-legs.sh resolves .exe.
      - name: Soak legs (${{ inputs.legs }}, ${{ inputs.duration_minutes }} min each)
        shell: msys2 {0}
        run: |
          export TEMP="$(cygpath -m "$CBM_CI_TEMP_ROOT")"
          export TMP="$TEMP"
          export TMPDIR="$(cygpath -u "$CBM_CI_TEMP_ROOT")"
          binary=build/c/codebase-memory-mcp
          if [ "${{ inputs.use_release_artifacts }}" = true ]; then
            binary="$(cygpath -u "$RUNNER_TEMP")/cbm-selected-artifact/codebase-memory-mcp.exe"
          fi
          scripts/soak-legs.sh --legs "${{ inputs.legs }}" "$binary" ${{ inputs.duration_minutes }}
      - name: Upload metrics
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: soak-quick-windows-amd64
          path: soak-results/
          retention-days: 14
      - name: Upload query-leak metrics
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: soak-query-leak-windows-amd64
          path: soak-results-query-leak/
          retention-days: 14

  soak-quick-windows-arm64:
    # Native ARM64 Windows soak (CLANGARM64, no sanitizer — ASan is unavailable
    # on native ARM64 Windows). Builds from source like soak-quick-windows.
    runs-on: windows-11-arm
    # Budget doctrine (see soak-quick): per-leg duration, two legs, 560.
    timeout-minutes: 560
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - uses: msys2/setup-msys2@66cd2cce69caa17b53920067426061ca1de3a884 # v2
        with:
          msystem: CLANGARM64
          path-type: inherit
          install: >-
            mingw-w64-clang-aarch64-clang
            mingw-w64-clang-aarch64-zlib
            mingw-w64-clang-aarch64-python3
            make
            git
            coreutils
            unzip
      # Same protected root the test legs and the VM soak use. Without it the
      # soak ran under the runner's inherited LocalAppData\Temp ACLs while the
      # local soak ran hardened — the two venues were soaking different shapes.
      - name: Create protected per-user temp root
        shell: pwsh
        run: |
          $root = & scripts/ci/new-protected-temp-root.ps1 `
            -Prefix 'cbm-ci-tmp-' `
            -ProtectDir (Join-Path $env:GITHUB_WORKSPACE "build\c")
          "CBM_CI_TEMP_ROOT=$root" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
      - name: Build
        if: ${{ !inputs.use_release_artifacts }}
        shell: msys2 {0}
        run: scripts/build.sh ${{ inputs.version && format('--version {0}', inputs.version) || '' }} CC=clang CXX=clang++
      - name: Download selected release container
        if: ${{ inputs.use_release_artifacts }}
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: binaries-windows-arm64
          path: ${{ runner.temp }}/cbm-selected-container
      - name: Extract selected release executable
        if: ${{ inputs.use_release_artifacts }}
        shell: msys2 {0}
        run: |
          ARTIFACT_DIR="$(cygpath -u "$RUNNER_TEMP")/cbm-selected-artifact"
          mkdir -p "$ARTIFACT_DIR"
          unzip -q "$(cygpath -u "$RUNNER_TEMP")/cbm-selected-container/codebase-memory-mcp-windows-arm64.zip" \
            codebase-memory-mcp.exe -d "$ARTIFACT_DIR"
          "$ARTIFACT_DIR/codebase-memory-mcp.exe" --version
      # Both release-gating legs via the ONE canonical soak entry; only the
      # protected-TEMP provisioning is venue text. soak-legs.sh resolves .exe.
      - name: Soak legs (${{ inputs.legs }}, ${{ inputs.duration_minutes }} min each)
        shell: msys2 {0}
        run: |
          export TEMP="$(cygpath -m "$CBM_CI_TEMP_ROOT")"
          export TMP="$TEMP"
          export TMPDIR="$(cygpath -u "$CBM_CI_TEMP_ROOT")"
          binary=build/c/codebase-memory-mcp
          if [ "${{ inputs.use_release_artifacts }}" = true ]; then
            binary="$(cygpath -u "$RUNNER_TEMP")/cbm-selected-artifact/codebase-memory-mcp.exe"
          fi
          scripts/soak-legs.sh --legs "${{ inputs.legs }}" "$binary" ${{ inputs.duration_minutes }}
      - name: Upload metrics
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: soak-quick-windows-arm64
          path: soak-results/
          retention-days: 14
      - name: Upload query-leak metrics
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: soak-query-leak-windows-arm64
          path: soak-results-query-leak/
          retention-days: 14

  # Portable Linux is a separate immutable product tuple. It participates when
  # a caller supplies selected release artifacts; the source-building nightly
  # retains its historical matrix and has no upstream binaries-* artifacts.
  soak-quick-linux-portable:
    if: ${{ inputs.use_release_artifacts }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - arch: amd64
            runner: ubuntu-latest
          - arch: arm64
            runner: ubuntu-24.04-arm
    runs-on: ${{ matrix.runner }}
    timeout-minutes: 560
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
      - name: Install soak dependencies
        run: sudo apt-get update && sudo apt-get install -y python3 git
      - name: Download selected portable release container
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: binaries-linux-${{ matrix.arch }}-portable
          path: ${{ runner.temp }}/cbm-selected-container
      - name: Extract selected portable executable
        run: |
          mkdir -p "$RUNNER_TEMP/cbm-selected-artifact"
          tar -xzf "$RUNNER_TEMP/cbm-selected-container/codebase-memory-mcp-linux-${{ matrix.arch }}-portable.tar.gz" \
            -C "$RUNNER_TEMP/cbm-selected-artifact" codebase-memory-mcp
          chmod +x "$RUNNER_TEMP/cbm-selected-artifact/codebase-memory-mcp"
          "$RUNNER_TEMP/cbm-selected-artifact/codebase-memory-mcp" --version
      - name: Soak selected portable legs (${{ inputs.legs }}, ${{ inputs.duration_minutes }} min each)
        run: |
          scripts/soak-legs.sh --legs "${{ inputs.legs }}" \
            "$RUNNER_TEMP/cbm-selected-artifact/codebase-memory-mcp" \
            ${{ inputs.duration_minutes }}
      - name: Upload metrics
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: soak-quick-linux-${{ matrix.arch }}-portable
          path: soak-results/
          retention-days: 14
      - name: Upload query-leak metrics
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: soak-query-leak-linux-${{ matrix.arch }}-portable
          path: soak-results-query-leak/
          retention-days: 14

  soak-asan:
    # Supplemental source-built instrumentation. This can find memory bugs but
    # is not evidence that the selected release bytes were soaked; the quick
    # jobs above own that release gate when use_release_artifacts=true.
    if: ${{ inputs.run_asan }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - os: ubuntu-latest
            goos: linux
            goarch: amd64
            cc: gcc
            cxx: g++
          - os: ubuntu-24.04-arm
            goos: linux
            goarch: arm64
            cc: gcc
            cxx: g++
          - os: macos-14
            goos: darwin
            goarch: arm64
            cc: cc
            cxx: c++
          - os: macos-15-intel
            goos: darwin
            goarch: amd64
            cc: cc
            cxx: c++
    runs-on: ${{ matrix.os }}
    # ASan soak runs a FIXED 15-min soak (hard-coded below, NOT driven by
    # inputs.duration_minutes), but the ASan-instrumented build is slow and
    # leak reporting adds teardown time. 60 keeps the budget comfortably above
    # the 15-min run so it is never truncated. (Same class of bug as the
    # soak-quick 30→240 mismatch above — keep the timeout above the run length.)
    timeout-minutes: 240
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
      - name: Install deps (Linux)
        if: startsWith(matrix.os, 'ubuntu')
        run: sudo apt-get update && sudo apt-get install -y zlib1g-dev python3 git
      - name: Build (ASan)
        run: |
          SANITIZE="-fsanitize=address,undefined -fno-omit-frame-pointer"
          scripts/build.sh ${{ inputs.version && format('--version {0}', inputs.version) || '' }} CC=${{ matrix.cc }} CXX=${{ matrix.cxx }} EXTRA_CFLAGS="$SANITIZE" EXTRA_LDFLAGS="$SANITIZE"
      - name: ASan soak (15 min)
        env:
          ASAN_OPTIONS: "detect_leaks=1:halt_on_error=0:log_path=soak-results/asan"
        # Single-leg variant of the canonical soak entry: the ASan build is too
        # slow for the two-leg sequence; leg selection is a documented flag, not
        # a divergent inline procedure.
        run: scripts/soak-legs.sh --legs quick build/c/codebase-memory-mcp 15
      - name: Upload metrics
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: soak-asan-${{ matrix.goos }}-${{ matrix.goarch }}
          path: soak-results/
          retention-days: 14

  soak-asan-windows:
    if: ${{ inputs.run_asan }}
    runs-on: windows-latest
    # FIXED 15-min soak (hard-coded below). MSYS2/Wine + ASan build is the
    # slowest path; 60 keeps the budget well above the run length so it is
    # never truncated.
    timeout-minutes: 240
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - uses: msys2/setup-msys2@66cd2cce69caa17b53920067426061ca1de3a884 # v2
        with:
          msystem: CLANG64
          path-type: inherit
          install: >-
            mingw-w64-clang-x86_64-clang
            mingw-w64-clang-x86_64-zlib
            mingw-w64-clang-x86_64-python3
            make
            git
            coreutils
      - name: Build (ASan)
        shell: msys2 {0}
        run: |
          SANITIZE="-fsanitize=address,undefined -fno-omit-frame-pointer"
          scripts/build.sh ${{ inputs.version && format('--version {0}', inputs.version) || '' }} CC=clang CXX=clang++ EXTRA_CFLAGS="$SANITIZE" EXTRA_LDFLAGS="$SANITIZE"
      - name: ASan soak (15 min, no leak detection)
        shell: msys2 {0}
        env:
          ASAN_OPTIONS: "detect_leaks=0:halt_on_error=0:log_path=soak-results/asan"
        # Single-leg variant of the canonical soak entry (see the unix ASan job).
        run: scripts/soak-legs.sh --legs quick build/c/codebase-memory-mcp 15
      - name: Upload metrics
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: soak-asan-windows-amd64
          path: soak-results/
          retention-days: 14