build-gfx11-rocm.yml

name: Build gfx11 + ROCm

on:
  push:
    branches: [gfx11]
  pull_request:
    types: [opened, synchronize, reopened]
  workflow_dispatch:
    inputs:
      build_set:
        description: 'Which variants to build: nightly (A: gfx11 + latest nightly), weekly (B+C: master + gfx11 on the frozen 7.14 release), or all.'
        required: false
        default: 'nightly'
        type: choice
        options: [nightly, weekly, all]
      rocm_version:
        description: 'Override ROCm version for ALL built variants (e.g. 7.14.0a20260608). Empty = per-variant channel auto-detect.'
        required: false
        default: ''
      create_release:
        description: 'Publish a dated per-variant GitHub Release (<base>-YYYYMMDD). Set true only for cron dispatches.'
        required: false
        default: 'false'
      roofline:
        description: 'Build with per-op roofline profiling support (GGML_HIP_ROOFLINE) and bundle the profiler libs (librocprofiler-sdk + libdw). Set false for lean builds.'
        required: false
        default: 'true'

# Three build variants (see the `setup` job matrix). Each publishes a DATED,
# persistent release tag (<base>-YYYYMMDD) so older builds stay downloadable for
# issue repro; prune old releases manually if storage becomes a concern.
#   A  gfx11  + latest nightly ROCm  -> gfx11-rocm-nightly-YYYYMMDD  (nightly)
#   B  master + 7.14 release ROCm    -> master-rocm-714-YYYYMMDD     (weekly)
#   C  gfx11  + 7.14 release ROCm    -> gfx11-rocm-714-YYYYMMDD      (weekly)
#
# ROCm channels resolve against TheRock's multiarch tarball host:
#   latest -> highest therock-dist-linux-multiarch-*.tar.gz overall (7.15 nightly today)
#   7.14   -> highest therock-dist-linux-multiarch-7.14.0a*.tar.gz. The 7.14 line is
#            FROZEN (last build 7.14.0a20260624, when 7.15 alphas began), so "7.14
#            release" is that final alpha; only the llama.cpp source changes weekly.
# An explicit rocm_version input overrides the channel for every variant.
#
# push/PR build only variant A's shape (triggering ref + latest) and never release.
# The 7.14 variants build via external cron (GitHub cron only fires from the default
# branch). On a host you control, with `gh` authenticated (token scope: repo + workflow):
#   # nightly (variant A)
#   7 13 * * *  gh workflow run build-gfx11-rocm.yml --repo AMD-Ecosystem/llama.cpp --ref gfx11 -f build_set=nightly -f create_release=true
#   # weekly Thursday (variants B + C), ready ahead of weekend regressions
#   7  9 * * 4  gh workflow run build-gfx11-rocm.yml --repo AMD-Ecosystem/llama.cpp --ref gfx11 -f build_set=weekly  -f create_release=true

env:
  # Default ON so push/PR/cron (non-dispatch) runs keep roofline, matching prior
  # always-on behavior; a workflow_dispatch can pass 'false' for a lean build.
  ROOFLINE: ${{ github.event.inputs.roofline || 'true' }}

jobs:
  setup:
    runs-on: ubuntu-24.04
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
    steps:
    - name: Compute build matrix
      id: set-matrix
      env:
        EVENT_NAME: ${{ github.event_name }}
        BUILD_SET: ${{ github.event.inputs.build_set || 'nightly' }}
        ROCM_OVERRIDE: ${{ github.event.inputs.rocm_version || '' }}
      run: |
        # Variant definitions. ref="" means "the ref that triggered this run"
        # (used for push/PR so CI builds the pushed code, not gfx11 tip).
        # rocm_channel is overridden wholesale when ROCM_OVERRIDE is set.
        # Each variant publishes a DATED, persistent release tag (<base>-YYYYMMDD)
        # so older nightlies stay reproducible; prune manually if storage grows.
        d="$(date -u +%Y%m%d)"
        a_ch="latest"; b_ch="7.14"; c_ch="7.14"
        if [ -n "$ROCM_OVERRIDE" ]; then
          a_ch="$ROCM_OVERRIDE"; b_ch="$ROCM_OVERRIDE"; c_ch="$ROCM_OVERRIDE"
        fi
        A="{\"variant\":\"A\",\"ref\":\"\",\"rocm_channel\":\"$a_ch\",\"tag\":\"gfx11-rocm-nightly-$d\"}"
        B="{\"variant\":\"B\",\"ref\":\"master\",\"rocm_channel\":\"$b_ch\",\"tag\":\"master-rocm-714-$d\"}"
        C="{\"variant\":\"C\",\"ref\":\"gfx11\",\"rocm_channel\":\"$c_ch\",\"tag\":\"gfx11-rocm-714-$d\"}"

        if [ "$EVENT_NAME" = "workflow_dispatch" ]; then
          case "$BUILD_SET" in
            nightly) entries="$A" ;;
            weekly)  entries="$B,$C" ;;
            all)     entries="$A,$B,$C" ;;
            *)       echo "Unknown build_set: $BUILD_SET" >&2; exit 1 ;;
          esac
        else
          # push / pull_request: single leg, triggering ref (ref=""), latest ROCm.
          entries="$A"
        fi

        matrix="{\"include\":[$entries]}"
        echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
        echo "Build matrix: $matrix"

  build-ubuntu:
    needs: setup
    runs-on: ubuntu-24.04
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.setup.outputs.matrix) }}
    # Single multiarch build per variant: one fat binary covering all current CI
    # arches, sourced from TheRock's multiarch tarball (arch-neutral host +
    # per-arch Tensile DBs). gfx1100-1103 (RDNA3 desktop + Hawk Point/Phoenix
    # 760M/780M), gfx1150/1151/1152/1153 (RDNA3.5 Strix APUs).
    env:
      GPU_TARGETS: gfx1100;gfx1101;gfx1102;gfx1103;gfx1150;gfx1151;gfx1152;gfx1153
      ROCM_VERSION: ${{ matrix.rocm_channel }}

    steps:
    - name: Free disk space
      # Remove unused runner files to free up disk space
      run: curl -fsSL https://raw.githubusercontent.com/kou/arrow/e49d8ae15583ceff03237571569099a6ad62be32/ci/scripts/util_free_space.sh | bash

    - name: Checkout repository
      uses: actions/checkout@v4
      with:
        # Empty ref (variant A on push/PR) checks out the triggering ref; B/C
        # pin master / gfx11 explicitly. Full history so rev-parse is meaningful.
        ref: ${{ matrix.ref }}
        fetch-depth: 0

    - name: Set up Node.js (for llama-ui build from source)
      uses: actions/setup-node@v6
      with:
        node-version: "24"

    - name: Clean up existing ROCm directory (safety precaution)
      run: |
        if [ -d "/opt/rocm" ]; then
          echo "Removing existing /opt/rocm directory..."
          sudo rm -rf /opt/rocm
        fi
        if [ -f "rocm.tar.gz" ]; then
          rm -f rocm.tar.gz
        fi
        echo "Cleanup completed successfully"

    - name: Install build dependencies
      run: |
        echo "Installing build dependencies..."
        sudo apt update
        sudo apt install -y cmake ninja-build unzip curl
        echo "Verifying installations..."
        cmake --version
        ninja --version
        echo "Build dependencies installation completed"

    - name: Download and extract multiarch ROCm directly to /opt/rocm
      run: |
        rocm_version="${{ env.ROCM_VERSION }}"
        base_url="https://rocm.nightlies.amd.com/tarball-multi-arch"

        # Channel resolution:
        #   latest        -> highest multiarch tarball overall
        #   X.Y (e.g 7.14) -> highest tarball on that minor line (frozen 7.14 ->
        #                    deterministic). Matched on the "multiarch-X.Y." prefix.
        #   X.Y.Z(a|rc)N  -> pinned full version, used verbatim (no detection)
        detect=false
        version_prefix=""
        if [ "$rocm_version" = "latest" ]; then
          detect=true
        elif [[ "$rocm_version" =~ ^[0-9]+\.[0-9]+$ ]]; then
          detect=true
          version_prefix="therock-dist-linux-multiarch-${rocm_version}."
        fi

        if [ "$detect" = true ]; then
          echo "Auto-detecting multiarch ROCm version (channel: $rocm_version)"
          # The multiarch host serves an HTML index (not S3 XML); scrape the
          # multiarch tarball names, then (for a X.Y channel) keep only that line.
          files=$(curl -s "$base_url/" \
            | grep -oE 'therock-dist-linux-multiarch-[0-9]+\.[0-9]+\.[0-9]+(a|rc)[0-9]+\.tar\.gz' \
            | sort -u)
          if [ -n "$version_prefix" ]; then
            files=$(printf '%s\n' "$files" | grep -F "$version_prefix" || true)
          fi

          if [ -z "$files" ]; then
            echo "No multiarch tarballs found for channel '$rocm_version'" >&2
            exit 1
          fi

          latest_file=""
          latest_major=0
          latest_minor=0
          latest_patch=0
          latest_rc=0
          latest_is_alpha=false

          while IFS= read -r file; do
            if [[ "$file" =~ therock-dist-linux-multiarch-([0-9]+\.[0-9]+\.[0-9]+(a|rc)[0-9]+)\.tar\.gz ]]; then
              version="${BASH_REMATCH[1]}"
              major=$(echo "$version" | cut -d. -f1)
              minor=$(echo "$version" | cut -d. -f2)
              patch=$(echo "$version" | cut -d. -f3 | sed 's/\(a\|rc\).*//')
              rc=$(echo "$version" | sed 's/.*\(a\|rc\)//')
              is_alpha=false
              if [[ "$version" =~ a ]]; then
                is_alpha=true
              fi

              is_newer=false
              if [ "$major" -gt "$latest_major" ]; then
                is_newer=true
              elif [ "$major" -eq "$latest_major" ] && [ "$minor" -gt "$latest_minor" ]; then
                is_newer=true
              elif [ "$major" -eq "$latest_major" ] && [ "$minor" -eq "$latest_minor" ] && [ "$patch" -gt "$latest_patch" ]; then
                is_newer=true
              elif [ "$major" -eq "$latest_major" ] && [ "$minor" -eq "$latest_minor" ] && [ "$patch" -eq "$latest_patch" ]; then
                if [ "$is_alpha" = true ] && [ "$latest_is_alpha" = false ]; then
                  is_newer=true
                elif [ "$is_alpha" = "$latest_is_alpha" ] && [ "$rc" -gt "$latest_rc" ]; then
                  is_newer=true
                fi
              fi

              if [ "$is_newer" = true ]; then
                latest_file="$file"
                latest_major="$major"
                latest_minor="$minor"
                latest_patch="$patch"
                latest_rc="$rc"
                latest_is_alpha="$is_alpha"
              fi
            fi
          done <<< "$files"

          echo "Found latest file: $latest_file"

          if [[ "$latest_file" =~ therock-dist-linux-multiarch-([0-9]+\.[0-9]+\.[0-9]+(a|rc)[0-9]+)\.tar\.gz ]]; then
            rocm_version="${BASH_REMATCH[1]}"
            echo "Detected latest ROCm version: $rocm_version"
          else
            echo "Failed to extract ROCm version from latest file: $latest_file"
            echo "Expected pattern: therock-dist-linux-multiarch-<version>.tar.gz"
            exit 1
          fi
        fi

        rocm_url="$base_url/therock-dist-linux-multiarch-${rocm_version}.tar.gz"
        echo "DETECTED_ROCM_VERSION=$rocm_version" >> $GITHUB_ENV

        # The multiarch tarball (~11.5 GB) ships device code for ALL 26 GPU
        # arches: every arch's .kpack plus per-arch rocBLAS/hipBLASLt Tensile DBs.
        # This consumer build only needs the 8 RDNA3/3.5 arches in GPU_TARGETS and
        # uses the GEMM (Tensile) path, which works without .kpack files. So we
        # stream-extract and prune at the tar level: drop ALL .kpack, and drop the
        # Tensile DBs of every arch not in our target set. This keeps the runner
        # disk footprint small (the 11.5 GB is streamed, never stored) and yields
        # a lean multiarch package. tar matches --exclude on pre-strip member
        # names, hence the leading "./".
        drop_arches="gfx900 gfx906 gfx908 gfx90a gfx942 gfx950 \
          gfx1010 gfx1011 gfx1012 gfx1030 gfx1031 gfx1032 gfx1033 gfx1034 gfx1035 gfx1036 \
          gfx1200 gfx1201"
        excludes=(--exclude='./.kpack' --exclude='./.kpack/*')
        for a in $drop_arches; do
          excludes+=("--exclude=./lib/*/library/${a}")
          excludes+=("--exclude=./lib/*/library/${a}/*")
          excludes+=("--exclude=./lib/*/library/*${a}*")
        done

        echo "Streaming multiarch ROCm from: $rocm_url (pruning .kpack + non-target arches)"
        sudo mkdir -p /opt/rocm
        curl -sL "$rocm_url" | sudo tar --use-compress-program=gzip -xf - \
          -C /opt/rocm --strip-components=1 "${excludes[@]}"

        echo "Retained rocBLAS Tensile arch dirs:"
        ls /opt/rocm/lib/rocblas/library/ 2>/dev/null || echo "(none)"
        echo "Retained hipBLASLt Tensile arch dirs:"
        ls /opt/rocm/lib/hipblaslt/library/ 2>/dev/null || echo "(none)"

    - name: Set ROCm environment variables
      run: |
        echo "Setting ROCm environment variables..."
        echo "HIP_PATH=/opt/rocm" >> $GITHUB_ENV
        echo "ROCM_PATH=/opt/rocm" >> $GITHUB_ENV
        echo "HIP_PLATFORM=amd" >> $GITHUB_ENV
        echo "HIP_CLANG_PATH=/opt/rocm/llvm/bin" >> $GITHUB_ENV
        echo "HIP_INCLUDE_PATH=/opt/rocm/include" >> $GITHUB_ENV
        echo "HIP_LIB_PATH=/opt/rocm/lib" >> $GITHUB_ENV
        echo "HIP_DEVICE_LIB_PATH=/opt/rocm/lib/llvm/amdgcn/bitcode" >> $GITHUB_ENV
        echo "/opt/rocm/bin:/opt/rocm/llvm/bin:$PATH" >> $GITHUB_PATH
        echo "LD_LIBRARY_PATH=/opt/rocm/lib:/opt/rocm/lib64:/opt/rocm/llvm/lib:${LD_LIBRARY_PATH:-}" >> $GITHUB_ENV
        echo "LIBRARY_PATH=/opt/rocm/lib:/opt/rocm/lib64:${LIBRARY_PATH:-}" >> $GITHUB_ENV
        echo "CPATH=/opt/rocm/include:${CPATH:-}" >> $GITHUB_ENV
        echo "PKG_CONFIG_PATH=/opt/rocm/lib/pkgconfig:${PKG_CONFIG_PATH:-}" >> $GITHUB_ENV
        echo "ROCm environment variables set successfully"

    - name: Record llama.cpp commit hash
      run: |
        commit_hash=$(git rev-parse --short=5 HEAD)
        echo "LLAMACPP_COMMIT_HASH=$commit_hash" >> $GITHUB_ENV
        echo "llama.cpp commit hash (5 digits): $commit_hash"
        echo "Current llama.cpp commit:"
        git log --oneline -1

    - name: Setup ccache
      # Auto-consumed by the build: llama.cpp's CMake sets ccache as the compile
      # launcher when GGML_CCACHE is ON (default) and ccache is on PATH, so no
      # cmake flags are needed. Persists across runs via actions/cache. Nightly
      # ROCm bumps change the compiler and will miss (expected); incremental
      # commits on the same ROCm hit. Only write the cache from pushes to gfx11
      # so PRs read but don't thrash the shared store.
      uses: ggml-org/ccache-action@v1.2.21
      with:
        # Per-variant key: the three variants use different source refs and ROCm
        # compilers, so a shared key would thrash. Keep each on its own store.
        key: gfx11-rocm-multiarch-${{ matrix.variant }}
        evict-old-files: 7d
        save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/gfx11' }}

    - name: Build Llama.cpp + ROCm
      run: |
        gpu_targets="${{ env.GPU_TARGETS }}"
        echo "Building multiarch binary (GPU_TARGETS=$gpu_targets)"

        mkdir build
        cd build

        cmake .. -G Ninja \
          -DCMAKE_C_COMPILER=/opt/rocm/llvm/bin/clang \
          -DCMAKE_CXX_COMPILER=/opt/rocm/llvm/bin/clang++ \
          -DCMAKE_CXX_FLAGS="-I/opt/rocm/include" \
          -DCMAKE_CROSSCOMPILING=ON \
          -DCMAKE_BUILD_TYPE=Release \
          -DGPU_TARGETS="$gpu_targets" \
          -DBUILD_SHARED_LIBS=ON \
          -DLLAMA_BUILD_TESTS=OFF \
          -DGGML_HIP=ON \
          -DGGML_OPENMP=OFF \
          -DGGML_CUDA_FORCE_CUBLAS=OFF \
          -DGGML_RPC=ON \
          -DGGML_HIP_ROCWMMA_FATTN=OFF \
          -DGGML_HIP_ROOFLINE=${{ env.ROOFLINE == 'true' && 'ON' || 'OFF' }} \
          -DLLAMA_BUILD_BORINGSSL=ON \
          -DGGML_NATIVE=OFF \
          -DGGML_STATIC=OFF \
          -DCMAKE_SYSTEM_NAME=Linux

        cmake --build . -j $(nproc)

    - name: Copy ROCm core libs to build directory
      run: |
        build_bin_path="build/bin"

        # Copy the rocblas/library folder and all its contents
        rocblas_lib_path="/opt/rocm/lib/rocblas/library"
        if [ -d "$rocblas_lib_path" ]; then
          echo "Copying rocblas/library folder and all contents..."
          dest_rocblas_path="$build_bin_path/rocblas/library"
          mkdir -p "$(dirname "$dest_rocblas_path")"
          cp -r "$rocblas_lib_path" "$(dirname "$dest_rocblas_path")/"
          echo "Copied: rocblas/library folder with all contents"
        else
          echo "Warning: rocblas/library folder not found at: $rocblas_lib_path"
        fi

        # Copy the hipblaslt/library folder and all its contents
        hipblaslt_lib_path="/opt/rocm/lib/hipblaslt/library"
        if [ -d "$hipblaslt_lib_path" ]; then
          echo "Copying hipblaslt/library folder and all contents..."
          dest_hipblaslt_path="$build_bin_path/hipblaslt/library"
          mkdir -p "$(dirname "$dest_hipblaslt_path")"
          cp -r "$hipblaslt_lib_path" "$(dirname "$dest_hipblaslt_path")/"
          echo "Copied: hipblaslt/library folder with all contents"
        else
          echo "Warning: hipblaslt/library folder not found at: $hipblaslt_lib_path"
        fi

        # Copy required ROCm libraries to build directory
        # If artifacts from ROCm or Llama.cpp change, you may need to update this list.
        echo "Copying required ROCm libraries to build directory..."
        cp -v /opt/rocm/lib/libhipblas.so* "$build_bin_path/" 2>/dev/null || echo "libhipblas.so* not found"
        cp -v /opt/rocm/lib/librocblas.so* "$build_bin_path/" 2>/dev/null || echo "librocblas.so* not found"
        cp -v /opt/rocm/lib/libamdhip64.so* "$build_bin_path/" 2>/dev/null || echo "libamdhip64.so* not found"
        cp -v /opt/rocm/lib/librocsolver.so* "$build_bin_path/" 2>/dev/null || echo "librocsolver.so* not found"
        cp -v /opt/rocm/lib/libroctx64.so* "$build_bin_path/" 2>/dev/null || echo "libroctx64.so* not found"
        cp -v /opt/rocm/lib/libhipblaslt.so* "$build_bin_path/" 2>/dev/null || echo "libhipblaslt.so* not found"
        # liborigami is a hard DT_NEEDED of libhipblaslt as of ROCm 7.15 (new
        # hipBLASLt kernel-selection lib). Omitting it makes llama-bench/llama-server
        # fail to load (liborigami.so.1: cannot open shared object file). The
        # transitive-closure backstop below would also catch it, but keep the
        # explicit copy so the intent is greppable alongside the other hipblaslt libs.
        cp -v /opt/rocm/lib/liborigami.so* "$build_bin_path/" 2>/dev/null || echo "liborigami.so* not found"
        cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_liblzma.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_liblzma.so* not found"
        cp -v /opt/rocm/lib/librocprofiler-register.so* "$build_bin_path/" 2>/dev/null || echo "librocprofiler-register.so* not found"
        # Roofline builds dlopen librocprofiler-sdk.so.1 at runtime (ggml-cuda-roofline.cpp);
        # the release must ship it plus its one bundle-missing dep libdw
        # (librocm_sysdeps_dw.so.1 — a hard DT_NEEDED of the sdk lib). The $ORIGIN RPATH
        # step below lets the dlopen resolve both locally with no LD_LIBRARY_PATH.
        if [ "${{ env.ROOFLINE }}" = "true" ]; then
          cp -v /opt/rocm/lib/librocprofiler-sdk.so* "$build_bin_path/" 2>/dev/null || echo "librocprofiler-sdk.so* not found"
          cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_dw.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_dw.so* not found"
        fi
        cp -v /opt/rocm/lib/libamd_comgr.so* "$build_bin_path/" 2>/dev/null || echo "libamd_comgr.so* not found"
        cp -v /opt/rocm/lib/libamd_comgr_loader.so* "$build_bin_path/" 2>/dev/null || echo "libamd_comgr_loader.so* not found"
        cp -v /opt/rocm/lib/libhsa-runtime64.so* "$build_bin_path/" 2>/dev/null || echo "libhsa-runtime64.so* not found"
        cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_numa.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_numa.so* not found"
        cp -v /opt/rocm/lib/librocroller.so* "$build_bin_path/" 2>/dev/null || echo "librocroller.so* not found"
        cp -v /opt/rocm/lib/librocm_kpack.so* "$build_bin_path/" 2>/dev/null || echo "librocm_kpack.so* not found"
        cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_z.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_z.so* not found"
        cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_zstd.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_zstd.so* not found"
        cp -v /opt/rocm/lib/llvm/lib/libLLVM.so* "$build_bin_path/" 2>/dev/null || echo "libLLVM.so* not found"
        cp -v /opt/rocm/lib/llvm/lib/libclang-cpp.so* "$build_bin_path/" 2>/dev/null || echo "libclang-cpp.so* not found"
        cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_elf.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_elf.so* not found"
        cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_drm.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_drm.so* not found"
        cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_drm_amdgpu.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_drm_amdgpu.so* not found"
        cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_bz2.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_bz2.so* not found"

        # Bundle libatomic (gcc runtime dep of llama-cli). Some self-hosted GPU
        # runners don't have libatomic1 installed, so ship it with the artifact;
        # the $ORIGIN RPATH step below makes the binary load this bundled copy.
        cp -v /usr/lib/x86_64-linux-gnu/libatomic.so.1* "$build_bin_path/" 2>/dev/null \
          || cp -v "$(gcc -print-file-name=libatomic.so.1)" "$build_bin_path/" 2>/dev/null \
          || echo "libatomic.so.1 not found"

        # Transitive-closure backstop for the hand-maintained copy list above.
        # A ROCm bump can add a new DT_NEEDED to a bundled lib (e.g. 7.15 added
        # liborigami.so.1 to libhipblaslt); if it isn't in the list, the release
        # ships broken. Here we ldd every bundled binary against the release dir
        # and, for any "=> not found" dep that exists somewhere under
        # /opt/rocm/lib, copy it in — repeating until no new libs are pulled in.
        # Only ROCm-provided deps are auto-bundled; genuinely external libs
        # (libc/libm/... provided by the runner) are left alone.
        echo "Resolving remaining ROCm deps via ldd closure..."
        rocm_lib_roots="/opt/rocm/lib /opt/rocm/lib/rocm_sysdeps/lib /opt/rocm/lib/llvm/lib"
        for pass in 1 2 3 4 5; do
          added=0
          # Collect unresolved sonames across all bundled binaries in one sweep.
          missing="$(cd "$build_bin_path" && \
            for f in *.so* llama-*; do
              [ -f "$f" ] && [ ! -L "$f" ] || continue
              LD_LIBRARY_PATH=. ldd "$f" 2>/dev/null
            done | awk '/=> not found/ {print $1}' | sort -u)"
          [ -z "$missing" ] && { echo "  pass $pass: closure complete"; break; }
          for soname in $missing; do
            # Already bundled? (a previous pass may have added it)
            [ -e "$build_bin_path/$soname" ] && continue
            src=""
            for root in $rocm_lib_roots; do
              cand="$(find "$root" -maxdepth 1 -name "$soname" 2>/dev/null | head -1)"
              [ -n "$cand" ] && { src="$cand"; break; }
            done
            if [ -n "$src" ]; then
              # Copy the real file and recreate the soname symlink chain.
              cp -vL "$src" "$build_bin_path/$soname"
              added=1
            else
              echo "  pass $pass: $soname not found under /opt/rocm — leaving to system loader"
            fi
          done
          [ "$added" -eq 0 ] && { echo "  pass $pass: no ROCm-resolvable deps left"; break; }
        done

        echo "Finished copying required ROCm libraries"

    - name: Set RPATH for portable distribution
      run: |
        sudo apt-get install -y patchelf
        cd build/bin
        # Set RPATH to $ORIGIN so all libraries (including the comgr stub loader) find deps locally
        for file in *.so* llama-*; do
          [ -f "$file" ] && [ ! -L "$file" ] && patchelf --set-rpath '$ORIGIN' "$file" 2>/dev/null || true
        done

    - name: Write build metadata into artifact
      run: |
        # Per-variant provenance travels inside the artifact so downstream matrix
        # jobs (test-gfx, create-release) read it without fragile matrix outputs.
        rocm_version="${DETECTED_ROCM_VERSION:-${{ env.ROCM_VERSION }}}"
        built_sha=$(git rev-parse HEAD)
        {
          echo "variant=${{ matrix.variant }}"
          echo "ref=${{ matrix.ref }}"
          echo "tag=${{ matrix.tag }}"
          echo "rocm_channel=${{ matrix.rocm_channel }}"
          echo "rocm_version=$rocm_version"
          echo "llamacpp_commit_hash=${LLAMACPP_COMMIT_HASH}"
          echo "llamacpp_commit_sha=$built_sha"
        } > build/bin/build-meta.txt
        echo "=== build-meta.txt ==="; cat build/bin/build-meta.txt

    - name: Verify bundled binaries have no unresolved deps
      run: |
        # GPU-free packaging gate. With RPATH=$ORIGIN set, ldd resolves bundled
        # libs from this dir and true system libs from default paths; anything
        # still "not found" means the artifact would fail to load at runtime.
        # This is exactly the failure that shipped in b20260724 (libhipblaslt
        # gained NEEDED liborigami.so.1 which wasn't bundled) — llama-cli happened
        # to load but llama-bench/llama-server did not. Fail the build here so a
        # missing dep can never reach a release again.
        cd build/bin
        status=0
        for f in *.so* llama-*; do
          [ -f "$f" ] && [ ! -L "$f" ] || continue
          missing="$(ldd "$f" 2>/dev/null | awk '/=> not found/ {print $1}')"
          if [ -n "$missing" ]; then
            status=1
            echo "❌ $f has unresolved dependencies:"
            echo "$missing" | sed 's/^/     /'
          fi
        done
        if [ "$status" -ne 0 ]; then
          echo ""
          echo "One or more bundled binaries have unresolved shared-library deps."
          echo "Add the missing lib(s) to the 'Copy ROCm core libs' step (or ensure"
          echo "the ldd-closure backstop can find them under /opt/rocm/lib)."
          exit 1
        fi
        echo "✅ All bundled binaries resolve their dependencies."

    - name: List build artifacts (including ROCm files)
      run: |
        cd build/bin
        echo "Final build artifacts (including ROCm library files):"
        ls -la

    - name: Upload build artifacts
      uses: actions/upload-artifact@v4
      with:
        # Variant-suffixed so the three legs don't collide on one artifact name.
        name: llama-ubuntu-rocm-multiarch-x64-${{ matrix.variant }}
        path: build/bin/
        retention-days: 30

  test-gfx:
    needs: [setup, build-ubuntu]
    # TEMPORARILY DISABLED: the linux-gfx1151-gpu-rocm self-hosted runner is
    # offline, so this job sits queued indefinitely and blocks create-release.
    # Re-enable by restoring `needs.build-ubuntu.result == 'success'` once the
    # runner is back online.
    if: false && needs.build-ubuntu.result == 'success'
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.setup.outputs.matrix) }}
    # Per-variant hardware test of the multiarch artifact on gfx1151. This is the
    # end-to-end safety net for the Tensile-only multiarch package: a real
    # llama-cli inference exercising the rocBLAS/hipBLASLt GEMM path on-device.
    runs-on: linux-gfx1151-gpu-rocm

    steps:
    - name: Checkout repository
      uses: actions/checkout@v4

    - name: Download build artifacts
      uses: actions/download-artifact@v4
      with:
        name: llama-ubuntu-rocm-multiarch-x64-${{ matrix.variant }}
        path: llama-binaries

    - name: Download test model
      run: |
        # Pinned to a fixed GitHub release asset instead of huggingface.co: some
        # self-hosted runners cannot reach huggingface.co (curl 35, connection
        # reset), but they can reach GitHub. Source:
        # https://github.com/jimw567/llamacpp-test-assets/releases/tag/test-assets
        model_url="https://github.com/jimw567/llamacpp-test-assets/releases/download/test-assets/Qwen3-0.6B-Q4_0.gguf"
        model_path="Qwen3-0.6B-Q4_0.gguf"
        echo "Downloading test model from: $model_url"
        curl -fL --retry 5 --retry-all-errors --retry-delay 5 -o "$model_path" "$model_url"
        if [ -f "$model_path" ]; then
          file_size=$(stat -c%s "$model_path")
          echo "Model downloaded successfully. Size: $file_size bytes"
        else
          echo "Failed to download model"
          exit 1
        fi

    - name: Set up library path
      run: |
        echo "LD_LIBRARY_PATH=$(pwd)/llama-binaries:$LD_LIBRARY_PATH" >> $GITHUB_ENV

    - name: Run llama-cli test
      run: |
        llama_cli_path="./llama-binaries/llama-cli"
        model_path="Qwen3-0.6B-Q4_0.gguf"
        output_file="llama_output.txt"

        chmod +x "$llama_cli_path"

        if [ ! -f "$llama_cli_path" ]; then
          echo "llama-cli not found at: $llama_cli_path"
          echo "Available files in llama-binaries:"
          find llama-binaries -type f
          exit 1
        fi

        # Use a prompt with a single correct answer and greedy decoding
        # (--temp 0) so the result is deterministic and verifiable.
        prompt="What is 2 + 2? Reply with only the number."
        echo "Running llama-cli test for gfx1151 (multiarch artifact)..."
        echo "Command: $llama_cli_path -m \"$model_path\" -ngl 99 --temp 0 -p \"$prompt\" -st -v"

        # Bound the run: a healthy 0.6B inference finishes in seconds. If the
        # GPU kernel launch hangs (driver/runner issue) the step would otherwise
        # never end and GitHub would never publish the logs. timeout forces a
        # clean failure and lets the captured output flush for diagnosis.
        set +e
        timeout 180 "$llama_cli_path" -m "$model_path" -ngl 99 --temp 0 -p "$prompt" -st -v > "$output_file" 2>&1
        exit_code=$?
        set -e

        echo "=== LLAMA-CLI OUTPUT ==="
        if [ -f "$output_file" ] && [ -s "$output_file" ]; then
          cat "$output_file"
        else
          echo "(empty)"
        fi
        echo "=== END OUTPUT ==="
        echo "Process exit code: $exit_code"

        if [ $exit_code -eq 124 ]; then
          echo "❌ llama-cli timed out after 180s (GPU kernel likely hung)"
          echo "=== GPU / driver diagnostics ==="
          rocminfo 2>&1 | grep -iE 'Name|Marketing|gfx|Uuid' | head -40 || echo "rocminfo failed"
          (dmesg 2>/dev/null | tail -40) || echo "dmesg unavailable"
          rm -f "$output_file"
          exit 1
        fi

        if [ $exit_code -ne 0 ]; then
          echo "❌ llama-cli exited with error code: $exit_code"
          echo "Checking for missing library dependencies..."
          ldd "$llama_cli_path" || echo "ldd command failed"
          rm -f "$output_file"
          exit 1
        fi

        # Functional checks against current llama.cpp output:
        #  (1) the ROCm GPU was selected,
        #  (2) the model layers were offloaded to it (incl. the output layer),
        #  (3) the model computed the correct answer to "What is 2 + 2?".
        # With greedy decoding (--temp 0) the answer is deterministic, so this
        # verifies the GPU math path end to end, not just that text was emitted.
        found_device=false
        found_offload=false
        found_answer=false

        if grep -q "using device ROCm0" "$output_file"; then
          found_device=true
        fi

        # Qwen3-0.6B has 28 transformer layers + output = 29 GPU assignments per
        # pass (verbose mode logs them more than once; require at least one pass).
        layers_on_gpu=$(grep -c "assigned to device ROCm0" "$output_file" || true)
        if [ "$layers_on_gpu" -ge 29 ] && grep -q "offloading output layer to GPU" "$output_file"; then
          found_offload=true
        fi

        # The parsed assistant answer ("content" field, printed with -v) must
        # contain 4 and no other digit, so "4" / "The answer is 4." pass while
        # "5", "14", "22" fail. This is the deterministic correctness check.
        # llama-cli now runs via an in-process server (upstream #24948); its
        # stream ends with a verbose stop chunk carrying an empty "content":"",
        # so drop empty matches before taking the last real answer chunk.
        answer_content=$(grep -oE '"content":"[^"]*"' "$output_file" | grep -v '"content":""' | tail -1)
        if echo "$answer_content" | grep -qE '4' && ! echo "$answer_content" | grep -qE '[0-35-9]'; then
          found_answer=true
        fi

        echo "=== TEST RESULTS ==="
        echo "ROCm GPU selected ('using device ROCm0'): $(if [ "$found_device" = true ]; then echo 'FOUND'; else echo 'NOT FOUND'; fi)"
        echo "Layers offloaded to GPU ('assigned to device ROCm0' x$layers_on_gpu + output layer): $(if [ "$found_offload" = true ]; then echo 'FOUND'; else echo 'NOT FOUND'; fi)"
        echo "Correct answer to '2 + 2' (parsed content: ${answer_content:-<none>}): $(if [ "$found_answer" = true ]; then echo 'FOUND'; else echo 'NOT FOUND'; fi)"

        rm -f "$output_file"

        if [ "$found_device" = true ] && [ "$found_offload" = true ] \
           && [ "$found_answer" = true ]; then
          echo "✅ Test PASSED - GPU offload + correct deterministic answer verified"
        else
          echo "❌ Test FAILED - Missing expected outputs"
          exit 1
        fi

    - name: Run llama-bench load check
      run: |
        # llama-cli above links hipBLASLt lazily, so it can load even when a
        # hipBLASLt dependency is missing from the bundle. llama-bench (and
        # llama-server) load hipBLASLt eagerly — this is the exact split that
        # let b20260724 pass the cli test but ship a broken bench/server. Run a
        # tiny llama-bench so a missing hipBLASLt dep fails the HW test too, not
        # just the GPU-free packaging gate in the build job.
        llama_bench_path="./llama-binaries/llama-bench"
        model_path="Qwen3-0.6B-Q4_0.gguf"
        chmod +x "$llama_bench_path"
        if [ ! -f "$llama_bench_path" ]; then
          echo "llama-bench not found at: $llama_bench_path"
          exit 1
        fi
        echo "Running minimal llama-bench (pp8/tg8, 1 rep) on gfx1151..."
        set +e
        timeout 180 "$llama_bench_path" -m "$model_path" -ngl 99 -p 8 -n 8 -r 1 > bench_output.txt 2>&1
        exit_code=$?
        set -e
        echo "=== LLAMA-BENCH OUTPUT ==="
        cat bench_output.txt || echo "(empty)"
        echo "=== END OUTPUT ==="
        if [ $exit_code -ne 0 ]; then
          echo "❌ llama-bench exited with code $exit_code"
          echo "Checking for missing library dependencies..."
          ldd "$llama_bench_path" || echo "ldd command failed"
          rm -f bench_output.txt
          exit 1
        fi
        rm -f bench_output.txt
        echo "✅ llama-bench loaded and ran (hipBLASLt dependency chain resolved)"

  create-release:
    needs: [setup, build-ubuntu]
    runs-on: ubuntu-24.04
    permissions:
      contents: write
    strategy:
      fail-fast: false
      # Each variant publishes to its own dated tag independently.
      matrix: ${{ fromJSON(needs.setup.outputs.matrix) }}
    # Publish only on cron/dispatch that sets create_release=true. Push/PR never
    # release. Require the build to succeed. NOTE: the gfx1151 hardware-test gate
    # (test-gfx in needs + needs.test-gfx.result == 'success') is TEMPORARILY
    # REMOVED while the self-hosted runner is offline and test-gfx is disabled
    # above. Restore both when the runner returns.
    if: |
      always() &&
      needs.build-ubuntu.result == 'success' &&
      github.event_name == 'workflow_dispatch' &&
      github.event.inputs.create_release == 'true'
    steps:
    - name: Checkout repository
      uses: actions/checkout@v4

    - name: Download this variant's build artifact
      uses: actions/download-artifact@v4
      with:
        name: llama-ubuntu-rocm-multiarch-x64-${{ matrix.variant }}
        path: ./artifact

    - name: Read build metadata
      id: meta
      run: |
        meta="./artifact/build-meta.txt"
        if [ ! -f "$meta" ]; then
          echo "ERROR: build-meta.txt missing from artifact" >&2
          exit 1
        fi
        # Re-export the recorded provenance as step outputs.
        while IFS='=' read -r k v; do
          [ -n "$k" ] && echo "$k=$v" >> "$GITHUB_OUTPUT"
        done < "$meta"
        echo "=== build-meta.txt ==="; cat "$meta"

    - name: Package release archive
      run: |
        TAG="${{ matrix.tag }}"
        archive="llama-${TAG}-ubuntu-rocm-multiarch-x64"
        # build-meta.txt is shipped inside the tarball as provenance; keep it.
        echo "Creating ${archive}.tar.gz from ./artifact"
        tar -czf "${archive}.tar.gz" -C ./artifact .
        ls -la *.tar.gz

    - name: Publish dated release
      env:
        GITHUB_TOKEN: ${{ github.token }}
        TAG: ${{ matrix.tag }}
        ROCM_VERSION: ${{ steps.meta.outputs.rocm_version }}
        LLAMACPP_COMMIT_HASH: ${{ steps.meta.outputs.llamacpp_commit_hash }}
        BUILT_SHA: ${{ steps.meta.outputs.llamacpp_commit_sha }}
        SRC_REF: ${{ matrix.ref }}
      run: |
        # Dated, persistent tag (<base>-YYYYMMDD): older builds stay downloadable
        # for issue repro; prune manually if storage becomes a concern. If a same-
        # day tag already exists (a re-run), replace it so the day reflects the
        # latest build at the exact commit this variant built.
        if gh release view "$TAG" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then
          echo "Deleting existing same-day release/tag $TAG (re-run)"
          gh release delete "$TAG" --repo "$GITHUB_REPOSITORY" --yes --cleanup-tag
        fi

        display_ref="${SRC_REF:-${GITHUB_REF_NAME}}"
        archive="llama-${TAG}-ubuntu-rocm-multiarch-x64.tar.gz"

        gh release create "$TAG" \
          --repo "$GITHUB_REPOSITORY" \
          --target "$BUILT_SHA" \
          --title "$TAG" \
          --notes "**Dated build**: $TAG (persistent; prune manually)
        **OS**: ubuntu
        **GPU Target(s)**: gfx1100, gfx1101, gfx1102, gfx1103, gfx1150, gfx1151, gfx1152, gfx1153 (single multiarch binary)
        **Source ref**: $display_ref
        **ROCm Version**: $ROCM_VERSION (multiarch)
        **Llama.cpp Commit**: $LLAMACPP_COMMIT_HASH ($BUILT_SHA)
        **Build Date**: $(date -u '+%Y-%m-%d %H:%M:%S UTC')

        Prebuilt llama.cpp ROCm binaries — one multiarch package covering the RDNA3 gfx110X family incl. Hawk Point/Phoenix (Radeon 760M/780M) and the RDNA3.5 gfx115x APUs (gfx1150/gfx1151/gfx1152/gfx1153). Built from TheRock's multiarch ROCm runtime, pruned to the CI target arches (per-arch Tensile databases bundled)." \
          "$archive"