Build gfx11 + ROCm #337

Workflow file for this run

.github/workflows/build-gfx11-rocm.yml at 03d2068

	name: Build gfx11 + ROCm

	on:
	push:
	branches: [gfx11]
	pull_request:
	types: [opened, synchronize, reopened]
	workflow_dispatch:
	inputs:
	build_set:
	description: 'Which variants to build: nightly (A: gfx11 + latest nightly), weekly (B+C: master + gfx11 on the frozen 7.14 release), or all.'
	required: false
	default: 'nightly'
	type: choice
	options: [nightly, weekly, all]
	rocm_version:
	description: 'Override ROCm version for ALL built variants (e.g. 7.14.0a20260608). Empty = per-variant channel auto-detect.'
	required: false
	default: ''
	create_release:
	description: 'Publish a dated per-variant GitHub Release (<base>-YYYYMMDD). Set true only for cron dispatches.'
	required: false
	default: 'false'
	roofline:
	description: 'Build with per-op roofline profiling support (GGML_HIP_ROOFLINE) and bundle the profiler libs (librocprofiler-sdk + libdw). Set false for lean builds.'
	required: false
	default: 'true'

	# Three build variants (see the `setup` job matrix). Each publishes a DATED,
	# persistent release tag (<base>-YYYYMMDD) so older builds stay downloadable for
	# issue repro; prune old releases manually if storage becomes a concern.
	# A gfx11 + latest nightly ROCm -> gfx11-rocm-nightly-YYYYMMDD (nightly)
	# B master + 7.14 release ROCm -> master-rocm-714-YYYYMMDD (weekly)
	# C gfx11 + 7.14 release ROCm -> gfx11-rocm-714-YYYYMMDD (weekly)
	#
	# ROCm channels resolve against TheRock's multiarch tarball host:
	# latest -> highest therock-dist-linux-multiarch-*.tar.gz overall (7.15 nightly today)
	# 7.14 -> highest therock-dist-linux-multiarch-7.14.0a*.tar.gz. The 7.14 line is
	# FROZEN (last build 7.14.0a20260624, when 7.15 alphas began), so "7.14
	# release" is that final alpha; only the llama.cpp source changes weekly.
	# An explicit rocm_version input overrides the channel for every variant.
	#
	# push/PR build only variant A's shape (triggering ref + latest) and never release.
	# The 7.14 variants build via external cron (GitHub cron only fires from the default
	# branch). On a host you control, with `gh` authenticated (token scope: repo + workflow):
	# # nightly (variant A)
	# 7 13 * * * gh workflow run build-gfx11-rocm.yml --repo AMD-Ecosystem/llama.cpp --ref gfx11 -f build_set=nightly -f create_release=true
	# # weekly Thursday (variants B + C), ready ahead of weekend regressions
	# 7 9 * * 4 gh workflow run build-gfx11-rocm.yml --repo AMD-Ecosystem/llama.cpp --ref gfx11 -f build_set=weekly -f create_release=true

	env:
	# Default ON so push/PR/cron (non-dispatch) runs keep roofline, matching prior
	# always-on behavior; a workflow_dispatch can pass 'false' for a lean build.
	ROOFLINE: ${{ github.event.inputs.roofline \|\| 'true' }}

	jobs:
	setup:
	runs-on: ubuntu-24.04
	outputs:
	matrix: ${{ steps.set-matrix.outputs.matrix }}
	steps:
	- name: Compute build matrix
	id: set-matrix
	env:
	EVENT_NAME: ${{ github.event_name }}
	BUILD_SET: ${{ github.event.inputs.build_set \|\| 'nightly' }}
	ROCM_OVERRIDE: ${{ github.event.inputs.rocm_version \|\| '' }}
	run: \|
	# Variant definitions. ref="" means "the ref that triggered this run"
	# (used for push/PR so CI builds the pushed code, not gfx11 tip).
	# rocm_channel is overridden wholesale when ROCM_OVERRIDE is set.
	# Each variant publishes a DATED, persistent release tag (<base>-YYYYMMDD)
	# so older nightlies stay reproducible; prune manually if storage grows.
	d="$(date -u +%Y%m%d)"
	a_ch="latest"; b_ch="7.14"; c_ch="7.14"
	if [ -n "$ROCM_OVERRIDE" ]; then
	a_ch="$ROCM_OVERRIDE"; b_ch="$ROCM_OVERRIDE"; c_ch="$ROCM_OVERRIDE"
	fi
	A="{\"variant\":\"A\",\"ref\":\"\",\"rocm_channel\":\"$a_ch\",\"tag\":\"gfx11-rocm-nightly-$d\"}"
	B="{\"variant\":\"B\",\"ref\":\"master\",\"rocm_channel\":\"$b_ch\",\"tag\":\"master-rocm-714-$d\"}"
	C="{\"variant\":\"C\",\"ref\":\"gfx11\",\"rocm_channel\":\"$c_ch\",\"tag\":\"gfx11-rocm-714-$d\"}"

	if [ "$EVENT_NAME" = "workflow_dispatch" ]; then
	case "$BUILD_SET" in
	nightly) entries="$A" ;;
	weekly) entries="$B,$C" ;;
	all) entries="$A,$B,$C" ;;
	*) echo "Unknown build_set: $BUILD_SET" >&2; exit 1 ;;
	esac
	else
	# push / pull_request: single leg, triggering ref (ref=""), latest ROCm.
	entries="$A"
	fi

	matrix="{\"include\":[$entries]}"
	echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
	echo "Build matrix: $matrix"

	build-ubuntu:
	needs: setup
	runs-on: ubuntu-24.04
	strategy:
	fail-fast: false
	matrix: ${{ fromJSON(needs.setup.outputs.matrix) }}
	# Single multiarch build per variant: one fat binary covering all current CI
	# arches, sourced from TheRock's multiarch tarball (arch-neutral host +
	# per-arch Tensile DBs). gfx1100-1103 (RDNA3 desktop + Hawk Point/Phoenix
	# 760M/780M), gfx1150/1151/1152/1153 (RDNA3.5 Strix APUs).
	env:
	GPU_TARGETS: gfx1100;gfx1101;gfx1102;gfx1103;gfx1150;gfx1151;gfx1152;gfx1153
	ROCM_VERSION: ${{ matrix.rocm_channel }}

	steps:
	- name: Free disk space
	# Remove unused runner files to free up disk space
	run: curl -fsSL https://raw.githubusercontent.com/kou/arrow/e49d8ae15583ceff03237571569099a6ad62be32/ci/scripts/util_free_space.sh \| bash

	- name: Checkout repository
	uses: actions/checkout@v4
	with:
	# Empty ref (variant A on push/PR) checks out the triggering ref; B/C
	# pin master / gfx11 explicitly. Full history so rev-parse is meaningful.
	ref: ${{ matrix.ref }}
	fetch-depth: 0

	- name: Set up Node.js (for llama-ui build from source)
	uses: actions/setup-node@v6
	with:
	node-version: "24"

	- name: Clean up existing ROCm directory (safety precaution)
	run: \|
	if [ -d "/opt/rocm" ]; then
	echo "Removing existing /opt/rocm directory..."
	sudo rm -rf /opt/rocm
	fi
	if [ -f "rocm.tar.gz" ]; then
	rm -f rocm.tar.gz
	fi
	echo "Cleanup completed successfully"

	- name: Install build dependencies
	run: \|
	echo "Installing build dependencies..."
	sudo apt update
	sudo apt install -y cmake ninja-build unzip curl
	echo "Verifying installations..."
	cmake --version
	ninja --version
	echo "Build dependencies installation completed"

	- name: Download and extract multiarch ROCm directly to /opt/rocm
	run: \|
	rocm_version="${{ env.ROCM_VERSION }}"
	base_url="https://rocm.nightlies.amd.com/tarball-multi-arch"

	# Channel resolution:
	# latest -> highest multiarch tarball overall
	# X.Y (e.g 7.14) -> highest tarball on that minor line (frozen 7.14 ->
	# deterministic). Matched on the "multiarch-X.Y." prefix.
	# X.Y.Z(a\|rc)N -> pinned full version, used verbatim (no detection)
	detect=false
	version_prefix=""
	if [ "$rocm_version" = "latest" ]; then
	detect=true
	elif [[ "$rocm_version" =~ ^[0-9]+\.[0-9]+$ ]]; then
	detect=true
	version_prefix="therock-dist-linux-multiarch-${rocm_version}."
	fi

	if [ "$detect" = true ]; then
	echo "Auto-detecting multiarch ROCm version (channel: $rocm_version)"
	# The multiarch host serves an HTML index (not S3 XML); scrape the
	# multiarch tarball names, then (for a X.Y channel) keep only that line.
	files=$(curl -s "$base_url/" \
	\| grep -oE 'therock-dist-linux-multiarch-[0-9]+\.[0-9]+\.[0-9]+(a\|rc)[0-9]+\.tar\.gz' \
	\| sort -u)
	if [ -n "$version_prefix" ]; then
	files=$(printf '%s\n' "$files" \| grep -F "$version_prefix" \|\| true)
	fi

	if [ -z "$files" ]; then
	echo "No multiarch tarballs found for channel '$rocm_version'" >&2
	exit 1
	fi

	latest_file=""
	latest_major=0
	latest_minor=0
	latest_patch=0
	latest_rc=0
	latest_is_alpha=false

	while IFS= read -r file; do
	if [[ "$file" =~ therock-dist-linux-multiarch-([0-9]+\.[0-9]+\.[0-9]+(a\|rc)[0-9]+)\.tar\.gz ]]; then
	version="${BASH_REMATCH[1]}"
	major=$(echo "$version" \| cut -d. -f1)
	minor=$(echo "$version" \| cut -d. -f2)
	patch=$(echo "$version" \| cut -d. -f3 \| sed 's/$a\\|rc$.*//')
	rc=$(echo "$version" \| sed 's/.*$a\\|rc$//')
	is_alpha=false
	if [[ "$version" =~ a ]]; then
	is_alpha=true
	fi

	is_newer=false
	if [ "$major" -gt "$latest_major" ]; then
	is_newer=true
	elif [ "$major" -eq "$latest_major" ] && [ "$minor" -gt "$latest_minor" ]; then
	is_newer=true
	elif [ "$major" -eq "$latest_major" ] && [ "$minor" -eq "$latest_minor" ] && [ "$patch" -gt "$latest_patch" ]; then
	is_newer=true
	elif [ "$major" -eq "$latest_major" ] && [ "$minor" -eq "$latest_minor" ] && [ "$patch" -eq "$latest_patch" ]; then
	if [ "$is_alpha" = true ] && [ "$latest_is_alpha" = false ]; then
	is_newer=true
	elif [ "$is_alpha" = "$latest_is_alpha" ] && [ "$rc" -gt "$latest_rc" ]; then
	is_newer=true
	fi
	fi

	if [ "$is_newer" = true ]; then
	latest_file="$file"
	latest_major="$major"
	latest_minor="$minor"
	latest_patch="$patch"
	latest_rc="$rc"
	latest_is_alpha="$is_alpha"
	fi
	fi
	done <<< "$files"

	echo "Found latest file: $latest_file"

	if [[ "$latest_file" =~ therock-dist-linux-multiarch-([0-9]+\.[0-9]+\.[0-9]+(a\|rc)[0-9]+)\.tar\.gz ]]; then
	rocm_version="${BASH_REMATCH[1]}"
	echo "Detected latest ROCm version: $rocm_version"
	else
	echo "Failed to extract ROCm version from latest file: $latest_file"
	echo "Expected pattern: therock-dist-linux-multiarch-<version>.tar.gz"
	exit 1
	fi
	fi

	rocm_url="$base_url/therock-dist-linux-multiarch-${rocm_version}.tar.gz"
	echo "DETECTED_ROCM_VERSION=$rocm_version" >> $GITHUB_ENV

	# The multiarch tarball (~11.5 GB) ships device code for ALL 26 GPU
	# arches: every arch's .kpack plus per-arch rocBLAS/hipBLASLt Tensile DBs.
	# This consumer build only needs the 8 RDNA3/3.5 arches in GPU_TARGETS and
	# uses the GEMM (Tensile) path, which works without .kpack files. So we
	# stream-extract and prune at the tar level: drop ALL .kpack, and drop the
	# Tensile DBs of every arch not in our target set. This keeps the runner
	# disk footprint small (the 11.5 GB is streamed, never stored) and yields
	# a lean multiarch package. tar matches --exclude on pre-strip member
	# names, hence the leading "./".
	drop_arches="gfx900 gfx906 gfx908 gfx90a gfx942 gfx950 \
	gfx1010 gfx1011 gfx1012 gfx1030 gfx1031 gfx1032 gfx1033 gfx1034 gfx1035 gfx1036 \
	gfx1200 gfx1201"
	excludes=(--exclude='./.kpack' --exclude='./.kpack/*')
	for a in $drop_arches; do
	excludes+=("--exclude=./lib/*/library/${a}")
	excludes+=("--exclude=./lib//library/${a}/")
	excludes+=("--exclude=./lib//library/${a}*")
	done

	echo "Streaming multiarch ROCm from: $rocm_url (pruning .kpack + non-target arches)"
	sudo mkdir -p /opt/rocm
	curl -sL "$rocm_url" \| sudo tar --use-compress-program=gzip -xf - \
	-C /opt/rocm --strip-components=1 "${excludes[@]}"

	echo "Retained rocBLAS Tensile arch dirs:"
	ls /opt/rocm/lib/rocblas/library/ 2>/dev/null \|\| echo "(none)"
	echo "Retained hipBLASLt Tensile arch dirs:"
	ls /opt/rocm/lib/hipblaslt/library/ 2>/dev/null \|\| echo "(none)"

	- name: Set ROCm environment variables
	run: \|
	echo "Setting ROCm environment variables..."
	echo "HIP_PATH=/opt/rocm" >> $GITHUB_ENV
	echo "ROCM_PATH=/opt/rocm" >> $GITHUB_ENV
	echo "HIP_PLATFORM=amd" >> $GITHUB_ENV
	echo "HIP_CLANG_PATH=/opt/rocm/llvm/bin" >> $GITHUB_ENV
	echo "HIP_INCLUDE_PATH=/opt/rocm/include" >> $GITHUB_ENV
	echo "HIP_LIB_PATH=/opt/rocm/lib" >> $GITHUB_ENV
	echo "HIP_DEVICE_LIB_PATH=/opt/rocm/lib/llvm/amdgcn/bitcode" >> $GITHUB_ENV
	echo "/opt/rocm/bin:/opt/rocm/llvm/bin:$PATH" >> $GITHUB_PATH
	echo "LD_LIBRARY_PATH=/opt/rocm/lib:/opt/rocm/lib64:/opt/rocm/llvm/lib:${LD_LIBRARY_PATH:-}" >> $GITHUB_ENV
	echo "LIBRARY_PATH=/opt/rocm/lib:/opt/rocm/lib64:${LIBRARY_PATH:-}" >> $GITHUB_ENV
	echo "CPATH=/opt/rocm/include:${CPATH:-}" >> $GITHUB_ENV
	echo "PKG_CONFIG_PATH=/opt/rocm/lib/pkgconfig:${PKG_CONFIG_PATH:-}" >> $GITHUB_ENV
	echo "ROCm environment variables set successfully"

	- name: Record llama.cpp commit hash
	run: \|
	commit_hash=$(git rev-parse --short=5 HEAD)
	echo "LLAMACPP_COMMIT_HASH=$commit_hash" >> $GITHUB_ENV
	echo "llama.cpp commit hash (5 digits): $commit_hash"
	echo "Current llama.cpp commit:"
	git log --oneline -1

	- name: Setup ccache
	# Auto-consumed by the build: llama.cpp's CMake sets ccache as the compile
	# launcher when GGML_CCACHE is ON (default) and ccache is on PATH, so no
	# cmake flags are needed. Persists across runs via actions/cache. Nightly
	# ROCm bumps change the compiler and will miss (expected); incremental
	# commits on the same ROCm hit. Only write the cache from pushes to gfx11
	# so PRs read but don't thrash the shared store.
	uses: ggml-org/ccache-action@v1.2.21
	with:
	# Per-variant key: the three variants use different source refs and ROCm
	# compilers, so a shared key would thrash. Keep each on its own store.
	key: gfx11-rocm-multiarch-${{ matrix.variant }}
	evict-old-files: 7d
	save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/gfx11' }}

	- name: Build Llama.cpp + ROCm
	run: \|
	gpu_targets="${{ env.GPU_TARGETS }}"
	echo "Building multiarch binary (GPU_TARGETS=$gpu_targets)"

	mkdir build
	cd build

	cmake .. -G Ninja \
	-DCMAKE_C_COMPILER=/opt/rocm/llvm/bin/clang \
	-DCMAKE_CXX_COMPILER=/opt/rocm/llvm/bin/clang++ \
	-DCMAKE_CXX_FLAGS="-I/opt/rocm/include" \
	-DCMAKE_CROSSCOMPILING=ON \
	-DCMAKE_BUILD_TYPE=Release \
	-DGPU_TARGETS="$gpu_targets" \
	-DBUILD_SHARED_LIBS=ON \
	-DLLAMA_BUILD_TESTS=OFF \
	-DGGML_HIP=ON \
	-DGGML_OPENMP=OFF \
	-DGGML_CUDA_FORCE_CUBLAS=OFF \
	-DGGML_RPC=ON \
	-DGGML_HIP_ROCWMMA_FATTN=OFF \
	-DGGML_HIP_ROOFLINE=${{ env.ROOFLINE == 'true' && 'ON' \|\| 'OFF' }} \
	-DLLAMA_BUILD_BORINGSSL=ON \
	-DGGML_NATIVE=OFF \
	-DGGML_STATIC=OFF \
	-DCMAKE_SYSTEM_NAME=Linux

	cmake --build . -j $(nproc)

	- name: Copy ROCm core libs to build directory
	run: \|
	build_bin_path="build/bin"

	# Copy the rocblas/library folder and all its contents
	rocblas_lib_path="/opt/rocm/lib/rocblas/library"
	if [ -d "$rocblas_lib_path" ]; then
	echo "Copying rocblas/library folder and all contents..."
	dest_rocblas_path="$build_bin_path/rocblas/library"
	mkdir -p "$(dirname "$dest_rocblas_path")"
	cp -r "$rocblas_lib_path" "$(dirname "$dest_rocblas_path")/"
	echo "Copied: rocblas/library folder with all contents"
	else
	echo "Warning: rocblas/library folder not found at: $rocblas_lib_path"
	fi

	# Copy the hipblaslt/library folder and all its contents
	hipblaslt_lib_path="/opt/rocm/lib/hipblaslt/library"
	if [ -d "$hipblaslt_lib_path" ]; then
	echo "Copying hipblaslt/library folder and all contents..."
	dest_hipblaslt_path="$build_bin_path/hipblaslt/library"
	mkdir -p "$(dirname "$dest_hipblaslt_path")"
	cp -r "$hipblaslt_lib_path" "$(dirname "$dest_hipblaslt_path")/"
	echo "Copied: hipblaslt/library folder with all contents"
	else
	echo "Warning: hipblaslt/library folder not found at: $hipblaslt_lib_path"
	fi

	# Copy required ROCm libraries to build directory
	# If artifacts from ROCm or Llama.cpp change, you may need to update this list.
	echo "Copying required ROCm libraries to build directory..."
	cp -v /opt/rocm/lib/libhipblas.so* "$build_bin_path/" 2>/dev/null \|\| echo "libhipblas.so* not found"
	cp -v /opt/rocm/lib/librocblas.so* "$build_bin_path/" 2>/dev/null \|\| echo "librocblas.so* not found"
	cp -v /opt/rocm/lib/libamdhip64.so* "$build_bin_path/" 2>/dev/null \|\| echo "libamdhip64.so* not found"
	cp -v /opt/rocm/lib/librocsolver.so* "$build_bin_path/" 2>/dev/null \|\| echo "librocsolver.so* not found"
	cp -v /opt/rocm/lib/libroctx64.so* "$build_bin_path/" 2>/dev/null \|\| echo "libroctx64.so* not found"
	cp -v /opt/rocm/lib/libhipblaslt.so* "$build_bin_path/" 2>/dev/null \|\| echo "libhipblaslt.so* not found"
	# liborigami is a hard DT_NEEDED of libhipblaslt as of ROCm 7.15 (new
	# hipBLASLt kernel-selection lib). Omitting it makes llama-bench/llama-server
	# fail to load (liborigami.so.1: cannot open shared object file). The
	# transitive-closure backstop below would also catch it, but keep the
	# explicit copy so the intent is greppable alongside the other hipblaslt libs.
	cp -v /opt/rocm/lib/liborigami.so* "$build_bin_path/" 2>/dev/null \|\| echo "liborigami.so* not found"
	cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_liblzma.so* "$build_bin_path/" 2>/dev/null \|\| echo "librocm_sysdeps_liblzma.so* not found"
	cp -v /opt/rocm/lib/librocprofiler-register.so* "$build_bin_path/" 2>/dev/null \|\| echo "librocprofiler-register.so* not found"
	# Roofline builds dlopen librocprofiler-sdk.so.1 at runtime (ggml-cuda-roofline.cpp);
	# the release must ship it plus its one bundle-missing dep libdw
	# (librocm_sysdeps_dw.so.1 — a hard DT_NEEDED of the sdk lib). The $ORIGIN RPATH
	# step below lets the dlopen resolve both locally with no LD_LIBRARY_PATH.
	if [ "${{ env.ROOFLINE }}" = "true" ]; then
	cp -v /opt/rocm/lib/librocprofiler-sdk.so* "$build_bin_path/" 2>/dev/null \|\| echo "librocprofiler-sdk.so* not found"
	cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_dw.so* "$build_bin_path/" 2>/dev/null \|\| echo "librocm_sysdeps_dw.so* not found"
	fi
	cp -v /opt/rocm/lib/libamd_comgr.so* "$build_bin_path/" 2>/dev/null \|\| echo "libamd_comgr.so* not found"
	cp -v /opt/rocm/lib/libamd_comgr_loader.so* "$build_bin_path/" 2>/dev/null \|\| echo "libamd_comgr_loader.so* not found"
	cp -v /opt/rocm/lib/libhsa-runtime64.so* "$build_bin_path/" 2>/dev/null \|\| echo "libhsa-runtime64.so* not found"
	cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_numa.so* "$build_bin_path/" 2>/dev/null \|\| echo "librocm_sysdeps_numa.so* not found"
	cp -v /opt/rocm/lib/librocroller.so* "$build_bin_path/" 2>/dev/null \|\| echo "librocroller.so* not found"
	cp -v /opt/rocm/lib/librocm_kpack.so* "$build_bin_path/" 2>/dev/null \|\| echo "librocm_kpack.so* not found"
	cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_z.so* "$build_bin_path/" 2>/dev/null \|\| echo "librocm_sysdeps_z.so* not found"
	cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_zstd.so* "$build_bin_path/" 2>/dev/null \|\| echo "librocm_sysdeps_zstd.so* not found"
	cp -v /opt/rocm/lib/llvm/lib/libLLVM.so* "$build_bin_path/" 2>/dev/null \|\| echo "libLLVM.so* not found"
	cp -v /opt/rocm/lib/llvm/lib/libclang-cpp.so* "$build_bin_path/" 2>/dev/null \|\| echo "libclang-cpp.so* not found"
	cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_elf.so* "$build_bin_path/" 2>/dev/null \|\| echo "librocm_sysdeps_elf.so* not found"
	cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_drm.so* "$build_bin_path/" 2>/dev/null \|\| echo "librocm_sysdeps_drm.so* not found"
	cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_drm_amdgpu.so* "$build_bin_path/" 2>/dev/null \|\| echo "librocm_sysdeps_drm_amdgpu.so* not found"
	cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_bz2.so* "$build_bin_path/" 2>/dev/null \|\| echo "librocm_sysdeps_bz2.so* not found"

	# Bundle libatomic (gcc runtime dep of llama-cli). Some self-hosted GPU
	# runners don't have libatomic1 installed, so ship it with the artifact;
	# the $ORIGIN RPATH step below makes the binary load this bundled copy.
	cp -v /usr/lib/x86_64-linux-gnu/libatomic.so.1* "$build_bin_path/" 2>/dev/null \
	\|\| cp -v "$(gcc -print-file-name=libatomic.so.1)" "$build_bin_path/" 2>/dev/null \
	\|\| echo "libatomic.so.1 not found"

	# Transitive-closure backstop for the hand-maintained copy list above.
	# A ROCm bump can add a new DT_NEEDED to a bundled lib (e.g. 7.15 added
	# liborigami.so.1 to libhipblaslt); if it isn't in the list, the release
	# ships broken. Here we ldd every bundled binary against the release dir
	# and, for any "=> not found" dep that exists somewhere under
	# /opt/rocm/lib, copy it in — repeating until no new libs are pulled in.
	# Only ROCm-provided deps are auto-bundled; genuinely external libs
	# (libc/libm/... provided by the runner) are left alone.
	echo "Resolving remaining ROCm deps via ldd closure..."
	rocm_lib_roots="/opt/rocm/lib /opt/rocm/lib/rocm_sysdeps/lib /opt/rocm/lib/llvm/lib"
	for pass in 1 2 3 4 5; do
	added=0
	# Collect unresolved sonames across all bundled binaries in one sweep.
	missing="$(cd "$build_bin_path" && \
	for f in .so llama-*; do
	[ -f "$f" ] && [ ! -L "$f" ] \|\| continue
	LD_LIBRARY_PATH=. ldd "$f" 2>/dev/null
	done \| awk '/=> not found/ {print $1}' \| sort -u)"
	[ -z "$missing" ] && { echo " pass $pass: closure complete"; break; }
	for soname in $missing; do
	# Already bundled? (a previous pass may have added it)
	[ -e "$build_bin_path/$soname" ] && continue
	src=""
	for root in $rocm_lib_roots; do
	cand="$(find "$root" -maxdepth 1 -name "$soname" 2>/dev/null \| head -1)"
	[ -n "$cand" ] && { src="$cand"; break; }
	done
	if [ -n "$src" ]; then
	# Copy the real file and recreate the soname symlink chain.
	cp -vL "$src" "$build_bin_path/$soname"
	added=1
	else
	echo " pass $pass: $soname not found under /opt/rocm — leaving to system loader"
	fi
	done
	[ "$added" -eq 0 ] && { echo " pass $pass: no ROCm-resolvable deps left"; break; }
	done

	echo "Finished copying required ROCm libraries"

	- name: Set RPATH for portable distribution
	run: \|
	sudo apt-get install -y patchelf
	cd build/bin
	# Set RPATH to $ORIGIN so all libraries (including the comgr stub loader) find deps locally
	for file in .so llama-*; do
	[ -f "$file" ] && [ ! -L "$file" ] && patchelf --set-rpath '$ORIGIN' "$file" 2>/dev/null \|\| true
	done

	- name: Write build metadata into artifact
	run: \|
	# Per-variant provenance travels inside the artifact so downstream matrix
	# jobs (test-gfx, create-release) read it without fragile matrix outputs.
	rocm_version="${DETECTED_ROCM_VERSION:-${{ env.ROCM_VERSION }}}"
	built_sha=$(git rev-parse HEAD)
	{
	echo "variant=${{ matrix.variant }}"
	echo "ref=${{ matrix.ref }}"
	echo "tag=${{ matrix.tag }}"
	echo "rocm_channel=${{ matrix.rocm_channel }}"
	echo "rocm_version=$rocm_version"
	echo "llamacpp_commit_hash=${LLAMACPP_COMMIT_HASH}"
	echo "llamacpp_commit_sha=$built_sha"
	} > build/bin/build-meta.txt
	echo "=== build-meta.txt ==="; cat build/bin/build-meta.txt

	- name: Verify bundled binaries have no unresolved deps
	run: \|
	# GPU-free packaging gate. With RPATH=$ORIGIN set, ldd resolves bundled
	# libs from this dir and true system libs from default paths; anything
	# still "not found" means the artifact would fail to load at runtime.
	# This is exactly the failure that shipped in b20260724 (libhipblaslt
	# gained NEEDED liborigami.so.1 which wasn't bundled) — llama-cli happened
	# to load but llama-bench/llama-server did not. Fail the build here so a
	# missing dep can never reach a release again.
	cd build/bin
	status=0
	for f in .so llama-*; do
	[ -f "$f" ] && [ ! -L "$f" ] \|\| continue
	missing="$(ldd "$f" 2>/dev/null \| awk '/=> not found/ {print $1}')"
	if [ -n "$missing" ]; then
	status=1
	echo "❌ $f has unresolved dependencies:"
	echo "$missing" \| sed 's/^/ /'
	fi
	done
	if [ "$status" -ne 0 ]; then
	echo ""
	echo "One or more bundled binaries have unresolved shared-library deps."
	echo "Add the missing lib(s) to the 'Copy ROCm core libs' step (or ensure"
	echo "the ldd-closure backstop can find them under /opt/rocm/lib)."
	exit 1
	fi
	echo "✅ All bundled binaries resolve their dependencies."

	- name: List build artifacts (including ROCm files)
	run: \|
	cd build/bin
	echo "Final build artifacts (including ROCm library files):"
	ls -la

	- name: Upload build artifacts
	uses: actions/upload-artifact@v4
	with:
	# Variant-suffixed so the three legs don't collide on one artifact name.
	name: llama-ubuntu-rocm-multiarch-x64-${{ matrix.variant }}
	path: build/bin/
	retention-days: 30

	test-gfx:
	needs: [setup, build-ubuntu]
	# TEMPORARILY DISABLED: the linux-gfx1151-gpu-rocm self-hosted runner is
	# offline, so this job sits queued indefinitely and blocks create-release.
	# Re-enable by restoring `needs.build-ubuntu.result == 'success'` once the
	# runner is back online.
	if: false && needs.build-ubuntu.result == 'success'
	strategy:
	fail-fast: false
	matrix: ${{ fromJSON(needs.setup.outputs.matrix) }}
	# Per-variant hardware test of the multiarch artifact on gfx1151. This is the
	# end-to-end safety net for the Tensile-only multiarch package: a real
	# llama-cli inference exercising the rocBLAS/hipBLASLt GEMM path on-device.
	runs-on: linux-gfx1151-gpu-rocm

	steps:
	- name: Checkout repository
	uses: actions/checkout@v4

	- name: Download build artifacts
	uses: actions/download-artifact@v4
	with:
	name: llama-ubuntu-rocm-multiarch-x64-${{ matrix.variant }}
	path: llama-binaries

	- name: Download test model
	run: \|
	# Pinned to a fixed GitHub release asset instead of huggingface.co: some
	# self-hosted runners cannot reach huggingface.co (curl 35, connection
	# reset), but they can reach GitHub. Source:
	# https://github.com/jimw567/llamacpp-test-assets/releases/tag/test-assets
	model_url="https://github.com/jimw567/llamacpp-test-assets/releases/download/test-assets/Qwen3-0.6B-Q4_0.gguf"
	model_path="Qwen3-0.6B-Q4_0.gguf"
	echo "Downloading test model from: $model_url"
	curl -fL --retry 5 --retry-all-errors --retry-delay 5 -o "$model_path" "$model_url"
	if [ -f "$model_path" ]; then
	file_size=$(stat -c%s "$model_path")
	echo "Model downloaded successfully. Size: $file_size bytes"
	else
	echo "Failed to download model"
	exit 1
	fi

	- name: Set up library path
	run: \|
	echo "LD_LIBRARY_PATH=$(pwd)/llama-binaries:$LD_LIBRARY_PATH" >> $GITHUB_ENV

	- name: Run llama-cli test
	run: \|
	llama_cli_path="./llama-binaries/llama-cli"
	model_path="Qwen3-0.6B-Q4_0.gguf"
	output_file="llama_output.txt"

	chmod +x "$llama_cli_path"

	if [ ! -f "$llama_cli_path" ]; then
	echo "llama-cli not found at: $llama_cli_path"
	echo "Available files in llama-binaries:"
	find llama-binaries -type f
	exit 1
	fi

	# Use a prompt with a single correct answer and greedy decoding
	# (--temp 0) so the result is deterministic and verifiable.
	prompt="What is 2 + 2? Reply with only the number."
	echo "Running llama-cli test for gfx1151 (multiarch artifact)..."
	echo "Command: $llama_cli_path -m \"$model_path\" -ngl 99 --temp 0 -p \"$prompt\" -st -v"

	# Bound the run: a healthy 0.6B inference finishes in seconds. If the
	# GPU kernel launch hangs (driver/runner issue) the step would otherwise
	# never end and GitHub would never publish the logs. timeout forces a
	# clean failure and lets the captured output flush for diagnosis.
	set +e
	timeout 180 "$llama_cli_path" -m "$model_path" -ngl 99 --temp 0 -p "$prompt" -st -v > "$output_file" 2>&1
	exit_code=$?
	set -e

	echo "=== LLAMA-CLI OUTPUT ==="
	if [ -f "$output_file" ] && [ -s "$output_file" ]; then
	cat "$output_file"
	else
	echo "(empty)"
	fi
	echo "=== END OUTPUT ==="
	echo "Process exit code: $exit_code"

	if [ $exit_code -eq 124 ]; then
	echo "❌ llama-cli timed out after 180s (GPU kernel likely hung)"
	echo "=== GPU / driver diagnostics ==="
	rocminfo 2>&1 \| grep -iE 'Name\|Marketing\|gfx\|Uuid' \| head -40 \|\| echo "rocminfo failed"
	(dmesg 2>/dev/null \| tail -40) \|\| echo "dmesg unavailable"
	rm -f "$output_file"
	exit 1
	fi

	if [ $exit_code -ne 0 ]; then
	echo "❌ llama-cli exited with error code: $exit_code"
	echo "Checking for missing library dependencies..."
	ldd "$llama_cli_path" \|\| echo "ldd command failed"
	rm -f "$output_file"
	exit 1
	fi

	# Functional checks against current llama.cpp output:
	# (1) the ROCm GPU was selected,
	# (2) the model layers were offloaded to it (incl. the output layer),
	# (3) the model computed the correct answer to "What is 2 + 2?".
	# With greedy decoding (--temp 0) the answer is deterministic, so this
	# verifies the GPU math path end to end, not just that text was emitted.
	found_device=false
	found_offload=false
	found_answer=false

	if grep -q "using device ROCm0" "$output_file"; then
	found_device=true
	fi

	# Qwen3-0.6B has 28 transformer layers + output = 29 GPU assignments per
	# pass (verbose mode logs them more than once; require at least one pass).
	layers_on_gpu=$(grep -c "assigned to device ROCm0" "$output_file" \|\| true)
	if [ "$layers_on_gpu" -ge 29 ] && grep -q "offloading output layer to GPU" "$output_file"; then
	found_offload=true
	fi

	# The parsed assistant answer ("content" field, printed with -v) must
	# contain 4 and no other digit, so "4" / "The answer is 4." pass while
	# "5", "14", "22" fail. This is the deterministic correctness check.
	# llama-cli now runs via an in-process server (upstream #24948); its
	# stream ends with a verbose stop chunk carrying an empty "content":"",
	# so drop empty matches before taking the last real answer chunk.
	answer_content=$(grep -oE '"content":"[^"]*"' "$output_file" \| grep -v '"content":""' \| tail -1)
	if echo "$answer_content" \| grep -qE '4' && ! echo "$answer_content" \| grep -qE '[0-35-9]'; then
	found_answer=true
	fi

	echo "=== TEST RESULTS ==="
	echo "ROCm GPU selected ('using device ROCm0'): $(if [ "$found_device" = true ]; then echo 'FOUND'; else echo 'NOT FOUND'; fi)"
	echo "Layers offloaded to GPU ('assigned to device ROCm0' x$layers_on_gpu + output layer): $(if [ "$found_offload" = true ]; then echo 'FOUND'; else echo 'NOT FOUND'; fi)"
	echo "Correct answer to '2 + 2' (parsed content: ${answer_content:-<none>}): $(if [ "$found_answer" = true ]; then echo 'FOUND'; else echo 'NOT FOUND'; fi)"

	rm -f "$output_file"

	if [ "$found_device" = true ] && [ "$found_offload" = true ] \
	&& [ "$found_answer" = true ]; then
	echo "✅ Test PASSED - GPU offload + correct deterministic answer verified"
	else
	echo "❌ Test FAILED - Missing expected outputs"
	exit 1
	fi

	- name: Run llama-bench load check
	run: \|
	# llama-cli above links hipBLASLt lazily, so it can load even when a
	# hipBLASLt dependency is missing from the bundle. llama-bench (and
	# llama-server) load hipBLASLt eagerly — this is the exact split that
	# let b20260724 pass the cli test but ship a broken bench/server. Run a
	# tiny llama-bench so a missing hipBLASLt dep fails the HW test too, not
	# just the GPU-free packaging gate in the build job.
	llama_bench_path="./llama-binaries/llama-bench"
	model_path="Qwen3-0.6B-Q4_0.gguf"
	chmod +x "$llama_bench_path"
	if [ ! -f "$llama_bench_path" ]; then
	echo "llama-bench not found at: $llama_bench_path"
	exit 1
	fi
	echo "Running minimal llama-bench (pp8/tg8, 1 rep) on gfx1151..."
	set +e
	timeout 180 "$llama_bench_path" -m "$model_path" -ngl 99 -p 8 -n 8 -r 1 > bench_output.txt 2>&1
	exit_code=$?
	set -e
	echo "=== LLAMA-BENCH OUTPUT ==="
	cat bench_output.txt \|\| echo "(empty)"
	echo "=== END OUTPUT ==="
	if [ $exit_code -ne 0 ]; then
	echo "❌ llama-bench exited with code $exit_code"
	echo "Checking for missing library dependencies..."
	ldd "$llama_bench_path" \|\| echo "ldd command failed"
	rm -f bench_output.txt
	exit 1
	fi
	rm -f bench_output.txt
	echo "✅ llama-bench loaded and ran (hipBLASLt dependency chain resolved)"

	create-release:
	needs: [setup, build-ubuntu]
	runs-on: ubuntu-24.04
	permissions:
	contents: write
	strategy:
	fail-fast: false
	# Each variant publishes to its own dated tag independently.
	matrix: ${{ fromJSON(needs.setup.outputs.matrix) }}
	# Publish only on cron/dispatch that sets create_release=true. Push/PR never
	# release. Require the build to succeed. NOTE: the gfx1151 hardware-test gate
	# (test-gfx in needs + needs.test-gfx.result == 'success') is TEMPORARILY
	# REMOVED while the self-hosted runner is offline and test-gfx is disabled
	# above. Restore both when the runner returns.
	if: \|
	always() &&
	needs.build-ubuntu.result == 'success' &&
	github.event_name == 'workflow_dispatch' &&
	github.event.inputs.create_release == 'true'
	steps:
	- name: Checkout repository
	uses: actions/checkout@v4

	- name: Download this variant's build artifact
	uses: actions/download-artifact@v4
	with:
	name: llama-ubuntu-rocm-multiarch-x64-${{ matrix.variant }}
	path: ./artifact

	- name: Read build metadata
	id: meta
	run: \|
	meta="./artifact/build-meta.txt"
	if [ ! -f "$meta" ]; then
	echo "ERROR: build-meta.txt missing from artifact" >&2
	exit 1
	fi
	# Re-export the recorded provenance as step outputs.
	while IFS='=' read -r k v; do
	[ -n "$k" ] && echo "$k=$v" >> "$GITHUB_OUTPUT"
	done < "$meta"
	echo "=== build-meta.txt ==="; cat "$meta"

	- name: Package release archive
	run: \|
	TAG="${{ matrix.tag }}"
	archive="llama-${TAG}-ubuntu-rocm-multiarch-x64"
	# build-meta.txt is shipped inside the tarball as provenance; keep it.
	echo "Creating ${archive}.tar.gz from ./artifact"
	tar -czf "${archive}.tar.gz" -C ./artifact .
	ls -la *.tar.gz

	- name: Publish dated release
	env:
	GITHUB_TOKEN: ${{ github.token }}
	TAG: ${{ matrix.tag }}
	ROCM_VERSION: ${{ steps.meta.outputs.rocm_version }}
	LLAMACPP_COMMIT_HASH: ${{ steps.meta.outputs.llamacpp_commit_hash }}
	BUILT_SHA: ${{ steps.meta.outputs.llamacpp_commit_sha }}
	SRC_REF: ${{ matrix.ref }}
	run: \|
	# Dated, persistent tag (<base>-YYYYMMDD): older builds stay downloadable
	# for issue repro; prune manually if storage becomes a concern. If a same-
	# day tag already exists (a re-run), replace it so the day reflects the
	# latest build at the exact commit this variant built.
	if gh release view "$TAG" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then
	echo "Deleting existing same-day release/tag $TAG (re-run)"
	gh release delete "$TAG" --repo "$GITHUB_REPOSITORY" --yes --cleanup-tag
	fi

	display_ref="${SRC_REF:-${GITHUB_REF_NAME}}"
	archive="llama-${TAG}-ubuntu-rocm-multiarch-x64.tar.gz"

	gh release create "$TAG" \
	--repo "$GITHUB_REPOSITORY" \
	--target "$BUILT_SHA" \
	--title "$TAG" \
	--notes "Dated build: $TAG (persistent; prune manually)
	OS: ubuntu
	GPU Target(s): gfx1100, gfx1101, gfx1102, gfx1103, gfx1150, gfx1151, gfx1152, gfx1153 (single multiarch binary)
	Source ref: $display_ref
	ROCm Version: $ROCM_VERSION (multiarch)
	Llama.cpp Commit: $LLAMACPP_COMMIT_HASH ($BUILT_SHA)
	Build Date: $(date -u '+%Y-%m-%d %H:%M:%S UTC')

	Prebuilt llama.cpp ROCm binaries — one multiarch package covering the RDNA3 gfx110X family incl. Hawk Point/Phoenix (Radeon 760M/780M) and the RDNA3.5 gfx115x APUs (gfx1150/gfx1151/gfx1152/gfx1153). Built from TheRock's multiarch ROCm runtime, pruned to the CI target arches (per-arch Tensile databases bundled)." \
	"$archive"

Navigation Menu

Uh oh!

Build gfx11 + ROCm #337

Workflow file

Build gfx11 + ROCm #337

Uh oh!

Workflow file for this run