Skip to content

Commit f43c5b4

Browse files
rsudermanAaronStGeorge
rsuderman
authored and
AaronStGeorge
committed
ggml-hrx: generalize kernel support and add llama support
1 parent 33a1f2b commit f43c5b4

164 files changed

Lines changed: 28006 additions & 2469 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎ggml/src/ggml-hrx/CMakeLists.txt‎

Lines changed: 75 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ if(HRX_SOURCE_DIR)
1515
set(GGML_LOOMC_LIB "${GGML_HRX_BUILD_DIR}/loom/binding/c/${CMAKE_SHARED_LIBRARY_PREFIX}loomc${CMAKE_SHARED_LIBRARY_SUFFIX}")
1616
set(GGML_HRX_LOOM_LINK "${GGML_HRX_BUILD_DIR}/loom/src/loom/tools/loom-link/loom-link${CMAKE_EXECUTABLE_SUFFIX}")
1717
set(GGML_HRX_LOOM_FORMAT "${GGML_HRX_BUILD_DIR}/loom/src/loom/tools/loom-format/loom-format${CMAKE_EXECUTABLE_SUFFIX}")
18+
set(GGML_HRX_IREE_BENCHMARK_LOOM "${GGML_HRX_BUILD_DIR}/loom/src/loom/tools/iree-benchmark-loom/iree-benchmark-loom${CMAKE_EXECUTABLE_SUFFIX}")
1819
set(GGML_HRX_DEPS_TARGET ggml-hrx-deps)
1920

2021
set(GGML_HRX_CMAKE_ARGS
@@ -41,9 +42,10 @@ if(HRX_SOURCE_DIR)
4142
SOURCE_DIR "${HRX_SOURCE_DIR}"
4243
PREFIX "${GGML_HRX_PREFIX}"
4344
CMAKE_ARGS ${GGML_HRX_CMAKE_ARGS}
44-
BUILD_COMMAND ${CMAKE_COMMAND} --build . --target hrx loomc_shared loom_tools_loom-link_loom-link loom_tools_loom-format_loom-format --config ${CMAKE_BUILD_TYPE}
45+
BUILD_ALWAYS TRUE
46+
BUILD_COMMAND ${CMAKE_COMMAND} --build . --target hrx loomc_shared loom_tools_loom-link_loom-link loom_tools_loom-format_loom-format loom_tools_iree-benchmark-loom_iree-benchmark-loom --config ${CMAKE_BUILD_TYPE}
4547
INSTALL_COMMAND ""
46-
BUILD_BYPRODUCTS "${GGML_HRX_LIB}" "${GGML_LOOMC_LIB}" "${GGML_HRX_LOOM_LINK}" "${GGML_HRX_LOOM_FORMAT}"
48+
BUILD_BYPRODUCTS "${GGML_HRX_LIB}" "${GGML_LOOMC_LIB}" "${GGML_HRX_LOOM_LINK}" "${GGML_HRX_LOOM_FORMAT}" "${GGML_HRX_IREE_BENCHMARK_LOOM}"
4749
UPDATE_COMMAND ""
4850
)
4951

@@ -64,6 +66,7 @@ else()
6466
find_package(loomc CONFIG REQUIRED)
6567
find_program(GGML_HRX_LOOM_LINK NAMES loom-link)
6668
find_program(GGML_HRX_LOOM_FORMAT NAMES loom-format)
69+
find_program(GGML_HRX_IREE_BENCHMARK_LOOM NAMES iree-benchmark-loom)
6770
endif()
6871

6972
find_package(Python3 REQUIRED COMPONENTS Interpreter)
@@ -86,8 +89,30 @@ elseif(NOT GGML_HRX_KERNEL_CORPUS_SOURCE_FORMAT STREQUAL "text")
8689
message(FATAL_ERROR "Unsupported GGML_HRX_KERNEL_CORPUS_SOURCE_FORMAT: ${GGML_HRX_KERNEL_CORPUS_SOURCE_FORMAT}")
8790
endif()
8891

89-
set(GGML_HRX_QWEN_KERNEL_CORPUS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/kernel-corpus/kernels/qwen_moe")
92+
set(GGML_HRX_QWEN_MOE_KERNEL_CORPUS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/kernel-corpus/kernels/qwen_moe")
93+
set(GGML_HRX_QWEN_MOE_KERNEL_CORPUS_MANIFEST "${GGML_HRX_QWEN_MOE_KERNEL_CORPUS_DIR}/manifest.json")
94+
set(GGML_HRX_HRX_KERNEL_CORPUS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/kernel-corpus/kernels/hrx")
95+
set(GGML_HRX_HRX_KERNEL_CORPUS_MANIFEST "${GGML_HRX_HRX_KERNEL_CORPUS_DIR}/manifest.json")
96+
set(GGML_HRX_QWEN_KERNEL_CORPUS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/kernel-corpus/kernels/qwen")
9097
set(GGML_HRX_QWEN_KERNEL_CORPUS_MANIFEST "${GGML_HRX_QWEN_KERNEL_CORPUS_DIR}/manifest.json")
98+
set(GGML_HRX_LOOM_LIBS_KERNEL_CORPUS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/kernel-corpus/kernels/loom-libs")
99+
set(GGML_HRX_LOOM_LIBS_KERNEL_CORPUS_MANIFEST "${GGML_HRX_LOOM_LIBS_KERNEL_CORPUS_DIR}/manifest.json")
100+
set(GGML_HRX_KERNEL_CORPUS_MANIFESTS
101+
"${GGML_HRX_QWEN_MOE_KERNEL_CORPUS_MANIFEST}"
102+
"${GGML_HRX_HRX_KERNEL_CORPUS_MANIFEST}"
103+
"${GGML_HRX_QWEN_KERNEL_CORPUS_MANIFEST}"
104+
"${GGML_HRX_LOOM_LIBS_KERNEL_CORPUS_MANIFEST}"
105+
)
106+
set(GGML_HRX_KERNEL_CORPUS_MANIFEST_ARGS
107+
--manifest "${GGML_HRX_QWEN_MOE_KERNEL_CORPUS_MANIFEST}"
108+
--corpus-dir "${GGML_HRX_QWEN_MOE_KERNEL_CORPUS_DIR}"
109+
--manifest "${GGML_HRX_HRX_KERNEL_CORPUS_MANIFEST}"
110+
--corpus-dir "${GGML_HRX_HRX_KERNEL_CORPUS_DIR}"
111+
--manifest "${GGML_HRX_QWEN_KERNEL_CORPUS_MANIFEST}"
112+
--corpus-dir "${GGML_HRX_QWEN_KERNEL_CORPUS_DIR}"
113+
--manifest "${GGML_HRX_LOOM_LIBS_KERNEL_CORPUS_MANIFEST}"
114+
--corpus-dir "${GGML_HRX_LOOM_LIBS_KERNEL_CORPUS_DIR}"
115+
)
91116
set(GGML_HRX_KERNEL_CORPUS_SOURCES_INC "${CMAKE_CURRENT_BINARY_DIR}/kernel-corpus-sources.inc")
92117
set(GGML_HRX_KERNEL_CORPUS_QWEN_INC "${CMAKE_CURRENT_BINARY_DIR}/kernel-corpus-qwen.inc")
93118
set(GGML_HRX_KERNEL_CORPUS_CATALOG_INC "${CMAKE_CURRENT_BINARY_DIR}/kernel-corpus-catalog.inc")
@@ -103,14 +128,13 @@ add_custom_command(
103128
--source-output "${GGML_HRX_KERNEL_CORPUS_SOURCES_INC}"
104129
--corpus-output "${GGML_HRX_KERNEL_CORPUS_QWEN_INC}"
105130
--catalog-output "${GGML_HRX_KERNEL_CORPUS_CATALOG_INC}"
106-
--manifest "${GGML_HRX_QWEN_KERNEL_CORPUS_MANIFEST}"
107-
--corpus-dir "${GGML_HRX_QWEN_KERNEL_CORPUS_DIR}"
131+
${GGML_HRX_KERNEL_CORPUS_MANIFEST_ARGS}
108132
--source-format "${GGML_HRX_KERNEL_CORPUS_SOURCE_FORMAT}"
109133
${GGML_HRX_KERNEL_CORPUS_TOOL_ARGS}
110134
--depfile "${GGML_HRX_KERNEL_CORPUS_DEPFILE}"
111135
DEPENDS
112136
"${CMAKE_CURRENT_SOURCE_DIR}/tools/generate_kernel_corpus.py"
113-
"${GGML_HRX_QWEN_KERNEL_CORPUS_MANIFEST}"
137+
${GGML_HRX_KERNEL_CORPUS_MANIFESTS}
114138
${GGML_HRX_KERNEL_CORPUS_TOOL_DEPENDS}
115139
DEPFILE "${GGML_HRX_KERNEL_CORPUS_DEPFILE}"
116140
VERBATIM
@@ -148,6 +172,8 @@ ggml_add_backend_library(ggml-hrx
148172
dispatch/command-plan.h
149173
dispatch/command-program-bindings.cpp
150174
dispatch/command-program-bindings.h
175+
dispatch/command-program-dump.cpp
176+
dispatch/command-program-dump.h
151177
dispatch/command-program-diagnostics.cpp
152178
dispatch/command-program-diagnostics.h
153179
dispatch/command-program.cpp
@@ -159,30 +185,51 @@ ggml_add_backend_library(ggml-hrx
159185
dispatch/dispatch.h
160186
dispatch/transient-allocator.cpp
161187
dispatch/transient-allocator.h
162-
dispatch_registration/dispatch-add.cpp
163-
dispatch_registration/dispatch-add.h
164-
dispatch_registration/dispatch-gather-add.cpp
165-
dispatch_registration/dispatch-gather-add.h
166-
dispatch_registration/dispatch-llm-matmul.cpp
167-
dispatch_registration/dispatch-llm-matmul.h
168-
dispatch_registration/dispatch-llm-profiles.h
169-
dispatch_registration/dispatch-llm-shapes.h
170-
dispatch_registration/dispatch-qwen-attention-postprocess.cpp
171-
dispatch_registration/dispatch-qwen-attention-postprocess.h
172-
dispatch_registration/dispatch-qwen-flash-attention.cpp
173-
dispatch_registration/dispatch-qwen-flash-attention.h
174-
dispatch_registration/dispatch-qwen-matmul.cpp
175-
dispatch_registration/dispatch-qwen-matmul.h
176-
dispatch_registration/dispatch-moe-router.cpp
177-
dispatch_registration/dispatch-moe-router.h
178-
dispatch_registration/dispatch-qwen-preamble.cpp
179-
dispatch_registration/dispatch-qwen-preamble.h
188+
dispatch_registration/common/dispatch-binary.cpp
189+
dispatch_registration/common/dispatch-binary.h
190+
dispatch_registration/common/dispatch-common.cpp
191+
dispatch_registration/common/dispatch-common.h
192+
dispatch_registration/common/dispatch-flash-attention.cpp
193+
dispatch_registration/common/dispatch-flash-attention.h
194+
dispatch_registration/common/dispatch-gated-mul-mat-id.cpp
195+
dispatch_registration/common/dispatch-gated-mul-mat-id.h
196+
dispatch_registration/common/dispatch-gated-mul-mat.cpp
197+
dispatch_registration/common/dispatch-gated-mul-mat.h
198+
dispatch_registration/common/dispatch-gather-add.cpp
199+
dispatch_registration/common/dispatch-gather-add.h
200+
dispatch_registration/common/dispatch-get-rows.cpp
201+
dispatch_registration/common/dispatch-get-rows.h
202+
dispatch_registration/common/dispatch-mul-mat-id.cpp
203+
dispatch_registration/common/dispatch-mul-mat-id-common.h
204+
dispatch_registration/common/dispatch-mul-mat-id.h
205+
dispatch_registration/common/dispatch-mul-mat.cpp
206+
dispatch_registration/common/dispatch-mul-mat-common.h
207+
dispatch_registration/common/dispatch-mul-mat-weight-format.h
208+
dispatch_registration/common/dispatch-mul-mat.h
209+
dispatch_registration/common/dispatch-rope-set-rows.cpp
210+
dispatch_registration/common/dispatch-rope-set-rows.h
211+
dispatch_registration/common/dispatch-rmsnorm.cpp
212+
dispatch_registration/common/dispatch-rmsnorm.h
213+
dispatch_registration/common/dispatch-unary.cpp
214+
dispatch_registration/common/dispatch-unary.h
180215
dispatch_registration/dispatch-registry.cpp
181216
dispatch_registration/dispatch-registry.h
182-
dispatch_registration/dispatch-rmsnorm.cpp
183-
dispatch_registration/dispatch-rmsnorm.h
184-
dispatch_registration/dispatch-routed-ffn.cpp
185-
dispatch_registration/dispatch-routed-ffn.h
217+
dispatch_registration/llm/dispatch-attention-qkv.cpp
218+
dispatch_registration/llm/dispatch-attention-qkv.h
219+
dispatch_registration/qwen/dispatch-llm-profiles.h
220+
dispatch_registration/qwen/dispatch-llm-shapes.h
221+
dispatch_registration/qwen/dispatch-moe-router.cpp
222+
dispatch_registration/qwen/dispatch-moe-router.h
223+
dispatch_registration/qwen/dispatch-qwen.cpp
224+
dispatch_registration/qwen/dispatch-qwen.h
225+
dispatch_registration/qwen/dispatch-qwen-attention-postprocess.cpp
226+
dispatch_registration/qwen/dispatch-qwen-attention-postprocess.h
227+
dispatch_registration/qwen/dispatch-qwen-matmul.cpp
228+
dispatch_registration/qwen/dispatch-qwen-matmul.h
229+
dispatch_registration/qwen/dispatch-qwen-rmsnorm.cpp
230+
dispatch_registration/qwen/dispatch-qwen-rmsnorm.h
231+
dispatch_registration/qwen/dispatch-routed-ffn.cpp
232+
dispatch_registration/qwen/dispatch-routed-ffn.h
186233
status.h
187234
graph/graph.cpp
188235
graph/graph-diagnostics.cpp

‎ggml/src/ggml-hrx/benchmarks/README.md‎

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
# HRX Loom Benchmarks
2+
3+
This directory contains model-scoped Loom benchmarks for HRX kernels. Benchmark sources stay separate from the production kernel corpus: they declare the kernels they need with `kernel.decl`, and the runner links those declarations against the production `.loom` files before benchmarking.
4+
5+
Use these benchmarks to measure production kernels with `iree-benchmark-loom` while keeping model-shaped benchmark cases out of the embedded kernel catalog. Each model has one `.loom` file under `loom/`, and each workload shape has a sidecar manifest next to it, for example `llama32_3b_f16.pp512.json`.
6+
7+
Generated model benchmarks are deduplicated by kernel shape. The sidecar manifest keeps a compact `dispatches` list with the generated benchmark name, kernel, compile/runtime parameters, source files, and `count`, so benchmark results can be weighted back to the full model without materializing one check case for every duplicate dispatch.
8+
9+
Generated model benchmarks are performance fixtures, not numerical validators. They launch the model-shaped kernels with synthetic buffers and intentionally omit `check.expect.*` assertions; backend model smoke tests and the kernel corpus checks remain responsible for numerical validation.
10+
11+
## Generating Llama Benchmarks
12+
13+
First dump the HRX command programs for each model run:
14+
15+
```sh
16+
GGML_HRX_DUMP_COMMAND_PROGRAM_DIR=/tmp/hrx-llama32-pp256-dumps \
17+
<build-dir>/bin/llama-bench \
18+
--model /home/rsuderman/Downloads/gguf/llama-3.2/Llama-3.2-3B-Instruct-F16.gguf \
19+
--device HRX0 \
20+
--n-gpu-layers -1 \
21+
--batch-size 64 \
22+
--ubatch-size 64 \
23+
--repetitions 1 \
24+
--no-warmup \
25+
--output jsonl \
26+
--n-prompt 256 \
27+
--n-gen 0 \
28+
--n-depth 0
29+
30+
GGML_HRX_DUMP_COMMAND_PROGRAM_DIR=/tmp/hrx-llama32-pp512-dumps \
31+
<build-dir>/bin/llama-bench \
32+
--model /home/rsuderman/Downloads/gguf/llama-3.2/Llama-3.2-3B-Instruct-F16.gguf \
33+
--device HRX0 \
34+
--n-gpu-layers -1 \
35+
--batch-size 64 \
36+
--ubatch-size 64 \
37+
--repetitions 1 \
38+
--no-warmup \
39+
--output jsonl \
40+
--n-prompt 512 \
41+
--n-gen 0 \
42+
--n-depth 0
43+
44+
GGML_HRX_DUMP_COMMAND_PROGRAM_DIR=/tmp/hrx-llama32-tg8-dumps \
45+
<build-dir>/bin/llama-bench \
46+
--model /home/rsuderman/Downloads/gguf/llama-3.2/Llama-3.2-3B-Instruct-F16.gguf \
47+
--device HRX0 \
48+
--n-gpu-layers -1 \
49+
--batch-size 64 \
50+
--ubatch-size 64 \
51+
--repetitions 1 \
52+
--no-warmup \
53+
--output jsonl \
54+
--n-prompt 0 \
55+
--n-gen 8 \
56+
--n-depth 0
57+
```
58+
59+
Then generate the shared model benchmark file and per-scenario sidecars:
60+
61+
```sh
62+
ggml/src/ggml-hrx/tools/benchmarks/generate-model-benchmarks.py \
63+
--model llama32_3b_f16 \
64+
--scenario-dump pp256=/tmp/hrx-llama32-pp256-dumps \
65+
--scenario-dump pp512=/tmp/hrx-llama32-pp512-dumps \
66+
--scenario-dump tg8=/tmp/hrx-llama32-tg8-dumps
67+
```
68+
69+
Review the generated invoked-kernel set with:
70+
71+
```sh
72+
jq '.kernel_counts' ggml/src/ggml-hrx/benchmarks/loom/llama32_3b_f16.<scenario>.json
73+
74+
ggml/src/ggml-hrx/tools/benchmarks/run-model-benchmarks.sh \
75+
--model llama32_3b_f16 \
76+
--scenario <scenario> \
77+
--build-dir <build-dir> \
78+
--list
79+
```
80+
81+
Run the generated benchmarks with:
82+
83+
```sh
84+
ggml/src/ggml-hrx/tools/benchmarks/run-model-benchmarks.sh \
85+
--model llama32_3b_f16 \
86+
--scenario <scenario> \
87+
--build-dir <build-dir> \
88+
--output-dir /home/rsuderman/codex/project-workspaces/llama.cpp/gates/hrx-loom-benchmarks/llama32-<scenario>
89+
```
90+
91+
Use `--dry-run-only` to stop after planning, `--list` to inspect the generated benchmark set, or `--benchmark @name` to run a single invocation. Set `LOOM_LINK` or `IREE_BENCHMARK_LOOM` to override tool discovery. Set `DEVICE` to override the default `amdgpu` HAL device.
92+
93+
The runner applies the same workload-argument specialization used by the HRX runtime after `loom-link`, and benchmarks the resulting `linked.runtime-specialized.loom` source by default. Use `--no-runtime-specialization` to benchmark the raw linked source.
94+
95+
`loom-link` also receives each captured compile parameter as `--config=<name>=<value>`. Some model kernels select templates from compile-time config values, so passing those configs during link keeps standalone benchmarks aligned with the HRX runtime path.
96+
97+
Use `--continue-on-failure` to record failed or timed-out standalone cases and keep running the rest of the model benchmark set. Use `--benchmark-timeout-sec` to bound each link, dry-run, and benchmark command. Each benchmark directory keeps `loom-link`, dry-run, and benchmark command lines plus stdout/stderr so failures can be reproduced directly.
98+
99+
Use `--profile-final-batch=false`, `--input-ring-count`, or repeated `--benchmark-extra-arg` flags when isolating `iree-benchmark-loom` behavior from production runtime behavior. After a run, summarize the weighted model time with:
100+
101+
```sh
102+
ggml/src/ggml-hrx/tools/benchmarks/summarize-model-benchmarks.py \
103+
/home/rsuderman/codex/project-workspaces/llama.cpp/gates/hrx-loom-benchmarks/llama32-pp512/results.jsonl
104+
```
105+
106+
The summary uses `operation_timing_ns.p50 * count` by default and writes `summary.json` plus `summary.md` next to the runner results.
107+
108+
## Generating Qwen 30B Benchmarks
109+
110+
Dump the HRX command programs with the Qwen 30B shard and the same pp256, pp512, and tg8 shapes used for model benchmarking:
111+
112+
```sh
113+
GGML_HRX_DUMP_COMMAND_PROGRAM_DIR=/tmp/hrx-qwen30b-pp256-dumps \
114+
<build-dir>/bin/llama-bench \
115+
--model /home/rsuderman/Downloads/gguf/qwen-30b/qwen3-30b-a3b-q4_k_m-00001-of-00020.gguf \
116+
--device HRX0 \
117+
--n-gpu-layers -1 \
118+
--batch-size 64 \
119+
--ubatch-size 64 \
120+
--repetitions 1 \
121+
--no-warmup \
122+
--output jsonl \
123+
--n-prompt 256 \
124+
--n-gen 0 \
125+
--n-depth 0
126+
127+
GGML_HRX_DUMP_COMMAND_PROGRAM_DIR=/tmp/hrx-qwen30b-pp512-dumps \
128+
<build-dir>/bin/llama-bench \
129+
--model /home/rsuderman/Downloads/gguf/qwen-30b/qwen3-30b-a3b-q4_k_m-00001-of-00020.gguf \
130+
--device HRX0 \
131+
--n-gpu-layers -1 \
132+
--batch-size 64 \
133+
--ubatch-size 64 \
134+
--repetitions 1 \
135+
--no-warmup \
136+
--output jsonl \
137+
--n-prompt 512 \
138+
--n-gen 0 \
139+
--n-depth 0
140+
141+
GGML_HRX_DUMP_COMMAND_PROGRAM_DIR=/tmp/hrx-qwen30b-tg8-dumps \
142+
<build-dir>/bin/llama-bench \
143+
--model /home/rsuderman/Downloads/gguf/qwen-30b/qwen3-30b-a3b-q4_k_m-00001-of-00020.gguf \
144+
--device HRX0 \
145+
--n-gpu-layers -1 \
146+
--batch-size 64 \
147+
--ubatch-size 64 \
148+
--repetitions 1 \
149+
--no-warmup \
150+
--output jsonl \
151+
--n-prompt 0 \
152+
--n-gen 8 \
153+
--n-depth 0
154+
```
155+
156+
Then regenerate the shared Qwen benchmark source and sidecars:
157+
158+
```sh
159+
ggml/src/ggml-hrx/tools/benchmarks/generate-model-benchmarks.py \
160+
--model qwen3_30b_a3b_q4_k_m \
161+
--scenario-dump pp256=/tmp/hrx-qwen30b-pp256-dumps \
162+
--scenario-dump pp512=/tmp/hrx-qwen30b-pp512-dumps \
163+
--scenario-dump tg8=/tmp/hrx-qwen30b-tg8-dumps
164+
```

0 commit comments

Comments
 (0)