|
| 1 | +"""Offload-runtime diagnostics for GPU memory faults. |
| 2 | +
|
| 3 | +Shared by the test harness, the benchmark runner and the case-optimization CI |
| 4 | +script -- all three run GPU cases and all three need the same answer when one |
| 5 | +faults. Kept out of test/ because bench.py depending on the test module to |
| 6 | +explain a crash would be the wrong way round. |
| 7 | +""" |
| 8 | + |
| 9 | +import collections |
| 10 | +import os |
| 11 | +import re |
| 12 | +import typing |
| 13 | + |
| 14 | +# The marker _handle_case attaches to the exception it raises, so that |
| 15 | +# classify_error can tell a GPU memory fault from any other execution failure. |
| 16 | +# Two constraints, both learned the hard way: |
| 17 | +# |
| 18 | +# * It must be one of the signatures below verbatim, because classify_error |
| 19 | +# recognises it by running the same matcher over the message. An earlier |
| 20 | +# version wrote "[gpu-memory-fault]" while the reader searched for "memory |
| 21 | +# access fault by gpu", so the two never matched and the feature was dead |
| 22 | +# while seven source-inspecting tests passed. |
| 23 | +# * No square brackets. main.py renders these messages through Rich, which |
| 24 | +# parses "[...]" as a style tag and deletes it -- which is why a CI log |
| 25 | +# showed a bare "Failed to execute MFC. " with the marker missing. |
| 26 | +GPU_FAULT_MARKER = "(memory access fault by GPU)" |
| 27 | + |
| 28 | +GPU_FAULT_SIGNATURES = ( |
| 29 | + # AMD/HSA -- Frontier, both CCE and AFAR builds. |
| 30 | + "memory access fault by gpu", |
| 31 | + "offload error: memory access fault", |
| 32 | + # NVHPC -- Phoenix. Worded nothing like the AMD ones, so matching only the |
| 33 | + # above meant 189 faults on a Phoenix gpu-acc shard were never recognised. |
| 34 | + # Only the specific error: NVHPC prefixes unrelated failures with |
| 35 | + # "Accelerator Fatal Error" too, including "call to cuMemAlloc returned |
| 36 | + # error 2: Out of memory", which is not a memory fault and must not be |
| 37 | + # classified as one. |
| 38 | + "cuda_error_illegal_address", |
| 39 | +) |
| 40 | + |
| 41 | + |
| 42 | +def is_gpu_memory_fault(text: str) -> bool: |
| 43 | + """Whether output shows a GPU memory fault, as opposed to any other failure. |
| 44 | +
|
| 45 | + Deliberately narrow. PMIX_ERR_NO_PERMISSIONS and friends appear in 16% of |
| 46 | + *passing* self-hosted jobs, so anything broader would fire constantly. |
| 47 | + """ |
| 48 | + lowered = (text or "").lower() |
| 49 | + |
| 50 | + return any(sig in lowered for sig in GPU_FAULT_SIGNATURES) |
| 51 | + |
| 52 | + |
| 53 | +def fault_diagnostic_env(base: dict) -> dict: |
| 54 | + """`base` plus the offload diagnostics that cost nothing until a fault. |
| 55 | +
|
| 56 | + These are set on EVERY run rather than on a retry. Both variables are |
| 57 | + inert in a healthy run -- they only produce output when the runtime is |
| 58 | + already aborting on a memory fault -- so paying for them up front makes the |
| 59 | + first failure informative instead of spending a whole extra run to learn |
| 60 | + the same thing. |
| 61 | +
|
| 62 | + That is the opposite of how this started. The original design retried a |
| 63 | + faulted case with diagnostics on, which measurement showed was the wrong |
| 64 | + shape: AFAR names the faulting kernel unaided in 189 of 189 faults, and |
| 65 | + NVHPC prints its file, function and line. |
| 66 | +
|
| 67 | + CCE names nothing on its own and no CRAY_ACC_* variable helps -- but that |
| 68 | + is a limit of CCE's trace, not of the machine. The ROCm debug agent works, |
| 69 | + one layer down at ROCr: HSA_TOOLS_LIB=librocm-debug-agent.so.2 prints |
| 70 | + "Disassembly for function s_tvd_rk$m_time_steppers_$ck_L486_6" -- the exact |
| 71 | + injected fault site -- plus the faulting instruction and per-wave register |
| 72 | + state, straight to the job log. It is deliberately not set here yet: its |
| 73 | + cost on a healthy run is unmeasured, and that decides always-on versus a |
| 74 | + documented recipe. See #1801. |
| 75 | +
|
| 76 | + Deliberately NOT set here: CRAY_ACC_DEBUG. It streams a line per launch and |
| 77 | + per transfer for the whole run, and because dispatch is async its tail is |
| 78 | + whatever ran next -- it blamed s_write_run_time_information in 81 of 102 |
| 79 | + traced faults and the true culprit in 0. A confident wrong suspect is worse |
| 80 | + than silence, and it is not free the way these two are. |
| 81 | +
|
| 82 | + Returns a new dict: these run in worker threads, and mutating a shared |
| 83 | + environment would leak settings into every concurrent case. |
| 84 | + """ |
| 85 | + env = { |
| 86 | + **base, |
| 87 | + # Says whether the faulting address was ever a real host allocation, |
| 88 | + # separating an overrun of a known array from a wild pointer. |
| 89 | + "OFFLOAD_TRACK_ALLOCATION_TRACES": "true", |
| 90 | + # Host stack traces for the most recent kernel launches. The runtime |
| 91 | + # advertises this itself in the fault message ("0 now, up to 8"). |
| 92 | + "OFFLOAD_TRACK_NUM_KERNEL_LAUNCH_TRACES": "8", |
| 93 | + } |
| 94 | + |
| 95 | + # The only thing that gives CCE a faulting kernel. Measured on Frontier |
| 96 | + # under --gpu acc: it prints "Disassembly for function |
| 97 | + # s_tvd_rk$m_time_steppers_$ck_L486_6", the exact injected fault site, with |
| 98 | + # the faulting instruction and per-wave registers -- where no CRAY_ACC_* |
| 99 | + # variable names it at all. |
| 100 | + # |
| 101 | + # Measured on all four lanes. The symbol form is set by the COMPILER, not by |
| 102 | + # the offload model -- which is the opposite of what it looks like from any |
| 103 | + # two of them: |
| 104 | + # |
| 105 | + # CCE acc s_tvd_rk$m_time_steppers_$ck_L486_6 |
| 106 | + # CCE mp s_tvd_rk$m_time_steppers_$ck_L486_16 (same scheme, counter differs) |
| 107 | + # AFAR mp __omp_offloading_..._QMm_time_steppersPs_tvd_rk_l486 (Flang) |
| 108 | + # |
| 109 | + # All three carry module, subroutine and line. The summarizer does not care |
| 110 | + # which -- its regex takes whatever the symbol is -- but anything that tries |
| 111 | + # to parse the symbol must not assume one scheme per offload model. |
| 112 | + # |
| 113 | + # Cost on a healthy run: one paired A/B put it at 4.5645 ns/gp/eq/rhs |
| 114 | + # against an agent-free spread of 4.5301-4.5614, i.e. 0.07% above a range |
| 115 | + # 0.69% wide -- inside the noise. That is n=1; the repeats were cancelled |
| 116 | + # deliberately rather than measured, so this is "no effect detected", not |
| 117 | + # "no effect". |
| 118 | + # |
| 119 | + # Unverified: how this interacts with the AFAR variables above on the |
| 120 | + # frontier_amd lane, where both are reachable. The agent is mutually |
| 121 | + # exclusive with ROCr core dumps, so it may likewise supersede libomptarget's |
| 122 | + # own fault report. Worst realistic case is one working diagnostic replacing |
| 123 | + # another strictly more detailed one; if a real AFAR fault shows otherwise, |
| 124 | + # gate this on the lane. |
| 125 | + agent = rocm_debug_agent_path() |
| 126 | + if agent is not None: |
| 127 | + env["HSA_TOOLS_LIB"] = ROCM_DEBUG_AGENT |
| 128 | + |
| 129 | + return env |
| 130 | + |
| 131 | + |
| 132 | +ROCM_DEBUG_AGENT = "librocm-debug-agent.so.2" |
| 133 | + |
| 134 | + |
| 135 | +def rocm_debug_agent_path() -> typing.Optional[str]: |
| 136 | + """Where the ROCm debug agent lives, or None if it is not reachable. |
| 137 | +
|
| 138 | + MUST be evaluated at call time, never cached at import. On Frontier the |
| 139 | + library is on disk the whole time, but /opt/rocm-*/lib only reaches |
| 140 | + LD_LIBRARY_PATH once `mfc.sh load` runs. A gate evaluated at import decides |
| 141 | + "absent" on the one machine this exists for, and does it indistinguishably |
| 142 | + from the Phoenix case where the library really is missing. |
| 143 | +
|
| 144 | + Probes for the file rather than dlopen'ing it: ctypes.CDLL would load a |
| 145 | + debug agent into the test harness's own process to answer a question about |
| 146 | + the subprocess. |
| 147 | + """ |
| 148 | + rocm_path = os.environ.get("ROCM_PATH", "") |
| 149 | + search = [os.path.join(rocm_path, "lib")] if rocm_path else [] |
| 150 | + search += os.environ.get("LD_LIBRARY_PATH", "").split(os.pathsep) |
| 151 | + |
| 152 | + for directory in search: |
| 153 | + if directory and os.path.isfile(os.path.join(directory, ROCM_DEBUG_AGENT)): |
| 154 | + return os.path.join(directory, ROCM_DEBUG_AGENT) |
| 155 | + |
| 156 | + return None |
| 157 | + |
| 158 | + |
| 159 | +def summarize_rocm_debug_agent(out: str, max_disasm: int = 14) -> str: |
| 160 | + """Collapse librocm-debug-agent output to a bounded, informative summary. |
| 161 | +
|
| 162 | + The agent repeats an identical disassembly block and a 115-line register |
| 163 | + dump per faulting wave -- 125 waves produced 14,635 lines on a 49x39 case. |
| 164 | + Only the kernel name, fault reason, stop-PC distribution and one |
| 165 | + representative wave carry information; the rest is duplicated. |
| 166 | +
|
| 167 | + A fixed tail cannot substitute. Measured on that log: the first 80 lines are |
| 168 | + one wave's registers and the last 80 are another's, and the kernel name -- |
| 169 | + the entire point -- appears in neither. The stop-PC histogram is kept |
| 170 | + because the waves halted at four distinct PCs whose modal one is a load |
| 171 | + while the injected fault is a write, so quoting a single PC without the |
| 172 | + distribution hands the reader the wrong instruction. |
| 173 | +
|
| 174 | + Returns '' when there is no agent report, so callers fall back to the raw |
| 175 | + output. |
| 176 | +
|
| 177 | + The format is NOT stable across ROCm versions, and the failure is silent -- |
| 178 | + no wave match means an empty summary and a fallback to tens of thousands of |
| 179 | + raw lines, with nothing saying why. Measured between two versions: |
| 180 | +
|
| 181 | + 6.3.1 wave_124: pc=0x7ff77e253408 (stopped, reason: MEMORY_VIOLATION) |
| 182 | + 7.2.0 wave_250: pc=0x7ff734dcbf3c (kernel_code_entry=0x... <...>, |
| 183 | + kernargs=0x...) (stopped, reason: MEMORY_VIOLATION) |
| 184 | +
|
| 185 | + 6.3.1 Memory access fault by GPU node-4 (Agent handle: ...) on address |
| 186 | + 7.2.0 OFFLOAD ERROR: memory access fault by GPU 4 (agent ...) at ... |
| 187 | +
|
| 188 | + An earlier version required pc= and "(stopped, reason:" to be adjacent and |
| 189 | + matched the fault line case-sensitively on "Memory". It returned nothing at |
| 190 | + all for 65,210 lines of real 7.2.0 output. Hence the tolerant separator, and |
| 191 | + reusing is_gpu_memory_fault rather than hardcoding one version's wording. |
| 192 | + Both formats are pinned by fixtures below. |
| 193 | +
|
| 194 | + Validated against three real reports, not one: |
| 195 | +
|
| 196 | + CCE acc ROCm 6.3.1 14,635 lines -> 37 |
| 197 | + CCE mp ROCm 6.3.1 13,826 lines -> 35 |
| 198 | + AFAR mp ROCm 7.2.0 65,210 lines -> 36 |
| 199 | +
|
| 200 | + and output from a run with no agent loaded still yields '', so the fallback |
| 201 | + is intact. The stop-PC histogram earns its place most on the CCE OpenMP |
| 202 | + lane, which halts at seven distinct PCs (62/21/19/10/10/2/1) against four |
| 203 | + for CCE OpenACC and one for AFAR: quoting a single PC would be wrong there |
| 204 | + six times in seven. |
| 205 | + """ |
| 206 | + waves = re.findall(r"^wave_\d+: pc=(0x[0-9a-f]+).*?\(stopped, reason: (\w+)\)", out, re.M) |
| 207 | + if not waves: |
| 208 | + return "" |
| 209 | + |
| 210 | + lines = out.splitlines() |
| 211 | + fault = next((line for line in lines if is_gpu_memory_fault(line)), None) |
| 212 | + kernels = sorted({m.group(1) for m in re.finditer(r"^Disassembly for function (.+):$", out, re.M)}) |
| 213 | + pcs = collections.Counter(pc for pc, _ in waves) |
| 214 | + reasons = collections.Counter(reason for _, reason in waves) |
| 215 | + |
| 216 | + summary = [f"=== GPU fault summary (rocm-debug-agent, {len(lines)} lines collapsed) ==="] |
| 217 | + if fault: |
| 218 | + summary.append(fault.strip()) |
| 219 | + summary.append("faulting kernel(s): " + (", ".join(kernels) or "<none reported>")) |
| 220 | + summary.append(f"faulting waves: {len(waves)} [" + ", ".join(f"{r} x{n}" for r, n in reasons.most_common()) + "]") |
| 221 | + summary.append("stop PCs: " + ", ".join(f"{pc} x{n}" for pc, n in pcs.most_common())) |
| 222 | + summary.append("NOTE: waves halt on fault detection, so the PC is near -- not necessarily at -- the offending instruction.") |
| 223 | + |
| 224 | + disasm_starts = [n for n, line in enumerate(lines) if line.startswith("Disassembly for function")] |
| 225 | + if disasm_starts: |
| 226 | + start = disasm_starts[0] |
| 227 | + end = next((n for n, line in enumerate(lines[start:], start) if line.startswith("End of disassembly")), start + max_disasm) |
| 228 | + summary += ["", f"--- disassembly (1 of {len(disasm_starts)} identical blocks) ---"] |
| 229 | + summary += lines[start : min(end + 1, start + max_disasm)] |
| 230 | + |
| 231 | + modal_pc = pcs.most_common(1)[0][0] |
| 232 | + start = next((n for n, line in enumerate(lines) if re.match(r"^wave_\d+: pc=" + re.escape(modal_pc) + r"(?![0-9a-f])", line)), None) |
| 233 | + if start is not None: |
| 234 | + summary += ["", f"--- representative wave (modal PC {modal_pc}, {pcs[modal_pc]} of {len(waves)} waves) ---"] |
| 235 | + summary += lines[start : start + max_disasm] |
| 236 | + summary.append(f" ... (registers for {len(waves) - 1} further waves suppressed)") |
| 237 | + |
| 238 | + return "\n".join(summary) |
0 commit comments