From 9bfacbc1680d4ad2143804927f34c00f10ef7d07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Jos=C3=A9=20Garc=C3=ADa=20Garc=C3=ADa?= Date: Wed, 26 Aug 2026 21:53:43 +0200 Subject: [PATCH] Name the end of the pipeline that actually broke MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Windows failed twice this month at the same line, and both logs said the same thing: bin/makepkg.conf: Write error: Broken pipe That is bsdtar, naming whatever file it happened to be reading when its output closed. The file was fine. What broke was the other end -- most likely xz, killed for memory during a silent three-minute -9 over the core payload -- and a process the kernel kills leaves no message at all, so nothing in the log was about it. Under pipefail the failing status does surface; the text does not, because it belongs to whichever element noticed first, and that is never the one that died. So both compression pipelines now capture PIPESTATUS and report every element that failed, by name, and a status above 128 as the signal it is. The status the caller sees is unchanged: report_pipeline returns the last failing one, which is what pipefail would have given it. Simulated by putting an xz on PATH that dies mid-write, against tests/package/test-core-package.sh: before : Write error (and a job-control dump) after packaging vdpm-…pkg.tar.xz: bsdtar exited 1 packaging vdpm-…pkg.tar.xz: xz killed by signal 9 both exiting 137. tests/ci/test-pipeline-report.sh drives the function the script defines, not a copy: silence when nothing failed, the killed compressor with its signal, the upstream SIGPIPE reported too, an ordinary exit as an exit, an element past the names given, and the status preserved. --- scripts/create-core-package.sh | 44 +++++++++++++++ tests/ci/test-pipeline-report.sh | 96 ++++++++++++++++++++++++++++++++ 2 files changed, 140 insertions(+) create mode 100755 tests/ci/test-pipeline-report.sh diff --git a/scripts/create-core-package.sh b/scripts/create-core-package.sh index f352d87..6426114 100755 --- a/scripts/create-core-package.sh +++ b/scripts/create-core-package.sh @@ -106,6 +106,40 @@ for path in ${client_paths[@]+"${client_paths[@]}"}; do mv "$core_root/$path" "$client_root/$path" done +# Which end of a compression pipeline actually broke. +# +# Under pipefail a failing status surfaces, but the text does not: whichever +# element notices first is the one that speaks, and that is never the one that +# died. When xz goes, bsdtar reports "Write error: Broken pipe" against +# whatever file it happened to be reading when its output closed -- and xz +# says nothing at all if the kernel killed it for memory, which is what -9 +# over the core payload invites. So name every element that failed, and how. +report_pipeline() { + local description=$1 name status index=0 failed=0 + shift + local -a names=() + while [[ $1 != -- ]]; do + names+=("$1") + shift + done + shift + for status in "$@"; do + name=${names[index]:-element $((index + 1))} + index=$((index + 1)) + ((status == 0)) && continue + # The last failing element's status, which is what pipefail would + # have surfaced: this says more than that, it does not say less. + failed=$status + if ((status > 128)); then + printf '%s: %s killed by signal %d\n' \ + "$description" "$name" "$((status - 128))" >&2 + else + printf '%s: %s exited %d\n' "$description" "$name" "$status" >&2 + fi + done + return "$failed" +} + set_tree_mtime() { local root=$1 timestamp if touch -h -d "@$source_date_epoch" "$root/.PKGINFO" 2>/dev/null; then @@ -171,22 +205,32 @@ EOF set_tree_mtime "$root" ( cd "$root" + set +e list_package_files .MTREE | LC_ALL=C sort -z | COPYFILE_DISABLE=1 LANG=C bsdtar -cnf - --format=mtree \ --no-acls --no-fflags --no-mac-metadata --no-xattrs \ --options='!all,use-set,type,uid,gid,mode,time,size,sha256,link' \ --uid 0 --gid 0 --null -T - | gzip -c -f -n > .MTREE + statuses=(${PIPESTATUS[@]+"${PIPESTATUS[@]}"}) + set -e + report_pipeline "writing .MTREE" \ + list_package_files sort bsdtar gzip -- ${statuses[@]+"${statuses[@]}"} ) set_tree_mtime "$root" ( cd "$root" + set +e list_package_files | LC_ALL=C sort -z | COPYFILE_DISABLE=1 LANG=C bsdtar -cnf - \ --no-acls --no-fflags --no-mac-metadata --no-xattrs \ --uid 0 --gid 0 --uname root --gname root --null -T - | xz -c -z -9 > "$path" + statuses=(${PIPESTATUS[@]+"${PIPESTATUS[@]}"}) + set -e + report_pipeline "packaging ${path##*/}" \ + list_package_files sort bsdtar xz -- ${statuses[@]+"${statuses[@]}"} ) "$script_directory/validate-core-package.sh" "$path" diff --git a/tests/ci/test-pipeline-report.sh b/tests/ci/test-pipeline-report.sh new file mode 100755 index 0000000..16accc3 --- /dev/null +++ b/tests/ci/test-pipeline-report.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# When a compression pipeline breaks, say which end broke. +# +# Under pipefail the failing status surfaces, but the message belongs to +# whichever element noticed first, and that is never the one that died: when +# xz goes, bsdtar reports "Write error: Broken pipe" against whatever file it +# was reading, and a compressor the kernel killed for memory leaves no +# message at all. Every log of this failure so far has named bsdtar and a +# file that was fine. + +set -euo pipefail + +repository_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P) +script="$repository_root/scripts/create-core-package.sh" + +failures=0 + +# The function as the script defines it, not a copy of it here. +report() +{ + bash -c ' + set -uo pipefail + '"$(sed -n '/^report_pipeline()/,/^}/p' "$script")"' + report_pipeline "$@" + ' report "$@" 2>&1 +} + +check() +{ + local description=$1 expected=$2 + shift 2 + local actual status=0 + actual=$(report "$@") || status=$? + case $expected in + ok) + if ((status != 0)) || [[ -n $actual ]]; then + printf 'FAIL: %s\n status %s, said: %s\n' \ + "$description" "$status" "$actual" >&2 + failures=$((failures + 1)) + fi + ;; + *) + if ((status == 0)); then + printf 'FAIL: %s was reported as success\n' "$description" >&2 + failures=$((failures + 1)) + elif [[ $actual != *"$expected"* ]]; then + printf 'FAIL: %s\n expected to contain: %s\n said: %s\n' \ + "$description" "$expected" "$actual" >&2 + failures=$((failures + 1)) + fi + ;; + esac +} + +check "a pipeline where nothing failed says nothing" ok \ + packaging a b c xz -- 0 0 0 0 + +# The case this exists for. 137 is SIGKILL, which is what the kernel's +# out-of-memory killer leaves behind, and it leaves nothing else. +check "a compressor killed for memory is named, with its signal" \ + "xz killed by signal 9" \ + packaging list_package_files sort bsdtar xz -- 141 0 141 137 + +# And the message that used to be the only one must not be the headline: the +# upstream SIGPIPE is a consequence, so it may be reported, but the killed +# element has to be there too. +actual=$(report packaging list_package_files sort bsdtar xz -- 141 0 141 137 || true) +if ! grep -q 'bsdtar killed by signal 13' <<<"$actual"; then + printf 'FAIL: the elements that took the SIGPIPE are not reported\n' >&2 + failures=$((failures + 1)) +fi + +check "an ordinary non-zero exit is reported as an exit" \ + "gzip exited 1" \ + "writing .MTREE" list_package_files sort bsdtar gzip -- 0 0 0 1 + +check "an element beyond the names given is still reported" \ + "element 5 exited 2" \ + packaging a b c d -- 0 0 0 0 2 + +# The status pipefail would have surfaced is still the status: a caller that +# was reading 137 out of this keeps reading 137. +status=0 +report packaging list_package_files sort bsdtar xz -- 141 0 1 137 >/dev/null 2>&1 || + status=$? +if ((status != 137)); then + printf 'FAIL: the last failing status is not preserved\n expected 137, got %s\n' \ + "$status" >&2 + failures=$((failures + 1)) +fi + +if ((failures)); then + printf '%d pipeline report check(s) failed\n' "$failures" >&2 + exit 1 +fi +printf 'pipeline failure reporting: all checks passed\n'