Skip to content

Commit 3b9e010

Browse files
Letdown2491Letdown2491
committed
smoke: the greeter gets memory headroom and the dump gets eyes
The chain dies the same way in every job: three seconds in, mid shader compile, no panic, no signal, no kernel line — and job verdicts flip run to run, which is a resource race wearing a workflow bug's clothes. A 4GB VM running wgpu over llvmpipe is tight, and systemd-oomd kills cgroups from userspace where nothing in the dump's greps reads. Four changes, one run: the VMs get 8GB; RUST_LOG drops to info so naga's flood stops costing journal memory and wall time; the wrapper traps TERM (systemd's polite half of a cgroup kill) and logs it; and the failure dump reads systemd-oomd's unit plus the unfiltered end of the boot journal, so whatever ends the session next has nowhere left to hide. Goldens re-blessed; suite green.
1 parent b4c7cd8 commit 3b9e010

5 files changed

Lines changed: 18 additions & 8 deletions

File tree

‎scripts/smoke.sh‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -256,6 +256,12 @@ bad() {
256256
guest 'journalctl -b --no-pager -t kuma-greeter | grep -a "wrapper:" | tail -30' || true
257257
guest 'journalctl -b --no-pager -t kuma-greeter | grep -aE "ERROR|panic" | tail -30' || true
258258
guest 'journalctl -b --no-pager -t kuma-greeter | tail -15' || true
259+
# oomd kills cgroups from userspace: no kernel oom-kill line,
260+
# just a line in its own unit that no tag grep above reads.
261+
guest 'journalctl -b --no-pager -u systemd-oomd | tail -10' || true
262+
# And whatever else ended the boot, its last words are here:
263+
# the unfiltered end of the journal, no tag can exclude it.
264+
guest 'journalctl -b --no-pager | tail -40' || true
259265
guest 'journalctl -b --no-pager | grep -iE "avc.*denied|selinux" | tail -20' || true
260266
# a compositor or GPU client dying without its own log line
261267
# leaves a kernel line instead: traps/segfault, oom-kill
@@ -957,7 +963,7 @@ smoke_published() {
957963
-global "ICH9-LPC.disable_s3=1")
958964
fi
959965
qemu-system-x86_64 \
960-
-enable-kvm -cpu host -smp 4 -m 4096 \
966+
-enable-kvm -cpu host -smp 4 -m 8192 \
961967
"${machine[@]}" "${globals[@]}" \
962968
-drive "if=pflash,format=raw,readonly=on,file=$code" \
963969
-drive "if=pflash,format=raw,file=$vars" \
@@ -2788,7 +2794,7 @@ dead_disk_run() {
27882794
fi || bad "cannot stage the OVMF vars"
27892795

27902796
qemu-system-x86_64 \
2791-
-enable-kvm -cpu host -smp 4 -m 4096 \
2797+
-enable-kvm -cpu host -smp 4 -m 8192 \
27922798
-machine q35 \
27932799
-drive "if=pflash,format=raw,readonly=on,file=$ovmf_code" \
27942800
-drive "if=pflash,format=raw,file=$dir/OVMF_VARS.fd" \
@@ -2979,7 +2985,7 @@ smoke_iso() {
29792985
cp "$ovmf_vars" "$dir/vars.fd"
29802986

29812987
env LIBGL_ALWAYS_SOFTWARE=1 qemu-system-x86_64 \
2982-
-enable-kvm -cpu host -smp 4 -m 4096 \
2988+
-enable-kvm -cpu host -smp 4 -m 8192 \
29832989
-drive "if=pflash,format=raw,readonly=on,file=$ovmf_code" \
29842990
-drive "if=pflash,format=raw,file=$dir/vars.fd" \
29852991
-cdrom "$iso" -boot d \
@@ -3100,7 +3106,7 @@ smoke_boot() {
31003106
cp "$ovmf_vars" "$dir/OVMF_VARS.fd" || bad "cannot stage the OVMF vars"
31013107

31023108
env LIBGL_ALWAYS_SOFTWARE=1 qemu-system-x86_64 \
3103-
-enable-kvm -cpu host -smp 4 -m 4096 \
3109+
-enable-kvm -cpu host -smp 4 -m 8192 \
31043110
-machine q35 \
31053111
-drive "if=pflash,format=raw,readonly=on,file=$ovmf_code" \
31063112
-drive "if=pflash,format=raw,file=$dir/OVMF_VARS.fd" \

‎src/containerfile/blocks.rs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -529,14 +529,18 @@ export XDG_CACHE_HOME=/var/lib/greetd/cache
529529
# and a 0700 greetd dir cannot be read back by the smoke's ssh user —
530530
# both measured, Oct 04). RUST_LOG=debug stays until the greeter
531531
# smoke passes; drop it to info after.
532-
export RUST_LOG=debug
532+
export RUST_LOG=info
533533
# bash dies on SIGPIPE when a builtin writes a broken pipe: if the
534534
# logger reader of this redirect ever stalls or dies, the wrapper's
535535
# next echo kills the whole session silently — greetd reads that as
536536
# "greeter exited without creating a session" and shoots a greeter
537537
# that is mid-initialization (measured, Oct 04). The greeter itself
538538
# is Rust and ignores SIGPIPE, which is why it outlived its wrapper.
539539
trap '' PIPE
540+
# A silent death needs a witness: TERM is what systemd sends before
541+
# the cgroup goes, and if the wrapper is killed with the scope its
542+
# last marker never lands. Log it and go.
543+
trap 'echo >&2 "wrapper: SIGTERM at $((SECONDS - started))s"; exit 143' TERM
540544
exec 2> >(exec logger -t kuma-greeter)
541545
542546
# The wrapper knows why the session ends: compositor death, socket

‎src/containerfile/goldens/everything-on.manifest‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ kuma-fstab-sync 8d91559a751bb9871641d2295536dac4fca72da4ca6b66787f5765cc0f191021
4949
kuma-fstab-sync.service 9d865e627355d6a5abb8ca39d00e62c41ef3effffe1eb235caeb16691622925b
5050
kuma-greeter 001555c61143bba5fc71ab136eadf5444ac9e0189bf20cc7fca15a6eb0224e25
5151
kuma-greeter-check ba4ff3ab5f84f6e5623a73a5a1c6f1384522652536f797bcad7048b3514fba5a
52-
kuma-greeter-session 7b2dc7faf6752a4445c50398fa57d0ab4da855ba4c1a4eca65eefadddb70e627
52+
kuma-greeter-session f037d181dbb7d473f6dc9d9ee78821dd2de636a85cee0834da754bd1bddbb1c5
5353
kuma-greeter-tmpfiles.conf ac57bb5086c057d537a424cd2d36924a739ef48e8b74b00baeb14bc4f8c4477e
5454
kuma-home-subvol b90db7dc4eabc2b9a95bdf90e7c93a432f6b818cb36c8135cde11138b39e2766
5555
kuma-home-subvol.service af9133fe3d2a930c432880a9937c71e8842e6d0fefbd75da065dc745240e4573

‎src/containerfile/goldens/niri.manifest‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ kuma-fstab-sync 8d91559a751bb9871641d2295536dac4fca72da4ca6b66787f5765cc0f191021
3737
kuma-fstab-sync.service 9d865e627355d6a5abb8ca39d00e62c41ef3effffe1eb235caeb16691622925b
3838
kuma-greeter 001555c61143bba5fc71ab136eadf5444ac9e0189bf20cc7fca15a6eb0224e25
3939
kuma-greeter-check ba4ff3ab5f84f6e5623a73a5a1c6f1384522652536f797bcad7048b3514fba5a
40-
kuma-greeter-session 7b2dc7faf6752a4445c50398fa57d0ab4da855ba4c1a4eca65eefadddb70e627
40+
kuma-greeter-session f037d181dbb7d473f6dc9d9ee78821dd2de636a85cee0834da754bd1bddbb1c5
4141
kuma-greeter-tmpfiles.conf ac57bb5086c057d537a424cd2d36924a739ef48e8b74b00baeb14bc4f8c4477e
4242
kuma-home-subvol b90db7dc4eabc2b9a95bdf90e7c93a432f6b818cb36c8135cde11138b39e2766
4343
kuma-home-subvol.service af9133fe3d2a930c432880a9937c71e8842e6d0fefbd75da065dc745240e4573

‎src/containerfile/goldens/nostr.manifest‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ kuma-fstab-sync 8d91559a751bb9871641d2295536dac4fca72da4ca6b66787f5765cc0f191021
3737
kuma-fstab-sync.service 9d865e627355d6a5abb8ca39d00e62c41ef3effffe1eb235caeb16691622925b
3838
kuma-greeter 001555c61143bba5fc71ab136eadf5444ac9e0189bf20cc7fca15a6eb0224e25
3939
kuma-greeter-check ba4ff3ab5f84f6e5623a73a5a1c6f1384522652536f797bcad7048b3514fba5a
40-
kuma-greeter-session 7b2dc7faf6752a4445c50398fa57d0ab4da855ba4c1a4eca65eefadddb70e627
40+
kuma-greeter-session f037d181dbb7d473f6dc9d9ee78821dd2de636a85cee0834da754bd1bddbb1c5
4141
kuma-greeter-tmpfiles.conf ac57bb5086c057d537a424cd2d36924a739ef48e8b74b00baeb14bc4f8c4477e
4242
kuma-home-subvol b90db7dc4eabc2b9a95bdf90e7c93a432f6b818cb36c8135cde11138b39e2766
4343
kuma-home-subvol.service af9133fe3d2a930c432880a9937c71e8842e6d0fefbd75da065dc745240e4573

0 commit comments

Comments
 (0)