Skip to content

Commit 0a22fb1

Browse files
committed
fix(windows): absorb transient file locks when retiring a running image
Release run 30385988475 lost three Windows smoke jobs to one phenomenon on two different surfaces. A Windows file that was just written or just executed can refuse deletion and rename for a moment while a scanner reads it or while the OS finishes reaping a child. Both conditions clear on their own within moments. It reads as nondeterminism because it is timing: windows-2025 passed both amd64 variants while windows-latest -- the SAME image, windows-2025-vs2026 -- failed both, and windows-11-arm failed too. PRODUCT: uninstall abandoned a live installation. With one binary per platform, uninstall must remove the very image it is executing, and on Windows a running image can only be renamed, never overwritten. That rename got a single attempt, so a transient sharing violation surfaced to the user as error: failed to remove ...; completed configuration/index cleanup may remain with the install left half-torn-down. install.ps1 already retries this exact operation ten times for this exact reason; the C path now does the same, and only for sharing/access/lock violations, so a file that is genuinely held still fails closed instead of spinning. Proven with a failure-injection seam rather than by racing a real scanner: 3 injected failures must still succeed, 64 must still fail AND leave the target in place. Deterministic, so it does not need CI to learn whether it works. HARNESS: the smoke killed itself silently. Phases that install a binary then `rm -rf` the directory hit the same lock. Under `set -euo pipefail` that returns non-zero and ends the run WITHOUT printing anything -- the logs jump straight from "OK 13h" to job cleanup with no FAIL line, which is what made this expensive to read. None of those 16 removals is an assertion; they are fixture cleanup, and _smoke.yml already states the principle ("a temp dir on an ephemeral runner must never fail the job") that smoke-test.sh did not honour. They now go through smoke_rmtree, which retries so the disk is actually reclaimed -- these fixtures hold ~300 MB binaries and runners are disk-tight -- then warns and continues. Verified: Windows VM full smoke ALL PASSED (previously died after 13h), macOS full smoke ALL PASSED, activation_transaction + cli 259 passed on Windows, all 10 shell contracts pass, lint-ci clean. Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
1 parent 0802689 commit 0a22fb1

4 files changed

Lines changed: 148 additions & 20 deletions

File tree

‎scripts/smoke-test.sh‎

Lines changed: 42 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,31 @@ smoke_file_sha256() {
6868
fi
6969
}
7070

71+
# Fixture cleanup, never an assertion. On Windows a directory holding an
72+
# executable that was just written or just run can refuse deletion for a moment
73+
# while a scanner or an unreaped child still holds it. Under `set -e` a plain
74+
# `rm -rf` then kills the run WITHOUT printing anything — three release smoke
75+
# jobs died exactly that way, silently, immediately after "OK 13h".
76+
#
77+
# Retry so the disk actually gets reclaimed (these fixtures hold ~300 MB
78+
# binaries and runners are disk-tight), then warn and continue: an ephemeral
79+
# temp dir must never decide the verdict.
80+
smoke_rmtree() {
81+
local target
82+
for target in "$@"; do
83+
[ -n "$target" ] || continue
84+
local attempt
85+
for attempt in 1 2 3 4 5 6 7 8 9 10; do
86+
rm -rf "$target" 2>/dev/null && break
87+
sleep 0.5
88+
done
89+
if [ -e "$target" ]; then
90+
echo "warn: could not remove smoke fixture $target (leaving it to the runner)"
91+
fi
92+
done
93+
return 0
94+
}
95+
7196
# Every platform ships ONE binary, Windows included: a fixture copy is complete
7297
# with nothing beside it.
7398
copy_smoke_binary() {
@@ -123,7 +148,7 @@ DRYRUN_HOME=""
123148
if command -v cygpath &>/dev/null; then
124149
TMPDIR=$(cygpath -m "$TMPDIR")
125150
fi
126-
trap 'rm -rf "$TMPDIR" "${DRYRUN_HOME:-}"' EXIT
151+
trap 'smoke_rmtree "$TMPDIR" "${DRYRUN_HOME:-}"' EXIT
127152

128153
CLI_STDERR=$(smoke_mktemp_file)
129154
cli() { "$BINARY" cli "$@" 2>"$CLI_STDERR"; }
@@ -970,7 +995,7 @@ chmod 755 "$INSTALL_DIR/codebase-memory-mcp"
970995
INSTALLED_VER=$("$INSTALL_DIR/codebase-memory-mcp" --version 2>&1)
971996
if ! echo "$INSTALLED_VER" | grep -qE 'v?[0-9]+\.[0-9]+|dev'; then
972997
echo "FAIL: installed binary --version failed: $INSTALLED_VER"
973-
rm -rf "$REPLACE_DIR"
998+
smoke_rmtree "$REPLACE_DIR"
974999
exit 1
9751000
fi
9761001

@@ -986,7 +1011,7 @@ chmod 755 "$INSTALL_DIR/codebase-memory-mcp"
9861011
REPLACED_VER=$("$INSTALL_DIR/codebase-memory-mcp" --version 2>&1)
9871012
if ! echo "$REPLACED_VER" | grep -qE 'v?[0-9]+\.[0-9]+|dev'; then
9881013
echo "FAIL: replaced binary --version failed: $REPLACED_VER"
989-
rm -rf "$REPLACE_DIR"
1014+
smoke_rmtree "$REPLACE_DIR"
9901015
exit 1
9911016
fi
9921017
echo "OK: binary replacement succeeded (version: $REPLACED_VER)"
@@ -1000,12 +1025,12 @@ chmod 755 "$INSTALL_DIR/codebase-memory-mcp"
10001025
READONLY_VER=$("$INSTALL_DIR/codebase-memory-mcp" --version 2>&1)
10011026
if ! echo "$READONLY_VER" | grep -qE 'v?[0-9]+\.[0-9]+|dev'; then
10021027
echo "FAIL: read-only replacement --version failed: $READONLY_VER"
1003-
rm -rf "$REPLACE_DIR"
1028+
smoke_rmtree "$REPLACE_DIR"
10041029
exit 1
10051030
fi
10061031
echo "OK: read-only binary replacement succeeded"
10071032

1008-
rm -rf "$REPLACE_DIR"
1033+
smoke_rmtree "$REPLACE_DIR"
10091034

10101035
echo ""
10111036
echo "=== Phase 7: MCP advanced tool calls ==="
@@ -2671,7 +2696,7 @@ if ! echo "$INSTALL_OUT" | grep -qi 'detected agents'; then
26712696
fi
26722697
echo "OK 9b-1: install with minimal agents exits cleanly"
26732698
retire_account_daemon "9b-1-cleanup"
2674-
rm -rf "$EMPTY_HOME"
2699+
smoke_rmtree "$EMPTY_HOME"
26752700

26762701
# 9b-2: Install twice (idempotent)
26772702
IDEM_HOME=$(smoke_mktemp_dir)
@@ -2695,7 +2720,7 @@ if [ "$COUNT" != "1" ]; then
26952720
fi
26962721
echo "OK 9b-2: double install is idempotent"
26972722
retire_account_daemon "9b-2-cleanup"
2698-
rm -rf "$IDEM_HOME"
2723+
smoke_rmtree "$IDEM_HOME"
26992724

27002725
# 9b-3: Uninstall without prior install
27012726
CLEAN_HOME=$(smoke_mktemp_dir)
@@ -2708,7 +2733,7 @@ if [ "$UNINSTALL_RC" -ge 128 ]; then
27082733
fi
27092734
echo "OK 9b-3: uninstall without install doesn't crash"
27102735
retire_account_daemon "9b-3-cleanup"
2711-
rm -rf "$CLEAN_HOME"
2736+
smoke_rmtree "$CLEAN_HOME"
27122737

27132738
# 9b-4: Install over corrupt JSON
27142739
CORRUPT_HOME=$(smoke_mktemp_dir)
@@ -2719,7 +2744,7 @@ run_no_crash 9b-4 env HOME="$CORRUPT_HOME" "$BINARY" install -y
27192744
# Should either fix it or handle gracefully — not crash
27202745
echo "OK 9b-4: install over corrupt JSON doesn't crash"
27212746
retire_account_daemon "9b-4-cleanup"
2722-
rm -rf "$CORRUPT_HOME"
2747+
smoke_rmtree "$CORRUPT_HOME"
27232748

27242749
# 9b-8: Double uninstall
27252750
DBL_HOME=$(smoke_mktemp_dir)
@@ -2734,7 +2759,7 @@ run_no_crash 9b-8-first env HOME="$DBL_HOME" "$DBL_UNINSTALLER" uninstall -y -n
27342759
run_no_crash 9b-8-second env HOME="$DBL_HOME" "$BINARY" uninstall -y -n
27352760
echo "OK 9b-8: double uninstall doesn't crash"
27362761
retire_account_daemon "9b-8-cleanup"
2737-
rm -rf "$DBL_HOME"
2762+
smoke_rmtree "$DBL_HOME"
27382763

27392764
# 9b-9: Non-interactive update without --standard/--ui should fail cleanly (not hang)
27402765
if [ "$(uname -s)" != "MINGW64_NT" ] 2>/dev/null; then
@@ -2748,7 +2773,7 @@ if [ "$(uname -s)" != "MINGW64_NT" ] 2>/dev/null; then
27482773
fi
27492774

27502775
retire_account_daemon "9-cleanup"
2751-
rm -rf "$FAKE_HOME" "$EMPTY_HOME"
2776+
smoke_rmtree "$FAKE_HOME" "$EMPTY_HOME"
27522777

27532778
if [ "$SMOKE_MODE" = "--agent-config-only" ]; then
27542779
echo ""
@@ -2848,7 +2873,7 @@ else
28482873
fi
28492874
fi
28502875

2851-
rm -rf "$SECURITY_DIR"
2876+
smoke_rmtree "$SECURITY_DIR"
28522877

28532878
echo ""
28542879
echo "=== Phase 11: process kill E2E ==="
@@ -3063,7 +3088,7 @@ sys.exit(0)
30633088
exit 1
30643089
fi
30653090

3066-
rm -rf "$UPDATE_HOME"
3091+
smoke_rmtree "$UPDATE_HOME"
30673092

30683093
else
30693094
# Local mode: basic binary replacement test (no download)
@@ -3083,7 +3108,7 @@ else
30833108
exit 1
30843109
fi
30853110
echo "OK 14: binary replacement + verify (local mode)"
3086-
rm -rf "$UPDATE_DIR"
3111+
smoke_rmtree "$UPDATE_DIR"
30873112
fi
30883113

30893114
# ── Phase 12 + 13: Download E2E + install script E2E (CI only) ──
@@ -3234,7 +3259,7 @@ else
32343259
echo "OK 12f: binary runs without signing ($DL_OS)"
32353260
fi
32363261

3237-
rm -rf "$DL_DIR"
3262+
smoke_rmtree "$DL_DIR"
32383263

32393264
echo ""
32403265
echo "=== Phase 13: install script E2E ==="
@@ -3301,7 +3326,7 @@ if [ "$DL_OS" != "windows" ] && [ -f "$REPO_ROOT/install.sh" ]; then
33013326
echo "OK 13f: PATH setup (rc file may not have been modified if already present)"
33023327
fi
33033328

3304-
rm -rf "$INSTALL_TEST_HOME" "$INSTALL_TEST_DIR"
3329+
smoke_rmtree "$INSTALL_TEST_HOME" "$INSTALL_TEST_DIR"
33053330

33063331
elif [ -f "$REPO_ROOT/install.ps1" ] && command -v powershell.exe &>/dev/null; then
33073332
echo "--- Phase 13: install.ps1 E2E (Windows) ---"
@@ -3356,7 +3381,7 @@ elif [ -f "$REPO_ROOT/install.ps1" ] && command -v powershell.exe &>/dev/null; t
33563381
exit 1
33573382
fi
33583383

3359-
rm -rf "$PS1_TEST_HOME" "$PS1_TEST_DIR"
3384+
smoke_rmtree "$PS1_TEST_HOME" "$PS1_TEST_DIR"
33603385
else
33613386
echo "SKIP Phase 13: no install script available for this platform"
33623387
fi

‎src/cli/activation_transaction.c‎

Lines changed: 51 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1118,6 +1118,40 @@ static bool activation_sync_directory(const cbm_activation_transaction_t *transa
11181118
#endif
11191119
}
11201120

1121+
#ifdef _WIN32
1122+
/* Renaming is how this transaction both publishes and retires -- and on Windows
1123+
* it is the ONLY mutation permitted on a running image, which is what lets a
1124+
* single binary replace or remove itself at all. It can still lose to a handle
1125+
* that is about to go away: an antivirus scan of a just-written executable, or
1126+
* a child process the OS has not finished reaping. Both surface as a sharing or
1127+
* access violation and both clear within moments.
1128+
*
1129+
* install.ps1 already retries this exact operation for this exact reason. The C
1130+
* path giving up on the first attempt is what let `uninstall` intermittently
1131+
* abandon a live installation with "cleanup may remain" -- observed on one
1132+
* runner while an identical job on the same image passed.
1133+
*
1134+
* Bounded on purpose: a file that is genuinely held must still fail, and fail
1135+
* closed, rather than spin. */
1136+
#define ACTIVATION_RENAME_ATTEMPTS 10U
1137+
#define ACTIVATION_RENAME_RETRY_MS 500U
1138+
1139+
static unsigned int activation_rename_failures_for_test;
1140+
1141+
static bool activation_rename_error_is_transient(DWORD error) {
1142+
return error == ERROR_SHARING_VIOLATION || error == ERROR_ACCESS_DENIED ||
1143+
error == ERROR_LOCK_VIOLATION;
1144+
}
1145+
#endif
1146+
1147+
void cbm_activation_transaction_rename_failures_set_for_test(unsigned int count) {
1148+
#ifdef _WIN32
1149+
activation_rename_failures_for_test = count;
1150+
#else
1151+
(void)count;
1152+
#endif
1153+
}
1154+
11211155
static bool activation_rename(const cbm_activation_transaction_t *transaction, const char *source,
11221156
const char *source_name, const char *destination,
11231157
const char *destination_name, bool replace_destination) {
@@ -1128,9 +1162,23 @@ static bool activation_rename(const cbm_activation_transaction_t *transaction, c
11281162
wchar_t *wide_source = activation_utf8_to_wide(source);
11291163
wchar_t *wide_destination = activation_utf8_to_wide(destination);
11301164
DWORD flags = MOVEFILE_WRITE_THROUGH | (replace_destination ? MOVEFILE_REPLACE_EXISTING : 0);
1131-
bool ok = wide_source && wide_destination &&
1132-
MoveFileExW(wide_source, wide_destination, flags) != 0 &&
1133-
activation_directory_still_valid(transaction);
1165+
bool moved = false;
1166+
for (unsigned int attempt = 0;
1167+
wide_source && wide_destination && !moved && attempt < ACTIVATION_RENAME_ATTEMPTS;
1168+
attempt++) {
1169+
if (activation_rename_failures_for_test > 0) {
1170+
activation_rename_failures_for_test--;
1171+
} else if (MoveFileExW(wide_source, wide_destination, flags) != 0) {
1172+
moved = true;
1173+
break;
1174+
} else if (!activation_rename_error_is_transient(GetLastError())) {
1175+
break;
1176+
}
1177+
if (attempt + 1U < ACTIVATION_RENAME_ATTEMPTS) {
1178+
Sleep(ACTIVATION_RENAME_RETRY_MS);
1179+
}
1180+
}
1181+
bool ok = moved && activation_directory_still_valid(transaction);
11341182
free(wide_source);
11351183
free(wide_destination);
11361184
return ok;

‎src/cli/activation_transaction.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,11 @@ typedef void (*cbm_activation_transaction_before_absent_publish_for_test_fn)(
3939
void cbm_activation_transaction_set_before_absent_publish_for_test(
4040
cbm_activation_transaction_before_absent_publish_for_test_fn hook, void *context);
4141

42+
/* Test-only seam: make the next `count` Windows rename attempts fail as though
43+
* another handle held the file, so the transient-lock retry can be proven
44+
* without racing a real scanner. Inert on POSIX and when count is 0. */
45+
void cbm_activation_transaction_rename_failures_set_for_test(unsigned int count);
46+
4247
/* Stage a candidate beside target_path (therefore on the same filesystem).
4348
* The staged file is private to the current account and executable. */
4449
cbm_activation_transaction_status_t cbm_activation_transaction_stage_bytes(

‎tests/test_activation_transaction.c‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -543,6 +543,53 @@ TEST(activation_transaction_removal_can_rollback_or_finalize) {
543543
PASS();
544544
}
545545

546+
#ifdef _WIN32
547+
/* A running Windows image can be renamed but never overwritten, so retiring it
548+
* is a rename -- and that rename can lose to a handle that is about to go away
549+
* (an antivirus scan of a fresh executable, a child the OS has not reaped).
550+
* Both clear on their own, so the retry must absorb them rather than abandon a
551+
* live installation. The failure seam substitutes for that timing instead of
552+
* waiting on a real scanner, which no test could schedule. */
553+
TEST(activation_transaction_removal_survives_transient_rename_locks) {
554+
char directory[ACTIVATION_TEST_PATH_CAP];
555+
char target[ACTIVATION_TEST_PATH_CAP];
556+
ASSERT_TRUE(activation_test_fixture(directory));
557+
ASSERT_TRUE(activation_test_path(target, directory, "cbm"));
558+
ASSERT_TRUE(activation_test_write(target, "old"));
559+
activation_test_validation_t absent = {
560+
.expect_absent = true,
561+
.expected_contents = NULL,
562+
};
563+
564+
cbm_activation_transaction_t *transaction = NULL;
565+
ASSERT_EQ(cbm_activation_transaction_stage_removal(target, &transaction),
566+
CBM_ACTIVATION_TRANSACTION_OK);
567+
/* Strictly inside the budget: the retire must still succeed. */
568+
cbm_activation_transaction_rename_failures_set_for_test(3U);
569+
ASSERT_EQ(cbm_activation_transaction_commit(transaction, activation_test_validate, &absent),
570+
CBM_ACTIVATION_TRANSACTION_OK);
571+
cbm_activation_transaction_rename_failures_set_for_test(0U);
572+
ASSERT_FALSE(activation_test_exists(target));
573+
ASSERT_EQ(cbm_activation_transaction_finalize(transaction), CBM_ACTIVATION_TRANSACTION_OK);
574+
ASSERT_EQ(cbm_activation_transaction_close(&transaction), CBM_ACTIVATION_TRANSACTION_OK);
575+
576+
/* Past the budget it must still FAIL, and leave the target in place: a
577+
* genuinely held file is not something to spin on or to report as removed. */
578+
ASSERT_TRUE(activation_test_write(target, "old"));
579+
ASSERT_EQ(cbm_activation_transaction_stage_removal(target, &transaction),
580+
CBM_ACTIVATION_TRANSACTION_OK);
581+
cbm_activation_transaction_rename_failures_set_for_test(64U);
582+
ASSERT_EQ(cbm_activation_transaction_commit(transaction, activation_test_validate, &absent),
583+
CBM_ACTIVATION_TRANSACTION_IO);
584+
cbm_activation_transaction_rename_failures_set_for_test(0U);
585+
ASSERT_TRUE(activation_test_exists(target));
586+
(void)cbm_activation_transaction_rollback(transaction);
587+
ASSERT_EQ(cbm_activation_transaction_close(&transaction), CBM_ACTIVATION_TRANSACTION_OK);
588+
ASSERT_EQ(th_rmtree(directory), 0);
589+
PASS();
590+
}
591+
#endif
592+
546593
TEST(activation_transaction_rejects_cross_account_writable_target_directory) {
547594
#ifndef _WIN32
548595
char directory[ACTIVATION_TEST_PATH_CAP];
@@ -972,6 +1019,9 @@ SUITE(activation_transaction) {
9721019
RUN_TEST(activation_transaction_stage_file_installs_new_target);
9731020
RUN_TEST(activation_transaction_stage_file_survives_long_target_path);
9741021
RUN_TEST(activation_transaction_removal_can_rollback_or_finalize);
1022+
#ifdef _WIN32
1023+
RUN_TEST(activation_transaction_removal_survives_transient_rename_locks);
1024+
#endif
9751025
RUN_TEST(activation_transaction_rejects_cross_account_writable_target_directory);
9761026
RUN_TEST(activation_transaction_rejects_windows_callback_allow_directory_ace);
9771027
RUN_TEST(activation_transaction_rejects_symlink_candidate_target_and_parent);

0 commit comments

Comments
 (0)