|
| 1 | +import XCTest |
| 2 | + |
| 3 | +@testable import CloudMachineCore |
| 4 | + |
| 5 | +/// When a series of refused attaches ends in a restart of the mount - and |
| 6 | +/// when it must not. |
| 7 | +final class AttachStallTests: XCTestCase { |
| 8 | + |
| 9 | + private let boot: TimeInterval = 1_792_999_292 |
| 10 | + private let start: TimeInterval = 1_793_000_000 |
| 11 | + /// The attach agent's tick. |
| 12 | + private let tick: TimeInterval = 15 * 60 |
| 13 | + |
| 14 | + private let refused = CMActionResult( |
| 15 | + succeeded: false, |
| 16 | + message: |
| 17 | + "Could not attach the image: hdiutil: attach failed - Resource temporarily unavailable", |
| 18 | + attachTemporarilyUnavailable: true) |
| 19 | + private let attached = CMActionResult(succeeded: true, message: "Attached") |
| 20 | + |
| 21 | + private final class Restarts: @unchecked Sendable { |
| 22 | + var count = 0 |
| 23 | + } |
| 24 | + |
| 25 | + /// Runs the attach outcomes one tick apart, carrying the record along. |
| 26 | + private func ticks( |
| 27 | + _ outcomes: [CMActionResult], from record: AttachStall.Record? = nil, |
| 28 | + restartResult: CMActionResult = CMActionResult(succeeded: true, message: "restarted") |
| 29 | + ) async -> (record: AttachStall.Record?, restarts: Int) { |
| 30 | + let restarts = Restarts() |
| 31 | + var record = record |
| 32 | + for (i, outcome) in outcomes.enumerated() { |
| 33 | + record = await AttachStall.handle( |
| 34 | + outcome, record: record, bootTime: boot, now: start + Double(i) * tick, |
| 35 | + restart: { |
| 36 | + restarts.count += 1 |
| 37 | + return restartResult |
| 38 | + } |
| 39 | + ).record |
| 40 | + } |
| 41 | + return (record, restarts.count) |
| 42 | + } |
| 43 | + |
| 44 | + func testRecognizesTheHdiutilRefusal() { |
| 45 | + XCTAssertTrue( |
| 46 | + BackupImageService.isTemporarilyUnavailable( |
| 47 | + stderr: |
| 48 | + "hdiutil: WARNING: 'hdiutil attach -nobrowse -mountpoint ...' is deprecated.\nhdiutil: attach failed - Resource temporarily unavailable\n" |
| 49 | + )) |
| 50 | + XCTAssertFalse( |
| 51 | + BackupImageService.isTemporarilyUnavailable( |
| 52 | + stderr: "hdiutil: attach failed - RPC version wrong")) |
| 53 | + } |
| 54 | + |
| 55 | + /// KNOWN BAD SAMPLE, 25-26 Oct 2026: every attach refused for 26 h, and |
| 56 | + /// nothing but a restart of the Mac helped. Three ticks (30 min) are enough. |
| 57 | + func testASeriesOfRefusalsRestartsTheMount() async { |
| 58 | + let result = await ticks([refused, refused]) |
| 59 | + XCTAssertEqual(result.restarts, 0) |
| 60 | + let third = await ticks([refused, refused, refused]) |
| 61 | + XCTAssertEqual(third.restarts, 1) |
| 62 | + } |
| 63 | + |
| 64 | + /// Refusals just after a start-up are normal (26.10 08:27) - three of them |
| 65 | + /// in quick succession are not yet a series. |
| 66 | + func testThreeQuickRefusalsAreNotASeries() async { |
| 67 | + let quick = AttachStall.addingFailure( |
| 68 | + to: AttachStall.addingFailure( |
| 69 | + to: AttachStall.addingFailure(to: nil, bootTime: boot, now: start), bootTime: boot, |
| 70 | + now: start + 60), bootTime: boot, now: start + 120) |
| 71 | + XCTAssertEqual(quick.failures, 3) |
| 72 | + XCTAssertFalse(AttachStall.shouldRestart(quick, bootTime: boot, now: start + 120)) |
| 73 | + } |
| 74 | + |
| 75 | + /// A success between the refusals ends the series. |
| 76 | + func testASuccessEndsTheSeries() async { |
| 77 | + let result = await ticks([refused, refused, attached, refused, refused]) |
| 78 | + XCTAssertEqual(result.restarts, 0) |
| 79 | + } |
| 80 | + |
| 81 | + /// Other failures (the buffer not ready, an orphaned mount point) are not |
| 82 | + /// what a fresh rclone fixes. |
| 83 | + func testOtherFailuresEndTheSeries() async { |
| 84 | + let other = CMActionResult(succeeded: false, message: "Drive is not mounted.") |
| 85 | + let result = await ticks([refused, refused, other, refused, refused]) |
| 86 | + XCTAssertEqual(result.restarts, 0) |
| 87 | + } |
| 88 | + |
| 89 | + /// "Busy with another operation" says nothing about the mount: it neither |
| 90 | + /// ends the series nor adds to it. |
| 91 | + func testBusyNeitherEndsNorExtendsTheSeries() async { |
| 92 | + let busy = CMActionResult(succeeded: false, message: "busy", didNotRun: true) |
| 93 | + let result = await ticks([refused, busy, refused, busy, refused]) |
| 94 | + XCTAssertEqual(result.restarts, 1) |
| 95 | + XCTAssertEqual(result.record?.failures, 0) |
| 96 | + } |
| 97 | + |
| 98 | + /// If a fresh rclone did not help, another one within the cooldown will |
| 99 | + /// not either - and every restart cuts the upload short. |
| 100 | + func testAtMostOneRestartPerCooldown() async { |
| 101 | + let hours = Int(AttachStall.cooldown / tick) |
| 102 | + let result = await ticks(Array(repeating: refused, count: hours - 1)) |
| 103 | + XCTAssertEqual(result.restarts, 1) |
| 104 | + let longer = await ticks(Array(repeating: refused, count: hours + 3)) |
| 105 | + XCTAssertEqual(longer.restarts, 2) |
| 106 | + } |
| 107 | + |
| 108 | + /// The cooldown holds across a short success, or restarts would follow |
| 109 | + /// every flap. |
| 110 | + func testCooldownSurvivesASuccess() async { |
| 111 | + let result = await ticks([refused, refused, refused, attached, refused, refused, refused]) |
| 112 | + XCTAssertEqual(result.restarts, 1) |
| 113 | + } |
| 114 | + |
| 115 | + /// A refusal of the restart (Time Machine backing up, tmutil silent) |
| 116 | + /// touched nothing - the next refused attach asks again, no cooldown. |
| 117 | + func testRefusedRestartDoesNotStartTheCooldown() async { |
| 118 | + let refusedRestart = CMActionResult(succeeded: false, message: "backing up", didNotRun: true) |
| 119 | + let result = await ticks( |
| 120 | + [refused, refused, refused, refused], restartResult: refusedRestart) |
| 121 | + XCTAssertEqual(result.restarts, 2) |
| 122 | + XCTAssertEqual(result.record?.restartedAt, 0) |
| 123 | + } |
| 124 | + |
| 125 | + /// A restart that ran and failed (the queue did not drain) still starts the |
| 126 | + /// cooldown: running the same drain every tick would not change its answer. |
| 127 | + func testFailedRestartStartsTheCooldown() async { |
| 128 | + let failed = CMActionResult(succeeded: false, message: "not drained") |
| 129 | + let result = await ticks( |
| 130 | + [refused, refused, refused, refused, refused, refused, refused], |
| 131 | + restartResult: failed) |
| 132 | + XCTAssertEqual(result.restarts, 1) |
| 133 | + } |
| 134 | + |
| 135 | + /// A record from before a restart of the Mac describes a mount that is gone. |
| 136 | + func testARecordFromAnotherBootIsIgnored() async { |
| 137 | + let old = AttachStall.Record( |
| 138 | + bootTime: boot - 86_400, firstFailureAt: start - 7200, failures: 9, restartedAt: start - 600) |
| 139 | + let result = await ticks([refused], from: old) |
| 140 | + XCTAssertEqual(result.restarts, 0) |
| 141 | + XCTAssertEqual(result.record?.failures, 1) |
| 142 | + XCTAssertEqual(result.record?.restartedAt, 0) |
| 143 | + } |
| 144 | + |
| 145 | + func testRecordTextRoundTrips() { |
| 146 | + let record = AttachStall.Record( |
| 147 | + bootTime: boot, firstFailureAt: start, failures: 4, restartedAt: start + 900) |
| 148 | + XCTAssertEqual(AttachStall.Record(text: record.text), record) |
| 149 | + XCTAssertNil(AttachStall.Record(text: "garbage")) |
| 150 | + XCTAssertNil(AttachStall.Record(text: "1 2 3")) |
| 151 | + } |
| 152 | +} |
0 commit comments