Odmontowanie osieroconego montowania po padzie rclone

mac-app/Sources/CloudMachineAgent/DriveCommands.swift

@@ -15,6 +15,9 @@ struct MountDrive: AsyncParsableCommand {
       "Mounts Google Drive with a write buffer. Stays in the foreground (for launchd)."))
 
   func run() async throws {
+    // A mount left behind by an rclone that quit (27.10) would otherwise be
+    // "already mounted" forever - see `OrphanMount`.
+    await OrphanMount.clearBeforeMount()
     if DriveBufferService.isMounted {
       print(L10n.tr("Already mounted: %@", DriveBufferService.mountPoint.path))
       return
@@ -235,6 +238,10 @@ struct BufferGuard: AsyncParsableCommand {
       "Buffer guard: pause above \(highGB) GB backlog / resume below \(lowGB) GB / min. free on disk \(minFreeGB) GB"
     )
 
+    // Beside the loop, not in it: a probe of the loop hanging on a dead mount
+    // must not hold up the one check that clears it.
+    Task.detached { await OrphanMount.watch() }
+
     // Forever: the guard has to outlive every backup, not just the first one.
     while true {
       // The mount runs for days and rclone never trims its own log; the

mac-app/Sources/CloudMachineCore/OrphanMount.swift

@@ -0,0 +1,133 @@
+import Foundation
+
+/// The way out of a Drive mount whose rclone is gone.
+///
+/// 25 Oct 2026 00:02 and 27 Oct 2026 00:19, both during a backup: rclone
+/// logged "Mount failed" / "CRITICAL: Fatal error: failed to umount FUSE fs"
+/// and quit, but the FUSE-T NFS mount stayed in the mount table. From then on
+/// every `stat` of the mount (Time Machine, `tmutil`, Xcode, our own agents)
+/// hung, and nothing brought rclone back: `mount-drive` read the mount table,
+/// saw the mount and said "Already mounted". On 27.10 that lasted 8 h, until
+/// the owner restarted the Mac. On 25.10 `umount -f` of the mount point by
+/// hand was enough - the agent brought rclone up 30 s later.
+///
+/// So the remedy is that `umount -f`, done by the agents, when the mount is
+/// listed and no live rclone holds it. Everything here avoids the file
+/// system: the mount table is read with `MNT_NOWAIT`, the process table with
+/// `pgrep`/`ps`, so a dead mount cannot hang the check that looks for it.
+///
+/// Two callers:
+/// - `buffer-guard` runs `watch()` in its own task, beside its main loop, so
+///   a probe of that loop hanging on the dead mount cannot hold it up. It
+///   waits `grace` before acting - the normal restart of rclone by launchd
+///   also has a moment with the mount listed and no rclone yet.
+/// - `mount-drive`, at start-up. launchd starts it only once the previous
+///   rclone has exited, so a listed mount with no rclone is an orphan.
+public enum OrphanMount {
+
+  /// Whether an rclone holding our mount is alive.
+  public enum RcloneState: Equatable {
+    case running
+    /// No such process, or only ones on their way out (`ps` state `E`/`Z`).
+    case gone
+    /// The process table could not be read - never a reason to unmount.
+    case unknown
+  }
+
+  public static let grace: TimeInterval = 60
+  public static let checkInterval: TimeInterval = 30
+  static let unmountTimeout: TimeInterval = 60
+
+  /// Since when the mount has been seen orphaned; `nil` = it is not.
+  public static func orphanedSince(
+    _ previous: TimeInterval?, mounted: Bool?, rclone: RcloneState, now: TimeInterval
+  ) -> TimeInterval? {
+    // Only a sure "mounted" and a sure "gone" make an orphan; any doubt
+    // resets the clock rather than counting towards an unmount.
+    guard mounted == true, rclone == .gone else { return nil }
+    return previous ?? now
+  }
+
+  /// Whether the orphan has lasted long enough to unmount it.
+  public static func shouldUnmount(orphanedSince: TimeInterval?, now: TimeInterval) -> Bool {
+    guard let orphanedSince else { return false }
+    return now - orphanedSince >= grace
+  }
+
+  /// `ps -o stat=` lines -> whether any of them is a process still running.
+  /// `E` = trying to exit, `Z` = zombie: neither serves the mount any more.
+  static func anyAlive(psStates: String) -> Bool {
+    psStates.split(whereSeparator: \.isNewline).contains { line in
+      let state = line.trimmingCharacters(in: .whitespaces)
+      return !state.isEmpty && !state.contains("E") && !state.contains("Z")
+    }
+  }
+
+  /// Our rclone, by the path it runs from and the `mount` subcommand.
+  static var rclonePattern: String {
+    NSRegularExpression.escapedPattern(for: CMTooling.managedRclonePath.path) + " mount "
+  }
+
+  public static func rcloneState() async -> RcloneState {
+    guard
+      let found = try? await ProcessRunner.run(
+        "/usr/bin/pgrep", ["-f", rclonePattern], timeout: 15)
+    else { return .unknown }
+    switch found.exitCode {
+    case 1: return .gone
+    case 0: break
+    default: return .unknown
+    }
+    let pids = found.stdout.split(whereSeparator: \.isWhitespace).map(String.init)
+    guard !pids.isEmpty,
+      let states = try? await ProcessRunner.run(
+        "/bin/ps", ["-o", "stat=", "-p", pids.joined(separator: ",")], timeout: 15)
+    else { return .unknown }
+    // `ps` exits 1 when the processes have gone in the meantime.
+    if !states.succeeded, states.stdout.isEmpty { return .gone }
+    return anyAlive(psStates: states.stdout) ? .running : .gone
+  }
+
+  /// `umount -f` of the mount point; `true` once it has left the mount table.
+  @discardableResult
+  public static func forceUnmount() async -> Bool {
+    let path = DriveBufferService.mountPoint.path
+    CMLogger.log("Orphaned Drive mount (no rclone holds it) - umount -f \(path)")
+    let result = try? await ProcessRunner.run(
+      "/sbin/umount", ["-f", path], timeout: unmountTimeout)
+    let gone = DriveBufferService.mountedState() == false
+    CMLogger.log(
+      gone
+        ? "Orphaned Drive mount removed"
+        : "Orphaned Drive mount NOT removed (umount -f: \(result.map { "exit code \($0.exitCode) \($0.stderr)" } ?? "no answer"))"
+    )
+    return gone
+  }
+
+  /// `mount-drive`'s check: unmount at once, no grace (see the type comment).
+  public static func clearBeforeMount() async {
+    guard DriveBufferService.mountedState() == true, await rcloneState() == .gone else { return }
+    await forceUnmount()
+  }
+
+  /// `buffer-guard`'s watch, forever.
+  public static func watch() async {
+    var since: TimeInterval?
+    while true {
+      try? await Task.sleep(nanoseconds: UInt64(checkInterval * 1_000_000_000))
+      let mounted = DriveBufferService.mountedState()
+      let rclone = mounted == true ? await rcloneState() : .unknown
+      let now = Date().timeIntervalSince1970
+      since = orphanedSince(since, mounted: mounted, rclone: rclone, now: now)
+      guard shouldUnmount(orphanedSince: since, now: now) else { continue }
+      since = nil
+      guard await forceUnmount() else { continue }
+      // launchd starts `mount-drive` again by itself once rclone has exited;
+      // a kickstart without `-k` only covers a job that is not running.
+      _ = try? await ProcessRunner.run(
+        "/bin/launchctl",
+        ["kickstart", "gui/\(getuid())/\(AgentRepair.prefix)\(MountRestart.mountAgent)"],
+        timeout: 30)
+    }
+  }
+}

mac-app/Tests/CloudMachineAppTests/OrphanMountTests.swift

@@ -0,0 +1,59 @@
+import XCTest
+
+@testable import CloudMachineCore
+
+/// When a mount without its rclone gets unmounted - and when it must not.
+final class OrphanMountTests: XCTestCase {
+
+  private let start: TimeInterval = 1_793_000_000
+
+  func testOrphanNeedsSureMountAndSureGone() {
+    XCTAssertEqual(
+      OrphanMount.orphanedSince(nil, mounted: true, rclone: .gone, now: start), start)
+    XCTAssertNil(OrphanMount.orphanedSince(nil, mounted: true, rclone: .running, now: start))
+    XCTAssertNil(OrphanMount.orphanedSince(nil, mounted: true, rclone: .unknown, now: start))
+    XCTAssertNil(OrphanMount.orphanedSince(nil, mounted: false, rclone: .gone, now: start))
+    XCTAssertNil(OrphanMount.orphanedSince(nil, mounted: nil, rclone: .gone, now: start))
+  }
+
+  func testOrphanKeepsItsFirstSighting() {
+    XCTAssertEqual(
+      OrphanMount.orphanedSince(start, mounted: true, rclone: .gone, now: start + 90), start)
+  }
+
+  /// Doubt in between starts the wait over instead of counting towards it.
+  func testDoubtResetsTheClock() {
+    var since = OrphanMount.orphanedSince(nil, mounted: true, rclone: .gone, now: start)
+    since = OrphanMount.orphanedSince(since, mounted: true, rclone: .unknown, now: start + 30)
+    since = OrphanMount.orphanedSince(since, mounted: true, rclone: .gone, now: start + 60)
+    XCTAssertEqual(since, start + 60)
+    XCTAssertFalse(OrphanMount.shouldUnmount(orphanedSince: since, now: start + 90))
+  }
+
+  /// launchd restarting rclone also leaves the mount listed for a moment.
+  func testWaitsOutTheGrace() {
+    XCTAssertFalse(OrphanMount.shouldUnmount(orphanedSince: nil, now: start))
+    XCTAssertFalse(
+      OrphanMount.shouldUnmount(orphanedSince: start, now: start + OrphanMount.grace - 1))
+    XCTAssertTrue(OrphanMount.shouldUnmount(orphanedSince: start, now: start + OrphanMount.grace))
+  }
+
+  func testExitingAndZombieProcessesDoNotCount() {
+    XCTAssertTrue(OrphanMount.anyAlive(psStates: "Ss\n"))
+    XCTAssertTrue(OrphanMount.anyAlive(psStates: "UE\nS+\n"))
+    XCTAssertFalse(OrphanMount.anyAlive(psStates: "UE\n"))
+    XCTAssertFalse(OrphanMount.anyAlive(psStates: "Z\n"))
+    XCTAssertFalse(OrphanMount.anyAlive(psStates: ""))
+  }
+
+  /// The pattern matches the command line `DriveBufferService` builds.
+  func testPatternMatchesTheMountCommandLine() throws {
+    let line =
+      ([CMTooling.managedRclonePath.path] + DriveBufferService.mountArguments()).joined(
+        separator: " ")
+    let regex = try NSRegularExpression(pattern: OrphanMount.rclonePattern)
+    XCTAssertNotNil(regex.firstMatch(in: line, range: NSRange(line.startIndex..., in: line)))
+    let other = "/opt/homebrew/bin/rclone mount gdrive: /tmp/x"
+    XCTAssertNil(regex.firstMatch(in: other, range: NSRange(other.startIndex..., in: other)))
+  }
+}