fix(core): self-heal a stale drive_file_id when an update hits a definitive 404 (#168)

An UPDATE against a drive_file_id whose Drive object is permanently gone
returned a 404 that classified as DriveErrorClassification::Other -> the
generic drive.unreachable: the op failed, the Activity view said "Drive
unavailable", the dead id was KEPT, and every later cycle re-planned the
same doomed update forever.

Verified against a real incident today: 4 files looping
drive.unreachable every cycle while neighboring uploads succeeded; their
recorded ids return hard 404 from the Drive API.

Changes:
- driven-drive: export error_is_not_found (walks the error chain for the
drive HTTP 404 marker) instead of duplicating string matching
downstream.
- executor: an update op whose failure is a definitive not-found now
clears the stale drive_file_id (mirroring reconcile's R3-P1-2 path),
drops the op, and surfaces a WARN drive.remote_file_missing outcome
instead of an error - the next scan re-plans a fresh CREATE and the file
self-heals. Create-path 404s (dest-folder) and the trash 404-is-ok rule
are untouched; transient 5xx behavior unchanged (regression-tested).
- types: new stable ErrorCode DriveRemoteFileMissing <->
drive.remote_file_missing (+ round-trip test).
- orchestrator: records the warn-level activity row for the new outcome;
does not fail the cycle (handled skip).
- ui: en-US.json errors entry ("Remote copy missing") + activity label
test.
- fake store: update-not-found fault injection + executor tests proving
id cleared, op dropped, warn row written, next cycle re-creates.

Gates: cargo fmt/clippy -D warnings clean; cargo test -p driven-core -p
driven-drive all green; vitest 489 passed; prettier clean.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

https://claude.ai/code/session_01JLB3E2Jm7knNJd37fVpH8X

Author: pmaxhogan

crates/driven-core/src/executor.rs

@@ -4981,6 +4981,47 @@ mod tests {
         );
     }
 
+    /// A self-healed stale `drive_file_id` must read truthfully in the activity
+    /// log: a WARN row keyed `drive.remote_file_missing` naming the file - NOT
+    /// an Error row, and NOT the `drive.unreachable` the 404 used to be reported
+    /// as. It must also NOT count as a failed op, since holding the source's
+    /// scan timestamps back would punish every other file for a condition the
+    /// executor already fixed.
+    #[test]
+    fn a_healed_missing_remote_copy_is_a_warn_row_and_not_a_failure() {
+        let src_id = crate::types::SourceId::new_v4();
+        let outcome = OpOutcome::Skipped {
+            relative_path: RelativePath::try_from("docs/report.pdf".to_string()).unwrap(),
+            reason: crate::executor::SkipReason::RemoteFileMissing,
+        };
+
+        let row = SyncOrchestrator::outcome_activity_row(src_id, 1_700_000_000_000, &outcome);
+        assert_eq!(
+            row.level,
+            ActivityLevel::Warn,
+            "a handled self-heal is a warning, not an error"
+        );
+        assert_eq!(row.event_type, "drive.remote_file_missing");
+        assert_eq!(row.message.as_deref(), Some("docs/report.pdf"));
+        assert_eq!(row.source_id, Some(src_id));
+
+        // Not a failed op: the cycle's timestamp-advance gate and the closing
+        // progress snapshot both leave it out.
+        assert!(!matches!(outcome, OpOutcome::Failed { .. }));
+        assert_eq!(
+            exec_progress_from(
+                &crate::types::PlanSummary {
+                    uploads: 1,
+                    ..Default::default()
+                },
+                std::slice::from_ref(&outcome),
+            )
+            .errors,
+            0,
+            "a healed missing remote copy must not be counted as a cycle error"
+        );
+    }
+
     #[tokio::test]
     async fn per_op_activity_survives_a_mid_plan_stop() {
         // R2-P2-1: activity is streamed PER OP (persisted immediately after each

crates/driven-core/src/orchestrator.rs

@@ -1027,6 +1027,15 @@ pub enum ErrorCode {
     /// `drive.unreachable` - Drive API down, unreachable, or 5xx
     /// circuit-open.
     DriveUnreachable,
+    /// `drive.remote_file_missing` - the backed-up copy this file's recorded
+    /// `drive_file_id` points at no longer exists on Drive (the user deleted
+    /// and purged it out-of-band), so the planned UPDATE returned a definitive
+    /// 404. NOT an error the user must act on: the executor clears the stale id
+    /// and the next scan re-uploads the file as a fresh create, so this is a
+    /// warn-level, self-healing outcome rather than a failure. Distinct from
+    /// [`Self::DriveUnreachable`], which a 404 used to be reported as - and
+    /// which is retried forever against an id that can never come back.
+    DriveRemoteFileMissing,
     /// `drive.resumable_session_invalid` - 4xx during resumable upload;
     /// caller must restart the session.
     DriveResumableSessionInvalid,
@@ -1148,6 +1157,7 @@ impl ErrorCode {
             ErrorCode::DriveUploadSizeLimit => "drive.upload_size_limit",
             ErrorCode::DriveChecksumMismatch => "drive.checksum_mismatch",
             ErrorCode::DriveUnreachable => "drive.unreachable",
+            ErrorCode::DriveRemoteFileMissing => "drive.remote_file_missing",
             ErrorCode::DriveResumableSessionInvalid => "drive.resumable_session_invalid",
             ErrorCode::DriveDestFolderMissing => "drive.dest_folder_missing",
             ErrorCode::DriveDestFolderPermissionDenied => "drive.dest_folder_permission_denied",
@@ -1202,6 +1212,7 @@ impl ErrorCode {
             "drive.upload_size_limit" => ErrorCode::DriveUploadSizeLimit,
             "drive.checksum_mismatch" => ErrorCode::DriveChecksumMismatch,
             "drive.unreachable" => ErrorCode::DriveUnreachable,
+            "drive.remote_file_missing" => ErrorCode::DriveRemoteFileMissing,
             "drive.resumable_session_invalid" => ErrorCode::DriveResumableSessionInvalid,
             "drive.dest_folder_missing" => ErrorCode::DriveDestFolderMissing,
             "drive.dest_folder_permission_denied" => ErrorCode::DriveDestFolderPermissionDenied,
@@ -1374,6 +1385,35 @@ mod tests {
         assert_eq!(rp.as_str(), "a/b.txt");
     }
 
+    // --- ErrorCode (SPEC s24 stable codes) ----------------------------------
+
+    /// `drive.remote_file_missing` (the self-healing stale-`drive_file_id`
+    /// outcome) must round-trip BOTH ways: `code()` is the persisted
+    /// `activity_log.event_type` and the UI's i18n key, and `from_code()` is
+    /// how a stored row is read back. A mapping added in only one direction
+    /// would silently degrade an existing row to "unknown code".
+    #[test]
+    fn drive_remote_file_missing_round_trips() {
+        assert_eq!(
+            ErrorCode::DriveRemoteFileMissing.code(),
+            "drive.remote_file_missing"
+        );
+        assert_eq!(
+            ErrorCode::from_code("drive.remote_file_missing"),
+            Some(ErrorCode::DriveRemoteFileMissing)
+        );
+        // ...and it is its OWN code, not folded into drive.unreachable (the
+        // bug this code exists to fix reported a definitive 404 as unreachable).
+        assert_ne!(
+            ErrorCode::DriveRemoteFileMissing.code(),
+            ErrorCode::DriveUnreachable.code()
+        );
+        assert_eq!(
+            ErrorCode::DriveRemoteFileMissing.to_string(),
+            "drive.remote_file_missing"
+        );
+    }
+
     // --- ScheduleConfig (V2 schedule windows) -------------------------------
 
     /// Monday 2024-01-01 00:00:00 UTC, in epoch ms. The dow formula resolves

crates/driven-core/src/types.rs

@@ -22,8 +22,8 @@
 //! - Transient faults (rate-limit, 5xx, network-drop) reset to "never
 //!   trip" once they fire; the next request after the trip succeeds.
 //! - "Stay-broken" faults (auth.invalid_grant, dest-folder missing /
-//!   readonly, trashed-visible-in-find) latch on first trigger and
-//!   remain set for the lifetime of the store.
+//!   readonly, update-target-not-found, trashed-visible-in-find) latch on
+//!   first trigger and remain set for the lifetime of the store.
 //! - `md5_mismatch_after` latches **on the affected entry**: when it
 //!   trips during a write, the entry's wrong md5 is stamped onto the
 //!   entry so every subsequent read of that entry (metadata,
@@ -175,6 +175,24 @@ impl InMemoryRemoteStore {
         self
     }
 
+    /// Latches "the object an UPDATE targets is gone": every `update()` and
+    /// every resumable session opened as [`crate::remote_store::ResumableKind::Update`]
+    /// fails with a real-shaped classified Drive 404, modelling a
+    /// `drive_file_id` whose object the user deleted (and purged) out-of-band.
+    ///
+    /// CREATES keep working, which is the point: the executor must clear the
+    /// stale id and re-upload the same path as a fresh create on the next
+    /// cycle (`drive.remote_file_missing`) instead of retrying a doomed update
+    /// forever. Read-only calls are unaffected.
+    ///
+    /// Latches for the lifetime of the store (a deleted Drive object does not
+    /// come back), so a test that wants the re-upload to succeed simply lets
+    /// the next cycle take the create path.
+    pub fn with_update_not_found(self) -> Self {
+        self.faults.update_not_found.store(true, Ordering::Release);
+        self
+    }
+
     /// Latches the destination-folder-readonly state. Every subsequent
     /// write-target request returns `drive.dest_folder_permission_denied`
     /// (SPEC s24). Read-only calls keep working - mirrors the user

crates/driven-drive/src/fake/fault_injection.rs

@@ -524,6 +524,14 @@ pub(crate) struct Faults {
     pub(crate) response_delay_nanos: AtomicU64,
     /// Latched once tripped (auth.invalid_grant is "stay-broken").
     pub(crate) invalid_grant_latched: std::sync::atomic::AtomicBool,
+    /// When true, every `update()` (and every resumable session opened with
+    /// [`ResumableKind::Update`]) fails with a REAL-SHAPED Drive 404 - the
+    /// classified error a `GoogleDriveStore` produces when the recorded
+    /// `drive_file_id` no longer exists (the user deleted or purged the object
+    /// out-of-band). Creates are unaffected, so a self-healing executor can
+    /// re-upload the same path on the next cycle. Latches for the lifetime of
+    /// the store; set by [`fault_injection::with_update_not_found`].
+    pub(crate) update_not_found: std::sync::atomic::AtomicBool,
     /// Dest-folder states are latched on construction by the builder.
     pub(crate) dest_folder_missing: std::sync::atomic::AtomicBool,
     pub(crate) dest_folder_readonly: std::sync::atomic::AtomicBool,
@@ -569,6 +577,7 @@ impl Default for Faults {
             daily_quota_latched: AtomicBool::new(false),
             response_delay_nanos: AtomicU64::new(0),
             invalid_grant_latched: AtomicBool::new(false),
+            update_not_found: AtomicBool::new(false),
             dest_folder_missing: AtomicBool::new(false),
             dest_folder_readonly: AtomicBool::new(false),
             trashed_visible_in_find: AtomicBool::new(false),
@@ -840,6 +849,25 @@ enum RequestKind {
     WriteTarget,
 }
 
+/// The error a real `GoogleDriveStore` produces when a request names a Drive
+/// object that no longer exists: a CLASSIFIED [`crate::google::DriveError`]
+/// built from a genuine Drive 404 body.
+///
+/// Built through [`crate::google::DriveError::from_response`] (not a plain
+/// `anyhow::anyhow!` string) precisely so the fake reproduces the real shape
+/// end-to-end: the executor's typed downcast classifies it
+/// [`crate::remote_store::DriveErrorClassification::Other`] AND
+/// [`crate::google::error_is_not_found`] finds `drive HTTP 404` in the source
+/// chain. A stringly fake message would satisfy neither.
+fn remote_file_missing_error(file_id: &str) -> anyhow::Error {
+    let body = format!(r#"{{"error":{{"code":404,"message":"File not found: {file_id}."}}}}"#);
+    anyhow::Error::new(crate::google::DriveError::from_response(
+        404,
+        body.as_bytes(),
+        None,
+    ))
+}
+
 /// Atomically decrement `counter` if it is non-zero and not `u64::MAX`.
 /// Returns `true` iff the decrement crossed from 1 to 0 (the "trip"
 /// edge). `u64::MAX` means "never trip" and is left alone.
@@ -1022,6 +1050,9 @@ impl RemoteStore for InMemoryRemoteStore {
         app_properties_patch: HashMap<String, String>,
     ) -> anyhow::Result<RemoteEntry> {
         self.check_request_faults(RequestKind::WriteTarget).await?;
+        if self.faults.update_not_found.load(Ordering::Acquire) {
+            return Err(remote_file_missing_error(file_id));
+        }
         let body = collect_body(body, self.oracle_on()).await?;
 
         let new_len = body.len();
@@ -1076,6 +1107,12 @@ impl RemoteStore for InMemoryRemoteStore {
                 guard.ensure_folder_parent(parent_id)?;
             }
             ResumableKind::Update { file_id } => {
+                // The update-target-missing fault applies here too: the real
+                // Drive validates the file id when the session is opened, so a
+                // gone object 404s before a single byte is pushed.
+                if self.faults.update_not_found.load(Ordering::Acquire) {
+                    return Err(remote_file_missing_error(file_id));
+                }
                 guard.ensure_object(file_id)?;
             }
         }

crates/driven-drive/src/fake/mod.rs

@@ -240,7 +240,7 @@ impl std::error::Error for DriveError {
     fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
         match self {
             // The `anyhow::Error` source is surfaced as the error chain so the
-            // `is_not_found` helper (and any caller) can walk to the
+            // `error_is_not_found` helper (and any caller) can walk to the
             // `drive HTTP <status>` cause. `anyhow::Error` impls
             // `AsRef<dyn Error + Send + Sync>`; coerce off the auto traits to
             // the `source()` return type.
@@ -1494,7 +1494,7 @@ impl RemoteStore for GoogleDriveStore {
             Ok(_) => Ok(()),
             Err(e) => {
                 // 404 -> already gone, treated as success (SPEC s3 `trash`).
-                if is_not_found(&e) {
+                if error_is_not_found(&e) {
                     Ok(())
                 } else {
                     Err(e)
@@ -1522,7 +1522,7 @@ impl RemoteStore for GoogleDriveStore {
         match result {
             Ok(_) => Ok(()),
             Err(e) => {
-                if is_not_found(&e) {
+                if error_is_not_found(&e) {
                     Ok(())
                 } else {
                     Err(e)
@@ -1758,8 +1758,26 @@ pub(crate) fn parse_retry_after(resp: &reqwest::Response) -> Option<u64> {
         .map(|secs| secs.saturating_mul(1000))
 }
 
-/// Whether an error is a Drive 404 (used to make `trash` idempotent).
-fn is_not_found(err: &anyhow::Error) -> bool {
+/// Whether `err` is a DEFINITIVE Drive "not found" (HTTP 404): the object the
+/// request named does not exist (deleted out-of-band, purged from the trash, or
+/// never ours).
+///
+/// The generic [`DriveErrorClassification`] of a 404 is
+/// [`DriveErrorClassification::Other`], which renders as
+/// `drive.unreachable: unclassified Drive error` - indistinguishable from any
+/// other unclassified failure. Callers that must react to a MISSING object
+/// specifically (the executor self-healing a stale `drive_file_id`, `trash`
+/// treating a 404 as an idempotent no-op) use this instead of re-deriving the
+/// classification: [`DriveError::from_response`] embeds the literal
+/// `drive HTTP 404` in the source chain, and [`DriveError`]'s
+/// `Error::source` exposes that chain, so walking it is the one reliable
+/// signal available across both the typed and stringly paths.
+///
+/// Deliberately narrow: only a real 404 matches. A 403 / 5xx / transport
+/// failure returns `false`, so a transient fault is never mistaken for a
+/// permanently-gone object.
+#[must_use]
+pub fn error_is_not_found(err: &anyhow::Error) -> bool {
     // `DriveError::from_response` embeds `drive HTTP 404` in the source chain.
     err.chain()
         .any(|c| c.to_string().contains("drive HTTP 404"))
@@ -1964,6 +1982,59 @@ pub(crate) fn clone_kind(kind: &ResumableKind) -> ResumableKind {
 mod tests {
     use super::*;
 
+    /// A 404 arriving as a CLASSIFIED Drive error (the shape every real
+    /// `GoogleDriveStore` call produces) must be detected by
+    /// [`error_is_not_found`], even though its classification is the generic
+    /// [`DriveErrorClassification::Other`] whose `Display` says
+    /// `drive.unreachable`. Callers (the executor's stale-`drive_file_id`
+    /// self-heal, `trash`'s idempotent no-op) depend on this distinction.
+    #[test]
+    fn error_is_not_found_detects_a_classified_404() {
+        let err = anyhow::Error::new(DriveError::from_response(
+            404,
+            br#"{"error":{"code":404,"message":"File not found: abc123."}}"#,
+            None,
+        ));
+        assert!(
+            error_is_not_found(&err),
+            "a classified Drive 404 must read as not-found, got: {err:?}"
+        );
+        // The generic classification stays `Other` (this helper is the decision
+        // point, not the classification) - guard against a future rework
+        // silently changing that assumption.
+        assert_eq!(
+            classification_of(&err),
+            Some(DriveErrorClassification::Other),
+            "a 404 classifies as Other; only `error_is_not_found` distinguishes it"
+        );
+    }
+
+    /// Nothing else may masquerade as not-found: a transient 5xx, a 403, a
+    /// plain `anyhow` message, and a wrapped error whose chain has no
+    /// `drive HTTP 404` all return `false`, so a temporary fault is never
+    /// mistaken for a permanently-gone object.
+    #[test]
+    fn error_is_not_found_rejects_non_404_errors() {
+        let five_xx = anyhow::Error::new(DriveError::from_response(503, b"unavailable", None));
+        assert!(
+            !error_is_not_found(&five_xx),
+            "a transient 5xx must NOT read as not-found"
+        );
+        let forbidden = anyhow::Error::new(DriveError::from_response(403, b"forbidden", None));
+        assert!(
+            !error_is_not_found(&forbidden),
+            "a 403 must NOT read as not-found"
+        );
+        assert!(
+            !error_is_not_found(&anyhow::anyhow!("drive: something else went wrong")),
+            "an untyped Drive error must NOT read as not-found"
+        );
+        assert!(
+            !error_is_not_found(&anyhow::anyhow!("boom").context("while updating")),
+            "a plain wrapped error must NOT read as not-found"
+        );
+    }
+
     #[test]
     fn google_store_new_builds_a_stream_client_with_the_ca() {
         // Issue #34: `GoogleDriveStore::new` derives its streaming client via

crates/driven-drive/src/google/mod.rs

@@ -1773,6 +1773,7 @@ Stable codes (V1):
 | `drive.upload_size_limit`    | File exceeds Drive's per-file size limit      |
 | `drive.checksum_mismatch`    | Verification failed after upload              |
 | `drive.unreachable`          | Drive API down / unreachable / 5xx-circuit-open |
+| `drive.remote_file_missing`  | The backed-up copy a recorded `drive_file_id` points at is gone (definitive 404 on an update); the stale id is cleared and the file re-uploads on the next scan (warn, self-healing) |
 | `drive.resumable_session_invalid` | 4xx during resumable upload — caller must restart session |
 | `local.file_locked`          | Couldn't open even with `FILE_SHARE_DELETE` (V1: locked file; VSS path failed too — see `local.vss_unavailable`) |
 | `local.vss_unavailable`      | Driven needs elevation to use VSS but isn't elevated |

design/SPEC.md

@@ -420,6 +420,7 @@ impl InMemoryRemoteStore {
     pub fn with_md5_mismatch_after(self, n: u64) -> Self;
     pub fn with_dest_folder_missing(self) -> Self;
     pub fn with_dest_folder_readonly(self) -> Self;
+    pub fn with_update_not_found(self) -> Self;
     pub fn with_fileid_recycle(self) -> Self;
 }
 ```

design/STRESS_HARNESS.md

@@ -527,6 +527,10 @@ fn error_code_is_network(code: ErrorCode) -> bool {
         | ErrorCode::DriveResumableSessionInvalid
         | ErrorCode::DriveDestFolderMissing
         | ErrorCode::DriveDestFolderPermissionDenied
+        // A missing remote copy is a self-healed, per-file warning (the next
+        // scan re-uploads it), and it says nothing about reachability - Drive
+        // answered, the object was simply gone. Non-network either way.
+        | ErrorCode::DriveRemoteFileMissing
         // Local filesystem / VSS errors -> red error. (The transient
         // vss_helper_pending skip is not an error condition, but it is not a
         // network one either, so it classifies here as non-network.)

src-tauri/src/tray.rs

@@ -45,6 +45,10 @@ describe("activityEventLabel (R1-P2-3)", () => {
     // those are localized via the shared error labels.
     expect(label("drive.checksum_mismatch")).toBe("Verification failed");
     expect(label("local.file_locked")).toBe("File in use");
+    // The self-healing stale-drive_file_id skip: a warn row whose event type is
+    // this code, so the Activity table must have a label for it rather than
+    // showing the raw dotted string.
+    expect(label("drive.remote_file_missing")).toBe("Remote copy missing");
   });
 
   it("safely falls back to the raw code for an unknown event type", () => {