fix(chaos): central md5+blake3 s6.3 content invariant + close M3.7 recheck-1

Closes the still-open M3.7 recheck round-1 codex findings (baseline 60d3a1c;
finding 1 disk-full was already resolved as an honest documented known-gap):

- F2 (P1) central no-data-loss now enforces STRESS_HARNESS s6.3 in full: for
  every synced row the live object must exist, its md5 must match drive_md5,
  AND (unencrypted + retained-bytes) the bytes must hash to hash_blake3. Adds
  blake3 dep + InMemoryRemoteStore::object_content; gates the byte-hash on
  !encryption_enabled (ciphertext != plaintext hash) and skips oracle-backed
  huge files (md5 is their proof). Mirrors an md5 check into the drive_side
  per-category checker.
- F3 (P1) drive-fileid-recycled asserts y.id == id_x so a regressed recycler
  fails instead of passing vacuously on the op-uuid check.
- F4 (P1) distinct chaos-fake-drive CI job (3-OS) via run-all --fault-injection
  + registry::fault_injection_registry (ROADMAP M3.7 / STRESS_HARNESS s7).
- F5 (P2) fake trash()/about() use content_len() so oracle-backed objects free
  their true logical size from bytes_stored.
- F6 (P2) mutator-drive-daily-quota runs hermetically via the real
  with_daily_quota_after injector; drops the real-creds gate + SKIP early-return.
- F7 (P2) capability probes target the target/chaos-fixtures volume, not temp.

Gates: fmt; build; clippy -D warnings; test --workspace (all green); hermetic
56 PASS/29 SKIP/0 FAIL; fault-injection 27 PASS/0 FAIL.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01J8meqeTo8bcZ3zjgKjBnJ4

Author: pmaxhogan

.github/workflows/chaos.yml

@@ -10,6 +10,13 @@ name: Chaos
 #                      the NTFS / Win32-shaped rows. None need real Drive or
 #                      elevation (the admin / VSS / real-Drive rows SKIP).
 #                      ROADMAP M3.7 acceptance: Linux + macOS + Windows.
+#   chaos-fake-drive - every PR, 3-OS matrix. The dedicated fault-injection gate
+#                      (STRESS_HARNESS s7 / ROADMAP M3.7): runs ONLY the s3.7
+#                      Drive-side hazards + s4.2/s5 drive-side mutator faults
+#                      against InMemoryRemoteStore (`run-all --fault-injection`).
+#                      These rows are also in the hermetic sweep; this focused
+#                      gate gives the separately-named, faster fault-injection
+#                      check the acceptance requires.
 #   chaos-real-drive - WIRED but SKIPPED until M4: gated on the
 #                      DRIVEN_E2E_REFRESH_TOKEN secret, which is absent until
 #                      the GoogleDriveStore (M4) lands. It skips cleanly, it
@@ -69,6 +76,31 @@ jobs:
       - name: chaos run-all (hermetic)
         run: cargo run -p driven-chaos -- run-all --hermetic
 
+  chaos-fake-drive:
+    name: chaos fake-drive (${{ matrix.os }})
+    # The dedicated fault-injection gate (ROADMAP M3.7 acceptance: a distinct
+    # chaos-fake-drive job on Linux + macOS + Windows). Runs ONLY the s3.7 /
+    # s4.2 / s5 fault-injection subset against InMemoryRemoteStore; capability
+    # gated rows (the daily-quota real-Drive path) SKIP cleanly, never red.
+    if: github.event_name != 'schedule'
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+      - name: Install Linux Tauri deps
+        if: runner.os == 'Linux'
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libwebkit2gtk-4.1-dev libxdo-dev libssl-dev \
+            libayatana-appindicator3-dev librsvg2-dev libsoup-3.0-dev javascriptcoregtk-4.1
+      - uses: Swatinem/rust-cache@v2
+      - name: chaos run-all (fault-injection)
+        run: cargo run -p driven-chaos -- run-all --fault-injection
+
   chaos-real-drive:
     name: chaos real-drive (M4-gated, skipped)
     # WIRED but SKIPPED until M4 (STRESS_HARNESS s7 "Gating on M4"): the

Cargo.lock

@@ -36,6 +36,9 @@ uuid.workspace = true
 # oracle instead of buffering / downloading 10-50 GB; same md-5 crate the core
 # + fake use, imported as `md5`.
 md5.workspace = true
+# Central s6.3 content-integrity invariant: hash a synced object's retained
+# bytes to the recorded plaintext blake3 (same crate + version the core uses).
+blake3 = "1"
 # Hermetic per-scenario StateRepo files live under a throwaway tempdir.
 tempfile = "3"
 

crates/driven-chaos/Cargo.toml

@@ -163,13 +163,17 @@ impl CapabilitySet {
     pub fn probe() -> Self {
         let real_drive_creds = std::env::var("DRIVEN_E2E_REFRESH_TOKEN").is_ok()
             && std::env::var("DRIVEN_E2E_DEST_FOLDER_ID").is_ok();
-        // The volume backing the harness temp dir is where every fixture is
-        // materialised, so free-disk + filesystem-type probes target it.
-        let temp_dir = std::env::temp_dir();
+        // Every fixture is materialised under `target/chaos-fixtures` (see
+        // runner::fixture_root_for), NOT the system temp dir, so the free-disk +
+        // filesystem-type probes must target THAT volume. On a multi-volume host
+        // `target/` and the temp dir can be different disks; a big-file gate
+        // keyed to the wrong one would mis-skip or fill the workspace volume
+        // (recheck P2-G).
+        let fixture_root = fixture_root_dir();
         let admin = driven_vss::is_elevated();
-        let free_disk_bytes = probe_free_disk_bytes(&temp_dir);
-        let ntfs_volume = probe_ntfs_volume(&temp_dir);
-        let case_sensitive_volume = probe_case_sensitive_volume(&temp_dir);
+        let free_disk_bytes = probe_free_disk_bytes(&fixture_root);
+        let ntfs_volume = probe_ntfs_volume(&fixture_root);
+        let case_sensitive_volume = probe_case_sensitive_volume(&fixture_root);
         let long_paths_enabled = probe_long_paths_enabled();
         // VSS needs Windows + elevation (driven-vss only exposes the COM
         // sequence on an elevated Windows host).
@@ -196,6 +200,21 @@ impl CapabilitySet {
     }
 }
 
+/// The volume root every chaos fixture is materialised under - the
+/// `target/chaos-fixtures` directory that matches `runner::fixture_root_for`.
+/// The free-disk and filesystem-type capability probes target this path rather
+/// than the system temp dir, so a multi-volume host gates the big-file rows on
+/// the disk the fixtures actually land on (recheck P2-G). It is created
+/// best-effort so the probes have an existing path to stat, and falls back to
+/// the temp dir when it cannot be created (for example, a read-only checkout).
+fn fixture_root_dir() -> std::path::PathBuf {
+    let root = std::path::PathBuf::from("target/chaos-fixtures");
+    match std::fs::create_dir_all(&root) {
+        Ok(()) => root,
+        Err(_) => std::env::temp_dir(),
+    }
+}
+
 /// Free bytes available to the current user on the volume backing `path`.
 /// Returns 0 (the conservative "no space" reading) on any probe failure.
 #[cfg(windows)]

crates/driven-chaos/src/capabilities.rs

@@ -50,6 +50,10 @@ pub enum Command {
     },
     /// `scenario run-all` - run every scenario, respecting capability gates.
     ScenarioRunAll,
+    /// `run-all --fault-injection` - run ONLY the fault-injection subset
+    /// (STRESS_HARNESS s3.7 / s4.2 / s5) the dedicated `chaos-fake-drive` CI
+    /// gate covers.
+    ScenarioRunFaultInjection,
     /// `fuzz [--seed N --duration D]` - property-style soak run.
     Fuzz {
         /// Seed for the weighted mutation distribution; `None` = `now()`.
@@ -133,14 +137,22 @@ pub fn parse(args: &[String]) -> anyhow::Result<Command> {
             })
         }
         Some("run-all") => {
-            // Accept and ignore a trailing `--hermetic` (the only mode there is).
+            // `--hermetic` is a no-op (the harness is hermetic by construction).
+            // `--fault-injection` narrows the run to the s3.7/s4.2/s5 fault
+            // subset the dedicated `chaos-fake-drive` CI gate runs.
+            let mut fault_injection = false;
             for flag in it {
                 match flag {
                     "--hermetic" => {}
+                    "--fault-injection" => fault_injection = true,
                     other => anyhow::bail!("unknown run-all flag: {other}"),
                 }
             }
-            Ok(Command::ScenarioRunAll)
+            if fault_injection {
+                Ok(Command::ScenarioRunFaultInjection)
+            } else {
+                Ok(Command::ScenarioRunAll)
+            }
         }
         Some("fuzz") => {
             let mut seed = None;
@@ -339,6 +351,17 @@ pub async fn run(command: Command, caps: &CapabilitySet) -> i32 {
                 exit_code::OK
             }
         }
+        Command::ScenarioRunFaultInjection => {
+            let report = runner::run_all(registry::fault_injection_registry(), caps).await;
+            print!("{}", report.render_json());
+            print!("{}", report.render_human());
+            persist_last_run(&report);
+            if report.any_failed() {
+                exit_code::FAIL
+            } else {
+                exit_code::OK
+            }
+        }
         Command::Fuzz {
             seed,
             duration_secs,

crates/driven-chaos/src/dispatch.rs

@@ -31,6 +31,20 @@ pub fn registry() -> Vec<Box<dyn Scenario>> {
     all
 }
 
+/// The fault-injection subset (STRESS_HARNESS s3.7 Drive-side hazards + s4.2/s5
+/// drive-side mutator faults) the dedicated `chaos-fake-drive` CI job runs
+/// (ROADMAP M3.7 acceptance: a distinct fake-drive gate on Linux + macOS +
+/// Windows that "adds the fault-injection scenarios" against
+/// `InMemoryRemoteStore`). These rows are ALSO covered by the full `run-all`
+/// hermetic sweep; this focused selection gives the separately-named,
+/// faster-feedback gate the acceptance requires.
+pub fn fault_injection_registry() -> Vec<Box<dyn Scenario>> {
+    let mut all: Vec<Box<dyn Scenario>> = Vec::new();
+    all.extend(scenarios::drive_side::scenarios());
+    all.extend(scenarios::mutator::scenarios());
+    all
+}
+
 /// Look one scenario up by its stable name (the `scenario run <name>` and
 /// `fixture create <name>` argument). Returns `None` if unknown.
 pub fn find(name: &str) -> Option<Box<dyn Scenario>> {

crates/driven-chaos/src/registry.rs

@@ -276,6 +276,18 @@ async fn check_invariants(
                 entry.size, row.size
             ));
         }
+        // md5 content integrity (s6.3): the live object's md5 must equal the
+        // recorded `drive_md5`. The central sweep (reporting::assert_invariants)
+        // additionally byte-hashes blake3 for the categories that delegate to
+        // it; the drive_side checker proves content via this md5 + the size
+        // check above (the typed fake's byte accessor is not plumbed through
+        // this per-category checker, which sees only the RemoteStore trait).
+        if let Some(drive_md5) = row.drive_md5 {
+            if entry.md5 != Some(drive_md5) {
+                no_data_loss = false;
+                notes.push(format!("data-loss: synced row {rel} drive md5 mismatch"));
+            }
+        }
         // The local file must still exist at the recorded size.
         let abs = std::path::Path::new(&source.local_path).join(rel.as_str());
         match std::fs::metadata(&abs) {
@@ -1123,6 +1135,15 @@ impl Scenario for DriveFileidRecycled {
             .collect();
         anyhow::ensure!(live.len() == 1, "exactly one live object (Y)");
         let y = &live[0];
+        // The whole point of this scenario is that Y REUSED X's trashed file_id;
+        // assert the recycle actually happened, so a regression in
+        // `with_fileid_recycle()` that hands Y a fresh id fails here instead of
+        // passing vacuously on the op-uuid check below.
+        anyhow::ensure!(
+            y.id == id_x,
+            "file_id was not recycled: Y id {} != X id {id_x}",
+            y.id
+        );
         let uuid_y = y
             .app_properties
             .get(CLIENT_OP_UUID_KEY)

crates/driven-chaos/src/scenarios/drive_side.rs

@@ -1027,17 +1027,16 @@ impl DriveMutatorScenario {
                 base.with_quota_exhausted_after(*after_bytes)
             }
             DriveKind::DailyQuota => {
-                // The fake exposes the storage-quota builder; the daily path is
-                // exercised here via the same quota builder with a 0-byte
-                // budget, but classified daily by the executor only if the fake
-                // emits a "daily" message. The fake's quota builder emits
-                // "quota_exhausted" (storage), NOT "daily" - so a faithful
-                // daily injection is NOT available on the current fake.
-                // Returning the plain remote here would silently mis-assert, so
-                // this kind is handled as a capability gap below (see
-                // run_assertions): we DO NOT fake a daily code we cannot
-                // produce.
-                base
+                // The fake's `with_daily_quota_after(n)` trips a faithful
+                // `403 dailyLimitExceeded` after `n` WriteTarget requests, then
+                // LATCHES (the daily window stays closed for the run). Its
+                // message carries "daily", so the executor's
+                // `classify_drive_error` maps it to `DriveError::DailyQuota` ->
+                // `ErrorCode::DriveDailyQuotaExhausted` (distinct from storage
+                // `quota_exhausted`). `after(4)` trips MID-run (a couple files
+                // land first), matching the sibling request-counted faults and
+                // the s9 "no data loss for files done before the failure" check.
+                base.with_daily_quota_after(4)
             }
             DriveKind::RateLimitStorm { after_requests } => {
                 base.with_rate_limit_after(*after_requests)
@@ -1067,23 +1066,12 @@ impl Scenario for DriveMutatorScenario {
         self.description
     }
     fn requires(&self) -> CapabilityRequirements {
-        // Most s4.2 scenarios run against the in-memory fake - no host
-        // capability needed. The daily-quota row is the exception: a faithful
-        // `dailyLimitExceeded` signal (distinct from `storageQuotaExceeded`)
-        // can ONLY be produced by real Drive - the InMemoryRemoteStore fault
-        // surface has no daily-vs-storage discriminator, and the executor
-        // classifies "daily" only from a message the fake does not emit. So,
-        // exactly like the sibling drive_side `daily-quota-exhausted` row, this
-        // gates on `cap:real_drive_creds` and SKIPs cleanly (with the missing
-        // capability recorded) until the live path is wired in M4. This is an
-        // honest capability-skip, NOT a faked or weakened code. Follow-up: add
-        // InMemoryRemoteStore::with_daily_quota_after to driven-drive so a fake
-        // variant becomes possible.
-        if matches!(self.kind, DriveKind::DailyQuota) {
-            CapabilityRequirements::of(vec![Capability::RealDriveCreds])
-        } else {
-            CapabilityRequirements::none()
-        }
+        // Every s4.2 scenario - including daily-quota - runs against the
+        // in-memory fake, so none need a host capability. The fake's
+        // `with_daily_quota_after` now emits a faithful `dailyLimitExceeded`
+        // (see `build_remote`), so the daily-quota row no longer gates on
+        // `cap:real_drive_creds`; it runs hermetically like its siblings.
+        CapabilityRequirements::none()
     }
 
     async fn setup(&self, ctx: &mut ScenarioContext) -> anyhow::Result<()> {
@@ -1092,26 +1080,11 @@ impl Scenario for DriveMutatorScenario {
     }
 
     async fn run_assertions(&self, _handle: &DrivenHandle) -> anyhow::Result<Outcome> {
-        // The current InMemoryRemoteStore fault surface cannot emit a faithful
-        // `daily` (vs storage) quota signal; the executor classifies "daily"
-        // only from a message the fake does not produce. Rather than assert a
-        // weakened/faked code, this scenario records the capability gap and
-        // returns a documented-skip outcome that the driver renders SKIPPED.
-        if matches!(self.kind, DriveKind::DailyQuota) {
-            return Ok(Outcome {
-                error_codes_seen: vec![],
-                final_drive_object_count: 0,
-                final_hash_matches_local: true,
-                notes: vec![
-                    "SKIP-by-capability: InMemoryRemoteStore exposes no faithful dailyLimitExceeded injector (only storageQuotaExceeded via with_quota_exhausted_after); the executor's daily classification needs a 'daily' message the fake does not emit. Not faking the code. Follow-up: add InMemoryRemoteStore::with_daily_quota_after to driven-drive fault_injection.".to_string(),
-                    "invariants: None - this is a capability SKIP early-return; no handle/source/remote was booted, so there is no single source+folder snapshot to sweep.".to_string(),
-                ],
-                // Capability SKIP: nothing was synced, so there is no terminal
-                // source+remote state for the canonical s6.3 sweep to read.
-                invariants: None,
-            });
-        }
-
+        // Every kind - including daily-quota - now runs the same hermetic body:
+        // `build_remote` rigs the kind's fault on the fake (the daily-quota row
+        // uses the real `with_daily_quota_after` injector), and the run drives
+        // cycles, collects the surfaced codes, and sweeps the canonical s6.3
+        // invariants. No kind early-returns a capability SKIP any more.
         let state_dir = tempfile::tempdir()?;
         let root_dir = tempfile::tempdir()?;
         // A modest multi-file population so a "after N bytes / N requests"
@@ -1667,27 +1640,38 @@ mod tests {
         );
     }
 
-    /// The daily-quota scenario honestly SKIPS (capability gap) rather than
-    /// faking a code the fake cannot emit.
+    /// The daily-quota scenario now runs hermetically against the fake's
+    /// `with_daily_quota_after` injector and surfaces the real
+    /// `DriveDailyQuotaExhausted` code (distinct from storage quota). It is no
+    /// longer a capability SKIP - the injector exists.
     #[tokio::test]
-    async fn drive_daily_quota_skips_by_capability() {
+    async fn drive_daily_quota_surfaces_code() {
         let scenario = DriveMutatorScenario::daily_quota_exhausted();
+        // It runs on a stock host - no real-Drive capability required.
+        assert!(
+            scenario.requires().required.is_empty(),
+            "daily-quota row needs no host capability now"
+        );
         let state_dir = tempfile::tempdir().expect("state dir");
         let remote = Arc::new(InMemoryRemoteStore::new());
         let handle = boot_handle(state_dir.path(), remote)
             .await
             .expect("boot placeholder handle");
         let outcome = scenario.run_assertions(&handle).await.expect("runs");
         assert!(
-            outcome.error_codes_seen.is_empty(),
-            "no code faked for the unavailable daily-quota injector"
+            outcome
+                .error_codes_seen
+                .contains(&ErrorCode::DriveDailyQuotaExhausted),
+            "drive.daily_quota_exhausted surfaced; saw {:?}",
+            outcome.error_codes_seen
         );
+        // The daily code is classified distinctly from storage quota.
         assert!(
-            outcome
-                .notes
-                .iter()
-                .any(|n| n.contains("SKIP-by-capability")),
-            "records the capability gap reason"
+            !outcome
+                .error_codes_seen
+                .contains(&ErrorCode::DriveQuotaExhausted),
+            "daily quota is not misclassified as storage quota; saw {:?}",
+            outcome.error_codes_seen
         );
     }
 }