fix(backup): survive iCloud's fresh-install metadata race; make restore resumable (build 11) (#21)

On a real device a fresh install's ubiquity container is cloud-only
until metadata syncs; the restore used plain filesystem walks, missed
the un-materialized note bodies, and installing the catalog blocked any
retry - notes came back with titles but no content (found on-device in
1.3 build 10; the dev-override simulator E2E structurally cannot catch
this).

- ICloudBackupModule: listCloudFiles via NSMetadataQuery (sees cloud-
  only items and nudges the sync; main-queue lifecycle with timeout and
  double-completion guard; /private symlink standardization; cloud-only
  directories classified by content type). ensureDownloaded now retries
  startDownloadingUbiquitousItem until the deadline instead of giving
  up when metadata has not arrived.
- backupEngine: manifest and backup catalog reads are download-forced
  too (the same race can hit them). Restore now MERGES the backup
  catalog - backup metadata adopted for missing notes and recovery
  stubs, local notes and tombstones untouched - and is resumable:
  resumeRestoreIfIncomplete heals notes-without-bodies and stub titles
  on every launch, gated to files iCloud still lists so a stale
  manifest entry cannot cause a retry-forever loop.
- useBackup runs the silent resume before the startup sync, so devices
  already damaged by build 10 self-heal on first launch of this build.

Verified: 48 node tests including the device repro (partial restore
resumes to completion) and the mirror-image race (stub titles heal);
simulator E2E of the exact broken device state - catalog with titles,
zero bodies, one a stub - fully healed on launch with no prompts.

Claude-Session: https://claude.ai/code/session_01Vy3wDR2rgpvNKrbBpCbrEY

Author: markm39

android/app/build.gradle

@@ -92,7 +92,7 @@ android {
         applicationId 'com.builderpro.opennotes'
         minSdkVersion rootProject.ext.minSdkVersion
         targetSdkVersion rootProject.ext.targetSdkVersion
-        versionCode 10
+        versionCode 11
         versionName "1.3"
 
         buildConfigField "String", "REACT_NATIVE_RELEASE_LEVEL", "\"${findProperty('reactNativeReleaseLevel') ?: 'stable'}\""

app.json

@@ -16,7 +16,7 @@
     "ios": {
       "bundleIdentifier": "com.builderpro.opennotes",
       "icon": "./assets/icon.png",
-      "buildNumber": "10",
+      "buildNumber": "11",
       "supportsTablet": true,
       "usesIcloudStorage": true,
       "entitlements": {
@@ -65,7 +65,7 @@
     },
     "android": {
       "package": "com.builderpro.opennotes",
-      "versionCode": 10,
+      "versionCode": 11,
       "adaptiveIcon": {
         "foregroundImage": "./assets/adaptive-icon.png",
         "backgroundColor": "#F7F7F4"

ios/OpenNotes.xcodeproj/project.pbxproj

@@ -364,7 +364,7 @@
 				CODE_SIGN_ENTITLEMENTS = OpenNotes/OpenNotes.entitlements;
 				CODE_SIGN_IDENTITY = "Apple Development";
 				CODE_SIGN_STYLE = Automatic;
-				CURRENT_PROJECT_VERSION = 10;
+				CURRENT_PROJECT_VERSION = 11;
 				DEVELOPMENT_TEAM = U2CPXQV7AJ;
 				ENABLE_BITCODE = NO;
 				GCC_PREPROCESSOR_DEFINITIONS = (
@@ -403,7 +403,7 @@
 				CODE_SIGN_ENTITLEMENTS = OpenNotes/OpenNotes.entitlements;
 				CODE_SIGN_IDENTITY = "Apple Development";
 				CODE_SIGN_STYLE = Automatic;
-				CURRENT_PROJECT_VERSION = 10;
+				CURRENT_PROJECT_VERSION = 11;
 				DEVELOPMENT_TEAM = U2CPXQV7AJ;
 				INFOPLIST_FILE = OpenNotes/Info.plist;
 				IPHONEOS_DEPLOYMENT_TARGET = 15.1;

ios/OpenNotes/ICloudBackupModule.m

@@ -32,4 +32,9 @@ @interface RCT_EXTERN_MODULE(ICloudBackupModule, NSObject)
                   resolver:(RCTPromiseResolveBlock)resolver
                   rejecter:(RCTPromiseRejectBlock)rejecter)
 
+RCT_EXTERN_METHOD(listCloudFiles:(NSString *)dir
+                  timeoutMs:(nonnull NSNumber *)timeoutMs
+                  resolver:(RCTPromiseResolveBlock)resolver
+                  rejecter:(RCTPromiseRejectBlock)rejecter)
+
 @end

ios/OpenNotes/ICloudBackupModule.swift

@@ -159,9 +159,10 @@ class ICloudBackupModule: NSObject {
     return path
   }
 
-  /// Ensures a file in the ubiquity container is downloaded locally (iCloud
-  /// may have evicted it). Resolves true when the file is readable, false when
-  /// the download did not complete within the timeout.
+  /// Ensures a file in the ubiquity container is downloaded locally. Handles
+  /// all three states of a fresh install: file present, iCloud placeholder
+  /// present, or cloud metadata not yet synced (startDownloading is retried
+  /// until the metadata arrives). Resolves true when the file is readable.
   @objc
   func ensureDownloaded(_ path: String,
                         timeoutMs: NSNumber,
@@ -170,36 +171,86 @@ class ICloudBackupModule: NSObject {
     DispatchQueue.global(qos: .utility).async {
       let url = URL(fileURLWithPath: Self.plainPath(path))
       let fileManager = FileManager.default
-
-      if fileManager.fileExists(atPath: url.path) {
-        resolver(true)
-        return
-      }
-
-      // A not-yet-downloaded ubiquitous file appears as ".<name>.icloud".
-      let placeholderName = ".\(url.lastPathComponent).icloud"
-      let placeholderUrl = url.deletingLastPathComponent().appendingPathComponent(placeholderName)
-      guard fileManager.fileExists(atPath: placeholderUrl.path) else {
-        resolver(false)
-        return
-      }
-
-      do {
-        try fileManager.startDownloadingUbiquitousItem(at: url)
-      } catch {
-        rejecter("E_DOWNLOAD_START", "Could not start iCloud download for \(url.lastPathComponent)", error)
-        return
-      }
-
       let deadline = Date().addingTimeInterval(timeoutMs.doubleValue / 1000.0)
+
       while Date() < deadline {
         if fileManager.fileExists(atPath: url.path) {
           resolver(true)
           return
         }
-        Thread.sleep(forTimeInterval: 0.2)
+        // Registers download interest. Throws while the item's cloud metadata
+        // has not synced down yet - keep retrying until the deadline.
+        try? fileManager.startDownloadingUbiquitousItem(at: url)
+        Thread.sleep(forTimeInterval: 0.4)
       }
-      resolver(false)
+      resolver(fileManager.fileExists(atPath: url.path))
+    }
+  }
+
+  /// Lists every item iCloud knows about under the container's Documents dir
+  /// via NSMetadataQuery - the canonical discovery API. Unlike a directory
+  /// walk it sees items whose contents have not been downloaded yet, and
+  /// running it nudges the metadata sync on a fresh install. Returns
+  /// [{ rel, size, downloaded }] with `rel` relative to the given directory.
+  @objc
+  func listCloudFiles(_ dir: String,
+                      timeoutMs: NSNumber,
+                      resolver: @escaping RCTPromiseResolveBlock,
+                      rejecter: @escaping RCTPromiseRejectBlock) {
+    DispatchQueue.main.async {
+      // Standardize through the /private symlink so prefix comparison cannot
+      // silently drop every result (/var vs /private/var).
+      let resolvedBase = URL(fileURLWithPath: Self.plainPath(dir))
+        .resolvingSymlinksInPath().path
+      let basePath = resolvedBase.hasSuffix("/") ? resolvedBase : resolvedBase + "/"
+      let query = NSMetadataQuery()
+      query.searchScopes = [
+        NSMetadataQueryUbiquitousDocumentsScope,
+        NSMetadataQueryUbiquitousDataScope,
+      ]
+      query.predicate = NSPredicate(format: "%K LIKE '*'", NSMetadataItemFSNameKey)
+
+      var finished = false
+      var observer: NSObjectProtocol?
+      func complete() {
+        guard !finished else { return }
+        finished = true
+        query.disableUpdates()
+        query.stop()
+        if let obs = observer { NotificationCenter.default.removeObserver(obs) }
+        var out: [[String: Any]] = []
+        for case let item as NSMetadataItem in query.results {
+          guard let rawPath = item.value(forAttribute: NSMetadataItemPathKey) as? String
+          else { continue }
+          let itemPath = URL(fileURLWithPath: rawPath).resolvingSymlinksInPath().path
+          guard itemPath.hasPrefix(basePath) else { continue }
+          // Cloud-only directories have no on-disk presence; classify by the
+          // metadata content type, falling back to the filesystem.
+          if let contentType = item.value(
+            forAttribute: NSMetadataItemContentTypeKey) as? String,
+            contentType == "public.folder" { continue }
+          var isDir: ObjCBool = false
+          if FileManager.default.fileExists(atPath: itemPath, isDirectory: &isDir),
+             isDir.boolValue { continue }
+          let rel = String(itemPath.dropFirst(basePath.count))
+          let size = (item.value(forAttribute: NSMetadataItemFSSizeKey) as? NSNumber)?.intValue ?? 0
+          let status = item.value(
+            forAttribute: NSMetadataUbiquitousItemDownloadingStatusKey) as? String
+          let downloaded = status == NSMetadataUbiquitousItemDownloadingStatusCurrent
+            || status == NSMetadataUbiquitousItemDownloadingStatusDownloaded
+          // size/downloaded are surfaced for diagnostics; JS keys off rel.
+          out.append(["rel": rel, "size": size, "downloaded": downloaded])
+        }
+        resolver(out)
+      }
+
+      observer = NotificationCenter.default.addObserver(
+        forName: .NSMetadataQueryDidFinishGathering, object: query, queue: .main
+      ) { _ in complete() }
+      DispatchQueue.main.asyncAfter(
+        deadline: .now() + timeoutMs.doubleValue / 1000.0
+      ) { complete() }
+      query.start()
     }
   }
 }

ios/OpenNotes/Info.plist

@@ -54,7 +54,7 @@
 		</dict>
 	</array>
 	<key>CFBundleVersion</key>
-	<string>10</string>
+	<string>11</string>
 	<key>ITSAppUsesNonExemptEncryption</key>
 	<false/>
 	<key>LSMinimumSystemVersion</key>

scripts/backupEngine.test.mjs

@@ -6,10 +6,13 @@ import {
   BACKUP_SUBDIR,
   checkRestoreAvailable,
   isSafeRelPath,
+  mergeBackupCatalog,
   noteIdForRel,
   restoreFromBackup,
+  resumeRestoreIfIncomplete,
   syncBackup,
 } from '../src/services/backupEngine.ts';
+import { RECOVERED_NOTE_TITLE } from '../src/services/catalogStore.ts';
 
 const CONTAINER = '/icloud/Documents';
 const BACKUP_DIR = `${CONTAINER}/${BACKUP_SUBDIR}`;
@@ -424,7 +427,9 @@ test('restore copies everything, installs the catalog, and round-trips a full ba
   const result = await restoreFromBackup(fresh);
   assert.deepEqual(result, { status: 'ok', restored: 3 });
   assert.equal(fresh.localFiles.get('notebook-bodies/note-a.body').contents, 'body-a');
-  assert.equal(fresh.localCatalog, catalogRaw(['note-a']));
+  const restoredCatalog = JSON.parse(fresh.localCatalog);
+  assert.deepEqual(restoredCatalog.notes.map((n) => n.id), ['note-a']);
+  assert.equal(restoredCatalog.notes[0].title, 'Title note-a');
 });
 
 test('EDGE: restore never overwrites files that already exist locally', async () => {
@@ -440,7 +445,7 @@ test('EDGE: restore never overwrites files that already exist locally', async ()
   assert.equal(env.localFiles.get('notebook-bodies/note-a.body').contents, 'local-version');
 });
 
-test('EDGE: restore never overwrites a non-empty local catalog', async () => {
+test('EDGE: restore merges into a non-empty local catalog without clobbering local notes', async () => {
   const env = makeEnv({
     localCatalog: catalogRaw(['note-local']),
     backup: {
@@ -449,7 +454,10 @@ test('EDGE: restore never overwrites a non-empty local catalog', async () => {
     },
   });
   await restoreFromBackup(env);
-  assert.equal(env.localCatalog, catalogRaw(['note-local']));
+  const merged = JSON.parse(env.localCatalog);
+  const ids = merged.notes.map((n) => n.id).sort();
+  assert.deepEqual(ids, ['note-backup', 'note-local']);
+  assert.equal(merged.notes.find((n) => n.id === 'note-local').title, 'Title note-local');
 });
 
 test('EDGE: evicted (undownloadable) files are counted as failures, rest still restores', async () => {
@@ -479,6 +487,111 @@ test('EDGE: restore with corrupt backup catalog still restores body files', asyn
   assert.equal(env.localCatalog, null);
 });
 
+test('mergeBackupCatalog: stubs healed, user notes kept, backup-only added, tombstones respected', () => {
+  const local = JSON.parse(catalogRaw(['note-user', 'note-stub']));
+  local.notes[1].title = RECOVERED_NOTE_TITLE;
+  local.deletedNoteIds = { 'note-deleted': '2026-08-30T00:00:00.000Z' };
+  const backup = JSON.parse(catalogRaw(['note-stub', 'note-cloud-only', 'note-deleted']));
+  const merged = mergeBackupCatalog(JSON.stringify(local), JSON.stringify(backup));
+  const titles = Object.fromEntries(merged.notes.map((n) => [n.id, n.title]));
+  assert.equal(titles['note-user'], 'Title note-user');
+  assert.equal(titles['note-stub'], 'Title note-stub');
+  assert.equal(titles['note-cloud-only'], 'Title note-cloud-only');
+  assert.equal('note-deleted' in titles, false);
+  assert.deepEqual(Object.keys(merged.deletedNoteIds), ['note-deleted']);
+});
+
+test('DEVICE REPRO: partial restore (cloud metadata race) resumes to completion on later launches', async () => {
+  // Fresh install on a real device: catalog synced fast, bodies were still
+  // cloud-only. The restore installed titles but no content. The resume pass
+  // must detect notes-without-bodies and pull them once downloadable.
+  const backup = {
+    [`${BACKUP_DIR}/${BACKUP_CATALOG_NAME}`]: catalogRaw(['note-a', 'note-b']),
+    [`${BACKUP_DIR}/${BACKUP_MANIFEST_NAME}`]: JSON.stringify({
+      version: 1,
+      files: {
+        'notebook-bodies/note-a.body': { size: 1, mtimeMs: 1 },
+        'notebook-bodies/note-b.body': { size: 1, mtimeMs: 1 },
+      },
+      lastBackupAt: 5,
+      noteCount: 2,
+    }),
+    [`${BACKUP_DIR}/notebook-bodies/note-a.body`]: 'body-a',
+    [`${BACKUP_DIR}/notebook-bodies/note-b.body`]: 'body-b',
+  };
+  const undownloadable = new Set([
+    'notebook-bodies/note-a.body',
+    'notebook-bodies/note-b.body',
+  ]);
+  const env = makeEnv({ backup, undownloadableRels: undownloadable });
+
+  const first = await restoreFromBackup(env);
+  assert.equal(first.status, 'partial');
+  assert.equal(first.restored, 0);
+  // Titles arrived via catalog merge, content did not - the reported state.
+  assert.equal(JSON.parse(env.localCatalog).notes.length, 2);
+  assert.equal(env.localFiles.size, 0);
+
+  // Next launch, iCloud metadata has synced: resume completes the restore.
+  undownloadable.clear();
+  const resumed = await resumeRestoreIfIncomplete(env);
+  assert.equal(resumed.status, 'ok');
+  assert.equal(resumed.restored, 2);
+  assert.equal(env.localFiles.get('notebook-bodies/note-a.body').contents, 'body-a');
+
+  // Fully healed: nothing further to resume.
+  assert.equal(await resumeRestoreIfIncomplete(env), null);
+});
+
+test('EDGE: resume does nothing for an empty library (prompt path owns that) or complete one', async () => {
+  const empty = makeEnv({
+    backup: { [`${BACKUP_DIR}/notebook-bodies/note-a.body`]: 'a' },
+  });
+  assert.equal(await resumeRestoreIfIncomplete(empty), null);
+
+  const complete = makeEnv({
+    local: { 'notebook-bodies/note-a.body': localFile('a') },
+    localCatalog: catalogRaw(['note-a']),
+    backup: { [`${BACKUP_DIR}/notebook-bodies/note-a.body`]: 'a' },
+  });
+  assert.equal(await resumeRestoreIfIncomplete(complete), null);
+});
+
+test('EDGE: a manifest-only rel (file gone from iCloud) never causes an endless resume loop', async () => {
+  const env = makeEnv({
+    local: { 'notebook-bodies/note-a.body': localFile('a') },
+    localCatalog: catalogRaw(['note-a', 'note-gone']),
+    backup: {
+      [`${BACKUP_DIR}/${BACKUP_MANIFEST_NAME}`]: JSON.stringify({
+        version: 1,
+        files: { 'notebook-bodies/note-gone.body': { size: 1, mtimeMs: 1 } },
+        lastBackupAt: 1,
+        noteCount: 2,
+      }),
+    },
+  });
+  // The file exists only in the stale manifest, not in any listing: gone.
+  assert.equal(await resumeRestoreIfIncomplete(env), null);
+});
+
+test('EDGE: stub titles heal from the backup catalog even when all bodies are present', async () => {
+  // The device race running the other way: bodies synced first, catalog was
+  // cloud-only during the first restore, reconciliation created stubs.
+  const local = JSON.parse(catalogRaw(['note-a']));
+  local.notes[0].title = RECOVERED_NOTE_TITLE;
+  const env = makeEnv({
+    local: { 'notebook-bodies/note-a.body': localFile('a') },
+    localCatalog: JSON.stringify(local),
+    backup: {
+      [`${BACKUP_DIR}/${BACKUP_CATALOG_NAME}`]: catalogRaw(['note-a']),
+      [`${BACKUP_DIR}/notebook-bodies/note-a.body`]: 'a',
+    },
+  });
+  const result = await resumeRestoreIfIncomplete(env);
+  assert.equal(result.status, 'ok');
+  assert.equal(JSON.parse(env.localCatalog).notes[0].title, 'Title note-a');
+});
+
 test('EDGE: an env that throws unexpectedly yields partial, not a crash', async () => {
   const env = makeEnv({ local: { 'notebook-bodies/a.body': localFile('a') } });
   env.listLocalDataFiles = async () => {

src/hooks/useBackup.ts

@@ -6,6 +6,7 @@ import {
   isBackupAvailable,
   isBackupEnabled,
   performBackupRestore,
+  resumeBackupRestoreIfIncomplete,
   scheduleBackupSync,
   setBackupEnabled,
 } from '../services/backupService';
@@ -65,6 +66,14 @@ export function useBackup(
     if (restorePromptShownRef.current) return;
     const availability = await checkBackupRestoreAvailable();
     if (!availability || restorePromptShownRef.current) {
+      // Heal any interrupted restore first: notes whose titles arrived but
+      // whose content files are still only in iCloud (a fresh install can
+      // race the metadata sync). Copies only what is missing; silent.
+      const resumed = await resumeBackupRestoreIfIncomplete();
+      if (resumed && resumed.status !== 'unavailable' && resumed.restored > 0) {
+        await catalogStore.invalidate();
+        await refreshLibrary();
+      }
       // No restore pending: safe to start the catch-up push. It retries a
       // backup that failed last session and mirrors pre-existing data after
       // an app update; a no-op copy-wise when the mirror is current. It is