fix(test): cold-boot Android emulator on ANR and missing package/activity services

Author: mikehardy

packages/app/__tests__/androidAdbRange.test.js

@@ -11,6 +11,7 @@ const {
   adbRangeDiagnosis,
   isMetroWaitFailure,
   shouldColdBootAndroidOnLaunchRetry,
+  errorProbeText,
 } = require('../../../tests/e2e/androidAdbRange');
 
 const repoRoot = path.resolve(__dirname, '../../..');
@@ -146,6 +147,9 @@ describe('shouldColdBootAndroidOnLaunchRetry', function () {
     expect(shouldColdBootAndroidOnLaunchRetry(new Error("cmd: Can't find service: activity"))).toBe(
       true,
     );
+    expect(shouldColdBootAndroidOnLaunchRetry(new Error("cmd: Can't find service: settings"))).toBe(
+      true,
+    );
     expect(
       shouldColdBootAndroidOnLaunchRetry(
         new Error("APP_UNREACHABLE Detox can't seem to connect to the test app(s)!"),
@@ -156,6 +160,38 @@ describe('shouldColdBootAndroidOnLaunchRetry', function () {
     ).toBe(true);
   });
 
+  it('cold-boots when ANR is only on jetOutput (Jet session desync path)', function () {
+    const err = new Error('Jet tests failed!');
+    err.jetOutput =
+      'com.github.anrwatchdog.ANRError: Application Not Responding for at least 5000 ms.';
+    expect(shouldColdBootAndroidOnLaunchRetry(err)).toBe(true);
+    // Full device-side probe must not see jetOutput (product titles can say "offline").
+    expect(errorProbeText(err)).toBe('Jet tests failed!');
+  });
+
+  it("cold-boots when Can't find service is only on Detox stderr (exit code 20)", function () {
+    const err = new Error('Process exited with code 20');
+    err.stderr = "cmd: Can't find service: package";
+    expect(shouldColdBootAndroidOnLaunchRetry(err)).toBe(true);
+    expect(errorProbeText(err)).toMatch(/Can't find service: package/);
+  });
+
+  it('does not cold-boot when jetOutput only contains a Mocha title with offline', function () {
+    const err = new Error('Jet tests failed!');
+    err.jetOutput =
+      '  1) database()\n       Reference\n         removes a node whilst offline:\n     Error: expected true to equal false';
+    expect(shouldColdBootAndroidOnLaunchRetry(err)).toBe(false);
+    expect(errorProbeText(err)).toBe('Jet tests failed!');
+  });
+
+  it('cold-boots when launch timeout is enriched with logcat ANR signal', function () {
+    expect(
+      shouldColdBootAndroidOnLaunchRetry(
+        new Error('[rnfb-e2e] launchApp timed out after 300000ms\nANR detected in recent logcat'),
+      ),
+    ).toBe(true);
+  });
+
   it('does not cold-boot for Jet session noise that is not device-side', function () {
     expect(shouldColdBootAndroidOnLaunchRetry(new Error('Jet WS closed 1006'))).toBe(false);
     expect(shouldColdBootAndroidOnLaunchRetry(new Error('launchApp timed out'))).toBe(false);

tests/e2e/androidAdbRange.js

@@ -56,8 +56,14 @@ const METRO_WAIT_FAILURE_RE = /Metro not responding|Metro bundle not available|p
 // Android cold-boot is only for device-side launch/health faults (ANR, offline, qemu/adb,
 // package-manager / ActivityManager collapse, install SIGTERM under load, unreachable app).
 // Metro wait failures are excluded via isMetroWaitFailure before this regex runs.
+// Includes Detox SPAWN_STDERR / Jet output phrasing: "Can't find service: package|activity|settings".
 const ANDROID_DEVICE_SIDE_LAUNCH_RE =
-  /ANR|\boffline\b|qemu-without-adb|did not become 'device'|cold-boot spawn did not register|unknown to FrontBoard|FBSOpenApplicationServiceErrorDomain|Can't find service: package|Can't find service: activity|terminated with SIGTERM|Process was killed with SIGTERM|APP_UNREACHABLE|Failed to run application/i;
+  /ANR|\boffline\b|qemu-without-adb|did not become 'device'|cold-boot spawn did not register|unknown to FrontBoard|FBSOpenApplicationServiceErrorDomain|Can't find service: (?:package|activity|settings)|terminated with SIGTERM|Process was killed with SIGTERM|APP_UNREACHABLE|Failed to run application/i;
+
+// jetOutput is Mocha/Jet log text (product titles like "whilst offline"). Only guest-death
+// phrases — never the full device-side token set (\boffline\b, SIGTERM, APP_UNREACHABLE, …).
+const ANDROID_JET_OUTPUT_GUEST_DEATH_RE =
+  /ANRError|Application Not Responding|Can't find service: (?:package|activity|settings)/i;
 
 function errorMessage(messageOrErr) {
   if (typeof messageOrErr === 'string') {
@@ -66,18 +72,44 @@ function errorMessage(messageOrErr) {
   return messageOrErr?.message || '';
 }
 
+// Detox often surfaces only "Process exited with code N" in err.message while the
+// actionable "Can't find service: …" lives on err.stderr / stdout. Full device-side
+// regex applies here only — not to jetOutput (see ANDROID_JET_OUTPUT_GUEST_DEATH_RE).
+function errorProbeText(messageOrErr) {
+  if (typeof messageOrErr === 'string') {
+    return messageOrErr;
+  }
+  if (!messageOrErr || typeof messageOrErr !== 'object') {
+    return '';
+  }
+  return [messageOrErr.message, messageOrErr.stderr, messageOrErr.stdout]
+    .filter(part => part != null && String(part).length > 0)
+    .map(String)
+    .join('\n');
+}
+
+function errorJetOutputText(messageOrErr) {
+  if (typeof messageOrErr === 'string' || !messageOrErr || typeof messageOrErr !== 'object') {
+    return '';
+  }
+  return messageOrErr.jetOutput != null ? String(messageOrErr.jetOutput) : '';
+}
+
 function isMetroWaitFailure(messageOrErr) {
   return METRO_WAIT_FAILURE_RE.test(errorMessage(messageOrErr));
 }
 
 // True only when an Android Jet retry should kill+relaunch qemu. Metro wait timeouts
 // must leave the emulator alone and re-wait for packager-status:running / bundle.
 function shouldColdBootAndroidOnLaunchRetry(messageOrErr) {
-  const message = errorMessage(messageOrErr);
-  if (isMetroWaitFailure(message)) {
+  // Metro exclusion stays message-primary so a polluted jetOutput cannot force a cold boot.
+  if (isMetroWaitFailure(messageOrErr)) {
     return false;
   }
-  return ANDROID_DEVICE_SIDE_LAUNCH_RE.test(message);
+  if (ANDROID_DEVICE_SIDE_LAUNCH_RE.test(errorProbeText(messageOrErr))) {
+    return true;
+  }
+  return ANDROID_JET_OUTPUT_GUEST_DEATH_RE.test(errorJetOutputText(messageOrErr));
 }
 
 function adbRangeDiagnosis(serial, avdName) {
@@ -116,6 +148,8 @@ module.exports = {
   adbRangeDiagnosis,
   isMetroWaitFailure,
   shouldColdBootAndroidOnLaunchRetry,
+  errorProbeText,
   METRO_WAIT_FAILURE_RE,
   ANDROID_DEVICE_SIDE_LAUNCH_RE,
+  ANDROID_JET_OUTPUT_GUEST_DEATH_RE,
 };

tests/e2e/firebase.test.js

@@ -531,6 +531,89 @@ function adbShell(serial, command, timeoutMs = 15000) {
   }).trim();
 }
 
+// After ANR / system_server death, boot_completed can still be 1 while package /
+// activity binders are gone — Detox then fails pm install with "Can't find service".
+function androidCoreServicesPresent(serial) {
+  try {
+    const list = adbShell(serial, 'service list', 15000);
+    return {
+      hasPackage: /\bpackage:/.test(list),
+      hasActivity: /\bactivity:/.test(list),
+      list,
+    };
+  } catch (err) {
+    return {
+      hasPackage: false,
+      hasActivity: false,
+      list: '',
+      error: err?.message || String(err),
+    };
+  }
+}
+
+// promisify-child-process leaves "Can't find service: …" on err.stderr while
+// err.message is only "Process exited with code N". Fold so cold-boot matching
+// and retry logs see the device-side fault without reading Detox SPAWN_STDERR alone.
+function foldDetoxChildProcessOutputIntoError(err) {
+  if (!err || typeof err !== 'object') {
+    return err;
+  }
+  const extras = [err.stderr, err.stdout]
+    .filter(part => part != null && String(part).trim().length > 0)
+    .map(part => String(part).trim());
+  if (extras.length === 0) {
+    return err;
+  }
+  const folded = extras.join('\n');
+  const current = String(err.message || '');
+  if (current.includes(folded) || folded.split('\n').every(line => current.includes(line))) {
+    return err;
+  }
+  err.message = current ? `${current}\n${folded}` : folded;
+  return err;
+}
+
+// launchApp timeouts / Jet desync often leave ANR only in Detox console + logcat,
+// or leave boot_completed=1 while package/activity binders are already gone.
+// Best-effort append so shouldColdBootAndroidOnLaunchRetry sees device-side faults.
+function enrichAndroidLaunchFailureWithGuestSignals(err) {
+  if (!err || typeof err !== 'object') {
+    return err;
+  }
+  if (shouldColdBootAndroidOnLaunchRetry(err)) {
+    return err;
+  }
+  try {
+    const serial = resolveAndroidSerial();
+    const services = androidCoreServicesPresent(serial);
+    if (!services.hasPackage || !services.hasActivity) {
+      const missing = [
+        !services.hasPackage ? "Can't find service: package" : null,
+        !services.hasActivity ? "Can't find service: activity" : null,
+      ]
+        .filter(Boolean)
+        .join('\n');
+      err.message = `${err.message || ''}\n${missing}`.trim();
+      console.warn(
+        `[rnfb-e2e] android guest-signal: core services missing package=${services.hasPackage} activity=${services.hasActivity}`,
+      );
+      return err;
+    }
+    const adb = resolveAdbPath();
+    const logTail = execSync(`${adb} -s ${serial} logcat -d -t 400`, {
+      encoding: 'utf8',
+      timeout: 15000,
+    });
+    if (/ANRError|Application Not Responding|ANR in /i.test(logTail)) {
+      err.message = `${err.message || ''}\nANR detected in recent logcat`.trim();
+      console.warn('[rnfb-e2e] android guest-signal: ANR found in recent logcat');
+    }
+  } catch (probeErr) {
+    console.warn(`[rnfb-e2e] android guest-signal probe failed: ${probeErr?.message || probeErr}`);
+  }
+  return err;
+}
+
 function parseGuestLoad1Min(loadavgLine) {
   const first = loadavgLine.trim().split(/\s+/)[0];
   const load = parseFloat(first);
@@ -827,6 +910,21 @@ async function waitForAndroidEmulatorReady() {
         continue;
       }
 
+      // Gate before wait-for-handler / pm install: binders must be listed.
+      const coreServices = androidCoreServicesPresent(serial);
+      if (!coreServices.hasPackage || !coreServices.hasActivity) {
+        stableLoadPolls = 0;
+        packageHandlerDone = false;
+        console.log(
+          `[rnfb-e2e] android-ready probe core-services package=${coreServices.hasPackage} ` +
+            `activity=${coreServices.hasActivity}` +
+            (coreServices.error ? ` error=${coreServices.error}` : '') +
+            ' (waiting for service list)',
+        );
+        await sleep(ANDROID_READY_POLL_MS);
+        continue;
+      }
+
       if (!packageHandlerDone) {
         console.log('[rnfb-e2e] android-ready probe waiting for package handler queue...');
         try {
@@ -1395,6 +1493,7 @@ async function killJetForLaunchRetry(jetProcess, reason = 'launch-retry') {
 async function launchAppWithRetry(launchArgs, { testsDir, onBeforeRelaunch } = {}) {
   const liveMetro = usesLiveMetro();
   let lastLaunchFailure;
+  let deleteAppAfterColdBoot = false;
 
   for (let launchAttempt = 1; launchAttempt <= LAUNCH_APP_MAX_ATTEMPTS; launchAttempt++) {
     try {
@@ -1411,6 +1510,8 @@ async function launchAppWithRetry(launchArgs, { testsDir, onBeforeRelaunch } = {
           );
           await coldBootAndroidEmulator();
           await waitForAndroidEmulatorReady();
+          // Reinstall after guest death — prior APK/instrumentation may be gone or corrupt.
+          deleteAppAfterColdBoot = true;
         }
         if (onBeforeRelaunch) {
           await onBeforeRelaunch();
@@ -1428,11 +1529,17 @@ async function launchAppWithRetry(launchArgs, { testsDir, onBeforeRelaunch } = {
         await new Promise(resolve => setTimeout(resolve, 2000));
       }
 
+      const deleteApp = launchAttempt === 1 || deleteAppAfterColdBoot;
+      deleteAppAfterColdBoot = false;
       await launchAppWithTimeout(launchArgs, {
-        deleteApp: launchAttempt === 1,
+        deleteApp,
       });
       return;
     } catch (err) {
+      foldDetoxChildProcessOutputIntoError(err);
+      if (detoxPlatformKey() === 'android') {
+        enrichAndroidLaunchFailureWithGuestSignals(err);
+      }
       lastLaunchFailure = err;
       console.warn(`[rnfb-e2e] launchApp failure reason=${err?.message || err}`);
       logLaunchInstallState(`after-launch-failure attempt=${launchAttempt}`);
@@ -1691,6 +1798,8 @@ describe('Jet Tests', function () {
           if (platform === 'ios' && process.platform === 'darwin') {
             await rebootIosSimulator(testsDir);
           } else if (platform === 'android') {
+            foldDetoxChildProcessOutputIntoError(lastFailure);
+            enrichAndroidLaunchFailureWithGuestSignals(lastFailure);
             if (shouldColdBootAndroidOnLaunchRetry(lastFailure)) {
               console.warn('[rnfb-e2e] Android device-side launch failure — cold booting emulator');
               await coldBootAndroidEmulator();
@@ -1717,6 +1826,10 @@ describe('Jet Tests', function () {
         await runJetE2eAttempt(attempt);
         break;
       } catch (err) {
+        foldDetoxChildProcessOutputIntoError(err);
+        if (platform === 'android') {
+          enrichAndroidLaunchFailureWithGuestSignals(err);
+        }
         lastFailure = err;
         logRetryEligibility(err, attempt);
         if (isRetryableCloudQuotaFailure(err?.jetOutput || '')) {