test: add retry for failed tests

Author: kacperzolkiewski

.maestro/scripts/run-tests.sh

@@ -21,7 +21,9 @@
 #   Android → --driver devicelab
 #   iOS     → default driver
 #
-# In CI on Android, driver startup crashes are retried twice (3 runs total).
+# Failed test runs are retried in CI only (iOS and Android). Override with
+# MAESTRO_MAX_RETRIES (default: 1 locally, 2 in CI). DeviceLab driver startup
+# crashes on CI Android get extra cleanup between attempts.
 
 set -euo pipefail
 
@@ -143,8 +145,12 @@ set_font_scale default
 # maestro-runner exits non-zero when the tag filter matches zero flows. That's
 # not a real failure for us (e.g. running a single flow that has no
 # accessibility variant), and letting it propagate aborts later test suites.
+is_ci() {
+  [ -n "${CI:-}" ] || [ -n "${GITHUB_ACTIONS:-}" ]
+}
+
 is_ci_android() {
-  [ "$PLATFORM" = android ] && { [ -n "${CI:-}" ] || [ -n "${GITHUB_ACTIONS:-}" ]; }
+  [ "$PLATFORM" = android ] && is_ci
 }
 
 is_driver_startup_crash() {
@@ -165,8 +171,8 @@ run_maestro() {
   local max_attempts rc tmp attempt=1 delayed_retry=0
   if [ -n "${MAESTRO_MAX_RETRIES:-}" ]; then
     max_attempts="$MAESTRO_MAX_RETRIES"
-  elif is_ci_android; then
-    max_attempts=3 # 1 initial run + 2 retries
+  elif is_ci; then
+    max_attempts=2 # 1 initial run + 1 retry
   else
     max_attempts=1
   fi
@@ -214,6 +220,15 @@ run_maestro() {
         rm -f "$tmp"
         continue
       fi
+    elif is_ci && [ "$attempt" -lt "$max_attempts" ]; then
+      echo "warn: test run failed (attempt $attempt/$max_attempts), retrying in 5s..." >&2
+      if [ "$PLATFORM" = android ]; then
+        cleanup_devicelab_driver
+      fi
+      sleep 5
+      attempt=$((attempt + 1))
+      rm -f "$tmp"
+      continue
     fi
 
     rm -f "$tmp"