ci(android): Run UI tests incl. replay on API 30 in AGP matrix, drop Sauce Labs UI job (#6492)

* ci(android): Run UI tests incl. replay on API 35 in AGP matrix, drop Sauce Labs UI job

Co-Authored-By: Roman Zavarnitsyn <roman.zavarnitsyn@sentry.io>

* ci(android): Pull replay snapshots even when the AGP matrix UI tests fail

android-emulator-runner runs each script line in its own shell and stops at
the first failing line, so a failing connectedReleaseAndroidTest skipped the
snapshot pull. Run the tests and the pull from one script that exits with the
tests' status.

#skip-changelog

* ci(android): Drop --daemon from the AGP matrix UI test run

Co-Authored-By: Roman Zavarnitsyn <roman.zavarnitsyn@sentry.io>

#skip-changelog

* ci(android): Run the AGP matrix UI tests on API 30 google_apis

API 30 google_apis captures replay frames with PixelCopy (verified in
#6495) and avoids the Android 15 ART method-tracing crash that the
profiling tests hit on API 35.

Co-Authored-By: Roman Zavarnitsyn <roman.zavarnitsyn@sentry.io>

* test(profiling): Run checkTimedOutProfile on emulators with a shorter timeout

The emulator skip was added in #3074 because the timed-out profile's
trace file was sometimes missing on emulators. Since #3478 timed-out
profiles are discarded and the test asserts no profile is sent, so it
no longer reads a trace file. Drop the skip, and add a test-only hook
to shorten the 30s profiling timeout so the test waits 5s instead of 35s.

Co-Authored-By: Roman Zavarnitsyn <roman.zavarnitsyn@sentry.io>

* test(uitest): Remove checkTimedOutProfile

The test only checks that a timed-out transaction profile is discarded,
which AndroidProfilerTest and AndroidTransactionProfilerTest already cover.
It was skipped on emulators, so nothing in CI ran it after the Sauce Labs
job was removed. Also reverts the test-only profiling timeout hook.

Co-Authored-By: Roman Zavarnitsyn <roman.zavarnitsyn@sentry.io>

---------

Co-authored-by: sentry-junior[bot] <264270552+sentry-junior[bot]@users.noreply.github.com>
Co-authored-by: Roman Zavarnitsyn <roman.zavarnitsyn@sentry.io>

Author: sentry-junior[bot]

.github/workflows/agp-matrix.yml

@@ -24,6 +24,8 @@ jobs:
     env:
       VERSION_AGP: ${{ matrix.agp }}
       APPLY_SENTRY_INTEGRATIONS: ${{ matrix.integrations }}
+      # Upload replay snapshots from one cell only. Forks don't have access to the upload secret.
+      UPLOAD_REPLAY_SNAPSHOTS: ${{ matrix.agp == '9.4.0' && matrix.integrations && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }}
 
     steps:
       - name: Checkout Repo
@@ -51,20 +53,21 @@ jobs:
           path: |
             ~/.android/avd/*
             ~/.android/adb*
-          key: avd-api-30-x86_64-aosp_atd
+          key: avd-api-30-x86_64-google_apis
 
       - name: Create AVD and generate snapshot for caching
         if: steps.avd-cache.outputs.cache-hit != 'true'
         uses: reactivecircus/android-emulator-runner@a421e43855164a8197daf9d8d40fe71c6996bb0d # pin@v2
         with:
           api-level: 30
-          target: aosp_atd
-          channel: canary # Necessary for ATDs
+          # Not an ATD image: PixelCopy returns no data on ATDs, which the replay tests need
+          target: google_apis
+          channel: canary
           arch: x86_64
           force-avd-creation: false
           disable-animations: true
           disable-spellchecker: true
-          emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
+          emulator-options: -memory 4096 -no-window -gpu auto -noaudio -no-boot-anim -camera-back none
           disk-size: 4096M
           script: echo "Generated AVD snapshot for caching."
 
@@ -77,15 +80,28 @@ jobs:
         uses: reactivecircus/android-emulator-runner@a421e43855164a8197daf9d8d40fe71c6996bb0d # pin@v2
         with:
           api-level: 30
-          target: aosp_atd
-          channel: canary # Necessary for ATDs
+          # Not an ATD image: PixelCopy returns no data on ATDs, which the replay tests need
+          target: google_apis
+          channel: canary
           arch: x86_64
           force-avd-creation: false
           disable-animations: true
           disable-spellchecker: true
-          emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
+          emulator-options: -memory 4096 -no-snapshot-save -no-window -gpu auto -noaudio -no-boot-anim -camera-back none
           disk-size: 4096M
-          script: ./gradlew sentry-android-integration-tests:sentry-uitest-android:connectedReleaseAndroidTest -Denvironment=github --daemon
+          script: ./scripts/test-ui-android.sh replay-snapshots
+
+      - name: Install Sentry CLI
+        if: ${{ !cancelled() && env.UPLOAD_REPLAY_SNAPSHOTS == 'true' }}
+        uses: getsentry/action-setup-cli@70d7e587b84c2e78cf4d37cd33d7b74fb3729c1b # v1
+
+      - name: Upload Replay Snapshots to Sentry
+        if: ${{ !cancelled() && env.UPLOAD_REPLAY_SNAPSHOTS == 'true' && hashFiles('replay-snapshots/*.png') != '' }}
+        run: sentry-cli snapshots upload ./replay-snapshots --app-id sentry-android-replay
+        env:
+          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+          SENTRY_ORG: sentry-sdks
+          SENTRY_PROJECT: sentry-android
 
       - name: Upload test results
         if: always()
@@ -96,6 +112,7 @@ jobs:
             **/build/reports/*
             **/build/outputs/*/connected/*
             **/build/outputs/mapping/release/*
+            replay-snapshots/
 
       - name: Test Report
         uses: phoenix-actions/test-reporting@7317eea6e13c47348dd0bb318669485157c518d6 # pin@v16

.github/workflows/integration-tests-ui.yml

@@ -0,0 +1,21 @@
+#!/bin/sh
+# Runs the sentry-uitest-android instrumentation tests on the connected device, then pulls the
+# replay frame snapshots that ReplaySnapshotTest writes, also when the tests fail.
+# android-emulator-runner runs each `script:` line in its own shell, so this stays one script to
+# keep the tests' exit code. Keep the device directory in sync with ReplaySnapshotTest.
+set -eu
+
+out_dir=$1
+device_dir=/sdcard/Download/sentry_replay_snapshots
+
+test_status=0
+./gradlew sentry-android-integration-tests:sentry-uitest-android:connectedReleaseAndroidTest || test_status=$?
+
+mkdir -p "$out_dir"
+if adb shell test -d "$device_dir"; then
+  adb pull "$device_dir/." "$out_dir"
+else
+  printf 'No replay snapshots on the device, skipping\n'
+fi
+
+exit "$test_status"

.sauce/sentry-uitest-android-ui.yml

@@ -7,13 +7,9 @@ By default the envelopes sent to relay are caught by a mock server which allows
 
 Simply run `./gradlew connectedCheck` to run all ui tests of all modules (requires a connected device, either physical or an emulator).
 
-# SauceLabs
-To run on saucelabs execute following commands (need also `SAUCE_USERNAME` and `SAUCE_ACCESS_KEY` environment variables):
-For End 2 End:
-```
-./gradlew :sentry-android-integration-tests:sentry-uitest-android:assembleRelease :sentry-android-integration-tests:sentry-uitest-android:assembleAndroidTest
-saucectl run -c .sauce/sentry-uitest-android-end2end.yml
-```
+# CI
+CI runs these tests on a GitHub Actions emulator in the AGP matrix workflow (`.github/workflows/agp-matrix.yml`).
+The emulator must not be an ATD image: PixelCopy returns no data there, and the replay tests need it.
 
 # Troubleshooting
 

scripts/test-ui-android.sh

@@ -30,7 +30,6 @@ android {
     // This doesn't work on some devices with Android 11+. Clearing package data resets permissions.
     // Check the readme for more info.
     testInstrumentationRunnerArguments["clearPackageData"] = "true"
-    buildConfigField("String", "ENVIRONMENT", "\"${System.getProperty("environment", "")}\"")
   }
 
   testOptions { execution = "ANDROIDX_TEST_ORCHESTRATOR" }
@@ -40,7 +39,6 @@ android {
     // Note that the viewBinding.enabled property is now deprecated.
     viewBinding = true
     compose = true
-    buildConfig = true
   }
 
   composeOptions { kotlinCompilerExtensionVersion = libs.versions.composeCompiler.get() }

sentry-android-integration-tests/sentry-uitest-android/README.md

@@ -125,6 +125,9 @@ fun waitUntilIdle() {
   Espresso.onIdle()
 }
 
+/** Whether sentry-android-replay is on the classpath (APPLY_SENTRY_INTEGRATIONS=true). */
+fun isReplayAvailable(): Boolean = classExists("io.sentry.android.replay.ReplayIntegration")
+
 fun classExists(className: String): Boolean {
   try {
     Class.forName(className)

sentry-android-integration-tests/sentry-uitest-android/build.gradle.kts

@@ -3,17 +3,14 @@ package io.sentry.uitest.android
 import androidx.lifecycle.Lifecycle
 import androidx.test.core.app.launchActivity
 import androidx.test.espresso.Espresso
-import androidx.test.espresso.IdlingPolicies
 import androidx.test.espresso.IdlingRegistry
 import androidx.test.espresso.action.ViewActions
 import androidx.test.espresso.matcher.ViewMatchers
 import androidx.test.ext.junit.runners.AndroidJUnit4
-import io.sentry.NoOpLogger
 import io.sentry.ProfilingTraceData
 import io.sentry.Sentry
 import io.sentry.SentryEvent
 import io.sentry.android.core.AndroidLogger
-import io.sentry.android.core.BuildInfoProvider
 import io.sentry.android.core.SentryAndroidOptions
 import io.sentry.assertEnvelopeProfile
 import io.sentry.assertEnvelopeTransaction
@@ -25,7 +22,6 @@ import kotlin.test.Test
 import kotlin.test.assertEquals
 import kotlin.test.assertNotNull
 import kotlin.test.assertTrue
-import org.junit.Assume
 import org.junit.Assume.assumeNotNull
 import org.junit.runner.RunWith
 
@@ -196,44 +192,6 @@ class EnvelopeTests : BaseUiTest() {
     }
   }
 
-  @Test
-  fun checkTimedOutProfile() {
-    Assume.assumeFalse(
-      "Sometimes traceFile does not exist for profiles on emulators and it fails." +
-        " Although, Android Runtime is the one that manages the file, so something" +
-        " must be wrong with Debug.startMethodTracing",
-      BuildInfoProvider(NoOpLogger.getInstance()).isEmulator ?: true,
-    )
-    // We increase the IdlingResources timeout to exceed the profiling timeout
-    IdlingPolicies.setIdlingResourceTimeout(1, TimeUnit.MINUTES)
-    initSentry(true) { options: SentryAndroidOptions ->
-      options.tracesSampleRate = 1.0
-      options.profilesSampleRate = 1.0
-    }
-    relayIdlingResource.increment()
-    val transaction = Sentry.startTransaction("timedOutProfile", "testTimeout")
-    Thread {
-        Thread.sleep(35_000)
-        transaction.finish()
-      }
-      .start()
-    // We call transaction.finish() after 35 seconds, and check profile times out after 30 seconds
-
-    relay.assert {
-      findEnvelope {
-          assertEnvelopeTransaction(it.items.toList(), AndroidLogger()).transaction ==
-            "timedOutProfile"
-        }
-        .assert {
-          val transactionItem: SentryTransaction = it.assertTransaction()
-          // Profile should not be present, as it timed out and is discarded
-          it.assertNoOtherItems()
-          assertEquals("timedOutProfile", transactionItem.transaction)
-        }
-      assertNoOtherEnvelopes()
-    }
-  }
-
   @Test
   fun sendProfiledTransaction() {
     // This is a dogfooding test

sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/BaseUiTest.kt

@@ -18,9 +18,7 @@ import shark.ReferencePattern
 class ReplayTest : BaseUiTest() {
   @Before
   fun setup() {
-    // we can't run on GH actions emulator, because they don't allow capturing screenshots properly
-    @Suppress("KotlinConstantConditions")
-    assumeThat(BuildConfig.ENVIRONMENT != "github", `is`(true))
+    assumeThat(isReplayAvailable(), `is`(true))
     // crash on swallowed Compose masking errors (e.g. broken obfuscated internals) so regressions
     // fail this on-device test instead of silently under-masking (see SentryReplayDebug)
     System.setProperty("io.sentry.replay.compose.fail-fast", "true")

sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/EnvelopeTests.kt

@@ -464,8 +464,7 @@ class UserFeedbackUiTest : BaseUiTest() {
 
   @Test
   fun userFeedbackSendEnvelope() {
-    // GH actions emulator don't allow capturing screenshots properly
-    val enableReplay = BuildConfig.ENVIRONMENT != "github"
+    val enableReplay = isReplayAvailable()
     initSentry(true) {
       // When sending the feedback we want to wait for relay to receive it.
       // We can't simply increment the idling resource,