ci(android): Run UI tests incl. replay on API 30 in AGP matrix, drop Sauce Labs UI job

.github/workflows/agp-matrix.yml

@@ -24,6 +24,8 @@ jobs:
     env:
       VERSION_AGP: ${{ matrix.agp }}
       APPLY_SENTRY_INTEGRATIONS: ${{ matrix.integrations }}
+      # Upload replay snapshots from one cell only. Forks don't have access to the upload secret.
+      UPLOAD_REPLAY_SNAPSHOTS: ${{ matrix.agp == '9.4.0' && matrix.integrations && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }}
 
     steps:
       - name: Checkout Repo
@@ -51,20 +53,21 @@ jobs:
           path: |
             ~/.android/avd/*
             ~/.android/adb*
-          key: avd-api-30-x86_64-aosp_atd
+          key: avd-api-30-x86_64-google_apis
 
       - name: Create AVD and generate snapshot for caching
         if: steps.avd-cache.outputs.cache-hit != 'true'
         uses: reactivecircus/android-emulator-runner@a421e43855164a8197daf9d8d40fe71c6996bb0d # pin@v2
         with:
           api-level: 30
-          target: aosp_atd
-          channel: canary # Necessary for ATDs
+          # Not an ATD image: PixelCopy returns no data on ATDs, which the replay tests need
+          target: google_apis
+          channel: canary
           arch: x86_64
           force-avd-creation: false
           disable-animations: true
           disable-spellchecker: true
-          emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
+          emulator-options: -memory 4096 -no-window -gpu auto -noaudio -no-boot-anim -camera-back none
           disk-size: 4096M
           script: echo "Generated AVD snapshot for caching."
 
@@ -77,15 +80,28 @@ jobs:
         uses: reactivecircus/android-emulator-runner@a421e43855164a8197daf9d8d40fe71c6996bb0d # pin@v2
         with:
           api-level: 30
-          target: aosp_atd
-          channel: canary # Necessary for ATDs
+          # Not an ATD image: PixelCopy returns no data on ATDs, which the replay tests need
+          target: google_apis
+          channel: canary
           arch: x86_64
           force-avd-creation: false
           disable-animations: true
           disable-spellchecker: true
-          emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
+          emulator-options: -memory 4096 -no-snapshot-save -no-window -gpu auto -noaudio -no-boot-anim -camera-back none
           disk-size: 4096M
-          script: ./gradlew sentry-android-integration-tests:sentry-uitest-android:connectedReleaseAndroidTest -Denvironment=github --daemon
+          script: ./scripts/test-ui-android.sh replay-snapshots
+
+      - name: Install Sentry CLI
+        if: ${{ !cancelled() && env.UPLOAD_REPLAY_SNAPSHOTS == 'true' }}
+        uses: getsentry/action-setup-cli@70d7e587b84c2e78cf4d37cd33d7b74fb3729c1b # v1
+
+      - name: Upload Replay Snapshots to Sentry
+        if: ${{ !cancelled() && env.UPLOAD_REPLAY_SNAPSHOTS == 'true' && hashFiles('replay-snapshots/*.png') != '' }}
+        run: sentry-cli snapshots upload ./replay-snapshots --app-id sentry-android-replay
+        env:
+          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+          SENTRY_ORG: sentry-sdks
+          SENTRY_PROJECT: sentry-android
 
       - name: Upload test results
         if: always()
@@ -96,6 +112,7 @@ jobs:
             **/build/reports/*
             **/build/outputs/*/connected/*
             **/build/outputs/mapping/release/*
+            replay-snapshots/
 
       - name: Test Report
         uses: phoenix-actions/test-reporting@7317eea6e13c47348dd0bb318669485157c518d6 # pin@v16

scripts/test-ui-android.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+# Runs the sentry-uitest-android instrumentation tests on the connected device, then pulls the
+# replay frame snapshots that ReplaySnapshotTest writes, also when the tests fail.
+# android-emulator-runner runs each `script:` line in its own shell, so this stays one script to
+# keep the tests' exit code. Keep the device directory in sync with ReplaySnapshotTest.
+set -eu
+
+out_dir=$1
+device_dir=/sdcard/Download/sentry_replay_snapshots
+
+test_status=0
+./gradlew sentry-android-integration-tests:sentry-uitest-android:connectedReleaseAndroidTest || test_status=$?
+
+mkdir -p "$out_dir"
+if adb shell test -d "$device_dir"; then
+  adb pull "$device_dir/." "$out_dir"
+else
+  printf 'No replay snapshots on the device, skipping\n'
+fi
+
+exit "$test_status"

sentry-android-integration-tests/sentry-uitest-android/README.md

@@ -7,13 +7,9 @@ By default the envelopes sent to relay are caught by a mock server which allows
 
 Simply run `./gradlew connectedCheck` to run all ui tests of all modules (requires a connected device, either physical or an emulator).
 
-# SauceLabs
-To run on saucelabs execute following commands (need also `SAUCE_USERNAME` and `SAUCE_ACCESS_KEY` environment variables):
-For End 2 End:
-```
-./gradlew :sentry-android-integration-tests:sentry-uitest-android:assembleRelease :sentry-android-integration-tests:sentry-uitest-android:assembleAndroidTest
-saucectl run -c .sauce/sentry-uitest-android-end2end.yml
-```
+# CI
+CI runs these tests on a GitHub Actions emulator in the AGP matrix workflow (`.github/workflows/agp-matrix.yml`).
+The emulator must not be an ATD image: PixelCopy returns no data there, and the replay tests need it.
 
 # Troubleshooting
 

sentry-android-integration-tests/sentry-uitest-android/build.gradle.kts

@@ -30,7 +30,6 @@ android {
     // This doesn't work on some devices with Android 11+. Clearing package data resets permissions.
     // Check the readme for more info.
     testInstrumentationRunnerArguments["clearPackageData"] = "true"
-    buildConfigField("String", "ENVIRONMENT", "\"${System.getProperty("environment", "")}\"")
   }
 
   testOptions { execution = "ANDROIDX_TEST_ORCHESTRATOR" }
@@ -40,7 +39,6 @@ android {
     // Note that the viewBinding.enabled property is now deprecated.
     viewBinding = true
     compose = true
-    buildConfig = true
   }
 
   composeOptions { kotlinCompilerExtensionVersion = libs.versions.composeCompiler.get() }

sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/BaseUiTest.kt

@@ -125,6 +125,9 @@ fun waitUntilIdle() {
   Espresso.onIdle()
 }
 
+/** Whether sentry-android-replay is on the classpath (APPLY_SENTRY_INTEGRATIONS=true). */
+fun isReplayAvailable(): Boolean = classExists("io.sentry.android.replay.ReplayIntegration")
+
 fun classExists(className: String): Boolean {
   try {
     Class.forName(className)

sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/EnvelopeTests.kt

@@ -3,17 +3,14 @@ package io.sentry.uitest.android
 import androidx.lifecycle.Lifecycle
 import androidx.test.core.app.launchActivity
 import androidx.test.espresso.Espresso
-import androidx.test.espresso.IdlingPolicies
 import androidx.test.espresso.IdlingRegistry
 import androidx.test.espresso.action.ViewActions
 import androidx.test.espresso.matcher.ViewMatchers
 import androidx.test.ext.junit.runners.AndroidJUnit4
-import io.sentry.NoOpLogger
 import io.sentry.ProfilingTraceData
 import io.sentry.Sentry
 import io.sentry.SentryEvent
 import io.sentry.android.core.AndroidLogger
-import io.sentry.android.core.BuildInfoProvider
 import io.sentry.android.core.SentryAndroidOptions
 import io.sentry.assertEnvelopeProfile
 import io.sentry.assertEnvelopeTransaction
@@ -25,7 +22,6 @@ import kotlin.test.Test
 import kotlin.test.assertEquals
 import kotlin.test.assertNotNull
 import kotlin.test.assertTrue
-import org.junit.Assume
 import org.junit.Assume.assumeNotNull
 import org.junit.runner.RunWith
 
@@ -196,44 +192,6 @@ class EnvelopeTests : BaseUiTest() {
     }
   }
 
-  @Test
-  fun checkTimedOutProfile() {
-    Assume.assumeFalse(
-      "Sometimes traceFile does not exist for profiles on emulators and it fails." +
-        " Although, Android Runtime is the one that manages the file, so something" +
-        " must be wrong with Debug.startMethodTracing",
-      BuildInfoProvider(NoOpLogger.getInstance()).isEmulator ?: true,
-    )
-    // We increase the IdlingResources timeout to exceed the profiling timeout
-    IdlingPolicies.setIdlingResourceTimeout(1, TimeUnit.MINUTES)
-    initSentry(true) { options: SentryAndroidOptions ->
-      options.tracesSampleRate = 1.0
-      options.profilesSampleRate = 1.0
-    }
-    relayIdlingResource.increment()
-    val transaction = Sentry.startTransaction("timedOutProfile", "testTimeout")
-    Thread {
-        Thread.sleep(35_000)
-        transaction.finish()
-      }
-      .start()
-    // We call transaction.finish() after 35 seconds, and check profile times out after 30 seconds
-
-    relay.assert {
-      findEnvelope {
-          assertEnvelopeTransaction(it.items.toList(), AndroidLogger()).transaction ==
-            "timedOutProfile"
-        }
-        .assert {
-          val transactionItem: SentryTransaction = it.assertTransaction()
-          // Profile should not be present, as it timed out and is discarded
-          it.assertNoOtherItems()
-          assertEquals("timedOutProfile", transactionItem.transaction)
-        }
-      assertNoOtherEnvelopes()
-    }
-  }
-
   @Test
   fun sendProfiledTransaction() {
     // This is a dogfooding test

sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/ReplayTest.kt

@@ -18,9 +18,7 @@ import shark.ReferencePattern
 class ReplayTest : BaseUiTest() {
   @Before
   fun setup() {
-    // we can't run on GH actions emulator, because they don't allow capturing screenshots properly
-    @Suppress("KotlinConstantConditions")
-    assumeThat(BuildConfig.ENVIRONMENT != "github", `is`(true))
+    assumeThat(isReplayAvailable(), `is`(true))
     // crash on swallowed Compose masking errors (e.g. broken obfuscated internals) so regressions
     // fail this on-device test instead of silently under-masking (see SentryReplayDebug)
     System.setProperty("io.sentry.replay.compose.fail-fast", "true")

sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/UserFeedbackUiTest.kt

@@ -464,8 +464,7 @@ class UserFeedbackUiTest : BaseUiTest() {
 
   @Test
   fun userFeedbackSendEnvelope() {
-    // GH actions emulator don't allow capturing screenshots properly
-    val enableReplay = BuildConfig.ENVIRONMENT != "github"
+    val enableReplay = isReplayAvailable()
     initSentry(true) {
       // When sending the feedback we want to wait for relay to receive it.
       // We can't simply increment the idling resource,

sentry-android-integration-tests/sentry-uitest-android/src/androidTestReplay/java/io/sentry/uitest/android/ReplaySnapshotTest.kt

@@ -12,17 +12,12 @@ import java.util.concurrent.CountDownLatch
 import java.util.concurrent.TimeUnit
 import kotlin.test.Test
 import kotlin.test.assertTrue
-import org.hamcrest.CoreMatchers.`is`
-import org.junit.Assume.assumeThat
 import org.junit.Before
 
 class ReplaySnapshotTest : BaseUiTest() {
 
   @Before
   fun setup() {
-    // GH Actions emulators don't support capturing screenshots for replay
-    @Suppress("KotlinConstantConditions")
-    assumeThat(BuildConfig.ENVIRONMENT != "github", `is`(true))
     // crash on swallowed Compose masking errors (e.g. broken obfuscated internals) so regressions
     // fail this on-device test instead of silently under-masking (see SentryReplayDebug)
     System.setProperty("io.sentry.replay.compose.fail-fast", "true")
@@ -33,7 +28,8 @@ class ReplaySnapshotTest : BaseUiTest() {
     val snapshotsDir =
       File(
           Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOWNLOADS),
-          "sauce_labs_custom_screenshots",
+          // pulled and uploaded by scripts/test-ui-android.sh in CI
+          "sentry_replay_snapshots",
         )
         .apply {
           deleteRecursively()