diff --git a/.github/workflows/agp-matrix.yml b/.github/workflows/agp-matrix.yml index 81127cb468f..9c08aa084d4 100644 --- a/.github/workflows/agp-matrix.yml +++ b/.github/workflows/agp-matrix.yml @@ -24,6 +24,8 @@ jobs: env: VERSION_AGP: ${{ matrix.agp }} APPLY_SENTRY_INTEGRATIONS: ${{ matrix.integrations }} + # Upload replay snapshots from one cell only. Forks don't have access to the upload secret. + UPLOAD_REPLAY_SNAPSHOTS: ${{ matrix.agp == '9.4.0' && matrix.integrations && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} steps: - name: Checkout Repo @@ -51,20 +53,21 @@ jobs: path: | ~/.android/avd/* ~/.android/adb* - key: avd-api-30-x86_64-aosp_atd + key: avd-api-30-x86_64-google_apis - name: Create AVD and generate snapshot for caching if: steps.avd-cache.outputs.cache-hit != 'true' uses: reactivecircus/android-emulator-runner@a421e43855164a8197daf9d8d40fe71c6996bb0d # pin@v2 with: api-level: 30 - target: aosp_atd - channel: canary # Necessary for ATDs + # Not an ATD image: PixelCopy returns no data on ATDs, which the replay tests need + target: google_apis + channel: canary arch: x86_64 force-avd-creation: false disable-animations: true disable-spellchecker: true - emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none + emulator-options: -memory 4096 -no-window -gpu auto -noaudio -no-boot-anim -camera-back none disk-size: 4096M script: echo "Generated AVD snapshot for caching." @@ -77,15 +80,28 @@ jobs: uses: reactivecircus/android-emulator-runner@a421e43855164a8197daf9d8d40fe71c6996bb0d # pin@v2 with: api-level: 30 - target: aosp_atd - channel: canary # Necessary for ATDs + # Not an ATD image: PixelCopy returns no data on ATDs, which the replay tests need + target: google_apis + channel: canary arch: x86_64 force-avd-creation: false disable-animations: true disable-spellchecker: true - emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none + emulator-options: -memory 4096 -no-snapshot-save -no-window -gpu auto -noaudio -no-boot-anim -camera-back none disk-size: 4096M - script: ./gradlew sentry-android-integration-tests:sentry-uitest-android:connectedReleaseAndroidTest -Denvironment=github --daemon + script: ./scripts/test-ui-android.sh replay-snapshots + + - name: Install Sentry CLI + if: ${{ !cancelled() && env.UPLOAD_REPLAY_SNAPSHOTS == 'true' }} + uses: getsentry/action-setup-cli@70d7e587b84c2e78cf4d37cd33d7b74fb3729c1b # v1 + + - name: Upload Replay Snapshots to Sentry + if: ${{ !cancelled() && env.UPLOAD_REPLAY_SNAPSHOTS == 'true' && hashFiles('replay-snapshots/*.png') != '' }} + run: sentry-cli snapshots upload ./replay-snapshots --app-id sentry-android-replay + env: + SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} + SENTRY_ORG: sentry-sdks + SENTRY_PROJECT: sentry-android - name: Upload test results if: always() @@ -96,6 +112,7 @@ jobs: **/build/reports/* **/build/outputs/*/connected/* **/build/outputs/mapping/release/* + replay-snapshots/ - name: Test Report uses: phoenix-actions/test-reporting@7317eea6e13c47348dd0bb318669485157c518d6 # pin@v16 diff --git a/.github/workflows/integration-tests-ui.yml b/.github/workflows/integration-tests-ui.yml deleted file mode 100644 index 5704347f497..00000000000 --- a/.github/workflows/integration-tests-ui.yml +++ /dev/null @@ -1,93 +0,0 @@ -name: 'Integration Tests' -on: - push: - branches: - - main - pull_request: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - test: - name: Ui tests - runs-on: ubuntu-latest - - # we copy the secret to the env variable in order to access it in the workflow - env: - SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }} - SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }} - - steps: - - name: Git checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: 'Set up Java: 17' - uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6.1.0 - with: - distribution: 'temurin' - java-version: '21' - - - name: Setup Gradle - uses: gradle/actions/setup-gradle@8da46b539cd8fe6ab27ca75128919a18d757604c # v6.4.1 - - # Clean, build and release a test apk, but only if we will run the benchmark - - name: Make assembleUiTests - if: env.SAUCE_USERNAME != null - run: make assembleUiTests - - - name: Install SauceLabs CLI - uses: saucelabs/saucectl-run-action@283660aa934c02723c497efa151d582a3acc5801 # pin@v4.5.0 - env: - GITHUB_TOKEN: ${{ github.token }} - with: - skip-run: true - if: env.SAUCE_USERNAME != null - - - name: Run Tests - id: saucelabs - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - exec &> >(tee -a "test_logs.txt") - saucectl run -c .sauce/sentry-uitest-android-ui.yml - if: env.SAUCE_USERNAME != null - continue-on-error: true - - - name: Verify Test Results - run: | - processCrashed=$(cat test_logs.txt | grep "Instrumentation run failed due to 'Process crashed.'" | wc -l) - if [[ ${{ steps.saucelabs.outcome }} == 'success' ]]; then - exit 0 - elif [[ "$processCrashed" -ne 0 ]]; then - exit 0 - else - exit 1 - fi - if: env.SAUCE_USERNAME != null - - - - name: Install Sentry CLI - if: ${{ !cancelled() && env.SAUCE_USERNAME != null }} - uses: getsentry/action-setup-cli@70d7e587b84c2e78cf4d37cd33d7b74fb3729c1b # v1 - - - name: Upload Replay Snapshots to Sentry - # Skip on PRs from forks, which don't have access to the upload secret - if: ${{ !cancelled() && env.SAUCE_USERNAME != null && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} - run: | - # find instead of `shopt -s globstar`, which macOS's bash 3.2 doesn't support - mkdir -p replay-snapshots - if [ -d artifacts ]; then - find artifacts -type f -name '*.png' -exec cp {} replay-snapshots/ \; - fi - if [ -n "$(ls -A replay-snapshots)" ]; then - sentry-cli snapshots upload ./replay-snapshots \ - --app-id sentry-android-replay - else - echo "No replay snapshot files found, skipping upload" - fi - env: - SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} - SENTRY_ORG: sentry-sdks - SENTRY_PROJECT: sentry-android diff --git a/.sauce/sentry-uitest-android-ui.yml b/.sauce/sentry-uitest-android-ui.yml deleted file mode 100644 index a00ee10614b..00000000000 --- a/.sauce/sentry-uitest-android-ui.yml +++ /dev/null @@ -1,36 +0,0 @@ -apiVersion: v1alpha -kind: espresso -sauce: - region: us-west-1 - # Controls how many suites are executed at the same time (sauce test env only). - concurrency: 4 - metadata: - build: sentry-uitest-android-ui-$GITHUB_REF-$GITHUB_SHA - tags: - - e2e - - android - -defaults: - timeout: 45m - -espresso: - app: ./sentry-android-integration-tests/sentry-uitest-android/build/outputs/apk/release/sentry-uitest-android-release.apk - testApp: ./sentry-android-integration-tests/sentry-uitest-android/build/outputs/apk/androidTest/release/sentry-uitest-android-release-androidTest.apk -suites: - - - name: "Android 15 Ui test (api 35)" - testOptions: - clearPackageData: true - useTestOrchestrator: true - devices: - - name: ".*" - platformVersion: "15" - -# Controls what artifacts to fetch when the suite on Sauce Cloud has finished. -artifacts: - download: - when: always - match: - - junit.xml - - "*.png" - directory: ./artifacts/ diff --git a/scripts/test-ui-android.sh b/scripts/test-ui-android.sh new file mode 100755 index 00000000000..4567f96afc4 --- /dev/null +++ b/scripts/test-ui-android.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# Runs the sentry-uitest-android instrumentation tests on the connected device, then pulls the +# replay frame snapshots that ReplaySnapshotTest writes, also when the tests fail. +# android-emulator-runner runs each `script:` line in its own shell, so this stays one script to +# keep the tests' exit code. Keep the device directory in sync with ReplaySnapshotTest. +set -eu + +out_dir=$1 +device_dir=/sdcard/Download/sentry_replay_snapshots + +test_status=0 +./gradlew sentry-android-integration-tests:sentry-uitest-android:connectedReleaseAndroidTest || test_status=$? + +mkdir -p "$out_dir" +if adb shell test -d "$device_dir"; then + adb pull "$device_dir/." "$out_dir" +else + printf 'No replay snapshots on the device, skipping\n' +fi + +exit "$test_status" diff --git a/sentry-android-integration-tests/sentry-uitest-android/README.md b/sentry-android-integration-tests/sentry-uitest-android/README.md index d6ccf2f1f93..145df22485c 100644 --- a/sentry-android-integration-tests/sentry-uitest-android/README.md +++ b/sentry-android-integration-tests/sentry-uitest-android/README.md @@ -7,13 +7,9 @@ By default the envelopes sent to relay are caught by a mock server which allows Simply run `./gradlew connectedCheck` to run all ui tests of all modules (requires a connected device, either physical or an emulator). -# SauceLabs -To run on saucelabs execute following commands (need also `SAUCE_USERNAME` and `SAUCE_ACCESS_KEY` environment variables): -For End 2 End: -``` -./gradlew :sentry-android-integration-tests:sentry-uitest-android:assembleRelease :sentry-android-integration-tests:sentry-uitest-android:assembleAndroidTest -saucectl run -c .sauce/sentry-uitest-android-end2end.yml -``` +# CI +CI runs these tests on a GitHub Actions emulator in the AGP matrix workflow (`.github/workflows/agp-matrix.yml`). +The emulator must not be an ATD image: PixelCopy returns no data there, and the replay tests need it. # Troubleshooting diff --git a/sentry-android-integration-tests/sentry-uitest-android/build.gradle.kts b/sentry-android-integration-tests/sentry-uitest-android/build.gradle.kts index 7bbbe39e04f..0af3e1bbb0d 100644 --- a/sentry-android-integration-tests/sentry-uitest-android/build.gradle.kts +++ b/sentry-android-integration-tests/sentry-uitest-android/build.gradle.kts @@ -30,7 +30,6 @@ android { // This doesn't work on some devices with Android 11+. Clearing package data resets permissions. // Check the readme for more info. testInstrumentationRunnerArguments["clearPackageData"] = "true" - buildConfigField("String", "ENVIRONMENT", "\"${System.getProperty("environment", "")}\"") } testOptions { execution = "ANDROIDX_TEST_ORCHESTRATOR" } @@ -40,7 +39,6 @@ android { // Note that the viewBinding.enabled property is now deprecated. viewBinding = true compose = true - buildConfig = true } composeOptions { kotlinCompilerExtensionVersion = libs.versions.composeCompiler.get() } diff --git a/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/BaseUiTest.kt b/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/BaseUiTest.kt index 62be835eb2b..529d840b3d3 100644 --- a/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/BaseUiTest.kt +++ b/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/BaseUiTest.kt @@ -125,6 +125,9 @@ fun waitUntilIdle() { Espresso.onIdle() } +/** Whether sentry-android-replay is on the classpath (APPLY_SENTRY_INTEGRATIONS=true). */ +fun isReplayAvailable(): Boolean = classExists("io.sentry.android.replay.ReplayIntegration") + fun classExists(className: String): Boolean { try { Class.forName(className) diff --git a/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/EnvelopeTests.kt b/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/EnvelopeTests.kt index 3fa2c904873..1d77c95663a 100644 --- a/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/EnvelopeTests.kt +++ b/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/EnvelopeTests.kt @@ -3,17 +3,14 @@ package io.sentry.uitest.android import androidx.lifecycle.Lifecycle import androidx.test.core.app.launchActivity import androidx.test.espresso.Espresso -import androidx.test.espresso.IdlingPolicies import androidx.test.espresso.IdlingRegistry import androidx.test.espresso.action.ViewActions import androidx.test.espresso.matcher.ViewMatchers import androidx.test.ext.junit.runners.AndroidJUnit4 -import io.sentry.NoOpLogger import io.sentry.ProfilingTraceData import io.sentry.Sentry import io.sentry.SentryEvent import io.sentry.android.core.AndroidLogger -import io.sentry.android.core.BuildInfoProvider import io.sentry.android.core.SentryAndroidOptions import io.sentry.assertEnvelopeProfile import io.sentry.assertEnvelopeTransaction @@ -25,7 +22,6 @@ import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertNotNull import kotlin.test.assertTrue -import org.junit.Assume import org.junit.Assume.assumeNotNull import org.junit.runner.RunWith @@ -196,44 +192,6 @@ class EnvelopeTests : BaseUiTest() { } } - @Test - fun checkTimedOutProfile() { - Assume.assumeFalse( - "Sometimes traceFile does not exist for profiles on emulators and it fails." + - " Although, Android Runtime is the one that manages the file, so something" + - " must be wrong with Debug.startMethodTracing", - BuildInfoProvider(NoOpLogger.getInstance()).isEmulator ?: true, - ) - // We increase the IdlingResources timeout to exceed the profiling timeout - IdlingPolicies.setIdlingResourceTimeout(1, TimeUnit.MINUTES) - initSentry(true) { options: SentryAndroidOptions -> - options.tracesSampleRate = 1.0 - options.profilesSampleRate = 1.0 - } - relayIdlingResource.increment() - val transaction = Sentry.startTransaction("timedOutProfile", "testTimeout") - Thread { - Thread.sleep(35_000) - transaction.finish() - } - .start() - // We call transaction.finish() after 35 seconds, and check profile times out after 30 seconds - - relay.assert { - findEnvelope { - assertEnvelopeTransaction(it.items.toList(), AndroidLogger()).transaction == - "timedOutProfile" - } - .assert { - val transactionItem: SentryTransaction = it.assertTransaction() - // Profile should not be present, as it timed out and is discarded - it.assertNoOtherItems() - assertEquals("timedOutProfile", transactionItem.transaction) - } - assertNoOtherEnvelopes() - } - } - @Test fun sendProfiledTransaction() { // This is a dogfooding test diff --git a/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/ReplayTest.kt b/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/ReplayTest.kt index 9c53f0a022d..86f30dc1a1a 100644 --- a/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/ReplayTest.kt +++ b/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/ReplayTest.kt @@ -18,9 +18,7 @@ import shark.ReferencePattern class ReplayTest : BaseUiTest() { @Before fun setup() { - // we can't run on GH actions emulator, because they don't allow capturing screenshots properly - @Suppress("KotlinConstantConditions") - assumeThat(BuildConfig.ENVIRONMENT != "github", `is`(true)) + assumeThat(isReplayAvailable(), `is`(true)) // crash on swallowed Compose masking errors (e.g. broken obfuscated internals) so regressions // fail this on-device test instead of silently under-masking (see SentryReplayDebug) System.setProperty("io.sentry.replay.compose.fail-fast", "true") diff --git a/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/UserFeedbackUiTest.kt b/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/UserFeedbackUiTest.kt index bfcaf2845b3..6bbbe95253a 100644 --- a/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/UserFeedbackUiTest.kt +++ b/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/UserFeedbackUiTest.kt @@ -464,8 +464,7 @@ class UserFeedbackUiTest : BaseUiTest() { @Test fun userFeedbackSendEnvelope() { - // GH actions emulator don't allow capturing screenshots properly - val enableReplay = BuildConfig.ENVIRONMENT != "github" + val enableReplay = isReplayAvailable() initSentry(true) { // When sending the feedback we want to wait for relay to receive it. // We can't simply increment the idling resource, diff --git a/sentry-android-integration-tests/sentry-uitest-android/src/androidTestReplay/java/io/sentry/uitest/android/ReplaySnapshotTest.kt b/sentry-android-integration-tests/sentry-uitest-android/src/androidTestReplay/java/io/sentry/uitest/android/ReplaySnapshotTest.kt index 6d45b2d1f9c..f1099d04348 100644 --- a/sentry-android-integration-tests/sentry-uitest-android/src/androidTestReplay/java/io/sentry/uitest/android/ReplaySnapshotTest.kt +++ b/sentry-android-integration-tests/sentry-uitest-android/src/androidTestReplay/java/io/sentry/uitest/android/ReplaySnapshotTest.kt @@ -12,17 +12,12 @@ import java.util.concurrent.CountDownLatch import java.util.concurrent.TimeUnit import kotlin.test.Test import kotlin.test.assertTrue -import org.hamcrest.CoreMatchers.`is` -import org.junit.Assume.assumeThat import org.junit.Before class ReplaySnapshotTest : BaseUiTest() { @Before fun setup() { - // GH Actions emulators don't support capturing screenshots for replay - @Suppress("KotlinConstantConditions") - assumeThat(BuildConfig.ENVIRONMENT != "github", `is`(true)) // crash on swallowed Compose masking errors (e.g. broken obfuscated internals) so regressions // fail this on-device test instead of silently under-masking (see SentryReplayDebug) System.setProperty("io.sentry.replay.compose.fail-fast", "true") @@ -33,7 +28,8 @@ class ReplaySnapshotTest : BaseUiTest() { val snapshotsDir = File( Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOWNLOADS), - "sauce_labs_custom_screenshots", + // pulled and uploaded by scripts/test-ui-android.sh in CI + "sentry_replay_snapshots", ) .apply { deleteRecursively()