test(ui): Add critical tests run by Maestro (#3802)

.github/workflows/integration-tests-ui-critical.yml

@@ -0,0 +1,106 @@
+name: UI Tests Critical
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  BASE_PATH: "sentry-android-integration-tests/sentry-uitest-android-critical"
+  BUILD_PATH: "build/outputs/apk/release"
+  APK_NAME: "sentry-uitest-android-critical-release.apk"
+  APK_ARTIFACT_NAME: "sentry-uitest-android-critical-release"
+  MAESTRO_VERSION: "1.39.0"
+
+jobs:
+  build:
+    name: Build sentry-uitest-android-critical
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Java 17
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: '17'
+
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@bb0c460cbf5354b0cddd15bacdf0d6aaa3e5a32b # pin@v3
+        with:
+          gradle-home-cache-cleanup: true
+
+      - name: Build debug APK
+        run: make assembleUiTestCriticalRelease
+
+      - name: Upload APK artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{env.APK_ARTIFACT_NAME}}
+          path: "${{env.BASE_PATH}}/${{env.BUILD_PATH}}/${{env.APK_NAME}}"
+          retention-days: 1
+
+  run-maestro-tests:
+    name: Run Maestro Tests
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup KVM
+        shell: bash
+        run: |
+          # check if virtualization is supported...
+          sudo apt install -y --no-install-recommends cpu-checker coreutils && echo "CPUs=$(nproc --all)" && kvm-ok
+          # allow access to KVM to run the emulator
+          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \
+            | sudo tee /etc/udev/rules.d/99-kvm4all.rules
+          sudo udevadm control --reload-rules
+          sudo udevadm trigger --name-match=kvm
+
+      - name: Download APK artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{env.APK_ARTIFACT_NAME}}
+
+      - name: Install Maestro
+        uses: dniHze/maestro-test-action@bda8a93211c86d0a05b7a4597c5ad134566fbde4 # [email protected]
+        with:
+          version: ${{env.MAESTRO_VERSION}}
+
+      - name: Run tests
+        uses: reactivecircus/android-emulator-runner@f0d1ed2dcad93c7479e8b2f2226c83af54494915 # [email protected]
+        with:
+          api-level: 30
+          force-avd-creation: false
+          disable-animations: true
+          disable-spellchecker: true
+          target: 'aosp_atd'
+          channel: canary # Necessary for ATDs
+          emulator-options: >
+            -no-window
+            -no-snapshot-save
+            -gpu swiftshader_indirect
+            -noaudio
+            -no-boot-anim
+            -camera-back none
+            -camera-front none
+            -timezone US/Pacific
+          script: |
+            adb install -r -d "${{env.APK_NAME}}"
+            maestro test "${{env.BASE_PATH}}/maestro" --debug-output "${{env.BASE_PATH}}/maestro-logs"
+
+      - name: Upload Maestro test results
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: maestro-logs
+          path: "${{env.BASE_PATH}}/maestro-logs"
+          retention-days: 1

.github/workflows/system-tests-backend.yml

@@ -46,19 +46,46 @@ jobs:
 
       - name: Exclude android modules from build
         run: |
-          sed -i -e '/.*"sentry-android-ndk",/d' -e '/.*"sentry-android",/d' -e '/.*"sentry-compose",/d' -e '/.*"sentry-android-core",/d' -e '/.*"sentry-android-fragment",/d' -e '/.*"sentry-android-navigation",/d' -e '/.*"sentry-android-okhttp",/d' -e '/.*"sentry-android-sqlite",/d' -e '/.*"sentry-android-timber",/d' -e '/.*"sentry-android-integration-tests:sentry-uitest-android-benchmark",/d' -e '/.*"sentry-android-integration-tests:sentry-uitest-android",/d' -e '/.*"sentry-android-integration-tests:test-app-sentry",/d' -e '/.*"sentry-samples:sentry-samples-android",/d' -e '/.*"sentry-android-replay",/d' settings.gradle.kts
+          sed -i \
+            -e '/.*"sentry-android-ndk",/d' \
+            -e '/.*"sentry-android",/d' \
+            -e '/.*"sentry-compose",/d' \
+            -e '/.*"sentry-android-core",/d' \
+            -e '/.*"sentry-android-fragment",/d' \
+            -e '/.*"sentry-android-navigation",/d' \
+            -e '/.*"sentry-android-okhttp",/d' \
+            -e '/.*"sentry-android-sqlite",/d' \
+            -e '/.*"sentry-android-timber",/d' \
+            -e '/.*"sentry-android-integration-tests:sentry-uitest-android-benchmark",/d' \
+            -e '/.*"sentry-android-integration-tests:sentry-uitest-android",/d' \
+            -e '/.*"sentry-android-integration-tests:sentry-uitest-android-critical",/d' \
+            -e '/.*"sentry-android-integration-tests:test-app-sentry",/d' \
+            -e '/.*"sentry-samples:sentry-samples-android",/d' \
+            -e '/.*"sentry-android-replay",/d' \
+            settings.gradle.kts
 
       - name: Exclude android modules from ignore list
         run: |
-          sed -i -e '/.*"sentry-uitest-android",/d' -e '/.*"sentry-uitest-android-benchmark",/d' -e '/.*"test-app-sentry",/d' -e '/.*"sentry-samples-android",/d' build.gradle.kts
+          sed -i \
+            -e '/.*"sentry-uitest-android",/d' \
+            -e '/.*"sentry-uitest-android-benchmark",/d' \
+            -e '/.*"sentry-uitest-android-critical",/d' \
+            -e '/.*"test-app-sentry",/d' \
+            -e '/.*"sentry-samples-android",/d' \
+            build.gradle.kts
 
       - name: Build server jar
         run: |
           ./gradlew :sentry-samples:${{ matrix.sample }}:bootJar
 
       - name: Start server and run integration test for sentry-cli commands
         run: |
-          test/system-test-sentry-server-start.sh > sentry-mock-server.txt 2>&1 & test/system-test-spring-server-start.sh "${{ matrix.sample }}" > spring-server.txt 2>&1 & test/wait-for-spring.sh && ./gradlew :sentry-samples:${{ matrix.sample }}:systemTest
+          test/system-test-sentry-server-start.sh \
+            > sentry-mock-server.txt 2>&1 & \
+          test/system-test-spring-server-start.sh "${{ matrix.sample }}" \
+            > spring-server.txt 2>&1 & \
+          test/wait-for-spring.sh && \
+          ./gradlew :sentry-samples:${{ matrix.sample }}:systemTest
 
       - name: Upload test results
         if: always()

Makefile

@@ -1,4 +1,4 @@
-.PHONY: all clean compile javadocs dryRelease update stop checkFormat format api assembleBenchmarkTestRelease assembleUiTestRelease createCoverageReports check preMerge publish
+.PHONY: all clean compile javadocs dryRelease update stop checkFormat format api assembleBenchmarkTestRelease assembleUiTestRelease assembleUiTestCriticalRelease createCoverageReports runUiTestCritical check preMerge publish
 
 all: stop clean javadocs compile createCoverageReports
 assembleBenchmarks: assembleBenchmarkTestRelease
@@ -53,6 +53,14 @@ assembleUiTestRelease:
 	./gradlew :sentry-android-integration-tests:sentry-uitest-android:assembleRelease
 	./gradlew :sentry-android-integration-tests:sentry-uitest-android:assembleAndroidTest -DtestBuildType=release
 
+# Assemble release of the uitest-android-critical module
+assembleUiTestCriticalRelease:
+	./gradlew :sentry-android-integration-tests:sentry-uitest-android-critical:assembleRelease
+
+# Run Maestro tests for the uitest-android-critical module
+runUiTestCritical:
+	./scripts/test-ui-critical.sh
+
 # Create coverage reports
 #  - Jacoco for Java & Android modules
 #  - Kover for KMP modules e.g sentry-compose

build.gradle.kts

@@ -68,6 +68,7 @@ apiValidation {
             "sentry-samples-spring-boot-webflux-jakarta",
             "sentry-uitest-android",
             "sentry-uitest-android-benchmark",
+            "sentry-uitest-android-critical",
             "test-app-plain",
             "test-app-sentry",
             "sentry-samples-netflix-dgs"

scripts/test-ui-critical.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+set -e
+
+echo "Checking if ADB is installed..."
+if ! command -v adb &> /dev/null; then
+    echo "ADB is not installed or not in PATH. Please install Android SDK platform tools and ensure ADB is in your PATH."
+    exit 1
+fi
+
+echo "Checking if an Android emulator is running..."
+if ! adb devices | grep -q "emulator"; then
+    echo "No Android emulator is currently running. Please start an emulator before running this script."
+    exit 1
+fi
+
+echo "Checking if Maestro is installed..."
+if ! command -v maestro &> /dev/null; then
+    echo "Maestro is not installed. Please install Maestro before running this script."
+    exit 1
+fi
+
+echo "Building the UI Test Critical app..."
+make assembleUiTestCriticalRelease
+
+echo "Installing the UI Test Critical app on the emulator..."
+baseDir="sentry-android-integration-tests/sentry-uitest-android-critical"
+buildDir="build/outputs/apk/release"
+apkName="sentry-uitest-android-critical-release.apk"
+appPath="${baseDir}/${buildDir}/${apkName}"
+adb install -r -d "$appPath"
+
+echo "Running the Maestro tests..."
+maestro test \
+  "${baseDir}/maestro" \
+  --debug-output "${baseDir}/maestro-logs"

sentry-android-integration-tests/sentry-uitest-android-critical/.gitignore

@@ -0,0 +1,2 @@
+/build
+/maestro-logs

sentry-android-integration-tests/sentry-uitest-android-critical/build.gradle.kts

@@ -0,0 +1,69 @@
+import io.gitlab.arturbosch.detekt.Detekt
+
+plugins {
+    id("com.android.application")
+    kotlin("android")
+}
+
+android {
+    compileSdk = Config.Android.compileSdkVersion
+    namespace = "io.sentry.uitest.android.critical"
+
+    signingConfigs {
+        getByName("debug") {
+            // Debug config remains unchanged
+        }
+    }
+
+    defaultConfig {
+        applicationId = "io.sentry.uitest.android.critical"
+        minSdk = Config.Android.minSdkVersionCompose
+        targetSdk = Config.Android.targetSdkVersion
+        versionCode = 1
+        versionName = "1.0"
+    }
+
+    buildTypes {
+        release {
+            isMinifyEnabled = false
+            signingConfig = signingConfigs.getByName("debug")
+            proguardFiles(
+                getDefaultProguardFile("proguard-android-optimize.txt"),
+                "proguard-rules.pro"
+            )
+        }
+    }
+    kotlinOptions {
+        jvmTarget = JavaVersion.VERSION_1_8.toString()
+    }
+    buildFeatures {
+        compose = true
+    }
+    composeOptions {
+        kotlinCompilerExtensionVersion = Config.androidComposeCompilerVersion
+    }
+    variantFilter {
+        if (Config.Android.shouldSkipDebugVariant(buildType.name)) {
+            ignore = true
+        }
+    }
+}
+
+dependencies {
+    implementation(kotlin(Config.kotlinStdLib, org.jetbrains.kotlin.config.KotlinCompilerVersion.VERSION))
+    implementation(Config.Libs.androidxCore)
+    implementation(Config.Libs.composeActivity)
+    implementation(Config.Libs.composeFoundation)
+    implementation(Config.Libs.composeMaterial)
+    implementation(Config.Libs.constraintLayout)
+    implementation(projects.sentryAndroidCore)
+}
+
+tasks.withType<Detekt> {
+    // Target version of the generated JVM bytecode. It is used for type resolution.
+    jvmTarget = JavaVersion.VERSION_1_8.toString()
+}
+
+kotlin {
+    explicitApi()
+}

sentry-android-integration-tests/sentry-uitest-android-critical/maestro/corruptEnvelope.yaml

@@ -0,0 +1,11 @@
+appId: io.sentry.uitest.android.critical
+---
+- launchApp
+- tapOn: "Write Corrupted Envelope"
+# The close here ensures the next corrupted envelope
+# will be present on the next app launch
+- tapOn: "Close SDK"
+- tapOn: "Write Corrupted Envelope"
+- stopApp
+- launchApp
+- assertVisible: "Welcome!"

sentry-android-integration-tests/sentry-uitest-android-critical/maestro/crash.yaml

@@ -0,0 +1,6 @@
+appId: io.sentry.uitest.android.critical
+---
+- launchApp
+- tapOn: "Crash"
+- launchApp
+- assertVisible: "Welcome!"

sentry-android-integration-tests/sentry-uitest-android-critical/proguard-rules.pro

@@ -0,0 +1,21 @@
+# Add project specific ProGuard rules here.
+# You can control the set of applied configuration files using the
+# proguardFiles setting in build.gradle.
+#
+# For more details, see
+#   http://developer.android.com/guide/developing/tools/proguard.html
+
+# If your project uses WebView with JS, uncomment the following
+# and specify the fully qualified class name to the JavaScript interface
+# class:
+#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
+#   public *;
+#}
+
+# Uncomment this to preserve the line number information for
+# debugging stack traces.
+#-keepattributes SourceFile,LineNumberTable
+
+# If you keep the line number information, uncomment this to
+# hide the original source file name.
+#-renamesourcefileattribute SourceFile

sentry-android-integration-tests/sentry-uitest-android-critical/src/main/AndroidManifest.xml

@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    xmlns:tools="http://schemas.android.com/tools">
+
+    <application
+        android:label="Sentry UI Tests Critical"
+        android:supportsRtl="true"
+        tools:targetApi="31">
+        <meta-data android:name="io.sentry.dsn" android:value="https://[email protected]/5428559" />
+        <meta-data android:name="io.sentry.debug" android:value="true" />
+        <activity
+            android:name=".MainActivity"
+            android:exported="true">
+            <intent-filter>
+                <action android:name="android.intent.action.MAIN" />
+                <category android:name="android.intent.category.LAUNCHER" />
+            </intent-filter>
+        </activity>
+    </application>
+
+</manifest>