From 52d91c05a7291e5684522343a07c2bb714ceef00 Mon Sep 17 00:00:00 2001 From: Leendert de Borst Date: Tue, 10 Feb 2026 20:27:02 +0100 Subject: [PATCH] Update tests --- .../net/aliasvault/app/AliasVaultUITests.kt | 81 +++++++- .../net/aliasvault/app/TestConfiguration.kt | 61 +++++- .../java/net/aliasvault/app/UITestHelpers.kt | 193 +++++++++++++++--- apps/mobile-app/android/scripts/e2e-test.sh | 26 ++- 4 files changed, 324 insertions(+), 37 deletions(-) diff --git a/apps/mobile-app/android/app/src/androidTest/java/net/aliasvault/app/AliasVaultUITests.kt b/apps/mobile-app/android/app/src/androidTest/java/net/aliasvault/app/AliasVaultUITests.kt index 031829cf6..915277811 100644 --- a/apps/mobile-app/android/app/src/androidTest/java/net/aliasvault/app/AliasVaultUITests.kt +++ b/apps/mobile-app/android/app/src/androidTest/java/net/aliasvault/app/AliasVaultUITests.kt @@ -2,11 +2,13 @@ package net.aliasvault.app import android.content.Intent import android.net.Uri +import android.util.Log import androidx.test.ext.junit.runners.AndroidJUnit4 import androidx.test.platform.app.InstrumentationRegistry import androidx.test.uiautomator.UiDevice import kotlinx.coroutines.runBlocking import net.aliasvault.app.UITestHelpers.assertTestIdExists +import net.aliasvault.app.UITestHelpers.dumpWindowHierarchy import net.aliasvault.app.UITestHelpers.existsByTestId import net.aliasvault.app.UITestHelpers.findByTestId import net.aliasvault.app.UITestHelpers.findByText @@ -14,6 +16,7 @@ import net.aliasvault.app.UITestHelpers.hideKeyboard import net.aliasvault.app.UITestHelpers.longSleep import net.aliasvault.app.UITestHelpers.scrollToTestId import net.aliasvault.app.UITestHelpers.scrollToText +import net.aliasvault.app.UITestHelpers.takeScreenshot import net.aliasvault.app.UITestHelpers.tapTestId import net.aliasvault.app.UITestHelpers.typeIntoTestId import net.aliasvault.app.UITestHelpers.waitForTestId @@ -24,9 +27,13 @@ import org.junit.Assert.assertTrue import org.junit.Assume.assumeTrue import org.junit.Before import org.junit.FixMethodOrder +import org.junit.Rule import org.junit.Test +import org.junit.rules.TestWatcher +import org.junit.runner.Description import org.junit.runner.RunWith import org.junit.runners.MethodSorters +import java.io.File /** * E2E UI Tests for AliasVault Android app. @@ -47,6 +54,53 @@ import org.junit.runners.MethodSorters class AliasVaultUITests { private lateinit var device: UiDevice private val packageName = "net.aliasvault.app" + private var currentTestName: String = "" + + companion object { + private const val TAG = "AliasVaultUITests" + } + + /** + * Rule to capture screenshots and UI hierarchy on test failure. + */ + @get:Rule + val screenshotOnFailureRule = object : TestWatcher() { + override fun failed(e: Throwable?, description: Description?) { + val testName = description?.methodName ?: "unknown" + Log.e(TAG, "Test '$testName' FAILED - capturing debug info") + + try { + if (::device.isInitialized) { + // Take screenshot + val screenshotFile = File("/sdcard/Download/test-failure-$testName.png") + device.takeScreenshot(screenshotFile) + Log.e(TAG, "Screenshot saved to: ${screenshotFile.absolutePath}") + + // Dump UI hierarchy for debugging + Log.e(TAG, "=== UI Hierarchy at failure ===") + device.dumpWindowHierarchy(System.err) + Log.e(TAG, "=== End UI Hierarchy ===") + } + } catch (ex: Exception) { + Log.e(TAG, "Failed to capture debug info: ${ex.message}") + } + } + + override fun starting(description: Description?) { + currentTestName = description?.methodName ?: "unknown" + Log.i(TAG, "=== Starting test: $currentTestName ===") + Log.i(TAG, "CI mode: ${TestConfiguration.isCI}") + Log.i( + TAG, + "Timeouts: DEFAULT=${TestConfiguration.DEFAULT_TIMEOUT_MS}ms, " + + "EXTENDED=${TestConfiguration.EXTENDED_TIMEOUT_MS}ms", + ) + } + + override fun finished(description: Description?) { + Log.i(TAG, "=== Finished test: ${description?.methodName} ===") + } + } @Before fun setUp() { @@ -55,6 +109,11 @@ class AliasVaultUITests { // Wake up device if sleeping device.wakeUp() + // Disable animations for more reliable tests in CI + // Note: Requires adb shell settings put global window_animation_scale 0 etc. + // This is done via the test script, but we log a reminder here + Log.i(TAG, "Ensure animations are disabled for reliable tests") + // Note: We don't clear app data here because pm clear kills the instrumentation process. // Instead, each test creates its own isolated test user via the API, and loginWithTestUser() // handles logging out any existing session before logging in with the new test user. @@ -63,7 +122,7 @@ class AliasVaultUITests { @After fun tearDown() { - // Take screenshot on failure would go here if needed + // Screenshots on failure are handled by the TestWatcher rule } // region Test Setup @@ -171,6 +230,9 @@ class AliasVaultUITests { // Verify item exists in list assertTrue("Should find item in list", verifyItemExistsInList(uniqueName)) + // Small delay before opening item - list may still be rendering + Thread.sleep(1000) + // Open and verify item details assertTrue("Should verify item details", openAndVerifyItem(uniqueName, "e2e-test@example.com")) @@ -752,6 +814,8 @@ class AliasVaultUITests { * Opens an item from the list and verifies its details. */ private fun openAndVerifyItem(name: String, expectedEmail: String? = null): Boolean { + println("[Helper] Opening item '$name' to verify details") + val itemCard = device.scrollToText(name) ?: device.findByText(name) if (itemCard == null) { @@ -759,18 +823,29 @@ class AliasVaultUITests { return false } + println("[Helper] Found item card, clicking...") itemCard.click() + // Wait for detail screen - use longer timeout for screen transition + Thread.sleep(500) // Small delay for click to register + println("[Helper] Waiting for detail screen (Login credentials text)...") if (device.waitForText("Login credentials", TestConfiguration.DEFAULT_TIMEOUT_MS) == null) { println("[Helper] Item detail screen did not appear") + println("[Helper] - items-screen visible: ${device.existsByTestId("items-screen")}") + println("[Helper] - add-edit-screen visible: ${device.existsByTestId("add-edit-screen")}") return false } + println("[Helper] Detail screen appeared") if (expectedEmail != null) { - if (device.waitForTextContains(expectedEmail) == null) { - println("[Helper] Expected email '$expectedEmail' not found") + println("[Helper] Looking for email '$expectedEmail' on detail screen...") + // Use DEFAULT_TIMEOUT_MS instead of SHORT_TIMEOUT_MS for email verification + // In headless CI mode, content rendering can be slow + if (device.waitForTextContains(expectedEmail, TestConfiguration.DEFAULT_TIMEOUT_MS) == null) { + println("[Helper] Expected email '$expectedEmail' not found within timeout") return false } + println("[Helper] Email found on detail screen") } println("[Helper] Item '$name' verified successfully") diff --git a/apps/mobile-app/android/app/src/androidTest/java/net/aliasvault/app/TestConfiguration.kt b/apps/mobile-app/android/app/src/androidTest/java/net/aliasvault/app/TestConfiguration.kt index 5f2ff766e..25c69306e 100644 --- a/apps/mobile-app/android/app/src/androidTest/java/net/aliasvault/app/TestConfiguration.kt +++ b/apps/mobile-app/android/app/src/androidTest/java/net/aliasvault/app/TestConfiguration.kt @@ -1,9 +1,58 @@ package net.aliasvault.app +import android.os.Bundle +import android.util.Log +import androidx.test.platform.app.InstrumentationRegistry + /** * Configuration for E2E UI tests. */ object TestConfiguration { + private const val TAG = "TestConfiguration" + + /** + * Get instrumentation arguments passed via gradle. + */ + private val instrumentationArgs: Bundle by lazy { + try { + InstrumentationRegistry.getArguments() + } catch (e: Exception) { + Log.w(TAG, "Failed to get instrumentation arguments: ${e.message}") + Bundle() + } + } + + /** + * Detect if running in CI environment (headless emulator). + * CI is detected by checking: + * 1. Instrumentation argument CI=true (passed from gradle) + * 2. GITHUB_ACTIONS env var + * 3. CI env var + */ + val isCI: Boolean by lazy { + // Check instrumentation argument first (most reliable) + val ciArg = instrumentationArgs.getString("CI") == "true" + + // Fallback to environment variables + val githubActions = System.getenv("GITHUB_ACTIONS") == "true" || + System.getProperty("GITHUB_ACTIONS") == "true" + val ciEnv = System.getenv("CI") == "true" || + System.getProperty("CI") == "true" + + val result = ciArg || githubActions || ciEnv + Log.i( + TAG, + "CI mode detected: $result (ciArg=$ciArg, GITHUB_ACTIONS=$githubActions, CI=$ciEnv)", + ) + result + } + + /** + * Multiplier for timeouts in CI (headless emulator is slower). + */ + private val timeoutMultiplier: Long + get() = if (isCI) 3L else 1L + /** * API URL for testing (defaults to local development server). * Can be overridden by setting the API_URL instrumentation argument. @@ -21,18 +70,24 @@ object TestConfiguration { /** * Default timeout for element waiting (milliseconds). + * Increased in CI mode where headless emulator is slower. */ - const val DEFAULT_TIMEOUT_MS = 10_000L + val DEFAULT_TIMEOUT_MS: Long + get() = 10_000L * timeoutMultiplier /** * Extended timeout for operations that may take longer (like login with network). + * Increased in CI mode where headless emulator is slower. */ - const val EXTENDED_TIMEOUT_MS = 30_000L + val EXTENDED_TIMEOUT_MS: Long + get() = 30_000L * timeoutMultiplier /** * Short timeout for quick checks (milliseconds). + * Increased in CI mode where headless emulator is slower. */ - const val SHORT_TIMEOUT_MS = 2_000L + val SHORT_TIMEOUT_MS: Long + get() = 2_000L * timeoutMultiplier /** * Default Argon2Id encryption settings matching server defaults. diff --git a/apps/mobile-app/android/app/src/androidTest/java/net/aliasvault/app/UITestHelpers.kt b/apps/mobile-app/android/app/src/androidTest/java/net/aliasvault/app/UITestHelpers.kt index 380c93897..5cc7a8b2f 100644 --- a/apps/mobile-app/android/app/src/androidTest/java/net/aliasvault/app/UITestHelpers.kt +++ b/apps/mobile-app/android/app/src/androidTest/java/net/aliasvault/app/UITestHelpers.kt @@ -5,6 +5,8 @@ import androidx.test.uiautomator.By import androidx.test.uiautomator.UiDevice import androidx.test.uiautomator.UiObject2 import androidx.test.uiautomator.Until +import java.io.File +import java.io.OutputStream /** * UI test helper functions for Android instrumented tests. @@ -13,6 +15,12 @@ import androidx.test.uiautomator.Until object UITestHelpers { private const val TAG = "UITestHelpers" + /** + * Enable verbose logging for CI debugging. + */ + private val verboseLogging: Boolean + get() = TestConfiguration.isCI + // region Element Finding /** @@ -136,51 +144,106 @@ object UITestHelpers { // region Actions + /** + * Maximum retries for flaky UI interactions in CI. + */ + private val maxRetries: Int + get() = if (TestConfiguration.isCI) 3 else 1 + /** * Tap on an element with testID. + * Includes retry logic for CI headless mode where taps can be flaky. */ fun UiDevice.tapTestId(testId: String): Boolean { - // Try immediate find first (no waiting) - val element = findByTestId(testId) - return if (element != null) { - element.click() - true - } else { - Log.e(TAG, "Failed to tap testId: $testId - element not found") - false + repeat(maxRetries) { attempt -> + val element = findByTestId(testId) + if (element != null) { + try { + element.click() + if (verboseLogging) { + Log.i(TAG, "Tapped testId: $testId (attempt ${attempt + 1})") + } + Thread.sleep(100) // Small delay after tap for UI to respond + return true + } catch (e: Exception) { + Log.w(TAG, "Tap failed on attempt ${attempt + 1}: ${e.message}") + Thread.sleep(200) + } + } else if (attempt < maxRetries - 1) { + if (verboseLogging) { + Log.w(TAG, "Element not found, waiting before retry ${attempt + 2}/$maxRetries") + } + Thread.sleep(500) + } } + Log.e(TAG, "Failed to tap testId: $testId after $maxRetries attempts") + return false } /** * Tap on an element with text. + * Includes retry logic for CI headless mode. */ fun UiDevice.tapText(text: String): Boolean { - // Try immediate find first (no waiting) - val element = findByText(text) - return if (element != null) { - element.click() - true - } else { - Log.e(TAG, "Failed to tap text: $text - element not found") - false + repeat(maxRetries) { attempt -> + val element = findByText(text) + if (element != null) { + try { + element.click() + if (verboseLogging) { + Log.i(TAG, "Tapped text: $text (attempt ${attempt + 1})") + } + Thread.sleep(100) + return true + } catch (e: Exception) { + Log.w(TAG, "Tap failed on attempt ${attempt + 1}: ${e.message}") + Thread.sleep(200) + } + } else if (attempt < maxRetries - 1) { + Thread.sleep(500) + } } + Log.e(TAG, "Failed to tap text: $text after $maxRetries attempts") + return false } /** * Type text into an element with testID. + * Includes retry logic and verification for CI headless mode. */ fun UiDevice.typeIntoTestId(testId: String, text: String): Boolean { - // Try immediate find first (no waiting) - val element = findByTestId(testId) - return if (element != null) { - element.click() - Thread.sleep(100) // Small delay for focus - element.text = text - true - } else { - Log.e(TAG, "Failed to type into testId: $testId - element not found") - false + repeat(maxRetries) { attempt -> + val element = findByTestId(testId) + if (element != null) { + try { + element.click() + Thread.sleep(150) // Slightly longer delay for focus in CI + element.text = text + Thread.sleep(100) + + // Verify text was entered (important for CI) + val verifyElement = findByTestId(testId) + if (verifyElement?.text == text) { + if (verboseLogging) { + Log.i(TAG, "Typed into testId: $testId (attempt ${attempt + 1})") + } + return true + } else if (verboseLogging) { + Log.w( + TAG, + "Text verification failed: expected '$text', got '${verifyElement?.text}'", + ) + } + } catch (e: Exception) { + Log.w(TAG, "Type failed on attempt ${attempt + 1}: ${e.message}") + } + Thread.sleep(300) + } else if (attempt < maxRetries - 1) { + Thread.sleep(500) + } } + Log.e(TAG, "Failed to type into testId: $testId after $maxRetries attempts") + return false } /** @@ -211,8 +274,16 @@ object UITestHelpers { testId: String, maxScrolls: Int = 5, ): UiObject2? { + // Longer settle time in CI mode where rendering is slower + val settleTime = if (TestConfiguration.isCI) 500L else 300L + repeat(maxScrolls) { - findByTestId(testId)?.let { return it } + findByTestId(testId)?.let { + if (verboseLogging) { + Log.i(TAG, "Found testId '$testId' after scroll") + } + return it + } swipe( displayWidth / 2, displayHeight * 3 / 4, @@ -220,7 +291,7 @@ object UITestHelpers { displayHeight / 4, 10, ) - Thread.sleep(300) // Wait for scroll to settle + Thread.sleep(settleTime) } return findByTestId(testId) } @@ -232,8 +303,16 @@ object UITestHelpers { text: String, maxScrolls: Int = 5, ): UiObject2? { + // Longer settle time in CI mode where rendering is slower + val settleTime = if (TestConfiguration.isCI) 500L else 300L + repeat(maxScrolls) { - findByText(text)?.let { return it } + findByText(text)?.let { + if (verboseLogging) { + Log.i(TAG, "Found text '$text' after scroll") + } + return it + } swipe( displayWidth / 2, displayHeight * 3 / 4, @@ -241,7 +320,7 @@ object UITestHelpers { displayHeight / 4, 10, ) - Thread.sleep(300) // Wait for scroll to settle + Thread.sleep(settleTime) } return findByText(text) } @@ -402,4 +481,58 @@ object UITestHelpers { } // endregion + + // region Debug Helpers + + /** + * Take a screenshot and save to file. + */ + fun UiDevice.takeScreenshot(file: File): Boolean { + return try { + takeScreenshot(file) + Log.i(TAG, "Screenshot saved to: ${file.absolutePath}") + true + } catch (e: Exception) { + Log.e(TAG, "Failed to take screenshot: ${e.message}") + false + } + } + + /** + * Dump the current window hierarchy for debugging. + */ + fun UiDevice.dumpWindowHierarchy(output: OutputStream) { + try { + dumpWindowHierarchy(output) + } catch (e: Exception) { + Log.e(TAG, "Failed to dump window hierarchy: ${e.message}") + } + } + + /** + * Log current screen state for debugging. + */ + fun UiDevice.logScreenState(context: String) { + if (!verboseLogging) return + + Log.i(TAG, "=== Screen state at: $context ===") + Log.i(TAG, " Display: ${displayWidth}x$displayHeight") + + // Check common screens + val screens = listOf( + "login-screen", + "unlock-screen", + "items-screen", + "add-edit-screen", + ) + + for (screen in screens) { + if (findByTestId(screen) != null) { + Log.i(TAG, " ✓ $screen is visible") + } + } + Log.i(TAG, "=== End screen state ===") + } + + // endregion } diff --git a/apps/mobile-app/android/scripts/e2e-test.sh b/apps/mobile-app/android/scripts/e2e-test.sh index 5ab1c1fce..d17800043 100755 --- a/apps/mobile-app/android/scripts/e2e-test.sh +++ b/apps/mobile-app/android/scripts/e2e-test.sh @@ -54,12 +54,22 @@ echo "Ensuring port forwarding..." adb -s "$EMULATOR_ID" reverse tcp:5092 tcp:5092 2>/dev/null || true adb -s "$EMULATOR_ID" reverse tcp:8081 tcp:8081 2>/dev/null || true +# Disable animations for reliable UI testing (critical for headless CI) +echo "Disabling animations for reliable testing..." +adb -s "$EMULATOR_ID" shell settings put global window_animation_scale 0.0 2>/dev/null || true +adb -s "$EMULATOR_ID" shell settings put global transition_animation_scale 0.0 2>/dev/null || true +adb -s "$EMULATOR_ID" shell settings put global animator_duration_scale 0.0 2>/dev/null || true + # Clear app data before running tests to ensure clean state # This must be done via adb (not from within instrumentation) to avoid crashing the test runner echo "Clearing app data for clean test state..." adb -s "$EMULATOR_ID" shell pm clear net.aliasvault.app 2>/dev/null || true sleep 1 +# Detect CI environment +IS_CI="${CI:-${GITHUB_ACTIONS:-false}}" +echo "CI mode: $IS_CI" + # Run tests echo "" echo "=== Running E2E Tests ===" @@ -68,8 +78,15 @@ TEST_OUTPUT_FILE="/tmp/android-test-output.log" # Use --console=plain to avoid ANSI escape codes that break parsing # Use --build-cache to leverage cached compilation artifacts +# Pass CI environment variable to tests for timeout adjustments +CI_ARG="" +if [ "$IS_CI" = "true" ]; then + CI_ARG="-Pandroid.testInstrumentationRunnerArguments.CI=true" +fi + ./gradlew :app:connectedDebugAndroidTest \ -Pandroid.testInstrumentationRunnerArguments.API_URL=http://10.0.2.2:5092 \ + $CI_ARG \ --console=plain \ --build-cache \ --stacktrace 2>&1 | tee "$TEST_OUTPUT_FILE" || TEST_EXIT_CODE=$? @@ -121,7 +138,7 @@ grep -E " > test.*\] SKIPPED" "$TEST_OUTPUT_FILE" 2>/dev/null | \ echo "" -# If tests failed, show failure details +# If tests failed, show failure details and pull screenshots if [ "$TEST_EXIT_CODE" -ne 0 ]; then echo "--- Failure Details ---" # Show assertion failures @@ -129,6 +146,13 @@ if [ "$TEST_EXIT_CODE" -ne 0 ]; then # Show test failure messages grep -B2 -A3 "FAILED" "$TEST_OUTPUT_FILE" 2>/dev/null | grep -v "^--$" | head -20 || true echo "" + + # Pull failure screenshots from device + echo "--- Pulling failure screenshots ---" + mkdir -p app/build/reports/androidTests/screenshots + adb -s "$EMULATOR_ID" pull /sdcard/Download/ app/build/reports/androidTests/screenshots/ 2>/dev/null || true + ls -la app/build/reports/androidTests/screenshots/*.png 2>/dev/null || echo "No screenshots found" + echo "" fi echo "=============================================="