From 6e1e2deb8242b556662ff8be748ac4e528a85b25 Mon Sep 17 00:00:00 2001 From: spaced4ndy <8711996+spaced4ndy@users.noreply.github.com> Date: Tue, 3 Nov 2026 19:55:14 +0000 Subject: [PATCH] ci: app tests reach the test names server and keep emulator crash data (#7932) * ci: pick the desktop test server's "To resolve names" role by the menu's first item That menu opens over the server address's QR code, where OCR can miss its "yes" (run 39242250150 failed its setup three times on it), so the smoke test finds the menu's first item, which repeats the field's value and reads well, and clicks the row under it. The check that the role is now yes stays. * ci: print the emulator's state and keep its crash reports when the Android smoke test fails When the emulator dies during the smoke test, the step only says "no emulator detected" and the emulator's own log stops without a reason. After a failed smoke test the job now prints adb's devices, the emulator process, the kernel log's tail and free memory, and uploads the emulator's crash reports with the smoke test results. * ci: the iOS simulator tests set private routing to unknown servers only, so the app can reach the test names server Master's 3126d9b54 made the iOS app's default private routing "always". Name lookups then go through a preset server, which can't reach the test SMP server on the runner's localhost, so every iOS case that resolves a SimpleX name would fail. The desktop and Android tests already set "unknown" before the first launch; the iOS smoke test now writes the same into the app group's defaults after installing the app, and the app applies it when its chat starts. * ci: iOS previews render default private routing in snapshot tests * ci: keep open_role_menu's comment above it in the desktop smoke test * ci: remove the Android emulator state step's comment * ci: iOS and Android test guides describe private routing and crash reports --------- Co-authored-by: spacedandy-agents-simplex <342056592+spacedandy-agents-simplex@users.noreply.github.com> --- .github/workflows/android-emu.yml | 15 ++++++++++++++ scripts/ci/android/README.md | 2 +- scripts/ci/desktop-linux/smoke-test.sh | 27 +++++++++++++++++++++++--- scripts/ci/ios-sim/README.md | 4 ++-- scripts/ci/ios-sim/smoke-test.sh | 7 +++++++ 5 files changed, 49 insertions(+), 6 deletions(-) diff --git a/.github/workflows/android-emu.yml b/.github/workflows/android-emu.yml index 7617693b2b..55b5909afc 100644 --- a/.github/workflows/android-emu.yml +++ b/.github/workflows/android-emu.yml @@ -304,6 +304,7 @@ jobs: run: scripts/ci/android/emulator.sh reverse 5223 - name: Run smoke test + id: smoke if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.backend.outcome != 'failure' && steps.reach.outcome != 'failure' }} env: BACKEND: ${{ steps.reach.outcome }} @@ -314,6 +315,20 @@ jobs: fi scripts/ci/android/smoke-test.sh "$RUNNER_TEMP/apk/$APK" "$RUNNER_TEMP/smoke-test" ${cases:-} + - name: Emulator state after a failed smoke test + if: ${{ !cancelled() && steps.smoke.outcome == 'failure' }} + run: | + adb devices + pgrep -af qemu-system || echo "no emulator process" + sudo dmesg | tail -n 40 + free -m + crashes=$(ls -d /tmp/android-"$USER"/emu-crash-* 2> /dev/null || true) + if [ -n "$crashes" ]; then + find $crashes -type f -exec ls -la {} + + mkdir -p "$RUNNER_TEMP/smoke-test/emulator-crashes" + cp -r $crashes "$RUNNER_TEMP/smoke-test/emulator-crashes/" + fi + - name: Stop test backend if: ${{ !cancelled() && steps.backend.outcome != 'skipped' }} run: scripts/ci/desktop-linux/backend.sh stop "$RUNNER_TEMP/backend" diff --git a/scripts/ci/android/README.md b/scripts/ci/android/README.md index 5b76b6d081..19e0e69bc1 100644 --- a/scripts/ci/android/README.md +++ b/scripts/ci/android/README.md @@ -20,7 +20,7 @@ Every run builds the core library, and the backend when it's asked for, from the | `release` (opt-in) | the latest release's arm64 APK, its signer checked against SimpleX's key, onboarding only | about 4 min | | `core-armv7a` (opt-in) | `build-core.sh` with `armv7a`: `lib:simplex-chat` and `lib:support` for armv7a-android, with GHC 8.10.7; the app and the emulator use only arm64 | not measured yet | -Results are artifacts: `android-smoke-test` (numbered screenshots, the accessibility tree and the text of each, `steps.log`, `checks.txt`, logcat as `app.log`, `emulator.log`), `android-release-smoke-test`, `android-claude-test`, and the backend's logs; `android-app`, the APK for manual tests and the Claude job, is kept 14 days. The job summaries list the checks and the Claude usage. +Results are artifacts: `android-smoke-test` (numbered screenshots, the accessibility tree and the text of each, `steps.log`, `checks.txt`, logcat as `app.log`, `emulator.log`, and after a failed test the emulator's crash reports in `emulator-crashes/`), `android-release-smoke-test`, `android-claude-test`, and the backend's logs; `android-app`, the APK for manual tests and the Claude job, is kept 14 days. The job summaries list the checks and the Claude usage. ## The emulator diff --git a/scripts/ci/desktop-linux/smoke-test.sh b/scripts/ci/desktop-linux/smoke-test.sh index 08d9242c23..c7f14f9b8a 100755 --- a/scripts/ci/desktop-linux/smoke-test.sh +++ b/scripts/ci/desktop-linux/smoke-test.sh @@ -231,6 +231,25 @@ create_local_chat() { "$ui" shot local-chat } +# The first item of the dropdown opened from the line at y $2, which repeats that line's value below it. +menu_first_item() { + local target line x y deadline=$((SECONDS + 10)) + target=$(normalise "$1") + while [ "$SECONDS" -lt "$deadline" ]; do + while IFS= read -r line; do + x=${line%%,*} + y=${line#*,} + y=${y%% *} + if [ "$x" -lt 390 ] && [ "$y" -gt $(($2 + 20)) ] && [ "$(normalise "${line#* }")" = "$target" ]; then + echo "$x $y" + return 0 + fi + done < <("$ui" text) + sleep 1 + done + return 1 +} + # Clicks a server role's value until its dropdown shows. The menu is open when an item reads exactly "yes", which no # role's value does ("default (yes)", "default (no)", "no"); it can take longer to show than OCR takes to read the screen. open_role_menu() { @@ -274,9 +293,11 @@ use_names_server() { # OCR misreads the short "no", the item after "yes" read -r x y < <(left_pane_position yes exact) || fail "the \"To receive\" menu closed" "$ui" click "$x" $((y + 48)) - open_role_menu "default (no)" || fail "no menu for the test server's \"To resolve names\"" - read -r x y < <(left_pane_position yes exact) || fail "the \"To resolve names\" menu closed" - "$ui" click "$x" "$y" + # this menu opens over the address's QR code, where OCR can miss "yes", so it is the row under the menu's first item + y=$(y_of "To resolve names") + "$ui" click-text "default (no)" 10 + read -r x y < <(menu_first_item "default (no)" "$y") || fail "no menu for the test server's \"To resolve names\"" + "$ui" click "$x" $((y + 48)) "$ui" shot names-server-roles # the first "default (yes)" and the first "yes" are on the lines after "To receive", so it is no in_line_with "default (yes)" "For private routing" || fail "the test server's \"To receive\" is still its default" diff --git a/scripts/ci/ios-sim/README.md b/scripts/ci/ios-sim/README.md index 82e8d99bcf..f8384412a2 100644 --- a/scripts/ci/ios-sim/README.md +++ b/scripts/ci/ios-sim/README.md @@ -38,7 +38,7 @@ Haiku 5.5 drives the app well: the smoke plan takes a few dozen turns and cents; ## Snapshots: `snapshots.sh` -At the end of the app job's smoke test, `smoke-test.sh` runs `snapshots.sh /snapshots` when the app it tested lies in the derived data of a build (`…/Build/Products/Debug-iphonesimulator`); the Claude job, which installs the app from a tarball, skips it. +At the end of the app job's smoke test, `smoke-test.sh` runs `snapshots.sh /snapshots` when the app it tested lies in the derived data of a build (`…/Build/Products/Debug-iphonesimulator`); the Claude job, which installs the app from a tarball, skips it. While the snapshots render, the smoke test removes its private routing setting (below), since the references show the default. `snapshots.sh` builds the unit-test target `SimpleX Snapshots` (`apps/ios/SimpleX Snapshots/SnapshotTests.swift`, hosted by the app) with `xcodebuild build-for-testing`, then runs it in the smoke test's simulator three times, after `xcrun simctl ui content_size` `large`, `extra-extra-extra-large` and `accessibility-medium`, and sets `large` back. The test hosts each preview listed in `previewScenes()` (the `previews` of a `PreviewProvider`, in a `VStack`) in a `UIHostingController` in a window of the screen's width and of the larger of two heights, the preview's ideal height and its height in a frame of the screen's size, in the light and dark themes without a wallpaper, ends text editing and removes the running animations, so a text cursor or a spinner is drawn the same in every run, and draws it with `drawHierarchy` at scale 1, one pixel per point. The chat item previews (`allSizes`) are rendered at the three sizes, the other previews at `large` only. @@ -63,7 +63,7 @@ When `apps/ios/SimpleX Snapshots/References` exists, `snapshots.sh` then runs `s On master's code it starts and passes these checks (the directory answers searches sent as messages); service requests to the directory, which only `f/directory-search` answers, are not checked here. -It writes what it seeded to `/seed.md` (the Claude sessions get it) and logs to `/*.log`; `stop`, `directory-stop`, `directory-start` and `listings` are its other commands. The app reaches it through the simulator, which shares the runner's network; pointing the app at the names server is a few steps in Settings, described in `claude-prompt.md`. +It writes what it seeded to `/seed.md` (the Claude sessions get it) and logs to `/*.log`; `stop`, `directory-stop`, `directory-start` and `listings` are its other commands. The app reaches it through the simulator, which shares the runner's network; pointing the app at the names server is a few steps in Settings, described in `claude-prompt.md`. Before the app first starts, the smoke test sets private routing to unknown servers only (`networkSMPProxyMode` `unknown` in the app group's defaults), since the preset proxies can't reach 127.0.0.1. ## Other files diff --git a/scripts/ci/ios-sim/smoke-test.sh b/scripts/ci/ios-sim/smoke-test.sh index 5e6682b2cc..b16425b28f 100755 --- a/scripts/ci/ios-sim/smoke-test.sh +++ b/scripts/ci/ios-sim/smoke-test.sh @@ -171,6 +171,10 @@ send_image() { } xcrun simctl install "$SIM_UDID" "$app" +# private routing to unknown servers only: the preset servers that would proxy name lookups can't reach the test SMP +# server on localhost +prefs=$(xcrun simctl get_app_container "$SIM_UDID" "$bundle_id" group.chat.simplex.app)/Library/Preferences/group.chat.simplex.app +xcrun simctl spawn "$SIM_UDID" defaults write "$prefs" networkSMPProxyMode -string unknown xcrun simctl launch "$SIM_UDID" "$bundle_id" # onboarding to the chat list @@ -210,7 +214,10 @@ echo "smoke test passed: onboarding reaches the chat list, which shows its onboa products=$(dirname "$app") if [[ "$products" == */Build/Products/Debug-iphonesimulator ]]; then + # the previews read the app's settings, and their references have the default private routing + xcrun simctl spawn "$SIM_UDID" defaults delete "$prefs" networkSMPProxyMode "$(dirname "$0")/snapshots.sh" "${products%/Build/Products/Debug-iphonesimulator}" "$UI_OUT/snapshots" + xcrun simctl spawn "$SIM_UDID" defaults write "$prefs" networkSMPProxyMode -string unknown fi if [ "$cases" = images ]; then