diff --git a/.github/workflows/android-emu.yml b/.github/workflows/android-emu.yml index 7617693b2b..55b5909afc 100644 --- a/.github/workflows/android-emu.yml +++ b/.github/workflows/android-emu.yml @@ -304,6 +304,7 @@ jobs: run: scripts/ci/android/emulator.sh reverse 5223 - name: Run smoke test + id: smoke if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.backend.outcome != 'failure' && steps.reach.outcome != 'failure' }} env: BACKEND: ${{ steps.reach.outcome }} @@ -314,6 +315,20 @@ jobs: fi scripts/ci/android/smoke-test.sh "$RUNNER_TEMP/apk/$APK" "$RUNNER_TEMP/smoke-test" ${cases:-} + - name: Emulator state after a failed smoke test + if: ${{ !cancelled() && steps.smoke.outcome == 'failure' }} + run: | + adb devices + pgrep -af qemu-system || echo "no emulator process" + sudo dmesg | tail -n 40 + free -m + crashes=$(ls -d /tmp/android-"$USER"/emu-crash-* 2> /dev/null || true) + if [ -n "$crashes" ]; then + find $crashes -type f -exec ls -la {} + + mkdir -p "$RUNNER_TEMP/smoke-test/emulator-crashes" + cp -r $crashes "$RUNNER_TEMP/smoke-test/emulator-crashes/" + fi + - name: Stop test backend if: ${{ !cancelled() && steps.backend.outcome != 'skipped' }} run: scripts/ci/desktop-linux/backend.sh stop "$RUNNER_TEMP/backend" diff --git a/scripts/ci/android/README.md b/scripts/ci/android/README.md index 5b76b6d081..19e0e69bc1 100644 --- a/scripts/ci/android/README.md +++ b/scripts/ci/android/README.md @@ -20,7 +20,7 @@ Every run builds the core library, and the backend when it's asked for, from the | `release` (opt-in) | the latest release's arm64 APK, its signer checked against SimpleX's key, onboarding only | about 4 min | | `core-armv7a` (opt-in) | `build-core.sh` with `armv7a`: `lib:simplex-chat` and `lib:support` for armv7a-android, with GHC 8.10.7; the app and the emulator use only arm64 | not measured yet | -Results are artifacts: `android-smoke-test` (numbered screenshots, the accessibility tree and the text of each, `steps.log`, `checks.txt`, logcat as `app.log`, `emulator.log`), `android-release-smoke-test`, `android-claude-test`, and the backend's logs; `android-app`, the APK for manual tests and the Claude job, is kept 14 days. The job summaries list the checks and the Claude usage. +Results are artifacts: `android-smoke-test` (numbered screenshots, the accessibility tree and the text of each, `steps.log`, `checks.txt`, logcat as `app.log`, `emulator.log`, and after a failed test the emulator's crash reports in `emulator-crashes/`), `android-release-smoke-test`, `android-claude-test`, and the backend's logs; `android-app`, the APK for manual tests and the Claude job, is kept 14 days. The job summaries list the checks and the Claude usage. ## The emulator diff --git a/scripts/ci/desktop-linux/smoke-test.sh b/scripts/ci/desktop-linux/smoke-test.sh index 08d9242c23..c7f14f9b8a 100755 --- a/scripts/ci/desktop-linux/smoke-test.sh +++ b/scripts/ci/desktop-linux/smoke-test.sh @@ -231,6 +231,25 @@ create_local_chat() { "$ui" shot local-chat } +# The first item of the dropdown opened from the line at y $2, which repeats that line's value below it. +menu_first_item() { + local target line x y deadline=$((SECONDS + 10)) + target=$(normalise "$1") + while [ "$SECONDS" -lt "$deadline" ]; do + while IFS= read -r line; do + x=${line%%,*} + y=${line#*,} + y=${y%% *} + if [ "$x" -lt 390 ] && [ "$y" -gt $(($2 + 20)) ] && [ "$(normalise "${line#* }")" = "$target" ]; then + echo "$x $y" + return 0 + fi + done < <("$ui" text) + sleep 1 + done + return 1 +} + # Clicks a server role's value until its dropdown shows. The menu is open when an item reads exactly "yes", which no # role's value does ("default (yes)", "default (no)", "no"); it can take longer to show than OCR takes to read the screen. open_role_menu() { @@ -274,9 +293,11 @@ use_names_server() { # OCR misreads the short "no", the item after "yes" read -r x y < <(left_pane_position yes exact) || fail "the \"To receive\" menu closed" "$ui" click "$x" $((y + 48)) - open_role_menu "default (no)" || fail "no menu for the test server's \"To resolve names\"" - read -r x y < <(left_pane_position yes exact) || fail "the \"To resolve names\" menu closed" - "$ui" click "$x" "$y" + # this menu opens over the address's QR code, where OCR can miss "yes", so it is the row under the menu's first item + y=$(y_of "To resolve names") + "$ui" click-text "default (no)" 10 + read -r x y < <(menu_first_item "default (no)" "$y") || fail "no menu for the test server's \"To resolve names\"" + "$ui" click "$x" $((y + 48)) "$ui" shot names-server-roles # the first "default (yes)" and the first "yes" are on the lines after "To receive", so it is no in_line_with "default (yes)" "For private routing" || fail "the test server's \"To receive\" is still its default" diff --git a/scripts/ci/ios-sim/README.md b/scripts/ci/ios-sim/README.md index 82e8d99bcf..f8384412a2 100644 --- a/scripts/ci/ios-sim/README.md +++ b/scripts/ci/ios-sim/README.md @@ -38,7 +38,7 @@ Haiku 5.5 drives the app well: the smoke plan takes a few dozen turns and cents; ## Snapshots: `snapshots.sh` -At the end of the app job's smoke test, `smoke-test.sh` runs `snapshots.sh /snapshots` when the app it tested lies in the derived data of a build (`…/Build/Products/Debug-iphonesimulator`); the Claude job, which installs the app from a tarball, skips it. +At the end of the app job's smoke test, `smoke-test.sh` runs `snapshots.sh /snapshots` when the app it tested lies in the derived data of a build (`…/Build/Products/Debug-iphonesimulator`); the Claude job, which installs the app from a tarball, skips it. While the snapshots render, the smoke test removes its private routing setting (below), since the references show the default. `snapshots.sh` builds the unit-test target `SimpleX Snapshots` (`apps/ios/SimpleX Snapshots/SnapshotTests.swift`, hosted by the app) with `xcodebuild build-for-testing`, then runs it in the smoke test's simulator three times, after `xcrun simctl ui content_size` `large`, `extra-extra-extra-large` and `accessibility-medium`, and sets `large` back. The test hosts each preview listed in `previewScenes()` (the `previews` of a `PreviewProvider`, in a `VStack`) in a `UIHostingController` in a window of the screen's width and of the larger of two heights, the preview's ideal height and its height in a frame of the screen's size, in the light and dark themes without a wallpaper, ends text editing and removes the running animations, so a text cursor or a spinner is drawn the same in every run, and draws it with `drawHierarchy` at scale 1, one pixel per point. The chat item previews (`allSizes`) are rendered at the three sizes, the other previews at `large` only. @@ -63,7 +63,7 @@ When `apps/ios/SimpleX Snapshots/References` exists, `snapshots.sh` then runs `s On master's code it starts and passes these checks (the directory answers searches sent as messages); service requests to the directory, which only `f/directory-search` answers, are not checked here. -It writes what it seeded to `/seed.md` (the Claude sessions get it) and logs to `/*.log`; `stop`, `directory-stop`, `directory-start` and `listings` are its other commands. The app reaches it through the simulator, which shares the runner's network; pointing the app at the names server is a few steps in Settings, described in `claude-prompt.md`. +It writes what it seeded to `/seed.md` (the Claude sessions get it) and logs to `/*.log`; `stop`, `directory-stop`, `directory-start` and `listings` are its other commands. The app reaches it through the simulator, which shares the runner's network; pointing the app at the names server is a few steps in Settings, described in `claude-prompt.md`. Before the app first starts, the smoke test sets private routing to unknown servers only (`networkSMPProxyMode` `unknown` in the app group's defaults), since the preset proxies can't reach 127.0.0.1. ## Other files diff --git a/scripts/ci/ios-sim/smoke-test.sh b/scripts/ci/ios-sim/smoke-test.sh index 5e6682b2cc..b16425b28f 100755 --- a/scripts/ci/ios-sim/smoke-test.sh +++ b/scripts/ci/ios-sim/smoke-test.sh @@ -171,6 +171,10 @@ send_image() { } xcrun simctl install "$SIM_UDID" "$app" +# private routing to unknown servers only: the preset servers that would proxy name lookups can't reach the test SMP +# server on localhost +prefs=$(xcrun simctl get_app_container "$SIM_UDID" "$bundle_id" group.chat.simplex.app)/Library/Preferences/group.chat.simplex.app +xcrun simctl spawn "$SIM_UDID" defaults write "$prefs" networkSMPProxyMode -string unknown xcrun simctl launch "$SIM_UDID" "$bundle_id" # onboarding to the chat list @@ -210,7 +214,10 @@ echo "smoke test passed: onboarding reaches the chat list, which shows its onboa products=$(dirname "$app") if [[ "$products" == */Build/Products/Debug-iphonesimulator ]]; then + # the previews read the app's settings, and their references have the default private routing + xcrun simctl spawn "$SIM_UDID" defaults delete "$prefs" networkSMPProxyMode "$(dirname "$0")/snapshots.sh" "${products%/Build/Products/Debug-iphonesimulator}" "$UI_OUT/snapshots" + xcrun simctl spawn "$SIM_UDID" defaults write "$prefs" networkSMPProxyMode -string unknown fi if [ "$cases" = images ]; then