mirror of
https://github.com/simplex-chat/simplex-chat.git
synced 2026-11-22 17:48:15 +00:00
ci: app tests reach the test names server and keep emulator crash data (#7932)
* ci: pick the desktop test server's "To resolve names" role by the menu's first item
That menu opens over the server address's QR code, where OCR can miss its "yes" (run 39242250150 failed its setup three times on it), so the smoke test finds the menu's first item, which repeats the field's value and reads well, and clicks the row under it. The check that the role is now yes stays.
* ci: print the emulator's state and keep its crash reports when the Android smoke test fails
When the emulator dies during the smoke test, the step only says "no emulator detected" and the emulator's own log stops without a reason. After a failed smoke test the job now prints adb's devices, the emulator process, the kernel log's tail and free memory, and uploads the emulator's crash reports with the smoke test results.
* ci: the iOS simulator tests set private routing to unknown servers only, so the app can reach the test names server
Master's 3126d9b54 made the iOS app's default private routing "always". Name lookups then go through a preset server, which can't reach the test SMP server on the runner's localhost, so every iOS case that resolves a SimpleX name would fail. The desktop and Android tests already set "unknown" before the first launch; the iOS smoke test now writes the same into the app group's defaults after installing the app, and the app applies it when its chat starts.
* ci: iOS previews render default private routing in snapshot tests
* ci: keep open_role_menu's comment above it in the desktop smoke test
* ci: remove the Android emulator state step's comment
* ci: iOS and Android test guides describe private routing and crash reports
---------
Co-authored-by: spacedandy-agents-simplex <342056592+spacedandy-agents-simplex@users.noreply.github.com>
This commit is contained in:
1 parent
2c09d06793
commit
6e1e2deb82
5 files changed
+49
-6
No files matched your search
@@ -304,6 +304,7 @@ jobs:
|
||||
run: scripts/ci/android/emulator.sh reverse 5223
|
||||
|
||||
- name: Run smoke test
|
||||
id: smoke
|
||||
if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.backend.outcome != 'failure' && steps.reach.outcome != 'failure' }}
|
||||
env:
|
||||
BACKEND: ${{ steps.reach.outcome }}
|
||||
@@ -314,6 +315,20 @@ jobs:
|
||||
fi
|
||||
scripts/ci/android/smoke-test.sh "$RUNNER_TEMP/apk/$APK" "$RUNNER_TEMP/smoke-test" ${cases:-}
|
||||
|
||||
- name: Emulator state after a failed smoke test
|
||||
if: ${{ !cancelled() && steps.smoke.outcome == 'failure' }}
|
||||
run: |
|
||||
adb devices
|
||||
pgrep -af qemu-system || echo "no emulator process"
|
||||
sudo dmesg | tail -n 40
|
||||
free -m
|
||||
crashes=$(ls -d /tmp/android-"$USER"/emu-crash-* 2> /dev/null || true)
|
||||
if [ -n "$crashes" ]; then
|
||||
find $crashes -type f -exec ls -la {} +
|
||||
mkdir -p "$RUNNER_TEMP/smoke-test/emulator-crashes"
|
||||
cp -r $crashes "$RUNNER_TEMP/smoke-test/emulator-crashes/"
|
||||
fi
|
||||
|
||||
- name: Stop test backend
|
||||
if: ${{ !cancelled() && steps.backend.outcome != 'skipped' }}
|
||||
run: scripts/ci/desktop-linux/backend.sh stop "$RUNNER_TEMP/backend"
|
||||
|
||||
@@ -20,7 +20,7 @@ Every run builds the core library, and the backend when it's asked for, from the
|
||||
| `release` (opt-in) | the latest release's arm64 APK, its signer checked against SimpleX's key, onboarding only | about 4 min |
|
||||
| `core-armv7a` (opt-in) | `build-core.sh` with `armv7a`: `lib:simplex-chat` and `lib:support` for armv7a-android, with GHC 8.10.7; the app and the emulator use only arm64 | not measured yet |
|
||||
|
||||
Results are artifacts: `android-smoke-test` (numbered screenshots, the accessibility tree and the text of each, `steps.log`, `checks.txt`, logcat as `app.log`, `emulator.log`), `android-release-smoke-test`, `android-claude-test`, and the backend's logs; `android-app`, the APK for manual tests and the Claude job, is kept 14 days. The job summaries list the checks and the Claude usage.
|
||||
Results are artifacts: `android-smoke-test` (numbered screenshots, the accessibility tree and the text of each, `steps.log`, `checks.txt`, logcat as `app.log`, `emulator.log`, and after a failed test the emulator's crash reports in `emulator-crashes/`), `android-release-smoke-test`, `android-claude-test`, and the backend's logs; `android-app`, the APK for manual tests and the Claude job, is kept 14 days. The job summaries list the checks and the Claude usage.
|
||||
|
||||
## The emulator
|
||||
|
||||
|
||||
@@ -231,6 +231,25 @@ create_local_chat() {
|
||||
"$ui" shot local-chat
|
||||
}
|
||||
|
||||
# The first item of the dropdown opened from the line at y $2, which repeats that line's value below it.
|
||||
menu_first_item() {
|
||||
local target line x y deadline=$((SECONDS + 10))
|
||||
target=$(normalise "$1")
|
||||
while [ "$SECONDS" -lt "$deadline" ]; do
|
||||
while IFS= read -r line; do
|
||||
x=${line%%,*}
|
||||
y=${line#*,}
|
||||
y=${y%% *}
|
||||
if [ "$x" -lt 390 ] && [ "$y" -gt $(($2 + 20)) ] && [ "$(normalise "${line#* }")" = "$target" ]; then
|
||||
echo "$x $y"
|
||||
return 0
|
||||
fi
|
||||
done < <("$ui" text)
|
||||
sleep 1
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# Clicks a server role's value until its dropdown shows. The menu is open when an item reads exactly "yes", which no
|
||||
# role's value does ("default (yes)", "default (no)", "no"); it can take longer to show than OCR takes to read the screen.
|
||||
open_role_menu() {
|
||||
@@ -274,9 +293,11 @@ use_names_server() {
|
||||
# OCR misreads the short "no", the item after "yes"
|
||||
read -r x y < <(left_pane_position yes exact) || fail "the \"To receive\" menu closed"
|
||||
"$ui" click "$x" $((y + 48))
|
||||
open_role_menu "default (no)" || fail "no menu for the test server's \"To resolve names\""
|
||||
read -r x y < <(left_pane_position yes exact) || fail "the \"To resolve names\" menu closed"
|
||||
"$ui" click "$x" "$y"
|
||||
# this menu opens over the address's QR code, where OCR can miss "yes", so it is the row under the menu's first item
|
||||
y=$(y_of "To resolve names")
|
||||
"$ui" click-text "default (no)" 10
|
||||
read -r x y < <(menu_first_item "default (no)" "$y") || fail "no menu for the test server's \"To resolve names\""
|
||||
"$ui" click "$x" $((y + 48))
|
||||
"$ui" shot names-server-roles
|
||||
# the first "default (yes)" and the first "yes" are on the lines after "To receive", so it is no
|
||||
in_line_with "default (yes)" "For private routing" || fail "the test server's \"To receive\" is still its default"
|
||||
|
||||
@@ -38,7 +38,7 @@ Haiku 5.5 drives the app well: the smoke plan takes a few dozen turns and cents;
|
||||
|
||||
## Snapshots: `snapshots.sh`
|
||||
|
||||
At the end of the app job's smoke test, `smoke-test.sh` runs `snapshots.sh <derived data> <smoke test dir>/snapshots` when the app it tested lies in the derived data of a build (`…/Build/Products/Debug-iphonesimulator`); the Claude job, which installs the app from a tarball, skips it.
|
||||
At the end of the app job's smoke test, `smoke-test.sh` runs `snapshots.sh <derived data> <smoke test dir>/snapshots` when the app it tested lies in the derived data of a build (`…/Build/Products/Debug-iphonesimulator`); the Claude job, which installs the app from a tarball, skips it. While the snapshots render, the smoke test removes its private routing setting (below), since the references show the default.
|
||||
|
||||
`snapshots.sh` builds the unit-test target `SimpleX Snapshots` (`apps/ios/SimpleX Snapshots/SnapshotTests.swift`, hosted by the app) with `xcodebuild build-for-testing`, then runs it in the smoke test's simulator three times, after `xcrun simctl ui <udid> content_size` `large`, `extra-extra-extra-large` and `accessibility-medium`, and sets `large` back. The test hosts each preview listed in `previewScenes()` (the `previews` of a `PreviewProvider`, in a `VStack`) in a `UIHostingController` in a window of the screen's width and of the larger of two heights, the preview's ideal height and its height in a frame of the screen's size, in the light and dark themes without a wallpaper, ends text editing and removes the running animations, so a text cursor or a spinner is drawn the same in every run, and draws it with `drawHierarchy` at scale 1, one pixel per point. The chat item previews (`allSizes`) are rendered at the three sizes, the other previews at `large` only.
|
||||
|
||||
@@ -63,7 +63,7 @@ When `apps/ios/SimpleX Snapshots/References` exists, `snapshots.sh` then runs `s
|
||||
|
||||
On master's code it starts and passes these checks (the directory answers searches sent as messages); service requests to the directory, which only `f/directory-search` answers, are not checked here.
|
||||
|
||||
It writes what it seeded to `<work dir>/seed.md` (the Claude sessions get it) and logs to `<work dir>/*.log`; `stop`, `directory-stop`, `directory-start` and `listings` are its other commands. The app reaches it through the simulator, which shares the runner's network; pointing the app at the names server is a few steps in Settings, described in `claude-prompt.md`.
|
||||
It writes what it seeded to `<work dir>/seed.md` (the Claude sessions get it) and logs to `<work dir>/*.log`; `stop`, `directory-stop`, `directory-start` and `listings` are its other commands. The app reaches it through the simulator, which shares the runner's network; pointing the app at the names server is a few steps in Settings, described in `claude-prompt.md`. Before the app first starts, the smoke test sets private routing to unknown servers only (`networkSMPProxyMode` `unknown` in the app group's defaults), since the preset proxies can't reach 127.0.0.1.
|
||||
|
||||
## Other files
|
||||
|
||||
|
||||
@@ -171,6 +171,10 @@ send_image() {
|
||||
}
|
||||
|
||||
xcrun simctl install "$SIM_UDID" "$app"
|
||||
# private routing to unknown servers only: the preset servers that would proxy name lookups can't reach the test SMP
|
||||
# server on localhost
|
||||
prefs=$(xcrun simctl get_app_container "$SIM_UDID" "$bundle_id" group.chat.simplex.app)/Library/Preferences/group.chat.simplex.app
|
||||
xcrun simctl spawn "$SIM_UDID" defaults write "$prefs" networkSMPProxyMode -string unknown
|
||||
xcrun simctl launch "$SIM_UDID" "$bundle_id"
|
||||
|
||||
# onboarding to the chat list
|
||||
@@ -210,7 +214,10 @@ echo "smoke test passed: onboarding reaches the chat list, which shows its onboa
|
||||
|
||||
products=$(dirname "$app")
|
||||
if [[ "$products" == */Build/Products/Debug-iphonesimulator ]]; then
|
||||
# the previews read the app's settings, and their references have the default private routing
|
||||
xcrun simctl spawn "$SIM_UDID" defaults delete "$prefs" networkSMPProxyMode
|
||||
"$(dirname "$0")/snapshots.sh" "${products%/Build/Products/Debug-iphonesimulator}" "$UI_OUT/snapshots"
|
||||
xcrun simctl spawn "$SIM_UDID" defaults write "$prefs" networkSMPProxyMode -string unknown
|
||||
fi
|
||||
|
||||
if [ "$cases" = images ]; then
|
||||
|
||||
Reference in new issue
Block a user