From 51bcb8fdf62d2c6edaeb4255d83dfa882e663930 Mon Sep 17 00:00:00 2001 From: spacedandy-agents-simplex <342056592+spacedandy-agents-simplex@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:15:27 +0000 Subject: [PATCH] ci: let Claude run the full desktop search after a scripted setup, with Haiku 5.5 on Claude Code 2.1.296 The prompt runs the whole plan with the toolbar at the bottom and, at the top, only the checks the layouts render differently. --- .github/workflows/desktop-linux.yml | 19 +++++---- scripts/ci/desktop-linux/claude-test.md | 52 ++++++++++++++++++------- scripts/ci/desktop-linux/claude-test.sh | 4 +- 3 files changed, 52 insertions(+), 23 deletions(-) diff --git a/.github/workflows/desktop-linux.yml b/.github/workflows/desktop-linux.yml index 0562f755c9..528846eaa0 100644 --- a/.github/workflows/desktop-linux.yml +++ b/.github/workflows/desktop-linux.yml @@ -312,10 +312,10 @@ jobs: retention-days: 3 if-no-files-found: warn - # Claude Code repeats the smoke test's directory search with the ui helper. Each run spends the Claude + # Claude Code repeats the smoke test's searches with the ui helper. Each run spends the Claude # subscription's usage limits, so it runs only when dispatched or asked for in the head commit's message. claude: - name: Claude searches the directory in the desktop app + name: Claude searches in the desktop app needs: [plan, core, backend] if: ${{ !cancelled() && !failure() && (github.event_name == 'workflow_dispatch' || contains(github.event.head_commit.message, '[claude-test]')) }} runs-on: ubuntu-24.04 @@ -327,8 +327,9 @@ jobs: group: desktop-linux-claude-${{ github.ref }} cancel-in-progress: true env: - CLAUDE_CODE_VERSION: 2.1.292 - CLAUDE_MODEL: claude-sonnet-5-5 + # the first release with Haiku 5.5's token counts right (2.1.293 added the model, 2.1.296 fixed its counts) + CLAUDE_CODE_VERSION: 2.1.296 + CLAUDE_MODEL: claude-haiku-5-5 steps: - name: Checkout code uses: actions/checkout@v5 @@ -339,8 +340,8 @@ jobs: - name: Install tools run: | sudo apt-get update - sudo apt-get install -y xvfb xdotool openbox imagemagick tesseract-ocr sqlite3 - dpkg-query -W xvfb xdotool openbox imagemagick tesseract-ocr sqlite3 + sudo apt-get install -y xvfb xdotool openbox imagemagick tesseract-ocr sqlite3 fonts-dejavu-core + dpkg-query -W xvfb xdotool openbox imagemagick tesseract-ocr sqlite3 fonts-dejavu-core - name: Download core library uses: actions/download-artifact@v7 @@ -389,8 +390,10 @@ jobs: printf '#!/bin/sh\nexec %s/scripts/ci/desktop-linux/ui "$@"\n' "$PWD" | sudo tee /usr/local/bin/ui sudo chmod +x /usr/local/bin/ui - - name: Start the app at the chat list - run: scripts/ci/desktop-linux/smoke-test.sh apps/multiplatform/release/main/app/simplex "$RUNNER_TEMP/claude-test/screens" onboarding + - name: Set the app up at the chat list + run: | + export NAMES_SERVER=$(cat "$RUNNER_TEMP/backend/names-server") + scripts/ci/desktop-linux/smoke-test.sh apps/multiplatform/release/main/app/simplex "$RUNNER_TEMP/claude-test/screens" setup # Claude's Bash is limited to single ui commands by the guard hook in claude-test.sh. Claude Code's credential # scrub stays off: on Linux it forces its whole Bash sandbox (bubblewrap, socat), which ui isn't set up for. diff --git a/scripts/ci/desktop-linux/claude-test.md b/scripts/ci/desktop-linux/claude-test.md index da8feed9ec..2bc77de2a1 100644 --- a/scripts/ci/desktop-linux/claude-test.md +++ b/scripts/ci/desktop-linux/claude-test.md @@ -1,33 +1,59 @@ -# Test: directory search in the SimpleX Chat desktop app +# Test: online search in the SimpleX Chat desktop app You are testing the SimpleX Chat desktop app, which runs on a virtual display on this machine. Drive it only with the `ui` helper, through Bash, one command per call: `ui [args]`. Anything else is refused; don't look for ways around that, and if something blocks the test, report it. The commands are listed at the end. -Your working directory is where your output goes. `ui shot ` saves `screens/NN-.png` and prints the text read from it, one phrase per line after its centre `x,y` in screen pixels. Read the PNG to see the screen (1440×900, the app window at the top left). OCR can misread or split words; when the text and the picture disagree, trust the picture. +Your working directory is where your output goes. `ui shot ` saves `screens/NN-.png` and prints the text read from it, one phrase per line after its centre `x,y` in screen pixels. Read the PNG to see the screen (1440×900, the app window at the top left). OCR can misread or split words; when the text and the picture disagree, trust the picture. "Above" means a smaller y. ## What is set up -- The app has a profile and shows its chat list. The toolbar and the search field "Search or paste SimpleX link" are at the bottom of the left pane: the default "Reachable app toolbars" layout. -- The app uses a test directory that lists a group, #ci_group ("CI seed group", red avatar), and a channel, #ci_channel ("CI seed channel", green avatar). Their avatars are as large as the app makes them, so the directory returns one of them per response: a search first shows one result and "Show more", which loads the other. +- The app has a profile and one chat, a group of its own called "seeded notes". The chat list is in the left pane (x 1 to 389). Its toolbar and the search field "Search or paste SimpleX link" are at the bottom: the default "Reachable app toolbars" layout. +- A test directory lists a group, #ci_group ("CI seeded group", a red avatar with a white G), and a channel, #ci_channel ("CI seeded channel", a green avatar with a white C). Their avatars are as large as the app makes them, so the directory returns one of them per response: a search first shows one result and "Show more", which loads the other. +- A test server resolves the SimpleX name `seeded.testing` to the address of a contact, ci_contact ("CI seeded contact", a blue avatar with a white N), and the app uses only that server to resolve names. ## The test -Do it twice: first in the default layout, then with the toolbar at the top. +Run all of A and B in the default layout, with the toolbar at the bottom. Then move the toolbar to the top and repeat A and B with only the checks marked **(both layouts)**: the two layouts lay the list out differently, but draw the rows themselves the same way. + +**A. Search in Directory.** 1. Click the search field and type `ci seed`. A row "Search in Directory" appears; click it. -2. The alert "Search in Directory?" asks before searching. The first time, click "Search"; the second time, "Search and don't show again". +2. The alert "Search in Directory?" asks before searching. In the first layout click "Search"; in the second, "Search and don't show again". 3. Wait for results, up to 60 s. If "Search failed, tap to retry" shows, tap it once and note it. 4. Use "Show more" until both #ci_group and #ci_channel are listed, and note how many pages that took. 5. Check, noting the evidence (screenshot names, positions): - - each result's row shows its name, its description, its member or subscriber count, and "Join group" (#ci_group) or "Join channel" (#ci_channel); - - each shows its avatar: run `ui avatar "#ci_group" red` and `ui avatar "#ci_channel" green`, and look at the screenshot; - - the "Directory" header is above its results; - - no "Show more" is left, and nothing is cut off, overlapping or garbled. -6. After the first layout: press Escape to clear the search. Click the profile picture at the left end of the bottom toolbar (about 27,730) to open the menu, open "Settings", then "Appearance", and turn off the switch of "Reachable app toolbars", at the right end of its row. Press Escape until the chat list shows again; its search field should now be at the top. Then repeat steps 1 to 5. + - each result's row shows its name, its description, its count ("2 members" or "1 subscriber") and "Join group" (#ci_group) or "Join channel" (#ci_channel); + - the count's text is smaller than the description's above it; + - each shows its avatar, filling the picture: run `ui avatar "#ci_group" red` and `ui avatar "#ci_channel" green`, and look at the screenshot; + - **(both layouts)** the "Directory" header is above its results; + - **(both layouts)** no "Show more" is left, and nothing is cut off, overlapping or garbled. +6. Press Escape to clear the search field. -You have at most 60 turns. Chain steps whose outcome you can predict with `ui do`, for example `ui do click-text "Show more" -- wait "#ci_channel" 60 -- shot top-page-2`, and look at the screen after each chain. +**B. The full search.** + +1. Click the search field and type `seeded`. A row "Connect to seeded.testing" appears. +2. Press Enter (`ui key Return`): on desktop it is the Search key. In the first layout the same alert asks first: click "Search". In the second layout it should not ask. +3. Wait up to 60 s for the results, and use "Show more" until both directory entries are listed. +4. Check, noting the evidence: + - **(both layouts)** three parts: the local chat "seeded notes"; a "Names" header with one row for ci_contact, showing `(@seeded.testing)` and "CI seeded contact"; and the "Directory" header with #ci_group and #ci_channel; + - "Connect to seeded.testing" is gone once the name is found; + - ci_contact shows its avatar: `ui avatar "ci_contact" blue`; + - **(both layouts)** each header is above all of its part's rows; + - **(both layouts)** the order of the parts: with the toolbar at the bottom the list reads upwards from the search field (the local chat nearest the field, then Names, then Directory furthest up); with the toolbar at the top it reads downwards from the field in the same order. +5. Press Escape to clear the search field. + +**Between the layouts:** click the profile picture at the left end of the bottom toolbar (about 27,730) to open the menu, open "Settings", then "Appearance", and turn off the switch of "Reachable app toolbars", at the right end of its row (about x 335). Press Escape until the chat list shows again; its search field should now be at the top. + +You have at most 60 turns, and the whole test fits in about 40 if you chain steps whose outcome you can predict with `ui do` and look at the screen after each chain. For example, in the first layout: + +- `ui do click-text "Search or paste SimpleX link" -- type "ci seed" -- click-text "Search in Directory" -- wait "Search and don't show again" -- click-text "Search" -- wait "Join" 60 -- shot bottom-directory-1` +- `ui do click-text "Show more" -- wait "#ci_group" 60 -- shot bottom-directory-2` (wait for whichever result is missing) +- `ui do avatar "#ci_group" red -- avatar "#ci_channel" green` +- `ui do key Escape -- click-text "Search or paste SimpleX link" -- type seeded -- wait "Connect to seeded.testing" -- key Return -- wait "Search and don't show again" -- click-text "Search" -- wait ci_contact 60 -- shot bottom-full-1` + +`ui do` stops at the first command that fails, so a failed chain tells you where it stopped. ## Report -Write `report.md` in your working directory: for each layout, each check of step 5 as PASS or FAIL with its evidence, the number of pages and retries, and anything else unexpected. End your final message with one line: `CLAUDE-TEST: PASS` if every check passed in both layouts, otherwise `CLAUDE-TEST: FAIL` followed by the checks that failed. +Write `report.md` in your working directory: for each layout and each of A and B, each check you ran as PASS or FAIL with its evidence, the number of pages and retries, and anything else unexpected. End your final message with one line: `CLAUDE-TEST: PASS` if every check you ran passed, otherwise `CLAUDE-TEST: FAIL` followed by the checks that failed. ## The ui helper diff --git a/scripts/ci/desktop-linux/claude-test.sh b/scripts/ci/desktop-linux/claude-test.sh index 118a64e0d5..b23234844b 100755 --- a/scripts/ci/desktop-linux/claude-test.sh +++ b/scripts/ci/desktop-linux/claude-test.sh @@ -1,9 +1,9 @@ #!/bin/bash -# Runs Claude Code to repeat the directory search with the ui helper, as claude-test.md describes, and passes when +# Runs Claude Code to repeat the smoke test's searches with the ui helper, as claude-test.md describes, and passes when # its final message ends with "CLAUDE-TEST: PASS". # Usage: scripts/ci/desktop-linux/claude-test.sh -# Expects the app at the chat list (smoke-test.sh ... onboarding) with UI_OUT under , claude and ui on PATH, +# Expects the app set up at the chat list (smoke-test.sh ... setup) with UI_OUT under , claude and ui on PATH, # CLAUDE_MODEL and CLAUDE_CODE_OAUTH_TOKEN. Claude works in : the screenshots, its report.md and the # prompt it was given end up there. The transcript and the guard hook's log (.guard.log) are kept # outside it, where Claude can't write.