From 35bc0b66d9ea10ce6838ce5b766ba68b838f551f Mon Sep 17 00:00:00 2001 From: spaced4ndy <8711996+spaced4ndy@users.noreply.github.com> Date: Mon, 19 Oct 2026 09:31:38 +0000 Subject: [PATCH] ci: app test workflows save no caches, reuse their builds as artifacts, and run Claude in its own job (#7767) --- .github/workflows/desktop-linux.yml | 220 +++++++++--------- .github/workflows/ios-sim.yml | 261 ++++++++++++++-------- scripts/ci/README.md | 37 +++ scripts/ci/desktop-linux/README.md | 35 +-- scripts/ci/desktop-linux/backend.sh | 37 +-- scripts/ci/desktop-linux/build-backend.sh | 1 - scripts/ci/desktop-linux/build-core.sh | 6 - scripts/ci/desktop-linux/claude-scope | 3 + scripts/ci/desktop-linux/claude-test.sh | 30 +-- scripts/ci/ios-sim/README.md | 36 +-- scripts/ci/ios-sim/backend.sh | 54 ++--- scripts/ci/ios-sim/build-backend.sh | 1 - scripts/ci/ios-sim/build-core.sh | 1 - scripts/ci/ios-sim/claude-prompt.md | 2 +- scripts/ci/ios-sim/claude-scope | 2 + scripts/ci/ios-sim/claude-test.sh | 28 ++- scripts/ci/ios-sim/claude-usage.sh | 20 +- scripts/ci/ios-sim/earlier-run.sh | 6 +- 18 files changed, 445 insertions(+), 335 deletions(-) create mode 100644 scripts/ci/README.md create mode 100644 scripts/ci/desktop-linux/claude-scope diff --git a/.github/workflows/desktop-linux.yml b/.github/workflows/desktop-linux.yml index f826ccbbfc..90a50f132c 100644 --- a/.github/workflows/desktop-linux.yml +++ b/.github/workflows/desktop-linux.yml @@ -15,7 +15,11 @@ on: type: boolean default: false backend: - description: Also start the test backend and test the released app + description: Also start the test backend + type: boolean + default: false + release: + description: Also test the latest released app, to tell a branch's failure from the runner's type: boolean default: false @@ -24,8 +28,8 @@ permissions: actions: read concurrency: - # a push that asks for no run doesn't cancel one that does - group: desktop-linux-${{ github.ref }}-${{ github.event_name == 'workflow_dispatch' || contains(github.event.head_commit.message, '[app-ci]') || contains(github.event.head_commit.message, '[app-ci:desktop]') || contains(github.event.head_commit.message, '[claude-test]') || contains(github.event.head_commit.message, '[claude-test:desktop]') }} + # a push that asks for no run must not cancel one that does + group: desktop-linux-${{ github.ref }}-${{ (github.event_name == 'workflow_dispatch' || contains(github.event.head_commit.message, '[app-ci]') || contains(github.event.head_commit.message, '[app-ci:desktop]') || contains(github.event.head_commit.message, '[claude-test]') || contains(github.event.head_commit.message, '[claude-test:desktop]')) && 'requested' || github.run_id }} cancel-in-progress: true env: @@ -38,6 +42,7 @@ env: jobs: plan: name: choose jobs, find earlier builds + # the concurrency group repeats this test, as it can't read job outputs if: >- github.event_name == 'workflow_dispatch' || contains(github.event.head_commit.message, '[app-ci]') || @@ -49,6 +54,10 @@ jobs: outputs: run-claude: ${{ steps.jobs.outputs.claude }} run-backend: ${{ steps.jobs.outputs.backend }} + run-release: ${{ steps.jobs.outputs.release }} + claude-backend: ${{ steps.scope.outputs.backend }} + core: ${{ steps.names.outputs.core }} + core-run: ${{ steps.core.outputs.run-id }} backend: ${{ steps.names.outputs.backend }} backend-run: ${{ steps.backend.outputs.run-id }} steps: @@ -57,19 +66,46 @@ jobs: env: CLAUDE: ${{ inputs.claude || contains(github.event.head_commit.message, '[claude-test]') || contains(github.event.head_commit.message, '[claude-test:desktop]') }} BACKEND: ${{ inputs.backend || contains(github.event.head_commit.message, '[ci-backend]') }} + RELEASE: ${{ inputs.release || contains(github.event.head_commit.message, '[ci-release]') }} run: | echo "claude=$CLAUDE" | tee -a "$GITHUB_OUTPUT" echo "backend=$BACKEND" | tee -a "$GITHUB_OUTPUT" + echo "release=$RELEASE" | tee -a "$GITHUB_OUTPUT" - name: Checkout code - if: steps.jobs.outputs.backend == 'true' - uses: actions/checkout@v5 + uses: actions/checkout@v6 + + - name: Claude scope + id: scope + if: steps.jobs.outputs.claude == 'true' + env: + BACKEND: ${{ steps.jobs.outputs.backend }} + run: | + backend=$(sed -n 's/^backend: *//p' scripts/ci/desktop-linux/claude-scope) + case "$backend" in + yes | no) echo "backend=$backend" | tee -a "$GITHUB_OUTPUT" ;; + *) + echo "::error::claude-scope must say backend: yes or backend: no" + exit 1 + ;; + esac + if [ "$backend" = yes ] && [ "$BACKEND" != true ]; then + echo "::error::claude-scope says the Claude session uses the test backend: ask for it too" + exit 1 + fi # artifacts are named after the sources they are built from, so a run uses one that an earlier run uploaded - name: Artifact names id: names - if: steps.jobs.outputs.backend == 'true' - run: echo "backend=desktop-linux-backend-${{ hashFiles('src/**', 'apps/simplex-chat/**', 'apps/simplex-directory-service/Main.hs', 'apps/simplex-directory-service/src/**', 'cabal.project', 'simplex-chat.cabal', 'scripts/ci/desktop-linux/build-backend.sh') }}" >> "$GITHUB_OUTPUT" + run: | + echo "core=desktop-linux-core-${{ hashFiles('src/**', 'cabal.project', 'simplex-chat.cabal', 'PRIVACY.md', 'scripts/desktop/build-lib-linux.sh', 'scripts/ci/desktop-linux/build-core.sh') }}" >> "$GITHUB_OUTPUT" + echo "backend=desktop-linux-backend-${{ hashFiles('src/**', 'apps/simplex-chat/**', 'apps/simplex-directory-service/Main.hs', 'apps/simplex-directory-service/src/**', 'cabal.project', 'simplex-chat.cabal', 'scripts/ci/desktop-linux/build-backend.sh') }}" >> "$GITHUB_OUTPUT" + + - name: Find core library + id: core + env: + GH_TOKEN: ${{ github.token }} + run: scripts/ci/desktop-linux/earlier-run.sh "${{ steps.names.outputs.core }}" - name: Find backend id: backend @@ -86,7 +122,7 @@ jobs: timeout-minutes: 150 steps: - name: Checkout code - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Show runner run: | @@ -96,26 +132,11 @@ jobs: df -h / - name: Setup Haskell - id: haskell uses: simplex-chat/setup-haskell-action@v2 with: ghc-version: ${{ env.GHC_VER }} cabal-version: ${{ env.CABAL_VER }} - # before the first backend store exists, the core library's has most of the same Hackage dependencies - - name: Restore cabal store - id: cabal-cache - uses: actions/cache/restore@v5 - with: - path: | - ${{ steps.haskell.outputs.cabal-store }} - dist-newstyle - key: desktop-linux-backend-store-ghc${{ env.GHC_VER }}-${{ hashFiles('cabal.project', 'simplex-chat.cabal') }}-${{ github.run_id }} - restore-keys: | - desktop-linux-backend-store-ghc${{ env.GHC_VER }}-${{ hashFiles('cabal.project', 'simplex-chat.cabal') }}- - desktop-linux-backend-store-ghc${{ env.GHC_VER }}- - desktop-linux-core-store-ghc${{ env.GHC_VER }}-${{ hashFiles('cabal.project', 'simplex-chat.cabal') }}- - - name: Install libraries run: | sudo apt-get update @@ -124,17 +145,8 @@ jobs: - name: Build backend run: scripts/ci/desktop-linux/build-backend.sh ~/backend/${{ env.BACKEND_PKG }} - - name: Save cabal store - if: ${{ !cancelled() }} - uses: actions/cache/save@v5 - with: - path: | - ${{ steps.haskell.outputs.cabal-store }} - dist-newstyle - key: ${{ steps.cabal-cache.outputs.cache-primary-key }} - - name: Upload backend - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v7 with: name: ${{ needs.plan.outputs.backend }} path: ~/backend/${{ env.BACKEND_PKG }} @@ -145,11 +157,12 @@ jobs: core: name: core library for Linux needs: plan + if: needs.plan.outputs.core-run == '' runs-on: ubuntu-24.04 timeout-minutes: 150 steps: - name: Checkout code - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Show runner run: | @@ -158,79 +171,39 @@ jobs: free -h df -h / - - name: Restore core library - id: core-cache - uses: actions/cache/restore@v5 - with: - path: ~/core/${{ env.CORE_PKG }} - key: desktop-linux-core-${{ hashFiles('src/**', 'cabal.project', 'simplex-chat.cabal', 'PRIVACY.md', 'scripts/desktop/build-lib-linux.sh', 'scripts/ci/desktop-linux/build-core.sh') }} - - name: Setup Haskell - id: haskell - if: steps.core-cache.outputs.cache-hit != 'true' uses: simplex-chat/setup-haskell-action@v2 with: ghc-version: ${{ env.GHC_VER }} cabal-version: ${{ env.CABAL_VER }} - - name: Restore cabal store - id: cabal-cache - if: steps.core-cache.outputs.cache-hit != 'true' - uses: actions/cache/restore@v5 - with: - path: | - ${{ steps.haskell.outputs.cabal-store }} - dist-newstyle - key: desktop-linux-core-store-ghc${{ env.GHC_VER }}-${{ hashFiles('cabal.project', 'simplex-chat.cabal') }}-${{ github.run_id }} - restore-keys: | - desktop-linux-core-store-ghc${{ env.GHC_VER }}-${{ hashFiles('cabal.project', 'simplex-chat.cabal') }}- - desktop-linux-core-store-ghc${{ env.GHC_VER }}- - - name: Install libraries - if: steps.core-cache.outputs.cache-hit != 'true' run: | sudo apt-get update sudo apt-get install -y libffi-dev libgmp-dev libnuma-dev libssl-dev zlib1g-dev patchelf - name: Build core library - if: steps.core-cache.outputs.cache-hit != 'true' run: scripts/ci/desktop-linux/build-core.sh ~/core/${{ env.CORE_PKG }} - - name: Save cabal store - if: ${{ !cancelled() && steps.core-cache.outputs.cache-hit != 'true' }} - uses: actions/cache/save@v5 - with: - path: | - ${{ steps.haskell.outputs.cabal-store }} - dist-newstyle - key: ${{ steps.cabal-cache.outputs.cache-primary-key }} - - - name: Save core library - if: steps.core-cache.outputs.cache-hit != 'true' - uses: actions/cache/save@v5 - with: - path: ~/core/${{ env.CORE_PKG }} - key: ${{ steps.core-cache.outputs.cache-primary-key }} - - name: Upload core library - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v7 with: - name: desktop-core-linux-x86_64 + name: ${{ needs.plan.outputs.core }} path: ~/core/${{ env.CORE_PKG }} compression-level: 0 - retention-days: 3 + retention-days: 30 if-no-files-found: error app: name: desktop app under Xvfb needs: [plan, core, backend] - # backend is skipped when it isn't asked for or an earlier run built it + # core and backend are skipped when an earlier run built them, and backend also when the run doesn't start it if: ${{ !cancelled() && !failure() && needs.plan.result == 'success' }} runs-on: ubuntu-24.04 timeout-minutes: 60 steps: - name: Checkout code - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: submodules: recursive @@ -250,10 +223,12 @@ jobs: dpkg-query -W xvfb xdotool openbox imagemagick tesseract-ocr sqlite3 fonts-dejavu-core - name: Download core library - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v8 with: - name: desktop-core-linux-x86_64 + name: ${{ needs.plan.outputs.core }} path: ~/core + run-id: ${{ needs.plan.outputs.core-run || github.run_id }} + github-token: ${{ github.token }} - name: Install core library id: core @@ -261,7 +236,7 @@ jobs: - name: Download backend if: needs.plan.outputs.run-backend == 'true' - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v8 with: name: ${{ needs.plan.outputs.backend }} path: ~/backend @@ -281,7 +256,6 @@ jobs: with: distribution: corretto java-version: ${{ env.JAVA_VER }} - cache: gradle - name: Build app id: build @@ -294,6 +268,21 @@ jobs: du -sh "$app" LD_LIBRARY_PATH="$app/lib/app/resources" ldd "$app/lib/app/resources/libapp-lib.so" + # for the Claude job; a tarball keeps the app's file modes, which an artifact drops + - name: Pack app + if: ${{ !cancelled() && steps.build.outcome == 'success' && needs.plan.outputs.run-claude == 'true' }} + run: tar -czf "$RUNNER_TEMP/desktop-app.tar.gz" -C apps/multiplatform/release/main/app simplex + + - name: Upload app + if: ${{ !cancelled() && steps.build.outcome == 'success' && needs.plan.outputs.run-claude == 'true' }} + uses: actions/upload-artifact@v7 + with: + name: desktop-app + path: ${{ runner.temp }}/desktop-app.tar.gz + compression-level: 0 + retention-days: 1 + if-no-files-found: error + - name: Run desktop tests if: ${{ !cancelled() && steps.build.outcome == 'success' }} working-directory: apps/multiplatform @@ -320,7 +309,7 @@ jobs: - name: Upload smoke test results if: ${{ !cancelled() }} - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v7 with: name: desktop-smoke-test path: ${{ runner.temp }}/smoke-test @@ -329,7 +318,7 @@ jobs: - name: Upload desktop test report if: ${{ !cancelled() }} - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v7 with: name: desktop-test-report path: | @@ -340,7 +329,7 @@ jobs: - name: Upload backend logs if: ${{ !cancelled() && steps.backend.outcome != 'skipped' }} - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v7 with: name: desktop-backend-logs path: ${{ runner.temp }}/backend/*.log @@ -352,8 +341,8 @@ jobs: # may use the token. claude: name: Claude tests the desktop app - needs: [plan, core, backend] - if: ${{ !cancelled() && !failure() && needs.plan.outputs.run-claude == 'true' }} + needs: [plan, app] + if: ${{ !cancelled() && needs.plan.outputs.run-claude == 'true' && needs.app.result == 'success' }} runs-on: ubuntu-24.04 timeout-minutes: 60 environment: @@ -363,14 +352,13 @@ jobs: group: desktop-linux-claude-${{ github.ref }} cancel-in-progress: true env: - # the first release with Haiku 5.5's token counts right (2.1.293 added the model, 2.1.296 fixed its counts) - CLAUDE_CODE_VERSION: 2.1.296 + # the same in both app workflows + CLAUDE_CODE_VERSION: 2.1.300 CLAUDE_MODEL: claude-haiku-5-5 steps: - name: Checkout code - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: - submodules: recursive persist-credentials: false - name: Install tools @@ -379,18 +367,20 @@ jobs: sudo apt-get install -y xvfb xdotool openbox imagemagick tesseract-ocr sqlite3 fonts-dejavu-core dpkg-query -W xvfb xdotool openbox imagemagick tesseract-ocr sqlite3 fonts-dejavu-core - - name: Download core library - uses: actions/download-artifact@v7 + - name: Download app + uses: actions/download-artifact@v8 with: - name: desktop-core-linux-x86_64 - path: ~/core + name: desktop-app + path: ${{ runner.temp }} - - name: Install core library - run: scripts/ci/desktop-linux/install-core.sh ~/core/${{ env.CORE_PKG }} + - name: Unpack app + run: | + mkdir -p "$RUNNER_TEMP/app" + tar -xzf "$RUNNER_TEMP/desktop-app.tar.gz" -C "$RUNNER_TEMP/app" - name: Download backend - if: needs.plan.outputs.run-backend == 'true' - uses: actions/download-artifact@v7 + if: needs.plan.outputs.claude-backend == 'yes' + uses: actions/download-artifact@v8 with: name: ${{ needs.plan.outputs.backend }} path: ~/backend @@ -398,21 +388,11 @@ jobs: github-token: ${{ github.token }} - name: Start test backend - if: needs.plan.outputs.run-backend == 'true' + id: backend + if: needs.plan.outputs.claude-backend == 'yes' timeout-minutes: 30 run: scripts/ci/desktop-linux/backend.sh start ~/backend/${{ env.BACKEND_PKG }} "$RUNNER_TEMP/backend" - - name: Setup Java - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ env.JAVA_VER }} - cache: gradle - - - name: Build app - working-directory: apps/multiplatform - run: ./gradlew --no-daemon --console=plain -Psimplex.assets.dir=../../assets :desktop:createDistributable - # ui goes on every shell's PATH, where Claude's Bash tool finds it whatever environment it builds - name: Install Claude Code and the ui command run: | @@ -422,7 +402,7 @@ jobs: sudo chmod +x /usr/local/bin/ui - name: Start the app - run: scripts/ci/desktop-linux/smoke-test.sh apps/multiplatform/release/main/app/simplex "$RUNNER_TEMP/claude-test/screens" start + run: scripts/ci/desktop-linux/smoke-test.sh "$RUNNER_TEMP/app/simplex" "$RUNNER_TEMP/claude-test/screens" start # Claude's Bash is limited to single ui commands by the guard hook in claude-test.sh. Claude Code's credential # scrub stays off: on Linux it forces its whole Bash sandbox (bubblewrap, socat), which ui isn't set up for. @@ -438,12 +418,12 @@ jobs: run: scripts/ci/desktop-linux/claude-usage.py "$RUNNER_TEMP/claude-transcript.jsonl" "$RUNNER_TEMP/claude-usage.json" - name: Stop test backend - if: ${{ !cancelled() && needs.plan.outputs.run-backend == 'true' }} + if: ${{ !cancelled() && steps.backend.outcome != 'skipped' }} run: scripts/ci/desktop-linux/backend.sh stop "$RUNNER_TEMP/backend" - name: Upload Claude test results if: ${{ !cancelled() }} - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v7 with: name: desktop-claude-test path: | @@ -456,8 +436,8 @@ jobs: if-no-files-found: warn - name: Upload backend logs - if: ${{ !cancelled() && needs.plan.outputs.run-backend == 'true' }} - uses: actions/upload-artifact@v6 + if: ${{ !cancelled() && steps.backend.outcome != 'skipped' }} + uses: actions/upload-artifact@v7 with: name: desktop-claude-backend-logs path: ${{ runner.temp }}/backend/*.log @@ -468,12 +448,12 @@ jobs: release: name: released app under Xvfb needs: plan - if: needs.plan.outputs.run-backend == 'true' + if: needs.plan.outputs.run-release == 'true' runs-on: ubuntu-24.04 timeout-minutes: 30 steps: - name: Checkout code - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: fetch-depth: 0 filter: tree:0 @@ -492,7 +472,7 @@ jobs: - name: Upload smoke test results if: ${{ !cancelled() }} - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v7 with: name: desktop-release-smoke-test path: ${{ runner.temp }}/smoke-test diff --git a/.github/workflows/ios-sim.yml b/.github/workflows/ios-sim.yml index 5d44162369..210a939fbc 100644 --- a/.github/workflows/ios-sim.yml +++ b/.github/workflows/ios-sim.yml @@ -39,25 +39,55 @@ env: jobs: plan: - name: find earlier builds + name: choose jobs, find earlier builds + # the concurrency group repeats this test, as it can't read job outputs if: >- github.event_name == 'workflow_dispatch' || contains(github.event.head_commit.message, '[app-ci]') || contains(github.event.head_commit.message, '[app-ci:ios]') || contains(github.event.head_commit.message, '[claude-test]') || contains(github.event.head_commit.message, '[claude-test:ios]') - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 10 outputs: + run-claude: ${{ steps.jobs.outputs.claude }} + run-backend: ${{ steps.jobs.outputs.backend }} + claude-backend: ${{ steps.scope.outputs.backend }} core: ${{ steps.names.outputs.core }} - backend: ${{ steps.names.outputs.backend }} core-run: ${{ steps.core.outputs.run-id }} + backend: ${{ steps.names.outputs.backend }} backend-run: ${{ steps.backend.outputs.run-id }} - claude: ${{ inputs.claude || contains(github.event.head_commit.message, '[claude-test]') || contains(github.event.head_commit.message, '[claude-test:ios]') }} - start-backend: ${{ inputs.backend || contains(github.event.head_commit.message, '[ci-backend]') }} steps: + - name: Choose jobs + id: jobs + env: + CLAUDE: ${{ inputs.claude || contains(github.event.head_commit.message, '[claude-test]') || contains(github.event.head_commit.message, '[claude-test:ios]') }} + BACKEND: ${{ inputs.backend || contains(github.event.head_commit.message, '[ci-backend]') }} + run: | + echo "claude=$CLAUDE" | tee -a "$GITHUB_OUTPUT" + echo "backend=$BACKEND" | tee -a "$GITHUB_OUTPUT" + - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v6 + + - name: Claude scope + id: scope + if: steps.jobs.outputs.claude == 'true' + env: + BACKEND: ${{ steps.jobs.outputs.backend }} + run: | + backend=$(sed -n 's/^backend: *//p' scripts/ci/ios-sim/claude-scope) + case "$backend" in + yes | no) echo "backend=$backend" | tee -a "$GITHUB_OUTPUT" ;; + *) + echo "::error::claude-scope must say backend: yes or backend: no" + exit 1 + ;; + esac + if [ "$backend" = yes ] && [ "$BACKEND" != true ]; then + echo "::error::claude-scope says the Claude sessions use the test backend: ask for it too" + exit 1 + fi # artifacts are named after the sources they are built from, so a run uses one that an earlier run uploaded - name: Artifact names @@ -74,6 +104,7 @@ jobs: - name: Find backend id: backend + if: steps.jobs.outputs.backend == 'true' env: GH_TOKEN: ${{ github.token }} run: scripts/ci/ios-sim/earlier-run.sh "${{ steps.names.outputs.backend }}" @@ -82,11 +113,11 @@ jobs: name: core library for the iOS simulator needs: plan if: needs.plan.outputs.core-run == '' - runs-on: macos-latest + runs-on: macos-26 timeout-minutes: 240 steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Show runner run: | @@ -100,29 +131,12 @@ jobs: - name: Use the macOS 26 SDK run: scripts/ci/ios-sim/use-macos26-sdk.sh - # without ~/.cabal, cabal 3.10 keeps its store in XDG directories, not where setup-haskell reports it - - name: Use ~/.cabal for cabal - run: echo "CABAL_DIR=$HOME/.cabal" >> "$GITHUB_ENV" - - name: Setup Haskell - id: haskell uses: simplex-chat/setup-haskell-action@v2 with: ghc-version: ${{ env.GHC_VER }} cabal-version: ${{ env.CABAL_VER }} - - name: Restore cabal store - id: cabal-cache - uses: actions/cache/restore@v4 - with: - path: | - ${{ steps.haskell.outputs.cabal-store }} - dist-newstyle - key: ios-sim-core-store-ghc${{ env.GHC_VER }}-${{ hashFiles('cabal.project', 'simplex-chat.cabal') }}-${{ github.run_id }} - restore-keys: | - ios-sim-core-store-ghc${{ env.GHC_VER }}-${{ hashFiles('cabal.project', 'simplex-chat.cabal') }}- - ios-sim-core-store-ghc${{ env.GHC_VER }}- - - name: Install libraries run: brew install openssl@3 libffi gmp @@ -130,20 +144,11 @@ jobs: id: build run: scripts/ci/ios-sim/build-core.sh ~/Downloads/${{ env.CORE_PKG }} - - name: Save cabal store - if: ${{ !cancelled() }} - uses: actions/cache/save@v4 - with: - path: | - ${{ steps.haskell.outputs.cabal-store }} - dist-newstyle - key: ${{ steps.cabal-cache.outputs.cache-primary-key }} - - name: Patch core library for the simulator run: scripts/ci/ios-sim/prepare-libs.sh - name: Upload core library - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: ${{ needs.plan.outputs.core }} path: ~/Downloads/${{ env.CORE_PKG }} @@ -154,12 +159,12 @@ jobs: backend: name: test backend for the app needs: plan - if: needs.plan.outputs.start-backend == 'true' && needs.plan.outputs.backend-run == '' - runs-on: macos-latest + if: needs.plan.outputs.run-backend == 'true' && needs.plan.outputs.backend-run == '' + runs-on: macos-26 timeout-minutes: 240 steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Show runner run: | @@ -171,42 +176,17 @@ jobs: - name: Use the macOS 26 SDK run: scripts/ci/ios-sim/use-macos26-sdk.sh - - name: Use ~/.cabal for cabal - run: echo "CABAL_DIR=$HOME/.cabal" >> "$GITHUB_ENV" - - name: Setup Haskell - id: haskell uses: simplex-chat/setup-haskell-action@v2 with: ghc-version: ${{ env.GHC_VER }} cabal-version: ${{ env.CABAL_VER }} - - name: Restore cabal store - id: cabal-cache - uses: actions/cache/restore@v4 - with: - path: | - ${{ steps.haskell.outputs.cabal-store }} - dist-newstyle - key: ios-sim-backend-store-ghc${{ env.GHC_VER }}-${{ hashFiles('cabal.project', 'simplex-chat.cabal') }}-${{ github.run_id }} - restore-keys: | - ios-sim-backend-store-ghc${{ env.GHC_VER }}-${{ hashFiles('cabal.project', 'simplex-chat.cabal') }}- - ios-sim-backend-store-ghc${{ env.GHC_VER }}- - - name: Build backend run: scripts/ci/ios-sim/build-backend.sh ~/Downloads/${{ env.BACKEND_PKG }} - - name: Save cabal store - if: ${{ !cancelled() }} - uses: actions/cache/save@v4 - with: - path: | - ${{ steps.haskell.outputs.cabal-store }} - dist-newstyle - key: ${{ steps.cabal-cache.outputs.cache-primary-key }} - - name: Upload backend - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: ${{ needs.plan.outputs.backend }} path: ~/Downloads/${{ env.BACKEND_PKG }} @@ -219,20 +199,13 @@ jobs: needs: [plan, core, backend] # core and backend are skipped when an earlier run built them, and backend also when the run doesn't start it if: ${{ !cancelled() && !failure() && needs.plan.result == 'success' }} - runs-on: macos-latest - environment: - name: ${{ needs.plan.outputs.claude == 'true' && 'app-sim-claude' || '' }} - deployment: false - timeout-minutes: 330 + runs-on: macos-26 + timeout-minutes: 120 env: - CLAUDE_RUN: ${{ needs.plan.outputs.claude }} - START_BACKEND: ${{ needs.plan.outputs.start-backend }} + START_BACKEND: ${{ needs.plan.outputs.run-backend }} steps: - # the job later holds the Claude token, so the checkout's GitHub token is not left in .git/config - name: Checkout code - uses: actions/checkout@v4 - with: - persist-credentials: false + uses: actions/checkout@v6 - name: Show runner run: | @@ -247,7 +220,7 @@ jobs: run: scripts/ci/ios-sim/use-macos26-sdk.sh - name: Download core library - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: ${{ needs.plan.outputs.core }} path: ~/Downloads @@ -261,7 +234,7 @@ jobs: - name: Download backend if: env.START_BACKEND == 'true' - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: ${{ needs.plan.outputs.backend }} path: ~/Downloads @@ -290,37 +263,34 @@ jobs: DEVELOPMENT_TEAM= \ PROVISIONING_PROFILE_SPECIFIER= + # for the Claude job; a tarball keeps the app's file modes, which an artifact drops + - name: Pack app + if: needs.plan.outputs.run-claude == 'true' + run: tar -czf "$RUNNER_TEMP/SimpleX.app.tar.gz" -C "$RUNNER_TEMP/DerivedData/Build/Products/Debug-iphonesimulator" SimpleX.app + + - name: Upload app + if: needs.plan.outputs.run-claude == 'true' + uses: actions/upload-artifact@v7 + with: + name: ios-sim-app + path: ${{ runner.temp }}/SimpleX.app.tar.gz + compression-level: 0 + retention-days: 1 + if-no-files-found: error + - name: Install AXe run: scripts/ci/ios-sim/install-axe.sh - name: Run smoke test run: scripts/ci/ios-sim/smoke-test.sh "$RUNNER_TEMP/DerivedData/Build/Products/Debug-iphonesimulator/SimpleX.app" "$RUNNER_TEMP/smoke-test" - - name: Install Claude Code - if: env.CLAUDE_RUN == 'true' - run: | - npm install -g @anthropic-ai/claude-code@2.1.300 - claude --version - - - name: Run the device test plan with Claude - if: env.CLAUDE_RUN == 'true' - timeout-minutes: 240 - env: - CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} - BACKEND_DIR: ${{ env.START_BACKEND == 'true' && format('{0}/backend', runner.temp) || '' }} - run: scripts/ci/ios-sim/claude-test.sh "$RUNNER_TEMP/claude-test" - - - name: Report Claude usage - if: ${{ always() && env.CLAUDE_RUN == 'true' }} - run: scripts/ci/ios-sim/claude-usage.sh "$RUNNER_TEMP/claude-test" - - name: Stop test backend if: ${{ !cancelled() && env.START_BACKEND == 'true' }} run: scripts/ci/ios-sim/backend.sh stop "$RUNNER_TEMP/backend" - name: Upload smoke test results if: ${{ !cancelled() }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: ios-sim-smoke-test path: ${{ runner.temp }}/smoke-test @@ -329,7 +299,7 @@ jobs: - name: Upload backend logs if: ${{ !cancelled() && env.START_BACKEND == 'true' }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: ios-sim-backend-logs path: | @@ -338,11 +308,104 @@ jobs: retention-days: 3 if-no-files-found: warn + # The app-sim-claude environment's branch rule decides which branches may use the token. + claude: + name: Claude tests the iOS app + needs: [plan, app] + if: ${{ !cancelled() && needs.plan.outputs.run-claude == 'true' && needs.app.result == 'success' }} + runs-on: macos-26 + timeout-minutes: 330 + environment: + name: app-sim-claude + deployment: false + concurrency: + group: ios-sim-claude-${{ github.ref }} + cancel-in-progress: true + env: + # the same in both app workflows + CLAUDE_CODE_VERSION: 2.1.300 + steps: + # the job holds the Claude token, so the checkout's GitHub token is not left in .git/config + - name: Checkout code + uses: actions/checkout@v6 + with: + persist-credentials: false + + - name: Show runner + run: | + sw_vers + xcodebuild -version + xcrun simctl list runtimes + + - name: Use the macOS 26 SDK + run: scripts/ci/ios-sim/use-macos26-sdk.sh + + - name: Download app + uses: actions/download-artifact@v8 + with: + name: ios-sim-app + path: ${{ runner.temp }} + + - name: Unpack app + run: tar -xzf "$RUNNER_TEMP/SimpleX.app.tar.gz" -C "$RUNNER_TEMP" + + - name: Download backend + if: needs.plan.outputs.claude-backend == 'yes' + uses: actions/download-artifact@v8 + with: + name: ${{ needs.plan.outputs.backend }} + path: ~/Downloads + run-id: ${{ needs.plan.outputs.backend-run || github.run_id }} + github-token: ${{ github.token }} + + - name: Start test backend + id: backend + if: needs.plan.outputs.claude-backend == 'yes' + timeout-minutes: 30 + run: scripts/ci/ios-sim/backend.sh start ~/Downloads/${{ env.BACKEND_PKG }} "$RUNNER_TEMP/backend" + + - name: Install AXe + run: scripts/ci/ios-sim/install-axe.sh + + - name: Run smoke test + run: scripts/ci/ios-sim/smoke-test.sh "$RUNNER_TEMP/SimpleX.app" "$RUNNER_TEMP/claude-test/smoke-test" + + - name: Install Claude Code + run: | + npm install -g "@anthropic-ai/claude-code@$CLAUDE_CODE_VERSION" + claude --version + + - name: Run the device test plan with Claude + timeout-minutes: 240 + env: + CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + BACKEND_DIR: ${{ steps.backend.outcome == 'success' && format('{0}/backend', runner.temp) || '' }} + run: scripts/ci/ios-sim/claude-test.sh "$RUNNER_TEMP/claude-test" + + - name: Report Claude usage + if: ${{ always() }} + run: scripts/ci/ios-sim/claude-usage.sh "$RUNNER_TEMP/claude-test" + + - name: Stop test backend + if: ${{ !cancelled() && steps.backend.outcome != 'skipped' }} + run: scripts/ci/ios-sim/backend.sh stop "$RUNNER_TEMP/backend" + - name: Upload Claude test results - if: ${{ always() && env.CLAUDE_RUN == 'true' }} - uses: actions/upload-artifact@v4 + if: ${{ always() }} + uses: actions/upload-artifact@v7 with: name: ios-sim-claude-test path: ${{ runner.temp }}/claude-test retention-days: 14 if-no-files-found: warn + + - name: Upload backend logs + if: ${{ !cancelled() && steps.backend.outcome != 'skipped' }} + uses: actions/upload-artifact@v7 + with: + name: ios-sim-claude-backend-logs + path: | + ${{ runner.temp }}/backend/*.log + ${{ runner.temp }}/backend/seed.md + retention-days: 3 + if-no-files-found: warn diff --git a/scripts/ci/README.md b/scripts/ci/README.md new file mode 100644 index 0000000000..edc3569c38 --- /dev/null +++ b/scripts/ci/README.md @@ -0,0 +1,37 @@ +# App tests on GitHub's runners + +Two workflows build the apps from a branch's own code and test them, only when asked: `ios-sim` (the iOS app in a simulator on a macOS runner) and `desktop-linux` (the desktop app on a Linux runner). Each runs a scripted smoke test and, on request, a Claude session that follows a test plan, and a test backend (relay, directory, names server, peers). Details: [iOS](ios-sim/README.md), [desktop](desktop-linux/README.md). + +## Run tags + +Put a tag in the message of the commit you push. Tags combine: each workflow runs everything its tags ask for. + +| Tag | iOS | Desktop | +|---|---|---| +| `[app-ci]` | scripted test | scripted test | +| `[app-ci:ios]` | scripted test | — | +| `[app-ci:desktop]` | — | scripted test | +| `[claude-test]` | scripted test, then Claude | scripted test, then Claude | +| `[claude-test:ios]` | scripted test, then Claude | — | +| `[claude-test:desktop]` | — | scripted test, then Claude | +| `[ci-backend]`, with one of the above | also the test backend | also the test backend | +| `[ci-release]`, with one of the above | — | also the latest release's onboarding, to tell a branch's failure from the runner's | + +Only the pushed head commit counts, and a tag counts anywhere in its message, body included: never write one in a message unless that push should run it. A push that asks for nothing starts no job and doesn't cancel a running test; a newer requested run on the same branch cancels the older one. Pushes to master and stable never start a run, because a squash merge's message lists the merged commits' subjects with their tags; run it by hand there. + +## Running a workflow by hand + +Actions → `ios-sim` or `desktop-linux` → Run workflow, on any branch, with checkboxes that are off by default: `claude` (the Claude session), `backend` (the test backend) and, on desktop, `release` (the latest release's onboarding). + +## How long a run takes + +| Run | iOS | Desktop | +|---|---|---| +| Scripted test, builds reused | about 25 min, plus the wait for a macOS runner, which the team's builds share (minutes to an hour) | about 10 min | +| The core rebuilt, after Haskell changes | 30–45 min more | about 25 min more | +| With the test backend | 20–45 min more when it is rebuilt, then about 10 min to start | about 30 min more when it is rebuilt, then a few minutes to start | +| With Claude on the smoke plan | 10–15 min more | about 4 min more | + +## The Claude job + +It needs the GitHub environment `app-sim-claude` with the secret `CLAUDE_CODE_OAUTH_TOKEN`, a Claude subscription token. The environment's branch rule alone decides which branches may use it: on a branch it doesn't allow, a run that asks for Claude fails at the Claude job's environment check, and the scripted test runs as usual. Each run spends the subscription's usage limits; the job summary gives its tokens and cost estimated at API prices. diff --git a/scripts/ci/desktop-linux/README.md b/scripts/ci/desktop-linux/README.md index a9080e1c87..f84c42b8c7 100644 --- a/scripts/ci/desktop-linux/README.md +++ b/scripts/ci/desktop-linux/README.md @@ -4,42 +4,29 @@ ## Running it -On any branch, in either of two ways: +The workflow runs only when asked, on any branch, by a run tag in the pushed commit's message or by hand from the Actions tab: [App tests on GitHub's runners](../README.md) lists the tags and inputs. -- **Dispatch:** Actions → desktop-linux → Run workflow, with the `claude` and `backend` checkboxes (both off by default). -- **Push** with a tag in the head commit's message, which is how agents in push sessions ask for a run: - -| Tag | Runs | -|---|---| -| `[app-ci]` | the scripted test, here and in the iOS app's workflow where the branch has it | -| `[app-ci:desktop]` | the scripted test, here only | -| `[claude-test]` | the scripted test and the Claude job, here and in the iOS app's workflow where the branch has it | -| `[claude-test:desktop]` | the scripted test and the Claude job, here only | -| `[ci-backend]` | with one of the above: also the test backend and the release job | - -A push without these tags (or with only iOS tags) starts a run whose jobs are all skipped, and it doesn't cancel a run in progress. A newer run that was asked for cancels an older one on the same branch. Pushes to master and stable never start it, because a squash merge's message lists the merged commits' subjects with their tags; dispatch it there instead. - -Every run builds the core library, and the backend when it's asked for, from the branch's own Haskell code. It reuses an earlier build only when the sources are unchanged: the core by a cache keyed on `src/**`, `cabal.project`, `simplex-chat.cabal`, `PRIVACY.md` and the build scripts; the backend by an artifact of the same branch named after a hash of its sources (30 days). So a branch that changes core and app code together tests both. +Every run builds the core library, and the backend when it's asked for, from the branch's own Haskell code. It reuses an earlier build only when the sources are unchanged, as an artifact of the same branch named after a hash of its sources (30 days): for the core `src/**`, `cabal.project`, `simplex-chat.cabal`, `PRIVACY.md` and the build scripts, for the backend `src/**`, the CLI's and the directory's sources, `cabal.project`, `simplex-chat.cabal` and its build script. So a branch that changes core and app code together tests both. The workflow saves no Actions caches, which this repository evicts within hours and shares with `build.yml`. ## Jobs | Job | Does | Time | |---|---|---| -| `plan` | chooses the jobs, finds an earlier backend build | 10 s | -| `core` | `scripts/desktop/build-lib-linux.sh` via `build-core.sh` | 15 s cached, about 23 min not | -| `app` | the app (`:desktop:createDistributable`), desktopTest, `smoke-test.sh` | about 10 min: build 4.5, desktopTest 0.5, smoke test 3.5 | -| `claude` (opt-in) | the app at its first screen, then Claude Code drives it | about 7 min, Claude 2.5 | -| `backend` (opt-in) | `build-backend.sh`: `simplex-chat`, `simplex-directory-service`, and `smp-server` from the simplexmq commit `cabal.project` pins | about 27 min, skipped when reused | +| `plan` | chooses the jobs, finds earlier core and backend builds | 10 s | +| `core` | `scripts/desktop/build-lib-linux.sh` via `build-core.sh` | skipped when reused, about 24 min not | +| `app` | the app (`:desktop:createDistributable`), desktopTest, `smoke-test.sh` | about 12 min: build 4.5, desktopTest 0.5, smoke test 3.5 | +| `claude` (opt-in) | after `app` passed: its distributable at the first screen, then Claude Code drives it | about 4 min, Claude 3 | +| `backend` (opt-in) | `build-backend.sh`: `simplex-chat`, `simplex-directory-service`, and `smp-server` from the simplexmq commit `cabal.project` pins | about 30 min, skipped when reused | | `release` (opt-in) | the latest release's .deb for this Ubuntu, onboarding only, to tell a branch's failure from the runner's | 1.5 min | -Queueing for a runner adds from under a minute to 20 minutes. Results are artifacts: `desktop-smoke-test` (numbered screenshots, the text OCR read from each, `steps.log`, `checks.txt`, `app.log`), `desktop-test-report`, `desktop-claude-test`, and the backend's logs. The job summaries list the checks, the desktopTest results and the Claude usage. +Queueing for a runner adds from under a minute to 20 minutes. Results are artifacts: `desktop-smoke-test` (numbered screenshots, the text OCR read from each, `steps.log`, `checks.txt`, `app.log`), `desktop-test-report`, `desktop-claude-test`, and the backend's logs; `desktop-app`, the distributable for the Claude job, is kept a day. The job summaries list the checks, the desktopTest results and the Claude usage. ## Scripts - `ui`: the UI driver. Compose for Desktop has no accessibility tree on Linux, so it finds text on screenshots by OCR (tesseract) and acts with xdotool on an Xvfb display. `ui` alone prints its commands: start (with app settings), shot, text, find, wait, click, click-text, scroll, type, key, avatar (a profile image's colour), ink (a text's height), do (several in a row), stop. `screen-text.py`, `screen-colour.py` and `screen-ink.py` are its helpers. - `smoke-test.sh [start|onboarding|setup|backend]`: the scripted cases; the default is onboarding and both layouts. `setup` also creates a local group and makes the test SMP server the app's only names server; `backend` does that, connects to the seeded name, and checks both layouts. - `backend.sh start ` / `stop `: the test backend, below. -- `claude-test.sh`, `claude-test.md` (the prompt), `claude-guard.py`, `claude-usage.py`: the Claude job. +- `claude-test.sh`, `claude-test.md` (the prompt), `claude-scope` (whether the prompt uses the test backend), `claude-guard.py`, `claude-usage.py`: the Claude job. - `build-core.sh`, `install-core.sh`, `build-backend.sh`, `earlier-run.sh`, `release-app.sh`, `test-summary.py`: builds, reuse and reports. ## The test backend @@ -54,8 +41,8 @@ It writes the directory's address to `/directory-link` and the SMP server's ## The Claude job -It needs the GitHub environment `app-sim-claude` with the secret `CLAUDE_CODE_OAUTH_TOKEN`, a Claude subscription token. Which branches may use it is decided only by that environment's branch rule in the repository settings: on a branch the rule doesn't allow, a requested Claude run fails at the environment check. The token goes only to the step that runs `claude`, and every `ui` call unsets it. With `[ci-backend]` the job also starts the test backend for prompts that use it, and waits for its build; this smoke path's prompt doesn't use it. The job installs Claude Code 2.1.296 and runs `claude-haiku-5-5` with `--max-turns 60` and `--max-budget-usd 3`. Its Bash runs only single `ui` commands (`claude-guard.py`, a PreToolUse hook: Claude Code runs read-only commands without asking in every permission mode). Before anything the session wrote is printed or uploaded, `claude-test.sh` checks it for the token; if it's there, the output, transcript, Claude Code's stderr and the guard log are deleted and the job fails. It passes when Claude's final message ends with `CLAUDE-TEST: PASS`; `report.md`, the transcript, the guard's log and `claude-usage.json` are in `desktop-claude-test`. +It needs the GitHub environment `app-sim-claude` with the secret `CLAUDE_CODE_OAUTH_TOKEN`, a Claude subscription token. Which branches may use it is decided only by that environment's branch rule in the repository settings: on a branch the rule doesn't allow, a requested Claude run fails at the environment check. It runs once the app job has passed, on that job's distributable, one per branch at a time. The token goes only to the step that runs `claude`; `claude-test.sh` passes it to the `claude` process alone, and every `ui` call unsets it too. The job starts the test backend when `claude-scope` says `backend: yes`; a run that asks for Claude must then ask for the backend too, or the plan job fails. On master it says `no`, since this prompt doesn't use the backend. The job installs Claude Code at the version the iOS workflow pins too, and runs `claude-haiku-5-5` with `--max-turns 60`, `--max-budget-usd 3` and the iOS sessions' other flags (`--restricted`, `--permission-prompts none`, `--effort high`). Its Bash runs only single `ui` commands (`claude-guard.py`, a PreToolUse hook: Claude Code runs read-only commands without asking in every permission mode). Before anything the session wrote is printed or uploaded, `claude-test.sh` checks it for the token; if it's there, the output, transcript, Claude Code's stderr and the guard log are deleted and the job fails. It passes when Claude's final message ends with `CLAUDE-TEST: PASS`; `report.md`, the transcript, the guard's log and `claude-usage.json` are in `desktop-claude-test`. ## Cost -The runners are free for this public repository. A Claude run spends the subscription's usage limits, which interactive sessions share: Haiku 5.5 does this smoke path in 15 to 17 turns, about 0.008 USD estimated at API prices (Claude Code's estimate, not a bill on a subscription). +The runners are free for this public repository. A Claude run spends the subscription's usage limits, which interactive sessions share: Haiku 5.5 does this smoke path in 15 to 30 turns, 0.01 to 0.02 USD estimated at API prices (Claude Code's estimate, not a bill on a subscription). diff --git a/scripts/ci/desktop-linux/backend.sh b/scripts/ci/desktop-linux/backend.sh index 836f09d275..842b3c890e 100755 --- a/scripts/ci/desktop-linux/backend.sh +++ b/scripts/ci/desktop-linux/backend.sh @@ -120,21 +120,27 @@ create_directory() { } start_directory() { + local pid deadline=$((SECONDS + 60)) # -y: the service adds its own tables to the database the CLI created, and would otherwise ask on stdin nohup "$bin/simplex-directory-service" -d "$work/directory" -y --super-users "$1" < /dev/null >> "$work/directory.log" 2>&1 & - echo $! > "$work/directory.pid" - sleep 5 - if ! kill -0 "$(cat "$work/directory.pid")" 2> /dev/null; then - echo "error: the directory service exited" - cat "$work/directory.log" - return 1 - fi + pid=$! + echo "$pid" > "$work/directory.pid" + # its output is block-buffered, so it shows that its chat has started by connecting to its servers + until grep -q "pid=$pid," <<< "$(ss -Htnp state established)"; do + if ! kill -0 "$pid" 2> /dev/null || [ "$SECONDS" -ge "$deadline" ]; then + echo "error: the directory service did not start" + cat "$work/directory.log" + return 1 + fi + sleep 1 + done } -stop_directory() { +# Waits for the process to exit, which completes its output, and kills it after 30 s. +stop_process() { local pid deadline - pid=$(cat "$work/directory.pid") - kill -INT "$pid" 2> /dev/null || true + pid=$(cat "$work/$1.pid") + kill "-$2" "$pid" 2> /dev/null || true deadline=$((SECONDS + 30)) while kill -0 "$pid" 2> /dev/null; do if [ "$SECONDS" -ge "$deadline" ]; then @@ -152,7 +158,7 @@ connect_admin() { admin_until "Welcome to $directory_name" 180 "/tail @'$directory_name' 10" local id id=$(db_query directory "SELECT contact_id FROM contacts WHERE local_display_name = '$admin'") - stop_directory + stop_process directory INT start_directory "$id:$admin" echo "directory restarted with super user $id:$admin" } @@ -311,12 +317,15 @@ start() { stop() { local p - for p in directory relay smp-server resolver; do + for p in directory relay smp-server; do if [ -f "$work/$p.pid" ]; then - kill -INT "$(cat "$work/$p.pid")" 2> /dev/null || true + stop_process "$p" INT fi done - sleep 3 + # Python, started in the background by a script, ignores SIGINT + if [ -f "$work/resolver.pid" ]; then + stop_process resolver TERM + fi for p in directory relay smp-server resolver; do echo "::group::$p log" cat "$work/$p.log" 2> /dev/null || true diff --git a/scripts/ci/desktop-linux/build-backend.sh b/scripts/ci/desktop-linux/build-backend.sh index adda90dcfe..fbe20bad8e 100755 --- a/scripts/ci/desktop-linux/build-backend.sh +++ b/scripts/ci/desktop-linux/build-backend.sh @@ -13,7 +13,6 @@ out="$(cd "$(dirname "$1")" && pwd)/$(basename "$1")" ghc --version cabal --version -du -sh ~/.cabal/store dist-newstyle 2> /dev/null || true cat > cabal.project.local << EOF ignore-project: False diff --git a/scripts/ci/desktop-linux/build-core.sh b/scripts/ci/desktop-linux/build-core.sh index 13ab52eb30..cda413e9f1 100755 --- a/scripts/ci/desktop-linux/build-core.sh +++ b/scripts/ci/desktop-linux/build-core.sh @@ -8,13 +8,10 @@ set -euo pipefail mkdir -p "$(dirname "$1")" out="$(cd "$(dirname "$1")" && pwd)/$(basename "$1")" arch=$(uname -m) -ghc_ver=$(ghc --numeric-version) -version=$(sed -n 's/^version: *//p' simplex-chat.cabal) libs_dir=apps/multiplatform/common/src/commonMain/cpp/desktop/libs/linux-$arch ghc --version cabal --version -du -sh ~/.local/state/cabal/store ~/.cabal/store dist-newstyle 2> /dev/null || true cat > cabal.project.local << EOF ignore-project: False @@ -24,9 +21,6 @@ package direct-sqlcipher flags: +openssl EOF -# a restored dist-newstyle can hold an older version's build directory, which breaks the script's simplex-chat-* glob -find "dist-newstyle/build/$arch-linux/ghc-$ghc_ver" -maxdepth 1 -name 'simplex-chat-*' ! -name "simplex-chat-$version" -exec rm -rf {} + 2> /dev/null || true - SKIP_VLC=1 scripts/desktop/build-lib-linux.sh ls -la "$libs_dir" diff --git a/scripts/ci/desktop-linux/claude-scope b/scripts/ci/desktop-linux/claude-scope new file mode 100644 index 0000000000..ca4c2da0f9 --- /dev/null +++ b/scripts/ci/desktop-linux/claude-scope @@ -0,0 +1,3 @@ +# What the Claude job needs for the session claude-test.md describes. +# backend: yes when the prompt uses the test backend; a run that asks for Claude must then start it too. +backend: no diff --git a/scripts/ci/desktop-linux/claude-test.sh b/scripts/ci/desktop-linux/claude-test.sh index d254226b75..12fde73a05 100755 --- a/scripts/ci/desktop-linux/claude-test.sh +++ b/scripts/ci/desktop-linux/claude-test.sh @@ -11,7 +11,9 @@ set -euo pipefail -: "${CLAUDE_CODE_OAUTH_TOKEN:?CLAUDE_CODE_OAUTH_TOKEN must be set}" +token=${CLAUDE_CODE_OAUTH_TOKEN:?CLAUDE_CODE_OAUTH_TOKEN must be set} +# only the claude process gets the token, nothing else this script starts +unset CLAUDE_CODE_OAUTH_TOKEN here=$(cd "$(dirname "$0")" && pwd) out=$(cd "$1" && pwd) transcript=$2 @@ -35,26 +37,28 @@ EOF claude --version cd "$out" status=0 -claude -p "$(cat prompt.md)" \ +CLAUDE_CODE_OAUTH_TOKEN=$token claude -p \ --model "$CLAUDE_MODEL" \ + --effort high \ + --restricted \ + --tools Bash,Read,Write,Edit \ + --allowedTools "Bash(ui)" "Bash(ui *)" "Read(/$out/**)" "Edit(/$out/**)" \ + --permission-mode dontAsk \ + --permission-prompts none \ + --strict-mcp-config \ --max-turns 60 \ --max-budget-usd 3 \ - --output-format stream-json \ - --verbose \ - --tools Bash,Read,Write \ - --permission-mode dontAsk \ - --allowedTools "Bash(ui *)" "Read(/$out/**)" "Edit(/$out/**)" \ - --disallowedTools "mcp__*" \ - --strict-mcp-config \ - --disable-slash-commands \ + --autocompact 200k \ --no-session-persistence \ --settings "$settings" \ - < /dev/null > "$transcript" 2> "$stderr" || status=$? + --output-format stream-json \ + --verbose \ + < prompt.md > "$transcript" 2> "$stderr" || status=$? # A second safeguard to the file tools' limits: what the session wrote is printed and uploaded only when it is known # not to hold the token. grep reads the token from a pipe, so it is in no process's arguments. leak=0 -printf '%s\n' "$CLAUDE_CODE_OAUTH_TOKEN" | grep -rqF -f - -- "$out" "$transcript" "$stderr" "$CLAUDE_GUARD_LOG" || leak=$? +printf '%s\n' "$token" | grep -rqF -f - -- "$out" "$transcript" "$stderr" "$CLAUDE_GUARD_LOG" || leak=$? if [ "$leak" -ne 1 ]; then rm -rf -- "$out" "$transcript" "$stderr" "$CLAUDE_GUARD_LOG" echo "error: the token is in the Claude session's output, or checking for it failed (grep exited $leak); the output, transcript, stderr and guard log are deleted" @@ -78,4 +82,4 @@ if [ "$status" -ne 0 ]; then exit 1 fi last=$(grep -v '^[[:space:]]*$' <<< "$result" | tail -n 1 || true) -grep -qxE '[*`]*CLAUDE-TEST: PASS[*`]*' <<< "$last" +grep -qxE '[*`]*CLAUDE-TEST: PASS[*`.]*[[:space:]]*' <<< "$last" diff --git a/scripts/ci/ios-sim/README.md b/scripts/ci/ios-sim/README.md index dd2c37c030..d84fd7d9e0 100644 --- a/scripts/ci/ios-sim/README.md +++ b/scripts/ci/ios-sim/README.md @@ -4,33 +4,33 @@ ## Running it -The workflow runs only when asked, on any branch: +The workflow runs only when asked, on any branch, by a run tag in the pushed commit's message or by hand from the Actions tab: [App tests on GitHub's runners](../README.md) lists the tags and inputs. -- **Dispatch** it from the Actions tab (workflow "ios-sim", "Run workflow"), with the inputs `claude` (also run the Claude session) and `backend` (also start the test backend), both off by default. -- **Push** a commit whose message carries a tag: `[app-ci]` runs the scripted tests of both apps (iOS and desktop), `[app-ci:ios]` only this one; `[claude-test]` (both apps) or `[claude-test:ios]` runs the scripted test and then the Claude session; add `[ci-backend]` to build and start the test backend. Only the pushed head commit counts, and a tag matches anywhere in its message, body included, so don't quote a tag in a message that shouldn't run it. - -A push without a tag starts nothing and doesn't cancel a running test; a newer requested run on the same branch cancels the older one. Pushes to master and stable never start it, because a squash merge's message lists the merged commits' subjects with their tags; dispatch it there instead. Which branches may run the Claude session is decided only by the `app-sim-claude` environment's branch rule in the repository settings: on a branch it doesn't allow, a run that asks for Claude fails at the environment check, and because the app job holds the environment, its scripted test fails with it. Ask for Claude only on branches the rule allows. - -Results: the job log, the job summary (Claude's usage) and the artifacts `ios-sim-smoke-test` (a screenshot, a 1000-px view and an accessibility tree per step, the app log), `ios-sim-backend-logs` and `ios-sim-claude-test` (per session: `prompt.md`, `report.md`, `shots/`, `transcript.jsonl`, `usage.json`). Read them with `gh run view --log` and `gh run download -n `. +Results: the job logs, the Claude job's summary (Claude's usage) and the artifacts `ios-sim-smoke-test` (a screenshot, a 1000-px view and an accessibility tree per step, the app log), `ios-sim-backend-logs` and `ios-sim-claude-test` (the Claude job's smoke test in `smoke-test/`, and per session: `prompt.md`, `report.md`, `shots/`, `transcript.jsonl`, `usage.json`). Read them with `gh run view --log` and `gh run download -n `. ## How long it takes, what it costs -- `plan` (ubuntu, 15 s) names the core library and the backend after their sources, `hashFiles` over the Haskell sources, `cabal.project`, `simplex-chat.cabal` and the build script, and reuses an artifact of the same name that an earlier run of the same branch uploaded (30 days). So every run builds the core library, and the backend when asked, from the branch's own Haskell code, and rebuilds only when those sources change. -- `core` (macOS, 35–45 min when not reused): `build-core.sh` builds `lib:simplex-chat` with cabal and the iOS flags, as the Nix iOS build does, into the package `scripts/ios/prepare.sh` expects, and `prepare-libs.sh` marks it for the simulator (`mac2ios`). -- `backend` (macOS, 20–30 min when not reused, only when asked): `build-backend.sh` builds `simplex-chat`, `simplex-directory-service` and `smp-server`. -- `app` (macOS): the Xcode build 8–22 min, the smoke test about 10 min, the backend start about 10 min when asked, Claude as long as its plan needs. +- `plan` (ubuntu-24.04, 10 s) reads what the run asks for, names the core library and the backend after their sources, `hashFiles` over the Haskell sources, `cabal.project`, `simplex-chat.cabal` and the build script, and reuses an artifact of the same name that an earlier run of the same branch uploaded (30 days). So every run builds the core library, and the backend when asked, from the branch's own Haskell code, and rebuilds only when those sources change. +- `core` (macOS, 30–45 min when not reused): `build-core.sh` builds `lib:simplex-chat` with cabal and the iOS flags, as the Nix iOS build does, into the package `scripts/ios/prepare.sh` expects, and `prepare-libs.sh` marks it for the simulator (`mac2ios`). +- `backend` (macOS, 20–45 min when not reused, only when asked): `build-backend.sh` builds `simplex-chat`, `simplex-directory-service` and `smp-server`. +- `app` (macOS): the Xcode build 8–22 min, the smoke test about 10 min, the backend start about 10 min when asked. +- `claude` (macOS, only when asked, after `app` passed): a new simulator with the app job's build and the smoke test, about 7 min, then the sessions (the smoke plan's took 2.6 min). -A run with reused builds and no extras takes about 26 minutes plus the wait for a macOS runner. Hosted minutes are free for this public repository; the limit is the repository's five concurrent macOS jobs, shared with PR builds (each `build.yml` run takes two), so queue waits can be long. Caches are evicted within hours (the repository is over its cache limit); the artifacts are what make runs fast. +A run with reused builds and no extras takes about 25 minutes plus the wait for a macOS runner. Hosted minutes are free for this public repository; the limit is the repository's five concurrent macOS jobs, shared with PR builds (each `build.yml` run takes two), so queue waits can be long. The workflow saves no Actions caches, which this repository evicts within hours and shares with `build.yml`: builds are reused only as the artifacts above, and a core or backend rebuild compiles its dependencies too. + +The macOS jobs run on `macos-26`, the image with Xcode 26 and the iPhone 17 simulator; `macos-latest` moves to each new macOS and its Xcode. ## The Claude session -It runs after the smoke test, in the same job, on the simulator the smoke test leaves running, once per layout listed in `claude-scope`, with Claude Code installed from npm at a pinned version. The job then references the environment `app-sim-claude`, which holds the secret `CLAUDE_CODE_OAUTH_TOKEN` (a Claude subscription token) and admits the branches its rule allows; jobs without Claude reference no environment. Only the Claude step gets the token, and `claude-test.sh` passes it to the `claude` processes alone. The job checks out without persisting its GitHub token and installs a pinned AXe (`install-axe.sh`), since it later holds the Claude token. +It runs in its own job, `claude`, once the app job has passed: the job installs that job's build (the artifact `ios-sim-app`, kept a day) in a new simulator, runs the smoke test to reach the sessions' starting state, then one session per layout listed in `claude-scope`, with Claude Code installed from npm at the version the desktop workflow pins too. Only this job references the environment `app-sim-claude`, which holds the secret `CLAUDE_CODE_OAUTH_TOKEN` (a Claude subscription token) and admits the branches its rule allows; one runs per branch at a time. Only the Claude step gets the token, and `claude-test.sh` passes it to the `claude` processes alone. The job checks out without persisting its GitHub token and installs a pinned AXe (`install-axe.sh`), since it holds the Claude token. It starts the test backend when `claude-scope` says `backend: yes`; a run that asks for Claude must then ask for the backend too, or the plan job fails. On master it says `no`, since the smoke plan doesn't use the backend. -- `claude-scope`: the model, the layouts, the cases per layout, optional entries per layout, `max-turns` and `max-budget-usd` per session. +Each session's final message ends with `CLAUDE-TEST: PASS`, or `CLAUDE-TEST: FAIL` and the failed cases; the step fails unless every session passes. + +- `claude-scope`: whether the cases use the test backend, the model, the layouts, the cases per layout, optional entries per layout, `max-turns` and `max-budget-usd` per session. - `device-test-plan.md`: the plan (`| ID | Do | Expect |` tables). On master it is a three-case smoke plan; a feature branch replaces it with its own and names its cases in `claude-scope`. - `claude-prompt.md`: what any session needs to know about the app, the `ui` helper and the test backend; `claude-test.sh` adds the `ui` help, the session's cases, layout, start state and limits, the seeded backend and the plan. -- `claude-guard.py`: a PreToolUse hook that lets through only one plain `ui` command per Bash call, since read-only commands run without asking in every permission mode and Bash inherits the token. Read, Write and Edit are allowed only in the session's own directory, so a session can't change `ui` or the scripts it runs. After each session `claude-test.sh` deletes the session's files if the token appears in them. -- `claude-usage.sh`: each session's turns, duration, tokens and cost "estimated at API prices" (Claude Code's own estimate, not a bill) to the job summary and `claude-usage.json`. +- `claude-guard.py`: a PreToolUse hook that lets through only one plain `ui` command per Bash call, since read-only commands run without asking in every permission mode and Bash inherits the token. Read, Write and Edit are allowed only in the session's own directory, so a session can't change `ui` or the scripts it runs. After each session `claude-test.sh` deletes the session's files, and fails, if the token appears in them or the check for it fails. +- `claude-usage.sh`: each session's turns, duration, tokens and cost "estimated at API prices" (Claude Code's own estimate, not a bill) to the job summary and `claude-usage.json`, or why they are unknown (a session cut short before its token check is not uploaded). Haiku 5.5 drives the app well: the smoke plan takes a few dozen turns and cents; a 50-case feature plan took 300–450 turns, 30–60 minutes and $1.1–1.7 per session, estimated at API prices. Every run spends the subscription's usage limits, which interactive sessions share. @@ -47,11 +47,11 @@ Haiku 5.5 drives the app well: the smoke plan takes a few dozen turns and cents; - a local SMP server with a `[NAMES]` section and `names-resolver.py`, the names `cinews.testing`, `ciboth.testing`, `cialice.testing` and `cibobby.testing` claimed by their owners, and the contacts cialice and Robert, online and accepting connections; its address goes to `/names-server`; - checks before the app starts: a search sent to the directory as a message must list `ci_group` and `ci_channel`, and `/_connect plan` must resolve each name from a fresh peer. -Not yet verified on master: the backend ran on `f/directory-search`'s code, and its checks were rewritten for master's (the directory answers searches sent as messages there; service requests need the feature branch). +On master's code it starts and passes these checks (the directory answers searches sent as messages); service requests to the directory, which only `f/directory-search` answers, are not checked here. It writes what it seeded to `/seed.md` (the Claude sessions get it) and logs to `/*.log`; `stop`, `directory-stop`, `directory-start` and `listings` are its other commands. The app reaches it through the simulator, which shares the runner's network; pointing the app at the names server is a few steps in Settings, described in `claude-prompt.md`. ## Other files -- `use-macos26-sdk.sh`: runner images from 20261013 add Command Line Tools with a macOS 27 SDK that Xcode 26.6's linker can't read; every macOS job points the default SDK back at macOS 26.5. +- `use-macos26-sdk.sh`: runner images from 20261013 add Command Line Tools with a macOS 27 SDK that Xcode 26.6's linker can't read; every macOS job points the default SDK back at macOS 26.5. The pinned `macos-26` label runs such an image. - `earlier-run.sh `: finds the newest run of this branch in this repository with that artifact; artifacts from forks and other branches are never used. diff --git a/scripts/ci/ios-sim/backend.sh b/scripts/ci/ios-sim/backend.sh index 02b293a9f9..3c4c5c4ca6 100755 --- a/scripts/ci/ios-sim/backend.sh +++ b/scripts/ci/ios-sim/backend.sh @@ -57,13 +57,6 @@ expect_out() { fi } -expect_no_error() { - if grep -qiE "error" "$work/out"; then - echo "error: the command failed" - return 1 - fi -} - # Each run keeps the client online for a few seconds, which also lets the handshakes it waits for progress. admin_until() { local pattern=$1 secs=$2 cmd=$3 @@ -114,22 +107,28 @@ create_directory() { } start_directory() { + local pid deadline=$((SECONDS + 60)) # -y: the service adds its own tables to the database the CLI created, and would otherwise ask on stdin nohup "$bin/simplex-directory-service" -d "$work/directory" -y --super-users "$1" < /dev/null >> "$work/directory.log" 2>&1 & - echo $! > "$work/directory.pid" + pid=$! + echo "$pid" > "$work/directory.pid" echo "$1" > "$work/super-user" - sleep 5 - if ! kill -0 "$(cat "$work/directory.pid")" 2> /dev/null; then - echo "error: the directory service exited" - cat "$work/directory.log" - return 1 - fi + # its output is block-buffered, so it shows that its chat has started by connecting to its servers + until lsof -a -p "$pid" -iTCP -sTCP:ESTABLISHED > /dev/null 2>&1; do + if ! kill -0 "$pid" 2> /dev/null || [ "$SECONDS" -ge "$deadline" ]; then + echo "error: the directory service did not start" + cat "$work/directory.log" + return 1 + fi + sleep 1 + done } -stop_directory() { +# Waits for the process to exit, which completes its output, and kills it after 30 s. +stop_process() { local pid deadline - pid=$(cat "$work/directory.pid") - kill -INT "$pid" 2> /dev/null || true + pid=$(cat "$work/$1.pid") + kill "-$2" "$pid" 2> /dev/null || true deadline=$((SECONDS + 30)) while kill -0 "$pid" 2> /dev/null; do if [ "$SECONDS" -ge "$deadline" ]; then @@ -147,7 +146,7 @@ connect_admin() { admin_until "Welcome to $directory_name" 180 "/tail @'$directory_name' 10" local id id=$(db_query directory "SELECT contact_id FROM contacts WHERE local_display_name = '$admin'") - stop_directory + stop_process directory INT start_directory "$id:$admin" echo "directory restarted with super user $id:$admin" } @@ -299,7 +298,7 @@ start_contact() { expect_out "^auto_accept on" register_name "$name" "$address" "" named "$db" "$display_name" 5 "/_set domain 1 $name" - expect_no_error + expect_out "^new contact address set$" # --headless is for relays only; a command with a long -t keeps the client online, accepting contacts, for 6 hours nohup "$bin/simplex-chat" -d "$work/$db" -e "/auto_accept on" -t 21600 --user-display-name "$display_name" < /dev/null > "$work/$db.log" 2>&1 & echo $! > "$work/$db.pid" @@ -323,7 +322,7 @@ seed_names() { link=$(channel_link ciboth) register_name ciboth.testing "$address" "$link" named "$admin" "$admin" 5 "/_set domain 1 ciboth.testing" - expect_no_error + expect_out "^new contact address set$" named "$admin" "$admin" 5 "/public group access #ciboth domain=ciboth.testing" expect_out "updated public group access: domain=ciboth.testing" @@ -334,6 +333,8 @@ seed_names() { # What the app will resolve, from a client that knows none of the names' owners. probe_names() { local target pattern + # -s gives its server the names role only for users that exist when the client starts + chat probe 1 "/user" --user-display-name ci_probe for target in "#cinews.testing|ok to connect via relays" "@cialice.testing|contact address: ok to connect" "@cibobby.testing|contact address: ok to connect" "ciboth.testing|You can also connect to @ciboth"; do pattern=${target#*|} target=${target%%|*} @@ -433,14 +434,17 @@ start() { } stop() { - local p processes="directory relay ci_alice ci_robert smp-server names-resolver" + local p processes="directory relay ci_alice ci_robert smp-server" for p in $processes; do if [ -f "$work/$p.pid" ]; then - kill -INT "$(cat "$work/$p.pid")" 2> /dev/null || true + stop_process "$p" INT fi done - sleep 3 - for p in $processes; do + # Python, started in the background by a script, ignores SIGINT + if [ -f "$work/names-resolver.pid" ]; then + stop_process names-resolver TERM + fi + for p in $processes names-resolver; do echo "::group::$p log" cat "$work/$p.log" 2> /dev/null || true echo "::endgroup::" @@ -459,7 +463,7 @@ case "${1:-}" in ;; directory-stop) work=$2 - stop_directory + stop_process directory INT ;; directory-start) work=$2 diff --git a/scripts/ci/ios-sim/build-backend.sh b/scripts/ci/ios-sim/build-backend.sh index 44b6c4c0c0..cf319a95dd 100755 --- a/scripts/ci/ios-sim/build-backend.sh +++ b/scripts/ci/ios-sim/build-backend.sh @@ -12,7 +12,6 @@ openssl_dir=$(brew --prefix openssl@3) ghc --version cabal --version -du -sh ~/.cabal/store dist-newstyle || true cat > cabal.project.local < cabal.project.local < (optional) tells that session which directory entries and names its cases use, so none meets an entry already known. +# backend: yes when the cases use the test backend; a run that asks for Claude must then start it too. +backend: no model: claude-haiku-5-5 layouts: reachable cases-reachable: S1-S3 diff --git a/scripts/ci/ios-sim/claude-test.sh b/scripts/ci/ios-sim/claude-test.sh index 5d72d03a9b..e069f33ef3 100755 --- a/scripts/ci/ios-sim/claude-test.sh +++ b/scripts/ci/ios-sim/claude-test.sh @@ -2,7 +2,8 @@ # Runs the device test plan with Claude Code on this runner after the smoke test, one session per layout, # as claude-scope says. A session drives the app only through the ui helper, and reads and writes only its -# own directory /: prompt.md, report.md, shots/, transcript.jsonl (stream-json). +# own directory, which ends up in /: prompt.md, report.md, shots/, transcript.jsonl (stream-json). +# It passes when every session's final message ends with "CLAUDE-TEST: PASS". # Usage: scripts/ci/ios-sim/claude-test.sh # Needs CLAUDE_CODE_OAUTH_TOKEN, SIM_UDID (the booted simulator with the app) and, when the test backend runs, BACKEND_DIR. @@ -14,6 +15,9 @@ unset CLAUDE_CODE_OAUTH_TOKEN mkdir -p "$1" out=$(cd "$1" && pwd) +# sessions run here and move to the output dir, which is uploaded, only after their files are checked for the token +staging=$out.sessions +mkdir -p "$staging" dir=$(cd "$(dirname "$0")" && pwd) : "${SIM_UDID:?SIM_UDID must be the simulator UDID}" BACKEND_DIR=${BACKEND_DIR:-} @@ -89,14 +93,15 @@ write_prompt() { status=0 previous="" for layout in $layouts; do - session=$out/$layout + session=$staging/$layout mkdir -p "$session/shots" + echo "$layout" >> "$out/sessions" ui layout "$layout" write_prompt "$layout" "$previous" > "$session/prompt.md" echo "::group::Claude session: $layout, $model, $max_turns turns, \$$max_budget" session_status=0 # outside the session's directory, so the session cannot change it - export CLAUDE_GUARD_LOG=$out/$layout-guard.log + export CLAUDE_GUARD_LOG=$staging/$layout-guard.log : > "$CLAUDE_GUARD_LOG" ( cd "$session" @@ -120,11 +125,16 @@ for layout in $layouts; do ) || session_status=$? echo "claude exited with $session_status" # the token must never be printed or uploaded, whatever the session did; grep reads it from a pipe, not from its arguments - if grep -rqF -f <(printf '%s\n' "$token") "$session" "$CLAUDE_GUARD_LOG"; then + leak=0 + grep -rqF -f <(printf '%s\n' "$token") "$session" "$CLAUDE_GUARD_LOG" || leak=$? + if [ "$leak" -ne 1 ]; then rm -rf "$session" "$CLAUDE_GUARD_LOG" - echo "::error::the $layout session's files contained the token, so they were deleted before the upload" + echo "::error::the $layout session's files contain the token, or checking for it failed (grep exited $leak), so they were deleted before the upload" session_status=1 else + mv "$session" "$CLAUDE_GUARD_LOG" "$out/" + session=$out/$layout + CLAUDE_GUARD_LOG=$out/$layout-guard.log tail -n 20 "$session/stderr.log" if [ -f "$session/report.md" ]; then cat "$session/report.md" @@ -133,6 +143,14 @@ for layout in $layouts; do fi echo "guard hook: $(grep -c '^allow' "$CLAUDE_GUARD_LOG" || true) allowed, $(grep -c '^refuse' "$CLAUDE_GUARD_LOG" || true) refused" grep '^refuse' "$CLAUDE_GUARD_LOG" || true + result=$(jq -r -R 'fromjson? | select(.type == "result") | .result // empty' "$session/transcript.jsonl" | tail -n 20 || true) + echo "final message, last lines:" + echo "$result" + last=$(grep -v '^[[:space:]]*$' <<< "$result" | tail -n 1 || true) + if ! grep -qxE '[*`]*CLAUDE-TEST: PASS[*`.]*[[:space:]]*' <<< "$last"; then + echo "::error::the $layout session's final message does not end with CLAUDE-TEST: PASS" + session_status=1 + fi fi echo "::endgroup::" if [ "$session_status" -ne 0 ]; then diff --git a/scripts/ci/ios-sim/claude-usage.sh b/scripts/ci/ios-sim/claude-usage.sh index 79132707cb..ee63e32ef4 100755 --- a/scripts/ci/ios-sim/claude-usage.sh +++ b/scripts/ci/ios-sim/claude-usage.sh @@ -19,10 +19,22 @@ mkdir -p "$out" echo "|---|---|---|---|---|---|---|---|---|---|---|" } >> "$summary" -for transcript in "$out"/*/transcript.jsonl; do - [ -f "$transcript" ] || continue - session=$(basename "$(dirname "$transcript")") +# claude-test.sh lists each session it starts in /sessions; a session's files reach only once +# they are checked for the token +while read -r session; do sessions=$((sessions + 1)) + transcript=$out/$session/transcript.jsonl + if [ ! -f "$transcript" ]; then + if [ -d "$out.sessions/$session" ]; then + reason="cut short before its files were checked for the token, so they are not uploaded" + else + reason="its files held the token or could not be checked, so they were deleted" + fi + echo "| $session | $reason: usage unknown | | | | | | | | | |" >> "$summary" + mkdir -p "$out/$session" + jq -n --arg s "$session" '{session: $s, result: "missing"}' > "$out/$session/usage.json" + continue + fi result=$(jq -c -R 'fromjson? | select(.type == "result")' "$transcript" | tail -n 1) if [ -z "$result" ]; then echo "| $session | no result event: the session timed out or crashed | | | | | | | | | |" >> "$summary" @@ -31,7 +43,7 @@ for transcript in "$out"/*/transcript.jsonl; do fi jq --arg s "$session" '{session: $s, subtype, is_error, num_turns, duration_ms, usage: (.usage | {input_tokens, output_tokens, cache_read_input_tokens, cache_creation_input_tokens}), modelUsage, total_cost_usd, permission_denials: (.permission_denials // [] | length)}' <<< "$result" > "$out/$session/usage.json" jq -r --arg s "$session" '"| \($s) | \(.subtype)\(if .is_error then " (error)" else "" end) | \(.num_turns) | \(.duration_ms / 60000 | floor) min | \(.usage.input_tokens) | \(.usage.output_tokens) | \(.usage.cache_read_input_tokens) | \(.usage.cache_creation_input_tokens) | \([.modelUsage // {} | to_entries[] | "\(.key) $\(.value.costUSD // 0 | . * 100 | round / 100)"] | join(", ")) | $\(.total_cost_usd // 0 | . * 100 | round / 100) | \(.permission_denials // [] | length) |"' <<< "$result" >> "$summary" -done +done < <(cat "$out/sessions" 2> /dev/null) if [ "$sessions" -eq 0 ]; then echo "No Claude session ran." >> "$summary" diff --git a/scripts/ci/ios-sim/earlier-run.sh b/scripts/ci/ios-sim/earlier-run.sh index 752bea6292..7081bd08b1 100755 --- a/scripts/ci/ios-sim/earlier-run.sh +++ b/scripts/ci/ios-sim/earlier-run.sh @@ -8,10 +8,10 @@ set -euo pipefail name=$1 -run_id=$(gh api "repos/$GITHUB_REPOSITORY/actions/artifacts?name=$name&per_page=100" --jq " +run_id=$(gh api "repos/$GITHUB_REPOSITORY/actions/artifacts?name=$name&per_page=100" | jq -r --arg branch "$GITHUB_REF_NAME" ' [.artifacts[] - | select((.expired | not) and .workflow_run.head_branch == \"$GITHUB_REF_NAME\" and .workflow_run.head_repository_id == .workflow_run.repository_id)] - | sort_by(.created_at) | last | .workflow_run.id // empty") + | select((.expired | not) and .workflow_run.head_branch == $branch and .workflow_run.head_repository_id == .workflow_run.repository_id)] + | sort_by(.created_at) | last | .workflow_run.id // empty') if [ -n "$run_id" ]; then echo "$name: from run $run_id" echo "run-id=$run_id" >> "$GITHUB_OUTPUT"