From 238fe03a910a0cc92fbb7b783d47397641f0cd1e Mon Sep 17 00:00:00 2001 From: sh-agents <350651734+sh-agents@users.noreply.github.com> Date: Wed, 4 Nov 2026 07:59:57 +0000 Subject: [PATCH] ci: build the app test cores with the tool --- .github/workflows/android-emu.yml | 18 +++++++---- .github/workflows/desktop-linux.yml | 8 ++--- .github/workflows/ios-sim.yml | 47 ++++++++++++++++++++++++----- scripts/ci/android/README.md | 8 ++--- scripts/ci/desktop-linux/README.md | 2 +- scripts/ci/ios-sim/README.md | 4 +-- 6 files changed, 63 insertions(+), 24 deletions(-) diff --git a/.github/workflows/android-emu.yml b/.github/workflows/android-emu.yml index 7617693b2b..ddb6dc51c6 100644 --- a/.github/workflows/android-emu.yml +++ b/.github/workflows/android-emu.yml @@ -32,7 +32,6 @@ env: GHC_VER: 9.6.3 CABAL_VER: 3.10.2.0 JAVA_VER: 17 - CORE_PKG: android-core-arm64-v8a.tar.gz BACKEND_PKG: desktop-linux-backend.tar.gz APK: android-foss-arm64-v8a-debug.apk @@ -92,7 +91,7 @@ jobs: - name: Artifact names id: names run: | - echo "core=android-core-${{ hashFiles('src/**', 'cabal.project', 'simplex-chat.cabal', 'PRIVACY.md', 'flake.nix', 'flake.lock', 'scripts/nix/**', 'scripts/ci/android/build-core.sh') }}" >> "$GITHUB_OUTPUT" + echo "core=android-core-${{ hashFiles('src/**', 'cabal.project', 'simplex-chat.cabal', 'PRIVACY.md', 'flake.nix', 'flake.lock', 'scripts/nix/**', 'scripts/simplex-chat-build.py', 'scripts/simplex_build/**', '!scripts/simplex_build/test_*.py', '!scripts/simplex_build/testing.py') }}" >> "$GITHUB_OUTPUT" echo "backend=desktop-linux-backend-${{ hashFiles('src/**', 'apps/simplex-chat/**', 'apps/simplex-directory-service/Main.hs', 'apps/simplex-directory-service/src/**', 'cabal.project', 'simplex-chat.cabal', 'scripts/ci/desktop-linux/build-backend.sh') }}" >> "$GITHUB_OUTPUT" - name: Find core library @@ -168,14 +167,18 @@ jobs: free -h df -h / + # the Nix sandbox needs user namespaces, which Ubuntu 24.04 restricts + - name: Allow user namespaces + run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + - name: Build core library - run: scripts/ci/android/build-core.sh ~/core/${{ env.CORE_PKG }} + run: scripts/simplex-chat-build.py build libs android --install-nix --out ~/core - name: Upload core library uses: actions/upload-artifact@v7 with: name: ${{ needs.plan.outputs.core }} - path: ~/core/${{ env.CORE_PKG }} + path: ~/core compression-level: 0 retention-days: 30 if-no-files-found: error @@ -200,8 +203,11 @@ jobs: free -h df -h / + - name: Allow user namespaces + run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + - name: Build core library - run: scripts/ci/android/build-core.sh ~/core/android-core-armeabi-v7a.tar.gz armv7a + run: scripts/simplex-chat-build.py build libs android --arch armv7a --install-nix --out ~/core app: name: Android app in the emulator @@ -242,7 +248,7 @@ jobs: - name: Install core library id: core - run: scripts/ci/android/install-core.sh ~/core/${{ env.CORE_PKG }} + run: scripts/simplex-chat-build.py install libs android ~/core - name: Download backend if: needs.plan.outputs.run-backend == 'true' diff --git a/.github/workflows/desktop-linux.yml b/.github/workflows/desktop-linux.yml index 21c51967ad..7c58ce2511 100644 --- a/.github/workflows/desktop-linux.yml +++ b/.github/workflows/desktop-linux.yml @@ -28,7 +28,7 @@ env: GHC_VER: 9.6.3 CABAL_VER: 3.10.2.0 JAVA_VER: 17 - CORE_PKG: desktop-core-linux-x86_64.tar.gz + CORE_PKG: simplex-chat-libs-linux-x86_64.zip BACKEND_PKG: desktop-linux-backend.tar.gz jobs: @@ -83,7 +83,7 @@ jobs: - name: Artifact names id: names run: | - echo "core=desktop-linux-core-${{ hashFiles('src/**', 'cabal.project', 'simplex-chat.cabal', 'PRIVACY.md', 'scripts/desktop/build-lib-linux.sh', 'scripts/ci/desktop-linux/build-core.sh') }}" >> "$GITHUB_OUTPUT" + echo "core=desktop-linux-core-${{ hashFiles('src/**', 'cabal.project', 'simplex-chat.cabal', 'PRIVACY.md', 'scripts/simplex-chat-build.py', 'scripts/simplex_build/**', '!scripts/simplex_build/test_*.py', '!scripts/simplex_build/testing.py') }}" >> "$GITHUB_OUTPUT" echo "backend=desktop-linux-backend-${{ hashFiles('src/**', 'apps/simplex-chat/**', 'apps/simplex-directory-service/Main.hs', 'apps/simplex-directory-service/src/**', 'cabal.project', 'simplex-chat.cabal', 'scripts/ci/desktop-linux/build-backend.sh') }}" >> "$GITHUB_OUTPUT" - name: Find core library @@ -168,7 +168,7 @@ jobs: sudo apt-get install -y libffi-dev libgmp-dev libnuma-dev libssl-dev zlib1g-dev patchelf - name: Build core library - run: scripts/ci/desktop-linux/build-core.sh ~/core/${{ env.CORE_PKG }} + run: scripts/simplex-chat-build.py build libs linux --host --out ~/core - name: Upload core library uses: actions/upload-artifact@v7 @@ -217,7 +217,7 @@ jobs: - name: Install core library id: core - run: scripts/ci/desktop-linux/install-core.sh ~/core/${{ env.CORE_PKG }} + run: scripts/simplex-chat-build.py install libs linux ~/core/${{ env.CORE_PKG }} - name: Download backend if: needs.plan.outputs.run-backend == 'true' diff --git a/.github/workflows/ios-sim.yml b/.github/workflows/ios-sim.yml index 4ddfabc6e1..d9660c1547 100644 --- a/.github/workflows/ios-sim.yml +++ b/.github/workflows/ios-sim.yml @@ -79,7 +79,7 @@ jobs: - name: Artifact names id: names run: | - echo "core=ios-sim-core-${{ hashFiles('src/**', 'cabal.project', 'simplex-chat.cabal', 'scripts/ci/ios-sim/build-core.sh') }}" >> "$GITHUB_OUTPUT" + echo "core=ios-sim-core-${{ hashFiles('src/**', 'cabal.project', 'simplex-chat.cabal', 'scripts/simplex-chat-build.py', 'scripts/simplex_build/**', '!scripts/simplex_build/test_*.py', '!scripts/simplex_build/testing.py') }}" >> "$GITHUB_OUTPUT" echo "backend=ios-sim-backend-${{ hashFiles('src/**', 'apps/simplex-chat/**', 'apps/simplex-directory-service/Main.hs', 'apps/simplex-directory-service/src/**', 'cabal.project', 'simplex-chat.cabal', 'scripts/ci/ios-sim/build-backend.sh') }}" >> "$GITHUB_OUTPUT" - name: Find core library @@ -95,9 +95,29 @@ jobs: GH_TOKEN: ${{ github.token }} run: scripts/ci/ios-sim/earlier-run.sh "${{ steps.names.outputs.backend }}" + # the app must open this encrypted database, made by the previous release with its own OpenSSL + old-database: + name: previous release's database + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Make the database + run: python3 scripts/simplex-chat-build.py test old-database --out "$RUNNER_TEMP/old-database" + + - name: Upload the database + uses: actions/upload-artifact@v7 + with: + name: ios-sim-old-database + path: ${{ runner.temp }}/old-database + retention-days: 3 + if-no-files-found: error + app: name: app in the iOS simulator - needs: plan + needs: [plan, old-database] runs-on: macos-26 timeout-minutes: 360 env: @@ -108,6 +128,9 @@ jobs: - name: Checkout code uses: actions/checkout@v6 + - name: Use the release Xcode + uses: ./.github/actions/xcode + - name: Show runner run: | sw_vers @@ -130,11 +153,11 @@ jobs: - name: Install libraries if: env.BUILD_CORE == 'true' - run: brew install openssl@3 libffi gmp + run: brew install libffi gmp - name: Build core library if: env.BUILD_CORE == 'true' - run: scripts/ci/ios-sim/build-core.sh ~/Downloads/${{ env.CORE_PKG }} + run: scripts/simplex-chat-build.py build libs ios-sim --out ~/Downloads - name: Upload core library if: env.BUILD_CORE == 'true' @@ -156,9 +179,7 @@ jobs: github-token: ${{ github.token }} - name: Prepare libraries - run: | - scripts/ci/ios-sim/prepare-libs.sh - scripts/ios/update-pbxproj.sh + run: scripts/simplex-chat-build.py install libs ios ~/Downloads/${{ env.CORE_PKG }} - name: Build backend if: env.BUILD_BACKEND == 'true' @@ -224,6 +245,15 @@ jobs: - name: Run smoke test run: scripts/ci/ios-sim/smoke-test.sh "$RUNNER_TEMP/DerivedData/Build/Products/Debug-iphonesimulator/SimpleX.app" "$RUNNER_TEMP/smoke-test" images + - name: Download the previous release's database + uses: actions/download-artifact@v8 + with: + name: ios-sim-old-database + path: ${{ runner.temp }}/old-database + + - name: Open the previous release's database + run: scripts/ci/ios-sim/old-database.sh "$RUNNER_TEMP/old-database" + - name: Stop test backend if: ${{ !cancelled() && env.START_BACKEND == 'true' }} run: scripts/ci/ios-sim/backend.sh stop "$RUNNER_TEMP/backend" @@ -271,6 +301,9 @@ jobs: with: persist-credentials: false + - name: Use the release Xcode + uses: ./.github/actions/xcode + - name: Show runner run: | sw_vers diff --git a/scripts/ci/android/README.md b/scripts/ci/android/README.md index 5b76b6d081..d5ca99838c 100644 --- a/scripts/ci/android/README.md +++ b/scripts/ci/android/README.md @@ -6,25 +6,25 @@ The workflow runs only when started by hand, on any branch: in the Actions tab, `android-emu` → Run workflow, or `gh workflow run android-emu.yml --ref ` with `-f claude=true`, `-f backend=true`, `-f release=true` or `-f armv7a=true`. [CI on GitHub's runners](../README.md) lists the inputs. A newer run on the same branch cancels the older one. -Every run builds the core library, and the backend when it's asked for, from the branch's own Haskell code. It reuses an earlier build only when the sources are unchanged, as an artifact of the same branch named after a hash of its sources (30 days): for the core `src/**`, `cabal.project`, `simplex-chat.cabal`, `PRIVACY.md`, the Nix files (`flake.nix`, `flake.lock`, `scripts/nix/**`) and `build-core.sh`; the backend is desktop-linux's, under its name and hash, so a backend either workflow built is reused by both. The workflow saves no Actions caches, which this repository evicts within hours and shares with `build.yml`. +Every run builds the core library, and the backend when it's asked for, from the branch's own Haskell code. It reuses an earlier build only when the sources are unchanged, as an artifact of the same branch named after a hash of its sources (30 days): for the core `src/**`, `cabal.project`, `simplex-chat.cabal`, `PRIVACY.md`, the Nix files (`flake.nix`, `flake.lock`, `scripts/nix/**`) and the build script with its package (`scripts/simplex-chat-build.py`, `scripts/simplex_build/**` without its tests); the backend is desktop-linux's, under its name and hash, so a backend either workflow built is reused by both. The workflow saves no Actions caches, which this repository evicts within hours and shares with `build.yml`. ## Jobs | Job | Does | Time | |---|---|---| | `plan` | reads the inputs, finds earlier core and backend builds | 10 s | -| `core` | `build-core.sh`: `lib:simplex-chat` and `lib:support` for aarch64-android with Nix, as `scripts/android/build-android.sh` builds them | skipped when reused, about 27 min not | +| `core` | `scripts/simplex-chat-build.py build libs android`: `lib:simplex-chat` and `lib:support` for aarch64-android with Nix, as `scripts/android/build-android.sh` builds them; `install libs android` puts them in the app | skipped when reused, about 27 min not | | `app` | the debug APK (`:android:assembleFossDebug` for arm64-v8a only), the emulator, `smoke-test.sh` | about 13 min: APK 7–8, emulator 1.5, smoke test 3.5 (4 with the backend case, after 2.5 to start the backend) | | `claude` (opt-in) | after `app` passed: its APK in a new emulator at the first screen, then Claude Code drives it | about 5 min, Claude 2.5 | | `backend` (opt-in) | desktop-linux's `build-backend.sh` | about 30 min, skipped when reused | | `release` (opt-in) | the latest release's arm64 APK, its signer checked against SimpleX's key, onboarding only | about 4 min | -| `core-armv7a` (opt-in) | `build-core.sh` with `armv7a`: `lib:simplex-chat` and `lib:support` for armv7a-android, with GHC 8.10.7; the app and the emulator use only arm64 | not measured yet | +| `core-armv7a` (opt-in) | `build libs android --arch armv7a`: `lib:simplex-chat` and `lib:support` for armv7a-android, with GHC 8.10.7; the app and the emulator use only arm64 | not measured yet | Results are artifacts: `android-smoke-test` (numbered screenshots, the accessibility tree and the text of each, `steps.log`, `checks.txt`, logcat as `app.log`, `emulator.log`), `android-release-smoke-test`, `android-claude-test`, and the backend's logs; `android-app`, the APK for manual tests and the Claude job, is kept 14 days. The job summaries list the checks and the Claude usage. ## The emulator -A Google APIs x86_64 system image of API 34 (revision 14) with KVM, on the x86_64 runner: the arm64 runner has no KVM, and Google publishes no Linux arm64 emulator. The image's ARM translation (`libndk_translation`) runs the app's arm64-v8a code, the only ABI besides armeabi-v7a the core is built for. `install-sdk.sh` installs the emulator, platform-tools and the image at pinned revisions from Google's repository, checking each archive's SHA-1 and SHA-256, and writes the AVD itself; `emulator.sh` boots it without a window. Screenshots come from the emulator console (`adb emu screenrecord screenshot`): without a window, the guest's `screencap` returns an empty or black image. +A Google APIs x86_64 system image of API 34 (revision 14) with KVM, on the x86_64 runner: the arm64 runner has no KVM, and Google publishes no Linux arm64 emulator. The image's ARM translation (`libndk_translation`) runs the app's arm64-v8a code, the only ABI besides armeabi-v7a the core is built for. The armv7a APK of build-apps.yml runs on the API 30 image (revision 16), the last x86_64 image whose translation also runs 32-bit ARM code (`install-sdk.sh 30`). `install-sdk.sh` installs the emulator, platform-tools and the image at pinned revisions from Google's repository, checking each archive's SHA-1 and SHA-256, and writes the AVD itself; `emulator.sh` boots it without a window. Screenshots come from the emulator console (`adb emu screenrecord screenshot`): without a window, the guest's `screencap` returns an empty or black image. ## Scripts diff --git a/scripts/ci/desktop-linux/README.md b/scripts/ci/desktop-linux/README.md index dfecf046e4..63cf5a9b71 100644 --- a/scripts/ci/desktop-linux/README.md +++ b/scripts/ci/desktop-linux/README.md @@ -13,7 +13,7 @@ Every run builds the core library, and the backend when it's asked for, from the | Job | Does | Time | |---|---|---| | `plan` | chooses the jobs, finds earlier core and backend builds | 10 s | -| `core` | `scripts/desktop/build-lib-linux.sh` via `build-core.sh` | skipped when reused, about 24 min not | +| `core` | `scripts/simplex-chat-build.py build libs linux --host`, then `install libs linux` in the app job | skipped when reused, about 24 min not | | `app` | the app (`:desktop:createDistributable`), desktopTest with the snapshots, `smoke-test.sh` | about 12 min: build 4.5, desktopTest 0.7, smoke test 3.5 | | `claude` (opt-in) | after `app` passed: its distributable at the first screen, then Claude Code drives it | about 4 min, Claude 3 | | `backend` (opt-in) | `build-backend.sh`: `simplex-chat`, `simplex-directory-service`, and `smp-server` from the simplexmq commit `cabal.project` pins | about 30 min, skipped when reused | diff --git a/scripts/ci/ios-sim/README.md b/scripts/ci/ios-sim/README.md index d9405b30c4..87206c7cdc 100644 --- a/scripts/ci/ios-sim/README.md +++ b/scripts/ci/ios-sim/README.md @@ -1,6 +1,6 @@ # iOS app in the simulator on GitHub's runners -`.github/workflows/ios-sim.yml` builds the core library and the iOS app from the branch's code on a macOS runner, starts the app in an iPhone 17 simulator, and drives it through onboarding to the chat list with a scripted smoke test; the app job's smoke test (`images`) then turns off the encryption of local files, creates a group and sends `tests/fixtures/test.jpg` and `logo-large-rgba.png`, each added to the photo library, picked, resized by the core, previewed and sent, and checks the pixel size of each sent file with `sips`. On request it also runs a Claude session that follows a device test plan, and a test backend (relay, directory, names server, contacts) for tests that need servers and peers. +`.github/workflows/ios-sim.yml` builds the core library and the iOS app from the branch's code on a macOS runner, starts the app in an iPhone 17 simulator, and drives it through onboarding to the chat list with a scripted smoke test; the app job's smoke test (`images`) then turns off the encryption of local files, creates a group and sends `tests/fixtures/test.jpg` and `logo-large-rgba.png`, each added to the photo library, picked, resized by the core, previewed and sent, and checks the pixel size of each sent file with `sips`. `old-database.sh` then gives the app an encrypted database made by the previous release's CLI and checks that it opens it with its key. On request it also runs a Claude session that follows a device test plan, and a test backend (relay, directory, names server, contacts) for tests that need servers and peers. ## Running it @@ -11,7 +11,7 @@ Results: the job logs, the Claude job's summary (Claude's usage) and the artifac ## How long it takes, what it costs - `plan` (ubuntu-24.04, 10 s) reads what the run asks for, names the core library and the backend after their sources, `hashFiles` over the Haskell sources, `cabal.project`, `simplex-chat.cabal` and the build script, and reuses an artifact of the same name that an earlier run of the same branch uploaded (30 days). So every run builds the core library, and the backend when asked, from the branch's own Haskell code, and rebuilds only when those sources change. -- `app` (macOS) first builds what no earlier run built, and uploads it for later runs: `build-core.sh` (30–45 min) builds `lib:simplex-chat` with cabal and the iOS flags, as the Nix iOS build does, into the package `scripts/ios/prepare.sh` expects; `build-backend.sh` (20–45 min, only when asked) builds `simplex-chat`, `simplex-directory-service` and `smp-server`. `prepare-libs.sh` then marks the core library for the simulator (`mac2ios`). The Xcode build takes 8–22 min, the smoke test about 10 min, the snapshots about 18 min more, the backend start about 10 min when asked. +- `app` (macOS) first builds what no earlier run built, and uploads it for later runs: `scripts/simplex-chat-build.py build libs ios-sim` (30–45 min) builds `lib:simplex-chat` with cabal and the iOS flags, as the Nix iOS build does, into the package of the Nix build; `build-backend.sh` (20–45 min, only when asked) builds `simplex-chat`, `simplex-directory-service` and `smp-server`. `install libs ios` then marks the core library for the simulator (`mac2ios`) and points the Xcode project at it. The Xcode build takes 8–22 min, the smoke test about 10 min, the snapshots about 18 min more, the backend start about 10 min when asked. - `claude` (macOS, only when asked, after `app` passed): a new simulator with the app job's build and the smoke test, about 7 min, then the sessions (the smoke plan's took 2.6 min). A run with reused builds and no extras takes about 25 minutes plus the wait for a macOS runner. Everything but the Claude sessions runs on one macOS runner, so a run waits for one runner, not one per build. Hosted minutes are free for this public repository; the limit is the repository's five concurrent macOS jobs, shared with PR builds (each `build.yml` run takes two), so queue waits can be long. The workflow saves no Actions caches, which this repository evicts within hours and shares with `build.yml`: builds are reused only as the artifacts above, and a core or backend rebuild compiles its dependencies too.