From 5f48972d7da836d3a1ef2cd7029be9f8bddead81 Mon Sep 17 00:00:00 2001 From: spacedandy-agents-simplex <342056592+spacedandy-agents-simplex@users.noreply.github.com> Date: Thu, 15 Oct 2026 11:28:56 +0000 Subject: [PATCH] ci: run a test directory with a group and a channel on the desktop app runner A backend job builds the CLI and the directory service without client_library, reused across runs as an artifact named by its sources (plan job, as on the iOS lane). The app job starts a relay and the directory on the preset servers, registers and approves ci_group and ci_channel from a CLI admin, and checks that a search request from a fresh client finds both. The app is not pointed at it yet; its build and desktopTest still run when the backend fails. --- .github/workflows/desktop-linux.yml | 134 ++++++++++++- scripts/ci/desktop-linux/backend.sh | 220 ++++++++++++++++++++++ scripts/ci/desktop-linux/build-backend.sh | 33 ++++ scripts/ci/desktop-linux/earlier-run.sh | 20 ++ 4 files changed, 399 insertions(+), 8 deletions(-) create mode 100755 scripts/ci/desktop-linux/backend.sh create mode 100755 scripts/ci/desktop-linux/build-backend.sh create mode 100755 scripts/ci/desktop-linux/earlier-run.sh diff --git a/.github/workflows/desktop-linux.yml b/.github/workflows/desktop-linux.yml index d794a0c07a..f5e85a5ead 100644 --- a/.github/workflows/desktop-linux.yml +++ b/.github/workflows/desktop-linux.yml @@ -8,6 +8,7 @@ on: permissions: contents: read + actions: read concurrency: group: desktop-linux-${{ github.ref }} @@ -18,8 +19,95 @@ env: CABAL_VER: 3.10.2.0 JAVA_VER: 17 CORE_PKG: desktop-core-linux-x86_64.tar.gz + BACKEND_PKG: desktop-linux-backend.tar.gz jobs: + plan: + name: find earlier builds + runs-on: ubuntu-24.04 + timeout-minutes: 10 + outputs: + backend: ${{ steps.names.outputs.backend }} + backend-run: ${{ steps.backend.outputs.run-id }} + steps: + - name: Checkout code + uses: actions/checkout@v5 + + # artifacts are named after the sources they are built from, so a run uses one that an earlier run uploaded + - name: Artifact names + id: names + run: echo "backend=desktop-linux-backend-${{ hashFiles('src/**', 'apps/simplex-chat/**', 'apps/simplex-directory-service/Main.hs', 'apps/simplex-directory-service/src/**', 'cabal.project', 'simplex-chat.cabal', 'scripts/ci/desktop-linux/build-backend.sh') }}" >> "$GITHUB_OUTPUT" + + - name: Find backend + id: backend + env: + GH_TOKEN: ${{ github.token }} + run: scripts/ci/desktop-linux/earlier-run.sh "${{ steps.names.outputs.backend }}" + + backend: + name: test backend for the app + needs: plan + if: needs.plan.outputs.backend-run == '' + runs-on: ubuntu-24.04 + timeout-minutes: 150 + steps: + - name: Checkout code + uses: actions/checkout@v5 + + - name: Show runner + run: | + lsb_release -ds + nproc + free -h + df -h / + + - name: Setup Haskell + id: haskell + uses: simplex-chat/setup-haskell-action@v2 + with: + ghc-version: ${{ env.GHC_VER }} + cabal-version: ${{ env.CABAL_VER }} + + # before the first backend store exists, the core library's has most of the same Hackage dependencies + - name: Restore cabal store + id: cabal-cache + uses: actions/cache/restore@v5 + with: + path: | + ${{ steps.haskell.outputs.cabal-store }} + dist-newstyle + key: desktop-linux-backend-store-ghc${{ env.GHC_VER }}-${{ hashFiles('cabal.project', 'simplex-chat.cabal') }}-${{ github.run_id }} + restore-keys: | + desktop-linux-backend-store-ghc${{ env.GHC_VER }}-${{ hashFiles('cabal.project', 'simplex-chat.cabal') }}- + desktop-linux-backend-store-ghc${{ env.GHC_VER }}- + desktop-linux-core-store-ghc${{ env.GHC_VER }}-${{ hashFiles('cabal.project', 'simplex-chat.cabal') }}- + + - name: Install libraries + run: | + sudo apt-get update + sudo apt-get install -y libffi-dev libgmp-dev libnuma-dev libssl-dev zlib1g-dev + + - name: Build backend + run: scripts/ci/desktop-linux/build-backend.sh ~/backend/${{ env.BACKEND_PKG }} + + - name: Save cabal store + if: ${{ !cancelled() }} + uses: actions/cache/save@v5 + with: + path: | + ${{ steps.haskell.outputs.cabal-store }} + dist-newstyle + key: ${{ steps.cabal-cache.outputs.cache-primary-key }} + + - name: Upload backend + uses: actions/upload-artifact@v6 + with: + name: ${{ needs.plan.outputs.backend }} + path: ~/backend/${{ env.BACKEND_PKG }} + compression-level: 0 + retention-days: 30 + if-no-files-found: error + core: name: core library for Linux runs-on: ubuntu-24.04 @@ -100,7 +188,9 @@ jobs: app: name: desktop app under Xvfb - needs: core + needs: [plan, core, backend] + # backend is skipped when an earlier run built it + if: ${{ !cancelled() && !failure() }} runs-on: ubuntu-24.04 timeout-minutes: 60 steps: @@ -118,6 +208,12 @@ jobs: echo "ANDROID_HOME=$ANDROID_HOME" ls "$ANDROID_HOME/platforms" "$ANDROID_HOME/build-tools" || true + - name: Install tools + run: | + sudo apt-get update + sudo apt-get install -y xvfb xdotool openbox imagemagick tesseract-ocr sqlite3 + dpkg-query -W xvfb xdotool openbox imagemagick tesseract-ocr sqlite3 + - name: Download core library uses: actions/download-artifact@v7 with: @@ -125,9 +221,24 @@ jobs: path: ~/core - name: Install core library + id: core run: scripts/ci/desktop-linux/install-core.sh ~/core/${{ env.CORE_PKG }} + - name: Download backend + uses: actions/download-artifact@v7 + with: + name: ${{ needs.plan.outputs.backend }} + path: ~/backend + run-id: ${{ needs.plan.outputs.backend-run || github.run_id }} + github-token: ${{ github.token }} + + - name: Start test backend + timeout-minutes: 30 + run: scripts/ci/desktop-linux/backend.sh start ~/backend/${{ env.BACKEND_PKG }} "$RUNNER_TEMP/backend" + + # the app and its tests are still built when the backend fails - name: Setup Java + if: ${{ !cancelled() && steps.core.outcome == 'success' }} uses: actions/setup-java@v5 with: distribution: corretto @@ -136,6 +247,7 @@ jobs: - name: Build app id: build + if: ${{ !cancelled() && steps.core.outcome == 'success' }} working-directory: apps/multiplatform run: | ./gradlew --no-daemon --console=plain -Psimplex.assets.dir=../../assets :desktop:createDistributable @@ -153,17 +265,14 @@ jobs: if: ${{ !cancelled() && steps.build.outcome == 'success' }} run: scripts/ci/desktop-linux/test-summary.py apps/multiplatform/common/build/test-results/desktopTest - - name: Install UI tools - if: ${{ !cancelled() && steps.build.outcome == 'success' }} - run: | - sudo apt-get update - sudo apt-get install -y xvfb xdotool openbox imagemagick tesseract-ocr - dpkg-query -W xvfb xdotool openbox imagemagick tesseract-ocr - - name: Run smoke test if: ${{ !cancelled() && steps.build.outcome == 'success' }} run: scripts/ci/desktop-linux/smoke-test.sh apps/multiplatform/release/main/app/simplex "$RUNNER_TEMP/smoke-test" + - name: Stop test backend + if: ${{ !cancelled() }} + run: scripts/ci/desktop-linux/backend.sh stop "$RUNNER_TEMP/backend" + - name: Upload smoke test results if: ${{ !cancelled() }} uses: actions/upload-artifact@v6 @@ -184,6 +293,15 @@ jobs: retention-days: 3 if-no-files-found: warn + - name: Upload backend logs + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v6 + with: + name: desktop-backend-logs + path: ${{ runner.temp }}/backend/*.log + retention-days: 3 + if-no-files-found: warn + release: name: released app under Xvfb runs-on: ubuntu-24.04 diff --git a/scripts/ci/desktop-linux/backend.sh b/scripts/ci/desktop-linux/backend.sh new file mode 100755 index 0000000000..6c2e236b33 --- /dev/null +++ b/scripts/ci/desktop-linux/backend.sh @@ -0,0 +1,220 @@ +#!/bin/bash + +# Test backend on the app runner: a chat relay and a directory service on the preset servers, with one group +# and one channel registered and approved by a CLI peer, and a search request that must find both. +# Usage: +# scripts/ci/desktop-linux/backend.sh start writes the directory's short address to /directory-link +# scripts/ci/desktop-linux/backend.sh stop stops the relay and the directory, prints their logs +# The relay and the directory keep running between start and stop. + +set -euo pipefail + +directory_name="SimpleX Directory" +admin=ci_admin +search_text="ci seed" + +# Runs one chat command in a CLI client that stays online for after it; the output is in $work/out. +chat() { + local db=$1 secs=$2 cmd=$3 + shift 3 + local status=0 + "$bin/simplex-chat" -d "$work/$db" -e "$cmd" -t "$secs" --execute-log all "$@" < /dev/null > "$work/out" 2>&1 || status=$? + cat "$work/out" >> "$work/$db-cli.log" + echo "::group::$db: $cmd" + cat "$work/out" + echo "::endgroup::" + return "$status" +} + +admin() { + chat "$admin" "$1" "$2" --user-display-name "$admin" +} + +expect_out() { + if ! grep -qE "$1" "$work/out"; then + echo "error: \"$1\" not in the output" + return 1 + fi +} + +# Each run keeps the client online for a few seconds, which also lets the handshakes it waits for progress. +admin_until() { + local pattern=$1 secs=$2 cmd=$3 + local deadline=$((SECONDS + secs)) + until admin 5 "$cmd" && grep -qE "$pattern" "$work/out"; do + if [ "$SECONDS" -ge "$deadline" ]; then + echo "error: \"$pattern\" not shown by \"$cmd\" after $secs s" + return 1 + fi + done +} + +db_query() { + sqlite3 "$work/$1_chat.db" "$2" +} + +start_relay() { + nohup "$bin/simplex-chat" -d "$work/relay" --relay --headless --user-display-name ci_relay < /dev/null > "$work/relay.log" 2>&1 & + echo $! > "$work/relay.pid" + local deadline=$((SECONDS + 120)) + until grep -qE '^https://[^ ]+/r#' "$work/relay.log"; do + if ! kill -0 "$(cat "$work/relay.pid")" 2> /dev/null || [ "$SECONDS" -ge "$deadline" ]; then + echo "error: the relay did not create its address" + cat "$work/relay.log" + return 1 + fi + sleep 1 + done + relay_link=$(grep -m 1 -oE '^https://[^ ]+/r#[^ ]+' "$work/relay.log") + echo "relay address: $relay_link" +} + +# The address is created here, like the directory tests do, with auto-accept on before the service subscribes to it. +create_directory() { + chat directory 1 "/ad pq_ratchet=on" --create-bot-display-name "$directory_name" + directory_link=$(awk '/Your new chat address is created!/ {getline; getline; print; exit}' "$work/out") + case "$directory_link" in + https://*/a#*) ;; + *) + echo "error: no short directory address in the output" + return 1 + ;; + esac + chat directory 1 "/auto_accept on" + expect_out "^auto_accept on" + echo "$directory_link" > "$work/directory-link" + echo "directory address: $directory_link" +} + +start_directory() { + # -y: the service adds its own tables to the database the CLI created, and would otherwise ask on stdin + nohup "$bin/simplex-directory-service" -d "$work/directory" -y --super-users "$1" < /dev/null >> "$work/directory.log" 2>&1 & + echo $! > "$work/directory.pid" + sleep 5 + if ! kill -0 "$(cat "$work/directory.pid")" 2> /dev/null; then + echo "error: the directory service exited" + cat "$work/directory.log" + return 1 + fi +} + +stop_directory() { + local pid deadline + pid=$(cat "$work/directory.pid") + kill -INT "$pid" 2> /dev/null || true + deadline=$((SECONDS + 30)) + while kill -0 "$pid" 2> /dev/null; do + if [ "$SECONDS" -ge "$deadline" ]; then + kill -KILL "$pid" 2> /dev/null || true + fi + sleep 1 + done +} + +# A super user is known by contact ID, which exists only after the admin connects, so the service starts +# with nobody and is restarted with the admin. +connect_admin() { + admin 10 "/c $directory_link" + expect_out "connection request sent!" + admin_until "Welcome to $directory_name" 180 "/tail @'$directory_name' 10" + local id + id=$(db_query directory "SELECT contact_id FROM contacts WHERE local_display_name = '$admin'") + stop_directory + start_directory "$id:$admin" + echo "directory restarted with super user $id:$admin" +} + +# The directory sends each registration to super users as a message with the exact /approve command. +approve() { + local name=$1 reply=$2 + admin_until "/approve [0-9]+:$name [0-9]+" 300 "/tail @'$directory_name' 20" + local cmd + cmd=$(grep -oE "/approve [0-9]+:$name [0-9]+" "$work/out" | tail -n 1) + admin 5 "@'$directory_name' $cmd" + admin_until "$reply" 120 "/tail @'$directory_name' 20" +} + +seed_group() { + admin 1 "/g ci_group CI seed group" + expect_out "group #ci_group \(CI seed group\) is created" + admin 5 "/a ci_group '$directory_name' admin" + expect_out "invitation to join the group #ci_group sent to" + approve ci_group "Group approved!" +} + +seed_channel() { + admin 1 "/relays name=ci_relay $relay_link" + expect_out "^ok$" + local relay_id + relay_id=$(db_query "$admin" "SELECT chat_relay_id FROM chat_relays WHERE display_name = 'ci_relay' AND deleted = 0") + admin 10 "/public group relays=$relay_id #ci_channel CI seed channel" + expect_out "group #ci_channel \(CI seed channel\) is created" + local group_id + group_id=$(db_query "$admin" "SELECT group_id FROM groups WHERE local_display_name = 'ci_channel'") + admin_until "relay id [0-9]+: active" 180 "/_get relays #$group_id" + admin 5 "/share chat #ci_channel @'$directory_name'" + expect_out "link to join channel #ci_channel" + approve ci_channel "Channel approved!" +} + +# What the app will send, from a client that never connected to the directory. +probe_search() { + chat probe 1 "/_service_request 1 $directory_link timeout=30 {\"type\":\"search\",\"searchText\":\"$search_text\"}" --user-display-name ci_probe + expect_out '"displayName":"ci_group"' + expect_out '"displayName":"ci_channel"' + echo "search for \"$search_text\" found ci_group and ci_channel" +} + +dump_directory() { + echo "::group::directory registrations" + db_query directory "SELECT group_id, group_reg_status FROM sx_directory_group_regs" || true + echo "::endgroup::" +} + +start() { + bin=$work/bin + mkdir -p "$bin" + tar -C "$bin" -xzf "$1" + "$bin/simplex-chat" --version + "$bin/simplex-directory-service" --version + + start_relay + create_directory + start_directory "0:nobody" + trap dump_directory EXIT + connect_admin + seed_group + seed_channel + probe_search +} + +stop() { + local p + for p in directory relay; do + if [ -f "$work/$p.pid" ]; then + kill -INT "$(cat "$work/$p.pid")" 2> /dev/null || true + fi + done + sleep 3 + for p in directory relay; do + echo "::group::$p log" + cat "$work/$p.log" 2> /dev/null || true + echo "::endgroup::" + done +} + +case "${1:-}" in + start) + work=$3 + mkdir -p "$work" + start "$2" + ;; + stop) + work=$2 + stop + ;; + *) + echo "usage: $0 start | stop " + exit 1 + ;; +esac diff --git a/scripts/ci/desktop-linux/build-backend.sh b/scripts/ci/desktop-linux/build-backend.sh new file mode 100755 index 0000000000..fcc8c08850 --- /dev/null +++ b/scripts/ci/desktop-linux/build-backend.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# Builds the CLI and the directory service that the app job runs as its test backend. +# Usage: scripts/ci/desktop-linux/build-backend.sh +# They need the server-capable configuration (no client_library), as build.yml's Linux job builds the CLI. + +set -euo pipefail + +mkdir -p "$(dirname "$1")" +out="$(cd "$(dirname "$1")" && pwd)/$(basename "$1")" + +ghc --version +cabal --version +du -sh ~/.cabal/store dist-newstyle 2> /dev/null || true + +cat > cabal.project.local << EOF +ignore-project: False +jobs: \$ncpus + +package direct-sqlcipher + flags: +openssl +EOF + +cabal build exe:simplex-chat exe:simplex-directory-service + +pkg=$(mktemp -d) +for exe in simplex-chat simplex-directory-service; do + cp "$(cabal list-bin "exe:$exe")" "$pkg" +done +ldd "$pkg"/* + +tar -C "$pkg" -czf "$out" . +ls -la "$out" diff --git a/scripts/ci/desktop-linux/earlier-run.sh b/scripts/ci/desktop-linux/earlier-run.sh new file mode 100755 index 0000000000..2bfedcd50e --- /dev/null +++ b/scripts/ci/desktop-linux/earlier-run.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# Finds the newest run of this branch in this repository that uploaded an unexpired artifact with this name, +# and writes its id as run-id to GITHUB_OUTPUT. Needs GH_TOKEN with actions: read. +# Usage: scripts/ci/desktop-linux/earlier-run.sh +# Artifacts from forks and other branches are not used: their workflows could upload anything under this name. + +set -euo pipefail + +name=$1 +run_id=$(gh api "repos/$GITHUB_REPOSITORY/actions/artifacts?name=$name&per_page=100" --jq " + [.artifacts[] + | select((.expired | not) and .workflow_run.head_branch == \"$GITHUB_REF_NAME\" and .workflow_run.head_repository_id == .workflow_run.repository_id)] + | sort_by(.created_at) | last | .workflow_run.id // empty") +if [ -n "$run_id" ]; then + echo "$name: from run $run_id" + echo "run-id=$run_id" >> "$GITHUB_OUTPUT" +else + echo "$name: not built yet" +fi