mirror of
https://github.com/simplex-chat/simplex-chat.git
synced 2026-11-05 14:57:42 +00:00
spacedandy-agents-simplex
8a9628bd9c
The app is built in the app job with that job's backend's directory address in DIRECTORY_SERVICE_LINK, and since master moved Claude into its own job, that job started a new backend with another address. The app job now uploads its stopped backend's work dir (without the binaries), and the Claude job restarts it with a new backend.sh resume: the same servers' keys and databases, so the same addresses. The relay, directory, names server and contacts start through the functions start uses, split out of it. The iOS Claude job's smoke test also gets BACKEND_DIR, as the app job's does. [claude-test] [ci-backend]
438 lines
17 KiB
Bash
Executable File
438 lines
17 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# Test backend on the app runner: a chat relay and a directory service on the preset servers, with one group
|
|
# and one channel registered and approved by a CLI peer, and another CLI peer that must find both by sending the
|
|
# directory the search text, and a third that must find both with a search request, as the app does.
|
|
# Both have profile images as large as the apps make them, so their search results don't fit in one service
|
|
# response, and the search request must page through them.
|
|
# For SimpleX names, an SMP server on 127.0.0.1 resolves .testing names through a local resolver, and a CLI peer
|
|
# with an address on it claims the name seeded.testing.
|
|
# Usage:
|
|
# scripts/ci/desktop-linux/backend.sh start <backend .tar.gz> <work dir> writes the directory's short address to <work dir>/directory-link
|
|
# and the names server's address to <work dir>/names-server
|
|
# scripts/ci/desktop-linux/backend.sh resume <backend .tar.gz> <work dir> starts again, on another runner, what start left stopped in
|
|
# <work dir>, with the same addresses
|
|
# scripts/ci/desktop-linux/backend.sh stop <work dir> stops the servers, prints their logs
|
|
# The relay, the directory, the SMP server and the resolver keep running between start and stop.
|
|
|
|
set -euo pipefail
|
|
|
|
here=$(cd "$(dirname "$0")" && pwd)
|
|
directory_name="SimpleX Directory"
|
|
admin=ci_admin
|
|
search_text="ci seed"
|
|
# the longest profile image data URI the apps make and the core accepts (maxProfileImageSize)
|
|
max_image_size=12500
|
|
# the largest service response body, compressed if it has to be (maxServiceBodyLength)
|
|
max_service_body=10737
|
|
name_label=seeded
|
|
resolver_port=8090
|
|
|
|
# Runs one chat command in a CLI client that stays online for <secs> after it; the output is in $work/out.
|
|
chat() {
|
|
local db=$1 secs=$2 cmd=$3
|
|
shift 3
|
|
local status=0
|
|
"$bin/simplex-chat" -d "$work/$db" -e "$cmd" -t "$secs" --execute-log all "$@" < /dev/null > "$work/out" 2>&1 || status=$?
|
|
cat "$work/out" >> "$work/$db-cli.log"
|
|
# commands with images are thousands of characters long
|
|
echo "::group::$db: ${cmd:0:200}"
|
|
cat "$work/out"
|
|
echo "::endgroup::"
|
|
return "$status"
|
|
}
|
|
|
|
admin() {
|
|
chat "$admin" "$1" "$2" --user-display-name "$admin"
|
|
}
|
|
|
|
expect_out() {
|
|
if ! grep -qE "$1" "$work/out"; then
|
|
echo "error: \"$1\" not in the output"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Each run keeps the client online for a few seconds, which also lets the handshakes it waits for progress.
|
|
until_shown() {
|
|
local client=$1 pattern=$2 secs=$3 cmd=$4
|
|
local deadline=$((SECONDS + secs))
|
|
until "$client" 5 "$cmd" && grep -qE "$pattern" "$work/out"; do
|
|
if [ "$SECONDS" -ge "$deadline" ]; then
|
|
echo "error: \"$pattern\" not shown by \"$cmd\" after $secs s"
|
|
return 1
|
|
fi
|
|
done
|
|
}
|
|
|
|
admin_until() {
|
|
until_shown admin "$@"
|
|
}
|
|
|
|
db_query() {
|
|
sqlite3 "$work/$1_chat.db" "$2"
|
|
}
|
|
|
|
# Prints a profile image data URI made as the apps make one: JPEG at quality 85, scaled down by the square root
|
|
# of its excess, at most halved per step, until it is at most max_image_size characters. The picture fills the
|
|
# whole square, a gradient between two shades of one colour with a large white letter; noise keeps JPEG from
|
|
# compressing it much, so the image ends up near the limit.
|
|
avatar() {
|
|
local top=$1 bottom=$2 letter=$3 source=$work/avatar-source.png file=$work/avatar.jpg size=512 uri font
|
|
font=$(fc-match -f '%{file}' 'DejaVu Sans:bold')
|
|
convert -size 512x512 gradient:"$top-$bottom" -seed 1 -attenuate 0.5 +noise Gaussian \
|
|
-font "$font" -pointsize 320 -fill white -gravity center -annotate +0+0 "$letter" "$source"
|
|
while :; do
|
|
convert "$source" -resize "${size}x${size}" -quality 85 "$file"
|
|
uri="data:image/jpg;base64,$(base64 -w 0 "$file")"
|
|
[ "${#uri}" -gt "$max_image_size" ] || break
|
|
size=$(awk -v s="$size" -v n="${#uri}" -v m="$max_image_size" 'BEGIN { r = sqrt(n / m); if (r > 2) r = 2; print int(s / r) }')
|
|
done
|
|
echo "avatar \"$letter\" over $top to $bottom: ${size}x${size} px, ${#uri} characters, mean colour $(convert "$file" -resize '1x1!' -format '%[pixel:p{0,0}]' info:)" >&2
|
|
echo "$uri"
|
|
}
|
|
|
|
run_relay() {
|
|
nohup "$bin/simplex-chat" -d "$work/relay" --relay --headless --user-display-name ci_relay < /dev/null >> "$work/relay.log" 2>&1 &
|
|
echo $! > "$work/relay.pid"
|
|
}
|
|
|
|
start_relay() {
|
|
run_relay
|
|
local deadline=$((SECONDS + 120))
|
|
until grep -qE '^https://[^ ]+/r#' "$work/relay.log"; do
|
|
if ! kill -0 "$(cat "$work/relay.pid")" 2> /dev/null || [ "$SECONDS" -ge "$deadline" ]; then
|
|
echo "error: the relay did not create its address"
|
|
cat "$work/relay.log"
|
|
return 1
|
|
fi
|
|
sleep 1
|
|
done
|
|
relay_link=$(grep -m 1 -oE '^https://[^ ]+/r#[^ ]+' "$work/relay.log")
|
|
echo "relay address: $relay_link"
|
|
}
|
|
|
|
# The address is created here, like the directory tests do, with auto-accept on before the service subscribes to it.
|
|
create_directory() {
|
|
chat directory 1 "/ad pq_ratchet=on" --create-bot-display-name "$directory_name"
|
|
directory_link=$(awk '/Your new chat address is created!/ {getline; getline; print; exit}' "$work/out")
|
|
case "$directory_link" in
|
|
https://*/a#*) ;;
|
|
*)
|
|
echo "error: no short directory address in the output"
|
|
return 1
|
|
;;
|
|
esac
|
|
chat directory 1 "/auto_accept on"
|
|
expect_out "^auto_accept on"
|
|
echo "$directory_link" > "$work/directory-link"
|
|
echo "directory address: $directory_link"
|
|
}
|
|
|
|
start_directory() {
|
|
# -y: the service adds its own tables to the database the CLI created, and would otherwise ask on stdin
|
|
nohup "$bin/simplex-directory-service" -d "$work/directory" -y --super-users "$1" < /dev/null >> "$work/directory.log" 2>&1 &
|
|
echo $! > "$work/directory.pid"
|
|
wait_connected directory
|
|
}
|
|
|
|
# A chat process shows that its chat has started by connecting to its servers: its output is block-buffered.
|
|
wait_connected() {
|
|
local pid deadline=$((SECONDS + 60))
|
|
pid=$(cat "$work/$1.pid")
|
|
until grep -q "pid=$pid," <<< "$(ss -Htnp state established)"; do
|
|
if ! kill -0 "$pid" 2> /dev/null || [ "$SECONDS" -ge "$deadline" ]; then
|
|
echo "error: $1 did not start"
|
|
cat "$work/$1.log"
|
|
return 1
|
|
fi
|
|
sleep 1
|
|
done
|
|
}
|
|
|
|
# Waits for the process to exit, which completes its output, and kills it after 30 s.
|
|
stop_process() {
|
|
local pid deadline
|
|
pid=$(cat "$work/$1.pid")
|
|
kill "-$2" "$pid" 2> /dev/null || true
|
|
deadline=$((SECONDS + 30))
|
|
while kill -0 "$pid" 2> /dev/null; do
|
|
if [ "$SECONDS" -ge "$deadline" ]; then
|
|
kill -KILL "$pid" 2> /dev/null || true
|
|
fi
|
|
sleep 1
|
|
done
|
|
}
|
|
|
|
# A super user is known by contact ID, which exists only after the admin connects, so the service starts
|
|
# with nobody and is restarted with the admin.
|
|
connect_admin() {
|
|
admin 10 "/c $directory_link"
|
|
expect_out "connection request sent!"
|
|
admin_until "Welcome to $directory_name" 180 "/tail @'$directory_name' 10"
|
|
local id
|
|
id=$(db_query directory "SELECT contact_id FROM contacts WHERE local_display_name = '$admin'")
|
|
stop_process directory INT
|
|
start_directory "$id:$admin"
|
|
echo "directory restarted with super user $id:$admin"
|
|
}
|
|
|
|
# The directory sends each registration to super users as a message with the exact /approve command.
|
|
approve() {
|
|
local name=$1 reply=$2
|
|
admin_until "/approve [0-9]+:$name [0-9]+" 300 "/tail @'$directory_name' 20"
|
|
local cmd
|
|
cmd=$(grep -oE "/approve [0-9]+:$name [0-9]+" "$work/out" | tail -n 1)
|
|
admin 5 "@'$directory_name' $cmd"
|
|
admin_until "$reply" 120 "/tail @'$directory_name' 20"
|
|
}
|
|
|
|
# The CLI's group commands can't set an image, so groups are created with the API commands, with the profiles
|
|
# that "/g" and "/public group" make plus the image.
|
|
group_json() {
|
|
local name=$1 descr=$2 image=$3 prefs=$4
|
|
echo "{\"displayName\":\"$name\",\"fullName\":\"\",\"shortDescr\":\"$descr\",\"image\":\"$image\",\"groupPreferences\":{\"directMessages\":{\"enable\":\"on\"},\"history\":{\"enable\":\"on\"}$prefs}}"
|
|
}
|
|
|
|
seed_group() {
|
|
local user_id
|
|
user_id=$(db_query "$admin" "SELECT user_id FROM users WHERE local_display_name = '$admin'")
|
|
group_image=$(avatar "rgb(235,80,70)" "rgb(150,25,30)" G)
|
|
admin 1 "/_group $user_id $(group_json ci_group "CI seeded group" "$group_image" "")"
|
|
expect_out "group #ci_group \(CI seeded group\) is created"
|
|
admin 5 "/a ci_group '$directory_name' admin"
|
|
expect_out "invitation to join the group #ci_group sent to"
|
|
approve ci_group "Group approved!"
|
|
}
|
|
|
|
seed_channel() {
|
|
admin 1 "/relays name=ci_relay $relay_link"
|
|
expect_out "^ok$"
|
|
local user_id relay_id
|
|
user_id=$(db_query "$admin" "SELECT user_id FROM users WHERE local_display_name = '$admin'")
|
|
relay_id=$(db_query "$admin" "SELECT chat_relay_id FROM chat_relays WHERE display_name = 'ci_relay' AND deleted = 0")
|
|
channel_image=$(avatar "rgb(80,200,90)" "rgb(25,120,45)" C)
|
|
admin 10 "/_public group $user_id $relay_id $(group_json ci_channel "CI seeded channel" "$channel_image" ",\"support\":{\"enable\":\"off\"}")"
|
|
expect_out "group #ci_channel \(CI seeded channel\) is created"
|
|
local group_id
|
|
group_id=$(db_query "$admin" "SELECT group_id FROM groups WHERE local_display_name = 'ci_channel'")
|
|
admin_until "relay id [0-9]+: active" 180 "/_get relays #$group_id"
|
|
admin 5 "/share chat #ci_channel @'$directory_name'"
|
|
expect_out "link to join channel #ci_channel"
|
|
approve ci_channel "Channel approved!"
|
|
}
|
|
|
|
# An SMP server on 127.0.0.1 whose [NAMES] section asks the local resolver, which answers from names.json.
|
|
start_names_server() {
|
|
local cfg=$work/smp/etc logs=$work/smp/var ini
|
|
echo '{}' > "$work/names.json"
|
|
SMP_SERVER_CFG_PATH=$cfg SMP_SERVER_LOG_PATH=$logs "$bin/smp-server" init -y --ip 127.0.0.1 --no-password --disable-web
|
|
ini=$cfg/smp-server.ini
|
|
# only 5223: 443 needs root
|
|
sed -i -e 's/^port = 5223,443$/port = 5223/' \
|
|
-e '/^\[NAMES\]/,/^\[/{s/^enable: off$/enable: on/;s|^# resolver_endpoint: http://127.0.0.1:8000$|resolver_endpoint: http://127.0.0.1:'"$resolver_port"'|}' "$ini"
|
|
if ! grep -qx 'port = 5223' "$ini" ||
|
|
! sed -n '/^\[NAMES\]/,/^\[/p' "$ini" | grep -qx 'enable: on' ||
|
|
! sed -n '/^\[NAMES\]/,/^\[/p' "$ini" | grep -qx "resolver_endpoint: http://127.0.0.1:$resolver_port"; then
|
|
echo "error: $ini has a different port or [NAMES] than expected:"
|
|
grep -E '^port|^\[NAMES\]|enable|resolver' "$ini"
|
|
return 1
|
|
fi
|
|
run_names_server
|
|
names_server="smp://$(cat "$cfg/fingerprint")@127.0.0.1"
|
|
echo "$names_server" > "$work/names-server"
|
|
echo "names server: $names_server"
|
|
}
|
|
|
|
run_names_server() {
|
|
local deadline
|
|
nohup python3 "$here/names-resolver.py" "$resolver_port" "$work/names.json" < /dev/null >> "$work/resolver.log" 2>&1 &
|
|
echo $! > "$work/resolver.pid"
|
|
SMP_SERVER_CFG_PATH=$work/smp/etc SMP_SERVER_LOG_PATH=$work/smp/var nohup "$bin/smp-server" start < /dev/null >> "$work/smp-server.log" 2>&1 &
|
|
echo $! > "$work/smp-server.pid"
|
|
deadline=$((SECONDS + 60))
|
|
until (exec 3<> /dev/tcp/127.0.0.1/5223) 2> /dev/null; do
|
|
if ! kill -0 "$(cat "$work/smp-server.pid")" 2> /dev/null || [ "$SECONDS" -ge "$deadline" ]; then
|
|
echo "error: the SMP server is not listening on 5223"
|
|
cat "$work/smp-server.log"
|
|
return 1
|
|
fi
|
|
sleep 1
|
|
done
|
|
}
|
|
|
|
# A CLI client started with -s uses only that server and, once its user exists, gives it every role, names included.
|
|
names_client() {
|
|
local db=$1
|
|
shift
|
|
chat "$db" "$@" --user-display-name "$db" -s "$names_server"
|
|
}
|
|
|
|
# ci_contact's address is on the names server; the resolver maps seeded.testing to it, and ci_contact claims the name.
|
|
seed_name() {
|
|
local image link
|
|
image=$(avatar "rgb(70,130,240)" "rgb(30,50,170)" N)
|
|
names_client ci_contact 1 "/_profile 1 {\"displayName\":\"ci_contact\",\"fullName\":\"\",\"shortDescr\":\"CI seeded contact\",\"image\":\"$image\"}"
|
|
names_client ci_contact 3 "/ad"
|
|
link=$(awk '/Your new chat address is created!/ {getline; getline; print; exit}' "$work/out")
|
|
case "$link" in
|
|
https://*/a#*) ;;
|
|
*)
|
|
echo "error: no short address for ci_contact in the output"
|
|
return 1
|
|
;;
|
|
esac
|
|
jq -n --arg n "$name_label" --arg l "$link" '{($n): {simplexContact: [$l]}}' > "$work/names.json"
|
|
names_client ci_contact 5 "/_set domain 1 $name_label.testing"
|
|
# what the app does on the Search key, from a client that knows nothing but the names server; it finds the
|
|
# address only if ci_contact's claim is in the address data
|
|
names_client name_probe 1 "/user"
|
|
names_client name_probe 5 "/_connect plan 1 @$name_label.testing"
|
|
expect_out "contact address: ok to connect"
|
|
echo "@$name_label.testing resolves to ci_contact's address"
|
|
}
|
|
|
|
searcher() {
|
|
chat searcher "$1" "$2" --user-display-name ci_searcher
|
|
}
|
|
|
|
# A new user connects to the directory and sends the search text; it replies with the count, then one message per
|
|
# group or channel found, with its image.
|
|
search_by_message() {
|
|
searcher 10 "/c $directory_link"
|
|
expect_out "connection request sent!"
|
|
until_shown searcher "Welcome to $directory_name" 180 "/tail @'$directory_name' 10"
|
|
searcher 5 "@'$directory_name' $search_text"
|
|
until_shown searcher "Found 2 group\(s\)" 120 "/tail @'$directory_name' 10"
|
|
until_shown searcher "ci_group" 60 "/tail @'$directory_name' 10"
|
|
until_shown searcher "ci_channel" 60 "/tail @'$directory_name' 10"
|
|
echo "search for \"$search_text\" by message found ci_group and ci_channel"
|
|
}
|
|
|
|
page_summary() {
|
|
local resp=$1 compressed=""
|
|
if command -v zstd > /dev/null; then
|
|
compressed=", $(printf '%s' "$resp" | zstd -3 -q -c | wc -c) bytes zstd-compressed"
|
|
fi
|
|
jq -r '[.entries[] | "\(.displayName) (image: \(.image // "" | length) characters)"] | join(", ") | "entries: " + .' <<< "$resp"
|
|
echo "body: ${#resp} bytes$compressed, limit $max_service_body"
|
|
}
|
|
|
|
# What the app will send, from a client that never connected to the directory, following the cursor to the last page.
|
|
# The images make the results too large for one response, so the directory must split them, and each must arrive
|
|
# once and with its whole image.
|
|
probe_search() {
|
|
local cursor="" pages=0 started resp req
|
|
: > "$work/probe-entries"
|
|
while :; do
|
|
pages=$((pages + 1))
|
|
if [ "$pages" -gt 10 ]; then
|
|
echo "error: the search is still returning pages after 10"
|
|
return 1
|
|
fi
|
|
req="{\"type\":\"search\",\"searchText\":\"$search_text\"${cursor:+,\"searchCursor\":$cursor}}"
|
|
started=$SECONDS
|
|
chat probe 1 "/_service_request 1 $directory_link timeout=30 $req" --user-display-name ci_probe
|
|
resp=$(sed -n 's/^service response: //p' "$work/out")
|
|
if [ -z "$resp" ]; then
|
|
echo "error: no service response for page $pages"
|
|
return 1
|
|
fi
|
|
echo "page $pages, in $((SECONDS - started)) s:"
|
|
page_summary "$resp"
|
|
jq -r '.entries[] | "\(.displayName) \(.image // "" | length)"' <<< "$resp" >> "$work/probe-entries"
|
|
cursor=$(jq -c '.searchCursor // empty' <<< "$resp")
|
|
[ -n "$cursor" ] || break
|
|
done
|
|
local expected="ci_channel ${#channel_image}
|
|
ci_group ${#group_image}"
|
|
if [ "$(sort "$work/probe-entries")" != "$expected" ]; then
|
|
echo "error: expected each result once with its whole image (name, image length):"
|
|
echo "$expected"
|
|
echo "got:"
|
|
sort "$work/probe-entries"
|
|
return 1
|
|
fi
|
|
if [ "$pages" -lt 2 ]; then
|
|
echo "error: the results fit in one response, so the search no longer tests paging"
|
|
return 1
|
|
fi
|
|
echo "search for \"$search_text\" found ci_group and ci_channel with their images, in $pages pages"
|
|
}
|
|
|
|
dump_directory() {
|
|
echo "::group::directory registrations"
|
|
db_query directory "SELECT group_id, group_reg_status FROM sx_directory_group_regs" || true
|
|
echo "::endgroup::"
|
|
}
|
|
|
|
start() {
|
|
bin=$work/bin
|
|
mkdir -p "$bin"
|
|
tar -C "$bin" -xzf "$1"
|
|
"$bin/simplex-chat" --version
|
|
"$bin/simplex-directory-service" --version
|
|
|
|
start_relay
|
|
create_directory
|
|
start_directory "0:nobody"
|
|
trap dump_directory EXIT
|
|
connect_admin
|
|
seed_group
|
|
seed_channel
|
|
search_by_message
|
|
probe_search
|
|
start_names_server
|
|
seed_name
|
|
}
|
|
|
|
# The servers' keys and the clients' databases stay in the work dir, so its addresses are the same again.
|
|
resume() {
|
|
bin=$work/bin
|
|
mkdir -p "$bin"
|
|
tar -C "$bin" -xzf "$1"
|
|
run_relay
|
|
wait_connected relay
|
|
start_directory "$(db_query directory "SELECT contact_id FROM contacts WHERE local_display_name = '$admin'"):$admin"
|
|
run_names_server
|
|
echo "resumed: directory $(cat "$work/directory-link"), names server $(cat "$work/names-server")"
|
|
}
|
|
|
|
stop() {
|
|
local p
|
|
for p in directory relay smp-server; do
|
|
if [ -f "$work/$p.pid" ]; then
|
|
stop_process "$p" INT
|
|
fi
|
|
done
|
|
# Python, started in the background by a script, ignores SIGINT
|
|
if [ -f "$work/resolver.pid" ]; then
|
|
stop_process resolver TERM
|
|
fi
|
|
for p in directory relay smp-server resolver; do
|
|
echo "::group::$p log"
|
|
cat "$work/$p.log" 2> /dev/null || true
|
|
echo "::endgroup::"
|
|
done
|
|
}
|
|
|
|
case "${1:-}" in
|
|
start)
|
|
work=$3
|
|
mkdir -p "$work"
|
|
start "$2"
|
|
;;
|
|
resume)
|
|
work=$3
|
|
resume "$2"
|
|
;;
|
|
stop)
|
|
work=$2
|
|
stop
|
|
;;
|
|
*)
|
|
echo "usage: $0 start <backend .tar.gz> <work dir> | resume <backend .tar.gz> <work dir> | stop <work dir>"
|
|
exit 1
|
|
;;
|
|
esac
|