mirror of
https://github.com/gadgethd/ukmesh.git
synced 2026-09-01 17:08:17 +00:00
hermes-gadgetandgadgethd
2fa099ff8b
* fix: unblock fresh database migrations (#44) * fix: remove retired CI components (#48) * fix: restore frontend CI and RF control (#49) --------- Co-authored-by: gadgethd <111318106+gadgethd@users.noreply.github.com>
402 lines
17 KiB
YAML
402 lines
17 KiB
YAML
name: CI
|
|
|
|
on:
|
|
push:
|
|
branches: ['**']
|
|
pull_request:
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
concurrency:
|
|
group: ci-${{ github.workflow }}-${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
backend:
|
|
name: Backend
|
|
runs-on: ubuntu-latest
|
|
services:
|
|
postgres:
|
|
image: timescale/timescaledb@sha256:22e8a5ae7aef121d1537afe946dd7cc5deeeb63ab36ce19849d671bd3b663509
|
|
env:
|
|
POSTGRES_PASSWORD: analysis-test-password
|
|
ports:
|
|
- 55433:5432
|
|
options: >-
|
|
--health-cmd "pg_isready -U postgres"
|
|
--health-interval 5s
|
|
--health-timeout 3s
|
|
--health-retries 10
|
|
redis:
|
|
image: redis@sha256:8b81dd37ff027bec4e516d41acfbe9fe2460070dc6d4a4570a2ac5b9d59df065
|
|
ports:
|
|
- 6379:6379
|
|
options: >-
|
|
--health-cmd "redis-cli ping"
|
|
--health-interval 5s
|
|
--health-timeout 3s
|
|
--health-retries 10
|
|
env:
|
|
TEST_REDIS_URL: redis://127.0.0.1:6379
|
|
TEST_ANALYSIS_DATABASE_URL: postgresql://postgres:analysis-test-password@127.0.0.1:55433/postgres
|
|
TEST_INGEST_DATABASE_URL: postgresql://postgres:analysis-test-password@127.0.0.1:55433/postgres
|
|
defaults:
|
|
run:
|
|
working-directory: backend
|
|
steps:
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
|
with:
|
|
node-version: 20
|
|
cache: npm
|
|
cache-dependency-path: backend/package-lock.json
|
|
- run: npm ci
|
|
- name: Audit backend dependencies
|
|
run: npm audit --audit-level=high
|
|
- run: npm run typecheck
|
|
- run: npm run contract:check
|
|
- run: npm test
|
|
- run: npm run test:analysis-integration
|
|
- run: npm run test:ingest-integration
|
|
- run: npm run build
|
|
|
|
frontend:
|
|
name: Frontend
|
|
runs-on: ubuntu-latest
|
|
defaults:
|
|
run:
|
|
working-directory: frontend
|
|
steps:
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
|
with:
|
|
node-version: 22
|
|
cache: npm
|
|
cache-dependency-path: frontend/package-lock.json
|
|
- run: npm ci
|
|
- name: Audit frontend dependencies
|
|
run: npm audit --audit-level=high
|
|
- run: npm test
|
|
- run: npm run build
|
|
- run: npx playwright install --with-deps chromium
|
|
- run: npm run test:e2e
|
|
|
|
workers-and-compose:
|
|
name: Workers and Compose
|
|
runs-on: ubuntu-latest
|
|
env:
|
|
POSTGRES_PASSWORD: ci-postgres-password
|
|
REDIS_PASSWORD: ci-redis-password
|
|
JWT_SECRET: ci-test-only-ci-test-only-ci-test-only-ci-test-only
|
|
OPERATOR_SITE_TOKEN: ci-operator-site-token-0123456789abcdef0123456789abcdef
|
|
MQTT_USERNAME: backend
|
|
MQTT_PASSWORD: ci-mqtt-password
|
|
ANUBIS_ED25519_PRIVATE_KEY_HEX: '0000000000000000000000000000000000000000000000000000000000000001'
|
|
HEALTHCHECK_SOURCE_REF: 952eec8e0c55d0749f9abef211c266f992c0bfc7
|
|
GRAFANA_ADMIN_PASSWORD: ci-grafana-password
|
|
SOURCE_REVISION: ${{ github.sha }}
|
|
MOSQUITTO_IMAGE: eclipse-mosquitto@sha256:9cfdd46ad59f3e3e5f592f6baf57ab23e1ad00605509d0f5c1e9b179c5314d87
|
|
steps:
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
|
with:
|
|
node-version: 22
|
|
- name: Validate release security waivers
|
|
run: node scripts/check-trivy-waivers.mjs
|
|
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
|
|
with:
|
|
python-version: '3.12'
|
|
- name: Test operator scripts
|
|
working-directory: scripts
|
|
run: |
|
|
npm ci
|
|
npm audit --audit-level=high
|
|
npm run typecheck
|
|
npm test
|
|
- name: Compile Python workers
|
|
run: python -m py_compile viewshed-worker/worker.py viewshed-worker/link_queue_v3.py viewshed-worker/viewshed_queue_v2.py viewshed-worker/backfill_profiles.py viewshed-worker/rf/*.py
|
|
- name: Build worker images
|
|
run: docker build -f viewshed-worker/Dockerfile -t meshcore-viewshed-worker:test viewshed-worker
|
|
- name: Test retained observed-link worker image
|
|
run: |
|
|
trap 'docker rm -f link-queue-test-redis >/dev/null 2>&1 || true' EXIT
|
|
docker run -d --name link-queue-test-redis \
|
|
-p 127.0.0.1:6380:6379 \
|
|
redis@sha256:8b81dd37ff027bec4e516d41acfbe9fe2460070dc6d4a4570a2ac5b9d59df065 redis-server --requirepass test-queue-password --maxmemory-policy noeviction
|
|
for _ in $(seq 1 30); do
|
|
if docker exec link-queue-test-redis redis-cli -a test-queue-password --no-auth-warning ping >/dev/null; then
|
|
break
|
|
fi
|
|
sleep 1
|
|
done
|
|
docker run --rm --network host --entrypoint python \
|
|
-e TEST_REDIS_URL=redis://127.0.0.1:6380 \
|
|
-e TEST_REDIS_PASSWORD=test-queue-password \
|
|
meshcore-viewshed-worker:test -m unittest discover -s /app/tests -v
|
|
docker run --rm --entrypoint python \
|
|
meshcore-viewshed-worker:test /app/tests/benchmark_rf.py --rays 12 --steps 1000
|
|
docker run --rm --network none --entrypoint python \
|
|
meshcore-viewshed-worker:test /app/tests/benchmark_rf_radial.py --rays 72 --steps 500 --repeats 1
|
|
- name: Test HopReach parity and progressive publication
|
|
run: |
|
|
docker run --rm \
|
|
--user "$(id -u):$(id -g)" \
|
|
-e HOME=/tmp \
|
|
-v "$PWD:/work" \
|
|
-w /work/third_party/hopreach \
|
|
golang:1.23-bookworm \
|
|
go test ./...
|
|
- name: Enforce HopReach UK performance release gate
|
|
run: |
|
|
profile_dir="$(mktemp -d)"
|
|
trap 'rm -rf "$profile_dir"' EXIT
|
|
docker run --rm \
|
|
--user "$(id -u):$(id -g)" \
|
|
-e HOME=/tmp \
|
|
-e HOPREACH_PROFILE_DIR=/profiles \
|
|
-v "$PWD:/work" \
|
|
-v "$profile_dir:/profiles" \
|
|
-w /work \
|
|
golang:1.23-bookworm \
|
|
/work/scripts/benchmark-hopreach.sh
|
|
- name: Bootstrap clean Mosquitto credentials
|
|
env:
|
|
MOSQUITTO_CONFIG_DIR: /tmp/meshcore-ci-mosquitto
|
|
run: |
|
|
install -d -m 700 "$MOSQUITTO_CONFIG_DIR"
|
|
cp mosquitto/mosquitto.conf "$MOSQUITTO_CONFIG_DIR/mosquitto.conf"
|
|
scripts/bootstrap-mosquitto.sh
|
|
for credential in passwd acl; do
|
|
mode="$(
|
|
docker run --rm \
|
|
--user 1000:1883 \
|
|
--cap-drop ALL \
|
|
--entrypoint stat \
|
|
-v "$MOSQUITTO_CONFIG_DIR:/mosquitto/config:ro" \
|
|
"$MOSQUITTO_IMAGE" \
|
|
-c '%a' "/mosquitto/config/$credential"
|
|
)"
|
|
test "$mode" = 640
|
|
done
|
|
- name: Validate Compose configuration
|
|
run: |
|
|
docker compose config --quiet
|
|
docker compose config --format json \
|
|
| jq -e '
|
|
(.services.timescaledb.command
|
|
| index("max_worker_processes=24") != null)
|
|
and (.networks.default.ipam.config[0].ip_range
|
|
== "172.30.0.128/25")' >/dev/null
|
|
- name: Validate tracked build and Compose inventory
|
|
run: |
|
|
while IFS= read -r context; do
|
|
[[ "$context" == *'${{'* ]] && continue
|
|
test -d "$context"
|
|
done < <(sed -n 's/^[[:space:]]*context: //p' .github/workflows/release.yml)
|
|
while IFS= read -r dockerfile; do
|
|
[[ "$dockerfile" == *'${{'* ]] && continue
|
|
test -f "$dockerfile"
|
|
done < <(sed -n 's/^[[:space:]]*file: //p' .github/workflows/release.yml)
|
|
compose_services="$(docker compose config --services)"
|
|
for service in \
|
|
db-migrate backend app-ukmesh website-ukmesh website-dev mesh-health-check \
|
|
mosquitto-reloader link-worker link-backfill-worker hopreach \
|
|
timescaledb mosquitto; do
|
|
grep -Fxq "$service" <<<"$compose_services"
|
|
done
|
|
- name: Build every application image and smoke an empty-volume stack
|
|
run: |
|
|
project="meshcore-ci-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
|
publisher_user="ci-e2e-publisher"
|
|
publisher_password="ci-e2e-publisher-password"
|
|
observer="CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC"
|
|
cleanup() {
|
|
docker compose \
|
|
--project-name "$project" \
|
|
--profile dev \
|
|
down --volumes --remove-orphans
|
|
}
|
|
trap cleanup EXIT
|
|
# Create each Argon2 entry independently. Updating an existing
|
|
# password file with this image is not reliable on hosted runners.
|
|
docker run --rm \
|
|
--user "$(id -u):$(id -g)" \
|
|
-v "$PWD/mosquitto:/mosquitto/config" \
|
|
"$MOSQUITTO_IMAGE" \
|
|
mosquitto_passwd -b -c /mosquitto/config/.passwd-backend \
|
|
"$MQTT_USERNAME" "$MQTT_PASSWORD"
|
|
docker run --rm \
|
|
--user "$(id -u):$(id -g)" \
|
|
-v "$PWD/mosquitto:/mosquitto/config" \
|
|
"$MOSQUITTO_IMAGE" \
|
|
mosquitto_passwd -b -c /mosquitto/config/.passwd-publisher \
|
|
"$publisher_user" "$publisher_password"
|
|
cp mosquitto/.passwd-backend mosquitto/passwd
|
|
cat mosquitto/.passwd-publisher >>mosquitto/passwd
|
|
unlink mosquitto/.passwd-backend
|
|
unlink mosquitto/.passwd-publisher
|
|
printf 'user %s\ntopic read meshcore/#\ntopic read ukmesh/#\ntopic read meshcore-test/#\n' \
|
|
"$MQTT_USERNAME" >mosquitto/acl
|
|
printf '\nuser %s\ntopic write meshcore-test/CI/%s/packets\ntopic write meshcore-test/CI/%s/status\n' \
|
|
"$publisher_user" "$observer" "$observer" >>mosquitto/acl
|
|
chmod 640 mosquitto/passwd mosquitto/acl
|
|
scripts/bootstrap-mosquitto.sh
|
|
docker build \
|
|
--build-arg SOURCE_REVISION="$SOURCE_REVISION" \
|
|
--build-arg VITE_APP_HOSTNAME=app.ukmesh.com \
|
|
-f Dockerfile -t meshcore-combined:test .
|
|
docker compose \
|
|
--project-name "$project" \
|
|
--profile dev \
|
|
build \
|
|
backend app-ukmesh website-ukmesh website-dev mesh-health-check \
|
|
mosquitto-reloader link-worker hopreach
|
|
docker compose \
|
|
--project-name "$project" \
|
|
--profile dev \
|
|
up --detach --no-build --scale link-backfill-worker=0 \
|
|
--wait --wait-timeout 360
|
|
# link-backfill-worker is intentionally a finite job. Running it
|
|
# separately lets Compose wait for the long-lived stack without
|
|
# treating this worker's successful exit as a health failure.
|
|
docker compose \
|
|
--project-name "$project" \
|
|
--profile dev \
|
|
run --rm --no-deps link-backfill-worker
|
|
health_session_id="$(
|
|
curl --fail --silent --request POST \
|
|
--header 'Content-Type: application/json' \
|
|
--data '{}' \
|
|
http://127.0.0.1:3090/api/sessions \
|
|
| jq -er '.id'
|
|
)"
|
|
sleep 1
|
|
docker compose --project-name "$project" exec -T \
|
|
-e HEALTH_SESSION_ID="$health_session_id" \
|
|
mesh-health-check node --input-type=module <<'NODE'
|
|
import { readFileSync } from 'node:fs';
|
|
const payload = JSON.parse(
|
|
readFileSync('/app/data/session-results.json', 'utf8'),
|
|
);
|
|
if (!payload.sessions?.some(({ id }) => id === process.env.HEALTH_SESSION_ID)) {
|
|
throw new Error('health-check session was not persisted by the non-root runtime');
|
|
}
|
|
NODE
|
|
curl --fail --silent http://127.0.0.1:3000/readyz \
|
|
| jq -e '.status == "ready"' >/dev/null
|
|
curl --fail --silent http://127.0.0.1:3003/ >/dev/null
|
|
curl --fail --silent http://127.0.0.1:3004/ >/dev/null
|
|
curl --fail --silent http://127.0.0.1:3003/readyz \
|
|
| jq -e '.status == "ready"' >/dev/null
|
|
curl --fail --silent http://127.0.0.1:3004/readyz \
|
|
| jq -e '.status == "ready"' >/dev/null
|
|
for origin in http://127.0.0.1:3003 http://127.0.0.1:3004; do
|
|
for path in /metrics /operations /observer-registrations /local-api/operations; do
|
|
test "$(curl --silent --output /dev/null --write-out '%{http_code}' "${origin}${path}")" = 404
|
|
done
|
|
done
|
|
backend_metrics="$(
|
|
docker compose --project-name "$project" exec -T backend \
|
|
wget -qO- http://127.0.0.1:9091/metrics
|
|
)"
|
|
grep -q '^meshcore_process_' <<<"$backend_metrics"
|
|
ws_log="$(mktemp)"
|
|
docker compose --project-name "$project" exec -T \
|
|
-e MQTT_E2E_OBSERVER="$observer" \
|
|
backend node --input-type=module >"$ws_log" <<'NODE' &
|
|
import WebSocket from 'ws';
|
|
const observer = process.env.MQTT_E2E_OBSERVER.toLowerCase();
|
|
const socket = new WebSocket('ws://website-ukmesh:8080/ws?network=test');
|
|
const timeout = setTimeout(() => {
|
|
console.error('timed out waiting for MQTT packet WebSocket fanout');
|
|
process.exit(1);
|
|
}, 15_000);
|
|
socket.on('message', (raw) => {
|
|
for (const line of String(raw).split('\n').filter(Boolean)) {
|
|
const message = JSON.parse(line);
|
|
if (
|
|
message.type === 'packet'
|
|
&& String(message.data?.rxNodeId ?? '').toLowerCase() === observer
|
|
) {
|
|
clearTimeout(timeout);
|
|
console.log('mqtt-packet-fanout-ok');
|
|
socket.close();
|
|
process.exit(0);
|
|
}
|
|
}
|
|
});
|
|
socket.on('error', (error) => {
|
|
console.error(error);
|
|
process.exit(1);
|
|
});
|
|
NODE
|
|
ws_pid=$!
|
|
sleep 2
|
|
docker compose --project-name "$project" exec -T mosquitto \
|
|
mosquitto_pub -h 127.0.0.1 -p 1883 \
|
|
-u "$publisher_user" -P "$publisher_password" \
|
|
-t "meshcore-test/CI/${observer}/status" \
|
|
-m '{"origin":"CI E2E observer","origin_id":"CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC","stats":{"battery_mv":4100,"uptime_secs":60}}'
|
|
docker compose --project-name "$project" exec -T mosquitto \
|
|
mosquitto_pub -h 127.0.0.1 -p 1883 \
|
|
-u "$publisher_user" -P "$publisher_password" \
|
|
-t "meshcore-test/CI/${observer}/packets" \
|
|
-m '{"packet_type":1,"RSSI":-75,"SNR":8,"direction":"rx","raw":"0D02AABBDEADBEEF"}'
|
|
readback=""
|
|
for _ in $(seq 1 30); do
|
|
readback="$(
|
|
docker compose --project-name "$project" exec -T timescaledb \
|
|
psql -U meshcore -d meshcore -Atc \
|
|
"SELECT EXISTS(SELECT 1 FROM nodes WHERE node_id = '${observer}' AND network = 'test'), EXISTS(SELECT 1 FROM packets WHERE rx_node_id = '${observer}' AND network = 'test'), EXISTS(SELECT 1 FROM packet_hourly_stats WHERE network = 'test')"
|
|
)"
|
|
if [ "$readback" = "t|t|t" ]; then
|
|
break
|
|
fi
|
|
sleep 1
|
|
done
|
|
test "$readback" = "t|t|t"
|
|
if ! wait "$ws_pid"; then
|
|
cat "$ws_log"
|
|
exit 1
|
|
fi
|
|
grep -q '^mqtt-packet-fanout-ok$' "$ws_log"
|
|
- name: Validate monitoring configuration
|
|
run: |
|
|
docker run --rm \
|
|
-v "$PWD/logging/prometheus.yml:/etc/prometheus/prometheus.yml:ro" \
|
|
-v "$PWD/logging/rules:/etc/prometheus/rules:ro" \
|
|
--entrypoint /bin/promtool \
|
|
prom/prometheus@sha256:63805ebb8d2b3920190daf1cb14a60871b16fd38bed42b857a3182bc621f4996 \
|
|
check config /etc/prometheus/prometheus.yml
|
|
docker run --rm \
|
|
-v "$PWD/logging/rules:/rules:ro" \
|
|
--entrypoint /bin/promtool \
|
|
prom/prometheus@sha256:63805ebb8d2b3920190daf1cb14a60871b16fd38bed42b857a3182bc621f4996 \
|
|
test rules /rules/meshcore.test.yml
|
|
docker run --rm \
|
|
-v "$PWD/logging:/etc/meshcore:ro" \
|
|
--entrypoint /bin/amtool \
|
|
prom/alertmanager@sha256:27c475db5fb156cab31d5c18a4251ac7ed567746a2483ff264516437a39b15ba \
|
|
check-config /etc/meshcore/alertmanager.yml
|
|
docker run --rm \
|
|
-v "$PWD/logging/alloy/config.alloy:/etc/alloy/config.alloy:ro" \
|
|
grafana/alloy@sha256:491b0578c04983fd54fe99b587b6fab4404dc46d0dc16677bd6b00cc1140b308 \
|
|
validate /etc/alloy/config.alloy
|
|
- name: Test compressed-chunk maintenance safety gates
|
|
run: scripts/test-vacuum-maintenance.sh
|
|
- name: Test immutable release rollback and schema compatibility gates
|
|
run: scripts/test-replace-container.sh
|
|
|
|
secrets:
|
|
name: Secret scan
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
with:
|
|
fetch-depth: 0
|
|
- name: Scan repository with gitleaks
|
|
uses: gitleaks/gitleaks-action@dcedce43c6f43de0b836d1fe38946645c9c638dc # v2
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|