From 2fa099ff8be4c0f8f22dbc353bea998a2b010192 Mon Sep 17 00:00:00 2001 From: hermes-gadget Date: Tue, 11 Aug 2026 03:38:57 +0100 Subject: [PATCH] fix: unblock fresh-DB migrations + retire removed CI components + restore frontend CI (#59) * fix: unblock fresh database migrations (#44) * fix: remove retired CI components (#48) * fix: restore frontend CI and RF control (#49) --------- Co-authored-by: gadgethd <111318106+gadgethd@users.noreply.github.com> --- .github/dependabot.yml | 11 -- .github/workflows/ci.yml | 127 +++++------------- .github/workflows/release.yml | 5 - backend/src/db/migrations.ts | 2 + .../src/db/packetBatch.integration.test.ts | 76 ++++++++++- frontend/src/styles/map-app.css | 2 +- frontend/test/e2e/owner.spec.ts | 5 +- frontend/test/e2e/public.spec.ts | 6 +- 8 files changed, 116 insertions(+), 118 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index d89db42..0683d3e 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -28,13 +28,6 @@ updates: groups: rf-worker-dependencies: patterns: ['*'] - - package-ecosystem: pip - directory: /ml-path-learner - schedule: - interval: weekly - groups: - ml-worker-dependencies: - patterns: ['*'] - package-ecosystem: docker directory: / schedule: @@ -43,10 +36,6 @@ updates: directory: /viewshed-worker schedule: interval: weekly - - package-ecosystem: docker - directory: /ml-path-learner - schedule: - interval: weekly - package-ecosystem: github-actions directory: / schedule: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d6b7fe3..775c114 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,8 +4,6 @@ on: push: branches: ['**'] pull_request: - schedule: - - cron: '17 2 * * *' permissions: contents: read @@ -87,18 +85,18 @@ jobs: workers-and-compose: name: Workers and Compose runs-on: ubuntu-latest - services: - postgres: - image: postgres@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777 - env: - POSTGRES_PASSWORD: ml-test-password - ports: - - 55432:5432 - options: >- - --health-cmd "pg_isready -U postgres" - --health-interval 5s - --health-timeout 3s - --health-retries 10 + env: + POSTGRES_PASSWORD: ci-postgres-password + REDIS_PASSWORD: ci-redis-password + JWT_SECRET: ci-test-only-ci-test-only-ci-test-only-ci-test-only + OPERATOR_SITE_TOKEN: ci-operator-site-token-0123456789abcdef0123456789abcdef + MQTT_USERNAME: backend + MQTT_PASSWORD: ci-mqtt-password + ANUBIS_ED25519_PRIVATE_KEY_HEX: '0000000000000000000000000000000000000000000000000000000000000001' + HEALTHCHECK_SOURCE_REF: 952eec8e0c55d0749f9abef211c266f992c0bfc7 + GRAFANA_ADMIN_PASSWORD: ci-grafana-password + SOURCE_REVISION: ${{ github.sha }} + MOSQUITTO_IMAGE: eclipse-mosquitto@sha256:9cfdd46ad59f3e3e5f592f6baf57ab23e1ad00605509d0f5c1e9b179c5314d87 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 @@ -117,11 +115,9 @@ jobs: npm run typecheck npm test - name: Compile Python workers - run: python -m py_compile viewshed-worker/worker.py viewshed-worker/link_queue_v3.py viewshed-worker/viewshed_queue_v2.py viewshed-worker/backfill_profiles.py viewshed-worker/rf/*.py ml-path-learner/worker.py + run: python -m py_compile viewshed-worker/worker.py viewshed-worker/link_queue_v3.py viewshed-worker/viewshed_queue_v2.py viewshed-worker/backfill_profiles.py viewshed-worker/rf/*.py - name: Build worker images - run: | - docker build -f viewshed-worker/Dockerfile -t meshcore-viewshed-worker:test viewshed-worker - docker build -f ml-path-learner/Dockerfile -t meshcore-ml-path-learner:test ml-path-learner + run: docker build -f viewshed-worker/Dockerfile -t meshcore-viewshed-worker:test viewshed-worker - name: Test retained observed-link worker image run: | trap 'docker rm -f link-queue-test-redis >/dev/null 2>&1 || true' EXIT @@ -164,17 +160,9 @@ jobs: -w /work \ golang:1.23-bookworm \ /work/scripts/benchmark-hopreach.sh - - name: Test ML worker in its image - run: | - docker run --rm --network host --entrypoint python \ - -e TEST_DATABASE_URL=postgresql://postgres:ml-test-password@127.0.0.1:55432/postgres \ - meshcore-ml-path-learner:test -m unittest discover -s /app/tests -v - name: Bootstrap clean Mosquitto credentials env: - MQTT_USERNAME: backend - MQTT_PASSWORD: ci-mqtt-password MOSQUITTO_CONFIG_DIR: /tmp/meshcore-ci-mosquitto - MOSQUITTO_IMAGE: eclipse-mosquitto@sha256:9cfdd46ad59f3e3e5f592f6baf57ab23e1ad00605509d0f5c1e9b179c5314d87 run: | install -d -m 700 "$MOSQUITTO_CONFIG_DIR" cp mosquitto/mosquitto.conf "$MOSQUITTO_CONFIG_DIR/mosquitto.conf" @@ -192,15 +180,6 @@ jobs: test "$mode" = 640 done - name: Validate Compose configuration - env: - POSTGRES_PASSWORD: ci-postgres-password - REDIS_PASSWORD: ci-redis-password - JWT_SECRET: ci-test-only-ci-test-only-ci-test-only-ci-test-only - OPERATOR_SITE_TOKEN: ci-operator-site-token-0123456789abcdef0123456789abcdef - MQTT_PASSWORD: ci-mqtt-password - ANUBIS_ED25519_PRIVATE_KEY_HEX: '0000000000000000000000000000000000000000000000000000000000000001' - HEALTHCHECK_SOURCE_REF: 952eec8e0c55d0749f9abef211c266f992c0bfc7 - GRAFANA_ADMIN_PASSWORD: ci-grafana-password run: | docker compose config --quiet docker compose config --format json \ @@ -209,19 +188,24 @@ jobs: | index("max_worker_processes=24") != null) and (.networks.default.ipam.config[0].ip_range == "172.30.0.128/25")' >/dev/null + - name: Validate tracked build and Compose inventory + run: | + while IFS= read -r context; do + [[ "$context" == *'${{'* ]] && continue + test -d "$context" + done < <(sed -n 's/^[[:space:]]*context: //p' .github/workflows/release.yml) + while IFS= read -r dockerfile; do + [[ "$dockerfile" == *'${{'* ]] && continue + test -f "$dockerfile" + done < <(sed -n 's/^[[:space:]]*file: //p' .github/workflows/release.yml) + compose_services="$(docker compose config --services)" + for service in \ + db-migrate backend app-ukmesh website-ukmesh website-dev mesh-health-check \ + mosquitto-reloader link-worker link-backfill-worker hopreach \ + timescaledb mosquitto; do + grep -Fxq "$service" <<<"$compose_services" + done - name: Build every application image and smoke an empty-volume stack - env: - POSTGRES_PASSWORD: ci-postgres-password - REDIS_PASSWORD: ci-redis-password - JWT_SECRET: ci-test-only-ci-test-only-ci-test-only-ci-test-only - OPERATOR_SITE_TOKEN: ci-operator-site-token-0123456789abcdef0123456789abcdef - MQTT_USERNAME: backend - MQTT_PASSWORD: ci-mqtt-password - ANUBIS_ED25519_PRIVATE_KEY_HEX: '0000000000000000000000000000000000000000000000000000000000000001' - HEALTHCHECK_SOURCE_REF: 952eec8e0c55d0749f9abef211c266f992c0bfc7 - GRAFANA_ADMIN_PASSWORD: ci-grafana-password - SOURCE_REVISION: ${{ github.sha }} - MOSQUITTO_IMAGE: eclipse-mosquitto@sha256:9cfdd46ad59f3e3e5f592f6baf57ab23e1ad00605509d0f5c1e9b179c5314d87 run: | project="meshcore-ci-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" publisher_user="ci-e2e-publisher" @@ -415,52 +399,3 @@ jobs: uses: gitleaks/gitleaks-action@dcedce43c6f43de0b836d1fe38946645c9c638dc # v2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - nightly-load: - name: Nightly realtime load - if: github.event_name == 'schedule' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 - with: - node-version: 20 - cache: npm - cache-dependency-path: backend/package-lock.json - - name: Start realtime stack - env: - POSTGRES_PASSWORD: nightly-postgres-password - REDIS_PASSWORD: nightly-redis-password - JWT_SECRET: nightly-jwt-secret - OPERATOR_SITE_TOKEN: nightly-operator-token-0123456789abcdef0123456789abcdef - MQTT_PASSWORD: nightly-mqtt-password - GRAFANA_ADMIN_PASSWORD: nightly-grafana-password - API_RATE_LIMIT_MAX: 1000000 - ANUBIS_ED25519_PRIVATE_KEY_HEX: '0000000000000000000000000000000000000000000000000000000000000001' - HEALTHCHECK_SOURCE_REF: 952eec8e0c55d0749f9abef211c266f992c0bfc7 - run: | - MQTT_USERNAME=backend MQTT_PASSWORD=nightly-mqtt-password scripts/bootstrap-mosquitto.sh - docker compose up -d --build timescaledb redis mosquitto db-migrate backend - - name: Wait for readiness - run: | - for _ in $(seq 1 60); do - if curl --fail --silent http://127.0.0.1:3000/readyz >/dev/null; then - exit 0 - fi - sleep 2 - done - docker compose logs backend - exit 1 - - name: Run realtime load budget - working-directory: backend - run: | - npm ci - npm run load:realtime -- --duration 30 --concurrency 20 --slow-ws-clients 10 --max-p95-ms 1500 - - name: Capture logs - if: always() - run: docker compose logs --no-color backend > nightly-backend.log - - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - if: always() - with: - name: nightly-load-logs - path: nightly-backend.log diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 04e8df6..0ef3bfa 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -76,11 +76,6 @@ jobs: file: third_party/hopreach/Dockerfile source_revision: 0230702be70a2729c5acc5640401f56ab9d65fd4 build_args: VERSION=v0.1.32-ukmesh.3 - - component: ml-path-learner - image: meshcore-ml-path-learner - context: ml-path-learner - file: ml-path-learner/Dockerfile - build_args: '' - component: mosquitto-reloader image: meshcore-mosquitto-reloader context: . diff --git a/backend/src/db/migrations.ts b/backend/src/db/migrations.ts index 0831fef..7a16f72 100644 --- a/backend/src/db/migrations.ts +++ b/backend/src/db/migrations.ts @@ -11,6 +11,8 @@ const NON_TRANSACTIONAL_DIRECTIVE = '-- meshcore:migration-mode non-transactiona export const NONEMPTY_PRIVATE_PREFIX_SUPERSESSION_APPROVAL = 'supersede-016-and-017-with-authoritative-privacy-and-026'; const EMPTY_DATABASE_SUPERSESSIONS = new Map([ + ['011_invalidate_pre_visibility_path_cache.sql', '044_health_current_remove_path_history.sql'], + ['015_public_visibility_generation.sql', '044_health_current_remove_path_history.sql'], ['016_private_prefixes.sql', '026_private_visibility_schema.sql'], ]); diff --git a/backend/src/db/packetBatch.integration.test.ts b/backend/src/db/packetBatch.integration.test.ts index 7b01f01..86c29cc 100644 --- a/backend/src/db/packetBatch.integration.test.ts +++ b/backend/src/db/packetBatch.integration.test.ts @@ -18,6 +18,71 @@ import { const { Pool } = pg; const databaseUrl = process.env['TEST_INGEST_DATABASE_URL']; +test('base schema and every migration apply to a brand-new database', { + skip: databaseUrl ? false : 'TEST_INGEST_DATABASE_URL is not configured', +}, async (t) => { + const adminPool = new Pool({ + connectionString: databaseUrl, + application_name: 'meshcore-fresh-migrations-admin-test', + max: 1, + }); + const databaseName = `meshcore_fresh_${process.pid}_${Date.now()}`; + const databaseIdentifier = `"${databaseName}"`; + const freshUrl = new URL(databaseUrl as string); + freshUrl.pathname = `/${databaseName}`; + let freshPool: pg.Pool | null = null; + + t.after(async () => { + await freshPool?.end(); + await adminPool.query(`DROP DATABASE IF EXISTS ${databaseIdentifier} WITH (FORCE)`); + await adminPool.end(); + }); + + await adminPool.query(`CREATE DATABASE ${databaseIdentifier}`); + freshPool = new Pool({ + connectionString: freshUrl.toString(), + application_name: 'meshcore-fresh-migrations-test', + max: 1, + }); + const baseSql = fs.readFileSync(new URL('./schema/base.sql', import.meta.url), 'utf8'); + await freshPool.query(baseSql); + + const executed = await runMigrations(freshPool); + assert.ok(executed.includes( + '011_invalidate_pre_visibility_path_cache.sql -> 044_health_current_remove_path_history.sql', + )); + assert.ok(executed.includes( + '015_public_visibility_generation.sql -> 044_health_current_remove_path_history.sql', + )); + assert.deepEqual(await runMigrations(freshPool), []); + + const compatibility = await freshPool.query<{ + migration_name: string; + disposition: string; + replacement_name: string | null; + }>(` + SELECT migration_name, disposition, replacement_name + FROM schema_migration_compatibility + WHERE migration_name IN ( + '011_invalidate_pre_visibility_path_cache.sql', + '015_public_visibility_generation.sql' + ) + ORDER BY migration_name + `); + assert.deepEqual(compatibility.rows, [ + { + migration_name: '011_invalidate_pre_visibility_path_cache.sql', + disposition: 'superseded-empty', + replacement_name: '044_health_current_remove_path_history.sql', + }, + { + migration_name: '015_public_visibility_generation.sql', + disposition: 'superseded-empty', + replacement_name: '044_health_current_remove_path_history.sql', + }, + ]); +}); + test('packet batch atomically coalesces observer, sighting, and stats writes', { skip: databaseUrl ? false : 'TEST_INGEST_DATABASE_URL is not configured', }, async (t) => { @@ -31,12 +96,17 @@ test('packet batch atomically coalesces observer, sighting, and stats writes', { const baseSql = fs.readFileSync(new URL('./schema/base.sql', import.meta.url), 'utf8'); await pool.query(baseSql); await pool.query(` + CREATE TABLE path_history_cache ( + packet_hash TEXT PRIMARY KEY + ); CREATE TABLE schema_migrations ( name TEXT PRIMARY KEY, applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); INSERT INTO schema_migrations (name) - VALUES ('016_stale_mqtt_observer_cleanup.sql') + VALUES ('011_invalidate_pre_visibility_path_cache.sql'), + ('015_public_visibility_generation.sql'), + ('016_stale_mqtt_observer_cleanup.sql') `); const compatibilityNode = 'migration-compatibility-fixture'; await pool.query( @@ -67,6 +137,10 @@ test('packet batch atomically coalesces observer, sighting, and stats writes', { assert.ok(migrationRuns.flat().some( (name) => name.startsWith('016_private_prefixes.sql -> 026_private_visibility_schema.sql'), )); + const retiredPathCache = await pool.query<{ exists: string | null }>( + "SELECT to_regclass('public.path_history_cache')::text AS exists", + ); + assert.equal(retiredPathCache.rows[0]?.exists, null); // Simulate a process loss after the idempotent CREATE INDEX CONCURRENTLY // succeeded but before its non-transactional ledger insert committed. The diff --git a/frontend/src/styles/map-app.css b/frontend/src/styles/map-app.css index a55f876..9c41333 100644 --- a/frontend/src/styles/map-app.css +++ b/frontend/src/styles/map-app.css @@ -13,7 +13,7 @@ position: absolute; left: 14px; bottom: 14px; - z-index: 780; + z-index: 810; width: min(360px, calc(100% - 28px)); color: var(--text-secondary); font: 11px/1.35 var(--font-mono); diff --git a/frontend/test/e2e/owner.spec.ts b/frontend/test/e2e/owner.spec.ts index a7efc03..6b4182d 100644 --- a/frontend/test/e2e/owner.spec.ts +++ b/frontend/test/e2e/owner.spec.ts @@ -56,7 +56,8 @@ test('session polling does not reset the repeater owner content', async ({ page }); await page.goto('/login'); - await expect(page.getByText('Alpha Repeater', { exact: true })).toBeVisible(); + const identities = page.getByLabel('Owned repeater identities'); + await expect(identities.getByText('Alpha Repeater', { exact: true })).toBeVisible(); await expect(page.locator('.owner-section-tabs')).toHaveCount(0); await expect(page.locator('.owner-settings')).toHaveCount(0); @@ -70,7 +71,7 @@ test('session polling does not reset the repeater owner content', async ({ page await page.clock.fastForward(15_001); await sessionRefresh; await page.clock.fastForward(100); - await expect(page.getByText('Alpha Repeater', { exact: true })).toBeVisible(); + await expect(identities.getByText('Alpha Repeater', { exact: true })).toBeVisible(); await expect(page.getByText('Unnamed', { exact: true })).toHaveCount(0); expect(liveRequests).toBe(initialLiveRequests + 1); }); diff --git a/frontend/test/e2e/public.spec.ts b/frontend/test/e2e/public.spec.ts index e499ff2..d1dfd54 100644 --- a/frontend/test/e2e/public.spec.ts +++ b/frontend/test/e2e/public.spec.ts @@ -49,7 +49,7 @@ test('phone layouts give primary content the full available width', async ({ pag await expect(page.locator('.uk-feed-right')).toBeHidden(); }); -test('tablet topology uses the full-width graph workspace', async ({ page }) => { +test('tablet topology uses the full-width map workspace', async ({ page }) => { await page.setViewportSize({ width: 768, height: 667 }); await page.route('**/api/topology**', (route) => route.fulfill({ json: { @@ -66,7 +66,9 @@ test('tablet topology uses the full-width graph workspace', async ({ page }) => await page.goto('/topology'); const workspace = page.locator('.topology-page__workspace'); expect(await workspace.evaluate((element) => getComputedStyle(element).gridTemplateColumns.split(' ').length)).toBe(1); - expect((await page.locator('.topology-page__graph').boundingBox())?.width ?? 0).toBeGreaterThan(700); + const map = page.getByLabel('Geographic repeater topology map'); + await expect(map).toBeVisible(); + expect((await map.boundingBox())?.width ?? 0).toBeGreaterThan(700); }); test('public site exposes its primary journeys', async ({ page }) => {