fix: unblock fresh-DB migrations + retire removed CI components + restore frontend CI (#59)

* fix: unblock fresh database migrations (#44) * fix: remove retired CI components (#48) * fix: restore frontend CI and RF control (#49) --------- Co-authored-by: gadgethd <111318106+gadgethd@users.noreply.github.com>
2026-09-01 17:08:17 +00:00 · 2026-08-11 03:38:57 +01:00
parent 1d6b9ae1ff
commit 2fa099ff8b
8 changed files with 116 additions and 118 deletions
@@ -28,13 +28,6 @@ updates:
    groups:
      rf-worker-dependencies:
        patterns: ['*']
-  - package-ecosystem: pip
-    directory: /ml-path-learner
-    schedule:
-      interval: weekly
-    groups:
-      ml-worker-dependencies:
-        patterns: ['*']
  - package-ecosystem: docker
    directory: /
    schedule:
@@ -43,10 +36,6 @@ updates:
    directory: /viewshed-worker
    schedule:
      interval: weekly
-  - package-ecosystem: docker
-    directory: /ml-path-learner
-    schedule:
-      interval: weekly
  - package-ecosystem: github-actions
    directory: /
    schedule:
@@ -4,8 +4,6 @@ on:
  push:
    branches: ['**']
  pull_request:
-  schedule:
-    - cron: '17 2 * * *'

 permissions:
  contents: read
@@ -87,18 +85,18 @@ jobs:
  workers-and-compose:
    name: Workers and Compose
    runs-on: ubuntu-latest
-    services:
-      postgres:
-        image: postgres@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777
-        env:
-          POSTGRES_PASSWORD: ml-test-password
-        ports:
-          - 55432:5432
-        options: >-
-          --health-cmd "pg_isready -U postgres"
-          --health-interval 5s
-          --health-timeout 3s
-          --health-retries 10
+    env:
+      POSTGRES_PASSWORD: ci-postgres-password
+      REDIS_PASSWORD: ci-redis-password
+      JWT_SECRET: ci-test-only-ci-test-only-ci-test-only-ci-test-only
+      OPERATOR_SITE_TOKEN: ci-operator-site-token-0123456789abcdef0123456789abcdef
+      MQTT_USERNAME: backend
+      MQTT_PASSWORD: ci-mqtt-password
+      ANUBIS_ED25519_PRIVATE_KEY_HEX: '0000000000000000000000000000000000000000000000000000000000000001'
+      HEALTHCHECK_SOURCE_REF: 952eec8e0c55d0749f9abef211c266f992c0bfc7
+      GRAFANA_ADMIN_PASSWORD: ci-grafana-password
+      SOURCE_REVISION: ${{ github.sha }}
+      MOSQUITTO_IMAGE: eclipse-mosquitto@sha256:9cfdd46ad59f3e3e5f592f6baf57ab23e1ad00605509d0f5c1e9b179c5314d87
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
@@ -117,11 +115,9 @@ jobs:
          npm run typecheck
          npm test
      - name: Compile Python workers
-        run: python -m py_compile viewshed-worker/worker.py viewshed-worker/link_queue_v3.py viewshed-worker/viewshed_queue_v2.py viewshed-worker/backfill_profiles.py viewshed-worker/rf/*.py ml-path-learner/worker.py
+        run: python -m py_compile viewshed-worker/worker.py viewshed-worker/link_queue_v3.py viewshed-worker/viewshed_queue_v2.py viewshed-worker/backfill_profiles.py viewshed-worker/rf/*.py
      - name: Build worker images
-        run: |
-          docker build -f viewshed-worker/Dockerfile -t meshcore-viewshed-worker:test viewshed-worker
-          docker build -f ml-path-learner/Dockerfile -t meshcore-ml-path-learner:test ml-path-learner
+        run: docker build -f viewshed-worker/Dockerfile -t meshcore-viewshed-worker:test viewshed-worker
      - name: Test retained observed-link worker image
        run: |
          trap 'docker rm -f link-queue-test-redis >/dev/null 2>&1 || true' EXIT
@@ -164,17 +160,9 @@ jobs:
            -w /work \
            golang:1.23-bookworm \
            /work/scripts/benchmark-hopreach.sh
-      - name: Test ML worker in its image
-        run: |
-          docker run --rm --network host --entrypoint python \
-            -e TEST_DATABASE_URL=postgresql://postgres:ml-test-password@127.0.0.1:55432/postgres \
-            meshcore-ml-path-learner:test -m unittest discover -s /app/tests -v
      - name: Bootstrap clean Mosquitto credentials
        env:
-          MQTT_USERNAME: backend
-          MQTT_PASSWORD: ci-mqtt-password
          MOSQUITTO_CONFIG_DIR: /tmp/meshcore-ci-mosquitto
-          MOSQUITTO_IMAGE: eclipse-mosquitto@sha256:9cfdd46ad59f3e3e5f592f6baf57ab23e1ad00605509d0f5c1e9b179c5314d87
        run: |
          install -d -m 700 "$MOSQUITTO_CONFIG_DIR"
          cp mosquitto/mosquitto.conf "$MOSQUITTO_CONFIG_DIR/mosquitto.conf"
@@ -192,15 +180,6 @@ jobs:
            test "$mode" = 640
          done
      - name: Validate Compose configuration
-        env:
-          POSTGRES_PASSWORD: ci-postgres-password
-          REDIS_PASSWORD: ci-redis-password
-          JWT_SECRET: ci-test-only-ci-test-only-ci-test-only-ci-test-only
-          OPERATOR_SITE_TOKEN: ci-operator-site-token-0123456789abcdef0123456789abcdef
-          MQTT_PASSWORD: ci-mqtt-password
-          ANUBIS_ED25519_PRIVATE_KEY_HEX: '0000000000000000000000000000000000000000000000000000000000000001'
-          HEALTHCHECK_SOURCE_REF: 952eec8e0c55d0749f9abef211c266f992c0bfc7
-          GRAFANA_ADMIN_PASSWORD: ci-grafana-password
        run: |
          docker compose config --quiet
          docker compose config --format json \
@@ -209,19 +188,24 @@ jobs:
                | index("max_worker_processes=24") != null)
              and (.networks.default.ipam.config[0].ip_range
                == "172.30.0.128/25")' >/dev/null
+      - name: Validate tracked build and Compose inventory
+        run: |
+          while IFS= read -r context; do
+            [[ "$context" == *'${{'* ]] && continue
+            test -d "$context"
+          done < <(sed -n 's/^[[:space:]]*context: //p' .github/workflows/release.yml)
+          while IFS= read -r dockerfile; do
+            [[ "$dockerfile" == *'${{'* ]] && continue
+            test -f "$dockerfile"
+          done < <(sed -n 's/^[[:space:]]*file: //p' .github/workflows/release.yml)
+          compose_services="$(docker compose config --services)"
+          for service in \
+            db-migrate backend app-ukmesh website-ukmesh website-dev mesh-health-check \
+            mosquitto-reloader link-worker link-backfill-worker hopreach \
+            timescaledb mosquitto; do
+            grep -Fxq "$service" <<<"$compose_services"
+          done
      - name: Build every application image and smoke an empty-volume stack
-        env:
-          POSTGRES_PASSWORD: ci-postgres-password
-          REDIS_PASSWORD: ci-redis-password
-          JWT_SECRET: ci-test-only-ci-test-only-ci-test-only-ci-test-only
-          OPERATOR_SITE_TOKEN: ci-operator-site-token-0123456789abcdef0123456789abcdef
-          MQTT_USERNAME: backend
-          MQTT_PASSWORD: ci-mqtt-password
-          ANUBIS_ED25519_PRIVATE_KEY_HEX: '0000000000000000000000000000000000000000000000000000000000000001'
-          HEALTHCHECK_SOURCE_REF: 952eec8e0c55d0749f9abef211c266f992c0bfc7
-          GRAFANA_ADMIN_PASSWORD: ci-grafana-password
-          SOURCE_REVISION: ${{ github.sha }}
-          MOSQUITTO_IMAGE: eclipse-mosquitto@sha256:9cfdd46ad59f3e3e5f592f6baf57ab23e1ad00605509d0f5c1e9b179c5314d87
        run: |
          project="meshcore-ci-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
          publisher_user="ci-e2e-publisher"
@@ -415,52 +399,3 @@ jobs:
        uses: gitleaks/gitleaks-action@dcedce43c6f43de0b836d1fe38946645c9c638dc # v2
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-  nightly-load:
-    name: Nightly realtime load
-    if: github.event_name == 'schedule'
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
-      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
-        with:
-          node-version: 20
-          cache: npm
-          cache-dependency-path: backend/package-lock.json
-      - name: Start realtime stack
-        env:
-          POSTGRES_PASSWORD: nightly-postgres-password
-          REDIS_PASSWORD: nightly-redis-password
-          JWT_SECRET: nightly-jwt-secret
-          OPERATOR_SITE_TOKEN: nightly-operator-token-0123456789abcdef0123456789abcdef
-          MQTT_PASSWORD: nightly-mqtt-password
-          GRAFANA_ADMIN_PASSWORD: nightly-grafana-password
-          API_RATE_LIMIT_MAX: 1000000
-          ANUBIS_ED25519_PRIVATE_KEY_HEX: '0000000000000000000000000000000000000000000000000000000000000001'
-          HEALTHCHECK_SOURCE_REF: 952eec8e0c55d0749f9abef211c266f992c0bfc7
-        run: |
-          MQTT_USERNAME=backend MQTT_PASSWORD=nightly-mqtt-password scripts/bootstrap-mosquitto.sh
-          docker compose up -d --build timescaledb redis mosquitto db-migrate backend
-      - name: Wait for readiness
-        run: |
-          for _ in $(seq 1 60); do
-            if curl --fail --silent http://127.0.0.1:3000/readyz >/dev/null; then
-              exit 0
-            fi
-            sleep 2
-          done
-          docker compose logs backend
-          exit 1
-      - name: Run realtime load budget
-        working-directory: backend
-        run: |
-          npm ci
-          npm run load:realtime -- --duration 30 --concurrency 20 --slow-ws-clients 10 --max-p95-ms 1500
-      - name: Capture logs
-        if: always()
-        run: docker compose logs --no-color backend > nightly-backend.log
-      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
-        if: always()
-        with:
-          name: nightly-load-logs
-          path: nightly-backend.log
@@ -76,11 +76,6 @@ jobs:
            file: third_party/hopreach/Dockerfile
            source_revision: 0230702be70a2729c5acc5640401f56ab9d65fd4
            build_args: VERSION=v0.1.32-ukmesh.3
-          - component: ml-path-learner
-            image: meshcore-ml-path-learner
-            context: ml-path-learner
-            file: ml-path-learner/Dockerfile
-            build_args: ''
          - component: mosquitto-reloader
            image: meshcore-mosquitto-reloader
            context: .
@@ -11,6 +11,8 @@ const NON_TRANSACTIONAL_DIRECTIVE = '-- meshcore:migration-mode non-transactiona
 export const NONEMPTY_PRIVATE_PREFIX_SUPERSESSION_APPROVAL =
  'supersede-016-and-017-with-authoritative-privacy-and-026';
 const EMPTY_DATABASE_SUPERSESSIONS = new Map<string, string>([
+  ['011_invalidate_pre_visibility_path_cache.sql', '044_health_current_remove_path_history.sql'],
+  ['015_public_visibility_generation.sql', '044_health_current_remove_path_history.sql'],
  ['016_private_prefixes.sql', '026_private_visibility_schema.sql'],
 ]);

@@ -18,6 +18,71 @@ import {
 const { Pool } = pg;
 const databaseUrl = process.env['TEST_INGEST_DATABASE_URL'];

+test('base schema and every migration apply to a brand-new database', {
+  skip: databaseUrl ? false : 'TEST_INGEST_DATABASE_URL is not configured',
+}, async (t) => {
+  const adminPool = new Pool({
+    connectionString: databaseUrl,
+    application_name: 'meshcore-fresh-migrations-admin-test',
+    max: 1,
+  });
+  const databaseName = `meshcore_fresh_${process.pid}_${Date.now()}`;
+  const databaseIdentifier = `"${databaseName}"`;
+  const freshUrl = new URL(databaseUrl as string);
+  freshUrl.pathname = `/${databaseName}`;
+  let freshPool: pg.Pool | null = null;
+
+  t.after(async () => {
+    await freshPool?.end();
+    await adminPool.query(`DROP DATABASE IF EXISTS ${databaseIdentifier} WITH (FORCE)`);
+    await adminPool.end();
+  });
+
+  await adminPool.query(`CREATE DATABASE ${databaseIdentifier}`);
+  freshPool = new Pool({
+    connectionString: freshUrl.toString(),
+    application_name: 'meshcore-fresh-migrations-test',
+    max: 1,
+  });
+  const baseSql = fs.readFileSync(new URL('./schema/base.sql', import.meta.url), 'utf8');
+  await freshPool.query(baseSql);
+
+  const executed = await runMigrations(freshPool);
+  assert.ok(executed.includes(
+    '011_invalidate_pre_visibility_path_cache.sql -> 044_health_current_remove_path_history.sql',
+  ));
+  assert.ok(executed.includes(
+    '015_public_visibility_generation.sql -> 044_health_current_remove_path_history.sql',
+  ));
+  assert.deepEqual(await runMigrations(freshPool), []);
+
+  const compatibility = await freshPool.query<{
+    migration_name: string;
+    disposition: string;
+    replacement_name: string | null;
+  }>(`
+    SELECT migration_name, disposition, replacement_name
+    FROM schema_migration_compatibility
+    WHERE migration_name IN (
+      '011_invalidate_pre_visibility_path_cache.sql',
+      '015_public_visibility_generation.sql'
+    )
+    ORDER BY migration_name
+  `);
+  assert.deepEqual(compatibility.rows, [
+    {
+      migration_name: '011_invalidate_pre_visibility_path_cache.sql',
+      disposition: 'superseded-empty',
+      replacement_name: '044_health_current_remove_path_history.sql',
+    },
+    {
+      migration_name: '015_public_visibility_generation.sql',
+      disposition: 'superseded-empty',
+      replacement_name: '044_health_current_remove_path_history.sql',
+    },
+  ]);
+});
+
 test('packet batch atomically coalesces observer, sighting, and stats writes', {
  skip: databaseUrl ? false : 'TEST_INGEST_DATABASE_URL is not configured',
 }, async (t) => {
@@ -31,12 +96,17 @@ test('packet batch atomically coalesces observer, sighting, and stats writes', {
  const baseSql = fs.readFileSync(new URL('./schema/base.sql', import.meta.url), 'utf8');
  await pool.query(baseSql);
  await pool.query(`
+    CREATE TABLE path_history_cache (
+      packet_hash TEXT PRIMARY KEY
+    );
    CREATE TABLE schema_migrations (
      name TEXT PRIMARY KEY,
      applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
    );
    INSERT INTO schema_migrations (name)
-    VALUES ('016_stale_mqtt_observer_cleanup.sql')
+    VALUES ('011_invalidate_pre_visibility_path_cache.sql'),
+           ('015_public_visibility_generation.sql'),
+           ('016_stale_mqtt_observer_cleanup.sql')
  `);
  const compatibilityNode = 'migration-compatibility-fixture';
  await pool.query(
@@ -67,6 +137,10 @@ test('packet batch atomically coalesces observer, sighting, and stats writes', {
  assert.ok(migrationRuns.flat().some(
    (name) => name.startsWith('016_private_prefixes.sql -> 026_private_visibility_schema.sql'),
  ));
+  const retiredPathCache = await pool.query<{ exists: string | null }>(
+    "SELECT to_regclass('public.path_history_cache')::text AS exists",
+  );
+  assert.equal(retiredPathCache.rows[0]?.exists, null);

  // Simulate a process loss after the idempotent CREATE INDEX CONCURRENTLY
  // succeeded but before its non-transactional ledger insert committed. The
@@ -13,7 +13,7 @@
  position: absolute;
  left: 14px;
  bottom: 14px;
-  z-index: 780;
+  z-index: 810;
  width: min(360px, calc(100% - 28px));
  color: var(--text-secondary);
  font: 11px/1.35 var(--font-mono);
@@ -56,7 +56,8 @@ test('session polling does not reset the repeater owner content', async ({ page
  });

  await page.goto('/login');
-  await expect(page.getByText('Alpha Repeater', { exact: true })).toBeVisible();
+  const identities = page.getByLabel('Owned repeater identities');
+  await expect(identities.getByText('Alpha Repeater', { exact: true })).toBeVisible();
  await expect(page.locator('.owner-section-tabs')).toHaveCount(0);
  await expect(page.locator('.owner-settings')).toHaveCount(0);

@@ -70,7 +71,7 @@ test('session polling does not reset the repeater owner content', async ({ page
  await page.clock.fastForward(15_001);
  await sessionRefresh;
  await page.clock.fastForward(100);
-  await expect(page.getByText('Alpha Repeater', { exact: true })).toBeVisible();
+  await expect(identities.getByText('Alpha Repeater', { exact: true })).toBeVisible();
  await expect(page.getByText('Unnamed', { exact: true })).toHaveCount(0);
  expect(liveRequests).toBe(initialLiveRequests + 1);
 });
@@ -49,7 +49,7 @@ test('phone layouts give primary content the full available width', async ({ pag
  await expect(page.locator('.uk-feed-right')).toBeHidden();
 });

-test('tablet topology uses the full-width graph workspace', async ({ page }) => {
+test('tablet topology uses the full-width map workspace', async ({ page }) => {
  await page.setViewportSize({ width: 768, height: 667 });
  await page.route('**/api/topology**', (route) => route.fulfill({
    json: {
@@ -66,7 +66,9 @@ test('tablet topology uses the full-width graph workspace', async ({ page }) =>
  await page.goto('/topology');
  const workspace = page.locator('.topology-page__workspace');
  expect(await workspace.evaluate((element) => getComputedStyle(element).gridTemplateColumns.split(' ').length)).toBe(1);
-  expect((await page.locator('.topology-page__graph').boundingBox())?.width ?? 0).toBeGreaterThan(700);
+  const map = page.getByLabel('Geographic repeater topology map');
+  await expect(map).toBeVisible();
+  expect((await map.boundingBox())?.width ?? 0).toBeGreaterThan(700);
 });

 test('public site exposes its primary journeys', async ({ page }) => {