From 5db382e181591b9bf736647eb903c699223cfdc8 Mon Sep 17 00:00:00 2001 From: gadgethd <111318106+gadgethd@users.noreply.github.com> Date: Sun, 2 Aug 2026 03:40:11 +0100 Subject: [PATCH] feat: complete codebase reliability upgrade --- .env.example | 66 +- .github/dependabot.yml | 33 + .github/workflows/ci.yml | 306 +- .github/workflows/frontend-mobile.yml | 4 +- .github/workflows/release.yml | 192 + .trivyignore.yaml | 3 + Dockerfile | 23 +- Dockerfile.app | 13 +- Dockerfile.backend | 17 +- Dockerfile.mesh-health-check | 25 +- Dockerfile.mosquitto-reloader | 9 + Dockerfile.website | 13 +- PLAN-repeater-tree-fix.md | 87 + README.md | 101 +- UI-AUDIT.md | 1022 +++ backend/backfill-transport-codes.mjs | 6 +- backend/package-lock.json | 545 -- backend/package.json | 12 +- backend/src/analysis/boundedRun.test.ts | 23 + backend/src/analysis/boundedRun.ts | 42 +- backend/src/analysis/publicationFence.ts | 57 + .../src/analysis/runState.integration.test.ts | 229 + backend/src/analysis/runState.ts | 489 +- backend/src/api/bootstrap/caches.ts | 6 + backend/src/api/bootstrap/limiters.test.ts | 12 + backend/src/api/bootstrap/limiters.ts | 28 + backend/src/api/caches.ts | 18 - backend/src/api/contracts.test.ts | 41 + backend/src/api/contracts.ts | 204 + backend/src/api/errors.test.ts | 35 + backend/src/api/errors.ts | 103 + backend/src/api/routeRegistry.test.ts | 32 + backend/src/api/routeRegistry.ts | 48 + backend/src/api/routes.ts | 33 +- backend/src/api/routes/activityTimeline.ts | 17 +- backend/src/api/routes/coverage.test.ts | 93 + backend/src/api/routes/coverage.ts | 62 +- .../src/api/routes/coveragePagination.test.ts | 9 +- backend/src/api/routes/coveragePagination.ts | 12 +- backend/src/api/routes/exports.ts | 35 +- backend/src/api/routes/health.ts | 3 +- backend/src/api/routes/misc.ts | 75 +- backend/src/api/routes/nodeStatus.ts | 24 +- backend/src/api/routes/nodes.ts | 412 +- backend/src/api/routes/owner.ts | 213 +- backend/src/api/routes/pathing.ts | 43 +- backend/src/api/routes/plannedCoverage.ts | 212 +- backend/src/api/routes/productFeatures.ts | 116 +- backend/src/api/routes/radio.test.ts | 13 + backend/src/api/routes/radio.ts | 95 +- backend/src/api/routes/rfValidation.ts | 51 +- backend/src/api/routes/spam.ts | 74 +- backend/src/api/routes/stats.ts | 4 + backend/src/api/routes/telemetry.ts | 50 +- backend/src/api/routes/topology.ts | 92 +- backend/src/api/utils/input.test.ts | 38 + backend/src/api/utils/input.ts | 163 + backend/src/api/utils/localOnly.ts | 30 +- backend/src/api/utils/networkFilters.test.ts | 10 +- backend/src/api/utils/networkFilters.ts | 56 +- backend/src/api/utils/observer.ts | 13 +- backend/src/backend-site/login.html | 53 + backend/src/backend-site/routes.test.ts | 126 + backend/src/backend-site/routes.ts | 391 +- backend/src/backend-site/template.html | 244 +- backend/src/backup/receipt.test.ts | 69 + backend/src/backup/receipt.ts | 155 + backend/src/cache/boundedTtlMap.ts | 38 +- backend/src/cache/policyRegistry.test.ts | 54 + backend/src/cache/policyRegistry.ts | 143 + backend/src/db/dataLifecycle.test.ts | 91 + backend/src/db/dataLifecycle.ts | 165 + backend/src/db/index.ts | 257 +- backend/src/db/migrations.test.ts | 47 + backend/src/db/migrations.ts | 362 +- .../023_security_delivery_and_diagnostics.sql | 34 + .../db/migrations/024_analysis_run_leases.sql | 32 + .../migrations/025_coverage_result_status.sql | 74 + .../026_private_visibility_schema.sql | 75 + .../db/migrations/027_ml_learner_leases.sql | 45 + .../src/db/migrations/028_ml_cursor_index.sql | 13 + .../029_stats_hourly_aggregates.sql | 36 + .../db/migrations/030_product_operations.sql | 89 + .../migrations/031_stats_chart_snapshots.sql | 18 + .../032_stats_snapshot_privacy_generation.sql | 38 + backend/src/db/migrations/README.md | 23 + backend/src/db/ownerAuth.ts | 17 +- .../src/db/packetBatch.integration.test.ts | 318 + backend/src/db/packetBatch.ts | 187 +- backend/src/db/schema/base.sql | 163 +- backend/src/features.test.ts | 52 + backend/src/features.ts | 59 + backend/src/health/status.ts | 494 +- backend/src/health/statusIntegrity.test.ts | 22 + backend/src/health/statusResources.test.ts | 79 + backend/src/http/requestScope.ts | 15 + backend/src/http/upstreamJson.test.ts | 90 + backend/src/http/upstreamJson.ts | 149 + backend/src/index.ts | 214 +- backend/src/lifecycle/coordinator.test.ts | 51 + backend/src/lifecycle/coordinator.ts | 100 + backend/src/metrics.test.ts | 56 + backend/src/metrics.ts | 397 + backend/src/metricsServer.ts | 36 + backend/src/mqtt/aclManager.ts | 37 +- backend/src/mqtt/client.ts | 111 +- backend/src/mqtt/connectionMonitor.ts | 49 +- backend/src/mqtt/spamDetector.ts | 32 +- backend/src/nodes/publicMap.test.ts | 94 + backend/src/nodes/publicMap.ts | 170 + .../src/operations/operatorOperations.test.ts | 30 + backend/src/operations/operatorOperations.ts | 944 +++ backend/src/owner/alertRules.test.ts | 46 + backend/src/owner/alertRules.ts | 325 +- backend/src/owner/ownerAccess.ts | 6 + backend/src/owner/ownerAclReconciler.ts | 13 +- backend/src/owner/ownerService.ts | 19 +- backend/src/path-beta/resolveCache.test.ts | 32 + backend/src/path-beta/resolveCache.ts | 73 +- backend/src/path-beta/resolver.ts | 76 +- backend/src/path-beta/workerPool.test.ts | 18 + backend/src/path-beta/workerPool.ts | 58 +- backend/src/path-learning/rebuild.ts | 112 +- backend/src/platform/config/boundedNumber.ts | 21 + backend/src/platform/config/database.test.ts | 50 + backend/src/platform/config/database.ts | 50 +- backend/src/queue/linkQueueV3.test.ts | 72 + backend/src/queue/linkQueueV3.ts | 37 + .../src/queue/publisher.integration.test.ts | 35 + backend/src/queue/publisher.ts | 189 +- backend/src/repositories/networkAnalysis.ts | 147 + backend/src/repositories/nodes.ts | 253 + backend/src/repositories/ownerAlerts.ts | 120 + backend/src/repositories/plannedCoverage.ts | 227 + backend/src/repositories/plannedNodes.test.ts | 24 + backend/src/repositories/plannedNodes.ts | 134 + .../src/repositories/productFeatures.test.ts | 26 + backend/src/repositories/productFeatures.ts | 165 + backend/src/security/operatorSession.test.ts | 18 + backend/src/security/operatorSession.ts | 76 + backend/src/security/outboundWebhook.test.ts | 54 + backend/src/security/outboundWebhook.ts | 176 + backend/src/spam/analyzer.ts | 75 +- backend/src/spam/repository.ts | 52 +- backend/src/stats/chartSnapshot.test.ts | 101 + backend/src/stats/chartSnapshot.ts | 152 + backend/src/stats/statsRepository.test.ts | 108 +- backend/src/stats/statsRepository.ts | 627 +- backend/src/stats/statsService.test.ts | 200 +- backend/src/stats/statsService.ts | 291 +- .../src/tools/backfillStatsRollups.test.ts | 43 + backend/src/tools/backfillStatsRollups.ts | 311 +- .../src/tools/cleanupOrphanedAnalysisRuns.ts | 156 + backend/src/tools/dataLifecycle.ts | 157 + backend/src/tools/generateOpenApi.ts | 159 + backend/src/tools/recomputeSpamMessages.ts | 108 +- backend/src/tools/seedStatsChartSnapshot.ts | 99 + backend/src/workers/alert-receiver.test.ts | 49 + backend/src/workers/alert-receiver.ts | 199 + backend/src/workers/health.ts | 7 + backend/src/workers/link-backfill.ts | 6 + backend/src/workers/link-recompute.ts | 5 + backend/src/workers/path-history.ts | 43 +- backend/src/workers/path-learning.ts | 6 + backend/src/workers/synthetic-monitor.ts | 4 + backend/src/workers/workerMetrics.ts | 9 + backend/src/ws/server.ts | 55 +- backup-receipts/README.md | 12 + backup-receipts/latest.json | 36 + backup-receipts/latest.json.sig | Bin 0 -> 384 bytes backup-receipts/verify.pem | 11 + docker-compose.live.yml | 61 + docker-compose.yml | 577 +- docker/mesh-health-check-entrypoint.sh | 5 + docker/mosquitto-reloader.py | 67 + docs/architecture.md | 96 +- docs/contributing.md | 5 +- docs/db-lifecycle.md | 63 + docs/frontend-map.md | 30 +- docs/link-model.md | 5 + docs/openapi.yaml | 7278 ++++++++++++++++- docs/operations.md | 304 + docs/performance-budgets.md | 113 + docs/rf-coverage-rollout.md | 143 + docs/runbook-backup-restore.md | 111 + docs/runbook-queue-recovery.md | 79 + docs/runbook-release-rollback.md | 109 + docs/runbook-srtm-alerts-observers.md | 86 + docs/security-closure.md | 44 + docs/security-waivers.md | 33 + frontend/package-lock.json | 237 +- frontend/package.json | 20 +- frontend/playwright.config.ts | 2 +- frontend/public/sw-runtime.js | 321 + frontend/public/sw.js | 112 +- frontend/scripts/check-css-structure.mjs | 176 + .../scripts/check-performance-budgets.mjs | 72 + frontend/scripts/soak-production.mjs | 367 + frontend/src/App.tsx | 169 +- .../components/FilterPanel/FilterPanel.tsx | 35 +- frontend/src/components/JsonLd.tsx | 2 +- frontend/src/components/LiveStatsSection.tsx | 76 +- frontend/src/components/Map/DeckGLOverlay.tsx | 7 +- .../components/Map/LinkQualitySparkline.tsx | 82 +- .../components/Map/LiveOverlayController.tsx | 13 +- frontend/src/components/Map/MapLibreMap.tsx | 187 +- frontend/src/components/Map/NodeLegend.tsx | 43 +- .../src/components/Map/NodePopupContent.tsx | 51 +- frontend/src/components/Map/NodeSearch.tsx | 104 +- .../components/Map/geojsonBuilders.test.ts | 49 + .../src/components/Map/geojsonBuilders.ts | 51 +- frontend/src/components/Map/mapConfig.ts | 74 + .../components/Map/mapSourceLayers.test.ts | 81 + .../src/components/Map/mapSourceLayers.ts | 156 +- frontend/src/components/Map/types.ts | 4 + frontend/src/components/SeoHead.tsx | 10 +- .../components/ServiceWorkerUpdatePrompt.tsx | 34 + .../src/components/StatsPanel/StatsPanel.tsx | 9 +- .../src/components/app/DisclaimerModal.tsx | 23 +- .../src/components/app/MobileControls.tsx | 53 +- .../src/components/app/WatchlistPanel.tsx | 41 +- .../components/owner/OwnerPortalSections.tsx | 169 +- .../stats/StatsDecodedPathDialog.tsx | 70 + .../src/components/stats/StatsPrimitives.tsx | 344 + frontend/src/components/ui/Combobox.tsx | 123 + frontend/src/components/ui/Dialog.tsx | 81 + frontend/src/components/ui/Tabs.tsx | 46 + frontend/src/config/features.ts | 12 + frontend/src/config/publicRoutes.test.ts | 20 + frontend/src/config/publicRoutes.ts | 121 + frontend/src/config/runtimeFeatures.test.ts | 59 + frontend/src/config/runtimeFeatures.ts | 161 + frontend/src/config/seo.ts | 68 +- frontend/src/hooks/packetFeed.ts | 35 +- .../src/hooks/useAppMessageHandler.test.ts | 116 + frontend/src/hooks/useAppMessageHandler.ts | 403 +- frontend/src/hooks/useCoverage.ts | 50 +- frontend/src/hooks/useDashboardStats.test.ts | 13 + frontend/src/hooks/useDashboardStats.ts | 22 +- frontend/src/hooks/useDocumentMeta.ts | 7 + frontend/src/hooks/useLinkState.test.ts | 84 + frontend/src/hooks/useLinkState.ts | 99 +- frontend/src/hooks/useNodes.test.ts | 141 +- frontend/src/hooks/useNodes.ts | 179 +- frontend/src/hooks/usePacketDetailData.ts | 194 +- frontend/src/hooks/useReducedMotion.ts | 24 + frontend/src/hooks/useStatsPageData.ts | 104 + frontend/src/hooks/useVisibilityPoll.test.ts | 72 + frontend/src/hooks/useVisibilityPoll.ts | 179 + frontend/src/hooks/useWatchlist.ts | 49 +- frontend/src/hooks/useWebSocket.ts | 28 +- frontend/src/main.tsx | 126 +- frontend/src/pages/NotFoundPage.tsx | 13 + frontend/src/pages/OpenSourcePage.tsx | 2 +- frontend/src/pages/OwnerPortalPage.tsx | 1153 +-- frontend/src/pages/SpamTransparencyPage.tsx | 141 +- frontend/src/pages/StatusPage.tsx | 2 +- frontend/src/pages/TopologyPage.tsx | 11 +- frontend/src/pages/docs-pages.css | 9 +- frontend/src/pages/network-intelligence.css | 15 +- frontend/src/pages/owner-portal.css | 57 +- frontend/src/pages/owner/OwnerMapView.tsx | 209 + .../src/pages/owner/OwnerPortalCharts.tsx | 458 ++ .../src/pages/owner/ownerPortalModel.test.ts | 46 + frontend/src/pages/owner/ownerPortalModel.ts | 286 + frontend/src/pages/path-modal.css | 2 - frontend/src/pages/site-content.css | 386 - frontend/src/pages/site-shell.css | 14 +- frontend/src/pages/spam-page.css | 6 +- frontend/src/pages/statsState.test.ts | 24 + frontend/src/pages/statsState.ts | 28 + frontend/src/pages/topologyModel.test.ts | 16 + frontend/src/pages/topologyModel.ts | 15 + frontend/src/pages/ukmesh/FeedPathViews.tsx | 4 +- .../src/pages/ukmesh/PacketDetailPanel.tsx | 8 +- .../src/pages/ukmesh/UKBestPracticePage.tsx | 4 +- frontend/src/pages/ukmesh/UKFeedPage.tsx | 13 +- .../src/pages/ukmesh/UKRepeaterSearchPage.tsx | 11 +- frontend/src/pages/ukmesh/feed-page.css | 7 - frontend/src/plugins/vite-seo.ts | 63 +- frontend/src/serviceWorkerRuntime.test.ts | 242 + frontend/src/serviceWorkerUpdates.ts | 108 + frontend/src/styles/globals.css | 170 +- frontend/src/styles/map-app.css | 99 +- frontend/src/styles/tokens.css | 12 +- frontend/src/utils/api.test.ts | 46 + frontend/src/utils/api.ts | 69 + .../src/utils/frameSnapshotScheduler.test.ts | 42 + frontend/src/utils/frameSnapshotScheduler.ts | 39 + frontend/src/utils/mapUrlState.ts | 2 + frontend/src/utils/nodeIds.ts | 9 + frontend/src/utils/requestLimiter.test.ts | 60 + frontend/src/utils/requestLimiter.ts | 107 + frontend/src/utils/scopedCache.test.ts | 57 + frontend/src/utils/scopedCache.ts | 222 + frontend/src/utils/terrainSampler.test.ts | 77 + frontend/src/utils/terrainSampler.ts | 388 +- frontend/src/vite-env.d.ts | 3 + frontend/test/e2e/accessibility.spec.ts | 364 + frontend/test/e2e/mobile.spec.ts | 127 +- frontend/test/e2e/owner.spec.ts | 66 +- frontend/test/e2e/public.spec.ts | 40 +- logging/alertmanager.yml | 22 + logging/alloy/config.alloy | 47 + logging/blackbox.yml | 16 + .../grafana/dashboards/backend-workers.json | 1 + logging/grafana/dashboards/db-metrics.json | 2 +- logging/grafana/dashboards/ingest-rate.json | 2 +- .../grafana/dashboards/ml-path-learner.json | 213 - logging/grafana/dashboards/mqtt.json | 1 + logging/grafana/dashboards/overview.json | 1 + .../grafana/dashboards/stats-recompute.json | 2 +- logging/grafana/dashboards/ws-clients.json | 2 +- .../provisioning/dashboards/provider.yaml | 4 +- .../provisioning/datasources/loki.yaml | 2 +- logging/prometheus.yml | 98 +- logging/promtail.yaml | 29 - logging/rules/meshcore.test.yml | 176 + logging/rules/meshcore.yml | 211 + ml-path-learner/.dockerignore | 8 + ml-path-learner/Dockerfile | 17 +- ml-path-learner/requirements.in | 5 + ml-path-learner/requirements.txt | 237 +- ml-path-learner/tests/__init__.py | 1 + ml-path-learner/tests/test_worker.py | 373 + ml-path-learner/worker.py | 669 +- mosquitto/mosquitto.conf | 8 +- nginx.app.conf | 13 +- nginx.website.conf | 28 +- plan.md | 2052 +++++ scripts/backup.sh | 275 + scripts/bootstrap-mosquitto.sh | 107 + scripts/check-trivy-waivers.mjs | 105 + scripts/generate-observer-key.ts | 64 +- scripts/package-lock.json | 681 ++ scripts/package.json | 4 +- scripts/replace-container.sh | 394 +- scripts/restore-drill.sh | 505 ++ scripts/test-alert-receiver.sh | 55 + scripts/test-alloy-cutover.sh | 131 + scripts/test-generate-observer-key.sh | 58 + scripts/test-replace-container.sh | 289 + scripts/test-vacuum-maintenance.sh | 121 + vacuum-compressed-chunks.sh | 262 +- viewshed-worker/.dockerignore | 8 + viewshed-worker/Dockerfile | 13 +- viewshed-worker/backfill_profiles.py | 26 +- viewshed-worker/dry_run_coverage.py | 356 + viewshed-worker/link_queue_v3.py | 309 +- viewshed-worker/queue_admin.py | 77 + viewshed-worker/requirements.in | 11 + viewshed-worker/requirements.txt | 317 +- viewshed-worker/rf/loss.py | 238 +- viewshed-worker/rf/terrain.py | 320 +- viewshed-worker/tests/benchmark_rf.py | 116 + viewshed-worker/tests/benchmark_rf_radial.py | 108 + .../tests/coverage_dry_run_input.sql | 33 + .../tests/fixtures/rf_radial_golden.json | 20 + .../tests/test_dry_run_coverage.py | 120 + viewshed-worker/tests/test_link_queue_v3.py | 136 + viewshed-worker/tests/test_loss.py | 172 + viewshed-worker/tests/test_rf_radial.py | 251 + viewshed-worker/tests/test_terrain.py | 172 + .../tests/test_viewshed_queue_v2.py | 89 + viewshed-worker/tests/test_worker_outcomes.py | 83 + viewshed-worker/viewshed_queue_v2.py | 487 ++ viewshed-worker/worker.py | 1091 ++- viewshed-worker/worker_metrics.py | 59 + 368 files changed, 44682 insertions(+), 5905 deletions(-) create mode 100644 .github/workflows/release.yml create mode 100644 .trivyignore.yaml create mode 100644 Dockerfile.mosquitto-reloader create mode 100644 PLAN-repeater-tree-fix.md create mode 100644 UI-AUDIT.md create mode 100644 backend/src/analysis/publicationFence.ts create mode 100644 backend/src/analysis/runState.integration.test.ts create mode 100644 backend/src/api/bootstrap/limiters.test.ts delete mode 100644 backend/src/api/caches.ts create mode 100644 backend/src/api/contracts.test.ts create mode 100644 backend/src/api/contracts.ts create mode 100644 backend/src/api/errors.test.ts create mode 100644 backend/src/api/errors.ts create mode 100644 backend/src/api/routeRegistry.test.ts create mode 100644 backend/src/api/routeRegistry.ts create mode 100644 backend/src/api/routes/coverage.test.ts create mode 100644 backend/src/api/routes/radio.test.ts create mode 100644 backend/src/api/utils/input.test.ts create mode 100644 backend/src/api/utils/input.ts create mode 100644 backend/src/backend-site/login.html create mode 100644 backend/src/backend-site/routes.test.ts create mode 100644 backend/src/backup/receipt.test.ts create mode 100644 backend/src/backup/receipt.ts create mode 100644 backend/src/cache/policyRegistry.test.ts create mode 100644 backend/src/cache/policyRegistry.ts create mode 100644 backend/src/db/dataLifecycle.test.ts create mode 100644 backend/src/db/dataLifecycle.ts create mode 100644 backend/src/db/migrations.test.ts create mode 100644 backend/src/db/migrations/023_security_delivery_and_diagnostics.sql create mode 100644 backend/src/db/migrations/024_analysis_run_leases.sql create mode 100644 backend/src/db/migrations/025_coverage_result_status.sql create mode 100644 backend/src/db/migrations/026_private_visibility_schema.sql create mode 100644 backend/src/db/migrations/027_ml_learner_leases.sql create mode 100644 backend/src/db/migrations/028_ml_cursor_index.sql create mode 100644 backend/src/db/migrations/029_stats_hourly_aggregates.sql create mode 100644 backend/src/db/migrations/030_product_operations.sql create mode 100644 backend/src/db/migrations/031_stats_chart_snapshots.sql create mode 100644 backend/src/db/migrations/032_stats_snapshot_privacy_generation.sql create mode 100644 backend/src/db/packetBatch.integration.test.ts create mode 100644 backend/src/features.test.ts create mode 100644 backend/src/health/statusIntegrity.test.ts create mode 100644 backend/src/health/statusResources.test.ts create mode 100644 backend/src/http/upstreamJson.test.ts create mode 100644 backend/src/http/upstreamJson.ts create mode 100644 backend/src/lifecycle/coordinator.test.ts create mode 100644 backend/src/lifecycle/coordinator.ts create mode 100644 backend/src/metrics.test.ts create mode 100644 backend/src/metricsServer.ts create mode 100644 backend/src/nodes/publicMap.test.ts create mode 100644 backend/src/nodes/publicMap.ts create mode 100644 backend/src/operations/operatorOperations.test.ts create mode 100644 backend/src/operations/operatorOperations.ts create mode 100644 backend/src/owner/alertRules.test.ts create mode 100644 backend/src/path-beta/resolveCache.test.ts create mode 100644 backend/src/platform/config/boundedNumber.ts create mode 100644 backend/src/platform/config/database.test.ts create mode 100644 backend/src/queue/publisher.integration.test.ts create mode 100644 backend/src/repositories/networkAnalysis.ts create mode 100644 backend/src/repositories/nodes.ts create mode 100644 backend/src/repositories/ownerAlerts.ts create mode 100644 backend/src/repositories/plannedCoverage.ts create mode 100644 backend/src/repositories/plannedNodes.test.ts create mode 100644 backend/src/repositories/plannedNodes.ts create mode 100644 backend/src/repositories/productFeatures.test.ts create mode 100644 backend/src/repositories/productFeatures.ts create mode 100644 backend/src/security/operatorSession.test.ts create mode 100644 backend/src/security/operatorSession.ts create mode 100644 backend/src/security/outboundWebhook.test.ts create mode 100644 backend/src/security/outboundWebhook.ts create mode 100644 backend/src/stats/chartSnapshot.test.ts create mode 100644 backend/src/stats/chartSnapshot.ts create mode 100644 backend/src/tools/backfillStatsRollups.test.ts create mode 100644 backend/src/tools/cleanupOrphanedAnalysisRuns.ts create mode 100644 backend/src/tools/dataLifecycle.ts create mode 100644 backend/src/tools/generateOpenApi.ts create mode 100644 backend/src/tools/seedStatsChartSnapshot.ts create mode 100644 backend/src/workers/alert-receiver.test.ts create mode 100644 backend/src/workers/alert-receiver.ts create mode 100644 backend/src/workers/workerMetrics.ts create mode 100644 backup-receipts/README.md create mode 100644 backup-receipts/latest.json create mode 100644 backup-receipts/latest.json.sig create mode 100644 backup-receipts/verify.pem create mode 100644 docker-compose.live.yml create mode 100644 docker/mosquitto-reloader.py create mode 100644 docs/performance-budgets.md create mode 100644 docs/rf-coverage-rollout.md create mode 100644 docs/runbook-backup-restore.md create mode 100644 docs/runbook-queue-recovery.md create mode 100644 docs/runbook-release-rollback.md create mode 100644 docs/runbook-srtm-alerts-observers.md create mode 100644 docs/security-closure.md create mode 100644 docs/security-waivers.md create mode 100644 frontend/public/sw-runtime.js create mode 100644 frontend/scripts/check-css-structure.mjs create mode 100644 frontend/scripts/check-performance-budgets.mjs create mode 100644 frontend/scripts/soak-production.mjs create mode 100644 frontend/src/components/Map/geojsonBuilders.test.ts create mode 100644 frontend/src/components/Map/mapSourceLayers.test.ts create mode 100644 frontend/src/components/ServiceWorkerUpdatePrompt.tsx create mode 100644 frontend/src/components/stats/StatsDecodedPathDialog.tsx create mode 100644 frontend/src/components/stats/StatsPrimitives.tsx create mode 100644 frontend/src/components/ui/Combobox.tsx create mode 100644 frontend/src/components/ui/Dialog.tsx create mode 100644 frontend/src/components/ui/Tabs.tsx create mode 100644 frontend/src/config/publicRoutes.test.ts create mode 100644 frontend/src/config/publicRoutes.ts create mode 100644 frontend/src/config/runtimeFeatures.test.ts create mode 100644 frontend/src/config/runtimeFeatures.ts create mode 100644 frontend/src/hooks/useAppMessageHandler.test.ts create mode 100644 frontend/src/hooks/useDashboardStats.test.ts create mode 100644 frontend/src/hooks/useLinkState.test.ts create mode 100644 frontend/src/hooks/useReducedMotion.ts create mode 100644 frontend/src/hooks/useStatsPageData.ts create mode 100644 frontend/src/hooks/useVisibilityPoll.test.ts create mode 100644 frontend/src/hooks/useVisibilityPoll.ts create mode 100644 frontend/src/pages/NotFoundPage.tsx create mode 100644 frontend/src/pages/owner/OwnerMapView.tsx create mode 100644 frontend/src/pages/owner/OwnerPortalCharts.tsx create mode 100644 frontend/src/pages/owner/ownerPortalModel.test.ts create mode 100644 frontend/src/pages/owner/ownerPortalModel.ts create mode 100644 frontend/src/pages/statsState.test.ts create mode 100644 frontend/src/pages/statsState.ts create mode 100644 frontend/src/pages/topologyModel.test.ts create mode 100644 frontend/src/pages/topologyModel.ts create mode 100644 frontend/src/serviceWorkerRuntime.test.ts create mode 100644 frontend/src/serviceWorkerUpdates.ts create mode 100644 frontend/src/utils/api.test.ts create mode 100644 frontend/src/utils/frameSnapshotScheduler.test.ts create mode 100644 frontend/src/utils/frameSnapshotScheduler.ts create mode 100644 frontend/src/utils/nodeIds.ts create mode 100644 frontend/src/utils/requestLimiter.test.ts create mode 100644 frontend/src/utils/requestLimiter.ts create mode 100644 frontend/src/utils/scopedCache.test.ts create mode 100644 frontend/src/utils/scopedCache.ts create mode 100644 frontend/src/utils/terrainSampler.test.ts create mode 100644 frontend/test/e2e/accessibility.spec.ts create mode 100644 logging/alertmanager.yml create mode 100644 logging/alloy/config.alloy create mode 100644 logging/blackbox.yml delete mode 100644 logging/grafana/dashboards/ml-path-learner.json delete mode 100644 logging/promtail.yaml create mode 100644 logging/rules/meshcore.test.yml create mode 100644 logging/rules/meshcore.yml create mode 100644 ml-path-learner/.dockerignore create mode 100644 ml-path-learner/requirements.in create mode 100644 ml-path-learner/tests/__init__.py create mode 100644 ml-path-learner/tests/test_worker.py create mode 100644 plan.md create mode 100755 scripts/backup.sh create mode 100755 scripts/bootstrap-mosquitto.sh create mode 100644 scripts/check-trivy-waivers.mjs create mode 100644 scripts/package-lock.json create mode 100755 scripts/restore-drill.sh create mode 100755 scripts/test-alert-receiver.sh create mode 100755 scripts/test-alloy-cutover.sh create mode 100755 scripts/test-generate-observer-key.sh create mode 100755 scripts/test-replace-container.sh create mode 100755 scripts/test-vacuum-maintenance.sh create mode 100644 viewshed-worker/.dockerignore create mode 100644 viewshed-worker/dry_run_coverage.py create mode 100644 viewshed-worker/queue_admin.py create mode 100644 viewshed-worker/requirements.in create mode 100644 viewshed-worker/tests/benchmark_rf.py create mode 100644 viewshed-worker/tests/benchmark_rf_radial.py create mode 100644 viewshed-worker/tests/coverage_dry_run_input.sql create mode 100644 viewshed-worker/tests/fixtures/rf_radial_golden.json create mode 100644 viewshed-worker/tests/test_dry_run_coverage.py create mode 100644 viewshed-worker/tests/test_link_queue_v3.py create mode 100644 viewshed-worker/tests/test_loss.py create mode 100644 viewshed-worker/tests/test_rf_radial.py create mode 100644 viewshed-worker/tests/test_terrain.py create mode 100644 viewshed-worker/tests/test_viewshed_queue_v2.py create mode 100644 viewshed-worker/tests/test_worker_outcomes.py create mode 100644 viewshed-worker/viewshed_queue_v2.py create mode 100644 viewshed-worker/worker_metrics.py diff --git a/.env.example b/.env.example index ea0ee8c..7e50f4a 100644 --- a/.env.example +++ b/.env.example @@ -17,6 +17,10 @@ OPERATOR_SITE_TOKEN=change_me_64_hex_chars POSTGRES_DB=meshcore POSTGRES_USER=meshcore POSTGRES_PASSWORD=change_me_strong_password +# Must exceed Timescale's configured background-worker count (currently 16) +# plus launcher/scheduler and logical-replication headroom. Requires a database +# restart when changed. +POSTGRES_MAX_WORKER_PROCESSES=24 # ── MQTT broker ─────────────────────────────────────────────────────────────── # Internal docker network URL for Mosquitto WebSocket listener @@ -39,8 +43,27 @@ STALE_MQTT_OBSERVER_CLEANUP_INTERVAL_MS=21600000 # Bound expensive statistics scans so they cannot exhaust the OLTP pool. STATS_DB_QUERY_CONCURRENCY=2 +# Aggregate reads remain off until the resumable backfill and shadow comparison +# have passed. Never leave shadow mode enabled after rollout validation. +STATS_AGGREGATE_READS_ENABLED=false +STATS_AGGREGATE_SHADOW_ENABLED=false WARMUP_NETWORKS=ukmesh +# Logical storage budgets used by operator health. The backend measures only +# these mounted data volumes; container root filesystem usage is not reported. +HEALTH_DATABASE_BUDGET_BYTES=322122547200 +HEALTH_REDIS_BUDGET_BYTES=4294967296 +HEALTH_TERRAIN_BUDGET_BYTES=53687091200 +HEALTH_LOG_BUDGET_BYTES=21474836480 + +# Destructive lifecycle actions are opt-in and additionally require the +# approval/backup/restore gates documented in docs/db-lifecycle.md. +DATA_LIFECYCLE_COMPRESSION_ENABLED=false +DATA_LIFECYCLE_RETENTION_ENABLED=false +DATA_LIFECYCLE_RETENTION_TARGETS= +BACKUP_RECEIPT_HOST_DIR=./backup-receipts +DATA_LIFECYCLE_RECEIPT_MAX_AGE_DAYS=7 + # ── Redis ───────────────────────────────────────────────────────────────────── # Docker Compose protects the internal Redis service with this password. The # application passes it separately, so REDIS_URL can remain a plain endpoint. @@ -58,6 +81,9 @@ JWT_SECRET=change_me_64_hex_chars # For local development, the defaults in docker-compose.yml cover localhost. # Set this to your production domains for deployment. ALLOWED_ORIGINS=https://app.example.com,https://www.example.com +# Per-client public API requests/minute. Keep 120 in production; only isolated +# load-test projects should raise this. +API_RATE_LIMIT_MAX=120 # ── Frontend hostname (build-time) ──────────────────────────────────────────── # If you're running two separate frontends (an analytics app and a public website) @@ -67,6 +93,12 @@ ALLOWED_ORIGINS=https://app.example.com,https://www.example.com # Leave blank to always show the analytics dashboard (simpler single-domain setup). VITE_APP_HOSTNAME=app.example.com +# RF coverage v6 uses bounded vectorized radial-prefix calculations. +COVERAGE_MODEL_VERSION=7 +COVERAGE_JOB_DEADLINE_SECONDS=600 +RF_PREFIX_ENDPOINT_BATCH=64 +RF_PREFIX_RAY_BATCH=8 + # ── MeshCore channel secrets ────────────────────────────────────────────────── # Comma-separated list of AES-128 channel secrets for decrypting GroupText packets. # Format: name:hex or bare hex (name defaults to first 6 chars of hex). @@ -79,7 +111,8 @@ MESHCORE_CHANNEL_SECRETS= # The app subscribes to observer packet MQTT and waits for users to send a # generated code to the configured MeshCore channel. HEALTHCHECK_APP_TITLE=UKMesh Health Check -HEALTHCHECK_SOURCE_REF=main +# Reviewed immutable upstream commit. Branches and tags are rejected at build. +HEALTHCHECK_SOURCE_REF=952eec8e0c55d0749f9abef211c266f992c0bfc7 HEALTHCHECK_TEST_CHANNEL_NAME=ukmeshtest # Optional source entry name in MESHCORE_CHANNEL_SECRETS when the stored secret # is named differently from the user-facing channel name. @@ -153,6 +186,9 @@ LINK_QUEUE_V3_PRODUCER_ENABLED=1 LINK_QUEUE_V3_MAX_JOBS=5000 LINK_QUEUE_V3_MAX_BYTES=67108864 LINK_QUEUE_V3_MAX_PAYLOAD_BYTES=32768 +LINK_QUEUE_V3_DEAD_MAX_JOBS=1000 +LINK_QUEUE_V3_DEAD_MAX_BYTES=33554432 +LINK_QUEUE_V3_DEAD_RETENTION_MS=2592000000 LINK_QUEUE_V3_MAX_ATTEMPTS=5 LINK_QUEUE_V3_LEASE_MS=120000 LINK_REBUILD_ADMISSION_TIMEOUT_MS=1800000 @@ -175,10 +211,28 @@ MQTT_AUDIT_RETENTION_DAYS=30 # Set true only when it is started with: # docker compose --profile viewshed up -d viewshed-worker VIEWSHED_ENABLED=false +# Public map layers are independently controlled at runtime and fail closed. +# Change these values and recreate only the backend container (no image rebuild); +# connected clients refresh /api/runtime-config within the configured TTL. +PUBLIC_FEATURE_INFERRED_NODES_ENABLED=true +PUBLIC_FEATURE_PACKET_ARCS_ENABLED=true +PUBLIC_FEATURE_HEATMAP_ENABLED=false +PUBLIC_FEATURE_CONFIG_TTL_SECONDS=30 OPENTOPODATA_API=https://api.opentopodata.org # Suppress duplicate advert-triggered coverage jobs after a successful enqueue. # A coordinate change or explicit coverage request bypasses this cooldown. VIEWSHED_JOB_COOLDOWN_SECONDS=300 +VIEWSHED_QUEUE_MAX=1000 +VIEWSHED_QUEUE_MAX_BYTES=16777216 +VIEWSHED_MAX_PAYLOAD_BYTES=4096 +VIEWSHED_QUEUE_MAX_ATTEMPTS=5 +VIEWSHED_QUEUE_LEASE_MS=900000 +VIEWSHED_QUEUE_DEAD_MAX_JOBS=100 +VIEWSHED_QUEUE_DEAD_MAX_BYTES=16777216 +VIEWSHED_QUEUE_DEAD_RETENTION_MS=2592000000 +SRTM_MAX_LINK_TILES=64 +SRTM_MAX_JOB_TILES=256 +SRTM_CACHE_MAX_BYTES=21474836480 # Synthetic monitoring and optional alert delivery SYNTHETIC_INTERVAL_MS=60000 @@ -240,3 +294,13 @@ SPAM_MESSAGE_ORIGIN_PATH_AMBIGUOUS_WEIGHT=0.4 SPAM_MESSAGE_ORIGIN_NEAR_HOP_MAX=2 SPAM_MESSAGE_ORIGIN_NEAR_HOP_SLACK=1 SPAM_MESSAGE_ORIGIN_PER_HOP_KM=12 +# Disabled ML learner safeguards (used only when the commented Compose service +# is deliberately enabled after a shadow-data run). +MAX_TRAINING_GOLD_ROWS=100000 +ML_LEASE_SECONDS=90 +ML_HEARTBEAT_SECONDS=20 +ML_RUN_DEADLINE_SECONDS=1200 +ML_MODEL_VERSION=lightgbm-path-v1 +ML_DATA_VERSION=gold-multibyte-v2 +# Numeric host group allowed to read the system journal (usually 999). +JOURNAL_GID=999 diff --git a/.github/dependabot.yml b/.github/dependabot.yml index d31a19a..d89db42 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -14,6 +14,39 @@ updates: groups: frontend-dependencies: patterns: ['*'] + - package-ecosystem: npm + directory: /scripts + schedule: + interval: weekly + groups: + scripts-dependencies: + patterns: ['*'] + - package-ecosystem: pip + directory: /viewshed-worker + schedule: + interval: weekly + groups: + rf-worker-dependencies: + patterns: ['*'] + - package-ecosystem: pip + directory: /ml-path-learner + schedule: + interval: weekly + groups: + ml-worker-dependencies: + patterns: ['*'] + - package-ecosystem: docker + directory: / + schedule: + interval: weekly + - package-ecosystem: docker + directory: /viewshed-worker + schedule: + interval: weekly + - package-ecosystem: docker + directory: /ml-path-learner + schedule: + interval: weekly - package-ecosystem: github-actions directory: / schedule: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c279ed8..f82a2fd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,12 +18,37 @@ jobs: backend: name: Backend runs-on: ubuntu-latest + services: + postgres: + image: timescale/timescaledb@sha256:22e8a5ae7aef121d1537afe946dd7cc5deeeb63ab36ce19849d671bd3b663509 + env: + POSTGRES_PASSWORD: analysis-test-password + ports: + - 55433:5432 + options: >- + --health-cmd "pg_isready -U postgres" + --health-interval 5s + --health-timeout 3s + --health-retries 10 + redis: + image: redis@sha256:8b81dd37ff027bec4e516d41acfbe9fe2460070dc6d4a4570a2ac5b9d59df065 + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 5s + --health-timeout 3s + --health-retries 10 + env: + TEST_REDIS_URL: redis://127.0.0.1:6379 + TEST_ANALYSIS_DATABASE_URL: postgresql://postgres:analysis-test-password@127.0.0.1:55433/postgres + TEST_INGEST_DATABASE_URL: postgresql://postgres:analysis-test-password@127.0.0.1:55433/postgres defaults: run: working-directory: backend steps: - - uses: actions/checkout@v7 - - uses: actions/setup-node@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 with: node-version: 20 cache: npm @@ -32,7 +57,11 @@ jobs: - name: Audit backend dependencies run: npm audit --audit-level=high - run: npm run typecheck + - run: npm run contract:check - run: npm test + - run: npm run test:analysis-integration + - run: npm run test:ingest-integration + - run: npm run test:queue-integration - run: npm run build frontend: @@ -42,15 +71,16 @@ jobs: run: working-directory: frontend steps: - - uses: actions/checkout@v7 - - uses: actions/setup-node@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 with: - node-version: 20 + node-version: 22 cache: npm cache-dependency-path: frontend/package-lock.json - run: npm ci - name: Audit frontend dependencies run: npm audit --audit-level=high + - run: npm test - run: npm run build - run: npx playwright install --with-deps chromium - run: npm run test:e2e @@ -58,13 +88,77 @@ jobs: workers-and-compose: name: Workers and Compose runs-on: ubuntu-latest + services: + postgres: + image: postgres@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777 + env: + POSTGRES_PASSWORD: ml-test-password + ports: + - 55432:5432 + options: >- + --health-cmd "pg_isready -U postgres" + --health-interval 5s + --health-timeout 3s + --health-retries 10 steps: - - uses: actions/checkout@v7 - - uses: actions/setup-python@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: 22 + - name: Validate release security waivers + run: node scripts/check-trivy-waivers.mjs + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 with: python-version: '3.12' + - name: Test operator scripts + working-directory: scripts + run: | + npm ci + npm audit --audit-level=high + npm run typecheck + npm test - name: Compile Python workers - run: python -m py_compile viewshed-worker/worker.py viewshed-worker/rf/*.py ml-path-learner/worker.py + run: python -m py_compile viewshed-worker/worker.py viewshed-worker/link_queue_v3.py viewshed-worker/viewshed_queue_v2.py viewshed-worker/backfill_profiles.py viewshed-worker/rf/*.py ml-path-learner/worker.py + - name: Build worker images + run: | + docker build -f viewshed-worker/Dockerfile -t meshcore-viewshed-worker:test viewshed-worker + docker build -f ml-path-learner/Dockerfile -t meshcore-ml-path-learner:test ml-path-learner + - name: Test viewshed worker in its GDAL image + run: | + trap 'docker rm -f link-queue-test-redis >/dev/null 2>&1 || true' EXIT + docker run -d --name link-queue-test-redis \ + -p 127.0.0.1:6380:6379 \ + redis@sha256:8b81dd37ff027bec4e516d41acfbe9fe2460070dc6d4a4570a2ac5b9d59df065 redis-server --requirepass test-queue-password --maxmemory-policy noeviction + for _ in $(seq 1 30); do + if docker exec link-queue-test-redis redis-cli -a test-queue-password --no-auth-warning ping >/dev/null; then + break + fi + sleep 1 + done + docker run --rm --network host --entrypoint python \ + -e TEST_REDIS_URL=redis://127.0.0.1:6380 \ + -e TEST_REDIS_PASSWORD=test-queue-password \ + meshcore-viewshed-worker:test -m unittest discover -s /app/tests -v + docker run --rm --entrypoint python \ + meshcore-viewshed-worker:test /app/tests/benchmark_rf.py --rays 12 --steps 1000 + docker run --rm --network none --entrypoint python \ + meshcore-viewshed-worker:test /app/tests/benchmark_rf_radial.py --rays 72 --steps 500 --repeats 1 + - name: Test ML worker in its image + run: | + docker run --rm --network host --entrypoint python \ + -e TEST_DATABASE_URL=postgresql://postgres:ml-test-password@127.0.0.1:55432/postgres \ + meshcore-ml-path-learner:test -m unittest discover -s /app/tests -v + - name: Bootstrap clean Mosquitto credentials + env: + MQTT_USERNAME: backend + MQTT_PASSWORD: ci-mqtt-password + MOSQUITTO_CONFIG_DIR: /tmp/meshcore-ci-mosquitto + run: | + install -d -m 700 "$MOSQUITTO_CONFIG_DIR" + cp mosquitto/mosquitto.conf "$MOSQUITTO_CONFIG_DIR/mosquitto.conf" + scripts/bootstrap-mosquitto.sh + test "$(stat -c '%a' "$MOSQUITTO_CONFIG_DIR/passwd")" = 640 + test "$(stat -c '%a' "$MOSQUITTO_CONFIG_DIR/acl")" = 640 - name: Validate Compose configuration env: POSTGRES_PASSWORD: ci-postgres-password @@ -73,18 +167,193 @@ jobs: OPERATOR_SITE_TOKEN: ci-operator-site-token-0123456789abcdef0123456789abcdef MQTT_PASSWORD: ci-mqtt-password ANUBIS_ED25519_PRIVATE_KEY_HEX: '0000000000000000000000000000000000000000000000000000000000000001' + HEALTHCHECK_SOURCE_REF: 952eec8e0c55d0749f9abef211c266f992c0bfc7 GRAFANA_ADMIN_PASSWORD: ci-grafana-password - run: docker compose config --quiet + run: | + docker compose config --quiet + docker compose config --format json \ + | jq -e '.services.timescaledb.command + | index("max_worker_processes=24") != null' >/dev/null + - name: Build every application image and smoke an empty-volume stack + env: + POSTGRES_PASSWORD: ci-postgres-password + REDIS_PASSWORD: ci-redis-password + JWT_SECRET: ci-jwt-secret + OPERATOR_SITE_TOKEN: ci-operator-site-token-0123456789abcdef0123456789abcdef + MQTT_USERNAME: backend + MQTT_PASSWORD: ci-mqtt-password + ANUBIS_ED25519_PRIVATE_KEY_HEX: '0000000000000000000000000000000000000000000000000000000000000001' + HEALTHCHECK_SOURCE_REF: 952eec8e0c55d0749f9abef211c266f992c0bfc7 + GRAFANA_ADMIN_PASSWORD: ci-grafana-password + SOURCE_REVISION: ${{ github.sha }} + MOSQUITTO_IMAGE: eclipse-mosquitto@sha256:9cfdd46ad59f3e3e5f592f6baf57ab23e1ad00605509d0f5c1e9b179c5314d87 + run: | + project="meshcore-ci-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + publisher_user="ci-e2e-publisher" + publisher_password="ci-e2e-publisher-password" + observer="CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC" + cleanup() { + docker compose \ + --project-name "$project" \ + --profile dev \ + --profile viewshed \ + down --volumes --remove-orphans + } + trap cleanup EXIT + scripts/bootstrap-mosquitto.sh + docker run --rm \ + --user "$(id -u):$(id -g)" \ + -v "$PWD/mosquitto:/mosquitto/config" \ + "$MOSQUITTO_IMAGE" \ + mosquitto_passwd -b /mosquitto/config/passwd \ + "$publisher_user" "$publisher_password" + printf '\nuser %s\ntopic write meshcore-test/CI/%s/packets\ntopic write meshcore-test/CI/%s/status\n' \ + "$publisher_user" "$observer" "$observer" >>mosquitto/acl + chmod 640 mosquitto/passwd mosquitto/acl + docker build \ + --build-arg SOURCE_REVISION="$SOURCE_REVISION" \ + --build-arg VITE_APP_HOSTNAME=app.ukmesh.com \ + -f Dockerfile -t meshcore-combined:test . + docker compose \ + --project-name "$project" \ + --profile dev \ + --profile viewshed \ + build \ + backend app-ukmesh website-ukmesh mesh-health-check \ + mosquitto-reloader link-worker + docker compose \ + --project-name "$project" \ + --profile dev \ + --profile viewshed \ + up --detach --no-build --wait --wait-timeout 360 + health_session_id="$( + curl --fail --silent --request POST \ + --header 'Content-Type: application/json' \ + --data '{}' \ + http://127.0.0.1:3090/api/sessions \ + | jq -er '.id' + )" + sleep 1 + docker compose --project-name "$project" exec -T \ + -e HEALTH_SESSION_ID="$health_session_id" \ + mesh-health-check node --input-type=module <<'NODE' + import { readFileSync } from 'node:fs'; + const payload = JSON.parse( + readFileSync('/app/data/session-results.json', 'utf8'), + ); + if (!payload.sessions?.some(({ id }) => id === process.env.HEALTH_SESSION_ID)) { + throw new Error('health-check session was not persisted by the non-root runtime'); + } + NODE + curl --fail --silent http://127.0.0.1:3000/readyz \ + | jq -e '.status == "ready"' >/dev/null + curl --fail --silent http://127.0.0.1:3003/ >/dev/null + curl --fail --silent http://127.0.0.1:3004/ >/dev/null + curl --fail --silent http://127.0.0.1:3003/readyz \ + | jq -e '.status == "ready"' >/dev/null + curl --fail --silent http://127.0.0.1:3004/readyz \ + | jq -e '.status == "ready"' >/dev/null + for origin in http://127.0.0.1:3003 http://127.0.0.1:3004; do + for path in /metrics /operations /observer-registrations /local-api/operations; do + test "$(curl --silent --output /dev/null --write-out '%{http_code}' "${origin}${path}")" = 404 + done + done + docker compose --project-name "$project" exec -T backend \ + wget -qO- http://127.0.0.1:9091/metrics \ + | grep -q '^meshcore_process_' + ws_log="$(mktemp)" + docker compose --project-name "$project" exec -T \ + -e MQTT_E2E_OBSERVER="$observer" \ + backend node --input-type=module >"$ws_log" <<'NODE' & + import WebSocket from 'ws'; + const observer = process.env.MQTT_E2E_OBSERVER.toLowerCase(); + const socket = new WebSocket('ws://website-ukmesh:8080/ws?network=test'); + const timeout = setTimeout(() => { + console.error('timed out waiting for MQTT packet WebSocket fanout'); + process.exit(1); + }, 15_000); + socket.on('message', (raw) => { + for (const line of String(raw).split('\n').filter(Boolean)) { + const message = JSON.parse(line); + if ( + message.type === 'packet' + && String(message.data?.rxNodeId ?? '').toLowerCase() === observer + ) { + clearTimeout(timeout); + console.log('mqtt-packet-fanout-ok'); + socket.close(); + process.exit(0); + } + } + }); + socket.on('error', (error) => { + console.error(error); + process.exit(1); + }); + NODE + ws_pid=$! + sleep 2 + docker compose --project-name "$project" exec -T mosquitto \ + mosquitto_pub -h 127.0.0.1 -p 1883 \ + -u "$publisher_user" -P "$publisher_password" \ + -t "meshcore-test/CI/${observer}/status" \ + -m '{"origin":"CI E2E observer","origin_id":"CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC","stats":{"battery_mv":4100,"uptime_secs":60}}' + docker compose --project-name "$project" exec -T mosquitto \ + mosquitto_pub -h 127.0.0.1 -p 1883 \ + -u "$publisher_user" -P "$publisher_password" \ + -t "meshcore-test/CI/${observer}/packets" \ + -m '{"packet_type":1,"RSSI":-75,"SNR":8,"direction":"rx","raw":"0D02AABBDEADBEEF"}' + readback="" + for _ in $(seq 1 30); do + readback="$( + docker compose --project-name "$project" exec -T timescaledb \ + psql -U meshcore -d meshcore -Atc \ + "SELECT EXISTS(SELECT 1 FROM nodes WHERE node_id = '${observer}' AND network = 'test'), EXISTS(SELECT 1 FROM packets WHERE rx_node_id = '${observer}' AND network = 'test'), EXISTS(SELECT 1 FROM packet_hourly_stats WHERE network = 'test')" + )" + if [ "$readback" = "t|t|t" ]; then + break + fi + sleep 1 + done + test "$readback" = "t|t|t" + wait "$ws_pid" + grep -q '^mqtt-packet-fanout-ok$' "$ws_log" + - name: Validate monitoring configuration + run: | + docker run --rm \ + -v "$PWD/logging/prometheus.yml:/etc/prometheus/prometheus.yml:ro" \ + -v "$PWD/logging/rules:/etc/prometheus/rules:ro" \ + --entrypoint /bin/promtool \ + prom/prometheus@sha256:63805ebb8d2b3920190daf1cb14a60871b16fd38bed42b857a3182bc621f4996 \ + check config /etc/prometheus/prometheus.yml + docker run --rm \ + -v "$PWD/logging/rules:/rules:ro" \ + --entrypoint /bin/promtool \ + prom/prometheus@sha256:63805ebb8d2b3920190daf1cb14a60871b16fd38bed42b857a3182bc621f4996 \ + test rules /rules/meshcore.test.yml + docker run --rm \ + -v "$PWD/logging:/etc/meshcore:ro" \ + --entrypoint /bin/amtool \ + prom/alertmanager@sha256:27c475db5fb156cab31d5c18a4251ac7ed567746a2483ff264516437a39b15ba \ + check-config /etc/meshcore/alertmanager.yml + docker run --rm \ + -v "$PWD/logging/alloy/config.alloy:/etc/alloy/config.alloy:ro" \ + grafana/alloy@sha256:491b0578c04983fd54fe99b587b6fab4404dc46d0dc16677bd6b00cc1140b308 \ + validate /etc/alloy/config.alloy + - name: Test compressed-chunk maintenance safety gates + run: scripts/test-vacuum-maintenance.sh + - name: Test immutable release rollback and schema compatibility gates + run: scripts/test-replace-container.sh secrets: name: Secret scan runs-on: ubuntu-latest steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: fetch-depth: 0 - name: Scan repository with gitleaks - uses: gitleaks/gitleaks-action@v2 + uses: gitleaks/gitleaks-action@dcedce43c6f43de0b836d1fe38946645c9c638dc # v2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -93,8 +362,8 @@ jobs: if: github.event_name == 'schedule' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v7 - - uses: actions/setup-node@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 with: node-version: 20 cache: npm @@ -107,10 +376,15 @@ jobs: OPERATOR_SITE_TOKEN: nightly-operator-token-0123456789abcdef0123456789abcdef MQTT_PASSWORD: nightly-mqtt-password GRAFANA_ADMIN_PASSWORD: nightly-grafana-password - run: docker compose up -d --build timescaledb redis mosquitto db-migrate backend + API_RATE_LIMIT_MAX: 1000000 + ANUBIS_ED25519_PRIVATE_KEY_HEX: '0000000000000000000000000000000000000000000000000000000000000001' + HEALTHCHECK_SOURCE_REF: 952eec8e0c55d0749f9abef211c266f992c0bfc7 + run: | + MQTT_USERNAME=backend MQTT_PASSWORD=nightly-mqtt-password scripts/bootstrap-mosquitto.sh + docker compose up -d --build timescaledb redis mosquitto db-migrate backend - name: Wait for readiness run: | - for attempt in $(seq 1 60); do + for _ in $(seq 1 60); do if curl --fail --silent http://127.0.0.1:3000/readyz >/dev/null; then exit 0 fi @@ -126,7 +400,7 @@ jobs: - name: Capture logs if: always() run: docker compose logs --no-color backend > nightly-backend.log - - uses: actions/upload-artifact@v7 + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() with: name: nightly-load-logs diff --git a/.github/workflows/frontend-mobile.yml b/.github/workflows/frontend-mobile.yml index 2cb3c5d..b043834 100644 --- a/.github/workflows/frontend-mobile.yml +++ b/.github/workflows/frontend-mobile.yml @@ -16,8 +16,8 @@ jobs: run: working-directory: frontend steps: - - uses: actions/checkout@v7 - - uses: actions/setup-node@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 with: node-version: 20 cache: npm diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..2273ef0 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,192 @@ +name: Signed container release + +on: + release: + types: [published] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: release-${{ github.ref }} + cancel-in-progress: false + +jobs: + policy: + name: Release policy + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: 22 + - name: Validate vulnerability waivers + run: node scripts/check-trivy-waivers.mjs + - name: Require a clean commit release + run: test -z "$(git status --porcelain --untracked-files=all)" + + image: + name: ${{ matrix.component }} + needs: policy + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write + attestations: write + strategy: + fail-fast: false + matrix: + include: + - component: backend + image: meshcore-backend + context: . + file: Dockerfile.backend + build_args: '' + - component: combined + image: meshcore-combined + context: . + file: Dockerfile + build_args: VITE_APP_HOSTNAME=app.ukmesh.com + - component: app + image: meshcore-app + context: . + file: Dockerfile.app + build_args: | + VITE_APP_HOSTNAME=app.ukmesh.com + VITE_NETWORK=ukmesh + VITE_SITE=ukmesh + - component: website + image: meshcore-website + context: . + file: Dockerfile.website + build_args: | + VITE_APP_HOSTNAME=app.ukmesh.com + VITE_SITE=ukmesh + - component: rf-worker + image: meshcore-rf-worker + context: viewshed-worker + file: viewshed-worker/Dockerfile + build_args: '' + - component: ml-path-learner + image: meshcore-ml-path-learner + context: ml-path-learner + file: ml-path-learner/Dockerfile + build_args: '' + - component: mosquitto-reloader + image: meshcore-mosquitto-reloader + context: . + file: Dockerfile.mosquitto-reloader + build_args: '' + - component: health-check + image: meshcore-health-check + context: . + file: Dockerfile.mesh-health-check + build_args: MESH_HEALTH_CHECK_REF=952eec8e0c55d0749f9abef211c266f992c0bfc7 + env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository_owner }}/${{ matrix.image }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - id: meta + uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=sha,format=long + type=ref,event=tag + - id: build + name: Build and push immutable image + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 + with: + context: ${{ matrix.context }} + file: ${{ matrix.file }} + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: | + SOURCE_REVISION=${{ github.sha }} + ${{ matrix.build_args }} + provenance: false + sbom: false + cache-from: type=gha,scope=${{ matrix.component }} + cache-to: type=gha,mode=max,scope=${{ matrix.component }} + - name: Gate exact digest on critical vulnerabilities + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 + with: + scan-type: image + image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ steps.build.outputs.digest }} + format: json + output: trivy-${{ matrix.component }}.json + exit-code: '1' + severity: CRITICAL + ignore-unfixed: false + trivyignores: .trivyignore.yaml + - name: Generate SPDX SBOM for exact digest + uses: anchore/sbom-action@e22c389904149dbc22b58101806040fa8d37a610 # v0 + with: + image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ steps.build.outputs.digest }} + format: spdx-json + output-file: sbom-${{ matrix.component }}.spdx.json + upload-artifact: false + - uses: sigstore/cosign-installer@f713795cb21599bc4e5c4b58cbad1da852d7eeb9 # v3 + - name: Sign and attach SBOM attestation + env: + DIGEST: ${{ steps.build.outputs.digest }} + run: | + image="${REGISTRY}/${IMAGE_NAME}@${DIGEST}" + cosign sign --yes "$image" + cosign attest --yes \ + --predicate "sbom-${{ matrix.component }}.spdx.json" \ + --type spdxjson "$image" + cosign verify "$image" \ + --certificate-identity-regexp "^https://github.com/${GITHUB_REPOSITORY}/.github/workflows/release.yml@" \ + --certificate-oidc-issuer "https://token.actions.githubusercontent.com" \ + > "signature-${{ matrix.component }}.json" + - id: provenance + name: Publish registry provenance + uses: actions/attest@f7c74d28b9d84cb8768d0b8ca14a4bac6ef463e6 # v4 + with: + subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + subject-digest: ${{ steps.build.outputs.digest }} + push-to-registry: true + - name: Write release inventory + env: + DIGEST: ${{ steps.build.outputs.digest }} + ATTESTATION_URL: ${{ steps.provenance.outputs.attestation-url }} + run: | + jq -n \ + --arg component "${{ matrix.component }}" \ + --arg source_revision "$GITHUB_SHA" \ + --arg image "${REGISTRY}/${IMAGE_NAME}@${DIGEST}" \ + --arg sbom "sbom-${{ matrix.component }}.spdx.json" \ + --arg scan "trivy-${{ matrix.component }}.json" \ + --arg signature "signature-${{ matrix.component }}.json" \ + --arg provenance "$ATTESTATION_URL" \ + '{ + format: "meshcore-release-inventory-v1", + component: $component, + source_revision: $source_revision, + image: $image, + sbom: $sbom, + scan: $scan, + signature: $signature, + provenance: $provenance + }' > "inventory-${{ matrix.component }}.json" + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: release-${{ github.sha }}-${{ matrix.component }} + if-no-files-found: error + retention-days: 90 + path: | + inventory-${{ matrix.component }}.json + sbom-${{ matrix.component }}.spdx.json + trivy-${{ matrix.component }}.json + signature-${{ matrix.component }}.json diff --git a/.trivyignore.yaml b/.trivyignore.yaml new file mode 100644 index 0000000..d2497fc --- /dev/null +++ b/.trivyignore.yaml @@ -0,0 +1,3 @@ +# Trivy's YAML ignore format is used so every exception can be narrowly scoped +# and automatically expire. There are deliberately no active waivers. +vulnerabilities: [] diff --git a/Dockerfile b/Dockerfile index d70b344..d88b7df 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Stage 1: Build frontend -FROM node:20-alpine AS frontend-builder +FROM node:22-alpine@sha256:16e22a550f3863206a3f701448c45f7912c6896a62de43add43bb9c86130c3e2 AS frontend-builder WORKDIR /build/frontend COPY frontend/package.json frontend/package-lock.json ./ RUN npm ci @@ -9,20 +9,27 @@ ENV VITE_APP_HOSTNAME=$VITE_APP_HOSTNAME RUN npm run build # Stage 2: Build backend -FROM node:20-alpine AS backend-builder +FROM node:20-alpine@sha256:fb4cd12c85ee03686f6af5362a0b0d56d50c58a04632e6c0fb8363f609372293 AS backend-builder WORKDIR /build/backend COPY backend/package.json backend/package-lock.json ./ -RUN npm ci +RUN npm ci --audit COPY backend/ ./ RUN npm run build # Stage 3: Runtime -FROM node:20-alpine AS runtime +FROM node:20-alpine@sha256:fb4cd12c85ee03686f6af5362a0b0d56d50c58a04632e6c0fb8363f609372293 AS runtime +ARG SOURCE_REVISION=unknown +LABEL org.opencontainers.image.revision="${SOURCE_REVISION}" \ + org.opencontainers.image.source="https://github.com/gadgethd/ukmesh" WORKDIR /app # Install production deps only COPY backend/package.json backend/package-lock.json ./ -RUN npm ci --omit=dev && npm cache clean --force +RUN npm ci --omit=dev --audit \ + && npm cache clean --force \ + && rm -rf /usr/local/lib/node_modules/npm \ + && rm -f /usr/local/bin/npm /usr/local/bin/npx +RUN mkdir -p /var/lib/meshcore-alerts && chown 1000:1000 /var/lib/meshcore-alerts # Copy compiled backend COPY --from=backend-builder /build/backend/dist ./dist @@ -31,10 +38,12 @@ COPY --from=backend-builder /build/backend/dist ./dist COPY --from=backend-builder /build/backend/src/db/schema ./dist/db/schema COPY --from=backend-builder /build/backend/src/db/migrations ./dist/db/migrations COPY --from=backend-builder /build/backend/src/db/owner-auth.sql ./dist/db/owner-auth.sql -COPY --from=backend-builder /build/backend/src/backend-site/template.html ./dist/backend-site/template.html +COPY --from=backend-builder /build/backend/src/backend-site/*.html ./dist/backend-site/ +COPY docs/openapi.yaml ./openapi.yaml # Copy frontend build into static dir served by backend COPY --from=frontend-builder /build/frontend/dist ./public -EXPOSE 3000 +EXPOSE 3000 9091 +USER 1000:1000 CMD ["node", "dist/index.js"] diff --git a/Dockerfile.app b/Dockerfile.app index 86b2441..83694c5 100644 --- a/Dockerfile.app +++ b/Dockerfile.app @@ -1,10 +1,11 @@ # App frontend — React SPA served via Nginx, proxies /api and /ws to backend -FROM node:20-alpine AS frontend-builder +FROM node:22-alpine@sha256:16e22a550f3863206a3f701448c45f7912c6896a62de43add43bb9c86130c3e2 AS frontend-builder WORKDIR /build/frontend COPY frontend/package.json frontend/package-lock.json ./ RUN npm ci COPY frontend/ ./ ARG VITE_APP_HOSTNAME +ARG VITE_BUILD_TARGET=app ARG VITE_NETWORK=ukmesh ARG VITE_SITE=ukmesh ARG VITE_OBSERVER_ID= @@ -14,6 +15,7 @@ ARG VITE_SITE_APP_URL= ARG VITE_SITE_HOME_URL= ARG VITE_VIEWSHED_ENABLED=false ENV VITE_APP_HOSTNAME=$VITE_APP_HOSTNAME +ENV VITE_BUILD_TARGET=$VITE_BUILD_TARGET ENV VITE_NETWORK=$VITE_NETWORK ENV VITE_SITE=$VITE_SITE ENV VITE_OBSERVER_ID=$VITE_OBSERVER_ID @@ -24,8 +26,13 @@ ENV VITE_SITE_HOME_URL=$VITE_SITE_HOME_URL ENV VITE_VIEWSHED_ENABLED=$VITE_VIEWSHED_ENABLED RUN npm run build -FROM nginx:alpine +FROM nginx:alpine@sha256:4a73073bd557c65b759505da037898b61f1be6cbcc3c2c3aeac22d2a470c1752 +ARG SOURCE_REVISION=unknown +LABEL org.opencontainers.image.revision="${SOURCE_REVISION}" \ + org.opencontainers.image.source="https://github.com/gadgethd/ukmesh" COPY --from=frontend-builder /build/frontend/dist /usr/share/nginx/html COPY nginx.app.conf /etc/nginx/conf.d/default.conf COPY nginx.security-headers.conf /etc/nginx/snippets/security-headers.conf -EXPOSE 80 +RUN chown -R nginx:nginx /var/cache/nginx /var/run /etc/nginx/conf.d +USER 101:101 +EXPOSE 8080 diff --git a/Dockerfile.backend b/Dockerfile.backend index a84fdbe..49fd989 100644 --- a/Dockerfile.backend +++ b/Dockerfile.backend @@ -1,19 +1,28 @@ # Backend only — API, WebSocket, MQTT ingestion, DB -FROM node:20-alpine AS backend-builder +FROM node:20-alpine@sha256:fb4cd12c85ee03686f6af5362a0b0d56d50c58a04632e6c0fb8363f609372293 AS backend-builder WORKDIR /build/backend COPY backend/package.json backend/package-lock.json ./ RUN npm ci --audit COPY backend/ ./ RUN npm run build -FROM node:20-alpine AS runtime +FROM node:20-alpine@sha256:fb4cd12c85ee03686f6af5362a0b0d56d50c58a04632e6c0fb8363f609372293 AS runtime +ARG SOURCE_REVISION=unknown +LABEL org.opencontainers.image.revision="${SOURCE_REVISION}" \ + org.opencontainers.image.source="https://github.com/gadgethd/ukmesh" WORKDIR /app COPY backend/package.json backend/package-lock.json ./ -RUN npm ci --omit=dev --audit && npm cache clean --force +RUN npm ci --omit=dev --audit \ + && npm cache clean --force \ + && rm -rf /usr/local/lib/node_modules/npm \ + && rm -f /usr/local/bin/npm /usr/local/bin/npx +RUN mkdir -p /var/lib/meshcore-alerts && chown 1000:1000 /var/lib/meshcore-alerts COPY --from=backend-builder /build/backend/dist ./dist COPY --from=backend-builder /build/backend/src/db/schema ./dist/db/schema COPY --from=backend-builder /build/backend/src/db/migrations ./dist/db/migrations COPY --from=backend-builder /build/backend/src/db/owner-auth.sql ./dist/db/owner-auth.sql +COPY --from=backend-builder /build/backend/src/backend-site/*.html ./dist/backend-site/ COPY docs/openapi.yaml ./openapi.yaml -EXPOSE 3000 +USER 1000:1000 +EXPOSE 3000 9091 CMD ["node", "dist/index.js"] diff --git a/Dockerfile.mesh-health-check b/Dockerfile.mesh-health-check index 60c738a..3d7919c 100644 --- a/Dockerfile.mesh-health-check +++ b/Dockerfile.mesh-health-check @@ -1,21 +1,24 @@ -FROM node:22-slim +FROM node:22-slim@sha256:6c74791e557ce11fc957704f6d4fe134a7bc8d6f5ca4403205b2966bd488f6b3 AS builder ARG MESH_HEALTH_CHECK_REPO=https://github.com/gadgethd/meshcore-health-check.git -ARG MESH_HEALTH_CHECK_REF=main +ARG MESH_HEALTH_CHECK_REF WORKDIR /app RUN apt-get update \ && apt-get install -y --no-install-recommends ca-certificates git \ && rm -rf /var/lib/apt/lists/* \ + && test -n "$MESH_HEALTH_CHECK_REF" \ + && echo "$MESH_HEALTH_CHECK_REF" | grep -Eq '^[0-9a-f]{40}$' \ && git init /tmp/mesh-health-check \ && git -C /tmp/mesh-health-check remote add origin "$MESH_HEALTH_CHECK_REPO" \ && git -C /tmp/mesh-health-check fetch --depth 1 origin "$MESH_HEALTH_CHECK_REF" \ && git -C /tmp/mesh-health-check checkout --detach FETCH_HEAD \ + && test "$(git -C /tmp/mesh-health-check rev-parse HEAD)" = "$MESH_HEALTH_CHECK_REF" \ && cp /tmp/mesh-health-check/package.json /tmp/mesh-health-check/package-lock.json ./ \ && mkdir -p ./scripts \ && cp /tmp/mesh-health-check/scripts/patch-meshcore-decoder.js ./scripts/patch-meshcore-decoder.js \ - && npm install --omit=dev --no-audit --fund=false \ + && npm ci --omit=dev --no-audit --fund=false \ && cp /tmp/mesh-health-check/server.js ./ \ && cp -a /tmp/mesh-health-check/lib ./lib \ && cp -a /tmp/mesh-health-check/public ./public \ @@ -23,13 +26,25 @@ RUN apt-get update \ && cp /tmp/mesh-health-check/README.md /tmp/mesh-health-check/HOWTO.md /tmp/mesh-health-check/AGENTS.md /tmp/mesh-health-check/CHANGES.md ./ \ && rm -rf /tmp/mesh-health-check \ && apt-get purge -y --auto-remove git \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* /usr/local/lib/node_modules/npm \ + && rm -f /usr/local/bin/npm /usr/local/bin/npx +FROM node:22-alpine@sha256:16e22a550f3863206a3f701448c45f7912c6896a62de43add43bb9c86130c3e2 +ARG SOURCE_REVISION=unknown +LABEL org.opencontainers.image.revision="${SOURCE_REVISION}" \ + org.opencontainers.image.source="https://github.com/gadgethd/ukmesh" +WORKDIR /app +COPY --from=builder --chown=node:node /app /app COPY docker/mesh-health-check-entrypoint.sh /usr/local/bin/mesh-health-check-entrypoint.sh -RUN chmod +x /usr/local/bin/mesh-health-check-entrypoint.sh +RUN rm -rf /usr/local/lib/node_modules/npm \ + && rm -f /usr/local/bin/npm /usr/local/bin/npx \ + && chmod +x /usr/local/bin/mesh-health-check-entrypoint.sh \ + && mkdir -p /app/data \ + && chown -R node:node /app/data ENV NODE_ENV=production EXPOSE 3090 +USER node ENTRYPOINT ["mesh-health-check-entrypoint.sh"] CMD ["node", "server.js"] diff --git a/Dockerfile.mosquitto-reloader b/Dockerfile.mosquitto-reloader new file mode 100644 index 0000000..16d3349 --- /dev/null +++ b/Dockerfile.mosquitto-reloader @@ -0,0 +1,9 @@ +FROM python:3.12-alpine@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df +ARG SOURCE_REVISION=unknown +LABEL org.opencontainers.image.revision="${SOURCE_REVISION}" \ + org.opencontainers.image.source="https://github.com/gadgethd/ukmesh" +WORKDIR /app +COPY docker/mosquitto-reloader.py /app/reloader.py +USER 65534:65534 +EXPOSE 8080 +CMD ["python3", "-u", "/app/reloader.py"] diff --git a/Dockerfile.website b/Dockerfile.website index b1197d9..b83a544 100644 --- a/Dockerfile.website +++ b/Dockerfile.website @@ -1,9 +1,10 @@ -FROM node:20-alpine AS frontend-builder +FROM node:22-alpine@sha256:16e22a550f3863206a3f701448c45f7912c6896a62de43add43bb9c86130c3e2 AS frontend-builder WORKDIR /build/frontend COPY frontend/package.json frontend/package-lock.json ./ RUN npm ci COPY frontend/ ./ ARG VITE_APP_HOSTNAME +ARG VITE_BUILD_TARGET=public-website ARG VITE_SITE=ukmesh ARG VITE_OBSERVER_ID= ARG VITE_NETWORK= @@ -13,6 +14,7 @@ ARG VITE_SITE_APP_URL= ARG VITE_SITE_HOME_URL= ARG VITE_VIEWSHED_ENABLED=false ENV VITE_APP_HOSTNAME=$VITE_APP_HOSTNAME +ENV VITE_BUILD_TARGET=$VITE_BUILD_TARGET ENV VITE_SITE=$VITE_SITE ENV VITE_OBSERVER_ID=$VITE_OBSERVER_ID ENV VITE_NETWORK=$VITE_NETWORK @@ -23,8 +25,13 @@ ENV VITE_SITE_HOME_URL=$VITE_SITE_HOME_URL ENV VITE_VIEWSHED_ENABLED=$VITE_VIEWSHED_ENABLED RUN npm run build -FROM nginx:alpine +FROM nginx:alpine@sha256:4a73073bd557c65b759505da037898b61f1be6cbcc3c2c3aeac22d2a470c1752 +ARG SOURCE_REVISION=unknown +LABEL org.opencontainers.image.revision="${SOURCE_REVISION}" \ + org.opencontainers.image.source="https://github.com/gadgethd/ukmesh" COPY --from=frontend-builder /build/frontend/dist /usr/share/nginx/html COPY nginx.website.conf /etc/nginx/conf.d/default.conf COPY nginx.security-headers.conf /etc/nginx/snippets/security-headers.conf -EXPOSE 80 +RUN chown -R nginx:nginx /var/cache/nginx /var/run /etc/nginx/conf.d +USER 101:101 +EXPOSE 8080 diff --git a/PLAN-repeater-tree-fix.md b/PLAN-repeater-tree-fix.md new file mode 100644 index 0000000..efb3b6c --- /dev/null +++ b/PLAN-repeater-tree-fix.md @@ -0,0 +1,87 @@ +# UKMesh — "Predicted Repeater Tree" Modal Fix Plan + +**Date:** 2026-08-01 +**Repo:** `~/meshcore-analytics` on 192.168.100.105 (remote: `github.com/gadgethd/ukmesh.git`, branch `main`) +**Deployed:** docker images `meshcore-analytics-app-ukmesh` (:3003) + `meshcore-analytics-website-ukmesh` (:3004), served via Cloudflare tunnel → Anubis + +--- + +## 1. What's wrong (from live site screenshot + code) + +The "Predicted Repeater Tree" modal on the live map / feed page: + +1. **Modal shell is unstyled** — no panel background/border/padding; title sits flush to the edge; the Close button renders as a giant full-width bright-cyan button; the modal looks broken/regressed. +2. **Tree content is confusing** — hops with no resolved repeater (raw 2-char trace fragments like `AD`, `91`) render as full "Unmatched repeater" nodes; every node's meta line dumps cryptic raw hex (`predicted repeater CBG D9993F88 D9 1/1 seen`); all dots are the same colour so matched/ambiguous/unmatched hops are indistinguishable. +3. **Header contradicts the tree** — says "1 branch 2/5 hops matched" while the tree shows 3 named repeaters (the ambiguous one isn't counted as "matched"). +4. **No map** — the modal shows only a text tree. The feed's right column already draws the path on a map; the stats page's Decoded Path modal (the good reference) shows a 420px map + node cards. The tree modal lost that context. + +## 2. Root cause (the regression) + +Commit **`83d770a`** ("fix(ui): UKMesh visual QA fixes — …", Aug 1, HEAD, deployed same day) **removed the 15 `.stats-page__path-modal*` rules from `frontend/src/styles/globals.css`** (they had lived there since the modal was first added in `9dd1e7a`, and were duplicated into `stats-page.css` since `93cc3e1`). + +Today those styles exist **only** in `frontend/src/pages/stats-page.css`, which is imported **only** by `StatsPage.tsx` (lazy route chunk). The feed modal (`frontend/src/pages/ukmesh/FeedDialogs.tsx`) uses the same `stats-page__path-modal*` classes but **never imports that CSS** → on the feed page the modal falls back to: +- `.ui-dialog-modal` wrapper (760px) — overridden to 1180px by `.uk-feed-path-modal` (feed-page.css, loaded) +- **no panel bg, no border, no radius, no padding, no flex column** (all from the missing `.stats-page__path-modal`) +- Close button = global `.disclaimer-modal__close` (full-width cyan accent button) + +Verified in the **deployed bundles** (fetched from live :3003): the feed chunk contains the class names but **zero** `path-modal` CSS rules; the main CSS has none either. So the live site == current source. The stats page's Decoded Path modal looks fine because StatsPage imports the CSS. + +## 3. Live data check (packet in screenshot) + +`GET /api/path-lazy/resolve?hash=BC1D6965…FD131&network=ukmesh` returns 1 branch: +- pos 0: `hash:"AD"`, nodeId **null** → unmatched +- pos 1: `hash:"91"`, nodeId **null** → unmatched +- pos 2: "Bluntisham RPTR" (nodeId D9993F88…) → matched +- pos 3: "Cambridge South" (nodeId 4E879CAB…, `ambiguous:true`) → matched-but-ambiguous +- pos 4: "EWR 🧭 Stotfold Omni" (FECE48AF…) → matched +- pos 5: "dm-stotfold 📡" → observer (RX) +- `matchedHops: 2, totalHops: 5` (ambiguous **not** counted as matched) + +Data itself is sane; presentation is the problem. + +## 4. Fix design + +### Phase 1 — Restore modal styling (the regression, must-do) +Extract the shared path-modal styles into a file both consumers import: +1. New `frontend/src/pages/path-modal.css` containing the `.stats-page__path-modal*` rules (panel, header, title, sub, close, map, list, node, mobile media query) currently in `stats-page.css` (lines ~402–545). +2. `StatsPage.tsx`: `import './path-modal.css';` (replaces the stats-page.css copy — delete the moved block). +3. `FeedDialogs.tsx`: `import '../path-modal.css';` +4. (Alternative if we want zero new files: `import '../stats-page.css'` in FeedDialogs — quicker but drags unrelated stats styles into the feed chunk and couples feed → stats page. Prefer the shared file.) + +Result: modal renders as a proper dark panel, compact Close button, mono uppercase accent title — identical look to the stats page modal. + +### Phase 2 — Tree readability (FeedPathViews.tsx + feed-page.css) +1. **Node identity/fallback:** + - Unmatched hop → title **"Unknown hop"** (not "Unmatched repeater"), meta shows only its short hash + seen count; grey **dashed** dot. + - Matched hop → repeater name (existing lookup), accent solid dot. + - Ambiguous hop → amber dot + "ambiguous" tag (data already has `ambiguous`). + - Observer → unchanged (green RX dot). +2. **Meta declutter** — matched nodes: `IATA · n/n seen` (drop role word, raw nodeId prefix, raw hash — name is already the title; put full nodeId in `title` tooltip). Unmatched: `hop AD · 1/1 seen`. Observer: `observer · E0218FD9`. +3. **Header** — `5 hops · 2 matched · 1 ambiguous` (compute ambiguous count from paths) + `N branches` only when >1. Drop the confusing bare "2/5 hops matched". +4. **Sizing/typography** — modal width 1180px → `min(960px, calc(100vw - 32px))` (matches stats modal); tree text 12px → 13–14px; more row padding. + +### Phase 3 — Put the map back in the modal (the "easy to see" win) +1. Reuse the existing `PathMap` (already used by `FeedMapPanel` / `PacketDetailPanel`) — render it in the modal body **above** the tree: header → map (`stats-page__path-modal-map` style, height ~340–420px) → tree. +2. Inputs already available: `lazyPath.paths[].coordinates`, `observerPositions` from packet, `nodeMap`. +3. Optional v1 skip: hover/click tree node → highlight on map (note for later). + +### Phase 4 — Housekeeping +- The July-29 feed refactor (untracked: `FeedDialogs.tsx`, `FeedPathViews.tsx`, `feedModel.ts`, `feedState.ts`, `PacketPathMap.tsx` + UKFeedPage rewrite) is already live in the deployed build but uncommitted. **Commit the refactor + this fix together** as one PR to `gadgethd/ukmesh` (branch from `main`). Do **not** `git add -A` (dirty tree carries unrelated changes; `.tokensave/` trap). + +## 5. Implementation steps (fix session) + +1. `cd ~/meshcore-analytics` — branch `fix/repeater-tree-modal` from main. +2. Create `frontend/src/pages/path-modal.css` (move block from stats-page.css), update imports in `StatsPage.tsx` + `FeedDialogs.tsx`. +3. Update `FeedPathViews.tsx` (labels, meta, header, dot variants) + `feed-page.css` (dot modifiers, typography, modal width). +4. Add modal map: embed `PathMap` in `FeedDialogs.tsx` body above `PacketPathTree`. +5. Update/extend `feedState.test.ts` if it covers tree status logic; add component-level checks where cheap. +6. Quality gates: `npx tsc --noEmit`, `npm test`, `npm run lint:css`, `npm run build`. +7. Deploy (with Ben's go-ahead): `docker compose -f docker-compose.yml -f docker-compose.live.yml build app-ukmesh website-ukmesh && … up -d` — or Ben deploys himself. **Restart via compose only, never bare `up`** (shared external network). +8. Verify live: open a packet's Repeater tree on app.ukmesh.com → screenshot; confirm styled panel, readable tree, map present. Screenshot to Ben. + +## 6. Risks / notes +- **Working tree is dirty** (many unrelated M files) — touch only the fix files; never reset/stash. +- Shared external network `meshcore-analytics_default` — compose down/up must keep the `live` overlay file. +- Anubis sits in front of the app — after deploy, first load may be challenged; bot policy already allows normal browsers. +- Website build (`public-website` target) shares `UKFeedPage` (`feed` route in main.tsx) — same fix covers both images; rebuild both. +- No backend changes needed (path-lazy API is fine). diff --git a/README.md b/README.md index c976b5c..01f3729 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ A real-time analytics platform for [MeshCore](https://meshcore.io) networks. It - RF coverage viewshed polygons per repeater using SRTM terrain data - Link intelligence overlay with directional observations and path-loss viability - Public repeater-topology explorer with hub ranking, graph components, likely bridge repeaters, isolated nodes, and multibyte-evidence filtering -- Map modes, shareable viewport/filter URLs, node detail drawer, and bounded activity replay +- Map modes, shareable viewport/filter URLs, selected-node popup, and bounded activity replay - Planned-repeater comparison with saved scenarios, share links, and overlap estimates - Regional health scoring and predicted-versus-observed RF validation - Local saved searches/watchlists for nodes, observers, regions, packet types, and incidents @@ -65,7 +65,7 @@ A real-time analytics platform for [MeshCore](https://meshcore.io) networks. It - Owner-facing dashboard: repeater summary, packet history, advert counts, direct sender map, heard-by list, link health, and alerts - Planned node placement tool: drop a marker on the map, preview estimated RF coverage before deploying hardware - Planned repeater registration/claim workflow improvements -- In-app owner alerts for low battery, stale links, advert drops, silence, and missing observers +- Owner alerts for offline duration, low battery voltage, and excessive predicted path loss, with durable delivery history and test delivery ### Phase 6 - Network intelligence expansion (complete) - Bounded topology graph with hubs, graph components, isolated repeaters, and likely bridge nodes @@ -114,13 +114,22 @@ cd ukmesh # 2. Copy and configure environment cp .env.example .env -# Edit .env — at minimum set POSTGRES_PASSWORD, JWT_SECRET, MQTT_PASSWORD, -# and REDIS_PASSWORD. +# Edit .env. Required values are POSTGRES_PASSWORD, JWT_SECRET, +# MQTT_PASSWORD, REDIS_PASSWORD, OPERATOR_SITE_TOKEN, +# ANUBIS_ED25519_PRIVATE_KEY_HEX, GRAFANA_ADMIN_PASSWORD, and the full +# 40-character HEALTHCHECK_SOURCE_REF. -# 3. Start everything +# 3. Create Mosquitto's backend credential and least-privilege ACL +set -a +. ./.env +set +a +scripts/bootstrap-mosquitto.sh + +# 4. Start everything docker compose up -d -# 4. Check logs +# 5. Confirm readiness and inspect logs +curl --fail http://127.0.0.1:3000/readyz docker compose logs -f backend ``` @@ -154,6 +163,7 @@ Copy `.env.example` to `.env` and fill in your values. All variables used by the | `POSTGRES_DB` | `meshcore` | TimescaleDB database name | | `POSTGRES_USER` | `meshcore` | TimescaleDB user | | `POSTGRES_PASSWORD` | *(required)* | TimescaleDB password | +| `POSTGRES_MAX_WORKER_PROCESSES` | `24` | PostgreSQL worker slots; keep above the Timescale background-worker setting plus launcher/scheduler headroom | | `MQTT_BROKER_URL` | `ws://mosquitto:9001` | Mosquitto WebSocket URL (internal) | | `MQTT_USERNAME` | `backend` | MQTT client username | | `MQTT_PASSWORD` | *(required)* | MQTT client password | @@ -161,6 +171,7 @@ Copy `.env.example` to `.env` and fill in your values. All variables used by the | `REDIS_URL` | `redis://redis:6379` | Redis URL for WebSocket pub/sub | | `JWT_SECRET` | *(required)* | Secret for JWT verification | | `ALLOWED_ORIGINS` | `http://localhost:3001,http://localhost:3002` | Comma-separated browser origins allowed for CORS and WebSocket | +| `API_RATE_LIMIT_MAX` | `120` | Per-client public API requests/minute; raise only in an isolated load-test project | | `VITE_APP_HOSTNAME` | *(blank — always shows dashboard)* | If set, only this hostname serves the analytics dashboard; all others serve the public website layout | | `MESHCORE_CHANNEL_SECRETS` | *(blank)* | Comma-separated channel secrets for decrypting GroupText packets. Format: `name:hex` or bare hex. The default MeshCore public channel key is always included. | | `OPENTOPODATA_API` | `https://api.opentopodata.org` | Elevation API endpoint for viewshed computation | @@ -172,26 +183,49 @@ Copy `.env.example` to `.env` and fill in your values. All variables used by the | `OWNER_ACL_UNMANAGED_USERS` | `backend,test,test2` | Exact broker accounts intentionally preserved outside owner grant management | | `OWNER_ACL_ALLOW_EMPTY_USERS` | *(empty)* | Explicitly reviewed owner accounts allowed to render with no publish grants | | `VIEWSHED_ENABLED` | `false` | Enable coverage/planned-repeater API only when the profile-gated viewshed worker is running | +| `PUBLIC_FEATURE_INFERRED_NODES_ENABLED` | `true` | Runtime kill switch for privacy-reviewed inferred map nodes | +| `PUBLIC_FEATURE_PACKET_ARCS_ENABLED` | `true` | Runtime kill switch for privacy-filtered live packet arcs | +| `PUBLIC_FEATURE_HEATMAP_ENABLED` | `false` | Runtime kill switch for the packet heatmap | +| `PUBLIC_FEATURE_CONFIG_TTL_SECONDS` | `30` | Client refresh interval for same-origin public feature configuration (bounded to 5–300 seconds) | | `COVERAGE_MODEL` | `rf_radial_100m` | Coverage model used by `viewshed-worker` | -| `COVERAGE_MODEL_VERSION` | `5` | Coverage schema/version gate used to trigger recomputation | +| `COVERAGE_MODEL_VERSION` | `7` | Coverage schema/version gate used to trigger recomputation | +| `RF_PREFIX_RAY_BATCH` | `8` | CPU RF rays evaluated together; raise cautiously because memory grows with the batch | | `CLOUDFLARE_TUNNEL_TOKEN` | *(optional)* | Cloudflare Zero Trust tunnel token | | `PORT` | `3000` | Internal app port | +The inferred-node and packet-arc layers are enabled after privacy/correctness +validation; the heatmap remains disabled by default. Each has an independent +runtime kill switch. To operate an immediate kill switch, +change the relevant value in `.env`, then recreate the backend without building +an image: + +```bash +docker compose up -d --no-build --force-recreate backend +``` + +Browsers load `/api/runtime-config` before the map starts and refresh it within +`PUBLIC_FEATURE_CONFIG_TTL_SECONDS`. A failed, malformed, or timed-out request +disables all three layers. + --- ## Mosquitto Setup Mosquitto is configured for WebSocket-only access with password authentication. -After first starting the stack, add the backend service password: +Create the backend credential and least-privilege read ACL before first startup: ```bash -# Add the backend client password (must match MQTT_PASSWORD in .env) -docker exec meshcore-analytics-mosquitto-1 \ - mosquitto_passwd -b /mosquitto/config/passwd backend your_password - -docker compose restart mosquitto +# Reads MQTT_USERNAME/MQTT_PASSWORD from the environment. +set -a +. ./.env +set +a +scripts/bootstrap-mosquitto.sh +docker compose up -d ``` +The bootstrap is idempotent, sets credential files to mode `0640`, refuses +symlinks or incomplete existing state, and never overwrites live credentials. + Do not create observer passwords manually. Provision every observer with `~/bin/newuser`; it installs and verifies the publish ACL before enabling the credentials. @@ -214,15 +248,17 @@ To expose the app and MQTT broker publicly without opening firewall ports: 3. Add to `.env`: `CLOUDFLARE_TUNNEL_TOKEN=` 4. Start with the tunnel profile: `docker compose --profile tunnel up -d` 5. Configure public hostnames in the Cloudflare dashboard (example): - - `app.example.com` → `http://app-ukmesh:80` - - `www.example.com` → `http://website-ukmesh:80` + - `app.example.com` → `http://anubis-app-ukmesh:8923` + - `www.example.com` → `http://anubis-website-ukmesh:8923` - `mqtt.example.com` → `http://mosquitto:9001` - - `healthcheck.example.com` → `http://mesh-health-check:3090` + - `healthcheck.example.com` → `http://anubis-mesh-health-check:8923` For UKMesh health checks, point `healthcheck.ukmesh.com` at -`http://mesh-health-check:3090` in the same tunnel. The container uses the +`http://anubis-mesh-health-check:8923` in the same tunnel. The container uses the internal Mosquitto WebSocket listener and persists observer/result state in the -`mesh_health_check_data` Docker volume. By default it uses +`mesh_health_check_data` Docker volume. A network-isolated one-shot Compose +initializer normalizes that volume's ownership before the capability-free, +non-root application starts. By default it uses `HEALTHCHECK_TEST_CHANNEL_NAME=ukmeshtest` and reuses the existing `test:...` entry from `MESHCORE_CHANNEL_SECRETS` via `HEALTHCHECK_TEST_CHANNEL_SECRET_SOURCE_NAME=test`. @@ -273,8 +309,8 @@ MeshCore Devices └─ app-ukmesh / website-ukmesh / website-dev (interactive dashboard + public site + owner portal) Python Workers - ├─ viewshed-worker (meshcore:viewshed_jobs) - ├─ link-worker (meshcore:link_jobs) + ├─ viewshed-worker (bounded meshcore:viewshed:v2 protocol; legacy drain reader) + ├─ link-worker (bounded meshcore:link:v3 protocol; legacy drain reader) ├─ SRTM terrain tiles (auto-downloaded) └─ node_coverage + node_links updates @@ -294,9 +330,10 @@ MeshCore Devices | Service | Image | Purpose | |---|---|---| -| `timescaledb` | `timescale/timescaledb:latest-pg16` | Time-series and relational data storage | -| `mosquitto` | `eclipse-mosquitto:2` | MQTT broker (WebSocket only) | -| `redis` | `redis:7-alpine` | WebSocket fan-out pub/sub and job queue | +| `timescaledb` | Digest-pinned TimescaleDB/PostgreSQL image from Compose | Time-series and relational data storage | +| `mosquitto` | Digest-pinned Eclipse Mosquitto image from Compose | MQTT broker (WebSocket only) | +| `mosquitto-reloader` | Locally built, least-privilege helper | Authenticated broker-local ACL reload | +| `redis` | Digest-pinned Redis image from Compose | WebSocket fan-out pub/sub and bounded job queues | | `backend` | Built from `Dockerfile.backend` | MQTT ingest, decoding, API, WebSocket | | `path-learning-worker` | Built from `Dockerfile.backend` | Hourly path-learning model rebuilds | | `path-history-worker` | Built from `Dockerfile.backend` | Historical path resolution backfill | @@ -310,14 +347,26 @@ MeshCore Devices | `mesh-health-check` | Built from the configured `gadgethd/meshcore-health-check` ref | MeshCore observer coverage health-check app | | `website-dev` | Built from `Dockerfile.website` | Isolated test/status site for `meshcore-test/*` traffic | | `cloudflared` | `cloudflare/cloudflared` | Optional Cloudflare Tunnel (use `--profile tunnel`) | +| `alloy`, `loki` | Digest-pinned observability images | Bounded journal/container log collection and storage | +| `prometheus`, `alertmanager`, `alert-receiver` | Digest-pinned observability images plus local receiver | Metrics, rules, grouped alert delivery, and delivery history | +| `grafana` | Digest-pinned Grafana image | Provisioned read-only operational dashboards | +| `postgres-exporter`, `redis-exporter`, `node-exporter`, `blackbox-exporter` | Digest-pinned exporters | Database, queue, host, and endpoint telemetry | --- ## Data Retention -- Packet retention policy is currently disabled. Historical data is kept indefinitely unless explicitly pruned. -- Node/link/coverage/path-learning/health tables are also retained indefinitely by default. -- Synthetic journey results are pruned to 14 days by the monitor. +- Compression and destructive retention ship disabled. Raw packets and status + samples remain intact until a table-specific, backup- and restore-gated + lifecycle rollout is approved. +- The proposed raw retention window is 180 days, preserving the longest + 120-day learner dependency. Privacy-safe hourly/daily aggregates and current + node/link/coverage/model state remain longer lived. +- Operational row-table cleanup is bounded and runs only when both + `DATA_LIFECYCLE_RETENTION_ENABLED=true` and the exact table appears in + `DATA_LIFECYCLE_RETENTION_TARGETS`. +- See `docs/db-lifecycle.md` for the exact dry-run inventory, compression, + retention, owner export, and restore requirements. --- diff --git a/UI-AUDIT.md b/UI-AUDIT.md new file mode 100644 index 0000000..c2dd19e --- /dev/null +++ b/UI-AUDIT.md @@ -0,0 +1,1022 @@ +# MeshCore Analytics — Frontend UI/UX Audit + +**Date:** 2026-08-01 +**Branch audited:** `fix/repeater-tree-modal` (working tree, uncommitted changes included) +**Scope:** `frontend/src/**` (57 `.tsx`, 63 `.ts`, 12 `.css`), `frontend/index.html`, `backend/src/backend-site/login.html` +**Type:** Read-only static audit. **No fixes were applied.** + +> **Resolution update — 2026-08-01:** The audit itself was read-only, but the +> follow-up implementation has now resolved UI-01 through UI-28 in the current +> working tree. Verification includes 38 frontend unit tests, 27 desktop/axe +> browser tests, 21 responsive/mobile browser tests, production builds, the CSS +> structure gate, and the asset-size budget. The browser suite covers every +> public route, keyboard/focus flows, touch-only map controls, and modal geometry +> from 320px through 1440px. The source gates report no undefined CSS custom +> properties, no text below the 10px floor, no duplicate selectors, no missing +> button `type`, no missing table-header `scope`, and no diff whitespace errors. + +--- + +## 0. How to use this document + +Each finding has a stable ID (`UI-01`…`UI-28`), a severity, exact file/line anchors, the *measured* evidence, the user-visible impact, and a concrete fix with acceptance criteria. Findings are independent unless a **Depends on** line says otherwise. + +**Recommended order of work:** §7 has a sequenced plan. Do not work top-to-bottom by ID — several P1 items share a root cause and are cheaper to fix together. + +**Before you start:** read §6 ("Verified correct — do not re-investigate"). Several plausible-looking problems were checked and are *not* bugs. Re-chasing them wastes time and risks regressions. + +--- + +## 1. Method + +1. **Full read** of `tokens.css`, `globals.css`, `map-app.css`, `site-shell.css`, `site-content.css`, `network-intelligence.css`, `path-modal.css`, plus targeted reads of `feed-page.css`, `spam-page.css`, `stats-page.css`, `docs-pages.css`, `owner-portal.css`. +2. **Undefined CSS custom property scan** — parsed every `--x:` definition and every `var(--x)` usage across all CSS/TSX, flagged usages with no definition and no fallback. +3. **WCAG 2.1 contrast computation** — a scripted sRGB relative-luminance calculator (script in §8) was run on every foreground/background pair discovered, including alpha compositing for `rgba()` and `opacity`. +4. **Typography scan** — every `font-size` and `font` shorthand across all CSS, normalised px/rem, flagged `< 11px`. +5. **Cascade verification** — checked the *built* CSS in `frontend/dist/assets/` and Vite's preload manifest to confirm real-world stylesheet load order, because specificity ties are resolved by load order here. +6. **Dead-code cross-check** — every class named in a finding was checked against all TSX/HTML to confirm it actually renders. Findings on dead classes are demoted to P3 and labelled. +7. **Structured a11y scan** — form-control labelling, ``, `role=` correctness, icon-only buttons, `` +- `frontend/src/pages/ukmesh/UKFeedPage.tsx:680` — `` + +Both are labelled only by the bare glyph `✕` (U+2715). Screen readers announce it as "multiplication X", "times", or nothing depending on the AT and its punctuation settings. + +Every other close control in the codebase is done correctly — `App.tsx:656` (`aria-label="Close"`), `MapLibreMap.tsx:1393` (`aria-label="Close node details"`), `WatchlistPanel.tsx:54`, `NodeSearch.tsx:172`, `Combobox.tsx:91`. These two are the outliers. + +**Fix** +```tsx +/* PacketDetailPanel.tsx:272 */ + + +/* UKFeedPage.tsx:680 */ + +``` +(The `type="button"` on the second also resolves one instance of **UI-20**.) + +--- + +### UI-14 — P2 — Feed packet search has no accessible name + +**File** `frontend/src/pages/ukmesh/UKFeedPage.tsx:530–536` + +```tsx + +``` + +No `