diff --git a/.env.example b/.env.example index 7e51ccd..759234f 100644 --- a/.env.example +++ b/.env.example @@ -26,6 +26,26 @@ POSTGRES_MAX_WORKER_PROCESSES=24 # Internal docker network URL for Mosquitto WebSocket listener MQTT_BROKER_URL=ws://mosquitto:9001 MQTT_CLIENT_ID=meshcore-analytics-ingest +MESHCORE_SITE_ID=ukmesh + +# Site-to-site owner packet sharing is available to every authenticated owner. +# It does not enable a destination. Each owner/node selection starts off until +# that owner turns it on. Destination rows are operator-managed in PostgreSQL; +# leave the encryption key empty until a destination is provisioned. +OWNER_PACKET_SHARE_ENCRYPTION_KEY= +OWNER_PACKET_SHARE_POLL_INTERVAL_MS=5000 +OWNER_PACKET_SHARE_BATCH_SIZE=25 +# Owner dashboard RX-strength cache warmer. Keeps the last-hop series cached for +# every active MQTT owner's nodes (7d and 30d windows) so the owner dashboard +# chart never pays for a cold scan on open. Interval in ms (invalid or <= 0 +# disables); NODE_DELAY paces the per-node loop within a pass. +OWNER_LAST_HOP_PREWARM_INTERVAL_MS=900000 +OWNER_LAST_HOP_PREWARM_NODE_DELAY_MS=150 +# Public feed endpoint settings. This is the authentication-locked UKmesh +# Mosquitto endpoint; opted-in active owners use their existing MQTT login for +# a read-only ACL grant. Account toggles remain off by default. +PUBLIC_FEED_PUBLISH_ENABLED=false +PUBLIC_FEED_BROKER_URL= # Credentials for the backend MQTT client MQTT_USERNAME=backend @@ -76,7 +96,7 @@ HEALTH_LOG_BUDGET_BYTES=21474836480 DATA_LIFECYCLE_COMPRESSION_ENABLED=false DATA_LIFECYCLE_RETENTION_ENABLED=false DATA_LIFECYCLE_COMPRESSION_TARGETS=packets,packet_paths,node_status_samples,node_neighbor_samples -DATA_LIFECYCLE_RETENTION_TARGETS=packets,worker_health_snapshots,frontend_error_events,owner_alert_deliveries,operational_check_results,observer_region_packet_sightings,observer_region_observer_sightings,link_job_commits,ml_model_variant_packet_results,packet_decryptions,observer_registration_requests,operator_audit_events +DATA_LIFECYCLE_RETENTION_TARGETS=packets,worker_health_snapshots,frontend_error_events,owner_alert_deliveries,owner_packet_share_deliveries,public_feed_deliveries,operational_check_results,observer_region_packet_sightings,observer_region_observer_sightings,link_job_commits,ml_model_variant_packet_results,packet_decryptions,observer_registration_requests,operator_audit_events BACKUP_RECEIPT_HOST_DIR=./backup-receipts DATA_LIFECYCLE_RECEIPT_MAX_AGE_DAYS=7 @@ -164,6 +184,12 @@ OWNER_ACL_UNMANAGED_USERS=backend,test,test2 # Explicit exception list for intentionally revoked/disabled accounts that # should retain an empty broker stanza during apply cutover. OWNER_ACL_ALLOW_EMPTY_USERS= +# Operator-approved read-only subscribers (comma-separated MQTT usernames). +# These accounts receive only the curated public packet namespace for opted-in, +# active owners; they never receive publish permission. +OWNER_ACL_PUBLIC_FEED_READ_USERS= +# Internal backend publish namespace for the curated public feed. +OWNER_ACL_SYSTEM_READWRITE_TOPICS=public/# # Set these after exporting the reviewed pre-reset inventory. Readiness fails # closed if database counts/grants, generations, or ACL readback differ. OWNER_AUTH_INVENTORY_HOST_DIR=./backup-receipts @@ -199,6 +225,9 @@ SPAM_MESSAGE_MAX_NORMALIZED_CHARS=512 SPAM_MESSAGE_MAX_INCIDENTS=100 SPAM_MESSAGE_MAX_CANDIDATE_CLUSTERS=64 SPAM_MESSAGE_MAX_EVIDENCE_MEMBERS=256 +# Per-run work budget for the spam analyzer. Quiet days finish in ~4s; busy +# spam days need ~30-60s (relay-path origin resolution per public incident). +# Raise (e.g. 60000) if the analyzer log shows status=timed_out. SPAM_MESSAGE_ANALYSIS_BUDGET_MS=5000 SPAM_MESSAGE_DB_TIMEOUT_MS=15000 STATS_UNIQUE_INFLIGHT_MAX=32 @@ -279,6 +308,15 @@ SPAM_MESSAGE_PUBLIC_MIN_SCORE=0.5 # sustained automated flood and scored up (unless it is plain test/connectivity # traffic). Catches templated floods even when they embed the word "test". SPAM_MESSAGE_FLOOD_MIN_MESSAGES=25 +# Rotating-ID botnet detection: many single-shot, machine-named sender IDs +# blasting the same text in bursts (see docs/spam-detection.md). A cluster +# qualifies when >= MIN_SENDERS distinct senders, >= SINGLE_SHOT_SHARE of them +# appear exactly once in the window, >= MACHINE_NAME_SHARE have keygen-style +# names, and >= MIN_BURST_SENDERS land inside one burst window. +SPAM_MESSAGE_ROTATING_MIN_SENDERS=5 +SPAM_MESSAGE_ROTATING_SINGLE_SHOT_SHARE=0.8 +SPAM_MESSAGE_ROTATING_MACHINE_NAME_SHARE=0.5 +SPAM_MESSAGE_ROTATING_MIN_BURST_SENDERS=3 # Channel labels excluded from detection (comma-separated). The test channel # carries repeated test traffic by design. NOTE: a flood that also hits other # channels (e.g. Public) is still caught via those copies. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 47389e9..47733f5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -225,6 +225,13 @@ jobs: publisher_user="ci-e2e-publisher" publisher_password="ci-e2e-publisher-password" observer="CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC" + cleanup() { + docker compose -f docker-compose.yml -f docker-compose.ci.yml \ + --project-name "$project" \ + --profile dev \ + down --volumes --remove-orphans + } + trap cleanup EXIT # Create each Argon2 entry independently. Updating an existing # password file with this image is not reliable on hosted runners. docker run --rm \ diff --git a/.gitignore b/.gitignore index 8da2091..52203e5 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,9 @@ __pycache__/ # Runtime logs (operational, not for source control) *.log +# Frozen local path evaluation data and per-packet predictions +/artifacts/path-accuracy/ + # Claude context file (may contain deployment-specific details) CLAUDE.md AI_MEMORY.md @@ -62,3 +65,10 @@ local-vs-github-pr-plan.md terrain-tiles/ /.env.bak-* +# Encrypted deployment-specific relay identities. +.private-config/ + +# Local planning documents / reports (tidy folder, never committed) +/plans/ +*.bak +*.bak-* diff --git a/.gitleaks.toml b/.gitleaks.toml index 4dd28ff..e623993 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -2,3 +2,38 @@ title = "UK Mesh gitleaks configuration" [extend] useDefault = true + +[[rules]] +id = "generic-api-key" + +[[rules.allowlists]] +description = "Documented MeshCore public-channel interoperability key" +paths = ['''^backend/backfill-transport-codes\.mjs$'''] +regexTarget = "line" +regexes = ['''(?i)const DEFAULT_PUBLIC_CHANNEL_KEY\s*=\s*'[0-9a-f]{32}'\s*;'''] +condition = "AND" + +[[rules.allowlists]] +description = "Reviewed public/community MeshCore channel secrets (channelRegistry.ts VALIDATED_CHANNELS) — each entry verified to decrypt real UK Mesh group text; classified intentionally public in the channel registry" +paths = ['''^backend/src/mqtt/channelRegistry\.ts$'''] +regexTarget = "line" +regexes = ['''\{ name: '[A-Za-z0-9_-]+',\s+secret: '[0-9a-f]{32}'\s*\},?'''] +condition = "AND" + +[[rules.allowlists]] +description = "Documented Public channel key in any file/historical commit (exact fingerprint)" +regexTarget = "secret" +regexes = ['''^8b3387e9c5cdea6ac9e5edbaa115cd72$'''] + +[[rules.allowlists]] +description = "Fake test fixtures (exact fingerprints)" +regexTarget = "secret" +regexes = [ + '''^0123456789abcdef0123456789abcdef$''', + '''^abcdef0123456789$''', +] + +[[rules.allowlists]] +description = "Deployed-commit SHA in live website manifests" +regexTarget = "secret" +regexes = ['''^bef003f072283fd5e3f14b29890f7610adf3fb9808e8c6e700bd48d3e7a8bc87$'''] diff --git a/DB-AUDIT-FIX-2026-08-13.md b/DB-AUDIT-FIX-2026-08-13.md deleted file mode 100644 index ecc2d2b..0000000 --- a/DB-AUDIT-FIX-2026-08-13.md +++ /dev/null @@ -1,230 +0,0 @@ -# ukmesh database reset-readiness fix report — 2026-08-13 - -## Verdict - -**Fresh-DB acceptance: PASS.** A database created from empty storage on the -CI/restore pinned TimescaleDB 2.25.1 digest applied the complete 49-file chain -(the 48 audited entries plus `051_reset_readiness.sql`), passed the full object -assertions, and treated a second migration run as a no-op. - -**Production reset authorization: BLOCKED only on independent off-host -publication.** The current encrypted local backup and isolated restore are -verified. The VPS has no distinct mounted off-host target, so the reset must not -start until Ben mounts and attests one and `sync-latest.sh` reports `verified`. -The local sync drill is useful evidence for the script, but is not represented -as off-host evidence. - -No production migration, SQL write, service restart, or production `compose up` -was performed. The only live-stack operations were read-only database dumps, -supported Redis/Mosquitto persistence checkpoints, and the read-only owner -inventory export. `.env` grant values were not changed. - -## Commits - -- `e7d16d4` — `fix(db): make reset schema and lifecycle reproducible` -- `c79f71e` — `fix(owner): preserve authorization inventory across reset` -- `e3a8ba3` — `fix(backup): restore split-stack recovery workflow` - -These commits are local only and were not pushed. - -## Finding status - -### F1 / G1 — identity materialized views: FIXED (`e7d16d4`) - -Migration 051 safely replaces fresh ordinary views with populated materialized -views, while treating already-materialized live objects as no-ops. It creates -the unique sighting `(node_id, network)` and link `(node_a_id, node_b_id)` -indexes, the reversed sighting index, and both link endpoint indexes. Fresh-DB -tests assert `pg_matviews`, all five definitions, and successful concurrent -refreshes of both views. - -### F2 / G5 — split-stack backup and recovery: REVISED (`e3a8ba3`) - -`backup.sh` now resolves TimescaleDB, Redis, and Mosquitto through the explicit -`meshcore-infra` project/directory. The encrypted recovery archive contains the -base/live/CI/infra-client/phase-4 app Compose files and a separate protected -infra configuration archive. Broker config is captured through its container, -and restore streams verified state into isolated named volumes, avoiding host -permission/bind differences. - -`sync-latest.sh` performs signed-receipt selection, full archive checksum -verification, partial-directory staging, atomic publication, a checksum -manifest, and post-publication verification. A same-filesystem functional drill -returned `verified` and its exact test directory was deleted. This finding is -REVISED rather than FIXED because no genuinely off-host mount exists yet. - -Canonical local backup evidence (secret recovery material, not committed): - -- ID: `backup-20260813T031400Z-2c1dd39b7173` -- completed: `2026-08-13T03:18:09Z` -- live schema captured: 50 -- encrypted bytes: 1,196,412,781 -- SHA-256: `6466077779be65d3f843843b3fbf57cd33abad671391c5744e93d3a0fe1911a7` -- datasets: analytics, owner auth, Redis, Mosquitto, app configuration, infra configuration -- exact-artifact isolated restore: PASS on the pinned 2.25.1 digest -- restored/current schema: 51; RTO: 414s; RPO at verification: 527s -- signed receipt: `backup-receipts/latest.json` (signature independently verified) - -The receipt's `source_revision` is the mandated pre-fix HEAD `2c1dd39` because -all gates were run before the local commits. The bundled scripts and migrations -are the exact working-tree content that passed the drill. - -### F3 / G3 — owner grant preservation: REVISED (`c79f71e`, `e3a8ba3`) - -The protected, gitignored baseline is -`/home/ben/ukmesh/backups/owner-grants-2026-08-13T021734Z.json` (mode 0600, -content hash `2d795007c523316cb4967262f89020813c213c727653fdabdfcd130fa0463fd6`). -It contains every reviewed active grant: 27 accounts, 44 grants, 5 -`operator-config`, 36 `operator-database`, and 3 legacy/null-method. Desired, -rendered, and applied ACL generations match and `lastError` is null. - -`owner-auth:inventory` exports/loads a checksum-bearing baseline and -`--require-complete` rejects count, method, config generation, ACL generation, -ACL readback, or last-error drift. `/readyz` applies the same baseline gate. -The recovery bundle includes `owner-auth.dump`; the runbook restores that dump, -mounts the baseline read-only, validates it before public admission, and leaves -`OWNER_MQTT_USERNAME_MAP` unchanged. - -The three legacy/null-method grants are deliberately preserved, not silently -dropped or promoted. Ben must choose promote-to-`operator-database` or revoke; -until then the recorded disposition is -`preserve-unmodified-pending-owner-review`. This outstanding policy choice does -not risk grant loss during a restore. - -### F4 / G4 — packet-path privacy transitions: FIXED (`e7d16d4`) - -Read/derivation SQL now evaluates path privacy from current privacy tables. -Because compatibility flags remain materialized, migration 051 also adds -ingest and node-transition rematerialization triggers plus receiver, source, -and path-prefix transition indexes. Integration tests cover public→private and -private→public both before and after source-packet deletion. - -### F5 / G2 — reproducible compression/capacity state: FIXED (`e7d16d4`) - -Migration 051 enables and schedules compression for all four hypertables: - -- `packets`: compress after 14 days; hard row retention at 30 days -- `packet_paths`: compress after 14 days; **no retention policy** -- `node_status_samples`: compress after 14 days -- `node_neighbor_samples`: compress after 1 day (7-day retention) - -The source registry uses the same values and keeps compression-only targets -separate from deletable targets. `packet_paths` is segmented by network and -ordered by descending time. Health metrics and alerts cover 30-day path row -rate, bytes-per-row/total size, and overdue uncompressed chunks. Fresh tests -assert exact Timescale settings/jobs and the packet-path no-retention invariant. - -### F6 — shared observation identity and post-retention reader: FIXED (`e7d16d4`) - -The classified source CTE generates one UUID used by both packet inserts. -Migration 051 repairs recent natural-key mismatches. Multibyte reconstruction -and observation-ID backfill now source durable rows from `packet_paths`. -Integration proves shared IDs, idempotency, and successful fact reconstruction -after the source packet is deleted. - -### F7 — reachable row-table retention: FIXED (`e7d16d4`, `c79f71e`) - -The closed worker allowlist, lifecycle declarations, Compose/default env wiring, -and tests now include `observer_registration_requests` at 365 days with an -explicit terminal-only predicate (`rejected`, `expired`, `provisioned`) and -`operator_audit_events` at 730 days. Migration 051 adds bounded-delete indexes. - -### F8 — indexed deterministic decryption retention: FIXED (`e7d16d4`) - -Migration 051 adds `packet_decryptions_created_at_idx`. The bounded worker -orders by timestamp, `tableoid`, and `ctid`, and deletes the exact selected -physical rows. Integration covers expired/retained rows and verifies the index -appears in the execution plan. - -### F9 — full fresh-DB/CI assertions: FIXED (`e7d16d4`) - -Fresh integration now checks all hypertables and chunk intervals, compression -and retention jobs, packet-path constraints/indexes/no-retention, materialized -views/indexes/concurrent refresh, every checksummed ledger row, rollback on -ledger failure, and second-run idempotency. Ingest coverage uses a non-null -two-byte path and asserts observation identity, nullable observer idempotency, -privacy transitions, packet-retention survival, and path-fact rebuilding. The -Compose MQTT fixture/readback now includes `packet_paths`. - -The audited request said 48 ledger entries; the correct current assertion is -49 because migration 051 was added and all previous 48 remain asserted. - -### F10 — migration transaction ownership: FIXED (`e7d16d4`) - -Migration 050 no longer contains file-level `BEGIN`/`COMMIT`. The runner owns -the transaction. An injected failure at the migration-050 ledger insert proves -its schema and policy changes roll back before a normal resume. - -The approved original migration-050 checksum is handled as one explicit shipped -content revision; arbitrary drift still fails closed. - -### F11 — hard 30-day packet bound: FIXED (`e7d16d4`) - -Packet chunks are one day and the closed health-worker target performs bounded -exact-row cleanup at 30 days. The integration test asserts the oldest retained -packet lies inside the boundary. Privacy/lifecycle documentation now describes -the hard bound rather than a 30–37-day chunk-bounded approximation. - -### F12 — telemetry-off coverage: FIXED (`e7d16d4`) - -Tests cover no `window`, `?telemetry=off`, non-off values, suppressed `fetch`, -and suppressed error/rejection/console handler installation. No production -frontend source was changed. - -## Required pre-reset gates - -1. Identity materialized-view migration: **PASS**. -2. Current post-split backup and independent restore: **PASS locally**; - independent off-host publication: **BLOCKED — mount required**. -3. Preserve every owner grant: **PASS**; all 44 are in both `owner-auth.dump` - and the protected baseline. Legacy disposition awaits Ben's policy decision. -4. Packet-path privacy rematerialization/current read derivation: **PASS**. -5. Migration-defined compression/capacity state plus empty-volume assertions on - TimescaleDB 2.25.1: **PASS**. - -Do not reset until gate 2's off-host sub-gate passes. - -## Acceptance evidence - -- Backend: **314/314 PASS**, TypeScript typecheck PASS, OpenAPI contract PASS - (62 API and 11 operator routes). -- Frontend: **96/96 PASS**, production build PASS; telemetry cases included. -- Fresh pinned TimescaleDB integration: **2/2 PASS**, no skips. -- App live-overlay Compose config: PASS. -- `meshcore-infra` Compose config: PASS. -- Prometheus rules: **24 rules PASS** under `promtool`. -- Backup receipt signature and 1.196 GB archive SHA-256: PASS. -- Exact canonical artifact restore: PASS; migration/readiness/integrity/owner - lookup checks all `passed`. -- Second migration run: no-op; ledger count 49 with no null checksums. -- Throwaway containers, networks, volumes, images, decrypted data, and local - sync test copy: removed. -- `.audit-input`: deleted. - -## Ben decisions and remaining operation - -1. Mount the true off-host target at `/mnt/offsite/meshcore`, create its reviewed - `.meshcore-offsite-target` attestation, copy the public receipt verification - key there or into the reviewed key location, then run `sync-latest.sh` without - `BACKUP_SYNC_ALLOW_LOCAL_TEST`. Verify `latest.sha256` independently. -2. Move/recoverably escrow the locally generated drill keys currently protected - under `/home/ben/ukmesh/backup-keys`; do not rely on the same VPS as the sole - key store. -3. Decide whether the three preserved legacy/null-method grants should be - promoted to `operator-database` or revoked. The reset path preserves them in - either case until an explicit decision. -4. Compression is resolved at 14 days for packets, packet paths, and status - samples, and 1 day for neighbor samples. Packet retention is an exact hard - 30-day bound. Change these only through a reviewed migration and matching - lifecycle registry update. - -No crontab was installed. If Ben approves `/mnt/offsite/meshcore` as the actual -mount and the current key locations, these are the exact proposed lines: - -```cron -17 2 * * * /usr/bin/flock -n /home/ben/ukmesh/backups/.backup.lock /usr/bin/env BACKUP_OUTPUT_DIR=/home/ben/ukmesh/backups BACKUP_ALLOW_LOCAL_STAGING=true BACKUP_ENCRYPTION_CERT=/home/ben/ukmesh/backup-keys/encrypt.pem BACKUP_RECEIPT_SIGNING_KEY=/home/ben/ukmesh/backup-keys/receipt-signing.pem MESHCORE_INFRA_DIR=/home/ben/ukmesh/meshcore-infra MESHCORE_INFRA_PROJECT_NAME=meshcore-infra /home/ben/ukmesh/meshcore-analytics/scripts/backup.sh >>/home/ben/ukmesh/backups/backup.log 2>&1 -47 3 * * * /usr/bin/flock -n /home/ben/ukmesh/backups/.sync.lock /usr/bin/env BACKUP_SOURCE_DIR=/home/ben/ukmesh/backups BACKUP_SYNC_TARGET_DIR=/mnt/offsite/meshcore BACKUP_RECEIPT_VERIFY_KEY=/home/ben/ukmesh/backup-keys/receipt-verify.pem /home/ben/ukmesh/meshcore-analytics/scripts/sync-latest.sh >>/home/ben/ukmesh/backups/sync.log 2>&1 -``` - -Before installing them, Ben should also approve backup-generation pruning; -neither script deletes historical recovery points by design. diff --git a/FEED-HISTORY-2026-08-06-bot.png b/FEED-HISTORY-2026-08-06-bot.png deleted file mode 100644 index 1b57427..0000000 Binary files a/FEED-HISTORY-2026-08-06-bot.png and /dev/null differ diff --git a/README.md b/README.md index 17daa2d..625606c 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ A real-time analytics platform for [MeshCore](https://meshcore.io) networks. It - Regional health scoring and predicted-versus-observed RF validation - Local saved searches/watchlists for nodes, observers, regions, packet types, and incidents - Privacy-filtered CSV/GeoJSON exports with a versioned OpenAPI contract -- Beta path prediction model with concurrent worker pool and hourly path-learning prior rebuilds +- Shared Viterbi path resolver with multibyte evidence, a concurrent worker pool, hourly prior rebuilds and [repeatable accuracy checks](docs/pathing.md) - Multibyte path-hash support (1-byte, 2-byte, 3-byte) throughout the live ingest and pathing stack - Decoded live packet feed (Advert, GroupText, DM, ACK, Path, Trace) - Stats pages and chart endpoints for packet rates, radios, hops, and activity @@ -22,6 +22,7 @@ A real-time analytics platform for [MeshCore](https://meshcore.io) networks. It - UK site Feed page for public MQTT observer traffic visibility - Repeater owner portal with MQTT username/password login and encrypted cookie session - Owner dashboard with repeater summary, direct sender map, live packets, advert trend, heard-by list, link health, and alerts +- Opt-in owner public MQTT feed with CoreScope-compatible MeshCore packet output - Multi-network ingestion (`meshcore/*` and `ukmesh/*`) with per-site filtering - Isolated test-feed support via `meshcore-test/*` and `test.ukmesh.com` - Multi-observer deduplication by packet hash @@ -164,20 +165,28 @@ Copy `.env.example` to `.env` and fill in your values. All variables used by the | `MQTT_BROKER_URL` | `ws://mosquitto:9001` | Mosquitto WebSocket URL (internal) | | `MQTT_USERNAME` | `backend` | MQTT client username | | `MQTT_PASSWORD` | *(required)* | MQTT client password | +| `MESHCORE_SITE_ID` | `ukmesh` | Stable identifier added to site-to-site packet relay metadata | | `REDIS_PASSWORD` | *(required)* | Password for the bundled Redis service; passed separately from the URL so reserved characters are safe | | `REDIS_URL` | `redis://redis:6379` | Redis URL for WebSocket pub/sub | | `JWT_SECRET` | *(required)* | Secret for JWT verification | | `ALLOWED_ORIGINS` | `http://localhost:3001,http://localhost:3002` | Comma-separated browser origins allowed for CORS and WebSocket | | `API_RATE_LIMIT_MAX` | `120` | Per-client public API requests/minute; raise only in an isolated load-test project | | `VITE_APP_HOSTNAME` | *(blank — always shows dashboard)* | If set, only this hostname serves the analytics dashboard; all others serve the public website layout | -| `MESHCORE_CHANNEL_SECRETS` | *(blank)* | Confidential channel keys for decrypting GroupText packets, managed through environment configuration. Format: `name:hex` or bare hex. The explicitly public `PUBLIC_CHANNELS` defaults are always included. | +| `MESHCORE_CHANNEL_SECRETS` | *(blank)* | Comma-separated channel secrets for decrypting GroupText packets. Format: `name:hex` or bare hex. The default MeshCore public channel key is always included. | | `OWNER_DATABASE_URL` | *(optional)* | Separate Postgres database URL for owner portal username → repeater mappings | | `OWNER_COOKIE_SECRET` | *(optional but recommended)* | Secret used to encrypt/sign the owner session cookie | | `OWNER_MQTT_USERNAME_MAP` | *(empty)* | Operator-managed owner grants in the format `user=nodeId1|nodeId2,...` | +| `OWNER_PACKET_SHARE_ENCRYPTION_KEY` | *(empty)* | Key used to encrypt destination MQTT passwords; required when a destination uses authentication | +| `OWNER_PACKET_SHARE_POLL_INTERVAL_MS` | `5000` | Packet-sharing delivery queue poll interval | +| `OWNER_PACKET_SHARE_BATCH_SIZE` | `25` | Maximum packet-sharing deliveries claimed per poll | +| `PUBLIC_FEED_PUBLISH_ENABLED` | `false` | Operator kill switch for the authentication-locked UKmesh MQTT feed; owner switches remain off by default | +| `PUBLIC_FEED_BROKER_URL` | *(empty)* | Deployment-only MQTT/MQTTS/WS/WSS endpoint shown in the owner dashboard; packets remain on the UKmesh broker and no separate publisher credential is used | | `OWNER_AUTHORIZATION_MODE` | `shadow` | `shadow` preserves read-only legacy ACL compatibility; `enforce` accepts verified database/config grants only | | `OWNER_ACL_MODE` | `shadow` | `shadow` renders and validates without changing Mosquitto; `apply` atomically installs and verifies the canonical ACL | | `OWNER_ACL_UNMANAGED_USERS` | `backend,test,test2` | Exact broker accounts intentionally preserved outside owner grant management | | `OWNER_ACL_ALLOW_EMPTY_USERS` | *(empty)* | Explicitly reviewed owner accounts allowed to render with no publish grants | +| `OWNER_ACL_PUBLIC_FEED_READ_USERS` | *(empty)* | Operator-approved read-only MQTT subscribers; they receive the curated public feed only | +| `OWNER_ACL_SYSTEM_READWRITE_TOPICS` | `public/#` | Broker topics the internal backend account may publish while rendering the public feed | | `RF_COVERAGE_ENABLED` | `true` | Compile the native HopReach layer into the production app image | | `HOPREACH_IMAGE` | `meshcore-analytics-hopreach:local` | Immutable HopReach calculator image override | | `HOPREACH_CPUS` | `4.0` | Calculator CPU limit | @@ -206,6 +215,146 @@ disables all three layers. --- +## Owner Packet Sharing + +Packet sharing is available to every authenticated UKMesh owner. The owner +dashboard can enable sharing for an owned observer and select only destination +rows that an operator has already provisioned in +`owner_packet_share_destinations`. Toggle and destination changes save +immediately. Each destination can also carry an HTTPS `website_url` and short +`description`; the owner UI renders those as a linked destination name with +hover text, so adding another site only requires another operator-managed +destination row. + +Availability does not enable forwarding. New owner/node selections start with +every destination off, and a destination is forwarded only after that owner +explicitly enables it. + +Destinations may set `allowed_owner_username` and `allowed_node_id` to restrict +visibility and forwarding to an exact account and observer. These restrictions +apply to API reads, saves, queue insertion and delivery. NULL scopes preserve +destinations available to all owners. + +ScotMesh is restricted to **node1**, uses IATA **MME**, and connects over TLS +WebSockets to `scotmesh-mqtt.mm7roq.compute.oarc.uk:443` with the same JWT +audience. Provisioning creates no enabled sharing rule. Its +`tx_mode = 'advert'` matches `set mqtt.tx advert`: received packets are included, +while transmitted packets must be the observer's own adverts. + +Prepare an encrypted relay identity for a verified node1 observer with the +deployment's `DATABASE_URL`, owner database settings and +`OWNER_PACKET_SHARE_ENCRYPTION_KEY` in the environment: + +```bash +npm --prefix backend run owner-packet-share:prepare-scotmesh -- \ + ACTIVE_OBSERVER_PUBLIC_KEY .private-config/scotmesh-node1.json +``` + +The command creates a new MeshCore relay signing key, stores only its encrypted +form in a file readable by its owner, and refuses to overwrite an existing +file. It does not change the radio's key or publish any packets. Apply migration +058 and deploy the owner-scoped backend **before** provisioning its destination +row; the previous backend cannot hide restricted destinations. Then run: + +```bash +npm --prefix backend run owner-packet-share:provision-scotmesh -- \ + .private-config/scotmesh-node1.json +``` + +The provisioning command validates the encrypted key, checks the verified +node1 grant again, and imports the complete configuration without enabling +forwarding. The schema +migration deliberately does not insert the restricted destination while the +previous backend may still be running. For `auth_mode = 'meshcore_jwt'`, +`broker_password_ciphertext` contains the encrypted 64-byte MeshCore signing +key, rather than a static broker password. The relay signs a fresh one-hour +token on each connection and renews it before expiry. ScotMesh's MQTT topic +and `origin_id` use the generated relay public key, while the original raw +radio packet and observer name are preserved. Turn the ScotMesh switch on in +node1's dashboard after provisioning. + +The relay uses outbound MQTT. `mqtts://` is the expected production transport; +`wss://`, `mqtt://`, and `ws://` are accepted for controlled testing. The +dashboard browser WebSocket remains a separate live-display transport. + +Destination rows are deliberately not writable through the owner API. Broker +passwords are stored as AES-GCM ciphertext. To prepare a password for an +operator SQL insert after `OWNER_PACKET_SHARE_ENCRYPTION_KEY` has been set: + +```bash +printf '%s' 'destination-password' \ + | OWNER_PACKET_SHARE_ENCRYPTION_KEY='deployment-secret' \ + npm --prefix backend run owner-packet-share:encrypt-secret +``` + +Store the resulting ciphertext in `broker_password_ciphertext`, then insert an +enabled destination with its fixed broker URL and topic template. Supported +template values are `{network}`, `{iata}`, `{observerId}`, and `{packetHash}`. +The owner toggle remains unavailable while no complete destination exists. + +For a broker that permits an anonymous publisher, the operator provisioning +shape is: + +```sql +INSERT INTO owner_packet_share_destinations ( + destination_id, display_name, website_url, description, + broker_url, topic_template, enabled +) VALUES ( + 'remote-site', 'Remote site', 'https://destination.example', + 'Short destination description', 'mqtts://broker.example:8883', + 'replicated/{network}/{observerId}/packets', TRUE +); +``` + +## Owner Public Feed + +The owner dashboard also has a separate account-level `Public feed` switch. +It is off by default and applies to every node currently owned by that account. +New or revoked owner-node grants are reconciled from the verified owner-auth +database automatically. Disabling the switch removes its read ACL immediately. +An account can enable the +switch only after at least one eligible public packet has been observed from +one of its nodes in the previous 24 hours. + +The feed is served by the authentication-locked UKmesh MQTT broker at the +deployment-configured endpoint. The backend republishes only eligible packets +from opted-in active owners into the curated CoreScope-compatible namespace +`public///packets` with the normal MeshCore JSON +payload. The canonical ingest namespace remains private to its owners and the +backend. + +An owner who has opted in and remains active receives one read-only ACL filter +for `public/+/+/packets`, so the owner can use the existing MQTT username and +password used to publish observer packets. A named subscriber configured in +`OWNER_ACL_PUBLIC_FEED_READ_USERS` receives the same read-only filter without +publish permission. The read grant is removed when the owner disables the +switch or has no eligible packet activity for 24 hours. The dashboard lists +the curated filter and explains that packets are observable over the internet +and that the site operator cannot verify what external subscribers do with the +data. +The public endpoint is deployment configuration and is not stored with +credentials in source control. + +For an ACL-based broker, the intended public-feed permission shape is: + +```text +user +topic read public/+/+/packets +``` + +Only the internal backend account receives `topic readwrite public/#` so it can +republish approved packets. The generated owner ACL applies directly to the +Mosquitto broker managed by this deployment. + +External subscriber accounts remain operator-managed and read-only. The +backend never receives their passwords and cannot inspect what a subscriber +does after receiving a packet. + +Authenticated destinations also set `broker_username` and the encrypted +`broker_password_ciphertext` value from the command above. + +--- + ## Mosquitto Setup Mosquitto is configured for WebSocket-only access with password authentication. @@ -350,19 +499,13 @@ MeshCore Devices ## Data Retention -- The reviewed core telemetry windows are 30 days for raw packets, 180 days - for status samples, and seven days for neighbour samples. Content-stripped - `packet_paths` remain available for path reconstruction and are never - retention-deleted. Privacy-safe aggregates and current node/link/model state - remain longer lived. -- Migration 050 declares the core Timescale retention policies. Repairing a - missing policy and enabling compression require a fresh signed - backup/restore receipt and per-table approval. When lifecycle actions are - enabled, startup fails closed if required core targets are missing from the - configured allowlists. -- Expired forensic data is available only by restoring a suitable pre-expiry - backup into an isolated, read-only environment; do not restore it over - production. +- Compression and destructive retention ship disabled. Raw packets and status + samples remain intact until a table-specific, backup- and restore-gated + lifecycle rollout is approved. +- The proposed raw retention window is 180 days, preserving the longest + 120-day learner dependency. Privacy-safe hourly/daily aggregates and current + node/link/model state remain longer lived. Legacy `node_coverage` is retained + for one release as inactive rollback data only. - Operational row-table cleanup is bounded and runs only when both `DATA_LIFECYCLE_RETENTION_ENABLED=true` and the exact table appears in `DATA_LIFECYCLE_RETENTION_TARGETS`. diff --git a/SEO-ROLLBACK-2026-08-08.txt b/SEO-ROLLBACK-2026-08-08.txt deleted file mode 100644 index 73828a2..0000000 --- a/SEO-ROLLBACK-2026-08-08.txt +++ /dev/null @@ -1,20 +0,0 @@ -=== UK Mesh deploy rollback pins — 2026-08-08 (full history) === -Restore: sed -i 's|^=.*|=|' .env && docker compose -f docker-compose.yml -f docker-compose.live.yml up -d --no-deps - -WEBSITE_IMAGE (website-ukmesh): - PRE-SEO (original live) sha256:148e80fbc02c24cc30981d5e57af4dff108ba409cdd93fd2cf3b1e4e0cecd585 [retagged ukmesh-website:live-orig] - SEO delta (00:15) sha256:5843f3aff6382be0c8f1105f967321cadfa1c9989906360e610498f966ab16bb - HEAD release (01:0x) sha256:0e37443ee557a66689de0fb64fc9541eb7ef7264b06bb98e9cdf80b22cd032b3 [CURRENT] - -BACKEND_IMAGE (backend): - PRE (original) meshcore-analytics-backend:dc12587f672d0f6edd8ea88246b7cbae0ceac9a306e56e2648610a287a0286ef (digest 87074c1ebafe) - HEAD release (00:5x) sha256:1468726cfbd446b52e44b2b5b0c943342a6c138514e4ade54f06cb6cb5498aa5 [CURRENT] - -WEBSITE_DEV_IMAGE (website-dev): - PRE sha256:1483797133b859019f93b58d6c2d42b35fcf80d708f56da3b668938fbbf21b11 - CURRENT (HEAD build) sha256:0e37443ee557a66689de0fb64fc9541eb7ef7264b06bb98e9cdf80b22cd032b3 - -Config changes today: - .env ALLOWED_ORIGINS += https://test.ukmesh.com (backup .env.bak-allowed-origins-*) - docker-compose.yml backend DATABASE_STATEMENT_TIMEOUT_MS 300000 -> 900000 (charts regen) - .env STATS_AGGREGATE_READS_ENABLED=true (was already true) diff --git a/SPRING-DASHBOARD-2026-08-06.png b/SPRING-DASHBOARD-2026-08-06.png deleted file mode 100644 index 404cc7f..0000000 Binary files a/SPRING-DASHBOARD-2026-08-06.png and /dev/null differ diff --git a/SPRING-INSTALL-AUDIT-2026-08-06.png b/SPRING-INSTALL-AUDIT-2026-08-06.png deleted file mode 100644 index 5c9adbd..0000000 Binary files a/SPRING-INSTALL-AUDIT-2026-08-06.png and /dev/null differ diff --git a/SPRING-MAPFEED-2026-08-06.png b/SPRING-MAPFEED-2026-08-06.png deleted file mode 100644 index f2d31b7..0000000 Binary files a/SPRING-MAPFEED-2026-08-06.png and /dev/null differ diff --git a/SPRING-PAGES-2026-08-06.png b/SPRING-PAGES-2026-08-06.png deleted file mode 100644 index 6f7068f..0000000 Binary files a/SPRING-PAGES-2026-08-06.png and /dev/null differ diff --git a/SPRING-TOPOLOGY-2026-08-06.png b/SPRING-TOPOLOGY-2026-08-06.png deleted file mode 100644 index 6cee94e..0000000 Binary files a/SPRING-TOPOLOGY-2026-08-06.png and /dev/null differ diff --git a/TERRAIN-V6-REPORT.md b/TERRAIN-V6-REPORT.md deleted file mode 100644 index 98e30b2..0000000 --- a/TERRAIN-V6-REPORT.md +++ /dev/null @@ -1,88 +0,0 @@ -# MapLibre 6 terrain regression report - -Date: 2026-08-13 UTC -Status: fixed and deployed to `app-ukmesh`, `website-ukmesh`, and `website-dev` - -## Root cause and fix - -MapLibre GL 6's terrain tile-pyramid selection produced a single raised DEM square when the local `raster-dem` source explicitly declared `minzoom: 5`. Upgrading MapLibre from 6.2.0 to 6.3.0 and correcting `tileSize` from 512 to the tiles' actual 256 pixels were necessary source corrections, but did not resolve the rendering defect by themselves. - -The decisive bare-MapLibre test rendered continuous multi-tile terrain with this source specification: - -- MapLibre GL 6.3.0 -- `tileSize: 256` -- `encoding: "terrarium"` -- `maxzoom: 12` -- no explicit `minzoom` -- terrain exaggeration 3 - -Removing only `minzoom: 5` from the application source fixed the full application while retaining hillshade, sky/atmosphere, and the deck.gl overlays. MapLibre can now select the required parent/child DEM pyramid levels itself. - -## Code changes - -Commit `18aed54ba0b4fd451c26ae6dc8fe687e07a51f1a` (`fix(map): restore multi-tile terrain on MapLibre 6`): - -- upgraded `maplibre-gl` from 6.2.0 to 6.3.0 in `frontend/package.json` and `frontend/package-lock.json`; -- changed `TERRAIN_DEM_SOURCE.tileSize` from 512 to 256; -- changed the two DEM source error-handler parameter types to `maplibregl.ErrorEvent`, preserving their behavior. - -Commit `d87b3d39c1a3f6a9ca742062515a2d316581ff3e` (`fix(map): let MapLibre select terrain DEM pyramid`): - -- removed the explicit `TERRAIN_DEM_SOURCE.minzoom` value. - -No backend code, CSS, hillshade behavior, sky behavior, or deck.gl behavior changed in the production fix. - -## Hypothesis protocol - -1. H1, shared hillshade source: removing both hillshade layer paths still failed to show continuous relief. Eliminated. -2. H2, sky/atmosphere: additionally removing `map.setSky(...)` still failed. Eliminated. -3. H3, deck.gl `MapboxOverlay`: additionally omitting both live overlay controllers still failed to provide the required continuous visual result. The diagnostic candidate was not a fixed first-loaded tile: center/candidate changed from `11/1017/652` / `11/1016/651`, to `11/1017/652` / `11/1017/651` after panning, and to `12/2035/1304` / `12/2035/1304` after zooming. Without hillshade the visual delta was weak, so this was not accepted as a fix. -4. H4, bare MapLibre 6.3: the first protocol variant, `tileSize=256`, absent `minzoom`, Terrarium encoding, exaggeration 3, rendered multi-tile terrain. Per the stop-on-success protocol, later variants were not run. - -The H4 camera-center tiles changed from `10/508/326` to `10/509/325` on pan and `11/1018/651` on zoom. Terrain-related pixel changes covered 16 distinct z10 tiles and non-null `queryTerrainElevation` samples covered 19 distinct z10 tiles. Therefore the rendered relief tracked the visible pyramid across many tiles; it was neither a center-tile-only bug nor a fixed first-loaded/cache-order tile. - -## Frontend gates - -Run from `frontend/` against the final source revision: - -- `npx tsc --noEmit`: passed cleanly. -- `npm test`: passed, 94/94 tests. -- `npm run build`: passed. -- CSS lint: the repository's known pre-existing duplicate-selector failures remain in untouched CSS (`owner-portal.css` and sibling files). `git diff --name-only` confirmed those files were not part of this change; no unrelated CSS was modified. - -## Images and deployment - -All images contain MapLibre 6.3.0 in `assets/maplibre-gl-OTXnCXpW.js` and carry source revision `d87b3d39c1a3f6a9ca742062515a2d316581ff3e`. - -| Service | Build tag | Pinned image digest | -| --- | --- | --- | -| `app-ukmesh` | `meshcore-analytics-app:v6fix-final` | `sha256:620db704760dbe91f471ce984c9805ec2257a13aa4097bb634beffbe07ccc386` | -| `website-ukmesh` | `meshcore-analytics-website:v6fix-final` | `sha256:2779d7c5da0062e8a2a5d4e2aea10ed66faaf0e492646a677a0eb30fa4a92b18` | -| `website-dev` | `meshcore-analytics-website-dev:v6fix-final` | `sha256:3674270ba54601e33ea693e95235791929f0bfd34b791cdbd016e8904a6c79e3` | - -The app was recreated with `up -d --no-deps app-ukmesh`. Both website services were deployed with `scripts/deploy-website.sh --force`; their drift checks, digest pins, recreation, and served-bundle verification passed. No other service was recreated. - -The refreshed manifests were captured at 2026-08-13T00:06:18Z. Their follow-up drift checks report 90/90 identical files for production and 76/76 identical files for staging. - -## Runtime proof - -The final diagnostic against `http://localhost:3003` recorded 15/15 terrain tile responses as HTTP 200, all `image/png`, across zoom levels 5, 9, 10, and 11. It recorded no failed responses, decode errors, or MapLibre terrain errors. The only browser noise was the expected local WebSocket 403 response. - -The final Teesside capture was taken three seconds after enabling terrain with RF Coverage disabled. It shows continuous relief across multiple adjacent terrain tiles without the former isolated hard-edged square. - -Public gates: - -- `https://ukmesh.com`: HTTP 200, `text/html` -- `https://test.ukmesh.com`: HTTP 200, `text/html` -- `https://ukmesh.com/terrain-tiles/5/15/9.png`: HTTP 200, `image/png` -- `https://test.ukmesh.com/terrain-tiles/5/15/9.png`: HTTP 200, `image/png` - -## Evidence - -- Decisive deployed Teesside screenshot: `/home/ben/ukmesh/meshcore-analytics/terrain-v6-evidence/terrain-v6-teesside-final.png` -- Final standard diagnostic screenshot: `/home/ben/ukmesh/meshcore-analytics/terrain-v6-evidence/terrain-diag2.png` -- Successful H4 screenshot: `/home/ben/ukmesh/meshcore-analytics/terrain-v6-evidence/h4-ts256-minabs-terr-e3-after.png` -- H4 measurements: `/home/ben/ukmesh/meshcore-analytics/terrain-v6-evidence/h4-ts256-minabs-terr-e3.json` -- H3 center/pan/zoom captures: `/home/ben/ukmesh/meshcore-analytics/terrain-v6-evidence/terrain-h3-center.png`, `terrain-h3-pan.png`, and `terrain-h3-zoom.png` - -All commits are local only; nothing was pushed. diff --git a/anubis/botPolicy.yaml b/anubis/botPolicy.yaml index 48688c6..dd1059d 100644 --- a/anubis/botPolicy.yaml +++ b/anubis/botPolicy.yaml @@ -21,6 +21,34 @@ bots: path_regex: ^/terrain-tiles/ action: ALLOW + # ── Link-preview bots: allow so shared links render embeds ──────────────── + # Discordbot/LinkedInBot UAs contain "Mozilla", which otherwise trips the + # generic-browser WEIGH rule (+10 -> challenge they cannot solve). + # These agents only fetch public pages for unfurling; UA-only ALLOW is fine. + # NOTE: AI-search carve-out (OAI-SearchBot etc.) intentionally not enabled; + # see FIX-PACK section 2.2 in ~/.review-notes/nemesh-seo-2026-10-11/ to enable. + - name: allow-link-preview-bots + user_agent_regex: >- + (?i)Discordbot|LinkedInBot|Slackbot|Twitterbot|TelegramBot|WhatsApp|facebookexternalhit|Facebot|Pinterestbot|redditbot + action: ALLOW + + # ── IndexNow key file: always allow ─────────────────────────────────────── + # Search engines fetch /assets/<32-hex>.txt from arbitrary IPs/UAs to verify + # site ownership for IndexNow submissions; those fetchers carry "Mozilla" + # in the UA and would otherwise be challenged (returning HTML instead of + # the key, which fails verification with a 422). Static file, exact pattern. + - name: allow-indexnow-key + path_regex: ^/(assets/)?[0-9a-f]{32}\.txt$ + action: ALLOW + + # ── Google tooling: GSC URL Inspection, Rich Results test, PageSpeed Insights ── + # These fetchers run from Google infra but not necessarily from the published + # googlebot IP ranges, so the UA+IP-gated googlebot allow rule can miss them. + - name: allow-google-tools + user_agent_regex: >- + (?i)Google-InspectionTool|Chrome-Lighthouse|Google-PageSpeedInsights|Google-Structured-Data-Testing-Tool|Google-Site-Verification + action: ALLOW + # ── Built-in deny rules for pathological bots ────────────────────────────── - import: (data)/bots/_deny-pathological.yaml - import: (data)/bots/aggressive-brazilian-scrapers.yaml diff --git a/backend/backfill-region-names.mjs b/backend/backfill-region-names.mjs new file mode 100644 index 0000000..cd1b812 --- /dev/null +++ b/backend/backfill-region-names.mjs @@ -0,0 +1,222 @@ +/** + * One-shot backfill: re-probe the REGION NAME for packets that already have + * transport_codes populated but region_scope NULL — i.e. history that predates + * a MESHCORE_REGION_NAMES extension. + * + * Companion to backfill-transport-codes.mjs: that script only processes rows + * where transport_codes IS NULL (all already filled), so extending the region + * name list needs this re-probe pass instead. + * + * Run from backend/: + * node backfill-region-names.mjs + * + * Env vars used (same as backend): + * DATABASE_URL, MESHCORE_CHANNEL_SECRETS, MESHCORE_REGION_NAMES + * + * Safe to re-run: only touches rows with region_scope IS NULL; decode is + * deterministic; window matches the original backfill (31 days). + */ + +import pg from 'pg'; +import { createRequire } from 'module'; +const require = createRequire(import.meta.url); + +const { + MeshCoreDecoder, + transportCodeMatchesRegion, +} = require('@michaelhart/meshcore-decoder'); + +const { Pool } = pg; + +// ── Config ────────────────────────────────────────────────────────────────── + +const DATABASE_URL = process.env.DATABASE_URL; +if (!DATABASE_URL) { + console.error('DATABASE_URL env var is required'); + process.exit(1); +} + +// This is the documented MeshCore public-channel material, not an application +// credential. Private/custom channel material must only enter through the +// environment and must never be committed. +const DEFAULT_PUBLIC_CHANNEL_KEY = '8b3387e9c5cdea6ac9e5edbaa115cd72'; +const CHANNEL_SECRETS = [DEFAULT_PUBLIC_CHANNEL_KEY]; +if (process.env.MESHCORE_CHANNEL_SECRETS) { + for (const entry of process.env.MESHCORE_CHANNEL_SECRETS.split(',').map((s) => s.trim()).filter(Boolean)) { + const colon = entry.indexOf(':'); + CHANNEL_SECRETS.push(colon > 0 ? entry.slice(colon + 1) : entry); + } +} + +const keyStore = MeshCoreDecoder.createKeyStore({ channelSecrets: CHANNEL_SECRETS }); + +const REGION_NAMES = (process.env.MESHCORE_REGION_NAMES ?? 'Europe,Global') + .split(',').map((s) => s.trim()).filter(Boolean) + .map((name) => (name.startsWith('#') ? name : `#${name}`)); + +console.log(`[backfill-names] region probe list: ${REGION_NAMES.join(', ')}`); + +// ── Decoder helpers (mirrors backfill-transport-codes.mjs) ────────────────── + +function hexToBytes(rawHex) { + const hex = rawHex.trim(); + if (!hex || hex.length % 2 !== 0) return null; + return Buffer.from(hex, 'hex'); +} + +function bytesToHex(buf) { + return buf.toString('hex').toUpperCase(); +} + +function parseTransportCodes(rawHex) { + const bytes = hexToBytes(rawHex); + if (!bytes || bytes.length < 2) return null; + + const routeType = bytes[0] & 0x03; + if (routeType !== 0 && routeType !== 3) return null; + + if (bytes.length < 5) return null; // 1 route byte + 4 transport code bytes + const tcHex = bytesToHex(bytes.subarray(1, 5)); + + // Decode with compat wrapper to get payload.raw for region probing + // Build compat raw hex: rewrite path length byte to the old 1-byte encoding + const encodedLengthByte = bytes[5]; + if (encodedLengthByte === undefined) return { tcHex, payloadRaw: null, payloadType: null }; + + const pathHashCount = encodedLengthByte & 0x3f; + const pathHashSize = (encodedLengthByte >> 6) + 1; + if (pathHashSize > 3) return { tcHex, payloadRaw: null, payloadType: null }; + + const pathByteLength = pathHashCount * pathHashSize; + if (pathByteLength > 63) return { tcHex, payloadRaw: null, payloadType: null }; + + // Build compat hex for the library decoder (replaces encoded length byte with plain byte count) + const compat = Buffer.from(bytes); + compat[5] = pathByteLength; + const compatHex = bytesToHex(compat); + + let decoded; + try { + decoded = MeshCoreDecoder.decode(compatHex, { keyStore }); + } catch { + return { tcHex, payloadRaw: null, payloadType: null }; + } + + return { + tcHex, + payloadRaw: decoded?.payload?.raw ?? null, + payloadType: decoded?.payloadType ?? null, + }; +} + +function probeRegion(tcHex, payloadType, payloadRaw) { + if (!tcHex || payloadType == null || !payloadRaw) return null; + const tcBuf = Buffer.from(tcHex, 'hex'); + if (tcBuf.length < 2) return null; + const tc0 = tcBuf[0] | (tcBuf[1] << 8); + for (const name of REGION_NAMES) { + if (transportCodeMatchesRegion(name, payloadType, payloadRaw, tc0)) { + return name; + } + } + return null; +} + +// ── Main ──────────────────────────────────────────────────────────────────── + +const pool = new Pool({ connectionString: DATABASE_URL }); + +async function run() { + const { rows } = await pool.query( + `SELECT DISTINCT ON (packet_hash) packet_hash, raw_hex + FROM packets + WHERE route_type IN (0, 3) + AND raw_hex IS NOT NULL + AND raw_hex != '' + AND transport_codes IS NOT NULL + AND region_scope IS NULL + AND time > NOW() - INTERVAL '31 days' + ORDER BY packet_hash, time DESC`, + ); + + console.log(`[backfill-names] ${rows.length} distinct packets to re-probe`); + if (rows.length === 0) { await pool.end(); return; } + + let processed = 0; + let matches = 0; + let noCode = 0; + let totalRowsUpdated = 0; + const byName = new Map(); + const BATCH = 200; + + for (let i = 0; i < rows.length; i += BATCH) { + const batch = rows.slice(i, i + BATCH); + const values = []; + + for (const row of batch) { + const result = parseTransportCodes(row.raw_hex); + if (!result) { noCode++; continue; } + + const region = probeRegion(result.tcHex, result.payloadType, result.payloadRaw); + if (!region) continue; + + matches++; + byName.set(region, (byName.get(region) ?? 0) + 1); + values.push({ hash: row.packet_hash, region }); + } + + if (values.length > 0) { + const hashes = values.map((v) => v.hash); + const regions = values.map((v) => v.region); + + const client = await pool.connect(); + try { + await client.query('SET timescaledb.max_tuples_decompressed_per_dml_transaction = 0'); + const r = await client.query( + `UPDATE packets AS p + SET region_scope = v.region + FROM unnest($1::text[], $2::text[]) AS v(hash, region) + WHERE p.packet_hash = v.hash + AND p.route_type IN (0, 3) + AND p.region_scope IS NULL`, + [hashes, regions], + ); + totalRowsUpdated += r.rowCount ?? 0; + } finally { + client.release(); + } + } + + processed += batch.length; + if (processed % 1000 === 0 || processed === rows.length) { + console.log(`[backfill-names] ${processed}/${rows.length} probed — ${matches} matches, ${totalRowsUpdated} rows updated`); + } + } + + console.log(`[backfill-names] done. ${matches} packets matched a region, ${noCode} undecodable.`); + if (byName.size > 0) { + console.log('[backfill-names] matches by name:'); + for (const [name, n] of [...byName.entries()].sort((a, b) => b[1] - a[1])) { + console.log(` ${name}: ${n} packets`); + } + } + + // Summary by region (31-day window, all rows) + const { rows: summary } = await pool.query( + `SELECT region_scope, COUNT(*) AS n + FROM packets + WHERE route_type IN (0, 3) + AND region_scope IS NOT NULL + AND time > NOW() - INTERVAL '31 days' + GROUP BY region_scope + ORDER BY n DESC`, + ); + if (summary.length > 0) { + console.log('[backfill-names] full region breakdown:'); + for (const r of summary) console.log(` ${r.region_scope}: ${r.n}`); + } + + await pool.end(); +} + +run().catch((err) => { console.error('[backfill-names] fatal:', err); process.exit(1); }); diff --git a/backend/package-lock.json b/backend/package-lock.json index f3e83c6..e6a04c8 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -561,10 +561,9 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "26.6.3", - "resolved": "https://registry.npmjs.org/@types/node/-/node-26.6.3.tgz", - "integrity": "sha512-dsqMQQoeTLqu9wynDD00q573mNzso3IdQOAfHRJqLCcmCFPoGo9A1bDpUcv/9tnKpErQWv9uKeGfl37EIS02Yg==", - "license": "MIT", + "version": "26.6.5", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.6.5.tgz", + "integrity": "sha512-uMzsv9stDdofIZBx4UGh4Z9ZgXnGC9seRDscilBcTHAeOFk67MZg/k9bWVfO+bk4qZmG+l52hRnLwUiH2LnJTg==", "dependencies": { "undici-types": "~8.9.0" } @@ -574,7 +573,6 @@ "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.23.1.tgz", "integrity": "sha512-fKVHpikPdg4GKks3JuLEhvwSyvwzF23hnabPy6DD8ljVbC7+6J5dQzdv4arV6jqq57djnMgs1HKBxX4P8aBI3A==", "dev": true, - "license": "MIT", "dependencies": { "@types/node": "*", "pg-protocol": "*", @@ -1272,7 +1270,6 @@ "version": "1.8.2", "resolved": "https://registry.npmjs.org/compression/-/compression-1.8.2.tgz", "integrity": "sha512-o8vI5RE5A6EVVOd9o41jKp41aJom+QTEO/Bx8MYNjexMo/Bv2WOjUfZr+aL0WnYSgymUy6zeguqLTsIhV0gMvQ==", - "license": "MIT", "dependencies": { "bytes": "3.1.2", "compressible": "~2.0.18", @@ -1394,7 +1391,6 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", - "license": "MIT", "engines": { "node": ">= 0.8", "npm": "1.2.8000 || >= 1.4.16" @@ -1575,10 +1571,9 @@ } }, "node_modules/express-rate-limit": { - "version": "8.7.0", - "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-8.7.0.tgz", - "integrity": "sha512-hOwV7WOxXfjRpAM1DSJWZDXx3GhplwD8IfwuwvogD8i1Qnkgosw/H45s4ZnFAUHDAhPjlY9hLBvJhKmGMyY26g==", - "license": "MIT", + "version": "8.7.1", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-8.7.1.tgz", + "integrity": "sha512-2FXbqkygil3mZiXzEtM0IjE/UBmBNot0J1GEVrJT4JbrvwOSoEtqtO+wHbiMQELWe1eNvHoYRg2dkjOid+6JkA==", "dependencies": { "debug": "^4.4.3", "ip-address": "^10.2.0" @@ -1924,9 +1919,9 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, "node_modules/ip-address": { - "version": "10.7.3", - "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.7.3.tgz", - "integrity": "sha512-A1kdq/tSb5QjvKvAMgIoEvDBIgL7qaqVP/jkvSwYYRZ9iEzvPpopxp2wQfu3SuZRHtpHNxMn8Fs0bS+gf5Xmwg==", + "version": "10.4.0", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.4.0.tgz", + "integrity": "sha512-oSK96Grm3aP6OrS263xVxbNDGVL7rzBtYdpGqlDG8iQdoenDoTs/nkki+DflYbAEE8Xl6o5YxhxlrKvI3nqKXQ==", "engines": { "node": ">= 12" } @@ -2020,7 +2015,6 @@ "version": "5.16.0", "resolved": "https://registry.npmjs.org/mqtt/-/mqtt-5.16.0.tgz", "integrity": "sha512-G0VsEtuJz5sJUgZJaaADMVfW7SIFPzkIuHR+d6rk+tBeSwdCfd9BaIRycRfd2AbyFDEmLlFM+SDQeti/+IbOnw==", - "license": "MIT", "dependencies": { "@types/readable-stream": "^4.0.21", "@types/ws": "^8.18.1", @@ -2339,25 +2333,22 @@ } }, "node_modules/proxy-addr": { - "version": "2.0.8", - "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.8.tgz", - "integrity": "sha512-5nnx0yGyVUcY6t9RnWcARWtwT9F1D8O9rt08htPvnd49W1IgZtmLkhu9WfMzQj1cFxjHIO6connUNVW5k7AVyQ==", + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", "dependencies": { "forwarded": "0.2.0", "ipaddr.js": "1.9.1" }, "engines": { "node": ">= 0.10" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" } }, "node_modules/qs": { - "version": "6.16.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.16.0.tgz", - "integrity": "sha512-h6fhOIaRrID2CbEY2fqs+7t+UXZo+MLAnU5gRIq85uFtdiUPCdsApMlHhXogKVM4HM2DVbIjGNTTYH2OcmP1vA==", + "version": "6.15.3", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.3.tgz", + "integrity": "sha512-O9gl3zCl5h5blw1KGUzQKhA5oUXSl8rwUIM5o0S3nCXMliSvy5Dzx7/DJcI+SwgICv+IneSZwhBh1oSyEHA71A==", + "license": "BSD-3-Clause", "dependencies": { "es-define-property": "^1.0.1", "side-channel": "^1.1.1" @@ -2724,7 +2715,6 @@ "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.23.15.tgz", "integrity": "sha512-Yiex1Ovn8z2xPpOWckIiysV1SSyRMY9BkLF++q0yKiDxCqRhosKfMg3janKkiLBwZ5c/YryloKwGZcrEmtwxKw==", "dev": true, - "license": "MIT", "dependencies": { "esbuild": "~0.28.0" }, @@ -2812,8 +2802,7 @@ "node_modules/undici-types": { "version": "8.9.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.9.0.tgz", - "integrity": "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg==", - "license": "MIT" + "integrity": "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg==" }, "node_modules/unpipe": { "version": "1.0.0", @@ -2890,7 +2879,6 @@ "version": "8.22.0", "resolved": "https://registry.npmjs.org/ws/-/ws-8.22.0.tgz", "integrity": "sha512-Ydggc987+RO0AnWtZ/7Wq9FtNvcrL1b/RO0ud9mWjUPgDrsAAwQSF51sm2hm1XofbU/4jkpGEsLFsZZxU+1DOg==", - "license": "MIT", "engines": { "node": ">=10.0.0" }, diff --git a/backend/package.json b/backend/package.json index 215279c..57127ae 100644 --- a/backend/package.json +++ b/backend/package.json @@ -8,6 +8,7 @@ "dev": "tsx watch src/index.ts", "spam:recompute": "node dist/tools/recomputeSpamSuspects.js", "spam:messages": "node dist/tools/recomputeSpamMessages.js", + "regions:rebuild": "node dist/tools/rebuildRegionParticipation.js", "db:migrate": "node dist/tools/migrate.js", "stats:backfill-rollups": "node dist/tools/backfillStatsRollups.js", "stats:backfill-multibyte": "node dist/tools/backfillMultibytePathFacts.js", @@ -16,8 +17,11 @@ "owner-auth:inventory": "node dist/tools/inventoryOwnerAuthorization.js", "owner-auth:revoke": "node dist/tools/revokeOwnerSessions.js", "owner-packet-share:encrypt-secret": "node dist/tools/encryptPacketShareSecret.js", + "owner-packet-share:prepare-scotmesh": "node dist/tools/prepareScotmesh.js", + "owner-packet-share:provision-scotmesh": "node dist/tools/provisionScotmesh.js", "analysis:cleanup-orphans": "node dist/tools/cleanupOrphanedAnalysisRuns.js", "bench:decode": "tsx src/tools/benchmarkPacketDecode.ts", + "path:accuracy": "tsx src/tools/pathAccuracy.ts", "load:realtime": "tsx src/tools/loadRealtime.ts", "test": "node --import tsx --test $(find src -name '*.test.ts' ! -name '*.integration.test.ts' -print) && bash ../scripts/test-alert-receiver.sh", "test:analysis-integration": "node --import tsx --test src/analysis/runState.integration.test.ts", diff --git a/backend/src/api/contracts.test.ts b/backend/src/api/contracts.test.ts index cfc1d18..b7e5a8b 100644 --- a/backend/src/api/contracts.test.ts +++ b/backend/src/api/contracts.test.ts @@ -57,22 +57,3 @@ test('slow path contracts cover status, mode query, and pending response', () => true, ); }); - -test('owner packet-sharing contracts cover bounded owner settings and destination metadata', () => { - const get = API_CONTRACTS.find((entry) => entry.method === 'GET' && entry.path === '/owner/packet-sharing'); - assert.equal(get?.access, 'owner'); - assert.deepEqual(get?.queryParameters?.[0]?.['schema'], { - type: 'string', - pattern: '^[0-9A-Fa-f]{64}$', - }); - assert.deepEqual((get?.responseSchema?.['required'] as string[] | undefined)?.sort(), [ - 'destinations', 'enabled', 'featureEnabled', 'nodeId', - ]); - - const post = API_CONTRACTS.find((entry) => entry.method === 'POST' && entry.path === '/owner/packet-sharing'); - assert.equal(post?.access, 'owner'); - assert.deepEqual((post?.requestSchema?.['required'] as string[] | undefined)?.sort(), [ - 'destinationIds', 'enabled', 'nodeId', - ]); - assert.equal(post?.additionalResponses?.['409']?.responseSchema['properties'] != null, true); -}); diff --git a/backend/src/api/contracts.ts b/backend/src/api/contracts.ts index 4d5c76f..25bea4d 100644 --- a/backend/src/api/contracts.ts +++ b/backend/src/api/contracts.ts @@ -119,6 +119,138 @@ const OWNER_PACKET_SHARING_SCHEMA = { additionalProperties: false, }; +const OWNER_PUBLIC_FEED_SCHEMA = { + type: 'object', + required: [ + 'featureEnabled', + 'configured', + 'brokerUrl', + 'enabled', + 'nodeCount', + 'eligible', + 'recentPacketCount', + 'topicFilters', + 'lastForwardedAt', + ], + properties: { + featureEnabled: { type: 'boolean' }, + configured: { type: 'boolean' }, + brokerUrl: { type: ['string', 'null'], format: 'uri', maxLength: 2_048 }, + enabled: { type: 'boolean' }, + nodeCount: { type: 'integer', minimum: 0 }, + eligible: { type: 'boolean' }, + recentPacketCount: { type: 'integer', minimum: 0 }, + topicFilters: { + type: 'array', + maxItems: 1, + items: { type: 'string', pattern: '^public/\\+/\\+/packets$' }, + }, + lastForwardedAt: { type: ['string', 'null'], format: 'date-time' }, + }, + additionalProperties: false, +}; + +const REGION_MEMBER_SCHEMA = { + type: 'object', + required: ['nodeId', 'name', 'role', 'lat', 'lon', 'packets', 'isBridge'], + properties: { + nodeId: { type: 'string', pattern: '^[0-9A-F]{64}$' }, + name: { type: ['string', 'null'] }, + role: { type: ['integer', 'null'] }, + lat: { type: ['number', 'null'], minimum: -90, maximum: 90 }, + lon: { type: ['number', 'null'], minimum: -180, maximum: 180 }, + packets: { type: 'integer', minimum: 0 }, + isBridge: { type: 'boolean' }, + }, + additionalProperties: false, +}; + +const REGION_SUMMARY_SCHEMA = { + type: 'object', + required: ['generatedAt', 'windowDays', 'regions'], + properties: { + generatedAt: { type: ['string', 'null'], format: 'date-time' }, + windowDays: { type: ['integer', 'null'], minimum: 1, maximum: 365 }, + regions: { + type: 'array', + items: { + type: 'object', + required: ['name', 'packetsTotal', 'memberCount', 'repeaterMemberCount', 'confidence', 'members'], + properties: { + name: { type: 'string', minLength: 1, maxLength: 64 }, + packetsTotal: { type: 'integer', minimum: 0 }, + memberCount: { type: 'integer', minimum: 0 }, + repeaterMemberCount: { type: 'integer', minimum: 0 }, + confidence: { type: 'string', enum: ['confirmed', 'probable', 'single'] }, + firstSeen: { type: ['string', 'null'], format: 'date-time' }, + lastSeen: { type: ['string', 'null'], format: 'date-time' }, + members: { type: 'array', items: REGION_MEMBER_SCHEMA }, + }, + additionalProperties: false, + }, + }, + }, + additionalProperties: false, +}; + +const REGION_REPEATERS_SCHEMA = { + type: 'object', + required: ['generatedAt', 'repeaters'], + properties: { + generatedAt: { type: ['string', 'null'], format: 'date-time' }, + repeaters: { + type: 'array', + items: { + type: 'object', + required: ['nodeId', 'status', 'evidence', 'scopedPackets', 'unscopedPackets7d', 'regions'], + properties: { + nodeId: { type: 'string', pattern: '^[0-9A-F]{64}$' }, + name: { type: ['string', 'null'] }, + role: { type: ['integer', 'null'] }, + lat: { type: ['number', 'null'], minimum: -90, maximum: 90 }, + lon: { type: ['number', 'null'], minimum: -180, maximum: 180 }, + status: { type: 'string', enum: ['green', 'red'] }, + evidence: { type: 'string' }, + scopedPackets: { type: 'integer', minimum: 0 }, + unscopedPackets7d: { type: 'integer', minimum: 0 }, + regions: { type: 'object', additionalProperties: { type: 'integer', minimum: 0 } }, + }, + additionalProperties: false, + }, + }, + }, + additionalProperties: false, +}; + +const REGION_SCOPE_SCHEMA = { + type: 'object', + required: ['nodeId', 'status', 'evidence', 'scopedPackets', 'unscopedPackets7d', 'regions'], + properties: { + nodeId: { type: 'string', pattern: '^[0-9A-F]{64}$' }, + name: { type: ['string', 'null'] }, + role: { type: ['integer', 'null'] }, + lat: { type: ['number', 'null'], minimum: -90, maximum: 90 }, + lon: { type: ['number', 'null'], minimum: -180, maximum: 180 }, + status: { type: 'string', enum: ['green', 'red', 'unobserved'] }, + evidence: { type: 'string' }, + scopedPackets: { type: 'integer', minimum: 0 }, + unscopedPackets7d: { type: 'integer', minimum: 0 }, + regions: { + type: 'array', + items: { + type: 'object', + required: ['region', 'packets'], + properties: { + region: { type: 'string', minLength: 1, maxLength: 64 }, + packets: { type: 'integer', minimum: 0 }, + }, + additionalProperties: false, + }, + }, + }, + additionalProperties: false, +}; + function humanize(path: string): string { return path .replace(/^\/v1\//, '') @@ -172,6 +304,9 @@ const PUBLIC_GET = [ '/planned-nodes', '/radio-history', '/radio-stats', + '/regions/repeaters', + '/regions/summary', + '/repeaters/:id/scope', '/repeaters/firmware', '/rf-validation', '/runtime-config', @@ -183,6 +318,7 @@ const PUBLIC_GET = [ '/spam/suspects', '/stats', '/stats/charts', + '/tags/recent', '/topology', '/v1', '/v1/exports/nodes.:format', @@ -197,6 +333,7 @@ const OWNER_GET = [ '/owner/live', '/owner/live-last-hop', '/owner/packet-sharing', + '/owner/public-feed', '/owner/session', ] as const; @@ -211,6 +348,7 @@ export const API_CONTRACTS: readonly ApiContract[] = [ '/owner/login', '/owner/logout', '/owner/packet-sharing', + '/owner/public-feed', ], 'owner'), ...contracts('DELETE', ['/coverage/planned/:planId'], 'public'), ...contracts('DELETE', ['/owner/alert-rules/:id'], 'owner'), @@ -239,6 +377,33 @@ export const API_CONTRACTS: readonly ApiContract[] = [ }, }; } + if (contract.path === '/regions/summary') { + return { + ...contract, + summary: 'Read per-region scoped-traffic participation summary', + responseSchema: REGION_SUMMARY_SCHEMA, + }; + } + if (contract.path === '/regions/repeaters') { + return { + ...contract, + summary: 'List repeaters classified by region-capability', + responseSchema: REGION_REPEATERS_SCHEMA, + }; + } + if (contract.path === '/repeaters/:id/scope') { + return { + ...contract, + summary: 'Read one repeater\'s region scoping detail', + responseSchema: REGION_SCOPE_SCHEMA, + additionalResponses: { + '404': { + description: 'The repeater is not known to the region-scope tables', + responseSchema: ERROR_SCHEMA, + }, + }, + }; + } if (contract.path === '/planned-nodes') { return { ...contract, summary: 'List explicitly published planned nodes', responseSchema: PLANNED_NODE_PAGE_SCHEMA }; } @@ -294,11 +459,6 @@ export const API_CONTRACTS: readonly ApiContract[] = [ }; } if (contract.path === '/owner/packet-sharing' && contract.method === 'POST') { - const destinationIdsSchema = { - type: 'array', - maxItems: 32, - items: { type: 'string', pattern: '^[a-z0-9][a-z0-9_-]{0,63}$' }, - }; return { ...contract, summary: 'Update owner packet-sharing settings', @@ -308,7 +468,11 @@ export const API_CONTRACTS: readonly ApiContract[] = [ properties: { nodeId: { type: 'string', pattern: '^[0-9A-Fa-f]{64}$' }, enabled: { type: 'boolean' }, - destinationIds: destinationIdsSchema, + destinationIds: { + type: 'array', + maxItems: 32, + items: { type: 'string', pattern: '^[a-z0-9][a-z0-9_-]{0,63}$' }, + }, }, additionalProperties: false, }, @@ -318,7 +482,11 @@ export const API_CONTRACTS: readonly ApiContract[] = [ properties: { nodeId: { type: 'string', pattern: '^[0-9A-Fa-f]{64}$' }, enabled: { type: 'boolean' }, - destinationIds: destinationIdsSchema, + destinationIds: { + type: 'array', + maxItems: 32, + items: { type: 'string', pattern: '^[a-z0-9][a-z0-9_-]{0,63}$' }, + }, }, additionalProperties: false, }, @@ -330,6 +498,35 @@ export const API_CONTRACTS: readonly ApiContract[] = [ }, }; } + if (contract.path === '/owner/public-feed') { + return { + ...contract, + summary: contract.method === 'GET' + ? 'Read owner public-feed settings' + : 'Update owner public-feed settings', + ...(contract.method === 'GET' + ? { responseSchema: OWNER_PUBLIC_FEED_SCHEMA } + : { + requestSchema: { + type: 'object', + required: ['enabled'], + properties: { enabled: { type: 'boolean' } }, + additionalProperties: false, + }, + responseSchema: OWNER_PUBLIC_FEED_SCHEMA, + additionalResponses: { + '409': { + description: 'The public feed is not configured or the owner has no eligible packet activity', + responseSchema: ERROR_SCHEMA, + }, + '503': { + description: 'The broker ACL could not be updated', + responseSchema: ERROR_SCHEMA, + }, + }, + }), + }; + } return contract; }); diff --git a/backend/src/api/routes.ts b/backend/src/api/routes.ts index a0ffa76..2747a55 100644 --- a/backend/src/api/routes.ts +++ b/backend/src/api/routes.ts @@ -25,6 +25,7 @@ import { getPacketDetail, getPublicVisibilityGeneration, getChannelMessageHistory, + getRecentMessageTags, getRecentPacketEvents, getRecentPackets, analyticsQuery, @@ -63,6 +64,7 @@ import radioRoutes from './routes/radio.js'; import { registerCoverageRoutes } from './routes/coverage.js'; import { registerPlannedCoverageRoutes } from './routes/plannedCoverage.js'; import { registerOwnerRoutes } from './routes/owner.js'; +import { reconcileOwnerAuthorization } from '../owner/ownerAclReconciler.js'; import { registerPathingRoutes } from './routes/pathing.js'; import { registerStatsRoutes } from './routes/stats.js'; import { registerTelemetryRoutes } from './routes/telemetry.js'; @@ -70,6 +72,7 @@ import { createNodeRepository } from '../repositories/nodes.js'; import { createPlannedCoverageRepository } from '../repositories/plannedCoverage.js'; import { registerSpamRoutes } from './routes/spam.js'; import { registerTopologyRoutes } from './routes/topology.js'; +import { registerRegionsRoutes } from './routes/regions.js'; import { registerActivityTimelineRoutes } from './routes/activityTimeline.js'; import { registerRfValidationRoutes } from './routes/rfValidation.js'; import { registerExportRoutes } from './routes/exports.js'; @@ -86,7 +89,6 @@ import { assertUniqueRouteRegistry } from './routeRegistry.js'; import { assertContractCoverage } from './contracts.js'; import { ApiInputError, wrapAsyncHandlers } from './errors.js'; import { withHeavyWorkAdmission } from '../analysis/heavyWorkAdmission.js'; -import { getOwnerAuthorizationSnapshot } from '../db/ownerAuth.js'; const router = Router(); // Anonymous cross-network aggregation is not a public API capability. Operator @@ -140,8 +142,9 @@ async function requireOwnerSession(req: Request, res: Response): Promise (await getOwnerAuthorizationSnapshot()) - .filter((owner) => owner.isActive && owner.nodeIds.length > 0), }); - -export const startOwnerLastHopPrewarm = () => ownerLastHopPrewarm.start(); -export const stopOwnerLastHopPrewarm = () => ownerLastHopPrewarm.stop(); registerPathingRoutes(router, { pathBetaLimiter: PATH_BETA_LIMITER, pathLearningLimiter: PATH_LEARNING_LIMITER, @@ -257,6 +257,11 @@ registerTopologyRoutes(router, { getPublicVisibilityGeneration, limiter: EXPENSIVE_LIMITER, }); +registerRegionsRoutes(router, { + query, + networkFilters, + limiter: EXPENSIVE_LIMITER, +}); registerActivityTimelineRoutes(router, { query, networkFilters, diff --git a/backend/src/api/routes/auditwave.integration.test.ts b/backend/src/api/routes/auditwave.integration.test.ts new file mode 100644 index 0000000..4eda65f --- /dev/null +++ b/backend/src/api/routes/auditwave.integration.test.ts @@ -0,0 +1,180 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { once } from 'node:events'; +import type { AddressInfo } from 'node:net'; +import test, { type TestContext } from 'node:test'; +import { promisify } from 'node:util'; +import pg, { type QueryResultRow } from 'pg'; +import express, { Router, type RequestHandler } from 'express'; +import { getNodeHistory } from '../../db/index.js'; +import { rfValidationRows } from '../../repositories/networkAnalysis.js'; +import { networkFilters } from '../utils/networkFilters.js'; +import { registerExportRoutes } from './exports.js'; + +// Real PostgreSQL evaluation of application SQL, with every application +// relation shadowed by a CTE. SELECT only; no fixture tables or DB writes. +const sqlTest = process.env['AUDITWAVE_SQL_TESTS'] === '1' ? test : test.skip; +const execFileAsync = promisify(execFile); +const key = (c: string) => c.repeat(64); +const A = key('A'), B = key('B'), C = key('C'), D = key('D'), F = key('F'); +function literal(value: unknown): string { + if (Array.isArray(value)) return `ARRAY[${value.map(literal).join(',')}]::text[]`; + if (typeof value === 'number') { assert.ok(Number.isFinite(value)); return String(value); } + return `'${String(value).replace(/'/g, "''")}'`; +} +function node(node_id: string, overrides: Record = {}) { + return { node_id, name: `Repeater ${node_id[0]}`, role: 2, network: 'ukmesh', + lat: 51, lon: -1, iata: 'LHR', last_seen: '2026-09-07T12:00:00Z', hardware_model: null, ...overrides }; +} +function packet(overrides: Record = {}) { + return { age: 3600, packet_hash: 'repeated', rx_node_id: A, src_node_id: B, + network: 'ukmesh', topic: 'ukmesh/LHR/observer/packets', topic_prefix: 'ukmesh', + visibility_ok: true, is_private: false, packet_type: 4, hop_count: 2, + rssi: null, snr: null, payload: { direction: 'rx' }, ...overrides }; +} +function link(node_a_id: string, node_b_id: string, overrides: Record = {}) { + return { node_a_id, node_b_id, age: 3600, observed_count: 1, + multibyte_observed_count: 0, itm_path_loss_db: 120, itm_viable: true, force_viable: false, ...overrides }; +} +type Fixture = { + packets?: ReturnType[]; + links?: ReturnType[]; + nodes?: ReturnType[]; + stalePrivacy?: boolean; + sighted?: boolean; +}; +function fixtureQuery(fixture: Fixture) { + return async (sql: string, params: unknown[] = []): Promise<{ rows: T[] }> => { + const bound = sql.replace(/\$(\d+)\b/g, (_p, index: string) => literal(params[Number(index) - 1])); + const fixtureSql = `WITH + packets AS ( + SELECT p.*, NOW() - age * INTERVAL '1 second' AS time + FROM jsonb_to_recordset(${literal(JSON.stringify(fixture.packets ?? []))}::jsonb) AS p( + age double precision, packet_hash text, rx_node_id text, src_node_id text, + network text, topic text, topic_prefix text, visibility_ok boolean, is_private boolean, + packet_type integer, hop_count integer, rssi double precision, snr double precision, payload jsonb) + ), node_identity_aliases AS ( + SELECT * FROM (VALUES (${literal(F)},${literal(A)})) a(source_node_id,canonical_node_id) + ), node_identity_nodes AS ( + SELECT * FROM jsonb_to_recordset(${literal(JSON.stringify(fixture.nodes ?? [node(A), node(B), node(C), node(D)]))}::jsonb) + AS n(node_id text,name text,role integer,network text,lat double precision,lon double precision, + iata text,last_seen timestamptz,hardware_model text) + ), node_identity_sightings AS ( + SELECT ${literal(B)} AS node_id,'ukmesh'::text AS network,NOW() AS last_seen_at + WHERE ${fixture.sighted ? 'TRUE' : 'FALSE'} + ), node_links AS ( + SELECT l.*, NOW() - age * INTERVAL '1 second' AS last_observed + FROM jsonb_to_recordset(${literal(JSON.stringify(fixture.links ?? []))}::jsonb) AS l( + node_a_id text,node_b_id text,age double precision,observed_count integer, + multibyte_observed_count integer,itm_path_loss_db double precision,itm_viable boolean,force_viable boolean) + ), packet_visibility_materialization_state AS ( + SELECT TRUE AS singleton,1 AS visibility_generation + ), public_visibility_state AS ( + SELECT TRUE AS singleton,${fixture.stalePrivacy ? 2 : 1} AS generation + ) SELECT COALESCE(jsonb_agg(to_jsonb(result)),'[]'::jsonb) FROM (${bound}) result;`; + const child = execFileAsync('docker', ['exec', '-i', '-e', + 'PGOPTIONS=-c default_transaction_read_only=on -c statement_timeout=4500', + 'meshcore-infra-timescaledb-1', 'psql', '-X', '-U', 'meshcore', '-d', 'meshcore', + '-v', 'ON_ERROR_STOP=1', '-At'], { timeout: 6_000, maxBuffer: 4 * 1024 * 1024 }); + child.child.stdin!.end(fixtureSql); + return { rows: JSON.parse((await child).stdout) as T[] }; + }; +} +async function history(t: TestContext, fixture: Fixture, id = A, network = 'ukmesh') { + const mocked = t.mock.method(pg.Pool.prototype, 'query', fixtureQuery(fixture)); + try { return await getNodeHistory(id, 24, network); } + finally { mocked.mock.restore(); } +} + +sqlTest('history selects RX and origin aliases once per observation, retains repeated hashes and canonicalizes source ids', async (t) => { + const rows = await history(t, { packets: [ + packet({ packet_hash: 'rx' }), packet({ packet_hash: 'src', rx_node_id: B, src_node_id: A }), + packet({ packet_hash: 'alias-rx', rx_node_id: F }), + packet({ packet_hash: 'alias-src', rx_node_id: B, src_node_id: F }), + packet({ packet_hash: 'self', rx_node_id: F, src_node_id: A }), + packet({ packet_hash: 'same' }), packet({ packet_hash: 'same', age: 3700 }), + packet({ packet_hash: 'unrelated', rx_node_id: B, src_node_id: C }), + ] }, ` ${F.toLowerCase()} `); + assert.equal(rows.length, 7); + assert.equal(rows.filter(r => r.packet_hash === 'self').length, 1); + assert.equal(rows.filter(r => r.packet_hash === 'same').length, 2); + assert.equal(rows.find(r => r.packet_hash === 'alias-src')!.src_node_id, A); + assert.deepEqual(Object.keys(rows[0]!).sort(), ['time','packet_hash','src_node_id','topic','packet_type','hop_count','rssi','snr','payload'].sort()); +}); + +sqlTest('history enforces the exact time window, public network compatibility and privacy for either direction', async (t) => { + for (const direction of ['rx', 'src']) { + const row = (change: Record) => packet({ ...(direction === 'src' ? { src_node_id: F, rx_node_id: B } : {}), ...change }); + const packets = [row({}), row({ network: 'northeast' }), row({ network: 'teesside' }), row({ age: 0 }), + row({ age: 86400 }), row({ age: -1 }), row({ network: 'test' }), row({ network: 'other' }), + row({ topic_prefix: 'meshcore-test' }), row({ topic_prefix: null, topic: 'meshcore-test/LHR/x/packets' }), + row({ visibility_ok: false }), row({ visibility_ok: null }), row({ is_private: true })]; + assert.equal((await history(t, { packets })).length, 4); + assert.deepEqual(await history(t, { packets, stalePrivacy: true }), []); + } + assert.equal((await history(t, { packets: [packet({ network: 'test', topic_prefix: 'meshcore-test' })] }, A, 'test')).length, 1); +}); + +sqlTest('history keeps the newest 500 observations and returns an honest empty set', async (t) => { + const rows = await history(t, { packets: Array.from({ length: 510 }, (_, i) => packet({ age: i, packet_hash: String(i) })) }); + assert.equal(rows.length, 500); + assert.equal(rows[0]!.packet_hash, '0'); + assert.equal(rows[499]!.packet_hash, '499'); + assert.deepEqual(await history(t, { packets: [packet()] }, key('0')), []); + assert.deepEqual(await history(t, { packets: [] }), []); +}); + +sqlTest('RF query combines aliases and preserves mismatch, override and stale weak-evidence classifications', async () => { + const { rows } = await rfValidationRows(fixtureQuery({ links: [ + link(A, B, { observed_count: 5, multibyte_observed_count: 1, itm_viable: false, itm_path_loss_db: 150 }), + link(B, F, { observed_count: 7, multibyte_observed_count: 2, itm_viable: false, itm_path_loss_db: 140 }), + link(A, C, { force_viable: true }), + link(B, C, { age: 8 * 86400, observed_count: 2 }), + link(A, D), + link(C, D, { age: 30 * 86400 }), + link(A, F, { observed_count: 1000, itm_viable: false }), + ] }), networkFilters('ukmesh'), 100); + assert.equal(rows.length, 4); + assert.deepEqual(rows.map(r => r.classification), ['observed_unexpected','operator_override','weak_model_evidence','match']); + assert.equal(rows[0]!.observed_count, 12); + assert.equal(rows[0]!.multibyte_observed_count, 3); + assert.equal(rows[0]!.itm_path_loss_db, 140); + assert.equal(rows[0]!.node_a_id, A); + assert.equal(rows[0]!.node_b_id, B); +}); + +sqlTest('RF scopes both endpoints before its limit, allowing recent sightings but excluding private, test and non-repeater nodes', async () => { + const fixture: Fixture = { nodes: [node(A), node(B, { network: 'other' }), node(C, { name: '🚫 Private' }), node(D, { role: 1 })], + links: [link(A, C, { observed_count: 99999, itm_viable: false }), link(A, D, { observed_count: 99999 }), link(A, B)], sighted: true }; + const { rows } = await rfValidationRows(fixtureQuery(fixture), networkFilters('ukmesh'), 1); + assert.equal(rows.length, 1); + assert.equal(rows[0]!.node_b_id, B); + assert.deepEqual((await rfValidationRows(fixtureQuery({ ...fixture, sighted: false }), networkFilters('ukmesh'), 1)).rows, []); + assert.deepEqual((await rfValidationRows(fixtureQuery({ ...fixture, nodes: [node(A), node(B, { network: 'test' })] }), networkFilters('ukmesh'), 1)).rows, []); +}); + +sqlTest('exports execute real SQL with the correct alias and omit private, unpositioned and out-of-scope nodes', async () => { + const app = express(), router = Router(); + const limiter: RequestHandler = (_req, _res, next) => next(); + registerExportRoutes(router, { networkFilters, + exportLimiter: limiter as Parameters[1]['exportLimiter'], + query: fixtureQuery({ nodes: [node(A), node(B, { network: 'other' }), node(C, { name: '🚫 Private' }), + node(D, { network: 'test' }), node(F, { lat: null })], sighted: true }), + }); + app.use(router); + const server = app.listen(0, '127.0.0.1'); + await once(server, 'listening'); + try { + const base = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + for (const format of ['csv', 'geojson']) { + const response = await fetch(`${base}/v1/exports/nodes.${format}?network=ukmesh&limit=3`); + assert.equal(response.status, 200); + const body = await response.text(); + assert.ok(body.includes(A) && body.includes(B)); + for (const id of [C, D, F]) assert.ok(!body.includes(id)); + } + } finally { + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + } +}); diff --git a/backend/src/api/routes/auditwave.performance.test.ts b/backend/src/api/routes/auditwave.performance.test.ts new file mode 100644 index 0000000..b2ada5b --- /dev/null +++ b/backend/src/api/routes/auditwave.performance.test.ts @@ -0,0 +1,93 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { mkdir, writeFile } from 'node:fs/promises'; +import { once } from 'node:events'; +import type { AddressInfo } from 'node:net'; +import test from 'node:test'; +import { promisify } from 'node:util'; +import pg, { type QueryResultRow } from 'pg'; +import express, { Router, type RequestHandler } from 'express'; +import { getNodeHistory } from '../../db/index.js'; +import { rfValidationRows } from '../../repositories/networkAnalysis.js'; +import { networkFilters } from '../utils/networkFilters.js'; +import { registerExportRoutes } from './exports.js'; + +const perfTest = process.env['AUDITWAVE_PERF_TESTS'] === '1' ? test : test.skip; +const execFileAsync = promisify(execFile); +const evidence = '../.fix-backups-auditwave-20260907/evidence'; +const summaries: unknown[] = []; +function literal(value: unknown): string { + if (Array.isArray(value)) return `ARRAY[${value.map(literal).join(',')}]::text[]`; + if (typeof value === 'number') { assert.ok(Number.isFinite(value)); return String(value); } + return `'${String(value).replace(/'/g, "''")}'`; +} +async function select(sql: string) { + const child = execFileAsync('docker', ['exec', '-i', '-e', + 'PGOPTIONS=-c default_transaction_read_only=on -c statement_timeout=4500', + 'meshcore-infra-timescaledb-1', 'psql', '-X', '-U', 'meshcore', '-d', 'meshcore', + '-v', 'ON_ERROR_STOP=1', '-At'], { timeout: 6_000, maxBuffer: 12 * 1024 * 1024 }); + child.child.stdin!.end(sql); + return JSON.parse((await child).stdout); +} +function measuredQuery(label: string) { + let calls = 0; + return async (sql: string, params: unknown[] = []): Promise<{ rows: T[] }> => { + const name = `${label}-${++calls}`; + const bound = sql.replace(/\$(\d+)\b/g, (_p, index: string) => literal(params[Number(index) - 1])); + await mkdir(evidence, { recursive: true }); + await writeFile(`${evidence}/${name}.sql`, bound); + const plan = await select(`EXPLAIN (ANALYZE, BUFFERS, SETTINGS, FORMAT JSON) ${bound}`); + await writeFile(`${evidence}/${name}-explain.json`, JSON.stringify(plan, null, 2)); + const rows = await select(`SELECT COALESCE(jsonb_agg(to_jsonb(result)),'[]'::jsonb) FROM (${bound}) result`); + const summary = { name, rows: rows.length, planningMs: plan[0]['Planning Time'], executionMs: plan[0]['Execution Time'] }; + summaries.push(summary); + await writeFile(`${evidence}/performance-summary.json`, JSON.stringify(summaries, null, 2)); + console.log(JSON.stringify(summary)); + assert.ok(plan[0]['Execution Time'] < 4500); + return { rows }; + }; +} + +perfTest('EXPLAIN actual RF repository queries on the live read-only database', { timeout: 20_000 }, async () => { + const { rows } = await rfValidationRows(measuredQuery('rf'), networkFilters('ukmesh'), 250); + assert.equal(rows.length, 250); + assert.ok(rows.some(row => row.classification === 'observed_unexpected')); +}); + +perfTest('EXPLAIN actual history queries for Lyneham, Gadget and an empty id, including maximum window', { timeout: 40_000 }, async (t) => { + const gadgetRows = await select("SELECT jsonb_agg(n) FROM (SELECT node_id,name FROM node_identity_nodes WHERE name ILIKE 'Gadget-RPT%') n"); + assert.ok(gadgetRows.length > 0); + for (const [name, id, hours] of [ + ['lyneham', 'C42E3FE2D99234E0E41967C1B70E49FA6F955AD5B8448A06EC2978AB0FF9E79E', 24], + ['gadget', gadgetRows[0].node_id, 24], + ['empty', '0'.repeat(64), 24], + ['lyneham-max-window', 'C42E3FE2D99234E0E41967C1B70E49FA6F955AD5B8448A06EC2978AB0FF9E79E', 672], + ] as const) { + const mocked = t.mock.method(pg.Pool.prototype, 'query', measuredQuery(`history-${name}`)); + try { + const rows = await getNodeHistory(id, hours, 'ukmesh'); + assert.ok(rows.length <= 500); + if (name === 'empty') assert.equal(rows.length, 0); + else assert.ok(rows.length > 0); + } finally { mocked.mock.restore(); } + } +}); + +perfTest('EXPLAIN actual node export SQL including its 5000-node maximum', { timeout: 15_000 }, async () => { + const app = express(), router = Router(); + const limiter: RequestHandler = (_req, _res, next) => next(); + registerExportRoutes(router, { networkFilters, + exportLimiter: limiter as Parameters[1]['exportLimiter'], query: measuredQuery('exports') }); + app.use(router); + const server = app.listen(0, '127.0.0.1'); + await once(server, 'listening'); + try { + const response = await fetch(`http://127.0.0.1:${(server.address() as AddressInfo).port}/v1/exports/nodes.geojson?network=ukmesh&limit=5000`); + assert.equal(response.status, 200); + const body = await response.json() as { features: unknown[] }; + assert.ok(body.features.length > 0 && body.features.length <= 5000); + } finally { + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + } +}); diff --git a/backend/src/api/routes/exports.test.ts b/backend/src/api/routes/exports.test.ts index cc62518..91aeb45 100644 --- a/backend/src/api/routes/exports.test.ts +++ b/backend/src/api/routes/exports.test.ts @@ -1,6 +1,10 @@ import assert from 'node:assert/strict'; import test from 'node:test'; -import { csvCell } from './exports.js'; +import { once } from 'node:events'; +import type { AddressInfo } from 'node:net'; +import express, { Router, type RequestHandler } from 'express'; +import { networkFilters } from '../utils/networkFilters.js'; +import { csvCell, registerExportRoutes } from './exports.js'; test('csvCell escapes delimiters, quotes, and newlines', () => { assert.equal(csvCell('simple'), 'simple'); @@ -9,3 +13,48 @@ test('csvCell escapes delimiters, quotes, and newlines', () => { assert.equal(csvCell('line one\nline two'), '"line one\nline two"'); assert.equal(csvCell(null), ''); }); + +test('node exports use the sightings alias and retain CSV/GeoJSON contracts', async () => { + const app = express(); + const router = Router(); + const row = { + node_id: 'A'.repeat(64), name: 'Repeater, "Alpha"', lat: 51.5, lon: -1.2, + role: 2, iata: 'LHR', last_seen: '2026-09-07T12:00:00Z', hardware_model: null, + }; + const limiter: RequestHandler = (_req, _res, next) => next(); + registerExportRoutes(router, { + networkFilters, + exportLimiter: limiter as Parameters[1]['exportLimiter'], + query: async (sql, params) => { + assert.match(sql, /FROM node_identity_nodes AS nodes/); + assert.match(sql, /s.node_id = nodes.node_id/); + assert.match(sql, /name NOT LIKE '%🚫%'/); + assert.match(sql, /lat IS NOT NULL AND lon IS NOT NULL/); + assert.match(sql, /LIMIT \$2/); + assert.deepEqual(params, [['ukmesh', 'northeast', 'teesside'], 3]); + return { rows: [row] as never[] }; + }, + }); + app.use('/api', router); + const server = app.listen(0, '127.0.0.1'); + await once(server, 'listening'); + try { + const base = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + const csv = await fetch(`${base}/api/v1/exports/nodes.csv?network=ukmesh&limit=3`); + assert.equal(csv.status, 200); + assert.match(csv.headers.get('content-type')!, /text\/csv/); + assert.equal(csv.headers.get('x-api-version'), '1'); + assert.equal(await csv.text(), `node_id,name,lat,lon,role,iata,last_seen,hardware_model\n${row.node_id},"Repeater, ""Alpha""",51.5,-1.2,2,LHR,2026-09-07T12:00:00Z,\n`); + const geo = await fetch(`${base}/api/v1/exports/nodes.geojson?network=ukmesh&limit=3`); + assert.equal(geo.status, 200); + assert.match(geo.headers.get('content-type')!, /application\/geo\+json/); + const body = await geo.json() as { type: string; features: Array<{ geometry: unknown; properties: { node_id: string } }> }; + assert.equal(body.type, 'FeatureCollection'); + assert.equal(body.features.length, 1); + assert.deepEqual(body.features[0]!.geometry, { type: 'Point', coordinates: [-1.2, 51.5] }); + assert.equal(body.features[0]!.properties.node_id, row.node_id); + } finally { + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + } +}); diff --git a/backend/src/api/routes/exports.ts b/backend/src/api/routes/exports.ts index ec2cea1..9062454 100644 --- a/backend/src/api/routes/exports.ts +++ b/backend/src/api/routes/exports.ts @@ -118,7 +118,7 @@ export function registerExportRoutes(router: Router, deps: Deps): void { role: number | null; iata: string | null; last_seen: string | null; hardware_model: string | null; }>( `SELECT node_id, name, lat, lon, role, iata, last_seen::text, hardware_model - FROM node_identity_nodes + FROM node_identity_nodes AS nodes WHERE lat IS NOT NULL AND lon IS NOT NULL AND (name IS NULL OR name NOT LIKE '%🚫%') ${filters.nodes} diff --git a/backend/src/api/routes/misc.test.ts b/backend/src/api/routes/misc.test.ts index c948e98..5b167a1 100644 --- a/backend/src/api/routes/misc.test.ts +++ b/backend/src/api/routes/misc.test.ts @@ -37,64 +37,3 @@ test('channel feed history route forwards its bounded channel and scope', async server.close(); } }); - -test('public MQTT node status omits owner-only and unknown telemetry fields', async () => { - const storedRow = { - node_id: 'b'.repeat(64), - name: 'Public MQTT node', - last_seen: '2026-09-23T12:00:00.000Z', - battery_mv: 3900, - uptime_secs: 4200, - channel_utilization: 8, - air_util_tx: 4, - rx_air_secs: 18, - tx_air_secs: 9, - packets_24h: '12', - hardware_model: 'private model', - firmware_version: 'private firmware', - stats: { - wifi_ssid: 'private wifi', - reset_reason: 'private reset detail', - fs_free_bytes: 1234, - mqtt: { broker_uri: 'mqtts://private', broker_username: 'private-user' }, - future_owner_field: 'private future value', - }, - future_status_field: 'private future value', - }; - const router = Router(); - registerMiscRoutes(router, { - query: async (text) => { - assert.doesNotMatch(text, /nss\.stats/); - return { rows: [storedRow as unknown as T] }; - }, - getRecentPackets: async () => [], - getRecentPacketEvents: async () => [], - getPacketDetail: async () => null, - getChannelMessageHistory: async () => [], - getPublicVisibilityGeneration: async () => 1, - packetDetailLimiter: passThroughLimiter as Parameters[1]['packetDetailLimiter'], - }); - const app = express(); - app.use(router); - const server = app.listen(0, '127.0.0.1'); - await once(server, 'listening'); - try { - const { port } = server.address() as AddressInfo; - const response = await fetch(`http://127.0.0.1:${port}/mqtt-nodes?network=ukmesh`); - assert.equal(response.status, 200); - assert.deepEqual(await response.json(), [{ - node_id: storedRow.node_id, - name: storedRow.name, - last_seen: storedRow.last_seen, - battery_mv: storedRow.battery_mv, - uptime_secs: storedRow.uptime_secs, - channel_utilization: storedRow.channel_utilization, - air_util_tx: storedRow.air_util_tx, - rx_air_secs: storedRow.rx_air_secs, - tx_air_secs: storedRow.tx_air_secs, - packets_24h: storedRow.packets_24h, - }]); - } finally { - server.close(); - } -}); diff --git a/backend/src/api/routes/misc.ts b/backend/src/api/routes/misc.ts index 6e13b0d..75a3c3f 100644 --- a/backend/src/api/routes/misc.ts +++ b/backend/src/api/routes/misc.ts @@ -16,7 +16,6 @@ import { decodePlannedNodeCursor, listPublicPlannedNodes, } from '../../repositories/plannedNodes.js'; -import { toPublicMqttNodeStatusDTO } from '../publicStatus.js'; import { readRecentWithPending } from '../../ram/liveRead.js'; type QueryFn = ( @@ -33,7 +32,22 @@ type GetRecentPacketsFn = ( readQuery?: QueryFn, overlapHashes?: string[], ) => Promise; -type GetRecentPacketEventsFn = (limit: number, network?: string, observer?: string) => Promise; +type GetRecentPacketEventsFn = ( + limit: number, + network?: string, + observer?: string, + region?: string, +) => Promise; + +// Region scope filter (e.g. 'eng-ne' / '#eng-ne' -> stored form '#eng-ne'). +const REGION_SCOPE_PATTERN = /^[a-z0-9][a-z0-9_-]{0,31}$/; + +function normalizeRegionScope(value: unknown): string | undefined { + if (typeof value !== 'string') return undefined; + const bare = value.trim().toLowerCase().replace(/^#/, ''); + if (!REGION_SCOPE_PATTERN.test(bare)) return undefined; + return `#${bare}`; +} type GetPacketDetailFn = (hash: string, network?: string) => Promise; type GetChannelMessageHistoryFn = ( channel: string, @@ -41,6 +55,7 @@ type GetChannelMessageHistoryFn = ( network?: string, observer?: string, ) => Promise; +type GetRecentMessageTagsFn = (limit: number, network?: string) => Promise; type MiscRouteDeps = { query: QueryFn; @@ -48,6 +63,7 @@ type MiscRouteDeps = { getRecentPacketEvents: GetRecentPacketEventsFn; getPacketDetail: GetPacketDetailFn; getChannelMessageHistory: GetChannelMessageHistoryFn; + getRecentMessageTags: GetRecentMessageTagsFn; getPublicVisibilityGeneration: () => Promise; packetDetailLimiter: ReturnType; }; @@ -59,6 +75,7 @@ export function registerMiscRoutes(router: Router, deps: MiscRouteDeps): void { getRecentPacketEvents, getPacketDetail, getChannelMessageHistory, + getRecentMessageTags, getPublicVisibilityGeneration, packetDetailLimiter, } = deps; @@ -83,6 +100,7 @@ export function registerMiscRoutes(router: Router, deps: MiscRouteDeps): void { max: 1000, }); const raw = parseBoolean(req.query['raw'], { name: 'raw' }); + const region = normalizeRegionScope(req.query['region']); const fields = parseEnum(req.query['fields'], { name: 'fields', values: ['slim', 'full'] as const, @@ -92,7 +110,7 @@ export function registerMiscRoutes(router: Router, deps: MiscRouteDeps): void { const network = resolvePublicNetworkScope(req.query['network'], req.headers); const observer = normalizeObserverQuery(req.query['observer']); const packets = raw - ? await getRecentPacketEvents(limit, network, observer) + ? await getRecentPacketEvents(limit, network, observer, region) : await readRecentWithPending( (overlapHashes) => getRecentPackets( limit, network, observer, fields, AbortSignal.timeout(30_000), undefined, overlapHashes, @@ -131,6 +149,27 @@ export function registerMiscRoutes(router: Router, deps: MiscRouteDeps): void { } }); + // Message tags (kind / speaker / topic / flags) for live feed enrichment. + // Tags are written ~2s after a packet by tagger-worker; the frontend patches + // rows by hash. Public visibility + network scope enforced in the query. + router.get('/tags/recent', async (req, res) => { + const limit = parseBoundedInteger(req.query['limit'], { + name: 'limit', + defaultValue: 200, + min: 1, + max: 500, + }); + try { + const network = resolvePublicNetworkScope(req.query['network'], req.headers); + const tags = await getRecentMessageTags(limit, network); + res.setHeader('Cache-Control', 'no-store'); + res.json(tags); + } catch (err) { + console.error('[api] GET /tags/recent', (err as Error).message); + res.status(500).json({ error: 'Internal server error' }); + } + }); + router.get('/packets/:hash', packetDetailLimiter, async (req, res) => { const hash = parseHexIdentifier(req.params['hash'], { name: 'hash', @@ -228,6 +267,7 @@ export function registerMiscRoutes(router: Router, deps: MiscRouteDeps): void { air_util_tx: number | null; rx_air_secs: number | null; tx_air_secs: number | null; + stats: Record | null; packets_24h: string; }>( @@ -241,6 +281,7 @@ export function registerMiscRoutes(router: Router, deps: MiscRouteDeps): void { nss.air_util_tx, nss.rx_air_secs, nss.tx_air_secs, + nss.stats, COALESCE(pc.packet_count, 0) AS packets_24h FROM node_identity_status_samples nss LEFT JOIN node_identity_nodes n ON n.node_id = nss.node_id @@ -258,9 +299,7 @@ export function registerMiscRoutes(router: Router, deps: MiscRouteDeps): void { ORDER BY nss.node_id, COALESCE(nss.uptime_secs, 0) DESC, nss.time DESC`, params, ); - res.json(result.rows - .filter((r) => Number(r.packets_24h) > 0) - .map(toPublicMqttNodeStatusDTO)); + res.json(result.rows.filter((r) => Number(r.packets_24h) > 0)); } catch (err) { console.error('[api] GET /mqtt-nodes', (err as Error).message); res.status(500).json({ error: 'Internal server error' }); diff --git a/backend/src/api/routes/nodeLinks.test.ts b/backend/src/api/routes/nodeLinks.test.ts new file mode 100644 index 0000000..abdcde5 --- /dev/null +++ b/backend/src/api/routes/nodeLinks.test.ts @@ -0,0 +1,56 @@ +import assert from 'node:assert/strict'; +import { once } from 'node:events'; +import type { AddressInfo } from 'node:net'; +import test from 'node:test'; +import express, { Router } from 'express'; +import { rateLimit } from 'express-rate-limit'; +import { networkFilters } from '../utils/networkFilters.js'; +import { createNodeRepository, type NodeLinkRow } from '../../repositories/nodes.js'; +import { registerNodeRoutes } from './nodes.js'; + +test('links endpoint serializes additive fields and retains them on a cache hit', async () => { + const nodeId = 'A'.repeat(64); + const rows: NodeLinkRow[] = [{ + peer_id: 'B'.repeat(64), peer_name: 'Peer', observed_count: 2000, + itm_path_loss_db: 112.1, count_this_to_peer: 700, count_peer_to_this: 1300, + last_observed: '2026-09-07T12:00:00.123Z', recent_count_72h: 0, + }, { + peer_id: 'C'.repeat(64), peer_name: null, observed_count: 20, + itm_path_loss_db: null, count_this_to_peer: 10, count_peer_to_this: 10, + last_observed: null, recent_count_72h: 12, + }]; + let calls = 0; + const repository = createNodeRepository(async () => ({ rows: [] })); + repository.listNodeLinks = async (id, scope) => { + calls += 1; + assert.equal(id.toUpperCase(), nodeId); + assert.deepEqual(scope.params, [['ukmesh', 'northeast', 'teesside']]); + return rows; + }; + const app = express(); + const router = Router(); + registerNodeRoutes(router, { + nodeRepository: repository, + getNodes: async () => [], getNodeHistory: async () => [], getNodeAdverts: async () => [], + getTopAdvertingRepeaters: async () => [], requireLocalOnly: () => false, + networkFilters, getPublicVisibilityGeneration: async () => 1, + inferredNodesCache: new Map(), inferredNodesInflight: new Map(), inferredNodesCacheTtlMs: 60000, + nodeLinksCache: new Map(), nodeLinksInflight: new Map(), nodeLinksCacheTtlMs: 60000, + nodesLimiter: rateLimit({ windowMs: 60000, limit: 100 }), + }); + app.use('/api', router); + const server = app.listen(0, '127.0.0.1'); + await once(server, 'listening'); + try { + const base = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + for (const id of [nodeId.toLowerCase(), nodeId]) { + const response = await fetch(`${base}/api/nodes/${id}/links?network=ukmesh`); + assert.equal(response.status, 200); + assert.deepEqual(await response.json(), rows); + } + assert.equal(calls, 1); + } finally { + server.closeAllConnections(); + await new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve())); + } +}); diff --git a/backend/src/api/routes/nodeStatus.ts b/backend/src/api/routes/nodeStatus.ts index df881dc..095642c 100644 --- a/backend/src/api/routes/nodeStatus.ts +++ b/backend/src/api/routes/nodeStatus.ts @@ -7,10 +7,6 @@ import { parseBoundedInteger, parseBoundedString, } from '../utils/input.js'; -import { - toPublicLatestNodeStatusDTO, - type PublicLatestNodeStatusDTO, -} from '../publicStatus.js'; const router = Router(); @@ -30,11 +26,28 @@ router.get('/node-status/latest', async (req, res) => { } const identityWhereClause = `WHERE ${identityConditions.join(' AND ')}`; - const result = await query( + const result = await query<{ + time: string; + node_id: string; + network: string | null; + battery_mv: number | null; + uptime_secs: number | null; + tx_air_secs: number | null; + rx_air_secs: number | null; + channel_utilization: number | null; + air_util_tx: number | null; + stats: Record | null; + name: string | null; + iata: string | null; + hardware_model: string | null; + firmware_version: string | null; + }>( `WITH identity_sources AS MATERIALIZED ( SELECT n.node_id AS canonical_id, n.name, n.iata, + n.hardware_model, + n.firmware_version, source.source_node_id FROM node_identity_nodes n CROSS JOIN LATERAL unnest(n.identity_source_ids) AS source(source_node_id) @@ -50,12 +63,15 @@ router.get('/node-status/latest', async (req, res) => { nss.rx_air_secs, nss.channel_utilization, nss.air_util_tx, + nss.stats, s.name, - s.iata + s.iata, + s.hardware_model, + s.firmware_version FROM identity_sources s JOIN LATERAL ( SELECT time, network, battery_mv, uptime_secs, tx_air_secs, - rx_air_secs, channel_utilization, air_util_tx + rx_air_secs, channel_utilization, air_util_tx, stats FROM node_status_samples WHERE node_id = s.source_node_id AND network = ANY($1::text[]) @@ -69,7 +85,7 @@ router.get('/node-status/latest', async (req, res) => { params, ); - res.json(result.rows.map(toPublicLatestNodeStatusDTO)); + res.json(result.rows); } catch (err) { console.error('[api] GET /node-status/latest', (err as Error).message); res.status(500).json({ error: 'Internal server error' }); diff --git a/backend/src/api/routes/owner.ts b/backend/src/api/routes/owner.ts index 3e82b66..4e80612 100644 --- a/backend/src/api/routes/owner.ts +++ b/backend/src/api/routes/owner.ts @@ -1,7 +1,6 @@ import type { Request, Response, Router } from 'express'; import { createOwnerRepository } from '../../owner/ownerRepository.js'; import { createOwnerService } from '../../owner/ownerService.js'; -import { createOwnerLastHopPrewarm, type OwnerLastHopPrewarmTarget } from '../../owner/ownerLastHopPrewarm.js'; import type { OwnerSession } from '../../owner/ownerSession.js'; import { createCsrfToken, readCookie, requireDoubleSubmitCsrf } from '../../security/operatorAuth.js'; import { resolveWebhookTarget } from '../../security/outboundWebhook.js'; @@ -11,6 +10,11 @@ import { } from '../../owner/packetShareConfig.js'; import { hasPacketShareEncryptionKey } from '../../owner/packetShareCrypto.js'; import { createPacketShareRepository, type PacketShareTransactionFn } from '../../owner/packetShareRepository.js'; +import { + getPublicFeedConfig, + PUBLIC_FEED_TOPIC_FILTER, +} from '../../owner/publicFeedConfig.js'; +import { createPublicFeedRepository } from '../../owner/publicFeedRepository.js'; import { deliverDueOwnerAlerts, queueOwnerTestDelivery, @@ -40,8 +44,11 @@ type IsSecureRequestFn = (req: Request) => boolean; type GetOwnerSessionFn = (req: Request) => OwnerSession | null; type RequireOwnerSessionFn = (req: Request, res: Response) => Promise; type GetOwnerCredentialGenerationFn = (mqttUsername: string) => Promise; +type ReconcileOwnerAuthorizationFn = () => Promise; type QueryFn = (text: string, params?: unknown[]) => Promise<{ rows: T[] }>; +type OwnerService = ReturnType; + type OwnerRouteDeps = { ownerCookieName: string; ownerLiveCacheTtlMs: number; @@ -63,16 +70,19 @@ type OwnerRouteDeps = { requireOwnerSession: RequireOwnerSessionFn; getOwnerCredentialGeneration: GetOwnerCredentialGenerationFn; invalidateOwnerNodeIdCache: (mqttUsername: string) => void; + reconcileOwnerAuthorization?: ReconcileOwnerAuthorizationFn; query: QueryFn; withTransaction?: PacketShareTransactionFn; - loadPrewarmOwners: () => Promise; }; -export function registerOwnerRoutes(router: Router, deps: OwnerRouteDeps) { +let registeredOwnerService: OwnerService | null = null; + +export function registerOwnerRoutes(router: Router, deps: OwnerRouteDeps): void { const repository = createOwnerRepository({ query: deps.query, }); const packetShareRepository = createPacketShareRepository(deps.query, deps.withTransaction); + const publicFeedRepository = createPublicFeedRepository(deps.query, deps.withTransaction); const service = createOwnerService({ ownerLiveCacheTtlMs: deps.ownerLiveCacheTtlMs, @@ -86,11 +96,7 @@ export function registerOwnerRoutes(router: Router, deps: OwnerRouteDeps) { repository, invalidateOwnerNodeIdCache: deps.invalidateOwnerNodeIdCache, }); - const prewarm = createOwnerLastHopPrewarm({ - loadOwners: deps.loadPrewarmOwners, - refresh: (owner, nodeId) => service.getOwnerLastHopStrength(owner.nodeIds, nodeId, true), - concurrency: process.env['OWNER_LAST_HOP_PREWARM_CONCURRENCY'], - }); + registeredOwnerService = service; const csrfCookieName = 'meshcore_owner_csrf'; const csrfProtection = requireDoubleSubmitCsrf(csrfCookieName); @@ -248,10 +254,16 @@ export function registerOwnerRoutes(router: Router, deps: OwnerRouteDeps) { maxLength: 64, pattern: /^[0-9a-fA-F]{64}$/, })?.toUpperCase(); + const requestedWindow = parseBoundedString(req.query['window'], { + name: 'window', + maxLength: 4, + pattern: /^(7d|30d)$/, + }); + const windowDays = requestedWindow === '30d' ? 30 : 7; try { const ownedNodeIds = await deps.requireOwnerSession(req, res); if (!ownedNodeIds) return; - res.json(await service.getOwnerLastHopStrength(ownedNodeIds, requestedNodeId)); + res.json(await service.getOwnerLastHopStrength(ownedNodeIds, requestedNodeId, windowDays)); } catch (err) { const message = (err as Error).message; if (message === 'NO_OWNED_NODES') { @@ -268,39 +280,32 @@ export function registerOwnerRoutes(router: Router, deps: OwnerRouteDeps) { }); router.get('/owner/packet-sharing', async (req, res) => { - let requestedNodeId: string | undefined; - try { - requestedNodeId = parseBoundedString(req.query['nodeId'], { - name: 'nodeId', - maxLength: 64, - pattern: /^[0-9a-fA-F]{64}$/, - })?.toUpperCase(); - } catch { - res.status(400).json({ error: 'Invalid packet sharing settings' }); - return; - } + const requestedNodeId = parseBoundedString(req.query['nodeId'], { + name: 'nodeId', + maxLength: 64, + pattern: /^[0-9a-fA-F]{64}$/, + })?.toUpperCase(); try { const ownedNodeIds = await deps.requireOwnerSession(req, res); if (!ownedNodeIds) return; const session = deps.getOwnerSession(req)!; - const ownedNodeIdSet = new Set(ownedNodeIds.map((nodeId) => nodeId.toUpperCase())); - const nodeId = requestedNodeId ?? ownedNodeIds[0]?.toUpperCase(); - if (!nodeId || !ownedNodeIdSet.has(nodeId)) { + const nodeId = requestedNodeId ?? ownedNodeIds[0]; + if (!nodeId || !ownedNodeIds.includes(nodeId)) { res.status(403).json({ error: 'Node is not owned by this session' }); return; } const [destinations, selectedIds, lastForwardedAt] = await Promise.all([ - packetShareRepository.listDestinations(), + packetShareRepository.listDestinations(session.mqttUsername, nodeId), packetShareRepository.listOwnerRules(session.mqttUsername, nodeId), packetShareRepository.listLastForwardedAt(session.mqttUsername, nodeId), ]); - const selected = new Set(selectedIds); + const selected = new Set(selectedIds.filter((id) => destinations.some((destination) => destination.destination_id === id))); const encryptionKeyAvailable = hasPacketShareEncryptionKey(); res.setHeader('Cache-Control', 'no-store'); res.json({ nodeId, featureEnabled: true, - enabled: selectedIds.length > 0, + enabled: selected.size > 0, destinations: destinations.map((destination) => publicDestinationView( destination, selected.has(destination.destination_id), @@ -329,38 +334,29 @@ export function registerOwnerRoutes(router: Router, deps: OwnerRouteDeps) { enabled?: unknown; destinationIds?: unknown; }; - if (Object.keys(rawBody).some((key) => !['nodeId', 'enabled', 'destinationIds'].includes(key))) { - res.status(400).json({ error: 'Invalid packet sharing settings' }); - return; - } - const nodeId = typeof body.nodeId === 'string' ? body.nodeId.trim().toUpperCase() : ''; - const ownedNodeIdSet = new Set(ownedNodeIds.map((ownedNodeId) => ownedNodeId.toUpperCase())); + const nodeId = String(body.nodeId ?? '').trim().toUpperCase(); const rawDestinationIds = body.destinationIds; const enabled = body.enabled; - if (!Array.isArray(rawDestinationIds) - || rawDestinationIds.length > 32 - || !rawDestinationIds.every((value) => typeof value === 'string') + const destinationIds = Array.isArray(rawDestinationIds) && rawDestinationIds.every((value) => typeof value === 'string') + ? Array.from(new Set(rawDestinationIds.map((value) => value.trim().toLowerCase()))) + : []; + if ( + !/^[0-9A-F]{64}$/.test(nodeId) + || !ownedNodeIds.includes(nodeId) || typeof enabled !== 'boolean' + || !Array.isArray(rawDestinationIds) + || !rawDestinationIds.every((value) => typeof value === 'string') + || destinationIds.length > 32 + || destinationIds.some((destinationId) => !/^[a-z0-9][a-z0-9_-]{0,63}$/.test(destinationId)) ) { res.status(400).json({ error: 'Invalid packet sharing settings' }); return; } - const destinationIds = Array.from(new Set(rawDestinationIds.map((value) => value.trim().toLowerCase()))); - if (!/^[0-9A-F]{64}$/.test(nodeId) - || destinationIds.some((destinationId) => !/^[a-z0-9][a-z0-9_-]{0,63}$/.test(destinationId))) { - res.status(400).json({ error: 'Invalid packet sharing settings' }); - return; - } - if (!ownedNodeIdSet.has(nodeId)) { - res.status(403).json({ error: 'Node is not owned by this session' }); - return; - } - const destinations = await packetShareRepository.listDestinations(); + const destinations = await packetShareRepository.listDestinations(session.mqttUsername, nodeId); const byId = new Map(destinations.map((destination) => [destination.destination_id, destination])); - const encryptionKeyAvailable = hasPacketShareEncryptionKey(); - if (enabled && destinationIds.some((destinationId) => { + if (destinationIds.some((destinationId) => { const destination = byId.get(destinationId); - return !destination || !destinationConfigured(destination, encryptionKeyAvailable); + return !destination || (enabled && !destinationConfigured(destination, hasPacketShareEncryptionKey())); })) { res.status(409).json({ error: 'One or more selected destinations are not configured' }); return; @@ -382,6 +378,95 @@ export function registerOwnerRoutes(router: Router, deps: OwnerRouteDeps) { } }); + router.get('/owner/public-feed', async (req, res) => { + try { + const ownedNodeIds = await deps.requireOwnerSession(req, res); + if (!ownedNodeIds) return; + const session = deps.getOwnerSession(req)!; + const status = await publicFeedRepository.accountStatus( + session.mqttUsername, + ownedNodeIds, + ); + const config = getPublicFeedConfig(); + const topicFilters = config.configured ? [PUBLIC_FEED_TOPIC_FILTER] : []; + res.setHeader('Cache-Control', 'no-store'); + res.json({ + featureEnabled: true, + configured: config.configured, + brokerUrl: config.brokerUrl, + enabled: status.enabled, + nodeCount: status.nodeCount, + eligible: status.eligible, + recentPacketCount: status.recentPacketCount, + topicFilters, + lastForwardedAt: status.lastForwardedAt, + }); + } catch (error) { + console.error('[api] GET /owner/public-feed', (error as Error).message); + res.status(500).json({ error: 'Internal server error' }); + } + }); + + router.post('/owner/public-feed', csrfProtection, async (req, res) => { + try { + const ownedNodeIds = await deps.requireOwnerSession(req, res); + if (!ownedNodeIds) return; + const session = deps.getOwnerSession(req)!; + const rawBody = req.body as unknown; + if (!rawBody || typeof rawBody !== 'object' || Array.isArray(rawBody)) { + res.status(400).json({ error: 'Invalid public feed settings' }); + return; + } + const enabled = (rawBody as { enabled?: unknown }).enabled; + if (typeof enabled !== 'boolean') { + res.status(400).json({ error: 'Invalid public feed settings' }); + return; + } + const config = getPublicFeedConfig(); + const currentStatus = await publicFeedRepository.accountStatus( + session.mqttUsername, + ownedNodeIds, + ); + if (enabled && !currentStatus.eligible) { + res.status(409).json({ error: 'Publish at least one eligible packet in the last 24 hours before enabling the public feed' }); + return; + } + if (enabled && !config.configured) { + res.status(409).json({ error: 'Public feed is not configured by the operator' }); + return; + } + await publicFeedRepository.setEnabled(session.mqttUsername, ownedNodeIds, enabled); + if (deps.reconcileOwnerAuthorization) { + try { + await deps.reconcileOwnerAuthorization(); + } catch (error) { + console.error('[api] public-feed ACL reconciliation failed', (error as Error).message); + res.status(503).json({ error: 'Public feed access could not be updated yet' }); + return; + } + } + const status = await publicFeedRepository.accountStatus( + session.mqttUsername, + ownedNodeIds, + ); + const topicFilters = config.configured ? [PUBLIC_FEED_TOPIC_FILTER] : []; + res.status(200).json({ + featureEnabled: true, + configured: config.configured, + brokerUrl: config.brokerUrl, + enabled: status.enabled, + nodeCount: status.nodeCount, + eligible: status.eligible, + recentPacketCount: status.recentPacketCount, + topicFilters, + lastForwardedAt: status.lastForwardedAt, + }); + } catch (error) { + console.error('[api] POST /owner/public-feed', (error as Error).message); + res.status(500).json({ error: 'Internal server error' }); + } + }); + router.get('/owner/alert-rules', async (req, res) => { try { const ownedNodeIds = await deps.requireOwnerSession(req, res); @@ -554,5 +639,11 @@ export function registerOwnerRoutes(router: Router, deps: OwnerRouteDeps) { }); res.json({ ok: true }); }); - return prewarm; +} + +// Exposes the registered owner service so the app bootstrap can start background +// work (e.g. the owner last-hop cache prewarm) against the same instance the +// routes use. Returns null when the routes have not been registered yet. +export function getRegisteredOwnerService(): OwnerService | null { + return registeredOwnerService; } diff --git a/backend/src/api/routes/ownerPacketSharing.test.ts b/backend/src/api/routes/ownerPacketSharing.test.ts index 9df24d1..3e46aab 100644 --- a/backend/src/api/routes/ownerPacketSharing.test.ts +++ b/backend/src/api/routes/ownerPacketSharing.test.ts @@ -23,7 +23,6 @@ const passThroughLimiter: RequestHandler = (_req, _res, next) => next(); function ownerRouteDeps( query: (sql: string, params?: unknown[]) => Promise<{ rows: QueryResultRow[] }>, mqttUsername = 'node1', - requireOwnerSession: (req: import('express').Request, res: import('express').Response) => Promise = async () => [nodeId], ) { return { ownerCookieName: 'meshcore_owner_session', @@ -43,7 +42,7 @@ function ownerRouteDeps( encryptOwnerSession: () => 'session', isSecureRequest: () => false, getOwnerSession: () => ({ v: 3 as const, mqttUsername, exp: Date.now() + 60_000, gen: 0 }), - requireOwnerSession, + requireOwnerSession: async () => [nodeId], getOwnerCredentialGeneration: async () => 0, invalidateOwnerNodeIdCache: () => undefined, query, @@ -55,7 +54,8 @@ async function withOwnerApi( query: (sql: string, params?: unknown[]) => Promise<{ rows: QueryResultRow[] }>, check: (baseUrl: string, writes: string[]) => Promise, selectedDestinationRows: QueryResultRow[] = [{ destination_id: 'remote-site' }], - requireOwnerSession?: (req: import('express').Request, res: import('express').Response) => Promise, + mqttUsername = 'node1', + destinationRows: QueryResultRow[] = [destination], ) { const writes: string[] = []; const router = Router(); @@ -69,13 +69,13 @@ async function withOwnerApi( : { rows: [] }; } if (sql.includes('FROM owner_packet_share_destinations')) { - return { rows: [destination] }; + return { rows: destinationRows }; } if (sql.includes('FROM owner_packet_share_rules')) { return { rows: selectedDestinationRows }; } return query(sql, params); - }, 'node1', requireOwnerSession)); + }, mqttUsername)); const app = express(); app.use(express.json()); app.use('/api', router); @@ -136,19 +136,6 @@ test('owner packet-sharing settings expose destinations and persist a selected d assert.equal(postResponse.status, 200); assert.deepEqual(await postResponse.json(), { nodeId, enabled: true, destinationIds: ['remote-site'] }); assert.equal(writes.length, 3); - - const disableResponse = await fetch(`${baseUrl}/api/owner/packet-sharing`, { - method: 'POST', - headers: { - 'content-type': 'application/json', - cookie: 'meshcore_owner_csrf=test-token', - 'x-csrf-token': 'test-token', - }, - body: JSON.stringify({ nodeId, enabled: false, destinationIds: [] }), - }); - assert.equal(disableResponse.status, 200); - assert.deepEqual(await disableResponse.json(), { nodeId, enabled: false, destinationIds: [] }); - assert.equal(writes.length, 5); }); }); @@ -191,39 +178,54 @@ test('owner packet-sharing settings are available to every authenticated owner a } }); -test('owner packet-sharing rejects nodes outside the authenticated session', async () => { - await withOwnerApi(async () => ({ rows: [] }), async (baseUrl, writes) => { - const response = await fetch(`${baseUrl}/api/owner/packet-sharing?nodeId=${'B'.repeat(64)}`); - assert.equal(response.status, 403); - assert.deepEqual(await response.json(), { error: 'Node is not owned by this session' }); - assert.equal(writes.length, 0); - }); -}); +const scotmesh = { + ...destination, + destination_id: 'scotmesh', + display_name: 'ScotMesh', + broker_url: 'wss://scotmesh-mqtt.mm7roq.compute.oarc.uk:443/', + topic_template: 'meshcore/MME/{observerId}/packets', + allowed_owner_username: 'node1', + allowed_node_id: nodeId, + auth_mode: 'meshcore_jwt', + auth_audience: 'scotmesh-mqtt.mm7roq.compute.oarc.uk', + tx_mode: 'advert', +}; -test('owner packet-sharing requires an authenticated session', async () => { +test('ScotMesh is visible to node1 while pending signing credentials', async () => { await withOwnerApi(async () => ({ rows: [] }), async (baseUrl) => { - const response = await fetch(`${baseUrl}/api/owner/packet-sharing`); - assert.equal(response.status, 401); - assert.deepEqual(await response.json(), { error: 'Unauthorized' }); - }, [{ destination_id: 'remote-site' }], async (_req, res) => { - res.status(401).json({ error: 'Unauthorized' }); - return null; - }); + const response = await fetch(`${baseUrl}/api/owner/packet-sharing?nodeId=${nodeId}`); + const body = await response.json(); + assert.equal(response.status, 200); + assert.equal(body.destinations[0].id, 'scotmesh'); + assert.equal(body.destinations[0].configured, false); + assert.equal(body.destinations[0].selected, false); + assert.equal(JSON.stringify(body).includes('broker_'), false); + }, [], 'node1', [scotmesh]); }); -test('owner packet-sharing rejects malformed settings without database writes', async () => { - await withOwnerApi(async () => ({ rows: [] }), async (baseUrl, writes) => { - const response = await fetch(`${baseUrl}/api/owner/packet-sharing`, { - method: 'POST', - headers: { - 'content-type': 'application/json', - cookie: 'meshcore_owner_csrf=test-token', - 'x-csrf-token': 'test-token', - }, - body: JSON.stringify({ nodeId, enabled: true, destinationIds: ['remote-site'], unexpected: true }), - }); - assert.equal(response.status, 400); - assert.deepEqual(await response.json(), { error: 'Invalid packet sharing settings' }); - assert.equal(writes.length, 0); +for (const username of ['node2', 'gnomemoe', 'NODE1', 'node1-other']) { + test(`ScotMesh is absent from ${username}'s API response, including stale selections`, async () => { + await withOwnerApi(async () => ({ rows: [] }), async (baseUrl) => { + const response = await fetch(`${baseUrl}/api/owner/packet-sharing?nodeId=${nodeId}`); + assert.equal(response.status, 200); + assert.equal(response.headers.get('cache-control'), 'no-store'); + assert.deepEqual(await response.json(), { nodeId, featureEnabled: true, enabled: false, destinations: [] }); + for (const enabled of [true, false]) { + const save = await fetch(`${baseUrl}/api/owner/packet-sharing`, { + method: 'POST', + headers: { 'content-type': 'application/json', cookie: 'meshcore_owner_csrf=token', 'x-csrf-token': 'token' }, + body: JSON.stringify({ nodeId, enabled, destinationIds: ['scotmesh'] }), + }); + assert.equal(save.status, 409); + assert.deepEqual(await save.json(), { error: 'One or more selected destinations are not configured' }); + } + }, [{ destination_id: 'scotmesh' }], username, [scotmesh]); }); +} + +test('ScotMesh is absent for a different node owned by node1', async () => { + await withOwnerApi(async () => ({ rows: [] }), async (baseUrl) => { + const response = await fetch(`${baseUrl}/api/owner/packet-sharing?nodeId=${nodeId}`); + assert.deepEqual((await response.json()).destinations, []); + }, [], 'node1', [{ ...scotmesh, allowed_node_id: 'C'.repeat(64) }]); }); diff --git a/backend/src/api/routes/ownerPublicFeed.test.ts b/backend/src/api/routes/ownerPublicFeed.test.ts new file mode 100644 index 0000000..7cfb7ce --- /dev/null +++ b/backend/src/api/routes/ownerPublicFeed.test.ts @@ -0,0 +1,185 @@ +import assert from 'node:assert/strict'; +import { once } from 'node:events'; +import type { AddressInfo } from 'node:net'; +import test from 'node:test'; +import express, { Router, type RequestHandler } from 'express'; +import type { QueryResultRow } from 'pg'; +import { registerOwnerRoutes } from './owner.js'; + +const nodeId = 'A'.repeat(64); +const passThroughLimiter: RequestHandler = (_req, _res, next) => next(); + +function ownerRouteDeps( + query: (sql: string, params?: unknown[]) => Promise<{ rows: QueryResultRow[] }>, + reconcileOwnerAuthorization?: () => Promise, +) { + return { + ownerCookieName: 'meshcore_owner_session', + ownerLiveCacheTtlMs: 1_000, + ownerLiveCache: new Map(), + ownerDashboardCacheTtlMs: 1_000, + ownerLastHopCacheTtlMs: 1_000, + ownerSessionTtlMs: 60_000, + mqttUsernameMaxLen: 128, + mqttPasswordMaxLen: 128, + ownerLoginLimiter: passThroughLimiter, + hasControlChars: () => false, + verifyMqttCredentials: async () => true, + resolveOwnerNodeIds: async () => [nodeId], + autoLinkOwnerNodeIds: async () => [nodeId], + buildOwnerDashboard: async () => ({ nodes: [] }), + encryptOwnerSession: () => 'session', + isSecureRequest: () => false, + getOwnerSession: () => ({ v: 3 as const, mqttUsername: 'node1', exp: Date.now() + 60_000, gen: 0 }), + requireOwnerSession: async () => [nodeId], + getOwnerCredentialGeneration: async () => 0, + invalidateOwnerNodeIdCache: () => undefined, + reconcileOwnerAuthorization, + query, + withTransaction: async (work: (runQuery: typeof query) => Promise) => work(query), + }; +} + +async function withOwnerApi( + query: (sql: string, params?: unknown[]) => Promise<{ rows: QueryResultRow[] }>, + check: (baseUrl: string) => Promise, + reconcileOwnerAuthorization?: () => Promise, +) { + const router = Router(); + registerOwnerRoutes(router, ownerRouteDeps(query, reconcileOwnerAuthorization)); + const app = express(); + app.use(express.json()); + app.use('/api', router); + const server = app.listen(0, '127.0.0.1'); + await once(server, 'listening'); + try { + await check(`http://127.0.0.1:${(server.address() as AddressInfo).port}`); + } finally { + server.closeAllConnections(); + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } +} + +test('public feed settings are account-level and default off', async () => { + await withOwnerApi(async (sql) => { + if (sql.includes('COALESCE(settings.enabled')) { + return { rows: [{ enabled: false, last_forwarded_at: null }] }; + } + return { rows: [] }; + }, async (baseUrl) => { + const response = await fetch(`${baseUrl}/api/owner/public-feed`); + assert.equal(response.status, 200); + assert.equal(response.headers.get('cache-control'), 'no-store'); + assert.deepEqual(await response.json(), { + featureEnabled: true, + configured: false, + brokerUrl: null, + enabled: false, + nodeCount: 1, + eligible: false, + recentPacketCount: 0, + topicFilters: [], + lastForwardedAt: null, + }); + }); +}); + +test('public feed rejects enabling until the operator supplies a broker configuration', async () => { + await withOwnerApi(async (sql) => { + if (sql.includes('COALESCE(settings.enabled')) { + return { rows: [{ enabled: false, recent_packet_count: 1, last_forwarded_at: null }] }; + } + return { rows: [] }; + }, async (baseUrl) => { + const response = await fetch(`${baseUrl}/api/owner/public-feed`, { + method: 'POST', + headers: { + 'content-type': 'application/json', + cookie: 'meshcore_owner_csrf=test-token', + 'x-csrf-token': 'test-token', + }, + body: JSON.stringify({ enabled: true }), + }); + assert.equal(response.status, 409); + assert.deepEqual(await response.json(), { error: 'Public feed is not configured by the operator' }); + }); +}); + +test('public feed rejects enabling when the owner has no eligible packet activity', async () => { + await withOwnerApi(async (sql) => { + if (sql.includes('COALESCE(settings.enabled')) { + return { rows: [{ enabled: false, recent_packet_count: 0, last_forwarded_at: null }] }; + } + return { rows: [] }; + }, async (baseUrl) => { + const response = await fetch(`${baseUrl}/api/owner/public-feed`, { + method: 'POST', + headers: { + 'content-type': 'application/json', + cookie: 'meshcore_owner_csrf=test-token', + 'x-csrf-token': 'test-token', + }, + body: JSON.stringify({ enabled: true }), + }); + assert.equal(response.status, 409); + assert.deepEqual(await response.json(), { + error: 'Publish at least one eligible packet in the last 24 hours before enabling the public feed', + }); + }); +}); + +test('public feed persists the account toggle and current owned-node mapping', async () => { + const previous = { ...process.env }; + Object.assign(process.env, { + PUBLIC_FEED_PUBLISH_ENABLED: 'true', + PUBLIC_FEED_BROKER_URL: 'mqtts://broker.example.test:8883', + PUBLIC_FEED_BROKER_USERNAME: 'publisher', + PUBLIC_FEED_BROKER_PASSWORD: 'unit-test-secret', + }); + let enabled = false; + const writes: string[] = []; + let aclReconciliations = 0; + try { + await withOwnerApi(async (sql) => { + if (sql.includes('COALESCE(settings.enabled')) { + return { rows: [{ enabled, recent_packet_count: 1, last_forwarded_at: null }] }; + } + if (sql.startsWith('INSERT INTO owner_public_feed_settings') + || sql.startsWith('DELETE FROM owner_public_feed_nodes') + || sql.startsWith('INSERT INTO owner_public_feed_nodes')) { + writes.push(sql); + if (sql.startsWith('INSERT INTO owner_public_feed_settings')) enabled = true; + } + return { rows: [] }; + }, async (baseUrl) => { + const response = await fetch(`${baseUrl}/api/owner/public-feed`, { + method: 'POST', + headers: { + 'content-type': 'application/json', + cookie: 'meshcore_owner_csrf=test-token', + 'x-csrf-token': 'test-token', + }, + body: JSON.stringify({ enabled: true }), + }); + assert.equal(response.status, 200); + assert.deepEqual(await response.json(), { + featureEnabled: true, + configured: true, + brokerUrl: 'mqtts://broker.example.test:8883', + enabled: true, + nodeCount: 1, + eligible: true, + recentPacketCount: 1, + topicFilters: ['public/+/+/packets'], + lastForwardedAt: null, + }); + assert.equal(writes.length, 3); + }, async () => { aclReconciliations += 1; }); + assert.equal(aclReconciliations, 1); + } finally { + for (const key of Object.keys(process.env)) { + if (!(key in previous)) delete process.env[key]; + } + Object.assign(process.env, previous); + } +}); diff --git a/backend/src/api/routes/productFeatures.test.ts b/backend/src/api/routes/productFeatures.test.ts new file mode 100644 index 0000000..9b7c8ec --- /dev/null +++ b/backend/src/api/routes/productFeatures.test.ts @@ -0,0 +1,82 @@ +import assert from 'node:assert/strict'; +import { once } from 'node:events'; +import type { AddressInfo } from 'node:net'; +import test from 'node:test'; +import express, { Router } from 'express'; +import type { QueryResultRow } from 'pg'; +import type { QueryFn } from '../../repositories/productFeatures.js'; +import { registerProductFeatureRoutes } from './productFeatures.js'; + +const nodeA = 'A'.repeat(64); +const nodeB = 'B'.repeat(64); + +async function withHistoryApi( + visibleIds: string[], + points: QueryResultRow[], + check: (baseUrl: string, historyParams: unknown[][]) => Promise, +) { + const historyParams: unknown[][] = []; + const query: QueryFn = async (sql: string, params?: unknown[]) => { + if (sql.includes('FROM node_identity_nodes')) { + return { rows: visibleIds.map((node_id) => ({ node_id })) as unknown as T[] }; + } + historyParams.push(params ?? []); + return { rows: points as T[] }; + }; + const app = express(); + const router = Router(); + registerProductFeatureRoutes(router, query); + app.use('/api', router); + const server = app.listen(0, '127.0.0.1'); + await once(server, 'listening'); + try { + await check(`http://127.0.0.1:${(server.address() as AddressInfo).port}`, historyParams); + } finally { + server.closeAllConnections(); + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } +} + +test('link history preserves its response contract and forwards the public packet scope', async () => { + const points = [{ time: '2026-09-07T12:00:00.123Z', snr: -7, rssi: -99, path_loss: 110, sample_count: 1 }]; + await withHistoryApi([nodeA, nodeB], points, async (baseUrl, params) => { + const response = await fetch(`${baseUrl}/api/links/${nodeB.toLowerCase()}:${nodeA.toLowerCase()}/history`); + assert.equal(response.status, 200); + assert.equal(response.headers.get('cache-control'), 'public, max-age=60, stale-while-revalidate=300'); + assert.deepEqual(await response.json(), { linkId: `${nodeA}:${nodeB}`, network: 'ukmesh', hours: 72, points }); + assert.deepEqual(params, [[nodeA, nodeB, '72', ['ukmesh', 'northeast', 'teesside']]]); + }); +}); + +test('a fabricated pairing of visible nodes with no traffic returns HTTP 200 and empty points', async () => { + await withHistoryApi([nodeA, nodeB], [], async (baseUrl, params) => { + const response = await fetch(`${baseUrl}/api/links/${nodeA}:${nodeB}/history?network=test&hours=168`); + assert.equal(response.status, 200); + assert.deepEqual(await response.json(), { linkId: `${nodeA}:${nodeB}`, network: 'test', hours: 168, points: [] }); + assert.deepEqual(params, [[nodeA, nodeB, '168', ['test']]]); + }); +}); + +test('history never reads packets when either endpoint is outside visibility scope', async () => { + await withHistoryApi([nodeA], [], async (baseUrl, params) => { + const response = await fetch(`${baseUrl}/api/links/${nodeA}:${nodeB}/history?hours=72`); + assert.equal(response.status, 404); + assert.deepEqual(await response.json(), { error: 'Link not found' }); + assert.deepEqual(params, []); + }); +}); + +test('history rejects windows above 168 hours before querying packets', async () => { + type Handler = (req: unknown, res: unknown) => Promise; + const handlers = new Map(); + const router = { + get: (path: string, handler: Handler) => handlers.set(path, handler), + post: () => undefined, + } as unknown as Router; + registerProductFeatureRoutes(router, async () => { + assert.fail('Invalid windows must not reach the database'); + }); + await assert.rejects(handlers.get('/links/:id/history')!({ + params: { id: `${nodeA}:${nodeB}` }, query: { hours: '169' }, + }, {}), /hours must be between 1 and 168/); +}); diff --git a/backend/src/api/routes/productFeatures.ts b/backend/src/api/routes/productFeatures.ts index 9d3ffb6..e071d67 100644 --- a/backend/src/api/routes/productFeatures.ts +++ b/backend/src/api/routes/productFeatures.ts @@ -85,7 +85,7 @@ export function registerProductFeatureRoutes(router: Router, query: QueryFn): vo res.status(404).json({ error: 'Link not found' }); return; } - const result = await linkHistoryRows(query, nodeA, nodeB, hours); + const result = await linkHistoryRows(query, nodeA, nodeB, hours, networks); res.setHeader('Cache-Control', 'public, max-age=60, stale-while-revalidate=300'); res.json({ linkId: `${nodeA}:${nodeB}`, network, hours, points: result.rows }); } catch (error) { diff --git a/backend/src/api/routes/regions.test.ts b/backend/src/api/routes/regions.test.ts new file mode 100644 index 0000000..2beda89 --- /dev/null +++ b/backend/src/api/routes/regions.test.ts @@ -0,0 +1,166 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { + confidenceFor, + isWithinUkDisplayEnvelope, + shapeRegionRepeatersPayload, + shapeRegionScopePayload, + shapeRegionSummaryPayload, + type RegionMemberRow, + type RegionStatRow, +} from './regions.js'; + +const NODE_ID = 'A'.repeat(64); +const PRIVATE_ID = 'B'.repeat(64); +const OTHER_ID = 'C'.repeat(64); + +test('confidenceFor applies confirmed / probable / single tiers', () => { + assert.equal(confidenceFor(12), 'confirmed'); + assert.equal(confidenceFor(4), 'confirmed'); + assert.equal(confidenceFor(3), 'probable'); + assert.equal(confidenceFor(2), 'probable'); + assert.equal(confidenceFor(1), 'single'); + assert.equal(confidenceFor(0), 'single'); +}); + +test('isWithinUkDisplayEnvelope keeps UK positions and rejects foreign, junk and placeholder coordinates', () => { + assert.equal(isWithinUkDisplayEnvelope(54.2, -1.2), true); + assert.equal(isWithinUkDisplayEnvelope(54.6, -5.9), true); + assert.equal(isWithinUkDisplayEnvelope(60.1, -1.3), true); + assert.equal(isWithinUkDisplayEnvelope(59.2, 9.5), false); + assert.equal(isWithinUkDisplayEnvelope(0, 0), false); + assert.equal(isWithinUkDisplayEnvelope(1910, -1426), false); + assert.equal(isWithinUkDisplayEnvelope(null, -1), false); +}); + +test('shapeRegionSummaryPayload nests members, applies confidence and drops opted-out nodes', () => { + const payload = shapeRegionSummaryPayload( + { finished_at: new Date('2026-10-06T12:00:00.000Z'), window_days: 92 }, + [ + { + region: 'yorkshire', packets_total: 8967, member_count: 117, repeater_member_count: 116, + first_seen: new Date('2026-09-05T00:01:44.000Z'), last_seen: new Date('2026-10-06T00:00:00.000Z'), + }, + { + region: 'ren', packets_total: 4, member_count: 23, repeater_member_count: 21, + first_seen: null, last_seen: null, + }, + ], + [ + { region: 'yorkshire', node_id: NODE_ID, packets: 7416, name: 'Hub One', role: 2, lat: 53.9, lon: -1.7 }, + { region: 'yorkshire', node_id: PRIVATE_ID, packets: 900, name: 'Opted Out 🚫', role: 2, lat: 54.1, lon: -1.2 }, + { region: 'yorkshire', node_id: OTHER_ID, packets: 120, name: null, role: null, lat: null, lon: null }, + { region: 'yorkshire', node_id: 'D'.repeat(64), packets: 50, name: 'Nordic Node', role: 2, lat: 59.2, lon: 9.5 }, + { region: 'yorkshire', node_id: 'E'.repeat(64), packets: 40, name: 'Null Island', role: 2, lat: 0, lon: 0 }, + ], + ); + assert.equal(payload.generatedAt, '2026-10-06T12:00:00.000Z'); + assert.equal(payload.windowDays, 92); + assert.equal(payload.regions.length, 2); + const yorkshire = payload.regions[0]!; + assert.equal(yorkshire.name, 'yorkshire'); + assert.equal(yorkshire.confidence, 'confirmed'); + assert.equal(yorkshire.firstSeen, '2026-09-05T00:01:44.000Z'); + assert.equal(yorkshire.members.length, 2); + assert.deepEqual(yorkshire.members.map((member) => member.nodeId), [NODE_ID, OTHER_ID]); + assert.equal(yorkshire.members.every((member) => member.isBridge === false), true); + assert.equal(payload.regions[1]?.confidence, 'confirmed'); +}); + +test('shapeRegionSummaryPayload tolerates a missing run row', () => { + const payload = shapeRegionSummaryPayload(null, [], []); + assert.equal(payload.generatedAt, null); + assert.equal(payload.windowDays, null); + assert.deepEqual(payload.regions, []); +}); + +test('shapeRegionRepeatersPayload filters opted-out nodes and normalises regions json', () => { + const payload = shapeRegionRepeatersPayload( + { finished_at: '2026-10-06T12:00:00.000Z', window_days: 92 }, + [ + { + node_id: NODE_ID, status: 'green', evidence: 'scoped', scoped_packets: 7781, unscoped_packets_7d: 123, + regions: { yorkshire: 7416, 'eng-ne': 220, junk: 0 as unknown as number }, name: 'Hub One', role: 2, lat: 53.9, lon: -1.7, + }, + { + node_id: PRIVATE_ID, status: 'green', evidence: 'scoped', scoped_packets: 10, unscoped_packets_7d: 2, + regions: { yorkshire: 10 }, name: 'Hidden 🚫', role: 2, lat: 54, lon: -1, + }, + { + node_id: OTHER_ID, status: 'red', evidence: 'unscoped-only', scoped_packets: 0, unscoped_packets_7d: 44, + regions: {}, name: 'Quiet Repeater', role: 2, lat: 53, lon: -2, + }, + { + node_id: 'F'.repeat(64), status: 'red', evidence: 'unscoped-only', scoped_packets: 0, unscoped_packets_7d: 900, + regions: {}, name: 'DK_7620_Lemvig_R02', role: 2, lat: 56.52, lon: 8.22, + }, + ], + ); + assert.equal(payload.repeaters.length, 2); + assert.deepEqual(payload.repeaters[0]?.regions, { yorkshire: 7416, 'eng-ne': 220 }); + assert.equal(payload.repeaters[1]?.status, 'red'); + assert.equal(payload.repeaters[1]?.evidence, 'unscoped-only'); +}); + +test('shapeRegionScopePayload reports green, unobserved and missing states', () => { + const green = shapeRegionScopePayload( + null, + { + node_id: NODE_ID, status: 'green', evidence: 'scoped', scoped_packets: 6816, unscoped_packets_7d: 54, + regions: { yorkshire: 6816 }, name: 'Hub One', role: 2, lat: 53.9, lon: -1.7, + }, + [{ region: 'yorkshire', packets: 6816 }, { region: 'eng-ne', packets: 33 }], + ); + assert.equal(green?.status, 'green'); + assert.deepEqual(green?.regions, [{ region: 'yorkshire', packets: 6816 }, { region: 'eng-ne', packets: 33 }]); + + const unobserved = shapeRegionScopePayload( + { node_id: OTHER_ID, name: 'New Repeater', role: 2, lat: 52, lon: -1 }, + null, + [], + ); + assert.equal(unobserved?.status, 'unobserved'); + assert.equal(unobserved?.scopedPackets, 0); + assert.deepEqual(unobserved?.regions, []); + + const redacted = shapeRegionScopePayload( + null, + { + node_id: PRIVATE_ID, status: 'red', evidence: 'unscoped-only', scoped_packets: 0, unscoped_packets_7d: 7, + regions: {}, name: 'Hidden 🚫', role: 2, lat: 54, lon: -1, + }, + [], + ); + assert.equal(redacted?.name, 'Private Node'); + assert.equal(redacted?.lat, null); + assert.equal(redacted?.lon, null); + assert.equal(redacted?.nodeId, PRIVATE_ID); + + assert.equal(shapeRegionScopePayload(null, null, []), null); +}); + +test('shapeRegionSummaryPayload flags universal bridge members at the 12-region threshold', () => { + const stats: RegionStatRow[] = []; + const members: RegionMemberRow[] = []; + for (let index = 0; index < 12; index += 1) { + const region = `bridge-${index}`; + stats.push({ + region, packets_total: 10, member_count: 2, repeater_member_count: 2, + first_seen: null, last_seen: null, + }); + members.push({ region, node_id: NODE_ID, packets: 10, name: 'Universal Hub', role: 2, lat: 54, lon: -1 }); + } + for (let index = 0; index < 11; index += 1) { + const region = `local-${index}`; + stats.push({ + region, packets_total: 5, member_count: 1, repeater_member_count: 1, + first_seen: null, last_seen: null, + }); + members.push({ region, node_id: OTHER_ID, packets: 5, name: 'Local Repeater', role: 2, lat: 53, lon: -2 }); + } + const payload = shapeRegionSummaryPayload(null, stats, members); + const bridgeRegion = payload.regions.find((region) => region.name === 'bridge-0'); + const localRegion = payload.regions.find((region) => region.name === 'local-0'); + assert.equal(bridgeRegion?.members[0]?.isBridge, true); + assert.equal(localRegion?.members[0]?.isBridge, false); +}); diff --git a/backend/src/api/routes/regions.ts b/backend/src/api/routes/regions.ts new file mode 100644 index 0000000..c59d197 --- /dev/null +++ b/backend/src/api/routes/regions.ts @@ -0,0 +1,408 @@ +import type { Router } from 'express'; +import type { QueryResultRow } from 'pg'; +import { resolvePublicNetworkScope } from '../../http/requestScope.js'; +import type { NetworkFilters } from '../utils/networkFilters.js'; +import { isPrivateNode } from '../utils/privateNode.js'; + +type QueryFn = ( + text: string, + params?: unknown[], +) => Promise<{ rows: T[] }>; + +export type RegionConfidence = 'confirmed' | 'probable' | 'single'; + +export type RegionMemberDto = { + nodeId: string; + name: string | null; + role: number | null; + lat: number | null; + lon: number | null; + packets: number; + isBridge: boolean; +}; + +export type RegionSummaryDto = { + name: string; + packetsTotal: number; + memberCount: number; + repeaterMemberCount: number; + confidence: RegionConfidence; + firstSeen: string | null; + lastSeen: string | null; + members: RegionMemberDto[]; +}; + +export type RegionRepeaterDto = { + nodeId: string; + name: string | null; + role: number | null; + lat: number | null; + lon: number | null; + status: 'green' | 'red'; + evidence: string; + scopedPackets: number; + unscopedPackets7d: number; + regions: Record; +}; + +export type RegionScopeDto = { + nodeId: string; + name: string | null; + role: number | null; + lat: number | null; + lon: number | null; + status: 'green' | 'red' | 'unobserved'; + evidence: string; + scopedPackets: number; + unscopedPackets7d: number; + regions: Array<{ region: string; packets: number }>; +}; + +/** + * Evidence tiers for a region's decoded packet count. The 2026-10 audit + * calibrated the chance-match floor at ~0.2 false hits per candidate name over + * the whole corpus: >=4 matches is cryptographically solid, 2-3 is plausible + * but not proven, a single match may be chance. + */ +export function confidenceFor(packetsTotal: number): RegionConfidence { + if (packetsTotal >= 4) return 'confirmed'; + if (packetsTotal >= 2) return 'probable'; + return 'single'; +} + +/** + * Display envelope for the UK regions layer. The mesh dataset also carries + * nodes from other deployments (Nordic SkyMesh, continental Europe) and a + * small number of junk/duplicate rows with placeholder coordinates, so the + * public UK map only renders nodes inside the UK. Deliberately coarse + * (GB + Northern Ireland); the Channel Islands are excluded. + */ +export const UK_DISPLAY_ENVELOPE = { + minLat: 49.5, + maxLat: 61.2, + minLon: -8.8, + maxLon: 2.2, +} as const; + +export function isWithinUkDisplayEnvelope(lat: number | null, lon: number | null): boolean { + if (lat == null || lon == null) return false; + if (!Number.isFinite(lat) || !Number.isFinite(lon)) return false; + // Reject null-island and similar placeholder coordinates. + if (Math.abs(lat) < 1e-6 && Math.abs(lon) < 1e-6) return false; + return lat >= UK_DISPLAY_ENVELOPE.minLat && lat <= UK_DISPLAY_ENVELOPE.maxLat + && lon >= UK_DISPLAY_ENVELOPE.minLon && lon <= UK_DISPLAY_ENVELOPE.maxLon; +} + +/** True only for rows carrying coordinates that fall outside the UK envelope. */ +function excludedByLocation(lat: number | null, lon: number | null): boolean { + return lat != null && lon != null && !isWithinUkDisplayEnvelope(lat, lon); +} + +interface RunRow extends QueryResultRow { + finished_at: Date | string | null; + window_days: number | null; +} + +export interface RegionStatRow extends QueryResultRow { + region: string; + packets_total: number; + member_count: number; + repeater_member_count: number; + first_seen: Date | string | null; + last_seen: Date | string | null; +} + +export interface RegionMemberRow extends QueryResultRow { + region: string; + node_id: string; + packets: number; + name: string | null; + role: number | null; + lat: number | null; + lon: number | null; +} + +export interface RegionRepeaterRow extends QueryResultRow { + node_id: string; + status: string; + evidence: string; + scoped_packets: number; + unscoped_packets_7d: number; + regions: Record | null; + name: string | null; + role: number | null; + lat: number | null; + lon: number | null; +} + +export interface RegionScopeNodeRow extends QueryResultRow { + node_id: string; + name: string | null; + role: number | null; + lat: number | null; + lon: number | null; +} + +interface ScopeMemberRow extends QueryResultRow { + region: string; + packets: number; +} + +function isoOrNull(value: Date | string | null | undefined): string | null { + if (value == null) return null; + const date = value instanceof Date ? value : new Date(value); + return Number.isFinite(date.getTime()) ? date.toISOString() : null; +} + +function numberOrNull(value: number | null | undefined): number | null { + return value == null ? null : Number(value); +} + +function normalizeRegions(value: Record | null): Record { + // node-postgres parses jsonb, but be defensive about non-numeric members. + const regions: Record = {}; + for (const [name, count] of Object.entries(value ?? {})) { + const numeric = Number(count); + if (Number.isFinite(numeric) && numeric > 0) regions[name] = numeric; + } + return regions; +} + +/** + * Universal "bridge" repeaters are configured to relay scoped traffic for most + * of the network's regions (the live distribution puts them at 17+ regions, + * with a clean gap down to 11 for ordinary multi-region repeaters), so their + * position carries no signal about any single region's footprint. Region + * shapes exclude bridge members; member lists keep them — they are real relays. + */ +export const BRIDGE_MIN_REGIONS = 12; + +export function shapeRegionSummaryPayload( + run: RunRow | null, + stats: RegionStatRow[], + memberRows: RegionMemberRow[], +): { generatedAt: string | null; windowDays: number | null; regions: RegionSummaryDto[] } { + const membersByRegion = new Map(); + for (const row of memberRows) { + if (isPrivateNode(row.name)) continue; + if (excludedByLocation(row.lat, row.lon)) continue; + const list = membersByRegion.get(row.region) ?? []; + list.push({ + nodeId: row.node_id, + name: row.name, + role: numberOrNull(row.role), + lat: numberOrNull(row.lat), + lon: numberOrNull(row.lon), + packets: Number(row.packets), + isBridge: false, + }); + membersByRegion.set(row.region, list); + } + // A node appears at most once per region list, so counting occurrences + // across lists yields its distinct-region membership count. + const regionsPerNode = new Map(); + for (const list of membersByRegion.values()) { + for (const member of list) { + regionsPerNode.set(member.nodeId, (regionsPerNode.get(member.nodeId) ?? 0) + 1); + } + } + for (const list of membersByRegion.values()) { + for (const member of list) { + member.isBridge = (regionsPerNode.get(member.nodeId) ?? 0) >= BRIDGE_MIN_REGIONS; + } + } + return { + generatedAt: isoOrNull(run?.finished_at), + windowDays: run?.window_days == null ? null : Number(run.window_days), + regions: stats.map((row) => ({ + name: row.region, + packetsTotal: Number(row.packets_total), + memberCount: Number(row.member_count), + repeaterMemberCount: Number(row.repeater_member_count), + confidence: confidenceFor(Number(row.packets_total)), + firstSeen: isoOrNull(row.first_seen), + lastSeen: isoOrNull(row.last_seen), + members: membersByRegion.get(row.region) ?? [], + })), + }; +} + +export function shapeRegionRepeatersPayload( + run: RunRow | null, + rows: RegionRepeaterRow[], +): { generatedAt: string | null; repeaters: RegionRepeaterDto[] } { + return { + generatedAt: isoOrNull(run?.finished_at), + repeaters: rows + .filter((row) => !isPrivateNode(row.name)) + .filter((row) => !excludedByLocation(row.lat, row.lon)) + .map((row) => ({ + nodeId: row.node_id, + name: row.name, + role: numberOrNull(row.role), + lat: numberOrNull(row.lat), + lon: numberOrNull(row.lon), + status: row.status === 'green' ? ('green' as const) : ('red' as const), + evidence: row.evidence, + scopedPackets: Number(row.scoped_packets), + unscopedPackets7d: Number(row.unscoped_packets_7d), + regions: normalizeRegions(row.regions), + })), + }; +} + +export function shapeRegionScopePayload( + nodeRow: RegionScopeNodeRow | null, + statusRow: RegionRepeaterRow | null, + memberRows: ScopeMemberRow[], +): RegionScopeDto | null { + if (!nodeRow && !statusRow) return null; + const nodeId = (statusRow?.node_id ?? nodeRow?.node_id ?? '').toUpperCase(); + let name = statusRow?.name ?? nodeRow?.name ?? null; + let lat = statusRow?.lat ?? nodeRow?.lat ?? null; + let lon = statusRow?.lon ?? nodeRow?.lon ?? null; + if (isPrivateNode(name)) { + // Public geometry must never be derived from opted-out nodes. + name = 'Private Node'; + lat = null; + lon = null; + } + return { + nodeId, + name, + role: numberOrNull(statusRow?.role ?? nodeRow?.role), + lat: numberOrNull(lat), + lon: numberOrNull(lon), + status: statusRow ? (statusRow.status === 'green' ? 'green' : 'red') : 'unobserved', + evidence: statusRow?.evidence ?? 'none', + scopedPackets: Number(statusRow?.scoped_packets ?? 0), + unscopedPackets7d: Number(statusRow?.unscoped_packets_7d ?? 0), + regions: memberRows.map((row) => ({ region: row.region, packets: Number(row.packets) })), + }; +} + +type RegionsRouteDeps = { + query: QueryFn; + networkFilters: (network?: string, observer?: string) => NetworkFilters; + limiter: ReturnType; +}; + +const REGIONS_CACHE_CONTROL = 'public, max-age=300, stale-while-revalidate=900'; + +// Nodes can briefly hold more than one row per network; pick the most recently +// seen row for stable public metadata. The network predicate comes from the +// shared networkFilters helper so the public ukmesh scope keeps the historical +// northeast/teesside labels in scope (matching every other public route). +function nodeMetaLateral(alias: string, filters: NetworkFilters): string { + return `LEFT JOIN LATERAL ( + SELECT n.name, n.role, n.lat, n.lon + FROM nodes n + WHERE n.node_id = ${alias}.node_id ${filters.nodesAlias('n')} + ORDER BY n.last_seen DESC NULLS LAST + LIMIT 1 + ) meta ON TRUE`; +} + +export function registerRegionsRoutes(router: Router, deps: RegionsRouteDeps): void { + router.get('/regions/summary', deps.limiter, async (req, res) => { + try { + const network = resolvePublicNetworkScope(req.query['network'], req.headers); + const filters = deps.networkFilters(network); + const [run, stats, members] = await Promise.all([ + deps.query('SELECT finished_at, window_days FROM region_scope_run LIMIT 1'), + deps.query( + `SELECT region, packets_total, member_count, repeater_member_count, first_seen, last_seen + FROM region_scope_stats + ORDER BY packets_total DESC, region ASC`, + ), + deps.query( + `SELECT m.region, m.node_id, m.packets, meta.name, meta.role, meta.lat, meta.lon + FROM region_scope_members m + ${nodeMetaLateral('m', filters)} + ORDER BY m.region ASC, m.packets DESC, m.node_id ASC`, + [...filters.params], + ), + ]); + res.setHeader('Cache-Control', REGIONS_CACHE_CONTROL); + res.json(shapeRegionSummaryPayload(run.rows[0] ?? null, stats.rows, members.rows)); + } catch (err) { + console.error('[api] GET /regions/summary', (err as Error).message); + res.status(500).json({ error: 'Internal server error' }); + } + }); + + router.get('/regions/repeaters', deps.limiter, async (req, res) => { + try { + const network = resolvePublicNetworkScope(req.query['network'], req.headers); + const filters = deps.networkFilters(network); + const [run, repeaters] = await Promise.all([ + deps.query('SELECT finished_at, window_days FROM region_scope_run LIMIT 1'), + deps.query( + `SELECT s.node_id, s.status, s.evidence, s.scoped_packets, s.unscoped_packets_7d, s.regions, + meta.name, meta.role, meta.lat, meta.lon + FROM repeater_scope_status s + ${nodeMetaLateral('s', filters)} + ORDER BY s.scoped_packets DESC, s.node_id ASC`, + [...filters.params], + ), + ]); + res.setHeader('Cache-Control', REGIONS_CACHE_CONTROL); + res.json(shapeRegionRepeatersPayload(run.rows[0] ?? null, repeaters.rows)); + } catch (err) { + console.error('[api] GET /regions/repeaters', (err as Error).message); + res.status(500).json({ error: 'Internal server error' }); + } + }); + + router.get('/repeaters/:id/scope', deps.limiter, async (req, res) => { + const rawId = String(req.params['id'] ?? '').trim().toUpperCase(); + if (!/^[0-9A-F]{64}$/.test(rawId)) { + res.status(400).json({ error: 'Invalid repeater id' }); + return; + } + try { + const network = resolvePublicNetworkScope(req.query['network'], req.headers); + const filters = deps.networkFilters(network); + const idParam = `$${filters.params.length + 1}`; + const [status, members, node] = await Promise.all([ + deps.query( + `SELECT s.node_id, s.status, s.evidence, s.scoped_packets, s.unscoped_packets_7d, s.regions, + meta.name, meta.role, meta.lat, meta.lon + FROM repeater_scope_status s + ${nodeMetaLateral('s', filters)} + WHERE s.node_id = ${idParam}`, + [...filters.params, rawId], + ), + deps.query( + `SELECT region, packets + FROM region_scope_members + WHERE node_id = $1 + ORDER BY packets DESC, region ASC`, + [rawId], + ), + deps.query( + `SELECT n.node_id, n.name, n.role, n.lat, n.lon + FROM nodes n + WHERE n.node_id = ${idParam} ${filters.nodesAlias('n')} + ORDER BY n.last_seen DESC NULLS LAST + LIMIT 1`, + [...filters.params, rawId], + ), + ]); + const payload = shapeRegionScopePayload( + node.rows[0] ?? null, + status.rows[0] ?? null, + members.rows, + ); + if (!payload) { + res.status(404).json({ error: 'Repeater not found' }); + return; + } + res.setHeader('Cache-Control', REGIONS_CACHE_CONTROL); + res.json(payload); + } catch (err) { + console.error('[api] GET /repeaters/:id/scope', (err as Error).message); + res.status(500).json({ error: 'Internal server error' }); + } + }); +} diff --git a/backend/src/api/routes/rfValidation.integration.test.ts b/backend/src/api/routes/rfValidation.integration.test.ts new file mode 100644 index 0000000..e2f3f50 --- /dev/null +++ b/backend/src/api/routes/rfValidation.integration.test.ts @@ -0,0 +1,22 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +// Opt in only after the authorized backend deployment. This is a GET against +// the actual route and its bounded read-only pool, without any DB writes. +test('live UKMesh RF validation returns populated results within five seconds at every supported size', { + skip: process.env['AUDITWAVE_LIVE_TESTS'] !== '1', + timeout: 20_000, +}, async () => { + for (const limit of [25, 100, 250]) { + const startedAt = performance.now(); + const response = await fetch(`http://127.0.0.1:3000/api/rf-validation?network=ukmesh&limit=${limit}`, { + signal: AbortSignal.timeout(5_000), + }); + const body = await response.json() as { summary: { evaluated: number } }; + const elapsedMs = performance.now() - startedAt; + assert.equal(response.status, 200); + assert.ok(elapsedMs < 5_000, `RF validation took ${elapsedMs.toFixed(0)}ms`); + assert.ok(body.summary.evaluated > 0 && body.summary.evaluated <= limit); + console.log(JSON.stringify({ limit, elapsedMs, evaluated: body.summary.evaluated })); + } +}); diff --git a/backend/src/api/routes/rfValidation.test.ts b/backend/src/api/routes/rfValidation.test.ts new file mode 100644 index 0000000..4a86693 --- /dev/null +++ b/backend/src/api/routes/rfValidation.test.ts @@ -0,0 +1,112 @@ +import assert from 'node:assert/strict'; +import { once } from 'node:events'; +import type { AddressInfo } from 'node:net'; +import test from 'node:test'; +import express, { Router, type RequestHandler } from 'express'; +import { networkFilters } from '../utils/networkFilters.js'; +import { registerRfValidationRoutes, RF_VALIDATION_REQUEST_TIMEOUT_MS } from './rfValidation.js'; + +const passThroughLimiter: RequestHandler = (_req, _res, next) => next(); +type RfQuery = NonNullable[2]>; +async function start(query: RfQuery) { + const router = Router(); + registerRfValidationRoutes(router, { + limiter: passThroughLimiter as Parameters[1]['limiter'], + networkFilters, + query: async () => { throw new Error('Must not use the unbounded shared query'); }, + }, query); + const app = express(); + app.use(router); + const server = app.listen(0, '127.0.0.1'); + await once(server, 'listening'); + return { + url: `http://127.0.0.1:${(server.address() as AddressInfo).port}/rf-validation?network=ukmesh&limit=100`, + close: async () => { + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + }, + }; +} + +for (const stage of [1, 2]) { + test(`RF validation propagates an HTTP disconnect during database read ${stage}`, { timeout: 3_000 }, async () => { + let calls = 0; + let resolveStarted!: () => void; + let resolveAborted!: () => void; + const started = new Promise((resolve) => { resolveStarted = resolve; }); + const aborted = new Promise((resolve) => { resolveAborted = resolve; }); + const server = await start(async (_sql, _params, signal) => { + assert.ok(signal); + if (++calls < stage) return { rows: [{ node_id: 'A', name: 'Alpha' }] as never[] }; + resolveStarted(); + return new Promise((_resolve, reject) => { + signal.addEventListener('abort', () => { + resolveAborted(); + reject(signal.reason); + }, { once: true }); + }); + }); + try { + const controller = new AbortController(); + const response = fetch(server.url, { signal: controller.signal }); + await started; + controller.abort(); + await assert.rejects(response, /aborted/i); + await aborted; + assert.equal(calls, stage); + } finally { await server.close(); } + }); +} + +test('RF deadline aborts the database operation and returns an uncached 503 within five seconds', { timeout: 6_000 }, async () => { + let aborted = false; + const server = await start(async (_sql, _params, signal) => new Promise((_resolve, reject) => { + signal!.addEventListener('abort', () => { aborted = true; reject(signal!.reason); }, { once: true }); + })); + try { + const started = performance.now(); + const response = await fetch(server.url); + assert.equal(response.status, 503); + assert.ok(aborted); + assert.ok(performance.now() - started < 5_000); + assert.equal(RF_VALIDATION_REQUEST_TIMEOUT_MS, 4_500); + assert.equal(response.headers.get('cache-control'), 'no-store'); + assert.equal(response.headers.get('retry-after'), '30'); + assert.match((await response.json() as { error: string }).error, /temporarily unavailable/); + } finally { await server.close(); } +}); + +test('RF server statement cancellation is reported as unavailable, not an empty successful panel', async () => { + const server = await start(async () => { throw Object.assign(new Error('statement cancelled'), { code: '57014' }); }); + try { + const response = await fetch(server.url); + assert.equal(response.status, 503); + assert.equal(response.headers.get('cache-control'), 'no-store'); + } finally { await server.close(); } +}); + +test('RF JSON keeps classification summaries and numeric counts, including an honest empty scope', async () => { + for (const empty of [false, true]) { + const server = await start(async (sql) => { + if (sql.includes('FROM node_identity_nodes')) { + return { rows: (empty ? [] : [{ node_id: 'A', name: 'Alpha' }, { node_id: 'B', name: null }]) as never[] }; + } + return { rows: [{ node_a_id: 'A', node_b_id: 'B', observed_count: '12', multibyte_observed_count: '3', + itm_path_loss_db: 130, itm_viable: false, force_viable: false, + last_observed: '2026-09-07T12:00:00Z', classification: 'observed_unexpected' }] as never[] }; + }); + try { + const response = await fetch(server.url); + assert.equal(response.status, 200); + const body = await response.json() as { windowDays: number; summary: { evaluated: number; observedUnexpected: number }; mismatches: Array<{ sourceName: string; observations: number; strongObservations: number }> }; + assert.equal(body.windowDays, 30); + assert.equal(body.summary.evaluated, empty ? 0 : 1); + assert.equal(body.summary.observedUnexpected, empty ? 0 : 1); + if (!empty) { + assert.equal(body.mismatches[0]!.sourceName, 'Alpha'); + assert.equal(body.mismatches[0]!.observations, 12); + assert.equal(body.mismatches[0]!.strongObservations, 3); + } + } finally { await server.close(); } + } +}); diff --git a/backend/src/api/routes/rfValidation.ts b/backend/src/api/routes/rfValidation.ts index f280be4..5e990fd 100644 --- a/backend/src/api/routes/rfValidation.ts +++ b/backend/src/api/routes/rfValidation.ts @@ -3,16 +3,21 @@ import type { QueryResultRow } from 'pg'; import { resolvePublicNetworkScope } from '../../http/requestScope.js'; import type { NetworkFilters } from '../utils/networkFilters.js'; import { parseBoundedInteger } from '../utils/input.js'; -import { rfValidationRows } from '../../repositories/networkAnalysis.js'; +import { queryRfValidation, rfValidationRows } from '../../repositories/networkAnalysis.js'; +export { RF_VALIDATION_DB_TIMEOUT_MS } from '../../repositories/networkAnalysis.js'; -type QueryFn = (text: string, params?: unknown[]) => Promise<{ rows: T[] }>; +type QueryFn = (text: string, params?: unknown[], signal?: AbortSignal) => Promise<{ rows: T[] }>; type Deps = { + // Kept compatible with the central route registration. RF reads use the + // dedicated read-only pool below instead of this general-purpose query. query: QueryFn; networkFilters: (network?: string, observer?: string) => NetworkFilters; limiter: ReturnType; }; -export function registerRfValidationRoutes(router: Router, deps: Deps): void { +export const RF_VALIDATION_REQUEST_TIMEOUT_MS = 4_500; + +export function registerRfValidationRoutes(router: Router, deps: Deps, rfQuery: QueryFn = queryRfValidation): void { router.get('/rf-validation', deps.limiter, async (req, res) => { const limit = parseBoundedInteger(req.query['limit'], { name: 'limit', @@ -20,10 +25,16 @@ export function registerRfValidationRoutes(router: Router, deps: Deps): void { min: 25, max: 250, }); + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(new Error('RF_VALIDATION_TIMEOUT')), RF_VALIDATION_REQUEST_TIMEOUT_MS); + const onClose = () => { + if (!res.writableEnded) controller.abort(new Error('RF_VALIDATION_DISCONNECTED')); + }; + res.once('close', onClose); try { const network = resolvePublicNetworkScope(req.query['network'], req.headers); const filters = deps.networkFilters(network); - const result = await rfValidationRows(deps.query, filters, limit); + const result = await rfValidationRows(rfQuery, filters, limit, controller.signal); const links = result.rows.map((row) => ({ source: row.node_a_id, target: row.node_b_id, @@ -42,7 +53,7 @@ export function registerRfValidationRoutes(router: Router, deps: Deps): void { res.json({ generatedAt: new Date().toISOString(), windowDays: 30, - methodology: 'Observed relay evidence compared with the stored ITM viability result; weak evidence is not treated as a proven model error.', + methodology: 'Relay links observed in the last 30 days, ranked by model disagreement and evidence strength. Stored observation counts and ITM results are combined across recently active identity aliases; weak evidence is not a proven model error.', summary: { evaluated: links.length, matches: links.length - mismatches.length, @@ -54,8 +65,19 @@ export function registerRfValidationRoutes(router: Router, deps: Deps): void { mismatches, }); } catch (err) { + if (res.destroyed) return; console.error('[api] GET /rf-validation', (err as Error).message); - res.status(500).json({ error: 'Internal server error' }); + res.setHeader('Cache-Control', 'no-store'); + if (controller.signal.aborted || (err as { code?: string }).code === '57014' + || /timeout|timed out/i.test((err as Error).message)) { + res.setHeader('Retry-After', '30'); + res.status(503).json({ error: 'RF validation is temporarily unavailable. Please retry shortly.' }); + } else { + res.status(500).json({ error: 'Internal server error' }); + } + } finally { + clearTimeout(timeout); + res.off('close', onClose); } }); } diff --git a/backend/src/cache/boundedTtlMap.test.ts b/backend/src/cache/boundedTtlMap.test.ts index 896dd4f..d18b996 100644 --- a/backend/src/cache/boundedTtlMap.test.ts +++ b/backend/src/cache/boundedTtlMap.test.ts @@ -60,38 +60,3 @@ test('bounded cache overwrite accounts weight once and reports hit/miss/eviction }); cache.shutdown(); }); - -test('sweeps respect replacement deadlines, access order, and backwards clock changes', () => { - let now = 100; - const cache = new BoundedTtlMap({ - maxEntries: 4, maxWeight: 100, ttlMs: 100, now: () => now, - }); - try { - cache.set('replaced', 'old'); - now = 150; - cache.set('replaced', 'new'); - now = 50; - cache.set('earlier', 'value'); - cache.get('replaced'); // LRU order must not determine expiry order. - now = 150; - cache.sweep(); - assert.equal(cache.has('earlier'), false); - assert.equal(cache.get('replaced'), 'new'); - now = 200; // The overwritten value's original expiry is harmless. - cache.sweep(); - assert.equal(cache.size, 1); - now = 250; - cache.sweep(); - assert.equal(cache.size, 0); - assert.equal(cache.weight(), 0); - assert.equal(cache.metrics().expiries, 2); - cache.clear(); - now = 0; - cache.set('after-clear', 'value'); - now = 100; - cache.sweep(); - assert.equal(cache.size, 0); - } finally { - cache.shutdown(); - } -}); diff --git a/backend/src/cache/boundedTtlMap.ts b/backend/src/cache/boundedTtlMap.ts index 0db8a7e..32b8df4 100644 --- a/backend/src/cache/boundedTtlMap.ts +++ b/backend/src/cache/boundedTtlMap.ts @@ -27,9 +27,6 @@ export class BoundedTtlMap extends Map { private readonly insertedAt = new Map(); private readonly weights = new Map(); private totalWeight = 0; - // A lower bound on the next expiry. Deletions may leave it conservatively - // early, but writes need not walk the whole cache while every entry is fresh. - private nextExpiryAt = Number.POSITIVE_INFINITY; private readonly timer: NodeJS.Timeout; private hits = 0; private misses = 0; @@ -93,9 +90,7 @@ export class BoundedTtlMap extends Map { this.sweep(); this.delete(key); super.set(key, value); - const insertedAt = this.options.now?.() ?? Date.now(); - this.insertedAt.set(key, insertedAt); - this.nextExpiryAt = Math.min(this.nextExpiryAt, insertedAt + this.options.ttlMs); + this.insertedAt.set(key, this.options.now?.() ?? Date.now()); this.weights.set(key, weight); this.totalWeight += weight; while (this.size > this.options.maxEntries || this.totalWeight > this.options.maxWeight) { @@ -127,21 +122,16 @@ export class BoundedTtlMap extends Map { this.insertedAt.clear(); this.weights.clear(); this.totalWeight = 0; - this.nextExpiryAt = Number.POSITIVE_INFINITY; this.recordMetric('cleared'); this.syncMetricGauges(); } sweep(now = this.options.now?.() ?? Date.now()): void { - if (now < this.nextExpiryAt) return; - this.nextExpiryAt = Number.POSITIVE_INFINITY; for (const [key, timestamp] of this.insertedAt) { if (now - timestamp >= this.options.ttlMs) { this.expiries += 1; this.recordMetric('expired'); this.delete(key); - } else { - this.nextExpiryAt = Math.min(this.nextExpiryAt, timestamp + this.options.ttlMs); } } } diff --git a/backend/src/cache/policyRegistry.ts b/backend/src/cache/policyRegistry.ts index a302dd2..7bfeaf8 100644 --- a/backend/src/cache/policyRegistry.ts +++ b/backend/src/cache/policyRegistry.ts @@ -123,6 +123,12 @@ export const CACHE_POLICY_REGISTRY: readonly CachePolicyRecord[] = Object.freeze scope: 'owner username', invalidation: 'authorization reconciliation + TTL', negativeCaching: 'empty verified grant list', singleFlight: 'ownerNodeIdInflight (cap 128)', }, + { + source: 'src/owner/ownerAccess.ts#authCache', + disposition: 'bounded-cache', maxEntries: 512, maxBytes: 2 << 20, ttlMs: 5 * 60_000, + scope: 'owner username + credential digest', invalidation: 'credential mismatch + TTL', + negativeCaching: 'positive results only', singleFlight: 'broker connection timeout', + }, { source: 'src/owner/ownerService.ts#ownerLastHopCache', disposition: 'bounded-cache', maxEntries: 1_024, maxBytes: 64 << 20, ttlMs: 60 * 60_000, diff --git a/backend/src/db/advertCount.test.ts b/backend/src/db/advertCount.test.ts index 17c54f9..cfdeaf3 100644 --- a/backend/src/db/advertCount.test.ts +++ b/backend/src/db/advertCount.test.ts @@ -11,7 +11,5 @@ test('advert count is folded into node UPSERT with durable canonical-hash dedupl assert.match(dbSource, /advert_count\s+= nodes\.advert_count \+ \(SELECT COUNT\(\*\) FROM advert_once\)/); assert.doesNotMatch(dbSource, /function incrementAdvertCount/); assert.match(mqttSource, /advertHash: canonicalPacketId/); - assert.match(mqttSource, /const advertSignatureValid = await verifyAdvertSignature\(decoded\)/); - assert.match(mqttSource, /if \(advertSignatureValid && network !== 'test'\)/); assert.doesNotMatch(mqttSource, /tryCountAdvert|countedAdvertHashes/); }); diff --git a/backend/src/db/dataLifecycle.ts b/backend/src/db/dataLifecycle.ts index d968008..0540a27 100644 --- a/backend/src/db/dataLifecycle.ts +++ b/backend/src/db/dataLifecycle.ts @@ -42,10 +42,10 @@ export const DATA_LIFECYCLE_POLICIES: readonly DataLifecyclePolicy[] = Object.fr { table: 'node_status_samples', timestampColumn: 'time', - retention: '180 days', + retention: '30 days', kind: 'hypertable', featureImpact: [ - 'owner status telemetry older than 180 days becomes restore-only', + 'owner status telemetry older than 30 days becomes restore-only', ], }, { @@ -113,6 +113,13 @@ export const DATA_LIFECYCLE_POLICIES: readonly DataLifecyclePolicy[] = Object.fr kind: 'row-table', featureImpact: ['terminal owner packet-sharing queue and audit rows older than 30 days are removed'], }, + { + table: 'public_feed_deliveries', + timestampColumn: 'created_at', + retention: '30 days', + kind: 'row-table', + featureImpact: ['terminal public-feed queue and audit rows older than 30 days are removed'], + }, { table: 'observer_registration_requests', timestampColumn: 'updated_at', diff --git a/backend/src/db/index.ts b/backend/src/db/index.ts index 9a51126..a856c39 100644 --- a/backend/src/db/index.ts +++ b/backend/src/db/index.ts @@ -146,6 +146,11 @@ async function queryPool( } } +export type DatabaseQueryFn = ( + text: string, + params?: unknown[], +) => Promise<{ rows: T[] }>; + export async function query( text: string, params?: unknown[], @@ -154,11 +159,6 @@ export async function query( return queryPool(pool, 'oltp', text, params, signal); } -export type DatabaseQueryFn = ( - text: string, - params?: unknown[], -) => Promise<{ rows: T[] }>; - export async function withTransaction(work: (transactionQuery: DatabaseQueryFn) => Promise): Promise { const client = await pool.connect(); const transactionQuery: DatabaseQueryFn = async ( @@ -644,13 +644,12 @@ export async function insertNodeStatusSample(sample: { channelUtilization?: number | null; airUtilTx?: number | null; stats?: Record | null; - status?: string | null; }): Promise { await pool.query( `WITH sample_insert AS ( INSERT INTO node_status_samples - (time, node_id, network, battery_mv, uptime_secs, tx_air_secs, rx_air_secs, channel_utilization, air_util_tx, stats, status) - VALUES (NOW(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + (time, node_id, network, battery_mv, uptime_secs, tx_air_secs, rx_air_secs, channel_utilization, air_util_tx, stats) + VALUES (NOW(), $1, $2, $3, $4, $5, $6, $7, $8, $9) RETURNING time ) UPDATE nodes @@ -668,7 +667,6 @@ export async function insertNodeStatusSample(sample: { sample.channelUtilization ?? null, sample.airUtilTx ?? null, sample.stats ? JSON.stringify(sample.stats) : null, - sample.status ?? null, ], ); } @@ -735,17 +733,35 @@ export async function getNodes( export async function getNodeHistory(nodeId: string, hours = 24, network = 'ukmesh') { const scope = buildScopePlaceholders(3, network); + const aliases = nodeAliasArraySql('$1'); + // Both consumers describe general activity, including for observers that + // are also repeaters. Include receptions and known origins for every role. + // Drive each direction with the canonical alias array and an equality + // lookup: a combined OR/ANY lets LIMIT choose a scan of time-ordered traffic + // for quiet nodes. Per-alias limits keep both node/time indexes useful even + // at 28 days; the final limit retains the newest 500 observations overall. + const packetsFor = (column: 'rx_node_id' | 'src_node_id') => ` + SELECT p.time, p.packet_hash, + COALESCE(src_alias.canonical_node_id, UPPER(BTRIM(p.src_node_id))) AS src_node_id, + topic, packet_type, hop_count, rssi, snr, payload + FROM packets p + LEFT JOIN node_identity_aliases src_alias + ON src_alias.source_node_id = UPPER(BTRIM(p.src_node_id)) + WHERE p.${column} = requested.node_id + ${column === 'src_node_id' ? `AND NOT COALESCE(p.rx_node_id = ANY(${aliases}), FALSE)` : ''} + AND p.time > NOW() - INTERVAL '1 hour' * $2 + AND p.time <= NOW() + ${buildPacketScopeClause(scope, 'p', network)} + ${buildPublicPacketPrivacyClause('p')} + ORDER BY time DESC LIMIT 500`; const res = await pool.query( - `SELECT p.time, p.packet_hash, - COALESCE(src_alias.canonical_node_id, UPPER(BTRIM(p.src_node_id))) AS src_node_id, - topic, packet_type, hop_count, rssi, snr, payload - FROM packets p - LEFT JOIN node_identity_aliases src_alias - ON src_alias.source_node_id = UPPER(BTRIM(p.src_node_id)) - WHERE p.rx_node_id = ANY(${nodeAliasArraySql('$1')}) - AND p.time > NOW() - INTERVAL '1 hour' * $2 - ${buildPacketScopeClause(scope, 'p', network)} - ${buildPublicPacketPrivacyClause('p')} + `SELECT recent.* + FROM unnest(${aliases}) requested(node_id) + CROSS JOIN LATERAL ( + (${packetsFor('rx_node_id')}) + UNION ALL + (${packetsFor('src_node_id')}) + ) recent ORDER BY time DESC LIMIT 500`, [nodeId, hours, ...scope.params] ); @@ -880,9 +896,11 @@ export async function getRecentPackets( ${includeObservationIds ? `ps._observation_ids, rp.network AS _policy_network, (SELECT generation::text FROM public_visibility_state WHERE singleton) AS _policy_generation, rp._representative_rank, rp._representative_id,` : ''} - ps.observer_node_ids, ps.observer_iatas, ps.rx_count, ps.tx_count + ps.observer_node_ids, ps.observer_iatas, ps.rx_count, ps.tx_count, + mt.tags, mt.confidence AS tag_confidence FROM recent_packets rp LEFT JOIN packet_stats ps ON ps.packet_hash = rp.packet_hash + LEFT JOIN message_tags mt ON mt.packet_hash = rp.packet_hash ${overlapParam ? `WHERE rp.packet_hash = ANY(${overlapParam}::text[]) OR rp.packet_hash IN (SELECT packet_hash FROM recent_packets ORDER BY time DESC, packet_hash LIMIT $1)` : ''} ORDER BY rp.time DESC, rp.packet_hash @@ -960,9 +978,11 @@ export async function getRecentMessages( ${includeObservationIds ? `ms._observation_ids, m.network AS _policy_network, (SELECT generation::text FROM public_visibility_state WHERE singleton) AS _policy_generation, m._representative_rank, m._representative_id,` : ''} - ms.observer_node_ids, ms.observer_iatas, ms.rx_count, ms.tx_count + ms.observer_node_ids, ms.observer_iatas, ms.rx_count, ms.tx_count, + mt.tags, mt.confidence AS tag_confidence FROM recent_msgs m LEFT JOIN msg_stats ms ON ms.packet_hash = m.packet_hash + LEFT JOIN message_tags mt ON mt.packet_hash = m.packet_hash ${overlapParam ? `WHERE m.packet_hash = ANY(${overlapParam}::text[]) OR m.packet_hash IN (SELECT packet_hash FROM recent_msgs ORDER BY time DESC, packet_hash LIMIT $1)` : ''} ORDER BY m.time DESC, m.packet_hash @@ -973,6 +993,32 @@ export async function getRecentMessages( return res.rows; } +/** + * Recent message tags for live feed enrichment. Tags are produced by the + * tagger-worker a few seconds after a packet arrives, so the live WS feed can + * never carry them inline; the frontend polls this and patches rows by hash. + * Only tags whose packet is publicly visible and in scope are returned. + */ +export async function getRecentMessageTags(limit = 200, network?: string) { + const scope = buildScopePlaceholders(2, network); + const res = await query( + `SELECT mt.packet_hash, mt.tags, mt.confidence, mt.tagged_at + FROM message_tags mt + WHERE mt.tagged_at > NOW() - INTERVAL '24 hours' + AND EXISTS ( + SELECT 1 FROM packets p + WHERE p.packet_hash = mt.packet_hash + AND p.time > NOW() - INTERVAL '24 hours' + ${buildPacketScopeClause(scope, 'p', network)} + ${buildPublicPacketPrivacyClause('p')} + ) + ORDER BY mt.tagged_at DESC + LIMIT $1`, + [limit, ...scope.params], + ); + return res.rows; +} + /** * Fetch one channel's decrypted GroupText history. The WebSocket snapshot stays * intentionally recent and bounded; this demand-driven read reaches back 90 @@ -1082,8 +1128,10 @@ export async function getChannelMessageHistory( CASE WHEN m.rx_node_id IS NOT NULL THEN ARRAY[m.rx_node_id]::text[] ELSE ARRAY[]::text[] END AS observer_node_ids, CASE WHEN NULLIF(TRIM(m.iata), '') IS NOT NULL THEN ARRAY[m.iata]::text[] ELSE ARRAY[]::text[] END AS observer_iatas, CASE WHEN COALESCE(m.payload->>'direction', 'rx') <> 'tx' THEN 1 ELSE 0 END::int AS rx_count, - CASE WHEN COALESCE(m.payload->>'direction', 'rx') = 'tx' THEN 1 ELSE 0 END::int AS tx_count + CASE WHEN COALESCE(m.payload->>'direction', 'rx') = 'tx' THEN 1 ELSE 0 END::int AS tx_count, + mt.tags, mt.confidence AS tag_confidence FROM channel_candidates m + LEFT JOIN message_tags mt ON mt.packet_hash = m.packet_hash WHERE 1 = 1 ${buildPublicPacketPrivacyClause('m')} ORDER BY m.time DESC`, @@ -1100,9 +1148,14 @@ export async function getChannelMessageHistory( }); } -export async function getRecentPacketEvents(limit = 200, network?: string, observer?: string) { +export async function getRecentPacketEvents(limit = 200, network?: string, observer?: string, regionScope?: string) { const scope = buildScopePlaceholders(2, network, observer); const params: unknown[] = [limit, ...scope.params]; + let regionClause = ''; + if (regionScope) { + params.push(regionScope); + regionClause = `AND p.region_scope = $${params.length}`; + } const res = await pool.query( `SELECT p.time, p.packet_hash, @@ -1111,7 +1164,8 @@ export async function getRecentPacketEvents(limit = 200, network?: string, obser p.topic, p.iata, p.packet_type, p.hop_count, p.rssi, p.snr, p.payload, p.payload->>'_summary' AS summary, - p.advert_count, p.path_hashes, p.path_hash_size_bytes + p.advert_count, p.path_hashes, p.path_hash_size_bytes, + p.region_scope FROM packets p LEFT JOIN node_identity_aliases rx_alias ON rx_alias.source_node_id = UPPER(BTRIM(p.rx_node_id)) @@ -1119,6 +1173,7 @@ export async function getRecentPacketEvents(limit = 200, network?: string, obser ON src_alias.source_node_id = UPPER(BTRIM(p.src_node_id)) WHERE p.time > NOW() - INTERVAL '24 hours' ${buildPacketScopeClause(scope, 'p', network)} + ${regionClause} ${buildPublicPacketPrivacyClause('p')} ORDER BY p.time DESC LIMIT $1`, @@ -1133,8 +1188,10 @@ export async function getPacketDetail(hash: string, network = 'ukmesh') { pool.query( `SELECT p.time, p.packet_hash, p.rx_node_id, p.src_node_id, p.topic, p.iata, p.packet_type, p.route_type, p.hop_count, p.rssi, p.snr, - p.payload, p.path_hashes, p.path_hash_size_bytes, p.raw_hex + p.payload, p.path_hashes, p.path_hash_size_bytes, p.raw_hex, + mt.tags, mt.confidence AS tag_confidence FROM packets p + LEFT JOIN message_tags mt ON mt.packet_hash = p.packet_hash WHERE p.packet_hash = $1 ${buildPacketScopeClause(scope, 'p', network)} ${buildPublicPacketPrivacyClause('p')} @@ -1176,6 +1233,8 @@ export async function getPacketDetail(hash: string, network = 'ukmesh') { pathHashes: row.path_hashes as string[] | null, pathHashSizeBytes: row.path_hash_size_bytes as number | null, rawHex: row.raw_hex as string | null, + tags: (row.tags as Record | null) ?? null, + tagConfidence: (row.tag_confidence as Record | null) ?? null, observations: observations.rows.map((r) => ({ rxNodeId: r.rx_node_id as string | null, iata: r.iata as string | null, diff --git a/backend/src/db/migrations/055_public_feed.sql b/backend/src/db/migrations/055_public_feed.sql new file mode 100644 index 0000000..3869aa1 --- /dev/null +++ b/backend/src/db/migrations/055_public_feed.sql @@ -0,0 +1,50 @@ +-- Account-level opt-in for the aggregate public MeshCore MQTT feed. +-- Ownership is stored as a materialised mapping because the owner-auth +-- database is intentionally separate from the analytics database. The +-- backend refreshes this mapping from verified owner grants periodically. + +CREATE TABLE IF NOT EXISTS owner_public_feed_settings ( + owner_username TEXT PRIMARY KEY CHECK (char_length(owner_username) BETWEEN 1 AND 128), + enabled BOOLEAN NOT NULL DEFAULT FALSE, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE TABLE IF NOT EXISTS owner_public_feed_nodes ( + owner_username TEXT NOT NULL REFERENCES owner_public_feed_settings(owner_username) ON DELETE CASCADE, + node_id TEXT NOT NULL CHECK (node_id ~ '^[0-9A-F]{64}$'), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (owner_username, node_id) +); + +CREATE INDEX IF NOT EXISTS owner_public_feed_nodes_node_idx + ON owner_public_feed_nodes (node_id, owner_username); + +CREATE TABLE IF NOT EXISTS public_feed_deliveries ( + id BIGSERIAL PRIMARY KEY, + node_id TEXT NOT NULL CHECK (node_id ~ '^[0-9A-F]{64}$'), + event_key TEXT NOT NULL UNIQUE, + packet_hash TEXT NOT NULL, + source_topic TEXT NOT NULL, + destination_topic TEXT NOT NULL, + payload JSONB NOT NULL, + status TEXT NOT NULL DEFAULT 'pending' + CHECK (status IN ('pending', 'delivering', 'succeeded', 'failed', 'dead_lettered')), + attempts INTEGER NOT NULL DEFAULT 0 CHECK (attempts >= 0 AND attempts <= 5), + next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + last_attempt_at TIMESTAMPTZ, + delivered_at TIMESTAMPTZ, + last_error TEXT, + claim_token TEXT, + claim_expires_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS public_feed_deliveries_due_idx + ON public_feed_deliveries (next_attempt_at, id) + WHERE status IN ('pending', 'failed') AND attempts < 5; + +CREATE INDEX IF NOT EXISTS public_feed_deliveries_retention_idx + ON public_feed_deliveries (created_at, id) + WHERE status IN ('succeeded', 'dead_lettered'); diff --git a/backend/src/db/migrations/060_region_scope_participation.sql b/backend/src/db/migrations/060_region_scope_participation.sql new file mode 100644 index 0000000..5d69495 --- /dev/null +++ b/backend/src/db/migrations/060_region_scope_participation.sql @@ -0,0 +1,53 @@ +-- 057: region-scope participation tables. +-- +-- Derived data owned solely by the regions:rebuild tool +-- (src/tools/rebuildRegionParticipation.ts), which refreshes all four tables +-- wholesale inside one transaction. This migration only owns the schema and is +-- intentionally additive: no existing table is touched. +-- +-- Region names are public hashtag regions (#name) whose transport key is +-- SHA256("#name")[0:16]; attribution is a candidate-name test against each +-- scoped packet's transport code. Names are stored without the leading '#'. +-- A node is 'green' when it was observed relaying cryptographically scoped +-- traffic and 'red' when only unscoped relay evidence (or none) exists. + +CREATE TABLE IF NOT EXISTS region_scope_stats ( + region TEXT PRIMARY KEY CHECK (char_length(region) BETWEEN 1 AND 64), + packets_total INTEGER NOT NULL DEFAULT 0 CHECK (packets_total >= 0), + member_count INTEGER NOT NULL DEFAULT 0 CHECK (member_count >= 0), + repeater_member_count INTEGER NOT NULL DEFAULT 0 CHECK (repeater_member_count >= 0), + first_seen TIMESTAMPTZ, + last_seen TIMESTAMPTZ +); + +CREATE TABLE IF NOT EXISTS region_scope_members ( + region TEXT NOT NULL REFERENCES region_scope_stats(region) ON DELETE CASCADE, + node_id TEXT NOT NULL CHECK (node_id ~ '^[0-9A-F]{64}$'), + packets INTEGER NOT NULL DEFAULT 0 CHECK (packets >= 0), + PRIMARY KEY (region, node_id) +); + +CREATE INDEX IF NOT EXISTS region_scope_members_node_idx + ON region_scope_members (node_id); + +CREATE TABLE IF NOT EXISTS repeater_scope_status ( + node_id TEXT PRIMARY KEY CHECK (node_id ~ '^[0-9A-F]{64}$'), + status TEXT NOT NULL CHECK (status IN ('green', 'red')), + evidence TEXT NOT NULL CHECK (evidence IN ('scoped', 'unscoped-only', 'none')), + scoped_packets INTEGER NOT NULL DEFAULT 0 CHECK (scoped_packets >= 0), + unscoped_packets_7d INTEGER NOT NULL DEFAULT 0 CHECK (unscoped_packets_7d >= 0), + regions JSONB NOT NULL DEFAULT '{}'::jsonb +); + +CREATE INDEX IF NOT EXISTS repeater_scope_status_status_idx + ON repeater_scope_status (status); + +-- Single-row run record: when the tool last refreshed the tables and over what +-- window. API responses surface this as generatedAt / windowDays. +CREATE TABLE IF NOT EXISTS region_scope_run ( + id BOOLEAN PRIMARY KEY DEFAULT TRUE CHECK (id), + finished_at TIMESTAMPTZ NOT NULL, + window_days INTEGER NOT NULL CHECK (window_days BETWEEN 1 AND 365), + scoped_packets INTEGER NOT NULL CHECK (scoped_packets >= 0), + decoded_packets INTEGER NOT NULL CHECK (decoded_packets >= 0) +); diff --git a/backend/src/db/migrations/061_packet_share_owner_scope_and_jwt.sql b/backend/src/db/migrations/061_packet_share_owner_scope_and_jwt.sql new file mode 100644 index 0000000..ad6bea0 --- /dev/null +++ b/backend/src/db/migrations/061_packet_share_owner_scope_and_jwt.sql @@ -0,0 +1,10 @@ +-- NULL scopes preserve existing destinations available to every owner. +ALTER TABLE owner_packet_share_destinations + ADD COLUMN allowed_owner_username TEXT CHECK (char_length(allowed_owner_username) BETWEEN 1 AND 128), + ADD COLUMN allowed_node_id TEXT CHECK (allowed_node_id ~ '^[0-9A-F]{64}$'), + ADD COLUMN auth_mode TEXT NOT NULL DEFAULT 'password' CHECK (auth_mode IN ('password', 'meshcore_jwt')), + ADD COLUMN auth_audience TEXT CHECK (char_length(auth_audience) BETWEEN 1 AND 253), + ADD COLUMN tx_mode TEXT NOT NULL DEFAULT 'on' CHECK (tx_mode IN ('on', 'advert')); + +-- Provision restricted destinations only after deploying the scoped backend. +-- The preceding backend lists all rows and cannot enforce these scopes. diff --git a/backend/src/db/nodeHistory.test.ts b/backend/src/db/nodeHistory.test.ts new file mode 100644 index 0000000..122f23e --- /dev/null +++ b/backend/src/db/nodeHistory.test.ts @@ -0,0 +1,37 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import pg from 'pg'; +import { getNodeHistory } from './index.js'; + +test('history retains its DTO, indexed alias predicates, time cap and fail-closed privacy in both directions', async (t) => { + const rows = [{ time: '2026-09-07T12:00:00Z', packet_hash: '1234', src_node_id: 'B'.repeat(64), + topic: 'ukmesh/LHR/observer/packets', packet_type: 4, hop_count: 1, rssi: -99, snr: 3, payload: {} }]; + let calls = 0; + t.mock.method(pg.Pool.prototype, 'query', async (sql: string, params: unknown[]) => { + calls += 1; + assert.match(sql, /WHERE p.rx_node_id = requested.node_id/); + assert.match(sql, /WHERE p.src_node_id = requested.node_id/); + assert.match(sql, /FROM unnest\(ARRAY\(/); + assert.match(sql, /UNION ALL/); + assert.match(sql, /NOT COALESCE\(p.rx_node_id = ANY\(ARRAY\(/); + assert.doesNotMatch(sql, /(?:UPPER|BTRIM)\(p\.(?:rx|src)_node_id\)\s*=\s*ANY/); + assert.match(sql, /canonical_node_id = alias.canonical_node_id/); + assert.match(sql, /p.time > NOW\(\) - INTERVAL '1 hour' \* \$2/); + assert.match(sql, /p.time <= NOW\(\)/); + assert.match(sql, /p.network = ANY\(\$3\)/); + assert.match(sql, /meshcore-test/); + assert.match(sql, /p.visibility_ok IS TRUE/); + assert.match(sql, /p.is_private IS NOT TRUE/); + assert.match(sql, /cached_visibility.visibility_generation = current_visibility.generation/); + assert.match(sql, /ORDER BY time DESC LIMIT 500/); + assert.deepEqual(params, ['A'.repeat(64), 24, ['ukmesh', 'northeast', 'teesside']]); + return { rows }; + }); + assert.equal(await getNodeHistory('A'.repeat(64), 24, 'ukmesh'), rows); + assert.equal(calls, 1); +}); + +test('history returns an empty array unchanged', async (t) => { + t.mock.method(pg.Pool.prototype, 'query', async () => ({ rows: [] })); + assert.deepEqual(await getNodeHistory('0'.repeat(64)), []); +}); diff --git a/backend/src/db/ownerAuth.ts b/backend/src/db/ownerAuth.ts index e4b1050..23f0d46 100644 --- a/backend/src/db/ownerAuth.ts +++ b/backend/src/db/ownerAuth.ts @@ -3,7 +3,6 @@ import fs from 'node:fs'; import { randomUUID } from 'node:crypto'; import { resolveDbAssetPath } from './assets.js'; import { boundedIntegerSetting } from '../platform/config/boundedNumber.js'; -import { planOperatorConfiguredOwnerGrantSync } from '../owner/ownerGrantReconciliation.js'; const { Pool } = pg; const OWNER_DB_NAME = process.env['OWNER_POSTGRES_DB'] ?? 'meshcore_owner_auth'; @@ -347,43 +346,35 @@ export async function syncOperatorConfiguredOwnerGrants( generation: string, ): Promise { const normalized = grants.map((grant) => normalizeGrant(grant.mqttUsername, grant.nodeId)); + const desiredKeys = new Set(normalized.map((grant) => `${grant.mqttUsername}\0${grant.nodeId}`)); const current = await ownerPool.query<{ mqtt_username: string; node_id: string; - revoked_at: string | Date | null; + revoked_at: string | null; verification_method: string | null; grant_generation: string | null; - updated_at: string | Date | null; }>( - `SELECT mqtt_username, node_id, revoked_at, verification_method, grant_generation, updated_at + `SELECT mqtt_username, node_id, revoked_at, verification_method, grant_generation FROM owner_account_nodes`, ); - const actions = planOperatorConfiguredOwnerGrantSync( - normalized, - current.rows.map((row) => ({ - mqttUsername: row.mqtt_username, - nodeId: row.node_id, - revokedAt: row.revoked_at, - verificationMethod: row.verification_method, - grantGeneration: row.grant_generation, - updatedAt: row.updated_at, - })), - generation, - ); - - for (const action of actions) { - if (action.type === 'upsert') { - await upsertVerifiedOwnerGrant(action.grant.mqttUsername, action.grant.nodeId, { - verificationMethod: 'operator-config', - actor: 'config-reconciler', - reason: action.reauthorize ? 'restored from operator configuration' : undefined, - generation, - reauthorize: action.reauthorize, - }); - continue; - } - await revokeOwnerGrant(action.grant.mqttUsername, action.grant.nodeId, { + for (const grant of normalized) { + const row = current.rows.find((candidate) => + candidate.mqtt_username === grant.mqttUsername && candidate.node_id === grant.nodeId); + if (row?.revoked_at) continue; + if (row?.verification_method === 'operator-database') continue; + if (row?.verification_method === 'operator-config' && row.grant_generation === generation) continue; + await upsertVerifiedOwnerGrant(grant.mqttUsername, grant.nodeId, { + verificationMethod: 'operator-config', + actor: 'config-reconciler', + generation, + }); + } + for (const row of current.rows) { + if (row.verification_method !== 'operator-config' + || row.revoked_at + || desiredKeys.has(`${row.mqtt_username}\0${row.node_id}`)) continue; + await revokeOwnerGrant(row.mqtt_username, row.node_id, { actor: 'config-reconciler', reason: 'removed from operator configuration', generation, diff --git a/backend/src/db/packetBatch.integration.test.ts b/backend/src/db/packetBatch.integration.test.ts index a8221c5..2682516 100644 --- a/backend/src/db/packetBatch.integration.test.ts +++ b/backend/src/db/packetBatch.integration.test.ts @@ -136,10 +136,13 @@ test('base schema and every migration apply to a brand-new database', { '052_owner_packet_sharing.sql', '053_owner_packet_share_destination_metadata.sql', '054_owner_packet_share_default_off.sql', + '055_public_feed.sql', '056_async_privacy_rematerialization.sql', '057_node_status_sample_status.sql', '058_ingest_observation_ledger.sql', '059_ingest_ledger_historical_coverage.sql', + '060_region_scope_participation.sql', + '061_packet_share_owner_scope_and_jwt.sql', ]); assert.deepEqual(await runMigrations(freshPool), []); diff --git a/backend/src/db/schema/base.sql b/backend/src/db/schema/base.sql index bceaa4f..850b80d 100644 --- a/backend/src/db/schema/base.sql +++ b/backend/src/db/schema/base.sql @@ -1,5 +1,5 @@ -- MeshCore Analytics — Database Schema --- TimescaleDB base schema; retention is managed by versioned lifecycle policy. +-- TimescaleDB, no automatic data retention (all data kept indefinitely) CREATE EXTENSION IF NOT EXISTS timescaledb; @@ -61,7 +61,7 @@ ALTER TABLE nodes ADD COLUMN IF NOT EXISTS last_predicted_online_at TIMESTAMPTZ; ALTER TABLE nodes ADD COLUMN IF NOT EXISTS last_path_evidence_at TIMESTAMPTZ; ALTER TABLE nodes ADD COLUMN IF NOT EXISTS last_mqtt_observer_seen_at TIMESTAMPTZ; --- ─── Packets hypertable ────────────────────────────────────────────────── +-- ─── Packets hypertable (no retention — data kept indefinitely) ────────── CREATE TABLE IF NOT EXISTS packets ( time TIMESTAMPTZ NOT NULL DEFAULT NOW(), @@ -83,6 +83,23 @@ CREATE TABLE IF NOT EXISTS packets ( SELECT create_hypertable('packets', 'time', if_not_exists => TRUE); +-- Remove any existing retention policy so packets are kept indefinitely. +DO $$ +DECLARE + _job_id INTEGER; +BEGIN + SELECT job_id INTO _job_id + FROM timescaledb_information.jobs + WHERE proc_name = 'policy_retention' + AND config->>'hypertable_id' = ( + SELECT id::text FROM _timescaledb_catalog.hypertable WHERE table_name = 'packets' + ); + + IF _job_id IS NOT NULL THEN + PERFORM remove_retention_policy('packets'); + END IF; +END $$; + ALTER TABLE packets ADD COLUMN IF NOT EXISTS advert_count INTEGER; ALTER TABLE packets ADD COLUMN IF NOT EXISTS path_hashes TEXT[]; ALTER TABLE packets ADD COLUMN IF NOT EXISTS path_hash_size_bytes INTEGER; @@ -229,8 +246,7 @@ CREATE TABLE IF NOT EXISTS node_status_samples ( rx_air_secs BIGINT, channel_utilization DOUBLE PRECISION, air_util_tx DOUBLE PRECISION, - stats JSONB, - status TEXT + stats JSONB ); SELECT create_hypertable('node_status_samples', 'time', if_not_exists => TRUE); diff --git a/backend/src/health/status.ts b/backend/src/health/status.ts index 3f7993d..26f8504 100644 --- a/backend/src/health/status.ts +++ b/backend/src/health/status.ts @@ -50,7 +50,7 @@ type WorkerSnapshot = { }; export type RetentionTarget = { - table: 'packets' | 'worker_health_snapshots' | 'frontend_error_events' | 'owner_alert_deliveries' | 'operational_check_results' | 'observer_region_packet_sightings' | 'observer_region_observer_sightings' | 'link_job_commits' | 'ml_model_variant_packet_results' | 'packet_decryptions' | 'observer_registration_requests' | 'operator_audit_events'; + table: 'packets' | 'worker_health_snapshots' | 'frontend_error_events' | 'owner_alert_deliveries' | 'owner_packet_share_deliveries' | 'public_feed_deliveries' | 'operational_check_results' | 'observer_region_packet_sightings' | 'observer_region_observer_sightings' | 'link_job_commits' | 'ml_model_variant_packet_results' | 'packet_decryptions' | 'observer_registration_requests' | 'operator_audit_events' | 'node_status_samples'; timestampColumn: 'ts' | 'time' | 'created_at' | 'updated_at' | 'completed_at' | 'last_seen'; retention: string; batchSize: number; @@ -59,8 +59,9 @@ export type RetentionTarget = { export const RETENTION_TARGETS: readonly RetentionTarget[] = [ // Timescale drops complete chunks; this bounded boundary cleanup makes the - // packet-content 30-day privacy limit exact for the partial oldest chunk. + // listed 30-day privacy limits exact for the partial oldest chunk. { table: 'packets', timestampColumn: 'time', retention: '30 days', batchSize: 25_000 }, + { table: 'node_status_samples', timestampColumn: 'time', retention: '30 days', batchSize: 25_000 }, { table: 'worker_health_snapshots', timestampColumn: 'ts', retention: '7 days', batchSize: 10_000 }, { table: 'frontend_error_events', timestampColumn: 'time', retention: '30 days', batchSize: 2_000 }, { @@ -70,6 +71,20 @@ export const RETENTION_TARGETS: readonly RetentionTarget[] = [ batchSize: 1_000, extraPredicate: `(status = 'succeeded' OR attempts >= 5)`, }, + { + table: 'owner_packet_share_deliveries', + timestampColumn: 'created_at', + retention: '30 days', + batchSize: 2_000, + extraPredicate: `status IN ('succeeded', 'dead_lettered')`, + }, + { + table: 'public_feed_deliveries', + timestampColumn: 'created_at', + retention: '30 days', + batchSize: 2_000, + extraPredicate: `status IN ('succeeded', 'dead_lettered')`, + }, { table: 'operational_check_results', timestampColumn: 'ts', retention: '14 days', batchSize: 2_000 }, { table: 'observer_region_packet_sightings', timestampColumn: 'last_seen', retention: '8 days', batchSize: 25_000 }, { table: 'observer_region_observer_sightings', timestampColumn: 'last_seen', retention: '31 days', batchSize: 5_000 }, @@ -157,7 +172,7 @@ async function refreshCoreTelemetryCapacityMetrics(): Promise { WITH lifecycle_targets (table_name, retention, compress_after) AS ( VALUES ('packets', INTERVAL '30 days', INTERVAL '15 days'), - ('node_status_samples', INTERVAL '180 days', INTERVAL '15 days'), + ('node_status_samples', INTERVAL '30 days', INTERVAL '15 days'), ('node_neighbor_samples', INTERVAL '7 days', INTERVAL '2 days') ) SELECT target.table_name, diff --git a/backend/src/index.ts b/backend/src/index.ts index e0f24a0..b3deceb 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -9,9 +9,10 @@ import { closeOwnerAuthDb, getOwnerAclReadiness, getOwnerAuthorizationInventory, + getOwnerAuthorizationSnapshot, initOwnerAuthDb, } from './db/ownerAuth.js'; -import { getMqttRuntimeStatus, startMqttClient, stopMqttClient, onPacket, onNodeSeen, onNodeUpsert } from './mqtt/client.js'; +import { getMqttRuntimeStatus, publishMqtt, startMqttClient, stopMqttClient, onPacket, onNodeSeen, onNodeUpsert } from './mqtt/client.js'; import { startRamWriter, closeRamWriter } from './ram/writerRuntime.js'; import { getRamIngestMode } from './ram/protocol.js'; import { @@ -20,7 +21,7 @@ import { } from './mqtt/connectionMonitor.js'; import { initWebSocketServer, closeWebSocketServer, broadcastPacket, broadcastNodeUpdate, broadcastNodeUpsert } from './ws/server.js'; import { createPublicRamSource } from './ram/publicSource.js'; -import apiRoutes, { startOwnerLastHopPrewarm, stopOwnerLastHopPrewarm } from './api/routes.js'; +import apiRoutes from './api/routes.js'; import { startRegisteredStatsWarmup } from './api/routes/stats.js'; import { initSpamMessageAnalyzer, stopSpamMessageAnalyzer } from './spam/analyzer.js'; import { @@ -31,6 +32,8 @@ import { import { createBackendSiteRoutes } from './backend-site/routes.js'; import { isTrustedProxyPeer } from './http/trustedProxy.js'; import { startOwnerAuthorizationReconciler, stopOwnerAuthorizationReconciler } from './owner/ownerAclReconciler.js'; +import { createOwnerLastHopPrewarm } from './owner/ownerLastHopPrewarm.js'; +import { getRegisteredOwnerService } from './api/routes/owner.js'; import { getAnalysisWorkloadStates } from './analysis/runState.js'; import { applySecurityHeaders } from './security/operatorAuth.js'; import { closeMetricsServer, startMetricsServer } from './metricsServer.js'; @@ -55,6 +58,8 @@ import { readAclFile } from './mqtt/aclManager.js'; import { parseOwnerGrantConfig } from './owner/ownerGrantConfig.js'; import { createPacketShareForwarder } from './owner/packetShareForwarder.js'; import { createPacketShareRepository } from './owner/packetShareRepository.js'; +import { createPublicFeedForwarder } from './owner/publicFeedForwarder.js'; +import { createPublicFeedRepository } from './owner/publicFeedRepository.js'; import { buildOwnerInventoryBaseline, loadOwnerInventoryBaseline, @@ -78,10 +83,23 @@ const SHUTDOWN_DEADLINE_MS = Math.min( Math.max(1_000, Number(process.env['SHUTDOWN_DEADLINE_MS'] ?? 30_000) || 30_000), ); const NODE_IDENTITY_REFRESH_INTERVAL_MS = 30 * 60_000; +const OWNER_LAST_HOP_PREWARM_INTERVAL_MS = Number(process.env['OWNER_LAST_HOP_PREWARM_INTERVAL_MS'] ?? 15 * 60_000); +const OWNER_LAST_HOP_PREWARM_NODE_DELAY_MS = Number(process.env['OWNER_LAST_HOP_PREWARM_NODE_DELAY_MS'] ?? 150); const lifecycle = new LifecycleCoordinator(SHUTDOWN_DEADLINE_MS); const packetShareForwarder = createPacketShareForwarder({ repository: createPacketShareRepository(query), }); +const publicFeedForwarder = createPublicFeedForwarder({ + repository: createPublicFeedRepository(query), + getOwnerAuthorizationSnapshot, + // The public namespace is on the same authenticated Mosquitto instance as + // ingest. Reusing this connection avoids a second credentialed publisher + // and keeps the public namespace behind the backend ACL only. + publisherFactory: () => ({ + publish: publishMqtt, + close: async () => undefined, + }), +}); let shutdownExitCode = 0; let shutdownDeadlineAt = 0; function shutdownTimeLeft(): number { @@ -89,6 +107,7 @@ function shutdownTimeLeft(): number { } let forceCloseHttpConnections = () => {}; let nodeIdentityRefreshTimer: NodeJS.Timeout | null = null; +let ownerLastHopPrewarmStop: (() => void) | null = null; lifecycle.register({ name: 'queue-admission', @@ -113,7 +132,10 @@ lifecycle.register({ lifecycle.register({ name: 'owner-last-hop-prewarm', stage: 10, - close: stopOwnerLastHopPrewarm, + close: () => { + ownerLastHopPrewarmStop?.(); + ownerLastHopPrewarmStop = null; + }, }); lifecycle.register({ name: 'mqtt-ingest', @@ -140,6 +162,11 @@ lifecycle.register({ stage: 30, close: packetShareForwarder.stop, }); +lifecycle.register({ + name: 'public-feed-forwarder', + stage: 30, + close: publicFeedForwarder.stop, +}); lifecycle.register({ name: 'operator-operations', stage: 30, @@ -222,7 +249,21 @@ async function main() { startRegisteredStatsWarmup(); await initOwnerAuthDb(); await startOwnerAuthorizationReconciler(); - startOwnerLastHopPrewarm(); + // Keep the owner dashboard RX-strength cache warm for every active MQTT owner + // so the panel (and its 7d/30d toggle) never pays for a cold scan on open. + const ownerService = getRegisteredOwnerService(); + if (ownerService) { + ownerLastHopPrewarmStop = createOwnerLastHopPrewarm({ + listOwnerNodes: getOwnerAuthorizationSnapshot, + refreshLastHop: (ownedNodeIds, nodeId, windowDays, minAgeMs) => ( + ownerService.refreshOwnerLastHopStrength(ownedNodeIds, nodeId, windowDays, minAgeMs) + ), + intervalMs: OWNER_LAST_HOP_PREWARM_INTERVAL_MS, + nodeDelayMs: OWNER_LAST_HOP_PREWARM_NODE_DELAY_MS, + }); + } else { + console.warn('[owner-last-hop-prewarm] owner service is not registered; prewarm disabled'); + } // 2. Start the audit-only MQTT connection monitor. if (MQTT_INGEST_ENABLED) { @@ -238,6 +279,9 @@ async function main() { void packetShareForwarder.enqueuePacket(packet, upstreamEnvelope).catch((error: unknown) => { console.error('[packet-share] packet handling failed:', error instanceof Error ? error.message : error); }); + void publicFeedForwarder.enqueuePacket(packet, upstreamEnvelope).catch((error: unknown) => { + console.error('[public-feed] packet handling failed:', error instanceof Error ? error.message : error); + }); } // RAM mode defers commit-dependent link admission to the writer's // post-commit hook (the link worker reads packet rows from PostgreSQL). @@ -442,6 +486,7 @@ async function main() { startMetricsServer(METRICS_PORT); packetShareForwarder.start(); + publicFeedForwarder.start(); if (getRamIngestMode() === 'ram') { void (async () => { let delayMs = 1_000; diff --git a/backend/src/metrics.ts b/backend/src/metrics.ts index 406a34e..cef7dab 100644 --- a/backend/src/metrics.ts +++ b/backend/src/metrics.ts @@ -130,6 +130,12 @@ export const mqttIngestOutcomesTotal = new Counter({ registers: [metricsRegistry], }); +export const mqttIngestQueueDepth = new Gauge({ + name: 'meshcore_mqtt_ingest_queue_depth', + help: 'MQTT messages waiting in the bounded ingest queue.', + registers: [metricsRegistry], +}); + export const mqttIngestActive = new Gauge({ name: 'meshcore_mqtt_ingest_active', help: 'MQTT messages currently being processed before acknowledgement.', @@ -149,6 +155,19 @@ export const packetShareQueueDepth = new Gauge({ registers: [metricsRegistry], }); +export const publicFeedOutcomesTotal = new Counter({ + name: 'meshcore_public_feed_outcomes_total', + help: 'Public MeshCore feed decisions and delivery outcomes.', + labelNames: ['outcome'] as const, + registers: [metricsRegistry], +}); + +export const publicFeedQueueDepth = new Gauge({ + name: 'meshcore_public_feed_queue_depth', + help: 'Public MeshCore feed deliveries currently claimed for processing.', + registers: [metricsRegistry], +}); + export const dbQueriesTotal = new Counter({ name: 'meshcore_db_queries_total', help: 'Database queries by pool and bounded outcome.', diff --git a/backend/src/mqtt/aclManager.test.ts b/backend/src/mqtt/aclManager.test.ts index adbcc57..07ba821 100644 --- a/backend/src/mqtt/aclManager.test.ts +++ b/backend/src/mqtt/aclManager.test.ts @@ -128,6 +128,60 @@ test('renderer grants only bounded broker uptime reads to the configured monitor ); }); +test('public-feed opt-in adds one curated read filter without granting public publish', () => { + const rendered = renderOwnerAcl('', [ + { + mqttUsername: 'active-owner', + nodeIds: [NODE_ID], + publicFeedRead: true, + }, + ], []); + + assert.match(rendered.content, /user active-owner\ntopic read public\/\+\/\+\/packets/); + assert.doesNotMatch(rendered.content, /topic read meshcore\/\+\/\+\/packets/); + assert.doesNotMatch(rendered.content, /topic (?:write|readwrite) public\/\+\/\+\/packets/); + assert.deepEqual(rendered.semantic, [{ + mqttUsername: 'active-owner', + nodeIds: [NODE_ID], + publicFeedRead: true, + }]); + validateRenderedOwnerAcl(rendered.content, rendered); +}); + +test('an explicit read-only subscriber replaces a legacy aggregate wildcard', () => { + const rendered = renderOwnerAcl( + 'user soulway\ntopic read meshcore/+/+/packets\n', + [{ + mqttUsername: 'soulway', + nodeIds: [], + publicFeedRead: true, + }], + ['soulway'], + ['soulway'], + ); + + assert.match(rendered.content, /user soulway\ntopic read public\/\+\/\+\/packets/); + assert.doesNotMatch(rendered.content, /topic read meshcore\/\+\/\+\/packets/); + assert.doesNotMatch(rendered.content, /topic write|topic readwrite/); + validateRenderedOwnerAcl(rendered.content, rendered); +}); + +test('only the retained backend account receives the public namespace write permission', () => { + const rendered = renderOwnerAcl( + 'user backend\ntopic readwrite meshcore/#\n', + [{ mqttUsername: 'soulway', nodeIds: [], publicFeedRead: true }], + ['backend'], + ['soulway'], + ['backend'], + ['public/#'], + ); + + assert.match(rendered.content, /user backend[\s\S]*topic readwrite public\/#/); + assert.match(rendered.content, /user soulway\ntopic read public\/\+\/\+\/packets/); + assert.doesNotMatch(rendered.content, /user soulway[\s\S]*topic (?:write|readwrite) public\/#/); + validateRenderedOwnerAcl(rendered.content, rendered); +}); + test('cutover validation blocks empty managed accounts unless explicitly reviewed', () => { const blocked = renderOwnerAcl('', [{ mqttUsername: 'revoked', nodeIds: [] }], []); assert.equal(blocked.validation.ok, false); diff --git a/backend/src/mqtt/aclManager.ts b/backend/src/mqtt/aclManager.ts index a6334d7..b25c980 100644 --- a/backend/src/mqtt/aclManager.ts +++ b/backend/src/mqtt/aclManager.ts @@ -3,7 +3,7 @@ import path from 'node:path'; import { createHash } from 'node:crypto'; import { ownerAclReloadTotal } from '../metrics.js'; -export const OWNER_ACL_RENDERER_VERSION = 'meshcore-owner-acl/v2-sys-read'; +export const OWNER_ACL_RENDERER_VERSION = 'meshcore-owner-acl/v5-public-feed-namespace'; const MANAGED_BEGIN = '# BEGIN MESHCORE OWNER ACL'; const MANAGED_END = '# END MESHCORE OWNER ACL'; const NODE_ID_RE = /^[0-9A-F]{64}$/; @@ -11,6 +11,8 @@ const NODE_ID_RE = /^[0-9A-F]{64}$/; export type OwnerAclGrant = { mqttUsername: string; nodeIds: string[]; + /** Grants read access to the curated public-feed namespace. */ + publicFeedRead?: boolean; }; export type AclDirectiveClass = @@ -40,6 +42,7 @@ export type OwnerAclRenderResult = { content: string; semantic: OwnerAclGrant[]; systemReadUsers: string[]; + systemReadwriteTopics: string[]; validation: { ok: boolean; ambiguousUsers: string[]; @@ -60,6 +63,13 @@ function normalizeNodeIds(nodeIds: string[]): string[] { return Array.from(new Set(nodeIds.map((value) => value.trim().toUpperCase()).filter((value) => NODE_ID_RE.test(value)))).sort(); } +function normalizeAclTopics(topics: Iterable): string[] { + return Array.from(new Set([...topics].map((value) => value.trim()).filter(Boolean).map((topic) => { + if (/\s|[\u0000\r\n]/.test(topic)) throw new Error(`INVALID_ACL_TOPIC:${topic}`); + return topic; + }))).sort(); +} + function stripManagedSection(content: string): { content: string; found: boolean; errors: string[] } { const lines = content.replace(/\r\n/g, '\n').split('\n'); const output: string[] = []; @@ -142,6 +152,9 @@ function buildManagedSection( for (const grant of grants) { lines.push(`user ${grant.mqttUsername}`); if (systemReadUsers.has(grant.mqttUsername)) lines.push('topic read $SYS/broker/uptime'); + if (grant.publicFeedRead) { + lines.push('topic read public/+/+/packets'); + } for (const nodeId of grant.nodeIds) { lines.push(`topic write meshcore/+/${nodeId}/packets`); lines.push(`topic write meshcore/+/${nodeId}/status`); @@ -168,17 +181,23 @@ export function renderOwnerAcl( unmanagedUsers: Iterable, allowedEmptyUsers: Iterable = [], systemReadUsers: Iterable = [], + systemReadwriteTopics: Iterable = [], ): OwnerAclRenderResult { const normalized = grants.map((grant) => ({ mqttUsername: normalizeUsername(grant.mqttUsername), nodeIds: normalizeNodeIds(grant.nodeIds), + ...(grant.publicFeedRead ? { + publicFeedRead: true, + } : {}), })); // An explicit operator grant is the declaration that an account has become // managed. This permits a staged owner to remain in the unmanaged list // until its grant is reviewed, without requiring a live .env edit alongside // the grant. const configuredUsernames = new Set( - normalized.filter((grant) => grant.nodeIds.length > 0).map((grant) => grant.mqttUsername), + normalized + .filter((grant) => grant.nodeIds.length > 0 || grant.publicFeedRead) + .map((grant) => grant.mqttUsername), ); const unmanaged = new Set( [...unmanagedUsers] @@ -194,12 +213,14 @@ export function renderOwnerAcl( [...systemReadUsers].map((value) => normalizeUsername(value)), )).sort(); const systemReadUserSet = new Set(normalizedSystemReadUsers); + const normalizedSystemReadwriteTopics = normalizeAclTopics(systemReadwriteTopics); const generation = createHash('sha256') .update(JSON.stringify({ renderer: OWNER_ACL_RENDERER_VERSION, grants: sortedNormalized, systemReadUsers: normalizedSystemReadUsers, + systemReadwriteTopics: normalizedSystemReadwriteTopics, })) .digest('hex'); const stripped = stripManagedSection(existingContent); @@ -233,6 +254,13 @@ export function renderOwnerAcl( ) { retainedLines.push('topic read $SYS/broker/uptime'); } + if (systemReadUserSet.has(stanza.username)) { + for (const topic of normalizedSystemReadwriteTopics) { + if (!retainedLines.some((line) => line.trim() === `topic readwrite ${topic}`)) { + retainedLines.push(`topic readwrite ${topic}`); + } + } + } output.push(...retainedLines); while (output.at(-1)?.trim() === '') output.pop(); } @@ -247,6 +275,7 @@ export function renderOwnerAcl( content, semantic: sortedNormalized, systemReadUsers: normalizedSystemReadUsers, + systemReadwriteTopics: normalizedSystemReadwriteTopics, validation: { ok: parsed.errors.length === 0 && parsed.duplicateUsers.length === 0 @@ -270,6 +299,12 @@ export function validateRenderedOwnerAcl(content: string, expected: OwnerAclRend throw new Error('OWNER_ACL_MANAGED_MARKERS_MISSING'); } for (const grant of expected.semantic) { + const stanza = parseAcl(content).stanzas.find((candidate) => candidate.username === grant.mqttUsername); + if (grant.publicFeedRead) { + if (!stanza?.directives.some((directive) => directive.line === 'topic read public/+/+/packets')) { + throw new Error(`OWNER_ACL_PUBLIC_FEED_READ_MISMATCH:${grant.mqttUsername}`); + } + } for (const nodeId of grant.nodeIds) { if (!content.includes(`topic write meshcore/+/${nodeId}/packets`) || !content.includes(`topic write meshcore/+/${nodeId}/status`) @@ -289,6 +324,14 @@ export function validateRenderedOwnerAcl(content: string, expected: OwnerAclRend throw new Error(`OWNER_ACL_SYSTEM_READ_MISMATCH:${username}`); } } + for (const username of expected.systemReadUsers) { + const stanza = parseAcl(content).stanzas.find((candidate) => candidate.username === username); + for (const topic of expected.systemReadwriteTopics) { + if (!stanza?.directives.some((directive) => directive.line === `topic readwrite ${topic}`)) { + throw new Error(`OWNER_ACL_SYSTEM_READWRITE_MISMATCH:${username}:${topic}`); + } + } + } } export function getNodeIdsForUserInAcl(content: string, mqttUsername: string): string[] { @@ -308,10 +351,10 @@ export function userExistsInAclContent(content: string, mqttUsername: string): b export function updateUserAclContent(content: string, mqttUsername: string, nodeIds: string[]): string { const parsed = parseAcl(content); - const grants = parsed.stanzas + const grants: OwnerAclGrant[] = parsed.stanzas .filter((stanza) => stanza.username !== mqttUsername) .map((stanza) => ({ mqttUsername: stanza.username, nodeIds: getNodeIdsForUserInAcl(content, stanza.username) })); - grants.push({ mqttUsername, nodeIds }); + grants.push({ mqttUsername, nodeIds, publicFeedRead: false }); return renderOwnerAcl('', grants, [], grants.filter((grant) => grant.nodeIds.length === 0).map((grant) => grant.mqttUsername)).content; } diff --git a/backend/src/mqtt/client.ts b/backend/src/mqtt/client.ts index bc47640..dc2cf46 100644 --- a/backend/src/mqtt/client.ts +++ b/backend/src/mqtt/client.ts @@ -370,6 +370,13 @@ function reportMqttIngestFailure(client: MqttClient, error: Error): void { } +/** Publish through the authenticated ingest connection on the internal broker. */ +export async function publishMqtt(topic: string, payload: string): Promise { + const client = mqttClients.find((candidate) => candidate.connected); + if (!client) throw new Error("MQTT_PUBLISH_NOT_CONNECTED"); + await client.publishAsync(topic, payload, { qos: 0, retain: false }); +} + export async function startMqttClient(): Promise { if (mqttStopping) return; assertSupportedIngestConnections(MQTT_INGEST_CONNECTIONS); diff --git a/backend/src/mqtt/decodePacket.test.ts b/backend/src/mqtt/decodePacket.test.ts index dac45ca..18f9ef3 100644 --- a/backend/src/mqtt/decodePacket.test.ts +++ b/backend/src/mqtt/decodePacket.test.ts @@ -1,10 +1,9 @@ import assert from 'node:assert/strict'; import test from 'node:test'; import { MeshCoreDecoder } from '@michaelhart/meshcore-decoder'; -import { decodePacketCompat, verifyAdvertSignature } from './decodePacket.js'; +import { decodePacketCompat } from './decodePacket.js'; const keyStore = MeshCoreDecoder.createKeyStore({ channelSecrets: [] }); -const VALID_ADVERT = '11007E7662676F7F0850A8A355BAAFBFC1EB7B4174C340442D7D7161C9474A2C94006CE7CF682E58408DD8FCC51906ECA98EBF94A037886BDADE7ECD09FD92B839491DF3809C9454F5286D1D3370AC31A34593D569E9A042A3B41FD331DFFB7E18599CE1E60992A076D50238C5B8F85757375354522F50756765744D65736820436F75676172'; test('preserves native two-byte path decoding', () => { // Header: ACK/Flood; path length 0x42 = two 2-byte hashes. @@ -56,11 +55,3 @@ test('does not expose metadata or an identity for malformed framing', () => { assert.equal(oddLength.metadataValid, false); assert.equal(oddLength.canonicalPacketId, undefined); }); - -test('only a valid Ed25519 advert may establish node identity', async () => { - const valid = decodePacketCompat(VALID_ADVERT, keyStore); - assert.equal(await verifyAdvertSignature(valid.decoded), true); - - const tampered = decodePacketCompat(`${VALID_ADVERT.slice(0, -2)}00`, keyStore); - assert.equal(await verifyAdvertSignature(tampered.decoded), false); -}); diff --git a/backend/src/mqtt/topic.test.ts b/backend/src/mqtt/topic.test.ts index 5ea9efb..dab309d 100644 --- a/backend/src/mqtt/topic.test.ts +++ b/backend/src/mqtt/topic.test.ts @@ -47,9 +47,6 @@ test('stores test-marker IATAs under the isolated test network scope', () => { test('rejects malformed and unsupported MQTT topics before persistence', () => { assert.equal(parseMqttTopic('meshcore/lhr/short/packets', prefixes, blocked), null); assert.equal(parseMqttTopic(`meshcore/lhr/${observer}/other`, prefixes, blocked), null); - // Raw packet bytes are the `raw` field in a /packets JSON envelope; /raw is - // not an ingest topic and must remain outside the observer topic contract. - assert.equal(parseMqttTopic(`meshcore/lhr/${observer}/raw`, prefixes, blocked), null); assert.equal(parseMqttTopic(`unknown/lhr/${observer}/packets`, prefixes, blocked), null); assert.equal(parseMqttTopic(`meshcore/lhr/${observer}/packets/extra`, prefixes, blocked), null); }); diff --git a/backend/src/owner/ownerAccess.ts b/backend/src/owner/ownerAccess.ts index 4d47d26..626d0aa 100644 --- a/backend/src/owner/ownerAccess.ts +++ b/backend/src/owner/ownerAccess.ts @@ -1,5 +1,5 @@ import mqtt from 'mqtt'; -import { randomBytes } from 'node:crypto'; +import { createHash, randomBytes } from 'node:crypto'; import { Redis } from 'ioredis'; import { BoundedTtlMap } from '../cache/boundedTtlMap.js'; import { getOwnerNodeIdsForUsername } from '../db/ownerAuth.js'; @@ -9,7 +9,6 @@ import { getRedisConnectionOptions, getRedisUrl } from '../platform/config/redis import { reconcileOwnerAuthorization } from './ownerAclReconciler.js'; import { groupOwnerNodes, type OwnerDashboardRow } from './ownerDashboard.js'; import { parseOwnerGrantConfig } from './ownerGrantConfig.js'; -import { createMqttCredentialVerifier } from './mqttCredentialVerifier.js'; function normalizeNodeIds(nodeIds: string[]): string[] { return Array.from(new Set( @@ -94,12 +93,50 @@ export async function autoLinkOwnerNodeIds(mqttUsername: string): Promise({ + name: 'owner_auth', + maxEntries: Number(process.env['OWNER_AUTH_CACHE_MAX_ENTRIES'] ?? 512), + maxWeight: 2 * 1024 * 1024, + ttlMs: AUTH_CACHE_TTL_MS, +}); + +function credentialHash(mqttUsername: string, mqttPassword: string): string { + return createHash('sha256').update(`${mqttUsername}\u0000${mqttPassword}`).digest('hex'); +} + +export function verifyMqttCredentials(mqttUsername: string, mqttPassword: string): Promise { + const hash = credentialHash(mqttUsername, mqttPassword); + const cached = authCache.get(mqttUsername); + if (cached && cached.credentialHash === hash && Date.now() - cached.ts < AUTH_CACHE_TTL_MS) { + return Promise.resolve(true); + } + return verifyMqttCredentialsViaBroker(mqttUsername, mqttPassword).then((ok) => { + if (ok) { + authCache.set(mqttUsername, { credentialHash: hash, ts: Date.now() }); + } else if (cached && cached.credentialHash === hash) { + // Same credential that previously worked now rejected (revoked) — drop it + // AND bump the credential generation so every live owner session minted + // under the old password is invalidated immediately. + authCache.delete(mqttUsername); + void bumpOwnerCredentialGeneration(mqttUsername); + } + return ok; + }); +} // ---- Credential generation (BUG-010) ---- -// Owner sessions record the credential generation that minted them. The -// operator revocation command bumps this Redis value after a password reset, -// and every authenticated owner request compares it with the cookie generation. +// Owner sessions are stateless signed cookies with a 30-day TTL. To make +// password revocation effective immediately, every session records the +// credential generation that minted it. When the broker rejects a previously +// valid password (the only signal the backend has that a password changed), +// the generation is bumped in Redis and all older sessions stop validating. const OWNER_CRED_GEN_KEY = (username: string) => `owner:credential-gen:${username}`; const OWNER_CRED_GEN_TTL_MS = 90 * 24 * 60 * 60 * 1000; @@ -117,28 +154,27 @@ function getCredGenRedis(): Redis { export async function getOwnerCredentialGeneration(mqttUsername: string): Promise { try { const value = await getCredGenRedis().get(OWNER_CRED_GEN_KEY(mqttUsername)); - if (value === null) return 0; - const parsed = Number(value); - if (!Number.isSafeInteger(parsed) || parsed < 0) { - throw new Error('INVALID_OWNER_CREDENTIAL_GENERATION'); - } - return parsed; + const parsed = value === null ? NaN : Number(value); + return Number.isFinite(parsed) && parsed > 0 ? parsed : 0; } catch (error) { - console.error('[owner-credgen] read failed', error instanceof Error ? error.message : error); - throw new Error('OWNER_CREDENTIAL_GENERATION_UNAVAILABLE', { cause: error }); + console.error('[owner-credgen] read failed, treating as 0', error instanceof Error ? error.message : error); + return 0; } } /** * Bump the credential generation, invalidating every session minted under an - * older password. Errors propagate so an operator reset cannot report success - * unless revocation was recorded. + * older password. Failures are logged but non-fatal: the next login still + * succeeds and re-records the current generation. */ export async function bumpOwnerCredentialGeneration(mqttUsername: string): Promise { - const redis = getCredGenRedis(); - const key = OWNER_CRED_GEN_KEY(mqttUsername); - await redis.incr(key); - await redis.pexpire(key, OWNER_CRED_GEN_TTL_MS); + try { + const redis = getCredGenRedis(); + await redis.incr(OWNER_CRED_GEN_KEY(mqttUsername)); + await redis.pexpire(OWNER_CRED_GEN_KEY(mqttUsername), OWNER_CRED_GEN_TTL_MS); + } catch (error) { + console.error('[owner-credgen] bump failed', error instanceof Error ? error.message : error); + } } export async function closeOwnerCredentialGenerationClient(): Promise { diff --git a/backend/src/owner/ownerAclReconciler.ts b/backend/src/owner/ownerAclReconciler.ts index a791dd0..29521a2 100644 --- a/backend/src/owner/ownerAclReconciler.ts +++ b/backend/src/owner/ownerAclReconciler.ts @@ -1,4 +1,5 @@ import fs from 'node:fs'; +import { query } from '../db/index.js'; import { getOwnerAuthorizationSnapshot, saveOwnerAclArtifact, @@ -14,6 +15,10 @@ import { writeAclAtomically, } from '../mqtt/aclManager.js'; import { ownerGrantConfigGeneration, parseOwnerGrantConfig } from './ownerGrantConfig.js'; +import { getPublicFeedConfig } from './publicFeedConfig.js'; +import { + getActivePublicFeedReaderUsernames, +} from './publicFeedRepository.js'; export type OwnerAclMode = 'shadow' | 'apply'; @@ -37,6 +42,20 @@ function allowedEmptyAclUsers(): string[] { .filter(Boolean); } +function configuredPublicFeedReaders(): string[] { + return String(process.env['OWNER_ACL_PUBLIC_FEED_READ_USERS'] ?? '') + .split(',') + .map((value) => value.trim()) + .filter(Boolean); +} + +function systemReadwriteAclTopics(): string[] { + return String(process.env['OWNER_ACL_SYSTEM_READWRITE_TOPICS'] ?? 'public/#') + .split(',') + .map((value) => value.trim()) + .filter(Boolean); +} + let inFlight: Promise | null = null; let reconcileTimer: NodeJS.Timeout | null = null; @@ -49,16 +68,42 @@ export function reconcileOwnerAuthorization(): Promise { await syncOperatorConfiguredOwnerGrants(configured, configGeneration); const snapshot = await getOwnerAuthorizationSnapshot(); + const publicFeedConfig = getPublicFeedConfig(); + const publicFeedReaders = publicFeedConfig.configured + ? await getActivePublicFeedReaderUsernames(query) + : new Set(); + const explicitPublicFeedReaders = configuredPublicFeedReaders(); + const explicitPublicFeedReaderSet = new Set(explicitPublicFeedReaders); + const grants = snapshot.map((account) => { + const publicFeedRead = explicitPublicFeedReaderSet.has(account.mqttUsername) + || (account.isActive && publicFeedReaders.has(account.mqttUsername)); + return { + mqttUsername: account.mqttUsername, + nodeIds: account.isActive ? account.nodeIds : [], + ...(publicFeedRead ? { publicFeedRead: true } : {}), + }; + }); + const snapshotUsernames = new Set(grants.map((grant) => grant.mqttUsername)); + for (const mqttUsername of explicitPublicFeedReaders) { + if (snapshotUsernames.has(mqttUsername)) continue; + // An explicit subscriber is managed even when the public feed is + // temporarily empty or disabled. That removes any stale broad ACL and + // leaves the account read-only with no topic access until a node is + // explicitly opted in and active. + grants.push({ + mqttUsername, + nodeIds: [], + publicFeedRead: true, + }); + } const currentContent = readAclFile(); const rendered = renderOwnerAcl( currentContent, - snapshot.map((account) => ({ - mqttUsername: account.mqttUsername, - nodeIds: account.isActive ? account.nodeIds : [], - })), + grants, unmanagedAclUsers(), - allowedEmptyAclUsers(), + [...allowedEmptyAclUsers(), ...explicitPublicFeedReaders], [String(process.env['MQTT_USERNAME'] ?? 'backend').trim() || 'backend'], + systemReadwriteAclTopics(), ); await updateOwnerAclState({ desiredGeneration: rendered.generation }); await saveOwnerAclArtifact({ diff --git a/backend/src/owner/ownerDashboard.test.ts b/backend/src/owner/ownerDashboard.test.ts index 68015b8..41f50cf 100644 --- a/backend/src/owner/ownerDashboard.test.ts +++ b/backend/src/owner/ownerDashboard.test.ts @@ -71,3 +71,35 @@ test('owner dashboard keeps unnamed canonical identities separate', () => { assert.deepEqual(nodes.map((node) => node.canonicalId).sort(), [first, second].sort()); assert.deepEqual(nodes.map((node) => node.members), [[first], [second]].sort()); }); + +test('owner dashboard prefers the freshest authorized key when same-name keys are both owned', () => { + // Regression (2026-09-18, Thinford Repeater): two same-name keys are both + // authorized; owner auth returns node ids ORDER BY node_id ASC and the stale + // key sorts first. The dashboard must select the freshest member so the live + // endpoints load the active key's data instead of the dead key's zeros. + const staleKey = '35CD1CDD'.repeat(8); + const liveKey = 'F518585D'.repeat(8); + + const nodes = groupOwnerNodes([ + row({ + canonical_id: liveKey, + name: 'Thinford Repeater', + members: [liveKey], + last_seen: '2026-09-18T11:38:09Z', + advert_count: 2147, + }), + row({ + canonical_id: staleKey, + name: 'Thinford Repeater', + members: [staleKey], + last_seen: '2026-04-14T08:54:50Z', + advert_count: 0, + }), + ], [staleKey, liveKey].sort()); + + assert.equal(nodes.length, 1); + assert.equal(nodes[0].node_id, liveKey); + assert.equal(nodes[0].canonicalId, liveKey); + assert.deepEqual(nodes[0].members, [liveKey, staleKey].sort()); + assert.equal(nodes[0].advert_count, 2147); +}); diff --git a/backend/src/owner/ownerDashboard.ts b/backend/src/owner/ownerDashboard.ts index 0706ad3..f0ae012 100644 --- a/backend/src/owner/ownerDashboard.ts +++ b/backend/src/owner/ownerDashboard.ts @@ -55,7 +55,7 @@ function compareRows(left: OwnerDashboardRow, right: OwnerDashboardRow): number * rows are also combined here because owner authorization can retain older * keys that the evidence-based global merge intentionally leaves separate. * Their keys stay visible in `members`, while the freshest row supplies the - * display position and metadata. + * display position, metadata, and the authorized key used by the live endpoints. */ export function groupOwnerNodes( rows: OwnerDashboardRow[], @@ -84,8 +84,13 @@ export function groupOwnerNodes( const ordered = [...group].sort(compareRows); const representative = ordered[0]!; const members = Array.from(new Set(group.flatMap((row) => row.members))).sort(); - const accessNodeId = authorized.find((nodeId) => members.includes(nodeId)) - ?? representative.canonical_id; + // owner-dashboard-freshest-member: prefer the freshest row that has an + // authorized key so the live endpoints load the most recently seen member + // (owner authorization can retain older keys that sort first by node_id). + const accessNodeId = ordered.reduce( + (found, row) => found ?? authorized.find((nodeId) => row.members.includes(nodeId)) ?? null, + null, + ) ?? representative.canonical_id; return { node_id: accessNodeId, diff --git a/backend/src/owner/ownerLastHopPrewarm.test.ts b/backend/src/owner/ownerLastHopPrewarm.test.ts deleted file mode 100644 index 95ffcef..0000000 --- a/backend/src/owner/ownerLastHopPrewarm.test.ts +++ /dev/null @@ -1,202 +0,0 @@ -import assert from 'node:assert/strict'; -import test from 'node:test'; -import { - createOwnerLastHopPrewarm, - ownerLastHopPrewarmConcurrency, -} from './ownerLastHopPrewarm.js'; - -function capturedLog() { - const entries: Array<{ level: string; message: string; detail: unknown }> = []; - return { - entries, - log: { - log(message: string, detail: unknown) { entries.push({ level: 'log', message, detail }); }, - warn(message: string, detail: unknown) { entries.push({ level: 'warn', message, detail }); }, - }, - }; -} - -test('prewarm concurrency defaults to two and cannot exceed the DB load cap', () => { - for (const value of [undefined, null, '', 'invalid', NaN, Infinity]) { - assert.equal(ownerLastHopPrewarmConcurrency(value), 2); - } - for (const value of [0, -1, 1, '1', 1.9]) assert.equal(ownerLastHopPrewarmConcurrency(value), 1); - for (const value of [2, '2', 12]) assert.equal(ownerLastHopPrewarmConcurrency(value), 2); -}); - -test('a pass counts completed owners and nodes, logs every ten, and continues after failures', async () => { - const captured = capturedLog(); - let active = 0; - let peak = 0; - const refreshed: string[] = []; - const prewarm = createOwnerLastHopPrewarm({ - concurrency: 99, - log: captured.log, - loadOwners: async () => [ - { mqttUsername: 'a', nodeIds: Array.from({ length: 8 }, (_, i) => String(i)) }, - { mqttUsername: 'b', nodeIds: Array.from({ length: 8 }, (_, i) => String(i + 8)) }, - { mqttUsername: 'c', nodeIds: Array.from({ length: 7 }, (_, i) => String(i + 16)) }, - { mqttUsername: 'empty', nodeIds: [] }, - ], - refresh: async (_owner, nodeId) => { - active++; - peak = Math.max(peak, active); - await new Promise((resolve) => setImmediate(resolve)); - active--; - refreshed.push(nodeId); - if (nodeId === '4' || nodeId === '18') throw new Error('test refresh failed'); - }, - }); - assert.deepEqual(await prewarm.run(), { - owners: 3, ownersTotal: 3, nodes: 23, nodesTotal: 23, refreshed: 21, failed: 2, - }); - assert.equal(peak, 2); - assert.equal(new Set(refreshed).size, 23); - assert.deepEqual(captured.entries.filter((entry) => entry.message.endsWith('progress')).map((entry) => entry.detail), [ - { owners: 1, ownersTotal: 3, nodes: 10, nodesTotal: 23, refreshed: 9, failed: 1 }, - { owners: 2, ownersTotal: 3, nodes: 20, nodesTotal: 23, refreshed: 18, failed: 2 }, - ]); -}); - -test('overlapping callers share one pass and duplicate nodes are refreshed once per owner', async () => { - let release!: () => void; - const gate = new Promise((resolve) => { release = resolve; }); - let loads = 0; - let calls = 0; - const prewarm = createOwnerLastHopPrewarm({ - log: capturedLog().log, - loadOwners: async () => { loads++; return [{ mqttUsername: 'a', nodeIds: ['one', 'one'] }]; }, - refresh: async () => { calls++; await gate; }, - }); - const first = prewarm.run(); - assert.equal(prewarm.run(), first); - await new Promise((resolve) => setImmediate(resolve)); - assert.equal(loads, 1); - assert.equal(calls, 1); - release(); - assert.equal((await first).refreshed, 1); -}); - -test('slow refresh warns while the refresh is still running, including one that fails', async (t) => { - t.mock.timers.enable({ apis: ['setTimeout'] }); - const captured = capturedLog(); - let clock = 0; - let rejectRefresh!: (error: Error) => void; - const gate = new Promise((_resolve, reject) => { rejectRefresh = reject; }); - const prewarm = createOwnerLastHopPrewarm({ - now: () => clock, - log: captured.log, - loadOwners: async () => [{ mqttUsername: 'a', nodeIds: ['slow'] }], - refresh: () => gate, - }); - const result = prewarm.run(); - await Promise.resolve(); - clock = 20_001; - t.mock.timers.tick(20_001); - assert.equal(captured.entries.filter((entry) => entry.message.includes('exceeds 20s')).length, 1); - rejectRefresh(new Error('late failure')); - assert.equal((await result).failed, 1); - assert.equal(captured.entries.filter((entry) => entry.message.includes('exceeds 20s')).length, 1); -}); - -test('stopping drains current refreshes and prevents further admissions', async () => { - let release!: () => void; - const gate = new Promise((resolve) => { release = resolve; }); - let calls = 0; - const prewarm = createOwnerLastHopPrewarm({ - concurrency: 1, - log: capturedLog().log, - loadOwners: async () => [{ mqttUsername: 'a', nodeIds: ['one', 'two'] }], - refresh: async () => { calls++; await gate; }, - }); - const result = prewarm.run(); - await Promise.resolve(); - const stopping = prewarm.stop(); - release(); - await stopping; - assert.equal(calls, 1); - assert.deepEqual(await result, { - owners: 0, ownersTotal: 1, nodes: 1, nodesTotal: 2, refreshed: 1, failed: 0, - }); - await assert.rejects(prewarm.run(), /PREWARM_STOPPED/); -}); - -test('scheduled passes wait for completion before starting their interval and start is idempotent', async (t) => { - t.mock.timers.enable({ apis: ['setTimeout'] }); - let release!: () => void; - const gate = new Promise((resolve) => { release = resolve; }); - let loads = 0; - let refreshes = 0; - const prewarm = createOwnerLastHopPrewarm({ - log: capturedLog().log, - loadOwners: async () => { loads++; return [{ mqttUsername: 'a', nodeIds: ['one'] }]; }, - refresh: async () => { refreshes++; await gate; }, - }); - t.after(() => prewarm.stop()); - prewarm.start(100); - prewarm.start(100); - await new Promise((resolve) => setImmediate(resolve)); - t.mock.timers.tick(1_000); - assert.equal(loads, 1, 'a long pass cannot overlap or accumulate interval ticks'); - assert.equal(refreshes, 1); - release(); - await new Promise((resolve) => setImmediate(resolve)); - t.mock.timers.tick(99); - assert.equal(loads, 1, 'the interval begins when the pass completes'); - t.mock.timers.tick(1); - await new Promise((resolve) => setImmediate(resolve)); - assert.equal(loads, 2); - assert.equal(refreshes, 2); - await prewarm.stop(); - t.mock.timers.tick(1_000); - assert.equal(loads, 2, 'shutdown cancels the next scheduled pass'); -}); - -test('a failed owner enumeration logs the failure and the next scheduled pass recovers', async (t) => { - t.mock.timers.enable({ apis: ['setTimeout'] }); - const captured = capturedLog(); - let loads = 0; - let refreshes = 0; - const prewarm = createOwnerLastHopPrewarm({ - log: captured.log, - loadOwners: async () => { - if (++loads === 1) throw new Error('owner inventory unavailable'); - return [{ mqttUsername: 'a', nodeIds: ['one'] }]; - }, - refresh: async () => { refreshes++; }, - }); - t.after(() => prewarm.stop()); - prewarm.start(100); - await new Promise((resolve) => setImmediate(resolve)); - assert.equal(refreshes, 0); - assert.deepEqual(captured.entries, [{ - level: 'warn', message: '[owner-last-hop-prewarm] pass failed', detail: 'owner inventory unavailable', - }]); - t.mock.timers.tick(100); - await new Promise((resolve) => setImmediate(resolve)); - assert.equal(loads, 2); - assert.equal(refreshes, 1); - assert.deepEqual(captured.entries.at(-1)?.detail, { - owners: 1, ownersTotal: 1, nodes: 1, nodesTotal: 1, refreshed: 1, failed: 0, - }); -}); - -test('shutdown during owner enumeration drains the pass without admitting a refresh', async () => { - let release!: (owners: Array<{ mqttUsername: string; nodeIds: string[] }>) => void; - const inventory = new Promise>((resolve) => { release = resolve; }); - let refreshes = 0; - const prewarm = createOwnerLastHopPrewarm({ - log: capturedLog().log, - loadOwners: () => inventory, - refresh: async () => { refreshes++; }, - }); - const pass = prewarm.run(); - const stopping = prewarm.stop(); - release([{ mqttUsername: 'a', nodeIds: ['one', 'two'] }]); - await stopping; - assert.equal(refreshes, 0); - assert.deepEqual(await pass, { - owners: 0, ownersTotal: 1, nodes: 0, nodesTotal: 2, refreshed: 0, failed: 0, - }); - await assert.rejects(prewarm.run(), /PREWARM_STOPPED/); -}); diff --git a/backend/src/owner/ownerLastHopPrewarm.ts b/backend/src/owner/ownerLastHopPrewarm.ts index dfbefcb..057f17a 100644 --- a/backend/src/owner/ownerLastHopPrewarm.ts +++ b/backend/src/owner/ownerLastHopPrewarm.ts @@ -1,122 +1,107 @@ -export type OwnerLastHopPrewarmTarget = { - mqttUsername: string; - nodeIds: string[]; +// Background cache warmer for the owner dashboard "RX Strength by Last Hop" panel. +// +// Every pass it refreshes the last-hop series cached for each node of every +// active MQTT owner, for both the 7d and 30d windows, so any owner opening the +// dashboard — on either time window — hits a warm cache instead of paying for a +// cold scan on their own request. The pass is sequential with a small delay +// between nodes to keep database load gentle, and individual failures are +// logged and skipped so one bad node can never stall the whole pass. +export type OwnerLastHopPrewarmDeps = { + listOwnerNodes: () => Promise>; + refreshLastHop: (ownedNodeIds: string[], nodeId: string, windowDays: number, minAgeMs: number) => Promise; + intervalMs: number; + nodeDelayMs: number; + windows?: number[]; }; -export type OwnerLastHopPrewarmProgress = { - owners: number; - ownersTotal: number; - nodes: number; - nodesTotal: number; - refreshed: number; - failed: number; -}; +const DEFAULT_WINDOWS = [7, 30]; +const INITIAL_DELAY_MS = 20_000; -export function ownerLastHopPrewarmConcurrency(value: unknown): number { - if (value == null || value === '') return 2; - const parsed = Number(value); - return Number.isFinite(parsed) ? Math.max(1, Math.min(2, Math.trunc(parsed))) : 2; +function sleep(ms: number): Promise { + return new Promise((resolve) => { setTimeout(resolve, ms); }); } -type PrewarmDeps = { - loadOwners: () => Promise; - refresh: (owner: OwnerLastHopPrewarmTarget, nodeId: string) => Promise; - concurrency?: unknown; - now?: () => number; - log?: Pick; -}; +export function createOwnerLastHopPrewarm(deps: OwnerLastHopPrewarmDeps): () => void { + if (!Number.isFinite(deps.intervalMs) || deps.intervalMs <= 0) { + console.warn('[owner-last-hop-prewarm] disabled (invalid interval)'); + return () => undefined; + } -/** One pass at a time, at most two node refreshes, and no overlapping schedules. */ -export function createOwnerLastHopPrewarm(deps: PrewarmDeps) { - const concurrency = ownerLastHopPrewarmConcurrency(deps.concurrency); - const now = deps.now ?? Date.now; - const log = deps.log ?? console; - let inFlight: Promise | null = null; - let timer: ReturnType | null = null; - let started = false; + const windows = deps.windows && deps.windows.length > 0 ? deps.windows : DEFAULT_WINDOWS; + // Entries refreshed within half a pass interval are left alone; anything older + // is rebuilt (incrementally when a previous series exists, fully when cold). + const minAgeMs = Math.max(30_000, Math.floor(deps.intervalMs / 2)); let stopped = false; + let running = false; + let initialTimer: NodeJS.Timeout | null = null; + let intervalTimer: NodeJS.Timeout | null = null; - async function pass(): Promise { - const owners = (await deps.loadOwners()) - .map((owner) => ({ ...owner, nodeIds: [...new Set(owner.nodeIds)] })) - .filter((owner) => owner.nodeIds.length > 0); - const jobs = owners.flatMap((owner) => owner.nodeIds.map((nodeId) => ({ owner, nodeId }))); - const remaining = new Map(owners.map((owner) => [owner, owner.nodeIds.length])); - const progress: OwnerLastHopPrewarmProgress = { - owners: 0, ownersTotal: owners.length, nodes: 0, nodesTotal: jobs.length, - refreshed: 0, failed: 0, - }; - log.log('[owner-last-hop-prewarm] started', { ...progress, concurrency }); - let next = 0; - async function work() { - while (!stopped && next < jobs.length) { - const { owner, nodeId } = jobs[next++]!; - const startedAt = now(); - let warned = false; - const warnSlow = () => { - warned = true; - log.warn('[owner-last-hop-prewarm] refresh exceeds 20s', { - owner: owner.mqttUsername, nodeId, elapsedMs: now() - startedAt, - }); - }; - const slowTimer = setTimeout(warnSlow, 20_000); - slowTimer.unref(); - try { - await deps.refresh(owner, nodeId); - progress.refreshed++; - } catch (error) { - progress.failed++; - log.warn('[owner-last-hop-prewarm] refresh failed', { - owner: owner.mqttUsername, nodeId, - error: error instanceof Error ? error.message : String(error), - }); - } finally { - clearTimeout(slowTimer); - if (!warned && now() - startedAt > 20_000) warnSlow(); - progress.nodes++; - const left = remaining.get(owner)! - 1; - remaining.set(owner, left); - if (left === 0) progress.owners++; - if (progress.nodes % 10 === 0 && progress.nodes < progress.nodesTotal) { - log.log('[owner-last-hop-prewarm] progress', { ...progress }); + async function runPass(): Promise { + if (stopped || running) return; + running = true; + const startedAt = Date.now(); + let owners = 0; + let nodes = 0; + let refreshed = 0; + let skipped = 0; + let failed = 0; + try { + const active = (await deps.listOwnerNodes()).filter((account) => account.isActive && account.nodeIds.length > 0); + owners = active.length; + for (const account of active) { + if (stopped) break; + for (const nodeId of account.nodeIds) { + if (stopped) break; + nodes += 1; + for (const windowDays of windows) { + try { + const didRefresh = await deps.refreshLastHop(account.nodeIds, nodeId, windowDays, minAgeMs); + if (didRefresh) refreshed += 1; + else skipped += 1; + } catch (error) { + failed += 1; + console.warn('[owner-last-hop-prewarm] refresh failed', { + mqttUsername: account.mqttUsername, + nodeId, + windowDays, + error: error instanceof Error ? error.message : String(error), + }); + } } + if (deps.nodeDelayMs > 0) await sleep(deps.nodeDelayMs); } } - } - await Promise.all(Array.from({ length: concurrency }, () => work())); - log.log(stopped ? '[owner-last-hop-prewarm] stopped' : '[owner-last-hop-prewarm] complete', { ...progress }); - return progress; - } - - function run(): Promise { - if (inFlight) return inFlight; - if (stopped) return Promise.reject(new Error('OWNER_LAST_HOP_PREWARM_STOPPED')); - inFlight = pass().finally(() => { inFlight = null; }); - return inFlight; - } - - function start(intervalMs = 30 * 60_000): void { - if (started || stopped) return; - started = true; - const scheduled = () => { - void run().catch((error: unknown) => { - log.warn('[owner-last-hop-prewarm] pass failed', error instanceof Error ? error.message : String(error)); - }).finally(() => { - if (!stopped) { - timer = setTimeout(scheduled, intervalMs); - timer.unref(); - } + console.log('[owner-last-hop-prewarm] pass complete', { + owners, + nodes, + refreshed, + skipped, + failed, + durationMs: Date.now() - startedAt, }); - }; - scheduled(); + } catch (error) { + console.warn('[owner-last-hop-prewarm] pass failed', error instanceof Error ? error.message : error); + } finally { + running = false; + } } - async function stop(): Promise { + console.log('[owner-last-hop-prewarm] enabled', { + intervalMs: deps.intervalMs, + nodeDelayMs: deps.nodeDelayMs, + windows, + minAgeMs, + }); + initialTimer = setTimeout(() => { void runPass(); }, INITIAL_DELAY_MS); + initialTimer.unref(); + intervalTimer = setInterval(() => { void runPass(); }, deps.intervalMs); + intervalTimer.unref(); + + return () => { stopped = true; - if (timer) clearTimeout(timer); - timer = null; - await inFlight?.catch(() => undefined); - } - - return { run, start, stop }; + if (initialTimer) clearTimeout(initialTimer); + if (intervalTimer) clearInterval(intervalTimer); + initialTimer = null; + intervalTimer = null; + }; } diff --git a/backend/src/owner/ownerRepository.integration.test.ts b/backend/src/owner/ownerRepository.integration.test.ts new file mode 100644 index 0000000..238289f --- /dev/null +++ b/backend/src/owner/ownerRepository.integration.test.ts @@ -0,0 +1,93 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import test from 'node:test'; +import { promisify } from 'node:util'; +import { createOwnerRepository } from './ownerRepository.js'; + +// SELECT-only fixtures shadow every source relation. No data or DDL writes. +const sqlTest = process.env['OWNER_SQL_TESTS'] === '1' ? test : test.skip; +const run = promisify(execFile); +const owner = 'A'.repeat(64), alias = 'F'.repeat(64), peer = 'B'.repeat(64), unrelated = 'C'.repeat(64); +const literal = (value: unknown) => `'${String(value).replace(/'/g, "''")}'`; +const packet = (overrides: Record = {}) => ({ age: 1, rx_node_id_raw: owner, src_node_id: peer, packet_hash: 'hash', packet_type: 4, route_type: 1, hop_count: 0, rssi: -100, snr: 5, payload: {}, ...overrides }); +const sample = (overrides: Record = {}) => ({ age: 1, node_id: owner, network: 'ukmesh', neighbors: [{ id: peer }], stats: { battery_mv: 4100 }, ...overrides }); + +async function execute(index: number, fixtures: { packets?: unknown[]; samples?: unknown[] }, requested = owner) { + const queries: string[] = []; + await createOwnerRepository({ query: async text => { queries.push(text); return { rows: [] }; } }).fetchOwnerLiveData(requested); + // Resolve only the selected test key without reading production mappings. + const canonical = requested.trim().toUpperCase() === alias ? owner : requested.trim().toUpperCase(); + const sql = queries[index]!.replaceAll('meshcore_canonical_node_id($1)', literal(canonical)).replaceAll('$1', literal(requested)); + const fixture = `WITH + node_identity_aliases AS (SELECT ${literal(alias)}::text AS source_node_id, ${literal(owner)}::text AS canonical_node_id), + node_identity_nodes AS ( + SELECT node_id, 'fixture-' || LEFT(node_id, 1) AS name, 'ukmesh'::text AS network, + 'MME'::text AS iata, 54.5::double precision AS lat, -1.3::double precision AS lon + FROM unnest(ARRAY[${literal(owner)},${literal(peer)}]::text[]) AS n(node_id) + ), node_identity_packets AS ( + SELECT p.*, NOW() - age * INTERVAL '1 second' AS time + FROM jsonb_to_recordset(${literal(JSON.stringify(fixtures.packets ?? []))}::jsonb) AS p( + age double precision, rx_node_id_raw text, src_node_id text, packet_hash text, + packet_type integer, route_type integer, hop_count integer, payload jsonb, + rssi double precision, snr double precision + ) + ), fixture_samples AS ( + SELECT s.*, NOW() - age * INTERVAL '1 second' AS time + FROM jsonb_to_recordset(${literal(JSON.stringify(fixtures.samples ?? []))}::jsonb) AS s( + age double precision, node_id text, network text, neighbors jsonb, stats jsonb, + battery_mv double precision, uptime_secs bigint, tx_air_secs double precision, + rx_air_secs double precision, channel_utilization double precision, air_util_tx double precision + ) + ), node_neighbor_samples AS (SELECT * FROM fixture_samples), + node_status_samples AS (SELECT * FROM fixture_samples) + SELECT COALESCE(jsonb_agg(to_jsonb(result)), '[]'::jsonb) FROM (${sql}) result;`; + const child = run('docker', ['exec', '-i', 'meshcore-infra-timescaledb-1', 'psql', '-X', '-U', 'meshcore', '-d', 'meshcore', '-At', '-v', 'ON_ERROR_STOP=1'], { timeout: 20_000, maxBuffer: 4 * 1024 * 1024 }); + child.child.stdin!.end(fixture); + return JSON.parse((await child).stdout) as Array>; +} + +sqlTest('packet feed merges aliases, deduplicates newest hashes and caps nine before enrichment', async () => { + const rows = await execute(2, { packets: [ + ...Array.from({ length: 12 }, (_, i) => packet({ age: i + 10, packet_hash: String(i), rx_node_id_raw: i % 2 ? alias : owner })), + packet({ age: 1, packet_hash: '0', rx_node_id_raw: alias, payload: { decrypted: { text: 'latest duplicate', sender: 'Alice' } } }), + packet({ age: 0, packet_hash: 'unrelated', rx_node_id_raw: unrelated }), packet({ age: 31 * 86400, packet_hash: 'expired' }), + ] }); + assert.equal(rows.length, 9); + assert.deepEqual(rows.map(row => row.packet_hash), ['0','1','2','3','4','5','6','7','8']); + assert.equal(rows[0]!.body, 'latest duplicate'); + assert.equal(rows[0]!.sender, 'Alice'); + assert.equal(rows[0]!.src_node_name, 'fixture-B'); + assert.deepEqual(await execute(2, { packets: [packet()] }, unrelated), []); +}); + +sqlTest('packet budget is global across aliases and null hashes retain fallback deduplication', async () => { + const rows = await execute(2, { packets: [ + ...Array.from({ length: 5000 }, (_, i) => packet({ age: i / 100, packet_hash: 'duplicate', rx_node_id_raw: i % 2 ? alias : owner })), + packet({ age: 100, packet_hash: 'outside-global-budget' }), + ] }); + assert.deepEqual(rows.map(row => row.packet_hash), ['duplicate']); + const nulls = await execute(2, { packets: [packet({ packet_hash: null }), packet({ packet_hash: null, rx_node_id_raw: alias }), packet({ packet_hash: null, payload: { decrypted: { text: 'different' } } })] }); + assert.equal(nulls.length, 2); +}); + +sqlTest('neighbours select newest alias report, preserve empty reports and bound time and network', async () => { + const rows = await execute(7, { samples: [sample({ age: 50 }), sample({ node_id: alias, age: 5, neighbors: [] }), sample({ node_id: unrelated, age: 0 }), sample({ network: 'test', age: 0 }), sample({ age: 8 * 86400 })] }, alias); + assert.equal(rows.length, 1); + assert.deepEqual(rows[0]!.neighbors, []); + assert.deepEqual(await execute(7, { samples: [sample({ age: 7 * 86400 }), sample({ network: 'test' }), sample({ node_id: unrelated })] }), []); +}); + +sqlTest('telemetry keeps raw identities in 24 hours, chronological order and nullable fields', async () => { + const rows = await execute(6, { samples: [sample({ age: 50, stats: { battery_mv: 4050 } }), sample({ age: 10, node_id: alias, stats: {}, battery_mv: 4200 }), sample({ age: 1, node_id: alias, stats: { battery_mv: 'unknown' } }), sample({ age: 86400 }), sample({ node_id: unrelated }), sample({ network: 'test' })] }); + assert.deepEqual(rows.map(row => row.battery_mv), [4050, 4200, null]); + assert.ok(Date.parse(rows[0]!.time) < Date.parse(rows[2]!.time)); + assert.deepEqual(await execute(6, { samples: [sample()] }, unrelated), []); +}); + +sqlTest('direct evidence excludes relays, self, tx reports, unrelated receivers and expired packets', async () => { + const rows = await execute(1, { packets: [packet(), packet({ rx_node_id_raw: alias, rssi: -90, snr: 1 }), packet({ hop_count: 1 }), packet({ src_node_id: owner }), packet({ payload: { direction: 'tx' } }), packet({ rx_node_id_raw: unrelated }), packet({ age: 86400 })] }); + assert.equal(rows.length, 1); + assert.equal(rows[0]!.packets_24h, 2); + assert.equal(rows[0]!.avg_rssi, -95); + assert.equal(rows[0]!.avg_snr, 3); +}); diff --git a/backend/src/owner/ownerRepository.ts b/backend/src/owner/ownerRepository.ts index 8416c12..782a38a 100644 --- a/backend/src/owner/ownerRepository.ts +++ b/backend/src/owner/ownerRepository.ts @@ -14,15 +14,17 @@ export type OwnerRepository = ReturnType; export function createOwnerRepository(deps: OwnerRepositoryDeps) { const { query } = deps; - async function fetchLastHopStrength(rxNodeIds: string[], allOwnerNodeIds: string[], since?: string) { + async function fetchLastHopStrength(rxNodeIds: string[], allOwnerNodeIds: string[], since?: string, windowDays = 7, bucketHours = 1) { // $1 = rxNodeIds (the selected node to filter received packets) // $2 = allOwnerNodeIds (all owned nodes to exclude from last-hop analysis) - // $3 = since (optional time filter) + // $3 = since (optional time filter) or windowDays for the default rolling window + // $4 = bucket size in hours: hourly for the 7d window, 12-hour chunks for 30d const params: unknown[] = [rxNodeIds, allOwnerNodeIds]; const timeFilter = since ? `p.time >= $3::timestamptz` - : `p.time > NOW() - INTERVAL '7 days'`; - if (since) params.push(since); + : `p.time > NOW() - (($3::int) * INTERVAL '1 day')`; + params.push(since ?? windowDays); + params.push(Math.max(1, Math.trunc(bucketHours))); return query<{ bucket: string; @@ -143,7 +145,7 @@ export function createOwnerRepository(deps: OwnerRepositoryDeps) { ), classified AS ( SELECT - time_bucket('1 hour', op.time)::text AS bucket, + time_bucket((($4::int) * INTERVAL '1 hour'), op.time)::text AS bucket, CASE WHEN op.hop_count = 0 AND op.src_node_id IS NOT NULL @@ -242,7 +244,7 @@ export function createOwnerRepository(deps: OwnerRepositoryDeps) { `SELECT node_id, name, network, iata, advert_count, last_seen, lat, lon, role, identity_source_ids AS members FROM node_identity_nodes - WHERE node_id = meshcore_canonical_node_id($1) + WHERE node_id = (SELECT meshcore_canonical_node_id($1)) LIMIT 1`, [selectedNodeId], ), @@ -255,6 +257,8 @@ export function createOwnerRepository(deps: OwnerRepositoryDeps) { lon: number | null; packets_24h: number; last_seen: string | null; + avg_rssi: number | null; + avg_snr: number | null; }>( `SELECT p.src_node_id AS node_id, @@ -264,7 +268,9 @@ export function createOwnerRepository(deps: OwnerRepositoryDeps) { n.lat, n.lon, COUNT(*)::int AS packets_24h, - MAX(p.time)::text AS last_seen + MAX(p.time)::text AS last_seen, + AVG(p.rssi)::double precision AS avg_rssi, + AVG(p.snr)::double precision AS avg_snr FROM node_identity_packets p LEFT JOIN node_identity_nodes n ON n.node_id = p.src_node_id WHERE p.rx_node_id_raw IN ( @@ -274,6 +280,7 @@ export function createOwnerRepository(deps: OwnerRepositoryDeps) { WHERE canonical_node_id = meshcore_canonical_node_id($1) ) AND p.hop_count = 0 + AND COALESCE(p.payload->>'direction', 'rx') <> 'tx' AND p.src_node_id IS NOT NULL AND p.src_node_id <> meshcore_canonical_node_id($1) AND p.time > NOW() - INTERVAL '24 hours' @@ -293,7 +300,12 @@ export function createOwnerRepository(deps: OwnerRepositoryDeps) { sender: string | null; body: string | null; }>( - `WITH recent AS ( + `WITH owner_ids AS MATERIALIZED ( + SELECT meshcore_canonical_node_id($1) AS node_id + UNION + SELECT source_node_id FROM node_identity_aliases + WHERE canonical_node_id = meshcore_canonical_node_id($1) + ), recent AS MATERIALIZED ( SELECT p.time, p.packet_type, @@ -302,14 +314,14 @@ export function createOwnerRepository(deps: OwnerRepositoryDeps) { p.packet_hash, p.src_node_id, p.payload - FROM node_identity_packets p - WHERE p.rx_node_id_raw IN ( - SELECT meshcore_canonical_node_id($1) - UNION - SELECT source_node_id FROM node_identity_aliases - WHERE canonical_node_id = meshcore_canonical_node_id($1) - ) - AND p.time > NOW() - INTERVAL '30 days' + FROM owner_ids + CROSS JOIN LATERAL ( + SELECT p.* FROM node_identity_packets p + WHERE p.rx_node_id_raw = owner_ids.node_id + AND p.time > NOW() - INTERVAL '30 days' + ORDER BY p.time DESC + LIMIT 5000 + ) p ORDER BY p.time DESC LIMIT 5000 ), @@ -338,6 +350,13 @@ export function createOwnerRepository(deps: OwnerRepositoryDeps) { ORDER BY recent.time DESC ) AS rn FROM recent + ), latest AS MATERIALIZED ( + -- Deduplicate and cap the feed before enriching names. Joining every + -- ranked packet to the identity view first created millions of pairs. + SELECT * FROM ranked + WHERE rn = 1 + ORDER BY time DESC + LIMIT 9 ) SELECT r.time::text AS time, @@ -355,11 +374,9 @@ export function createOwnerRepository(deps: OwnerRepositoryDeps) { r.payload->'decoded'->>'text', r.payload->>'message' ) AS body - FROM ranked r + FROM latest r LEFT JOIN node_identity_nodes src ON src.node_id = r.src_node_id - WHERE r.rn = 1 - ORDER BY r.time DESC - LIMIT 9`, + ORDER BY r.time DESC`, [selectedNodeId], ), query<{ @@ -595,11 +612,16 @@ export function createOwnerRepository(deps: OwnerRepositoryDeps) { WHEN jsonb_typeof(stats->'tx_publish_calls') = 'number' THEN (stats->>'tx_publish_calls')::double precision ELSE NULL END AS tx_publish_calls - FROM node_identity_status_samples - WHERE node_id = meshcore_canonical_node_id($1) + FROM node_status_samples + WHERE node_id IN ( + SELECT meshcore_canonical_node_id($1) + UNION + SELECT source_node_id FROM node_identity_aliases + WHERE canonical_node_id = meshcore_canonical_node_id($1) + ) AND network = COALESCE( (SELECT network FROM node_identity_nodes - WHERE node_id = meshcore_canonical_node_id($1) + WHERE node_id = (SELECT meshcore_canonical_node_id($1)) LIMIT 1), 'ukmesh' ) @@ -608,53 +630,37 @@ export function createOwnerRepository(deps: OwnerRepositoryDeps) { [selectedNodeId], ), query<{ - id: string; - rssi: number | null; - snr: number | null; - last_seen: string | null; - sample_time: string; + time: string; + neighbors: unknown; }>( - `WITH latest AS ( - SELECT time, neighbors - FROM node_neighbor_samples - WHERE meshcore_canonical_node_id(node_id) = meshcore_canonical_node_id($1) - AND network = COALESCE( - (SELECT network FROM node_identity_nodes - WHERE node_id = meshcore_canonical_node_id($1) - LIMIT 1), - 'ukmesh' - ) - ORDER BY time DESC + `WITH owner_ids AS MATERIALIZED ( + SELECT meshcore_canonical_node_id($1) AS node_id + UNION + SELECT source_node_id FROM node_identity_aliases + WHERE canonical_node_id = meshcore_canonical_node_id($1) + ), latest AS ( + -- Probe the (node_id, time) index per identity member, including + -- absent publishers, instead of canonicalising the entire history. + SELECT sample.time, sample.neighbors + FROM owner_ids + CROSS JOIN LATERAL ( + SELECT time, neighbors + FROM node_neighbor_samples + WHERE node_id = owner_ids.node_id + AND time > NOW() - INTERVAL '7 days' + AND network = COALESCE( + (SELECT network FROM node_identity_nodes + WHERE node_id = (SELECT meshcore_canonical_node_id($1)) + LIMIT 1), + 'ukmesh' + ) + ORDER BY time DESC + LIMIT 1 + ) sample + ORDER BY sample.time DESC LIMIT 1 ) - SELECT - COALESCE(item->>'id', item->>'node_id', item->>'pubkey', item->>'public_key') AS id, - CASE - WHEN jsonb_typeof(item->'rssi') = 'number' THEN (item->>'rssi')::double precision - WHEN jsonb_typeof(item->'RSSI') = 'number' THEN (item->>'RSSI')::double precision - ELSE NULL - END AS rssi, - CASE - WHEN jsonb_typeof(item->'snr') = 'number' THEN (item->>'snr')::double precision - WHEN jsonb_typeof(item->'SNR') = 'number' THEN (item->>'SNR')::double precision - WHEN jsonb_typeof(item->'snr_db') = 'number' THEN (item->>'snr_db')::double precision - ELSE NULL - END AS snr, - COALESCE( - CASE WHEN jsonb_typeof(item->'last_seen') IN ('string', 'number') THEN item->>'last_seen' ELSE NULL END, - CASE WHEN jsonb_typeof(item->'lastSeen') IN ('string', 'number') THEN item->>'lastSeen' ELSE NULL END, - CASE WHEN jsonb_typeof(item->'heard_secs_ago') = 'number' - THEN (latest.time - ((item->>'heard_secs_ago')::double precision * INTERVAL '1 second'))::text - ELSE NULL END - ) AS last_seen, - latest.time::text AS sample_time - FROM latest - CROSS JOIN LATERAL jsonb_array_elements( - CASE WHEN jsonb_typeof(latest.neighbors) = 'array' THEN latest.neighbors ELSE '[]'::jsonb END - ) AS neighbor(item) - WHERE COALESCE(item->>'id', item->>'node_id', item->>'pubkey', item->>'public_key') IS NOT NULL - ORDER BY last_seen DESC NULLS LAST - LIMIT 32`, + SELECT time::text AS time, neighbors FROM latest`, [selectedNodeId], ), query<{ packets_24h: number }>( diff --git a/backend/src/owner/ownerService.test.ts b/backend/src/owner/ownerService.test.ts index 12d238a..27030ff 100644 --- a/backend/src/owner/ownerService.test.ts +++ b/backend/src/owner/ownerService.test.ts @@ -1,177 +1,52 @@ import assert from 'node:assert/strict'; import test from 'node:test'; -import { createOwnerService } from './ownerService.js'; -import { createOwnerRepository } from './ownerRepository.js'; +import { buildOwnerHeardNeighbors } from './ownerService.js'; -function lastHopService(fetch: ReturnType['fetchLastHopStrength']) { - const repository = createOwnerRepository({ query: async () => ({ rows: [] }) }); - repository.fetchLastHopStrength = fetch; - return createOwnerService({ - repository, - ownerLiveCache: new Map(), - ownerLiveCacheTtlMs: 15_000, - ownerDashboardCacheTtlMs: 20_000, - ownerLastHopCacheTtlMs: 60_000, - verifyMqttCredentials: async () => true, - resolveOwnerNodeIds: async () => [], - autoLinkOwnerNodeIds: async () => [], - buildOwnerDashboard: async () => ({ nodes: [] }), - invalidateOwnerNodeIdCache: () => {}, +const time = '2026-09-08T01:00:00Z'; +const direct = [{ node_id: 'direct', last_seen: time, avg_rssi: -103.5, avg_snr: 7.25 }]; + +test('absent neighbour publisher uses real direct packet signal with explicit provenance', () => { + assert.deepEqual(buildOwnerHeardNeighbors(undefined, direct), { + heardNeighbors: [{ id: 'direct', last_seen_at: '2026-09-08T01:00:00.000Z', rssi: -103.5, snr: 7.25, sampled_at: null }], + heardNeighborsSource: 'direct_packets', heardNeighborsSampledAt: null, }); -} - -test('last-hop prewarm and foreground reads share one in-flight refresh', async () => { - let calls = 0; - let release!: () => void; - const gate = new Promise((resolve) => { release = resolve; }); - const service = lastHopService(async () => { calls++; await gate; return { rows: [] }; }); - const prewarm = service.getOwnerLastHopStrength(['a'], 'a', true); - const foreground = service.getOwnerLastHopStrength(['a'], 'a'); - assert.equal(calls, 1); - release(); - assert.deepEqual(await prewarm, { points: [] }); - assert.deepEqual(await foreground, { points: [] }); - await service.getOwnerLastHopStrength(['a'], 'a'); - assert.equal(calls, 1); - await service.getOwnerLastHopStrength(['a'], 'a', true); - assert.equal(calls, 2); + assert.equal(buildOwnerHeardNeighbors(undefined, []).heardNeighborsSource, 'none'); + assert.equal(buildOwnerHeardNeighbors({ time, neighbors: null }, direct).heardNeighborsSource, 'direct_packets'); }); -test('last-hop cache respects the complete ownership exclusion scope', async () => { - const scopes: string[][] = []; - const service = lastHopService(async (_nodeIds, ownedNodeIds) => { scopes.push(ownedNodeIds); return { rows: [] }; }); - await service.getOwnerLastHopStrength(['a', 'b'], 'a'); - await service.getOwnerLastHopStrength(['a', 'c'], 'a'); - await service.getOwnerLastHopStrength(['b', 'a'], 'a'); - assert.deepEqual(scopes, [['a', 'b'], ['a', 'c']]); - await assert.rejects(service.getOwnerLastHopStrength(['b'], 'a', true), /NODE_NOT_OWNED/); +test('published lists, including empty lists, take precedence over direct packet fallback', () => { + const empty = buildOwnerHeardNeighbors({ time, neighbors: [] }, direct); + assert.equal(empty.heardNeighborsSource, 'reported'); + assert.equal(empty.heardNeighborsSampledAt, '2026-09-08T01:00:00.000Z'); + assert.deepEqual(empty.heardNeighbors, []); + const result = buildOwnerHeardNeighbors({ time, neighbors: [{ pubkey: 'reported', RSSI: -95, SNR: 0 }] }, direct); + assert.equal(result.heardNeighborsSource, 'reported'); + assert.deepEqual(result.heardNeighbors.map(n => n.id), ['reported']); + assert.equal(result.heardNeighbors[0]!.snr, 0); }); -test('a failed last-hop refresh releases single-flight state for retry', async () => { - let calls = 0; - const service = lastHopService(async () => { - if (++calls === 1) throw new Error('query failed'); - return { rows: [] }; - }); - await assert.rejects(service.getOwnerLastHopStrength(['a'], 'a', true), /query failed/); - assert.deepEqual(await service.getOwnerLastHopStrength(['a'], 'a', true), { points: [] }); - assert.equal(calls, 2); +test('neighbour timestamps normalize ISO, epoch seconds, milliseconds and relative ages before sorting', () => { + const epoch = Date.parse(time) / 1_000; + const result = buildOwnerHeardNeighbors({ time, neighbors: [ + { id: 'iso', last_seen: '2026-09-08T00:00:00-01:00', rssi: 'not a number' }, + { node_id: 'seconds', lastSeen: epoch - 1, snr_db: 1 }, + { public_key: 'milliseconds', last_seen: (epoch - 2) * 1_000 }, + { id: 'relative', heard_secs_ago: 3 }, + { id: 'unknown', last_seen: 'invalid', rssi: Infinity, snr: null }, + { id: 'fallback-timestamp', last_seen: 'invalid', lastSeen: String(epoch - 4) }, + { id: 'negative-age', heard_secs_ago: -10 }, + null, [], 'invalid', {}, { id: '' }, + ] }, direct); + assert.deepEqual(result.heardNeighbors.map(n => n.id), ['iso', 'seconds', 'milliseconds', 'relative', 'fallback-timestamp', 'unknown', 'negative-age']); + assert.equal(result.heardNeighbors[0]!.rssi, null); + assert.equal(result.heardNeighbors[3]!.last_seen_at, '2026-09-08T00:59:57.000Z'); + assert.equal(result.heardNeighbors[5]!.last_seen_at, null); + assert.equal(result.heardNeighbors[5]!.rssi, null); }); -function lastHopRow(bucket: string, sampleCount = 1) { - return { bucket, last_hop_node_id: 'peer', last_hop_name: 'Peer', resolution: 'resolved' as const, - avg_snr: 4, avg_rssi: -90, sample_count: sampleCount }; -} - -test('a warm refresh updates the current bucket, retains history and trims the rolling seven-day window', async (t) => { - const now = Date.parse('2026-10-02T12:00:00Z'); - t.mock.timers.enable({ apis: ['Date'], now }); - const expiring = new Date(now - 7 * 24 * 60 * 60_000 + 1_000).toISOString(); - const previous = '2026-10-02T10:00:00.000Z'; - const latest = '2026-10-02T11:00:00.000Z'; - const since: Array = []; - const service = lastHopService(async (_nodes, _scope, cursor) => { - since.push(cursor); - return { rows: cursor === undefined - ? [lastHopRow(expiring), lastHopRow(previous), lastHopRow(latest)] - : [lastHopRow(latest, 9), lastHopRow('2026-10-02T12:00:00.000Z', 2)] }; - }); - assert.equal((await service.getOwnerLastHopStrength(['a'], 'a')).points.length, 3); - t.mock.timers.tick(2_000); - const refreshed = await service.getOwnerLastHopStrength(['a'], 'a', true); - assert.deepEqual(since, [undefined, latest]); - assert.deepEqual(refreshed.points.map(point => [point.bucket, point.sampleCount]), [ - [previous, 1], [latest, 9], ['2026-10-02T12:00:00.000Z', 2], - ]); - assert.deepEqual(await service.getOwnerLastHopStrength(['a'], 'a'), refreshed); - assert.equal(since.length, 2, 'the completed warm refresh serves the next foreground read'); -}); - -test('warm last-hop refreshes replace groups when peer names or resolution change', async (t) => { - t.mock.timers.enable({ apis: ['Date'], now: Date.parse('2026-10-02T12:00:00Z') }); - const previous = '2026-10-02T10:00:00.000Z'; - const latest = '2026-10-02T11:00:00.000Z'; - const historical = lastHopRow(previous, 3); - const renamed = { ...lastHopRow(latest, 4), last_hop_name: 'Renamed peer' }; - const resolved = { ...lastHopRow(latest, 2), last_hop_node_id: 'new-peer', last_hop_name: 'New peer' }; - const service = lastHopService(async (_nodes, _scope, cursor) => ({ rows: cursor === undefined - ? [historical, lastHopRow(latest, 4), { - ...lastHopRow(latest, 2), last_hop_node_id: null, last_hop_name: 'Unresolved', resolution: 'unresolved', - }] - : [renamed, resolved] })); - await service.getOwnerLastHopStrength(['a'], 'a'); - const refreshed = await service.getOwnerLastHopStrength(['a'], 'a', true); - assert.deepEqual(refreshed.points.map(point => [point.bucket, point.lastHopNodeId, point.lastHopName, point.resolution, point.sampleCount]), [ - [previous, 'peer', 'Peer', 'resolved', 3], - [latest, 'peer', 'Renamed peer', 'resolved', 4], - [latest, 'new-peer', 'New peer', 'resolved', 2], - ]); - assert.deepEqual(await service.getOwnerLastHopStrength(['a'], 'a'), refreshed); -}); - -test('an empty warm last-hop result removes obsolete current groups and preserves earlier history', async (t) => { - t.mock.timers.enable({ apis: ['Date'], now: Date.parse('2026-10-02T12:00:00Z') }); - const previous = '2026-10-02T10:00:00.000Z'; - const latest = '2026-10-02T11:00:00.000Z'; - const service = lastHopService(async (_nodes, _scope, cursor) => ({ rows: cursor === undefined - ? [lastHopRow(previous, 3), lastHopRow(latest, 2)] : [] })); - await service.getOwnerLastHopStrength(['a'], 'a'); - const refreshed = await service.getOwnerLastHopStrength(['a'], 'a', true); - assert.deepEqual(refreshed.points.map(point => [point.bucket, point.sampleCount]), [[previous, 3]]); -}); - -test('a failed warm refresh preserves the prior foreground cache and allows a later retry', async (t) => { - t.mock.timers.enable({ apis: ['Date'], now: Date.parse('2026-10-02T12:00:00Z') }); - let calls = 0; - const service = lastHopService(async () => { - if (++calls === 2) throw new Error('warm query failed'); - return { rows: [lastHopRow('2026-10-02T11:00:00.000Z', calls)] }; - }); - const cached = await service.getOwnerLastHopStrength(['a'], 'a'); - await assert.rejects(service.getOwnerLastHopStrength(['a'], 'a', true), /warm query failed/); - assert.deepEqual(await service.getOwnerLastHopStrength(['a'], 'a'), cached); - assert.equal(calls, 2, 'failed refreshes must not evict useful cached data'); - assert.equal((await service.getOwnerLastHopStrength(['a'], 'a', true)).points[0]?.sampleCount, 3); -}); - -test('expired last-hop caches start a fresh bounded-window query instead of reusing an old cursor', async (t) => { - t.mock.timers.enable({ apis: ['Date'], now: Date.parse('2026-10-02T12:00:00Z') }); - const since: Array = []; - const service = lastHopService(async (_nodes, _scope, cursor) => { - since.push(cursor); - return { rows: [lastHopRow('2026-10-02T11:00:00.000Z')] }; - }); - await service.getOwnerLastHopStrength(['a'], 'a'); - t.mock.timers.tick(60_000); - await service.getOwnerLastHopStrength(['a'], 'a', true); - assert.deepEqual(since, [undefined, undefined]); -}); - -test('fresh foreground last-hop data stays responsive while a background warm refresh is pending', async (t) => { - t.mock.timers.enable({ apis: ['Date'], now: Date.parse('2026-10-02T12:00:00Z') }); - let release!: () => void; - const gate = new Promise(resolve => { release = resolve; }); - let calls = 0; - const service = lastHopService(async () => { - const count = ++calls; - if (count === 2) await gate; - return { rows: [lastHopRow('2026-10-02T11:00:00.000Z', count)] }; - }); - const cached = await service.getOwnerLastHopStrength(['a'], 'a'); - const warming = service.getOwnerLastHopStrength(['a'], 'a', true); - t.after(async () => { release(); await warming; }); - const joinedWarm = service.getOwnerLastHopStrength(['a'], 'a', true); - let foregroundSettled = false; - const foreground = service.getOwnerLastHopStrength(['a'], 'a').then(result => { - foregroundSettled = true; - return result; - }); - await new Promise(resolve => setImmediate(resolve)); - assert.equal(foregroundSettled, true, 'a fresh cache hit must not wait for a slow background query'); - assert.deepEqual(await foreground, cached); - assert.equal(calls, 2, 'overlapping warm requests still share the one refresh'); - release(); - assert.deepEqual(await joinedWarm, await warming); - assert.equal((await service.getOwnerLastHopStrength(['a'], 'a')).points[0]?.sampleCount, 2); - assert.equal(calls, 2); +test('neighbour cap keeps the latest 32 rather than the first 32 reported entries', () => { + const result = buildOwnerHeardNeighbors({ time, neighbors: Array.from({ length: 40 }, (_, i) => ({ id: String(i), heard_secs_ago: 39 - i })) }, []); + assert.equal(result.heardNeighbors.length, 32); + assert.equal(result.heardNeighbors[0]!.id, '39'); + assert.equal(result.heardNeighbors[31]!.id, '8'); }); diff --git a/backend/src/owner/ownerService.ts b/backend/src/owner/ownerService.ts index 83c32ac..7c1f6bd 100644 --- a/backend/src/owner/ownerService.ts +++ b/backend/src/owner/ownerService.ts @@ -58,6 +58,47 @@ function neighborTimestamp(value: unknown): string | null { return isoTimestamp(value); } +export function buildOwnerHeardNeighbors( + sample: { time: string; neighbors: unknown } | undefined, + directPeers: Array<{ node_id: string; last_seen: string | null; avg_rssi: number | null; avg_snr: number | null }>, +) { + const sampledAt = isoTimestamp(sample?.time); + const reported = sample !== undefined && Array.isArray(sample.neighbors); + // An explicitly empty published list is authoritative. The fallback is only + // for nodes that have not supplied a usable list in the reporting window. + const neighbors = reported ? (sample.neighbors as unknown[]).flatMap((entry) => { + if (!entry || typeof entry !== 'object' || Array.isArray(entry)) return []; + const item = entry as Record; + const id = [item['id'], item['node_id'], item['pubkey'], item['public_key']] + .find((value): value is string => typeof value === 'string' && value.trim() !== ''); + if (!id) return []; + const signal = (...values: unknown[]) => values.find((value): value is number => typeof value === 'number' && Number.isFinite(value)) ?? null; + const age = signal(item['heard_secs_ago']); + const relativeTime = sampledAt && age !== null && age >= 0 + ? neighborTimestamp(Date.parse(sampledAt) / 1_000 - age) + : null; + return [{ + id, + rssi: signal(item['rssi'], item['RSSI']), + snr: signal(item['snr'], item['SNR'], item['snr_db']), + last_seen_at: neighborTimestamp(item['last_seen']) ?? neighborTimestamp(item['lastSeen']) ?? relativeTime, + sampled_at: sampledAt, + }]; + }) : directPeers.map((peer) => ({ + id: peer.node_id, + rssi: nullableNumber(peer.avg_rssi), + snr: nullableNumber(peer.avg_snr), + last_seen_at: isoTimestamp(peer.last_seen), + sampled_at: null, + })); + neighbors.sort((a, b) => (Date.parse(b.last_seen_at ?? '') || 0) - (Date.parse(a.last_seen_at ?? '') || 0)); + return { + heardNeighbors: neighbors.slice(0, 32), + heardNeighborsSource: reported ? 'reported' as const : directPeers.length > 0 ? 'direct_packets' as const : 'none' as const, + heardNeighborsSampledAt: reported ? sampledAt : null, + }; +} + type OwnerServiceDeps = { ownerLiveCacheTtlMs: number; ownerLiveCache: Map; @@ -90,13 +131,6 @@ export function createOwnerService(deps: OwnerServiceDeps) { maxWeight: 64 * 1024 * 1024, ttlMs: ownerLastHopCacheTtlMs, }); - const ownerLastHopInflight = new Map>(); - - function lastHopCacheKey(nodeId: string, ownedNodeIds: string[]): string { - // Last-hop analysis excludes every node owned by this owner, so a shared - // node must never reuse a result from a different ownership scope. - return `${nodeId}|${[...new Set(ownedNodeIds)].sort().join(',')}`; - } const ownerDashboardCache = new BoundedTtlMap Date.parse(point.bucket) >= cutoff) .sort((a, b) => a.bucket.localeCompare(b.bucket)); } - function replaceLastHopPoints(existing: LastHopStrengthPoint[], recent: LastHopStrengthPoint[], since: string): LastHopStrengthPoint[] { - // The inclusive query recomputes every group from this bucket onward. Peer - // renames, resolution changes or removals must replace the old groups too. - const cursor = Date.parse(since); - return trimLastHopPoints([ - ...existing.filter((point) => Date.parse(point.bucket) < cursor), - ...recent, - ]); + function mergeLastHopPoints(existing: LastHopStrengthPoint[], recent: LastHopStrengthPoint[], windowDays: number): LastHopStrengthPoint[] { + const merged = new Map(); + for (const point of existing) { + merged.set(`${point.bucket}|${point.lastHopNodeId ?? ''}|${point.lastHopName}|${point.resolution}`, point); + } + for (const point of recent) { + merged.set(`${point.bucket}|${point.lastHopNodeId ?? ''}|${point.lastHopName}|${point.resolution}`, point); + } + return trimLastHopPoints(Array.from(merged.values()), windowDays); } function latestBucketOf(points: LastHopStrengthPoint[]): string | null { return points.length > 0 ? points[points.length - 1]!.bucket : null; } - async function buildOwnerLastHopResponse(nodeId: string, allOwnedNodeIds: string[], since?: string): Promise { - const lastHopStrengthResult = await repository.fetchLastHopStrength([nodeId], allOwnedNodeIds, since); + // Longer windows aggregate into larger chunks: the 30d series renders as + // 12-hour buckets (~60 points) instead of ~720 hourly points per repeater. + function lastHopBucketHours(windowDays: number): number { + return windowDays > 7 ? 12 : 1; + } + + async function buildOwnerLastHopResponse(nodeId: string, allOwnedNodeIds: string[], since?: string, windowDays = 7): Promise { + const lastHopStrengthResult = await repository.fetchLastHopStrength([nodeId], allOwnedNodeIds, since, windowDays, lastHopBucketHours(windowDays)); return mapLastHopRows(lastHopStrengthResult.rows); } @@ -283,20 +324,7 @@ export function createOwnerService(deps: OwnerServiceDeps) { last_seen: row.last_seen ? new Date(row.last_seen).toISOString() : null, })); - const heardNeighbors = heardNeighborsResult.rows - .map((row) => ({ - id: row.id, - rssi: nullableNumber(row.rssi), - snr: nullableNumber(row.snr), - last_seen_at: neighborTimestamp(row.last_seen), - sampled_at: isoTimestamp(row.sample_time), - })) - .sort((a, b) => { - const aTime = a.last_seen_at ? Date.parse(a.last_seen_at) : 0; - const bTime = b.last_seen_at ? Date.parse(b.last_seen_at) : 0; - return bTime - aTime; - }) - .slice(0, 32); + const neighborData = buildOwnerHeardNeighbors(heardNeighborsResult.rows[0], filteredIncomingRows); const linkHealth = linkHealthResult.rows.map((row) => ({ ...row, @@ -517,7 +545,7 @@ export function createOwnerService(deps: OwnerServiceDeps) { advertTrend24h, telemetry24h, status, - heardNeighbors, + ...neighborData, packetsSent24h: Number(packetsSentResult.rows[0]?.packets_24h ?? 0), packetsReceived24h: Number(packetsReceivedResult.rows[0]?.packets_24h ?? 0), alerts, @@ -530,7 +558,7 @@ export function createOwnerService(deps: OwnerServiceDeps) { return responseData; } - async function getOwnerLastHopStrength(ownedNodeIds: string[], requestedNodeId?: string, refresh = false): Promise { + async function getOwnerLastHopStrength(ownedNodeIds: string[], requestedNodeId?: string, windowDays = 7): Promise { if (ownedNodeIds.length < 1) { throw new Error('NO_OWNED_NODES'); } @@ -542,35 +570,62 @@ export function createOwnerService(deps: OwnerServiceDeps) { throw new Error('NODE_NOT_OWNED'); } - const cacheKey = lastHopCacheKey(selectedNodeId, ownedNodeIds); - // Serve fresh foreground hits while the background pass fills a replacement. + // Window is part of the cache key: the 7d and 30d series build and merge + // independently instead of trimming each other's points. + const cacheKey = `${windowDays}d:${selectedNodeId}`; const cacheEntry = ownerLastHopCache.get(cacheKey); - if (!refresh && cacheEntry && Date.now() - cacheEntry.ts < ownerLastHopCacheTtlMs) { + if (cacheEntry && Date.now() - cacheEntry.ts < ownerLastHopCacheTtlMs) { return cacheEntry.data; } - const inFlight = ownerLastHopInflight.get(cacheKey); - if (inFlight) return inFlight; - const load = async () => { - let responseData: OwnerLastHopResponse; - if (!cacheEntry || !cacheEntry.latestBucket) { - responseData = await buildOwnerLastHopResponse(selectedNodeId, ownedNodeIds); - } else { - const recent = await buildOwnerLastHopResponse(selectedNodeId, ownedNodeIds, cacheEntry.latestBucket); - responseData = { - points: replaceLastHopPoints(cacheEntry.data.points, recent.points, cacheEntry.latestBucket), - }; - } - ownerLastHopCache.set(cacheKey, { - ts: Date.now(), data: responseData, latestBucket: latestBucketOf(responseData.points), - }); - return responseData; - }; - const tracked = load().finally(() => { - if (ownerLastHopInflight.get(cacheKey) === tracked) ownerLastHopInflight.delete(cacheKey); + let responseData: OwnerLastHopResponse; + if (!cacheEntry || !cacheEntry.latestBucket) { + responseData = await buildOwnerLastHopResponse(selectedNodeId, ownedNodeIds, undefined, windowDays); + } else { + const recent = await buildOwnerLastHopResponse(selectedNodeId, ownedNodeIds, cacheEntry.latestBucket, windowDays); + responseData = { + points: mergeLastHopPoints(cacheEntry.data.points, recent.points, windowDays), + }; + } + + ownerLastHopCache.set(cacheKey, { + ts: Date.now(), + data: responseData, + latestBucket: latestBucketOf(responseData.points), }); - ownerLastHopInflight.set(cacheKey, tracked); - return tracked; + return responseData; + } + + // Forced refresh used by the background prewarm loop: rebuilds (or incrementally + // extends) the cached last-hop series for one node/window pair without the + // cache-freshness shortcut, so warm entries never expire out of the bounded TTL. + // Returns false when a recent entry was left untouched because of minAgeMs. + async function refreshOwnerLastHopStrength(ownedNodeIds: string[], nodeId: string, windowDays = 7, minAgeMs = 0): Promise { + if (!ownedNodeIds.includes(nodeId)) { + throw new Error('NODE_NOT_OWNED'); + } + const cacheKey = `${windowDays}d:${nodeId}`; + const cacheEntry = ownerLastHopCache.get(cacheKey); + if (cacheEntry && minAgeMs > 0 && Date.now() - cacheEntry.ts < minAgeMs) { + return false; + } + + let responseData: OwnerLastHopResponse; + if (cacheEntry && cacheEntry.latestBucket) { + const recent = await buildOwnerLastHopResponse(nodeId, ownedNodeIds, cacheEntry.latestBucket, windowDays); + responseData = { + points: mergeLastHopPoints(cacheEntry.data.points, recent.points, windowDays), + }; + } else { + responseData = await buildOwnerLastHopResponse(nodeId, ownedNodeIds, undefined, windowDays); + } + + ownerLastHopCache.set(cacheKey, { + ts: Date.now(), + data: responseData, + latestBucket: latestBucketOf(responseData.points), + }); + return true; } function clearOwnerSession(mqttUsername: string): void { @@ -579,7 +634,11 @@ export function createOwnerService(deps: OwnerServiceDeps) { ownerDashboardCache.delete(cacheKey); for (const nodeId of nodeIds) { ownerLiveCache.delete(nodeId); - ownerLastHopCache.delete(lastHopCacheKey(nodeId, nodeIds)); + for (const lastHopKey of Array.from(ownerLastHopCache.keys())) { + if (lastHopKey.endsWith(`:${nodeId}`)) { + ownerLastHopCache.delete(lastHopKey); + } + } } invalidateOwnerNodeIdCache(mqttUsername); } @@ -589,6 +648,7 @@ export function createOwnerService(deps: OwnerServiceDeps) { getSessionDashboard, getOwnerLiveData, getOwnerLastHopStrength, + refreshOwnerLastHopStrength, clearOwnerSession, }; } diff --git a/backend/src/owner/packetShareAuth.test.ts b/backend/src/owner/packetShareAuth.test.ts new file mode 100644 index 0000000..d98c2e6 --- /dev/null +++ b/backend/src/owner/packetShareAuth.test.ts @@ -0,0 +1,52 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { verifyAuthToken } from '@michaelhart/meshcore-decoder'; +import { generatePacketShareIdentity, packetShareCredentials } from './packetShareAuth.js'; +import { decryptPacketShareSecret, encryptPacketShareSecret } from './packetShareCrypto.js'; +import { destinationConfigured, type PacketShareDestinationRow } from './packetShareConfig.js'; + +const encryptionKey = 'test-packet-share-encryption-key'; + +test('generated MeshCore relay keys sign verifiable, renewable ScotMesh JWTs', async () => { + const identity = await generatePacketShareIdentity(encryptionKey); + assert.match(identity.username, /^v1_[0-9A-F]{64}$/); + assert.match(decryptPacketShareSecret(identity.privateKeyCiphertext, encryptionKey), /^[0-9a-f]{128}$/); + const destination: PacketShareDestinationRow = { + destination_id: 'scotmesh', display_name: 'ScotMesh', + broker_url: 'wss://scotmesh-mqtt.mm7roq.compute.oarc.uk:443/', + topic_template: 'meshcore/MME/{observerId}/packets', + broker_username: identity.username, broker_password_ciphertext: identity.privateKeyCiphertext, + auth_mode: 'meshcore_jwt', auth_audience: 'scotmesh-mqtt.mm7roq.compute.oarc.uk', enabled: true, + }; + assert.equal(destinationConfigured(destination), true); + const now = Math.floor(Date.now() / 1_000); + const credentials = await packetShareCredentials(destination, now, encryptionKey); + const claims = await verifyAuthToken(credentials.password!, identity.username.slice(3)); + assert.equal(claims?.aud, destination.auth_audience); + assert.equal(claims?.publicKey, identity.username.slice(3)); + assert.equal(claims?.iat, now); + assert.equal(claims?.exp, now + 3_600); + const renewed = await packetShareCredentials(destination, now + 60, encryptionKey); + assert.notEqual(renewed.password, credentials.password); + assert.equal(renewed.expiresAt, now + 3_660); + const otherIdentity = await generatePacketShareIdentity(encryptionKey); + await assert.rejects(packetShareCredentials({ ...destination, broker_username: otherIdentity.username }, now, encryptionKey), /does not match/); + assert.equal(destinationConfigured({ ...destination, auth_audience: 'another-broker.example' }), false); + assert.equal(destinationConfigured({ ...destination, broker_url: 'ws://scotmesh-mqtt.mm7roq.compute.oarc.uk:443/' }), false); + assert.equal(destinationConfigured(destination, false), false); +}); + +test('existing password and anonymous destinations keep their authentication', async () => { + const destination: PacketShareDestinationRow = { + destination_id: 'remote', display_name: 'Remote', broker_url: 'mqtts://example.test:8883', + topic_template: 'packets', broker_username: 'owner', + broker_password_ciphertext: encryptPacketShareSecret(' existing-password ', encryptionKey), enabled: true, + }; + const credentials = await packetShareCredentials(destination, undefined, encryptionKey); + assert.equal(credentials.username, 'owner'); + assert.equal(credentials.password, ' existing-password '); + assert.equal(credentials.expiresAt, Infinity); + const anonymous = await packetShareCredentials({ ...destination, broker_username: null, broker_password_ciphertext: null }); + assert.equal(anonymous.username, undefined); + assert.equal(anonymous.password, undefined); +}); diff --git a/backend/src/owner/packetShareAuth.ts b/backend/src/owner/packetShareAuth.ts new file mode 100644 index 0000000..e146c5f --- /dev/null +++ b/backend/src/owner/packetShareAuth.ts @@ -0,0 +1,56 @@ +import { createHash, randomBytes } from 'node:crypto'; +import { createAuthToken, Utils } from '@michaelhart/meshcore-decoder'; +import type { PacketShareDestinationRow } from './packetShareConfig.js'; +import { decryptPacketShareSecret, encryptPacketShareSecret } from './packetShareCrypto.js'; + +const TOKEN_TTL_SECONDS = 3_600; + +/** orlp/ed25519 uses a clamped SHA-512 expansion of a random 32-byte seed. */ +export async function generatePacketShareIdentity(encryptionKey?: string): Promise<{ + username: string; + privateKeyCiphertext: string; +}> { + const privateKey = createHash('sha512').update(randomBytes(32)).digest(); + privateKey[0] = privateKey[0]! & 248; + privateKey[31] = (privateKey[31]! & 63) | 64; + try { + const privateKeyHex = privateKey.toString('hex'); + const publicKey = (await Utils.derivePublicKey(privateKeyHex)).toUpperCase(); + return { + username: `v1_${publicKey}`, + privateKeyCiphertext: encryptPacketShareSecret(privateKeyHex, encryptionKey), + }; + } finally { + privateKey.fill(0); + } +} + +export async function packetShareCredentials( + destination: PacketShareDestinationRow, + nowSeconds = Math.floor(Date.now() / 1_000), + encryptionKey?: string, +): Promise<{ username?: string; password?: string; expiresAt: number }> { + const username = destination.broker_username?.trim() || undefined; + const secret = destination.broker_password_ciphertext + ? decryptPacketShareSecret(destination.broker_password_ciphertext, encryptionKey) + : undefined; + if (destination.auth_mode !== 'meshcore_jwt') { + return { username, password: secret, expiresAt: Infinity }; + } + const privateKey = secret?.trim(); + if (!username || !/^v1_[0-9A-F]{64}$/.test(username) + || !privateKey || !/^[0-9a-fA-F]{128}$/.test(privateKey) + || !destination.auth_audience) throw new Error('MeshCore signing credentials are incomplete'); + const publicKey = username.slice(3); + if ((await Utils.derivePublicKey(privateKey)).toUpperCase() !== publicKey) { + throw new Error('MeshCore signing key does not match the public key'); + } + const expiresAt = nowSeconds + TOKEN_TTL_SECONDS; + const password = await createAuthToken({ + publicKey, + aud: destination.auth_audience, + iat: nowSeconds, + exp: expiresAt, + }, privateKey, publicKey); + return { username, password, expiresAt }; +} diff --git a/backend/src/owner/packetShareConfig.test.ts b/backend/src/owner/packetShareConfig.test.ts index 2377750..51eb5ee 100644 --- a/backend/src/owner/packetShareConfig.test.ts +++ b/backend/src/owner/packetShareConfig.test.ts @@ -3,9 +3,12 @@ import test from 'node:test'; import { buildPacketSharePayload, destinationConfigured, + destinationAcceptsPacket, + destinationAllowedForOwner, isValidPacketShareWebsiteUrl, isForwardablePacket, renderPacketShareTopic, + renderDestinationTopic, } from './packetShareConfig.js'; import { decryptPacketShareSecret, encryptPacketShareSecret } from './packetShareCrypto.js'; import type { LivePacket } from '../types/index.js'; @@ -111,3 +114,37 @@ test('packet share secrets round-trip with authenticated encryption', () => { assert.equal(decryptPacketShareSecret(encrypted, 'test-encryption-key'), 'broker-password'); assert.throws(() => decryptPacketShareSecret(encrypted, 'wrong-encryption-key')); }); + +test('account and observer restrictions require exact matches', () => { + const scope = { allowed_owner_username: 'node1', allowed_node_id: packet.rxNodeId }; + assert.equal(destinationAllowedForOwner(scope, 'node1', packet.rxNodeId!), true); + assert.equal(destinationAllowedForOwner(scope, 'node2', packet.rxNodeId!), false); + assert.equal(destinationAllowedForOwner(scope, 'NODE1', packet.rxNodeId!), false); + assert.equal(destinationAllowedForOwner(scope, 'node1', 'C'.repeat(64)), false); +}); + +test('advert TX mode preserves RX but rejects other transmissions and relayed adverts', () => { + const destination = { tx_mode: 'advert' as const }; + assert.equal(destinationAcceptsPacket(destination, { ...packet, direction: 'rx' }, {}), true); + assert.equal(destinationAcceptsPacket(destination, { ...packet, direction: 'tx' }, {}), false); + assert.equal(destinationAcceptsPacket(destination, { ...packet, packetType: 4, direction: 'tx', srcNodeId: packet.rxNodeId }, {}), true); + assert.equal(destinationAcceptsPacket(destination, { ...packet, packetType: 4, direction: 'tx', srcNodeId: 'C'.repeat(64) }, {}), false); + assert.equal(destinationAcceptsPacket(destination, packet, { direction: 'tx' }), false); + assert.equal(destinationAcceptsPacket(destination, { ...packet, direction: 'unknown' }, {}), false); +}); + +test('ScotMesh uses MME and the relay identity in the MQTT topic and origin_id', () => { + const relayPublicKey = 'D'.repeat(64); + const destination = { + destination_id: 'scotmesh', display_name: 'ScotMesh', enabled: true, + broker_url: 'wss://scotmesh-mqtt.mm7roq.compute.oarc.uk:443/', + topic_template: 'meshcore/MME/{observerId}/packets', + broker_username: `v1_${relayPublicKey}`, broker_password_ciphertext: null, + auth_mode: 'meshcore_jwt' as const, + }; + assert.equal(renderDestinationTopic(destination, packet), `meshcore/MME/${relayPublicKey}/packets`); + const payload = buildPacketSharePayload(packet, { raw: '0102', origin: 'Gadget-RPT-TBeam', origin_id: packet.rxNodeId }, destination); + assert.equal(payload.origin_id, relayPublicKey); + assert.equal(payload.origin, 'Gadget-RPT-TBeam'); + assert.equal(payload.raw, '0102'); +}); diff --git a/backend/src/owner/packetShareConfig.ts b/backend/src/owner/packetShareConfig.ts index 5dad676..92c58ce 100644 --- a/backend/src/owner/packetShareConfig.ts +++ b/backend/src/owner/packetShareConfig.ts @@ -15,6 +15,11 @@ export type PacketShareDestinationRow = { topic_template: string | null; broker_username: string | null; broker_password_ciphertext: string | null; + allowed_owner_username?: string | null; + allowed_node_id?: string | null; + auth_mode?: 'password' | 'meshcore_jwt'; + auth_audience?: string | null; + tx_mode?: 'on' | 'advert'; enabled: boolean; }; @@ -79,7 +84,7 @@ export function isValidPacketShareTopicTemplate(template: string | null): boolea } export function destinationConfigured( - destination: Pick, + destination: Pick, encryptionKeyAvailable = true, ): boolean { if (!destination.enabled @@ -89,9 +94,37 @@ export function destinationConfigured( const password = destination.broker_password_ciphertext?.trim() || ''; if (Boolean(username) !== Boolean(password)) return false; if (password && (!encryptionKeyAvailable || !isPacketShareSecretCiphertext(password))) return false; + if (destination.auth_mode === 'meshcore_jwt') { + const url = new URL(destination.broker_url!.trim()); + if (url.protocol !== 'wss:' + || destination.auth_audience !== url.hostname + || !/^v1_[0-9A-F]{64}$/.test(username) + || !password) return false; + } return true; } +export function destinationAllowedForOwner( + destination: Pick, + ownerUsername: string, + nodeId: string, +): boolean { + return (destination.allowed_owner_username == null || destination.allowed_owner_username === ownerUsername) + && (destination.allowed_node_id == null || destination.allowed_node_id === nodeId); +} + +/** Match mqtt.tx advert: receive capture stays enabled; TX includes only own adverts. */ +export function destinationAcceptsPacket( + destination: Pick, + packet: LivePacket, + upstreamEnvelope: Record, +): boolean { + if (destination.tx_mode !== 'advert') return true; + const direction = String(packet.direction ?? upstreamEnvelope['direction'] ?? 'rx').trim().toLowerCase(); + if (direction === 'rx') return true; + return direction === 'tx' && packet.packetType === 4 && packet.srcNodeId === packet.rxNodeId; +} + export function renderPacketShareTopic(template: string, packet: LivePacket): string { if (!isValidPacketShareTopicTemplate(template)) { throw new Error('invalid packet share destination topic'); @@ -111,6 +144,13 @@ export function renderPacketShareTopic(template: string, packet: LivePacket): st return rendered; } +export function renderDestinationTopic(destination: PacketShareDestinationRow, packet: LivePacket): string { + const observerId = destination.auth_mode === 'meshcore_jwt' + ? destination.broker_username?.slice(3) + : packet.rxNodeId; + return renderPacketShareTopic(destination.topic_template!, { ...packet, rxNodeId: observerId }); +} + function copyEnvelopeValue(target: PacketShareForwardPayload, source: Record, key: string): void { const value = source[key]; if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') { @@ -138,6 +178,7 @@ function packetDateParts(packet: LivePacket, timestamp: unknown): { time: string export function buildPacketSharePayload( packet: LivePacket, upstreamEnvelope: Record, + destination?: PacketShareDestinationRow, ): PacketShareForwardPayload { const envelope: PacketShareForwardPayload = {}; for (const key of [ @@ -150,10 +191,12 @@ export function buildPacketSharePayload( if (!('timestamp' in envelope)) envelope.timestamp = new Date(packet.ts).toISOString(); if (!('hash' in envelope)) envelope.hash = packet.packetHash; if (!('type' in envelope)) envelope.type = 'PACKET'; - if (!('direction' in envelope)) envelope.direction = 'rx'; + if (!('direction' in envelope)) envelope.direction = packet.direction === 'tx' ? 'tx' : 'rx'; if (!('origin_id' in envelope) && packet.rxNodeId) envelope.origin_id = packet.rxNodeId; if (!('packet_type' in envelope) && packet.packetType != null) envelope.packet_type = String(packet.packetType); if (!('len' in envelope) && typeof envelope.raw === 'string') envelope.len = String(Math.floor(envelope.raw.length / 2)); + // JWT brokers authorize both the topic and origin_id against the relay identity. + if (destination?.auth_mode === 'meshcore_jwt') envelope.origin_id = destination.broker_username!.slice(3); const dateParts = packetDateParts(packet, envelope.timestamp); if (dateParts.time) envelope.time = dateParts.time; diff --git a/backend/src/owner/packetShareForwarder.test.ts b/backend/src/owner/packetShareForwarder.test.ts index 25b2abb..637a073 100644 --- a/backend/src/owner/packetShareForwarder.test.ts +++ b/backend/src/owner/packetShareForwarder.test.ts @@ -3,8 +3,23 @@ import test from 'node:test'; import type { LivePacket } from '../types/index.js'; import type { PacketSharePublisher } from './packetShareForwarder.js'; import { createPacketShareForwarder } from './packetShareForwarder.js'; +import { generatePacketShareIdentity } from './packetShareAuth.js'; import type { PacketShareRepository } from './packetShareRepository.js'; +async function withGuard(promise: Promise, label: string): Promise { + let timer: NodeJS.Timeout | undefined; + try { + return await Promise.race([ + promise, + new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(new Error(`${label} hung`)), 5_000); + }), + ]); + } finally { + if (timer) clearTimeout(timer); + } +} + const packet: LivePacket = { id: 'packet-event', packetHash: 'B'.repeat(16), @@ -97,3 +112,182 @@ test('forwarder does not queue a privacy-rejected packet', async () => { await forwarder.stop(); assert.equal(targets, 0); }); + +test('forwarder rejects a stale queued ScotMesh delivery for another account or observer', async () => { + let published = 0; + let failed = 0; + let claimed = false; + const repository = { + reclaimExpiredClaims: async () => 0, + claimDue: async () => { + if (claimed) return []; + claimed = true; + return [ + { id: '1', owner_username: 'node2', node_id: packet.rxNodeId, destination_id: 'scotmesh', attempts: 1 }, + { id: '2', owner_username: 'node1', node_id: 'C'.repeat(64), destination_id: 'scotmesh', attempts: 1 }, + ]; + }, + getDestination: async () => ({ allowed_owner_username: 'node1', allowed_node_id: packet.rxNodeId }), + markFailed: async () => { failed += 1; }, + } as unknown as PacketShareRepository; + const forwarder = createPacketShareForwarder({ repository, publisherFactory: () => { + published += 1; + throw new Error('unauthorized destination reached publisher'); + } }); + await forwarder.processDue(); + await forwarder.stop(); + assert.equal(failed, 2); + assert.equal(published, 0); +}); + +test('ScotMesh queues only the scoped repeater and applies advert TX mode with its relay identity', async () => { + const encryptionKey = 'test-packet-share-encryption-key'; + const previousKey = process.env['OWNER_PACKET_SHARE_ENCRYPTION_KEY']; + process.env['OWNER_PACKET_SHARE_ENCRYPTION_KEY'] = encryptionKey; + try { + const identity = await generatePacketShareIdentity(encryptionKey); + const target = { + destination_id: 'scotmesh', display_name: 'ScotMesh', enabled: true, + broker_url: 'wss://scotmesh-mqtt.mm7roq.compute.oarc.uk:443/', + topic_template: 'meshcore/MME/{observerId}/packets', + broker_username: identity.username, broker_password_ciphertext: identity.privateKeyCiphertext, + allowed_owner_username: 'node1', allowed_node_id: packet.rxNodeId, + owner_username: 'node1', node_id: packet.rxNodeId, + auth_mode: 'meshcore_jwt' as const, auth_audience: 'scotmesh-mqtt.mm7roq.compute.oarc.uk', tx_mode: 'advert' as const, + }; + const queued: Array<{ destinationTopic: string; payload: Record }> = []; + const repository = { + activeTargets: async () => [target, { ...target, owner_username: 'node2' }], + enqueueDelivery: async (input: typeof queued[number]) => { queued.push(input); return true; }, + reclaimExpiredClaims: async () => 0, + claimDue: async () => [], + } as unknown as PacketShareRepository; + const forwarder = createPacketShareForwarder({ repository }); + await forwarder.enqueuePacket({ ...packet, direction: 'rx' }, { raw: '0102' }); + await forwarder.enqueuePacket({ ...packet, direction: 'tx' }, { raw: '0102' }); + await forwarder.enqueuePacket({ ...packet, direction: 'tx', packetType: 4, srcNodeId: packet.rxNodeId }, { raw: '0102' }); + await forwarder.enqueuePacket({ ...packet, direction: 'tx', packetType: 4, srcNodeId: 'C'.repeat(64) }, { raw: '0102' }); + await forwarder.enqueuePacket({ ...packet, rxNodeId: 'C'.repeat(64) }, { raw: '0102' }); + await forwarder.stop(); + assert.equal(queued.length, 2); + for (const row of queued) { + assert.equal(row.destinationTopic, `meshcore/MME/${identity.username.slice(3)}/packets`); + assert.equal(row.payload['origin_id'], identity.username.slice(3)); + } + assert.equal(queued[0]?.payload['direction'], 'rx'); + assert.equal(queued[1]?.payload['direction'], 'tx'); + } finally { + if (previousKey === undefined) delete process.env['OWNER_PACKET_SHARE_ENCRYPTION_KEY']; + else process.env['OWNER_PACKET_SHARE_ENCRYPTION_KEY'] = previousKey; + } +}); + +const wedgeDestination = { + destination_id: 'remote-site', + display_name: 'Remote site', + broker_url: 'mqtts://remote.example.test:8883', + topic_template: 'replicated/{network}/{observerId}/packets', + broker_username: null, + broker_password_ciphertext: null, + enabled: true, +}; + +const wedgeRow = { + id: '1', + owner_username: 'node1', + node_id: packet.rxNodeId!, + destination_id: 'remote-site', + event_key: 'k', + packet_hash: packet.packetHash, + source_topic: packet.topic, + destination_topic: `replicated/ukmesh/${'A'.repeat(64)}/packets`, + payload: { raw: '0102' } as Record, + attempts: 1, +}; + +test('forwarder survives a hung publish: times out, fails for retry, then delivers on a fresh publisher', async () => { + let claims = 0; + const failed: Array<{ id: string; message: string }> = []; + const succeeded: string[] = []; + const repository = { + reclaimExpiredClaims: async () => 0, + claimDue: async () => { + claims += 1; + return claims <= 2 ? [wedgeRow] : []; + }, + getDestination: async () => wedgeDestination, + markSucceeded: async (id: string) => { succeeded.push(id); }, + markFailed: async (id: string, _claimToken: string, _attempts: number, message: string) => { + failed.push({ id, message }); + }, + } as unknown as PacketShareRepository; + const published: string[] = []; + let factories = 0; + const forwarder = createPacketShareForwarder({ + repository, + publisherFactory: () => { + factories += 1; + if (factories === 1) { + return { + // Settles only after the publish timeout has fired: the late + // rejection must be swallowed, not crash the process. + publish: () => new Promise((_resolve, reject) => { + setTimeout(() => reject(new Error('late publish failure')), 50); + }), + close: async () => undefined, + }; + } + return { + publish: async (topic: string) => { published.push(topic); }, + close: async () => undefined, + }; + }, + publishAckTimeoutMs: 20, + deliverTimeoutMs: 2_000, + }); + + await withGuard(forwarder.processDue(), 'first processDue'); + assert.equal(failed.length, 1, 'hung publish must fail the delivery after the timeout'); + assert.match(failed[0]!.message, /publish timed out/); + await new Promise((resolve) => setTimeout(resolve, 80)); + + await withGuard(forwarder.processDue(), 'second processDue'); + await forwarder.stop(); + + assert.equal(factories, 2, 'retry must use a fresh publisher'); + assert.equal(succeeded.length, 1, 'retry should deliver'); + assert.equal(published.length, 1); + assert.equal(published[0], `replicated/ukmesh/${'A'.repeat(64)}/packets`); +}); + +test('forwarder aborts a delivery past the per-row deadline even when the publisher never settles', async () => { + let claims = 0; + const failed: Array<{ id: string; message: string }> = []; + const repository = { + reclaimExpiredClaims: async () => 0, + claimDue: async () => { + claims += 1; + return claims === 1 ? [wedgeRow] : []; + }, + getDestination: async () => wedgeDestination, + markSucceeded: async () => undefined, + markFailed: async (id: string, _claimToken: string, _attempts: number, message: string) => { + failed.push({ id, message }); + }, + } as unknown as PacketShareRepository; + const forwarder = createPacketShareForwarder({ + repository, + publisherFactory: () => ({ + publish: () => new Promise(() => undefined), + close: async () => undefined, + }), + publishAckTimeoutMs: 10_000, + deliverTimeoutMs: 30, + }); + + await withGuard(forwarder.processDue(), 'processDue'); + await forwarder.stop(); + + assert.equal(failed.length, 1, 'the per-row deadline must release the claim'); + assert.match(failed[0]!.message, /delivery timed out/); +}); diff --git a/backend/src/owner/packetShareForwarder.ts b/backend/src/owner/packetShareForwarder.ts index 6251233..29fdc37 100644 --- a/backend/src/owner/packetShareForwarder.ts +++ b/backend/src/owner/packetShareForwarder.ts @@ -4,19 +4,24 @@ import type { LivePacket } from '../types/index.js'; import { buildPacketSharePayload, destinationConfigured, + destinationAcceptsPacket, + destinationAllowedForOwner, isForwardablePacket, isValidPacketShareBrokerUrl, packetShareEventKey, - renderPacketShareTopic, + renderDestinationTopic, type PacketShareDestinationRow, } from './packetShareConfig.js'; -import { decryptPacketShareSecret, hasPacketShareEncryptionKey } from './packetShareCrypto.js'; +import { hasPacketShareEncryptionKey } from './packetShareCrypto.js'; +import { packetShareCredentials } from './packetShareAuth.js'; import type { PacketShareDeliveryRow, PacketShareRepository } from './packetShareRepository.js'; import { packetShareOutcomesTotal, packetShareQueueDepth } from '../metrics.js'; const DEFAULT_POLL_INTERVAL_MS = 5_000; const DEFAULT_BATCH_SIZE = 25; const PUBLISH_TIMEOUT_MS = 10_000; +const DEFAULT_PUBLISH_ACK_TIMEOUT_MS = 10_000; +const DEFAULT_DELIVER_TIMEOUT_MS = 30_000; export type PacketSharePublisher = { publish: (topic: string, payload: string) => Promise; @@ -32,6 +37,21 @@ function positiveIntegerSetting(name: string, fallback: number, max: number): nu return Number.isFinite(parsed) && parsed > 0 ? Math.min(max, Math.floor(parsed)) : fallback; } +// A hung publisher promise must never be able to block the single-flight +// delivery loop. The 2026-10-12 wedge: a QoS-1 publish on a silently-dead +// connection never settled, freezing delivery for every destination until +// the process was restarted. +function withTimeout(promise: Promise, timeoutMs: number, message: string): Promise { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error(`${message} (after ${timeoutMs}ms)`)), timeoutMs); + timer.unref(); + promise.then( + (value) => { clearTimeout(timer); resolve(value); }, + (error) => { clearTimeout(timer); reject(error); }, + ); + }); +} + function destinationUrl(destination: PacketShareDestinationRow): URL { if (!destination.broker_url) throw new Error('destination broker URL is missing'); if (!isValidPacketShareBrokerUrl(destination.broker_url)) { @@ -47,12 +67,18 @@ function destinationFingerprint(destination: PacketShareDestinationRow): string destination.topic_template, destination.broker_username, destination.broker_password_ciphertext, + destination.auth_mode, + destination.auth_audience, ]); } -function connectMqttPublisher(destination: PacketShareDestinationRow): PacketSharePublisher { +function connectMqttPublisher( + destination: PacketShareDestinationRow, + publishAckTimeoutMs: number, +): PacketSharePublisher { let client: MqttClient | null = null; let connecting: Promise | null = null; + let credentialsExpireAt = Infinity; const closeClient = async (force = true): Promise => { const current = client; @@ -62,51 +88,56 @@ function connectMqttPublisher(destination: PacketShareDestinationRow): PacketSha }; const connect = async (): Promise => { - if (client?.connected) return client; + if (client?.connected && Date.now() / 1_000 < credentialsExpireAt - 60) return client; if (connecting) return connecting; - const url = destinationUrl(destination); - const encryptedPassword = destination.broker_password_ciphertext?.trim() || ''; - const password = encryptedPassword - ? decryptPacketShareSecret(encryptedPassword) - : undefined; - const created = mqtt.connect(url.toString(), { - clientId: `meshcore-share-${randomUUID().replaceAll('-', '').slice(0, 24)}`, - username: destination.broker_username?.trim() || undefined, - password, - clean: true, - reconnectPeriod: 0, - connectTimeout: PUBLISH_TIMEOUT_MS, - }); - client = created; - created.on('error', (error) => { - console.error(`[packet-share] MQTT destination ${destination.destination_id} error:`, error.message); - }); - connecting = new Promise((resolve, reject) => { - let settled = false; - let timer: NodeJS.Timeout; - const cleanup = () => { - clearTimeout(timer); - created.removeListener('connect', onConnect); - created.removeListener('close', onClose); - }; - const finish = (error?: Error) => { - if (settled) return; - settled = true; - cleanup(); - if (error) { - if (client === created) client = null; - created.end(true); - reject(error); - } else { - resolve(created); - } - }; - const onConnect = () => finish(); - const onClose = () => finish(new Error('destination MQTT connection closed before connect')); - timer = setTimeout(() => finish(new Error('destination MQTT connection timed out')), PUBLISH_TIMEOUT_MS); - created.once('connect', onConnect); - created.once('close', onClose); - }).finally(() => { + connecting = (async () => { + if (client) { + const expired = client; + client = null; + await expired.endAsync(true).catch(() => undefined); + } + const url = destinationUrl(destination); + const credentials = await packetShareCredentials(destination); + credentialsExpireAt = credentials.expiresAt; + const created = mqtt.connect(url.toString(), { + clientId: `meshcore-share-${randomUUID().replaceAll('-', '').slice(0, 24)}`, + username: credentials.username, + password: credentials.password, + clean: true, + reconnectPeriod: 0, + connectTimeout: PUBLISH_TIMEOUT_MS, + }); + client = created; + created.on('error', (error) => { + console.error(`[packet-share] MQTT destination ${destination.destination_id} error:`, error.message); + }); + return new Promise((resolve, reject) => { + let settled = false; + let timer: NodeJS.Timeout; + const cleanup = () => { + clearTimeout(timer); + created.removeListener('connect', onConnect); + created.removeListener('close', onClose); + }; + const finish = (error?: Error) => { + if (settled) return; + settled = true; + cleanup(); + if (error) { + if (client === created) client = null; + created.end(true); + reject(error); + } else { + resolve(created); + } + }; + const onConnect = () => finish(); + const onClose = () => finish(new Error('destination MQTT connection closed before connect')); + timer = setTimeout(() => finish(new Error('destination MQTT connection timed out')), PUBLISH_TIMEOUT_MS); + created.once('connect', onConnect); + created.once('close', onClose); + }); + })().finally(() => { connecting = null; }); return connecting; @@ -115,7 +146,18 @@ function connectMqttPublisher(destination: PacketShareDestinationRow): PacketSha return { async publish(topic: string, payload: string): Promise { const current = await connect(); - await current.publishAsync(topic, payload, { qos: 1 }); + try { + await withTimeout( + current.publishAsync(topic, payload, { qos: 1 }), + publishAckTimeoutMs, + 'destination MQTT publish timed out', + ); + } catch (error) { + // A publish left in an unknown state must not linger: discard the + // client so the next attempt reconnects with fresh credentials. + await closeClient(true); + throw error; + } if (!current.connected) { await closeClient(); throw new Error('destination MQTT connection closed after publish'); @@ -130,10 +172,19 @@ export type PacketShareForwarderDeps = { publisherFactory?: PacketSharePublisherFactory; pollIntervalMs?: number; batchSize?: number; + publishAckTimeoutMs?: number; + deliverTimeoutMs?: number; }; export function createPacketShareForwarder(deps: PacketShareForwarderDeps) { - const publisherFactory = deps.publisherFactory ?? connectMqttPublisher; + const publishAckTimeoutMs = deps.publishAckTimeoutMs ?? positiveIntegerSetting( + 'OWNER_PACKET_SHARE_PUBLISH_TIMEOUT_MS', DEFAULT_PUBLISH_ACK_TIMEOUT_MS, 120_000, + ); + const deliverTimeoutMs = deps.deliverTimeoutMs ?? positiveIntegerSetting( + 'OWNER_PACKET_SHARE_DELIVER_TIMEOUT_MS', DEFAULT_DELIVER_TIMEOUT_MS, 600_000, + ); + const publisherFactory: PacketSharePublisherFactory = deps.publisherFactory + ?? ((destination) => connectMqttPublisher(destination, publishAckTimeoutMs)); const pollIntervalMs = deps.pollIntervalMs ?? positiveIntegerSetting( 'OWNER_PACKET_SHARE_POLL_INTERVAL_MS', DEFAULT_POLL_INTERVAL_MS, 60_000, ); @@ -167,13 +218,15 @@ export function createPacketShareForwarder(deps: PacketShareForwarderDeps) { if (targets.length < 1) return; for (const target of targets) { + if (!destinationAllowedForOwner(target, target.owner_username, packet.rxNodeId!) + || !destinationAcceptsPacket(target, packet, upstreamEnvelope)) continue; if (!destinationConfigured(target, hasPacketShareEncryptionKey())) { packetShareOutcomesTotal.inc({ outcome: 'unconfigured' }); continue; } let destinationTopic: string; try { - destinationTopic = renderPacketShareTopic(target.topic_template!, packet); + destinationTopic = renderDestinationTopic(target, packet); } catch (error) { packetShareOutcomesTotal.inc({ outcome: 'invalid_destination' }); console.error('[packet-share] invalid destination topic:', error instanceof Error ? error.message : error); @@ -181,7 +234,7 @@ export function createPacketShareForwarder(deps: PacketShareForwarderDeps) { } try { const eventKey = packetShareEventKey(target.owner_username, target.destination_id, packet); - const payload = buildPacketSharePayload(packet, upstreamEnvelope); + const payload = buildPacketSharePayload(packet, upstreamEnvelope, target); const inserted = await deps.repository.enqueueDelivery({ ownerUsername: target.owner_username, nodeId: packet.rxNodeId!, @@ -202,14 +255,19 @@ export function createPacketShareForwarder(deps: PacketShareForwarderDeps) { async function deliver(row: PacketShareDeliveryRow, claimToken: string): Promise { const destination = await deps.repository.getDestination(row.destination_id); - if (!destination || !destinationConfigured(destination, hasPacketShareEncryptionKey())) { + if (!destination || !destinationAllowedForOwner(destination, row.owner_username, row.node_id) + || !destinationConfigured(destination, hasPacketShareEncryptionKey())) { await deps.repository.markFailed(row.id, claimToken, row.attempts, 'destination is missing or incomplete'); packetShareOutcomesTotal.inc({ outcome: 'delivery_error' }); return; } try { const publisher = await publisherFor(destination); - await publisher.publish(row.destination_topic, JSON.stringify(row.payload)); + await withTimeout( + publisher.publish(row.destination_topic, JSON.stringify(row.payload)), + publishAckTimeoutMs, + 'destination MQTT publish timed out', + ); await deps.repository.markSucceeded(row.id, claimToken); packetShareOutcomesTotal.inc({ outcome: 'delivered' }); } catch (error) { @@ -236,7 +294,23 @@ export function createPacketShareForwarder(deps: PacketShareForwarderDeps) { const claimToken = randomUUID(); const rows = await deps.repository.claimDue(batchSize, claimToken); packetShareQueueDepth.set(rows.length); - await Promise.all(rows.map((row) => deliver(row, claimToken))); + await Promise.all(rows.map(async (row) => { + try { + await withTimeout( + deliver(row, claimToken), + deliverTimeoutMs, + `delivery timed out (row ${row.id}, ${row.destination_id})`, + ); + } catch (error) { + // Last-resort guard: a delivery that never settled must not wedge + // the loop. Release the claim so the row retries on a backoff. + const message = error instanceof Error ? error.message : String(error); + console.error('[packet-share] delivery aborted:', message); + await deps.repository.markFailed(row.id, claimToken, row.attempts, message) + .catch(() => undefined); + packetShareOutcomesTotal.inc({ outcome: 'delivery_error' }); + } + })); } catch (error) { packetShareOutcomesTotal.inc({ outcome: 'worker_error' }); console.error('[packet-share] worker error:', error instanceof Error ? error.message : error); diff --git a/backend/src/owner/packetShareRepository.integration.test.ts b/backend/src/owner/packetShareRepository.integration.test.ts new file mode 100644 index 0000000..fa2a8df --- /dev/null +++ b/backend/src/owner/packetShareRepository.integration.test.ts @@ -0,0 +1,63 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import { randomUUID } from 'node:crypto'; +import test from 'node:test'; +import pg from 'pg'; +import { createPacketShareRepository } from './packetShareRepository.js'; +import type { DatabaseQueryFn } from '../db/index.js'; +import type { LivePacket } from '../types/index.js'; + +const databaseUrl = process.env['PACKET_SHARE_TEST_DATABASE_URL']; + +test('PostgreSQL enforces ScotMesh owner and observer scope for reads, rules and queue insertion', { + skip: !databaseUrl, +}, async () => { + const schema = `packet_share_test_${randomUUID().replaceAll('-', '')}`; + const pool = new pg.Pool({ connectionString: databaseUrl, max: 1, options: `-c search_path=${schema}` }); + const nodeId = 'A'.repeat(64); + const packet: LivePacket = { + id: 'event', packetHash: 'B'.repeat(16), rxNodeId: nodeId, + topic: `meshcore/LHR/${nodeId}/packets`, ts: Date.now(), visibilityOk: true, + }; + try { + await pool.query(`CREATE SCHEMA ${schema}`); + for (const migration of [ + '052_owner_packet_sharing.sql', '053_owner_packet_share_destination_metadata.sql', + '054_owner_packet_share_default_off.sql', '061_packet_share_owner_scope_and_jwt.sql', + ]) { + await pool.query(await fs.readFile(new URL(`../db/migrations/${migration}`, import.meta.url), 'utf8')); + } + await pool.query( + `INSERT INTO owner_packet_share_destinations (destination_id, display_name, enabled, allowed_owner_username, allowed_node_id) + VALUES ('scotmesh', 'ScotMesh', TRUE, 'node1', $1), ('existing', 'Existing', TRUE, NULL, NULL)`, [nodeId], + ); + const query: DatabaseQueryFn = async (sql, params) => pool.query(sql, params); + const repository = createPacketShareRepository(query); + assert.deepEqual((await repository.listDestinations('node1', nodeId)).map((row) => row.destination_id), ['existing', 'scotmesh']); + assert.deepEqual((await repository.listDestinations('node2', nodeId)).map((row) => row.destination_id), ['existing']); + assert.deepEqual((await repository.listDestinations('node1', 'C'.repeat(64))).map((row) => row.destination_id), ['existing']); + assert.deepEqual(await repository.replaceOwnerRules('node2', nodeId, ['scotmesh'], true), []); + assert.deepEqual(await repository.replaceOwnerRules('node1', 'C'.repeat(64), ['scotmesh'], true), []); + assert.deepEqual(await repository.replaceOwnerRules('node1', nodeId, ['scotmesh'], true), ['scotmesh']); + // Simulate old rules created before scope enforcement; they must not become targets. + await pool.query( + `INSERT INTO owner_packet_share_rules (owner_username, node_id, destination_id, enabled) + VALUES ('node2', $1, 'scotmesh', TRUE), ('node1', $2, 'scotmesh', TRUE)`, [nodeId, 'C'.repeat(64)], + ); + assert.deepEqual(await repository.listOwnerRules('node2', nodeId), []); + assert.deepEqual((await repository.activeTargets(nodeId)).map((row) => row.owner_username), ['node1']); + assert.deepEqual(await repository.activeTargets('C'.repeat(64)), []); + for (const [ownerUsername, observer, expected] of [ + ['node1', nodeId, true], ['node2', nodeId, false], ['node1', 'C'.repeat(64), false], + ] as const) { + assert.equal(await repository.enqueueDelivery({ + ownerUsername, nodeId: observer, destinationId: 'scotmesh', + eventKey: `${ownerUsername}:${observer}`, packet: { ...packet, rxNodeId: observer }, + destinationTopic: `meshcore/MME/${observer}/packets`, payload: { raw: '0102' }, + }), expected); + } + } finally { + await pool.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`); + await pool.end(); + } +}); diff --git a/backend/src/owner/packetShareRepository.ts b/backend/src/owner/packetShareRepository.ts index d9c7c6c..4d35a42 100644 --- a/backend/src/owner/packetShareRepository.ts +++ b/backend/src/owner/packetShareRepository.ts @@ -1,6 +1,6 @@ import type { DatabaseQueryFn } from '../db/index.js'; import type { LivePacket } from '../types/index.js'; -import type { PacketShareDestinationRow, PacketShareTarget } from './packetShareConfig.js'; +import { destinationAllowedForOwner, type PacketShareDestinationRow, type PacketShareTarget } from './packetShareConfig.js'; export type PacketShareQueryFn = DatabaseQueryFn; export type PacketShareTransactionFn = (work: (query: PacketShareQueryFn) => Promise) => Promise; @@ -22,22 +22,27 @@ export function createPacketShareRepository( query: PacketShareQueryFn, transaction?: PacketShareTransactionFn, ) { - async function listDestinations(): Promise { + async function listDestinations(ownerUsername: string, nodeId: string): Promise { const result = await query( `SELECT destination_id, display_name, website_url, description, broker_url, topic_template, - broker_username, broker_password_ciphertext, enabled + broker_username, broker_password_ciphertext, enabled, + allowed_owner_username, allowed_node_id, auth_mode, auth_audience, tx_mode FROM owner_packet_share_destinations + WHERE (allowed_owner_username IS NULL OR allowed_owner_username = $1) + AND (allowed_node_id IS NULL OR allowed_node_id = $2) ORDER BY display_name, destination_id`, + [ownerUsername, nodeId], ); - return result.rows; + return result.rows.filter((row) => destinationAllowedForOwner(row, ownerUsername, nodeId)); } async function getDestination(destinationId: string): Promise { const result = await query( `SELECT destination_id, display_name, website_url, description, broker_url, topic_template, - broker_username, broker_password_ciphertext, enabled + broker_username, broker_password_ciphertext, enabled, + allowed_owner_username, allowed_node_id, auth_mode, auth_audience, tx_mode FROM owner_packet_share_destinations WHERE destination_id = $1`, [destinationId], @@ -47,10 +52,13 @@ export function createPacketShareRepository( async function listOwnerRules(ownerUsername: string, nodeId: string): Promise { const result = await query<{ destination_id: string }>( - `SELECT destination_id - FROM owner_packet_share_rules - WHERE owner_username = $1 AND node_id = $2 AND enabled = TRUE - ORDER BY destination_id`, + `SELECT rules.destination_id + FROM owner_packet_share_rules rules + JOIN owner_packet_share_destinations destination USING (destination_id) + WHERE rules.owner_username = $1 AND rules.node_id = $2 AND rules.enabled = TRUE + AND (destination.allowed_owner_username IS NULL OR destination.allowed_owner_username = $1) + AND (destination.allowed_node_id IS NULL OR destination.allowed_node_id = $2) + ORDER BY rules.destination_id`, [ownerUsername, nodeId], ); return result.rows.map((row) => row.destination_id); @@ -77,17 +85,20 @@ export function createPacketShareRepository( destination.website_url, destination.description, destination.broker_url, destination.topic_template, destination.broker_username, destination.broker_password_ciphertext, - destination.enabled + destination.enabled, destination.allowed_owner_username, destination.allowed_node_id, + destination.auth_mode, destination.auth_audience, destination.tx_mode FROM owner_packet_share_rules rules JOIN owner_packet_share_destinations destination ON destination.destination_id = rules.destination_id WHERE rules.node_id = $1 AND rules.enabled = TRUE AND destination.enabled = TRUE + AND (destination.allowed_owner_username IS NULL OR destination.allowed_owner_username = rules.owner_username) + AND (destination.allowed_node_id IS NULL OR destination.allowed_node_id = rules.node_id) ORDER BY destination.destination_id`, [nodeId], ); - return result.rows; + return result.rows.filter((row) => destinationAllowedForOwner(row, row.owner_username, row.node_id)); } async function replaceOwnerRules( @@ -122,6 +133,8 @@ export function createPacketShareRepository( FROM owner_packet_share_destinations WHERE destination_id = ANY($3::text[]) AND enabled = TRUE + AND (allowed_owner_username IS NULL OR allowed_owner_username = $1) + AND (allowed_node_id IS NULL OR allowed_node_id = $2) RETURNING destination_id`, [ownerUsername, nodeId, destinationIds], ); @@ -158,6 +171,8 @@ export function createPacketShareRepository( AND rules.destination_id = $3 AND rules.enabled = TRUE AND destination.enabled = TRUE + AND (destination.allowed_owner_username IS NULL OR destination.allowed_owner_username = rules.owner_username) + AND (destination.allowed_node_id IS NULL OR destination.allowed_node_id = rules.node_id) ) ON CONFLICT (event_key) DO NOTHING RETURNING id`, diff --git a/backend/src/owner/publicFeedConfig.test.ts b/backend/src/owner/publicFeedConfig.test.ts new file mode 100644 index 0000000..a046eee --- /dev/null +++ b/backend/src/owner/publicFeedConfig.test.ts @@ -0,0 +1,71 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import type { LivePacket } from '../types/index.js'; +import { + buildPublicFeedPacket, + getPublicFeedConfig, + isValidPublicFeedBrokerUrl, + publicFeedPacketIsEligible, +} from './publicFeedConfig.js'; + +const packet: LivePacket = { + id: 'public-feed-event', + packetHash: 'B'.repeat(16), + rxNodeId: 'A'.repeat(64), + topic: `meshcore/LHR/${'A'.repeat(64)}/packets`, + iata: 'LHR', + network: 'ukmesh', + packetType: 5, + visibilityOk: true, + ts: Date.parse('2026-09-20T12:00:00Z'), +}; + +test('public feed configuration requires the explicit operator switch and broker endpoint', () => { + assert.equal(getPublicFeedConfig({}).configured, false); + assert.equal(getPublicFeedConfig({ + PUBLIC_FEED_PUBLISH_ENABLED: 'true', + PUBLIC_FEED_BROKER_URL: 'mqtts://broker.example.test:8883', + }).configured, true); + assert.equal(getPublicFeedConfig({ + PUBLIC_FEED_PUBLISH_ENABLED: 'true', + PUBLIC_FEED_BROKER_URL: 'mqtts://user:pass@broker.example.test:8883', + }).configured, false); +}); + +test('public feed serializer emits CoreScope topic and numeric JSON fields without a wrapper', () => { + const outbound = buildPublicFeedPacket(packet, { + raw: '04719600004AA03688E8F83CEEB7B40DE40D9E2DB29FED705273C5C1FF3DBC375E5916AB5F24649A54AA7BCF3664', + origin: '49Segre-Room-Obs', + timestamp: '2026-09-20T12:00:00.000Z', + SNR: '12.2', + RSSI: '-30', + score: '1000', + direction: 'rx', + body: 'decoded content must not cross the public-feed boundary', + }); + assert.equal(outbound.topic, `public/LHR/${'A'.repeat(64)}/packets`); + assert.deepEqual(outbound.payload, { + raw: '04719600004AA03688E8F83CEEB7B40DE40D9E2DB29FED705273C5C1FF3DBC375E5916AB5F24649A54AA7BCF3664', + origin: '49Segre-Room-Obs', + timestamp: '2026-09-20T12:00:00.000Z', + SNR: 12.2, + RSSI: -30, + score: 1000, + direction: 'rx', + }); +}); + +test('public feed serializer rejects bare or malformed raw packets and invalid regions', () => { + assert.throws(() => buildPublicFeedPacket(packet, { raw: 'not-hex' })); + assert.throws(() => buildPublicFeedPacket(packet, { raw: 'ABC' })); + assert.throws(() => buildPublicFeedPacket({ ...packet, iata: 'unknown region' }, { raw: '0102' })); + assert.equal(publicFeedPacketIsEligible({ ...packet, visibilityOk: false }), false); + assert.equal(publicFeedPacketIsEligible({ ...packet, iata: undefined }), false); +}); + +test('public feed broker validation accepts MQTT transports without embedded credentials', () => { + assert.equal(isValidPublicFeedBrokerUrl('mqtts://broker.example.test:8883'), true); + assert.equal(isValidPublicFeedBrokerUrl('wss://broker.example.test/mqtt'), true); + assert.equal(isValidPublicFeedBrokerUrl('mqtts://user:pass@broker.example.test:8883'), false); + assert.equal(isValidPublicFeedBrokerUrl('https://broker.example.test'), false); +}); diff --git a/backend/src/owner/publicFeedConfig.ts b/backend/src/owner/publicFeedConfig.ts new file mode 100644 index 0000000..c76f3b6 --- /dev/null +++ b/backend/src/owner/publicFeedConfig.ts @@ -0,0 +1,200 @@ +import type { LivePacket } from '../types/index.js'; + +const PUBLIC_FEED_PROTOCOLS = new Set(['mqtt:', 'mqtts:', 'ws:', 'wss:']); +const MAX_RAW_HEX_LENGTH = 128 * 1024; +const MAX_PAYLOAD_BYTES = 64 * 1024; +const MAX_ORIGIN_LENGTH = 256; + +/** Curated namespace exposed to authenticated public-feed subscribers. */ +export const PUBLIC_FEED_TOPIC_FILTER = 'public/+/+/packets'; +const PUBLIC_FEED_TOPIC_RE = /^public\/[A-Z0-9]{2,8}\/[0-9A-F]{64}\/packets$/; + +export type PublicFeedConfig = { + operatorEnabled: boolean; + configured: boolean; + brokerUrl: string | null; + /** + * Kept for backwards-compatible config parsing. The public feed is served + * by the UKmesh broker itself, so it never uses a separate publisher + * credential. + */ + username: string | null; + password: string | null; +}; + +export type PublicFeedPacketPayload = { + raw: string; + origin?: string; + timestamp?: string; + SNR?: number; + RSSI?: number; + score?: number; + direction?: 'rx' | 'tx'; +}; + +export type PublicFeedPacket = { + topic: string; + payload: PublicFeedPacketPayload; +}; + +function envBoolean(value: string | undefined, fallback: boolean): boolean { + if (value == null || value.trim() === '') return fallback; + return ['1', 'true', 'yes', 'on'].includes(value.trim().toLowerCase()); +} + +function validBrokerUrl(rawUrl: string): boolean { + try { + const url = new URL(rawUrl); + return PUBLIC_FEED_PROTOCOLS.has(url.protocol) + && Boolean(url.hostname) + && !url.username + && !url.password; + } catch { + return false; + } +} + +export function getPublicFeedConfig(env: NodeJS.ProcessEnv = process.env): PublicFeedConfig { + const operatorEnabled = envBoolean(env['PUBLIC_FEED_PUBLISH_ENABLED'], false); + const brokerUrl = String(env['PUBLIC_FEED_BROKER_URL'] ?? '').trim() || null; + const username = String(env['PUBLIC_FEED_BROKER_USERNAME'] ?? '').trim() || null; + const password = env['PUBLIC_FEED_BROKER_PASSWORD'] ?? null; + const configured = operatorEnabled + && brokerUrl !== null + && validBrokerUrl(brokerUrl); + return { + operatorEnabled, + configured, + brokerUrl, + username, + password, + }; +} + +function finiteNumber(value: unknown): number | undefined { + if (typeof value === 'number') return Number.isFinite(value) ? value : undefined; + if (typeof value !== 'string' || value.trim() === '') return undefined; + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : undefined; +} + +function boundedNumber(value: unknown, minimum: number, maximum: number): number | undefined { + const parsed = finiteNumber(value); + return parsed !== undefined && parsed >= minimum && parsed <= maximum ? parsed : undefined; +} + +function timestampValue(value: unknown, fallbackMs: number): string { + const candidate = typeof value === 'string' || typeof value === 'number' + ? new Date(value) + : new Date(fallbackMs); + const date = Number.isFinite(candidate.getTime()) ? candidate : new Date(fallbackMs); + if (!Number.isFinite(date.getTime())) throw new Error('public feed packet timestamp is invalid'); + return date.toISOString(); +} + +function packetRegion(packet: LivePacket): string { + const region = String(packet.iata ?? '').trim().toUpperCase(); + if (!/^[A-Z0-9]{2,8}$/.test(region)) { + throw new Error('public feed packet region is invalid'); + } + return region; +} + +function packetObserverId(packet: LivePacket): string { + const observerId = String(packet.rxNodeId ?? '').trim().toUpperCase(); + if (!/^[0-9A-F]{64}$/.test(observerId)) { + throw new Error('public feed packet observer ID is invalid'); + } + return observerId; +} + +function packetRaw(upstreamEnvelope: Record): string { + const raw = typeof upstreamEnvelope['raw'] === 'string' + ? upstreamEnvelope['raw'].trim().toUpperCase() + : ''; + if ( + raw.length < 2 + || raw.length > MAX_RAW_HEX_LENGTH + || raw.length % 2 !== 0 + || !/^[0-9A-F]+$/.test(raw) + ) { + throw new Error('public feed packet raw bytes are missing or invalid'); + } + return raw; +} + +function copyOrigin(payload: PublicFeedPacketPayload, value: unknown): void { + if (typeof value !== 'string') return; + const origin = value.trim().slice(0, MAX_ORIGIN_LENGTH); + if (origin) payload.origin = origin; +} + +function copyRadioMetric( + payload: PublicFeedPacketPayload, + key: 'SNR' | 'RSSI' | 'score', + value: unknown, + minimum: number, + maximum: number, +): void { + const parsed = boundedNumber(value, minimum, maximum); + if (parsed !== undefined) payload[key] = parsed; +} + +/** + * Produce the canonical CoreScope-compatible public packet envelope. Only the + * original raw packet and bounded radio metadata are copied; decoded content + * and the private relay wrapper never cross this boundary. + */ +export function buildPublicFeedPacket( + packet: LivePacket, + upstreamEnvelope: Record, +): PublicFeedPacket { + const payload: PublicFeedPacketPayload = { + raw: packetRaw(upstreamEnvelope), + timestamp: timestampValue(upstreamEnvelope['timestamp'], packet.ts), + }; + copyOrigin(payload, upstreamEnvelope['origin']); + copyRadioMetric(payload, 'SNR', upstreamEnvelope['SNR'], -200, 200); + copyRadioMetric(payload, 'RSSI', upstreamEnvelope['RSSI'], -200, 100); + copyRadioMetric(payload, 'score', upstreamEnvelope['score'], -1_000_000, 1_000_000); + + const direction = typeof upstreamEnvelope['direction'] === 'string' + ? upstreamEnvelope['direction'].trim().toLowerCase() + : typeof packet.direction === 'string' + ? packet.direction.trim().toLowerCase() + : 'rx'; + payload.direction = direction === 'tx' ? 'tx' : 'rx'; + + if (Buffer.byteLength(JSON.stringify(payload), 'utf8') > MAX_PAYLOAD_BYTES) { + throw new Error('public feed packet payload exceeds configured limit'); + } + + return { + topic: `public/${packetRegion(packet)}/${packetObserverId(packet)}/packets`, + payload, + }; +} + +export function isPublicFeedPacketTopic(topic: string): boolean { + return PUBLIC_FEED_TOPIC_RE.test(topic.trim()); +} + +export function publicFeedEventKey(packet: LivePacket): string { + const observerId = packetObserverId(packet); + return `${observerId}:${packet.packetHash}`; +} + +export function publicFeedPacketIsEligible(packet: LivePacket): boolean { + return packet.visibilityOk === true + && packet.network !== 'test' + && typeof packet.rxNodeId === 'string' + && /^[0-9A-F]{64}$/.test(packet.rxNodeId) + && /^[0-9A-F]{16,128}$/.test(packet.packetHash) + && /^[A-Z0-9]{2,8}$/.test(String(packet.iata ?? '').trim().toUpperCase()); +} + +export function isValidPublicFeedBrokerUrl(rawUrl: string | null | undefined): boolean { + return typeof rawUrl === 'string' + && rawUrl.trim().length > 0 + && validBrokerUrl(rawUrl.trim()); +} diff --git a/backend/src/owner/publicFeedForwarder.test.ts b/backend/src/owner/publicFeedForwarder.test.ts new file mode 100644 index 0000000..068f527 --- /dev/null +++ b/backend/src/owner/publicFeedForwarder.test.ts @@ -0,0 +1,131 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import type { LivePacket } from '../types/index.js'; +import { createPublicFeedForwarder, type PublicFeedPublisher } from './publicFeedForwarder.js'; +import type { PublicFeedRepository } from './publicFeedRepository.js'; + +const packet: LivePacket = { + id: 'public-feed-event', + packetHash: 'B'.repeat(16), + rxNodeId: 'A'.repeat(64), + topic: `meshcore/LHR/${'A'.repeat(64)}/packets`, + iata: 'LHR', + network: 'ukmesh', + packetType: 5, + visibilityOk: true, + ts: Date.parse('2026-09-20T12:00:00Z'), +}; + +function configuredEnv(): Record { + return { + PUBLIC_FEED_PUBLISH_ENABLED: 'true', + PUBLIC_FEED_BROKER_URL: 'mqtts://broker.example.test:8883', + PUBLIC_FEED_BROKER_USERNAME: 'publisher', + PUBLIC_FEED_BROKER_PASSWORD: 'secret', + }; +} + +test('public feed queues one aggregate CoreScope packet and publishes it through the MQTT seam', async () => { + const previous = { ...process.env }; + Object.assign(process.env, configuredEnv()); + const enqueued: Array> = []; + const published: Array<{ topic: string; payload: string }> = []; + let claimed = false; + const repository = { + hasActiveNode: async () => true, + enqueueDelivery: async (input: Record) => { + enqueued.push(input); + return true; + }, + syncOwnership: async () => undefined, + accountStatus: async () => ({ enabled: true, nodeCount: 1, lastForwardedAt: null }), + setEnabled: async () => undefined, + discardInactiveDeliveries: async () => 0, + reclaimExpiredClaims: async () => 0, + claimDue: async () => { + if (claimed || enqueued.length < 1) return []; + claimed = true; + const row = enqueued[0]!; + return [{ + id: '1', + node_id: packet.rxNodeId!, + event_key: String(row['eventKey']), + packet_hash: packet.packetHash, + source_topic: packet.topic, + destination_topic: String(row['destinationTopic']), + payload: row['payload'] as Record, + attempts: 1, + }]; + }, + markSucceeded: async () => undefined, + markFailed: async () => undefined, + markDiscarded: async () => undefined, + } as unknown as PublicFeedRepository; + const publisher: PublicFeedPublisher = { + publish: async (topic, payload) => { published.push({ topic, payload }); }, + close: async () => undefined, + }; + const forwarder = createPublicFeedForwarder({ + repository, + getOwnerAuthorizationSnapshot: async () => [], + publisherFactory: () => publisher, + }); + + try { + await forwarder.enqueuePacket(packet, { + raw: '0102', + origin: 'Test observer', + SNR: '12.2', + RSSI: '-30', + score: '1000', + direction: 'rx', + body: 'private decoded text', + }); + await forwarder.processDue(); + await forwarder.stop(); + } finally { + for (const key of Object.keys(process.env)) { + if (!(key in previous)) delete process.env[key]; + } + Object.assign(process.env, previous); + } + + assert.equal(enqueued.length, 1); + assert.equal(published.length, 1); + assert.equal(published[0]?.topic, `public/LHR/${'A'.repeat(64)}/packets`); + const outbound = JSON.parse(published[0]!.payload) as Record; + assert.equal(outbound['raw'], '0102'); + assert.equal(outbound['SNR'], 12.2); + assert.equal(outbound['RSSI'], -30); + assert.equal(outbound['score'], 1000); + assert.equal('body' in outbound, false); + assert.equal('relay' in outbound, false); + assert.equal('packet' in outbound, false); +}); + +test('public feed does not enqueue a packet while the broker is not operator-enabled', async () => { + const previous = { ...process.env }; + delete process.env.PUBLIC_FEED_PUBLISH_ENABLED; + delete process.env.PUBLIC_FEED_BROKER_URL; + delete process.env.PUBLIC_FEED_BROKER_USERNAME; + delete process.env.PUBLIC_FEED_BROKER_PASSWORD; + let enqueues = 0; + const repository = { + hasActiveNode: async () => { throw new Error('must not check ownership when unconfigured'); }, + enqueueDelivery: async () => { enqueues++; return true; }, + } as unknown as PublicFeedRepository; + const forwarder = createPublicFeedForwarder({ + repository, + getOwnerAuthorizationSnapshot: async () => [], + }); + try { + await forwarder.enqueuePacket(packet, { raw: '0102' }); + await forwarder.stop(); + } finally { + for (const key of Object.keys(process.env)) { + if (!(key in previous)) delete process.env[key]; + } + Object.assign(process.env, previous); + } + assert.equal(enqueues, 0); +}); diff --git a/backend/src/owner/publicFeedForwarder.ts b/backend/src/owner/publicFeedForwarder.ts new file mode 100644 index 0000000..c839eef --- /dev/null +++ b/backend/src/owner/publicFeedForwarder.ts @@ -0,0 +1,305 @@ +import mqtt, { type MqttClient } from 'mqtt'; +import { randomUUID } from 'node:crypto'; +import type { LivePacket } from '../types/index.js'; +import { + buildPublicFeedPacket, + getPublicFeedConfig, + isPublicFeedPacketTopic, + publicFeedEventKey, + publicFeedPacketIsEligible, + type PublicFeedConfig, +} from './publicFeedConfig.js'; +import type { PublicFeedDeliveryRow, PublicFeedRepository, PublicFeedAuthorizationSnapshot } from './publicFeedRepository.js'; +import { publicFeedOutcomesTotal, publicFeedQueueDepth } from '../metrics.js'; + +const DEFAULT_POLL_INTERVAL_MS = 5_000; +const DEFAULT_BATCH_SIZE = 25; +const DEFAULT_OWNERSHIP_REFRESH_INTERVAL_MS = 15_000; +const PUBLISH_TIMEOUT_MS = 10_000; + +export type PublicFeedPublisher = { + publish: (topic: string, payload: string) => Promise; + close: () => Promise; +}; + +export type PublicFeedPublisherFactory = (config: PublicFeedConfig) => PublicFeedPublisher; + +function positiveIntegerSetting(name: string, fallback: number, max: number): number { + const parsed = Number(process.env[name]); + return Number.isFinite(parsed) && parsed > 0 ? Math.min(max, Math.floor(parsed)) : fallback; +} + +function configUrl(config: PublicFeedConfig): URL { + if (!config.brokerUrl) throw new Error('public feed broker URL is missing'); + const url = new URL(config.brokerUrl); + if (!['mqtt:', 'mqtts:', 'ws:', 'wss:'].includes(url.protocol) || !url.hostname) { + throw new Error('public feed broker URL is invalid'); + } + return url; +} + +function configFingerprint(config: PublicFeedConfig): string { + return JSON.stringify([config.brokerUrl, config.username, config.password]); +} + +function connectPublicFeedPublisher(config: PublicFeedConfig): PublicFeedPublisher { + let client: MqttClient | null = null; + let connecting: Promise | null = null; + + const closeClient = async (force = true): Promise => { + const current = client; + client = null; + connecting = null; + if (current) await current.endAsync(force).catch(() => undefined); + }; + + const connect = async (): Promise => { + if (client?.connected) return client; + if (connecting) return connecting; + const url = configUrl(config); + if (!config.username || !config.password) throw new Error('public feed publisher credentials are missing'); + const created = mqtt.connect(url.toString(), { + clientId: `meshcore-public-feed-${randomUUID().replaceAll('-', '').slice(0, 24)}`, + username: config.username, + password: config.password, + clean: true, + reconnectPeriod: 0, + connectTimeout: PUBLISH_TIMEOUT_MS, + }); + client = created; + created.on('error', (error) => { + console.error('[public-feed] MQTT publisher error:', error.message); + }); + connecting = new Promise((resolve, reject) => { + let settled = false; + let timer: NodeJS.Timeout; + const cleanup = () => { + clearTimeout(timer); + created.removeListener('connect', onConnect); + created.removeListener('close', onClose); + }; + const finish = (error?: Error) => { + if (settled) return; + settled = true; + cleanup(); + if (error) { + if (client === created) client = null; + created.end(true); + reject(error); + } else { + resolve(created); + } + }; + const onConnect = () => finish(); + const onClose = () => finish(new Error('public feed MQTT connection closed before connect')); + timer = setTimeout(() => finish(new Error('public feed MQTT connection timed out')), PUBLISH_TIMEOUT_MS); + created.once('connect', onConnect); + created.once('close', onClose); + }).finally(() => { + connecting = null; + }); + return connecting; + }; + + return { + async publish(topic: string, payload: string): Promise { + const current = await connect(); + // CoreScope subscribes at QoS 0. The feed is an observation stream, so + // retaining messages or requesting broker redelivery is undesirable. + await current.publishAsync(topic, payload, { qos: 0, retain: false }); + if (!current.connected) { + await closeClient(); + throw new Error('public feed MQTT connection closed after publish'); + } + }, + close: closeClient, + }; +} + +export type PublicFeedForwarderDeps = { + repository: PublicFeedRepository; + getOwnerAuthorizationSnapshot: () => Promise; + publisherFactory?: PublicFeedPublisherFactory; + pollIntervalMs?: number; + batchSize?: number; + ownershipRefreshIntervalMs?: number; +}; + +export function createPublicFeedForwarder(deps: PublicFeedForwarderDeps) { + const publisherFactory = deps.publisherFactory ?? connectPublicFeedPublisher; + const pollIntervalMs = deps.pollIntervalMs ?? positiveIntegerSetting( + 'PUBLIC_FEED_POLL_INTERVAL_MS', DEFAULT_POLL_INTERVAL_MS, 60_000, + ); + const batchSize = deps.batchSize ?? positiveIntegerSetting( + 'PUBLIC_FEED_BATCH_SIZE', DEFAULT_BATCH_SIZE, 100, + ); + const ownershipRefreshIntervalMs = deps.ownershipRefreshIntervalMs ?? positiveIntegerSetting( + 'PUBLIC_FEED_OWNERSHIP_REFRESH_INTERVAL_MS', DEFAULT_OWNERSHIP_REFRESH_INTERVAL_MS, 300_000, + ); + const publishers = new Map(); + let pollTimer: NodeJS.Timeout | null = null; + let ownershipTimer: NodeJS.Timeout | null = null; + let processing: Promise | null = null; + let ownershipRefresh: Promise | null = null; + let stopping = false; + + async function publisherFor(config: PublicFeedConfig): Promise { + const fingerprint = configFingerprint(config); + const existing = publishers.get('public-feed'); + if (existing?.fingerprint === fingerprint) return existing.publisher; + if (existing) { + publishers.delete('public-feed'); + await existing.publisher.close().catch(() => undefined); + } + const created = publisherFactory(config); + publishers.set('public-feed', { publisher: created, fingerprint }); + return created; + } + + async function refreshOwnership(): Promise { + if (stopping) return; + if (ownershipRefresh) { + await ownershipRefresh; + return; + } + ownershipRefresh = (async () => { + try { + const snapshot = await deps.getOwnerAuthorizationSnapshot(); + await deps.repository.syncOwnership(snapshot); + } catch (error) { + publicFeedOutcomesTotal.inc({ outcome: 'ownership_sync_error' }); + console.error('[public-feed] ownership refresh failed:', error instanceof Error ? error.message : error); + } finally { + ownershipRefresh = null; + } + })(); + await ownershipRefresh; + } + + async function enqueuePacket(packet: LivePacket, upstreamEnvelope: Record): Promise { + if (!publicFeedPacketIsEligible(packet)) { + publicFeedOutcomesTotal.inc({ outcome: 'skipped' }); + return; + } + const config = getPublicFeedConfig(); + if (!config.configured) { + publicFeedOutcomesTotal.inc({ outcome: 'unconfigured' }); + return; + } + if (!await deps.repository.hasActiveNode(packet.rxNodeId!)) return; + + let outbound: ReturnType; + try { + outbound = buildPublicFeedPacket(packet, upstreamEnvelope); + } catch (error) { + publicFeedOutcomesTotal.inc({ outcome: 'invalid_packet' }); + console.error('[public-feed] packet rejected:', error instanceof Error ? error.message : error); + return; + } + + try { + const inserted = await deps.repository.enqueueDelivery({ + nodeId: packet.rxNodeId!, + eventKey: publicFeedEventKey(packet), + packet, + destinationTopic: outbound.topic, + payload: outbound.payload, + }); + publicFeedOutcomesTotal.inc({ outcome: inserted ? 'queued' : 'duplicate' }); + } catch (error) { + publicFeedOutcomesTotal.inc({ outcome: 'queue_error' }); + console.error('[public-feed] queue error:', error instanceof Error ? error.message : error); + return; + } + void processDue(); + } + + async function deliver(row: PublicFeedDeliveryRow, claimToken: string): Promise { + // Drop deliveries created by an older deployment before they can publish + // to the canonical ingest namespace. The curated public namespace is the + // only destination this worker is allowed to emit. + if (!isPublicFeedPacketTopic(row.destination_topic)) { + await deps.repository.markDiscarded(row.id, claimToken, 'public feed destination namespace is invalid'); + publicFeedOutcomesTotal.inc({ outcome: 'invalid_destination' }); + return; + } + if (!await deps.repository.hasActiveNode(row.node_id)) { + await deps.repository.markDiscarded(row.id, claimToken, 'public feed is no longer enabled for this node'); + publicFeedOutcomesTotal.inc({ outcome: 'disabled' }); + return; + } + const config = getPublicFeedConfig(); + if (!config.configured) { + await deps.repository.markFailed(row.id, claimToken, row.attempts, 'public feed broker is not configured'); + publicFeedOutcomesTotal.inc({ outcome: 'unconfigured' }); + return; + } + try { + const publisher = await publisherFor(config); + await publisher.publish(row.destination_topic, JSON.stringify(row.payload)); + await deps.repository.markSucceeded(row.id, claimToken); + publicFeedOutcomesTotal.inc({ outcome: 'delivered' }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + const entry = publishers.get('public-feed'); + if (entry) { + publishers.delete('public-feed'); + await entry.publisher.close().catch(() => undefined); + } + await deps.repository.markFailed(row.id, claimToken, row.attempts, message); + publicFeedOutcomesTotal.inc({ outcome: 'delivery_error' }); + } + } + + async function processDue(): Promise { + if (stopping) return; + if (processing) { + await processing; + return; + } + processing = (async () => { + try { + await deps.repository.discardInactiveDeliveries(); + await deps.repository.reclaimExpiredClaims(); + if (!getPublicFeedConfig().configured) return; + const claimToken = randomUUID(); + const rows = await deps.repository.claimDue(batchSize, claimToken); + publicFeedQueueDepth.set(rows.length); + await Promise.all(rows.map((row) => deliver(row, claimToken))); + } catch (error) { + publicFeedOutcomesTotal.inc({ outcome: 'worker_error' }); + console.error('[public-feed] worker error:', error instanceof Error ? error.message : error); + } finally { + publicFeedQueueDepth.set(0); + processing = null; + } + })(); + await processing; + } + + function start(): void { + if (pollTimer || stopping) return; + stopping = false; + pollTimer = setInterval(() => void processDue(), pollIntervalMs); + pollTimer.unref(); + ownershipTimer = setInterval(() => void refreshOwnership(), ownershipRefreshIntervalMs); + ownershipTimer.unref(); + void refreshOwnership(); + void processDue(); + } + + async function stop(): Promise { + stopping = true; + if (pollTimer) clearInterval(pollTimer); + if (ownershipTimer) clearInterval(ownershipTimer); + pollTimer = null; + ownershipTimer = null; + if (processing) await processing.catch(() => undefined); + if (ownershipRefresh) await ownershipRefresh.catch(() => undefined); + const current = [...publishers.values()].map(({ publisher }) => publisher); + publishers.clear(); + await Promise.all(current.map((publisher) => publisher.close().catch(() => undefined))); + } + + return { enqueuePacket, processDue, refreshOwnership, start, stop }; +} diff --git a/backend/src/owner/publicFeedRepository.test.ts b/backend/src/owner/publicFeedRepository.test.ts new file mode 100644 index 0000000..a8ae229 --- /dev/null +++ b/backend/src/owner/publicFeedRepository.test.ts @@ -0,0 +1,63 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { + createPublicFeedRepository, + getActivePublicFeedNodeIds, + getActivePublicFeedReaderUsernames, +} from './publicFeedRepository.js'; + +const nodeId = 'A'.repeat(64); + +test('active public-feed readers require opt-in and a visible packet in the last 24 hours', async () => { + let sql = ''; + const readers = await getActivePublicFeedReaderUsernames(async (text) => { + sql = text; + return { rows: [{ owner_username: 'alice' }] }; + }); + + assert.deepEqual([...readers], ['alice']); + assert.match(sql, /settings\.enabled = TRUE/); + assert.match(sql, /packet\.time > NOW\(\) - INTERVAL '24 hours'/); + assert.match(sql, /packet\.visibility_ok IS TRUE/); + assert.match(sql, /packet\.is_private IS NOT TRUE/); + assert.match(sql, /packet\.network IS DISTINCT FROM 'test'/); +}); + +test('active public-feed node IDs require opt-in and recent eligible activity', async () => { + let sql = ''; + const nodeIds = await getActivePublicFeedNodeIds(async (text) => { + sql = text; + return { rows: [{ node_id: 'b'.repeat(64) }, { node_id: 'A'.repeat(64) }] }; + }); + + assert.deepEqual(nodeIds, ['A'.repeat(64), 'B'.repeat(64)]); + assert.match(sql, /settings\.enabled = TRUE/); + assert.match(sql, /packet\.time > NOW\(\) - INTERVAL '24 hours'/); + assert.match(sql, /packet\.visibility_ok IS TRUE/); + assert.match(sql, /packet\.is_private IS NOT TRUE/); + assert.match(sql, /packet\.network IS DISTINCT FROM 'test'/); +}); + +test('owner public-feed status reports recent eligibility separately from opt-in state', async () => { + let params: unknown[] | undefined; + const repository = createPublicFeedRepository(async (sql, queryParams) => { + assert.match(sql, /recent_activity/); + params = queryParams; + return { + rows: [{ + enabled: true, + recent_packet_count: 2, + last_forwarded_at: '2026-09-20T12:00:00.000Z', + }], + }; + }); + + assert.deepEqual(await repository.accountStatus('alice', [nodeId]), { + enabled: true, + nodeCount: 1, + eligible: true, + recentPacketCount: 2, + lastForwardedAt: '2026-09-20T12:00:00.000Z', + }); + assert.deepEqual(params, ['alice', [nodeId]]); +}); diff --git a/backend/src/owner/publicFeedRepository.ts b/backend/src/owner/publicFeedRepository.ts new file mode 100644 index 0000000..84be856 --- /dev/null +++ b/backend/src/owner/publicFeedRepository.ts @@ -0,0 +1,450 @@ +import type { DatabaseQueryFn } from '../db/index.js'; +import type { LivePacket } from '../types/index.js'; + +export type PublicFeedQueryFn = DatabaseQueryFn; +export type PublicFeedTransactionFn = (work: (query: PublicFeedQueryFn) => Promise) => Promise; + +export type PublicFeedAuthorizationSnapshot = { + mqttUsername: string; + isActive: boolean; + nodeIds: string[]; +}; + +export type PublicFeedDeliveryRow = { + id: string; + node_id: string; + event_key: string; + packet_hash: string; + source_topic: string; + destination_topic: string; + payload: Record; + attempts: number; +}; + +const NORMALIZED_NODE_ID_RE = /^[0-9A-F]{64}$/; + +function normalizedNodeIds(nodeIds: string[]): string[] { + return Array.from(new Set( + nodeIds + .map((nodeId) => nodeId.trim().toUpperCase()) + .filter((nodeId) => NORMALIZED_NODE_ID_RE.test(nodeId)), + )); +} + +/** + * A public-feed reader is granted only while its opted-in account has produced + * at least one public packet observation in the rolling 24-hour window. This + * set is consumed by the broker ACL reconciler; it is deliberately derived + * from the materialised verified owner mapping rather than owner credentials. + */ +export async function getActivePublicFeedReaderUsernames( + query: PublicFeedQueryFn, +): Promise> { + const result = await query<{ owner_username: string }>( + `SELECT DISTINCT feed_node.owner_username + FROM owner_public_feed_nodes feed_node + JOIN owner_public_feed_settings settings + ON settings.owner_username = feed_node.owner_username + AND settings.enabled = TRUE + WHERE EXISTS ( + SELECT 1 + FROM packets packet + WHERE packet.rx_node_id = feed_node.node_id + AND packet.time > NOW() - INTERVAL '24 hours' + AND packet.network IS DISTINCT FROM 'test' + AND packet.visibility_ok IS TRUE + AND packet.is_private IS NOT TRUE + )`, + ); + return new Set(result.rows.map((row) => row.owner_username)); +} + +/** + * Return the observer IDs that are both opted in and currently contributing + * eligible public packets. The forwarder uses this set to decide which + * observers may be republished into the curated public namespace. + */ +export async function getActivePublicFeedNodeIds( + query: PublicFeedQueryFn, +): Promise { + const result = await query<{ node_id: string }>( + `SELECT DISTINCT feed_node.node_id + FROM owner_public_feed_nodes feed_node + JOIN owner_public_feed_settings settings + ON settings.owner_username = feed_node.owner_username + AND settings.enabled = TRUE + WHERE EXISTS ( + SELECT 1 + FROM packets packet + WHERE packet.rx_node_id = feed_node.node_id + AND packet.time > NOW() - INTERVAL '24 hours' + AND packet.network IS DISTINCT FROM 'test' + AND packet.visibility_ok IS TRUE + AND packet.is_private IS NOT TRUE + ) + ORDER BY feed_node.node_id`, + ); + return normalizedNodeIds(result.rows.map((row) => row.node_id)).sort(); +} + +export function createPublicFeedRepository( + query: PublicFeedQueryFn, + transaction?: PublicFeedTransactionFn, +) { + async function accountStatus(ownerUsername: string, nodeIds: string[]): Promise<{ + enabled: boolean; + nodeCount: number; + eligible: boolean; + recentPacketCount: number; + lastForwardedAt: string | null; + }> { + const activeNodeIds = normalizedNodeIds(nodeIds); + const result = await query<{ + enabled: boolean | null; + recent_packet_count: number | string | null; + last_forwarded_at: Date | string | null; + }>( + `WITH recent_activity AS ( + SELECT COUNT(*)::int AS recent_packet_count + FROM packets packet + WHERE packet.rx_node_id = ANY($2::text[]) + AND packet.time > NOW() - INTERVAL '24 hours' + AND packet.network IS DISTINCT FROM 'test' + AND packet.visibility_ok IS TRUE + AND packet.is_private IS NOT TRUE + ) + SELECT COALESCE(settings.enabled, FALSE) AS enabled, + recent_activity.recent_packet_count, + MAX(delivery.delivered_at) AS last_forwarded_at + FROM (SELECT $1::text AS owner_username) requested + CROSS JOIN recent_activity + LEFT JOIN owner_public_feed_settings settings + ON settings.owner_username = requested.owner_username + LEFT JOIN owner_public_feed_nodes feed_node + ON feed_node.owner_username = requested.owner_username + LEFT JOIN public_feed_deliveries delivery + ON delivery.node_id = feed_node.node_id + AND delivery.status = 'succeeded' + AND delivery.delivered_at IS NOT NULL + GROUP BY settings.enabled, recent_activity.recent_packet_count`, + [ownerUsername, activeNodeIds], + ); + const row = result.rows[0]; + const recentPacketCount = Number(row?.recent_packet_count ?? 0); + return { + enabled: row?.enabled === true, + nodeCount: nodeIds.length, + eligible: recentPacketCount > 0, + recentPacketCount: Number.isFinite(recentPacketCount) ? recentPacketCount : 0, + lastForwardedAt: row?.last_forwarded_at + ? new Date(row.last_forwarded_at).toISOString() + : null, + }; + } + + async function activeNodeIds(): Promise { + return getActivePublicFeedNodeIds(query); + } + + async function setEnabled( + ownerUsername: string, + nodeIds: string[], + enabled: boolean, + ): Promise { + const update = async (runQuery: PublicFeedQueryFn): Promise => { + await runQuery( + `INSERT INTO owner_public_feed_settings (owner_username, enabled, updated_at) + VALUES ($1, $2, NOW()) + ON CONFLICT (owner_username) DO UPDATE SET + enabled = EXCLUDED.enabled, + updated_at = NOW()`, + [ownerUsername, enabled], + ); + await runQuery( + `DELETE FROM owner_public_feed_nodes + WHERE owner_username = $1`, + [ownerUsername], + ); + if (enabled && nodeIds.length > 0) { + await runQuery( + `INSERT INTO owner_public_feed_nodes (owner_username, node_id) + SELECT $1, node_id + FROM unnest($2::text[]) AS selected(node_id) + ON CONFLICT (owner_username, node_id) DO NOTHING`, + [ownerUsername, nodeIds], + ); + } + if (!enabled) { + await runQuery( + `UPDATE public_feed_deliveries delivery + SET status = 'dead_lettered', + last_error = 'public feed disabled', + claim_token = NULL, + claim_expires_at = NULL, + updated_at = NOW() + WHERE delivery.status IN ('pending', 'failed') + AND NOT EXISTS ( + SELECT 1 + FROM owner_public_feed_nodes active_node + WHERE active_node.node_id = delivery.node_id + )`, + ); + } + }; + if (transaction) { + await transaction(update); + return; + } + await update(query); + } + + /** + * Rebuild the materialised owner-to-node mapping from the verified owner + * database. The snapshot is supplied by the owner-auth module so this + * analytics database never becomes an independent source of ownership. + */ + async function syncOwnership(snapshot: PublicFeedAuthorizationSnapshot[]): Promise { + const pairSet = new Set(); + const usernames: string[] = []; + const nodeIds: string[] = []; + for (const account of snapshot) { + if (!account.isActive) continue; + const username = account.mqttUsername.trim(); + if (!username) continue; + for (const rawNodeId of account.nodeIds) { + const nodeId = rawNodeId.trim().toUpperCase(); + const pair = `${username}\u0000${nodeId}`; + if (!/^[0-9A-F]{64}$/.test(nodeId) || pairSet.has(pair)) continue; + pairSet.add(pair); + usernames.push(username); + nodeIds.push(nodeId); + } + } + + const update = async (runQuery: PublicFeedQueryFn): Promise => { + await runQuery( + `DELETE FROM owner_public_feed_nodes feed_node + WHERE NOT EXISTS ( + SELECT 1 + FROM unnest($1::text[], $2::text[]) AS requested(owner_username, node_id) + WHERE requested.owner_username = feed_node.owner_username + AND requested.node_id = feed_node.node_id + ) + OR NOT EXISTS ( + SELECT 1 + FROM owner_public_feed_settings settings + WHERE settings.owner_username = feed_node.owner_username + AND settings.enabled = TRUE + )`, + [usernames, nodeIds], + ); + if (usernames.length > 0) { + await runQuery( + `INSERT INTO owner_public_feed_nodes (owner_username, node_id) + SELECT requested.owner_username, requested.node_id + FROM unnest($1::text[], $2::text[]) AS requested(owner_username, node_id) + JOIN owner_public_feed_settings settings + ON settings.owner_username = requested.owner_username + AND settings.enabled = TRUE + ON CONFLICT (owner_username, node_id) DO NOTHING`, + [usernames, nodeIds], + ); + } + await runQuery( + `UPDATE public_feed_deliveries delivery + SET status = 'dead_lettered', + last_error = 'public feed ownership changed', + claim_token = NULL, + claim_expires_at = NULL, + updated_at = NOW() + WHERE delivery.status IN ('pending', 'failed') + AND NOT EXISTS ( + SELECT 1 + FROM owner_public_feed_nodes active_node + WHERE active_node.node_id = delivery.node_id + )`, + ); + }; + if (transaction) { + await transaction(update); + return; + } + await update(query); + } + + async function hasActiveNode(nodeId: string): Promise { + const result = await query( + `SELECT 1 + FROM owner_public_feed_nodes feed_node + JOIN owner_public_feed_settings settings + ON settings.owner_username = feed_node.owner_username + WHERE feed_node.node_id = $1 + AND settings.enabled = TRUE + LIMIT 1`, + [nodeId], + ); + return result.rows.length > 0; + } + + async function enqueueDelivery(input: { + nodeId: string; + eventKey: string; + packet: LivePacket; + destinationTopic: string; + payload: Record; + }): Promise { + const result = await query( + `INSERT INTO public_feed_deliveries ( + node_id, event_key, packet_hash, source_topic, destination_topic, payload + ) + SELECT $1, $2, $3, $4, $5, $6::jsonb + WHERE EXISTS ( + SELECT 1 + FROM owner_public_feed_nodes feed_node + JOIN owner_public_feed_settings settings + ON settings.owner_username = feed_node.owner_username + WHERE feed_node.node_id = $1 + AND settings.enabled = TRUE + ) + ON CONFLICT (event_key) DO NOTHING + RETURNING id`, + [ + input.nodeId, + input.eventKey, + input.packet.packetHash, + input.packet.topic, + input.destinationTopic, + JSON.stringify(input.payload), + ], + ); + return result.rows.length > 0; + } + + async function reclaimExpiredClaims(now = new Date()): Promise { + const result = await query<{ count: string }>( + `WITH reclaimed AS ( + UPDATE public_feed_deliveries + SET status = CASE WHEN attempts >= 5 THEN 'dead_lettered' ELSE 'failed' END, + next_attempt_at = NOW(), + claim_token = NULL, + claim_expires_at = NULL, + updated_at = NOW() + WHERE status = 'delivering' + AND claim_expires_at < $1 + RETURNING id + ) SELECT COUNT(*)::text AS count FROM reclaimed`, + [now], + ); + return Number(result.rows[0]?.count ?? 0); + } + + async function discardInactiveDeliveries(): Promise { + const result = await query<{ id: string }>( + `UPDATE public_feed_deliveries delivery + SET status = 'dead_lettered', + last_error = 'public feed is no longer enabled for this node', + claim_token = NULL, + claim_expires_at = NULL, + updated_at = NOW() + WHERE delivery.status IN ('pending', 'failed') + AND NOT EXISTS ( + SELECT 1 + FROM owner_public_feed_nodes feed_node + JOIN owner_public_feed_settings settings + ON settings.owner_username = feed_node.owner_username + WHERE feed_node.node_id = delivery.node_id + AND settings.enabled = TRUE + ) + RETURNING delivery.id::text`, + ); + return result.rows.length; + } + + async function claimDue(limit: number, claimToken: string): Promise { + const result = await query( + `WITH due AS ( + SELECT delivery.id + FROM public_feed_deliveries delivery + WHERE delivery.status IN ('pending', 'failed') + AND delivery.attempts < 5 + AND delivery.next_attempt_at <= NOW() + AND EXISTS ( + SELECT 1 + FROM owner_public_feed_nodes feed_node + JOIN owner_public_feed_settings settings + ON settings.owner_username = feed_node.owner_username + WHERE feed_node.node_id = delivery.node_id + AND settings.enabled = TRUE + ) + ORDER BY delivery.next_attempt_at, delivery.id + FOR UPDATE SKIP LOCKED + LIMIT $1 + ) + UPDATE public_feed_deliveries delivery + SET status = 'delivering', + attempts = delivery.attempts + 1, + last_attempt_at = NOW(), + claim_token = $2, + claim_expires_at = NOW() + INTERVAL '30 seconds', + updated_at = NOW() + FROM due + WHERE delivery.id = due.id + RETURNING delivery.id::text, delivery.node_id, delivery.event_key, + delivery.packet_hash, delivery.source_topic, + delivery.destination_topic, delivery.payload, delivery.attempts`, + [limit, claimToken], + ); + return result.rows; + } + + async function markSucceeded(id: string, claimToken: string): Promise { + await query( + `UPDATE public_feed_deliveries + SET status = 'succeeded', delivered_at = NOW(), last_error = NULL, + claim_token = NULL, claim_expires_at = NULL, updated_at = NOW() + WHERE id = $1 AND claim_token = $2`, + [id, claimToken], + ); + } + + async function markFailed(id: string, claimToken: string, attempts: number, error: string): Promise { + const terminal = attempts >= 5; + const delaySeconds = Math.min(300, 2 ** Math.max(0, attempts - 1)); + await query( + `UPDATE public_feed_deliveries + SET status = $3, + next_attempt_at = NOW() + ($4::text || ' seconds')::interval, + last_error = $5, claim_token = NULL, claim_expires_at = NULL, + updated_at = NOW() + WHERE id = $1 AND claim_token = $2`, + [id, claimToken, terminal ? 'dead_lettered' : 'failed', String(delaySeconds), error.slice(0, 500)], + ); + } + + async function markDiscarded(id: string, claimToken: string, error: string): Promise { + await query( + `UPDATE public_feed_deliveries + SET status = 'dead_lettered', last_error = $3, + claim_token = NULL, claim_expires_at = NULL, updated_at = NOW() + WHERE id = $1 AND claim_token = $2`, + [id, claimToken, error.slice(0, 500)], + ); + } + + return { + accountStatus, + activeNodeIds, + setEnabled, + syncOwnership, + hasActiveNode, + enqueueDelivery, + reclaimExpiredClaims, + discardInactiveDeliveries, + claimDue, + markSucceeded, + markFailed, + markDiscarded, + }; +} + +export type PublicFeedRepository = ReturnType; diff --git a/backend/src/path-beta/championPriors.test.ts b/backend/src/path-beta/championPriors.test.ts new file mode 100644 index 0000000..d377b3e --- /dev/null +++ b/backend/src/path-beta/championPriors.test.ts @@ -0,0 +1,30 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { buildChampionPriors, loadChampionPriors } from './championPriors.js'; + +test('combines regional counts and keeps position indices aligned with source edges', () => { + const priors = buildChampionPriors({ + positionPrefixes: [{ prefix: 'aa', position: 0, node_id: 'AA01', count: '3' }, { prefix: 'AA', position: 1, node_id: 'AA01', count: 2 }], + corridors: [{ src_node_id: 'SRC', rx_node_id: 'RX', position: 0, node_id: 'AA01', count: 3 }], + transitions: [{ from_node_id: 'AA01', to_node_id: 'BB01', count: 5 }, { from_node_id: 'AA01', to_node_id: 'BB01', count: '7' }], + positionTransitions: [{ position: 0, from_node_id: 'SRC', to_node_id: 'AA01', count: 3 }, { position: 1, from_node_id: 'AA01', to_node_id: 'BB01', count: 2 }], + }); + assert.equal(priors.prefixCounts.get('AA|AA01'), 5); + assert.equal(priors.positionPrefixCounts.get('AA|0|AA01'), 3); + assert.equal(priors.transitionCounts.get('AA01|BB01'), 12); + assert.equal(priors.corridorCounts.get('SRC|RX|0|AA01'), 3); + assert.equal(priors.positionTransitionCounts.get('0|SRC|AA01'), 3); + assert.equal(priors.verifiedEdgeCounts.get('AA01|BB01'), 2); + assert.equal(priors.verifiedEdgeCounts.has('BB01|AA01'), false); +}); + +test('loads all four count tables and fails visibly if a bounded model would be truncated', async () => { + const tables: string[] = []; + await loadChampionPriors('ukmesh', async (sql, params) => { + assert.equal(params[0], 'ukmesh'); + tables.push(sql.match(/FROM (\w+)/)![1]!); + return { rows: [] }; + }); + assert.deepEqual(tables.sort(), ['path_corridor_priors', 'path_position_prefix_priors', 'path_position_transition_priors', 'path_transition_priors']); + await assert.rejects(loadChampionPriors('ukmesh', async () => ({ rows: new Array(100_001) })), /PATH_CHAMPION_PRIOR_LIMIT/); +}); diff --git a/backend/src/path-beta/championPriors.ts b/backend/src/path-beta/championPriors.ts new file mode 100644 index 0000000..148bfa0 --- /dev/null +++ b/backend/src/path-beta/championPriors.ts @@ -0,0 +1,73 @@ +/** Count-based evidence published by the multibyte path learner. */ +export type ChampionPriorRows = { + positionPrefixes: { prefix: string; position: number; node_id: string; count: number | string }[]; + corridors: { src_node_id: string; rx_node_id: string; position: number; node_id: string; count: number | string }[]; + transitions: { from_node_id: string; to_node_id: string; count: number | string }[]; + positionTransitions: { position: number; from_node_id: string; to_node_id: string; count: number | string }[]; +}; + +export type ChampionPriors = { + prefixCounts: Map; + positionPrefixCounts: Map; + corridorCounts: Map; + transitionCounts: Map; + positionTransitionCounts: Map; + verifiedEdgeCounts: Map; +}; + +type PriorQuery = >( + sql: string, params: unknown[], signal?: AbortSignal, +) => Promise<{ rows: T[] }>; + +/** Loaded once per cached resolver context, never once per packet. */ +export async function loadChampionPriors(network: string, query: PriorQuery, signal?: AbortSignal): Promise { + // Refuse an incomplete model instead of silently removing rare choices. + const limit = 100_000; + const read = async >(sql: string): Promise => { + const result = await query(`${sql} LIMIT $2`, [network, limit + 1], signal); + if (result.rows.length > limit) throw new Error('PATH_CHAMPION_PRIOR_LIMIT'); + return result.rows; + }; + const [positionPrefixes, corridors, transitions, positionTransitions] = await Promise.all([ + read( + 'SELECT prefix, position, node_id, count FROM path_position_prefix_priors WHERE network = $1'), + read( + 'SELECT src_node_id, rx_node_id, position, node_id, count FROM path_corridor_priors WHERE network = $1'), + read( + 'SELECT from_node_id, to_node_id, count FROM path_transition_priors WHERE network = $1'), + read( + 'SELECT position, from_node_id, to_node_id, count FROM path_position_transition_priors WHERE network = $1'), + ]); + return buildChampionPriors({ positionPrefixes, corridors, transitions, positionTransitions }); +} + +export function buildChampionPriors(rows: ChampionPriorRows): ChampionPriors { + const result: ChampionPriors = { + prefixCounts: new Map(), + positionPrefixCounts: new Map(), + corridorCounts: new Map(), + transitionCounts: new Map(), + positionTransitionCounts: new Map(), + verifiedEdgeCounts: new Map(), + }; + const add = (map: Map, key: string, value: number | string) => { + const count = Number(value); + if (Number.isFinite(count) && count > 0) map.set(key, (map.get(key) ?? 0) + count); + }; + for (const row of rows.positionPrefixes) { + const prefix = row.prefix.toUpperCase().slice(0, 2); + add(result.prefixCounts, `${prefix}|${row.node_id}`, row.count); + add(result.positionPrefixCounts, `${prefix}|${row.position}|${row.node_id}`, row.count); + } + for (const row of rows.corridors) { + add(result.corridorCounts, `${row.src_node_id}|${row.rx_node_id}|${row.position}|${row.node_id}`, row.count); + } + for (const row of rows.transitions) { + add(result.transitionCounts, `${row.from_node_id}|${row.to_node_id}`, row.count); + } + for (const row of rows.positionTransitions) { + add(result.positionTransitionCounts, `${row.position}|${row.from_node_id}|${row.to_node_id}`, row.count); + add(result.verifiedEdgeCounts, `${row.from_node_id}|${row.to_node_id}`, row.count); + } + return result; +} diff --git a/backend/src/path-beta/resolveWorker.ts b/backend/src/path-beta/resolveWorker.ts index babd981..60692d3 100644 --- a/backend/src/path-beta/resolveWorker.ts +++ b/backend/src/path-beta/resolveWorker.ts @@ -4,9 +4,7 @@ * keeping the main event loop free during CPU-heavy path computation. */ import { parentPort } from 'worker_threads'; -import { query } from '../db/index.js'; -import { lazyResolvePath } from '../path-lazy/lazyResolver.js'; -import { resolveBetaPathForPacketHash, resolveMultiObserverBetaPath } from './resolver.js'; +import { resolveBetaPathForPacketHash, resolveMultiObserverBetaPath, resolveLazyPathForPacketHash } from './resolver.js'; import { getHeldPath, setHeldPath, type HeldPathEntry } from './resolveCache.js'; if (!parentPort) throw new Error('resolveWorker must run as a worker thread'); @@ -26,7 +24,7 @@ parentPort.on('message', (msg: WorkerMessage) => { const stickyMap = msg.stickyMap ? new Map(Object.entries(msg.stickyMap)) : undefined; const heldPath = msg.heldPath ?? getHeldPath(msg.packetHash, msg.network); const run = msg.type === 'resolveLazy' - ? lazyResolvePath(msg.packetHash, msg.network, query) + ? resolveLazyPathForPacketHash(msg.packetHash, msg.network) : msg.type === 'resolveMulti' ? resolveMultiObserverBetaPath(msg.packetHash, msg.network, stickyMap, msg.stickyAgeFraction, { heldPath }) : resolveBetaPathForPacketHash( diff --git a/backend/src/path-beta/resolver.ts b/backend/src/path-beta/resolver.ts index 277ce6e..c5503f8 100644 --- a/backend/src/path-beta/resolver.ts +++ b/backend/src/path-beta/resolver.ts @@ -33,6 +33,13 @@ import type { PathPacket, } from './types.js'; import type { HeldPathEntry } from './resolveCache.js'; +import { loadChampionPriors } from './championPriors.js'; +import type { LazyPath, LazyPathNode, LazyPathResult } from '../path-lazy/lazyResolver.js'; +import { expandResolverScope } from '../networks.js'; + +function resolverScope(network: string): string[] | null { + return network === 'all' ? null : expandResolverScope(network); +} const contextCache = new BoundedTtlMap({ name: 'path_context', @@ -96,7 +103,7 @@ async function buildLearningModel( network: string, signal?: AbortSignal, ): Promise { - const [prefixRows, transitionRows, edgeRows, motifRows, calibrationRows] = await Promise.all([ + const [prefixRows, transitionRows, edgeRows, motifRows, calibrationRows, champion] = await Promise.all([ query( `SELECT prefix, receiver_region, prev_prefix, node_id, count FROM path_prefix_priors @@ -157,6 +164,7 @@ async function buildLearningModel( [network], signal, ), + loadChampionPriors(network, query, signal), ]); const prefixProbabilities = buildOneBytePrefixProbabilities(prefixRows.rows); @@ -208,6 +216,7 @@ async function buildLearningModel( const calibration = calibrationRows.rows[0]; return { + champion, prefixProbabilities, transitionProbabilities, edgeScores, @@ -258,9 +267,9 @@ async function loadContext( query( `SELECT node_id, name, lat, lon, iata, role, elevation_m, last_seen::text AS last_seen FROM nodes - WHERE ($1 = 'all' OR network = $1) + WHERE ($1::text[] IS NULL OR network = ANY($1)) AND (name IS NULL OR name NOT LIKE '%🚫%')`, - [network], + [resolverScope(network)], options?.signal, ), query<{ @@ -281,8 +290,8 @@ async function loadContext( WHERE (nl.itm_viable IS NOT NULL OR nl.force_viable = true) AND (a.name IS NULL OR a.name NOT LIKE '%🚫%') AND (b.name IS NULL OR b.name NOT LIKE '%🚫%') - AND ($1 = 'all' OR (a.network = $1 AND b.network = $1))`, - [network], + AND ($1::text[] IS NULL OR (a.network = ANY($1) AND b.network = ANY($1)))`, + [resolverScope(network)], options?.signal, ), buildLearningModel(network, options?.signal), @@ -340,32 +349,36 @@ function matchesObserverPathHash(rx: MeshNode | null, hash: string | undefined): return Boolean(normalized && nodePathHash(rx.node_id, normalized) === normalized); } -function trimObserverTerminalHop(hops: string[], rx: MeshNode | null): string[] { +function trimObserverTerminalHop(hops: string[], rx: MeshNode | null, packet: PathPacket): string[] { + // RX paths describe the transmitting relays. A short-hash collision with + // the receiver is not evidence that it appended itself to that path. + if (packet.direction === 'rx') return hops; if (!rx || rx.role !== 2 || hops.length <= 1) return hops; return matchesObserverPathHash(rx, hops[hops.length - 1]) ? hops.slice(0, -1) : hops; } -function preparePacketObservation(packet: PathPacket, rx: MeshNode | null): PreparedPacketObservation { - const hashes = (packet.path_hashes ?? []).map(normalizePathHash).filter(Boolean); +export function preparePacketObservation(packet: PathPacket, rx: MeshNode | null): PreparedPacketObservation { + const hashes = (packet.path_hashes ?? []).map(normalizePathHash); const expectedHexLength = packet.path_hash_size_bytes == null ? null : packet.path_hash_size_bytes * 2; - const validatedHashes = expectedHexLength == null - ? hashes - : hashes.filter((hash) => hash.length === expectedHexLength); + const invalidHashes = (expectedHexLength != null && ![2, 4, 6].includes(expectedHexLength)) + || hashes.some((hash) => !/^(?:[0-9A-F]{2}){1,3}$/.test(hash) + || (expectedHexLength != null && hash.length !== expectedHexLength)); const rawHops = packet.hop_count == null - ? validatedHashes - : validatedHashes.slice(0, Math.max(0, packet.hop_count)); + ? hashes + : hashes.slice(0, Math.max(0, packet.hop_count)); const ignoreForPathing = Boolean( - rx?.role === 2 + invalidHashes || (packet.route_type != null && packet.route_type !== 0 && packet.route_type !== 1) + || (packet.direction !== 'rx' && rx?.role === 2 && rawHops.length >= 3 && matchesObserverPathHash(rx, rawHops[0]) - && matchesObserverPathHash(rx, rawHops[rawHops.length - 1]), + && matchesObserverPathHash(rx, rawHops[rawHops.length - 1])), ); return { packet, rx, hashes, rawHops, - hops: trimObserverTerminalHop(rawHops, rx), + hops: trimObserverTerminalHop(rawHops, rx, packet), ignoreForPathing, }; } @@ -410,6 +423,26 @@ function toObserverEntry(prepared: PreparedPacketObservation): BetaObserverEntry }; } +/** Flood copies can reach the same observer along different routes. Keep + * those variants; collapsing them to one row loses valid multibyte evidence. */ +export function prepareObservationEntries(rows: readonly PathPacket[], context: BetaResolveContext): BetaObserverEntry[] { + const variants = new Map(); + const observers = new Set(); + for (const row of rows) { + if (!row.rx_node_id) continue; + const prepared = preparePacketObservation(row, context.nodesById.get(row.rx_node_id) ?? null); + if (prepared.ignoreForPathing || prepared.hops.length === 0 || prepared.hops.length >= MAX_BETA_HOPS) continue; + const entry = toObserverEntry(prepared); + if (!entry) continue; + observers.add(entry.observerId); + const key = `${entry.observerId}|${entry.hops.join(',')}`; + const previous = variants.get(key); + if (!previous || (!previous.packet.src_node_id && row.src_node_id)) variants.set(key, entry); + } + if (observers.size > PATH_MULTI_MAX_OBSERVERS) throw new Error('PATH_HISTORY_LIMIT'); + return [...variants.values()]; +} + function findSharedPathPrefix(pathsByObserver: readonly string[][]): string[] { if (pathsByObserver.length === 0) return []; const first = pathsByObserver[0]!; @@ -654,10 +687,10 @@ export async function resolveBetaPathForPacketHash( ): Promise { throwIfResolutionAborted(options); const packetResult = await query( - `SELECT packet_hash, rx_node_id, src_node_id, packet_type, hop_count, path_hashes, path_hash_size_bytes + `SELECT packet_hash, rx_node_id, src_node_id, packet_type, route_type, payload->>'direction' AS direction, hop_count, path_hashes, path_hash_size_bytes FROM packets WHERE packet_hash = $1 - AND ($2 = 'all' OR network = $2) + AND ($2::text[] IS NULL OR network = ANY($2)) ${observer ? 'AND rx_node_id = $3' : ''} AND NOT EXISTS ( SELECT 1 FROM nodes private_node @@ -679,7 +712,7 @@ export async function resolveBetaPathForPacketHash( hop_count ASC NULLS LAST, time ASC LIMIT 32`, - observer ? [packetHash, network, observer] : [packetHash, network], + observer ? [packetHash, resolverScope(network), observer] : [packetHash, resolverScope(network)], options?.signal, ); if (packetResult.rows.length === 0) return null; @@ -731,7 +764,7 @@ export async function resolveBetaPathForPacketHash( : null; const projection = projectCanonicalPathForObserver(selectedEntry, decoded, context, source); const payload = payloadFromProjection(packetHash, selectedEntry, projection, { - canonicalPath: buildCanonicalPath(decoded), + canonicalPath: buildCanonicalPath(decoded).slice(0, selectedEntry.hops.length), observers: [{ observerId: selectedEntry.observerId }], network, }); @@ -829,21 +862,7 @@ async function resolveMultiObserverBetaPathFromRows( stickyAgeFraction?: number, options?: PathResolutionOptions, ): Promise { - const byObserver = new Map(); - for (const row of rows) { - if (!row.rx_node_id) continue; - const prepared = preparePacketObservation(row, context.nodesById.get(row.rx_node_id) ?? null); - if (prepared.ignoreForPathing) continue; - const previous = byObserver.get(row.rx_node_id); - byObserver.set(row.rx_node_id, previous ? mergeObserverEvidence(previous, prepared) : prepared); - } - if (byObserver.size > PATH_MULTI_MAX_OBSERVERS) throw new Error('PATH_HISTORY_LIMIT'); - - const entries = [...byObserver.values()] - .map(toObserverEntry) - .filter((entry): entry is BetaObserverEntry => ( - entry !== null && entry.hops.length > 0 && entry.hops.length < MAX_BETA_HOPS - )); + const entries = prepareObservationEntries(rows, context); if (entries.length === 0) return null; const groups = groupCompatibleObservations(entries); @@ -886,8 +905,8 @@ async function resolveMultiObserverBetaPathFromRows( : null; const projection = projectCanonicalPathForObserver(entry, decoded, context, source); const payload = payloadFromProjection(packetHash, entry, projection, { - canonicalPath: buildCanonicalPath(decoded), - observers: group.members.map((member) => ({ observerId: member.observerId })), + canonicalPath: buildCanonicalPath(decoded).slice(0, entry.hops.length), + observers: [...new Set(group.members.map((member) => member.observerId))].map((observerId) => ({ observerId })), network, }); results.push(maskResolvedPayload(payload, hiddenCoordMask)); @@ -913,10 +932,10 @@ async function resolveMultiObserverBetaPathFromRows( ok: true, packetHash, network, - observerCount: entries.length, + observerCount: new Set(entries.map((entry) => entry.observerId)).size, sharedPrefixLength, canonicalPath: firstCanonicalPath, - observers: entries.map((entry) => ({ observerId: entry.observerId })), + observers: [...new Set(entries.map((entry) => entry.observerId))].map((observerId) => ({ observerId })), confidence: results[0]?.confidence ?? null, results, ...(regionLinks.length > 0 ? { regionLinks } : {}), @@ -924,19 +943,15 @@ async function resolveMultiObserverBetaPathFromRows( }; } -export async function resolveMultiObserverBetaPath( - packetHash: string, - network: string, - stickyMap?: Map, - stickyAgeFraction?: number, - options?: PathResolutionOptions, -): Promise { +async function loadMultiObserverInputs( + packetHash: string, network: string, options?: PathResolutionOptions, +): Promise<{ rows: PathPacket[]; context: BetaResolveContext } | null> { throwIfResolutionAborted(options); const allResult = await query( - `SELECT packet_hash, rx_node_id, src_node_id, packet_type, hop_count, path_hashes, path_hash_size_bytes + `SELECT packet_hash, rx_node_id, src_node_id, packet_type, route_type, payload->>'direction' AS direction, hop_count, path_hashes, path_hash_size_bytes FROM packets WHERE packet_hash = $1 - AND ($2 = 'all' OR network = $2) + AND ($2::text[] IS NULL OR network = ANY($2)) AND time >= NOW() - ($3::int * INTERVAL '1 hour') AND rx_node_id IS NOT NULL AND NOT EXISTS ( @@ -959,7 +974,7 @@ export async function resolveMultiObserverBetaPath( hop_count ASC NULLS LAST, time ASC LIMIT $4`, - [packetHash, network, PATH_MULTI_HISTORY_WINDOW_HOURS, PATH_MULTI_MAX_SCAN_ROWS + 1], + [packetHash, resolverScope(network), PATH_MULTI_HISTORY_WINDOW_HOURS, PATH_MULTI_MAX_SCAN_ROWS + 1], options?.signal, ); if (allResult.rows.length === 0) return null; @@ -971,11 +986,23 @@ export async function resolveMultiObserverBetaPath( signal: options?.signal, }); throwIfResolutionAborted(options); + return { rows: allResult.rows, context }; +} + +export async function resolveMultiObserverBetaPath( + packetHash: string, + network: string, + stickyMap?: Map, + stickyAgeFraction?: number, + options?: PathResolutionOptions, +): Promise { + const inputs = await loadMultiObserverInputs(packetHash, network, options); + if (!inputs) return null; return resolveMultiObserverBetaPathFromRows( packetHash, network, - allResult.rows, - context, + inputs.rows, + inputs.context, stickyMap, stickyAgeFraction, options, @@ -987,6 +1014,54 @@ export type MultiObserverPathBatch = { limitedPacketHashes: Set; }; +/** The packet detail view and live overlay share candidates, priors and decode. */ +export async function resolveLazyPathForPacketHash( + packetHash: string, network: string, options?: PathResolutionOptions, +): Promise { + const inputs = await loadMultiObserverInputs(packetHash, network, options); + if (!inputs) return null; + const { context } = inputs; + const entries = prepareObservationEntries(inputs.rows, context); + return decodeLazyPathEntries(packetHash, entries, context, options); +} + +export function decodeLazyPathEntries( + packetHash: string, entries: readonly BetaObserverEntry[], context: BetaResolveContext, + options?: PathResolutionOptions, +): LazyPathResult | null { + const observerCount = new Set(entries.map((entry) => entry.observerId)).size; + const paths: LazyPath[] = []; + for (const group of groupCompatibleObservations(entries)) { + throwIfResolutionAborted(options); + const decoded = decodeBetaCanonicalGroup(group, context); + const canonicalPath: LazyPathNode[] = buildCanonicalPath(decoded).map((hop) => ({ + ...hop, + appearances: new Set(group.members.filter((member) => member.hops.length > hop.position).map((member) => member.observerId)).size, + totalObservations: observerCount, + isObserver: false, + })); + for (const member of group.members) { + // Each observer's route ends at its own reception. Interleaving other + // observers into one polyline invents observer-to-relay radio hops. + const observerPath = canonicalPath.slice(0, member.hops.length); + observerPath.push({ + position: member.hops.length, hash: member.observerId, nodeId: member.observerId, + name: member.rx.name, lat: member.rx.lat, lon: member.rx.lon, + appearances: 1, totalObservations: observerCount, ambiguous: false, isObserver: true, + }); + paths.push({ + canonicalPath: observerPath, + coordinates: observerPath.filter((hop) => hop.lat != null && hop.lon != null) + .map((hop) => [hop.lat!, hop.lon!]), + matchedHops: observerPath.filter((hop) => !hop.isObserver && hop.nodeId != null && !hop.ambiguous).length, + totalHops: member.hops.length, + observerIds: [member.observerId], + }); + } + } + return paths.length > 0 ? { packetHash, observerCount, paths } : null; +} + export async function createMultiObserverPathBatchResolver( network: string, visibilityGeneration: number, @@ -1014,17 +1089,17 @@ export async function createMultiObserverPathBatchResolver( if (packetHashes.length === 0) return { results, limitedPacketHashes }; const packetResult = await namedQuery( - 'path-history-observations-v1', + 'path-history-observations-v2', `SELECT observation.packet_hash, observation.rx_node_id, observation.src_node_id, - observation.packet_type, observation.hop_count, observation.path_hashes, + observation.packet_type, observation.route_type, observation.direction, observation.hop_count, observation.path_hashes, observation.path_hash_size_bytes FROM unnest($1::text[]) WITH ORDINALITY AS requested(packet_hash, selection_order) CROSS JOIN LATERAL ( - SELECT p.packet_hash, p.rx_node_id, p.src_node_id, p.packet_type, + SELECT p.packet_hash, p.rx_node_id, p.src_node_id, p.packet_type, p.route_type, p.payload->>'direction' AS direction, p.hop_count, p.path_hashes, p.path_hash_size_bytes, p.time FROM packets p WHERE p.packet_hash = requested.packet_hash - AND ($2 = 'all' OR p.network = $2) + AND ($2::text[] IS NULL OR p.network = ANY($2)) AND p.time >= $3 AND p.time <= $4 AND p.rx_node_id IS NOT NULL @@ -1055,7 +1130,7 @@ export async function createMultiObserverPathBatchResolver( CASE WHEN observation.src_node_id IS NOT NULL THEN 1 ELSE 0 END DESC, observation.hop_count ASC NULLS LAST, observation.time ASC`, - [packetHashes, network, windowStart, windowEnd, PATH_MULTI_MAX_SCAN_ROWS + 1], + [packetHashes, resolverScope(network), windowStart, windowEnd, PATH_MULTI_MAX_SCAN_ROWS + 1], batchSignal, ); batchSignal?.throwIfAborted(); diff --git a/backend/src/path-beta/resolverContext.test.ts b/backend/src/path-beta/resolverContext.test.ts index 42568a0..c1f99b2 100644 --- a/backend/src/path-beta/resolverContext.test.ts +++ b/backend/src/path-beta/resolverContext.test.ts @@ -4,9 +4,14 @@ import { buildOneBytePrefixProbabilities, canReusePathContext, mergeObserverEvidence, + preparePacketObservation, + prepareObservationEntries, + decodeLazyPathEntries, type PreparedPacketObservation, } from './resolver.js'; import type { PathPacket } from './types.js'; +import type { BetaResolveContext, MeshNode } from './types.js'; +import { buildChampionPriors } from './championPriors.js'; function prepared(size: number, hops: string[]): PreparedPacketObservation { const packet: PathPacket = { @@ -31,6 +36,53 @@ test('retains same-observer multibyte anchors while keeping the longer path tail assert.equal(merged.packet.path_hash_size_bytes, 1, 'the longer observation remains the projection row'); }); +const receiver: MeshNode = { node_id: 'AA0011', name: null, lat: 51, lon: 1, role: 2, iata: null, elevation_m: null, last_seen: null }; + +test('RX prefix collisions retain the final relay; TX strips the appended observer', () => { + const raw = { ...prepared(1, ['BB', 'AA']).packet, rx_node_id: receiver.node_id }; + assert.deepEqual(preparePacketObservation({ ...raw, direction: 'rx' }, receiver).hops, ['BB', 'AA']); + assert.deepEqual(preparePacketObservation({ ...raw, direction: 'tx' }, receiver).hops, ['BB']); +}); + +test('direct-routing destinations and malformed middle hashes cannot become observed relay chains', () => { + assert.equal(preparePacketObservation({ ...prepared(1, ['AA', 'BB']).packet, route_type: 2 }, receiver).ignoreForPathing, true); + const bad = preparePacketObservation(prepared(1, ['AA', 'zz', 'BB']).packet, receiver); + assert.equal(bad.ignoreForPathing, true); + assert.equal(bad.rawHops.length, 3); +}); + +test('keeps distinct routes at the same observer while de-duplicating identical receptions', () => { + const a = { ...prepared(3, ['BB0011', 'CC0011']).packet, rx_node_id: receiver.node_id }; + const b = { ...a, path_hashes: ['BB0022', 'CC0022'] }; + const entries = prepareObservationEntries([a, b, a], { + nodesById: new Map([[receiver.node_id, receiver]]), + } as BetaResolveContext); + assert.equal(entries.length, 2); + assert.deepEqual(entries.map((entry) => entry.hops), [a.path_hashes, b.path_hashes]); + const long = { ...a, path_hash_size_bytes: 1, hop_count: 40, path_hashes: new Array(40).fill('BB') }; + assert.equal(prepareObservationEntries([long], { nodesById: new Map([[receiver.node_id, receiver]]) } as BetaResolveContext)[0]?.hops.length, 40); +}); + +test('packet-detail projections end at their own observers without creating cross-observer edges', () => { + const relayA = { ...receiver, node_id: 'BB0011' }; + const relayB = { ...receiver, node_id: 'CC0022' }; + const rxLong = { ...receiver, node_id: 'DD0033' }; + const ctx: BetaResolveContext = { + loadedAt: 0, visibilityGeneration: 0, + nodesById: new Map([relayA, relayB, receiver, rxLong].map((node) => [node.node_id, node])), + repeaterNodes: [relayA, relayB, receiver, rxLong], linkMetrics: new Map(), + learningModel: { prefixProbabilities: new Map(), transitionProbabilities: new Map(), edgeScores: new Map(), motifProbabilities: new Map(), + confidenceScale: 1, confidenceBias: 0, bucketHours: 6, + champion: buildChampionPriors({ positionPrefixes: [], corridors: [], transitions: [], positionTransitions: [] }) }, + }; + const short = { ...prepared(3, ['BB0011']).packet, rx_node_id: receiver.node_id, direction: 'rx' }; + const long = { ...prepared(3, ['BB0011', 'CC0022']).packet, rx_node_id: rxLong.node_id, direction: 'rx' }; + const result = decodeLazyPathEntries('packet', prepareObservationEntries([short, long], ctx), ctx)!; + assert.equal(result.observerCount, 2); + assert.deepEqual(result.paths.find((path) => path.observerIds[0] === receiver.node_id)?.canonicalPath.map((node) => node.nodeId), ['BB0011', receiver.node_id]); + assert.deepEqual(result.paths.find((path) => path.observerIds[0] === rxLong.node_id)?.canonicalPath.map((node) => node.nodeId), ['BB0011', 'CC0022', rxLong.node_id]); +}); + test('collapses multibyte path facts into calibrated one-byte prefix priors', () => { const probabilities = buildOneBytePrefixProbabilities([ { prefix: 'AA01', receiver_region: 'LON', prev_prefix: 'BB01', node_id: 'node-a', count: 3 }, diff --git a/backend/src/path-beta/sharedDecoder.test.ts b/backend/src/path-beta/sharedDecoder.test.ts index 8fd8f3c..8f784f7 100644 --- a/backend/src/path-beta/sharedDecoder.test.ts +++ b/backend/src/path-beta/sharedDecoder.test.ts @@ -9,6 +9,7 @@ import { type BetaSharedDecode, } from './sharedDecoder.js'; import type { BetaResolveContext, MeshNode, PathPacket } from './types.js'; +import { buildChampionPriors } from './championPriors.js'; const meshNode = (node_id: string, lat: number, lon: number, iata: string | null = null): MeshNode => ({ node_id, @@ -21,6 +22,79 @@ const meshNode = (node_id: string, lat: number, lon: number, iata: string | null last_seen: null, }); +test('source and learned route-position evidence resolve an otherwise equal one-byte collision', () => { + const source = meshNode('SRC', 51, 1); + const good = meshNode('AA01', 51.1, 1); + const other = meshNode('AA02', 51.1, 1); + const observer = entry('RX', ['AA']); + observer.rx = meshNode('RX', 51.2, 1); + observer.packet.src_node_id = source.node_id; + const ctx = context([source, other, good, observer.rx]); + ctx.learningModel.champion = buildChampionPriors({ + positionPrefixes: [{ prefix: 'AA', position: 0, node_id: good.node_id, count: 10 }], + corridors: [{ src_node_id: 'SRC', rx_node_id: 'RX', position: 0, node_id: good.node_id, count: 4 }], + transitions: [{ from_node_id: 'SRC', to_node_id: good.node_id, count: 10 }], + positionTransitions: [{ position: 0, from_node_id: 'SRC', to_node_id: good.node_id, count: 10 }], + }); + assert.equal(decodeBetaCanonicalGroup({ canonicalHashes: ['AA'], members: [observer] }, ctx).hops.get(0)?.nodeId, good.node_id); +}); + +test('confirmed identities survive missing coordinates and terrain disagreement', () => { + const absent = meshNode('AA0001', 0, 0); + const remote = meshNode('BB0002', 57, 1); + const observer = entry('RX', ['AA0001', 'BB0002']); + observer.rx = meshNode('RX', 51, 1); + const ctx = context([absent, remote, observer.rx]); + const result = decodeBetaCanonicalGroup({ canonicalHashes: observer.hops, members: [observer] }, ctx); + assert.deepEqual([...result.hops.values()].map((hop) => hop.nodeId), ['AA0001', 'BB0002']); + assert.equal(result.hops.get(0)?.lat, null); + assert.equal(result.hops.get(0)?.lon, null); +}); + +test('later evidence can refine a held hop at the beginning of a long route', () => { + const nodes = ['AA01', 'AA02', 'BB01', 'CC01', 'DD01'].map((id) => meshNode(id, 51, 1)); + const observer = entry('RX', ['AA', 'BB', 'CC', 'DD']); + observer.rx = meshNode('RX', 51.1, 1); + const ctx = context([...nodes, observer.rx]); + ctx.learningModel.prefixProbabilities.set('unknown|AA||AA02', 1); + const result = decodeBetaCanonicalGroupWithHeldPath({ canonicalHashes: observer.hops, members: [observer] }, ctx, + ['AA01', 'BB01', 'CC01', 'DD01']); + assert.equal(result.refined, true); + assert.equal(result.hops.get(0)?.nodeId, 'AA02'); +}); + +test('a missing-position candidate needs a real directed edge to its observer', () => { + const missing = meshNode('AA01', 0, 0); + const plausible = meshNode('AA02', 51.1, 1); + const observer = entry('RX', ['AA']); + observer.rx = meshNode('RX', 51.2, 1); + const ctx = context([missing, plausible, observer.rx]); + const rows = { + positionPrefixes: [{ prefix: 'AA', position: 0, node_id: 'AA01', count: 1000 }], + corridors: [], transitions: [], positionTransitions: [], + }; + ctx.learningModel.champion = buildChampionPriors(rows); + assert.equal(decodeBetaCanonicalGroup({ canonicalHashes: ['AA'], members: [observer] }, ctx).hops.get(0)?.nodeId, 'AA02'); + ctx.learningModel.champion = buildChampionPriors({ ...rows, transitions: [{ from_node_id: 'AA01', to_node_id: 'RX', count: 10 }] }); + const hop = decodeBetaCanonicalGroup({ canonicalHashes: ['AA'], members: [observer] }, ctx).hops.get(0); + assert.equal(hop?.nodeId, 'AA01'); + assert.equal(hop?.lat, null); +}); + +test('a relay sharing the source prefix does not erase the source endpoint', () => { + const source = meshNode('AA0001', 51, 1); + const relay = meshNode('AA0002', 51.1, 1); + const observer = entry('RX', ['AA']); + observer.rx = meshNode('RX', 51.2, 1); + const ctx = context([source, relay, observer.rx]); + const decoded: BetaSharedDecode = { + canonicalHashes: ['AA'], hopConfidences: new Map([[0, 1]]), + hops: new Map([[0, { hash: 'AA', nodeId: relay.node_id, name: relay.name, lat: relay.lat, lon: relay.lon, margin: 1, ambiguous: false }]]), + }; + assert.deepEqual(projectCanonicalPathForObserver(observer, decoded, ctx, source).purplePath, + [[51, 1], [51.1, 1], [51.2, 1]]); +}); + function packet(rx_node_id: string, path_hashes: string[], packet_type = 0): PathPacket { return { packet_hash: 'packet', diff --git a/backend/src/path-beta/sharedDecoder.ts b/backend/src/path-beta/sharedDecoder.ts index a8ae2f6..1ccb8c6 100644 --- a/backend/src/path-beta/sharedDecoder.ts +++ b/backend/src/path-beta/sharedDecoder.ts @@ -1,6 +1,7 @@ import { buildObserverBounds, decodePathWithScore, + distanceKm, evaluatePhysicalHop, indexCandidatesByHash, scoreFixedPath, @@ -17,9 +18,11 @@ import { ML_DOMINANT_THRESHOLD, NULL_BASELINE, SCORE, + CORRIDOR_INTERPOLATION_KM, + OBSERVER_DISTANCE_DECAY_KM, } from '../path-shared/scoring.js'; import { nodePathHash, normalizePathHash } from '../path-hash/utils.js'; -import { clamp, hasCoords, linkKey } from './geometry.js'; +import { clamp, hasCoords, isValidMapCoord, linkKey } from './geometry.js'; import type { BetaResolveContext, MeshNode, PathPacket } from './types.js'; export type BetaObserverEntry = { @@ -207,16 +210,27 @@ export function decodeBetaCanonicalGroup( .filter(hasCoords) .map((node) => ({ lat: node.lat!, lon: node.lon! })); if (sourceNode) observerCoordinates.push({ lat: sourceNode.lat!, lon: sourceNode.lon! }); - const bounds = buildObserverBounds(observerCoordinates, pathingConfig.maxHopKm); - const candidateEvidence: CandidateNodeEvidence[] = context.repeaterNodes.map((node) => ({ + // A flood can wander many hops beyond the receiving observer's region. + // A one-hop region bound silently discarded the start of long routes. + const bounds = buildObserverBounds(observerCoordinates, pathingConfig.maxHopKm * Math.max(1, canonicalHashes.length)); + const candidateNodes = [...context.repeaterNodes]; + if (context.learningModel.champion) { + for (const node of context.nodesById.values()) { + if (!isValidMapCoord(node.lat, node.lon) && (node.role === null || node.role === 2) + && context.learningModel.champion.prefixCounts.has(`${node.node_id.slice(0, 2).toUpperCase()}|${node.node_id}`)) { + candidateNodes.push(node); + } + } + } + const candidateEvidence: CandidateNodeEvidence[] = candidateNodes.map((node) => ({ nodeId: node.node_id, name: node.name, - lat: node.lat, - lon: node.lon, + lat: hasCoords(node) ? node.lat : null, + lon: hasCoords(node) ? node.lon : null, elevationM: node.elevation_m, })); - const candidatesByHash = indexCandidatesByHash(canonicalHashes, candidateEvidence, bounds); - const confirmed = confirmedMultibyteAssignments(group, context.repeaterNodes); + const candidatesByHash = indexCandidatesByHash(canonicalHashes, candidateEvidence, bounds, Boolean(context.learningModel.champion)); + const confirmed = confirmedMultibyteAssignments(group, [...context.nodesById.values()]); // A confirmed relay may sit outside the observer-only candidate bound on a // long path. Preserve that direct evidence explicitly before locking it. for (const [position, nodeId] of confirmed) { @@ -242,6 +256,19 @@ export function decodeBetaCanonicalGroup( const bucketHours = context.learningModel.bucketHours || 6; const bucket = currentHourBucket(bucketHours); const stickyStrength = clamp(1 - Number(stickyAgeFraction ?? 0), 0, 1); + const champion = context.learningModel.champion; + const confirmedEdges = new Set(); + for (let position = 1; position < canonicalHashes.length; position++) { + const from = confirmed.get(position - 1); + const to = confirmed.get(position); + if (from && to) confirmedEdges.add(`${from}|${to}`); + } + for (const member of group.members) { + const first = confirmed.get(0); + const last = confirmed.get(member.hops.length - 1); + if (first && member.packet.src_node_id) confirmedEdges.add(`${member.packet.src_node_id}|${first}`); + if (last) confirmedEdges.add(`${last}|${member.observerId}`); + } const evidence: PathDecoderEvidence = { weights: SCORE, @@ -259,10 +286,13 @@ export function decodeBetaCanonicalGroup( nodeId: sourceNode.node_id, elevationM: sourceNode.elevation_m, } : undefined, - endpointTransitionsGateOnly: true, + endpointTransitionsGateOnly: !champion, + scoreDirectAnchorTransitions: Boolean(champion), + requireUnpositionedTransitionEvidence: Boolean(champion), directAnchors: (position) => anchors.get(position) ?? [], prefixProbability: (nodeId, hash, previousHash) => { const prefix = normalizePathHash(hash).slice(0, 2); + if (champion) return champion.prefixCounts.get(`${prefix}|${nodeId}`) ?? 0; const previous = normalizePathHash(previousHash).slice(0, 2); const learned = regionalMaximum(regions, (region) => ( context.learningModel.prefixProbabilities.get(`${region}|${prefix}|${previous}|${nodeId}`) @@ -271,14 +301,55 @@ export function decodeBetaCanonicalGroup( const sticky = stickyMap?.get(prefix) === nodeId ? stickyStrength : 0; return Math.max(learned, sticky); }, + positionPrefixCount: (position, nodeId, hash) => ( + champion?.positionPrefixCounts.get(`${hash.slice(0, 2)}|${position}|${nodeId}`) ?? 0 + ), + corridorPriorCount: (position, nodeId) => { + let best = 0; + for (const member of group.members) { + if (position >= member.hops.length || !member.packet.src_node_id) continue; + best = Math.max(best, champion?.corridorCounts.get( + `${member.packet.src_node_id}|${member.observerId}|${position}|${nodeId}`, + ) ?? 0); + } + return best; + }, + observerDistance: (candidate) => { + if (!champion) return 0; + let nearest = Infinity; + for (const member of group.members) { + if (hasCoords(member.rx)) nearest = Math.min(nearest, distanceKm(candidate, { lat: member.rx.lat!, lon: member.rx.lon! })); + } + return Number.isFinite(nearest) ? -nearest / OBSERVER_DISTANCE_DECAY_KM : 0; + }, + corridorInterpolation: (position, _pathLength, candidate) => { + if (!champion || !sourceNode) return 0; + let nearest = Infinity; + for (const member of group.members) { + if (position >= member.hops.length || !hasCoords(member.rx)) continue; + const fraction = member.hops.length <= 1 ? 0.5 : position / (member.hops.length - 1); + nearest = Math.min(nearest, distanceKm(candidate, { + lat: sourceNode.lat! + (member.rx.lat! - sourceNode.lat!) * fraction, + lon: sourceNode.lon! + (member.rx.lon! - sourceNode.lon!) * fraction, + })); + } + return Number.isFinite(nearest) ? -nearest / CORRIDOR_INTERPOLATION_KM : 0; + }, mlPrefixScore: () => 0, directedEdgeScore: (fromNodeId, toNodeId) => regionalMaximum(regions, (region) => ( context.learningModel.edgeScores.get(`${region}|${bucket}|${fromNodeId}|${toNodeId}`) ?? context.learningModel.edgeScores.get(`${region}|-1|${fromNodeId}|${toNodeId}`) )), - transitionProbability: (fromNodeId, toNodeId) => regionalMaximum(regions, (region) => ( + transitionProbability: (fromNodeId, toNodeId) => champion + ? champion.transitionCounts.get(`${fromNodeId}|${toNodeId}`) ?? 0 + : regionalMaximum(regions, (region) => ( context.learningModel.transitionProbabilities.get(`${region}|${fromNodeId}|${toNodeId}`) )), + positionConditionalTransition: (position, fromNodeId, toNodeId) => ( + champion?.positionTransitionCounts.get(`${position}|${fromNodeId}|${toNodeId}`) ?? 0 + ), + isItmViable: (fromNodeId, toNodeId) => champion != null + && context.linkMetrics.get(linkKey(fromNodeId, toNodeId))?.itm_viable === true, motifProbability: (fromNodeId, toNodeId) => regionalMaximum(regions, (region) => ( context.learningModel.motifProbabilities.get(`${region}|${bucket}|2|${fromNodeId}>${toNodeId}`) ?? context.learningModel.motifProbabilities.get(`${region}|-1|2|${fromNodeId}>${toNodeId}`) @@ -303,7 +374,11 @@ export function decodeBetaCanonicalGroup( antennaHeightM: pathingConfig.defaultAntennaHeightM, }); return { - possible: physics.possible, + // Radio observations identify an actual edge. Terrain and current + // coordinates are estimates and cannot disprove that identity. + possible: confirmedEdges.has(`${from.nodeId}|${to.nodeId}`) + || (champion?.verifiedEdgeCounts.get(`${from.nodeId}|${to.nodeId}`) ?? 0) >= 2 + || physics.possible, score: pathingConfig.physicsSoftMarginWeight * physics.softMargin, }; }, @@ -311,6 +386,19 @@ export function decodeBetaCanonicalGroup( const decoded = decodePathWithScore(canonicalHashes, evidence); const hops = decoded.hops; + // Identity in a collision-free multibyte observation is direct evidence, + // even when the node has since moved or has no usable map coordinates. + // Preserve those IDs if missing geometry prevents a complete trellis solve. + for (const [position, nodeId] of confirmed) { + const node = context.nodesById.get(nodeId); + if (!node) continue; + hops.set(position, { + hash: canonicalHashes[position]!, nodeId, name: node.name, + lat: hasCoords(node) ? node.lat : null, + lon: hasCoords(node) ? node.lon : null, + margin: Infinity, ambiguous: false, + }); + } const hopConfidences = new Map(); for (const [position, hop] of hops) hopConfidences.set(position, marginConfidence(hop)); return { @@ -326,6 +414,7 @@ export function decodeBetaCanonicalGroup( function hardHopPossible(from: MeshNode, to: MeshNode, context: BetaResolveContext): boolean { if (!hasCoords(from) || !hasCoords(to)) return false; + if ((context.learningModel.champion?.verifiedEdgeCounts.get(`${from.node_id}|${to.node_id}`) ?? 0) >= 2) return true; return evaluatePhysicalHop( { lat: from.lat!, lon: from.lon!, elevationM: from.elevation_m }, { lat: to.lat!, lon: to.lon!, elevationM: to.elevation_m }, @@ -378,27 +467,30 @@ function materializeHeldDecode( group: BetaCanonicalGroup, context: BetaResolveContext, nodeIds: readonly string[], + evidence: BetaSharedDecode, ): BetaSharedDecode { const canonicalHashes = group.canonicalHashes.map(normalizePathHash).filter(Boolean); const hops = new Map(); const hopConfidences = new Map(); for (let position = 0; position < canonicalHashes.length; position++) { const node = context.nodesById.get(nodeIds[position]!); + const current = evidence.hops.get(position); + const unchanged = current?.nodeId === node?.node_id; hops.set(position, { hash: canonicalHashes[position]!, nodeId: node?.node_id ?? null, name: node?.name ?? null, lat: node?.lat ?? null, lon: node?.lon ?? null, - margin: Infinity, - ambiguous: false, + margin: unchanged ? current!.margin : 0, + ambiguous: !unchanged || current!.ambiguous, }); - hopConfidences.set(position, node ? 1 : 0); + hopConfidences.set(position, unchanged ? evidence.hopConfidences.get(position) ?? 0 : 0); } - return { canonicalHashes, hops, hopConfidences, score: 0 }; + return { canonicalHashes, hops, hopConfidences, score: evidence.baselineScore }; } -/** Hold a physical path and refine only the two observer-nearest positions. */ +/** Reconsider the whole chain when later observers supply better evidence. */ export function decodeBetaCanonicalGroupWithHeldPath( group: BetaCanonicalGroup, context: BetaResolveContext, @@ -415,35 +507,18 @@ export function decodeBetaCanonicalGroupWithHeldPath( }; } - let current = [...heldNodeIds]; - let refined = false; - const lastPosition = current.length - 1; - for (let position = lastPosition; position >= Math.max(0, lastPosition - 1); position--) { - const fixed = new Map(); - current.forEach((nodeId, index) => { if (index !== position) fixed.set(index, nodeId); }); - const candidate = decodeBetaCanonicalGroup( - group, - context, - stickyMap, - stickyAgeFraction, - fixed, - current, - ); - const candidateHop = candidate.hops.get(position); - if (candidateHop?.nodeId - && candidateHop.nodeId !== current[position] - && Number.isFinite(candidate.score) - && Number.isFinite(candidate.baselineScore) - && candidate.score! >= candidate.baselineScore! + HELD_PATH_REFINEMENT_MARGIN) { - current[position] = candidateHop.nodeId; - refined = true; - } + const candidate = decodeBetaCanonicalGroup(group, context, stickyMap, stickyAgeFraction, undefined, heldNodeIds); + const changed = heldNodeIds.some((nodeId, position) => candidate.hops.get(position)?.nodeId !== nodeId); + if (changed && Number.isFinite(candidate.score) + && (!Number.isFinite(candidate.baselineScore) + || candidate.score! >= candidate.baselineScore! + HELD_PATH_REFINEMENT_MARGIN)) { + return { ...candidate, held: true, physical: true, refined: true }; } return { - ...materializeHeldDecode(group, context, current), + ...materializeHeldDecode(group, context, heldNodeIds, candidate), held: true, physical: true, - refined, + refined: false, }; } @@ -454,11 +529,10 @@ type PositionedPoint = { receiver: boolean; }; -function shouldPrependSource(entry: BetaObserverEntry, source: MeshNode | null): source is MeshNode { - if (!hasCoords(source)) return false; - if (entry.packet.packet_type === 4) return true; - const firstHash = entry.hops[0]; - return Boolean(firstHash && nodePathHash(source.node_id, firstHash) !== normalizePathHash(firstHash)); +function shouldPrependSource(source: MeshNode | null, decoded: BetaSharedDecode): source is MeshNode { + // A one-byte prefix match is not an identity match. Only omit the source + // when the first resolved relay already represents that exact node. + return hasCoords(source) && decoded.hops.get(0)?.nodeId !== source.node_id; } /** Project one observer's DTO path from its group's canonical decode without inventing fallback edges. */ @@ -469,7 +543,7 @@ export function projectCanonicalPathForObserver( source: MeshNode | null, ): BetaPathProjection { const points: PositionedPoint[] = []; - if (shouldPrependSource(entry, source)) { + if (shouldPrependSource(source, decoded)) { points.push({ coordinate: [source.lat!, source.lon!], nodeId: source.node_id, diff --git a/backend/src/path-beta/types.ts b/backend/src/path-beta/types.ts index 0d0320a..3989e01 100644 --- a/backend/src/path-beta/types.ts +++ b/backend/src/path-beta/types.ts @@ -1,3 +1,5 @@ +import type { ChampionPriors } from './championPriors.js'; + export type MeshNode = { node_id: string; name: string | null; @@ -21,6 +23,7 @@ export type LinkMetrics = { }; export type PathLearningModel = { + champion?: ChampionPriors; prefixProbabilities: Map; transitionProbabilities: Map; edgeScores: Map; @@ -32,6 +35,8 @@ export type PathLearningModel = { export type PathPacket = { + direction?: string | null; + route_type?: number | null; packet_hash: string; rx_node_id: string | null; src_node_id: string | null; diff --git a/backend/src/path-beta/workerPool.test.ts b/backend/src/path-beta/workerPool.test.ts index 230b24d..6bc77a8 100644 --- a/backend/src/path-beta/workerPool.test.ts +++ b/backend/src/path-beta/workerPool.test.ts @@ -29,23 +29,13 @@ test('worker pool bounds queued interactive work without losing accepted jobs', } }); -test('worker pool timeout includes queue and execution time and replaces the worker', { timeout: 10_000 }, async (t) => { - // Advance the pool deadline deterministically; replacement thread startup - // is not a 750ms performance assertion on a shared test host. - t.mock.timers.enable({ apis: ['setTimeout'] }); +test('worker pool timeout includes queue and execution time and replaces the worker', async () => { const pool = new WorkerPool(fixtureUrl, 1, 1, 1, 750); try { - const active = pool.run({ value: 'late', delayMs: 1_500 }); - const queued = pool.run({ value: 'queued', delayMs: 1 }); - const expired = Promise.all([ - assert.rejects(active, WorkerPoolTimeoutError), - assert.rejects(queued, WorkerPoolTimeoutError), - ]); - t.mock.timers.tick(749); - assert.deepEqual(pool.snapshot(), { active: 1, interactiveQueued: 1, backgroundQueued: 0 }); - t.mock.timers.tick(1); - await expired; - assert.deepEqual(pool.snapshot(), { active: 0, interactiveQueued: 0, backgroundQueued: 0 }); + await assert.rejects( + pool.run({ value: 'late', delayMs: 1_500 }), + WorkerPoolTimeoutError, + ); assert.equal(await pool.run({ value: 'recovered', delayMs: 1 }), 'recovered'); } finally { await pool.close(); diff --git a/backend/src/path-core/decoder.test.ts b/backend/src/path-core/decoder.test.ts index fba4a29..e79ccb1 100644 --- a/backend/src/path-core/decoder.test.ts +++ b/backend/src/path-core/decoder.test.ts @@ -5,6 +5,8 @@ import { decodePath, evaluatePhysicalHop, indexCandidatesByHash, + decodePathWithScore, + scoreFixedPath, type CandidateNode, type PathDecoderEvidence, } from './decoder.js'; @@ -25,6 +27,36 @@ const node = (nodeId: string, lat: number, lon: number): CandidateNode => ({ lon, }); +test('indexes every matching width, de-duplicates repeated hashes and preserves elevation', () => { + const indexed = indexCandidatesByHash(['AA', 'AABBCC', 'AA'], [ + { nodeId: 'AABBCC00', name: null, lat: 51, lon: 0, elevationM: 456 }, + ], null); + assert.equal(indexed.get('AA')?.length, 1); + assert.equal(indexed.get('AABBCC')?.[0]?.elevationM, 456); +}); + +test('direct radio evidence can override a geographic cutoff without bypassing other gates', () => { + const decoded = decodePath(['AA', 'BB'], evidence(new Map([ + ['AA', [node('A', 51, 1)]], ['BB', [node('B', 53, 1), node('C', 54, 1)]], + ]), { maxHopKm: 100, physicalTransition: (from, to) => ({ possible: from.nodeId === 'A' && to.nodeId === 'B', score: 0 }) })); + assert.deepEqual([...decoded.values()].map((hop) => hop.nodeId), ['A', 'B']); +}); + +test('cached transitions preserve the complete chain score and score each edge once', () => { + const counts = new Map(); + const ev = evidence(new Map([ + ['AA', [node('A', 51, 1), node('B', 51, 1)]], + ['BB', [node('C', 51.1, 1), node('D', 51.1, 1)]], + ]), { directedEdgeScore: (a, b) => { + const key = `${a}|${b}`; + counts.set(key, (counts.get(key) ?? 0) + 1); + return 0; + } }); + const result = decodePathWithScore(['AA', 'BB'], ev); + assert.deepEqual([...counts.values()], [1, 1, 1, 1]); + assert.equal(result.score, scoreFixedPath(['AA', 'BB'], [...result.hops.values()].map((h) => h.nodeId!), ev)); +}); + function evidence( candidatesByHash: ReadonlyMap, overrides: Partial = {}, diff --git a/backend/src/path-core/decoder.ts b/backend/src/path-core/decoder.ts index 937880f..b6544b7 100644 --- a/backend/src/path-core/decoder.ts +++ b/backend/src/path-core/decoder.ts @@ -74,7 +74,7 @@ export type PathDecoderEvidence = { unresolvedBaseline: number; ambiguityDelta: number; maxColumnCandidates: number; - candidatesByHash: ReadonlyMap; + candidatesByHash: ReadonlyMap; directAnchors: (position: number, hash: string) => readonly ObserverAnchor[]; sourceAnchor?: ObserverAnchor; terminalAnchors?: readonly ObserverAnchor[]; @@ -88,6 +88,10 @@ export type PathDecoderEvidence = { ) => { possible: boolean; score: number }; /** Endpoint anchors use only hard gates plus the bounded physics tiebreaker. */ endpointTransitionsGateOnly?: boolean; + /** Include directed evidence for each directly receiving observer. */ + scoreDirectAnchorTransitions?: boolean; + /** Missing coordinates require observed directed edges on both sides. */ + requireUnpositionedTransitionEvidence?: boolean; prefixProbability: (nodeId: string, hash: string, prevHash: string | null) => number; positionPrefixCount?: (position: number, nodeId: string, hash: string) => number; corridorPriorCount?: (position: number, nodeId: string) => number; @@ -213,24 +217,34 @@ export function indexCandidatesByHash( hashes: readonly string[], candidates: readonly CandidateNodeEvidence[], bounds: GeographicBounds | null, -): Map { - const result = new Map(); + includeUnpositioned = false, +): Map { + const result = new Map(); + const hashesByByte = new Map(); + for (const hash of new Set(hashes)) { + const byte = hash.slice(0, 2).toUpperCase(); + const values = hashesByByte.get(byte) ?? []; + values.push(hash); + hashesByByte.set(byte, values); + } for (const candidate of candidates) { - if (candidate.lat == null || candidate.lon == null - || candidate.lat === 0 || candidate.lon === 0) continue; - const point = { lat: candidate.lat, lon: candidate.lon }; - if (bounds && !isInBounds(point, bounds)) continue; + const positioned = !(candidate.lat == null || candidate.lon == null + || !Number.isFinite(candidate.lat) || !Number.isFinite(candidate.lon) + || (candidate.lat === 0 && candidate.lon === 0)); + if (!positioned && !includeUnpositioned) continue; + const point = { lat: candidate.lat!, lon: candidate.lon! }; + if (positioned && bounds && !isInBounds(point, bounds)) continue; const upperNodeId = candidate.nodeId.toUpperCase(); - for (const hash of hashes) { + for (const hash of hashesByByte.get(upperNodeId.slice(0, 2)) ?? []) { if (!upperNodeId.startsWith(hash)) continue; if (!result.has(hash)) result.set(hash, []); result.get(hash)!.push({ nodeId: candidate.nodeId, name: candidate.name, - lat: candidate.lat, - lon: candidate.lon, + lat: positioned ? candidate.lat : null, + lon: positioned ? candidate.lon : null, + elevationM: candidate.elevationM, }); - break; } } return result; @@ -249,9 +263,16 @@ function emissionScore( const positioned = candidate.lat != null && candidate.lon != null; let directPhysicsScore = 0; - if (anchors.length > 0) { - if (!positioned) return -Infinity; - if (minDistanceToSet({ lat: candidate.lat!, lon: candidate.lon! }, anchors) > evidence.maxHopKm) { + if (anchors.length > 0 && !positioned) { + if (!evidence.requireUnpositionedTransitionEvidence || !evidence.scoreDirectAnchorTransitions) return -Infinity; + directPhysicsScore = Math.max(...anchors.map((anchor) => endpointTransitionScore( + candidate, endpointCandidate(anchor), evidence, position + 1, + ))); + if (!Number.isFinite(directPhysicsScore)) return -Infinity; + } + if (anchors.length > 0 && positioned) { + if (!evidence.physicalTransition + && minDistanceToSet({ lat: candidate.lat!, lon: candidate.lon! }, anchors) > evidence.maxHopKm) { return -Infinity; } if (evidence.physicalTransition) { @@ -262,7 +283,9 @@ function emissionScore( endpointCandidate(anchor) as CandidateNode, distanceKm(candidate as CandidateNode, anchor), ); - if (physics.possible) bestPhysics = Math.max(bestPhysics, physics.score); + if (physics.possible) bestPhysics = Math.max(bestPhysics, evidence.scoreDirectAnchorTransitions + ? endpointTransitionScore(candidate, endpointCandidate(anchor), evidence, position + 1) + : physics.score); } if (!isFinite(bestPhysics)) return -Infinity; directPhysicsScore = bestPhysics; @@ -296,7 +319,7 @@ function emissionScore( * Math.log1p(Math.max(0, evidence.corridorPriorCount(position, candidate.nodeId))); } - if (evidence.weights.corridorInterpolation !== 0 && evidence.corridorInterpolation) { + if (positioned && evidence.weights.corridorInterpolation !== 0 && evidence.corridorInterpolation) { score += evidence.weights.corridorInterpolation * evidence.corridorInterpolation(position, pathLength, candidate as CandidateNode); } @@ -310,13 +333,16 @@ function transitionScore( evidence: PathDecoderEvidence, ): number { if (previous.nodeId === null || current.nodeId === null) return 0; + if (evidence.requireUnpositionedTransitionEvidence + && (previous.lat == null || previous.lon == null || current.lat == null || current.lon == null) + && evidence.transitionProbability(previous.nodeId, current.nodeId) <= 0) return -Infinity; let score = 0; if (previous.lat != null && previous.lon != null && current.lat != null && current.lon != null) { const distance = distanceKm( { lat: previous.lat, lon: previous.lon }, { lat: current.lat, lon: current.lon }, ); - if (distance > evidence.maxHopKm) return -Infinity; + if (!evidence.physicalTransition && distance > evidence.maxHopKm) return -Infinity; if (evidence.physicalTransition) { const physics = evidence.physicalTransition( previous as CandidateNode, @@ -355,11 +381,12 @@ function endpointTransitionScore( from: DecoderCandidate, to: DecoderCandidate, evidence: PathDecoderEvidence, + position = 0, ): number { - if (!evidence.endpointTransitionsGateOnly) return transitionScore(0, from, to, evidence); + if (!evidence.endpointTransitionsGateOnly) return transitionScore(position, from, to, evidence); if (from.lat == null || from.lon == null || to.lat == null || to.lon == null) return -Infinity; const distance = distanceKm(from as CandidateNode, to as CandidateNode); - if (distance > evidence.maxHopKm) return -Infinity; + if (!evidence.physicalTransition && distance > evidence.maxHopKm) return -Infinity; if (!evidence.physicalTransition) return 0; const physics = evidence.physicalTransition(from as CandidateNode, to as CandidateNode, distance); return physics.possible ? physics.score : -Infinity; @@ -493,6 +520,19 @@ export function decodePathWithScore( ))); } + // Forward and backward passes use identical transitions. Cache each edge + // once: terrain checks, geographic distances and learned lookups dominate + // the cost of decoding densely colliding one-byte columns. + const transitions: number[][][] = columns.map((column, position) => position === 0 ? [] + : columns[position - 1]!.map(() => new Array(column.length).fill(NaN))); + const edgeScore = (position: number, previous: number, current: number): number => { + const row = transitions[position]![previous]!; + if (Number.isNaN(row[current])) row[current] = transitionScore( + position, columns[position - 1]![previous]!, columns[position]![current]!, evidence, + ); + return row[current]!; + }; + // Best prefix score ending at columns[position][candidate]. const forward: number[][] = columns.map((column) => new Array(column.length).fill(-Infinity)); const previousCandidate: number[][] = columns.map((column) => new Array(column.length).fill(-1)); @@ -512,12 +552,7 @@ export function decodePathWithScore( let bestPrevious = -1; for (let previous = 0; previous < columns[position - 1]!.length; previous++) { if (!isFinite(forward[position - 1]![previous]!)) continue; - const transition = transitionScore( - position, - columns[position - 1]![previous]!, - columns[position]![current]!, - evidence, - ); + const transition = edgeScore(position, previous, current); if (!isFinite(transition)) continue; const total = forward[position - 1]![previous]! + transition; if (total > best) { @@ -547,12 +582,7 @@ export function decodePathWithScore( for (let next = 0; next < columns[position + 1]!.length; next++) { const nextEmission = emissions[position + 1]![next]!; if (!isFinite(nextEmission) || !isFinite(backward[position + 1]![next]!)) continue; - const transition = transitionScore( - position + 1, - columns[position]![current]!, - columns[position + 1]![next]!, - evidence, - ); + const transition = edgeScore(position + 1, current, next); if (!isFinite(transition)) continue; const total = transition + nextEmission + backward[position + 1]![next]!; if (total > best) best = total; diff --git a/backend/src/path-lazy/evaluate.ts b/backend/src/path-lazy/evaluate.ts index 08e5c3b..68dbc40 100644 --- a/backend/src/path-lazy/evaluate.ts +++ b/backend/src/path-lazy/evaluate.ts @@ -1,5 +1,7 @@ /** - * Gold-packet accuracy harness for all path-resolution jobs. + * Historical path diagnostics, retained for debugging old jobs. + * This harness does not enforce a held-out split and filters long routes. + * Use tools/pathAccuracy.ts for reproducible accuracy comparisons. * * Ground truth: multibyte packets (path_hash_size_bytes > 1) whose every relay * hash uniquely resolves to exactly one positioned node in the network scope. @@ -585,6 +587,7 @@ function printStratification( } async function main() { + console.warn('[evaluate] Historical diagnostics only; use npm run path:accuracy for held-out accuracy measurements.'); const network = process.argv[2] ?? 'ukmesh'; const sampleSize = Number(process.argv[3] ?? 400); const selectedJobs = selectedJobNames(); diff --git a/backend/src/path-lazy/lazyResolver.ts b/backend/src/path-lazy/lazyResolver.ts index 8a17b09..8f75912 100644 --- a/backend/src/path-lazy/lazyResolver.ts +++ b/backend/src/path-lazy/lazyResolver.ts @@ -1,5 +1,7 @@ /** - * Lazy path resolver — called after propagation has settled. + * Previous lazy resolver, retained for read-only accuracy comparisons. + * Production packet details use resolveLazyPathForPacketHash and the same + * evidence provider as the live overlays; this implementation is not routed. * * Uses path_hashes embedded in packets by relaying nodes, combined with * geographic anchoring from MQTT observer positions, to reconstruct the diff --git a/backend/src/path-learning/rebuild.test.ts b/backend/src/path-learning/rebuild.test.ts index 9462aac..703ae95 100644 --- a/backend/src/path-learning/rebuild.test.ts +++ b/backend/src/path-learning/rebuild.test.ts @@ -1,7 +1,8 @@ import assert from 'node:assert/strict'; import { readFile } from 'node:fs/promises'; import test from 'node:test'; -import { pathLearningInputHash } from './rebuild.js'; +import { pathLearningInputHash, resolvePathForPacket } from './rebuild.js'; +import { buildNodePathHashIndex } from '../path-hash/utils.js'; const baseInput = { modelNetwork: 'ukmesh', @@ -35,6 +36,17 @@ test('path-learning input hash fences bounds, source rows, and privacy generatio }), baseline); }); +test('training keeps original positions across missing relays and preserves verified loops', () => { + const a = { node_id: 'AA0011', lat: 51, lon: 1, elevation_m: null, iata: null }; + const b = { ...a, node_id: 'BB0022' }; + const rx = { ...a, node_id: 'CC0033' }; + const index = buildNodePathHashIndex([a, b, rx]); + const gap = resolvePathForPacket(['AA0011', 'FFFFFF', 'BB0022'], undefined, rx, index, new Set()); + assert.deepEqual(gap.map((hop) => hop.position), [0, 2]); + const loop = resolvePathForPacket(['AA0011', 'BB0022', 'AA0011'], undefined, rx, index, new Set()); + assert.deepEqual(loop.map((hop) => hop.node.node_id), ['AA0011', 'BB0022', 'AA0011']); +}); + test('rebuild has no full-table-per-network replacement path and retains all fallback', async () => { const source = await readFile(new URL('./rebuild.ts', import.meta.url), 'utf8'); assert.doesNotMatch(source, /DELETE FROM path_[a-z_]+_priors WHERE network = \$1/); diff --git a/backend/src/path-learning/rebuild.ts b/backend/src/path-learning/rebuild.ts index b4cb80f..85863ad 100644 --- a/backend/src/path-learning/rebuild.ts +++ b/backend/src/path-learning/rebuild.ts @@ -32,6 +32,7 @@ type LearningNode = { }; type LearningPacket = { + direction?: string | null; time: string; rx_node_id: string; src_node_id: string | null; @@ -47,6 +48,7 @@ type LearningLink = { }; type ResolvedHop = { + position: number; prefix: string; node: LearningNode; isUnique: boolean; @@ -91,7 +93,7 @@ const MAX_CORRIDOR_CHOICES_PER_GROUP = 16; const MAX_POSITION_TRANSITIONS_PER_GROUP = 16; const HOUR_BUCKET_SIZE = 6; const PREFIX_AMBIGUITY_RADIUS_KM = 45; -const PATH_LEARNING_ALGORITHM_VERSION = 'path-learning-v4-delta'; +const PATH_LEARNING_ALGORITHM_VERSION = 'path-learning-v5-contiguous'; type PrefixPriorWriteRow = { prefix: string; @@ -218,7 +220,7 @@ function localPrefixAmbiguityPenalty( return clamp(raw * 0.12, 0, 0.24); } -function resolvePathForPacket( +export function resolvePathForPacket( pathHashes: string[], srcNode: LearningNode | undefined, rxNode: LearningNode, @@ -232,7 +234,7 @@ function resolvePathForPacket( for (let i = pathHashes.length - 1; i >= 0; i--) { const prefix = normalizePathHash(pathHashes[i]); const allCandidates = getNodesForPathHash(pathHashIndex, prefix); - const candidates = allCandidates.filter((n) => !visited.has(n.node_id)); + const candidates = allCandidates.filter((n) => allCandidates.length === 1 || !visited.has(n.node_id)); if (candidates.length === 0) continue; let best: LearningNode | null = null; @@ -251,7 +253,7 @@ function resolvePathForPacket( } if (best) { - resolved.unshift({ prefix, node: best, isUnique: allCandidates.length === 1 }); + resolved.unshift({ position: i, prefix, node: best, isUnique: allCandidates.length === 1 }); visited.add(best.node_id); prev = best; } @@ -500,6 +502,7 @@ export async function rebuildPathLearningModels(): Promise { AND path_hashes IS NOT NULL AND cardinality(path_hashes) > 0 AND path_hash_size_bytes >= 2 + AND route_type IN (0, 1) AND mod(hashtext(packet_hash)::bigint + 2147483648, 100) < ${TRAINING_SPLIT_PERCENT} ORDER BY network LIMIT 33`, @@ -592,12 +595,13 @@ async function rebuildNetworkUnderLease( const packetsResult = await analyticsQuery( `SELECT DISTINCT ON (packet_hash, rx_node_id, src_node_id, path_hashes) - time, rx_node_id, src_node_id, path_hashes + time, rx_node_id, src_node_id, path_hashes, payload->>'direction' AS direction FROM packets WHERE rx_node_id IS NOT NULL AND path_hashes IS NOT NULL AND cardinality(path_hashes) > 0 AND path_hash_size_bytes >= 2 + AND route_type IN (0, 1) AND time > $${sourceNetwork ? 2 : 1}::timestamptz AND time <= $${sourceNetwork ? 3 : 2}::timestamptz AND mod(hashtext(packet_hash)::bigint + 2147483648, 100) < ${TRAINING_SPLIT_PERCENT} @@ -708,7 +712,7 @@ async function rebuildNetworkUnderLease( if (hashes.length === 0) continue; const rx = nodesById.get(packet.rx_node_id); if (!rx) continue; - if (hashes.length > 0 && rx.node_id.toUpperCase().startsWith(hashes[hashes.length - 1]!)) { + if (packet.direction !== 'rx' && hashes.length > 0 && rx.node_id.toUpperCase().startsWith(hashes[hashes.length - 1]!)) { hashes = hashes.slice(0, -1); } if (hashes.length === 0) continue; @@ -722,9 +726,13 @@ async function rebuildNetworkUnderLease( const ts = new Date(packet.time); const bucket = hourBucket(ts); - const fullNodes: LearningPathNode[] = [ + const resolvedByPosition = new Map(resolved.map((hop) => [hop.position, hop])); + const fullNodes: (LearningPathNode | null)[] = [ ...(src ? [{ node: src, isVerified: true }] : []), - ...resolved.map((hop) => ({ node: hop.node, isVerified: hop.isUnique })), + ...hashes.map((_, position) => { + const hop = resolvedByPosition.get(position); + return hop ? { node: hop.node, isVerified: hop.isUnique } : null; + }), { node: rx, isVerified: true }, ]; let successfulEdges = 0; @@ -732,7 +740,7 @@ async function rebuildNetworkUnderLease( for (let i = 0; i < fullNodes.length - 1; i++) { const from = fullNodes[i]!; const to = fullNodes[i + 1]!; - if (!from.isVerified || !to.isVerified) continue; + if (!from?.isVerified || !to?.isVerified) continue; totalEdges++; if (confirmedLinks.has(linkKey(from.node.node_id, to.node.node_id))) successfulEdges++; } @@ -746,7 +754,7 @@ async function rebuildNetworkUnderLease( for (let i = 0; i < resolved.length; i++) { const hop = resolved[i]!; if (!hop.isUnique) continue; - const prevPrefix = i > 0 ? resolved[i - 1]!.prefix : ''; + const prevPrefix = hop.position > 0 ? hashes[hop.position - 1]! : ''; const prefixGroup = `${hop.prefix}|${region}|${prevPrefix}`; const choiceKey = `${prefixGroup}|${hop.node.node_id}`; increment(prefixChoiceCounts, choiceKey); @@ -783,7 +791,7 @@ async function rebuildNetworkUnderLease( for (let i = 0; i < fullNodes.length - 1; i++) { const from = fullNodes[i]!; const to = fullNodes[i + 1]!; - if (!from.isVerified || !to.isVerified) continue; + if (!from?.isVerified || !to?.isVerified) continue; const group = `${from.node.node_id}|${region}`; const edgeKey = `${group}|${to.node.node_id}`; increment(transitionCounts, edgeKey); @@ -802,7 +810,7 @@ async function rebuildNetworkUnderLease( if (i < fullNodes.length - 2) { const next = fullNodes[i + 2]!; - if (!next.isVerified) continue; + if (!next?.isVerified) continue; const motif3Group = `${region}|${bucket}|${from.node.node_id}>${to.node.node_id}`; const motif3Key = `${motif3Group}|${from.node.node_id}>${to.node.node_id}>${next.node.node_id}`; increment(motif3Counts, motif3Key); diff --git a/backend/src/path-shared/scoring.ts b/backend/src/path-shared/scoring.ts index cb32407..7c1893c 100644 --- a/backend/src/path-shared/scoring.ts +++ b/backend/src/path-shared/scoring.ts @@ -1,16 +1,14 @@ /** * Shared scoring weights and prior key-formats for the path resolvers. * - * The shared decoder scores candidate hops with prefix/transition/edge/motif - * priors, ML scores, node_links, and hop distance. The lazy resolver uses it - * today; the beta resolvers will adopt it in later extraction phases. + * All production path surfaces use the shared decoder and count-based + * multibyte priors, observer anchors, geographic and terrain evidence. * * This module is the single source of truth for: * - the prior key formats (must match how `path-learning/rebuild.ts` stores them); * - the relative scoring weights used by the Viterbi decoder. * - * `path-beta/resolver.ts` predates this module and still has its own inline - * weights; later phases will converge it here. New code should import from here. + * Evidence providers should import this table rather than duplicating weights. */ /** Generous but realistic upper bound for a single LoRa hop, in km. */ diff --git a/backend/src/platform/config/pathing.ts b/backend/src/platform/config/pathing.ts index 4ba3f4d..a72ae12 100644 --- a/backend/src/platform/config/pathing.ts +++ b/backend/src/platform/config/pathing.ts @@ -1,5 +1,5 @@ export const pathingConfig = { - maxBetaHops: 25, + maxBetaHops: 64, purpleThreshold: 0.45, earthEffectiveRadiusM: 6_371_000 / (1 - 0.25), defaultAntennaHeightM: 15, diff --git a/backend/src/regions/memberShare.test.ts b/backend/src/regions/memberShare.test.ts new file mode 100644 index 0000000..4a21da9 --- /dev/null +++ b/backend/src/regions/memberShare.test.ts @@ -0,0 +1,31 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { MEMBER_MIN_SHARE_PCT, meetsMemberShare } from './memberShare.js'; + +test('MEMBER_MIN_SHARE_PCT is the calibrated 10% floor', () => { + assert.equal(MEMBER_MIN_SHARE_PCT, 10); +}); + +test('meetsMemberShare accepts the floor exactly and anything above it', () => { + assert.equal(meetsMemberShare(10, 100), true); + assert.equal(meetsMemberShare(11, 100), true); + assert.equal(meetsMemberShare(1, 10), true); + // yorkshire-scale: 9599 packets, floor lands at 960 + assert.equal(meetsMemberShare(960, 9599), true); + assert.equal(meetsMemberShare(7065, 9599), true); +}); + +test('meetsMemberShare rejects token relays below the floor', () => { + assert.equal(meetsMemberShare(9, 100), false); + assert.equal(meetsMemberShare(959, 9599), false); + assert.equal(meetsMemberShare(1, 1000), false); + assert.equal(meetsMemberShare(0, 100), false); +}); + +test('meetsMemberShare is defensive about degenerate inputs', () => { + assert.equal(meetsMemberShare(5, 0), false); + assert.equal(meetsMemberShare(0, 0), false); + assert.equal(meetsMemberShare(Number.NaN, 100), false); + assert.equal(meetsMemberShare(5, Number.NaN), false); + assert.equal(meetsMemberShare(5, Number.POSITIVE_INFINITY), false); +}); diff --git a/backend/src/regions/memberShare.ts b/backend/src/regions/memberShare.ts new file mode 100644 index 0000000..3d70d90 --- /dev/null +++ b/backend/src/regions/memberShare.ts @@ -0,0 +1,23 @@ +/** + * Membership floor for the region-scope participation tables. + * + * A node only counts as a member of a region when it relayed at least + * MEMBER_MIN_SHARE_PCT percent of that region's scoped packets. Nodes that + * merely pick up the odd scoped packet (a handful out of thousands) are not + * part of a region's relay pattern: including them diluted member lists and + * stretched the map hulls. Calibrated against the 2026-10 live corpus: the + * 10% floor removed ~36% of member rows and cleaned 7 of 24 hulls without + * dropping any region's shape. + */ +export const MEMBER_MIN_SHARE_PCT = 10; + +/** + * True when `memberPackets` is at least MEMBER_MIN_SHARE_PCT% of + * `regionPacketsTotal`. Integer-safe (no float division) and defensive about + * degenerate inputs. + */ +export function meetsMemberShare(memberPackets: number, regionPacketsTotal: number): boolean { + if (!Number.isFinite(memberPackets) || !Number.isFinite(regionPacketsTotal)) return false; + if (regionPacketsTotal <= 0) return false; + return memberPackets * 100 >= regionPacketsTotal * MEMBER_MIN_SHARE_PCT; +} diff --git a/backend/src/regions/regionNames.ts b/backend/src/regions/regionNames.ts new file mode 100644 index 0000000..cbe4738 --- /dev/null +++ b/backend/src/regions/regionNames.ts @@ -0,0 +1,55 @@ +// Canonical probe list for MeshCore region-scope decoding on the UK mesh. +// +// Public hashtag regions derive their transport key from the region name alone +// (firmware TransportKeyStore::getAutoKeyFor() = SHA256("#name")[0:16]), so a +// packet's transport code can be attributed by testing candidate names — there +// is no key material involved and no reverse mapping. Names that never match +// simply stay unattributed; the rebuild tool logs the unknown share so the list +// can be extended as new regions appear. +// +// The list covers every region name observed on the ukmesh corpus plus the +// wider vocabulary used by neighbouring networks (CoreScope, ScotMesh). Order +// is normalised to sorted() at module load so probing is deterministic: if two +// names ever produced the same transport code the earlier name wins. + +const OBSERVED_AND_CANDIDATE_NAMES = ` +yorkshire northwest eng-ne eng-yh eng-yorkshire gb uk ren durham eng-tyneside-admin +wales-admin northumbria-admin eng-cum eng-oxf-admin sta-admin ioi eng-cam northwest-admin +north-west eng-nw-admin nw bristol bristol-admin eng-bristol bath avon wessex west-country +westcountry south-west southwest south-coast bournemouth southampton portsmouth brighton +plymouth exeter dorset-admin cornwall-admin devon-admin somerset-admin norwich norfolk-admin +suffolk-admin ipswich east-anglia cambridge peterborough lincoln lincolnshire-admin hull +grimsby scunthorpe doncaster sheffield leeds york bradford halifax huddersfield wakefield +barnsley rotherham chesterfield manchester merseyside liverpool lancashire-admin bolton +wigan preston blackpool carlisle chester crewe warrington stockport oldham rochdale burnley +blackburn teesside middlesbrough darlington sunderland gateshead newcastle tyneside +northumberland eng-tyne eng-tees eng-ncl eng-dur eng-nwk eng-nwest eng-northwest north-east +northeast northeast-admin eng-ne-admin yorkshire-admin selkirkshire cleveland west-yorkshire +south-yorkshire north-yorkshire east-yorkshire ceredigion south-east southeast +southeast-admin east-midlands west-midlands midlands the-midlands mids nw-england +eng-north-west north-west-england norwest wirral cheshire-admin stoke staffordshire-admin +shropshire-admin derbyshire-admin nottinghamshire-admin leicestershire-admin +worcestershire-admin warwickshire-admin hertfordshire-admin bedfordshire-admin +buckinghamshire-admin oxfordshire-admin berkshire-admin hampshire-admin surrey-admin +kent-admin sussex-admin essex-admin sw-admin +sco wls iom fif hig tay edi noc cen gla fal dun per inv sti ioi-admin europe global +`; + +export const REGION_NAMES: readonly string[] = [ + ...new Set(OBSERVED_AND_CANDIDATE_NAMES.split(/\s+/).filter(Boolean)), +].sort(); + +/** + * Build the effective probe list: canonical names plus any extras from a + * comma-separated env value (same convention as the ingest-side + * MESHCORE_REGION_NAMES probe). Extra names keep their case — region keys are + * case-sensitive — and the leading '#' is tolerated for convenience. + */ +export function regionProbeNamesWithEnv(envValue: string | undefined): string[] { + const extra = (envValue ?? '') + .split(',') + .map((value) => value.trim()) + .filter(Boolean) + .map((name) => (name.startsWith('#') ? name.slice(1) : name)); + return [...new Set([...REGION_NAMES, ...extra])].sort(); +} diff --git a/backend/src/regions/scopeDecode.test.ts b/backend/src/regions/scopeDecode.test.ts new file mode 100644 index 0000000..f31b24a --- /dev/null +++ b/backend/src/regions/scopeDecode.test.ts @@ -0,0 +1,74 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { MeshCoreDecoder } from '@michaelhart/meshcore-decoder'; +import { decodePacketCompat } from '../mqtt/decodePacket.js'; +import { REGION_NAMES } from './regionNames.js'; +import { decodeRegionForPacket, transportCode0Of } from './scopeDecode.js'; + +// Real route-type 0/3 frames captured from the ukmesh corpus. Each was +// verified against the live database region_scope column and the +// @michaelhart/meshcore-decoder library during the 2026-10 region-scope audit. +const SAMPLES: Array<{ region: string; raw: string }> = [ + { + region: 'eng-ne', + raw: '04313C000081F51858D28648D4184042345B3FD371B1E3B17B1D86B1F7', + }, + { + region: 'yorkshire', + raw: '14CD3D0000854D52E4A8AB2FBBC5B1BBDE0C8C197F805A2BC47E8F027BED00573888281E957B3DEE0496717D73274BD5877B39047BE57D9FE91E09E08345F9818829DDD1750C3D3CC44A708ECB7931CE6DAA2275E9E11F386D677348E375CD419A327D6AC77EE68267BC1FB08CC4732EEDB6EED385D9764022ADC98C5397DA100AFB4A8160A6DAABB59806C96E4C0D9CF0C66948404FF8FA', + }, + { + region: 'eng-yh', + raw: '08F5FE0000818C197F65378C55A3A9D3BE9347E808CA4D012626A27E4701E12EAE0210BCBD0F4C7EC378552F9A', + }, + { + region: 'gb', + raw: '141E380000416D11118EEA9A9C8AAE7EA6AB13A622CCA86A3BFD1CD8F98FC7DF8FF5FE3344DF7B230A6A0891362A348567669EE0A7A3F9229884F51D4A6A8EA833999DA6E5D36B16CDC1FA3C52D391D8BBECE77280C26A70A3C7804FAB3F343A1342EF8CA0C2423A10498CBE460D3C3503546D303E5102037FB87A3C57E4673B576FEAD716ED94756EE7F1', + }, +]; + +// Documented MeshCore public channel material, not an application credential. +const keyStore = MeshCoreDecoder.createKeyStore({ + channelSecrets: ['8b3387e9c5cdea6ac9e5edbaa115cd72'], +}); + +test('transportCode0Of reads the first little-endian uint16 and rejects malformed input', () => { + assert.equal(transportCode0Of('CD3D0000'), 0x3dcd); + assert.equal(transportCode0Of('01FF'), 0xff01); + assert.equal(transportCode0Of(''), null); + assert.equal(transportCode0Of('CD3'), null); + assert.equal(transportCode0Of('ZZZZ'), null); +}); + +test('decodeRegionForPacket attributes captured scoped packets to their region', () => { + for (const sample of SAMPLES) { + const compat = decodePacketCompat(sample.raw, keyStore); + assert.equal(compat.metadataValid, true, `${sample.region}: frame must decode cleanly`); + assert.ok(compat.transportCodes, `${sample.region}: transport codes must be present`); + const tc0 = transportCode0Of(compat.transportCodes); + assert.notEqual(tc0, null); + const decoded = compat.decoded; + assert.ok(decoded?.payload?.raw, `${sample.region}: payload raw must be present`); + const name = decodeRegionForPacket( + REGION_NAMES, + decoded.payloadType, + decoded.payload.raw, + tc0, + ); + assert.equal(name, sample.region); + } +}); + +test('decodeRegionForPacket returns null when no candidate matches or inputs are missing', () => { + const [first] = SAMPLES; + const compat = decodePacketCompat(first.raw, keyStore); + assert.ok(compat.decoded?.payload?.raw); + const tc0 = transportCode0Of(compat.transportCodes!); + assert.equal( + decodeRegionForPacket(['definitely-not-a-region'], compat.decoded.payloadType, compat.decoded.payload.raw, tc0), + null, + ); + assert.equal(decodeRegionForPacket(REGION_NAMES, null, compat.decoded.payload.raw, tc0), null); + assert.equal(decodeRegionForPacket(REGION_NAMES, compat.decoded.payloadType, null, tc0), null); + assert.equal(decodeRegionForPacket(REGION_NAMES, compat.decoded.payloadType, compat.decoded.payload.raw, null), null); +}); diff --git a/backend/src/regions/scopeDecode.ts b/backend/src/regions/scopeDecode.ts new file mode 100644 index 0000000..dfb1488 --- /dev/null +++ b/backend/src/regions/scopeDecode.ts @@ -0,0 +1,39 @@ +import { transportCodeMatchesRegion } from '@michaelhart/meshcore-decoder'; + +/** + * First transport code (uint16, little-endian) from a 4-byte transport-codes + * hex string, as extracted from a route-type 0/3 packet. Returns null when the + * hex is too short or malformed. + */ +export function transportCode0Of(transportCodesHex: string): number | null { + const hex = transportCodesHex.trim(); + if (!/^[0-9A-Fa-f]{4,}$/.test(hex)) return null; + const low = Number.parseInt(hex.slice(0, 2), 16); + const high = Number.parseInt(hex.slice(2, 4), 16); + return low | (high << 8); +} + +/** + * Attribute a decoded scoped packet (route 0/3, transport codes present) to one + * of the candidate public region names, or null when no candidate matches. + * + * The match is `HMAC-SHA256(SHA256("#name")[0:16], payloadType || payload)` + * truncated to a little-endian uint16, compared against the packet's first + * transport code — the same derivation the firmware uses when flooding within + * a region, executed per candidate name (see @michaelhart/meshcore-decoder + * region-transport, which mirrors TransportKeyStore::getAutoKeyFor()). + */ +export function decodeRegionForPacket( + probeNames: readonly string[], + payloadType: number | null | undefined, + payloadRawHex: string | null | undefined, + transportCode0: number | null, +): string | null { + if (transportCode0 == null || payloadType == null || !payloadRawHex) return null; + for (const name of probeNames) { + if (transportCodeMatchesRegion(name, payloadType, payloadRawHex, transportCode0)) { + return name; + } + } + return null; +} diff --git a/backend/src/repositories/networkAnalysis.test.ts b/backend/src/repositories/networkAnalysis.test.ts new file mode 100644 index 0000000..25de5c8 --- /dev/null +++ b/backend/src/repositories/networkAnalysis.test.ts @@ -0,0 +1,86 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import type { QueryResultRow } from 'pg'; +import { networkFilters } from '../api/utils/networkFilters.js'; +import { rfValidationRows, queryRfValidation, rfValidationPool, RF_VALIDATION_DB_TIMEOUT_MS } from './networkAnalysis.js'; + +test('RF validation scopes endpoints without joining both identity-node projections', async () => { + const statements: string[] = []; + const controller = new AbortController(); + const query = async ( + sql: string, + _params?: unknown[], + signal?: AbortSignal, + ): Promise<{ rows: T[] }> => { + assert.equal(signal, controller.signal); + statements.push(sql); + if (sql.includes('FROM node_identity_nodes')) { + return { rows: [ + { node_id: 'A', name: 'Alpha' }, + { node_id: 'B', name: 'Bravo' }, + { node_id: 'C', name: 'Charlie' }, + ] as unknown as T[] }; + } + return { rows: [ + { + node_a_id: 'A', node_b_id: 'OUTSIDE', observed_count: '99', + multibyte_observed_count: '5', itm_path_loss_db: 100, + itm_viable: false, force_viable: false, + last_observed: '2026-08-20 10:00:00+00', classification: 'observed_unexpected', + }, + { + node_a_id: 'A', node_b_id: 'B', observed_count: '20', + multibyte_observed_count: '2', itm_path_loss_db: 110, + itm_viable: false, force_viable: false, + last_observed: '2026-08-20 09:00:00+00', classification: 'observed_unexpected', + }, + { + node_a_id: 'B', node_b_id: 'C', observed_count: '10', + multibyte_observed_count: '1', itm_path_loss_db: 120, + itm_viable: true, force_viable: false, + last_observed: '2026-08-20 08:00:00+00', classification: 'match', + }, + ] as unknown as T[] }; + }; + + const result = await rfValidationRows(query, networkFilters('ukmesh'), 1, controller.signal); + assert.equal(statements.length, 2); + assert.doesNotMatch(statements[1]!, /JOIN\s+node_identity_nodes/i); + assert.deepEqual(result.rows, [{ + node_a_id: 'A', node_b_id: 'B', observed_count: '20', + multibyte_observed_count: '2', itm_path_loss_db: 110, + itm_viable: false, force_viable: false, + last_observed: '2026-08-20 09:00:00+00', classification: 'observed_unexpected', + name_a: 'Alpha', name_b: 'Bravo', + }]); +}); + +test('empty RF scope does not run a link scan', async () => { + let calls = 0; + assert.deepEqual(await rfValidationRows(async () => { calls += 1; return { rows: [] }; }, networkFilters('test'), 100), { rows: [] }); + assert.equal(calls, 1); +}); + +test('RF pool has an independent server timeout and read-only sessions', () => { + assert.equal(rfValidationPool.options.statement_timeout, RF_VALIDATION_DB_TIMEOUT_MS); + assert.equal(RF_VALIDATION_DB_TIMEOUT_MS, 3_500); + assert.equal(rfValidationPool.options.max, 2); + assert.match(rfValidationPool.options.options!, /default_transaction_read_only=on/); +}); + +test('RF cancellation destroys only its checked-out connection; successful reads release normally', async (t) => { + const released: Array = []; + let pending = false; + t.mock.method(rfValidationPool, 'connect', async () => ({ + release: (destroy?: boolean) => released.push(destroy), + query: async () => pending ? new Promise(() => {}) : { rows: [{ ok: true }] }, + })); + assert.deepEqual(await queryRfValidation('SELECT true AS ok'), { rows: [{ ok: true }] }); + const controller = new AbortController(); + pending = true; + const result = queryRfValidation('SELECT true AS ok', [], controller.signal); + await new Promise((resolve) => setImmediate(resolve)); + controller.abort(new Error('disconnected')); + await assert.rejects(result, /disconnected/); + assert.deepEqual(released, [false, true]); +}); diff --git a/backend/src/repositories/networkAnalysis.ts b/backend/src/repositories/networkAnalysis.ts index fe21792..e393055 100644 --- a/backend/src/repositories/networkAnalysis.ts +++ b/backend/src/repositories/networkAnalysis.ts @@ -1,9 +1,11 @@ -import type { QueryResultRow } from 'pg'; +import pg, { type QueryResultRow } from 'pg'; +import { databaseConfig } from '../platform/config/database.js'; import type { NetworkFilters } from '../api/utils/networkFilters.js'; type QueryFn = ( text: string, params?: unknown[], + signal?: AbortSignal, ) => Promise<{ rows: T[] }>; export type TopologyRow = { @@ -330,57 +332,150 @@ export async function standaloneTopologyRows(query: QueryFn, filters: NetworkFil ); } +export const RF_VALIDATION_DB_TIMEOUT_MS = 3_500; + +// RF validation must not inherit the shared pool's potentially unlimited +// analytics timeout. This small, lazy pool is read-only and bounds execution +// on the server even if the requesting browser or backend disconnects. +export const rfValidationPool = new pg.Pool({ + connectionString: process.env['DATABASE_URL'], + application_name: `${databaseConfig.applicationName}-rf-validation`, + options: `-c default_transaction_read_only=on${databaseConfig.schema ? ` -c search_path=${databaseConfig.schema},public` : ''}`, + max: 2, + connectionTimeoutMillis: 500, + idleTimeoutMillis: 10_000, + allowExitOnIdle: true, + statement_timeout: RF_VALIDATION_DB_TIMEOUT_MS, + query_timeout: RF_VALIDATION_DB_TIMEOUT_MS + 250, +}); +rfValidationPool.on('error', (error) => { + console.error('[rf-validation] database connection error', error.message); +}); + +export async function queryRfValidation( + text: string, + params?: unknown[], + signal?: AbortSignal, +): Promise<{ rows: T[] }> { + signal?.throwIfAborted(); + const client = await rfValidationPool.connect(); + let released = false; + let onAbort: (() => void) | undefined; + try { + signal?.throwIfAborted(); + const aborted = new Promise((_resolve, reject) => { + if (!signal) return; + onAbort = () => { + // Destroy rather than return a connection with an in-flight query. + // The server timeout remains the independent cancellation backstop. + released = true; + client.release(true); + reject(signal.reason); + }; + signal.addEventListener('abort', onAbort, { once: true }); + }); + return await Promise.race([client.query(text, params), aborted]); + } finally { + if (onAbort) signal?.removeEventListener('abort', onAbort); + if (!released) client.release(signal?.aborted === true); + } +} + +type RfValidationRow = { + node_a_id: string; + node_b_id: string; + name_a: string | null; + name_b: string | null; + observed_count: string | number; + multibyte_observed_count: string | number; + itm_path_loss_db: number | null; + itm_viable: boolean | null; + force_viable: boolean; + last_observed: string; + classification: string; +}; + export async function rfValidationRows( query: QueryFn, filters: NetworkFilters, limit: number, -) { - const limitParam = `$${filters.params.length + 1}`; - return query<{ - node_a_id: string; - node_b_id: string; - name_a: string | null; - name_b: string | null; - observed_count: string; - multibyte_observed_count: string; - itm_path_loss_db: number | null; - itm_viable: boolean | null; - force_viable: boolean; - last_observed: string; - classification: string; - }>( - `SELECT - nl.node_a_id, nl.node_b_id, a.name AS name_a, b.name AS name_b, - nl.observed_count, nl.multibyte_observed_count, nl.itm_path_loss_db, - nl.itm_viable, nl.force_viable, nl.last_observed::text, + signal?: AbortSignal, +): Promise<{ rows: RfValidationRow[] }> { + // The identity-node view estimates ~1 eligible row where production has + // thousands. Joining it twice to identity links caused millions of nested + // loop comparisons. Resolve scope once, then pass the actual ids into the + // link query so its hash joins have accurate input cardinality. + const nodes = await query<{ node_id: string; name: string | null }>( + `SELECT n.node_id, n.name + FROM node_identity_nodes n + WHERE (n.role IS NULL OR n.role = 2) + AND (n.name IS NULL OR n.name NOT LIKE '%🚫%') + ${filters.nodesAlias('n')}`, + filters.params, + signal, + ); + signal?.throwIfAborted(); + const names = new Map(nodes.rows.map((node) => [node.node_id, node.name])); + if (names.size === 0) return { rows: [] }; + const result = await query>( + `WITH eligible_ids AS MATERIALIZED ( + SELECT unnest($1::text[]) AS node_id + ), recent_links AS MATERIALIZED ( + SELECT + COALESCE(la.canonical_node_id, UPPER(BTRIM(nl.node_a_id))) AS raw_a, + COALESCE(lb.canonical_node_id, UPPER(BTRIM(nl.node_b_id))) AS raw_b, + nl.observed_count, nl.multibyte_observed_count, nl.itm_path_loss_db, + nl.itm_viable, nl.force_viable, nl.last_observed + FROM node_links nl + LEFT JOIN node_identity_aliases la + ON la.source_node_id = UPPER(BTRIM(nl.node_a_id)) + LEFT JOIN node_identity_aliases lb + ON lb.source_node_id = UPPER(BTRIM(nl.node_b_id)) + WHERE nl.last_observed > NOW() - INTERVAL '30 days' + ), aggregated_links AS ( + SELECT LEAST(raw_a, raw_b) AS node_a_id, + GREATEST(raw_a, raw_b) AS node_b_id, + SUM(observed_count)::integer AS observed_count, + SUM(multibyte_observed_count)::integer AS multibyte_observed_count, + MIN(itm_path_loss_db) AS itm_path_loss_db, + BOOL_OR(itm_viable) AS itm_viable, + BOOL_OR(force_viable) AS force_viable, + MAX(last_observed) AS last_observed + FROM recent_links + WHERE raw_a <> raw_b + AND raw_a IN (SELECT node_id FROM eligible_ids) + AND raw_b IN (SELECT node_id FROM eligible_ids) + GROUP BY LEAST(raw_a, raw_b), GREATEST(raw_a, raw_b) + ) + SELECT node_a_id, node_b_id, + observed_count, multibyte_observed_count, itm_path_loss_db, + itm_viable, force_viable, last_observed::text, CASE - WHEN nl.force_viable THEN 'operator_override' - WHEN nl.itm_viable = false AND nl.multibyte_observed_count > 0 THEN 'observed_unexpected' - WHEN nl.itm_viable = false AND nl.observed_count >= 10 THEN 'observed_unexpected' - WHEN nl.itm_viable = true AND nl.observed_count <= 2 AND nl.last_observed < NOW() - INTERVAL '7 days' THEN 'weak_model_evidence' + WHEN force_viable THEN 'operator_override' + WHEN itm_viable = false AND multibyte_observed_count > 0 THEN 'observed_unexpected' + WHEN itm_viable = false AND observed_count >= 10 THEN 'observed_unexpected' + WHEN itm_viable = true AND observed_count <= 2 AND last_observed < NOW() - INTERVAL '7 days' THEN 'weak_model_evidence' ELSE 'match' END AS classification - FROM node_identity_links nl - JOIN node_identity_nodes a ON a.node_id = nl.node_a_id - JOIN node_identity_nodes b ON b.node_id = nl.node_b_id - WHERE nl.last_observed > NOW() - INTERVAL '30 days' - AND (a.name IS NULL OR a.name NOT LIKE '%🚫%') - AND (b.name IS NULL OR b.name NOT LIKE '%🚫%') - AND (a.role IS NULL OR a.role = 2) - AND (b.role IS NULL OR b.role = 2) - ${filters.nodesAlias('a')} - ${filters.nodesAlias('b')} + FROM aggregated_links ORDER BY CASE - WHEN nl.itm_viable = false AND nl.multibyte_observed_count > 0 THEN 0 - WHEN nl.force_viable THEN 1 - WHEN nl.itm_viable = false AND nl.observed_count >= 10 THEN 2 - WHEN nl.itm_viable = true AND nl.observed_count <= 2 THEN 3 + WHEN itm_viable = false AND multibyte_observed_count > 0 THEN 0 + WHEN force_viable THEN 1 + WHEN itm_viable = false AND observed_count >= 10 THEN 2 + WHEN itm_viable = true AND observed_count <= 2 THEN 3 ELSE 4 END, - nl.multibyte_observed_count DESC, - nl.observed_count DESC - LIMIT ${limitParam}`, - [...filters.params, limit], + multibyte_observed_count DESC, observed_count DESC, node_a_id, node_b_id + LIMIT $2`, + [[...names.keys()], Math.max(1, Math.min(250, Math.trunc(limit) || 100))], + signal, ); + signal?.throwIfAborted(); + // Enrich only the bounded result, and fail closed if an endpoint is absent + // from the public scope (including when a test/custom query supplies rows). + return { rows: result.rows + .filter((row) => names.has(row.node_a_id) && names.has(row.node_b_id)) + .slice(0, limit) + .map((row) => ({ ...row, name_a: names.get(row.node_a_id)!, name_b: names.get(row.node_b_id)! })) }; } diff --git a/backend/src/repositories/nodeLinks.integration.test.ts b/backend/src/repositories/nodeLinks.integration.test.ts new file mode 100644 index 0000000..678f9d1 --- /dev/null +++ b/backend/src/repositories/nodeLinks.integration.test.ts @@ -0,0 +1,150 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import test from 'node:test'; +import { promisify } from 'node:util'; +import { networkFilters } from '../api/utils/networkFilters.js'; +import { createNodeRepository } from './nodes.js'; + +// Opt-in PostgreSQL fixture execution: SELECT only, with all production +// relations shadowed by CTEs. No tables, functions or data are written. +const sqlTest = process.env['NODE_LINKS_SQL_TESTS'] === '1' ? test : test.skip; +const execFileAsync = promisify(execFile); +const source = 'A'.repeat(64); +const peer = 'B'.repeat(64); +const quietPeer = 'C'.repeat(64); +const sourceAlias = 'F'.repeat(64); +const peerAlias = '2'.repeat(64); +const unrelated = 'D'.repeat(64); + +function packet(overrides: Record = {}) { + return { + seconds_ago: 3600, rx_node_id: source, src_node_id: peer, + network: 'ukmesh', topic: 'ukmesh/MME/observer/packets', topic_prefix: 'ukmesh', + visibility_ok: true, is_private: false, packet_hash: 'same-hash', + hop_count: 4, snr: null, rssi: null, payload: { direction: 'rx' }, + ...overrides, + }; +} + +function literal(value: unknown): string { + if (Array.isArray(value)) return `ARRAY[${value.map(literal).join(',')}]::text[]`; + return `'${String(value).replace(/'/g, "''")}'`; +} + +async function links(packets: ReturnType[], options: { + request?: string; network?: string; stalePrivacy?: boolean; + sourceName?: string; peerName?: string; peerNetwork?: string; sighted?: boolean; +} = {}) { + const nodes = [ + { node_id: source, name: options.sourceName ?? 'Source', network: 'ukmesh' }, + { node_id: peer, name: options.peerName ?? 'Peer', network: options.peerNetwork ?? 'ukmesh' }, + { node_id: quietPeer, name: 'Quiet', network: 'ukmesh' }, + { node_id: unrelated, name: 'Test', network: 'test' }, + ]; + const aliases = [ + { source_node_id: sourceAlias, canonical_node_id: source }, + { source_node_id: peerAlias, canonical_node_id: peer }, + ]; + const identityLinks = [ + { node_a_id: source, node_b_id: peer, observed_count: 9999, + itm_viable: true, force_viable: false, itm_path_loss_db: 112.1, + count_a_to_b: 99, count_b_to_a: 9900, last_observed: '2026-09-01T12:34:56.789+01:00' }, + { node_a_id: quietPeer, node_b_id: source, observed_count: 50000, + itm_viable: false, force_viable: true, itm_path_loss_db: null, + count_a_to_b: 40000, count_b_to_a: 10000, last_observed: null }, + { node_a_id: source, node_b_id: unrelated, observed_count: 3, + itm_viable: false, force_viable: false, itm_path_loss_db: null, + count_a_to_b: 2, count_b_to_a: 1, last_observed: null }, + ]; + const calls: string[] = []; + const repo = createNodeRepository(async (sql, params) => { + calls.push(sql); + const bound = sql.replace(/\$(\d+)\b/g, (_placeholder, index: string) => literal(params![Number(index) - 1])); + const fixture = `WITH + packets AS ( + SELECT p.*, NOW() - seconds_ago * INTERVAL '1 second' AS time + FROM jsonb_to_recordset(${literal(JSON.stringify(packets))}::jsonb) AS p( + seconds_ago double precision, rx_node_id text, src_node_id text, + network text, topic text, topic_prefix text, visibility_ok boolean, + is_private boolean, packet_hash text, hop_count integer, + snr double precision, rssi double precision, payload jsonb + ) + ), node_identity_aliases AS ( + SELECT * FROM jsonb_to_recordset(${literal(JSON.stringify(aliases))}::jsonb) + AS a(source_node_id text, canonical_node_id text) + ), node_identity_nodes AS ( + SELECT * FROM jsonb_to_recordset(${literal(JSON.stringify(nodes))}::jsonb) + AS n(node_id text, name text, network text) + ), nodes AS (SELECT * FROM node_identity_nodes), + node_identity_links AS ( + SELECT * FROM jsonb_to_recordset(${literal(JSON.stringify(identityLinks))}::jsonb) + AS l(node_a_id text, node_b_id text, observed_count integer, + itm_viable boolean, force_viable boolean, itm_path_loss_db double precision, + count_a_to_b integer, count_b_to_a integer, last_observed timestamptz) + ), node_identity_sightings AS ( + SELECT ${literal(peer)}::text AS node_id, 'ukmesh'::text AS network, NOW() AS last_seen_at + WHERE ${options.sighted ? 'TRUE' : 'FALSE'} + ), packet_visibility_materialization_state AS ( + SELECT TRUE AS singleton, 1 AS visibility_generation + ), public_visibility_state AS ( + SELECT TRUE AS singleton, ${options.stalePrivacy ? 2 : 1} AS generation + ) + SELECT COALESCE(jsonb_agg(to_jsonb(result)), '[]'::jsonb) FROM (${bound}) result;`; + const child = execFileAsync('docker', [ + 'exec', '-i', 'meshcore-infra-timescaledb-1', 'psql', '-X', + '-U', 'meshcore', '-d', 'meshcore', '-v', 'ON_ERROR_STOP=1', '-At', + ], { timeout: 20_000, maxBuffer: 2 * 1024 * 1024 }); + child.child.stdin!.end(fixture); + return { rows: JSON.parse((await child).stdout) }; + }); + return { rows: await repo.listNodeLinks(options.request ?? source, networkFilters(options.network ?? 'ukmesh')), calls }; +} + +sqlTest('contacts aggregate both alias directions, relayed packets and repeated hashes; preserve ISO, orientation and quiet links', async () => { + const packets = [packet(), packet({ rx_node_id: peer, src_node_id: source }), + packet({ rx_node_id: sourceAlias, src_node_id: peerAlias }), + packet({ rx_node_id: peerAlias, src_node_id: sourceAlias }), + packet({ payload: { direction: 'tx' } })]; + const result = await links(packets); + assert.equal(result.rows.length, 2); + assert.equal(result.rows.find(r => r.peer_id === peer)!.recent_count_72h, 5); + assert.equal(result.rows.find(r => r.peer_id === peer)!.last_observed, '2026-09-01T11:34:56.789Z'); + assert.deepEqual(result.rows.find(r => r.peer_id === quietPeer), { + peer_id: quietPeer, peer_name: 'Quiet', observed_count: 50000, itm_path_loss_db: null, + count_this_to_peer: 10000, count_peer_to_this: 40000, last_observed: null, recent_count_72h: 0, + }); + assert.deepEqual((await links(packets, { request: ` ${sourceAlias.toLowerCase()} ` })).rows, result.rows); + assert.equal(result.calls.filter(sql => sql.includes('FROM packets p')).length, 1); +}); + +sqlTest('contacts enforce exact 72h boundaries, public network compatibility and privacy without requiring radio metrics', async () => { + const packets = [ + packet({ seconds_ago: 72 * 3600 - 1 }), packet({ seconds_ago: 0 }), + packet({ network: 'northeast' }), packet({ network: 'teesside' }), + packet({ seconds_ago: 72 * 3600 }), packet({ seconds_ago: -1 }), + packet({ src_node_id: null }), packet({ rx_node_id: null }), + packet({ src_node_id: unrelated }), packet({ rx_node_id: unrelated }), + packet({ network: 'test' }), packet({ network: 'other' }), + packet({ topic_prefix: 'meshcore-test' }), + packet({ topic_prefix: null, topic: 'meshcore-test/MME/observer/packets' }), + packet({ visibility_ok: false }), packet({ visibility_ok: null }), packet({ is_private: true }), + ]; + const result = await links(packets); + assert.equal(result.rows.find(r => r.peer_id === peer)!.recent_count_72h, 4); + assert.ok((await links(packets, { stalePrivacy: true })).rows.every(r => r.recent_count_72h === 0)); +}); + +sqlTest('contacts do not inherit the sparkline 1000-point cap', async () => { + const result = await links(Array.from({ length: 1005 }, () => packet())); + assert.equal(result.rows.find(r => r.peer_id === peer)!.recent_count_72h, 1005); +}); + +sqlTest('source and peer visibility, force viability and recent network sightings remain intact', async () => { + assert.equal((await links([packet()], { sourceName: '🚫 Source' })).calls.length, 1); + assert.deepEqual((await links([packet()], { sourceName: '🚫 Source' })).rows, []); + assert.deepEqual((await links([packet()], { network: 'test' })).rows, []); + assert.deepEqual((await links([packet()], { peerName: '🚫 Peer' })).rows.map(r => r.peer_id), [quietPeer]); + assert.deepEqual((await links([packet()], { peerNetwork: 'other' })).rows.map(r => r.peer_id), [quietPeer]); + assert.equal((await links([packet()], { peerNetwork: 'other', sighted: true })).rows.find(r => r.peer_id === peer)!.recent_count_72h, 1); + assert.deepEqual((await links([packet()], { peerNetwork: 'test', sighted: true })).rows.map(r => r.peer_id), [quietPeer]); +}); diff --git a/backend/src/repositories/nodes.test.ts b/backend/src/repositories/nodes.test.ts index 268a082..ba701ae 100644 --- a/backend/src/repositories/nodes.test.ts +++ b/backend/src/repositories/nodes.test.ts @@ -1,7 +1,8 @@ import assert from 'node:assert/strict'; import test from 'node:test'; import { networkFilters } from '../api/utils/networkFilters.js'; -import { createNodeRepository } from './nodes.js'; +import type { QueryResultRow } from 'pg'; +import { createNodeRepository, type NodeLinkRow } from './nodes.js'; test('public map projects path evidence as the effective seen and online state', async () => { let capturedSql = ''; @@ -34,3 +35,42 @@ test('public map projects path evidence as the effective seen and online state', 101, ]); }); + +test('node links preserve metrics and fetch one scoped aggregate for the entire peer set', async () => { + const nodeId = 'A'.repeat(64); + const { recent_count_72h, ...link } = { + peer_id: 'B'.repeat(64), peer_name: 'Peer', observed_count: 999, + itm_path_loss_db: 100, count_this_to_peer: 500, count_peer_to_this: 499, + last_observed: '2026-09-07T12:00:00.123Z', recent_count_72h: 0, + } satisfies NodeLinkRow; + const calls: Array<{ sql: string; params?: unknown[] }> = []; + const rows = [{ ...link, recent_count_72h }]; + const repository = createNodeRepository(async (sql: string, params?: unknown[]) => { + calls.push({ sql, params }); + return { rows: (calls.length === 1 ? [link] : rows) as unknown as T[] }; + }); + assert.deepEqual(await repository.listNodeLinks(nodeId, networkFilters('ukmesh')), rows); + assert.equal(calls.length, 2); + assert.match(calls[0]!.sql, /to_char\(rl.last_observed AT TIME ZONE 'UTC'/); + const aggregate = calls[1]!; + assert.deepEqual(aggregate.params, [['ukmesh', 'northeast', 'teesside'], nodeId, JSON.stringify([link])]); + assert.equal(aggregate.sql.match(/FROM packets p/g)?.length, 1); + assert.match(aggregate.sql, /COUNT\(\*\)::integer AS recent_count_72h/); + assert.match(aggregate.sql, /GROUP BY pa.peer_id/); + assert.match(aggregate.sql, /LEFT JOIN recent_counts/); + assert.match(aggregate.sql, /COALESCE\(rc.recent_count_72h, 0\)::integer/); + assert.match(aggregate.sql, /p.src_node_id IS NOT NULL/); + assert.match(aggregate.sql, /p.time > NOW\(\) - INTERVAL '72 hours'/); + assert.match(aggregate.sql, /p.time <= NOW\(\)/); + assert.match(aggregate.sql, /p.network = ANY\(\$1\)/); + assert.match(aggregate.sql, /p.visibility_ok IS TRUE/); + assert.match(aggregate.sql, /p.is_private IS NOT TRUE/); + assert.match(aggregate.sql, /cached_visibility.visibility_generation = current_visibility.generation/); +}); + +test('empty or invisible link sets never scan packets', async () => { + let calls = 0; + const repository = createNodeRepository(async () => { calls += 1; return { rows: [] }; }); + assert.deepEqual(await repository.listNodeLinks('A'.repeat(64), networkFilters('ukmesh')), []); + assert.equal(calls, 1); +}); diff --git a/backend/src/repositories/nodes.ts b/backend/src/repositories/nodes.ts index 4f33683..dfa3f55 100644 --- a/backend/src/repositories/nodes.ts +++ b/backend/src/repositories/nodes.ts @@ -1,5 +1,5 @@ import type { QueryResultRow } from 'pg'; -import type { NetworkFilters } from '../api/utils/networkFilters.js'; +import { nodeAliasArraySql, type NetworkFilters } from '../api/utils/networkFilters.js'; import { publicMapFreshPredicate } from '../nodes/publicMap.js'; import { nodeEffectiveLastSeenSql, @@ -25,6 +25,8 @@ export type NodeLinkRow = { itm_path_loss_db: number | null; count_this_to_peer: number; count_peer_to_this: number; + last_observed: string | null; + recent_count_72h: number; }; export type TestDiagnosticsData = { @@ -230,40 +232,102 @@ export function createNodeRepository(query: QueryFn): NodeRepository { async listNodeLinks(nodeId, filters) { const idParam = `$${filters.params.length + 1}`; - return (await query( - `WITH source_node AS MATERIALIZED ( + const links = (await query>( + `WITH requested_identity AS MATERIALIZED ( + SELECT COALESCE( + (SELECT canonical_node_id FROM node_identity_aliases + WHERE source_node_id = UPPER(BTRIM(${idParam}))), + UPPER(BTRIM(${idParam})) + ) AS node_id + ), link_nodes AS MATERIALIZED ( + SELECT node_id, name, network FROM node_identity_nodes + ), source_node AS MATERIALIZED ( SELECT node_id - FROM node_identity_nodes sn - WHERE node_id = meshcore_canonical_node_id(${idParam}) + FROM link_nodes sn + WHERE node_id = (SELECT node_id FROM requested_identity) AND (name IS NULL OR name NOT LIKE '%🚫%') ${filters.nodesAlias('sn')} ), relevant_links AS MATERIALIZED ( SELECT - CASE WHEN nl.node_a_id = meshcore_canonical_node_id(${idParam}) + CASE WHEN nl.node_a_id = (SELECT node_id FROM requested_identity) THEN nl.node_b_id ELSE nl.node_a_id END AS peer_id, nl.observed_count, + nl.last_observed, nl.itm_path_loss_db, - CASE WHEN nl.node_a_id = meshcore_canonical_node_id(${idParam}) + CASE WHEN nl.node_a_id = (SELECT node_id FROM requested_identity) THEN nl.count_a_to_b ELSE nl.count_b_to_a END AS count_this_to_peer, - CASE WHEN nl.node_a_id = meshcore_canonical_node_id(${idParam}) + CASE WHEN nl.node_a_id = (SELECT node_id FROM requested_identity) THEN nl.count_b_to_a ELSE nl.count_a_to_b END AS count_peer_to_this FROM node_identity_links nl - WHERE (nl.node_a_id = meshcore_canonical_node_id(${idParam}) - OR nl.node_b_id = meshcore_canonical_node_id(${idParam})) + WHERE (nl.node_a_id = (SELECT node_id FROM requested_identity) + OR nl.node_b_id = (SELECT node_id FROM requested_identity)) AND (nl.itm_viable = TRUE OR nl.force_viable = TRUE) AND EXISTS (SELECT 1 FROM source_node) ) SELECT rl.peer_id, peer.name AS peer_name, rl.observed_count, - rl.itm_path_loss_db, rl.count_this_to_peer, rl.count_peer_to_this + rl.itm_path_loss_db, rl.count_this_to_peer, rl.count_peer_to_this, + to_char(rl.last_observed AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"') AS last_observed FROM relevant_links rl - JOIN node_identity_nodes peer ON peer.node_id = rl.peer_id + JOIN link_nodes peer ON peer.node_id = rl.peer_id WHERE (peer.name IS NULL OR peer.name NOT LIKE '%🚫%') ${filters.nodesAlias('peer')} ORDER BY rl.observed_count DESC`, [...filters.params, nodeId], )).rows; + if (links.length === 0) return []; + + // Resolve the visible link set first. The materialized identity views are + // expensive; share their node projection above, then keep this one packet + // aggregate indexed and bounded independently of those view plans. + // last_observed is the worker's latest observation in either direction, + // not necessarily direct. Count all public packet rows for canonical src/rx + // pairs (including relayed packets), not distinct hashes or lifetime hits. + // The sparkline uses the same pairing, but additionally needs signal data + // and caps its points; neither restriction applies to this 72-hour count. + const linksParam = `$${filters.params.length + 2}`; + return (await query( + `WITH source_ids AS MATERIALIZED ( + SELECT ${nodeAliasArraySql(idParam)} AS ids + ), visible_links AS MATERIALIZED ( + SELECT * FROM jsonb_to_recordset(${linksParam}::jsonb) AS link( + peer_id text, peer_name text, observed_count integer, + itm_path_loss_db double precision, count_this_to_peer integer, + count_peer_to_this integer, last_observed text + ) + ), peer_aliases AS MATERIALIZED ( + SELECT vl.peer_id, UNNEST(${nodeAliasArraySql('vl.peer_id')}) AS source_node_id + FROM visible_links vl + ), peer_ids AS MATERIALIZED ( + SELECT ARRAY(SELECT source_node_id FROM peer_aliases) AS ids + ), recent_counts AS ( + SELECT pa.peer_id, COUNT(*)::integer AS recent_count_72h + FROM packets p + JOIN peer_aliases pa ON pa.source_node_id = CASE + WHEN p.rx_node_id = ANY((SELECT ids FROM source_ids)::text[]) + THEN p.src_node_id ELSE p.rx_node_id END + WHERE p.time > NOW() - INTERVAL '72 hours' + AND p.time <= NOW() + AND p.src_node_id IS NOT NULL + AND ( + (p.rx_node_id = ANY((SELECT ids FROM source_ids)::text[]) + AND p.src_node_id = ANY((SELECT ids FROM peer_ids)::text[])) + OR (p.src_node_id = ANY((SELECT ids FROM source_ids)::text[]) + AND p.rx_node_id = ANY((SELECT ids FROM peer_ids)::text[])) + ) + ${filters.packetsAlias('p')} + GROUP BY pa.peer_id + ) + SELECT vl.peer_id, vl.peer_name, vl.observed_count, + vl.itm_path_loss_db, vl.count_this_to_peer, vl.count_peer_to_this, + vl.last_observed, + COALESCE(rc.recent_count_72h, 0)::integer AS recent_count_72h + FROM visible_links vl + LEFT JOIN recent_counts rc ON rc.peer_id = vl.peer_id + ORDER BY vl.observed_count DESC`, + [...filters.params, nodeId, JSON.stringify(links)], + )).rows; }, }; } diff --git a/backend/src/repositories/productFeatures.integration.test.ts b/backend/src/repositories/productFeatures.integration.test.ts new file mode 100644 index 0000000..fadbba7 --- /dev/null +++ b/backend/src/repositories/productFeatures.integration.test.ts @@ -0,0 +1,170 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import test from 'node:test'; +import { promisify } from 'node:util'; +import { linkHistoryRows, type QueryFn } from './productFeatures.js'; + +// Opt-in, read-only PostgreSQL checks. Every relation used by the production +// query is shadowed by a VALUES/JSON fixture CTE; no tables or rows are written. +// LINK_HISTORY_SQL_TESTS=1 node --import tsx --test src/repositories/productFeatures.integration.test.ts +const sqlTest = process.env['LINK_HISTORY_SQL_TESTS'] === '1' ? test : test.skip; +const execFileAsync = promisify(execFile); +const nodeA = 'A'.repeat(64); +const nodeB = 'B'.repeat(64); +const aliasA = 'F'.repeat(64); +const aliasB = '2'.repeat(64); +const unrelated = 'C'.repeat(64); +const baseTime = Date.now() - 2 * 60 * 60_000; + +type Packet = { + time: string; + observation_id: string; + rx_node_id: string | null; + src_node_id: string | null; + network: string; + snr: number | null; + rssi: number | null; + payload: { direction?: string } | null; + visibility_ok: boolean | null; + is_private: boolean | null; + hop_count: number; +}; + +function packet(index: number, overrides: Partial = {}): Packet { + return { + time: new Date(baseTime + index * 1_000).toISOString(), + observation_id: `00000000-0000-0000-0000-${String(index).padStart(12, '0')}`, + rx_node_id: nodeA, + src_node_id: nodeB, + network: 'ukmesh', + snr: -index, + rssi: -99, + payload: { direction: 'rx' }, + visibility_ok: true, + is_private: false, + hop_count: 0, + ...overrides, + }; +} + +function literal(value: unknown): string { + if (Array.isArray(value)) return `ARRAY[${value.map(literal).join(',')}]::text[]`; + return `'${String(value).replace(/'/g, "''")}'`; +} + +async function history( + packets: Packet[], + options: { nodeA?: string; nodeB?: string; hours?: number; networks?: string[]; noPathLoss?: boolean; stalePrivacy?: boolean } = {}, +) { + const aliases = [ + { source_node_id: aliasA, canonical_node_id: nodeA }, + { source_node_id: aliasB, canonical_node_id: nodeB }, + ]; + const links = options.noPathLoss ? [] : [ + { node_a_id: nodeA, node_b_id: nodeB, itm_path_loss_db: 140 }, + { node_a_id: nodeB, node_b_id: aliasA, itm_path_loss_db: 110 }, + { node_a_id: aliasB, node_b_id: nodeA, itm_path_loss_db: 120 }, + { node_a_id: nodeA, node_b_id: unrelated, itm_path_loss_db: 10 }, + ]; + const query: QueryFn = async (sql, params) => { + const bound = sql.replace(/\$(\d+)\b/g, (_placeholder, index: string) => literal(params![Number(index) - 1])); + const fixtureSql = `WITH + packets AS ( + SELECT * FROM jsonb_to_recordset(${literal(JSON.stringify(packets))}::jsonb) AS p( + time timestamptz, observation_id uuid, rx_node_id text, src_node_id text, + network text, snr double precision, rssi double precision, payload jsonb, + visibility_ok boolean, is_private boolean, hop_count integer + ) + ), node_identity_aliases AS ( + SELECT * FROM jsonb_to_recordset(${literal(JSON.stringify(aliases))}::jsonb) + AS a(source_node_id text, canonical_node_id text) + ), node_links AS ( + SELECT * FROM jsonb_to_recordset(${literal(JSON.stringify(links))}::jsonb) + AS l(node_a_id text, node_b_id text, itm_path_loss_db double precision) + ), packet_visibility_materialization_state AS ( + SELECT TRUE AS singleton, 1 AS visibility_generation + ), public_visibility_state AS ( + SELECT TRUE AS singleton, ${options.stalePrivacy ? 2 : 1} AS generation + ) + SELECT COALESCE(jsonb_agg(to_jsonb(points)), '[]'::jsonb) + FROM (${bound}) points;`; + // Use stdin to keep large fixtures within the OS argument-size limit. + const child = execFileAsync('docker', [ + 'exec', '-i', 'meshcore-infra-timescaledb-1', + 'psql', '-X', '-U', 'meshcore', '-d', 'meshcore', '-v', 'ON_ERROR_STOP=1', '-At', + ], { maxBuffer: 2 * 1024 * 1024, timeout: 20_000 }); + child.child.stdin!.end(fixtureSql); + const { stdout } = await child; + return { rows: JSON.parse(stdout) }; + }; + return (await linkHistoryRows( + query, options.nodeA ?? nodeA, options.nodeB ?? nodeB, options.hours ?? 72, + options.networks ?? ['ukmesh', 'northeast', 'teesside'], + )).rows; +} + +sqlTest('packet history expands both aliases, includes relayed receptions and preserves ISO radio points', async () => { + const packets = [ + packet(1), + packet(2, { rx_node_id: nodeB, src_node_id: nodeA }), + packet(3, { rx_node_id: aliasA, src_node_id: aliasB, hop_count: 4 }), + packet(4, { rx_node_id: aliasB, src_node_id: aliasA, hop_count: 2 }), + ]; + const expected = packets.map(({ time, snr, rssi }) => ({ time, snr, rssi, path_loss: 110, sample_count: 1 })); + assert.deepEqual(await history(packets), expected); + assert.deepEqual(await history(packets, { nodeA: aliasA.toLowerCase(), nodeB: ` ${aliasB} ` }), expected); + assert.deepEqual(await history(packets, { nodeA: nodeB, nodeB: nodeA }), expected); +}); + +sqlTest('packet history excludes null sources, unrelated peers, TX, absent metrics and hidden receptions', async () => { + const good = packet(1, { payload: null, snr: 0, rssi: null }); + const rssiOnly = packet(2, { snr: null }); + const rows = await history([ + good, rssiOnly, + packet(3, { src_node_id: null }), + packet(4, { src_node_id: unrelated }), + packet(5, { rx_node_id: unrelated }), + packet(6, { rx_node_id: null }), + packet(7, { payload: { direction: 'tx' } }), + packet(8, { snr: null, rssi: null }), + packet(9, { visibility_ok: false }), + packet(10, { visibility_ok: null }), + packet(11, { is_private: true }), + ]); + assert.deepEqual(rows.map(({ time, snr, rssi }) => ({ time, snr, rssi })), + [good, rssiOnly].map(({ time, snr, rssi }) => ({ time, snr, rssi }))); + assert.deepEqual(await history([good], { stalePrivacy: true }), []); +}); + +sqlTest('packet history enforces network isolation and the requested time window', async () => { + const packets = [ + packet(1), + packet(2, { network: 'northeast' }), + packet(3, { network: 'teesside' }), + packet(4, { network: 'test' }), + packet(5, { time: new Date(Date.now() - 73 * 60 * 60_000).toISOString() }), + packet(6, { time: new Date(Date.now() + 60 * 60_000).toISOString() }), + ]; + assert.deepEqual((await history(packets)).map(({ snr }) => snr), [-1, -2, -3]); + assert.deepEqual((await history(packets, { networks: ['test'] })).map(({ snr }) => snr), [-4]); + assert.deepEqual(await history(packets, { hours: 1 }), []); + assert.deepEqual((await history(packets, { hours: 168 })).map(({ snr }) => snr), [-5, -1, -2, -3]); +}); + +sqlTest('packet history returns the latest 1000 observations in chronological order', async () => { + const packets = Array.from({ length: 1005 }, (_, index) => packet(index)); + const rows = await history(packets.reverse()); + assert.equal(rows.length, 1000); + assert.equal(rows[0].snr, -5); + assert.equal(rows.at(-1)!.snr, -1004); + assert.deepEqual(rows.map(({ time }) => time), [...rows.map(({ time }) => time)].sort()); + assert.ok(rows.every((row) => row.sample_count === 1)); +}); + +sqlTest('packet history keeps radio samples without a model link and returns empty for no traffic', async () => { + assert.deepEqual(await history([packet(1)], { noPathLoss: true }), [{ + time: packet(1).time, snr: -1, rssi: -99, path_loss: null, sample_count: 1, + }]); + assert.deepEqual(await history([]), []); + assert.deepEqual(await history([packet(1)], { nodeB: unrelated }), []); +}); diff --git a/backend/src/repositories/productFeatures.test.ts b/backend/src/repositories/productFeatures.test.ts index 9660222..570f456 100644 --- a/backend/src/repositories/productFeatures.test.ts +++ b/backend/src/repositories/productFeatures.test.ts @@ -1,6 +1,6 @@ import assert from 'node:assert/strict'; import test from 'node:test'; -import { visibleLinkNodeIds, type QueryFn } from './productFeatures.js'; +import { linkHistoryRows, visibleLinkNodeIds, type QueryFn } from './productFeatures.js'; test('public link history authorizes both endpoints through network and privacy scope', async () => { let capturedSql = ''; @@ -24,3 +24,23 @@ test('public link history authorizes both endpoints through network and privacy assert.match(capturedSql, /sighting\.last_seen_at > NOW\(\) - INTERVAL '30 days'/); assert.doesNotMatch(capturedSql, /SELECT .* FROM node_link_radio_reports/is); }); + +test('link history binds both identities, the time window and packet network scope', async () => { + const calls: Array<{ sql: string; params: unknown[] | undefined }> = []; + const query: QueryFn = async (sql, params) => { + calls.push({ sql, params }); + return { rows: [] }; + }; + const nodeA = 'A'.repeat(64); + const nodeB = 'B'.repeat(64); + + assert.deepEqual(await linkHistoryRows(query, nodeA, nodeB, 72, ['test']), { rows: [] }); + assert.equal(calls.length, 1); + assert.deepEqual(calls[0].params, [nodeA, nodeB, '72', ['test']]); + // Regressions here would either revive the orphaned source, hide aliases, + // or turn indexed packet lookups into per-row identity function calls. + assert.match(calls[0].sql, /FROM packets p/); + assert.match(calls[0].sql, /FROM node_identity_aliases alias/); + assert.doesNotMatch(calls[0].sql, /node_identity_link_radio_reports/); + assert.doesNotMatch(calls[0].sql, /meshcore_canonical_node_id\(p\./); +}); diff --git a/backend/src/repositories/productFeatures.ts b/backend/src/repositories/productFeatures.ts index 6f28762..1d3ec77 100644 --- a/backend/src/repositories/productFeatures.ts +++ b/backend/src/repositories/productFeatures.ts @@ -1,4 +1,5 @@ import type { QueryResultRow } from 'pg'; +import { nodeAliasArraySql, publicPacketPrivacySql } from '../api/utils/networkFilters.js'; export type QueryFn = ( text: string, @@ -102,7 +103,11 @@ export async function linkHistoryRows( nodeA: string, nodeB: string, hours: number, + networks: string[], ) { + // Resolve identities before scanning packets so both endpoint predicates can + // use the raw-key indexes. These are source/receiver receptions, including + // relayed packets; their SNR is not proof of a direct RF link between peers. return query<{ time: string; snr: number | null; @@ -110,17 +115,40 @@ export async function linkHistoryRows( path_loss: number | null; sample_count: number; }>( - `SELECT reports.last_seen::text AS time, reports.last_snr_db AS snr, - NULL::double precision AS rssi, links.itm_path_loss_db AS path_loss, - reports.sample_count - FROM node_identity_link_radio_reports reports - LEFT JOIN node_identity_links links - ON links.node_a_id = reports.node_a_id AND links.node_b_id = reports.node_b_id - WHERE reports.node_a_id = meshcore_canonical_node_id($1) - AND reports.node_b_id = meshcore_canonical_node_id($2) - AND reports.last_seen > NOW() - ($3::text || ' hours')::interval - ORDER BY reports.last_seen ASC`, - [nodeA, nodeB, String(hours)], + `WITH ids AS MATERIALIZED ( + SELECT ${nodeAliasArraySql('$1')} AS a_ids, + ${nodeAliasArraySql('$2')} AS b_ids + ), recent AS ( + SELECT p.time, p.observation_id, p.snr, p.rssi + FROM packets p + WHERE p.time > NOW() - ($3::text || ' hours')::interval + AND p.time <= NOW() + AND p.network = ANY($4::text[]) + AND p.src_node_id IS NOT NULL + AND ( + -- Scalar initplans keep the alias arrays in the packet index predicates. + (p.rx_node_id = ANY((SELECT a_ids FROM ids)::text[]) + AND p.src_node_id = ANY((SELECT b_ids FROM ids)::text[])) + OR (p.rx_node_id = ANY((SELECT b_ids FROM ids)::text[]) + AND p.src_node_id = ANY((SELECT a_ids FROM ids)::text[])) + ) + AND p.payload->>'direction' IS DISTINCT FROM 'tx' + AND (p.snr IS NOT NULL OR p.rssi IS NOT NULL) + AND ${publicPacketPrivacySql('p')} + ORDER BY p.time DESC, p.observation_id DESC + LIMIT 1000 + ), path_loss AS MATERIALIZED ( + -- Match the identity view's MIN without materializing every network link. + SELECT MIN(links.itm_path_loss_db) AS value + FROM node_links links CROSS JOIN ids + WHERE (links.node_a_id = ANY(ids.a_ids) AND links.node_b_id = ANY(ids.b_ids)) + OR (links.node_a_id = ANY(ids.b_ids) AND links.node_b_id = ANY(ids.a_ids)) + ) + SELECT to_char(recent.time AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"') AS time, + recent.snr, recent.rssi, path_loss.value AS path_loss, 1::integer AS sample_count + FROM recent CROSS JOIN path_loss + ORDER BY recent.time ASC, recent.observation_id ASC`, + [nodeA, nodeB, String(hours), networks], ); } diff --git a/backend/src/spam/cluster.test.ts b/backend/src/spam/cluster.test.ts index d012268..7ee35d3 100644 --- a/backend/src/spam/cluster.test.ts +++ b/backend/src/spam/cluster.test.ts @@ -268,3 +268,34 @@ test('canonical URL index prevents interleaved decoys evicting variant spam', () assert.ok(target); assert.equal(target.messageCount, 8); }); + +test('a rotating-ID botnet flood (many single-shot machine-named senders) is caught', () => { + // Real-world pattern (Oct 2026): the anti-Kirby spammer rotated through + // fresh keygen-style node identities, each posting the same slogan once. + // These are not "many independent people": every identity appears exactly + // once in the window and the names are machine-generated — so the cluster + // must qualify as an incident, not be dismissed as broad chatter. + const text = 'still clinging to the trademarks like a desperate leech'; + const ids = [ + '0CYT64KV', '1KG3RWU7', '16V0OP2W', '2H6WCDJW', '5ZA1J31M', + '6KE2E6P6', '7F32S5OK', 'C23R8J39', 'DP8INYG2', 'G6B8QHA3', + ]; + const records = ids.map((id, i) => rec(text, id, i)); + const incidents = clusterMessages(records, cfg()); + assert.equal(incidents.length, 1, 'the rotating-ID flood should form one incident'); + assert.equal(incidents[0]!.messageCount, ids.length); + assert.ok( + incidents[0]!.score >= DEFAULT_SPAM_MESSAGE_CONFIG.publicMinScore, + `expected score above the public floor, got ${incidents[0]!.score}`, + ); + assert.ok(incidents[0]!.reasons.some((r) => /single-shot/i.test(r))); +}); + +test('single-shot human chatter is still treated as broad chatter', () => { + // Guard for the rotating-identity exemption: many one-shot senders with + // human names (a meme / coordinated greeting) must stay chatter. + const text = 'happy birthday to the repeater that never sleeps'; + const names = ['Alice', 'Bob', 'Carol', 'Dave', 'Erin', 'Frank', 'Grace', 'Heidi']; + const records = names.map((n, i) => rec(text, n, i)); + assert.equal(clusterMessages(records, cfg()).length, 0); +}); diff --git a/backend/src/spam/cluster.ts b/backend/src/spam/cluster.ts index cacc02b..4f9bf74 100644 --- a/backend/src/spam/cluster.ts +++ b/backend/src/spam/cluster.ts @@ -2,7 +2,7 @@ import { createHash } from 'node:crypto'; import type { Incident, IncidentStatus, MessageRecord } from './types.js'; import type { SpamMessageConfig } from './config.js'; import { messageSimilarity, usernameSimilarity } from './similarity.js'; -import { normalizeUsername } from './normalize.js'; +import { normalizeUsername, looksMachineGeneratedName } from './normalize.js'; // Trivial / connectivity tokens that are normal mesh chatter, never spam on // their own. A cluster made up solely of these (and short) is not an incident. @@ -108,6 +108,27 @@ function maxBurstDensity(sorted: MessageRecord[], windowMs: number): number { return best; } +/** Max number of distinct non-empty senders inside any single sliding burst window. */ +function maxBurstSenders(sorted: MessageRecord[], windowMs: number): number { + const counts = new Map(); + let best = 0; + let start = 0; + for (let end = 0; end < sorted.length; end++) { + const key = normalizeUsername(sorted[end]!.sender) || '∅'; + counts.set(key, (counts.get(key) ?? 0) + 1); + while (sorted[end]!.observedAt - sorted[start]!.observedAt > windowMs) { + const k = normalizeUsername(sorted[start]!.sender) || '∅'; + const remaining = (counts.get(k) ?? 1) - 1; + if (remaining <= 0) counts.delete(k); + else counts.set(k, remaining); + start += 1; + } + const distinctInWindow = counts.size - (counts.has('∅') ? 1 : 0); + best = Math.max(best, distinctInWindow); + } + return best; +} + /** Summary of who is sending the messages in a cluster. */ interface SenderProfile { distinctSenders: number; @@ -115,13 +136,28 @@ interface SenderProfile { dominantShare: number; /** >=2 distinct senders that are fuzzy-similar variants of each other. */ rotatingSimilarNames: boolean; + /** Share of distinct senders appearing exactly once in the analysis window. */ + singleShotShare: number; + /** Share of distinct senders whose name looks machine-generated (keygen-style). */ + machineNameShare: number; + /** Most distinct senders seen inside any single burst window. */ + burstSenderCount: number; + /** Many single-shot, machine-named senders inside one burst = rotating-ID + * botnet, not broad human participation. */ + rotatingIdentities: boolean; } -function profileSenders(members: MessageRecord[], cfg: SpamMessageConfig): SenderProfile { +function profileSenders( + members: MessageRecord[], + cfg: SpamMessageConfig, + corpusSenderCounts: ReadonlyMap, +): SenderProfile { const counts = new Map(); + const rawByKey = new Map(); for (const m of members) { const key = normalizeUsername(m.sender) || '∅'; counts.set(key, (counts.get(key) ?? 0) + 1); + if (!rawByKey.has(key)) rawByKey.set(key, m.sender); } const distinct = [...counts.keys()].filter((k) => k !== '∅'); const distinctSenders = distinct.length || (counts.has('∅') ? 1 : 0); @@ -138,7 +174,38 @@ function profileSenders(members: MessageRecord[], cfg: SpamMessageConfig): Sende } } } - return { distinctSenders, dominantShare, rotatingSimilarNames }; + + // Rotating-identity (botnet) profile: how many of these "distinct senders" + // are actually single-shot, machine-named identities rather than people? + // A one-shot identity exists only for this flood, and a keygen-style name + // reinforces that the identity was generated, not typed by a human. + let singleShot = 0; + let machineNamed = 0; + for (const key of distinct) { + const corpusCount = corpusSenderCounts.get(key) ?? counts.get(key) ?? 1; + if (corpusCount <= 1) singleShot += 1; + if (looksMachineGeneratedName(rawByKey.get(key) ?? '')) machineNamed += 1; + } + const distinctCount = distinct.length; + const singleShotShare = distinctCount > 0 ? singleShot / distinctCount : 0; + const machineNameShare = distinctCount > 0 ? machineNamed / distinctCount : 0; + const burstSenderCount = maxBurstSenders(members, cfg.burstWindowMs); + + const rotatingIdentities = + distinctCount >= cfg.rotatingMinSenders && + singleShotShare >= cfg.rotatingSingleShotShare && + machineNameShare >= cfg.rotatingMachineNameShare && + burstSenderCount >= cfg.rotatingMinBurstSenders; + + return { + distinctSenders, + dominantShare, + rotatingSimilarNames, + singleShotShare, + machineNameShare, + burstSenderCount, + rotatingIdentities, + }; } /** Whether the canonical text is only trivial/connectivity tokens. */ @@ -194,17 +261,21 @@ function qualifies( if (isBenignChatter(canonical) && !hasUrl && !hasMarker) return false; // Many *dissimilar* people sending the same thing = legitimate (a meme, a - // greeting, a coordinated test) — not one actor abusing the network. + // greeting, a coordinated test) — not one actor abusing the network. The + // exception: "many senders" made of single-shot machine-named identities is + // a rotating-ID botnet, not people (each identity exists only for the flood). const broadChatter = sender.distinctSenders > cfg.maxIndependentSenders && !sender.rotatingSimilarNames && + !sender.rotatingIdentities && sender.dominantShare < cfg.dominantSenderShare; if (broadChatter) return false; const senderSuspicious = sender.distinctSenders <= 1 || sender.dominantShare >= cfg.dominantSenderShare || - sender.rotatingSimilarNames; + sender.rotatingSimilarNames || + sender.rotatingIdentities; // A few dissimilar senders, no link, no marker -> not enough to call spam. if (!senderSuspicious && !hasUrl && !hasMarker) return false; @@ -234,7 +305,8 @@ function scoreIncident( if (hasMarker) score += 0.25; if (urlRepeats) score += 0.2; else if (hasUrl) score += 0.1; - if (sender.rotatingSimilarNames) score += 0.25; + if (sender.rotatingIdentities) score += 0.3; + else if (sender.rotatingSimilarNames) score += 0.25; else if (sender.distinctSenders <= 1 && sender.dominantShare >= 0.8) score += 0.1; // Connectivity/test floods with no link, marker or name-rotation are noise, @@ -242,7 +314,8 @@ function scoreIncident( // Only content that is *dominated* by benign tokens counts (so a templated // flood that merely contains the word "test" is not let off the hook). const looksLikeTesting = - isBenignDominated(canonicalText) && !hasUrl && !hasMarker && !sender.rotatingSimilarNames; + isBenignDominated(canonicalText) && !hasUrl && !hasMarker && + !sender.rotatingSimilarNames && !sender.rotatingIdentities; // A sustained, high-volume repeat of the same templated message is itself a // strong automated-abuse signal, regardless of what it says — but we never @@ -258,7 +331,13 @@ function scoreIncident( reasons.push(`${count} near-duplicate messages`); reasons.push(`up to ${burst} within ${Math.round(cfg.burstWindowMs / 60000)} min`); if (isFlood) reasons.push('sustained high-volume flood'); - if (sender.rotatingSimilarNames) reasons.push(`${sender.distinctSenders} similar (rotating) sender names`); + if (sender.rotatingIdentities) { + reasons.push( + `${sender.distinctSenders} single-shot identities ` + + `(${Math.round(sender.machineNameShare * 100)}% machine-style names, ` + + `${sender.burstSenderCount} within ${Math.round(cfg.burstWindowMs / 60000)} min) posted near-identical text`, + ); + } else if (sender.rotatingSimilarNames) reasons.push(`${sender.distinctSenders} similar (rotating) sender names`); else if (sender.distinctSenders <= 1) reasons.push('single repeating sender'); if (urlRepeats) reasons.push('same link repeated across messages'); else if (hasUrl) reasons.push('contains a link'); @@ -268,7 +347,11 @@ function scoreIncident( return { score, reasons }; } -function finalizeIncident(members: MessageRecord[], cfg: SpamMessageConfig): Incident | null { +function finalizeIncident( + members: MessageRecord[], + cfg: SpamMessageConfig, + corpusSenderCounts: ReadonlyMap, +): Incident | null { const sorted = [...members].sort((a, b) => a.observedAt - b.observedAt); const rep = modalNormalized(sorted); const network = sorted[0]!.network; @@ -287,7 +370,7 @@ function finalizeIncident(members: MessageRecord[], cfg: SpamMessageConfig): Inc } const canonical = rep.norm.normalized; - const sender = profileSenders(sorted, cfg); + const sender = profileSenders(sorted, cfg, corpusSenderCounts); if (!qualifies(canonical, hasMarker, hasUrl, sender, cfg)) return null; const { score, reasons } = scoreIncident(sorted, hasMarker, hasUrl, sender, cfg); @@ -330,6 +413,13 @@ export function clusterMessages(records: MessageRecord[], cfg: SpamMessageConfig const sorted = [...eligible] .slice(0, cfg.maxMessagesPerRun) .sort((a, b) => a.observedAt - b.observedAt); + // Corpus-wide sender frequency within this analysis batch: rotating-ID + // detection needs to know whether each sender is a one-shot identity. + const corpusSenderCounts = new Map(); + for (const record of sorted) { + const key = normalizeUsername(record.sender) || '∅'; + corpusSenderCounts.set(key, (corpusSenderCounts.get(key) ?? 0) + 1); + } const open: OpenCluster[] = []; const closed: OpenCluster[] = []; const exactIndex = new Map(); @@ -445,7 +535,7 @@ export function clusterMessages(records: MessageRecord[], cfg: SpamMessageConfig if (cl.members.length < cfg.minTransmissions) continue; const sortedMembers = [...cl.members].sort((a, b) => a.observedAt - b.observedAt); if (maxBurstDensity(sortedMembers, cfg.burstWindowMs) < cfg.minBurst) continue; - const incident = finalizeIncident(cl.members, cfg); + const incident = finalizeIncident(cl.members, cfg, corpusSenderCounts); if (incident) incidents.push(incident); } diff --git a/backend/src/spam/config.ts b/backend/src/spam/config.ts index 23aa656..edfe0c1 100644 --- a/backend/src/spam/config.ts +++ b/backend/src/spam/config.ts @@ -62,6 +62,18 @@ export interface SpamMessageConfig { /** At/above this message count a qualified cluster is treated as a sustained * flood and scored up — an automated, high-volume abuse signal on its own. */ floodMinMessages: number; + /** Rotating-identity (botnet) detection: a cluster whose senders are many + * single-shot, machine-named identities posting within one burst is an + * automated flood, not broad human participation. Minimum distinct senders + * before the rotating-identity path can apply. */ + rotatingMinSenders: number; + /** ...share of those senders that must appear exactly once in the analysis + * window (a one-shot identity, not an established member). */ + rotatingSingleShotShare: number; + /** ...share of sender names that must look machine-generated (keygen-style). */ + rotatingMachineNameShare: number; + /** ...distinct senders required inside a single burst window. */ + rotatingMinBurstSenders: number; /** Channel labels excluded from spam detection (the test channel by design * carries repeated test traffic that is not abuse). */ excludeChannels: string[]; @@ -132,6 +144,10 @@ export function loadSpamMessageConfig(): SpamMessageConfig { maxIndependentSenders: envNum('SPAM_MESSAGE_MAX_INDEPENDENT_SENDERS', 4), dominantSenderShare: envNum('SPAM_MESSAGE_DOMINANT_SENDER_SHARE', 0.6), floodMinMessages: envNum('SPAM_MESSAGE_FLOOD_MIN_MESSAGES', 25), + rotatingMinSenders: envNum('SPAM_MESSAGE_ROTATING_MIN_SENDERS', 5), + rotatingSingleShotShare: envNum('SPAM_MESSAGE_ROTATING_SINGLE_SHOT_SHARE', 0.8), + rotatingMachineNameShare: envNum('SPAM_MESSAGE_ROTATING_MACHINE_NAME_SHARE', 0.5), + rotatingMinBurstSenders: envNum('SPAM_MESSAGE_ROTATING_MIN_BURST_SENDERS', 3), excludeChannels: (process.env['SPAM_MESSAGE_EXCLUDE_CHANNELS'] ?? 'test') .split(',') .map((s) => s.trim().toLowerCase()) diff --git a/backend/src/spam/normalize.test.ts b/backend/src/spam/normalize.test.ts index 01fab62..b0c6339 100644 --- a/backend/src/spam/normalize.test.ts +++ b/backend/src/spam/normalize.test.ts @@ -1,6 +1,6 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; -import { normalizeMessage, normalizeUsername, canonicalizeUrl, collapseRepeats, SPAM_MARKER } from './normalize.js'; +import { normalizeMessage, normalizeUsername, canonicalizeUrl, collapseRepeats, looksMachineGeneratedName, SPAM_MARKER } from './normalize.js'; test('lowercases, trims and collapses whitespace', () => { const n = normalizeMessage(' Hello WORLD '); @@ -57,3 +57,20 @@ test('handles null / empty input safely', () => { assert.deepEqual(n.tokens, []); assert.equal(normalizeUsername(undefined), ''); }); + +test('machine-generated names are detected (keygen-style botnet ids)', () => { + // Observed rotating-ID spammer style: separator-free base32-ish tokens. + for (const id of ['0CYT64KV', '1KG3RWU7', '43DBCEEF5C', '5ZA1J31M', '16V0OP2W', 'H5M6N2P8', 'BKGL7DMF', '8SPLFZAP']) { + assert.equal(looksMachineGeneratedName(id), true, `${id} should look machine-generated`); + } +}); + +test('human and amateur-radio handles are not machine-generated', () => { + for (const name of [ + 'M7GQW', 'MU6GBG', 'G0SYR', '2E0ABC', 'GW4ABC', 'GB3WF', + 'Rob M7JCN V3', 'Dave-S75', 'Grog 🚲', 'Simeon', 'Timberland', + 'Delta4zer0', 'eyrie', 'Alice', '12345678', 'test', + ]) { + assert.equal(looksMachineGeneratedName(name), false, `${name} should not look machine-generated`); + } +}); diff --git a/backend/src/spam/normalize.ts b/backend/src/spam/normalize.ts index 160108a..1bdc3bd 100644 --- a/backend/src/spam/normalize.ts +++ b/backend/src/spam/normalize.ts @@ -111,3 +111,53 @@ export function normalizeUsername(input: string | null | undefined): string { name = collapseRepeats(name); return name.replace(/\s+/g, ' ').trim(); } + +/** + * Heuristic: does a display name look machine-generated (a keygen-style random + * token) rather than a human handle? One signal among several for + * rotating-identity botnet detection — never sufficient on its own. + * + * Observed botnet style: separator-free base32-ish tokens such as + * "0CYT64KV", "1KG3RWU7", "43DBCEEF5C", "5ZA1J31M", "16V0OP2W". + * Human / amateur-radio handles ("M7GQW", "MU6GBG", "2E0ABC", "Rob M7JCN V3", + * "Dave-S75", "Grog 🚲") must NOT match: composite names carry separators and + * callsigns keep a single digit next to letters. + */ +export function looksMachineGeneratedName(input: string | null | undefined): boolean { + const raw = (input ?? '').normalize('NFKC').trim(); + if (raw.length === 0) return false; + + // Strip pictographic decoration, then require a single alphanumeric token: + // any remaining separator/punctuation marks the name as composite (human). + const stripped = raw + .replace(/[\p{Extended_Pictographic}\uFE0F\u200D]/gu, '') + .trim(); + if (stripped.length === 0) return false; + if (/[^A-Za-z0-9]/.test(stripped)) return false; + + const compact = stripped; + if (compact.length < 6 || compact.length > 14) return false; + + const upper = compact.toUpperCase(); + const letterChars = compact.replace(/[^A-Za-z]/g, ''); + const letters = upper.replace(/[^A-Z]/g, ''); + const digits = upper.replace(/[^0-9]/g, ''); + // At least one digit and mostly-uppercase token (keygen style); human-ish + // mixed case is rejected. One digit is enough — the botnet also uses + // single-digit ids like "BKGL7DMF"; callsign shapes are excluded below. + if (letters.length < 3 || digits.length === 0) return false; + + // Mostly-uppercase token (keygen style); human-ish mixed case is rejected. + const upperCount = (letterChars.match(/[A-Z]/g) ?? []).length; + if (upperCount / letterChars.length < 0.7) return false; + + // Amateur-callsign shapes (M7GQW, G0SYR, MU6GBG, 2E0ABC, GW4ABC) are human. + if (/^[A-Z]{1,3}[0-9][A-Z]{1,4}$/.test(upper)) return false; + if (/^[0-9][A-Z][0-9][A-Z]{1,3}$/.test(upper)) return false; + + // Keygen strings are vowel-poor. + const vowels = (letters.match(/[AEIOUY]/g) ?? []).length; + if (vowels / letters.length > 0.34) return false; + + return true; +} diff --git a/backend/src/stats/statsRepository.test.ts b/backend/src/stats/statsRepository.test.ts index 428f339..2434d60 100644 --- a/backend/src/stats/statsRepository.test.ts +++ b/backend/src/stats/statsRepository.test.ts @@ -2,25 +2,10 @@ import assert from 'node:assert/strict'; import { test } from 'node:test'; import type { QueryResultRow } from 'pg'; import { - bindChartSnapshot, compareAggregateShadowRows, createStatsRepository, } from './statsRepository.js'; -test('pins repeated chart NOW expressions to one timestamp and parameter', () => { - const asOf = new Date('2026-09-23T10:15:00.000Z'); - const snapshot = bindChartSnapshot( - "SELECT * FROM packets WHERE time > NOW() - INTERVAL '24 hours' AND time <= NOW()", - ['ukmesh'], - asOf, - ); - assert.equal( - snapshot.text, - "SELECT * FROM packets WHERE time > $2::timestamptz - INTERVAL '24 hours' AND time <= $2::timestamptz", - ); - assert.deepEqual(snapshot.params, ['ukmesh', asOf.toISOString()]); -}); - test('multibyte fact cutover removes both raw decode scans and binds the privacy generation', async () => { const calls: Array<{ text: string; params?: unknown[] }> = []; const query = async ( @@ -195,89 +180,6 @@ test('canonical charts coalesce six high-volume dimensions into one maintained a sql.includes("time_bucket('1 hour', p.time) AS bucket, COUNT(*)::int AS count"))); }); -test('per-network shadowing runs before reads are enabled and stays off for other networks', async () => { - const calls: Array<{ text: string; params?: unknown[] }> = []; - const query = async ( - text: string, - params?: unknown[], - ): Promise<{ rows: T[] }> => { - calls.push({ text, params }); - return { rows: [] }; - }; - const shadowRepository = createStatsRepository({ - query, - networkFilters: () => ({ - params: ['ukmesh'], - packets: 'AND network = $1', - packetsAlias: (alias: string) => `AND ${alias}.network = $1`, - nodes: 'AND network = $1', - nodesAlias: (alias: string) => `AND ${alias}.network = $1`, - }), - aggregateShadowNetworks: ['ukmesh'], - }); - - await shadowRepository.fetchChartsData('ukmesh', undefined); - assert.ok(calls.some((call) => call.text.includes('FROM packet_hourly_stats'))); - assert.ok(calls.some((call) => call.text.includes('WITH scoped AS MATERIALIZED'))); - assert.ok(calls.every((call) => !/\bNOW\(\)/i.test(call.text))); - - calls.length = 0; - const readRepository = createStatsRepository({ - query, - networkFilters: () => ({ - params: ['ukmesh'], - packets: 'AND network = $1', - packetsAlias: (alias: string) => `AND ${alias}.network = $1`, - nodes: 'AND network = $1', - nodesAlias: (alias: string) => `AND ${alias}.network = $1`, - }), - aggregateReadNetworks: ['ukmesh'], - }); - await readRepository.fetchChartsData('ukmesh', undefined); - assert.ok(calls.some((call) => call.text.includes('FROM packet_hourly_stats'))); - - calls.length = 0; - await readRepository.fetchChartsData('test', undefined); - assert.equal(calls.some((call) => call.text.includes('FROM packet_hourly_stats')), false); -}); - -test('Compose aggregate switch accepts a comma-separated network value', async () => { - const previousReads = process.env['STATS_AGGREGATE_READS_ENABLED']; - const previousShadow = process.env['STATS_AGGREGATE_SHADOW_ENABLED']; - process.env['STATS_AGGREGATE_READS_ENABLED'] = 'ukmesh'; - process.env['STATS_AGGREGATE_SHADOW_ENABLED'] = 'false'; - try { - const calls: string[] = []; - const query = async ( - text: string, - ): Promise<{ rows: T[] }> => { - calls.push(text); - return { rows: [] }; - }; - const repository = createStatsRepository({ - query, - networkFilters: () => ({ - params: ['ukmesh'], - packets: 'AND network = $1', - packetsAlias: (alias: string) => `AND ${alias}.network = $1`, - nodes: 'AND network = $1', - nodesAlias: (alias: string) => `AND ${alias}.network = $1`, - }), - }); - await repository.fetchChartsData('ukmesh', undefined); - assert.ok(calls.some((sql) => sql.includes('FROM packet_hourly_stats'))); - - calls.length = 0; - await repository.fetchChartsData('test', undefined); - assert.equal(calls.some((sql) => sql.includes('FROM packet_hourly_stats')), false); - } finally { - if (previousReads === undefined) delete process.env['STATS_AGGREGATE_READS_ENABLED']; - else process.env['STATS_AGGREGATE_READS_ENABLED'] = previousReads; - if (previousShadow === undefined) delete process.env['STATS_AGGREGATE_SHADOW_ENABLED']; - else process.env['STATS_AGGREGATE_SHADOW_ENABLED'] = previousShadow; - } -}); - test('map summary uses the same coordinate, role, and 14-day freshness rules as the map', async () => { const calls: Array<{ text: string; params?: unknown[] }> = []; const query = async ( diff --git a/backend/src/stats/statsRepository.ts b/backend/src/stats/statsRepository.ts index 4a18b66..847d919 100644 --- a/backend/src/stats/statsRepository.ts +++ b/backend/src/stats/statsRepository.ts @@ -15,7 +15,6 @@ import { ensureMultibyteFactsCoverWindow } from './multibytePathFacts.js'; type QueryFn = ( text: string, params?: unknown[], - signal?: AbortSignal, ) => Promise<{ rows: T[] }>; type StatsRepositoryDeps = { @@ -23,8 +22,6 @@ type StatsRepositoryDeps = { query: QueryFn; aggregateReadsEnabled?: boolean; aggregateShadowEnabled?: boolean; - aggregateReadNetworks?: readonly string[]; - aggregateShadowNetworks?: readonly string[]; }; export type StatsRepository = ReturnType; @@ -54,40 +51,6 @@ function normalizedDimensionKey(row: QueryResultRow): string { )); } -function settingInteger(name: string, fallback: number, minimum: number, maximum: number): number { - const raw = process.env[name]; - if (raw === undefined || raw.trim() === '') return fallback; - const parsed = Number(raw); - if (!Number.isSafeInteger(parsed) || parsed < minimum || parsed > maximum) { - console.error(`[stats-query-budget] invalid ${name}; using ${fallback}`); - return fallback; - } - return parsed; -} - -export function bindChartSnapshot( - text: string, - params: unknown[], - asOf: Date, -): { text: string; params: unknown[] } { - if (!/\bNOW\(\)/i.test(text)) return { text, params }; - const snapshotParam = `$${params.length + 1}::timestamptz`; - return { - text: text.replace(/\bNOW\(\)/gi, snapshotParam), - params: [...params, asOf.toISOString()], - }; -} - -function networkSwitches(...values: Array): Set { - return new Set( - values - .flatMap((raw) => String(raw ?? '').split(',')) - .map((network) => network.trim().toLowerCase()) - .filter((network) => network !== 'true' && network !== 'false') - .filter(Boolean), - ); -} - /** * Shadow reads use one pinned cutoff, but a packet transaction that commits * between the aggregate and legacy snapshots can still create a tiny count @@ -139,9 +102,6 @@ export function compareAggregateShadowRows( export function createStatsRepository(deps: StatsRepositoryDeps) { const { networkFilters } = deps; - const queryBudgetMs = settingInteger('STATS_QUERY_DURATION_BUDGET_MS', 60_000, 1_000, 600_000); - const queryTimeoutMs = settingInteger('STATS_CHART_QUERY_TIMEOUT_MS', 120_000, 1_000, 300_000); - const maxResultRows = settingInteger('STATS_CHART_MAX_RESULT_ROWS', 100_000, 1, 1_000_000); const queryConcurrency = Math.max( 1, Math.min(8, Math.trunc(Number(process.env['STATS_DB_QUERY_CONCURRENCY'] ?? 2) || 2)), @@ -169,56 +129,14 @@ export function createStatsRepository(deps: StatsRepositoryDeps) { const query: QueryFn = async ( text: string, params?: unknown[], - parentSignal?: AbortSignal, ) => { - const controller = new AbortController(); - const startedAt = performance.now(); - let timeoutFired = false; - let acquired = false; - const abortFromParent = () => controller.abort(parentSignal?.reason); - if (parentSignal?.aborted) abortFromParent(); - else parentSignal?.addEventListener('abort', abortFromParent, { once: true }); - const timeout = setTimeout(() => { - timeoutFired = true; - controller.abort(new Error(`STATS_QUERY_TIMEOUT_${queryTimeoutMs}MS`)); - }, queryTimeoutMs); + await acquireQuerySlot(); try { - await acquireQuerySlot(); - acquired = true; - controller.signal.throwIfAborted(); - const result = await deps.query(text, params, controller.signal); - const durationMs = Math.round(performance.now() - startedAt); - const source = text.match(/\bFROM\s+([a-z_][a-z0-9_.]*)/i)?.[1] ?? 'unknown'; - if (durationMs > queryBudgetMs || result.rows.length > maxResultRows) { - console.error('[stats-query-budget] exceeded', { - source, - durationMs, - durationBudgetMs: queryBudgetMs, - returnedRows: result.rows.length, - returnedRowBudget: maxResultRows, - }); - throw new Error('STATS_QUERY_BUDGET_EXCEEDED'); - } - return result; - } catch (error) { - const durationMs = Math.round(performance.now() - startedAt); - if (timeoutFired || durationMs > queryBudgetMs) { - const source = text.match(/\bFROM\s+([a-z_][a-z0-9_.]*)/i)?.[1] ?? 'unknown'; - console.error('[stats-query-budget] query failed after budget', { - source, - durationMs, - durationBudgetMs: queryBudgetMs, - queryTimeoutMs, - }); - } - throw error; + return await deps.query(text, params); } finally { - clearTimeout(timeout); - parentSignal?.removeEventListener('abort', abortFromParent); - if (acquired) releaseQuerySlot(); + releaseQuerySlot(); } }; - const limitedQuery: QueryFn = query; async function loadChartSnapshot(scope: string, visibilityGeneration: number) { return loadStoredChartSnapshot(query, scope, visibilityGeneration); @@ -239,13 +157,9 @@ export function createStatsRepository(deps: StatsRepositoryDeps) { ); } - async function fetchObserverRegionSummary( - network: string | undefined, - observer: string | undefined, - queryFn: QueryFn = query, - ) { + async function fetchObserverRegionSummary(network: string | undefined, observer: string | undefined) { const filters = networkFilters(network, observer, { includePrivacy: false }); - return queryFn(` + return query(` SELECT COALESCE(NULLIF(TRIM(UPPER(p.iata)), ''), 'UNK') AS iata, COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.time > NOW() - INTERVAL '24 hours') AS packets_24h, @@ -315,28 +229,10 @@ export function createStatsRepository(deps: StatsRepositoryDeps) { transportCodes: QueryResultRow[]; }; - // Compose already forwards these two settings. Preserve `true` as global - // enablement while allowing a comma-separated per-network rollout. - const readSwitch = String(process.env['STATS_AGGREGATE_READS_ENABLED'] ?? '').trim().toLowerCase(); - const shadowSwitch = String(process.env['STATS_AGGREGATE_SHADOW_ENABLED'] ?? '').trim().toLowerCase(); - const aggregateReadNetworks = new Set([ - ...networkSwitches(process.env['STATS_AGGREGATE_READ_NETWORKS'], readSwitch), - ...(deps.aggregateReadNetworks ?? []).map((network) => network.trim().toLowerCase()).filter(Boolean), - ]); - const aggregateShadowNetworks = new Set([ - ...networkSwitches(process.env['STATS_AGGREGATE_SHADOW_NETWORKS'], shadowSwitch), - ...(deps.aggregateShadowNetworks ?? []).map((network) => network.trim().toLowerCase()).filter(Boolean), - ]); - const aggregateReadsEnabledFor = (network: string | undefined): boolean => ( - deps.aggregateReadsEnabled - ?? (readSwitch === 'true' - || aggregateReadNetworks.has((network ?? 'public').trim().toLowerCase())) - ); - const aggregateShadowEnabledFor = (network: string | undefined): boolean => ( - deps.aggregateShadowEnabled - ?? (shadowSwitch === 'true' - || aggregateShadowNetworks.has((network ?? 'public').trim().toLowerCase())) - ); + const aggregateReadsEnabled = deps.aggregateReadsEnabled + ?? process.env['STATS_AGGREGATE_READS_ENABLED'] === 'true'; + const aggregateShadowEnabled = deps.aggregateShadowEnabled + ?? process.env['STATS_AGGREGATE_SHADOW_ENABLED'] === 'true'; const aggregateShadowInFlight = new Map>(); const aggregateShadowLastStartedAt = new Map(); const aggregateShadowMinimumIntervalMs = 5 * 60_000; @@ -378,7 +274,6 @@ export function createStatsRepository(deps: StatsRepositoryDeps) { async function fetchAggregateChartParts( network: string | undefined, asOf: Date, - signal?: AbortSignal, ): Promise { const scope = aggregateScope(network); const filters = networkFilters(network, undefined); @@ -584,7 +479,6 @@ export function createStatsRepository(deps: StatsRepositoryDeps) { bounds.fullStart7d.toISOString(), bounds.fullEnd.toISOString(), ], - signal, ); const row = result.rows[0]; return { @@ -793,26 +687,6 @@ export function createStatsRepository(deps: StatsRepositoryDeps) { const filters = networkFilters(network, observer); const totalFilters = networkFilters(network, observer, { includePrivacy: false }); const aggregateAsOf = new Date(); - const chartAbort = new AbortController(); - const chartQuery: QueryFn = async ( - text: string, - params: unknown[] = filters.params, - signal: AbortSignal = chartAbort.signal, - ) => { - const snapshot = bindChartSnapshot(text, params, aggregateAsOf); - try { - return await limitedQuery(snapshot.text, snapshot.params, signal); - } catch (error) { - if (!chartAbort.signal.aborted) chartAbort.abort(error); - throw error; - } - }; - // Shadow the repository query helper for this function so every raw chart - // query, including observer summaries and fact readiness checks, shares one - // bounded asOf timestamp. - const query = chartQuery; - const aggregateReadsEnabled = aggregateReadsEnabledFor(network); - const aggregateShadowEnabled = aggregateShadowEnabledFor(network); const multibyteCutoffPlaceholder = `$${filters.params.length + 1}`; const multibyteGenerationPlaceholder = `$${filters.params.length + 2}`; const multibyteFactParams = [ @@ -944,31 +818,15 @@ export function createStatsRepository(deps: StatsRepositoryDeps) { ORDER BY b.day`, multibyteFactParams, ); - const aggregatePartsPromise = (aggregateReadsEnabled || aggregateShadowEnabled) && !observer - ? fetchAggregateChartParts(network, aggregateAsOf, chartAbort.signal) + const aggregatePartsPromise = aggregateReadsEnabled && !observer + ? fetchAggregateChartParts(network, aggregateAsOf) : null; - if (aggregatePartsPromise && aggregateReadsEnabled) { - void aggregatePartsPromise.catch((error: unknown) => { - if (!chartAbort.signal.aborted) chartAbort.abort(error); - }); - } - if (aggregatePartsPromise && aggregateShadowEnabled) { - void aggregatePartsPromise - .then((aggregate) => scheduleAggregateShadow(network, aggregateAsOf, aggregate)) - .catch((error: unknown) => { - console.warn('[stats-aggregate-shadow] aggregate fetch failed', { - network: network ?? 'public', - error: error instanceof Error ? error.message : String(error), - }); - }); - } - const aggregateReadPartsPromise = aggregateReadsEnabled ? aggregatePartsPromise : null; const aggregateRows = ( key: keyof AggregateChartParts, legacy: () => Promise<{ rows: QueryResultRow[] }>, ): Promise<{ rows: QueryResultRow[] }> => ( - aggregateReadPartsPromise - ? aggregateReadPartsPromise.then((parts) => ({ rows: parts[key] })) + aggregatePartsPromise + ? aggregatePartsPromise.then((parts) => ({ rows: parts[key] })) : legacy() ); @@ -1082,7 +940,7 @@ export function createStatsRepository(deps: StatsRepositoryDeps) { WHERE p.time > NOW() - INTERVAL '24 hours' AND p.src_node_id IS NOT NULL ${filters.packetsAlias('p')}) AS unique_radios_24h `, filters.params), - fetchObserverRegionSummary(network, observer, query), + fetchObserverRegionSummary(network, observer), query(` SELECT COALESCE(NULLIF(TRIM(UPPER(p.iata)), ''), 'UNK') AS iata, @@ -1207,6 +1065,12 @@ export function createStatsRepository(deps: StatsRepositoryDeps) { transportCodesResult, pathDecodeTrendResult, }; + if (aggregatePartsPromise && aggregateShadowEnabled) { + const aggregate = await aggregatePartsPromise; + // The comparison is bounded to the six switched dimensions, uses one + // materialized recent packet scan and runs off the response path. + scheduleAggregateShadow(network, aggregateAsOf, aggregate); + } return response; } @@ -1357,17 +1221,49 @@ export function createStatsRepository(deps: StatsRepositoryDeps) { ${filters.packetsAlias('p')} GROUP BY COALESCE(rx_alias.canonical_node_id, UPPER(BTRIM(p.rx_node_id))) ), + prefix_map AS MATERIALIZED ( + -- 3-byte path-hash prefix -> canonical node id, so relay evidence + -- (a repeater's prefix appearing in a packet's path_hashes) can be + -- attributed to the node that forwarded the packet. + SELECT LEFT(node_id, 6) AS prefix, node_id AS canonical_node_id + FROM node_identity_nodes + UNION + SELECT LEFT(source_node_id, 6) AS prefix, canonical_node_id + FROM node_identity_aliases + ), tx AS ( - SELECT COALESCE(src_alias.canonical_node_id, UPPER(BTRIM(p.src_node_id))) AS node_id, - COUNT(p.packet_hash)::text AS tx_24h, - MAX(p.time)::text AS last_tx - FROM packets p - LEFT JOIN node_identity_aliases src_alias - ON src_alias.source_node_id = UPPER(BTRIM(p.src_node_id)) - WHERE p.time > NOW() - INTERVAL '24 hours' - AND p.src_node_id IS NOT NULL - ${filters.packetsAlias('p')} - GROUP BY COALESCE(src_alias.canonical_node_id, UPPER(BTRIM(p.src_node_id))) + -- tx_24h = distinct packets the node transmitted in the last 24 hours: + -- * packets it originated (alias-resolved src rows), and + -- * packets it relayed (its 3-byte prefix present in path_hashes). + -- Counting originations only showed "tx 0" for repeaters that relayed + -- hundreds of packets/day but advertise rarely (2026-09-22 fix). + SELECT node_id, + COUNT(DISTINCT packet_hash)::text AS tx_24h, + MAX(time)::text AS last_tx + FROM ( + SELECT COALESCE(src_alias.canonical_node_id, UPPER(BTRIM(p.src_node_id))) AS node_id, + p.packet_hash, + p.time + FROM packets p + LEFT JOIN node_identity_aliases src_alias + ON src_alias.source_node_id = UPPER(BTRIM(p.src_node_id)) + WHERE p.time > NOW() - INTERVAL '24 hours' + AND p.src_node_id IS NOT NULL + ${filters.packetsAlias('p')} + UNION ALL + SELECT prefix_map.canonical_node_id AS node_id, + p.packet_hash, + p.time + FROM packets p + CROSS JOIN LATERAL unnest(p.path_hashes) AS path_element + JOIN prefix_map ON prefix_map.prefix = path_element + WHERE p.time > NOW() - INTERVAL '24 hours' + AND p.path_hash_size_bytes >= 3 + AND p.path_hashes IS NOT NULL + AND LENGTH(path_element) = 6 + ${filters.packetsAlias('p')} + ) attributed + GROUP BY node_id ) SELECT n.node_id, diff --git a/backend/src/tools/inventoryOwnerAuthorization.ts b/backend/src/tools/inventoryOwnerAuthorization.ts index 86e09d5..355678f 100644 --- a/backend/src/tools/inventoryOwnerAuthorization.ts +++ b/backend/src/tools/inventoryOwnerAuthorization.ts @@ -1,3 +1,7 @@ +import { + closeDb, + query, +} from '../db/index.js'; import { closeOwnerAuthDb, getOwnerAclReadiness, @@ -6,6 +10,10 @@ import { } from '../db/ownerAuth.js'; import { parseAcl, readAclFile, renderOwnerAcl } from '../mqtt/aclManager.js'; import { parseOwnerGrantConfig } from '../owner/ownerGrantConfig.js'; +import { getPublicFeedConfig } from '../owner/publicFeedConfig.js'; +import { + getActivePublicFeedReaderUsernames, +} from '../owner/publicFeedRepository.js'; import { buildOwnerInventoryBaseline, loadOwnerInventoryBaseline, @@ -27,18 +35,53 @@ async function main(): Promise { const aclContent = readAclFile(); const parsedAcl = parseAcl(aclContent); const configured = parseOwnerGrantConfig(String(process.env['OWNER_MQTT_USERNAME_MAP'] ?? '')); + const publicFeedConfig = getPublicFeedConfig(); + const publicFeedReaders = publicFeedConfig.configured + ? await getActivePublicFeedReaderUsernames(query) + : new Set(); + const explicitPublicFeedReaders = String(process.env['OWNER_ACL_PUBLIC_FEED_READ_USERS'] ?? '') + .split(',') + .map((value) => value.trim()) + .filter(Boolean); + const explicitPublicFeedReaderSet = new Set(explicitPublicFeedReaders); + const grants = snapshot.map((account) => { + const publicFeedRead = explicitPublicFeedReaderSet.has(account.mqttUsername) + || (account.isActive && publicFeedReaders.has(account.mqttUsername)); + return { + mqttUsername: account.mqttUsername, + nodeIds: account.isActive ? account.nodeIds : [], + ...(publicFeedRead ? { publicFeedRead: true } : {}), + }; + }); + const snapshotUsernames = new Set(grants.map((grant) => grant.mqttUsername)); + for (const mqttUsername of explicitPublicFeedReaders) { + if (snapshotUsernames.has(mqttUsername)) continue; + grants.push({ + mqttUsername, + nodeIds: [], + publicFeedRead: true, + }); + } const unmanagedUsers = String(process.env['OWNER_ACL_UNMANAGED_USERS'] ?? 'backend,test,test2') .split(',') .map((value) => value.trim()) .filter(Boolean); const rendered = renderOwnerAcl( aclContent, - snapshot.map((account) => ({ - mqttUsername: account.mqttUsername, - nodeIds: account.nodeIds, - })), + grants, unmanagedUsers, - String(process.env['OWNER_ACL_ALLOW_EMPTY_USERS'] ?? '').split(',').map((value) => value.trim()).filter(Boolean), + [ + ...String(process.env['OWNER_ACL_ALLOW_EMPTY_USERS'] ?? '') + .split(',') + .map((value) => value.trim()) + .filter(Boolean), + ...explicitPublicFeedReaders, + ], + [String(process.env['MQTT_USERNAME'] ?? 'backend').trim() || 'backend'], + String(process.env['OWNER_ACL_SYSTEM_READWRITE_TOPICS'] ?? 'public/#') + .split(',') + .map((value) => value.trim()) + .filter(Boolean), ); const configKeys = new Set(configured.map((grant) => `${grant.mqttUsername}\0${grant.nodeId}`)); const legacyAclGrants = parsedAcl.stanzas.flatMap((stanza) => @@ -117,4 +160,7 @@ main() console.error('[owner-auth-inventory]', error.message); process.exitCode = 1; }) - .finally(() => closeOwnerAuthDb().catch(() => undefined)); +.finally(async () => { + await closeDb().catch(() => undefined); + await closeOwnerAuthDb().catch(() => undefined); +}); diff --git a/backend/src/tools/pathAccuracy.test.ts b/backend/src/tools/pathAccuracy.test.ts new file mode 100644 index 0000000..9ee5cb8 --- /dev/null +++ b/backend/src/tools/pathAccuracy.test.ts @@ -0,0 +1,22 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { compareAccuracy } from './pathAccuracy.js'; + +const result = { + summary: { snapshotSha256: 'frozen', split: 'test', mode: 'single' }, + cases: [{ packetHash: 'packet', receiver: 'rx', source: null, truth: ['A', 'B'], prediction: ['A', 'C'] }], +}; + +test('accuracy comparisons reject changed snapshots, missing cases or changed labels', () => { + assert.throws(() => compareAccuracy(result, { ...result, summary: { ...result.summary, snapshotSha256: 'different' } }), /snapshotSha256/); + assert.throws(() => compareAccuracy(result, { ...result, cases: [] }), /cases/); + assert.throws(() => compareAccuracy(result, { ...result, cases: [{ ...result.cases[0]!, truth: ['A', 'C'] }] }), /ground truth/); +}); + +test('paired comparison counts a correction without excluding the previously wrong hop', () => { + const comparison = compareAccuracy(result, { ...result, cases: [{ ...result.cases[0]!, prediction: ['A', 'B'] }] }); + assert.equal(comparison.improved, 1); + assert.equal(comparison.regressed, 0); + assert.equal(comparison.hopGainPercentagePoints, 50); + assert.equal(comparison.routeGainPercentagePoints, 100); +}); diff --git a/backend/src/tools/pathAccuracy.ts b/backend/src/tools/pathAccuracy.ts new file mode 100644 index 0000000..fb18079 --- /dev/null +++ b/backend/src/tools/pathAccuracy.ts @@ -0,0 +1,342 @@ +/** Read-only, frozen-snapshot comparison of live decoders on hidden three-byte routes. + * + * Capture: node --env-file=../.env --import tsx src/tools/pathAccuracy.ts capture /tmp/path-snapshot.json + * Replay: node --import tsx src/tools/pathAccuracy.ts replay /tmp/path-snapshot.json /tmp/result.json [decoder.ts] [dev|test] + * A baseline decoder can be taken from a git archive with its relative imports intact. + */ +import { readFile, writeFile, mkdir, symlink } from 'node:fs/promises'; +import { createHash } from 'node:crypto'; +import { execFileSync } from 'node:child_process'; +import { resolve, join } from 'node:path'; +import { pathToFileURL, fileURLToPath } from 'node:url'; +import pg from 'pg'; +import { buildChampionPriors, type ChampionPriorRows } from '../path-beta/championPriors.js'; +import type { BetaResolveContext, MeshNode, PathPacket, LinkMetrics } from '../path-beta/types.js'; +import type { BetaObserverEntry } from '../path-beta/sharedDecoder.js'; +import { linkKey } from '../path-beta/geometry.js'; +import { expandResolverScope } from '../networks.js'; + +type Observation = PathPacket & { time: string; split: number; network: string }; +type Snapshot = { + capturedAt: string; + network: string; + windowDays: number; + calibration: Record[]; + nodes: MeshNode[]; + links: (LinkMetrics & { node_a_id: string; node_b_id: string })[]; + prefixes: { prefix: string; prev_prefix: string; receiver_region: string; node_id: string; count: number }[]; + transitions: { from_node_id: string; to_node_id: string; receiver_region: string; count: number; probability: number }[]; + priors: ChampionPriorRows; + observations: Observation[]; +}; + +async function capture(file: string) { + const client = new pg.Client({ + connectionString: process.env['DATABASE_URL'], + host: process.env['PGHOST'] ?? '127.0.0.1', + user: process.env['POSTGRES_USER'] ?? 'meshcore', + password: process.env['POSTGRES_PASSWORD'], + database: process.env['POSTGRES_DB'] ?? 'meshcore', + application_name: 'path-accuracy-readonly', + statement_timeout: 120_000, + }); + await client.connect(); + try { + await client.query('BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY'); + const network = process.env['PATH_ACCURACY_NETWORK'] ?? 'ukmesh'; + const scope = network === 'all' ? null : expandResolverScope(network); + const windowDays = Number(process.env['PATH_ACCURACY_DAYS'] ?? 14); + if (!Number.isInteger(windowDays) || windowDays < 1 || windowDays > 45) throw new Error('invalid window days'); + const rows = async (sql: string, params: unknown[] = [network]): Promise => + (await client.query(sql, params)).rows; + const capturedAt = (await rows<{ t: string }>('SELECT now()::text AS t', []))[0]!.t; + const calibration = await rows('SELECT * FROM path_model_calibration WHERE network = $1'); + const nodes = await rows(`SELECT node_id, name, lat, lon, iata, role, elevation_m, last_seen::text + FROM nodes WHERE ($1::text[] IS NULL OR network = ANY($1)) AND (name IS NULL OR name NOT LIKE '%🚫%') ORDER BY node_id`, [scope]); + const links = await rows(`SELECT nl.node_a_id, nl.node_b_id, nl.observed_count, + nl.multibyte_observed_count, nl.itm_path_loss_db, nl.itm_viable, nl.count_a_to_b, nl.count_b_to_a + FROM node_links nl JOIN nodes a ON a.node_id = nl.node_a_id JOIN nodes b ON b.node_id = nl.node_b_id + WHERE ($1::text[] IS NULL OR (a.network = ANY($1) AND b.network = ANY($1))) AND (nl.itm_viable IS NOT NULL OR nl.force_viable = true) + AND (a.name IS NULL OR a.name NOT LIKE '%🚫%') AND (b.name IS NULL OR b.name NOT LIKE '%🚫%')`, [scope]); + const prefixes = await rows(`SELECT prefix, prev_prefix, receiver_region, node_id, count + FROM path_prefix_priors WHERE network = $1 ORDER BY count DESC LIMIT 6000`); + const transitions = await rows(`SELECT from_node_id, to_node_id, receiver_region, count, probability + FROM path_transition_priors WHERE network = $1 ORDER BY count DESC`); + const priors: ChampionPriorRows = { + positionPrefixes: await rows('SELECT prefix, position, node_id, count FROM path_position_prefix_priors WHERE network = $1'), + corridors: await rows('SELECT src_node_id, rx_node_id, position, node_id, count FROM path_corridor_priors WHERE network = $1'), + transitions, + positionTransitions: await rows('SELECT position, from_node_id, to_node_id, count FROM path_position_transition_priors WHERE network = $1'), + }; + // The learner trains only on buckets 0..69. Split at packet hash so every + // observation of a held-out packet is excluded from all learned priors. + const gold = await rows<{ packet_hash: string }>(`SELECT DISTINCT packet_hash FROM packets + WHERE ($1::text[] IS NULL OR network = ANY($1)) AND time > now() - $2 * interval '1 day' + AND path_hash_size_bytes = 3 AND cardinality(path_hashes) >= 2 + AND mod(hashtext(packet_hash)::bigint + 2147483648, 100) >= 70`, [scope, windowDays]); + console.log(`capturing ${gold.length} held-out packet hashes`); + const observations = await rows(`SELECT time::text, network, packet_hash, rx_node_id, src_node_id, + packet_type, route_type, payload->>'direction' AS direction, hop_count, path_hashes, path_hash_size_bytes, + mod(hashtext(packet_hash)::bigint + 2147483648, 100)::int AS split + FROM packets WHERE ($1::text[] IS NULL OR network = ANY($1)) AND time > now() - $2 * interval '1 day' + AND packet_hash = ANY($3::text[]) ORDER BY time, packet_hash, rx_node_id`, + [scope, windowDays, gold.map((row) => row.packet_hash)]); + const snapshot: Snapshot = { capturedAt, network, windowDays, calibration, nodes, links, prefixes, transitions, priors, observations }; + await writeFile(file, JSON.stringify(snapshot)); + await client.query('COMMIT'); + console.log(JSON.stringify({ file, nodes: nodes.length, links: links.length, observations: observations.length })); + } finally { + await client.end(); + } +} + +async function archiveBaseline(directory: string, ref: string) { + const cwd = fileURLToPath(new URL('../../../', import.meta.url)); + const revision = execFileSync('git', ['rev-parse', '--verify', `${ref}^{commit}`], { cwd, encoding: 'utf8' }).trim(); + const destination = resolve(directory); + await mkdir(destination); + const archive = execFileSync('git', ['archive', revision, 'backend/src', 'backend/package.json'], { cwd, maxBuffer: 64 * 1024 * 1024 }); + execFileSync('tar', ['-x', '-C', destination], { input: archive }); + await symlink(fileURLToPath(new URL('../../node_modules', import.meta.url)), join(destination, 'backend/node_modules')); + const resolver = join(destination, 'backend/src/path-beta/resolver.ts'); + const original = await readFile(resolver, 'utf8'); + // Expose an existing pure function to the replay harness. Its body and all + // scoring/configuration files remain byte-for-byte at the baseline commit. + await writeFile(resolver, original.replace(/^function preparePacketObservation\(/m, 'export function preparePacketObservation(')); + const manifest = { revision, resolverSha256: createHash('sha256').update(original).digest('hex'), modification: 'export preparePacketObservation only' }; + await writeFile(join(destination, 'baseline.json'), JSON.stringify(manifest, null, 2)); + console.log(JSON.stringify({ directory: destination, ...manifest })); +} + +function makeContext(snapshot: Snapshot): BetaResolveContext { + const choiceCounts = new Map(); + const totals = new Map(); + for (const row of snapshot.prefixes) { + const group = `${row.receiver_region}|${row.prefix.toUpperCase().slice(0, 2)}|${row.prev_prefix.toUpperCase().slice(0, 2)}`; + const key = `${group}|${row.node_id}`; + choiceCounts.set(key, (choiceCounts.get(key) ?? 0) + Number(row.count)); + totals.set(group, (totals.get(group) ?? 0) + Number(row.count)); + } + const prefixProbabilities = new Map([...choiceCounts].map(([key, count]) => [key, count / totals.get(key.slice(0, key.lastIndexOf('|')))!])); + return { + loadedAt: Date.parse(snapshot.capturedAt), visibilityGeneration: 0, + nodesById: new Map(snapshot.nodes.map((node) => [node.node_id, node])), + repeaterNodes: snapshot.nodes.filter((n) => n.lat != null && n.lon != null && n.lat !== 0 && n.lon !== 0 && (n.role == null || n.role === 2)), + linkMetrics: new Map(snapshot.links.map((link) => [linkKey(link.node_a_id, link.node_b_id), link])), + learningModel: { + champion: buildChampionPriors(snapshot.priors), prefixProbabilities, + transitionProbabilities: new Map(snapshot.transitions.slice(0, 6000).map((r) => [`${r.receiver_region}|${r.from_node_id}|${r.to_node_id}`, Number(r.probability)])), + edgeScores: new Map(), motifProbabilities: new Map(), confidenceScale: 1, confidenceBias: 0, bucketHours: 6, + }, + }; +} + +/** Replay the former lazy SQL adapter against the same immutable evidence. */ +function snapshotQuery(snapshot: Snapshot, observations: Observation[]) { + const membership = new WeakMap>(); + return async (sql: string, params: unknown[] = []) => { + const includes = (ids: unknown, id: string) => { + if (!Array.isArray(ids)) return false; + let set = membership.get(ids); + if (!set) { set = new Set(ids); membership.set(ids, set); } + return set.has(id); + }; + if (/FROM packets\b/i.test(sql)) { + if (/SELECT DISTINCT rx_node_id, hop_count/i.test(sql)) return { rows: observations }; + const byObserver = new Map(); + for (const row of [...observations].sort((a, b) => (b.path_hashes?.length ?? 0) - (a.path_hashes?.length ?? 0))) { + if (!byObserver.has(row.rx_node_id)) byObserver.set(row.rx_node_id, row); + } + return { rows: [...byObserver.values()] }; + } + if (/FROM nodes\b/i.test(sql)) return { rows: snapshot.nodes.filter((node) => /node_id = ANY\(\$1\)/i.test(sql) + ? includes(params[0], node.node_id) && node.lat != null && node.lon != null && node.lat !== 0 && node.lon !== 0 + : params.slice(1, -1).some((prefix) => typeof prefix === 'string' && node.node_id.toUpperCase().startsWith(prefix.replace('%', '')))) }; + if (/FROM node_links\b/i.test(sql)) return { rows: snapshot.links.filter((link) => includes(params[0], link.node_a_id) && includes(params[0], link.node_b_id)) }; + if (/FROM path_position_prefix_priors\b/i.test(sql)) return { rows: snapshot.priors.positionPrefixes.filter((row) => includes(params[1], row.prefix)) }; + if (/FROM path_corridor_priors\b/i.test(sql)) return { rows: snapshot.priors.corridors.filter((row) => includes(params[1], row.src_node_id) && includes(params[2], row.rx_node_id)) }; + if (/FROM path_position_transition_priors\b/i.test(sql)) return { rows: snapshot.priors.positionTransitions.filter((row) => includes(params[1], row.from_node_id) && includes(params[1], row.to_node_id)) }; + if (/FROM path_transition_priors\b/i.test(sql)) { + const counts = new Map(); + for (const row of snapshot.transitions) { + if (!includes(params[1], row.from_node_id) || !includes(params[1], row.to_node_id)) continue; + const key = `${row.from_node_id}|${row.to_node_id}`; + const previous = counts.get(key); + counts.set(key, { from_node_id: row.from_node_id, to_node_id: row.to_node_id, count: (previous?.count ?? 0) + Number(row.count) }); + } + return { rows: [...counts.values()] }; + } + if (/FROM path_model_calibration\b/i.test(sql)) return { rows: snapshot.calibration }; + throw new Error(`Unimplemented snapshot query: ${sql}`); + }; +} + +async function replay(file: string, output: string, decoderFile?: string, split = 'dev') { + if (!['dev', 'test'].includes(split)) throw new Error('split must be dev or test'); + const raw = await readFile(file, 'utf8'); + const snapshot = JSON.parse(raw) as Snapshot; + const context = makeContext(snapshot); + const decoder = await import(pathToFileURL(resolve(decoderFile ?? 'src/path-beta/sharedDecoder.ts')).href) as + typeof import('../path-beta/sharedDecoder.js') & Partial; + const preparation = await import(new URL(decoder.lazyResolvePath ? '../path-beta/resolver.ts' : './resolver.ts', + pathToFileURL(resolve(decoderFile ?? 'src/path-beta/sharedDecoder.ts'))).href) as typeof import('../path-beta/resolver.js'); + const mode = process.env['PATH_ACCURACY_MODE'] === 'multi' ? 'multi' : 'single'; + const threeByteIndex = new Map(); + for (const node of snapshot.nodes) { + const hash = node.node_id.toUpperCase().slice(0, 6); + threeByteIndex.set(hash, [...(threeByteIndex.get(hash) ?? []), node]); + } + const byPacket = new Map(); + for (const obs of snapshot.observations) { + if ((obs.split < 85) !== (split === 'dev')) continue; + byPacket.set(obs.packet_hash, [...(byPacket.get(obs.packet_hash) ?? []), obs]); + } + let excluded = 0; + const cases: { packetHash: string; receiver: string; truth: string[]; prediction: (string | null)[]; ms: number; rawMs: number; rawCorrect: boolean; source: string | null }[] = []; + for (const [packetHash, observations] of byPacket) { + const candidates = observations.filter((row) => row.path_hash_size_bytes === 3) + .sort((a, b) => (b.path_hashes?.length ?? 0) - (a.path_hashes?.length ?? 0) || b.time.localeCompare(a.time)); + let selected: { obs: Observation; hashes: string[]; truth: string[]; rx: MeshNode } | undefined; + for (const obs of candidates) { + if (obs.route_type != null && obs.route_type !== 0 && obs.route_type !== 1) continue; + const rx = context.nodesById.get(obs.rx_node_id ?? ''); + if (!rx || rx.lat == null || rx.lon == null || rx.lat === 0 || rx.lon === 0) continue; + let hashes = (obs.path_hashes ?? []).map((h) => h.toUpperCase()); + if (hashes.some((h) => !/^[0-9A-F]{6}$/.test(h))) continue; + if (obs.direction !== 'rx' && hashes.length > 1 && rx.role === 2 && rx.node_id.toUpperCase().startsWith(hashes.at(-1)!)) hashes = hashes.slice(0, -1); + if (hashes.length < 2 || hashes.length > 25) continue; + const nodes = hashes.map((h) => threeByteIndex.get(h) ?? []); + if (nodes.some((matches) => matches.length !== 1 || matches[0]!.lat == null || matches[0]!.lon == null)) continue; + selected = { obs, rx, hashes, truth: nodes.map((n) => n[0]!.node_id) }; + break; + } + if (!selected) { excluded++; continue; } + const { obs, rx, truth } = selected; + const decode = async (size: number) => { + const visible = (mode === 'multi' ? observations.filter((row) => Math.abs(Date.parse(row.time) - Date.parse(obs.time)) <= 60_000) : [obs]) + .map((row) => ({ ...row, path_hashes: row.path_hashes?.map((h) => size === 1 ? h.slice(0, 2) : h) ?? null, + path_hash_size_bytes: size === 1 ? 1 : row.path_hash_size_bytes })); + if (decoder.lazyResolvePath) { + const query = snapshotQuery(snapshot, visible); + const result = await decoder.lazyResolvePath(packetHash, snapshot.network, + query as Parameters[2], { referenceTime: obs.time }); + const resolved = result?.paths.find((p) => p.observerIds.includes(rx.node_id)); + return new Map(resolved?.canonicalPath.filter((h) => !h.isObserver).map((h) => [h.position, h.nodeId]) ?? []); + } + const byObserver = new Map>(); + for (const row of visible) { + const observer = context.nodesById.get(row.rx_node_id ?? ''); + if (!observer || observer.lat == null || observer.lon == null) continue; + const prepared = preparation.preparePacketObservation(row, observer); + if (prepared.ignoreForPathing || prepared.hops.length === 0) continue; + const previous = byObserver.get(observer.node_id); + byObserver.set(observer.node_id, previous ? preparation.mergeObserverEvidence(previous, prepared) : prepared); + } + const entries: BetaObserverEntry[] = preparation.prepareObservationEntries + ? preparation.prepareObservationEntries(visible, context) + : [...byObserver.values()].map((p) => ({ + observerId: p.rx!.node_id, rx: p.rx!, hashes: p.hashes, hops: p.hops, packet: p.packet, + })); + const target = preparation.preparePacketObservation({ ...obs, path_hashes: obs.path_hashes?.map((h) => size === 1 ? h.slice(0, 2) : h) ?? null, + path_hash_size_bytes: size }, rx); + const groups = decoder.groupCompatibleObservations(entries); + const group = groups.find((g) => g.members.some((e) => e.observerId === rx.node_id + && e.hops.length === target.hops.length && e.hops.every((h, i) => h === target.hops[i]))) + ?? groups.find((g) => g.members.some((e) => e.observerId === rx.node_id)); + if (!group) return new Map(); + return new Map([...decoder.decodeBetaCanonicalGroup(group, context).hops] + .map(([position, hop]) => [position, hop.nodeId])); + }; + const start = performance.now(); + const result = await decode(1); + const ms = performance.now() - start; + const rawStart = performance.now(); + const rawResult = await decode(3); + const rawMs = performance.now() - rawStart; + cases.push({ packetHash, receiver: rx.node_id, source: obs.src_node_id, truth, + prediction: truth.map((_, i) => result.get(i) ?? null), ms, rawMs, + rawCorrect: truth.every((id, i) => rawResult.get(i) === id) }); + } + const routeCorrect = cases.filter((c) => c.truth.every((id, i) => c.prediction[i] === id)).length; + const totalHops = cases.reduce((n, c) => n + c.truth.length, 0); + const correctHops = cases.reduce((n, c) => n + c.truth.filter((id, i) => c.prediction[i] === id).length, 0); + const times = cases.map((c) => c.ms).sort((a, b) => a - b); + const summary = { snapshotSha256: createHash('sha256').update(raw).digest('hex'), capturedAt: snapshot.capturedAt, + decoder: decoderFile ?? 'current', split, mode, routes: cases.length, excluded, routeCorrect, totalHops, correctHops, + routeAccuracy: routeCorrect / cases.length, hopAccuracy: correctHops / totalHops, + rawThreeByteCorrect: cases.filter((c) => c.rawCorrect).length, + uniqueRoutes: new Set(cases.map((c) => c.truth.join('|'))).size, + medianMs: times[Math.floor(times.length * 0.5)], p95Ms: times[Math.floor(times.length * 0.95)], p99Ms: times[Math.floor(times.length * 0.99)] }; + const sourceRoot = new URL('../', pathToFileURL(resolve(decoderFile ?? 'src/path-beta/sharedDecoder.ts'))); + const sourceFiles = ['path-beta/sharedDecoder.ts', 'path-beta/resolver.ts', 'path-core/decoder.ts', 'path-shared/scoring.ts', 'platform/config/pathing.ts']; + const sourceHashes = Object.fromEntries(await Promise.all(sourceFiles.map(async (name) => [name, + createHash('sha256').update(await readFile(new URL(name, sourceRoot))).digest('hex')]))); + await writeFile(output, JSON.stringify({ summary, sourceHashes, cases }, null, 2)); + console.log(JSON.stringify(summary, null, 2)); +} + +type AccuracyCase = { packetHash: string; receiver: string; source: string | null; truth: string[]; prediction: (string | null)[] }; +type AccuracyResult = { summary: { snapshotSha256: string; split: string; mode: string; [key: string]: unknown }; cases: AccuracyCase[] }; + +export function compareAccuracy(baseline: AccuracyResult, current: AccuracyResult) { + for (const key of ['snapshotSha256', 'split', 'mode'] as const) { + if (baseline.summary[key] !== current.summary[key]) throw new Error(`comparison mismatch: ${key}`); + } + if (baseline.cases.length === 0 || baseline.cases.length !== current.cases.length) throw new Error('comparison mismatch: cases'); + const oldCases = new Map(baseline.cases.map((c) => [`${c.packetHash}|${c.receiver}`, c])); + let improved = 0; + let regressed = 0; + let hopGain = 0; + let totalHops = 0; + const routeClusters = new Map(); + const strata: Record = {}; + for (const next of current.cases) { + const previous = oldCases.get(`${next.packetHash}|${next.receiver}`); + if (!previous || JSON.stringify(previous.truth) !== JSON.stringify(next.truth)) throw new Error('comparison mismatch: ground truth'); + const before = previous.truth.every((id, i) => previous.prediction[i] === id); + const after = next.truth.every((id, i) => next.prediction[i] === id); + if (after && !before) improved++; + if (before && !after) regressed++; + totalHops += next.truth.length; + hopGain += next.truth.filter((id, i) => next.prediction[i] === id).length - previous.truth.filter((id, i) => previous.prediction[i] === id).length; + const key = next.truth.join('|'); + routeClusters.set(key, [...(routeClusters.get(key) ?? []), Number(after) - Number(before)]); + for (const label of [next.source ? 'known-source' : 'unknown-source', next.truth.length <= 3 ? 'hops-2-3' : next.truth.length <= 6 ? 'hops-4-6' : next.truth.length <= 12 ? 'hops-7-12' : 'hops-13+']) { + const bucket = strata[label] ?? { routes: 0, baselineCorrect: 0, currentCorrect: 0 }; + bucket.routes++; bucket.baselineCorrect += Number(before); bucket.currentCorrect += Number(after); + strata[label] = bucket; + } + } + // Resample unique relay chains, giving recurrent copies of the same route + // one vote. This is stricter than treating every packet as independent. + const means = [...routeClusters.values()].map((values) => values.reduce((a, b) => a + b, 0) / values.length); + let seed = 0x5eeda11; + const random = () => { seed ^= seed << 13; seed ^= seed >>> 17; seed ^= seed << 5; return (seed >>> 0) / 2 ** 32; }; + const bootstrap: number[] = []; + for (let iteration = 0; iteration < 2000; iteration++) { + let sum = 0; + for (let i = 0; i < means.length; i++) sum += means[Math.floor(random() * means.length)]!; + bootstrap.push(100 * sum / means.length); + } + bootstrap.sort((a, b) => a - b); + return { baseline: baseline.summary, current: current.summary, improved, regressed, + routeGainPercentagePoints: 100 * (improved - regressed) / current.cases.length, + hopGainPercentagePoints: 100 * hopGain / totalHops, + uniqueRouteGain95IntervalPercentagePoints: [bootstrap[50], bootstrap[1950]], strata }; +} + +async function main() { + const [command, file, output, decoder, split] = process.argv.slice(2); + if (!file) throw new Error('expected capture, baseline, replay or compare; see docs/pathing.md'); + if (command === 'capture') await capture(file); + else if (command === 'baseline' && output) await archiveBaseline(file, output); + else if (command === 'replay' && output) await replay(file, output, decoder === 'current' ? undefined : decoder, split); + else if (command === 'compare' && output && decoder) { + const result = compareAccuracy(JSON.parse(await readFile(file, 'utf8')), JSON.parse(await readFile(output, 'utf8'))); + await writeFile(decoder, JSON.stringify(result, null, 2)); + console.log(JSON.stringify(result, null, 2)); + if (result.routeGainPercentagePoints <= 0 || result.hopGainPercentagePoints <= 0) process.exitCode = 1; + } else throw new Error('invalid command'); +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) await main(); diff --git a/backend/src/tools/prepareScotmesh.ts b/backend/src/tools/prepareScotmesh.ts new file mode 100644 index 0000000..6baab13 --- /dev/null +++ b/backend/src/tools/prepareScotmesh.ts @@ -0,0 +1,48 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { generatePacketShareIdentity } from '../owner/packetShareAuth.js'; +import { getOwnerNodeIdsForUsername, closeOwnerAuthDb } from '../db/ownerAuth.js'; + +async function main(): Promise { + const [nodeIdArgument, outputFile] = process.argv.slice(2); + const nodeId = nodeIdArgument?.trim().toUpperCase(); + if (!nodeId || !/^[0-9A-F]{64}$/.test(nodeId) || !outputFile) { + throw new Error('Usage: prepareScotmesh NODE_ID OUTPUT_FILE'); + } + try { + const nodeIds = await getOwnerNodeIdsForUsername('node1'); + if (!nodeIds.includes(nodeId)) throw new Error('Node is not verified for node1'); + // Opening exclusively prevents an accidental rerun from rotating an identity. + await fs.mkdir(path.dirname(outputFile), { recursive: true, mode: 0o700 }); + const file = await fs.open(outputFile, 'wx', 0o600); + try { + const identity = await generatePacketShareIdentity(); + const config = { + destination_id: 'scotmesh', + display_name: 'ScotMesh', + description: 'MME; received packets and own transmitted adverts', + broker_url: 'wss://scotmesh-mqtt.mm7roq.compute.oarc.uk:443/', + topic_template: 'meshcore/MME/{observerId}/packets', + allowed_owner_username: 'node1', + allowed_node_id: nodeId, + auth_mode: 'meshcore_jwt', + auth_audience: 'scotmesh-mqtt.mm7roq.compute.oarc.uk', + tx_mode: 'advert', + broker_username: identity.username, + broker_password_ciphertext: identity.privateKeyCiphertext, + enabled: true, + }; + await file.writeFile(`${JSON.stringify(config, null, 2)}\n`); + console.log(JSON.stringify({ destination: 'scotmesh', owner: 'node1', nodeId, username: identity.username, outputFile })); + } finally { + await file.close(); + } + } finally { + await closeOwnerAuthDb(); + } +} + +main().catch((error: unknown) => { + console.error('[scotmesh] preparation failed:', error instanceof Error ? error.message : error); + process.exitCode = 1; +}); diff --git a/backend/src/tools/provisionScotmesh.ts b/backend/src/tools/provisionScotmesh.ts new file mode 100644 index 0000000..d9cbc8d --- /dev/null +++ b/backend/src/tools/provisionScotmesh.ts @@ -0,0 +1,61 @@ +import fs from 'node:fs/promises'; +import pg from 'pg'; +import { closeOwnerAuthDb, getOwnerNodeIdsForUsername } from '../db/ownerAuth.js'; +import { databaseConfig } from '../platform/config/database.js'; +import { packetShareCredentials } from '../owner/packetShareAuth.js'; +import { destinationConfigured, type PacketShareDestinationRow } from '../owner/packetShareConfig.js'; +import { hasPacketShareEncryptionKey } from '../owner/packetShareCrypto.js'; + +async function main(): Promise { + const [inputFile] = process.argv.slice(2); + if (!inputFile) throw new Error('Usage: provisionScotmesh CONFIG_FILE'); + const pool = new pg.Pool({ + connectionString: process.env['DATABASE_URL'], + max: 1, + options: databaseConfig.schema ? `-c search_path=${databaseConfig.schema},public` : undefined, + }); + try { + const destination = JSON.parse(await fs.readFile(inputFile, 'utf8')) as PacketShareDestinationRow; + if (destination.destination_id !== 'scotmesh' + || destination.allowed_owner_username !== 'node1' + || !destination.allowed_node_id || !/^[0-9A-F]{64}$/.test(destination.allowed_node_id) + || destination.broker_url !== 'wss://scotmesh-mqtt.mm7roq.compute.oarc.uk:443/' + || destination.topic_template !== 'meshcore/MME/{observerId}/packets' + || destination.auth_mode !== 'meshcore_jwt' + || destination.auth_audience !== 'scotmesh-mqtt.mm7roq.compute.oarc.uk' + || destination.tx_mode !== 'advert' + || !destinationConfigured(destination, hasPacketShareEncryptionKey())) { + throw new Error('Invalid node1 ScotMesh configuration'); + } + if (!(await getOwnerNodeIdsForUsername('node1')).includes(destination.allowed_node_id)) { + throw new Error('Observer is no longer verified for node1'); + } + // Decrypt and validate the persisted identity before installing credentials. + await packetShareCredentials(destination); + await pool.query( + `INSERT INTO owner_packet_share_destinations ( + destination_id, display_name, description, broker_url, topic_template, + broker_username, broker_password_ciphertext, allowed_owner_username, + allowed_node_id, auth_mode, auth_audience, tx_mode, enabled + ) VALUES ('scotmesh', 'ScotMesh', $1, $2, $3, $4, $5, 'node1', $6, 'meshcore_jwt', $7, 'advert', TRUE) + ON CONFLICT (destination_id) DO UPDATE SET + display_name = EXCLUDED.display_name, description = EXCLUDED.description, + broker_url = EXCLUDED.broker_url, topic_template = EXCLUDED.topic_template, + broker_username = EXCLUDED.broker_username, broker_password_ciphertext = EXCLUDED.broker_password_ciphertext, + allowed_owner_username = EXCLUDED.allowed_owner_username, allowed_node_id = EXCLUDED.allowed_node_id, + auth_mode = EXCLUDED.auth_mode, auth_audience = EXCLUDED.auth_audience, + tx_mode = EXCLUDED.tx_mode, enabled = TRUE, updated_at = NOW()`, + [destination.description, destination.broker_url, destination.topic_template, + destination.broker_username, destination.broker_password_ciphertext, + destination.allowed_node_id, destination.auth_audience], + ); + console.log(JSON.stringify({ destination: 'scotmesh', owner: 'node1', nodeId: destination.allowed_node_id, iata: 'MME' })); + } finally { + await Promise.all([pool.end(), closeOwnerAuthDb()]); + } +} + +main().catch((error: unknown) => { + console.error('[scotmesh] provisioning failed:', error instanceof Error ? error.message : error); + process.exitCode = 1; +}); diff --git a/backend/src/tools/rebuildRegionParticipation.ts b/backend/src/tools/rebuildRegionParticipation.ts new file mode 100644 index 0000000..c01e105 --- /dev/null +++ b/backend/src/tools/rebuildRegionParticipation.ts @@ -0,0 +1,405 @@ +/** + * Rebuild the region-scope participation tables. + * + * For every distinct transport-routed (route 0/3) packet in the window: + * - decode the region via the candidate-name transport-code probe + * - collect relay path elements (union across ALL observation rows) + * - resolve elements to nodes (unique prefix match only; drop ambiguous) + * Then tally node -> region -> distinct packets, plus unscoped relay evidence + * (route 1, multibyte, 7d) for the green/red classification, and refresh: + * region_scope_stats, region_scope_members, repeater_scope_status, + * region_scope_run + * + * Membership floor: a node counts as a region member only when it relayed at + * least MEMBER_MIN_SHARE_PCT (regions/memberShare.ts, currently 10%) of that + * region's scoped packets. Token relays (the odd packet out of thousands) are + * excluded from member lists, member counts and per-node region maps so the + * region shapes and lists reflect each region's actual relay pattern. + * + * Run from backend/ (dist build): node dist/tools/rebuildRegionParticipation.js + * or via: npm run regions:rebuild + * + * Env: + * DATABASE_URL required (same as the backend) + * REGION_SCOPE_WINDOW_DAYS scoped window in days (default 92, 7..365) + * MESHCORE_REGION_NAMES extra probe names (comma-separated) + * MESHCORE_CHANNEL_SECRETS extra channel secrets for the decoder keyStore + * REGION_SCOPE_DRY_RUN=1 compute + log but do not write + * + * All four tables are refreshed wholesale inside ONE transaction (delete + + * insert), so readers never observe an empty or partial state. If the scoped + * corpus is non-empty but nothing decodes, the tool aborts without writing + * (a broken probe list must never wipe good data). + */ + +import pg from 'pg'; +import { MeshCoreDecoder } from '@michaelhart/meshcore-decoder'; +import { decodePacketCompat } from '../mqtt/decodePacket.js'; +import { MEMBER_MIN_SHARE_PCT, meetsMemberShare } from '../regions/memberShare.js'; +import { regionProbeNamesWithEnv } from '../regions/regionNames.js'; +import { decodeRegionForPacket, transportCode0Of } from '../regions/scopeDecode.js'; + +const { Pool } = pg; + +// Documented MeshCore public channel material, not an application credential. +const DEFAULT_PUBLIC_CHANNEL_KEY = '8b3387e9c5cdea6ac9e5edbaa115cd72'; +const DEFAULT_WINDOW_DAYS = 92; +const UNRESOLVED_ELEMENT_LOG_LIMIT = 12; + +interface ScopedPacketRow { + packet_hash: string; + t0: Date; + t1: Date; + raw: string | null; + elems: string[] | null; +} + +interface NodeMetaRow { + node_id: string; + role: number | null; +} + +interface UnscopedElementRow { + e: string; + c: string; +} + +function boundedWindowDays(raw: string | undefined): number { + const value = Number(raw ?? DEFAULT_WINDOW_DAYS); + if (!Number.isFinite(value)) return DEFAULT_WINDOW_DAYS; + return Math.max(7, Math.min(365, Math.trunc(value))); +} + +function buildChannelSecrets(): string[] { + const secrets = [DEFAULT_PUBLIC_CHANNEL_KEY]; + for (const entry of (process.env['MESHCORE_CHANNEL_SECRETS'] ?? '') + .split(',').map((value) => value.trim()).filter(Boolean)) { + const colon = entry.indexOf(':'); + secrets.push(colon > 0 ? entry.slice(colon + 1) : entry); + } + return secrets; +} + +function resolveElement(prefixIndex: Map, element: string): string | null { + const normalized = element.trim().toUpperCase(); + if (!/^[0-9A-F]{2,6}$/.test(normalized)) return null; + const candidates = prefixIndex.get(normalized); + if (!candidates || candidates.length === 0) return null; + const hits = candidates.filter((candidate) => candidate.startsWith(normalized)); + return hits.length === 1 ? hits[0] : null; +} + +async function main(): Promise { + if (!process.env['DATABASE_URL']) throw new Error('DATABASE_URL is required'); + const windowDays = boundedWindowDays(process.env['REGION_SCOPE_WINDOW_DAYS']); + const dryRun = process.env['REGION_SCOPE_DRY_RUN'] === '1'; + const probeNames = regionProbeNamesWithEnv(process.env['MESHCORE_REGION_NAMES']); + const keyStore = MeshCoreDecoder.createKeyStore({ channelSecrets: buildChannelSecrets() }); + + const pool = new Pool({ + connectionString: process.env['DATABASE_URL'], + application_name: 'meshcore-regions-rebuild', + max: 2, + statement_timeout: 0, + query_timeout: 0, + }); + + try { + const startedAt = Date.now(); + console.log(`[regions] probe list: ${probeNames.length} names; window ${windowDays}d${dryRun ? ' (dry run)' : ''}`); + + const { rows: scopedRows } = await pool.query( + `SELECT p.packet_hash, + min(p.time) AS t0, + max(p.time) AS t1, + (array_agg(p.raw_hex ORDER BY p.time DESC) + FILTER (WHERE p.route_type IN (0, 3) AND p.raw_hex <> ''))[1] AS raw, + (array_agg(DISTINCT e) FILTER (WHERE e IS NOT NULL))[1:80] AS elems + FROM packets p + LEFT JOIN LATERAL unnest(p.path_hashes) AS e ON TRUE + WHERE p.route_type IN (0, 3) + AND p.time > now() - make_interval(days => $1) + GROUP BY p.packet_hash`, + [windowDays], + ); + console.log(`[regions] scoped packets: ${scopedRows.length} (${Math.round((Date.now() - startedAt) / 1000)}s)`); + if (scopedRows.length === 0) throw new Error('no scoped packets found in window; refusing to wipe tables'); + + const regionTotals = new Map(); + const regionFirstSeen = new Map(); + const regionLastSeen = new Map(); + const packetRegion = new Map(); + let decodedCount = 0; + for (const row of scopedRows) { + let region: string | null = null; + if (row.raw) { + const compat = decodePacketCompat(row.raw, keyStore); + if (compat.metadataValid && compat.transportCodes) { + const tc0 = transportCode0Of(compat.transportCodes); + region = decodeRegionForPacket( + probeNames, + compat.decoded?.payloadType, + compat.decoded?.payload?.raw, + tc0, + ); + } + } + packetRegion.set(row.packet_hash, region); + if (region) { + decodedCount += 1; + regionTotals.set(region, (regionTotals.get(region) ?? 0) + 1); + const first = regionFirstSeen.get(region); + if (!first || row.t0 < first) regionFirstSeen.set(region, row.t0); + const last = regionLastSeen.get(region); + if (!last || row.t1 > last) regionLastSeen.set(region, row.t1); + } + } + const unknownCount = scopedRows.length - decodedCount; + console.log(`[regions] decoded ${decodedCount}/${scopedRows.length} (${unknownCount} unknown)`); + if (decodedCount === 0) throw new Error('no scoped packets decoded; refusing to wipe tables'); + + const { rows: nodeRows } = await pool.query( + 'SELECT node_id, role FROM nodes', + ); + const nodeRoles = new Map(); + const prefixIndex = new Map(); + for (const row of nodeRows) { + const nodeId = (row.node_id ?? '').trim().toUpperCase(); + if (!/^[0-9A-F]+$/.test(nodeId)) continue; + nodeRoles.set(nodeId, row.role); + for (const length of [2, 4, 6]) { + const key = nodeId.slice(0, length); + const list = prefixIndex.get(key); + if (list) list.push(nodeId); + else prefixIndex.set(key, [nodeId]); + } + } + + const scopedByNode = new Map>>(); + const scopedTotal = new Map>(); + const elementStats = new Map(); + const unresolvedSamples: string[] = []; + for (const row of scopedRows) { + const region = packetRegion.get(row.packet_hash) ?? null; + for (const element of row.elems ?? []) { + if (!element) continue; + const lengthKey = `elem_len_${element.length}`; + elementStats.set(lengthKey, (elementStats.get(lengthKey) ?? 0) + 1); + const nodeId = resolveElement(prefixIndex, element); + if (!nodeId) { + const unresolvedKey = `unresolved_len_${element.length}`; + elementStats.set(unresolvedKey, (elementStats.get(unresolvedKey) ?? 0) + 1); + if (unresolvedSamples.length < UNRESOLVED_ELEMENT_LOG_LIMIT) unresolvedSamples.push(element); + continue; + } + let total = scopedTotal.get(nodeId); + if (!total) { + total = new Set(); + scopedTotal.set(nodeId, total); + } + total.add(row.packet_hash); + if (region) { + let byRegion = scopedByNode.get(nodeId); + if (!byRegion) { + byRegion = new Map>(); + scopedByNode.set(nodeId, byRegion); + } + let hashes = byRegion.get(region); + if (!hashes) { + hashes = new Set(); + byRegion.set(region, hashes); + } + hashes.add(row.packet_hash); + } + } + } + console.log(`[regions] element stats: ${JSON.stringify(Object.fromEntries([...elementStats.entries()].sort()))}`); + + const { rows: unscopedRows } = await pool.query( + `SELECT e, count(DISTINCT p.packet_hash) AS c + FROM packets p, unnest(p.path_hashes) AS e + WHERE p.route_type = 1 + AND p.path_hash_size_bytes >= 2 + AND p.time > now() - interval '7 days' + AND length(e) >= 4 + GROUP BY e`, + ); + const unscoped = new Map(); + for (const row of unscopedRows) { + if (!row.e) continue; + const nodeId = resolveElement(prefixIndex, row.e); + if (nodeId) unscoped.set(nodeId, (unscoped.get(nodeId) ?? 0) + Number(row.c)); + } + console.log(`[regions] unscoped relay evidence (7d): ${unscoped.size} nodes, ${[...unscoped.values()].reduce((sum, value) => sum + value, 0)} relay counts`); + + // Region view (members = relays with resolved ids carrying at least + // MEMBER_MIN_SHARE_PCT of the region's scoped packets). + interface MemberOutput { + region: string; + nodeId: string; + packets: number; + role: number | null; + } + const members: MemberOutput[] = []; + const regionView = new Map(); + let memberRowsTotal = 0; + for (const [nodeId, byRegion] of scopedByNode) { + for (const [region, hashes] of byRegion) { + memberRowsTotal += 1; + if (!meetsMemberShare(hashes.size, regionTotals.get(region) ?? 0)) continue; + const role = nodeRoles.get(nodeId) ?? null; + members.push({ region, nodeId, packets: hashes.size, role }); + const view = regionView.get(region) ?? { memberCount: 0, repeaterMemberCount: 0 }; + view.memberCount += 1; + if (role === 2) view.repeaterMemberCount += 1; + regionView.set(region, view); + } + } + console.log(`[regions] member share floor >=${MEMBER_MIN_SHARE_PCT}%: kept ${members.length}/${memberRowsTotal} member rows`); + + // Node classification over every node with any relay evidence. + interface ClassificationOutput { + nodeId: string; + status: 'green' | 'red'; + evidence: 'scoped' | 'unscoped-only' | 'none'; + scopedPackets: number; + unscopedPackets7d: number; + regions: Record; + } + const classification: ClassificationOutput[] = []; + const candidateIds = new Set([...scopedTotal.keys(), ...unscoped.keys()]); + for (const nodeId of candidateIds) { + const scoped = scopedTotal.get(nodeId); + const byRegion = scopedByNode.get(nodeId); + const regions: Record = {}; + if (byRegion) { + for (const [region, hashes] of [...byRegion.entries()].sort((a, b) => b[1].size - a[1].size)) { + if (!meetsMemberShare(hashes.size, regionTotals.get(region) ?? 0)) continue; + regions[region] = hashes.size; + } + } + classification.push({ + nodeId, + status: scoped ? 'green' : 'red', + evidence: scoped ? 'scoped' : (unscoped.get(nodeId) ?? 0) > 0 ? 'unscoped-only' : 'none', + scopedPackets: scoped ? scoped.size : 0, + unscopedPackets7d: unscoped.get(nodeId) ?? 0, + regions, + }); + } + + const greenCount = classification.filter((entry) => entry.status === 'green').length; + const redCount = classification.length - greenCount; + console.log(`[regions] regions (scoped packets -> members >=${MEMBER_MIN_SHARE_PCT}%):`); + for (const [region, total] of [...regionTotals.entries()].sort((a, b) => b[1] - a[1])) { + const view = regionView.get(region) ?? { memberCount: 0, repeaterMemberCount: 0 }; + console.log(` ${region}: packets=${total} members=${view.memberCount} repeaters=${view.repeaterMemberCount}`); + } + console.log(`[regions] classification: green=${greenCount} red=${redCount}`); + if (unresolvedSamples.length > 0) { + console.log(`[regions] unresolved element samples: ${unresolvedSamples.join(', ')}`); + } + + if (dryRun) { + console.log(JSON.stringify({ + status: 'dry-run', + scopedPackets: scopedRows.length, + decodedPackets: decodedCount, + regions: regionTotals.size, + members: members.length, + classification: classification.length, + })); + return; + } + + const client = await pool.connect(); + try { + await client.query('BEGIN'); + await client.query('DELETE FROM region_scope_members'); + await client.query('DELETE FROM region_scope_stats'); + await client.query('DELETE FROM repeater_scope_status'); + await client.query('DELETE FROM region_scope_run'); + + const regionNames = [...regionTotals.keys()]; + await client.query( + `INSERT INTO region_scope_stats + (region, packets_total, member_count, repeater_member_count, first_seen, last_seen) + SELECT region, packets_total, member_count, repeater_member_count, first_seen, last_seen + FROM unnest($1::text[], $2::int[], $3::int[], $4::int[], $5::timestamptz[], $6::timestamptz[]) + AS t(region, packets_total, member_count, repeater_member_count, first_seen, last_seen)`, + [ + regionNames, + regionNames.map((region) => regionTotals.get(region) ?? 0), + regionNames.map((region) => regionView.get(region)?.memberCount ?? 0), + regionNames.map((region) => regionView.get(region)?.repeaterMemberCount ?? 0), + regionNames.map((region) => regionFirstSeen.get(region) ?? null), + regionNames.map((region) => regionLastSeen.get(region) ?? null), + ], + ); + + const MEMBER_BATCH = 2_000; + for (let offset = 0; offset < members.length; offset += MEMBER_BATCH) { + const batch = members.slice(offset, offset + MEMBER_BATCH); + await client.query( + `INSERT INTO region_scope_members (region, node_id, packets) + SELECT region, node_id, packets + FROM unnest($1::text[], $2::text[], $3::int[]) AS t(region, node_id, packets)`, + [ + batch.map((member) => member.region), + batch.map((member) => member.nodeId), + batch.map((member) => member.packets), + ], + ); + } + + const STATUS_BATCH = 2_000; + for (let offset = 0; offset < classification.length; offset += STATUS_BATCH) { + const batch = classification.slice(offset, offset + STATUS_BATCH); + await client.query( + `INSERT INTO repeater_scope_status + (node_id, status, evidence, scoped_packets, unscoped_packets_7d, regions) + SELECT node_id, status, evidence, scoped_packets, unscoped_packets_7d, regions::jsonb + FROM unnest($1::text[], $2::text[], $3::text[], $4::int[], $5::int[], $6::text[]) + AS t(node_id, status, evidence, scoped_packets, unscoped_packets_7d, regions)`, + [ + batch.map((entry) => entry.nodeId), + batch.map((entry) => entry.status), + batch.map((entry) => entry.evidence), + batch.map((entry) => entry.scopedPackets), + batch.map((entry) => entry.unscopedPackets7d), + batch.map((entry) => JSON.stringify(entry.regions)), + ], + ); + } + + await client.query( + `INSERT INTO region_scope_run (id, finished_at, window_days, scoped_packets, decoded_packets) + VALUES (TRUE, now(), $1, $2, $3)`, + [windowDays, scopedRows.length, decodedCount], + ); + await client.query('COMMIT'); + } catch (error) { + await client.query('ROLLBACK').catch(() => undefined); + throw error; + } finally { + client.release(); + } + + console.log(JSON.stringify({ + status: 'complete', + scopedPackets: scopedRows.length, + decodedPackets: decodedCount, + regions: regionTotals.size, + members: members.length, + classification: classification.length, + elapsedSeconds: Math.round((Date.now() - startedAt) / 1000), + })); + } finally { + await pool.end(); + } +} + +main().catch((error: unknown) => { + console.error('[regions] failed:', error instanceof Error ? error.message : error); + process.exit(1); +}); diff --git a/backend/src/ws/server.ts b/backend/src/ws/server.ts index 264f292..f0ef7b3 100644 --- a/backend/src/ws/server.ts +++ b/backend/src/ws/server.ts @@ -114,6 +114,8 @@ function slimInitialPacket(raw: InitialStateRow): InitialStateRow { path_hash_size_bytes: raw['path_hash_size_bytes'], rx_count: raw['rx_count'], tx_count: raw['tx_count'], + tags: raw['tags'], + tag_confidence: raw['tag_confidence'], }; } diff --git a/backup-receipts/latest.json b/backup-receipts/latest.json index d589130..b238c87 100644 --- a/backup-receipts/latest.json +++ b/backup-receipts/latest.json @@ -1,14 +1,14 @@ { "format": "meshcore-restore-receipt-v1", - "receipt_id": "restore-20260813T032656Z-20260813T031400Z-2c1dd39b7173", - "backup_id": "backup-20260813T031400Z-2c1dd39b7173", - "backup_completed_at": "2026-08-13T03:18:09.000Z", - "restore_started_at": "2026-08-13T03:20:02.000Z", - "restore_verified_at": "2026-08-13T03:26:56.000Z", - "restore_duration_seconds": 414, - "recovery_point_age_seconds": 527, - "archive_sha256": "6466077779be65d3f843843b3fbf57cd33abad671391c5744e93d3a0fe1911a7", - "source_revision": "2c1dd39b7173e2cb97164a6e9dbe9e1f600b6aaa", + "receipt_id": "restore-20260906T181708Z-20260906T173725Z-1eefb50b31fd", + "backup_id": "backup-20260906T173725Z-1eefb50b31fd", + "backup_completed_at": "2026-09-06T17:45:59.000Z", + "restore_started_at": "2026-09-06T18:06:24.000Z", + "restore_verified_at": "2026-09-06T18:17:08.000Z", + "restore_duration_seconds": 644, + "recovery_point_age_seconds": 1869, + "archive_sha256": "310356e22c0e260e78556efcbfd0a520e6dd2c4fc74c461e40460f3a8f0a550d", + "source_revision": "1eefb50b31fda8d79ef383a658764671c65cb46a", "schema_version": 51, "datasets": [ "analytics", @@ -21,11 +21,11 @@ "evidence": { "integrity": { "packet_sample_rows": 1000, - "packet_sample_md5": "d975a0fd34eee068df4eabf80f7d52ae", + "packet_sample_md5": "6dee09805b37d0fa114aecf0db10fac6", "node_sample_rows": 1000, - "node_sample_md5": "5d1a8fea57b96209ad404739fbaa6628" + "node_sample_md5": "9754a27fe3280d2c578dd75fdfaa2300" }, - "owner_lookup_count": 47 + "owner_lookup_count": 50 }, "checks": { "migrations": "passed", diff --git a/backup-receipts/latest.json.sig b/backup-receipts/latest.json.sig index b26b674..786afe3 100644 Binary files a/backup-receipts/latest.json.sig and b/backup-receipts/latest.json.sig differ diff --git a/docker-compose.yml b/docker-compose.yml index 0fd9f4d..22b7b78 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -76,24 +76,36 @@ services: MQTT_USERNAME: ${MQTT_USERNAME:-backend} MQTT_PASSWORD: ${MQTT_PASSWORD} MQTT_CLIENT_ID: ${MQTT_CLIENT_ID:-meshcore-analytics-ingest} + MESHCORE_SITE_ID: ${MESHCORE_SITE_ID:-ukmesh} REDIS_URL: ${REDIS_URL:-redis://redis:6379} REDIS_PASSWORD: ${REDIS_PASSWORD:?REDIS_PASSWORD required} JWT_SECRET: ${JWT_SECRET:?JWT_SECRET required} PORT: ${PORT:-3000} METRICS_PORT: ${METRICS_PORT:-9091} MESHCORE_CHANNEL_SECRETS: ${MESHCORE_CHANNEL_SECRETS:-} + MESHCORE_REGION_NAMES: ${MESHCORE_REGION_NAMES:-eng-ne,Europe,Global} OWNER_DATABASE_URL: ${OWNER_DATABASE_URL:-postgresql://${POSTGRES_USER:-meshcore}:${POSTGRES_PASSWORD}@timescaledb:5432/${OWNER_POSTGRES_DB:-meshcore_owner_auth}} OWNER_MQTT_USERNAME_MAP: ${OWNER_MQTT_USERNAME_MAP:-} OWNER_AUTHORIZATION_MODE: ${OWNER_AUTHORIZATION_MODE:-shadow} OWNER_ACL_MODE: ${OWNER_ACL_MODE:-shadow} OWNER_ACL_UNMANAGED_USERS: ${OWNER_ACL_UNMANAGED_USERS:-backend,test,test2} OWNER_ACL_ALLOW_EMPTY_USERS: ${OWNER_ACL_ALLOW_EMPTY_USERS:-} + OWNER_ACL_PUBLIC_FEED_READ_USERS: ${OWNER_ACL_PUBLIC_FEED_READ_USERS:-} + OWNER_ACL_SYSTEM_READWRITE_TOPICS: ${OWNER_ACL_SYSTEM_READWRITE_TOPICS:-public/#} OWNER_AUTH_INVENTORY_BASELINE_PATH: ${OWNER_AUTH_INVENTORY_BASELINE_PATH:-} OWNER_ACL_SHADOW_PATH: /tmp/meshcore-owner-acl.shadow MOSQUITTO_ACL_PATH: /mosquitto/config/acl OWNER_ACL_RELOAD_URL: http://mosquitto-reloader:8080/reload OWNER_ACL_RELOAD_TOKEN: ${JWT_SECRET:?JWT_SECRET required} OWNER_COOKIE_SECRET: ${OWNER_COOKIE_SECRET:-} + OWNER_PACKET_SHARE_ENCRYPTION_KEY: ${OWNER_PACKET_SHARE_ENCRYPTION_KEY:-} + OWNER_PACKET_SHARE_POLL_INTERVAL_MS: ${OWNER_PACKET_SHARE_POLL_INTERVAL_MS:-5000} + OWNER_PACKET_SHARE_BATCH_SIZE: ${OWNER_PACKET_SHARE_BATCH_SIZE:-25} + # The public feed is the authentication-locked UKmesh Mosquitto endpoint. + # The URL is display/configuration metadata; packets already flow through + # this broker, and opted-in owners receive a read-only ACL grant. + PUBLIC_FEED_PUBLISH_ENABLED: ${PUBLIC_FEED_PUBLISH_ENABLED:-false} + PUBLIC_FEED_BROKER_URL: ${PUBLIC_FEED_BROKER_URL:-} OPERATOR_SITE_TOKEN: ${OPERATOR_SITE_TOKEN:?OPERATOR_SITE_TOKEN required} TRUSTED_PROXY_PEERS: ${TRUSTED_PROXY_PEERS:-172.30.0.10,172.30.0.11,172.30.0.12} ALLOWED_ORIGINS: ${ALLOWED_ORIGINS:-http://localhost:3003,http://localhost:3004,https://ukmesh.com,https://www.ukmesh.com,https://app.ukmesh.com,https://test.ukmesh.com} @@ -272,7 +284,7 @@ services: PRIVACY_REMAT_MAX_ATTEMPTS: ${PRIVACY_REMAT_MAX_ATTEMPTS:-8} HEALTH_VOLUME_PATHS: database=/health-volumes/database=${HEALTH_DATABASE_BUDGET_BYTES:-322122547200},redis=/health-volumes/redis=${HEALTH_REDIS_BUDGET_BYTES:-4294967296},terrain=/health-volumes/terrain=${HEALTH_TERRAIN_BUDGET_BYTES:-53687091200},logs=/health-volumes/logs=${HEALTH_LOG_BUDGET_BYTES:-21474836480} DATA_LIFECYCLE_RETENTION_ENABLED: "true" - DATA_LIFECYCLE_RETENTION_TARGETS: ${DATA_LIFECYCLE_RETENTION_TARGETS:-packets,worker_health_snapshots,frontend_error_events,owner_alert_deliveries,operational_check_results,observer_region_packet_sightings,observer_region_observer_sightings,link_job_commits,ml_model_variant_packet_results,packet_decryptions,observer_registration_requests,operator_audit_events} + DATA_LIFECYCLE_RETENTION_TARGETS: ${DATA_LIFECYCLE_RETENTION_TARGETS:-packets,worker_health_snapshots,frontend_error_events,owner_alert_deliveries,owner_packet_share_deliveries,public_feed_deliveries,operational_check_results,observer_region_packet_sightings,observer_region_observer_sightings,link_job_commits,ml_model_variant_packet_results,packet_decryptions,observer_registration_requests,operator_audit_events} NODE_ENV: production depends_on: db-migrate: @@ -283,6 +295,29 @@ services: - srtm_data:/health-volumes/terrain:ro - loki_data:/health-volumes/logs:ro + # Tags new public-channel messages with TypeSafe/Jev (shadow-mode trial, + # added 2026-09-17 by Hermes for Ben). Results land in message_tags; nothing + # user-visible reads it yet. Monitoring: tagger_state + container logs. + tagger-worker: + <<: [*internal-hosts, *application-hardening] + image: ${TAGGER_WORKER_IMAGE:-meshcore-analytics-tagger-worker:local} + build: + context: ./tagger-worker + dockerfile: Dockerfile + restart: always + logging: *json-log-limits + mem_limit: "256m" + memswap_limit: "256m" + cpus: 1.0 + environment: + DATABASE_URL: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@timescaledb:5432/${POSTGRES_DB:-meshcore} + DATABASE_APPLICATION_NAME: ukmesh-tagger-worker + TYPESAFE_API_KEY: ${TYPESAFE_API_KEY:-} + TAGGER_START: ${TAGGER_START:-} + TAGGER_POLL_S: ${TAGGER_POLL_S:-5} + TAGGER_CALL_FLOOR_S: ${TAGGER_CALL_FLOOR_S:-0.5} + PYTHONDONTWRITEBYTECODE: "1" + privacy-remat-worker: <<: [*internal-hosts, *application-hardening] image: ${BACKEND_IMAGE:-meshcore-analytics-backend:local} @@ -573,6 +608,7 @@ services: MQTT_WS_PATH: / MQTT_TLS: "false" DASH_BROKER_HOST: ${HEALTHCHECK_DASH_BROKER_HOST:-mqtt.ukmesh.com} + WEBSOCKET_ALLOWED_ORIGINS: ${HEALTHCHECK_WEBSOCKET_ALLOWED_ORIGINS:-https://healthcheck.ukmesh.com} TEST_CHANNEL_NAME: ${HEALTHCHECK_TEST_CHANNEL_NAME:-ukmeshtest} TEST_CHANNEL_SECRET: ${HEALTHCHECK_TEST_CHANNEL_SECRET:-} TEST_CHANNEL_SECRET_SOURCE_NAME: ${HEALTHCHECK_TEST_CHANNEL_SECRET_SOURCE_NAME:-test} @@ -935,7 +971,7 @@ services: volumes: - alert_receiver_data:/var/lib/meshcore-alerts healthcheck: - test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:8080/healthz >/dev/null"] + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:8080/readyz >/dev/null"] interval: 30s timeout: 5s retries: 5 diff --git a/docs/db-lifecycle.md b/docs/db-lifecycle.md index 68133eb..bee4067 100644 --- a/docs/db-lifecycle.md +++ b/docs/db-lifecycle.md @@ -59,7 +59,7 @@ Content-bearing packets have a hard 30-day bound. Timescale drops complete chunks, while the health worker deletes exact expired rows from the partial boundary chunk in deterministic 25,000-row batches; new packet chunks are one day. Content-free path reconstruction reads `packet_paths`, so it continues -after the source packet expires. Status samples retain 180 days and neighbor +after the source packet expires. Status samples retain 30 days and neighbor samples seven days. Privacy-filtered aggregates, model parameters, current node/link/coverage state, and privacy state remain longer lived. @@ -73,100 +73,44 @@ It reports exact expired rows, expired/compressed Timescale chunks, relation bytes, oldest/newest timestamps, and the affected features for every target. The inventory can be expensive by design; run it away from peak ingest. -Compression and retention are separate, table-at-a-time changes. Migration -050 declares Timescale retention policies for packets, status samples, and -neighbour samples. The base schema previously removed the packet policy during -schema initialization; it no longer does. If a policy is absent, use the -receipt-gated apply command below after verifying a named backup and completing -an isolated restore drill. Compression remains a separate receipt-gated -operator action. The application fails startup when an enabled core lifecycle -action omits a required target. - -Each gated per-table apply command requires: +Compression and retention are separate, table-at-a-time changes. Both require: - a named, fresh database backup; -- a signed restore receipt verified within the configured receipt age (seven - days by default); +- a successful isolated restore verification from the last 30 days; - the target in `DATA_LIFECYCLE_RETENTION_TARGETS` for deletion or `DATA_LIFECYCLE_COMPRESSION_TARGETS` for compression; - the action flag set to `true`; and - an exact per-table approval argument. -The receipt path, detached signature, and verification key are mounted into -the backend as `DATA_LIFECYCLE_RESTORE_RECEIPT_PATH`, -`DATA_LIFECYCLE_RESTORE_RECEIPT_SIGNATURE_PATH`, and -`DATA_LIFECYCLE_RECEIPT_VERIFY_KEY_PATH`. Confirm those files are from the -completed isolated restore drill before proceeding. The signed receipt contains -the backup identifier and restore checks. - -First inspect the read-only inventory and record the named backup and restore -receipt for the change: +Example compression rollout: ```bash -docker compose exec backend npm run db:lifecycle -``` - -After approval, enable the reviewed core retention windows with one explicit -command per table. The packet content window is 30 days in the current -repository policy; status telemetry is 180 days and neighbour telemetry is -seven days. `packet_paths` is never a retention target: - -```bash -docker compose exec -e DATA_LIFECYCLE_RETENTION_ENABLED=true backend npm run db:lifecycle -- \ - --apply-retention --target=packets \ - --approve=apply-data-lifecycle-retention-packets -docker compose exec -e DATA_LIFECYCLE_RETENTION_ENABLED=true backend npm run db:lifecycle -- \ - --apply-retention --target=node_status_samples \ - --approve=apply-data-lifecycle-retention-node_status_samples -docker compose exec -e DATA_LIFECYCLE_RETENTION_ENABLED=true backend npm run db:lifecycle -- \ - --apply-retention --target=node_neighbor_samples \ - --approve=apply-data-lifecycle-retention-node_neighbor_samples -``` - -Apply compression per table after checking the inventory and database/WAL -headroom. The reviewed policy compresses packets, path rows, and status after -14 days, and neighbour samples after one day: - -```bash -docker compose exec -e DATA_LIFECYCLE_COMPRESSION_ENABLED=true backend npm run db:lifecycle -- \ +DATA_LIFECYCLE_COMPRESSION_ENABLED=true +DATA_LIFECYCLE_COMPRESSION_TARGETS=packets,packet_paths,node_status_samples,node_neighbor_samples +DATA_LIFECYCLE_BACKUP_REFERENCE=backup-20260729 +DATA_LIFECYCLE_RESTORE_VERIFIED_AT=2026-07-29T12:00:00Z +docker compose exec backend npm run db:lifecycle -- \ --apply-compression --target=packets \ --approve=apply-data-lifecycle-compression-packets -docker compose exec -e DATA_LIFECYCLE_COMPRESSION_ENABLED=true backend npm run db:lifecycle -- \ - --apply-compression --target=packet_paths \ - --approve=apply-data-lifecycle-compression-packet_paths -docker compose exec -e DATA_LIFECYCLE_COMPRESSION_ENABLED=true backend npm run db:lifecycle -- \ - --apply-compression --target=node_status_samples \ - --approve=apply-data-lifecycle-compression-node_status_samples -docker compose exec -e DATA_LIFECYCLE_COMPRESSION_ENABLED=true backend npm run db:lifecycle -- \ - --apply-compression --target=node_neighbor_samples \ - --approve=apply-data-lifecycle-compression-node_neighbor_samples ``` -Measure query CPU, ingest WAL, and storage after cold-chunk compression. Check -the read-only inventory and Prometheus chunk/byte gauges after each change. -Set the production configuration flags only after every required core target -is present; missing targets make the application fail closed. +Measure query CPU, ingest WAL, and storage after cold-chunk compression. Only +after aggregate cutover and another inventory may retention be enabled: -## Expired forensic data - -Expired packet content, status samples, and neighbour history are not served by -the production application. For forensic access, restore a named backup that -predates the expiry into an isolated restore environment, keep the database and -application read-only, disable MQTT ingest and all background writers, and -limit access to the authorized investigation. Never restore old data over the -production database or reconnect an isolated forensic copy to public APIs. -The backup and restore workflow is documented in -[`runbook-backup-restore.md`](runbook-backup-restore.md). - -Prometheus tracks retained hypertable bytes/chunks, expired chunks, -compression-overdue chunks, and host free-space trends. Follow the matching -first-response steps in [`operations.md`](operations.md) for growth, -retention, compression, or disk exhaustion alerts. +```bash +docker compose exec backend npm run db:lifecycle -- \ + --apply-retention --target=packets \ + --approve=apply-data-lifecycle-retention-packets +``` Hypertable retention uses a Timescale policy. Row-table deletion is bounded and performed by the health worker only for targets explicitly listed in `DATA_LIFECYCLE_RETENTION_TARGETS`. Failed/pending owner alert deliveries are -not discarded while they remain retryable. +not discarded while they remain retryable. Owner packet-sharing deliveries +follow the same rule and retain their bounded relay payload for 30 days after +they reach a terminal status. +Public-feed delivery rows follow the same rule and retain their bounded public +packet envelope for 30 days after reaching a terminal status. Compression state is migration-defined: `packets`, `packet_paths`, and `node_status_samples` compress after 14 days (matching the reviewed live diff --git a/docs/decryption.md b/docs/decryption.md index e6830c2..daff654 100644 --- a/docs/decryption.md +++ b/docs/decryption.md @@ -6,20 +6,16 @@ how historical traffic was retroactively decrypted. ## Key store - `backend/src/mqtt/channelRegistry.ts` is the single source of truth: - - `PUBLIC_CHANNELS` — 41 recovered/community-known values intentionally - classified as public and baked into the image. Each was validated against - real human-readable UK Mesh group text. + - `VALIDATED_CHANNELS` — default secrets baked into the image (39 entries + incl. Public; recovered 2026-08-06 and validated to decrypt real + human-readable group text). - `buildCombinedKeyStore()` — merges baked defaults with - `MESHCORE_CHANNEL_SECRETS` (managed environment configuration, with - comma-separated `name:hex` or bare hex entries; dedupes by secret). - Environment-supplied entries are classified as confidential; never commit - their values or copy them into `PUBLIC_CHANNELS`. + `MESHCORE_CHANNEL_SECRETS` (env, comma-separated `name:hex` or bare hex; + dedupes by secret) for secrets that shouldn't be committed. - `buildSummary()` / `identifyChannel()` — shared by ingest and offline tools. -- The repo is **public**. `PUBLIC_CHANNELS` is the explicit source classification; - the scanner baseline in `.gitleaksignore` records exact historical - commit/path/rule/line fingerprints for reviewed public values, test fixtures, - and generated revision identifiers. New findings remain blocking by default. +- ⚠️ The repo is **public**: keys in `channelRegistry.ts` are public. Keep + non-derivable/community keys in the env var only. - Reading env at startup only — after a change: `docker compose -f docker-compose.yml -f docker-compose.live.yml up -d --no-deps --force-recreate backend`. diff --git a/docs/openapi.yaml b/docs/openapi.yaml index b68b273..91afe4b 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -3822,6 +3822,421 @@ } } }, + "/api/owner/public-feed": { + "get": { + "summary": "Read owner public-feed settings", + "operationId": "get-owner-public-feed", + "tags": [ + "owner" + ], + "x-access": "owner", + "parameters": [], + "responses": { + "200": { + "description": "Successful response", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "featureEnabled", + "configured", + "brokerUrl", + "enabled", + "nodeCount", + "eligible", + "recentPacketCount", + "topicFilters", + "lastForwardedAt" + ], + "properties": { + "featureEnabled": { + "type": "boolean" + }, + "configured": { + "type": "boolean" + }, + "brokerUrl": { + "type": [ + "string", + "null" + ], + "format": "uri", + "maxLength": 2048 + }, + "enabled": { + "type": "boolean" + }, + "nodeCount": { + "type": "integer", + "minimum": 0 + }, + "eligible": { + "type": "boolean" + }, + "recentPacketCount": { + "type": "integer", + "minimum": 0 + }, + "topicFilters": { + "type": "array", + "maxItems": 1, + "items": { + "type": "string", + "pattern": "^public/\\+/\\+/packets$" + } + }, + "lastForwardedAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + } + }, + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Invalid or out-of-bounds request", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "type": "string" + } + }, + "additionalProperties": false + } + } + } + }, + "401": { + "description": "Authentication required", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "type": "string" + } + }, + "additionalProperties": false + } + } + } + }, + "403": { + "description": "Authentication or CSRF validation failed", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "type": "string" + } + }, + "additionalProperties": false + } + } + } + }, + "429": { + "description": "Rate limit exceeded", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "type": "string" + } + }, + "additionalProperties": false + } + } + } + }, + "500": { + "description": "Internal server error", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "type": "string" + } + }, + "additionalProperties": false + } + } + } + } + }, + "security": [ + { + "ownerSession": [] + } + ] + }, + "post": { + "summary": "Update owner public-feed settings", + "operationId": "post-owner-public-feed", + "tags": [ + "owner" + ], + "x-access": "owner", + "parameters": [], + "responses": { + "200": { + "description": "Successful response", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "featureEnabled", + "configured", + "brokerUrl", + "enabled", + "nodeCount", + "eligible", + "recentPacketCount", + "topicFilters", + "lastForwardedAt" + ], + "properties": { + "featureEnabled": { + "type": "boolean" + }, + "configured": { + "type": "boolean" + }, + "brokerUrl": { + "type": [ + "string", + "null" + ], + "format": "uri", + "maxLength": 2048 + }, + "enabled": { + "type": "boolean" + }, + "nodeCount": { + "type": "integer", + "minimum": 0 + }, + "eligible": { + "type": "boolean" + }, + "recentPacketCount": { + "type": "integer", + "minimum": 0 + }, + "topicFilters": { + "type": "array", + "maxItems": 1, + "items": { + "type": "string", + "pattern": "^public/\\+/\\+/packets$" + } + }, + "lastForwardedAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + } + }, + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Invalid or out-of-bounds request", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "type": "string" + } + }, + "additionalProperties": false + } + } + } + }, + "401": { + "description": "Authentication required", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "type": "string" + } + }, + "additionalProperties": false + } + } + } + }, + "403": { + "description": "Authentication or CSRF validation failed", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "type": "string" + } + }, + "additionalProperties": false + } + } + } + }, + "409": { + "description": "The public feed is not configured or the owner has no eligible packet activity", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "type": "string" + } + }, + "additionalProperties": false + } + } + } + }, + "429": { + "description": "Rate limit exceeded", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "type": "string" + } + }, + "additionalProperties": false + } + } + } + }, + "500": { + "description": "Internal server error", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "type": "string" + } + }, + "additionalProperties": false + } + } + } + }, + "503": { + "description": "The broker ACL could not be updated", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "type": "string" + } + }, + "additionalProperties": false + } + } + } + } + }, + "security": [ + { + "ownerSession": [] + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "enabled" + ], + "properties": { + "enabled": { + "type": "boolean" + } + }, + "additionalProperties": false + } + } + } + } + } + }, "/api/owner/session": { "get": { "summary": "Read owner session", @@ -4984,6 +5399,584 @@ } } }, + "/api/regions/repeaters": { + "get": { + "summary": "List repeaters classified by region-capability", + "operationId": "get-regions-repeaters", + "tags": [ + "public" + ], + "x-access": "public", + "parameters": [], + "responses": { + "200": { + "description": "Successful response", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "generatedAt", + "repeaters" + ], + "properties": { + "generatedAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "repeaters": { + "type": "array", + "items": { + "type": "object", + "required": [ + "nodeId", + "status", + "evidence", + "scopedPackets", + "unscopedPackets7d", + "regions" + ], + "properties": { + "nodeId": { + "type": "string", + "pattern": "^[0-9A-F]{64}$" + }, + "name": { + "type": [ + "string", + "null" + ] + }, + "role": { + "type": [ + "integer", + "null" + ] + }, + "lat": { + "type": [ + "number", + "null" + ], + "minimum": -90, + "maximum": 90 + }, + "lon": { + "type": [ + "number", + "null" + ], + "minimum": -180, + "maximum": 180 + }, + "status": { + "type": "string", + "enum": [ + "green", + "red" + ] + }, + "evidence": { + "type": "string" + }, + "scopedPackets": { + "type": "integer", + "minimum": 0 + }, + "unscopedPackets7d": { + "type": "integer", + "minimum": 0 + }, + "regions": { + "type": "object", + "additionalProperties": { + "type": "integer", + "minimum": 0 + } + } + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Invalid or out-of-bounds request", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "type": "string" + } + }, + "additionalProperties": false + } + } + } + }, + "429": { + "description": "Rate limit exceeded", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "type": "string" + } + }, + "additionalProperties": false + } + } + } + }, + "500": { + "description": "Internal server error", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "type": "string" + } + }, + "additionalProperties": false + } + } + } + } + } + } + }, + "/api/regions/summary": { + "get": { + "summary": "Read per-region scoped-traffic participation summary", + "operationId": "get-regions-summary", + "tags": [ + "public" + ], + "x-access": "public", + "parameters": [], + "responses": { + "200": { + "description": "Successful response", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "generatedAt", + "windowDays", + "regions" + ], + "properties": { + "generatedAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "windowDays": { + "type": [ + "integer", + "null" + ], + "minimum": 1, + "maximum": 365 + }, + "regions": { + "type": "array", + "items": { + "type": "object", + "required": [ + "name", + "packetsTotal", + "memberCount", + "repeaterMemberCount", + "confidence", + "members" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 64 + }, + "packetsTotal": { + "type": "integer", + "minimum": 0 + }, + "memberCount": { + "type": "integer", + "minimum": 0 + }, + "repeaterMemberCount": { + "type": "integer", + "minimum": 0 + }, + "confidence": { + "type": "string", + "enum": [ + "confirmed", + "probable", + "single" + ] + }, + "firstSeen": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "lastSeen": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "members": { + "type": "array", + "items": { + "type": "object", + "required": [ + "nodeId", + "name", + "role", + "lat", + "lon", + "packets", + "isBridge" + ], + "properties": { + "nodeId": { + "type": "string", + "pattern": "^[0-9A-F]{64}$" + }, + "name": { + "type": [ + "string", + "null" + ] + }, + "role": { + "type": [ + "integer", + "null" + ] + }, + "lat": { + "type": [ + "number", + "null" + ], + "minimum": -90, + "maximum": 90 + }, + "lon": { + "type": [ + "number", + "null" + ], + "minimum": -180, + "maximum": 180 + }, + "packets": { + "type": "integer", + "minimum": 0 + }, + "isBridge": { + "type": "boolean" + } + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Invalid or out-of-bounds request", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "type": "string" + } + }, + "additionalProperties": false + } + } + } + }, + "429": { + "description": "Rate limit exceeded", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "type": "string" + } + }, + "additionalProperties": false + } + } + } + }, + "500": { + "description": "Internal server error", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "type": "string" + } + }, + "additionalProperties": false + } + } + } + } + } + } + }, + "/api/repeaters/{id}/scope": { + "get": { + "summary": "Read one repeater's region scoping detail", + "operationId": "get-repeaters-id-scope", + "tags": [ + "public" + ], + "x-access": "public", + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "minLength": 1, + "maxLength": 200 + } + } + ], + "responses": { + "200": { + "description": "Successful response", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "nodeId", + "status", + "evidence", + "scopedPackets", + "unscopedPackets7d", + "regions" + ], + "properties": { + "nodeId": { + "type": "string", + "pattern": "^[0-9A-F]{64}$" + }, + "name": { + "type": [ + "string", + "null" + ] + }, + "role": { + "type": [ + "integer", + "null" + ] + }, + "lat": { + "type": [ + "number", + "null" + ], + "minimum": -90, + "maximum": 90 + }, + "lon": { + "type": [ + "number", + "null" + ], + "minimum": -180, + "maximum": 180 + }, + "status": { + "type": "string", + "enum": [ + "green", + "red", + "unobserved" + ] + }, + "evidence": { + "type": "string" + }, + "scopedPackets": { + "type": "integer", + "minimum": 0 + }, + "unscopedPackets7d": { + "type": "integer", + "minimum": 0 + }, + "regions": { + "type": "array", + "items": { + "type": "object", + "required": [ + "region", + "packets" + ], + "properties": { + "region": { + "type": "string", + "minLength": 1, + "maxLength": 64 + }, + "packets": { + "type": "integer", + "minimum": 0 + } + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Invalid or out-of-bounds request", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "type": "string" + } + }, + "additionalProperties": false + } + } + } + }, + "404": { + "description": "The repeater is not known to the region-scope tables", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "type": "string" + } + }, + "additionalProperties": false + } + } + } + }, + "429": { + "description": "Rate limit exceeded", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "type": "string" + } + }, + "additionalProperties": false + } + } + } + }, + "500": { + "description": "Internal server error", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "type": "string" + } + }, + "additionalProperties": false + } + } + } + } + } + } + }, "/api/repeaters/firmware": { "get": { "summary": "Read repeaters firmware", @@ -5897,6 +6890,87 @@ } } }, + "/api/tags/recent": { + "get": { + "summary": "Read tags recent", + "operationId": "get-tags-recent", + "tags": [ + "public" + ], + "x-access": "public", + "parameters": [], + "responses": { + "200": { + "description": "Successful response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true + } + } + } + }, + "400": { + "description": "Invalid or out-of-bounds request", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "type": "string" + } + }, + "additionalProperties": false + } + } + } + }, + "429": { + "description": "Rate limit exceeded", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "type": "string" + } + }, + "additionalProperties": false + } + } + } + }, + "500": { + "description": "Internal server error", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "type": "string" + } + }, + "additionalProperties": false + } + } + } + } + } + } + }, "/api/telemetry/frontend-error": { "post": { "summary": "Create or run telemetry frontend error", diff --git a/docs/operations.md b/docs/operations.md index 4ce7fde..fac288e 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -119,7 +119,7 @@ cd backend npm run load:realtime -- --duration 30 --concurrency 25 --max-p95-ms 1500 ``` -Use `--mqtt-messages 5000` to exercise MQTT ingest backpressure with isolated, rejected test envelopes, and `--slow-ws-clients 10` to hold non-reading WebSocket clients during the run. MQTT mode uses `MQTT_BROKER_URL`, `MQTT_USERNAME`, and `MQTT_PASSWORD`; never point it at a broker outside the deployment under test. +Use `--mqtt-messages 5000` to exercise the bounded MQTT ingest queue with isolated, rejected test envelopes, and `--slow-ws-clients 10` to hold non-reading WebSocket clients during the run. MQTT mode uses `MQTT_BROKER_URL`, `MQTT_USERNAME`, and `MQTT_PASSWORD`; never point it at a broker outside the deployment under test. The internal HopReach compatibility load path is covered by `backend/src/api/hopreachCompatibility.test.ts`. Its UK-size test performs ten @@ -137,13 +137,8 @@ docker compose exec backend node dist/tools/backfillStatsRollups.js --apply --ho ``` The first command is a read-only inventory. The checkpoint and the rollup -slice commit together, so interruption resumes without double-counting. The -`STATS_AGGREGATE_READS_ENABLED` and `STATS_AGGREGATE_SHADOW_ENABLED` settings -accept comma-separated network names; `true` retains the all-network mode. -Start with `STATS_AGGREGATE_SHADOW_ENABLED=ukmesh` while reads remain disabled. -The bounded aggregate query and its legacy comparison run off the response path -even in this shadow-only mode. Set `STATS_AGGREGATE_READS_ENABLED=ukmesh` only -after backfill and a clean parity window. Once the +slice commit together, so interruption resumes without double-counting. Set +`STATS_AGGREGATE_READS_ENABLED=true` only after backfill. Once the aggregate-writing backend is live, reconcile the partial cutover hour: ```bash @@ -158,8 +153,8 @@ gap above 48 hours requires a newly reviewed historical backfill. Transactions that arrive during a rebuild increment the reconstructed rows after its lock is released. -A short validation window may set -`STATS_AGGREGATE_SHADOW_ENABLED=ukmesh`. The comparison pins one cutoff, scans +A short validation window may additionally set +`STATS_AGGREGATE_SHADOW_ENABLED=true`. The comparison pins one cutoff, scans the recent source rows once, and runs off the response path with one in-flight comparison per scope and a five-minute minimum interval. Dimension keys must match exactly. Count differences may be at most five packets or 0.1%, @@ -169,15 +164,6 @@ snapshots. Treat any `[stats-aggregate-shadow] mismatch` or back off. Shadow mode deliberately adds a raw validation scan and must not remain enabled. -Chart queries also share one `asOf` cutoff. `STATS_QUERY_DURATION_BUDGET_MS` -defaults to 60 seconds, `STATS_CHART_QUERY_TIMEOUT_MS` defaults to 120 seconds, -and `STATS_CHART_MAX_RESULT_ROWS` defaults to 100,000. A query that exceeds a -budget fails the chart refresh and is logged with a `[stats-query-budget]` -record; the existing stale snapshot remains available. The result-row limit is -an output-size guard. It does not measure rows scanned inside PostgreSQL, so -keep reviewing query plans and database scan telemetry before enabling reads on -additional networks. - Production aggregate reads combine maintained full-hour rows with raw boundary fragments, preserving the legacy exact rolling 24-hour/seven-day windows while keeping raw work bounded to partial hours. @@ -242,43 +228,6 @@ The public `/health` page polls the aggregated `/api/health` contract every minu - [SRTM cache, alert delivery, observer review, and planned publication](runbook-srtm-alerts-observers.md) - [Database compression and retention gates](db-lifecycle.md) -### Owner authorization baseline refresh - -`/readyz` fails closed when the current owner accounts, grants, configured grants, -or ACL generations differ from `OWNER_AUTH_INVENTORY_BASELINE_PATH`. This means -an owner grant or `OWNER_MQTT_USERNAME_MAP` entry changed after the baseline was -recorded. Drift is expected after every approved grant change and is not itself -an incident, but it must be reviewed before the baseline is refreshed. - -1. Identify the change from the audit trail and confirm it was approved: - - ```bash - docker exec -e PGOPTIONS='-c default_transaction_read_only=on' \ - meshcore-infra-timescaledb-1 psql -U meshcore -d meshcore_owner_auth -X \ - -c "SELECT occurred_at, event_type, source, actor FROM owner_grant_audit \ - WHERE occurred_at > now() - interval '30 days' ORDER BY occurred_at DESC" - ``` - -2. Generate and validate a fresh baseline with the running image (read-only - against the database; it only writes the new baseline file): - - ```bash - scripts/refresh-owner-baseline.sh - ``` - -3. Activate it after reviewing the printed counts/generations: - - ```bash - scripts/refresh-owner-baseline.sh --apply --recreate - curl -fsS http://127.0.0.1:3000/readyz - ``` - -The script refuses to overwrite an existing baseline, backs up `.env` before -editing, keeps the previous baseline for rollback, and validates the new file -against the live inventory before activation. To roll back, point -`OWNER_AUTH_INVENTORY_BASELINE_PATH` at the previous file and recreate the -backend. - Use the `/operations`, `/observer-registrations`, and operator audit pages for normal mutations. They enforce authorization, CSRF, typed confirmation, idempotency, capacity, and audit rules that direct Redis/SQL edits bypass. @@ -321,29 +270,6 @@ the reloader logs. Do not regenerate credentials over an existing set. If ACL reconciliation caused the alert, keep the last verified ACL and roll back that change. -### Denied PUBLISH on `/raw` (#269 / #213) - -**Disposition:** keep these denials. They are expected log noise from publishers -using a topic outside the supported observer contract; the evidence does not -show a packet-ingest gap. - -The B4 review brief reports about 233 Mosquitto `Denied PUBLISH` lines from -`ESP32_a806F8` for `meshcore/LBA/8CA23B78…/raw`, associated with a granted node, -plus a separate `meshmonitor-observer EXT` `/raw` family. The count and client -identities above come from that supplied log review; they were not re-counted -against the live broker during this code change. - -The renderer in `backend/src/mqtt/aclManager.ts` grants owner writes to -`packets`, `status`, `neighbors`, and `neighbours`. `backend/src/mqtt/topic.ts` -accepts those suffixes and rejects `/raw`. Packet bytes are carried in the -`raw` JSON field on the `/packets` envelope and decoded in -`backend/src/mqtt/client.ts`; there is no `/raw` topic consumer. Granting the -extra suffix would authorize a broker topic that the backend does not ingest. - -Reopen this ruling if a supported publisher is denied on `/packets`, or if the -ingest contract is deliberately extended with a `/raw` consumer. The topic -parser test keeps the current rejection explicit. - ### MeshIngestSilent Check backend MQTT readiness, broker connected-client events, public topic @@ -412,15 +338,6 @@ Inspect waiting/active connection metrics, slow route latency and PostgreSQL activity. Shed optional analysis/load, identify the bounded slow query, and avoid increasing pool size until connection and memory headroom are measured. -### DatabaseVacuumBacklog - -Inspect the `relname` table label, `n_dead_tup`, `n_live_tup`, and recent -autovacuum activity in PostgreSQL. A table with more than 50,000 dead tuples -is actionable even when its live-row estimate is small (as can happen for a -TimescaleDB chunk after deletes); run the reviewed bounded vacuum procedure and -confirm that both the alert and `/api/health` clear. Do not drop chunks or -delete the database volume as a response. - ### DatabaseWalRateHigh Identify current maintenance, backfill, compression or ingest activity. Pause @@ -434,36 +351,6 @@ databases and signed backups. Prune only explicitly identified unused build cache/images or let the bounded SRTM/log policies converge; never run a broad volume prune. -### HostDiskExhaustionForecast - -Treat a sustained forecast of disk exhaustion within 24 hours as an incident -before the low-space threshold fires. Identify the filesystem and its largest -consumers, compare the database hypertable and chunk growth gauges, pause -optional backfills, and preserve backup capacity. Do not remove database -chunks or volumes to make space; follow the signed backup and lifecycle gates. - -### CoreTelemetryGrowthHigh - -Compare the per-table byte and chunk gauges with the retention and compression -policies, then run the read-only `npm run db:lifecycle` inventory during a low -traffic window. Check for missed retention/compression jobs and forecast the -next 30 days of disk use. Changes to destructive retention require the current -signed backup/restore receipt and the per-table approval in `docs/db-lifecycle.md`. - -### CoreTelemetryRetentionOverdue - -Use read-only queries against `timescaledb_information.jobs` and -`timescaledb_information.chunks` to identify the missing or failing policy for -the labelled table. Verify backup and isolated restore evidence before running -the gated per-table lifecycle command. Keep expired forensic access isolated -and read-only; never expose restored expired rows through production APIs. - -### CoreTelemetryCompressionOverdue - -Inspect the labelled table's compression policy and Timescale job history, as -well as worker capacity and recent database/WAL pressure. Apply compression -only through the per-table, receipt-gated command in `docs/db-lifecycle.md`. - ### PacketPathsCapacity Compare the health-worker `meshcore_packet_paths_rows_30d`, @@ -473,16 +360,6 @@ is overdue, inspect the `packet_paths` compression job and Timescale worker headroom; never add a retention job for this table. For genuine growth, forecast the next 12 months against free disk before changing storage or ingest policy. -### Durable MQTT ingestion - -The MQTT client uses the library's inbound message callback for backpressure; -QoS 1 PUBACK follows successful database persistence. A database failure is -returned as a handler error, leaving the broker delivery unacknowledged for -redelivery and keeping readiness failed until a durable write succeeds. Inspect -the `failure`, `packet_persist_failure`, `status_persist_failure`, or -`neighbors_persist_failure` ingest outcomes and resolve the database issue; -do not clear the broker session or discard its inflight delivery. - ### BackupReceiptMissing Check the receipt mount, signature/key permissions, and backup job result. @@ -503,15 +380,16 @@ identify the malformed observer/client version without weakening validation. ## Spring-clean additions (2026-08-06) -### MQTT ingest resilience +### MQTT ingest resilience (`90d0dce`) -The MQTT client keeps a stable `clean=false` QoS-1 session and uses MQTT.js -message handling as the backpressure boundary. It sends PUBACK only after -packet, status, or neighbour persistence completes. Persistence failures are -propagated to MQTT.js and close the connection without PUBACK so the persistent -broker session redelivers the message. The process-local packet dedupe marker -is written only after packet-batch commit. The broker session is -`meshcore-analytics-ingest` (Mosquitto reports `c0`). +Ingest was silently lossy: failed packet-batch DB writes were logged + +DISCARDED (no retry), and the MQTT client used a CLEAN session (missed +messages on reconnect). Fix: idempotent transient batch retries + stable +`clean=false` QoS-1 MQTT session (broker sees `meshcore-analytics-ingest` +with `c0`) + ingest outcome/retry metrics. The backend does NOT dedupe +inserts (DISTINCT ON is readers/backfills only) — envelope-vs-packet count +differences vs services counting every MQTT message are expected, not loss. +Full audit: `INGEST-AUDIT-2026-08-06.md` in the repo root. ### Feed history contract diff --git a/docs/pathing-accuracy-2026-09-05.json b/docs/pathing-accuracy-2026-09-05.json new file mode 100644 index 0000000..5733df7 --- /dev/null +++ b/docs/pathing-accuracy-2026-09-05.json @@ -0,0 +1,374 @@ +{ + "baselineCommit": "1eefb50b31fda8d79ef383a658764671c65cb46a", + "snapshotSha256": "f92b5be6776f589ab3c237875c449989545ba56a6c15d9e0ec220b6cf2047632", + "capturedAt": "2026-09-05 01:18:07.69378+00", + "trainingBuckets": [ + 0, + 69 + ], + "developmentBuckets": [ + 70, + 84 + ], + "evaluationBuckets": [ + 85, + 99 + ], + "trainingSplitVerified": "Running worker had no PATH_LEARNING_TRAINING_SPLIT_PERCENT override; source default is 70.", + "readOnlySnapshot": true, + "comparisons": { + "lazy-single": { + "baseline": { + "snapshotSha256": "f92b5be6776f589ab3c237875c449989545ba56a6c15d9e0ec220b6cf2047632", + "capturedAt": "2026-09-05 01:18:07.69378+00", + "decoder": "1eefb50:lazy", + "split": "test", + "mode": "single", + "routes": 1377, + "excluded": 410, + "routeCorrect": 1035, + "totalHops": 9190, + "correctHops": 8351, + "routeAccuracy": 0.7516339869281046, + "hopAccuracy": 0.9087051142546246, + "rawThreeByteCorrect": 1184, + "uniqueRoutes": 1292, + "medianMs": 34.32457699999213, + "p95Ms": 64.22827799999504, + "p99Ms": 107.54097700000057 + }, + "current": { + "snapshotSha256": "f92b5be6776f589ab3c237875c449989545ba56a6c15d9e0ec220b6cf2047632", + "capturedAt": "2026-09-05 01:18:07.69378+00", + "decoder": "current", + "split": "test", + "mode": "single", + "routes": 1377, + "excluded": 410, + "routeCorrect": 1131, + "totalHops": 9190, + "correctHops": 8710, + "routeAccuracy": 0.8213507625272332, + "hopAccuracy": 0.9477693144722524, + "rawThreeByteCorrect": 1377, + "uniqueRoutes": 1292, + "medianMs": 11.225374000001466, + "p95Ms": 30.753426999999647, + "p99Ms": 41.991972999996506 + }, + "improved": 106, + "regressed": 10, + "routeGainPercentagePoints": 6.971677559912854, + "hopGainPercentagePoints": 3.9064200217627856, + "uniqueRouteGain95IntervalPercentagePoints": [ + 5.882352941176471, + 9.055727554179567 + ], + "strata": { + "known-source": { + "routes": 295, + "baselineCorrect": 227, + "currentCorrect": 234 + }, + "hops-7-12": { + "routes": 440, + "baselineCorrect": 284, + "currentCorrect": 338 + }, + "hops-2-3": { + "routes": 144, + "baselineCorrect": 131, + "currentCorrect": 130 + }, + "unknown-source": { + "routes": 1082, + "baselineCorrect": 808, + "currentCorrect": 897 + }, + "hops-4-6": { + "routes": 689, + "baselineCorrect": 597, + "currentCorrect": 604 + }, + "hops-13+": { + "routes": 104, + "baselineCorrect": 23, + "currentCorrect": 59 + } + } + }, + "lazy-multi": { + "baseline": { + "snapshotSha256": "f92b5be6776f589ab3c237875c449989545ba56a6c15d9e0ec220b6cf2047632", + "capturedAt": "2026-09-05 01:18:07.69378+00", + "decoder": "1eefb50:lazy", + "split": "test", + "mode": "multi", + "routes": 1377, + "excluded": 410, + "routeCorrect": 1078, + "totalHops": 9190, + "correctHops": 8617, + "routeAccuracy": 0.7828612926652142, + "hopAccuracy": 0.9376496191512513, + "rawThreeByteCorrect": 1246, + "uniqueRoutes": 1292, + "medianMs": 54.398984999992535, + "p95Ms": 150.54681999999957, + "p99Ms": 206.64727600000333 + }, + "current": { + "snapshotSha256": "f92b5be6776f589ab3c237875c449989545ba56a6c15d9e0ec220b6cf2047632", + "capturedAt": "2026-09-05 01:18:07.69378+00", + "decoder": "current", + "split": "test", + "mode": "multi", + "routes": 1377, + "excluded": 410, + "routeCorrect": 1135, + "totalHops": 9190, + "correctHops": 8716, + "routeAccuracy": 0.8242556281771968, + "hopAccuracy": 0.9484221980413493, + "rawThreeByteCorrect": 1377, + "uniqueRoutes": 1292, + "medianMs": 10.617765000000873, + "p95Ms": 30.299431999999797, + "p99Ms": 42.795510999999806 + }, + "improved": 71, + "regressed": 14, + "routeGainPercentagePoints": 4.139433551198257, + "hopGainPercentagePoints": 1.0772578890097932, + "uniqueRouteGain95IntervalPercentagePoints": [ + 3.018575851393189, + 5.804953560371517 + ], + "strata": { + "known-source": { + "routes": 295, + "baselineCorrect": 222, + "currentCorrect": 234 + }, + "hops-7-12": { + "routes": 440, + "baselineCorrect": 314, + "currentCorrect": 338 + }, + "hops-2-3": { + "routes": 144, + "baselineCorrect": 128, + "currentCorrect": 130 + }, + "unknown-source": { + "routes": 1082, + "baselineCorrect": 856, + "currentCorrect": 901 + }, + "hops-4-6": { + "routes": 689, + "baselineCorrect": 592, + "currentCorrect": 607 + }, + "hops-13+": { + "routes": 104, + "baselineCorrect": 44, + "currentCorrect": 60 + } + } + }, + "beta-single": { + "baseline": { + "snapshotSha256": "f92b5be6776f589ab3c237875c449989545ba56a6c15d9e0ec220b6cf2047632", + "capturedAt": "2026-09-05 01:18:07.69378+00", + "decoder": "1eefb50:beta", + "split": "test", + "mode": "single", + "routes": 1377, + "excluded": 410, + "routeCorrect": 479, + "totalHops": 9190, + "correctHops": 6315, + "routeAccuracy": 0.3478576615831518, + "hopAccuracy": 0.6871599564744287, + "rawThreeByteCorrect": 1165, + "uniqueRoutes": 1292, + "medianMs": 2.602124000000913, + "p95Ms": 9.193194999999832, + "p99Ms": 14.868608999999196 + }, + "current": { + "snapshotSha256": "f92b5be6776f589ab3c237875c449989545ba56a6c15d9e0ec220b6cf2047632", + "capturedAt": "2026-09-05 01:18:07.69378+00", + "decoder": "current", + "split": "test", + "mode": "single", + "routes": 1377, + "excluded": 410, + "routeCorrect": 1131, + "totalHops": 9190, + "correctHops": 8710, + "routeAccuracy": 0.8213507625272332, + "hopAccuracy": 0.9477693144722524, + "rawThreeByteCorrect": 1377, + "uniqueRoutes": 1292, + "medianMs": 11.225374000001466, + "p95Ms": 30.753426999999647, + "p99Ms": 41.991972999996506 + }, + "improved": 684, + "regressed": 32, + "routeGainPercentagePoints": 47.34931009440813, + "hopGainPercentagePoints": 26.06093579978237, + "uniqueRouteGain95IntervalPercentagePoints": [ + 45.04643962848297, + 51.0061919504644 + ], + "strata": { + "known-source": { + "routes": 295, + "baselineCorrect": 118, + "currentCorrect": 234 + }, + "hops-7-12": { + "routes": 440, + "baselineCorrect": 86, + "currentCorrect": 338 + }, + "hops-2-3": { + "routes": 144, + "baselineCorrect": 90, + "currentCorrect": 130 + }, + "unknown-source": { + "routes": 1082, + "baselineCorrect": 361, + "currentCorrect": 897 + }, + "hops-4-6": { + "routes": 689, + "baselineCorrect": 301, + "currentCorrect": 604 + }, + "hops-13+": { + "routes": 104, + "baselineCorrect": 2, + "currentCorrect": 59 + } + } + }, + "beta-multi": { + "baseline": { + "snapshotSha256": "f92b5be6776f589ab3c237875c449989545ba56a6c15d9e0ec220b6cf2047632", + "capturedAt": "2026-09-05 01:18:07.69378+00", + "decoder": "1eefb50:beta", + "split": "test", + "mode": "multi", + "routes": 1377, + "excluded": 410, + "routeCorrect": 485, + "totalHops": 9190, + "correctHops": 6318, + "routeAccuracy": 0.3522149600580973, + "hopAccuracy": 0.6874863982589772, + "rawThreeByteCorrect": 1142, + "uniqueRoutes": 1292, + "medianMs": 2.552361000000019, + "p95Ms": 6.777016999999432, + "p99Ms": 10.883194999998523 + }, + "current": { + "snapshotSha256": "f92b5be6776f589ab3c237875c449989545ba56a6c15d9e0ec220b6cf2047632", + "capturedAt": "2026-09-05 01:18:07.69378+00", + "decoder": "current", + "split": "test", + "mode": "multi", + "routes": 1377, + "excluded": 410, + "routeCorrect": 1135, + "totalHops": 9190, + "correctHops": 8716, + "routeAccuracy": 0.8242556281771968, + "hopAccuracy": 0.9484221980413493, + "rawThreeByteCorrect": 1377, + "uniqueRoutes": 1292, + "medianMs": 10.617765000000873, + "p95Ms": 30.299431999999797, + "p99Ms": 42.795510999999806 + }, + "improved": 679, + "regressed": 29, + "routeGainPercentagePoints": 47.20406681190995, + "hopGainPercentagePoints": 26.093579978237216, + "uniqueRouteGain95IntervalPercentagePoints": [ + 44.81424148606811, + 50.77399380804953 + ], + "strata": { + "known-source": { + "routes": 295, + "baselineCorrect": 118, + "currentCorrect": 234 + }, + "hops-7-12": { + "routes": 440, + "baselineCorrect": 90, + "currentCorrect": 338 + }, + "hops-2-3": { + "routes": 144, + "baselineCorrect": 89, + "currentCorrect": 130 + }, + "unknown-source": { + "routes": 1082, + "baselineCorrect": 367, + "currentCorrect": 901 + }, + "hops-4-6": { + "routes": 689, + "baselineCorrect": 304, + "currentCorrect": 607 + }, + "hops-13+": { + "routes": 104, + "baselineCorrect": 2, + "currentCorrect": 60 + } + } + } + }, + "productionSmoke": { + "readOnlyDatabaseFence": true, + "coldContextDetailMs": 3232.646745, + "oneByteWarmDetailMs": 286.86554200000046, + "threeByteWarmDetailMs": 500.3961670000008, + "oneByteObserverPaths": 7, + "threeByteObserverPaths": 19, + "allSingleMultiAndDetailResponsesPresent": true, + "observerProjectionChecksPassed": true, + "note": "Two packet smoke test during concurrent test/replay load; not a latency distribution." + }, + "validation": { + "backendBuild": "passed", + "backendTypecheck": "passed", + "frontendTypecheck": "passed", + "pathTests": { + "passed": 56, + "failed": 0 + }, + "frontendPathTests": { + "passed": 9, + "failed": 0 + }, + "fullBackendSuite": { + "passed": 336, + "failed": 3, + "preExistingUntrackedFailures": [ + "backend/src/analysis/runState.test.ts", + "backend/src/api/routes/rfValidation.test.ts", + "backend/src/repositories/networkAnalysis.test.ts" + ] + } + } +} diff --git a/docs/pathing-accuracy-2026-09-05.md b/docs/pathing-accuracy-2026-09-05.md new file mode 100644 index 0000000..1918b7e --- /dev/null +++ b/docs/pathing-accuracy-2026-09-05.md @@ -0,0 +1,173 @@ +# Path accuracy verification — 5 September 2026 + +The revised resolver is more accurate than both production algorithms at +commit `1eefb50b31fda8d79ef383a658764671c65cb46a` on the same held-out three-byte +flood observations degraded to one-byte prefixes. All production path views +now use the revised evidence provider. + +## Final results + +The evaluation contains **1,377 packets, 9,190 relay positions and 1,292 unique +relay chains**. Entire-route accuracy requires every relay identity to match; +unresolved positions count as errors. + +| Resolver | One-byte exact routes | One-byte correct hops | Original three-byte routes preserved | +| --- | ---: | ---: | ---: | +| Previous beta, single observer | 34.79% (479) | 68.72% (6,315) | 1,165 / 1,377 | +| Previous detail resolver, single observer | 75.16% (1,035) | 90.87% (8,351) | 1,184 / 1,377 | +| Revised, single observer | **82.14% (1,131)** | **94.78% (8,710)** | **1,377 / 1,377** | +| Previous beta, multiple observers | 35.22% (485) | 68.75% (6,318) | 1,142 / 1,377 | +| Previous detail resolver, multiple observers | 78.29% (1,078) | 93.76% (8,617) | 1,246 / 1,377 | +| Revised, multiple observers | **82.43% (1,135)** | **94.84% (8,716)** | **1,377 / 1,377** | + +Against the stronger previous multi-observer implementation, the revision +corrects **71 routes** and regresses **14**: a net gain of 57 routes, or +**19.1% fewer incorrect routes**. It corrects 99 additional relay positions. +The route gain is 4.14 percentage points; a deterministic bootstrap giving +each unique relay chain equal weight yields a 95% interval of **+3.02 to ++5.80 percentage points** for that unique-chain metric. This interval is not +an estimate of accuracy on every real one-byte packet. + +The multi-observer gain appears in every reported route-length and source +stratum. On 104 routes with at least 13 relays, exact matches rise from 44 to +60. On 1,082 packets without a known source, they rise from 856 to 901. +Some individual cases still regress; one-byte identities remain ambiguous. + +[Machine-readable aggregate results](pathing-accuracy-2026-09-05.json) include +all paired comparisons, counts, intervals and strata. + +## What changed + +- The live beta resolver previously omitted the learner's position-prefix, + corridor and position-transition tables and used transition probabilities + where the scoring weights expected counts. Those signals now reach the + shared decoder, including directed source and receiver edges. +- Candidate bounds grow with route length. Long flood routes no longer lose + their distant first relays to an observer-centered, one-hop bounding box. +- Unique multibyte identities survive missing coordinates and contradictory + terrain estimates. Repeated, verified directed training edges can override + geographic rejection of inferred links. Unpositioned inferred candidates + need directed link evidence and retain null coordinates. +- Mixed hash widths index every matching candidate, preserving elevation. + Forward/backward passes cache transition scores to keep inference bounded. +- RX prefix collisions no longer remove a real terminal relay. Direct-routing + instructions are excluded from observed flood paths. Invalid hashes cannot + shift later positions. One-byte processing supports up to 63 hops. +- Distinct flood copies heard by one observer are retained. Confirmed branches + stay separate; observer-specific projections stop at their own receivers. +- Map labels use each branch's identities, so co-located relays cannot borrow + another branch's label or collapse distinct decoded routes by coordinates. + A relay sharing the source's byte prefix no longer removes that source + endpoint from the projected path. +- Late evidence can refine any position in a held chain. Holding a choice + does not manufacture perfect confidence. +- Packet details, overlays, slow finalization and history batches share the + decoder and evidence provider. Historical UKMesh network labels remain in + scope. Public visibility checks remain in place. +- Training preserves gaps and verified loops, excludes direct-routing paths, + and uses RX/TX metadata. A missing relay cannot create a learned shortcut. + +## Method and reproducibility + +The read-only, repeatable-read snapshot was captured at +**2026-09-05 01:18:07.69378 UTC**, over the preceding 14 days. It includes +14,542 public node records, 155,581 link records, the published prior tables +and 39,786 observations belonging to 3,489 candidate held-out packet hashes. +UKMesh's historical `northeast` and `teesside` labels are included. + +Snapshot SHA-256: + +```text +f92b5be6776f589ab3c237875c449989545ba56a6c15d9e0ec220b6cf2047632 +``` + +The running learner used the default PostgreSQL hash buckets 0–69 for +training; its environment had no training-split override. Buckets 70–84 were +used for development and 85–99 for final evaluation. Both splits are disjoint +from training at packet-hash level, including every observer's copy of the +packet. The geographic and learned weights were not tuned on final results. + +Ground truth requires a three-byte **flood** route, uniquely matching public +node identities and a positioned observer. Nodes without current coordinates +remain eligible as known identities. Direct routes, missing/colliding node +identities and paths without sufficient usable truth exclude 410 of the +1,787 final candidate packet hashes. There is no distance-based exclusion. +This sample covers 2–21 relay positions; longer one-byte paths are supported +and tested structurally but cannot have equivalent three-byte paths beyond +the protocol's path-byte limit. + +Every one-byte replay truncates all visible sibling observations, not just the +target path. Multi mode admits siblings within 60 seconds on either side of +the selected observation. Both algorithms receive the same snapshot and +observation window. The archived baseline changes only the export visibility +of its existing packet-preparation helper, not its implementation or scores. +The former detail resolver's SQL adapter is replayed against the frozen +snapshot. Node IDs, rather than rendered coordinates, determine correctness. + +This is **packet-held-out validation using current metadata**, not a temporal +forecast or a recreation of past coordinates. Recurrent corridors may occur +in both training and evaluation. Learned priors are frozen identically for +both algorithms, and unique-chain bootstrap results reduce the influence of +recurrent route copies. It does not establish that every live one-byte path, +or every confidence value, has the reported accuracy. + +The local snapshot, per-packet results, comparisons, source hashes and test +logs are under `artifacts/path-accuracy/2026-09-05-final/` (ignored by Git). +The aggregate JSON linked above is tracked. [Pathing documentation](pathing.md) +contains capture, baseline archive, replay and comparison commands. The older +`path-lazy/evaluate.ts` is marked as historical diagnostics because its +original sampling did not enforce held-out membership. + +## Runtime and validation + +The revised offline decode measured **10.62 ms median, 30.30 ms p95 and +42.80 ms p99** in multi-observer mode. Single-observer p95 was 30.75 ms. +These measure decoding against a loaded context, including observation +preparation; they exclude database loading and network latency. Baseline lazy +replay timings include its snapshot-query adapter and should not be used as +a production latency comparison. + +A separate smoke test ran the actual database-backed single, multi and detail +resolver entry points with PostgreSQL enforcing +`default_transaction_read_only=on`. One-byte and three-byte samples produced +all three responses, with correct observer endpoints. Warm detail requests +were approximately 287 ms and 500 ms; the first request loading context took +3.23 seconds. These are two samples under concurrent test load, not latency +percentiles. The existing worker pool, context caching and propagation +settling window remain in use. + +Validation completed: + +- Backend TypeScript build/typecheck and frontend TypeScript check passed. +- **56 path, learning, cache, worker, DTO, API and accuracy-harness tests passed.** +- **9 relevant frontend path/rendering tests passed.** +- Full backend run: **336 passed, 3 failed** in pre-existing, untracked files. + `analysis/runState.test.ts` imports a missing `normalizeRetiredAnalysisState`; + `repositories/networkAnalysis.test.ts` expects an unprovided abort signal; + `api/routes/rfValidation.test.ts` waits indefinitely for that signal and its + test process was terminated after confirming the failed assertion. These + files were present before this work and were not modified. +- Database-backed smoke tests were read-only; no prior tables were rebuilt. + +Deployed live on **2026-09-05 at 02:03 UTC**. The backend, path-learning worker, +and app now run the reviewed path implementation. Both `ukmesh` and `all` +models have published `path-learning-v5-contiguous` priors. The accuracy comparison +above still uses the same frozen priors for both implementations. + +Production verification passed: six single/multi/detail API checks across public +one-byte and three-byte packets, all eight identities in the sampled three-byte +route, 64 served JS/CSS assets, public app readiness and HTML, and live packet +streaming through the app proxy. The private-packet probe returns 404. The API and +app are healthy, with no container restarts or OOM events after the final rollout. +Other running containers retained their original IDs. + +The rollout exposed an August owner-inventory snapshot that the current readiness +check rejected. A fresh inventory was validated against the unchanged pre-deploy +broker policy and pinned; the original snapshot was preserved. An initial smoke +probe accidentally selected a private packet and triggered the deployment guard's +rollback. Public samples were then checked in the release image before the +successful retry. No access grants were changed. + +Image IDs, checks, exact build-source archive, and rollback references are recorded +in [the local deployment evidence](../artifacts/path-accuracy/2026-09-05-deploy/completion.json) +and its adjacent `manifest.json`. Previous service images remain tagged for rollback. diff --git a/docs/pathing.md b/docs/pathing.md index 7b41640..5da9acb 100644 --- a/docs/pathing.md +++ b/docs/pathing.md @@ -1,69 +1,117 @@ # Pathing -## Purpose +Production packet details, single-observer overlays, multi-observer overlays, +slow finalization and history batches use `path-core/decoder.ts` and the +evidence provider in `path-beta/sharedDecoder.ts`. The previous lazy resolver +is retained only for accuracy comparisons; workers no longer route to it. -Pathing resolves packet routes into confident relay chains. The production -pipeline (2026-08) is the Viterbi champion (`vit_src`) from the pathing -experiment: it walks a candidate trellis built from 1-byte hop prefixes and -scores emissions/transitions from seven calibrated signals, including -positional prefix-frequency priors, corridor interpolation, and **ITM -terrain feasibility** (no hop is placed where line-of-sight physics rules it -out). Measured on the gold holdout set: **97.27% route accuracy / 99.24% -hop accuracy**. +The September 2026 changes and comparison against commit `1eefb50` are in +[the accuracy report](pathing-accuracy-2026-09-05.md). The previous 97.27% route / +99.24% hop claim came from a different, narrower experiment and should not be +used as the accuracy of today's traffic. -## Main layers +## Evidence and decoding -- `backend/src/path-beta/` - - resolver implementation (`resolver.ts`) — Viterbi candidate trellis - - worker pool - - geometry helpers (terrain-aware arcs) - - fallback logic -- `backend/src/pathing/` - - service/repository orchestration for API-facing pathing endpoints - - `pathingPublicDto.ts` — the public DTO projector. **Any new resolver - field must be added here or the API silently drops it.** -- `backend/src/pathing/pathingService.ts` - - cache and resolver orchestration +The learner builds priors from independently identified multibyte relays. +The live context loads position-prefix, source/receiver corridor, directed +transition and position-specific transition counts. The scoring weights use +counts, not transition probabilities. Source edges have transition position +0; the first relay-to-relay edge has position 1. Observer endpoint edges use +the same directed evidence. -## Production behaviour (2026-08-06) +A Viterbi trellis finds a coherent chain of node IDs matching the observed +prefixes. Geography, corridor interpolation, direct receiver anchors and ITM +viability contribute to ranking. Candidate bounds grow with path length so +long flood routes can extend beyond one hop from their observers. Columns +remain capped at 64 candidates. Forward and backward passes share cached +transition scores. Resolution remains in the bounded worker pool; network +context is cached for 15 minutes, and the propagation settling window remains. -- **Slow-mode resolution**: packets wait out their propagation window before - the path is finalised (`469c84e`), so late observations don't re-shape a - settled route. -- **Physics gates + held-path refinement** (`5dd15d9`): candidate hops that - violate physical feasibility are gated; held paths are refined in place. -- **Canonical path DTO**: `canonicalPath` is the authoritative route in API - responses; the frontend renders it as blue paths. Path identity colours - live-path arcs by the resolved packet (`f6f8290`). -- Terrain-aware live paths + airborne-hop markers (`a76ab1e`); live path arcs - stay above terrain (`7bcabcc`). +Collision-free two- and three-byte observations lock node identities. +Recorded identities survive absent/stale coordinates and contrary terrain +estimates. For inferred routes, repeatedly verified directed training edges +can override a geometric rejection; otherwise physics gates still apply. +A node without coordinates needs observed directed transitions to participate +in an inferred route. Its coordinates stay null so the map breaks the line. -## Evidence priorities +## Observation handling -In practice the resolver should rank evidence roughly like this: +Flood paths describe recorded relays; direct paths describe routing +instructions and are excluded from observed-route inference and training. +See the [MeshCore packet format](https://github.com/meshcore-dev/MeshCore/blob/main/docs/packet_format.md) +and [firmware path handling](https://github.com/meshcore-dev/MeshCore/blob/main/src/Mesh.cpp). -- physically plausible link support (ITM/LOS) -- multibyte path evidence -- radio-neighbour evidence -- weaker observational hints +RX observations retain their last relay even if its short prefix matches the +observer. TX observations remove an appended observer prefix. Legacy rows +without direction retain the previous terminal-trimming convention. +Malformed hashes reject an observation rather than shifting later hops. +Valid one-byte paths support the protocol's 63-hop count instead of stopping +after 24 hops. -## API ownership +An observer can hear several flood copies over different routes. Multi-observer +inference keeps those variants and de-duplicates identical receptions. +Compatible prefixes share a decode; conflicting multibyte routes stay separate. +A coarse observation compatible with several confirmed routes keeps its own +group. Each projected path ends at its own observer, without adding edges to +another observer's branch. +Map labels use each branch's own relay identities; coordinates alone cannot +identify co-located nodes. -- `backend/src/api/routes/pathing.ts` - - thin HTTP wrapper -- `backend/src/pathing/pathingService.ts` - - cache and resolver orchestration -- `backend/src/pathing/pathingRepository.ts` - - DB-backed path history and learning queries +A held path is reconsidered across the entire chain when evidence changes. +A better chain must clear the existing score margin; holding an assignment +does not turn its confidence into 100%. -## Contributor rule +The learner preserves original positions across missing or ambiguous relays +and cannot learn a shortcut across a missing relay. Its version is now +`path-learning-v5-contiguous`; the next normal rebuild republishes corrected +priors without a schema migration. -If a path looks wrong, first determine whether the issue is: +## Ownership and scope -- evidence weighting -- physical graph quality (ITM feasibility) -- ambiguous short-hash matching -- cache reuse -- a missing field in the public DTO projector (`pathingPublicDto.ts`) +- `path-beta/resolver.ts`: scoped evidence loading, observation preparation, + cached context and API projections. +- `path-beta/championPriors.ts`: count aggregation and bounded prior loading. +- `path-core/decoder.ts`: candidate indexing, scoring, Viterbi traversal and + ambiguity margins. +- `path-learning/rebuild.ts`: multibyte training and atomic publication. +- `pathing/pathingPublicDto.ts`: explicit public DTO projection. Add new + resolver fields here before clients can receive them. -Do not debug path rendering as if it were a frontend-only problem. +UKMesh retains historical `northeast` and `teesside` evidence through the +shared network scope. Private-node exclusions and visibility-generation cache +fences continue to apply. Test networks remain isolated. + +## Reproducing accuracy checks + +Run from `backend/`. Capture uses a repeatable-read, read-only transaction. +Replays are offline and do not update priors, online state or production +caches. Local snapshots and traces belong under `artifacts/path-accuracy/`, +which is ignored by Git. + +```sh +node --env-file=../.env --import tsx src/tools/pathAccuracy.ts capture /tmp/path-snapshot.json +npm run path:accuracy -- baseline /tmp/path-baseline 1eefb50 +npm run path:accuracy -- replay /tmp/path-snapshot.json /tmp/new.json current test +npm run path:accuracy -- replay /tmp/path-snapshot.json /tmp/old.json /tmp/path-baseline/backend/src/path-lazy/lazyResolver.ts test +npm run path:accuracy -- compare /tmp/old.json /tmp/new.json /tmp/comparison.json +``` + +Use `PATH_ACCURACY_MODE=multi` for multi-observer replay. Substitute +`path-beta/sharedDecoder.ts` in the baseline path to compare the old beta +resolver. `PATH_ACCURACY_DAYS` selects the capture window (default 14, maximum +45); `PATH_ACCURACY_NETWORK` defaults to `ukmesh`. + +The harness reserves PostgreSQL packet-hash buckets 70–84 for development and +85–99 for evaluation. **Verify that the running learner uses the default 70% +training split before capture**; training on more than 70% would contaminate +these held-out packets. Node/link metadata is a current snapshot, so this is +a packet-held-out comparison, not a historical simulation of metadata at the +time of each reception. + +Truth comes only from three-byte flood paths with uniquely matching public +node identities and a positioned observer. One-byte replay hides the extra +bytes in every sibling observation. Cases with unresolved or colliding truth +are reported as excluded; geographic distance does not exclude a route. +Comparisons require identical snapshots and labels, count unresolved guesses +as errors, report regressions and corrections, and bootstrap unique relay +chains rather than treating recurrent copies as independent routes. diff --git a/docs/privacy.md b/docs/privacy.md index ac70e40..bc50a82 100644 --- a/docs/privacy.md +++ b/docs/privacy.md @@ -36,6 +36,7 @@ The service stores radio traffic observed on the MeshCore network. |---|---|---| | `packets` — message content + raw bytes + metadata | **at most 30 days** (Timescale chunk policy + bounded exact-row boundary cleanup) | network diagnostics; content is not kept long-term | | `packet_decryptions` — full decrypted messages | **30 days** (operational retention loop) | diagnostics only | +| `owner_packet_share_deliveries` — validated packet payload and delivery metadata | **30 days after terminal delivery** (operational retention loop) | durable site-to-site delivery and deduplication; retryable rows remain until terminal | | `packet_paths` — path metadata (node IDs, timing, signal, hashes) | **indefinite, content-stripped** | raw pathing analytics is the product | | `multibyte_path_facts`, `multihop_paths_weekly`, `path_*_priors` — derived path stats | indefinite | derived, pseudonymous | | `packet_daily_stats` / `packet_hourly_stats` | indefinite | anonymous aggregates | @@ -64,23 +65,7 @@ The service stores radio traffic observed on the MeshCore network. deletion procedure in the ops playbook (rows by src/rx node id across packets, packet_paths, decryptions, path facts). -## 3. Device status telemetry - -Status samples retain a JSON telemetry object for the authenticated owner -portal. The unauthenticated `/node-status/latest` and `/mqtt-nodes` APIs expose -only their explicit public fields: node identity, sample/availability data, -battery, uptime, airtime, channel utilization, and recent packet counts. They -do not return the telemetry object or forward unrecognized fields. Wi-Fi -identifiers, MQTT broker/account details, reset and configuration fields, -firmware metadata, and filesystem details are available only through -owner-authenticated endpoints. Unrecognized keys stay in storage until they -are deliberately added to a response contract. - -Existing status samples do not need a database rewrite: public responses are -projected through the same field allowlist regardless of when the sample was -stored. - -## 4. Cookies +## 3. Cookies LocalStorage-only choices (cookie consent, theme). No third-party cookies. PECR does not apply to the mesh radio data (not a public electronic diff --git a/docs/runbook-backup-restore.md b/docs/runbook-backup-restore.md index df8cb1c..a473b3a 100644 --- a/docs/runbook-backup-restore.md +++ b/docs/runbook-backup-restore.md @@ -59,12 +59,6 @@ export MESHCORE_INFRA_PROJECT_NAME=meshcore-infra scripts/backup.sh ``` -`MESHCORE_INFRA_COMPOSE_FILE` can select a non-default external Compose file; -`MESHCORE_INFRA_PROJECT_DIR` can set its project directory explicitly. The -shared resolver checks that Compose project and requires one running container -for each exact `timescaledb`, `redis`, and `mosquitto` service. It does not -search container names across projects. - For the returned receipt and archive: ```bash diff --git a/docs/runbook-release-rollback.md b/docs/runbook-release-rollback.md index aeeba7c..d411347 100644 --- a/docs/runbook-release-rollback.md +++ b/docs/runbook-release-rollback.md @@ -45,8 +45,6 @@ export COSIGN_CERTIFICATE_OIDC_ISSUER='https://token.actions.githubusercontent.c export RESTORE_RECEIPT_PATH=/secure/receipts/latest.json export RESTORE_RECEIPT_VERIFY_KEY=/secure/receipts/verify.pem export RELEASE_RECEIPT_SIGNING_KEY=/secure/release/receipt-signing.pem -export MESHCORE_INFRA_DIR=/home/ben/ukmesh/meshcore-infra -export MESHCORE_INFRA_PROJECT_NAME=meshcore-infra scripts/replace-container.sh backend \ --image="$BACKEND_IMAGE" \ @@ -54,13 +52,6 @@ scripts/replace-container.sh backend \ --source-revision="$SOURCE_REVISION" ``` -If the infrastructure Compose file has another path, set -`MESHCORE_INFRA_COMPOSE_FILE` and, when needed, -`MESHCORE_INFRA_PROJECT_DIR`. The release script validates that external -project and resolves its single running `timescaledb` container before it -starts the migration job; the post-migration schema check uses that exact -container ID. - Repeat for each application service, passing its digest as `--image` and the same signed backend digest as `--backend-image`. The script verifies signatures and revision labels, runs migrations, starts the prior backend against the diff --git a/docs/runbook-srtm-alerts-observers.md b/docs/runbook-srtm-alerts-observers.md index 33f0ca0..c4e53d9 100644 --- a/docs/runbook-srtm-alerts-observers.md +++ b/docs/runbook-srtm-alerts-observers.md @@ -29,18 +29,8 @@ dead-letter reason. Alertmanager groups by alert name/severity, sends firing and resolved events to the internal receiver, and retains its own state for 120 hours. The receiver -stores bounded, rotated summaries in `alert_receiver_data`. When -`ALERT_FORWARD_URL` is configured, the receiver fsyncs each forwarding record -into `alerts.jsonl.queue` before returning HTTP 202, then retries with bounded -backoff and moves exhausted deliveries into a `.dead.json` record. Pending items -and the last successful delivery time survive receiver restarts. - -If `ALERT_FORWARD_URL` is unset or invalid, the receiver operates in archive-only -mode. It still accepts alerts after the local receipt is persisted, but both -`/healthz` and `/readyz` return 503 so Compose and operators see degraded -delivery. Health JSON includes `pending_count`, `queue_oldest_age_seconds`, -`dead_letter_count`, `last_success_at`, and `last_error`. Resolve any dead-letter -or aged-queue condition before treating a firing alert as delivered. +stores bounded, rotated summaries in `alert_receiver_data`; optional forwarding +uses `ALERT_FORWARD_URL`. ```bash docker compose exec -T alertmanager amtool \ diff --git a/docs/spam-detection.md b/docs/spam-detection.md index 67d130a..6d95169 100644 --- a/docs/spam-detection.md +++ b/docs/spam-detection.md @@ -100,8 +100,10 @@ A cluster only becomes an incident when it is genuine abuse, not normal traffic: *or* the spam marker. Trivial/connectivity messages (`test`, `ack`, `morning`, `gm`, …) never qualify on their own. - **Requires a suspicious sender pattern** — a single repeating sender, one - dominant sender, or **rotating similar usernames**. Many *dissimilar* people - sending the same thing is treated as broad legitimate chatter, not one abuser. + dominant sender, **rotating similar usernames**, or a **rotating-ID botnet** + (see below). Many *dissimilar* **people** sending the same thing is treated + as broad legitimate chatter, not one abuser — but "many senders" made of + single-shot machine-named identities is a botnet, not people. - The **test channel is excluded by default** (it carries repeated test traffic by design). Configurable via `SPAM_MESSAGE_EXCLUDE_CHANNELS`. A flood that *also* hits another channel (e.g. Public) is still caught via those copies. @@ -116,6 +118,29 @@ A cluster only becomes an incident when it is genuine abuse, not normal traffic: test/connectivity flood is never rewarded, so it stays below the public floor no matter how large it grows. +### Rotating-ID botnets + +A distinct abuse class: the **same canned line blasted from many fresh sender +IDs** — one message per ID, in timed bursts (a handful of IDs within minutes), +then quiet. Per-sender rate logic sees nothing; the tell is one text arriving +from many *one-shot* identities. A cluster is flagged `rotatingIdentities` +when ALL of: + +- ≥ `SPAM_MESSAGE_ROTATING_MIN_SENDERS` distinct senders, +- ≥ `SPAM_MESSAGE_ROTATING_SINGLE_SHOT_SHARE` of them appear exactly once in the + analysis window (one-shot identities, not established members), +- ≥ `SPAM_MESSAGE_ROTATING_MACHINE_NAME_SHARE` of the names look + machine-generated (`looksMachineGeneratedName` in `normalize.ts`: + separator-free base32-ish tokens, ≥1 digit, vowel-poor, mostly uppercase — + amateur-callsign shapes and human composites are excluded), and +- ≥ `SPAM_MESSAGE_ROTATING_MIN_BURST_SENDERS` distinct senders inside one burst + window. + +Flagged clusters are exempted from the broad-chatter rejection, scored +0.3, and +carry the reason "N single-shot identities (X% machine-style names, N within +10 min) posted near-identical text". First production catch: the Oct-2026 +anti-Kirby campaign — 9 incidents, 8–15 sender IDs each, confidence 0.54–0.81. + ## Incident lifecycle - **active / ongoing** while the last matching message is within @@ -145,6 +170,10 @@ All knobs live in `backend/src/spam/config.ts` and read from the environment. | `SPAM_MESSAGE_MAX_INDEPENDENT_SENDERS` | `4` | Above this (dissimilar) → broad chatter | | `SPAM_MESSAGE_DOMINANT_SENDER_SHARE` | `0.6` | One sender at this share qualifies | | `SPAM_MESSAGE_FLOOD_MIN_MESSAGES` | `25` | ≥ this many near-duplicates → sustained-flood boost | +| `SPAM_MESSAGE_ROTATING_MIN_SENDERS` | `5` | Min distinct senders for rotating-ID detection | +| `SPAM_MESSAGE_ROTATING_SINGLE_SHOT_SHARE` | `0.8` | Share of senders seen exactly once in the window | +| `SPAM_MESSAGE_ROTATING_MACHINE_NAME_SHARE` | `0.5` | Share of names that must look machine-generated | +| `SPAM_MESSAGE_ROTATING_MIN_BURST_SENDERS` | `3` | Min distinct senders inside one burst window | | `SPAM_MESSAGE_EXCLUDE_CHANNELS` | `test` | Channel labels skipped (comma list) | | `SPAM_MESSAGE_PUBLIC_MIN_SCORE` | `0.5` | Confidence floor for public list / status | | `SPAM_MESSAGE_ONGOING_WINDOW_MIN` | `30` | How long an incident stays "ongoing" | @@ -171,8 +200,9 @@ Two separate confidences are shown: - **Detection confidence** (incident `confidence`, 0–1) — how strongly the pattern resembles spam. Built from volume, burst density, sustained-flood - size, repeated links, the spam marker, and rotating sender names; content - dominated by connectivity/test tokens is dampened. `High ≥ 0.66`, + size, repeated links, the spam marker, rotating sender names, and + rotating-identity evidence; content dominated by connectivity/test tokens is + dampened. `High ≥ 0.66`, `Medium ≥ 0.4`, else `Low`. Only incidents at/above `SPAM_MESSAGE_PUBLIC_MIN_SCORE` appear publicly by default. - **Origin confidence** (origin `confidence` + `level`) — how trustworthy the @@ -248,6 +278,13 @@ docker compose exec backend node dist/tools/recomputeSpamMessages.js --hours 720 docker compose exec backend node dist/tools/recomputeSpamMessages.js --hours 720 --apply ``` +If the analyzer log shows `status=timed_out`, the run's work exceeded +`SPAM_MESSAGE_ANALYSIS_BUDGET_MS` — the 5 s default is tuned for quiet days; +busy spam days need ~30–60 s (each public incident resolves relay paths for +origin). Raise it in `.env` (e.g. `SPAM_MESSAGE_ANALYSIS_BUDGET_MS=60000`) and +recreate the backend. The recompute tool loads more rows than the analyzer and +may need `SPAM_MESSAGE_MAX_MESSAGES` raised for wide windows. + ## Tests Pure logic (normalization, fuzzy matching, username similarity, clustering, diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 9daa2eb..9f24a4f 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -14,6 +14,7 @@ "@deck.gl/mapbox": "^9.4.0", "fuse.js": "^7.5.0", "maplibre-gl": "^6.11.2", + "polygon-clipping": "^0.15.7", "react": "19.3.0", "react-aria-components": "^1.21.1", "react-dom": "19.3.0", @@ -2146,6 +2147,15 @@ "node": ">=20" } }, + "node_modules/polygon-clipping": { + "version": "0.15.7", + "resolved": "https://registry.npmjs.org/polygon-clipping/-/polygon-clipping-0.15.7.tgz", + "integrity": "sha512-nhfdr83ECBg6xtqOAJab1tbksbBAOMUltN60bU+llHVOL0e5Onm1WpAXXWXVB39L8AJFssoIhEVuy/S90MmotA==", + "dependencies": { + "robust-predicates": "^3.0.2", + "splaytree": "^3.1.0" + } + }, "node_modules/postcss": { "version": "8.5.28", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.28.tgz", @@ -2390,6 +2400,11 @@ "protocol-buffers-schema": "^3.3.1" } }, + "node_modules/robust-predicates": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/robust-predicates/-/robust-predicates-3.0.3.tgz", + "integrity": "sha512-NS3levdsRIUOmiJ8FZWCP7LG3QpJyrs/TE0Zpf1yvZu8cAJJ6QMW92H1c7kWpdIHo8RvmLxN/o2JXTKHp74lUA==" + }, "node_modules/rolldown": { "version": "1.2.9", "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.2.9.tgz", @@ -2439,6 +2454,14 @@ "node": ">=0.10.0" } }, + "node_modules/splaytree": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/splaytree/-/splaytree-3.2.3.tgz", + "integrity": "sha512-7OXrNWzy6CK+r7Ch9OLPBDTKfB6XlWHjX4P0RU5B3IgFuWPeYN0XtRtlexGRjgbQxpfaUve6jTAwBGWuGntz/w==", + "engines": { + "node": ">=18.20 || >=20" + } + }, "node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", diff --git a/frontend/package.json b/frontend/package.json index 43edf13..bbc8d41 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -21,6 +21,7 @@ "@deck.gl/mapbox": "^9.4.0", "fuse.js": "^7.5.0", "maplibre-gl": "^6.11.2", + "polygon-clipping": "^0.15.7", "react": "19.3.0", "react-aria-components": "^1.21.1", "react-dom": "19.3.0", diff --git a/frontend/playwright.config.ts b/frontend/playwright.config.ts index 801a9c3..5058377 100644 --- a/frontend/playwright.config.ts +++ b/frontend/playwright.config.ts @@ -1,8 +1,4 @@ import { defineConfig, devices } from '@playwright/test'; -import { portBase, publicOrigin, dashboardOrigin } from './test/e2e/localServers.js'; - -// Parallel worktrees can select isolated ports instead of reusing a stale server. -const reuseExistingServer = !process.env['CI'] && !process.env['PLAYWRIGHT_PORT_BASE']; export default defineConfig({ testDir: './test/e2e', @@ -17,18 +13,18 @@ export default defineConfig({ projects: [ { name: 'public-desktop', - testMatch: /(?:accessibility|owner|public|feed-virtualizer)\.spec\.ts/, - use: { ...devices['Desktop Chrome'], baseURL: publicOrigin }, + testMatch: /(?:accessibility|owner|public)\.spec\.ts/, + use: { ...devices['Desktop Chrome'], baseURL: 'http://127.0.0.1:4173' }, }, { name: 'dashboard-desktop', testMatch: /dashboard\.spec\.ts/, - use: { ...devices['Desktop Chrome'], baseURL: dashboardOrigin }, + use: { ...devices['Desktop Chrome'], baseURL: 'http://127.0.0.1:4174' }, }, { name: 'dashboard-mobile', testMatch: /dashboard\.spec\.ts/, - use: { ...devices['Pixel 7'], baseURL: dashboardOrigin }, + use: { ...devices['Pixel 7'], baseURL: 'http://127.0.0.1:4174' }, }, { name: 'mobile-regression', @@ -38,19 +34,19 @@ export default defineConfig({ ], webServer: [ { - command: `VITE_APP_HOSTNAME=app.invalid VITE_SITE=ukmesh npm run dev -- --config playwright.vite.config.ts --host 127.0.0.1 --port ${portBase} --strictPort`, - port: portBase, - reuseExistingServer, + command: 'VITE_APP_HOSTNAME=app.invalid VITE_SITE=ukmesh npm run dev -- --host 127.0.0.1 --port 4173', + port: 4173, + reuseExistingServer: !process.env['CI'], }, { - command: `VITE_APP_HOSTNAME=127.0.0.1 VITE_SITE=ukmesh VITE_NETWORK=ukmesh VITE_RF_COVERAGE_ENABLED=true npm run dev -- --config playwright.vite.config.ts --host 127.0.0.1 --port ${portBase + 1} --strictPort`, - port: portBase + 1, - reuseExistingServer, + command: 'VITE_APP_HOSTNAME=127.0.0.1 VITE_SITE=ukmesh VITE_NETWORK=ukmesh VITE_RF_COVERAGE_ENABLED=true npm run dev -- --host 127.0.0.1 --port 4174', + port: 4174, + reuseExistingServer: !process.env['CI'], }, { - command: `VITE_APP_HOSTNAME=app.invalid VITE_SITE=dev npm run dev -- --config playwright.vite.config.ts --host 127.0.0.1 --port ${portBase + 2} --strictPort`, - port: portBase + 2, - reuseExistingServer, + command: 'VITE_APP_HOSTNAME=app.invalid VITE_SITE=dev npm run dev -- --host 127.0.0.1 --port 4175', + port: 4175, + reuseExistingServer: !process.env['CI'], }, ], }); diff --git a/frontend/public/sw-runtime.js b/frontend/public/sw-runtime.js index 88f2ba0..7bb7321 100644 --- a/frontend/public/sw-runtime.js +++ b/frontend/public/sw-runtime.js @@ -1,10 +1,10 @@ export const SW_CACHE_POLICY = Object.freeze({ - version: 'v7', - tileCache: 'meshcore-tiles-v7', - appCache: 'meshcore-app-v7', - metadataCache: 'meshcore-meta-v7', - previousTileCache: 'meshcore-tiles-v6', - previousAppCache: 'meshcore-app-v6', + version: 'v8', + tileCache: 'meshcore-tiles-v8', + appCache: 'meshcore-app-v8', + metadataCache: 'meshcore-meta-v8', + previousTileCache: 'meshcore-tiles-v7', + previousAppCache: 'meshcore-app-v7', maxTileEntries: 6_000, maxTileBytes: 96 * 1024 * 1024, pruneTargetRatio: 0.9, @@ -93,7 +93,7 @@ export function createTileMetadata(initial = []) { export function createServiceWorkerRuntime(environment) { const policy = { ...SW_CACHE_POLICY, ...(environment.policy ?? {}) }; const now = environment.now ?? Date.now; - const metadataUrl = new URL('/__meshcore_sw_metadata_v7__', environment.origin).toString(); + const metadataUrl = new URL('/__meshcore_sw_metadata_v8__', environment.origin).toString(); let metadata; let metadataState = { version: 1, diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 2308e8b..39312be 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -16,6 +16,7 @@ import { rfNodeCoverageState, useRfCoverage, type RfCoverageTierName } from './h import { useDashboardStats, type DashboardStats } from './hooks/useDashboardStats.js'; import { linkStateStore } from './hooks/useLinkState.js'; import { useAppMessageHandler } from './hooks/useAppMessageHandler.js'; +import { useMessageTags } from './hooks/useMessageTags.js'; import { HEATMAP_CAPABLE, PACKET_ARCS_CAPABLE, @@ -469,6 +470,8 @@ export const App: React.FC = () => { { network: networkFilter, observer: observerFilter }, scopeState.nodeEpoch, ); + // Live tag enrichment for the feed panel (tags land ~2s after a packet). + useMessageTags({ network: networkFilter, observer: observerFilter }, realtimeScopeKey); const wsState = wsConnection.readyState; wsConnectedRef.current = wsState === 'connected'; diff --git a/frontend/src/components/ConfirmedNeighbours.test.ts b/frontend/src/components/ConfirmedNeighbours.test.ts new file mode 100644 index 0000000..e841425 --- /dev/null +++ b/frontend/src/components/ConfirmedNeighbours.test.ts @@ -0,0 +1,77 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import React from 'react'; +import { renderToStaticMarkup } from 'react-dom/server'; +import { ConfirmedNeighbours, NeighbourContact, sortNeighbourLinks, timeAgo, type NodeLink } from './ConfirmedNeighbours.js'; + +const now = Date.parse('2026-09-07T12:00:00.000Z'); +const day = 86400_000; +const link: NodeLink = { + peer_id: 'A'.repeat(64), peer_name: 'Quiet repeater', observed_count: 107238, + itm_path_loss_db: 112.1, count_this_to_peer: 64146, count_peer_to_this: 43092, + last_observed: new Date(now - 2 * day).toISOString(), recent_count_72h: 0, +}; + +function contact(overrides: Partial = {}) { + return renderToStaticMarkup(React.createElement(NeighbourContact, { link: { ...link, ...overrides }, now })); +} + +test('last-contact dots follow the exact 24-hour and seven-day boundaries', () => { + for (const [age, colour, relative] of [ + [day - 1, 'online', '23h ago'], [day, 'amber', '1d ago'], + [7 * day, 'amber', '7d ago'], [7 * day + 1, 'danger', '7d ago'], + ] as const) { + const iso = new Date(now - age).toISOString(); + const html = contact({ last_observed: iso }); + assert.ok(html.includes(`background:var(--${colour})`)); + assert.ok(html.includes(`dateTime="${iso}"`)); + assert.ok(html.includes(relative)); + assert.match(html, /Last contact:/); + } +}); + +test('frequency depends only on recent contact, including zero with a high lifetime count', () => { + for (const [count, bucket] of [[0, 'RARE'], [1, 'OCCASIONAL'], [9, 'OCCASIONAL'], [10, 'ACTIVE'], [1001, 'ACTIVE']] as const) { + const html = contact({ recent_count_72h: count }); + assert.ok(html.includes(`>${bucket}`)); + assert.ok(html.includes(`${count} ${count === 1 ? 'contact' : 'contacts'} in last 72h`)); + } +}); + +test('unknown timestamps are neutral and clock skew never produces negative relative times', () => { + for (const last_observed of [null, '', 'invalid']) { + const html = contact({ last_observed }); + assert.match(html, /Last contact: Unknown/); + assert.ok(html.includes('background:var(--text-muted)')); + assert.doesNotMatch(html, / { + const input: NodeLink[] = [ + { ...link, peer_id: 'unknown', last_observed: null }, + { ...link, peer_id: 'frequent', recent_count_72h: 20 }, + { ...link, peer_id: 'newest', last_observed: new Date(now).toISOString(), recent_count_72h: 2 }, + ]; + const snapshot = structuredClone(input); + assert.deepEqual(sortNeighbourLinks(input, 'contact').map(x => x.peer_id), ['newest', 'frequent', 'unknown']); + assert.deepEqual(sortNeighbourLinks(input, 'frequency').map(x => x.peer_id), ['frequent', 'newest', 'unknown']); + assert.deepEqual(input, snapshot); + assert.deepEqual(sortNeighbourLinks([{ ...link, peer_id: 'b', last_observed: null }, { ...link, peer_id: 'a', last_observed: null }], 'contact').map(x => x.peer_id), ['a', 'b']); +}); + +test('card renders contact, lifetime context, sort labels, empty state and safe peer names', () => { + const html = renderToStaticMarkup(React.createElement(ConfirmedNeighbours, { links: [{ ...link, peer_name: '' }] })); + for (const text of ['107238× seen', '112 dB loss', 'TX: 64146', 'RX: 43092', 'RARE', 'Sort neighbours', 'Last contact', 'Most frequent', 'Contact includes relayed traffic.']) { + assert.ok(html.includes(text), text); + } + assert.ok(html.includes('<script>bad</script>')); + assert.ok(html.includes('flex-wrap:wrap')); + assert.ok(html.includes('min-width:0;overflow-wrap:anywhere')); + assert.match(renderToStaticMarkup(React.createElement(ConfirmedNeighbours, { links: [] })), /No neighbours found/); +}); diff --git a/frontend/src/components/ConfirmedNeighbours.tsx b/frontend/src/components/ConfirmedNeighbours.tsx new file mode 100644 index 0000000..aa6d70e --- /dev/null +++ b/frontend/src/components/ConfirmedNeighbours.tsx @@ -0,0 +1,123 @@ +import { useMemo, useState } from 'react'; + +export interface NodeLink { + peer_id: string; + peer_name: string | null; + observed_count: number; + itm_path_loss_db: number | null; + count_this_to_peer: number; + count_peer_to_this: number; + last_observed: string | null; + recent_count_72h: number; +} + +// Shared with the repeater page's existing last-seen and packet timestamps. +export function timeAgo(iso: string, now = Date.now()): string { + const timestamp = Date.parse(iso); + if (!Number.isFinite(timestamp)) return 'Unknown'; + const secs = Math.max(0, Math.floor((now - timestamp) / 1000)); + if (secs < 60) return `${secs}s ago`; + if (secs < 3600) return `${Math.floor(secs / 60)}m ago`; + if (secs < 86400) return `${Math.floor(secs / 3600)}h ago`; + return `${Math.floor(secs / 86400)}d ago`; +} + +function contactTime(link: NodeLink): number { + const timestamp = link.last_observed ? Date.parse(link.last_observed) : NaN; + return Number.isFinite(timestamp) ? timestamp : -Infinity; +} + +type NeighbourSort = 'contact' | 'frequency'; + +export function sortNeighbourLinks(links: NodeLink[], sort: NeighbourSort): NodeLink[] { + return [...links].sort((a, b) => { + if (sort === 'frequency' && a.recent_count_72h !== b.recent_count_72h) { + return b.recent_count_72h - a.recent_count_72h; + } + const aTime = contactTime(a); + const bTime = contactTime(b); + if (aTime !== bTime) return aTime > bTime ? -1 : 1; + return a.peer_id.localeCompare(b.peer_id); + }); +} + +export function NeighbourContact({ link, now = Date.now() }: { link: NodeLink; now?: number }) { + const timestamp = contactTime(link); + const known = Number.isFinite(timestamp); + const age = Math.max(0, now - timestamp); + const colour = !known ? 'var(--text-muted)' + : age < 86400_000 ? 'var(--online)' + : age <= 7 * 86400_000 ? 'var(--amber)' : 'var(--danger)'; + // Last contact is any-direction link-worker evidence, not "last direct". + // Recent counts are public src/rx packet rows, including relayed traffic. + // A healthy but quiet link is deliberately RARE even with many lifetime hits. + const frequency = link.recent_count_72h >= 10 ? 'ACTIVE' + : link.recent_count_72h >= 1 ? 'OCCASIONAL' : 'RARE'; + const frequencyColour = frequency === 'ACTIVE' ? 'var(--online)' + : frequency === 'OCCASIONAL' ? 'var(--amber)' : 'var(--text-muted)'; + + return ( +
+ + + + {frequency} + {link.recent_count_72h} {link.recent_count_72h === 1 ? 'contact' : 'contacts'} in last 72h + +
+ ); +} + +export function ConfirmedNeighbours({ links }: { links: NodeLink[] }) { + const [sort, setSort] = useState('contact'); + const sortedLinks = useMemo(() => sortNeighbourLinks(links, sort), [links, sort]); + + if (links.length === 0) { + return

No neighbours found for this node.

; + } + + return ( + <> +

+ Nodes with confirmed two-way communication. Contact includes relayed traffic. + {' '}Active: 10+ contacts in 72h · Occasional: 1–9 · Rare: 0. +

+ +
+ {sortedLinks.map(link => ( +
+
+ + {link.peer_name || `${link.peer_id.slice(0, 12)}...`} + + + {link.peer_id.slice(0, 16)}…{link.peer_id.slice(-8)} + +
+ +
+ {link.observed_count}× seen + {link.itm_path_loss_db !== null && · {Math.round(link.itm_path_loss_db)} dB loss} + · TX: {link.count_this_to_peer} + · RX: {link.count_peer_to_this} +
+
+ ))} +
+ + ); +} diff --git a/frontend/src/components/Map/RegionsMap.tsx b/frontend/src/components/Map/RegionsMap.tsx new file mode 100644 index 0000000..2e86d6a --- /dev/null +++ b/frontend/src/components/Map/RegionsMap.tsx @@ -0,0 +1,342 @@ +import { useCallback, useEffect, useState } from 'react'; +import * as maplibregl from 'maplibre-gl'; +import { MapLibreMap } from './MapLibreMap.js'; +import { DEFAULT_CENTER } from './mapConfig.js'; +import { + buildRegionAreasGeoJSON, + buildRegionRepeatersGeoJSON, + type RegionRepeaterFilters, +} from './regionOverlay.js'; +import type { RegionRepeater, RegionSummary } from '../../pages/regionsModel.js'; + +const REGION_AREAS_FILL_LAYER = 'region-areas-fill'; +const REGION_AREAS_LINE_LAYER = 'region-areas-line'; +const REGION_REPEATER_HIT_LAYER = 'region-repeater-hit'; +const REGION_REPEATER_HALO_LAYER = 'region-repeater-halo'; +const REGION_REPEATER_LAYER = 'region-repeater'; +const REGIONS_INITIAL_VIEW = { lat: DEFAULT_CENTER[0], lon: -2.3, zoom: 5.7 } as const; + +type RegionsMapProps = { + regions: readonly RegionSummary[]; + includeLowConfidence: boolean; + repeaters: readonly RegionRepeater[]; + shapesVisible: boolean; + repeatersVisible: boolean; + roleFilter: 'repeaters' | 'all'; + statusFilter: 'all' | 'green' | 'red'; + regionFilter: string | null; + memberIds: ReadonlySet | null; + selectedRegion: string | null; + selectedNodeId: string | null; + network?: string; + observer?: string; + privacyGeneration: number; + onRegionSelect: (region: string) => void; + onRepeaterSelect: (nodeId: string) => void; +}; + +type RegionsMapOverlayProps = Omit & { + map: maplibregl.Map; +}; + +function getFirstSymbolLayerId(map: maplibregl.Map): string | undefined { + for (const layer of map.getStyle().layers) { + if (layer.type === 'symbol') return layer.id; + } + return undefined; +} + +function addRegionsLayer(map: maplibregl.Map, layer: maplibregl.LayerSpecification): void { + if (map.getLayer(layer.id)) return; + const beforeId = getFirstSymbolLayerId(map); + if (beforeId) map.addLayer(layer, beforeId); + else map.addLayer(layer); +} + +const regionAreaBase = { + type: 'fill' as const, + source: 'region-areas', +}; + +const regionLineBase = { + type: 'line' as const, + source: 'region-areas', + layout: { + 'line-cap': 'round' as const, + 'line-join': 'round' as const, + }, +}; + +const regionRepeaterBase = { + type: 'circle' as const, + source: 'region-repeaters', +}; + +function RegionsMapOverlay({ + map, + regions, + repeaters, + shapesVisible, + repeatersVisible, + roleFilter, + statusFilter, + regionFilter, + memberIds, + selectedRegion, + selectedNodeId, + onRegionSelect, + onRepeaterSelect, +}: RegionsMapOverlayProps): null { + useEffect(() => { + map.addSource('region-areas', { + type: 'geojson', + data: { type: 'FeatureCollection', features: [] }, + }); + map.addSource('region-repeaters', { + type: 'geojson', + data: { type: 'FeatureCollection', features: [] }, + }); + + addRegionsLayer(map, { + ...regionAreaBase, + id: REGION_AREAS_FILL_LAYER, + paint: { + 'fill-color': ['get', 'color'], + 'fill-opacity': ['get', 'fill_opacity'], + }, + }); + addRegionsLayer(map, { + ...regionLineBase, + id: REGION_AREAS_LINE_LAYER, + paint: { + 'line-color': ['get', 'color'], + 'line-width': ['get', 'line_width'], + 'line-opacity': ['get', 'line_opacity'], + }, + }); + addRegionsLayer(map, { + ...regionRepeaterBase, + id: REGION_REPEATER_HIT_LAYER, + paint: { + 'circle-radius': ['interpolate', ['linear'], ['zoom'], 5, 9, 8, 11, 12, 15], + 'circle-color': '#ffffff', + 'circle-opacity': 0, + 'circle-stroke-opacity': 0, + }, + }); + addRegionsLayer(map, { + ...regionRepeaterBase, + id: REGION_REPEATER_HALO_LAYER, + paint: { + 'circle-radius': ['+', ['get', 'radius'], 5], + 'circle-color': [ + 'case', + ['get', 'selected'], '#f8fafc', + ['==', ['get', 'status'], 'green'], '#22c55e', + '#ef4444', + ], + 'circle-opacity': ['case', ['get', 'selected'], 0.3, 0.08], + 'circle-blur': 0.55, + }, + }); + addRegionsLayer(map, { + ...regionRepeaterBase, + id: REGION_REPEATER_LAYER, + paint: { + 'circle-radius': [ + 'interpolate', ['linear'], ['zoom'], + 5, ['case', ['==', ['get', 'status'], 'green'], 3.6, 3], + 8, ['case', ['==', ['get', 'status'], 'green'], 5.4, 4.2], + 12, ['case', ['==', ['get', 'status'], 'green'], 7.5, 6], + ], + 'circle-color': [ + 'case', + ['get', 'selected'], '#f8fafc', + ['==', ['get', 'status'], 'green'], '#22c55e', + '#ef4444', + ], + 'circle-opacity': [ + 'case', + ['get', 'selected'], 1, + ['==', ['get', 'tier'], 'confirmed'], 0.95, + ['==', ['get', 'tier'], 'probable'], 0.8, + ['==', ['get', 'tier'], 'single'], 0.55, + 0.72, + ], + 'circle-stroke-color': [ + 'case', + ['get', 'selected'], '#ffffff', + ['==', ['get', 'status'], 'green'], '#052e16', + '#450a0a', + ], + 'circle-stroke-width': ['case', ['get', 'selected'], 2.4, 1.1], + 'circle-stroke-opacity': 0.92, + }, + }); + + const handleRepeaterClick = (event: maplibregl.MapLayerMouseEvent) => { + const nodeId = event.features?.[0]?.properties?.['node_id']; + if (typeof nodeId === 'string' && nodeId.length > 0) onRepeaterSelect(nodeId); + }; + const handleAreaClick = (event: maplibregl.MapLayerMouseEvent) => { + // A repeater dot sitting inside a shape must win over the shape itself. + if (map.queryRenderedFeatures(event.point, { layers: [REGION_REPEATER_HIT_LAYER] }).length > 0) return; + const region = event.features?.[0]?.properties?.['region']; + if (typeof region === 'string' && region.length > 0) onRegionSelect(region); + }; + const handlePointerEnter = () => { map.getCanvas().style.cursor = 'pointer'; }; + const handlePointerLeave = () => { map.getCanvas().style.cursor = ''; }; + map.on('click', REGION_REPEATER_HIT_LAYER, handleRepeaterClick); + map.on('click', REGION_AREAS_FILL_LAYER, handleAreaClick); + map.on('mouseenter', REGION_REPEATER_HIT_LAYER, handlePointerEnter); + map.on('mouseleave', REGION_REPEATER_HIT_LAYER, handlePointerLeave); + map.on('mouseenter', REGION_AREAS_FILL_LAYER, handlePointerEnter); + map.on('mouseleave', REGION_AREAS_FILL_LAYER, handlePointerLeave); + + const navigation = new maplibregl.NavigationControl({ showCompass: false }); + map.addControl(navigation, 'top-right'); + + return () => { + map.off('click', REGION_REPEATER_HIT_LAYER, handleRepeaterClick); + map.off('click', REGION_AREAS_FILL_LAYER, handleAreaClick); + map.off('mouseenter', REGION_REPEATER_HIT_LAYER, handlePointerEnter); + map.off('mouseleave', REGION_REPEATER_HIT_LAYER, handlePointerLeave); + map.off('mouseenter', REGION_AREAS_FILL_LAYER, handlePointerEnter); + map.off('mouseleave', REGION_AREAS_FILL_LAYER, handlePointerLeave); + try { map.removeControl(navigation); } catch { /* map may already be removing */ } + try { + for (const layerId of [ + REGION_REPEATER_LAYER, + REGION_REPEATER_HALO_LAYER, + REGION_REPEATER_HIT_LAYER, + REGION_AREAS_LINE_LAYER, + REGION_AREAS_FILL_LAYER, + ]) { + if (map.getLayer(layerId)) map.removeLayer(layerId); + } + if (map.getSource('region-repeaters')) map.removeSource('region-repeaters'); + if (map.getSource('region-areas')) map.removeSource('region-areas'); + } catch { /* MapLibreMap owns final map teardown */ } + }; + }, [map, onRegionSelect, onRepeaterSelect]); + + useEffect(() => { + const areasSource = map.getSource('region-areas') as maplibregl.GeoJSONSource | undefined; + areasSource?.setData(buildRegionAreasGeoJSON(regions, selectedRegion)); + }, [map, regions, selectedRegion]); + + useEffect(() => { + const repeatersSource = map.getSource('region-repeaters') as maplibregl.GeoJSONSource | undefined; + if (!repeatersSource) return; + const filters: RegionRepeaterFilters = { + roleFilter, + statusFilter, + regionFilter, + memberIds, + selectedNodeId, + }; + const featureCollection = buildRegionRepeatersGeoJSON(repeaters, filters); + // Radius is a data-driven paint input, so carry a fixed per-feature radius. + repeatersSource.setData({ + ...featureCollection, + features: featureCollection.features.map((feature) => ({ + ...feature, + properties: { ...feature.properties, radius: feature.properties['status'] === 'green' ? 4.5 : 3.8 }, + })), + }); + }, [map, repeaters, roleFilter, statusFilter, regionFilter, memberIds, selectedNodeId]); + + useEffect(() => { + for (const [layerId, visible] of [ + [REGION_AREAS_FILL_LAYER, shapesVisible], + [REGION_AREAS_LINE_LAYER, shapesVisible], + [REGION_REPEATER_HIT_LAYER, repeatersVisible], + [REGION_REPEATER_HALO_LAYER, repeatersVisible], + [REGION_REPEATER_LAYER, repeatersVisible], + ] as const) { + if (map.getLayer(layerId)) { + map.setLayoutProperty(layerId, 'visibility', visible ? 'visible' : 'none'); + } + } + }, [map, shapesVisible, repeatersVisible]); + + return null; +} + +export function RegionsMap({ + regions, + includeLowConfidence, + repeaters, + shapesVisible, + repeatersVisible, + roleFilter, + statusFilter, + regionFilter, + memberIds, + selectedRegion, + selectedNodeId, + network, + observer, + privacyGeneration, + onRegionSelect, + onRepeaterSelect, +}: RegionsMapProps): React.ReactElement { + const [map, setMap] = useState(null); + const handleMapReady = useCallback((nextMap: maplibregl.Map) => setMap(nextMap), []); + const shapedRegions = regions.filter((region) => region.members.some((member) => member.lat != null && member.lon != null)).length; + const locatedRepeaters = repeaters.filter((repeater) => Number.isFinite(repeater.lat) && Number.isFinite(repeater.lon)).length; + + return ( +
+ + {map && ( + + )} +
+ Region scoping + {shapedRegions} region shapes · {locatedRepeaters} mapped repeaters + + {regionFilter + ? `Filtered to ${regionFilter}` + : includeLowConfidence + ? 'All observed regions' + : 'Established regions · low-confidence hidden'} + +
+
+ Region-capable repeater · solid = strong evidence + No scoped relay observed + Region footprint (convex hull of members relaying ≥10% of region traffic; bridge repeaters excluded) +
+
+ ); +} diff --git a/frontend/src/components/Map/regionOverlay.test.ts b/frontend/src/components/Map/regionOverlay.test.ts new file mode 100644 index 0000000..121a614 --- /dev/null +++ b/frontend/src/components/Map/regionOverlay.test.ts @@ -0,0 +1,109 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { buildRegionAreasGeoJSON, buildRegionRepeatersGeoJSON, type RegionRepeaterFilters } from './regionOverlay.js'; +import type { RegionRepeater, RegionSummary } from '../../pages/regionsModel.js'; + +function repeater(overrides: Partial = {}): RegionRepeater { + return { + nodeId: 'A'.repeat(64), + name: 'Repeater', + role: 2, + lat: 54, + lon: -1, + status: 'green', + evidence: 'scoped', + scopedPackets: 10, + unscopedPackets7d: 0, + regions: { yorkshire: 10 }, + ...overrides, + }; +} + +const baseFilters: RegionRepeaterFilters = { + roleFilter: 'all', + statusFilter: 'all', + regionFilter: null, + memberIds: null, + selectedNodeId: null, +}; + +test('buildRegionRepeatersGeoJSON filters by role, status, region and coordinates', () => { + const repeaters = [ + repeater(), + repeater({ nodeId: 'B'.repeat(64), role: 1, regions: { yorkshire: 5 } }), + repeater({ nodeId: 'C'.repeat(64), status: 'red', scopedPackets: 0, regions: {} }), + repeater({ nodeId: 'D'.repeat(64), lat: null, lon: null }), + ]; + const all = buildRegionRepeatersGeoJSON(repeaters, baseFilters); + assert.equal(all.features.length, 3); + assert.equal(all.features.some((feature) => feature.properties?.['node_id'] === 'D'.repeat(64)), false); + + const repeatersOnly = buildRegionRepeatersGeoJSON(repeaters, { ...baseFilters, roleFilter: 'repeaters' }); + assert.deepEqual(repeatersOnly.features.map((feature) => feature.properties?.['status']), ['green', 'red']); + + const greenOnly = buildRegionRepeatersGeoJSON(repeaters, { ...baseFilters, statusFilter: 'green' }); + assert.equal(greenOnly.features.length, 2); + + const byMembers = buildRegionRepeatersGeoJSON(repeaters, { + ...baseFilters, regionFilter: 'yorkshire', memberIds: new Set(['A'.repeat(64)]), + }); + assert.equal(byMembers.features.length, 1); + assert.equal(byMembers.features[0]?.properties?.['node_id'], 'A'.repeat(64)); + + const byRegionMap = buildRegionRepeatersGeoJSON(repeaters, { ...baseFilters, regionFilter: 'yorkshire' }); + assert.equal(byRegionMap.features.length, 2); +}); + +test('buildRegionRepeatersGeoJSON carries selection and tier properties', () => { + const collection = buildRegionRepeatersGeoJSON( + [repeater(), repeater({ nodeId: 'E'.repeat(64), scopedPackets: 1 })], + { ...baseFilters, selectedNodeId: 'A'.repeat(64) }, + ); + const selected = collection.features.find((feature) => feature.properties?.['node_id'] === 'A'.repeat(64)); + const single = collection.features.find((feature) => feature.properties?.['node_id'] === 'E'.repeat(64)); + assert.equal(selected?.properties?.['selected'], true); + assert.equal(selected?.properties?.['tier'], 'confirmed'); + assert.equal(single?.properties?.['tier'], 'single'); +}); + +test('buildRegionAreasGeoJSON skips regions without a ring and marks selection', () => { + const shapeable: RegionSummary = { + name: 'yorkshire', packetsTotal: 100, memberCount: 3, repeaterMemberCount: 3, + confidence: 'confirmed', firstSeen: null, lastSeen: null, + members: [ + { nodeId: 'A', name: 'A', role: 2, lat: 54, lon: -1, packets: 9 }, + { nodeId: 'B', name: 'B', role: 2, lat: 54.2, lon: -0.8, packets: 8 }, + { nodeId: 'C', name: 'C', role: 2, lat: 53.8, lon: -1.4, packets: 7 }, + ], + }; + const tooSmall: RegionSummary = { ...shapeable, name: 'micro', members: [shapeable.members[0]!] }; + const collection = buildRegionAreasGeoJSON([shapeable, tooSmall], 'yorkshire'); + assert.equal(collection.features.length, 1); + assert.equal(collection.features[0]?.properties?.['region'], 'yorkshire'); + assert.equal(collection.features[0]?.properties?.['selected'], true); + assert.match(String(collection.features[0]?.properties?.['color']), /^#[0-9a-f]{6}$/); + const ring = collection.features[0]?.geometry.coordinates[0]!; + assert.deepEqual(ring[0], ring[ring.length - 1]); +}); + +test('buildRegionAreasGeoJSON hides other regions when one is selected', () => { + const base: RegionSummary = { + name: 'yorkshire', packetsTotal: 100, memberCount: 3, repeaterMemberCount: 3, + confidence: 'confirmed', firstSeen: null, lastSeen: null, + members: [ + { nodeId: 'A', name: 'A', role: 2, lat: 54, lon: -1, packets: 9 }, + { nodeId: 'B', name: 'B', role: 2, lat: 54.2, lon: -0.8, packets: 8 }, + { nodeId: 'C', name: 'C', role: 2, lat: 53.8, lon: -1.4, packets: 7 }, + ], + }; + const northwest: RegionSummary = { + ...base, + name: 'northwest', + members: base.members.map((member) => ({ ...member, lat: member.lat! + 0.5, lon: member.lon! - 1 })), + }; + assert.equal(buildRegionAreasGeoJSON([base, northwest], null).features.length, 2); + const selectedOnly = buildRegionAreasGeoJSON([base, northwest], 'northwest'); + assert.equal(selectedOnly.features.length, 1); + assert.equal(selectedOnly.features[0]?.properties?.['region'], 'northwest'); + assert.equal(selectedOnly.features[0]?.properties?.['selected'], true); +}); diff --git a/frontend/src/components/Map/regionOverlay.ts b/frontend/src/components/Map/regionOverlay.ts new file mode 100644 index 0000000..fe32091 --- /dev/null +++ b/frontend/src/components/Map/regionOverlay.ts @@ -0,0 +1,99 @@ +import { + confidenceFor, + regionColor, + regionRing, + type RegionRepeater, + type RegionSummary, +} from '../../pages/regionsModel.js'; + +type RegionFeatureProperties = { + [key: string]: string | number | boolean | null; +}; + +export type RegionRepeaterFilters = { + roleFilter: 'repeaters' | 'all'; + statusFilter: 'all' | 'green' | 'red'; + regionFilter: string | null; + memberIds: ReadonlySet | null; + selectedNodeId: string | null; +}; + +function hasCoordinates(repeater: RegionRepeater): repeater is RegionRepeater & { lat: number; lon: number } { + const { lat, lon } = repeater; + return lat != null && lon != null + && Number.isFinite(lat) && Number.isFinite(lon) + && lat >= -90 && lat <= 90 + && lon >= -180 && lon <= 180 + && !(Math.abs(lat) < 1e-6 && Math.abs(lon) < 1e-6); +} + +export function buildRegionAreasGeoJSON( + regions: readonly RegionSummary[], + selectedRegion: string | null, +): GeoJSON.FeatureCollection { + // Selecting a region hides every other region's shape so the chosen + // footprint is unambiguous; clearing the selection restores all shapes. + const visibleRegions = selectedRegion + ? regions.filter((region) => region.name === selectedRegion) + : regions; + return { + type: 'FeatureCollection', + features: visibleRegions.flatMap((region) => { + const ring = regionRing(region); + if (!ring) return []; + const selected = region.name === selectedRegion; + return [{ + type: 'Feature' as const, + geometry: { type: 'Polygon' as const, coordinates: [ring] }, + properties: { + region: region.name, + color: regionColor(region.name), + selected, + fill_opacity: selected ? 0.3 : 0.13, + line_opacity: selected ? 1 : 0.55, + line_width: selected ? 2.4 : 1.3, + packets: region.packetsTotal, + members: region.memberCount, + }, + }]; + }), + }; +} + +export function buildRegionRepeatersGeoJSON( + repeaters: readonly RegionRepeater[], + filters: RegionRepeaterFilters, +): GeoJSON.FeatureCollection { + return { + type: 'FeatureCollection', + features: repeaters + .filter(hasCoordinates) + .filter((repeater) => { + if (filters.roleFilter === 'repeaters' && repeater.role !== 2) return false; + if (filters.statusFilter !== 'all' && repeater.status !== filters.statusFilter) return false; + if (filters.regionFilter) { + if (filters.memberIds) { + if (!filters.memberIds.has(repeater.nodeId)) return false; + } else if (!((repeater.regions[filters.regionFilter] ?? 0) > 0)) { + return false; + } + } + return true; + }) + .map((repeater) => ({ + type: 'Feature' as const, + geometry: { type: 'Point' as const, coordinates: [repeater.lon, repeater.lat] }, + properties: { + node_id: repeater.nodeId, + name: repeater.name, + label: repeater.name ?? repeater.nodeId.slice(0, 8), + status: repeater.status, + tier: repeater.status === 'green' ? confidenceFor(repeater.scopedPackets) : 'red', + scoped_packets: repeater.scopedPackets, + unscoped_packets_7d: repeater.unscopedPackets7d, + region_count: Object.keys(repeater.regions).length, + selected: repeater.nodeId === filters.selectedNodeId, + }, + })), + }; +} diff --git a/frontend/src/components/PacketFeed.tsx b/frontend/src/components/PacketFeed.tsx index e8d1ce6..288be05 100644 --- a/frontend/src/components/PacketFeed.tsx +++ b/frontend/src/components/PacketFeed.tsx @@ -1,6 +1,6 @@ import React, { useEffect, useRef, useState } from 'react'; import { useMessages, useNodeMap } from '../hooks/useNodes.js'; -import { aggregatedPacketObserverIataLabel } from '../hooks/packetFeed.js'; +import { aggregatedPacketObserverIataLabel, messageTagKind, messageTagTitle } from '../hooks/packetFeed.js'; import { useOverlayStore } from '../store/overlayStore.js'; import { useWatchlist } from '../hooks/useWatchlist.js'; import type { AggregatedPacket } from '../hooks/useNodes.js'; @@ -91,6 +91,11 @@ const PacketFeedItem: React.FC = React.memo(({ } + {messageTagKind(p) && ( + + {messageTagKind(p)} + + )}
+ ) : null}
{node.name ?? 'Unnamed node'} {node.members.length} member key{node.members.length === 1 ? '' : 's'} @@ -334,6 +616,7 @@ export const OwnerPortalPage: React.FC = () => { ))}
+
{live?.ownerNode.name ?? 'Unnamed'}{nodeRoleLabel(live?.ownerNode.role ?? null)}
{live?.ownerNode.network ?? '-'}Network
@@ -348,7 +631,7 @@ export const OwnerPortalPage: React.FC = () => {
{live?.packetsSent24h ?? 0}Packets Sent (24h)
{live?.packetsReceived24h ?? 0}Packets Received (24h)
- {liveError ?

Live data error: {liveError}

: null} +
@@ -359,6 +642,7 @@ export const OwnerPortalPage: React.FC = () => {
+ {TELEMETRY_SERIES.map((series) => ( { value={formatUptime(latestTelemetry?.uptimeSecs ?? null)} meta={latestTelemetry?.uptimeSecs == null ? 'No telemetry yet' : `${latestTelemetry.uptimeSecs}s reported`} /> - 0)} /> + + + 0)} compareSeries={compareChartSeries} /> +
@@ -385,9 +672,11 @@ export const OwnerPortalPage: React.FC = () => {

Node status

Latest nullable diagnostics reported by the node.

- {live?.status?.sampled_at ? fmtTs(live.status.sampled_at) : 'No sample'} + {live ? (live.status?.sampled_at ? fmtTs(live.status.sampled_at) : 'No sample') : '—'} - + + +
@@ -395,14 +684,18 @@ export const OwnerPortalPage: React.FC = () => {

Direct Sender Map

+
+

Alerts

+
+ {live?.alerts.length === 0 ?

No active alerts.

: null} {(live?.alerts ?? []).map((alert, idx) => (
{alert.level.toUpperCase()} @@ -410,15 +703,21 @@ export const OwnerPortalPage: React.FC = () => {
))}
+

Advert Trend

- + + {(live?.advertTrend24h ?? []).some((point) => point.adverts > 0) + ? + :

No adverts observed in the last 24 hours.

} +

RF Link Health

+
{(live?.linkHealth ?? []).slice(0, 8).map((link) => (
@@ -438,10 +737,12 @@ export const OwnerPortalPage: React.FC = () => {

No link health data has been calculated for this node yet.

) : null}
+

Direct Senders

+
{(live?.incomingPeers ?? []).slice(0, 8).map((peer) => (
@@ -459,6 +760,7 @@ export const OwnerPortalPage: React.FC = () => {

No direct sender nodes found in the last 24 hours.

) : null}
+
@@ -468,6 +770,7 @@ export const OwnerPortalPage: React.FC = () => {

Other nodes that received packets transmitted by this repeater (last 7d)

+
{(live?.heardBy ?? []).slice(0, 8).map((peer) => (
@@ -487,20 +790,24 @@ export const OwnerPortalPage: React.FC = () => {

No nodes have received packets from this node in the last 7 days.

) : null}
+

Heard neighbors

-

The latest neighbor sample, sorted by last-seen recency (up to 32 nodes).

+

Reported neighbors or directly received senders, sorted by last-seen recency (up to 32 nodes).

- + + +

Live Packets Received By {nodeRoleLabel(live?.ownerNode.role ?? null)}

+
{(live?.recentPackets ?? []).map((packet, idx) => (
@@ -518,11 +825,30 @@ export const OwnerPortalPage: React.FC = () => {
))} {(live?.recentPackets ?? []).length === 0 ? ( -

No packets received by this node yet.

+

No packets received by this node in the last 30 days.

) : null}
+
+ + {isPacketSharingOwner(ownerSessionKey) ? ( + + ) : null} + ) : null} diff --git a/frontend/src/pages/RegionsPage.tsx b/frontend/src/pages/RegionsPage.tsx new file mode 100644 index 0000000..7514d58 --- /dev/null +++ b/frontend/src/pages/RegionsPage.tsx @@ -0,0 +1,380 @@ +import React, { useCallback, useEffect, useMemo, useState } from 'react'; +import { LoadingIndicator } from '../components/LoadingIndicator.js'; +import { RegionsMap } from '../components/Map/RegionsMap.js'; +import { getCurrentSite } from '../config/site.js'; +import type { SiteConfig } from '../config/site.js'; +import { useRuntimeFeatures } from '../config/runtimeFeatures.js'; +import { fetchJson, withScopeParams } from '../utils/api.js'; +import { + compactNumber, + confidenceFor, + formatGeneratedAt, + isEstablished, + memberIdSet, + regionColor, + regionsForDisplay, + type RegionRepeater, + type RegionRepeatersPayload, + type RegionSummary, + type RegionSummaryPayload, +} from './regionsModel.js'; +import './regions-page.css'; + +const CONFIDENCE_LABELS: Record = { + confirmed: 'Strong evidence — 4 or more scoped packets', + probable: 'Moderate evidence — 2-3 scoped packets', + single: 'Single observation — could be chance', +}; + +export const RegionsPage: React.FC = () => { + const site = getCurrentSite(); + const { privacyGeneration } = useRuntimeFeatures(); + const network = site.networkFilter ?? site.network; + const observer = site.observerId; + const [summary, setSummary] = useState(null); + const [repeatersPayload, setRepeatersPayload] = useState(null); + const [error, setError] = useState(null); + const [selectedRegion, setSelectedRegion] = useState(null); + const [selectedNodeId, setSelectedNodeId] = useState(null); + const [shapesVisible, setShapesVisible] = useState(true); + const [repeatersVisible, setRepeatersVisible] = useState(true); + const [roleFilter, setRoleFilter] = useState<'repeaters' | 'all'>('repeaters'); + const [statusFilter, setStatusFilter] = useState<'all' | 'green' | 'red'>('all'); + const [includeLowConfidence, setIncludeLowConfidence] = useState(false); + + useEffect(() => { + const controller = new AbortController(); + setSummary(null); + setRepeatersPayload(null); + setError(null); + setSelectedRegion(null); + setSelectedNodeId(null); + Promise.all([ + fetchJson( + withScopeParams('/api/regions/summary', { network, observer }), + { signal: controller.signal, cache: 'no-store' }, + { timeoutMs: 15_000, maxBytes: 8 * 1024 * 1024 }, + ), + fetchJson( + withScopeParams('/api/regions/repeaters', { network, observer }), + { signal: controller.signal, cache: 'no-store' }, + { timeoutMs: 15_000, maxBytes: 8 * 1024 * 1024 }, + ), + ]) + .then(([nextSummary, nextRepeaters]) => { + if (controller.signal.aborted) return; + setSummary(nextSummary); + setRepeatersPayload(nextRepeaters); + }) + .catch((reason: unknown) => { + if ((reason as DOMException).name !== 'AbortError') setError((reason as Error).message); + }); + return () => controller.abort(); + }, [network, observer, privacyGeneration]); + + const regions = useMemo(() => summary?.regions ?? [], [summary]); + const displayRegions = useMemo( + () => regionsForDisplay(regions, includeLowConfidence), + [regions, includeLowConfidence], + ); + const hiddenRegionCount = regions.length - displayRegions.length; + const repeaters = useMemo(() => repeatersPayload?.repeaters ?? [], [repeatersPayload]); + const stats = useMemo(() => { + let scopedPackets = 0; + let memberships = 0; + for (const region of regions) { + scopedPackets += region.packetsTotal; + memberships += region.memberCount; + } + let green = 0; + let red = 0; + for (const repeater of repeaters) { + if (repeater.status === 'green') green += 1; + else red += 1; + } + return { regions: regions.length, scopedPackets, memberships, green, red }; + }, [regions, repeaters]); + const selectedRegionData = useMemo( + () => regions.find((region) => region.name === selectedRegion) ?? null, + [regions, selectedRegion], + ); + const selectedRepeater = useMemo( + () => repeaters.find((repeater) => repeater.nodeId === selectedNodeId) ?? null, + [repeaters, selectedNodeId], + ); + const memberIds = useMemo( + () => (selectedRegionData ? memberIdSet(selectedRegionData) : null), + [selectedRegionData], + ); + const handleRegionSelect = useCallback((region: string) => { + setSelectedRegion(region); + setSelectedNodeId(null); + }, []); + const handleRepeaterSelect = useCallback((nodeId: string) => { + setSelectedNodeId(nodeId); + }, []); + const handleLowConfidenceToggle = useCallback((checked: boolean) => { + setIncludeLowConfidence(checked); + if (!checked) { + // A region that just became hidden must not stay selected. + setSelectedRegion((current) => { + const target = current ? regions.find((region) => region.name === current) : undefined; + return target && !isEstablished(target) ? null : current; + }); + } + }, [regions]); + const generatedLabel = formatGeneratedAt(summary?.generatedAt ?? null); + + return ( +
+
+
+

Network intelligence

+

MeshCore regions

+

+ MeshCore scoped flood traffic carries a per-region transport code. This map shows the + regions active on the UK mesh, the repeaters observed relaying scoped traffic (green), + and repeaters with no scoped relay observed (red). Attribution is a cryptographic + candidate-name test against the region transport code — private regions stay + unattributed by design. Region names are community-defined keys, so shapes show where + scoped traffic is relayed (bridge repeaters excluded) rather than official boundaries. +

+
+
+ + + + + + +
+
+ + {error &&
{error}
} + {!summary && !error && } + + {summary && ( + <> +
+
{stats.regions}observed regions
+
{compactNumber(stats.scopedPackets)}scoped packets
+
{compactNumber(stats.green)}region-capable nodes
+
{compactNumber(stats.red)}no scoped relay observed
+
{compactNumber(stats.memberships)}relay memberships
+
+ +
+ + + +
+ + )} +
+ ); +}; + +export function RepeaterPanel({ repeater, site, onClear }: { + repeater: RegionRepeater; + site: SiteConfig; + onClear: () => void; +}): React.ReactElement { + const regionEntries = Object.entries(repeater.regions).sort((a, b) => b[1] - a[1]).slice(0, 12); + const tier = repeater.status === 'green' ? confidenceFor(repeater.scopedPackets) : null; + const regionCount = Object.keys(repeater.regions).length; + return ( +
+

Selected repeater

+ {repeater.name ?? repeater.nodeId.slice(0, 12)} + + {repeater.status === 'green' ? 'Region-capable' : 'No scoped relay observed'} + + {repeater.status === 'green' ? ( + regionCount > 0 ? ( + + Relayed {compactNumber(repeater.scopedPackets)} cryptographically scoped packets + across {regionCount} region{regionCount === 1 ? '' : 's'}. + + ) : ( + + Relayed {compactNumber(repeater.scopedPackets)} cryptographically scoped packets, but no + single region exceeded the 10% membership floor. + + ) + ) : ( + + Unscoped relay evidence: {compactNumber(repeater.unscopedPackets7d)} packets in the last + 7 days. No region-scoped traffic was observed relaying through it. + + )} + {tier && {CONFIDENCE_LABELS[tier]}} + {regionEntries.length > 0 && ( +
+ {regionEntries.map(([name, packets]) => ( + {name} {compactNumber(packets)} + ))} +
+ )} + {repeater.lat != null && repeater.lon != null && ( + + Zoom to repeater on live map + + )} + +
+ ); +} + +export function RegionPanel({ region, onMemberSelect, onClear }: { + region: RegionSummary; + onMemberSelect: (nodeId: string) => void; + onClear: () => void; +}): React.ReactElement { + const topMembers = [...region.members].sort((a, b) => b.packets - a.packets).slice(0, 10); + return ( +
+

Selected region

+ {region.name} + {CONFIDENCE_LABELS[region.confidence]} + + {compactNumber(region.packetsTotal)} scoped packets · {region.memberCount} significant relay + members ({region.repeaterMemberCount} repeaters) + + {topMembers.length > 0 ? ( + <> + Top relay members +
    + {topMembers.map((member) => ( +
  1. + +
    2. + ))} +
+ + ) : ( + No significant relay members — no repeater relayed at least 10% of this region's traffic, or its relay hops could not be attributed. + )} +

+ Members are repeaters relaying at least 10% of this region's scoped packets; shapes are the + convex hull of their positions, excluding universal bridge repeaters that relay many regions — + an approximation of the region's active footprint. +

+ +
+ ); +} diff --git a/frontend/src/pages/TopologyPage.browser.test.ts b/frontend/src/pages/TopologyPage.browser.test.ts new file mode 100644 index 0000000..fd4a75f --- /dev/null +++ b/frontend/src/pages/TopologyPage.browser.test.ts @@ -0,0 +1,71 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { chromium, expect } from '@playwright/test'; + +const browserTest = process.env['AUDITWAVE_BROWSER_TESTS'] === '1' ? test : test.skip; +const topology = { + generatedAt: '2026-09-07T12:00:00Z', windowDays: 30, limited: false, + summary: { nodes: 2, links: 1, observations: 12, connectedComponents: 1, likelyBridges: 0, isolatedNodes: 0 }, + nodes: [ + { nodeId: 'A'.repeat(64), name: 'Alpha', lat: 52, lon: -1, degree: 1, observations: 12 }, + { nodeId: 'B'.repeat(64), name: 'Bravo', lat: 52.1, lon: -1.1, degree: 1, observations: 12 }, + ], + links: [{ source: 'A'.repeat(64), target: 'B'.repeat(64), observations: 12, strongObservations: 3, pathLossDb: 130, lastObserved: '2026-09-07T12:00:00Z' }], + analysis: { connectedComponents: 1, bridgeNodeIds: [], isolatedNodeIds: [] }, +}; +const rf = { + methodology: 'Observed relay evidence compared with stored ITM.', + summary: { evaluated: 1, matches: 0, mismatches: 1, observedUnexpected: 1, operatorOverrides: 0, weakModelEvidence: 0 }, + mismatches: [{ source: 'A', target: 'B', sourceName: 'Alpha', targetName: 'Bravo', + observations: 12, strongObservations: 3, pathLossDb: 130, classification: 'observed_unexpected' }], +}; + +for (const failure of ['http503', 'timeout'] as const) { + browserTest(`deployed topology retains graph and RF panel after ${failure}, then retry restores results`, { timeout: 35_000 }, async () => { + const browser = await chromium.launch({ headless: true, args: ['--use-angle=swiftshader', '--enable-unsafe-swiftshader'] }); + try { + const context = await browser.newContext({ serviceWorkers: 'block' }); + const page = await context.newPage(); + const errors: string[] = []; + page.on('pageerror', error => errors.push(error.message)); + let attempts = 0; + await context.route('**/*', async route => { + const url = new URL(route.request().url()); + // Only load assets from the authorized local website. All API data, + // map tiles and telemetry are intercepted; this sends no external calls. + if (url.hostname !== '127.0.0.1') { await route.abort(); return; } + if (url.pathname === '/api/topology') { await route.fulfill({ json: topology }); return; } + if (url.pathname === '/api/runtime-config') { + await route.fulfill({ json: { version: 1, packetArcs: false, heatmap: false, privacyGeneration: 0, refreshAfterSeconds: 300 } }); + return; + } + if (url.pathname === '/api/rf-validation') { + attempts += 1; + if (attempts === 1) { + if (failure === 'http503') await route.fulfill({ status: 503, json: { error: 'temporarily unavailable' } }); + // Leave the first request pending to exercise the real browser's + // AbortSignal timeout, not an artificial immediate network error. + return; + } + await route.fulfill({ json: rf }); + return; + } + if (url.pathname.startsWith('/api/')) { await route.fulfill({ status: 200, json: {} }); return; } + await route.continue(); + }); + await page.goto('http://127.0.0.1:3004/topology', { waitUntil: 'domcontentloaded' }); + await expect(page.getByRole('heading', { name: 'Predicted versus observed RF' })).toBeVisible(); + await expect(page.locator('.topology-page__workspace')).toBeVisible(); + await expect(page.locator('.topology-page__workspace canvas')).toBeVisible(); + if (failure === 'timeout') await expect(page.getByText('Loading RF validation…')).toBeVisible(); + await expect(page.getByRole('alert').filter({ hasText: 'RF validation is temporarily unavailable' })).toBeVisible({ timeout: 18_000 }); + await expect(page.locator('.topology-page__workspace')).toBeVisible(); + await page.getByRole('button', { name: 'Retry RF validation' }).click(); + await expect(page.getByRole('table', { name: 'RF model mismatches' })).toBeVisible(); + await expect(page.getByRole('cell', { name: '130.0 dB', exact: true })).toBeVisible(); + assert.equal(attempts, 2); + assert.deepEqual(errors, []); + await context.close(); + } finally { await browser.close(); } + }); +} diff --git a/frontend/src/pages/TopologyPage.test.ts b/frontend/src/pages/TopologyPage.test.ts new file mode 100644 index 0000000..37dfc79 --- /dev/null +++ b/frontend/src/pages/TopologyPage.test.ts @@ -0,0 +1,52 @@ +import assert from 'node:assert/strict'; +import { registerHooks } from 'node:module'; +import test from 'node:test'; +import React from 'react'; +import { renderToStaticMarkup } from 'react-dom/server'; + +// Render the real RF panel without loading the browser's map or Vite env. +// Request, retry and cleanup behavior is exercised in the browser test. +const hooks = registerHooks({ + resolve(specifier, context, nextResolve) { + if (specifier.endsWith('.css')) return { url: 'data:text/javascript,export default {}', shortCircuit: true }; + if (specifier === '../components/Map/TopologyMap.js') { + return { url: 'data:text/javascript,export const TopologyMap=()=>null', shortCircuit: true }; + } + if (specifier === '../config/site.js') { + return { url: 'data:text/javascript,export const getCurrentSite=()=>({network:"ukmesh"})', shortCircuit: true }; + } + return nextResolve(specifier, context); + }, +}); +const { RfValidationPanel } = await import('./TopologyPage.js'); +hooks.deregister(); +const render = (data: Parameters[0]['data'], unavailable = false) => ( + renderToStaticMarkup(React.createElement(RfValidationPanel, { data, unavailable, onRetry: () => {} })) +); + +test('RF panel remains visible while loading and on failure with an accessible retry action', () => { + const loading = render(null); + assert.match(loading, /Predicted versus observed RF/); + assert.match(loading, /Loading RF validation/); + const failed = render(null, true); + assert.match(failed, /Predicted versus observed RF/); + assert.match(failed, /role="alert"/); + assert.match(failed, /RF validation is temporarily unavailable/); + assert.match(failed, / + + ) : !data ? ( + + ) : ( + <> +
+
{data.summary.evaluated}links evaluated
+
{data.summary.observedUnexpected}unexpected observed links
+
{data.summary.operatorOverrides}operator overrides
+
{data.summary.weakModelEvidence}weak evidence
+
+ {data.mismatches.length > 0 && ( +
+
+ Link + Observations + Modelled path loss + Classification +
+ {data.mismatches.slice(0, 20).map((link) => ( +
+ {link.sourceName ?? link.source.slice(0, 8)} ↔ {link.targetName ?? link.target.slice(0, 8)} + {link.observations} obs · {link.strongObservations} strong + {link.pathLossDb == null ? 'No model dB' : `${link.pathLossDb.toFixed(1)} dB`} + {link.classification.replace(/_/g, ' ')} +
+ ))} +
+ )} + {data.summary.evaluated === 0 &&

No recent public repeater links are available for RF validation.

} + + )} + + ); +} diff --git a/frontend/src/pages/docs-pages.css b/frontend/src/pages/docs-pages.css index 8a3564f..9e99791 100644 --- a/frontend/src/pages/docs-pages.css +++ b/frontend/src/pages/docs-pages.css @@ -512,10 +512,6 @@ line-height: 1.45; } -.cookie-banner__body a { - color: var(--accent); -} - .cookie-banner__button { flex-shrink: 0; min-width: 88px; diff --git a/frontend/src/pages/owner-portal.css b/frontend/src/pages/owner-portal.css index fff350c..27351f0 100644 --- a/frontend/src/pages/owner-portal.css +++ b/frontend/src/pages/owner-portal.css @@ -31,6 +31,269 @@ color: #ff8f8f; margin-top: 10px; } +.owner-sharing-panel { + margin-top: 16px; +} +.owner-sharing-panel__intro { + min-width: 0; +} +.owner-sharing-panel__eyebrow { + margin: 0 0 5px; + color: var(--accent); + font-family: var(--font-mono); + font-size: 10px; + font-weight: 700; + letter-spacing: 0.12em; + text-transform: uppercase; +} +.owner-sharing-switch { + position: relative; + display: inline-flex; + align-items: center; + flex: 0 0 auto; + width: 52px; + height: 32px; + cursor: pointer; +} +.owner-sharing-switch input { + position: absolute; + width: 1px; + height: 1px; + opacity: 0; +} +.owner-sharing-switch__track { + position: relative; + display: block; + width: 52px; + height: 32px; + border: 1px solid var(--border); + border-radius: 999px; + background: rgba(107, 138, 170, 0.32); + box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.24); + transition: background 160ms ease, border-color 160ms ease, box-shadow 160ms ease; +} +.owner-sharing-switch__thumb { + position: absolute; + top: 3px; + left: 3px; + width: 24px; + height: 24px; + border-radius: 50%; + background: #fff; + box-shadow: 0 2px 5px rgba(0, 0, 0, 0.32); + transition: transform 160ms ease; +} +.owner-sharing-switch input:checked + .owner-sharing-switch__track { + border-color: var(--online); + background: var(--online); + box-shadow: 0 0 0 3px color-mix(in srgb, var(--online) 16%, transparent), inset 0 1px 2px rgba(0, 0, 0, 0.12); +} +.owner-sharing-switch input:checked + .owner-sharing-switch__track .owner-sharing-switch__thumb { + transform: translateX(20px); +} +.owner-sharing-switch input:focus-visible + .owner-sharing-switch__track { + outline: 2px solid var(--accent); + outline-offset: 3px; +} +.owner-sharing-switch input:disabled + .owner-sharing-switch__track { + cursor: not-allowed; + opacity: 0.48; +} +.owner-sharing-state { + display: flex; + align-items: flex-start; + gap: 10px; + margin: 2px 0 16px; + padding: 12px 14px; + border: 1px solid var(--border); + border-radius: var(--radius); + background: rgba(107, 138, 170, 0.08); +} +.owner-sharing-state--on { + border-color: color-mix(in srgb, var(--online) 48%, var(--border)); + background: color-mix(in srgb, var(--online) 8%, transparent); +} +.owner-sharing-state__dot { + flex: 0 0 auto; + width: 8px; + height: 8px; + margin-top: 5px; + border-radius: 50%; + background: var(--text-muted); + box-shadow: 0 0 0 3px rgba(107, 138, 170, 0.12); +} +.owner-sharing-state--on .owner-sharing-state__dot { + background: var(--online); + box-shadow: 0 0 0 3px color-mix(in srgb, var(--online) 16%, transparent); +} +.owner-sharing-state strong { + display: block; + color: var(--text-primary); + font-size: 13px; +} +.owner-sharing-state p { + margin: 3px 0 0; + color: var(--text-secondary); + font-size: 12px; + line-height: 1.4; +} +.owner-sharing-empty { + margin: 0 0 12px; +} +.owner-sharing-destinations { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(220px, 1fr)); + gap: 10px; +} +.owner-sharing-destination { + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; + padding: 12px; + border: 1px solid var(--border); + border-radius: var(--radius); + background: var(--bg-panel-alt); + color: var(--text-primary); +} +.owner-sharing-destination--pending { + opacity: 0.72; +} +.owner-sharing-destination__body { + display: grid; + gap: 3px; + min-width: 0; +} +.owner-sharing-destination__link { + overflow: hidden; + color: var(--accent); + font-weight: 600; + text-overflow: ellipsis; + text-decoration: none; + white-space: nowrap; +} +.owner-sharing-destination__link:hover, +.owner-sharing-destination__link:focus-visible { + text-decoration: underline; +} +.owner-sharing-destination small { + color: var(--text-secondary); + font-size: 11px; +} +.owner-sharing-destination:focus-within { + border-color: var(--border-bright); +} +.owner-sharing-saving { + margin-top: 14px; +} +.owner-sharing-error { + color: #ff8f8f; + margin-top: 10px; +} +.owner-public-feed-panel { + margin-top: 16px; +} +.owner-public-feed-switch-control { + display: flex; + align-items: center; + gap: 9px; + flex: 0 0 auto; +} +.owner-public-feed-switch__label { + color: var(--text-muted); + font-family: var(--font-mono); + font-size: 10px; + font-weight: 700; + letter-spacing: 0.1em; + text-transform: uppercase; +} +.owner-public-feed-switch { + margin-top: 0; +} +.owner-public-feed-warning { + display: grid; + gap: 4px; + margin: 0 0 14px; + padding: 11px 14px; + border: 1px solid color-mix(in srgb, var(--amber) 32%, var(--border)); + border-left: 3px solid var(--amber); + border-radius: var(--radius); + background: color-mix(in srgb, var(--amber) 6%, var(--bg-panel)); +} +.owner-public-feed-warning__title { + display: flex; + align-items: center; + gap: 8px; + color: var(--text-primary); + font-size: 12px; +} +.owner-public-feed-warning__title > span { + display: inline-grid; + place-items: center; + width: 16px; + height: 16px; + border: 1px solid color-mix(in srgb, var(--amber) 65%, transparent); + border-radius: 50%; + color: var(--amber); + font-family: var(--font-mono); + font-size: 11px; + font-weight: 700; +} +.owner-public-feed-warning strong { + color: var(--text-primary); +} +.owner-public-feed-warning p { + margin: 0; + color: var(--text-secondary); + font-size: 12px; + line-height: 1.45; +} +.owner-public-feed-details { + display: grid; + grid-template-columns: repeat(4, minmax(0, 1fr)); + gap: 10px; + padding-top: 13px; + border-top: 1px solid var(--border); + color: var(--text-secondary); + font-size: 11px; +} +.owner-public-feed-details__item { + display: grid; + gap: 3px; + min-width: 0; +} +.owner-public-feed-details__label { + color: var(--text-muted); + font-family: var(--font-mono); + font-size: 10px; + letter-spacing: 0.08em; + text-transform: uppercase; +} +.owner-public-feed-details code { + color: var(--accent); + overflow-wrap: anywhere; +} +/* Keep the iOS switch usable when the owner dashboard narrows to a phone. */ +@media (max-width: 480px) { + .owner-sharing-panel .owner-panel__head { + align-items: center; + } + .owner-sharing-panel__intro .prose-note { + max-width: 27ch; + } + .owner-sharing-destination { + align-items: flex-start; + } + .owner-public-feed-panel .owner-panel__head { + align-items: center; + } + .owner-public-feed-panel .owner-sharing-panel__intro .prose-note { + max-width: 27ch; + } + .owner-public-feed-details { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } +} .owner-head { display: flex; align-items: center; @@ -377,6 +640,31 @@ font-size: 10px; font-family: var(--font-mono); } +.owner-telemetry-metric__window-toggle { + display: inline-flex; + align-items: center; + gap: 4px; +} +.owner-telemetry-metric__window-btn { + border: 1px solid var(--border); + background: rgba(255,255,255,0.03); + color: var(--text-secondary); + border-radius: 999px; + padding: 1px 7px; + font-size: 10px; + line-height: 1.5; + font-family: var(--font-mono); + cursor: pointer; +} +.owner-telemetry-metric__window-btn:hover { + border-color: var(--border-bright); + color: var(--text-primary); +} +.owner-telemetry-metric__window-btn--active { + border-color: rgba(0,196,255,0.45); + background: rgba(0,196,255,0.12); + color: var(--text-primary); +} .owner-telemetry-metric__series-list { align-items: flex-start; min-width: 0; @@ -460,6 +748,7 @@ margin: 16px 0; } .owner-node-identity { + position: relative; padding: 12px; border: 1px solid var(--border); border-radius: var(--radius); @@ -481,6 +770,40 @@ .owner-node-identity--selected:hover { border-color: rgba(0, 196, 255, 0.8); } +.owner-node-identity__compare { + position: absolute; + top: 10px; + right: 10px; + width: 22px; + height: 22px; + display: inline-flex; + align-items: center; + justify-content: center; + padding: 0; + border: 1px solid var(--border); + border-radius: 999px; + background: var(--bg-panel); + color: var(--text-secondary); + font-size: 13px; + line-height: 1; + cursor: pointer; + transition: border-color 120ms ease, color 120ms ease, background-color 120ms ease; +} +.owner-node-identity__compare:hover { + border-color: var(--accent); + color: var(--text-primary); +} +.owner-node-identity__compare--active { + border-color: rgba(0, 196, 255, 0.8); + background: rgba(0, 196, 255, 0.16); + color: #7dd8ff; +} +.owner-node-identity--comparing { + border-color: rgba(0, 196, 255, 0.35); +} +.owner-node-identity:has(.owner-node-identity__compare) .owner-node-identity__head { + padding-right: 28px; +} .owner-node-identity__head { display: flex; justify-content: space-between; diff --git a/frontend/src/pages/owner/OwnerHeardNeighbors.tsx b/frontend/src/pages/owner/OwnerHeardNeighbors.tsx index faeb32e..5cb0cc1 100644 --- a/frontend/src/pages/owner/OwnerHeardNeighbors.tsx +++ b/frontend/src/pages/owner/OwnerHeardNeighbors.tsx @@ -6,7 +6,11 @@ function signalValue(value: number | null, suffix: string): string { return `${value.toFixed(1)} ${suffix}`; } -export const OwnerHeardNeighbors: React.FC<{ neighbors: HeardNeighbor[] }> = ({ neighbors }) => { +export const OwnerHeardNeighbors: React.FC<{ + neighbors: HeardNeighbor[]; + source?: 'reported' | 'direct_packets' | 'none'; + sampledAt?: string | null; +}> = ({ neighbors, source, sampledAt: reportedSampleTime }) => { const rows = useMemo( () => [...neighbors] .sort((a, b) => { @@ -18,19 +22,24 @@ export const OwnerHeardNeighbors: React.FC<{ neighbors: HeardNeighbor[] }> = ({ .slice(0, 32), [neighbors], ); - const sampledAt = rows.find((neighbor) => neighbor.sampled_at)?.sampled_at ?? null; + const sampledAt = reportedSampleTime ?? rows.find((neighbor) => neighbor.sampled_at)?.sampled_at ?? null; + const directPackets = source === 'direct_packets'; if (rows.length === 0) { - return

No neighbor sample has been received for this node yet.

; + return

{source === 'reported' + ? `The latest reported neighbor list is empty.${sampledAt ? ` Sampled ${new Date(sampledAt).toLocaleString()}.` : ''}` + : 'No neighbor list was reported in the last 7 days and no direct senders were observed in the last 24 hours.'}

; } return (
-
Latest sample: {sampledAt ? new Date(sampledAt).toLocaleString() : '—'}
+
{directPackets + ? 'No neighbor list reported in the last 7 days. Showing direct senders from received packets in the last 24 hours, with average signal strength.' + : `Latest reported sample: ${sampledAt ? new Date(sampledAt).toLocaleString() : '—'}`}
- + {rows.map((neighbor, index) => ( diff --git a/frontend/src/pages/owner/OwnerPanelState.test.ts b/frontend/src/pages/owner/OwnerPanelState.test.ts new file mode 100644 index 0000000..19c397b --- /dev/null +++ b/frontend/src/pages/owner/OwnerPanelState.test.ts @@ -0,0 +1,49 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import React from 'react'; +import { renderToStaticMarkup } from 'react-dom/server'; +import { OwnerPanelState } from './OwnerPanelState.js'; +import { OwnerHeardNeighbors } from './OwnerHeardNeighbors.js'; + +function panel(hasData: boolean, error: string | null, loading = false) { + return renderToStaticMarkup(React.createElement(OwnerPanelState, { + label: 'Direct Senders', hasData, error, loading, onRetry: () => {}, + children: React.createElement('p', null, 'No direct senders observed.'), + })); +} + +test('owner panels distinguish loading and unavailable from a successful empty result', () => { + assert.match(panel(false, null), /Loading Direct Senders/); + assert.doesNotMatch(panel(false, null), /No direct senders observed/); + assert.match(panel(false, 'timeout'), /role="alert"/); + assert.match(panel(false, 'timeout'), /Retry Direct Senders/); + assert.doesNotMatch(panel(false, 'timeout'), /No direct senders observed/); + assert.match(panel(true, null), /No direct senders observed/); + assert.doesNotMatch(panel(true, null), /Loading|temporarily unavailable/); +}); + +test('refresh failures keep prior content visibly marked as out of date', () => { + const html = panel(true, 'HTTP 503'); + assert.match(html, /last successful data/); + assert.match(html, /No direct senders observed/); + assert.match(html, /Retry Direct Senders/); +}); + +test('background refresh keeps successful content in place without adding a loading row', () => { + const html = panel(true, null, true); + assert.match(html, /No direct senders observed/); + assert.doesNotMatch(html, /Loading|Refreshing/); +}); + +test('neighbour panel describes packet fallback and does not misreport a published empty list', () => { + const html = renderToStaticMarkup(React.createElement(OwnerHeardNeighbors, { + source: 'direct_packets', neighbors: [{ id: 'peer', rssi: -95.5, snr: 0, last_seen_at: null, sampled_at: null }], + })); + assert.match(html, /received packets in the last 24 hours/); + assert.match(html, /Avg RSSI/); + assert.match(html, /-95.5 dBm/); + assert.match(html, /0.0 dB/); + const empty = renderToStaticMarkup(React.createElement(OwnerHeardNeighbors, { source: 'reported', neighbors: [] })); + assert.match(empty, /latest reported neighbor list is empty/); + assert.doesNotMatch(empty, /No neighbor list was reported/); +}); diff --git a/frontend/src/pages/owner/OwnerPanelState.tsx b/frontend/src/pages/owner/OwnerPanelState.tsx new file mode 100644 index 0000000..6efdac0 --- /dev/null +++ b/frontend/src/pages/owner/OwnerPanelState.tsx @@ -0,0 +1,28 @@ +import React from 'react'; +import { LoadingIndicator } from '../../components/LoadingIndicator.js'; + +export type OwnerPanelStateProps = { + label: string; + hasData: boolean; + loading: boolean; + error: string | null; + onRetry: () => void; + children: React.ReactNode; +}; + +export function OwnerPanelState({ label, hasData, error, onRetry, children }: OwnerPanelStateProps) { + // Keep the last successful snapshot mounted during background polls. Adding + // a spinner above it changes the panel height and makes the whole dashboard + // jump on every refresh. + return <> + {error ? ( +
+

{label} is temporarily unavailable.{hasData ? ' Showing the last successful data; it may be out of date.' : ' Please try again shortly.'}

+ +
+ ) : !hasData ? ( + + ) : null} + {hasData ? children : null} + ; +} diff --git a/frontend/src/pages/owner/OwnerPortalCharts.tsx b/frontend/src/pages/owner/OwnerPortalCharts.tsx index b1250ef..e974222 100644 --- a/frontend/src/pages/owner/OwnerPortalCharts.tsx +++ b/frontend/src/pages/owner/OwnerPortalCharts.tsx @@ -1,15 +1,18 @@ import React, { useEffect, useMemo, useState } from 'react'; import { Area, AreaChart, Line, LineChart, ResponsiveContainer, Tooltip, XAxis, YAxis } from 'recharts'; import { + LAST_HOP_WINDOW_OPTIONS, OWNER_AXIS_COLOR as AXIS_COLOR, OWNER_LABEL_COLOR as LABEL_COLOR, OWNER_TOOLTIP_BG as TIP_BG, OWNER_TOOLTIP_BORDER as TIP_BORDER, formatCompactTs, formatDurationMs, + lastHopWindowLabel, readExcludedLastHopSeries, writeExcludedLastHopSeries, type LastHopStrengthPoint, + type LastHopWindow, type OwnerLiveResponse, } from './ownerPortalModel.js'; export const TrendBars: React.FC<{ points: Array<{ bucket: string; adverts: number }> }> = ({ points }) => { @@ -93,6 +96,25 @@ const OwnerTelemetryTooltip: React.FC<{ const LAST_HOP_COLORS = ['#00c4ff', '#6ddc7a', '#ff9f43', '#f472b6', '#a78bfa', '#facc15']; const MIN_LAST_HOP_SAMPLES = 10; +const LastHopWindowToggle: React.FC<{ + value: LastHopWindow; + onChange: (value: LastHopWindow) => void; +}> = ({ value, onChange }) => ( +
+ {LAST_HOP_WINDOW_OPTIONS.map((option) => ( + + ))} +
+); + const LastHopStrengthTooltip: React.FC<{ active?: boolean; payload?: Array<{ dataKey: string; value: number; color: string; payload: Record }>; @@ -118,7 +140,16 @@ const LastHopStrengthTooltip: React.FC<{ ); }; -export const LastHopStrengthChart: React.FC<{ nodeId: string; points: LastHopStrengthPoint[]; isPassiveRepeater?: boolean }> = ({ nodeId, points, isPassiveRepeater }) => { +export const LastHopStrengthChart: React.FC<{ + nodeId: string; + points: LastHopStrengthPoint[]; + timeWindow: LastHopWindow; + onTimeWindowChange: (value: LastHopWindow) => void; + isPassiveRepeater?: boolean; + compareSeries?: Array<{ id: string; name: string; points: LastHopStrengthPoint[] }>; +}> = ({ nodeId, points, timeWindow, onTimeWindowChange, isPassiveRepeater, compareSeries }) => { + const compareMode = Boolean(compareSeries && compareSeries.length >= 2); + const bucketLabel = timeWindow === '30d' ? '12h' : 'hourly'; const [selectedSeriesKey, setSelectedSeriesKey] = useState(null); const [excludedSeriesKeys, setExcludedSeriesKeys] = useState(() => readExcludedLastHopSeries(nodeId)); const visiblePoints = useMemo( @@ -133,53 +164,91 @@ export const LastHopStrengthChart: React.FC<{ nodeId: string; points: LastHopStr }, [nodeId]); const chartState = useMemo(() => { - const totals = new Map; resolution: LastHopStrengthPoint['resolution']; - }>(); - for (const point of visiblePoints) { - const key = point.lastHopNodeId ?? `unresolved:${point.lastHopName}`; - const existing = totals.get(key); - if (existing) { - existing.totalSamples += point.sampleCount; - existing.buckets.add(point.bucket); - existing.bucketCount = existing.buckets.size; - } else { - totals.set(key, { + }; + const totals = new Map(); + const bucketed = new Map>(); + const addSample = (bucket: string, key: string, value: number | null, samples: number) => { + const entry = bucketed.get(bucket) ?? { bucket }; + entry[key] = value; + entry[`${key}__samples`] = samples; + bucketed.set(bucket, entry); + }; + + if (compareMode && compareSeries) { + // Compare mode: one line per selected repeater. Each bucket value is the + // sample-weighted average SNR across that repeater's last-hop series. + const accumulators = new Map>(); + for (const node of compareSeries) { + const key = `cmp:${node.id}`; + const nodeTotals = totals.get(key) ?? { key, - label: point.lastHopName, - totalSamples: point.sampleCount, - bucketCount: 1, - buckets: new Set([point.bucket]), - resolution: point.resolution, - }); + label: node.name, + totalSamples: 0, + bucketCount: 0, + buckets: new Set(), + resolution: 'resolved' as const, + }; + totals.set(key, nodeTotals); + const perBucket = accumulators.get(key) ?? new Map(); + accumulators.set(key, perBucket); + for (const point of node.points) { + if (point.resolution === 'unresolved' || point.avgSnr == null) continue; + const samples = Math.max(1, point.sampleCount); + const acc = perBucket.get(point.bucket) ?? { sum: 0, samples: 0 }; + acc.sum += point.avgSnr * samples; + acc.samples += samples; + perBucket.set(point.bucket, acc); + nodeTotals.totalSamples += point.sampleCount; + nodeTotals.buckets.add(point.bucket); + } + nodeTotals.bucketCount = nodeTotals.buckets.size; + } + for (const [key, perBucket] of accumulators) { + for (const [bucket, acc] of perBucket) { + if (acc.samples < 1) continue; + addSample(bucket, key, Number((acc.sum / acc.samples).toFixed(2)), acc.samples); + } + } + } else { + for (const point of visiblePoints) { + const key = point.lastHopNodeId ?? `unresolved:${point.lastHopName}`; + const existing = totals.get(key); + if (existing) { + existing.totalSamples += point.sampleCount; + existing.buckets.add(point.bucket); + existing.bucketCount = existing.buckets.size; + } else { + totals.set(key, { + key, + label: point.lastHopName, + totalSamples: point.sampleCount, + bucketCount: 1, + buckets: new Set([point.bucket]), + resolution: point.resolution, + }); + } + addSample(point.bucket, key, point.avgSnr, point.sampleCount); } } const allSeries = Array.from(totals.values()) - .filter((series) => series.totalSamples >= MIN_LAST_HOP_SAMPLES) - .sort((a, b) => b.totalSamples - a.totalSamples || a.label.localeCompare(b.label)); + .filter((series) => (compareMode ? series.totalSamples > 0 : series.totalSamples >= MIN_LAST_HOP_SAMPLES)) + .sort(compareMode + ? (a, b) => a.label.localeCompare(b.label) + : (a, b) => b.totalSamples - a.totalSamples || a.label.localeCompare(b.label)); const includedSeries = allSeries.filter((series) => !excludedSet.has(series.key)); const selected = selectedSeriesKey ? includedSeries.filter((series) => series.key === selectedSeriesKey) - : includedSeries.slice(0, 6); - const selectedKeys = new Set(selected.map((series) => series.key)); + : includedSeries.slice(0, compareMode ? 8 : 6); const seriesByKey = new Map(allSeries.map((series) => [series.key, series] as const)); - const bucketed = new Map>(); - for (const point of visiblePoints) { - const key = point.lastHopNodeId ?? `unresolved:${point.lastHopName}`; - if (!selectedKeys.has(key)) continue; - const entry = bucketed.get(point.bucket) ?? { bucket: point.bucket }; - entry[key] = point.avgSnr; - entry[`${key}__samples`] = point.sampleCount; - bucketed.set(point.bucket, entry); - } - return { allSeries, includedSeries, @@ -188,7 +257,7 @@ export const LastHopStrengthChart: React.FC<{ nodeId: string; points: LastHopStr seriesByKey, data: Array.from(bucketed.values()).sort((a, b) => String(a.bucket).localeCompare(String(b.bucket))), }; - }, [excludedSet, selectedSeriesKey, visiblePoints]); + }, [compareMode, compareSeries, excludedSet, selectedSeriesKey, visiblePoints]); useEffect(() => { if (selectedSeriesKey && !chartState.allSeries.some((series) => series.key === selectedSeriesKey)) { @@ -203,8 +272,11 @@ export const LastHopStrengthChart: React.FC<{ nodeId: string; points: LastHopStr }, [excludedSet, selectedSeriesKey]); useEffect(() => { + // Compare-mode exclusions are session-only; the persisted list is keyed to + // last-hop series, not repeaters. + if (compareMode) return; writeExcludedLastHopSeries(nodeId, excludedSeriesKeys); - }, [excludedSeriesKeys, nodeId]); + }, [excludedSeriesKeys, nodeId, compareMode]); const toggleSeriesExclusion = (seriesKey: string) => { setExcludedSeriesKeys((current) => ( @@ -228,23 +300,27 @@ export const LastHopStrengthChart: React.FC<{ nodeId: string; points: LastHopStr
-

RX Strength by Last Hop (Node names are predicted based on the first two hex characters in the path)

-

Average SNR over the last 7 days

+

RX Strength by Last Hop {compareMode ? '(One line per selected repeater)' : '(Node names are predicted based on the first two hex characters in the path)'}

+

{compareMode ? 'Comparing the selected repeaters' : `Average SNR over the last ${lastHopWindowLabel(timeWindow)}`}

- {isPassiveRepeater - ? 'This node does not report received packets via MQTT — rx signal data is captured by its companion node' - : 'No last-hop repeaters with 10+ samples yet'} + {compareMode + ? 'No comparable RX samples for the selected repeaters yet' + : isPassiveRepeater + ? 'This node does not report received packets via MQTT — rx signal data is captured by its companion node' + : 'No last-hop repeaters with 10+ samples yet'}
- Last 7d + Minimum 10 samples
@@ -267,15 +343,19 @@ export const LastHopStrengthChart: React.FC<{ nodeId: string; points: LastHopStr
-

RX Strength by Last Hop (Node names are predicted based on the first two hex characters in the path)

-

{latestActive || (chartState.includedSeries.length > 0 ? 'Average SNR over the last 7 days' : 'All eligible repeaters are currently hidden')}

+

RX Strength by Last Hop {compareMode ? '(One line per selected repeater)' : '(Node names are predicted based on the first two hex characters in the path)'}

+

{compareMode + ? `Comparing ${chartState.includedSeries.length} repeater${chartState.includedSeries.length === 1 ? '' : 's'} · average SNR over the last ${lastHopWindowLabel(timeWindow)}` + : (latestActive || (chartState.includedSeries.length > 0 ? `Average SNR over the last ${lastHopWindowLabel(timeWindow)}` : 'All eligible repeaters are currently hidden'))}

{chartState.series.length}
{chartState.series.length > 0 && chartState.data.length > 0 ? ( @@ -310,11 +390,13 @@ export const LastHopStrengthChart: React.FC<{ nodeId: string; points: LastHopStr )}
- Last 7d + {selectedSeriesKey ? 'Focused on one repeater' - : `${chartState.includedSeries.length} shown of ${chartState.allSeries.length}`} + : compareMode + ? `${chartState.includedSeries.length} of ${chartState.allSeries.length} repeaters` + : `${chartState.includedSeries.length} shown of ${chartState.allSeries.length}`} {chartState.excludedCount > 0 ? ` · ${chartState.excludedCount} hidden` : ''}
@@ -350,7 +432,7 @@ export const LastHopStrengthChart: React.FC<{ nodeId: string; points: LastHopStr type="button" className={`owner-telemetry-metric__series-btn${selectedSeriesKey === series.key ? ' owner-telemetry-metric__series-btn--active' : ''}`} onClick={() => setSelectedSeriesKey(series.key)} - title={`${series.label} · ${series.bucketCount} hourly buckets · ${series.totalSamples} packets in last 7d`} + title={`${series.label} · ${series.bucketCount} ${bucketLabel} buckets · ${series.totalSamples} packets in last ${timeWindow}`} > {series.label} ({series.totalSamples}) diff --git a/frontend/src/pages/owner/ownerPortalModel.test.ts b/frontend/src/pages/owner/ownerPortalModel.test.ts index bdbeaf2..9711e48 100644 --- a/frontend/src/pages/owner/ownerPortalModel.test.ts +++ b/frontend/src/pages/owner/ownerPortalModel.test.ts @@ -9,20 +9,8 @@ import { isOwnerSessionResponse, isValidMapCoord, nodeRoleLabel, - ownerLoginCredentials, - ownerLoginValidationError, } from './ownerPortalModel.js'; -test('owner login trims only the username and preserves password whitespace', () => { - assert.equal(ownerLoginValidationError(' owner ', ' password '), null); - assert.deepEqual(ownerLoginCredentials(' owner ', ' password '), { - mqttUsername: 'owner', - mqttPassword: ' password ', - }); - assert.match(ownerLoginValidationError('owner', '\tpassword') ?? '', /invalid characters/); - assert.match(ownerLoginValidationError('owner', 'x'.repeat(129)) ?? '', /too long/); -}); - test('owner map coordinate policy rejects placeholders and invalid coordinates', () => { assert.equal(isValidMapCoord(51.5, -0.1), true); assert.equal(isValidMapCoord(0, 0), false); diff --git a/frontend/src/pages/owner/ownerPortalModel.ts b/frontend/src/pages/owner/ownerPortalModel.ts index f9ce3b3..5c344aa 100644 --- a/frontend/src/pages/owner/ownerPortalModel.ts +++ b/frontend/src/pages/owner/ownerPortalModel.ts @@ -21,24 +21,6 @@ export function nodeRoleLabel(role: number | null): string { return 'Repeater'; } -export type OwnerLoginCredentials = { mqttUsername: string; mqttPassword: string }; - -export function ownerLoginValidationError(mqttUsername: string, mqttPassword: string): string | null { - const normalizedUsername = mqttUsername.trim(); - if (!normalizedUsername || !mqttPassword) return 'Enter your MQTT username and password.'; - if (normalizedUsername.length > 128 || mqttPassword.length > 128) { - return 'MQTT username or password is too long.'; - } - if (/[\u0000-\u001F\u007F]/.test(normalizedUsername) || /[\u0000-\u001F\u007F]/.test(mqttPassword)) { - return 'MQTT username or password contains invalid characters.'; - } - return null; -} - -export function ownerLoginCredentials(mqttUsername: string, mqttPassword: string): OwnerLoginCredentials { - return { mqttUsername: mqttUsername.trim(), mqttPassword }; -} - export type OwnerDashboard = { nodes: OwnerNode[]; }; @@ -49,6 +31,39 @@ export type OwnerSessionResponse = { mqttUsername?: string | null; }; +export type OwnerPacketSharingDestination = { + id: string; + name: string; + websiteUrl: string | null; + description: string | null; + configured: boolean; + selected: boolean; + lastForwardedAt: string | null; +}; + +export type OwnerPacketSharing = { + nodeId: string; + featureEnabled: boolean; + enabled: boolean; + destinations: OwnerPacketSharingDestination[]; +}; + +export type OwnerPublicFeed = { + featureEnabled: boolean; + configured: boolean; + brokerUrl: string | null; + enabled: boolean; + nodeCount: number; + eligible: boolean; + recentPacketCount: number; + topicFilters: string[]; + lastForwardedAt: string | null; +}; + +export function isPacketSharingOwner(username: string | null | undefined): boolean { + return typeof username === 'string' && username.trim().length > 0; +} + const OWNER_SESSION_EVENT = 'meshcore-owner-session'; const LAST_HOP_EXCLUDED_COOKIE = 'meshcore-owner-last-hop-hidden-v1'; const LAST_HOP_COOKIE_MAX_AGE = 60 * 60 * 24 * 365; @@ -208,6 +223,8 @@ export type OwnerLiveResponse = { }>; status: OwnerStatus | null; heardNeighbors: HeardNeighbor[]; + heardNeighborsSource?: 'reported' | 'direct_packets' | 'none'; + heardNeighborsSampledAt?: string | null; packetsSent24h: number; packetsReceived24h: number; alerts: Array<{ level: 'info' | 'warn' | 'error'; message: string }>; @@ -224,12 +241,21 @@ export type LastHopStrengthPoint = { sampleCount: number; }; +export type LastHopWindow = '7d' | '30d'; + +export const LAST_HOP_WINDOW_OPTIONS: readonly LastHopWindow[] = ['7d', '30d']; + +export function lastHopWindowLabel(value: LastHopWindow): string { + return value === '30d' ? '30 days' : '7 days'; +} + export type OwnerLastHopStrengthResponse = { points: LastHopStrengthPoint[]; }; -// Scope: authenticated owner session + public privacy generation. This cache is -// cleared at logout/session expiry and bounded to 128 series or 12 MiB. +// Scope: authenticated owner session + public privacy generation + selected time +// window. This cache is cleared at logout/session expiry and bounded to 128 +// series or 12 MiB. export const lastHopSeriesCache = new ScopedCache({ name: 'owner-last-hop', ttlMs: 5 * 60_000, @@ -254,6 +280,92 @@ export function isOwnerSessionResponse(value: unknown): value is OwnerSessionRes && (value['mqttUsername'] == null || typeof value['mqttUsername'] === 'string'); } +export function isOwnerPacketSharingResponse(value: unknown): value is OwnerPacketSharing { + if (!isRecord(value) + || typeof value['nodeId'] !== 'string' + || !/^[0-9A-Fa-f]{64}$/.test(value['nodeId']) + || typeof value['featureEnabled'] !== 'boolean' + || typeof value['enabled'] !== 'boolean' + || !Array.isArray(value['destinations'])) return false; + return value['destinations'].every((destination) => isRecord(destination) + && typeof destination['id'] === 'string' + && /^[a-z0-9][a-z0-9_-]{0,63}$/.test(destination['id']) + && typeof destination['name'] === 'string' + && destination['name'].length >= 1 + && destination['name'].length <= 120 + && (destination['websiteUrl'] == null || isSafePacketShareWebsiteUrl(destination['websiteUrl'])) + && (destination['description'] == null || ( + typeof destination['description'] === 'string' + && destination['description'].length >= 1 + && destination['description'].length <= 160 + )) + && typeof destination['configured'] === 'boolean' + && typeof destination['selected'] === 'boolean' + && (destination['lastForwardedAt'] == null || ( + typeof destination['lastForwardedAt'] === 'string' + && destination['lastForwardedAt'].length <= 64 + && Number.isFinite(Date.parse(destination['lastForwardedAt'])) + ))); +} + +function isSafePacketShareWebsiteUrl(value: unknown): value is string { + if (typeof value !== 'string' || value.length < 1 || value.length > 2_048) return false; + try { + const url = new URL(value); + return url.protocol === 'https:' && Boolean(url.hostname) && !url.username && !url.password; + } catch { + return false; + } +} + +export async function fetchOwnerPacketSharing( + nodeId: string, + signal?: AbortSignal, +): Promise { + return fetchJson( + `/api/owner/packet-sharing?nodeId=${encodeURIComponent(nodeId)}`, + { cache: 'no-store', signal }, + { timeoutMs: 10_000, maxBytes: 128 * 1024, validate: isOwnerPacketSharingResponse }, + ); +} + +export function isOwnerPublicFeedResponse(value: unknown): value is OwnerPublicFeed { + return isRecord(value) + && typeof value['featureEnabled'] === 'boolean' + && typeof value['configured'] === 'boolean' + && (value['brokerUrl'] == null || ( + typeof value['brokerUrl'] === 'string' + && value['brokerUrl'].length <= 2_048 + )) + && typeof value['enabled'] === 'boolean' + && typeof value['nodeCount'] === 'number' + && Number.isInteger(value['nodeCount']) + && value['nodeCount'] >= 0 + && typeof value['eligible'] === 'boolean' + && typeof value['recentPacketCount'] === 'number' + && Number.isInteger(value['recentPacketCount']) + && value['recentPacketCount'] >= 0 + && Array.isArray(value['topicFilters']) + && value['topicFilters'].length <= 1 + && value['topicFilters'].every((topic) => ( + typeof topic === 'string' + && /^public\/\+\/\+\/packets$/.test(topic) + )) + && (value['lastForwardedAt'] == null || ( + typeof value['lastForwardedAt'] === 'string' + && value['lastForwardedAt'].length <= 64 + && Number.isFinite(Date.parse(value['lastForwardedAt'])) + )); +} + +export async function fetchOwnerPublicFeed(signal?: AbortSignal): Promise { + return fetchJson( + '/api/owner/public-feed', + { cache: 'no-store', signal }, + { timeoutMs: 10_000, maxBytes: 32 * 1024, validate: isOwnerPublicFeedResponse }, + ); +} + export function isOwnerLiveResponse(value: unknown): value is OwnerLiveResponse { if (!isRecord(value)) return false; const status = value['status']; diff --git a/frontend/src/pages/owner/packetShareModel.test.ts b/frontend/src/pages/owner/packetShareModel.test.ts new file mode 100644 index 0000000..d5c25a2 --- /dev/null +++ b/frontend/src/pages/owner/packetShareModel.test.ts @@ -0,0 +1,54 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { isOwnerPacketSharingResponse, isOwnerPublicFeedResponse, isPacketSharingOwner } from './ownerPortalModel.js'; + +const valid = { + nodeId: 'A'.repeat(64), + featureEnabled: true, + enabled: false, + destinations: [{ + id: 'remote-site', + name: 'Remote site', + websiteUrl: 'https://remote.example.test', + description: 'A remote packet site', + configured: true, + selected: false, + lastForwardedAt: null, + }], +}; + +test('owner packet-sharing response guard accepts the bounded settings DTO', () => { + assert.equal(isOwnerPacketSharingResponse(valid), true); + assert.equal(isOwnerPacketSharingResponse({ ...valid, nodeId: 'short' }), false); + assert.equal(isOwnerPacketSharingResponse({ ...valid, destinations: [{ ...valid.destinations[0], id: 'Remote Site' }] }), false); + assert.equal(isOwnerPacketSharingResponse({ ...valid, destinations: [{ ...valid.destinations[0], selected: 'yes' }] }), false); + assert.equal(isOwnerPacketSharingResponse({ ...valid, destinations: [{ ...valid.destinations[0], websiteUrl: 'javascript:alert(1)' }] }), false); + assert.equal(isOwnerPacketSharingResponse({ ...valid, destinations: [{ ...valid.destinations[0], lastForwardedAt: 'not-a-date' }] }), false); + assert.equal(isOwnerPacketSharingResponse({ ...valid, featureEnabled: 'true' }), false); +}); + +test('shows packet sharing for every authenticated owner account', () => { + assert.equal(isPacketSharingOwner('node1'), true); + assert.equal(isPacketSharingOwner('gnomemoe'), true); + assert.equal(isPacketSharingOwner('node2'), true); + assert.equal(isPacketSharingOwner('new-owner'), true); + assert.equal(isPacketSharingOwner(null), false); +}); + +test('owner public-feed response guard accepts the account-level settings DTO', () => { + const valid = { + featureEnabled: true, + configured: true, + brokerUrl: 'wss://mqtt.example.test:443', + enabled: false, + nodeCount: 2, + eligible: true, + recentPacketCount: 1, + topicFilters: ['public/+/+/packets'], + lastForwardedAt: null, + }; + assert.equal(isOwnerPublicFeedResponse(valid), true); + assert.equal(isOwnerPublicFeedResponse({ ...valid, nodeCount: -1 }), false); + assert.equal(isOwnerPublicFeedResponse({ ...valid, topicFilters: ['meshcore/#'] }), false); + assert.equal(isOwnerPublicFeedResponse({ ...valid, lastForwardedAt: 'invalid' }), false); +}); diff --git a/frontend/src/pages/regions-page.css b/frontend/src/pages/regions-page.css new file mode 100644 index 0000000..844b36d --- /dev/null +++ b/frontend/src/pages/regions-page.css @@ -0,0 +1,79 @@ +.regions-page { padding-top: 48px; padding-bottom: 64px; } +.regions-page__header { display: flex; align-items: flex-end; justify-content: space-between; gap: 32px; margin-bottom: 24px; } +.regions-page__header h1 { margin: 4px 0 10px; } +.regions-page__header p { max-width: 760px; color: var(--text-secondary); } +.regions-page__eyebrow { color: var(--accent) !important; font: 600 11px/1 var(--font-mono); letter-spacing: 0.13em; text-transform: uppercase; } +.regions-page__controls { display: flex; flex-wrap: wrap; align-items: center; justify-content: flex-end; gap: 14px; max-width: 420px; } +.regions-page__toggle { display: flex; align-items: center; gap: 8px; white-space: nowrap; color: var(--text-secondary); font-size: 12px; cursor: pointer; } +.regions-page__select { + appearance: none; + -webkit-appearance: none; + min-width: 128px; + padding: 8px 34px 8px 10px; + border: 1px solid rgba(107, 174, 210, 0.58); + border-radius: var(--radius); + background-color: #0d1828; + background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='14' height='14' viewBox='0 0 14 14' fill='none' stroke='%23d6e8ff' stroke-width='1.6' stroke-linecap='round' stroke-linejoin='round'%3E%3Cpath d='m3 5 4 4 4-4'/%3E%3C/svg%3E"); + background-repeat: no-repeat; + background-position: right 10px center; + background-size: 14px; + color: #e8f4ff; + color-scheme: dark; + font: 12px var(--font-body); + cursor: pointer; +} +.regions-page__select:hover { border-color: var(--accent); background-color: #11243a; } +.regions-page__select:focus-visible { outline: 2px solid var(--accent); outline-offset: 2px; } +.regions-page__select option { background: #0d1828; color: #e8f4ff; } +.regions-page__stats { display: grid; grid-template-columns: repeat(5, 1fr); gap: 10px; margin-bottom: 12px; } +.regions-page__stats > div { display: flex; flex-direction: column; padding: 14px 16px; background: var(--bg-panel); border: 1px solid var(--border); border-radius: var(--radius-lg); } +.regions-page__stats strong { color: var(--text-primary); font: 600 22px/1.2 var(--font-mono); } +.regions-page__stats span { margin-top: 4px; color: var(--text-muted); font-size: 11px; } +.regions-page__workspace { display: grid; grid-template-columns: minmax(0, 1fr) 280px; gap: 12px; } +.regions-page__map { position: relative; height: min(68vh, 680px); min-height: 520px; overflow: hidden; background: #07101d; border: 1px solid var(--border); border-radius: var(--radius-lg); } +.regions-page__map .map-area { height: 100%; } +.regions-page__map-meta { position: absolute; top: 14px; left: 14px; z-index: 2; display: grid; gap: 2px; max-width: min(330px, calc(100% - 28px)); padding: 9px 11px; border: 1px solid rgba(104, 177, 216, 0.3); border-radius: var(--radius); background: rgba(2, 8, 16, 0.84); box-shadow: 0 8px 20px rgba(0, 0, 0, 0.24); color: var(--text-muted); font: 10px/1.35 var(--font-mono); pointer-events: none; } +.regions-page__map-meta strong { color: var(--text-primary); font: 600 11px var(--font-mono); letter-spacing: 0.06em; text-transform: uppercase; } +.regions-page__map-legend { position: absolute; right: 14px; bottom: 14px; z-index: 2; display: grid; gap: 4px; max-width: min(390px, calc(100% - 28px)); padding: 9px 11px; border: 1px solid rgba(104, 177, 216, 0.3); border-radius: var(--radius); background: rgba(2, 8, 16, 0.84); box-shadow: 0 8px 20px rgba(0, 0, 0, 0.24); color: var(--text-muted); font: 10px/1.35 var(--font-mono); pointer-events: none; } +.regions-page__legend-swatch { display: inline-block; width: 9px; height: 9px; margin: 0 7px 2px 0; border-radius: 50%; background: #22c55e; vertical-align: middle; } +.regions-page__legend-swatch--green { background: #22c55e; } +.regions-page__legend-swatch--red { background: #ef4444; } +.regions-page__legend-swatch--area { width: 18px; height: 10px; border-radius: 2px; border: 1px solid #22d3ee; background: rgba(34, 211, 238, 0.22); } +.regions-page__map .maplibregl-ctrl-group { border: 1px solid rgba(104, 177, 216, 0.35); border-radius: var(--radius); background: rgba(2, 8, 16, 0.84); box-shadow: 0 8px 20px rgba(0, 0, 0, 0.24); } +.regions-page__map .maplibregl-ctrl button { filter: invert(0.9); } +.regions-page__side { background: #07101d; border: 1px solid var(--border); border-radius: var(--radius-lg); padding: 18px; } +.regions-page__side h2 { margin: 0 0 12px; font-size: 15px; } +.regions-page__selected { display: flex; flex-direction: column; gap: 7px; color: var(--text-secondary); font-size: 12px; } +.regions-page__selected strong { color: var(--accent); font-size: 14px; overflow-wrap: anywhere; } +.regions-page__region-dot { display: inline-block; width: 9px; height: 9px; margin-right: 7px; border-radius: 50%; vertical-align: middle; } +.regions-page__badge { width: fit-content; padding: 5px 8px; border: 1px solid var(--border); border-radius: var(--radius); font: 10px/1.4 var(--font-mono); } +.regions-page__badge--green { color: #22c55e; border-color: rgba(34, 197, 94, 0.55); } +.regions-page__badge--red { color: #ef4444; border-color: rgba(239, 68, 68, 0.55); } +.regions-page__badge--muted { color: var(--text-muted); } +.regions-page__chips { display: flex; flex-wrap: wrap; gap: 6px; } +.regions-page__chip { padding: 4px 7px; border: 1px solid var(--border); border-radius: var(--radius); color: var(--text-secondary); font: 10px/1.3 var(--font-mono); } +.regions-page__chip strong { color: var(--text-primary); font-size: 10px; } +.regions-page__list { margin: 0; padding: 0; list-style: none; } +.regions-page__list li button { display: grid; grid-template-columns: minmax(0, 1fr) auto; align-items: center; gap: 7px; width: 100%; padding: 8px 5px; border: 0; border-bottom: 1px solid var(--border); background: transparent; color: var(--text-secondary); text-align: left; cursor: pointer; } +.regions-page__list li button:hover { color: var(--accent); background: var(--bg-hover); } +.regions-page__list li span { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.regions-page__list li strong { color: var(--text-primary); font: 11px var(--font-mono); } +.regions-page__list li i { display: inline-block; width: 8px; height: 8px; margin-right: 7px; border-radius: 50%; vertical-align: middle; } +.regions-page__low-tag { margin-left: 6px; padding: 1px 5px; border: 1px solid rgba(148, 163, 184, 0.42); border-radius: var(--radius); color: var(--text-muted); font: 9px/1.4 var(--font-mono); font-style: normal; letter-spacing: 0.08em; text-transform: uppercase; vertical-align: middle; } +.regions-page__note { margin-top: 4px; color: var(--text-muted); font-size: 10px; } +.regions-page__error { padding: 14px; border: 1px solid #7f1d1d; border-radius: var(--radius); color: #fca5a5; } +.regions-page__clear { margin-top: 8px; padding: 7px; border: 1px solid var(--border); border-radius: var(--radius); background: var(--bg-active); color: var(--text-secondary); cursor: pointer; } + +@media (max-width: 900px) { + .regions-page__workspace { grid-template-columns: 1fr; } + .regions-page__stats { grid-template-columns: repeat(3, 1fr); } +} + +@media (max-width: 640px) { + .regions-page { padding-top: 28px; } + .regions-page__header { align-items: flex-start; flex-direction: column; gap: 12px; } + .regions-page__controls { justify-content: flex-start; } + .regions-page__stats { grid-template-columns: 1fr 1fr; } + .regions-page__map { min-height: 430px; height: 62vh; } + .regions-page__map-legend { right: 10px; bottom: 10px; max-width: calc(100% - 20px); } +} diff --git a/frontend/src/pages/regionsModel.test.ts b/frontend/src/pages/regionsModel.test.ts new file mode 100644 index 0000000..c7801c3 --- /dev/null +++ b/frontend/src/pages/regionsModel.test.ts @@ -0,0 +1,138 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { + confidenceFor, + convexHull, + isEstablished, + memberIdSet, + regionColor, + regionRing, + regionsForDisplay, + type RegionMember, + type RegionSummary, +} from './regionsModel.js'; + +function member( + nodeId: string, role: number | null, lat: number | null, lon: number | null, + packets = 5, isBridge = false, +): RegionMember { + return { nodeId, name: nodeId, role, lat, lon, packets, isBridge }; +} + +function region(members: RegionMember[], overrides: Partial = {}): RegionSummary { + return { + name: 'yorkshire', + packetsTotal: 100, + memberCount: members.length, + repeaterMemberCount: members.filter((entry) => entry.role === 2).length, + confidence: 'confirmed', + firstSeen: null, + lastSeen: null, + members, + ...overrides, + }; +} + +test('convexHull returns the square hull with interior points excluded', () => { + const hull = convexHull([[0, 0], [2, 0], [2, 2], [0, 2], [1, 1], [0.5, 1.5]]); + assert.equal(hull.length, 4); + const corners = new Set(hull.map(([x, y]) => `${x},${y}`)); + for (const corner of ['0,0', '2,0', '2,2', '0,2']) assert.ok(corners.has(corner), `missing ${corner}`); +}); + +test('convexHull collapses collinear and duplicate points', () => { + assert.deepEqual(convexHull([[0, 0], [1, 1], [2, 2], [1, 1]]), [[0, 0], [2, 2]]); + assert.deepEqual(convexHull([[5, 5]]), [[5, 5]]); + assert.deepEqual(convexHull([]), []); +}); + +test('regionRing builds a closed ring preferring repeater members', () => { + const ring = regionRing(region([ + member('A', 2, 54.0, -1.0), + member('B', 2, 54.2, -0.8), + member('C', 2, 53.8, -1.4), + member('D', 1, 60.0, 5.0), + ])); + assert.ok(ring); + assert.equal(ring.length, 4); + assert.deepEqual(ring[0], ring[ring.length - 1]); + for (const [, lat] of ring) assert.ok(lat < 55, 'far companion must not shape the repeater hull'); +}); + +test('regionRing falls back to all located members when repeaters are scarce', () => { + const ring = regionRing(region([ + member('A', 2, 54.0, -1.0), + member('B', 1, 54.2, -0.8), + member('C', 1, 53.8, -1.4), + ])); + assert.ok(ring); + assert.equal(ring.length, 4); +}); + +test('regionRing returns null for degenerate inputs', () => { + assert.equal(regionRing(region([member('A', 2, 54, -1), member('B', 2, 54.1, -0.9)])), null); + assert.equal(regionRing(region([ + member('A', 2, 54, -1), + member('B', 2, null, null), + member('C', 2, 54.02, -1.02), + ])), null); + assert.equal(regionRing(region([member('A', 2, 54, -1), member('B', 2, 54.1, -0.9), member('C', 2, 54.2, -0.8)])), null); +}); + +test('regionRing ignores null-island placeholder members', () => { + const ring = regionRing(region([ + member('A', 2, 54.0, -1.0), + member('B', 2, 54.2, -0.8), + member('C', 2, 53.8, -1.4), + member('JUNK', 2, 0, 0), + ])); + assert.ok(ring); + for (const [lon, lat] of ring) { + assert.ok(lat > 50 && lon < 0, 'placeholder coordinates must not shape the hull'); + } +}); + +test('regionColor is stable and well-formed', () => { + assert.equal(regionColor('yorkshire'), regionColor('yorkshire')); + assert.match(regionColor('eng-ne'), /^#[0-9a-f]{6}$/); +}); + +test('confidenceFor mirrors the backend evidence tiers', () => { + assert.equal(confidenceFor(9), 'confirmed'); + assert.equal(confidenceFor(4), 'confirmed'); + assert.equal(confidenceFor(3), 'probable'); + assert.equal(confidenceFor(2), 'probable'); + assert.equal(confidenceFor(1), 'single'); +}); + +test('memberIdSet collects every member id', () => { + const ids = memberIdSet(region([member('A', 2, 54, -1), member('B', 1, 53, -2)])); + assert.deepEqual([...ids].sort(), ['A', 'B']); +}); + +test('isEstablished and regionsForDisplay gate low-confidence regions', () => { + assert.equal(isEstablished({ packetsTotal: 3 }), true); + assert.equal(isEstablished({ packetsTotal: 2 }), false); + const strong = region([member('A', 2, 54, -1)], { name: 'yorkshire', packetsTotal: 100 }); + const weak = region([member('B', 2, 53, -2)], { name: 'micro', packetsTotal: 1 }); + assert.deepEqual(regionsForDisplay([strong, weak], false).map((entry) => entry.name), ['yorkshire']); + assert.deepEqual(regionsForDisplay([strong, weak], true).map((entry) => entry.name), ['yorkshire', 'micro']); +}); + +test('regionRing excludes universal bridge members from the hull', () => { + const ring = regionRing(region([ + member('A', 2, 54.0, -1.0), + member('B', 2, 54.2, -0.8), + member('C', 2, 53.8, -1.4), + member('HUB', 2, 60.0, 5.0, 9000, true), + ])); + assert.ok(ring); + for (const [, lat] of ring) assert.ok(lat < 55, 'bridge members must not shape the hull'); + + const allBridges = regionRing(region([ + member('A', 2, 54.0, -1.0, 5, true), + member('B', 2, 54.2, -0.8, 5, true), + member('C', 2, 53.8, -1.4, 5, true), + ])); + assert.equal(allBridges, null); +}); diff --git a/frontend/src/pages/regionsModel.ts b/frontend/src/pages/regionsModel.ts new file mode 100644 index 0000000..4e56eab --- /dev/null +++ b/frontend/src/pages/regionsModel.ts @@ -0,0 +1,167 @@ +// Region-scope model helpers for the public /regions page. +// +// Pure functions only — the map overlay builders and page components consume +// these; tests run them under node without a DOM. + +export type RegionConfidence = 'confirmed' | 'probable' | 'single'; + +export type RegionMember = { + nodeId: string; + name: string | null; + role: number | null; + lat: number | null; + lon: number | null; + packets: number; + /** True for universal bridge repeaters that relay most of the network's regions. */ + isBridge?: boolean; +}; + +export type RegionSummary = { + name: string; + packetsTotal: number; + memberCount: number; + repeaterMemberCount: number; + confidence: RegionConfidence; + firstSeen: string | null; + lastSeen: string | null; + members: RegionMember[]; +}; + +export type RegionRepeater = { + nodeId: string; + name: string | null; + role: number | null; + lat: number | null; + lon: number | null; + status: 'green' | 'red'; + evidence: string; + scopedPackets: number; + unscopedPackets7d: number; + regions: Record; +}; + +export type RegionSummaryPayload = { + generatedAt: string | null; + windowDays: number | null; + regions: RegionSummary[]; +}; + +export type RegionRepeatersPayload = { + generatedAt: string | null; + repeaters: RegionRepeater[]; +}; + +const REGION_PALETTE = [ + '#22d3ee', '#a78bfa', '#34d399', '#fbbf24', '#f472b6', '#60a5fa', + '#f87171', '#4ade80', '#facc15', '#c084fc', '#38bdf8', '#fb923c', +] as const; + +/** Stable per-region colour: same name always maps to the same palette entry. */ +export function regionColor(name: string): string { + let hash = 0; + for (let index = 0; index < name.length; index += 1) { + hash = (hash * 31 + name.charCodeAt(index)) >>> 0; + } + return REGION_PALETTE[hash % REGION_PALETTE.length]!; +} + +/** Mirrors the backend evidence tiers (>=4 confirmed, 2-3 probable, 1 single). */ +export function confidenceFor(packets: number): RegionConfidence { + if (packets >= 4) return 'confirmed'; + if (packets >= 2) return 'probable'; + return 'single'; +} + +/** + * Display threshold for the default map and list: regions with fewer than + * three decoded packets are chance-match grade — the 2026-10 calibration + * measured ~15% of candidate names picking up a single spurious hit over the + * whole corpus, so 1-2 observations stay hidden unless the user opts in. + */ +export const REGION_ESTABLISHED_MIN_PACKETS = 3; + +export function isEstablished(region: Pick): boolean { + return region.packetsTotal >= REGION_ESTABLISHED_MIN_PACKETS; +} + +/** Regions for the default view; low-confidence ones appear only when asked for. */ +export function regionsForDisplay( + regions: readonly RegionSummary[], + includeLowConfidence: boolean, +): RegionSummary[] { + return includeLowConfidence ? [...regions] : regions.filter(isEstablished); +} + +export function compactNumber(value: number): string { + return new Intl.NumberFormat('en-GB', { notation: 'compact', maximumFractionDigits: 1 }).format(value); +} + +export type Coordinate = [number, number]; + +function hasCoordinates(member: { lat: number | null; lon: number | null }): member is { lat: number; lon: number } { + const { lat, lon } = member; + return lat != null && lon != null + && Number.isFinite(lat) && Number.isFinite(lon) + && lat >= -90 && lat <= 90 + && lon >= -180 && lon <= 180 + && !(Math.abs(lat) < 1e-6 && Math.abs(lon) < 1e-6); +} + +/** + * Andrew's monotone chain over [lon, lat] points. Returns the hull vertices + * counter-clockwise without repeating the first point; collinear inputs yield + * fewer than three vertices (callers treat that as "no polygon"). + */ +export function convexHull(points: ReadonlyArray): Coordinate[] { + const unique = [...new Map(points.map(([x, y]) => [`${x},${y}`, [x, y] as Coordinate])).values()] + .sort((a, b) => a[0] - b[0] || a[1] - b[1]); + if (unique.length <= 2) return unique; + const cross = (o: Coordinate, a: Coordinate, b: Coordinate): number => + (a[0] - o[0]) * (b[1] - o[1]) - (a[1] - o[1]) * (b[0] - o[0]); + const lower: Coordinate[] = []; + for (const point of unique) { + while (lower.length >= 2 && cross(lower[lower.length - 2]!, lower[lower.length - 1]!, point) <= 0) lower.pop(); + lower.push(point); + } + const upper: Coordinate[] = []; + for (let index = unique.length - 1; index >= 0; index -= 1) { + const point = unique[index]!; + while (upper.length >= 2 && cross(upper[upper.length - 2]!, upper[upper.length - 1]!, point) <= 0) upper.pop(); + upper.push(point); + } + lower.pop(); + upper.pop(); + return lower.concat(upper); +} + +/** + * Closed GeoJSON ring for a region: the convex hull of its resolved relay + * members (repeater-role members preferred). Universal bridge repeaters are + * excluded — they relay most regions' traffic, so their positions carry no + * signal about this region's footprint. Null when fewer than three located + * non-bridge members remain or the hull degenerates to a line. + */ +export function regionRing(region: RegionSummary): Coordinate[] | null { + const located = region.members.filter((member) => member.isBridge !== true && hasCoordinates(member)); + const repeaterMembers = located.filter((member) => member.role === 2); + const source = repeaterMembers.length >= 3 ? repeaterMembers : located; + if (source.length < 3) return null; + const hull = convexHull(source.map((member) => [member.lon, member.lat] as Coordinate)); + if (hull.length < 3) return null; + const ring = hull.map(([lon, lat]) => [Number(lon.toFixed(5)), Number(lat.toFixed(5))] as Coordinate); + return [...ring, ring[0]!]; +} + +/** Node ids of a region's resolved relay members, for map filtering. */ +export function memberIdSet(region: RegionSummary): Set { + return new Set(region.members.map((member) => member.nodeId)); +} + +export function formatGeneratedAt(value: string | null): string | null { + if (!value) return null; + const date = new Date(value); + if (!Number.isFinite(date.getTime())) return null; + return date.toLocaleString('en-GB', { + day: 'numeric', month: 'short', hour: '2-digit', minute: '2-digit', + }); +} diff --git a/frontend/src/pages/ukmesh/PacketDetailPanel.tsx b/frontend/src/pages/ukmesh/PacketDetailPanel.tsx index 4e6ac3a..bfc1ca3 100644 --- a/frontend/src/pages/ukmesh/PacketDetailPanel.tsx +++ b/frontend/src/pages/ukmesh/PacketDetailPanel.tsx @@ -3,6 +3,7 @@ import { LoadingIndicator } from '../../components/LoadingIndicator.js'; import type { MeshNode } from '../../hooks/useNodes.js'; import type { FeedPacket } from './UKFeedPage.js'; import { usePacketDetailData, type RadioState } from '../../hooks/usePacketDetailData.js'; +import type { MessageTags, TagConfidence } from '../../hooks/packetFeed.js'; import { PathMap, type LazyPath, @@ -335,6 +336,32 @@ export const PacketDetailPanel: React.FC<{ + {/* Message tags (tagger-worker / Jev) */} + {(() => { + const tags = (packet.tags ?? detail?.tags) as MessageTags | undefined | null; + if (!tags || Object.keys(tags).length === 0) return null; + const conf = (packet.tag_confidence ?? detail?.tagConfidence) as TagConfidence | undefined | null; + const chips: string[] = []; + if (tags.kind) chips.push(tags.kind); + if (tags.topic) chips.push(`topic: ${tags.topic}`); + if (tags.speaker) chips.push(`speaker: ${tags.speaker}`); + if ((tags.safety ?? 0) >= 0.5) chips.push('⚠ safety'); + if ((tags.directed ?? 0) >= 0.5) chips.push('directed'); + if ((tags.mentions_location ?? 0) >= 0.5) chips.push('mentions location'); + return ( +
+
+ Message tags{typeof conf?.kind === 'number' ? ` · kind ${(conf.kind * 100).toFixed(0)}%` : ''} +
+
+ {chips.map((chip) => ( + {chip} + ))} +
+
+ ); + })()} + {/* Observer table */} {detail?.observations && detail.observations.length > 0 && (
diff --git a/frontend/src/pages/ukmesh/UKFeedPage.tsx b/frontend/src/pages/ukmesh/UKFeedPage.tsx index c436839..1fa0806 100644 --- a/frontend/src/pages/ukmesh/UKFeedPage.tsx +++ b/frontend/src/pages/ukmesh/UKFeedPage.tsx @@ -1,6 +1,5 @@ import React, { useMemo, useCallback, useEffect, useState, useRef } from 'react'; import './feed-page.css'; -import { useMeasuredVirtualRows } from '../../hooks/useMeasuredVirtualRows.js'; import { getCurrentSite } from '../../config/site.js'; import { useWebSocket } from '../../hooks/useWebSocket.js'; import { @@ -10,6 +9,8 @@ import { } from '../../hooks/useNodes.js'; import { mapMessageRows, type RecentPacketRow } from '../../hooks/packetFeed.js'; import { useAppMessageHandler } from '../../hooks/useAppMessageHandler.js'; +import { useMessageTags } from '../../hooks/useMessageTags.js'; +import { messageTagKind, messageTagTitle } from '../../hooks/packetFeed.js'; import { ApiResponseError, chartStatsEndpoint, @@ -84,6 +85,8 @@ export const UKFeedPage: React.FC = () => { return requested && (/^(?:[0-9]{1,3}|unknown)$/.test(requested)) ? requested : null; }); const [now, setNow] = useState(() => Date.now()); + const [viewportHeight, setViewportHeight] = useState(() => window.innerHeight); + const [listScrollTop, setListScrollTop] = useState(0); const [detailOpen, setDetailOpen] = useState(false); const [historicalMessages, setHistoricalMessages] = useState([]); const [historyStatus, setHistoryStatus] = useState<'idle' | 'loading' | 'ready' | 'error'>('idle'); @@ -113,6 +116,9 @@ export const UKFeedPage: React.FC = () => { const wsConnection = useWebSocket(handleWSMessage, scope, scopeEpoch); + // Live tag enrichment: tags land ~2s after a packet, so patch rows by hash. + useMessageTags(scope, scopeKey); + useEffect(() => { historyRequestRef.current?.abort(); historyRequestRef.current = null; @@ -163,6 +169,12 @@ export const UKFeedPage: React.FC = () => { return () => window.clearInterval(timer); }, []); + useEffect(() => { + const handleResize = () => setViewportHeight(window.innerHeight); + window.addEventListener('resize', handleResize); + return () => window.removeEventListener('resize', handleResize); + }, []); + // Selected-packet path cache. Requests are demand-driven and scope-fenced. const [, setLazyCacheVersion] = useState(0); const settleTimerRef = useRef | null>(null); @@ -307,11 +319,21 @@ export const UKFeedPage: React.FC = () => { const recentPackets = useMemo(() => { return filteredPackets.slice(0, MAX_PACKETS); }, [filteredPackets]); - const rowKeys = useMemo( - () => recentPackets.map((packet) => `${scopeKey}:${packet.packet_hash}`), - [recentPackets, scopeKey], - ); - const virtualRows = useMeasuredVirtualRows(rowKeys, packetListRef); + const virtualRows = useMemo(() => { + // Rows are natural-height; tagged rows add one ~18px chips line under the + // meta section. Keep the estimate aligned with the taller (tagged) case so + // spacers never underestimate and rows cannot overlap (2026-09-17). + const rowHeight = 76; + const visibleHeight = Math.max(320, viewportHeight - 200); + const start = Math.max(0, Math.floor(listScrollTop / rowHeight) - 5); + const end = Math.min(recentPackets.length, start + Math.ceil(visibleHeight / rowHeight) + 10); + return { + rows: recentPackets.slice(start, end), + start, + top: start * rowHeight, + bottom: Math.max(0, (recentPackets.length - end) * rowHeight), + }; + }, [listScrollTop, recentPackets, viewportHeight]); // Always derive selectedPacket from the live list so new MQTT observers are picked up const selectedPacket = useMemo( @@ -609,19 +631,17 @@ export const UKFeedPage: React.FC = () => {
setListScrollTop(event.currentTarget.scrollTop)} > {recentPackets.length > 0 ? <> + {messageTagKind(packet) && ( +
+ + {messageTagKind(packet)} + +
+ )}

{packetSummary(packet, nodeMap)}

@@ -651,7 +678,8 @@ export const UKFeedPage: React.FC = () => {

No public packets have arrived yet.

)} - {/* Selected details live outside the measured packet list. */} + {/* Selected details live outside the fixed-height virtual rows. This + keeps expansion height from corrupting spacer calculations. */} {selectedPacket && (
@@ -676,6 +704,11 @@ export const UKFeedPage: React.FC = () => { {selectedPacket.hop_count} hop{selectedPacket.hop_count !== 1 ? 's' : ''} )} + {messageTagKind(selectedPacket) && ( + + {messageTagKind(selectedPacket)} + + )}

{packetSummary(selectedPacket, nodeMap)}

@@ -735,6 +768,11 @@ export const UKFeedPage: React.FC = () => { {selectedPacket.hop_count} hop{selectedPacket.hop_count !== 1 ? 's' : ''} )} + {messageTagKind(selectedPacket) && ( + + {messageTagKind(selectedPacket)} + + )}

{packetSummary(selectedPacket, nodeMap)}

diff --git a/frontend/src/pages/ukmesh/UKLayout.tsx b/frontend/src/pages/ukmesh/UKLayout.tsx index 30f9b6c..508da89 100644 --- a/frontend/src/pages/ukmesh/UKLayout.tsx +++ b/frontend/src/pages/ukmesh/UKLayout.tsx @@ -20,7 +20,7 @@ export const UKLayout: React.FC = () => { showFeed showRepeaterSearch showCompanion - showRegions={false} + showRegions showTopology showSpam showAbout={false} diff --git a/frontend/src/pages/ukmesh/UKRepeaterSearchPage.tsx b/frontend/src/pages/ukmesh/UKRepeaterSearchPage.tsx index e895e11..db8fcb1 100644 --- a/frontend/src/pages/ukmesh/UKRepeaterSearchPage.tsx +++ b/frontend/src/pages/ukmesh/UKRepeaterSearchPage.tsx @@ -5,6 +5,7 @@ import { useRuntimeFeatures } from '../../config/runtimeFeatures.js'; import { fetchJson, withScopeParams } from '../../utils/api.js'; import { ScopedCache } from '../../utils/scopedCache.js'; import { Combobox, type ComboboxOption } from '../../components/ui/Combobox.js'; +import { ConfirmedNeighbours, timeAgo, type NodeLink } from '../../components/ConfirmedNeighbours.js'; interface MeshNode { node_id: string; @@ -21,15 +22,6 @@ interface MeshNode { elevation_m?: number; } -interface NodeLink { - peer_id: string; - peer_name: string | null; - observed_count: number; - itm_path_loss_db: number | null; - count_this_to_peer: number; - count_peer_to_this: number; -} - interface PacketHistory { time: string; packet_hash: string; @@ -164,14 +156,6 @@ async function loadCompleteNodeSnapshot( throw new Error('Repeater index exceeded its page budget'); } -function timeAgo(iso: string): string { - const secs = Math.floor((Date.now() - new Date(iso).getTime()) / 1000); - if (secs < 60) return `${secs}s ago`; - if (secs < 3600) return `${Math.floor(secs / 60)}m ago`; - if (secs < 86400) return `${Math.floor(secs / 3600)}h ago`; - return `${Math.floor(secs / 86400)}d ago`; -} - function predictNextAdvert(packets: T[]): { nextAdvert: Date; avgInterval: number; samples: number } | null { if (packets.length < 2) return null; @@ -630,35 +614,7 @@ export const UKRepeaterSearchPage: React.FC = () => { Confirmed Neighbours {links.length > 0 && {links.length}} - {links.length === 0 ? ( -

No neighbours found for this node.

- ) : ( - <> -

Nodes with confirmed two-way communication

-
- {links.map(link => ( -
-
- - {link.peer_name || `${link.peer_id.slice(0, 12)}...`} - - - {link.peer_id.slice(0, 16)}…{link.peer_id.slice(-8)} - -
-
- Seen {link.observed_count}× - {link.itm_path_loss_db !== null && ( - · {Math.round(link.itm_path_loss_db)} dB loss - )} - · TX: {link.count_this_to_peer} - · RX: {link.count_peer_to_this} -
-
- ))} -
- - )} +
diff --git a/frontend/src/pages/ukmesh/feed-page.css b/frontend/src/pages/ukmesh/feed-page.css index 293dbfc..0b2d21e 100644 --- a/frontend/src/pages/ukmesh/feed-page.css +++ b/frontend/src/pages/ukmesh/feed-page.css @@ -1173,3 +1173,30 @@ border-bottom: 1px solid var(--border); } } + +/* ── Message tags (tagger-worker / Jev) ────────────────────────────────────── */ +.uk-feed-packet-row__tags { + display: flex; + flex-wrap: wrap; + gap: 4px; + margin: 2px 0 0; +} +.msg-tag-chip { + display: inline-flex; + align-items: center; + padding: 0 6px; + border-radius: 999px; + border: 1px solid rgba(38, 198, 162, 0.45); + background: rgba(38, 198, 162, 0.12); + color: #9fe8d6; + font-size: 10px; + line-height: 16px; + letter-spacing: 0.04em; + text-transform: uppercase; + white-space: nowrap; +} +.feed-detail__tags { + display: flex; + flex-wrap: wrap; + gap: 6px; +} diff --git a/frontend/src/pages/ukmesh/feedModel.ts b/frontend/src/pages/ukmesh/feedModel.ts index edaeaf0..650d9f7 100644 --- a/frontend/src/pages/ukmesh/feedModel.ts +++ b/frontend/src/pages/ukmesh/feedModel.ts @@ -1,4 +1,5 @@ import type { AggregatedPacket, MeshNode } from '../../hooks/useNodes.js'; +import type { MessageTags, TagConfidence } from '../../hooks/packetFeed.js'; import { ScopedCache } from '../../utils/scopedCache.js'; import type { LazyPathNode, LazyPathResult } from './PacketDetailPanel.js'; @@ -21,6 +22,8 @@ export type FeedPacket = { tx_count?: number; summary?: string | null; path_hashes?: string[] | null; + tags?: MessageTags; + tag_confidence?: TagConfidence; }; export const TYPE_LABELS: Record = { @@ -217,6 +220,8 @@ export function aggregatedPacketToFeedPacket(packet: AggregatedPacket): FeedPack tx_count: packet.txCount, summary: packet.summary ?? null, path_hashes: (packet.path as string[] | undefined) ?? null, + tags: packet.tags, + tag_confidence: packet.tagConfidence, }; } @@ -251,7 +256,7 @@ export function mergeFeedPacketObservations(current: FeedPacket, next: FeedPacke ...(next.observer_iatas ?? []), next.iata, ]); - return { + const merged: FeedPacket = { ...latest, first_seen_time: firstSeenCandidates.length > 0 ? new Date(Math.min(...firstSeenCandidates)).toISOString() @@ -267,6 +272,13 @@ export function mergeFeedPacketObservations(current: FeedPacket, next: FeedPacke rx_count: Math.max(current.rx_count ?? 0, next.rx_count ?? 0), tx_count: Math.max(current.tx_count ?? 0, next.tx_count ?? 0), }; + // Tags only exist once the tagger has run; keep the key absent when untagged + // so feed-row shapes stay stable for callers and tests. + const tags = latest.tags ?? current.tags ?? next.tags; + if (tags !== undefined) merged.tags = tags; + const tagConfidence = latest.tag_confidence ?? current.tag_confidence ?? next.tag_confidence; + if (tagConfidence !== undefined) merged.tag_confidence = tagConfidence; + return merged; } export function packetObserverIatas(packet: FeedPacket, nodeMap: Map): string[] { diff --git a/frontend/src/styles/map-app.css b/frontend/src/styles/map-app.css index 2a092a2..03320d2 100644 --- a/frontend/src/styles/map-app.css +++ b/frontend/src/styles/map-app.css @@ -2220,3 +2220,28 @@ top: calc(48px + 56px + 8px); } } + +/* ── Message tags (tagger-worker / Jev) ────────────────────────────────────── */ +.msg-tag-chip { + display: inline-flex; + align-items: center; + padding: 0 6px; + border-radius: 999px; + border: 1px solid rgba(38, 198, 162, 0.45); + background: rgba(38, 198, 162, 0.12); + color: #9fe8d6; + font-size: 10px; + line-height: 16px; + letter-spacing: 0.04em; + text-transform: uppercase; + white-space: nowrap; +} +.packet-item__tag { + margin-left: 2px; +} +.packet-item__tags { + display: flex; + flex-wrap: wrap; + gap: 4px; + margin-top: 2px; +} diff --git a/frontend/test/e2e/accessibility.spec.ts b/frontend/test/e2e/accessibility.spec.ts index a3c4ca3..8da4ea0 100644 --- a/frontend/test/e2e/accessibility.spec.ts +++ b/frontend/test/e2e/accessibility.spec.ts @@ -1,4 +1,3 @@ -import { dashboardOrigin } from './localServers.js'; import { expect, test, type Page, type Route } from '@playwright/test'; import AxeBuilder from '@axe-core/playwright'; @@ -258,7 +257,7 @@ for (const route of publicRoutes) { test('dashboard dialogs trap focus, close with Escape, and restore the trigger', async ({ page }) => { await installFixtures(page, false); - await page.goto(`${dashboardOrigin}/`); + await page.goto('http://127.0.0.1:4174/'); const initialDialog = page.getByRole('dialog', { name: 'Data disclaimer' }); await expect(initialDialog).toBeVisible(); @@ -336,7 +335,7 @@ test('repeater search exposes keyboard selection and tolerates IME composition', test('reduced-motion preference removes repeating CSS animation', async ({ page }) => { await page.emulateMedia({ reducedMotion: 'reduce' }); await installFixtures(page); - await page.goto(`${dashboardOrigin}/`); + await page.goto('http://127.0.0.1:4174/'); const repeating = await page.evaluate(() => Array.from(document.querySelectorAll('*')) .filter((element) => { const style = getComputedStyle(element); @@ -349,7 +348,7 @@ test('reduced-motion preference removes repeating CSS animation', async ({ page test('keyboard-only dashboard flow selects a node, reads details, and toggles a packet row', async ({ page }) => { await installFixtures(page); await installDashboardWebSocket(page); - await page.goto(`${dashboardOrigin}/`); + await page.goto('http://127.0.0.1:4174/'); const packet = page.locator('.packet-item').filter({ hasText: 'Keyboard fixture message' }); await expect(packet).toHaveAttribute('aria-label', /Pin GRP packet/); diff --git a/frontend/test/e2e/dashboard.spec.ts b/frontend/test/e2e/dashboard.spec.ts index 932ada8..8f7ad19 100644 --- a/frontend/test/e2e/dashboard.spec.ts +++ b/frontend/test/e2e/dashboard.spec.ts @@ -1,10 +1,38 @@ -import { expect, test } from '@playwright/test'; +import { expect, test, type Page } from '@playwright/test'; import AxeBuilder from '@axe-core/playwright'; -import { installMapRoutes, installTerrainRoutes, RF_COVERAGE_TILE } from './mapFixtures.js'; + +const TEST_MAP_STYLE = { + version: 8, + sources: { + openmaptiles: { + type: 'vector', + url: 'https://tiles.openfreemap.org/planet', + }, + }, + glyphs: 'https://tiles.openfreemap.org/fonts/{fontstack}/{range}.pbf', + layers: [ + { id: 'bg', type: 'background', paint: { 'background-color': '#080d14' } }, + ], +}; + +async function installMapRoutes(page: Page, delayPlanetMetadata = false) { + await page.route('https://tiles.openfreemap.org/**', async (route) => { + const pathname = new URL(route.request().url()).pathname; + if (pathname === '/styles/dark' || pathname === '/styles/positron') { + await route.fulfill({ json: TEST_MAP_STYLE }); + return; + } + if (pathname === '/planet' && delayPlanetMetadata) { + await new Promise((resolve) => setTimeout(resolve, 1_500)); + await route.continue(); + return; + } + await route.abort(); + }); +} test.beforeEach(async ({ page }) => { await installMapRoutes(page); - await installTerrainRoutes(page); await page.route('**/*basemaps.cartocdn.com/**', () => { throw new Error('CARTO tiles should not be requested after OpenFreeMap migration'); }); @@ -53,8 +81,7 @@ test.beforeEach(async ({ page }) => { }); let metaPolls = 0; await page.route('**/rf-coverage/meta.json*', async (route) => { - // Two fixture stages; later polls retain the completed stage and tile revision. - metaPolls = Math.min(metaPolls + 1, 2); + metaPolls += 1; const tiles = [{ image: 'tiles/standard/0-0.png', bounds: { South: 49, North: 61, West: -9, East: 3 } }]; if (metaPolls > 1) tiles.push({ image: 'tiles/standard/0-1.png', bounds: { South: 49, North: 61, West: 3, East: 4 } }); await route.fulfill({ json: { @@ -99,7 +126,7 @@ test.beforeEach(async ({ page }) => { } })); await page.route('**/rf-coverage/tiles/**/*.png*', (route) => route.fulfill({ contentType: 'image/png', - body: RF_COVERAGE_TILE, + body: Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M/wHwAF/gL+AvnqWQAAAABJRU5ErkJggg==', 'base64'), })); await page.route('**/api/**', async (route) => { const path = new URL(route.request().url()).pathname; @@ -133,9 +160,6 @@ test('HopReach tiles arrive progressively while the 4,600-node map remains inter { timeout: 15_000 }, ).toBe(true); await expect(page.getByRole('button', { name: 'Precision' })).toBeVisible({ timeout: 8_000 }); - if (testInfo.project.name === 'dashboard-mobile') { - await page.getByRole('button', { name: 'Close menu' }).click(); - } await page.getByRole('button', { name: 'Precision' }).click(); await expect(page.getByRole('button', { name: 'Precision' })).toHaveAttribute('aria-pressed', 'true'); await expect.poll( @@ -171,19 +195,6 @@ test('enabled HopReach coverage survives metadata winning the initial map-load r }); test('RF coverage remains available with 3D terrain', async ({ page }, testInfo) => { - // Real DEM decoding and software 3D rendering take longer than rejected HTML tiles. - test.setTimeout(60_000); - const tileErrors: string[] = []; - page.on('console', (message) => { - if (message.text().startsWith('[terrain] source error:') - || message.text().startsWith('[rf-coverage] unable to load source tile')) { - tileErrors.push(message.text()); - } - }); - const firstTerrainTile = page.waitForResponse((response) => - new URL(response.url()).pathname.startsWith('/terrain-tiles/')); - const firstCoverageTile = page.waitForResponse((response) => - new URL(response.url()).pathname.startsWith('/rf-coverage/tiles/')); await page.goto('/?layers=feed%2Cterrain%2Ccoverage', { waitUntil: 'domcontentloaded' }); await page.getByText('Live Map', { exact: true }).waitFor({ state: 'visible', timeout: 15_000 }); if (testInfo.project.name === 'dashboard-mobile') { @@ -195,24 +206,9 @@ test('RF coverage remains available with 3D terrain', async ({ page }, testInfo) await expect(coverage).toHaveAttribute('aria-pressed', 'true'); await expect(terrain).toHaveAttribute('aria-pressed', 'true'); await expect(page).toHaveURL(/layers=[^&]*terrain[^&]*coverage/); - const tileResponses = await Promise.all([firstTerrainTile, firstCoverageTile]); - for (const response of tileResponses) { - expect(response.status()).toBe(200); - expect(response.headers()['content-type']).toContain('image/png'); - } - const tileSizes = await page.evaluate((urls) => Promise.all(urls.map(async (url) => { - const image = await createImageBitmap(await (await fetch(url)).blob()); - const size = { width: image.width, height: image.height }; - image.close(); - return size; - })), tileResponses.map((response) => response.url())); - expect(tileSizes).toEqual([{ width: 256, height: 256 }, { width: 256, height: 256 }]); - expect(tileErrors).toEqual([]); }); -test('map modes update layers and produce a shareable URL', async ({ page }, testInfo) => { - // Two 4,600-node bootstraps and an axe scan share this budget, including cold Vite compilation. - test.setTimeout(45_000); +test('map modes update layers and produce a shareable URL', async ({ page }) => { await page.goto('/'); await expect(page.getByText('Live Map', { exact: true })).toBeVisible({ timeout: 15_000 }); const mapArea = page.locator('.map-area'); @@ -223,9 +219,6 @@ test('map modes update layers and produce a shareable URL', async ({ page }, tes .filter((violation) => violation.impact === 'critical' || violation.impact === 'serious') .map((violation) => ({ id: violation.id, targets: violation.nodes.map((node) => node.target.join(' ')) }))).toEqual([]); - if (testInfo.project.name === 'dashboard-mobile') { - await page.getByRole('button', { name: 'Layers' }).first().click(); - } const diagnose = page.getByRole('button', { name: 'Diagnose' }).first(); await diagnose.click(); await expect(diagnose).toHaveAttribute('aria-pressed', 'true'); @@ -234,9 +227,5 @@ test('map modes update layers and produce a shareable URL', async ({ page }, tes await expect(page).toHaveURL(/layers=.*clashes/); await page.reload(); - await expect(page.getByText('Live Map', { exact: true })).toBeVisible({ timeout: 15_000 }); - if (testInfo.project.name === 'dashboard-mobile') { - await page.getByRole('button', { name: 'Layers' }).first().click(); - } await expect(page.getByRole('button', { name: 'Diagnose' }).first()).toHaveAttribute('aria-pressed', 'true'); }); diff --git a/frontend/test/e2e/mobile.spec.ts b/frontend/test/e2e/mobile.spec.ts index cddac63..49212fc 100644 --- a/frontend/test/e2e/mobile.spec.ts +++ b/frontend/test/e2e/mobile.spec.ts @@ -1,6 +1,4 @@ -import { publicOrigin, dashboardOrigin, devOrigin } from './localServers.js'; import { expect, test, type Page } from '@playwright/test'; -import { installMapRoutes, installTerrainRoutes } from './mapFixtures.js'; const PHONE = { width: 375, height: 667 }; @@ -77,8 +75,6 @@ const repeaterNodes = Array.from({ length: 12 }, (_, index) => ({ })); async function installApiFixtures(page: Page) { - await installMapRoutes(page); - await installTerrainRoutes(page); await page.addInitScript(() => { localStorage.setItem('meshcore-cookie-consent-v1', '1'); localStorage.setItem('meshcore-disclaimer-dismissed', '1'); @@ -151,16 +147,16 @@ async function expectNoViewportOverflow(page: Page) { } const routedViews = [ - ['UK home', `${publicOrigin}/`], - ['UK feed', `${publicOrigin}/feed`], - ['UK repeaters', `${publicOrigin}/repeater`], - ['UK companions', `${publicOrigin}/companion`], - ['UK install', `${publicOrigin}/install`], - ['UK open source', `${publicOrigin}/open-source`], - ['UK stats', `${publicOrigin}/stats`], - ['UK login', `${publicOrigin}/login`], - ['UK docs', `${publicOrigin}/docs`], - ['test site home', `${devOrigin}/`], + ['UK home', 'http://127.0.0.1:4173/'], + ['UK feed', 'http://127.0.0.1:4173/feed'], + ['UK repeaters', 'http://127.0.0.1:4173/repeater'], + ['UK companions', 'http://127.0.0.1:4173/companion'], + ['UK install', 'http://127.0.0.1:4173/install'], + ['UK open source', 'http://127.0.0.1:4173/open-source'], + ['UK stats', 'http://127.0.0.1:4173/stats'], + ['UK login', 'http://127.0.0.1:4173/login'], + ['UK docs', 'http://127.0.0.1:4173/docs'], + ['test site home', 'http://127.0.0.1:4175/'], ] as const; for (const [name, url] of routedViews) { @@ -180,7 +176,7 @@ for (const [name, url] of routedViews) { test('shared navigation stays compact and dismisses without navigation', async ({ page }) => { await page.setViewportSize(PHONE); await installApiFixtures(page); - await page.goto(`${publicOrigin}/feed`, { waitUntil: 'networkidle' }); + await page.goto('http://127.0.0.1:4173/feed', { waitUntil: 'networkidle' }); const openMenu = page.getByRole('button', { name: 'Open menu' }); await openMenu.click(); @@ -200,7 +196,7 @@ test('shared navigation stays compact and dismisses without navigation', async ( test('shared navigation collapses before tablet links can wrap', async ({ page }) => { await page.setViewportSize({ width: 768, height: 667 }); await installApiFixtures(page); - await page.goto(`${publicOrigin}/feed`, { waitUntil: 'networkidle' }); + await page.goto('http://127.0.0.1:4173/feed', { waitUntil: 'networkidle' }); const menu = page.locator('#site-navigation'); const openMenu = page.getByRole('button', { name: 'Open menu' }); @@ -216,7 +212,7 @@ test('shared navigation collapses before tablet links can wrap', async ({ page } test('phone home and feed give primary content the full available width', async ({ page }) => { await page.setViewportSize(PHONE); await installApiFixtures(page); - await page.goto(`${publicOrigin}/`, { waitUntil: 'networkidle' }); + await page.goto('http://127.0.0.1:4173/', { waitUntil: 'networkidle' }); const introBox = await page.locator('.site-home__intro').boundingBox(); const panelBox = await page.locator('.site-home__panel').boundingBox(); @@ -227,7 +223,7 @@ test('phone home and feed give primary content the full available width', async )).toBe(1); await expectNoViewportOverflow(page); - await page.goto(`${publicOrigin}/feed`, { waitUntil: 'networkidle' }); + await page.goto('http://127.0.0.1:4173/feed', { waitUntil: 'networkidle' }); const channelsBox = await page.locator('.uk-feed-channels').boundingBox(); const chatBox = await page.locator('.uk-feed-chat').boundingBox(); expect(channelsBox?.height ?? PHONE.height).toBeLessThan(64); @@ -239,14 +235,14 @@ test('phone home and feed give primary content the full available width', async test('tablet home and feed avoid squeezed desktop columns', async ({ page }) => { await page.setViewportSize({ width: 768, height: 667 }); await installApiFixtures(page); - await page.goto(`${publicOrigin}/`, { waitUntil: 'networkidle' }); + await page.goto('http://127.0.0.1:4173/', { waitUntil: 'networkidle' }); expect(await page.locator('.site-home__grid').evaluate( (element) => getComputedStyle(element).gridTemplateColumns.split(' ').length, )).toBe(1); expect((await page.locator('.site-home__intro').boundingBox())?.width ?? 0).toBeGreaterThan(720); - await page.goto(`${publicOrigin}/feed`, { waitUntil: 'networkidle' }); + await page.goto('http://127.0.0.1:4173/feed', { waitUntil: 'networkidle' }); expect((await page.locator('.uk-feed-chat').boundingBox())?.width ?? 0).toBeGreaterThan(740); await expect(page.locator('.uk-feed-right')).toBeHidden(); }); @@ -254,7 +250,7 @@ test('tablet home and feed avoid squeezed desktop columns', async ({ page }) => test('map controls and disclaimer leave the map usable on a phone', async ({ page }) => { await page.setViewportSize(PHONE); await installApiFixtures(page); - await page.goto(`${dashboardOrigin}/`, { waitUntil: 'networkidle' }); + await page.goto('http://127.0.0.1:4174/', { waitUntil: 'networkidle' }); await page.getByRole('button', { name: /Layers/ }).click(); const lineOfSight = page.getByRole('button', { name: 'Line of sight' }); @@ -278,7 +274,7 @@ test('map controls and disclaimer leave the map usable on a phone', async ({ pag test('mobile map menu floats over the map; feed hides behind the Live chip', async ({ page }) => { await page.setViewportSize(PHONE); await installApiFixtures(page); - await page.goto(`${dashboardOrigin}/`, { waitUntil: 'networkidle' }); + await page.goto('http://127.0.0.1:4174/', { waitUntil: 'networkidle' }); // Default state: compact floating bar, packet feed hidden. const controlsBox = await page.locator('.mobile-controls').boundingBox(); @@ -305,17 +301,17 @@ test('mobile map menu floats over the map; feed hides behind the Live chip', asy await expectNoViewportOverflow(page); }); -for (const width of [320, 375, 640, 768, 1024, 1280, 1440]) { - test(`live map stays inside the ${width}px viewport and remains UK-only`, async ({ page }) => { - await installApiFixtures(page); - const apiRequests: string[] = []; - page.on('request', (request) => { - const url = new URL(request.url()); - if (url.pathname.startsWith('/api/')) apiRequests.push(url.href); - }); +test('live map stays inside every supported viewport and remains UK-only', async ({ page }) => { + await installApiFixtures(page); + const apiRequests: string[] = []; + page.on('request', (request) => { + const url = new URL(request.url()); + if (url.pathname.startsWith('/api/')) apiRequests.push(url.href); + }); + for (const width of [320, 375, 640, 768, 1024, 1280, 1440]) { await page.setViewportSize({ width, height: width <= 375 ? 667 : 800 }); - await page.goto(`${dashboardOrigin}/?network=teesside`, { waitUntil: 'networkidle' }); + await page.goto('http://127.0.0.1:4174/?network=teesside', { waitUntil: 'networkidle' }); await expect(page.locator('.map-area')).toBeVisible(); await expect(page.getByLabel('Network region')).toHaveCount(0); @@ -333,16 +329,17 @@ for (const width of [320, 375, 640, 768, 1024, 1280, 1440]) { expect(controlsBox!.x + controlsBox!.width).toBeLessThanOrEqual(width + 0.5); } await expectNoViewportOverflow(page); - expect(apiRequests.some((url) => new URL(url).searchParams.get('network') === 'ukmesh')).toBe(true); - expect(apiRequests.some((url) => new URL(url).searchParams.get('network') === 'teesside')).toBe(false); - expect(apiRequests.some((url) => new URL(url).pathname === '/api/observers/health')).toBe(false); - }); -} + } + + expect(apiRequests.some((url) => new URL(url).searchParams.get('network') === 'ukmesh')).toBe(true); + expect(apiRequests.some((url) => new URL(url).searchParams.get('network') === 'teesside')).toBe(false); + expect(apiRequests.some((url) => new URL(url).pathname === '/api/observers/health')).toBe(false); +}); test('repeater search results stay bounded and scroll inside the menu', async ({ page }) => { await page.setViewportSize(PHONE); await installApiFixtures(page); - await page.goto(`${publicOrigin}/repeater`, { waitUntil: 'networkidle' }); + await page.goto('http://127.0.0.1:4173/repeater', { waitUntil: 'networkidle' }); await page.getByRole('combobox', { name: 'Search repeaters' }).fill('Repeater'); const results = page.locator('.repeater-search-box__results'); @@ -359,7 +356,7 @@ test('repeater search results stay bounded and scroll inside the menu', async ({ test('stats path modal fits the viewport and keeps its close action reachable', async ({ page }) => { await page.setViewportSize(PHONE); await installApiFixtures(page); - await page.goto(`${publicOrigin}/stats`, { waitUntil: 'networkidle' }); + await page.goto('http://127.0.0.1:4173/stats', { waitUntil: 'networkidle' }); await page.getByRole('tab', { name: 'Paths' }).click(); await page.getByRole('button', { name: 'North -> South' }).click(); @@ -373,7 +370,7 @@ test('stats path modal fits the viewport and keeps its close action reachable', test('repeater-tree modal remains centred and fully reachable at every target width', async ({ page }) => { await installApiFixtures(page); - await page.goto(`${publicOrigin}/feed`, { waitUntil: 'networkidle' }); + await page.goto('http://127.0.0.1:4173/feed', { waitUntil: 'networkidle' }); await page.evaluate(() => { const overlay = document.createElement('div'); @@ -402,7 +399,7 @@ test('repeater-tree modal remains centred and fully reachable at every target wi test('narrow 320px pages keep primary content inside the viewport', async ({ page }) => { await page.setViewportSize({ width: 320, height: 568 }); await installApiFixtures(page); - await page.goto(`${publicOrigin}/`, { waitUntil: 'networkidle' }); + await page.goto('http://127.0.0.1:4173/', { waitUntil: 'networkidle' }); await expectNoViewportOverflow(page); const introBox = await page.locator('.site-home__intro').boundingBox(); expect(introBox?.width ?? 320).toBeLessThan(320); diff --git a/frontend/test/e2e/owner.spec.ts b/frontend/test/e2e/owner.spec.ts index 6809d17..ce9b57f 100644 --- a/frontend/test/e2e/owner.spec.ts +++ b/frontend/test/e2e/owner.spec.ts @@ -1,8 +1,21 @@ -import { expect, test } from '@playwright/test'; -import { installMapRoutes } from './mapFixtures.js'; +import { expect, test, type Page } from '@playwright/test'; const NODE_ID = 'A'.repeat(64); +const TEST_MAP_STYLE = { + version: 8, + sources: { + openmaptiles: { + type: 'vector', + url: 'https://tiles.openfreemap.org/planet', + }, + }, + glyphs: 'https://tiles.openfreemap.org/fonts/{fontstack}/{range}.pbf', + layers: [ + { id: 'bg', type: 'background', paint: { 'background-color': '#080d14' } }, + ], +}; + const dashboard = { nodes: [{ node_id: NODE_ID, @@ -19,13 +32,17 @@ const dashboard = { }], }; -test.beforeEach(async ({ page }) => { - await installMapRoutes(page); - await page.route('**/api/runtime-config', (route) => route.fulfill({ json: { - version: 1, packetArcs: false, heatmap: false, privacyGeneration: 1, refreshAfterSeconds: 30, - } })); +async function installMapRoutes(page: Page) { + await page.route('https://tiles.openfreemap.org/**', async (route) => { + const pathname = new URL(route.request().url()).pathname; + if (pathname === '/styles/dark' || pathname === '/styles/positron') { + await route.fulfill({ json: TEST_MAP_STYLE }); + return; + } + await route.abort(); + }); await page.route('**/*basemaps.cartocdn.com/**', (route) => route.abort()); -}); +} test('session polling does not reset the repeater owner content', async ({ page }) => { let sessionRequests = 0; @@ -33,6 +50,7 @@ test('session polling does not reset the repeater owner content', async ({ page let liveRequestLimit: number | null = null; await page.clock.install({ time: new Date('2026-07-16T12:00:00Z') }); + await installMapRoutes(page); await page.route('**/api/owner/session', async (route) => { sessionRequests += 1; await route.fulfill({ json: { ok: true, dashboard, mqttUsername: 'alpha-owner' } }); @@ -65,7 +83,7 @@ test('session polling does not reset the repeater owner content', async ({ page await page.goto('/login'); const identities = page.getByLabel('Owned repeater identities'); - await expect(identities.getByText('Alpha Repeater', { exact: true })).toBeVisible({ timeout: 15_000 }); + await expect(identities.getByText('Alpha Repeater', { exact: true })).toBeVisible(); await expect(page.locator('.owner-section-tabs')).toHaveCount(0); await expect(page.locator('.owner-settings')).toHaveCount(0); @@ -100,6 +118,7 @@ test('owner map construction remains one across repeated live polls', async ({ p } }).observe(document, { childList: true, subtree: true }); }); + await installMapRoutes(page); await page.route('**/api/owner/session', (route) => route.fulfill({ json: { ok: true, dashboard, mqttUsername: 'alpha-owner' }, })); @@ -124,7 +143,7 @@ test('owner map construction remains one across repeated live polls', async ({ p await page.route('**/api/owner/live-last-hop?**', (route) => route.fulfill({ json: { points: [] } })); await page.goto('/login'); - await expect(page.getByRole('heading', { name: 'Direct Sender Map' })).toBeVisible({ timeout: 15_000 }); + await expect(page.getByRole('heading', { name: 'Direct Sender Map' })).toBeVisible(); await expect(page.getByRole('heading', { name: 'Node Telemetry' })).toBeVisible(); await expect(page.locator('.owner-section-tabs')).toHaveCount(0); await expect(page.locator('.owner-settings')).toHaveCount(0); diff --git a/frontend/vite.config.ts b/frontend/vite.config.ts index 67e471f..2fef6ae 100644 --- a/frontend/vite.config.ts +++ b/frontend/vite.config.ts @@ -1,4 +1,4 @@ -import { readFileSync } from 'node:fs'; +import { readFileSync, readdirSync } from 'node:fs'; import { fileURLToPath } from 'node:url'; import { join } from 'node:path'; import { defineConfig, type Plugin } from 'vite'; @@ -11,21 +11,22 @@ import viteSeoPlugin from './src/plugins/vite-seo.js'; * worker asset is ever emitted. At runtime the browser requests * /assets/maplibre-gl-worker.mjs (which imports ./maplibre-gl-shared.mjs), * both 404 (nginx SPA fallback serves HTML), and the map renders dead (no - * tiles, no node dots). Emit the production worker and its shared module so - * the runtime requests resolve. The main module is already bundled by Vite; - * the development builds are not runtime dependencies of this worker. + * tiles, no node dots). Emit every .mjs from the maplibre dist so the runtime + * requests resolve. */ function maplibreWorkerPlugin(): Plugin { return { name: 'maplibre-worker-emit', generateBundle() { const distDir = fileURLToPath(new URL('./node_modules/maplibre-gl/dist', import.meta.url)); - for (const file of ['maplibre-gl-worker.mjs', 'maplibre-gl-shared.mjs']) { - this.emitFile({ - type: 'asset', - fileName: `assets/${file}`, - source: readFileSync(join(distDir, file)), - }); + for (const file of readdirSync(distDir)) { + if (file.endsWith('.mjs') && !file.endsWith('.map')) { + this.emitFile({ + type: 'asset', + fileName: `assets/${file}`, + source: readFileSync(join(distDir, file)), + }); + } } }, }; diff --git a/tagger-worker/Dockerfile b/tagger-worker/Dockerfile new file mode 100644 index 0000000..1887f14 --- /dev/null +++ b/tagger-worker/Dockerfile @@ -0,0 +1,8 @@ +FROM python:3.12-slim +RUN pip install --no-cache-dir psycopg2-binary==2.9.10 +WORKDIR /app +COPY tagger_worker.py /app/tagger_worker.py +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 +RUN useradd -m -u 10001 tagger && chown -R tagger:tagger /app +USER tagger +CMD ["python3", "-u", "/app/tagger_worker.py"] diff --git a/tagger-worker/tagger_worker.py b/tagger-worker/tagger_worker.py new file mode 100644 index 0000000..ee5c548 --- /dev/null +++ b/tagger-worker/tagger_worker.py @@ -0,0 +1,347 @@ +#!/usr/bin/env python3 +"""UKMesh realtime message tagger - TypeSafe/Jev. + +Polls the packets table for new public-channel messages, asks Jev to tag each +one (speaker / kind / topic + flags), and stores the result in message_tags. +Tags are displayed on the public feed and app.ukmesh.com detail panels; private +or out-of-scope packets are never sent to the API (public visibility predicate + +network scope applied in the query). Monitoring data lives in tagger_state +(heartbeat + counters). + +Failure handling: a message that fails all inline API attempts is persisted to +tagger_retry and retried on every subsequent poll (up to TAGGER_MAX_RETRY_ATTEMPTS) +so a TypeSafe outage cannot silently drop tags (2026-09-17 hardening; the old +20s overlap window lost 6 messages during two 503 bursts). +""" +import json +import os +import signal +import sys +import time +import urllib.error +import urllib.request + +import psycopg2 + +API_URL = os.environ.get('TYPESAFE_API_URL', 'https://api.typesafe.ai/v1/systemone') +API_KEY = os.environ.get('TYPESAFE_API_KEY', '').strip() +DATABASE_URL = os.environ['DATABASE_URL'] +POLL_S = float(os.environ.get('TAGGER_POLL_S', '5')) +CALL_FLOOR_S = float(os.environ.get('TAGGER_CALL_FLOOR_S', '0.5')) +BATCH_LIMIT = int(os.environ.get('TAGGER_BATCH_LIMIT', '200')) +OVERLAP_S = int(os.environ.get('TAGGER_OVERLAP_S', '120')) +MAX_RETRY_ATTEMPTS = int(os.environ.get('TAGGER_MAX_RETRY_ATTEMPTS', '30')) +RETRY_BATCH = int(os.environ.get('TAGGER_RETRY_BATCH', '20')) +MODEL = os.environ.get('TYPESAFE_MODEL', 'jev-latest') +START = os.environ.get('TAGGER_START', '').strip() +NETWORKS = [n.strip() for n in os.environ.get('TAGGER_NETWORKS', 'ukmesh,northeast,teesside').split(',') if n.strip()] + +QUESTIONS = { + "speaker": {"type": "choice", "instructions": "Who most likely sent this message?", + "criteria": {"human": "A human operator typing a message", + "bot": "An automated bot or script", + "room-server": "A room server, repeater, or bridge auto-response", + "unclear": "Cannot tell from the message"}}, + "kind": {"type": "choice", "instructions": "What is the primary kind of this message?", + "criteria": {"chat": "Social conversation, greeting, banter, or sharing (links, pictures, files, images)", + "radio-check": "Checking whether/how their signal is received", + "test-ping": "A test or ping", + "ack": "Acknowledgement of a previous signal", + "position": "Sharing their whereabouts or movements (e.g. 'I am near X', 'heading to Y', 'going off air for a bit') - not announcements about infrastructure or network status", + "coverage": "Reporting range, hops, or propagation", + "help": "Asking for help or information from others", + "noise": "No meaningful content (gibberish, stray characters, emoji only)", + "spam": "Only unsolicited promotion, scams, or abuse. IMPORTANT: a bare link, picture, or file with no promotional text is NEVER spam (mesh users share links/images routinely); when choosing between spam and chat for a bare link, choose chat", + "automation": "Automated telemetry, home-automation, or sensor notification", + "command": "A command to a system or bot (e.g. !test, !path, !help)"}}, + "topic": {"type": "choice", "instructions": "What is this message mainly about?", + "criteria": {"weather": "Weather or conditions", + "meetups": "Meetups, events, or social plans", + "travel": "Travel, journeys, or being on the move", + "gear": "Radios, antennas, equipment, or tech gear", + "network": "Network coverage, repeaters, hops, or connectivity", + "help": "Getting help, support, or answering questions", + "humour": "Jokes, banter, or humour", + "general": "General chatter that fits nothing else"}}, + "mentions_location": {"type": "noul", "instructions": "Does the message mention a geographic place (town, area, or landmark)?"}, + "directed": {"type": "noul", "instructions": "Is the message addressed to a specific person or node (by name or @mention)?"}, + "safety": {"type": "noul", "instructions": "Is this message safety or emergency related?"}, +} + +DDL = """ +CREATE TABLE IF NOT EXISTS message_tags ( + packet_hash text PRIMARY KEY, + packet_time timestamptz NOT NULL, + tagged_at timestamptz NOT NULL DEFAULT now(), + model text, + latency_ms integer, + tags jsonb NOT NULL, + confidence jsonb +); +CREATE TABLE IF NOT EXISTS tagger_state ( + id int PRIMARY KEY CHECK (id = 1), + cursor_time timestamptz, + seen bigint NOT NULL DEFAULT 0, + tagged bigint NOT NULL DEFAULT 0, + errors bigint NOT NULL DEFAULT 0, + http_429 bigint NOT NULL DEFAULT 0, + last_error text, + avg_latency_ms_50 real, + started_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz +); +INSERT INTO tagger_state (id) VALUES (1) ON CONFLICT DO NOTHING; +ALTER TABLE tagger_state ADD COLUMN IF NOT EXISTS retried bigint NOT NULL DEFAULT 0; +ALTER TABLE tagger_state ADD COLUMN IF NOT EXISTS given_up bigint NOT NULL DEFAULT 0; +CREATE TABLE IF NOT EXISTS tagger_retry ( + packet_hash text PRIMARY KEY, + packet_time timestamptz NOT NULL, + attempts int NOT NULL DEFAULT 0, + last_error text, + first_failed_at timestamptz NOT NULL DEFAULT now(), + exhausted boolean NOT NULL DEFAULT false +); +""" + +_running = True + + +def _stop(*_a): + global _running + _running = False + + +signal.signal(signal.SIGTERM, _stop) +signal.signal(signal.SIGINT, _stop) + + +def log(msg): + print(f"[tagger] {msg}", flush=True) + + +def api_tag(state): + payload = {'state': state, 'model': MODEL, 'questions': QUESTIONS} + req = urllib.request.Request(API_URL, data=json.dumps(payload).encode(), + headers={'Authorization': f'Bearer {API_KEY}', + 'Content-Type': 'application/json'}) + t0 = time.time() + with urllib.request.urlopen(req, timeout=90) as r: + resp = json.loads(r.read().decode()) + return resp, (time.time() - t0) * 1000.0 + + +def flatten(answers): + tags, conf = {}, {} + for k, v in (answers or {}).items(): + if not isinstance(v, dict): + continue + if 'choice' in v: + tags[k] = v.get('choice') + elif 'noul' in v: + tags[k] = v.get('noul') + elif 'score' in v: + tags[k] = v.get('score') + if 'confidence' in v: + conf[k] = v.get('confidence') + return tags, conf + + +def tag_message(cur, ph, ts): + """Tag one message with up to 3 inline attempts. Returns (ok, latency_ms, err).""" + cur.execute(""" + SELECT payload->'decrypted'->>'sender', payload->'decrypted'->>'message' + FROM packets WHERE packet_hash=%s LIMIT 1 + """, (ph,)) + row = cur.fetchone() + if row is None or row[1] is None: + return True, None, None # nothing taggable (row vanished / no message) + sender, message = row + state = f"Message from node '{sender or 'unknown'}': \"{(message or '')[:4000]}\"" + for attempt in (1, 2, 3): + try: + resp, ms = api_tag(state) + tags, conf = flatten(resp.get('answers')) + cur.execute("""INSERT INTO message_tags + (packet_hash, packet_time, tagged_at, model, latency_ms, tags, confidence) + VALUES (%s, %s, now(), %s, %s, %s, %s) + ON CONFLICT (packet_hash) DO NOTHING""", + (ph, ts, MODEL, int(ms), json.dumps(tags), json.dumps(conf))) + return True, ms, None + except urllib.error.HTTPError as e: + if e.code == 429: + cur.execute("UPDATE tagger_state SET http_429=http_429+1 WHERE id=1") + time.sleep(min(CALL_FLOOR_S * (2 ** attempt), 30)) + continue + err = f'http {e.code}: {e.read().decode()[:160]}' + time.sleep(2 * attempt) + except Exception as ex: # noqa: BLE001 + err = str(ex)[:200] + time.sleep(2 * attempt) + return False, None, err + + +def process_retries(cur): + """Retry previously failed messages. Returns (ok_count, fail_count, pending).""" + cur.execute(""" + SELECT packet_hash, packet_time, attempts FROM tagger_retry + WHERE NOT exhausted AND attempts < %s + ORDER BY first_failed_at ASC LIMIT %s + """, (MAX_RETRY_ATTEMPTS, RETRY_BATCH)) + due = cur.fetchall() + ok = fail = 0 + for ph, ts, attempts in due: + cur.execute("SELECT 1 FROM message_tags WHERE packet_hash=%s", (ph,)) + if cur.fetchone(): + cur.execute("DELETE FROM tagger_retry WHERE packet_hash=%s", (ph,)) + continue + good, ms, err = tag_message(cur, ph, ts) + if good: + cur.execute("DELETE FROM tagger_retry WHERE packet_hash=%s", (ph,)) + cur.execute("UPDATE tagger_state SET retried=retried+1 WHERE id=1") + ok += 1 + else: + fail += 1 + if attempts + 1 >= MAX_RETRY_ATTEMPTS: + cur.execute("""UPDATE tagger_retry SET attempts=attempts+1, last_error=%s, + exhausted=true WHERE packet_hash=%s""", (err, ph)) + cur.execute("UPDATE tagger_state SET given_up=given_up+1 WHERE id=1") + else: + cur.execute("""UPDATE tagger_retry SET attempts=attempts+1, last_error=%s + WHERE packet_hash=%s""", (err, ph)) + time.sleep(CALL_FLOOR_S) + cur.execute("SELECT count(*) FROM tagger_retry WHERE NOT exhausted") + pending = cur.fetchone()[0] + return ok, fail, pending + + +def main(): + if not API_KEY: + log('FATAL: TYPESAFE_API_KEY is empty') + sys.exit(2) + conn = psycopg2.connect(DATABASE_URL) + conn.autocommit = True + cur = conn.cursor() + cur.execute(DDL) + cur.execute("SELECT cursor_time FROM tagger_state WHERE id=1") + cursor = cur.fetchone()[0] + if cursor is None: + if START: + cur.execute("UPDATE tagger_state SET cursor_time=%s::timestamptz WHERE id=1", (START,)) + else: + cur.execute("UPDATE tagger_state SET cursor_time=now() - interval '5 minutes' WHERE id=1") + cur.execute("SELECT cursor_time FROM tagger_state WHERE id=1") + cursor = cur.fetchone()[0] + log(f"started; model={MODEL} poll={POLL_S}s floor={CALL_FLOOR_S}s overlap={OVERLAP_S}s " + f"networks={','.join(NETWORKS)} cursor={cursor.isoformat()}") + + lat_hist = [] + last_log = 0.0 + while _running: + loop_start = time.time() + try: + retry_ok, retry_fail, retry_pending = process_retries(cur) + + cur.execute(""" + SELECT * FROM ( + SELECT DISTINCT ON (packet_hash) + packet_hash, time, + payload->'decrypted'->>'sender' AS sender, + payload->'decrypted'->>'message' AS message + FROM packets + WHERE packet_type = 5 + AND payload->'decrypted'->>'message' IS NOT NULL + AND time > %s::timestamptz - make_interval(secs => %s) + AND time <= now() + AND network = ANY(%s::text[]) + AND visibility_ok IS TRUE + AND is_private IS NOT TRUE + AND EXISTS ( + SELECT 1 + FROM packet_visibility_materialization_state cached_visibility + JOIN public_visibility_state current_visibility + ON current_visibility.singleton = cached_visibility.singleton + WHERE cached_visibility.singleton = TRUE + AND cached_visibility.visibility_generation = current_visibility.generation + ) + ORDER BY packet_hash, time DESC + ) x ORDER BY time ASC LIMIT %s + """, (cursor, OVERLAP_S, NETWORKS, BATCH_LIMIT)) + rows = cur.fetchall() + new = errs = 0 + max_t = cursor + for ph, ts, sender, message in rows: + if ts > max_t: + max_t = ts + cur.execute("SELECT 1 FROM message_tags WHERE packet_hash=%s", (ph,)) + if cur.fetchone(): + continue + state = f"Message from node '{sender or 'unknown'}': \"{(message or '')[:4000]}\"" + ans, ms, err = None, None, None + for attempt in (1, 2, 3): + try: + resp, ms = api_tag(state) + ans = resp.get('answers') + break + except urllib.error.HTTPError as e: + if e.code == 429: + cur.execute("UPDATE tagger_state SET http_429=http_429+1 WHERE id=1") + time.sleep(min(CALL_FLOOR_S * (2 ** attempt), 30)) + continue + err = f'http {e.code}: {e.read().decode()[:160]}' + time.sleep(2 * attempt) + except Exception as ex: # noqa: BLE001 + err = str(ex)[:200] + time.sleep(2 * attempt) + if ans: + tags, conf = flatten(ans) + cur.execute("""INSERT INTO message_tags + (packet_hash, packet_time, tagged_at, model, latency_ms, tags, confidence) + VALUES (%s, %s, now(), %s, %s, %s, %s) + ON CONFLICT (packet_hash) DO NOTHING""", + (ph, ts, MODEL, int(ms), json.dumps(tags), json.dumps(conf))) + new += 1 + lat_hist.append(ms) + lat_hist = lat_hist[-50:] + else: + errs += 1 + cur.execute("UPDATE tagger_state SET errors=errors+1, last_error=%s WHERE id=1", (err,)) + # Persist for retry instead of letting the cursor move past it. + cur.execute("""INSERT INTO tagger_retry (packet_hash, packet_time, attempts, last_error) + VALUES (%s, %s, 1, %s) + ON CONFLICT (packet_hash) DO UPDATE + SET attempts=tagger_retry.attempts+1, last_error=EXCLUDED.last_error""", + (ph, ts, err)) + time.sleep(CALL_FLOOR_S) + if max_t and max_t != cursor: + cur.execute("UPDATE tagger_state SET cursor_time=%s WHERE id=1", (max_t,)) + cursor = max_t + avg = (sum(lat_hist) / len(lat_hist)) if lat_hist else None + cur.execute("""UPDATE tagger_state SET seen=seen+%s, tagged=tagged+%s, + avg_latency_ms_50=%s, updated_at=now() WHERE id=1""", + (new + errs, new, avg)) + if new or errs or retry_ok or retry_fail or (time.time() - last_log) > 120: + log(f"batch={len(rows)} new={new} errs={errs} retry_ok={retry_ok} " + f"retry_pending={retry_pending} avg50={avg and round(avg)}ms cursor={cursor.isoformat()}") + last_log = time.time() + if new + errs == 0 and retry_ok + retry_fail == 0: + time.sleep(POLL_S) + except Exception as ex: # noqa: BLE001 + log(f"loop error: {ex}") + try: + conn.rollback() + except Exception: # noqa: BLE001 + pass + time.sleep(POLL_S) + try: + conn = psycopg2.connect(DATABASE_URL) + conn.autocommit = True + cur = conn.cursor() + except Exception as ex2: # noqa: BLE001 + log(f"reconnect failed: {ex2}") + time.sleep(15) + _ = loop_start + log('shutting down') + + +if __name__ == '__main__': + main() diff --git a/viewshed-worker/tests/test_side_effect_markers.py b/viewshed-worker/tests/test_side_effect_markers.py index 12c2846..69766cd 100644 --- a/viewshed-worker/tests/test_side_effect_markers.py +++ b/viewshed-worker/tests/test_side_effect_markers.py @@ -25,23 +25,14 @@ class FakeRedis: def __init__(self): self.data = {} self.published = [] - self.publish_count = 0 - self.fail_publish_at = None - self.fail_marker_set = False def exists(self, key): return 1 if key in self.data else 0 def set(self, key, value, ex=None): - if self.fail_marker_set and key.startswith('viewshed:side-effects:'): - self.fail_marker_set = False - raise RuntimeError('injected completion-marker failure') self.data[key] = value def publish(self, channel, message): - self.publish_count += 1 - if self.publish_count == self.fail_publish_at: - raise RuntimeError('injected notification failure') self.published.append((channel, json.loads(message))) @@ -76,15 +67,6 @@ class FakeDb: return FakeCursor(self.rows) -class SequenceDb: - """Return one row set per cursor so process_job retry paths are realistic.""" - def __init__(self, cursor_rows): - self.cursor_rows = iter(cursor_rows) - - def cursor(self): - return FakeCursor(next(self.cursor_rows)) - - class SideEffectMarkerTest(unittest.TestCase): def test_marker_round_trip(self): r = FakeRedis() @@ -138,56 +120,6 @@ class SideEffectMarkerTest(unittest.TestCase): self.assertEqual(r.published, []) self.assertFalse(worker.side_effects_complete(r, 'E' * 64)) - def test_post_commit_faults_replay_on_retry_at_every_boundary(self): - """Coverage is already committed; any incomplete Redis effect must retry.""" - node = 'F' * 64 - geom = {'type': 'Polygon', 'coordinates': []} - strength = {'s1': {'type': 'Polygon', 'coordinates': []}} - - for boundary in ('link_admission', 'coverage_notification', 'node_notification', 'marker'): - with self.subTest(boundary=boundary): - r = FakeRedis() - r.fail_publish_at = { - 'coverage_notification': 1, - 'node_notification': 2, - }.get(boundary) - r.fail_marker_set = boundary == 'marker' - db = SequenceDb([ - [(None, 2, None)], # first delivery: node eligibility - [(geom, strength, 5_000.0, 12.0)], # replay: coverage row - [(54.0, -1.5)], # replay: node position - [(None, 2, None)], # retry: node eligibility - [(geom, strength, 5_000.0, 12.0)], # replay: coverage row - [(54.0, -1.5)], # replay: node position - ]) - enqueue_calls = 0 - - def enqueue(*args, **kwargs): - nonlocal enqueue_calls - enqueue_calls += 1 - if boundary == 'link_admission' and enqueue_calls == 1: - raise RuntimeError('injected link admission failure') - return 1 - - job = {'node_id': node, 'lat': 54.0, 'lon': -1.5} - with mock.patch.object(worker, 'already_calculated', return_value=True), \ - mock.patch.object(worker, 'WORKER_MODE', 'all'), \ - mock.patch.object(worker, 'enqueue_physical_link_jobs_for_node', side_effect=enqueue): - with self.assertRaises(RuntimeError): - worker.process_job(db, r, job) - self.assertFalse(worker.side_effects_complete(r, node)) - - # The next delivery takes the already-calculated branch and - # reconciles the missing side effects before returning. - worker.process_job(db, r, job) - - self.assertTrue(worker.side_effects_complete(r, node)) - expected_notifications = { - 'node_notification': 3, - 'marker': 4, - }.get(boundary, 2) - self.assertEqual(len(r.published), expected_notifications) - if __name__ == '__main__': unittest.main()
IDRSSISNRLast seen
ID{directPackets ? 'Avg RSSI' : 'RSSI'}{directPackets ? 'Avg SNR' : 'SNR'}Last seen