From f438411a27e35fc2415cff59c7a9bddb23239d1d Mon Sep 17 00:00:00 2001 From: Kpa-clawbot Date: Sun, 29 Mar 2026 15:53:51 -0700 Subject: [PATCH] chore: remove deprecated Node.js backend (-11,291 lines) (#265) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Removes all deprecated Node.js backend server code. The Go server (`cmd/server/`) has been the production backend — the Node.js server was kept "just in case" but is no longer needed. ### Removed (19 files, -11,291 lines) **Backend server (6 files):** `server.js`, `db.js`, `decoder.js`, `server-helpers.js`, `packet-store.js`, `iata-coords.js` **Backend tests (9 files):** `test-decoder.js`, `test-decoder-spec.js`, `test-server-helpers.js`, `test-server-routes.js`, `test-packet-store.js`, `test-db.js`, `test-db-migration.js`, `test-regional-filter.js`, `test-regional-integration.js` **Backend tooling (4 files):** `tools/e2e-test.js`, `tools/frontend-test.js`, `benchmark.js`, `benchmark-ab.sh` ### Updated - `AGENTS.md` — Rewritten architecture section for Go, explicit deprecation warnings - `test-all.sh` — Only runs frontend tests - `package.json` — Updated test:unit - `scripts/validate.sh` — Removed Node.js server syntax check - `docker/supervisord.conf` — Points to Go binary ### NOT touched - `public/` (active frontend) ✅ - `test-e2e-playwright.js` (frontend E2E tests) ✅ - Frontend test files (`test-packet-filter.js`, `test-aging.js`, `test-frontend-helpers.js`) ✅ - `package.json` / Playwright deps ✅ ### Follow-up - Server-only npm deps (express, better-sqlite3, mqtt, ws, supertest) can be cleaned from package.json separately - `Dockerfile.node` can be removed separately --------- Co-authored-by: you --- AGENTS.md | 34 +- benchmark-ab.sh | 131 -- benchmark.js | 246 --- db.js | 935 ----------- decoder.js | 439 ----- docker/supervisord.conf | 2 +- iata-coords.js | 90 - package.json | 2 +- packet-store.js | 752 --------- scripts/combined-coverage.sh | 31 +- scripts/validate.sh | 1 - server-helpers.js | 323 ---- server.js | 3067 ---------------------------------- test-all.sh | 17 +- test-db-migration.js | 321 ---- test-db.js | 512 ------ test-decoder-spec.js | 625 ------- test-decoder.js | 630 ------- test-packet-store.js | 552 ------ test-regional-filter.js | 135 -- test-regional-integration.js | 96 -- test-server-helpers.js | 319 ---- test-server-routes.js | 1279 -------------- tools/e2e-test.js | 476 ------ tools/frontend-test.js | 320 ---- 25 files changed, 34 insertions(+), 11301 deletions(-) delete mode 100755 benchmark-ab.sh delete mode 100644 benchmark.js delete mode 100644 db.js delete mode 100644 decoder.js delete mode 100644 iata-coords.js delete mode 100644 packet-store.js delete mode 100644 server-helpers.js delete mode 100644 server.js delete mode 100644 test-db-migration.js delete mode 100644 test-db.js delete mode 100644 test-decoder-spec.js delete mode 100644 test-decoder.js delete mode 100644 test-packet-store.js delete mode 100644 test-regional-filter.js delete mode 100644 test-regional-integration.js delete mode 100644 test-server-helpers.js delete mode 100644 test-server-routes.js delete mode 100644 tools/e2e-test.js delete mode 100644 tools/frontend-test.js diff --git a/AGENTS.md b/AGENTS.md index 4698c413..a0faa808 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,14 +4,20 @@ Guide for AI agents working on this codebase. Read this before writing any code. ## Architecture -Single Node.js server + static frontend. No build step. No framework. No bundler. +Go backend + static frontend. No build step. No framework. No bundler. + +**⚠️ The Node.js server (server.js) is DEPRECATED and has been removed. All backend code is in Go.** +**⚠️ DO NOT create or modify any Node.js server files. All backend changes go in `cmd/server/` or `cmd/ingestor/`.** ``` -server.js — Express API + MQTT ingestion + WebSocket broadcast -decoder.js — MeshCore packet parser (header, path, payload, adverts) -packet-store.js — In-memory packet store + query engine (backed by SQLite) -db.js — SQLite schema + prepared statements -public/ — Frontend (vanilla JS, one file per page) +cmd/server/ — Go API server (REST + WebSocket broadcast + static file serving) + main.go — Entry point, flags, SPA handler + routes.go — All /api/* endpoints + store.go — In-memory packet store + analytics + SQLite queries + config.go — Configuration loading + decoder.go — MeshCore packet decoder +cmd/ingestor/ — Go MQTT ingestor (separate binary, writes to shared SQLite DB) +public/ — Frontend (vanilla JS, one file per page) — ACTIVE, NOT DEPRECATED app.js — SPA router, shared globals, theme loading roles.js — ROLE_COLORS, TYPE_COLORS, health thresholds, shared helpers nodes.js — Nodes list + side pane + full detail page @@ -28,17 +34,25 @@ public/ — Frontend (vanilla JS, one file per page) live.css — Live page styles home.css — Home page styles index.html — SPA shell, script/style tags with cache busters +test-fixtures/ — Real data SQLite fixture from staging (used for E2E tests) +scripts/ — Tooling (coverage collector, fixture capture, frontend instrumentation) ``` ### Data Flow -1. MQTT brokers → server.js ingests packets → decoder.js parses → packet-store.js stores in memory + SQLite -2. WebSocket broadcasts new packets to connected browsers -3. Frontend fetches via REST API, filters/sorts client-side +1. MQTT brokers → Go ingestor (`cmd/ingestor/`) ingests packets → decodes → writes to SQLite +2. Go server (`cmd/server/`) polls SQLite for new packets, broadcasts via WebSocket +3. Frontend fetches via REST API (`/api/*`), filters/sorts client-side + +### What's Deprecated (DO NOT TOUCH) +The following were part of the old Node.js backend and have been removed: +- `server.js`, `db.js`, `decoder.js`, `server-helpers.js`, `packet-store.js`, `iata-coords.js` +- All `test-server-*.js`, `test-decoder*.js`, `test-db*.js`, `test-regional*.js` files +- If you see references to these in comments or docs, they're stale — ignore them ## Rules — Read These First ### 1. No commit without tests -Every change that touches logic MUST have unit tests. Run `node test-packet-filter.js && node test-aging.js` before pushing. If you add new logic, add tests to the appropriate test file or create a new one. No exceptions. +Every change that touches logic MUST have tests. For Go backend: `cd cmd/server && go test ./...` and `cd cmd/ingestor && go test ./...`. For frontend: `node test-packet-filter.js && node test-aging.js && node test-frontend-helpers.js`. If you add new logic, add tests. No exceptions. ### 2. No commit without browser validation After pushing, verify the change works in an actual browser. Use `browser profile=openclaw` against the running instance. Take a screenshot if the change is visual. If you can't validate it, say so — don't claim it works. diff --git a/benchmark-ab.sh b/benchmark-ab.sh deleted file mode 100755 index 8d972614..00000000 --- a/benchmark-ab.sh +++ /dev/null @@ -1,131 +0,0 @@ -#!/bin/bash -# A/B benchmark: old (pre-perf) vs new (current) -# Usage: ./benchmark-ab.sh -set -e - -PORT_OLD=13003 -PORT_NEW=13004 -RUNS=3 -DB_PATH="$(pwd)/data/meshcore.db" - -OLD_COMMIT="23caae4" -NEW_COMMIT="$(git rev-parse HEAD)" - -echo "═══════════════════════════════════════════════════════" -echo " A/B Benchmark: Pre-optimization vs Current" -echo "═══════════════════════════════════════════════════════" -echo "OLD: $OLD_COMMIT (v2.0.1 — before any perf work)" -echo "NEW: $NEW_COMMIT (current)" -echo "Runs per endpoint: $RUNS" -echo "" - -# Get a real node pubkey for testing -ORIG_DIR="$(pwd)" -PUBKEY=$(sqlite3 "$DB_PATH" "SELECT public_key FROM nodes ORDER BY last_seen DESC LIMIT 1") -echo "Test node: ${PUBKEY:0:16}..." -echo "" - -# Setup old version in temp dir -OLD_DIR=$(mktemp -d) -echo "Cloning old version to $OLD_DIR..." -git worktree add "$OLD_DIR" "$OLD_COMMIT" --quiet 2>/dev/null || { - git worktree add "$OLD_DIR" "$OLD_COMMIT" --detach --quiet -} -# Copy config + db symlink -# Copy config + db + share node_modules -cp config.json "$OLD_DIR/" -mkdir -p "$OLD_DIR/data" -cp "$ORIG_DIR/data/meshcore.db" "$OLD_DIR/data/meshcore.db" -ln -sf "$ORIG_DIR/node_modules" "$OLD_DIR/node_modules" - -ENDPOINTS=( - "Stats|/api/stats" - "Packets(50)|/api/packets?limit=50" - "PacketsGrouped|/api/packets?limit=50&groupByHash=true" - "NodesList|/api/nodes?limit=50" - "NodeDetail|/api/nodes/$PUBKEY" - "NodeHealth|/api/nodes/$PUBKEY/health" - "NodeAnalytics|/api/nodes/$PUBKEY/analytics?days=7" - "BulkHealth|/api/nodes/bulk-health?limit=50" - "NetworkStatus|/api/nodes/network-status" - "Channels|/api/channels" - "Observers|/api/observers" - "RF|/api/analytics/rf" - "Topology|/api/analytics/topology" - "ChannelAnalytics|/api/analytics/channels" - "HashSizes|/api/analytics/hash-sizes" -) - -bench_endpoint() { - local port=$1 path=$2 runs=$3 nocache=$4 - local total=0 - for i in $(seq 1 $runs); do - local url="http://127.0.0.1:$port$path" - if [ "$nocache" = "1" ]; then - if echo "$path" | grep -q '?'; then - url="${url}&nocache=1" - else - url="${url}?nocache=1" - fi - fi - local ms=$(curl -s -o /dev/null -w "%{time_total}" "$url" 2>/dev/null) - local ms_int=$(echo "$ms * 1000" | bc | cut -d. -f1) - total=$((total + ms_int)) - done - echo $((total / runs)) -} - -# Launch old server -echo "Starting OLD server (port $PORT_OLD)..." -cd "$OLD_DIR" -PORT=$PORT_OLD node server.js &>/dev/null & -OLD_PID=$! -cd - >/dev/null - -# Launch new server -echo "Starting NEW server (port $PORT_NEW)..." -PORT=$PORT_NEW node server.js &>/dev/null & -NEW_PID=$! - -# Wait for both -sleep 12 # old server has no memory store; new needs prewarm - -# Verify -curl -s "http://127.0.0.1:$PORT_OLD/api/stats" >/dev/null 2>&1 || { echo "OLD server failed to start"; kill $OLD_PID $NEW_PID 2>/dev/null; exit 1; } -curl -s "http://127.0.0.1:$PORT_NEW/api/stats" >/dev/null 2>&1 || { echo "NEW server failed to start"; kill $OLD_PID $NEW_PID 2>/dev/null; exit 1; } - -echo "" -echo "Warming up caches on new server..." -for ep in "${ENDPOINTS[@]}"; do - path="${ep#*|}" - curl -s -o /dev/null "http://127.0.0.1:$PORT_NEW$path" 2>/dev/null -done -sleep 2 - -printf "\n%-22s %9s %9s %9s %9s\n" "Endpoint" "Old(ms)" "New-cold" "New-cache" "Speedup" -printf "%-22s %9s %9s %9s %9s\n" "──────────────────────" "─────────" "─────────" "─────────" "─────────" - -for ep in "${ENDPOINTS[@]}"; do - name="${ep%%|*}" - path="${ep#*|}" - - old_ms=$(bench_endpoint $PORT_OLD "$path" $RUNS 0) - new_cold=$(bench_endpoint $PORT_NEW "$path" $RUNS 1) - new_cached=$(bench_endpoint $PORT_NEW "$path" $RUNS 0) - - if [ "$old_ms" -gt 0 ] && [ "$new_cached" -gt 0 ]; then - speedup="${old_ms}/${new_cached}" - speedup_x=$(echo "scale=0; $old_ms / $new_cached" | bc 2>/dev/null || echo "?") - printf "%-22s %7dms %7dms %7dms %7d×\n" "$name" "$old_ms" "$new_cold" "$new_cached" "$speedup_x" - else - printf "%-22s %7dms %7dms %7dms %9s\n" "$name" "$old_ms" "$new_cold" "$new_cached" "∞" - fi -done - -echo "" -echo "═══════════════════════════════════════════════════════" - -# Cleanup -kill $OLD_PID $NEW_PID 2>/dev/null -git worktree remove "$OLD_DIR" --force 2>/dev/null -echo "Done." diff --git a/benchmark.js b/benchmark.js deleted file mode 100644 index e7b39899..00000000 --- a/benchmark.js +++ /dev/null @@ -1,246 +0,0 @@ -#!/usr/bin/env node -'use strict'; - -/** - * Benchmark suite for meshcore-analyzer. - * Launches two server instances — one with in-memory store, one with pure SQLite — - * and compares performance side by side. - * - * Usage: node benchmark.js [--runs 5] [--json] - */ - -const http = require('http'); -const { spawn } = require('child_process'); -const path = require('path'); - -const args = process.argv.slice(2); -const RUNS = Number(args.find((a, i) => args[i - 1] === '--runs') || 5); -const JSON_OUT = args.includes('--json'); - -const PORT_MEM = 13001; // In-memory store -const PORT_SQL = 13002; // SQLite-only - -const ENDPOINTS = [ - { name: 'Stats', path: '/api/stats' }, - { name: 'Packets (50)', path: '/api/packets?limit=50' }, - { name: 'Packets (100)', path: '/api/packets?limit=100' }, - { name: 'Packets grouped', path: '/api/packets?limit=100&groupByHash=true' }, - { name: 'Packets filtered', path: '/api/packets?limit=50&type=5' }, - { name: 'Packets timestamps', path: '/api/packets/timestamps?since=2020-01-01' }, - { name: 'Nodes list', path: '/api/nodes?limit=50' }, - { name: 'Node detail', path: '/api/nodes/__FIRST_NODE__' }, - { name: 'Node health', path: '/api/nodes/__FIRST_NODE__/health' }, - { name: 'Bulk health', path: '/api/nodes/bulk-health?limit=50' }, - { name: 'Network status', path: '/api/nodes/network-status' }, - { name: 'Observers', path: '/api/observers' }, - { name: 'Channels', path: '/api/channels' }, - { name: 'RF Analytics', path: '/api/analytics/rf' }, - { name: 'Topology', path: '/api/analytics/topology' }, - { name: 'Channel Analytics', path: '/api/analytics/channels' }, - { name: 'Hash Sizes', path: '/api/analytics/hash-sizes' }, - { name: 'Subpaths 2-hop', path: '/api/analytics/subpaths?minLen=2&maxLen=2&limit=50' }, - { name: 'Subpaths 3-hop', path: '/api/analytics/subpaths?minLen=3&maxLen=3&limit=30' }, - { name: 'Subpaths 4-hop', path: '/api/analytics/subpaths?minLen=4&maxLen=4&limit=20' }, - { name: 'Subpaths 5-8 hop', path: '/api/analytics/subpaths?minLen=5&maxLen=8&limit=15' }, -]; - -function fetch(url) { - return new Promise((resolve, reject) => { - const t0 = process.hrtime.bigint(); - const req = http.get(url, (res) => { - let body = ''; - res.on('data', c => body += c); - res.on('end', () => { - const ms = Number(process.hrtime.bigint() - t0) / 1e6; - resolve({ ms, bytes: Buffer.byteLength(body), status: res.statusCode, body }); - }); - }); - req.on('error', reject); - req.setTimeout(60000, () => { req.destroy(); reject(new Error('timeout')); }); - }); -} - -function median(arr) { const s = [...arr].sort((a,b)=>a-b); return s[Math.floor(s.length/2)]; } -function p95(arr) { const s = [...arr].sort((a,b)=>a-b); return s[Math.floor(s.length*0.95)]; } -function avg(arr) { return arr.reduce((a,b)=>a+b,0)/arr.length; } -function fmt(ms) { return ms >= 1000 ? (ms/1000).toFixed(1)+'s' : ms.toFixed(1)+'ms'; } -function fmtSize(b) { return b >= 1048576 ? (b/1048576).toFixed(1)+'MB' : b >= 1024 ? (b/1024).toFixed(0)+'KB' : b+'B'; } - -function launchServer(port, env = {}) { - return new Promise((resolve, reject) => { - const child = spawn('node', ['server.js'], { - cwd: __dirname, - env: { ...process.env, PORT: String(port), ...env }, - stdio: ['ignore', 'pipe', 'pipe'], - }); - let started = false; - const timeout = setTimeout(() => { if (!started) { child.kill(); reject(new Error('Server start timeout')); } }, 30000); - - child.stdout.on('data', (d) => { - if (!started && (d.toString().includes('listening') || d.toString().includes('running'))) { - started = true; clearTimeout(timeout); resolve(child); - } - }); - child.stderr.on('data', (d) => { - if (!started && (d.toString().includes('listening') || d.toString().includes('running'))) { - started = true; clearTimeout(timeout); resolve(child); - } - }); - child.on('exit', (code) => { if (!started) { clearTimeout(timeout); reject(new Error(`Server exited with ${code}`)); } }); - - // Fallback: wait longer (SQLite-only mode pre-warms subpaths ~6s) - setTimeout(() => { - if (!started) { - started = true; clearTimeout(timeout); - resolve(child); - } - }, 15000); - }); -} - -async function waitForServer(port, maxMs = 20000) { - const t0 = Date.now(); - while (Date.now() - t0 < maxMs) { - try { - const r = await fetch(`http://127.0.0.1:${port}/api/stats`); - if (r.status === 200) return true; - } catch {} - await new Promise(r => setTimeout(r, 500)); - } - throw new Error(`Server on port ${port} didn't start`); -} - -async function benchmarkEndpoints(port, endpoints, nocache = false) { - const results = []; - for (const ep of endpoints) { - const suffix = nocache ? (ep.path.includes('?') ? '&nocache=1' : '?nocache=1') : ''; - const url = `http://127.0.0.1:${port}${ep.path}${suffix}`; - - // Warm-up - try { await fetch(url); } catch {} - - const times = []; - let bytes = 0; - let failed = false; - - for (let i = 0; i < RUNS; i++) { - try { - const r = await fetch(url); - if (r.status !== 200) { failed = true; break; } - times.push(r.ms); - bytes = r.bytes; - } catch { failed = true; break; } - } - - if (failed || !times.length) { - results.push({ name: ep.name, failed: true }); - } else { - results.push({ - name: ep.name, - avg: Math.round(avg(times) * 10) / 10, - p50: Math.round(median(times) * 10) / 10, - p95: Math.round(p95(times) * 10) / 10, - bytes - }); - } - } - return results; -} - -async function run() { - console.log(`\nCoreScope Benchmark — ${RUNS} runs per endpoint`); - console.log('Launching servers...\n'); - - // Launch both servers - let memServer, sqlServer; - try { - console.log(' Starting in-memory server (port ' + PORT_MEM + ')...'); - memServer = await launchServer(PORT_MEM, {}); - await waitForServer(PORT_MEM); - console.log(' ✅ In-memory server ready'); - - console.log(' Starting SQLite-only server (port ' + PORT_SQL + ')...'); - sqlServer = await launchServer(PORT_SQL, { NO_MEMORY_STORE: '1' }); - await waitForServer(PORT_SQL); - console.log(' ✅ SQLite-only server ready\n'); - } catch (e) { - console.error('Failed to start servers:', e.message); - if (memServer) memServer.kill(); - if (sqlServer) sqlServer.kill(); - process.exit(1); - } - - // Get first node pubkey - let firstNode = ''; - try { - const r = await fetch(`http://127.0.0.1:${PORT_MEM}/api/nodes?limit=1`); - const data = JSON.parse(r.body); - firstNode = data.nodes?.[0]?.public_key || ''; - } catch {} - - const endpoints = ENDPOINTS.map(e => ({ - ...e, - path: e.path.replace('__FIRST_NODE__', firstNode), - })); - - // Get packet count - try { - const r = await fetch(`http://127.0.0.1:${PORT_MEM}/api/stats`); - const stats = JSON.parse(r.body); - console.log(`Dataset: ${(stats.totalPackets || '?').toLocaleString()} packets\n`); - } catch {} - - // Run benchmarks - console.log('Benchmarking in-memory store (nocache for true compute cost)...'); - const memResults = await benchmarkEndpoints(PORT_MEM, endpoints, true); - - console.log('Benchmarking SQLite-only (nocache)...'); - const sqlResults = await benchmarkEndpoints(PORT_SQL, endpoints, true); - - // Also test cached in-memory for the full picture - console.log('Benchmarking in-memory store (cached)...'); - const memCachedResults = await benchmarkEndpoints(PORT_MEM, endpoints, false); - - // Kill servers - memServer.kill(); - sqlServer.kill(); - - if (JSON_OUT) { - console.log(JSON.stringify({ memoryNocache: memResults, sqliteNocache: sqlResults, memoryCached: memCachedResults }, null, 2)); - return; - } - - // Print results - const W = 94; - console.log(`\n${'═'.repeat(W)}`); - console.log(' 🏁 BENCHMARK RESULTS: SQLite vs In-Memory Store'); - console.log(`${'═'.repeat(W)}`); - console.log(`${'Endpoint'.padEnd(24)} ${'SQLite'.padStart(9)} ${'Memory'.padStart(9)} ${'Cached'.padStart(9)} ${'Speedup'.padStart(9)} ${'Size (SQL)'.padStart(10)} ${'Size (Mem)'.padStart(10)}`); - console.log(`${'─'.repeat(24)} ${'─'.repeat(9)} ${'─'.repeat(9)} ${'─'.repeat(9)} ${'─'.repeat(9)} ${'─'.repeat(10)} ${'─'.repeat(10)}`); - - for (let i = 0; i < endpoints.length; i++) { - const sql = sqlResults[i]; - const mem = memResults[i]; - const cached = memCachedResults[i]; - if (!sql || sql.failed || !mem || mem.failed) { - console.log(`${endpoints[i].name.padEnd(24)} ${'FAILED'.padStart(9)}`); - continue; - } - - const speedup = sql.avg > 0 && mem.avg > 0 ? Math.round(sql.avg / mem.avg) + '×' : '—'; - const cachedStr = cached && !cached.failed ? fmt(cached.avg) : '—'; - - console.log( - `${sql.name.padEnd(24)} ${fmt(sql.avg).padStart(9)} ${fmt(mem.avg).padStart(9)} ${cachedStr.padStart(9)} ${speedup.padStart(9)} ${fmtSize(sql.bytes).padStart(10)} ${fmtSize(mem.bytes).padStart(10)}` - ); - } - - // Summary - const sqlTotal = sqlResults.filter(r => !r.failed).reduce((s, r) => s + r.avg, 0); - const memTotal = memResults.filter(r => !r.failed).reduce((s, r) => s + r.avg, 0); - console.log(`${'─'.repeat(24)} ${'─'.repeat(9)} ${'─'.repeat(9)} ${'─'.repeat(9)} ${'─'.repeat(9)}`); - console.log(`${'TOTAL'.padEnd(24)} ${fmt(sqlTotal).padStart(9)} ${fmt(memTotal).padStart(9)} ${''.padStart(9)} ${(Math.round(sqlTotal/memTotal)+'×').padStart(9)}`); - console.log(`\n${'═'.repeat(W)}\n`); -} - -run().catch(e => { console.error(e); process.exit(1); }); diff --git a/db.js b/db.js deleted file mode 100644 index c8352cd1..00000000 --- a/db.js +++ /dev/null @@ -1,935 +0,0 @@ -const Database = require('better-sqlite3'); -const path = require('path'); -const fs = require('fs'); - -// Ensure data directory exists -const dbPath = process.env.DB_PATH || path.join(__dirname, 'data', 'meshcore.db'); -const dataDir = path.dirname(dbPath); -if (!fs.existsSync(dataDir)) fs.mkdirSync(dataDir, { recursive: true }); - -const db = new Database(dbPath); -db.pragma('journal_mode = WAL'); -db.pragma('foreign_keys = ON'); -db.pragma('wal_autocheckpoint = 0'); // Disable auto-checkpoint — manual checkpoint on timer to avoid random event loop spikes - -// --- Migration: drop legacy tables (replaced by transmissions + observations in v2.3.0) --- -// Drop paths first (has FK to packets) -const legacyTables = ['paths', 'packets']; -for (const t of legacyTables) { - const exists = db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name=?`).get(t); - if (exists) { - console.log(`[migration] Dropping legacy table: ${t}`); - db.exec(`DROP TABLE IF EXISTS ${t}`); - } -} - -// --- Schema --- -db.exec(` - CREATE TABLE IF NOT EXISTS nodes ( - public_key TEXT PRIMARY KEY, - name TEXT, - role TEXT, - lat REAL, - lon REAL, - last_seen TEXT, - first_seen TEXT, - advert_count INTEGER DEFAULT 0, - battery_mv INTEGER, - temperature_c REAL - ); - - CREATE TABLE IF NOT EXISTS observers ( - id TEXT PRIMARY KEY, - name TEXT, - iata TEXT, - last_seen TEXT, - first_seen TEXT, - packet_count INTEGER DEFAULT 0, - model TEXT, - firmware TEXT, - client_version TEXT, - radio TEXT, - battery_mv INTEGER, - uptime_secs INTEGER, - noise_floor INTEGER - ); - - CREATE TABLE IF NOT EXISTS inactive_nodes ( - public_key TEXT PRIMARY KEY, - name TEXT, - role TEXT, - lat REAL, - lon REAL, - last_seen TEXT, - first_seen TEXT, - advert_count INTEGER DEFAULT 0, - battery_mv INTEGER, - temperature_c REAL - ); - - CREATE INDEX IF NOT EXISTS idx_nodes_last_seen ON nodes(last_seen); - CREATE INDEX IF NOT EXISTS idx_observers_last_seen ON observers(last_seen); - CREATE INDEX IF NOT EXISTS idx_inactive_nodes_last_seen ON inactive_nodes(last_seen); - - CREATE TABLE IF NOT EXISTS transmissions ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - raw_hex TEXT NOT NULL, - hash TEXT NOT NULL UNIQUE, - first_seen TEXT NOT NULL, - route_type INTEGER, - payload_type INTEGER, - payload_version INTEGER, - decoded_json TEXT, - created_at TEXT DEFAULT (datetime('now')) - ); - - CREATE INDEX IF NOT EXISTS idx_transmissions_hash ON transmissions(hash); - CREATE INDEX IF NOT EXISTS idx_transmissions_first_seen ON transmissions(first_seen); - CREATE INDEX IF NOT EXISTS idx_transmissions_payload_type ON transmissions(payload_type); -`); - -// --- Determine schema version --- -let schemaVersion = db.pragma('user_version', { simple: true }) || 0; - -// Migrate from old schema_version table to pragma user_version -if (schemaVersion === 0) { - try { - const row = db.prepare('SELECT version FROM schema_version ORDER BY version DESC LIMIT 1').get(); - if (row && row.version >= 3) { - db.pragma(`user_version = ${row.version}`); - schemaVersion = row.version; - db.exec('DROP TABLE IF EXISTS schema_version'); - } - } catch {} -} - -// Detect v3 schema by column presence (handles crash between migration and version write) -if (schemaVersion === 0) { - try { - const cols = db.pragma('table_info(observations)').map(c => c.name); - if (cols.includes('observer_idx') && !cols.includes('observer_id')) { - db.pragma('user_version = 3'); - schemaVersion = 3; - console.log('[migration-v3] Detected already-migrated schema, set user_version = 3'); - } - } catch {} -} - -// --- v3 migration: lean observations table --- -function needsV3Migration() { - if (schemaVersion >= 3) return false; - // Check if observations table exists with old observer_id TEXT column - const obsExists = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='observations'").get(); - if (!obsExists) return false; - const cols = db.pragma('table_info(observations)').map(c => c.name); - return cols.includes('observer_id'); -} - -function runV3Migration() { - const startTime = Date.now(); - console.log('[migration-v3] Starting observations table optimization...'); - - // a. Backup DB - const backupPath = dbPath + `.pre-v3-backup-${Date.now()}`; - try { - console.log(`[migration-v3] Backing up DB to ${backupPath}...`); - fs.copyFileSync(dbPath, backupPath); - console.log(`[migration-v3] Backup complete (${Date.now() - startTime}ms)`); - } catch (e) { - console.error(`[migration-v3] Backup failed, aborting migration: ${e.message}`); - return false; - } - - try { - // b. Create lean table - let stepStart = Date.now(); - db.exec(` - CREATE TABLE observations_v3 ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - transmission_id INTEGER NOT NULL REFERENCES transmissions(id), - observer_idx INTEGER, - direction TEXT, - snr REAL, - rssi REAL, - score INTEGER, - path_json TEXT, - timestamp INTEGER NOT NULL - ) - `); - console.log(`[migration-v3] Created observations_v3 table (${Date.now() - stepStart}ms)`); - - // c. Migrate data - stepStart = Date.now(); - const result = db.prepare(` - INSERT INTO observations_v3 (id, transmission_id, observer_idx, direction, snr, rssi, score, path_json, timestamp) - SELECT o.id, o.transmission_id, obs.rowid, o.direction, o.snr, o.rssi, o.score, o.path_json, - CAST(strftime('%s', o.timestamp) AS INTEGER) - FROM observations o - LEFT JOIN observers obs ON obs.id = o.observer_id - `).run(); - console.log(`[migration-v3] Migrated ${result.changes} rows (${Date.now() - stepStart}ms)`); - - // d. Drop view, old table, rename - stepStart = Date.now(); - db.exec('DROP VIEW IF EXISTS packets_v'); - db.exec('DROP TABLE observations'); - db.exec('ALTER TABLE observations_v3 RENAME TO observations'); - console.log(`[migration-v3] Replaced observations table (${Date.now() - stepStart}ms)`); - - // f. Create indexes - stepStart = Date.now(); - db.exec(` - CREATE INDEX idx_observations_transmission_id ON observations(transmission_id); - CREATE INDEX idx_observations_observer_idx ON observations(observer_idx); - CREATE INDEX idx_observations_timestamp ON observations(timestamp); - CREATE UNIQUE INDEX idx_observations_dedup ON observations(transmission_id, observer_idx, COALESCE(path_json, '')); - `); - console.log(`[migration-v3] Created indexes (${Date.now() - stepStart}ms)`); - - // g. Set schema version - - db.pragma('user_version = 3'); - schemaVersion = 3; - - // h. Rebuild view (done below in common code) - - // i. VACUUM + checkpoint - stepStart = Date.now(); - db.exec('VACUUM'); - db.pragma('wal_checkpoint(TRUNCATE)'); - console.log(`[migration-v3] VACUUM + checkpoint complete (${Date.now() - stepStart}ms)`); - - console.log(`[migration-v3] Migration complete! Total time: ${Date.now() - startTime}ms`); - return true; - } catch (e) { - console.error(`[migration-v3] Migration failed: ${e.message}`); - console.error('[migration-v3] Restore from backup if needed: ' + dbPath + '.pre-v3-backup'); - // Try to clean up v3 table if it exists - try { db.exec('DROP TABLE IF EXISTS observations_v3'); } catch {} - return false; - } -} - -const isV3 = schemaVersion >= 3; - -if (!isV3 && needsV3Migration()) { - runV3Migration(); -} - -// If user_version < 3 and no migration happened (fresh DB or migration skipped), create old-style table -if (schemaVersion < 3) { - const obsExists = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='observations'").get(); - if (!obsExists) { - // Fresh DB — create v3 schema directly - db.exec(` - CREATE TABLE observations ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - transmission_id INTEGER NOT NULL REFERENCES transmissions(id), - observer_idx INTEGER, - direction TEXT, - snr REAL, - rssi REAL, - score INTEGER, - path_json TEXT, - timestamp INTEGER NOT NULL - ); - CREATE INDEX idx_observations_transmission_id ON observations(transmission_id); - CREATE INDEX idx_observations_observer_idx ON observations(observer_idx); - CREATE INDEX idx_observations_timestamp ON observations(timestamp); - CREATE UNIQUE INDEX idx_observations_dedup ON observations(transmission_id, observer_idx, COALESCE(path_json, '')); - `); - - db.pragma('user_version = 3'); - schemaVersion = 3; - } else { - // Old-style observations table exists but migration wasn't run (or failed) - // Ensure indexes exist for old schema - db.exec(` - CREATE INDEX IF NOT EXISTS idx_observations_hash ON observations(hash); - CREATE INDEX IF NOT EXISTS idx_observations_transmission_id ON observations(transmission_id); - CREATE INDEX IF NOT EXISTS idx_observations_observer_id ON observations(observer_id); - CREATE INDEX IF NOT EXISTS idx_observations_timestamp ON observations(timestamp); - `); - // Dedup cleanup for old schema - try { - db.exec(`DROP INDEX IF EXISTS idx_observations_dedup`); - db.exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_observations_dedup ON observations(hash, observer_id, COALESCE(path_json, ''))`); - db.exec(`DELETE FROM observations WHERE id NOT IN (SELECT MIN(id) FROM observations GROUP BY hash, observer_id, COALESCE(path_json, ''))`); - } catch {} - } -} - -// --- Create/rebuild packets_v view --- -db.exec('DROP VIEW IF EXISTS packets_v'); -if (schemaVersion >= 3) { - db.exec(` - CREATE VIEW packets_v AS - SELECT o.id, t.raw_hex, - datetime(o.timestamp, 'unixepoch') AS timestamp, - obs.id AS observer_id, obs.name AS observer_name, - o.direction, o.snr, o.rssi, o.score, t.hash, t.route_type, - t.payload_type, t.payload_version, o.path_json, t.decoded_json, - t.created_at - FROM observations o - JOIN transmissions t ON t.id = o.transmission_id - LEFT JOIN observers obs ON obs.rowid = o.observer_idx - `); -} else { - db.exec(` - CREATE VIEW packets_v AS - SELECT o.id, t.raw_hex, o.timestamp, o.observer_id, o.observer_name, - o.direction, o.snr, o.rssi, o.score, t.hash, t.route_type, - t.payload_type, t.payload_version, o.path_json, t.decoded_json, - t.created_at - FROM observations o - JOIN transmissions t ON t.id = o.transmission_id - `); -} - -// --- Migrations for existing DBs --- -const observerCols = db.pragma('table_info(observers)').map(c => c.name); -for (const col of ['model', 'firmware', 'client_version', 'radio', 'battery_mv', 'uptime_secs', 'noise_floor']) { - if (!observerCols.includes(col)) { - const type = ['battery_mv', 'uptime_secs', 'noise_floor'].includes(col) ? 'INTEGER' : 'TEXT'; - db.exec(`ALTER TABLE observers ADD COLUMN ${col} ${type}`); - console.log(`[migration] Added observers.${col}`); - } -} - -// --- Cleanup corrupted nodes on startup --- -// Remove nodes with obviously invalid data (short pubkeys, control chars in names, etc.) -{ - const cleaned = db.prepare(` - DELETE FROM nodes WHERE - length(public_key) < 16 - OR public_key GLOB '*[^0-9a-fA-F]*' - OR (lat IS NOT NULL AND (lat < -90 OR lat > 90)) - OR (lon IS NOT NULL AND (lon < -180 OR lon > 180)) - `).run(); - if (cleaned.changes > 0) console.log(`[cleanup] Removed ${cleaned.changes} corrupted node(s) from DB`); -} - -// --- One-time migration: recalculate advert_count to count unique transmissions only --- -{ - db.exec(`CREATE TABLE IF NOT EXISTS _migrations (name TEXT PRIMARY KEY)`); - const done = db.prepare(`SELECT 1 FROM _migrations WHERE name = 'advert_count_unique_v1'`).get(); - if (!done) { - const start = Date.now(); - console.log('[migration] Recalculating advert_count (unique transmissions only)...'); - db.prepare(` - UPDATE nodes SET advert_count = ( - SELECT COUNT(*) FROM transmissions t - WHERE t.payload_type = 4 - AND t.decoded_json LIKE '%' || nodes.public_key || '%' - ) - `).run(); - db.prepare(`INSERT INTO _migrations (name) VALUES ('advert_count_unique_v1')`).run(); - console.log(`[migration] advert_count recalculated in ${Date.now() - start}ms`); - } -} - -// --- One-time migration: add telemetry columns to nodes and inactive_nodes --- -{ - const done = db.prepare(`SELECT 1 FROM _migrations WHERE name = 'node_telemetry_v1'`).get(); - if (!done) { - console.log('[migration] Adding telemetry columns to nodes/inactive_nodes...'); - const nodeCols = db.pragma('table_info(nodes)').map(c => c.name); - if (!nodeCols.includes('battery_mv')) db.exec(`ALTER TABLE nodes ADD COLUMN battery_mv INTEGER`); - if (!nodeCols.includes('temperature_c')) db.exec(`ALTER TABLE nodes ADD COLUMN temperature_c REAL`); - const inactiveCols = db.pragma('table_info(inactive_nodes)').map(c => c.name); - if (!inactiveCols.includes('battery_mv')) db.exec(`ALTER TABLE inactive_nodes ADD COLUMN battery_mv INTEGER`); - if (!inactiveCols.includes('temperature_c')) db.exec(`ALTER TABLE inactive_nodes ADD COLUMN temperature_c REAL`); - db.prepare(`INSERT INTO _migrations (name) VALUES ('node_telemetry_v1')`).run(); - console.log('[migration] node telemetry columns added'); - } -} - -// --- Prepared statements --- -const stmts = { - upsertNode: db.prepare(` - INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen) - VALUES (@public_key, @name, @role, @lat, @lon, @last_seen, @first_seen) - ON CONFLICT(public_key) DO UPDATE SET - name = COALESCE(@name, name), - role = COALESCE(@role, role), - lat = COALESCE(@lat, lat), - lon = COALESCE(@lon, lon), - last_seen = @last_seen - `), - incrementAdvertCount: db.prepare(` - UPDATE nodes SET advert_count = advert_count + 1 WHERE public_key = @public_key - `), - updateNodeTelemetry: db.prepare(` - UPDATE nodes SET - battery_mv = COALESCE(@battery_mv, battery_mv), - temperature_c = COALESCE(@temperature_c, temperature_c) - WHERE public_key = @public_key - `), - upsertObserver: db.prepare(` - INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count, model, firmware, client_version, radio, battery_mv, uptime_secs, noise_floor) - VALUES (@id, @name, @iata, @last_seen, @first_seen, 1, @model, @firmware, @client_version, @radio, @battery_mv, @uptime_secs, @noise_floor) - ON CONFLICT(id) DO UPDATE SET - name = COALESCE(@name, name), - iata = COALESCE(@iata, iata), - last_seen = @last_seen, - packet_count = packet_count + 1, - model = COALESCE(@model, model), - firmware = COALESCE(@firmware, firmware), - client_version = COALESCE(@client_version, client_version), - radio = COALESCE(@radio, radio), - battery_mv = COALESCE(@battery_mv, battery_mv), - uptime_secs = COALESCE(@uptime_secs, uptime_secs), - noise_floor = COALESCE(@noise_floor, noise_floor) - `), - updateObserverStatus: db.prepare(` - INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count, model, firmware, client_version, radio, battery_mv, uptime_secs, noise_floor) - VALUES (@id, @name, @iata, @last_seen, @first_seen, 0, @model, @firmware, @client_version, @radio, @battery_mv, @uptime_secs, @noise_floor) - ON CONFLICT(id) DO UPDATE SET - name = COALESCE(@name, name), - iata = COALESCE(@iata, iata), - last_seen = @last_seen, - model = COALESCE(@model, model), - firmware = COALESCE(@firmware, firmware), - client_version = COALESCE(@client_version, client_version), - radio = COALESCE(@radio, radio), - battery_mv = COALESCE(@battery_mv, battery_mv), - uptime_secs = COALESCE(@uptime_secs, uptime_secs), - noise_floor = COALESCE(@noise_floor, noise_floor) - `), - getPacket: db.prepare(`SELECT * FROM packets_v WHERE id = ?`), - getNode: db.prepare(`SELECT * FROM nodes WHERE public_key = ?`), - getRecentPacketsForNode: db.prepare(` - SELECT * FROM packets_v WHERE decoded_json LIKE ? OR decoded_json LIKE ? OR decoded_json LIKE ? OR decoded_json LIKE ? - ORDER BY timestamp DESC LIMIT 20 - `), - getObservers: db.prepare(`SELECT * FROM observers ORDER BY last_seen DESC`), - countPackets: db.prepare(`SELECT COUNT(*) as count FROM observations`), - countNodes: db.prepare(`SELECT COUNT(*) as count FROM nodes`), - countActiveNodes: db.prepare(`SELECT COUNT(*) as count FROM nodes WHERE last_seen > ?`), - countActiveNodesByRole: db.prepare(`SELECT COUNT(*) as count FROM nodes WHERE role = ? AND last_seen > ?`), - countObservers: db.prepare(`SELECT COUNT(*) as count FROM observers`), - countRecentPackets: schemaVersion >= 3 - ? db.prepare(`SELECT COUNT(*) as count FROM observations WHERE timestamp > CAST(strftime('%s', ?) AS INTEGER)`) - : db.prepare(`SELECT COUNT(*) as count FROM observations WHERE timestamp > ?`), - getTransmissionByHash: db.prepare(`SELECT id, first_seen FROM transmissions WHERE hash = ?`), - insertTransmission: db.prepare(` - INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) - VALUES (@raw_hex, @hash, @first_seen, @route_type, @payload_type, @payload_version, @decoded_json) - `), - updateTransmissionFirstSeen: db.prepare(`UPDATE transmissions SET first_seen = @first_seen WHERE id = @id`), - insertObservation: schemaVersion >= 3 - ? db.prepare(` - INSERT OR IGNORE INTO observations (transmission_id, observer_idx, direction, snr, rssi, score, path_json, timestamp) - VALUES (@transmission_id, @observer_idx, @direction, @snr, @rssi, @score, @path_json, @timestamp) - `) - : db.prepare(` - INSERT OR IGNORE INTO observations (transmission_id, hash, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) - VALUES (@transmission_id, @hash, @observer_id, @observer_name, @direction, @snr, @rssi, @score, @path_json, @timestamp) - `), - getObserverRowid: db.prepare(`SELECT rowid FROM observers WHERE id = ?`), -}; - -// --- In-memory observer map (observer_id text → rowid integer) --- -const observerIdToRowid = new Map(); -if (schemaVersion >= 3) { - const rows = db.prepare('SELECT id, rowid FROM observers').all(); - for (const r of rows) observerIdToRowid.set(r.id, r.rowid); -} - -// --- In-memory dedup set for v3 --- -const dedupSet = new Map(); // key → timestamp (for cleanup) -const DEDUP_TTL_MS = 5 * 60 * 1000; // 5 minutes - -function cleanupDedupSet() { - const cutoff = Date.now() - DEDUP_TTL_MS; - for (const [key, ts] of dedupSet) { - if (ts < cutoff) dedupSet.delete(key); - } -} - -// Periodic cleanup every 60s -setInterval(cleanupDedupSet, 60000).unref(); - -function resolveObserverIdx(observerId) { - if (!observerId) return null; - let rowid = observerIdToRowid.get(observerId); - if (rowid !== undefined) return rowid; - // Try DB lookup (observer may have been inserted elsewhere) - const row = stmts.getObserverRowid.get(observerId); - if (row) { - observerIdToRowid.set(observerId, row.rowid); - return row.rowid; - } - return null; -} - -// --- Helper functions --- - -function insertTransmission(data) { - const hash = data.hash; - if (!hash) return null; - - const timestamp = data.timestamp || new Date().toISOString(); - let transmissionId; - - let isNew = false; - const existing = stmts.getTransmissionByHash.get(hash); - if (existing) { - transmissionId = existing.id; - if (timestamp < existing.first_seen) { - stmts.updateTransmissionFirstSeen.run({ id: transmissionId, first_seen: timestamp }); - } - } else { - isNew = true; - const result = stmts.insertTransmission.run({ - raw_hex: data.raw_hex || '', - hash, - first_seen: timestamp, - route_type: data.route_type ?? null, - payload_type: data.payload_type ?? null, - payload_version: data.payload_version ?? null, - decoded_json: data.decoded_json || null, - }); - transmissionId = result.lastInsertRowid; - } - - let obsResult; - if (schemaVersion >= 3) { - const observerIdx = resolveObserverIdx(data.observer_id); - const epochTs = typeof timestamp === 'number' ? timestamp : Math.floor(new Date(timestamp).getTime() / 1000); - - // In-memory dedup check - const dedupKey = `${transmissionId}|${observerIdx}|${data.path_json || ''}`; - if (dedupSet.has(dedupKey)) { - return { transmissionId, observationId: 0, isNew }; - } - - obsResult = stmts.insertObservation.run({ - transmission_id: transmissionId, - observer_idx: observerIdx, - direction: data.direction || null, - snr: data.snr ?? null, - rssi: data.rssi ?? null, - score: data.score ?? null, - path_json: data.path_json || null, - timestamp: epochTs, - }); - dedupSet.set(dedupKey, Date.now()); - } else { - obsResult = stmts.insertObservation.run({ - transmission_id: transmissionId, - hash, - observer_id: data.observer_id || null, - observer_name: data.observer_name || null, - direction: data.direction || null, - snr: data.snr ?? null, - rssi: data.rssi ?? null, - score: data.score ?? null, - path_json: data.path_json || null, - timestamp, - }); - } - - return { transmissionId, observationId: obsResult.lastInsertRowid, isNew }; -} - -function incrementAdvertCount(publicKey) { - stmts.incrementAdvertCount.run({ public_key: publicKey }); -} - -function updateNodeTelemetry(data) { - stmts.updateNodeTelemetry.run({ - public_key: data.public_key, - battery_mv: data.battery_mv ?? null, - temperature_c: data.temperature_c ?? null, - }); -} - -function upsertNode(data) { - const now = new Date().toISOString(); - stmts.upsertNode.run({ - public_key: data.public_key, - name: data.name || null, - role: data.role || null, - lat: data.lat ?? null, - lon: data.lon ?? null, - last_seen: data.last_seen || now, - first_seen: data.first_seen || now, - }); -} - -function upsertObserver(data) { - const now = new Date().toISOString(); - stmts.upsertObserver.run({ - id: data.id, - name: data.name || null, - iata: data.iata || null, - last_seen: data.last_seen || now, - first_seen: data.first_seen || now, - model: data.model || null, - firmware: data.firmware || null, - client_version: data.client_version || null, - radio: data.radio || null, - battery_mv: data.battery_mv || null, - uptime_secs: data.uptime_secs || null, - noise_floor: data.noise_floor || null, - }); - // Update in-memory map for v3 - if (schemaVersion >= 3 && !observerIdToRowid.has(data.id)) { - const row = stmts.getObserverRowid.get(data.id); - if (row) observerIdToRowid.set(data.id, row.rowid); - } -} - -function updateObserverStatus(data) { - const now = new Date().toISOString(); - stmts.updateObserverStatus.run({ - id: data.id, - name: data.name || null, - iata: data.iata || null, - last_seen: data.last_seen || now, - first_seen: data.first_seen || now, - model: data.model || null, - firmware: data.firmware || null, - client_version: data.client_version || null, - radio: data.radio || null, - battery_mv: data.battery_mv || null, - uptime_secs: data.uptime_secs || null, - noise_floor: data.noise_floor || null, - }); -} - -function getPackets({ limit = 50, offset = 0, type, route, hash, since } = {}) { - let where = []; - let params = {}; - if (type !== undefined) { where.push('payload_type = @type'); params.type = type; } - if (route !== undefined) { where.push('route_type = @route'); params.route = route; } - if (hash) { where.push('hash = @hash'); params.hash = hash; } - if (since) { where.push('timestamp > @since'); params.since = since; } - const clause = where.length ? 'WHERE ' + where.join(' AND ') : ''; - const rows = db.prepare(`SELECT * FROM packets_v ${clause} ORDER BY timestamp DESC LIMIT @limit OFFSET @offset`).all({ ...params, limit, offset }); - const total = db.prepare(`SELECT COUNT(*) as count FROM packets_v ${clause}`).get(params).count; - return { rows, total }; -} - -function getTransmission(id) { - try { - return db.prepare('SELECT * FROM transmissions WHERE id = ?').get(id) || null; - } catch { return null; } -} - -function getPacket(id) { - const packet = stmts.getPacket.get(id); - if (!packet) return null; - return packet; -} - -function getNodes({ limit = 50, offset = 0, sortBy = 'last_seen' } = {}) { - const allowed = ['last_seen', 'name', 'advert_count', 'first_seen']; - const col = allowed.includes(sortBy) ? sortBy : 'last_seen'; - const dir = col === 'name' ? 'ASC' : 'DESC'; - const rows = db.prepare(`SELECT * FROM nodes ORDER BY ${col} ${dir} LIMIT ? OFFSET ?`).all(limit, offset); - const total = stmts.countNodes.get().count; - return { rows, total }; -} - -function getNode(pubkey) { - const node = stmts.getNode.get(pubkey); - if (!node) return null; - // Match by: pubkey anywhere, name in sender/text fields, name as text prefix ("Name: msg") - const namePattern = node.name ? `%${node.name}%` : `%${pubkey}%`; - const textPrefix = node.name ? `%"text":"${node.name}:%` : `%${pubkey}%`; - node.recentPackets = stmts.getRecentPacketsForNode.all( - `%${pubkey}%`, - namePattern, - textPrefix, - `%"sender":"${node.name || pubkey}"%` - ); - return node; -} - -function getObservers() { - return stmts.getObservers.all(); -} - -function getStats() { - const oneHourAgo = new Date(Date.now() - 3600000).toISOString(); - const sevenDaysAgo = new Date(Date.now() - 7 * 24 * 3600000).toISOString(); - // Try to get transmission count from normalized schema - let totalTransmissions = null; - try { - totalTransmissions = db.prepare('SELECT COUNT(*) as count FROM transmissions').get().count; - } catch {} - return { - totalPackets: totalTransmissions || stmts.countPackets.get().count, - totalTransmissions, - totalObservations: stmts.countPackets.get().count, - totalNodes: stmts.countActiveNodes.get(sevenDaysAgo).count, - totalNodesAllTime: stmts.countNodes.get().count, - totalObservers: stmts.countObservers.get().count, - packetsLastHour: stmts.countRecentPackets.get(oneHourAgo).count, - packetsLast24h: stmts.countRecentPackets.get(new Date(Date.now() - 24 * 3600000).toISOString()).count, - }; -} - -// --- Run directly --- -if (require.main === module) { - console.log('Stats:', getStats()); -} - -// Remove phantom nodes created by autoLearnHopNodes before this fix. -// Real MeshCore pubkeys are 32 bytes (64 hex chars). Phantom nodes have only -// the hop prefix as their public_key (typically 4-8 hex chars). -// Threshold: public_key <= 16 hex chars (8 bytes) is too short to be real. -function removePhantomNodes() { - const result = db.prepare(`DELETE FROM nodes WHERE LENGTH(public_key) <= 16`).run(); - if (result.changes > 0) { - console.log(`[cleanup] Removed ${result.changes} phantom node(s) with short public_key prefixes`); - } - return result.changes; -} - -function searchNodes(query, limit = 10) { - return db.prepare(` - SELECT * FROM nodes - WHERE name LIKE @q OR public_key LIKE @prefix - ORDER BY last_seen DESC - LIMIT @limit - `).all({ q: `%${query}%`, prefix: `${query}%`, limit }); -} - -function getNodeHealth(pubkey) { - const node = stmts.getNode.get(pubkey); - if (!node) return null; - - const todayStart = new Date(); - todayStart.setUTCHours(0, 0, 0, 0); - const todayISO = todayStart.toISOString(); - - const keyPattern = `%${pubkey}%`; - // Also match by node name in decoded_json (channel messages have sender name, not pubkey) - const namePattern = node.name ? `%${node.name.replace(/[%_]/g, '')}%` : null; - const whereClause = namePattern - ? `(decoded_json LIKE @keyPattern OR decoded_json LIKE @namePattern)` - : `decoded_json LIKE @keyPattern`; - const params = namePattern ? { keyPattern, namePattern } : { keyPattern }; - - // Observers that heard this node - const observers = db.prepare(` - SELECT observer_id, observer_name, - AVG(snr) as avgSnr, AVG(rssi) as avgRssi, COUNT(*) as packetCount - FROM packets_v - WHERE ${whereClause} AND observer_id IS NOT NULL - GROUP BY observer_id - ORDER BY packetCount DESC - `).all(params); - - // Stats - const packetsToday = db.prepare(` - SELECT COUNT(*) as count FROM packets_v WHERE ${whereClause} AND timestamp > @since - `).get({ ...params, since: todayISO }).count; - - const avgStats = db.prepare(` - SELECT AVG(snr) as avgSnr FROM packets_v WHERE ${whereClause} - `).get(params); - - const lastHeard = db.prepare(` - SELECT MAX(timestamp) as lastHeard FROM packets_v WHERE ${whereClause} - `).get(params).lastHeard; - - // Avg hops from path_json - const pathRows = db.prepare(` - SELECT path_json FROM packets_v WHERE ${whereClause} AND path_json IS NOT NULL - `).all(params); - - let totalHops = 0, hopCount = 0; - for (const row of pathRows) { - try { - const hops = JSON.parse(row.path_json); - if (Array.isArray(hops)) { totalHops += hops.length; hopCount++; } - } catch {} - } - const avgHops = hopCount > 0 ? Math.round(totalHops / hopCount) : 0; - - const totalPackets = db.prepare(` - SELECT COUNT(*) as count FROM packets_v WHERE ${whereClause} - `).get(params).count; - - // Recent 10 packets - const recentPackets = db.prepare(` - SELECT * FROM packets_v WHERE ${whereClause} ORDER BY timestamp DESC LIMIT 10 - `).all(params); - - return { - node, - observers, - stats: { totalPackets, packetsToday, avgSnr: avgStats.avgSnr, avgHops, lastHeard }, - recentPackets, - }; -} - -function getNodeAnalytics(pubkey, days) { - const node = stmts.getNode.get(pubkey); - if (!node) return null; - - const now = new Date(); - const from = new Date(now.getTime() - days * 86400000); - const fromISO = from.toISOString(); - const toISO = now.toISOString(); - - const keyPattern = `%${pubkey}%`; - const namePattern = node.name ? `%${node.name.replace(/[%_]/g, '')}%` : null; - const whereClause = namePattern - ? `(decoded_json LIKE @keyPattern OR decoded_json LIKE @namePattern)` - : `decoded_json LIKE @keyPattern`; - const timeWhere = `${whereClause} AND timestamp > @fromISO`; - const params = namePattern ? { keyPattern, namePattern, fromISO } : { keyPattern, fromISO }; - - // Activity timeline - const activityTimeline = db.prepare(` - SELECT strftime('%Y-%m-%dT%H:00:00Z', timestamp) as bucket, COUNT(*) as count - FROM packets_v WHERE ${timeWhere} GROUP BY bucket ORDER BY bucket - `).all(params); - - // SNR trend - const snrTrend = db.prepare(` - SELECT timestamp, snr, rssi, observer_id, observer_name - FROM packets_v WHERE ${timeWhere} AND snr IS NOT NULL ORDER BY timestamp - `).all(params); - - // Packet type breakdown - const packetTypeBreakdown = db.prepare(` - SELECT payload_type, COUNT(*) as count FROM packets_v WHERE ${timeWhere} GROUP BY payload_type - `).all(params); - - // Observer coverage - const observerCoverage = db.prepare(` - SELECT observer_id, observer_name, COUNT(*) as packetCount, - AVG(snr) as avgSnr, AVG(rssi) as avgRssi, MIN(timestamp) as firstSeen, MAX(timestamp) as lastSeen - FROM packets_v WHERE ${timeWhere} AND observer_id IS NOT NULL - GROUP BY observer_id ORDER BY packetCount DESC - `).all(params); - - // Hop distribution - const pathRows = db.prepare(` - SELECT path_json FROM packets_v WHERE ${timeWhere} AND path_json IS NOT NULL - `).all(params); - - const hopCounts = {}; - let totalWithPath = 0, relayedCount = 0; - for (const row of pathRows) { - try { - const hops = JSON.parse(row.path_json); - if (Array.isArray(hops)) { - const h = hops.length; - const key = h >= 4 ? '4+' : String(h); - hopCounts[key] = (hopCounts[key] || 0) + 1; - totalWithPath++; - if (h > 1) relayedCount++; - } - } catch {} - } - const hopDistribution = Object.entries(hopCounts).map(([hops, count]) => ({ hops, count })) - .sort((a, b) => a.hops.localeCompare(b.hops, undefined, { numeric: true })); - - // Peer interactions from decoded_json - const decodedRows = db.prepare(` - SELECT decoded_json, timestamp FROM packets_v WHERE ${timeWhere} AND decoded_json IS NOT NULL - `).all(params); - - const peerMap = {}; - for (const row of decodedRows) { - try { - const d = JSON.parse(row.decoded_json); - // Look for sender/recipient pubkeys that aren't this node - const candidates = []; - if (d.sender_key && d.sender_key !== pubkey) candidates.push({ key: d.sender_key, name: d.sender_name || d.sender_short_name }); - if (d.recipient_key && d.recipient_key !== pubkey) candidates.push({ key: d.recipient_key, name: d.recipient_name || d.recipient_short_name }); - if (d.pubkey && d.pubkey !== pubkey) candidates.push({ key: d.pubkey, name: d.name }); - for (const c of candidates) { - if (!c.key) continue; - if (!peerMap[c.key]) peerMap[c.key] = { peer_key: c.key, peer_name: c.name || c.key.slice(0, 12), messageCount: 0, lastContact: row.timestamp }; - peerMap[c.key].messageCount++; - if (row.timestamp > peerMap[c.key].lastContact) peerMap[c.key].lastContact = row.timestamp; - } - } catch {} - } - const peerInteractions = Object.values(peerMap).sort((a, b) => b.messageCount - a.messageCount).slice(0, 20); - - // Uptime heatmap - const uptimeHeatmap = db.prepare(` - SELECT CAST(strftime('%w', timestamp) AS INTEGER) as dayOfWeek, - CAST(strftime('%H', timestamp) AS INTEGER) as hour, COUNT(*) as count - FROM packets_v WHERE ${timeWhere} GROUP BY dayOfWeek, hour - `).all(params); - - // Computed stats - const totalPackets = db.prepare(`SELECT COUNT(*) as count FROM packets_v WHERE ${timeWhere}`).get(params).count; - const uniqueObservers = observerCoverage.length; - const uniquePeers = peerInteractions.length; - const avgPacketsPerDay = days > 0 ? Math.round(totalPackets / days * 10) / 10 : totalPackets; - - // Availability: distinct hours with packets / total hours - const distinctHours = activityTimeline.length; - const totalHours = days * 24; - const availabilityPct = totalHours > 0 ? Math.round(distinctHours / totalHours * 1000) / 10 : 0; - - // Longest silence - const timestamps = db.prepare(` - SELECT timestamp FROM packets_v WHERE ${timeWhere} ORDER BY timestamp - `).all(params).map(r => new Date(r.timestamp).getTime()); - - let longestSilenceMs = 0, longestSilenceStart = null; - for (let i = 1; i < timestamps.length; i++) { - const gap = timestamps[i] - timestamps[i - 1]; - if (gap > longestSilenceMs) { longestSilenceMs = gap; longestSilenceStart = new Date(timestamps[i - 1]).toISOString(); } - } - - // Signal grade - const snrValues = snrTrend.map(r => r.snr); - const snrMean = snrValues.length > 0 ? snrValues.reduce((a, b) => a + b, 0) / snrValues.length : 0; - const snrStdDev = snrValues.length > 1 ? Math.sqrt(snrValues.reduce((s, v) => s + (v - snrMean) ** 2, 0) / snrValues.length) : 0; - let signalGrade = 'D'; - if (snrMean > 15 && snrStdDev < 2) signalGrade = 'A'; - else if (snrMean > 15) signalGrade = 'A-'; - else if (snrMean > 12 && snrStdDev < 3) signalGrade = 'B+'; - else if (snrMean > 8) signalGrade = 'B'; - else if (snrMean > 3) signalGrade = 'C'; - - const relayPct = totalWithPath > 0 ? Math.round(relayedCount / totalWithPath * 1000) / 10 : 0; - - return { - node, - timeRange: { from: fromISO, to: toISO, days }, - activityTimeline, - snrTrend, - packetTypeBreakdown, - observerCoverage, - hopDistribution, - peerInteractions, - uptimeHeatmap, - computedStats: { - availabilityPct, longestSilenceMs, longestSilenceStart, signalGrade, - snrMean: Math.round(snrMean * 10) / 10, snrStdDev: Math.round(snrStdDev * 10) / 10, - relayPct, totalPackets, uniqueObservers, uniquePeers, avgPacketsPerDay - } - }; -} - -// Move stale nodes to inactive_nodes table based on retention.nodeDays config. -function moveStaleNodes(nodeDays) { - if (!nodeDays || nodeDays <= 0) return 0; - const cutoff = new Date(Date.now() - nodeDays * 24 * 3600000).toISOString(); - const move = db.transaction(() => { - db.prepare(`INSERT OR REPLACE INTO inactive_nodes SELECT * FROM nodes WHERE last_seen < ?`).run(cutoff); - const result = db.prepare(`DELETE FROM nodes WHERE last_seen < ?`).run(cutoff); - return result.changes; - }); - const moved = move(); - if (moved > 0) { - console.log(`[retention] Moved ${moved} node(s) to inactive_nodes (not seen in ${nodeDays} days)`); - } - return moved; -} - -module.exports = { db, schemaVersion, observerIdToRowid, resolveObserverIdx, insertTransmission, upsertNode, incrementAdvertCount, updateNodeTelemetry, upsertObserver, updateObserverStatus, getPackets, getPacket, getTransmission, getNodes, getNode, getObservers, getStats, searchNodes, getNodeHealth, getNodeAnalytics, removePhantomNodes, moveStaleNodes }; diff --git a/decoder.js b/decoder.js deleted file mode 100644 index cf267860..00000000 --- a/decoder.js +++ /dev/null @@ -1,439 +0,0 @@ -/** - * MeshCore Packet Decoder - * Custom implementation — does NOT use meshcore-decoder library (known path_length bug). - * - * Packet layout (per firmware docs/packet_format.md): - * [header(1)] [transportCodes?(4)] [pathLength(1)] [path hops] [payload...] - * - * Header byte (LSB first): - * bits 1-0: routeType (0=TRANSPORT_FLOOD, 1=FLOOD, 2=DIRECT, 3=TRANSPORT_DIRECT) - * bits 5-2: payloadType - * bits 7-6: payloadVersion - * - * Path length byte: - * bits 5-0: hash_count (number of hops, 0-63) - * bits 7-6: (value >> 6) + 1 = hash_size (1-4 bytes per hop hash) - */ - -'use strict'; - -// --- Constants --- - -const ROUTE_TYPES = { - 0: 'TRANSPORT_FLOOD', - 1: 'FLOOD', - 2: 'DIRECT', - 3: 'TRANSPORT_DIRECT', -}; - -const PAYLOAD_TYPES = { - 0x00: 'REQ', - 0x01: 'RESPONSE', - 0x02: 'TXT_MSG', - 0x03: 'ACK', - 0x04: 'ADVERT', - 0x05: 'GRP_TXT', - 0x06: 'GRP_DATA', - 0x07: 'ANON_REQ', - 0x08: 'PATH', - 0x09: 'TRACE', - 0x0A: 'MULTIPART', - 0x0B: 'CONTROL', - 0x0F: 'RAW_CUSTOM', -}; - -// Route types that carry transport codes (2x uint16_t, 4 bytes total) -const TRANSPORT_ROUTES = new Set([0, 3]); // TRANSPORT_FLOOD, TRANSPORT_DIRECT - -// --- Header parsing --- - -function decodeHeader(byte) { - return { - routeType: byte & 0x03, - routeTypeName: ROUTE_TYPES[byte & 0x03] || 'UNKNOWN', - payloadType: (byte >> 2) & 0x0F, - payloadTypeName: PAYLOAD_TYPES[(byte >> 2) & 0x0F] || 'UNKNOWN', - payloadVersion: (byte >> 6) & 0x03, - }; -} - -// --- Path parsing --- - -function decodePath(pathByte, buf, offset) { - const hashSize = (pathByte >> 6) + 1; // 1-4 bytes per hash - const hashCount = pathByte & 0x3F; // 0-63 hops - const available = buf.length - offset; - // Cap to what the buffer actually holds — corrupt packets may claim more hops than exist - const safeCount = Math.min(hashCount, Math.floor(available / hashSize)); - const totalBytes = safeCount * hashSize; - const hops = []; - - for (let i = 0; i < safeCount; i++) { - hops.push(buf.subarray(offset + i * hashSize, offset + i * hashSize + hashSize).toString('hex').toUpperCase()); - } - - return { - hashSize, - hashCount: safeCount, - hops, - bytesConsumed: totalBytes, - truncated: safeCount < hashCount, - }; -} - -// --- Payload decoders --- - -/** REQ / RESPONSE / TXT_MSG: dest(1) + src(1) + MAC(2) + encrypted (PAYLOAD_VER_1, per Mesh.cpp) */ -function decodeEncryptedPayload(buf) { - if (buf.length < 4) return { error: 'too short', raw: buf.toString('hex') }; - return { - destHash: buf.subarray(0, 1).toString('hex'), - srcHash: buf.subarray(1, 2).toString('hex'), - mac: buf.subarray(2, 4).toString('hex'), - encryptedData: buf.subarray(4).toString('hex'), - }; -} - -/** ACK: checksum(4) — CRC of message timestamp + text + sender pubkey (per Mesh.cpp createAck) */ -function decodeAck(buf) { - if (buf.length < 4) return { error: 'too short', raw: buf.toString('hex') }; - return { - ackChecksum: buf.subarray(0, 4).toString('hex'), - }; -} - -/** ADVERT: pubkey(32) + timestamp(4 LE) + signature(64) + appdata */ -function decodeAdvert(buf) { - if (buf.length < 100) return { error: 'too short for advert', raw: buf.toString('hex') }; - const pubKey = buf.subarray(0, 32).toString('hex'); - const timestamp = buf.readUInt32LE(32); - const signature = buf.subarray(36, 100).toString('hex'); - const appdata = buf.subarray(100); - - const result = { pubKey, timestamp, timestampISO: new Date(timestamp * 1000).toISOString(), signature }; - - if (appdata.length > 0) { - const flags = appdata[0]; - const advType = flags & 0x0F; // lower nibble is enum type, not individual bits - result.flags = { - raw: flags, - type: advType, - chat: advType === 1, - repeater: advType === 2, - room: advType === 3, - sensor: advType === 4, - hasLocation: !!(flags & 0x10), - hasFeat1: !!(flags & 0x20), - hasFeat2: !!(flags & 0x40), - hasName: !!(flags & 0x80), - }; - - let off = 1; - if (result.flags.hasLocation && appdata.length >= off + 8) { - result.lat = appdata.readInt32LE(off) / 1e6; - result.lon = appdata.readInt32LE(off + 4) / 1e6; - off += 8; - } - if (result.flags.hasFeat1 && appdata.length >= off + 2) { - result.feat1 = appdata.readUInt16LE(off); - off += 2; - } - if (result.flags.hasFeat2 && appdata.length >= off + 2) { - result.feat2 = appdata.readUInt16LE(off); - off += 2; - } - if (result.flags.hasName) { - // Find null terminator to separate name from trailing telemetry bytes - let nameEnd = appdata.length; - for (let i = off; i < appdata.length; i++) { - if (appdata[i] === 0x00) { nameEnd = i; break; } - } - let name = appdata.subarray(off, nameEnd).toString('utf8'); - name = name.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/g, ''); - result.name = name; - off = nameEnd; - // Skip null terminator(s) - while (off < appdata.length && appdata[off] === 0x00) off++; - } - - // Telemetry bytes after name: battery_mv(2 LE) + temperature_c(2 LE, signed, /100) - // Only sensor nodes (advType=4) carry telemetry bytes. - if (result.flags.sensor && off + 4 <= appdata.length) { - const batteryMv = appdata.readUInt16LE(off); - const tempRaw = appdata.readInt16LE(off + 2); - const tempC = tempRaw / 100.0; - if (batteryMv > 0 && batteryMv <= 10000) { - result.battery_mv = batteryMv; - } - // Raw int16 / 100 → °C; accept -50°C to 100°C (raw: -5000 to 10000) - if (tempRaw >= -5000 && tempRaw <= 10000) { - result.temperature_c = tempC; - } - } - } - - return result; -} - -/** - * Check if text contains non-printable characters (binary garbage). - * Returns true if more than 2 non-printable chars found (excluding \n, \t). - */ -function hasNonPrintableChars(text) { - if (!text) return false; - let count = 0; - for (let i = 0; i < text.length; i++) { - const code = text.charCodeAt(i); - if (code < 0x20 && code !== 0x0A && code !== 0x09) count++; - else if (code === 0xFFFD) count++; // Unicode replacement char (invalid UTF-8) - if (count > 2) return true; - } - return false; -} - -/** GRP_TXT: channel_hash(1) + MAC(2) + encrypted */ -function decodeGrpTxt(buf, channelKeys) { - if (buf.length < 3) return { error: 'too short', raw: buf.toString('hex') }; - const channelHash = buf[0]; - const channelHashHex = channelHash.toString(16).padStart(2, '0').toUpperCase(); - const mac = buf.subarray(1, 3).toString('hex'); - const encryptedData = buf.subarray(3).toString('hex'); - - const hasKeys = channelKeys && Object.keys(channelKeys).length > 0; - - // Try decryption with known channel keys - if (hasKeys && encryptedData.length >= 10) { - try { - const { ChannelCrypto } = require('@michaelhart/meshcore-decoder/dist/crypto/channel-crypto'); - for (const [name, key] of Object.entries(channelKeys)) { - const result = ChannelCrypto.decryptGroupTextMessage(encryptedData, mac, key); - if (result.success && result.data) { - const text = result.data.sender && result.data.message - ? `${result.data.sender}: ${result.data.message}` - : result.data.message || ''; - // Validate decrypted text is printable UTF-8 (not binary garbage) - if (hasNonPrintableChars(text)) { - return { - type: 'GRP_TXT', channelHash, channelHashHex, channel: name, - decryptionStatus: 'decryption_failed', text: null, mac, encryptedData, - }; - } - return { - type: 'CHAN', - channel: name, - channelHash, - channelHashHex, - decryptionStatus: 'decrypted', - sender: result.data.sender || null, - text, - sender_timestamp: result.data.timestamp, - flags: result.data.flags, - }; - } - } - } catch (e) { /* decryption failed, fall through */ } - - return { type: 'GRP_TXT', channelHash, channelHashHex, decryptionStatus: 'decryption_failed', mac, encryptedData }; - } - - return { type: 'GRP_TXT', channelHash, channelHashHex, decryptionStatus: 'no_key', mac, encryptedData }; -} - -/** ANON_REQ: dest(1) + ephemeral_pubkey(32) + MAC(2) + encrypted */ -function decodeAnonReq(buf) { - if (buf.length < 35) return { error: 'too short', raw: buf.toString('hex') }; - return { - destHash: buf.subarray(0, 1).toString('hex'), - ephemeralPubKey: buf.subarray(1, 33).toString('hex'), - mac: buf.subarray(33, 35).toString('hex'), - encryptedData: buf.subarray(35).toString('hex'), - }; -} - -/** PATH: dest(1) + src(1) + MAC(2) + path_data */ -function decodePath_payload(buf) { - if (buf.length < 4) return { error: 'too short', raw: buf.toString('hex') }; - return { - destHash: buf.subarray(0, 1).toString('hex'), - srcHash: buf.subarray(1, 2).toString('hex'), - mac: buf.subarray(2, 4).toString('hex'), - pathData: buf.subarray(4).toString('hex'), - }; -} - -/** TRACE: tag(4) + authCode(4) + flags(1) + pathData (per Mesh.cpp onRecvPacket TRACE) */ -function decodeTrace(buf) { - if (buf.length < 9) return { error: 'too short', raw: buf.toString('hex') }; - return { - tag: buf.readUInt32LE(0), - authCode: buf.subarray(4, 8).toString('hex'), - flags: buf[8], - pathData: buf.subarray(9).toString('hex'), - }; -} - -// Dispatcher -function decodePayload(type, buf, channelKeys) { - switch (type) { - case 0x00: return { type: 'REQ', ...decodeEncryptedPayload(buf) }; - case 0x01: return { type: 'RESPONSE', ...decodeEncryptedPayload(buf) }; - case 0x02: return { type: 'TXT_MSG', ...decodeEncryptedPayload(buf) }; - case 0x03: return { type: 'ACK', ...decodeAck(buf) }; - case 0x04: return { type: 'ADVERT', ...decodeAdvert(buf) }; - case 0x05: return { type: 'GRP_TXT', ...decodeGrpTxt(buf, channelKeys) }; - case 0x07: return { type: 'ANON_REQ', ...decodeAnonReq(buf) }; - case 0x08: return { type: 'PATH', ...decodePath_payload(buf) }; - case 0x09: return { type: 'TRACE', ...decodeTrace(buf) }; - default: return { type: 'UNKNOWN', raw: buf.toString('hex') }; - } -} - -// --- Main decoder --- - -function decodePacket(hexString, channelKeys) { - const hex = hexString.replace(/\s+/g, ''); - const buf = Buffer.from(hex, 'hex'); - - if (buf.length < 2) throw new Error('Packet too short (need at least header + pathLength)'); - - const header = decodeHeader(buf[0]); - let offset = 1; - - // Transport codes for TRANSPORT_FLOOD / TRANSPORT_DIRECT — BEFORE path_length per spec - let transportCodes = null; - if (TRANSPORT_ROUTES.has(header.routeType)) { - if (buf.length < offset + 4) throw new Error('Packet too short for transport codes'); - transportCodes = { - code1: buf.subarray(offset, offset + 2).toString('hex').toUpperCase(), - code2: buf.subarray(offset + 2, offset + 4).toString('hex').toUpperCase(), - }; - offset += 4; - } - - // Path length byte — AFTER transport codes per spec - const pathByte = buf[offset++]; - - // Path - const path = decodePath(pathByte, buf, offset); - offset += path.bytesConsumed; - - // Payload (rest of buffer) - const payloadBuf = buf.subarray(offset); - const payload = decodePayload(header.payloadType, payloadBuf, channelKeys); - - return { - header: { - routeType: header.routeType, - routeTypeName: header.routeTypeName, - payloadType: header.payloadType, - payloadTypeName: header.payloadTypeName, - payloadVersion: header.payloadVersion, - }, - transportCodes, - path: { - hashSize: path.hashSize, - hashCount: path.hashCount, - hops: path.hops, - truncated: path.truncated, - }, - payload, - raw: hex.toUpperCase(), - }; -} - -// --- ADVERT validation --- - -const VALID_ROLES = new Set(['repeater', 'companion', 'room', 'sensor']); - -/** - * Validate decoded ADVERT data before upserting into the DB. - * Returns { valid: true } or { valid: false, reason: string }. - */ -function validateAdvert(advert) { - if (!advert || advert.error) return { valid: false, reason: advert?.error || 'null advert' }; - - // pubkey must be at least 16 hex chars (8 bytes) and not all zeros - const pk = advert.pubKey || ''; - if (pk.length < 16) return { valid: false, reason: `pubkey too short (${pk.length} hex chars)` }; - if (/^0+$/.test(pk)) return { valid: false, reason: 'pubkey is all zeros' }; - - // lat/lon must be in valid ranges if present - if (advert.lat != null) { - if (!Number.isFinite(advert.lat) || advert.lat < -90 || advert.lat > 90) { - return { valid: false, reason: `invalid lat: ${advert.lat}` }; - } - } - if (advert.lon != null) { - if (!Number.isFinite(advert.lon) || advert.lon < -180 || advert.lon > 180) { - return { valid: false, reason: `invalid lon: ${advert.lon}` }; - } - } - - // name must not contain control chars (except space) or be garbage - if (advert.name != null) { - // eslint-disable-next-line no-control-regex - if (/[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/.test(advert.name)) { - return { valid: false, reason: 'name contains control characters' }; - } - // Reject names that are mostly non-printable or suspiciously long - if (advert.name.length > 64) { - return { valid: false, reason: `name too long (${advert.name.length} chars)` }; - } - } - - // role derivation check — flags byte should produce a known role - if (advert.flags) { - const role = advert.flags.repeater ? 'repeater' : advert.flags.room ? 'room' : advert.flags.sensor ? 'sensor' : 'companion'; - if (!VALID_ROLES.has(role)) return { valid: false, reason: `unknown role: ${role}` }; - } - - // timestamp: decoded but not currently used for node storage — skip validation - - return { valid: true }; -} - -module.exports = { decodePacket, validateAdvert, hasNonPrintableChars, ROUTE_TYPES, PAYLOAD_TYPES, VALID_ROLES }; - -// --- Tests --- -if (require.main === module) { - console.log('=== Test 1: ADVERT, FLOOD, 5 hops (2-byte hashes), "Kpa Roof Solar" ==='); - const pkt1 = decodePacket( - '11451000D818206D3AAC152C8A91F89957E6D30CA51F36E28790228971C473B755F244F718754CF5EE4A2FD58D944466E42CDED140C66D0CC590183E32BAF40F112BE8F3F2BDF6012B4B2793C52F1D36F69EE054D9A05593286F78453E56C0EC4A3EB95DDA2A7543FCCC00B939CACC009278603902FC12BCF84B706120526F6F6620536F6C6172' - ); - console.log(JSON.stringify(pkt1, null, 2)); - console.log(); - - // Assertions - const assert = (cond, msg) => { if (!cond) throw new Error('ASSERT FAILED: ' + msg); }; - assert(pkt1.header.routeTypeName === 'FLOOD', 'route should be FLOOD'); - assert(pkt1.header.payloadTypeName === 'ADVERT', 'payload should be ADVERT'); - assert(pkt1.path.hashSize === 2, 'hashSize should be 2'); - assert(pkt1.path.hashCount === 5, 'hashCount should be 5'); - assert(pkt1.path.hops[0] === '1000', 'first hop should be 1000'); - assert(pkt1.path.hops[1] === 'D818', 'second hop should be D818'); - assert(pkt1.transportCodes === null, 'FLOOD has no transport codes'); - assert(pkt1.payload.name === 'Kpa Roof Solar', 'name should be "Kpa Roof Solar"'); - console.log('✅ Test 1 passed\n'); - - console.log('=== Test 2: ADVERT, FLOOD, 0 hops (zero-path) ==='); - // Build a minimal advert: header=0x11 (FLOOD+ADVERT), pathLen=0x00 (1-byte hashes, 0 hops) - // Then a minimal advert payload: 32-byte pubkey + 4-byte ts + 64-byte sig + flags(1) - const fakePubKey = '00'.repeat(32); - const fakeTs = '78563412'; // LE = 0x12345678 - const fakeSig = 'AA'.repeat(64); - const flags = '00'; // no location, no name - const pkt2hex = '1100' + fakePubKey + fakeTs + fakeSig + flags; - const pkt2 = decodePacket(pkt2hex); - console.log(JSON.stringify(pkt2, null, 2)); - console.log(); - - assert(pkt2.header.routeTypeName === 'FLOOD', 'route should be FLOOD'); - assert(pkt2.header.payloadTypeName === 'ADVERT', 'payload should be ADVERT'); - assert(pkt2.path.hashSize === 1, 'hashSize should be 1'); - assert(pkt2.path.hashCount === 0, 'hashCount should be 0'); - assert(pkt2.path.hops.length === 0, 'no hops'); - assert(pkt2.payload.timestamp === 0x12345678, 'timestamp'); - console.log('✅ Test 2 passed\n'); - - console.log('All tests passed ✅'); -} diff --git a/docker/supervisord.conf b/docker/supervisord.conf index f55f34d4..2e5efbbc 100644 --- a/docker/supervisord.conf +++ b/docker/supervisord.conf @@ -15,7 +15,7 @@ stderr_logfile=/dev/stderr stderr_logfile_maxbytes=0 [program:corescope] -command=node /app/server.js +command=/app/corescope-server directory=/app autostart=true autorestart=true diff --git a/iata-coords.js b/iata-coords.js deleted file mode 100644 index 57ebd947..00000000 --- a/iata-coords.js +++ /dev/null @@ -1,90 +0,0 @@ -// IATA airport coordinates for regional node filtering -// Used by resolve-hops to determine if a node is geographically near an observer's region -const IATA_COORDS = { - // US West Coast - SJC: { lat: 37.3626, lon: -121.9290 }, - SFO: { lat: 37.6213, lon: -122.3790 }, - OAK: { lat: 37.7213, lon: -122.2208 }, - SEA: { lat: 47.4502, lon: -122.3088 }, - PDX: { lat: 45.5898, lon: -122.5951 }, - LAX: { lat: 33.9425, lon: -118.4081 }, - SAN: { lat: 32.7338, lon: -117.1933 }, - SMF: { lat: 38.6954, lon: -121.5908 }, - MRY: { lat: 36.5870, lon: -121.8430 }, - EUG: { lat: 44.1246, lon: -123.2119 }, - RDD: { lat: 40.5090, lon: -122.2934 }, - MFR: { lat: 42.3742, lon: -122.8735 }, - FAT: { lat: 36.7762, lon: -119.7181 }, - SBA: { lat: 34.4262, lon: -119.8405 }, - RNO: { lat: 39.4991, lon: -119.7681 }, - BOI: { lat: 43.5644, lon: -116.2228 }, - LAS: { lat: 36.0840, lon: -115.1537 }, - PHX: { lat: 33.4373, lon: -112.0078 }, - SLC: { lat: 40.7884, lon: -111.9778 }, - // US Mountain/Central - DEN: { lat: 39.8561, lon: -104.6737 }, - DFW: { lat: 32.8998, lon: -97.0403 }, - IAH: { lat: 29.9844, lon: -95.3414 }, - AUS: { lat: 30.1975, lon: -97.6664 }, - MSP: { lat: 44.8848, lon: -93.2223 }, - // US East Coast - ATL: { lat: 33.6407, lon: -84.4277 }, - ORD: { lat: 41.9742, lon: -87.9073 }, - JFK: { lat: 40.6413, lon: -73.7781 }, - EWR: { lat: 40.6895, lon: -74.1745 }, - BOS: { lat: 42.3656, lon: -71.0096 }, - MIA: { lat: 25.7959, lon: -80.2870 }, - IAD: { lat: 38.9531, lon: -77.4565 }, - CLT: { lat: 35.2144, lon: -80.9473 }, - DTW: { lat: 42.2124, lon: -83.3534 }, - MCO: { lat: 28.4312, lon: -81.3081 }, - BNA: { lat: 36.1263, lon: -86.6774 }, - RDU: { lat: 35.8801, lon: -78.7880 }, - // Canada - YVR: { lat: 49.1967, lon: -123.1815 }, - YYZ: { lat: 43.6777, lon: -79.6248 }, - YYC: { lat: 51.1215, lon: -114.0076 }, - YEG: { lat: 53.3097, lon: -113.5800 }, - YOW: { lat: 45.3225, lon: -75.6692 }, - // Europe - LHR: { lat: 51.4700, lon: -0.4543 }, - CDG: { lat: 49.0097, lon: 2.5479 }, - FRA: { lat: 50.0379, lon: 8.5622 }, - AMS: { lat: 52.3105, lon: 4.7683 }, - MUC: { lat: 48.3537, lon: 11.7750 }, - SOF: { lat: 42.6952, lon: 23.4062 }, - // Asia/Pacific - NRT: { lat: 35.7720, lon: 140.3929 }, - HND: { lat: 35.5494, lon: 139.7798 }, - ICN: { lat: 37.4602, lon: 126.4407 }, - SYD: { lat: -33.9461, lon: 151.1772 }, - MEL: { lat: -37.6690, lon: 144.8410 }, -}; - -// Haversine distance in km -function haversineKm(lat1, lon1, lat2, lon2) { - const R = 6371; - const dLat = (lat2 - lat1) * Math.PI / 180; - const dLon = (lon2 - lon1) * Math.PI / 180; - const a = Math.sin(dLat / 2) ** 2 + - Math.cos(lat1 * Math.PI / 180) * Math.cos(lat2 * Math.PI / 180) * - Math.sin(dLon / 2) ** 2; - return R * 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a)); -} - -// Default radius for "near region" — LoRa max realistic range ~300km -const DEFAULT_REGION_RADIUS_KM = 300; - -/** - * Check if a node is geographically within radius of an IATA region center. - * Returns { near: boolean, distKm: number } or null if can't determine. - */ -function nodeNearRegion(nodeLat, nodeLon, iata, radiusKm = DEFAULT_REGION_RADIUS_KM) { - const center = IATA_COORDS[iata]; - if (!center) return null; - if (nodeLat == null || nodeLon == null || (nodeLat === 0 && nodeLon === 0)) return null; - const distKm = haversineKm(nodeLat, nodeLon, center.lat, center.lon); - return { near: distKm <= radiusKm, distKm: Math.round(distKm) }; -} - -module.exports = { IATA_COORDS, haversineKm, nodeNearRegion, DEFAULT_REGION_RADIUS_KM }; diff --git a/package.json b/package.json index 6c488086..fae2bd87 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "main": "index.js", "scripts": { "test": "npx c8 --reporter=text --reporter=text-summary sh test-all.sh", - "test:unit": "node test-packet-filter.js && node test-aging.js && node test-regional-filter.js", + "test:unit": "node test-packet-filter.js && node test-aging.js && node test-frontend-helpers.js", "test:coverage": "npx c8 --reporter=text --reporter=html sh test-all.sh", "test:full-coverage": "sh scripts/combined-coverage.sh" }, diff --git a/packet-store.js b/packet-store.js deleted file mode 100644 index b9e3fbee..00000000 --- a/packet-store.js +++ /dev/null @@ -1,752 +0,0 @@ -'use strict'; - -/** - * In-memory packet store — loads transmissions + observations from SQLite on startup, - * serves reads from RAM, writes to both RAM + SQLite. - * M3: Restructured around transmissions (deduped by hash) with observations. - * Caps memory at configurable limit (default 1GB). - */ -class PacketStore { - constructor(dbModule, config = {}) { - this.dbModule = dbModule; // The full db module (has .db, .insertTransmission, .getPacket) - this.db = dbModule.db; // Raw better-sqlite3 instance for queries - this.maxBytes = (config.maxMemoryMB || 1024) * 1024 * 1024; - this.estPacketBytes = config.estimatedPacketBytes || 450; - this.maxPackets = Math.floor(this.maxBytes / this.estPacketBytes); - - // SQLite-only mode: skip RAM loading, all reads go to DB - this.sqliteOnly = process.env.NO_MEMORY_STORE === '1'; - - // Primary storage: transmissions sorted by first_seen DESC (newest first) - // Each transmission looks like a packet for backward compat - this.packets = []; - - // Indexes - this.byId = new Map(); // observation_id → observation object (backward compat for packet detail links) - this.byTxId = new Map(); // transmission_id → transmission object - this.byHash = new Map(); // hash → transmission object (1:1) - this.byObserver = new Map(); // observer_id → [observation objects] - this.byNode = new Map(); // pubkey → [transmission objects] (deduped) - - // Track which hashes are indexed per node pubkey (avoid dupes in byNode) - this._nodeHashIndex = new Map(); // pubkey → Set - this._advertByObserver = new Map(); // pubkey → Set (ADVERT-only, for region filtering) - - this.loaded = false; - this.stats = { totalLoaded: 0, totalObservations: 0, evicted: 0, inserts: 0, queries: 0 }; - } - - /** Load all packets from SQLite into memory */ - load() { - if (this.sqliteOnly) { - console.log('[PacketStore] SQLite-only mode (NO_MEMORY_STORE=1) — all reads go to database'); - this.loaded = true; - return this; - } - - const t0 = Date.now(); - - // Check if normalized schema exists - const hasTransmissions = this.db.prepare( - "SELECT name FROM sqlite_master WHERE type='table' AND name='transmissions'" - ).get(); - - if (hasTransmissions) { - this._loadNormalized(); - } else { - this._loadLegacy(); - } - - this.stats.totalLoaded = this.packets.length; - this.loaded = true; - const elapsed = Date.now() - t0; - console.log(`[PacketStore] Loaded ${this.packets.length} transmissions (${this.stats.totalObservations} observations) in ${elapsed}ms (${Math.round(this.packets.length * this.estPacketBytes / 1024 / 1024)}MB est)`); - return this; - } - - /** Load from normalized transmissions + observations tables */ - _loadNormalized() { - // Detect v3 schema (observer_idx instead of observer_id in observations) - const obsCols = this.db.pragma('table_info(observations)').map(c => c.name); - const isV3 = obsCols.includes('observer_idx'); - - const sql = isV3 - ? `SELECT t.id AS transmission_id, t.raw_hex, t.hash, t.first_seen, t.route_type, - t.payload_type, t.payload_version, t.decoded_json, - o.id AS observation_id, obs.id AS observer_id, obs.name AS observer_name, o.direction, - o.snr, o.rssi, o.score, o.path_json, datetime(o.timestamp, 'unixepoch') AS obs_timestamp - FROM transmissions t - LEFT JOIN observations o ON o.transmission_id = t.id - LEFT JOIN observers obs ON obs.rowid = o.observer_idx - ORDER BY t.first_seen DESC, o.timestamp DESC` - : `SELECT t.id AS transmission_id, t.raw_hex, t.hash, t.first_seen, t.route_type, - t.payload_type, t.payload_version, t.decoded_json, - o.id AS observation_id, o.observer_id, o.observer_name, o.direction, - o.snr, o.rssi, o.score, o.path_json, o.timestamp AS obs_timestamp - FROM transmissions t - LEFT JOIN observations o ON o.transmission_id = t.id - ORDER BY t.first_seen DESC, o.timestamp DESC`; - - for (const row of this.db.prepare(sql).iterate()) { - if (this.packets.length >= this.maxPackets && !this.byHash.has(row.hash)) break; - - let tx = this.byHash.get(row.hash); - if (!tx) { - tx = { - id: row.transmission_id, - raw_hex: row.raw_hex, - hash: row.hash, - first_seen: row.first_seen, - timestamp: row.first_seen, - route_type: row.route_type, - payload_type: row.payload_type, - decoded_json: row.decoded_json, - observations: [], - observation_count: 0, - // Filled from first observation for backward compat - observer_id: null, - observer_name: null, - snr: null, - rssi: null, - path_json: null, - direction: null, - }; - this.byHash.set(row.hash, tx); - this.byHash.set(row.hash, tx); - this.packets.push(tx); - this.byTxId.set(tx.id, tx); - this._indexByNode(tx); - } - - if (row.observation_id != null) { - const obs = { - id: row.observation_id, - transmission_id: tx.id, - hash: tx.hash, - observer_id: row.observer_id, - observer_name: row.observer_name, - direction: row.direction, - snr: row.snr, - rssi: row.rssi, - score: row.score, - path_json: row.path_json, - timestamp: row.obs_timestamp, - }; - - // Dedup: skip if same observer + same path already loaded - const isDupeLoad = tx.observations.some(o => o.observer_id === obs.observer_id && (o.path_json || '') === (obs.path_json || '')); - if (isDupeLoad) continue; - - tx.observations.push(obs); - tx.observation_count++; - - // Fill first observation data into transmission for backward compat - if (tx.observer_id == null && obs.observer_id) { - tx.observer_id = obs.observer_id; - tx.observer_name = obs.observer_name; - tx.snr = obs.snr; - tx.rssi = obs.rssi; - tx.path_json = obs.path_json; - tx.direction = obs.direction; - } - - // byId maps observation IDs for packet detail links - this.byId.set(obs.id, obs); - - // byObserver - if (obs.observer_id) { - if (!this.byObserver.has(obs.observer_id)) this.byObserver.set(obs.observer_id, []); - this.byObserver.get(obs.observer_id).push(obs); - } - - this.stats.totalObservations++; - } - } - - // Post-load: set each transmission's display path to the LONGEST observation path - // (most representative of mesh topology — short paths are just nearby observers) - for (const tx of this.packets) { - if (tx.observations.length > 0) { - let best = tx.observations[0]; - let bestLen = 0; - try { bestLen = JSON.parse(best.path_json || '[]').length; } catch {} - for (let i = 1; i < tx.observations.length; i++) { - let len = 0; - try { len = JSON.parse(tx.observations[i].path_json || '[]').length; } catch {} - if (len > bestLen) { best = tx.observations[i]; bestLen = len; } - } - tx.observer_id = best.observer_id; - tx.observer_name = best.observer_name; - tx.snr = best.snr; - tx.rssi = best.rssi; - tx.path_json = best.path_json; - tx.direction = best.direction; - } - } - - // Post-load: build ADVERT-by-observer index (needs all observations loaded first) - for (const tx of this.packets) { - if (tx.payload_type === 4 && tx.decoded_json) { - try { - const d = JSON.parse(tx.decoded_json); - if (d.pubKey) this._indexAdvertObservers(d.pubKey, tx); - } catch {} - } - } - console.log(`[PacketStore] ADVERT observer index: ${this._advertByObserver.size} nodes tracked`); - } - - /** Fallback: load from legacy packets table */ - _loadLegacy() { - for (const row of this.db.prepare( - 'SELECT * FROM packets_v ORDER BY timestamp DESC' - ).iterate()) { - if (this.packets.length >= this.maxPackets) break; - this._indexLegacy(row); - } - } - - /** Index a legacy packet row (old flat structure) — builds transmission + observation */ - _indexLegacy(pkt) { - let tx = this.byHash.get(pkt.hash); - if (!tx) { - tx = { - id: pkt.id, - raw_hex: pkt.raw_hex, - hash: pkt.hash, - first_seen: pkt.timestamp, - timestamp: pkt.timestamp, - route_type: pkt.route_type, - payload_type: pkt.payload_type, - decoded_json: pkt.decoded_json, - observations: [], - observation_count: 0, - observer_id: pkt.observer_id, - observer_name: pkt.observer_name, - snr: pkt.snr, - rssi: pkt.rssi, - path_json: pkt.path_json, - direction: pkt.direction, - }; - this.byHash.set(pkt.hash, tx); - this.byHash.set(pkt.hash, tx); - this.packets.push(tx); - this.byTxId.set(tx.id, tx); - this._indexByNode(tx); - } - - if (pkt.timestamp < tx.first_seen) { - tx.first_seen = pkt.timestamp; - tx.timestamp = pkt.timestamp; - } - // Update display path if new observation has longer path - let newPathLen = 0, curPathLen = 0; - try { newPathLen = JSON.parse(pkt.path_json || '[]').length; } catch {} - try { curPathLen = JSON.parse(tx.path_json || '[]').length; } catch {} - if (newPathLen > curPathLen) { - tx.observer_id = pkt.observer_id; - tx.observer_name = pkt.observer_name; - tx.path_json = pkt.path_json; - } - - const obs = { - id: pkt.id, - transmission_id: tx.id, - observer_id: pkt.observer_id, - observer_name: pkt.observer_name, - direction: pkt.direction, - snr: pkt.snr, - rssi: pkt.rssi, - score: pkt.score, - path_json: pkt.path_json, - timestamp: pkt.timestamp, - }; - // Dedup: skip if same observer + same path already recorded for this transmission - const isDupe = tx.observations.some(o => o.observer_id === obs.observer_id && (o.path_json || '') === (obs.path_json || '')); - if (isDupe) return tx; - - tx.observations.push(obs); - tx.observation_count++; - - this.byId.set(pkt.id, obs); - - if (pkt.observer_id) { - if (!this.byObserver.has(pkt.observer_id)) this.byObserver.set(pkt.observer_id, []); - this.byObserver.get(pkt.observer_id).push(obs); - } - - this.stats.totalObservations++; - } - - /** Extract node pubkeys from decoded_json and index transmission in byNode */ - _indexByNode(tx) { - if (!tx.decoded_json) return; - try { - const decoded = JSON.parse(tx.decoded_json); - const keys = new Set(); - if (decoded.pubKey) keys.add(decoded.pubKey); - if (decoded.destPubKey) keys.add(decoded.destPubKey); - if (decoded.srcPubKey) keys.add(decoded.srcPubKey); - for (const k of keys) { - if (!this._nodeHashIndex.has(k)) this._nodeHashIndex.set(k, new Set()); - if (this._nodeHashIndex.get(k).has(tx.hash)) continue; - this._nodeHashIndex.get(k).add(tx.hash); - if (!this.byNode.has(k)) this.byNode.set(k, []); - this.byNode.get(k).push(tx); - } - } catch {} - } - - /** Track which observers saw an ADVERT from a given pubkey */ - _indexAdvertObservers(pubkey, tx) { - if (!this._advertByObserver.has(pubkey)) this._advertByObserver.set(pubkey, new Set()); - const s = this._advertByObserver.get(pubkey); - for (const obs of tx.observations) { - if (obs.observer_id) s.add(obs.observer_id); - } - } - - /** Get node pubkeys whose ADVERTs were seen by any of the given observer IDs */ - getNodesByAdvertObservers(observerIds) { - const result = new Set(); - for (const [pubkey, observers] of this._advertByObserver) { - for (const obsId of observerIds) { - if (observers.has(obsId)) { result.add(pubkey); break; } - } - } - return result; - } - - /** Remove oldest transmissions when over memory limit */ - _evict() { - while (this.packets.length > this.maxPackets) { - const old = this.packets.pop(); - this.byHash.delete(old.hash); - this.byHash.delete(old.hash); - this.byTxId.delete(old.id); - // Remove observations from byId and byObserver - for (const obs of old.observations) { - this.byId.delete(obs.id); - if (obs.observer_id && this.byObserver.has(obs.observer_id)) { - const arr = this.byObserver.get(obs.observer_id).filter(o => o.id !== obs.id); - if (arr.length) this.byObserver.set(obs.observer_id, arr); else this.byObserver.delete(obs.observer_id); - } - } - // Skip node index cleanup (expensive, low value) - this.stats.evicted++; - } - } - - /** Insert a new packet (to both memory and SQLite) */ - insert(packetData) { - // Write to normalized tables and get the transmission ID - const txResult = this.dbModule.insertTransmission ? this.dbModule.insertTransmission(packetData) : null; - const transmissionId = txResult ? txResult.transmissionId : null; - const observationId = txResult ? txResult.observationId : null; - - // Build row directly from packetData — avoids view ID mismatch issues - const row = { - id: observationId, - raw_hex: packetData.raw_hex, - hash: packetData.hash, - timestamp: packetData.timestamp, - route_type: packetData.route_type, - payload_type: packetData.payload_type, - payload_version: packetData.payload_version, - decoded_json: packetData.decoded_json, - observer_id: packetData.observer_id, - observer_name: packetData.observer_name, - snr: packetData.snr, - rssi: packetData.rssi, - path_json: packetData.path_json, - direction: packetData.direction, - }; - if (!this.sqliteOnly) { - // Update or create transmission in memory - let tx = this.byHash.get(row.hash); - if (!tx) { - tx = { - id: transmissionId || row.id, - raw_hex: row.raw_hex, - hash: row.hash, - first_seen: row.timestamp, - timestamp: row.timestamp, - route_type: row.route_type, - payload_type: row.payload_type, - decoded_json: row.decoded_json, - observations: [], - observation_count: 0, - observer_id: row.observer_id, - observer_name: row.observer_name, - snr: row.snr, - rssi: row.rssi, - path_json: row.path_json, - direction: row.direction, - }; - this.byHash.set(row.hash, tx); - this.byHash.set(row.hash, tx); - this.packets.unshift(tx); // newest first - this.byTxId.set(tx.id, tx); - this._indexByNode(tx); - } else { - // Update first_seen if earlier - if (row.timestamp < tx.first_seen) { - tx.first_seen = row.timestamp; - tx.timestamp = row.timestamp; - } - // Update display path if new observation has longer path - let newPathLen = 0, curPathLen = 0; - try { newPathLen = JSON.parse(row.path_json || '[]').length; } catch {} - try { curPathLen = JSON.parse(tx.path_json || '[]').length; } catch {} - if (newPathLen > curPathLen) { - tx.observer_id = row.observer_id; - tx.observer_name = row.observer_name; - tx.path_json = row.path_json; - } - } - - // Add observation - const obs = { - id: row.id, - transmission_id: tx.id, - hash: tx.hash, - observer_id: row.observer_id, - observer_name: row.observer_name, - direction: row.direction, - snr: row.snr, - rssi: row.rssi, - score: row.score, - path_json: row.path_json, - timestamp: row.timestamp, - }; - // Dedup: skip if same observer + same path already recorded for this transmission - const isDupe = tx.observations.some(o => o.observer_id === obs.observer_id && (o.path_json || '') === (obs.path_json || '')); - if (!isDupe) { - tx.observations.push(obs); - tx.observation_count++; - } - - // Update transmission's display fields if this is first observation - if (tx.observations.length === 1) { - tx.observer_id = obs.observer_id; - tx.observer_name = obs.observer_name; - tx.snr = obs.snr; - tx.rssi = obs.rssi; - tx.path_json = obs.path_json; - } - - this.byId.set(obs.id, obs); - if (obs.observer_id) { - if (!this.byObserver.has(obs.observer_id)) this.byObserver.set(obs.observer_id, []); - this.byObserver.get(obs.observer_id).push(obs); - } - - this.stats.totalObservations++; - - // Update ADVERT observer index for live ingestion - if (tx.payload_type === 4 && obs.observer_id && tx.decoded_json) { - try { - const d = JSON.parse(tx.decoded_json); - if (d.pubKey) { - if (!this._advertByObserver.has(d.pubKey)) this._advertByObserver.set(d.pubKey, new Set()); - this._advertByObserver.get(d.pubKey).add(obs.observer_id); - } - } catch {} - } - - this._evict(); - this.stats.inserts++; - } - return observationId || transmissionId; - } - - /** - * Find ALL packets referencing a node — by pubkey index + name + pubkey text search. - * Returns unique transmissions (deduped). - * @param {string} nodeIdOrName - pubkey or friendly name - * @param {Array} [fromPackets] - packet array to filter (defaults to this.packets) - * @returns {{ packets: Array, pubkey: string, nodeName: string }} - */ - findPacketsForNode(nodeIdOrName, fromPackets) { - let pubkey = nodeIdOrName; - let nodeName = nodeIdOrName; - - // Always resolve to get both pubkey and name - try { - const row = this.db.prepare("SELECT public_key, name FROM nodes WHERE public_key = ? OR name = ? LIMIT 1").get(nodeIdOrName, nodeIdOrName); - if (row) { pubkey = row.public_key; nodeName = row.name || nodeIdOrName; } - } catch {} - - // Combine: index hits + text search - const indexed = this.byNode.get(pubkey); - const hashSet = indexed ? new Set(indexed.map(t => t.hash)) : new Set(); - const source = fromPackets || this.packets; - const packets = source.filter(t => - hashSet.has(t.hash) || - (t.decoded_json && (t.decoded_json.includes(nodeName) || t.decoded_json.includes(pubkey))) - ); - - return { packets, pubkey, nodeName }; - } - - /** Count transmissions and observations for a node */ - countForNode(pubkey) { - const txs = this.byNode.get(pubkey) || []; - let observations = 0; - for (const tx of txs) observations += tx.observation_count; - return { transmissions: txs.length, observations }; - } - - /** Query packets with filters — all from memory (or SQLite in fallback mode) */ - query({ limit = 50, offset = 0, type, route, region, observer, hash, since, until, node, order = 'DESC' } = {}) { - this.stats.queries++; - - if (this.sqliteOnly) return this._querySQLite({ limit, offset, type, route, region, observer, hash, since, until, node, order }); - - let results = this.packets; - - // Use indexes for single-key filters when possible - if (hash && !type && !route && !region && !observer && !since && !until && !node) { - const tx = this.byHash.get(hash); - results = tx ? [tx] : []; - } else if (observer && !type && !route && !region && !hash && !since && !until && !node) { - // For observer filter, find unique transmissions where any observation matches - results = this._transmissionsForObserver(observer); - } else if (node && !type && !route && !region && !observer && !hash && !since && !until) { - results = this.findPacketsForNode(node).packets; - } else { - // Apply filters sequentially - if (type !== undefined) { - const t = Number(type); - results = results.filter(p => p.payload_type === t); - } - if (route !== undefined) { - const r = Number(route); - results = results.filter(p => p.route_type === r); - } - if (observer) results = this._transmissionsForObserver(observer, results); - if (hash) { - const h = hash.toLowerCase(); - const tx = this.byHash.get(h); - results = tx ? results.filter(p => p.hash === h) : []; - } - if (since) results = results.filter(p => p.timestamp > since); - if (until) results = results.filter(p => p.timestamp < until); - if (region) { - const regionObservers = new Set(); - try { - const obs = this.db.prepare('SELECT id FROM observers WHERE iata = ?').all(region); - obs.forEach(o => regionObservers.add(o.id)); - } catch {} - results = results.filter(p => - p.observations.some(o => regionObservers.has(o.observer_id)) - ); - } - if (node) { - results = this.findPacketsForNode(node, results).packets; - } - } - - const total = results.length; - - // Sort - if (order === 'ASC') { - results = results.slice().sort((a, b) => { - if (a.timestamp < b.timestamp) return -1; - if (a.timestamp > b.timestamp) return 1; - return 0; - }); - } - // Default DESC — packets array is already sorted newest-first - - // Paginate - const paginated = results.slice(Number(offset), Number(offset) + Number(limit)); - return { packets: paginated, total }; - } - - /** Find unique transmissions that have at least one observation from given observer */ - _transmissionsForObserver(observerId, fromTransmissions) { - if (fromTransmissions) { - return fromTransmissions.filter(tx => - tx.observations.some(o => o.observer_id === observerId) - ); - } - // Use byObserver index: get observations, then unique transmissions - const obs = this.byObserver.get(observerId) || []; - const seen = new Set(); - const result = []; - for (const o of obs) { - const txId = o.transmission_id; - if (!seen.has(txId)) { - seen.add(txId); - const tx = this.byTxId.get(txId); - if (tx) result.push(tx); - } - } - return result; - } - - /** Query with groupByHash — now trivial since packets ARE transmissions */ - queryGrouped({ limit = 50, offset = 0, type, route, region, observer, hash, since, until, node } = {}) { - this.stats.queries++; - - if (this.sqliteOnly) return this._queryGroupedSQLite({ limit, offset, type, route, region, observer, hash, since, until, node }); - - // Get filtered transmissions - const { packets: filtered, total: filteredTotal } = this.query({ - limit: 999999, offset: 0, type, route, region, observer, hash, since, until, node - }); - - // Already grouped by hash — just format for backward compat - const sorted = filtered.map(tx => ({ - hash: tx.hash, - first_seen: tx.first_seen || tx.timestamp, - count: tx.observation_count, - observer_count: new Set(tx.observations.map(o => o.observer_id).filter(Boolean)).size, - latest: tx.observations.length ? tx.observations.reduce((max, o) => o.timestamp > max ? o.timestamp : max, tx.observations[0].timestamp) : tx.timestamp, - observer_id: tx.observer_id, - observer_name: tx.observer_name, - path_json: tx.path_json, - payload_type: tx.payload_type, - route_type: tx.route_type, - raw_hex: tx.raw_hex, - decoded_json: tx.decoded_json, - observation_count: tx.observation_count, - snr: tx.snr, - rssi: tx.rssi, - })).sort((a, b) => b.latest.localeCompare(a.latest)); - - const total = sorted.length; - const paginated = sorted.slice(Number(offset), Number(offset) + Number(limit)); - return { packets: paginated, total }; - } - - /** Get timestamps for sparkline */ - getTimestamps(since) { - if (this.sqliteOnly) { - return this.db.prepare('SELECT timestamp FROM packets_v WHERE timestamp > ? ORDER BY timestamp ASC').all(since).map(r => r.timestamp); - } - const results = []; - for (const p of this.packets) { - if (p.timestamp <= since) break; - results.push(p.timestamp); - } - return results.reverse(); - } - - /** Get a single packet by ID — checks observation IDs first (backward compat) */ - getById(id) { - if (this.sqliteOnly) return this.db.prepare('SELECT * FROM packets_v WHERE id = ?').get(id) || null; - const obs = this.byId.get(id) || null; - return this._enrichObs(obs); - } - - /** Get a transmission by its transmission table ID */ - getByTxId(id) { - if (this.sqliteOnly) return this.db.prepare('SELECT * FROM transmissions WHERE id = ?').get(id) || null; - return this.byTxId.get(id) || null; - } - - /** Get all siblings of a packet (same hash) — returns enriched observations array */ - getSiblings(hash) { - const h = hash.toLowerCase(); - if (this.sqliteOnly) return this.db.prepare('SELECT * FROM packets_v WHERE hash = ? ORDER BY timestamp DESC').all(h); - const tx = this.byHash.get(h); - return tx ? tx.observations.map(o => this._enrichObs(o)) : []; - } - - /** Get all transmissions (backward compat — returns packets array) */ - all() { - if (this.sqliteOnly) return this.db.prepare('SELECT * FROM packets_v ORDER BY timestamp DESC').all(); - return this.packets; - } - - /** Get all transmissions matching a filter function */ - filter(fn) { - if (this.sqliteOnly) return this.db.prepare('SELECT * FROM packets_v ORDER BY timestamp DESC').all().filter(fn); - return this.packets.filter(fn); - } - - /** Enrich a lean observation with transmission fields (for API responses) */ - _enrichObs(obs) { - if (!obs) return null; - const tx = this.byTxId.get(obs.transmission_id); - if (!tx) return obs; - return { - ...obs, - hash: tx.hash, - raw_hex: tx.raw_hex, - payload_type: tx.payload_type, - decoded_json: tx.decoded_json, - route_type: tx.route_type, - }; - } - - /** Enrich an array of observations with transmission fields */ - enrichObservations(observations) { - if (!observations || !observations.length) return observations; - return observations.map(o => this._enrichObs(o)); - } - - /** Memory stats */ - getStats() { - return { - ...this.stats, - inMemory: this.sqliteOnly ? 0 : this.packets.length, - sqliteOnly: this.sqliteOnly, - maxPackets: this.maxPackets, - estimatedMB: this.sqliteOnly ? 0 : Math.round(this.packets.length * this.estPacketBytes / 1024 / 1024), - maxMB: Math.round(this.maxBytes / 1024 / 1024), - indexes: { - byHash: this.byHash.size, - byObserver: this.byObserver.size, - byNode: this.byNode.size, - advertByObserver: this._advertByObserver.size, - } - }; - } - - /** SQLite fallback: query with filters */ - _querySQLite({ limit, offset, type, route, region, observer, hash, since, until, node, order }) { - const where = []; const params = []; - if (type !== undefined) { where.push('payload_type = ?'); params.push(Number(type)); } - if (route !== undefined) { where.push('route_type = ?'); params.push(Number(route)); } - if (observer) { where.push('observer_id = ?'); params.push(observer); } - if (hash) { where.push('hash = ?'); params.push(hash.toLowerCase()); } - if (since) { where.push('timestamp > ?'); params.push(since); } - if (until) { where.push('timestamp < ?'); params.push(until); } - if (region) { where.push('observer_id IN (SELECT id FROM observers WHERE iata = ?)'); params.push(region); } - if (node) { try { const nr = this.db.prepare('SELECT public_key FROM nodes WHERE public_key = ? OR name = ? LIMIT 1').get(node, node); const pk = nr ? nr.public_key : node; where.push('decoded_json LIKE ?'); params.push('%' + pk + '%'); } catch(e) { where.push('decoded_json LIKE ?'); params.push('%' + node + '%'); } } - const w = where.length ? 'WHERE ' + where.join(' AND ') : ''; - const total = this.db.prepare(`SELECT COUNT(*) as c FROM packets_v ${w}`).get(...params).c; - const packets = this.db.prepare(`SELECT * FROM packets_v ${w} ORDER BY timestamp ${order === 'ASC' ? 'ASC' : 'DESC'} LIMIT ? OFFSET ?`).all(...params, limit, offset); - return { packets, total }; - } - - /** SQLite fallback: grouped query */ - _queryGroupedSQLite({ limit, offset, type, route, region, observer, hash, since, until, node }) { - const where = []; const params = []; - if (type !== undefined) { where.push('payload_type = ?'); params.push(Number(type)); } - if (route !== undefined) { where.push('route_type = ?'); params.push(Number(route)); } - if (observer) { where.push('observer_id = ?'); params.push(observer); } - if (hash) { where.push('hash = ?'); params.push(hash.toLowerCase()); } - if (since) { where.push('timestamp > ?'); params.push(since); } - if (until) { where.push('timestamp < ?'); params.push(until); } - if (region) { where.push('observer_id IN (SELECT id FROM observers WHERE iata = ?)'); params.push(region); } - if (node) { try { const nr = this.db.prepare('SELECT public_key FROM nodes WHERE public_key = ? OR name = ? LIMIT 1').get(node, node); const pk = nr ? nr.public_key : node; where.push('decoded_json LIKE ?'); params.push('%' + pk + '%'); } catch(e) { where.push('decoded_json LIKE ?'); params.push('%' + node + '%'); } } - const w = where.length ? 'WHERE ' + where.join(' AND ') : ''; - - const sql = `SELECT hash, COUNT(*) as count, COUNT(DISTINCT observer_id) as observer_count, - MAX(timestamp) as latest, MIN(observer_id) as observer_id, MIN(observer_name) as observer_name, - MIN(path_json) as path_json, MIN(payload_type) as payload_type, MIN(route_type) as route_type, - MIN(raw_hex) as raw_hex, MIN(decoded_json) as decoded_json, MIN(snr) as snr, MIN(rssi) as rssi - FROM packets_v ${w} GROUP BY hash ORDER BY latest DESC LIMIT ? OFFSET ?`; - const packets = this.db.prepare(sql).all(...params, limit, offset); - - const countSql = `SELECT COUNT(DISTINCT hash) as c FROM packets_v ${w}`; - const total = this.db.prepare(countSql).get(...params).c; - return { packets, total }; - } -} - -module.exports = PacketStore; diff --git a/scripts/combined-coverage.sh b/scripts/combined-coverage.sh index 435bb902..eb0eb039 100644 --- a/scripts/combined-coverage.sh +++ b/scripts/combined-coverage.sh @@ -1,27 +1,10 @@ #!/bin/sh -# Run server-side tests with c8, then frontend coverage with nyc +# Combined coverage: Go backend + frontend via Playwright +# TODO: Update to use Go server binary instead of removed Node.js server. +# The old flow used `node server.js` — now use the Go binary from cmd/server/. set -e -# 1. Server-side coverage (existing) -npx c8 --reporter=json --reports-dir=.nyc_output node tools/e2e-test.js - -# 2. Instrument frontend -sh scripts/instrument-frontend.sh - -# 3. Start instrumented server -COVERAGE=1 PORT=13581 node server.js & -SERVER_PID=$! -sleep 5 - -# 4. Run Playwright tests (exercises frontend code) -BASE_URL=http://localhost:13581 node test-e2e-playwright.js || true -BASE_URL=http://localhost:13581 node test-e2e-interactions.js || true - -# 5. Collect browser coverage -BASE_URL=http://localhost:13581 node scripts/collect-frontend-coverage.js - -# 6. Kill server -kill $SERVER_PID 2>/dev/null || true - -# 7. Generate combined report -npx nyc report --reporter=text-summary --reporter=text +echo "⚠️ combined-coverage.sh needs updating for Go server migration." +echo " The Node.js server (server.js) has been removed." +echo " Update this script to start the Go binary instead." +exit 1 diff --git a/scripts/validate.sh b/scripts/validate.sh index 4041a689..4db51634 100755 --- a/scripts/validate.sh +++ b/scripts/validate.sh @@ -3,7 +3,6 @@ set -e echo "=== Syntax check ===" -node -c server.js for f in public/*.js; do node -c "$f"; done echo "✅ All JS files parse OK" diff --git a/server-helpers.js b/server-helpers.js deleted file mode 100644 index 568efdf9..00000000 --- a/server-helpers.js +++ /dev/null @@ -1,323 +0,0 @@ -'use strict'; - -const path = require('path'); -const fs = require('fs'); -const crypto = require('crypto'); - -// Config file loading -const CONFIG_PATHS = [ - path.join(__dirname, 'config.json'), - path.join(__dirname, 'data', 'config.json') -]; - -function loadConfigFile(configPaths) { - const paths = configPaths || CONFIG_PATHS; - for (const p of paths) { - try { return JSON.parse(fs.readFileSync(p, 'utf8')); } catch {} - } - return {}; -} - -// Theme file loading -const THEME_PATHS = [ - path.join(__dirname, 'theme.json'), - path.join(__dirname, 'data', 'theme.json') -]; - -function loadThemeFile(themePaths) { - const paths = themePaths || THEME_PATHS; - for (const p of paths) { - try { return JSON.parse(fs.readFileSync(p, 'utf8')); } catch {} - } - return {}; -} - -// Health thresholds -function buildHealthConfig(config) { - const _ht = (config && config.healthThresholds) || {}; - return { - infraDegraded: _ht.infraDegradedHours || 24, - infraSilent: _ht.infraSilentHours || 72, - nodeDegraded: _ht.nodeDegradedHours || 1, - nodeSilent: _ht.nodeSilentHours || 24 - }; -} - -function getHealthMs(role, HEALTH) { - const H = 3600000; - const isInfra = role === 'repeater' || role === 'room'; - return { - degradedMs: (isInfra ? HEALTH.infraDegraded : HEALTH.nodeDegraded) * H, - silentMs: (isInfra ? HEALTH.infraSilent : HEALTH.nodeSilent) * H - }; -} - -// Hash size flip-flop detection (pure — operates on provided maps) -function isHashSizeFlipFlop(seq, allSizes) { - if (!seq || seq.length < 3) return false; - if (!allSizes || allSizes.size < 2) return false; - let transitions = 0; - for (let i = 1; i < seq.length; i++) { - if (seq[i] !== seq[i - 1]) transitions++; - } - return transitions >= 2; -} - -// Compute content hash from raw hex -function computeContentHash(rawHex) { - try { - const buf = Buffer.from(rawHex, 'hex'); - if (buf.length < 2) return rawHex.slice(0, 16); - const pathByte = buf[1]; - const hashSize = ((pathByte >> 6) & 0x3) + 1; - const hashCount = pathByte & 0x3F; - const pathBytes = hashSize * hashCount; - const payloadStart = 2 + pathBytes; - const payload = buf.subarray(payloadStart); - const toHash = Buffer.concat([Buffer.from([buf[0]]), payload]); - return crypto.createHash('sha256').update(toHash).digest('hex').slice(0, 16); - } catch { return rawHex.slice(0, 16); } -} - -// Distance helper (degrees) -function geoDist(lat1, lon1, lat2, lon2) { - return Math.sqrt((lat1 - lat2) ** 2 + (lon1 - lon2) ** 2); -} - -// Derive hashtag channel key -function deriveHashtagChannelKey(channelName) { - return crypto.createHash('sha256').update(channelName).digest('hex').slice(0, 32); -} - -// Build hex breakdown ranges for packet detail view -function buildBreakdown(rawHex, decoded, decodePacketFn, channelKeys) { - if (!rawHex) return {}; - const buf = Buffer.from(rawHex, 'hex'); - const ranges = []; - - ranges.push({ start: 0, end: 0, color: 'red', label: 'Header' }); - if (buf.length < 2) return { ranges }; - - ranges.push({ start: 1, end: 1, color: 'orange', label: 'Path Length' }); - - const header = decodePacketFn ? decodePacketFn(rawHex, channelKeys || {}) : null; - let offset = 2; - - if (header && header.transportCodes) { - ranges.push({ start: 2, end: 5, color: 'blue', label: 'Transport Codes' }); - offset = 6; - } - - const pathByte = buf[1]; - const hashSize = (pathByte >> 6) + 1; - const hashCount = pathByte & 0x3F; - const pathBytes = hashSize * hashCount; - if (pathBytes > 0) { - ranges.push({ start: offset, end: offset + pathBytes - 1, color: 'green', label: 'Path' }); - } - const payloadStart = offset + pathBytes; - - if (payloadStart < buf.length) { - ranges.push({ start: payloadStart, end: buf.length - 1, color: 'yellow', label: 'Payload' }); - - if (decoded && decoded.type === 'ADVERT') { - const ps = payloadStart; - const subRanges = []; - subRanges.push({ start: ps, end: ps + 31, color: '#FFD700', label: 'PubKey' }); - subRanges.push({ start: ps + 32, end: ps + 35, color: '#FFA500', label: 'Timestamp' }); - subRanges.push({ start: ps + 36, end: ps + 99, color: '#FF6347', label: 'Signature' }); - if (buf.length > ps + 100) { - subRanges.push({ start: ps + 100, end: ps + 100, color: '#7FFFD4', label: 'Flags' }); - let off = ps + 101; - const flags = buf[ps + 100]; - if (flags & 0x10 && buf.length >= off + 8) { - subRanges.push({ start: off, end: off + 3, color: '#87CEEB', label: 'Latitude' }); - subRanges.push({ start: off + 4, end: off + 7, color: '#87CEEB', label: 'Longitude' }); - off += 8; - } - if (flags & 0x80 && off < buf.length) { - subRanges.push({ start: off, end: buf.length - 1, color: '#DDA0DD', label: 'Name' }); - } - } - ranges.push(...subRanges); - } - } - - return { ranges }; -} - -// Disambiguate hop prefixes to full nodes -function disambiguateHops(hops, allNodes, maxHopDist) { - const MAX_HOP_DIST = maxHopDist || 1.8; - - if (!allNodes._prefixIdx) { - allNodes._prefixIdx = {}; - allNodes._prefixIdxName = {}; - for (const n of allNodes) { - const pk = n.public_key.toLowerCase(); - for (let len = 1; len <= 3; len++) { - const p = pk.slice(0, len * 2); - if (!allNodes._prefixIdx[p]) allNodes._prefixIdx[p] = []; - allNodes._prefixIdx[p].push(n); - if (!allNodes._prefixIdxName[p]) allNodes._prefixIdxName[p] = n; - } - } - } - - const resolved = hops.map(hop => { - const h = hop.toLowerCase(); - const withCoords = (allNodes._prefixIdx[h] || []).filter(n => n.lat && n.lon && !(n.lat === 0 && n.lon === 0)); - if (withCoords.length === 1) { - return { hop, name: withCoords[0].name, lat: withCoords[0].lat, lon: withCoords[0].lon, pubkey: withCoords[0].public_key, known: true }; - } else if (withCoords.length > 1) { - return { hop, name: hop, lat: null, lon: null, pubkey: null, known: false, candidates: withCoords }; - } - const nameMatch = allNodes._prefixIdxName[h]; - return { hop, name: nameMatch?.name || hop, lat: null, lon: null, pubkey: nameMatch?.public_key || null, known: false }; - }); - - let lastPos = null; - for (const r of resolved) { - if (r.known && r.lat) { lastPos = [r.lat, r.lon]; continue; } - if (!r.candidates) continue; - if (lastPos) r.candidates.sort((a, b) => geoDist(a.lat, a.lon, lastPos[0], lastPos[1]) - geoDist(b.lat, b.lon, lastPos[0], lastPos[1])); - const best = r.candidates[0]; - r.name = best.name; r.lat = best.lat; r.lon = best.lon; r.pubkey = best.public_key; r.known = true; - lastPos = [r.lat, r.lon]; - } - - let nextPos = null; - for (let i = resolved.length - 1; i >= 0; i--) { - const r = resolved[i]; - if (r.known && r.lat) { nextPos = [r.lat, r.lon]; continue; } - if (!r.candidates || !nextPos) continue; - r.candidates.sort((a, b) => geoDist(a.lat, a.lon, nextPos[0], nextPos[1]) - geoDist(b.lat, b.lon, nextPos[0], nextPos[1])); - const best = r.candidates[0]; - r.name = best.name; r.lat = best.lat; r.lon = best.lon; r.pubkey = best.public_key; r.known = true; - nextPos = [r.lat, r.lon]; - } - - // Distance sanity check - for (let i = 0; i < resolved.length; i++) { - const r = resolved[i]; - if (!r.lat) continue; - const prev = i > 0 && resolved[i-1].lat ? resolved[i-1] : null; - const next = i < resolved.length-1 && resolved[i+1].lat ? resolved[i+1] : null; - if (!prev && !next) continue; - const dPrev = prev ? geoDist(r.lat, r.lon, prev.lat, prev.lon) : 0; - const dNext = next ? geoDist(r.lat, r.lon, next.lat, next.lon) : 0; - if ((prev && dPrev > MAX_HOP_DIST) && (next && dNext > MAX_HOP_DIST)) { r.unreliable = true; r.lat = null; r.lon = null; } - else if (prev && !next && dPrev > MAX_HOP_DIST) { r.unreliable = true; r.lat = null; r.lon = null; } - else if (!prev && next && dNext > MAX_HOP_DIST) { r.unreliable = true; r.lat = null; r.lon = null; } - } - - return resolved.map(r => ({ hop: r.hop, name: r.name, lat: r.lat, lon: r.lon, pubkey: r.pubkey, known: !!r.known, ambiguous: !!r.candidates, unreliable: !!r.unreliable })); -} - -// Update hash_size maps for a single packet -function updateHashSizeForPacket(p, hashSizeMap, hashSizeAllMap, hashSizeSeqMap) { - if (p.payload_type === 4 && p.raw_hex) { - try { - const d = typeof p.decoded_json === 'string' ? JSON.parse(p.decoded_json || '{}') : (p.decoded_json || {}); - const pk = d.pubKey || d.public_key; - if (pk) { - const pathByte = parseInt(p.raw_hex.slice(2, 4), 16); - const hs = ((pathByte >> 6) & 0x3) + 1; - hashSizeMap.set(pk, hs); - if (!hashSizeAllMap.has(pk)) hashSizeAllMap.set(pk, new Set()); - hashSizeAllMap.get(pk).add(hs); - if (!hashSizeSeqMap.has(pk)) hashSizeSeqMap.set(pk, []); - hashSizeSeqMap.get(pk).push(hs); - } - } catch {} - } else if (p.path_json && p.decoded_json) { - try { - const d = typeof p.decoded_json === 'string' ? JSON.parse(p.decoded_json) : p.decoded_json; - const pk = d.pubKey || d.public_key; - if (pk && !hashSizeMap.has(pk)) { - const hops = typeof p.path_json === 'string' ? JSON.parse(p.path_json) : p.path_json; - if (hops.length > 0) { - const pathByte = p.raw_hex ? parseInt(p.raw_hex.slice(2, 4), 16) : -1; - const hs = pathByte >= 0 ? ((pathByte >> 6) & 0x3) + 1 : (hops[0].length / 2); - if (hs >= 1 && hs <= 4) hashSizeMap.set(pk, hs); - } - } - } catch {} - } -} - -// Rebuild all hash size maps from packet store -function rebuildHashSizeMap(packets, hashSizeMap, hashSizeAllMap, hashSizeSeqMap) { - hashSizeMap.clear(); - hashSizeAllMap.clear(); - hashSizeSeqMap.clear(); - - // Pass 1: ADVERT packets - for (const p of packets) { - if (p.payload_type === 4 && p.raw_hex) { - try { - const d = JSON.parse(p.decoded_json || '{}'); - const pk = d.pubKey || d.public_key; - if (pk) { - const pathByte = parseInt(p.raw_hex.slice(2, 4), 16); - const hs = ((pathByte >> 6) & 0x3) + 1; - if (!hashSizeMap.has(pk)) hashSizeMap.set(pk, hs); - if (!hashSizeAllMap.has(pk)) hashSizeAllMap.set(pk, new Set()); - hashSizeAllMap.get(pk).add(hs); - if (!hashSizeSeqMap.has(pk)) hashSizeSeqMap.set(pk, []); - hashSizeSeqMap.get(pk).push(hs); - } - } catch {} - } - } - for (const [, seq] of hashSizeSeqMap) seq.reverse(); - - // Pass 2: fallback from path hops - for (const p of packets) { - if (p.path_json) { - try { - const hops = JSON.parse(p.path_json); - if (hops.length > 0) { - const hopLen = hops[0].length / 2; - if (hopLen >= 1 && hopLen <= 4) { - const pathByte = p.raw_hex ? parseInt(p.raw_hex.slice(2, 4), 16) : -1; - const hs = pathByte >= 0 ? ((pathByte >> 6) & 0x3) + 1 : hopLen; - if (p.decoded_json) { - const d = JSON.parse(p.decoded_json); - const pk = d.pubKey || d.public_key; - if (pk && !hashSizeMap.has(pk)) hashSizeMap.set(pk, hs); - } - } - } - } catch {} - } - } -} - -// API key middleware factory -function requireApiKey(apiKey) { - return function(req, res, next) { - if (!apiKey) return next(); - const provided = req.headers['x-api-key'] || req.query.apiKey; - if (provided === apiKey) return next(); - return res.status(401).json({ error: 'Invalid or missing API key' }); - }; -} - -module.exports = { - loadConfigFile, - loadThemeFile, - buildHealthConfig, - getHealthMs, - isHashSizeFlipFlop, - computeContentHash, - geoDist, - deriveHashtagChannelKey, - buildBreakdown, - disambiguateHops, - updateHashSizeForPacket, - rebuildHashSizeMap, - requireApiKey, - CONFIG_PATHS, - THEME_PATHS -}; diff --git a/server.js b/server.js deleted file mode 100644 index 31cd6d74..00000000 --- a/server.js +++ /dev/null @@ -1,3067 +0,0 @@ -'use strict'; - -const express = require('express'); -const http = require('http'); -const https = require('https'); -const { WebSocketServer } = require('ws'); -const mqtt = require('mqtt'); -const path = require('path'); -const fs = require('fs'); -const helpers = require('./server-helpers'); -const { loadConfigFile, loadThemeFile, buildHealthConfig, getHealthMs: _getHealthMs, - isHashSizeFlipFlop, computeContentHash, geoDist, deriveHashtagChannelKey, - buildBreakdown: _buildBreakdown, disambiguateHops: _disambiguateHops, - updateHashSizeForPacket: _updateHashSizeForPacket, - rebuildHashSizeMap: _rebuildHashSizeMap, - requireApiKey: _requireApiKeyFactory, - CONFIG_PATHS, THEME_PATHS } = helpers; -const config = loadConfigFile(); -const decoder = require('./decoder'); -const PAYLOAD_TYPES = decoder.PAYLOAD_TYPES; -const hasNonPrintableChars = decoder.hasNonPrintableChars; -const { nodeNearRegion, IATA_COORDS } = require('./iata-coords'); -const { execSync } = require('child_process'); - -// Version + git commit for /api/stats and /api/health -const APP_VERSION = (() => { - try { return require('./package.json').version; } catch { return 'unknown'; } -})(); -const GIT_COMMIT = (() => { - // 1. .git-commit file (baked by Docker / CI) - try { - const c = fs.readFileSync(path.join(__dirname, '.git-commit'), 'utf8').trim(); - if (c && c !== 'unknown') return c; - } catch { /* ignore */ } - // 2. git rev-parse at runtime - try { return execSync('git rev-parse --short HEAD', { encoding: 'utf8', timeout: 3000 }).trim(); } catch { /* ignore */ } - return 'unknown'; -})(); - -// Health thresholds — configurable with sensible defaults -const HEALTH = buildHealthConfig(config); -function getHealthMs(role) { return _getHealthMs(role, HEALTH); } -const MAX_HOP_DIST_SERVER = config.maxHopDist || 1.8; -const PacketStore = require('./packet-store'); - -// --- Precomputed hash_size map (updated on new packets, not per-request) --- -const _hashSizeMap = new Map(); // pubkey → latest hash_size (number) -const _hashSizeAllMap = new Map(); // pubkey → Set of all hash_sizes seen -const _hashSizeSeqMap = new Map(); // pubkey → array of hash_sizes in chronological order (oldest first) -function _rebuildHashSizeMapLocal() { - _rebuildHashSizeMap(pktStore.packets, _hashSizeMap, _hashSizeAllMap, _hashSizeSeqMap); -} -function _isHashSizeFlipFlop(pubkey) { - return isHashSizeFlipFlop(_hashSizeSeqMap.get(pubkey), _hashSizeAllMap.get(pubkey)); -} - -function _updateHashSizeForPacketLocal(p) { - _updateHashSizeForPacket(p, _hashSizeMap, _hashSizeAllMap, _hashSizeSeqMap); -} - -// API key middleware for write endpoints -const API_KEY = config.apiKey || null; -const requireApiKey = _requireApiKeyFactory(API_KEY); - -const db = require('./db'); -const pktStore = new PacketStore(db, config.packetStore || {}).load(); -_rebuildHashSizeMapLocal(); - -// Backfill: fix roles for nodes whose adverts were decoded with old bitfield flags -// ADV_TYPE is a 4-bit enum (0=none, 1=chat, 2=repeater, 3=room, 4=sensor), not individual bits -(function _backfillRoles() { - const ADV_ROLES = { 1: 'companion', 2: 'repeater', 3: 'room', 4: 'sensor' }; - let fixed = 0; - for (const p of pktStore.packets) { - if (p.payload_type !== 4 || !p.raw_hex) continue; - try { - const d = JSON.parse(p.decoded_json || '{}'); - const pk = d.pubKey || d.public_key; - if (!pk) continue; - const appStart = p.raw_hex.length - (d.flags?.raw != null ? 2 : 0); // flags byte position varies - const flagsByte = d.flags?.raw; - if (flagsByte == null) continue; - const advType = flagsByte & 0x0F; - const correctRole = ADV_ROLES[advType] || 'companion'; - const node = db.db.prepare('SELECT role FROM nodes WHERE public_key = ?').get(pk); - if (node && node.role !== correctRole) { - db.db.prepare('UPDATE nodes SET role = ? WHERE public_key = ?').run(correctRole, pk); - fixed++; - } - } catch {} - } - if (fixed > 0) console.log(`[backfill] Fixed ${fixed} node roles (advert type enum vs bitfield)`); -})(); - -// --- Shared cached node list (refreshed every 30s, avoids repeated SQLite queries) --- -let _cachedAllNodes = null; -let _cachedAllNodesWithRole = null; -let _cachedAllNodesTs = 0; -const NODES_CACHE_MS = 30000; -function getCachedNodes(includeRole) { - const now = Date.now(); - if (!_cachedAllNodes || now - _cachedAllNodesTs > NODES_CACHE_MS) { - _cachedAllNodes = db.db.prepare('SELECT public_key, name, lat, lon FROM nodes WHERE name IS NOT NULL').all(); - _cachedAllNodesWithRole = db.db.prepare('SELECT public_key, name, lat, lon, role FROM nodes WHERE name IS NOT NULL').all(); - _cachedAllNodesTs = now; - // Clear prefix index so disambiguateHops rebuilds it on fresh data - delete _cachedAllNodes._prefixIdx; - delete _cachedAllNodes._prefixIdxName; - delete _cachedAllNodesWithRole._prefixIdx; - delete _cachedAllNodesWithRole._prefixIdxName; - } - return includeRole ? _cachedAllNodesWithRole : _cachedAllNodes; -} - -const configuredChannelKeys = config.channelKeys || {}; -const hashChannels = Array.isArray(config.hashChannels) ? config.hashChannels : []; - -const derivedHashChannelKeys = {}; -for (const rawChannel of hashChannels) { - if (typeof rawChannel !== 'string') continue; - const trimmed = rawChannel.trim(); - if (!trimmed) continue; - const channelName = trimmed.startsWith('#') ? trimmed : `#${trimmed}`; - if (Object.prototype.hasOwnProperty.call(configuredChannelKeys, channelName)) continue; - derivedHashChannelKeys[channelName] = deriveHashtagChannelKey(channelName); -} - -// Load rainbow table of pre-computed channel keys (common MeshCore channel names) -let rainbowKeys = {}; -try { - const rainbowPath = path.join(__dirname, 'channel-rainbow.json'); - if (fs.existsSync(rainbowPath)) { - rainbowKeys = JSON.parse(fs.readFileSync(rainbowPath, 'utf8')); - console.log(`[channels] Loaded ${Object.keys(rainbowKeys).length} rainbow table entries`); - } -} catch (e) { - console.warn('[channels] Failed to load channel-rainbow.json:', e.message); -} - -// Merge: rainbow (lowest priority) -> derived from hashChannels -> explicit config (highest priority) -const channelKeys = { ...rainbowKeys, ...derivedHashChannelKeys, ...configuredChannelKeys }; - -const totalKeys = Object.keys(channelKeys).length; -const derivedCount = Object.keys(derivedHashChannelKeys).length; -const rainbowCount = Object.keys(rainbowKeys).length; -console.log(`[channels] ${totalKeys} channel key(s) (${derivedCount} derived from hashChannels, ${rainbowCount} from rainbow table)`); - -// --- Cache TTL config (seconds → ms) --- -const _ttlCfg = config.cacheTTL || {}; -const TTL = { - stats: (_ttlCfg.stats || 10) * 1000, - nodeDetail: (_ttlCfg.nodeDetail || 300) * 1000, - nodeHealth: (_ttlCfg.nodeHealth || 300) * 1000, - nodeList: (_ttlCfg.nodeList || 90) * 1000, - bulkHealth: (_ttlCfg.bulkHealth || 600) * 1000, - networkStatus: (_ttlCfg.networkStatus || 600) * 1000, - observers: (_ttlCfg.observers || 300) * 1000, - channels: (_ttlCfg.channels || 15) * 1000, - channelMessages: (_ttlCfg.channelMessages || 10) * 1000, - analyticsRF: (_ttlCfg.analyticsRF || 1800) * 1000, - analyticsTopology: (_ttlCfg.analyticsTopology || 1800) * 1000, - analyticsChannels: (_ttlCfg.analyticsChannels || 1800) * 1000, - analyticsHashSizes: (_ttlCfg.analyticsHashSizes || 3600) * 1000, - analyticsSubpaths: (_ttlCfg.analyticsSubpaths || 3600) * 1000, - analyticsSubpathDetail: (_ttlCfg.analyticsSubpathDetail || 3600) * 1000, - nodeAnalytics: (_ttlCfg.nodeAnalytics || 60) * 1000, - nodeSearch: (_ttlCfg.nodeSearch || 10) * 1000, - invalidationDebounce: (_ttlCfg.invalidationDebounce || 30) * 1000, -}; - -// --- TTL Cache --- -class TTLCache { - constructor() { this.store = new Map(); this.hits = 0; this.misses = 0; this.staleHits = 0; this.recomputes = 0; this._inflight = new Map(); } - get(key) { - const entry = this.store.get(key); - if (!entry) { this.misses++; return undefined; } - if (Date.now() > entry.expires) { - // Stale-while-revalidate: return stale data if within grace period (2× TTL) - if (Date.now() < entry.expires + entry.ttl) { - this.staleHits++; - return entry.value; - } - this.store.delete(key); this.misses++; return undefined; - } - this.hits++; - return entry.value; - } - // Check if entry is stale (expired but within grace). Caller should trigger async recompute. - isStale(key) { - const entry = this.store.get(key); - if (!entry) return false; - return Date.now() > entry.expires && Date.now() < entry.expires + entry.ttl; - } - // Recompute guard: ensures only one recompute per key at a time - recompute(key, fn) { - if (this._inflight.has(key)) return; - this._inflight.set(key, true); - this.recomputes++; - try { fn(); } catch (e) { console.error(`[cache] recompute error for ${key}:`, e.message); } - this._inflight.delete(key); - } - set(key, value, ttlMs) { - this.store.set(key, { value, expires: Date.now() + ttlMs, ttl: ttlMs }); - } - invalidate(prefix) { - for (const key of this.store.keys()) { - if (key.startsWith(prefix)) this.store.delete(key); - } - } - debouncedInvalidateBulkHealth() { - if (this._bulkHealthTimer) return; - this._bulkHealthTimer = setTimeout(() => { - this._bulkHealthTimer = null; - this.invalidate('bulk-health'); - }, 30000); - } - debouncedInvalidateAll() { - if (this._debounceTimer) return; - this._debounceTimer = setTimeout(() => { - this._debounceTimer = null; - // Only invalidate truly time-sensitive caches - this.invalidate('channels'); // chat messages need freshness - this.invalidate('observers'); // observer packet counts - // node:, health:, bulk-health, analytics: all have long TTLs — let them expire naturally - }, TTL.invalidationDebounce); - } - clear() { this.store.clear(); } - get size() { return this.store.size; } -} -const cache = new TTLCache(); - -const app = express(); - -function createServer(app, cfg) { - const tls = cfg.https || {}; - if (!tls.cert || !tls.key) { - return { server: http.createServer(app), isHttps: false }; - } - - try { - const certPath = path.resolve(tls.cert); - const keyPath = path.resolve(tls.key); - const options = { - cert: fs.readFileSync(certPath), - key: fs.readFileSync(keyPath), - }; - console.log(`[https] enabled (cert: ${certPath}, key: ${keyPath})`); - return { server: https.createServer(options, app), isHttps: true }; - } catch (e) { - console.error(`[https] failed to load TLS cert/key, falling back to HTTP: ${e.message}`); - return { server: http.createServer(app), isHttps: false }; - } -} - -const { server, isHttps } = createServer(app, config); - -// --- Performance Instrumentation --- -const perfStats = { - requests: 0, - totalMs: 0, - endpoints: {}, // { path: { count, totalMs, maxMs, avgMs, p95: [], lastSlow } } - slowQueries: [], // last 50 requests > 100ms - startedAt: Date.now(), - reset() { - this.requests = 0; this.totalMs = 0; this.endpoints = {}; this.slowQueries = []; this.startedAt = Date.now(); - } -}; - -app.use((req, res, next) => { - if (!req.path.startsWith('/api/')) return next(); - // Benchmark mode: bypass cache when ?nocache=1 - if (req.query.nocache === '1') { - const origGet = cache.get.bind(cache); - cache.get = () => null; - res.on('finish', () => { cache.get = origGet; }); - } - const start = process.hrtime.bigint(); - const origEnd = res.end; - res.end = function(...args) { - const ms = Number(process.hrtime.bigint() - start) / 1e6; - perfStats.requests++; - perfStats.totalMs += ms; - // Normalize parameterized routes - const key = req.route ? req.route.path : req.path.replace(/[0-9a-f]{8,}/gi, ':id'); - if (!perfStats.endpoints[key]) perfStats.endpoints[key] = { count: 0, totalMs: 0, maxMs: 0, recent: [] }; - const ep = perfStats.endpoints[key]; - ep.count++; - ep.totalMs += ms; - if (ms > ep.maxMs) ep.maxMs = ms; - ep.recent.push(ms); - if (ep.recent.length > 100) ep.recent.shift(); - if (ms > 100) { - perfStats.slowQueries.push({ path: req.path, ms: Math.round(ms * 10) / 10, time: new Date().toISOString(), status: res.statusCode }); - if (perfStats.slowQueries.length > 50) perfStats.slowQueries.shift(); - } - origEnd.apply(res, args); - }; - next(); -}); - -// Expose cache TTL config to frontend -app.get('/api/config/cache', (req, res) => { - res.json(config.cacheTTL || {}); -}); - -// Expose all client-side config (roles, thresholds, tiles, limits, etc.) -app.get('/api/config/client', (req, res) => { - res.json({ - roles: config.roles || null, - healthThresholds: { - infraDegradedMs: HEALTH.infraDegraded * 3600000, - infraSilentMs: HEALTH.infraSilent * 3600000, - nodeDegradedMs: HEALTH.nodeDegraded * 3600000, - nodeSilentMs: HEALTH.nodeSilent * 3600000 - }, - tiles: config.tiles || null, - snrThresholds: config.snrThresholds || null, - distThresholds: config.distThresholds || null, - maxHopDist: config.maxHopDist || null, - limits: config.limits || null, - perfSlowMs: config.perfSlowMs || null, - wsReconnectMs: config.wsReconnectMs || null, - cacheInvalidateMs: config.cacheInvalidateMs || null, - externalUrls: config.externalUrls || null, - propagationBufferMs: (config.liveMap || {}).propagationBufferMs || 5000 - }); -}); - -app.get('/api/config/regions', (req, res) => { - // Merge config regions with any IATA codes seen from observers - const regions = { ...(config.regions || {}) }; - try { - const rows = db.db.prepare("SELECT DISTINCT iata FROM observers WHERE iata IS NOT NULL").all(); - for (const r of rows) { - if (r.iata && !regions[r.iata]) regions[r.iata] = r.iata; // fallback to code itself - } - } catch {} - res.json(regions); -}); - -// Helper: get set of observer IDs matching region filter (comma-separated IATA codes) -function getObserverIdsForRegions(regionParam) { - if (!regionParam) return null; // null = no filter - const codes = regionParam.split(',').map(s => s.trim()).filter(Boolean); - if (codes.length === 0) return null; - const ids = new Set(); - const observers = db.getObservers(); - for (const o of observers) { - if (o.iata && codes.includes(o.iata)) ids.add(o.id); - } - return ids; -} - -app.get('/api/config/theme', (req, res) => { - const cfg = loadConfigFile(); - const theme = loadThemeFile(); - res.json({ - branding: { - siteName: 'CoreScope', - tagline: 'Real-time MeshCore LoRa mesh network analyzer', - ...(cfg.branding || {}), - ...(theme.branding || {}) - }, - theme: { - accent: '#4a9eff', - accentHover: '#6db3ff', - navBg: '#0f0f23', - navBg2: '#1a1a2e', - ...(cfg.theme || {}), - ...(theme.theme || {}) - }, - themeDark: { - ...(cfg.themeDark || {}), - ...(theme.themeDark || {}) - }, - nodeColors: { - repeater: '#dc2626', - companion: '#2563eb', - room: '#16a34a', - sensor: '#d97706', - observer: '#8b5cf6', - ...(cfg.nodeColors || {}), - ...(theme.nodeColors || {}) - }, - typeColors: { - ...(cfg.typeColors || {}), - ...(theme.typeColors || {}) - }, - home: theme.home || cfg.home || null, - }); -}); - - - -app.get('/api/config/map', (req, res) => { - const defaults = config.mapDefaults || {}; - res.json({ - center: defaults.center || [37.45, -122.0], - zoom: defaults.zoom || 9 - }); -}); - -app.get('/api/perf', (req, res) => { - const summary = {}; - for (const [path, ep] of Object.entries(perfStats.endpoints)) { - const sorted = [...ep.recent].sort((a, b) => a - b); - const p95 = sorted[Math.floor(sorted.length * 0.95)] || 0; - const p50 = sorted[Math.floor(sorted.length * 0.5)] || 0; - summary[path] = { - count: ep.count, - avgMs: Math.round(ep.totalMs / ep.count * 10) / 10, - p50Ms: Math.round(p50 * 10) / 10, - p95Ms: Math.round(p95 * 10) / 10, - maxMs: Math.round(ep.maxMs * 10) / 10, - }; - } - // Sort by total time spent (count * avg) descending - const sorted = Object.entries(summary).sort((a, b) => (b[1].count * b[1].avgMs) - (a[1].count * a[1].avgMs)); - res.json({ - uptime: Math.round((Date.now() - perfStats.startedAt) / 1000), - totalRequests: perfStats.requests, - avgMs: perfStats.requests ? Math.round(perfStats.totalMs / perfStats.requests * 10) / 10 : 0, - endpoints: Object.fromEntries(sorted), - slowQueries: perfStats.slowQueries.slice(-20), - cache: { size: cache.size, hits: cache.hits, misses: cache.misses, staleHits: cache.staleHits, recomputes: cache.recomputes, hitRate: cache.hits + cache.staleHits + cache.misses > 0 ? Math.round((cache.hits + cache.staleHits) / (cache.hits + cache.staleHits + cache.misses) * 1000) / 10 : 0 }, - packetStore: pktStore.getStats(), - sqlite: (() => { - try { - const walInfo = db.db.pragma('wal_checkpoint(PASSIVE)'); - const pageSize = db.db.pragma('page_size', { simple: true }); - const pageCount = db.db.pragma('page_count', { simple: true }); - const freelistCount = db.db.pragma('freelist_count', { simple: true }); - const dbSizeMB = Math.round(pageSize * pageCount / 1048576 * 10) / 10; - const freelistMB = Math.round(pageSize * freelistCount / 1048576 * 10) / 10; - const fs = require('fs'); - const dbPath = process.env.DB_PATH || require('path').join(__dirname, 'data', 'meshcore.db'); - let walSizeMB = 0; - try { walSizeMB = Math.round(fs.statSync(dbPath + '-wal').size / 1048576 * 10) / 10; } catch {} - const stats = db.getStats(); - return { - dbSizeMB, - walSizeMB, - freelistMB, - walPages: walInfo[0] ? { total: walInfo[0].busy + walInfo[0].checkpointed, checkpointed: walInfo[0].checkpointed, busy: walInfo[0].busy } : null, - rows: { transmissions: stats.totalTransmissions, observations: stats.totalObservations, nodes: stats.totalNodes, observers: stats.totalObservers }, - }; - } catch (e) { return { error: e.message }; } - })(), - }); -}); - -app.post('/api/perf/reset', requireApiKey, (req, res) => { perfStats.reset(); res.json({ ok: true }); }); - -// --- Event Loop Lag Monitoring --- -let evtLoopLag = 0, evtLoopMax = 0, evtLoopSamples = []; -const EL_INTERVAL = 1000; -let _elLast = process.hrtime.bigint(); -setInterval(() => { - const now = process.hrtime.bigint(); - const delta = Number(now - _elLast) / 1e6; // ms - const lag = Math.max(0, delta - EL_INTERVAL); - evtLoopLag = lag; - if (lag > evtLoopMax) evtLoopMax = lag; - evtLoopSamples.push(lag); - if (evtLoopSamples.length > 60) evtLoopSamples.shift(); // last 60s - _elLast = now; -}, EL_INTERVAL).unref(); - -// Manual WAL checkpoint every 5 minutes (auto-checkpoint disabled to avoid random event loop spikes) -setInterval(() => { - try { - const t0 = Date.now(); - db.db.pragma('wal_checkpoint(PASSIVE)'); // PASSIVE = non-blocking, won't stall writers - const ms = Date.now() - t0; - if (ms > 50) console.log(`[wal] checkpoint: ${ms}ms`); - } catch (e) { console.error('[wal] checkpoint error:', e.message); } -}, 300000).unref(); - -// Daily TRUNCATE checkpoint at 2:00 AM UTC — reclaims WAL file space -setInterval(() => { - const h = new Date().getUTCHours(); - const m = new Date().getUTCMinutes(); - if (h === 2 && m === 0) { - try { - const t0 = Date.now(); - db.db.pragma('wal_checkpoint(TRUNCATE)'); - console.log(`[wal] daily TRUNCATE checkpoint: ${Date.now() - t0}ms`); - } catch (e) { console.error('[wal] TRUNCATE checkpoint error:', e.message); } - } -}, 60000).unref(); - -// --- Node Retention: move stale nodes to inactive_nodes --- -const RETENTION_NODE_DAYS = (config.retention && config.retention.nodeDays) || 7; -db.moveStaleNodes(RETENTION_NODE_DAYS); -setInterval(() => { - db.moveStaleNodes(RETENTION_NODE_DAYS); -}, 3600000).unref(); // hourly - -// --- Health / Telemetry Endpoint --- -app.get('/api/health', (req, res) => { - const mem = process.memoryUsage(); - const uptime = process.uptime(); - const sorted = [...evtLoopSamples].sort((a, b) => a - b); - const wsClients = wss ? wss.clients.size : 0; - const pktStoreSize = pktStore ? pktStore.all().length : 0; - const pktStoreMB = pktStore ? Math.round(pktStore.all().length * 430 / 1024 / 1024 * 10) / 10 : 0; - - res.json({ - status: 'ok', - engine: 'node', - version: APP_VERSION, - commit: GIT_COMMIT, - uptime: Math.round(uptime), - uptimeHuman: `${Math.floor(uptime / 3600)}h ${Math.floor((uptime % 3600) / 60)}m`, - memory: { - rss: Math.round(mem.rss / 1024 / 1024), - heapUsed: Math.round(mem.heapUsed / 1024 / 1024), - heapTotal: Math.round(mem.heapTotal / 1024 / 1024), - external: Math.round(mem.external / 1024 / 1024), - }, - eventLoop: { - currentLagMs: Math.round(evtLoopLag * 10) / 10, - maxLagMs: Math.round(evtLoopMax * 10) / 10, - p50Ms: Math.round((sorted[Math.floor(sorted.length * 0.5)] || 0) * 10) / 10, - p95Ms: Math.round((sorted[Math.floor(sorted.length * 0.95)] || 0) * 10) / 10, - p99Ms: Math.round((sorted[Math.floor(sorted.length * 0.99)] || 0) * 10) / 10, - }, - cache: { - entries: cache.size, - hits: cache.hits, - misses: cache.misses, - staleHits: cache.staleHits, - recomputes: cache.recomputes, - hitRate: cache.hits + cache.staleHits + cache.misses > 0 ? Math.round((cache.hits + cache.staleHits) / (cache.hits + cache.staleHits + cache.misses) * 1000) / 10 : 0, - }, - websocket: { - clients: wsClients, - }, - packetStore: { - packets: pktStoreSize, - estimatedMB: pktStoreMB, - }, - perf: { - totalRequests: perfStats.requests, - avgMs: perfStats.requests > 0 ? Math.round(perfStats.totalMs / perfStats.requests * 10) / 10 : 0, - slowQueries: perfStats.slowQueries.length, - recentSlow: perfStats.slowQueries.slice(-5), - }, - }); -}); - -// --- WebSocket --- -const wss = new WebSocketServer({ server }); - -function broadcast(msg) { - const data = JSON.stringify(msg); - wss.clients.forEach(c => { if (c.readyState === 1) c.send(data); }); -} - -// Resolve path hops to known nodes (≥2 bytes / 4 hex chars) — never creates phantom nodes. -// Hops that can't be resolved are displayed as raw hex prefixes by the hop-resolver. -const hopNodeCache = new Set(); // Avoid repeated DB lookups for known hops -// Track when nodes were last seen as relay hops in packet paths (full pubkey → ISO timestamp) -const lastPathSeenMap = new Map(); - -// Sequential hop disambiguation — delegates to server-helpers.js (single source of truth) -function disambiguateHops(hops, allNodes) { - return _disambiguateHops(hops, allNodes, MAX_HOP_DIST_SERVER); -} - -// Cache hop prefix → full pubkey for lastPathSeenMap resolution -const hopPrefixToKey = new Map(); -// Negative cache: prefixes known to be ambiguous (match multiple nodes) — never resolve these -const ambiguousHopPrefixes = new Set(); - -// Check if a hop prefix uniquely resolves to a single node. Returns the public_key or null. -function resolveUniquePrefixMatch(hopLower) { - if (ambiguousHopPrefixes.has(hopLower)) return null; - if (hopPrefixToKey.has(hopLower)) return hopPrefixToKey.get(hopLower); - // Count matches — only use if exactly one node matches - const matches = db.db.prepare("SELECT public_key FROM nodes WHERE LOWER(public_key) LIKE ? LIMIT 2").all(hopLower + '%'); - if (matches.length === 1) { - hopPrefixToKey.set(hopLower, matches[0].public_key); - return matches[0].public_key; - } - if (matches.length > 1) { - ambiguousHopPrefixes.add(hopLower); - } - return null; -} - -function autoLearnHopNodes(hops, now) { - for (const hop of hops) { - if (hop.length < 4) continue; // Skip 1-byte hops — too ambiguous - if (hopNodeCache.has(hop)) continue; - resolveUniquePrefixMatch(hop.toLowerCase()); - // Cache either way to avoid repeated DB lookups — but never create phantom nodes. - // Unresolved hops are displayed as raw prefixes by the hop-resolver. - hopNodeCache.add(hop); - } -} - -// Update lastPathSeenMap for all hops in a packet path (including 1-byte hops) -function updatePathSeenTimestamps(hops, now) { - for (const hop of hops) { - const hopLower = hop.toLowerCase(); - const fullKey = resolveUniquePrefixMatch(hopLower); - if (fullKey) { - lastPathSeenMap.set(fullKey, now); - } - } -} - -// --- MQTT --- -// Build list of MQTT sources: supports single config.mqtt (legacy) or config.mqttSources array -const mqttSources = []; -if (config.mqttSources && Array.isArray(config.mqttSources)) { - mqttSources.push(...config.mqttSources); -} else if (config.mqtt && config.mqtt.broker) { - // Legacy single-broker config - mqttSources.push({ - name: 'default', - broker: config.mqtt.broker, - topics: [config.mqtt.topic, 'meshcore/#'], - }); -} - -if (process.env.NODE_ENV === 'test') { - console.log('[mqtt] Skipping MQTT connections in test mode'); -} else { -for (const source of mqttSources) { - try { - const opts = { reconnectPeriod: 5000 }; - if (source.username) opts.username = source.username; - if (source.password) opts.password = source.password; - if (source.rejectUnauthorized === false) opts.rejectUnauthorized = false; - - const client = mqtt.connect(source.broker, opts); - const tag = source.name || source.broker; - - client.on('connect', () => { - console.log(`MQTT [${tag}] connected to ${source.broker}`); - const topics = Array.isArray(source.topics) ? source.topics : [source.topics || 'meshcore/#']; - for (const t of topics) { - client.subscribe(t, { qos: 0 }, (err) => { - if (err) console.error(`MQTT [${tag}] subscribe error for ${t}:`, err); - else console.log(`MQTT [${tag}] subscribed to ${t}`); - }); - } - }); - client.on('error', (e) => console.error(`MQTT [${tag}] error:`, e.message)); - client.on('offline', () => console.log(`MQTT [${tag}] offline`)); - client.on('message', (topic, message) => { - try { - const msg = JSON.parse(message.toString()); - const parts = topic.split('/'); - const now = new Date().toISOString(); - - // IATA filter: if source has iataFilter, only accept matching regions - const region = parts[1] || null; - if (source.iataFilter && Array.isArray(source.iataFilter) && region) { - if (!source.iataFilter.includes(region)) return; - } - - // --- Status topic: meshcore///status --- - if (parts[3] === 'status' && parts[2]) { - const observerId = parts[2]; - const name = msg.origin || null; - const iata = region; - // Parse radio string: "freq,bw,sf,cr" - let radioInfo = null; - if (msg.radio) { - const rp = msg.radio.split(','); - radioInfo = { freq: parseFloat(rp[0]), bw: parseFloat(rp[1]), sf: parseInt(rp[2]), cr: parseInt(rp[3]) }; - } - db.updateObserverStatus({ - id: observerId, - name: name, - iata: iata, - model: msg.model || null, - firmware: msg.firmware_version || null, - client_version: msg.client_version || null, - radio: msg.radio || null, - battery_mv: msg.stats?.battery_mv || null, - uptime_secs: msg.stats?.uptime_secs || null, - noise_floor: msg.stats?.noise_floor || null, - }); - console.log(`MQTT [${tag}] status: ${name || observerId} (${iata}) - ${msg.status}`); - return; - } - - // --- Format 1: Raw packet logging (meshcoretomqtt / Cisien format) --- - // Topic: meshcore///packets, payload: { raw, SNR, RSSI, hash } - if (msg.raw && typeof msg.raw === 'string') { - const decoded = decoder.decodePacket(msg.raw, channelKeys); - const observerId = parts[2] || null; - const region = parts[1] || null; - - const pktData = { - raw_hex: msg.raw, - timestamp: now, - observer_id: observerId, - observer_name: msg.origin || null, - snr: msg.SNR ?? null, - rssi: msg.RSSI ?? null, - hash: computeContentHash(msg.raw), - route_type: decoded.header.routeType, - payload_type: decoded.header.payloadType, - payload_version: decoded.header.payloadVersion, - path_json: JSON.stringify(decoded.path.hops), - decoded_json: JSON.stringify(decoded.payload), - }; - const packetId = pktStore.insert(pktData); _updateHashSizeForPacketLocal(pktData); - let txResult; - try { txResult = db.insertTransmission(pktData); } catch (e) { console.error('[dual-write] transmission insert error:', e.message); } - - if (decoded.path.hops.length > 0) { - // Auto-create stub nodes from 2+ byte path hops - autoLearnHopNodes(decoded.path.hops, now); - // Track when each resolved hop node was last seen relaying - updatePathSeenTimestamps(decoded.path.hops, now); - } - - if (decoded.header.payloadTypeName === 'ADVERT' && decoded.payload.pubKey) { - const p = decoded.payload; - const validation = decoder.validateAdvert(p); - if (validation.valid) { - const role = p.flags ? (p.flags.repeater ? 'repeater' : p.flags.room ? 'room' : p.flags.sensor ? 'sensor' : 'companion') : 'companion'; - db.upsertNode({ public_key: p.pubKey, name: p.name || null, role, lat: p.lat, lon: p.lon, last_seen: now }); - if (txResult && txResult.isNew) db.incrementAdvertCount(p.pubKey); - // Update telemetry if present in advert - if (p.battery_mv != null || p.temperature_c != null) { - db.updateNodeTelemetry({ public_key: p.pubKey, battery_mv: p.battery_mv ?? null, temperature_c: p.temperature_c ?? null }); - } - // Invalidate this node's caches on advert - cache.invalidate('node:' + p.pubKey); - cache.invalidate('health:' + p.pubKey); - cache.debouncedInvalidateBulkHealth(); - - // Cross-reference: if this node's pubkey matches an existing observer, backfill observer name - if (p.name && p.pubKey) { - const existingObs = db.db.prepare('SELECT id FROM observers WHERE id = ?').get(p.pubKey); - if (existingObs) db.updateObserverStatus({ id: p.pubKey, name: p.name }); - } - } else { - console.warn(`[advert] Skipping corrupted ADVERT from ${tag}: ${validation.reason} (raw: ${msg.raw.slice(0, 40)}…)`); - } - } - - if (observerId) { - db.upsertObserver({ id: observerId, name: msg.origin || null, iata: region }); - } - - - // Invalidate caches on new data - cache.debouncedInvalidateAll(); - - const fullPacket = pktStore.getById(packetId) || pktStore.byHash.get(pktData.hash) || pktData; - const tx = pktStore.byHash.get(pktData.hash); - const observation_count = tx ? tx.observation_count : 1; - const broadcastData = { id: packetId, raw: msg.raw, decoded, snr: msg.SNR, rssi: msg.RSSI, hash: pktData.hash, observer: observerId, observer_name: msg.origin || null, path_json: pktData.path_json, packet: fullPacket, observation_count }; - broadcast({ type: 'packet', data: broadcastData }); - - if (decoded.header.payloadTypeName === 'GRP_TXT') { - broadcast({ type: 'message', data: broadcastData }); - } - return; - } - - // --- Format 2: Companion bridge (ipnet-mesh/meshcore-mqtt) --- - // Topics: meshcore/advertisement, meshcore/message/channel/, meshcore/message/direct/, etc. - // Skip status/connection topics - if (topic === 'meshcore/status' || topic === 'meshcore/events/connection') return; - - // Handle self_info - local node identity - if (topic === 'meshcore/self_info') { - const info = msg.payload || msg; - const pubKey = info.pubkey || info.pub_key || info.public_key; - if (pubKey) { - db.upsertNode({ public_key: pubKey, name: info.name || 'L1 Pro (Local)', role: info.role || 'companion', lat: info.lat ?? null, lon: info.lon ?? null, last_seen: now }); - } - return; - } - - // Extract event type from topic - const eventType = parts.slice(1).join('/'); - - // Handle advertisements - if (topic === 'meshcore/advertisement') { - const advert = msg.payload || msg; - if (advert.pubkey || advert.pub_key || advert.public_key || advert.name) { - const pubKey = advert.pubkey || advert.pub_key || advert.public_key || `node-${(advert.name||'unknown').toLowerCase().replace(/[^a-z0-9]/g, '')}`; - const name = advert.name || advert.node_name || null; - const lat = advert.lat ?? advert.latitude ?? null; - const lon = advert.lon ?? advert.lng ?? advert.longitude ?? null; - const role = advert.role || (advert.flags?.repeater ? 'repeater' : advert.flags?.room ? 'room' : 'companion'); - - // Validate companion bridge adverts too - const bridgeAdvert = { pubKey: pubKey, name, lat, lon, timestamp: Math.floor(Date.now() / 1000), flags: advert.flags || null }; - const validation = decoder.validateAdvert(bridgeAdvert); - if (!validation.valid) { - console.warn(`[advert] Skipping corrupted companion ADVERT: ${validation.reason}`); - return; - } - - db.upsertNode({ public_key: pubKey, name, role, lat, lon, last_seen: now }); - - const advertPktData = { - raw_hex: null, - timestamp: now, - observer_id: 'companion', - observer_name: 'L1 Pro (BLE)', - snr: advert.SNR ?? advert.snr ?? null, - rssi: advert.RSSI ?? advert.rssi ?? null, - hash: 'advert', - route_type: 1, // FLOOD - payload_type: 4, // ADVERT - payload_version: 0, - path_json: JSON.stringify([]), - decoded_json: JSON.stringify(advert), - }; - const packetId = pktStore.insert(advertPktData); _updateHashSizeForPacketLocal(advertPktData); - let txResult; - try { txResult = db.insertTransmission(advertPktData); } catch (e) { console.error('[dual-write] transmission insert error:', e.message); } - if (txResult && txResult.isNew) db.incrementAdvertCount(pubKey); - broadcast({ type: 'packet', data: { id: packetId, hash: advertPktData.hash, raw: advertPktData.raw_hex, decoded: { header: { payloadTypeName: 'ADVERT' }, payload: advert } } }); - } - return; - } - - // Handle channel messages - if (topic.startsWith('meshcore/message/channel/')) { - const channelMsg = msg.payload || msg; - const channelIdx = channelMsg.channel_idx ?? msg.attributes?.channel_idx ?? topic.split('/').pop(); - const channelHash = `ch${channelIdx}`; - // Extract sender name from "Name: message" format - const senderName = channelMsg.text?.split(':')[0] || null; - // Create/update node for sender - if (senderName) { - const senderKey = `sender-${senderName.toLowerCase().replace(/[^a-z0-9]/g, '')}`; - db.upsertNode({ public_key: senderKey, name: senderName, role: 'companion', lat: null, lon: null, last_seen: now }); - } - const chPktData = { - raw_hex: null, - timestamp: now, - observer_id: 'companion', - observer_name: 'L1 Pro (BLE)', - snr: channelMsg.SNR ?? channelMsg.snr ?? null, - rssi: channelMsg.RSSI ?? channelMsg.rssi ?? null, - hash: channelHash, - route_type: 1, - payload_type: 5, // GRP_TXT - payload_version: 0, - path_json: JSON.stringify([]), - decoded_json: JSON.stringify(channelMsg), - }; - const packetId = pktStore.insert(chPktData); _updateHashSizeForPacketLocal(chPktData); - try { db.insertTransmission(chPktData); } catch (e) { console.error('[dual-write] transmission insert error:', e.message); } - broadcast({ type: 'packet', data: { id: packetId, hash: chPktData.hash, raw: chPktData.raw_hex, decoded: { header: { payloadTypeName: 'GRP_TXT' }, payload: channelMsg } } }); - broadcast({ type: 'message', data: { id: packetId, hash: chPktData.hash, decoded: { header: { payloadTypeName: 'GRP_TXT' }, payload: channelMsg } } }); - return; - } - - // Handle direct messages - if (topic.startsWith('meshcore/message/direct/')) { - const dm = msg.payload || msg; - const dmPktData = { - raw_hex: null, - timestamp: dm.timestamp || now, - observer_id: 'companion', - snr: dm.snr ?? null, - rssi: dm.rssi ?? null, - hash: null, - route_type: 0, - payload_type: 2, // TXT_MSG - payload_version: 0, - path_json: JSON.stringify(dm.hops || []), - decoded_json: JSON.stringify(dm), - }; - const packetId = pktStore.insert(dmPktData); _updateHashSizeForPacketLocal(dmPktData); - try { db.insertTransmission(dmPktData); } catch (e) { console.error('[dual-write] transmission insert error:', e.message); } - broadcast({ type: 'packet', data: { id: packetId, hash: dmPktData.hash, raw: dmPktData.raw_hex, decoded: { header: { payloadTypeName: 'TXT_MSG' }, payload: dm } } }); - return; - } - - // Handle traceroute - if (topic.startsWith('meshcore/traceroute/')) { - const trace = msg.payload || msg; - const tracePktData = { - raw_hex: null, - timestamp: now, - observer_id: 'companion', - snr: null, - rssi: null, - hash: null, - route_type: 1, - payload_type: 8, // PATH/TRACE - payload_version: 0, - path_json: JSON.stringify(trace.hops || trace.path || []), - decoded_json: JSON.stringify(trace), - }; - const packetId = pktStore.insert(tracePktData); _updateHashSizeForPacketLocal(tracePktData); - try { db.insertTransmission(tracePktData); } catch (e) { console.error('[dual-write] transmission insert error:', e.message); } - broadcast({ type: 'packet', data: { id: packetId, hash: tracePktData.hash, raw: tracePktData.raw_hex, decoded: { header: { payloadTypeName: 'TRACE' }, payload: trace } } }); - return; - } - - } catch (e) { - if (topic !== 'meshcore/status' && topic !== 'meshcore/events/connection') { - console.error(`MQTT [${tag}] handler error [${topic}]:`, e.message); - try { console.error(' payload:', message.toString().substring(0, 200)); } catch {} - } - } - }); - } catch (e) { - console.error(`MQTT [${source.name || source.broker}] connection failed (non-fatal):`, e.message); - } -} -} // end NODE_ENV !== 'test' - -// --- Express --- -app.use(express.json()); - -// REST API - -app.get('/api/stats', (req, res) => { - const stats = db.getStats(); - // Get role counts (active nodes only — same 7-day window as totalNodes) - const sevenDaysAgo = new Date(Date.now() - 7 * 24 * 3600000).toISOString(); - const counts = {}; - for (const role of ['repeater', 'room', 'companion', 'sensor']) { - const r = db.db.prepare(`SELECT COUNT(*) as count FROM nodes WHERE role = ? AND last_seen > ?`).get(role, sevenDaysAgo); - counts[role + 's'] = r.count; - } - res.json({ ...stats, engine: 'node', version: APP_VERSION, commit: GIT_COMMIT, counts }); -}); - -app.get('/api/packets', (req, res) => { - const { limit = 50, offset = 0, type, route, region, observer, hash, since, until, groupByHash, node, nodes } = req.query; - const order = req.query.order === 'asc' ? 'ASC' : 'DESC'; - - // Multi-node filter: comma-separated pubkeys - if (nodes) { - const pubkeys = nodes.split(',').map(s => s.trim()).filter(Boolean); - const allPackets = new Map(); - for (const pk of pubkeys) { - const { packets: found } = pktStore.findPacketsForNode(pk); - for (const p of found) allPackets.set(p.id, p); - } - let results = [...allPackets.values()].sort((a, b) => order === 'DESC' ? b.timestamp.localeCompare(a.timestamp) : a.timestamp.localeCompare(b.timestamp)); - // Apply additional filters (type/observer filtering done client-side; server only filters for nodes query path) - if (region) results = results.filter(p => (p.observer_id || '').includes(region) || (p.decoded_json || '').includes(region)); - if (since) results = results.filter(p => p.timestamp >= since); - if (until) results = results.filter(p => p.timestamp <= until); - const total = results.length; - const paged = results.slice(Number(offset), Number(offset) + Number(limit)); - return res.json({ packets: paged, total, limit: Number(limit), offset: Number(offset) }); - } - - // groupByHash is now the default behavior (transmissions ARE grouped) — keep param for compat - if (groupByHash === 'true') { - return res.json(pktStore.queryGrouped({ limit, offset, type, route, region, observer, hash, since, until, node })); - } - - const expand = req.query.expand; - const result = pktStore.query({ limit, offset, type, route, region, observer, hash, since, until, node, order }); - - // Strip observations[] from default response for bandwidth; include with ?expand=observations - if (expand !== 'observations') { - result.packets = result.packets.map(p => { - const { observations, ...rest } = p; - return rest; - }); - } - - res.json(result); -}); - -// Lightweight endpoint: just timestamps for timeline sparkline -app.get('/api/packets/timestamps', (req, res) => { - const { since } = req.query; - if (!since) return res.status(400).json({ error: 'since required' }); - res.json(pktStore.getTimestamps(since)); -}); - -app.get('/api/packets/:id', (req, res) => { - const param = req.params.id; - const isHash = /^[0-9a-f]{16}$/i.test(param); - let packet; - if (isHash) { - // Hash-based lookup - const tx = pktStore.byHash.get(param.toLowerCase()); - packet = tx || null; - } - if (!packet) { - const id = Number(param); - if (!isNaN(id)) { - // Try transmission ID first (what the UI sends), then observation ID, then legacy - packet = pktStore.getByTxId(id) || pktStore.getById(id) || db.getPacket(id); - } - } - if (!packet) return res.status(404).json({ error: 'Not found' }); - - // Note: packet.path_json reflects the first observer's path (earliest first_seen). - // Individual observation paths are in siblingObservations below. - - const pathHops = packet.paths || []; - let decoded; - try { decoded = JSON.parse(packet.decoded_json); } catch { decoded = null; } - - // Build byte breakdown - const breakdown = buildBreakdown(packet.raw_hex, decoded); - - // Include sibling observations for this transmission - const transmission = packet.hash ? pktStore.byHash.get(packet.hash) : null; - const siblingObservations = transmission ? pktStore.enrichObservations(transmission.observations) : []; - const observation_count = transmission ? transmission.observation_count : 1; - - res.json({ packet, path: pathHops, breakdown, observation_count, observations: siblingObservations }); -}); - -function buildBreakdown(rawHex, decoded) { - return _buildBreakdown(rawHex, decoded, decoder.decodePacket, channelKeys); -} - -// Decode-only endpoint (no DB insert) -app.post('/api/decode', (req, res) => { - try { - const { hex } = req.body; - if (!hex) return res.status(400).json({ error: 'hex is required' }); - const decoded = decoder.decodePacket(hex.trim().replace(/\s+/g, ''), channelKeys); - res.json({ decoded }); - } catch (e) { - res.status(400).json({ error: e.message }); - } -}); - -app.post('/api/packets', requireApiKey, (req, res) => { - try { - const { hex, observer, snr, rssi, region, hash } = req.body; - if (!hex) return res.status(400).json({ error: 'hex is required' }); - - const decoded = decoder.decodePacket(hex, channelKeys); - const now = new Date().toISOString(); - - const apiPktData = { - raw_hex: hex.toUpperCase(), - timestamp: now, - observer_id: observer || null, - snr: snr ?? null, - rssi: rssi ?? null, - hash: computeContentHash(hex), - route_type: decoded.header.routeType, - payload_type: decoded.header.payloadType, - payload_version: decoded.header.payloadVersion, - path_json: JSON.stringify(decoded.path.hops), - decoded_json: JSON.stringify(decoded.payload), - }; - const packetId = pktStore.insert(apiPktData); _updateHashSizeForPacketLocal(apiPktData); - let txResult; - try { txResult = db.insertTransmission(apiPktData); } catch (e) { console.error('[dual-write] transmission insert error:', e.message); } - - if (decoded.path.hops.length > 0) { - const _now = new Date().toISOString(); - autoLearnHopNodes(decoded.path.hops, _now); - updatePathSeenTimestamps(decoded.path.hops, _now); - } - - if (decoded.header.payloadTypeName === 'ADVERT' && decoded.payload.pubKey) { - const p = decoded.payload; - const validation = decoder.validateAdvert(p); - if (validation.valid) { - const role = p.flags ? (p.flags.repeater ? 'repeater' : p.flags.room ? 'room' : p.flags.sensor ? 'sensor' : 'companion') : 'companion'; - db.upsertNode({ public_key: p.pubKey, name: p.name || null, role, lat: p.lat, lon: p.lon, last_seen: now }); - if (txResult && txResult.isNew) db.incrementAdvertCount(p.pubKey); - // Update telemetry if present in advert - if (p.battery_mv != null || p.temperature_c != null) { - db.updateNodeTelemetry({ public_key: p.pubKey, battery_mv: p.battery_mv ?? null, temperature_c: p.temperature_c ?? null }); - } - } else { - console.warn(`[advert] Skipping corrupted ADVERT (API): ${validation.reason}`); - } - } - - if (observer) { - db.upsertObserver({ id: observer, iata: region || null }); - } - - - // Invalidate caches on new data - cache.debouncedInvalidateAll(); - - broadcast({ type: 'packet', data: { id: packetId, hash: apiPktData.hash, raw: apiPktData.raw_hex, decoded } }); - - res.json({ id: packetId, decoded }); - } catch (e) { - res.status(400).json({ error: e.message }); - } -}); - -app.get('/api/nodes', (req, res) => { - const { limit = 50, offset = 0, role, region, lastHeard, sortBy = 'lastSeen', search, before } = req.query; - - let where = []; - let params = {}; - - if (role) { where.push('role = @role'); params.role = role; } - if (search) { where.push('name LIKE @search'); params.search = `%${search}%`; } - if (before) { where.push('first_seen <= @before'); params.before = before; } - if (lastHeard) { - const durations = { '1h': 3600000, '6h': 21600000, '24h': 86400000, '7d': 604800000, '30d': 2592000000 }; - const ms = durations[lastHeard]; - if (ms) { where.push('last_seen > @since'); params.since = new Date(Date.now() - ms).toISOString(); } - } - - // Region filtering: if region param is set, only include nodes whose ADVERTs were seen by regional observers - const regionObsIds = getObserverIdsForRegions(region); - let regionNodeKeys = null; - if (regionObsIds && regionObsIds.size > 0) { - regionNodeKeys = pktStore.getNodesByAdvertObservers(regionObsIds); - } - - const clause = where.length ? 'WHERE ' + where.join(' AND ') : ''; - const sortMap = { name: 'name ASC', lastSeen: 'last_seen DESC', packetCount: 'advert_count DESC' }; - const order = sortMap[sortBy] || 'last_seen DESC'; - - let nodes, total, filteredAll; - if (regionNodeKeys) { - const allNodes = db.db.prepare(`SELECT * FROM nodes ${clause} ORDER BY ${order}`).all(params); - filteredAll = allNodes.filter(n => regionNodeKeys.has(n.public_key)); - total = filteredAll.length; - nodes = filteredAll.slice(Number(offset), Number(offset) + Number(limit)); - } else { - nodes = db.db.prepare(`SELECT * FROM nodes ${clause} ORDER BY ${order} LIMIT @limit OFFSET @offset`).all({ ...params, limit: Number(limit), offset: Number(offset) }); - total = db.db.prepare(`SELECT COUNT(*) as count FROM nodes ${clause}`).get(params).count; - filteredAll = null; - } - - const counts = {}; - if (filteredAll) { - for (const r of ['repeater', 'room', 'companion', 'sensor']) { - counts[r + 's'] = filteredAll.filter(n => n.role === r).length; - } - } else { - for (const r of ['repeater', 'room', 'companion', 'sensor']) { - counts[r + 's'] = db.db.prepare(`SELECT COUNT(*) as count FROM nodes WHERE role = ?`).get(r).count; - } - } - - // Use precomputed hash_size map (rebuilt at startup, updated on new packets) - for (const node of nodes) { - node.hash_size = _hashSizeMap.get(node.public_key) || null; - const allSizes = _hashSizeAllMap.get(node.public_key); - node.hash_size_inconsistent = _isHashSizeFlipFlop(node.public_key); - if (allSizes && allSizes.size > 1) node.hash_sizes_seen = [...allSizes].sort(); - // Compute lastHeard from in-memory packets (more accurate than DB last_seen) - const nodePkts = pktStore.byNode.get(node.public_key); - if (nodePkts && nodePkts.length > 0) { - let latest = null; - for (const p of nodePkts) { - if (!latest || p.timestamp > latest) latest = p.timestamp; - } - if (latest) node.last_heard = latest; - } - // Also check if this node was seen as a relay hop in any packet path - const pathSeen = lastPathSeenMap.get(node.public_key); - if (pathSeen && (!node.last_heard || pathSeen > node.last_heard)) { - node.last_heard = pathSeen; - } - } - - res.json({ nodes, total, counts }); -}); - -app.get('/api/nodes/search', (req, res) => { - const q = req.query.q || ''; - if (!q.trim()) return res.json({ nodes: [] }); - const nodes = db.searchNodes(q.trim()); - res.json({ nodes }); -}); - -// Bulk health summary for analytics — single query approach (MUST be before :pubkey routes) -app.get('/api/nodes/bulk-health', (req, res) => { - const limit = Math.min(Number(req.query.limit) || 50, 200); - const regionKey = req.query.region || ''; - const _ck = 'bulk-health:' + limit + ':r=' + regionKey; - const _c = cache.get(_ck); if (_c) return res.json(_c); - - // Region filtering - const regionObsIds = getObserverIdsForRegions(req.query.region); - let regionNodeKeys = null; - let regionalHashes = null; - if (regionObsIds) { - regionalHashes = new Set(); - for (const obsId of regionObsIds) { - const obs = pktStore.byObserver.get(obsId); - if (obs) for (const o of obs) regionalHashes.add(o.hash); - } - regionNodeKeys = new Set(); - for (const [pubkey, hashes] of pktStore._nodeHashIndex) { - for (const h of hashes) { - if (regionalHashes.has(h)) { regionNodeKeys.add(pubkey); break; } - } - } - } - - let nodes = db.db.prepare(`SELECT * FROM nodes ORDER BY last_seen DESC LIMIT ?`).all(regionNodeKeys ? 500 : limit); - if (regionNodeKeys) { - nodes = nodes.filter(n => regionNodeKeys.has(n.public_key)).slice(0, limit); - } - if (nodes.length === 0) { cache.set(_ck, [], TTL.bulkHealth); return res.json([]); } - - const todayStart = new Date(); - todayStart.setUTCHours(0, 0, 0, 0); - const todayISO = todayStart.toISOString(); - - const results = []; - for (const node of nodes) { - const packets = pktStore.byNode.get(node.public_key) || []; - let packetsToday = 0, snrSum = 0, snrCount = 0, lastHeard = null; - const observers = {}; - let totalObservations = 0; - - for (const pkt of packets) { - totalObservations += pkt.observation_count || 1; - if (pkt.timestamp > todayISO) packetsToday++; - if (pkt.snr != null) { snrSum += pkt.snr; snrCount++; } - if (!lastHeard || pkt.timestamp > lastHeard) lastHeard = pkt.timestamp; - if (pkt.observer_id) { - if (!observers[pkt.observer_id]) { - observers[pkt.observer_id] = { name: pkt.observer_name, snrSum: 0, snrCount: 0, rssiSum: 0, rssiCount: 0, count: 0 }; - } - const obs = observers[pkt.observer_id]; - obs.count++; - if (pkt.snr != null) { obs.snrSum += pkt.snr; obs.snrCount++; } - if (pkt.rssi != null) { obs.rssiSum += pkt.rssi; obs.rssiCount++; } - } - } - - const observerRows = Object.entries(observers) - .map(([id, o]) => ({ - observer_id: id, observer_name: o.name, - avgSnr: o.snrCount ? o.snrSum / o.snrCount : null, - avgRssi: o.rssiCount ? o.rssiSum / o.rssiCount : null, - packetCount: o.count - })) - .sort((a, b) => b.packetCount - a.packetCount); - - results.push({ - public_key: node.public_key, name: node.name, role: node.role, - lat: node.lat, lon: node.lon, - stats: { - totalTransmissions: packets.length, - totalObservations, - totalPackets: packets.length, // backward compat - packetsToday, avgSnr: snrCount ? snrSum / snrCount : null, lastHeard - }, - observers: observerRows - }); - } - - cache.set(_ck, results, TTL.bulkHealth); - res.json(results); -}); - -app.get('/api/nodes/network-status', (req, res) => { - const now = Date.now(); - let allNodes = db.db.prepare('SELECT public_key, name, role, last_seen FROM nodes').all(); - - // Region filtering - const regionObsIds = getObserverIdsForRegions(req.query.region); - if (regionObsIds) { - const regionalHashes = new Set(); - for (const obsId of regionObsIds) { - const obs = pktStore.byObserver.get(obsId); - if (obs) for (const o of obs) regionalHashes.add(o.hash); - } - const regionNodeKeys = new Set(); - for (const [pubkey, hashes] of pktStore._nodeHashIndex) { - for (const h of hashes) { - if (regionalHashes.has(h)) { regionNodeKeys.add(pubkey); break; } - } - } - allNodes = allNodes.filter(n => regionNodeKeys.has(n.public_key)); - } - - let active = 0, degraded = 0, silent = 0; - const roleCounts = {}; - allNodes.forEach(n => { - const r = n.role || 'unknown'; - roleCounts[r] = (roleCounts[r] || 0) + 1; - const ls = n.last_seen ? new Date(n.last_seen).getTime() : 0; - const age = now - ls; - const isInfra = r === 'repeater' || r === 'room'; - const { degradedMs, silentMs } = getHealthMs(r); - if (age < degradedMs) active++; - else if (age < silentMs) degraded++; - else silent++; - }); - res.json({ total: allNodes.length, active, degraded, silent, roleCounts }); -}); - -app.get('/api/nodes/:pubkey', (req, res) => { - const pubkey = req.params.pubkey; - const _ck = 'node:' + pubkey; - const _c = cache.get(_ck); if (_c) return res.json(_c); - const node = db.db.prepare('SELECT * FROM nodes WHERE public_key = ?').get(pubkey); - if (!node) return res.status(404).json({ error: 'Not found' }); - node.hash_size = _hashSizeMap.get(pubkey) || null; - const allSizes = _hashSizeAllMap.get(pubkey); - node.hash_size_inconsistent = _isHashSizeFlipFlop(pubkey); - if (allSizes && allSizes.size > 1) node.hash_sizes_seen = [...allSizes].sort(); - const recentAdverts = (pktStore.byNode.get(pubkey) || []).slice(-20).reverse(); - const _nResult = { node, recentAdverts }; - cache.set(_ck, _nResult, TTL.nodeDetail); - res.json(_nResult); -}); - -// --- Analytics API --- -// --- RF Analytics --- -app.get('/api/analytics/rf', (req, res) => { - const { region } = req.query; - const regionObsIds = getObserverIdsForRegions(region); - const _ck = 'analytics:rf' + (region ? ':' + region : ''); - const _c = cache.get(_ck); if (_c) return res.json(_c); - const PTYPES = { 0:'REQ',1:'RESPONSE',2:'TXT_MSG',3:'ACK',4:'ADVERT',5:'GRP_TXT',7:'ANON_REQ',8:'PATH',9:'TRACE',11:'CONTROL' }; - - // Step 1: Get ALL regional observations (no SNR requirement) — for general stats - // Step 2: Filter by SNR for signal-specific stats - // When no region filter, use all transmissions directly for backward compat - let allRegional, signalPackets; - if (regionObsIds) { - // Collect observations from regional observers via byObserver index - allRegional = []; - for (const obsId of regionObsIds) { - const obs = pktStore.byObserver.get(obsId); - if (obs) allRegional.push(...obs); - } - signalPackets = allRegional.filter(p => p.snr != null); - } else { - // No region filter — flatten all observations from all transmissions - allRegional = []; - for (const tx of pktStore.packets) { - if (tx.observations && tx.observations.length) { - allRegional.push(...tx.observations); - } else { - allRegional.push(tx); // legacy packets without observations - } - } - signalPackets = allRegional.filter(p => p.snr != null); - } - - // Unique transmission hashes in the regional set - const regionalHashes = new Set(allRegional.map(p => p.hash).filter(Boolean)); - - const snrVals = signalPackets.map(p => p.snr).filter(v => v != null); - const rssiVals = signalPackets.map(p => p.rssi).filter(v => v != null); - // Packet sizes from ALL regional observations (use unique hashes to avoid double-counting) - const seenSizeHashes = new Set(); - const packetSizes = []; - for (const p of allRegional) { - const raw = p.raw_hex || (p.transmission_id ? (pktStore.byTxId.get(p.transmission_id) || {}).raw_hex : null); - if (raw && p.hash && !seenSizeHashes.has(p.hash)) { - seenSizeHashes.add(p.hash); - packetSizes.push(raw.length / 2); - } - } - - const sorted = arr => [...arr].sort((a, b) => a - b); - const median = arr => { const s = sorted(arr); return s.length ? s[Math.floor(s.length/2)] : 0; }; - const stddev = (arr, avg) => Math.sqrt(arr.reduce((s, v) => s + (v - avg) ** 2, 0) / Math.max(arr.length, 1)); - const arrMin = arr => { let m = Infinity; for (const v of arr) if (v < m) m = v; return m === Infinity ? 0 : m; }; - const arrMax = arr => { let m = -Infinity; for (const v of arr) if (v > m) m = v; return m === -Infinity ? 0 : m; }; - - const snrAvg = snrVals.reduce((a, b) => a + b, 0) / Math.max(snrVals.length, 1); - const rssiAvg = rssiVals.reduce((a, b) => a + b, 0) / Math.max(rssiVals.length, 1); - - // Packets per hour — from ALL regional observations - const hourBuckets = {}; - allRegional.forEach(p => { - const ts = p.timestamp || p.obs_timestamp; - if (!ts) return; - const hr = ts.slice(0, 13); - hourBuckets[hr] = (hourBuckets[hr] || 0) + 1; - }); - const packetsPerHour = Object.entries(hourBuckets).sort().map(([hour, count]) => ({ hour, count })); - - // Payload type distribution — from ALL regional (unique by hash to count transmissions) - const seenTypeHashes = new Set(); - const typeBuckets = {}; - allRegional.forEach(p => { - if (p.hash && !seenTypeHashes.has(p.hash)) { - seenTypeHashes.add(p.hash); - typeBuckets[p.payload_type] = (typeBuckets[p.payload_type] || 0) + 1; - } - }); - const payloadTypes = Object.entries(typeBuckets) - .map(([type, count]) => ({ type: +type, name: PTYPES[type] || `UNK(${type})`, count })) - .sort((a, b) => b.count - a.count); - - // SNR by payload type — from signal-filtered subset - const snrByType = {}; - signalPackets.forEach(p => { - const name = PTYPES[p.payload_type] || `UNK(${p.payload_type})`; - if (!snrByType[name]) snrByType[name] = { vals: [] }; - snrByType[name].vals.push(p.snr); - }); - const snrByTypeArr = Object.entries(snrByType).map(([name, d]) => ({ - name, count: d.vals.length, - avg: d.vals.reduce((a, b) => a + b, 0) / d.vals.length, - min: arrMin(d.vals), max: arrMax(d.vals) - })).sort((a, b) => b.count - a.count); - - // Signal over time — from signal-filtered subset - const sigTime = {}; - signalPackets.forEach(p => { - const ts = p.timestamp || p.obs_timestamp; - if (!ts) return; - const hr = ts.slice(0, 13); - if (!sigTime[hr]) sigTime[hr] = { snrs: [], count: 0 }; - sigTime[hr].snrs.push(p.snr); - sigTime[hr].count++; - }); - const signalOverTime = Object.entries(sigTime).sort().map(([hour, d]) => ({ - hour, count: d.count, avgSnr: d.snrs.reduce((a, b) => a + b, 0) / d.snrs.length - })); - - // Scatter data (SNR vs RSSI) — downsample to max 500 points - const scatterAll = signalPackets.filter(p => p.snr != null && p.rssi != null); - const scatterStep = Math.max(1, Math.floor(scatterAll.length / 500)); - const scatterData = scatterAll.filter((_, i) => i % scatterStep === 0).map(p => ({ snr: p.snr, rssi: p.rssi })); - - // Pre-compute histograms server-side so we don't send raw arrays - function buildHistogram(values, bins) { - if (!values.length) return { bins: [], min: 0, max: 0 }; - const min = arrMin(values), max = arrMax(values); - const range = max - min || 1; - const binWidth = range / bins; - const counts = new Array(bins).fill(0); - for (const v of values) { - const idx = Math.min(Math.floor((v - min) / binWidth), bins - 1); - counts[idx]++; - } - return { bins: counts.map((count, i) => ({ x: min + i * binWidth, w: binWidth, count })), min, max }; - } - - const snrHistogram = buildHistogram(snrVals, 20); - const rssiHistogram = buildHistogram(rssiVals, 20); - const sizeHistogram = buildHistogram(packetSizes, 25); - - const times = allRegional.map(p => new Date(p.timestamp || p.obs_timestamp).getTime()).filter(t => !isNaN(t)); - const timeSpanHours = times.length ? (arrMax(times) - arrMin(times)) / 3600000 : 0; - - const _rfResult = { - totalPackets: signalPackets.length, - totalAllPackets: allRegional.length, - totalTransmissions: regionalHashes.size, - snr: snrVals.length ? { min: arrMin(snrVals), max: arrMax(snrVals), avg: snrAvg, median: median(snrVals), stddev: stddev(snrVals, snrAvg) } : { min: 0, max: 0, avg: 0, median: 0, stddev: 0 }, - rssi: rssiVals.length ? { min: arrMin(rssiVals), max: arrMax(rssiVals), avg: rssiAvg, median: median(rssiVals), stddev: stddev(rssiVals, rssiAvg) } : { min: 0, max: 0, avg: 0, median: 0, stddev: 0 }, - snrValues: snrHistogram, rssiValues: rssiHistogram, packetSizes: sizeHistogram, - minPacketSize: packetSizes.length ? arrMin(packetSizes) : 0, - maxPacketSize: packetSizes.length ? arrMax(packetSizes) : 0, - avgPacketSize: packetSizes.length ? Math.round(packetSizes.reduce((a, b) => a + b, 0) / packetSizes.length) : 0, - packetsPerHour, payloadTypes, snrByType: snrByTypeArr, signalOverTime, scatterData, timeSpanHours - }; - cache.set(_ck, _rfResult, TTL.analyticsRF); - res.json(_rfResult); -}); - -// --- Topology Analytics --- -app.get('/api/analytics/topology', (req, res) => { - const { region } = req.query; - const regionObsIds = getObserverIdsForRegions(region); - const _ck = 'analytics:topology' + (region ? ':' + region : ''); - const _c = cache.get(_ck); if (_c) return res.json(_c); - const packets = pktStore.filter(p => p.path_json && p.path_json !== '[]' && (!regionObsIds || regionObsIds.has(p.observer_id))); - const allNodes = getCachedNodes(false); - - // Build prefix map for O(1) hop resolution (same pattern as distance endpoint) - const topoPrefixMap = new Map(); - for (const n of allNodes) { - const pk = n.public_key.toLowerCase(); - for (let len = 2; len <= pk.length; len++) { - const pfx = pk.slice(0, len); - if (!topoPrefixMap.has(pfx)) topoPrefixMap.set(pfx, []); - topoPrefixMap.get(pfx).push(n); - } - } - const topoHopCache = new Map(); - const resolveHop = (hop, contextPositions) => { - if (topoHopCache.has(hop)) return topoHopCache.get(hop); - const h = hop.toLowerCase(); - const candidates = topoPrefixMap.get(h); - if (!candidates || candidates.length === 0) { topoHopCache.set(hop, null); return null; } - let result; - if (candidates.length === 1) { result = { name: candidates[0].name, pubkey: candidates[0].public_key }; } - else if (contextPositions && contextPositions.length > 0) { - const cLat = contextPositions.reduce((s, p) => s + p.lat, 0) / contextPositions.length; - const cLon = contextPositions.reduce((s, p) => s + p.lon, 0) / contextPositions.length; - const withLoc = candidates.filter(c => c.lat && c.lon && !(c.lat === 0 && c.lon === 0)); - if (withLoc.length) { - withLoc.sort((a, b) => Math.hypot(a.lat - cLat, a.lon - cLon) - Math.hypot(b.lat - cLat, b.lon - cLon)); - result = { name: withLoc[0].name, pubkey: withLoc[0].public_key }; - } else { result = { name: candidates[0].name, pubkey: candidates[0].public_key }; } - } else { result = { name: candidates[0].name, pubkey: candidates[0].public_key }; } - // Only cache when no context positions (context-dependent results vary) - if (!contextPositions || contextPositions.length === 0) topoHopCache.set(hop, result); - return result; - }; - - // Hop distribution - const hopCounts = {}; - const allHopsList = []; - const hopSnr = {}; - const hopFreq = {}; - const pairFreq = {}; - packets.forEach(p => { - const hops = p._parsedPath || (p._parsedPath = JSON.parse(p.path_json)); - const n = hops.length; - hopCounts[n] = (hopCounts[n] || 0) + 1; - allHopsList.push(n); - if (!hopSnr[n]) hopSnr[n] = []; - if (p.snr != null) hopSnr[n].push(p.snr); - hops.forEach(h => { hopFreq[h] = (hopFreq[h] || 0) + 1; }); - for (let i = 0; i < hops.length - 1; i++) { - const pair = [hops[i], hops[i + 1]].sort().join('|'); - pairFreq[pair] = (pairFreq[pair] || 0) + 1; - } - }); - - const hopDistribution = Object.entries(hopCounts) - .map(([hops, count]) => ({ hops: +hops, count })) - .filter(h => h.hops <= 25) - .sort((a, b) => a.hops - b.hops); - - const avgHops = allHopsList.length ? allHopsList.reduce((a, b) => a + b, 0) / allHopsList.length : 0; - const medianHops = allHopsList.length ? [...allHopsList].sort((a, b) => a - b)[Math.floor(allHopsList.length / 2)] : 0; - const maxHops = allHopsList.length ? Math.max(...allHopsList) : 0; - - // Top repeaters - const topRepeaters = Object.entries(hopFreq) - .sort((a, b) => b[1] - a[1]) - .slice(0, 20) - .map(([hop, count]) => { - const resolved = resolveHop(hop); - return { hop, count, name: resolved?.name || null, pubkey: resolved?.pubkey || null }; - }); - - // Top pairs - const topPairs = Object.entries(pairFreq) - .sort((a, b) => b[1] - a[1]) - .slice(0, 15) - .map(([pair, count]) => { - const [a, b] = pair.split('|'); - const rA = resolveHop(a), rB = resolveHop(b); - return { hopA: a, hopB: b, count, nameA: rA?.name, nameB: rB?.name, pubkeyA: rA?.pubkey, pubkeyB: rB?.pubkey }; - }); - - // Hops vs SNR - const hopsVsSnr = Object.entries(hopSnr) - .filter(([h]) => +h <= 20) - .map(([hops, snrs]) => ({ - hops: +hops, count: snrs.length, - avgSnr: snrs.reduce((a, b) => a + b, 0) / snrs.length - })) - .sort((a, b) => a.hops - b.hops); - - // Reachability: per-observer hop distances + cross-observer comparison + best path - const observerMap = new Map(); packets.forEach(p => { if (p.observer_id) observerMap.set(p.observer_id, p.observer_name); }); const observers = [...observerMap].map(([observer_id, observer_name]) => ({ observer_id, observer_name })); - - // Per-observer: node → min hop distance seen from that observer - const perObserver = {}; // observer_id → { hop_hex → { minDist, maxDist, count } } - const bestPath = {}; // hop_hex → { minDist, observer } - const crossObserver = {}; // hop_hex → [ { observer_id, observer_name, minDist, count } ] - - packets.forEach(p => { - const obsId = p.observer_id; - if (!perObserver[obsId]) perObserver[obsId] = {}; - const hops = p._parsedPath || (p._parsedPath = JSON.parse(p.path_json)); - hops.forEach((h, i) => { - const dist = hops.length - i; - if (!perObserver[obsId][h]) perObserver[obsId][h] = { minDist: dist, maxDist: dist, count: 0 }; - const entry = perObserver[obsId][h]; - entry.minDist = Math.min(entry.minDist, dist); - entry.maxDist = Math.max(entry.maxDist, dist); - entry.count++; - }); - }); - - // Build cross-observer and best-path from perObserver - for (const [obsId, nodes] of Object.entries(perObserver)) { - const obsName = observers.find(o => o.observer_id === obsId)?.observer_name || obsId; - for (const [hop, data] of Object.entries(nodes)) { - // Cross-observer - if (!crossObserver[hop]) crossObserver[hop] = []; - crossObserver[hop].push({ observer_id: obsId, observer_name: obsName, minDist: data.minDist, count: data.count }); - // Best path - if (!bestPath[hop] || data.minDist < bestPath[hop].minDist) { - bestPath[hop] = { minDist: data.minDist, observer_id: obsId, observer_name: obsName }; - } - } - } - - // Format per-observer reachability (grouped by distance) - const perObserverReach = {}; - for (const [obsId, nodes] of Object.entries(perObserver)) { - const obsInfo = observers.find(o => o.observer_id === obsId); - const byDist = {}; - for (const [hop, data] of Object.entries(nodes)) { - const d = data.minDist; - if (d > 15) continue; - if (!byDist[d]) byDist[d] = []; - const r = resolveHop(hop); - byDist[d].push({ hop, name: r?.name || null, pubkey: r?.pubkey || null, count: data.count, distRange: data.minDist === data.maxDist ? null : `${data.minDist}-${data.maxDist}` }); - } - perObserverReach[obsId] = { - observer_name: obsInfo?.observer_name || obsId, - rings: Object.entries(byDist).map(([dist, nodes]) => ({ hops: +dist, nodes: nodes.sort((a, b) => b.count - a.count) })).sort((a, b) => a.hops - b.hops) - }; - } - - // Cross-observer: nodes seen by multiple observers - const multiObsNodes = Object.entries(crossObserver) - .filter(([, obs]) => obs.length > 1) - .map(([hop, obs]) => { - const r = resolveHop(hop); - return { hop, name: r?.name || null, pubkey: r?.pubkey || null, observers: obs.sort((a, b) => a.minDist - b.minDist) }; - }) - .sort((a, b) => b.observers.length - a.observers.length) - .slice(0, 50); - - // Best path: sorted by distance - const bestPathList = Object.entries(bestPath) - .map(([hop, data]) => { - const r = resolveHop(hop); - return { hop, name: r?.name || null, pubkey: r?.pubkey || null, ...data }; - }) - .sort((a, b) => a.minDist - b.minDist) - .slice(0, 50); - - const _topoResult = { - uniqueNodes: db.getStats().totalNodes, - avgHops, medianHops, maxHops, - hopDistribution, topRepeaters, topPairs, hopsVsSnr, - observers: observers.map(o => ({ id: o.observer_id, name: o.observer_name || o.observer_id })), - perObserverReach, - multiObsNodes, - bestPathList - }; - cache.set(_ck, _topoResult, TTL.analyticsTopology); - res.json(_topoResult); -}); - -// --- Channel Analytics --- -app.get('/api/analytics/channels', (req, res) => { - const { region } = req.query; - const regionObsIds = getObserverIdsForRegions(region); - const _ck = 'analytics:channels' + (region ? ':' + region : ''); - const _c = cache.get(_ck); if (_c) return res.json(_c); - const packets = pktStore.filter(p => p.payload_type === 5 && p.decoded_json && (!regionObsIds || regionObsIds.has(p.observer_id))); - - const channels = {}; - const senderCounts = {}; - const msgLengths = []; - const timeline = {}; - - packets.forEach(p => { - try { - const d = p._parsedDecoded || (p._parsedDecoded = typeof p.decoded_json === 'string' ? JSON.parse(p.decoded_json) : p.decoded_json); - const hash = d.channelHash || d.channel_hash || '?'; - const name = d.channelName || (d.type === 'CHAN' ? (d.channel || `ch${hash}`) : `ch${hash}`); - const encrypted = !d.text && !d.sender; - // Use channel name as key when available to distinguish channels with same hash (#108) - const chKey = (d.type === 'CHAN' && d.channel) ? `${hash}_${d.channel}` : String(hash); - - if (!channels[chKey]) channels[chKey] = { hash, name, messages: 0, senders: new Set(), lastActivity: p.timestamp, encrypted }; - channels[chKey].messages++; - channels[chKey].lastActivity = p.timestamp; - if (!encrypted) channels[chKey].encrypted = false; - - if (d.sender) { - channels[chKey].senders.add(d.sender); - senderCounts[d.sender] = (senderCounts[d.sender] || 0) + 1; - } - if (d.text) msgLengths.push(d.text.length); - - // Timeline - const hr = p.timestamp.slice(0, 13); - const key = hr + '|' + (name || `ch${hash}`); - timeline[key] = (timeline[key] || 0) + 1; - } catch {} - }); - - const channelList = Object.values(channels) - .map(c => ({ ...c, senders: c.senders.size })) - .sort((a, b) => b.messages - a.messages); - - const topSenders = Object.entries(senderCounts) - .sort((a, b) => b[1] - a[1]) - .slice(0, 15) - .map(([name, count]) => ({ name, count })); - - const channelTimeline = Object.entries(timeline) - .map(([key, count]) => { - const [hour, channel] = key.split('|'); - return { hour, channel, count }; - }) - .sort((a, b) => a.hour.localeCompare(b.hour)); - - const _chanResult = { - activeChannels: channelList.length, - decryptable: channelList.filter(c => !c.encrypted).length, - channels: channelList, - topSenders, - channelTimeline, - msgLengths - }; - cache.set(_ck, _chanResult, TTL.analyticsChannels); - res.json(_chanResult); -}); - -app.get('/api/analytics/distance', (req, res) => { - const { region } = req.query; - const regionObsIds = getObserverIdsForRegions(region); - const _ck = 'analytics:distance' + (region ? ':' + region : ''); - const _c = cache.get(_ck); if (_c) return res.json(_c); - - const arrMin = arr => { let m = Infinity; for (const v of arr) if (v < m) m = v; return m === Infinity ? 0 : m; }; - const arrMax = arr => { let m = -Infinity; for (const v of arr) if (v > m) m = v; return m === -Infinity ? 0 : m; }; - const median = arr => { if (!arr.length) return 0; const s = [...arr].sort((a,b)=>a-b); return s[Math.floor(s.length/2)]; }; - - function haversine(lat1, lon1, lat2, lon2) { - const R = 6371; - const dLat = (lat2 - lat1) * Math.PI / 180; - const dLon = (lon2 - lon1) * Math.PI / 180; - const a = Math.sin(dLat/2)**2 + Math.cos(lat1*Math.PI/180) * Math.cos(lat2*Math.PI/180) * Math.sin(dLon/2)**2; - return R * 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1-a)); - } - - const allNodes = getCachedNodes(true); - const nodeByPk = new Map(allNodes.map(n => [n.public_key, n])); - - // Build prefix map for O(1) hop resolution instead of O(N) filter per hop - const prefixMap = new Map(); // lowercase prefix → [nodes] - for (const n of allNodes) { - const pk = n.public_key.toLowerCase(); - // Index all prefixes from length 2 to full key length - for (let len = 2; len <= pk.length; len++) { - const pfx = pk.slice(0, len); - if (!prefixMap.has(pfx)) prefixMap.set(pfx, []); - prefixMap.get(pfx).push(n); - } - } - - // Cache resolved hops to avoid re-resolving same hex prefix - const hopCache = new Map(); - const resolveHop = (hop) => { - if (hopCache.has(hop)) return hopCache.get(hop); - const h = hop.toLowerCase(); - const candidates = prefixMap.get(h); - let result = null; - if (candidates && candidates.length === 1) result = candidates[0]; - else if (candidates && candidates.length > 1) { - const withLoc = candidates.filter(c => c.lat && c.lon && !(c.lat === 0 && c.lon === 0)); - result = withLoc.length ? withLoc[0] : candidates[0]; - } - hopCache.set(hop, result); - return result; - }; - - // Pre-compute repeater status - const repeaterSet = new Set(); - for (const n of allNodes) { - if (n.role && n.role.toLowerCase().includes('repeater')) repeaterSet.add(n.public_key); - } - const validGps = n => n && n.lat != null && n.lon != null && !(n.lat === 0 && n.lon === 0); - const isRepeater = n => n && repeaterSet.has(n.public_key); - - const packets = pktStore.filter(p => p.path_json && p.path_json !== '[]' && (!regionObsIds || regionObsIds.has(p.observer_id))); - - // Collect hops with distances - const allHops = []; // { from, to, dist, type, snr, hash, timestamp } - const pathTotals = []; // { hash, totalDist, hopCount, timestamp, hops: [{from,to,dist}] } - const catDists = { 'R↔R': [], 'C↔R': [], 'C↔C': [] }; - const distByHour = {}; // hourBucket → [distances] - - for (const p of packets) { - let hops; - try { - hops = p._parsedPath || (p._parsedPath = JSON.parse(p.path_json)); - } catch { continue; } - if (!hops.length) continue; - - // Resolve all hops to nodes - const resolved = hops.map(h => resolveHop(h)); - - // Also try to resolve sender from decoded_json - let senderNode = null; - if (p.decoded_json) { - try { - const dec = p._parsedDecoded || (p._parsedDecoded = typeof p.decoded_json === 'string' ? JSON.parse(p.decoded_json) : p.decoded_json); - if (dec.pubKey) senderNode = nodeByPk.get(dec.pubKey) || null; - } catch {} - } - - // Build chain: sender → hop0 → hop1 → ... → observer - // For distance we only measure consecutive hops where both have valid GPS - const chain = []; - if (senderNode && validGps(senderNode)) chain.push(senderNode); - for (const r of resolved) { - if (r && validGps(r)) chain.push(r); - } - - if (chain.length < 2) continue; - - const hourBucket = p.timestamp ? new Date(p.timestamp).toISOString().slice(0, 13) : null; - let pathDist = 0; - const pathHops = []; - - for (let i = 0; i < chain.length - 1; i++) { - const a = chain[i], b = chain[i + 1]; - const dist = haversine(a.lat, a.lon, b.lat, b.lon); - if (dist > 300) continue; // sanity: skip > 300km (LoRa record ~250km) - - const aRep = isRepeater(a), bRep = isRepeater(b); - let type; - if (aRep && bRep) type = 'R↔R'; - else if (!aRep && !bRep) type = 'C↔C'; - else type = 'C↔R'; - - const hop = { fromName: a.name, fromPk: a.public_key, toName: b.name, toPk: b.public_key, dist: Math.round(dist * 100) / 100, type, snr: p.snr || null, hash: p.hash, timestamp: p.timestamp }; - allHops.push(hop); - catDists[type].push(dist); - pathDist += dist; - pathHops.push({ fromName: a.name, fromPk: a.public_key, toName: b.name, toPk: b.public_key, dist: hop.dist }); - - if (hourBucket) { - if (!distByHour[hourBucket]) distByHour[hourBucket] = []; - distByHour[hourBucket].push(dist); - } - } - - if (pathHops.length > 0) { - pathTotals.push({ hash: p.hash, totalDist: Math.round(pathDist * 100) / 100, hopCount: pathHops.length, timestamp: p.timestamp, hops: pathHops }); - } - } - - // Top longest hops - allHops.sort((a, b) => b.dist - a.dist); - const topHops = allHops.slice(0, 50); - - // Top longest paths - pathTotals.sort((a, b) => b.totalDist - a.totalDist); - const topPaths = pathTotals.slice(0, 20); - - // Category stats - const catStats = {}; - for (const [cat, dists] of Object.entries(catDists)) { - if (!dists.length) { catStats[cat] = { count: 0, avg: 0, median: 0, min: 0, max: 0 }; continue; } - const avg = dists.reduce((s, v) => s + v, 0) / dists.length; - catStats[cat] = { count: dists.length, avg: Math.round(avg * 100) / 100, median: Math.round(median(dists) * 100) / 100, min: Math.round(arrMin(dists) * 100) / 100, max: Math.round(arrMax(dists) * 100) / 100 }; - } - - // Histogram of all hop distances - const allDists = allHops.map(h => h.dist); - let distHistogram = []; - if (allDists.length) { - const hMin = arrMin(allDists), hMax = arrMax(allDists); - const binCount = 25; - const binW = (hMax - hMin) / binCount || 1; - const bins = new Array(binCount).fill(0); - for (const d of allDists) { - const idx = Math.min(Math.floor((d - hMin) / binW), binCount - 1); - bins[idx]++; - } - distHistogram = { bins: bins.map((count, i) => ({ x: Math.round((hMin + i * binW) * 10) / 10, w: Math.round(binW * 10) / 10, count })), min: hMin, max: hMax }; - } - - // Distance over time - const timeEntries = Object.entries(distByHour).sort((a, b) => a[0].localeCompare(b[0])); - const distOverTime = timeEntries.map(([hour, dists]) => ({ - hour, - avg: Math.round((dists.reduce((s, v) => s + v, 0) / dists.length) * 100) / 100, - count: dists.length - })); - - // Summary - const totalDists = allHops.map(h => h.dist); - const summary = { - totalHops: allHops.length, - totalPaths: pathTotals.length, - avgDist: totalDists.length ? Math.round((totalDists.reduce((s, v) => s + v, 0) / totalDists.length) * 100) / 100 : 0, - maxDist: totalDists.length ? Math.round(arrMax(totalDists) * 100) / 100 : 0, - }; - - const _distResult = { summary, topHops, topPaths, catStats, distHistogram, distOverTime }; - cache.set(_ck, _distResult, TTL.analyticsTopology); - res.json(_distResult); -}); - -app.get('/api/analytics/hash-sizes', (req, res) => { - const { region } = req.query; - const regionObsIds = getObserverIdsForRegions(region); - const _ck = 'analytics:hash-sizes' + (region ? ':' + region : ''); - const _c = cache.get(_ck); if (_c) return res.json(_c); - // Get all packets with raw_hex and non-empty paths from memory store - const packets = pktStore.filter(p => p.raw_hex && p.path_json && p.path_json !== '[]' && (!regionObsIds || regionObsIds.has(p.observer_id))); - - const distribution = { 1: 0, 2: 0, 3: 0 }; - const byHour = {}; // hour bucket → { 1: n, 2: n, 3: n } - const byNode = {}; // node name/prefix → { hashSize, packets, lastSeen } - const uniqueHops = {}; // hop hex → { size, count, resolvedName } - - // Resolve all known nodes for hop matching — use prefix map for O(1) lookup - const allNodes = getCachedNodes(false); - const hsPrefixMap = new Map(); - for (const n of allNodes) { - const pk = n.public_key.toLowerCase(); - for (let len = 2; len <= pk.length; len++) { - const pfx = pk.slice(0, len); - if (!hsPrefixMap.has(pfx)) hsPrefixMap.set(pfx, []); - hsPrefixMap.get(pfx).push(n); - } - } - - for (const p of packets) { - const pathByte = parseInt(p.raw_hex.slice(2, 4), 16); - // Check if this packet has transport codes (route type 0 or 3) - const header = parseInt(p.raw_hex.slice(0, 2), 16); - const routeType = header & 0x03; - let pathByteIdx = 1; // normally byte index 1 - if (routeType === 0 || routeType === 3) pathByteIdx = 5; // skip 4 transport code bytes - const actualPathByte = parseInt(p.raw_hex.slice(pathByteIdx * 2, pathByteIdx * 2 + 2), 16); - - const hashSize = ((actualPathByte >> 6) & 0x3) + 1; - const hashCount = actualPathByte & 0x3F; - if (hashSize > 3) continue; // reserved - - distribution[hashSize] = (distribution[hashSize] || 0) + 1; - - // Hourly buckets - const hour = p.timestamp.slice(0, 13); // "2026-03-18T04" - if (!byHour[hour]) byHour[hour] = { 1: 0, 2: 0, 3: 0 }; - byHour[hour][hashSize]++; - - // Track unique hops with their sizes - const hops = JSON.parse(p.path_json); - for (const hop of hops) { - if (!uniqueHops[hop]) { - const hopLower = hop.toLowerCase(); - const candidates = hsPrefixMap.get(hopLower); - const match = candidates && candidates.length ? candidates[0] : null; - uniqueHops[hop] = { size: Math.ceil(hop.length / 2), count: 0, name: match?.name || null, pubkey: match?.public_key || null }; - } - uniqueHops[hop].count++; - } - - // Try to identify originator from decoded_json for advert packets - if (p.payload_type === 4) { - try { - const d = p._parsedDecoded || (p._parsedDecoded = typeof p.decoded_json === 'string' ? JSON.parse(p.decoded_json) : p.decoded_json); - const name = d.name || (d.pubKey || d.public_key || '').slice(0, 8); - if (name) { - if (!byNode[name]) byNode[name] = { hashSize, packets: 0, lastSeen: p.timestamp, pubkey: d.pubKey || d.public_key || null }; - byNode[name].packets++; - byNode[name].hashSize = hashSize; - byNode[name].lastSeen = p.timestamp; - } - } catch {} - } - } - - // Sort hourly data - const hourly = Object.entries(byHour) - .sort(([a], [b]) => a.localeCompare(b)) - .map(([hour, sizes]) => ({ hour, ...sizes })); - - // Top hops by frequency - const topHops = Object.entries(uniqueHops) - .sort(([, a], [, b]) => b.count - a.count) - .slice(0, 50) - .map(([hex, data]) => ({ hex, ...data })); - - // Nodes that use non-default (>1 byte) hash sizes - const multiByteNodes = Object.entries(byNode) - .filter(([, v]) => v.hashSize > 1) - .sort(([, a], [, b]) => b.packets - a.packets) - .map(([name, data]) => ({ name, ...data })); - - const _hsResult = { - total: packets.length, - distribution, - hourly, - topHops, - multiByteNodes - }; - cache.set(_ck, _hsResult, TTL.analyticsHashSizes); - res.json(_hsResult); -}); - -// Resolve path hop hex prefixes to node names -app.get('/api/resolve-hops', (req, res) => { - const hops = (req.query.hops || '').split(',').filter(Boolean); - const observerId = req.query.observer || null; - const originLat = req.query.originLat ? parseFloat(req.query.originLat) : null; - const originLon = req.query.originLon ? parseFloat(req.query.originLon) : null; - if (!hops.length) return res.json({ resolved: {} }); - - const allNodes = getCachedNodes(false); - const allObservers = db.getObservers(); - - // Build observer IATA lookup and regional observer sets - const observerIataMap = {}; // observer_id → iata - const observersByIata = {}; // iata → Set - for (const obs of allObservers) { - if (obs.iata) { - observerIataMap[obs.id] = obs.iata; - if (!observersByIata[obs.iata]) observersByIata[obs.iata] = new Set(); - observersByIata[obs.iata].add(obs.id); - } - } - - // Determine this packet's region from its observer - const packetIata = observerId ? observerIataMap[observerId] : null; - const regionalObserverIds = packetIata ? observersByIata[packetIata] : null; - - // Helper: check if a node is near the packet's region using layered filtering - // Layer 1: Node has lat/lon → geographic distance to IATA center (bridge-proof) - // Layer 2: Node has no lat/lon → observer-based (was ADVERT seen by regional observer) - // Returns: { near: boolean, method: 'geo'|'observer'|'none', distKm?: number } - const nodeInRegion = (candidate) => { - // Layer 1: Geographic check (ground truth, bridge-proof) - if (packetIata && candidate.lat && candidate.lon && !(candidate.lat === 0 && candidate.lon === 0)) { - const geoCheck = nodeNearRegion(candidate.lat, candidate.lon, packetIata); - if (geoCheck) return { near: geoCheck.near, method: 'geo', distKm: geoCheck.distKm }; - } - // Layer 2: Observer-based check (fallback for nodes without GPS) - if (regionalObserverIds) { - const nodeObservers = pktStore._advertByObserver.get(candidate.public_key); - if (nodeObservers) { - for (const obsId of nodeObservers) { - if (regionalObserverIds.has(obsId)) return { near: true, method: 'observer' }; - } - } - return { near: false, method: 'observer' }; - } - // No region info available - return { near: false, method: 'none' }; - }; - - // Build observer geographic position - let observerLat = null, observerLon = null; - if (observerId) { - const obsNode = allNodes.find(n => n.name === observerId); - if (obsNode && obsNode.lat && obsNode.lon && !(obsNode.lat === 0 && obsNode.lon === 0)) { - observerLat = obsNode.lat; - observerLon = obsNode.lon; - } else { - const obsNodes = db.db.prepare(` - SELECT n.lat, n.lon FROM packets_v p - JOIN nodes n ON n.public_key = json_extract(p.decoded_json, '$.pubKey') - WHERE (p.observer_id = ? OR p.observer_name = ?) - AND p.payload_type = 4 - AND n.lat IS NOT NULL AND n.lat != 0 AND n.lon != 0 - GROUP BY n.public_key - ORDER BY COUNT(*) DESC - LIMIT 20 - `).all(observerId, observerId); - if (obsNodes.length) { - observerLat = obsNodes.reduce((s, n) => s + n.lat, 0) / obsNodes.length; - observerLon = obsNodes.reduce((s, n) => s + n.lon, 0) / obsNodes.length; - } - } - } - - const resolved = {}; - // First pass: find all candidates for each hop, split into regional and global - for (const hop of hops) { - const hopLower = hop.toLowerCase(); - const hopByteLen = Math.ceil(hop.length / 2); // 2 hex chars = 1 byte - const allCandidates = allNodes.filter(n => n.public_key.toLowerCase().startsWith(hopLower)); - - if (allCandidates.length === 0) { - resolved[hop] = { name: null, candidates: [], conflicts: [] }; - } else if (allCandidates.length === 1) { - const c = allCandidates[0]; - const regionCheck = nodeInRegion(c); - resolved[hop] = { name: c.name, pubkey: c.public_key, - candidates: [{ name: c.name, pubkey: c.public_key, lat: c.lat, lon: c.lon, regional: regionCheck.near, filterMethod: regionCheck.method, distKm: regionCheck.distKm }], - conflicts: [] }; - } else { - // Multiple candidates — apply layered regional filtering - const checked = allCandidates.map(c => { - const r = nodeInRegion(c); - return { ...c, regional: r.near, filterMethod: r.method, distKm: r.distKm }; - }); - const regional = checked.filter(c => c.regional); - // Sort by distance to region center — closest first - regional.sort((a, b) => (a.distKm || 9999) - (b.distKm || 9999)); - const candidates = regional.length > 0 ? regional : checked; - const globalFallback = regional.length === 0 && checked.length > 0; - - const conflicts = candidates.map(c => ({ - name: c.name, pubkey: c.public_key, lat: c.lat, lon: c.lon, - regional: c.regional, filterMethod: c.filterMethod, distKm: c.distKm - })); - - if (candidates.length === 1) { - resolved[hop] = { name: candidates[0].name, pubkey: candidates[0].public_key, - candidates: conflicts, conflicts, globalFallback, - filterMethod: candidates[0].filterMethod }; - } else { - resolved[hop] = { name: candidates[0].name, pubkey: candidates[0].public_key, - ambiguous: true, candidates: conflicts, conflicts, globalFallback, - hopBytes: hopByteLen, totalGlobal: allCandidates.length, totalRegional: regional.length, - filterMethods: [...new Set(candidates.map(c => c.filterMethod))] }; - } - } - } - - const dist = (lat1, lon1, lat2, lon2) => Math.sqrt((lat1 - lat2) ** 2 + (lon1 - lon2) ** 2); - - // Forward pass: resolve each ambiguous hop using previous hop's position - const hopPositions = {}; - // Seed unambiguous positions - for (const hop of hops) { - const r = resolved[hop]; - if (r && !r.ambiguous && r.pubkey) { - const node = allNodes.find(n => n.public_key === r.pubkey); - if (node && node.lat && node.lon && !(node.lat === 0 && node.lon === 0)) { - hopPositions[hop] = { lat: node.lat, lon: node.lon }; - } - } - } - - let lastPos = (originLat != null && originLon != null) ? { lat: originLat, lon: originLon } : null; - for (let hi = 0; hi < hops.length; hi++) { - const hop = hops[hi]; - if (hopPositions[hop]) { - lastPos = hopPositions[hop]; - continue; - } - const r = resolved[hop]; - if (!r || !r.ambiguous) continue; - const withLoc = r.candidates.filter(c => c.lat && c.lon && !(c.lat === 0 && c.lon === 0)); - if (!withLoc.length) continue; - - let anchor = lastPos; - if (!anchor && hi === hops.length - 1 && observerLat != null) { - anchor = { lat: observerLat, lon: observerLon }; - } - if (anchor) { - withLoc.sort((a, b) => dist(a.lat, a.lon, anchor.lat, anchor.lon) - dist(b.lat, b.lon, anchor.lat, anchor.lon)); - } - r.name = withLoc[0].name; - r.pubkey = withLoc[0].pubkey; - hopPositions[hop] = { lat: withLoc[0].lat, lon: withLoc[0].lon }; - lastPos = hopPositions[hop]; - } - - // Backward pass: resolve any remaining ambiguous hops using next hop's position - let nextPos = observerLat != null ? { lat: observerLat, lon: observerLon } : null; - for (let hi = hops.length - 1; hi >= 0; hi--) { - const hop = hops[hi]; - if (hopPositions[hop]) { - nextPos = hopPositions[hop]; - continue; - } - const r = resolved[hop]; - if (!r || !r.ambiguous) continue; - const withLoc = r.candidates.filter(c => c.lat && c.lon && !(c.lat === 0 && c.lon === 0)); - if (!withLoc.length || !nextPos) continue; - withLoc.sort((a, b) => dist(a.lat, a.lon, nextPos.lat, nextPos.lon) - dist(b.lat, b.lon, nextPos.lat, nextPos.lon)); - r.name = withLoc[0].name; - r.pubkey = withLoc[0].pubkey; - hopPositions[hop] = { lat: withLoc[0].lat, lon: withLoc[0].lon }; - nextPos = hopPositions[hop]; - } - - // Sanity check: drop hops impossibly far from both neighbors - const MAX_HOP_DIST = MAX_HOP_DIST_SERVER; - for (let i = 0; i < hops.length; i++) { - const pos = hopPositions[hops[i]]; - if (!pos) continue; - const prev = i > 0 ? hopPositions[hops[i-1]] : null; - const next = i < hops.length-1 ? hopPositions[hops[i+1]] : null; - if (!prev && !next) continue; - const dPrev = prev ? dist(pos.lat, pos.lon, prev.lat, prev.lon) : 0; - const dNext = next ? dist(pos.lat, pos.lon, next.lat, next.lon) : 0; - const tooFarPrev = prev && dPrev > MAX_HOP_DIST; - const tooFarNext = next && dNext > MAX_HOP_DIST; - if ((tooFarPrev && tooFarNext) || (tooFarPrev && !next) || (tooFarNext && !prev)) { - const r = resolved[hops[i]]; - if (r) { r.unreliable = true; } - delete hopPositions[hops[i]]; - } - } - - res.json({ resolved, region: packetIata || null }); -}); - -// channelHashNames removed — we only use decoded channel names now - -app.get('/api/channels', (req, res) => { - const { region } = req.query; - const regionObsIds = getObserverIdsForRegions(region); - const _ck = 'channels' + (region ? ':' + region : ''); - const _c = cache.get(_ck); if (_c) return res.json(_c); - // Single pass: only scan type-5 packets via filter (already in memory) - const channelMap = {}; - - for (const pkt of pktStore.all()) { - if (pkt.payload_type !== 5) continue; - if (regionObsIds && !regionObsIds.has(pkt.observer_id)) continue; - let decoded; - try { decoded = JSON.parse(pkt.decoded_json); } catch { continue; } - - // Only show decrypted messages — skip encrypted garbage - if (decoded.type !== 'CHAN') continue; - - const channelName = decoded.channel || 'unknown'; - // Filter out garbage-decrypted channel names (pre-#197 data still in DB) - if (hasNonPrintableChars(channelName)) continue; - if (hasNonPrintableChars(decoded.text)) continue; - const key = channelName; - - if (!channelMap[key]) { - channelMap[key] = { - hash: key, - name: channelName, - lastMessage: null, lastSender: null, messageCount: 0, lastActivity: pkt.timestamp, - }; - } - channelMap[key].messageCount++; - if (pkt.timestamp >= channelMap[key].lastActivity) { - channelMap[key].lastActivity = pkt.timestamp; - if (decoded.text) { - const colonIdx = decoded.text.indexOf(': '); - channelMap[key].lastMessage = colonIdx > 0 ? decoded.text.slice(colonIdx + 2) : decoded.text; - channelMap[key].lastSender = decoded.sender || null; - } - } - } - - const _chResult = { channels: Object.values(channelMap) }; - cache.set(_ck, _chResult, TTL.channels); - res.json(_chResult); -}); - -app.get('/api/channels/:hash/messages', (req, res) => { - const _ck = 'channels:' + req.params.hash + ':' + (req.query.limit||100) + ':' + (req.query.offset||0); - const _c = cache.get(_ck); if (_c) return res.json(_c); - const { limit = 100, offset = 0 } = req.query; - const channelHash = req.params.hash; - const packets = pktStore.filter(p => p.payload_type === 5).sort((a,b) => a.timestamp > b.timestamp ? 1 : -1); - - // Group by message content + timestamp to deduplicate repeats - const msgMap = new Map(); - for (const pkt of packets) { - let decoded; - try { decoded = JSON.parse(pkt.decoded_json); } catch { continue; } - // Only decrypted messages - if (decoded.type !== 'CHAN') continue; - const ch = decoded.channel || 'unknown'; - if (ch !== channelHash) continue; - - const sender = decoded.sender || (decoded.text ? decoded.text.split(': ')[0] : null) || pkt.observer_name || pkt.observer_id || 'Unknown'; - const text = decoded.text || decoded.encryptedData || ''; - // Use server observation timestamp for dedup — sender_timestamp is unreliable (device clocks are wildly inaccurate) - const ts = pkt.timestamp; - const dedupeKey = `${sender}:${pkt.hash}`; - - if (msgMap.has(dedupeKey)) { - const existing = msgMap.get(dedupeKey); - existing.repeats++; - if (pkt.observer_name && !existing.observers.includes(pkt.observer_name)) { - existing.observers.push(pkt.observer_name); - } - } else { - // Parse sender and message from "sender: message" format - let displaySender = sender; - let displayText = text; - if (decoded.text) { - const colonIdx = decoded.text.indexOf(': '); - if (colonIdx > 0 && colonIdx < 50) { - displaySender = decoded.text.slice(0, colonIdx); - displayText = decoded.text.slice(colonIdx + 2); - } - } - msgMap.set(dedupeKey, { - sender: displaySender, - text: displayText, - timestamp: pkt.timestamp, - sender_timestamp: decoded.sender_timestamp || null, - packetId: pkt.id, - packetHash: pkt.hash, - repeats: 1, - observers: [pkt.observer_name || pkt.observer_id].filter(Boolean), - hops: decoded.path_len || (pkt.path_json ? JSON.parse(pkt.path_json).length : 0), - snr: pkt.snr || (decoded.SNR !== undefined ? decoded.SNR : null), - }); - } - } - - const allMessages = [...msgMap.values()]; - const total = allMessages.length; - // Return the latest messages (tail), not the oldest (head) - const start = Math.max(0, total - Number(limit) - Number(offset)); - const end = total - Number(offset); - const messages = allMessages.slice(Math.max(0, start), Math.max(0, end)); - const _msgResult = { messages, total }; - cache.set(_ck, _msgResult, TTL.channelMessages); - res.json(_msgResult); -}); - -app.get('/api/observers', (req, res) => { - const _c = cache.get('observers'); if (_c) return res.json(_c); - const observers = db.getObservers(); - const oneHourAgo = new Date(Date.now() - 3600000).toISOString(); - // Batch-fetch all node locations in one query - const allNodes = db.db.prepare("SELECT public_key, lat, lon, role FROM nodes").all(); - const nodeMap = new Map(); - for (const n of allNodes) nodeMap.set(n.public_key?.toLowerCase(), n); - const result = observers.map(o => { - const obsPackets = pktStore.byObserver.get(o.id) || []; - // byObserver is NOT uniformly sorted — initial DB load is DESC but live - // ingestion appends newest at the end. Full scan required. - let count = 0; - for (const obs of obsPackets) { - if (obs.timestamp > oneHourAgo) count++; - } - const node = nodeMap.get(o.id?.toLowerCase()); - return { ...o, packetsLastHour: count, lat: node?.lat || null, lon: node?.lon || null, nodeRole: node?.role || null }; - }); - const _oResult = { observers: result, server_time: new Date().toISOString() }; - cache.set('observers', _oResult, TTL.observers); - res.json(_oResult); -}); - -// Observer detail -app.get('/api/observers/:id', (req, res) => { - const id = req.params.id; - const obs = db.db.prepare('SELECT * FROM observers WHERE id = ?').get(id); - if (!obs) return res.status(404).json({ error: 'Observer not found' }); - const oneHourAgo = new Date(Date.now() - 3600000).toISOString(); - const obsPackets = pktStore.byObserver.get(id) || []; - const packetsLastHour = obsPackets.filter(p => p.timestamp > oneHourAgo).length; - res.json({ ...obs, packetsLastHour }); -}); - -// Observer analytics -app.get('/api/observers/:id/analytics', (req, res) => { - const id = req.params.id; - const days = parseInt(req.query.days) || 7; - const since = new Date(Date.now() - days * 86400000).toISOString(); - const obsPackets = pktStore.enrichObservations((pktStore.byObserver.get(id) || []).filter(p => p.timestamp >= since)).sort((a, b) => b.timestamp.localeCompare(a.timestamp)); - - // Timeline: packets per hour (last N days, bucketed) - const bucketMs = days <= 1 ? 3600000 : days <= 7 ? 3600000 * 4 : 86400000; - const buckets = {}; - for (const p of obsPackets) { - const t = Math.floor(new Date(p.timestamp).getTime() / bucketMs) * bucketMs; - buckets[t] = (buckets[t] || 0) + 1; - } - const timeline = Object.entries(buckets) - .sort((a, b) => a[0] - b[0]) - .map(([t, count]) => { - const d = new Date(parseInt(t)); - const label = days <= 1 - ? d.toLocaleTimeString('en-US', { hour: '2-digit', minute: '2-digit' }) - : days <= 7 - ? d.toLocaleDateString('en-US', { weekday: 'short', hour: '2-digit' }) - : d.toLocaleDateString('en-US', { month: 'short', day: 'numeric' }); - return { label, count }; - }); - - // Packet type breakdown - const packetTypes = {}; - for (const p of obsPackets) { - packetTypes[p.payload_type] = (packetTypes[p.payload_type] || 0) + 1; - } - - // Unique nodes per time bucket - const nodeBuckets = {}; - for (const p of obsPackets) { - const t = Math.floor(new Date(p.timestamp).getTime() / bucketMs) * bucketMs; - if (!nodeBuckets[t]) nodeBuckets[t] = new Set(); - try { - const decoded = typeof p.decoded_json === 'string' ? JSON.parse(p.decoded_json) : p.decoded_json; - if (decoded && decoded.pubKey) nodeBuckets[t].add(decoded.pubKey); - if (decoded && decoded.srcHash) nodeBuckets[t].add(decoded.srcHash); - if (decoded && decoded.destHash) nodeBuckets[t].add(decoded.destHash); - } catch {} - const hops = typeof p.path_json === 'string' ? JSON.parse(p.path_json) : (p.path_json || []); - for (const h of hops) nodeBuckets[t].add(h); - } - const nodesTimeline = Object.entries(nodeBuckets) - .sort((a, b) => a[0] - b[0]) - .map(([t, nodes]) => { - const d = new Date(parseInt(t)); - const label = days <= 1 - ? d.toLocaleTimeString('en-US', { hour: '2-digit', minute: '2-digit' }) - : days <= 7 - ? d.toLocaleDateString('en-US', { weekday: 'short', hour: '2-digit' }) - : d.toLocaleDateString('en-US', { month: 'short', day: 'numeric' }); - return { label, count: nodes.size }; - }); - - // SNR distribution - const snrBuckets = {}; - for (const p of obsPackets) { - if (p.snr == null) continue; - const bucket = Math.floor(p.snr / 2) * 2; // 2dB buckets - const range = bucket + ' to ' + (bucket + 2); - snrBuckets[bucket] = snrBuckets[bucket] || { range, count: 0 }; - snrBuckets[bucket].count++; - } - const snrDistribution = Object.values(snrBuckets).sort((a, b) => parseFloat(a.range) - parseFloat(b.range)); - - // Recent packets (last 20) — obsPackets filtered from pktStore, newest-first - const recentPackets = obsPackets.slice(0, 20); - - res.json({ timeline, packetTypes, nodesTimeline, snrDistribution, recentPackets }); -}); - -app.get('/api/traces/:hash', (req, res) => { - const packets = (pktStore.getSiblings(req.params.hash) || []).sort((a,b) => a.timestamp > b.timestamp ? 1 : -1); - const traces = packets.map(p => ({ - observer: p.observer_id, - observer_name: p.observer_name || null, - time: p.timestamp, - snr: p.snr, - rssi: p.rssi, - path_json: p.path_json || null - })); - res.json({ traces }); -}); - -app.get('/api/nodes/:pubkey/health', (req, res) => { - const pubkey = req.params.pubkey; - const _ck = 'health:' + pubkey; - const _c = cache.get(_ck); if (_c) return res.json(_c); - - const node = db.db.prepare('SELECT * FROM nodes WHERE public_key = ?').get(pubkey); - if (!node) return res.status(404).json({ error: 'Not found' }); - - const todayStart = new Date(); todayStart.setUTCHours(0, 0, 0, 0); - const todayISO = todayStart.toISOString(); - - // Single reusable lookup for all packets referencing this node - const packets = pktStore.findPacketsForNode(pubkey).packets; - - // Observers - const obsMap = {}; - let snrSum = 0, snrN = 0, totalHops = 0, hopCount = 0, lastHeard = null, packetsToday = 0; - for (const p of packets) { - if (p.timestamp > todayISO) packetsToday++; - if (p.snr != null) { snrSum += p.snr; snrN++; } - if (!lastHeard || p.timestamp > lastHeard) lastHeard = p.timestamp; - if (p.path_json) { - try { const h = JSON.parse(p.path_json); if (Array.isArray(h)) { totalHops += h.length; hopCount++; } } catch {} - } - if (p.observer_id) { - if (!obsMap[p.observer_id]) obsMap[p.observer_id] = { observer_name: p.observer_name, snrSum: 0, snrN: 0, rssiSum: 0, rssiN: 0, packetCount: 0 }; - const o = obsMap[p.observer_id]; o.packetCount++; - if (p.snr != null) { o.snrSum += p.snr; o.snrN++; } - if (p.rssi != null) { o.rssiSum += p.rssi; o.rssiN++; } - } - } - - // Build observer iata lookup - const allObservers = db.getObservers(); - const obsIataMap = {}; - for (const obs of allObservers) { if (obs.iata) obsIataMap[obs.id] = obs.iata; } - - const observers = Object.entries(obsMap).map(([observer_id, o]) => ({ - observer_id, observer_name: o.observer_name, packetCount: o.packetCount, - avgSnr: o.snrN ? o.snrSum / o.snrN : null, avgRssi: o.rssiN ? o.rssiSum / o.rssiN : null, - iata: obsIataMap[observer_id] || null - })).sort((a, b) => b.packetCount - a.packetCount); - - const recentPackets = packets.slice(0, 20); - - // Count transmissions vs observations - const counts = pktStore.countForNode(pubkey); - const recentWithoutObs = recentPackets.map(p => { - const { observations, ...rest } = p; - return { ...rest, observation_count: p.observation_count || 1 }; - }); - - const result = { - node: node.node || node, observers, - stats: { - totalTransmissions: counts.transmissions, - totalObservations: counts.observations, - totalPackets: counts.transmissions, // backward compat - packetsToday, avgSnr: snrN ? snrSum / snrN : null, avgHops: hopCount > 0 ? Math.round(totalHops / hopCount) : 0, lastHeard - }, - recentPackets: recentWithoutObs - }; - cache.set(_ck, result, TTL.nodeHealth); - res.json(result); -}); - -app.get('/api/nodes/:pubkey/paths', (req, res) => { - const pubkey = req.params.pubkey; - const _ck = 'nodePaths:' + pubkey; - const _c = cache.get(_ck); if (_c) return res.json(_c); - - const node = db.db.prepare('SELECT public_key, name, lat, lon FROM nodes WHERE public_key = ?').get(pubkey); - if (!node) return res.status(404).json({ error: 'Not found' }); - - const prefix1 = pubkey.slice(0, 2).toLowerCase(); - const prefix2 = pubkey.slice(0, 4).toLowerCase(); - - const allNodes = getCachedNodes(false); - - // Scan all transmissions for paths containing this node's prefix - const matchingTx = []; - for (const [, tx] of pktStore.byHash) { - if (!tx.path_json) continue; - const hops = tx._parsedPath || (tx.path_json ? (() => { try { return tx._parsedPath = JSON.parse(tx.path_json); } catch { return null; } })() : null); - if (!Array.isArray(hops) || !hops.length) continue; - const found = hops.some(h => { - const hl = (typeof h === 'string' ? h : '').toLowerCase(); - return hl === prefix1 || hl === prefix2 || hl.startsWith(prefix2); - }); - if (found) matchingTx.push({ tx, hops }); - } - - // Resolve and group paths using shared disambiguateHops (prefix-indexed) - - // Group by resolved path signature - const pathGroups = {}; - let totalTransmissions = 0; - const _pathsDisambigCache = {}; - for (const { tx, hops } of matchingTx) { - totalTransmissions++; - // Use disambiguateHops (has prefix index) instead of inline allNodes.filter per hop - const cacheKey = hops.join(','); - const resolved = _pathsDisambigCache[cacheKey] || (_pathsDisambigCache[cacheKey] = disambiguateHops(hops, allNodes)); - const resolvedHops = resolved.map(r => ({ prefix: r.hop, name: r.name, pubkey: r.pubkey || null, lat: r.lat || null, lon: r.lon || null })); - const key = resolvedHops.map(h => h.pubkey || h.prefix).join('→'); - if (!pathGroups[key]) pathGroups[key] = { hops: resolvedHops, count: 0, lastSeen: null, sampleHash: tx.hash }; - pathGroups[key].count++; - const ts = tx.timestamp; - if (!pathGroups[key].lastSeen || ts > pathGroups[key].lastSeen) { - pathGroups[key].lastSeen = ts; - pathGroups[key].sampleHash = tx.hash; - } - } - - const paths = Object.values(pathGroups) - .sort((a, b) => b.count - a.count) - .slice(0, 50); - - const result = { - node: { public_key: node.public_key, name: node.name, lat: node.lat, lon: node.lon }, - paths, - totalPaths: Object.keys(pathGroups).length, - totalTransmissions - }; - cache.set(_ck, result, TTL.nodeHealth); - res.json(result); -}); - -app.get('/api/nodes/:pubkey/analytics', (req, res) => { - const pubkey = req.params.pubkey; - const days = Math.min(Math.max(Number(req.query.days) || 7, 1), 365); - const _ck = `node-analytics:${pubkey}:${days}`; - const _c = cache.get(_ck); if (_c) return res.json(_c); - - const node = db.db.prepare('SELECT * FROM nodes WHERE public_key = ?').get(pubkey); - if (!node) return res.status(404).json({ error: 'Not found' }); - - const now = new Date(); - const fromISO = new Date(now.getTime() - days * 86400000).toISOString(); - const toISO = now.toISOString(); - - // Read from in-memory index + name search, filter by time range - const allPkts = pktStore.findPacketsForNode(pubkey).packets; - const packets = allPkts.filter(p => p.timestamp > fromISO); - - // Activity timeline - const timelineBuckets = {}; - for (const p of packets) { const b = p.timestamp.slice(0, 13) + ':00:00Z'; timelineBuckets[b] = (timelineBuckets[b] || 0) + 1; } - const activityTimeline = Object.entries(timelineBuckets).sort().map(([bucket, count]) => ({ bucket, count })); - - // SNR trend - const snrTrend = packets.filter(p => p.snr != null).map(p => ({ - timestamp: p.timestamp, snr: p.snr, rssi: p.rssi, observer_id: p.observer_id, observer_name: p.observer_name - })); - - // Packet type breakdown - const typeBuckets = {}; - for (const p of packets) { typeBuckets[p.payload_type] = (typeBuckets[p.payload_type] || 0) + 1; } - const packetTypeBreakdown = Object.entries(typeBuckets).map(([payload_type, count]) => ({ payload_type: +payload_type, count })); - - // Observer coverage - const obsMap = {}; - for (const p of packets) { - if (!p.observer_id) continue; - if (!obsMap[p.observer_id]) obsMap[p.observer_id] = { observer_name: p.observer_name, packetCount: 0, snrSum: 0, snrN: 0, rssiSum: 0, rssiN: 0, first: p.timestamp, last: p.timestamp }; - const o = obsMap[p.observer_id]; o.packetCount++; - if (p.snr != null) { o.snrSum += p.snr; o.snrN++; } - if (p.rssi != null) { o.rssiSum += p.rssi; o.rssiN++; } - if (p.timestamp < o.first) o.first = p.timestamp; - if (p.timestamp > o.last) o.last = p.timestamp; - } - const observerCoverage = Object.entries(obsMap).map(([observer_id, o]) => ({ - observer_id, observer_name: o.observer_name, packetCount: o.packetCount, - avgSnr: o.snrN ? o.snrSum / o.snrN : null, avgRssi: o.rssiN ? o.rssiSum / o.rssiN : null, - firstSeen: o.first, lastSeen: o.last - })).sort((a, b) => b.packetCount - a.packetCount); - - // Hop distribution - const hopCounts = {}; - let totalWithPath = 0, relayedCount = 0; - for (const p of packets) { - if (!p.path_json) continue; - try { - const hops = JSON.parse(p.path_json); - if (Array.isArray(hops)) { - const h = hops.length; const key = h >= 4 ? '4+' : String(h); - hopCounts[key] = (hopCounts[key] || 0) + 1; - totalWithPath++; if (h > 1) relayedCount++; - } - } catch {} - } - const hopDistribution = Object.entries(hopCounts).map(([hops, count]) => ({ hops, count })) - .sort((a, b) => a.hops.localeCompare(b.hops, undefined, { numeric: true })); - - // Peer interactions - const peerMap = {}; - for (const p of packets) { - if (!p.decoded_json) continue; - try { - const d = JSON.parse(p.decoded_json); - const candidates = []; - if (d.sender_key && d.sender_key !== pubkey) candidates.push({ key: d.sender_key, name: d.sender_name || d.sender_short_name }); - if (d.recipient_key && d.recipient_key !== pubkey) candidates.push({ key: d.recipient_key, name: d.recipient_name || d.recipient_short_name }); - if (d.pubkey && d.pubkey !== pubkey) candidates.push({ key: d.pubkey, name: d.name }); - for (const c of candidates) { - if (!c.key) continue; - if (!peerMap[c.key]) peerMap[c.key] = { peer_key: c.key, peer_name: c.name || c.key.slice(0, 12), messageCount: 0, lastContact: p.timestamp }; - peerMap[c.key].messageCount++; - if (p.timestamp > peerMap[c.key].lastContact) peerMap[c.key].lastContact = p.timestamp; - } - } catch {} - } - const peerInteractions = Object.values(peerMap).sort((a, b) => b.messageCount - a.messageCount).slice(0, 20); - - // Uptime heatmap - const heatmap = []; - for (const p of packets) { - const d = new Date(p.timestamp); - heatmap.push({ dayOfWeek: d.getUTCDay(), hour: d.getUTCHours() }); - } - const heatBuckets = {}; - for (const h of heatmap) { const k = `${h.dayOfWeek}:${h.hour}`; heatBuckets[k] = (heatBuckets[k] || 0) + 1; } - const uptimeHeatmap = Object.entries(heatBuckets).map(([k, count]) => { - const [d, h] = k.split(':'); return { dayOfWeek: +d, hour: +h, count }; - }); - - // Computed stats - const totalPackets = packets.length; - const distinctHours = activityTimeline.length; - const availabilityPct = days * 24 > 0 ? Math.round(distinctHours / (days * 24) * 1000) / 10 : 0; - const avgPacketsPerDay = days > 0 ? Math.round(totalPackets / days * 10) / 10 : totalPackets; - - // Longest silence - const timestamps = packets.map(p => new Date(p.timestamp).getTime()).sort((a, b) => a - b); - let longestSilenceMs = 0, longestSilenceStart = null; - for (let i = 1; i < timestamps.length; i++) { - const gap = timestamps[i] - timestamps[i - 1]; - if (gap > longestSilenceMs) { longestSilenceMs = gap; longestSilenceStart = new Date(timestamps[i - 1]).toISOString(); } - } - - // Signal grade - const snrValues = snrTrend.map(r => r.snr); - const snrMean = snrValues.length > 0 ? snrValues.reduce((a, b) => a + b, 0) / snrValues.length : 0; - const snrStdDev = snrValues.length > 1 ? Math.sqrt(snrValues.reduce((s, v) => s + (v - snrMean) ** 2, 0) / snrValues.length) : 0; - let signalGrade = 'D'; - if (snrMean > 15 && snrStdDev < 2) signalGrade = 'A'; - else if (snrMean > 15) signalGrade = 'A-'; - else if (snrMean > 12 && snrStdDev < 3) signalGrade = 'B+'; - else if (snrMean > 8) signalGrade = 'B'; - else if (snrMean > 3) signalGrade = 'C'; - const relayPct = totalWithPath > 0 ? Math.round(relayedCount / totalWithPath * 1000) / 10 : 0; - - const result = { - node: node.node || node, - timeRange: { from: fromISO, to: toISO, days }, - activityTimeline, snrTrend, packetTypeBreakdown, observerCoverage, hopDistribution, peerInteractions, uptimeHeatmap, - computedStats: { - availabilityPct, longestSilenceMs, longestSilenceStart, signalGrade, - snrMean: Math.round(snrMean * 10) / 10, snrStdDev: Math.round(snrStdDev * 10) / 10, - relayPct, totalPackets, uniqueObservers: observerCoverage.length, uniquePeers: peerInteractions.length, avgPacketsPerDay - } - }; - cache.set(_ck, result, TTL.nodeAnalytics); - res.json(result); -}); - -// Pre-compute all subpath data in a single pass (shared across all subpath queries) -let _subpathsComputing = null; -function computeAllSubpaths() { - const _c = cache.get('analytics:subpaths:master'); - if (_c) return _c; - if (_subpathsComputing) return _subpathsComputing; // deduplicate concurrent calls - - const t0 = Date.now(); - const packets = pktStore.filter(p => p.path_json && p.path_json !== '[]'); - const allNodes = getCachedNodes(false); - - const disambigCache = {}; - function cachedDisambiguate(hops) { - const key = hops.join(','); - if (disambigCache[key]) return disambigCache[key]; - const result = disambiguateHops(hops, allNodes); - disambigCache[key] = result; - return result; - } - - // Single pass: extract ALL subpaths (lengths 2-8) at once - const subpathsByLen = {}; // len → { path → { count, raw } } - let totalPaths = 0; - - for (const pkt of packets) { - const hops = pkt._parsedPath || (pkt.path_json ? (() => { try { return pkt._parsedPath = JSON.parse(pkt.path_json); } catch { return null; } })() : null); - if (!Array.isArray(hops) || hops.length < 2) continue; - totalPaths++; - - const resolved = cachedDisambiguate(hops); - const named = resolved.map(r => r.name); - - for (let len = 2; len <= Math.min(8, named.length); len++) { - if (!subpathsByLen[len]) subpathsByLen[len] = {}; - for (let start = 0; start <= named.length - len; start++) { - const sub = named.slice(start, start + len).join(' → '); - const raw = hops.slice(start, start + len).join(','); - if (!subpathsByLen[len][sub]) subpathsByLen[len][sub] = { count: 0, raw }; - subpathsByLen[len][sub].count++; - } - } - } - - const master = { subpathsByLen, totalPaths }; - cache.set('analytics:subpaths:master', master, TTL.analyticsSubpaths); - _subpathsComputing = master; // keep ref for concurrent callers - setTimeout(() => { _subpathsComputing = null; }, 100); // release after brief window - return master; -} - -// Subpath frequency analysis — reads from pre-computed master -app.get('/api/analytics/subpaths', (req, res) => { - const regionKey = req.query.region || ''; - const _ck = 'analytics:subpaths:' + (req.query.minLen||2) + ':' + (req.query.maxLen||8) + ':' + (req.query.limit||100) + ':r=' + regionKey; - const _c = cache.get(_ck); if (_c) return res.json(_c); - - const minLen = Math.max(2, Number(req.query.minLen) || 2); - const maxLen = Number(req.query.maxLen) || 8; - const limit = Number(req.query.limit) || 100; - - const regionObsIds = getObserverIdsForRegions(req.query.region); - if (regionObsIds) { - // Region-filtered subpath computation - const regionalHashes = new Set(); - for (const obsId of regionObsIds) { - const obs = pktStore.byObserver.get(obsId); - if (obs) for (const o of obs) regionalHashes.add(o.hash); - } - const packets = pktStore.filter(p => p.path_json && p.path_json !== '[]' && regionalHashes.has(p.hash)); - const allNodes = getCachedNodes(false); - const subpathsByLen = {}; - let totalPaths = 0; - for (const pkt of packets) { - const hops = pkt._parsedPath || (pkt.path_json ? (() => { try { return pkt._parsedPath = JSON.parse(pkt.path_json); } catch { return null; } })() : null); - if (!Array.isArray(hops) || hops.length < 2) continue; - totalPaths++; - const resolved = disambiguateHops(hops, allNodes); - const named = resolved.map(r => r.name); - for (let len = minLen; len <= Math.min(maxLen, named.length); len++) { - if (!subpathsByLen[len]) subpathsByLen[len] = {}; - for (let start = 0; start <= named.length - len; start++) { - const sub = named.slice(start, start + len).join(' \u2192 '); - const raw = hops.slice(start, start + len).join(','); - if (!subpathsByLen[len][sub]) subpathsByLen[len][sub] = { count: 0, raw }; - subpathsByLen[len][sub].count++; - } - } - } - const merged = {}; - for (let len = minLen; len <= maxLen; len++) { - const bucket = subpathsByLen[len] || {}; - for (const [path, data] of Object.entries(bucket)) { - if (!merged[path]) merged[path] = { count: 0, raw: data.raw }; - merged[path].count += data.count; - } - } - const ranked = Object.entries(merged) - .map(([path, data]) => ({ path, rawHops: data.raw.split(','), count: data.count, hops: path.split(' \u2192 ').length, pct: totalPaths > 0 ? Math.round(data.count / totalPaths * 1000) / 10 : 0 })) - .sort((a, b) => b.count - a.count) - .slice(0, limit); - const result = { subpaths: ranked, totalPaths }; - cache.set(_ck, result, TTL.analyticsSubpaths); - return res.json(result); - } - - const { subpathsByLen, totalPaths } = computeAllSubpaths(); - - // Merge requested length ranges - const merged = {}; - for (let len = minLen; len <= maxLen; len++) { - const bucket = subpathsByLen[len] || {}; - for (const [path, data] of Object.entries(bucket)) { - if (!merged[path]) merged[path] = { count: 0, raw: data.raw }; - merged[path].count += data.count; - } - } - - const ranked = Object.entries(merged) - .map(([path, data]) => ({ - path, - rawHops: data.raw.split(','), - count: data.count, - hops: path.split(' → ').length, - pct: totalPaths > 0 ? Math.round(data.count / totalPaths * 1000) / 10 : 0 - })) - .sort((a, b) => b.count - a.count) - .slice(0, limit); - - const _spResult = { subpaths: ranked, totalPaths }; - cache.set(_ck, _spResult, TTL.analyticsSubpaths); - res.json(_spResult); -}); - -// Subpath detail — stats for a specific subpath (by raw hop prefixes) -app.get('/api/analytics/subpath-detail', (req, res) => { - const _sdck = 'analytics:subpath-detail:' + (req.query.hops || ''); - const _sdc = cache.get(_sdck); if (_sdc) return res.json(_sdc); - const rawHops = (req.query.hops || '').split(',').filter(Boolean); - if (rawHops.length < 2) return res.json({ error: 'Need at least 2 hops' }); - - const packets = pktStore.filter(p => p.path_json && p.path_json !== '[]'); - const allNodes = getCachedNodes(false); - - // Disambiguate the requested hops - const resolvedHops = disambiguateHops(rawHops, allNodes); - - const matching = []; - const parentPaths = {}; - const hourBuckets = new Array(24).fill(0); - let snrSum = 0, snrCount = 0, rssiSum = 0, rssiCount = 0; - const observers = {}; - const _detailCache = {}; - - for (const pkt of packets) { - let hops; - try { hops = JSON.parse(pkt.path_json); } catch { continue; } - if (!Array.isArray(hops) || hops.length < rawHops.length) continue; - - // Check if rawHops appears as a contiguous subsequence - let found = false; - for (let i = 0; i <= hops.length - rawHops.length; i++) { - let match = true; - for (let j = 0; j < rawHops.length; j++) { - if (hops[i + j].toLowerCase() !== rawHops[j].toLowerCase()) { match = false; break; } - } - if (match) { found = true; break; } - } - if (!found) continue; - - matching.push(pkt); - const hr = new Date(pkt.timestamp).getUTCHours(); - hourBuckets[hr]++; - if (pkt.snr != null) { snrSum += pkt.snr; snrCount++; } - if (pkt.rssi != null) { rssiSum += pkt.rssi; rssiCount++; } - if (pkt.observer_name) observers[pkt.observer_name] = (observers[pkt.observer_name] || 0) + 1; - - // Track full parent paths (disambiguated, cached) - const cacheKey = hops.join(','); - if (!_detailCache[cacheKey]) _detailCache[cacheKey] = disambiguateHops(hops, allNodes); - const fullPath = _detailCache[cacheKey].map(r => r.name).join(' → '); - parentPaths[fullPath] = (parentPaths[fullPath] || 0) + 1; - } - - // Use disambiguated nodes for map - const nodes = resolvedHops.map(r => ({ hop: r.hop, name: r.name, lat: r.lat, lon: r.lon, pubkey: r.pubkey })); - - const topParents = Object.entries(parentPaths) - .sort((a, b) => b[1] - a[1]) - .slice(0, 15) - .map(([path, count]) => ({ path, count })); - - const topObservers = Object.entries(observers) - .sort((a, b) => b[1] - a[1]) - .slice(0, 10) - .map(([name, count]) => ({ name, count })); - - const _sdResult = { - hops: rawHops, - nodes, - totalMatches: matching.length, - firstSeen: matching.length ? matching[0].timestamp : null, - lastSeen: matching.length ? matching[matching.length - 1].timestamp : null, - signal: { - avgSnr: snrCount ? Math.round(snrSum / snrCount * 10) / 10 : null, - avgRssi: rssiCount ? Math.round(rssiSum / rssiCount) : null, - samples: snrCount - }, - hourDistribution: hourBuckets, - parentPaths: topParents, - observers: topObservers - }; - cache.set(_sdck, _sdResult, TTL.analyticsSubpathDetail); - res.json(_sdResult); -}); - -// IATA coordinates for client-side regional filtering -app.get('/api/iata-coords', (req, res) => { - res.json({ coords: IATA_COORDS }); -}); - -// Audio Lab: representative packets bucketed by type -app.get('/api/audio-lab/buckets', (req, res) => { - const buckets = {}; - const byType = {}; - for (const tx of pktStore.packets) { - if (!tx.raw_hex) continue; - let typeName = 'UNKNOWN'; - try { const d = JSON.parse(tx.decoded_json || '{}'); typeName = d.type || (PAYLOAD_TYPES[tx.payload_type] || 'UNKNOWN'); } catch {} - if (!byType[typeName]) byType[typeName] = []; - byType[typeName].push(tx); - } - for (const [type, pkts] of Object.entries(byType)) { - const sorted = pkts.sort((a, b) => (a.raw_hex || '').length - (b.raw_hex || '').length); - const count = Math.min(8, sorted.length); - const picked = []; - for (let i = 0; i < count; i++) { - const idx = Math.floor((i / count) * sorted.length); - const tx = sorted[idx]; - picked.push({ - hash: tx.hash, raw_hex: tx.raw_hex, decoded_json: tx.decoded_json, - observation_count: tx.observation_count || 1, payload_type: tx.payload_type, - path_json: tx.path_json, observer_id: tx.observer_id, timestamp: tx.timestamp, - }); - } - buckets[type] = picked; - } - res.json({ buckets }); -}); - -// Static files + SPA fallback -const publicDir = process.env.COVERAGE === '1' ? 'public-instrumented' : 'public'; -app.use(express.static(path.join(__dirname, publicDir), { - etag: false, - lastModified: false, - setHeaders: (res, filePath) => { - if (filePath.endsWith('.js') || filePath.endsWith('.css') || filePath.endsWith('.html')) { - res.setHeader('Cache-Control', 'no-cache, no-store, must-revalidate'); - } - } -})); -app.get('/{*splat}', (req, res) => { - const indexPath = path.join(__dirname, 'public', 'index.html'); - const fs = require('fs'); - if (fs.existsSync(indexPath)) { - res.sendFile(indexPath); - } else { - res.status(200).send('

CoreScope

Frontend not yet built.

'); - } -}); - -// --- Start --- -const listenPort = process.env.PORT || config.port; -if (require.main === module) { -// Clean up phantom nodes created by the old autoLearnHopNodes behavior (fixes #133) -db.removePhantomNodes(); -server.listen(listenPort, () => { - const protocol = isHttps ? 'https' : 'http'; - console.log(`CoreScope running on ${protocol}://localhost:${listenPort}`); - // Log theme file location - let themeFound = false; - for (const p of THEME_PATHS) { - try { fs.accessSync(p); console.log(`[theme] Loaded from ${p}`); themeFound = true; break; } catch {} - } - if (!themeFound) console.log(`[theme] No theme.json found. Place it next to config.json or in data/ to customize.`); - // Pre-warm expensive caches via self-requests (yields event loop between each) - setTimeout(() => { - const port = listenPort; - const warmClient = isHttps ? https : http; - const warmEndpoints = [ - // Subpaths (heaviest — must go first so cache is ready) - '/api/analytics/subpaths?minLen=2&maxLen=2&limit=50', - '/api/analytics/subpaths?minLen=3&maxLen=3&limit=30', - '/api/analytics/subpaths?minLen=4&maxLen=4&limit=20', - '/api/analytics/subpaths?minLen=5&maxLen=8&limit=15', - // Other analytics - '/api/observers', - '/api/nodes?limit=10000&lastHeard=259200', - '/api/analytics/rf', - '/api/analytics/topology', - '/api/analytics/channels', - '/api/analytics/hash-sizes', - '/api/analytics/distance', - '/api/nodes/bulk-health?limit=50', - ]; - let warmed = 0; - const tw = Date.now(); - const warmNext = () => { - if (warmed >= warmEndpoints.length) { - console.log(`[pre-warm] ${warmEndpoints.length} endpoints in ${Date.now() - tw}ms`); - return; - } - const ep = warmEndpoints[warmed++]; - const requestOptions = { hostname: '127.0.0.1', port, path: ep }; - if (isHttps) requestOptions.rejectUnauthorized = false; - warmClient.get(requestOptions, (res) => { - res.resume(); - res.on('end', () => setImmediate(warmNext)); - }).on('error', () => setImmediate(warmNext)); - }; - warmNext(); - }, 5000); // 5s delay — let initial client page load complete first -}); -} // end if (require.main === module) - -// --- Graceful Shutdown --- -let _shuttingDown = false; -function shutdown(signal) { - if (_shuttingDown) return; - _shuttingDown = true; - console.log(`\n[shutdown] received ${signal}, closing gracefully…`); - - // Terminate WebSocket clients first — open WS connections would prevent - // server.close() from ever completing its callback otherwise. - if (wss) { - for (const client of wss.clients) { - try { client.terminate(); } catch {} - } - wss.close(); - console.log('[shutdown] WebSocket server closed'); - } - - // Force-drain all keep-alive HTTP connections so server.close() fires promptly. - // closeAllConnections() is available since Node 18.2 (we're on Node 22). - server.closeAllConnections(); - server.close(() => console.log('[shutdown] HTTP server closed')); - - // Checkpoint WAL and close SQLite synchronously — performed unconditionally, - // not gated on server.close(), so the DB is always cleanly flushed. - try { - db.db.pragma('wal_checkpoint(TRUNCATE)'); - db.db.close(); - console.log('[shutdown] database closed'); - } catch (e) { - console.error('[shutdown] database close error:', e.message); - } - - process.exit(0); -} - -process.on('SIGTERM', () => shutdown('SIGTERM')); -process.on('SIGINT', () => shutdown('SIGINT')); - -module.exports = { app, server, wss, pktStore, db, cache, lastPathSeenMap, hopPrefixToKey, ambiguousHopPrefixes, resolveUniquePrefixMatch }; diff --git a/test-all.sh b/test-all.sh index b9669e7f..e31f2166 100755 --- a/test-all.sh +++ b/test-all.sh @@ -9,27 +9,12 @@ echo "" # Unit tests (deterministic, fast) echo "── Unit Tests ──" -node test-decoder.js -node test-decoder-spec.js -node test-packet-store.js node test-packet-filter.js node test-aging.js node test-frontend-helpers.js -node test-regional-filter.js -node test-server-helpers.js -node test-server-routes.js -node test-db.js -node test-db-migration.js - -# Integration tests (spin up temp servers) -echo "" -echo "── Integration Tests ──" -node tools/e2e-test.js -node tools/frontend-test.js +node test-perf-go-runtime.js echo "" echo "═══════════════════════════════════════" echo " All tests passed" echo "═══════════════════════════════════════" -node test-server-routes.js -# test trigger diff --git a/test-db-migration.js b/test-db-migration.js deleted file mode 100644 index 19a637a6..00000000 --- a/test-db-migration.js +++ /dev/null @@ -1,321 +0,0 @@ -'use strict'; - -// Test v3 migration: create old-schema DB, run db.js to migrate, verify results -const path = require('path'); -const fs = require('fs'); -const os = require('os'); -const { execSync } = require('child_process'); -const Database = require('better-sqlite3'); - -let passed = 0, failed = 0; -function assert(cond, msg) { - if (cond) { passed++; console.log(` ✅ ${msg}`); } - else { failed++; console.error(` ❌ ${msg}`); } -} - -console.log('── db.js v3 migration tests ──\n'); - -// Helper: create a DB with old (v2) schema and test data -function createOldSchemaDB(dbPath) { - const db = new Database(dbPath); - db.pragma('journal_mode = WAL'); - db.pragma('foreign_keys = ON'); - - db.exec(` - CREATE TABLE nodes ( - public_key TEXT PRIMARY KEY, - name TEXT, - role TEXT, - lat REAL, - lon REAL, - last_seen TEXT, - first_seen TEXT, - advert_count INTEGER DEFAULT 0 - ); - - CREATE TABLE observers ( - id TEXT PRIMARY KEY, - name TEXT, - iata TEXT, - last_seen TEXT, - first_seen TEXT, - packet_count INTEGER DEFAULT 0, - model TEXT, - firmware TEXT, - client_version TEXT, - radio TEXT, - battery_mv INTEGER, - uptime_secs INTEGER, - noise_floor INTEGER - ); - - CREATE TABLE transmissions ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - raw_hex TEXT NOT NULL, - hash TEXT NOT NULL UNIQUE, - first_seen TEXT NOT NULL, - route_type INTEGER, - payload_type INTEGER, - payload_version INTEGER, - decoded_json TEXT, - created_at TEXT DEFAULT (datetime('now')) - ); - - CREATE TABLE observations ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - transmission_id INTEGER NOT NULL REFERENCES transmissions(id), - hash TEXT NOT NULL, - observer_id TEXT, - observer_name TEXT, - direction TEXT, - snr REAL, - rssi REAL, - score INTEGER, - path_json TEXT, - timestamp TEXT NOT NULL, - created_at TEXT DEFAULT (datetime('now')) - ); - - CREATE INDEX idx_transmissions_hash ON transmissions(hash); - CREATE INDEX idx_observations_hash ON observations(hash); - CREATE INDEX idx_observations_transmission_id ON observations(transmission_id); - CREATE INDEX idx_observations_observer_id ON observations(observer_id); - CREATE INDEX idx_observations_timestamp ON observations(timestamp); - CREATE UNIQUE INDEX idx_observations_dedup ON observations(hash, observer_id, COALESCE(path_json, '')); - `); - - // Insert test observers - db.prepare(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) VALUES (?, ?, ?, ?, ?, ?)`).run( - 'aabbccdd11223344aabbccdd11223344aabbccdd11223344aabbccdd11223344', 'Observer Alpha', 'SFO', - '2025-06-01T12:00:00Z', '2025-01-01T00:00:00Z', 100 - ); - db.prepare(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) VALUES (?, ?, ?, ?, ?, ?)`).run( - 'deadbeef12345678deadbeef12345678deadbeef12345678deadbeef12345678', 'Observer Beta', 'LAX', - '2025-06-01T11:00:00Z', '2025-02-01T00:00:00Z', 50 - ); - - // Insert test transmissions - db.prepare(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) VALUES (?, ?, ?, ?, ?, ?)`).run( - '0400aabbccdd', 'hash-mig-001', '2025-06-01T10:00:00Z', 1, 4, '{"type":"ADVERT"}' - ); - db.prepare(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) VALUES (?, ?, ?, ?, ?, ?)`).run( - '0400deadbeef', 'hash-mig-002', '2025-06-01T10:30:00Z', 2, 5, '{"type":"GRP_TXT"}' - ); - - // Insert test observations (old schema: has hash, observer_id, observer_name, text timestamp) - db.prepare(`INSERT INTO observations (transmission_id, hash, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run( - 1, 'hash-mig-001', 'aabbccdd11223344aabbccdd11223344aabbccdd11223344aabbccdd11223344', 'Observer Alpha', - 'rx', 12.5, -80, 85, '["aabb","ccdd"]', '2025-06-01T10:00:00Z' - ); - db.prepare(`INSERT INTO observations (transmission_id, hash, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run( - 1, 'hash-mig-001', 'deadbeef12345678deadbeef12345678deadbeef12345678deadbeef12345678', 'Observer Beta', - 'rx', 8.0, -92, 70, '["aabb"]', '2025-06-01T10:01:00Z' - ); - db.prepare(`INSERT INTO observations (transmission_id, hash, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run( - 2, 'hash-mig-002', 'aabbccdd11223344aabbccdd11223344aabbccdd11223344aabbccdd11223344', 'Observer Alpha', - 'rx', 15.0, -75, 90, null, '2025-06-01T10:30:00Z' - ); - - db.close(); -} - -// Helper: require db.js in a child process with a given DB_PATH, return schema info -function runDbModule(dbPath) { - const scriptPath = path.join(os.tmpdir(), 'meshcore-mig-test-script.js'); - fs.writeFileSync(scriptPath, ` - process.env.DB_PATH = ${JSON.stringify(dbPath)}; - const db = require(${JSON.stringify(path.resolve(__dirname, 'db'))}); - const cols = db.db.pragma('table_info(observations)').map(c => c.name); - const sv = db.db.pragma('user_version', { simple: true }); - const obsCount = db.db.prepare('SELECT COUNT(*) as c FROM observations').get().c; - const viewRows = db.db.prepare('SELECT * FROM packets_v ORDER BY id').all(); - const rawObs = db.db.prepare('SELECT * FROM observations ORDER BY id').all(); - console.log(JSON.stringify({ - columns: cols, - schemaVersion: sv || 0, - obsCount, - viewRows, - rawObs - })); - db.db.close(); - `); - const result = execSync(`node ${JSON.stringify(scriptPath)}`, { - cwd: __dirname, - encoding: 'utf8', - timeout: 30000, - }); - fs.unlinkSync(scriptPath); - const lines = result.trim().split('\n'); - for (let i = lines.length - 1; i >= 0; i--) { - try { return JSON.parse(lines[i]); } catch {} - } - throw new Error('No JSON output from child process: ' + result); -} - -// --- Test 1: Migration from old schema --- -console.log('Migration from old schema:'); -{ - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'meshcore-mig-test-')); - const dbPath = path.join(tmpDir, 'test-mig.db'); - - createOldSchemaDB(dbPath); - - // Run db.js which should trigger migration - const info = runDbModule(dbPath); - - // Verify schema - assert(info.schemaVersion === 3, 'schema version is 3 after migration'); - assert(info.columns.includes('observer_idx'), 'has observer_idx column'); - assert(!info.columns.includes('observer_id'), 'no observer_id column'); - assert(!info.columns.includes('observer_name'), 'no observer_name column'); - assert(!info.columns.includes('hash'), 'no hash column'); - - // Verify row count - assert(info.obsCount === 3, `all 3 rows migrated (got ${info.obsCount})`); - - // Verify raw observation data - const obs0 = info.rawObs[0]; - assert(typeof obs0.timestamp === 'number', 'timestamp is integer'); - assert(obs0.timestamp === Math.floor(new Date('2025-06-01T10:00:00Z').getTime() / 1000), 'timestamp epoch correct'); - assert(obs0.observer_idx !== null, 'observer_idx populated'); - - // Verify view backward compat - const vr0 = info.viewRows[0]; - assert(vr0.observer_id === 'aabbccdd11223344aabbccdd11223344aabbccdd11223344aabbccdd11223344', 'view observer_id correct'); - assert(vr0.observer_name === 'Observer Alpha', 'view observer_name correct'); - assert(typeof vr0.timestamp === 'string', 'view timestamp is string'); - assert(vr0.hash === 'hash-mig-001', 'view hash correct'); - assert(vr0.snr === 12.5, 'view snr correct'); - assert(vr0.path_json === '["aabb","ccdd"]', 'view path_json correct'); - - // Third row has null path_json - const vr2 = info.viewRows[2]; - assert(vr2.path_json === null, 'null path_json preserved'); - - // Verify backup file created - const backups1 = fs.readdirSync(tmpDir).filter(f => f.includes('.pre-v3-backup-')); - assert(backups1.length === 1, 'backup file exists'); - - fs.rmSync(tmpDir, { recursive: true }); -} - -// --- Test 2: Migration doesn't re-run --- -console.log('\nMigration idempotency:'); -{ - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'meshcore-mig-test2-')); - const dbPath = path.join(tmpDir, 'test-mig2.db'); - - createOldSchemaDB(dbPath); - - // First run — triggers migration - let info = runDbModule(dbPath); - assert(info.schemaVersion === 3, 'first run migrates to v3'); - - // Second run — should NOT re-run migration (no backup overwrite, same data) - const backups2pre = fs.readdirSync(tmpDir).filter(f => f.includes('.pre-v3-backup-')); - const backupMtime = fs.statSync(path.join(tmpDir, backups2pre[0])).mtimeMs; - info = runDbModule(dbPath); - assert(info.schemaVersion === 3, 'second run still v3'); - assert(info.obsCount === 3, 'rows still intact'); - - fs.rmSync(tmpDir, { recursive: true }); -} - -// --- Test 3: Each migration creates a unique backup --- -console.log('\nUnique backup per migration:'); -{ - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'meshcore-mig-test3-')); - const dbPath = path.join(tmpDir, 'test-mig3.db'); - - createOldSchemaDB(dbPath); - - const info = runDbModule(dbPath); - - // Migration should have completed - assert(info.columns.includes('observer_idx'), 'migration completed'); - assert(info.schemaVersion === 3, 'schema version is 3'); - - // A timestamped backup should exist - const backups = fs.readdirSync(tmpDir).filter(f => f.includes('.pre-v3-backup-')); - assert(backups.length === 1, 'exactly one backup created'); - assert(fs.statSync(path.join(tmpDir, backups[0])).size > 0, 'backup is non-empty'); - - fs.rmSync(tmpDir, { recursive: true }); -} - -// --- Test 4: v3 ingestion via child process --- -console.log('\nv3 ingestion test:'); -{ - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'meshcore-mig-test4-')); - const dbPath = path.join(tmpDir, 'test-v3-ingest.db'); - - const scriptPath = path.join(os.tmpdir(), 'meshcore-ingest-test-script.js'); - fs.writeFileSync(scriptPath, ` - process.env.DB_PATH = ${JSON.stringify(dbPath)}; - const db = require(${JSON.stringify(path.resolve(__dirname, 'db'))}); - - db.upsertObserver({ id: 'test-obs', name: 'Test Obs' }); - - const r = db.insertTransmission({ - raw_hex: '0400ff', - hash: 'h-001', - timestamp: '2025-06-01T12:00:00Z', - observer_id: 'test-obs', - observer_name: 'Test Obs', - direction: 'rx', - snr: 10, - rssi: -85, - path_json: '["aa"]', - route_type: 1, - payload_type: 4, - }); - - const r2 = db.insertTransmission({ - raw_hex: '0400ff', - hash: 'h-001', - timestamp: '2025-06-01T12:00:00Z', - observer_id: 'test-obs', - direction: 'rx', - snr: 10, - rssi: -85, - path_json: '["aa"]', - }); - - const pkt = db.db.prepare('SELECT * FROM packets_v WHERE hash = ?').get('h-001'); - - console.log(JSON.stringify({ - r1_ok: r !== null && r.transmissionId > 0, - r2_deduped: r2.observationId === 0, - obs_count: db.db.prepare('SELECT COUNT(*) as c FROM observations').get().c, - view_observer_id: pkt.observer_id, - view_observer_name: pkt.observer_name, - view_ts_type: typeof pkt.timestamp, - })); - db.db.close(); - `); - - const result = execSync(`node ${JSON.stringify(scriptPath)}`, { - cwd: __dirname, encoding: 'utf8', timeout: 30000, - }); - fs.unlinkSync(scriptPath); - const lines = result.trim().split('\n'); - let info; - for (let i = lines.length - 1; i >= 0; i--) { - try { info = JSON.parse(lines[i]); break; } catch {} - } - - assert(info.r1_ok, 'first insertion succeeded'); - assert(info.r2_deduped, 'duplicate caught by dedup'); - assert(info.obs_count === 1, 'only one observation row'); - assert(info.view_observer_id === 'test-obs', 'view resolves observer_id'); - assert(info.view_observer_name === 'Test Obs', 'view resolves observer_name'); - assert(info.view_ts_type === 'string', 'view timestamp is string'); - - fs.rmSync(tmpDir, { recursive: true }); -} - -console.log(`\n═══════════════════════════════════════`); -console.log(` PASSED: ${passed}`); -console.log(` FAILED: ${failed}`); -console.log(`═══════════════════════════════════════`); -if (failed > 0) process.exit(1); diff --git a/test-db.js b/test-db.js deleted file mode 100644 index 4b69e002..00000000 --- a/test-db.js +++ /dev/null @@ -1,512 +0,0 @@ -'use strict'; - -// Test db.js functions with a temp database -const path = require('path'); -const fs = require('fs'); -const os = require('os'); - -const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'meshcore-db-test-')); -const dbPath = path.join(tmpDir, 'test.db'); -process.env.DB_PATH = dbPath; - -// Now require db.js — it will use our temp DB -const db = require('./db'); - -let passed = 0, failed = 0; -function assert(cond, msg) { - if (cond) { passed++; console.log(` ✅ ${msg}`); } - else { failed++; console.error(` ❌ ${msg}`); } -} - -function cleanup() { - try { db.db.close(); } catch {} - try { fs.rmSync(tmpDir, { recursive: true }); } catch {} -} - -console.log('── db.js tests ──\n'); - -// --- Schema --- -console.log('Schema:'); -{ - const tables = db.db.prepare("SELECT name FROM sqlite_master WHERE type='table'").all().map(r => r.name); - assert(tables.includes('nodes'), 'nodes table exists'); - assert(tables.includes('observers'), 'observers table exists'); - assert(tables.includes('transmissions'), 'transmissions table exists'); - assert(tables.includes('observations'), 'observations table exists'); -} - -// --- upsertNode --- -console.log('\nupsertNode:'); -{ - db.upsertNode({ public_key: 'aabbccdd11223344aabbccdd11223344', name: 'TestNode', role: 'repeater', lat: 37.0, lon: -122.0 }); - const node = db.getNode('aabbccdd11223344aabbccdd11223344'); - assert(node !== null, 'node inserted'); - assert(node.name === 'TestNode', 'name correct'); - assert(node.role === 'repeater', 'role correct'); - assert(node.lat === 37.0, 'lat correct'); - - // Update - db.upsertNode({ public_key: 'aabbccdd11223344aabbccdd11223344', name: 'UpdatedNode', role: 'room' }); - const node2 = db.getNode('aabbccdd11223344aabbccdd11223344'); - assert(node2.name === 'UpdatedNode', 'name updated'); - assert(node2.name === 'UpdatedNode', 'name updated'); - assert(node2.advert_count === 0, 'advert_count unchanged by upsertNode'); - - // advert_count only increments via incrementAdvertCount - db.incrementAdvertCount('aabbccdd11223344aabbccdd11223344'); - const node3 = db.getNode('aabbccdd11223344aabbccdd11223344'); - assert(node3.advert_count === 1, 'advert_count incremented via incrementAdvertCount'); -} - -// --- upsertObserver --- -console.log('\nupsertObserver:'); -{ - db.upsertObserver({ id: 'obs-1', name: 'Observer One', iata: 'SFO' }); - const observers = db.getObservers(); - assert(observers.length >= 1, 'observer inserted'); - assert(observers.some(o => o.id === 'obs-1'), 'observer found by id'); - assert(observers.find(o => o.id === 'obs-1').name === 'Observer One', 'observer name correct'); - - // Upsert again - db.upsertObserver({ id: 'obs-1', name: 'Observer Updated' }); - const obs2 = db.getObservers().find(o => o.id === 'obs-1'); - assert(obs2.name === 'Observer Updated', 'observer name updated'); - assert(obs2.packet_count === 2, 'packet_count incremented'); -} - -// --- updateObserverStatus --- -console.log('\nupdateObserverStatus:'); -{ - db.updateObserverStatus({ id: 'obs-2', name: 'Status Observer', iata: 'LAX', model: 'T-Deck' }); - const obs = db.getObservers().find(o => o.id === 'obs-2'); - assert(obs !== null, 'observer created via status update'); - assert(obs.model === 'T-Deck', 'model set'); - assert(obs.packet_count === 0, 'packet_count stays 0 for status update'); -} - -// --- insertTransmission --- -console.log('\ninsertTransmission:'); -{ - const result = db.insertTransmission({ - raw_hex: '0400aabbccdd', - hash: 'hash-001', - timestamp: '2025-01-01T00:00:00Z', - observer_id: 'obs-1', - observer_name: 'Observer One', - direction: 'rx', - snr: 10.5, - rssi: -85, - route_type: 1, - payload_type: 4, - payload_version: 1, - path_json: '["aabb","ccdd"]', - decoded_json: '{"type":"ADVERT","pubKey":"aabbccdd11223344aabbccdd11223344","name":"TestNode"}', - }); - assert(result !== null, 'transmission inserted'); - assert(result.transmissionId > 0, 'has transmissionId'); - assert(result.observationId > 0, 'has observationId'); - - // Duplicate hash = same transmission, new observation - const result2 = db.insertTransmission({ - raw_hex: '0400aabbccdd', - hash: 'hash-001', - timestamp: '2025-01-01T00:01:00Z', - observer_id: 'obs-2', - observer_name: 'Observer Two', - direction: 'rx', - snr: 8.0, - rssi: -90, - route_type: 1, - payload_type: 4, - path_json: '["aabb"]', - decoded_json: '{"type":"ADVERT","pubKey":"aabbccdd11223344aabbccdd11223344","name":"TestNode"}', - }); - assert(result2.transmissionId === result.transmissionId, 'same transmissionId for duplicate hash'); - - // No hash = null - const result3 = db.insertTransmission({ raw_hex: '0400' }); - assert(result3 === null, 'no hash returns null'); -} - -// --- getPackets --- -console.log('\ngetPackets:'); -{ - const { rows, total } = db.getPackets({ limit: 10 }); - assert(total >= 1, 'has packets'); - assert(rows.length >= 1, 'returns rows'); - assert(rows[0].hash === 'hash-001', 'correct hash'); - - // Filter by type - const { rows: r2 } = db.getPackets({ type: 4 }); - assert(r2.length >= 1, 'filter by type works'); - - const { rows: r3 } = db.getPackets({ type: 99 }); - assert(r3.length === 0, 'filter by nonexistent type returns empty'); - - // Filter by hash - const { rows: r4 } = db.getPackets({ hash: 'hash-001' }); - assert(r4.length >= 1, 'filter by hash works'); -} - -// --- getPacket --- -console.log('\ngetPacket:'); -{ - const { rows } = db.getPackets({ limit: 1 }); - const pkt = db.getPacket(rows[0].id); - assert(pkt !== null, 'getPacket returns packet'); - assert(pkt.hash === 'hash-001', 'correct packet'); - - const missing = db.getPacket(999999); - assert(missing === null, 'missing packet returns null'); -} - -// --- getTransmission --- -console.log('\ngetTransmission:'); -{ - const tx = db.getTransmission(1); - assert(tx !== null, 'getTransmission returns data'); - assert(tx.hash === 'hash-001', 'correct hash'); - - const missing = db.getTransmission(999999); - assert(missing === null, 'missing transmission returns null'); -} - -// --- getNodes --- -console.log('\ngetNodes:'); -{ - const { rows, total } = db.getNodes({ limit: 10 }); - assert(total >= 1, 'has nodes'); - assert(rows.length >= 1, 'returns node rows'); - - // Sort by name - const { rows: r2 } = db.getNodes({ sortBy: 'name' }); - assert(r2.length >= 1, 'sort by name works'); - - // Invalid sort falls back to last_seen - const { rows: r3 } = db.getNodes({ sortBy: 'DROP TABLE nodes' }); - assert(r3.length >= 1, 'invalid sort is safe'); -} - -// --- getNode --- -console.log('\ngetNode:'); -{ - const node = db.getNode('aabbccdd11223344aabbccdd11223344'); - assert(node !== null, 'getNode returns node'); - assert(Array.isArray(node.recentPackets), 'has recentPackets'); - - const missing = db.getNode('nonexistent'); - assert(missing === null, 'missing node returns null'); -} - -// --- searchNodes --- -console.log('\nsearchNodes:'); -{ - const results = db.searchNodes('Updated'); - assert(results.length >= 1, 'search by name'); - - const r2 = db.searchNodes('aabbcc'); - assert(r2.length >= 1, 'search by pubkey prefix'); - - const r3 = db.searchNodes('nonexistent_xyz'); - assert(r3.length === 0, 'no results for nonexistent'); -} - -// --- getStats --- -console.log('\ngetStats:'); -{ - const stats = db.getStats(); - assert(stats.totalNodes >= 1, 'totalNodes'); - assert(stats.totalObservers >= 1, 'totalObservers'); - assert(typeof stats.totalPackets === 'number', 'totalPackets is number'); - assert(typeof stats.packetsLastHour === 'number', 'packetsLastHour is number'); - assert(typeof stats.totalNodesAllTime === 'number', 'totalNodesAllTime is number'); - assert(stats.totalNodesAllTime >= stats.totalNodes, 'totalNodesAllTime >= totalNodes'); -} - -// --- getStats active node filtering --- -console.log('\ngetStats active node filtering:'); -{ - // Insert a node with last_seen 30 days ago (should be excluded from totalNodes) - const thirtyDaysAgo = new Date(Date.now() - 30 * 24 * 3600000).toISOString(); - db.upsertNode({ public_key: 'deadnode0000000000000000deadnode00', name: 'DeadNode', role: 'repeater', last_seen: thirtyDaysAgo, first_seen: thirtyDaysAgo }); - - // Insert a node with last_seen now (should be included) - db.upsertNode({ public_key: 'livenode0000000000000000livenode00', name: 'LiveNode', role: 'companion', last_seen: new Date().toISOString() }); - - const stats = db.getStats(); - assert(stats.totalNodesAllTime > stats.totalNodes, 'dead node excluded from totalNodes but included in totalNodesAllTime'); - - // Verify the dead node is in totalNodesAllTime - const allTime = stats.totalNodesAllTime; - assert(allTime >= 3, 'totalNodesAllTime includes dead + live nodes'); - - // Verify active count doesn't include the 30-day-old node - // The dead node's last_seen is 30 days ago, window is 7 days - const nodeInDb = db.getNode('deadnode0000000000000000deadnode00'); - assert(nodeInDb !== null, 'dead node exists in DB'); - const liveNode = db.getNode('livenode0000000000000000livenode00'); - assert(liveNode !== null, 'live node exists in DB'); -} - -// --- getNodeHealth --- -console.log('\ngetNodeHealth:'); -{ - const health = db.getNodeHealth('aabbccdd11223344aabbccdd11223344'); - assert(health !== null, 'returns health data'); - assert(health.node.name === 'UpdatedNode', 'has node info'); - assert(typeof health.stats.totalPackets === 'number', 'has totalPackets stat'); - assert(Array.isArray(health.observers), 'has observers array'); - assert(Array.isArray(health.recentPackets), 'has recentPackets array'); - - const missing = db.getNodeHealth('nonexistent'); - assert(missing === null, 'missing node returns null'); -} - -// --- getNodeAnalytics --- -console.log('\ngetNodeAnalytics:'); -{ - const analytics = db.getNodeAnalytics('aabbccdd11223344aabbccdd11223344', 7); - assert(analytics !== null, 'returns analytics'); - assert(analytics.node.name === 'UpdatedNode', 'has node info'); - assert(Array.isArray(analytics.activityTimeline), 'has activityTimeline'); - assert(Array.isArray(analytics.snrTrend), 'has snrTrend'); - assert(Array.isArray(analytics.packetTypeBreakdown), 'has packetTypeBreakdown'); - assert(Array.isArray(analytics.observerCoverage), 'has observerCoverage'); - assert(Array.isArray(analytics.hopDistribution), 'has hopDistribution'); - assert(Array.isArray(analytics.peerInteractions), 'has peerInteractions'); - assert(Array.isArray(analytics.uptimeHeatmap), 'has uptimeHeatmap'); - assert(typeof analytics.computedStats.availabilityPct === 'number', 'has availabilityPct'); - assert(typeof analytics.computedStats.signalGrade === 'string', 'has signalGrade'); - - const missing = db.getNodeAnalytics('nonexistent', 7); - assert(missing === null, 'missing node returns null'); -} - -// --- seed --- -console.log('\nseed:'); -{ - if (typeof db.seed === 'function') { - // Already has data, should return false - const result = db.seed(); - assert(result === false, 'seed returns false when data exists'); - } else { - console.log(' (skipped — seed not exported)'); - } -} - -// --- v3 schema tests (fresh DB should be v3) --- -console.log('\nv3 schema:'); -{ - assert(db.schemaVersion >= 3, 'fresh DB creates v3 schema'); - - // observations table should have observer_idx, not observer_id - const cols = db.db.pragma('table_info(observations)').map(c => c.name); - assert(cols.includes('observer_idx'), 'observations has observer_idx column'); - assert(!cols.includes('observer_id'), 'observations does NOT have observer_id column'); - assert(!cols.includes('observer_name'), 'observations does NOT have observer_name column'); - assert(!cols.includes('hash'), 'observations does NOT have hash column'); - assert(!cols.includes('created_at'), 'observations does NOT have created_at column'); - - // timestamp should be integer - const obsRow = db.db.prepare('SELECT typeof(timestamp) as t FROM observations LIMIT 1').get(); - if (obsRow) { - assert(obsRow.t === 'integer', 'timestamp is stored as integer'); - } - - // packets_v view should still expose observer_id, observer_name, ISO timestamp - const viewRow = db.db.prepare('SELECT * FROM packets_v LIMIT 1').get(); - if (viewRow) { - assert('observer_id' in viewRow, 'packets_v exposes observer_id'); - assert('observer_name' in viewRow, 'packets_v exposes observer_name'); - assert(typeof viewRow.timestamp === 'string', 'packets_v timestamp is ISO string'); - } - - // user_version is 3 - const sv = db.db.pragma('user_version', { simple: true }); - assert(sv === 3, 'user_version is 3'); -} - -// --- v3 ingestion: observer resolved via observer_idx --- -console.log('\nv3 ingestion with observer resolution:'); -{ - // Insert a new observer - db.upsertObserver({ id: 'obs-v3-test', name: 'V3 Test Observer' }); - - // Insert observation referencing that observer - const result = db.insertTransmission({ - raw_hex: '0400deadbeef', - hash: 'hash-v3-001', - timestamp: '2025-06-01T12:00:00Z', - observer_id: 'obs-v3-test', - observer_name: 'V3 Test Observer', - direction: 'rx', - snr: 12.0, - rssi: -80, - route_type: 1, - payload_type: 4, - path_json: '["aabb"]', - }); - assert(result !== null, 'v3 insertion succeeded'); - assert(result.transmissionId > 0, 'v3 has transmissionId'); - - // Verify via packets_v view - const pkt = db.db.prepare('SELECT * FROM packets_v WHERE hash = ?').get('hash-v3-001'); - assert(pkt !== null, 'v3 packet found via view'); - assert(pkt.observer_id === 'obs-v3-test', 'v3 observer_id resolved in view'); - assert(pkt.observer_name === 'V3 Test Observer', 'v3 observer_name resolved in view'); - assert(typeof pkt.timestamp === 'string', 'v3 timestamp is ISO string in view'); - assert(pkt.timestamp.includes('2025-06-01'), 'v3 timestamp date correct'); - - // Raw observation should have integer timestamp - const obs = db.db.prepare('SELECT * FROM observations ORDER BY id DESC LIMIT 1').get(); - assert(typeof obs.timestamp === 'number', 'v3 raw observation timestamp is integer'); - assert(obs.observer_idx !== null, 'v3 observation has observer_idx'); -} - -// --- v3 dedup --- -console.log('\nv3 dedup:'); -{ - // Insert same observation again — should be deduped - const result = db.insertTransmission({ - raw_hex: '0400deadbeef', - hash: 'hash-v3-001', - timestamp: '2025-06-01T12:00:00Z', - observer_id: 'obs-v3-test', - direction: 'rx', - snr: 12.0, - rssi: -80, - path_json: '["aabb"]', - }); - assert(result.observationId === 0, 'duplicate caught by in-memory dedup'); - - // Different observer = not a dupe - db.upsertObserver({ id: 'obs-v3-test-2', name: 'V3 Test Observer 2' }); - const result2 = db.insertTransmission({ - raw_hex: '0400deadbeef', - hash: 'hash-v3-001', - timestamp: '2025-06-01T12:01:00Z', - observer_id: 'obs-v3-test-2', - direction: 'rx', - snr: 9.0, - rssi: -88, - path_json: '["ccdd"]', - }); - assert(result2.observationId > 0, 'different observer is not a dupe'); -} - -// --- removePhantomNodes --- -console.log('\nremovePhantomNodes:'); -{ - // Insert phantom nodes (short public_keys like hop prefixes) - db.upsertNode({ public_key: 'aabb', name: null, role: 'repeater' }); - db.upsertNode({ public_key: 'ccddee', name: null, role: 'repeater' }); - db.upsertNode({ public_key: 'ff001122', name: null, role: 'repeater' }); - db.upsertNode({ public_key: '0011223344556677', name: null, role: 'repeater' }); // 16 chars — still phantom - - // Verify they exist - assert(db.getNode('aabb') !== null, 'phantom node aabb exists before cleanup'); - assert(db.getNode('ccddee') !== null, 'phantom node ccddee exists before cleanup'); - assert(db.getNode('ff001122') !== null, 'phantom node ff001122 exists before cleanup'); - assert(db.getNode('0011223344556677') !== null, 'phantom 16-char exists before cleanup'); - - // Verify real node still exists - assert(db.getNode('aabbccdd11223344aabbccdd11223344') !== null, 'real node exists before cleanup'); - - // Run cleanup - const removed = db.removePhantomNodes(); - assert(removed === 4, `removed 4 phantom nodes (got ${removed})`); - - // Verify phantoms are gone - assert(db.getNode('aabb') === null, 'phantom aabb removed'); - assert(db.getNode('ccddee') === null, 'phantom ccddee removed'); - assert(db.getNode('ff001122') === null, 'phantom ff001122 removed'); - assert(db.getNode('0011223344556677') === null, 'phantom 16-char removed'); - - // Verify real node is still there - assert(db.getNode('aabbccdd11223344aabbccdd11223344') !== null, 'real node preserved after cleanup'); - - // Running again should remove 0 - const removed2 = db.removePhantomNodes(); - assert(removed2 === 0, 'second cleanup removes nothing'); -} - -// --- stats exclude phantom nodes --- -console.log('\nstats exclude phantom nodes:'); -{ - const statsBefore = db.getStats(); - const countBefore = statsBefore.totalNodesAllTime; - - // Insert a phantom — should be cleanable - db.upsertNode({ public_key: 'deadbeef', name: null, role: 'repeater' }); - const statsWithPhantom = db.getStats(); - assert(statsWithPhantom.totalNodesAllTime === countBefore + 1, 'phantom inflates totalNodesAllTime'); - - // Clean it - db.removePhantomNodes(); - const statsAfter = db.getStats(); - assert(statsAfter.totalNodesAllTime === countBefore, 'phantom removed from totalNodesAllTime'); -} - -// --- moveStaleNodes --- -console.log('\nmoveStaleNodes:'); -{ - // Verify inactive_nodes table exists - const tables = db.db.prepare("SELECT name FROM sqlite_master WHERE type='table'").all().map(r => r.name); - assert(tables.includes('inactive_nodes'), 'inactive_nodes table exists'); - - // Verify inactive_nodes has same columns as nodes - const nodesCols = db.db.pragma('table_info(nodes)').map(c => c.name).sort(); - const inactiveCols = db.db.pragma('table_info(inactive_nodes)').map(c => c.name).sort(); - assert(JSON.stringify(nodesCols) === JSON.stringify(inactiveCols), 'inactive_nodes has same columns as nodes'); - - // Insert a stale node (last_seen 30 days ago) and a fresh node - const thirtyDaysAgo = new Date(Date.now() - 30 * 24 * 3600000).toISOString(); - const now = new Date().toISOString(); - db.upsertNode({ public_key: 'stale00000000000000000000stale000', name: 'StaleNode', role: 'repeater', last_seen: thirtyDaysAgo, first_seen: thirtyDaysAgo }); - db.upsertNode({ public_key: 'fresh00000000000000000000fresh000', name: 'FreshNode', role: 'companion', last_seen: now, first_seen: now }); - - // Verify both exist in nodes - assert(db.getNode('stale00000000000000000000stale000') !== null, 'stale node exists before move'); - assert(db.getNode('fresh00000000000000000000fresh000') !== null, 'fresh node exists before move'); - - // Move stale nodes (7 day threshold) - const moved = db.moveStaleNodes(7); - assert(moved >= 1, `moveStaleNodes moved at least 1 node (got ${moved})`); - - // Stale node should be gone from nodes - assert(db.getNode('stale00000000000000000000stale000') === null, 'stale node removed from nodes'); - - // Fresh node should still be in nodes - assert(db.getNode('fresh00000000000000000000fresh000') !== null, 'fresh node still in nodes'); - - // Stale node should be in inactive_nodes - const inactive = db.db.prepare('SELECT * FROM inactive_nodes WHERE public_key = ?').get('stale00000000000000000000stale000'); - assert(inactive !== null, 'stale node exists in inactive_nodes'); - assert(inactive.name === 'StaleNode', 'stale node name preserved in inactive_nodes'); - assert(inactive.role === 'repeater', 'stale node role preserved in inactive_nodes'); - - // Fresh node should NOT be in inactive_nodes - const freshInactive = db.db.prepare('SELECT * FROM inactive_nodes WHERE public_key = ?').get('fresh00000000000000000000fresh000'); - assert(!freshInactive, 'fresh node not in inactive_nodes'); - - // Running again should move 0 (already moved) - const moved2 = db.moveStaleNodes(7); - assert(moved2 === 0, 'second moveStaleNodes moves nothing'); - - // With nodeDays=0 should be a no-op - const moved3 = db.moveStaleNodes(0); - assert(moved3 === 0, 'moveStaleNodes(0) is a no-op'); - - // With null should be a no-op - const moved4 = db.moveStaleNodes(null); - assert(moved4 === 0, 'moveStaleNodes(null) is a no-op'); -} - -cleanup(); -delete process.env.DB_PATH; - -console.log(`\n═══════════════════════════════════════`); -console.log(` PASSED: ${passed}`); -console.log(` FAILED: ${failed}`); -console.log(`═══════════════════════════════════════`); -if (failed > 0) process.exit(1); diff --git a/test-decoder-spec.js b/test-decoder-spec.js deleted file mode 100644 index 853826fa..00000000 --- a/test-decoder-spec.js +++ /dev/null @@ -1,625 +0,0 @@ -/** - * Spec-driven tests for MeshCore decoder. - * - * Section 1: Spec assertions (from firmware/docs/packet_format.md + payloads.md) - * Section 2: Golden fixtures (from production data at analyzer.00id.net) - */ - -'use strict'; - -const { decodePacket, validateAdvert, ROUTE_TYPES, PAYLOAD_TYPES } = require('./decoder'); - -let passed = 0; -let failed = 0; -let noted = 0; - -function assert(condition, msg) { - if (condition) { passed++; } - else { failed++; console.error(` FAIL: ${msg}`); } -} - -function assertEq(actual, expected, msg) { - if (actual === expected) { passed++; } - else { failed++; console.error(` FAIL: ${msg} — expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`); } -} - -function assertDeepEq(actual, expected, msg) { - const a = JSON.stringify(actual); - const b = JSON.stringify(expected); - if (a === b) { passed++; } - else { failed++; console.error(` FAIL: ${msg}\n expected: ${b}\n got: ${a}`); } -} - -function note(msg) { - noted++; - console.log(` NOTE: ${msg}`); -} - -// ═══════════════════════════════════════════════════════════ -// Section 1: Spec-based assertions -// ═══════════════════════════════════════════════════════════ - -console.log('── Spec Tests: Header Parsing ──'); - -// Header byte: bits 1-0 = routeType, bits 5-2 = payloadType, bits 7-6 = payloadVersion -{ - // 0x11 = 0b00_0100_01 → routeType=1(FLOOD), payloadType=4(ADVERT), version=0 - const p = decodePacket('1100' + '00'.repeat(101)); // min advert = 100 bytes payload - assertEq(p.header.routeType, 1, 'header: routeType from bits 1-0'); - assertEq(p.header.payloadType, 4, 'header: payloadType from bits 5-2'); - assertEq(p.header.payloadVersion, 0, 'header: payloadVersion from bits 7-6'); - assertEq(p.header.routeTypeName, 'FLOOD', 'header: routeTypeName'); - assertEq(p.header.payloadTypeName, 'ADVERT', 'header: payloadTypeName'); -} - -// All four route types -{ - const routeNames = { 0: 'TRANSPORT_FLOOD', 1: 'FLOOD', 2: 'DIRECT', 3: 'TRANSPORT_DIRECT' }; - for (const [val, name] of Object.entries(routeNames)) { - assertEq(ROUTE_TYPES[val], name, `ROUTE_TYPES[${val}] = ${name}`); - } -} - -// All payload types from spec -{ - const specTypes = { - 0x00: 'REQ', 0x01: 'RESPONSE', 0x02: 'TXT_MSG', 0x03: 'ACK', - 0x04: 'ADVERT', 0x05: 'GRP_TXT', 0x07: 'ANON_REQ', - 0x08: 'PATH', 0x09: 'TRACE', - }; - for (const [val, name] of Object.entries(specTypes)) { - assertEq(PAYLOAD_TYPES[val], name, `PAYLOAD_TYPES[${val}] = ${name}`); - } -} - -// Spec defines 0x06=GRP_DATA, 0x0A=MULTIPART, 0x0B=CONTROL, 0x0F=RAW_CUSTOM — decoder may not have them -{ - if (!PAYLOAD_TYPES[0x06]) note('Decoder missing PAYLOAD_TYPE 0x06 (GRP_DATA) — spec defines it'); - if (!PAYLOAD_TYPES[0x0A]) note('Decoder missing PAYLOAD_TYPE 0x0A (MULTIPART) — spec defines it'); - if (!PAYLOAD_TYPES[0x0B]) note('Decoder missing PAYLOAD_TYPE 0x0B (CONTROL) — spec defines it'); - if (!PAYLOAD_TYPES[0x0F]) note('Decoder missing PAYLOAD_TYPE 0x0F (RAW_CUSTOM) — spec defines it'); -} - -console.log('── Spec Tests: Path Byte Parsing ──'); - -// path_length: bits 5-0 = hop count, bits 7-6 = hash_size - 1 -{ - // 0x00: 0 hops, 1-byte hashes - const p0 = decodePacket('0500' + '00'.repeat(10)); - assertEq(p0.path.hashCount, 0, 'path 0x00: hashCount=0'); - assertEq(p0.path.hashSize, 1, 'path 0x00: hashSize=1'); - assertDeepEq(p0.path.hops, [], 'path 0x00: no hops'); -} - -{ - // 0x05: 5 hops, 1-byte hashes → 5 path bytes - const p5 = decodePacket('0505' + 'AABBCCDDEE' + '00'.repeat(10)); - assertEq(p5.path.hashCount, 5, 'path 0x05: hashCount=5'); - assertEq(p5.path.hashSize, 1, 'path 0x05: hashSize=1'); - assertEq(p5.path.hops.length, 5, 'path 0x05: 5 hops'); - assertEq(p5.path.hops[0], 'AA', 'path 0x05: first hop'); - assertEq(p5.path.hops[4], 'EE', 'path 0x05: last hop'); -} - -{ - // 0x45: 5 hops, 2-byte hashes (bits 7-6 = 01) → 10 path bytes - const p45 = decodePacket('0545' + 'AA11BB22CC33DD44EE55' + '00'.repeat(10)); - assertEq(p45.path.hashCount, 5, 'path 0x45: hashCount=5'); - assertEq(p45.path.hashSize, 2, 'path 0x45: hashSize=2'); - assertEq(p45.path.hops.length, 5, 'path 0x45: 5 hops'); - assertEq(p45.path.hops[0], 'AA11', 'path 0x45: first hop (2-byte)'); -} - -{ - // 0x8A: 10 hops, 3-byte hashes (bits 7-6 = 10) → 30 path bytes - const p8a = decodePacket('058A' + 'AA11FF'.repeat(10) + '00'.repeat(10)); - assertEq(p8a.path.hashCount, 10, 'path 0x8A: hashCount=10'); - assertEq(p8a.path.hashSize, 3, 'path 0x8A: hashSize=3'); - assertEq(p8a.path.hops.length, 10, 'path 0x8A: 10 hops'); -} - -console.log('── Spec Tests: Transport Codes ──'); - -{ - // Route type 0 (TRANSPORT_FLOOD) and 3 (TRANSPORT_DIRECT) should have 4-byte transport codes - // Route type 0: header=0x14 = payloadType 5 (GRP_TXT), routeType 0 (TRANSPORT_FLOOD) - // Format: header(1) + transportCodes(4) + pathByte(1) + payload - const hex = '14' + 'AABB' + 'CCDD' + '00' + '1A' + '00'.repeat(10); // transport codes + pathByte + GRP_TXT payload - const p = decodePacket(hex); - assertEq(p.header.routeType, 0, 'transport: routeType=0 (TRANSPORT_FLOOD)'); - assert(p.transportCodes !== null, 'transport: transportCodes present for TRANSPORT_FLOOD'); - assertEq(p.transportCodes.code1, 'AABB', 'transport: code1'); - assertEq(p.transportCodes.code2, 'CCDD', 'transport: code2'); -} - -{ - // Route type 1 (FLOOD) should NOT have transport codes - const p = decodePacket('0500' + '00'.repeat(10)); - assertEq(p.transportCodes, null, 'no transport codes for FLOOD'); -} - -console.log('── Spec Tests: Advert Payload ──'); - -// Advert: pubkey(32) + timestamp(4 LE) + signature(64) + appdata -{ - const pubkey = 'AA'.repeat(32); - const timestamp = '78563412'; // 0x12345678 LE = 305419896 - const signature = 'BB'.repeat(64); - // flags: 0x92 = repeater(2) | hasLocation(0x10) | hasName(0x80) - const flags = '92'; - // lat: 37000000 = 0x02353A80 LE → 80 3A 35 02 - const lat = '40933402'; - // lon: -122100000 = 0xF8B9E260 LE → 60 E2 B9 F8 - const lon = 'E0E6B8F8'; - const name = Buffer.from('TestNode').toString('hex'); - - const hex = '1200' + pubkey + timestamp + signature + flags + lat + lon + name; - const p = decodePacket(hex); - - assertEq(p.payload.type, 'ADVERT', 'advert: payload type'); - assertEq(p.payload.pubKey, pubkey.toLowerCase(), 'advert: 32-byte pubkey'); - assertEq(p.payload.timestamp, 0x12345678, 'advert: uint32 LE timestamp'); - assertEq(p.payload.signature, signature.toLowerCase().repeat(1), 'advert: 64-byte signature'); - - // Flags - assertEq(p.payload.flags.raw, 0x92, 'advert flags: raw byte'); - assertEq(p.payload.flags.type, 2, 'advert flags: type enum = 2 (repeater)'); - assertEq(p.payload.flags.repeater, true, 'advert flags: repeater'); - assertEq(p.payload.flags.room, false, 'advert flags: not room'); - assertEq(p.payload.flags.chat, false, 'advert flags: not chat'); - assertEq(p.payload.flags.sensor, false, 'advert flags: not sensor'); - assertEq(p.payload.flags.hasLocation, true, 'advert flags: hasLocation (bit 4)'); - assertEq(p.payload.flags.hasName, true, 'advert flags: hasName (bit 7)'); - - // Location: int32 at 1e6 scale - assert(Math.abs(p.payload.lat - 37.0) < 0.001, 'advert: lat decoded from int32/1e6'); - assert(Math.abs(p.payload.lon - (-122.1)) < 0.001, 'advert: lon decoded from int32/1e6'); - - // Name - assertEq(p.payload.name, 'TestNode', 'advert: name from remaining appdata'); -} - -// Advert type enum values per spec -{ - // type 0 = none (companion), 1 = chat/companion, 2 = repeater, 3 = room, 4 = sensor - const makeAdvert = (flagsByte) => { - const hex = '1200' + 'AA'.repeat(32) + '00000000' + 'BB'.repeat(64) + flagsByte.toString(16).padStart(2, '0'); - return decodePacket(hex).payload; - }; - - const t1 = makeAdvert(0x01); - assertEq(t1.flags.type, 1, 'advert type 1 = chat/companion'); - assertEq(t1.flags.chat, true, 'type 1: chat=true'); - - const t2 = makeAdvert(0x02); - assertEq(t2.flags.type, 2, 'advert type 2 = repeater'); - assertEq(t2.flags.repeater, true, 'type 2: repeater=true'); - - const t3 = makeAdvert(0x03); - assertEq(t3.flags.type, 3, 'advert type 3 = room'); - assertEq(t3.flags.room, true, 'type 3: room=true'); - - const t4 = makeAdvert(0x04); - assertEq(t4.flags.type, 4, 'advert type 4 = sensor'); - assertEq(t4.flags.sensor, true, 'type 4: sensor=true'); -} - -// Advert with no location, no name (flags = 0x02, just repeater) -{ - const hex = '1200' + 'CC'.repeat(32) + '00000000' + 'DD'.repeat(64) + '02'; - const p = decodePacket(hex).payload; - assertEq(p.flags.hasLocation, false, 'advert no location: hasLocation=false'); - assertEq(p.flags.hasName, false, 'advert no name: hasName=false'); - assertEq(p.lat, undefined, 'advert no location: lat undefined'); - assertEq(p.name, undefined, 'advert no name: name undefined'); -} - -// Telemetry: sensor node with battery + positive temperature -{ - const pubkey = 'AA'.repeat(32); - const sig = 'BB'.repeat(64); - const flags = '84'; // sensor(4) | hasName(0x80) - const name = Buffer.from('S1').toString('hex') + '00'; // null-terminated - const battBuf = Buffer.alloc(2); battBuf.writeUInt16LE(3700); - const tempBuf = Buffer.alloc(2); tempBuf.writeInt16LE(2850); // 28.50°C - const hex = '1200' + pubkey + '00000000' + sig + flags + name + - battBuf.toString('hex') + tempBuf.toString('hex'); - const p = decodePacket(hex).payload; - assertEq(p.battery_mv, 3700, 'telemetry: battery_mv decoded'); - assert(Math.abs(p.temperature_c - 28.50) < 0.01, 'telemetry: temperature_c positive'); -} - -// Telemetry: sensor node with 0°C must still emit temperature_c -{ - const pubkey = 'CC'.repeat(32); - const sig = 'DD'.repeat(64); - const flags = '84'; // sensor(4) | hasName(0x80) - const name = Buffer.from('S2').toString('hex') + '00'; - const battBuf = Buffer.alloc(2); battBuf.writeUInt16LE(3600); - const tempBuf = Buffer.alloc(2); // 0°C - const hex = '1200' + pubkey + '00000000' + sig + flags + name + - battBuf.toString('hex') + tempBuf.toString('hex'); - const p = decodePacket(hex).payload; - assert(p.temperature_c === 0, 'telemetry: 0°C is valid and emitted'); -} - -// Telemetry: non-sensor node with trailing bytes must NOT decode telemetry -{ - const pubkey = 'EE'.repeat(32); - const sig = 'FF'.repeat(64); - const flags = '82'; // repeater(2) | hasName(0x80) - const name = Buffer.from('R1').toString('hex') + '00'; - const extraBytes = 'B40ED403'; // battery-like and temp-like bytes - const hex = '1200' + pubkey + '00000000' + sig + flags + name + extraBytes; - const p = decodePacket(hex).payload; - assertEq(p.battery_mv, undefined, 'telemetry: non-sensor node: battery_mv must be undefined'); - assertEq(p.temperature_c, undefined, 'telemetry: non-sensor node: temperature_c must be undefined'); -} - -console.log('── Spec Tests: Encrypted Payload Format ──'); - -// Spec says v1 encrypted payloads: dest(1)+src(1)+MAC(2)+cipher — decoder matches this. -{ - const hex = '0100' + 'AA' + 'BB' + 'CCDD' + '00'.repeat(10); - const p = decodePacket(hex); - assertEq(p.payload.destHash, 'aa', 'encrypted payload: dest is 1 byte'); - assertEq(p.payload.srcHash, 'bb', 'encrypted payload: src is 1 byte'); - assertEq(p.payload.mac, 'ccdd', 'encrypted payload: MAC is 2 bytes'); -} - -console.log('── Spec Tests: validateAdvert ──'); - -{ - const good = { pubKey: 'aa'.repeat(32), flags: { repeater: true, room: false, sensor: false } }; - assertEq(validateAdvert(good).valid, true, 'validateAdvert: good advert'); - - assertEq(validateAdvert(null).valid, false, 'validateAdvert: null'); - assertEq(validateAdvert({ error: 'bad' }).valid, false, 'validateAdvert: error advert'); - assertEq(validateAdvert({ pubKey: 'aa' }).valid, false, 'validateAdvert: short pubkey'); - assertEq(validateAdvert({ pubKey: '00'.repeat(32) }).valid, false, 'validateAdvert: all-zero pubkey'); - - const badLat = { pubKey: 'aa'.repeat(32), lat: 999 }; - assertEq(validateAdvert(badLat).valid, false, 'validateAdvert: invalid lat'); - - const badLon = { pubKey: 'aa'.repeat(32), lon: -999 }; - assertEq(validateAdvert(badLon).valid, false, 'validateAdvert: invalid lon'); - - const badName = { pubKey: 'aa'.repeat(32), name: 'test\x00name' }; - assertEq(validateAdvert(badName).valid, false, 'validateAdvert: control chars in name'); - - const longName = { pubKey: 'aa'.repeat(32), name: 'x'.repeat(65) }; - assertEq(validateAdvert(longName).valid, false, 'validateAdvert: name too long'); -} - -// ═══════════════════════════════════════════════════════════ -// Section 2: Golden fixtures (from production) -// ═══════════════════════════════════════════════════════════ - -console.log('── Golden Tests: Production Packets ──'); - -const goldenFixtures = [ - { - "raw_hex": "0A00D69FD7A5A7475DB07337749AE61FA53A4788E976", - "payload_type": 2, - "route_type": 2, - "decoded": "{\"type\":\"TXT_MSG\",\"destHash\":\"d6\",\"srcHash\":\"9f\",\"mac\":\"d7a5\",\"encryptedData\":\"a7475db07337749ae61fa53a4788e976\"}", - "path": { - "hashSize": 1, - "hashCount": 0, - "hops": [] - } - }, - { - "raw_hex": "0A009FD605771EE2EB0CDC46D100232B455947E3C2D4B9DD0B8880EACA99A3C5F7EF63183D6D", - "payload_type": 2, - "route_type": 2, - "decoded": "{\"type\":\"TXT_MSG\",\"destHash\":\"9f\",\"srcHash\":\"d6\",\"mac\":\"0577\",\"encryptedData\":\"1ee2eb0cdc46d100232b455947e3c2d4b9dd0b8880eaca99a3c5f7ef63183d6d\"}", - "path": { - "hashSize": 1, - "hashCount": 0, - "hops": [] - } - }, - { - "raw_hex": "120046D62DE27D4C5194D7821FC5A34A45565DCC2537B300B9AB6275255CEFB65D840CE5C169C94C9AED39E8BCB6CB6EB0335497A198B33A1A610CD3B03D8DCFC160900E5244280323EE0B44CACAB8F02B5B38B91CFA18BD067B0B5E63E94CFC85F758A8530B9240933402E0E6B8F84D5252322D52", - "payload_type": 4, - "route_type": 2, - "decoded": "{\"type\":\"ADVERT\",\"pubKey\":\"46d62de27d4c5194d7821fc5a34a45565dcc2537b300b9ab6275255cefb65d84\",\"timestamp\":1774314764,\"timestampISO\":\"2026-03-24T01:12:44.000Z\",\"signature\":\"c94c9aed39e8bcb6cb6eb0335497a198b33a1a610cd3b03d8dcfc160900e5244280323ee0b44cacab8f02b5b38b91cfa18bd067b0b5e63e94cfc85f758a8530b\",\"flags\":{\"raw\":146,\"type\":2,\"chat\":false,\"repeater\":true,\"room\":false,\"sensor\":false,\"hasLocation\":true,\"hasName\":true},\"lat\":37,\"lon\":-122.1,\"name\":\"MRR2-R\"}", - "path": { - "hashSize": 1, - "hashCount": 0, - "hops": [] - } - }, - { - "raw_hex": "120073CFF971E1CB5754A742C152B2D2E0EB108A19B246D663ED8898A72C4A5AD86EA6768E66694B025EDF6939D5C44CFF719C5D5520E5F06B20680A83AD9C2C61C3227BBB977A85EE462F3553445FECF8EDD05C234ECE217272E503F14D6DF2B1B9B133890C923CDF3002F8FDC1F85045414BF09F8CB3", - "payload_type": 4, - "route_type": 2, - "decoded": "{\"type\":\"ADVERT\",\"pubKey\":\"73cff971e1cb5754a742c152b2d2e0eb108a19b246d663ed8898a72c4a5ad86e\",\"timestamp\":1720612518,\"timestampISO\":\"2024-07-10T11:55:18.000Z\",\"signature\":\"694b025edf6939d5c44cff719c5d5520e5f06b20680a83ad9c2c61c3227bbb977a85ee462f3553445fecf8edd05c234ece217272e503f14d6df2b1b9b133890c\",\"flags\":{\"raw\":146,\"type\":2,\"chat\":false,\"repeater\":true,\"room\":false,\"sensor\":false,\"hasLocation\":true,\"hasName\":true},\"lat\":36.757308,\"lon\":-121.504264,\"name\":\"PEAK🌳\"}", - "path": { - "hashSize": 1, - "hashCount": 0, - "hops": [] - } - }, - { - "raw_hex": "06001f33e1bef15f5596b394adf03a77d46b89afa2e3", - "payload_type": 1, - "route_type": 2, - "decoded": "{\"type\":\"RESPONSE\",\"destHash\":\"1f\",\"srcHash\":\"33\",\"mac\":\"e1be\",\"encryptedData\":\"f15f5596b394adf03a77d46b89afa2e3\"}", - "path": { - "hashSize": 1, - "hashCount": 0, - "hops": [] - } - }, - { - "raw_hex": "0200331fe52805e05cf6f4bae6a094ac258d57baf045", - "payload_type": 0, - "route_type": 2, - "decoded": "{\"type\":\"REQ\",\"destHash\":\"33\",\"srcHash\":\"1f\",\"mac\":\"e528\",\"encryptedData\":\"05e05cf6f4bae6a094ac258d57baf045\"}", - "path": { - "hashSize": 1, - "hashCount": 0, - "hops": [] - } - }, - { - "raw_hex": "15001ABC314305D3CCC94EB3F398D3054B4E95899229027B027E450FD68B4FA4E0A0126AC1", - "payload_type": 5, - "route_type": 1, - "decoded": "{\"type\":\"GRP_TXT\",\"channelHash\":26,\"mac\":\"bc31\",\"encryptedData\":\"4305d3ccc94eb3f398d3054b4e95899229027b027e450fd68b4fa4e0a0126ac1\"}", - "path": { - "hashSize": 1, - "hashCount": 0, - "hops": [] - } - }, - { - "raw_hex": "010673a210206cb51e42fee24c4847a99208b9fc1d7ab36c42b10748", - "payload_type": 0, - "route_type": 1, - "decoded": "{\"type\":\"REQ\",\"destHash\":\"1e\",\"srcHash\":\"42\",\"mac\":\"fee2\",\"encryptedData\":\"4c4847a99208b9fc1d7ab36c42b10748\"}", - "path": { - "hashSize": 1, - "hashCount": 6, - "hops": [ - "73", - "A2", - "10", - "20", - "6C", - "B5" - ] - } - }, - { - "raw_hex": "0101731E42FEE24C4847A99208293810E4A3E335640D8E", - "payload_type": 0, - "route_type": 1, - "decoded": "{\"type\":\"REQ\",\"destHash\":\"1e\",\"srcHash\":\"42\",\"mac\":\"fee2\",\"encryptedData\":\"4c4847a99208293810e4a3e335640d8e\"}", - "path": { - "hashSize": 1, - "hashCount": 1, - "hops": [ - "73" - ] - } - }, - { - "raw_hex": "0106FB10844070101E42BA859D1D939362F79D3F3865333629FF92E9", - "payload_type": 0, - "route_type": 1, - "decoded": "{\"type\":\"REQ\",\"destHash\":\"1e\",\"srcHash\":\"42\",\"mac\":\"ba85\",\"encryptedData\":\"9d1d939362f79d3f3865333629ff92e9\"}", - "path": { - "hashSize": 1, - "hashCount": 6, - "hops": [ - "FB", - "10", - "84", - "40", - "70", - "10" - ] - } - }, - { - "raw_hex": "0102FB101E42BA859D1D939362F79D3F3865333629FF92D9", - "payload_type": 0, - "route_type": 1, - "decoded": "{\"type\":\"REQ\",\"destHash\":\"1e\",\"srcHash\":\"42\",\"mac\":\"ba85\",\"encryptedData\":\"9d1d939362f79d3f3865333629ff92d9\"}", - "path": { - "hashSize": 1, - "hashCount": 2, - "hops": [ - "FB", - "10" - ] - } - }, - { - "raw_hex": "22009FD65B38857C5A7F6F0F28E999CF2632C03ACCCC", - "payload_type": 8, - "route_type": 2, - "decoded": "{\"type\":\"PATH\",\"destHash\":\"9f\",\"srcHash\":\"d6\",\"mac\":\"5b38\",\"pathData\":\"857c5a7f6f0f28e999cf2632c03acccc\"}", - "path": { - "hashSize": 1, - "hashCount": 0, - "hops": [] - } - }, - { - "raw_hex": "0506701085AD8573D69F96FA7DD3B1AC3702794035442D9CDAD436D4", - "payload_type": 1, - "route_type": 1, - "decoded": "{\"type\":\"RESPONSE\",\"destHash\":\"d6\",\"srcHash\":\"9f\",\"mac\":\"96fa\",\"encryptedData\":\"7dd3b1ac3702794035442d9cdad436d4\"}", - "path": { - "hashSize": 1, - "hashCount": 6, - "hops": [ - "70", - "10", - "85", - "AD", - "85", - "73" - ] - } - }, - { - "raw_hex": "0500D69F96FA7DD3B1AC3702794035442D9CDAD43654", - "payload_type": 1, - "route_type": 1, - "decoded": "{\"type\":\"RESPONSE\",\"destHash\":\"d6\",\"srcHash\":\"9f\",\"mac\":\"96fa\",\"encryptedData\":\"7dd3b1ac3702794035442d9cdad43654\"}", - "path": { - "hashSize": 1, - "hashCount": 0, - "hops": [] - } - }, - { - "raw_hex": "1E009FD6DFC543C53E826A2B789B072FF9CBE922E57EA093E5643A0CA813E79F42EE9108F855B72A3E0B599C9AC80D3A211E7C7BA2", - "payload_type": 7, - "route_type": 2, - "decoded": "{\"type\":\"ANON_REQ\",\"destHash\":\"9f\",\"ephemeralPubKey\":\"d6dfc543c53e826a2b789b072ff9cbe922e57ea093e5643a0ca813e79f42ee91\",\"mac\":\"08f8\",\"encryptedData\":\"55b72a3e0b599c9ac80d3a211e7c7ba2\"}", - "path": { - "hashSize": 1, - "hashCount": 0, - "hops": [] - } - }, - { - "raw_hex": "110146B7F1C45F2ED5888335F79E27085D0DE871A7C8ECB1EF5313435EBD0825BACDC181E3C1695556F51A89C9895E2114D1FECA91B58F82CBBBC1DD2B868ADDC0F7EB8C310D0887C2A2283D6F7D01A5E97B6C2F6A4CC899F27AFA513CC6B295E34ADC84A1F1019240933402E0E6B8F84D6574726F2D52", - "payload_type": 4, - "route_type": 1, - "decoded": "{\"type\":\"ADVERT\",\"pubKey\":\"b7f1c45f2ed5888335f79e27085d0de871a7c8ecb1ef5313435ebd0825bacdc1\",\"timestamp\":1774314369,\"timestampISO\":\"2026-03-24T01:06:09.000Z\",\"signature\":\"5556f51a89c9895e2114d1feca91b58f82cbbbc1dd2b868addc0f7eb8c310d0887c2a2283d6f7d01a5e97b6c2f6a4cc899f27afa513cc6b295e34adc84a1f101\",\"flags\":{\"raw\":146,\"type\":2,\"chat\":false,\"repeater\":true,\"room\":false,\"sensor\":false,\"hasLocation\":true,\"hasName\":true},\"lat\":37,\"lon\":-122.1,\"name\":\"Metro-R\"}", - "path": { - "hashSize": 1, - "hashCount": 1, - "hops": [ - "46" - ] - } - }, - { - "raw_hex": "15001A901C5D927D90572BAF6135D226F91D180AD4F7B90DF20F82EEEA920312D9CCFD9C3F8CA9EFBEB1C37DFA31265F73483BD0640EC94E247902F617B2C320BFA332F50441AD234D8324A48ABAA9A16EB15BD50F2D67029F2424E0836010A635EB45B5DFDB4CDC080C09FC849040AB4B82769E0F", - "payload_type": 5, - "route_type": 1, - "decoded": "{\"type\":\"GRP_TXT\",\"channelHash\":26,\"mac\":\"901c\",\"encryptedData\":\"5d927d90572baf6135d226f91d180ad4f7b90df20f82eeea920312d9ccfd9c3f8ca9efbeb1c37dfa31265f73483bd0640ec94e247902f617b2c320bfa332f50441ad234d8324a48abaa9a16eb15bd50f2d67029f2424e0836010a635eb45b5dfdb4cdc080c09fc849040ab4b82769e0f\"}", - "path": { - "hashSize": 1, - "hashCount": 0, - "hops": [] - } - }, - { - "raw_hex": "0A00D69F0E65C6CCDEBE8391ED093D3C76E2D064F525", - "payload_type": 2, - "route_type": 2, - "decoded": "{\"type\":\"TXT_MSG\",\"destHash\":\"d6\",\"srcHash\":\"9f\",\"mac\":\"0e65\",\"encryptedData\":\"c6ccdebe8391ed093d3c76e2d064f525\"}", - "path": { - "hashSize": 1, - "hashCount": 0, - "hops": [] - } - }, - { - "raw_hex": "0A00D69F940E0BA255095E9540EE6E23895DA80AAC60", - "payload_type": 2, - "route_type": 2, - "decoded": "{\"type\":\"TXT_MSG\",\"destHash\":\"d6\",\"srcHash\":\"9f\",\"mac\":\"940e\",\"encryptedData\":\"0ba255095e9540ee6e23895da80aac60\"}", - "path": { - "hashSize": 1, - "hashCount": 0, - "hops": [] - } - }, - { - "raw_hex": "06001f5d5acf699ea80c7ca1a9349b8af9a1b47d4a1a", - "payload_type": 1, - "route_type": 2, - "decoded": "{\"type\":\"RESPONSE\",\"destHash\":\"1f\",\"srcHash\":\"5d\",\"mac\":\"5acf\",\"encryptedData\":\"699ea80c7ca1a9349b8af9a1b47d4a1a\"}", - "path": { - "hashSize": 1, - "hashCount": 0, - "hops": [] - } - } -]; - -// One special case: the advert with 1 hop from prod had raw_hex starting with "110146" -// but the API reported path ["46"]. Let me re-check — header 0x11 = routeType 1, payloadType 4. -// pathByte 0x01 = 1 hop, 1-byte hash. Next byte is 0x46 = the hop. Correct. -// However, the raw_hex I captured from the API was "110146B7F1..." but the actual prod JSON showed path ["46"]. -// I need to use the correct raw_hex. Let me fix fixture 15 (Metro-R advert). - -for (let i = 0; i < goldenFixtures.length; i++) { - const fix = goldenFixtures[i]; - const expected = typeof fix.decoded === "string" ? JSON.parse(fix.decoded) : fix.decoded; - const label = `golden[${i}] ${expected.type}`; - - try { - const result = decodePacket(fix.raw_hex); - - // Verify header matches expected route/payload type - assertEq(result.header.routeType, fix.route_type, `${label}: routeType`); - assertEq(result.header.payloadType, fix.payload_type, `${label}: payloadType`); - - // Verify path hops - assertDeepEq(result.path.hops, (fix.path.hops || fix.path), `${label}: path hops`); - - // Verify payload matches prod decoded output - // Compare key fields rather than full deep equality (to handle minor serialization diffs) - - assertEq(result.payload.type, expected.type, `${label}: payload type`); - - if (expected.type === 'ADVERT') { - assertEq(result.payload.pubKey, expected.pubKey, `${label}: pubKey`); - assertEq(result.payload.timestamp, expected.timestamp, `${label}: timestamp`); - assertEq(result.payload.signature, expected.signature, `${label}: signature`); - if (expected.flags) { - assertEq(result.payload.flags.raw, expected.flags.raw, `${label}: flags.raw`); - assertEq(result.payload.flags.type, expected.flags.type, `${label}: flags.type`); - assertEq(result.payload.flags.hasLocation, expected.flags.hasLocation, `${label}: hasLocation`); - assertEq(result.payload.flags.hasName, expected.flags.hasName, `${label}: hasName`); - } - if (expected.lat != null) assert(Math.abs(result.payload.lat - expected.lat) < 0.001, `${label}: lat`); - if (expected.lon != null) assert(Math.abs(result.payload.lon - expected.lon) < 0.001, `${label}: lon`); - if (expected.name) assertEq(result.payload.name, expected.name, `${label}: name`); - - // Spec checks on advert structure - assert(result.payload.pubKey.length === 64, `${label}: pubKey is 32 bytes (64 hex chars)`); - assert(result.payload.signature.length === 128, `${label}: signature is 64 bytes (128 hex chars)`); - } else if (expected.type === 'GRP_TXT' || expected.type === 'CHAN') { - assertEq(result.payload.channelHash, expected.channelHash, `${label}: channelHash`); - // If decoded as CHAN (with channel key), check sender/text; otherwise check mac/encrypted - if (expected.type === 'GRP_TXT') { - assertEq(result.payload.mac, expected.mac, `${label}: mac`); - assertEq(result.payload.encryptedData, expected.encryptedData, `${label}: encryptedData`); - } - } else if (expected.type === 'ANON_REQ') { - assertEq(result.payload.destHash, expected.destHash, `${label}: destHash`); - assertEq(result.payload.ephemeralPubKey, expected.ephemeralPubKey, `${label}: ephemeralPubKey`); - assertEq(result.payload.mac, expected.mac, `${label}: mac`); - } else { - // Encrypted payload types: REQ, RESPONSE, TXT_MSG, PATH - assertEq(result.payload.destHash, expected.destHash, `${label}: destHash`); - assertEq(result.payload.srcHash, expected.srcHash, `${label}: srcHash`); - assertEq(result.payload.mac, expected.mac, `${label}: mac`); - if (expected.encryptedData) assertEq(result.payload.encryptedData, expected.encryptedData, `${label}: encryptedData`); - if (expected.pathData) assertEq(result.payload.pathData, expected.pathData, `${label}: pathData`); - } - } catch (e) { - failed++; - console.error(` FAIL: ${label} — threw: ${e.message}`); - } -} - -// ═══════════════════════════════════════════════════════════ -// Summary -// ═══════════════════════════════════════════════════════════ - -console.log(''); -console.log(`═══ Results: ${passed} passed, ${failed} failed, ${noted} notes ═══`); -if (failed > 0) process.exit(1); diff --git a/test-decoder.js b/test-decoder.js deleted file mode 100644 index 28efcfd4..00000000 --- a/test-decoder.js +++ /dev/null @@ -1,630 +0,0 @@ -/* Unit tests for decoder.js */ -'use strict'; -const assert = require('assert'); -const { decodePacket, validateAdvert, ROUTE_TYPES, PAYLOAD_TYPES, VALID_ROLES } = require('./decoder'); - -let passed = 0, failed = 0; -function test(name, fn) { - try { fn(); passed++; console.log(` ✅ ${name}`); } - catch (e) { failed++; console.log(` ❌ ${name}: ${e.message}`); } -} - -// === Constants === -console.log('\n=== Constants ==='); -test('ROUTE_TYPES has 4 entries', () => assert.strictEqual(Object.keys(ROUTE_TYPES).length, 4)); -test('PAYLOAD_TYPES has 13 entries', () => assert.strictEqual(Object.keys(PAYLOAD_TYPES).length, 13)); -test('VALID_ROLES has repeater, companion, room, sensor', () => { - for (const r of ['repeater', 'companion', 'room', 'sensor']) assert(VALID_ROLES.has(r)); -}); - -// === Header decoding === -console.log('\n=== Header decoding ==='); -test('FLOOD + ADVERT = 0x11', () => { - const p = decodePacket('1100' + '00'.repeat(101)); - assert.strictEqual(p.header.routeType, 1); - assert.strictEqual(p.header.routeTypeName, 'FLOOD'); - assert.strictEqual(p.header.payloadType, 4); - assert.strictEqual(p.header.payloadTypeName, 'ADVERT'); -}); - -test('TRANSPORT_FLOOD = routeType 0', () => { - // header=0x00 (TRANSPORT_FLOOD + REQ), transportCodes=AABB+CCDD, pathByte=0x00, payload - const hex = '00' + 'AABB' + 'CCDD' + '00' + '00'.repeat(16); - const p = decodePacket(hex); - assert.strictEqual(p.header.routeType, 0); - assert.strictEqual(p.header.routeTypeName, 'TRANSPORT_FLOOD'); - assert.notStrictEqual(p.transportCodes, null); - assert.strictEqual(p.transportCodes.code1, 'AABB'); - assert.strictEqual(p.transportCodes.code2, 'CCDD'); -}); - -test('TRANSPORT_DIRECT = routeType 3', () => { - const hex = '03' + '1122' + '3344' + '00' + '00'.repeat(16); - const p = decodePacket(hex); - assert.strictEqual(p.header.routeType, 3); - assert.strictEqual(p.header.routeTypeName, 'TRANSPORT_DIRECT'); - assert.strictEqual(p.transportCodes.code1, '1122'); -}); - -test('DIRECT = routeType 2, no transport codes', () => { - const hex = '0200' + '00'.repeat(16); - const p = decodePacket(hex); - assert.strictEqual(p.header.routeType, 2); - assert.strictEqual(p.header.routeTypeName, 'DIRECT'); - assert.strictEqual(p.transportCodes, null); -}); - -test('payload version extracted', () => { - // 0xC1 = 11_0000_01 → version=3, payloadType=0, routeType=1 - const hex = 'C100' + '00'.repeat(16); - const p = decodePacket(hex); - assert.strictEqual(p.header.payloadVersion, 3); -}); - -// === Path decoding === -console.log('\n=== Path decoding ==='); -test('hashSize=1, hashCount=3', () => { - // pathByte = 0x03 → (0>>6)+1=1, 3&0x3F=3 - const hex = '1103' + 'AABBCC' + '00'.repeat(101); - const p = decodePacket(hex); - assert.strictEqual(p.path.hashSize, 1); - assert.strictEqual(p.path.hashCount, 3); - assert.strictEqual(p.path.hops.length, 3); - assert.strictEqual(p.path.hops[0], 'AA'); - assert.strictEqual(p.path.hops[1], 'BB'); - assert.strictEqual(p.path.hops[2], 'CC'); -}); - -test('hashSize=2, hashCount=2', () => { - // pathByte = 0x42 → (1>>0=1)+1=2, 2&0x3F=2 - const hex = '1142' + 'AABB' + 'CCDD' + '00'.repeat(101); - const p = decodePacket(hex); - assert.strictEqual(p.path.hashSize, 2); - assert.strictEqual(p.path.hashCount, 2); - assert.strictEqual(p.path.hops[0], 'AABB'); - assert.strictEqual(p.path.hops[1], 'CCDD'); -}); - -test('hashSize=4 from pathByte 0xC1', () => { - // 0xC1 = 11_000001 → hashSize=(3)+1=4, hashCount=1 - const hex = '11C1' + 'DEADBEEF' + '00'.repeat(101); - const p = decodePacket(hex); - assert.strictEqual(p.path.hashSize, 4); - assert.strictEqual(p.path.hashCount, 1); - assert.strictEqual(p.path.hops[0], 'DEADBEEF'); -}); - -test('zero hops', () => { - const hex = '1100' + '00'.repeat(101); - const p = decodePacket(hex); - assert.strictEqual(p.path.hashCount, 0); - assert.strictEqual(p.path.hops.length, 0); -}); - -// === Payload types === -console.log('\n=== ADVERT payload ==='); -test('ADVERT with name and location', () => { - const pkt = decodePacket( - '11451000D818206D3AAC152C8A91F89957E6D30CA51F36E28790228971C473B755F244F718754CF5EE4A2FD58D944466E42CDED140C66D0CC590183E32BAF40F112BE8F3F2BDF6012B4B2793C52F1D36F69EE054D9A05593286F78453E56C0EC4A3EB95DDA2A7543FCCC00B939CACC009278603902FC12BCF84B706120526F6F6620536F6C6172' - ); - assert.strictEqual(pkt.payload.type, 'ADVERT'); - assert.strictEqual(pkt.payload.name, 'Kpa Roof Solar'); - assert(pkt.payload.pubKey.length === 64); - assert(pkt.payload.timestamp > 0); - assert(pkt.payload.timestampISO); - assert(pkt.payload.signature.length === 128); -}); - -test('ADVERT flags: chat type=1', () => { - const pubKey = 'AB'.repeat(32); - const ts = '01000000'; - const sig = 'CC'.repeat(64); - const flags = '01'; // type=1 → chat - const hex = '1100' + pubKey + ts + sig + flags; - const p = decodePacket(hex); - assert.strictEqual(p.payload.flags.type, 1); - assert.strictEqual(p.payload.flags.chat, true); - assert.strictEqual(p.payload.flags.repeater, false); -}); - -test('ADVERT flags: repeater type=2', () => { - const pubKey = 'AB'.repeat(32); - const ts = '01000000'; - const sig = 'CC'.repeat(64); - const flags = '02'; - const hex = '1100' + pubKey + ts + sig + flags; - const p = decodePacket(hex); - assert.strictEqual(p.payload.flags.type, 2); - assert.strictEqual(p.payload.flags.repeater, true); -}); - -test('ADVERT flags: room type=3', () => { - const pubKey = 'AB'.repeat(32); - const ts = '01000000'; - const sig = 'CC'.repeat(64); - const flags = '03'; - const hex = '1100' + pubKey + ts + sig + flags; - const p = decodePacket(hex); - assert.strictEqual(p.payload.flags.type, 3); - assert.strictEqual(p.payload.flags.room, true); -}); - -test('ADVERT flags: sensor type=4', () => { - const pubKey = 'AB'.repeat(32); - const ts = '01000000'; - const sig = 'CC'.repeat(64); - const flags = '04'; - const hex = '1100' + pubKey + ts + sig + flags; - const p = decodePacket(hex); - assert.strictEqual(p.payload.flags.type, 4); - assert.strictEqual(p.payload.flags.sensor, true); -}); - -test('ADVERT flags: hasLocation', () => { - const pubKey = 'AB'.repeat(32); - const ts = '01000000'; - const sig = 'CC'.repeat(64); - // flags=0x12 → type=2(repeater), hasLocation=true - const flags = '12'; - const lat = '40420f00'; // 1000000 → 1.0 degrees - const lon = '80841e00'; // 2000000 → 2.0 degrees - const hex = '1100' + pubKey + ts + sig + flags + lat + lon; - const p = decodePacket(hex); - assert.strictEqual(p.payload.flags.hasLocation, true); - assert.strictEqual(p.payload.lat, 1.0); - assert.strictEqual(p.payload.lon, 2.0); -}); - -test('ADVERT flags: hasName', () => { - const pubKey = 'AB'.repeat(32); - const ts = '01000000'; - const sig = 'CC'.repeat(64); - // flags=0x82 → type=2(repeater), hasName=true - const flags = '82'; - const name = Buffer.from('MyNode').toString('hex'); - const hex = '1100' + pubKey + ts + sig + flags + name; - const p = decodePacket(hex); - assert.strictEqual(p.payload.flags.hasName, true); - assert.strictEqual(p.payload.name, 'MyNode'); -}); - -test('ADVERT too short', () => { - const hex = '1100' + '00'.repeat(50); - const p = decodePacket(hex); - assert(p.payload.error); -}); - -console.log('\n=== GRP_TXT payload ==='); -test('GRP_TXT basic decode', () => { - // payloadType=5 → (5<<2)|1 = 0x15 - const hex = '1500' + 'FF' + 'AABB' + 'CCDDEE'; - const p = decodePacket(hex); - assert.strictEqual(p.payload.type, 'GRP_TXT'); - assert.strictEqual(p.payload.channelHash, 0xFF); - assert.strictEqual(p.payload.mac, 'aabb'); -}); - -test('GRP_TXT too short', () => { - const hex = '1500' + 'FF' + 'AA'; - const p = decodePacket(hex); - assert(p.payload.error); -}); - -test('GRP_TXT has channelHashHex field', () => { - const hex = '1500' + '1A' + 'AABB' + 'CCDDEE'; - const p = decodePacket(hex); - assert.strictEqual(p.payload.channelHashHex, '1A'); -}); - -test('GRP_TXT channelHashHex zero-pads single digit', () => { - const hex = '1500' + '03' + 'AABB' + 'CCDDEE'; - const p = decodePacket(hex); - assert.strictEqual(p.payload.channelHashHex, '03'); -}); - -test('GRP_TXT decryptionStatus is no_key when no keys provided', () => { - const hex = '1500' + 'FF' + 'AABB' + 'CCDDEE112233'; - const p = decodePacket(hex); - assert.strictEqual(p.payload.decryptionStatus, 'no_key'); -}); - -test('GRP_TXT decryptionStatus is no_key when keys empty', () => { - const hex = '1500' + 'FF' + 'AABB' + 'CCDDEE112233'; - const p = decodePacket(hex, {}); - assert.strictEqual(p.payload.decryptionStatus, 'no_key'); -}); - -test('GRP_TXT decryptionStatus is decryption_failed with bad keys', () => { - const hex = '1500' + 'FF' + 'AABB' + 'CCDDEE112233'; - const p = decodePacket(hex, { '#test': 'deadbeefdeadbeefdeadbeefdeadbeef' }); - assert.strictEqual(p.payload.decryptionStatus, 'decryption_failed'); -}); - -test('GRP_TXT decryptionStatus is no_key when encrypted data too short', () => { - // encryptedData < 10 hex chars (5 bytes) — not enough to attempt decryption - const hex = '1500' + 'FF' + 'AABB' + 'CCDD'; - const p = decodePacket(hex, { '#test': 'deadbeefdeadbeefdeadbeefdeadbeef' }); - assert.strictEqual(p.payload.decryptionStatus, 'no_key'); -}); - -test('GRP_TXT decryptionStatus is decrypted when key matches', () => { - // Mock the ChannelCrypto module to simulate successful decryption - const cryptoPath = require.resolve('@michaelhart/meshcore-decoder/dist/crypto/channel-crypto'); - const originalModule = require.cache[cryptoPath]; - require.cache[cryptoPath] = { - id: cryptoPath, - exports: { - ChannelCrypto: { - decryptGroupTextMessage: () => ({ - success: true, - data: { sender: 'TestUser', message: 'Hello world', timestamp: 1700000000, flags: 0 }, - }), - }, - }, - }; - try { - const hex = '1500' + 'FF' + 'AABB' + 'CCDDEE112233'; - const p = decodePacket(hex, { '#general': 'aabbccddaabbccddaabbccddaabbccdd' }); - assert.strictEqual(p.payload.decryptionStatus, 'decrypted'); - assert.strictEqual(p.payload.type, 'CHAN'); - assert.strictEqual(p.payload.channelHashHex, 'FF'); - assert.strictEqual(p.payload.channel, '#general'); - assert.strictEqual(p.payload.sender, 'TestUser'); - assert.strictEqual(p.payload.text, 'TestUser: Hello world'); - assert.strictEqual(p.payload.sender_timestamp, 1700000000); - assert.strictEqual(p.payload.flags, 0); - assert.strictEqual(p.payload.channelHash, 0xFF); - } finally { - if (originalModule) require.cache[cryptoPath] = originalModule; - else delete require.cache[cryptoPath]; - } -}); - -test('GRP_TXT decrypted without sender formats text correctly', () => { - const cryptoPath = require.resolve('@michaelhart/meshcore-decoder/dist/crypto/channel-crypto'); - const originalModule = require.cache[cryptoPath]; - require.cache[cryptoPath] = { - id: cryptoPath, - exports: { - ChannelCrypto: { - decryptGroupTextMessage: () => ({ - success: true, - data: { sender: null, message: 'Broadcast msg', timestamp: 1700000001, flags: 1 }, - }), - }, - }, - }; - try { - const hex = '1500' + '0A' + 'AABB' + 'CCDDEE112233'; - const p = decodePacket(hex, { '#alerts': 'deadbeefdeadbeefdeadbeefdeadbeef' }); - assert.strictEqual(p.payload.decryptionStatus, 'decrypted'); - assert.strictEqual(p.payload.sender, null); - assert.strictEqual(p.payload.text, 'Broadcast msg'); - assert.strictEqual(p.payload.channelHashHex, '0A'); - } finally { - if (originalModule) require.cache[cryptoPath] = originalModule; - else delete require.cache[cryptoPath]; - } -}); - -test('GRP_TXT decrypted tries multiple keys, first match wins', () => { - const cryptoPath = require.resolve('@michaelhart/meshcore-decoder/dist/crypto/channel-crypto'); - const originalModule = require.cache[cryptoPath]; - let callCount = 0; - require.cache[cryptoPath] = { - id: cryptoPath, - exports: { - ChannelCrypto: { - decryptGroupTextMessage: (ciphertext, mac, key) => { - callCount++; - if (key === 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb') { - return { success: true, data: { sender: 'Bob', message: 'Found it', timestamp: 0, flags: 0 } }; - } - return { success: false }; - }, - }, - }, - }; - try { - const hex = '1500' + 'FF' + 'AABB' + 'CCDDEE112233'; - const p = decodePacket(hex, { - '#wrong': 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', - '#right': 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', - }); - assert.strictEqual(p.payload.decryptionStatus, 'decrypted'); - assert.strictEqual(p.payload.channel, '#right'); - assert.strictEqual(p.payload.sender, 'Bob'); - assert.strictEqual(callCount, 2); - } finally { - if (originalModule) require.cache[cryptoPath] = originalModule; - else delete require.cache[cryptoPath]; - } -}); - -console.log('\n=== TXT_MSG payload ==='); -test('TXT_MSG decode', () => { - // payloadType=2 → (2<<2)|1 = 0x09 - const hex = '0900' + '00'.repeat(20); - const p = decodePacket(hex); - assert.strictEqual(p.payload.type, 'TXT_MSG'); - assert(p.payload.destHash); - assert(p.payload.srcHash); - assert(p.payload.mac); -}); - -console.log('\n=== ACK payload ==='); -test('ACK decode', () => { - // payloadType=3 → (3<<2)|1 = 0x0D - const hex = '0D00' + '00'.repeat(18); - const p = decodePacket(hex); - assert.strictEqual(p.payload.type, 'ACK'); - assert(p.payload.ackChecksum); -}); - -test('ACK too short', () => { - const hex = '0D00' + '00'.repeat(3); - const p = decodePacket(hex); - assert(p.payload.error); -}); - -console.log('\n=== REQ payload ==='); -test('REQ decode', () => { - // payloadType=0 → (0<<2)|1 = 0x01 - const hex = '0100' + '00'.repeat(20); - const p = decodePacket(hex); - assert.strictEqual(p.payload.type, 'REQ'); -}); - -console.log('\n=== RESPONSE payload ==='); -test('RESPONSE decode', () => { - // payloadType=1 → (1<<2)|1 = 0x05 - const hex = '0500' + '00'.repeat(20); - const p = decodePacket(hex); - assert.strictEqual(p.payload.type, 'RESPONSE'); -}); - -console.log('\n=== ANON_REQ payload ==='); -test('ANON_REQ decode', () => { - // payloadType=7 → (7<<2)|1 = 0x1D - const hex = '1D00' + '00'.repeat(50); - const p = decodePacket(hex); - assert.strictEqual(p.payload.type, 'ANON_REQ'); - assert(p.payload.destHash); - assert(p.payload.ephemeralPubKey); - assert(p.payload.mac); -}); - -test('ANON_REQ too short', () => { - const hex = '1D00' + '00'.repeat(20); - const p = decodePacket(hex); - assert(p.payload.error); -}); - -console.log('\n=== PATH payload ==='); -test('PATH decode', () => { - // payloadType=8 → (8<<2)|1 = 0x21 - const hex = '2100' + '00'.repeat(20); - const p = decodePacket(hex); - assert.strictEqual(p.payload.type, 'PATH'); - assert(p.payload.destHash); - assert(p.payload.srcHash); -}); - -test('PATH too short', () => { - const hex = '2100' + '00'.repeat(1); - const p = decodePacket(hex); - assert(p.payload.error); -}); - -console.log('\n=== TRACE payload ==='); -test('TRACE decode', () => { - // payloadType=9 → (9<<2)|1 = 0x25 - const hex = '2500' + '00'.repeat(12); - const p = decodePacket(hex); - assert.strictEqual(p.payload.type, 'TRACE'); - assert(p.payload.tag !== undefined); - assert(p.payload.authCode !== undefined); - assert.strictEqual(p.payload.flags, 0); -}); - -test('TRACE too short', () => { - const hex = '2500' + '00'.repeat(5); - const p = decodePacket(hex); - assert(p.payload.error); -}); - -console.log('\n=== UNKNOWN payload ==='); -test('Unknown payload type', () => { - // payloadType=6 → (6<<2)|1 = 0x19 - const hex = '1900' + 'DEADBEEF'; - const p = decodePacket(hex); - assert.strictEqual(p.payload.type, 'UNKNOWN'); - assert(p.payload.raw); -}); - -// === Edge cases === -console.log('\n=== Edge cases ==='); -test('Packet too short throws', () => { - assert.throws(() => decodePacket('FF'), /too short/); -}); - -test('Packet with spaces in hex', () => { - const hex = '11 00 ' + '00'.repeat(101); - const p = decodePacket(hex); - assert.strictEqual(p.header.payloadTypeName, 'ADVERT'); -}); - -test('Transport route too short throws', () => { - assert.throws(() => decodePacket('0000'), /too short for transport/); -}); - -test('Corrupt packet #183 — TRANSPORT_DIRECT with correct field order', () => { - const hex = 'BBAD6797EC8751D500BF95A1A776EF580E665BCBF6A0BBE03B5E730707C53489B8C728FD3FB902397197E1263CEC21E52465362243685DBBAD6797EC8751C90A75D9FD8213155D'; - const p = decodePacket(hex); - assert.strictEqual(p.header.routeType, 3, 'routeType should be TRANSPORT_DIRECT'); - assert.strictEqual(p.header.payloadTypeName, 'UNKNOWN'); - // transport codes are bytes 1-4, pathByte=0x87 at byte 5 - assert.strictEqual(p.transportCodes.code1, 'AD67'); - assert.strictEqual(p.transportCodes.code2, '97EC'); - // pathByte 0x87: hashSize=3, hashCount=7 - assert.strictEqual(p.path.hashSize, 3); - assert.strictEqual(p.path.hashCount, 7); - assert.strictEqual(p.path.hops.length, 7); - // No empty strings in hops - assert(p.path.hops.every(h => h.length > 0), 'no empty hops'); -}); - -test('path.truncated is false for normal packets', () => { - const hex = '1100' + '00'.repeat(101); - const p = decodePacket(hex); - assert.strictEqual(p.path.truncated, false); -}); - -test('path overflow with hashSize=2', () => { - // FLOOD + REQ, pathByte=0x45 → hashSize=2, hashCount=5, needs 10 bytes of path - // Only provide 7 bytes after pathByte → fits 3 full 2-byte hops - const hex = '0145' + 'AABBCCDDEEFF77'; - const p = decodePacket(hex); - assert.strictEqual(p.path.hashCount, 3); - assert.strictEqual(p.path.truncated, true); - assert.strictEqual(p.path.hops.length, 3); - assert.strictEqual(p.path.hops[0], 'AABB'); - assert.strictEqual(p.path.hops[1], 'CCDD'); - assert.strictEqual(p.path.hops[2], 'EEFF'); -}); - -// === Real packets from API === -console.log('\n=== Real packets ==='); -test('Real GRP_TXT packet', () => { - const p = decodePacket('150115D96CFF1FC90E7917B91729B76C1B509AE7789BBBD87D5AC3837E6C1487B47B0958AED8C7A6'); - assert.strictEqual(p.header.payloadTypeName, 'GRP_TXT'); - assert.strictEqual(p.header.routeTypeName, 'FLOOD'); - assert.strictEqual(p.path.hashCount, 1); -}); - -test('Real ADVERT packet FLOOD with 3 hops', () => { - const p = decodePacket('11036CEF52206D763E1EACFD52FBAD4EF926887D0694C42A618AAF480A67C41120D3785950EFE0C1'); - assert.strictEqual(p.header.payloadTypeName, 'ADVERT'); - assert.strictEqual(p.header.routeTypeName, 'FLOOD'); - assert.strictEqual(p.path.hashCount, 3); - assert.strictEqual(p.path.hashSize, 1); - // Payload is too short for full ADVERT but decoder handles it - assert.strictEqual(p.payload.type, 'ADVERT'); -}); - -test('Real DIRECT TXT_MSG packet', () => { - // 0x0A = DIRECT(2) + TXT_MSG(2) - const p = decodePacket('0A403220AD034C0394C2C449810E3D86399C53AEE7FE355BA67002FFC3627B1175A257A181AE'); - assert.strictEqual(p.header.payloadTypeName, 'TXT_MSG'); - assert.strictEqual(p.header.routeTypeName, 'DIRECT'); -}); - -// === validateAdvert === -console.log('\n=== validateAdvert ==='); -test('valid advert', () => { - const a = { pubKey: 'AB'.repeat(16), flags: { repeater: true, room: false, sensor: false } }; - assert.deepStrictEqual(validateAdvert(a), { valid: true }); -}); - -test('null advert', () => { - assert.strictEqual(validateAdvert(null).valid, false); -}); - -test('advert with error', () => { - assert.strictEqual(validateAdvert({ error: 'bad' }).valid, false); -}); - -test('pubkey too short', () => { - assert.strictEqual(validateAdvert({ pubKey: 'AABB' }).valid, false); -}); - -test('pubkey all zeros', () => { - assert.strictEqual(validateAdvert({ pubKey: '0'.repeat(64) }).valid, false); -}); - -test('invalid lat', () => { - assert.strictEqual(validateAdvert({ pubKey: 'AB'.repeat(16), lat: 200 }).valid, false); -}); - -test('invalid lon', () => { - assert.strictEqual(validateAdvert({ pubKey: 'AB'.repeat(16), lon: -200 }).valid, false); -}); - -test('name with control chars', () => { - assert.strictEqual(validateAdvert({ pubKey: 'AB'.repeat(16), name: 'test\x00bad' }).valid, false); -}); - -test('name too long', () => { - assert.strictEqual(validateAdvert({ pubKey: 'AB'.repeat(16), name: 'A'.repeat(65) }).valid, false); -}); - -test('valid name', () => { - assert.strictEqual(validateAdvert({ pubKey: 'AB'.repeat(16), name: 'My Node' }).valid, true); -}); - -test('valid lat/lon', () => { - const r = validateAdvert({ pubKey: 'AB'.repeat(16), lat: 37.3, lon: -121.9 }); - assert.strictEqual(r.valid, true); -}); - -test('NaN lat invalid', () => { - assert.strictEqual(validateAdvert({ pubKey: 'AB'.repeat(16), lat: NaN }).valid, false); -}); - -// --- GRP_TXT garbage detection (fixes #197) --- - -test('GRP_TXT decrypted garbage text marked as decryption_failed', () => { - const cryptoPath = require.resolve('@michaelhart/meshcore-decoder/dist/crypto/channel-crypto'); - const originalModule = require.cache[cryptoPath]; - require.cache[cryptoPath] = { - id: cryptoPath, - exports: { - ChannelCrypto: { - decryptGroupTextMessage: () => ({ - success: true, - data: { sender: 'Node', message: '\x01\x02\x03\x80\x81', timestamp: 1700000000, flags: 0 }, - }), - }, - }, - }; - try { - const hex = '1500' + 'FF' + 'AABB' + 'CCDDEE112233'; - const p = decodePacket(hex, { '#general': 'aabbccddaabbccddaabbccddaabbccdd' }); - assert.strictEqual(p.payload.decryptionStatus, 'decryption_failed'); - assert.strictEqual(p.payload.text, null); - assert.strictEqual(p.payload.channelHashHex, 'FF'); - assert.strictEqual(p.payload.channel, '#general'); - } finally { - if (originalModule) require.cache[cryptoPath] = originalModule; - else delete require.cache[cryptoPath]; - } -}); - -test('GRP_TXT valid text still marked as decrypted', () => { - const cryptoPath = require.resolve('@michaelhart/meshcore-decoder/dist/crypto/channel-crypto'); - const originalModule = require.cache[cryptoPath]; - require.cache[cryptoPath] = { - id: cryptoPath, - exports: { - ChannelCrypto: { - decryptGroupTextMessage: () => ({ - success: true, - data: { sender: 'Alice', message: 'Hello\nworld', timestamp: 1700000000, flags: 0 }, - }), - }, - }, - }; - try { - const hex = '1500' + 'FF' + 'AABB' + 'CCDDEE112233'; - const p = decodePacket(hex, { '#general': 'aabbccddaabbccddaabbccddaabbccdd' }); - assert.strictEqual(p.payload.decryptionStatus, 'decrypted'); - assert.strictEqual(p.payload.text, 'Alice: Hello\nworld'); - } finally { - if (originalModule) require.cache[cryptoPath] = originalModule; - else delete require.cache[cryptoPath]; - } -}); - -// === Summary === -console.log(`\n${passed} passed, ${failed} failed`); -if (failed > 0) process.exit(1); diff --git a/test-packet-store.js b/test-packet-store.js deleted file mode 100644 index 9a25fe03..00000000 --- a/test-packet-store.js +++ /dev/null @@ -1,552 +0,0 @@ -/* Unit tests for packet-store.js — uses a mock db module */ -'use strict'; -const assert = require('assert'); -const PacketStore = require('./packet-store'); - -let passed = 0, failed = 0; -function test(name, fn) { - try { fn(); passed++; console.log(` ✅ ${name}`); } - catch (e) { failed++; console.log(` ❌ ${name}: ${e.message}`); } -} - -// Mock db module — minimal stubs for PacketStore -function createMockDb() { - let txIdCounter = 1; - let obsIdCounter = 1000; - return { - db: { - pragma: (query) => { - if (query.includes('table_info(observations)')) return [{ name: 'observer_idx' }]; - return []; - }, - prepare: (sql) => ({ - get: (...args) => { - if (sql.includes('sqlite_master')) return { name: 'transmissions' }; - if (sql.includes('nodes')) return null; - if (sql.includes('observers')) return []; - return null; - }, - all: (...args) => [], - iterate: (...args) => [][Symbol.iterator](), - }), - }, - insertTransmission: (data) => ({ - transmissionId: txIdCounter++, - observationId: obsIdCounter++, - }), - }; -} - -function makePacketData(overrides = {}) { - return { - raw_hex: 'AABBCCDD', - hash: 'abc123', - timestamp: new Date().toISOString(), - route_type: 1, - payload_type: 5, - payload_version: 0, - decoded_json: JSON.stringify({ pubKey: 'DEADBEEF'.repeat(8) }), - observer_id: 'obs1', - observer_name: 'Observer1', - snr: 8.5, - rssi: -45, - path_json: '["AA","BB"]', - direction: 'rx', - ...overrides, - }; -} - -// === Constructor === -console.log('\n=== PacketStore constructor ==='); -test('creates empty store', () => { - const store = new PacketStore(createMockDb()); - assert.strictEqual(store.packets.length, 0); - assert.strictEqual(store.loaded, false); -}); - -test('respects maxMemoryMB config', () => { - const store = new PacketStore(createMockDb(), { maxMemoryMB: 512 }); - assert.strictEqual(store.maxBytes, 512 * 1024 * 1024); -}); - -// === Load === -console.log('\n=== Load ==='); -test('load sets loaded flag', () => { - const store = new PacketStore(createMockDb()); - store.load(); - assert.strictEqual(store.loaded, true); -}); - -test('sqliteOnly mode skips RAM', () => { - const orig = process.env.NO_MEMORY_STORE; - process.env.NO_MEMORY_STORE = '1'; - const store = new PacketStore(createMockDb()); - store.load(); - assert.strictEqual(store.sqliteOnly, true); - assert.strictEqual(store.packets.length, 0); - process.env.NO_MEMORY_STORE = orig || ''; - if (!orig) delete process.env.NO_MEMORY_STORE; -}); - -// === Insert === -console.log('\n=== Insert ==='); -test('insert adds packet to memory', () => { - const store = new PacketStore(createMockDb()); - store.load(); - store.insert(makePacketData()); - assert.strictEqual(store.packets.length, 1); - assert.strictEqual(store.stats.inserts, 1); -}); - -test('insert deduplicates by hash', () => { - const store = new PacketStore(createMockDb()); - store.load(); - store.insert(makePacketData({ hash: 'dup1' })); - store.insert(makePacketData({ hash: 'dup1', observer_id: 'obs2' })); - assert.strictEqual(store.packets.length, 1); - assert.strictEqual(store.packets[0].observations.length, 2); - assert.strictEqual(store.packets[0].observation_count, 2); -}); - -test('insert dedup: same observer+path skipped', () => { - const store = new PacketStore(createMockDb()); - store.load(); - store.insert(makePacketData({ hash: 'dup2' })); - store.insert(makePacketData({ hash: 'dup2' })); // same observer_id + path_json - assert.strictEqual(store.packets[0].observations.length, 1); -}); - -test('insert indexes by node pubkey', () => { - const store = new PacketStore(createMockDb()); - store.load(); - const pk = 'DEADBEEF'.repeat(8); - store.insert(makePacketData({ hash: 'n1', decoded_json: JSON.stringify({ pubKey: pk }) })); - assert(store.byNode.has(pk)); - assert.strictEqual(store.byNode.get(pk).length, 1); -}); - -test('insert indexes byObserver', () => { - const store = new PacketStore(createMockDb()); - store.load(); - store.insert(makePacketData({ observer_id: 'obs-test' })); - assert(store.byObserver.has('obs-test')); -}); - -test('insert updates first_seen for earlier timestamp', () => { - const store = new PacketStore(createMockDb()); - store.load(); - store.insert(makePacketData({ hash: 'ts1', timestamp: '2025-01-02T00:00:00Z', observer_id: 'o1' })); - store.insert(makePacketData({ hash: 'ts1', timestamp: '2025-01-01T00:00:00Z', observer_id: 'o2' })); - assert.strictEqual(store.packets[0].first_seen, '2025-01-01T00:00:00Z'); -}); - -test('insert indexes ADVERT observer', () => { - const store = new PacketStore(createMockDb()); - store.load(); - const pk = 'AA'.repeat(32); - store.insert(makePacketData({ hash: 'adv1', payload_type: 4, decoded_json: JSON.stringify({ pubKey: pk }), observer_id: 'obs-adv' })); - assert(store._advertByObserver.has(pk)); - assert(store._advertByObserver.get(pk).has('obs-adv')); -}); - -// === Query === -console.log('\n=== Query ==='); -test('query returns all packets', () => { - const store = new PacketStore(createMockDb()); - store.load(); - store.insert(makePacketData({ hash: 'q1' })); - store.insert(makePacketData({ hash: 'q2' })); - const r = store.query(); - assert.strictEqual(r.total, 2); - assert.strictEqual(r.packets.length, 2); -}); - -test('query by type filter', () => { - const store = new PacketStore(createMockDb()); - store.load(); - store.insert(makePacketData({ hash: 'qt1', payload_type: 4 })); - store.insert(makePacketData({ hash: 'qt2', payload_type: 5 })); - const r = store.query({ type: 4 }); - assert.strictEqual(r.total, 1); - assert.strictEqual(r.packets[0].payload_type, 4); -}); - -test('query by route filter', () => { - const store = new PacketStore(createMockDb()); - store.load(); - store.insert(makePacketData({ hash: 'qr1', route_type: 0 })); - store.insert(makePacketData({ hash: 'qr2', route_type: 1 })); - const r = store.query({ route: 1 }); - assert.strictEqual(r.total, 1); -}); - -test('query by hash (index path)', () => { - const store = new PacketStore(createMockDb()); - store.load(); - store.insert(makePacketData({ hash: 'qh1' })); - store.insert(makePacketData({ hash: 'qh2' })); - const r = store.query({ hash: 'qh1' }); - assert.strictEqual(r.total, 1); - assert.strictEqual(r.packets[0].hash, 'qh1'); -}); - -test('query by observer (index path)', () => { - const store = new PacketStore(createMockDb()); - store.load(); - store.insert(makePacketData({ hash: 'qo1', observer_id: 'obsA' })); - store.insert(makePacketData({ hash: 'qo2', observer_id: 'obsB' })); - const r = store.query({ observer: 'obsA' }); - assert.strictEqual(r.total, 1); -}); - -test('query with limit and offset', () => { - const store = new PacketStore(createMockDb()); - store.load(); - for (let i = 0; i < 10; i++) store.insert(makePacketData({ hash: `ql${i}`, observer_id: `o${i}` })); - const r = store.query({ limit: 3, offset: 2 }); - assert.strictEqual(r.packets.length, 3); - assert.strictEqual(r.total, 10); -}); - -test('query by since filter', () => { - const store = new PacketStore(createMockDb()); - store.load(); - store.insert(makePacketData({ hash: 'qs1', timestamp: '2025-01-01T00:00:00Z' })); - store.insert(makePacketData({ hash: 'qs2', timestamp: '2025-06-01T00:00:00Z', observer_id: 'o2' })); - const r = store.query({ since: '2025-03-01T00:00:00Z' }); - assert.strictEqual(r.total, 1); -}); - -test('query by until filter', () => { - const store = new PacketStore(createMockDb()); - store.load(); - store.insert(makePacketData({ hash: 'qu1', timestamp: '2025-01-01T00:00:00Z' })); - store.insert(makePacketData({ hash: 'qu2', timestamp: '2025-06-01T00:00:00Z', observer_id: 'o2' })); - const r = store.query({ until: '2025-03-01T00:00:00Z' }); - assert.strictEqual(r.total, 1); -}); - -test('query ASC order', () => { - const store = new PacketStore(createMockDb()); - store.load(); - store.insert(makePacketData({ hash: 'qa1', timestamp: '2025-06-01T00:00:00Z' })); - store.insert(makePacketData({ hash: 'qa2', timestamp: '2025-01-01T00:00:00Z', observer_id: 'o2' })); - const r = store.query({ order: 'ASC' }); - assert(r.packets[0].timestamp < r.packets[1].timestamp); -}); - -// === queryGrouped === -console.log('\n=== queryGrouped ==='); -test('queryGrouped returns grouped data', () => { - const store = new PacketStore(createMockDb()); - store.load(); - store.insert(makePacketData({ hash: 'qg1' })); - store.insert(makePacketData({ hash: 'qg1', observer_id: 'obs2' })); - store.insert(makePacketData({ hash: 'qg2', observer_id: 'obs3' })); - const r = store.queryGrouped(); - assert.strictEqual(r.total, 2); - const g1 = r.packets.find(p => p.hash === 'qg1'); - assert(g1); - assert.strictEqual(g1.observation_count, 2); - assert.strictEqual(g1.observer_count, 2); -}); - -// === getNodesByAdvertObservers === -console.log('\n=== getNodesByAdvertObservers ==='); -test('finds nodes by observer', () => { - const store = new PacketStore(createMockDb()); - store.load(); - const pk = 'BB'.repeat(32); - store.insert(makePacketData({ hash: 'nao1', payload_type: 4, decoded_json: JSON.stringify({ pubKey: pk }), observer_id: 'obs-x' })); - const result = store.getNodesByAdvertObservers(['obs-x']); - assert(result.has(pk)); -}); - -test('returns empty for unknown observer', () => { - const store = new PacketStore(createMockDb()); - store.load(); - const result = store.getNodesByAdvertObservers(['nonexistent']); - assert.strictEqual(result.size, 0); -}); - -// === Other methods === -console.log('\n=== Other methods ==='); -test('getById returns observation', () => { - const store = new PacketStore(createMockDb()); - store.load(); - const id = store.insert(makePacketData({ hash: 'gbi1' })); - const obs = store.getById(id); - assert(obs); -}); - -test('getSiblings returns observations for hash', () => { - const store = new PacketStore(createMockDb()); - store.load(); - store.insert(makePacketData({ hash: 'sib1' })); - store.insert(makePacketData({ hash: 'sib1', observer_id: 'obs2' })); - const sibs = store.getSiblings('sib1'); - assert.strictEqual(sibs.length, 2); -}); - -test('getSiblings empty for unknown hash', () => { - const store = new PacketStore(createMockDb()); - store.load(); - assert.deepStrictEqual(store.getSiblings('nope'), []); -}); - -test('all() returns packets', () => { - const store = new PacketStore(createMockDb()); - store.load(); - store.insert(makePacketData({ hash: 'all1' })); - assert.strictEqual(store.all().length, 1); -}); - -test('filter() works', () => { - const store = new PacketStore(createMockDb()); - store.load(); - store.insert(makePacketData({ hash: 'f1', payload_type: 4 })); - store.insert(makePacketData({ hash: 'f2', payload_type: 5, observer_id: 'o2' })); - assert.strictEqual(store.filter(p => p.payload_type === 4).length, 1); -}); - -test('countForNode returns counts', () => { - const store = new PacketStore(createMockDb()); - store.load(); - const pk = 'CC'.repeat(32); - store.insert(makePacketData({ hash: 'cn1', decoded_json: JSON.stringify({ pubKey: pk }) })); - store.insert(makePacketData({ hash: 'cn1', decoded_json: JSON.stringify({ pubKey: pk }), observer_id: 'o2' })); - const c = store.countForNode(pk); - assert.strictEqual(c.transmissions, 1); - assert.strictEqual(c.observations, 2); -}); - -test('getStats returns stats object', () => { - const store = new PacketStore(createMockDb()); - store.load(); - const s = store.getStats(); - assert.strictEqual(s.inMemory, 0); - assert(s.indexes); - assert.strictEqual(s.sqliteOnly, false); -}); - -test('getTimestamps returns timestamps', () => { - const store = new PacketStore(createMockDb()); - store.load(); - store.insert(makePacketData({ hash: 'gt1', timestamp: '2025-06-01T00:00:00Z' })); - store.insert(makePacketData({ hash: 'gt2', timestamp: '2025-06-02T00:00:00Z', observer_id: 'o2' })); - const ts = store.getTimestamps('2025-05-01T00:00:00Z'); - assert.strictEqual(ts.length, 2); -}); - -// === Eviction === -console.log('\n=== Eviction ==='); -test('evicts oldest when over maxPackets', () => { - const store = new PacketStore(createMockDb(), { maxMemoryMB: 1, estimatedPacketBytes: 500000 }); - // maxPackets will be very small - store.load(); - for (let i = 0; i < 10; i++) store.insert(makePacketData({ hash: `ev${i}`, observer_id: `o${i}` })); - assert(store.packets.length <= store.maxPackets); - assert(store.stats.evicted > 0); -}); - -// === findPacketsForNode === -console.log('\n=== findPacketsForNode ==='); -test('finds by pubkey', () => { - const store = new PacketStore(createMockDb()); - store.load(); - const pk = 'DD'.repeat(32); - store.insert(makePacketData({ hash: 'fpn1', decoded_json: JSON.stringify({ pubKey: pk }) })); - store.insert(makePacketData({ hash: 'fpn2', decoded_json: JSON.stringify({ pubKey: 'other' }), observer_id: 'o2' })); - const r = store.findPacketsForNode(pk); - assert.strictEqual(r.packets.length, 1); - assert.strictEqual(r.pubkey, pk); -}); - -test('finds by text search in decoded_json', () => { - const store = new PacketStore(createMockDb()); - store.load(); - store.insert(makePacketData({ hash: 'fpn3', decoded_json: JSON.stringify({ name: 'MySpecialNode' }) })); - const r = store.findPacketsForNode('MySpecialNode'); - assert.strictEqual(r.packets.length, 1); -}); - -// === Memory optimization: observation deduplication === -console.log('\n=== Observation deduplication (transmission_id refs) ==='); - -test('observations don\'t duplicate transmission fields', () => { - const store = new PacketStore(createMockDb()); - store.load(); - store.insert(makePacketData({ hash: 'dedup1', raw_hex: 'FF00FF00', decoded_json: '{"pubKey":"ABCD"}' })); - const tx = store.byHash.get('dedup1'); - assert(tx, 'transmission should exist'); - assert(tx.observations.length >= 1, 'should have at least 1 observation'); - const obs = tx.observations[0]; - // Observation should NOT have its own copies of transmission fields - assert(!obs.hasOwnProperty('raw_hex'), 'obs should not have own raw_hex'); - assert(!obs.hasOwnProperty('decoded_json'), 'obs should not have own decoded_json'); - // Observation should reference its parent transmission - assert(obs.hasOwnProperty('transmission_id'), 'obs should have transmission_id'); -}); - -test('transmission fields accessible through lookup', () => { - const store = new PacketStore(createMockDb()); - store.load(); - store.insert(makePacketData({ hash: 'lookup1', raw_hex: 'DEADBEEF', decoded_json: '{"pubKey":"CAFE"}' })); - const tx = store.byHash.get('lookup1'); - const obs = tx.observations[0]; - // Look up the transmission via the observation's transmission_id - const parentTx = store.byTxId.get(obs.transmission_id); - assert(parentTx, 'should find parent transmission via transmission_id'); - assert.strictEqual(parentTx.raw_hex, 'DEADBEEF'); - assert.strictEqual(parentTx.decoded_json, '{"pubKey":"CAFE"}'); - assert.strictEqual(parentTx.hash, 'lookup1'); -}); - -test('query results still contain transmission fields (backward compat)', () => { - const store = new PacketStore(createMockDb()); - store.load(); - store.insert(makePacketData({ hash: 'compat1', raw_hex: 'AABB', decoded_json: '{"test":true}' })); - const r = store.query(); - assert.strictEqual(r.total, 1); - const pkt = r.packets[0]; - // Query results (transmissions) should still have these fields - assert.strictEqual(pkt.raw_hex, 'AABB'); - assert.strictEqual(pkt.decoded_json, '{"test":true}'); - assert.strictEqual(pkt.hash, 'compat1'); -}); - -test('all() results contain transmission fields', () => { - const store = new PacketStore(createMockDb()); - store.load(); - store.insert(makePacketData({ hash: 'allcompat1', raw_hex: 'CCDD', decoded_json: '{"x":1}' })); - const pkts = store.all(); - assert.strictEqual(pkts.length, 1); - assert.strictEqual(pkts[0].raw_hex, 'CCDD'); - assert.strictEqual(pkts[0].decoded_json, '{"x":1}'); -}); - -test('multiple observations share one transmission', () => { - const store = new PacketStore(createMockDb()); - store.load(); - store.insert(makePacketData({ hash: 'shared1', observer_id: 'obs-A', raw_hex: 'FFFF' })); - store.insert(makePacketData({ hash: 'shared1', observer_id: 'obs-B', raw_hex: 'FFFF' })); - store.insert(makePacketData({ hash: 'shared1', observer_id: 'obs-C', raw_hex: 'FFFF' })); - // Only 1 transmission should exist - assert.strictEqual(store.packets.length, 1); - const tx = store.byHash.get('shared1'); - assert.strictEqual(tx.observations.length, 3); - // All observations should reference the same transmission_id - const txId = tx.observations[0].transmission_id; - assert(txId != null, 'transmission_id should be set'); - assert.strictEqual(tx.observations[1].transmission_id, txId); - assert.strictEqual(tx.observations[2].transmission_id, txId); - // Only 1 entry in byTxId for this transmission - assert(store.byTxId.has(txId), 'byTxId should have the shared transmission'); -}); - -test('getSiblings still returns observation data after dedup', () => { - const store = new PacketStore(createMockDb()); - store.load(); - store.insert(makePacketData({ hash: 'sibdedup1', observer_id: 'obs-X', snr: 5.0 })); - store.insert(makePacketData({ hash: 'sibdedup1', observer_id: 'obs-Y', snr: 9.0 })); - const sibs = store.getSiblings('sibdedup1'); - assert.strictEqual(sibs.length, 2); - // Each sibling should have observer-specific fields - const obsIds = sibs.map(s => s.observer_id).sort(); - assert.deepStrictEqual(obsIds, ['obs-X', 'obs-Y']); -}); - -test('queryGrouped still returns transmission fields after dedup', () => { - const store = new PacketStore(createMockDb()); - store.load(); - store.insert(makePacketData({ hash: 'grpdedup1', raw_hex: 'AABB', decoded_json: '{"g":1}', observer_id: 'o1' })); - store.insert(makePacketData({ hash: 'grpdedup1', observer_id: 'o2' })); - const r = store.queryGrouped(); - assert.strictEqual(r.total, 1); - const g = r.packets[0]; - assert.strictEqual(g.raw_hex, 'AABB'); - assert.strictEqual(g.decoded_json, '{"g":1}'); - assert.strictEqual(g.observation_count, 2); -}); - -test('memory estimate reflects deduplication savings', () => { - const store = new PacketStore(createMockDb()); - store.load(); - // Insert 50 unique transmissions, each with 5 observers - const longHex = 'AA'.repeat(200); - const longJson = JSON.stringify({ pubKey: 'BB'.repeat(32), name: 'TestNode', data: 'X'.repeat(200) }); - for (let i = 0; i < 50; i++) { - for (let j = 0; j < 5; j++) { - store.insert(makePacketData({ - hash: `mem${i}`, - observer_id: `obs-mem-${j}`, - raw_hex: longHex, - decoded_json: longJson, - })); - } - } - assert.strictEqual(store.packets.length, 50); - // Verify observations don't bloat memory with duplicate strings - let obsWithRawHex = 0; - for (const tx of store.packets) { - for (const obs of tx.observations) { - if (obs.hasOwnProperty('raw_hex')) obsWithRawHex++; - } - } - assert.strictEqual(obsWithRawHex, 0, 'no observation should have own raw_hex property'); -}); - -// === Regression: packetsLastHour must count live-appended observations (#182) === -console.log('\n=== packetsLastHour byObserver regression (#182) ==='); - -test('byObserver counts recent packets regardless of insertion order', () => { - const store = new PacketStore(createMockDb()); - store.load(); - - const twoHoursAgo = new Date(Date.now() - 7200000).toISOString(); - const thirtyMinAgo = new Date(Date.now() - 1800000).toISOString(); - const fiveMinAgo = new Date(Date.now() - 300000).toISOString(); - - // Simulate initial DB load: oldest packets pushed first (as if loaded DESC then reversed) - store.insert(makePacketData({ hash: 'old1', timestamp: twoHoursAgo, observer_id: 'obs-hr' })); - // Simulate live-ingested packet (appended at end, most recent) - store.insert(makePacketData({ hash: 'new1', timestamp: thirtyMinAgo, observer_id: 'obs-hr' })); - store.insert(makePacketData({ hash: 'new2', timestamp: fiveMinAgo, observer_id: 'obs-hr' })); - - const obsPackets = store.byObserver.get('obs-hr'); - assert.strictEqual(obsPackets.length, 3, 'should have 3 observations'); - - // Count packets in the last hour — the same way the fixed /api/observers does - const oneHourAgo = new Date(Date.now() - 3600000).toISOString(); - let count = 0; - for (const obs of obsPackets) { - if (obs.timestamp > oneHourAgo) count++; - } - assert.strictEqual(count, 2, 'should count 2 recent packets, not 0 (regression #182)'); -}); - -test('byObserver early-break bug: old item at front must not abort count', () => { - const store = new PacketStore(createMockDb()); - store.load(); - - const twoHoursAgo = new Date(Date.now() - 7200000).toISOString(); - const tenMinAgo = new Date(Date.now() - 600000).toISOString(); - - // Old observation first, then recent — simulates the mixed-order array - store.insert(makePacketData({ hash: 'h1', timestamp: twoHoursAgo, observer_id: 'obs-bug' })); - store.insert(makePacketData({ hash: 'h2', timestamp: tenMinAgo, observer_id: 'obs-bug' })); - - const obsPackets = store.byObserver.get('obs-bug'); - const oneHourAgo = new Date(Date.now() - 3600000).toISOString(); - - // BUGGY code (break on first old item) would return 0 here - let count = 0; - for (const obs of obsPackets) { - if (obs.timestamp > oneHourAgo) count++; - } - assert.strictEqual(count, 1, 'must not skip recent packet after old one'); -}); - -// === Summary === -console.log(`\n${passed} passed, ${failed} failed`); -if (failed > 0) process.exit(1); diff --git a/test-regional-filter.js b/test-regional-filter.js deleted file mode 100644 index ec76faef..00000000 --- a/test-regional-filter.js +++ /dev/null @@ -1,135 +0,0 @@ -#!/usr/bin/env node -// Test: Regional hop resolution filtering -// Validates that resolve-hops correctly filters candidates by geography and observer region - -const { IATA_COORDS, haversineKm, nodeNearRegion } = require('./iata-coords'); - -let pass = 0, fail = 0; - -function assert(condition, msg) { - if (condition) { pass++; console.log(` ✅ ${msg}`); } - else { fail++; console.error(` ❌ FAIL: ${msg}`); } -} - -// === 1. Haversine distance tests === -console.log('\n=== Haversine Distance ==='); - -const sjcToSea = haversineKm(37.3626, -121.9290, 47.4502, -122.3088); -assert(sjcToSea > 1100 && sjcToSea < 1150, `SJC→SEA = ${Math.round(sjcToSea)}km (expect ~1125km)`); - -const sjcToOak = haversineKm(37.3626, -121.9290, 37.7213, -122.2208); -assert(sjcToOak > 40 && sjcToOak < 55, `SJC→OAK = ${Math.round(sjcToOak)}km (expect ~48km)`); - -const sjcToSjc = haversineKm(37.3626, -121.9290, 37.3626, -121.9290); -assert(sjcToSjc === 0, `SJC→SJC = ${sjcToSjc}km (expect 0)`); - -const sjcToEug = haversineKm(37.3626, -121.9290, 44.1246, -123.2119); -assert(sjcToEug > 750 && sjcToEug < 780, `SJC→EUG = ${Math.round(sjcToEug)}km (expect ~762km)`); - -// === 2. nodeNearRegion tests === -console.log('\n=== Node Near Region ==='); - -// Node in San Jose, check against SJC region -const sjNode = nodeNearRegion(37.35, -121.95, 'SJC'); -assert(sjNode && sjNode.near, `San Jose node near SJC: ${sjNode.distKm}km`); - -// Node in Seattle, check against SJC region — should NOT be near -const seaNode = nodeNearRegion(47.45, -122.30, 'SJC'); -assert(seaNode && !seaNode.near, `Seattle node NOT near SJC: ${seaNode.distKm}km`); - -// Node in Seattle, check against SEA region — should be near -const seaNodeSea = nodeNearRegion(47.45, -122.30, 'SEA'); -assert(seaNodeSea && seaNodeSea.near, `Seattle node near SEA: ${seaNodeSea.distKm}km`); - -// Node in Eugene, check against EUG — should be near -const eugNode = nodeNearRegion(44.05, -123.10, 'EUG'); -assert(eugNode && eugNode.near, `Eugene node near EUG: ${eugNode.distKm}km`); - -// Eugene node should NOT be near SJC (~762km) -const eugNodeSjc = nodeNearRegion(44.05, -123.10, 'SJC'); -assert(eugNodeSjc && !eugNodeSjc.near, `Eugene node NOT near SJC: ${eugNodeSjc.distKm}km`); - -// Node with no location — returns null -const noLoc = nodeNearRegion(null, null, 'SJC'); -assert(noLoc === null, 'Null lat/lon returns null'); - -// Node at 0,0 — returns null -const zeroLoc = nodeNearRegion(0, 0, 'SJC'); -assert(zeroLoc === null, 'Zero lat/lon returns null'); - -// Unknown IATA — returns null -const unkIata = nodeNearRegion(37.35, -121.95, 'ZZZ'); -assert(unkIata === null, 'Unknown IATA returns null'); - -// === 3. Edge cases: nodes just inside/outside 300km radius === -console.log('\n=== Boundary Tests (300km radius) ==='); - -// Sacramento is ~145km from SJC — inside -const smfNode = nodeNearRegion(38.58, -121.49, 'SJC'); -assert(smfNode && smfNode.near, `Sacramento near SJC: ${smfNode.distKm}km (expect ~145)`); - -// Fresno is ~235km from SJC — inside -const fatNode = nodeNearRegion(36.74, -119.79, 'SJC'); -assert(fatNode && fatNode.near, `Fresno near SJC: ${fatNode.distKm}km (expect ~235)`); - -// Redding is ~400km from SJC — outside -const rddNode = nodeNearRegion(40.59, -122.39, 'SJC'); -assert(rddNode && !rddNode.near, `Redding NOT near SJC: ${rddNode.distKm}km (expect ~400)`); - -// === 4. Simulate the core issue: 1-byte hop with cross-regional collision === -console.log('\n=== Cross-Regional Collision Simulation ==='); - -// Two nodes with pubkeys starting with "D6": one in SJC area, one in SEA area -const candidates = [ - { name: 'Redwood Mt. Tam', pubkey: 'D6...sjc', lat: 37.92, lon: -122.60 }, // Marin County, CA - { name: 'VE7RSC North Repeater', pubkey: 'D6...sea', lat: 49.28, lon: -123.12 }, // Vancouver, BC - { name: 'KK7RXY Lynden', pubkey: 'D6...bel', lat: 48.94, lon: -122.47 }, // Bellingham, WA -]; - -// Packet observed in SJC region -const packetIata = 'SJC'; -const geoFiltered = candidates.filter(c => { - const check = nodeNearRegion(c.lat, c.lon, packetIata); - return check && check.near; -}); -assert(geoFiltered.length === 1, `Geo filter SJC: ${geoFiltered.length} candidates (expect 1)`); -assert(geoFiltered[0].name === 'Redwood Mt. Tam', `Winner: ${geoFiltered[0].name} (expect Redwood Mt. Tam)`); - -// Packet observed in SEA region -const seaFiltered = candidates.filter(c => { - const check = nodeNearRegion(c.lat, c.lon, 'SEA'); - return check && check.near; -}); -assert(seaFiltered.length === 2, `Geo filter SEA: ${seaFiltered.length} candidates (expect 2 — Vancouver + Bellingham)`); - -// Packet observed in EUG region — Eugene is ~300km from SEA nodes -const eugFiltered = candidates.filter(c => { - const check = nodeNearRegion(c.lat, c.lon, 'EUG'); - return check && check.near; -}); -assert(eugFiltered.length === 0, `Geo filter EUG: ${eugFiltered.length} candidates (expect 0 — all too far)`); - -// === 5. Layered fallback logic === -console.log('\n=== Layered Fallback ==='); - -const nodeWithGps = { lat: 37.92, lon: -122.60 }; // has GPS -const nodeNoGps = { lat: null, lon: null }; // no GPS -const observerSawNode = true; // observer-based filter says yes - -// Layer 1: GPS check -const gpsCheck = nodeNearRegion(nodeWithGps.lat, nodeWithGps.lon, 'SJC'); -assert(gpsCheck && gpsCheck.near, 'Layer 1 (GPS): node with GPS near SJC'); - -// Layer 2: No GPS, fall back to observer -const gpsCheckNoLoc = nodeNearRegion(nodeNoGps.lat, nodeNoGps.lon, 'SJC'); -assert(gpsCheckNoLoc === null, 'Layer 2: no GPS returns null → use observer-based fallback'); - -// Bridged WA node with GPS — should be REJECTED by SJC even though observer saw it -const bridgedWaNode = { lat: 47.45, lon: -122.30 }; // Seattle -const bridgedCheck = nodeNearRegion(bridgedWaNode.lat, bridgedWaNode.lon, 'SJC'); -assert(bridgedCheck && !bridgedCheck.near, `Bridge test: WA node rejected by SJC geo filter (${bridgedCheck.distKm}km)`); - -// === Summary === -console.log(`\n${'='.repeat(40)}`); -console.log(`Results: ${pass} passed, ${fail} failed`); -process.exit(fail > 0 ? 1 : 0); diff --git a/test-regional-integration.js b/test-regional-integration.js deleted file mode 100644 index 40d9932d..00000000 --- a/test-regional-integration.js +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/env node -// Integration test: Verify layered filtering works against live prod API -// Tests that resolve-hops returns regional metadata and correct filtering - -const https = require('https'); -const BASE = 'https://analyzer.00id.net'; - -function apiGet(path) { - return new Promise((resolve, reject) => { - https.get(BASE + path, { timeout: 10000 }, (res) => { - let data = ''; - res.on('data', d => data += d); - res.on('end', () => { try { resolve(JSON.parse(data)); } catch (e) { reject(e); } }); - }).on('error', reject); - }); -} - -let pass = 0, fail = 0; -function assert(condition, msg) { - if (condition) { pass++; console.log(` ✅ ${msg}`); } - else { fail++; console.error(` ❌ FAIL: ${msg}`); } -} - -async function run() { - console.log('\n=== Integration: resolve-hops API with regional filtering ===\n'); - - // 1. Get a packet with short hops and a known observer - const packets = await apiGet('/api/packets?limit=100&groupByHash=true'); - const pkt = packets.packets.find(p => { - const path = JSON.parse(p.path_json || '[]'); - return path.length > 0 && path.some(h => h.length <= 2) && p.observer_id; - }); - - if (!pkt) { - console.log(' ⚠ No packets with short hops found — skipping API tests'); - return; - } - - const path = JSON.parse(pkt.path_json); - const shortHops = path.filter(h => h.length <= 2); - console.log(` Using packet ${pkt.hash.slice(0,12)} observed by ${pkt.observer_name || pkt.observer_id.slice(0,12)}`); - console.log(` Path: ${path.join(' → ')} (${shortHops.length} short hops)`); - - // 2. Resolve WITH observer (should get regional filtering) - const withObs = await apiGet(`/api/resolve-hops?hops=${path.join(',')}&observer=${pkt.observer_id}`); - - assert(withObs.region != null, `Response includes region: ${withObs.region}`); - - // 3. Check that conflicts have filterMethod field - let hasFilterMethod = false; - let hasDistKm = false; - for (const [hop, info] of Object.entries(withObs.resolved)) { - if (info.conflicts && info.conflicts.length > 0) { - for (const c of info.conflicts) { - if (c.filterMethod) hasFilterMethod = true; - if (c.distKm != null) hasDistKm = true; - } - } - if (info.filterMethods) { - assert(Array.isArray(info.filterMethods), `Hop ${hop}: filterMethods is array: ${JSON.stringify(info.filterMethods)}`); - } - } - assert(hasFilterMethod, 'At least one conflict has filterMethod'); - - // 4. Resolve WITHOUT observer (no regional filtering) - const withoutObs = await apiGet(`/api/resolve-hops?hops=${path.join(',')}`); - assert(withoutObs.region === null, `Without observer: region is null`); - - // 5. Compare: with observer should have same or fewer candidates per ambiguous hop - for (const hop of shortHops) { - const withInfo = withObs.resolved[hop]; - const withoutInfo = withoutObs.resolved[hop]; - if (withInfo && withoutInfo && withInfo.conflicts && withoutInfo.conflicts) { - const withCount = withInfo.totalRegional || withInfo.conflicts.length; - const withoutCount = withoutInfo.totalGlobal || withoutInfo.conflicts.length; - assert(withCount <= withoutCount + 1, - `Hop ${hop}: regional(${withCount}) <= global(${withoutCount}) — ${withInfo.name || '?'}`); - } - } - - // 6. Check that geo-filtered candidates have distKm - for (const [hop, info] of Object.entries(withObs.resolved)) { - if (info.conflicts) { - const geoFiltered = info.conflicts.filter(c => c.filterMethod === 'geo'); - for (const c of geoFiltered) { - assert(c.distKm != null, `Hop ${hop} candidate ${c.name}: has distKm=${c.distKm}km (geo filter)`); - } - } - } - - console.log(`\n${'='.repeat(40)}`); - console.log(`Results: ${pass} passed, ${fail} failed`); - process.exit(fail > 0 ? 1 : 0); -} - -run().catch(e => { console.error('Test error:', e); process.exit(1); }); diff --git a/test-server-helpers.js b/test-server-helpers.js deleted file mode 100644 index 6086d583..00000000 --- a/test-server-helpers.js +++ /dev/null @@ -1,319 +0,0 @@ -'use strict'; - -const helpers = require('./server-helpers'); -const path = require('path'); -const fs = require('fs'); -const os = require('os'); - -let passed = 0, failed = 0; -function assert(cond, msg) { - if (cond) { passed++; console.log(` ✅ ${msg}`); } - else { failed++; console.error(` ❌ ${msg}`); } -} - -console.log('── server-helpers tests ──\n'); - -// --- loadConfigFile --- -console.log('loadConfigFile:'); -{ - // Returns {} when no files exist - const result = helpers.loadConfigFile(['/nonexistent/path.json']); - assert(typeof result === 'object' && Object.keys(result).length === 0, 'returns {} for missing files'); - - // Loads valid JSON - const tmp = path.join(os.tmpdir(), `test-config-${Date.now()}.json`); - fs.writeFileSync(tmp, JSON.stringify({ hello: 'world' })); - const result2 = helpers.loadConfigFile([tmp]); - assert(result2.hello === 'world', 'loads valid JSON file'); - fs.unlinkSync(tmp); - - // Falls back to second path - const tmp2 = path.join(os.tmpdir(), `test-config2-${Date.now()}.json`); - fs.writeFileSync(tmp2, JSON.stringify({ fallback: true })); - const result3 = helpers.loadConfigFile(['/nonexistent.json', tmp2]); - assert(result3.fallback === true, 'falls back to second path'); - fs.unlinkSync(tmp2); - - // Handles malformed JSON - const tmp3 = path.join(os.tmpdir(), `test-config3-${Date.now()}.json`); - fs.writeFileSync(tmp3, 'not json{{{'); - const result4 = helpers.loadConfigFile([tmp3]); - assert(Object.keys(result4).length === 0, 'returns {} for malformed JSON'); - fs.unlinkSync(tmp3); -} - -// --- loadThemeFile --- -console.log('\nloadThemeFile:'); -{ - const result = helpers.loadThemeFile(['/nonexistent/theme.json']); - assert(typeof result === 'object' && Object.keys(result).length === 0, 'returns {} for missing files'); - - const tmp = path.join(os.tmpdir(), `test-theme-${Date.now()}.json`); - fs.writeFileSync(tmp, JSON.stringify({ theme: { accent: '#ff0000' } })); - const result2 = helpers.loadThemeFile([tmp]); - assert(result2.theme.accent === '#ff0000', 'loads theme file'); - fs.unlinkSync(tmp); -} - -// --- buildHealthConfig --- -console.log('\nbuildHealthConfig:'); -{ - const h = helpers.buildHealthConfig({}); - assert(h.infraDegraded === 24, 'default infraDegraded'); - assert(h.infraSilent === 72, 'default infraSilent'); - assert(h.nodeDegraded === 1, 'default nodeDegraded'); - assert(h.nodeSilent === 24, 'default nodeSilent'); - - const h2 = helpers.buildHealthConfig({ healthThresholds: { infraDegradedHours: 2 } }); - assert(h2.infraDegraded === 2, 'custom infraDegraded'); - assert(h2.nodeDegraded === 1, 'other defaults preserved'); - - const h3 = helpers.buildHealthConfig(null); - assert(h3.infraDegraded === 24, 'handles null config'); -} - -// --- getHealthMs --- -console.log('\ngetHealthMs:'); -{ - const HEALTH = helpers.buildHealthConfig({}); - - const rep = helpers.getHealthMs('repeater', HEALTH); - assert(rep.degradedMs === 24 * 3600000, 'repeater uses infra degraded'); - assert(rep.silentMs === 72 * 3600000, 'repeater uses infra silent'); - - const room = helpers.getHealthMs('room', HEALTH); - assert(room.degradedMs === 24 * 3600000, 'room uses infra degraded'); - - const comp = helpers.getHealthMs('companion', HEALTH); - assert(comp.degradedMs === 1 * 3600000, 'companion uses node degraded'); - assert(comp.silentMs === 24 * 3600000, 'companion uses node silent'); - - const sensor = helpers.getHealthMs('sensor', HEALTH); - assert(sensor.degradedMs === 1 * 3600000, 'sensor uses node degraded'); - - const undef = helpers.getHealthMs(undefined, HEALTH); - assert(undef.degradedMs === 1 * 3600000, 'undefined role uses node degraded'); -} - -// --- isHashSizeFlipFlop --- -console.log('\nisHashSizeFlipFlop:'); -{ - assert(helpers.isHashSizeFlipFlop(null, null) === false, 'null seq returns false'); - assert(helpers.isHashSizeFlipFlop([1, 2], new Set([1, 2])) === false, 'too few samples'); - assert(helpers.isHashSizeFlipFlop([1, 1, 1], new Set([1])) === false, 'single size'); - assert(helpers.isHashSizeFlipFlop([1, 1, 1, 2, 2, 2], new Set([1, 2])) === false, 'clean upgrade (1 transition)'); - assert(helpers.isHashSizeFlipFlop([1, 2, 1], new Set([1, 2])) === true, 'flip-flop detected'); - assert(helpers.isHashSizeFlipFlop([1, 2, 1, 2], new Set([1, 2])) === true, 'repeated flip-flop'); - assert(helpers.isHashSizeFlipFlop([2, 1, 2], new Set([1, 2])) === true, 'reverse flip-flop'); - assert(helpers.isHashSizeFlipFlop([1, 2, 3], new Set([1, 2, 3])) === true, 'three sizes, 2 transitions'); -} - -// --- computeContentHash --- -console.log('\ncomputeContentHash:'); -{ - // Minimal packet: header + path byte + payload - // header=0x04, path_byte=0x00 (hash_size=1, 0 hops), payload=0xABCD - const hex1 = '0400abcd'; - const h1 = helpers.computeContentHash(hex1); - assert(typeof h1 === 'string' && h1.length === 16, 'returns 16-char hash'); - - // Same payload, different path should give same hash - // header=0x04, path_byte=0x41 (hash_size=2, 1 hop), path=0x1234, payload=0xABCD - const hex2 = '04411234abcd'; - const h2 = helpers.computeContentHash(hex2); - assert(h1 === h2, 'same content different path = same hash'); - - // Different payload = different hash - const hex3 = '0400ffff'; - const h3 = helpers.computeContentHash(hex3); - assert(h3 !== h1, 'different payload = different hash'); - - // Very short hex - const h4 = helpers.computeContentHash('04'); - assert(h4 === '04', 'short hex returns prefix'); - - // Invalid hex - const h5 = helpers.computeContentHash('xyz'); - assert(typeof h5 === 'string', 'handles invalid hex gracefully'); -} - -// --- geoDist --- -console.log('\ngeoDist:'); -{ - assert(helpers.geoDist(0, 0, 0, 0) === 0, 'same point = 0'); - assert(helpers.geoDist(0, 0, 3, 4) === 5, 'pythagorean triple'); - assert(helpers.geoDist(37.7749, -122.4194, 37.7749, -122.4194) === 0, 'SF to SF = 0'); - const d = helpers.geoDist(37.0, -122.0, 38.0, -122.0); - assert(Math.abs(d - 1.0) < 0.001, '1 degree latitude diff'); -} - -// --- deriveHashtagChannelKey --- -console.log('\nderiveHashtagChannelKey:'); -{ - const k1 = helpers.deriveHashtagChannelKey('test'); - assert(typeof k1 === 'string' && k1.length === 32, 'returns 32-char key'); - const k2 = helpers.deriveHashtagChannelKey('test'); - assert(k1 === k2, 'deterministic'); - const k3 = helpers.deriveHashtagChannelKey('other'); - assert(k3 !== k1, 'different input = different key'); -} - -// --- buildBreakdown --- -console.log('\nbuildBreakdown:'); -{ - const r1 = helpers.buildBreakdown(null, null, null, null); - assert(JSON.stringify(r1) === '{}', 'null rawHex returns empty'); - - const r2 = helpers.buildBreakdown('04', null, null, null); - assert(r2.ranges.length === 1, 'single-byte returns header only'); - assert(r2.ranges[0].label === 'Header', 'header range'); - - // 2 bytes: header + path byte, no payload - const r3 = helpers.buildBreakdown('0400', null, null, null); - assert(r3.ranges.length === 2, 'two bytes: header + path length'); - assert(r3.ranges[1].label === 'Path Length', 'path length range'); - - // With payload: header=04, path_byte=00, payload=abcd - const r4 = helpers.buildBreakdown('0400abcd', null, null, null); - assert(r4.ranges.some(r => r.label === 'Payload'), 'has payload range'); - - // With path hops: header=04, path_byte=0x41 (size=2, count=1), path=1234, payload=ff - const r5 = helpers.buildBreakdown('04411234ff', null, null, null); - assert(r5.ranges.some(r => r.label === 'Path'), 'has path range'); - - // ADVERT with enough payload - // flags=0x90 (0x10=GPS + 0x80=Name) - const advertHex = '0400' + 'aa'.repeat(32) + 'bb'.repeat(4) + 'cc'.repeat(64) + '90' + 'dddddddddddddddd' + '48656c6c6f'; - const r6 = helpers.buildBreakdown(advertHex, { type: 'ADVERT' }, null, null); - assert(r6.ranges.some(r => r.label === 'PubKey'), 'ADVERT has PubKey sub-range'); - assert(r6.ranges.some(r => r.label === 'Flags'), 'ADVERT has Flags sub-range'); - assert(r6.ranges.some(r => r.label === 'Latitude'), 'ADVERT with GPS flag has Latitude'); - assert(r6.ranges.some(r => r.label === 'Name'), 'ADVERT with name flag has Name'); -} - -// --- disambiguateHops --- -console.log('\ndisambiguateHops:'); -{ - const nodes = [ - { public_key: 'aabb11223344', name: 'Node-A', lat: 37.0, lon: -122.0 }, - { public_key: 'ccdd55667788', name: 'Node-C', lat: 37.1, lon: -122.1 }, - ]; - // Single unique match - const r1 = helpers.disambiguateHops(['aabb'], nodes); - assert(r1.length === 1, 'resolves single hop'); - assert(r1[0].name === 'Node-A', 'resolves to correct node'); - assert(r1[0].pubkey === 'aabb11223344', 'includes pubkey'); - - // Unknown hop - delete nodes._prefixIdx; delete nodes._prefixIdxName; - const r2 = helpers.disambiguateHops(['ffff'], nodes); - assert(r2[0].name === 'ffff', 'unknown hop uses hex as name'); - - // Multiple hops - delete nodes._prefixIdx; delete nodes._prefixIdxName; - const r3 = helpers.disambiguateHops(['aabb', 'ccdd'], nodes); - assert(r3.length === 2, 'resolves multiple hops'); - assert(r3[0].name === 'Node-A' && r3[1].name === 'Node-C', 'both resolved'); -} - -// --- updateHashSizeForPacket --- -console.log('\nupdateHashSizeForPacket:'); -{ - const map = new Map(), allMap = new Map(), seqMap = new Map(); - - // ADVERT packet (payload_type=4) - // path byte 0x40 = hash_size 2 (bits 7-6 = 01) - const p1 = { - payload_type: 4, - raw_hex: '0440' + 'aa'.repeat(100), - decoded_json: JSON.stringify({ pubKey: 'abc123' }), - path_json: null - }; - helpers.updateHashSizeForPacket(p1, map, allMap, seqMap); - assert(map.get('abc123') === 2, 'ADVERT sets hash_size=2'); - assert(allMap.get('abc123').has(2), 'all map has size 2'); - assert(seqMap.get('abc123')[0] === 2, 'seq map records size'); - - // Non-ADVERT with path_json fallback - const map2 = new Map(), allMap2 = new Map(), seqMap2 = new Map(); - const p2 = { - payload_type: 1, - raw_hex: '0140ff', // path byte 0x40 = hash_size 2 - decoded_json: JSON.stringify({ pubKey: 'def456' }), - path_json: JSON.stringify(['aabb']) - }; - helpers.updateHashSizeForPacket(p2, map2, allMap2, seqMap2); - assert(map2.get('def456') === 2, 'non-ADVERT falls back to path byte'); - - // Already-parsed decoded_json (object, not string) - const map3 = new Map(), allMap3 = new Map(), seqMap3 = new Map(); - const p3 = { - payload_type: 4, - raw_hex: '04c0' + 'aa'.repeat(100), // 0xC0 = bits 7-6 = 11 = hash_size 4 - decoded_json: { pubKey: 'ghi789' }, - path_json: null - }; - helpers.updateHashSizeForPacket(p3, map3, allMap3, seqMap3); - assert(map3.get('ghi789') === 4, 'handles object decoded_json'); -} - -// --- rebuildHashSizeMap --- -console.log('\nrebuildHashSizeMap:'); -{ - const map = new Map(), allMap = new Map(), seqMap = new Map(); - const packets = [ - // Newest first (as packet store provides) - { payload_type: 4, raw_hex: '0480' + 'bb'.repeat(50), decoded_json: JSON.stringify({ pubKey: 'node1' }), path_json: null }, - { payload_type: 4, raw_hex: '0440' + 'aa'.repeat(50), decoded_json: JSON.stringify({ pubKey: 'node1' }), path_json: null }, - ]; - helpers.rebuildHashSizeMap(packets, map, allMap, seqMap); - assert(map.get('node1') === 3, 'first seen (newest) wins for map'); - assert(allMap.get('node1').size === 2, 'all map has both sizes'); - // Seq should be reversed to chronological: [2, 3] - const seq = seqMap.get('node1'); - assert(seq[0] === 2 && seq[1] === 3, 'sequence is chronological (reversed)'); - - // Pass 2 fallback: node without advert - const map2 = new Map(), allMap2 = new Map(), seqMap2 = new Map(); - const packets2 = [ - { payload_type: 1, raw_hex: '0140ff', decoded_json: JSON.stringify({ pubKey: 'node2' }), path_json: JSON.stringify(['aabb']) }, - ]; - helpers.rebuildHashSizeMap(packets2, map2, allMap2, seqMap2); - assert(map2.get('node2') === 2, 'pass 2 fallback from path'); -} - -// --- requireApiKey --- -console.log('\nrequireApiKey:'); -{ - // No API key configured - const mw1 = helpers.requireApiKey(null); - let nextCalled = false; - mw1({headers: {}, query: {}}, {}, () => { nextCalled = true; }); - assert(nextCalled, 'no key configured = passes through'); - - // Valid key - const mw2 = helpers.requireApiKey('secret123'); - nextCalled = false; - mw2({headers: {'x-api-key': 'secret123'}, query: {}}, {}, () => { nextCalled = true; }); - assert(nextCalled, 'valid header key passes'); - - // Valid key via query - nextCalled = false; - mw2({headers: {}, query: {apiKey: 'secret123'}}, {}, () => { nextCalled = true; }); - assert(nextCalled, 'valid query key passes'); - - // Invalid key - let statusCode = null, jsonBody = null; - const mockRes = { - status(code) { statusCode = code; return { json(body) { jsonBody = body; } }; } - }; - nextCalled = false; - mw2({headers: {'x-api-key': 'wrong'}, query: {}}, mockRes, () => { nextCalled = true; }); - assert(!nextCalled && statusCode === 401, 'invalid key returns 401'); -} - -console.log(`\n═══════════════════════════════════════`); -console.log(` PASSED: ${passed}`); -console.log(` FAILED: ${failed}`); -console.log(`═══════════════════════════════════════`); -if (failed > 0) process.exit(1); diff --git a/test-server-routes.js b/test-server-routes.js deleted file mode 100644 index d1f448d5..00000000 --- a/test-server-routes.js +++ /dev/null @@ -1,1279 +0,0 @@ -#!/usr/bin/env node -'use strict'; - -// Server route integration tests via supertest -process.env.NODE_ENV = 'test'; -process.env.SEED_DB = 'true'; // Seed test data - -const request = require('supertest'); -const { app, server, wss, pktStore, db, cache, lastPathSeenMap, hopPrefixToKey, ambiguousHopPrefixes, resolveUniquePrefixMatch } = require('./server'); - -let passed = 0, failed = 0; - -async function t(name, fn) { - try { - await fn(); - passed++; - } catch (e) { - failed++; - console.error(`FAIL: ${name} — ${e.message}`); - } -} - -function assert(cond, msg) { if (!cond) throw new Error(msg || 'assertion failed'); } - -// Seed additional test data for branch coverage -function seedTestData() { - const now = new Date().toISOString(); - const yesterday = new Date(Date.now() - 86400000).toISOString(); - - // Add nodes with various roles and locations - const nodes = [ - { public_key: 'aabb' + '0'.repeat(60), name: 'TestRepeater1', role: 'repeater', lat: 37.7749, lon: -122.4194, last_seen: now, first_seen: yesterday }, - { public_key: 'ccdd' + '0'.repeat(60), name: 'TestRoom1', role: 'room', lat: 40.7128, lon: -74.0060, last_seen: now, first_seen: yesterday }, - { public_key: 'eeff' + '0'.repeat(60), name: 'TestCompanion1', role: 'companion', lat: 0, lon: 0, last_seen: yesterday, first_seen: yesterday }, - { public_key: '1122' + '0'.repeat(60), name: 'TestSensor1', role: 'sensor', lat: 51.5074, lon: -0.1278, last_seen: now, first_seen: yesterday }, - // Node with same 2-char prefix as TestRepeater1 to test ambiguous resolution - { public_key: 'aabb' + '1'.repeat(60), name: 'TestRepeater2', role: 'repeater', lat: 34.0522, lon: -118.2437, last_seen: now, first_seen: yesterday }, - ]; - for (const n of nodes) { - try { db.upsertNode(n); } catch {} - } - - // Add observer - try { db.upsertObserver({ id: 'test-obs-1', name: 'TestObs', iata: 'SFO', last_seen: now, first_seen: yesterday }); } catch {} - try { db.upsertObserver({ id: 'test-obs-2', name: 'TestObs2', iata: 'NYC', last_seen: now, first_seen: yesterday }); } catch {} - - // Add packets with paths and decoded data - const packets = [ - { - raw_hex: '11451000D818206D3AAC152C8A91F89957E6D30CA51F36E28790228971C473B755F244F718754CF5EE4A2FD58D944466E42CDED140C66D0CC590183E32BAF40F112BE8F3F2BDF6012B4B2793C52F1D36F69EE054D9A05593286F78453E56C0EC4A3EB95DDA2A7543FCCC00B939CACC009278603902FC12BCF84B706120526F6F6620536F6C6172', - timestamp: now, observer_id: 'test-obs-1', snr: 10.5, rssi: -85, - hash: 'test-hash-001', route_type: 1, payload_type: 4, payload_version: 1, - path_json: JSON.stringify(['aabb', 'ccdd']), - decoded_json: JSON.stringify({ type: 'ADVERT', name: 'TestRepeater1', pubKey: 'aabb' + '0'.repeat(60), role: 'repeater', lat: 37.7749, lon: -122.4194, flags: { repeater: true } }), - }, - { - raw_hex: '2233445566778899AABBCCDD', - timestamp: yesterday, observer_id: 'test-obs-1', snr: -5, rssi: -110, - hash: 'test-hash-002', route_type: 0, payload_type: 5, payload_version: 1, - path_json: JSON.stringify(['aabb', 'ccdd', 'eeff']), - decoded_json: JSON.stringify({ type: 'TXT_MSG', text: 'Hello test', channelHash: 'ch01', channel_hash: 'ch01', srcName: 'TestCompanion1' }), - }, - { - raw_hex: 'AABBCCDD00112233', - timestamp: now, observer_id: 'test-obs-2', snr: 8, rssi: -70, - hash: 'test-hash-003', route_type: 3, payload_type: 4, payload_version: 1, - path_json: JSON.stringify(['1122', 'aabb']), - decoded_json: JSON.stringify({ type: 'ADVERT', name: 'TestSensor1', pubKey: '1122' + '0'.repeat(60), role: 'sensor', lat: 51.5074, lon: -0.1278, flags: { sensor: true } }), - }, - { - raw_hex: 'FF00FF00FF00FF00', - timestamp: now, observer_id: 'test-obs-1', snr: 15, rssi: -60, - hash: 'test-hash-001', route_type: 1, payload_type: 4, payload_version: 1, - path_json: JSON.stringify(['aabb', 'ccdd']), - decoded_json: JSON.stringify({ type: 'ADVERT', name: 'TestRepeater1', pubKey: 'aabb' + '0'.repeat(60) }), - }, - { - raw_hex: '5566778899AABB00', - timestamp: now, observer_id: 'test-obs-2', snr: 3, rssi: -90, - hash: 'test-hash-004', route_type: 0, payload_type: 5, payload_version: 1, - path_json: JSON.stringify(['eeff', 'aabb', 'ccdd', '1122']), - decoded_json: JSON.stringify({ type: 'TXT_MSG', text: 'Another msg', channelHash: 'ch02', srcName: 'TestRoom1' }), - }, - ]; - - for (const pkt of packets) { - try { pktStore.insert(pkt); } catch {} - try { db.insertTransmission(pkt); } catch {} - } - - // Seed another packet with CHAN type for channel messages - const chanPkt = { - raw_hex: 'AA00BB00CC00DD00', - timestamp: now, observer_id: 'test-obs-1', observer_name: 'TestObs', snr: 5, rssi: -80, - hash: 'test-hash-005', route_type: 0, payload_type: 5, payload_version: 1, - path_json: JSON.stringify(['aabb', 'ccdd']), - decoded_json: JSON.stringify({ type: 'CHAN', channel: 'ch01', text: 'UserA: Hello world', sender: 'UserA', sender_timestamp: now, SNR: 5 }), - }; - try { pktStore.insert(chanPkt); } catch {} - try { db.insertTransmission(chanPkt); } catch {} - - // Another CHAN message for dedup code path coverage - const chanPkt3 = { - raw_hex: 'FF00EE00DD00CC00', - timestamp: now, observer_id: 'test-obs-1', observer_name: 'TestObs', snr: 7, rssi: -75, - hash: 'test-hash-006', route_type: 1, payload_type: 5, payload_version: 1, - path_json: JSON.stringify([]), - decoded_json: JSON.stringify({ type: 'CHAN', channel: 'ch01', text: 'UserB: Test msg', sender: 'UserB' }), - }; - try { pktStore.insert(chanPkt3); } catch {} - try { db.insertTransmission(chanPkt3); } catch {} - - // Duplicate of same message from different observer (for dedup/repeats coverage) - const chanPkt2 = { - raw_hex: 'AA00BB00CC00DD00', - timestamp: now, observer_id: 'test-obs-2', observer_name: 'TestObs2', snr: 3, rssi: -90, - hash: 'test-hash-005', route_type: 0, payload_type: 5, payload_version: 1, - path_json: JSON.stringify(['aabb', 'ccdd']), - decoded_json: JSON.stringify({ type: 'CHAN', channel: 'ch01', text: 'UserA: Hello world', sender: 'UserA' }), - }; - try { pktStore.insert(chanPkt2); } catch {} - try { db.insertTransmission(chanPkt2); } catch {} - - // Seed a CHAN packet with garbage-decrypted name (pre-#197 data) — should be filtered out - const garbageChanPkt = { - raw_hex: 'GARBAGE0CHANNEL1', - timestamp: now, observer_id: 'test-obs-1', observer_name: 'TestObs', snr: 2, rssi: -95, - hash: 'test-hash-garbage-chan', route_type: 0, payload_type: 5, payload_version: 1, - path_json: JSON.stringify([]), - decoded_json: JSON.stringify({ type: 'CHAN', channel: 'garb\x01\x02\x03age', text: 'SomeUser: hello\x04\x05\x06', sender: 'SomeUser' }), - }; - try { pktStore.insert(garbageChanPkt); } catch {} - try { db.insertTransmission(garbageChanPkt); } catch {} - - // Seed a CHAN packet with clean name but garbage text — should also be filtered out - const garbageTextPkt = { - raw_hex: 'GARBAGE0TEXT0001', - timestamp: now, observer_id: 'test-obs-1', observer_name: 'TestObs', snr: 2, rssi: -95, - hash: 'test-hash-garbage-text', route_type: 0, payload_type: 5, payload_version: 1, - path_json: JSON.stringify([]), - decoded_json: JSON.stringify({ type: 'CHAN', channel: 'cleanChan', text: '\x00\x01\x02\x03garbage binary', sender: 'User' }), - }; - try { pktStore.insert(garbageTextPkt); } catch {} - try { db.insertTransmission(garbageTextPkt); } catch {} - - // Packet with sender_key/recipient_key for peer interaction coverage in db.getNodeAnalytics - const peerPkt = { - raw_hex: 'DEADBEEF00112233', - timestamp: yesterday, observer_id: 'test-obs-1', observer_name: 'TestObs', snr: 8, rssi: -82, - hash: 'test-hash-007', route_type: 0, payload_type: 2, payload_version: 1, - path_json: JSON.stringify(['aabb']), - decoded_json: JSON.stringify({ type: 'TXT_MSG', sender_key: 'aabb' + '0'.repeat(60), sender_name: 'TestRepeater1', recipient_key: 'ccdd' + '0'.repeat(60), recipient_name: 'TestRoom1', text: 'hello' }), - }; - try { pktStore.insert(peerPkt); } catch {} - try { db.insertTransmission(peerPkt); } catch {} - - // Clear cache so fresh data is picked up - cache.clear(); -} - -seedTestData(); - -(async () => { - console.log('── Server Route Tests ──'); - - // --- Config routes --- - await t('GET /api/config/cache', async () => { - const r = await request(app).get('/api/config/cache').expect(200); - assert(r.body && typeof r.body === 'object', 'should return object'); - }); - - await t('GET /api/config/client', async () => { - const r = await request(app).get('/api/config/client').expect(200); - assert(typeof r.body === 'object', 'should return config'); - }); - - await t('GET /api/config/regions', async () => { - const r = await request(app).get('/api/config/regions').expect(200); - assert(typeof r.body === 'object', 'should return regions'); - }); - - await t('GET /api/config/theme', async () => { - const r = await request(app).get('/api/config/theme').expect(200); - assert(typeof r.body === 'object', 'should return theme object'); - }); - - await t('GET /api/config/map', async () => { - const r = await request(app).get('/api/config/map').expect(200); - assert(typeof r.body === 'object', 'should return map config'); - }); - - // --- Health --- - await t('GET /api/health', async () => { - const r = await request(app).get('/api/health').expect(200); - assert(r.body.status, 'should have status'); - assert(r.body.engine === 'node', 'health should include engine=node'); - assert(typeof r.body.version === 'string', 'health should include version'); - assert(typeof r.body.commit === 'string', 'health should include commit'); - }); - - // --- Stats --- - await t('GET /api/stats', async () => { - const r = await request(app).get('/api/stats').expect(200); - assert(typeof r.body === 'object', 'should return stats'); - assert(r.body.engine === 'node', 'stats should include engine=node'); - assert(typeof r.body.version === 'string', 'stats should include version'); - assert(typeof r.body.commit === 'string', 'stats should include commit'); - }); - - // --- Perf --- - await t('GET /api/perf', async () => { - const r = await request(app).get('/api/perf').expect(200); - assert(typeof r.body === 'object', 'should return perf data'); - }); - - await t('POST /api/perf/reset', async () => { - const r = await request(app).post('/api/perf/reset'); - assert(r.status === 200 || r.status === 403, 'should return 200 or 403'); - }); - - // --- Nodes --- - await t('GET /api/nodes default', async () => { - const r = await request(app).get('/api/nodes').expect(200); - assert(Array.isArray(r.body) || r.body.nodes, 'should return nodes'); - }); - - await t('GET /api/nodes with limit', async () => { - await request(app).get('/api/nodes?limit=5').expect(200); - }); - - await t('GET /api/nodes with offset', async () => { - await request(app).get('/api/nodes?limit=2&offset=1').expect(200); - }); - - await t('GET /api/nodes with role=repeater', async () => { - await request(app).get('/api/nodes?role=repeater').expect(200); - }); - - await t('GET /api/nodes with role=room', async () => { - await request(app).get('/api/nodes?role=room').expect(200); - }); - - await t('GET /api/nodes with region=SFO', async () => { - await request(app).get('/api/nodes?region=SFO').expect(200); - }); - - await t('GET /api/nodes with search', async () => { - await request(app).get('/api/nodes?search=Test').expect(200); - }); - - await t('GET /api/nodes with lastHeard', async () => { - await request(app).get('/api/nodes?lastHeard=86400').expect(200); - }); - - await t('GET /api/nodes with sortBy=name', async () => { - await request(app).get('/api/nodes?sortBy=name').expect(200); - }); - - await t('GET /api/nodes with sortBy=role', async () => { - await request(app).get('/api/nodes?sortBy=role').expect(200); - }); - - await t('GET /api/nodes with before cursor', async () => { - await request(app).get('/api/nodes?before=2099-01-01T00:00:00Z').expect(200); - }); - - await t('GET /api/nodes with large limit', async () => { - await request(app).get('/api/nodes?limit=10000&lastHeard=259200').expect(200); - }); - - await t('GET /api/nodes/search with q', async () => { - const r = await request(app).get('/api/nodes/search?q=Test').expect(200); - assert(Array.isArray(r.body) || typeof r.body === 'object', 'should return results'); - }); - - await t('GET /api/nodes/search without q', async () => { - await request(app).get('/api/nodes/search').expect(200); - }); - - await t('GET /api/nodes/bulk-health', async () => { - const r = await request(app).get('/api/nodes/bulk-health').expect(200); - assert(typeof r.body === 'object', 'should return bulk health'); - }); - - await t('GET /api/nodes/network-status', async () => { - const r = await request(app).get('/api/nodes/network-status').expect(200); - assert(typeof r.body === 'object', 'should return network status'); - }); - - cache.clear(); // Clear to avoid cache hits for regional queries - await t('GET /api/nodes/network-status with region', async () => { - await request(app).get('/api/nodes/network-status?region=SFO').expect(200); - }); - - cache.clear(); - await t('GET /api/nodes/bulk-health with region', async () => { - await request(app).get('/api/nodes/bulk-health?region=SFO').expect(200); - }); - - // Test with real node pubkey - const testPubkey = 'aabb' + '0'.repeat(60); - - await t('GET /api/nodes/:pubkey — existing', async () => { - const r = await request(app).get(`/api/nodes/${testPubkey}`); - assert(r.status === 200 || r.status === 404, 'should find or not find'); - }); - - await t('GET /api/nodes/:pubkey — nonexistent', async () => { - await request(app).get('/api/nodes/' + '0'.repeat(64)).expect(404); - }); - - await t('GET /api/nodes/:pubkey/health — existing', async () => { - const r = await request(app).get(`/api/nodes/${testPubkey}/health`); - assert(r.status === 200 || r.status === 404, 'should handle'); - }); - - await t('GET /api/nodes/:pubkey/health — nonexistent', async () => { - const r = await request(app).get('/api/nodes/nonexistent/health'); - assert(r.status === 404 || r.status === 200, 'should handle missing node'); - }); - - await t('GET /api/nodes/:pubkey/paths — existing', async () => { - const r = await request(app).get(`/api/nodes/${testPubkey}/paths`); - assert(r.status === 200 || r.status === 404, 'should handle'); - }); - - await t('GET /api/nodes/:pubkey/paths — nonexistent', async () => { - const r = await request(app).get('/api/nodes/nonexistent/paths'); - assert(r.status === 404 || r.status === 200, 'should handle missing'); - }); - - await t('GET /api/nodes/:pubkey/paths with days param', async () => { - await request(app).get(`/api/nodes/${testPubkey}/paths?days=7`); - }); - - await t('GET /api/nodes/:pubkey/analytics — existing', async () => { - const r = await request(app).get(`/api/nodes/${testPubkey}/analytics`); - assert(r.status === 200 || r.status === 404, 'should handle'); - }); - - await t('GET /api/nodes/:pubkey/analytics with days', async () => { - await request(app).get(`/api/nodes/${testPubkey}/analytics?days=7`); - }); - - await t('GET /api/nodes/:pubkey/analytics — nonexistent', async () => { - const r = await request(app).get('/api/nodes/nonexistent/analytics'); - assert(r.status === 404 || r.status === 200, 'should handle missing'); - }); - - // --- Packets --- - await t('GET /api/packets default', async () => { - const r = await request(app).get('/api/packets').expect(200); - assert(typeof r.body === 'object', 'should return packets'); - }); - - await t('GET /api/packets with limit', async () => { - await request(app).get('/api/packets?limit=5').expect(200); - }); - - await t('GET /api/packets with offset', async () => { - await request(app).get('/api/packets?limit=5&offset=0').expect(200); - }); - - await t('GET /api/packets with type', async () => { - await request(app).get('/api/packets?type=ADVERT').expect(200); - }); - - await t('GET /api/packets with route', async () => { - await request(app).get('/api/packets?route=1').expect(200); - }); - - await t('GET /api/packets with observer', async () => { - await request(app).get('/api/packets?observer=test-obs-1').expect(200); - }); - - await t('GET /api/packets with region', async () => { - await request(app).get('/api/packets?region=SFO').expect(200); - }); - - await t('GET /api/packets with hash', async () => { - await request(app).get('/api/packets?hash=test-hash-001').expect(200); - }); - - await t('GET /api/packets with since/until', async () => { - await request(app).get('/api/packets?since=2020-01-01T00:00:00Z&until=2099-01-01T00:00:00Z').expect(200); - }); - - await t('GET /api/packets with groupByHash', async () => { - await request(app).get('/api/packets?groupByHash=true').expect(200); - }); - - await t('GET /api/packets with node filter', async () => { - await request(app).get(`/api/packets?node=${testPubkey}`).expect(200); - }); - - await t('GET /api/packets with nodes (multi)', async () => { - await request(app).get(`/api/packets?nodes=${testPubkey},ccdd${'0'.repeat(60)}`).expect(200); - }); - - await t('GET /api/packets with order asc', async () => { - await request(app).get('/api/packets?order=asc').expect(200); - }); - - await t('GET /api/packets/timestamps without since', async () => { - await request(app).get('/api/packets/timestamps').expect(400); - }); - - await t('GET /api/packets/timestamps with since', async () => { - const r = await request(app).get('/api/packets/timestamps?since=2020-01-01T00:00:00Z').expect(200); - assert(typeof r.body === 'object', 'should return timestamps'); - }); - - await t('GET /api/packets/:id — id 1', async () => { - const r = await request(app).get('/api/packets/1'); - assert(r.status === 200 || r.status === 404, 'should handle'); - }); - - await t('GET /api/packets/:id — nonexistent', async () => { - const r = await request(app).get('/api/packets/999999'); - assert(r.status === 404 || r.status === 200, 'should handle missing packet'); - }); - - // --- POST /api/decode --- - await t('POST /api/decode without hex', async () => { - await request(app).post('/api/decode').send({}).expect(400); - }); - - await t('POST /api/decode with invalid hex', async () => { - await request(app).post('/api/decode').send({ hex: 'zzzz' }).expect(400); - }); - - await t('POST /api/decode with valid hex', async () => { - const r = await request(app).post('/api/decode') - .send({ hex: '11451000D818206D3AAC152C8A91F89957E6D30CA51F36E28790228971C473B755F244F718754CF5EE4A2FD58D944466E42CDED140C66D0CC590183E32BAF40F112BE8F3F2BDF6012B4B2793C52F1D36F69EE054D9A05593286F78453E56C0EC4A3EB95DDA2A7543FCCC00B939CACC009278603902FC12BCF84B706120526F6F6620536F6C6172' }); - assert(r.status === 200 || r.status === 400, 'should not crash'); - }); - - // --- POST /api/packets --- - await t('POST /api/packets without hex', async () => { - await request(app).post('/api/packets').send({}).expect(400); - }); - - await t('POST /api/packets with hex (no api key configured)', async () => { - const r = await request(app).post('/api/packets') - .send({ hex: '11451000D818206D3AAC152C8A91F89957E6D30CA51F36E28790228971C473B755F244F718754CF5EE4A2FD58D944466E42CDED140C66D0CC590183E32BAF40F112BE8F3F2BDF6012B4B2793C52F1D36F69EE054D9A05593286F78453E56C0EC4A3EB95DDA2A7543FCCC00B939CACC009278603902FC12BCF84B706120526F6F6620536F6C6172', observer: 'test-obs-1', region: 'SFO' }); - assert(r.status === 200 || r.status === 400 || r.status === 403, 'should handle'); - }); - - await t('POST /api/packets with invalid hex', async () => { - const r = await request(app).post('/api/packets').send({ hex: 'zzzz' }); - assert(r.status === 400, 'should reject invalid hex'); - }); - - // --- Channels (clear cache first to ensure fresh data) --- - cache.clear(); - await t('GET /api/channels', async () => { - const r = await request(app).get('/api/channels').expect(200); - assert(typeof r.body === 'object', 'should return channels'); - }); - - await t('GET /api/channels filters garbage channel names', async () => { - cache.clear(); - const r = await request(app).get('/api/channels').expect(200); - const names = r.body.channels.map(c => c.name); - assert(!names.some(n => n.includes('\x01') || n.includes('\x02') || n.includes('\x03')), - 'garbage channel names should be filtered out'); - assert(!names.includes('cleanChan'), - 'channels with garbage text should be filtered out'); - assert(names.includes('ch01'), 'valid channel ch01 should still be present'); - }); - - await t('GET /api/channels with region', async () => { - await request(app).get('/api/channels?region=SFO').expect(200); - }); - - await t('GET /api/channels/:hash/messages', async () => { - const r = await request(app).get('/api/channels/ch01/messages').expect(200); - assert(typeof r.body === 'object', 'should return messages'); - }); - - await t('GET /api/channels/:hash/messages with params', async () => { - await request(app).get('/api/channels/ch01/messages?limit=5&offset=0').expect(200); - }); - - await t('GET /api/channels/:hash/messages with region', async () => { - await request(app).get('/api/channels/ch01/messages?region=SFO').expect(200); - }); - - await t('GET /api/channels/:hash/messages nonexistent', async () => { - await request(app).get('/api/channels/nonexistent/messages').expect(200); - }); - - // --- Observers --- - await t('GET /api/observers', async () => { - const r = await request(app).get('/api/observers').expect(200); - assert(typeof r.body === 'object', 'should return observers'); - }); - - await t('GET /api/observers — packetsLastHour regression #182', async () => { - // Clear cache so the handler re-computes counts - cache.clear(); - const r = await request(app).get('/api/observers').expect(200); - const observers = r.body.observers || []; - const obs1 = observers.find(o => o.id === 'test-obs-1'); - if (obs1) { - assert(typeof obs1.packetsLastHour === 'number', 'packetsLastHour must be a number'); - // test-obs-1 has recent seeded packets — must not be zero - assert(obs1.packetsLastHour > 0, `packetsLastHour should be > 0, got ${obs1.packetsLastHour} (regression #182)`); - } - }); - - await t('GET /api/observers/:id — existing', async () => { - const r = await request(app).get('/api/observers/test-obs-1'); - assert(r.status === 200 || r.status === 404, 'should handle'); - }); - - await t('GET /api/observers/:id — nonexistent', async () => { - const r = await request(app).get('/api/observers/nonexistent'); - assert(r.status === 404 || r.status === 200, 'should handle missing observer'); - }); - - await t('GET /api/observers/:id/analytics — existing', async () => { - const r = await request(app).get('/api/observers/test-obs-1/analytics'); - assert(r.status === 200 || r.status === 404, 'should handle'); - }); - - await t('GET /api/observers/:id/analytics — nonexistent', async () => { - const r = await request(app).get('/api/observers/nonexistent/analytics'); - assert(r.status === 404 || r.status === 200, 'should handle'); - }); - - // --- Traces --- - await t('GET /api/traces/:hash — existing', async () => { - const r = await request(app).get('/api/traces/test-hash-001'); - assert(r.status === 200 || r.status === 404, 'should handle'); - }); - - await t('GET /api/traces/:hash — nonexistent', async () => { - const r = await request(app).get('/api/traces/nonexistent'); - assert(r.status === 200 || r.status === 404, 'should handle trace lookup'); - }); - - // --- Analytics (clear cache before regional tests) --- - cache.clear(); - await t('GET /api/analytics/rf', async () => { - const r = await request(app).get('/api/analytics/rf').expect(200); - assert(typeof r.body === 'object', 'should return RF analytics'); - }); - - await t('GET /api/analytics/rf with region', async () => { - await request(app).get('/api/analytics/rf?region=SFO').expect(200); - }); - - await t('GET /api/analytics/rf with region NYC', async () => { - await request(app).get('/api/analytics/rf?region=NYC').expect(200); - }); - - await t('GET /api/analytics/topology', async () => { - const r = await request(app).get('/api/analytics/topology').expect(200); - assert(typeof r.body === 'object', 'should return topology'); - }); - - await t('GET /api/analytics/topology uniqueNodes matches stats totalNodes (#155)', async () => { - const topo = await request(app).get('/api/analytics/topology').expect(200); - const stats = await request(app).get('/api/stats').expect(200); - assert(topo.body.uniqueNodes === stats.body.totalNodes, - `uniqueNodes (${topo.body.uniqueNodes}) should match stats totalNodes (${stats.body.totalNodes})`); - }); - - await t('GET /api/analytics/topology with region', async () => { - await request(app).get('/api/analytics/topology?region=SFO').expect(200); - }); - - await t('GET /api/analytics/channels', async () => { - const r = await request(app).get('/api/analytics/channels').expect(200); - assert(typeof r.body === 'object', 'should return channel analytics'); - }); - - await t('GET /api/analytics/channels with region', async () => { - await request(app).get('/api/analytics/channels?region=SFO').expect(200); - }); - - await t('GET /api/analytics/hash-sizes', async () => { - const r = await request(app).get('/api/analytics/hash-sizes').expect(200); - assert(typeof r.body === 'object', 'should return hash sizes'); - }); - - await t('GET /api/analytics/hash-sizes with region', async () => { - await request(app).get('/api/analytics/hash-sizes?region=SFO').expect(200); - }); - - await t('GET /api/analytics/subpaths', async () => { - const r = await request(app).get('/api/analytics/subpaths').expect(200); - assert(typeof r.body === 'object', 'should return subpaths'); - }); - - await t('GET /api/analytics/subpaths with params', async () => { - await request(app).get('/api/analytics/subpaths?minLen=2&maxLen=3&limit=10').expect(200); - }); - - await t('GET /api/analytics/subpaths with region', async () => { - await request(app).get('/api/analytics/subpaths?region=SFO').expect(200); - }); - - await t('GET /api/analytics/subpath-detail with hops', async () => { - const r = await request(app).get('/api/analytics/subpath-detail?hops=aabb,ccdd'); - assert(r.status === 200 || r.status === 400, 'should handle'); - }); - - await t('GET /api/analytics/subpath-detail without hops', async () => { - const r = await request(app).get('/api/analytics/subpath-detail'); - assert(r.status === 200 || r.status === 400, 'should handle missing hops'); - }); - - await t('GET /api/analytics/distance', async () => { - const r = await request(app).get('/api/analytics/distance').expect(200); - assert(typeof r.body === 'object', 'should return distance analytics'); - }); - - await t('GET /api/analytics/distance with region', async () => { - await request(app).get('/api/analytics/distance?region=SFO').expect(200); - }); - - // --- Resolve hops --- - await t('GET /api/resolve-hops with hops', async () => { - const r = await request(app).get('/api/resolve-hops?hops=aabb,ccdd').expect(200); - assert(typeof r.body === 'object', 'should return resolved hops'); - }); - - await t('GET /api/resolve-hops without hops', async () => { - await request(app).get('/api/resolve-hops').expect(200); - }); - - await t('GET /api/resolve-hops with region and observer', async () => { - await request(app).get('/api/resolve-hops?hops=aabb,ccdd®ion=SFO&observer=test-obs-1').expect(200); - }); - - await t('GET /api/resolve-hops with prefixes (legacy)', async () => { - await request(app).get('/api/resolve-hops?prefixes=aabb,ccdd').expect(200); - }); - - await t('GET /api/resolve-hops ambiguous prefix', async () => { - // 'aabb' matches both TestRepeater1 and TestRepeater2 - const r = await request(app).get('/api/resolve-hops?hops=aabb,ccdd,1122®ion=SFO&observer=test-obs-1').expect(200); - assert(typeof r.body === 'object', 'should resolve hops'); - }); - - await t('GET /api/resolve-hops with packet context', async () => { - await request(app).get('/api/resolve-hops?hops=aabb,eeff,ccdd®ion=SFO&observer=test-obs-1&packetHash=test-hash-001').expect(200); - }); - - // --- IATA coords --- - await t('GET /api/iata-coords', async () => { - const r = await request(app).get('/api/iata-coords').expect(200); - assert(r.body && 'coords' in r.body, 'should have coords key'); - }); - - // --- Audio lab --- - await t('GET /api/audio-lab/buckets', async () => { - const r = await request(app).get('/api/audio-lab/buckets').expect(200); - assert(r.body && 'buckets' in r.body, 'should have buckets key'); - }); - - // --- SPA fallback --- - await t('GET /nodes SPA fallback', async () => { - const r = await request(app).get('/nodes'); - assert([200, 304, 404].includes(r.status), 'should not crash'); - }); - - // --- Cache behavior: hit same endpoint twice --- - await t('Cache hit: /api/nodes/bulk-health twice', async () => { - await request(app).get('/api/nodes/bulk-health').expect(200); - await request(app).get('/api/nodes/bulk-health').expect(200); - }); - - await t('Cache hit: /api/analytics/rf twice', async () => { - await request(app).get('/api/analytics/rf').expect(200); - await request(app).get('/api/analytics/rf').expect(200); - }); - - await t('Cache hit: /api/analytics/topology twice', async () => { - await request(app).get('/api/analytics/topology').expect(200); - await request(app).get('/api/analytics/topology').expect(200); - }); - - // ── WebSocket tests ── - await t('WebSocket connection', async () => { - const WebSocket = require('ws'); - await new Promise((resolve) => { - if (server.address()) return resolve(); - server.listen(0, resolve); - }); - const port = server.address().port; - const ws = new WebSocket(`ws://127.0.0.1:${port}`); - await new Promise((resolve, reject) => { - ws.on('open', resolve); - ws.on('error', reject); - setTimeout(() => reject(new Error('WS timeout')), 3000); - }); - assert(ws.readyState === WebSocket.OPEN, 'WS should be open'); - ws.close(); - await new Promise(r => setTimeout(r, 100)); - }); - - // ── Additional query parameter branches ── - await t('GET /api/nodes?sortBy=lastSeen', async () => { - await request(app).get('/api/nodes?sortBy=lastSeen').expect(200); - }); - await t('GET /api/nodes multi-filter', async () => { - await request(app).get('/api/nodes?role=repeater®ion=SFO&lastHeard=86400&search=Test').expect(200); - }); - await t('GET /api/packets?type=4', async () => { - await request(app).get('/api/packets?type=4').expect(200); - }); - await t('GET /api/packets multi-filter', async () => { - await request(app).get('/api/packets?type=5&route=1&observer=test-obs-1').expect(200); - }); - await t('GET /api/packets?node=TestRepeater1', async () => { - await request(app).get('/api/packets?node=TestRepeater1').expect(200); - }); - await t('GET /api/packets?groupByHash=true&type=4&observer=test-obs-1', async () => { - await request(app).get('/api/packets?groupByHash=true&type=4&observer=test-obs-1').expect(200); - }); - await t('GET /api/packets?groupByHash=true®ion=SFO', async () => { - await request(app).get('/api/packets?groupByHash=true®ion=SFO').expect(200); - }); - await t('GET /api/nodes?sortBy=name&search=nonexistent', async () => { - const r = await request(app).get('/api/nodes?sortBy=name&search=nonexistent').expect(200); - assert(r.body.nodes.length === 0, 'no nodes'); - }); - await t('GET /api/nodes?before=2000-01-01', async () => { - const r = await request(app).get('/api/nodes?before=2000-01-01T00:00:00Z').expect(200); - assert(r.body.nodes.length === 0, 'no nodes'); - }); - - // ── Node health/analytics/paths for nodes WITH packets ── - const testRepeaterKey = 'aabb' + '0'.repeat(60); - - await t('GET /api/nodes/:pubkey/health with packets', async () => { - cache.clear(); - const r = await request(app).get(`/api/nodes/${testRepeaterKey}/health`).expect(200); - assert(r.body.node && r.body.stats, 'should have node+stats'); - }); - await t('GET /api/nodes/:pubkey/analytics?days=30', async () => { - cache.clear(); - const r = await request(app).get(`/api/nodes/${testRepeaterKey}/analytics?days=30`).expect(200); - assert(r.body.computedStats, 'should have computedStats'); - }); - await t('GET /api/nodes/:pubkey/analytics?days=1', async () => { - cache.clear(); - await request(app).get(`/api/nodes/${testRepeaterKey}/analytics?days=1`).expect(200); - }); - await t('GET /api/nodes/:pubkey/paths with packets', async () => { - cache.clear(); - const r = await request(app).get(`/api/nodes/${testRepeaterKey}/paths`).expect(200); - assert(r.body.paths !== undefined, 'should have paths'); - }); - await t('GET /api/nodes/:pubkey existing', async () => { - await request(app).get(`/api/nodes/${testRepeaterKey}`).expect(200); - }); - await t('GET /api/nodes/:pubkey 404', async () => { - await request(app).get('/api/nodes/' + '0'.repeat(63) + '1').expect(404); - }); - - // ── Observer analytics ── - await t('GET /api/observers/test-obs-1/analytics', async () => { - cache.clear(); - const r = await request(app).get('/api/observers/test-obs-1/analytics').expect(200); - assert(r.body.timeline !== undefined, 'should have timeline'); - }); - await t('GET /api/observers/test-obs-1/analytics?days=1', async () => { - cache.clear(); - await request(app).get('/api/observers/test-obs-1/analytics?days=1').expect(200); - }); - - // ── Traces ── - await t('GET /api/traces/:hash existing', async () => { - const r = await request(app).get('/api/traces/test-hash-001').expect(200); - assert(r.body.traces, 'should have traces'); - }); - - // ── Resolve hops ── - await t('GET /api/resolve-hops?hops=aabb', async () => { - await request(app).get('/api/resolve-hops?hops=aabb').expect(200); - }); - await t('GET /api/resolve-hops?hops=ffff', async () => { - await request(app).get('/api/resolve-hops?hops=ffff').expect(200); - }); - - // ── Analytics with nonexistent region ── - for (const ep of ['rf', 'topology', 'channels', 'distance', 'hash-sizes']) { - await t(`GET /api/analytics/${ep}?region=ZZZZ`, async () => { - cache.clear(); - await request(app).get(`/api/analytics/${ep}?region=ZZZZ`).expect(200); - }); - } - - // ── Subpath endpoints ── - await t('GET /api/analytics/subpaths?minLen=2&maxLen=2', async () => { - cache.clear(); - await request(app).get('/api/analytics/subpaths?minLen=2&maxLen=2&limit=5').expect(200); - }); - await t('GET /api/analytics/subpath-detail?hops=aabb,ccdd,eeff', async () => { - cache.clear(); - await request(app).get('/api/analytics/subpath-detail?hops=aabb,ccdd,eeff').expect(200); - }); - - // ── POST /api/packets with observer+region ── - await t('POST /api/packets with observer+region', async () => { - const r = await request(app).post('/api/packets') - .send({ hex: '11451000D818206D3AAC152C8A91F89957E6D30CA51F36E28790228971C473B755F244F718754CF5EE4A2FD58D944466E42CDED140C66D0CC590183E32BAF40F112BE8F3F2BDF6012B4B2793C52F1D36F69EE054D9A05593286F78453E56C0EC4A3EB95DDA2A7543FCCC00B939CACC009278603902FC12BCF84B706120526F6F6620536F6C6172', observer: 'test-api-obs', region: 'LAX', snr: 12, rssi: -75 }); - assert([200, 400, 403].includes(r.status), 'should handle'); - }); - - // ── POST /api/decode with whitespace ── - await t('POST /api/decode whitespace hex', async () => { - const r = await request(app).post('/api/decode').send({ hex: ' 1145 1000 D818 206D ' }); - assert([200, 400].includes(r.status), 'should handle'); - }); - - // ── Direct db function calls ── - await t('db.searchNodes', () => { - assert(Array.isArray(db.searchNodes('Test', 10)), 'should return array'); - assert(db.searchNodes('zzzznonexistent', 5).length === 0, 'empty for no match'); - }); - await t('db.getNodeHealth existing', () => { - const h = db.getNodeHealth(testRepeaterKey); - assert(h && h.node && h.stats, 'should have node+stats'); - }); - await t('db.getNodeHealth nonexistent', () => { - assert(db.getNodeHealth('nonexistent') === null, 'null for missing'); - }); - await t('db.getNodeAnalytics existing', () => { - const a = db.getNodeAnalytics(testRepeaterKey, 7); - assert(a && a.computedStats, 'should have computedStats'); - }); - await t('db.getNodeAnalytics nonexistent', () => { - assert(db.getNodeAnalytics('nonexistent', 7) === null, 'null for missing'); - }); - await t('db.getNodeAnalytics for named node', () => { - const a = db.getNodeAnalytics('ccdd' + '0'.repeat(60), 30); - assert(a !== null, 'should return analytics'); - }); - await t('db.getNodeHealth for named node', () => { - assert(db.getNodeHealth('ccdd' + '0'.repeat(60)) !== null, 'should have health'); - }); - await t('db.updateObserverStatus', () => { - db.updateObserverStatus({ id: 'test-status-obs', name: 'StatusObs', iata: 'LAX', model: 'test', firmware: '1.0', client_version: '2.0', radio: '915,125,7,5', battery_mv: 3700, uptime_secs: 86400, noise_floor: -120 }); - }); - - // ── Packet store direct methods ── - await t('pktStore.getById missing', () => { assert(pktStore.getById(999999) === null); }); - await t('pktStore.getSiblings missing', () => { assert(pktStore.getSiblings('nonexistent').length === 0); }); - await t('pktStore.getTimestamps', () => { assert(Array.isArray(pktStore.getTimestamps('2000-01-01T00:00:00Z'))); }); - await t('pktStore.all', () => { assert(Array.isArray(pktStore.all())); }); - await t('pktStore.filter', () => { assert(Array.isArray(pktStore.filter(p => p.payload_type === 4))); }); - await t('pktStore.getStats', () => { const s = pktStore.getStats(); assert(s.inMemory !== undefined && s.indexes); }); - await t('pktStore.queryGrouped', () => { assert(pktStore.queryGrouped({ limit: 5, type: 4 }).packets !== undefined); }); - await t('pktStore.queryGrouped region+since', () => { assert(pktStore.queryGrouped({ limit: 5, region: 'SFO', since: '2000-01-01' }).packets !== undefined); }); - await t('pktStore.countForNode existing', () => { assert(pktStore.countForNode(testRepeaterKey).transmissions !== undefined); }); - await t('pktStore.countForNode missing', () => { assert(pktStore.countForNode('nonexistent').transmissions === 0); }); - await t('pktStore.findPacketsForNode', () => { assert(pktStore.findPacketsForNode(testRepeaterKey).packets !== undefined); }); - await t('pktStore._transmissionsForObserver with fromTx', () => { - assert(Array.isArray(pktStore._transmissionsForObserver('test-obs-1', pktStore.all()))); - }); - - // ── Cache SWR (stale-while-revalidate) ── - await t('Cache stale-while-revalidate', async () => { - // 50ms TTL, grace period = 100ms - cache.set('swr-test', { data: 42 }, 50); - await new Promise(r => setTimeout(r, 60)); // Wait past expiry but within grace - const stale = cache.get('swr-test'); - assert(stale && stale.data === 42, 'should return stale value within grace'); - }); - await t('Cache fully expired (past grace)', async () => { - cache.set('swr-expired', { data: 99 }, 10); - await new Promise(r => setTimeout(r, 30)); // Past 2× TTL - const expired = cache.get('swr-expired'); - assert(expired === undefined, 'should return undefined past grace'); - }); - await t('Cache isStale', async () => { - cache.set('stale-test', { data: 1 }, 50); - await new Promise(r => setTimeout(r, 60)); - assert(cache.isStale('stale-test') === true, 'should be stale'); - }); - await t('Cache recompute', () => { - let ran = false; - cache.recompute('recompute-test', () => { ran = true; }); - assert(ran === true, 'recompute fn should run'); - }); - await t('Cache debouncedInvalidateAll', async () => { - cache.debouncedInvalidateAll(); - await new Promise(r => setTimeout(r, 200)); - }); - - // ── Cache operations ── - await t('Cache set/get/invalidate', () => { - cache.set('test-key', { data: 1 }, 60000); - assert(cache.get('test-key').data === 1); - cache.invalidate('test-key'); - }); - await t('Cache invalidate+refetch', async () => { - cache.clear(); - await request(app).get('/api/stats').expect(200); - await request(app).get('/api/stats').expect(200); - }); - - // ── Channel messages fresh (dedup coverage) ── - await t('GET /api/channels/ch01/messages fresh', async () => { - cache.clear(); - const r = await request(app).get('/api/channels/ch01/messages').expect(200); - assert(r.body.messages !== undefined, 'should have messages'); - }); - await t('GET /api/channels/ch01/messages?limit=1&offset=0', async () => { - cache.clear(); - await request(app).get('/api/channels/ch01/messages?limit=1&offset=0').expect(200); - }); - - // ── Multi-filter packet queries ── - await t('GET /api/packets?type=4&node=TestRepeater1', async () => { - await request(app).get('/api/packets?type=4&node=TestRepeater1').expect(200); - }); - await t('GET /api/packets all filters', async () => { - await request(app).get('/api/packets?type=4&route=1&since=2000-01-01T00:00:00Z&until=2099-01-01T00:00:00Z&observer=test-obs-1&hash=test-hash-001').expect(200); - }); - await t('GET /api/packets?type=5®ion=SFO&node=TestRepeater1', async () => { - await request(app).get('/api/packets?type=5®ion=SFO&node=TestRepeater1&since=2000-01-01T00:00:00Z').expect(200); - }); - - // ── Perf/nocache bypass ── - await t('GET /api/stats?nocache=1', async () => { - await request(app).get('/api/stats?nocache=1').expect(200); - }); - await t('GET /api/nodes?nocache=1', async () => { - await request(app).get('/api/nodes?nocache=1').expect(200); - }); - - // ── More route branch coverage ── - await t('GET /api/packets/:id by hash', async () => { - await request(app).get('/api/packets/testhash00000001').expect(404); // 16 hex chars - }); - await t('GET /api/packets/:id by string non-hash', async () => { - await request(app).get('/api/packets/not-a-hash-or-number').expect(404); - }); - - // ── SPA fallback paths ── - for (const path of ['/map', '/packets', '/analytics', '/live']) { - await t(`GET ${path} SPA fallback`, async () => { - const r = await request(app).get(path); - assert([200, 304, 404].includes(r.status)); - }); - } - - // ── Network status ── - await t('GET /api/nodes/network-status?region=ZZZZ', async () => { - cache.clear(); - await request(app).get('/api/nodes/network-status?region=ZZZZ').expect(200); - }); - - // ── Bulk health variants ── - await t('GET /api/nodes/bulk-health?limit=2', async () => { - cache.clear(); - await request(app).get('/api/nodes/bulk-health?limit=2').expect(200); - }); - await t('GET /api/nodes/bulk-health?region=NYC', async () => { - cache.clear(); - await request(app).get('/api/nodes/bulk-health?region=NYC').expect(200); - }); - - // ── Decoder: various payload types ── - await t('POST /api/decode REQ', async () => { - const r = await request(app).post('/api/decode').send({ hex: '0000' + 'AA'.repeat(20) }); - assert(r.status === 200 && r.body.decoded.payload.type === 'REQ'); - }); - await t('POST /api/decode RESPONSE', async () => { - const r = await request(app).post('/api/decode').send({ hex: '0400' + 'AA'.repeat(20) }); - assert(r.status === 200 && r.body.decoded.payload.type === 'RESPONSE'); - }); - await t('POST /api/decode TXT_MSG', async () => { - const r = await request(app).post('/api/decode').send({ hex: '0800' + 'AA'.repeat(20) }); - assert(r.status === 200 && r.body.decoded.payload.type === 'TXT_MSG'); - }); - await t('POST /api/decode ACK', async () => { - const r = await request(app).post('/api/decode').send({ hex: '0C00' + 'BB'.repeat(6) }); - assert(r.status === 200 && r.body.decoded.payload.type === 'ACK'); - }); - await t('POST /api/decode GRP_TXT', async () => { - const r = await request(app).post('/api/decode').send({ hex: '1500' + 'CC'.repeat(10) }); - assert(r.status === 200 && r.body.decoded.payload.type === 'GRP_TXT'); - }); - await t('POST /api/decode ANON_REQ', async () => { - const r = await request(app).post('/api/decode').send({ hex: '1D00' + 'DD'.repeat(40) }); - assert(r.status === 200 && r.body.decoded.payload.type === 'ANON_REQ'); - }); - await t('POST /api/decode PATH', async () => { - const r = await request(app).post('/api/decode').send({ hex: '2100' + 'EE'.repeat(10) }); - assert(r.status === 200 && r.body.decoded.payload.type === 'PATH'); - }); - await t('POST /api/decode TRACE', async () => { - const r = await request(app).post('/api/decode').send({ hex: '2500' + 'FF'.repeat(12) }); - assert(r.status === 200 && r.body.decoded.payload.type === 'TRACE'); - }); - await t('POST /api/decode UNKNOWN type', async () => { - const r = await request(app).post('/api/decode').send({ hex: '3C00' + '00'.repeat(10) }); - assert(r.status === 200 && r.body.decoded.payload.type === 'UNKNOWN'); - }); - await t('POST /api/decode TRANSPORT_FLOOD', async () => { - const r = await request(app).post('/api/decode').send({ hex: '1200AABBCCDD' + '00'.repeat(101) }); - assert([200, 400].includes(r.status)); - }); - await t('POST /api/decode minimal ADVERT', async () => { - await request(app).post('/api/decode').send({ hex: '1100' + '00'.repeat(100) }).expect(200); - }); - await t('POST /api/decode too-short', async () => { - await request(app).post('/api/decode').send({ hex: 'AA' }).expect(400); - }); - // Short payloads triggering error branches - await t('POST /api/decode GRP_TXT too short', async () => { - await request(app).post('/api/decode').send({ hex: '1500AABB' }).expect(200); - }); - await t('POST /api/decode ADVERT too short', async () => { - await request(app).post('/api/decode').send({ hex: '1100' + 'AA'.repeat(10) }).expect(200); - }); - await t('POST /api/decode TRACE too short', async () => { - await request(app).post('/api/decode').send({ hex: '2500' + 'FF'.repeat(5) }).expect(200); - }); - await t('POST /api/decode PATH too short', async () => { - await request(app).post('/api/decode').send({ hex: '2100' + 'EE'.repeat(2) }).expect(200); - }); - await t('POST /api/decode ANON_REQ too short', async () => { - await request(app).post('/api/decode').send({ hex: '1D00' + 'DD'.repeat(10) }).expect(200); - }); - - // ── server-helpers: disambiguateHops direct tests ── - await t('disambiguateHops ambiguous multi-candidate', () => { - const helpers = require('./server-helpers'); - const allNodes = [ - { public_key: 'aabb' + '0'.repeat(60), name: 'Node1', lat: 37.7, lon: -122.4 }, - { public_key: 'aabb' + '1'.repeat(60), name: 'Node2', lat: 34.0, lon: -118.2 }, - { public_key: 'ccdd' + '0'.repeat(60), name: 'Node3', lat: 40.7, lon: -74.0 }, - ]; - const resolved = helpers.disambiguateHops(['aabb', 'ccdd'], allNodes); - assert(resolved.length === 2 && resolved[0].name); - }); - await t('disambiguateHops backward pass', () => { - const helpers = require('./server-helpers'); - // Ambiguous first hop, known second → backward pass resolves first - const allNodes = [ - { public_key: 'aa' + '3'.repeat(62), name: 'ANode1', lat: 37.7, lon: -122.4 }, - { public_key: 'aa' + '4'.repeat(62), name: 'ANode2', lat: 34.0, lon: -118.2 }, - { public_key: 'bb' + '3'.repeat(62), name: 'BNode', lat: 40.7, lon: -74.0 }, - ]; - const resolved = helpers.disambiguateHops(['aa', 'bb'], allNodes); - assert(resolved.length === 2); - }); - await t('disambiguateHops distance unreliable', () => { - const helpers = require('./server-helpers'); - const allNodes = [ - { public_key: 'aa' + '5'.repeat(62), name: 'Near1', lat: 37.7, lon: -122.4 }, - { public_key: 'bb' + '5'.repeat(62), name: 'FarAway', lat: -33.8, lon: 151.2 }, - { public_key: 'cc' + '5'.repeat(62), name: 'Near2', lat: 37.8, lon: -122.3 }, - ]; - const resolved = helpers.disambiguateHops(['aa', 'bb', 'cc'], allNodes, 0.5); - assert(resolved[1].unreliable === true, 'middle node should be unreliable'); - }); - await t('disambiguateHops unknown prefix', () => { - const helpers = require('./server-helpers'); - const allNodes = [{ public_key: 'aabb' + '0'.repeat(60), name: 'Node1', lat: 37.7, lon: -122.4 }]; - const resolved = helpers.disambiguateHops(['ffff', 'aabb'], allNodes); - assert(resolved[0].known === false); - }); - await t('disambiguateHops single known match', () => { - const helpers = require('./server-helpers'); - const allNodes = [{ public_key: 'ccdd' + '6'.repeat(60), name: 'UniqueNode', lat: 40.7, lon: -74.0 }]; - const resolved = helpers.disambiguateHops(['ccdd'], allNodes); - assert(resolved[0].known === true && resolved[0].name === 'UniqueNode'); - }); - await t('disambiguateHops no-coord node', () => { - const helpers = require('./server-helpers'); - const allNodes = [{ public_key: 'aabb' + '7'.repeat(60), name: 'NoCoord', lat: 0, lon: 0 }]; - const resolved = helpers.disambiguateHops(['aabb'], allNodes); - assert(resolved.length === 1); - }); - - // ── isHashSizeFlipFlop ── - await t('isHashSizeFlipFlop true', () => { - const h = require('./server-helpers'); - assert(h.isHashSizeFlipFlop([1, 2, 1, 2], new Set([1, 2])) === true); - }); - await t('isHashSizeFlipFlop false stable', () => { - const h = require('./server-helpers'); - assert(h.isHashSizeFlipFlop([1, 1, 1], new Set([1])) === false); - }); - await t('isHashSizeFlipFlop false short/null', () => { - const h = require('./server-helpers'); - assert(h.isHashSizeFlipFlop([1, 2], new Set([1, 2])) === false); - assert(h.isHashSizeFlipFlop(null, null) === false); - }); - - // ── lastPathSeenMap: repeater hop tracking ── - await t('node appearing only as path hop gets last_heard', async () => { - // Create a node that has NO packets in pktStore (only exists in DB) - const hopPubkey = 'ffaa' + '0'.repeat(60); - db.upsertNode({ public_key: hopPubkey, name: 'HopOnlyRepeater', role: 'repeater', lat: 0, lon: 0, last_seen: '2020-01-01T00:00:00.000Z' }); - - // Simulate it being seen as a path hop - const recentTime = new Date().toISOString(); - lastPathSeenMap.set(hopPubkey, recentTime); - - const res = await request(app).get('/api/nodes?search=HopOnlyRepeater'); - assert(res.status === 200); - assert(res.body.nodes.length >= 1, 'should find the hop-only node'); - const node = res.body.nodes.find(n => n.public_key === hopPubkey); - assert(node, 'node should exist in results'); - assert(node.last_heard === recentTime, 'last_heard should come from lastPathSeenMap'); - - // Cleanup - lastPathSeenMap.delete(hopPubkey); - }); - - await t('last_heard from path hop preferred over stale last_seen', async () => { - const hopPubkey = 'ffbb' + '0'.repeat(60); - const staleTime = '2020-01-01T00:00:00.000Z'; - const freshTime = new Date().toISOString(); - db.upsertNode({ public_key: hopPubkey, name: 'StaleRepeater', role: 'repeater', lat: 0, lon: 0, last_seen: staleTime }); - - // Path hop seen recently - lastPathSeenMap.set(hopPubkey, freshTime); - - const res = await request(app).get('/api/nodes?search=StaleRepeater'); - assert(res.status === 200); - const node = res.body.nodes.find(n => n.public_key === hopPubkey); - assert(node, 'node should exist'); - assert(node.last_heard === freshTime, 'last_heard should be fresh path time, not stale DB time'); - assert(node.last_seen === staleTime, 'last_seen (DB) should still be stale'); - - lastPathSeenMap.delete(hopPubkey); - }); - - await t('last_heard from pktStore preferred over older path hop', async () => { - const hopPubkey = 'aabb' + '0'.repeat(60); // TestRepeater1 — has packets in pktStore - const oldPathTime = '2019-01-01T00:00:00.000Z'; - lastPathSeenMap.set(hopPubkey, oldPathTime); - - const res = await request(app).get('/api/nodes?search=TestRepeater1'); - assert(res.status === 200); - const node = res.body.nodes.find(n => n.public_key === hopPubkey); - assert(node, 'node should exist'); - // pktStore should have a more recent timestamp than our old path time - assert(node.last_heard > oldPathTime, 'pktStore timestamp should win over older path hop time'); - - lastPathSeenMap.delete(hopPubkey); - }); - - await t('bulk-health cache invalidated after advert', () => { - // Set a fake bulk-health cache entry - cache.set('bulk-health:50:r=', { fake: true }, 60000); - assert(cache.get('bulk-health:50:r='), 'cache should have bulk-health entry'); - - // Simulate what happens on advert: cache.invalidate('bulk-health') - cache.invalidate('bulk-health'); - assert(!cache.get('bulk-health:50:r='), 'bulk-health cache should be invalidated after advert'); - }); - - // ── Issue #126: hash prefix collision — ambiguous hop attribution ── - - await t('ambiguous hop prefix does NOT update lastPathSeenMap for either node (fixes #126)', async () => { - // Two nodes sharing prefix 'ab12': collision scenario like 1CC4 vs 1C82 - const nodeA = 'ab12' + 'a'.repeat(60); - const nodeB = 'ab12' + 'b'.repeat(60); - db.upsertNode({ public_key: nodeA, name: 'CollidingNodeA', role: 'repeater', lat: 37.0, lon: -122.0, last_seen: '2020-01-01T00:00:00.000Z' }); - db.upsertNode({ public_key: nodeB, name: 'CollidingNodeB', role: 'repeater', lat: 38.0, lon: -121.0, last_seen: '2020-01-01T00:00:00.000Z' }); - - // Clear caches to start fresh - hopPrefixToKey.delete('ab12'); - ambiguousHopPrefixes.delete('ab12'); - lastPathSeenMap.delete(nodeA); - lastPathSeenMap.delete(nodeB); - - // Attempt to resolve the ambiguous prefix — should return null - const result = resolveUniquePrefixMatch('ab12'); - assert(result === null, 'ambiguous prefix should resolve to null'); - assert(ambiguousHopPrefixes.has('ab12'), 'ab12 should be in ambiguous prefix cache'); - assert(!hopPrefixToKey.has('ab12'), 'ab12 should NOT be in hopPrefixToKey cache'); - - // Verify neither node gets last_heard updated - assert(!lastPathSeenMap.has(nodeA), 'lastPathSeenMap should NOT have nodeA (ambiguous prefix)'); - assert(!lastPathSeenMap.has(nodeB), 'lastPathSeenMap should NOT have nodeB (ambiguous prefix)'); - - // Subsequent resolution calls should also return null (cached negative result) - const result2 = resolveUniquePrefixMatch('ab12'); - assert(result2 === null, 'cached ambiguous prefix should still resolve to null'); - - // Cleanup - hopPrefixToKey.delete('ab12'); - ambiguousHopPrefixes.delete('ab12'); - }); - - await t('unique hop prefix still resolves and updates lastPathSeenMap', async () => { - // A node with a unique prefix that no other node shares - const uniqueNode = 'eeee' + 'f'.repeat(60); - db.upsertNode({ public_key: uniqueNode, name: 'UniqueHopNode', role: 'repeater', lat: 40.0, lon: -74.0, last_seen: '2020-01-01T00:00:00.000Z' }); - - // Clear caches - hopPrefixToKey.delete('eeee'); - ambiguousHopPrefixes.delete('eeee'); - lastPathSeenMap.delete(uniqueNode); - - // Resolve the unique prefix — should return the full key - const result = resolveUniquePrefixMatch('eeee'); - assert(result === uniqueNode, 'unique prefix should resolve to the full public_key'); - assert(hopPrefixToKey.get('eeee') === uniqueNode, 'unique prefix should be cached in hopPrefixToKey'); - assert(!ambiguousHopPrefixes.has('eeee'), 'unique prefix should NOT be in ambiguous cache'); - - // Cleanup - hopPrefixToKey.delete('eeee'); - lastPathSeenMap.delete(uniqueNode); - }); - - await t('1-byte ambiguous prefix does NOT update dead node (issue #126 regression)', async () => { - // Simulate the exact scenario: two nodes with shared 1-byte prefix '1c' - const deadNode = '1c' + 'c4' + 'dd'.repeat(30); - const liveNode = '1c' + '82' + 'ee'.repeat(30); - const staleTime = new Date(Date.now() - 8 * 86400000).toISOString(); // 8 days ago - const recentTime = new Date().toISOString(); - - db.upsertNode({ public_key: deadNode, name: 'DeadNodeR5D4', role: 'repeater', lat: 35.0, lon: -120.0, last_seen: staleTime }); - db.upsertNode({ public_key: liveNode, name: 'LiveNodeK2S0', role: 'repeater', lat: 35.1, lon: -120.1, last_seen: recentTime }); - - // Clear all caches - hopPrefixToKey.delete('1c'); - ambiguousHopPrefixes.delete('1c'); - lastPathSeenMap.delete(deadNode); - lastPathSeenMap.delete(liveNode); - - // The 1-byte prefix '1c' matches both nodes — should be ambiguous - const result = resolveUniquePrefixMatch('1c'); - assert(result === null, '1-byte prefix 1c should be ambiguous (matches 2 nodes)'); - - // Fetch DeadNodeR5D4 — should NOT have last_heard from pathSeenMap - const res = await request(app).get('/api/nodes?search=DeadNodeR5D4'); - assert(res.status === 200); - const node = res.body.nodes.find(n => n.public_key === deadNode); - assert(node, 'dead node should exist in DB'); - // last_heard should be null/undefined or equal to the stale last_seen — NOT recent - const lastHeard = node.last_heard ? new Date(node.last_heard).getTime() : 0; - const sevenDaysAgo = Date.now() - 7 * 86400000; - assert(lastHeard < sevenDaysAgo, 'dead node last_heard should NOT be recent (hash collision guard)'); - - // Cleanup - hopPrefixToKey.delete('1c'); - ambiguousHopPrefixes.delete('1c'); - lastPathSeenMap.delete(deadNode); - lastPathSeenMap.delete(liveNode); - }); - - // ── Cache hit rate includes stale hits ── - await t('Cache hitRate includes staleHits in formula', async () => { - cache.clear(); - cache.hits = 0; - cache.misses = 0; - cache.staleHits = 0; - // Simulate: 3 hits, 2 stale hits, 5 misses => rate = (3+2)/(3+2+5) = 50% - cache.hits = 3; - cache.staleHits = 2; - cache.misses = 5; - const r = await request(app).get('/api/health').expect(200); - assert(r.body.cache.hitRate === 50, 'hitRate should be (hits+staleHits)/(hits+staleHits+misses) = 50%, got ' + r.body.cache.hitRate); - // Reset - cache.hits = 0; - cache.misses = 0; - cache.staleHits = 0; - }); - - // ── Summary ── - console.log(`\n═══ Server Route Tests: ${passed} passed, ${failed} failed ═══`); - if (failed > 0) process.exit(1); - process.exit(0); -})(); diff --git a/tools/e2e-test.js b/tools/e2e-test.js deleted file mode 100644 index 01b0e8e8..00000000 --- a/tools/e2e-test.js +++ /dev/null @@ -1,476 +0,0 @@ -#!/usr/bin/env node -'use strict'; - -/** - * MeshCore Analyzer — End-to-End Validation Test (M12) - * - * Starts the server with a temp DB, injects 100+ synthetic packets, - * validates every API endpoint, WebSocket broadcasts, and optionally MQTT. - */ - -const { spawn, execSync } = require('child_process'); -const path = require('path'); -const fs = require('fs'); -const os = require('os'); -const crypto = require('crypto'); -const WebSocket = require('ws'); - -const PROJECT_DIR = path.join(__dirname, '..'); -const PORT = 13579; // avoid conflict with dev server -const BASE = `http://localhost:${PORT}`; - -// ── Helpers ────────────────────────────────────────────────────────── - -let passed = 0, failed = 0; -const failures = []; - -function assert(cond, label) { - if (cond) { passed++; } - else { failed++; failures.push(label); console.error(` ❌ FAIL: ${label}`); } -} - -async function get(path) { - const r = await fetch(`${BASE}${path}`); - return { status: r.status, data: await r.json() }; -} - -async function post(path, body) { - const r = await fetch(`${BASE}${path}`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(body), - }); - return { status: r.status, data: await r.json() }; -} - -function sleep(ms) { return new Promise(r => setTimeout(r, ms)); } - -// ── Packet generation (inline from generate-packets.js logic) ──────── - -const OBSERVERS = [ - { id: 'E2E-SJC-1', iata: 'SJC' }, - { id: 'E2E-SFO-2', iata: 'SFO' }, - { id: 'E2E-OAK-3', iata: 'OAK' }, -]; - -const NODE_NAMES = [ - 'TestNode Alpha', 'TestNode Beta', 'TestNode Gamma', 'TestNode Delta', - 'TestNode Epsilon', 'TestNode Zeta', 'TestNode Eta', 'TestNode Theta', -]; - -function rand(a, b) { return Math.random() * (b - a) + a; } -function randInt(a, b) { return Math.floor(rand(a, b + 1)); } -function pick(a) { return a[randInt(0, a.length - 1)]; } -function randomBytes(n) { return crypto.randomBytes(n); } - -function pubkeyFor(name) { - return crypto.createHash('sha256').update(name).digest(); -} - -function encodeHeader(routeType, payloadType, ver = 0) { - return (routeType & 0x03) | ((payloadType & 0x0F) << 2) | ((ver & 0x03) << 6); -} - -function buildPath(hopCount, hashSize = 2) { - const pathByte = ((hashSize - 1) << 6) | (hopCount & 0x3F); - const hops = crypto.randomBytes(hashSize * hopCount); - return { pathByte, hops }; -} - -function buildAdvert(name, role) { - const pubKey = pubkeyFor(name); - const ts = Buffer.alloc(4); ts.writeUInt32LE(Math.floor(Date.now() / 1000)); - const sig = randomBytes(64); - let flags = 0x80 | 0x10; // hasName + hasLocation - if (role === 'repeater') flags |= 0x02; - else if (role === 'room') flags |= 0x04; - else if (role === 'sensor') flags |= 0x08; - else flags |= 0x01; - const nameBuf = Buffer.from(name, 'utf8'); - const appdata = Buffer.alloc(9 + nameBuf.length); - appdata[0] = flags; - appdata.writeInt32LE(Math.round(37.34 * 1e6), 1); - appdata.writeInt32LE(Math.round(-121.89 * 1e6), 5); - nameBuf.copy(appdata, 9); - const payload = Buffer.concat([pubKey, ts, sig, appdata]); - const header = encodeHeader(1, 0x04, 0); // FLOOD + ADVERT - const { pathByte, hops } = buildPath(randInt(0, 3)); - return Buffer.concat([Buffer.from([header, pathByte]), hops, payload]); -} - -function buildGrpTxt(channelHash = 0) { - const mac = randomBytes(2); - const enc = randomBytes(randInt(10, 40)); - const payload = Buffer.concat([Buffer.from([channelHash]), mac, enc]); - const header = encodeHeader(1, 0x05, 0); // FLOOD + GRP_TXT - const { pathByte, hops } = buildPath(randInt(0, 3)); - return Buffer.concat([Buffer.from([header, pathByte]), hops, payload]); -} - -function buildAck() { - const payload = randomBytes(18); - const header = encodeHeader(2, 0x03, 0); - const { pathByte, hops } = buildPath(randInt(0, 2)); - return Buffer.concat([Buffer.from([header, pathByte]), hops, payload]); -} - -function buildTxtMsg() { - const payload = Buffer.concat([randomBytes(6), randomBytes(6), randomBytes(4), randomBytes(20)]); - const header = encodeHeader(2, 0x02, 0); - const { pathByte, hops } = buildPath(randInt(0, 2)); - return Buffer.concat([Buffer.from([header, pathByte]), hops, payload]); -} - -// ── Main ───────────────────────────────────────────────────────────── - -async function main() { - // 1. Create temp DB - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'meshcore-e2e-')); - const dbPath = path.join(tmpDir, 'test.db'); - console.log(`Temp DB: ${dbPath}`); - - // 2. Start server - console.log('Starting server...'); - const srv = spawn('node', ['server.js'], { - cwd: PROJECT_DIR, - env: { ...process.env, DB_PATH: dbPath, PORT: String(PORT) }, - stdio: ['ignore', 'pipe', 'pipe'], - }); - - let serverOutput = ''; - srv.stdout.on('data', d => { serverOutput += d; }); - srv.stderr.on('data', d => { serverOutput += d; }); - - // We need the server to respect PORT env — check if config is hard-coded - // The server uses config.port from config.json. We need to patch that or - // monkey-patch. Let's just use port 3000 if the server doesn't read PORT env. - // Actually let me check... - - const cleanup = () => { - try { srv.kill('SIGTERM'); } catch {} - try { fs.unlinkSync(dbPath); fs.rmdirSync(tmpDir); } catch {} - }; - - process.on('SIGINT', () => { cleanup(); process.exit(1); }); - process.on('uncaughtException', (e) => { console.error(e); cleanup(); process.exit(1); }); - - // 3. Wait for server ready - let ready = false; - for (let i = 0; i < 30; i++) { - await sleep(500); - try { - const r = await fetch(`${BASE}/api/stats`); - if (r.ok) { ready = true; break; } - } catch {} - } - - if (!ready) { - console.error('Server did not start in time. Output:', serverOutput); - cleanup(); - process.exit(1); - } - console.log('Server ready.\n'); - - // 4. Connect WebSocket - const wsMessages = []; - const ws = new WebSocket(`ws://localhost:${PORT}`); - await new Promise((resolve, reject) => { - ws.on('open', resolve); - ws.on('error', reject); - setTimeout(() => reject(new Error('WS timeout')), 5000); - }); - ws.on('message', (data) => { - try { wsMessages.push(JSON.parse(data.toString())); } catch {} - }); - console.log('WebSocket connected.\n'); - - // 5. Generate and inject packets - const roles = ['repeater', 'room', 'companion', 'sensor']; - const injected = []; - const advertNodes = {}; // name -> {role, pubkey, count} - const grpTxtCount = { total: 0, byChannel: {} }; - const observerCounts = {}; // id -> count - const hashToObservers = {}; // hash -> Set(observer) - - // Generate ADVERT packets — ensure at least one of each role - for (let ri = 0; ri < roles.length; ri++) { - const name = NODE_NAMES[ri]; - const role = roles[ri]; - const buf = buildAdvert(name, role); - const hex = buf.toString('hex').toUpperCase(); - const hash = crypto.createHash('md5').update(hex).digest('hex').slice(0, 16); - const obs = OBSERVERS[ri % OBSERVERS.length]; - injected.push({ hex, observer: obs.id, region: obs.iata, hash, snr: 5.0, rssi: -80 }); - advertNodes[name] = { role, pubkey: pubkeyFor(name).toString('hex'), count: 1 }; - observerCounts[obs.id] = (observerCounts[obs.id] || 0) + 1; - if (!hashToObservers[hash]) hashToObservers[hash] = new Set(); - hashToObservers[hash].add(obs.id); - } - - // More ADVERTs - for (let i = 0; i < 40; i++) { - const name = pick(NODE_NAMES); - const role = pick(roles); - const buf = buildAdvert(name, role); - const hex = buf.toString('hex').toUpperCase(); - const hash = crypto.createHash('md5').update(hex).digest('hex').slice(0, 16); - // Multi-observer: 30% chance heard by 2 observers - const obsCount = Math.random() < 0.3 ? 2 : 1; - const shuffled = [...OBSERVERS].sort(() => Math.random() - 0.5); - for (let o = 0; o < obsCount; o++) { - const obs = shuffled[o]; - injected.push({ hex, observer: obs.id, region: obs.iata, hash, snr: rand(-2, 10), rssi: rand(-110, -60) }); - observerCounts[obs.id] = (observerCounts[obs.id] || 0) + 1; - if (!hashToObservers[hash]) hashToObservers[hash] = new Set(); - hashToObservers[hash].add(obs.id); - } - if (!advertNodes[name]) advertNodes[name] = { role, pubkey: pubkeyFor(name).toString('hex'), count: 0 }; - advertNodes[name].count++; - } - - // GRP_TXT packets - for (let i = 0; i < 30; i++) { - const ch = randInt(0, 3); - const buf = buildGrpTxt(ch); - const hex = buf.toString('hex').toUpperCase(); - const hash = crypto.createHash('md5').update(hex).digest('hex').slice(0, 16); - const obs = pick(OBSERVERS); - injected.push({ hex, observer: obs.id, region: obs.iata, hash, snr: 3.0, rssi: -90 }); - grpTxtCount.total++; - grpTxtCount.byChannel[ch] = (grpTxtCount.byChannel[ch] || 0) + 1; - observerCounts[obs.id] = (observerCounts[obs.id] || 0) + 1; - if (!hashToObservers[hash]) hashToObservers[hash] = new Set(); - hashToObservers[hash].add(obs.id); - } - - // ACK + TXT_MSG - for (let i = 0; i < 20; i++) { - const buf = i < 10 ? buildAck() : buildTxtMsg(); - const hex = buf.toString('hex').toUpperCase(); - const hash = crypto.createHash('md5').update(hex).digest('hex').slice(0, 16); - const obs = pick(OBSERVERS); - injected.push({ hex, observer: obs.id, region: obs.iata, hash, snr: 1.0, rssi: -95 }); - observerCounts[obs.id] = (observerCounts[obs.id] || 0) + 1; - if (!hashToObservers[hash]) hashToObservers[hash] = new Set(); - hashToObservers[hash].add(obs.id); - } - - // Find a hash with multiple observers for trace testing - let traceHash = null; - for (const [h, obs] of Object.entries(hashToObservers)) { - if (obs.size >= 2) { traceHash = h; break; } - } - // If none, create one explicitly - if (!traceHash) { - const buf = buildAck(); - const hex = buf.toString('hex').toUpperCase(); - traceHash = crypto.createHash('md5').update(hex).digest('hex').slice(0, 16); - injected.push({ hex, observer: OBSERVERS[0].id, region: OBSERVERS[0].iata, hash: traceHash, snr: 5, rssi: -80 }); - injected.push({ hex, observer: OBSERVERS[1].id, region: OBSERVERS[1].iata, hash: traceHash, snr: 3, rssi: -90 }); - observerCounts[OBSERVERS[0].id] = (observerCounts[OBSERVERS[0].id] || 0) + 1; - observerCounts[OBSERVERS[1].id] = (observerCounts[OBSERVERS[1].id] || 0) + 1; - } - - console.log(`Injecting ${injected.length} packets...`); - let injectOk = 0, injectFail = 0; - for (const pkt of injected) { - const r = await post('/api/packets', pkt); - if (r.status === 200) injectOk++; - else { injectFail++; if (injectFail <= 3) console.error(' Inject fail:', r.data); } - } - console.log(`Injected: ${injectOk} ok, ${injectFail} fail\n`); - assert(injectFail === 0, 'All packets injected successfully'); - assert(injected.length >= 100, `Injected 100+ packets (got ${injected.length})`); - - // Wait a moment for WS messages to arrive - await sleep(500); - - // ── Validate ─────────────────────────────────────────────────────── - - // 5a. Stats - console.log('── Stats ──'); - const stats = (await get('/api/stats')).data; - // totalPackets includes seed packet, so should be >= injected.length - assert(stats.totalPackets > 0, `stats.totalPackets (${stats.totalPackets}) >= ${injected.length}`); - assert(stats.totalNodes > 0, `stats.totalNodes > 0 (${stats.totalNodes})`); - assert(stats.totalObservers >= OBSERVERS.length, `stats.totalObservers >= ${OBSERVERS.length} (${stats.totalObservers})`); - console.log(` totalPackets=${stats.totalPackets} totalNodes=${stats.totalNodes} totalObservers=${stats.totalObservers}\n`); - - // 5b. Packets API - basic list - console.log('── Packets API ──'); - const pktsAll = (await get('/api/packets?limit=200')).data; - assert(pktsAll.total > 0, `packets total (${pktsAll.total}) > 0`); - assert(pktsAll.packets.length > 0, 'packets array not empty'); - - // Filter by type (ADVERT = 4) - const pktsAdvert = (await get('/api/packets?type=4&limit=200')).data; - assert(pktsAdvert.total > 0, `filter by type=ADVERT returns results (${pktsAdvert.total})`); - assert(pktsAdvert.packets.every(p => p.payload_type === 4), 'all filtered packets are ADVERT'); - - // Filter by observer - const testObs = OBSERVERS[0].id; - const pktsObs = (await get(`/api/packets?observer=${testObs}&limit=200`)).data; - assert(pktsObs.total > 0, `filter by observer=${testObs} returns results`); - assert(pktsObs.packets.length > 0, 'observer filter returns packets'); - - // Filter by region - const pktsRegion = (await get('/api/packets?region=SJC&limit=200')).data; - assert(pktsRegion.total > 0, 'filter by region=SJC returns results'); - - // Pagination - const page1 = (await get('/api/packets?limit=5&offset=0')).data; - const page2 = (await get('/api/packets?limit=5&offset=5')).data; - assert(page1.packets.length === 5, 'pagination: page1 has 5'); - assert(page2.packets.length === 5, 'pagination: page2 has 5'); - if (page1.packets.length && page2.packets.length) { - assert(page1.packets[0].id !== page2.packets[0].id, 'pagination: pages are different'); - } - - // groupByHash - const grouped = (await get('/api/packets?groupByHash=true&limit=200')).data; - assert(grouped.total > 0, `groupByHash returns results (${grouped.total})`); - assert(grouped.packets[0].hash !== undefined, 'groupByHash entries have hash'); - assert(grouped.packets[0].count !== undefined, 'groupByHash entries have count'); - // Find a multi-observer group - const multiObs = grouped.packets.find(p => p.observer_count >= 2); - assert(!!multiObs, 'groupByHash has entry with observer_count >= 2'); - console.log(' ✓ Packets API checks passed\n'); - - // 5c. Packet detail - console.log('── Packet Detail ──'); - const firstPkt = pktsAll.packets[0]; - const detail = (await get(`/api/packets/${firstPkt.id}`)).data; - assert(detail.packet !== undefined, 'detail has packet'); - assert(detail.breakdown !== undefined, 'detail has breakdown'); - assert(detail.breakdown.ranges !== undefined, 'breakdown has ranges'); - assert(detail.breakdown.ranges.length > 0, 'breakdown has color ranges'); - assert(detail.breakdown.ranges[0].color !== undefined, 'ranges have color field'); - assert(detail.breakdown.ranges[0].start !== undefined, 'ranges have start field'); - console.log(` ✓ Detail: ${detail.breakdown.ranges.length} color ranges\n`); - - // 5d. Nodes - console.log('── Nodes ──'); - const nodesResp = (await get('/api/nodes?limit=50')).data; - assert(nodesResp.total > 0, `nodes total > 0 (${nodesResp.total})`); - assert(nodesResp.nodes.length > 0, 'nodes array not empty'); - assert(nodesResp.counts !== undefined, 'nodes response has counts'); - - // Role filtering - const repNodes = (await get('/api/nodes?role=repeater')).data; - assert(repNodes.nodes.every(n => n.role === 'repeater'), 'role filter works for repeater'); - - // Node detail - const someNode = nodesResp.nodes[0]; - const nodeDetail = (await get(`/api/nodes/${someNode.public_key}`)).data; - assert(nodeDetail.node !== undefined, 'node detail has node'); - assert(nodeDetail.node.public_key === someNode.public_key, 'node detail matches pubkey'); - assert(nodeDetail.recentAdverts !== undefined, 'node detail has recentAdverts'); - console.log(` ✓ Nodes: ${nodesResp.total} total, detail works\n`); - - // 5e. Channels - console.log('── Channels ──'); - const chResp = (await get('/api/channels')).data; - const chList = chResp.channels || []; - assert(Array.isArray(chList), 'channels response is array'); - if (chList.length > 0) { - const someCh = chList[0]; - assert(someCh.messageCount > 0, `channel has messages (${someCh.messageCount})`); - const msgResp = (await get(`/api/channels/${encodeURIComponent(someCh.hash)}/messages`)).data; - assert(msgResp.messages.length > 0, 'channel has message list'); - assert(msgResp.messages[0].sender !== undefined, 'message has sender'); - console.log(` ✓ Channels: ${chList.length} channels\n`); - } else { - console.log(` ⚠ Channels: 0 (synthetic packets don't produce decodable channel messages)\n`); - } - - // 5f. Observers - console.log('── Observers ──'); - const obsResp = (await get('/api/observers')).data; - assert(obsResp.observers.length >= OBSERVERS.length, `observers >= ${OBSERVERS.length} (${obsResp.observers.length})`); - for (const expObs of OBSERVERS) { - const found = obsResp.observers.find(o => o.id === expObs.id); - assert(!!found, `observer ${expObs.id} exists`); - if (found) { - assert(found.packet_count > 0, `observer ${expObs.id} has packet_count > 0 (${found.packet_count})`); - } - } - console.log(` ✓ Observers: ${obsResp.observers.length}\n`); - - // 5g. Traces - console.log('── Traces ──'); - if (traceHash) { - const traceResp = (await get(`/api/traces/${traceHash}`)).data; - assert(Array.isArray(traceResp.traces), 'trace response is array'); - if (traceResp.traces.length >= 2) { - const traceObservers = new Set(traceResp.traces.map(t => t.observer)); - assert(traceObservers.size >= 2, `trace has >= 2 distinct observers (${traceObservers.size})`); - } - console.log(` ✓ Traces: ${traceResp.traces.length} entries for hash\n`); - } else { - console.log(' ⚠ No multi-observer hash available for trace test\n'); - } - - // 5h. WebSocket - console.log('── WebSocket ──'); - assert(wsMessages.length > 0, `WebSocket received messages (${wsMessages.length})`); - assert(wsMessages.length >= injected.length * 0.5, `WS got >= 50% of injected (${wsMessages.length}/${injected.length})`); - const wsPacketMsgs = wsMessages.filter(m => m.type === 'packet'); - assert(wsPacketMsgs.length > 0, 'WS has packet-type messages'); - console.log(` ✓ WebSocket: ${wsMessages.length} messages received\n`); - - // 6. MQTT (optional) - console.log('── MQTT ──'); - let mqttAvailable = false; - try { - execSync('which mosquitto_pub', { stdio: 'ignore' }); - mqttAvailable = true; - } catch {} - - if (mqttAvailable) { - console.log(' mosquitto_pub found, testing MQTT path...'); - // Would need a running mosquitto broker — skip if not running - try { - const mqttMod = require('mqtt'); - const mc = mqttMod.connect('mqtt://localhost:1883', { connectTimeout: 2000 }); - await new Promise((resolve, reject) => { - mc.on('connect', resolve); - mc.on('error', reject); - setTimeout(() => reject(new Error('timeout')), 2000); - }); - const mqttHex = buildAdvert('MQTTTestNode', 'repeater').toString('hex').toUpperCase(); - const mqttHash = 'mqtt-test-hash-001'; - mc.publish('meshcore/SJC/MQTT-OBS-1/packets', JSON.stringify({ - raw: mqttHex, SNR: 8.0, RSSI: -75, hash: mqttHash, - })); - await sleep(1000); - mc.end(); - const mqttTrace = (await get(`/api/traces/${mqttHash}`)).data; - assert(mqttTrace.traces.length >= 1, 'MQTT packet appeared in traces'); - console.log(' ✓ MQTT path works\n'); - } catch (e) { - console.log(` ⚠ MQTT broker not reachable: ${e.message}\n`); - } - } else { - console.log(' ⚠ mosquitto not available, skipping MQTT test\n'); - } - - // 7. Summary - ws.close(); - cleanup(); - - console.log('═══════════════════════════════════════'); - console.log(` PASSED: ${passed}`); - console.log(` FAILED: ${failed}`); - if (failures.length) { - console.log(' Failures:'); - failures.forEach(f => console.log(` - ${f}`)); - } - console.log('═══════════════════════════════════════'); - - process.exit(failed > 0 ? 1 : 0); -} - -main().catch(e => { - console.error('Fatal:', e); - process.exit(1); -}); diff --git a/tools/frontend-test.js b/tools/frontend-test.js deleted file mode 100644 index 9178b39d..00000000 --- a/tools/frontend-test.js +++ /dev/null @@ -1,320 +0,0 @@ -#!/usr/bin/env node -'use strict'; - -/** - * MeshCore Analyzer — Frontend Smoke Tests (M13) - * - * Starts the server with a temp DB, injects synthetic packets, - * then validates HTML pages, JS syntax, and API data shapes. - */ - -const { spawn } = require('child_process'); -const path = require('path'); -const fs = require('fs'); -const os = require('os'); -const crypto = require('crypto'); - -const PROJECT_DIR = path.join(__dirname, '..'); -const PORT = 13580; -const BASE = `http://localhost:${PORT}`; - -// ── Helpers ────────────────────────────────────────────────────────── - -let passed = 0, failed = 0; -const failures = []; - -function assert(cond, label) { - if (cond) { passed++; } - else { failed++; failures.push(label); console.error(` ❌ FAIL: ${label}`); } -} - -function sleep(ms) { return new Promise(r => setTimeout(r, ms)); } - -async function get(urlPath) { - const r = await fetch(`${BASE}${urlPath}`); - return { status: r.status, data: await r.json() }; -} - -async function getHtml(urlPath) { - const r = await fetch(`${BASE}${urlPath}`); - return { status: r.status, text: await r.text() }; -} - -async function post(urlPath, body) { - const r = await fetch(`${BASE}${urlPath}`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(body), - }); - return { status: r.status, data: await r.json() }; -} - -// ── Packet builders (from e2e-test.js) ─────────────────────────────── - -function rand(a, b) { return Math.random() * (b - a) + a; } -function randInt(a, b) { return Math.floor(rand(a, b + 1)); } -function pick(a) { return a[randInt(0, a.length - 1)]; } - -function pubkeyFor(name) { - return crypto.createHash('sha256').update(name).digest(); -} - -function encodeHeader(routeType, payloadType, ver = 0) { - return (routeType & 0x03) | ((payloadType & 0x0F) << 2) | ((ver & 0x03) << 6); -} - -function buildPath(hopCount, hashSize = 2) { - const pathByte = ((hashSize - 1) << 6) | (hopCount & 0x3F); - const hops = crypto.randomBytes(hashSize * hopCount); - return { pathByte, hops }; -} - -function buildAdvert(name, role) { - const pubKey = pubkeyFor(name); - const ts = Buffer.alloc(4); ts.writeUInt32LE(Math.floor(Date.now() / 1000)); - const sig = crypto.randomBytes(64); - let flags = 0x80 | 0x10; - if (role === 'repeater') flags |= 0x02; - else if (role === 'room') flags |= 0x04; - else if (role === 'sensor') flags |= 0x08; - else flags |= 0x01; - const nameBuf = Buffer.from(name, 'utf8'); - const appdata = Buffer.alloc(9 + nameBuf.length); - appdata[0] = flags; - appdata.writeInt32LE(Math.round(37.34 * 1e6), 1); - appdata.writeInt32LE(Math.round(-121.89 * 1e6), 5); - nameBuf.copy(appdata, 9); - const payload = Buffer.concat([pubKey, ts, sig, appdata]); - const header = encodeHeader(1, 0x04, 0); - const { pathByte, hops } = buildPath(randInt(0, 3)); - return Buffer.concat([Buffer.from([header, pathByte]), hops, payload]); -} - -function buildGrpTxt(channelHash = 0) { - const mac = crypto.randomBytes(2); - const enc = crypto.randomBytes(randInt(10, 40)); - const payload = Buffer.concat([Buffer.from([channelHash]), mac, enc]); - const header = encodeHeader(1, 0x05, 0); - const { pathByte, hops } = buildPath(randInt(0, 3)); - return Buffer.concat([Buffer.from([header, pathByte]), hops, payload]); -} - -function buildAck() { - const payload = crypto.randomBytes(18); - const header = encodeHeader(2, 0x03, 0); - const { pathByte, hops } = buildPath(randInt(0, 2)); - return Buffer.concat([Buffer.from([header, pathByte]), hops, payload]); -} - -// ── Main ───────────────────────────────────────────────────────────── - -const OBSERVERS = [ - { id: 'FE-SJC-1', iata: 'SJC' }, - { id: 'FE-SFO-2', iata: 'SFO' }, -]; - -const NODE_NAMES = ['FENode Alpha', 'FENode Beta', 'FENode Gamma', 'FENode Delta']; - -async function main() { - // 1. Temp DB - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'meshcore-fe-')); - const dbPath = path.join(tmpDir, 'test.db'); - console.log(`Temp DB: ${dbPath}`); - - // 2. Start server - console.log('Starting server...'); - const srv = spawn('node', ['server.js'], { - cwd: PROJECT_DIR, - env: { ...process.env, DB_PATH: dbPath, PORT: String(PORT) }, - stdio: ['ignore', 'pipe', 'pipe'], - }); - - let serverOutput = ''; - srv.stdout.on('data', d => { serverOutput += d; }); - srv.stderr.on('data', d => { serverOutput += d; }); - - const cleanup = () => { - try { srv.kill('SIGTERM'); } catch {} - try { fs.unlinkSync(dbPath); fs.rmdirSync(tmpDir); } catch {} - }; - - process.on('SIGINT', () => { cleanup(); process.exit(1); }); - - // 3. Wait for ready - let ready = false; - for (let i = 0; i < 30; i++) { - await sleep(500); - try { - const r = await fetch(`${BASE}/api/stats`); - if (r.ok) { ready = true; break; } - } catch {} - } - if (!ready) { - console.error('Server did not start. Output:', serverOutput); - cleanup(); - process.exit(1); - } - console.log('Server ready.\n'); - - // 4. Inject test data - const injected = []; - const roles = ['repeater', 'room', 'companion', 'sensor']; - for (let i = 0; i < NODE_NAMES.length; i++) { - const buf = buildAdvert(NODE_NAMES[i], roles[i]); - const hex = buf.toString('hex').toUpperCase(); - const hash = crypto.createHash('md5').update(hex).digest('hex').slice(0, 16); - const obs = OBSERVERS[i % OBSERVERS.length]; - injected.push({ hex, observer: obs.id, region: obs.iata, hash, snr: 5.0, rssi: -80 }); - } - for (let i = 0; i < 20; i++) { - const buf = buildGrpTxt(i % 3); - const hex = buf.toString('hex').toUpperCase(); - const hash = crypto.createHash('md5').update(hex).digest('hex').slice(0, 16); - const obs = pick(OBSERVERS); - injected.push({ hex, observer: obs.id, region: obs.iata, hash, snr: 3.0, rssi: -90 }); - } - for (let i = 0; i < 10; i++) { - const buf = buildAck(); - const hex = buf.toString('hex').toUpperCase(); - const hash = crypto.createHash('md5').update(hex).digest('hex').slice(0, 16); - const obs = pick(OBSERVERS); - injected.push({ hex, observer: obs.id, region: obs.iata, hash, snr: 1.0, rssi: -95 }); - } - - console.log(`Injecting ${injected.length} packets...`); - let injectFail = 0; - for (const pkt of injected) { - const r = await post('/api/packets', pkt); - if (r.status !== 200) injectFail++; - } - assert(injectFail === 0, `All ${injected.length} packets injected`); - console.log(`Injected: ${injected.length - injectFail} ok, ${injectFail} fail\n`); - - // ── HTML & Nav Tests ─────────────────────────────────────────────── - console.log('── HTML & Navigation ──'); - const { status: htmlStatus, text: html } = await getHtml('/'); - assert(htmlStatus === 200, 'index.html returns 200'); - assert(html.includes(''); - - const expectedLinks = ['#/packets', '#/map', '#/channels', '#/nodes', '#/traces', '#/observers']; - for (const link of expectedLinks) { - assert(html.includes(`href="${link}"`), `nav contains link to ${link}`); - } - - // ── JS File References ───────────────────────────────────────────── - console.log('\n── JS File References ──'); - const jsFiles = ['app.js', 'packets.js', 'map.js', 'channels.js', 'nodes.js', 'traces.js', 'observers.js']; - for (const jsFile of jsFiles) { - assert(html.includes(`src="${jsFile}`) || html.includes(`src="${jsFile}?`), `index.html references ${jsFile}`); - } - - // ── JS Syntax Validation ─────────────────────────────────────────── - console.log('\n── JS Syntax Validation ──'); - for (const jsFile of jsFiles) { - const jsPath = path.join(PROJECT_DIR, 'public', jsFile); - try { - const source = fs.readFileSync(jsPath, 'utf8'); - // Use the vm module's Script to check for syntax errors - new (require('vm')).Script(source, { filename: jsFile }); - assert(true, `${jsFile} has valid syntax`); - } catch (e) { - assert(false, `${jsFile} syntax error: ${e.message}`); - } - } - - // ── JS Files Fetchable from Server ───────────────────────────────── - console.log('\n── JS Files Served ──'); - for (const jsFile of jsFiles) { - const resp = await getHtml(`/${jsFile}`); - assert(resp.status === 200, `${jsFile} served with 200`); - assert(resp.text.length > 0, `${jsFile} is non-empty`); - } - - // ── API Data Shape Validation ────────────────────────────────────── - console.log('\n── API: /api/stats ──'); - const stats = (await get('/api/stats')).data; - assert(typeof stats.totalPackets === 'number', 'stats.totalPackets is number'); - assert(typeof stats.totalNodes === 'number', 'stats.totalNodes is number'); - assert(typeof stats.totalObservers === 'number', 'stats.totalObservers is number'); - assert(stats.totalPackets > 0, `stats.totalPackets > 0 (${stats.totalPackets})`); - - console.log('\n── API: /api/packets (packets page) ──'); - const pkts = (await get('/api/packets?limit=10')).data; - assert(typeof pkts.total === 'number', 'packets response has total'); - assert(Array.isArray(pkts.packets), 'packets response has packets array'); - assert(pkts.packets.length > 0, 'packets array non-empty'); - const pkt0 = pkts.packets[0]; - assert(pkt0.id !== undefined, 'packet has id'); - assert(pkt0.raw_hex !== undefined, 'packet has raw_hex'); - assert(pkt0.payload_type !== undefined, 'packet has payload_type'); - assert(pkt0.observer_id !== undefined, 'packet has observer_id'); - - // Packet detail (byte breakdown) - const detail = (await get(`/api/packets/${pkt0.id}`)).data; - assert(detail.packet !== undefined, 'packet detail has packet'); - assert(detail.breakdown !== undefined, 'packet detail has breakdown'); - assert(Array.isArray(detail.breakdown.ranges), 'breakdown has ranges array'); - - console.log('\n── API: /api/packets?groupByHash (map page) ──'); - const grouped = (await get('/api/packets?groupByHash=true&limit=10')).data; - assert(typeof grouped.total === 'number', 'groupByHash has total'); - assert(Array.isArray(grouped.packets), 'groupByHash has packets array'); - - console.log('\n── API: /api/channels (channels page) ──'); - const ch = (await get('/api/channels')).data; - assert(Array.isArray(ch.channels), 'channels response has channels array'); - if (ch.channels.length > 0) { - assert(ch.channels[0].hash !== undefined, 'channel has hash'); - assert(ch.channels[0].messageCount !== undefined, 'channel has messageCount'); - const chMsgs = (await get(`/api/channels/${ch.channels[0].hash}/messages`)).data; - assert(Array.isArray(chMsgs.messages || []), 'channel messages is array'); - } else { - console.log(' ⚠ No channels (synthetic packets are not decodable channel messages)'); - } - - console.log('\n── API: /api/nodes (nodes page) ──'); - const nodes = (await get('/api/nodes?limit=10')).data; - assert(typeof nodes.total === 'number', 'nodes has total'); - assert(Array.isArray(nodes.nodes), 'nodes has nodes array'); - assert(nodes.nodes.length > 0, 'nodes non-empty'); - const n0 = nodes.nodes[0]; - assert(n0.public_key !== undefined, 'node has public_key'); - assert(n0.name !== undefined, 'node has name'); - - // Node detail - const nd = (await get(`/api/nodes/${n0.public_key}`)).data; - assert(nd.node !== undefined, 'node detail has node'); - assert(nd.recentAdverts !== undefined, 'node detail has recentAdverts'); - - console.log('\n── API: /api/observers (observers page) ──'); - const obs = (await get('/api/observers')).data; - assert(Array.isArray(obs.observers), 'observers is array'); - assert(obs.observers.length > 0, 'observers non-empty'); - assert(obs.observers[0].id !== undefined, 'observer has id'); - assert(obs.observers[0].packet_count !== undefined, 'observer has packet_count'); - - console.log('\n── API: /api/traces (traces page) ──'); - // Use a known hash from injected packets - const knownHash = crypto.createHash('md5').update(injected[0].hex).digest('hex').slice(0, 16); - const traces = (await get(`/api/traces/${knownHash}`)).data; - assert(Array.isArray(traces.traces), 'traces is array'); - - // ── Summary ──────────────────────────────────────────────────────── - cleanup(); - - console.log('\n═══════════════════════════════════════'); - console.log(` PASSED: ${passed}`); - console.log(` FAILED: ${failed}`); - if (failures.length) { - console.log(' Failures:'); - failures.forEach(f => console.log(` - ${f}`)); - } - console.log('═══════════════════════════════════════'); - - process.exit(failed > 0 ? 1 : 0); -} - -main().catch(e => { - console.error('Fatal:', e); - process.exit(1); -});