From 9f00151ae1db9dbef26b80e2917e34bc43d5d8d3 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 17 Mar 2026 22:56:24 +0000 Subject: [PATCH] Perf & UX improvements: stats cache, CPU fix, observer names on feed - Pre-compute /api/stats/charts every 30 min with inflight dedup; cache served instantly to all visitors - Fix CPU usage stat: systemStats() was called twice per health request, giving near-zero second reading - De-duplicate observer region packet counts by packet_hash (was counting per observer row) - Show repeater names on feed instead of rx counter / "heard by N" - Observer names span full row width on /feed packet list Co-Authored-By: Claude Sonnet 4.6 --- backend/src/api/routes.ts | 173 ++++++++++++++--------- backend/src/health/status.ts | 11 +- frontend/src/components/PacketFeed.tsx | 14 +- frontend/src/pages/ukmesh/UKFeedPage.tsx | 10 +- frontend/src/styles/globals.css | 1 + 5 files changed, 135 insertions(+), 74 deletions(-) diff --git a/backend/src/api/routes.ts b/backend/src/api/routes.ts index 8147ea0..64b3e39 100644 --- a/backend/src/api/routes.ts +++ b/backend/src/api/routes.ts @@ -21,10 +21,13 @@ const STATS_CACHE_TTL_MS = 15_000; // stats don't need sub-second fres const INFERRED_NODES_CACHE_TTL_MS = 60_000; // 7-day packet scan, changes slowly const PATH_HISTORY_CACHE_TTL_MS = 60_000; // history cache is rebuilt by worker, not real-time const COVERAGE_CACHE_TTL_MS = 30_000; // geometry changes only on coverage rebuild +const CHARTS_CACHE_TTL_MS = 30 * 60_000; // 30 min — background refresh keeps it warm const statsCache = new Map(); const inferredNodesCache = new Map(); const pathHistoryCache = new Map(); const coverageCache = new Map(); +const chartsCache = new Map(); +const chartsInflight = new Map>(); const OWNER_SESSION_TTL_MS = 30 * 24 * 60 * 60 * 1000; const MQTT_USERNAME_MAX_LEN = 128; const MQTT_PASSWORD_MAX_LEN = 128; @@ -2179,25 +2182,20 @@ router.post('/telemetry/frontend-error', async (req, res) => { } }); -// GET /api/stats/charts -router.get('/stats/charts', STATS_CHARTS_LIMITER, async (req, res) => { - try { - const requestedNetwork = resolveRequestNetwork(req.query['network'], req.headers); - const network = requestedNetwork === 'all' ? undefined : requestedNetwork; - const observer = normalizeObserverQuery(req.query['observer']); - const filters = networkFilters(network, observer); +async function computeChartsData(network: string | undefined, observer: string | undefined): Promise { + const filters = networkFilters(network, observer); - const PAYLOAD_LABELS: Record = { - 0: 'Request', 1: 'Response', 2: 'DM', 3: 'Ack', - 4: 'Advert', 5: 'GroupText', 6: 'GroupData', - 7: 'AnonReq', 8: 'Path', 9: 'Trace', 11: 'Control', - }; + const PAYLOAD_LABELS: Record = { + 0: 'Request', 1: 'Response', 2: 'DM', 3: 'Ack', + 4: 'Advert', 5: 'GroupText', 6: 'GroupData', + 7: 'AnonReq', 8: 'Path', 9: 'Trace', 11: 'Control', + }; - const [ - phResult, pdResult, rhResult, rdResult, - ptResult, rpResult, hdResult, pcResult, sumResult, orSummaryResult, orSeriesResult, - pathHashWidthsResult, multibyteSummaryResult, - ] = await Promise.all([ + const [ + phResult, pdResult, rhResult, rdResult, + ptResult, rpResult, hdResult, pcResult, sumResult, orSummaryResult, orSeriesResult, + pathHashWidthsResult, multibyteSummaryResult, + ] = await Promise.all([ // packets per rolling hour — last 24h (sampled every 5 minutes) query(` WITH buckets AS ( @@ -2316,12 +2314,12 @@ router.get('/stats/charts', STATS_CHARTS_LIMITER, async (req, res) => { (SELECT COUNT(*) FROM nodes n WHERE (n.role IS NULL OR n.role = 2) AND n.last_seen > NOW() - INTERVAL '7 days' ${filters.nodesAlias('n')}) AS active_repeaters, (SELECT COUNT(*) FROM nodes n WHERE (n.role IS NULL OR n.role = 2) AND n.last_seen <= NOW() - INTERVAL '7 days' AND n.last_seen > NOW() - INTERVAL '14 days' ${filters.nodesAlias('n')}) AS stale_repeaters `, filters.params), - // observer regions summary — last 7d + // observer regions summary — last 7d (de-duped per region by packet_hash) query(` SELECT COALESCE(NULLIF(TRIM(UPPER(n.iata)), ''), 'UNK') AS iata, - COUNT(*) FILTER (WHERE p.time > NOW() - INTERVAL '24 hours') AS packets_24h, - COUNT(*) AS packets_7d, + COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.time > NOW() - INTERVAL '24 hours') AS packets_24h, + COUNT(DISTINCT p.packet_hash) AS packets_7d, COUNT(DISTINCT LOWER(p.rx_node_id)) FILTER (WHERE n.last_seen > NOW() - INTERVAL '1 minute') AS active_observers, COUNT(DISTINCT LOWER(p.rx_node_id)) AS observers, MAX(p.time)::text AS last_packet_at @@ -2335,12 +2333,12 @@ router.get('/stats/charts', STATS_CHARTS_LIMITER, async (req, res) => { GROUP BY 1 ORDER BY packets_7d DESC, iata ASC `, filters.params), - // observer regions sparkline series — last 7d + // observer regions sparkline series — last 7d (de-duped per region by packet_hash) query(` SELECT COALESCE(NULLIF(TRIM(UPPER(n.iata)), ''), 'UNK') AS iata, time_bucket('1 day', p.time) AS day, - COUNT(*) AS count + COUNT(DISTINCT p.packet_hash) AS count FROM packets p LEFT JOIN nodes n ON LOWER(n.node_id) = LOWER(p.rx_node_id) WHERE p.time > NOW() - INTERVAL '7 days' @@ -2560,48 +2558,95 @@ router.get('/stats/charts', STATS_CHARTS_LIMITER, async (req, res) => { const latestFullyDecodedNodes = maskDecodedPathNodes(multibyteRow?.latest_fully_decoded_nodes); const longestFullyDecodedNodes = maskDecodedPathNodes(multibyteRow?.longest_fully_decoded_nodes); - res.json({ - packetsPerHour: phResult.rows.map(r => ({ hour: fmtHourMinute(r.hour), count: Number(r.count) })), - packetsPerDay: pdResult.rows.map(r => ({ day: fmtDay(r.day), count: Number(r.count) })), - radiosPerHour: rhResult.rows.map(r => ({ hour: fmtHourMinute(r.hour), count: Number(r.count) })), - radiosPerDay: rdResult.rows.map(r => ({ day: fmtDay(r.day), count: Number(r.count) })), - packetTypes: ptResult.rows.map(r => ({ label: PAYLOAD_LABELS[Number(r.packet_type)] ?? `Type${r.packet_type}`, count: Number(r.count) })), - repeatersPerDay: rpResult.rows.map(r => ({ hour: fmtDay(r.hour), count: Number(r.count ?? 0) })), - hopDistribution: hdResult.rows.map(r => ({ hops: Number(r.hops), count: Number(r.count) })), - // Back-compat for cached older frontend bundles that still read topChatters. - topChatters: [], - prefixCollisions: pcResult.rows.map(r => ({ - prefix: String(r.prefix ?? '').toUpperCase(), - repeats: Number(r.repeats), - })), - observerRegions: Array.from(observerRegionsByIata.values()), - pathHashes: { - last24hHops: pathHashStats, - multibytePackets24h: Number(multibyteRow?.multibyte_packets_24h ?? 0), - fullyDecodedMultibyte24h: Number(multibyteRow?.fully_decoded_multibyte_24h ?? 0), - latestMultibyteAt: multibyteRow?.latest_multibyte_at ?? null, - latestMultibyteHash: multibyteRow?.latest_multibyte_hash ?? null, - latestFullyDecodedAt: multibyteRow?.latest_fully_decoded_at ?? null, - latestFullyDecodedHash: multibyteRow?.latest_fully_decoded_hash ?? null, - latestFullyDecodedHops: Number(multibyteRow?.latest_fully_decoded_hops ?? 0) || null, - latestFullyDecodedPath: multibyteRow?.latest_fully_decoded_path ?? null, - latestFullyDecodedNodes, - longestFullyDecodedAt: multibyteRow?.longest_fully_decoded_at ?? null, - longestFullyDecodedHash: multibyteRow?.longest_fully_decoded_hash ?? null, - longestFullyDecodedHops: Number(multibyteRow?.longest_fully_decoded_hops ?? 0) || null, - longestFullyDecodedPath: multibyteRow?.longest_fully_decoded_path ?? null, - longestFullyDecodedNodes, - }, - summary: { - totalPackets24h: Number(sumResult.rows[0].total_24h), - totalPackets7d: Number(sumResult.rows[0].total_7d), - uniqueRadios24h: Number(sumResult.rows[0].unique_radios_24h), - activeRepeaters: Number(sumResult.rows[0].active_repeaters ?? 0), - staleRepeaters: Number(sumResult.rows[0].stale_repeaters ?? 0), - peakHour: peakRow ? fmtHour(peakRow.hour) : null, - peakHourCount: peakRow ? Number(peakRow.count) : 0, - }, - }); + return { + packetsPerHour: phResult.rows.map(r => ({ hour: fmtHourMinute(r.hour), count: Number(r.count) })), + packetsPerDay: pdResult.rows.map(r => ({ day: fmtDay(r.day), count: Number(r.count) })), + radiosPerHour: rhResult.rows.map(r => ({ hour: fmtHourMinute(r.hour), count: Number(r.count) })), + radiosPerDay: rdResult.rows.map(r => ({ day: fmtDay(r.day), count: Number(r.count) })), + packetTypes: ptResult.rows.map(r => ({ label: PAYLOAD_LABELS[Number(r.packet_type)] ?? `Type${r.packet_type}`, count: Number(r.count) })), + repeatersPerDay: rpResult.rows.map(r => ({ hour: fmtDay(r.hour), count: Number(r.count ?? 0) })), + hopDistribution: hdResult.rows.map(r => ({ hops: Number(r.hops), count: Number(r.count) })), + topChatters: [], + prefixCollisions: pcResult.rows.map(r => ({ + prefix: String(r.prefix ?? '').toUpperCase(), + repeats: Number(r.repeats), + })), + observerRegions: Array.from(observerRegionsByIata.values()), + pathHashes: { + last24hHops: pathHashStats, + multibytePackets24h: Number(multibyteRow?.multibyte_packets_24h ?? 0), + fullyDecodedMultibyte24h: Number(multibyteRow?.fully_decoded_multibyte_24h ?? 0), + latestMultibyteAt: multibyteRow?.latest_multibyte_at ?? null, + latestMultibyteHash: multibyteRow?.latest_multibyte_hash ?? null, + latestFullyDecodedAt: multibyteRow?.latest_fully_decoded_at ?? null, + latestFullyDecodedHash: multibyteRow?.latest_fully_decoded_hash ?? null, + latestFullyDecodedHops: Number(multibyteRow?.latest_fully_decoded_hops ?? 0) || null, + latestFullyDecodedPath: multibyteRow?.latest_fully_decoded_path ?? null, + latestFullyDecodedNodes, + longestFullyDecodedAt: multibyteRow?.longest_fully_decoded_at ?? null, + longestFullyDecodedHash: multibyteRow?.longest_fully_decoded_hash ?? null, + longestFullyDecodedHops: Number(multibyteRow?.longest_fully_decoded_hops ?? 0) || null, + longestFullyDecodedPath: multibyteRow?.longest_fully_decoded_path ?? null, + longestFullyDecodedNodes, + }, + summary: { + totalPackets24h: Number(sumResult.rows[0].total_24h), + totalPackets7d: Number(sumResult.rows[0].total_7d), + uniqueRadios24h: Number(sumResult.rows[0].unique_radios_24h), + activeRepeaters: Number(sumResult.rows[0].active_repeaters ?? 0), + staleRepeaters: Number(sumResult.rows[0].stale_repeaters ?? 0), + peakHour: peakRow ? fmtHour(peakRow.hour) : null, + peakHourCount: peakRow ? Number(peakRow.count) : 0, + }, + }; +} + +async function getCachedChartsData(network: string | undefined, observer: string | undefined): Promise { + const key = `${network ?? 'all'}:${observer ?? ''}`; + const cached = chartsCache.get(key); + if (cached && Date.now() - cached.ts < CHARTS_CACHE_TTL_MS) return cached.data; + + // Deduplicate concurrent requests for the same key + const inflight = chartsInflight.get(key); + if (inflight) return inflight; + + const promise = computeChartsData(network, observer).then((data) => { + chartsCache.set(key, { ts: Date.now(), data }); + chartsInflight.delete(key); + return data; + }).catch((err) => { + chartsInflight.delete(key); + throw err; + }); + + chartsInflight.set(key, promise); + return promise; +} + +// Pre-warm charts cache for common networks on startup, then refresh every 30 minutes. +// This ensures the first visitor always gets a cached response. +{ + const CHARTS_WARMUP_NETWORKS = (process.env['WARMUP_NETWORKS'] ?? 'teesside,ukmesh') + .split(',').map((s: string) => s.trim()).filter(Boolean); + + const warmCharts = () => { + for (const net of CHARTS_WARMUP_NETWORKS) { + getCachedChartsData(net, undefined).catch(() => { /* best-effort */ }); + } + }; + + // Delay slightly so the DB pool is ready + setTimeout(warmCharts, 5_000); + setInterval(warmCharts, CHARTS_CACHE_TTL_MS); +} + +// GET /api/stats/charts +router.get('/stats/charts', STATS_CHARTS_LIMITER, async (req, res) => { + try { + const requestedNetwork = resolveRequestNetwork(req.query['network'], req.headers); + const network = requestedNetwork === 'all' ? undefined : requestedNetwork; + const observer = normalizeObserverQuery(req.query['observer']); + res.json(await getCachedChartsData(network, observer)); } catch (err) { console.error('[api] GET /stats/charts', (err as Error).message); res.status(500).json({ error: 'Internal server error' }); diff --git a/backend/src/health/status.ts b/backend/src/health/status.ts index 1ebf45a..1623976 100644 --- a/backend/src/health/status.ts +++ b/backend/src/health/status.ts @@ -106,7 +106,7 @@ function systemStats() { }; } -async function currentWorkers(): Promise { +async function currentWorkers(precomputedStats?: ReturnType): Promise { const r = redis(); const [ viewshedDepth, @@ -144,7 +144,7 @@ async function currentWorkers(): Promise { ), ]); - const stats = systemStats(); + const stats = precomputedStats ?? systemStats(); const load = stats.cpu.load_1m; const memPct = stats.memory.used_pct; const diskPct = stats.disk.used_pct; @@ -241,8 +241,11 @@ export async function captureWorkerHealthSnapshot(): Promise { } export async function getWorkerHealthOverview() { + // Compute system stats once — cpuUsagePct() diffs against lastCpuSample, + // so calling it twice in one request gives a garbage near-zero second reading. + const sysStats = systemStats(); const [workers, history, errors1h, ingest, pathHashWidths, multibyteSummary] = await Promise.all([ - currentWorkers(), + currentWorkers(sysStats), query<{ ts: string; worker_name: string; @@ -367,7 +370,7 @@ export async function getWorkerHealthOverview() { const multibyteRow = multibyteSummary.rows[0]; return { - system: systemStats(), + system: sysStats, workers, history: history.rows, frontend_errors_1h: Number(errors1h.rows[0]?.count ?? 0), diff --git a/frontend/src/components/PacketFeed.tsx b/frontend/src/components/PacketFeed.tsx index eadaa4b..97e0150 100644 --- a/frontend/src/components/PacketFeed.tsx +++ b/frontend/src/components/PacketFeed.tsx @@ -69,6 +69,16 @@ export const PacketFeed: React.FC = React.memo(({ packets, nodes, mqttObs ? (p.advertCount === 1 ? 'NEW' : `${p.advertCount}`) : undefined; + // Resolve observer IDs to names for display + const MAX_OBSERVER_NAMES = 3; + const observerNames = p.observerIds.map((id) => { + const node = nodes.get(id); + return node?.name ?? node?.iata ?? id.slice(0, 6); + }); + const observerDisplay = observerNames.length > MAX_OBSERVER_NAMES + ? [...observerNames.slice(0, MAX_OBSERVER_NAMES), `+${observerNames.length - MAX_OBSERVER_NAMES}`] + : observerNames; + const isPinned = pinnedPacketId === p.id; return ( @@ -99,7 +109,9 @@ export const PacketFeed: React.FC = React.memo(({ packets, nodes, mqttObs ↑{p.hopCount} )} - {p.rxCount > 0 && {p.rxCount}rx} + {observerDisplay.length > 0 && ( + {observerDisplay.join(' · ')} + )} {p.txCount > 0 && {p.txCount}tx} {isPinned && ●} diff --git a/frontend/src/pages/ukmesh/UKFeedPage.tsx b/frontend/src/pages/ukmesh/UKFeedPage.tsx index 5402791..f766bd2 100644 --- a/frontend/src/pages/ukmesh/UKFeedPage.tsx +++ b/frontend/src/pages/ukmesh/UKFeedPage.tsx @@ -303,11 +303,11 @@ export const UKFeedPage: React.FC = () => { if (selectedIata === 'all') return true; return packetObserverIatas(packet, nodeMap).includes(selectedIata); }); - const observerDisplay = observerIds.length > 1 - ? `heard by ${observerIds.length}` - : observerIds.length === 1 - ? (nodeMap.get(observerIds[0])?.name ?? shortNode(observerIds[0])) - : 'unknown'; + const names = observerIds.map((id) => { + const node = nodeMap.get(id); + return node?.name ?? node?.iata ?? shortNode(id); + }); + const observerDisplay = names.length === 0 ? 'unknown' : names.join(' · '); return (
diff --git a/frontend/src/styles/globals.css b/frontend/src/styles/globals.css index 266a440..c2c5acf 100644 --- a/frontend/src/styles/globals.css +++ b/frontend/src/styles/globals.css @@ -2330,6 +2330,7 @@ html, body, #root { color: var(--accent); font-weight: 500; font-size: 0.85rem; + grid-column: 1 / -1; } .uk-feed-packet-row__summary {