From 1f0e4addde3ddcda2714df95abdc74cc4bb95e92 Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 9 Mar 2026 02:20:00 +0000 Subject: [PATCH] Improve stats charts and path hash reporting --- backend/src/api/routes.ts | 154 +++++++++++++++++++++++++++--- backend/src/db/index.ts | 23 +++-- frontend/src/pages/HealthPage.tsx | 37 ------- frontend/src/pages/StatsPage.tsx | 47 +++++++++ 4 files changed, 201 insertions(+), 60 deletions(-) diff --git a/backend/src/api/routes.ts b/backend/src/api/routes.ts index 682a9a0..60f8baf 100644 --- a/backend/src/api/routes.ts +++ b/backend/src/api/routes.ts @@ -320,8 +320,13 @@ function networkFilters(network?: string, observer?: string): NetworkFilters { packetsAlias: (alias: string) => { const prefix = `${alias}.`; const conditions: string[] = []; - if (networkParam) conditions.push(`${prefix}network = ${networkParam}`); - else conditions.push(`${prefix}network IS DISTINCT FROM 'test'`); + if (networkParam) { + conditions.push(`${prefix}network = ${networkParam}`); + conditions.push(`split_part(${prefix}topic, '/', 1) <> 'meshcore-test'`); + } else { + conditions.push(`${prefix}network IS DISTINCT FROM 'test'`); + conditions.push(`split_part(${prefix}topic, '/', 1) <> 'meshcore-test'`); + } if (observerParam) conditions.push(`LOWER(${prefix}rx_node_id) = LOWER(${observerParam})`); return conditions.length > 0 ? `AND ${conditions.join(' AND ')}` : ''; }, @@ -1144,13 +1149,27 @@ router.get('/stats/charts', async (req, res) => { const [ phResult, pdResult, rhResult, rdResult, ptResult, rpResult, hdResult, pcResult, sumResult, orSummaryResult, orSeriesResult, + pathHashWidthsResult, multibyteSummaryResult, ] = await Promise.all([ - // packets per hour — last 24h (all observed packet rows) + // packets per rolling hour — last 24h (sampled every 5 minutes) query(` - SELECT time_bucket('1 hour', time) AS hour, COUNT(*) AS count - FROM packets - WHERE time > NOW() - INTERVAL '24 hours' ${filters.packets} - GROUP BY hour ORDER BY hour + WITH buckets AS ( + SELECT generate_series( + date_trunc('minute', NOW() - INTERVAL '24 hours'), + date_trunc('minute', NOW()), + INTERVAL '5 minutes' + ) AS bucket + ) + SELECT b.bucket AS hour, COALESCE(c.count, 0) AS count + FROM buckets b + LEFT JOIN LATERAL ( + SELECT COUNT(*)::int AS count + FROM packets p + WHERE p.time > b.bucket - INTERVAL '1 hour' + AND p.time <= b.bucket + ${filters.packetsAlias('p')} + ) c ON TRUE + ORDER BY b.bucket `, filters.params), // packets per day — last 7d (all observed packet rows) query(` @@ -1159,12 +1178,26 @@ router.get('/stats/charts', async (req, res) => { WHERE time > NOW() - INTERVAL '7 days' ${filters.packets} GROUP BY day ORDER BY day `, filters.params), - // unique radios heard per hour — last 24h (distinct transmitting nodes) + // unique radios heard per rolling hour — last 24h (sampled every 5 minutes) query(` - SELECT time_bucket('1 hour', time) AS hour, COUNT(DISTINCT src_node_id) AS count - FROM packets - WHERE time > NOW() - INTERVAL '24 hours' AND src_node_id IS NOT NULL ${filters.packets} - GROUP BY hour ORDER BY hour + WITH buckets AS ( + SELECT generate_series( + date_trunc('minute', NOW() - INTERVAL '24 hours'), + date_trunc('minute', NOW()), + INTERVAL '5 minutes' + ) AS bucket + ) + SELECT b.bucket AS hour, COALESCE(c.count, 0) AS count + FROM buckets b + LEFT JOIN LATERAL ( + SELECT COUNT(DISTINCT p.src_node_id)::int AS count + FROM packets p + WHERE p.time > b.bucket - INTERVAL '1 hour' + AND p.time <= b.bucket + AND p.src_node_id IS NOT NULL + ${filters.packetsAlias('p')} + ) c ON TRUE + ORDER BY b.bucket `, filters.params), // unique radios heard per day — last 7d query(` @@ -1268,6 +1301,69 @@ router.get('/stats/charts', async (req, res) => { GROUP BY 1, 2 ORDER BY iata ASC, day ASC `, filters.params), + query<{ + hash_hex_len: string; + hop_count: string; + }>( + `SELECT length(h)::text AS hash_hex_len, COUNT(*)::text AS hop_count + FROM packets p + CROSS JOIN LATERAL unnest(p.path_hashes) AS h + WHERE p.time > NOW() - INTERVAL '24 hours' + ${filters.packetsAlias('p')} + GROUP BY 1`, + filters.params, + ), + query<{ + latest_multibyte_at: string | null; + multibyte_packets_24h: string; + fully_decoded_multibyte_24h: string; + }>( + `WITH multibyte AS ( + SELECT DISTINCT ON (p.packet_hash) + p.packet_hash, + p.time, + p.path_hashes + FROM packets p + WHERE p.time > NOW() - INTERVAL '24 hours' + AND p.path_hash_size_bytes > 1 + AND COALESCE(array_length(p.path_hashes, 1), 0) > 0 + ${filters.packetsAlias('p')} + ORDER BY p.packet_hash, + COALESCE(array_length(p.path_hashes, 1), 0) DESC, + COALESCE(p.hop_count, 0) DESC, + p.time DESC + ), + hop_matches AS ( + SELECT + m.packet_hash, + h.ord, + COUNT(DISTINCT n.node_id)::int AS match_count, + MIN(n.node_id) FILTER (WHERE n.node_id IS NOT NULL) AS matched_node_id + FROM multibyte m + CROSS JOIN LATERAL unnest(m.path_hashes) WITH ORDINALITY AS h(hash, ord) + LEFT JOIN nodes n + ON UPPER(LEFT(n.node_id, LENGTH(h.hash))) = UPPER(h.hash) + AND (n.role IS NULL OR n.role = 2) + AND n.lat IS NOT NULL + AND n.lon IS NOT NULL + GROUP BY 1, 2 + ), + packet_eval AS ( + SELECT + m.packet_hash, + BOOL_AND(hm.match_count = 1) AS all_unique, + COUNT(hm.ord)::int AS hop_count, + COUNT(DISTINCT hm.matched_node_id) FILTER (WHERE hm.matched_node_id IS NOT NULL)::int AS unique_nodes + FROM multibyte m + JOIN hop_matches hm ON hm.packet_hash = m.packet_hash + GROUP BY m.packet_hash + ) + SELECT + (SELECT MAX(time)::text FROM multibyte) AS latest_multibyte_at, + (SELECT COUNT(*)::text FROM multibyte) AS multibyte_packets_24h, + (SELECT COUNT(*)::text FROM packet_eval WHERE all_unique AND unique_nodes = hop_count) AS fully_decoded_multibyte_24h`, + filters.params, + ), ]); const peakRow = phResult.rows.reduce( @@ -1279,6 +1375,10 @@ router.get('/stats/charts', async (req, res) => { const d = new Date(ts); return `${d.getHours().toString().padStart(2, '0')}:00`; }; + const fmtHourMinute = (ts: Date | string) => { + const d = new Date(ts); + return `${d.getHours().toString().padStart(2, '0')}:${d.getMinutes().toString().padStart(2, '0')}`; + }; const fmtDay = (ts: Date | string) => { const d = new Date(ts); return d.toLocaleDateString('en-GB', { weekday: 'short', month: 'short', day: 'numeric' }); @@ -1315,10 +1415,30 @@ router.get('/stats/charts', async (req, res) => { }); } + const widthToBucket: Record = { + 2: 'one_byte', + 4: 'two_byte', + 6: 'three_byte', + }; + const pathHashStats = { + one_byte: 0, + two_byte: 0, + three_byte: 0, + }; + + for (const row of pathHashWidthsResult.rows) { + const width = Number(row.hash_hex_len ?? 0); + const bucket = widthToBucket[width]; + if (!bucket) continue; + pathHashStats[bucket] += Number(row.hop_count ?? 0); + } + + const multibyteRow = multibyteSummaryResult.rows[0]; + res.json({ - packetsPerHour: phResult.rows.map(r => ({ hour: fmtHour(r.hour), count: Number(r.count) })), + packetsPerHour: phResult.rows.map(r => ({ hour: fmtHourMinute(r.hour), count: Number(r.count) })), packetsPerDay: pdResult.rows.map(r => ({ day: fmtDay(r.day), count: Number(r.count) })), - radiosPerHour: rhResult.rows.map(r => ({ hour: fmtHour(r.hour), count: Number(r.count) })), + radiosPerHour: rhResult.rows.map(r => ({ hour: fmtHourMinute(r.hour), count: Number(r.count) })), radiosPerDay: rdResult.rows.map(r => ({ day: fmtDay(r.day), count: Number(r.count) })), packetTypes: ptResult.rows.map(r => ({ label: PAYLOAD_LABELS[Number(r.packet_type)] ?? `Type${r.packet_type}`, count: Number(r.count) })), repeatersPerDay: rpResult.rows.map(r => ({ hour: fmtDay(r.hour), count: Number(r.count ?? 0) })), @@ -1330,6 +1450,12 @@ router.get('/stats/charts', async (req, res) => { repeats: Number(r.repeats), })), observerRegions: Array.from(observerRegionsByIata.values()), + pathHashes: { + last24hHops: pathHashStats, + multibytePackets24h: Number(multibyteRow?.multibyte_packets_24h ?? 0), + fullyDecodedMultibyte24h: Number(multibyteRow?.fully_decoded_multibyte_24h ?? 0), + latestMultibyteAt: multibyteRow?.latest_multibyte_at ?? null, + }, summary: { totalPackets24h: Number(sumResult.rows[0].total_24h), totalPackets7d: Number(sumResult.rows[0].total_7d), diff --git a/backend/src/db/index.ts b/backend/src/db/index.ts index a619a89..fb8f8f1 100644 --- a/backend/src/db/index.ts +++ b/backend/src/db/index.ts @@ -48,13 +48,18 @@ function buildScopePlaceholders(startIndex: number, network?: string, observer?: function buildPacketScopeClause( placeholders: ScopePlaceholders, alias?: string, + network?: string, ): string { const prefix = alias ? `${alias}.` : ''; const conditions: string[] = []; if (placeholders.networkParam) { conditions.push(`${prefix}network = ${placeholders.networkParam}`); + if (network !== 'test') { + conditions.push(`split_part(${prefix}topic, '/', 1) <> 'meshcore-test'`); + } } else { conditions.push(`${prefix}network IS DISTINCT FROM 'test'`); + conditions.push(`split_part(${prefix}topic, '/', 1) <> 'meshcore-test'`); } if (placeholders.observerParam) { conditions.push(`LOWER(${prefix}rx_node_id) = LOWER(${placeholders.observerParam})`); @@ -227,7 +232,7 @@ export async function getRecentPackets(limit = 200, network?: string, observer?: WHERE p2.packet_hash = p.packet_hash AND p2.time > NOW() - INTERVAL '5 minutes' AND p2.rx_node_id IS NOT NULL - ${buildPacketScopeClause(scope, 'p2')} + ${buildPacketScopeClause(scope, 'p2', network)} ) AS observer_node_ids, ( SELECT COUNT(*)::int @@ -235,7 +240,7 @@ export async function getRecentPackets(limit = 200, network?: string, observer?: WHERE p2.packet_hash = p.packet_hash AND p2.time > NOW() - INTERVAL '5 minutes' AND COALESCE(p2.payload->>'direction', 'rx') <> 'tx' - ${buildPacketScopeClause(scope, 'p2')} + ${buildPacketScopeClause(scope, 'p2', network)} ) AS rx_count, ( SELECT COUNT(*)::int @@ -243,11 +248,11 @@ export async function getRecentPackets(limit = 200, network?: string, observer?: WHERE p2.packet_hash = p.packet_hash AND p2.time > NOW() - INTERVAL '5 minutes' AND COALESCE(p2.payload->>'direction', 'rx') = 'tx' - ${buildPacketScopeClause(scope, 'p2')} + ${buildPacketScopeClause(scope, 'p2', network)} ) AS tx_count FROM packets p WHERE p.time > NOW() - INTERVAL '5 minutes' - ${buildPacketScopeClause(scope, 'p')} + ${buildPacketScopeClause(scope, 'p', network)} ORDER BY p.packet_hash, CASE WHEN p.payload ? 'appData' THEN 1 ELSE 0 END DESC, CASE WHEN p.src_node_id IS NOT NULL THEN 1 ELSE 0 END DESC, @@ -273,7 +278,7 @@ export async function getRecentPacketEvents(limit = 200, network?: string, obser p.advert_count, p.path_hashes, p.path_hash_size_bytes FROM packets p WHERE p.time > NOW() - INTERVAL '24 hours' - ${buildPacketScopeClause(scope, 'p')} + ${buildPacketScopeClause(scope, 'p', network)} ORDER BY p.time DESC LIMIT $1`, params, @@ -296,7 +301,7 @@ export async function getLastNPackets(n: number, network?: string, observer?: st WHERE p2.packet_hash = p.packet_hash AND p2.time > NOW() - INTERVAL '24 hours' AND p2.rx_node_id IS NOT NULL - ${buildPacketScopeClause(scope, 'p2')} + ${buildPacketScopeClause(scope, 'p2', network)} ) AS observer_node_ids, ( SELECT COUNT(*)::int @@ -304,7 +309,7 @@ export async function getLastNPackets(n: number, network?: string, observer?: st WHERE p2.packet_hash = p.packet_hash AND p2.time > NOW() - INTERVAL '24 hours' AND COALESCE(p2.payload->>'direction', 'rx') <> 'tx' - ${buildPacketScopeClause(scope, 'p2')} + ${buildPacketScopeClause(scope, 'p2', network)} ) AS rx_count, ( SELECT COUNT(*)::int @@ -312,10 +317,10 @@ export async function getLastNPackets(n: number, network?: string, observer?: st WHERE p2.packet_hash = p.packet_hash AND p2.time > NOW() - INTERVAL '24 hours' AND COALESCE(p2.payload->>'direction', 'rx') = 'tx' - ${buildPacketScopeClause(scope, 'p2')} + ${buildPacketScopeClause(scope, 'p2', network)} ) AS tx_count FROM packets p - WHERE p.time > NOW() - INTERVAL '24 hours' ${buildPacketScopeClause(scope, 'p')} + WHERE p.time > NOW() - INTERVAL '24 hours' ${buildPacketScopeClause(scope, 'p', network)} ORDER BY p.packet_hash, CASE WHEN payload ? 'appData' THEN 1 ELSE 0 END DESC, CASE WHEN src_node_id IS NOT NULL THEN 1 ELSE 0 END DESC, diff --git a/frontend/src/pages/HealthPage.tsx b/frontend/src/pages/HealthPage.tsx index 02b5565..7222411 100644 --- a/frontend/src/pages/HealthPage.tsx +++ b/frontend/src/pages/HealthPage.tsx @@ -33,15 +33,6 @@ type HealthPayload = { stale_threshold_minutes: number; global_last_packet_at: string | null; }; - path_hashes: { - last_24h_hops: { - one_byte: number; - two_byte: number; - three_byte: number; - }; - multibyte_packets_24h: number; - latest_multibyte_at: string | null; - }; }; function timeAgo(ts: string | null): string { @@ -200,34 +191,6 @@ export const HealthPage: React.FC = () => {
Disk{fmtGb(data?.system.disk.used_gb)} / {fmtGb(data?.system.disk.total_gb)}
- -
-

Path Hashes

-
-
- {fmtInt(data?.path_hashes.last_24h_hops.one_byte)} - 1-byte Hops (24h) -
-
- {fmtInt(data?.path_hashes.last_24h_hops.two_byte)} - 2-byte Hops (24h) -
-
- {fmtInt(data?.path_hashes.last_24h_hops.three_byte)} - 3-byte Hops (24h) -
-
- {fmtInt(data?.path_hashes.multibyte_packets_24h)} - Multibyte Packets (24h) -
-
-
-
- Latest Multibyte Packet - {data?.path_hashes.latest_multibyte_at ? timeAgo(data.path_hashes.latest_multibyte_at) : 'not seen yet'} -
-
-
); diff --git a/frontend/src/pages/StatsPage.tsx b/frontend/src/pages/StatsPage.tsx index 41a31fd..d605c42 100644 --- a/frontend/src/pages/StatsPage.tsx +++ b/frontend/src/pages/StatsPage.tsx @@ -62,6 +62,16 @@ interface ChartData { lastPacketAt: string | null; series: { day: string; count: number }[]; }[]; + pathHashes: { + last24hHops: { + one_byte: number; + two_byte: number; + three_byte: number; + }; + multibytePackets24h: number; + fullyDecodedMultibyte24h: number; + latestMultibyteAt: string | null; + }; summary: { totalPackets24h: number; totalPackets7d: number; @@ -120,6 +130,7 @@ export const StatsPage: React.FC = () => { }, [site.networkFilter, site.observerId]); const fmt = (n: number) => n.toLocaleString(); + const pct = (num: number, den: number) => den > 0 ? `${Math.round((num / den) * 100)}%` : '0%'; const timeAgo = (ts: string | null) => { if (!ts) return 'never'; const diff = Math.max(0, Date.now() - Date.parse(ts)); @@ -244,6 +255,42 @@ export const StatsPage: React.FC = () => { )} +
+
+ Path hashes + observed hop widths across the last 24 hours +
+
+
+ {fmt(data.pathHashes.last24hHops.one_byte)} + 1-byte Hops (24h) +
+
+ {fmt(data.pathHashes.last24hHops.two_byte)} + 2-byte Hops (24h) +
+
+ {fmt(data.pathHashes.last24hHops.three_byte)} + 3-byte Hops (24h) +
+
+ {fmt(data.pathHashes.multibytePackets24h)} + Multibyte Packets (24h) +
+
+ {fmt(data.pathHashes.fullyDecodedMultibyte24h)} + Fully Decoded (24h) + {pct(data.pathHashes.fullyDecodedMultibyte24h, data.pathHashes.multibytePackets24h)} of multibyte packets +
+
+
+
+ Latest Multibyte Packet + {data.pathHashes.latestMultibyteAt ? timeAgo(data.pathHashes.latestMultibyteAt) : 'not seen yet'} +
+
+
+ {/* ── Packets over time ────────────────────────────────────────── */}