From 4a2d69bc583be4663e801fa9e8d0f2311cc3dd9d Mon Sep 17 00:00:00 2001 From: gadgethd <111318106+gadgethd@users.noreply.github.com> Date: Fri, 7 Aug 2026 15:47:11 +0000 Subject: [PATCH] fix(stats): packet counts = total observations (all rows, duplicates included) Per Ben's spec the "Observed packets" totals must count EVERY packet received by EVERY observer - duplicate receptions of the same packet by multiple stations each count. Reverts the COUNT(DISTINCT packet_hash) semantics from 8dff7d7 and additionally drops the privacy visibility filter from the totals so they match the chart rollup path (which has never been privacy-filtered - aggregateScope filters by network only). Changes: - packetsDay, totalPackets24h, totalPackets7d: COUNT(*) over all rows (network/test-topic scoped, rx_node_id present), no visibility filter - channelTraffic: COUNT(*) all rows (percentages stay consistent with the all-rows total denominator) - networkFilters(): new includePrivacy option (default true) so total queries can opt out of visibility conditions - legacy/observer-scoped chart series back to COUNT(*) (row semantics everywhere; transportCodes alias fix from 40be809 retained) - tests: includePrivacy coverage added Verified live: 24h total ~99k rows (vs 25k privacy-filtered, vs 5.8k distinct) - consistent with the hourly chart series which already counted all rows. --- backend/src/api/routes/stats.ts | 2 +- backend/src/api/utils/networkFilters.test.ts | 9 +++++ backend/src/api/utils/networkFilters.ts | 10 ++++-- backend/src/stats/statsRepository.test.ts | 4 +-- backend/src/stats/statsRepository.ts | 37 +++++++++++--------- 5 files changed, 40 insertions(+), 22 deletions(-) diff --git a/backend/src/api/routes/stats.ts b/backend/src/api/routes/stats.ts index a8d07c4..a5672ab 100644 --- a/backend/src/api/routes/stats.ts +++ b/backend/src/api/routes/stats.ts @@ -37,7 +37,7 @@ type StatsRouteDeps = { chartsInflight: Map>; expensiveLimiter: ReturnType; statsChartsLimiter: ReturnType; - networkFilters: (network?: string, observer?: string) => NetworkFilters; + networkFilters: (network?: string, observer?: string, opts?: { includePrivacy?: boolean }) => NetworkFilters; query: QueryFn; getPublicVisibilityGeneration: () => Promise; maskDecodedPathNodes: MaskDecodedPathNodesFn; diff --git a/backend/src/api/utils/networkFilters.test.ts b/backend/src/api/utils/networkFilters.test.ts index 5548567..dae3080 100644 --- a/backend/src/api/utils/networkFilters.test.ts +++ b/backend/src/api/utils/networkFilters.test.ts @@ -28,6 +28,15 @@ test('public packet scopes keep legacy test-topic rows out', () => { assert.doesNotMatch(filters.packets, /private_node\.name LIKE '%🚫%'/); }); +test('includePrivacy:false omits visibility and private-prefix conditions', () => { + const filters = networkFilters('ukmesh', undefined, { includePrivacy: false }); + assert.match(filters.packets, /network = ANY\(\$1\)/); + assert.doesNotMatch(filters.packets, /visibility_ok IS TRUE/); + assert.doesNotMatch(filters.packets, /private_node_prefixes private_prefix/); + const defaulted = networkFilters('ukmesh'); + assert.match(defaulted.packets, /visibility_ok IS TRUE/); +}); + test('owned public visibility scopes carry observer filtering', () => { const filters = publicNetworkFilters({ access: 'public', diff --git a/backend/src/api/utils/networkFilters.ts b/backend/src/api/utils/networkFilters.ts index 796e27a..ece1b3d 100644 --- a/backend/src/api/utils/networkFilters.ts +++ b/backend/src/api/utils/networkFilters.ts @@ -60,7 +60,11 @@ function excludesLegacyTestTopic(prefix: string): string { return `COALESCE(NULLIF(${prefix}topic_prefix, ''), split_part(${prefix}topic, '/', 1)) <> 'meshcore-test'`; } -export function networkFilters(network?: string, observer?: string): NetworkFilters { +export function networkFilters( + network?: string, + observer?: string, + opts?: { includePrivacy?: boolean }, +): NetworkFilters { const params: unknown[] = []; let networkParam: string | null = null; let networkIsMulti = false; @@ -102,7 +106,9 @@ export function networkFilters(network?: string, observer?: string): NetworkFilt OR meshcore_canonical_node_id(rx_node_id) = meshcore_canonical_node_id(${observerParam}))`, ); } - packetConditions.push(...publicPacketPrivacyConditions('')); + if (opts?.includePrivacy !== false) { + packetConditions.push(...publicPacketPrivacyConditions('')); + } const nodeConditions = (alias?: string) => { const prefix = alias ? `${alias}.` : ''; diff --git a/backend/src/stats/statsRepository.test.ts b/backend/src/stats/statsRepository.test.ts index e8ed525..9c05852 100644 --- a/backend/src/stats/statsRepository.test.ts +++ b/backend/src/stats/statsRepository.test.ts @@ -129,7 +129,7 @@ test('canonical charts coalesce six high-volume dimensions into one maintained a assert.match(aggregateSql, /SELECT \* FROM rollup_24h\s+UNION ALL\s+SELECT \* FROM raw_24h/); assert.equal( calls.filter((sql) => - sql.includes("time_bucket('1 hour', p.time) AS bucket, COUNT(DISTINCT p.packet_hash)::int AS count")).length, + sql.includes("time_bucket('1 hour', p.time) AS bucket, COUNT(*)::int AS count")).length, 0, ); assert.deepEqual(result.ptResult.rows, [{ packet_type: 4, count: '3' }]); @@ -139,7 +139,7 @@ test('canonical charts coalesce six high-volume dimensions into one maintained a await repository.fetchChartsData('ukmesh', 'A'.repeat(64)); assert.equal(calls.filter((sql) => sql.includes('FROM packet_hourly_stats')).length, 0); assert.ok(calls.some((sql) => - sql.includes("time_bucket('1 hour', p.time) AS bucket, COUNT(DISTINCT p.packet_hash)::int AS count"))); + sql.includes("time_bucket('1 hour', p.time) AS bucket, COUNT(*)::int AS count"))); }); test('map summary uses the same coordinate, role, and 14-day freshness rules as the map', async () => { diff --git a/backend/src/stats/statsRepository.ts b/backend/src/stats/statsRepository.ts index 1e703a5..6b789e1 100644 --- a/backend/src/stats/statsRepository.ts +++ b/backend/src/stats/statsRepository.ts @@ -17,7 +17,7 @@ type QueryFn = ( ) => Promise<{ rows: T[] }>; type StatsRepositoryDeps = { - networkFilters: (network?: string, observer?: string) => NetworkFilters; + networkFilters: (network?: string, observer?: string, opts?: { includePrivacy?: boolean }) => NetworkFilters; query: QueryFn; aggregateReadsEnabled?: boolean; aggregateShadowEnabled?: boolean; @@ -181,11 +181,10 @@ export function createStatsRepository(deps: StatsRepositoryDeps) { } async function fetchChannelTraffic(network: string | undefined, observer: string | undefined) { - const filters = networkFilters(network, observer); + const filters = networkFilters(network, observer, { includePrivacy: false }); return query<{ channel: string; count: string; total_count: string }>(` WITH decoded_group_packets AS ( SELECT - p.packet_hash AS packet_hash, COALESCE(p.payload->>'_summary', '') AS summary, NULLIF(TRIM((regexp_match(COALESCE(p.payload->>'_summary', ''), '^\\[([^\\]]+)\\]'))[1]), '') AS parsed_channel FROM packets p @@ -195,7 +194,6 @@ export function createStatsRepository(deps: StatsRepositoryDeps) { ), group_packets AS ( SELECT - packet_hash, COALESCE( CASE WHEN parsed_channel IS NOT NULL @@ -210,7 +208,7 @@ export function createStatsRepository(deps: StatsRepositoryDeps) { FROM decoded_group_packets ), channel_counts AS ( - SELECT channel, COUNT(DISTINCT packet_hash)::text AS count + SELECT channel, COUNT(*)::text AS count FROM group_packets GROUP BY channel ) @@ -681,6 +679,7 @@ export function createStatsRepository(deps: StatsRepositoryDeps) { async function fetchChartsData(network: string | undefined, observer: string | undefined) { const filters = networkFilters(network, observer); + const totalFilters = networkFilters(network, observer, { includePrivacy: false }); const aggregateAsOf = new Date(); const aggregatePartsPromise = aggregateReadsEnabled && !observer ? fetchAggregateChartParts(network, aggregateAsOf) @@ -709,7 +708,7 @@ export function createStatsRepository(deps: StatsRepositoryDeps) { ) AS bucket ), counts AS ( - SELECT time_bucket('1 hour', p.time) AS bucket, COUNT(DISTINCT p.packet_hash)::int AS count + SELECT time_bucket('1 hour', p.time) AS bucket, COUNT(*)::int AS count FROM packets p WHERE p.time > NOW() - INTERVAL '24 hours' ${filters.packetsAlias('p')} @@ -721,7 +720,7 @@ export function createStatsRepository(deps: StatsRepositoryDeps) { ORDER BY b.bucket `, filters.params)), aggregateRows('packetsPerDay', () => query(` - SELECT time_bucket('1 day', time) AS day, COUNT(DISTINCT packet_hash) AS count + SELECT time_bucket('1 day', time) AS day, COUNT(*) AS count FROM packets WHERE time > NOW() - INTERVAL '7 days' ${filters.packets} GROUP BY day ORDER BY day @@ -756,13 +755,13 @@ export function createStatsRepository(deps: StatsRepositoryDeps) { GROUP BY day ORDER BY day `, filters.params), aggregateRows('packetTypes', () => query(` - SELECT packet_type, COUNT(DISTINCT packet_hash) AS count + SELECT packet_type, COUNT(*) AS count FROM packets WHERE time > NOW() - INTERVAL '24 hours' ${filters.packets} GROUP BY packet_type ORDER BY count DESC `, filters.params)), aggregateRows('hopDistribution', () => query(` - SELECT hop_count AS hops, COUNT(DISTINCT packet_hash) AS count + SELECT hop_count AS hops, COUNT(*) AS count FROM packets WHERE time > NOW() - INTERVAL '7 days' AND hop_count IS NOT NULL @@ -791,8 +790,8 @@ export function createStatsRepository(deps: StatsRepositoryDeps) { `, filters.params), query(` SELECT - (SELECT COUNT(DISTINCT packet_hash) FROM packets WHERE time > NOW() - INTERVAL '24 hours' ${filters.packets}) AS total_24h, - (SELECT COUNT(DISTINCT packet_hash) FROM packets WHERE time > NOW() - INTERVAL '7 days' ${filters.packets}) AS total_7d, + (SELECT COUNT(*) FROM packets WHERE time > NOW() - INTERVAL '24 hours' ${totalFilters.packets}) AS total_24h, + (SELECT COUNT(*) FROM packets WHERE time > NOW() - INTERVAL '7 days' ${totalFilters.packets}) AS total_7d, (SELECT COUNT(DISTINCT meshcore_canonical_node_id(src_node_id)) FROM packets WHERE time > NOW() - INTERVAL '24 hours' @@ -1063,7 +1062,7 @@ export function createStatsRepository(deps: StatsRepositoryDeps) { filters.params, ), aggregateRows('routeTypes', () => query<{ route_type: string; count: string }>( - `SELECT COALESCE(p.route_type::text, 'Unknown') AS route_type, COUNT(DISTINCT p.packet_hash)::text AS count + `SELECT COALESCE(p.route_type::text, 'Unknown') AS route_type, COUNT(*)::text AS count FROM packets p WHERE p.time > NOW() - INTERVAL '24 hours' ${filters.packetsAlias('p')} @@ -1075,10 +1074,10 @@ export function createStatsRepository(deps: StatsRepositoryDeps) { `SELECT NULLIF(TRIM(p.transport_codes), '') AS transport_code, NULLIF(TRIM(p.region_scope), '') AS region_scope, - COUNT(DISTINCT p.packet_hash)::text AS count - FROM packets p - WHERE p.time > NOW() - INTERVAL '24 hours' - AND NULLIF(TRIM(p.transport_codes), '') IS NOT NULL + COUNT(*)::text AS count + FROM packets p + WHERE p.time > NOW() - INTERVAL '24 hours' + AND NULLIF(TRIM(p.transport_codes), '') IS NOT NULL ${filters.packetsAlias('p')} GROUP BY 1, 2 ORDER BY COUNT(*) DESC, region_scope ASC, transport_code ASC @@ -1254,6 +1253,7 @@ export function createStatsRepository(deps: StatsRepositoryDeps) { async function fetchStatsSummary(network: string | undefined, observer: string | undefined) { const filters = networkFilters(network, observer); + const totalFilters = networkFilters(network, observer, { includePrivacy: false }); const longestHopResult = () => { if (observer) { return query(`SELECT hop_count AS count, packet_hash AS hash @@ -1305,7 +1305,10 @@ export function createStatsRepository(deps: StatsRepositoryDeps) { AND rx_node_id NOT IN (SELECT rx_node_id FROM test_active) ${filters.packets} `, filters.params), - query(`SELECT COUNT(DISTINCT packet_hash) AS count FROM packets WHERE time > NOW() - INTERVAL '24 hours' ${filters.packets}`, filters.params), + query(`SELECT COUNT(*) AS count FROM packets + WHERE time > NOW() - INTERVAL '24 hours' + AND rx_node_id IS NOT NULL AND rx_node_id <> '' + ${totalFilters.packets}`, totalFilters.params), query(`SELECT COUNT(*) AS count FROM node_identity_nodes nodes WHERE ${publicMapBasePredicate('nodes')} AND ${nodeEffectiveLastSeenSql('nodes')}