dandri/ukmesh
1
0
Fork 0
mirror of https://github.com/gadgethd/ukmesh.git synced 2026-09-02 19:23:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(stats): count distinct packets (not observation rows) in all packet-count stats

Browse Source 
The packets table stores one row per observer per packet, so COUNT(*)
overcounted by the average observation multiplicity (~4.3x public, ~9.5x
overall). A packet heard by 5 stations counted 5 times.

Converted to COUNT(DISTINCT packet_hash) on every surface that presents a
"packets" number:
- /api/stats summary packetsDay (app + home dashboard card)
- charts summary totalPackets24h / totalPackets7d (StatsPage cards)
- legacy/observer-scoped chart series: packetsPerHour, packetsPerDay,
  packetTypes, hopDistribution, routeTypes, transportCodes
- channelTraffic channel counts (keeps allPct consistent with the
  distinct denominator)

The canonical rollup path (packet_hourly_stats, incrementally maintained)
remains row-based - it cannot maintain distinct counts incrementally and
stays as the trend-shape source for the public charts.

Verified: public distinct 24h = 5,835 vs 25,236 public rows vs 99,079
total rows (74% of traffic is from privacy-marked nodes).
This commit is contained in:
gadgethd
2026-08-07 14:06:28 +00:00
parent 8bc4b0dce6
commit 8dff7d79d0
2 changed files with 17 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/stats/statsRepository.test.ts
+2 -2
View File
@@ -129,7 +129,7 @@ test('canonical charts coalesce six high-volume dimensions into one maintained a
assert.match(aggregateSql, /SELECT \* FROM rollup_24h\s+UNION ALL\s+SELECT \* FROM raw_24h/);
assert.equal(
calls.filter((sql) =>
sql.includes("time_bucket('1 hour', p.time) AS bucket, COUNT(*)::int AS count")).length,
sql.includes("time_bucket('1 hour', p.time) AS bucket, COUNT(DISTINCT p.packet_hash)::int AS count")).length,
0,
);
assert.deepEqual(result.ptResult.rows, [{ packet_type: 4, count: '3' }]);
@@ -139,7 +139,7 @@ test('canonical charts coalesce six high-volume dimensions into one maintained a
await repository.fetchChartsData('ukmesh', 'A'.repeat(64));
assert.equal(calls.filter((sql) => sql.includes('FROM packet_hourly_stats')).length, 0);
assert.ok(calls.some((sql) =>
sql.includes("time_bucket('1 hour', p.time) AS bucket, COUNT(*)::int AS count")));
sql.includes("time_bucket('1 hour', p.time) AS bucket, COUNT(DISTINCT p.packet_hash)::int AS count")));
});
test('map summary uses the same coordinate, role, and 14-day freshness rules as the map', async () => {
backend/src/stats/statsRepository.ts
+15 -13
View File
@@ -185,6 +185,7 @@ export function createStatsRepository(deps: StatsRepositoryDeps) {
return query<{ channel: string; count: string; total_count: string }>(`
WITH decoded_group_packets AS (
SELECT
p.packet_hash AS packet_hash,
COALESCE(p.payload->>'_summary', '') AS summary,
NULLIF(TRIM((regexp_match(COALESCE(p.payload->>'_summary', ''), '^\\[([^\\]]+)\\]'))[1]), '') AS parsed_channel
FROM packets p
@@ -194,6 +195,7 @@ export function createStatsRepository(deps: StatsRepositoryDeps) {
),
group_packets AS (
SELECT
packet_hash,
COALESCE(
CASE
WHEN parsed_channel IS NOT NULL
@@ -208,7 +210,7 @@ export function createStatsRepository(deps: StatsRepositoryDeps) {
FROM decoded_group_packets
),
channel_counts AS (
SELECT channel, COUNT(*)::text AS count
SELECT channel, COUNT(DISTINCT p.packet_hash)::text AS count
FROM group_packets
GROUP BY channel
)
@@ -707,7 +709,7 @@ export function createStatsRepository(deps: StatsRepositoryDeps) {
) AS bucket
),
counts AS (
SELECT time_bucket('1 hour', p.time) AS bucket, COUNT(*)::int AS count
SELECT time_bucket('1 hour', p.time) AS bucket, COUNT(DISTINCT p.packet_hash)::int AS count
FROM packets p
WHERE p.time > NOW() - INTERVAL '24 hours'
${filters.packetsAlias('p')}
@@ -719,7 +721,7 @@ export function createStatsRepository(deps: StatsRepositoryDeps) {
ORDER BY b.bucket
`, filters.params)),
aggregateRows('packetsPerDay', () => query(`
SELECT time_bucket('1 day', time) AS day, COUNT(*) AS count
SELECT time_bucket('1 day', time) AS day, COUNT(DISTINCT packet_hash) AS count
FROM packets
WHERE time > NOW() - INTERVAL '7 days' ${filters.packets}
GROUP BY day ORDER BY day
@@ -754,13 +756,13 @@ export function createStatsRepository(deps: StatsRepositoryDeps) {
GROUP BY day ORDER BY day
`, filters.params),
aggregateRows('packetTypes', () => query(`
SELECT packet_type, COUNT(*) AS count
SELECT packet_type, COUNT(DISTINCT packet_hash) AS count
FROM packets
WHERE time > NOW() - INTERVAL '24 hours' ${filters.packets}
GROUP BY packet_type ORDER BY count DESC
`, filters.params)),
aggregateRows('hopDistribution', () => query(`
SELECT hop_count AS hops, COUNT(*) AS count
SELECT hop_count AS hops, COUNT(DISTINCT packet_hash) AS count
FROM packets
WHERE time > NOW() - INTERVAL '7 days'
AND hop_count IS NOT NULL
@@ -789,8 +791,8 @@ export function createStatsRepository(deps: StatsRepositoryDeps) {
`, filters.params),
query(`
SELECT
(SELECT COUNT(*) FROM packets WHERE time > NOW() - INTERVAL '24 hours' ${filters.packets}) AS total_24h,
(SELECT COUNT(*) FROM packets WHERE time > NOW() - INTERVAL '7 days' ${filters.packets}) AS total_7d,
(SELECT COUNT(DISTINCT packet_hash) FROM packets WHERE time > NOW() - INTERVAL '24 hours' ${filters.packets}) AS total_24h,
(SELECT COUNT(DISTINCT packet_hash) FROM packets WHERE time > NOW() - INTERVAL '7 days' ${filters.packets}) AS total_7d,
(SELECT COUNT(DISTINCT meshcore_canonical_node_id(src_node_id))
FROM packets
WHERE time > NOW() - INTERVAL '24 hours'
@@ -1061,7 +1063,7 @@ export function createStatsRepository(deps: StatsRepositoryDeps) {
filters.params,
),
aggregateRows('routeTypes', () => query<{ route_type: string; count: string }>(
`SELECT COALESCE(p.route_type::text, 'Unknown') AS route_type, COUNT(*)::text AS count
`SELECT COALESCE(p.route_type::text, 'Unknown') AS route_type, COUNT(DISTINCT p.packet_hash)::text AS count
FROM packets p
WHERE p.time > NOW() - INTERVAL '24 hours'
${filters.packetsAlias('p')}
@@ -1073,10 +1075,10 @@ export function createStatsRepository(deps: StatsRepositoryDeps) {
`SELECT
NULLIF(TRIM(p.transport_codes), '') AS transport_code,
NULLIF(TRIM(p.region_scope), '') AS region_scope,
COUNT(*)::text AS count
FROM packets p
WHERE p.time > NOW() - INTERVAL '24 hours'
AND NULLIF(TRIM(p.transport_codes), '') IS NOT NULL
COUNT(DISTINCT p.packet_hash)::text AS count
FROM packets
WHERE time > NOW() - INTERVAL '24 hours'
AND NULLIF(TRIM(p.transport_codes), '') IS NOT NULL
${filters.packetsAlias('p')}
GROUP BY 1, 2
ORDER BY COUNT(*) DESC, region_scope ASC, transport_code ASC
@@ -1303,7 +1305,7 @@ export function createStatsRepository(deps: StatsRepositoryDeps) {
AND rx_node_id NOT IN (SELECT rx_node_id FROM test_active)
${filters.packets}
`, filters.params),
query(`SELECT COUNT(*) AS count FROM packets WHERE time > NOW() - INTERVAL '24 hours' ${filters.packets}`, filters.params),
query(`SELECT COUNT(DISTINCT packet_hash) AS count FROM packets WHERE time > NOW() - INTERVAL '24 hours' ${filters.packets}`, filters.params),
query(`SELECT COUNT(*) AS count FROM node_identity_nodes nodes
WHERE ${publicMapBasePredicate('nodes')}
AND ${nodeEffectiveLastSeenSql('nodes')}