dandri/ukmesh
1
0 Fork 0
mirror of https://github.com/gadgethd/ukmesh.git synced 2026-10-22 22:11:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(stats): packet counts = total observations (all rows, duplicates included)

Browse Source 
Per Ben's spec the "Observed packets" totals must count EVERY packet
received by EVERY observer - duplicate receptions of the same packet by
multiple stations each count. Reverts the COUNT(DISTINCT packet_hash)
semantics from 8dff7d7 and additionally drops the privacy visibility
filter from the totals so they match the chart rollup path (which has
never been privacy-filtered - aggregateScope filters by network only).

Changes:
- packetsDay, totalPackets24h, totalPackets7d: COUNT(*) over all rows
  (network/test-topic scoped, rx_node_id present), no visibility filter
- channelTraffic: COUNT(*) all rows (percentages stay consistent with
  the all-rows total denominator)
- networkFilters(): new includePrivacy option (default true) so total
  queries can opt out of visibility conditions
- legacy/observer-scoped chart series back to COUNT(*) (row semantics
  everywhere; transportCodes alias fix from 40be809 retained)
- tests: includePrivacy coverage added

Verified live: 24h total ~99k rows (vs 25k privacy-filtered, vs 5.8k
distinct) - consistent with the hourly chart series which already
counted all rows.
This commit is contained in:
gadgethd committed 2026-08-07 15:47:11 +00:00
1 parent 40be809330
commit 4a2d69bc58
5 files changed
+40 -22
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

backend/src/api/routes/stats.ts
+1 -1
View File
@@ -37,7 +37,7 @@ type StatsRouteDeps = {
chartsInflight: Map<string, Promise<unknown>>;
expensiveLimiter: ReturnType<typeof import('express-rate-limit').rateLimit>;
statsChartsLimiter: ReturnType<typeof import('express-rate-limit').rateLimit>;
networkFilters: (network?: string, observer?: string) => NetworkFilters;
networkFilters: (network?: string, observer?: string, opts?: { includePrivacy?: boolean }) => NetworkFilters;
query: QueryFn;
getPublicVisibilityGeneration: () => Promise<number>;
maskDecodedPathNodes: MaskDecodedPathNodesFn;
backend/src/api/utils/networkFilters.test.ts
+9
View File
@@ -28,6 +28,15 @@ test('public packet scopes keep legacy test-topic rows out', () => {
assert.doesNotMatch(filters.packets, /private_node\.name LIKE '%🚫%'/);
});
test('includePrivacy:false omits visibility and private-prefix conditions', () => {
const filters = networkFilters('ukmesh', undefined, { includePrivacy: false });
assert.match(filters.packets, /network = ANY\(\$1\)/);
assert.doesNotMatch(filters.packets, /visibility_ok IS TRUE/);
assert.doesNotMatch(filters.packets, /private_node_prefixes private_prefix/);
const defaulted = networkFilters('ukmesh');
assert.match(defaulted.packets, /visibility_ok IS TRUE/);
});
test('owned public visibility scopes carry observer filtering', () => {
const filters = publicNetworkFilters({
access: 'public',
backend/src/api/utils/networkFilters.ts
+8 -2
View File
@@ -60,7 +60,11 @@ function excludesLegacyTestTopic(prefix: string): string {
return `COALESCE(NULLIF(${prefix}topic_prefix, ''), split_part(${prefix}topic, '/', 1)) <> 'meshcore-test'`;
}
export function networkFilters(network?: string, observer?: string): NetworkFilters {
export function networkFilters(
network?: string,
observer?: string,
opts?: { includePrivacy?: boolean },
): NetworkFilters {
const params: unknown[] = [];
let networkParam: string | null = null;
let networkIsMulti = false;
@@ -102,7 +106,9 @@ export function networkFilters(network?: string, observer?: string): NetworkFilt
OR meshcore_canonical_node_id(rx_node_id) = meshcore_canonical_node_id(${observerParam}))`,
);
}
packetConditions.push(...publicPacketPrivacyConditions(''));
if (opts?.includePrivacy !== false) {
packetConditions.push(...publicPacketPrivacyConditions(''));
}
const nodeConditions = (alias?: string) => {
const prefix = alias ? `${alias}.` : '';
backend/src/stats/statsRepository.test.ts
+2 -2
View File
@@ -129,7 +129,7 @@ test('canonical charts coalesce six high-volume dimensions into one maintained a
assert.match(aggregateSql, /SELECT \* FROM rollup_24h\s+UNION ALL\s+SELECT \* FROM raw_24h/);
assert.equal(
calls.filter((sql) =>
sql.includes("time_bucket('1 hour', p.time) AS bucket, COUNT(DISTINCT p.packet_hash)::int AS count")).length,
sql.includes("time_bucket('1 hour', p.time) AS bucket, COUNT(*)::int AS count")).length,
0,
);
assert.deepEqual(result.ptResult.rows, [{ packet_type: 4, count: '3' }]);
@@ -139,7 +139,7 @@ test('canonical charts coalesce six high-volume dimensions into one maintained a
await repository.fetchChartsData('ukmesh', 'A'.repeat(64));
assert.equal(calls.filter((sql) => sql.includes('FROM packet_hourly_stats')).length, 0);
assert.ok(calls.some((sql) =>
sql.includes("time_bucket('1 hour', p.time) AS bucket, COUNT(DISTINCT p.packet_hash)::int AS count")));
sql.includes("time_bucket('1 hour', p.time) AS bucket, COUNT(*)::int AS count")));
});
test('map summary uses the same coordinate, role, and 14-day freshness rules as the map', async () => {
backend/src/stats/statsRepository.ts
+20 -17
View File
@@ -17,7 +17,7 @@ type QueryFn = <T extends QueryResultRow = QueryResultRow>(
) => Promise<{ rows: T[] }>;
type StatsRepositoryDeps = {
networkFilters: (network?: string, observer?: string) => NetworkFilters;
networkFilters: (network?: string, observer?: string, opts?: { includePrivacy?: boolean }) => NetworkFilters;
query: QueryFn;
aggregateReadsEnabled?: boolean;
aggregateShadowEnabled?: boolean;
@@ -181,11 +181,10 @@ export function createStatsRepository(deps: StatsRepositoryDeps) {
}
async function fetchChannelTraffic(network: string | undefined, observer: string | undefined) {
const filters = networkFilters(network, observer);
const filters = networkFilters(network, observer, { includePrivacy: false });
return query<{ channel: string; count: string; total_count: string }>(`
WITH decoded_group_packets AS (
SELECT
p.packet_hash AS packet_hash,
COALESCE(p.payload->>'_summary', '') AS summary,
NULLIF(TRIM((regexp_match(COALESCE(p.payload->>'_summary', ''), '^\\[([^\\]]+)\\]'))[1]), '') AS parsed_channel
FROM packets p
@@ -195,7 +194,6 @@ export function createStatsRepository(deps: StatsRepositoryDeps) {
),
group_packets AS (
SELECT
packet_hash,
COALESCE(
CASE
WHEN parsed_channel IS NOT NULL
@@ -210,7 +208,7 @@ export function createStatsRepository(deps: StatsRepositoryDeps) {
FROM decoded_group_packets
),
channel_counts AS (
SELECT channel, COUNT(DISTINCT packet_hash)::text AS count
SELECT channel, COUNT(*)::text AS count
FROM group_packets
GROUP BY channel
)
@@ -681,6 +679,7 @@ export function createStatsRepository(deps: StatsRepositoryDeps) {
async function fetchChartsData(network: string | undefined, observer: string | undefined) {
const filters = networkFilters(network, observer);
const totalFilters = networkFilters(network, observer, { includePrivacy: false });
const aggregateAsOf = new Date();
const aggregatePartsPromise = aggregateReadsEnabled && !observer
? fetchAggregateChartParts(network, aggregateAsOf)
@@ -709,7 +708,7 @@ export function createStatsRepository(deps: StatsRepositoryDeps) {
) AS bucket
),
counts AS (
SELECT time_bucket('1 hour', p.time) AS bucket, COUNT(DISTINCT p.packet_hash)::int AS count
SELECT time_bucket('1 hour', p.time) AS bucket, COUNT(*)::int AS count
FROM packets p
WHERE p.time > NOW() - INTERVAL '24 hours'
${filters.packetsAlias('p')}
@@ -721,7 +720,7 @@ export function createStatsRepository(deps: StatsRepositoryDeps) {
ORDER BY b.bucket
`, filters.params)),
aggregateRows('packetsPerDay', () => query(`
SELECT time_bucket('1 day', time) AS day, COUNT(DISTINCT packet_hash) AS count
SELECT time_bucket('1 day', time) AS day, COUNT(*) AS count
FROM packets
WHERE time > NOW() - INTERVAL '7 days' ${filters.packets}
GROUP BY day ORDER BY day
@@ -756,13 +755,13 @@ export function createStatsRepository(deps: StatsRepositoryDeps) {
GROUP BY day ORDER BY day
`, filters.params),
aggregateRows('packetTypes', () => query(`
SELECT packet_type, COUNT(DISTINCT packet_hash) AS count
SELECT packet_type, COUNT(*) AS count
FROM packets
WHERE time > NOW() - INTERVAL '24 hours' ${filters.packets}
GROUP BY packet_type ORDER BY count DESC
`, filters.params)),
aggregateRows('hopDistribution', () => query(`
SELECT hop_count AS hops, COUNT(DISTINCT packet_hash) AS count
SELECT hop_count AS hops, COUNT(*) AS count
FROM packets
WHERE time > NOW() - INTERVAL '7 days'
AND hop_count IS NOT NULL
@@ -791,8 +790,8 @@ export function createStatsRepository(deps: StatsRepositoryDeps) {
`, filters.params),
query(`
SELECT
(SELECT COUNT(DISTINCT packet_hash) FROM packets WHERE time > NOW() - INTERVAL '24 hours' ${filters.packets}) AS total_24h,
(SELECT COUNT(DISTINCT packet_hash) FROM packets WHERE time > NOW() - INTERVAL '7 days' ${filters.packets}) AS total_7d,
(SELECT COUNT(*) FROM packets WHERE time > NOW() - INTERVAL '24 hours' ${totalFilters.packets}) AS total_24h,
(SELECT COUNT(*) FROM packets WHERE time > NOW() - INTERVAL '7 days' ${totalFilters.packets}) AS total_7d,
(SELECT COUNT(DISTINCT meshcore_canonical_node_id(src_node_id))
FROM packets
WHERE time > NOW() - INTERVAL '24 hours'
@@ -1063,7 +1062,7 @@ export function createStatsRepository(deps: StatsRepositoryDeps) {
filters.params,
),
aggregateRows('routeTypes', () => query<{ route_type: string; count: string }>(
`SELECT COALESCE(p.route_type::text, 'Unknown') AS route_type, COUNT(DISTINCT p.packet_hash)::text AS count
`SELECT COALESCE(p.route_type::text, 'Unknown') AS route_type, COUNT(*)::text AS count
FROM packets p
WHERE p.time > NOW() - INTERVAL '24 hours'
${filters.packetsAlias('p')}
@@ -1075,10 +1074,10 @@ export function createStatsRepository(deps: StatsRepositoryDeps) {
`SELECT
NULLIF(TRIM(p.transport_codes), '') AS transport_code,
NULLIF(TRIM(p.region_scope), '') AS region_scope,
COUNT(DISTINCT p.packet_hash)::text AS count
FROM packets p
WHERE p.time > NOW() - INTERVAL '24 hours'
AND NULLIF(TRIM(p.transport_codes), '') IS NOT NULL
COUNT(*)::text AS count
FROM packets p
WHERE p.time > NOW() - INTERVAL '24 hours'
AND NULLIF(TRIM(p.transport_codes), '') IS NOT NULL
${filters.packetsAlias('p')}
GROUP BY 1, 2
ORDER BY COUNT(*) DESC, region_scope ASC, transport_code ASC
@@ -1254,6 +1253,7 @@ export function createStatsRepository(deps: StatsRepositoryDeps) {
async function fetchStatsSummary(network: string | undefined, observer: string | undefined) {
const filters = networkFilters(network, observer);
const totalFilters = networkFilters(network, observer, { includePrivacy: false });
const longestHopResult = () => {
if (observer) {
return query(`SELECT hop_count AS count, packet_hash AS hash
@@ -1305,7 +1305,10 @@ export function createStatsRepository(deps: StatsRepositoryDeps) {
AND rx_node_id NOT IN (SELECT rx_node_id FROM test_active)
${filters.packets}
`, filters.params),
query(`SELECT COUNT(DISTINCT packet_hash) AS count FROM packets WHERE time > NOW() - INTERVAL '24 hours' ${filters.packets}`, filters.params),
query(`SELECT COUNT(*) AS count FROM packets
WHERE time > NOW() - INTERVAL '24 hours'
AND rx_node_id IS NOT NULL AND rx_node_id <> ''
${totalFilters.packets}`, totalFilters.params),
query(`SELECT COUNT(*) AS count FROM node_identity_nodes nodes
WHERE ${publicMapBasePredicate('nodes')}
AND ${nodeEffectiveLastSeenSql('nodes')}