fix(map): remove inferred nodes and restore historic hops

This commit is contained in:
gadgethd committed 2026-08-03 22:47:50 +01:00
1 parent 209989832e
commit bcaa35fd27
28 files changed
+403 -272

No files matched your search

+31 -70
View File
@@ -19,6 +19,14 @@ import {
dbQueryDuration,
updateDbPoolMetrics,
} from '../metrics.js';
import {
nodeEffectiveLastSeenSql,
nodeEffectiveOnlineSql,
} from '../nodes/presence.js';
import {
reactivateHistoricPathNodes,
type HistoricPathNode,
} from '../repositories/pathEvidence.js';
const { Pool } = pg;
const COORDINATE_RECALC_THRESHOLD_M = Number(process.env['NODE_COORDINATE_RECALC_THRESHOLD_M'] ?? 25);
@@ -370,10 +378,18 @@ export async function refreshRecentPathEvidence(
),
node_hashes AS (
SELECT 2 AS path_hash_size_bytes, UPPER(LEFT(node_id, 4)) AS hash, node_id
FROM nodes WHERE (role = 2 OR role IS NULL) AND ${pathEvidenceNodeScope}
FROM nodes
WHERE (role = 2 OR role IS NULL) AND ${pathEvidenceNodeScope}
AND lat BETWEEN -90 AND 90
AND lon BETWEEN -180 AND 180
AND NOT (ABS(lat) < 1e-9 AND ABS(lon) < 1e-9)
UNION ALL
SELECT 3 AS path_hash_size_bytes, UPPER(LEFT(node_id, 6)) AS hash, node_id
FROM nodes WHERE (role = 2 OR role IS NULL) AND ${pathEvidenceNodeScope}
FROM nodes
WHERE (role = 2 OR role IS NULL) AND ${pathEvidenceNodeScope}
AND lat BETWEEN -90 AND 90
AND lon BETWEEN -180 AND 180
AND NOT (ABS(lat) < 1e-9 AND ABS(lon) < 1e-9)
),
unique_node_hashes AS (
SELECT path_hash_size_bytes, hash, MIN(node_id) AS node_id
@@ -409,8 +425,8 @@ export async function refreshRecentPathEvidence(
/**
* Real-time counterpart to {@link refreshRecentPathEvidence}: given the multibyte
* path hashes of a single freshly-ingested packet, credit each repeater whose
* prefix uniquely matches with `last_path_evidence_at = seenAt`. Returns the node
* IDs that were updated so the caller can broadcast a live "seen now" update.
* prefix uniquely matches with `last_path_evidence_at = seenAt`. Returns the
* preserved historic node rows so the caller can broadcast their coordinates.
*
* Only 2- and 3-byte hashes are accepted (single-byte is too collision-prone), and
* prefixes shared by more than one repeater are skipped (ambiguous). Best-effort:
@@ -422,52 +438,14 @@ export async function recordMultibyteEvidence(
seenAt: Date,
routeType?: number,
network?: string,
): Promise<string[]> {
// On Direct/TransportDirect packets the path is a future route, not an
// observed relay path. Crediting it would invent node presence.
if ((routeType !== 0 && routeType !== 1) || (sizeBytes !== 2 && sizeBytes !== 3)) return [];
const prefixLen = sizeBytes * 2; // hex chars: 2 bytes → 4, 3 bytes → 6
const hashes = Array.from(new Set(
pathHashes
.map((h) => String(h).trim().toUpperCase())
.filter((h) => h.length === prefixLen && /^[0-9A-F]+$/.test(h)),
));
if (hashes.length === 0) return [];
const pathEvidenceNodeScope = network === 'test'
? "n.network = 'test'"
: "n.network IS DISTINCT FROM 'test'";
// prefixLen is a server-controlled integer (4 or 6); inlining it lets Postgres
// use the upper(left(node_id,N)) functional indexes (idx_nodes_path_hash_2/3).
const res = await pool.query<{ node_id: string }>(
`WITH input(hash) AS (
SELECT UNNEST($1::text[])
),
candidates AS (
SELECT n.node_id, i.hash
FROM nodes n
JOIN input i ON UPPER(LEFT(n.node_id, ${prefixLen})) = i.hash
WHERE (n.role = 2 OR n.role IS NULL) AND ${pathEvidenceNodeScope}
),
unique_hashes AS (
SELECT hash FROM candidates GROUP BY hash HAVING COUNT(*) = 1
),
matched AS (
SELECT c.node_id
FROM candidates c
JOIN unique_hashes u ON u.hash = c.hash
),
updated AS (
UPDATE nodes n
SET last_path_evidence_at = $2::timestamptz
FROM matched m
WHERE n.node_id = m.node_id
AND (n.last_path_evidence_at IS NULL OR n.last_path_evidence_at < $2::timestamptz)
RETURNING n.node_id
)
SELECT node_id FROM updated`,
[hashes, seenAt.toISOString()],
);
return res.rows.map((r) => r.node_id);
): Promise<HistoricPathNode[]> {
return reactivateHistoricPathNodes(query, {
pathHashes,
sizeBytes,
seenAt,
routeType,
network,
});
}
export async function upsertNode(nodeId: string, updates: {
@@ -726,30 +704,13 @@ export async function getNodes(
-- multibyte relay evidence are independent proofs of presence. Always
-- expose the newest proof; preferring observer metadata with COALESCE
-- could make a freshly advertised node appear days older on the map.
GREATEST(
n.last_seen,
n.last_rx_at,
n.last_status_at,
n.last_path_evidence_at
) AS last_seen,
COALESCE(
CASE
WHEN GREATEST(n.last_rx_at, n.last_status_at) IS NOT NULL
THEN GREATEST(n.last_rx_at, n.last_status_at) > NOW() - INTERVAL '15 minutes'
ELSE NULL
END,
CASE
WHEN n.last_path_evidence_at IS NOT NULL
AND n.last_path_evidence_at > NOW() - INTERVAL '60 minutes'
THEN TRUE
ELSE n.is_online
END
) AS is_online,
${nodeEffectiveLastSeenSql('n')} AS last_seen,
${nodeEffectiveOnlineSql('n')} AS is_online,
n.advert_count
${optionalFields}
FROM nodes n
${whereClause}
ORDER BY GREATEST(n.last_seen, n.last_rx_at, n.last_status_at, n.last_path_evidence_at) DESC`,
ORDER BY ${nodeEffectiveLastSeenSql('n')} DESC`,
scope.params
);
return res.rows;
+5 -5
View File
@@ -954,19 +954,19 @@ async function handleMessage(topic: string, rawPayload: Buffer): Promise<void> {
invalidateResolveCache(finalHash);
// A repeater appearing in a multibyte (2–3 byte) path hash almost certainly
// relayed this packet, so treat it as proof it is online right now: refresh its
// last_path_evidence_at and broadcast a live "seen now" update. Non-MQTT
// repeaters (no direct reception) only get refreshed this way. Best-effort.
// relayed this packet. Resolve only a unique historic row with stored
// coordinates, then broadcast the full row so clients which no longer hold
// the stale node can restore it at its original location. Best-effort.
if (decodedPathHashSizeBytes != null && decodedPathHashSizeBytes >= 2 && path && path.length > 0) {
try {
const nodeIds = await recordMultibyteEvidence(
const historicNodes = await recordMultibyteEvidence(
path,
decodedPathHashSizeBytes,
new Date(),
decodedRouteType,
network,
);
for (const nodeId of nodeIds) emitNode(nodeId, { network });
for (const node of historicNodes) emitNodeUpsert({ ...node });
} catch (err) {
console.error('[mqtt] recordMultibyteEvidence error:', (err as Error).message);
}
+26
View File
@@ -0,0 +1,26 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import {
nodeEffectiveLastSeenSql,
nodeEffectiveOnlineSql,
} from './presence.js';
test('effective node presence gives fresh path evidence online precedence', () => {
const lastSeenSql = nodeEffectiveLastSeenSql('node');
const onlineSql = nodeEffectiveOnlineSql('node', '$4::timestamptz');
assert.match(lastSeenSql, /node\.last_seen/);
assert.match(lastSeenSql, /node\.last_rx_at/);
assert.match(lastSeenSql, /node\.last_status_at/);
assert.match(lastSeenSql, /node\.last_path_evidence_at/);
assert.match(
onlineSql,
/node\.last_path_evidence_at > \$4::timestamptz - INTERVAL '60 minutes'[\s\S]*THEN TRUE/,
);
assert.ok(
onlineSql.indexOf('last_path_evidence_at') < onlineSql.indexOf('last_rx_at'),
'fresh path evidence must be evaluated before older direct observations',
);
assert.match(onlineSql, /ELSE node\.is_online/);
assert.throws(() => nodeEffectiveOnlineSql('node; DROP TABLE nodes'));
});
+37
View File
@@ -0,0 +1,37 @@
function safeAlias(alias: string): string {
if (!/^[a-z][a-z0-9_]*$/i.test(alias)) {
throw new Error(`invalid SQL alias: ${alias}`);
}
return alias;
}
/** Newest direct or path-derived proof that a node was present. */
export function nodeEffectiveLastSeenSql(alias = 'n'): string {
const table = safeAlias(alias);
return `GREATEST(
${table}.last_seen,
${table}.last_rx_at,
${table}.last_status_at,
${table}.last_path_evidence_at
)`;
}
/**
* Path evidence is an independent online signal. It must take precedence over
* an older direct-RX/status timestamp rather than being hidden by it.
*/
export function nodeEffectiveOnlineSql(
alias = 'n',
referenceSql = 'NOW()',
): string {
const table = safeAlias(alias);
return `CASE
WHEN ${table}.last_path_evidence_at IS NOT NULL
AND ${table}.last_path_evidence_at > ${referenceSql} - INTERVAL '60 minutes'
THEN TRUE
WHEN GREATEST(${table}.last_rx_at, ${table}.last_status_at) IS NOT NULL
THEN GREATEST(${table}.last_rx_at, ${table}.last_status_at)
> ${referenceSql} - INTERVAL '15 minutes'
ELSE ${table}.is_online
END`;
}
+4 -1
View File
@@ -26,7 +26,10 @@ test('public map predicate is the single role, coordinate, privacy, and freshnes
assert.match(sql, /NOT \(ABS\(n\.lat\) < 5 AND ABS\(n\.lon\) < 5\)/);
assert.match(sql, /n\.name NOT LIKE '%🚫%'/);
assert.match(sql, /n\.role IS NULL OR n\.role NOT IN \(1, 3\)/);
assert.match(sql, /GREATEST\(n\.last_seen, n\.last_path_evidence_at\)/);
assert.match(sql, /n\.last_seen/);
assert.match(sql, /n\.last_rx_at/);
assert.match(sql, /n\.last_status_at/);
assert.match(sql, /n\.last_path_evidence_at/);
assert.match(sql, /> \$2::timestamptz - INTERVAL '28 days'/);
assert.match(sql, /<= \$2::timestamptz/);
});
+4 -2
View File
@@ -1,3 +1,5 @@
import { nodeEffectiveLastSeenSql } from './presence.js';
export const PUBLIC_MAP_ALLOWED_FIELDS = [
'node_id',
'name',
@@ -47,9 +49,9 @@ export function publicMapFreshPredicate(
const table = safeAlias(alias);
return `(
${publicMapBasePredicate(table)}
AND GREATEST(${table}.last_seen, ${table}.last_path_evidence_at)
AND ${nodeEffectiveLastSeenSql(table)}
> ${referenceSql} - INTERVAL '28 days'
AND GREATEST(${table}.last_seen, ${table}.last_path_evidence_at)
AND ${nodeEffectiveLastSeenSql(table)}
<= ${referenceSql}
)`;
}
+36
View File
@@ -0,0 +1,36 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { networkFilters } from '../api/utils/networkFilters.js';
import { createNodeRepository } from './nodes.js';
test('public map projects path evidence as the effective seen and online state', async () => {
let capturedSql = '';
let capturedParams: unknown[] | undefined;
const repository = createNodeRepository(async (sql, params) => {
capturedSql = sql;
capturedParams = params;
return { rows: [] };
});
const snapshot = '2026-08-03T12:00:00.000Z';
await repository.listPublicMapRows(
['node_id', 'last_seen', 'is_online'],
networkFilters('ukmesh'),
snapshot,
null,
100,
);
assert.match(capturedSql, /GREATEST\([\s\S]*n\.last_path_evidence_at[\s\S]*\)::text AS last_seen/);
assert.match(
capturedSql,
/n\.last_path_evidence_at > \$2::timestamptz - INTERVAL '60 minutes'[\s\S]*THEN TRUE[\s\S]*AS is_online/,
);
assert.doesNotMatch(capturedSql, /n\.last_seen::text AS last_seen/);
assert.deepEqual(capturedParams, [
['ukmesh', 'northeast', 'teesside'],
snapshot,
null,
101,
]);
});
+13 -1
View File
@@ -1,6 +1,10 @@
import type { QueryResultRow } from 'pg';
import type { NetworkFilters } from '../api/utils/networkFilters.js';
import { publicMapFreshPredicate } from '../nodes/publicMap.js';
import {
nodeEffectiveLastSeenSql,
nodeEffectiveOnlineSql,
} from '../nodes/presence.js';
type QueryFn = <T extends QueryResultRow = QueryResultRow>(
text: string,
@@ -182,7 +186,15 @@ export function createNodeRepository(query: QueryFn): NodeRepository {
const cursorParameter = snapshotParameter + 1;
const limitParameter = cursorParameter + 1;
const selectedFields = fields
.map((field) => field === 'last_seen' ? 'n.last_seen::text AS last_seen' : `n.${field}`)
.map((field) => {
if (field === 'last_seen') {
return `${nodeEffectiveLastSeenSql('n')}::text AS last_seen`;
}
if (field === 'is_online') {
return `${nodeEffectiveOnlineSql('n', `$${snapshotParameter}::timestamptz`)} AS is_online`;
}
return `n.${field}`;
})
.join(', ');
const result = await query<Record<string, unknown>>(
`SELECT ${selectedFields}
@@ -0,0 +1,94 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import {
reactivateHistoricPathNodes,
type HistoricPathNode,
type QueryFn,
} from './pathEvidence.js';
const NODE_ID = `ABCDEF${'1'.repeat(58)}`;
const SEEN_AT = new Date('2026-08-03T12:34:56.000Z');
test('historic path activation resolves only unique coordinate-bearing relays', async () => {
let capturedSql = '';
let capturedParams: unknown[] | undefined;
const expected: HistoricPathNode = {
node_id: NODE_ID,
name: 'Historic relay',
lat: 54.5,
lon: -1.2,
iata: 'MME',
role: 2,
last_seen: SEEN_AT.toISOString(),
is_online: true,
hardware_model: null,
public_key: NODE_ID,
advert_count: 4,
elevation_m: 120,
network: 'ukmesh',
};
const query: QueryFn = async (sql, params) => {
capturedSql = sql;
capturedParams = params;
return { rows: [expected] };
};
const rows = await reactivateHistoricPathNodes(query, {
pathHashes: ['abcdef', 'ABCDEF', 'not-hex', '1234'],
sizeBytes: 3,
seenAt: SEEN_AT,
routeType: 1,
network: 'ukmesh',
});
assert.deepEqual(rows, [expected]);
assert.deepEqual(capturedParams, [['ABCDEF'], SEEN_AT.toISOString()]);
assert.match(capturedSql, /UPPER\(LEFT\(n\.node_id, 6\)\) = i\.hash/);
assert.match(capturedSql, /n\.network IS DISTINCT FROM 'test'/);
assert.match(capturedSql, /n\.lat BETWEEN -90 AND 90/);
assert.match(capturedSql, /n\.lon BETWEEN -180 AND 180/);
assert.match(capturedSql, /NOT \(ABS\(n\.lat\) < 1e-9 AND ABS\(n\.lon\) < 1e-9\)/);
assert.match(capturedSql, /HAVING COUNT\(\*\) = 1/);
assert.match(capturedSql, /SET last_path_evidence_at = \$2::timestamptz/);
assert.match(capturedSql, /n\.lat,[\s\S]*n\.lon/);
assert.match(capturedSql, /TRUE AS is_online/);
});
test('historic path activation rejects future routes, short hashes, and missing evidence', async () => {
let calls = 0;
const query: QueryFn = async () => {
calls += 1;
return { rows: [] };
};
assert.deepEqual(await reactivateHistoricPathNodes(query, {
pathHashes: ['ABCDEF'], sizeBytes: 3, seenAt: SEEN_AT, routeType: 2,
}), []);
assert.deepEqual(await reactivateHistoricPathNodes(query, {
pathHashes: ['AB'], sizeBytes: 1, seenAt: SEEN_AT, routeType: 1,
}), []);
assert.deepEqual(await reactivateHistoricPathNodes(query, {
pathHashes: ['ZZZZ'], sizeBytes: 2, seenAt: SEEN_AT, routeType: 0,
}), []);
assert.equal(calls, 0);
});
test('historic test-network activation cannot match public inventory rows', async () => {
let capturedSql = '';
const query: QueryFn = async (sql) => {
capturedSql = sql;
return { rows: [] };
};
await reactivateHistoricPathNodes(query, {
pathHashes: ['ABCD'],
sizeBytes: 2,
seenAt: SEEN_AT,
routeType: 0,
network: 'test',
});
assert.match(capturedSql, /UPPER\(LEFT\(n\.node_id, 4\)\) = i\.hash/);
assert.match(capturedSql, /n\.network = 'test'/);
assert.doesNotMatch(capturedSql, /IS DISTINCT FROM 'test'/);
});
+107
View File
@@ -0,0 +1,107 @@
import type { QueryResultRow } from 'pg';
export type QueryFn = <T extends QueryResultRow = QueryResultRow>(
text: string,
params?: unknown[],
) => Promise<{ rows: T[] }>;
export type HistoricPathNode = QueryResultRow & {
node_id: string;
name: string | null;
lat: number;
lon: number;
iata: string | null;
role: number | null;
last_seen: string | Date;
is_online: true;
hardware_model: string | null;
public_key: string | null;
advert_count: number | null;
elevation_m: number | null;
network: string | null;
};
/**
* Resolve trustworthy multibyte relay hashes against the historic inventory.
* Only a unique, coordinate-bearing repeater can be reactivated; its stored
* coordinates are returned unchanged for the live node upsert.
*/
export async function reactivateHistoricPathNodes(
query: QueryFn,
input: {
pathHashes: string[];
sizeBytes: number;
seenAt: Date;
routeType?: number;
network?: string;
},
): Promise<HistoricPathNode[]> {
const { pathHashes, sizeBytes, seenAt, routeType, network } = input;
// Direct routes contain a future route. Only Flood/TransportFlood paths are
// evidence that the listed relay actually handled this packet.
if ((routeType !== 0 && routeType !== 1) || (sizeBytes !== 2 && sizeBytes !== 3)) return [];
const prefixLength = sizeBytes * 2;
const hashes = Array.from(new Set(
pathHashes
.map((hash) => String(hash).trim().toUpperCase())
.filter((hash) => hash.length === prefixLength && /^[0-9A-F]+$/.test(hash)),
));
if (hashes.length === 0) return [];
const networkScope = network === 'test'
? "n.network = 'test'"
: "n.network IS DISTINCT FROM 'test'";
// prefixLength is restricted above to 4 or 6, allowing the matching
// functional indexes to be used without interpolating caller input.
const result = await query<HistoricPathNode>(
`WITH input(hash) AS (
SELECT UNNEST($1::text[])
),
candidates AS (
SELECT n.node_id, i.hash
FROM nodes n
JOIN input i ON UPPER(LEFT(n.node_id, ${prefixLength})) = i.hash
WHERE (n.role = 2 OR n.role IS NULL)
AND ${networkScope}
AND n.lat BETWEEN -90 AND 90
AND n.lon BETWEEN -180 AND 180
AND NOT (ABS(n.lat) < 1e-9 AND ABS(n.lon) < 1e-9)
),
unique_hashes AS (
SELECT hash
FROM candidates
GROUP BY hash
HAVING COUNT(*) = 1
),
matched AS (
SELECT c.node_id
FROM candidates c
JOIN unique_hashes u ON u.hash = c.hash
),
updated AS (
UPDATE nodes n
SET last_path_evidence_at = $2::timestamptz
FROM matched m
WHERE n.node_id = m.node_id
AND (n.last_path_evidence_at IS NULL OR n.last_path_evidence_at < $2::timestamptz)
RETURNING
n.node_id,
n.name,
n.lat,
n.lon,
COALESCE(n.observer_iata, n.iata) AS iata,
n.role,
n.last_path_evidence_at AS last_seen,
TRUE AS is_online,
n.hardware_model,
n.public_key,
n.advert_count,
n.elevation_m,
n.network
)
SELECT * FROM updated`,
[hashes, seenAt.toISOString()],
);
return result.rows;
}
+6 -2
View File
@@ -168,7 +168,8 @@ test('map summary uses the same coordinate, role, and 14-day freshness rules as
call.text.includes("<= NOW() - INTERVAL '14 days'"),
)?.text;
const mapSql = calls.find((call) =>
call.text.includes("GREATEST(nodes.last_seen, nodes.last_path_evidence_at)")
call.text.includes('nodes.last_path_evidence_at')
&& call.text.includes('nodes.last_rx_at')
&& call.text.includes("> NOW() - INTERVAL '28 days'")
&& !call.text.includes("<= NOW() - INTERVAL '14 days'"),
)?.text;
@@ -180,7 +181,10 @@ test('map summary uses the same coordinate, role, and 14-day freshness rules as
assert.match(sql, /nodes\.lon BETWEEN -180 AND 180/);
assert.match(sql, /NOT \(ABS\(nodes\.lat\) < 5 AND ABS\(nodes\.lon\) < 5\)/);
assert.match(sql, /\(nodes\.role IS NULL OR nodes\.role NOT IN \(1, 3\)\)/);
assert.match(sql, /GREATEST\(nodes\.last_seen, nodes\.last_path_evidence_at\)/);
assert.match(sql, /nodes\.last_seen/);
assert.match(sql, /nodes\.last_rx_at/);
assert.match(sql, /nodes\.last_status_at/);
assert.match(sql, /nodes\.last_path_evidence_at/);
assert.match(sql, /nodes\.name NOT LIKE/);
assert.doesNotMatch(sql, /INTERVAL '7 days'/);
}
+3 -2
View File
@@ -5,6 +5,7 @@ import {
publicMapBasePredicate,
publicMapFreshPredicate,
} from '../nodes/publicMap.js';
import { nodeEffectiveLastSeenSql } from '../nodes/presence.js';
import {
loadStoredChartSnapshot,
saveStoredChartSnapshot,
@@ -1295,9 +1296,9 @@ export function createStatsRepository(deps: StatsRepositoryDeps) {
query(`SELECT COUNT(*) AS count FROM packets WHERE time > NOW() - INTERVAL '24 hours' ${filters.packets}`, filters.params),
query(`SELECT COUNT(*) AS count FROM nodes
WHERE ${publicMapBasePredicate('nodes')}
AND GREATEST(nodes.last_seen, nodes.last_path_evidence_at)
AND ${nodeEffectiveLastSeenSql('nodes')}
<= NOW() - INTERVAL '14 days'
AND GREATEST(nodes.last_seen, nodes.last_path_evidence_at)
AND ${nodeEffectiveLastSeenSql('nodes')}
> NOW() - INTERVAL '28 days'
${filters.nodes}`, filters.params),
query(`SELECT COUNT(*) AS count FROM nodes