Support multibyte path hashes across live pathing

This commit is contained in:
Ben committed 2026-03-07 15:51:29 +00:00
1 parent 814c94d01f
commit cae0580609
15 files changed
+2313 -113

No files matched your search

+1855
View File
File diff suppressed because it is too large. Load diff
+1 -1
View File
@@ -9,7 +9,7 @@
"start": "node dist/index.js"
},
"dependencies": {
"@michaelhart/meshcore-decoder": "^0.1.7",
"@michaelhart/meshcore-decoder": "^0.2.7",
"cors": "^2.8.5",
"express": "^4.19.2",
"express-rate-limit": "^7.4.1",
+7 -2
View File
@@ -137,7 +137,7 @@ export async function getRecentPackets(limit = 200) {
export async function getLastNPackets(n: number, network?: string) {
// DISTINCT ON deduplicates by hash (same packet heard by multiple observers),
// picking the most recent observation per hash within the last 24 hours.
// preferring the richest observation per hash within the last 24 hours.
const nFilter = network ? 'AND network = $2' : '';
const params: unknown[] = network ? [n, network] : [n];
const res = await pool.query(
@@ -146,7 +146,12 @@ export async function getLastNPackets(n: number, network?: string) {
packet_type, hop_count, payload, advert_count, path_hashes
FROM packets
WHERE time > NOW() - INTERVAL '24 hours' ${nFilter}
ORDER BY packet_hash, time DESC
ORDER BY packet_hash,
CASE WHEN payload ? 'appData' THEN 1 ELSE 0 END DESC,
CASE WHEN src_node_id IS NOT NULL THEN 1 ELSE 0 END DESC,
CASE WHEN packet_type = 4 THEN 1 ELSE 0 END DESC,
CASE WHEN payload->>'direction' = 'tx' THEN 1 ELSE 0 END DESC,
time DESC
) deduped
ORDER BY time DESC LIMIT $1`,
params
+57 -1
View File
@@ -221,7 +221,7 @@ export async function captureWorkerHealthSnapshot(): Promise<void> {
}
export async function getWorkerHealthOverview() {
const [workers, history, errors1h, ingest] = await Promise.all([
const [workers, history, errors1h, ingest, pathHashWidths, multibyteSummary] = await Promise.all([
currentWorkers(),
query<{
ts: string;
@@ -272,6 +272,38 @@ export async function getWorkerHealthOverview() {
(SELECT MAX(time)::text FROM packets) AS global_last_packet_at
FROM active_rx`,
),
query<{
hash_hex_len: string;
hop_count: string;
}>(
`SELECT length(h)::text AS hash_hex_len, COUNT(*)::text AS hop_count
FROM packets p
CROSS JOIN LATERAL unnest(p.path_hashes) AS h
WHERE p.time > NOW() - INTERVAL '24 hours'
GROUP BY 1`,
),
query<{
latest_multibyte_at: string | null;
multibyte_packets_24h: string;
}>(
`SELECT
MAX(time) FILTER (
WHERE EXISTS (
SELECT 1
FROM unnest(path_hashes) AS h
WHERE length(h) > 2
)
)::text AS latest_multibyte_at,
COUNT(*) FILTER (
WHERE EXISTS (
SELECT 1
FROM unnest(path_hashes) AS h
WHERE length(h) > 2
)
)::text AS multibyte_packets_24h
FROM packets
WHERE time > NOW() - INTERVAL '24 hours'`,
),
]);
const ingestRow = ingest.rows[0];
@@ -279,6 +311,25 @@ export async function getWorkerHealthOverview() {
const activeNodes = Number(ingestRow?.active_nodes ?? 0);
const maxStaleMinutes = Number(ingestRow?.max_stale_minutes ?? 0);
const staleThresholdMinutes = Number(ingestRow?.stale_threshold_minutes ?? 15);
const widthToBucket: Record<number, 'one_byte' | 'two_byte' | 'three_byte'> = {
2: 'one_byte',
4: 'two_byte',
6: 'three_byte',
};
const pathHashStats = {
one_byte: 0,
two_byte: 0,
three_byte: 0,
};
for (const row of pathHashWidths.rows) {
const width = Number(row.hash_hex_len ?? 0);
const bucket = widthToBucket[width];
if (!bucket) continue;
pathHashStats[bucket] += Number(row.hop_count ?? 0);
}
const multibyteRow = multibyteSummary.rows[0];
return {
system: systemStats(),
@@ -292,5 +343,10 @@ export async function getWorkerHealthOverview() {
stale_threshold_minutes: staleThresholdMinutes,
global_last_packet_at: ingestRow?.global_last_packet_at ?? null,
},
path_hashes: {
last_24h_hops: pathHashStats,
multibyte_packets_24h: Number(multibyteRow?.multibyte_packets_24h ?? 0),
latest_multibyte_at: multibyteRow?.latest_multibyte_at ?? null,
},
};
}
+71 -10
View File
@@ -6,6 +6,7 @@ import type {
} from '@michaelhart/meshcore-decoder';
import { insertPacket, upsertNode, incrementAdvertCount, query } from '../db/index.js';
import type { LivePacket } from '../types/index.js';
import { decodePacketCompat } from './decodePacket.js';
type PacketCallback = (packet: LivePacket) => void;
type NodeCallback = (nodeId: string) => void;
@@ -127,7 +128,7 @@ const keyStore = MeshCoreDecoder.createKeyStore({
/** Identify which channel a GroupText was sent on by trying each single-key keyStore. */
function identifyChannel(rawHex: string): string | undefined {
for (const entry of channelEntries) {
const d = MeshCoreDecoder.decode(rawHex, { keyStore: entry.keyStore });
const { decoded: d } = decodePacketCompat(rawHex, entry.keyStore);
const p = d?.payload?.decoded as GroupTextPayload | undefined;
if (p?.decrypted) return entry.name;
}
@@ -176,6 +177,23 @@ function buildSummary(payloadType: number, decoded: unknown, rawHex?: string): s
}
}
function buildAdvertFallbackPayload(originId: string, originName?: string): Record<string, unknown> {
const appData: Record<string, unknown> = {
flags: 0,
deviceRole: 2,
hasLocation: false,
hasName: Boolean(originName),
};
if (originName) appData['name'] = originName;
return {
type: 4,
version: 0,
isValid: false,
publicKey: originId,
appData,
};
}
export function startMqttClient(): void {
const brokerUrl = process.env['MQTT_BROKER_URL'] ?? 'ws://mosquitto:9001';
@@ -223,9 +241,10 @@ async function handleMessage(topic: string, rawPayload: Buffer): Promise<void> {
}
// ── Status: update observer node metadata ─────────────────────────────────
const origin = json['origin'] as string | undefined;
const originId = json['origin_id'] as string | undefined;
if (suffix === 'status') {
const origin = json['origin'] as string | undefined;
const originId = json['origin_id'] as string | undefined;
const model = json['model'] as string | undefined;
const firmware = json['firmware_version'] as string | undefined;
@@ -265,13 +284,13 @@ async function handleMessage(topic: string, rawPayload: Buffer): Promise<void> {
if (rawHex) {
try {
const decoded = MeshCoreDecoder.decode(rawHex, { keyStore });
const { decoded, pathHashes, pathHashCount } = decodePacketCompat(rawHex, keyStore);
if (decoded) {
decodedHash = decoded.messageHash;
decodedHops = decoded.pathLength;
if (decoded.path && Array.isArray(decoded.path) && decoded.path.length > 0) {
path = decoded.path as string[];
decodedHops = pathHashCount ?? decoded.pathLength;
if (pathHashes && pathHashes.length > 0) {
path = pathHashes;
}
const decodedInner = decoded.payload?.decoded;
@@ -330,6 +349,27 @@ async function handleMessage(topic: string, rawPayload: Buffer): Promise<void> {
}
}
// MeshCore v1.14.0 multibyte-path adverts can currently reach us with packet_type=4
// on the MQTT envelope while the community decoder still classifies the raw payload
// as another type. For self-originated TX adverts, use the MQTT origin metadata as a
// safe fallback so the repeater still appears correctly on the site.
const hasDecodedAdvertPayload = Boolean(
innerPayload
&& typeof innerPayload === 'object'
&& 'appData' in innerPayload
&& srcNodeId
);
const useTxAdvertFallback = direction === 'tx'
&& packetType === 4
&& Boolean(originId)
&& !hasDecodedAdvertPayload;
if (useTxAdvertFallback && originId) {
srcNodeId = originId;
summary ??= origin;
innerPayload = buildAdvertFallbackPayload(originId, origin);
}
// Use decoder hash as primary key — it's derived from packet content and is the
// same for TX and RX of the same packet, enabling frontend deduplication.
// Fall back to mctomqtt hash (RX only) when decode fails, then UUID.
@@ -340,6 +380,27 @@ async function handleMessage(topic: string, rawPayload: Buffer): Promise<void> {
void upsertNode(observerKey, { iata, network });
emitNode(observerKey);
if (useTxAdvertFallback && originId) {
await upsertNode(originId, {
name: origin,
iata,
publicKey: originId,
network,
});
if (tryCountAdvert(finalHash)) {
advertCount = await incrementAdvertCount(originId);
}
emitNodeUpsert({
node_id: originId,
name: origin,
iata,
public_key: originId,
last_seen: new Date().toISOString(),
is_online: true,
advert_count: advertCount,
});
}
{
const livePacket: LivePacket = {
id: crypto.randomUUID(),
@@ -401,9 +462,9 @@ export async function backfillHistoricalLinks(
let queued = 0;
for (const row of res.rows) {
try {
const decoded = MeshCoreDecoder.decode(row.raw_hex, { keyStore });
if (decoded?.path && Array.isArray(decoded.path) && decoded.path.length > 0) {
queueFn(row.rx_node_id, row.src_node_id ?? undefined, decoded.path as string[], decoded.pathLength);
const compat = decodePacketCompat(row.raw_hex, keyStore);
if (compat.pathHashes && compat.pathHashes.length > 0) {
queueFn(row.rx_node_id, row.src_node_id ?? undefined, compat.pathHashes, compat.pathHashCount);
queued++;
}
} catch {
+86
View File
@@ -0,0 +1,86 @@
import { MeshCoreDecoder } from '@michaelhart/meshcore-decoder';
type KeyStore = ReturnType<typeof MeshCoreDecoder.createKeyStore>;
type PathMeta = {
encodedLengthByte: number;
pathLengthOffset: number;
pathHashCount: number;
pathHashSize: number;
pathByteLength: number;
pathHashes: string[];
};
export type CompatDecodedPacket = {
decoded: ReturnType<typeof MeshCoreDecoder.decode>;
pathHashes?: string[];
pathHashCount?: number;
pathHashSize?: number;
};
function bytesToHex(bytes: Uint8Array): string {
return Array.from(bytes, (b) => b.toString(16).padStart(2, '0')).join('').toUpperCase();
}
function hexToBytes(rawHex: string): Uint8Array | null {
const hex = rawHex.trim();
if (!hex || (hex.length % 2) !== 0 || !/^[0-9A-Fa-f]+$/.test(hex)) return null;
return Uint8Array.from(Buffer.from(hex, 'hex'));
}
function parsePathMeta(rawHex: string): PathMeta | null {
const bytes = hexToBytes(rawHex);
if (!bytes || bytes.length < 2) return null;
const routeType = bytes[0]! & 0x03;
let offset = 1;
if (routeType === 0 || routeType === 3) offset += 4;
if (bytes.length <= offset) return null;
const encodedLengthByte = bytes[offset]!;
const pathHashCount = encodedLengthByte & 0x3f;
const pathHashSize = (encodedLengthByte >> 6) + 1;
if (pathHashSize > 3) return null;
const pathByteLength = pathHashCount * pathHashSize;
const pathStart = offset + 1;
const pathEnd = pathStart + pathByteLength;
if (bytes.length < pathEnd) return null;
const pathHashes: string[] = [];
for (let i = 0; i < pathHashCount; i++) {
const start = pathStart + (i * pathHashSize);
const end = start + pathHashSize;
pathHashes.push(bytesToHex(bytes.subarray(start, end)));
}
return {
encodedLengthByte,
pathLengthOffset: offset,
pathHashCount,
pathHashSize,
pathByteLength,
pathHashes,
};
}
function buildCompatRawHex(rawHex: string, meta: PathMeta): string {
if (meta.encodedLengthByte < 64) return rawHex;
const bytes = hexToBytes(rawHex);
if (!bytes) return rawHex;
const compat = new Uint8Array(bytes);
compat[meta.pathLengthOffset] = meta.pathByteLength;
return bytesToHex(compat);
}
export function decodePacketCompat(rawHex: string, keyStore: KeyStore): CompatDecodedPacket {
const meta = parsePathMeta(rawHex);
const compatRawHex = meta ? buildCompatRawHex(rawHex, meta) : rawHex;
const decoded = MeshCoreDecoder.decode(compatRawHex, { keyStore });
return {
decoded,
pathHashes: meta?.pathHashes ?? (decoded.path as string[] | null) ?? undefined,
pathHashCount: meta?.pathHashCount ?? decoded.pathLength ?? undefined,
pathHashSize: meta?.pathHashSize ?? 1,
};
}
+38 -46
View File
@@ -1,4 +1,11 @@
import { MIN_LINK_OBSERVATIONS, query } from '../db/index.js';
import {
buildNodePathHashIndex,
countNodesForPathHash,
getNodesForPathHash,
nodePathHash,
normalizePathHash,
} from '../path-hash/utils.js';
type MeshNode = {
node_id: string;
@@ -371,14 +378,15 @@ function buildFallbackPrefixPath(
const repeaters = Array.from(nodesById.values()).filter(
(n) => hasCoords(n) && (n.role === null || n.role === 2),
);
const pathHashIndex = buildNodePathHashIndex(repeaters);
const pickedNearRx: MeshNode[] = [];
const visited = new Set<string>([rx.node_id]);
let prev = rx;
for (const h of [...hopHashes].reverse()) {
const prefix = h.slice(0, 2).toUpperCase();
const candidates = repeaters
.filter((n) => !visited.has(n.node_id) && n.node_id.slice(0, 2).toUpperCase() === prefix)
const prefix = normalizePathHash(h);
const candidates = getNodesForPathHash(pathHashIndex, prefix)
.filter((n) => !visited.has(n.node_id))
.sort((a, b) => {
const score = (c: MeshNode) => {
const distPenalty = distKm(c, prev) / 50;
@@ -422,13 +430,7 @@ function enumeratePrefixContinuations(
const candidates = Array.from(nodesById.values()).filter(
(n) => hasCoords(n) && (n.role === null || n.role === 2),
);
const byPrefix = new Map<string, MeshNode[]>();
for (const n of candidates) {
const p = n.node_id.slice(0, 2).toUpperCase();
const arr = byPrefix.get(p);
if (arr) arr.push(n);
else byPrefix.set(p, [n]);
}
const pathHashIndex = buildNodePathHashIndex(candidates);
const start = nodesById.get(startNodeId);
const end = nodesById.get(endNodeId);
@@ -482,8 +484,8 @@ function enumeratePrefixContinuations(
return;
}
const prefix = remainingPrefixes[idx]!.slice(0, 2).toUpperCase();
const nodesForPrefix = (byPrefix.get(prefix) ?? [])
const prefix = normalizePathHash(remainingPrefixes[idx]);
const nodesForPrefix = getNodesForPathHash(pathHashIndex, prefix)
.filter((n) => !visited.has(n.node_id) && !blocked.has(n.node_id) && n.node_id !== end.node_id && distKm(n, current) <= MAX_PERMUTATION_HOP_KM)
.sort((a, b) => distKm(a, current) - distKm(b, current));
if (nodesForPrefix.length < 1) {
@@ -544,8 +546,9 @@ function resolveBetaPath(
context: BetaResolveContext,
options?: { forceIncludeSource?: boolean; disableSourcePrepend?: boolean; blockedNodeIds?: string[] },
): { path: [number, number][]; confidence: number; segmentConfidence: number[]; nodeIds: string[] } | null {
if (!hasCoords(rx) || pathHashes.length === 0) return null;
if (pathHashes.length >= MAX_BETA_HOPS) return null;
const normalizedHashes = pathHashes.map(normalizePathHash).filter(Boolean);
if (!hasCoords(rx) || normalizedHashes.length === 0) return null;
if (normalizedHashes.length >= MAX_BETA_HOPS) return null;
const rxLat = rx.lat!;
const rxLon = rx.lon!;
@@ -554,16 +557,7 @@ function resolveBetaPath(
const candidatesPool = Array.from(context.nodesById.values()).filter(
(n) => hasCoords(n) && (n.role === null || n.role === 2) && !blockedNodeIds.has(n.node_id),
);
const prefixCounts = new Map<string, number>();
const prefixBuckets = new Map<string, MeshNode[]>();
for (const n of candidatesPool) {
const p = n.node_id.slice(0, 2).toUpperCase();
prefixCounts.set(p, (prefixCounts.get(p) ?? 0) + 1);
const existing = prefixBuckets.get(p);
if (existing) existing.push(n);
else prefixBuckets.set(p, [n]);
}
const pathHashIndex = buildNodePathHashIndex(candidatesPool);
const totalDist = hasCoords(src) ? distKm(src, rx) : 0;
const corridorMaxKm = Math.max(10, Math.min(80, totalDist * 0.35));
@@ -652,9 +646,8 @@ function resolveBetaPath(
return null;
}
function localPrefixAmbiguityPenalty(candidate: MeshNode, prevNode: MeshNode): number {
const prefix = candidate.node_id.slice(0, 2).toUpperCase();
const peers = prefixBuckets.get(prefix) ?? [];
function localPrefixAmbiguityPenalty(candidate: MeshNode, prevNode: MeshNode, pathHash: string): number {
const peers = getNodesForPathHash(pathHashIndex, pathHash);
if (peers.length <= 1) return 0;
const inRangeKm = Math.max(PREFIX_AMBIGUITY_FLOOR_KM, nodeRange(candidate.node_id, context.coverageByNode), nodeRange(prevNode.node_id, context.coverageByNode));
@@ -677,9 +670,8 @@ function resolveBetaPath(
return clamp(localPenalty + hopPenalty, 0, 0.30);
}
function clashPressure(candidate: MeshNode): number {
const prefix = candidate.node_id.slice(0, 2).toUpperCase();
const peers = prefixBuckets.get(prefix) ?? [];
function clashPressure(candidate: MeshNode, pathHash: string): number {
const peers = getNodesForPathHash(pathHashIndex, pathHash);
if (peers.length <= 1) return 0;
let raw = 0;
for (const peer of peers) {
@@ -691,7 +683,7 @@ function resolveBetaPath(
return clamp(raw / 3, 0, 1);
}
function inCorridor(candidate: MeshNode, prevNode: MeshNode): boolean {
function inCorridor(candidate: MeshNode, prevNode: MeshNode, pathHash: string): boolean {
if (!hasCoords(src)) return true;
const bx = src.lon! - rxLon;
const by = src.lat! - rxLat;
@@ -700,7 +692,7 @@ function resolveBetaPath(
const px = candidate.lon! - rxLon;
const py = candidate.lat! - rxLat;
const t = (px * bx + py * by) / segLen2;
const pressure = clashPressure(candidate);
const pressure = clashPressure(candidate, pathHash);
const tPadding = 0.15 + (1 - pressure) * 0.15;
if (t < -tPadding || t > 1 + tPadding) return false;
@@ -718,7 +710,7 @@ function resolveBetaPath(
}
function getCandidates(prefix: string, prevPrefix: string, prevNode: MeshNode, nextTowardRx: string | null): Array<{ node: MeshNode; conf: number }> {
const all = candidatesPool.filter((n) => n.node_id.toUpperCase().startsWith(prefix));
const all = getNodesForPathHash(pathHashIndex, prefix);
if (all.length === 0) return [];
function align(c: MeshNode): number {
@@ -733,7 +725,7 @@ function resolveBetaPath(
}
function sortScore(c: MeshNode): number {
const corridorBonus = inCorridor(c, prevNode) ? 0.25 : -0.6;
const corridorBonus = inCorridor(c, prevNode, prefix) ? 0.25 : -0.6;
return align(c) - distKm(c, prevNode) / 50 + corridorBonus;
}
@@ -760,7 +752,7 @@ function resolveBetaPath(
const motifBoost = motifPrior([c.node_id, prevNode.node_id]) * 0.2
+ (nextTowardRx ? motifPrior([c.node_id, prevNode.node_id, nextTowardRx]) * 0.25 : 0);
const edgeBoost = edgePrior(c.node_id, prevNode.node_id) * 0.3;
const ambiguityPenalty = localPrefixAmbiguityPenalty(c, prevNode);
const ambiguityPenalty = localPrefixAmbiguityPenalty(c, prevNode, prefix);
const confirmedFloor = strongConfirmedFloor(meta);
const baseConf = confirmedLinkConfidence(meta, c.node_id, prevNode.node_id, {
prefix: priorBoost,
@@ -775,7 +767,7 @@ function resolveBetaPath(
const reachable = all
.filter((c) => {
if (usedIds.has(c.node_id)) return false;
if (!inCorridor(c, prevNode)) return false;
if (!inCorridor(c, prevNode, prefix)) return false;
const meta = context.linkMetrics.get(linkKey(c.node_id, prevNode.node_id));
const reachOk = canReach(c, prevNode, context.coverageByNode);
const losOk = hasLoS(c, prevNode);
@@ -792,7 +784,7 @@ function resolveBetaPath(
const motifBoost = motifPrior([c.node_id, prevNode.node_id]) * 0.18
+ (nextTowardRx ? motifPrior([c.node_id, prevNode.node_id, nextTowardRx]) * 0.2 : 0);
const edgeBoost = edgePrior(c.node_id, prevNode.node_id) * 0.28;
const ambiguityPenalty = localPrefixAmbiguityPenalty(c, prevNode);
const ambiguityPenalty = localPrefixAmbiguityPenalty(c, prevNode, prefix);
return {
node: c,
conf: Math.max(0.08, 0.2 + prior * 0.34 + prefixBoost + transitionBoost + motifBoost + edgeBoost - distancePenalty - ambiguityPenalty - (all.length - 1) * 0.01),
@@ -802,7 +794,7 @@ function resolveBetaPath(
const fallback = all
.filter((c) => {
if (usedIds.has(c.node_id)) return false;
if (!inCorridor(c, prevNode)) return false;
if (!inCorridor(c, prevNode, prefix)) return false;
if (distKm(c, prevNode) >= MAX_HOP_KM * 0.5) return false;
const meta = context.linkMetrics.get(linkKey(c.node_id, prevNode.node_id));
return hasLoS(c, prevNode) || isWeakOrBetter(meta);
@@ -815,7 +807,7 @@ function resolveBetaPath(
const transitionBoost = transitionPrior(c.node_id, prevNode.node_id) * 0.16;
const motifBoost = motifPrior([c.node_id, prevNode.node_id]) * 0.12;
const edgeBoost = edgePrior(c.node_id, prevNode.node_id) * 0.18;
const ambiguityPenalty = localPrefixAmbiguityPenalty(c, prevNode);
const ambiguityPenalty = localPrefixAmbiguityPenalty(c, prevNode, prefix);
return {
node: c,
conf: Math.max(0.03, 0.04 + prior * 0.2 + prefixBoost + transitionBoost + motifBoost + edgeBoost - ambiguityPenalty) / Math.max(1, all.length),
@@ -825,15 +817,15 @@ function resolveBetaPath(
return [...confirmed, ...reachable, ...fallback];
}
const ambiguity = pathHashes.reduce((sum, h) => sum + (prefixCounts.get(h.slice(0, 2).toUpperCase()) ?? 0), 0);
let budget = Math.max(3_000, Math.min(308_232, 1_000 + pathHashes.length * 3_000 + ambiguity * 800));
const ambiguity = normalizedHashes.reduce((sum, h) => sum + countNodesForPathHash(pathHashIndex, h), 0);
let budget = Math.max(3_000, Math.min(308_232, 1_000 + normalizedHashes.length * 3_000 + ambiguity * 800));
function solve(hopIdx: number, prevNode: MeshNode, nextTowardRx: string | null, visited: Set<string>): HopResult[] | null {
if (hopIdx < 0) return [];
if (--budget <= 0) return null;
const prefix = pathHashes[hopIdx]!.slice(0, 2).toUpperCase();
const prevPrefix = hopIdx > 0 ? pathHashes[hopIdx - 1]!.slice(0, 2).toUpperCase() : '';
const prefix = normalizedHashes[hopIdx]!;
const prevPrefix = hopIdx > 0 ? normalizedHashes[hopIdx - 1]! : '';
const options = getCandidates(prefix, prevPrefix, prevNode, nextTowardRx).filter((o) => !visited.has(o.node.node_id));
for (const opt of options) {
@@ -845,7 +837,7 @@ function resolveBetaPath(
return null;
}
const raw = solve(pathHashes.length - 1, rx, null, new Set([rx.node_id]));
const raw = solve(normalizedHashes.length - 1, rx, null, new Set([rx.node_id]));
if (!raw) return null;
const hops = [...raw].reverse();
@@ -858,8 +850,8 @@ function resolveBetaPath(
const calibratedConfidence = rawConfidence * context.learningModel.confidenceScale + context.learningModel.confidenceBias;
const confidence = clamp(calibratedConfidence, 0, 1);
const srcPrefix = hasCoords(src) ? src.node_id.slice(0, 2).toUpperCase() : null;
const firstHopPrefix = pathHashes[0]?.slice(0, 2).toUpperCase() ?? null;
const firstHopPrefix = normalizedHashes[0] ?? null;
const srcPrefix = hasCoords(src) && firstHopPrefix ? nodePathHash(src.node_id, firstHopPrefix) : null;
const prependSource = options?.disableSourcePrepend
? false
: Boolean(options?.forceIncludeSource
+71
View File
@@ -0,0 +1,71 @@
type NodeWithId = {
node_id: string;
};
export type NodePathHashIndex<T extends NodeWithId> = Map<string, T[]>;
export const SUPPORTED_PATH_HASH_HEX_LENGTHS = [2, 4, 6] as const;
export function normalizePathHash(pathHash: string | null | undefined): string {
return String(pathHash ?? '').trim().toUpperCase();
}
export function nodePathHash(nodeId: string, pathHashOrLength: string | number): string {
const length = typeof pathHashOrLength === 'number'
? pathHashOrLength
: normalizePathHash(pathHashOrLength).length;
if (!Number.isFinite(length) || length <= 0) return '';
return nodeId.slice(0, length).toUpperCase();
}
export function nodeMatchesPathHash(nodeId: string, pathHash: string | null | undefined): boolean {
const normalized = normalizePathHash(pathHash);
return normalized.length > 0 && nodeId.toUpperCase().startsWith(normalized);
}
export function collectPathHashLengths(pathHashes: Iterable<string | null | undefined>): number[] {
const lengths = new Set<number>();
for (const pathHash of pathHashes) {
const normalized = normalizePathHash(pathHash);
if (normalized.length > 0) lengths.add(normalized.length);
}
return Array.from(lengths).sort((a, b) => a - b);
}
export function buildNodePathHashIndex<T extends NodeWithId>(
nodes: Iterable<T>,
hashLengths: Iterable<number> = SUPPORTED_PATH_HASH_HEX_LENGTHS,
): NodePathHashIndex<T> {
const index: NodePathHashIndex<T> = new Map();
const lengths = Array.from(new Set(
Array.from(hashLengths)
.map((length) => Number(length))
.filter((length) => Number.isInteger(length) && length > 0),
)).sort((a, b) => a - b);
for (const node of nodes) {
for (const length of lengths) {
if (node.node_id.length < length) continue;
const key = nodePathHash(node.node_id, length);
const bucket = index.get(key);
if (bucket) bucket.push(node);
else index.set(key, [node]);
}
}
return index;
}
export function getNodesForPathHash<T extends NodeWithId>(
index: NodePathHashIndex<T>,
pathHash: string | null | undefined,
): T[] {
return index.get(normalizePathHash(pathHash)) ?? [];
}
export function countNodesForPathHash<T extends NodeWithId>(
index: NodePathHashIndex<T>,
pathHash: string | null | undefined,
): number {
return getNodesForPathHash(index, pathHash).length;
}
+17 -17
View File
@@ -1,4 +1,10 @@
import { MIN_LINK_OBSERVATIONS, query } from '../db/index.js';
import {
buildNodePathHashIndex,
getNodesForPathHash,
nodePathHash,
normalizePathHash,
} from '../path-hash/utils.js';
type LearningNode = {
node_id: string;
@@ -74,10 +80,10 @@ function recencyScore(lastSeenMs: number | undefined, nowMs: number): number {
function localPrefixAmbiguityPenalty(
anchor: LearningNode,
target: LearningNode,
prefixMap: Map<string, LearningNode[]>,
pathHash: string,
pathHashIndex: Map<string, LearningNode[]>,
): number {
const prefix = target.node_id.slice(0, 2).toUpperCase();
const samePrefixNodes = prefixMap.get(prefix) ?? [];
const samePrefixNodes = getNodesForPathHash(pathHashIndex, pathHash);
if (samePrefixNodes.length <= 1) return 0;
const targetDist = distKm(anchor, target);
@@ -99,7 +105,7 @@ function resolvePathForPacket(
pathHashes: string[],
srcNode: LearningNode | undefined,
rxNode: LearningNode,
prefixMap: Map<string, LearningNode[]>,
pathHashIndex: Map<string, LearningNode[]>,
confirmedLinks: Set<string>,
): ResolvedHop[] {
const resolved: ResolvedHop[] = [];
@@ -107,8 +113,8 @@ function resolvePathForPacket(
let prev = rxNode;
for (let i = pathHashes.length - 1; i >= 0; i--) {
const prefix = pathHashes[i]!.slice(0, 2).toUpperCase();
const candidates = (prefixMap.get(prefix) ?? []).filter((n) => !visited.has(n.node_id));
const prefix = normalizePathHash(pathHashes[i]);
const candidates = getNodesForPathHash(pathHashIndex, prefix).filter((n) => !visited.has(n.node_id));
if (candidates.length === 0) continue;
let best: LearningNode | null = null;
@@ -118,7 +124,7 @@ function resolvePathForPacket(
const confirmed = confirmedLinks.has(linkKey(candidate.node_id, prev.node_id)) ? 1 : 0;
const distanceScore = distancePrior(candidate, prev);
const srcScore = srcNode ? (distKm(srcNode, prev) - distKm(srcNode, candidate)) / 100 : 0;
const ambiguityPenalty = localPrefixAmbiguityPenalty(prev, candidate, prefixMap);
const ambiguityPenalty = localPrefixAmbiguityPenalty(prev, candidate, prefix, pathHashIndex);
const score = confirmed * 2.5 + distanceScore * 1.3 + srcScore - ambiguityPenalty;
if (score > bestScore) {
best = candidate;
@@ -213,13 +219,7 @@ async function rebuildNetwork(modelNetwork: string, sourceNetwork: string | unde
);
const nodesById = new Map(nodesResult.rows.map((n) => [n.node_id, n]));
const prefixMap = new Map<string, LearningNode[]>();
for (const node of nodesResult.rows) {
const prefix = node.node_id.slice(0, 2).toUpperCase();
const existing = prefixMap.get(prefix);
if (existing) existing.push(node);
else prefixMap.set(prefix, [node]);
}
const pathHashIndex = buildNodePathHashIndex(nodesResult.rows);
const linksResult = await query<LearningLink>(
`SELECT nl.node_a_id, nl.node_b_id, nl.itm_path_loss_db, nl.count_a_to_b, nl.count_b_to_a
@@ -273,14 +273,14 @@ async function rebuildNetwork(modelNetwork: string, sourceNetwork: string | unde
let confidenceSum = 0;
for (const packet of packetsResult.rows) {
const hashes = packet.path_hashes?.map((h) => h.slice(0, 2).toUpperCase()) ?? [];
const hashes = packet.path_hashes?.map(normalizePathHash).filter(Boolean) ?? [];
if (hashes.length === 0) continue;
const rx = nodesById.get(packet.rx_node_id);
if (!rx) continue;
const src = packet.src_node_id ? nodesById.get(packet.src_node_id) : undefined;
const region = rx.iata ?? 'unknown';
const resolved = resolvePathForPacket(hashes, src, rx, prefixMap, confirmedLinks);
const resolved = resolvePathForPacket(hashes, src, rx, pathHashIndex, confirmedLinks);
if (resolved.length === 0) continue;
const ts = new Date(packet.time);
@@ -429,7 +429,7 @@ async function rebuildNetwork(modelNetwork: string, sourceNetwork: string | unde
const dKm = distKm(fromNode, toNode);
if (dKm > 55 && pathLoss > 150) consistencyPenalty += 0.14;
}
consistencyPenalty += localPrefixAmbiguityPenalty(fromNode, toNode, prefixMap);
consistencyPenalty += localPrefixAmbiguityPenalty(fromNode, toNode, nodePathHash(toNode.node_id, 2), pathHashIndex);
}
if (observed < 3 && missing >= 3) consistencyPenalty += 0.1;
if (directional < 0.06 && observed >= 5) consistencyPenalty += 0.06;
+1 -1
View File
@@ -58,7 +58,7 @@ export interface LivePacket {
direction?: string; // 'rx' | 'tx' from mctomqtt
summary?: string; // human-readable decoded content
payload?: Record<string, unknown>;
path?: string[]; // relay node 2-char hex prefixes in hop order
path?: string[]; // relay hop hashes in packet order (1/2/3-byte => 2/4/6 hex chars)
advertCount?: number; // for Advert packets: persistent DB count after this event
ts: number;
}
+20 -16
View File
@@ -1,5 +1,12 @@
import 'node:process';
import { initDb, query } from '../db/index.js';
import {
buildNodePathHashIndex,
countNodesForPathHash,
getNodesForPathHash,
nodeMatchesPathHash,
normalizePathHash,
} from '../path-hash/utils.js';
const RUN_INTERVAL_MS = Number(process.env['PATH_SIM_INTERVAL_MS'] ?? 6 * 60 * 60 * 1000);
const PERM_BUCKET_CAP = Number(process.env['PATH_SIM_PERM_BUCKET_CAP'] ?? 50);
@@ -274,14 +281,14 @@ function lookupCalibration(model: LearningModel, network: string): { scale: numb
function localHexClashPenalty(
candidate: SimNode,
current: SimNode,
pathHash: string,
prefixCounts: Map<string, number>,
byPrefix: Map<string, SimNode[]>,
weakAdjacency: Map<string, Set<string>>,
): number {
const prefix = candidate.node_id.slice(0, 2).toUpperCase();
const total = prefixCounts.get(prefix) ?? 0;
const total = prefixCounts.get(pathHash) ?? 0;
if (total <= 1) return 0;
const peers = byPrefix.get(prefix) ?? [];
const peers = byPrefix.get(pathHash) ?? [];
let raw = 0;
const inRangeKm = 75;
for (const peer of peers) {
@@ -316,7 +323,7 @@ function scoreCandidate(ctx: CandidateScoreContext, prefixCounts: Map<string, nu
? lookupMotifPrior(model, ctx.packetNetwork, ctx.receiverRegion, ctx.hourBucket, [candidate.node_id, current.node_id, ctx.nextTowardRx])
: 0;
const ambiguityPenalty = localHexClashPenalty(candidate, current, prefixCounts, byPrefix, weakAdjacency);
const ambiguityPenalty = localHexClashPenalty(candidate, current, ctx.prefix, prefixCounts, byPrefix, weakAdjacency);
const weakAdjacencyBoost = weakAdjacency.get(candidate.node_id)?.has(current.node_id) ? 0.04 : 0;
const veryHighLossPenalty = pathLoss != null && pathLoss > 145 ? 0.12 : 0;
@@ -350,11 +357,11 @@ function getCandidatesForPrefix(
receiverRegion: string,
hourBucket: number,
): Array<{ node: SimNode; conf: number }> {
const pool = context.byPrefix.get(prefix) ?? [];
const pool = getNodesForPathHash(context.byPrefix, prefix);
const scored: Array<{ node: SimNode; conf: number }> = [];
for (const node of pool) {
if (visited.has(node.node_id)) continue;
if (node.node_id === srcNodeId && prefix !== srcNodeId.slice(0, 2).toUpperCase()) continue;
if (node.node_id === srcNodeId && !nodeMatchesPathHash(srcNodeId, prefix)) continue;
const d = distKm(node, current);
if (d > strategy.maxHopKm) continue;
const conf = scoreCandidate({
@@ -396,7 +403,7 @@ function enumerateContinuationsMl(
return { fullCount: 0, partialCount: 0, longestPrefixDepth: 0, truncated: false, bestConfidence: 0 };
}
const prefixes = pathHashes.map((h) => h.slice(0, 2).toUpperCase());
const prefixes = pathHashes.map(normalizePathHash).filter(Boolean);
const receiverRegion = rx.iata ?? 'unknown';
const hourBucket = currentHourBucket(packetTime, context.config);
const calibration = lookupCalibration(context.model, packetNetwork);
@@ -500,16 +507,13 @@ async function loadNodes(): Promise<{ byId: Map<string, SimNode>; byPrefix: Map<
);
const byId = new Map<string, SimNode>();
const byPrefix = new Map<string, SimNode[]>();
const prefixCounts = new Map<string, number>();
for (const row of nodeRows.rows) {
byId.set(row.node_id, row);
const prefix = row.node_id.slice(0, 2).toUpperCase();
const arr = byPrefix.get(prefix);
if (arr) arr.push(row);
else byPrefix.set(prefix, [row]);
prefixCounts.set(prefix, (prefixCounts.get(prefix) ?? 0) + 1);
}
const byPrefix = buildNodePathHashIndex(nodeRows.rows);
const prefixCounts = new Map<string, number>(
Array.from(byPrefix.entries()).map(([prefix, nodes]) => [prefix, nodes.length]),
);
return { byId, byPrefix, prefixCounts };
}
@@ -824,7 +828,7 @@ async function runOnce(
bump(packetTypeTotals, String(row.packet_type ?? -1));
const rx = row.rx_node_id ? context.byId.get(row.rx_node_id) : undefined;
const src = row.src_node_id ? context.byId.get(row.src_node_id) : undefined;
const hashes = row.path_hashes ?? [];
const hashes = row.path_hashes?.map(normalizePathHash).filter(Boolean) ?? [];
const hops = row.hop_count != null ? hashes.slice(0, Math.max(0, row.hop_count)) : hashes;
const isDirectNoPath = hops.length === 0 && row.hop_count === 0;
const packetNetwork = (row.network ?? NETWORK ?? 'teesside').trim().toLowerCase() || 'teesside';
@@ -838,7 +842,7 @@ async function runOnce(
bump(skipReasons, 'no_path_hashes_for_multihop_or_unknown_hops');
continue;
}
if (hops.length > 0 && hops.some((h) => !context.byPrefix.has(h.slice(0, 2).toUpperCase()))) {
if (hops.length > 0 && hops.some((h) => countNodesForPathHash(context.byPrefix, h) < 1)) {
bump(skipReasons, 'unmapped_prefix_no_logged_repeater');
continue;
}
@@ -13,10 +13,11 @@ export const DisclaimerModal: React.FC<DisclaimerModalProps> = ({ onClose }) =>
<h3>Packet paths</h3>
<p>
The relay paths shown on this dashboard are a best estimate. MeshCore packets include
only the first 2 hex characters of each relay node&apos;s ID, so when resolving a path we
match those 2 characters against known nodes. If multiple nodes share the same prefix
the closest candidate is chosen, but the actual path the packet took may have been
different.
relay node ID hashes in the width chosen by the sending node. Depending on firmware and
settings that can be 1, 2, or 3 bytes per hop, so path resolution matches the exact
hash width present in the packet against known nodes. If multiple nodes still share the
same hash at that width, the closest or strongest candidate is chosen, but the actual
path the packet took may have been different.
</p>
</section>
<section>
+33 -9
View File
@@ -26,7 +26,7 @@ export interface LivePacketData {
direction?: string;
summary?: string;
payload?: Record<string, unknown>;
path?: string[]; // relay node 2-char hex prefixes in hop order
path?: string[]; // relay hop hashes in packet order (1/2/3-byte => 2/4/6 hex chars)
advertCount?: number; // for Advert packets: persistent count from DB
ts: number;
}
@@ -64,6 +64,18 @@ interface HashRecord {
ts: number;
}
function packetInfoScore(packet: Pick<AggregatedPacket, 'packetType' | 'srcNodeId' | 'summary' | 'hopCount' | 'path' | 'advertCount'>): number {
let score = 0;
if (packet.summary) score += 4;
if (packet.srcNodeId) score += 3;
if (packet.packetType === 4) score += 2;
else if (packet.packetType !== undefined) score += 1;
if (packet.hopCount !== undefined) score += 1;
if (packet.path && packet.path.length > 0) score += 1;
if ((packet.advertCount ?? 0) > 0) score += 1;
return score;
}
export function useNodes() {
const [nodes, setNodes] = useState<Map<string, MeshNode>>(new Map());
const [packets, setPackets] = useState<AggregatedPacket[]>([]);
@@ -106,9 +118,9 @@ export function useNodes() {
for (const row of data.packets) {
if (seen.has(row.packet_hash)) continue;
seen.add(row.packet_hash);
// For Advert packets the payload stores the decoded inner payload with appData.name
const appData = row.payload?.['appData'] as Record<string, unknown> | undefined;
const summary = row.packet_type === 4 ? appData?.['name'] as string | undefined : undefined;
const summary = (appData?.['name'] as string | undefined)
?? (row.payload?.['origin'] as string | undefined);
initialPackets.push({
id: row.packet_hash,
packetHash: row.packet_hash,
@@ -134,12 +146,24 @@ export function useNodes() {
if (idx >= 0) {
// Known packet — increment count, bubble to top
const current = prev[idx]!;
const candidate: AggregatedPacket = {
...current,
packetType: packet.packetType ?? current.packetType,
srcNodeId: packet.srcNodeId ?? current.srcNodeId,
summary: packet.summary ?? (packet.payload?.['origin'] as string | undefined) ?? current.summary,
hopCount: packet.hopCount ?? current.hopCount,
path: packet.path ?? current.path,
advertCount: Math.max(current.advertCount ?? 0, packet.advertCount ?? 0) || undefined,
rxCount: current.rxCount + (packet.direction !== 'tx' ? 1 : 0),
txCount: current.txCount + (packet.direction === 'tx' ? 1 : 0),
ts: packet.ts,
};
const useCandidate = packetInfoScore(candidate) >= packetInfoScore(current);
const entry: AggregatedPacket = {
...prev[idx]!,
rxCount: prev[idx]!.rxCount + (packet.direction !== 'tx' ? 1 : 0),
txCount: prev[idx]!.txCount + (packet.direction === 'tx' ? 1 : 0),
// Adopt summary if we now have one and didn't before
summary: prev[idx]!.summary ?? packet.summary,
...(useCandidate ? candidate : current),
rxCount: current.rxCount + (packet.direction !== 'tx' ? 1 : 0),
txCount: current.txCount + (packet.direction === 'tx' ? 1 : 0),
ts: packet.ts,
};
const next = prev.filter((_, i) => i !== idx);
@@ -152,7 +176,7 @@ export function useNodes() {
packetType: packet.packetType,
rxNodeId: packet.rxNodeId,
srcNodeId: packet.srcNodeId,
summary: packet.summary,
summary: packet.summary ?? (packet.payload?.['origin'] as string | undefined),
hopCount: packet.hopCount,
path: packet.path,
rxCount: packet.direction !== 'tx' ? 1 : 0,
+37
View File
@@ -33,6 +33,15 @@ type HealthPayload = {
stale_threshold_minutes: number;
global_last_packet_at: string | null;
};
path_hashes: {
last_24h_hops: {
one_byte: number;
two_byte: number;
three_byte: number;
};
multibyte_packets_24h: number;
latest_multibyte_at: string | null;
};
};
function timeAgo(ts: string | null): string {
@@ -192,6 +201,34 @@ export const HealthPage: React.FC = () => {
<div className="health-kv"><span>Disk</span><strong>{fmtGb(data?.system.disk.used_gb)} / {fmtGb(data?.system.disk.total_gb)}</strong></div>
</div>
</section>
<section className="prose-section">
<h2>Path Hashes</h2>
<div className="site-stats-grid site-stats-grid--4 health-system-grid">
<div className="site-stat">
<span className="site-stat__value">{fmtInt(data?.path_hashes.last_24h_hops.one_byte)}</span>
<span className="site-stat__label">1-byte Hops (24h)</span>
</div>
<div className="site-stat">
<span className="site-stat__value">{fmtInt(data?.path_hashes.last_24h_hops.two_byte)}</span>
<span className="site-stat__label">2-byte Hops (24h)</span>
</div>
<div className="site-stat">
<span className="site-stat__value">{fmtInt(data?.path_hashes.last_24h_hops.three_byte)}</span>
<span className="site-stat__label">3-byte Hops (24h)</span>
</div>
<div className="site-stat">
<span className="site-stat__value">{fmtInt(data?.path_hashes.multibyte_packets_24h)}</span>
<span className="site-stat__label">Multibyte Packets (24h)</span>
</div>
</div>
<div className="health-meta">
<div className="health-kv">
<span>Latest Multibyte Packet</span>
<strong>{data?.path_hashes.latest_multibyte_at ? timeAgo(data.path_hashes.latest_multibyte_at) : 'not seen yet'}</strong>
</div>
</div>
</section>
</div>
</>
);
+14 -6
View File
@@ -674,13 +674,19 @@ def process_link_job(db, r_client, job: dict):
((a['lon'] - b['lon']) * 111.32 * cos_m) ** 2
)
def local_prefix_ambiguity_penalty(prefix: str, target_id: str, target_node: dict, anchor_node: dict, pool: list[tuple[str, dict]]) -> float:
def normalize_path_hash(value) -> str:
return str(value or '').strip().upper()
def node_matches_path_hash(node_id: str, path_hash: str) -> bool:
return bool(path_hash) and node_id.upper().startswith(path_hash)
def local_prefix_ambiguity_penalty(path_hash: str, target_id: str, target_node: dict, anchor_node: dict, pool: list[tuple[str, dict]]) -> float:
target_dist = node_dist(target_node, anchor_node)
raw = 0.0
for cand_id, cand_node in pool:
if cand_id == target_id:
continue
if not cand_id.upper().startswith(prefix):
if not node_matches_path_hash(cand_id, path_hash):
continue
cand_dist = node_dist(cand_node, anchor_node)
if cand_dist > PREFIX_AMBIGUITY_RADIUS_KM:
@@ -699,11 +705,13 @@ def process_link_job(db, r_client, job: dict):
prev = rx
visited = {rx_node_id}
for prefix in reversed(path_hashes):
prefix = prefix[:2].upper()
for raw_hash in reversed(path_hashes):
path_hash = normalize_path_hash(raw_hash)
if not path_hash:
continue
candidates = [
(nid, nd) for nid, nd in all_nodes.items()
if nid.upper().startswith(prefix)
if node_matches_path_hash(nid, path_hash)
and nid not in visited
and (nd['role'] is None or nd['role'] == 2)
and nd['name'] and '🚫' not in nd['name']
@@ -719,7 +727,7 @@ def process_link_job(db, r_client, job: dict):
for nid, nd in candidates:
confirmed_bonus = 2.5 if confirmed_link(nid, prev_id) else 0.0
distance_score = -node_dist(nd, prev) / 12.0
ambiguity_penalty = local_prefix_ambiguity_penalty(prefix, nid, nd, prev, candidates)
ambiguity_penalty = local_prefix_ambiguity_penalty(path_hash, nid, nd, prev, candidates)
score = confirmed_bonus + distance_score - ambiguity_penalty
if score > best_score:
best_score = score