diff --git a/backend/src/mqtt/channelRegistry.ts b/backend/src/mqtt/channelRegistry.ts new file mode 100644 index 0000000..714220f --- /dev/null +++ b/backend/src/mqtt/channelRegistry.ts @@ -0,0 +1,163 @@ +/** + * Channel registry — the single source of truth for MeshCore channel secrets. + * + * VALIDATED_CHANNELS are the default secrets baked into the service (all + * recovered/community-known channels, verified to decrypt real group text). + * MESHCORE_CHANNEL_SECRETS (env, 'name:hex' or bare 'hex', comma-separated) + * APPENDS extra channels on top — useful for community-shared secrets that + * shouldn't be committed. + * + * Also hosts the summary + channel-identification helpers so the ingest + * pipeline and offline tools (e.g. backfillDecrypt) share one implementation. + */ +import { MeshCoreDecoder } from '@michaelhart/meshcore-decoder'; +import type { GroupTextPayload } from '@michaelhart/meshcore-decoder'; +import { decodePacketCompat } from './decodePacket.js'; +import { BoundedTtlMap } from '../cache/boundedTtlMap.js'; + +export interface ChannelEntry { + name: string; + secret: string; + keyStore: ReturnType; +} + +/** + * Default channels — recovered via the meshcore wordlist/derivation audit + * (2026-08-06) and validated: each key decrypts real human-readable group + * text (sane epoch timestamp + printable text) on the UK Mesh network. + */ +export const VALIDATED_CHANNELS: ReadonlyArray<{ name: string; secret: string }> = [ + { name: 'Public', secret: '8b3387e9c5cdea6ac9e5edbaa115cd72' }, + { name: 'test', secret: '9cd8fcf22a47333b591d96a2b848b73f' }, + { name: 'bot', secret: 'eb50a1bcb3e4e5d7bf69a57c9dada211' }, + { name: 'yorkshire', secret: 'ea19e9567eec1b148c174f0cb9d04dd7' }, + { name: 'liverpool', secret: '2a223c8f7b4190b329f46d54d3a9c350' }, + { name: 'london', secret: '9881d2b7ab9105a41a8d0f6ba449447e' }, + { name: 'nottingham', secret: '90f554aa06b30c705e7ed3bdd59b38e2' }, + { name: 'northeast', secret: '3b1ca0ae6003193eb9f91984eefef5dc' }, + { name: 'kent', secret: 'dea8bb392991eeee26806e4d2cdf8dd4' }, + { name: 'wales', secret: '809573d8134fa262d284400a788f63d9' }, + { name: 'cornwall', secret: '88b113ee8130e832d0bf15a5c7be0ddd' }, + { name: 'devon', secret: 'cbd7337937131f7db971cfdb24431ae3' }, + { name: 'dorset', secret: 'd2de225d13760cf2e13aa368ef9e5b6b' }, + { name: 'cumbria', secret: 'ec3be8009f600059d6cd1dca32c74e1c' }, + { name: 'yorks', secret: '81eb980a4884e5859bb42b135261200e' }, + { name: 'leicester', secret: '2ca4f626387ca26ec7c2ae45641f9628' }, + { name: 'dartford', secret: '848469a776c39f8bf2b774def5cd697b' }, + { name: 'uckfield', secret: '15ab8ecee7b54d18adffd4b83be33682' }, + { name: 'huddersfield', secret: '549a4cd97d96c130d359748c72f0e321' }, + { name: 'derbyshire', secret: '4c3e09555ad74fde17f3a82febb610c4' }, + { name: 'lincolnshire', secret: '2db23106132a517f0b9ea47b4d822c4f' }, + { name: 'surrey', secret: '5e12bb2d2b8f660ab179676cbc663cd1' }, + { name: 'midlands', secret: 'ba3707a0760d156169aea2a6c82e2d6a' }, + { name: 'hamradio', secret: '83c8b01997654265938da8765cbc7db9' }, + { name: 'mesh', secret: '5b664cde0b08b220612113db980650f3' }, + { name: 'public2', secret: '8b4b705b080c0d943b1c80f6b3ef6b6d' }, + { name: 'test2', secret: '9f86d081884c7d659a2feaa0c55ad015' }, + { name: 'thenorf', secret: '1f19cc976112b77f623ec970e4805e87' }, + { name: 'g8py', secret: 'f4624be48a6c80309b4c2f6f3eb36e99' }, + { name: 'echo', secret: '5d25cf40b1f5b4a7eb0cf9703634a948' }, + { name: 'denhaag', secret: '84eed36f62c7b22527dbf8883585ad14' }, + { name: 'dublin', secret: '8792638977132bc05a1f72d6bb913694' }, + { name: 'glasgow', secret: 'ea6b521f794e7a0e4ff41fe5d0e3f838' }, + { name: 'york', secret: 'd90a1f4941a7892855e473c8f9a10195' }, + { name: 'ireland', secret: '1b2a12acc5db1517d9d407946756b1da' }, + { name: 'scilly', secret: 'eb9ca9764bd08712f596a054e32e336c' }, + { name: 'brentwood', secret: 'b4e5450da06c21e5338b91b3b18206ad' }, + { name: 'marple', secret: '3238ea75dcefd77a31ce907037901f61' }, + { name: 'uk', secret: '22b2eed34b5cc429ce1dc5e88635ff84' }, +]; + +/** Build the channel entry list: committed defaults + env extras (dedup by secret). */ +export function buildChannelEntries(envValue?: string): ChannelEntry[] { + const entries: ChannelEntry[] = VALIDATED_CHANNELS.map(({ name, secret }) => ({ + name, + secret, + keyStore: MeshCoreDecoder.createKeyStore({ channelSecrets: [secret] }), + })); + const seen = new Set(entries.map((e) => e.secret.toLowerCase())); + for (const raw of (envValue ?? '').split(',').map((s) => s.trim()).filter(Boolean)) { + const colon = raw.indexOf(':'); + const name = colon > 0 ? raw.slice(0, colon) : raw.slice(0, 6); + const secret = colon > 0 ? raw.slice(colon + 1) : raw; + if (seen.has(secret.toLowerCase())) continue; + seen.add(secret.toLowerCase()); + entries.push({ name, secret, keyStore: MeshCoreDecoder.createKeyStore({ channelSecrets: [secret] }) }); + } + return entries; +} + +/** Combined keyStore used for decryption — one decode call per packet tries every secret. */ +export function buildCombinedKeyStore(entries: ChannelEntry[]) { + return MeshCoreDecoder.createKeyStore({ channelSecrets: entries.map((e) => e.secret) }); +} + +// Small cache so relay copies of the same GroupText don't trigger re-decodes +const channelCache = new BoundedTtlMap({ + name: 'mqtt_channels', + maxEntries: 200, + maxWeight: 2 * 1024 * 1024, + ttlMs: 10 * 60_000, + weightOf: (key, value) => key.length * 2 + (value?.length ?? 0) * 2, +}); + +/** Identify which channel a GroupText was sent on by trying each single-key keyStore. */ +export function identifyChannel(rawHex: string, entries: ChannelEntry[]): string | undefined { + // Single channel — must be it, no re-decode needed + if (entries.length === 1) return entries[0]!.name; + + if (channelCache.has(rawHex)) return channelCache.get(rawHex) ?? undefined; + + let result: string | undefined; + for (const entry of entries) { + const { decoded: d, metadataValid } = decodePacketCompat(rawHex, entry.keyStore); + if (!metadataValid) continue; + const p = d?.payload?.decoded as GroupTextPayload | undefined; + if (p?.decrypted) { result = entry.name; break; } + } + + channelCache.set(rawHex, result ?? null); + return result; +} + +/** Build a short human-readable summary from a decoded payload. */ +export function buildSummary(payloadType: number, decoded: unknown, rawHex?: string, entries?: ChannelEntry[]): string | undefined { + if (!decoded) return undefined; + + switch (payloadType) { + case 4: { + const p = decoded as { appData?: { name?: string } }; + const name = p.appData?.name; + return name ? `${name}` : undefined; + } + case 5: { + const p = decoded as GroupTextPayload; + if (p.decrypted) { + const sender = p.decrypted.sender ?? '?'; + const channel = rawHex && entries ? identifyChannel(rawHex, entries) : undefined; + const prefix = channel ? `[${channel}] ` : ''; + return `${prefix}${sender}: ${p.decrypted.message}`; + } + return '[encrypted]'; + } + case 2: { + const p = decoded as { decrypted?: { message?: string } }; + if (p.decrypted?.message) return `${p.decrypted.message}`; + return '[encrypted DM]'; + } + case 3: { + const p = decoded as { checksum: string }; + return `ACK ${p.checksum.slice(0, 4)}`; + } + case 8: { + const p = decoded as { pathLength: number }; + return `${p.pathLength} hop path`; + } + case 9: { + const p = decoded as { pathHashes: unknown[] }; + return `trace ${p.pathHashes.length} hops`; + } + default: + return undefined; + } +} diff --git a/backend/src/mqtt/client.ts b/backend/src/mqtt/client.ts index 910ab01..65ad0d9 100644 --- a/backend/src/mqtt/client.ts +++ b/backend/src/mqtt/client.ts @@ -1,9 +1,5 @@ import mqtt, { type MqttClient } from 'mqtt'; -import { MeshCoreDecoder, calcRegionKey, transportCodeMatchesRegion } from '@michaelhart/meshcore-decoder'; -import type { - AdvertPayload, GroupTextPayload, TextMessagePayload, - TracePayload, PathPayload, AckPayload, -} from '@michaelhart/meshcore-decoder'; +import { calcRegionKey, transportCodeMatchesRegion } from '@michaelhart/meshcore-decoder'; import { insertNodeStatusSample, insertPacket, upsertNode, incrementAdvertCount, query, insertOrUpdateSpamSuspect, recordMultibyteEvidence } from '../db/index.js'; import { closePacketBatch, flush as flushPacketBatch } from '../db/packetBatch.js'; import { evaluateAdvert, initSpamDetector } from './spamDetector.js'; @@ -11,9 +7,9 @@ import { invalidateResolveCache, setResolveCache, getStickyNodeMap, mergeStickyN import { resolvePool } from '../path-beta/resolvePool.js'; import { scheduleSlowResolution } from '../path-beta/slowMode.js'; import { pathingConfig } from '../platform/config/pathing.js'; -import { BoundedTtlMap } from '../cache/boundedTtlMap.js'; import type { LivePacket } from '../types/index.js'; import { decodePacketCompat } from './decodePacket.js'; +import { buildChannelEntries, buildCombinedKeyStore, buildSummary } from './channelRegistry.js'; import { shouldDiscardUnverifiedTxAdvert, statusEnvelopeTargetsObserver } from './identityBinding.js'; import { parseMqttTopic } from './topic.js'; import { @@ -354,41 +350,10 @@ function tryCountAdvert(hash: string): boolean { * MESHCORE_CHANNEL_SECRETS supports 'name:hex' or bare 'hex' entries (comma-separated). * The default public channel is always included. */ -interface ChannelEntry { - name: string; - secret: string; - keyStore: ReturnType; -} - -const channelEntries: ChannelEntry[] = [ - { - name: 'Public', - secret: '8b3387e9c5cdea6ac9e5edbaa115cd72', - keyStore: MeshCoreDecoder.createKeyStore({ channelSecrets: ['8b3387e9c5cdea6ac9e5edbaa115cd72'] }), - }, - ...(process.env['MESHCORE_CHANNEL_SECRETS'] - ?.split(',').map((s) => s.trim()).filter(Boolean) - .map((entry) => { - const colon = entry.indexOf(':'); - const name = colon > 0 ? entry.slice(0, colon) : entry.slice(0, 6); - const secret = colon > 0 ? entry.slice(colon + 1) : entry; - return { name, secret, keyStore: MeshCoreDecoder.createKeyStore({ channelSecrets: [secret] }) }; - }) ?? []), -]; +const channelEntries = buildChannelEntries(process.env['MESHCORE_CHANNEL_SECRETS']); // Combined keyStore used for decryption (all secrets, single decode call per packet) -const keyStore = MeshCoreDecoder.createKeyStore({ - channelSecrets: channelEntries.map((e) => e.secret), -}); - -// Small cache so relay copies of the same GroupText don't trigger re-decodes -const channelCache = new BoundedTtlMap({ - name: 'mqtt_channels', - maxEntries: 200, - maxWeight: 2 * 1024 * 1024, - ttlMs: 10 * 60_000, - weightOf: (key, value) => key.length * 2 + (value?.length ?? 0) * 2, -}); +const keyStore = buildCombinedKeyStore(channelEntries); function boundedEnvInteger(name: string, fallback: number, min: number, max: number): number { const parsed = Number(process.env[name]); @@ -459,66 +424,6 @@ function enqueueMqttMessage(topic: string, rawPayload: Buffer): void { drainMqttIngestQueue(); } -/** Identify which channel a GroupText was sent on by trying each single-key keyStore. */ -function identifyChannel(rawHex: string): string | undefined { - // Single channel — must be it, no re-decode needed - if (channelEntries.length === 1) return channelEntries[0]!.name; - - if (channelCache.has(rawHex)) return channelCache.get(rawHex) ?? undefined; - - let result: string | undefined; - for (const entry of channelEntries) { - const { decoded: d, metadataValid } = decodePacketCompat(rawHex, entry.keyStore); - if (!metadataValid) continue; - const p = d?.payload?.decoded as GroupTextPayload | undefined; - if (p?.decrypted) { result = entry.name; break; } - } - - channelCache.set(rawHex, result ?? null); - return result; -} - -/** Build a short human-readable summary from a decoded payload. */ -function buildSummary(payloadType: number, decoded: unknown, rawHex?: string): string | undefined { - if (!decoded) return undefined; - - switch (payloadType) { - case 4: { - const p = decoded as AdvertPayload; - const name = p.appData?.name; - return name ? `${name}` : undefined; - } - case 5: { - const p = decoded as GroupTextPayload; - if (p.decrypted) { - const sender = p.decrypted.sender ?? '?'; - const channel = rawHex ? identifyChannel(rawHex) : undefined; - const prefix = channel ? `[${channel}] ` : ''; - return `${prefix}${sender}: ${p.decrypted.message}`; - } - return '[encrypted]'; - } - case 2: { - const p = decoded as TextMessagePayload; - if (p.decrypted?.message) return `${p.decrypted.message}`; - return '[encrypted DM]'; - } - case 3: { - const p = decoded as AckPayload; - return `ACK ${p.checksum.slice(0, 4)}`; - } - case 8: { - const p = decoded as PathPayload; - return `${p.pathLength} hop path`; - } - case 9: { - const p = decoded as TracePayload; - return `trace ${p.pathHashes.length} hops`; - } - default: - return undefined; - } -} export async function startMqttClient(): Promise { if (mqttStopping) return; @@ -762,7 +667,7 @@ async function handleMessage(topic: string, rawPayload: Buffer): Promise { } const decodedInner = decoded.payload?.decoded; - summary = buildSummary(decoded.payloadType, decodedInner, rawHex); + summary = buildSummary(decoded.payloadType, decodedInner, rawHex, channelEntries); if (decoded.payloadType === 4) { const inner = decodedInner as unknown as Record | undefined; diff --git a/backend/src/tools/backfillDecrypt.ts b/backend/src/tools/backfillDecrypt.ts new file mode 100644 index 0000000..dc1ee8e --- /dev/null +++ b/backend/src/tools/backfillDecrypt.ts @@ -0,0 +1,108 @@ +/** + * Backfill decryption — decrypt stored raw group-text packets retroactively. + * + * Keyset-paginated on packet_hash (uses packets_hash_idx) so each batch is a + * small range scan instead of re-scanning the whole table. Resumable via + * --cursor=. Idempotent: skips hashes already decrypted. + * + * Usage (inside the backend image): + * node dist/tools/backfillDecrypt.js [--days=90] [--batch=500] [--cursor=] + */ +import { buildChannelEntries, buildCombinedKeyStore, buildSummary } from '../mqtt/channelRegistry.js'; +import { decodePacketCompat } from '../mqtt/decodePacket.js'; +import { pool } from '../db/index.js'; + +function log(msg: string): void { + const line = `[backfill ${new Date().toISOString()}] ${msg}`; + process.stdout.write(line + '\n'); +} + +const args = process.argv.slice(2); +const daysArg = args.find((a) => a.startsWith('--days=')); +const batchArg = args.find((a) => a.startsWith('--batch=')); +const cursorArg = args.find((a) => a.startsWith('--cursor=')); +const DAYS = daysArg ? Number(daysArg.split('=')[1]) : 90; +const BATCH = batchArg ? Number(batchArg.split('=')[1]) : 500; +let cursor = cursorArg ? cursorArg.split('=')[1] : ''; + +const channelEntries = buildChannelEntries(process.env['MESHCORE_CHANNEL_SECRETS']); +const keyStore = buildCombinedKeyStore(channelEntries); +log(`channels=${channelEntries.length} days=${DAYS} batch=${BATCH} cursor=${cursor || 'start'}`); + +process.on('unhandledRejection', (err) => { + log(`FATAL unhandledRejection: ${err instanceof Error ? err.stack : String(err)}`); + process.exit(1); +}); + +async function fetchBatch(gt: string, limit: number): Promise> { + const res = await pool.query( + `SELECT DISTINCT packet_hash, raw_hex + FROM packets + WHERE packet_type = 5 + AND raw_hex IS NOT NULL + AND time > NOW() - ($1::int * INTERVAL '1 day') + AND payload->>'decrypted' IS NULL + AND packet_hash > $2 + ORDER BY packet_hash + LIMIT $3`, + [DAYS, gt, limit], + ); + return res.rows as Array<{ packet_hash: string; raw_hex: string }>; +} + +async function applyDecryptions(rows: Array<{ packet_hash: string; decrypted: unknown; summary: string }>): Promise { + await pool.query( + `UPDATE packets + SET payload = payload || jsonb_build_object( + 'decrypted', $1::jsonb, + '_summary', $2::text + ) + WHERE packet_hash = ANY($3::text[])`, + [JSON.stringify(rows.map((r) => r.decrypted)), JSON.stringify(rows.map((r) => r.summary)), rows.map((r) => r.packet_hash)], + ); +} + +async function main(): Promise { + let processed = 0; + let decrypted = 0; + let failed = 0; + const startedAt = Date.now(); + + for (;;) { + const rows = await fetchBatch(cursor, BATCH); + if (rows.length === 0) break; + + const updates: Array<{ packet_hash: string; decrypted: unknown; summary: string }> = []; + for (const row of rows) { + processed += 1; + try { + const { decoded, metadataValid } = decodePacketCompat(row.raw_hex, keyStore); + if (!metadataValid || !decoded?.payload?.decoded) { failed += 1; continue; } + const inner = decoded.payload.decoded as { decrypted?: { sender?: string; message?: string } } | undefined; + if (!inner?.decrypted) { failed += 1; continue; } + const summary = buildSummary(decoded.payloadType, inner, row.raw_hex, channelEntries); + updates.push({ packet_hash: row.packet_hash, decrypted: inner.decrypted, summary: summary ?? '[decrypted]' }); + decrypted += 1; + } catch (err) { + failed += 1; + } + } + + if (updates.length > 0) { + await applyDecryptions(updates); + } + cursor = rows[rows.length - 1]!.packet_hash; + + if (processed % (BATCH * 10) === 0 || rows.length < BATCH) { + log(`progress: ${processed} scanned, ${decrypted} decrypted, ${failed} undecryptable — ${Math.round((Date.now() - startedAt) / 1000)}s — cursor=${cursor.slice(0, 12)}…`); + } + } + + log(`DONE — scanned=${processed} decrypted=${decrypted} undecryptable=${failed} in ${Math.round((Date.now() - startedAt) / 1000)}s`); + await pool.end(); +} + +main().catch((err) => { + log(`FATAL: ${err instanceof Error ? err.stack : String(err)}`); + process.exit(1); +}); diff --git a/frontend/src/pages/ukmesh/UKFeedPage.tsx b/frontend/src/pages/ukmesh/UKFeedPage.tsx index 0f9bd48..5ef4fa6 100644 --- a/frontend/src/pages/ukmesh/UKFeedPage.tsx +++ b/frontend/src/pages/ukmesh/UKFeedPage.tsx @@ -21,6 +21,7 @@ import type { LazyPathResult } from './PacketDetailPanel.js'; import { FeedMapPanel } from './FeedPathViews.js'; import { FeedDialogs } from './FeedDialogs.js'; import { + MESSAGE_SCOPE_CHANNELS, FEED_PATH_MAX_CONCURRENCY, LAZY_SETTLE_MS, MAX_PACKETS, @@ -62,7 +63,9 @@ export const UKFeedPage: React.FC = () => { const [selectedIata, setSelectedIata] = useState(() => localStorage.getItem('uk-feed-iata') ?? 'all'); const [selectedMessageScope, setSelectedMessageScope] = useState(() => { const stored = localStorage.getItem('uk-feed-message-scope'); - return stored === 'public' || stored === 'test' ? stored : 'all'; + return stored === 'all' || (stored !== null && (MESSAGE_SCOPE_CHANNELS as readonly string[]).includes(stored)) + ? (stored as MessageScope) + : 'all'; }); const [messagesOnly, setMessagesOnly] = useState(() => localStorage.getItem('uk-feed-messages-only') === '1'); const [regionOptions, setRegionOptions] = useState([]); @@ -447,27 +450,23 @@ export const UKFeedPage: React.FC = () => { {/* ── Channels sidebar ───────────────────────────────────────── */}