feat: first-class MeshCore channel decryption

- channelRegistry: 39 validated channel secrets committed as the service
  default (Public + 38 community/hashtag channels recovered via the
  derivation audit); MESHCORE_CHANNEL_SECRETS env appends on top
- shared buildSummary/identifyChannel (single implementation for ingest
  and offline tools)
- tools/backfillDecrypt: keyset-paginated retroactive decryption of the
  stored raw_hex corpus (idempotent, resumable)
- feed: channel scope sidebar now lists every decrypted channel
This commit is contained in:
gadgethd committed 2026-08-06 01:00:05 +00:00
1 parent f6f8290c70
commit 61fdcd9d11
5 files changed
+335 -117

No files matched your search

+163
View File
@@ -0,0 +1,163 @@
/**
* Channel registry — the single source of truth for MeshCore channel secrets.
*
* VALIDATED_CHANNELS are the default secrets baked into the service (all
* recovered/community-known channels, verified to decrypt real group text).
* MESHCORE_CHANNEL_SECRETS (env, 'name:hex' or bare 'hex', comma-separated)
* APPENDS extra channels on top — useful for community-shared secrets that
* shouldn't be committed.
*
* Also hosts the summary + channel-identification helpers so the ingest
* pipeline and offline tools (e.g. backfillDecrypt) share one implementation.
*/
import { MeshCoreDecoder } from '@michaelhart/meshcore-decoder';
import type { GroupTextPayload } from '@michaelhart/meshcore-decoder';
import { decodePacketCompat } from './decodePacket.js';
import { BoundedTtlMap } from '../cache/boundedTtlMap.js';
export interface ChannelEntry {
name: string;
secret: string;
keyStore: ReturnType<typeof MeshCoreDecoder.createKeyStore>;
}
/**
* Default channels — recovered via the meshcore wordlist/derivation audit
* (2026-08-06) and validated: each key decrypts real human-readable group
* text (sane epoch timestamp + printable text) on the UK Mesh network.
*/
export const VALIDATED_CHANNELS: ReadonlyArray<{ name: string; secret: string }> = [
{ name: 'Public', secret: '8b3387e9c5cdea6ac9e5edbaa115cd72' },
{ name: 'test', secret: '9cd8fcf22a47333b591d96a2b848b73f' },
{ name: 'bot', secret: 'eb50a1bcb3e4e5d7bf69a57c9dada211' },
{ name: 'yorkshire', secret: 'ea19e9567eec1b148c174f0cb9d04dd7' },
{ name: 'liverpool', secret: '2a223c8f7b4190b329f46d54d3a9c350' },
{ name: 'london', secret: '9881d2b7ab9105a41a8d0f6ba449447e' },
{ name: 'nottingham', secret: '90f554aa06b30c705e7ed3bdd59b38e2' },
{ name: 'northeast', secret: '3b1ca0ae6003193eb9f91984eefef5dc' },
{ name: 'kent', secret: 'dea8bb392991eeee26806e4d2cdf8dd4' },
{ name: 'wales', secret: '809573d8134fa262d284400a788f63d9' },
{ name: 'cornwall', secret: '88b113ee8130e832d0bf15a5c7be0ddd' },
{ name: 'devon', secret: 'cbd7337937131f7db971cfdb24431ae3' },
{ name: 'dorset', secret: 'd2de225d13760cf2e13aa368ef9e5b6b' },
{ name: 'cumbria', secret: 'ec3be8009f600059d6cd1dca32c74e1c' },
{ name: 'yorks', secret: '81eb980a4884e5859bb42b135261200e' },
{ name: 'leicester', secret: '2ca4f626387ca26ec7c2ae45641f9628' },
{ name: 'dartford', secret: '848469a776c39f8bf2b774def5cd697b' },
{ name: 'uckfield', secret: '15ab8ecee7b54d18adffd4b83be33682' },
{ name: 'huddersfield', secret: '549a4cd97d96c130d359748c72f0e321' },
{ name: 'derbyshire', secret: '4c3e09555ad74fde17f3a82febb610c4' },
{ name: 'lincolnshire', secret: '2db23106132a517f0b9ea47b4d822c4f' },
{ name: 'surrey', secret: '5e12bb2d2b8f660ab179676cbc663cd1' },
{ name: 'midlands', secret: 'ba3707a0760d156169aea2a6c82e2d6a' },
{ name: 'hamradio', secret: '83c8b01997654265938da8765cbc7db9' },
{ name: 'mesh', secret: '5b664cde0b08b220612113db980650f3' },
{ name: 'public2', secret: '8b4b705b080c0d943b1c80f6b3ef6b6d' },
{ name: 'test2', secret: '9f86d081884c7d659a2feaa0c55ad015' },
{ name: 'thenorf', secret: '1f19cc976112b77f623ec970e4805e87' },
{ name: 'g8py', secret: 'f4624be48a6c80309b4c2f6f3eb36e99' },
{ name: 'echo', secret: '5d25cf40b1f5b4a7eb0cf9703634a948' },
{ name: 'denhaag', secret: '84eed36f62c7b22527dbf8883585ad14' },
{ name: 'dublin', secret: '8792638977132bc05a1f72d6bb913694' },
{ name: 'glasgow', secret: 'ea6b521f794e7a0e4ff41fe5d0e3f838' },
{ name: 'york', secret: 'd90a1f4941a7892855e473c8f9a10195' },
{ name: 'ireland', secret: '1b2a12acc5db1517d9d407946756b1da' },
{ name: 'scilly', secret: 'eb9ca9764bd08712f596a054e32e336c' },
{ name: 'brentwood', secret: 'b4e5450da06c21e5338b91b3b18206ad' },
{ name: 'marple', secret: '3238ea75dcefd77a31ce907037901f61' },
{ name: 'uk', secret: '22b2eed34b5cc429ce1dc5e88635ff84' },
];
/** Build the channel entry list: committed defaults + env extras (dedup by secret). */
export function buildChannelEntries(envValue?: string): ChannelEntry[] {
const entries: ChannelEntry[] = VALIDATED_CHANNELS.map(({ name, secret }) => ({
name,
secret,
keyStore: MeshCoreDecoder.createKeyStore({ channelSecrets: [secret] }),
}));
const seen = new Set(entries.map((e) => e.secret.toLowerCase()));
for (const raw of (envValue ?? '').split(',').map((s) => s.trim()).filter(Boolean)) {
const colon = raw.indexOf(':');
const name = colon > 0 ? raw.slice(0, colon) : raw.slice(0, 6);
const secret = colon > 0 ? raw.slice(colon + 1) : raw;
if (seen.has(secret.toLowerCase())) continue;
seen.add(secret.toLowerCase());
entries.push({ name, secret, keyStore: MeshCoreDecoder.createKeyStore({ channelSecrets: [secret] }) });
}
return entries;
}
/** Combined keyStore used for decryption — one decode call per packet tries every secret. */
export function buildCombinedKeyStore(entries: ChannelEntry[]) {
return MeshCoreDecoder.createKeyStore({ channelSecrets: entries.map((e) => e.secret) });
}
// Small cache so relay copies of the same GroupText don't trigger re-decodes
const channelCache = new BoundedTtlMap<string, string | null>({
name: 'mqtt_channels',
maxEntries: 200,
maxWeight: 2 * 1024 * 1024,
ttlMs: 10 * 60_000,
weightOf: (key, value) => key.length * 2 + (value?.length ?? 0) * 2,
});
/** Identify which channel a GroupText was sent on by trying each single-key keyStore. */
export function identifyChannel(rawHex: string, entries: ChannelEntry[]): string | undefined {
// Single channel — must be it, no re-decode needed
if (entries.length === 1) return entries[0]!.name;
if (channelCache.has(rawHex)) return channelCache.get(rawHex) ?? undefined;
let result: string | undefined;
for (const entry of entries) {
const { decoded: d, metadataValid } = decodePacketCompat(rawHex, entry.keyStore);
if (!metadataValid) continue;
const p = d?.payload?.decoded as GroupTextPayload | undefined;
if (p?.decrypted) { result = entry.name; break; }
}
channelCache.set(rawHex, result ?? null);
return result;
}
/** Build a short human-readable summary from a decoded payload. */
export function buildSummary(payloadType: number, decoded: unknown, rawHex?: string, entries?: ChannelEntry[]): string | undefined {
if (!decoded) return undefined;
switch (payloadType) {
case 4: {
const p = decoded as { appData?: { name?: string } };
const name = p.appData?.name;
return name ? `${name}` : undefined;
}
case 5: {
const p = decoded as GroupTextPayload;
if (p.decrypted) {
const sender = p.decrypted.sender ?? '?';
const channel = rawHex && entries ? identifyChannel(rawHex, entries) : undefined;
const prefix = channel ? `[${channel}] ` : '';
return `${prefix}${sender}: ${p.decrypted.message}`;
}
return '[encrypted]';
}
case 2: {
const p = decoded as { decrypted?: { message?: string } };
if (p.decrypted?.message) return `${p.decrypted.message}`;
return '[encrypted DM]';
}
case 3: {
const p = decoded as { checksum: string };
return `ACK ${p.checksum.slice(0, 4)}`;
}
case 8: {
const p = decoded as { pathLength: number };
return `${p.pathLength} hop path`;
}
case 9: {
const p = decoded as { pathHashes: unknown[] };
return `trace ${p.pathHashes.length} hops`;
}
default:
return undefined;
}
}
+5 -100
View File
@@ -1,9 +1,5 @@
import mqtt, { type MqttClient } from 'mqtt';
import { MeshCoreDecoder, calcRegionKey, transportCodeMatchesRegion } from '@michaelhart/meshcore-decoder';
import type {
AdvertPayload, GroupTextPayload, TextMessagePayload,
TracePayload, PathPayload, AckPayload,
} from '@michaelhart/meshcore-decoder';
import { calcRegionKey, transportCodeMatchesRegion } from '@michaelhart/meshcore-decoder';
import { insertNodeStatusSample, insertPacket, upsertNode, incrementAdvertCount, query, insertOrUpdateSpamSuspect, recordMultibyteEvidence } from '../db/index.js';
import { closePacketBatch, flush as flushPacketBatch } from '../db/packetBatch.js';
import { evaluateAdvert, initSpamDetector } from './spamDetector.js';
@@ -11,9 +7,9 @@ import { invalidateResolveCache, setResolveCache, getStickyNodeMap, mergeStickyN
import { resolvePool } from '../path-beta/resolvePool.js';
import { scheduleSlowResolution } from '../path-beta/slowMode.js';
import { pathingConfig } from '../platform/config/pathing.js';
import { BoundedTtlMap } from '../cache/boundedTtlMap.js';
import type { LivePacket } from '../types/index.js';
import { decodePacketCompat } from './decodePacket.js';
import { buildChannelEntries, buildCombinedKeyStore, buildSummary } from './channelRegistry.js';
import { shouldDiscardUnverifiedTxAdvert, statusEnvelopeTargetsObserver } from './identityBinding.js';
import { parseMqttTopic } from './topic.js';
import {
@@ -354,41 +350,10 @@ function tryCountAdvert(hash: string): boolean {
* MESHCORE_CHANNEL_SECRETS supports 'name:hex' or bare 'hex' entries (comma-separated).
* The default public channel is always included.
*/
interface ChannelEntry {
name: string;
secret: string;
keyStore: ReturnType<typeof MeshCoreDecoder.createKeyStore>;
}
const channelEntries: ChannelEntry[] = [
{
name: 'Public',
secret: '8b3387e9c5cdea6ac9e5edbaa115cd72',
keyStore: MeshCoreDecoder.createKeyStore({ channelSecrets: ['8b3387e9c5cdea6ac9e5edbaa115cd72'] }),
},
...(process.env['MESHCORE_CHANNEL_SECRETS']
?.split(',').map((s) => s.trim()).filter(Boolean)
.map((entry) => {
const colon = entry.indexOf(':');
const name = colon > 0 ? entry.slice(0, colon) : entry.slice(0, 6);
const secret = colon > 0 ? entry.slice(colon + 1) : entry;
return { name, secret, keyStore: MeshCoreDecoder.createKeyStore({ channelSecrets: [secret] }) };
}) ?? []),
];
const channelEntries = buildChannelEntries(process.env['MESHCORE_CHANNEL_SECRETS']);
// Combined keyStore used for decryption (all secrets, single decode call per packet)
const keyStore = MeshCoreDecoder.createKeyStore({
channelSecrets: channelEntries.map((e) => e.secret),
});
// Small cache so relay copies of the same GroupText don't trigger re-decodes
const channelCache = new BoundedTtlMap<string, string | null>({
name: 'mqtt_channels',
maxEntries: 200,
maxWeight: 2 * 1024 * 1024,
ttlMs: 10 * 60_000,
weightOf: (key, value) => key.length * 2 + (value?.length ?? 0) * 2,
});
const keyStore = buildCombinedKeyStore(channelEntries);
function boundedEnvInteger(name: string, fallback: number, min: number, max: number): number {
const parsed = Number(process.env[name]);
@@ -459,66 +424,6 @@ function enqueueMqttMessage(topic: string, rawPayload: Buffer): void {
drainMqttIngestQueue();
}
/** Identify which channel a GroupText was sent on by trying each single-key keyStore. */
function identifyChannel(rawHex: string): string | undefined {
// Single channel — must be it, no re-decode needed
if (channelEntries.length === 1) return channelEntries[0]!.name;
if (channelCache.has(rawHex)) return channelCache.get(rawHex) ?? undefined;
let result: string | undefined;
for (const entry of channelEntries) {
const { decoded: d, metadataValid } = decodePacketCompat(rawHex, entry.keyStore);
if (!metadataValid) continue;
const p = d?.payload?.decoded as GroupTextPayload | undefined;
if (p?.decrypted) { result = entry.name; break; }
}
channelCache.set(rawHex, result ?? null);
return result;
}
/** Build a short human-readable summary from a decoded payload. */
function buildSummary(payloadType: number, decoded: unknown, rawHex?: string): string | undefined {
if (!decoded) return undefined;
switch (payloadType) {
case 4: {
const p = decoded as AdvertPayload;
const name = p.appData?.name;
return name ? `${name}` : undefined;
}
case 5: {
const p = decoded as GroupTextPayload;
if (p.decrypted) {
const sender = p.decrypted.sender ?? '?';
const channel = rawHex ? identifyChannel(rawHex) : undefined;
const prefix = channel ? `[${channel}] ` : '';
return `${prefix}${sender}: ${p.decrypted.message}`;
}
return '[encrypted]';
}
case 2: {
const p = decoded as TextMessagePayload;
if (p.decrypted?.message) return `${p.decrypted.message}`;
return '[encrypted DM]';
}
case 3: {
const p = decoded as AckPayload;
return `ACK ${p.checksum.slice(0, 4)}`;
}
case 8: {
const p = decoded as PathPayload;
return `${p.pathLength} hop path`;
}
case 9: {
const p = decoded as TracePayload;
return `trace ${p.pathHashes.length} hops`;
}
default:
return undefined;
}
}
export async function startMqttClient(): Promise<void> {
if (mqttStopping) return;
@@ -762,7 +667,7 @@ async function handleMessage(topic: string, rawPayload: Buffer): Promise<void> {
}
const decodedInner = decoded.payload?.decoded;
summary = buildSummary(decoded.payloadType, decodedInner, rawHex);
summary = buildSummary(decoded.payloadType, decodedInner, rawHex, channelEntries);
if (decoded.payloadType === 4) {
const inner = decodedInner as unknown as Record<string, unknown> | undefined;
+108
View File
@@ -0,0 +1,108 @@
/**
* Backfill decryption — decrypt stored raw group-text packets retroactively.
*
* Keyset-paginated on packet_hash (uses packets_hash_idx) so each batch is a
* small range scan instead of re-scanning the whole table. Resumable via
* --cursor=<packet_hash>. Idempotent: skips hashes already decrypted.
*
* Usage (inside the backend image):
* node dist/tools/backfillDecrypt.js [--days=90] [--batch=500] [--cursor=<hash>]
*/
import { buildChannelEntries, buildCombinedKeyStore, buildSummary } from '../mqtt/channelRegistry.js';
import { decodePacketCompat } from '../mqtt/decodePacket.js';
import { pool } from '../db/index.js';
function log(msg: string): void {
const line = `[backfill ${new Date().toISOString()}] ${msg}`;
process.stdout.write(line + '\n');
}
const args = process.argv.slice(2);
const daysArg = args.find((a) => a.startsWith('--days='));
const batchArg = args.find((a) => a.startsWith('--batch='));
const cursorArg = args.find((a) => a.startsWith('--cursor='));
const DAYS = daysArg ? Number(daysArg.split('=')[1]) : 90;
const BATCH = batchArg ? Number(batchArg.split('=')[1]) : 500;
let cursor = cursorArg ? cursorArg.split('=')[1] : '';
const channelEntries = buildChannelEntries(process.env['MESHCORE_CHANNEL_SECRETS']);
const keyStore = buildCombinedKeyStore(channelEntries);
log(`channels=${channelEntries.length} days=${DAYS} batch=${BATCH} cursor=${cursor || 'start'}`);
process.on('unhandledRejection', (err) => {
log(`FATAL unhandledRejection: ${err instanceof Error ? err.stack : String(err)}`);
process.exit(1);
});
async function fetchBatch(gt: string, limit: number): Promise<Array<{ packet_hash: string; raw_hex: string }>> {
const res = await pool.query(
`SELECT DISTINCT packet_hash, raw_hex
FROM packets
WHERE packet_type = 5
AND raw_hex IS NOT NULL
AND time > NOW() - ($1::int * INTERVAL '1 day')
AND payload->>'decrypted' IS NULL
AND packet_hash > $2
ORDER BY packet_hash
LIMIT $3`,
[DAYS, gt, limit],
);
return res.rows as Array<{ packet_hash: string; raw_hex: string }>;
}
async function applyDecryptions(rows: Array<{ packet_hash: string; decrypted: unknown; summary: string }>): Promise<void> {
await pool.query(
`UPDATE packets
SET payload = payload || jsonb_build_object(
'decrypted', $1::jsonb,
'_summary', $2::text
)
WHERE packet_hash = ANY($3::text[])`,
[JSON.stringify(rows.map((r) => r.decrypted)), JSON.stringify(rows.map((r) => r.summary)), rows.map((r) => r.packet_hash)],
);
}
async function main(): Promise<void> {
let processed = 0;
let decrypted = 0;
let failed = 0;
const startedAt = Date.now();
for (;;) {
const rows = await fetchBatch(cursor, BATCH);
if (rows.length === 0) break;
const updates: Array<{ packet_hash: string; decrypted: unknown; summary: string }> = [];
for (const row of rows) {
processed += 1;
try {
const { decoded, metadataValid } = decodePacketCompat(row.raw_hex, keyStore);
if (!metadataValid || !decoded?.payload?.decoded) { failed += 1; continue; }
const inner = decoded.payload.decoded as { decrypted?: { sender?: string; message?: string } } | undefined;
if (!inner?.decrypted) { failed += 1; continue; }
const summary = buildSummary(decoded.payloadType, inner, row.raw_hex, channelEntries);
updates.push({ packet_hash: row.packet_hash, decrypted: inner.decrypted, summary: summary ?? '[decrypted]' });
decrypted += 1;
} catch (err) {
failed += 1;
}
}
if (updates.length > 0) {
await applyDecryptions(updates);
}
cursor = rows[rows.length - 1]!.packet_hash;
if (processed % (BATCH * 10) === 0 || rows.length < BATCH) {
log(`progress: ${processed} scanned, ${decrypted} decrypted, ${failed} undecryptable — ${Math.round((Date.now() - startedAt) / 1000)}s — cursor=${cursor.slice(0, 12)}…`);
}
}
log(`DONE — scanned=${processed} decrypted=${decrypted} undecryptable=${failed} in ${Math.round((Date.now() - startedAt) / 1000)}s`);
await pool.end();
}
main().catch((err) => {
log(`FATAL: ${err instanceof Error ? err.stack : String(err)}`);
process.exit(1);
});
+15 -16
View File
@@ -21,6 +21,7 @@ import type { LazyPathResult } from './PacketDetailPanel.js';
import { FeedMapPanel } from './FeedPathViews.js';
import { FeedDialogs } from './FeedDialogs.js';
import {
MESSAGE_SCOPE_CHANNELS,
FEED_PATH_MAX_CONCURRENCY,
LAZY_SETTLE_MS,
MAX_PACKETS,
@@ -62,7 +63,9 @@ export const UKFeedPage: React.FC = () => {
const [selectedIata, setSelectedIata] = useState<string>(() => localStorage.getItem('uk-feed-iata') ?? 'all');
const [selectedMessageScope, setSelectedMessageScope] = useState<MessageScope>(() => {
const stored = localStorage.getItem('uk-feed-message-scope');
return stored === 'public' || stored === 'test' ? stored : 'all';
return stored === 'all' || (stored !== null && (MESSAGE_SCOPE_CHANNELS as readonly string[]).includes(stored))
? (stored as MessageScope)
: 'all';
});
const [messagesOnly, setMessagesOnly] = useState<boolean>(() => localStorage.getItem('uk-feed-messages-only') === '1');
const [regionOptions, setRegionOptions] = useState<string[]>([]);
@@ -447,27 +450,23 @@ export const UKFeedPage: React.FC = () => {
{/* ── Channels sidebar ───────────────────────────────────────── */}
<nav className="uk-feed-channels">
<div className="uk-feed-channels__header">Channels</div>
<button
<button
type="button"
className={`uk-feed-channel-item${selectedMessageScope === 'all' ? ' uk-feed-channel-item--active' : ''}`}
onClick={() => setSelectedMessageScope('all')}
>
All
</button>
<button
type="button"
className={`uk-feed-channel-item${selectedMessageScope === 'public' ? ' uk-feed-channel-item--active' : ''}`}
onClick={() => setSelectedMessageScope('public')}
>
Public
</button>
<button
type="button"
className={`uk-feed-channel-item${selectedMessageScope === 'test' ? ' uk-feed-channel-item--active' : ''}`}
onClick={() => setSelectedMessageScope('test')}
>
Test
</button>
{MESSAGE_SCOPE_CHANNELS.map((channel) => (
<button
key={channel}
type="button"
className={`uk-feed-channel-item${selectedMessageScope === channel ? ' uk-feed-channel-item--active' : ''}`}
onClick={() => setSelectedMessageScope(channel)}
>
{channel === 'public' ? 'Public' : channel}
</button>
))}
<div className="uk-feed-channels__divider" />
<div className="uk-feed-channels__header">Regions</div>
+44 -1
View File
@@ -38,7 +38,50 @@ export const TYPE_LABELS: Record<number, string> = {
};
export const MAX_PACKETS = 500;
export type MessageScope = 'all' | 'public' | 'test';
/** Decrypted/visible channel scopes — matches the [Channel] summary prefix (lowercased).
* Kept in sync with backend/src/mqtt/channelRegistry.ts VALIDATED_CHANNELS. */
export const MESSAGE_SCOPE_CHANNELS = [
'public',
'test',
'bot',
'yorkshire',
'liverpool',
'london',
'nottingham',
'northeast',
'kent',
'wales',
'cornwall',
'devon',
'dorset',
'cumbria',
'yorks',
'leicester',
'dartford',
'uckfield',
'huddersfield',
'derbyshire',
'lincolnshire',
'surrey',
'midlands',
'hamradio',
'mesh',
'public2',
'test2',
'thenorf',
'g8py',
'echo',
'denhaag',
'dublin',
'glasgow',
'york',
'ireland',
'scilly',
'brentwood',
'marple',
'uk',
] as const;
export type MessageScope = 'all' | (typeof MESSAGE_SCOPE_CHANNELS)[number];
export type PathTreeBranchNode = LazyPathNode & {
treeKey: string;
branchIndexes: Set<number>;