Files
meshcore-analyzer/cmd/ingestor/coverage_gate_test.go
T
efitenandClaude Opus 5 9e13e0b05f feat(ingestor): full-packet RF observations from mobile clients (#1905)
## What

A CoreDrive RX drive already carries far more RF information than
reaches CoreScope, and it was being discarded twice: once in the mobile
app (every packet it could not attribute to a directly-heard node was
dropped before queueing) and once here (the ingestor decodes the
*complete* packet, then keeps only
`heard_key`/`snr`/`rssi`/`lat`/`lon`).

This captures what was being thrown away, at **zero extra airtime** —
nothing new is transmitted.

- **`transmissions.code1` / `code2`** — the transport codes were decoded
on every packet and used only to derive `scope_name`, then dropped.
Storing them turns "which repeater forwards which scope" from a re-parse
into a query.
- **An async backfill** re-parses the `raw_hex` already on disk, so
months of scope history become queryable with no new data collection.
- **`client_rx_observations`** — a new diagnostic table holding every
decodable packet a phone heard, with route type, transport codes, scope
name, path-hash size, the full forwarder chain and the forwarder.

## Why it is safe for existing deployments

Both halves are **opt-in and default off**
(`clientRxObservations.enabled`, and `fullRfLog` on the app side), so an
existing deployment sees no behaviour change and no volume change on
upgrade.

The coverage invariant is untouched: `client_receptions` keeps its rule
— 0-hop advert pubkey or FLOOD `path[last]`, ≥2-byte hash — and an
unattributable packet writes **zero** coverage rows. `deriveHeardKey`,
`buildClientReception` and `InsertClientReception` are unmodified except
for one guard described below.

## Performance justification (touches the ingest hot path)

- **Backfill:** keyset-paginated by `id` in 5000-row batches, a single
forward scan, `rows.Close()` before `Begin()` so it never deadlocks
against `SetMaxOpenConns(1)`, and commits per batch so live ingest
interleaves. Termination is driven by rows *scanned*, not rows decoded —
an earlier count-based loop would have stopped at the first batch
containing an undecodable row and then written its completion guard,
permanently stranding the rest.
- **Guard row is written if and only if the loop ran to genuine
exhaustion.** Every error path leaves it unwritten so the next startup
retries.
- **Per-packet cost:** one extra INSERT on the client topic when
enabled, gated behind an opt-in flag. No new work on the observer path.
- **New indexes** cover the prune (`rx_at`), the flood-grouping
(`pkt_hash, rx_at`), the per-repeater query (`forwarder, rx_at`) and the
scope query (`scope_name, rx_at`). Retention has its own shorter window
— this table is diagnostic, not archival.

## Two firmware-derived correctness points

- **`pkt_hash` is `ComputeContentHash()`**, byte-identical to
`transmissions.hash`, so dark-traffic queries are a plain equality join
rather than a translation layer.
- **TRACE packets are refused.** TRACE repurposes the header path bytes
as per-hop SNR values, so deriving a `heard_key` from them invents a
node that never existed. `packetpath.PathBytesAreHops` existed but was
never wired into the client path; it became reachable only because the
app half now publishes packets it previously dropped locally.

## Testing

Full ingestor suite green. Notable coverage: a FLOOD-routed TRACE writes
zero coverage rows and NULL `forwarder`; a `direction: "tx"` message
writes no observation; a DIRECT route never sets `forwarder`; two
forwarder copies of one flood remain two rows; the backfill's
multi-batch path is exercised with an undecodable row in the first page;
and a forced error asserts the migration guard stays unwritten.

---------

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-02 18:35:20 +02:00

134 lines
5.9 KiB
Go

package main
import "testing"
// testCompanionPK is a valid lowercase-hex companion pubkey for coverage tests.
// The topic segment must be hex (clientPubkeyRe) or handleClientPacket drops it.
const testCompanionPK = "a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2"
// clientCoverageMsg builds a valid mobile client-RX coverage message on the
// dedicated topic meshcore/client/<pubkey>/packets. The raw hex is a relayed
// advert with GPS, so handleClientPacket would write exactly one
// client_receptions row when the feature is enabled (see
// TestHandleClientPacketAdvertWritesReception).
func clientCoverageMsg() *mockMessage {
advertHex := "11451000D818206D3AAC152C8A91F89957E6D30CA51F36E28790228971C473B755F244F718754CF5EE4A2FD58D944466E42CDED140C66D0CC590183E32BAF40F112BE8F3F2BDF6012B4B2793C52F1D36F69EE054D9A05593286F78453E56C0EC4A3EB95DDA2A7543FCCC00B939CACC009278603902FC12BCF84B706120526F6F6620536F6C6172"
payload := []byte(`{"raw":"` + advertHex + `","direction":"rx","timestamp":"2026-06-09T12:00:00Z","origin":"MyMob","SNR":-7.0,"RSSI":-92.0,"gps":{"lat":51.05,"lon":3.72,"acc_m":8.0}}`)
return &mockMessage{topic: "meshcore/client/" + testCompanionPK + "/packets", payload: payload}
}
func clientReceptionCount(t *testing.T, s *Store) int {
t.Helper()
var n int
if err := s.db.QueryRow(`SELECT COUNT(*) FROM client_receptions`).Scan(&n); err != nil {
t.Fatal(err)
}
return n
}
// TestClientRxCoverageEnabledDefault verifies the gate helper defaults OFF for
// nil/absent config and is only true when explicitly enabled.
func TestClientRxCoverageEnabledDefault(t *testing.T) {
if (&Config{}).ClientRxCoverageEnabled() {
t.Fatal("nil ClientRxCoverage must report disabled")
}
if (&Config{ClientRxCoverage: &ClientRxCoverageConfig{Enabled: false}}).ClientRxCoverageEnabled() {
t.Fatal("Enabled:false must report disabled")
}
if !(&Config{ClientRxCoverage: &ClientRxCoverageConfig{Enabled: true}}).ClientRxCoverageEnabled() {
t.Fatal("Enabled:true must report enabled")
}
}
// TestClientRxCoverageGateOff drives handleMessage with the feature OFF: the
// client-topic message must fall through and write no client_receptions rows.
func TestClientRxCoverageGateOff(t *testing.T) {
store := newTestStore(t)
source := MQTTSource{Name: "test"}
cfg := &Config{} // ClientRxCoverage nil ⇒ disabled
handleMessage(store, "test", source, clientCoverageMsg(), nil, nil, cfg)
if n := clientReceptionCount(t, store); n != 0 {
t.Fatalf("feature OFF: expected 0 client_receptions rows, got %d", n)
}
}
// TestClientRxCoverageGateOn drives handleMessage with the feature ON: the
// client-topic message must be dispatched and write exactly one row.
func TestClientRxCoverageGateOn(t *testing.T) {
store := newTestStore(t)
source := MQTTSource{Name: "test"}
cfg := &Config{ClientRxCoverage: &ClientRxCoverageConfig{Enabled: true}}
handleMessage(store, "test", source, clientCoverageMsg(), nil, nil, cfg)
if n := clientReceptionCount(t, store); n != 1 {
t.Fatalf("feature ON: expected 1 client_receptions row, got %d", n)
}
}
// TestClientRxCoverageGateOffDoesNotFallThroughToObserverPath is the
// CRITICAL-adjacent regression test: with the feature OFF, a
// meshcore/client/<pubkey>/packets message must be dropped outright, never
// fall through to the observer packet path below. Before the fix, the
// enable-gate lived INSIDE the topic match
// (cfg.ClientRxCoverageEnabled() && parts[1]=="client" && ...), so a disabled
// gate made the whole condition false and the message fell through: the
// observer path takes parts[1] ("client") as a region and parts[2] (the
// companion pubkey) as an observer id, creating a bogus "client" region and
// registering the phone as an observer — worse now that fullRfLog multiplies
// client-topic volume. Asserts zero client_receptions rows (already covered
// by TestClientRxCoverageGateOff) AND zero observer/region pollution.
func TestClientRxCoverageGateOffDoesNotFallThroughToObserverPath(t *testing.T) {
store := newTestStore(t)
source := MQTTSource{Name: "test"}
cfg := &Config{} // ClientRxCoverage nil ⇒ disabled
handleMessage(store, "test", source, clientCoverageMsg(), nil, nil, cfg)
if n := clientReceptionCount(t, store); n != 0 {
t.Fatalf("feature OFF: expected 0 client_receptions rows, got %d", n)
}
var observerRows int
if err := store.db.QueryRow(`SELECT COUNT(*) FROM observers WHERE id = ?`, testCompanionPK).Scan(&observerRows); err != nil {
t.Fatal(err)
}
if observerRows != 0 {
t.Fatalf("feature OFF: the companion pubkey must not be registered as an observer, got %d rows", observerRows)
}
var clientRegionRows int
if err := store.db.QueryRow(`SELECT COUNT(*) FROM observers WHERE iata = 'client'`).Scan(&clientRegionRows); err != nil {
t.Fatal(err)
}
if clientRegionRows != 0 {
t.Fatalf("feature OFF: no observer should be registered under a bogus 'client' region, got %d rows", clientRegionRows)
}
var txRows int
if err := store.db.QueryRow(`SELECT COUNT(*) FROM transmissions`).Scan(&txRows); err != nil {
t.Fatal(err)
}
if txRows != 0 {
t.Fatalf("feature OFF: the client-topic packet must not be ingested as an ordinary observer packet, got %d transmissions rows", txRows)
}
}
// TestClientRxCoverageBlacklistedDropped verifies the #1 fix: a blacklisted
// operator cannot skirt the observer blacklist via the client topic. With the
// feature ON but the companion pubkey blacklisted, no row is written. Without
// the gate the client dispatch runs before the blacklist check and inserts.
func TestClientRxCoverageBlacklistedDropped(t *testing.T) {
store := newTestStore(t)
source := MQTTSource{Name: "test"}
cfg := &Config{
ClientRxCoverage: &ClientRxCoverageConfig{Enabled: true},
ObserverBlacklist: []string{testCompanionPK},
}
handleMessage(store, "test", source, clientCoverageMsg(), nil, nil, cfg)
if n := clientReceptionCount(t, store); n != 0 {
t.Fatalf("blacklisted companion: expected 0 client_receptions rows, got %d", n)
}
}