Files
meshcore-analyzer/cmd/server/advert_stats_test.go
T
n30nex b9fb3d244c feat(analytics): split adverts using recorded route evidence (#2088)
Red commits: `dc7dcbe`, `76e3217` ([assertion
failures](https://github.com/Kpa-clawbot/CoreScope/actions/runs/37059977385)).

Fixes #2041; supplies #2085's backend contract.

Records at most two route bits per retained advert. Node flood counts
now use the same evidence, including mixed and transport-flood adverts.
Analytics failures log/count errors while core observation, path and
liveness writes continue. The unrelated recent-advert limit clamp is
removed.

Direct adverts (empty path) describes the observed remaining path across
valid hash widths. Firmware can forward an advert down to an empty path;
this cannot establish original send mode or RF distance. The `zero_hop`
API spelling remains compatible.

Successful post-upgrade evidence writes survive observation replacement
and restart. Older overwritten frames and failed writes can leave gaps.
The UI explains automatic background backfill.

## Measured validation

Synthetic Windows workloads; timings are not production guarantees:

- 300K retained adverts, one-hour window: median 64.35→39.84 ms; 13.98
MB→145 KB allocated. Seven-day allocations unchanged; five paired
samples.
- 1M transmissions/11M observations: backfill 5m43s. Simple 10 Hz
writer: all 3,423 observations persisted; p99 28 ms, maximum 323 ms.
- Simulated pre-backfill cursor: retained evidence ready 4m21s; full
catch-up 6m41s, 122 cache invalidations. Normal completed-migration
startup bulk-loads current evidence.

Portable opt-in scale tests and window benchmarks are included. All 183
frontend suites, full server tests, targeted races, lint and real-API
desktop/mobile checks passed. Full [Linux
CI](https://github.com/Kpa-clawbot/CoreScope/actions/runs/37063210229)
passed at `36def9c`; three independent reviews are clear. Local Windows
testing hit the unchanged symlink-privilege test. No configuration
changes.

Preflight overrides: external script/OpenClaw profile unavailable;
repository checks and Chromium used.
2026-10-03 12:16:55 +02:00

183 lines
7.5 KiB
Go

package main
import (
"encoding/json"
"fmt"
"net/http/httptest"
"testing"
"time"
"github.com/meshcore-analyzer/packetpath"
)
// insertAdvertTx seeds one advert transmission row - the single place that
// knows the INSERT column list, shared by every test in this file.
func insertAdvertTx(t *testing.T, db *DB, pubkey, hash string, rt int, ts time.Time) {
t.Helper()
if _, err := db.conn.Exec(
`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, from_pubkey) VALUES ('00', ?, ?, ?, ?, ?)`,
hash, ts.Format("2006-01-02T15:04:05.000Z"), rt, payloadTypeAdvert, pubkey); err != nil {
t.Fatalf("insert transmission: %v", err)
}
// Fixtures represent ingestor output: canonical routing alone is not evidence.
if _, err := db.conn.Exec(`CREATE TABLE IF NOT EXISTS advert_route_evidence(id INTEGER PRIMARY KEY AUTOINCREMENT,tx_id INTEGER,bit INTEGER,UNIQUE(tx_id,bit))`); err != nil {
t.Fatal(err)
}
if rt == RouteFlood {
if _, err := db.conn.Exec(`INSERT INTO advert_route_evidence(tx_id,bit) SELECT id,1 FROM transmissions WHERE hash=?`, hash); err != nil {
t.Fatal(err)
}
}
}
func advertTS(hoursAgo float64) string {
return time.Now().UTC().Add(-time.Duration(hoursAgo * float64(time.Hour))).Format("2006-01-02T15:04:05.000Z")
}
// Flood adverts inside the window count; zero-hop (DIRECT) and out-of-window
// ones do not; duplicate hashes collapse; broken timestamps are skipped.
func TestCountFloodAdverts(t *testing.T) {
now := time.Now()
entries := []floodAdvertEntry{
{ts: advertTS(1), mask: packetpath.AdvertFlood, hash: "a1"},
{ts: advertTS(2), mask: packetpath.AdvertFlood, hash: "a1"}, // dup hash: one advert, two rows
{ts: advertTS(3), mask: packetpath.AdvertFlood, hash: "a2"},
{ts: advertTS(4), mask: packetpath.AdvertDirectEmptyPath, hash: "a3"}, // direct-only evidence: excluded
{ts: advertTS(9 * 24), mask: packetpath.AdvertFlood, hash: "a4"}, // outside 7d window
{ts: "not-a-time", mask: packetpath.AdvertFlood, hash: "a5"}, // unparseable: skipped
{ts: advertTS(5), mask: 0, hash: "a6"}, // no known evidence: excluded
}
if got := countFloodAdverts(entries, now, 7*24); got != 2 {
t.Fatalf("want 2 flood adverts in window, got %d", got)
}
// Hash-less entries dedup by timestamp instead of collapsing into one.
hashless := []floodAdvertEntry{
{ts: advertTS(1), mask: packetpath.AdvertFlood},
{ts: advertTS(2), mask: packetpath.AdvertFlood},
}
if got := countFloodAdverts(hashless, now, 7*24); got != 2 {
t.Fatalf("want 2 hash-less flood adverts, got %d", got)
}
}
// The wire contract: GET /api/nodes/{pubkey} carries flood_advert_count_7d,
// counting recent flood adverts only (zero-hop and out-of-window excluded).
func TestNodeDetailIncludesFloodAdvertCount(t *testing.T) {
srv, router := setupTestServer(t)
if _, err := srv.db.conn.Exec(`CREATE TABLE advert_route_evidence(id INTEGER PRIMARY KEY AUTOINCREMENT,tx_id INTEGER,bit INTEGER,UNIQUE(tx_id,bit))`); err != nil {
t.Fatal(err)
}
now := time.Now().UTC()
ins := func(hash string, rt int, ts time.Time) {
insertAdvertTx(t, srv.db, "aabbccdd11223344", hash, rt, ts)
}
// The shared fixture may already seed adverts for this node, so assert the
// DELTA our inserts cause rather than an absolute count.
fetch := func() float64 {
t.Helper()
req := httptest.NewRequest("GET", "/api/nodes/aabbccdd11223344", nil)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
if w.Code != 200 {
t.Fatalf("expected 200, got %d", w.Code)
}
var body map[string]interface{}
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
t.Fatalf("bad JSON: %v", err)
}
node, ok := body["node"].(map[string]interface{})
if !ok {
t.Fatal("expected node object")
}
got, ok := node["flood_advert_count_7d"].(float64)
if !ok {
t.Fatalf("flood_advert_count_7d missing or not a number: %v", node["flood_advert_count_7d"])
}
return got
}
before := fetch()
ins("fa1", RouteFlood, now.Add(-2*time.Hour))
ins("fa2", RouteFlood, now.Add(-30*time.Hour))
ins("za1", 0, now.Add(-1*time.Hour)) // zero-hop: excluded
ins("fa3", RouteFlood, now.Add(-9*24*time.Hour)) // outside the 7d window
// Inside the SQL date floor (window + 1d slack) but outside the exact 7d
// window - only the Go-side check rejects this one.
ins("fa4", RouteFlood, now.Add(-time.Duration(7.5*24)*time.Hour))
if got := fetch(); got != before+2 {
t.Fatalf("want flood_advert_count_7d = %v+2, got %v", before, got)
}
}
// The row cap saturates the count instead of failing: with a cap of 2, three
// qualifying flood adverts count as 2 (newest rows win via ORDER BY id DESC).
func TestCountFloodAdvertsForNode_RowCapSaturates(t *testing.T) {
db := setupTestDB(t)
now := time.Now().UTC()
for i, h := range []string{"cap1", "cap2", "cap3"} {
insertAdvertTx(t, db, "capnode11223344", h, RouteFlood, now.Add(-time.Duration(i+1)*time.Hour))
}
n, err := db.CountFloodAdvertsForNode("capnode11223344", 7*24, 2)
if err != nil {
t.Fatalf("count: %v", err)
}
if n != 2 {
t.Fatalf("want saturated count 2, got %d", n)
}
}
func TestNodeDetailFloodCountAgreesWithRouteEvidence(t *testing.T) {
for _, firstRoute := range []int{RouteDirect, RouteFlood, RouteTransportFlood} {
t.Run(fmt.Sprint(firstRoute), func(t *testing.T) {
srv, router := setupTestServer(t)
if _, err := srv.db.conn.Exec(`CREATE TABLE IF NOT EXISTS advert_route_evidence(id INTEGER PRIMARY KEY AUTOINCREMENT, tx_id INTEGER, bit INTEGER, UNIQUE(tx_id,bit))`); err != nil {
t.Fatal(err)
}
const pubkey = "aabbccdd11223344"
if _, err := srv.db.conn.Exec(`DELETE FROM transmissions WHERE from_pubkey=?`, pubkey); err != nil {
t.Fatal(err)
}
insertAdvertTx(t, srv.db, pubkey, "mixed-advert", firstRoute, time.Now().Add(-time.Hour))
if _, err := srv.db.conn.Exec(`INSERT INTO advert_route_evidence(tx_id,bit) SELECT id,1 FROM transmissions WHERE hash='mixed-advert' AND NOT EXISTS(SELECT 1 FROM advert_route_evidence WHERE tx_id=transmissions.id AND bit=1) UNION ALL SELECT id,2 FROM transmissions WHERE hash='mixed-advert'`); err != nil {
t.Fatal(err)
}
w := httptest.NewRecorder()
router.ServeHTTP(w, httptest.NewRequest("GET", "/api/nodes/"+pubkey, nil))
var response NodeDetailResponse
if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil {
t.Fatal(err)
}
if w.Code != 200 || len(response.RecentAdverts) != 1 || response.RecentAdverts[0]["advert_kind"] != "mixed" {
t.Fatalf("unexpected node response: %d %+v", w.Code, response)
}
if response.Node["flood_advert_count_7d"] != float64(1) {
t.Errorf("mixed advert must contribute one flood count regardless of first route %d: %v", firstRoute, response.Node["flood_advert_count_7d"])
}
})
}
}
func TestNodeDetailLegacyFloodCountIsUnavailable(t *testing.T) {
srv, router := setupTestServer(t)
if _, err := srv.db.conn.Exec(`DROP TABLE IF EXISTS advert_route_evidence`); err != nil {
t.Fatal(err)
}
insertAdvertTx(t, srv.db, "aabbccdd11223344", "legacy-flood", RouteFlood, time.Now().Add(-time.Hour))
if _, err := srv.db.conn.Exec(`DROP TABLE advert_route_evidence`); err != nil {
t.Fatal(err)
}
w := httptest.NewRecorder()
router.ServeHTTP(w, httptest.NewRequest("GET", "/api/nodes/aabbccdd11223344", nil))
var response NodeDetailResponse
if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil {
t.Fatal(err)
}
if w.Code != 200 {
t.Fatal(w.Code)
}
if _, ok := response.Node["flood_advert_count_7d"]; ok {
t.Errorf("missing evidence table must not report a proven count: %v", response.Node["flood_advert_count_7d"])
}
}