mirror of
https://github.com/Kpa-clawbot/meshcore-analyzer.git
synced 2026-06-01 23:44:15 +00:00
9383201c07
Red commit:
https://github.com/Kpa-clawbot/CoreScope/commit/eae179b99b5fd34924547632aa8f8025c405aa53
(CI: pending — opens with this PR)
Finishes #1283. RED test `TestServerSourceHasNoCachedRWCalls` goes from
failing (13 writer call-sites) to GREEN (zero). Per #1287 Option 4
(https://github.com/Kpa-clawbot/CoreScope/issues/1287#issuecomment-4485099992):
ingestor owns the neighbor graph build + persist; server reads the
snapshot.
**Category A — Schema migrations** → new `internal/dbschema` package.
`dbschema.Apply(rw)` runs in `cmd/ingestor` startup (in `OpenStore`).
`dbschema.AssertReady(ro)` runs in `cmd/server/main.go` and
FATAL-LOG-EXITS if any expected column/index/table is missing — the
operator must restart the ingestor first. Covers indexes,
`neighbor_edges`, `observations.resolved_path`,
`observers.{inactive,last_packet_at,iata}`,
`(inactive_)nodes.foreign_advert`, `transmissions.from_pubkey`.
**Category B — Backfill** → ingestor.
`BackfillFromPubkey` and observer-blacklist soft-delete moved to
`cmd/ingestor/maintenance.go`. Server keeps an inert
`fromPubkeyBackfillSnapshot` stub for `/api/healthz` API compatibility.
**Category C — Neighbor-graph persistence (Option 4)** → ingestor
writes, server reads.
- Ingestor (`cmd/ingestor/neighbor_builder.go`): every 60s scans
`observations + transmissions`, extracts edges (originator↔first-hop for
ADVERTs; observer↔last-hop for all), resolves hop prefixes via a
node-table prefix index, upserts into `neighbor_edges`.
- Server (`cmd/server/neighbor_recomputer.go`): every 60s re-reads
`neighbor_edges` and atomic-swaps the resulting `NeighborGraph` into
`s.graph`. Initial load is synchronous on startup. All server-side
incremental edge writers (the two `asyncPersistResolvedPathsAndEdges`
paths in `cmd/server/store.go`) are gone.
- Neighbor-edge daily prune (`PruneNeighborEdges`) moved to ingestor.
**Why Option 4**: clean read/write separation, no startup CPU spike
(server loads existing snapshot instead of rebuilding from history), no
IPC/delta-protocol churn. Staleness budget ~60s — same model as the
analytics recomputers in #1240 / #1248 / #672 axis 2.
**Recomputer interval default for neighbor graph**: 60s
(`NeighborGraphRecomputerDefaultInterval`,
`NeighborEdgesBuilderInterval`).
**Invariants added**:
- `TestServerSourceHasNoCachedRWCalls` (RED commit eae179b9): grep
enforces zero `cachedRW(`, `mode=rw`, or `sql.Open(_journal_mode=WAL…)`
in non-test `cmd/server/` sources.
- `TestServerStartupRequiresMigratedSchema`: server refuses to start
against an unmigrated DB.
- `TestNeighborGraphRecomputerLoadsSnapshot`: post-write snapshot is
picked up on the next refresh.
- `TestNeighborEdgesBuilderUpsertsFromObservations`: end-to-end pipeline
writes the expected edge.
`grep cachedRW cmd/server/*.go | grep -v _test.go` → 0 matches.
Fixes #1287.
---------
Co-authored-by: MeshCore Bot <bot@meshcore.local>
Co-authored-by: Kpa-clawbot <Kpa-clawbot@users.noreply.github.com>
Co-authored-by: corescope-bot <bot@corescope.local>
133 lines
4.2 KiB
Go
133 lines
4.2 KiB
Go
package main
|
|
|
|
import (
|
|
"database/sql"
|
|
"path/filepath"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/meshcore-analyzer/dbschema"
|
|
_ "modernc.org/sqlite"
|
|
)
|
|
|
|
// TestNeighborGraphRecomputerLoadsSnapshot enforces #1287 Option 4:
|
|
// the server LOADS its in-memory neighbor graph from the SQLite
|
|
// snapshot the ingestor writes. After a write to neighbor_edges (here
|
|
// done synthetically), the recomputer's atomic-swap must reflect it.
|
|
func TestNeighborGraphRecomputerLoadsSnapshot(t *testing.T) {
|
|
dir := t.TempDir()
|
|
dbPath := filepath.Join(dir, "neighbor_recomp.db")
|
|
|
|
// Bootstrap a WAL DB with the neighbor_edges table.
|
|
rw, err := sql.Open("sqlite", "file:"+dbPath+"?_journal_mode=WAL")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer rw.Close()
|
|
if _, err := rw.Exec(`CREATE TABLE neighbor_edges (
|
|
node_a TEXT NOT NULL,
|
|
node_b TEXT NOT NULL,
|
|
count INTEGER DEFAULT 1,
|
|
last_seen TEXT,
|
|
PRIMARY KEY (node_a, node_b)
|
|
)`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Stage one edge.
|
|
now := time.Now().UTC().Format(time.RFC3339)
|
|
if _, err := rw.Exec(
|
|
`INSERT INTO neighbor_edges (node_a, node_b, count, last_seen) VALUES (?, ?, ?, ?)`,
|
|
"aaa", "bbb", 5, now,
|
|
); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Server opens read-only and refreshes via the recomputer.
|
|
d, err := OpenDB(dbPath)
|
|
if err != nil {
|
|
t.Fatalf("OpenDB: %v", err)
|
|
}
|
|
defer d.conn.Close()
|
|
store := &PacketStore{db: d}
|
|
store.graph.Store(NewNeighborGraph())
|
|
|
|
store.refreshNeighborGraphFromSnapshot()
|
|
g := store.graph.Load()
|
|
if g == nil {
|
|
t.Fatal("graph nil after refresh")
|
|
}
|
|
if got := len(g.AllEdges()); got != 1 {
|
|
t.Fatalf("expected 1 edge after first refresh, got %d", got)
|
|
}
|
|
|
|
// Add another row, refresh, assert the new total.
|
|
if _, err := rw.Exec(
|
|
`INSERT INTO neighbor_edges (node_a, node_b, count, last_seen) VALUES (?, ?, ?, ?)`,
|
|
"ccc", "ddd", 2, now,
|
|
); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
store.refreshNeighborGraphFromSnapshot()
|
|
g = store.graph.Load()
|
|
if got := len(g.AllEdges()); got != 2 {
|
|
t.Fatalf("expected 2 edges after second refresh, got %d", got)
|
|
}
|
|
}
|
|
|
|
// TestServerStartupRequiresMigratedSchema enforces #1287: the server
|
|
// MUST refuse to start if the ingestor hasn't run schema migrations.
|
|
// AssertReady on a DB missing the required columns returns an error
|
|
// listing every missing surface; main.go then calls log.Fatalf.
|
|
func TestServerStartupRequiresMigratedSchema(t *testing.T) {
|
|
dir := t.TempDir()
|
|
dbPath := filepath.Join(dir, "unmigrated.db")
|
|
|
|
// Bootstrap with ONLY transmissions/observations (the things
|
|
// server tries to read) but WITHOUT the columns dbschema asserts
|
|
// (resolved_path, inactive, last_packet_at, iata, foreign_advert,
|
|
// from_pubkey, neighbor_edges).
|
|
rw, err := sql.Open("sqlite", "file:"+dbPath+"?_journal_mode=WAL")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer rw.Close()
|
|
for _, s := range []string{
|
|
`CREATE TABLE transmissions (id INTEGER PRIMARY KEY, hash TEXT, payload_type INTEGER)`,
|
|
`CREATE TABLE observations (id INTEGER PRIMARY KEY, transmission_id INTEGER)`,
|
|
`CREATE TABLE observers (id TEXT PRIMARY KEY, name TEXT)`,
|
|
`CREATE TABLE nodes (public_key TEXT PRIMARY KEY)`,
|
|
`CREATE TABLE inactive_nodes (public_key TEXT PRIMARY KEY)`,
|
|
} {
|
|
if _, err := rw.Exec(s); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
// Open the read-only server handle and call AssertReady directly
|
|
// (production path: main.go does this before any business logic).
|
|
d, err := OpenDB(dbPath)
|
|
if err != nil {
|
|
t.Fatalf("OpenDB: %v", err)
|
|
}
|
|
defer d.conn.Close()
|
|
|
|
// The package-level dbschema.AssertReady requires every missing
|
|
// surface to be reported. We hit it directly through the same
|
|
// path main.go uses.
|
|
if err := assertReadyForTest(d); err == nil {
|
|
t.Fatal("expected AssertReady to fail against an unmigrated DB; server would have started against an incomplete schema")
|
|
}
|
|
}
|
|
|
|
// assertReadyForTest is the same call main.go makes — declared here so
|
|
// the test stays decoupled from any future inlining or rename.
|
|
func assertReadyForTest(d *DB) error {
|
|
return dbschemaAssertReadyShim(d)
|
|
}
|
|
|
|
// dbschemaAssertReadyShim wraps the package import so tests don't
|
|
// directly depend on the import being present (production wires it
|
|
// via main.go).
|
|
func dbschemaAssertReadyShim(d *DB) error { return dbschema.AssertReady(d.conn) }
|