mirror of
https://github.com/Kpa-clawbot/meshcore-analyzer.git
synced 2026-05-24 18:55:19 +00:00
Kpa-clawbot
9383201c07
Red commit:
https://github.com/Kpa-clawbot/CoreScope/commit/eae179b99b5fd34924547632aa8f8025c405aa53
(CI: pending — opens with this PR)
Finishes #1283. RED test `TestServerSourceHasNoCachedRWCalls` goes from
failing (13 writer call-sites) to GREEN (zero). Per #1287 Option 4
(https://github.com/Kpa-clawbot/CoreScope/issues/1287#issuecomment-4485099992):
ingestor owns the neighbor graph build + persist; server reads the
snapshot.
**Category A — Schema migrations** → new `internal/dbschema` package.
`dbschema.Apply(rw)` runs in `cmd/ingestor` startup (in `OpenStore`).
`dbschema.AssertReady(ro)` runs in `cmd/server/main.go` and
FATAL-LOG-EXITS if any expected column/index/table is missing — the
operator must restart the ingestor first. Covers indexes,
`neighbor_edges`, `observations.resolved_path`,
`observers.{inactive,last_packet_at,iata}`,
`(inactive_)nodes.foreign_advert`, `transmissions.from_pubkey`.
**Category B — Backfill** → ingestor.
`BackfillFromPubkey` and observer-blacklist soft-delete moved to
`cmd/ingestor/maintenance.go`. Server keeps an inert
`fromPubkeyBackfillSnapshot` stub for `/api/healthz` API compatibility.
**Category C — Neighbor-graph persistence (Option 4)** → ingestor
writes, server reads.
- Ingestor (`cmd/ingestor/neighbor_builder.go`): every 60s scans
`observations + transmissions`, extracts edges (originator↔first-hop for
ADVERTs; observer↔last-hop for all), resolves hop prefixes via a
node-table prefix index, upserts into `neighbor_edges`.
- Server (`cmd/server/neighbor_recomputer.go`): every 60s re-reads
`neighbor_edges` and atomic-swaps the resulting `NeighborGraph` into
`s.graph`. Initial load is synchronous on startup. All server-side
incremental edge writers (the two `asyncPersistResolvedPathsAndEdges`
paths in `cmd/server/store.go`) are gone.
- Neighbor-edge daily prune (`PruneNeighborEdges`) moved to ingestor.
**Why Option 4**: clean read/write separation, no startup CPU spike
(server loads existing snapshot instead of rebuilding from history), no
IPC/delta-protocol churn. Staleness budget ~60s — same model as the
analytics recomputers in #1240 / #1248 / #672 axis 2.
**Recomputer interval default for neighbor graph**: 60s
(`NeighborGraphRecomputerDefaultInterval`,
`NeighborEdgesBuilderInterval`).
**Invariants added**:
- `TestServerSourceHasNoCachedRWCalls` (RED commit eae179b9): grep
enforces zero `cachedRW(`, `mode=rw`, or `sql.Open(_journal_mode=WAL…)`
in non-test `cmd/server/` sources.
- `TestServerStartupRequiresMigratedSchema`: server refuses to start
against an unmigrated DB.
- `TestNeighborGraphRecomputerLoadsSnapshot`: post-write snapshot is
picked up on the next refresh.
- `TestNeighborEdgesBuilderUpsertsFromObservations`: end-to-end pipeline
writes the expected edge.
`grep cachedRW cmd/server/*.go | grep -v _test.go` → 0 matches.
Fixes #1287.
---------
Co-authored-by: MeshCore Bot <bot@meshcore.local>
Co-authored-by: Kpa-clawbot <Kpa-clawbot@users.noreply.github.com>
Co-authored-by: corescope-bot <bot@corescope.local>
85 lines
2.4 KiB
Go
85 lines
2.4 KiB
Go
// Test that the migrate binary brings the e2e fixture DB up to the
|
|
// shape required by cmd/server's dbschema.AssertReady. Regression test
|
|
// for PR #1289 / fix for the CI "Server failed to start within 30s"
|
|
// failure: AssertReady fired against the unmigrated fixture and the
|
|
// server fatal-logged before opening its HTTP listener.
|
|
package main
|
|
|
|
import (
|
|
"database/sql"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
|
|
"github.com/meshcore-analyzer/dbschema"
|
|
_ "modernc.org/sqlite"
|
|
)
|
|
|
|
// fixtureCandidates lists possible locations of the committed e2e
|
|
// fixture DB relative to this test's package directory. We resolve
|
|
// against runtime cwd which is cmd/migrate when `go test` runs.
|
|
var fixtureCandidates = []string{
|
|
"../../test-fixtures/e2e-fixture.db",
|
|
}
|
|
|
|
func locateFixture(t *testing.T) string {
|
|
t.Helper()
|
|
for _, p := range fixtureCandidates {
|
|
if _, err := os.Stat(p); err == nil {
|
|
abs, _ := filepath.Abs(p)
|
|
return abs
|
|
}
|
|
}
|
|
t.Skipf("e2e fixture not found (looked in: %v)", fixtureCandidates)
|
|
return ""
|
|
}
|
|
|
|
func copyFile(t *testing.T, src, dst string) {
|
|
t.Helper()
|
|
in, err := os.Open(src)
|
|
if err != nil {
|
|
t.Fatalf("open src: %v", err)
|
|
}
|
|
defer in.Close()
|
|
out, err := os.Create(dst)
|
|
if err != nil {
|
|
t.Fatalf("create dst: %v", err)
|
|
}
|
|
defer out.Close()
|
|
if _, err := io.Copy(out, in); err != nil {
|
|
t.Fatalf("copy: %v", err)
|
|
}
|
|
}
|
|
|
|
// TestMigrateBringsFixtureToReady is the gate test for the CI bug.
|
|
// Before the fix landed, AssertReady against the committed fixture
|
|
// returned an error ("missing: inactive_nodes.foreign_advert" etc.).
|
|
// After Apply(), AssertReady must return nil.
|
|
func TestMigrateBringsFixtureToReady(t *testing.T) {
|
|
src := locateFixture(t)
|
|
dst := filepath.Join(t.TempDir(), "fixture-copy.db")
|
|
copyFile(t, src, dst)
|
|
|
|
db, err := sql.Open("sqlite", dst)
|
|
if err != nil {
|
|
t.Fatalf("open: %v", err)
|
|
}
|
|
defer db.Close()
|
|
|
|
// Sanity: the committed fixture is missing at least one expected
|
|
// migration column. If this stops being true, either someone
|
|
// pre-migrated the fixture (and this test no longer protects #1289)
|
|
// or AssertReady's required set changed.
|
|
if err := dbschema.AssertReady(db); err == nil {
|
|
t.Logf("note: fixture already passes AssertReady; skipping pre-condition assertion")
|
|
}
|
|
|
|
if err := dbschema.Apply(db, t.Logf); err != nil {
|
|
t.Fatalf("Apply: %v", err)
|
|
}
|
|
if err := dbschema.AssertReady(db); err != nil {
|
|
t.Fatalf("AssertReady after Apply: %v", err)
|
|
}
|
|
}
|