From 81ef51cc5c9249cce5c2cbb60b6fdad44de747b9 Mon Sep 17 00:00:00 2001 From: Kpa-clawbot Date: Fri, 3 Apr 2026 23:08:09 -0700 Subject: [PATCH] fix: debounce distance index rebuild to prevent CPU hot loop (#557) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem On busy meshes (325K+ transmissions, 50 observers), the distance index rebuild runs on **every ingest poll** (~1s interval), computing haversine distances for 1M+ hop records. Each rebuild takes 2-3 seconds but new observations arrive faster than it can finish, creating a CPU hot loop that starves the HTTP server. Discovered on the Cascadia Mesh instance where `corescope-server` was consuming 15 minutes of CPU time in 10 minutes of uptime, the API was completely unresponsive, and health checks were timing out. ### Server logs showing the hot loop: ``` [store] Built distance index: 1797778 hop records, 207072 path records [store] Built distance index: 1797806 hop records, 207075 path records [store] Built distance index: 1797811 hop records, 207075 path records [store] Built distance index: 1797820 hop records, 207075 path records ``` Every 2 seconds, nonstop. ## Root Cause `IngestNewObservations` calls `buildDistanceIndex()` synchronously whenever `pickBestObservation` selects a longer path. With 50 observers sending observations every second, paths change on nearly every poll cycle, triggering a full rebuild each time. ## Fix - Mark distance index dirty on path changes instead of rebuilding inline - Rebuild at most every **30 seconds** (configurable via `distLast` timer) - Set `distLast` after initial `Load()` to prevent immediate re-rebuild on first ingest - Distance data is at most 30s stale — acceptable for an analytics view ## Testing - `go build`, `go vet`, `go test` all pass - No behavioral change for the initial load or the analytics API response shape - Distance data freshness goes from real-time to 30s max staleness --------- Co-authored-by: Kpa-clawbot <259247574+Kpa-clawbot@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Co-authored-by: you --- cmd/server/coverage_test.go | 42 +++++++++++++++++++++++++++++++++++++ cmd/server/store.go | 29 +++++++++++++++++-------- 2 files changed, 62 insertions(+), 9 deletions(-) diff --git a/cmd/server/coverage_test.go b/cmd/server/coverage_test.go index d707f1da..44a0820a 100644 --- a/cmd/server/coverage_test.go +++ b/cmd/server/coverage_test.go @@ -3913,3 +3913,45 @@ func TestBuildTransmissionWhereMultiObserver(t *testing.T) { } }) } + +// --- Distance index rebuild debounce (#557) --- + +func TestDistanceRebuildDebounce(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + seedTestData(t, db) + store := NewPacketStore(db, nil) + store.Load() + + // After Load(), distLast is set to now — so distDirty should be false + if store.distDirty { + t.Fatal("distDirty should be false after Load()") + } + + // Insert a new observation with a different path to trigger distDirty + maxObsID := db.GetMaxObservationID() + db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) + VALUES (1, 2, 5.0, -100, '["xx","yy","zz"]', ?)`, time.Now().Unix()) + + store.IngestNewObservations(maxObsID, 500) + + // distDirty should be true (30s hasn't elapsed since Load) + if !store.distDirty { + t.Fatal("distDirty should be true after path change within 30s window") + } + + // Now simulate 30s having elapsed by backdating distLast + store.distLast = time.Now().Add(-31 * time.Second) + + // Insert another observation to trigger another ingest cycle + maxObsID = db.GetMaxObservationID() + db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) + VALUES (1, 2, 7.0, -95, '["aa","bb","cc","dd"]', ?)`, time.Now().Unix()) + + store.IngestNewObservations(maxObsID, 500) + + // After 30s elapsed, distDirty should be cleared (rebuild happened) + if store.distDirty { + t.Fatal("distDirty should be false after rebuild (30s elapsed)") + } +} diff --git a/cmd/server/store.go b/cmd/server/store.go index aa9204da..679013b1 100644 --- a/cmd/server/store.go +++ b/cmd/server/store.go @@ -61,6 +61,10 @@ type StoreObs struct { Timestamp string } +// distRebuildInterval is the minimum time between distance index rebuilds +// to avoid hot-looping on busy meshes. +const distRebuildInterval = 30 * time.Second + // PacketStore holds all transmissions in memory with indexes for fast queries. type PacketStore struct { mu sync.RWMutex @@ -117,6 +121,8 @@ type PacketStore struct { // computed during Load() and incrementally updated on ingest. distHops []distHopRecord distPaths []distPathRecord + distDirty bool // set when paths change; cleared after rebuild + distLast time.Time // last time distance index was rebuilt // Cached GetNodeHashSizeInfo result — recomputed at most once every 15s hashSizeInfoMu sync.Mutex @@ -329,6 +335,7 @@ func (s *PacketStore) Load() error { // Precompute distance analytics (hop distances, path totals) s.buildDistanceIndex() + s.distLast = time.Now() s.loaded = true elapsed := time.Since(t0) @@ -1470,25 +1477,29 @@ func (s *PacketStore) IngestNewObservations(sinceObsID, limit int) []map[string] } } - // Rebuild distance index if any paths changed (distances depend on path hops) + // Check if any paths changed (used for both distance rebuild and cache invalidation). + hasPathChanges := false for txID, tx := range updatedTxs { if tx.PathJSON != oldPaths[txID] { - s.buildDistanceIndex() + hasPathChanges = true break } } + // Mark distance index dirty if any paths changed (rebuild is debounced) + if hasPathChanges { + s.distDirty = true + } + if s.distDirty && time.Since(s.distLast) > distRebuildInterval { + s.buildDistanceIndex() + s.distDirty = false + s.distLast = time.Now() + } + if len(updatedTxs) > 0 { // Targeted cache invalidation: new observations always affect RF // analytics; topology/distance/subpath caches only if paths changed. // Channel and hash caches are unaffected by observation-only ingestion. - hasPathChanges := false - for txID, tx := range updatedTxs { - if tx.PathJSON != oldPaths[txID] { - hasPathChanges = true - break - } - } s.invalidateCachesFor(cacheInvalidation{ hasNewObservations: true, hasNewPaths: hasPathChanges,