fix: debounce distance index rebuild to prevent CPU hot loop (#557)

## Problem

On busy meshes (325K+ transmissions, 50 observers), the distance index
rebuild runs on **every ingest poll** (~1s interval), computing
haversine distances for 1M+ hop records. Each rebuild takes 2-3 seconds
but new observations arrive faster than it can finish, creating a CPU
hot loop that starves the HTTP server.

Discovered on the Cascadia Mesh instance where `corescope-server` was
consuming 15 minutes of CPU time in 10 minutes of uptime, the API was
completely unresponsive, and health checks were timing out.

### Server logs showing the hot loop:
```
[store] Built distance index: 1797778 hop records, 207072 path records
[store] Built distance index: 1797806 hop records, 207075 path records
[store] Built distance index: 1797811 hop records, 207075 path records
[store] Built distance index: 1797820 hop records, 207075 path records
```
Every 2 seconds, nonstop.

## Root Cause

`IngestNewObservations` calls `buildDistanceIndex()` synchronously
whenever `pickBestObservation` selects a longer path. With 50 observers
sending observations every second, paths change on nearly every poll
cycle, triggering a full rebuild each time.

## Fix

- Mark distance index dirty on path changes instead of rebuilding inline
- Rebuild at most every **30 seconds** (configurable via `distLast`
timer)
- Set `distLast` after initial `Load()` to prevent immediate re-rebuild
on first ingest
- Distance data is at most 30s stale — acceptable for an analytics view

## Testing

- `go build`, `go vet`, `go test` all pass
- No behavioral change for the initial load or the analytics API
response shape
- Distance data freshness goes from real-time to 30s max staleness

---------

Co-authored-by: Kpa-clawbot <259247574+Kpa-clawbot@users.noreply.github.com>
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Co-authored-by: you <you@example.com>

cmd/server/coverage_test.go

@@ -3913,3 +3913,45 @@ func TestBuildTransmissionWhereMultiObserver(t *testing.T) {
 		}
 	})
 }
+
+// --- Distance index rebuild debounce (#557) ---
+
+func TestDistanceRebuildDebounce(t *testing.T) {
+	db := setupTestDB(t)
+	defer db.Close()
+	seedTestData(t, db)
+	store := NewPacketStore(db, nil)
+	store.Load()
+
+	// After Load(), distLast is set to now — so distDirty should be false
+	if store.distDirty {
+		t.Fatal("distDirty should be false after Load()")
+	}
+
+	// Insert a new observation with a different path to trigger distDirty
+	maxObsID := db.GetMaxObservationID()
+	db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
+		VALUES (1, 2, 5.0, -100, '["xx","yy","zz"]', ?)`, time.Now().Unix())
+
+	store.IngestNewObservations(maxObsID, 500)
+
+	// distDirty should be true (30s hasn't elapsed since Load)
+	if !store.distDirty {
+		t.Fatal("distDirty should be true after path change within 30s window")
+	}
+
+	// Now simulate 30s having elapsed by backdating distLast
+	store.distLast = time.Now().Add(-31 * time.Second)
+
+	// Insert another observation to trigger another ingest cycle
+	maxObsID = db.GetMaxObservationID()
+	db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
+		VALUES (1, 2, 7.0, -95, '["aa","bb","cc","dd"]', ?)`, time.Now().Unix())
+
+	store.IngestNewObservations(maxObsID, 500)
+
+	// After 30s elapsed, distDirty should be cleared (rebuild happened)
+	if store.distDirty {
+		t.Fatal("distDirty should be false after rebuild (30s elapsed)")
+	}
+}

cmd/server/store.go

@@ -61,6 +61,10 @@ type StoreObs struct {
 	Timestamp      string
 }
 
+// distRebuildInterval is the minimum time between distance index rebuilds
+// to avoid hot-looping on busy meshes.
+const distRebuildInterval = 30 * time.Second
+
 // PacketStore holds all transmissions in memory with indexes for fast queries.
 type PacketStore struct {
 	mu            sync.RWMutex
@@ -117,6 +121,8 @@ type PacketStore struct {
 	// computed during Load() and incrementally updated on ingest.
 	distHops  []distHopRecord
 	distPaths []distPathRecord
+	distDirty bool      // set when paths change; cleared after rebuild
+	distLast  time.Time // last time distance index was rebuilt
 
 	// Cached GetNodeHashSizeInfo result — recomputed at most once every 15s
 	hashSizeInfoMu    sync.Mutex
@@ -329,6 +335,7 @@ func (s *PacketStore) Load() error {
 
 	// Precompute distance analytics (hop distances, path totals)
 	s.buildDistanceIndex()
+	s.distLast = time.Now()
 
 	s.loaded = true
 	elapsed := time.Since(t0)
@@ -1470,25 +1477,29 @@ func (s *PacketStore) IngestNewObservations(sinceObsID, limit int) []map[string]
 		}
 	}
 
-	// Rebuild distance index if any paths changed (distances depend on path hops)
+	// Check if any paths changed (used for both distance rebuild and cache invalidation).
+	hasPathChanges := false
 	for txID, tx := range updatedTxs {
 		if tx.PathJSON != oldPaths[txID] {
-			s.buildDistanceIndex()
+			hasPathChanges = true
 			break
 		}
 	}
 
+	// Mark distance index dirty if any paths changed (rebuild is debounced)
+	if hasPathChanges {
+		s.distDirty = true
+	}
+	if s.distDirty && time.Since(s.distLast) > distRebuildInterval {
+		s.buildDistanceIndex()
+		s.distDirty = false
+		s.distLast = time.Now()
+	}
+
 	if len(updatedTxs) > 0 {
 		// Targeted cache invalidation: new observations always affect RF
 		// analytics; topology/distance/subpath caches only if paths changed.
 		// Channel and hash caches are unaffected by observation-only ingestion.
-		hasPathChanges := false
-		for txID, tx := range updatedTxs {
-			if tx.PathJSON != oldPaths[txID] {
-				hasPathChanges = true
-				break
-			}
-		}
 		s.invalidateCachesFor(cacheInvalidation{
 			hasNewObservations: true,
 			hasNewPaths:        hasPathChanges,