From 6cdbf7e3f6ef9786e756e3f39e1f5d5528d2eb35 Mon Sep 17 00:00:00 2001 From: Kpa-clawbot <259247574+Kpa-clawbot@users.noreply.github.com> Date: Fri, 27 Mar 2026 09:49:46 -0700 Subject: [PATCH] perf(go): remove debug logging, update history Remove temporary rf-cache debug logs. Update hicks history with endpoint optimization learnings. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .squad/agents/hicks/history.md | 4 ++++ cmd/server/store.go | 7 ------- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.squad/agents/hicks/history.md b/.squad/agents/hicks/history.md index 42d0e7bd..354c661d 100644 --- a/.squad/agents/hicks/history.md +++ b/.squad/agents/hicks/history.md @@ -17,3 +17,7 @@ User: User - Fixed #133 (node count keeps climbing): `db.getStats().totalNodes` used `SELECT COUNT(*) FROM nodes` which counts every node ever seen ā€” 6800+ on a ~200-400 node mesh. Changed `totalNodes` to count only nodes with `last_seen` within 7 days. Added `totalNodesAllTime` for the full historical count. Also filtered role counts in `/api/stats` to the same 7-day window. Added `countActiveNodes` and `countActiveNodesByRole` prepared statements in db.js. 6 new tests (95 total in test-db.js). The existing `idx_nodes_last_seen` index covers the new queries. - Go server FULL API parity: Rewrote QueryGroupedPackets from packets_v VIEW scan (8s on 1.2M rows) to transmission-centric query (<100ms). Fixed GetStats to use 7-day window for totalNodes + added totalNodesAllTime. Split GetRoleCounts into 7-day (for /api/stats) and all-time (for /api/nodes). Added packetsLastHour + node lat/lon/role to /api/observers via batch queries (GetObserverPacketCounts, GetNodeLocations). Added multi-node filter support (/api/packets?nodes=pk1,pk2). Fixed /api/packets/:id to return parsed path_json in path field. Populated bulk-health per-node stats from SQL. Updated test seed data to use dynamic timestamps for 7-day filter compatibility. All 42+ tests pass, go vet clean. - Fixed #133 ROOT CAUSE (phantom nodes): `autoLearnHopNodes` in server.js was calling `db.upsertNode()` for every unresolved hop prefix, creating thousands of fake "repeater" nodes with short public_keys (just the 2-4 byte hop prefix). Removed the `upsertNode` call entirely ā€” unresolved hops are now simply cached to skip repeat DB lookups, and display as raw hex prefixes via hop-resolver. Added `db.removePhantomNodes()` that deletes nodes with `LENGTH(public_key) <= 16` (real pubkeys are 64 hex chars). Called at server startup to purge existing phantoms. 14 new test assertions (109 total in test-db.js). +- Fixed #126 (offline node showing on map due to hash prefix collision): `updatePathSeenTimestamps()` and `autoLearnHopNodes()` used `LIKE prefix%` DB queries that non-deterministically picked the first match when multiple nodes shared a hash prefix (e.g. `1CC4` and `1C82` both start with `1C` under 1-byte hash_size). Extracted `resolveUniquePrefixMatch()` that checks for uniqueness ā€” ambiguous prefixes (matching 2+ nodes) are skipped and cached in a negative-cache Set. This prevents dead nodes from getting `last_heard` updates from packets that actually belong to a different node. 3 new tests (207 total in test-server-routes.js). +- Fixed #123 (channel hash for undecrypted GRP_TXT): Added `channelHashHex` (zero-padded uppercase hex) and `decryptionStatus` ('decrypted'|'no_key'|'decryption_failed') fields to `decodeGrpTxt` in decoder.js. Distinguishes between "no channel keys configured" vs "keys tried but decryption failed." Frontend packets.js updated: list preview shows "šŸ”’ Ch 0xXX (status)", detail pane hex breakdown and message area show channel hash with status label. 6 new tests (58 total in test-decoder.js). +- Ported in-memory packet store to Go (`cmd/server/store.go`). PacketStore loads all transmissions + observations from SQLite at startup via streaming query (no .all()), builds 5 indexes (byHash, byTxID, byObsID, byObserver, byNode), picks longest-path observation per transmission for display fields. QueryPackets and QueryGroupedPackets serve from memory with full filter support (type, route, observer, hash, since, until, region, node). Poller ingests new transmissions into store via IngestNewFromDB. Server/routes fall back to direct DB queries when store is nil (backward-compatible with tests). All 42+ existing tests pass, go vet clean, go build clean. System Go 1.17 requires using Go 1.22.5 at C:\go1.22\go\bin. +- Fixed 3 critically slow Go endpoints by switching from SQLite queries against packets_v VIEW (1.2M rows) to in-memory PacketStore queries. `/api/channels` 7.2sā†’37ms (195Ɨ), `/api/channels/:hash/messages` 8.2sā†’36ms (228Ɨ), `/api/analytics/rf` 4.2sā†’90ms avg (47Ɨ). Key optimizations: (1) byPayloadType index reduces channels scan from 52K to 17K packets, (2) struct-based JSON decode avoids map[string]interface{} allocations, (3) per-transmission work hoisted out of 1.2M observation loop for RF, (4) eliminated second-pass time.Parse over 1.2M observations (track min/max timestamps as strings instead), (5) pre-allocated slices with capacity hints, (6) 15-second TTL cache for RF analytics (separate mutex to avoid contention with store RWMutex). Cache invalidation is TTL-only because live mesh generates continuous ingest events. Also fixed `/api/analytics/channels` to use store. All handlers fall back to DB when store is nil (test compat). diff --git a/cmd/server/store.go b/cmd/server/store.go index d22cdb04..27e4ff33 100644 --- a/cmd/server/store.go +++ b/cmd/server/store.go @@ -1236,25 +1236,18 @@ func (s *PacketStore) GetChannelMessages(channelHash string, limit, offset int) // GetAnalyticsRF returns full RF analytics computed from in-memory observations. func (s *PacketStore) GetAnalyticsRF(region string) map[string]interface{} { - // Check cache first s.cacheMu.Lock() if cached, ok := s.rfCache[region]; ok && time.Now().Before(cached.expiresAt) { s.cacheMu.Unlock() - log.Printf("[rf-cache] HIT region=%q ttl=%.1fs", region, time.Until(cached.expiresAt).Seconds()) return cached.data } s.cacheMu.Unlock() - // Compute fresh result - t0 := time.Now() result := s.computeAnalyticsRF(region) - elapsed := time.Since(t0) - // Cache result s.cacheMu.Lock() s.rfCache[region] = &cachedResult{data: result, expiresAt: time.Now().Add(s.rfCacheTTL)} s.cacheMu.Unlock() - log.Printf("[rf-cache] MISS region=%q computed in %v, cached for %v", region, elapsed, s.rfCacheTTL) return result }