From 8d89f7d3e973f70fa8d4ef4b2815e6175169e6bb Mon Sep 17 00:00:00 2001
From: you <you@example.com>
Date: Sun, 19 Apr 2026 00:03:37 +0000
Subject: [PATCH] fix(#791): drop singleton subpath entries to save ~150MB at
 scale
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

After the initial subpath index build, iterate and delete all entries
where count==1. At scale, 60-70% of subpath keys are singletons that
appear only once — they add no analytical value and consume significant
memory (each entry is a string key + map bucket overhead).

Newly-seen singletons during incremental ingest will remain in the
index until the next restart. This is acceptable since singleton
subpaths are noise for ranking/display purposes.

Add a startup log line showing how many entries were kept vs dropped.
---
 cmd/server/store.go | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/cmd/server/store.go b/cmd/server/store.go
index 52167978..d2354d4d 100644
--- a/cmd/server/store.go
+++ b/cmd/server/store.go
@@ -2442,8 +2442,21 @@ func (s *PacketStore) buildSubpathIndex() {
 			s.spTotalPaths++
 		}
 	}
-	log.Printf("[store] Built subpath index: %d unique raw subpaths from %d paths",
-		len(s.spIndex), s.spTotalPaths)
+	// Drop singleton entries (count==1) to save memory. At scale, 60-70% of
+	// subpath keys appear only once — they're noise and not useful for analytics.
+	// Singletons that appear after startup via incremental ingest will remain
+	// in the index until the next restart; this is acceptable since they only
+	// matter for display ranking and the count is still correct for non-singletons.
+	total := len(s.spIndex)
+	dropped := 0
+	for key, count := range s.spIndex {
+		if count == 1 {
+			delete(s.spIndex, key)
+			dropped++
+		}
+	}
+	log.Printf("[store] Subpath index: kept %d/%d entries (dropped %d singletons)",
+		total-dropped, total, dropped)
 }
 
 // buildPathHopIndex scans all packets and populates byPathHop.