mirror of
https://github.com/Kpa-clawbot/meshcore-analyzer.git
synced 2026-07-28 15:59:21 +00:00
fix(#791): drop singleton subpath entries to save ~150MB at scale
After the initial subpath index build, iterate and delete all entries where count==1. At scale, 60-70% of subpath keys are singletons that appear only once — they add no analytical value and consume significant memory (each entry is a string key + map bucket overhead). Newly-seen singletons during incremental ingest will remain in the index until the next restart. This is acceptable since singleton subpaths are noise for ranking/display purposes. Add a startup log line showing how many entries were kept vs dropped.
This commit is contained in:
+15
-2
@@ -2442,8 +2442,21 @@ func (s *PacketStore) buildSubpathIndex() {
|
||||
s.spTotalPaths++
|
||||
}
|
||||
}
|
||||
log.Printf("[store] Built subpath index: %d unique raw subpaths from %d paths",
|
||||
len(s.spIndex), s.spTotalPaths)
|
||||
// Drop singleton entries (count==1) to save memory. At scale, 60-70% of
|
||||
// subpath keys appear only once — they're noise and not useful for analytics.
|
||||
// Singletons that appear after startup via incremental ingest will remain
|
||||
// in the index until the next restart; this is acceptable since they only
|
||||
// matter for display ranking and the count is still correct for non-singletons.
|
||||
total := len(s.spIndex)
|
||||
dropped := 0
|
||||
for key, count := range s.spIndex {
|
||||
if count == 1 {
|
||||
delete(s.spIndex, key)
|
||||
dropped++
|
||||
}
|
||||
}
|
||||
log.Printf("[store] Subpath index: kept %d/%d entries (dropped %d singletons)",
|
||||
total-dropped, total, dropped)
|
||||
}
|
||||
|
||||
// buildPathHopIndex scans all packets and populates byPathHop.
|
||||
|
||||
Reference in New Issue
Block a user