Files
meshcore-analyzer/cmd/server/config.go
Kpa-clawbot 77d8f35a04 feat: implement packet store eviction/aging to prevent OOM (#273)
## Summary

The in-memory `PacketStore` had **no eviction or aging** — it grew
unbounded until OOM killed the process. At ~3K packets/hour and ~5KB per
packet (not the 450 bytes previously estimated), an 8GB VM would OOM in
a few days.

## Changes

### Time-based eviction
- Configurable via `config.json`: `"packetStore": { "retentionHours": 24
}`
- Packets older than the retention window are evicted from the head of
the sorted slice

### Memory-based cap
- Configurable via `"packetStore": { "maxMemoryMB": 1024 }`
- Hard ceiling — evicts oldest packets when estimated memory exceeds the
cap

### Index cleanup
When a `StoreTx` is evicted, ALL associated data is removed from:
- `byHash`, `byTxID`, `byObsID`, `byObserver`, `byNode`, `byPayloadType`
- `nodeHashes`, `distHops`, `distPaths`, `spIndex`

### Periodic execution
- Background ticker runs eviction every 60 seconds
- Analytics caches and hash size cache are invalidated after eviction

### Stats fixes
- `estimatedMB` now uses ~5KB/packet + ~500B/observation (was 430B +
200B)
- `evicted` counter reflects actual evictions (was hardcoded to 0)
- Removed fake `maxPackets: 2386092` and `maxMB: 1024` from stats

### Config example
```json
{
  "packetStore": {
    "retentionHours": 24,
    "maxMemoryMB": 1024
  }
}
```

Both values default to 0 (unlimited) for backward compatibility.

## Tests
- 7 new tests in `eviction_test.go` covering time-based, memory-based,
index cleanup, thread safety, config parsing, and no-op when disabled
- All existing tests pass unchanged

Co-authored-by: Kpa-clawbot <kpabap+clawdbot@gmail.com>
2026-03-30 03:42:11 +00:00

195 lines
5.8 KiB
Go

package main
import (
"encoding/json"
"os"
"path/filepath"
)
// Config mirrors the Node.js config.json structure (read-only fields).
type Config struct {
Port int `json:"port"`
APIKey string `json:"apiKey"`
DBPath string `json:"dbPath"`
Branding map[string]interface{} `json:"branding"`
Theme map[string]interface{} `json:"theme"`
ThemeDark map[string]interface{} `json:"themeDark"`
NodeColors map[string]interface{} `json:"nodeColors"`
TypeColors map[string]interface{} `json:"typeColors"`
Home map[string]interface{} `json:"home"`
MapDefaults struct {
Center []float64 `json:"center"`
Zoom int `json:"zoom"`
} `json:"mapDefaults"`
Regions map[string]string `json:"regions"`
Roles map[string]interface{} `json:"roles"`
HealthThresholds *HealthThresholds `json:"healthThresholds"`
Tiles map[string]interface{} `json:"tiles"`
SnrThresholds map[string]interface{} `json:"snrThresholds"`
DistThresholds map[string]interface{} `json:"distThresholds"`
MaxHopDist *float64 `json:"maxHopDist"`
Limits map[string]interface{} `json:"limits"`
PerfSlowMs *int `json:"perfSlowMs"`
WsReconnectMs *int `json:"wsReconnectMs"`
CacheInvalidMs *int `json:"cacheInvalidateMs"`
ExternalUrls map[string]interface{} `json:"externalUrls"`
LiveMap struct {
PropagationBufferMs int `json:"propagationBufferMs"`
} `json:"liveMap"`
CacheTTL map[string]interface{} `json:"cacheTTL"`
Retention *RetentionConfig `json:"retention,omitempty"`
PacketStore *PacketStoreConfig `json:"packetStore,omitempty"`
}
// PacketStoreConfig controls in-memory packet store limits.
type PacketStoreConfig struct {
RetentionHours float64 `json:"retentionHours"` // max age of packets in hours (0 = unlimited)
MaxMemoryMB int `json:"maxMemoryMB"` // hard memory ceiling in MB (0 = unlimited)
}
type RetentionConfig struct {
NodeDays int `json:"nodeDays"`
}
// NodeDaysOrDefault returns the configured retention.nodeDays or 7 if not set.
func (c *Config) NodeDaysOrDefault() int {
if c.Retention != nil && c.Retention.NodeDays > 0 {
return c.Retention.NodeDays
}
return 7
}
type HealthThresholds struct {
InfraDegradedHours float64 `json:"infraDegradedHours"`
InfraSilentHours float64 `json:"infraSilentHours"`
NodeDegradedHours float64 `json:"nodeDegradedHours"`
NodeSilentHours float64 `json:"nodeSilentHours"`
}
// ThemeFile mirrors theme.json overlay.
type ThemeFile struct {
Branding map[string]interface{} `json:"branding"`
Theme map[string]interface{} `json:"theme"`
ThemeDark map[string]interface{} `json:"themeDark"`
NodeColors map[string]interface{} `json:"nodeColors"`
TypeColors map[string]interface{} `json:"typeColors"`
Home map[string]interface{} `json:"home"`
}
func LoadConfig(baseDirs ...string) (*Config, error) {
if len(baseDirs) == 0 {
baseDirs = []string{"."}
}
paths := make([]string, 0)
for _, d := range baseDirs {
paths = append(paths, filepath.Join(d, "config.json"))
paths = append(paths, filepath.Join(d, "data", "config.json"))
}
cfg := &Config{Port: 3000}
for _, p := range paths {
data, err := os.ReadFile(p)
if err != nil {
continue
}
if err := json.Unmarshal(data, cfg); err != nil {
continue
}
return cfg, nil
}
return cfg, nil // defaults
}
func LoadTheme(baseDirs ...string) *ThemeFile {
if len(baseDirs) == 0 {
baseDirs = []string{"."}
}
for _, d := range baseDirs {
for _, name := range []string{"theme.json"} {
p := filepath.Join(d, name)
data, err := os.ReadFile(p)
if err != nil {
p = filepath.Join(d, "data", name)
data, err = os.ReadFile(p)
if err != nil {
continue
}
}
var t ThemeFile
if json.Unmarshal(data, &t) == nil {
return &t
}
}
}
return &ThemeFile{}
}
func (c *Config) GetHealthThresholds() HealthThresholds {
h := HealthThresholds{
InfraDegradedHours: 24,
InfraSilentHours: 72,
NodeDegradedHours: 1,
NodeSilentHours: 24,
}
if c.HealthThresholds != nil {
if c.HealthThresholds.InfraDegradedHours > 0 {
h.InfraDegradedHours = c.HealthThresholds.InfraDegradedHours
}
if c.HealthThresholds.InfraSilentHours > 0 {
h.InfraSilentHours = c.HealthThresholds.InfraSilentHours
}
if c.HealthThresholds.NodeDegradedHours > 0 {
h.NodeDegradedHours = c.HealthThresholds.NodeDegradedHours
}
if c.HealthThresholds.NodeSilentHours > 0 {
h.NodeSilentHours = c.HealthThresholds.NodeSilentHours
}
}
return h
}
// GetHealthMs returns degraded/silent thresholds in ms for a given role.
func (h HealthThresholds) GetHealthMs(role string) (degradedMs, silentMs int) {
const hourMs = 3600000
if role == "repeater" || role == "room" {
return int(h.InfraDegradedHours * hourMs), int(h.InfraSilentHours * hourMs)
}
return int(h.NodeDegradedHours * hourMs), int(h.NodeSilentHours * hourMs)
}
// ToClientMs returns the thresholds as ms for the frontend.
func (h HealthThresholds) ToClientMs() map[string]int {
const hourMs = 3600000
return map[string]int{
"infraDegradedMs": int(h.InfraDegradedHours * hourMs),
"infraSilentMs": int(h.InfraSilentHours * hourMs),
"nodeDegradedMs": int(h.NodeDegradedHours * hourMs),
"nodeSilentMs": int(h.NodeSilentHours * hourMs),
}
}
func (c *Config) ResolveDBPath(baseDir string) string {
if c.DBPath != "" {
return c.DBPath
}
if v := os.Getenv("DB_PATH"); v != "" {
return v
}
return filepath.Join(baseDir, "data", "meshcore.db")
}
func (c *Config) PropagationBufferMs() int {
if c.LiveMap.PropagationBufferMs > 0 {
return c.LiveMap.PropagationBufferMs
}
return 5000
}