diff --git a/README.md b/README.md index 5035b87e..d3db6e1c 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ The Go backend serves all 40+ API endpoints from an in-memory packet store with | Memory (56K packets) | **~300 MB** (vs 1.3 GB on Node.js) | | WebSocket broadcast | **Real-time** to all connected browsers | | Channel decryption | **AES-128-ECB** with rainbow table | +| GOMEMLIMIT (memory-constrained hosts) | **set to ≥1.5× working set** (e.g. 1536 MiB on a 2 GB Pi for a ~1 GB store). Lower values trigger a GC death-spiral. Configure via the `GOMEMLIMIT` env var or `runtime.maxMemoryMB` in `config.json`; env wins. Applies to both server and ingestor. See [#1010](https://github.com/Kpa-clawbot/CoreScope/issues/1010). | See [PERFORMANCE.md](PERFORMANCE.md) for full benchmarks. diff --git a/cmd/ingestor/config.go b/cmd/ingestor/config.go index a999fd18..10e81195 100644 --- a/cmd/ingestor/config.go +++ b/cmd/ingestor/config.go @@ -53,6 +53,7 @@ type Config struct { HashRegions []string `json:"hashRegions,omitempty"` Retention *RetentionConfig `json:"retention,omitempty"` Metrics *MetricsConfig `json:"metrics,omitempty"` + Runtime *RuntimeConfig `json:"runtime,omitempty"` GeoFilter *GeoFilterConfig `json:"geo_filter,omitempty"` ForeignAdverts *ForeignAdvertConfig `json:"foreignAdverts,omitempty"` ValidateSignatures *bool `json:"validateSignatures,omitempty"` @@ -151,6 +152,15 @@ type MetricsConfig struct { SampleIntervalSec int `json:"sampleIntervalSec"` } +// RuntimeConfig holds Go runtime tuning knobs (#1010). +type RuntimeConfig struct { + // MaxMemoryMB is the soft memory limit (GOMEMLIMIT) in MiB applied via + // runtime/debug.SetMemoryLimit at startup. The GOMEMLIMIT environment + // variable, when set, takes precedence over this value. 0/unset means + // no limit is applied and default Go runtime behavior is preserved. + MaxMemoryMB int `json:"maxMemoryMB"` +} + // DBConfig is the shared SQLite vacuum/maintenance config (#919, #921). type DBConfig = dbconfig.DBConfig diff --git a/cmd/ingestor/main.go b/cmd/ingestor/main.go index 0832d69c..9e33884f 100644 --- a/cmd/ingestor/main.go +++ b/cmd/ingestor/main.go @@ -51,6 +51,25 @@ func main() { log.Fatalf("config: %v", err) } + // Apply Go runtime soft memory limit (GOMEMLIMIT). See #1010. + // Precedence: GOMEMLIMIT env > runtime.maxMemoryMB > unset (default). + { + _, envSet := os.LookupEnv("GOMEMLIMIT") + runtimeMaxMB := 0 + if cfg.Runtime != nil { + runtimeMaxMB = cfg.Runtime.MaxMemoryMB + } + limit, source := applyMemoryLimit(runtimeMaxMB, envSet) + switch source { + case "env": + log.Printf("[memlimit] using GOMEMLIMIT from environment (%s)", os.Getenv("GOMEMLIMIT")) + case "config": + log.Printf("[memlimit] runtime.maxMemoryMB=%d → SetMemoryLimit(%d MiB)", runtimeMaxMB, limit/(1024*1024)) + default: + log.Printf("[memlimit] unset → default (no soft memory limit; recommend setting GOMEMLIMIT or runtime.maxMemoryMB to ≥1.5× working set to avoid OOM-kill)") + } + } + sources := cfg.ResolvedSources() store, err := OpenStoreWithInterval(cfg.DBPath, cfg.MetricsSampleInterval()) diff --git a/cmd/ingestor/memlimit.go b/cmd/ingestor/memlimit.go new file mode 100644 index 00000000..509eeabb --- /dev/null +++ b/cmd/ingestor/memlimit.go @@ -0,0 +1,26 @@ +package main + +import "runtime/debug" + +// applyMemoryLimit configures Go's soft memory limit (GOMEMLIMIT) for the +// ingestor process. See #1010. +// +// Precedence: +// 1. GOMEMLIMIT env var (parsed by the runtime at startup) — we do not +// override; report source="env" with limit=0. +// 2. runtimeMaxMB > 0 (from config runtime.maxMemoryMB) — set limit of +// runtimeMaxMB MiB via debug.SetMemoryLimit; source="config". +// 3. Otherwise no limit applied; source="none" (default behavior). +// +// Returns the limit (bytes) we set, or 0 if we did not set one. +func applyMemoryLimit(runtimeMaxMB int, envSet bool) (int64, string) { + if envSet { + return 0, "env" + } + if runtimeMaxMB <= 0 { + return 0, "none" + } + limit := int64(runtimeMaxMB) * 1024 * 1024 + debug.SetMemoryLimit(limit) + return limit, "config" +} diff --git a/cmd/ingestor/memlimit_test.go b/cmd/ingestor/memlimit_test.go new file mode 100644 index 00000000..493bbb8f --- /dev/null +++ b/cmd/ingestor/memlimit_test.go @@ -0,0 +1,71 @@ +package main + +import ( + "runtime/debug" + "testing" +) + +// TestApplyMemoryLimit_FromEnv: when GOMEMLIMIT env var is set, the runtime +// already parsed it. Our function MUST NOT override and MUST report env source. +func TestApplyMemoryLimit_FromEnv(t *testing.T) { + t.Setenv("GOMEMLIMIT", "850MiB") + defer debug.SetMemoryLimit(-1) + + limit, source := applyMemoryLimit(512, true /* envSet */) + if source != "env" { + t.Fatalf("expected source=env, got %q", source) + } + if limit != 0 { + t.Fatalf("expected limit=0 (not set by us), got %d", limit) + } +} + +// TestApplyMemoryLimit_FromConfig: when env is unset and runtime.maxMemoryMB +// is set, derive a limit of exactly runtimeMaxMB * 1 MiB (no headroom — the +// ingestor's working set is bounded by MQTT batch decode, not packet store). +func TestApplyMemoryLimit_FromConfig(t *testing.T) { + defer debug.SetMemoryLimit(-1) + + limit, source := applyMemoryLimit(512, false /* envSet */) + if source != "config" { + t.Fatalf("expected source=config, got %q", source) + } + want := int64(512) * 1024 * 1024 + if limit != want { + t.Fatalf("expected limit=%d, got %d", want, limit) + } + cur := debug.SetMemoryLimit(-1) + if cur != want { + t.Fatalf("runtime memory limit not set: want=%d got=%d", want, cur) + } +} + +// TestApplyMemoryLimit_None: neither env nor config — no limit applied, +// default behavior preserved. +func TestApplyMemoryLimit_None(t *testing.T) { + defer debug.SetMemoryLimit(-1) + debug.SetMemoryLimit(int64(1<<63 - 1)) // math.MaxInt64 = "no limit" + + limit, source := applyMemoryLimit(0, false) + if source != "none" { + t.Fatalf("expected source=none, got %q", source) + } + if limit != 0 { + t.Fatalf("expected limit=0, got %d", limit) + } +} + +// TestApplyMemoryLimit_EnvWinsOverConfig: env set AND config set → env wins, +// our function does not override. Locks the precedence triage specified. +func TestApplyMemoryLimit_EnvWinsOverConfig(t *testing.T) { + t.Setenv("GOMEMLIMIT", "1GiB") + defer debug.SetMemoryLimit(-1) + + limit, source := applyMemoryLimit(512, true /* envSet */) + if source != "env" { + t.Fatalf("expected source=env when both set, got %q", source) + } + if limit != 0 { + t.Fatalf("expected limit=0 when env wins, got %d", limit) + } +} diff --git a/cmd/server/config.go b/cmd/server/config.go index e56a9b9d..aece9ba3 100644 --- a/cmd/server/config.go +++ b/cmd/server/config.go @@ -86,6 +86,11 @@ type Config struct { PacketStore *PacketStoreConfig `json:"packetStore,omitempty"` + // Runtime holds Go runtime tuning knobs (#1010). + // Currently exposes runtime.maxMemoryMB which sets a soft memory limit + // (GOMEMLIMIT) via runtime/debug.SetMemoryLimit at startup. The + // GOMEMLIMIT environment variable, when set, takes precedence. + Runtime *RuntimeConfig `json:"runtime,omitempty"` GeoFilter *GeoFilterConfig `json:"geo_filter,omitempty"` Areas map[string]AreaEntry `json:"areas,omitempty"` @@ -237,6 +242,16 @@ type PacketStoreConfig struct { // GeoFilterConfig is an alias for the shared geofilter.Config type. type GeoFilterConfig = geofilter.Config +// RuntimeConfig holds Go runtime tuning knobs (#1010). +type RuntimeConfig struct { + // MaxMemoryMB sets the Go soft memory limit (GOMEMLIMIT) in MiB via + // runtime/debug.SetMemoryLimit at startup. Takes precedence over the + // implicit limit derived from packetStore.maxMemoryMB. The GOMEMLIMIT + // environment variable, when set, takes precedence over this value. + // 0/unset preserves default behavior. + MaxMemoryMB int `json:"maxMemoryMB"` +} + type RetentionConfig struct { NodeDays int `json:"nodeDays"` ObserverDays int `json:"observerDays"` diff --git a/cmd/server/main.go b/cmd/server/main.go index 419d6c5b..e6a5ee82 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -109,22 +109,37 @@ func main() { log.Printf("[security] WARNING: API key is weak or a known default — write endpoints are vulnerable") } - // Apply Go runtime soft memory limit (#836). - // Honors GOMEMLIMIT if set; otherwise derives from packetStore.maxMemoryMB. + // Apply Go runtime soft memory limit (#836, #1010). + // Precedence: GOMEMLIMIT env > runtime.maxMemoryMB > derived from packetStore.maxMemoryMB. { _, envSet := os.LookupEnv("GOMEMLIMIT") + runtimeMaxMB := 0 + if cfg.Runtime != nil { + runtimeMaxMB = cfg.Runtime.MaxMemoryMB + } maxMB := 0 if cfg.PacketStore != nil { maxMB = cfg.PacketStore.MaxMemoryMB } - limit, source := applyMemoryLimit(maxMB, envSet) + // runtime.maxMemoryMB (explicit) wins over packetStore-derived (implicit). + effectiveMB := maxMB + usedRuntimeCfg := false + if !envSet && runtimeMaxMB > 0 { + effectiveMB = runtimeMaxMB + usedRuntimeCfg = true + } + limit, source := applyMemoryLimit(effectiveMB, envSet) switch source { case "env": log.Printf("[memlimit] using GOMEMLIMIT from environment (%s)", os.Getenv("GOMEMLIMIT")) case "derived": - log.Printf("[memlimit] derived from packetStore.maxMemoryMB=%d → %d MiB (1.5x headroom)", maxMB, limit/(1024*1024)) + if usedRuntimeCfg { + log.Printf("[memlimit] runtime.maxMemoryMB=%d → %d MiB (1.5x headroom)", runtimeMaxMB, limit/(1024*1024)) + } else { + log.Printf("[memlimit] derived from packetStore.maxMemoryMB=%d → %d MiB (1.5x headroom)", maxMB, limit/(1024*1024)) + } default: - log.Printf("[memlimit] no soft memory limit set (GOMEMLIMIT unset, packetStore.maxMemoryMB=0); recommend setting one to avoid container OOM-kill") + log.Printf("[memlimit] unset → default (no soft memory limit; recommend setting GOMEMLIMIT or runtime.maxMemoryMB to ≥1.5× working set to avoid OOM-kill)") } warnIfMemlimitUnderprovisioned(limit) } diff --git a/config.example.json b/config.example.json index 4e0062f7..4a105508 100644 --- a/config.example.json +++ b/config.example.json @@ -295,6 +295,10 @@ "_comment": "In-memory packet store. maxMemoryMB caps RAM usage. retentionHours: only packets younger than this are kept in memory (0 = unlimited). hotStartupHours: hours loaded synchronously at startup; background loader fills the remaining retentionHours window. 0 = disabled (loads full retentionHours synchronously, legacy behavior). Set to a positive value (e.g. 24) to reduce startup time on large DBs.", "_comment_gomemlimit": "On startup the server reads GOMEMLIMIT from the environment if set; otherwise it derives a Go runtime soft memory limit of maxMemoryMB * 1.5 and applies it via debug.SetMemoryLimit. This forces aggressive GC under cgroup pressure so the process self-throttles before the kernel SIGKILLs it. To override, set GOMEMLIMIT explicitly (e.g. GOMEMLIMIT=850MiB). See issue #836." }, + "runtime": { + "maxMemoryMB": 0, + "_comment_runtime_maxMemoryMB": "Go soft memory limit (GOMEMLIMIT) in MiB applied via runtime/debug.SetMemoryLimit at startup. Precedence: GOMEMLIMIT env var > runtime.maxMemoryMB > packetStore.maxMemoryMB-derived (server only). 0 (default) preserves existing behavior. Set on memory-constrained deployments (2 GB Pi, 4 GB VMs) to trigger earlier GC and avoid container OOM-kill. Floor recommendation: ≥1.5× working set; setting below the working set causes a GC death-spiral. Applies to BOTH cmd/server and cmd/ingestor. See #1010." + }, "resolvedPath": { "backfillHours": 24, "_comment": "How far back (hours) the async backfill scans for observations with NULL resolved_path. Default: 24. Set higher to backfill older data, lower to speed up startup."