feat(memlimit): GOMEMLIMIT via runtime.maxMemoryMB in server + ingestor (#1010) (#1595)

Red commit: 929da3c6dc — CI:
https://github.com/Kpa-clawbot/CoreScope/commit/929da3c6dcc1b619c27478291125d1c91323db8f/checks

Fixes #1010.

## What
Adds `GOMEMLIMIT` support to both `cmd/server` and `cmd/ingestor` per
the locked triage scope on #1010.

Precedence (env wins):
1. `GOMEMLIMIT` env var
2. `runtime.maxMemoryMB` config field (new)
3. Server only: implicit `packetStore.maxMemoryMB * 1.5` (existing #836
behavior, unchanged when `runtime.maxMemoryMB` is absent)
4. Otherwise unset — default Go behavior preserved (backwards
compatible)

Each startup logs a `[memlimit]` line echoing the effective
source/limit, or an "unset → default" note when neither is set.

## Changes
- `cmd/ingestor/memlimit.go` — new, `applyMemoryLimit(runtimeMaxMB,
envSet)`.
- `cmd/ingestor/memlimit_test.go` — new, env/config/none/precedence
assertions.
- `cmd/ingestor/config.go` — new `RuntimeConfig{MaxMemoryMB int}` field.
- `cmd/ingestor/main.go` — wires `applyMemoryLimit` into startup right
after `LoadConfig`.
- `cmd/server/config.go` — new `RuntimeConfig` + `cfg.Runtime` field.
- `cmd/server/main.go` — adds explicit `runtime.maxMemoryMB` precedence
over packetStore-derived; existing `warnIfMemlimitUnderprovisioned`
(#1264) unchanged.
- `config.example.json` — new `runtime` block with
`_comment_runtime_maxMemoryMB` per the Config Documentation Rule.
- `README.md` — sizing-table row with ≥1.5× working set floor +
death-spiral warning.

## TDD
- Red: `929da3c6` — ingestor `applyMemoryLimit` stub returns
`(0,"none")`; four tests fail on assertions (`expected source=env, got
"none"`, etc.) — no compile errors.
- Green: `953ec9d8` — implements ingestor `applyMemoryLimit`, wires
startup, threads `runtime.maxMemoryMB` through server too.

## Preflight
`bash ~/.openclaw/skills/pr-preflight/scripts/run-all.sh origin/master`
→ clean (all gates pass, all warnings pass).

## Out of scope
- `pprof`-verified GC-trigger acceptance criterion from the original
issue — requires production tracing; the triage scope is the
operator-tunable plumbing.
- Container auto-detection of cgroup memory limit (already covered by
#1264's `warnIfMemlimitUnderprovisioned`).

---------

Co-authored-by: corescope-bot <bot@corescope>

README.md

@@ -21,6 +21,7 @@ The Go backend serves all 40+ API endpoints from an in-memory packet store with
 | Memory (56K packets) | **~300 MB** (vs 1.3 GB on Node.js) |
 | WebSocket broadcast | **Real-time** to all connected browsers |
 | Channel decryption | **AES-128-ECB** with rainbow table |
+| GOMEMLIMIT (memory-constrained hosts) | **set to ≥1.5× working set** (e.g. 1536 MiB on a 2 GB Pi for a ~1 GB store). Lower values trigger a GC death-spiral. Configure via the `GOMEMLIMIT` env var or `runtime.maxMemoryMB` in `config.json`; env wins. Applies to both server and ingestor. See [#1010](https://github.com/Kpa-clawbot/CoreScope/issues/1010). |
 
 See [PERFORMANCE.md](PERFORMANCE.md) for full benchmarks.
 

cmd/ingestor/config.go

@@ -53,6 +53,7 @@ type Config struct {
 	HashRegions     []string          `json:"hashRegions,omitempty"`
 	Retention       *RetentionConfig  `json:"retention,omitempty"`
 	Metrics         *MetricsConfig    `json:"metrics,omitempty"`
+	Runtime         *RuntimeConfig    `json:"runtime,omitempty"`
 	GeoFilter            *GeoFilterConfig     `json:"geo_filter,omitempty"`
 	ForeignAdverts       *ForeignAdvertConfig `json:"foreignAdverts,omitempty"`
 	ValidateSignatures   *bool             `json:"validateSignatures,omitempty"`
@@ -151,6 +152,15 @@ type MetricsConfig struct {
 	SampleIntervalSec int `json:"sampleIntervalSec"`
 }
 
+// RuntimeConfig holds Go runtime tuning knobs (#1010).
+type RuntimeConfig struct {
+	// MaxMemoryMB is the soft memory limit (GOMEMLIMIT) in MiB applied via
+	// runtime/debug.SetMemoryLimit at startup. The GOMEMLIMIT environment
+	// variable, when set, takes precedence over this value. 0/unset means
+	// no limit is applied and default Go runtime behavior is preserved.
+	MaxMemoryMB int `json:"maxMemoryMB"`
+}
+
 // DBConfig is the shared SQLite vacuum/maintenance config (#919, #921).
 type DBConfig = dbconfig.DBConfig
 

cmd/ingestor/main.go

@@ -51,6 +51,25 @@ func main() {
 		log.Fatalf("config: %v", err)
 	}
 
+	// Apply Go runtime soft memory limit (GOMEMLIMIT). See #1010.
+	// Precedence: GOMEMLIMIT env > runtime.maxMemoryMB > unset (default).
+	{
+		_, envSet := os.LookupEnv("GOMEMLIMIT")
+		runtimeMaxMB := 0
+		if cfg.Runtime != nil {
+			runtimeMaxMB = cfg.Runtime.MaxMemoryMB
+		}
+		limit, source := applyMemoryLimit(runtimeMaxMB, envSet)
+		switch source {
+		case "env":
+			log.Printf("[memlimit] using GOMEMLIMIT from environment (%s)", os.Getenv("GOMEMLIMIT"))
+		case "config":
+			log.Printf("[memlimit] runtime.maxMemoryMB=%d → SetMemoryLimit(%d MiB)", runtimeMaxMB, limit/(1024*1024))
+		default:
+			log.Printf("[memlimit] unset → default (no soft memory limit; recommend setting GOMEMLIMIT or runtime.maxMemoryMB to ≥1.5× working set to avoid OOM-kill)")
+		}
+	}
+
 	sources := cfg.ResolvedSources()
 
 	store, err := OpenStoreWithInterval(cfg.DBPath, cfg.MetricsSampleInterval())

cmd/ingestor/memlimit.go

@@ -0,0 +1,26 @@
+package main
+
+import "runtime/debug"
+
+// applyMemoryLimit configures Go's soft memory limit (GOMEMLIMIT) for the
+// ingestor process. See #1010.
+//
+// Precedence:
+//  1. GOMEMLIMIT env var (parsed by the runtime at startup) — we do not
+//     override; report source="env" with limit=0.
+//  2. runtimeMaxMB > 0 (from config runtime.maxMemoryMB) — set limit of
+//     runtimeMaxMB MiB via debug.SetMemoryLimit; source="config".
+//  3. Otherwise no limit applied; source="none" (default behavior).
+//
+// Returns the limit (bytes) we set, or 0 if we did not set one.
+func applyMemoryLimit(runtimeMaxMB int, envSet bool) (int64, string) {
+	if envSet {
+		return 0, "env"
+	}
+	if runtimeMaxMB <= 0 {
+		return 0, "none"
+	}
+	limit := int64(runtimeMaxMB) * 1024 * 1024
+	debug.SetMemoryLimit(limit)
+	return limit, "config"
+}

cmd/ingestor/memlimit_test.go

@@ -0,0 +1,71 @@
+package main
+
+import (
+	"runtime/debug"
+	"testing"
+)
+
+// TestApplyMemoryLimit_FromEnv: when GOMEMLIMIT env var is set, the runtime
+// already parsed it. Our function MUST NOT override and MUST report env source.
+func TestApplyMemoryLimit_FromEnv(t *testing.T) {
+	t.Setenv("GOMEMLIMIT", "850MiB")
+	defer debug.SetMemoryLimit(-1)
+
+	limit, source := applyMemoryLimit(512, true /* envSet */)
+	if source != "env" {
+		t.Fatalf("expected source=env, got %q", source)
+	}
+	if limit != 0 {
+		t.Fatalf("expected limit=0 (not set by us), got %d", limit)
+	}
+}
+
+// TestApplyMemoryLimit_FromConfig: when env is unset and runtime.maxMemoryMB
+// is set, derive a limit of exactly runtimeMaxMB * 1 MiB (no headroom — the
+// ingestor's working set is bounded by MQTT batch decode, not packet store).
+func TestApplyMemoryLimit_FromConfig(t *testing.T) {
+	defer debug.SetMemoryLimit(-1)
+
+	limit, source := applyMemoryLimit(512, false /* envSet */)
+	if source != "config" {
+		t.Fatalf("expected source=config, got %q", source)
+	}
+	want := int64(512) * 1024 * 1024
+	if limit != want {
+		t.Fatalf("expected limit=%d, got %d", want, limit)
+	}
+	cur := debug.SetMemoryLimit(-1)
+	if cur != want {
+		t.Fatalf("runtime memory limit not set: want=%d got=%d", want, cur)
+	}
+}
+
+// TestApplyMemoryLimit_None: neither env nor config — no limit applied,
+// default behavior preserved.
+func TestApplyMemoryLimit_None(t *testing.T) {
+	defer debug.SetMemoryLimit(-1)
+	debug.SetMemoryLimit(int64(1<<63 - 1)) // math.MaxInt64 = "no limit"
+
+	limit, source := applyMemoryLimit(0, false)
+	if source != "none" {
+		t.Fatalf("expected source=none, got %q", source)
+	}
+	if limit != 0 {
+		t.Fatalf("expected limit=0, got %d", limit)
+	}
+}
+
+// TestApplyMemoryLimit_EnvWinsOverConfig: env set AND config set → env wins,
+// our function does not override. Locks the precedence triage specified.
+func TestApplyMemoryLimit_EnvWinsOverConfig(t *testing.T) {
+	t.Setenv("GOMEMLIMIT", "1GiB")
+	defer debug.SetMemoryLimit(-1)
+
+	limit, source := applyMemoryLimit(512, true /* envSet */)
+	if source != "env" {
+		t.Fatalf("expected source=env when both set, got %q", source)
+	}
+	if limit != 0 {
+		t.Fatalf("expected limit=0 when env wins, got %d", limit)
+	}
+}

cmd/server/config.go

@@ -86,6 +86,11 @@ type Config struct {
 
 	PacketStore *PacketStoreConfig `json:"packetStore,omitempty"`
 
+	// Runtime holds Go runtime tuning knobs (#1010).
+	// Currently exposes runtime.maxMemoryMB which sets a soft memory limit
+	// (GOMEMLIMIT) via runtime/debug.SetMemoryLimit at startup. The
+	// GOMEMLIMIT environment variable, when set, takes precedence.
+	Runtime *RuntimeConfig `json:"runtime,omitempty"`
 	GeoFilter *GeoFilterConfig `json:"geo_filter,omitempty"`
 
 	Areas map[string]AreaEntry `json:"areas,omitempty"`
@@ -237,6 +242,16 @@ type PacketStoreConfig struct {
 // GeoFilterConfig is an alias for the shared geofilter.Config type.
 type GeoFilterConfig = geofilter.Config
 
+// RuntimeConfig holds Go runtime tuning knobs (#1010).
+type RuntimeConfig struct {
+	// MaxMemoryMB sets the Go soft memory limit (GOMEMLIMIT) in MiB via
+	// runtime/debug.SetMemoryLimit at startup. Takes precedence over the
+	// implicit limit derived from packetStore.maxMemoryMB. The GOMEMLIMIT
+	// environment variable, when set, takes precedence over this value.
+	// 0/unset preserves default behavior.
+	MaxMemoryMB int `json:"maxMemoryMB"`
+}
+
 type RetentionConfig struct {
 	NodeDays      int `json:"nodeDays"`
 	ObserverDays  int `json:"observerDays"`

cmd/server/main.go

@@ -109,22 +109,37 @@ func main() {
 		log.Printf("[security] WARNING: API key is weak or a known default — write endpoints are vulnerable")
 	}
 
-	// Apply Go runtime soft memory limit (#836).
-	// Honors GOMEMLIMIT if set; otherwise derives from packetStore.maxMemoryMB.
+	// Apply Go runtime soft memory limit (#836, #1010).
+	// Precedence: GOMEMLIMIT env > runtime.maxMemoryMB > derived from packetStore.maxMemoryMB.
 	{
 		_, envSet := os.LookupEnv("GOMEMLIMIT")
+		runtimeMaxMB := 0
+		if cfg.Runtime != nil {
+			runtimeMaxMB = cfg.Runtime.MaxMemoryMB
+		}
 		maxMB := 0
 		if cfg.PacketStore != nil {
 			maxMB = cfg.PacketStore.MaxMemoryMB
 		}
-		limit, source := applyMemoryLimit(maxMB, envSet)
+		// runtime.maxMemoryMB (explicit) wins over packetStore-derived (implicit).
+		effectiveMB := maxMB
+		usedRuntimeCfg := false
+		if !envSet && runtimeMaxMB > 0 {
+			effectiveMB = runtimeMaxMB
+			usedRuntimeCfg = true
+		}
+		limit, source := applyMemoryLimit(effectiveMB, envSet)
 		switch source {
 		case "env":
 			log.Printf("[memlimit] using GOMEMLIMIT from environment (%s)", os.Getenv("GOMEMLIMIT"))
 		case "derived":
-			log.Printf("[memlimit] derived from packetStore.maxMemoryMB=%d → %d MiB (1.5x headroom)", maxMB, limit/(1024*1024))
+			if usedRuntimeCfg {
+				log.Printf("[memlimit] runtime.maxMemoryMB=%d → %d MiB (1.5x headroom)", runtimeMaxMB, limit/(1024*1024))
+			} else {
+				log.Printf("[memlimit] derived from packetStore.maxMemoryMB=%d → %d MiB (1.5x headroom)", maxMB, limit/(1024*1024))
+			}
 		default:
-			log.Printf("[memlimit] no soft memory limit set (GOMEMLIMIT unset, packetStore.maxMemoryMB=0); recommend setting one to avoid container OOM-kill")
+			log.Printf("[memlimit] unset → default (no soft memory limit; recommend setting GOMEMLIMIT or runtime.maxMemoryMB to ≥1.5× working set to avoid OOM-kill)")
 		}
 		warnIfMemlimitUnderprovisioned(limit)
 	}

config.example.json

@@ -295,6 +295,10 @@
     "_comment": "In-memory packet store. maxMemoryMB caps RAM usage. retentionHours: only packets younger than this are kept in memory (0 = unlimited). hotStartupHours: hours loaded synchronously at startup; background loader fills the remaining retentionHours window. 0 = disabled (loads full retentionHours synchronously, legacy behavior). Set to a positive value (e.g. 24) to reduce startup time on large DBs.",
     "_comment_gomemlimit": "On startup the server reads GOMEMLIMIT from the environment if set; otherwise it derives a Go runtime soft memory limit of maxMemoryMB * 1.5 and applies it via debug.SetMemoryLimit. This forces aggressive GC under cgroup pressure so the process self-throttles before the kernel SIGKILLs it. To override, set GOMEMLIMIT explicitly (e.g. GOMEMLIMIT=850MiB). See issue #836."
   },
+  "runtime": {
+    "maxMemoryMB": 0,
+    "_comment_runtime_maxMemoryMB": "Go soft memory limit (GOMEMLIMIT) in MiB applied via runtime/debug.SetMemoryLimit at startup. Precedence: GOMEMLIMIT env var > runtime.maxMemoryMB > packetStore.maxMemoryMB-derived (server only). 0 (default) preserves existing behavior. Set on memory-constrained deployments (2 GB Pi, 4 GB VMs) to trigger earlier GC and avoid container OOM-kill. Floor recommendation: ≥1.5× working set; setting below the working set causes a GC death-spiral. Applies to BOTH cmd/server and cmd/ingestor. See #1010."
+  },
   "resolvedPath": {
     "backfillHours": 24,
     "_comment": "How far back (hours) the async backfill scans for observations with NULL resolved_path. Default: 24. Set higher to backfill older data, lower to speed up startup."