package main import ( "log" "os" "runtime/debug" "strconv" "strings" ) // cgroupUnlimitedThreshold is the sentinel above which a cgroup memory value // means "no limit". cgroup v1 encodes unlimited as math.MaxInt64 (page-aligned // near 1<<63); 1<<62 is a safe upper bound that excludes all real limits while // staying well below the unlimited sentinel. const cgroupUnlimitedThreshold = int64(1 << 62) // applyMemoryLimit configures Go's soft memory limit (GOMEMLIMIT). // // Behavior: // - If envSet is true (GOMEMLIMIT env var present), the runtime has already // parsed it; we leave it alone and report source="env" with limit=0. // - Otherwise, if maxMemoryMB > 0, we derive a limit of maxMemoryMB * 1.5 MiB // and set it via debug.SetMemoryLimit. This forces aggressive GC under // cgroup pressure so the process self-throttles before SIGKILL. See #836. // - Otherwise, no limit is applied; source="none". // // Returns the limit (in bytes) we actually set, or 0 if we did not set one, // plus a short source identifier ("env" | "derived" | "none") for logging. func applyMemoryLimit(maxMemoryMB int, envSet bool) (int64, string) { if envSet { return 0, "env" } if maxMemoryMB <= 0 { return 0, "none" } // 1.5x headroom over the steady-state packet store budget covers // transient peaks (cold-load row-scan / decode pipeline, Go's NextGC // trigger at ~2x live heap). See issue #836 heap profile. limit := int64(maxMemoryMB) * 1024 * 1024 * 3 / 2 debug.SetMemoryLimit(limit) return limit, "derived" } // readCgroupMemoryMBFn is the package-level hook used by // warnIfMemlimitUnderprovisioned. Tests override it to inject deterministic // cgroup values without needing a Linux kernel with cgroup mounts. var readCgroupMemoryMBFn = readCgroupMemoryMB // readCgroupMemoryMB returns the container's memory limit from cgroup, in MiB. // Returns 0 when unavailable (non-Linux, unlimited, or read error). func readCgroupMemoryMB() int64 { // cgroup v2: single file, value in bytes or literal "max" if b, err := os.ReadFile("/sys/fs/cgroup/memory.max"); err == nil { s := strings.TrimSpace(string(b)) if s != "max" { if v, err := strconv.ParseInt(s, 10, 64); err == nil && v > 0 { return v / (1024 * 1024) } } } // cgroup v1: values near math.MaxInt64 represent "unlimited" if b, err := os.ReadFile("/sys/fs/cgroup/memory/memory.limit_in_bytes"); err == nil { if v, err := strconv.ParseInt(strings.TrimSpace(string(b)), 10, 64); err == nil { if v > 0 && v < cgroupUnlimitedThreshold { return v / (1024 * 1024) } } } return 0 } // memlimitUnderprovisioned reports whether effectiveMB is less than half of // cgroupMB. Extracted for unit testing the comparison boundary. func memlimitUnderprovisioned(effectiveMB, cgroupMB int64) bool { return effectiveMB > 0 && cgroupMB > 0 && effectiveMB*2 < cgroupMB } // warnIfMemlimitUnderprovisioned logs a warning when GOMEMLIMIT is below 50% // of the container cgroup memory limit, which causes the Go GC to thrash. // In one reported incident (#1264) 82% of CPU was GC with a 1536 MiB limit // on a 7.7 GB container — all endpoints 3-100x slower until maxMemoryMB was // bumped and the process restarted. // // limitBytes is the value returned by applyMemoryLimit: // - source="derived": the limit we set ourselves (> 0) // - source="env": 0 — we did not touch the runtime; read it back below // - source="none": 0 — no limit set at all; runtime default is math.MaxInt64, // which the >= cgroupUnlimitedThreshold guard below catches and skips func warnIfMemlimitUnderprovisioned(limitBytes int64) { cgroupMB := readCgroupMemoryMBFn() if cgroupMB <= 0 { return } effective := limitBytes if effective <= 0 { // Either GOMEMLIMIT was set via env (source="env") or no limit was // configured (source="none"). Read the runtime's current value: // - env case: returns whatever the operator set // - none case: returns math.MaxInt64, caught by the guard below // debug.SetMemoryLimit(-1) leaves the limit unchanged and returns it. effective = debug.SetMemoryLimit(-1) } if effective <= 0 || effective >= cgroupUnlimitedThreshold { return } effectiveMB := effective / (1024 * 1024) if memlimitUnderprovisioned(effectiveMB, cgroupMB) { log.Printf("[memlimit] WARN: GOMEMLIMIT=%d MiB is <50%% of container limit %d MiB — "+ "GC may thrash under load; consider bumping packetStore.maxMemoryMB "+ "(suggested: ~%d MiB, roughly 2/3 of container limit)", effectiveMB, cgroupMB, cgroupMB*2/3) } }