From 0af70cf2c10d022f8b07eae711cf91e364367bc3 Mon Sep 17 00:00:00 2001 From: Zach Date: Wed, 18 Nov 2026 11:35:41 -0600 Subject: [PATCH] fix(build): pin musl thread stacks at 8 MiB and check the artifacts (#2410) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Pins musl thread stacks at 8 MiB on every link, and fixes a crash in **native Alpine builds** (`make build`, `make test`, or `go test` on an Alpine host). **The released images are not affected.** An earlier version of this description said public endpoints could crash the shipped image. That was wrong, and review caught it: see "Who is affected". ## Root cause - musl sizes every non-main thread's stack from the ELF `PT_GNU_STACK` size, with a floor of 128 KiB and a cap of 8 MiB (glibc defaults to 8 MiB). - cgo calls run on the calling thread's stack, and SQLite's `multiSelect` recurses about 850 bytes per `UNION ALL` term. - `lookupPrefixes` (up to 400 terms) and `sharedEightBytePrefixes` (up to 450 terms) overflow a 128 KiB stack and SIGSEGV the whole process. ## Who is affected | Link | `PT_GNU_STACK` without the flag | Thread stack | Wide-UNION tests | |---|---|---|---| | Alpine gcc + GNU ld (`make build` / `make test` / `go test` on Alpine) | `0` | 128 KiB | **SIGSEGV** | | zig 0.16.0, x86_64-linux-musl (Dockerfile, `crossbuild`, release job) | `0x1000000` (16 MiB) | 8 MiB | pass 3/3 | | zig 0.16.0, aarch64-linux-musl | `0x1000000` (16 MiB) | 8 MiB | pass 3/3, run on an arm64 node | So for the released images the flag pins a value that zig's default already provides. It stops being a property of a linker default, and a build check now fails if it is lost. For native Alpine links it is the fix: with the flag those tests pass. Control: a zig arm64 link with `-Wl,-z,stack-size=131072` gets 128 KiB threads and the same tests SIGSEGV in cgo on the arm64 node. ## Fix - Every musl link passes `-Wl,-z,stack-size=8388608`: the `Dockerfile`, the `Makefile` (`build` and `test` on Linux, and `crossbuild`) and the release job in `deploy.yml`. - This reserves address space only. Memory is committed as pages are touched. - **The artifacts are checked.** The Dockerfile's builder stage and the release job's "Verify release binaries" step read `PT_GNU_STACK` with `readelf` and fail unless it is exactly 8 MiB. Verified: the builder stage passes for `linux/amd64` and `linux/arm64`, and fails with `--build-arg STACK_LDFLAG=` (`/corescope-server: PT_GNU_STACK is not 8 MiB`, memsz `0x1000000`). ## Testing - `TestMuslThreadStackWideUnionQueries` (`cmd/server/musl_thread_stack_linux_test.go`, Linux only) runs both queries off the main thread and asserts exact results, so a failed chunk cannot pass as "nothing matched". Under `golang:1.27-alpine` with gcc: SIGSEGV without the flag, pass with it. On CI's glibc runners it passes either way, which is why the artifact checks above exist. - `tests/unit/test-issue-1956-release-routing.js`: the release verify step must check both binaries, and a `0x000000` or `0x1000000` header must fail the release. - `tests/unit/test-dockerfile-buildplatform.js`: all three `go build` lines carry `STACK_LDFLAG`, and the `readelf` check covers all three binaries. - `sh test-all.sh` passes. `go vet` passes for `cmd/server` on Linux with gofmt clean. - Not verified: a Windows `go vet`. The fix there is the `_linux_test.go` file name; no Windows machine was available to run it. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --- ## Merge order These four PRs were split out of #2396 and are **independent — they can merge in any order.** Verified: every pair is conflict-free, and all four merged together pass `go vet`, `go test ./...` (all 16 modules) and `sh test-all.sh`. (That combined run predates the review fixes on #2409 and #2410.) Suggested order, none of it urgent: 1. **#2408** — never lose rows during startup / a failed poll. 2. **#2407** — explicit `observers.idx` primary key. 3. **#2409** — content-hash migration moved into the ingestor. 4. **#2410** — pin musl thread stacks at 8 MiB. Released images are not affected; it fixes native Alpine builds. After these land, #2396 (Postgres) will be rebased onto master. --------- Co-authored-by: Claude Opus 5.5 Co-authored-by: efiten --- .github/workflows/deploy.yml | 7 +- Dockerfile | 28 +++++- Makefile | 24 ++++- cmd/server/musl_thread_stack_linux_test.go | 91 +++++++++++++++++++ tests/unit/test-dockerfile-buildplatform.js | 19 ++++ tests/unit/test-issue-1956-release-routing.js | 21 ++++- 6 files changed, 179 insertions(+), 11 deletions(-) create mode 100644 cmd/server/musl_thread_stack_linux_test.go diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index a22c3425..6361c781 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1167,18 +1167,21 @@ jobs: - name: Build corescope-decrypt (static, linux/amd64) run: | cd cmd/decrypt - CGO_ENABLED=1 GOOS=linux GOARCH=amd64 CC="zig cc -target x86_64-linux-musl" go build -trimpath -tags netgo,osusergo,sqlite_omit_load_extension -ldflags="-s -w -extldflags '-static -Wl,-s' -X main.version=${{ github.ref_name }}" -o ../../corescope-decrypt-linux-amd64 . + CGO_ENABLED=1 GOOS=linux GOARCH=amd64 CC="zig cc -target x86_64-linux-musl" go build -trimpath -tags netgo,osusergo,sqlite_omit_load_extension -ldflags="-s -w -extldflags '-static -Wl,-s -Wl,-z,stack-size=8388608' -X main.version=${{ github.ref_name }}" -o ../../corescope-decrypt-linux-amd64 . - name: Build corescope-decrypt (static, linux/arm64) run: | cd cmd/decrypt - CGO_ENABLED=1 GOOS=linux GOARCH=arm64 CC="zig cc -target aarch64-linux-musl" go build -trimpath -tags netgo,osusergo,sqlite_omit_load_extension -ldflags="-s -w -extldflags '-static -Wl,-s' -X main.version=${{ github.ref_name }}" -o ../../corescope-decrypt-linux-arm64 . + CGO_ENABLED=1 GOOS=linux GOARCH=arm64 CC="zig cc -target aarch64-linux-musl" go build -trimpath -tags netgo,osusergo,sqlite_omit_load_extension -ldflags="-s -w -extldflags '-static -Wl,-s -Wl,-z,stack-size=8388608' -X main.version=${{ github.ref_name }}" -o ../../corescope-decrypt-linux-arm64 . - name: Verify release binaries are static and runnable run: | set -e for f in corescope-decrypt-linux-amd64 corescope-decrypt-linux-arm64; do file "$f" | grep -q 'statically linked' || { echo "::error::$f is not statically linked"; exit 1; } + # musl sizes thread stacks from PT_GNU_STACK; the build's + # -Wl,-z,stack-size flag must have reached the artifact. + readelf -lW "$f" | grep -Eq 'GNU_STACK .* 0x0*800000 +RW' || { echo "::error::$f: PT_GNU_STACK is not 8 MiB, musl thread stacks are not pinned"; exit 1; } done # amd64 runs natively on the runner; arm64 is only checked structurally # here (the container image job exercises arm64 under QEMU). diff --git a/Dockerfile b/Dockerfile index 08857fcc..8d979c30 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,6 +18,17 @@ FROM --platform=$BUILDPLATFORM golang:1.27-alpine AS builder ARG APP_VERSION=unknown ARG GIT_COMMIT=unknown +# musl sizes every non-main thread's stack from the ELF PT_GNU_STACK size +# (floor 128 KiB, cap 8 MiB). cgo calls into SQLite run on the calling thread's +# stack, and a deep query (multiSelect recursing ~850 B per UNION ALL term) +# overflows 128 KiB and SIGSEGVs the process. zig's linker already writes a +# 16 MiB PT_GNU_STACK, so these binaries got 8 MiB threads before this flag +# existed; GNU ld writes 0, which is where the crash was seen (a native Alpine +# gcc link). The flag pins the size so it no longer rests on a linker default, +# and the readelf check after the builds fails the image if it is ever lost. +# Reserved address space, not resident memory. Go rejects this flag in +# #cgo LDFLAGS, so it must go in -extldflags. +ARG STACK_LDFLAG="-Wl,-z,stack-size=8388608" ARG BUILD_TIME=unknown # Provided by buildx for multi-arch builds ARG TARGETOS @@ -33,7 +44,8 @@ ENV GO_BUILD_TAGS=netgo,osusergo,sqlite_omit_load_extension \ # TARGETARCH because this stage is pinned to BUILDPLATFORM — it has to work when # someone builds on an arm64 machine too. ARG ZIG_VERSION=0.16.0 -RUN apk add --no-cache curl xz && \ +# binutils: readelf, for the thread-stack check after the builds. +RUN apk add --no-cache curl xz binutils && \ case "$(uname -m)" in \ x86_64) ZA=x86_64; ZSHA=70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d00 ;; \ aarch64) ZA=aarch64; ZSHA=ea4b09bfb22ec6f6c6ceac57ab63efb6b46e17ab08d21f69f3a48b38e1534f17 ;; \ @@ -81,7 +93,7 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/tmp/zig-cache \ GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ go build -trimpath -tags ${GO_BUILD_TAGS} \ - -ldflags "-s -w -extldflags '-static -Wl,-s' -X main.Version=${APP_VERSION} -X main.Commit=${GIT_COMMIT} -X main.BuildTime=${BUILD_TIME}" \ + -ldflags "-s -w -extldflags '-static -Wl,-s ${STACK_LDFLAG}' -X main.Version=${APP_VERSION} -X main.Commit=${GIT_COMMIT} -X main.BuildTime=${BUILD_TIME}" \ -o /corescope-server . # Build ingestor @@ -106,7 +118,7 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/tmp/zig-cache \ GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ go build -trimpath -tags ${GO_BUILD_TAGS} \ - -ldflags "-s -w -extldflags '-static -Wl,-s'" \ + -ldflags "-s -w -extldflags '-static -Wl,-s ${STACK_LDFLAG}'" \ -o /corescope-ingestor . # Build decrypt CLI @@ -123,9 +135,17 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/tmp/zig-cache \ GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ go build -trimpath -tags ${GO_BUILD_TAGS} \ - -ldflags "-s -w -extldflags '-static -Wl,-s' -X main.version=${APP_VERSION}" \ + -ldflags "-s -w -extldflags '-static -Wl,-s ${STACK_LDFLAG}' -X main.version=${APP_VERSION}" \ -o /corescope-decrypt . +# The thread-stack size must be in every binary's PT_GNU_STACK (see +# STACK_LDFLAG). The Go test for it runs on glibc in CI, where it passes with +# or without the flag, so the artifact is checked here. +RUN for f in /corescope-server /corescope-ingestor /corescope-decrypt; do \ + readelf -lW "$f" | grep -Eq 'GNU_STACK .* 0x0*800000 +RW' || \ + { echo "$f: PT_GNU_STACK is not 8 MiB, musl thread stacks are not pinned" >&2; readelf -lW "$f" | grep GNU_STACK >&2; exit 1; }; \ + done + # Runtime image FROM alpine:3.24 diff --git a/Makefile b/Makefile index cc633614..dbd9c6bd 100644 --- a/Makefile +++ b/Makefile @@ -41,6 +41,24 @@ GO_BUILD_TAGS ?= netgo,osusergo,sqlite_omit_load_extension GO_BUILD_FLAGS ?= -trimpath GO_LDFLAGS_OPTIMS ?= -s -w +# musl sizes every non-main thread's stack from the ELF PT_GNU_STACK size +# (floor 128 KiB, cap 8 MiB). cgo calls into SQLite run on the calling thread's +# stack, and a deep query (multiSelect recursing ~850 B per UNION ALL term) +# overflows 128 KiB and SIGSEGVs the process. +# - Native musl links (`make build` / `make test` on Alpine: gcc + GNU ld) +# write a zero PT_GNU_STACK and get the 128 KiB floor. The flag is the fix. +# - zig links (crossbuild, the Dockerfile, the release job) already write +# 16 MiB, so those binaries had 8 MiB threads without it. There the flag +# pins the size instead of leaving it to a linker default. +# Reserved address space, not resident memory. Go rejects this flag in +# #cgo LDFLAGS, so it must go in -extldflags. Harmless on glibc; GNU ld/lld +# only, so the host build and `make test` apply it on Linux only. +STACK_LDFLAG ?= -Wl,-z,stack-size=8388608 +ifeq ($(GOOS),linux) +HOST_EXTLDFLAGS := -extldflags '$(STACK_LDFLAG)' +HOST_TEST_LDFLAGS := -ldflags=-extldflags=$(STACK_LDFLAG) +endif + CMDS := server ingestor decrypt migrate DIST := dist @@ -77,7 +95,7 @@ build-%: @mkdir -p $(DIST) cd cmd/$* && GOOS=$(GOOS) GOARCH=$(GOARCH) $(GO) build \ -tags $(GO_BUILD_TAGS) $(GO_BUILD_FLAGS) \ - -ldflags "$(GO_LDFLAGS_OPTIMS) $(LDFLAGS_$*)" \ + -ldflags "$(GO_LDFLAGS_OPTIMS) $(HOST_EXTLDFLAGS) $(LDFLAGS_$*)" \ -o ../../$(DIST)/corescope-$* . # -- cross builds -------------------------------------------------------------- @@ -94,7 +112,7 @@ crossbuild-$(2)-$(3)-$(4): @command -v zig >/dev/null || { echo "zig not found: needed to cross-compile cgo. See https://ziglang.org/download/"; exit 1; } cd cmd/$(2) && CGO_ENABLED=1 GOOS=$(3) GOARCH=$(4) CC="zig cc -target $(1)" \ $$(GO) build -tags $$(GO_BUILD_TAGS) $$(GO_BUILD_FLAGS) \ - -ldflags '$$(GO_LDFLAGS_OPTIMS) -extldflags "-static -Wl,-s" $$(LDFLAGS_$(2))' \ + -ldflags '$$(GO_LDFLAGS_OPTIMS) -extldflags "-static -Wl,-s $$(STACK_LDFLAG)" $$(LDFLAGS_$(2))' \ -o ../../$$(DIST)/corescope-$(2)-$(3)-$(4) . endef @@ -105,7 +123,7 @@ $(foreach c,$(CMDS),\ # -- checks -------------------------------------------------------------------- test: - bash scripts/allmod.sh test ./... + bash scripts/allmod.sh test $(HOST_TEST_LDFLAGS) ./... vet: bash scripts/allmod.sh vet ./... diff --git a/cmd/server/musl_thread_stack_linux_test.go b/cmd/server/musl_thread_stack_linux_test.go new file mode 100644 index 00000000..2720259a --- /dev/null +++ b/cmd/server/musl_thread_stack_linux_test.go @@ -0,0 +1,91 @@ +package main + +import ( + "fmt" + "os" + "runtime" + "strings" + "syscall" + "testing" +) + +// Regression for musl thread stacks. musl sizes every non-main thread's stack +// from the ELF PT_GNU_STACK size (floor 128 KiB), and cgo calls into SQLite +// run on the calling thread's stack. SQLite's multiSelect recurses ~850 B per +// UNION ALL term, so the wide chunked queries below overflow 128 KiB off the +// main thread and SIGSEGV the process. +// +// Which links are exposed: a native musl link with gcc and GNU ld (`go test` +// or `make build` on Alpine) writes a zero PT_GNU_STACK and crashes here +// without -Wl,-z,stack-size=8388608. zig's linker writes 16 MiB by default, so +// the zig-linked release binaries pass with or without the flag, and so does +// any glibc link. That makes this a test for Alpine hosts (`make test` passes +// the flag); the shipped artifacts are guarded by the readelf checks in the +// Dockerfile and the release job instead. +// +// Linux only: it needs the thread id to stay off the main thread. + +// offMainThread runs fn on an OS thread that is not the process's main thread. +// The main thread has the kernel-sized stack, so it would hide the bug. A +// goroutine locked to the main thread cannot be moved off it, so it runs fn in +// a fresh goroutine instead; a fresh goroutine is never on the main thread +// while the locking goroutine still holds it. +func offMainThread(fn func()) { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + if syscall.Gettid() != os.Getpid() { + fn() + return + } + done := make(chan struct{}) + go func() { + defer close(done) + runtime.LockOSThread() + fn() + }() + <-done +} + +func TestMuslThreadStackWideUnionQueries(t *testing.T) { + srv := seedClientNames(t) + + // 249 keys nothing matches and one that resolves, in the second chunk. + keys := make([]string, 0, 250) + for i := 0; i < 249; i++ { + keys = append(keys, fmt.Sprintf("%08x", 0x30000000+i)) + } + keys = append(keys, "c0cd") + // 160 targets nothing shares a prefix with, and one whose 8-byte prefix is + // also the seeded node cnnAdvertPK's. + targets := make([]string, 0, 161) + for i := 0; i < 160; i++ { + targets = append(targets, fmt.Sprintf("%016x", 0x4000000000000000+uint64(i))+strings.Repeat("0", 48)) + } + sharedTarget := cnnAdvertPK[:16] + strings.Repeat("f", 48) + targets = append(targets, sharedTarget) + + var hits map[string]prefixHit + var shared map[string]bool + offMainThread(func() { + hits = srv.lookupPrefixes(keys) + shared = srv.sharedEightBytePrefixes(targets) + }) + + // A failed lookupPrefixes chunk drops its keys, and a failed + // sharedEightBytePrefixes chunk marks every prefix in it shared. Both + // results are checked exactly, so neither failure passes as "no match". + if len(hits) != len(keys) { + t.Fatalf("lookupPrefixes returned %d keys, want %d (a chunk failed)", len(hits), len(keys)) + } + if h := hits["c0cd"]; h.pubkey != cnnOnlyPK || h.name != "Self Named" { + t.Fatalf("c0cd = %+v, want %s / Self Named", h, cnnOnlyPK) + } + for _, k := range keys[:249] { + if h := hits[k]; h.matches != 0 { + t.Fatalf("key %s: unexpected match %+v", k, h) + } + } + if len(shared) != 1 || !shared[sharedTarget[:16]] { + t.Fatalf("sharedEightBytePrefixes = %v, want only %s (a chunk failed?)", shared, sharedTarget[:16]) + } +} diff --git a/tests/unit/test-dockerfile-buildplatform.js b/tests/unit/test-dockerfile-buildplatform.js index c396d2ec..746cf698 100644 --- a/tests/unit/test-dockerfile-buildplatform.js +++ b/tests/unit/test-dockerfile-buildplatform.js @@ -40,5 +40,24 @@ test('no stage hard-codes a platform', () => { assert.deepStrictEqual(pinned, [], 'found: ' + pinned.join(' | ')); }); +console.log('\n=== Dockerfile: musl thread stacks ==='); + +// The Go regression test runs on glibc in CI and cannot fail there, so the +// image build is what guards the flag: it must be on every link line, and the +// build must read it back from every binary. +test('every go build passes STACK_LDFLAG and the result is checked with readelf', () => { + const joined = dockerfile.replace(/\\\n/g, ' '); + const builds = joined.split('\n').filter((l) => /\bgo build\b/.test(l)); + assert.strictEqual(builds.length, 3, 'expected server, ingestor and decrypt builds'); + for (const b of builds) assert.ok(b.includes('${STACK_LDFLAG}'), 'missing STACK_LDFLAG: ' + b.trim().slice(0, 120)); + assert.ok(/^ARG STACK_LDFLAG="-Wl,-z,stack-size=8388608"$/m.test(dockerfile), 'STACK_LDFLAG must be 8 MiB'); + const check = joined.split('\n').find((l) => /readelf -lW/.test(l) && /GNU_STACK/.test(l)); + assert.ok(check, 'no readelf PT_GNU_STACK check'); + for (const bin of ['/corescope-server', '/corescope-ingestor', '/corescope-decrypt']) { + assert.ok(check.includes(bin), bin + ' is not checked'); + } + assert.ok(check.includes('0x0*800000'), 'the check must require exactly 8 MiB'); +}); + console.log(`\n${passed} passed, ${failed} failed`); if (failed) process.exit(1); diff --git a/tests/unit/test-issue-1956-release-routing.js b/tests/unit/test-issue-1956-release-routing.js index bb520ee6..f7edcf50 100644 --- a/tests/unit/test-issue-1956-release-routing.js +++ b/tests/unit/test-issue-1956-release-routing.js @@ -96,6 +96,7 @@ function runSteps(source, context, edge, mutateFails = false) { tar() { log tar "$@"; } go() { log go "$GOOS" "$GOARCH" "$CGO_ENABLED" "$CC" "$@"; } file() { log file "$@"; echo "$1: ELF 64-bit LSB executable, statically linked"; } + readelf() { log readelf "$@"; echo " GNU_STACK 0x000000 0x0000000000000000 0x0000000000000000 0x000000 $GNU_STACK_MEMSZ RW 0x0"; } # Stand-in for the jq filters the release steps use. Each branch mirrors one # filter, so a filter that changes shape without the test knowing fails here # instead of silently returning nothing. @@ -148,7 +149,8 @@ function runSteps(source, context, edge, mutateFails = false) { ] }), GH_API_SHA: edge === null ? 'a'.repeat(40) : edge, - MUTATE_FAILS: String(mutateFails) + MUTATE_FAILS: String(mutateFails), + GNU_STACK_MEMSZ: process.env.TEST_GNU_STACK_MEMSZ || '0x800000' } }); if (mutateFails && result.status !== 0) return { commands: commands(), failed: true }; @@ -313,9 +315,24 @@ assert.deepEqual(builds.map(command => command.slice(1, 5)), [ ['linux', 'arm64', '1', 'zig cc -target aarch64-linux-musl'], ]); for (const command of builds) { - assert.ok(command.includes("-ldflags=-s -w -extldflags '-static -Wl,-s' -X main.version=v9.8.7"), 'binary version must come from tag, and the artifact must stay static'); + // -Wl,-z,stack-size=8388608 pins musl thread stacks at 8 MiB, so deep SQLite + // queries do not depend on the linker's default PT_GNU_STACK size. + assert.ok(command.includes("-ldflags=-s -w -extldflags '-static -Wl,-s -Wl,-z,stack-size=8388608' -X main.version=v9.8.7"), 'binary version must come from tag, the artifact must stay static, and musl threads must get 8 MiB stacks'); assert.ok(command.includes('-tags'), 'netgo/osusergo/sqlite_omit_load_extension must survive'); } +// The flag is only worth something if it reached the artifact: the verify step +// must read PT_GNU_STACK of both binaries and fail on any other size (zig's own +// default is 16 MiB, GNU ld's is 0). +assert.deepEqual(runSteps(release, context(), null).commands.filter(command => command[0] === 'readelf').map(command => command[2]), + ['corescope-decrypt-linux-amd64', 'corescope-decrypt-linux-arm64'], 'both release binaries must have their thread-stack size checked'); +for (const memsz of ['0x000000', '0x1000000']) { + process.env.TEST_GNU_STACK_MEMSZ = memsz; + try { + assert.throws(() => runSteps(release, context(), null), /Verify release binaries/, `a ${memsz} PT_GNU_STACK must fail the release`); + } finally { + delete process.env.TEST_GNU_STACK_MEMSZ; + } +} const upload = steps(release).filter(step => step.includes('uses: softprops/action-gh-release@v2')); assert.equal(upload.length, 1, 'publish both architectures together, before the release becomes immutable'); assert.equal(value(upload[0], 'fail_on_unmatched_files', 10), 'true', 'missing assets must prevent publication');