diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index a22c3425..6361c781 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1167,18 +1167,21 @@ jobs: - name: Build corescope-decrypt (static, linux/amd64) run: | cd cmd/decrypt - CGO_ENABLED=1 GOOS=linux GOARCH=amd64 CC="zig cc -target x86_64-linux-musl" go build -trimpath -tags netgo,osusergo,sqlite_omit_load_extension -ldflags="-s -w -extldflags '-static -Wl,-s' -X main.version=${{ github.ref_name }}" -o ../../corescope-decrypt-linux-amd64 . + CGO_ENABLED=1 GOOS=linux GOARCH=amd64 CC="zig cc -target x86_64-linux-musl" go build -trimpath -tags netgo,osusergo,sqlite_omit_load_extension -ldflags="-s -w -extldflags '-static -Wl,-s -Wl,-z,stack-size=8388608' -X main.version=${{ github.ref_name }}" -o ../../corescope-decrypt-linux-amd64 . - name: Build corescope-decrypt (static, linux/arm64) run: | cd cmd/decrypt - CGO_ENABLED=1 GOOS=linux GOARCH=arm64 CC="zig cc -target aarch64-linux-musl" go build -trimpath -tags netgo,osusergo,sqlite_omit_load_extension -ldflags="-s -w -extldflags '-static -Wl,-s' -X main.version=${{ github.ref_name }}" -o ../../corescope-decrypt-linux-arm64 . + CGO_ENABLED=1 GOOS=linux GOARCH=arm64 CC="zig cc -target aarch64-linux-musl" go build -trimpath -tags netgo,osusergo,sqlite_omit_load_extension -ldflags="-s -w -extldflags '-static -Wl,-s -Wl,-z,stack-size=8388608' -X main.version=${{ github.ref_name }}" -o ../../corescope-decrypt-linux-arm64 . - name: Verify release binaries are static and runnable run: | set -e for f in corescope-decrypt-linux-amd64 corescope-decrypt-linux-arm64; do file "$f" | grep -q 'statically linked' || { echo "::error::$f is not statically linked"; exit 1; } + # musl sizes thread stacks from PT_GNU_STACK; the build's + # -Wl,-z,stack-size flag must have reached the artifact. + readelf -lW "$f" | grep -Eq 'GNU_STACK .* 0x0*800000 +RW' || { echo "::error::$f: PT_GNU_STACK is not 8 MiB, musl thread stacks are not pinned"; exit 1; } done # amd64 runs natively on the runner; arm64 is only checked structurally # here (the container image job exercises arm64 under QEMU). diff --git a/Dockerfile b/Dockerfile index 08857fcc..8d979c30 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,6 +18,17 @@ FROM --platform=$BUILDPLATFORM golang:1.27-alpine AS builder ARG APP_VERSION=unknown ARG GIT_COMMIT=unknown +# musl sizes every non-main thread's stack from the ELF PT_GNU_STACK size +# (floor 128 KiB, cap 8 MiB). cgo calls into SQLite run on the calling thread's +# stack, and a deep query (multiSelect recursing ~850 B per UNION ALL term) +# overflows 128 KiB and SIGSEGVs the process. zig's linker already writes a +# 16 MiB PT_GNU_STACK, so these binaries got 8 MiB threads before this flag +# existed; GNU ld writes 0, which is where the crash was seen (a native Alpine +# gcc link). The flag pins the size so it no longer rests on a linker default, +# and the readelf check after the builds fails the image if it is ever lost. +# Reserved address space, not resident memory. Go rejects this flag in +# #cgo LDFLAGS, so it must go in -extldflags. +ARG STACK_LDFLAG="-Wl,-z,stack-size=8388608" ARG BUILD_TIME=unknown # Provided by buildx for multi-arch builds ARG TARGETOS @@ -33,7 +44,8 @@ ENV GO_BUILD_TAGS=netgo,osusergo,sqlite_omit_load_extension \ # TARGETARCH because this stage is pinned to BUILDPLATFORM — it has to work when # someone builds on an arm64 machine too. ARG ZIG_VERSION=0.16.0 -RUN apk add --no-cache curl xz && \ +# binutils: readelf, for the thread-stack check after the builds. +RUN apk add --no-cache curl xz binutils && \ case "$(uname -m)" in \ x86_64) ZA=x86_64; ZSHA=70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d00 ;; \ aarch64) ZA=aarch64; ZSHA=ea4b09bfb22ec6f6c6ceac57ab63efb6b46e17ab08d21f69f3a48b38e1534f17 ;; \ @@ -81,7 +93,7 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/tmp/zig-cache \ GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ go build -trimpath -tags ${GO_BUILD_TAGS} \ - -ldflags "-s -w -extldflags '-static -Wl,-s' -X main.Version=${APP_VERSION} -X main.Commit=${GIT_COMMIT} -X main.BuildTime=${BUILD_TIME}" \ + -ldflags "-s -w -extldflags '-static -Wl,-s ${STACK_LDFLAG}' -X main.Version=${APP_VERSION} -X main.Commit=${GIT_COMMIT} -X main.BuildTime=${BUILD_TIME}" \ -o /corescope-server . # Build ingestor @@ -106,7 +118,7 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/tmp/zig-cache \ GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ go build -trimpath -tags ${GO_BUILD_TAGS} \ - -ldflags "-s -w -extldflags '-static -Wl,-s'" \ + -ldflags "-s -w -extldflags '-static -Wl,-s ${STACK_LDFLAG}'" \ -o /corescope-ingestor . # Build decrypt CLI @@ -123,9 +135,17 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/tmp/zig-cache \ GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ go build -trimpath -tags ${GO_BUILD_TAGS} \ - -ldflags "-s -w -extldflags '-static -Wl,-s' -X main.version=${APP_VERSION}" \ + -ldflags "-s -w -extldflags '-static -Wl,-s ${STACK_LDFLAG}' -X main.version=${APP_VERSION}" \ -o /corescope-decrypt . +# The thread-stack size must be in every binary's PT_GNU_STACK (see +# STACK_LDFLAG). The Go test for it runs on glibc in CI, where it passes with +# or without the flag, so the artifact is checked here. +RUN for f in /corescope-server /corescope-ingestor /corescope-decrypt; do \ + readelf -lW "$f" | grep -Eq 'GNU_STACK .* 0x0*800000 +RW' || \ + { echo "$f: PT_GNU_STACK is not 8 MiB, musl thread stacks are not pinned" >&2; readelf -lW "$f" | grep GNU_STACK >&2; exit 1; }; \ + done + # Runtime image FROM alpine:3.24 diff --git a/Makefile b/Makefile index cc633614..dbd9c6bd 100644 --- a/Makefile +++ b/Makefile @@ -41,6 +41,24 @@ GO_BUILD_TAGS ?= netgo,osusergo,sqlite_omit_load_extension GO_BUILD_FLAGS ?= -trimpath GO_LDFLAGS_OPTIMS ?= -s -w +# musl sizes every non-main thread's stack from the ELF PT_GNU_STACK size +# (floor 128 KiB, cap 8 MiB). cgo calls into SQLite run on the calling thread's +# stack, and a deep query (multiSelect recursing ~850 B per UNION ALL term) +# overflows 128 KiB and SIGSEGVs the process. +# - Native musl links (`make build` / `make test` on Alpine: gcc + GNU ld) +# write a zero PT_GNU_STACK and get the 128 KiB floor. The flag is the fix. +# - zig links (crossbuild, the Dockerfile, the release job) already write +# 16 MiB, so those binaries had 8 MiB threads without it. There the flag +# pins the size instead of leaving it to a linker default. +# Reserved address space, not resident memory. Go rejects this flag in +# #cgo LDFLAGS, so it must go in -extldflags. Harmless on glibc; GNU ld/lld +# only, so the host build and `make test` apply it on Linux only. +STACK_LDFLAG ?= -Wl,-z,stack-size=8388608 +ifeq ($(GOOS),linux) +HOST_EXTLDFLAGS := -extldflags '$(STACK_LDFLAG)' +HOST_TEST_LDFLAGS := -ldflags=-extldflags=$(STACK_LDFLAG) +endif + CMDS := server ingestor decrypt migrate DIST := dist @@ -77,7 +95,7 @@ build-%: @mkdir -p $(DIST) cd cmd/$* && GOOS=$(GOOS) GOARCH=$(GOARCH) $(GO) build \ -tags $(GO_BUILD_TAGS) $(GO_BUILD_FLAGS) \ - -ldflags "$(GO_LDFLAGS_OPTIMS) $(LDFLAGS_$*)" \ + -ldflags "$(GO_LDFLAGS_OPTIMS) $(HOST_EXTLDFLAGS) $(LDFLAGS_$*)" \ -o ../../$(DIST)/corescope-$* . # -- cross builds -------------------------------------------------------------- @@ -94,7 +112,7 @@ crossbuild-$(2)-$(3)-$(4): @command -v zig >/dev/null || { echo "zig not found: needed to cross-compile cgo. See https://ziglang.org/download/"; exit 1; } cd cmd/$(2) && CGO_ENABLED=1 GOOS=$(3) GOARCH=$(4) CC="zig cc -target $(1)" \ $$(GO) build -tags $$(GO_BUILD_TAGS) $$(GO_BUILD_FLAGS) \ - -ldflags '$$(GO_LDFLAGS_OPTIMS) -extldflags "-static -Wl,-s" $$(LDFLAGS_$(2))' \ + -ldflags '$$(GO_LDFLAGS_OPTIMS) -extldflags "-static -Wl,-s $$(STACK_LDFLAG)" $$(LDFLAGS_$(2))' \ -o ../../$$(DIST)/corescope-$(2)-$(3)-$(4) . endef @@ -105,7 +123,7 @@ $(foreach c,$(CMDS),\ # -- checks -------------------------------------------------------------------- test: - bash scripts/allmod.sh test ./... + bash scripts/allmod.sh test $(HOST_TEST_LDFLAGS) ./... vet: bash scripts/allmod.sh vet ./... diff --git a/cmd/server/musl_thread_stack_linux_test.go b/cmd/server/musl_thread_stack_linux_test.go new file mode 100644 index 00000000..2720259a --- /dev/null +++ b/cmd/server/musl_thread_stack_linux_test.go @@ -0,0 +1,91 @@ +package main + +import ( + "fmt" + "os" + "runtime" + "strings" + "syscall" + "testing" +) + +// Regression for musl thread stacks. musl sizes every non-main thread's stack +// from the ELF PT_GNU_STACK size (floor 128 KiB), and cgo calls into SQLite +// run on the calling thread's stack. SQLite's multiSelect recurses ~850 B per +// UNION ALL term, so the wide chunked queries below overflow 128 KiB off the +// main thread and SIGSEGV the process. +// +// Which links are exposed: a native musl link with gcc and GNU ld (`go test` +// or `make build` on Alpine) writes a zero PT_GNU_STACK and crashes here +// without -Wl,-z,stack-size=8388608. zig's linker writes 16 MiB by default, so +// the zig-linked release binaries pass with or without the flag, and so does +// any glibc link. That makes this a test for Alpine hosts (`make test` passes +// the flag); the shipped artifacts are guarded by the readelf checks in the +// Dockerfile and the release job instead. +// +// Linux only: it needs the thread id to stay off the main thread. + +// offMainThread runs fn on an OS thread that is not the process's main thread. +// The main thread has the kernel-sized stack, so it would hide the bug. A +// goroutine locked to the main thread cannot be moved off it, so it runs fn in +// a fresh goroutine instead; a fresh goroutine is never on the main thread +// while the locking goroutine still holds it. +func offMainThread(fn func()) { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + if syscall.Gettid() != os.Getpid() { + fn() + return + } + done := make(chan struct{}) + go func() { + defer close(done) + runtime.LockOSThread() + fn() + }() + <-done +} + +func TestMuslThreadStackWideUnionQueries(t *testing.T) { + srv := seedClientNames(t) + + // 249 keys nothing matches and one that resolves, in the second chunk. + keys := make([]string, 0, 250) + for i := 0; i < 249; i++ { + keys = append(keys, fmt.Sprintf("%08x", 0x30000000+i)) + } + keys = append(keys, "c0cd") + // 160 targets nothing shares a prefix with, and one whose 8-byte prefix is + // also the seeded node cnnAdvertPK's. + targets := make([]string, 0, 161) + for i := 0; i < 160; i++ { + targets = append(targets, fmt.Sprintf("%016x", 0x4000000000000000+uint64(i))+strings.Repeat("0", 48)) + } + sharedTarget := cnnAdvertPK[:16] + strings.Repeat("f", 48) + targets = append(targets, sharedTarget) + + var hits map[string]prefixHit + var shared map[string]bool + offMainThread(func() { + hits = srv.lookupPrefixes(keys) + shared = srv.sharedEightBytePrefixes(targets) + }) + + // A failed lookupPrefixes chunk drops its keys, and a failed + // sharedEightBytePrefixes chunk marks every prefix in it shared. Both + // results are checked exactly, so neither failure passes as "no match". + if len(hits) != len(keys) { + t.Fatalf("lookupPrefixes returned %d keys, want %d (a chunk failed)", len(hits), len(keys)) + } + if h := hits["c0cd"]; h.pubkey != cnnOnlyPK || h.name != "Self Named" { + t.Fatalf("c0cd = %+v, want %s / Self Named", h, cnnOnlyPK) + } + for _, k := range keys[:249] { + if h := hits[k]; h.matches != 0 { + t.Fatalf("key %s: unexpected match %+v", k, h) + } + } + if len(shared) != 1 || !shared[sharedTarget[:16]] { + t.Fatalf("sharedEightBytePrefixes = %v, want only %s (a chunk failed?)", shared, sharedTarget[:16]) + } +} diff --git a/tests/unit/test-dockerfile-buildplatform.js b/tests/unit/test-dockerfile-buildplatform.js index c396d2ec..746cf698 100644 --- a/tests/unit/test-dockerfile-buildplatform.js +++ b/tests/unit/test-dockerfile-buildplatform.js @@ -40,5 +40,24 @@ test('no stage hard-codes a platform', () => { assert.deepStrictEqual(pinned, [], 'found: ' + pinned.join(' | ')); }); +console.log('\n=== Dockerfile: musl thread stacks ==='); + +// The Go regression test runs on glibc in CI and cannot fail there, so the +// image build is what guards the flag: it must be on every link line, and the +// build must read it back from every binary. +test('every go build passes STACK_LDFLAG and the result is checked with readelf', () => { + const joined = dockerfile.replace(/\\\n/g, ' '); + const builds = joined.split('\n').filter((l) => /\bgo build\b/.test(l)); + assert.strictEqual(builds.length, 3, 'expected server, ingestor and decrypt builds'); + for (const b of builds) assert.ok(b.includes('${STACK_LDFLAG}'), 'missing STACK_LDFLAG: ' + b.trim().slice(0, 120)); + assert.ok(/^ARG STACK_LDFLAG="-Wl,-z,stack-size=8388608"$/m.test(dockerfile), 'STACK_LDFLAG must be 8 MiB'); + const check = joined.split('\n').find((l) => /readelf -lW/.test(l) && /GNU_STACK/.test(l)); + assert.ok(check, 'no readelf PT_GNU_STACK check'); + for (const bin of ['/corescope-server', '/corescope-ingestor', '/corescope-decrypt']) { + assert.ok(check.includes(bin), bin + ' is not checked'); + } + assert.ok(check.includes('0x0*800000'), 'the check must require exactly 8 MiB'); +}); + console.log(`\n${passed} passed, ${failed} failed`); if (failed) process.exit(1); diff --git a/tests/unit/test-issue-1956-release-routing.js b/tests/unit/test-issue-1956-release-routing.js index bb520ee6..f7edcf50 100644 --- a/tests/unit/test-issue-1956-release-routing.js +++ b/tests/unit/test-issue-1956-release-routing.js @@ -96,6 +96,7 @@ function runSteps(source, context, edge, mutateFails = false) { tar() { log tar "$@"; } go() { log go "$GOOS" "$GOARCH" "$CGO_ENABLED" "$CC" "$@"; } file() { log file "$@"; echo "$1: ELF 64-bit LSB executable, statically linked"; } + readelf() { log readelf "$@"; echo " GNU_STACK 0x000000 0x0000000000000000 0x0000000000000000 0x000000 $GNU_STACK_MEMSZ RW 0x0"; } # Stand-in for the jq filters the release steps use. Each branch mirrors one # filter, so a filter that changes shape without the test knowing fails here # instead of silently returning nothing. @@ -148,7 +149,8 @@ function runSteps(source, context, edge, mutateFails = false) { ] }), GH_API_SHA: edge === null ? 'a'.repeat(40) : edge, - MUTATE_FAILS: String(mutateFails) + MUTATE_FAILS: String(mutateFails), + GNU_STACK_MEMSZ: process.env.TEST_GNU_STACK_MEMSZ || '0x800000' } }); if (mutateFails && result.status !== 0) return { commands: commands(), failed: true }; @@ -313,9 +315,24 @@ assert.deepEqual(builds.map(command => command.slice(1, 5)), [ ['linux', 'arm64', '1', 'zig cc -target aarch64-linux-musl'], ]); for (const command of builds) { - assert.ok(command.includes("-ldflags=-s -w -extldflags '-static -Wl,-s' -X main.version=v9.8.7"), 'binary version must come from tag, and the artifact must stay static'); + // -Wl,-z,stack-size=8388608 pins musl thread stacks at 8 MiB, so deep SQLite + // queries do not depend on the linker's default PT_GNU_STACK size. + assert.ok(command.includes("-ldflags=-s -w -extldflags '-static -Wl,-s -Wl,-z,stack-size=8388608' -X main.version=v9.8.7"), 'binary version must come from tag, the artifact must stay static, and musl threads must get 8 MiB stacks'); assert.ok(command.includes('-tags'), 'netgo/osusergo/sqlite_omit_load_extension must survive'); } +// The flag is only worth something if it reached the artifact: the verify step +// must read PT_GNU_STACK of both binaries and fail on any other size (zig's own +// default is 16 MiB, GNU ld's is 0). +assert.deepEqual(runSteps(release, context(), null).commands.filter(command => command[0] === 'readelf').map(command => command[2]), + ['corescope-decrypt-linux-amd64', 'corescope-decrypt-linux-arm64'], 'both release binaries must have their thread-stack size checked'); +for (const memsz of ['0x000000', '0x1000000']) { + process.env.TEST_GNU_STACK_MEMSZ = memsz; + try { + assert.throws(() => runSteps(release, context(), null), /Verify release binaries/, `a ${memsz} PT_GNU_STACK must fail the release`); + } finally { + delete process.env.TEST_GNU_STACK_MEMSZ; + } +} const upload = steps(release).filter(step => step.includes('uses: softprops/action-gh-release@v2')); assert.equal(upload.length, 1, 'publish both architectures together, before the release becomes immutable'); assert.equal(value(upload[0], 'fail_on_unmatched_files', 10), 'true', 'missing assets must prevent publication');