diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 8b42e2d3..737fee93 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -124,7 +124,11 @@ jobs: uses: actions/setup-go@v6 with: go-version: '1.27' - cache-dependency-path: cmd/ingestor/go.sum + # All modules: a -race build of a cgo package is the slowest thing in + # this pipeline, and setup-go caches ~/.cache/go-build as well as the + # module cache, so a key that misses on an unrelated module hurts most + # here. Same reasoning as go-test below. + cache-dependency-path: "**/go.sum" - name: go test -race run: | @@ -153,9 +157,10 @@ jobs: uses: actions/setup-go@v6 with: go-version: '1.27' - cache-dependency-path: | - cmd/server/go.sum - cmd/ingestor/go.sum + # All modules, not just two: the cgo SQLite build is expensive to + # redo, and setup-go caches ~/.cache/go-build as well as the module + # cache, so a key that misses on an unrelated module hurts. + cache-dependency-path: "**/go.sum" - name: Enforce gofmt + go vet (issue #1859) run: | @@ -202,10 +207,22 @@ jobs: go test ./... echo "--- Channel library tests passed ---" cd ../../cmd/decrypt - CGO_ENABLED=0 go build -ldflags="-s -w" -o corescope-decrypt . + go build -ldflags="-s -w" -o corescope-decrypt . go test ./... echo "--- Decrypt CLI tests passed ---" + - name: Test migrate CLI + dbschema + run: | + set -e -o pipefail + # Neither module had CI test execution before the mattn/go-sqlite3 + # migration; both open the database, and dbschema owns the migrations. + cd internal/dbschema + go test ./... + cd ../../cmd/migrate + go build -o /dev/null . + go test ./... + echo "--- migrate + dbschema tests passed ---" + - name: Verify Dockerfile COPY invariants (issue #1316) run: bash scripts/check-dockerfile-internal-pkgs.sh @@ -329,7 +346,7 @@ jobs: uses: actions/setup-go@v6 with: go-version: '1.27' - cache-dependency-path: cmd/server/go.sum + cache-dependency-path: "**/go.sum" - name: Build Go server run: | @@ -682,14 +699,58 @@ jobs: docker compose -f "$STAGING_COMPOSE_FILE" -p corescope-staging build "$STAGING_SERVICE" echo "Built Go staging image ✅" + # Needed by the PR two-arch gate below as well as the GHCR push, so the + # condition covers both rather than push/tag only. - name: Set up Docker Buildx - if: ${{ github.event_name == 'push' || startsWith(github.ref, 'refs/tags/v') }} + if: ${{ github.event_name == 'pull_request' || github.event_name == 'push' || startsWith(github.ref, 'refs/tags/v') }} uses: docker/setup-buildx-action@v3 - - name: Set up QEMU (arm64 runtime stage) - if: ${{ github.event_name == 'push' || startsWith(github.ref, 'refs/tags/v') }} + - name: Set up QEMU (arm64 runtime stage + arm64 smoke) + if: ${{ github.event_name == 'pull_request' || github.event_name == 'push' || startsWith(github.ref, 'refs/tags/v') }} uses: docker/setup-qemu-action@v3 + # Since the SQLite driver became cgo, the container build depends on a zig + # cross-toolchain and static musl linking. The GHCR step below is + # push/tag-only, so without this a PR never exercises either, and the + # first signal would arrive on master. Builds both architectures and + # actually runs the arm64 binaries under QEMU. + - name: Two-arch build (no publish) + if: ${{ github.event_name == 'pull_request' }} + uses: docker/build-push-action@v6 + with: + context: . + push: false + platforms: linux/amd64,linux/arm64 + tags: corescope:pr-${{ github.event.pull_request.number }} + build-args: | + APP_VERSION=${{ steps.meta.outputs.app_version }} + GIT_COMMIT=${{ steps.meta.outputs.git_commit }} + BUILD_TIME=${{ steps.meta.outputs.build_time }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Smoke the arm64 binaries under QEMU + if: ${{ github.event_name == 'pull_request' }} + run: | + set -e + # Export the arm64 binaries so the runner's own `file` can confirm + # they are static, then execute one under QEMU. Cheap: the build above + # already populated the layer cache. + docker buildx build . --platform linux/arm64 \ + --build-arg APP_VERSION=smoke --cache-from type=gha \ + --output type=local,dest=./arm64-image + for b in corescope-server corescope-ingestor corescope-decrypt; do + file "./arm64-image/app/$b" | tee /dev/stderr | grep -q 'statically linked' \ + || { echo "::error::$b is not statically linked"; exit 1; } + file "./arm64-image/app/$b" | grep -q 'ARM aarch64' \ + || { echo "::error::$b is not an arm64 binary"; exit 1; } + done + docker run --rm --platform linux/arm64 \ + -v "$PWD/arm64-image/app:/bin-under-test:ro" alpine:3.20 \ + /bin-under-test/corescope-decrypt --version + rm -rf ./arm64-image + echo "arm64 static binaries run under QEMU ✅" + - name: Log in to GHCR if: ${{ github.event_name == 'push' || startsWith(github.ref, 'refs/tags/v') }} uses: docker/login-action@v3 @@ -745,16 +806,37 @@ jobs: uses: actions/setup-go@v6 with: go-version: '1.27' + cache-dependency-path: | + cmd/decrypt/go.sum + internal/channel/go.sum + + # The SQLite driver is cgo now (github.com/mattn/go-sqlite3), so the Go + # toolchain alone cannot cross-compile these. zig cc is the C compiler + # that can target both architectures; musl makes the result static. + - name: Set up Zig + uses: mlugg/setup-zig@v2 + with: + version: 0.16.0 - name: Build corescope-decrypt (static, linux/amd64) run: | cd cmd/decrypt - CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w -X main.version=${{ github.ref_name }}" -o ../../corescope-decrypt-linux-amd64 . + CGO_ENABLED=1 GOOS=linux GOARCH=amd64 CC="zig cc -target x86_64-linux-musl" go build -trimpath -tags netgo,osusergo,sqlite_omit_load_extension -ldflags="-s -w -extldflags '-static -Wl,-s' -X main.version=${{ github.ref_name }}" -o ../../corescope-decrypt-linux-amd64 . - name: Build corescope-decrypt (static, linux/arm64) run: | cd cmd/decrypt - CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -ldflags="-s -w -X main.version=${{ github.ref_name }}" -o ../../corescope-decrypt-linux-arm64 . + CGO_ENABLED=1 GOOS=linux GOARCH=arm64 CC="zig cc -target aarch64-linux-musl" go build -trimpath -tags netgo,osusergo,sqlite_omit_load_extension -ldflags="-s -w -extldflags '-static -Wl,-s' -X main.version=${{ github.ref_name }}" -o ../../corescope-decrypt-linux-arm64 . + + - name: Verify release binaries are static and runnable + run: | + set -e + for f in corescope-decrypt-linux-amd64 corescope-decrypt-linux-arm64; do + file "$f" | grep -q 'statically linked' || { echo "::error::$f is not statically linked"; exit 1; } + done + # amd64 runs natively on the runner; arm64 is only checked structurally + # here (the container image job exercises arm64 under QEMU). + ./corescope-decrypt-linux-amd64 --version - name: Upload release assets # Standard releases upload both assets to a draft before publishing. diff --git a/.gitignore b/.gitignore index 548c25d6..33def577 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,10 @@ config-lincomatic.json theme.json firmware/ coverage/ +dist/ +# internal/prunequeue writes these next to the database; the server tests leave +# them in cmd/server/. Runtime queue state, never repo content. +prune-requests/ public-instrumented/ .nyc_output/ .setup-state diff --git a/.squad/decisions/decisions.md b/.squad/decisions/decisions.md index d5aef085..37c8bb32 100644 --- a/.squad/decisions/decisions.md +++ b/.squad/decisions/decisions.md @@ -211,7 +211,9 @@ Reconciled `manage.sh` and `docker-compose.yml` Docker volume names: Standalone Go MQTT ingestor service. Separate process from Node.js web server that handles MQTT packet ingestion + writes to shared SQLite DB. **Architecture:** -- Single binary, no CGO (uses `modernc.org/sqlite` pure Go) +- Single statically-linked binary. Was CGO-free via `modernc.org/sqlite`; now cgo, + using `github.com/mattn/go-sqlite3` for read performance, cross-compiled with + `zig cc` against musl so the artifact stays a single self-contained file - Reads same `config.json` (mqttSources array) - Shares SQLite DB with Node.js (WAL mode for concurrent access) - Format 1 (raw packet) MQTT only — companion bridge stays in Node.js diff --git a/AGENTS.md b/AGENTS.md index 6fc1db4b..fb25c6d5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -90,7 +90,9 @@ Every change must consider performance impact BEFORE implementation. This codeba No proof = no merge. ### 1. No commit without tests -Every change that touches logic MUST have tests. For Go backend: `cd cmd/server && go test ./...` and `cd cmd/ingestor && go test ./...`. For frontend: `node test-packet-filter.js && node test-aging.js && node test-frontend-helpers.js`. If you add new logic, add tests. No exceptions. +Every change that touches logic MUST have tests. For Go backend: `cd cmd/server && go test ./...` and `cd cmd/ingestor && go test ./...`, or `make test` for all 14 modules. For frontend: `node test-packet-filter.js && node test-aging.js && node test-frontend-helpers.js`. If you add new logic, add tests. No exceptions. + +The SQLite driver (`github.com/mattn/go-sqlite3`) is cgo. **`CGO_ENABLED=0` still *builds*** — that is the trap. It links a stub, and the binary dies on the first query with `go-sqlite3 requires cgo to work. This is a stub`, so a green build proves nothing. A plain `GOOS=linux go build` likewise cannot cross-compile it. Use `make build` / `make crossbuild` — the latter needs [`zig`](https://ziglang.org/download/) as the cross C compiler and produces static musl binaries. Never re-add `CGO_ENABLED=0`. ### 2. No commit without browser validation After pushing, verify the change works in an actual browser. Use `browser profile=openclaw` against the running instance. Take a screenshot if the change is visual. If you can't validate it, say so — don't claim it works. diff --git a/Dockerfile b/Dockerfile index 4e70794c..3042081d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,13 @@ -# Build stage always runs natively on the builder's arch ($BUILDPLATFORM) -# and cross-compiles to $TARGETOS/$TARGETARCH via Go toolchain. No QEMU. +# syntax=docker/dockerfile:1 +# Build stage always runs natively on the builder's arch ($BUILDPLATFORM) and +# cross-compiles to $TARGETOS/$TARGETARCH. No QEMU for compilation. +# +# The SQLite driver is github.com/mattn/go-sqlite3, which is cgo, so the Go +# toolchain alone can no longer cross-compile this: it needs a C compiler that +# can target the other architecture. `zig cc` is that compiler. Targeting musl +# makes the result fully static (see -extldflags below), so the runtime stage +# has no libc dependency on the base image at all. +# # BUILDPLATFORM is auto-set by buildx; default to linux/amd64 so plain # `docker build` (without buildx) doesn't fail on an empty platform string. ARG BUILDPLATFORM=linux/amd64 @@ -12,7 +20,40 @@ ARG BUILD_TIME=unknown ARG TARGETOS ARG TARGETARCH -# Build server (pure-Go sqlite — no CGO needed, cross-compiles cleanly) +# Keep these in step with the Makefile: netgo/osusergo preserve the pure-Go +# resolver and user lookup the binaries had under CGO_ENABLED=0, and +# sqlite_omit_load_extension drops the dlopen path so -static links cleanly. +ENV GO_BUILD_TAGS=netgo,osusergo,sqlite_omit_load_extension \ + ZIG_GLOBAL_CACHE_DIR=/tmp/zig-cache + +# Pinned zig, checksum-verified. Arch comes from `uname -m` rather than +# TARGETARCH because this stage is pinned to BUILDPLATFORM — it has to work when +# someone builds on an arm64 machine too. +ARG ZIG_VERSION=0.16.0 +RUN apk add --no-cache curl xz && \ + case "$(uname -m)" in \ + x86_64) ZA=x86_64; ZSHA=70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d00 ;; \ + aarch64) ZA=aarch64; ZSHA=ea4b09bfb22ec6f6c6ceac57ab63efb6b46e17ab08d21f69f3a48b38e1534f17 ;; \ + *) echo "unsupported builder arch $(uname -m)" >&2; exit 1 ;; \ + esac && \ + curl -sSLo /tmp/zig.tar.xz "https://ziglang.org/download/${ZIG_VERSION}/zig-${ZA}-linux-${ZIG_VERSION}.tar.xz" && \ + echo "${ZSHA} /tmp/zig.tar.xz" | sha256sum -c - && \ + mkdir -p /opt/zig && tar -xJf /tmp/zig.tar.xz -C /opt/zig --strip-components=1 && \ + ln -s /opt/zig/zig /usr/local/bin/zig && rm /tmp/zig.tar.xz && \ + zig version + +# zigcc resolves TARGETARCH to a zig target triple once, so the three build +# steps below stay readable and cannot disagree with each other. +RUN printf '%s\n' '#!/bin/sh' \ + 'case "$TARGETARCH" in' \ + ' amd64) t=x86_64-linux-musl ;;' \ + ' arm64) t=aarch64-linux-musl ;;' \ + ' *) echo "unsupported TARGETARCH=$TARGETARCH" >&2; exit 1 ;;' \ + 'esac' \ + 'exec zig cc -target "$t" "$@"' > /usr/local/bin/zigcc && chmod +x /usr/local/bin/zigcc +ENV CC=zigcc CGO_ENABLED=1 + +# Build server WORKDIR /build/server COPY cmd/server/go.mod cmd/server/go.sum ./ COPY internal/geofilter/ ../../internal/geofilter/ @@ -24,10 +65,18 @@ COPY internal/prunequeue/ ../../internal/prunequeue/ COPY internal/perfio/ ../../internal/perfio/ COPY internal/mbcapqueue/ ../../internal/mbcapqueue/ COPY internal/lora/ ../../internal/lora/ -RUN go mod download +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/tmp/zig-cache \ + go mod download COPY cmd/server/ ./ -RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ - go build -ldflags "-X main.Version=${APP_VERSION} -X main.Commit=${GIT_COMMIT} -X main.BuildTime=${BUILD_TIME}" -o /corescope-server . +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/tmp/zig-cache \ + GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ + go build -trimpath -tags ${GO_BUILD_TAGS} \ + -ldflags "-s -w -extldflags '-static -Wl,-s' -X main.Version=${APP_VERSION} -X main.Commit=${GIT_COMMIT} -X main.BuildTime=${BUILD_TIME}" \ + -o /corescope-server . # Build ingestor WORKDIR /build/ingestor @@ -40,19 +89,35 @@ COPY internal/dbschema/ ../../internal/dbschema/ COPY internal/prunequeue/ ../../internal/prunequeue/ COPY internal/perfio/ ../../internal/perfio/ COPY internal/mbcapqueue/ ../../internal/mbcapqueue/ -RUN go mod download +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/tmp/zig-cache \ + go mod download COPY cmd/ingestor/ ./ -RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ - go build -o /corescope-ingestor . +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/tmp/zig-cache \ + GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ + go build -trimpath -tags ${GO_BUILD_TAGS} \ + -ldflags "-s -w -extldflags '-static -Wl,-s'" \ + -o /corescope-ingestor . # Build decrypt CLI WORKDIR /build/decrypt COPY cmd/decrypt/go.mod cmd/decrypt/go.sum ./ COPY internal/channel/ ../../internal/channel/ -RUN go mod download +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/tmp/zig-cache \ + go mod download COPY cmd/decrypt/ ./ -RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ - go build -ldflags="-s -w" -o /corescope-decrypt . +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/tmp/zig-cache \ + GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ + go build -trimpath -tags ${GO_BUILD_TAGS} \ + -ldflags "-s -w -extldflags '-static -Wl,-s' -X main.version=${APP_VERSION}" \ + -o /corescope-decrypt . # Runtime image FROM alpine:3.20 @@ -61,7 +126,7 @@ RUN apk add --no-cache mosquitto mosquitto-clients supervisor caddy wget WORKDIR /app -# Go binaries +# Go binaries (statically linked; they do not use this image's libc) COPY --from=builder /corescope-server /corescope-ingestor /corescope-decrypt /app/ # Frontend assets + config diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..65c25df4 --- /dev/null +++ b/Makefile @@ -0,0 +1,139 @@ +# corescope build entry point. +# +# This repo is 14 Go modules wired with `replace ../../internal/*` and no +# go.work, so every recipe changes into its module. The build flags live here +# because the SQLite driver is cgo now (github.com/mattn/go-sqlite3) and the +# flags stopped being something you can safely retype at each call site. +# +# Cross-compilation uses `zig cc` as the C toolchain, targeting musl so the +# binaries are fully static and run on the alpine runtime image (or scratch) +# with no libc dependency at all. +# +# Quick reference: +# make build # all four binaries for the host +# make build-server # just one +# make crossbuild # linux/amd64 + linux/arm64, static, into dist/ +# make test / vet / fmt-check # across all modules +# make docker-build # multi-arch image via buildx + +GO ?= $(shell command -v go) +GIT_VERSION ?= $(shell git describe --tags --match "v*" 2>/dev/null || echo unknown) +GIT_COMMIT ?= $(shell git rev-parse --short HEAD 2>/dev/null || echo unknown) +BUILD_TIME ?= $(shell date -u +%Y-%m-%dT%H:%M:%SZ) + +GOENV_GOOS := $(shell $(GO) env GOOS) +GOENV_GOARCH := $(shell $(GO) env GOARCH) +GOOS ?= $(GOENV_GOOS) +GOARCH ?= $(GOENV_GOARCH) + +# Exported: a bare make variable never reaches the go toolchain. +export CGO_ENABLED ?= 1 + +# netgo/osusergo keep the pure-Go resolver and user lookup the binaries had +# under CGO_ENABLED=0, so enabling cgo for SQLite does not quietly switch DNS +# resolution to the C resolver. sqlite_omit_load_extension drops the dlopen +# path, which is what lets -extldflags -static link cleanly. -Wl,-s on the +# cross builds strips the musl objects zig links in: Go's own -s -w does not +# reach them, and they account for well over half the binary. +GO_BUILD_TAGS ?= netgo,osusergo,sqlite_omit_load_extension +GO_BUILD_FLAGS ?= -trimpath +GO_LDFLAGS_OPTIMS ?= -s -w + +CMDS := server ingestor decrypt migrate +DIST := dist + +# Per-binary version stamping. Only these two read build metadata; ingestor and +# migrate take none. Kept as make variables so crossbuild and the host build +# cannot drift apart. +LDFLAGS_server := -X main.Version=$(GIT_VERSION) -X main.Commit=$(GIT_COMMIT) -X main.BuildTime=$(BUILD_TIME) +LDFLAGS_ingestor := +LDFLAGS_decrypt := -X main.version=$(GIT_VERSION) +LDFLAGS_migrate := + +# zig target triples for the platforms we ship. musl, so the result is static. +ZIG_TARGET_linux_amd64 := x86_64-linux-musl +ZIG_TARGET_linux_arm64 := aarch64-linux-musl +CROSS_PLATFORMS := linux/amd64 linux/arm64 + +DOCKER_IMAGE ?= ghcr.io/kpa-clawbot/corescope +DOCKER_TAG ?= $(GIT_VERSION) +DOCKER_PLATFORMS ?= linux/amd64,linux/arm64 + +.PHONY: all build crossbuild test vet fmt-check tidy clean docker-build docker-push help + +all: build + +help: + @echo "targets: build crossbuild test vet fmt-check tidy clean docker-build docker-push" + @echo " build-{$(shell echo $(CMDS) | tr ' ' ',')}" + +# -- host builds --------------------------------------------------------------- + +build: $(addprefix build-,$(CMDS)) + +build-%: + @mkdir -p $(DIST) + cd cmd/$* && GOOS=$(GOOS) GOARCH=$(GOARCH) $(GO) build \ + -tags $(GO_BUILD_TAGS) $(GO_BUILD_FLAGS) \ + -ldflags "$(GO_LDFLAGS_OPTIMS) $(LDFLAGS_$*)" \ + -o ../../$(DIST)/corescope-$* . + +# -- cross builds -------------------------------------------------------------- +# +# The host build above deliberately leaves CC alone: native builds, `go vet` and +# `-race` must use the host compiler. Only these recipes hand the build to zig. + +crossbuild: $(foreach p,$(CROSS_PLATFORMS),$(foreach c,$(CMDS),crossbuild-$(c)-$(subst /,-,$(p)))) + +define CROSSBUILD_RULE +.PHONY: crossbuild-$(2)-$(3)-$(4) +crossbuild-$(2)-$(3)-$(4): + @mkdir -p $$(DIST) + @command -v zig >/dev/null || { echo "zig not found: needed to cross-compile cgo. See https://ziglang.org/download/"; exit 1; } + cd cmd/$(2) && CGO_ENABLED=1 GOOS=$(3) GOARCH=$(4) CC="zig cc -target $(1)" \ + $$(GO) build -tags $$(GO_BUILD_TAGS) $$(GO_BUILD_FLAGS) \ + -ldflags '$$(GO_LDFLAGS_OPTIMS) -extldflags "-static -Wl,-s" $$(LDFLAGS_$(2))' \ + -o ../../$$(DIST)/corescope-$(2)-$(3)-$(4) . +endef + +$(foreach c,$(CMDS),\ + $(eval $(call CROSSBUILD_RULE,$(ZIG_TARGET_linux_amd64),$(c),linux,amd64))\ + $(eval $(call CROSSBUILD_RULE,$(ZIG_TARGET_linux_arm64),$(c),linux,arm64))) + +# -- checks -------------------------------------------------------------------- + +test: + bash scripts/allmod.sh test ./... + +vet: + bash scripts/allmod.sh vet ./... + +fmt-check: + @unformatted=$$(gofmt -l $$(git ls-files '*.go' | grep -vi 'Dockerfile.go')); \ + if [ -n "$$unformatted" ]; then echo "gofmt required on:"; echo "$$unformatted"; exit 1; fi; \ + echo "gofmt: clean" + +tidy: + bash scripts/allmod.sh mod tidy + +clean: + rm -rf $(DIST) + +# -- docker -------------------------------------------------------------------- + +docker-build: + docker buildx build . \ + --platform=$(DOCKER_PLATFORMS) \ + --build-arg=APP_VERSION=$(GIT_VERSION) \ + --build-arg=GIT_COMMIT=$(GIT_COMMIT) \ + --build-arg=BUILD_TIME=$(BUILD_TIME) \ + -t $(DOCKER_IMAGE):$(DOCKER_TAG) + +docker-push: DOCKER_PUSH_FLAG := --push +docker-push: + docker buildx build . \ + --platform=$(DOCKER_PLATFORMS) \ + --build-arg=APP_VERSION=$(GIT_VERSION) \ + --build-arg=GIT_COMMIT=$(GIT_COMMIT) \ + --build-arg=BUILD_TIME=$(BUILD_TIME) \ + -t $(DOCKER_IMAGE):$(DOCKER_TAG) $(DOCKER_PUSH_FLAG) diff --git a/README.md b/README.md index 5ba0b3c4..b06d54a4 100644 --- a/README.md +++ b/README.md @@ -234,6 +234,22 @@ corescope/ ## For Developers +### Building + +```bash +make build # all four binaries for your machine, into dist/ +make build-server # just one +make crossbuild # static linux/amd64 + linux/arm64 binaries +``` + +The SQLite driver is [`mattn/go-sqlite3`](https://github.com/mattn/go-sqlite3), which +is cgo, so a plain `GOOS=linux go build` from a Mac will not work: cross-compiling +needs a C compiler that can target the other platform. `make crossbuild` uses +[`zig`](https://ziglang.org/download/) as that compiler (install it and it just +works) and links statically against musl, so the result is one self-contained file +that runs on Alpine or scratch. The container build does the same thing — see +`Dockerfile`. + ### Test Suite **380 Go tests** covering the backend, plus **150+ Node.js tests** for the frontend and legacy logic, plus **49 Playwright E2E tests** for browser validation. @@ -243,6 +259,9 @@ corescope/ cd cmd/server && go test ./... -v cd cmd/ingestor && go test ./... -v +# Or across all 14 modules at once +make test + # Node.js frontend + integration tests npm test diff --git a/cmd/decrypt/README.md b/cmd/decrypt/README.md index 3aa70a48..d6bf5ee7 100644 --- a/cmd/decrypt/README.md +++ b/cmd/decrypt/README.md @@ -37,7 +37,7 @@ chmod +x corescope-decrypt-linux-amd64 ```bash cd cmd/decrypt -CGO_ENABLED=0 go build -ldflags="-s -w" -o corescope-decrypt . +go build -ldflags="-s -w" -o corescope-decrypt . # cgo: the SQLite driver needs it ``` The binary is statically linked — no dependencies, runs on any Linux. diff --git a/cmd/decrypt/go.mod b/cmd/decrypt/go.mod index 185a76ae..226dbcf0 100644 --- a/cmd/decrypt/go.mod +++ b/cmd/decrypt/go.mod @@ -3,20 +3,8 @@ module github.com/corescope/decrypt go 1.22 require ( + github.com/mattn/go-sqlite3 v1.14.52 github.com/meshcore-analyzer/channel v0.0.0 - modernc.org/sqlite v1.34.5 -) - -require ( - github.com/dustin/go-humanize v1.0.1 // indirect - github.com/google/uuid v1.6.0 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect - github.com/ncruces/go-strftime v0.1.9 // indirect - github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect - golang.org/x/sys v0.22.0 // indirect - modernc.org/libc v1.55.3 // indirect - modernc.org/mathutil v1.6.0 // indirect - modernc.org/memory v1.8.0 // indirect ) replace github.com/meshcore-analyzer/channel => ../../internal/channel diff --git a/cmd/decrypt/go.sum b/cmd/decrypt/go.sum index 5424fe41..75bf6eb5 100644 --- a/cmd/decrypt/go.sum +++ b/cmd/decrypt/go.sum @@ -1,43 +1,2 @@ -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo= -github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= -github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= -golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= -golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= -modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ= -modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ= -modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y= -modernc.org/ccgo/v4 v4.19.2/go.mod h1:ysS3mxiMV38XGRTTcgo0DQTeTmAO4oCmJl1nX9VFI3s= -modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= -modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= -modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw= -modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU= -modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U= -modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w= -modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= -modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= -modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= -modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= -modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= -modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= -modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc= -modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss= -modernc.org/sqlite v1.34.5 h1:Bb6SR13/fjp15jt70CL4f18JIN7p7dnMExd+UFnF15g= -modernc.org/sqlite v1.34.5/go.mod h1:YLuNmX9NKs8wRNK2ko1LW1NGYcc9FkBO69JOt1AR9JE= -modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= -modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= -modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= -modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +github.com/mattn/go-sqlite3 v1.14.52 h1:wVbm2Qnf4OXkqhBTSPuCRZDRnxfbVrrmiCEroVdog8U= +github.com/mattn/go-sqlite3 v1.14.52/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= diff --git a/cmd/decrypt/main.go b/cmd/decrypt/main.go index 9a19ca69..8c897d6f 100644 --- a/cmd/decrypt/main.go +++ b/cmd/decrypt/main.go @@ -18,8 +18,8 @@ import ( "strings" "time" + _ "github.com/mattn/go-sqlite3" "github.com/meshcore-analyzer/channel" - _ "modernc.org/sqlite" ) // Version info (set via ldflags). @@ -121,7 +121,7 @@ LIMITATIONS key := channel.DeriveKey(ch) chHash := channel.ChannelHash(key) - db, err := sql.Open("sqlite", *dbPath+"?mode=ro") + db, err := sql.Open("sqlite3", "file:"+*dbPath+"?mode=ro") if err != nil { log.Fatalf("Failed to open database: %v", err) } diff --git a/cmd/ingestor/README.md b/cmd/ingestor/README.md index 5cf5536f..0ad2986b 100644 --- a/cmd/ingestor/README.md +++ b/cmd/ingestor/README.md @@ -12,7 +12,7 @@ MQTT Broker(s) → Go Ingestor → SQLite DB ← Node.js Web Server ``` - **Single static binary** — no runtime dependencies, no CGO -- **SQLite** via `modernc.org/sqlite` (pure Go) +- **SQLite** via `github.com/mattn/go-sqlite3` (cgo; cross-compiled with `zig cc`, see the root `Makefile`) - **MQTT** via `github.com/eclipse/paho.mqtt.golang` - Runs **alongside** the Node.js server — they share the DB file - Does NOT serve HTTP/WebSocket — that stays in Node.js diff --git a/cmd/ingestor/db.go b/cmd/ingestor/db.go index facd529a..81af4e08 100644 --- a/cmd/ingestor/db.go +++ b/cmd/ingestor/db.go @@ -14,9 +14,9 @@ import ( "sync/atomic" "time" + _ "github.com/mattn/go-sqlite3" "github.com/meshcore-analyzer/dbschema" "github.com/meshcore-analyzer/packetpath" - _ "modernc.org/sqlite" ) // DBStats tracks operational metrics for the ingestor database. @@ -130,7 +130,7 @@ func OpenStoreWithInterval(dbPath string, sampleIntervalSec int) (*Store, error) return nil, fmt.Errorf("creating data dir: %w", err) } - db, err := sql.Open("sqlite", dbPath+"?_pragma=auto_vacuum(INCREMENTAL)&_pragma=journal_mode(WAL)&_pragma=foreign_keys(ON)&_pragma=busy_timeout(5000)") + db, err := sql.Open("sqlite3", dbschema.WriterDSN(dbPath)) if err != nil { return nil, fmt.Errorf("opening db: %w", err) } @@ -141,7 +141,7 @@ func OpenStoreWithInterval(dbPath string, sampleIntervalSec int) (*Store, error) db.SetMaxOpenConns(1) db.SetMaxIdleConns(1) - log.Printf("SQLite config: busy_timeout=5000ms, max_open_conns=1, max_idle_conns=1, journal=WAL") + log.Printf("SQLite config: busy_timeout=5000ms, max_open_conns=1, max_idle_conns=1, journal=WAL, synchronous=FULL") if err := applySchema(db); err != nil { return nil, fmt.Errorf("applying schema: %w", err) diff --git a/cmd/ingestor/db_test.go b/cmd/ingestor/db_test.go index 1cea8a57..9cd4fbfe 100644 --- a/cmd/ingestor/db_test.go +++ b/cmd/ingestor/db_test.go @@ -1627,7 +1627,7 @@ func TestObsTimestampIndexMigration(t *testing.T) { // Build a bare-bones DB that mimics an old installation: // observations table exists but idx_observations_timestamp does NOT. - db, err := sql.Open("sqlite", path) + db, err := sql.Open("sqlite3", path) if err != nil { t.Fatal(err) } @@ -2676,7 +2676,7 @@ func TestCleanupLegacyNullHashTimestamp(t *testing.T) { path := tempDBPath(t) // Create a bare-bones DB with legacy bad data - db, err := sql.Open("sqlite", path+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)") + db, err := sql.Open("sqlite3", path+"?_journal_mode=WAL&_busy_timeout=5000") if err != nil { t.Fatal(err) } @@ -2806,7 +2806,7 @@ func TestBackfillPathJSONAsync(t *testing.T) { dbPath := filepath.Join(dir, "async_test.db") // Bootstrap schema manually so we can insert test data BEFORE OpenStore - db, err := sql.Open("sqlite", dbPath+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)") + db, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL&_busy_timeout=5000") if err != nil { t.Fatal(err) } @@ -2995,7 +2995,7 @@ func TestBackfillPathJSONAsync_BracketRowsTerminate(t *testing.T) { // Bootstrap a minimal schema directly so we can seed pre-existing '[]' rows // before OpenStore runs. - db, err := sql.Open("sqlite", dbPath+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)") + db, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL&_busy_timeout=5000") if err != nil { t.Fatal(err) } diff --git a/cmd/ingestor/go.mod b/cmd/ingestor/go.mod index 729c8e9d..1d5cc637 100644 --- a/cmd/ingestor/go.mod +++ b/cmd/ingestor/go.mod @@ -6,7 +6,6 @@ require ( github.com/eclipse/paho.mqtt.golang v1.5.0 github.com/meshcore-analyzer/geofilter v0.0.0 github.com/meshcore-analyzer/sigvalidate v0.0.0 - modernc.org/sqlite v1.34.5 ) replace github.com/meshcore-analyzer/geofilter => ../../internal/geofilter @@ -30,24 +29,18 @@ require github.com/meshcore-analyzer/dbschema v0.0.0 replace github.com/meshcore-analyzer/dbschema => ../../internal/dbschema require ( - github.com/dustin/go-humanize v1.0.1 // indirect - github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.5.3 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect - github.com/ncruces/go-strftime v0.1.9 // indirect - github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect golang.org/x/net v0.27.0 // indirect golang.org/x/sync v0.7.0 // indirect - golang.org/x/sys v0.22.0 // indirect - modernc.org/libc v1.55.3 // indirect - modernc.org/mathutil v1.6.0 // indirect - modernc.org/memory v1.8.0 // indirect ) require github.com/meshcore-analyzer/prunequeue v0.0.0 replace github.com/meshcore-analyzer/prunequeue => ../../internal/prunequeue -require github.com/meshcore-analyzer/mbcapqueue v0.0.0 +require ( + github.com/mattn/go-sqlite3 v1.14.52 + github.com/meshcore-analyzer/mbcapqueue v0.0.0 +) replace github.com/meshcore-analyzer/mbcapqueue => ../../internal/mbcapqueue diff --git a/cmd/ingestor/go.sum b/cmd/ingestor/go.sum index cc13ef47..554a4827 100644 --- a/cmd/ingestor/go.sum +++ b/cmd/ingestor/go.sum @@ -1,51 +1,10 @@ -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/eclipse/paho.mqtt.golang v1.5.0 h1:EH+bUVJNgttidWFkLLVKaQPGmkTUfQQqjOsyvMGvD6o= github.com/eclipse/paho.mqtt.golang v1.5.0/go.mod h1:du/2qNQVqJf/Sqs4MEL77kR8QTqANF7XU7Fk0aOTAgk= -github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo= -github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= -github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= -golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +github.com/mattn/go-sqlite3 v1.14.52 h1:wVbm2Qnf4OXkqhBTSPuCRZDRnxfbVrrmiCEroVdog8U= +github.com/mattn/go-sqlite3 v1.14.52/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= -golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= -modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ= -modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ= -modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y= -modernc.org/ccgo/v4 v4.19.2/go.mod h1:ysS3mxiMV38XGRTTcgo0DQTeTmAO4oCmJl1nX9VFI3s= -modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= -modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= -modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw= -modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU= -modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U= -modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w= -modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= -modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= -modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= -modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= -modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= -modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= -modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc= -modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss= -modernc.org/sqlite v1.34.5 h1:Bb6SR13/fjp15jt70CL4f18JIN7p7dnMExd+UFnF15g= -modernc.org/sqlite v1.34.5/go.mod h1:YLuNmX9NKs8wRNK2ko1LW1NGYcc9FkBO69JOt1AR9JE= -modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= -modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= -modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= -modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/cmd/ingestor/issue1283_test.go b/cmd/ingestor/issue1283_test.go index f19936ba..4cca0164 100644 --- a/cmd/ingestor/issue1283_test.go +++ b/cmd/ingestor/issue1283_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" ) // TestIngestorPruneOldPackets enforces #1283: the writer for @@ -64,7 +64,7 @@ func TestIngestorVacuumOnStartupMigratesNONEtoINCREMENTAL(t *testing.T) { path := filepath.Join(dir, "vac.db") // Create a NONE-auto_vacuum DB (simulates an older deployment). - seed, err := sql.Open("sqlite", path+"?_pragma=journal_mode(WAL)") + seed, err := sql.Open("sqlite3", path+"?_journal_mode=WAL") if err != nil { t.Fatal(err) } diff --git a/cmd/ingestor/pragma_parity_test.go b/cmd/ingestor/pragma_parity_test.go new file mode 100644 index 00000000..e90e4d97 --- /dev/null +++ b/cmd/ingestor/pragma_parity_test.go @@ -0,0 +1,55 @@ +package main + +import ( + "path/filepath" + "testing" +) + +// TestOpenStorePragmas pins the pragmas the writer connection runs with. +// +// These were silently at risk during the modernc.org/sqlite → +// github.com/mattn/go-sqlite3 migration in two different ways: +// +// - The DSN dialect changed. modernc understood only `_pragma=name(value)`; +// mattn understands only `_`-prefixed parameters and ignores the other +// form outright. A driver-only rename would have dropped every pragma on +// the floor with no error anywhere. +// +// - mattn runs `PRAGMA synchronous = NORMAL` unconditionally, defaulting the +// mode itself to NORMAL rather than leaving SQLite's own FULL in place. So +// the writer would have quietly moved from FULL (2) to NORMAL (1), +// weakening durability under power loss. `_synchronous=FULL` in the DSN is +// what holds it — this test is what notices if it is ever dropped. +// +// Read through the store's own connection on purpose: a separate connection, or +// the log line in OpenStoreWithInterval, would prove nothing about what the +// writer is actually using. +func TestOpenStorePragmas(t *testing.T) { + store, err := OpenStore(filepath.Join(t.TempDir(), "pragma.db")) + if err != nil { + t.Fatalf("OpenStore: %v", err) + } + defer store.db.Close() + + for _, want := range []struct { + pragma string + value string + why string + }{ + {"journal_mode", "wal", "concurrent reader (cmd/server) requires WAL"}, + {"synchronous", "2", "FULL; mattn defaults to 1 (NORMAL) unless the DSN says otherwise"}, + {"auto_vacuum", "2", "INCREMENTAL; maintenance.go drives incremental_vacuum"}, + {"foreign_keys", "1", "schema relies on FK enforcement"}, + {"busy_timeout", "5000", "writer serialises behind the reader"}, + {"cache_size", "-2000", "2 MiB of C-allocated page cache, pinned: it sits outside GOMEMLIMIT"}, + } { + var got string + if err := store.db.QueryRow("PRAGMA " + want.pragma).Scan(&got); err != nil { + t.Errorf("PRAGMA %s: %v", want.pragma, err) + continue + } + if got != want.value { + t.Errorf("PRAGMA %s = %q, want %q (%s)", want.pragma, got, want.value, want.why) + } + } +} diff --git a/cmd/ingestor/prune_chunked_test.go b/cmd/ingestor/prune_chunked_test.go index a54130b0..a3be3261 100644 --- a/cmd/ingestor/prune_chunked_test.go +++ b/cmd/ingestor/prune_chunked_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" ) // seedAgedTransmissions inserts n transmissions stamped `age` days in the diff --git a/cmd/migrate/go.mod b/cmd/migrate/go.mod index b380d96c..1d24e0a4 100644 --- a/cmd/migrate/go.mod +++ b/cmd/migrate/go.mod @@ -3,20 +3,8 @@ module github.com/corescope/migrate go 1.22 require ( + github.com/mattn/go-sqlite3 v1.14.52 github.com/meshcore-analyzer/dbschema v0.0.0 - modernc.org/sqlite v1.34.5 ) replace github.com/meshcore-analyzer/dbschema => ../../internal/dbschema - -require ( - github.com/dustin/go-humanize v1.0.1 // indirect - github.com/google/uuid v1.6.0 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect - github.com/ncruces/go-strftime v0.1.9 // indirect - github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect - golang.org/x/sys v0.22.0 // indirect - modernc.org/libc v1.55.3 // indirect - modernc.org/mathutil v1.6.0 // indirect - modernc.org/memory v1.8.0 // indirect -) diff --git a/cmd/migrate/go.sum b/cmd/migrate/go.sum index 5424fe41..75bf6eb5 100644 --- a/cmd/migrate/go.sum +++ b/cmd/migrate/go.sum @@ -1,43 +1,2 @@ -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo= -github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= -github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= -golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= -golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= -modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ= -modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ= -modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y= -modernc.org/ccgo/v4 v4.19.2/go.mod h1:ysS3mxiMV38XGRTTcgo0DQTeTmAO4oCmJl1nX9VFI3s= -modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= -modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= -modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw= -modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU= -modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U= -modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w= -modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= -modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= -modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= -modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= -modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= -modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= -modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc= -modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss= -modernc.org/sqlite v1.34.5 h1:Bb6SR13/fjp15jt70CL4f18JIN7p7dnMExd+UFnF15g= -modernc.org/sqlite v1.34.5/go.mod h1:YLuNmX9NKs8wRNK2ko1LW1NGYcc9FkBO69JOt1AR9JE= -modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= -modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= -modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= -modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +github.com/mattn/go-sqlite3 v1.14.52 h1:wVbm2Qnf4OXkqhBTSPuCRZDRnxfbVrrmiCEroVdog8U= +github.com/mattn/go-sqlite3 v1.14.52/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= diff --git a/cmd/migrate/main.go b/cmd/migrate/main.go index c10f75a1..d6f7eae0 100644 --- a/cmd/migrate/main.go +++ b/cmd/migrate/main.go @@ -18,8 +18,8 @@ import ( "flag" "log" + _ "github.com/mattn/go-sqlite3" "github.com/meshcore-analyzer/dbschema" - _ "modernc.org/sqlite" ) func main() { @@ -33,7 +33,10 @@ func main() { log.SetFlags(log.LstdFlags | log.Lmsgprefix) log.SetPrefix("[migrate] ") - db, err := sql.Open("sqlite", *dbPath) + // Same DSN as the ingestor. A bare path here would mean synchronous=NORMAL + // under mattn, quietly weakening durability for a process that writes — and + // this one runs dbschema.Apply, which now also deletes duplicate rows. + db, err := sql.Open("sqlite3", dbschema.WriterDSN(*dbPath)) if err != nil { log.Fatalf("open %s: %v", *dbPath, err) } diff --git a/cmd/migrate/main_test.go b/cmd/migrate/main_test.go index 63956fbc..b32b5c9b 100644 --- a/cmd/migrate/main_test.go +++ b/cmd/migrate/main_test.go @@ -12,8 +12,8 @@ import ( "path/filepath" "testing" + _ "github.com/mattn/go-sqlite3" "github.com/meshcore-analyzer/dbschema" - _ "modernc.org/sqlite" ) // fixtureCandidates lists possible locations of the committed e2e @@ -61,7 +61,7 @@ func TestMigrateBringsFixtureToReady(t *testing.T) { dst := filepath.Join(t.TempDir(), "fixture-copy.db") copyFile(t, src, dst) - db, err := sql.Open("sqlite", dst) + db, err := sql.Open("sqlite3", dst) if err != nil { t.Fatalf("open: %v", err) } diff --git a/cmd/server/bounded_load_test.go b/cmd/server/bounded_load_test.go index 6da1d458..ef383d54 100644 --- a/cmd/server/bounded_load_test.go +++ b/cmd/server/bounded_load_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" ) // createTestDB creates a temporary SQLite database with N transmissions (1 obs each). @@ -149,7 +149,7 @@ func createTestDBWithAgedPackets(t *testing.T, numRecent, numOld int) string { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL") + conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") if err != nil { t.Fatal(err) } @@ -162,8 +162,8 @@ func createTestDBWithAgedPackets(t *testing.T, numRecent, numOld int) string { } execOrFail(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT)`) execOrFail(`CREATE TABLE observations (id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT)`) - execOrFail(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`) - execOrFail(`CREATE TABLE nodes (pubkey TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, frequency REAL)`) + execOrFail(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) + execOrFail(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, frequency REAL)`) execOrFail(`CREATE TABLE schema_version (version INTEGER)`) execOrFail(`INSERT INTO schema_version (version) VALUES (1)`) execOrFail(`CREATE INDEX idx_tx_first_seen ON transmissions(first_seen)`) @@ -171,7 +171,7 @@ func createTestDBWithAgedPackets(t *testing.T, numRecent, numOld int) string { now := time.Now().UTC() id := 1 // Single transaction for all inserts — see createTestDBAt for the rationale - // (modernc.org/sqlite auto-commit per Exec fsyncs per row). numOld+numRecent + // (auto-commit per Exec fsyncs per row). numOld+numRecent // is small here today, but wrapping keeps the fixture robust if callers scale // it up, and is consistent with the other builders. if _, err := conn.Exec("BEGIN"); err != nil { @@ -308,7 +308,7 @@ func BenchmarkLoad_30K_Unlimited(b *testing.B) { // createTestDBAt is like createTestDB but writes to a specific path. func createTestDBAt(tb testing.TB, dbPath string, numTx int) { tb.Helper() - conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL") + conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") if err != nil { tb.Fatal(err) } @@ -331,9 +331,9 @@ func createTestDBAt(tb testing.TB, dbPath string, numTx int) { direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT )`) - execOrFail(`CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`) + execOrFail(`CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) execOrFail(`CREATE TABLE IF NOT EXISTS nodes ( - pubkey TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, + public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, frequency REAL )`) execOrFail(`CREATE TABLE IF NOT EXISTS schema_version (version INTEGER)`) @@ -352,7 +352,7 @@ func createTestDBAt(tb testing.TB, dbPath string, numTx int) { defer obsStmt.Close() base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) - // Wrap the inserts in a single transaction. Without this, modernc.org/sqlite + // Wrap the inserts in a single transaction. Without this, SQLite // (pure-Go driver) auto-commits every Exec → one fsync per row → ~2N fsyncs // for N transmissions (tx + obs). At numTx=5000 that is ~10k fsyncs and the // fixture blows past the test timeout (the #1741 hang). A single @@ -379,7 +379,7 @@ func createTestDBAt(tb testing.TB, dbPath string, numTx int) { // createTestDBWithObs creates a test DB with realistic observation counts (1–5 per tx). func createTestDBWithObs(tb testing.TB, dbPath string, numTx int) { tb.Helper() - conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL") + conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") if err != nil { tb.Fatal(err) } @@ -398,9 +398,9 @@ func createTestDBWithObs(tb testing.TB, dbPath string, numTx int) { id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT )`) - execOrFail(`CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`) + execOrFail(`CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) execOrFail(`CREATE TABLE IF NOT EXISTS nodes ( - pubkey TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, + public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, frequency REAL )`) execOrFail(`CREATE TABLE IF NOT EXISTS schema_version (version INTEGER)`) @@ -423,7 +423,7 @@ func createTestDBWithObs(tb testing.TB, dbPath string, numTx int) { obsID := 1 base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) // Single transaction for all inserts — see createTestDBAt for the rationale - // (modernc.org/sqlite auto-commit per Exec would fsync per row; at numTx=30000 + // (auto-commit per Exec would fsync per row; at numTx=30000 // the benchmarks would otherwise stall for minutes). One BEGIN/COMMIT. if _, err := conn.Exec("BEGIN"); err != nil { tb.Fatalf("test DB BEGIN: %v", err) diff --git a/cmd/server/chunked_load_id_zero_test.go b/cmd/server/chunked_load_id_zero_test.go index e6cf74bb..7b1fb87e 100644 --- a/cmd/server/chunked_load_id_zero_test.go +++ b/cmd/server/chunked_load_id_zero_test.go @@ -29,7 +29,7 @@ import ( func createTestDBWithIDZero(tb testing.TB, dbPath string, extraTx int) { tb.Helper() - conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL") + conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") if err != nil { tb.Fatal(err) } @@ -48,9 +48,9 @@ func createTestDBWithIDZero(tb testing.TB, dbPath string, extraTx int) { direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT )`, - `CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`, + `CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE IF NOT EXISTS nodes ( - pubkey TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, + public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, frequency REAL )`, `CREATE TABLE IF NOT EXISTS schema_version (version INTEGER)`, diff --git a/cmd/server/chunked_load_oldest_test.go b/cmd/server/chunked_load_oldest_test.go index 8b9a5f82..8b3edce7 100644 --- a/cmd/server/chunked_load_oldest_test.go +++ b/cmd/server/chunked_load_oldest_test.go @@ -31,7 +31,7 @@ import ( // This mirrors the freshen-fixture-shifted e2e DB exactly. func createTestDBReverseTime(tb testing.TB, dbPath string, numTx int) { tb.Helper() - conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL") + conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") if err != nil { tb.Fatal(err) } @@ -50,9 +50,9 @@ func createTestDBReverseTime(tb testing.TB, dbPath string, numTx int) { direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT )`, - `CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`, + `CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE IF NOT EXISTS nodes ( - pubkey TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, + public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, frequency REAL )`, `CREATE TABLE IF NOT EXISTS schema_version (version INTEGER)`, diff --git a/cmd/server/coverage_test.go b/cmd/server/coverage_test.go index 5e790dbf..907ee448 100644 --- a/cmd/server/coverage_test.go +++ b/cmd/server/coverage_test.go @@ -15,14 +15,14 @@ import ( "time" "github.com/gorilla/mux" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" ) // --- helpers --- func setupTestDBv2(t *testing.T) *DB { t.Helper() - conn, err := sql.Open("sqlite", ":memory:") + conn, err := sql.Open("sqlite3", ":memory:") if err != nil { t.Fatal(err) } diff --git a/cmd/server/db.go b/cmd/server/db.go index 9ed1b9af..450933bf 100644 --- a/cmd/server/db.go +++ b/cmd/server/db.go @@ -13,9 +13,9 @@ import ( "sync" "time" + _ "github.com/mattn/go-sqlite3" "github.com/meshcore-analyzer/dbschema" "github.com/meshcore-analyzer/geofilter" - _ "modernc.org/sqlite" ) // routeTypeTransport covers TRANSPORT_FLOOD (0) and TRANSPORT_DIRECT (3) — @@ -103,10 +103,25 @@ type channelMessagesCacheEntry struct { exp time.Time } -// OpenDB opens a read-only SQLite connection with WAL mode. +// OpenDB opens a read-only SQLite connection. +// +// The DSN used to pass _journal_mode=WAL and _busy_timeout=5000, which +// modernc.org/sqlite silently ignored: it understood only the +// _pragma=name(value) form. Under github.com/mattn/go-sqlite3 those parameters +// are honoured, and setting journal_mode on a read-only handle is a write — it +// would succeed only because the database is already WAL. Both are gone: +// mattn's own busy_timeout default is already 5000ms, so the read handle keeps +// the timeout (which it had silently lacked) without the pointless write. +// +// What remains is deliberate. mode=ro is the read-only invariant from +// #1283/#1289 and works because mattn's C wrapper ORs SQLITE_OPEN_URI into the +// open flags — see TestOpenDBRefusesMissingDatabase, which fails if that ever +// stops holding. _cache_size pins SQLite's C-allocated page cache at 2 MiB per +// connection; it sits outside GOMEMLIMIT, so it is bounded here rather than +// left to the driver's default (see memlimit.go). func OpenDB(path string) (*DB, error) { - dsn := fmt.Sprintf("file:%s?mode=ro&_journal_mode=WAL&_busy_timeout=5000", path) - conn, err := sql.Open("sqlite", dsn) + dsn := fmt.Sprintf("file:%s?mode=ro&_cache_size=-2000", path) + conn, err := sql.Open("sqlite3", dsn) if err != nil { return nil, err } diff --git a/cmd/server/db_test.go b/cmd/server/db_test.go index 8dc492b0..8b08e8e5 100644 --- a/cmd/server/db_test.go +++ b/cmd/server/db_test.go @@ -11,13 +11,13 @@ import ( "testing" "time" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" ) // setupTestDB creates an in-memory SQLite database with the v3 schema. func setupTestDB(t *testing.T) *DB { t.Helper() - conn, err := sql.Open("sqlite", ":memory:") + conn, err := sql.Open("sqlite3", ":memory:") if err != nil { t.Fatal(err) } @@ -1198,7 +1198,7 @@ func TestGetNodesFiltering(t *testing.T) { // where observations use observer_id TEXT instead of observer_idx INTEGER. func setupTestDBV2(t *testing.T) *DB { t.Helper() - conn, err := sql.Open("sqlite", ":memory:") + conn, err := sql.Open("sqlite3", ":memory:") if err != nil { t.Fatal(err) } @@ -1226,7 +1226,8 @@ func setupTestDBV2(t *testing.T) *DB { last_seen TEXT, first_seen TEXT, packet_count INTEGER DEFAULT 0, - last_packet_at TEXT DEFAULT NULL + last_packet_at TEXT DEFAULT NULL, + inactive INTEGER DEFAULT 0 ); CREATE TABLE transmissions ( @@ -1454,7 +1455,7 @@ func TestOpenDBValid(t *testing.T) { dbPath := filepath.Join(dir, "test.db") // Create DB with a table using a writable connection first - conn, err := sql.Open("sqlite", dbPath) + conn, err := sql.Open("sqlite3", dbPath) if err != nil { t.Fatal(err) } @@ -1463,6 +1464,7 @@ func TestOpenDBValid(t *testing.T) { conn.Close() t.Fatal(err) } + ensurePreparable(t, conn) conn.Close() // Now test OpenDB (read-only) @@ -1495,7 +1497,7 @@ func TestDetectSchemaScopeName(t *testing.T) { dbPath := filepath.Join(dir, "detect.db") // Create file-based DB with the scope_name and default_scope columns. - conn, err := sql.Open("sqlite", dbPath) + conn, err := sql.Open("sqlite3", dbPath) if err != nil { t.Fatal(err) } @@ -1512,6 +1514,7 @@ func TestDetectSchemaScopeName(t *testing.T) { conn.Close() t.Fatalf("create observations: %v", err) } + ensurePreparable(t, conn) conn.Close() db, err := OpenDB(dbPath) @@ -1529,7 +1532,7 @@ func TestDetectSchemaScopeName(t *testing.T) { // Verify the flags stay false when the columns are absent. dbPath2 := filepath.Join(dir, "detect2.db") - conn2, err := sql.Open("sqlite", dbPath2) + conn2, err := sql.Open("sqlite3", dbPath2) if err != nil { t.Fatal(err) } @@ -1537,6 +1540,7 @@ func TestDetectSchemaScopeName(t *testing.T) { conn2.Exec(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY, hash TEXT)`) conn2.Exec(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY)`) conn2.Exec(`CREATE TABLE observations (id INTEGER PRIMARY KEY)`) + ensurePreparable(t, conn2) conn2.Close() db2, err := OpenDB(dbPath2) @@ -2233,7 +2237,7 @@ func TestPerObservationRawHexEnrich(t *testing.T) { } func TestGetScopeStats(t *testing.T) { - conn, err := sql.Open("sqlite", ":memory:") + conn, err := sql.Open("sqlite3", ":memory:") if err != nil { t.Fatalf("sql.Open: %v", err) } @@ -2293,7 +2297,7 @@ func TestGetScopeStats(t *testing.T) { // scope_name states (NULL is unscoped, an empty string is unknown scope, a // name is a named scope). func TestGetScopeStatsAdvertsByRole(t *testing.T) { - conn, err := sql.Open("sqlite", ":memory:") + conn, err := sql.Open("sqlite3", ":memory:") if err != nil { t.Fatalf("sql.Open: %v", err) } @@ -2377,7 +2381,7 @@ func TestGetScopeStatsAdvertsByRole(t *testing.T) { } func TestGetScopeStatsAdvertsByRoleEmpty(t *testing.T) { - conn, err := sql.Open("sqlite", ":memory:") + conn, err := sql.Open("sqlite3", ":memory:") if err != nil { t.Fatalf("sql.Open: %v", err) } @@ -2463,7 +2467,7 @@ func (f *failingQuerier) QueryContext(ctx context.Context, query string, args .. // a v3 database for the whole process lifetime. func TestDetectSchemaFailsLoudOnProbeError(t *testing.T) { dbPath := filepath.Join(t.TempDir(), "v3.db") - conn, err := sql.Open("sqlite", dbPath) + conn, err := sql.Open("sqlite3", dbPath) if err != nil { t.Fatal(err) } @@ -2477,7 +2481,7 @@ func TestDetectSchemaFailsLoudOnProbeError(t *testing.T) { conn.Exec(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY)`) conn.Close() - real, err := sql.Open("sqlite", "file:"+dbPath+"?mode=ro") + real, err := sql.Open("sqlite3", "file:"+dbPath+"?mode=ro") if err != nil { t.Fatal(err) } @@ -2499,7 +2503,7 @@ func TestDetectSchemaV3AndV2(t *testing.T) { dir := t.TempDir() v3 := filepath.Join(dir, "v3.db") - c, err := sql.Open("sqlite", v3) + c, err := sql.Open("sqlite3", v3) if err != nil { t.Fatal(err) } @@ -2507,6 +2511,7 @@ func TestDetectSchemaV3AndV2(t *testing.T) { c.Exec(`CREATE TABLE observations (id INTEGER PRIMARY KEY, observer_idx INTEGER)`) c.Exec(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY, hash TEXT)`) c.Exec(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY)`) + ensurePreparable(t, c) c.Close() db, err := OpenDB(v3) if err != nil { @@ -2518,7 +2523,7 @@ func TestDetectSchemaV3AndV2(t *testing.T) { db.Close() v2 := filepath.Join(dir, "v2.db") - c2, err := sql.Open("sqlite", v2) + c2, err := sql.Open("sqlite3", v2) if err != nil { t.Fatal(err) } @@ -2526,6 +2531,7 @@ func TestDetectSchemaV3AndV2(t *testing.T) { c2.Exec(`CREATE TABLE observations (id INTEGER PRIMARY KEY, observer_id INTEGER)`) c2.Exec(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY, hash TEXT)`) c2.Exec(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY)`) + ensurePreparable(t, c2) c2.Close() db2, err := OpenDB(v2) if err != nil { diff --git a/cmd/server/go.mod b/cmd/server/go.mod index ae93a43f..7518c678 100644 --- a/cmd/server/go.mod +++ b/cmd/server/go.mod @@ -7,7 +7,6 @@ require ( github.com/gorilla/websocket v1.5.3 github.com/meshcore-analyzer/geofilter v0.0.0 github.com/meshcore-analyzer/sigvalidate v0.0.0 - modernc.org/sqlite v1.34.5 ) replace github.com/meshcore-analyzer/geofilter => ../../internal/geofilter @@ -34,23 +33,12 @@ require github.com/meshcore-analyzer/lora v0.0.0 replace github.com/meshcore-analyzer/lora => ../../internal/lora -require ( - github.com/dustin/go-humanize v1.0.1 // indirect - github.com/google/uuid v1.6.0 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect - github.com/ncruces/go-strftime v0.1.9 // indirect - github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect - golang.org/x/sys v0.22.0 // indirect - modernc.org/libc v1.55.3 // indirect - modernc.org/mathutil v1.6.0 // indirect - modernc.org/memory v1.8.0 // indirect -) - require github.com/meshcore-analyzer/prunequeue v0.0.0 replace github.com/meshcore-analyzer/prunequeue => ../../internal/prunequeue require ( + github.com/mattn/go-sqlite3 v1.14.52 github.com/meshcore-analyzer/mbcapqueue v0.0.0 golang.org/x/sync v0.10.0 ) diff --git a/cmd/server/go.sum b/cmd/server/go.sum index 45b74210..8c505bf8 100644 --- a/cmd/server/go.sum +++ b/cmd/server/go.sum @@ -1,49 +1,8 @@ -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo= -github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= -github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= -golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +github.com/mattn/go-sqlite3 v1.14.52 h1:wVbm2Qnf4OXkqhBTSPuCRZDRnxfbVrrmiCEroVdog8U= +github.com/mattn/go-sqlite3 v1.14.52/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= -golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= -modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ= -modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ= -modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y= -modernc.org/ccgo/v4 v4.19.2/go.mod h1:ysS3mxiMV38XGRTTcgo0DQTeTmAO4oCmJl1nX9VFI3s= -modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= -modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= -modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw= -modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU= -modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U= -modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w= -modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= -modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= -modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= -modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= -modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= -modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= -modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc= -modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss= -modernc.org/sqlite v1.34.5 h1:Bb6SR13/fjp15jt70CL4f18JIN7p7dnMExd+UFnF15g= -modernc.org/sqlite v1.34.5/go.mod h1:YLuNmX9NKs8wRNK2ko1LW1NGYcc9FkBO69JOt1AR9JE= -modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= -modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= -modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= -modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/cmd/server/hexgrid.go b/cmd/server/hexgrid.go index 93c25f3b..689661d8 100644 --- a/cmd/server/hexgrid.go +++ b/cmd/server/hexgrid.go @@ -8,7 +8,9 @@ import ( ) // Pure-Go hexagonal binning for RX coverage display. We deliberately avoid the -// CGO-based uber/h3-go (this project builds with CGO_ENABLED=0). Points are +// CGO-based uber/h3-go. (That predates the SQLite driver move to cgo; this +// stays pure Go because it needs no C library, not because cgo is unavailable.) +// Points are // projected to Web Mercator and snapped to a pointy-top hex grid whose size // depends on the display resolution. Cell ids are "res:q:r" (axial coords). // At city/region scale this looks like H3/mapme.sh coverage without any deps. diff --git a/cmd/server/hot_startup_test.go b/cmd/server/hot_startup_test.go index d848da8e..238369c3 100644 --- a/cmd/server/hot_startup_test.go +++ b/cmd/server/hot_startup_test.go @@ -13,7 +13,7 @@ import ( "time" "github.com/gorilla/mux" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" ) // createTestDBMultiDay creates a test DB with packets spread across numDays days. @@ -24,7 +24,7 @@ func createTestDBMultiDay(t *testing.T, numDays, txPerDay int) string { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL") + conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") if err != nil { t.Fatal(err) } @@ -37,8 +37,8 @@ func createTestDBMultiDay(t *testing.T, numDays, txPerDay int) string { } execOrFail(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT)`) execOrFail(`CREATE TABLE observations (id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT)`) - execOrFail(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`) - execOrFail(`CREATE TABLE nodes (pubkey TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, frequency REAL)`) + execOrFail(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) + execOrFail(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, frequency REAL)`) execOrFail(`CREATE TABLE schema_version (version INTEGER)`) execOrFail(`INSERT INTO schema_version (version) VALUES (1)`) execOrFail(`CREATE INDEX idx_tx_first_seen ON transmissions(first_seen)`) diff --git a/cmd/server/issue1690_cold_load_test.go b/cmd/server/issue1690_cold_load_test.go index 2b1f8500..1668e04c 100644 --- a/cmd/server/issue1690_cold_load_test.go +++ b/cmd/server/issue1690_cold_load_test.go @@ -19,7 +19,7 @@ import ( "testing" "time" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" ) // createTestDBWithLastSeen seeds a DB with the post-fix schema (last_seen diff --git a/cmd/server/issue1809_bg_load_race_test.go b/cmd/server/issue1809_bg_load_race_test.go index 14ba1850..eca19727 100644 --- a/cmd/server/issue1809_bg_load_race_test.go +++ b/cmd/server/issue1809_bg_load_race_test.go @@ -138,7 +138,7 @@ func createTestDBSpreadOverDays(t *testing.T, dbPath string, numTx, spanDays int // (adv #11: prior code carried a duplicated annotation on every line)." func seedTestDBRows(t *testing.T, dbPath string, numTx, obsPerTx int, rowTimes func(i int) (string, int64)) { t.Helper() - conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL") + conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") if err != nil { t.Fatal(err) } @@ -164,9 +164,9 @@ func seedTestDBRows(t *testing.T, dbPath string, numTx, obsPerTx int, rowTimes f path_json TEXT, timestamp TEXT, raw_hex TEXT )`) // PREFLIGHT: async=true reason="unit-test fixture seeder" - execOrFail(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`) + execOrFail(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) // PREFLIGHT: async=true reason="unit-test fixture seeder" - execOrFail(`CREATE TABLE nodes (pubkey TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, frequency REAL)`) + execOrFail(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, frequency REAL)`) // PREFLIGHT: async=true reason="unit-test fixture seeder" execOrFail(`CREATE TABLE schema_version (version INTEGER)`) execOrFail(`INSERT INTO schema_version (version) VALUES (1)`) diff --git a/cmd/server/issue673_test.go b/cmd/server/issue673_test.go index b5ac5517..85337c9d 100644 --- a/cmd/server/issue673_test.go +++ b/cmd/server/issue673_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" ) const issue673NodePK = "7502f19f44cad6d7b626e1d811c00a914af452636182ccded3fd019803395ec9" diff --git a/cmd/server/load_graph_invariant_test.go b/cmd/server/load_graph_invariant_test.go index e128756b..07ae649f 100644 --- a/cmd/server/load_graph_invariant_test.go +++ b/cmd/server/load_graph_invariant_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" ) // TestLoad_PanicsWhenGraphNotLoadedAndEdgesExist pins the startup-ordering @@ -22,7 +22,7 @@ func TestLoad_PanicsWhenGraphNotLoadedAndEdgesExist(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - rw, err := sql.Open("sqlite", "file:"+dbPath+"?_journal_mode=WAL") + rw, err := sql.Open("sqlite3", "file:"+dbPath+"?_journal_mode=WAL") if err != nil { t.Fatal(err) } @@ -49,7 +49,7 @@ func TestLoad_PanicsWhenGraphNotLoadedAndEdgesExist(t *testing.T) { path_json TEXT, timestamp TEXT, raw_hex TEXT, resolved_path TEXT )`) // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" - exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`) + exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" exec(`CREATE TABLE nodes ( public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, diff --git a/cmd/server/loadchunk_ambiguous_prefix_test.go b/cmd/server/loadchunk_ambiguous_prefix_test.go index 9a0d4265..3c0e1596 100644 --- a/cmd/server/loadchunk_ambiguous_prefix_test.go +++ b/cmd/server/loadchunk_ambiguous_prefix_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" ) // createTestDBAmbiguousPrefix builds a fixture where TWO repeaters share the @@ -24,7 +24,7 @@ func createTestDBAmbiguousPrefix(t *testing.T, relayA, relayB, hop, firstSeen st dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL") + conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") if err != nil { t.Fatal(err) } @@ -54,7 +54,7 @@ func createTestDBAmbiguousPrefix(t *testing.T, relayA, relayB, hop, firstSeen st resolved_path TEXT )`) // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" - exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`) + exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" exec(`CREATE TABLE nodes ( public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, diff --git a/cmd/server/loadchunk_pathjson_fallback_test.go b/cmd/server/loadchunk_pathjson_fallback_test.go index 518fe161..9d28b620 100644 --- a/cmd/server/loadchunk_pathjson_fallback_test.go +++ b/cmd/server/loadchunk_pathjson_fallback_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" ) // createTestDBPathJSONNoResolvedPath builds a fixture that mirrors the LIVE @@ -25,7 +25,7 @@ func createTestDBPathJSONNoResolvedPath(t *testing.T, relayPubkey, hopPrefix, fi dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL") + conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") if err != nil { t.Fatal(err) } @@ -56,7 +56,7 @@ func createTestDBPathJSONNoResolvedPath(t *testing.T, relayPubkey, hopPrefix, fi resolved_path TEXT )`) // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" - exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`) + exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) // Production nodes schema uses public_key (not pubkey) — getAllNodes / // buildPrefixMap reads public_key, role, advert_count, first_seen. // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" diff --git a/cmd/server/loadchunk_resolved_path_1558_test.go b/cmd/server/loadchunk_resolved_path_1558_test.go index b9ede3c4..9cd83ecd 100644 --- a/cmd/server/loadchunk_resolved_path_1558_test.go +++ b/cmd/server/loadchunk_resolved_path_1558_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" ) // createTestDBWithResolvedPath creates a fixture DB containing numTx old @@ -20,7 +20,7 @@ func createTestDBWithResolvedPath(t *testing.T, numTx int, relayPubkeys []string dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL") + conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") if err != nil { t.Fatal(err) } @@ -47,8 +47,8 @@ func createTestDBWithResolvedPath(t *testing.T, numTx int, relayPubkeys []string raw_hex TEXT, resolved_path TEXT )`) - exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`) - exec(`CREATE TABLE nodes (pubkey TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, frequency REAL)`) + exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) + exec(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, frequency REAL)`) exec(`CREATE TABLE schema_version (version INTEGER)`) exec(`INSERT INTO schema_version (version) VALUES (1)`) exec(`CREATE INDEX idx_tx_first_seen ON transmissions(first_seen)`) diff --git a/cmd/server/memlimit.go b/cmd/server/memlimit.go index 4ff59060..f8e3df5c 100644 --- a/cmd/server/memlimit.go +++ b/cmd/server/memlimit.go @@ -36,6 +36,21 @@ func applyMemoryLimit(maxMemoryMB int, envSet bool) (int64, string) { // 1.5x headroom over the steady-state packet store budget covers // transient peaks (cold-load row-scan / decode pipeline, Go's NextGC // trigger at ~2x live heap). See issue #836 heap profile. + // + // GOMEMLIMIT covers memory the Go runtime manages — heap, stacks and + // runtime structures — and nothing else. Since the SQLite driver became + // cgo (github.com/mattn/go-sqlite3), what SQLite allocates in C is + // outside it, so this limit is not an RSS ceiling: actual RSS is this + // plus the native allocations plus whatever the runtime has reserved + // but not returned. + // + // The page cache, at least, is deliberately bounded: both DSNs pin + // _cache_size=-2000, i.e. ~2 MiB per connection, ~8 MiB across + // SetMaxOpenConns(4). That fits inside the headroom above rather than + // eating into the store budget. It does not bound SQLite's other native + // allocations, so if the connection count or _cache_size grows + // materially, or a workload starts holding many prepared statements, + // this derivation needs revisiting against measured RSS. limit := int64(maxMemoryMB) * 1024 * 1024 * 3 / 2 debug.SetMemoryLimit(limit) return limit, "derived" diff --git a/cmd/server/memory.go b/cmd/server/memory.go index 2ffe4d7c..8ad52b51 100644 --- a/cmd/server/memory.go +++ b/cmd/server/memory.go @@ -32,9 +32,22 @@ import ( // eviction; both can shrink when packets age out. goHeapInuseMB and goSysMB // fluctuate with GC. // -// cgoBytesMB intentionally absent: this build uses the pure-Go -// modernc.org/sqlite driver, so there is no cgo allocator to measure. -// Reintroduce only if we ever switch back to mattn/go-sqlite3. +// There is no cgo-allocation field, but for a different reason than before: +// the driver is github.com/mattn/go-sqlite3 now, so SQLite's page cache IS a C +// allocation — the Go runtime simply exposes no byte counter for it. +// +// processRSSMB does include it, since VmRSS counts every resident page whoever +// allocated it. Resist the temptation to read `processRSSMB - goSysMB` as "the +// C share": goSysMB is address space the runtime has reserved, not what is +// resident, so the subtraction mixes two different things and can even go +// negative. It is a smell test, not a measurement. To size native usage, +// measure RSS against a build with the store disabled, or profile. +// +// What is bounded is SQLite's page cache specifically: both DSNs pin +// _cache_size=-2000, i.e. ~2 MiB per connection, so roughly 8 MiB across +// SetMaxOpenConns(4) here and 2 MiB in the ingestor. That is a cap on the page +// cache, not on everything SQLite allocates — statement and schema memory sit +// outside it. See memlimit.go for why any of it matters to GOMEMLIMIT. type MemorySnapshot struct { ProcessRSSMB float64 `json:"processRSSMB"` GoHeapInuseMB float64 `json:"goHeapInuseMB"` diff --git a/cmd/server/multibyte_capability_test.go b/cmd/server/multibyte_capability_test.go index 1325510a..53267533 100644 --- a/cmd/server/multibyte_capability_test.go +++ b/cmd/server/multibyte_capability_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" ) // recentTS returns a timestamp string N hours ago, ensuring test data @@ -20,7 +20,7 @@ func recentTS(hoursAgo int) string { // setupCapabilityTestDB creates a minimal in-memory DB with nodes table. func setupCapabilityTestDB(t *testing.T) *DB { t.Helper() - conn, err := sql.Open("sqlite", ":memory:") + conn, err := sql.Open("sqlite3", ":memory:") if err != nil { t.Fatal(err) } @@ -446,7 +446,7 @@ func TestMultiByteCapability_AdopterEvidenceTakesPrecedence(t *testing.T) { // setupPersistTestDB creates an in-memory DB with multibyte_sup/multibyte_evidence columns. func setupPersistTestDB(t *testing.T) *DB { t.Helper() - conn, err := sql.Open("sqlite", ":memory:") + conn, err := sql.Open("sqlite3", ":memory:") if err != nil { t.Fatal(err) } diff --git a/cmd/server/neighbor_api_test.go b/cmd/server/neighbor_api_test.go index 4a146eaf..e3ab0b96 100644 --- a/cmd/server/neighbor_api_test.go +++ b/cmd/server/neighbor_api_test.go @@ -9,7 +9,7 @@ import ( "time" "github.com/gorilla/mux" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" ) // ─── Helpers ─────────────────────────────────────────────────────────────────── @@ -467,7 +467,7 @@ func TestBuildNodeInfoMap_ObserverEnrichment(t *testing.T) { tmpDir := t.TempDir() dbPath := tmpDir + "/test.db" - conn, err := sql.Open("sqlite", dbPath) + conn, err := sql.Open("sqlite3", dbPath) if err != nil { t.Fatal(err) } @@ -476,15 +476,16 @@ func TestBuildNodeInfoMap_ObserverEnrichment(t *testing.T) { // Create tables for _, stmt := range []string{ "CREATE TABLE nodes (public_key TEXT, name TEXT, role TEXT, lat REAL, lon REAL)", - "CREATE TABLE observers (id TEXT, name TEXT, iata TEXT)", + "CREATE TABLE observers (id TEXT, name TEXT, iata TEXT, inactive INTEGER)", "INSERT INTO nodes VALUES ('AAAA1111', 'Repeater-1', 'repeater', 0, 0)", - "INSERT INTO observers VALUES ('BBBB2222', 'Observer-Alpha', '')", - "INSERT INTO observers VALUES ('AAAA1111', 'Obs-also-repeater', '')", + "INSERT INTO observers (id, name, iata) VALUES ('BBBB2222', 'Observer-Alpha', '')", + "INSERT INTO observers (id, name, iata) VALUES ('AAAA1111', 'Obs-also-repeater', '')", } { if _, err := conn.Exec(stmt); err != nil { t.Fatalf("exec %q: %v", stmt, err) } } + ensurePreparable(t, conn) conn.Close() // Open via our DB wrapper @@ -542,20 +543,21 @@ func TestBuildNodeInfoMap_FirstSeenIsCached(t *testing.T) { dbPath := tmpDir + "/test.db" // Seed via rw connection. - rw, err := sql.Open("sqlite", dbPath) + rw, err := sql.Open("sqlite3", dbPath) if err != nil { t.Fatal(err) } defer rw.Close() for _, stmt := range []string{ "CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, advert_count INTEGER)", - "CREATE TABLE observers (id TEXT, name TEXT, iata TEXT)", + "CREATE TABLE observers (id TEXT, name TEXT, iata TEXT, inactive INTEGER)", "INSERT INTO nodes VALUES ('AAAA1111', 'Repeater-1', 'repeater', 0, 0, '', '2024-01-01T00:00:00Z', 0)", } { if _, err := rw.Exec(stmt); err != nil { t.Fatalf("seed exec %q: %v", stmt, err) } } + ensurePreparable(t, rw) db, err := OpenDB(dbPath) if err != nil { @@ -608,20 +610,21 @@ func TestGetAllNodes_FirstSeenSchemaFallback(t *testing.T) { dbPath := tmpDir + "/test.db" // Seed a nodes table WITHOUT first_seen (advert_count + last_seen present). - rw, err := sql.Open("sqlite", dbPath) + rw, err := sql.Open("sqlite3", dbPath) if err != nil { t.Fatal(err) } defer rw.Close() for _, stmt := range []string{ "CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, advert_count INTEGER)", - "CREATE TABLE observers (id TEXT, name TEXT, iata TEXT)", + "CREATE TABLE observers (id TEXT, name TEXT, iata TEXT, inactive INTEGER)", "INSERT INTO nodes VALUES ('BBBB2222', 'Repeater-2', 'repeater', 0, 0, '2024-02-02T00:00:00Z', 3)", } { if _, err := rw.Exec(stmt); err != nil { t.Fatalf("seed exec %q: %v", stmt, err) } } + ensurePreparable(t, rw) db, err := OpenDB(dbPath) if err != nil { diff --git a/cmd/server/neighbor_persist_legacy_test.go b/cmd/server/neighbor_persist_legacy_test.go index efb8d6dd..798b96e4 100644 --- a/cmd/server/neighbor_persist_legacy_test.go +++ b/cmd/server/neighbor_persist_legacy_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" ) // TestNeighborPersist_LegacyEdgeInvariant (#1638 adv-#1): edges loaded from @@ -18,7 +18,7 @@ import ( func TestNeighborPersist_LegacyEdgeInvariant(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "neighbor_legacy.db") - rw, err := sql.Open("sqlite", "file:"+dbPath+"?_journal_mode=WAL") + rw, err := sql.Open("sqlite3", "file:"+dbPath+"?_journal_mode=WAL") if err != nil { t.Fatal(err) } @@ -79,7 +79,7 @@ func TestNeighborPersist_LegacyEdgeInvariant(t *testing.T) { func TestNeighborPersist_LegacyEdgeMergeOnReload(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "neighbor_legacy_merge.db") - rw, err := sql.Open("sqlite", "file:"+dbPath+"?_journal_mode=WAL") + rw, err := sql.Open("sqlite3", "file:"+dbPath+"?_journal_mode=WAL") if err != nil { t.Fatal(err) } diff --git a/cmd/server/neighbor_recomputer_test.go b/cmd/server/neighbor_recomputer_test.go index f78770a4..cdcf7b8f 100644 --- a/cmd/server/neighbor_recomputer_test.go +++ b/cmd/server/neighbor_recomputer_test.go @@ -6,8 +6,8 @@ import ( "testing" "time" + _ "github.com/mattn/go-sqlite3" "github.com/meshcore-analyzer/dbschema" - _ "modernc.org/sqlite" ) // TestNeighborGraphRecomputerLoadsSnapshot enforces #1287 Option 4: @@ -19,7 +19,7 @@ func TestNeighborGraphRecomputerLoadsSnapshot(t *testing.T) { dbPath := filepath.Join(dir, "neighbor_recomp.db") // Bootstrap a WAL DB with the neighbor_edges table. - rw, err := sql.Open("sqlite", "file:"+dbPath+"?_journal_mode=WAL") + rw, err := sql.Open("sqlite3", "file:"+dbPath+"?_journal_mode=WAL") if err != nil { t.Fatal(err) } @@ -33,6 +33,7 @@ func TestNeighborGraphRecomputerLoadsSnapshot(t *testing.T) { )`); err != nil { t.Fatal(err) } + ensurePreparable(t, rw) // Stage one edge. now := time.Now().UTC().Format(time.RFC3339) @@ -85,9 +86,15 @@ func TestServerStartupRequiresMigratedSchema(t *testing.T) { // Bootstrap with ONLY transmissions/observations (the things // server tries to read) but WITHOUT the columns dbschema asserts - // (resolved_path, inactive, last_packet_at, iata, foreign_advert, - // from_pubkey, neighbor_edges). - rw, err := sql.Open("sqlite", "file:"+dbPath+"?_journal_mode=WAL") + // (resolved_path, last_packet_at, iata, foreign_advert, from_pubkey, + // neighbor_edges). + // + // observers.inactive used to be in that list. It no longer is: OpenDB + // prepares statements eagerly under github.com/mattn/go-sqlite3 and one of + // them selects on inactive, so a fixture without it cannot reach + // AssertReady at all. The other missing surfaces above still make + // AssertReady fail, which is what this test asserts. + rw, err := sql.Open("sqlite3", "file:"+dbPath+"?_journal_mode=WAL") if err != nil { t.Fatal(err) } @@ -95,7 +102,7 @@ func TestServerStartupRequiresMigratedSchema(t *testing.T) { for _, s := range []string{ `CREATE TABLE transmissions (id INTEGER PRIMARY KEY, hash TEXT, payload_type INTEGER)`, `CREATE TABLE observations (id INTEGER PRIMARY KEY, transmission_id INTEGER)`, - `CREATE TABLE observers (id TEXT PRIMARY KEY, name TEXT)`, + `CREATE TABLE observers (id TEXT PRIMARY KEY, name TEXT, inactive INTEGER)`, `CREATE TABLE nodes (public_key TEXT PRIMARY KEY)`, `CREATE TABLE inactive_nodes (public_key TEXT PRIMARY KEY)`, } { @@ -104,6 +111,8 @@ func TestServerStartupRequiresMigratedSchema(t *testing.T) { } } + ensurePreparable(t, rw) + // Open the read-only server handle and call AssertReady directly // (production path: main.go does this before any business logic). d, err := OpenDB(dbPath) diff --git a/cmd/server/node_reach_bench_test.go b/cmd/server/node_reach_bench_test.go index 4159c80e..bbf24a34 100644 --- a/cmd/server/node_reach_bench_test.go +++ b/cmd/server/node_reach_bench_test.go @@ -6,7 +6,7 @@ import ( "fmt" "testing" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" ) // benchReachDB builds an in-memory DB with nObs observations. matchEvery @@ -24,13 +24,13 @@ func benchReachDB(b *testing.B, nObs, matchEvery int, lowerHops bool) *DB { // path_json is uppercase; this measures the worst case Carmack flagged). matchPath, fillerPath = `["aa","01fa","bb"]`, `["aa","cc","bb"]` } - conn, err := sql.Open("sqlite", ":memory:") + conn, err := sql.Open("sqlite3", ":memory:") if err != nil { b.Fatal(err) } schema := []string{ `CREATE TABLE transmissions (id INTEGER PRIMARY KEY, hash TEXT, first_seen TEXT, payload_type INTEGER, from_pubkey TEXT)`, - `CREATE TABLE observers (id TEXT PRIMARY KEY, name TEXT)`, + `CREATE TABLE observers (id TEXT PRIMARY KEY, name TEXT, inactive INTEGER)`, `CREATE TABLE observations (id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_idx INTEGER, snr REAL, path_json TEXT, timestamp INTEGER)`, `CREATE INDEX idx_obs_ts ON observations(timestamp)`, } @@ -155,12 +155,12 @@ func BenchmarkNodeReachAttribute(b *testing.B) { // must surface an error, not a swallowed nil. Lives in this file because // the bench callers in the same file rely on the same signature. func TestScanReachRows_ErrorReturn(t *testing.T) { - conn, err := sql.Open("sqlite", ":memory:") + conn, err := sql.Open("sqlite3", ":memory:") if err != nil { t.Fatalf("open: %v", err) } // PREFLIGHT: async=true reason="test-only in-memory scratch schema, immediately closed" - if _, err := conn.Exec(`CREATE TABLE observations (id INTEGER); CREATE TABLE transmissions (id INTEGER); CREATE TABLE observers (rowid INTEGER, id TEXT)`); err != nil { + if _, err := conn.Exec(`CREATE TABLE observations (id INTEGER); CREATE TABLE transmissions (id INTEGER); CREATE TABLE observers (rowid INTEGER, id TEXT, inactive INTEGER)`); err != nil { t.Fatalf("schema: %v", err) } conn.Close() // force QueryContext to fail diff --git a/cmd/server/node_reach_endpoint_test.go b/cmd/server/node_reach_endpoint_test.go index 6094077b..a7fc6d4a 100644 --- a/cmd/server/node_reach_endpoint_test.go +++ b/cmd/server/node_reach_endpoint_test.go @@ -11,7 +11,7 @@ import ( "time" "github.com/gorilla/mux" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" ) func serveReach(srv *Server, path string) *httptest.ResponseRecorder { @@ -57,7 +57,7 @@ func resetReachState(t *testing.T, servers ...*Server) { // build the zero-reach case (identifiable node, no matching observations). func newReachIntegrationDB(t *testing.T, obsPath string) (*DB, string) { t.Helper() - conn, err := sql.Open("sqlite", ":memory:") + conn, err := sql.Open("sqlite3", ":memory:") if err != nil { t.Fatal(err) } @@ -68,7 +68,7 @@ func newReachIntegrationDB(t *testing.T, obsPath string) (*DB, string) { stmts := []string{ `CREATE TABLE nodes (public_key TEXT, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, advert_count INTEGER)`, `CREATE TABLE transmissions (id INTEGER PRIMARY KEY, from_pubkey TEXT, payload_type INTEGER)`, - `CREATE TABLE observers (id TEXT)`, + `CREATE TABLE observers (id TEXT, inactive INTEGER)`, `CREATE TABLE observations (id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_idx INTEGER, snr REAL, path_json TEXT, timestamp INTEGER)`, `CREATE TABLE neighbor_edges (node_a TEXT, node_b TEXT, count INTEGER)`, } diff --git a/cmd/server/node_reach_test.go b/cmd/server/node_reach_test.go index 606f8e2c..55938919 100644 --- a/cmd/server/node_reach_test.go +++ b/cmd/server/node_reach_test.go @@ -6,20 +6,20 @@ import ( "strconv" "testing" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" ) // newReachScanTestDB builds a minimal observer_idx-schema DB with two rows whose // path contains "01FA" and one that does not, for scanReachRows coverage. func newReachScanTestDB(t *testing.T) *DB { t.Helper() - conn, err := sql.Open("sqlite", ":memory:") + conn, err := sql.Open("sqlite3", ":memory:") if err != nil { t.Fatal(err) } stmts := []string{ `CREATE TABLE transmissions (id INTEGER PRIMARY KEY, from_pubkey TEXT, payload_type INTEGER)`, - `CREATE TABLE observers (id TEXT)`, + `CREATE TABLE observers (id TEXT, inactive INTEGER)`, `CREATE TABLE observations (id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_idx INTEGER, snr REAL, path_json TEXT, timestamp INTEGER)`, `INSERT INTO observers (id) VALUES ('OBS1')`, // rowid 1 `INSERT INTO transmissions (id, from_pubkey, payload_type) VALUES (1,'FF00',4),(2,'',5),(3,'',5)`, diff --git a/cmd/server/perf_io.go b/cmd/server/perf_io.go index 53415e39..dcf1372e 100644 --- a/cmd/server/perf_io.go +++ b/cmd/server/perf_io.go @@ -260,7 +260,7 @@ func (s *Server) handlePerfSqlite(w http.ResponseWriter, r *http.Request) { resp.CacheSize = cacheSize // Cache hit rate: derived from PacketStore cache (rw_cache). We don't - // have a direct SQLite cache counter via the modernc driver, so we + // have a direct SQLite cache counter through the driver, so we // surface the closest available proxy — the in-process row cache. if s.store != nil { cs := s.store.GetCacheStatsTyped() diff --git a/cmd/server/preparable_schema_test.go b/cmd/server/preparable_schema_test.go new file mode 100644 index 00000000..0540091d --- /dev/null +++ b/cmd/server/preparable_schema_test.go @@ -0,0 +1,77 @@ +package main + +import ( + "database/sql" + "strings" + "testing" +) + +// The statements compiled by prepareStatements reference a set of tables and +// columns. Under modernc.org/sqlite that never mattered for test fixtures: +// newStmt only stored the SQL string and compiled it lazily on first use, so a +// fixture could omit half the schema and still open. github.com/mattn/go-sqlite3 +// calls sqlite3_prepare_v2 inside Prepare, so OpenDB now fails on an incomplete +// schema — which is the production behaviour we want (#1901), but it means +// fixtures have to declare what they are prepared against. +// +// ensurePreparable is the seam. Fixtures that deliberately build a partial +// schema (to exercise schema detection, say) call it on the writable connection +// just before OpenDB, and it fills in whatever is missing without disturbing the +// shapes the test actually cares about. +var preparableTables = []string{ + `CREATE TABLE IF NOT EXISTS transmissions (id INTEGER PRIMARY KEY, hash TEXT)`, + `CREATE TABLE IF NOT EXISTS observations (id INTEGER PRIMARY KEY, timestamp TEXT)`, + `CREATE TABLE IF NOT EXISTS nodes (public_key TEXT PRIMARY KEY)`, + `CREATE TABLE IF NOT EXISTS observers (id TEXT)`, +} + +// preparableColumns are added when absent. A fixture that already declared the +// table keeps its own definition, so only the missing columns get appended. +var preparableColumns = []struct{ table, column, decl string }{ + {"transmissions", "hash", "TEXT"}, + {"observations", "timestamp", "TEXT"}, + {"nodes", "name", "TEXT"}, + {"nodes", "role", "TEXT"}, + {"nodes", "last_seen", "TEXT"}, + {"observers", "inactive", "INTEGER"}, +} + +// ensurePreparable makes conn's schema sufficient for prepareStatements. +// It is idempotent and never overwrites an existing table or column. +func ensurePreparable(tb testing.TB, conn *sql.DB) { + tb.Helper() + for _, ddl := range preparableTables { + if _, err := conn.Exec(ddl); err != nil { + tb.Fatalf("ensurePreparable: %v\nSQL: %s", err, ddl) + } + } + for _, c := range preparableColumns { + _, err := conn.Exec("ALTER TABLE " + c.table + " ADD COLUMN " + c.column + " " + c.decl) + // "duplicate column name" means the fixture already declared it. + if err != nil && !strings.Contains(err.Error(), "duplicate column name") { + tb.Fatalf("ensurePreparable: add %s.%s: %v", c.table, c.column, err) + } + } +} + +// TestEnsurePreparableMatchesPrepareStatements keeps the helper honest. If a new +// prepared statement references a table or column ensurePreparable does not +// create, it fails here rather than as a confusing OpenDB error in whichever +// fixture happens to be thinnest. +func TestEnsurePreparableMatchesPrepareStatements(t *testing.T) { + conn, err := sql.Open("sqlite3", ":memory:") + if err != nil { + t.Fatal(err) + } + defer conn.Close() + // prepareStatements compiles against a pooled connection; pin the pool so + // the in-memory database the DDL landed in is the one it prepares against. + conn.SetMaxOpenConns(1) + ensurePreparable(t, conn) + + db := &DB{conn: conn} + if err := db.prepareStatements(); err != nil { + t.Fatalf("prepareStatements against the minimal preparable schema: %v\n"+ + "Add the missing table/column to preparableTables or preparableColumns.", err) + } +} diff --git a/cmd/server/readonly_invariant_test.go b/cmd/server/readonly_invariant_test.go index f97baa48..0f671a2a 100644 --- a/cmd/server/readonly_invariant_test.go +++ b/cmd/server/readonly_invariant_test.go @@ -10,7 +10,7 @@ import ( "strings" "testing" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" ) // TestServerSourceHasNoCachedRWCalls enforces issue #1287: after the @@ -129,7 +129,7 @@ func TestServerDBConnIsReadOnly(t *testing.T) { // Bootstrap a minimal DB with the ingestor-style WAL opener so the // server can attach in read-only mode. - if err := bootstrapMinimalDB(path); err != nil { + if err := bootstrapMinimalDB(t, path); err != nil { t.Fatalf("bootstrap: %v", err) } @@ -149,9 +149,9 @@ func TestServerDBConnIsReadOnly(t *testing.T) { // need, opened with WAL so the read-only opener in OpenDB can attach. // Kept in *_test.go so it does NOT add any write capability to the // production server binary. -func bootstrapMinimalDB(path string) error { +func bootstrapMinimalDB(tb testing.TB, path string) error { dsn := fmt.Sprintf("file:%s?_journal_mode=WAL&_busy_timeout=5000", path) - rw, err := sql.Open("sqlite", dsn) + rw, err := sql.Open("sqlite3", dsn) if err != nil { return err } @@ -159,6 +159,9 @@ func bootstrapMinimalDB(path string) error { if _, err := rw.Exec(`CREATE TABLE IF NOT EXISTS nodes (public_key TEXT PRIMARY KEY, name TEXT)`); err != nil { return err } + // prepareStatements compiles eagerly under mattn; give it the rest of the + // surface it references so OpenDB gets far enough to test read-onlyness. + ensurePreparable(tb, rw) return nil } @@ -211,3 +214,37 @@ func nodeTableWritePattern(verb, trailer string) *regexp.Regexp { } return regexp.MustCompile(expr) } + +// TestOpenDBRefusesMissingDatabase pins the behaviour that makes mode=ro +// meaningful: OpenDB must fail on a path that does not exist rather than +// creating an empty database there. +// +// This is worth a test of its own because the guarantee is not obvious from the +// call site. github.com/mattn/go-sqlite3 always passes +// SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE to sqlite3_open_v2; what restricts it +// is the mode=ro parameter in the URI, which only takes effect because mattn's +// C wrapper ORs SQLITE_OPEN_URI into the flags (sqlite3.go _sqlite3_open_v2). +// Lose the file: prefix, or the URI flag, and this silently degrades into a +// read-write open that manufactures a fresh empty database — which the server +// would then happily serve. cmd/decrypt shipped exactly that bug for a while by +// building its DSN without the file: prefix. +func TestOpenDBRefusesMissingDatabase(t *testing.T) { + dir := t.TempDir() + missing := filepath.Join(dir, "absent.db") + + db, err := OpenDB(missing) + if err == nil { + db.Close() + t.Fatal("OpenDB succeeded against a nonexistent database; mode=ro is not in effect") + } + + // Neither the database itself nor a file literally named after the DSN + // (what happens when URI handling is off) may have been created. + entries, rerr := os.ReadDir(dir) + if rerr != nil { + t.Fatalf("read temp dir: %v", rerr) + } + for _, e := range entries { + t.Errorf("OpenDB created %q; a read-only open must not write anything", e.Name()) + } +} diff --git a/cmd/server/resolved_index_test.go b/cmd/server/resolved_index_test.go index 30f2669d..90c2bfb8 100644 --- a/cmd/server/resolved_index_test.go +++ b/cmd/server/resolved_index_test.go @@ -10,7 +10,7 @@ import ( "testing" "time" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" ) // --- Unit tests --- @@ -1096,7 +1096,7 @@ func TestLRU_CapacityAfterBulkDelete(t *testing.T) { // TestConfirmResolvedPathContains_SpecialChars verifies that pubkeys containing // SQL LIKE wildcards (%, _) don't cause false positives with the INSTR approach. func TestConfirmResolvedPathContains_SpecialChars(t *testing.T) { - db, err := sql.Open("sqlite", ":memory:") + db, err := sql.Open("sqlite3", ":memory:") if err != nil { t.Fatal(err) } diff --git a/cmd/server/scope_audit_test.go b/cmd/server/scope_audit_test.go index 3ebd811b..83ae7212 100644 --- a/cmd/server/scope_audit_test.go +++ b/cmd/server/scope_audit_test.go @@ -31,7 +31,7 @@ var ( // fiction, so the join behaves the way it does against a real database. func setupScopeConformanceDB(t *testing.T) *DB { t.Helper() - conn, err := sql.Open("sqlite", ":memory:") + conn, err := sql.Open("sqlite3", ":memory:") if err != nil { t.Fatal(err) } diff --git a/cmd/server/store_memory_diagnostics_test.go b/cmd/server/store_memory_diagnostics_test.go index 989a76cc..e1253d2b 100644 --- a/cmd/server/store_memory_diagnostics_test.go +++ b/cmd/server/store_memory_diagnostics_test.go @@ -95,7 +95,7 @@ func TestObsRawHexNotRetainedOnLoad(t *testing.T) { const txHex = "deadbeefcafe" const obsHex = "c0ffee0102" // distinct from txHex: proves we DON'T keep it - conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL") + conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") if err != nil { t.Fatal(err) } @@ -109,9 +109,9 @@ func TestObsRawHexNotRetainedOnLoad(t *testing.T) { observer_name TEXT, direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT )`, - `CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`, + `CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE nodes ( - pubkey TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, + public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, frequency REAL )`, `CREATE INDEX idx_tx_first_seen ON transmissions(first_seen)`, diff --git a/cmd/server/topology_dedup_test.go b/cmd/server/topology_dedup_test.go index eac7036c..b875eb01 100644 --- a/cmd/server/topology_dedup_test.go +++ b/cmd/server/topology_dedup_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" ) // TestTopologyDedup_RepeatersMergeByPubkey verifies that topRepeaters @@ -15,7 +15,7 @@ import ( func TestTopologyDedup_RepeatersMergeByPubkey(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL") + conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") if err != nil { t.Fatal(err) } @@ -34,7 +34,7 @@ func TestTopologyDedup_RepeatersMergeByPubkey(t *testing.T) { id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT )`) - exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`) + exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) exec(`CREATE TABLE nodes ( public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, frequency REAL @@ -139,7 +139,7 @@ func TestTopologyDedup_RepeatersMergeByPubkey(t *testing.T) { func TestTopologyDedup_AmbiguousPrefixNotMerged(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL") + conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") if err != nil { t.Fatal(err) } @@ -158,7 +158,7 @@ func TestTopologyDedup_AmbiguousPrefixNotMerged(t *testing.T) { id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT )`) - exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`) + exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) exec(`CREATE TABLE nodes ( public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, frequency REAL @@ -245,7 +245,7 @@ func TestTopologyDedup_AmbiguousPrefixNotMerged(t *testing.T) { func TestTopologyDedup_PairsMergeByPubkey(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL") + conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") if err != nil { t.Fatal(err) } @@ -264,7 +264,7 @@ func TestTopologyDedup_PairsMergeByPubkey(t *testing.T) { id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT )`) - exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`) + exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) exec(`CREATE TABLE nodes ( public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, frequency REAL diff --git a/docs/client-rx-coverage.md b/docs/client-rx-coverage.md index cc8d96aa..61191a24 100644 --- a/docs/client-rx-coverage.md +++ b/docs/client-rx-coverage.md @@ -203,7 +203,8 @@ server-side (read-only). Each feature: ``` - Hex binning is a pure-Go pointy-top grid over Web Mercator (`cmd/server/hexgrid.go`). We do **not** - use `uber/h3-go` because it is CGO and the project builds with `CGO_ENABLED=0`. Latitude is only + use `uber/h3-go`, which would add a C dependency for no benefit (the SQLite driver + is cgo since the mattn/go-sqlite3 move, but that is a library we need). Latitude is only defined within ±85.05° (Web Mercator limit) and is clamped to that range. - `z` (Leaflet zoom) selects the hex resolution (zoom-adaptive). Raw points never leave the server (privacy: contributors' tracks are not exposed). diff --git a/docs/sqlite-driver-migration.md b/docs/sqlite-driver-migration.md new file mode 100644 index 00000000..f7245164 --- /dev/null +++ b/docs/sqlite-driver-migration.md @@ -0,0 +1,245 @@ +# SQLite driver: modernc.org/sqlite → github.com/mattn/go-sqlite3 + +The database driver changed from `modernc.org/sqlite v1.34.5` (pure Go, SQLite +3.46.0) to `github.com/mattn/go-sqlite3 v1.14.52` (cgo, bundled SQLite 3.53.4). + +This is the record of why, what it measured, and the five behavioural +differences that had to be handled — several of which fail silently if you get +them wrong, so read this before touching the DSNs or the build. + +## Why + +corescope is read-heavy: the server chunk-loads a graph at startup and fans out +neighbor/topology/analytics queries per request. modernc's pure-Go SQLite is a +transpilation of the C amalgamation and pays for it on exactly those paths. + +## What it measured + +Head-to-head on the same 120k-transmission / 240k-observation database, running +corescope's own hot-path SQL under both drivers (Apple M4, `-count=5`, medians): + +| Workload | modernc | mattn | Change | +|---|---:|---:|---| +| Chunk load (the `cmd/server/chunked_load.go` v3 join, 20k transmissions) | 449 ms | 196 ms | **2.3× faster** | +| Aggregate scan (240k-row join + `GROUP BY`, stands in for analytics) | 276 ms | 137 ms | **2.0× faster** | +| 1500 prepared-statement lookups (`prepareStatements` round trips) | 512 ms | 403 ms | **1.3× faster** | + +Allocation counts drop with it: 1.12M vs 1.64M allocs (−32%) and 21 MB vs 30 MB +(−30%) on the chunk load, 20.3k vs 27.3k on the lookups. + +## Confirmed on production + +The table above comes from a standalone harness. @efiten then ran both drivers +against a real instance — 11,077,038 observations, 9.7 GB database, 4-core arm64 +— as server-only containers against the same live volume, one at a time. Round 2 +reverses the order so the page-cache advantage goes to the old driver: + +| | audit 7d | audit 24h | background fill (13 chunks) | start → /api/health | +|---|---:|---:|---:|---:| +| modernc, round 1 | 16.67s | 2.27s | 130.2s | 16.6s | +| mattn, round 1 | 7.87s | 1.34s | 93.8s | 13.5s | +| mattn, round 2 | 8.15s | 1.35s | 96.4s | 13.0s | +| modernc, round 2 | 13.46s | 2.29s | 137.8s | 15.5s | + +Warm, the old driver improves to 13.46s on the 7d audit and still loses by +~1.8×. Chunk load is ~1.4×. `/api/nodes?limit=500` is 0.039s against 0.037s — +i.e. nothing. + +**Real-world gains are smaller than the harness suggests: ~1.4-1.8× on the +paths that matter, not 2-2.3×.** The shape holds — scans and joins gain, small +lookups do not — but quote these numbers, not the harness ones. + +**Build time is the counterweight**, cold and native on that machine: +**52s on master, 163s on this branch**. An instance that builds its own image +pays that on every deploy. + +## The cost: the build is cgo now + +`CGO_ENABLED=0` still *builds*, which is the trap: mattn links a stub, and the +binary dies on its first query with `go-sqlite3 requires cgo to work. This is a +stub`. A green build proves nothing here. `GOOS=linux go build` from a Mac, on +the other hand, genuinely cannot cross-compile any more — cgo needs a C compiler +that can target the other platform. That compiler is +[`zig`](https://ziglang.org/download/): + +```bash +make build # host +make crossbuild # static linux/amd64 + linux/arm64 via `zig cc -target …-linux-musl` +``` + +Targeting musl and linking with `-extldflags "-static -Wl,-s"` keeps the output a +single self-contained binary, so the `alpine:3.20` runtime image no longer +depends on the base image's libc at all. `-Wl,-s` matters: Go's own `-s -w` does +not reach the musl objects zig links in, and without it the server binary is +19.8 MB instead of 12.1 MB. + +The `Dockerfile` builder stage installs a checksum-pinned zig and does the same +thing, still on a single `$BUILDPLATFORM` builder with no QEMU for compilation. +Build-cache mounts are not optional there: compiling the SQLite amalgamation +twice from cold takes over half an hour. + +## The five behavioural differences + +### 1. Statement preparation is eager + +modernc's `newStmt` only stored the SQL and compiled lazily on first use; mattn +calls `sqlite3_prepare_v2` inside `Prepare`. SQL referencing a missing table or +column now fails at **open** time. + +This is the migration's largest single effect: 59 server tests failed on it, +purely from fixtures with partial schemas. `OpenDB` keeps failing loudly (that is +the #1901 behaviour we want in production, and `cmd/server/main.go` also gates on +`dbschema.AssertReady`); the fixtures instead declare what they are prepared +against, via `ensurePreparable` in `cmd/server/preparable_schema_test.go`. If you +add a prepared statement referencing something new, +`TestEnsurePreparableMatchesPrepareStatements` tells you to extend that helper. + +It also surfaced nine `nodes(pubkey …)` declarations across seven files, when +production has only ever had `public_key`. Lazy compilation had hidden the +mismatch. + +### 2. It exposed a real bug in the observation UPSERT + +`stmtInsertObservation` resolves `ON CONFLICT(transmission_id, observer_idx, +COALESCE(path_json, ''))` against the unique expression index +`idx_observations_dedup` — which `cmd/ingestor/db.go` only ever created inside +the branch that creates the `observations` table for the first time. Databases +whose table predates that branch never had one, so the UPSERT had no conflict +target. modernc failed on the first insert; mattn fails at `OpenStore`. Same bug, +found earlier. + +`internal/dbschema` now creates it unconditionally. Because the index is what was +supposed to prevent duplicates, a database that never had it can already hold +rows violating it — the repo's own `test-fixtures/e2e-fixture.db` held one — so +duplicates are collapsed first. Refusing would not have been safer: without the +index the ingestor cannot prepare its UPSERT, so it cannot start at all. + +The collapse has to *replay* the UPSERT, and getting that subtly wrong is easy. +`DO UPDATE SET snr = COALESCE(excluded.snr, snr)` means the **incoming** value +wins when it is non-NULL, so down a group in id order the survivor ends up with +the **last** non-NULL value, not the first. It also names exactly five columns — +`snr`, `rssi`, `score`, `raw_hex`, `resolved_path` — so every other column keeps +the surviving row's own value; merging those too would invent history the +ingestor would never have written. An earlier version of this change took the +first non-NULL value and merged every column, which silently discarded newer +readings. + +Merge, delete and index creation all share one transaction. Split apart, a +writer inserting a duplicate in the gap makes the index creation fail while the +deletions stay committed — rows destroyed and no index to show for it. + +The repair logs what it is about to destroy — group count, rows to remove, and +the first 20 group keys with the id it keeps — before deleting anything, because +a bare row count is not enough to reconstruct from if the merge direction is +ever wrong again. It also says up front that it holds the write lock, since on a +large table that pause at ingestor startup is otherwise unexplained. + +Two hazards worth knowing before an upgrade: + +- **Reads inside the repair must use the transaction, not the pool.** + `cmd/ingestor` runs `SetMaxOpenConns(1)`, so a query issued against the pool + while the repair holds its transaction waits for a connection that + transaction has checked out, forever. It deadlocked a staging ingestor at + boot — logs stop after "Repairing now", the write lock is free, the process + is simply blocked. Anything reading in there takes a `Querier` and is passed + `tx`. +- **NULL is not a duplicate.** `GROUP BY` folds NULLs into one group; a UNIQUE + index treats them as distinct, so a row with a NULL in an indexed column can + never violate it. Grouping without excluding them does not delete the rows — + the `DELETE` joins on `observer_idx = observer_idx`, which NULL never + satisfies — it is the *merge* that does the damage, matching nothing and + writing NULL over the survivor's real readings. The rows stay put and their + measurements disappear. On an 11.2M-row instance 198 of 222 reported groups + were `observer_idx IS NULL`. + +The failing `CREATE UNIQUE INDEX` that triggers all this is itself a stall: on +11.2M rows it held the write lock long enough for a concurrent writer to hit the +full 5s `busy_timeout`. So an instance that needs the repair pauses writers for +seconds *before* the repair starts. + +The repair itself does not hold the write lock throughout, tempting as that is +to assume. The grouping scan (~18s at that size) reads, and a concurrent writer +can proceed during it; the lock is taken when the repair first writes. A writer +that gets in between makes the repair fail and roll back, which is the intended +outcome — but "holds the write lock until it completes" is the wrong mental +model. + +The collapse itself is measured at 4.1s on 2.4M synthetic rows holding 5 +duplicates. That is well short of a real deployment: an 11M-row instance has not +been measured to completion, and the collapse has not been run against a +database an ingestor is actively writing to. + +Note `COALESCE(path_json, '')` makes a NULL path and an empty-string path the +same key, but `NULL` and `'[]'` different keys. See +`internal/dbschema/dedup_index_test.go`, where the last-wins and atomicity +properties each have a test. + +### 3. `synchronous` silently dropped from FULL to NORMAL + +mattn defaults `synchronousMode` to `NORMAL` and runs `PRAGMA synchronous = +NORMAL` unconditionally, where SQLite's own compile default (what modernc left in +place) is FULL. In WAL mode that changes durability under power loss. + +The writer DSN pins `_synchronous=FULL`, and lives in one place — +`dbschema.WriterDSN` — because there are two writers. `cmd/migrate` originally +kept a bare path and so silently wrote at NORMAL, which is exactly what a second +copy of a DSN buys you. `TestOpenStorePragmas` reads every pragma back +**through the store's own connection**, and `TestWriterDSNPragmas` covers the DSN +itself; a separate `sqlite3` session or the startup log line would prove +nothing. + +### 4. The DSN dialect is different, and each driver ignores the other's + +modernc understood only `_pragma=name(value)`; mattn understands only +`_`-prefixed parameters. Neither errors on the other's form, so a driver-only +rename would have dropped every pragma on the floor in silence. All five +`_pragma=` DSNs were rewritten (one production, four test seeds). + +Also removed: `_journal_mode=WAL` on the server's read handle. modernc had been +ignoring it all along; mattn honours it, and setting `journal_mode` on a +read-only connection is a write. Dropping `_busy_timeout` with it costs nothing — +mattn's default is already 5000 ms, so the read handle finally *gets* the busy +timeout it had silently lacked. + +### 5. `mode=ro` still works — but not for the reason you would guess + +mattn always passes `SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE` to +`sqlite3_open_v2`, and the bundled amalgamation has `SQLITE_USE_URI=0`. What +makes `file:…?mode=ro` work anyway is that mattn's C wrapper `_sqlite3_open_v2` +ORs `SQLITE_OPEN_URI` into the flags itself. So the read-only invariant from +#1283/#1289 holds with no build flags — but it depends on the `file:` prefix +being present. `TestOpenDBRefusesMissingDatabase` fails if any of that stops +holding. + +`cmd/decrypt` had been building its DSN *without* the `file:` prefix, so both +drivers stripped the query string and its `mode=ro` had never applied — a missing +path was created read-write. Fixed in passing; it was never a migration +regression. + +## Memory + +`runtime/debug.SetMemoryLimit` (GOMEMLIMIT) covers what the Go runtime manages — +heap, stacks, runtime structures — and nothing else, so it is not an RSS ceiling +now that SQLite allocates in C. + +What is bounded is the page cache specifically: both DSNs pin +`_cache_size=-2000`, i.e. ~2 MiB per connection, so ~8 MiB across the server's +`SetMaxOpenConns(4)` and ~2 MiB in the ingestor, comfortably inside the 1.5× +headroom `applyMemoryLimit` derives. That caps the page cache, not everything +SQLite allocates — statement and schema memory sit outside it — so revisit +against measured RSS if the connection count or `_cache_size` grows, or if a +workload starts holding many prepared statements. + +There is no cgo-bytes metric because Go exposes no counter for one. Do not read +`processRSSMB - goSysMB` as the C share either: `goSysMB` is reserved address +space rather than resident memory, so the subtraction mixes two different +quantities. It is a smell test, not a measurement. + +## Things that did not change + +No modernc-specific API was in use — no `RegisterFunction`, no `*sqlite.Conn`, no +`modernc.org/sqlite/lib` error constants, no `sql.Register`. No `time.Time` is +ever bound as a query argument (every retention cutoff is pre-formatted), so +driver time handling is not in play. Both drivers convert declared +`DATE`/`DATETIME`/`TIMESTAMP` columns to `time.Time`, so `/api/dropped-packets` +keeps emitting `dropped_at` as RFC3339 exactly as before. diff --git a/internal/dbschema/dbschema.go b/internal/dbschema/dbschema.go index 8518be28..433ee8de 100644 --- a/internal/dbschema/dbschema.go +++ b/internal/dbschema/dbschema.go @@ -43,6 +43,9 @@ func Apply(rw *sql.DB, logf Logger) error { if err := ensureServerIndexes(rw); err != nil { return fmt.Errorf("ensure server indexes: %w", err) } + if err := ensureObservationsDedupIndex(rw, logf); err != nil { + return fmt.Errorf("ensure observations dedup index: %w", err) + } if err := ensureNeighborEdgesTable(rw); err != nil { return fmt.Errorf("ensure neighbor_edges: %w", err) } @@ -189,9 +192,27 @@ func AssertReady(ro *sql.DB) error { return nil } +// Querier is the read surface shared by *sql.DB and *sql.Tx. +// +// Not *sql.Conn: it exposes only QueryContext/QueryRowContext, so it does not +// satisfy this. Widen the interface to the Context variants if a caller ever +// needs one. +// +// It exists so schema probes can run on whichever handle the caller already +// holds. Taking *sql.DB unconditionally is a deadlock waiting to happen: a +// caller inside a transaction has the connection checked out, and on a pool +// capped at one connection — which is what cmd/ingestor runs — a probe against +// the pool waits forever for the connection its own transaction is holding. +type Querier interface { + Query(query string, args ...any) (*sql.Rows, error) + QueryRow(query string, args ...any) *sql.Row +} + // TableHasColumn reports whether the given table has the given column. // Exported because tests and the read-side need it without re-implementing. -func TableHasColumn(db *sql.DB, table, column string) (bool, error) { +// +// Pass the transaction, not the pool, when you are inside one. See Querier. +func TableHasColumn(db Querier, table, column string) (bool, error) { rows, err := db.Query(fmt.Sprintf("PRAGMA table_info(%s)", table)) if err != nil { return false, err diff --git a/internal/dbschema/dbschema_test.go b/internal/dbschema/dbschema_test.go index 576c8eec..5145777b 100644 --- a/internal/dbschema/dbschema_test.go +++ b/internal/dbschema/dbschema_test.go @@ -6,7 +6,7 @@ import ( "strings" "testing" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" ) // minimalDB bootstraps a SQLite DB with just enough tables for the @@ -16,7 +16,7 @@ func minimalDB(t *testing.T) *sql.DB { t.Helper() dir := t.TempDir() dbPath := filepath.Join(dir, "schema.db") - db, err := sql.Open("sqlite", "file:"+dbPath+"?_journal_mode=WAL") + db, err := sql.Open("sqlite3", "file:"+dbPath+"?_journal_mode=WAL") if err != nil { t.Fatal(err) } diff --git a/internal/dbschema/dedup_index.go b/internal/dbschema/dedup_index.go new file mode 100644 index 00000000..fb0f728f --- /dev/null +++ b/internal/dbschema/dedup_index.go @@ -0,0 +1,314 @@ +package dbschema + +import ( + "database/sql" + "errors" + "fmt" + "strconv" + + sqlite3 "github.com/mattn/go-sqlite3" +) + +// upsertMergedColumns are the columns the ingestor's observation UPSERT writes +// on conflict, in cmd/ingestor/db.go: +// +// ON CONFLICT(...) DO UPDATE SET +// snr = COALESCE(excluded.snr, snr), +// rssi = COALESCE(excluded.rssi, rssi), +// score = COALESCE(excluded.score, score), +// raw_hex = COALESCE(excluded.raw_hex, raw_hex), +// resolved_path = COALESCE(excluded.resolved_path, resolved_path) +// +// Two things follow, and collapseDuplicatesAndIndex has to honour both to be a +// faithful replay rather than an approximation of it. +// +// First, `excluded` is the incoming row, so a non-NULL later value REPLACES an +// earlier one. Applied down a group in insertion order the result is the LAST +// non-NULL value, not the first. +// +// Second, every other column is absent from the SET list, so the UPSERT never +// touches it: the surviving row keeps its own direction, timestamp, path_json +// and so on. Merging those too would invent history the ingestor would not have +// written. +var upsertMergedColumns = []string{"snr", "rssi", "score", "raw_hex", "resolved_path"} + +// dupGroupsDDL materialises one row per duplicate group: the group key plus the +// earliest id, which is the row that is kept and merged into. +// +// Materialised rather than left as a repeated subquery on purpose. In the +// subquery form every merged column's UPDATE re-ran this GROUP BY over the whole +// observations table — measured at 9.7s on 2.4M rows holding only 5 duplicates, +// nearly all of it the same scan performed repeatedly. This runs inside a write +// transaction and holds the write lock for its duration, so the repetition is +// worth removing. +// The WHERE clause is the whole correctness of this query, and leaving it out +// destroys data. +// +// GROUP BY folds all NULLs into one group. A UNIQUE index does the opposite: +// SQLite treats NULLs as distinct, so a row with NULL in any indexed column can +// never violate it. Group without excluding them and the repair calls rows +// duplicates that the index would have accepted, then deletes them to build an +// index that did not need them gone. +// +// Not hypothetical: on an 11.2M-row instance, 198 of the 222 groups this +// reported had observer_idx IS NULL — 238 of the 262 rows it planned to delete. +// They were direction='tx' rows, and the index built without complaint once +// they were left alone. +// +// transmission_id is NOT NULL in the schema and COALESCE(path_json, ”) can +// never be NULL, so observer_idx is the only one that needs the guard; it is +// written out in full anyway, because the next person to add a column to this +// index should see the rule rather than infer it. +const dupGroupsDDL = `CREATE TEMP TABLE dedup_groups AS + SELECT transmission_id, observer_idx, COALESCE(path_json, '') AS p, + MIN(id) AS keep, COUNT(*) AS n + FROM observations + WHERE transmission_id IS NOT NULL + AND observer_idx IS NOT NULL + GROUP BY transmission_id, observer_idx, COALESCE(path_json, '') + HAVING COUNT(*) > 1` + +const dedupIndexDDL = `CREATE UNIQUE INDEX IF NOT EXISTS idx_observations_dedup ` + + `ON observations(transmission_id, observer_idx, COALESCE(path_json, ''))` + +// ensureObservationsDedupIndex creates the unique expression index that the +// ingestor's observation UPSERT resolves its ON CONFLICT target against: +// +// ON CONFLICT(transmission_id, observer_idx, COALESCE(path_json, '')) +// +// cmd/ingestor/db.go creates this index, but only inside the branch that +// creates the observations table for the first time. Any database whose +// observations table predates that branch therefore never got one, so the +// UPSERT had no conflict target to resolve against. Under modernc.org/sqlite +// that surfaced late — statements compiled lazily, so it failed on the first +// insert. github.com/mattn/go-sqlite3 compiles inside Prepare, so it now +// surfaces at OpenStore. Same bug either way; creating the index here repairs +// the database rather than just moving the error. +// +// Because the index is what was supposed to prevent duplicates, a database that +// never had it can already hold rows violating it — the repo's own +// test-fixtures/e2e-fixture.db did. Those rows are collapsed first. Refusing +// would not be safer: without the index the ingestor cannot prepare its UPSERT, +// so it cannot run at all. +// +// Skipped on v2 schemas (observer_id rather than observer_idx): the UPSERT does +// not apply there, and indexing a column that does not exist would fail. +func ensureObservationsDedupIndex(rw *sql.DB, logf Logger) error { + hasIdx, err := TableHasColumn(rw, "observations", "observer_idx") + if err != nil { + return err + } + if !hasIdx { + return nil + } + // Fast path: no duplicates, so the index just builds. Atomic on its own. + _, err = rw.Exec(dedupIndexDDL) + if err == nil { + return nil + } + if !isConstraintViolation(err) { + return err + } + + // Deleting rows is not something to do quietly, and on a large table this + // holds the write lock long enough that an operator watching startup + // deserves to know why before it happens rather than after. + logf("[dbschema] idx_observations_dedup cannot be created: duplicate observations exist. " + + "Repairing now — on a large observations table this takes tens of seconds, and " + + "writers are blocked for part of it.") + + removed, err := collapseDuplicatesAndIndex(rw, logf) + if err != nil { + return fmt.Errorf("collapse duplicate observations: %w", err) + } + logf("[dbschema] collapsed %d duplicate observation row(s); idx_observations_dedup created", removed) + return nil +} + +// isConstraintViolation reports whether err is SQLite refusing a constraint. +// +// Deliberately a typed check. This used to match on +// strings.Contains(err, "UNIQUE constraint failed"), which made the entire +// repair path — including the row deletion — hinge on the driver's prose, in +// the same change that swapped the driver. A reworded or wrapped message would +// silently skip the repair and surface later as an OpenStore failure with no +// hint as to why. CREATE UNIQUE INDEX can only hit a constraint error because +// the data violates the uniqueness it asks for, so the primary code is the +// right granularity. +func isConstraintViolation(err error) bool { + var se sqlite3.Error + if errors.As(err, &se) { + return se.Code == sqlite3.ErrConstraint + } + return false +} + +// collapseDuplicatesAndIndex merges duplicate observation rows into the lowest +// id of each group, deletes the rest, and creates the unique index — all in one +// transaction. It returns the number of rows deleted. +// +// The index creation belongs inside the same transaction as the deletion. With +// them separated, a writer inserting a duplicate in the gap makes the index +// creation fail while leaving the deletions committed: rows destroyed and no +// index to show for it. One transaction makes the repair all-or-nothing. +func collapseDuplicatesAndIndex(rw *sql.DB, logf Logger) (int64, error) { + tx, err := rw.Begin() + if err != nil { + return 0, err + } + defer func() { _ = tx.Rollback() }() + + // Every reference to the TEMP table goes through tx, so its whole lifecycle + // sits on one connection: created here, dropped before COMMIT below, and + // removed by ROLLBACK on any error path (a TEMP table created inside a + // transaction does not survive its rollback — verified, not assumed). + // + // It has to be tx, not rw. A TEMP table belongs to a single connection and + // database/sql hands out pooled ones, so an `rw.Exec` drop can land on a + // different connection and leave the table behind on the one that has it. + // The leading DROP is still here because this package cannot prove nothing + // else ever left one on this connection. + if _, err := tx.Exec(`DROP TABLE IF EXISTS temp.dedup_groups`); err != nil { + return 0, fmt.Errorf("drop stale dedup_groups: %w", err) + } + if _, err := tx.Exec(dupGroupsDDL); err != nil { + return 0, fmt.Errorf("build dedup_groups: %w", err) + } + if _, err := tx.Exec(`CREATE INDEX temp.dedup_groups_key + ON dedup_groups(transmission_id, observer_idx, p)`); err != nil { + return 0, fmt.Errorf("index dedup_groups: %w", err) + } + + // The group keys exist right now and cease to after the delete. Record them + // before destroying anything: a row count is not enough to reconstruct from + // if the merge direction is ever wrong again. + if err := logDuplicateGroups(tx, logf); err != nil { + return 0, err + } + + // Only merge columns this database actually has. Apply runs this step + // before ensureResolvedPathColumn and ensureObservationsRawHexColumn, so on + // a database old enough to be missing the dedup index, resolved_path and + // raw_hex may not exist yet either. + cols, err := existingColumns(tx, "observations", upsertMergedColumns) + if err != nil { + return 0, err + } + + // Replay the UPSERT: for each merged column take the LAST non-NULL value in + // the group by id. The subquery spans the whole group, the survivor + // included, so when every row is NULL the result is NULL — which is what + // the UPSERT would also have left behind. + for _, c := range cols { + q := `UPDATE observations SET ` + c + ` = ( + SELECT o2.` + c + ` FROM observations o2 + WHERE o2.transmission_id = observations.transmission_id + AND o2.observer_idx = observations.observer_idx + AND COALESCE(o2.path_json, '') = COALESCE(observations.path_json, '') + AND o2.` + c + ` IS NOT NULL + ORDER BY o2.id DESC LIMIT 1) + WHERE id IN (SELECT keep FROM dedup_groups)` + if _, err := tx.Exec(q); err != nil { + return 0, fmt.Errorf("merge %s: %w", c, err) + } + } + + res, err := tx.Exec(`DELETE FROM observations WHERE id IN ( + SELECT o.id FROM observations o + JOIN dedup_groups d + ON d.transmission_id = o.transmission_id + AND d.observer_idx = o.observer_idx + AND d.p = COALESCE(o.path_json, '') + WHERE o.id <> d.keep)`) + if err != nil { + return 0, err + } + removed, err := res.RowsAffected() + if err != nil { + return 0, err + } + + // Same transaction: if this fails, the deletions above roll back with it. + if _, err := tx.Exec(`DROP TABLE temp.dedup_groups`); err != nil { + return 0, fmt.Errorf("drop dedup_groups: %w", err) + } + + if _, err := tx.Exec(dedupIndexDDL); err != nil { + return 0, fmt.Errorf("create idx_observations_dedup after collapsing %d row(s): %w", removed, err) + } + if err := tx.Commit(); err != nil { + return 0, err + } + return removed, nil +} + +// existingColumns filters want down to the columns table actually has, keeping +// the given order. +// +// Takes a Querier, not *sql.DB, and callers inside a transaction must pass the +// transaction. Probing the pool from inside one deadlocks the ingestor, whose +// pool is capped at a single connection: the probe waits for the connection the +// transaction is holding, and nothing ever releases it. +func existingColumns(rw Querier, table string, want []string) ([]string, error) { + out := make([]string, 0, len(want)) + for _, c := range want { + ok, err := TableHasColumn(rw, table, c) + if err != nil { + return nil, err + } + if ok { + out = append(out, c) + } + } + return out, nil +} + +// maxLoggedDupGroups bounds the audit log. A database missing the index for +// long enough can have a great many duplicate groups, and an unbounded dump at +// startup is its own operational problem. +const maxLoggedDupGroups = 20 + +// logDuplicateGroups records what is about to be collapsed, while dedup_groups +// still holds it. Reads through tx because the TEMP table lives on that +// transaction's connection. +func logDuplicateGroups(tx *sql.Tx, logf Logger) error { + var groups, extra int64 + if err := tx.QueryRow(`SELECT COUNT(*), COALESCE(SUM(n - 1), 0) FROM dedup_groups`). + Scan(&groups, &extra); err != nil { + return fmt.Errorf("count dedup_groups: %w", err) + } + logf("[dbschema] %d duplicate observation group(s), %d row(s) to remove", groups, extra) + + rows, err := tx.Query(`SELECT transmission_id, observer_idx, p, keep, n + FROM dedup_groups ORDER BY keep LIMIT ?`, maxLoggedDupGroups) + if err != nil { + return fmt.Errorf("list dedup_groups: %w", err) + } + defer rows.Close() + var listed int64 + for rows.Next() { + var txID, keep, n int64 + var observerIdx sql.NullInt64 + var pathJSON string + if err := rows.Scan(&txID, &observerIdx, &pathJSON, &keep, &n); err != nil { + return fmt.Errorf("scan dedup_groups: %w", err) + } + // Print NULL as NULL. Rendering observerIdx.Int64 unconditionally shows + // a NULL as 0, which is the one value that would hide the grouping bug + // above from the only person able to notice it. + idx := "NULL" + if observerIdx.Valid { + idx = strconv.FormatInt(observerIdx.Int64, 10) + } + logf("[dbschema] transmission_id=%d observer_idx=%s path_json=%q rows=%d keeping id=%d", + txID, idx, pathJSON, n, keep) + listed++ + } + if err := rows.Err(); err != nil { + return fmt.Errorf("iterate dedup_groups: %w", err) + } + if groups > listed { + logf("[dbschema] ... and %d more group(s) not listed", groups-listed) + } + return nil +} diff --git a/internal/dbschema/dedup_index_test.go b/internal/dbschema/dedup_index_test.go new file mode 100644 index 00000000..550e328d --- /dev/null +++ b/internal/dbschema/dedup_index_test.go @@ -0,0 +1,540 @@ +package dbschema + +import ( + "database/sql" + "errors" + "fmt" + "path/filepath" + "strings" + "testing" + "time" + + sqlite3 "github.com/mattn/go-sqlite3" +) + +// observationsDB builds a database with an observations table but deliberately +// no idx_observations_dedup — the shape of every database created before +// cmd/ingestor/db.go started making that index, and of any database whose +// observations table it never ran against. +func observationsDB(t *testing.T) *sql.DB { + t.Helper() + db, err := sql.Open("sqlite3", "file:"+filepath.Join(t.TempDir(), "obs.db")+"?_journal_mode=WAL") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { db.Close() }) + // One connection, like cmd/ingestor. An unbounded pool hides any code that + // queries the pool while holding a transaction: it quietly opens a second + // connection instead of deadlocking, so the suite passes and production + // hangs. Every fixture here must match the tightest pool in production. + db.SetMaxOpenConns(1) + // The full observations shape, including all five columns the ingestor's + // UPSERT merges. An earlier fixture omitted score, raw_hex and + // resolved_path, so removing those three from upsertMergedColumns left the + // entire suite green — three fifths of the merge was untested. + if _, err := db.Exec(`CREATE TABLE observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + transmission_id INTEGER NOT NULL, + observer_idx INTEGER, + direction TEXT, + snr REAL, + rssi REAL, + score INTEGER, + path_json TEXT, + timestamp INTEGER, + raw_hex TEXT, + resolved_path TEXT + )`); err != nil { + t.Fatal(err) + } + return db +} + +func dedupIndexExists(t *testing.T, db *sql.DB) bool { + t.Helper() + var n int + if err := db.QueryRow( + `SELECT COUNT(*) FROM sqlite_master WHERE type='index' AND name='idx_observations_dedup'`, + ).Scan(&n); err != nil { + t.Fatal(err) + } + return n == 1 +} + +func TestEnsureObservationsDedupIndexOnCleanTable(t *testing.T) { + db := observationsDB(t) + if _, err := db.Exec( + `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES (1, 1, '[]', 10), (1, 2, '[]', 10)`, + ); err != nil { + t.Fatal(err) + } + if err := ensureObservationsDedupIndex(db, t.Logf); err != nil { + t.Fatalf("ensureObservationsDedupIndex: %v", err) + } + if !dedupIndexExists(t, db) { + t.Error("index was not created") + } +} + +// The in-transaction DROP only matters on the success path: on failure ROLLBACK +// removes the temp table regardless. Replacing that DROP with a no-op survived +// every other test here, because none of them looked at the temp schema after a +// repair that worked. +// +// It matters because the connection goes back to the pool. A leftover +// dedup_groups is stale data the next repair would otherwise inherit. +func TestCollapseLeavesNoTempTableAfterSuccess(t *testing.T) { + db := observationsDB(t) // SetMaxOpenConns(1), so this is the same connection + if _, err := db.Exec(`INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp) VALUES + (1, 21, 9, '[]', 10), (2, 21, 9, '[]', 10)`); err != nil { + t.Fatal(err) + } + if err := ensureObservationsDedupIndex(db, t.Logf); err != nil { + t.Fatalf("ensureObservationsDedupIndex: %v", err) + } + var n int + if err := db.QueryRow( + `SELECT COUNT(*) FROM temp.sqlite_master WHERE name LIKE 'dedup_groups%'`).Scan(&n); err != nil { + t.Fatal(err) + } + if n != 0 { + t.Errorf("%d temp dedup_groups object(s) left on the connection after a successful repair", n) + } +} + +// The interesting case: duplicates already present, which is what blocked the +// index from being created in the first place. They must be collapsed, and the +// survivor must inherit the non-NULL fields of the rows that went away — the +// same merge the ingestor's ON CONFLICT ... DO UPDATE SET x = COALESCE(...) +// would have performed had the index existed. +func TestEnsureObservationsDedupIndexCollapsesDuplicates(t *testing.T) { + db := observationsDB(t) + if _, err := db.Exec(`INSERT INTO observations (id, transmission_id, observer_idx, direction, snr, rssi, path_json, timestamp) VALUES + (1, 5, 3, 'rx', NULL, -90, '[]', 100), + (2, 5, 3, NULL, 7.5, NULL, '[]', 100), + (3, 5, 3, NULL, NULL, NULL, NULL, 100), + (4, 9, 1, 'rx', 1.0, -80, '["AA"]', 200)`); err != nil { + t.Fatal(err) + } + + if err := ensureObservationsDedupIndex(db, t.Logf); err != nil { + t.Fatalf("ensureObservationsDedupIndex: %v", err) + } + if !dedupIndexExists(t, db) { + t.Fatal("index was not created after collapsing duplicates") + } + + // Rows 1 and 2 share the key (5, 3, '[]') and collapse into id 1. + // + // Row 3 does NOT join them: its path_json is NULL, and COALESCE(path_json,'') + // makes that the empty string, a different key from '[]'. That is the + // ingestor's conflict target verbatim, so an unrecorded path and an + // explicitly empty path are distinct observations here — worth pinning, + // because from the outside it looks like it ought to be one group. + // Row 4 has its own key and is untouched. + var ids string + if err := db.QueryRow(`SELECT GROUP_CONCAT(id) FROM (SELECT id FROM observations ORDER BY id)`).Scan(&ids); err != nil { + t.Fatal(err) + } + if ids != "1,3,4" { + t.Errorf("surviving ids = %q, want \"1,3,4\" (lowest id of each group; NULL path_json is its own group)", ids) + } + + var direction sql.NullString + var snr, rssi sql.NullFloat64 + if err := db.QueryRow(`SELECT direction, snr, rssi FROM observations WHERE id = 1`).Scan(&direction, &snr, &rssi); err != nil { + t.Fatal(err) + } + if direction.String != "rx" { + t.Errorf("direction = %q, want \"rx\" (its own value)", direction.String) + } + if snr.Float64 != 7.5 { + t.Errorf("snr = %v, want 7.5 (merged from the row that was removed)", snr.Float64) + } + if rssi.Float64 != -90 { + t.Errorf("rssi = %v, want -90 (its own value, not overwritten by a NULL)", rssi.Float64) + } +} + +// The merge has to replay the UPSERT, and the UPSERT's +// `COALESCE(excluded.x, x)` means a later non-NULL value REPLACES an earlier +// one. Complementary NULLs (as above) cannot tell "first non-NULL wins" from +// "last non-NULL wins" — both produce the same answer — so this asserts the +// direction explicitly with values that conflict. +func TestEnsureObservationsDedupIndexKeepsLatestValues(t *testing.T) { + db := observationsDB(t) + // One group, three rows, every merged column non-NULL and different. + // path_json is identical so they collide; direction is NOT in the UPSERT's + // SET list, so the survivor must keep its own. + // All five merged columns differ across the group, so each one proves the + // direction independently. direction/timestamp are NOT in the UPSERT's SET + // list and must keep the survivor's own values. + if _, err := db.Exec(`INSERT INTO observations (id, transmission_id, observer_idx, direction, snr, rssi, score, path_json, timestamp, raw_hex, resolved_path) VALUES + (1, 7, 2, 'first', 1.0, -10, 11, '[]', 100, 'aa', '["a"]'), + (2, 7, 2, 'second', 7.0, -20, 22, '[]', 200, 'bb', '["b"]'), + (3, 7, 2, 'third', 9.0, -30, 33, '[]', 300, 'cc', '["c"]')`); err != nil { + t.Fatal(err) + } + if err := ensureObservationsDedupIndex(db, t.Logf); err != nil { + t.Fatalf("ensureObservationsDedupIndex: %v", err) + } + + var direction, rawHex, resolvedPath string + var snr, rssi float64 + var score, timestamp int64 + if err := db.QueryRow(`SELECT direction, snr, rssi, score, raw_hex, resolved_path, timestamp + FROM observations WHERE id = 1`).Scan( + &direction, &snr, &rssi, &score, &rawHex, &resolvedPath, ×tamp); err != nil { + t.Fatal(err) + } + // Merged: last non-NULL down the group wins. + for _, c := range []struct { + name string + got, want any + }{ + {"snr", snr, 9.0}, + {"rssi", rssi, -30.0}, + {"score", score, int64(33)}, + {"raw_hex", rawHex, "cc"}, + {"resolved_path", resolvedPath, `["c"]`}, + } { + if c.got != c.want { + t.Errorf("%s = %v, want %v (last non-NULL: COALESCE(excluded.x, x) lets later rows win)", c.name, c.got, c.want) + } + } + // Not merged: the UPSERT never SETs these, so the survivor keeps its own. + if direction != "first" { + t.Errorf("direction = %q, want \"first\": not in the UPSERT's SET list", direction) + } + if timestamp != 100 { + t.Errorf("timestamp = %d, want 100: not in the UPSERT's SET list", timestamp) + } + var n int + if err := db.QueryRow(`SELECT COUNT(*) FROM observations`).Scan(&n); err != nil { + t.Fatal(err) + } + if n != 1 { + t.Errorf("observations = %d, want 1", n) + } +} + +// The repair must be all-or-nothing. If the index cannot be created, the +// deletions must not survive: rows destroyed with no index to show for it is +// the worst outcome available. +func TestCollapseDuplicatesAndIndexIsAtomic(t *testing.T) { + db := observationsDB(t) + // Conflicting readings, so the merge has something to write. Checking only + // the row count let a version that committed the merge early pass: the rows + // came back but their values did not. + if _, err := db.Exec(`INSERT INTO observations (id, transmission_id, observer_idx, snr, rssi, path_json, timestamp) VALUES + (1, 1, 1, 1.0, -10, '[]', 10), + (2, 1, 1, 2.0, -20, '[]', 10)`); err != nil { + t.Fatal(err) + } + // Occupy the index name with a TABLE. `CREATE INDEX IF NOT EXISTS` only + // shrugs when an *index* of that name exists; a table of that name is an + // error, so the CREATE fails after the merge and delete have already run. + if _, err := db.Exec(`CREATE TABLE idx_observations_dedup (x INTEGER)`); err != nil { + t.Fatal(err) + } + + if _, err := collapseDuplicatesAndIndex(db, t.Logf); err == nil { + t.Fatal("expected index creation to fail") + } + + var n int + if err := db.QueryRow(`SELECT COUNT(*) FROM observations`).Scan(&n); err != nil { + t.Fatal(err) + } + if n != 2 { + t.Errorf("observations = %d, want 2: a failed index creation must roll the deletions back", n) + } + // The merge must roll back too, not just the delete. + var snr, rssi float64 + if err := db.QueryRow(`SELECT snr, rssi FROM observations WHERE id = 1`).Scan(&snr, &rssi); err != nil { + t.Fatal(err) + } + if snr != 1.0 || rssi != -10 { + t.Errorf("id=1 snr=%v rssi=%v, want 1/-10: the merge must roll back with the delete", snr, rssi) + } + // And nothing may be left in the temp schema for the next user of this + // connection. Replacing the in-transaction DROP with a no-op used to pass. + var temps int + if err := db.QueryRow(`SELECT COUNT(*) FROM temp.sqlite_master WHERE name LIKE 'dedup_groups%'`).Scan(&temps); err != nil { + t.Fatal(err) + } + if temps != 0 { + t.Errorf("temp.dedup_groups survived a failed repair (%d objects)", temps) + } +} + +// Running twice must be a no-op: Apply runs on every ingestor start. +func TestEnsureObservationsDedupIndexIsIdempotent(t *testing.T) { + db := observationsDB(t) + if _, err := db.Exec(`INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES (1, 1, '[]', 10), (1, 1, '[]', 10)`); err != nil { + t.Fatal(err) + } + for i := range 2 { + if err := ensureObservationsDedupIndex(db, t.Logf); err != nil { + t.Fatalf("pass %d: %v", i+1, err) + } + } + var n int + if err := db.QueryRow(`SELECT COUNT(*) FROM observations`).Scan(&n); err != nil { + t.Fatal(err) + } + if n != 1 { + t.Errorf("observations = %d, want 1", n) + } +} + +// v2 schemas key observations by observer_id, not observer_idx. The ingestor's +// UPSERT does not apply there, and indexing a missing column would error, so the +// step must skip rather than fail. +func TestEnsureObservationsDedupIndexSkipsV2Schema(t *testing.T) { + db, err := sql.Open("sqlite3", "file:"+filepath.Join(t.TempDir(), "v2.db")+"?_journal_mode=WAL") + if err != nil { + t.Fatal(err) + } + defer db.Close() + if _, err := db.Exec(`CREATE TABLE observations (id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_id TEXT)`); err != nil { + t.Fatal(err) + } + if err := ensureObservationsDedupIndex(db, t.Logf); err != nil { + t.Fatalf("expected a skip on a v2 schema, got: %v", err) + } + if dedupIndexExists(t, db) { + t.Error("index must not be created on a v2 schema") + } +} + +// Nothing covered the branch that DECIDES to repair. TestCollapseDuplicates... +// calls collapseDuplicatesAndIndex directly, so a broken error check in +// ensureObservationsDedupIndex would leave every one of those tests green while +// production silently skipped the repair and failed later at OpenStore. +// +// This asserts the decision: duplicates present, the real driver's real error, +// and the repair actually taken. It is the test that would catch the driver +// rewording its constraint message. +func TestEnsureObservationsDedupIndexTakesRepairPathOnRealDriverError(t *testing.T) { + db := observationsDB(t) + if _, err := db.Exec(`INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp) VALUES + (1, 4, 4, '[]', 10), (2, 4, 4, '[]', 10)`); err != nil { + t.Fatal(err) + } + + // The error the fast path actually gets. If isConstraintViolation stops + // recognising this, the repair below never runs. + _, createErr := db.Exec(dedupIndexDDL) + if createErr == nil { + t.Fatal("expected CREATE UNIQUE INDEX to fail over duplicates") + } + if !isConstraintViolation(createErr) { + t.Fatalf("isConstraintViolation did not recognise the driver's own error: %v (%T)", createErr, createErr) + } + + // The above passes just as happily with a strings.Contains check, because + // the current driver still says "UNIQUE constraint failed". These pin the + // typed behaviour itself: a constraint error whose text says nothing of the + // sort must still be recognised, a wrapped one must be unwrapped, and an + // unrelated error carrying the magic phrase must NOT trigger a repair that + // deletes rows. + typed := sqlite3.Error{Code: sqlite3.ErrConstraint, ExtendedCode: sqlite3.ErrConstraintUnique} + if !isConstraintViolation(typed) { + t.Error("a bare sqlite3.Error with ErrConstraint was not recognised") + } + if !isConstraintViolation(fmt.Errorf("create index: %w", typed)) { + t.Error("a wrapped constraint error was not recognised; errors.As should unwrap it") + } + if isConstraintViolation(errors.New("UNIQUE constraint failed: index 'x'")) { + t.Error("a plain error carrying the message was treated as a constraint violation — " + + "that is the string match this check exists to replace") + } + if isConstraintViolation(sqlite3.Error{Code: sqlite3.ErrBusy}) { + t.Error("SQLITE_BUSY was treated as a constraint violation; a locked database must not start deleting rows") + } + + var repaired bool + logf := func(format string, args ...interface{}) { + repaired = true + t.Logf(format, args...) + } + if err := ensureObservationsDedupIndex(db, logf); err != nil { + t.Fatalf("ensureObservationsDedupIndex: %v", err) + } + if !repaired { + t.Error("repair path was not taken: no log output, so the constraint error was not recognised") + } + if !dedupIndexExists(t, db) { + t.Error("index missing after repair") + } + var n int + if err := db.QueryRow(`SELECT COUNT(*) FROM observations`).Scan(&n); err != nil { + t.Fatal(err) + } + if n != 1 { + t.Errorf("observations = %d, want 1", n) + } +} + +// The audit log has to name what it destroyed, not just count it. +func TestCollapseLogsGroupKeysBeforeDeleting(t *testing.T) { + db := observationsDB(t) + if _, err := db.Exec(`INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp) VALUES + (1, 11, 3, '["AA"]', 10), (2, 11, 3, '["AA"]', 10), (3, 11, 3, '["AA"]', 10)`); err != nil { + t.Fatal(err) + } + var out []string + logf := func(format string, args ...interface{}) { + out = append(out, fmt.Sprintf(format, args...)) + } + if err := ensureObservationsDedupIndex(db, logf); err != nil { + t.Fatal(err) + } + joined := strings.Join(out, "\n") + for _, want := range []string{ + "1 duplicate observation group(s), 2 row(s) to remove", + "transmission_id=11", + `path_json="[\"AA\"]"`, + "keeping id=1", + } { + if !strings.Contains(joined, want) { + t.Errorf("audit log missing %q; got:\n%s", want, joined) + } + } +} + +// Content is not the point — order is. The log exists so that a repair which +// destroys the wrong rows leaves a record of what it destroyed, which requires +// the keys to be written BEFORE the delete, not after. Asserting content alone +// passed with the logging moved below the DELETE. +// +// Forcing the repair to fail after the merge proves the ordering: if the keys +// were logged before the failure, they were logged before the delete. +func TestCollapseLogsGroupKeysEvenWhenTheRepairFails(t *testing.T) { + db := observationsDB(t) + if _, err := db.Exec(`INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp) VALUES + (1, 12, 4, '["BB"]', 10), (2, 12, 4, '["BB"]', 10)`); err != nil { + t.Fatal(err) + } + // Fail at the DELETE specifically. Failing later (at CREATE INDEX) does not + // pin the ordering: logging moved to just after the DELETE would still have + // run. A BEFORE DELETE trigger that aborts puts the failure exactly at the + // step the log is supposed to precede. + if _, err := db.Exec(`CREATE TRIGGER block_delete BEFORE DELETE ON observations + BEGIN SELECT RAISE(ABORT, 'no deletes'); END`); err != nil { + t.Fatal(err) + } + + var out []string + logf := func(format string, args ...interface{}) { out = append(out, fmt.Sprintf(format, args...)) } + if _, err := collapseDuplicatesAndIndex(db, logf); err == nil { + t.Fatal("expected the repair to fail") + } + + joined := strings.Join(out, "\n") + if !strings.Contains(joined, "transmission_id=12") || !strings.Contains(joined, "keeping id=1") { + t.Errorf("group keys were not logged before the repair failed, so a failed or wrong "+ + "repair leaves no record of what it touched; got:\n%s", joined) + } +} + +// A pool of one is what cmd/ingestor runs. Anything in the repair that queries +// the pool while holding the transaction waits for a connection the transaction +// itself has checked out, and never gets it: the ingestor hangs at boot, after +// logging that it is repairing, with the database untouched and ingest dead. +// +// Found on staging, not here, because every fixture used an unbounded pool. +// Runs in a goroutine so a regression fails in seconds with a usable message +// rather than hanging until the package timeout. +func TestCollapseDoesNotDeadlockOnSingleConnectionPool(t *testing.T) { + db := observationsDB(t) // SetMaxOpenConns(1) + if _, err := db.Exec(`INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp) VALUES + (1, 3, 1, '[]', 10), (2, 3, 1, '[]', 10)`); err != nil { + t.Fatal(err) + } + + done := make(chan error, 1) + go func() { done <- ensureObservationsDedupIndex(db, func(string, ...interface{}) {}) }() + + select { + case err := <-done: + if err != nil { + t.Fatalf("ensureObservationsDedupIndex: %v", err) + } + case <-time.After(15 * time.Second): + t.Fatal("deadlock: the repair is querying the pool while holding its own transaction — " + + "pass tx, not rw, to anything that reads inside collapseDuplicatesAndIndex") + } + if !dedupIndexExists(t, db) { + t.Error("index missing") + } +} + +// GROUP BY folds NULLs together; a UNIQUE index keeps them apart. Rows with a +// NULL in an indexed column can never violate the index, so the repair must not +// treat them as duplicates at all. +// +// The damage is not the obvious one. Both the DELETE and the merge's correlated +// subquery join on `observer_idx = observer_idx`, and NULL = NULL is not true, +// so the rows are never actually deleted — the *merge* is what destroys data: +// the subquery matches nothing and writes NULL over the survivor's real +// readings. Measured with the guard removed, a row holding snr=4.5 rssi=-70 +// came back with both NULL and its row still in place, so nothing looks missing +// while the measurements are gone. Assert the values, not just the row count: +// an earlier version of this test checked survival alone and passed against the +// bug. +// +// On an 11.2M-row instance, 198 of 222 reported groups were observer_idx IS +// NULL — direction='tx' rows. +func TestCollapseLeavesNullKeyedRowsAlone(t *testing.T) { + db := observationsDB(t) + if _, err := db.Exec(`INSERT INTO observations (id, transmission_id, observer_idx, direction, snr, rssi, path_json, timestamp) VALUES + (1, 1, NULL, 'tx', 4.5, -70, '[]', 10), + (2, 1, NULL, 'tx', 5.5, -60, '[]', 10), + (3, 2, 7, 'rx', 1.0, -80, '[]', 20), + (4, 2, 7, 'rx', 2.0, -90, '[]', 20)`); err != nil { + t.Fatal(err) + } + + var logged []string + logf := func(format string, args ...interface{}) { logged = append(logged, fmt.Sprintf(format, args...)) } + if err := ensureObservationsDedupIndex(db, logf); err != nil { + t.Fatalf("ensureObservationsDedupIndex: %v", err) + } + + // Only the observer_idx=7 pair was a real violation: one row removed there, + // both NULL rows left entirely alone. + var ids string + if err := db.QueryRow(`SELECT GROUP_CONCAT(id) FROM (SELECT id FROM observations ORDER BY id)`).Scan(&ids); err != nil { + t.Fatal(err) + } + if ids != "1,2,3" { + t.Errorf("surviving ids = %q, want \"1,2,3\": NULL observer_idx rows never violate the unique index", ids) + } + + // The readings on the NULL rows must be exactly as inserted. + for _, want := range []struct { + id int64 + snr, rssi float64 + }{{1, 4.5, -70}, {2, 5.5, -60}} { + var snr, rssi sql.NullFloat64 + if err := db.QueryRow(`SELECT snr, rssi FROM observations WHERE id = ?`, want.id).Scan(&snr, &rssi); err != nil { + t.Fatal(err) + } + if !snr.Valid || !rssi.Valid { + t.Errorf("id=%d: snr/rssi wiped to NULL — the merge matched nothing and overwrote real readings", want.id) + continue + } + if snr.Float64 != want.snr || rssi.Float64 != want.rssi { + t.Errorf("id=%d: snr=%v rssi=%v, want %v/%v", want.id, snr.Float64, rssi.Float64, want.snr, want.rssi) + } + } + + if !dedupIndexExists(t, db) { + t.Error("index missing — proof the NULL rows were never in its way") + } + if strings.Contains(strings.Join(logged, "\n"), "observer_idx=0") { + t.Error("a NULL observer_idx was logged as 0, which hides exactly this class of bug") + } +} diff --git a/internal/dbschema/go.mod b/internal/dbschema/go.mod index dcaac4c6..3dca3dc9 100644 --- a/internal/dbschema/go.mod +++ b/internal/dbschema/go.mod @@ -2,15 +2,4 @@ module github.com/meshcore-analyzer/dbschema go 1.22 -require ( - github.com/dustin/go-humanize v1.0.1 // indirect - github.com/google/uuid v1.6.0 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect - github.com/ncruces/go-strftime v0.1.9 // indirect - github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect - golang.org/x/sys v0.22.0 // indirect - modernc.org/libc v1.55.3 // indirect - modernc.org/mathutil v1.6.0 // indirect - modernc.org/memory v1.8.0 // indirect - modernc.org/sqlite v1.34.5 // indirect -) +require github.com/mattn/go-sqlite3 v1.14.52 diff --git a/internal/dbschema/go.sum b/internal/dbschema/go.sum index 2b43db39..75bf6eb5 100644 --- a/internal/dbschema/go.sum +++ b/internal/dbschema/go.sum @@ -1,21 +1,2 @@ -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= -github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U= -modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w= -modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= -modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= -modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= -modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= -modernc.org/sqlite v1.34.5 h1:Bb6SR13/fjp15jt70CL4f18JIN7p7dnMExd+UFnF15g= -modernc.org/sqlite v1.34.5/go.mod h1:YLuNmX9NKs8wRNK2ko1LW1NGYcc9FkBO69JOt1AR9JE= +github.com/mattn/go-sqlite3 v1.14.52 h1:wVbm2Qnf4OXkqhBTSPuCRZDRnxfbVrrmiCEroVdog8U= +github.com/mattn/go-sqlite3 v1.14.52/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= diff --git a/internal/dbschema/writer_dsn.go b/internal/dbschema/writer_dsn.go new file mode 100644 index 00000000..fb0f7acd --- /dev/null +++ b/internal/dbschema/writer_dsn.go @@ -0,0 +1,35 @@ +package dbschema + +// WriterDSN builds the DSN every writer must open the database with. +// +// It lives here because this package is the one thing cmd/ingestor and +// cmd/migrate both import, and they are the only two writers. Keeping the +// string in one place is not tidiness: cmd/migrate previously opened with a +// bare path, which under github.com/mattn/go-sqlite3 silently means +// synchronous=NORMAL, so the durability the ingestor DSN carefully pins was +// lost the moment anyone ran the migrate CLI against the same database. Two +// copies of a DSN is how that happens. +// +// Every parameter is deliberate: +// +// - _journal_mode=WAL — cmd/server reads concurrently and needs WAL. +// - _synchronous=FULL — SQLite's own default, and what modernc.org/sqlite +// left in place. mattn defaults this connection to NORMAL and executes the +// pragma unconditionally, so it must be stated to be kept. NORMAL can lose +// recent transactions on power loss. +// - _auto_vacuum=incremental — cmd/ingestor/maintenance.go drives +// incremental_vacuum. Only takes effect on a database created with it. +// - _foreign_keys=on — the schema relies on FK enforcement. +// - _busy_timeout=5000 — writers serialise behind the reader. +// - _cache_size=-2000 — 2000 KiB of page cache per connection. This is +// SQLite's own default, pinned so it cannot drift: the page cache is a C +// allocation and therefore sits outside GOMEMLIMIT. +func WriterDSN(path string) string { + return path + + "?_journal_mode=WAL" + + "&_synchronous=FULL" + + "&_auto_vacuum=incremental" + + "&_foreign_keys=on" + + "&_busy_timeout=5000" + + "&_cache_size=-2000" +} diff --git a/internal/dbschema/writer_dsn_test.go b/internal/dbschema/writer_dsn_test.go new file mode 100644 index 00000000..1e83fa02 --- /dev/null +++ b/internal/dbschema/writer_dsn_test.go @@ -0,0 +1,43 @@ +package dbschema + +import ( + "database/sql" + "path/filepath" + "testing" + + _ "github.com/mattn/go-sqlite3" +) + +// TestWriterDSNPragmas pins what every writer connection actually gets. +// +// cmd/ingestor has its own end-to-end version of this through the store, but +// this one covers the DSN itself, which is what cmd/migrate also opens with. +// The synchronous line matters most: mattn defaults it to NORMAL and runs the +// pragma unconditionally, so if it ever drops out of the DSN the durability +// change is silent. +func TestWriterDSNPragmas(t *testing.T) { + db, err := sql.Open("sqlite3", WriterDSN(filepath.Join(t.TempDir(), "w.db"))) + if err != nil { + t.Fatal(err) + } + defer db.Close() + db.SetMaxOpenConns(1) + + for _, want := range []struct{ pragma, value, why string }{ + {"journal_mode", "wal", "cmd/server reads concurrently"}, + {"synchronous", "2", "FULL; mattn defaults to 1 (NORMAL) unless the DSN says otherwise"}, + {"auto_vacuum", "2", "INCREMENTAL; maintenance.go drives incremental_vacuum"}, + {"foreign_keys", "1", "the schema relies on FK enforcement"}, + {"busy_timeout", "5000", "writers serialise behind the reader"}, + {"cache_size", "-2000", "C-allocated page cache, pinned: it sits outside GOMEMLIMIT"}, + } { + var got string + if err := db.QueryRow("PRAGMA " + want.pragma).Scan(&got); err != nil { + t.Errorf("PRAGMA %s: %v", want.pragma, err) + continue + } + if got != want.value { + t.Errorf("PRAGMA %s = %q, want %q (%s)", want.pragma, got, want.value, want.why) + } + } +} diff --git a/scripts/allmod.sh b/scripts/allmod.sh new file mode 100644 index 00000000..ad54f93c --- /dev/null +++ b/scripts/allmod.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Run a go command in every module of this multi-module repo. +# Usage: bash scripts/allmod.sh vet ./... +# bash scripts/allmod.sh test ./... +set -uo pipefail +cd "$(dirname "$0")/.." +rc=0 +while IFS= read -r mod; do + dir=$(dirname "$mod") + printf '== %-26s ' "$dir" + out=$( (cd "$dir" && go "$@" 2>&1) ) + if [ $? -eq 0 ]; then + echo "OK" + else + echo "FAIL" + echo "$out" | sed 's/^/ /' + rc=1 + fi +done < <(find . -name go.mod -not -path './node_modules/*' | sort) +exit $rc diff --git a/scripts/migrate-fixture-hashes.go b/scripts/migrate-fixture-hashes.go index 04973e11..a0acded9 100644 --- a/scripts/migrate-fixture-hashes.go +++ b/scripts/migrate-fixture-hashes.go @@ -10,7 +10,7 @@ import ( "log" "os" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" ) func computeContentHash(rawHex string) string { @@ -69,7 +69,7 @@ func main() { } dbPath := os.Args[1] - db, err := sql.Open("sqlite", dbPath) + db, err := sql.Open("sqlite3", dbPath) if err != nil { log.Fatal(err) } diff --git a/test-issue-1956-release-routing.js b/test-issue-1956-release-routing.js index 0f73a4d7..97b20fa4 100644 --- a/test-issue-1956-release-routing.js +++ b/test-issue-1956-release-routing.js @@ -56,6 +56,12 @@ function runSteps(source, context, edge, mutateFails = false) { const log = path.join(dir, 'commands'); fs.writeFileSync(log, ''); fs.mkdirSync(path.join(dir, 'cmd/decrypt'), { recursive: true }); + // go() only logs, so nothing real is produced; the release job's own + // static/runnable verification step still needs these to exist. + for (const arch of ['amd64', 'arm64']) { + fs.writeFileSync(path.join(dir, `corescope-decrypt-linux-${arch}`), + '#!/bin/sh\necho "corescope-decrypt stub"\n', { mode: 0o755 }); + } const stubs = ` log() { node -e 'require("fs").appendFileSync(process.env.COMMAND_LOG, JSON.stringify(process.argv.slice(1))+"\\n")' "$@"; } crane() { @@ -69,7 +75,8 @@ function runSteps(source, context, edge, mutateFails = false) { } gh() { log gh "$@"; } tar() { log tar "$@"; } - go() { log go "$GOOS" "$GOARCH" "$CGO_ENABLED" "$@"; } + go() { log go "$GOOS" "$GOARCH" "$CGO_ENABLED" "$CC" "$@"; } + file() { log file "$@"; echo "$1: ELF 64-bit LSB executable, statically linked"; } jq() { node -e 'const fs=require("fs"); const assert=require("assert/strict"); assert.equal(process.argv[1], ".config.Labels[\\"org.opencontainers.image.revision\\"] // \\\"\\\""); console.log(JSON.parse(fs.readFileSync(0,"utf8")).config.Labels["org.opencontainers.image.revision"] || "")' "$2" } @@ -151,8 +158,19 @@ for (const [ref, event] of [['refs/heads/master', 'push'], ['refs/heads/master', const jobs = route(context(ref, event, { images_published: true })); assert.equal(jobs['release-artifacts'].result, 'skipped', `${event}: no GitHub release`); assert.equal(jobs['build-and-publish'].result, 'success', `${event}: tag-only input must not skip branch/PR checks`); - const publishing = steps(block(deploy, 'build-and-publish', 2)).find(step => step.includes('uses: docker/build-push-action')); - assert.equal(Boolean(evaluate(value(publishing, 'if', 8), context(ref, event))), event === 'push', `${event}: GHCR publishing`); + // There is more than one build-push-action step now: a PR-only two-arch build + // that must NOT publish, and the GHCR push. Pin the pushing one by `push: true` + // rather than by being first in the job. + const buildSteps = steps(block(deploy, 'build-and-publish', 2)).filter(step => step.includes('uses: docker/build-push-action')); + const publishing = buildSteps.filter(step => value(step, 'push', 10) === 'true'); + assert.equal(publishing.length, 1, 'exactly one step may publish to GHCR'); + assert.equal(Boolean(evaluate(value(publishing[0], 'if', 8), context(ref, event))), event === 'push', `${event}: GHCR publishing`); + // The cross-toolchain gate: since the SQLite driver became cgo, a PR must + // still build both architectures, and must do it without publishing. + const prBuild = buildSteps.filter(step => value(step, 'push', 10) === 'false'); + assert.equal(prBuild.length, 1, 'PRs must get exactly one non-publishing two-arch build'); + assert.equal(value(prBuild[0], 'platforms', 10), 'linux/amd64,linux/arm64', 'the PR gate must cover both shipped architectures'); + assert.equal(Boolean(evaluate(value(prBuild[0], 'if', 8), context(ref, event))), event === 'pull_request', `${event}: PR-only two-arch gate`); } assert.equal(route(context(), 'go-test')['release-artifacts'].result, 'skipped', 'failed Go validation must block release'); const dispatchInput = block(deploy, 'images_published', 6); @@ -161,8 +179,17 @@ assert.equal(value(dispatchInput, 'default', 8), 'false', 'manual and fallback d const release = block(deploy, 'release-artifacts', 2); const builds = runSteps(release, context(), null).commands.filter(command => command[0] === 'go'); -assert.deepEqual(builds.map(command => command.slice(1, 4)), [['linux', 'amd64', '0'], ['linux', 'arm64', '0']]); -for (const command of builds) assert.ok(command.includes('-ldflags=-s -w -X main.version=v9.8.7'), 'binary version must come from tag'); +// CGO_ENABLED=1 since the SQLite driver became github.com/mattn/go-sqlite3, and +// CC must be zig targeting musl — that is what makes the artifact static and +// cross-buildable. A silent revert to the Go-only toolchain fails here. +assert.deepEqual(builds.map(command => command.slice(1, 5)), [ + ['linux', 'amd64', '1', 'zig cc -target x86_64-linux-musl'], + ['linux', 'arm64', '1', 'zig cc -target aarch64-linux-musl'], +]); +for (const command of builds) { + assert.ok(command.includes("-ldflags=-s -w -extldflags '-static -Wl,-s' -X main.version=v9.8.7"), 'binary version must come from tag, and the artifact must stay static'); + assert.ok(command.includes('-tags'), 'netgo/osusergo/sqlite_omit_load_extension must survive'); +} const upload = steps(release).filter(step => step.includes('uses: softprops/action-gh-release@v2')); assert.equal(upload.length, 1, 'publish both architectures together, before the release becomes immutable'); assert.equal(value(upload[0], 'fail_on_unmatched_files', 10), 'true', 'missing assets must prevent publication');