feat(backup): add protected database and config download (#154)

This commit is contained in:
n30nex
2026-09-23 20:13:40 -04:00
committed by GitHub
parent 4d2642abec
commit a25e2675c8
21 changed files with 1103 additions and 6 deletions
+2 -1
View File
@@ -9,7 +9,8 @@ COPY . .
RUN go build -o beacon ./cmd/beacon
FROM alpine:3.19
RUN apk add --no-cache ca-certificates tzdata
ARG POSTGRES_CLIENT_MAJOR=16
RUN apk add --no-cache ca-certificates tzdata postgresql${POSTGRES_CLIENT_MAJOR}-client
WORKDIR /app
COPY --from=builder /app/beacon .
CMD ["./beacon"]
+11
View File
@@ -47,10 +47,16 @@ jobs:
- name: Test
run: go test ./...
- name: Verify backup startup against PostgreSQL 16
env:
BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable
run: go test ./cmd/beacon -run '^TestBackupConfigurationPostgres$' -count=1 -v
- name: Verify backup command against PostgreSQL 16
run: |
CGO_ENABLED=0 go build -o "$RUNNER_TEMP/beacon-backup" ./cmd/beacon-backup
CGO_ENABLED=0 go test -c -o "$RUNNER_TEMP/backup.test" ./internal/backup
CGO_ENABLED=0 go test -c -o "$RUNNER_TEMP/backup-http.test" ./internal/api/router
docker run --rm --network host \
-e PGHOST=127.0.0.1 -e PGUSER=postgres -e PGDATABASE=postgres \
-e PGPASSWORD=backup-ci-only -e PGSSLMODE=disable \
@@ -58,6 +64,11 @@ jobs:
-v "$RUNNER_TEMP/beacon-backup:/beacon-backup:ro" \
-v "$RUNNER_TEMP/backup.test:/backup.test:ro" \
postgres:16-alpine /backup.test -test.run '^TestExportPostgres$' -test.v
docker run --rm --network host \
-e PGHOST=127.0.0.1 -e PGUSER=postgres -e PGDATABASE=postgres \
-e PGPASSWORD=backup-ci-only -e PGSSLMODE=disable -e BEACON_BACKUP_TEST_POSTGRES=1 \
-v "$RUNNER_TEMP/backup-http.test:/backup-http.test:ro" \
postgres:16-alpine /backup-http.test -test.run '^TestBackupDownloadPostgres$' -test.v
- name: Install govulncheck
run: go install golang.org/x/vuln/cmd/govulncheck@latest
+2 -2
View File
@@ -28,8 +28,8 @@ in PostgreSQL, and streams live events to WebSocket clients.
For deployment instructions including the frontend app, see the deployment docs.
For a bounded private database and saved-config bundle, see
[backup export](docs/backup-export.md). This is a standalone export tool; the
backup web interface and import workflow are separate follow-ups.
[backup export](docs/backup-export.md). A standalone export tool is available; the
protected download API is opt-in; browser login and import remain separate follow-ups.
---
+45
View File
@@ -0,0 +1,45 @@
// Copyright 2026 Beacon Contributors
// SPDX-License-Identifier: AGPL-3.0-or-later
package main
import (
"context"
"log/slog"
"time"
"github.com/MeshCore-Beacon/beacon-server/internal/backup"
"github.com/MeshCore-Beacon/beacon-server/internal/config"
"github.com/jackc/pgx/v5/pgxpool"
)
// A failed optional export prerequisite must not stop ingest or the public API.
// Empty options leave the authenticated backup endpoint unavailable (503).
func configureBackup(ctx context.Context, cfg *config.Config, pool *pgxpool.Pool, dsn, configPath string) backup.Options {
if !cfg.Backup.Enabled {
return backup.Options{}
}
if cfg.Auth.APIKey == "" {
slog.Error("backup unavailable: configure an admin bearer key", "component", "backup")
return backup.Options{}
}
service, err := backup.ConnectionService(dsn)
if err != nil {
slog.Error("backup unavailable: POSTGRES_DSN must be a supported single-host PostgreSQL URL; keyword DSNs remain valid for Beacon", "component", "backup")
return backup.Options{}
}
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
var serverVersion int
if err := pool.QueryRow(ctx, "SELECT current_setting('server_version_num')::integer").Scan(&serverVersion); err != nil {
slog.Error("backup unavailable: could not determine PostgreSQL server version", "component", "backup")
return backup.Options{}
}
if err := backup.CheckClient(ctx, serverVersion); err != nil {
// CheckClient never returns process output or connection settings.
slog.Error("backup unavailable", "component", "backup", "error", err)
return backup.Options{}
}
return backup.Options{ConfigPath: configPath, MaxBytes: backup.DefaultMaxBytes,
Timeout: backup.DefaultTimeout, Version: version, ConnectionService: service}
}
+58
View File
@@ -0,0 +1,58 @@
// Copyright 2026 Beacon Contributors
// SPDX-License-Identifier: AGPL-3.0-or-later
package main
import (
"context"
"os"
"path/filepath"
"runtime"
"testing"
"time"
"github.com/MeshCore-Beacon/beacon-server/internal/config"
"github.com/jackc/pgx/v5/pgxpool"
)
func TestBackupPreconditionsDegradeWithoutDatabaseOrExit(t *testing.T) {
cfg := &config.Config{}
if got := configureBackup(context.Background(), cfg, nil, "", "saved.yaml"); got.ConnectionService != "" {
t.Fatal("disabled backup configured")
}
cfg.Backup.Enabled = true
if got := configureBackup(context.Background(), cfg, nil, "", "saved.yaml"); got.ConnectionService != "" {
t.Fatal("missing admin key configured")
}
cfg.Auth.APIKey = "test-only-key"
if got := configureBackup(context.Background(), cfg, nil, "host=localhost user=test dbname=test", "saved.yaml"); got.ConnectionService != "" {
t.Fatal("keyword connection configured for backup")
}
}
func TestBackupConfigurationPostgres(t *testing.T) {
dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN")
if dsn == "" || runtime.GOOS == "windows" {
t.Skip("requires BEACON_TEST_POSTGRES_DSN and a Unix test client")
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
pool, err := pgxpool.New(ctx, dsn)
if err != nil {
t.Fatal("test database configuration unavailable")
}
defer pool.Close()
bin := t.TempDir()
t.Setenv("PATH", bin)
cfg := &config.Config{}
cfg.Backup.Enabled, cfg.Auth.APIKey = true, "test-only-key"
for _, major := range []string{"999", "15"} {
if err := os.WriteFile(filepath.Join(bin, "pg_dump"), []byte("#!/bin/sh\necho 'pg_dump (PostgreSQL) "+major+".0'\n"), 0700); err != nil {
t.Fatal(err)
}
got := configureBackup(ctx, cfg, pool, dsn, "saved.yaml")
if (got.ConnectionService != "") != (major == "999") {
t.Fatalf("unexpected backup availability for test client major %s", major)
}
}
}
+7 -2
View File
@@ -125,13 +125,15 @@ func main() {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
pool, err := pgxpool.New(ctx, getEnv("POSTGRES_DSN"))
dsn := getEnv("POSTGRES_DSN")
pool, err := pgxpool.New(ctx, dsn)
if err != nil {
// Parse errors can embed the complete DSN, including its password.
slog.Error("invalid PostgreSQL connection configuration; check POSTGRES_DSN", "component", "startup")
os.Exit(1)
}
defer pool.Close()
backupOpts := configureBackup(ctx, cfg, pool, dsn, configPath)
if err := db.RunMigrations(ctx, pool); err != nil {
slog.Error("migrations failed", "component", "startup", "error", err)
@@ -308,7 +310,10 @@ func main() {
MaxConnsPerIP: resolved.MaxConnsPerIP,
MaxConnectsPerMinute: resolved.MaxConnectsPerMinute,
CORS: cfg.CORS, Server: cfg.Server, Auth: cfg.Auth, RateLimit: resolved.RateLimit,
AdminRoutes: map[string]http.Handler{"/accounts": handlers.AccountsRouter(store)},
AdminRoutes: map[string]http.Handler{
"/accounts": handlers.AccountsRouter(store),
"/backup": handlers.BackupRouter(backupOpts),
},
})
srv := &http.Server{
+6
View File
@@ -112,6 +112,12 @@ telemetry:
retention: 672h # 4 weeks
resolution: 1h # store one snapshot per observer per hour
# Optional authenticated database/config downloads. Disabled unless explicitly enabled.
# Requires auth.api_key, a supported PostgreSQL URL and a compatible pg_dump client.
# A failed prerequisite leaves this feature unavailable; ingest and other APIs continue.
#backup:
# enabled: false
packets:
retention: 720h # 30 days
+56
View File
@@ -31,6 +31,62 @@ without publishing an output. Normal failure, timeout and handled interruption
remove temporary files; a power loss or SIGKILL can leave a private
`.beacon-backup-*` staging directory for the operator to inspect.
## Protected download API
Set `backup.enabled: true` in saved YAML and configure the existing admin bearer
key to enable `GET /api/v1/admin/backup`. Supply the key only in the `Authorization:
Bearer ...` header over HTTPS. No query parameters or request body are accepted.
This is an operator API, not account login or a browser admin panel. Keep it off
on public previews that do not need private exports.
The server requires `pg_dump` on its own PATH when this option is enabled. The
Docker image includes PostgreSQL 16's client by default. Its Alpine 3.19 base does
not provide PostgreSQL 17/18 packages: changing `POSTGRES_CLIENT_MAJOR` alone is
not an upgrade path. For a newer database, supply a runtime with a compatible
client. Startup compares `pg_dump --version` with `server_version_num` and leaves
backup unavailable if the client is missing, unrecognized or older than the server.
Match the source database major version. A client on the host or in a different container is not
sufficient. No Docker socket or host-command bridge is used.
The download uses the startup `POSTGRES_DSN`, not the standalone export's ambient
`PGDATABASE`. This first adapter supports a single-host `postgres://` or
`postgresql://` URL with an explicit user and database. Passwords, IPv6 and URL
escaping are supported. It copies TLS settings, `passfile`, `connect_timeout`,
`application_name`, `target_session_attrs` and `options`; known `pool_*` controls
are omitted from libpq. Supported PG environment defaults are captured at startup
and URL values override them. Prefer explicit `sslmode=verify-full` and a trusted
`sslrootcert` for remote databases; password-file/TLS paths must be readable in
the same runtime. Their contents are not included in the archive.
Keyword DSNs, service indirection, multiple hosts, duplicate/unknown query options,
newer protocol negotiation settings and service-file values with line breaks or
edge whitespace are rejected rather than silently changing targets or TLS policy.
Ambient `PGSERVICE`, `PGSERVICEFILE`, `PGSSLNEGOTIATION`, `PGMINPROTOCOLVERSION`,
`PGMAXPROTOCOLVERSION` and `PGTZ` are also unsupported for this adapter. The
standalone command retains native PG* support for those deployments. Errors do
not echo the DSN. Any failed backup prerequisite logs its reason and leaves only
this feature unavailable; Beacon still starts ingest, WebSocket and other APIs.
Fix the prerequisite and restart to enable downloads. Native connection settings live only in a 0600 service file in
private staging; process arguments contain no connection string or credential,
and no request changes Beacon's environment.
Only one export or transfer is allowed at a time; another request receives 409.
The existing ten-minute/1 GiB SQL/1 MiB YAML limits apply, and disconnecting
cancels an in-progress export. The completed transfer has a ten-minute write
deadline. Temporary files are removed after completion/failure/cancellation;
abrupt process termination can leave private staging. Responses use `no-store`
and a fixed attachment filename. A failed export returns an error without any
partial archive. Authentication missing/invalid is 401, unconfigured access is
503, invalid input is 400, export failure is 500, timeout is 504 and exceeding the
configured export size limit is 507. Other export failures retain sanitized
operator diagnostics in the server log; raw pg_dump stderr remains discarded.
Provision a private writable temporary directory (`TMPDIR` on Unix) with space
for roughly twice the SQL limit plus overhead. A small read-only-container tmpfs
is insufficient for a large backup. Exports include saved YAML verbatim and may
include keys/message data; the exclusions below still apply. This API does not
implement archive upload, import/overwrite, schedules or remote storage.
## Format 1
The tar contains exactly three regular files with fixed names:
+94
View File
@@ -338,6 +338,100 @@ const docTemplate = `{
}
}
},
"/admin/backup": {
"get": {
"security": [
{
"AdminKey": []
}
],
"description": "Opt-in operator export. Returns only a completed bundle; one export/transfer at a time. Contains secrets. Login sessions, external deployment files and import are outside this endpoint.",
"produces": [
"application/gzip"
],
"tags": [
"Admin"
],
"summary": "Download a private database and saved-config backup",
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "file"
},
"headers": {
"Content-Disposition": {
"type": "string",
"description": "attachment; filename=beacon-backup.tar.gz"
}
}
},
"400": {
"description": "Bad Request",
"schema": {
"type": "object",
"additionalProperties": {
"$ref": "#/definitions/internal_api_handlers.APIError"
}
}
},
"401": {
"description": "Unauthorized",
"schema": {
"type": "object",
"additionalProperties": {
"$ref": "#/definitions/internal_api_handlers.APIError"
}
}
},
"409": {
"description": "Conflict",
"schema": {
"type": "object",
"additionalProperties": {
"$ref": "#/definitions/internal_api_handlers.APIError"
}
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"type": "object",
"additionalProperties": {
"$ref": "#/definitions/internal_api_handlers.APIError"
}
}
},
"503": {
"description": "Service Unavailable",
"schema": {
"type": "object",
"additionalProperties": {
"$ref": "#/definitions/internal_api_handlers.APIError"
}
}
},
"504": {
"description": "Gateway Timeout",
"schema": {
"type": "object",
"additionalProperties": {
"$ref": "#/definitions/internal_api_handlers.APIError"
}
}
},
"507": {
"description": "Insufficient Storage",
"schema": {
"type": "object",
"additionalProperties": {
"$ref": "#/definitions/internal_api_handlers.APIError"
}
}
}
}
}
},
"/admin/config": {
"get": {
"security": [
+94
View File
@@ -336,6 +336,100 @@
}
}
},
"/admin/backup": {
"get": {
"security": [
{
"AdminKey": []
}
],
"description": "Opt-in operator export. Returns only a completed bundle; one export/transfer at a time. Contains secrets. Login sessions, external deployment files and import are outside this endpoint.",
"produces": [
"application/gzip"
],
"tags": [
"Admin"
],
"summary": "Download a private database and saved-config backup",
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "file"
},
"headers": {
"Content-Disposition": {
"type": "string",
"description": "attachment; filename=beacon-backup.tar.gz"
}
}
},
"400": {
"description": "Bad Request",
"schema": {
"type": "object",
"additionalProperties": {
"$ref": "#/definitions/internal_api_handlers.APIError"
}
}
},
"401": {
"description": "Unauthorized",
"schema": {
"type": "object",
"additionalProperties": {
"$ref": "#/definitions/internal_api_handlers.APIError"
}
}
},
"409": {
"description": "Conflict",
"schema": {
"type": "object",
"additionalProperties": {
"$ref": "#/definitions/internal_api_handlers.APIError"
}
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"type": "object",
"additionalProperties": {
"$ref": "#/definitions/internal_api_handlers.APIError"
}
}
},
"503": {
"description": "Service Unavailable",
"schema": {
"type": "object",
"additionalProperties": {
"$ref": "#/definitions/internal_api_handlers.APIError"
}
}
},
"504": {
"description": "Gateway Timeout",
"schema": {
"type": "object",
"additionalProperties": {
"$ref": "#/definitions/internal_api_handlers.APIError"
}
}
},
"507": {
"description": "Insufficient Storage",
"schema": {
"type": "object",
"additionalProperties": {
"$ref": "#/definitions/internal_api_handlers.APIError"
}
}
}
}
}
},
"/admin/config": {
"get": {
"security": [
+63
View File
@@ -1525,6 +1525,69 @@ paths:
summary: Get an operator account
tags:
- Admin
/admin/backup:
get:
description: Opt-in operator export. Returns only a completed bundle; one export/transfer
at a time. Contains secrets. Login sessions, external deployment files and
import are outside this endpoint.
produces:
- application/gzip
responses:
"200":
description: OK
headers:
Content-Disposition:
description: attachment; filename=beacon-backup.tar.gz
type: string
schema:
type: file
"400":
description: Bad Request
schema:
additionalProperties:
$ref: '#/definitions/internal_api_handlers.APIError'
type: object
"401":
description: Unauthorized
schema:
additionalProperties:
$ref: '#/definitions/internal_api_handlers.APIError'
type: object
"409":
description: Conflict
schema:
additionalProperties:
$ref: '#/definitions/internal_api_handlers.APIError'
type: object
"500":
description: Internal Server Error
schema:
additionalProperties:
$ref: '#/definitions/internal_api_handlers.APIError'
type: object
"503":
description: Service Unavailable
schema:
additionalProperties:
$ref: '#/definitions/internal_api_handlers.APIError'
type: object
"504":
description: Gateway Timeout
schema:
additionalProperties:
$ref: '#/definitions/internal_api_handlers.APIError'
type: object
"507":
description: Insufficient Storage
schema:
additionalProperties:
$ref: '#/definitions/internal_api_handlers.APIError'
type: object
security:
- AdminKey: []
summary: Download a private database and saved-config backup
tags:
- Admin
/admin/config:
get:
description: Returns current CORS options, auth configuration status and configured
+107
View File
@@ -0,0 +1,107 @@
// Copyright 2026 Beacon Contributors
// SPDX-License-Identifier: AGPL-3.0-or-later
package handlers
import (
"context"
"errors"
"io"
"log/slog"
"net/http"
"os"
"path/filepath"
"strconv"
"sync/atomic"
"time"
"github.com/MeshCore-Beacon/beacon-server/internal/backup"
"github.com/go-chi/chi/v5"
)
// BackupRouter returns an operator-only subrouter with one bounded export or
// transfer at a time. Its caller must apply the admin authentication middleware.
// The options are startup-owned; the request selects no targets or paths.
func BackupRouter(opts backup.Options) http.Handler {
r := chi.NewRouter()
r.Get("/", backupDownload(opts, backup.Export))
return r
}
// backupDownload godoc
//
// @Summary Download a private database and saved-config backup
// @Description Opt-in operator export. Returns only a completed bundle; one export/transfer at a time. Contains secrets. Login sessions, external deployment files and import are outside this endpoint.
// @Tags Admin
// @Produce application/gzip
// @Security AdminKey
// @Success 200 {file} binary
// @Header 200 {string} Content-Disposition "attachment; filename=beacon-backup.tar.gz"
// @Failure 400 {object} map[string]APIError
// @Failure 401 {object} map[string]APIError
// @Failure 409 {object} map[string]APIError
// @Failure 500 {object} map[string]APIError
// @Failure 503 {object} map[string]APIError
// @Failure 504 {object} map[string]APIError
// @Failure 507 {object} map[string]APIError
// @Router /admin/backup [get]
func backupDownload(opts backup.Options, export func(context.Context, backup.Options) error) http.HandlerFunc {
var busy atomic.Bool
return func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Cache-Control", "no-store")
if opts.ConnectionService == "" {
respondError(w, 503, "backup download is not configured")
return
}
if r.URL.RawQuery != "" || r.ContentLength != 0 || len(r.TransferEncoding) != 0 {
respondError(w, 400, "backup download accepts no query parameters or body")
return
}
if !busy.CompareAndSwap(false, true) {
respondError(w, 409, "a backup download is already in progress")
return
}
defer busy.Store(false)
dir, err := os.MkdirTemp("", "beacon-download-")
if err != nil {
respondError(w, 500, "cannot stage backup")
return
}
defer os.RemoveAll(dir)
requestOpts := opts
requestOpts.OutputPath = filepath.Join(dir, "backup.tar.gz")
if err := export(r.Context(), requestOpts); err != nil {
if errors.Is(err, backup.ErrTooLarge) {
respondError(w, http.StatusInsufficientStorage, "backup exceeds the configured export size limit")
} else if errors.Is(err, context.DeadlineExceeded) {
respondError(w, 504, "backup export timed out")
} else if r.Context().Err() == nil {
// Export discards pg_dump stderr and never returns connection settings.
slog.Error("backup export failed", "component", "backup", "error", err)
respondError(w, 500, "backup export failed; check private client configuration and capacity")
}
return
}
file, err := os.Open(requestOpts.OutputPath)
if err != nil {
respondError(w, 500, "cannot read completed backup")
return
}
defer file.Close()
info, err := file.Stat()
if err != nil {
respondError(w, 500, "cannot inspect completed backup")
return
}
// Keep slow downloads from retaining the only slot and private files
// indefinitely. The native server supports ResponseController deadlines.
controller := http.NewResponseController(w)
_ = controller.SetWriteDeadline(time.Now().Add(backup.DefaultTimeout))
defer controller.SetWriteDeadline(time.Time{})
w.Header().Set("Content-Type", "application/gzip")
w.Header().Set("Content-Disposition", `attachment; filename="beacon-backup.tar.gz"`)
w.Header().Set("X-Content-Type-Options", "nosniff")
w.Header().Set("Content-Length", strconv.FormatInt(info.Size(), 10))
_, _ = io.Copy(w, file)
}
}
+106
View File
@@ -0,0 +1,106 @@
// Copyright 2026 Beacon Contributors
// SPDX-License-Identifier: AGPL-3.0-or-later
package handlers
import (
"context"
"errors"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/MeshCore-Beacon/beacon-server/internal/backup"
)
func TestBackupDownload(t *testing.T) {
opts := backup.Options{ConnectionService: "synthetic-private-service", MaxBytes: 1024, Timeout: time.Second, ConfigPath: "saved.yaml"}
var output string
export := func(ctx context.Context, got backup.Options) error {
if got.ConfigPath != opts.ConfigPath || got.ConnectionService != opts.ConnectionService || got.MaxBytes != opts.MaxBytes || got.Timeout != opts.Timeout {
t.Fatal("startup options changed")
}
output = got.OutputPath
return os.WriteFile(output, []byte("completed fixture"), 0600)
}
handler := backupDownload(opts, export)
for _, query := range []string{"?database=other", "?path=/private", "?command=anything"} {
w := httptest.NewRecorder()
handler(w, httptest.NewRequest("GET", "/backup"+query, nil))
if w.Code != 400 || output != "" {
t.Fatal("caller-selected arguments accepted")
}
}
w := httptest.NewRecorder()
handler(w, httptest.NewRequest("GET", "/backup", strings.NewReader("body")))
if w.Code != 400 || output != "" {
t.Fatal("request body accepted")
}
w = httptest.NewRecorder()
handler(w, httptest.NewRequest("GET", "/backup", nil))
if w.Code != 200 || w.Body.String() != "completed fixture" || w.Header().Get("Content-Type") != "application/gzip" ||
w.Header().Get("Cache-Control") != "no-store" || w.Header().Get("Content-Length") != "17" || w.Header().Get("Content-Disposition") != `attachment; filename="beacon-backup.tar.gz"` {
t.Fatalf("unexpected download: %d %v", w.Code, w.Header())
}
if _, err := os.Stat(filepath.Dir(output)); !os.IsNotExist(err) {
t.Fatal("download staging retained")
}
for _, failure := range []error{errors.New("PRIVATE_CANARY"), context.DeadlineExceeded, backup.ErrTooLarge} {
handler = backupDownload(opts, func(_ context.Context, got backup.Options) error {
output = got.OutputPath
return failure
})
w = httptest.NewRecorder()
handler(w, httptest.NewRequest("GET", "/backup", nil))
want := 500
if errors.Is(failure, context.DeadlineExceeded) {
want = 504
}
if errors.Is(failure, backup.ErrTooLarge) {
want = 507
}
if w.Code != want || strings.Contains(w.Body.String(), "PRIVATE_CANARY") || w.Header().Get("Content-Disposition") != "" {
t.Fatal("failure leaked diagnostics or published a download")
}
if _, err := os.Stat(filepath.Dir(output)); !os.IsNotExist(err) {
t.Fatal("failed staging retained")
}
}
w = httptest.NewRecorder()
BackupRouter(backup.Options{}).ServeHTTP(w, httptest.NewRequest("GET", "/", nil))
if w.Code != 503 {
t.Fatal("unconfigured download enabled")
}
}
func TestBackupDownloadConcurrencyAndCancellation(t *testing.T) {
started, finished := make(chan struct{}), make(chan struct{})
var output string
handler := backupDownload(backup.Options{ConnectionService: "fixture"}, func(ctx context.Context, opts backup.Options) error {
output = opts.OutputPath
close(started)
<-ctx.Done()
return ctx.Err()
})
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go func() {
defer close(finished)
handler(httptest.NewRecorder(), httptest.NewRequest("GET", "/backup", nil).WithContext(ctx))
}()
<-started
w := httptest.NewRecorder()
handler(w, httptest.NewRequest("GET", "/backup", nil))
if w.Code != http.StatusConflict {
t.Fatal("concurrent export started")
}
cancel()
<-finished
if _, err := os.Stat(filepath.Dir(output)); !os.IsNotExist(err) {
t.Fatal("cancelled staging retained")
}
}
@@ -0,0 +1,172 @@
// Copyright 2026 Beacon Contributors
// SPDX-License-Identifier: AGPL-3.0-or-later
package router
import (
"archive/tar"
"bytes"
"compress/gzip"
"context"
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"io"
"net"
"net/http"
"net/http/httptest"
"net/url"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"testing"
"time"
"github.com/MeshCore-Beacon/beacon-server/db"
"github.com/MeshCore-Beacon/beacon-server/internal/api/handlers"
"github.com/MeshCore-Beacon/beacon-server/internal/backup"
"github.com/MeshCore-Beacon/beacon-server/internal/config"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
)
// This opt-in test only creates/restores/drops its own randomly named databases.
func TestBackupDownloadPostgres(t *testing.T) {
if os.Getenv("BEACON_BACKUP_TEST_POSTGRES") != "1" {
t.Skip("requires pg_dump/psql and an explicitly selected private PostgreSQL test server")
}
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()
admin, err := pgx.Connect(ctx, "")
if err != nil {
t.Fatal("cannot connect to private PostgreSQL")
}
defer admin.Close(context.Background())
sourceName, targetName := "beacon_download_"+strings.ToLower(rand.Text()), "beacon_download_"+strings.ToLower(rand.Text())
for _, name := range []string{sourceName, targetName} {
ident := pgx.Identifier{name}.Sanitize()
if _, err := admin.Exec(ctx, "CREATE DATABASE "+ident); err != nil {
t.Fatal("cannot create private fixture")
}
defer func() {
if _, err := admin.Exec(ctx, "DROP DATABASE "+ident+" WITH (FORCE)"); err != nil {
t.Error("cannot remove private fixture")
}
}()
}
cfg, err := pgxpool.ParseConfig("")
if err != nil {
t.Fatal("invalid fixture configuration")
}
cfg.ConnConfig.Database = sourceName
pool, err := pgxpool.NewWithConfig(ctx, cfg)
if err != nil {
t.Fatal("cannot open source fixture")
}
defer pool.Close()
if err := db.RunMigrations(ctx, pool); err != nil {
t.Fatal(err)
}
if _, err := pool.Exec(ctx, "CREATE TABLE download_fixture (id bigint GENERATED ALWAYS AS IDENTITY, message text); INSERT INTO download_fixture(message) VALUES ('café 雪')"); err != nil {
t.Fatal(err)
}
dsn := (&url.URL{Scheme: "postgres", Host: net.JoinHostPort(cfg.ConnConfig.Host, strconv.Itoa(int(cfg.ConnConfig.Port))),
User: url.UserPassword(cfg.ConnConfig.User, cfg.ConnConfig.Password), Path: "/" + sourceName,
RawQuery: "sslmode=disable&pool_max_conns=2"}).String()
service, err := backup.ConnectionService(dsn)
if err != nil {
t.Fatal(err)
}
dir := t.TempDir()
configPath := filepath.Join(dir, "config.yaml")
saved := []byte("# verbatim saved file\nbackup:\n enabled: true\n")
if err := os.WriteFile(configPath, saved, 0600); err != nil {
t.Fatal(err)
}
t.Setenv("TMPDIR", dir)
server := httptest.NewServer(New(nil, nil, nil, Options{
Auth: config.AuthConfig{APIKey: "fixture-key"},
AdminRoutes: map[string]http.Handler{"/backup": handlers.BackupRouter(backup.Options{
ConnectionService: service, ConfigPath: configPath, MaxBytes: 16 << 20, Timeout: 30 * time.Second, Version: "http-fixture",
})},
}))
defer server.Close()
request, _ := http.NewRequestWithContext(ctx, "GET", server.URL+"/api/v1/admin/backup", nil)
request.Header.Set("Authorization", "Bearer fixture-key")
response, err := server.Client().Do(request)
if err != nil {
t.Fatal(err)
}
defer response.Body.Close()
if response.StatusCode != 200 || response.Header.Get("Cache-Control") != "no-store" {
t.Fatalf("download status=%d", response.StatusCode)
}
gz, err := gzip.NewReader(response.Body)
if err != nil {
t.Fatal(err)
}
defer gz.Close()
tarReader, members := tar.NewReader(gz), map[string][]byte{}
for {
header, err := tarReader.Next()
if err == io.EOF {
break
}
if err != nil || header.Typeflag != tar.TypeReg || header.Mode != 0600 {
t.Fatal("invalid member")
}
members[header.Name], err = io.ReadAll(tarReader)
if err != nil {
t.Fatal(err)
}
}
if _, err := io.Copy(io.Discard, gz); err != nil {
t.Fatal(err)
}
var manifest backup.Manifest
if json.Unmarshal(members["manifest.json"], &manifest) != nil || len(members) != 3 || manifest.FormatVersion != 1 || !bytes.Equal(members["config.yaml"], saved) {
t.Fatal("invalid manifest/config")
}
for _, file := range manifest.Files {
sum := sha256.Sum256(members[file.Name])
if file.Size != int64(len(members[file.Name])) || file.SHA256 != hex.EncodeToString(sum[:]) {
t.Fatal("checksum mismatch")
}
}
if bytes.Contains(members["database.sql"], []byte("beacon_backup]")) {
t.Fatal("connection settings leaked into bundle")
}
restore := exec.CommandContext(ctx, "psql", "-X", "--set=ON_ERROR_STOP=on", "--single-transaction")
restore.Env = append(os.Environ(), "PGDATABASE="+targetName)
restore.Stdin = bytes.NewReader(members["database.sql"])
if err := restore.Run(); err != nil {
t.Fatal("cannot restore downloaded SQL into empty fixture")
}
targetCfg := admin.Config().Copy()
targetCfg.Database = targetName
target, err := pgx.ConnectConfig(ctx, targetCfg)
if err != nil {
t.Fatal("cannot connect to restored fixture")
}
defer target.Close(context.Background())
const fingerprint = `SELECT md5(string_agg(table_name||column_name||data_type, ',' ORDER BY table_name,ordinal_position)) FROM information_schema.columns WHERE table_schema='public'`
var before, after, message string
if pool.QueryRow(ctx, fingerprint).Scan(&before) != nil || target.QueryRow(ctx, fingerprint).Scan(&after) != nil || before != after {
t.Fatal("schema differs after restore")
}
if target.QueryRow(ctx, "SELECT message FROM download_fixture").Scan(&message) != nil || message != "café 雪" {
t.Fatal("wrong database or changed data")
}
var nextID int
if target.QueryRow(ctx, "INSERT INTO download_fixture(message) VALUES ('next') RETURNING id").Scan(&nextID) != nil || nextID != 2 {
t.Fatal("identity sequence changed")
}
paths, _ := filepath.Glob(filepath.Join(dir, "beacon-download-*"))
if len(paths) != 0 {
t.Fatal("private HTTP staging retained")
}
t.Log("HTTP bundle restored current Beacon schema and Unicode data; native client ignored ambient PGDATABASE")
}
+38
View File
@@ -0,0 +1,38 @@
// Copyright 2026 Beacon Contributors
// SPDX-License-Identifier: AGPL-3.0-or-later
package router
import (
"net/http"
"net/http/httptest"
"testing"
"github.com/MeshCore-Beacon/beacon-server/internal/api/handlers"
"github.com/MeshCore-Beacon/beacon-server/internal/backup"
"github.com/MeshCore-Beacon/beacon-server/internal/config"
)
func TestBackupAuthenticationBeforeExport(t *testing.T) {
for _, key := range []string{"", "fixture-key"} {
handler := New(nil, nil, nil, Options{Auth: config.AuthConfig{APIKey: key}, AdminRoutes: map[string]http.Handler{"/backup": handlers.BackupRouter(backup.Options{ConnectionService: "fixture"})}})
for _, token := range []string{"", "wrong", "fixture-key"} {
r := httptest.NewRequest("GET", "/api/v1/admin/backup?database=other", nil)
if token != "" {
r.Header.Set("Authorization", "Bearer "+token)
}
w := httptest.NewRecorder()
handler.ServeHTTP(w, r)
want := 503
if key != "" {
want = 401
if token == key {
want = 400
}
}
if w.Code != want || w.Header().Get("Cache-Control") != "no-store" {
t.Fatalf("status=%d want=%d", w.Code, want)
}
}
}
}
+40
View File
@@ -0,0 +1,40 @@
// Copyright 2026 Beacon Contributors
// SPDX-License-Identifier: AGPL-3.0-or-later
package backup
import (
"context"
"errors"
"fmt"
"os/exec"
"regexp"
"strconv"
)
var dumpVersion = regexp.MustCompile(`^pg_dump \(PostgreSQL\) ([0-9]+)(?:\.[0-9]+)+(?:[ \r\n]|$)`)
// CheckClient checks the client inside this runtime before enabling downloads.
// pg_dump cannot export a server newer than its own major version.
func CheckClient(ctx context.Context, serverVersion int) error {
output, err := exec.CommandContext(ctx, "pg_dump", "--version").Output()
if err != nil {
return errors.New("cannot run pg_dump --version; install the PostgreSQL client in this runtime")
}
return checkClientVersion(output, serverVersion)
}
func checkClientVersion(output []byte, serverVersion int) error {
match := dumpVersion.FindSubmatch(output)
if match == nil || serverVersion < 100000 {
return errors.New("cannot verify pg_dump and PostgreSQL server major versions")
}
client, err := strconv.Atoi(string(match[1]))
if err != nil {
return errors.New("cannot verify pg_dump major version")
}
if client < serverVersion/10000 {
return fmt.Errorf("pg_dump major %d is older than PostgreSQL server major %d; install a compatible client and restart Beacon", client, serverVersion/10000)
}
return nil
}
+31
View File
@@ -0,0 +1,31 @@
// Copyright 2026 Beacon Contributors
// SPDX-License-Identifier: AGPL-3.0-or-later
package backup
import (
"strings"
"testing"
)
func TestCheckClientVersion(t *testing.T) {
for _, tc := range []struct {
output string
server int
valid bool
}{
{"pg_dump (PostgreSQL) 16.10\n", 160010, true},
{"pg_dump (PostgreSQL) 17.6 (Debian 17.6)\n", 160010, true},
{"pg_dump (PostgreSQL) 16.10\n", 170006, false},
{"pg_dump (PostgreSQL) 16.10\n", 180000, false},
{"pg_dump (PostgreSQL) 18.0\n", 180000, true},
{"PRIVATE_CANARY", 160010, false},
{"pg_dump (PostgreSQL) 16.bad", 160010, false},
{"pg_dump (PostgreSQL) 16.10\n", 0, false},
} {
err := checkClientVersion([]byte(tc.output), tc.server)
if (err == nil) != tc.valid || (err != nil && strings.Contains(err.Error(), "PRIVATE_CANARY")) {
t.Fatalf("server %d: %v", tc.server, err)
}
}
}
+97
View File
@@ -0,0 +1,97 @@
// Copyright 2026 Beacon Contributors
// SPDX-License-Identifier: AGPL-3.0-or-later
package backup
import (
"errors"
"net/url"
"os"
"sort"
"strconv"
"strings"
"unicode"
)
// ConnectionService captures a single-host PostgreSQL URL for pg_dump. The
// result contains secrets and must only be written to private staging, never
// logged or passed as a process argument. Unsupported settings fail closed.
func ConnectionService(dsn string) (string, error) {
return connectionService(dsn, os.Environ())
}
func connectionService(dsn string, environ []string) (string, error) {
invalid := errors.New("backup requires a single-host PostgreSQL URL with explicit user and database and supported libpq settings")
u, err := url.Parse(dsn)
if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") || u.User == nil || u.User.Username() == "" ||
u.Hostname() == "" || strings.Contains(u.Hostname(), ",") || strings.TrimLeft(u.Path, "/") == "" || u.Fragment != "" {
return "", invalid
}
settings := map[string]string{"host": u.Hostname(), "port": "5432", "user": u.User.Username(),
"dbname": strings.TrimLeft(u.Path, "/"), "sslmode": "prefer", "gssencmode": "disable"}
// Match the pgx startup environment for the supported native client settings.
// Target fields come exclusively from the URL. Service indirection and newer
// protocol options need separate compatibility work, not a guessed translation.
envKeys := map[string]string{"PGPASSWORD": "password", "PGPASSFILE": "passfile", "PGAPPNAME": "application_name",
"PGCONNECT_TIMEOUT": "connect_timeout", "PGSSLMODE": "sslmode", "PGSSLKEY": "sslkey", "PGSSLCERT": "sslcert",
"PGSSLROOTCERT": "sslrootcert", "PGSSLPASSWORD": "sslpassword", "PGSSLSNI": "sslsni",
"PGTARGETSESSIONATTRS": "target_session_attrs", "PGOPTIONS": "options"}
for _, entry := range environ {
key, value, _ := strings.Cut(entry, "=")
if value == "" {
continue
}
switch key {
case "PGSERVICE", "PGSERVICEFILE", "PGSSLNEGOTIATION", "PGMINPROTOCOLVERSION", "PGMAXPROTOCOLVERSION", "PGTZ":
return "", invalid
}
if name := envKeys[key]; name != "" {
settings[name] = value
}
if key == "PGPORT" && u.Port() == "" {
settings["port"] = value
}
}
if port := u.Port(); port != "" {
settings["port"] = port
}
if port, err := strconv.Atoi(settings["port"]); err != nil || port < 1 || port > 65535 {
return "", invalid
}
if password, ok := u.User.Password(); ok {
settings["password"] = password
}
query, err := url.ParseQuery(u.RawQuery)
if err != nil {
return "", invalid
}
for key, values := range query {
if len(values) != 1 || values[0] == "" {
return "", invalid
}
switch key {
case "pool_max_conns", "pool_min_conns", "pool_min_idle_conns", "pool_max_conn_lifetime", "pool_max_conn_idle_time", "pool_health_check_period", "pool_max_conn_lifetime_jitter":
continue // pgx-only pool controls; main has already validated them.
case "sslmode", "sslcert", "sslkey", "sslrootcert", "sslpassword", "sslsni", "passfile", "connect_timeout", "application_name", "target_session_attrs", "options":
settings[key] = values[0]
default:
return "", invalid
}
}
var result strings.Builder
result.WriteString("[beacon_backup]\n")
keys := make([]string, 0, len(settings))
for key, value := range settings {
// libpq service files have line-based, unquoted values. Reject anything
// their parser would trim or interpret as another entry.
if strings.TrimSpace(value) != value || strings.IndexFunc(value, unicode.IsControl) >= 0 || len(value) > 1000 {
return "", invalid
}
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
result.WriteString(key + "=" + settings[key] + "\n")
}
return result.String(), nil
}
+48
View File
@@ -0,0 +1,48 @@
// Copyright 2026 Beacon Contributors
// SPDX-License-Identifier: AGPL-3.0-or-later
package backup
import (
"strings"
"testing"
)
func TestConnectionService(t *testing.T) {
service, err := connectionService("postgresql://user:p%23ass%3Dword@[::1]:5544/db%20name?sslmode=verify-full&sslrootcert=%2Fprivate%2Fca.pem&passfile=%2Fprivate%2Fpass&pool_max_conns=2", []string{
"PGHOST=wrong", "PGDATABASE=wrong", "PGUSER=wrong", "PGPASSWORD=wrong", "PGSSLMODE=disable", "PGPASSFILE=/wrong", "PGSSLKEY=/private/key.pem"})
if err != nil {
t.Fatal(err)
}
for _, line := range []string{"[beacon_backup]", "host=::1", "port=5544", "user=user", "password=p#ass=word", "dbname=db name",
"sslmode=verify-full", "sslrootcert=/private/ca.pem", "passfile=/private/pass", "sslkey=/private/key.pem"} {
if !strings.Contains(service, line+"\n") {
t.Fatalf("missing expected setting %q", line)
}
}
if strings.Contains(service, "wrong") || strings.Contains(service, "pool_") {
t.Fatal("unrelated target or pool settings leaked into libpq")
}
service, err = connectionService("postgres://user@host/db", []string{"PGPASSWORD=from-environment", "PGPORT=5555", "PGSSLMODE=require"})
if err != nil || !strings.Contains(service, "password=from-environment\n") || !strings.Contains(service, "port=5555\n") || !strings.Contains(service, "sslmode=require\n") {
t.Fatal("supported environment defaults were not captured")
}
}
func TestConnectionServiceRejectsAmbiguousSettings(t *testing.T) {
for _, dsn := range []string{
"host=host user=user dbname=db", "postgres://host/db", "postgres://user@host/", "postgres://user@a,b/db",
"postgres://user@host/db?service=other", "postgres://user@host/db?host=other", "postgres://user@host/db?sslmode=require&sslmode=disable",
"postgres://user@host/db?unknown=x", "postgres://user:secret%0Ahost%3Dother@host/db", "postgres://user:%20secret@host/db",
"postgres://user@host:99999/db", "postgres://user@host/db#fragment", "postgres://user@host/db?sslnegotiation=direct",
} {
if _, err := connectionService(dsn, nil); err == nil || strings.Contains(err.Error(), "secret") {
t.Fatal("unsupported connection must fail without echoing input")
}
}
for _, env := range []string{"PGSERVICE=other", "PGSERVICEFILE=/private", "PGSSLNEGOTIATION=direct", "PGMINPROTOCOLVERSION=3.0", "PGTZ=UTC", "PGPORT=5432,5433"} {
if _, err := connectionService("postgres://user@host/db", []string{env}); err == nil {
t.Fatal("unsupported ambient connection settings accepted")
}
}
}
+20 -1
View File
@@ -17,6 +17,7 @@ import (
"os"
"os/exec"
"path/filepath"
"strings"
"time"
)
@@ -36,6 +37,9 @@ type Options struct {
MaxBytes int64 // Maximum uncompressed database dump size.
Timeout time.Duration
Version string
// ConnectionService overrides PG* using a private service file. Construct it
// with ConnectionService at startup; empty retains the standalone PG* mode.
ConnectionService string
}
// Manifest describes format 1; hashes cover the uncompressed archive members.
@@ -72,7 +76,7 @@ func export(ctx context.Context, opts Options, command func(context.Context) *ex
if opts.ConfigPath == "" || opts.OutputPath == "" || opts.MaxBytes <= 0 || opts.MaxBytes > 1<<40 || opts.Timeout <= 0 {
return errors.New("config, output, positive timeout and max-bytes (at most 1 TiB) are required")
}
if os.Getenv("PGDATABASE") == "" {
if opts.ConnectionService == "" && os.Getenv("PGDATABASE") == "" {
return errors.New("PGDATABASE must explicitly name the database; POSTGRES_DSN is not used")
}
ctx, cancel := context.WithTimeout(ctx, opts.Timeout)
@@ -104,6 +108,21 @@ func export(ctx context.Context, opts Options, command func(context.Context) *ex
hash := sha256.New()
output := &limitedWriter{ctx: ctx, cancel: cancel, dst: io.MultiWriter(dump, hash), remaining: opts.MaxBytes}
cmd := command(ctx)
if opts.ConnectionService != "" {
servicePath := filepath.Join(temp, "pg_service.conf")
if err := os.WriteFile(servicePath, []byte(opts.ConnectionService), 0600); err != nil {
return errors.New("cannot prepare private backup connection")
}
// Do not let unrelated PG* values redirect the client, and never mutate
// the Beacon process environment while another request is running.
cmd.Env = nil
for _, entry := range os.Environ() {
if !strings.HasPrefix(strings.ToUpper(entry), "PG") {
cmd.Env = append(cmd.Env, entry)
}
}
cmd.Env = append(cmd.Env, "PGSERVICEFILE="+servicePath, "PGSERVICE=beacon_backup")
}
cmd.Stdout, cmd.Stderr = output, io.Discard
cmd.WaitDelay = time.Second
err = cmd.Run()
+6
View File
@@ -18,6 +18,7 @@ import (
// Config is the top-level structure of the Beacon config file.
type Config struct {
Auth AuthConfig `yaml:"auth"`
Backup BackupConfig `yaml:"backup"`
Log LogConfig `yaml:"log"`
Server ServerConfig `yaml:"server"`
IATAs map[string]IATAConfig `yaml:"iatas"`
@@ -38,6 +39,11 @@ type Config struct {
Observers ObserversConfig `yaml:"observers"`
}
// BackupConfig enables protected downloads. Disabled by default.
type BackupConfig struct {
Enabled bool `yaml:"enabled"`
}
// AuthConfig holds the operator key for the protected admin subtree.
// The key is excluded from JSON; it must not be exposed by configuration APIs.
type AuthConfig struct {