mirror of
https://github.com/MeshCore-Beacon/beacon-server.git
synced 2026-10-16 00:49:57 +00:00
feat(backup): verify native archives offline without extraction (#160)
This commit is contained in:
+67
-14
@@ -1,13 +1,15 @@
|
||||
// Copyright 2026 Beacon Contributors
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
// beacon-backup creates a private database and saved-config export.
|
||||
// beacon-backup creates or verifies a private database and saved-config export.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
@@ -18,22 +20,73 @@ import (
|
||||
var version = "dev"
|
||||
|
||||
func main() {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
code := run(ctx, os.Args[1:], os.Stdout, os.Stderr)
|
||||
stop()
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
func run(ctx context.Context, args []string, stdout, stderr io.Writer) int {
|
||||
var opts backup.Options
|
||||
flag.StringVar(&opts.ConfigPath, "config", "config.yaml", "saved YAML file to include verbatim (may contain secrets)")
|
||||
flag.StringVar(&opts.OutputPath, "output", "", "new private .tar.gz destination (required; never overwritten)")
|
||||
flag.Int64Var(&opts.MaxBytes, "max-bytes", backup.DefaultMaxBytes, "maximum uncompressed database dump size")
|
||||
flag.DurationVar(&opts.Timeout, "timeout", backup.DefaultTimeout, "maximum export duration")
|
||||
flag.Parse()
|
||||
if flag.NArg() != 0 {
|
||||
fmt.Fprintln(os.Stderr, "unexpected positional arguments; connection settings use PG* environment variables")
|
||||
os.Exit(2)
|
||||
var verifyPath string
|
||||
flags := flag.NewFlagSet("beacon-backup", flag.ContinueOnError)
|
||||
flags.SetOutput(stderr)
|
||||
flags.StringVar(&opts.ConfigPath, "config", "config.yaml", "saved YAML file to export verbatim (may contain secrets)")
|
||||
flags.StringVar(&opts.OutputPath, "output", "", "new private .tar.gz export destination (never overwritten)")
|
||||
flags.StringVar(&verifyPath, "verify", "", "verify an existing .tar.gz without extraction or database access")
|
||||
flags.Int64Var(&opts.MaxBytes, "max-bytes", backup.DefaultMaxBytes, "maximum uncompressed database dump size")
|
||||
flags.DurationVar(&opts.Timeout, "timeout", backup.DefaultTimeout, "maximum export or verification duration")
|
||||
if err := flags.Parse(args); err != nil {
|
||||
if errors.Is(err, flag.ErrHelp) {
|
||||
return 0
|
||||
}
|
||||
return 2
|
||||
}
|
||||
if flags.NArg() != 0 || opts.Timeout <= 0 {
|
||||
fmt.Fprintln(stderr, "positive timeout and no positional arguments required")
|
||||
return 2
|
||||
}
|
||||
verifyMode, exportFlag := false, false
|
||||
flags.Visit(func(f *flag.Flag) {
|
||||
verifyMode = verifyMode || f.Name == "verify"
|
||||
exportFlag = exportFlag || f.Name == "config" || f.Name == "output"
|
||||
})
|
||||
if verifyMode {
|
||||
if verifyPath == "" || exportFlag {
|
||||
fmt.Fprintln(stderr, "verify requires an archive path and cannot be combined with config or output")
|
||||
return 2
|
||||
}
|
||||
// Reject directories/devices/FIFOs before opening, and check the opened
|
||||
// descriptor too. A symlink to a regular, operator-selected file is fine.
|
||||
info, err := os.Stat(verifyPath)
|
||||
if err != nil || !info.Mode().IsRegular() {
|
||||
fmt.Fprintln(stderr, "verification requires a readable regular archive file")
|
||||
return 1
|
||||
}
|
||||
file, err := os.Open(verifyPath)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "cannot open backup archive")
|
||||
return 1
|
||||
}
|
||||
defer file.Close()
|
||||
if info, err = file.Stat(); err != nil || !info.Mode().IsRegular() {
|
||||
fmt.Fprintln(stderr, "verification requires a regular archive file")
|
||||
return 1
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, opts.Timeout)
|
||||
defer cancel()
|
||||
if _, err := backup.Verify(ctx, file, opts.MaxBytes); err != nil {
|
||||
fmt.Fprintln(stderr, "verification failed:", err)
|
||||
return 1
|
||||
}
|
||||
fmt.Fprintln(stdout, "Archive structure and checksums verified. No files extracted or SQL executed.")
|
||||
return 0
|
||||
}
|
||||
opts.Version = version
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
if err := backup.Export(ctx, opts); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "backup failed:", err)
|
||||
os.Exit(1)
|
||||
fmt.Fprintln(stderr, "backup failed:", err)
|
||||
return 1
|
||||
}
|
||||
fmt.Println("Backup complete. Store this bundle privately; it may contain keys and message data.")
|
||||
fmt.Fprintln(stdout, "Backup complete. Store this bundle privately; it may contain keys and message data.")
|
||||
return 0
|
||||
}
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
// Copyright 2026 Beacon Contributors
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/MeshCore-Beacon/beacon-server/internal/backup"
|
||||
)
|
||||
|
||||
func TestVerifyCommandIsReadOnlyAndOffline(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
t.Setenv("PATH", dir) // no pg_dump, psql or shell helper is available
|
||||
t.Setenv("PGDATABASE", "")
|
||||
archive := filepath.Join(dir, "private.tar.gz")
|
||||
m := backup.Manifest{FormatVersion: 1, CreatedAt: time.Now().UTC(), DatabaseFormat: "postgresql-plain-sql", ToolVersion: "PRIVATE_CANARY"}
|
||||
data := map[string][]byte{"database.sql": []byte("SELECT 'PRIVATE_CANARY';"), "config.yaml": {}}
|
||||
for _, name := range []string{"database.sql", "config.yaml"} {
|
||||
hash := sha256.Sum256(data[name])
|
||||
m.Files = append(m.Files, backup.File{Name: name, Size: int64(len(data[name])), SHA256: hex.EncodeToString(hash[:])})
|
||||
}
|
||||
data["manifest.json"], _ = json.Marshal(m)
|
||||
var raw bytes.Buffer
|
||||
gz := gzip.NewWriter(&raw)
|
||||
tw := tar.NewWriter(gz)
|
||||
for _, name := range []string{"manifest.json", "database.sql", "config.yaml"} {
|
||||
if err := tw.WriteHeader(&tar.Header{Name: name, Size: int64(len(data[name])), Mode: 0600}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := tw.Write(data[name]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := tw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := gz.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(archive, raw.Bytes(), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
args []string
|
||||
code int
|
||||
}{
|
||||
{[]string{"-verify", archive}, 0},
|
||||
{[]string{"-verify", archive, "-max-bytes", "1"}, 1},
|
||||
{[]string{"-verify", archive, "-timeout", "0"}, 2},
|
||||
{[]string{"-verify", archive, "-output", filepath.Join(dir, "must-not-exist")}, 2},
|
||||
{[]string{"-verify", archive, "-config", "missing.yaml"}, 2},
|
||||
{[]string{"-verify", "", "-output", filepath.Join(dir, "must-not-exist")}, 2},
|
||||
{[]string{"-verify", dir}, 1},
|
||||
{[]string{"-verify", filepath.Join(dir, "PRIVATE_CANARY-missing")}, 1},
|
||||
{[]string{"-verify", archive, "unexpected"}, 2},
|
||||
} {
|
||||
var stdout, stderr bytes.Buffer
|
||||
if got := run(context.Background(), tc.args, &stdout, &stderr); got != tc.code || strings.Contains(stdout.String()+stderr.String(), "PRIVATE_CANARY") {
|
||||
t.Fatalf("exit %d, want %d; diagnostic %s", got, tc.code, stderr.String())
|
||||
}
|
||||
}
|
||||
files, err := os.ReadDir(dir)
|
||||
if err != nil || len(files) != 1 || files[0].Name() != "private.tar.gz" {
|
||||
t.Fatal("verification extracted files or created output")
|
||||
}
|
||||
unchanged, err := os.ReadFile(archive)
|
||||
if err != nil || !bytes.Equal(unchanged, raw.Bytes()) {
|
||||
t.Fatal("input archive changed")
|
||||
}
|
||||
}
|
||||
+47
-14
@@ -1,13 +1,14 @@
|
||||
# Database and saved-config export
|
||||
# Database and saved-config export and verification
|
||||
|
||||
`beacon-backup` is the export foundation for issue #72. It produces a private,
|
||||
versioned `.tar.gz` using PostgreSQL's `pg_dump`. It does not yet provide a web
|
||||
interface, account login, scheduled/remote storage or automatic import.
|
||||
`beacon-backup` is the export and archive-validation foundation for issue #72. It
|
||||
produces a private, versioned `.tar.gz` using PostgreSQL's `pg_dump` and can verify
|
||||
an existing bundle offline. It does not yet provide a web interface, account
|
||||
login, scheduled/remote storage or automatic import.
|
||||
|
||||
Build it with `go build ./cmd/beacon-backup`. Install `pg_dump` in the same runtime
|
||||
as this command; an installation on the Docker host does not install it inside an
|
||||
app container. Use a client of the same major version as the source PostgreSQL
|
||||
server; an older client cannot dump a newer server.
|
||||
Build it with `go build ./cmd/beacon-backup`. For exports, install `pg_dump` in the
|
||||
same runtime as this command; an installation on the Docker host does not install
|
||||
it inside an app container. Use a client of the same major version as the source
|
||||
PostgreSQL server; an older client cannot dump a newer server.
|
||||
|
||||
Set libpq's standard `PGHOST`, `PGPORT`, `PGDATABASE`, `PGUSER` and TLS settings.
|
||||
`PGDATABASE` is required and must be a plain database name. Prefer a private
|
||||
@@ -115,12 +116,43 @@ waiting indefinitely behind schema changes. Export failure publishes no backup;
|
||||
check the client version, privileges, connection settings and available capacity
|
||||
privately. The command deliberately does not expose raw client diagnostics.
|
||||
|
||||
## Verify an existing archive
|
||||
|
||||
```sh
|
||||
beacon-backup -verify /private/beacon-20260913.tar.gz
|
||||
```
|
||||
|
||||
Verification reads a regular file without extracting members, writing files,
|
||||
executing SQL or connecting to PostgreSQL, Redis or MQTT. It needs neither a
|
||||
configuration file nor `pg_dump` or connection credentials. Do not combine
|
||||
`-verify` with the export-only `-config` or `-output` flags.
|
||||
|
||||
The verifier accepts the native format-1 bundle: exactly the three fixed regular
|
||||
members with 0600 modes and USTAR headers (or the size-only PAX header needed for
|
||||
SQL over 8 GiB). It rejects duplicates, unexpected paths, links, devices, other
|
||||
extended metadata, incomplete tar/gzip endings and trailing or concatenated
|
||||
streams. Manifest fields must be known, with no duplicate keys or case aliases;
|
||||
the payload sizes and SHA-256 hashes must match the bytes actually read.
|
||||
|
||||
The same `-max-bytes` SQL limit applies, with a separate 1 MiB config limit and
|
||||
64 KiB manifest limit. Compressed input is capped at the SQL limit plus 1% and
|
||||
2 MiB, matching the exporter. Headers and decompressed data are bounded too;
|
||||
SQL and config are hashed as streams, without keeping their contents in memory.
|
||||
The ten-minute default `-timeout` and interrupt handling check cancellation
|
||||
between reads; they cannot interrupt a blocked filesystem read.
|
||||
|
||||
Exit status 0 means the archive structure and checksums passed. Failure returns
|
||||
a nonzero status with no payload, manifest values or input path in its diagnostic.
|
||||
The manifest is not signed: verification does **not** establish authenticity,
|
||||
safe SQL, configuration validity or restorability. It does not make an untrusted
|
||||
dump safe to restore, and does not provide import/overwrite or deployment recovery.
|
||||
|
||||
## Restore verification
|
||||
|
||||
Use trusted bundles only: PostgreSQL dumps can contain executable SQL. Inspect
|
||||
the fixed members, verify the gzip stream and the manifest's payload sizes and
|
||||
hashes, and extract into private staging. For a **new, empty disposable database**,
|
||||
with its own explicit `PGDATABASE` and target-role connection settings:
|
||||
Use trusted bundles only: PostgreSQL dumps can contain executable SQL. Run the
|
||||
archive verification above, then extract into private staging. For a
|
||||
**new, empty disposable database**, with its own explicit `PGDATABASE` and
|
||||
target-role connection settings:
|
||||
|
||||
```sh
|
||||
psql -X --set ON_ERROR_STOP=on --single-transaction --file database.sql
|
||||
@@ -137,8 +169,9 @@ PostgreSQL references: [pg_dump](https://www.postgresql.org/docs/16/app-pgdump.h
|
||||
[password files](https://www.postgresql.org/docs/16/libpq-pgpass.html).
|
||||
|
||||
CI runs the compiled command and its PostgreSQL round-trip test with a dedicated
|
||||
PostgreSQL 16 service. To repeat it privately, build the command, set the `PG*`
|
||||
connection variables for an isolated test server and set
|
||||
PostgreSQL 16 service. The test also verifies the export offline with no client on
|
||||
PATH or connection settings, and rejects a truncated copy. To repeat it privately,
|
||||
build the command, set the `PG*` connection variables for an isolated test server and set
|
||||
`BEACON_BACKUP_TEST_POSTGRES=1` plus `BEACON_BACKUP_TEST_BINARY` to the command's
|
||||
absolute path. Run `go test ./internal/backup -run '^TestExportPostgres$' -v`.
|
||||
The test role needs permission to create/drop its two randomly named databases;
|
||||
|
||||
@@ -115,6 +115,19 @@ CREATE MATERIALIZED VIEW backup_materialized AS SELECT count(*) AS count FROM ba
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
verify := exec.CommandContext(ctx, binary, "-verify", output, "-max-bytes", "16777216", "-timeout", "30s")
|
||||
verify.Env = []string{"PATH=" + dir} // verification needs no PostgreSQL client or connection
|
||||
message, err := verify.CombinedOutput()
|
||||
if err != nil || !bytes.Contains(message, []byte("structure and checksums verified")) || bytes.Contains(message, []byte("café")) {
|
||||
t.Fatal("exact command could not verify its export offline")
|
||||
}
|
||||
corrupt := filepath.Join(dir, "corrupt.tar.gz")
|
||||
if err := os.WriteFile(corrupt, original[:len(original)-1], 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := exec.CommandContext(ctx, binary, "-verify", corrupt).Run(); err == nil {
|
||||
t.Fatal("exact command accepted a truncated export")
|
||||
}
|
||||
for _, failure := range []string{"existing", "size", "connection", "timeout"} {
|
||||
t.Run(failure, func(t *testing.T) {
|
||||
failedOutput := filepath.Join(dir, failure+".tar.gz")
|
||||
@@ -212,5 +225,5 @@ CREATE MATERIALIZED VIEW backup_materialized AS SELECT count(*) AS count FROM ba
|
||||
if err := target.QueryRow(ctx, "INSERT INTO backup_fixture (message) VALUES ('next') RETURNING id").Scan(&nextID); err != nil || nextID != 3 {
|
||||
t.Fatal("identity sequence did not survive restore")
|
||||
}
|
||||
t.Log("exact command restored all migrations, columns, Unicode/bytea/JSON data, relationships, views and identity sequence")
|
||||
t.Log("exact command verified its export offline and restored all migrations, columns, Unicode/bytea/JSON data, relationships, views and identity sequence")
|
||||
}
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
// Copyright 2026 Beacon Contributors
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package backup
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bufio"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const maxManifestBytes = 64 << 10
|
||||
|
||||
var ErrInvalidArchive = errors.New("invalid backup archive")
|
||||
|
||||
// Verify streams a format-1 bundle without extracting files or executing SQL.
|
||||
// maxBytes bounds the SQL payload (up to 1 TiB); the saved YAML and manifest have
|
||||
// separate fixed limits. The caller owns input and its deadline. Cancellation is
|
||||
// checked between reads; the supplied Reader must not block indefinitely.
|
||||
// The returned manifest is untrusted metadata, not proof of origin or restorability.
|
||||
func Verify(ctx context.Context, input io.Reader, maxBytes int64) (*Manifest, error) {
|
||||
if input == nil || maxBytes <= 0 || maxBytes > 1<<40 {
|
||||
return nil, errors.New("input and positive max-bytes (at most 1 TiB) are required")
|
||||
}
|
||||
// Match the exporter's compressed-output allowance. ByteReader prevents gzip
|
||||
// from swallowing a following stream before we can reject it.
|
||||
compressed := &verifyReader{ctx: ctx, r: input, limit: maxBytes + maxBytes/100 + 2*maxConfigBytes}
|
||||
buffered := bufio.NewReader(compressed)
|
||||
gz, err := gzip.NewReader(buffered)
|
||||
if err != nil {
|
||||
return nil, verificationError(ctx, err, "gzip header")
|
||||
}
|
||||
defer gz.Close()
|
||||
gz.Multistream(false)
|
||||
decoded := &verifyReader{ctx: ctx, r: gz}
|
||||
tr := tar.NewReader(decoded)
|
||||
files := make(map[string]File, 3)
|
||||
var metadata []byte
|
||||
var padding int64
|
||||
for {
|
||||
before := decoded.n
|
||||
// Native headers are USTAR, or one size-only PAX record for SQL >8 GiB.
|
||||
// Bound Next itself: it otherwise hides and allocates extension records.
|
||||
decoded.limit = before + padding + 3*512
|
||||
h, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
// tar.Reader also accepts EOF without both closing zero blocks.
|
||||
if decoded.n-before != padding+2*512 {
|
||||
return nil, verificationError(ctx, nil, "missing tar terminator")
|
||||
}
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, verificationError(ctx, err, "tar header")
|
||||
}
|
||||
headerBytes := decoded.n - before - padding
|
||||
ordinary := h.Format == tar.FormatUSTAR && headerBytes == 512 && len(h.PAXRecords) == 0
|
||||
large := h.Format == tar.FormatPAX && headerBytes == 3*512 && len(h.PAXRecords) == 1 && h.PAXRecords["size"] == strconv.FormatInt(h.Size, 10)
|
||||
if (!ordinary && !large) || (h.Typeflag != tar.TypeReg && h.Typeflag != tar.TypeRegA) || h.Linkname != "" || h.Mode != 0600 {
|
||||
return nil, verificationError(ctx, nil, "unsupported member type or metadata")
|
||||
}
|
||||
if _, duplicate := files[h.Name]; duplicate || len(files) == 3 {
|
||||
return nil, verificationError(ctx, nil, "duplicate or extra member")
|
||||
}
|
||||
var limit int64
|
||||
switch h.Name {
|
||||
case "manifest.json":
|
||||
limit = maxManifestBytes
|
||||
case "database.sql":
|
||||
limit = maxBytes
|
||||
if h.Size == 0 {
|
||||
return nil, verificationError(ctx, nil, "empty database dump")
|
||||
}
|
||||
case "config.yaml":
|
||||
limit = maxConfigBytes
|
||||
default:
|
||||
return nil, verificationError(ctx, nil, "unexpected member name")
|
||||
}
|
||||
if h.Size < 0 || h.Size > limit {
|
||||
return nil, ErrTooLarge
|
||||
}
|
||||
decoded.limit = decoded.n + h.Size
|
||||
digest := sha256.New()
|
||||
if h.Name == "manifest.json" {
|
||||
metadata, err = io.ReadAll(tr)
|
||||
} else {
|
||||
_, err = io.Copy(digest, tr)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, verificationError(ctx, err, "member data")
|
||||
}
|
||||
files[h.Name] = File{Name: h.Name, Size: h.Size, SHA256: hex.EncodeToString(digest.Sum(nil))}
|
||||
padding = (512 - h.Size%512) % 512
|
||||
}
|
||||
// Read through the gzip checksum/trailer, but allow no data after tar EOF.
|
||||
decoded.limit = decoded.n
|
||||
var tail [1]byte
|
||||
if _, err := decoded.Read(tail[:]); err != io.EOF {
|
||||
return nil, verificationError(ctx, err, "gzip completion or trailing tar data")
|
||||
}
|
||||
if _, err := buffered.ReadByte(); err != io.EOF {
|
||||
return nil, verificationError(ctx, err, "trailing compressed data")
|
||||
}
|
||||
if len(files) != 3 || uniqueManifestKeys(json.NewDecoder(bytes.NewReader(metadata)), 0) != nil {
|
||||
return nil, verificationError(ctx, nil, "missing members or invalid manifest")
|
||||
}
|
||||
var manifest Manifest
|
||||
decoder := json.NewDecoder(bytes.NewReader(metadata))
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(&manifest); err != nil || decoder.Decode(new(any)) != io.EOF || manifest.FormatVersion != 1 || manifest.DatabaseFormat != "postgresql-plain-sql" || manifest.CreatedAt.IsZero() || len(manifest.Files) != 2 {
|
||||
return nil, verificationError(ctx, nil, "unsupported or incomplete manifest")
|
||||
}
|
||||
for _, declared := range manifest.Files {
|
||||
actual, exists := files[declared.Name]
|
||||
digest, err := hex.DecodeString(declared.SHA256)
|
||||
if !exists || declared.Name == "manifest.json" || declared.Size != actual.Size || err != nil || len(digest) != sha256.Size || !strings.EqualFold(declared.SHA256, actual.SHA256) {
|
||||
return nil, verificationError(ctx, nil, "manifest size or checksum mismatch")
|
||||
}
|
||||
delete(files, declared.Name) // duplicate manifest file records fail as well.
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &manifest, nil
|
||||
}
|
||||
|
||||
// Avoid returning filenames, manifest values, reader errors or payload text.
|
||||
func verificationError(ctx context.Context, err error, stage string) error {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
if errors.Is(err, ErrTooLarge) {
|
||||
return ErrTooLarge
|
||||
}
|
||||
return fmt.Errorf("%w: %s", ErrInvalidArchive, stage)
|
||||
}
|
||||
|
||||
type verifyReader struct {
|
||||
ctx context.Context
|
||||
r io.Reader
|
||||
n, limit int64
|
||||
}
|
||||
|
||||
func (r *verifyReader) Read(p []byte) (int, error) {
|
||||
if err := r.ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if r.n > r.limit {
|
||||
return 0, ErrTooLarge
|
||||
}
|
||||
if int64(len(p)) > r.limit-r.n+1 {
|
||||
p = p[:r.limit-r.n+1]
|
||||
}
|
||||
n, err := r.r.Read(p)
|
||||
r.n += int64(n)
|
||||
if r.n > r.limit {
|
||||
return 0, ErrTooLarge
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
// encoding/json accepts duplicate keys and case aliases. Reject both before
|
||||
// decoding the fixed schema, so another reader cannot interpret different values.
|
||||
func uniqueManifestKeys(d *json.Decoder, depth int) error {
|
||||
if depth > 8 {
|
||||
return ErrInvalidArchive
|
||||
}
|
||||
token, err := d.Token()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if delim, ok := token.(json.Delim); ok {
|
||||
keys := map[string]bool{}
|
||||
for d.More() {
|
||||
if delim == '{' {
|
||||
key, err := d.Token()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
name, ok := key.(string)
|
||||
// Schema keys are ASCII. Unicode simple-fold aliases (such as
|
||||
// long-s for s) also match encoding/json fields, even in lowercase.
|
||||
if !ok || keys[name] || strings.IndexFunc(name, func(r rune) bool { return r > 127 || r >= 'A' && r <= 'Z' }) >= 0 {
|
||||
return ErrInvalidArchive
|
||||
}
|
||||
keys[name] = true
|
||||
}
|
||||
if err := uniqueManifestKeys(d, depth+1); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
_, err = d.Token()
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,257 @@
|
||||
// Copyright 2026 Beacon Contributors
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package backup
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type verifyEntry struct {
|
||||
name string
|
||||
data []byte
|
||||
kind byte
|
||||
pax map[string]string
|
||||
}
|
||||
|
||||
func verifyFixture(t testing.TB) []verifyEntry {
|
||||
t.Helper()
|
||||
sql, config := []byte(testSQL), []byte("secret: PRIVATE_CANARY\n")
|
||||
m := Manifest{FormatVersion: 1, CreatedAt: time.Now().UTC(), ToolVersion: "fixture", DatabaseFormat: "postgresql-plain-sql", Excluded: []string{"deployment_environment"}}
|
||||
for _, entry := range []verifyEntry{{name: "database.sql", data: sql}, {name: "config.yaml", data: config}} {
|
||||
hash := sha256.Sum256(entry.data)
|
||||
m.Files = append(m.Files, File{Name: entry.name, Size: int64(len(entry.data)), SHA256: hex.EncodeToString(hash[:])})
|
||||
}
|
||||
metadata, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return []verifyEntry{{name: "manifest.json", data: metadata}, {name: "database.sql", data: sql}, {name: "config.yaml", data: config}}
|
||||
}
|
||||
|
||||
func verifyTar(t testing.TB, entries []verifyEntry) []byte {
|
||||
t.Helper()
|
||||
var data bytes.Buffer
|
||||
tw := tar.NewWriter(&data)
|
||||
for _, e := range entries {
|
||||
h := &tar.Header{Name: e.name, Size: int64(len(e.data)), Mode: 0600, Typeflag: e.kind}
|
||||
if e.kind == tar.TypeSymlink || e.kind == tar.TypeLink {
|
||||
h.Linkname = "../PRIVATE_CANARY"
|
||||
}
|
||||
if e.pax != nil {
|
||||
h.Format, h.PAXRecords = tar.FormatPAX, e.pax
|
||||
}
|
||||
if err := tw.WriteHeader(h); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := tw.Write(e.data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := tw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return data.Bytes()
|
||||
}
|
||||
|
||||
func verifyGzip(t testing.TB, data []byte) []byte {
|
||||
t.Helper()
|
||||
var encoded bytes.Buffer
|
||||
gz := gzip.NewWriter(&encoded)
|
||||
if _, err := gz.Write(data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := gz.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return encoded.Bytes()
|
||||
}
|
||||
|
||||
func TestVerifyExportAndReorderedMembers(t *testing.T) {
|
||||
opts := setup(t)
|
||||
if err := export(context.Background(), opts, helper(t, "ok")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
original, err := os.ReadFile(opts.OutputPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := Verify(context.Background(), bytes.NewReader(original), opts.MaxBytes)
|
||||
if err != nil || m.ToolVersion != opts.Version || len(m.Files) != 2 {
|
||||
t.Fatalf("native export: %v %v", m, err)
|
||||
}
|
||||
entries := verifyFixture(t)
|
||||
entries[0], entries[2] = entries[2], entries[0]
|
||||
if _, err := Verify(context.Background(), bytes.NewReader(verifyGzip(t, verifyTar(t, entries))), int64(len(testSQL))); err != nil {
|
||||
t.Fatal("valid reordered members:", err)
|
||||
}
|
||||
if current, err := os.ReadFile(opts.OutputPath); err != nil || !bytes.Equal(original, current) {
|
||||
t.Fatal("verification changed input")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifyRejectsInvalidMembersAndManifest(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
change func([]verifyEntry) []verifyEntry
|
||||
}{
|
||||
{"missing", func(e []verifyEntry) []verifyEntry { return e[:2] }},
|
||||
{"duplicate", func(e []verifyEntry) []verifyEntry { return append(e, e[1]) }},
|
||||
{"traversal", func(e []verifyEntry) []verifyEntry { e[2].name = "../PRIVATE_CANARY"; return e }},
|
||||
{"absolute", func(e []verifyEntry) []verifyEntry { e[2].name = "/config.yaml"; return e }},
|
||||
{"windows path", func(e []verifyEntry) []verifyEntry { e[2].name = `C:\config.yaml`; return e }},
|
||||
{"directory prefix", func(e []verifyEntry) []verifyEntry { e[2].name = "./config.yaml"; return e }},
|
||||
{"symlink", func(e []verifyEntry) []verifyEntry { e[2].kind, e[2].data = tar.TypeSymlink, nil; return e }},
|
||||
{"hardlink", func(e []verifyEntry) []verifyEntry { e[2].kind, e[2].data = tar.TypeLink, nil; return e }},
|
||||
{"device", func(e []verifyEntry) []verifyEntry { e[2].kind, e[2].data = tar.TypeChar, nil; return e }},
|
||||
{"directory", func(e []verifyEntry) []verifyEntry { e[2].kind, e[2].data = tar.TypeDir, nil; return e }},
|
||||
{"payload tamper", func(e []verifyEntry) []verifyEntry { e[1].data[0] ^= 1; return e }},
|
||||
{"large sql", func(e []verifyEntry) []verifyEntry { e[1].data = bytes.Repeat([]byte("x"), len(testSQL)+1); return e }},
|
||||
{"large config", func(e []verifyEntry) []verifyEntry { e[2].data = bytes.Repeat([]byte("x"), maxConfigBytes+1); return e }},
|
||||
{"large manifest", func(e []verifyEntry) []verifyEntry {
|
||||
e[0].data = bytes.Repeat([]byte("x"), maxManifestBytes+1)
|
||||
return e
|
||||
}},
|
||||
{"extended attributes", func(e []verifyEntry) []verifyEntry {
|
||||
e[1].pax = map[string]string{"SCHILY.xattr.secret": "PRIVATE_CANARY"}
|
||||
return e
|
||||
}},
|
||||
{"hidden oversized metadata", func(e []verifyEntry) []verifyEntry {
|
||||
e[1].pax = map[string]string{"comment": strings.Repeat("x", 8000)}
|
||||
return e
|
||||
}},
|
||||
{"unknown version", func(e []verifyEntry) []verifyEntry {
|
||||
e[0].data = bytes.Replace(e[0].data, []byte(`"format_version":1`), []byte(`"format_version":2`), 1)
|
||||
return e
|
||||
}},
|
||||
{"duplicate key", func(e []verifyEntry) []verifyEntry {
|
||||
e[0].data = append([]byte(`{"format_version":2,`), e[0].data[1:]...)
|
||||
return e
|
||||
}},
|
||||
{"case alias", func(e []verifyEntry) []verifyEntry {
|
||||
e[0].data = bytes.Replace(e[0].data, []byte("format_version"), []byte("FORMAT_VERSION"), 1)
|
||||
return e
|
||||
}},
|
||||
{"unicode case alias", func(e []verifyEntry) []verifyEntry {
|
||||
e[0].data = bytes.Replace(e[0].data, []byte("database_format"), []byte(`databa\u017fe_format`), 1)
|
||||
return e
|
||||
}},
|
||||
{"duplicate file record", func(e []verifyEntry) []verifyEntry {
|
||||
var m Manifest
|
||||
_ = json.Unmarshal(e[0].data, &m)
|
||||
m.Files[1] = m.Files[0]
|
||||
e[0].data, _ = json.Marshal(m)
|
||||
return e
|
||||
}},
|
||||
{"claimed size", func(e []verifyEntry) []verifyEntry {
|
||||
var m Manifest
|
||||
_ = json.Unmarshal(e[0].data, &m)
|
||||
m.Files[0].Size++
|
||||
e[0].data, _ = json.Marshal(m)
|
||||
return e
|
||||
}},
|
||||
{"unknown field", func(e []verifyEntry) []verifyEntry {
|
||||
e[0].data = append([]byte(`{"private":"PRIVATE_CANARY",`), e[0].data[1:]...)
|
||||
return e
|
||||
}},
|
||||
{"second json value", func(e []verifyEntry) []verifyEntry { e[0].data = append(e[0].data, []byte(`{}`)...); return e }},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
data := verifyGzip(t, verifyTar(t, tc.change(verifyFixture(t))))
|
||||
if _, err := Verify(context.Background(), bytes.NewReader(data), int64(len(testSQL))); err == nil || strings.Contains(err.Error(), "PRIVATE_CANARY") {
|
||||
t.Fatalf("expected a sanitized rejection, got %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifyFraming(t *testing.T) {
|
||||
raw := verifyTar(t, verifyFixture(t))
|
||||
valid := verifyGzip(t, raw)
|
||||
badCRC := bytes.Clone(valid)
|
||||
badCRC[len(badCRC)-8] ^= 1
|
||||
cases := map[string][]byte{
|
||||
"not gzip": []byte("PRIVATE_CANARY"), "truncated gzip": valid[:len(valid)-1], "bad checksum": badCRC,
|
||||
"no tar terminator": verifyGzip(t, raw[:len(raw)-1024]), "one tar terminator": verifyGzip(t, raw[:len(raw)-512]),
|
||||
"truncated payload": verifyGzip(t, raw[:1500]), "trailing tar bytes": verifyGzip(t, append(bytes.Clone(raw), 0)),
|
||||
"trailing compressed bytes": append(bytes.Clone(valid), 0), "second gzip stream": append(bytes.Clone(valid), verifyGzip(t, nil)...),
|
||||
}
|
||||
for name, data := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if _, err := Verify(context.Background(), bytes.NewReader(data), DefaultMaxBytes); err == nil || strings.Contains(err.Error(), "PRIVATE_CANARY") {
|
||||
t.Fatalf("expected sanitized framing error, got %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifySizeOnlyPAX(t *testing.T) {
|
||||
// A native dump over 8 GiB requires a size-only PAX header. Generate the
|
||||
// real header without allocating its body; verification must reach the
|
||||
// truncated member, rather than reject the exporter's metadata format.
|
||||
var raw bytes.Buffer
|
||||
tw := tar.NewWriter(&raw)
|
||||
if err := tw.WriteHeader(&tar.Header{Name: "database.sql", Mode: 0600, Size: 1 << 34}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h, err := tar.NewReader(bytes.NewReader(raw.Bytes())).Next()
|
||||
if err != nil || h.Format != tar.FormatPAX || len(h.PAXRecords) != 1 || h.PAXRecords["size"] != "17179869184" {
|
||||
t.Fatalf("fixture must contain native size-only PAX metadata: %v %v", h, err)
|
||||
}
|
||||
data := verifyGzip(t, raw.Bytes())
|
||||
if _, err := Verify(context.Background(), bytes.NewReader(data), 1<<35); !errors.Is(err, ErrInvalidArchive) || !strings.Contains(err.Error(), "member data") {
|
||||
t.Fatalf("native PAX header was rejected before reading the payload: %v", err)
|
||||
}
|
||||
if _, err := Verify(context.Background(), bytes.NewReader(data), DefaultMaxBytes); !errors.Is(err, ErrTooLarge) {
|
||||
t.Fatalf("PAX size did not respect the SQL limit: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifyBoundsAndCancellation(t *testing.T) {
|
||||
for _, limit := range []int64{0, -1, 1<<40 + 1} {
|
||||
if _, err := Verify(context.Background(), bytes.NewReader(nil), limit); err == nil {
|
||||
t.Fatal("invalid limit accepted")
|
||||
}
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err := Verify(ctx, bytes.NewReader(nil), 1); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("cancellation lost: %v", err)
|
||||
}
|
||||
for _, size := range []int{2, 3} {
|
||||
r := &verifyReader{ctx: context.Background(), r: bytes.NewReader(make([]byte, size)), limit: 2}
|
||||
_, err := io.Copy(io.Discard, r)
|
||||
if (size == 3 && !errors.Is(err, ErrTooLarge)) || (size == 2 && err != nil) {
|
||||
t.Fatalf("read limit boundary: %d %v", size, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func FuzzVerify(f *testing.F) {
|
||||
f.Add([]byte("invalid gzip"))
|
||||
f.Add(verifyGzip(f, verifyTar(f, verifyFixture(f))))
|
||||
f.Add(verifyTar(f, verifyFixture(f)))
|
||||
f.Fuzz(func(t *testing.T, data []byte) {
|
||||
if len(data) > 2<<20 {
|
||||
t.Skip()
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
_, _ = Verify(ctx, bytes.NewReader(data), 4096)
|
||||
// Fuzz tar/manifest parsing too, without every mutation first failing
|
||||
// the outer gzip checksum.
|
||||
_, _ = Verify(ctx, bytes.NewReader(verifyGzip(t, data)), 4096)
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user