feat(backup): verify native archives offline without extraction (#160)

This commit is contained in:
n30nex
2026-09-23 20:18:45 -04:00
committed by GitHub
parent 8a3fa7e6bf
commit 20691dc5a3
6 changed files with 672 additions and 29 deletions
+67 -14
View File
@@ -1,13 +1,15 @@
// Copyright 2026 Beacon Contributors
// SPDX-License-Identifier: AGPL-3.0-or-later
// beacon-backup creates a private database and saved-config export.
// beacon-backup creates or verifies a private database and saved-config export.
package main
import (
"context"
"errors"
"flag"
"fmt"
"io"
"os"
"os/signal"
"syscall"
@@ -18,22 +20,73 @@ import (
var version = "dev"
func main() {
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
code := run(ctx, os.Args[1:], os.Stdout, os.Stderr)
stop()
os.Exit(code)
}
func run(ctx context.Context, args []string, stdout, stderr io.Writer) int {
var opts backup.Options
flag.StringVar(&opts.ConfigPath, "config", "config.yaml", "saved YAML file to include verbatim (may contain secrets)")
flag.StringVar(&opts.OutputPath, "output", "", "new private .tar.gz destination (required; never overwritten)")
flag.Int64Var(&opts.MaxBytes, "max-bytes", backup.DefaultMaxBytes, "maximum uncompressed database dump size")
flag.DurationVar(&opts.Timeout, "timeout", backup.DefaultTimeout, "maximum export duration")
flag.Parse()
if flag.NArg() != 0 {
fmt.Fprintln(os.Stderr, "unexpected positional arguments; connection settings use PG* environment variables")
os.Exit(2)
var verifyPath string
flags := flag.NewFlagSet("beacon-backup", flag.ContinueOnError)
flags.SetOutput(stderr)
flags.StringVar(&opts.ConfigPath, "config", "config.yaml", "saved YAML file to export verbatim (may contain secrets)")
flags.StringVar(&opts.OutputPath, "output", "", "new private .tar.gz export destination (never overwritten)")
flags.StringVar(&verifyPath, "verify", "", "verify an existing .tar.gz without extraction or database access")
flags.Int64Var(&opts.MaxBytes, "max-bytes", backup.DefaultMaxBytes, "maximum uncompressed database dump size")
flags.DurationVar(&opts.Timeout, "timeout", backup.DefaultTimeout, "maximum export or verification duration")
if err := flags.Parse(args); err != nil {
if errors.Is(err, flag.ErrHelp) {
return 0
}
return 2
}
if flags.NArg() != 0 || opts.Timeout <= 0 {
fmt.Fprintln(stderr, "positive timeout and no positional arguments required")
return 2
}
verifyMode, exportFlag := false, false
flags.Visit(func(f *flag.Flag) {
verifyMode = verifyMode || f.Name == "verify"
exportFlag = exportFlag || f.Name == "config" || f.Name == "output"
})
if verifyMode {
if verifyPath == "" || exportFlag {
fmt.Fprintln(stderr, "verify requires an archive path and cannot be combined with config or output")
return 2
}
// Reject directories/devices/FIFOs before opening, and check the opened
// descriptor too. A symlink to a regular, operator-selected file is fine.
info, err := os.Stat(verifyPath)
if err != nil || !info.Mode().IsRegular() {
fmt.Fprintln(stderr, "verification requires a readable regular archive file")
return 1
}
file, err := os.Open(verifyPath)
if err != nil {
fmt.Fprintln(stderr, "cannot open backup archive")
return 1
}
defer file.Close()
if info, err = file.Stat(); err != nil || !info.Mode().IsRegular() {
fmt.Fprintln(stderr, "verification requires a regular archive file")
return 1
}
ctx, cancel := context.WithTimeout(ctx, opts.Timeout)
defer cancel()
if _, err := backup.Verify(ctx, file, opts.MaxBytes); err != nil {
fmt.Fprintln(stderr, "verification failed:", err)
return 1
}
fmt.Fprintln(stdout, "Archive structure and checksums verified. No files extracted or SQL executed.")
return 0
}
opts.Version = version
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
if err := backup.Export(ctx, opts); err != nil {
fmt.Fprintln(os.Stderr, "backup failed:", err)
os.Exit(1)
fmt.Fprintln(stderr, "backup failed:", err)
return 1
}
fmt.Println("Backup complete. Store this bundle privately; it may contain keys and message data.")
fmt.Fprintln(stdout, "Backup complete. Store this bundle privately; it may contain keys and message data.")
return 0
}
+82
View File
@@ -0,0 +1,82 @@
// Copyright 2026 Beacon Contributors
// SPDX-License-Identifier: AGPL-3.0-or-later
package main
import (
"archive/tar"
"bytes"
"compress/gzip"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/MeshCore-Beacon/beacon-server/internal/backup"
)
func TestVerifyCommandIsReadOnlyAndOffline(t *testing.T) {
dir := t.TempDir()
t.Setenv("PATH", dir) // no pg_dump, psql or shell helper is available
t.Setenv("PGDATABASE", "")
archive := filepath.Join(dir, "private.tar.gz")
m := backup.Manifest{FormatVersion: 1, CreatedAt: time.Now().UTC(), DatabaseFormat: "postgresql-plain-sql", ToolVersion: "PRIVATE_CANARY"}
data := map[string][]byte{"database.sql": []byte("SELECT 'PRIVATE_CANARY';"), "config.yaml": {}}
for _, name := range []string{"database.sql", "config.yaml"} {
hash := sha256.Sum256(data[name])
m.Files = append(m.Files, backup.File{Name: name, Size: int64(len(data[name])), SHA256: hex.EncodeToString(hash[:])})
}
data["manifest.json"], _ = json.Marshal(m)
var raw bytes.Buffer
gz := gzip.NewWriter(&raw)
tw := tar.NewWriter(gz)
for _, name := range []string{"manifest.json", "database.sql", "config.yaml"} {
if err := tw.WriteHeader(&tar.Header{Name: name, Size: int64(len(data[name])), Mode: 0600}); err != nil {
t.Fatal(err)
}
if _, err := tw.Write(data[name]); err != nil {
t.Fatal(err)
}
}
if err := tw.Close(); err != nil {
t.Fatal(err)
}
if err := gz.Close(); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(archive, raw.Bytes(), 0600); err != nil {
t.Fatal(err)
}
for _, tc := range []struct {
args []string
code int
}{
{[]string{"-verify", archive}, 0},
{[]string{"-verify", archive, "-max-bytes", "1"}, 1},
{[]string{"-verify", archive, "-timeout", "0"}, 2},
{[]string{"-verify", archive, "-output", filepath.Join(dir, "must-not-exist")}, 2},
{[]string{"-verify", archive, "-config", "missing.yaml"}, 2},
{[]string{"-verify", "", "-output", filepath.Join(dir, "must-not-exist")}, 2},
{[]string{"-verify", dir}, 1},
{[]string{"-verify", filepath.Join(dir, "PRIVATE_CANARY-missing")}, 1},
{[]string{"-verify", archive, "unexpected"}, 2},
} {
var stdout, stderr bytes.Buffer
if got := run(context.Background(), tc.args, &stdout, &stderr); got != tc.code || strings.Contains(stdout.String()+stderr.String(), "PRIVATE_CANARY") {
t.Fatalf("exit %d, want %d; diagnostic %s", got, tc.code, stderr.String())
}
}
files, err := os.ReadDir(dir)
if err != nil || len(files) != 1 || files[0].Name() != "private.tar.gz" {
t.Fatal("verification extracted files or created output")
}
unchanged, err := os.ReadFile(archive)
if err != nil || !bytes.Equal(unchanged, raw.Bytes()) {
t.Fatal("input archive changed")
}
}
+47 -14
View File
@@ -1,13 +1,14 @@
# Database and saved-config export
# Database and saved-config export and verification
`beacon-backup` is the export foundation for issue #72. It produces a private,
versioned `.tar.gz` using PostgreSQL's `pg_dump`. It does not yet provide a web
interface, account login, scheduled/remote storage or automatic import.
`beacon-backup` is the export and archive-validation foundation for issue #72. It
produces a private, versioned `.tar.gz` using PostgreSQL's `pg_dump` and can verify
an existing bundle offline. It does not yet provide a web interface, account
login, scheduled/remote storage or automatic import.
Build it with `go build ./cmd/beacon-backup`. Install `pg_dump` in the same runtime
as this command; an installation on the Docker host does not install it inside an
app container. Use a client of the same major version as the source PostgreSQL
server; an older client cannot dump a newer server.
Build it with `go build ./cmd/beacon-backup`. For exports, install `pg_dump` in the
same runtime as this command; an installation on the Docker host does not install
it inside an app container. Use a client of the same major version as the source
PostgreSQL server; an older client cannot dump a newer server.
Set libpq's standard `PGHOST`, `PGPORT`, `PGDATABASE`, `PGUSER` and TLS settings.
`PGDATABASE` is required and must be a plain database name. Prefer a private
@@ -115,12 +116,43 @@ waiting indefinitely behind schema changes. Export failure publishes no backup;
check the client version, privileges, connection settings and available capacity
privately. The command deliberately does not expose raw client diagnostics.
## Verify an existing archive
```sh
beacon-backup -verify /private/beacon-20260913.tar.gz
```
Verification reads a regular file without extracting members, writing files,
executing SQL or connecting to PostgreSQL, Redis or MQTT. It needs neither a
configuration file nor `pg_dump` or connection credentials. Do not combine
`-verify` with the export-only `-config` or `-output` flags.
The verifier accepts the native format-1 bundle: exactly the three fixed regular
members with 0600 modes and USTAR headers (or the size-only PAX header needed for
SQL over 8 GiB). It rejects duplicates, unexpected paths, links, devices, other
extended metadata, incomplete tar/gzip endings and trailing or concatenated
streams. Manifest fields must be known, with no duplicate keys or case aliases;
the payload sizes and SHA-256 hashes must match the bytes actually read.
The same `-max-bytes` SQL limit applies, with a separate 1 MiB config limit and
64 KiB manifest limit. Compressed input is capped at the SQL limit plus 1% and
2 MiB, matching the exporter. Headers and decompressed data are bounded too;
SQL and config are hashed as streams, without keeping their contents in memory.
The ten-minute default `-timeout` and interrupt handling check cancellation
between reads; they cannot interrupt a blocked filesystem read.
Exit status 0 means the archive structure and checksums passed. Failure returns
a nonzero status with no payload, manifest values or input path in its diagnostic.
The manifest is not signed: verification does **not** establish authenticity,
safe SQL, configuration validity or restorability. It does not make an untrusted
dump safe to restore, and does not provide import/overwrite or deployment recovery.
## Restore verification
Use trusted bundles only: PostgreSQL dumps can contain executable SQL. Inspect
the fixed members, verify the gzip stream and the manifest's payload sizes and
hashes, and extract into private staging. For a **new, empty disposable database**,
with its own explicit `PGDATABASE` and target-role connection settings:
Use trusted bundles only: PostgreSQL dumps can contain executable SQL. Run the
archive verification above, then extract into private staging. For a
**new, empty disposable database**, with its own explicit `PGDATABASE` and
target-role connection settings:
```sh
psql -X --set ON_ERROR_STOP=on --single-transaction --file database.sql
@@ -137,8 +169,9 @@ PostgreSQL references: [pg_dump](https://www.postgresql.org/docs/16/app-pgdump.h
[password files](https://www.postgresql.org/docs/16/libpq-pgpass.html).
CI runs the compiled command and its PostgreSQL round-trip test with a dedicated
PostgreSQL 16 service. To repeat it privately, build the command, set the `PG*`
connection variables for an isolated test server and set
PostgreSQL 16 service. The test also verifies the export offline with no client on
PATH or connection settings, and rejects a truncated copy. To repeat it privately,
build the command, set the `PG*` connection variables for an isolated test server and set
`BEACON_BACKUP_TEST_POSTGRES=1` plus `BEACON_BACKUP_TEST_BINARY` to the command's
absolute path. Run `go test ./internal/backup -run '^TestExportPostgres$' -v`.
The test role needs permission to create/drop its two randomly named databases;
+14 -1
View File
@@ -115,6 +115,19 @@ CREATE MATERIALIZED VIEW backup_materialized AS SELECT count(*) AS count FROM ba
if err != nil {
t.Fatal(err)
}
verify := exec.CommandContext(ctx, binary, "-verify", output, "-max-bytes", "16777216", "-timeout", "30s")
verify.Env = []string{"PATH=" + dir} // verification needs no PostgreSQL client or connection
message, err := verify.CombinedOutput()
if err != nil || !bytes.Contains(message, []byte("structure and checksums verified")) || bytes.Contains(message, []byte("café")) {
t.Fatal("exact command could not verify its export offline")
}
corrupt := filepath.Join(dir, "corrupt.tar.gz")
if err := os.WriteFile(corrupt, original[:len(original)-1], 0600); err != nil {
t.Fatal(err)
}
if err := exec.CommandContext(ctx, binary, "-verify", corrupt).Run(); err == nil {
t.Fatal("exact command accepted a truncated export")
}
for _, failure := range []string{"existing", "size", "connection", "timeout"} {
t.Run(failure, func(t *testing.T) {
failedOutput := filepath.Join(dir, failure+".tar.gz")
@@ -212,5 +225,5 @@ CREATE MATERIALIZED VIEW backup_materialized AS SELECT count(*) AS count FROM ba
if err := target.QueryRow(ctx, "INSERT INTO backup_fixture (message) VALUES ('next') RETURNING id").Scan(&nextID); err != nil || nextID != 3 {
t.Fatal("identity sequence did not survive restore")
}
t.Log("exact command restored all migrations, columns, Unicode/bytea/JSON data, relationships, views and identity sequence")
t.Log("exact command verified its export offline and restored all migrations, columns, Unicode/bytea/JSON data, relationships, views and identity sequence")
}
+205
View File
@@ -0,0 +1,205 @@
// Copyright 2026 Beacon Contributors
// SPDX-License-Identifier: AGPL-3.0-or-later
package backup
import (
"archive/tar"
"bufio"
"bytes"
"compress/gzip"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"strconv"
"strings"
)
const maxManifestBytes = 64 << 10
var ErrInvalidArchive = errors.New("invalid backup archive")
// Verify streams a format-1 bundle without extracting files or executing SQL.
// maxBytes bounds the SQL payload (up to 1 TiB); the saved YAML and manifest have
// separate fixed limits. The caller owns input and its deadline. Cancellation is
// checked between reads; the supplied Reader must not block indefinitely.
// The returned manifest is untrusted metadata, not proof of origin or restorability.
func Verify(ctx context.Context, input io.Reader, maxBytes int64) (*Manifest, error) {
if input == nil || maxBytes <= 0 || maxBytes > 1<<40 {
return nil, errors.New("input and positive max-bytes (at most 1 TiB) are required")
}
// Match the exporter's compressed-output allowance. ByteReader prevents gzip
// from swallowing a following stream before we can reject it.
compressed := &verifyReader{ctx: ctx, r: input, limit: maxBytes + maxBytes/100 + 2*maxConfigBytes}
buffered := bufio.NewReader(compressed)
gz, err := gzip.NewReader(buffered)
if err != nil {
return nil, verificationError(ctx, err, "gzip header")
}
defer gz.Close()
gz.Multistream(false)
decoded := &verifyReader{ctx: ctx, r: gz}
tr := tar.NewReader(decoded)
files := make(map[string]File, 3)
var metadata []byte
var padding int64
for {
before := decoded.n
// Native headers are USTAR, or one size-only PAX record for SQL >8 GiB.
// Bound Next itself: it otherwise hides and allocates extension records.
decoded.limit = before + padding + 3*512
h, err := tr.Next()
if err == io.EOF {
// tar.Reader also accepts EOF without both closing zero blocks.
if decoded.n-before != padding+2*512 {
return nil, verificationError(ctx, nil, "missing tar terminator")
}
break
}
if err != nil {
return nil, verificationError(ctx, err, "tar header")
}
headerBytes := decoded.n - before - padding
ordinary := h.Format == tar.FormatUSTAR && headerBytes == 512 && len(h.PAXRecords) == 0
large := h.Format == tar.FormatPAX && headerBytes == 3*512 && len(h.PAXRecords) == 1 && h.PAXRecords["size"] == strconv.FormatInt(h.Size, 10)
if (!ordinary && !large) || (h.Typeflag != tar.TypeReg && h.Typeflag != tar.TypeRegA) || h.Linkname != "" || h.Mode != 0600 {
return nil, verificationError(ctx, nil, "unsupported member type or metadata")
}
if _, duplicate := files[h.Name]; duplicate || len(files) == 3 {
return nil, verificationError(ctx, nil, "duplicate or extra member")
}
var limit int64
switch h.Name {
case "manifest.json":
limit = maxManifestBytes
case "database.sql":
limit = maxBytes
if h.Size == 0 {
return nil, verificationError(ctx, nil, "empty database dump")
}
case "config.yaml":
limit = maxConfigBytes
default:
return nil, verificationError(ctx, nil, "unexpected member name")
}
if h.Size < 0 || h.Size > limit {
return nil, ErrTooLarge
}
decoded.limit = decoded.n + h.Size
digest := sha256.New()
if h.Name == "manifest.json" {
metadata, err = io.ReadAll(tr)
} else {
_, err = io.Copy(digest, tr)
}
if err != nil {
return nil, verificationError(ctx, err, "member data")
}
files[h.Name] = File{Name: h.Name, Size: h.Size, SHA256: hex.EncodeToString(digest.Sum(nil))}
padding = (512 - h.Size%512) % 512
}
// Read through the gzip checksum/trailer, but allow no data after tar EOF.
decoded.limit = decoded.n
var tail [1]byte
if _, err := decoded.Read(tail[:]); err != io.EOF {
return nil, verificationError(ctx, err, "gzip completion or trailing tar data")
}
if _, err := buffered.ReadByte(); err != io.EOF {
return nil, verificationError(ctx, err, "trailing compressed data")
}
if len(files) != 3 || uniqueManifestKeys(json.NewDecoder(bytes.NewReader(metadata)), 0) != nil {
return nil, verificationError(ctx, nil, "missing members or invalid manifest")
}
var manifest Manifest
decoder := json.NewDecoder(bytes.NewReader(metadata))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&manifest); err != nil || decoder.Decode(new(any)) != io.EOF || manifest.FormatVersion != 1 || manifest.DatabaseFormat != "postgresql-plain-sql" || manifest.CreatedAt.IsZero() || len(manifest.Files) != 2 {
return nil, verificationError(ctx, nil, "unsupported or incomplete manifest")
}
for _, declared := range manifest.Files {
actual, exists := files[declared.Name]
digest, err := hex.DecodeString(declared.SHA256)
if !exists || declared.Name == "manifest.json" || declared.Size != actual.Size || err != nil || len(digest) != sha256.Size || !strings.EqualFold(declared.SHA256, actual.SHA256) {
return nil, verificationError(ctx, nil, "manifest size or checksum mismatch")
}
delete(files, declared.Name) // duplicate manifest file records fail as well.
}
if err := ctx.Err(); err != nil {
return nil, err
}
return &manifest, nil
}
// Avoid returning filenames, manifest values, reader errors or payload text.
func verificationError(ctx context.Context, err error, stage string) error {
if ctx.Err() != nil {
return ctx.Err()
}
if errors.Is(err, ErrTooLarge) {
return ErrTooLarge
}
return fmt.Errorf("%w: %s", ErrInvalidArchive, stage)
}
type verifyReader struct {
ctx context.Context
r io.Reader
n, limit int64
}
func (r *verifyReader) Read(p []byte) (int, error) {
if err := r.ctx.Err(); err != nil {
return 0, err
}
if r.n > r.limit {
return 0, ErrTooLarge
}
if int64(len(p)) > r.limit-r.n+1 {
p = p[:r.limit-r.n+1]
}
n, err := r.r.Read(p)
r.n += int64(n)
if r.n > r.limit {
return 0, ErrTooLarge
}
return n, err
}
// encoding/json accepts duplicate keys and case aliases. Reject both before
// decoding the fixed schema, so another reader cannot interpret different values.
func uniqueManifestKeys(d *json.Decoder, depth int) error {
if depth > 8 {
return ErrInvalidArchive
}
token, err := d.Token()
if err != nil {
return err
}
if delim, ok := token.(json.Delim); ok {
keys := map[string]bool{}
for d.More() {
if delim == '{' {
key, err := d.Token()
if err != nil {
return err
}
name, ok := key.(string)
// Schema keys are ASCII. Unicode simple-fold aliases (such as
// long-s for s) also match encoding/json fields, even in lowercase.
if !ok || keys[name] || strings.IndexFunc(name, func(r rune) bool { return r > 127 || r >= 'A' && r <= 'Z' }) >= 0 {
return ErrInvalidArchive
}
keys[name] = true
}
if err := uniqueManifestKeys(d, depth+1); err != nil {
return err
}
}
_, err = d.Token()
}
return err
}
+257
View File
@@ -0,0 +1,257 @@
// Copyright 2026 Beacon Contributors
// SPDX-License-Identifier: AGPL-3.0-or-later
package backup
import (
"archive/tar"
"bytes"
"compress/gzip"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"io"
"os"
"strings"
"testing"
"time"
)
type verifyEntry struct {
name string
data []byte
kind byte
pax map[string]string
}
func verifyFixture(t testing.TB) []verifyEntry {
t.Helper()
sql, config := []byte(testSQL), []byte("secret: PRIVATE_CANARY\n")
m := Manifest{FormatVersion: 1, CreatedAt: time.Now().UTC(), ToolVersion: "fixture", DatabaseFormat: "postgresql-plain-sql", Excluded: []string{"deployment_environment"}}
for _, entry := range []verifyEntry{{name: "database.sql", data: sql}, {name: "config.yaml", data: config}} {
hash := sha256.Sum256(entry.data)
m.Files = append(m.Files, File{Name: entry.name, Size: int64(len(entry.data)), SHA256: hex.EncodeToString(hash[:])})
}
metadata, err := json.Marshal(m)
if err != nil {
t.Fatal(err)
}
return []verifyEntry{{name: "manifest.json", data: metadata}, {name: "database.sql", data: sql}, {name: "config.yaml", data: config}}
}
func verifyTar(t testing.TB, entries []verifyEntry) []byte {
t.Helper()
var data bytes.Buffer
tw := tar.NewWriter(&data)
for _, e := range entries {
h := &tar.Header{Name: e.name, Size: int64(len(e.data)), Mode: 0600, Typeflag: e.kind}
if e.kind == tar.TypeSymlink || e.kind == tar.TypeLink {
h.Linkname = "../PRIVATE_CANARY"
}
if e.pax != nil {
h.Format, h.PAXRecords = tar.FormatPAX, e.pax
}
if err := tw.WriteHeader(h); err != nil {
t.Fatal(err)
}
if _, err := tw.Write(e.data); err != nil {
t.Fatal(err)
}
}
if err := tw.Close(); err != nil {
t.Fatal(err)
}
return data.Bytes()
}
func verifyGzip(t testing.TB, data []byte) []byte {
t.Helper()
var encoded bytes.Buffer
gz := gzip.NewWriter(&encoded)
if _, err := gz.Write(data); err != nil {
t.Fatal(err)
}
if err := gz.Close(); err != nil {
t.Fatal(err)
}
return encoded.Bytes()
}
func TestVerifyExportAndReorderedMembers(t *testing.T) {
opts := setup(t)
if err := export(context.Background(), opts, helper(t, "ok")); err != nil {
t.Fatal(err)
}
original, err := os.ReadFile(opts.OutputPath)
if err != nil {
t.Fatal(err)
}
m, err := Verify(context.Background(), bytes.NewReader(original), opts.MaxBytes)
if err != nil || m.ToolVersion != opts.Version || len(m.Files) != 2 {
t.Fatalf("native export: %v %v", m, err)
}
entries := verifyFixture(t)
entries[0], entries[2] = entries[2], entries[0]
if _, err := Verify(context.Background(), bytes.NewReader(verifyGzip(t, verifyTar(t, entries))), int64(len(testSQL))); err != nil {
t.Fatal("valid reordered members:", err)
}
if current, err := os.ReadFile(opts.OutputPath); err != nil || !bytes.Equal(original, current) {
t.Fatal("verification changed input")
}
}
func TestVerifyRejectsInvalidMembersAndManifest(t *testing.T) {
for _, tc := range []struct {
name string
change func([]verifyEntry) []verifyEntry
}{
{"missing", func(e []verifyEntry) []verifyEntry { return e[:2] }},
{"duplicate", func(e []verifyEntry) []verifyEntry { return append(e, e[1]) }},
{"traversal", func(e []verifyEntry) []verifyEntry { e[2].name = "../PRIVATE_CANARY"; return e }},
{"absolute", func(e []verifyEntry) []verifyEntry { e[2].name = "/config.yaml"; return e }},
{"windows path", func(e []verifyEntry) []verifyEntry { e[2].name = `C:\config.yaml`; return e }},
{"directory prefix", func(e []verifyEntry) []verifyEntry { e[2].name = "./config.yaml"; return e }},
{"symlink", func(e []verifyEntry) []verifyEntry { e[2].kind, e[2].data = tar.TypeSymlink, nil; return e }},
{"hardlink", func(e []verifyEntry) []verifyEntry { e[2].kind, e[2].data = tar.TypeLink, nil; return e }},
{"device", func(e []verifyEntry) []verifyEntry { e[2].kind, e[2].data = tar.TypeChar, nil; return e }},
{"directory", func(e []verifyEntry) []verifyEntry { e[2].kind, e[2].data = tar.TypeDir, nil; return e }},
{"payload tamper", func(e []verifyEntry) []verifyEntry { e[1].data[0] ^= 1; return e }},
{"large sql", func(e []verifyEntry) []verifyEntry { e[1].data = bytes.Repeat([]byte("x"), len(testSQL)+1); return e }},
{"large config", func(e []verifyEntry) []verifyEntry { e[2].data = bytes.Repeat([]byte("x"), maxConfigBytes+1); return e }},
{"large manifest", func(e []verifyEntry) []verifyEntry {
e[0].data = bytes.Repeat([]byte("x"), maxManifestBytes+1)
return e
}},
{"extended attributes", func(e []verifyEntry) []verifyEntry {
e[1].pax = map[string]string{"SCHILY.xattr.secret": "PRIVATE_CANARY"}
return e
}},
{"hidden oversized metadata", func(e []verifyEntry) []verifyEntry {
e[1].pax = map[string]string{"comment": strings.Repeat("x", 8000)}
return e
}},
{"unknown version", func(e []verifyEntry) []verifyEntry {
e[0].data = bytes.Replace(e[0].data, []byte(`"format_version":1`), []byte(`"format_version":2`), 1)
return e
}},
{"duplicate key", func(e []verifyEntry) []verifyEntry {
e[0].data = append([]byte(`{"format_version":2,`), e[0].data[1:]...)
return e
}},
{"case alias", func(e []verifyEntry) []verifyEntry {
e[0].data = bytes.Replace(e[0].data, []byte("format_version"), []byte("FORMAT_VERSION"), 1)
return e
}},
{"unicode case alias", func(e []verifyEntry) []verifyEntry {
e[0].data = bytes.Replace(e[0].data, []byte("database_format"), []byte(`databa\u017fe_format`), 1)
return e
}},
{"duplicate file record", func(e []verifyEntry) []verifyEntry {
var m Manifest
_ = json.Unmarshal(e[0].data, &m)
m.Files[1] = m.Files[0]
e[0].data, _ = json.Marshal(m)
return e
}},
{"claimed size", func(e []verifyEntry) []verifyEntry {
var m Manifest
_ = json.Unmarshal(e[0].data, &m)
m.Files[0].Size++
e[0].data, _ = json.Marshal(m)
return e
}},
{"unknown field", func(e []verifyEntry) []verifyEntry {
e[0].data = append([]byte(`{"private":"PRIVATE_CANARY",`), e[0].data[1:]...)
return e
}},
{"second json value", func(e []verifyEntry) []verifyEntry { e[0].data = append(e[0].data, []byte(`{}`)...); return e }},
} {
t.Run(tc.name, func(t *testing.T) {
data := verifyGzip(t, verifyTar(t, tc.change(verifyFixture(t))))
if _, err := Verify(context.Background(), bytes.NewReader(data), int64(len(testSQL))); err == nil || strings.Contains(err.Error(), "PRIVATE_CANARY") {
t.Fatalf("expected a sanitized rejection, got %v", err)
}
})
}
}
func TestVerifyFraming(t *testing.T) {
raw := verifyTar(t, verifyFixture(t))
valid := verifyGzip(t, raw)
badCRC := bytes.Clone(valid)
badCRC[len(badCRC)-8] ^= 1
cases := map[string][]byte{
"not gzip": []byte("PRIVATE_CANARY"), "truncated gzip": valid[:len(valid)-1], "bad checksum": badCRC,
"no tar terminator": verifyGzip(t, raw[:len(raw)-1024]), "one tar terminator": verifyGzip(t, raw[:len(raw)-512]),
"truncated payload": verifyGzip(t, raw[:1500]), "trailing tar bytes": verifyGzip(t, append(bytes.Clone(raw), 0)),
"trailing compressed bytes": append(bytes.Clone(valid), 0), "second gzip stream": append(bytes.Clone(valid), verifyGzip(t, nil)...),
}
for name, data := range cases {
t.Run(name, func(t *testing.T) {
if _, err := Verify(context.Background(), bytes.NewReader(data), DefaultMaxBytes); err == nil || strings.Contains(err.Error(), "PRIVATE_CANARY") {
t.Fatalf("expected sanitized framing error, got %v", err)
}
})
}
}
func TestVerifySizeOnlyPAX(t *testing.T) {
// A native dump over 8 GiB requires a size-only PAX header. Generate the
// real header without allocating its body; verification must reach the
// truncated member, rather than reject the exporter's metadata format.
var raw bytes.Buffer
tw := tar.NewWriter(&raw)
if err := tw.WriteHeader(&tar.Header{Name: "database.sql", Mode: 0600, Size: 1 << 34}); err != nil {
t.Fatal(err)
}
h, err := tar.NewReader(bytes.NewReader(raw.Bytes())).Next()
if err != nil || h.Format != tar.FormatPAX || len(h.PAXRecords) != 1 || h.PAXRecords["size"] != "17179869184" {
t.Fatalf("fixture must contain native size-only PAX metadata: %v %v", h, err)
}
data := verifyGzip(t, raw.Bytes())
if _, err := Verify(context.Background(), bytes.NewReader(data), 1<<35); !errors.Is(err, ErrInvalidArchive) || !strings.Contains(err.Error(), "member data") {
t.Fatalf("native PAX header was rejected before reading the payload: %v", err)
}
if _, err := Verify(context.Background(), bytes.NewReader(data), DefaultMaxBytes); !errors.Is(err, ErrTooLarge) {
t.Fatalf("PAX size did not respect the SQL limit: %v", err)
}
}
func TestVerifyBoundsAndCancellation(t *testing.T) {
for _, limit := range []int64{0, -1, 1<<40 + 1} {
if _, err := Verify(context.Background(), bytes.NewReader(nil), limit); err == nil {
t.Fatal("invalid limit accepted")
}
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := Verify(ctx, bytes.NewReader(nil), 1); !errors.Is(err, context.Canceled) {
t.Fatalf("cancellation lost: %v", err)
}
for _, size := range []int{2, 3} {
r := &verifyReader{ctx: context.Background(), r: bytes.NewReader(make([]byte, size)), limit: 2}
_, err := io.Copy(io.Discard, r)
if (size == 3 && !errors.Is(err, ErrTooLarge)) || (size == 2 && err != nil) {
t.Fatalf("read limit boundary: %d %v", size, err)
}
}
}
func FuzzVerify(f *testing.F) {
f.Add([]byte("invalid gzip"))
f.Add(verifyGzip(f, verifyTar(f, verifyFixture(f))))
f.Add(verifyTar(f, verifyFixture(f)))
f.Fuzz(func(t *testing.T, data []byte) {
if len(data) > 2<<20 {
t.Skip()
}
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
_, _ = Verify(ctx, bytes.NewReader(data), 4096)
// Fuzz tar/manifest parsing too, without every mutation first failing
// the outer gzip checksum.
_, _ = Verify(ctx, bytes.NewReader(verifyGzip(t, data)), 4096)
})
}