feat: /api/backup — one-click SQLite database export (#474) (#1022)

## Summary

Implements `GET /api/backup` — one-click SQLite database export per
#474.

Operators can now grab a complete, consistent snapshot of the analyzer
DB with a single authenticated request — no SSH, no scripts, no DB
tooling.

## Endpoint

```
GET /api/backup
X-API-Key: <key>            # required
→ 200 OK
  Content-Type: application/octet-stream
  Content-Disposition: attachment; filename="corescope-backup-<unix>.db"
  <body: complete SQLite database file>
```

## Approach

Uses SQLite's `VACUUM INTO 'path'` to produce an atomic, defragmented
copy of the database into a fresh file:

- **Consistent**: VACUUM INTO runs at read isolation — the snapshot
reflects a single point in time even while the ingestor is writing to
the WAL.
- **Non-blocking**: writers continue uninterrupted; we never hold a
write lock.
- **Works on read-only connections**: verified manually against a
WAL-mode source DB (`mode=ro` connection successfully produces a
snapshot).
- **No corruption risk**: even if the live on-disk DB has issues, VACUUM
INTO surfaces what the server can read rather than copying broken pages
byte-for-byte.

The snapshot is staged in `os.MkdirTemp(...)` and removed after the
response body is fully streamed (deferred cleanup). Requesting client IP
is logged for audit.

The issue suggested an alternative in-memory rebuild path; `VACUUM INTO`
is simpler, faster, and produces a strictly more accurate copy of what
the server actually sees, so going with it.

## Security

- Mounted under `requireAPIKey` middleware — same gate as other admin
endpoints (`/api/admin/prune`, `/api/perf/reset`).
- Returns 401 without a valid `X-API-Key` header.
- Returns 403 if no API key is configured server-side.
- `X-Content-Type-Options: nosniff` set on the response.

## TDD

- **Red** (`99548f2`): `cmd/server/backup_test.go` adds
`TestBackupRequiresAPIKey` + `TestBackupReturnsValidSQLiteSnapshot`.
Stub handler returns 200 with no body so the tests fail on assertions
(Content-Type / Content-Disposition / SQLite magic header), not on
import or build errors.
- **Green** (`837b2fe`): real implementation lands; both tests pass;
full `go test ./...` suite stays green.

## Files

- `cmd/server/backup.go` — handler implementation
- `cmd/server/backup_test.go` — red-then-green tests
- `cmd/server/routes.go` — route registration under `requireAPIKey`
- `cmd/server/openapi.go` — OpenAPI metadata so `/api/openapi`
advertises the endpoint

## Out of scope (follow-ups)

- Rate limiting (issue suggested 1 req/min). Not added here —
admin-key-gated endpoint with a fast snapshot path is acceptable for v1;
happy to add a token-bucket limiter in a follow-up if operators report
hammering.
- UI button to trigger the download (frontend work — separate PR).

Fixes #474

---------

Co-authored-by: corescope-bot <bot@corescope.local>
This commit is contained in:
Kpa-clawbot
2026-05-03 17:56:42 -07:00
committed by GitHub
parent 51b9fed15e
commit b06adf9f2a
4 changed files with 146 additions and 0 deletions
+89
View File
@@ -0,0 +1,89 @@
package main
import (
"fmt"
"io"
"log"
"net/http"
"os"
"path/filepath"
"strings"
"time"
)
// handleBackup streams a consistent SQLite snapshot of the analyzer DB.
//
// Requires API-key authentication (mounted via requireAPIKey in routes.go).
//
// Strategy: SQLite's `VACUUM INTO 'path'` produces an atomic, defragmented
// copy of the current database into a new file. It runs at READ ISOLATION
// against the source DB (works on our read-only connection) and never
// blocks concurrent writers — the ingestor keeps writing to the WAL while
// the snapshot is taken from a consistent read transaction.
//
// Response:
//
// 200 OK
// Content-Type: application/octet-stream
// Content-Disposition: attachment; filename="corescope-backup-<unix>.db"
// <body: complete SQLite database file>
//
// The temp file is removed after the response is fully written, regardless
// of whether the client successfully consumed the stream.
func (s *Server) handleBackup(w http.ResponseWriter, r *http.Request) {
if s.db == nil || s.db.conn == nil {
writeError(w, http.StatusServiceUnavailable, "database unavailable")
return
}
ts := time.Now().UTC().Unix()
clientIP := r.Header.Get("X-Forwarded-For")
if clientIP == "" {
clientIP = r.RemoteAddr
}
log.Printf("[backup] generating backup for client %s", clientIP)
// Stage the snapshot in the OS temp dir so we never touch the live DB
// directory (avoids confusing operators / accidental WAL clobber).
tmpDir, err := os.MkdirTemp("", "corescope-backup-")
if err != nil {
writeError(w, http.StatusInternalServerError, "tempdir failed: "+err.Error())
return
}
defer func() {
if rmErr := os.RemoveAll(tmpDir); rmErr != nil {
log.Printf("[backup] cleanup error: %v", rmErr)
}
}()
snapshotPath := filepath.Join(tmpDir, fmt.Sprintf("corescope-backup-%d.db", ts))
// SQLite parses the path literal — escape any single quotes defensively.
// (mkdtemp output won't contain quotes, but be paranoid for future-proofing.)
escaped := strings.ReplaceAll(snapshotPath, "'", "''")
if _, err := s.db.conn.ExecContext(r.Context(), fmt.Sprintf("VACUUM INTO '%s'", escaped)); err != nil {
writeError(w, http.StatusInternalServerError, "snapshot failed: "+err.Error())
return
}
f, err := os.Open(snapshotPath)
if err != nil {
writeError(w, http.StatusInternalServerError, "open snapshot failed: "+err.Error())
return
}
defer f.Close()
stat, err := f.Stat()
if err == nil {
w.Header().Set("Content-Length", fmt.Sprintf("%d", stat.Size()))
}
w.Header().Set("Content-Type", "application/octet-stream")
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"corescope-backup-%d.db\"", ts))
w.Header().Set("X-Content-Type-Options", "nosniff")
w.WriteHeader(http.StatusOK)
if _, err := io.Copy(w, f); err != nil {
// Headers already flushed; just log. Client will see truncated stream.
log.Printf("[backup] stream error: %v", err)
}
}
+55
View File
@@ -0,0 +1,55 @@
package main
import (
"net/http"
"net/http/httptest"
"strings"
"testing"
)
// sqliteMagic is the 16-byte file header identifying a valid SQLite 3 database.
// See https://www.sqlite.org/fileformat.html#magic_header_string
const sqliteMagic = "SQLite format 3\x00"
func TestBackupRequiresAPIKey(t *testing.T) {
_, router := setupTestServerWithAPIKey(t, "test-secret-key-strong-enough")
req := httptest.NewRequest("GET", "/api/backup", nil)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
if w.Code != http.StatusUnauthorized {
t.Fatalf("expected 401 without API key, got %d (body: %s)", w.Code, w.Body.String())
}
}
func TestBackupReturnsValidSQLiteSnapshot(t *testing.T) {
const apiKey = "test-secret-key-strong-enough"
_, router := setupTestServerWithAPIKey(t, apiKey)
req := httptest.NewRequest("GET", "/api/backup", nil)
req.Header.Set("X-API-Key", apiKey)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("expected 200, got %d (body: %s)", w.Code, w.Body.String())
}
ct := w.Header().Get("Content-Type")
if ct != "application/octet-stream" {
t.Errorf("expected Content-Type application/octet-stream, got %q", ct)
}
cd := w.Header().Get("Content-Disposition")
if !strings.HasPrefix(cd, "attachment;") || !strings.Contains(cd, "filename=\"corescope-backup-") || !strings.HasSuffix(cd, ".db\"") {
t.Errorf("expected Content-Disposition attachment with corescope-backup-<ts>.db filename, got %q", cd)
}
body := w.Body.Bytes()
if len(body) < len(sqliteMagic) {
t.Fatalf("backup body too short (%d bytes) — expected SQLite file", len(body))
}
if got := string(body[:len(sqliteMagic)]); got != sqliteMagic {
t.Fatalf("expected SQLite magic header %q, got %q", sqliteMagic, got)
}
}
+1
View File
@@ -45,6 +45,7 @@ func routeDescriptions() map[string]routeMeta {
"POST /api/perf/reset": {Summary: "Reset performance stats", Tag: "admin", Auth: true},
"POST /api/admin/prune": {Summary: "Prune old data", Description: "Deletes packets and nodes older than the configured retention period.", Tag: "admin", Auth: true},
"GET /api/debug/affinity": {Summary: "Debug neighbor affinity scores", Tag: "admin", Auth: true},
"GET /api/backup": {Summary: "Download SQLite backup", Description: "Streams a consistent SQLite snapshot of the analyzer DB (VACUUM INTO). Response is application/octet-stream with attachment filename corescope-backup-<unix>.db.", Tag: "admin", Auth: true},
// Packets
"GET /api/packets": {Summary: "List packets", Description: "Returns decoded packets with filtering, sorting, and pagination.", Tag: "packets",
+1
View File
@@ -132,6 +132,7 @@ func (s *Server) RegisterRoutes(r *mux.Router) {
r.Handle("/api/admin/prune", s.requireAPIKey(http.HandlerFunc(s.handleAdminPrune))).Methods("POST")
r.Handle("/api/debug/affinity", s.requireAPIKey(http.HandlerFunc(s.handleDebugAffinity))).Methods("GET")
r.Handle("/api/dropped-packets", s.requireAPIKey(http.HandlerFunc(s.handleDroppedPackets))).Methods("GET")
r.Handle("/api/backup", s.requireAPIKey(http.HandlerFunc(s.handleBackup))).Methods("GET")
// Packet endpoints
r.HandleFunc("/api/packets/observations", s.handleBatchObservations).Methods("POST")