From b06adf9f2a04494bac2a1fc1caa26f6304bed587 Mon Sep 17 00:00:00 2001 From: Kpa-clawbot Date: Sun, 3 May 2026 17:56:42 -0700 Subject: [PATCH] =?UTF-8?q?feat:=20/api/backup=20=E2=80=94=20one-click=20S?= =?UTF-8?q?QLite=20database=20export=20(#474)=20(#1022)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Implements `GET /api/backup` — one-click SQLite database export per #474. Operators can now grab a complete, consistent snapshot of the analyzer DB with a single authenticated request — no SSH, no scripts, no DB tooling. ## Endpoint ``` GET /api/backup X-API-Key: # required → 200 OK Content-Type: application/octet-stream Content-Disposition: attachment; filename="corescope-backup-.db" ``` ## Approach Uses SQLite's `VACUUM INTO 'path'` to produce an atomic, defragmented copy of the database into a fresh file: - **Consistent**: VACUUM INTO runs at read isolation — the snapshot reflects a single point in time even while the ingestor is writing to the WAL. - **Non-blocking**: writers continue uninterrupted; we never hold a write lock. - **Works on read-only connections**: verified manually against a WAL-mode source DB (`mode=ro` connection successfully produces a snapshot). - **No corruption risk**: even if the live on-disk DB has issues, VACUUM INTO surfaces what the server can read rather than copying broken pages byte-for-byte. The snapshot is staged in `os.MkdirTemp(...)` and removed after the response body is fully streamed (deferred cleanup). Requesting client IP is logged for audit. The issue suggested an alternative in-memory rebuild path; `VACUUM INTO` is simpler, faster, and produces a strictly more accurate copy of what the server actually sees, so going with it. ## Security - Mounted under `requireAPIKey` middleware — same gate as other admin endpoints (`/api/admin/prune`, `/api/perf/reset`). - Returns 401 without a valid `X-API-Key` header. - Returns 403 if no API key is configured server-side. - `X-Content-Type-Options: nosniff` set on the response. ## TDD - **Red** (`99548f2`): `cmd/server/backup_test.go` adds `TestBackupRequiresAPIKey` + `TestBackupReturnsValidSQLiteSnapshot`. Stub handler returns 200 with no body so the tests fail on assertions (Content-Type / Content-Disposition / SQLite magic header), not on import or build errors. - **Green** (`837b2fe`): real implementation lands; both tests pass; full `go test ./...` suite stays green. ## Files - `cmd/server/backup.go` — handler implementation - `cmd/server/backup_test.go` — red-then-green tests - `cmd/server/routes.go` — route registration under `requireAPIKey` - `cmd/server/openapi.go` — OpenAPI metadata so `/api/openapi` advertises the endpoint ## Out of scope (follow-ups) - Rate limiting (issue suggested 1 req/min). Not added here — admin-key-gated endpoint with a fast snapshot path is acceptable for v1; happy to add a token-bucket limiter in a follow-up if operators report hammering. - UI button to trigger the download (frontend work — separate PR). Fixes #474 --------- Co-authored-by: corescope-bot --- cmd/server/backup.go | 89 +++++++++++++++++++++++++++++++++++++++ cmd/server/backup_test.go | 55 ++++++++++++++++++++++++ cmd/server/openapi.go | 1 + cmd/server/routes.go | 1 + 4 files changed, 146 insertions(+) create mode 100644 cmd/server/backup.go create mode 100644 cmd/server/backup_test.go diff --git a/cmd/server/backup.go b/cmd/server/backup.go new file mode 100644 index 00000000..91915ed1 --- /dev/null +++ b/cmd/server/backup.go @@ -0,0 +1,89 @@ +package main + +import ( + "fmt" + "io" + "log" + "net/http" + "os" + "path/filepath" + "strings" + "time" +) + +// handleBackup streams a consistent SQLite snapshot of the analyzer DB. +// +// Requires API-key authentication (mounted via requireAPIKey in routes.go). +// +// Strategy: SQLite's `VACUUM INTO 'path'` produces an atomic, defragmented +// copy of the current database into a new file. It runs at READ ISOLATION +// against the source DB (works on our read-only connection) and never +// blocks concurrent writers — the ingestor keeps writing to the WAL while +// the snapshot is taken from a consistent read transaction. +// +// Response: +// +// 200 OK +// Content-Type: application/octet-stream +// Content-Disposition: attachment; filename="corescope-backup-.db" +// +// +// The temp file is removed after the response is fully written, regardless +// of whether the client successfully consumed the stream. +func (s *Server) handleBackup(w http.ResponseWriter, r *http.Request) { + if s.db == nil || s.db.conn == nil { + writeError(w, http.StatusServiceUnavailable, "database unavailable") + return + } + + ts := time.Now().UTC().Unix() + clientIP := r.Header.Get("X-Forwarded-For") + if clientIP == "" { + clientIP = r.RemoteAddr + } + log.Printf("[backup] generating backup for client %s", clientIP) + + // Stage the snapshot in the OS temp dir so we never touch the live DB + // directory (avoids confusing operators / accidental WAL clobber). + tmpDir, err := os.MkdirTemp("", "corescope-backup-") + if err != nil { + writeError(w, http.StatusInternalServerError, "tempdir failed: "+err.Error()) + return + } + defer func() { + if rmErr := os.RemoveAll(tmpDir); rmErr != nil { + log.Printf("[backup] cleanup error: %v", rmErr) + } + }() + + snapshotPath := filepath.Join(tmpDir, fmt.Sprintf("corescope-backup-%d.db", ts)) + + // SQLite parses the path literal — escape any single quotes defensively. + // (mkdtemp output won't contain quotes, but be paranoid for future-proofing.) + escaped := strings.ReplaceAll(snapshotPath, "'", "''") + if _, err := s.db.conn.ExecContext(r.Context(), fmt.Sprintf("VACUUM INTO '%s'", escaped)); err != nil { + writeError(w, http.StatusInternalServerError, "snapshot failed: "+err.Error()) + return + } + + f, err := os.Open(snapshotPath) + if err != nil { + writeError(w, http.StatusInternalServerError, "open snapshot failed: "+err.Error()) + return + } + defer f.Close() + + stat, err := f.Stat() + if err == nil { + w.Header().Set("Content-Length", fmt.Sprintf("%d", stat.Size())) + } + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"corescope-backup-%d.db\"", ts)) + w.Header().Set("X-Content-Type-Options", "nosniff") + w.WriteHeader(http.StatusOK) + + if _, err := io.Copy(w, f); err != nil { + // Headers already flushed; just log. Client will see truncated stream. + log.Printf("[backup] stream error: %v", err) + } +} diff --git a/cmd/server/backup_test.go b/cmd/server/backup_test.go new file mode 100644 index 00000000..024ef3cb --- /dev/null +++ b/cmd/server/backup_test.go @@ -0,0 +1,55 @@ +package main + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +// sqliteMagic is the 16-byte file header identifying a valid SQLite 3 database. +// See https://www.sqlite.org/fileformat.html#magic_header_string +const sqliteMagic = "SQLite format 3\x00" + +func TestBackupRequiresAPIKey(t *testing.T) { + _, router := setupTestServerWithAPIKey(t, "test-secret-key-strong-enough") + + req := httptest.NewRequest("GET", "/api/backup", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusUnauthorized { + t.Fatalf("expected 401 without API key, got %d (body: %s)", w.Code, w.Body.String()) + } +} + +func TestBackupReturnsValidSQLiteSnapshot(t *testing.T) { + const apiKey = "test-secret-key-strong-enough" + _, router := setupTestServerWithAPIKey(t, apiKey) + + req := httptest.NewRequest("GET", "/api/backup", nil) + req.Header.Set("X-API-Key", apiKey) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d (body: %s)", w.Code, w.Body.String()) + } + + ct := w.Header().Get("Content-Type") + if ct != "application/octet-stream" { + t.Errorf("expected Content-Type application/octet-stream, got %q", ct) + } + + cd := w.Header().Get("Content-Disposition") + if !strings.HasPrefix(cd, "attachment;") || !strings.Contains(cd, "filename=\"corescope-backup-") || !strings.HasSuffix(cd, ".db\"") { + t.Errorf("expected Content-Disposition attachment with corescope-backup-.db filename, got %q", cd) + } + + body := w.Body.Bytes() + if len(body) < len(sqliteMagic) { + t.Fatalf("backup body too short (%d bytes) — expected SQLite file", len(body)) + } + if got := string(body[:len(sqliteMagic)]); got != sqliteMagic { + t.Fatalf("expected SQLite magic header %q, got %q", sqliteMagic, got) + } +} diff --git a/cmd/server/openapi.go b/cmd/server/openapi.go index 8149ce1c..650d50c2 100644 --- a/cmd/server/openapi.go +++ b/cmd/server/openapi.go @@ -45,6 +45,7 @@ func routeDescriptions() map[string]routeMeta { "POST /api/perf/reset": {Summary: "Reset performance stats", Tag: "admin", Auth: true}, "POST /api/admin/prune": {Summary: "Prune old data", Description: "Deletes packets and nodes older than the configured retention period.", Tag: "admin", Auth: true}, "GET /api/debug/affinity": {Summary: "Debug neighbor affinity scores", Tag: "admin", Auth: true}, + "GET /api/backup": {Summary: "Download SQLite backup", Description: "Streams a consistent SQLite snapshot of the analyzer DB (VACUUM INTO). Response is application/octet-stream with attachment filename corescope-backup-.db.", Tag: "admin", Auth: true}, // Packets "GET /api/packets": {Summary: "List packets", Description: "Returns decoded packets with filtering, sorting, and pagination.", Tag: "packets", diff --git a/cmd/server/routes.go b/cmd/server/routes.go index 292905ed..457da763 100644 --- a/cmd/server/routes.go +++ b/cmd/server/routes.go @@ -132,6 +132,7 @@ func (s *Server) RegisterRoutes(r *mux.Router) { r.Handle("/api/admin/prune", s.requireAPIKey(http.HandlerFunc(s.handleAdminPrune))).Methods("POST") r.Handle("/api/debug/affinity", s.requireAPIKey(http.HandlerFunc(s.handleDebugAffinity))).Methods("GET") r.Handle("/api/dropped-packets", s.requireAPIKey(http.HandlerFunc(s.handleDroppedPackets))).Methods("GET") + r.Handle("/api/backup", s.requireAPIKey(http.HandlerFunc(s.handleBackup))).Methods("GET") // Packet endpoints r.HandleFunc("/api/packets/observations", s.handleBatchObservations).Methods("POST")