fix: enable SQLite incremental auto-vacuum so DB shrinks after retention (#919) (#920)

Closes #919 ## Summary Enables SQLite incremental auto-vacuum so the database file actually shrinks after retention reaper deletes old data. Previously, `DELETE` operations freed pages internally but never returned disk space to the OS. ## Changes ### 1. Auto-vacuum on new databases - `PRAGMA auto_vacuum = INCREMENTAL` set via DSN pragma before `journal_mode(WAL)` in the ingestor's `OpenStoreWithInterval` - Must be set before any tables are created; DSN ordering ensures this ### 2. Post-reaper incremental vacuum - `PRAGMA incremental_vacuum(N)` runs after every retention reaper cycle (packets, metrics, observers, neighbor edges) - N defaults to 1024 pages, configurable via `db.incrementalVacuumPages` - Noop on `auto_vacuum=NONE` databases (safe before migration) - Added to both server and ingestor ### 3. Opt-in full VACUUM for existing databases - Startup check logs a clear warning if `auto_vacuum != INCREMENTAL` - `db.vacuumOnStartup: true` config triggers one-time `PRAGMA auto_vacuum = INCREMENTAL; VACUUM` - Logs start/end time for operator visibility ### 4. Documentation - `docs/user-guide/configuration.md`: retention section notes that lowering retention doesn't immediately shrink the DB - `docs/user-guide/database.md`: new guide covering WAL, auto-vacuum, migration, manual VACUUM ### 5. Tests - `TestNewDBHasIncrementalAutoVacuum` — fresh DB gets `auto_vacuum=2` - `TestExistingDBHasAutoVacuumNone` — old DB stays at `auto_vacuum=0` - `TestVacuumOnStartupMigratesDB` — full VACUUM sets `auto_vacuum=2` - `TestIncrementalVacuumReducesFreelist` — DELETE + vacuum shrinks freelist - `TestCheckAutoVacuumLogs` — handles both modes without panic - `TestConfigIncrementalVacuumPages` — config defaults and overrides ## Migration path for existing databases 1. On startup, CoreScope logs: `[db] auto_vacuum=NONE — DB needs one-time VACUUM...` 2. Set `db.vacuumOnStartup: true` in config.json 3. Restart — VACUUM runs (blocks startup, minutes on large DBs) 4. Remove `vacuumOnStartup` after migration ## Test results ``` ok github.com/corescope/server 19.448s ok github.com/corescope/ingestor 30.682s ``` --------- Co-authored-by: you <you@example.com>
2026-07-17 12:02:26 +00:00 · 2026-04-30 23:45:00 -07:00
parent 43f17ed770
commit aeae7813bc
10 changed files with 561 additions and 1 deletions
@@ -41,6 +41,7 @@ type Config struct {
 	Metrics         *MetricsConfig    `json:"metrics,omitempty"`
 	GeoFilter            *GeoFilterConfig  `json:"geo_filter,omitempty"`
 	ValidateSignatures   *bool             `json:"validateSignatures,omitempty"`
+	DB                   *DBConfig         `json:"db,omitempty"`
 }

 // GeoFilterConfig is an alias for the shared geofilter.Config type.
@@ -58,6 +59,20 @@ type MetricsConfig struct {
 	SampleIntervalSec int `json:"sampleIntervalSec"`
 }

+// DBConfig controls SQLite vacuum and maintenance behavior (#919).
+type DBConfig struct {
+	VacuumOnStartup        bool `json:"vacuumOnStartup"`        // one-time full VACUUM on startup if auto_vacuum is not INCREMENTAL
+	IncrementalVacuumPages int  `json:"incrementalVacuumPages"` // pages returned to OS per reaper cycle (default 1024)
+}
+
+// IncrementalVacuumPages returns the configured pages per vacuum or 1024 default.
+func (c *Config) IncrementalVacuumPages() int {
+	if c.DB != nil && c.DB.IncrementalVacuumPages > 0 {
+		return c.DB.IncrementalVacuumPages
+	}
+	return 1024
+}
+
 // ShouldValidateSignatures returns true (default) unless explicitly disabled.
 func (c *Config) ShouldValidateSignatures() bool {
 	if c.ValidateSignatures != nil {
@@ -59,7 +59,7 @@ func OpenStoreWithInterval(dbPath string, sampleIntervalSec int) (*Store, error)
 		return nil, fmt.Errorf("creating data dir: %w", err)
 	}

-	db, err := sql.Open("sqlite", dbPath+"?_pragma=journal_mode(WAL)&_pragma=foreign_keys(ON)&_pragma=busy_timeout(5000)")
+	db, err := sql.Open("sqlite", dbPath+"?_pragma=auto_vacuum(INCREMENTAL)&_pragma=journal_mode(WAL)&_pragma=foreign_keys(ON)&_pragma=busy_timeout(5000)")
 	if err != nil {
 		return nil, fmt.Errorf("opening db: %w", err)
 	}
@@ -85,6 +85,9 @@ func OpenStoreWithInterval(dbPath string, sampleIntervalSec int) (*Store, error)
 }

 func applySchema(db *sql.DB) error {
+	// auto_vacuum=INCREMENTAL is set via DSN pragma (must be before journal_mode).
+	// Logging of current mode is handled by CheckAutoVacuum — no duplicate log here.
+
 	schema := `
 		CREATE TABLE IF NOT EXISTS nodes (
 			public_key TEXT PRIMARY KEY,
@@ -788,6 +791,58 @@ func (s *Store) PruneOldMetrics(retentionDays int) (int64, error) {
 	return n, nil
 }

+// CheckAutoVacuum inspects the current auto_vacuum mode and logs a warning
+// if not INCREMENTAL. Performs opt-in full VACUUM if db.vacuumOnStartup is set (#919).
+func (s *Store) CheckAutoVacuum(cfg *Config) {
+	var autoVacuum int
+	if err := s.db.QueryRow("PRAGMA auto_vacuum").Scan(&autoVacuum); err != nil {
+		log.Printf("[db] warning: could not read auto_vacuum: %v", err)
+		return
+	}
+
+	if autoVacuum == 2 {
+		log.Printf("[db] auto_vacuum=INCREMENTAL")
+		return
+	}
+
+	modes := map[int]string{0: "NONE", 1: "FULL", 2: "INCREMENTAL"}
+	mode := modes[autoVacuum]
+	if mode == "" {
+		mode = fmt.Sprintf("UNKNOWN(%d)", autoVacuum)
+	}
+
+	log.Printf("[db] auto_vacuum=%s — DB needs one-time VACUUM to enable incremental auto-vacuum. "+
+		"Set db.vacuumOnStartup: true in config to migrate (will block startup for several minutes on large DBs). "+
+		"See https://github.com/Kpa-clawbot/CoreScope/issues/919", mode)
+
+	if cfg.DB != nil && cfg.DB.VacuumOnStartup {
+		// WARNING: Full VACUUM creates a temporary copy of the entire DB file.
+		// Requires ~2× the DB file size in free disk space or it will fail.
+		log.Printf("[db] vacuumOnStartup=true — starting one-time full VACUUM (ensure 2x DB size free disk space)...")
+		start := time.Now()
+
+		if _, err := s.db.Exec("PRAGMA auto_vacuum = INCREMENTAL"); err != nil {
+			log.Printf("[db] VACUUM failed: could not set auto_vacuum: %v", err)
+			return
+		}
+		if _, err := s.db.Exec("VACUUM"); err != nil {
+			log.Printf("[db] VACUUM failed: %v", err)
+			return
+		}
+
+		elapsed := time.Since(start)
+		log.Printf("[db] VACUUM complete in %v — auto_vacuum is now INCREMENTAL", elapsed.Round(time.Millisecond))
+	}
+}
+
+// RunIncrementalVacuum returns free pages to the OS (#919).
+// Safe to call on auto_vacuum=NONE databases (noop).
+func (s *Store) RunIncrementalVacuum(pages int) {
+	if _, err := s.db.Exec(fmt.Sprintf("PRAGMA incremental_vacuum(%d)", pages)); err != nil {
+		log.Printf("[vacuum] incremental_vacuum error: %v", err)
+	}
+}
+
 // Checkpoint forces a WAL checkpoint to release the WAL lock file,
 // preventing lock contention with a new process starting up.
 func (s *Store) Checkpoint() {
@@ -57,6 +57,9 @@ func main() {
 	defer store.Close()
 	log.Printf("SQLite opened: %s", cfg.DBPath)

+	// Check auto_vacuum mode and optionally migrate (#919)
+	store.CheckAutoVacuum(cfg)
+
 	// Node retention: move stale nodes to inactive_nodes on startup
 	nodeDays := cfg.NodeDaysOrDefault()
 	store.MoveStaleNodes(nodeDays)
@@ -69,12 +72,15 @@ func main() {
 	metricsDays := cfg.MetricsRetentionDays()
 	store.PruneOldMetrics(metricsDays)
 	store.PruneDroppedPackets(metricsDays)
+	vacuumPages := cfg.IncrementalVacuumPages()
+	store.RunIncrementalVacuum(vacuumPages)

 	// Daily ticker for node retention
 	retentionTicker := time.NewTicker(1 * time.Hour)
 	go func() {
 		for range retentionTicker.C {
 			store.MoveStaleNodes(nodeDays)
+			store.RunIncrementalVacuum(vacuumPages)
 		}
 	}()

@@ -83,8 +89,10 @@ func main() {
 	go func() {
 		time.Sleep(90 * time.Second) // stagger after metrics prune
 		store.RemoveStaleObservers(observerDays)
+		store.RunIncrementalVacuum(vacuumPages)
 		for range observerRetentionTicker.C {
 			store.RemoveStaleObservers(observerDays)
+			store.RunIncrementalVacuum(vacuumPages)
 		}
 	}()

@@ -94,6 +102,7 @@ func main() {
 		for range metricsRetentionTicker.C {
 			store.PruneOldMetrics(metricsDays)
 			store.PruneDroppedPackets(metricsDays)
+			store.RunIncrementalVacuum(vacuumPages)
 		}
 	}()

@@ -62,6 +62,8 @@ type Config struct {

 	Retention *RetentionConfig `json:"retention,omitempty"`

+	DB *DBConfig `json:"db,omitempty"`
+
 	PacketStore *PacketStoreConfig `json:"packetStore,omitempty"`

 	GeoFilter *GeoFilterConfig `json:"geo_filter,omitempty"`
@@ -129,6 +131,20 @@ type RetentionConfig struct {
 	MetricsDays   int `json:"metricsDays"`
 }

+// DBConfig controls SQLite vacuum and maintenance behavior (#919).
+type DBConfig struct {
+	VacuumOnStartup        bool `json:"vacuumOnStartup"`        // one-time full VACUUM on startup if auto_vacuum is not INCREMENTAL
+	IncrementalVacuumPages int  `json:"incrementalVacuumPages"` // pages returned to OS per reaper cycle (default 1024)
+}
+
+// IncrementalVacuumPages returns the configured pages per vacuum or 1024 default.
+func (c *Config) IncrementalVacuumPages() int {
+	if c.DB != nil && c.DB.IncrementalVacuumPages > 0 {
+		return c.DB.IncrementalVacuumPages
+	}
+	return 1024
+}
+
 // MetricsRetentionDays returns configured metrics retention or 30 days default.
 func (c *Config) MetricsRetentionDays() int {
 	if c.Retention != nil && c.Retention.MetricsDays > 0 {
@@ -0,0 +1,262 @@
+package main
+
+import (
+	"database/sql"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	_ "modernc.org/sqlite"
+)
+
+// createFreshIngestorDB creates a SQLite DB using the ingestor's applySchema logic
+// (simulated here) with auto_vacuum=INCREMENTAL set before tables.
+func createFreshDBWithAutoVacuum(t *testing.T, path string) *sql.DB {
+	t.Helper()
+	// auto_vacuum must be set via DSN before journal_mode creates the DB file
+	db, err := sql.Open("sqlite", path+"?_pragma=auto_vacuum(INCREMENTAL)&_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)")
+	if err != nil {
+		t.Fatal(err)
+	}
+	db.SetMaxOpenConns(1)
+
+	// Create minimal schema
+	_, err = db.Exec(`
+		CREATE TABLE transmissions (
+			id INTEGER PRIMARY KEY AUTOINCREMENT,
+			raw_hex TEXT NOT NULL,
+			hash TEXT NOT NULL UNIQUE,
+			first_seen TEXT NOT NULL,
+			route_type INTEGER,
+			payload_type INTEGER,
+			payload_version INTEGER,
+			decoded_json TEXT,
+			created_at TEXT DEFAULT (datetime('now')),
+			channel_hash TEXT
+		);
+		CREATE TABLE observations (
+			id INTEGER PRIMARY KEY AUTOINCREMENT,
+			transmission_id INTEGER NOT NULL REFERENCES transmissions(id),
+			observer_idx INTEGER,
+			direction TEXT,
+			snr REAL,
+			rssi REAL,
+			score INTEGER,
+			path_json TEXT,
+			timestamp INTEGER NOT NULL
+		);
+	`)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return db
+}
+
+func TestNewDBHasIncrementalAutoVacuum(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "test.db")
+
+	db := createFreshDBWithAutoVacuum(t, path)
+	defer db.Close()
+
+	var autoVacuum int
+	if err := db.QueryRow("PRAGMA auto_vacuum").Scan(&autoVacuum); err != nil {
+		t.Fatal(err)
+	}
+	if autoVacuum != 2 {
+		t.Fatalf("expected auto_vacuum=2 (INCREMENTAL), got %d", autoVacuum)
+	}
+}
+
+func TestExistingDBHasAutoVacuumNone(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "test.db")
+
+	// Create DB WITHOUT setting auto_vacuum (simulates old DB)
+	db, err := sql.Open("sqlite", path+"?_pragma=journal_mode(WAL)")
+	if err != nil {
+		t.Fatal(err)
+	}
+	db.SetMaxOpenConns(1)
+	_, err = db.Exec("CREATE TABLE dummy (id INTEGER PRIMARY KEY)")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var autoVacuum int
+	if err := db.QueryRow("PRAGMA auto_vacuum").Scan(&autoVacuum); err != nil {
+		t.Fatal(err)
+	}
+	db.Close()
+
+	if autoVacuum != 0 {
+		t.Fatalf("expected auto_vacuum=0 (NONE) for old DB, got %d", autoVacuum)
+	}
+}
+
+func TestVacuumOnStartupMigratesDB(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "test.db")
+
+	// Create DB without auto_vacuum (old DB)
+	db, err := sql.Open("sqlite", path+"?_pragma=journal_mode(WAL)")
+	if err != nil {
+		t.Fatal(err)
+	}
+	db.SetMaxOpenConns(1)
+	_, err = db.Exec("CREATE TABLE dummy (id INTEGER PRIMARY KEY)")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var before int
+	db.QueryRow("PRAGMA auto_vacuum").Scan(&before)
+	if before != 0 {
+		t.Fatalf("precondition: expected auto_vacuum=0, got %d", before)
+	}
+	db.Close()
+
+	// Simulate vacuumOnStartup migration using openRW
+	rw, err := openRW(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := rw.Exec("PRAGMA auto_vacuum = INCREMENTAL"); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := rw.Exec("VACUUM"); err != nil {
+		t.Fatal(err)
+	}
+	rw.Close()
+
+	// Verify migration
+	db2, err := sql.Open("sqlite", path+"?mode=ro")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db2.Close()
+
+	var after int
+	if err := db2.QueryRow("PRAGMA auto_vacuum").Scan(&after); err != nil {
+		t.Fatal(err)
+	}
+	if after != 2 {
+		t.Fatalf("expected auto_vacuum=2 after VACUUM migration, got %d", after)
+	}
+}
+
+func TestIncrementalVacuumReducesFreelist(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "test.db")
+
+	db := createFreshDBWithAutoVacuum(t, path)
+
+	// Insert a bunch of data
+	now := time.Now().UTC().Format(time.RFC3339)
+	for i := 0; i < 500; i++ {
+		_, err := db.Exec(
+			"INSERT INTO transmissions (raw_hex, hash, first_seen) VALUES (?, ?, ?)",
+			strings.Repeat("AA", 200), // ~400 bytes each
+			"hash_"+string(rune('A'+i%26))+string(rune('0'+i/26)),
+			now,
+		)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// Get file size before delete
+	db.Close()
+	infoBefore, _ := os.Stat(path)
+	sizeBefore := infoBefore.Size()
+
+	// Reopen and delete all
+	db, err := sql.Open("sqlite", path+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)")
+	if err != nil {
+		t.Fatal(err)
+	}
+	db.SetMaxOpenConns(1)
+	defer db.Close()
+
+	_, err = db.Exec("DELETE FROM transmissions")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Check freelist before vacuum
+	var freelistBefore int64
+	db.QueryRow("PRAGMA freelist_count").Scan(&freelistBefore)
+	if freelistBefore == 0 {
+		t.Fatal("expected non-zero freelist after DELETE")
+	}
+
+	// Run incremental vacuum
+	_, err = db.Exec("PRAGMA incremental_vacuum(10000)")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Check freelist after vacuum
+	var freelistAfter int64
+	db.QueryRow("PRAGMA freelist_count").Scan(&freelistAfter)
+	if freelistAfter >= freelistBefore {
+		t.Fatalf("expected freelist to shrink: before=%d after=%d", freelistBefore, freelistAfter)
+	}
+
+	// Checkpoint WAL and check file size shrunk
+	db.Exec("PRAGMA wal_checkpoint(TRUNCATE)")
+	db.Close()
+	infoAfter, _ := os.Stat(path)
+	sizeAfter := infoAfter.Size()
+	if sizeAfter >= sizeBefore {
+		t.Logf("warning: file did not shrink (before=%d after=%d) — may depend on page reuse", sizeBefore, sizeAfter)
+	}
+}
+
+func TestCheckAutoVacuumLogs(t *testing.T) {
+	// This test verifies checkAutoVacuum doesn't panic on various configs
+	dir := t.TempDir()
+	path := filepath.Join(dir, "test.db")
+
+	// Create a fresh DB with auto_vacuum=INCREMENTAL
+	dbConn := createFreshDBWithAutoVacuum(t, path)
+	db := &DB{conn: dbConn, path: path}
+	cfg := &Config{}
+
+	// Should not panic
+	checkAutoVacuum(db, cfg, path)
+	dbConn.Close()
+
+	// Create a DB without auto_vacuum
+	path2 := filepath.Join(dir, "test2.db")
+	dbConn2, _ := sql.Open("sqlite", path2+"?_pragma=journal_mode(WAL)")
+	dbConn2.SetMaxOpenConns(1)
+	dbConn2.Exec("CREATE TABLE dummy (id INTEGER PRIMARY KEY)")
+	db2 := &DB{conn: dbConn2, path: path2}
+
+	// Should log warning but not panic
+	checkAutoVacuum(db2, cfg, path2)
+	dbConn2.Close()
+}
+
+func TestConfigIncrementalVacuumPages(t *testing.T) {
+	// Default
+	cfg := &Config{}
+	if cfg.IncrementalVacuumPages() != 1024 {
+		t.Fatalf("expected default 1024, got %d", cfg.IncrementalVacuumPages())
+	}
+
+	// Custom
+	cfg.DB = &DBConfig{IncrementalVacuumPages: 512}
+	if cfg.IncrementalVacuumPages() != 512 {
+		t.Fatalf("expected 512, got %d", cfg.IncrementalVacuumPages())
+	}
+
+	// Zero should return default
+	cfg.DB.IncrementalVacuumPages = 0
+	if cfg.IncrementalVacuumPages() != 1024 {
+		t.Fatalf("expected default 1024 for zero, got %d", cfg.IncrementalVacuumPages())
+	}
+}
@@ -148,6 +148,9 @@ func main() {
 			stats.TotalTransmissions, stats.TotalObservations, stats.TotalNodes, stats.TotalObservers)
 	}

+	// Check auto_vacuum mode and optionally migrate (#919)
+	checkAutoVacuum(database, cfg, resolvedDB)
+
 	// In-memory packet store
 	store := NewPacketStore(database, cfg.PacketStore, cfg.CacheTTL)
 	if err := store.Load(); err != nil {
@@ -266,6 +269,7 @@ func main() {
 	defer stopEviction()

 	// Auto-prune old packets if retention.packetDays is configured
+	vacuumPages := cfg.IncrementalVacuumPages()
 	var stopPrune func()
 	if cfg.Retention != nil && cfg.Retention.PacketDays > 0 {
 		days := cfg.Retention.PacketDays
@@ -286,6 +290,9 @@ func main() {
 				log.Printf("[prune] error: %v", err)
 			} else {
 				log.Printf("[prune] deleted %d transmissions older than %d days", n, days)
+				if n > 0 {
+					runIncrementalVacuum(resolvedDB, vacuumPages)
+				}
 			}
 			for {
 				select {
@@ -294,6 +301,9 @@ func main() {
 						log.Printf("[prune] error: %v", err)
 					} else {
 						log.Printf("[prune] deleted %d transmissions older than %d days", n, days)
+						if n > 0 {
+							runIncrementalVacuum(resolvedDB, vacuumPages)
+						}
 					}
 				case <-pruneDone:
 					return
@@ -321,10 +331,12 @@ func main() {
 			}()
 			time.Sleep(2 * time.Minute) // stagger after packet prune
 			database.PruneOldMetrics(metricsDays)
+			runIncrementalVacuum(resolvedDB, vacuumPages)
 			for {
 				select {
 				case <-metricsPruneTicker.C:
 					database.PruneOldMetrics(metricsDays)
+					runIncrementalVacuum(resolvedDB, vacuumPages)
 				case <-metricsPruneDone:
 					return
 				}
@@ -354,10 +366,12 @@ func main() {
 				}()
 				time.Sleep(3 * time.Minute) // stagger after metrics prune
 				database.RemoveStaleObservers(observerDays)
+				runIncrementalVacuum(resolvedDB, vacuumPages)
 				for {
 					select {
 					case <-observerPruneTicker.C:
 						database.RemoveStaleObservers(observerDays)
+						runIncrementalVacuum(resolvedDB, vacuumPages)
 					case <-observerPruneDone:
 						return
 					}
@@ -388,6 +402,7 @@ func main() {
 			g := store.graph
 			store.mu.RUnlock()
 			PruneNeighborEdges(dbPath, g, maxAgeDays)
+			runIncrementalVacuum(resolvedDB, vacuumPages)
 			for {
 				select {
 				case <-edgePruneTicker.C:
@@ -395,6 +410,7 @@ func main() {
 					g := store.graph
 					store.mu.RUnlock()
 					PruneNeighborEdges(dbPath, g, maxAgeDays)
+					runIncrementalVacuum(resolvedDB, vacuumPages)
 				case <-edgePruneDone:
 					return
 				}
@@ -0,0 +1,84 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"time"
+)
+
+// checkAutoVacuum inspects the current auto_vacuum mode and logs a warning
+// if it's not INCREMENTAL. Optionally performs a one-time full VACUUM if
+// the operator has set db.vacuumOnStartup: true in config (#919).
+func checkAutoVacuum(db *DB, cfg *Config, dbPath string) {
+	var autoVacuum int
+	if err := db.conn.QueryRow("PRAGMA auto_vacuum").Scan(&autoVacuum); err != nil {
+		log.Printf("[db] warning: could not read auto_vacuum: %v", err)
+		return
+	}
+
+	if autoVacuum == 2 {
+		log.Printf("[db] auto_vacuum=INCREMENTAL")
+		return
+	}
+
+	modes := map[int]string{0: "NONE", 1: "FULL", 2: "INCREMENTAL"}
+	mode := modes[autoVacuum]
+	if mode == "" {
+		mode = fmt.Sprintf("UNKNOWN(%d)", autoVacuum)
+	}
+
+	log.Printf("[db] auto_vacuum=%s — DB needs one-time VACUUM to enable incremental auto-vacuum. "+
+		"Set db.vacuumOnStartup: true in config to migrate (will block startup for several minutes on large DBs). "+
+		"See https://github.com/Kpa-clawbot/CoreScope/issues/919", mode)
+
+	if cfg.DB != nil && cfg.DB.VacuumOnStartup {
+		// WARNING: Full VACUUM creates a temporary copy of the entire DB file.
+		// Requires ~2× the DB file size in free disk space or it will fail.
+		log.Printf("[db] vacuumOnStartup=true — starting one-time full VACUUM (ensure 2x DB size free disk space)...")
+		start := time.Now()
+
+		rw, err := openRW(dbPath)
+		if err != nil {
+			log.Printf("[db] VACUUM failed: could not open RW connection: %v", err)
+			return
+		}
+		defer rw.Close()
+
+		if _, err := rw.Exec("PRAGMA auto_vacuum = INCREMENTAL"); err != nil {
+			log.Printf("[db] VACUUM failed: could not set auto_vacuum: %v", err)
+			return
+		}
+		if _, err := rw.Exec("VACUUM"); err != nil {
+			log.Printf("[db] VACUUM failed: %v", err)
+			return
+		}
+
+		elapsed := time.Since(start)
+		log.Printf("[db] VACUUM complete in %v — auto_vacuum is now INCREMENTAL", elapsed.Round(time.Millisecond))
+
+		// Re-check
+		var newMode int
+		if err := db.conn.QueryRow("PRAGMA auto_vacuum").Scan(&newMode); err == nil {
+			if newMode == 2 {
+				log.Printf("[db] auto_vacuum=INCREMENTAL (confirmed after VACUUM)")
+			} else {
+				log.Printf("[db] warning: auto_vacuum=%d after VACUUM — expected 2", newMode)
+			}
+		}
+	}
+}
+
+// runIncrementalVacuum runs PRAGMA incremental_vacuum(N) on a read-write
+// connection. Safe to call on auto_vacuum=NONE databases (noop).
+func runIncrementalVacuum(dbPath string, pages int) {
+	rw, err := openRW(dbPath)
+	if err != nil {
+		log.Printf("[vacuum] could not open RW connection: %v", err)
+		return
+	}
+	defer rw.Close()
+
+	if _, err := rw.Exec(fmt.Sprintf("PRAGMA incremental_vacuum(%d)", pages)); err != nil {
+		log.Printf("[vacuum] incremental_vacuum error: %v", err)
+	}
+}
@@ -9,6 +9,11 @@
    "packetDays": 30,
    "_comment": "nodeDays: nodes not seen in N days moved to inactive_nodes (default 7). observerDays: observers not sending data in N days are removed (-1 = keep forever, default 14). packetDays: transmissions older than N days are deleted (0 = disabled)."
  },
+  "db": {
+    "vacuumOnStartup": false,
+    "incrementalVacuumPages": 1024,
+    "_comment": "vacuumOnStartup: run one-time full VACUUM to enable incremental auto-vacuum on existing DBs (blocks startup for minutes on large DBs; requires 2x DB file size in free disk space). incrementalVacuumPages: free pages returned to OS after each retention reaper cycle (default 1024). See #919."
+  },
  "https": {
    "cert": "/path/to/cert.pem",
    "key": "/path/to/key.pem",
@@ -98,6 +98,22 @@ How long (in hours) before a node is marked degraded or silent:
 | `retention.nodeDays` | `7` | Nodes not seen in N days move to inactive |
 | `retention.packetDays` | `30` | Packets older than N days are deleted daily |

+> **Note:** Lowering retention does **not** immediately shrink the database file.
+> SQLite marks deleted pages as free but does not return them to the filesystem
+> unless [incremental auto-vacuum](database.md) is enabled. New databases created
+> after v0.x.x have auto-vacuum enabled automatically. Existing databases require
+> a one-time migration — see the [Database](database.md) guide.
+
+## Database
+
+| Field | Default | Description |
+|-------|---------|-------------|
+| `db.vacuumOnStartup` | `false` | Run a one-time full `VACUUM` on startup to enable incremental auto-vacuum (blocks for minutes on large DBs) |
+| `db.incrementalVacuumPages` | `1024` | Free pages returned to the OS after each retention reaper cycle |
+
+See [Database](database.md) for details on SQLite auto-vacuum, WAL, and manual maintenance.
+See [#919](https://github.com/Kpa-clawbot/CoreScope/issues/919) for background.
+
 ## Channel decryption

 | Field | Description |
@@ -0,0 +1,82 @@
+# Database
+
+CoreScope uses SQLite in WAL (Write-Ahead Log) mode for both the server
+(read-only) and ingestor (read-write).
+
+## WAL mode
+
+WAL mode allows concurrent reads while writes happen. It is set automatically
+at connection time via `PRAGMA journal_mode=WAL`. No operator action needed.
+
+The WAL file (`meshcore.db-wal`) grows during writes and is checkpointed
+(merged back into the main DB) periodically and at clean shutdown.
+
+## Auto-vacuum
+
+By default, SQLite does not shrink the database file after `DELETE` operations.
+Deleted pages are marked free and reused by future writes, but the file size
+on disk stays the same. This is surprising when lowering retention settings.
+
+### New databases
+
+Databases created after this feature was added automatically have
+`PRAGMA auto_vacuum = INCREMENTAL`. After each retention reaper cycle,
+CoreScope runs `PRAGMA incremental_vacuum(N)` to return free pages to the OS.
+
+### Existing databases
+
+The `auto_vacuum` mode is stored in the database header and can only be changed
+by rewriting the entire file with `VACUUM`. CoreScope will **not** do this
+automatically — on large databases (5+ GB seen in the wild) it takes minutes
+and holds an exclusive lock.
+
+**To migrate an existing database:**
+
+1. At startup, CoreScope logs a warning:
+   ```
+   [db] auto_vacuum=NONE — DB needs one-time VACUUM to enable incremental auto-vacuum.
+   ```
+2. **Ensure at least 2× the database file size in free disk space.** Full VACUUM
+   creates a temporary copy of the entire file — on a near-full disk it will fail.
+3. Set `db.vacuumOnStartup: true` in your `config.json`:
+   ```json
+   {
+     "db": {
+       "vacuumOnStartup": true
+     }
+   }
+   ```
+4. Restart CoreScope. The one-time `VACUUM` will run and block startup.
+5. After migration, remove or set `vacuumOnStartup: false` — it's not needed again.
+
+### Configuration
+
+| Field | Default | Description |
+|-------|---------|-------------|
+| `db.vacuumOnStartup` | `false` | One-time full VACUUM to enable incremental auto-vacuum |
+| `db.incrementalVacuumPages` | `1024` | Pages returned to OS per reaper cycle |
+
+## Manual VACUUM
+
+You can also run a manual vacuum from the SQLite CLI:
+
+```bash
+sqlite3 data/meshcore.db "PRAGMA auto_vacuum = INCREMENTAL; VACUUM;"
+```
+
+This is equivalent to `vacuumOnStartup: true` but can be done offline.
+
+> ⚠️ Full VACUUM requires **2× the database file size** in free disk space (it
+> creates a temporary copy). Check with `ls -lh data/meshcore.db` before running.
+
+## Checking current mode
+
+```bash
+sqlite3 data/meshcore.db "PRAGMA auto_vacuum;"
+```
+
+- `0` = NONE (default for old databases)
+- `1` = FULL (automatic, but slower writes)
+- `2` = INCREMENTAL (recommended — CoreScope triggers vacuum after deletes)
+
+See [#919](https://github.com/Kpa-clawbot/CoreScope/issues/919) for background on this feature.