From aeae7813bcaa31c1939290105fd89ed9e51a0f7c Mon Sep 17 00:00:00 2001 From: Kpa-clawbot Date: Thu, 30 Apr 2026 23:45:00 -0700 Subject: [PATCH] fix: enable SQLite incremental auto-vacuum so DB shrinks after retention (#919) (#920) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #919 ## Summary Enables SQLite incremental auto-vacuum so the database file actually shrinks after retention reaper deletes old data. Previously, `DELETE` operations freed pages internally but never returned disk space to the OS. ## Changes ### 1. Auto-vacuum on new databases - `PRAGMA auto_vacuum = INCREMENTAL` set via DSN pragma before `journal_mode(WAL)` in the ingestor's `OpenStoreWithInterval` - Must be set before any tables are created; DSN ordering ensures this ### 2. Post-reaper incremental vacuum - `PRAGMA incremental_vacuum(N)` runs after every retention reaper cycle (packets, metrics, observers, neighbor edges) - N defaults to 1024 pages, configurable via `db.incrementalVacuumPages` - Noop on `auto_vacuum=NONE` databases (safe before migration) - Added to both server and ingestor ### 3. Opt-in full VACUUM for existing databases - Startup check logs a clear warning if `auto_vacuum != INCREMENTAL` - `db.vacuumOnStartup: true` config triggers one-time `PRAGMA auto_vacuum = INCREMENTAL; VACUUM` - Logs start/end time for operator visibility ### 4. Documentation - `docs/user-guide/configuration.md`: retention section notes that lowering retention doesn't immediately shrink the DB - `docs/user-guide/database.md`: new guide covering WAL, auto-vacuum, migration, manual VACUUM ### 5. Tests - `TestNewDBHasIncrementalAutoVacuum` — fresh DB gets `auto_vacuum=2` - `TestExistingDBHasAutoVacuumNone` — old DB stays at `auto_vacuum=0` - `TestVacuumOnStartupMigratesDB` — full VACUUM sets `auto_vacuum=2` - `TestIncrementalVacuumReducesFreelist` — DELETE + vacuum shrinks freelist - `TestCheckAutoVacuumLogs` — handles both modes without panic - `TestConfigIncrementalVacuumPages` — config defaults and overrides ## Migration path for existing databases 1. On startup, CoreScope logs: `[db] auto_vacuum=NONE — DB needs one-time VACUUM...` 2. Set `db.vacuumOnStartup: true` in config.json 3. Restart — VACUUM runs (blocks startup, minutes on large DBs) 4. Remove `vacuumOnStartup` after migration ## Test results ``` ok github.com/corescope/server 19.448s ok github.com/corescope/ingestor 30.682s ``` --------- Co-authored-by: you --- cmd/ingestor/config.go | 15 ++ cmd/ingestor/db.go | 57 ++++++- cmd/ingestor/main.go | 9 ++ cmd/server/config.go | 16 ++ cmd/server/db_vacuum_test.go | 262 +++++++++++++++++++++++++++++++ cmd/server/main.go | 16 ++ cmd/server/vacuum.go | 84 ++++++++++ config.example.json | 5 + docs/user-guide/configuration.md | 16 ++ docs/user-guide/database.md | 82 ++++++++++ 10 files changed, 561 insertions(+), 1 deletion(-) create mode 100644 cmd/server/db_vacuum_test.go create mode 100644 cmd/server/vacuum.go create mode 100644 docs/user-guide/database.md diff --git a/cmd/ingestor/config.go b/cmd/ingestor/config.go index 70c18fbe..910d3b95 100644 --- a/cmd/ingestor/config.go +++ b/cmd/ingestor/config.go @@ -41,6 +41,7 @@ type Config struct { Metrics *MetricsConfig `json:"metrics,omitempty"` GeoFilter *GeoFilterConfig `json:"geo_filter,omitempty"` ValidateSignatures *bool `json:"validateSignatures,omitempty"` + DB *DBConfig `json:"db,omitempty"` } // GeoFilterConfig is an alias for the shared geofilter.Config type. @@ -58,6 +59,20 @@ type MetricsConfig struct { SampleIntervalSec int `json:"sampleIntervalSec"` } +// DBConfig controls SQLite vacuum and maintenance behavior (#919). +type DBConfig struct { + VacuumOnStartup bool `json:"vacuumOnStartup"` // one-time full VACUUM on startup if auto_vacuum is not INCREMENTAL + IncrementalVacuumPages int `json:"incrementalVacuumPages"` // pages returned to OS per reaper cycle (default 1024) +} + +// IncrementalVacuumPages returns the configured pages per vacuum or 1024 default. +func (c *Config) IncrementalVacuumPages() int { + if c.DB != nil && c.DB.IncrementalVacuumPages > 0 { + return c.DB.IncrementalVacuumPages + } + return 1024 +} + // ShouldValidateSignatures returns true (default) unless explicitly disabled. func (c *Config) ShouldValidateSignatures() bool { if c.ValidateSignatures != nil { diff --git a/cmd/ingestor/db.go b/cmd/ingestor/db.go index bada26c8..93d3dc53 100644 --- a/cmd/ingestor/db.go +++ b/cmd/ingestor/db.go @@ -59,7 +59,7 @@ func OpenStoreWithInterval(dbPath string, sampleIntervalSec int) (*Store, error) return nil, fmt.Errorf("creating data dir: %w", err) } - db, err := sql.Open("sqlite", dbPath+"?_pragma=journal_mode(WAL)&_pragma=foreign_keys(ON)&_pragma=busy_timeout(5000)") + db, err := sql.Open("sqlite", dbPath+"?_pragma=auto_vacuum(INCREMENTAL)&_pragma=journal_mode(WAL)&_pragma=foreign_keys(ON)&_pragma=busy_timeout(5000)") if err != nil { return nil, fmt.Errorf("opening db: %w", err) } @@ -85,6 +85,9 @@ func OpenStoreWithInterval(dbPath string, sampleIntervalSec int) (*Store, error) } func applySchema(db *sql.DB) error { + // auto_vacuum=INCREMENTAL is set via DSN pragma (must be before journal_mode). + // Logging of current mode is handled by CheckAutoVacuum — no duplicate log here. + schema := ` CREATE TABLE IF NOT EXISTS nodes ( public_key TEXT PRIMARY KEY, @@ -788,6 +791,58 @@ func (s *Store) PruneOldMetrics(retentionDays int) (int64, error) { return n, nil } +// CheckAutoVacuum inspects the current auto_vacuum mode and logs a warning +// if not INCREMENTAL. Performs opt-in full VACUUM if db.vacuumOnStartup is set (#919). +func (s *Store) CheckAutoVacuum(cfg *Config) { + var autoVacuum int + if err := s.db.QueryRow("PRAGMA auto_vacuum").Scan(&autoVacuum); err != nil { + log.Printf("[db] warning: could not read auto_vacuum: %v", err) + return + } + + if autoVacuum == 2 { + log.Printf("[db] auto_vacuum=INCREMENTAL") + return + } + + modes := map[int]string{0: "NONE", 1: "FULL", 2: "INCREMENTAL"} + mode := modes[autoVacuum] + if mode == "" { + mode = fmt.Sprintf("UNKNOWN(%d)", autoVacuum) + } + + log.Printf("[db] auto_vacuum=%s — DB needs one-time VACUUM to enable incremental auto-vacuum. "+ + "Set db.vacuumOnStartup: true in config to migrate (will block startup for several minutes on large DBs). "+ + "See https://github.com/Kpa-clawbot/CoreScope/issues/919", mode) + + if cfg.DB != nil && cfg.DB.VacuumOnStartup { + // WARNING: Full VACUUM creates a temporary copy of the entire DB file. + // Requires ~2× the DB file size in free disk space or it will fail. + log.Printf("[db] vacuumOnStartup=true — starting one-time full VACUUM (ensure 2x DB size free disk space)...") + start := time.Now() + + if _, err := s.db.Exec("PRAGMA auto_vacuum = INCREMENTAL"); err != nil { + log.Printf("[db] VACUUM failed: could not set auto_vacuum: %v", err) + return + } + if _, err := s.db.Exec("VACUUM"); err != nil { + log.Printf("[db] VACUUM failed: %v", err) + return + } + + elapsed := time.Since(start) + log.Printf("[db] VACUUM complete in %v — auto_vacuum is now INCREMENTAL", elapsed.Round(time.Millisecond)) + } +} + +// RunIncrementalVacuum returns free pages to the OS (#919). +// Safe to call on auto_vacuum=NONE databases (noop). +func (s *Store) RunIncrementalVacuum(pages int) { + if _, err := s.db.Exec(fmt.Sprintf("PRAGMA incremental_vacuum(%d)", pages)); err != nil { + log.Printf("[vacuum] incremental_vacuum error: %v", err) + } +} + // Checkpoint forces a WAL checkpoint to release the WAL lock file, // preventing lock contention with a new process starting up. func (s *Store) Checkpoint() { diff --git a/cmd/ingestor/main.go b/cmd/ingestor/main.go index 481c7cc1..b0b94bed 100644 --- a/cmd/ingestor/main.go +++ b/cmd/ingestor/main.go @@ -57,6 +57,9 @@ func main() { defer store.Close() log.Printf("SQLite opened: %s", cfg.DBPath) + // Check auto_vacuum mode and optionally migrate (#919) + store.CheckAutoVacuum(cfg) + // Node retention: move stale nodes to inactive_nodes on startup nodeDays := cfg.NodeDaysOrDefault() store.MoveStaleNodes(nodeDays) @@ -69,12 +72,15 @@ func main() { metricsDays := cfg.MetricsRetentionDays() store.PruneOldMetrics(metricsDays) store.PruneDroppedPackets(metricsDays) + vacuumPages := cfg.IncrementalVacuumPages() + store.RunIncrementalVacuum(vacuumPages) // Daily ticker for node retention retentionTicker := time.NewTicker(1 * time.Hour) go func() { for range retentionTicker.C { store.MoveStaleNodes(nodeDays) + store.RunIncrementalVacuum(vacuumPages) } }() @@ -83,8 +89,10 @@ func main() { go func() { time.Sleep(90 * time.Second) // stagger after metrics prune store.RemoveStaleObservers(observerDays) + store.RunIncrementalVacuum(vacuumPages) for range observerRetentionTicker.C { store.RemoveStaleObservers(observerDays) + store.RunIncrementalVacuum(vacuumPages) } }() @@ -94,6 +102,7 @@ func main() { for range metricsRetentionTicker.C { store.PruneOldMetrics(metricsDays) store.PruneDroppedPackets(metricsDays) + store.RunIncrementalVacuum(vacuumPages) } }() diff --git a/cmd/server/config.go b/cmd/server/config.go index 6039d41e..f21ef207 100644 --- a/cmd/server/config.go +++ b/cmd/server/config.go @@ -62,6 +62,8 @@ type Config struct { Retention *RetentionConfig `json:"retention,omitempty"` + DB *DBConfig `json:"db,omitempty"` + PacketStore *PacketStoreConfig `json:"packetStore,omitempty"` GeoFilter *GeoFilterConfig `json:"geo_filter,omitempty"` @@ -129,6 +131,20 @@ type RetentionConfig struct { MetricsDays int `json:"metricsDays"` } +// DBConfig controls SQLite vacuum and maintenance behavior (#919). +type DBConfig struct { + VacuumOnStartup bool `json:"vacuumOnStartup"` // one-time full VACUUM on startup if auto_vacuum is not INCREMENTAL + IncrementalVacuumPages int `json:"incrementalVacuumPages"` // pages returned to OS per reaper cycle (default 1024) +} + +// IncrementalVacuumPages returns the configured pages per vacuum or 1024 default. +func (c *Config) IncrementalVacuumPages() int { + if c.DB != nil && c.DB.IncrementalVacuumPages > 0 { + return c.DB.IncrementalVacuumPages + } + return 1024 +} + // MetricsRetentionDays returns configured metrics retention or 30 days default. func (c *Config) MetricsRetentionDays() int { if c.Retention != nil && c.Retention.MetricsDays > 0 { diff --git a/cmd/server/db_vacuum_test.go b/cmd/server/db_vacuum_test.go new file mode 100644 index 00000000..6dad269f --- /dev/null +++ b/cmd/server/db_vacuum_test.go @@ -0,0 +1,262 @@ +package main + +import ( + "database/sql" + "os" + "path/filepath" + "strings" + "testing" + "time" + + _ "modernc.org/sqlite" +) + +// createFreshIngestorDB creates a SQLite DB using the ingestor's applySchema logic +// (simulated here) with auto_vacuum=INCREMENTAL set before tables. +func createFreshDBWithAutoVacuum(t *testing.T, path string) *sql.DB { + t.Helper() + // auto_vacuum must be set via DSN before journal_mode creates the DB file + db, err := sql.Open("sqlite", path+"?_pragma=auto_vacuum(INCREMENTAL)&_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)") + if err != nil { + t.Fatal(err) + } + db.SetMaxOpenConns(1) + + // Create minimal schema + _, err = db.Exec(` + CREATE TABLE transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + raw_hex TEXT NOT NULL, + hash TEXT NOT NULL UNIQUE, + first_seen TEXT NOT NULL, + route_type INTEGER, + payload_type INTEGER, + payload_version INTEGER, + decoded_json TEXT, + created_at TEXT DEFAULT (datetime('now')), + channel_hash TEXT + ); + CREATE TABLE observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + transmission_id INTEGER NOT NULL REFERENCES transmissions(id), + observer_idx INTEGER, + direction TEXT, + snr REAL, + rssi REAL, + score INTEGER, + path_json TEXT, + timestamp INTEGER NOT NULL + ); + `) + if err != nil { + t.Fatal(err) + } + return db +} + +func TestNewDBHasIncrementalAutoVacuum(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "test.db") + + db := createFreshDBWithAutoVacuum(t, path) + defer db.Close() + + var autoVacuum int + if err := db.QueryRow("PRAGMA auto_vacuum").Scan(&autoVacuum); err != nil { + t.Fatal(err) + } + if autoVacuum != 2 { + t.Fatalf("expected auto_vacuum=2 (INCREMENTAL), got %d", autoVacuum) + } +} + +func TestExistingDBHasAutoVacuumNone(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "test.db") + + // Create DB WITHOUT setting auto_vacuum (simulates old DB) + db, err := sql.Open("sqlite", path+"?_pragma=journal_mode(WAL)") + if err != nil { + t.Fatal(err) + } + db.SetMaxOpenConns(1) + _, err = db.Exec("CREATE TABLE dummy (id INTEGER PRIMARY KEY)") + if err != nil { + t.Fatal(err) + } + + var autoVacuum int + if err := db.QueryRow("PRAGMA auto_vacuum").Scan(&autoVacuum); err != nil { + t.Fatal(err) + } + db.Close() + + if autoVacuum != 0 { + t.Fatalf("expected auto_vacuum=0 (NONE) for old DB, got %d", autoVacuum) + } +} + +func TestVacuumOnStartupMigratesDB(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "test.db") + + // Create DB without auto_vacuum (old DB) + db, err := sql.Open("sqlite", path+"?_pragma=journal_mode(WAL)") + if err != nil { + t.Fatal(err) + } + db.SetMaxOpenConns(1) + _, err = db.Exec("CREATE TABLE dummy (id INTEGER PRIMARY KEY)") + if err != nil { + t.Fatal(err) + } + + var before int + db.QueryRow("PRAGMA auto_vacuum").Scan(&before) + if before != 0 { + t.Fatalf("precondition: expected auto_vacuum=0, got %d", before) + } + db.Close() + + // Simulate vacuumOnStartup migration using openRW + rw, err := openRW(path) + if err != nil { + t.Fatal(err) + } + if _, err := rw.Exec("PRAGMA auto_vacuum = INCREMENTAL"); err != nil { + t.Fatal(err) + } + if _, err := rw.Exec("VACUUM"); err != nil { + t.Fatal(err) + } + rw.Close() + + // Verify migration + db2, err := sql.Open("sqlite", path+"?mode=ro") + if err != nil { + t.Fatal(err) + } + defer db2.Close() + + var after int + if err := db2.QueryRow("PRAGMA auto_vacuum").Scan(&after); err != nil { + t.Fatal(err) + } + if after != 2 { + t.Fatalf("expected auto_vacuum=2 after VACUUM migration, got %d", after) + } +} + +func TestIncrementalVacuumReducesFreelist(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "test.db") + + db := createFreshDBWithAutoVacuum(t, path) + + // Insert a bunch of data + now := time.Now().UTC().Format(time.RFC3339) + for i := 0; i < 500; i++ { + _, err := db.Exec( + "INSERT INTO transmissions (raw_hex, hash, first_seen) VALUES (?, ?, ?)", + strings.Repeat("AA", 200), // ~400 bytes each + "hash_"+string(rune('A'+i%26))+string(rune('0'+i/26)), + now, + ) + if err != nil { + t.Fatal(err) + } + } + + // Get file size before delete + db.Close() + infoBefore, _ := os.Stat(path) + sizeBefore := infoBefore.Size() + + // Reopen and delete all + db, err := sql.Open("sqlite", path+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)") + if err != nil { + t.Fatal(err) + } + db.SetMaxOpenConns(1) + defer db.Close() + + _, err = db.Exec("DELETE FROM transmissions") + if err != nil { + t.Fatal(err) + } + + // Check freelist before vacuum + var freelistBefore int64 + db.QueryRow("PRAGMA freelist_count").Scan(&freelistBefore) + if freelistBefore == 0 { + t.Fatal("expected non-zero freelist after DELETE") + } + + // Run incremental vacuum + _, err = db.Exec("PRAGMA incremental_vacuum(10000)") + if err != nil { + t.Fatal(err) + } + + // Check freelist after vacuum + var freelistAfter int64 + db.QueryRow("PRAGMA freelist_count").Scan(&freelistAfter) + if freelistAfter >= freelistBefore { + t.Fatalf("expected freelist to shrink: before=%d after=%d", freelistBefore, freelistAfter) + } + + // Checkpoint WAL and check file size shrunk + db.Exec("PRAGMA wal_checkpoint(TRUNCATE)") + db.Close() + infoAfter, _ := os.Stat(path) + sizeAfter := infoAfter.Size() + if sizeAfter >= sizeBefore { + t.Logf("warning: file did not shrink (before=%d after=%d) — may depend on page reuse", sizeBefore, sizeAfter) + } +} + +func TestCheckAutoVacuumLogs(t *testing.T) { + // This test verifies checkAutoVacuum doesn't panic on various configs + dir := t.TempDir() + path := filepath.Join(dir, "test.db") + + // Create a fresh DB with auto_vacuum=INCREMENTAL + dbConn := createFreshDBWithAutoVacuum(t, path) + db := &DB{conn: dbConn, path: path} + cfg := &Config{} + + // Should not panic + checkAutoVacuum(db, cfg, path) + dbConn.Close() + + // Create a DB without auto_vacuum + path2 := filepath.Join(dir, "test2.db") + dbConn2, _ := sql.Open("sqlite", path2+"?_pragma=journal_mode(WAL)") + dbConn2.SetMaxOpenConns(1) + dbConn2.Exec("CREATE TABLE dummy (id INTEGER PRIMARY KEY)") + db2 := &DB{conn: dbConn2, path: path2} + + // Should log warning but not panic + checkAutoVacuum(db2, cfg, path2) + dbConn2.Close() +} + +func TestConfigIncrementalVacuumPages(t *testing.T) { + // Default + cfg := &Config{} + if cfg.IncrementalVacuumPages() != 1024 { + t.Fatalf("expected default 1024, got %d", cfg.IncrementalVacuumPages()) + } + + // Custom + cfg.DB = &DBConfig{IncrementalVacuumPages: 512} + if cfg.IncrementalVacuumPages() != 512 { + t.Fatalf("expected 512, got %d", cfg.IncrementalVacuumPages()) + } + + // Zero should return default + cfg.DB.IncrementalVacuumPages = 0 + if cfg.IncrementalVacuumPages() != 1024 { + t.Fatalf("expected default 1024 for zero, got %d", cfg.IncrementalVacuumPages()) + } +} diff --git a/cmd/server/main.go b/cmd/server/main.go index 22dc600e..31fbcd4f 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -148,6 +148,9 @@ func main() { stats.TotalTransmissions, stats.TotalObservations, stats.TotalNodes, stats.TotalObservers) } + // Check auto_vacuum mode and optionally migrate (#919) + checkAutoVacuum(database, cfg, resolvedDB) + // In-memory packet store store := NewPacketStore(database, cfg.PacketStore, cfg.CacheTTL) if err := store.Load(); err != nil { @@ -266,6 +269,7 @@ func main() { defer stopEviction() // Auto-prune old packets if retention.packetDays is configured + vacuumPages := cfg.IncrementalVacuumPages() var stopPrune func() if cfg.Retention != nil && cfg.Retention.PacketDays > 0 { days := cfg.Retention.PacketDays @@ -286,6 +290,9 @@ func main() { log.Printf("[prune] error: %v", err) } else { log.Printf("[prune] deleted %d transmissions older than %d days", n, days) + if n > 0 { + runIncrementalVacuum(resolvedDB, vacuumPages) + } } for { select { @@ -294,6 +301,9 @@ func main() { log.Printf("[prune] error: %v", err) } else { log.Printf("[prune] deleted %d transmissions older than %d days", n, days) + if n > 0 { + runIncrementalVacuum(resolvedDB, vacuumPages) + } } case <-pruneDone: return @@ -321,10 +331,12 @@ func main() { }() time.Sleep(2 * time.Minute) // stagger after packet prune database.PruneOldMetrics(metricsDays) + runIncrementalVacuum(resolvedDB, vacuumPages) for { select { case <-metricsPruneTicker.C: database.PruneOldMetrics(metricsDays) + runIncrementalVacuum(resolvedDB, vacuumPages) case <-metricsPruneDone: return } @@ -354,10 +366,12 @@ func main() { }() time.Sleep(3 * time.Minute) // stagger after metrics prune database.RemoveStaleObservers(observerDays) + runIncrementalVacuum(resolvedDB, vacuumPages) for { select { case <-observerPruneTicker.C: database.RemoveStaleObservers(observerDays) + runIncrementalVacuum(resolvedDB, vacuumPages) case <-observerPruneDone: return } @@ -388,6 +402,7 @@ func main() { g := store.graph store.mu.RUnlock() PruneNeighborEdges(dbPath, g, maxAgeDays) + runIncrementalVacuum(resolvedDB, vacuumPages) for { select { case <-edgePruneTicker.C: @@ -395,6 +410,7 @@ func main() { g := store.graph store.mu.RUnlock() PruneNeighborEdges(dbPath, g, maxAgeDays) + runIncrementalVacuum(resolvedDB, vacuumPages) case <-edgePruneDone: return } diff --git a/cmd/server/vacuum.go b/cmd/server/vacuum.go new file mode 100644 index 00000000..a53556a5 --- /dev/null +++ b/cmd/server/vacuum.go @@ -0,0 +1,84 @@ +package main + +import ( + "fmt" + "log" + "time" +) + +// checkAutoVacuum inspects the current auto_vacuum mode and logs a warning +// if it's not INCREMENTAL. Optionally performs a one-time full VACUUM if +// the operator has set db.vacuumOnStartup: true in config (#919). +func checkAutoVacuum(db *DB, cfg *Config, dbPath string) { + var autoVacuum int + if err := db.conn.QueryRow("PRAGMA auto_vacuum").Scan(&autoVacuum); err != nil { + log.Printf("[db] warning: could not read auto_vacuum: %v", err) + return + } + + if autoVacuum == 2 { + log.Printf("[db] auto_vacuum=INCREMENTAL") + return + } + + modes := map[int]string{0: "NONE", 1: "FULL", 2: "INCREMENTAL"} + mode := modes[autoVacuum] + if mode == "" { + mode = fmt.Sprintf("UNKNOWN(%d)", autoVacuum) + } + + log.Printf("[db] auto_vacuum=%s — DB needs one-time VACUUM to enable incremental auto-vacuum. "+ + "Set db.vacuumOnStartup: true in config to migrate (will block startup for several minutes on large DBs). "+ + "See https://github.com/Kpa-clawbot/CoreScope/issues/919", mode) + + if cfg.DB != nil && cfg.DB.VacuumOnStartup { + // WARNING: Full VACUUM creates a temporary copy of the entire DB file. + // Requires ~2× the DB file size in free disk space or it will fail. + log.Printf("[db] vacuumOnStartup=true — starting one-time full VACUUM (ensure 2x DB size free disk space)...") + start := time.Now() + + rw, err := openRW(dbPath) + if err != nil { + log.Printf("[db] VACUUM failed: could not open RW connection: %v", err) + return + } + defer rw.Close() + + if _, err := rw.Exec("PRAGMA auto_vacuum = INCREMENTAL"); err != nil { + log.Printf("[db] VACUUM failed: could not set auto_vacuum: %v", err) + return + } + if _, err := rw.Exec("VACUUM"); err != nil { + log.Printf("[db] VACUUM failed: %v", err) + return + } + + elapsed := time.Since(start) + log.Printf("[db] VACUUM complete in %v — auto_vacuum is now INCREMENTAL", elapsed.Round(time.Millisecond)) + + // Re-check + var newMode int + if err := db.conn.QueryRow("PRAGMA auto_vacuum").Scan(&newMode); err == nil { + if newMode == 2 { + log.Printf("[db] auto_vacuum=INCREMENTAL (confirmed after VACUUM)") + } else { + log.Printf("[db] warning: auto_vacuum=%d after VACUUM — expected 2", newMode) + } + } + } +} + +// runIncrementalVacuum runs PRAGMA incremental_vacuum(N) on a read-write +// connection. Safe to call on auto_vacuum=NONE databases (noop). +func runIncrementalVacuum(dbPath string, pages int) { + rw, err := openRW(dbPath) + if err != nil { + log.Printf("[vacuum] could not open RW connection: %v", err) + return + } + defer rw.Close() + + if _, err := rw.Exec(fmt.Sprintf("PRAGMA incremental_vacuum(%d)", pages)); err != nil { + log.Printf("[vacuum] incremental_vacuum error: %v", err) + } +} diff --git a/config.example.json b/config.example.json index 5672ed31..9345de1a 100644 --- a/config.example.json +++ b/config.example.json @@ -9,6 +9,11 @@ "packetDays": 30, "_comment": "nodeDays: nodes not seen in N days moved to inactive_nodes (default 7). observerDays: observers not sending data in N days are removed (-1 = keep forever, default 14). packetDays: transmissions older than N days are deleted (0 = disabled)." }, + "db": { + "vacuumOnStartup": false, + "incrementalVacuumPages": 1024, + "_comment": "vacuumOnStartup: run one-time full VACUUM to enable incremental auto-vacuum on existing DBs (blocks startup for minutes on large DBs; requires 2x DB file size in free disk space). incrementalVacuumPages: free pages returned to OS after each retention reaper cycle (default 1024). See #919." + }, "https": { "cert": "/path/to/cert.pem", "key": "/path/to/key.pem", diff --git a/docs/user-guide/configuration.md b/docs/user-guide/configuration.md index eda7910d..05aaebd2 100644 --- a/docs/user-guide/configuration.md +++ b/docs/user-guide/configuration.md @@ -98,6 +98,22 @@ How long (in hours) before a node is marked degraded or silent: | `retention.nodeDays` | `7` | Nodes not seen in N days move to inactive | | `retention.packetDays` | `30` | Packets older than N days are deleted daily | +> **Note:** Lowering retention does **not** immediately shrink the database file. +> SQLite marks deleted pages as free but does not return them to the filesystem +> unless [incremental auto-vacuum](database.md) is enabled. New databases created +> after v0.x.x have auto-vacuum enabled automatically. Existing databases require +> a one-time migration — see the [Database](database.md) guide. + +## Database + +| Field | Default | Description | +|-------|---------|-------------| +| `db.vacuumOnStartup` | `false` | Run a one-time full `VACUUM` on startup to enable incremental auto-vacuum (blocks for minutes on large DBs) | +| `db.incrementalVacuumPages` | `1024` | Free pages returned to the OS after each retention reaper cycle | + +See [Database](database.md) for details on SQLite auto-vacuum, WAL, and manual maintenance. +See [#919](https://github.com/Kpa-clawbot/CoreScope/issues/919) for background. + ## Channel decryption | Field | Description | diff --git a/docs/user-guide/database.md b/docs/user-guide/database.md new file mode 100644 index 00000000..feaf6c1e --- /dev/null +++ b/docs/user-guide/database.md @@ -0,0 +1,82 @@ +# Database + +CoreScope uses SQLite in WAL (Write-Ahead Log) mode for both the server +(read-only) and ingestor (read-write). + +## WAL mode + +WAL mode allows concurrent reads while writes happen. It is set automatically +at connection time via `PRAGMA journal_mode=WAL`. No operator action needed. + +The WAL file (`meshcore.db-wal`) grows during writes and is checkpointed +(merged back into the main DB) periodically and at clean shutdown. + +## Auto-vacuum + +By default, SQLite does not shrink the database file after `DELETE` operations. +Deleted pages are marked free and reused by future writes, but the file size +on disk stays the same. This is surprising when lowering retention settings. + +### New databases + +Databases created after this feature was added automatically have +`PRAGMA auto_vacuum = INCREMENTAL`. After each retention reaper cycle, +CoreScope runs `PRAGMA incremental_vacuum(N)` to return free pages to the OS. + +### Existing databases + +The `auto_vacuum` mode is stored in the database header and can only be changed +by rewriting the entire file with `VACUUM`. CoreScope will **not** do this +automatically — on large databases (5+ GB seen in the wild) it takes minutes +and holds an exclusive lock. + +**To migrate an existing database:** + +1. At startup, CoreScope logs a warning: + ``` + [db] auto_vacuum=NONE — DB needs one-time VACUUM to enable incremental auto-vacuum. + ``` +2. **Ensure at least 2× the database file size in free disk space.** Full VACUUM + creates a temporary copy of the entire file — on a near-full disk it will fail. +3. Set `db.vacuumOnStartup: true` in your `config.json`: + ```json + { + "db": { + "vacuumOnStartup": true + } + } + ``` +4. Restart CoreScope. The one-time `VACUUM` will run and block startup. +5. After migration, remove or set `vacuumOnStartup: false` — it's not needed again. + +### Configuration + +| Field | Default | Description | +|-------|---------|-------------| +| `db.vacuumOnStartup` | `false` | One-time full VACUUM to enable incremental auto-vacuum | +| `db.incrementalVacuumPages` | `1024` | Pages returned to OS per reaper cycle | + +## Manual VACUUM + +You can also run a manual vacuum from the SQLite CLI: + +```bash +sqlite3 data/meshcore.db "PRAGMA auto_vacuum = INCREMENTAL; VACUUM;" +``` + +This is equivalent to `vacuumOnStartup: true` but can be done offline. + +> ⚠️ Full VACUUM requires **2× the database file size** in free disk space (it +> creates a temporary copy). Check with `ls -lh data/meshcore.db` before running. + +## Checking current mode + +```bash +sqlite3 data/meshcore.db "PRAGMA auto_vacuum;" +``` + +- `0` = NONE (default for old databases) +- `1` = FULL (automatic, but slower writes) +- `2` = INCREMENTAL (recommended — CoreScope triggers vacuum after deletes) + +See [#919](https://github.com/Kpa-clawbot/CoreScope/issues/919) for background on this feature.