From 8a0862523d3454c0d532ced0248c7576eb79237f Mon Sep 17 00:00:00 2001 From: efiten Date: Wed, 1 Apr 2026 16:06:54 +0200 Subject: [PATCH] fix: add migration for missing observations.timestamp index (#332) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem On installations where the database predates the `idx_observations_timestamp` index, `/api/stats` takes 30s+ because `GetStoreStats()` runs two full table scans: ```sql SELECT COUNT(*) FROM observations WHERE timestamp > ? -- last hour SELECT COUNT(*) FROM observations WHERE timestamp > ? -- last 24h ``` The index is only created in the `if !obsExists` block, so any database where the `observations` table already existed before that code was added never gets it. ## Fix Adds a one-time migration (`obs_timestamp_index_v1`) that runs at ingestor startup: ```sql CREATE INDEX IF NOT EXISTS idx_observations_timestamp ON observations(timestamp) ``` On large installations this index creation may take a few seconds on first startup after the upgrade, but subsequent stats queries become instant. ## Test plan - [ ] Restart ingestor on an older database and confirm `[migration] observations timestamp index created` appears in logs - [ ] Confirm `/api/stats` response time drops from 30s+ to <100ms 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Sonnet 4.6 --- cmd/ingestor/db.go | 11 ++++ cmd/ingestor/db_test.go | 114 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 125 insertions(+) diff --git a/cmd/ingestor/db.go b/cmd/ingestor/db.go index d09747ab..bcc52ff3 100644 --- a/cmd/ingestor/db.go +++ b/cmd/ingestor/db.go @@ -280,6 +280,17 @@ func applySchema(db *sql.DB) error { log.Println("[migration] node telemetry columns added") } + // One-time migration: add timestamp index on observations for fast stats queries. + // Older databases created before this index was added suffer from full table scans + // on COUNT(*) WHERE timestamp > ?, causing /api/stats to take 30s+. + row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'obs_timestamp_index_v1'") + if row.Scan(&migDone) != nil { + log.Println("[migration] Adding timestamp index on observations...") + db.Exec(`CREATE INDEX IF NOT EXISTS idx_observations_timestamp ON observations(timestamp)`) + db.Exec(`INSERT INTO _migrations (name) VALUES ('obs_timestamp_index_v1')`) + log.Println("[migration] observations timestamp index created") + } + return nil } diff --git a/cmd/ingestor/db_test.go b/cmd/ingestor/db_test.go index 3debc706..2ac915d2 100644 --- a/cmd/ingestor/db_test.go +++ b/cmd/ingestor/db_test.go @@ -1457,3 +1457,117 @@ func TestExtractObserverMetaNestedNilSkipsTopLevel(t *testing.T) { t.Error("nested nil should suppress top-level fallback") } } + +func TestObsTimestampIndexMigration(t *testing.T) { + // Case 1: new DB — OpenStore should create idx_observations_timestamp as part + // of the observations table schema. + t.Run("NewDB", func(t *testing.T) { + s, err := OpenStore(tempDBPath(t)) + if err != nil { + t.Fatal(err) + } + defer s.Close() + + var count int + err = s.db.QueryRow( + "SELECT COUNT(*) FROM sqlite_master WHERE type='index' AND name='idx_observations_timestamp'", + ).Scan(&count) + if err != nil { + t.Fatal(err) + } + if count != 1 { + t.Error("idx_observations_timestamp should exist on a new DB") + } + + var migCount int + err = s.db.QueryRow( + "SELECT COUNT(*) FROM _migrations WHERE name='obs_timestamp_index_v1'", + ).Scan(&migCount) + if err != nil { + t.Fatal(err) + } + // On a new DB the index is created inline (not via migration), so the + // migration row may or may not be recorded — just verify the index exists. + _ = migCount + }) + + // Case 2: existing DB that has the observations table but lacks the index + // and lacks the _migrations entry — simulates an older installation. + t.Run("MigrationPath", func(t *testing.T) { + path := tempDBPath(t) + + // Build a bare-bones DB that mimics an old installation: + // observations table exists but idx_observations_timestamp does NOT. + db, err := sql.Open("sqlite", path) + if err != nil { + t.Fatal(err) + } + _, err = db.Exec(` + CREATE TABLE IF NOT EXISTS _migrations (name TEXT PRIMARY KEY); + CREATE TABLE IF NOT EXISTS transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + raw_hex TEXT NOT NULL, + hash TEXT NOT NULL UNIQUE, + first_seen TEXT NOT NULL, + route_type INTEGER, + payload_type INTEGER, + payload_version INTEGER, + decoded_json TEXT, + created_at TEXT DEFAULT (datetime('now')) + ); + CREATE TABLE IF NOT EXISTS observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + transmission_id INTEGER NOT NULL REFERENCES transmissions(id), + observer_idx INTEGER, + direction TEXT, + snr REAL, + rssi REAL, + score INTEGER, + path_json TEXT, + timestamp INTEGER NOT NULL + ); + `) + if err != nil { + db.Close() + t.Fatal(err) + } + // Confirm the index is absent before OpenStore runs. + var preCount int + db.QueryRow( + "SELECT COUNT(*) FROM sqlite_master WHERE type='index' AND name='idx_observations_timestamp'", + ).Scan(&preCount) + db.Close() + if preCount != 0 { + t.Fatalf("pre-condition failed: idx_observations_timestamp should not exist yet, got count=%d", preCount) + } + + // Now open via OpenStore — the migration should add the index. + s, err := OpenStore(path) + if err != nil { + t.Fatal(err) + } + defer s.Close() + + var idxCount int + err = s.db.QueryRow( + "SELECT COUNT(*) FROM sqlite_master WHERE type='index' AND name='idx_observations_timestamp'", + ).Scan(&idxCount) + if err != nil { + t.Fatal(err) + } + if idxCount != 1 { + t.Error("idx_observations_timestamp should exist after migration on old DB") + } + + var migCount int + err = s.db.QueryRow( + "SELECT COUNT(*) FROM _migrations WHERE name='obs_timestamp_index_v1'", + ).Scan(&migCount) + if err != nil { + t.Fatal(err) + } + if migCount != 1 { + t.Errorf("migration obs_timestamp_index_v1 should be recorded, got count=%d", migCount) + } + }) +}