Files
meshcore-analyzer/cmd/server/ensure_indexes_test.go
T
efiten 11d2026bb1 feat(startup): hot startup — load hotStartupHours synchronously, fill retentionHours in background (#1187)
Closes #1183

## Summary

- Adds `packetStore.hotStartupHours` config key (float64, default 0 =
disabled). When set, `Load()` loads only that many hours of data
synchronously, reducing startup time on large DBs. Background goroutine
fills the remaining `retentionHours` window in daily chunks after
startup completes.
- A background goroutine (`loadBackgroundChunks`) fills the remaining
`retentionHours` window in daily chunks after startup completes.
Analytics indexes are rebuilt once at the end.
- `QueryPackets` and `QueryGroupedPackets` check `oldestLoaded` and fall
back to `db.QueryPackets()` for any query whose `Since`/`Until` predates
the in-memory window — covering days 8–30 permanently (beyond
`retentionHours`) and the background-fill gap during startup.
- `/api/perf` gains `hotStartupHours`, `backgroundLoadComplete`, and
`backgroundLoadProgress` fields inside `packetStore` so operators can
monitor the fill.

### Drive-by fixes

- E2E: added `gotoPackets` navigation helper used across packet-related
tests
- E2E: rewrote stripe assertion to check per-row stripe parity rather
than a fragile computed-style comparison
- E2E: theme test updated to use `#/home` as the initial route (was
`#/`)
- `db.go`: removed the RFC3339→unix-timestamp subquery path in
`buildTransmissionWhere`; `t.first_seen` is now always compared directly
as a string for both RFC3339 and non-RFC3339 inputs

## Configuration

```json
"packetStore": {
  "retentionHours": 168,
  "hotStartupHours": 24
}
```

`hotStartupHours: 0` (default) preserves existing behavior exactly.
Recommended for large DBs to reduce startup time; set to 0 to disable
(loads full retentionHours at startup, legacy behavior).

## Test plan

- [x] `TestHotStartupConfig_Clamp` — clamping when `hotStartupHours >
retentionHours`
- [x] `TestHotStartupConfig_ZeroIsDisabled` — zero leaves feature
disabled
- [x] `TestHotStartup_LoadsOnlyHotWindow` — only hot-window packets in
memory after `Load()`
- [x] `TestHotStartup_DisabledWhenZero` — all retention packets loaded
when disabled
- [x] `TestHotStartup_loadChunk_AddsOlderData` — chunk merges correctly,
ASC order maintained
- [x] `TestHotStartup_BackgroundFillsToRetention` — background goroutine
fills to `retentionHours`
- [x] `TestHotStartup_ChunkErrorRecovery` — chunk SQL failure logged and
skipped, loop terminates
- [x] `TestHotStartup_SQLFallback_TriggeredForOldDate` — query before
`oldestLoaded` routes to SQL
- [x] `TestHotStartup_SQLFallback_NotTriggeredForRecentDate` — recent
query stays in-memory
- [x] `TestHotStartup_PerfStats` — new fields present in
`GetPerfStoreStats()` (backs the perf endpoint)
- [x] `TestHotStartup_PerfStoreHTTP` — HTTP-level: GET /api/perf returns
`hotStartupHours`, `backgroundLoadComplete`, `backgroundLoadProgress` in
`packetStore`

🤖 Generated with [Claude Code](https://claude.com/claude-code)

---------

Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
Co-authored-by: openclaw-bot <bot@openclaw.local>
Co-authored-by: CoreScope Bot <bot@corescope.local>
2026-05-15 22:46:25 -07:00

76 lines
2.8 KiB
Go

package main
import (
"database/sql"
"path/filepath"
"testing"
_ "modernc.org/sqlite"
)
// TestEnsureServerIndexes_CreatesObservationsIndexes guards against
// regression of PR #1187 r3 MUST-FIX 2: legacy server-only DBs that lack
// the ingestor-created observation indexes used to full-scan the
// `SELECT ... FROM observations WHERE timestamp >= ?` subquery added to
// buildTransmissionWhere by 63cc1bc3. ensureServerIndexes must create
// idx_observations_timestamp (and the join companions) so the hot-startup
// chunk loader and RFC3339 since/until path don't full-scan observations.
func TestEnsureServerIndexes_CreatesObservationsIndexes(t *testing.T) {
dir := t.TempDir()
dbPath := filepath.Join(dir, "schema_only.db")
conn, err := sql.Open("sqlite", dbPath)
if err != nil {
t.Fatalf("open: %v", err)
}
// Minimal legacy server-only schema: tables present, no extra indexes.
stmts := []string{
`CREATE TABLE transmissions (id INTEGER PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT)`,
// v3 schema (observer_idx) — matches the ingestor-created shape
// and the path that 63cc1bc3 / hot-startup loadChunk traverse.
`CREATE TABLE observations (id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_idx INTEGER, direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT)`,
`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT)`,
`CREATE TABLE nodes (pubkey TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, frequency REAL)`,
`CREATE TABLE schema_version (version INTEGER)`,
`INSERT INTO schema_version (version) VALUES (1)`,
}
for _, s := range stmts {
if _, err := conn.Exec(s); err != nil {
t.Fatalf("setup %q: %v", s, err)
}
}
conn.Close()
if err := ensureServerIndexes(dbPath); err != nil {
t.Fatalf("ensureServerIndexes: %v", err)
}
// Reopen and query sqlite_master for the indexes we expect.
conn2, err := sql.Open("sqlite", dbPath)
if err != nil {
t.Fatalf("reopen: %v", err)
}
defer conn2.Close()
required := []string{
"idx_transmissions_first_seen",
"idx_transmissions_hash",
"idx_transmissions_payload_type",
"idx_observations_timestamp",
"idx_observations_transmission_id",
"idx_observations_observer_idx",
}
for _, name := range required {
var found string
err := conn2.QueryRow(`SELECT name FROM sqlite_master WHERE type='index' AND name=?`, name).Scan(&found)
if err != nil {
t.Errorf("index %s not created (err=%v) — ensureServerIndexes must create it to avoid full scans on the SQL fallback path", name, err)
continue
}
if found != name {
t.Errorf("index lookup mismatch: want %s got %s", name, found)
}
}
}