mirror of
https://github.com/Kpa-clawbot/meshcore-analyzer.git
synced 2026-05-25 16:44:02 +00:00
11d2026bb1
Closes #1183 ## Summary - Adds `packetStore.hotStartupHours` config key (float64, default 0 = disabled). When set, `Load()` loads only that many hours of data synchronously, reducing startup time on large DBs. Background goroutine fills the remaining `retentionHours` window in daily chunks after startup completes. - A background goroutine (`loadBackgroundChunks`) fills the remaining `retentionHours` window in daily chunks after startup completes. Analytics indexes are rebuilt once at the end. - `QueryPackets` and `QueryGroupedPackets` check `oldestLoaded` and fall back to `db.QueryPackets()` for any query whose `Since`/`Until` predates the in-memory window — covering days 8–30 permanently (beyond `retentionHours`) and the background-fill gap during startup. - `/api/perf` gains `hotStartupHours`, `backgroundLoadComplete`, and `backgroundLoadProgress` fields inside `packetStore` so operators can monitor the fill. ### Drive-by fixes - E2E: added `gotoPackets` navigation helper used across packet-related tests - E2E: rewrote stripe assertion to check per-row stripe parity rather than a fragile computed-style comparison - E2E: theme test updated to use `#/home` as the initial route (was `#/`) - `db.go`: removed the RFC3339→unix-timestamp subquery path in `buildTransmissionWhere`; `t.first_seen` is now always compared directly as a string for both RFC3339 and non-RFC3339 inputs ## Configuration ```json "packetStore": { "retentionHours": 168, "hotStartupHours": 24 } ``` `hotStartupHours: 0` (default) preserves existing behavior exactly. Recommended for large DBs to reduce startup time; set to 0 to disable (loads full retentionHours at startup, legacy behavior). ## Test plan - [x] `TestHotStartupConfig_Clamp` — clamping when `hotStartupHours > retentionHours` - [x] `TestHotStartupConfig_ZeroIsDisabled` — zero leaves feature disabled - [x] `TestHotStartup_LoadsOnlyHotWindow` — only hot-window packets in memory after `Load()` - [x] `TestHotStartup_DisabledWhenZero` — all retention packets loaded when disabled - [x] `TestHotStartup_loadChunk_AddsOlderData` — chunk merges correctly, ASC order maintained - [x] `TestHotStartup_BackgroundFillsToRetention` — background goroutine fills to `retentionHours` - [x] `TestHotStartup_ChunkErrorRecovery` — chunk SQL failure logged and skipped, loop terminates - [x] `TestHotStartup_SQLFallback_TriggeredForOldDate` — query before `oldestLoaded` routes to SQL - [x] `TestHotStartup_SQLFallback_NotTriggeredForRecentDate` — recent query stays in-memory - [x] `TestHotStartup_PerfStats` — new fields present in `GetPerfStoreStats()` (backs the perf endpoint) - [x] `TestHotStartup_PerfStoreHTTP` — HTTP-level: GET /api/perf returns `hotStartupHours`, `backgroundLoadComplete`, `backgroundLoadProgress` in `packetStore` 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com> Co-authored-by: openclaw-bot <bot@openclaw.local> Co-authored-by: CoreScope Bot <bot@corescope.local>
76 lines
2.8 KiB
Go
76 lines
2.8 KiB
Go
package main
|
|
|
|
import (
|
|
"database/sql"
|
|
"path/filepath"
|
|
"testing"
|
|
|
|
_ "modernc.org/sqlite"
|
|
)
|
|
|
|
// TestEnsureServerIndexes_CreatesObservationsIndexes guards against
|
|
// regression of PR #1187 r3 MUST-FIX 2: legacy server-only DBs that lack
|
|
// the ingestor-created observation indexes used to full-scan the
|
|
// `SELECT ... FROM observations WHERE timestamp >= ?` subquery added to
|
|
// buildTransmissionWhere by 63cc1bc3. ensureServerIndexes must create
|
|
// idx_observations_timestamp (and the join companions) so the hot-startup
|
|
// chunk loader and RFC3339 since/until path don't full-scan observations.
|
|
func TestEnsureServerIndexes_CreatesObservationsIndexes(t *testing.T) {
|
|
dir := t.TempDir()
|
|
dbPath := filepath.Join(dir, "schema_only.db")
|
|
|
|
conn, err := sql.Open("sqlite", dbPath)
|
|
if err != nil {
|
|
t.Fatalf("open: %v", err)
|
|
}
|
|
|
|
// Minimal legacy server-only schema: tables present, no extra indexes.
|
|
stmts := []string{
|
|
`CREATE TABLE transmissions (id INTEGER PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT)`,
|
|
// v3 schema (observer_idx) — matches the ingestor-created shape
|
|
// and the path that 63cc1bc3 / hot-startup loadChunk traverse.
|
|
`CREATE TABLE observations (id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_idx INTEGER, direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT)`,
|
|
`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT)`,
|
|
`CREATE TABLE nodes (pubkey TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, frequency REAL)`,
|
|
`CREATE TABLE schema_version (version INTEGER)`,
|
|
`INSERT INTO schema_version (version) VALUES (1)`,
|
|
}
|
|
for _, s := range stmts {
|
|
if _, err := conn.Exec(s); err != nil {
|
|
t.Fatalf("setup %q: %v", s, err)
|
|
}
|
|
}
|
|
conn.Close()
|
|
|
|
if err := ensureServerIndexes(dbPath); err != nil {
|
|
t.Fatalf("ensureServerIndexes: %v", err)
|
|
}
|
|
|
|
// Reopen and query sqlite_master for the indexes we expect.
|
|
conn2, err := sql.Open("sqlite", dbPath)
|
|
if err != nil {
|
|
t.Fatalf("reopen: %v", err)
|
|
}
|
|
defer conn2.Close()
|
|
|
|
required := []string{
|
|
"idx_transmissions_first_seen",
|
|
"idx_transmissions_hash",
|
|
"idx_transmissions_payload_type",
|
|
"idx_observations_timestamp",
|
|
"idx_observations_transmission_id",
|
|
"idx_observations_observer_idx",
|
|
}
|
|
for _, name := range required {
|
|
var found string
|
|
err := conn2.QueryRow(`SELECT name FROM sqlite_master WHERE type='index' AND name=?`, name).Scan(&found)
|
|
if err != nil {
|
|
t.Errorf("index %s not created (err=%v) — ensureServerIndexes must create it to avoid full scans on the SQL fallback path", name, err)
|
|
continue
|
|
}
|
|
if found != name {
|
|
t.Errorf("index lookup mismatch: want %s got %s", name, found)
|
|
}
|
|
}
|
|
}
|