mirror of
https://github.com/Kpa-clawbot/meshcore-analyzer.git
synced 2026-06-06 19:31:39 +00:00
11d2026bb1
Closes #1183 ## Summary - Adds `packetStore.hotStartupHours` config key (float64, default 0 = disabled). When set, `Load()` loads only that many hours of data synchronously, reducing startup time on large DBs. Background goroutine fills the remaining `retentionHours` window in daily chunks after startup completes. - A background goroutine (`loadBackgroundChunks`) fills the remaining `retentionHours` window in daily chunks after startup completes. Analytics indexes are rebuilt once at the end. - `QueryPackets` and `QueryGroupedPackets` check `oldestLoaded` and fall back to `db.QueryPackets()` for any query whose `Since`/`Until` predates the in-memory window — covering days 8–30 permanently (beyond `retentionHours`) and the background-fill gap during startup. - `/api/perf` gains `hotStartupHours`, `backgroundLoadComplete`, and `backgroundLoadProgress` fields inside `packetStore` so operators can monitor the fill. ### Drive-by fixes - E2E: added `gotoPackets` navigation helper used across packet-related tests - E2E: rewrote stripe assertion to check per-row stripe parity rather than a fragile computed-style comparison - E2E: theme test updated to use `#/home` as the initial route (was `#/`) - `db.go`: removed the RFC3339→unix-timestamp subquery path in `buildTransmissionWhere`; `t.first_seen` is now always compared directly as a string for both RFC3339 and non-RFC3339 inputs ## Configuration ```json "packetStore": { "retentionHours": 168, "hotStartupHours": 24 } ``` `hotStartupHours: 0` (default) preserves existing behavior exactly. Recommended for large DBs to reduce startup time; set to 0 to disable (loads full retentionHours at startup, legacy behavior). ## Test plan - [x] `TestHotStartupConfig_Clamp` — clamping when `hotStartupHours > retentionHours` - [x] `TestHotStartupConfig_ZeroIsDisabled` — zero leaves feature disabled - [x] `TestHotStartup_LoadsOnlyHotWindow` — only hot-window packets in memory after `Load()` - [x] `TestHotStartup_DisabledWhenZero` — all retention packets loaded when disabled - [x] `TestHotStartup_loadChunk_AddsOlderData` — chunk merges correctly, ASC order maintained - [x] `TestHotStartup_BackgroundFillsToRetention` — background goroutine fills to `retentionHours` - [x] `TestHotStartup_ChunkErrorRecovery` — chunk SQL failure logged and skipped, loop terminates - [x] `TestHotStartup_SQLFallback_TriggeredForOldDate` — query before `oldestLoaded` routes to SQL - [x] `TestHotStartup_SQLFallback_NotTriggeredForRecentDate` — recent query stays in-memory - [x] `TestHotStartup_PerfStats` — new fields present in `GetPerfStoreStats()` (backs the perf endpoint) - [x] `TestHotStartup_PerfStoreHTTP` — HTTP-level: GET /api/perf returns `hotStartupHours`, `backgroundLoadComplete`, `backgroundLoadProgress` in `packetStore` 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com> Co-authored-by: openclaw-bot <bot@openclaw.local> Co-authored-by: CoreScope Bot <bot@corescope.local>
76 lines
2.8 KiB
Go
76 lines
2.8 KiB
Go
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
)
|
|
|
|
// ensureServerIndexes creates the indexes that the SQL fallback path in
|
|
// QueryPackets / QueryGroupedPackets and the background hot-startup chunk
|
|
// loader depend on. Mirrors the indexes the ingestor creates (see
|
|
// cmd/ingestor/db.go applySchema). Safe to call on every server start
|
|
// because every CREATE INDEX uses IF NOT EXISTS. Needed because DBs
|
|
// created by an old server-only build (pre-ingestor) won't have the
|
|
// ingestor's indexes, which would cause full table scans on the SQL
|
|
// fallback path during hot startup.
|
|
func ensureServerIndexes(dbPath string) error {
|
|
rw, err := cachedRW(dbPath)
|
|
if err != nil {
|
|
return fmt.Errorf("open rw for index ensure: %w", err)
|
|
}
|
|
stmts := []string{
|
|
`CREATE INDEX IF NOT EXISTS idx_transmissions_first_seen ON transmissions(first_seen)`,
|
|
`CREATE INDEX IF NOT EXISTS idx_transmissions_hash ON transmissions(hash)`,
|
|
`CREATE INDEX IF NOT EXISTS idx_transmissions_payload_type ON transmissions(payload_type)`,
|
|
// PR #1187 r3: commit 63cc1bc3 restored the RFC3339 since/until path
|
|
// to a SELECT … FROM observations WHERE timestamp >= ? subquery in
|
|
// buildTransmissionWhere. Without these indexes the subquery
|
|
// full-scans observations on legacy server-only DBs (the ingestor
|
|
// already creates them; see cmd/ingestor/db.go applySchema).
|
|
`CREATE INDEX IF NOT EXISTS idx_observations_timestamp ON observations(timestamp)`,
|
|
`CREATE INDEX IF NOT EXISTS idx_observations_transmission_id ON observations(transmission_id)`,
|
|
}
|
|
for _, s := range stmts {
|
|
if _, err := rw.Exec(s); err != nil {
|
|
return fmt.Errorf("ensure index %q: %w", s, err)
|
|
}
|
|
}
|
|
|
|
// observer_idx column exists in v3 schema only; observer_id is the
|
|
// v2 equivalent. Probe the schema and create the matching index.
|
|
rows, err := rw.Query(`PRAGMA table_info(observations)`)
|
|
if err != nil {
|
|
return fmt.Errorf("pragma table_info(observations): %w", err)
|
|
}
|
|
var hasObserverIdx, hasObserverID bool
|
|
for rows.Next() {
|
|
var cid int
|
|
var name, ctype string
|
|
var notnull, pk int
|
|
var dflt interface{}
|
|
if err := rows.Scan(&cid, &name, &ctype, ¬null, &dflt, &pk); err != nil {
|
|
rows.Close()
|
|
return fmt.Errorf("scan table_info: %w", err)
|
|
}
|
|
switch strings.ToLower(name) {
|
|
case "observer_idx":
|
|
hasObserverIdx = true
|
|
case "observer_id":
|
|
hasObserverID = true
|
|
}
|
|
}
|
|
rows.Close()
|
|
|
|
if hasObserverIdx {
|
|
if _, err := rw.Exec(`CREATE INDEX IF NOT EXISTS idx_observations_observer_idx ON observations(observer_idx)`); err != nil {
|
|
return fmt.Errorf("ensure idx_observations_observer_idx: %w", err)
|
|
}
|
|
}
|
|
if hasObserverID {
|
|
if _, err := rw.Exec(`CREATE INDEX IF NOT EXISTS idx_observations_observer_id ON observations(observer_id)`); err != nil {
|
|
return fmt.Errorf("ensure idx_observations_observer_id: %w", err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|