mirror of
https://github.com/Kpa-clawbot/meshcore-analyzer.git
synced 2026-06-05 20:01:23 +00:00
efiten
11d2026bb1
Closes #1183 ## Summary - Adds `packetStore.hotStartupHours` config key (float64, default 0 = disabled). When set, `Load()` loads only that many hours of data synchronously, reducing startup time on large DBs. Background goroutine fills the remaining `retentionHours` window in daily chunks after startup completes. - A background goroutine (`loadBackgroundChunks`) fills the remaining `retentionHours` window in daily chunks after startup completes. Analytics indexes are rebuilt once at the end. - `QueryPackets` and `QueryGroupedPackets` check `oldestLoaded` and fall back to `db.QueryPackets()` for any query whose `Since`/`Until` predates the in-memory window — covering days 8–30 permanently (beyond `retentionHours`) and the background-fill gap during startup. - `/api/perf` gains `hotStartupHours`, `backgroundLoadComplete`, and `backgroundLoadProgress` fields inside `packetStore` so operators can monitor the fill. ### Drive-by fixes - E2E: added `gotoPackets` navigation helper used across packet-related tests - E2E: rewrote stripe assertion to check per-row stripe parity rather than a fragile computed-style comparison - E2E: theme test updated to use `#/home` as the initial route (was `#/`) - `db.go`: removed the RFC3339→unix-timestamp subquery path in `buildTransmissionWhere`; `t.first_seen` is now always compared directly as a string for both RFC3339 and non-RFC3339 inputs ## Configuration ```json "packetStore": { "retentionHours": 168, "hotStartupHours": 24 } ``` `hotStartupHours: 0` (default) preserves existing behavior exactly. Recommended for large DBs to reduce startup time; set to 0 to disable (loads full retentionHours at startup, legacy behavior). ## Test plan - [x] `TestHotStartupConfig_Clamp` — clamping when `hotStartupHours > retentionHours` - [x] `TestHotStartupConfig_ZeroIsDisabled` — zero leaves feature disabled - [x] `TestHotStartup_LoadsOnlyHotWindow` — only hot-window packets in memory after `Load()` - [x] `TestHotStartup_DisabledWhenZero` — all retention packets loaded when disabled - [x] `TestHotStartup_loadChunk_AddsOlderData` — chunk merges correctly, ASC order maintained - [x] `TestHotStartup_BackgroundFillsToRetention` — background goroutine fills to `retentionHours` - [x] `TestHotStartup_ChunkErrorRecovery` — chunk SQL failure logged and skipped, loop terminates - [x] `TestHotStartup_SQLFallback_TriggeredForOldDate` — query before `oldestLoaded` routes to SQL - [x] `TestHotStartup_SQLFallback_NotTriggeredForRecentDate` — recent query stays in-memory - [x] `TestHotStartup_PerfStats` — new fields present in `GetPerfStoreStats()` (backs the perf endpoint) - [x] `TestHotStartup_PerfStoreHTTP` — HTTP-level: GET /api/perf returns `hotStartupHours`, `backgroundLoadComplete`, `backgroundLoadProgress` in `packetStore` 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com> Co-authored-by: openclaw-bot <bot@openclaw.local> Co-authored-by: CoreScope Bot <bot@corescope.local>
51 lines
2.7 KiB
Bash
Executable File
51 lines
2.7 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# freshen-fixture.sh — Shift all timestamps in the fixture DB to be relative to now.
|
|
# Preserves the relative ordering between timestamps.
|
|
# Usage: bash tools/freshen-fixture.sh <path-to-fixture.db>
|
|
set -euo pipefail
|
|
|
|
DB="${1:?Usage: freshen-fixture.sh <path-to-fixture.db>}"
|
|
|
|
if [ ! -f "$DB" ]; then
|
|
echo "ERROR: DB file not found: $DB" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# Find the max timestamp across all time columns, compute offset, shift everything forward.
|
|
sqlite3 "$DB" <<'SQL'
|
|
-- Shift all timestamps forward so the newest is ~now, preserving relative ordering.
|
|
-- Use strftime to produce RFC3339 format (T separator + Z suffix) for correct comparison.
|
|
UPDATE nodes SET last_seen = strftime('%Y-%m-%dT%H:%M:%SZ', last_seen,
|
|
(SELECT printf('+%d seconds', CAST((julianday('now') - julianday(MAX(last_seen))) * 86400 AS INTEGER)) FROM nodes)
|
|
) WHERE last_seen IS NOT NULL;
|
|
|
|
UPDATE transmissions SET first_seen = strftime('%Y-%m-%dT%H:%M:%SZ', first_seen,
|
|
(SELECT printf('+%d seconds', CAST((julianday('now') - julianday(MAX(first_seen))) * 86400 AS INTEGER)) FROM transmissions)
|
|
) WHERE first_seen IS NOT NULL;
|
|
|
|
-- Sync observations.timestamp (Unix seconds) to match their transmission's freshened first_seen.
|
|
-- Observations with timestamp=0 break the SQL since-filter in buildTransmissionWhere.
|
|
UPDATE observations SET timestamp = CAST(strftime('%s',
|
|
(SELECT first_seen FROM transmissions WHERE id = transmission_id)
|
|
) AS INTEGER)
|
|
WHERE timestamp = 0 OR timestamp IS NULL;
|
|
|
|
-- Observers: shift last_seen too so they don't get auto-pruned by RemoveStaleObservers
|
|
-- on server startup (default 14d threshold marks all >14d observers inactive=1, which
|
|
-- the /api/observers filter then excludes — leaving the map page with no observer markers).
|
|
UPDATE observers SET last_seen = strftime('%Y-%m-%dT%H:%M:%SZ', last_seen,
|
|
(SELECT printf('+%d seconds', CAST((julianday('now') - julianday(MAX(last_seen))) * 86400 AS INTEGER)) FROM observers)
|
|
) WHERE last_seen IS NOT NULL;
|
|
SQL
|
|
|
|
# Defensive: clear any stale inactive=1 flags. Column may not exist on fresh fixtures
|
|
# (added by server migration on first startup); silently no-op if missing.
|
|
sqlite3 "$DB" "UPDATE observers SET inactive = 0 WHERE inactive = 1;" 2>/dev/null || true
|
|
|
|
# neighbor_edges may not exist in all fixture versions
|
|
sqlite3 "$DB" "UPDATE neighbor_edges SET last_seen = strftime('%Y-%m-%dT%H:%M:%SZ', last_seen, (SELECT printf('+%d seconds', CAST((julianday('now') - julianday(MAX(last_seen))) * 86400 AS INTEGER)) FROM neighbor_edges)) WHERE last_seen IS NOT NULL;" 2>/dev/null || true
|
|
|
|
echo "Fixture timestamps freshened in $DB"
|
|
sqlite3 "$DB" "SELECT 'nodes: min=' || MIN(last_seen) || ' max=' || MAX(last_seen) FROM nodes;"
|
|
sqlite3 "$DB" "SELECT 'observers: count=' || COUNT(*) || ' max=' || MAX(last_seen) FROM observers;"
|