mirror of
https://github.com/Kpa-clawbot/meshcore-analyzer.git
synced 2026-04-26 18:55:10 +00:00
9e90548637
## Summary Remove `ResolvedPath []*string` field from `StoreTx` and `StoreObs` structs, replacing it with a compact membership index + on-demand SQL decode. This eliminates the dominant heap cost identified in profiling (#791, #799). **Spec:** #800 (consolidated from two rounds of expert + implementer review on #799) Closes #800 Closes #791 ## Design ### Removed - `StoreTx.ResolvedPath []*string` - `StoreObs.ResolvedPath []*string` - `TransmissionResp.ResolvedPath`, `ObservationResp.ResolvedPath` struct fields ### Added | Structure | Purpose | Est. cost at 1M obs | |---|---|---:| | `resolvedPubkeyIndex map[uint64][]int` | FNV-1a(pubkey) → []txID forward index | 50–120 MB | | `resolvedPubkeyReverse map[int][]uint64` | txID → []hashes for clean removal | ~40 MB | | `apiResolvedPathLRU` (10K entries) | FIFO cache for on-demand API decode | ~2 MB | ### Decode-window discipline `resolved_path` JSON decoded once per packet. Consumers fed in order, temp slice dropped — never stored on struct: 1. `addToByNode` — relay node indexing 2. `touchRelayLastSeen` — relay liveness DB updates 3. `byPathHop` resolved-key entries 4. `resolvedPubkeyIndex` + reverse insert 5. WebSocket broadcast map (raw JSON bytes) 6. Persist batch (raw JSON bytes for SQL UPDATE) ### Collision safety When the forward index returns candidates, a batched SQL query confirms exact pubkey presence using `LIKE '%"pubkey"%'` on the `resolved_path` column. ### Feature flag `useResolvedPathIndex` (default `true`). Off-path is conservative: all candidates kept, index not consulted. For one-release rollback safety. ## Files changed | File | Changes | |---|---| | `resolved_index.go` | **New** — index structures, LRU cache, on-demand SQL helpers, collision safety | | `store.go` | Remove RP fields, decode-window discipline in Load/Ingest, on-demand txToMap/obsToMap/enrichObs, eviction cleanup via SQL, memory accounting update | | `types.go` | Remove RP fields from TransmissionResp/ObservationResp | | `routes.go` | Replace `nodeInResolvedPath` with `nodeInResolvedPathViaIndex`, remove RP from mapSlice helpers | | `neighbor_persist.go` | Refactor backfill: reverse-map removal → forward+reverse insert → LRU invalidation | ## Tests added (27 new) **Unit:** - `TestStoreTx_ResolvedPathFieldAbsent` — reflection guard - `TestResolvedPubkeyIndex_BuildFromLoad` — forward+reverse consistency - `TestResolvedPubkeyIndex_HashCollision` — SQL collision safety - `TestResolvedPubkeyIndex_IngestUpdate` — maps reflect new ingests - `TestResolvedPubkeyIndex_RemoveOnEvict` — clean removal via reverse map - `TestResolvedPubkeyIndex_PerObsCoverage` — non-best obs pubkeys indexed - `TestAddToByNode_WithoutResolvedPathField` - `TestTouchRelayLastSeen_WithoutResolvedPathField` - `TestWebSocketBroadcast_IncludesResolvedPath` - `TestBackfill_InvalidatesLRU` - `TestEviction_ByNodeCleanup_OnDemandSQL` - `TestExtractResolvedPubkeys`, `TestMergeResolvedPubkeys` - `TestResolvedPubkeyHash_Deterministic` - `TestLRU_EvictionOnFull` **Endpoint:** - `TestPathsThroughNode_NilResolvedPathFallback` - `TestPacketsAPI_OnDemandResolvedPath` - `TestPacketsAPI_OnDemandResolvedPath_LRUHit` - `TestPacketsAPI_OnDemandResolvedPath_Empty` **Feature flag:** - `TestFeatureFlag_OffPath_PreservesOldBehavior` - `TestFeatureFlag_Toggle_NoStateLeak` **Concurrency:** - `TestReverseMap_NoLeakOnPartialFailure` - `TestDecodeWindow_LockHoldTimeBounded` - `TestLivePolling_LRUUnderConcurrentIngest` **Regression:** - `TestRepeaterLiveness_StillAccurate` **Benchmarks:** - `BenchmarkLoad_BeforeAfter` - `BenchmarkResolvedPubkeyIndex_Memory` - `BenchmarkPathsThroughNode_Latency` - `BenchmarkLivePolling_UnderIngest` ## Benchmark results ``` BenchmarkResolvedPubkeyIndex_Memory/pubkeys=50K 429ms 103MB 777K allocs BenchmarkResolvedPubkeyIndex_Memory/pubkeys=500K 4205ms 896MB 7.67M allocs BenchmarkLoad_BeforeAfter 65ms 20MB 202K allocs BenchmarkPathsThroughNode_Latency 3.9µs 0B 0 allocs BenchmarkLivePolling_UnderIngest 5.4µs 545B 7 allocs ``` Key: per-obs `[]*string` overhead completely eliminated. At 1M obs with 3 hops average, this saves ~72 bytes/obs × 1M = ~68 MB just from the slice headers + pointers, plus the JSON-decoded string data (~900 MB at scale per profiling). ## Design choices - **FNV-1a instead of xxhash**: stdlib availability, no external dependency. Performance is equivalent for this use case (pubkey strings are short). - **FIFO LRU instead of true LRU**: simpler implementation, adequate for the access pattern (mostly sequential obs IDs from live polling). - **Grouped packets view omits resolved_path**: cold path, not worth SQL round-trip per page render. - **Backfill pending check uses reverse-map presence** instead of per-obs field: if a tx has any indexed pubkeys, its observations are considered resolved. Closes #807 --------- Co-authored-by: you <you@example.com>
391 lines
12 KiB
Go
391 lines
12 KiB
Go
package main
|
|
|
|
import (
|
|
"encoding/json"
|
|
"log"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
|
|
"github.com/meshcore-analyzer/geofilter"
|
|
)
|
|
|
|
// Config mirrors the Node.js config.json structure (read-only fields).
|
|
type Config struct {
|
|
Port int `json:"port"`
|
|
APIKey string `json:"apiKey"`
|
|
DBPath string `json:"dbPath"`
|
|
|
|
// NodeBlacklist is a list of public keys to exclude from all API responses.
|
|
// Blacklisted nodes are hidden from node lists, search, detail, map, and stats.
|
|
// Use this to filter out trolls, nodes with offensive names, or nodes
|
|
// reporting deliberately false data (e.g. wrong GPS position) that the
|
|
// operator refuses to fix.
|
|
NodeBlacklist []string `json:"nodeBlacklist"`
|
|
|
|
// blacklistSetCached is the lazily-built set version of NodeBlacklist.
|
|
blacklistSetCached map[string]bool
|
|
blacklistOnce sync.Once
|
|
|
|
Branding map[string]interface{} `json:"branding"`
|
|
Theme map[string]interface{} `json:"theme"`
|
|
ThemeDark map[string]interface{} `json:"themeDark"`
|
|
NodeColors map[string]interface{} `json:"nodeColors"`
|
|
TypeColors map[string]interface{} `json:"typeColors"`
|
|
Home map[string]interface{} `json:"home"`
|
|
|
|
MapDefaults struct {
|
|
Center []float64 `json:"center"`
|
|
Zoom int `json:"zoom"`
|
|
} `json:"mapDefaults"`
|
|
|
|
Regions map[string]string `json:"regions"`
|
|
|
|
Roles map[string]interface{} `json:"roles"`
|
|
HealthThresholds *HealthThresholds `json:"healthThresholds"`
|
|
Tiles map[string]interface{} `json:"tiles"`
|
|
SnrThresholds map[string]interface{} `json:"snrThresholds"`
|
|
DistThresholds map[string]interface{} `json:"distThresholds"`
|
|
MaxHopDist *float64 `json:"maxHopDist"`
|
|
Limits map[string]interface{} `json:"limits"`
|
|
PerfSlowMs *int `json:"perfSlowMs"`
|
|
WsReconnectMs *int `json:"wsReconnectMs"`
|
|
CacheInvalidMs *int `json:"cacheInvalidateMs"`
|
|
ExternalUrls map[string]interface{} `json:"externalUrls"`
|
|
|
|
LiveMap struct {
|
|
PropagationBufferMs int `json:"propagationBufferMs"`
|
|
} `json:"liveMap"`
|
|
|
|
CacheTTL map[string]interface{} `json:"cacheTTL"`
|
|
|
|
Retention *RetentionConfig `json:"retention,omitempty"`
|
|
|
|
PacketStore *PacketStoreConfig `json:"packetStore,omitempty"`
|
|
|
|
GeoFilter *GeoFilterConfig `json:"geo_filter,omitempty"`
|
|
|
|
Timestamps *TimestampConfig `json:"timestamps,omitempty"`
|
|
|
|
DebugAffinity bool `json:"debugAffinity,omitempty"`
|
|
|
|
ResolvedPath *ResolvedPathConfig `json:"resolvedPath,omitempty"`
|
|
NeighborGraph *NeighborGraphConfig `json:"neighborGraph,omitempty"`
|
|
}
|
|
|
|
// weakAPIKeys is the blocklist of known default/example API keys that must be rejected.
|
|
var weakAPIKeys = map[string]bool{
|
|
"your-secret-api-key-here": true,
|
|
"change-me": true,
|
|
"example": true,
|
|
"test": true,
|
|
"password": true,
|
|
"admin": true,
|
|
"apikey": true,
|
|
"api-key": true,
|
|
"secret": true,
|
|
"default": true,
|
|
}
|
|
|
|
// IsWeakAPIKey returns true if the key is in the blocklist or shorter than 16 characters.
|
|
func IsWeakAPIKey(key string) bool {
|
|
if key == "" {
|
|
return false // empty is handled separately (endpoints disabled)
|
|
}
|
|
if weakAPIKeys[strings.ToLower(key)] {
|
|
return true
|
|
}
|
|
if len(key) < 16 {
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
// ResolvedPathConfig controls async backfill behavior.
|
|
type ResolvedPathConfig struct {
|
|
BackfillHours int `json:"backfillHours"` // how far back (hours) to scan for NULL resolved_path (default 24)
|
|
}
|
|
|
|
// NeighborGraphConfig controls neighbor edge pruning.
|
|
type NeighborGraphConfig struct {
|
|
MaxAgeDays int `json:"maxAgeDays"` // edges older than this are pruned (default 5)
|
|
}
|
|
|
|
// PacketStoreConfig controls in-memory packet store limits.
|
|
type PacketStoreConfig struct {
|
|
RetentionHours float64 `json:"retentionHours"` // max age of packets in hours (0 = unlimited)
|
|
MaxMemoryMB int `json:"maxMemoryMB"` // hard memory ceiling in MB (0 = unlimited)
|
|
MaxResolvedPubkeyIndexEntries int `json:"maxResolvedPubkeyIndexEntries"` // warning threshold for index size (0 = 5M default)
|
|
}
|
|
|
|
// GeoFilterConfig is an alias for the shared geofilter.Config type.
|
|
type GeoFilterConfig = geofilter.Config
|
|
|
|
type RetentionConfig struct {
|
|
NodeDays int `json:"nodeDays"`
|
|
ObserverDays int `json:"observerDays"`
|
|
PacketDays int `json:"packetDays"`
|
|
MetricsDays int `json:"metricsDays"`
|
|
}
|
|
|
|
// MetricsRetentionDays returns configured metrics retention or 30 days default.
|
|
func (c *Config) MetricsRetentionDays() int {
|
|
if c.Retention != nil && c.Retention.MetricsDays > 0 {
|
|
return c.Retention.MetricsDays
|
|
}
|
|
return 30
|
|
}
|
|
|
|
// BackfillHours returns configured backfill window or 24h default.
|
|
func (c *Config) BackfillHours() int {
|
|
if c.ResolvedPath != nil && c.ResolvedPath.BackfillHours > 0 {
|
|
return c.ResolvedPath.BackfillHours
|
|
}
|
|
return 24
|
|
}
|
|
|
|
// NeighborMaxAgeDays returns configured max edge age or 30 days default.
|
|
func (c *Config) NeighborMaxAgeDays() int {
|
|
if c.NeighborGraph != nil && c.NeighborGraph.MaxAgeDays > 0 {
|
|
return c.NeighborGraph.MaxAgeDays
|
|
}
|
|
return 5
|
|
}
|
|
|
|
type TimestampConfig struct {
|
|
DefaultMode string `json:"defaultMode"` // "ago" | "absolute"
|
|
Timezone string `json:"timezone"` // "local" | "utc"
|
|
FormatPreset string `json:"formatPreset"` // "iso" | "iso-seconds" | "locale"
|
|
CustomFormat string `json:"customFormat"` // freeform, only used when AllowCustomFormat=true
|
|
AllowCustomFormat bool `json:"allowCustomFormat"` // admin gate
|
|
}
|
|
|
|
func defaultTimestampConfig() TimestampConfig {
|
|
return TimestampConfig{
|
|
DefaultMode: "ago",
|
|
Timezone: "local",
|
|
FormatPreset: "iso",
|
|
CustomFormat: "",
|
|
AllowCustomFormat: false,
|
|
}
|
|
}
|
|
|
|
// NodeDaysOrDefault returns the configured retention.nodeDays or 7 if not set.
|
|
func (c *Config) NodeDaysOrDefault() int {
|
|
if c.Retention != nil && c.Retention.NodeDays > 0 {
|
|
return c.Retention.NodeDays
|
|
}
|
|
return 7
|
|
}
|
|
|
|
// ObserverDaysOrDefault returns the configured retention.observerDays or 14 if not set.
|
|
// A value of -1 means observers are never removed.
|
|
func (c *Config) ObserverDaysOrDefault() int {
|
|
if c.Retention != nil && c.Retention.ObserverDays != 0 {
|
|
return c.Retention.ObserverDays
|
|
}
|
|
return 14
|
|
}
|
|
|
|
type HealthThresholds struct {
|
|
InfraDegradedHours float64 `json:"infraDegradedHours"`
|
|
InfraSilentHours float64 `json:"infraSilentHours"`
|
|
NodeDegradedHours float64 `json:"nodeDegradedHours"`
|
|
NodeSilentHours float64 `json:"nodeSilentHours"`
|
|
}
|
|
|
|
// ThemeFile mirrors theme.json overlay.
|
|
type ThemeFile struct {
|
|
Branding map[string]interface{} `json:"branding"`
|
|
Theme map[string]interface{} `json:"theme"`
|
|
ThemeDark map[string]interface{} `json:"themeDark"`
|
|
NodeColors map[string]interface{} `json:"nodeColors"`
|
|
TypeColors map[string]interface{} `json:"typeColors"`
|
|
Home map[string]interface{} `json:"home"`
|
|
}
|
|
|
|
func LoadConfig(baseDirs ...string) (*Config, error) {
|
|
if len(baseDirs) == 0 {
|
|
baseDirs = []string{"."}
|
|
}
|
|
paths := make([]string, 0)
|
|
for _, d := range baseDirs {
|
|
paths = append(paths, filepath.Join(d, "config.json"))
|
|
paths = append(paths, filepath.Join(d, "data", "config.json"))
|
|
}
|
|
|
|
cfg := &Config{Port: 3000}
|
|
for _, p := range paths {
|
|
data, err := os.ReadFile(p)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
if err := json.Unmarshal(data, cfg); err != nil {
|
|
continue
|
|
}
|
|
cfg.NormalizeTimestampConfig()
|
|
return cfg, nil
|
|
}
|
|
cfg.NormalizeTimestampConfig()
|
|
return cfg, nil // defaults
|
|
}
|
|
|
|
func LoadTheme(baseDirs ...string) *ThemeFile {
|
|
if len(baseDirs) == 0 {
|
|
baseDirs = []string{"."}
|
|
}
|
|
for _, d := range baseDirs {
|
|
for _, name := range []string{"theme.json"} {
|
|
p := filepath.Join(d, name)
|
|
data, err := os.ReadFile(p)
|
|
if err != nil {
|
|
p = filepath.Join(d, "data", name)
|
|
data, err = os.ReadFile(p)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
}
|
|
var t ThemeFile
|
|
if json.Unmarshal(data, &t) == nil {
|
|
return &t
|
|
}
|
|
}
|
|
}
|
|
return &ThemeFile{}
|
|
}
|
|
|
|
func (c *Config) GetHealthThresholds() HealthThresholds {
|
|
h := HealthThresholds{
|
|
InfraDegradedHours: 24,
|
|
InfraSilentHours: 72,
|
|
NodeDegradedHours: 1,
|
|
NodeSilentHours: 24,
|
|
}
|
|
if c.HealthThresholds != nil {
|
|
if c.HealthThresholds.InfraDegradedHours > 0 {
|
|
h.InfraDegradedHours = c.HealthThresholds.InfraDegradedHours
|
|
}
|
|
if c.HealthThresholds.InfraSilentHours > 0 {
|
|
h.InfraSilentHours = c.HealthThresholds.InfraSilentHours
|
|
}
|
|
if c.HealthThresholds.NodeDegradedHours > 0 {
|
|
h.NodeDegradedHours = c.HealthThresholds.NodeDegradedHours
|
|
}
|
|
if c.HealthThresholds.NodeSilentHours > 0 {
|
|
h.NodeSilentHours = c.HealthThresholds.NodeSilentHours
|
|
}
|
|
}
|
|
return h
|
|
}
|
|
|
|
// GetHealthMs returns degraded/silent thresholds in ms for a given role.
|
|
func (h HealthThresholds) GetHealthMs(role string) (degradedMs, silentMs int) {
|
|
const hourMs = 3600000
|
|
if role == "repeater" || role == "room" {
|
|
return int(h.InfraDegradedHours * hourMs), int(h.InfraSilentHours * hourMs)
|
|
}
|
|
return int(h.NodeDegradedHours * hourMs), int(h.NodeSilentHours * hourMs)
|
|
}
|
|
|
|
// ToClientMs returns the thresholds as ms for the frontend.
|
|
func (h HealthThresholds) ToClientMs() map[string]int {
|
|
const hourMs = 3600000
|
|
return map[string]int{
|
|
"infraDegradedMs": int(h.InfraDegradedHours * hourMs),
|
|
"infraSilentMs": int(h.InfraSilentHours * hourMs),
|
|
"nodeDegradedMs": int(h.NodeDegradedHours * hourMs),
|
|
"nodeSilentMs": int(h.NodeSilentHours * hourMs),
|
|
}
|
|
}
|
|
|
|
func (c *Config) ResolveDBPath(baseDir string) string {
|
|
if c.DBPath != "" {
|
|
return c.DBPath
|
|
}
|
|
if v := os.Getenv("DB_PATH"); v != "" {
|
|
return v
|
|
}
|
|
return filepath.Join(baseDir, "data", "meshcore.db")
|
|
}
|
|
|
|
|
|
func (c *Config) NormalizeTimestampConfig() {
|
|
defaults := defaultTimestampConfig()
|
|
if c.Timestamps == nil {
|
|
log.Printf("[config] timestamps not configured - using defaults (ago/local/iso)")
|
|
c.Timestamps = &defaults
|
|
return
|
|
}
|
|
|
|
origMode := c.Timestamps.DefaultMode
|
|
mode := strings.ToLower(strings.TrimSpace(origMode))
|
|
switch mode {
|
|
case "ago", "absolute":
|
|
c.Timestamps.DefaultMode = mode
|
|
default:
|
|
log.Printf("[config] warning: timestamps.defaultMode=%q is invalid, using %q", origMode, defaults.DefaultMode)
|
|
c.Timestamps.DefaultMode = defaults.DefaultMode
|
|
}
|
|
|
|
origTimezone := c.Timestamps.Timezone
|
|
timezone := strings.ToLower(strings.TrimSpace(origTimezone))
|
|
switch timezone {
|
|
case "local", "utc":
|
|
c.Timestamps.Timezone = timezone
|
|
default:
|
|
log.Printf("[config] warning: timestamps.timezone=%q is invalid, using %q", origTimezone, defaults.Timezone)
|
|
c.Timestamps.Timezone = defaults.Timezone
|
|
}
|
|
|
|
origPreset := c.Timestamps.FormatPreset
|
|
formatPreset := strings.ToLower(strings.TrimSpace(origPreset))
|
|
switch formatPreset {
|
|
case "iso", "iso-seconds", "locale":
|
|
c.Timestamps.FormatPreset = formatPreset
|
|
default:
|
|
log.Printf("[config] warning: timestamps.formatPreset=%q is invalid, using %q", origPreset, defaults.FormatPreset)
|
|
c.Timestamps.FormatPreset = defaults.FormatPreset
|
|
}
|
|
}
|
|
|
|
func (c *Config) GetTimestampConfig() TimestampConfig {
|
|
if c == nil || c.Timestamps == nil {
|
|
return defaultTimestampConfig()
|
|
}
|
|
return *c.Timestamps
|
|
}
|
|
func (c *Config) PropagationBufferMs() int {
|
|
if c.LiveMap.PropagationBufferMs > 0 {
|
|
return c.LiveMap.PropagationBufferMs
|
|
}
|
|
return 5000
|
|
}
|
|
|
|
// blacklistSet lazily builds and caches the nodeBlacklist as a set for O(1) lookups.
|
|
// Uses sync.Once to eliminate the data race on first concurrent access.
|
|
func (c *Config) blacklistSet() map[string]bool {
|
|
c.blacklistOnce.Do(func() {
|
|
if len(c.NodeBlacklist) == 0 {
|
|
return
|
|
}
|
|
m := make(map[string]bool, len(c.NodeBlacklist))
|
|
for _, pk := range c.NodeBlacklist {
|
|
trimmed := strings.ToLower(strings.TrimSpace(pk))
|
|
if trimmed != "" {
|
|
m[trimmed] = true
|
|
}
|
|
}
|
|
c.blacklistSetCached = m
|
|
})
|
|
return c.blacklistSetCached
|
|
}
|
|
|
|
// IsBlacklisted returns true if the given public key is in the nodeBlacklist.
|
|
func (c *Config) IsBlacklisted(pubkey string) bool {
|
|
if c == nil || len(c.NodeBlacklist) == 0 {
|
|
return false
|
|
}
|
|
return c.blacklistSet()[strings.ToLower(strings.TrimSpace(pubkey))]
|
|
}
|