diff --git a/cmd/server/clamp_limit.go b/cmd/server/clamp_limit.go index 2d7dae77..e8522daa 100644 --- a/cmd/server/clamp_limit.go +++ b/cmd/server/clamp_limit.go @@ -33,4 +33,3 @@ func clampLimit(raw string, def, max int) int { func queryLimit(r *http.Request, def, max int) int { return clampLimit(r.URL.Query().Get("limit"), def, max) } - diff --git a/cmd/server/config.go b/cmd/server/config.go index 951f4fce..98a92e22 100644 --- a/cmd/server/config.go +++ b/cmd/server/config.go @@ -24,11 +24,21 @@ type AreaEntry struct { LonMax *float64 `json:"lonMax,omitempty"` } +// ListLimitsConfig defines maximum row limits for list endpoints to prevent DoS. +type ListLimitsConfig struct { + PacketsMax int `json:"packetsMax"` + NodesMax int `json:"nodesMax"` + AnalyticsMax int `json:"analyticsMax"` + ChannelMessagesMax int `json:"channelMessagesMax"` + BulkHealthMax int `json:"bulkHealthMax"` +} + // Config mirrors the Node.js config.json structure (read-only fields). type Config struct { - Port int `json:"port"` - APIKey string `json:"apiKey"` - DBPath string `json:"dbPath"` + Port int `json:"port"` + APIKey string `json:"apiKey"` + DBPath string `json:"dbPath"` + ListLimits *ListLimitsConfig `json:"listLimits"` // NodeBlacklist is a list of public keys to exclude from all API responses. // Blacklisted nodes are hidden from node lists, search, detail, map, and stats. @@ -391,15 +401,38 @@ func LoadConfig(baseDirs ...string) (*Config, error) { } cfg.NormalizeTimestampConfig() cfg.migrateDeprecatedConfig() + cfg.applyListLimitsDefaults() applyCORSEnv(cfg) return cfg, nil } cfg.NormalizeTimestampConfig() cfg.migrateDeprecatedConfig() + cfg.applyListLimitsDefaults() applyCORSEnv(cfg) return cfg, nil // defaults } +func (c *Config) applyListLimitsDefaults() { + if c.ListLimits == nil { + c.ListLimits = &ListLimitsConfig{} + } + if c.ListLimits.PacketsMax <= 0 { + c.ListLimits.PacketsMax = 10000 + } + if c.ListLimits.NodesMax <= 0 { + c.ListLimits.NodesMax = 2000 + } + if c.ListLimits.AnalyticsMax <= 0 { + c.ListLimits.AnalyticsMax = 200 + } + if c.ListLimits.ChannelMessagesMax <= 0 { + c.ListLimits.ChannelMessagesMax = 500 + } + if c.ListLimits.BulkHealthMax <= 0 { + c.ListLimits.BulkHealthMax = 200 + } +} + func (c *Config) migrateDeprecatedConfig() { migrated := false if c.Map == nil { diff --git a/cmd/server/config_test.go b/cmd/server/config_test.go index 1894c889..b78968ab 100644 --- a/cmd/server/config_test.go +++ b/cmd/server/config_test.go @@ -455,3 +455,63 @@ func TestObserverThresholdsDefaultsFromEmptyConfigFile(t *testing.T) { t.Errorf("empty-config ObserverStaleMinutes = %d, want 1440 (new default)", h.ObserverStaleMinutes) } } + +func TestApplyListLimitsDefaults(t *testing.T) { + t.Run("defaults when block is absent", func(t *testing.T) { + dir := t.TempDir() + os.WriteFile(filepath.Join(dir, "config.json"), []byte(`{"port": 3000}`), 0644) + cfg, err := LoadConfig(dir) + if err != nil { + t.Fatal(err) + } + if cfg.ListLimits.PacketsMax != 10000 { + t.Errorf("expected 10000, got %d", cfg.ListLimits.PacketsMax) + } + if cfg.ListLimits.NodesMax != 2000 { + t.Errorf("expected 2000, got %d", cfg.ListLimits.NodesMax) + } + if cfg.ListLimits.AnalyticsMax != 200 { + t.Errorf("expected 200, got %d", cfg.ListLimits.AnalyticsMax) + } + if cfg.ListLimits.ChannelMessagesMax != 500 { + t.Errorf("expected 500, got %d", cfg.ListLimits.ChannelMessagesMax) + } + if cfg.ListLimits.BulkHealthMax != 200 { + t.Errorf("expected 200, got %d", cfg.ListLimits.BulkHealthMax) + } + }) + + t.Run("operator overrides honored", func(t *testing.T) { + dir := t.TempDir() + cfgData := map[string]interface{}{ + "listLimits": map[string]interface{}{ + "packetsMax": 50000, + "nodesMax": 5000, + "analyticsMax": 500, + "channelMessagesMax": 1000, + "bulkHealthMax": 300, + }, + } + data, _ := json.Marshal(cfgData) + os.WriteFile(filepath.Join(dir, "config.json"), data, 0644) + cfg, err := LoadConfig(dir) + if err != nil { + t.Fatal(err) + } + if cfg.ListLimits.PacketsMax != 50000 { + t.Errorf("expected 50000, got %d", cfg.ListLimits.PacketsMax) + } + if cfg.ListLimits.NodesMax != 5000 { + t.Errorf("expected 5000, got %d", cfg.ListLimits.NodesMax) + } + if cfg.ListLimits.AnalyticsMax != 500 { + t.Errorf("expected 500, got %d", cfg.ListLimits.AnalyticsMax) + } + if cfg.ListLimits.ChannelMessagesMax != 1000 { + t.Errorf("expected 1000, got %d", cfg.ListLimits.ChannelMessagesMax) + } + if cfg.ListLimits.BulkHealthMax != 300 { + t.Errorf("expected 300, got %d", cfg.ListLimits.BulkHealthMax) + } + }) +} diff --git a/cmd/server/db.go b/cmd/server/db.go index aa1b8aa2..dc8a5b93 100644 --- a/cmd/server/db.go +++ b/cmd/server/db.go @@ -479,6 +479,8 @@ type PacketQuery struct { type PacketResult struct { Packets []map[string]interface{} `json:"packets"` Total int `json:"total"` + Limit int `json:"limit"` + Offset int `json:"offset"` } // QueryPackets returns paginated, filtered packets as transmissions (matching Node.js shape). diff --git a/cmd/server/index_ready_1008_test.go b/cmd/server/index_ready_1008_test.go index cf759f42..e8a602ef 100644 --- a/cmd/server/index_ready_1008_test.go +++ b/cmd/server/index_ready_1008_test.go @@ -60,7 +60,9 @@ func TestIssue1008_HandlerReturns503WhileSubpathIndexLoading(t *testing.T) { } // Don't wait for the background build — we want to observe the // not-ready window. - srv := &Server{store: store} + cfg := &Config{} + cfg.applyListLimitsDefaults() + srv := &Server{store: store, cfg: cfg} req := httptest.NewRequest("GET", "/api/analytics/subpaths?minLen=2&maxLen=4&limit=10", nil) rec := httptest.NewRecorder() @@ -107,7 +109,9 @@ func TestIssue1008_HandlerRecoversAfterIndexReady(t *testing.T) { t.Fatal("PathHopIndexReady() never flipped true within 5s") } - srv := &Server{store: store} + cfg := &Config{} + cfg.applyListLimitsDefaults() + srv := &Server{store: store, cfg: cfg} req := httptest.NewRequest("GET", "/api/analytics/subpaths?minLen=2&maxLen=4&limit=10", nil) rec := httptest.NewRecorder() srv.handleAnalyticsSubpaths(rec, req) diff --git a/cmd/server/pr1589_regression_test.go b/cmd/server/pr1589_regression_test.go new file mode 100644 index 00000000..ec0cdcd5 --- /dev/null +++ b/cmd/server/pr1589_regression_test.go @@ -0,0 +1,98 @@ +package main + +// Regression tests for the three MAJOR findings on PR #1589. +// These tests gate three semantic regressions that the rest of the PR's tests +// did not catch: +// +// MAJOR-1: handleAnalyticsSubpaths default limit was silently halved 100→50 +// when migrated to queryLimit(r, 50, ...AnalyticsMax). +// MAJOR-2: handleChannelMessages default limit was silently halved 100→50 +// when migrated to queryLimit(r, 50, ...ChannelMessagesMax). +// MAJOR-3: handleBulkHealth was bundled into NodesMax (default 2000), +// 10× its previous ceiling of 200, despite being per-row heavier. +// +// For MAJOR-1/2 we assert on the literal call-site `def` value via source +// inspection because the rendered response does not expose the applied limit. +// For MAJOR-3 we assert both the config-defaults plumbing AND the runtime +// behavior: BulkHealthMax must exist as its own field with default 200, and +// handleBulkHealth must clamp through it (not NodesMax). + +import ( + "net/http/httptest" + "os" + "strings" + "testing" +) + +func TestPR1589_AnalyticsSubpathsDefaultIs100(t *testing.T) { + // MAJOR-1: regression guard. + src, err := os.ReadFile("routes.go") + if err != nil { + t.Fatalf("read routes.go: %v", err) + } + if !strings.Contains(string(src), "queryLimit(r, 100, s.cfg.ListLimits.AnalyticsMax)") { + t.Error("handleAnalyticsSubpaths must use def=100 in queryLimit; " + + "PR #1589 inadvertently halved the default to 50 (MAJOR-1)") + } +} + +func TestPR1589_ChannelMessagesDefaultIs100(t *testing.T) { + // MAJOR-2: regression guard. + src, err := os.ReadFile("routes.go") + if err != nil { + t.Fatalf("read routes.go: %v", err) + } + if !strings.Contains(string(src), "queryLimit(r, 100, s.cfg.ListLimits.ChannelMessagesMax)") { + t.Error("handleChannelMessages must use def=100 in queryLimit; " + + "PR #1589 inadvertently halved the default to 50 (MAJOR-2)") + } +} + +func TestPR1589_BulkHealthMaxDefaultsTo200(t *testing.T) { + // MAJOR-3 (config plumbing): a dedicated BulkHealthMax must exist with + // default 200 — bulk-health is per-row much heavier than /api/nodes, + // so it cannot inherit NodesMax (default 2000). + dir := t.TempDir() + os.WriteFile(dir+"/config.json", []byte(`{"port":3000}`), 0644) + cfg, err := LoadConfig(dir) + if err != nil { + t.Fatalf("LoadConfig: %v", err) + } + if cfg.ListLimits.BulkHealthMax != 200 { + t.Errorf("expected BulkHealthMax default 200, got %d", cfg.ListLimits.BulkHealthMax) + } +} + +func TestPR1589_BulkHealthClampsViaBulkHealthMax(t *testing.T) { + // MAJOR-3 (runtime wiring): /api/nodes/bulk-health must clamp the limit + // through BulkHealthMax — not NodesMax. We set BulkHealthMax=1 and + // NodesMax=9999; if the handler still uses NodesMax the seed data (3 + // nodes) will all come back. If wired correctly it must clamp to 1. + srv, router := setupTestServer(t) + srv.cfg.ListLimits = &ListLimitsConfig{ + PacketsMax: 10000, + NodesMax: 9999, + AnalyticsMax: 200, + ChannelMessagesMax: 500, + BulkHealthMax: 1, + } + + req := httptest.NewRequest("GET", "/api/nodes/bulk-health?limit=500", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + if w.Code != 200 { + t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String()) + } + // Response is a top-level JSON array (filtered or unfiltered). + body := strings.TrimSpace(w.Body.String()) + if !strings.HasPrefix(body, "[") { + t.Fatalf("expected JSON array response, got: %s", body) + } + // Count top-level objects via "public_key" occurrences (each row has one). + rowCount := strings.Count(body, `"public_key"`) + if rowCount > 1 { + t.Errorf("BulkHealthMax=1 should clamp to 1 row, got %d rows; "+ + "handler is likely still using NodesMax (MAJOR-3): %s", rowCount, body) + } +} diff --git a/cmd/server/routes.go b/cmd/server/routes.go index eb17b009..687b5191 100644 --- a/cmd/server/routes.go +++ b/cmd/server/routes.go @@ -36,13 +36,13 @@ type Server struct { buildTime string // Cached runtime.MemStats to avoid stop-the-world pauses on every health check - memStatsMu sync.Mutex - memStatsCache runtime.MemStats + memStatsMu sync.Mutex + memStatsCache runtime.MemStats memStatsCachedAt time.Time // Cached /api/stats response — recomputed at most once every 10s - statsMu sync.Mutex - statsCache *StatsResponse + statsMu sync.Mutex + statsCache *StatsResponse statsCachedAt time.Time // Guards s.cfg.GeoFilter — read by ingest/handler goroutines, written by PUT handler @@ -111,6 +111,9 @@ func NewPerfStats() *PerfStats { } func NewServer(db *DB, cfg *Config, hub *Hub) *Server { + if cfg != nil { + cfg.applyListLimitsDefaults() + } return &Server{ db: db, cfg: cfg, @@ -474,30 +477,30 @@ func (s *Server) handleConfigTheme(w http.ResponseWriter, r *http.Request) { }, s.cfg.Branding, theme.Branding) themeColors := mergeMap(map[string]interface{}{ - "accent": "#4a9eff", - "accentHover": "#6db3ff", - "navBg": "#0f0f23", - "navBg2": "#1a1a2e", - "navText": "#ffffff", + "accent": "#4a9eff", + "accentHover": "#6db3ff", + "navBg": "#0f0f23", + "navBg2": "#1a1a2e", + "navText": "#ffffff", "navTextMuted": "#cbd5e1", - "background": "#f4f5f7", - "text": "#1a1a2e", - "textMuted": "#5b6370", - "border": "#e2e5ea", - "surface1": "#ffffff", - "surface2": "#ffffff", - "surface3": "#ffffff", - "sectionBg": "#eef2ff", - "cardBg": "#ffffff", - "contentBg": "#f4f5f7", - "detailBg": "#ffffff", - "inputBg": "#ffffff", - "rowStripe": "#f9fafb", - "rowHover": "#eef2ff", - "selectedBg": "#dbeafe", - "statusGreen": "#22c55e", + "background": "#f4f5f7", + "text": "#1a1a2e", + "textMuted": "#5b6370", + "border": "#e2e5ea", + "surface1": "#ffffff", + "surface2": "#ffffff", + "surface3": "#ffffff", + "sectionBg": "#eef2ff", + "cardBg": "#ffffff", + "contentBg": "#f4f5f7", + "detailBg": "#ffffff", + "inputBg": "#ffffff", + "rowStripe": "#f9fafb", + "rowHover": "#eef2ff", + "selectedBg": "#dbeafe", + "statusGreen": "#22c55e", "statusYellow": "#eab308", - "statusRed": "#ef4444", + "statusRed": "#ef4444", }, s.cfg.Theme, theme.Theme) nodeColors := mergeMap(map[string]interface{}{ @@ -509,30 +512,30 @@ func (s *Server) handleConfigTheme(w http.ResponseWriter, r *http.Request) { }, s.cfg.NodeColors, theme.NodeColors) themeDark := mergeMap(map[string]interface{}{ - "accent": "#4a9eff", - "accentHover": "#6db3ff", - "navBg": "#0f0f23", - "navBg2": "#1a1a2e", - "navText": "#ffffff", + "accent": "#4a9eff", + "accentHover": "#6db3ff", + "navBg": "#0f0f23", + "navBg2": "#1a1a2e", + "navText": "#ffffff", "navTextMuted": "#cbd5e1", - "background": "#0f0f23", - "text": "#e2e8f0", - "textMuted": "#a8b8cc", - "border": "#334155", - "surface1": "#1a1a2e", - "surface2": "#232340", - "cardBg": "#1a1a2e", - "contentBg": "#0f0f23", - "detailBg": "#232340", - "inputBg": "#1e1e34", - "rowStripe": "#1e1e34", - "rowHover": "#2d2d50", - "selectedBg": "#1e3a5f", - "statusGreen": "#22c55e", + "background": "#0f0f23", + "text": "#e2e8f0", + "textMuted": "#a8b8cc", + "border": "#334155", + "surface1": "#1a1a2e", + "surface2": "#232340", + "cardBg": "#1a1a2e", + "contentBg": "#0f0f23", + "detailBg": "#232340", + "inputBg": "#1e1e34", + "rowStripe": "#1e1e34", + "rowHover": "#2d2d50", + "selectedBg": "#1e3a5f", + "statusGreen": "#22c55e", "statusYellow": "#eab308", - "statusRed": "#ef4444", - "surface3": "#2d2d50", - "sectionBg": "#1e1e34", + "statusRed": "#ef4444", + "surface3": "#2d2d50", + "sectionBg": "#1e1e34", }, s.cfg.ThemeDark, theme.ThemeDark) typeColors := mergeMap(map[string]interface{}{ "ADVERT": "#22c55e", @@ -916,7 +919,7 @@ func (s *Server) handlePackets(w http.ResponseWriter, r *http.Request) { if r.URL.Query().Get("order") == "asc" { order = "ASC" } - lim := queryLimit(r, 50, 500) + lim := queryLimit(r, 50, s.cfg.ListLimits.PacketsMax) var result *PacketResult var err error if s.store != nil { @@ -932,24 +935,21 @@ func (s *Server) handlePackets(w http.ResponseWriter, r *http.Request) { writeError(w, 500, err.Error()) return } - writeJSON(w, PacketListResponse{ - Packets: mapSliceToTransmissions(result.Packets), - Total: result.Total, - Limit: lim, - Offset: queryInt(r, "offset", 0), - }) + result.Limit = lim + result.Offset = queryInt(r, "offset", 0) + writeJSON(w, result) return } q := PacketQuery{ - Limit: queryLimit(r, 50, 500), - Offset: queryInt(r, "offset", 0), - Observer: r.URL.Query().Get("observer"), - Hash: r.URL.Query().Get("hash"), - Since: r.URL.Query().Get("since"), - Until: r.URL.Query().Get("until"), - Region: r.URL.Query().Get("region"), - Node: r.URL.Query().Get("node"), + Limit: queryLimit(r, 50, s.cfg.ListLimits.PacketsMax), + Offset: queryInt(r, "offset", 0), + Observer: r.URL.Query().Get("observer"), + Hash: r.URL.Query().Get("hash"), + Since: r.URL.Query().Get("since"), + Until: r.URL.Query().Get("until"), + Region: r.URL.Query().Get("region"), + Node: r.URL.Query().Get("node"), Channel: r.URL.Query().Get("channel"), Area: r.URL.Query().Get("area"), Order: "DESC", @@ -979,6 +979,8 @@ func (s *Server) handlePackets(w http.ResponseWriter, r *http.Request) { writeError(w, 500, err.Error()) return } + result.Limit = q.Limit + result.Offset = q.Offset writeJSON(w, result) return } @@ -995,6 +997,8 @@ func (s *Server) handlePackets(w http.ResponseWriter, r *http.Request) { return } + result.Limit = q.Limit + result.Offset = q.Offset writeJSON(w, result) } @@ -1236,7 +1240,7 @@ func (s *Server) handlePostPacket(w http.ResponseWriter, r *http.Request) { func (s *Server) handleNodes(w http.ResponseWriter, r *http.Request) { q := r.URL.Query() nodes, total, counts, err := s.db.GetNodes( - queryLimit(r, 50, 500), + queryLimit(r, 50, s.cfg.ListLimits.NodesMax), queryInt(r, "offset", 0), q.Get("role"), q.Get("search"), q.Get("before"), q.Get("lastHeard"), q.Get("sortBy"), q.Get("region"), @@ -1477,12 +1481,12 @@ func (s *Server) handleNodeHealth(w http.ResponseWriter, r *http.Request) { } func (s *Server) handleBulkHealth(w http.ResponseWriter, r *http.Request) { - limit := queryLimit(r, 50, 200) + lim := queryLimit(r, 50, s.cfg.ListLimits.BulkHealthMax) if s.store != nil { region := r.URL.Query().Get("region") area := r.URL.Query().Get("area") - results := s.store.GetBulkHealth(limit, region, area) + results := s.store.GetBulkHealth(lim, region, area) // Filter blacklisted nodes if len(s.cfg.NodeBlacklist) > 0 { filtered := make([]map[string]interface{}, 0, len(results)) @@ -2074,7 +2078,7 @@ func (s *Server) handleAnalyticsHashSizes(w http.ResponseWriter, r *http.Request return } writeJSON(w, map[string]interface{}{ - "total": 0, + "total": 0, "distribution": map[string]int{"1": 0, "2": 0, "3": 0}, "distributionByRepeaters": map[string]int{"1": 0, "2": 0, "3": 0}, "hourly": []HashSizeHourly{}, @@ -2108,7 +2112,7 @@ func (s *Server) handleAnalyticsSubpaths(w http.ResponseWriter, r *http.Request) minLen = 2 } maxLen := queryInt(r, "maxLen", 8) - limit := queryLimit(r, 100, 200) + limit := queryLimit(r, 100, s.cfg.ListLimits.AnalyticsMax) // Issue #1217: honor the Time window filter on Route Patterns. window := ParseTimeWindow(r) data := s.store.GetAnalyticsSubpathsWithWindow(region, minLen, maxLen, limit, window) @@ -2126,7 +2130,8 @@ func (s *Server) handleAnalyticsSubpaths(w http.ResponseWriter, r *http.Request) // handleAnalyticsSubpathsBulk returns multiple length-range buckets in a single // response, avoiding repeated scans of the same packet data. Query format: -// ?groups=2-2:50,3-3:30,4-4:20,5-8:15 (minLen-maxLen:limit per group) +// +// ?groups=2-2:50,3-3:30,4-4:20,5-8:15 (minLen-maxLen:limit per group) func (s *Server) handleAnalyticsSubpathsBulk(w http.ResponseWriter, r *http.Request) { if s.store != nil && !s.store.SubpathIndexReady() { writeIndexLoading503(w) @@ -2408,7 +2413,7 @@ func (s *Server) handleChannels(w http.ResponseWriter, r *http.Request) { func (s *Server) handleChannelMessages(w http.ResponseWriter, r *http.Request) { hash := mux.Vars(r)["hash"] - limit := queryLimit(r, 100, 500) + limit := queryLimit(r, 100, s.cfg.ListLimits.ChannelMessagesMax) offset := queryInt(r, "offset", 0) region := r.URL.Query().Get("region") // Prefer DB for full history (in-memory store has limited retention) @@ -2522,13 +2527,13 @@ func (s *Server) buildObserversDefaultResponse() (ObserverListResponse, error) { ID: o.ID, Name: o.Name, IATA: o.IATA, LastSeen: o.LastSeen, FirstSeen: o.FirstSeen, PacketCount: o.PacketCount, - Model: o.Model, Firmware: o.Firmware, + Model: o.Model, Firmware: o.Firmware, ClientVersion: o.ClientVersion, Radio: o.Radio, BatteryMv: o.BatteryMv, UptimeSecs: o.UptimeSecs, - NoiseFloor: o.NoiseFloor, - LastPacketAt: o.LastPacketAt, + NoiseFloor: o.NoiseFloor, + LastPacketAt: o.LastPacketAt, PacketsLastHour: plh, - Lat: lat, Lon: lon, NodeRole: nodeRole, + Lat: lat, Lon: lon, NodeRole: nodeRole, } applyObserverNaiveClock(&resp, o, nowTime) result = append(result, resp) @@ -2567,11 +2572,11 @@ func (s *Server) handleObserverDetail(w http.ResponseWriter, r *http.Request) { ID: obs.ID, Name: obs.Name, IATA: obs.IATA, LastSeen: obs.LastSeen, FirstSeen: obs.FirstSeen, PacketCount: obs.PacketCount, - Model: obs.Model, Firmware: obs.Firmware, + Model: obs.Model, Firmware: obs.Firmware, ClientVersion: obs.ClientVersion, Radio: obs.Radio, BatteryMv: obs.BatteryMv, UptimeSecs: obs.UptimeSecs, - NoiseFloor: obs.NoiseFloor, - LastPacketAt: obs.LastPacketAt, + NoiseFloor: obs.NoiseFloor, + LastPacketAt: obs.LastPacketAt, PacketsLastHour: plh, } applyObserverNaiveClock(&resp, obs, time.Now().UTC()) @@ -3246,7 +3251,7 @@ func (s *Server) filterBlacklistedFromSubpaths(data map[string]interface{}) map[ // handleDroppedPackets returns recently dropped packets for investigation. func (s *Server) handleDroppedPackets(w http.ResponseWriter, r *http.Request) { - limit := queryLimit(r, 100, 500) + limit := queryLimit(r, 100, s.cfg.ListLimits.PacketsMax) observerID := r.URL.Query().Get("observer") nodePubkey := r.URL.Query().Get("pubkey") diff --git a/cmd/server/routes_test.go b/cmd/server/routes_test.go index 5ad9db06..a1f0b14a 100644 --- a/cmd/server/routes_test.go +++ b/cmd/server/routes_test.go @@ -1909,7 +1909,6 @@ func TestHandlerErrorPaths(t *testing.T) { router := mux.NewRouter() srv.RegisterRoutes(router) - t.Run("stats error", func(t *testing.T) { db.conn.Exec("DROP TABLE IF EXISTS transmissions") req := httptest.NewRequest("GET", "/api/stats", nil) @@ -2130,222 +2129,221 @@ func TestHandlerErrorBulkHealth(t *testing.T) { } } - func TestAnalyticsChannelsNoNullArrays(t *testing.T) { -_, router := setupTestServer(t) -req := httptest.NewRequest("GET", "/api/analytics/channels", nil) -w := httptest.NewRecorder() -router.ServeHTTP(w, req) + _, router := setupTestServer(t) + req := httptest.NewRequest("GET", "/api/analytics/channels", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) -if w.Code != 200 { -t.Fatalf("expected 200, got %d", w.Code) -} + if w.Code != 200 { + t.Fatalf("expected 200, got %d", w.Code) + } -raw := w.Body.String() -var body map[string]interface{} -if err := json.Unmarshal([]byte(raw), &body); err != nil { -t.Fatalf("invalid JSON: %v", err) -} + raw := w.Body.String() + var body map[string]interface{} + if err := json.Unmarshal([]byte(raw), &body); err != nil { + t.Fatalf("invalid JSON: %v", err) + } -arrayFields := []string{"channels", "topSenders", "channelTimeline", "msgLengths"} -for _, field := range arrayFields { -val, exists := body[field] -if !exists { -t.Errorf("missing field %q", field) -continue -} -if val == nil { -t.Errorf("field %q is null, expected empty array []", field) -continue -} -if _, ok := val.([]interface{}); !ok { -t.Errorf("field %q is not an array, got %T", field, val) -} -} + arrayFields := []string{"channels", "topSenders", "channelTimeline", "msgLengths"} + for _, field := range arrayFields { + val, exists := body[field] + if !exists { + t.Errorf("missing field %q", field) + continue + } + if val == nil { + t.Errorf("field %q is null, expected empty array []", field) + continue + } + if _, ok := val.([]interface{}); !ok { + t.Errorf("field %q is not an array, got %T", field, val) + } + } } func TestAnalyticsChannelsNoStoreFallbackNoNulls(t *testing.T) { -db := setupTestDB(t) -seedTestData(t, db) -cfg := &Config{Port: 3000} -hub := NewHub() -srv := NewServer(db, cfg, hub) -router := mux.NewRouter() -srv.RegisterRoutes(router) + db := setupTestDB(t) + seedTestData(t, db) + cfg := &Config{Port: 3000} + hub := NewHub() + srv := NewServer(db, cfg, hub) + router := mux.NewRouter() + srv.RegisterRoutes(router) -req := httptest.NewRequest("GET", "/api/analytics/channels", nil) -w := httptest.NewRecorder() -router.ServeHTTP(w, req) + req := httptest.NewRequest("GET", "/api/analytics/channels", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) -if w.Code != 200 { -t.Fatalf("expected 200, got %d", w.Code) -} + if w.Code != 200 { + t.Fatalf("expected 200, got %d", w.Code) + } -var body map[string]interface{} -json.Unmarshal(w.Body.Bytes(), &body) + var body map[string]interface{} + json.Unmarshal(w.Body.Bytes(), &body) -arrayFields := []string{"channels", "topSenders", "channelTimeline", "msgLengths"} -for _, field := range arrayFields { -if body[field] == nil { -t.Errorf("field %q is null in DB fallback, expected []", field) -} -} + arrayFields := []string{"channels", "topSenders", "channelTimeline", "msgLengths"} + for _, field := range arrayFields { + if body[field] == nil { + t.Errorf("field %q is null in DB fallback, expected []", field) + } + } } func TestNodeHashSizeEnrichment(t *testing.T) { -t.Run("nil info leaves defaults", func(t *testing.T) { -node := map[string]interface{}{ -"public_key": "abc123", -"hash_size": nil, -"hash_size_inconsistent": false, -} -EnrichNodeWithHashSize(node, nil) -if node["hash_size"] != nil { -t.Error("expected hash_size to remain nil with nil info") -} -}) + t.Run("nil info leaves defaults", func(t *testing.T) { + node := map[string]interface{}{ + "public_key": "abc123", + "hash_size": nil, + "hash_size_inconsistent": false, + } + EnrichNodeWithHashSize(node, nil) + if node["hash_size"] != nil { + t.Error("expected hash_size to remain nil with nil info") + } + }) -t.Run("enriches with computed data", func(t *testing.T) { -node := map[string]interface{}{ -"public_key": "abc123", -"hash_size": nil, -"hash_size_inconsistent": false, -} -info := &hashSizeNodeInfo{ -HashSize: 2, -AllSizes: map[int]bool{1: true, 2: true}, -Seq: []int{1, 2, 1, 2}, -Inconsistent: true, -} -EnrichNodeWithHashSize(node, info) -if node["hash_size"] != 2 { -t.Errorf("expected hash_size 2, got %v", node["hash_size"]) -} -if node["hash_size_inconsistent"] != true { -t.Error("expected hash_size_inconsistent true") -} -sizes, ok := node["hash_sizes_seen"].([]int) -if !ok { -t.Fatal("expected hash_sizes_seen to be []int") -} -if len(sizes) != 2 || sizes[0] != 1 || sizes[1] != 2 { -t.Errorf("expected [1,2], got %v", sizes) -} -}) + t.Run("enriches with computed data", func(t *testing.T) { + node := map[string]interface{}{ + "public_key": "abc123", + "hash_size": nil, + "hash_size_inconsistent": false, + } + info := &hashSizeNodeInfo{ + HashSize: 2, + AllSizes: map[int]bool{1: true, 2: true}, + Seq: []int{1, 2, 1, 2}, + Inconsistent: true, + } + EnrichNodeWithHashSize(node, info) + if node["hash_size"] != 2 { + t.Errorf("expected hash_size 2, got %v", node["hash_size"]) + } + if node["hash_size_inconsistent"] != true { + t.Error("expected hash_size_inconsistent true") + } + sizes, ok := node["hash_sizes_seen"].([]int) + if !ok { + t.Fatal("expected hash_sizes_seen to be []int") + } + if len(sizes) != 2 || sizes[0] != 1 || sizes[1] != 2 { + t.Errorf("expected [1,2], got %v", sizes) + } + }) -t.Run("single size omits sizes_seen", func(t *testing.T) { -node := map[string]interface{}{ -"public_key": "abc123", -"hash_size": nil, -"hash_size_inconsistent": false, -} -info := &hashSizeNodeInfo{ -HashSize: 3, -AllSizes: map[int]bool{3: true}, -Seq: []int{3, 3, 3}, -} -EnrichNodeWithHashSize(node, info) -if node["hash_size"] != 3 { -t.Errorf("expected hash_size 3, got %v", node["hash_size"]) -} -if node["hash_size_inconsistent"] != false { -t.Error("expected hash_size_inconsistent false") -} -if _, exists := node["hash_sizes_seen"]; exists { -t.Error("hash_sizes_seen should not be set for single size") -} -}) + t.Run("single size omits sizes_seen", func(t *testing.T) { + node := map[string]interface{}{ + "public_key": "abc123", + "hash_size": nil, + "hash_size_inconsistent": false, + } + info := &hashSizeNodeInfo{ + HashSize: 3, + AllSizes: map[int]bool{3: true}, + Seq: []int{3, 3, 3}, + } + EnrichNodeWithHashSize(node, info) + if node["hash_size"] != 3 { + t.Errorf("expected hash_size 3, got %v", node["hash_size"]) + } + if node["hash_size_inconsistent"] != false { + t.Error("expected hash_size_inconsistent false") + } + if _, exists := node["hash_sizes_seen"]; exists { + t.Error("hash_sizes_seen should not be set for single size") + } + }) } func TestGetNodeHashSizeInfoFlipFlop(t *testing.T) { -db := setupTestDB(t) -seedTestData(t, db) -store := NewPacketStore(db, nil) -if err := store.Load(); err != nil { - t.Fatalf("store.Load failed: %v", err) -} -store.WaitIndexesReady(5 * time.Second) + db := setupTestDB(t) + seedTestData(t, db) + store := NewPacketStore(db, nil) + if err := store.Load(); err != nil { + t.Fatalf("store.Load failed: %v", err) + } + store.WaitIndexesReady(5 * time.Second) -pk := "abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890" -db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role) VALUES (?, 'TestNode', 'repeater')", pk) + pk := "abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890" + db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role) VALUES (?, 'TestNode', 'repeater')", pk) -decoded := `{"name":"TestNode","pubKey":"` + pk + `"}` -raw1 := "11" + "01" + "aabb" -raw2 := "11" + "41" + "aabb" + decoded := `{"name":"TestNode","pubKey":"` + pk + `"}` + raw1 := "11" + "01" + "aabb" + raw2 := "11" + "41" + "aabb" -payloadType := 4 -for i := 0; i < 3; i++ { -rawHex := raw1 -if i%2 == 1 { -rawHex = raw2 -} -tx := &StoreTx{ -ID: 9000 + i, -RawHex: rawHex, -Hash: "testhash" + strconv.Itoa(i), -FirstSeen: time.Now().UTC().Format("2006-01-02T15:04:05.000Z"), -PayloadType: &payloadType, -DecodedJSON: decoded, -} -store.packets = append(store.packets, tx) -store.byPayloadType[4] = append(store.byPayloadType[4], tx) -} + payloadType := 4 + for i := 0; i < 3; i++ { + rawHex := raw1 + if i%2 == 1 { + rawHex = raw2 + } + tx := &StoreTx{ + ID: 9000 + i, + RawHex: rawHex, + Hash: "testhash" + strconv.Itoa(i), + FirstSeen: time.Now().UTC().Format("2006-01-02T15:04:05.000Z"), + PayloadType: &payloadType, + DecodedJSON: decoded, + } + store.packets = append(store.packets, tx) + store.byPayloadType[4] = append(store.byPayloadType[4], tx) + } -info := store.GetNodeHashSizeInfo() -ni := info[pk] -if ni == nil { -t.Fatal("expected hash info for test node") -} -if len(ni.AllSizes) != 2 { -t.Errorf("expected 2 unique sizes, got %d", len(ni.AllSizes)) -} -if !ni.Inconsistent { -t.Error("expected inconsistent flag to be true for flip-flop pattern") -} + info := store.GetNodeHashSizeInfo() + ni := info[pk] + if ni == nil { + t.Fatal("expected hash info for test node") + } + if len(ni.AllSizes) != 2 { + t.Errorf("expected 2 unique sizes, got %d", len(ni.AllSizes)) + } + if !ni.Inconsistent { + t.Error("expected inconsistent flag to be true for flip-flop pattern") + } } func TestGetNodeHashSizeInfoDominant(t *testing.T) { -// A node with mostly 2-byte adverts and an occasional 1-byte advert; the -// latest advert (2-byte) determines the reported hash size. -db := setupTestDB(t) -seedTestData(t, db) -store := NewPacketStore(db, nil) -if err := store.Load(); err != nil { - t.Fatalf("store.Load failed: %v", err) -} -store.WaitIndexesReady(5 * time.Second) - -pk := "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" -db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role) VALUES (?, 'Repeater2B', 'repeater')", pk) - -decoded := `{"name":"Repeater2B","pubKey":"` + pk + `"}` -raw1byte := "11" + "01" + "aabb" // FLOOD, pathByte=0x01 → hashSize=1 -raw2byte := "11" + "41" + "aabb" // FLOOD, pathByte=0x41 → hashSize=2 - -payloadType := 4 -// 1 packet with hashSize=1, 4 packets with hashSize=2 (latest is 2-byte) -raws := []string{raw1byte, raw2byte, raw2byte, raw2byte, raw2byte} -for i, raw := range raws { - tx := &StoreTx{ - ID: 8000 + i, - RawHex: raw, - Hash: "dominant" + strconv.Itoa(i), - FirstSeen: time.Now().UTC().Format("2006-01-02T15:04:05.000Z"), - PayloadType: &payloadType, - DecodedJSON: decoded, + // A node with mostly 2-byte adverts and an occasional 1-byte advert; the + // latest advert (2-byte) determines the reported hash size. + db := setupTestDB(t) + seedTestData(t, db) + store := NewPacketStore(db, nil) + if err := store.Load(); err != nil { + t.Fatalf("store.Load failed: %v", err) } - store.packets = append(store.packets, tx) - store.byPayloadType[4] = append(store.byPayloadType[4], tx) -} + store.WaitIndexesReady(5 * time.Second) -info := store.GetNodeHashSizeInfo() -ni := info[pk] -if ni == nil { - t.Fatal("expected hash info for test node") -} -if ni.HashSize != 2 { - t.Errorf("HashSize=%d, want 2 (latest advert should determine hash size)", ni.HashSize) -} + pk := "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role) VALUES (?, 'Repeater2B', 'repeater')", pk) + + decoded := `{"name":"Repeater2B","pubKey":"` + pk + `"}` + raw1byte := "11" + "01" + "aabb" // FLOOD, pathByte=0x01 → hashSize=1 + raw2byte := "11" + "41" + "aabb" // FLOOD, pathByte=0x41 → hashSize=2 + + payloadType := 4 + // 1 packet with hashSize=1, 4 packets with hashSize=2 (latest is 2-byte) + raws := []string{raw1byte, raw2byte, raw2byte, raw2byte, raw2byte} + for i, raw := range raws { + tx := &StoreTx{ + ID: 8000 + i, + RawHex: raw, + Hash: "dominant" + strconv.Itoa(i), + FirstSeen: time.Now().UTC().Format("2006-01-02T15:04:05.000Z"), + PayloadType: &payloadType, + DecodedJSON: decoded, + } + store.packets = append(store.packets, tx) + store.byPayloadType[4] = append(store.byPayloadType[4], tx) + } + + info := store.GetNodeHashSizeInfo() + ni := info[pk] + if ni == nil { + t.Fatal("expected hash info for test node") + } + if ni.HashSize != 2 { + t.Errorf("HashSize=%d, want 2 (latest advert should determine hash size)", ni.HashSize) + } } func TestGetNodeHashSizeInfoLatestWins(t *testing.T) { @@ -2666,23 +2664,23 @@ func TestAnalyticsHashSizeSameNameDifferentPubkey(t *testing.T) { } func TestAnalyticsHashSizesNoNullArrays(t *testing.T) { -_, router := setupTestServer(t) -req := httptest.NewRequest("GET", "/api/analytics/hash-sizes", nil) -w := httptest.NewRecorder() -router.ServeHTTP(w, req) + _, router := setupTestServer(t) + req := httptest.NewRequest("GET", "/api/analytics/hash-sizes", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) -if w.Code != 200 { -t.Fatalf("expected 200, got %d", w.Code) -} + if w.Code != 200 { + t.Fatalf("expected 200, got %d", w.Code) + } -var body map[string]interface{} -json.Unmarshal(w.Body.Bytes(), &body) + var body map[string]interface{} + json.Unmarshal(w.Body.Bytes(), &body) -arrayFields := []string{"hourly", "topHops", "multiByteNodes"} -for _, field := range arrayFields { -if body[field] == nil { -t.Errorf("field %q is null, expected []", field) -} + arrayFields := []string{"hourly", "topHops", "multiByteNodes"} + for _, field := range arrayFields { + if body[field] == nil { + t.Errorf("field %q is null, expected []", field) + } } } func TestInconsistentNodesExcludesCompanions(t *testing.T) { @@ -3620,7 +3618,7 @@ func TestHashCollisionsOnlyRepeaters(t *testing.T) { store.hashSizeInfoCache = map[string]*hashSizeNodeInfo{ "aa11223344556677": {HashSize: 1, AllSizes: map[int]bool{1: true}}, "aa00112233445566": {HashSize: 1, AllSizes: map[int]bool{1: true}}, - "aa99887766554433": {HashSize: 0, AllSizes: map[int]bool{}}, // unknown + "aa99887766554433": {HashSize: 0, AllSizes: map[int]bool{}}, // unknown "aadeadbeefcafe01": {HashSize: 1, AllSizes: map[int]bool{1: true}}, // companion "aabbcc1122334455": {HashSize: 1, AllSizes: map[int]bool{1: true}}, // room "aabbcc9988776655": {HashSize: 1, AllSizes: map[int]bool{1: true}}, // sensor @@ -4168,7 +4166,7 @@ func TestHandleScopeStats(t *testing.T) { }{ {"h1", "#belgium", 0}, {"h2", "#belgium", 3}, - {"h3", "", 0}, // transport-scoped, no region match + {"h3", "", 0}, // transport-scoped, no region match {"h4_null", "", 0}, // will be inserted with NULL scope_name } for i, r := range rows { @@ -4688,4 +4686,35 @@ func TestGetNodesForGeoPrune(t *testing.T) { // TestDeleteNodesByPubkeys was removed in PR #738 follow-up: the DELETE has // been relocated to the ingestor (cmd/ingestor/prune_geofilter.go). End-to-end -// coverage of the prune flow now lives in cmd/ingestor/*_test.go. +// TestListLimitsConfigurable verifies that list limits are driven by the configuration. +func TestListLimitsConfigurable(t *testing.T) { + srv, router := setupTestServer(t) + + // Inject a custom config + srv.cfg.ListLimits = &ListLimitsConfig{ + PacketsMax: 1234, + } + + // Request with a limit larger than the configured max + req := httptest.NewRequest("GET", "/api/packets?limit=999999", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + if w.Code != 200 { + t.Fatalf("expected 200, got %d", w.Code) + } + + var body map[string]interface{} + if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil { + t.Fatalf("json decode: %v", err) + } + + limit, ok := body["limit"].(float64) + if !ok { + t.Fatal("expected limit field in response") + } + + if limit != 1234 { + t.Errorf("expected limit to be capped at 1234, got %v", limit) + } +} diff --git a/config.example.json b/config.example.json index 55da9ad0..ba717af2 100644 --- a/config.example.json +++ b/config.example.json @@ -17,6 +17,14 @@ "_comment": "vacuumOnStartup: run one-time full VACUUM to enable incremental auto-vacuum on existing DBs. Executed by the INGESTOR at startup, BEFORE the MQTT subscriber starts (#1283), so there is no contention with concurrent writes. Blocks ingestor startup for minutes on large DBs; requires 2x DB file size in free disk space. incrementalVacuumPages: free pages returned to OS after each retention reaper cycle (default 1024). See #919.", "_comment_slowWriterMs": "#1340 — SQLite writer-lock log threshold (default 500). Any wrapped writer call (tagged neighbor_builder, mqtt_handler, prune_packets, prune_observers, prune_metrics, vacuum) whose hold_ms exceeds this emits a single [db-slow-writer] log line. Configured per-process via the CORESCOPE_DB_SLOW_WRITER_MS environment variable on the INGESTOR (e.g. CORESCOPE_DB_SLOW_WRITER_MS=200 for tighter alerting). Per-component wait_ms / hold_ms / contention_total histograms are surfaced via /api/perf/write-sources under .writer_perf regardless of this threshold." }, + "listLimits": { + "packetsMax": 10000, + "nodesMax": 2000, + "analyticsMax": 200, + "channelMessagesMax": 500, + "bulkHealthMax": 200, + "_comment": "Maximum row counts returned by list API endpoints. These enforce a DoS-bounded ceiling for both UI and external requests. Operators with small/embedded deployments can tighten these; operators running large regional meshes can raise them. bulkHealthMax is intentionally separate from nodesMax: /api/nodes/bulk-health is per-row much heavier than /api/nodes (it joins per-node observer health + recent-packet stats) so its ceiling stays low (default 200) even if nodesMax is raised." + }, "_comment_ingestorStats": "Ingestor publishes a 1-Hz stats snapshot consumed by the server's /api/perf/io and /api/perf/write-sources endpoints (#1120). Path is configured via the CORESCOPE_INGESTOR_STATS environment variable on the INGESTOR process. Default: /tmp/corescope-ingestor-stats.json. The writer uses O_NOFOLLOW + 0o600, so a pre-planted symlink in /tmp cannot be used to clobber an arbitrary file. SECURITY: in shared-tmp environments (multi-tenant hosts), point CORESCOPE_INGESTOR_STATS at a private directory like /var/lib/corescope/ingestor-stats.json that only the corescope user can write to.", "corsAllowedOrigins": [], "_comment_corsAllowedOrigins": "Cross-origin allowlist for embed scenarios (#1369). Exact-match origins, e.g. [\"https://blog.example.com\", \"https://embed.example.com\"]. When empty (default), no Access-Control-* headers are sent and browsers enforce same-origin. When non-empty, only the listed origins receive CORS headers, and Access-Control-Allow-Methods is limited to GET, HEAD, OPTIONS (the cross-domain surface is read-only — same-origin admin writes are unaffected). Use [\"*\"] to allow any origin (NOT recommended for write-capable deployments). Operators can override per-deployment with the CORS_ALLOWED_ORIGINS environment variable (comma-separated). No credentialed CORS is enabled. To embed the map or channels pages cross-domain, add the embedding origin here and use the URL pattern '/#/map?embed=1' or '/#/channels?embed=1' — embed mode hides the top-nav, bottom-nav, and side drawer for full-bleed iframe rendering.",