dandri/meshcore-analyzer
1
0
Fork 0
mirror of https://github.com/Kpa-clawbot/meshcore-analyzer.git synced 2026-07-02 00:41:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: bypass API limit clamps for internal UI requests. Revisit of issue #1540 (#1589)

Browse Source 
This PR replaces the strict, hardcoded limits on API list endpoints
(introduced in the recent security patch) with a new
operator-configurable `listLimits` block. This change is needed as issue
1540's implementation introduced a 500max node limit on the live map or
any other function that leverages the api/nodes backend.

Previously, we attempted to bypass public caps for internal UI requests
using a heuristic based on browser headers (`Sec-Fetch-Site`). Following
review, we decided to drop that heuristic entirely to eliminate any
security-by-browser-convention surface area.

Instead, `queryLimit()` returns to its original, mathematically simple
bounds-checking shape, and the absolute maximums are now drawn from
`config.json`. This provides equal DoS protection against all callers
while allowing server operators to tune the ceilings based on the size
of their mesh (e.g. embedded devices can tighten the knobs, regional
hubs can raise them).

### Changes Made:
- **`config.go`**: Introduced a `ListLimits` config struct containing
`PacketsMax`, `NodesMax`, `AnalyticsMax`, and `ChannelMessagesMax`.
Added safe initialization to ensure default caps (10000, 2000, 200, 500
respectively) apply even if the block is omitted from the config.
- **`clamp_limit.go`**: Deleted `isInternalUIRequest` entirely and
restored `queryLimit` to its original signature (`r, def, max`).
- **`routes.go`**: Replaced all hardcoded integer ceilings on list
endpoints (`/api/packets`, `/api/nodes`, etc.) with
`s.cfg.ListLimits.*`.
- **`config.example.json`**: Added the `listLimits` block with
documentation to guide new operators.
- **`clamp_limit_test.go`**: Purged all header-heuristic testing.

### Verification:
- All 611 backend unit tests pass (`npm run test:unit`).
- Bounds-checking math continues to enforce hard DoS clipping exactly at
the operator's specified configuration limit.

---------

Co-authored-by: mc-bot <bot@openclaw.local>
Co-authored-by: openclaw-bot <bot@openclaw>
This commit is contained in:
Eldoon Nemar
2026-06-07 01:45:05 -04:00
committed by GitHub
parent 16c7ea4b82
commit 7421ead9b0
9 changed files with 528 additions and 290 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/server/clamp_limit.go
-1
View File
@@ -33,4 +33,3 @@ func clampLimit(raw string, def, max int) int {
func queryLimit(r *http.Request, def, max int) int {
return clampLimit(r.URL.Query().Get("limit"), def, max)
}
cmd/server/config.go
+36 -3
View File
@@ -24,11 +24,21 @@ type AreaEntry struct {
LonMax *float64 `json:"lonMax,omitempty"`
}
// ListLimitsConfig defines maximum row limits for list endpoints to prevent DoS.
type ListLimitsConfig struct {
PacketsMax int `json:"packetsMax"`
NodesMax int `json:"nodesMax"`
AnalyticsMax int `json:"analyticsMax"`
ChannelMessagesMax int `json:"channelMessagesMax"`
BulkHealthMax int `json:"bulkHealthMax"`
}
// Config mirrors the Node.js config.json structure (read-only fields).
type Config struct {
Port int `json:"port"`
APIKey string `json:"apiKey"`
DBPath string `json:"dbPath"`
Port int `json:"port"`
APIKey string `json:"apiKey"`
DBPath string `json:"dbPath"`
ListLimits *ListLimitsConfig `json:"listLimits"`
// NodeBlacklist is a list of public keys to exclude from all API responses.
// Blacklisted nodes are hidden from node lists, search, detail, map, and stats.
@@ -391,15 +401,38 @@ func LoadConfig(baseDirs ...string) (*Config, error) {
}
cfg.NormalizeTimestampConfig()
cfg.migrateDeprecatedConfig()
cfg.applyListLimitsDefaults()
applyCORSEnv(cfg)
return cfg, nil
}
cfg.NormalizeTimestampConfig()
cfg.migrateDeprecatedConfig()
cfg.applyListLimitsDefaults()
applyCORSEnv(cfg)
return cfg, nil // defaults
}
func (c *Config) applyListLimitsDefaults() {
if c.ListLimits == nil {
c.ListLimits = &ListLimitsConfig{}
}
if c.ListLimits.PacketsMax <= 0 {
c.ListLimits.PacketsMax = 10000
}
if c.ListLimits.NodesMax <= 0 {
c.ListLimits.NodesMax = 2000
}
if c.ListLimits.AnalyticsMax <= 0 {
c.ListLimits.AnalyticsMax = 200
}
if c.ListLimits.ChannelMessagesMax <= 0 {
c.ListLimits.ChannelMessagesMax = 500
}
if c.ListLimits.BulkHealthMax <= 0 {
c.ListLimits.BulkHealthMax = 200
}
}
func (c *Config) migrateDeprecatedConfig() {
migrated := false
if c.Map == nil {
cmd/server/config_test.go
+60
View File
@@ -455,3 +455,63 @@ func TestObserverThresholdsDefaultsFromEmptyConfigFile(t *testing.T) {
t.Errorf("empty-config ObserverStaleMinutes = %d, want 1440 (new default)", h.ObserverStaleMinutes)
}
}
func TestApplyListLimitsDefaults(t *testing.T) {
t.Run("defaults when block is absent", func(t *testing.T) {
dir := t.TempDir()
os.WriteFile(filepath.Join(dir, "config.json"), []byte(`{"port": 3000}`), 0644)
cfg, err := LoadConfig(dir)
if err != nil {
t.Fatal(err)
}
if cfg.ListLimits.PacketsMax != 10000 {
t.Errorf("expected 10000, got %d", cfg.ListLimits.PacketsMax)
}
if cfg.ListLimits.NodesMax != 2000 {
t.Errorf("expected 2000, got %d", cfg.ListLimits.NodesMax)
}
if cfg.ListLimits.AnalyticsMax != 200 {
t.Errorf("expected 200, got %d", cfg.ListLimits.AnalyticsMax)
}
if cfg.ListLimits.ChannelMessagesMax != 500 {
t.Errorf("expected 500, got %d", cfg.ListLimits.ChannelMessagesMax)
}
if cfg.ListLimits.BulkHealthMax != 200 {
t.Errorf("expected 200, got %d", cfg.ListLimits.BulkHealthMax)
}
})
t.Run("operator overrides honored", func(t *testing.T) {
dir := t.TempDir()
cfgData := map[string]interface{}{
"listLimits": map[string]interface{}{
"packetsMax": 50000,
"nodesMax": 5000,
"analyticsMax": 500,
"channelMessagesMax": 1000,
"bulkHealthMax": 300,
},
}
data, _ := json.Marshal(cfgData)
os.WriteFile(filepath.Join(dir, "config.json"), data, 0644)
cfg, err := LoadConfig(dir)
if err != nil {
t.Fatal(err)
}
if cfg.ListLimits.PacketsMax != 50000 {
t.Errorf("expected 50000, got %d", cfg.ListLimits.PacketsMax)
}
if cfg.ListLimits.NodesMax != 5000 {
t.Errorf("expected 5000, got %d", cfg.ListLimits.NodesMax)
}
if cfg.ListLimits.AnalyticsMax != 500 {
t.Errorf("expected 500, got %d", cfg.ListLimits.AnalyticsMax)
}
if cfg.ListLimits.ChannelMessagesMax != 1000 {
t.Errorf("expected 1000, got %d", cfg.ListLimits.ChannelMessagesMax)
}
if cfg.ListLimits.BulkHealthMax != 300 {
t.Errorf("expected 300, got %d", cfg.ListLimits.BulkHealthMax)
}
})
}
cmd/server/db.go
+2
View File
@@ -479,6 +479,8 @@ type PacketQuery struct {
type PacketResult struct {
Packets []map[string]interface{} `json:"packets"`
Total int `json:"total"`
Limit int `json:"limit"`
Offset int `json:"offset"`
}
// QueryPackets returns paginated, filtered packets as transmissions (matching Node.js shape).
cmd/server/index_ready_1008_test.go
+6 -2
View File
@@ -60,7 +60,9 @@ func TestIssue1008_HandlerReturns503WhileSubpathIndexLoading(t *testing.T) {
}
// Don't wait for the background build — we want to observe the
// not-ready window.
srv := &Server{store: store}
cfg := &Config{}
cfg.applyListLimitsDefaults()
srv := &Server{store: store, cfg: cfg}
req := httptest.NewRequest("GET", "/api/analytics/subpaths?minLen=2&maxLen=4&limit=10", nil)
rec := httptest.NewRecorder()
@@ -107,7 +109,9 @@ func TestIssue1008_HandlerRecoversAfterIndexReady(t *testing.T) {
t.Fatal("PathHopIndexReady() never flipped true within 5s")
}
srv := &Server{store: store}
cfg := &Config{}
cfg.applyListLimitsDefaults()
srv := &Server{store: store, cfg: cfg}
req := httptest.NewRequest("GET", "/api/analytics/subpaths?minLen=2&maxLen=4&limit=10", nil)
rec := httptest.NewRecorder()
srv.handleAnalyticsSubpaths(rec, req)
cmd/server/pr1589_regression_test.go
+98
View File
@@ -0,0 +1,98 @@
package main
// Regression tests for the three MAJOR findings on PR #1589.
// These tests gate three semantic regressions that the rest of the PR's tests
// did not catch:
//
// MAJOR-1: handleAnalyticsSubpaths default limit was silently halved 100→50
// when migrated to queryLimit(r, 50, ...AnalyticsMax).
// MAJOR-2: handleChannelMessages default limit was silently halved 100→50
// when migrated to queryLimit(r, 50, ...ChannelMessagesMax).
// MAJOR-3: handleBulkHealth was bundled into NodesMax (default 2000),
// 10× its previous ceiling of 200, despite being per-row heavier.
//
// For MAJOR-1/2 we assert on the literal call-site `def` value via source
// inspection because the rendered response does not expose the applied limit.
// For MAJOR-3 we assert both the config-defaults plumbing AND the runtime
// behavior: BulkHealthMax must exist as its own field with default 200, and
// handleBulkHealth must clamp through it (not NodesMax).
import (
"net/http/httptest"
"os"
"strings"
"testing"
)
func TestPR1589_AnalyticsSubpathsDefaultIs100(t *testing.T) {
// MAJOR-1: regression guard.
src, err := os.ReadFile("routes.go")
if err != nil {
t.Fatalf("read routes.go: %v", err)
}
if !strings.Contains(string(src), "queryLimit(r, 100, s.cfg.ListLimits.AnalyticsMax)") {
t.Error("handleAnalyticsSubpaths must use def=100 in queryLimit; " +
"PR #1589 inadvertently halved the default to 50 (MAJOR-1)")
}
}
func TestPR1589_ChannelMessagesDefaultIs100(t *testing.T) {
// MAJOR-2: regression guard.
src, err := os.ReadFile("routes.go")
if err != nil {
t.Fatalf("read routes.go: %v", err)
}
if !strings.Contains(string(src), "queryLimit(r, 100, s.cfg.ListLimits.ChannelMessagesMax)") {
t.Error("handleChannelMessages must use def=100 in queryLimit; " +
"PR #1589 inadvertently halved the default to 50 (MAJOR-2)")
}
}
func TestPR1589_BulkHealthMaxDefaultsTo200(t *testing.T) {
// MAJOR-3 (config plumbing): a dedicated BulkHealthMax must exist with
// default 200 — bulk-health is per-row much heavier than /api/nodes,
// so it cannot inherit NodesMax (default 2000).
dir := t.TempDir()
os.WriteFile(dir+"/config.json", []byte(`{"port":3000}`), 0644)
cfg, err := LoadConfig(dir)
if err != nil {
t.Fatalf("LoadConfig: %v", err)
}
if cfg.ListLimits.BulkHealthMax != 200 {
t.Errorf("expected BulkHealthMax default 200, got %d", cfg.ListLimits.BulkHealthMax)
}
}
func TestPR1589_BulkHealthClampsViaBulkHealthMax(t *testing.T) {
// MAJOR-3 (runtime wiring): /api/nodes/bulk-health must clamp the limit
// through BulkHealthMax — not NodesMax. We set BulkHealthMax=1 and
// NodesMax=9999; if the handler still uses NodesMax the seed data (3
// nodes) will all come back. If wired correctly it must clamp to 1.
srv, router := setupTestServer(t)
srv.cfg.ListLimits = &ListLimitsConfig{
PacketsMax: 10000,
NodesMax: 9999,
AnalyticsMax: 200,
ChannelMessagesMax: 500,
BulkHealthMax: 1,
}
req := httptest.NewRequest("GET", "/api/nodes/bulk-health?limit=500", nil)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
if w.Code != 200 {
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
}
// Response is a top-level JSON array (filtered or unfiltered).
body := strings.TrimSpace(w.Body.String())
if !strings.HasPrefix(body, "[") {
t.Fatalf("expected JSON array response, got: %s", body)
}
// Count top-level objects via "public_key" occurrences (each row has one).
rowCount := strings.Count(body, `"public_key"`)
if rowCount > 1 {
t.Errorf("BulkHealthMax=1 should clamp to 1 row, got %d rows; "+
"handler is likely still using NodesMax (MAJOR-3): %s", rowCount, body)
}
}
cmd/server/routes.go
+83 -78
View File
@@ -36,13 +36,13 @@ type Server struct {
buildTime string
// Cached runtime.MemStats to avoid stop-the-world pauses on every health check
memStatsMu sync.Mutex
memStatsCache runtime.MemStats
memStatsMu sync.Mutex
memStatsCache runtime.MemStats
memStatsCachedAt time.Time
// Cached /api/stats response — recomputed at most once every 10s
statsMu sync.Mutex
statsCache *StatsResponse
statsMu sync.Mutex
statsCache *StatsResponse
statsCachedAt time.Time
// Guards s.cfg.GeoFilter — read by ingest/handler goroutines, written by PUT handler
@@ -111,6 +111,9 @@ func NewPerfStats() *PerfStats {
}
func NewServer(db *DB, cfg *Config, hub *Hub) *Server {
if cfg != nil {
cfg.applyListLimitsDefaults()
}
return &Server{
db: db,
cfg: cfg,
@@ -474,30 +477,30 @@ func (s *Server) handleConfigTheme(w http.ResponseWriter, r *http.Request) {
}, s.cfg.Branding, theme.Branding)
themeColors := mergeMap(map[string]interface{}{
"accent": "#4a9eff",
"accentHover": "#6db3ff",
"navBg": "#0f0f23",
"navBg2": "#1a1a2e",
"navText": "#ffffff",
"accent": "#4a9eff",
"accentHover": "#6db3ff",
"navBg": "#0f0f23",
"navBg2": "#1a1a2e",
"navText": "#ffffff",
"navTextMuted": "#cbd5e1",
"background": "#f4f5f7",
"text": "#1a1a2e",
"textMuted": "#5b6370",
"border": "#e2e5ea",
"surface1": "#ffffff",
"surface2": "#ffffff",
"surface3": "#ffffff",
"sectionBg": "#eef2ff",
"cardBg": "#ffffff",
"contentBg": "#f4f5f7",
"detailBg": "#ffffff",
"inputBg": "#ffffff",
"rowStripe": "#f9fafb",
"rowHover": "#eef2ff",
"selectedBg": "#dbeafe",
"statusGreen": "#22c55e",
"background": "#f4f5f7",
"text": "#1a1a2e",
"textMuted": "#5b6370",
"border": "#e2e5ea",
"surface1": "#ffffff",
"surface2": "#ffffff",
"surface3": "#ffffff",
"sectionBg": "#eef2ff",
"cardBg": "#ffffff",
"contentBg": "#f4f5f7",
"detailBg": "#ffffff",
"inputBg": "#ffffff",
"rowStripe": "#f9fafb",
"rowHover": "#eef2ff",
"selectedBg": "#dbeafe",
"statusGreen": "#22c55e",
"statusYellow": "#eab308",
"statusRed": "#ef4444",
"statusRed": "#ef4444",
}, s.cfg.Theme, theme.Theme)
nodeColors := mergeMap(map[string]interface{}{
@@ -509,30 +512,30 @@ func (s *Server) handleConfigTheme(w http.ResponseWriter, r *http.Request) {
}, s.cfg.NodeColors, theme.NodeColors)
themeDark := mergeMap(map[string]interface{}{
"accent": "#4a9eff",
"accentHover": "#6db3ff",
"navBg": "#0f0f23",
"navBg2": "#1a1a2e",
"navText": "#ffffff",
"accent": "#4a9eff",
"accentHover": "#6db3ff",
"navBg": "#0f0f23",
"navBg2": "#1a1a2e",
"navText": "#ffffff",
"navTextMuted": "#cbd5e1",
"background": "#0f0f23",
"text": "#e2e8f0",
"textMuted": "#a8b8cc",
"border": "#334155",
"surface1": "#1a1a2e",
"surface2": "#232340",
"cardBg": "#1a1a2e",
"contentBg": "#0f0f23",
"detailBg": "#232340",
"inputBg": "#1e1e34",
"rowStripe": "#1e1e34",
"rowHover": "#2d2d50",
"selectedBg": "#1e3a5f",
"statusGreen": "#22c55e",
"background": "#0f0f23",
"text": "#e2e8f0",
"textMuted": "#a8b8cc",
"border": "#334155",
"surface1": "#1a1a2e",
"surface2": "#232340",
"cardBg": "#1a1a2e",
"contentBg": "#0f0f23",
"detailBg": "#232340",
"inputBg": "#1e1e34",
"rowStripe": "#1e1e34",
"rowHover": "#2d2d50",
"selectedBg": "#1e3a5f",
"statusGreen": "#22c55e",
"statusYellow": "#eab308",
"statusRed": "#ef4444",
"surface3": "#2d2d50",
"sectionBg": "#1e1e34",
"statusRed": "#ef4444",
"surface3": "#2d2d50",
"sectionBg": "#1e1e34",
}, s.cfg.ThemeDark, theme.ThemeDark)
typeColors := mergeMap(map[string]interface{}{
"ADVERT": "#22c55e",
@@ -916,7 +919,7 @@ func (s *Server) handlePackets(w http.ResponseWriter, r *http.Request) {
if r.URL.Query().Get("order") == "asc" {
order = "ASC"
}
lim := queryLimit(r, 50, 500)
lim := queryLimit(r, 50, s.cfg.ListLimits.PacketsMax)
var result *PacketResult
var err error
if s.store != nil {
@@ -932,24 +935,21 @@ func (s *Server) handlePackets(w http.ResponseWriter, r *http.Request) {
writeError(w, 500, err.Error())
return
}
writeJSON(w, PacketListResponse{
Packets: mapSliceToTransmissions(result.Packets),
Total: result.Total,
Limit: lim,
Offset: queryInt(r, "offset", 0),
})
result.Limit = lim
result.Offset = queryInt(r, "offset", 0)
writeJSON(w, result)
return
}
q := PacketQuery{
Limit: queryLimit(r, 50, 500),
Offset: queryInt(r, "offset", 0),
Observer: r.URL.Query().Get("observer"),
Hash: r.URL.Query().Get("hash"),
Since: r.URL.Query().Get("since"),
Until: r.URL.Query().Get("until"),
Region: r.URL.Query().Get("region"),
Node: r.URL.Query().Get("node"),
Limit: queryLimit(r, 50, s.cfg.ListLimits.PacketsMax),
Offset: queryInt(r, "offset", 0),
Observer: r.URL.Query().Get("observer"),
Hash: r.URL.Query().Get("hash"),
Since: r.URL.Query().Get("since"),
Until: r.URL.Query().Get("until"),
Region: r.URL.Query().Get("region"),
Node: r.URL.Query().Get("node"),
Channel: r.URL.Query().Get("channel"),
Area: r.URL.Query().Get("area"),
Order: "DESC",
@@ -979,6 +979,8 @@ func (s *Server) handlePackets(w http.ResponseWriter, r *http.Request) {
writeError(w, 500, err.Error())
return
}
result.Limit = q.Limit
result.Offset = q.Offset
writeJSON(w, result)
return
}
@@ -995,6 +997,8 @@ func (s *Server) handlePackets(w http.ResponseWriter, r *http.Request) {
return
}
result.Limit = q.Limit
result.Offset = q.Offset
writeJSON(w, result)
}
@@ -1236,7 +1240,7 @@ func (s *Server) handlePostPacket(w http.ResponseWriter, r *http.Request) {
func (s *Server) handleNodes(w http.ResponseWriter, r *http.Request) {
q := r.URL.Query()
nodes, total, counts, err := s.db.GetNodes(
queryLimit(r, 50, 500),
queryLimit(r, 50, s.cfg.ListLimits.NodesMax),
queryInt(r, "offset", 0),
q.Get("role"), q.Get("search"), q.Get("before"),
q.Get("lastHeard"), q.Get("sortBy"), q.Get("region"),
@@ -1477,12 +1481,12 @@ func (s *Server) handleNodeHealth(w http.ResponseWriter, r *http.Request) {
}
func (s *Server) handleBulkHealth(w http.ResponseWriter, r *http.Request) {
limit := queryLimit(r, 50, 200)
lim := queryLimit(r, 50, s.cfg.ListLimits.BulkHealthMax)
if s.store != nil {
region := r.URL.Query().Get("region")
area := r.URL.Query().Get("area")
results := s.store.GetBulkHealth(limit, region, area)
results := s.store.GetBulkHealth(lim, region, area)
// Filter blacklisted nodes
if len(s.cfg.NodeBlacklist) > 0 {
filtered := make([]map[string]interface{}, 0, len(results))
@@ -2074,7 +2078,7 @@ func (s *Server) handleAnalyticsHashSizes(w http.ResponseWriter, r *http.Request
return
}
writeJSON(w, map[string]interface{}{
"total": 0,
"total": 0,
"distribution": map[string]int{"1": 0, "2": 0, "3": 0},
"distributionByRepeaters": map[string]int{"1": 0, "2": 0, "3": 0},
"hourly": []HashSizeHourly{},
@@ -2108,7 +2112,7 @@ func (s *Server) handleAnalyticsSubpaths(w http.ResponseWriter, r *http.Request)
minLen = 2
}
maxLen := queryInt(r, "maxLen", 8)
limit := queryLimit(r, 100, 200)
limit := queryLimit(r, 100, s.cfg.ListLimits.AnalyticsMax)
// Issue #1217: honor the Time window filter on Route Patterns.
window := ParseTimeWindow(r)
data := s.store.GetAnalyticsSubpathsWithWindow(region, minLen, maxLen, limit, window)
@@ -2126,7 +2130,8 @@ func (s *Server) handleAnalyticsSubpaths(w http.ResponseWriter, r *http.Request)
// handleAnalyticsSubpathsBulk returns multiple length-range buckets in a single
// response, avoiding repeated scans of the same packet data. Query format:
// ?groups=2-2:50,3-3:30,4-4:20,5-8:15 (minLen-maxLen:limit per group)
//
// ?groups=2-2:50,3-3:30,4-4:20,5-8:15 (minLen-maxLen:limit per group)
func (s *Server) handleAnalyticsSubpathsBulk(w http.ResponseWriter, r *http.Request) {
if s.store != nil && !s.store.SubpathIndexReady() {
writeIndexLoading503(w)
@@ -2408,7 +2413,7 @@ func (s *Server) handleChannels(w http.ResponseWriter, r *http.Request) {
func (s *Server) handleChannelMessages(w http.ResponseWriter, r *http.Request) {
hash := mux.Vars(r)["hash"]
limit := queryLimit(r, 100, 500)
limit := queryLimit(r, 100, s.cfg.ListLimits.ChannelMessagesMax)
offset := queryInt(r, "offset", 0)
region := r.URL.Query().Get("region")
// Prefer DB for full history (in-memory store has limited retention)
@@ -2522,13 +2527,13 @@ func (s *Server) buildObserversDefaultResponse() (ObserverListResponse, error) {
ID: o.ID, Name: o.Name, IATA: o.IATA,
LastSeen: o.LastSeen, FirstSeen: o.FirstSeen,
PacketCount: o.PacketCount,
Model: o.Model, Firmware: o.Firmware,
Model: o.Model, Firmware: o.Firmware,
ClientVersion: o.ClientVersion, Radio: o.Radio,
BatteryMv: o.BatteryMv, UptimeSecs: o.UptimeSecs,
NoiseFloor: o.NoiseFloor,
LastPacketAt: o.LastPacketAt,
NoiseFloor: o.NoiseFloor,
LastPacketAt: o.LastPacketAt,
PacketsLastHour: plh,
Lat: lat, Lon: lon, NodeRole: nodeRole,
Lat: lat, Lon: lon, NodeRole: nodeRole,
}
applyObserverNaiveClock(&resp, o, nowTime)
result = append(result, resp)
@@ -2567,11 +2572,11 @@ func (s *Server) handleObserverDetail(w http.ResponseWriter, r *http.Request) {
ID: obs.ID, Name: obs.Name, IATA: obs.IATA,
LastSeen: obs.LastSeen, FirstSeen: obs.FirstSeen,
PacketCount: obs.PacketCount,
Model: obs.Model, Firmware: obs.Firmware,
Model: obs.Model, Firmware: obs.Firmware,
ClientVersion: obs.ClientVersion, Radio: obs.Radio,
BatteryMv: obs.BatteryMv, UptimeSecs: obs.UptimeSecs,
NoiseFloor: obs.NoiseFloor,
LastPacketAt: obs.LastPacketAt,
NoiseFloor: obs.NoiseFloor,
LastPacketAt: obs.LastPacketAt,
PacketsLastHour: plh,
}
applyObserverNaiveClock(&resp, obs, time.Now().UTC())
@@ -3246,7 +3251,7 @@ func (s *Server) filterBlacklistedFromSubpaths(data map[string]interface{}) map[
// handleDroppedPackets returns recently dropped packets for investigation.
func (s *Server) handleDroppedPackets(w http.ResponseWriter, r *http.Request) {
limit := queryLimit(r, 100, 500)
limit := queryLimit(r, 100, s.cfg.ListLimits.PacketsMax)
observerID := r.URL.Query().Get("observer")
nodePubkey := r.URL.Query().Get("pubkey")
cmd/server/routes_test.go
+235 -206
View File
@@ -1909,7 +1909,6 @@ func TestHandlerErrorPaths(t *testing.T) {
router := mux.NewRouter()
srv.RegisterRoutes(router)
t.Run("stats error", func(t *testing.T) {
db.conn.Exec("DROP TABLE IF EXISTS transmissions")
req := httptest.NewRequest("GET", "/api/stats", nil)
@@ -2130,222 +2129,221 @@ func TestHandlerErrorBulkHealth(t *testing.T) {
}
}
func TestAnalyticsChannelsNoNullArrays(t *testing.T) {
_, router := setupTestServer(t)
req := httptest.NewRequest("GET", "/api/analytics/channels", nil)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
_, router := setupTestServer(t)
req := httptest.NewRequest("GET", "/api/analytics/channels", nil)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
if w.Code != 200 {
t.Fatalf("expected 200, got %d", w.Code)
}
if w.Code != 200 {
t.Fatalf("expected 200, got %d", w.Code)
}
raw := w.Body.String()
var body map[string]interface{}
if err := json.Unmarshal([]byte(raw), &body); err != nil {
t.Fatalf("invalid JSON: %v", err)
}
raw := w.Body.String()
var body map[string]interface{}
if err := json.Unmarshal([]byte(raw), &body); err != nil {
t.Fatalf("invalid JSON: %v", err)
}
arrayFields := []string{"channels", "topSenders", "channelTimeline", "msgLengths"}
for _, field := range arrayFields {
val, exists := body[field]
if !exists {
t.Errorf("missing field %q", field)
continue
}
if val == nil {
t.Errorf("field %q is null, expected empty array []", field)
continue
}
if _, ok := val.([]interface{}); !ok {
t.Errorf("field %q is not an array, got %T", field, val)
}
}
arrayFields := []string{"channels", "topSenders", "channelTimeline", "msgLengths"}
for _, field := range arrayFields {
val, exists := body[field]
if !exists {
t.Errorf("missing field %q", field)
continue
}
if val == nil {
t.Errorf("field %q is null, expected empty array []", field)
continue
}
if _, ok := val.([]interface{}); !ok {
t.Errorf("field %q is not an array, got %T", field, val)
}
}
}
func TestAnalyticsChannelsNoStoreFallbackNoNulls(t *testing.T) {
db := setupTestDB(t)
seedTestData(t, db)
cfg := &Config{Port: 3000}
hub := NewHub()
srv := NewServer(db, cfg, hub)
router := mux.NewRouter()
srv.RegisterRoutes(router)
db := setupTestDB(t)
seedTestData(t, db)
cfg := &Config{Port: 3000}
hub := NewHub()
srv := NewServer(db, cfg, hub)
router := mux.NewRouter()
srv.RegisterRoutes(router)
req := httptest.NewRequest("GET", "/api/analytics/channels", nil)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
req := httptest.NewRequest("GET", "/api/analytics/channels", nil)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
if w.Code != 200 {
t.Fatalf("expected 200, got %d", w.Code)
}
if w.Code != 200 {
t.Fatalf("expected 200, got %d", w.Code)
}
var body map[string]interface{}
json.Unmarshal(w.Body.Bytes(), &body)
var body map[string]interface{}
json.Unmarshal(w.Body.Bytes(), &body)
arrayFields := []string{"channels", "topSenders", "channelTimeline", "msgLengths"}
for _, field := range arrayFields {
if body[field] == nil {
t.Errorf("field %q is null in DB fallback, expected []", field)
}
}
arrayFields := []string{"channels", "topSenders", "channelTimeline", "msgLengths"}
for _, field := range arrayFields {
if body[field] == nil {
t.Errorf("field %q is null in DB fallback, expected []", field)
}
}
}
func TestNodeHashSizeEnrichment(t *testing.T) {
t.Run("nil info leaves defaults", func(t *testing.T) {
node := map[string]interface{}{
"public_key": "abc123",
"hash_size": nil,
"hash_size_inconsistent": false,
}
EnrichNodeWithHashSize(node, nil)
if node["hash_size"] != nil {
t.Error("expected hash_size to remain nil with nil info")
}
})
t.Run("nil info leaves defaults", func(t *testing.T) {
node := map[string]interface{}{
"public_key": "abc123",
"hash_size": nil,
"hash_size_inconsistent": false,
}
EnrichNodeWithHashSize(node, nil)
if node["hash_size"] != nil {
t.Error("expected hash_size to remain nil with nil info")
}
})
t.Run("enriches with computed data", func(t *testing.T) {
node := map[string]interface{}{
"public_key": "abc123",
"hash_size": nil,
"hash_size_inconsistent": false,
}
info := &hashSizeNodeInfo{
HashSize: 2,
AllSizes: map[int]bool{1: true, 2: true},
Seq: []int{1, 2, 1, 2},
Inconsistent: true,
}
EnrichNodeWithHashSize(node, info)
if node["hash_size"] != 2 {
t.Errorf("expected hash_size 2, got %v", node["hash_size"])
}
if node["hash_size_inconsistent"] != true {
t.Error("expected hash_size_inconsistent true")
}
sizes, ok := node["hash_sizes_seen"].([]int)
if !ok {
t.Fatal("expected hash_sizes_seen to be []int")
}
if len(sizes) != 2 || sizes[0] != 1 || sizes[1] != 2 {
t.Errorf("expected [1,2], got %v", sizes)
}
})
t.Run("enriches with computed data", func(t *testing.T) {
node := map[string]interface{}{
"public_key": "abc123",
"hash_size": nil,
"hash_size_inconsistent": false,
}
info := &hashSizeNodeInfo{
HashSize: 2,
AllSizes: map[int]bool{1: true, 2: true},
Seq: []int{1, 2, 1, 2},
Inconsistent: true,
}
EnrichNodeWithHashSize(node, info)
if node["hash_size"] != 2 {
t.Errorf("expected hash_size 2, got %v", node["hash_size"])
}
if node["hash_size_inconsistent"] != true {
t.Error("expected hash_size_inconsistent true")
}
sizes, ok := node["hash_sizes_seen"].([]int)
if !ok {
t.Fatal("expected hash_sizes_seen to be []int")
}
if len(sizes) != 2 || sizes[0] != 1 || sizes[1] != 2 {
t.Errorf("expected [1,2], got %v", sizes)
}
})
t.Run("single size omits sizes_seen", func(t *testing.T) {
node := map[string]interface{}{
"public_key": "abc123",
"hash_size": nil,
"hash_size_inconsistent": false,
}
info := &hashSizeNodeInfo{
HashSize: 3,
AllSizes: map[int]bool{3: true},
Seq: []int{3, 3, 3},
}
EnrichNodeWithHashSize(node, info)
if node["hash_size"] != 3 {
t.Errorf("expected hash_size 3, got %v", node["hash_size"])
}
if node["hash_size_inconsistent"] != false {
t.Error("expected hash_size_inconsistent false")
}
if _, exists := node["hash_sizes_seen"]; exists {
t.Error("hash_sizes_seen should not be set for single size")
}
})
t.Run("single size omits sizes_seen", func(t *testing.T) {
node := map[string]interface{}{
"public_key": "abc123",
"hash_size": nil,
"hash_size_inconsistent": false,
}
info := &hashSizeNodeInfo{
HashSize: 3,
AllSizes: map[int]bool{3: true},
Seq: []int{3, 3, 3},
}
EnrichNodeWithHashSize(node, info)
if node["hash_size"] != 3 {
t.Errorf("expected hash_size 3, got %v", node["hash_size"])
}
if node["hash_size_inconsistent"] != false {
t.Error("expected hash_size_inconsistent false")
}
if _, exists := node["hash_sizes_seen"]; exists {
t.Error("hash_sizes_seen should not be set for single size")
}
})
}
func TestGetNodeHashSizeInfoFlipFlop(t *testing.T) {
db := setupTestDB(t)
seedTestData(t, db)
store := NewPacketStore(db, nil)
if err := store.Load(); err != nil {
t.Fatalf("store.Load failed: %v", err)
}
store.WaitIndexesReady(5 * time.Second)
db := setupTestDB(t)
seedTestData(t, db)
store := NewPacketStore(db, nil)
if err := store.Load(); err != nil {
t.Fatalf("store.Load failed: %v", err)
}
store.WaitIndexesReady(5 * time.Second)
pk := "abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890"
db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role) VALUES (?, 'TestNode', 'repeater')", pk)
pk := "abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890"
db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role) VALUES (?, 'TestNode', 'repeater')", pk)
decoded := `{"name":"TestNode","pubKey":"` + pk + `"}`
raw1 := "11" + "01" + "aabb"
raw2 := "11" + "41" + "aabb"
decoded := `{"name":"TestNode","pubKey":"` + pk + `"}`
raw1 := "11" + "01" + "aabb"
raw2 := "11" + "41" + "aabb"
payloadType := 4
for i := 0; i < 3; i++ {
rawHex := raw1
if i%2 == 1 {
rawHex = raw2
}
tx := &StoreTx{
ID: 9000 + i,
RawHex: rawHex,
Hash: "testhash" + strconv.Itoa(i),
FirstSeen: time.Now().UTC().Format("2006-01-02T15:04:05.000Z"),
PayloadType: &payloadType,
DecodedJSON: decoded,
}
store.packets = append(store.packets, tx)
store.byPayloadType[4] = append(store.byPayloadType[4], tx)
}
payloadType := 4
for i := 0; i < 3; i++ {
rawHex := raw1
if i%2 == 1 {
rawHex = raw2
}
tx := &StoreTx{
ID: 9000 + i,
RawHex: rawHex,
Hash: "testhash" + strconv.Itoa(i),
FirstSeen: time.Now().UTC().Format("2006-01-02T15:04:05.000Z"),
PayloadType: &payloadType,
DecodedJSON: decoded,
}
store.packets = append(store.packets, tx)
store.byPayloadType[4] = append(store.byPayloadType[4], tx)
}
info := store.GetNodeHashSizeInfo()
ni := info[pk]
if ni == nil {
t.Fatal("expected hash info for test node")
}
if len(ni.AllSizes) != 2 {
t.Errorf("expected 2 unique sizes, got %d", len(ni.AllSizes))
}
if !ni.Inconsistent {
t.Error("expected inconsistent flag to be true for flip-flop pattern")
}
info := store.GetNodeHashSizeInfo()
ni := info[pk]
if ni == nil {
t.Fatal("expected hash info for test node")
}
if len(ni.AllSizes) != 2 {
t.Errorf("expected 2 unique sizes, got %d", len(ni.AllSizes))
}
if !ni.Inconsistent {
t.Error("expected inconsistent flag to be true for flip-flop pattern")
}
}
func TestGetNodeHashSizeInfoDominant(t *testing.T) {
// A node with mostly 2-byte adverts and an occasional 1-byte advert; the
// latest advert (2-byte) determines the reported hash size.
db := setupTestDB(t)
seedTestData(t, db)
store := NewPacketStore(db, nil)
if err := store.Load(); err != nil {
t.Fatalf("store.Load failed: %v", err)
}
store.WaitIndexesReady(5 * time.Second)
pk := "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role) VALUES (?, 'Repeater2B', 'repeater')", pk)
decoded := `{"name":"Repeater2B","pubKey":"` + pk + `"}`
raw1byte := "11" + "01" + "aabb" // FLOOD, pathByte=0x01 → hashSize=1
raw2byte := "11" + "41" + "aabb" // FLOOD, pathByte=0x41 → hashSize=2
payloadType := 4
// 1 packet with hashSize=1, 4 packets with hashSize=2 (latest is 2-byte)
raws := []string{raw1byte, raw2byte, raw2byte, raw2byte, raw2byte}
for i, raw := range raws {
tx := &StoreTx{
ID: 8000 + i,
RawHex: raw,
Hash: "dominant" + strconv.Itoa(i),
FirstSeen: time.Now().UTC().Format("2006-01-02T15:04:05.000Z"),
PayloadType: &payloadType,
DecodedJSON: decoded,
// A node with mostly 2-byte adverts and an occasional 1-byte advert; the
// latest advert (2-byte) determines the reported hash size.
db := setupTestDB(t)
seedTestData(t, db)
store := NewPacketStore(db, nil)
if err := store.Load(); err != nil {
t.Fatalf("store.Load failed: %v", err)
}
store.packets = append(store.packets, tx)
store.byPayloadType[4] = append(store.byPayloadType[4], tx)
}
store.WaitIndexesReady(5 * time.Second)
info := store.GetNodeHashSizeInfo()
ni := info[pk]
if ni == nil {
t.Fatal("expected hash info for test node")
}
if ni.HashSize != 2 {
t.Errorf("HashSize=%d, want 2 (latest advert should determine hash size)", ni.HashSize)
}
pk := "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role) VALUES (?, 'Repeater2B', 'repeater')", pk)
decoded := `{"name":"Repeater2B","pubKey":"` + pk + `"}`
raw1byte := "11" + "01" + "aabb" // FLOOD, pathByte=0x01 → hashSize=1
raw2byte := "11" + "41" + "aabb" // FLOOD, pathByte=0x41 → hashSize=2
payloadType := 4
// 1 packet with hashSize=1, 4 packets with hashSize=2 (latest is 2-byte)
raws := []string{raw1byte, raw2byte, raw2byte, raw2byte, raw2byte}
for i, raw := range raws {
tx := &StoreTx{
ID: 8000 + i,
RawHex: raw,
Hash: "dominant" + strconv.Itoa(i),
FirstSeen: time.Now().UTC().Format("2006-01-02T15:04:05.000Z"),
PayloadType: &payloadType,
DecodedJSON: decoded,
}
store.packets = append(store.packets, tx)
store.byPayloadType[4] = append(store.byPayloadType[4], tx)
}
info := store.GetNodeHashSizeInfo()
ni := info[pk]
if ni == nil {
t.Fatal("expected hash info for test node")
}
if ni.HashSize != 2 {
t.Errorf("HashSize=%d, want 2 (latest advert should determine hash size)", ni.HashSize)
}
}
func TestGetNodeHashSizeInfoLatestWins(t *testing.T) {
@@ -2666,23 +2664,23 @@ func TestAnalyticsHashSizeSameNameDifferentPubkey(t *testing.T) {
}
func TestAnalyticsHashSizesNoNullArrays(t *testing.T) {
_, router := setupTestServer(t)
req := httptest.NewRequest("GET", "/api/analytics/hash-sizes", nil)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
_, router := setupTestServer(t)
req := httptest.NewRequest("GET", "/api/analytics/hash-sizes", nil)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
if w.Code != 200 {
t.Fatalf("expected 200, got %d", w.Code)
}
if w.Code != 200 {
t.Fatalf("expected 200, got %d", w.Code)
}
var body map[string]interface{}
json.Unmarshal(w.Body.Bytes(), &body)
var body map[string]interface{}
json.Unmarshal(w.Body.Bytes(), &body)
arrayFields := []string{"hourly", "topHops", "multiByteNodes"}
for _, field := range arrayFields {
if body[field] == nil {
t.Errorf("field %q is null, expected []", field)
}
arrayFields := []string{"hourly", "topHops", "multiByteNodes"}
for _, field := range arrayFields {
if body[field] == nil {
t.Errorf("field %q is null, expected []", field)
}
}
}
func TestInconsistentNodesExcludesCompanions(t *testing.T) {
@@ -3620,7 +3618,7 @@ func TestHashCollisionsOnlyRepeaters(t *testing.T) {
store.hashSizeInfoCache = map[string]*hashSizeNodeInfo{
"aa11223344556677": {HashSize: 1, AllSizes: map[int]bool{1: true}},
"aa00112233445566": {HashSize: 1, AllSizes: map[int]bool{1: true}},
"aa99887766554433": {HashSize: 0, AllSizes: map[int]bool{}}, // unknown
"aa99887766554433": {HashSize: 0, AllSizes: map[int]bool{}}, // unknown
"aadeadbeefcafe01": {HashSize: 1, AllSizes: map[int]bool{1: true}}, // companion
"aabbcc1122334455": {HashSize: 1, AllSizes: map[int]bool{1: true}}, // room
"aabbcc9988776655": {HashSize: 1, AllSizes: map[int]bool{1: true}}, // sensor
@@ -4168,7 +4166,7 @@ func TestHandleScopeStats(t *testing.T) {
}{
{"h1", "#belgium", 0},
{"h2", "#belgium", 3},
{"h3", "", 0}, // transport-scoped, no region match
{"h3", "", 0}, // transport-scoped, no region match
{"h4_null", "", 0}, // will be inserted with NULL scope_name
}
for i, r := range rows {
@@ -4688,4 +4686,35 @@ func TestGetNodesForGeoPrune(t *testing.T) {
// TestDeleteNodesByPubkeys was removed in PR #738 follow-up: the DELETE has
// been relocated to the ingestor (cmd/ingestor/prune_geofilter.go). End-to-end
// coverage of the prune flow now lives in cmd/ingestor/*_test.go.
// TestListLimitsConfigurable verifies that list limits are driven by the configuration.
func TestListLimitsConfigurable(t *testing.T) {
srv, router := setupTestServer(t)
// Inject a custom config
srv.cfg.ListLimits = &ListLimitsConfig{
PacketsMax: 1234,
}
// Request with a limit larger than the configured max
req := httptest.NewRequest("GET", "/api/packets?limit=999999", nil)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
if w.Code != 200 {
t.Fatalf("expected 200, got %d", w.Code)
}
var body map[string]interface{}
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
t.Fatalf("json decode: %v", err)
}
limit, ok := body["limit"].(float64)
if !ok {
t.Fatal("expected limit field in response")
}
if limit != 1234 {
t.Errorf("expected limit to be capped at 1234, got %v", limit)
}
}
config.example.json
+8
View File
@@ -17,6 +17,14 @@
"_comment": "vacuumOnStartup: run one-time full VACUUM to enable incremental auto-vacuum on existing DBs. Executed by the INGESTOR at startup, BEFORE the MQTT subscriber starts (#1283), so there is no contention with concurrent writes. Blocks ingestor startup for minutes on large DBs; requires 2x DB file size in free disk space. incrementalVacuumPages: free pages returned to OS after each retention reaper cycle (default 1024). See #919.",
"_comment_slowWriterMs": "#1340 — SQLite writer-lock log threshold (default 500). Any wrapped writer call (tagged neighbor_builder, mqtt_handler, prune_packets, prune_observers, prune_metrics, vacuum) whose hold_ms exceeds this emits a single [db-slow-writer] log line. Configured per-process via the CORESCOPE_DB_SLOW_WRITER_MS environment variable on the INGESTOR (e.g. CORESCOPE_DB_SLOW_WRITER_MS=200 for tighter alerting). Per-component wait_ms / hold_ms / contention_total histograms are surfaced via /api/perf/write-sources under .writer_perf regardless of this threshold."
},
"listLimits": {
"packetsMax": 10000,
"nodesMax": 2000,
"analyticsMax": 200,
"channelMessagesMax": 500,
"bulkHealthMax": 200,
"_comment": "Maximum row counts returned by list API endpoints. These enforce a DoS-bounded ceiling for both UI and external requests. Operators with small/embedded deployments can tighten these; operators running large regional meshes can raise them. bulkHealthMax is intentionally separate from nodesMax: /api/nodes/bulk-health is per-row much heavier than /api/nodes (it joins per-node observer health + recent-packet stats) so its ceiling stays low (default 200) even if nodesMax is raised."
},
"_comment_ingestorStats": "Ingestor publishes a 1-Hz stats snapshot consumed by the server's /api/perf/io and /api/perf/write-sources endpoints (#1120). Path is configured via the CORESCOPE_INGESTOR_STATS environment variable on the INGESTOR process. Default: /tmp/corescope-ingestor-stats.json. The writer uses O_NOFOLLOW + 0o600, so a pre-planted symlink in /tmp cannot be used to clobber an arbitrary file. SECURITY: in shared-tmp environments (multi-tenant hosts), point CORESCOPE_INGESTOR_STATS at a private directory like /var/lib/corescope/ingestor-stats.json that only the corescope user can write to.",
"corsAllowedOrigins": [],
"_comment_corsAllowedOrigins": "Cross-origin allowlist for embed scenarios (#1369). Exact-match origins, e.g. [\"https://blog.example.com\", \"https://embed.example.com\"]. When empty (default), no Access-Control-* headers are sent and browsers enforce same-origin. When non-empty, only the listed origins receive CORS headers, and Access-Control-Allow-Methods is limited to GET, HEAD, OPTIONS (the cross-domain surface is read-only — same-origin admin writes are unaffected). Use [\"*\"] to allow any origin (NOT recommended for write-capable deployments). Operators can override per-deployment with the CORS_ALLOWED_ORIGINS environment variable (comma-separated). No credentialed CORS is enabled. To embed the map or channels pages cross-domain, add the embedding origin here and use the URL pattern '/#/map?embed=1' or '/#/channels?embed=1' — embed mode hides the top-nav, bottom-nav, and side drawer for full-bleed iframe rendering.",