feat(mqtt): per-source status endpoint + Observers panel (#1682)

## Summary Adds MQTT source status visibility per #1043 acceptance criteria: - **Ingestor:** per-source counter registry (`cmd/ingestor/source_status.go`) tracking `connected`, `lastConnectUnix`, `lastDisconnectUnix`, `lastPacketUnix`, `connectCount`, `disconnectCount`, `packetsTotal`, `packetsLast5m` (sliding 5-min window via per-second buckets keyed by unix second — no stale-leak), `lastError`. Wired at the existing OnConnect / ConnectionLost / DefaultPublish callsites alongside the liveness watchdog. Idempotent registration so counters survive reconnects. Snapshot emitted in the existing stats file under `source_statuses` (additive, `omitempty`). - **Backend:** new `GET /api/mqtt/status` handler reads the ingestor stats file and returns the per-source list. **Broker passwords are masked** via a regex over the `scheme://user:pass@host` form (covers mqtt/mqtts/tcp/ssl/ws/wss). Mask is also applied to `lastError` as defense-in-depth (broker libs occasionally quote the failing URL). OpenAPI completeness gate satisfied with a `routeDescriptions` entry. - **Frontend:** small self-contained panel (`public/mqtt-status-panel.js`) mounted above the Observers table. Auto-refreshes every 10s, color-codes each row (green = connected + recent packet, yellow = connected idle, red = disconnected), and tears down its timer on SPA route change. ## TDD - Red commit `f19a93b5` — stub `/api/mqtt/status` handler + assertion test that the broker password is `****`-redacted. Test fails on the assertion (handler passes the URL through verbatim). Compile-clean — assertion-fail, not build-fail. - Green commit `77042e41` — `maskBrokerURL` helper + table-driven unit tests across all schemes + handler rewires to mask both `Broker` and `LastError`. - Subsequent commits land the ingestor wiring and the frontend panel. ## Tests ``` $ cd cmd/server && go test -run 'TestMqttStatus|TestMaskBrokerURL' -v ./... PASS: TestMqttStatus_MasksBrokerPassword PASS: TestMqttStatus_EmptyWhenNoStatsFile PASS: TestMaskBrokerURL_Patterns (10 subtests) $ cd cmd/ingestor && go test -run 'TestSourceStatus|TestSnapshotSourceStatuses' -v ./... PASS: TestSourceStatus_BasicLifecycle PASS: TestSourceStatus_Disconnect PASS: TestSnapshotSourceStatuses_ReturnsAll $ node test-mqtt-status-panel.js 7 passed, 0 failed ``` Full `go test ./...` clean in both `cmd/server` and `cmd/ingestor`. ## Preflight overrides - `cross-stack`: justified — issue #1043 is intrinsically full-stack (ingestor stats → server endpoint → observers panel). Per-stack split would land an unreachable endpoint or a fetch with no backend. - `check-xss-sinks` (public/mqtt-status-panel.js:55): justified — the flagged `innerHTML=` is a fully-static literal (empty-state placeholder, no payload data interpolated). All payload-bearing `innerHTML=` sites in this file run through `escapeHTML` (defined in the same file); the test `renderPanel never echoes a plaintext password (defense-in-depth)` exercises the rendered HTML against payload strings. ## Acceptance criteria - [x] `/api/mqtt/status` returns per-source connection state — `cmd/server/mqtt_status.go` - [x] UI panel shows all configured sources with live status — `public/mqtt-status-panel.js` - [x] Connection state updates on reconnect/disconnect events — `MarkConnect` / `MarkDisconnect` wired in `cmd/ingestor/main.go` - [x] Broker URLs don't expose passwords in the API response — `maskBrokerURL` + 13 test cases - [x] Works with 1-N sources — registry is keyed per-source, snapshot iterates the map **Partial fix for #1043** — per-packet `mqtt_source` attribution (the issue's "Follow-up" section) is **deferred** per the `mc-bot-triaged:v1` triage and the autofix comment ("Per-packet attribution deferred to follow-up issue"). That work requires a new observation-row column and DB schema migration, both explicitly out of scope for this PR. Refs #1043 --------- Co-authored-by: openclaw-bot <bot@openclaw.local>
2026-08-28 03:24:17 +00:00 · 2026-06-12 08:11:02 -07:00
parent 2ef7d2437d
commit efd66ea3f5
13 changed files with 919 additions and 0 deletions
@@ -0,0 +1,144 @@
+package main
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/url"
+	"os"
+	"regexp"
+	"strings"
+)
+
+// mqttBrokerSchemes is the set of broker URL schemes whose embedded
+// `user:pass@host` credentials we want to redact. We URL-parse for these
+// (defense vs. passwords containing `@`); other strings fall through to
+// the legacy regex pass for embedded user:pass occurrences in free-form
+// error strings.
+var mqttBrokerSchemes = map[string]bool{
+	"mqtt": true, "mqtts": true, "tcp": true, "ssl": true, "ws": true, "wss": true,
+}
+
+// mqttBrokerURLRe locates a broker URL (with credentials) embedded inside
+// a larger free-form string — e.g. an error message that quotes the
+// failing broker. Each match is fed through url.Parse + redaction. We
+// match greedily up through the LAST `@` followed by a host-shaped token
+// so passwords containing `@` are not truncated (#1682 adversarial r1).
+//
+// Go's RE2 has no lookahead; we capture the host tail and emit it
+// unchanged in the replacement.
+var mqttBrokerURLRe = regexp.MustCompile(`(?i)(?:mqtt|mqtts|tcp|ssl|ws|wss)://[^\s]*`)
+
+// maskBrokerURL returns the broker URL with any inline password redacted.
+// `mqtt://user:secret@host:1883` -> `mqtt://user:****@host:1883`.
+// `mqtt://user:p@ss@host` -> `mqtt://user:****@host` (password with `@`).
+// URLs without inline credentials are returned unchanged.
+//
+// Primary strategy: url.Parse — handles passwords with `@`, `:`, etc.
+// Fallback: regex sweep for free-form strings (e.g. error messages that
+// quote a URL fragment but aren't standalone-parseable).
+func maskBrokerURL(s string) string {
+	if s == "" {
+		return s
+	}
+	// Fast path: the whole string is the broker URL.
+	if masked, ok := redactBrokerURL(s); ok {
+		return masked
+	}
+	// Fallback: free-form string (e.g. error message) containing a URL.
+	// Find embedded broker URLs and redact each in-place.
+	return mqttBrokerURLRe.ReplaceAllStringFunc(s, func(m string) string {
+		if out, ok := redactBrokerURL(m); ok {
+			return out
+		}
+		return m
+	})
+}
+
+// redactBrokerURL parses s as a URL and, if it has an mqtt-family scheme
+// with userinfo containing a password, returns the URL with the password
+// replaced by `****`. Returns ok=false when s is not such a URL.
+func redactBrokerURL(s string) (string, bool) {
+	u, err := url.Parse(s)
+	if err != nil || u.Scheme == "" || u.User == nil {
+		return s, false
+	}
+	if !mqttBrokerSchemes[strings.ToLower(u.Scheme)] {
+		return s, false
+	}
+	if _, hasPass := u.User.Password(); !hasPass {
+		return s, false
+	}
+	// Re-assemble manually rather than via url.UserPassword + u.String()
+	// because the latter percent-encodes the `*` mask token into `%2A`,
+	// defeating the user-visible redaction marker. We only need to swap
+	// the userinfo segment of the original string.
+	hostAndAfter := s
+	if idx := strings.LastIndex(s, "@"); idx >= 0 {
+		hostAndAfter = s[idx+1:]
+	}
+	// Preserve original scheme casing (url.Parse lowercases u.Scheme).
+	schemeEnd := strings.Index(s, "://")
+	if schemeEnd < 0 {
+		return s, false
+	}
+	return s[:schemeEnd] + "://" + u.User.Username() + ":****@" + hostAndAfter, true
+}
+
+// MqttSourceStatus is the per-MQTT-source status row surfaced via
+// /api/mqtt/status. Mirrors the on-disk shape the ingestor publishes
+// (cmd/ingestor SourceStatusSnapshot) but with the broker URL credentials
+// redacted before serving — operators must not see the broker password
+// in the API response (#1043 acceptance criterion).
+type MqttSourceStatus struct {
+	Name               string `json:"name"`
+	Broker             string `json:"broker"`
+	Connected          bool   `json:"connected"`
+	LastConnectUnix    int64  `json:"lastConnectUnix"`
+	LastDisconnectUnix int64  `json:"lastDisconnectUnix"`
+	LastPacketUnix     int64  `json:"lastPacketUnix"`
+	ConnectCount       int64  `json:"connectCount"`
+	DisconnectCount    int64  `json:"disconnectCount"`
+	PacketsTotal       int64  `json:"packetsTotal"`
+	PacketsLast5m      int64  `json:"packetsLast5m"`
+	LastError          string `json:"lastError,omitempty"`
+}
+
+// MqttStatusResponse is the JSON envelope returned by /api/mqtt/status.
+type MqttStatusResponse struct {
+	Sources  []MqttSourceStatus `json:"sources"`
+	SampleAt string             `json:"sampleAt"`
+}
+
+// ingestorMqttStatusEnvelope is the partial shape the server decodes from
+// the ingestor stats file (additive — older ingestors omit the field).
+type ingestorMqttStatusEnvelope struct {
+	SampledAt      string             `json:"sampledAt"`
+	SourceStatuses []MqttSourceStatus `json:"source_statuses"`
+}
+
+// handleMqttStatus serves GET /api/mqtt/status. Reads the ingestor stats
+// file, masks broker-URL passwords, and returns the per-source status
+// list. Returns an empty list (200 OK) when the stats file is missing
+// or unparseable — the UI panel renders a "no data yet" state.
+func (s *Server) handleMqttStatus(w http.ResponseWriter, r *http.Request) {
+	resp := MqttStatusResponse{Sources: []MqttSourceStatus{}, SampleAt: ""}
+	data, err := os.ReadFile(IngestorStatsPath())
+	if err != nil {
+		writeJSON(w, resp)
+		return
+	}
+	var env ingestorMqttStatusEnvelope
+	if err := json.Unmarshal(data, &env); err != nil {
+		writeJSON(w, resp)
+		return
+	}
+	resp.SampleAt = env.SampledAt
+	for _, src := range env.SourceStatuses {
+		src.Broker = maskBrokerURL(src.Broker)
+		// Broker libraries occasionally quote the failing URL in the
+		// error string — redact there too as defense-in-depth.
+		src.LastError = maskBrokerURL(src.LastError)
+		resp.Sources = append(resp.Sources, src)
+	}
+	writeJSON(w, resp)
+}
@@ -0,0 +1,142 @@
+package main
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// TestMqttStatus_MasksBrokerPassword (#1043) asserts the /api/mqtt/status
+// handler never leaks the broker password embedded in a mqtt:// URL.
+// Operators viewing the API response (or the Observers panel that
+// consumes it) must see `****` in place of the inline credential.
+//
+// Test shape: write a stub ingestor stats file with one source whose
+// broker URL contains a plaintext password, invoke the handler, assert
+// the JSON response (a) contains the username + host, (b) does NOT
+// contain the password substring.
+func TestMqttStatus_MasksBrokerPassword(t *testing.T) {
+	const password = "hunter2supersecret"
+	const rawBroker = "mqtt://obsuser:" + password + "@broker.example.com:1883"
+
+	tmp := t.TempDir()
+	statsPath := filepath.Join(tmp, "ingestor-stats.json")
+	t.Setenv("CORESCOPE_INGESTOR_STATS", statsPath)
+
+	// Stub stats file: one MQTT source with a credentialed broker URL.
+	stub := map[string]any{
+		"sampledAt": "2026-06-12T12:30:00Z",
+		"source_statuses": []map[string]any{{
+			"name":            "local",
+			"broker":          rawBroker,
+			"connected":       true,
+			"lastPacketUnix":  1717977000,
+			"connectCount":    1,
+			"disconnectCount": 0,
+			"packetsTotal":    42,
+			"packetsLast5m":   7,
+		}},
+	}
+	data, err := json.Marshal(stub)
+	if err != nil {
+		t.Fatalf("marshal stub: %v", err)
+	}
+	if err := os.WriteFile(statsPath, data, 0o600); err != nil {
+		t.Fatalf("write stub: %v", err)
+	}
+
+	srv := &Server{}
+	req := httptest.NewRequest(http.MethodGet, "/api/mqtt/status", nil)
+	rec := httptest.NewRecorder()
+	srv.handleMqttStatus(rec, req)
+
+	if rec.Code != http.StatusOK {
+		t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
+	}
+	body := rec.Body.String()
+	t.Logf("response body: %s", body)
+
+	if strings.Contains(body, password) {
+		t.Errorf("response leaks broker password %q in body: %s", password, body)
+	}
+	// Sanity: the response still identifies the source by name + host.
+	if !strings.Contains(body, "broker.example.com") {
+		t.Errorf("response missing broker host: %s", body)
+	}
+	if !strings.Contains(body, "obsuser") {
+		t.Errorf("response missing broker username: %s", body)
+	}
+	// Mask token must be present so operators can tell credentials were
+	// redacted vs the broker URL never having a password to begin with.
+	if !strings.Contains(body, "****") {
+		t.Errorf("response missing redaction marker '****': %s", body)
+	}
+}
+
+// TestMqttStatus_EmptyWhenNoStatsFile asserts the handler returns an empty
+// list (200 OK) when the ingestor stats file is missing — the UI panel
+// renders a "no data yet" state in that case.
+func TestMqttStatus_EmptyWhenNoStatsFile(t *testing.T) {
+	tmp := t.TempDir()
+	t.Setenv("CORESCOPE_INGESTOR_STATS", filepath.Join(tmp, "does-not-exist.json"))
+
+	srv := &Server{}
+	req := httptest.NewRequest(http.MethodGet, "/api/mqtt/status", nil)
+	rec := httptest.NewRecorder()
+	srv.handleMqttStatus(rec, req)
+
+	if rec.Code != http.StatusOK {
+		t.Fatalf("status = %d, want 200", rec.Code)
+	}
+	var resp MqttStatusResponse
+	if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
+		t.Fatalf("unmarshal: %v; body=%s", err, rec.Body.String())
+	}
+	if len(resp.Sources) != 0 {
+		t.Errorf("Sources len = %d, want 0", len(resp.Sources))
+	}
+}
+
+// TestMaskBrokerURL_Patterns is a unit table-driven test for the masking
+// helper. Kept separate from the handler test so a regression in the
+// regex localizes immediately.
+func TestMaskBrokerURL_Patterns(t *testing.T) {
+	cases := []struct {
+		name, in, want string
+	}{
+		{"plain mqtt no creds", "mqtt://broker.example.com:1883", "mqtt://broker.example.com:1883"},
+		{"mqtt with creds", "mqtt://u:secret@broker.example.com:1883", "mqtt://u:****@broker.example.com:1883"},
+		{"mqtts with creds", "mqtts://u:secret@broker.example.com:8883", "mqtts://u:****@broker.example.com:8883"},
+		{"tcp with creds", "tcp://u:p@host:1883", "tcp://u:****@host:1883"},
+		{"ssl with creds", "ssl://u:p@host:8883", "ssl://u:****@host:8883"},
+		{"ws with creds", "ws://u:p@host:8080/mqtt", "ws://u:****@host:8080/mqtt"},
+		{"wss with creds", "wss://u:p@host:443/mqtt", "wss://u:****@host:443/mqtt"},
+		{"uppercase scheme", "MQTT://u:p@host:1883", "MQTT://u:****@host:1883"},
+		{"empty", "", ""},
+		{"long password", "mqtt://obsuser:hunter2supersecretXYZ123@host:1883", "mqtt://obsuser:****@host:1883"},
+		{"no scheme bare host", "host:1883", "host:1883"},
+		// Adversarial r1 review (#1682): password contains @. The previous
+		// regex-only impl matched only up to the FIRST @, exposing "ss" as
+		// part of the path: "mqtt://user:****@ss@host". url.Parse handles
+		// this correctly because Go interprets the LAST @ as the userinfo
+		// boundary.
+		{"password with single @", "mqtt://user:p@ss@host:1883", "mqtt://user:****@host:1883"},
+		{"password with multiple @", "mqtt://user:p@ss@wo@host:1883", "mqtt://user:****@host:1883"},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			got := maskBrokerURL(c.in)
+			if got != c.want {
+				t.Errorf("maskBrokerURL(%q) = %q, want %q", c.in, got, c.want)
+			}
+			// Inline secret must never survive.
+			if c.in != c.want && strings.Contains(got, "secret") {
+				t.Errorf("output still contains 'secret': %q", got)
+			}
+		})
+	}
+}
@@ -42,6 +42,7 @@ func routeDescriptions() map[string]routeMeta {
 		"GET /api/health":     {Summary: "Health check", Description: "Returns server health, uptime, and memory stats.", Tag: "admin"},
 		"GET /api/stats":      {Summary: "Network statistics", Description: "Returns aggregate stats (node counts, packet counts, observer counts). Cached for 10s.", Tag: "admin"},
 		"GET /api/perf":       {Summary: "Performance statistics", Description: "Returns per-endpoint request timing and slow query log.", Tag: "admin"},
+		"GET /api/mqtt/status": {Summary: "MQTT source status", Description: "Returns per-MQTT-source connection state and counters (lastConnectUnix, lastPacketUnix, packetsTotal, etc.). Broker URL passwords are masked. Sourced from the ingestor stats file; empty list when unavailable. (#1043)", Tag: "admin"},
 		"POST /api/perf/reset": {Summary: "Reset performance stats", Tag: "admin", Auth: true},
 		// "POST /api/admin/prune" removed in #1283 (ingestor owns prune).
 		"GET /api/debug/affinity": {Summary: "Debug neighbor affinity scores", Tag: "admin", Auth: true},
@@ -230,6 +230,7 @@ func (s *Server) RegisterRoutes(r *mux.Router) {
 	r.HandleFunc("/api/perf/io", s.handlePerfIO).Methods("GET")
 	r.HandleFunc("/api/perf/sqlite", s.handlePerfSqlite).Methods("GET")
 	r.HandleFunc("/api/perf/write-sources", s.handlePerfWriteSources).Methods("GET")
+	r.HandleFunc("/api/mqtt/status", s.handleMqttStatus).Methods("GET")
 	r.Handle("/api/perf/reset", s.requireAPIKey(http.HandlerFunc(s.handlePerfReset))).Methods("POST")
 	// /api/admin/prune removed in #1283 — pruning is owned by the
 	// ingestor process (scheduled tickers + startup pass). Operators