Files
meshcore-analyzer/cmd/ingestor/config_test.go
T
Eric Muehlstein 29432d4fe0 feat(ingestor): document and test ws:// / wss:// WebSocket MQTT broker support (#902)
## Summary

CoreScope's ingestor already supports WebSocket MQTT connections today —
`paho.mqtt.golang` v1.5.0 handles `ws://` and `wss://` natively via
gorilla/websocket. However this support was **undocumented, untested,
and had a TLS gap** for `wss://` connections.

This PR closes those gaps without any breaking changes.

## Changes

### `cmd/ingestor/config.go`
- Added godoc comment to `ResolvedSources()` explaining all four
supported schemes and which ones require translation vs. pass-through
- `ws://` and `wss://` explicitly documented as native paho schemes
requiring no mapping

### `cmd/ingestor/main.go`
- Extended TLS config to cover `wss://` in addition to `ssl://`
- Before: `wss://` connections would use paho's default TLS (no explicit
`tls.Config` set), which works for valid certs but doesn't apply the
same predictable setup as `ssl://`
- After: both `ssl://` and `wss://` get `tls.Config{}` (system CA pool),
matching behavior; `rejectUnauthorized: false` still works for
self-signed certs on both schemes

### `cmd/ingestor/config_test.go`
Two new tests:
- `TestResolvedSourcesSchemeMapping`: validates all six scheme
variations (`mqtt://`, `mqtts://`, `tcp://`, `ssl://`, `ws://`,
`wss://`) including paths like `wss://host/mqtt`
- `TestLoadConfigWSSource`: full round-trip of a dual-source config (TCP
+ wss:// with username/password), verifies scheme unchanged through
`LoadConfig` and `ResolvedSources`

### `config.example.json`
- Added `wsmqtt` example entry showing `wss://` with username/password
- Updated `_comment_mqttSources` to enumerate all supported schemes:
`mqtt://`, `mqtts://`, `ws://`, `wss://`

## Motivation

We run
[meshcore-mqtt-broker](https://github.com/andrewjfreyer/meshcore-mqtt-broker)
(a WebSocket MQTT bridge with JWT auth) alongside Mosquitto, and
subscribe to both via `mqttSources`. The dual-source config works in
production but nothing in the docs or example config made this
discoverable for other operators.

## Testing

```
cd cmd/ingestor && go test ./...
ok    github.com/corescope/ingestor  1.568s
```

All existing tests pass. Two new tests added.

## No breaking changes

- Existing configs: no change in behavior
- `ws://` / `wss://` configs that were already working: same behavior +
explicit TLS setup for `wss://`
2026-05-28 14:58:52 -07:00

487 lines
13 KiB
Go

package main
import (
"os"
"path/filepath"
"testing"
)
func TestLoadConfigValidJSON(t *testing.T) {
dir := t.TempDir()
cfgPath := filepath.Join(dir, "config.json")
os.WriteFile(cfgPath, []byte(`{
"dbPath": "/tmp/test.db",
"mqttSources": [
{"name": "s1", "broker": "tcp://localhost:1883", "topics": ["meshcore/#"]}
]
}`), 0o644)
cfg, err := LoadConfig(cfgPath)
if err != nil {
t.Fatal(err)
}
if cfg.DBPath != "/tmp/test.db" {
t.Errorf("dbPath=%s, want /tmp/test.db", cfg.DBPath)
}
if len(cfg.MQTTSources) != 1 {
t.Fatalf("mqttSources len=%d, want 1", len(cfg.MQTTSources))
}
if cfg.MQTTSources[0].Broker != "tcp://localhost:1883" {
t.Errorf("broker=%s", cfg.MQTTSources[0].Broker)
}
}
func TestLoadConfigMissingFile(t *testing.T) {
t.Setenv("DB_PATH", "")
t.Setenv("MQTT_BROKER", "")
cfg, err := LoadConfig("/nonexistent/path/config.json")
if err != nil {
t.Fatalf("missing config should not error (zero-config mode), got: %v", err)
}
if cfg.DBPath != "data/meshcore.db" {
t.Errorf("dbPath=%s, want data/meshcore.db", cfg.DBPath)
}
// Should default to localhost MQTT
if len(cfg.MQTTSources) != 1 {
t.Fatalf("mqttSources len=%d, want 1", len(cfg.MQTTSources))
}
if cfg.MQTTSources[0].Broker != "mqtt://localhost:1883" {
t.Errorf("default broker=%s, want mqtt://localhost:1883", cfg.MQTTSources[0].Broker)
}
if cfg.MQTTSources[0].Name != "local" {
t.Errorf("default source name=%s, want local", cfg.MQTTSources[0].Name)
}
}
func TestLoadConfigMalformedJSON(t *testing.T) {
dir := t.TempDir()
cfgPath := filepath.Join(dir, "bad.json")
os.WriteFile(cfgPath, []byte(`{not valid json`), 0o644)
_, err := LoadConfig(cfgPath)
if err == nil {
t.Error("expected error for malformed JSON")
}
}
func TestLoadConfigEnvVarDBPath(t *testing.T) {
t.Setenv("DB_PATH", "/override/db.sqlite")
t.Setenv("MQTT_BROKER", "")
dir := t.TempDir()
cfgPath := filepath.Join(dir, "config.json")
os.WriteFile(cfgPath, []byte(`{"dbPath": "original.db"}`), 0o644)
cfg, err := LoadConfig(cfgPath)
if err != nil {
t.Fatal(err)
}
if cfg.DBPath != "/override/db.sqlite" {
t.Errorf("dbPath=%s, want /override/db.sqlite", cfg.DBPath)
}
}
func TestLoadConfigEnvVarMQTTBroker(t *testing.T) {
t.Setenv("MQTT_BROKER", "tcp://env-broker:1883")
t.Setenv("MQTT_TOPIC", "custom/topic")
dir := t.TempDir()
cfgPath := filepath.Join(dir, "config.json")
os.WriteFile(cfgPath, []byte(`{"dbPath": "test.db"}`), 0o644)
cfg, err := LoadConfig(cfgPath)
if err != nil {
t.Fatal(err)
}
if len(cfg.MQTTSources) != 1 {
t.Fatalf("mqttSources len=%d, want 1", len(cfg.MQTTSources))
}
src := cfg.MQTTSources[0]
if src.Name != "env" {
t.Errorf("name=%s, want env", src.Name)
}
if src.Broker != "tcp://env-broker:1883" {
t.Errorf("broker=%s", src.Broker)
}
if len(src.Topics) != 1 || src.Topics[0] != "custom/topic" {
t.Errorf("topics=%v, want [custom/topic]", src.Topics)
}
}
func TestLoadConfigEnvVarMQTTBrokerDefaultTopic(t *testing.T) {
t.Setenv("MQTT_BROKER", "tcp://env-broker:1883")
t.Setenv("MQTT_TOPIC", "")
dir := t.TempDir()
cfgPath := filepath.Join(dir, "config.json")
os.WriteFile(cfgPath, []byte(`{"dbPath": "test.db"}`), 0o644)
cfg, err := LoadConfig(cfgPath)
if err != nil {
t.Fatal(err)
}
if cfg.MQTTSources[0].Topics[0] != "meshcore/#" {
t.Errorf("default topic=%s, want meshcore/#", cfg.MQTTSources[0].Topics[0])
}
}
func TestLoadConfigLegacyMQTT(t *testing.T) {
t.Setenv("DB_PATH", "")
t.Setenv("MQTT_BROKER", "")
dir := t.TempDir()
cfgPath := filepath.Join(dir, "config.json")
os.WriteFile(cfgPath, []byte(`{
"dbPath": "test.db",
"mqtt": {"broker": "tcp://legacy:1883", "topic": "old/topic"}
}`), 0o644)
cfg, err := LoadConfig(cfgPath)
if err != nil {
t.Fatal(err)
}
if len(cfg.MQTTSources) != 1 {
t.Fatalf("mqttSources len=%d, want 1", len(cfg.MQTTSources))
}
src := cfg.MQTTSources[0]
if src.Name != "default" {
t.Errorf("name=%s, want default", src.Name)
}
if src.Broker != "tcp://legacy:1883" {
t.Errorf("broker=%s", src.Broker)
}
if len(src.Topics) != 2 || src.Topics[0] != "old/topic" || src.Topics[1] != "meshcore/#" {
t.Errorf("topics=%v, want [old/topic meshcore/#]", src.Topics)
}
}
func TestLoadConfigLegacyMQTTNotUsedWhenSourcesExist(t *testing.T) {
t.Setenv("DB_PATH", "")
t.Setenv("MQTT_BROKER", "")
dir := t.TempDir()
cfgPath := filepath.Join(dir, "config.json")
os.WriteFile(cfgPath, []byte(`{
"dbPath": "test.db",
"mqtt": {"broker": "tcp://legacy:1883", "topic": "old/topic"},
"mqttSources": [{"name": "modern", "broker": "tcp://modern:1883", "topics": ["m/#"]}]
}`), 0o644)
cfg, err := LoadConfig(cfgPath)
if err != nil {
t.Fatal(err)
}
if len(cfg.MQTTSources) != 1 {
t.Fatalf("mqttSources len=%d, want 1", len(cfg.MQTTSources))
}
if cfg.MQTTSources[0].Name != "modern" {
t.Errorf("should use modern source, got name=%s", cfg.MQTTSources[0].Name)
}
}
func TestLoadConfigDefaultDBPath(t *testing.T) {
t.Setenv("DB_PATH", "")
t.Setenv("MQTT_BROKER", "")
dir := t.TempDir()
cfgPath := filepath.Join(dir, "config.json")
os.WriteFile(cfgPath, []byte(`{}`), 0o644)
cfg, err := LoadConfig(cfgPath)
if err != nil {
t.Fatal(err)
}
if cfg.DBPath != "data/meshcore.db" {
t.Errorf("dbPath=%s, want data/meshcore.db", cfg.DBPath)
}
}
func TestLoadConfigLegacyMQTTEmptyBroker(t *testing.T) {
t.Setenv("DB_PATH", "")
t.Setenv("MQTT_BROKER", "")
dir := t.TempDir()
cfgPath := filepath.Join(dir, "config.json")
os.WriteFile(cfgPath, []byte(`{
"dbPath": "test.db",
"mqtt": {"broker": "", "topic": "t"}
}`), 0o644)
cfg, err := LoadConfig(cfgPath)
if err != nil {
t.Fatal(err)
}
if len(cfg.MQTTSources) != 1 || cfg.MQTTSources[0].Name != "local" {
t.Errorf("mqttSources should default to local broker when legacy broker is empty, got %v", cfg.MQTTSources)
}
}
func TestResolvedSources(t *testing.T) {
cfg := &Config{
MQTTSources: []MQTTSource{
{Name: "a", Broker: "tcp://a:1883"},
{Name: "b", Broker: "tcp://b:1883"},
},
}
sources := cfg.ResolvedSources()
if len(sources) != 2 {
t.Fatalf("len=%d, want 2", len(sources))
}
if sources[0].Name != "a" || sources[1].Name != "b" {
t.Errorf("sources=%v", sources)
}
}
func TestResolvedSourcesEmpty(t *testing.T) {
cfg := &Config{}
sources := cfg.ResolvedSources()
if len(sources) != 0 {
t.Errorf("len=%d, want 0", len(sources))
}
}
func TestLoadConfigWithAllFields(t *testing.T) {
t.Setenv("DB_PATH", "")
t.Setenv("MQTT_BROKER", "")
dir := t.TempDir()
cfgPath := filepath.Join(dir, "config.json")
reject := false
_ = reject
os.WriteFile(cfgPath, []byte(`{
"dbPath": "mydb.db",
"logLevel": "debug",
"mqttSources": [{
"name": "full",
"broker": "tcp://full:1883",
"username": "user1",
"password": "pass1",
"rejectUnauthorized": false,
"topics": ["a/#", "b/#"],
"iataFilter": ["SJC", "LAX"]
}]
}`), 0o644)
cfg, err := LoadConfig(cfgPath)
if err != nil {
t.Fatal(err)
}
if cfg.LogLevel != "debug" {
t.Errorf("logLevel=%s, want debug", cfg.LogLevel)
}
src := cfg.MQTTSources[0]
if src.Username != "user1" {
t.Errorf("username=%s", src.Username)
}
if src.Password != "pass1" {
t.Errorf("password=%s", src.Password)
}
if src.RejectUnauthorized == nil || *src.RejectUnauthorized != false {
t.Error("rejectUnauthorized should be false")
}
if len(src.IATAFilter) != 2 || src.IATAFilter[0] != "SJC" {
t.Errorf("iataFilter=%v", src.IATAFilter)
}
}
func TestConnectTimeoutOrDefault(t *testing.T) {
// Default when unset
s := MQTTSource{}
if got := s.ConnectTimeoutOrDefault(); got != 30 {
t.Errorf("default: got %d, want 30", got)
}
// Custom value
s.ConnectTimeoutSec = 5
if got := s.ConnectTimeoutOrDefault(); got != 5 {
t.Errorf("custom: got %d, want 5", got)
}
// Zero treated as unset
s.ConnectTimeoutSec = 0
if got := s.ConnectTimeoutOrDefault(); got != 30 {
t.Errorf("zero: got %d, want 30", got)
}
}
func TestConnectTimeoutFromJSON(t *testing.T) {
dir := t.TempDir()
cfgPath := dir + "/config.json"
os.WriteFile(cfgPath, []byte(`{"mqttSources":[{"name":"s1","broker":"tcp://b:1883","topics":["#"],"connectTimeoutSec":5}]}`), 0644)
cfg, err := LoadConfig(cfgPath)
if err != nil {
t.Fatal(err)
}
if got := cfg.MQTTSources[0].ConnectTimeoutOrDefault(); got != 5 {
t.Errorf("from JSON: got %d, want 5", got)
}
}
func TestObserverIATAWhitelist(t *testing.T) {
// Config with whitelist set
cfg := Config{
ObserverIATAWhitelist: []string{"ARN", "got"},
}
// Matching (case-insensitive)
if !cfg.IsObserverIATAAllowed("ARN") {
t.Error("ARN should be allowed")
}
if !cfg.IsObserverIATAAllowed("arn") {
t.Error("arn (lowercase) should be allowed")
}
if !cfg.IsObserverIATAAllowed("GOT") {
t.Error("GOT should be allowed")
}
// Non-matching
if cfg.IsObserverIATAAllowed("SJC") {
t.Error("SJC should NOT be allowed")
}
// Empty string not allowed
if cfg.IsObserverIATAAllowed("") {
t.Error("empty IATA should NOT be allowed")
}
}
func TestObserverIATAWhitelistEmpty(t *testing.T) {
// No whitelist = allow all
cfg := Config{}
if !cfg.IsObserverIATAAllowed("SJC") {
t.Error("with no whitelist, all IATAs should be allowed")
}
if !cfg.IsObserverIATAAllowed("") {
t.Error("with no whitelist, even empty IATA should be allowed")
}
}
func TestObserverIATAWhitelistJSON(t *testing.T) {
json := `{
"dbPath": "test.db",
"observerIATAWhitelist": ["ARN", "GOT"]
}`
tmp := t.TempDir() + "/config.json"
os.WriteFile(tmp, []byte(json), 0644)
cfg, err := LoadConfig(tmp)
if err != nil {
t.Fatal(err)
}
if len(cfg.ObserverIATAWhitelist) != 2 {
t.Fatalf("expected 2 entries, got %d", len(cfg.ObserverIATAWhitelist))
}
if !cfg.IsObserverIATAAllowed("ARN") {
t.Error("ARN should be allowed after loading from JSON")
}
}
func TestMQTTSourceRegionField(t *testing.T) {
dir := t.TempDir()
cfgPath := filepath.Join(dir, "config.json")
os.WriteFile(cfgPath, []byte(`{
"dbPath": "/tmp/test.db",
"mqttSources": [
{"name": "cascadia", "broker": "tcp://localhost:1883", "topics": ["meshcore/#"], "region": "PDX"}
]
}`), 0o644)
cfg, err := LoadConfig(cfgPath)
if err != nil {
t.Fatal(err)
}
if cfg.MQTTSources[0].Region != "PDX" {
t.Fatalf("expected region PDX, got %q", cfg.MQTTSources[0].Region)
}
}
// TestResolvedSourcesSchemeMapping verifies that mqtt:// and mqtts:// are translated
// to the paho-native tcp:// and ssl:// schemes, while ws:// and wss:// pass through
// unchanged (paho handles WebSocket connections natively).
func TestResolvedSourcesSchemeMapping(t *testing.T) {
tests := []struct {
input string
want string
}{
{"mqtt://host:1883", "tcp://host:1883"},
{"mqtts://host:8883", "ssl://host:8883"},
{"tcp://host:1883", "tcp://host:1883"},
{"ssl://host:8883", "ssl://host:8883"},
{"ws://host:9001", "ws://host:9001"},
{"wss://host:9001", "wss://host:9001"},
{"ws://host:9001/mqtt", "ws://host:9001/mqtt"},
{"wss://host:9001/mqtt", "wss://host:9001/mqtt"},
}
for _, tt := range tests {
cfg := &Config{
MQTTSources: []MQTTSource{
{Name: "test", Broker: tt.input, Topics: []string{"meshcore/#"}},
},
}
sources := cfg.ResolvedSources()
if got := sources[0].Broker; got != tt.want {
t.Errorf("ResolvedSources(%q) = %q, want %q", tt.input, got, tt.want)
}
}
}
// TestLoadConfigWSSource verifies that a WebSocket MQTT source round-trips through
// LoadConfig correctly — username/password preserved, scheme unchanged.
func TestLoadConfigWSSource(t *testing.T) {
t.Setenv("DB_PATH", "")
t.Setenv("MQTT_BROKER", "")
dir := t.TempDir()
cfgPath := filepath.Join(dir, "config.json")
os.WriteFile(cfgPath, []byte(`{
"dbPath": "test.db",
"mqttSources": [
{
"name": "local-tcp",
"broker": "mqtt://localhost:1883",
"topics": ["meshcore/#"]
},
{
"name": "wsmqtt-ws",
"broker": "wss://wsmqtt.example.com/mqtt",
"username": "corescope",
"password": "s3cr3t",
"topics": ["meshcore/#"]
}
]
}`), 0o644)
cfg, err := LoadConfig(cfgPath)
if err != nil {
t.Fatal(err)
}
if len(cfg.MQTTSources) != 2 {
t.Fatalf("mqttSources len=%d, want 2", len(cfg.MQTTSources))
}
tcp := cfg.MQTTSources[0]
if tcp.Name != "local-tcp" {
t.Errorf("name=%s, want local-tcp", tcp.Name)
}
ws := cfg.MQTTSources[1]
if ws.Name != "wsmqtt-ws" {
t.Errorf("name=%s, want wsmqtt-ws", ws.Name)
}
if ws.Broker != "wss://wsmqtt.example.com/mqtt" {
t.Errorf("broker=%s, want wss://wsmqtt.example.com/mqtt", ws.Broker)
}
if ws.Username != "corescope" {
t.Errorf("username=%s, want corescope", ws.Username)
}
if ws.Password != "s3cr3t" {
t.Errorf("password=%s, want s3cr3t", ws.Password)
}
sources := cfg.ResolvedSources()
if sources[1].Broker != "wss://wsmqtt.example.com/mqtt" {
t.Errorf("ResolvedSources wss broker=%s, want unchanged", sources[1].Broker)
}
}