mirror of
https://github.com/Kpa-clawbot/meshcore-analyzer.git
synced 2026-06-06 14:51:42 +00:00
29432d4fe0
## Summary
CoreScope's ingestor already supports WebSocket MQTT connections today —
`paho.mqtt.golang` v1.5.0 handles `ws://` and `wss://` natively via
gorilla/websocket. However this support was **undocumented, untested,
and had a TLS gap** for `wss://` connections.
This PR closes those gaps without any breaking changes.
## Changes
### `cmd/ingestor/config.go`
- Added godoc comment to `ResolvedSources()` explaining all four
supported schemes and which ones require translation vs. pass-through
- `ws://` and `wss://` explicitly documented as native paho schemes
requiring no mapping
### `cmd/ingestor/main.go`
- Extended TLS config to cover `wss://` in addition to `ssl://`
- Before: `wss://` connections would use paho's default TLS (no explicit
`tls.Config` set), which works for valid certs but doesn't apply the
same predictable setup as `ssl://`
- After: both `ssl://` and `wss://` get `tls.Config{}` (system CA pool),
matching behavior; `rejectUnauthorized: false` still works for
self-signed certs on both schemes
### `cmd/ingestor/config_test.go`
Two new tests:
- `TestResolvedSourcesSchemeMapping`: validates all six scheme
variations (`mqtt://`, `mqtts://`, `tcp://`, `ssl://`, `ws://`,
`wss://`) including paths like `wss://host/mqtt`
- `TestLoadConfigWSSource`: full round-trip of a dual-source config (TCP
+ wss:// with username/password), verifies scheme unchanged through
`LoadConfig` and `ResolvedSources`
### `config.example.json`
- Added `wsmqtt` example entry showing `wss://` with username/password
- Updated `_comment_mqttSources` to enumerate all supported schemes:
`mqtt://`, `mqtts://`, `ws://`, `wss://`
## Motivation
We run
[meshcore-mqtt-broker](https://github.com/andrewjfreyer/meshcore-mqtt-broker)
(a WebSocket MQTT bridge with JWT auth) alongside Mosquitto, and
subscribe to both via `mqttSources`. The dual-source config works in
production but nothing in the docs or example config made this
discoverable for other operators.
## Testing
```
cd cmd/ingestor && go test ./...
ok github.com/corescope/ingestor 1.568s
```
All existing tests pass. Two new tests added.
## No breaking changes
- Existing configs: no change in behavior
- `ws://` / `wss://` configs that were already working: same behavior +
explicit TLS setup for `wss://`
487 lines
13 KiB
Go
487 lines
13 KiB
Go
package main
|
|
|
|
import (
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
)
|
|
|
|
func TestLoadConfigValidJSON(t *testing.T) {
|
|
dir := t.TempDir()
|
|
cfgPath := filepath.Join(dir, "config.json")
|
|
os.WriteFile(cfgPath, []byte(`{
|
|
"dbPath": "/tmp/test.db",
|
|
"mqttSources": [
|
|
{"name": "s1", "broker": "tcp://localhost:1883", "topics": ["meshcore/#"]}
|
|
]
|
|
}`), 0o644)
|
|
|
|
cfg, err := LoadConfig(cfgPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if cfg.DBPath != "/tmp/test.db" {
|
|
t.Errorf("dbPath=%s, want /tmp/test.db", cfg.DBPath)
|
|
}
|
|
if len(cfg.MQTTSources) != 1 {
|
|
t.Fatalf("mqttSources len=%d, want 1", len(cfg.MQTTSources))
|
|
}
|
|
if cfg.MQTTSources[0].Broker != "tcp://localhost:1883" {
|
|
t.Errorf("broker=%s", cfg.MQTTSources[0].Broker)
|
|
}
|
|
}
|
|
|
|
func TestLoadConfigMissingFile(t *testing.T) {
|
|
t.Setenv("DB_PATH", "")
|
|
t.Setenv("MQTT_BROKER", "")
|
|
|
|
cfg, err := LoadConfig("/nonexistent/path/config.json")
|
|
if err != nil {
|
|
t.Fatalf("missing config should not error (zero-config mode), got: %v", err)
|
|
}
|
|
if cfg.DBPath != "data/meshcore.db" {
|
|
t.Errorf("dbPath=%s, want data/meshcore.db", cfg.DBPath)
|
|
}
|
|
// Should default to localhost MQTT
|
|
if len(cfg.MQTTSources) != 1 {
|
|
t.Fatalf("mqttSources len=%d, want 1", len(cfg.MQTTSources))
|
|
}
|
|
if cfg.MQTTSources[0].Broker != "mqtt://localhost:1883" {
|
|
t.Errorf("default broker=%s, want mqtt://localhost:1883", cfg.MQTTSources[0].Broker)
|
|
}
|
|
if cfg.MQTTSources[0].Name != "local" {
|
|
t.Errorf("default source name=%s, want local", cfg.MQTTSources[0].Name)
|
|
}
|
|
}
|
|
|
|
func TestLoadConfigMalformedJSON(t *testing.T) {
|
|
dir := t.TempDir()
|
|
cfgPath := filepath.Join(dir, "bad.json")
|
|
os.WriteFile(cfgPath, []byte(`{not valid json`), 0o644)
|
|
|
|
_, err := LoadConfig(cfgPath)
|
|
if err == nil {
|
|
t.Error("expected error for malformed JSON")
|
|
}
|
|
}
|
|
|
|
func TestLoadConfigEnvVarDBPath(t *testing.T) {
|
|
t.Setenv("DB_PATH", "/override/db.sqlite")
|
|
t.Setenv("MQTT_BROKER", "")
|
|
|
|
dir := t.TempDir()
|
|
cfgPath := filepath.Join(dir, "config.json")
|
|
os.WriteFile(cfgPath, []byte(`{"dbPath": "original.db"}`), 0o644)
|
|
|
|
cfg, err := LoadConfig(cfgPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if cfg.DBPath != "/override/db.sqlite" {
|
|
t.Errorf("dbPath=%s, want /override/db.sqlite", cfg.DBPath)
|
|
}
|
|
}
|
|
|
|
func TestLoadConfigEnvVarMQTTBroker(t *testing.T) {
|
|
t.Setenv("MQTT_BROKER", "tcp://env-broker:1883")
|
|
t.Setenv("MQTT_TOPIC", "custom/topic")
|
|
|
|
dir := t.TempDir()
|
|
cfgPath := filepath.Join(dir, "config.json")
|
|
os.WriteFile(cfgPath, []byte(`{"dbPath": "test.db"}`), 0o644)
|
|
|
|
cfg, err := LoadConfig(cfgPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(cfg.MQTTSources) != 1 {
|
|
t.Fatalf("mqttSources len=%d, want 1", len(cfg.MQTTSources))
|
|
}
|
|
src := cfg.MQTTSources[0]
|
|
if src.Name != "env" {
|
|
t.Errorf("name=%s, want env", src.Name)
|
|
}
|
|
if src.Broker != "tcp://env-broker:1883" {
|
|
t.Errorf("broker=%s", src.Broker)
|
|
}
|
|
if len(src.Topics) != 1 || src.Topics[0] != "custom/topic" {
|
|
t.Errorf("topics=%v, want [custom/topic]", src.Topics)
|
|
}
|
|
}
|
|
|
|
func TestLoadConfigEnvVarMQTTBrokerDefaultTopic(t *testing.T) {
|
|
t.Setenv("MQTT_BROKER", "tcp://env-broker:1883")
|
|
t.Setenv("MQTT_TOPIC", "")
|
|
|
|
dir := t.TempDir()
|
|
cfgPath := filepath.Join(dir, "config.json")
|
|
os.WriteFile(cfgPath, []byte(`{"dbPath": "test.db"}`), 0o644)
|
|
|
|
cfg, err := LoadConfig(cfgPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if cfg.MQTTSources[0].Topics[0] != "meshcore/#" {
|
|
t.Errorf("default topic=%s, want meshcore/#", cfg.MQTTSources[0].Topics[0])
|
|
}
|
|
}
|
|
|
|
func TestLoadConfigLegacyMQTT(t *testing.T) {
|
|
t.Setenv("DB_PATH", "")
|
|
t.Setenv("MQTT_BROKER", "")
|
|
|
|
dir := t.TempDir()
|
|
cfgPath := filepath.Join(dir, "config.json")
|
|
os.WriteFile(cfgPath, []byte(`{
|
|
"dbPath": "test.db",
|
|
"mqtt": {"broker": "tcp://legacy:1883", "topic": "old/topic"}
|
|
}`), 0o644)
|
|
|
|
cfg, err := LoadConfig(cfgPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(cfg.MQTTSources) != 1 {
|
|
t.Fatalf("mqttSources len=%d, want 1", len(cfg.MQTTSources))
|
|
}
|
|
src := cfg.MQTTSources[0]
|
|
if src.Name != "default" {
|
|
t.Errorf("name=%s, want default", src.Name)
|
|
}
|
|
if src.Broker != "tcp://legacy:1883" {
|
|
t.Errorf("broker=%s", src.Broker)
|
|
}
|
|
if len(src.Topics) != 2 || src.Topics[0] != "old/topic" || src.Topics[1] != "meshcore/#" {
|
|
t.Errorf("topics=%v, want [old/topic meshcore/#]", src.Topics)
|
|
}
|
|
}
|
|
|
|
func TestLoadConfigLegacyMQTTNotUsedWhenSourcesExist(t *testing.T) {
|
|
t.Setenv("DB_PATH", "")
|
|
t.Setenv("MQTT_BROKER", "")
|
|
|
|
dir := t.TempDir()
|
|
cfgPath := filepath.Join(dir, "config.json")
|
|
os.WriteFile(cfgPath, []byte(`{
|
|
"dbPath": "test.db",
|
|
"mqtt": {"broker": "tcp://legacy:1883", "topic": "old/topic"},
|
|
"mqttSources": [{"name": "modern", "broker": "tcp://modern:1883", "topics": ["m/#"]}]
|
|
}`), 0o644)
|
|
|
|
cfg, err := LoadConfig(cfgPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(cfg.MQTTSources) != 1 {
|
|
t.Fatalf("mqttSources len=%d, want 1", len(cfg.MQTTSources))
|
|
}
|
|
if cfg.MQTTSources[0].Name != "modern" {
|
|
t.Errorf("should use modern source, got name=%s", cfg.MQTTSources[0].Name)
|
|
}
|
|
}
|
|
|
|
func TestLoadConfigDefaultDBPath(t *testing.T) {
|
|
t.Setenv("DB_PATH", "")
|
|
t.Setenv("MQTT_BROKER", "")
|
|
|
|
dir := t.TempDir()
|
|
cfgPath := filepath.Join(dir, "config.json")
|
|
os.WriteFile(cfgPath, []byte(`{}`), 0o644)
|
|
|
|
cfg, err := LoadConfig(cfgPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if cfg.DBPath != "data/meshcore.db" {
|
|
t.Errorf("dbPath=%s, want data/meshcore.db", cfg.DBPath)
|
|
}
|
|
}
|
|
|
|
func TestLoadConfigLegacyMQTTEmptyBroker(t *testing.T) {
|
|
t.Setenv("DB_PATH", "")
|
|
t.Setenv("MQTT_BROKER", "")
|
|
|
|
dir := t.TempDir()
|
|
cfgPath := filepath.Join(dir, "config.json")
|
|
os.WriteFile(cfgPath, []byte(`{
|
|
"dbPath": "test.db",
|
|
"mqtt": {"broker": "", "topic": "t"}
|
|
}`), 0o644)
|
|
|
|
cfg, err := LoadConfig(cfgPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(cfg.MQTTSources) != 1 || cfg.MQTTSources[0].Name != "local" {
|
|
t.Errorf("mqttSources should default to local broker when legacy broker is empty, got %v", cfg.MQTTSources)
|
|
}
|
|
}
|
|
|
|
func TestResolvedSources(t *testing.T) {
|
|
cfg := &Config{
|
|
MQTTSources: []MQTTSource{
|
|
{Name: "a", Broker: "tcp://a:1883"},
|
|
{Name: "b", Broker: "tcp://b:1883"},
|
|
},
|
|
}
|
|
sources := cfg.ResolvedSources()
|
|
if len(sources) != 2 {
|
|
t.Fatalf("len=%d, want 2", len(sources))
|
|
}
|
|
if sources[0].Name != "a" || sources[1].Name != "b" {
|
|
t.Errorf("sources=%v", sources)
|
|
}
|
|
}
|
|
|
|
func TestResolvedSourcesEmpty(t *testing.T) {
|
|
cfg := &Config{}
|
|
sources := cfg.ResolvedSources()
|
|
if len(sources) != 0 {
|
|
t.Errorf("len=%d, want 0", len(sources))
|
|
}
|
|
}
|
|
|
|
func TestLoadConfigWithAllFields(t *testing.T) {
|
|
t.Setenv("DB_PATH", "")
|
|
t.Setenv("MQTT_BROKER", "")
|
|
|
|
dir := t.TempDir()
|
|
cfgPath := filepath.Join(dir, "config.json")
|
|
reject := false
|
|
_ = reject
|
|
os.WriteFile(cfgPath, []byte(`{
|
|
"dbPath": "mydb.db",
|
|
"logLevel": "debug",
|
|
"mqttSources": [{
|
|
"name": "full",
|
|
"broker": "tcp://full:1883",
|
|
"username": "user1",
|
|
"password": "pass1",
|
|
"rejectUnauthorized": false,
|
|
"topics": ["a/#", "b/#"],
|
|
"iataFilter": ["SJC", "LAX"]
|
|
}]
|
|
}`), 0o644)
|
|
|
|
cfg, err := LoadConfig(cfgPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if cfg.LogLevel != "debug" {
|
|
t.Errorf("logLevel=%s, want debug", cfg.LogLevel)
|
|
}
|
|
src := cfg.MQTTSources[0]
|
|
if src.Username != "user1" {
|
|
t.Errorf("username=%s", src.Username)
|
|
}
|
|
if src.Password != "pass1" {
|
|
t.Errorf("password=%s", src.Password)
|
|
}
|
|
if src.RejectUnauthorized == nil || *src.RejectUnauthorized != false {
|
|
t.Error("rejectUnauthorized should be false")
|
|
}
|
|
if len(src.IATAFilter) != 2 || src.IATAFilter[0] != "SJC" {
|
|
t.Errorf("iataFilter=%v", src.IATAFilter)
|
|
}
|
|
}
|
|
|
|
func TestConnectTimeoutOrDefault(t *testing.T) {
|
|
// Default when unset
|
|
s := MQTTSource{}
|
|
if got := s.ConnectTimeoutOrDefault(); got != 30 {
|
|
t.Errorf("default: got %d, want 30", got)
|
|
}
|
|
|
|
// Custom value
|
|
s.ConnectTimeoutSec = 5
|
|
if got := s.ConnectTimeoutOrDefault(); got != 5 {
|
|
t.Errorf("custom: got %d, want 5", got)
|
|
}
|
|
|
|
// Zero treated as unset
|
|
s.ConnectTimeoutSec = 0
|
|
if got := s.ConnectTimeoutOrDefault(); got != 30 {
|
|
t.Errorf("zero: got %d, want 30", got)
|
|
}
|
|
}
|
|
|
|
func TestConnectTimeoutFromJSON(t *testing.T) {
|
|
dir := t.TempDir()
|
|
cfgPath := dir + "/config.json"
|
|
os.WriteFile(cfgPath, []byte(`{"mqttSources":[{"name":"s1","broker":"tcp://b:1883","topics":["#"],"connectTimeoutSec":5}]}`), 0644)
|
|
cfg, err := LoadConfig(cfgPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := cfg.MQTTSources[0].ConnectTimeoutOrDefault(); got != 5 {
|
|
t.Errorf("from JSON: got %d, want 5", got)
|
|
}
|
|
}
|
|
|
|
func TestObserverIATAWhitelist(t *testing.T) {
|
|
// Config with whitelist set
|
|
cfg := Config{
|
|
ObserverIATAWhitelist: []string{"ARN", "got"},
|
|
}
|
|
|
|
// Matching (case-insensitive)
|
|
if !cfg.IsObserverIATAAllowed("ARN") {
|
|
t.Error("ARN should be allowed")
|
|
}
|
|
if !cfg.IsObserverIATAAllowed("arn") {
|
|
t.Error("arn (lowercase) should be allowed")
|
|
}
|
|
if !cfg.IsObserverIATAAllowed("GOT") {
|
|
t.Error("GOT should be allowed")
|
|
}
|
|
|
|
// Non-matching
|
|
if cfg.IsObserverIATAAllowed("SJC") {
|
|
t.Error("SJC should NOT be allowed")
|
|
}
|
|
|
|
// Empty string not allowed
|
|
if cfg.IsObserverIATAAllowed("") {
|
|
t.Error("empty IATA should NOT be allowed")
|
|
}
|
|
}
|
|
|
|
func TestObserverIATAWhitelistEmpty(t *testing.T) {
|
|
// No whitelist = allow all
|
|
cfg := Config{}
|
|
if !cfg.IsObserverIATAAllowed("SJC") {
|
|
t.Error("with no whitelist, all IATAs should be allowed")
|
|
}
|
|
if !cfg.IsObserverIATAAllowed("") {
|
|
t.Error("with no whitelist, even empty IATA should be allowed")
|
|
}
|
|
}
|
|
|
|
func TestObserverIATAWhitelistJSON(t *testing.T) {
|
|
json := `{
|
|
"dbPath": "test.db",
|
|
"observerIATAWhitelist": ["ARN", "GOT"]
|
|
}`
|
|
tmp := t.TempDir() + "/config.json"
|
|
os.WriteFile(tmp, []byte(json), 0644)
|
|
cfg, err := LoadConfig(tmp)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(cfg.ObserverIATAWhitelist) != 2 {
|
|
t.Fatalf("expected 2 entries, got %d", len(cfg.ObserverIATAWhitelist))
|
|
}
|
|
if !cfg.IsObserverIATAAllowed("ARN") {
|
|
t.Error("ARN should be allowed after loading from JSON")
|
|
}
|
|
}
|
|
|
|
func TestMQTTSourceRegionField(t *testing.T) {
|
|
dir := t.TempDir()
|
|
cfgPath := filepath.Join(dir, "config.json")
|
|
os.WriteFile(cfgPath, []byte(`{
|
|
"dbPath": "/tmp/test.db",
|
|
"mqttSources": [
|
|
{"name": "cascadia", "broker": "tcp://localhost:1883", "topics": ["meshcore/#"], "region": "PDX"}
|
|
]
|
|
}`), 0o644)
|
|
|
|
cfg, err := LoadConfig(cfgPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if cfg.MQTTSources[0].Region != "PDX" {
|
|
t.Fatalf("expected region PDX, got %q", cfg.MQTTSources[0].Region)
|
|
}
|
|
}
|
|
|
|
// TestResolvedSourcesSchemeMapping verifies that mqtt:// and mqtts:// are translated
|
|
// to the paho-native tcp:// and ssl:// schemes, while ws:// and wss:// pass through
|
|
// unchanged (paho handles WebSocket connections natively).
|
|
func TestResolvedSourcesSchemeMapping(t *testing.T) {
|
|
tests := []struct {
|
|
input string
|
|
want string
|
|
}{
|
|
{"mqtt://host:1883", "tcp://host:1883"},
|
|
{"mqtts://host:8883", "ssl://host:8883"},
|
|
{"tcp://host:1883", "tcp://host:1883"},
|
|
{"ssl://host:8883", "ssl://host:8883"},
|
|
{"ws://host:9001", "ws://host:9001"},
|
|
{"wss://host:9001", "wss://host:9001"},
|
|
{"ws://host:9001/mqtt", "ws://host:9001/mqtt"},
|
|
{"wss://host:9001/mqtt", "wss://host:9001/mqtt"},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
cfg := &Config{
|
|
MQTTSources: []MQTTSource{
|
|
{Name: "test", Broker: tt.input, Topics: []string{"meshcore/#"}},
|
|
},
|
|
}
|
|
sources := cfg.ResolvedSources()
|
|
if got := sources[0].Broker; got != tt.want {
|
|
t.Errorf("ResolvedSources(%q) = %q, want %q", tt.input, got, tt.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestLoadConfigWSSource verifies that a WebSocket MQTT source round-trips through
|
|
// LoadConfig correctly — username/password preserved, scheme unchanged.
|
|
func TestLoadConfigWSSource(t *testing.T) {
|
|
t.Setenv("DB_PATH", "")
|
|
t.Setenv("MQTT_BROKER", "")
|
|
|
|
dir := t.TempDir()
|
|
cfgPath := filepath.Join(dir, "config.json")
|
|
os.WriteFile(cfgPath, []byte(`{
|
|
"dbPath": "test.db",
|
|
"mqttSources": [
|
|
{
|
|
"name": "local-tcp",
|
|
"broker": "mqtt://localhost:1883",
|
|
"topics": ["meshcore/#"]
|
|
},
|
|
{
|
|
"name": "wsmqtt-ws",
|
|
"broker": "wss://wsmqtt.example.com/mqtt",
|
|
"username": "corescope",
|
|
"password": "s3cr3t",
|
|
"topics": ["meshcore/#"]
|
|
}
|
|
]
|
|
}`), 0o644)
|
|
|
|
cfg, err := LoadConfig(cfgPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(cfg.MQTTSources) != 2 {
|
|
t.Fatalf("mqttSources len=%d, want 2", len(cfg.MQTTSources))
|
|
}
|
|
|
|
tcp := cfg.MQTTSources[0]
|
|
if tcp.Name != "local-tcp" {
|
|
t.Errorf("name=%s, want local-tcp", tcp.Name)
|
|
}
|
|
|
|
ws := cfg.MQTTSources[1]
|
|
if ws.Name != "wsmqtt-ws" {
|
|
t.Errorf("name=%s, want wsmqtt-ws", ws.Name)
|
|
}
|
|
if ws.Broker != "wss://wsmqtt.example.com/mqtt" {
|
|
t.Errorf("broker=%s, want wss://wsmqtt.example.com/mqtt", ws.Broker)
|
|
}
|
|
if ws.Username != "corescope" {
|
|
t.Errorf("username=%s, want corescope", ws.Username)
|
|
}
|
|
if ws.Password != "s3cr3t" {
|
|
t.Errorf("password=%s, want s3cr3t", ws.Password)
|
|
}
|
|
|
|
sources := cfg.ResolvedSources()
|
|
if sources[1].Broker != "wss://wsmqtt.example.com/mqtt" {
|
|
t.Errorf("ResolvedSources wss broker=%s, want unchanged", sources[1].Broker)
|
|
}
|
|
}
|