meshcore-analyzer/cmd/ingestor/config.go

package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"os"
	"strings"
	"sync"

	"github.com/meshcore-analyzer/dbconfig"
	"github.com/meshcore-analyzer/geofilter"
)

// MQTTSource represents a single MQTT broker connection.
type MQTTSource struct {
	Name               string   `json:"name"`
	Broker             string   `json:"broker"`
	Username           string   `json:"username,omitempty"`
	Password           string   `json:"password,omitempty"`
	RejectUnauthorized *bool    `json:"rejectUnauthorized,omitempty"`
	Topics             []string `json:"topics"`
	IATAFilter         []string `json:"iataFilter,omitempty"`
	ConnectTimeoutSec  int      `json:"connectTimeoutSec,omitempty"`
	Region             string   `json:"region,omitempty"`
}

// ConnectTimeoutOrDefault returns the per-source connect timeout in seconds,
// or 30 if not set (matching the WaitTimeout default from #926).
func (s MQTTSource) ConnectTimeoutOrDefault() int {
	if s.ConnectTimeoutSec > 0 {
		return s.ConnectTimeoutSec
	}
	return 30
}

// MQTTLegacy is the old single-broker config format.
type MQTTLegacy struct {
	Broker string `json:"broker"`
	Topic  string `json:"topic"`
}

// Config holds the ingestor configuration, compatible with the Node.js config.json format.
type Config struct {
	DBPath          string            `json:"dbPath"`
	MQTT            *MQTTLegacy       `json:"mqtt,omitempty"`
	MQTTSources     []MQTTSource      `json:"mqttSources,omitempty"`
	LogLevel        string            `json:"logLevel,omitempty"`
	ChannelKeysPath string            `json:"channelKeysPath,omitempty"`
	ChannelKeys     map[string]string `json:"channelKeys,omitempty"`
	HashChannels    []string          `json:"hashChannels,omitempty"`
	HashRegions     []string          `json:"hashRegions,omitempty"`
	Retention       *RetentionConfig  `json:"retention,omitempty"`
	Metrics         *MetricsConfig    `json:"metrics,omitempty"`
	GeoFilter            *GeoFilterConfig     `json:"geo_filter,omitempty"`
	ForeignAdverts       *ForeignAdvertConfig `json:"foreignAdverts,omitempty"`
	ValidateSignatures   *bool             `json:"validateSignatures,omitempty"`
	DB                   *DBConfig         `json:"db,omitempty"`

	// ObserverIATAWhitelist restricts which observer IATA regions are processed.
	// When non-empty, only observers whose IATA code (from the MQTT topic) matches
	// one of these entries are accepted. Case-insensitive. An empty list means all
	// IATA codes are allowed. This applies globally, unlike the per-source iataFilter.
	ObserverIATAWhitelist []string `json:"observerIATAWhitelist,omitempty"`

	// obsIATAWhitelistCached is the lazily-built uppercase set for O(1) lookups.
	obsIATAWhitelistCached map[string]bool
	obsIATAWhitelistOnce   sync.Once

	// ObserverBlacklist is a list of observer public keys to drop at ingest.
	// Messages from blacklisted observers are silently discarded — no DB writes,
	// no UpsertObserver, no observations, no metrics.
	ObserverBlacklist []string `json:"observerBlacklist,omitempty"`

	// obsBlacklistSetCached is the lazily-built lowercase set for O(1) lookups.
	obsBlacklistSetCached map[string]bool
	obsBlacklistOnce      sync.Once

	// NeighborEdgesMaxAgeDays controls neighbor_edges row retention
	// (#1287 — moved from cmd/server). 0 = default 5.
	NeighborEdgesMaxAgeDays int `json:"neighborEdgesMaxAgeDays,omitempty"`
}

// NeighborEdgesDaysOrDefault returns the configured pruning window or 5.
func (c *Config) NeighborEdgesDaysOrDefault() int {
	if c == nil || c.NeighborEdgesMaxAgeDays <= 0 {
		return 5
	}
	return c.NeighborEdgesMaxAgeDays
}

// GeoFilterConfig is an alias for the shared geofilter.Config type.
type GeoFilterConfig = geofilter.Config

// ForeignAdvertConfig controls how the ingestor handles ADVERTs whose GPS lies
// outside the configured geofilter polygon (#730). Modes:
//   - "flag" (default): store the advert/node and tag it foreign for visibility.
//   - "drop":           silently discard the advert (legacy behavior).
type ForeignAdvertConfig struct {
	Mode string `json:"mode,omitempty"`
}

// IsDropMode reports whether the foreign-advert config is set to "drop".
// Defaults to false ("flag" mode) when nil or unset.
func (f *ForeignAdvertConfig) IsDropMode() bool {
	if f == nil {
		return false
	}
	return strings.EqualFold(strings.TrimSpace(f.Mode), "drop")
}

// RetentionConfig controls how long stale nodes are kept before being moved to inactive_nodes.
type RetentionConfig struct {
	NodeDays     int `json:"nodeDays"`
	ObserverDays int `json:"observerDays"`
	MetricsDays  int `json:"metricsDays"`
	// PacketDays is the retention window for transmissions (#1283).
	// Ownership moved from cmd/server to cmd/ingestor; 0 disables.
	PacketDays int `json:"packetDays"`
}

// PacketDaysOrZero returns the configured retention.packetDays or 0
// (disabled) if not set.
func (c *Config) PacketDaysOrZero() int {
	if c.Retention != nil && c.Retention.PacketDays > 0 {
		return c.Retention.PacketDays
	}
	return 0
}

// MetricsConfig controls observer metrics collection.
type MetricsConfig struct {
	SampleIntervalSec int `json:"sampleIntervalSec"`
}

// DBConfig is the shared SQLite vacuum/maintenance config (#919, #921).
type DBConfig = dbconfig.DBConfig

// IncrementalVacuumPages returns the configured pages per vacuum or 1024 default.
func (c *Config) IncrementalVacuumPages() int {
	if c.DB != nil && c.DB.IncrementalVacuumPages > 0 {
		return c.DB.IncrementalVacuumPages
	}
	return 1024
}

// ShouldValidateSignatures returns true (default) unless explicitly disabled.
func (c *Config) ShouldValidateSignatures() bool {
	if c.ValidateSignatures != nil {
		return *c.ValidateSignatures
	}
	return true
}

// MetricsSampleInterval returns the configured sample interval or 300s default.
func (c *Config) MetricsSampleInterval() int {
	if c.Metrics != nil && c.Metrics.SampleIntervalSec > 0 {
		return c.Metrics.SampleIntervalSec
	}
	return 300
}

// MetricsRetentionDays returns configured metrics retention or 30 days default.
func (c *Config) MetricsRetentionDays() int {
	if c.Retention != nil && c.Retention.MetricsDays > 0 {
		return c.Retention.MetricsDays
	}
	return 30
}

// NodeDaysOrDefault returns the configured retention.nodeDays or 7 if not set.
func (c *Config) NodeDaysOrDefault() int {
	if c.Retention != nil && c.Retention.NodeDays > 0 {
		return c.Retention.NodeDays
	}
	return 7
}

// ObserverDaysOrDefault returns the configured retention.observerDays or 14 if not set.
// A value of -1 means observers are never removed.
func (c *Config) ObserverDaysOrDefault() int {
	if c.Retention != nil && c.Retention.ObserverDays != 0 {
		return c.Retention.ObserverDays
	}
	return 14
}

// IsObserverBlacklisted returns true if the given observer ID is in the observerBlacklist.
func (c *Config) IsObserverBlacklisted(id string) bool {
	if c == nil || len(c.ObserverBlacklist) == 0 {
		return false
	}
	c.obsBlacklistOnce.Do(func() {
		m := make(map[string]bool, len(c.ObserverBlacklist))
		for _, pk := range c.ObserverBlacklist {
			trimmed := strings.ToLower(strings.TrimSpace(pk))
			if trimmed != "" {
				m[trimmed] = true
			}
		}
		c.obsBlacklistSetCached = m
	})
	return c.obsBlacklistSetCached[strings.ToLower(strings.TrimSpace(id))]
}

// IsObserverIATAAllowed returns true if the given IATA code is permitted.
// When ObserverIATAWhitelist is empty, all codes are allowed.
func (c *Config) IsObserverIATAAllowed(iata string) bool {
	if c == nil || len(c.ObserverIATAWhitelist) == 0 {
		return true
	}
	c.obsIATAWhitelistOnce.Do(func() {
		m := make(map[string]bool, len(c.ObserverIATAWhitelist))
		for _, code := range c.ObserverIATAWhitelist {
			trimmed := strings.ToUpper(strings.TrimSpace(code))
			if trimmed != "" {
				m[trimmed] = true
			}
		}
		c.obsIATAWhitelistCached = m
	})
	return c.obsIATAWhitelistCached[strings.ToUpper(strings.TrimSpace(iata))]
}

// LoadConfig reads configuration from a JSON file, with env var overrides.
// If the config file does not exist, sensible defaults are used (zero-config startup).
func LoadConfig(path string) (*Config, error) {
	var cfg Config

	data, err := os.ReadFile(path)
	if err != nil {
		if !errors.Is(err, os.ErrNotExist) {
			return nil, fmt.Errorf("reading config %s: %w", path, err)
		}
		// Config file doesn't exist — use defaults (zero-config mode)
		log.Printf("config file %s not found, using sensible defaults", path)
	} else {
		if err := json.Unmarshal(data, &cfg); err != nil {
			return nil, fmt.Errorf("parsing config %s: %w", path, err)
		}
	}

	// Env var overrides
	if v := os.Getenv("DB_PATH"); v != "" {
		cfg.DBPath = v
	}
	if v := os.Getenv("MQTT_BROKER"); v != "" {
		// Single broker from env — create a source
		topic := os.Getenv("MQTT_TOPIC")
		if topic == "" {
			topic = "meshcore/#"
		}
		cfg.MQTTSources = []MQTTSource{{
			Name:   "env",
			Broker: v,
			Topics: []string{topic},
		}}
	}

	// Default DB path
	if cfg.DBPath == "" {
		cfg.DBPath = "data/meshcore.db"
	}

	// Normalize: convert legacy single mqtt config to mqttSources
	if len(cfg.MQTTSources) == 0 && cfg.MQTT != nil && cfg.MQTT.Broker != "" {
		cfg.MQTTSources = []MQTTSource{{
			Name:   "default",
			Broker: cfg.MQTT.Broker,
			Topics: []string{cfg.MQTT.Topic, "meshcore/#"},
		}}
	}

	// Default MQTT source: connect to localhost broker when no sources configured
	if len(cfg.MQTTSources) == 0 {
		cfg.MQTTSources = []MQTTSource{{
			Name:   "local",
			Broker: "mqtt://localhost:1883",
			Topics: []string{"meshcore/#"},
		}}
		log.Printf("no MQTT sources configured, defaulting to mqtt://localhost:1883")
	}

	return &cfg, nil
}

// ResolvedSources returns the final list of MQTT sources to connect to.
func (c *Config) ResolvedSources() []MQTTSource {
	for i := range c.MQTTSources {
		// paho uses tcp:// and ssl:// not mqtt:// and mqtts://
		b := c.MQTTSources[i].Broker
		if strings.HasPrefix(b, "mqtt://") {
			c.MQTTSources[i].Broker = "tcp://" + b[7:]
		} else if strings.HasPrefix(b, "mqtts://") {
			c.MQTTSources[i].Broker = "ssl://" + b[8:]
		}
	}
	return c.MQTTSources
}