From 594aff7cf033d2541cfb78309d3d39666dc5652d Mon Sep 17 00:00:00 2001 From: you Date: Sun, 5 Apr 2026 21:46:19 +0000 Subject: [PATCH] feat: zero-config defaults + deployment docs (M3-M4, #610) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit M3 — Sensible defaults: - Ingestor starts with no config.json (zero-config mode) - Missing config file logs a warning instead of crashing - Default MQTT broker: mqtt://localhost:1883 when no sources configured - Server already had sensible defaults (port 3000, etc.) M4 — Documentation: - Add comprehensive docs/deployment.md (system requirements, Docker, MQTT setup, TLS, monitoring, backup/restore, troubleshooting) - Update README.md with pre-built image quickstart - Link to DEPLOY.md and docs/deployment.md --- README.md | 21 +- cmd/ingestor/config.go | 29 ++- cmd/ingestor/config_test.go | 26 ++- cmd/ingestor/main.go | 3 - docs/deployment.md | 416 ++++++++++++++++++++++++++++++++++++ 5 files changed, 477 insertions(+), 18 deletions(-) create mode 100644 docs/deployment.md diff --git a/README.md b/README.md index fedc4502..be49cf0a 100644 --- a/README.md +++ b/README.md @@ -74,9 +74,24 @@ Full experience on your phone — proper touch controls, iOS safe area support, ## Quick Start -### Docker (Recommended) +### Pre-built Image (Recommended) -No Go installation needed — everything builds inside the container. +No build step required — just run: + +```bash +docker run -d --name corescope \ + -p 80:80 \ + -v corescope-data:/app/data \ + -e DISABLE_CADDY=true \ + ghcr.io/kpa-clawbot/corescope:latest +``` + +Open `http://localhost` — done. No config file needed; CoreScope starts with sensible defaults. + +See [DEPLOY.md](DEPLOY.md) for image tags, Docker Compose, and migration from `manage.sh`. +See [docs/deployment.md](docs/deployment.md) for the full deployment guide — MQTT setup, HTTPS options, backups, monitoring, and troubleshooting. + +### Build from Source ```bash git clone https://github.com/Kpa-clawbot/CoreScope.git @@ -95,8 +110,6 @@ The setup wizard walks you through config, domain, HTTPS, build, and run. ./manage.sh help # All commands ``` -See [docs/DEPLOYMENT.md](docs/DEPLOYMENT.md) for the full deployment guide — HTTPS options (auto cert, bring your own, Cloudflare Tunnel), MQTT security, backups, and troubleshooting. - ### Configure Copy `config.example.json` to `config.json` and edit: diff --git a/cmd/ingestor/config.go b/cmd/ingestor/config.go index 4cda02aa..41e65c2f 100644 --- a/cmd/ingestor/config.go +++ b/cmd/ingestor/config.go @@ -3,6 +3,7 @@ package main import ( "encoding/json" "fmt" + "log" "os" "strings" @@ -79,15 +80,21 @@ func (c *Config) NodeDaysOrDefault() int { } // LoadConfig reads configuration from a JSON file, with env var overrides. +// If the config file does not exist, sensible defaults are used (zero-config startup). func LoadConfig(path string) (*Config, error) { + var cfg Config + data, err := os.ReadFile(path) if err != nil { - return nil, fmt.Errorf("reading config %s: %w", path, err) - } - - var cfg Config - if err := json.Unmarshal(data, &cfg); err != nil { - return nil, fmt.Errorf("parsing config %s: %w", path, err) + if !os.IsNotExist(err) { + return nil, fmt.Errorf("reading config %s: %w", path, err) + } + // Config file doesn't exist — use defaults (zero-config mode) + log.Printf("config file %s not found, using sensible defaults", path) + } else { + if err := json.Unmarshal(data, &cfg); err != nil { + return nil, fmt.Errorf("parsing config %s: %w", path, err) + } } // Env var overrides @@ -121,6 +128,16 @@ func LoadConfig(path string) (*Config, error) { }} } + // Default MQTT source: connect to localhost broker when no sources configured + if len(cfg.MQTTSources) == 0 { + cfg.MQTTSources = []MQTTSource{{ + Name: "local", + Broker: "mqtt://localhost:1883", + Topics: []string{"meshcore/#"}, + }} + log.Printf("no MQTT sources configured, defaulting to mqtt://localhost:1883") + } + return &cfg, nil } diff --git a/cmd/ingestor/config_test.go b/cmd/ingestor/config_test.go index baef1a4d..76b76f10 100644 --- a/cmd/ingestor/config_test.go +++ b/cmd/ingestor/config_test.go @@ -32,9 +32,25 @@ func TestLoadConfigValidJSON(t *testing.T) { } func TestLoadConfigMissingFile(t *testing.T) { - _, err := LoadConfig("/nonexistent/path/config.json") - if err == nil { - t.Error("expected error for missing file") + t.Setenv("DB_PATH", "") + t.Setenv("MQTT_BROKER", "") + + cfg, err := LoadConfig("/nonexistent/path/config.json") + if err != nil { + t.Fatalf("missing config should not error (zero-config mode), got: %v", err) + } + if cfg.DBPath != "data/meshcore.db" { + t.Errorf("dbPath=%s, want data/meshcore.db", cfg.DBPath) + } + // Should default to localhost MQTT + if len(cfg.MQTTSources) != 1 { + t.Fatalf("mqttSources len=%d, want 1", len(cfg.MQTTSources)) + } + if cfg.MQTTSources[0].Broker != "mqtt://localhost:1883" { + t.Errorf("default broker=%s, want mqtt://localhost:1883", cfg.MQTTSources[0].Broker) + } + if cfg.MQTTSources[0].Name != "local" { + t.Errorf("default source name=%s, want local", cfg.MQTTSources[0].Name) } } @@ -196,8 +212,8 @@ func TestLoadConfigLegacyMQTTEmptyBroker(t *testing.T) { if err != nil { t.Fatal(err) } - if len(cfg.MQTTSources) != 0 { - t.Errorf("mqttSources should be empty when legacy broker is empty, got %d", len(cfg.MQTTSources)) + if len(cfg.MQTTSources) != 1 || cfg.MQTTSources[0].Name != "local" { + t.Errorf("mqttSources should default to local broker when legacy broker is empty, got %v", cfg.MQTTSources) } } diff --git a/cmd/ingestor/main.go b/cmd/ingestor/main.go index ee5bf82d..3ee0d232 100644 --- a/cmd/ingestor/main.go +++ b/cmd/ingestor/main.go @@ -49,9 +49,6 @@ func main() { } sources := cfg.ResolvedSources() - if len(sources) == 0 { - log.Fatal("no MQTT sources configured — set mqttSources in config or MQTT_BROKER env var") - } store, err := OpenStoreWithInterval(cfg.DBPath, cfg.MetricsSampleInterval()) if err != nil { diff --git a/docs/deployment.md b/docs/deployment.md new file mode 100644 index 00000000..9aaa8bf7 --- /dev/null +++ b/docs/deployment.md @@ -0,0 +1,416 @@ +# CoreScope Deployment Guide + +Comprehensive guide to deploying and operating CoreScope. For a quick start, see [DEPLOY.md](../DEPLOY.md). + +## Table of Contents + +- [System Requirements](#system-requirements) +- [Docker Deployment](#docker-deployment) +- [Configuration Reference](#configuration-reference) +- [MQTT Setup](#mqtt-setup) +- [TLS / HTTPS](#tls--https) +- [Monitoring & Health Checks](#monitoring--health-checks) +- [Backup & Restore](#backup--restore) +- [Troubleshooting](#troubleshooting) + +--- + +## System Requirements + +| Resource | Minimum | Recommended | +|----------|---------|-------------| +| RAM | 256 MB | 512 MB+ | +| Disk | 500 MB (image + DB) | 2 GB+ for long-term data | +| CPU | 1 core | 2+ cores | +| Architecture | `linux/amd64`, `linux/arm64` | — | +| Docker | 20.10+ | Latest stable | + +CoreScope runs well on Raspberry Pi 4/5 (ARM64). The Go server uses ~300 MB RAM for 56K+ packets. + +--- + +## Docker Deployment + +### Quick Start (one command) + +```bash +docker run -d --name corescope \ + -p 80:80 \ + -v corescope-data:/app/data \ + -e DISABLE_CADDY=true \ + ghcr.io/kpa-clawbot/corescope:latest +``` + +Open `http://localhost` — you'll see an empty dashboard ready to receive packets. + +No `config.json` is required. The server starts with sensible defaults: +- HTTP on port 3000 (Caddy proxies port 80 → 3000 internally) +- Internal Mosquitto MQTT broker on port 1883 +- Ingestor connects to `mqtt://localhost:1883` automatically +- SQLite database at `/app/data/meshcore.db` + +### Docker Compose (recommended for production) + +Download the example compose file: + +```bash +curl -sL https://raw.githubusercontent.com/Kpa-clawbot/CoreScope/master/docker-compose.example.yml \ + -o docker-compose.yml +docker compose up -d +``` + +#### Compose environment variables + +| Variable | Default | Description | +|----------|---------|-------------| +| `HTTP_PORT` | `80` | Host port for the web UI | +| `DATA_DIR` | `./data` | Host path for persistent data | +| `DISABLE_CADDY` | `false` | Set `true` when behind a reverse proxy | +| `DISABLE_MOSQUITTO` | `false` | Set `true` to use an external MQTT broker | + +### Image tags + +| Tag | Use case | +|-----|----------| +| `v3.4.1` | Pinned release — recommended for production | +| `v3.4` | Latest patch in the v3.4.x series | +| `v3` | Latest minor+patch in v3.x | +| `latest` | Latest release tag | +| `edge` | Built from master on every push — unstable | + +### Updating + +```bash +docker compose pull +docker compose up -d +``` + +For `docker run` users: + +```bash +docker pull ghcr.io/kpa-clawbot/corescope:latest +docker stop corescope && docker rm corescope +docker run -d --name corescope ... # same flags as before +``` + +Data is preserved in the volume — updates are non-destructive. + +--- + +## Configuration Reference + +CoreScope uses a layered configuration system (highest priority wins): + +1. **Environment variables** — `MQTT_BROKER`, `DB_PATH`, etc. +2. **`/app/data/config.json`** — full config file (volume-mounted) +3. **Built-in defaults** — work out of the box with no config + +### Environment variable overrides + +| Variable | Default | Description | +|----------|---------|-------------| +| `MQTT_BROKER` | `mqtt://localhost:1883` | MQTT broker URL (overrides config file) | +| `MQTT_TOPIC` | `meshcore/#` | MQTT topic subscription pattern | +| `DB_PATH` | `data/meshcore.db` | SQLite database path | +| `DISABLE_CADDY` | `false` | Skip the internal Caddy reverse proxy | +| `DISABLE_MOSQUITTO` | `false` | Skip the internal Mosquitto broker | + +### config.json + +For advanced configuration, create a `config.json` and mount it at `/app/data/config.json`: + +```bash +docker run -d --name corescope \ + -p 80:80 \ + -v corescope-data:/app/data \ + -v ./config.json:/app/data/config.json:ro \ + ghcr.io/kpa-clawbot/corescope:latest +``` + +See `config.example.json` in the repository for all available options including: +- MQTT sources (multiple brokers) +- Channel encryption keys +- Branding and theming +- Health thresholds +- Region filters +- Retention policies +- Geo-filtering + +--- + +## MQTT Setup + +CoreScope receives MeshCore packets via MQTT. The container ships with an internal Mosquitto broker — no setup needed for basic use. + +### Internal broker (default) + +The built-in Mosquitto broker listens on port 1883 inside the container. Point your MeshCore gateways at it: + +```bash +# Expose MQTT port for external gateways +docker run -d --name corescope \ + -p 80:80 -p 1883:1883 \ + -v corescope-data:/app/data \ + -e DISABLE_CADDY=true \ + ghcr.io/kpa-clawbot/corescope:latest +``` + +### External broker + +To use your own MQTT broker (Mosquitto, EMQX, HiveMQ, etc.): + +1. Disable the internal broker: + ```bash + -e DISABLE_MOSQUITTO=true + ``` + +2. Point the ingestor at your broker: + ```bash + -e MQTT_BROKER=mqtt://your-broker:1883 + ``` + + Or via `config.json`: + ```json + { + "mqttSources": [ + { + "name": "my-broker", + "broker": "mqtt://your-broker:1883", + "username": "user", + "password": "pass", + "topics": ["meshcore/#"] + } + ] + } + ``` + +### Multiple brokers + +CoreScope can connect to multiple MQTT brokers simultaneously: + +```json +{ + "mqttSources": [ + { + "name": "local", + "broker": "mqtt://localhost:1883", + "topics": ["meshcore/#"] + }, + { + "name": "remote", + "broker": "mqtts://remote-broker:8883", + "username": "reader", + "password": "secret", + "topics": ["meshcore/+/+/packets"] + } + ] +} +``` + +### MQTT topic format + +MeshCore gateways typically publish to `meshcore///packets`. The default subscription `meshcore/#` catches all of them. + +--- + +## TLS / HTTPS + +### Option 1: External reverse proxy (recommended) + +Run CoreScope with `DISABLE_CADDY=true` and place nginx, Traefik, or Cloudflare Tunnel in front: + +```nginx +# nginx example +server { + listen 443 ssl; + server_name corescope.example.com; + + ssl_certificate /etc/ssl/certs/corescope.pem; + ssl_certificate_key /etc/ssl/private/corescope.key; + + location / { + proxy_pass http://localhost:80; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host $host; + } +} +``` + +The `Upgrade` and `Connection` headers are required for WebSocket support. + +### Option 2: Built-in Caddy (auto-TLS) + +The container includes Caddy for automatic Let's Encrypt certificates: + +1. Create a Caddyfile: + ``` + corescope.example.com { + reverse_proxy localhost:3000 + } + ``` + +2. Mount it and expose TLS ports: + ```bash + docker run -d --name corescope \ + -p 80:80 -p 443:443 \ + -v corescope-data:/app/data \ + -v caddy-certs:/data/caddy \ + -v ./Caddyfile:/etc/caddy/Caddyfile:ro \ + ghcr.io/kpa-clawbot/corescope:latest + ``` + +Caddy handles certificate issuance and renewal automatically. + +--- + +## Monitoring & Health Checks + +### Docker health check + +The container includes a built-in health check that hits `/api/stats`: + +```bash +docker inspect --format='{{.State.Health.Status}}' corescope +``` + +Docker reports `healthy` or `unhealthy` automatically. The check runs every 30 seconds. + +### Manual health check + +```bash +curl -f http://localhost/api/stats +``` + +Returns JSON with packet counts, node counts, and uptime: + +```json +{ + "totalPackets": 56234, + "activeNodes": 142, + "uptimeSeconds": 86400, + "mqttConnected": true +} +``` + +### Log monitoring + +```bash +# All logs +docker compose logs -f + +# Server only +docker compose logs -f | grep '\[server\]' + +# Ingestor only +docker compose logs -f | grep '\[ingestor\]' +``` + +### Resource monitoring + +```bash +docker stats corescope +``` + +--- + +## Backup & Restore + +### Backup + +All persistent data lives in `/app/data`. The critical file is the SQLite database: + +```bash +# Copy from the Docker volume +docker cp corescope:/app/data/meshcore.db ./backup-$(date +%Y%m%d).db + +# Or if using a bind mount +cp ./data/meshcore.db ./backup-$(date +%Y%m%d).db +``` + +Optional files to back up: +- `config.json` — custom configuration +- `theme.json` — custom theme/branding + +### Restore + +```bash +# Stop the container +docker stop corescope + +# Replace the database +docker cp ./backup.db corescope:/app/data/meshcore.db + +# Restart +docker start corescope +``` + +### Automated backups + +```bash +# cron: daily backup at 3 AM, keep 7 days +0 3 * * * docker cp corescope:/app/data/meshcore.db /backups/corescope-$(date +\%Y\%m\%d).db && find /backups -name "corescope-*.db" -mtime +7 -delete +``` + +--- + +## Troubleshooting + +### Container starts but dashboard is empty + +This is normal on first start with no MQTT sources configured. The dashboard shows data once packets arrive via MQTT. Either: +- Point a MeshCore gateway at the container's MQTT broker (port 1883) +- Configure an external MQTT source in `config.json` + +### "no MQTT connections established" in logs + +The ingestor couldn't connect to any MQTT broker. Check: +1. Is the internal Mosquitto running? (`DISABLE_MOSQUITTO` should be `false`) +2. Is the external broker reachable? Test with `mosquitto_sub -h broker -t meshcore/#` +3. Are credentials correct in `config.json`? + +### WebSocket disconnects / real-time updates stop + +If behind a reverse proxy, ensure WebSocket upgrade headers are forwarded: +```nginx +proxy_http_version 1.1; +proxy_set_header Upgrade $http_upgrade; +proxy_set_header Connection "upgrade"; +``` + +Also check proxy timeouts — set them to at least 300s for long-lived WebSocket connections. + +### High memory usage + +The in-memory packet store grows with retained packets. Configure retention limits in `config.json`: + +```json +{ + "packetStore": { + "retentionHours": 24, + "maxMemoryMB": 512 + }, + "retention": { + "nodeDays": 7, + "packetDays": 30 + } +} +``` + +### Database locked errors + +SQLite doesn't support concurrent writers well. Ensure only one CoreScope instance accesses the database file. If running multiple containers, each needs its own database. + +### Container unhealthy + +Check logs: `docker compose logs --tail 50`. Common causes: +- Port 3000 already in use inside the container +- Database file permissions (must be writable by the container user) +- Corrupted database — restore from backup + +### ARM / Raspberry Pi issues + +- Use `linux/arm64` images (Pi 4 and 5). Pi 3 (armv7) is not supported. +- First pull may be slow — the multi-arch manifest selects the right image automatically. +- If memory is tight, set `packetStore.maxMemoryMB` to limit RAM usage.