From 1f83df8583e919ad79e4faba5e3905e1137a0a14 Mon Sep 17 00:00:00 2001 From: MrAlders0n <55921894+MrAlders0n@users.noreply.github.com> Date: Fri, 2 Oct 2026 21:43:03 -0400 Subject: [PATCH] docs: move operator docs to beacon-docs (#206) * docs: keep only server-specific contributor notes * docs: trim README to the code and link to beacon-docs * chore: strip tracking parameters from two airport website URLs --- .github/ISSUE_TEMPLATE/bug_report.md | 2 +- .github/ISSUE_TEMPLATE/feature_request.md | 2 +- .github/PULL_REQUEST_TEMPLATE.md | 2 +- .github/workflows/docker-publish.yml | 2 +- CODE_OF_CONDUCT.md | 10 +- CONTRIBUTING.md | 183 ++---- CONTRIBUTORS.md | 2 +- PROFILING.md | 70 --- README.md | 667 ++-------------------- SECURITY.md | 2 +- SHOULDERS.md | 46 +- config.yaml.example | 7 +- docs/backup-export.md | 136 ----- docs/packet-summaries.md | 22 - env.example | 5 +- internal/iatadb/gen/airports.csv | 4 +- 16 files changed, 162 insertions(+), 1000 deletions(-) delete mode 100644 PROFILING.md delete mode 100644 docs/backup-export.md delete mode 100644 docs/packet-summaries.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 64ec816..c7bf29c 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -34,5 +34,5 @@ paste logs here ## Additional context -Any other context — config snippets (redact credentials), MQTT broker details, +Any other context: config snippets (redact credentials), MQTT broker details, mesh topology, etc. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 433c423..f2fdb7f 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -15,7 +15,7 @@ related to an existing issue or limitation, link it here. ## Proposed solution -How you'd like it to work. API shape, config options, WS event format — as much +How you'd like it to work. API shape, config options, WS event format, as much detail as you have. ## Alternatives considered diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 16463f3..ef6d6c0 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -23,7 +23,7 @@ issues: `Closes #123` or `Related to #456`. - [ ] API changes include swagger annotations and `swag init` has been run - [ ] `docs/` is committed if swagger was regenerated - [ ] New dependencies are added to `SHOULDERS.md` -- [ ] I have read [CONTRIBUTING.md](CONTRIBUTING.md) +- [ ] I have read [CONTRIBUTING.md](../CONTRIBUTING.md) ## Testing notes diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 0cc526c..70f27ec 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -38,7 +38,7 @@ jobs: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} labels: | org.opencontainers.image.title=beacon-server - org.opencontainers.image.description=Beacon — MeshCore LoRa mesh observation server + org.opencontainers.image.description=Beacon: MeshCore LoRa mesh observation server org.opencontainers.image.licenses=AGPL-3.0-or-later flavor: | latest=false diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index c861031..e50faa7 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -25,7 +25,7 @@ Examples of behaviour that contributes to a positive environment: Examples of unacceptable behaviour: -- Hate speech or discriminatory language or jokes of any kind — including but +- Hate speech or discriminatory language or jokes of any kind, including but not limited to those targeting race, ethnicity, nationality, gender identity or expression, sexual orientation, disability, religion, age, or socioeconomic background @@ -34,7 +34,7 @@ Examples of unacceptable behaviour: - Trolling, insulting or derogatory comments, and personal or political attacks - Public or private harassment of any kind - Deliberate misgendering or use of rejected names -- Publishing others' private information — such as a physical or email address — +- Publishing others' private information, such as a physical or email address, without their explicit permission - Threats of violence or incitement of violence toward any individual or group - Dismissing or minimising reports of harassment or discrimination @@ -55,15 +55,15 @@ when appropriate. ## Scope -This Code of Conduct applies within all project spaces — GitHub issues, pull -requests, discussions, and the MeshCore Canada Discord server — and also when an +This Code of Conduct applies within all project spaces (GitHub issues, pull +requests, discussions, and the MeshCore Canada Discord server) and also when an individual is representing the project in public spaces. ## Reporting Instances of abusive, harassing, or otherwise unacceptable behaviour may be reported by contacting the maintainers directly on the MeshCore Canada Discord -server at https://discord.gg/Gz3KvJx2hf — reach out to **dedskelly** privately. +server at https://discord.gg/Gz3KvJx2hf; reach out to **dedskelly** privately. All reports will be handled with discretion and confidentiality. Maintainers are obligated to respect the privacy and safety of the reporter. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 382b735..4670283 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,40 +6,16 @@ before opening a PR. --- -## Before you start - -**Open or comment on an issue before starting work.** This avoids duplicate -effort and lets maintainers flag if something is already in progress or out of -scope. For small bug fixes a brief comment is fine; for larger features please -discuss the approach first. - -**One thing per PR.** Each pull request should cover one logical change — a bug -fix, a new endpoint, a refactor, a new test. PRs that touch many unrelated parts -of the codebase are hard to review and hard to revert if something goes wrong. - -**No fully AI-generated contributions.** We welcome developers who use AI tools -to assist their work, but PRs should reflect the author's own understanding and -judgement. PRs that appear to be unreviewed AI output may be closed without -further comment. - ---- - -## Branches - -- `main` — stable releases only, protected. Never target this directly. -- `dev` — active development. All PRs target `dev`. - ## Workflow -1. Fork or create a branch from `dev` -2. Make your changes -3. Run the checklist below -4. Open a pull request against `dev` with a clear description of what changed - and why, referencing any related issues +Branches, the pull request flow, commit message format and the release process are the same +for every Beacon repo and live in +[beacon-docs/CONTRIBUTING.md](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/CONTRIBUTING.md). +In short: open or comment on an issue first, branch from `dev`, keep each PR to one change, +write conventional commit messages, and open the PR against `dev`. Running beacon-web against +a local server is described there too. ---- - -## Checklist before opening a PR +Before opening a PR here: ``` go build ./... # must compile @@ -75,13 +51,31 @@ swag init -g cmd/beacon/main.go -o docs --parseInternal --parseDependency --- +## Backup export tests + +CI runs export, restore round-trip, offline verify and truncated-archive rejection against +PostgreSQL 16. Locally, with `PG*` pointing at a throwaway server whose role can create +databases: + +```sh +go build -o "$PWD/beacon-backup" ./cmd/beacon-backup +BEACON_BACKUP_TEST_POSTGRES=1 BEACON_BACKUP_TEST_BINARY="$PWD/beacon-backup" \ + go test ./internal/backup -run '^TestExportPostgres$' -v +``` + +The test creates, migrates, restores and drops two randomly named databases. The export +format and operator usage are documented in +[Backup and export](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/docs/backup-export.md). + +--- + ## Database changes All schema changes must include a proper migration path: - Add a new migration file to `db/migrations/` following the existing naming convention (e.g. `002_add_observation_count.sql`). `001_baseline.sql` is the - 2.0.0 schema; do not modify existing migration files — append only via new files. + 2.0.0 schema; do not modify existing migration files; append only via new files. - Update `db/queries/queries.sql` with any new or modified queries - Re-run `sqlc generate` to regenerate `db/sqlc/` - Update the store layer in `db/` to expose the new functionality @@ -89,7 +83,7 @@ All schema changes must include a proper migration path: - Historical stats read hourly rollups, never raw observations; follow [docs/historical-stats.md](docs/historical-stats.md) to add a metric -Never edit files under `db/sqlc/` by hand — they are generated by sqlc and will +Never edit files under `db/sqlc/` by hand; they are generated by sqlc and will be overwritten. If you need to work around a sqlc limitation, document it clearly in the query comment. @@ -125,7 +119,7 @@ docs: - Add or update `// @Summary`, `// @Param`, `// @Success`, `// @Failure`, and `// @Router` comments on the handler function -- Response types are defined in `internal/api/` — add new types there, not +- Response types are defined in `internal/api/`; add new types there, not inline in handlers - After changing any handler or API type, rerun the `swag init` command from the checklist and commit the updated `docs/` directory @@ -214,55 +208,37 @@ AIRPORTS_CSV=/path/to/airports.csv go run ./internal/iatadb/gen --- -## Commit messages - -Use the conventional commits format: - -``` -feat(routes): add observation_count to known routes response -fix(ingest): correct lat/lon divisor for advert payloads -refactor(hub): collapse RegionIATAs into IATAs in Scope -test(api): add unit tests for NodeTypeName and NodeTypeFromString -chore: update airports.csv -docs: expand CONTRIBUTING.md -``` - -Scopes are optional but helpful for larger codebases. Common scopes: `api`, -`db`, `ingest`, `hub`, `ws`, `handlers`, `config`, `keystore`. - ---- - ## Project structure ``` -cmd/beacon/ — main entry point, wiring, startup -cmd/beacon-backup/ — standalone backup export/verify command -db/ — store layer: sqlc-generated code + thin mapping layer - migrations/ — SQL schema (2.0.0 baseline + numbered files, append only) - queries/ — SQL queries (input to sqlc) - sqlc/ — generated Go code (do not edit by hand) +cmd/beacon/ main entry point, wiring, startup +cmd/beacon-backup/ standalone backup export/verify command +db/ store layer: sqlc-generated code + thin mapping layer + migrations/ SQL schema (2.0.0 baseline + numbered files, append only) + queries/ SQL queries (input to sqlc) + sqlc/ generated Go code (do not edit by hand) internal/ - api/ — response types and Reader interface - handlers/ — HTTP handlers (validation, routing, response) - middleware/ — auth, rate limiting, logging - router/ — chi router wiring - background/ — periodic maintenance (retention, rollups, age-out) - backup/ — database + config bundle export and verification - borders/ — foreign-repeater classification against IATA borders - cache/ — Redis caching layer over the Reader - config/ — config loading and scope key derivation - hub/ — WebSocket fan-out broker - iatadb/ — in-memory IATA airport lookup - ingest/ — MQTT packet ingestion and side effects - keystore/ — channel key lookup - logging/ — slog setup - lora/ — LoRa airtime calculation - meshmapper/ — MeshMapper scope and zone imports - presence/ — coalesced last-seen writes - profiling/ — opt-in CPU profiling - scopestore/ — transport scope key lookup - ws/ — WebSocket connection handling -docs/ — generated swagger and design notes (swagger: do not edit by hand) + api/ response types and Reader interface + handlers/ HTTP handlers (validation, routing, response) + middleware/ auth, rate limiting, logging + router/ chi router wiring + background/ periodic maintenance (retention, rollups, age-out) + backup/ database + config bundle export and verification + borders/ foreign-repeater classification against IATA borders + cache/ Redis caching layer over the Reader + config/ config loading and scope key derivation + hub/ WebSocket fan-out broker + iatadb/ in-memory IATA airport lookup + ingest/ MQTT packet ingestion and side effects + keystore/ channel key lookup + logging/ slog setup + lora/ LoRa airtime calculation + meshmapper/ MeshMapper scope and zone imports + presence/ coalesced last-seen writes + profiling/ opt-in CPU profiling + scopestore/ transport scope key lookup + ws/ WebSocket connection handling +docs/ generated swagger and design notes (swagger: do not edit by hand) ``` Key patterns to understand before contributing: @@ -272,7 +248,7 @@ Key patterns to understand before contributing: put business logic here. - **Ingest layer** (`internal/ingest/`): processes raw MQTT packets, calls the store, and broadcasts hub events. The `DB` interface in `ingest.go` defines - exactly what the ingest layer needs from the store — keep it minimal. + exactly what the ingest layer needs from the store; keep it minimal. - **Hub** (`internal/hub/`): pure fan-out broker. Events are pre-serialised JSON before entering the hub so the broadcast loop never touches encoding. - **Reader interface** (`internal/api/reader.go`): defines everything the API @@ -282,54 +258,15 @@ Key patterns to understand before contributing: ## Releases -Merges from `dev` to `main` are done by maintainers and represent a versioned -release. Do not open PRs directly against `main`. - -### Cutting a release - -> NOTE: releases are done manually, not from GH. Commits for releases should be -> signed. - -1. Ensure all changes are committed and CI is green on `dev` -2. Bump the version string in the `@version` swagger annotation in - `cmd/beacon/main.go` -3. Regenerate swagger docs: +The flow is in +[Releases and versioning](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/docs/releases.md#cutting-a-release). +The server-specific step is bumping `@version` in `cmd/beacon/main.go` and regenerating Swagger: ```bash swag init -g cmd/beacon/main.go -o docs --parseInternal --parseDependency ``` -4. Commit the version bump and updated docs: - -``` -chore: bump version to vX.Y.Z -``` - -5. Merge `dev` into `main` (fast-forward only): - -```bash -git checkout main -git merge --ff-only dev -``` - -6. Tag the release and push: - -```bash -git tag vX.Y.Z -git push origin main --tags -``` - -Pushing the tag triggers the release CI workflow, which builds binaries for all -supported platforms and attaches them to a draft GitHub release. - -7. Open the draft release on GitHub, paste the release notes, and publish. - -8. Rebase `dev` on `main` to keep histories in sync for future releases: - -```bash -git checkout dev -git rebase main -``` +--- ## Recognition diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 929596e..0f7e5d9 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -4,7 +4,7 @@ Beacon is built by its contributors. Thank you to everyone who has helped. ## Core -- [ded](https://hackers.town/@ded) — co-founder, lead developer +- [ded](https://hackers.town/@ded), co-founder, lead developer ## Contributors diff --git a/PROFILING.md b/PROFILING.md deleted file mode 100644 index 6fbe7ae..0000000 --- a/PROFILING.md +++ /dev/null @@ -1,70 +0,0 @@ -# Production CPU profiling - -Beacon can record bounded CPU profiles on Linux and macOS. It's off by default, writes -private files only (there's no HTTP endpoint or port), and a profiling failure stops the -recorder with a warning without affecting ingest. - -## Enable a session - -Create a dedicated `0700` directory writable by the container user, mount it, and set both -variables: - -```yaml -services: - app: - environment: - BEACON_CPU_PROFILE_DIR: /profiles - BEACON_CPU_PROFILE_UNTIL: "2026-10-02T12:00:00Z" # fixed deadline - volumes: - - ./profiles:/profiles -``` - -- The deadline is RFC 3339, at most 72 hours ahead. It survives restarts and is inactive - once past. Don't generate a fresh deadline on every start. -- Changing either setting needs a restart. -- Start with a deadline about a minute out, and compare process CPU, packet freshness, queue - drops and database timeouts with an unprofiled period before running a longer session. -- One Beacon process per directory. Keep it out of web roots, public backups and source - control, and don't run another CPU profiler in the process meanwhile. - -## What gets captured - -- A 30-second sample at start, another 35 minutes later, then every 30 minutes (the extra - five minutes keeps a maintenance trigger from landing in the cooldown). -- An extra sample when route reconfirmation maintenance starts (including its retention - step). -- A five-minute cooldown between capture starts. A periodic sample due during it runs when it - ends; a triggered one is skipped, and any sample cancels a periodic one that's waiting. -- Background task stacks carry a `task` label. -- Shutdown or the deadline ends the active sample early and saves it. - -Limits: 8 MiB per profile; 256 MiB and 512 files per directory, counting metadata, leftovers -and unrelated regular files (subdirectories are ignored). The recorder reserves room for a -full capture first and stops at a limit. Nothing is deleted automatically. Files are `0600`; -a `.partial` file is an interrupted capture. - -## Reading the output - -Each `.pprof` has a `.pprof.json` sidecar with UTC start and end, the trigger, Go version, -process CPU counters, goroutine count, and database-pool counters before and after (pool -values are cumulative, so subtract). No SQL text or credentials. - -Copy completed files off the host privately, noting the image revision and digest. Then: - -```sh -go tool pprof -top capture.pprof -go tool pprof -top -cum capture.pprof -go tool pprof -tags capture.pprof -``` - -Profiles show CPU inside Beacon, not PostgreSQL work or lock waits. Correlate each interval -with host/container CPU and I/O, PostgreSQL wait states, observation throughput and Beacon's -timeout/drop logs. Don't turn on SQL text logging or run full-table counts for this; the -recorder itself makes no database queries. - -Review quiet, burst and maintenance captures separately: maintenance samples bias an -aggregate. Thirty-second windows can miss brief stalls, so a missing stack isn't proof it -never runs. - -When the deadline passes, look for `CPU profiling stopped` in the log, export the files, and -remove the variables and mount at the next deployment. diff --git a/README.md b/README.md index c83ab37..dca6275 100644 --- a/README.md +++ b/README.md @@ -14,8 +14,8 @@ in PostgreSQL, and streams live events to WebSocket clients. - Subscribes to MeshCore MQTT brokers and decodes incoming LoRa packets using [meshcore-go](https://github.com/meshcore-go/meshcore-go) - Stores packets, observations, nodes, observers, traces, routes and channel - messages in PostgreSQL (more backends to come) -- Deduplicates observations across multiple brokers (same packet heard by two + messages in PostgreSQL +- Deduplicates observations across multiple brokers (the same packet heard by two brokers is one observation per observer) - Decrypts group text messages for known channel keys - Detects firmware capability flags from path hash sizes @@ -25,16 +25,9 @@ in PostgreSQL, and streams live events to WebSocket clients. - Seeds regions, IATA display names, and channel keys from a YAML config file on startup -More documentation: - -- [beacon-docs](https://github.com/MeshCore-Beacon/beacon-docs): deployment with the - frontend and reverse proxy examples -- [Backup and export](docs/backup-export.md): private database and config bundles -- [Historical stats](docs/historical-stats.md): how the hourly rollups work -- [Packet summaries](docs/packet-summaries.md): the `summary` field on packet lists -- [CPU profiling](PROFILING.md): bounded production captures - ---- +This repo is the code. Deploying, configuring and operating Beacon is documented in +[beacon-docs](https://github.com/MeshCore-Beacon/beacon-docs); see +[Documentation](#documentation) below. ## Stack @@ -51,612 +44,68 @@ More documentation: | Config | YAML via gopkg.in/yaml.v3 | | Env | godotenv | ---- +## Running it locally -## Getting started - -### Prerequisites - -- Go 1.26+ -- Docker and Docker Compose - -### 1. Clone and configure +You need Go 1.26+ and a PostgreSQL 16 database. Docker is the easy way to get the database. ```bash -git clone https://github.com/MeshCore-Beacon/beacon-server.git -cd beacon-server +git clone https://github.com/MeshCore-Beacon/beacon-server.git && cd beacon-server cp env.example .env cp config.yaml.example config.yaml -``` - -Edit `.env` with your broker credentials and database DSN. Edit `config.yaml` to -define your regions, IATA display names, channel keys, and retention settings. - -### 2. Start PostgreSQL - -Any PostgreSQL 16 database works. For a local throwaway one: - -```bash docker run -d --name beacon-postgres -p 5432:5432 \ - -e POSTGRES_USER=beacon -e POSTGRES_PASSWORD=beacon -e POSTGRES_DB=beacon \ - postgres:16-alpine -``` - -Database migrations are applied automatically on startup. - -### 3. Run - -```bash + -e POSTGRES_USER=beacon -e POSTGRES_PASSWORD=beacon -e POSTGRES_DB=beacon postgres:16-alpine go run ./cmd/beacon ``` -Or pull and run the Docker image: - -```bash -docker pull ghcr.io/meshcore-beacon/beacon-server:latest -``` - -The image is public on GitHub Container Registry — no `docker login` required. -`latest` tracks stable releases; pin a version tag (e.g. `2.0.0`) for -predictable upgrades, and `dev` follows the development branch. - -Beacon will: - -- Load `.env` and `config.yaml` -- Connect to PostgreSQL and seed config data -- Connect to the configured MQTT brokers -- Start the HTTP server on `LISTEN_ADDR` (default `:8080`) - -### Upgrading from 1.x - -2.0.0 starts from a new schema baseline and **needs a fresh, empty database**; 1.x history -does not carry over. Pointed at a 1.x database it refuses to start with `database schema -predates Beacon 2.0.0; 2.0.0 needs a fresh database` (Docker: stop the stack and move or -remove the Postgres data directory). - -Config changes to review: - -- Every manually configured `scopes:` entry needs a `region`. -- `meshmapper.scopes.sources` is ignored; MeshMapper refresh intervals must be within - 1h–24h for scopes and 24h–168h for zones, or startup fails. -- REST rate limiting is on by default. Behind a reverse proxy, set - `server.trusted_proxies` (see [Reverse proxies and rate limits](#reverse-proxies-and-rate-limits)). -- `packets.retention` now defaults to 7 days. - -### Cold start - -Path resolution, capability detection and known routes depend on nodes having advertised -to a local observer. On a fresh database every hop shows `"confidence": "none"` and -`supportsMultibytePaths` is `false` until adverts populate `node_short_ids`. That's -expected and fills in as the mesh is observed. - -GRP_TXT packets whose channel key isn't known yet are stored hash-only. After adding the -key to `config.yaml`, the next restart decrypts the matching history; look for -`backfilled N previously-undecrypted channel message(s)` in the log. - ---- - -## Configuration - -### Environment variables (`.env`) - -| Variable | Default | Description | -| ------------------------ | ------------- | ------------------------------------------------------------ | -| `LISTEN_ADDR` | `:8080` | HTTP listen address | -| `POSTGRES_DSN` | — | PostgreSQL connection string | -| `REDIS_ADDR` | — | Redis address (`host:port`). Leave unset to disable caching. | -| `REDIS_PASSWORD` | — | Redis password (optional) | -| `REDIS_DB` | `0` | Redis database index | -| `CONFIG_PATH` | `config.yaml` | Path to YAML config file | -| `MQTT_BROKER_1_URL` | — | Broker 1 WebSocket URL (e.g. `wss://mqtt1.example.com:443`) | -| `MQTT_BROKER_1_USERNAME` | — | Broker 1 username | -| `MQTT_BROKER_1_PASSWORD` | — | Broker 1 password | -| `MQTT_BROKER_2_URL` | — | Broker 2 WebSocket URL | -| `MQTT_BROKER_2_USERNAME` | — | Broker 2 username | -| `MQTT_BROKER_2_PASSWORD` | — | Broker 2 password | -| `BEACON_API_KEY` | — | Admin bearer key; overrides `auth.api_key` when set | -| `LOG_LEVEL` | `info` | `debug`, `info`, `warn` or `error`; overrides `log.level` | -| `LOG_FORMAT` | `text` | `text` or `json`; overrides `log.format` | -| `BEACON_CPU_PROFILE_DIR` | — | Private CPU capture directory (see [PROFILING.md](PROFILING.md)) | -| `BEACON_CPU_PROFILE_UNTIL` | — | RFC 3339 deadline for CPU captures, at most 72h ahead | - -### Config file (`config.yaml`) - -```yaml -# MeshMapper integrations (optional, no API key). Import transport scopes and -# IATA border outlines from MeshMapper instead of maintaining scopes: and -# iatas.*.borderFile by hand. Both cover every known IATA that MeshMapper lists; -# no sources to configure. See config.yaml.example for limits and behaviour. -#meshmapper: -# scopes: -# enabled: true -# zones: -# enabled: true -# import_groups: true # MeshMapper zone groups become regions -# channels: -# enabled: true # import each region's public hashtag channels - -# Optional IATA overrides — auto-created on first packet arrival, -# only needed if you want to customise display name or coordinates. -# borderFile points to a GeoJSON Feature (Polygon or MultiPolygon) for the -# region border map feature; relative paths resolve against this config -# file's own directory. Validated at boot — invalid geometry fails startup. -iatas: - YVR: - name: Vancouver International - lat: 49.1967 - lng: -123.1815 - borderFile: borders/yvr.geojson # optional; or use meshmapper.zones - -# Super-regions grouping multiple IATAs. -regions: - - slug: western-canada - name: Western Canada - display_order: 1 - center_lat: 51.0 - center_lng: -114.0 - zoom_level: 5 - iatas: [YVR, YYJ, YYC, YEG] - -# Channel keys for decrypting group messages. -channel_keys: - # Hashtag channels: Beacon derives the PSK from the tag name automatically. - # secret = SHA256("#tag")[:16], channel_hash = SHA256(secret)[0] - # Tag names should be provided without the # prefix. Plain names show under - # every region; {name, region: } limits a channel to one region. - hashtags: - - meshcore - - name: vancouver-mesh - region: western-canada - - # Explicit keys: channel hash (hex) and key (hex), with optional display name. - # The public MeshCore channel key is included in config.yaml.example. - keys: - "11": - key: "8b3387e9c5cdea6ac9e5edbaa115cd72" - name: "Public" - -# Regional transport scopes for matching TRANSPORT_FLOOD packets. -# Plain names have # prepended automatically (e.g. "bc" → "#bc"). -# region (required) is a configured region slug; region-filtered scope lists -# and scope stats show the scope under that region's IATAs. Matching stays global. -# Optional when meshmapper.scopes covers your regions. -scopes: - - name: bc - region: western-canada - - name: "#west" - region: western-canada - -# Observer telemetry storage settings. -telemetry: - retention: 744h # how long to keep telemetry snapshots (default: 31 days) - resolution: 1h # snapshot frequency per observer; duplicates within window are dropped (default: 1h) - -# Packet and observation retention. -packets: - retention: 168h # packets, observations, channel messages (default: 7 days, minimum 24h) - -# Hourly analytics rollups behind the stats endpoints; they outlive raw packets. -analytics: - rollup_retention: 2160h # also the longest /stats/series window (default: 90 days, minimum 24h) - -# Presence write coalescing. -# Observer last_seen and packet last_heard_at bumps are batched in memory and -# flushed on an interval instead of writing one row per observation. An -# unclean shutdown loses at most one interval of presence freshness. -presence: - flush_interval: 30s # how often coalesced bumps are flushed (default: 30s) - packet_ttl: 30s # how long a quiet packet stays coalesced before writing through again (default: 30s) - -# Direct proxy peers allowed to set the client IP through X-Real-IP. -server: - trusted_proxies: [172.30.0.0/24] # default: [] (direct access) - -# Per-client REST limits for /api/v1/*. -ratelimit: - enabled: true # default: true - requests_per_minute: 300 # default: 300 - -# WebSocket settings. -websocket: - max_connections_per_ip: 5 # default: 5 - max_connects_per_minute: 10 # upgrade attempts per IP (default: 10) - allowed_origins: [https://beacon.example.com] # other sites allowed to open /ws (default: same host only) - -# Node staleness, deletion, and clock-drift thresholds. -nodes: - mark_foreign: false # optional indication for repeaters outside configured IATA borders - stale_threshold: 24h # mark a node "stale" in the API after this long unseen (default: 24h) - delete_after: 720h # delete a node entirely after this long unseen (default: 30 days) - clock_drift_threshold: 5m # |device clock - server clock| above which clockOutOfSync=true for a repeater/room server (default: 5m) - -# Optional observer age-out (disabled by default; set e.g. 720h to opt in). -# Enable only when one Beacon ingest process owns the database: presence-cache -# preparation is local to that process, including both of its MQTT workers. -# Deletes at most 1000 observers per background.cleanup interval, only when -# last_seen/last_status_at are old and no observations, telemetry or ownership remain. -# Broker/location/scope metadata cascades; a returning observer receives a new ID. -observers: - delete_after: 0s # omitted or nonpositive disables deletion - -# Redis caching layer (optional). -# Caches read-heavy, slow-changing responses to reduce PostgreSQL load. -# Connection details (address, password, database) are set via environment -# variables. Leave REDIS_ADDR unset to disable caching entirely. -# TTLs are duration strings e.g. "30m", "1h". Per-category TTLs override -# the global ttl. Any unset category inherits ttl. Default: 1h. -cache: - ttl: "1h" - ttls: - stats: "1h" # stats endpoints (hourly rollups; keys change when a new hour is rolled) - reference: "1h" # IATAs, regions, scopes - nodes: "1h" # node detail (also explicitly invalidated on upsert) - observers: "1h" # observer detail (also explicitly invalidated on upsert) - -# Geographic ingest filter (optional). -# Drop packets from observers outside the specified area. -# Country codes are ISO 3166-1 alpha-2. Continent codes: AF AN AS EU NA OC SA. -# If both are set an IATA passes if it matches either (OR semantics). -# Omit entirely to accept all IATAs (default). -ingest: - allow_countries: [CA, US] # only store packets from these countries - allow_continents: [NA] # or: accept all of North America -``` - -IATAs are auto-created on first packet arrival. The config file adds display -names, coordinates, and optional region borders. Regions and channel keys -must be defined here unless MeshMapper imports them (`meshmapper.zones.import_groups`, -`meshmapper.channels`). - -### Admin API - -The `/api/v1/admin` subtree requires `Authorization: Bearer `. Everything else, -including the WebSocket and CORS preflights, is public. - -- Set the key with `BEACON_API_KEY` or `auth.api_key`. A set environment variable wins; an - explicitly empty one disables admin access. No key is generated for you. -- The key must be at least 16 characters with no inner whitespace (surrounding whitespace is - trimmed). An unusable key fails startup. Changing it needs a restart. -- With no key, admin requests return 503. A missing, wrong or duplicated header returns 401 - with `WWW-Authenticate: Bearer`. After auth, unknown paths are 404 and unsupported methods - 405. -- Use a long random key, send it only in the header (never the URL or body), and keep it out - of source control and logs. Terminate HTTPS at the proxy and keep Beacon's listener - private to it. - -Endpoints: - -- **`GET /admin/config`** returns the CORS options with defaults applied, `auth.configured` - and `ingest.broker_count` (configured broker workers, not connection status). No - credentials, broker addresses, channel material or database settings. -- **`PUT /admin/config`** accepts only `{"cors":{"allowed_origins":[...]}}` and replaces the - origin list immediately. It is **runtime-only**: nothing is written, and a restart reloads - the file. The response carries `config`, `persisted: false` and `requires_restart: false`. - Send 1–32 ASCII http(s) origins of at most 512 bytes each, with an optional single - hostname wildcard; a lone `*` allows all. Empty or null lists, paths, queries, credentials, - control characters, unknown fields and bodies over 16 KiB are rejected. Concurrent updates - apply one at a time; requests already in flight may see the old list. -- **`GET/POST /admin/accounts`, `GET/DELETE /admin/accounts/{id}`** manage operator account - records (no login, session or token). POST takes `{"name": "..."}` (body ≤ 4 KiB; name - trimmed, case-sensitive, ≤ 128 characters, no control characters, unique among active - accounts). DELETE deactivates: 204, or 404 if missing, 409 if already inactive. The name - can then be reused. The list returns active and inactive accounts, newest first, - unpaginated. -- **`GET /admin/backup`** is opt-in; see [Backup and export](docs/backup-export.md). - -Browser admin clients need the matching methods in the saved `cors.allowed_methods`. The -default `GET, HEAD, OPTIONS` is read-only. For an admin UI, use -`[GET, HEAD, OPTIONS, POST, PUT, DELETE]`, restrict `cors.allowed_origins` to that UI and -allow the `Authorization` and `Content-Type` headers, or preflight blocks requests that work -from curl. CORS controls browser access, not authentication. - -### Reverse proxies and rate limits - -REST requests under `/api/v1` are rate limited per client IP (300/min by -default; IPv6 clients share a /64). Exhausted clients get `429` with -`rate_limited` and `Retry-After`. WebSocket upgrades are limited separately by -`websocket.max_connects_per_minute`. - -Beacon never trusts `X-Forwarded-For` or `True-Client-IP`. Behind a reverse -proxy, list the proxy's address in `server.trusted_proxies` (CIDR, e.g. -`127.0.0.1/32` or the Docker network subnet) and have the proxy overwrite -`X-Real-IP` with the connecting client's address: - -```nginx -proxy_set_header X-Real-IP $remote_addr; -``` - -```caddy -reverse_proxy app:8080 { - header_up X-Real-IP {remote_host} -} -``` - -Without this every visitor shares the proxy's budget and the site gets 429s -under normal load. Beacon logs a warning at startup when limits are on and no -proxy is trusted, and once at runtime when a request carries forwarding headers -it is ignoring. Full nginx and Caddy examples are in -[beacon-docs](https://github.com/MeshCore-Beacon/beacon-docs). - -### Analytics retention - -Historical stats read hourly rollups kept for `analytics.rollup_retention` (default 90 -days), independently of `packets.retention`. Each UTC hour is rolled about 95 minutes after -it closes, and packet cleanup holds back up to 24h of raw rows until their hours are rolled. -An hour that lost raw rows first is reported as partial, without values. Drill-downs, -sub-hour observer activity and observer comparison still read raw data. Details: -[docs/historical-stats.md](docs/historical-stats.md). - -### Logging - -Logs go to stderr; collect and rotate them with Docker or systemd. Set `log.level` -(`debug`, `info`, `warn`, `error`; default `info`) and `log.format` (`text` or `json`) in -`config.yaml`, or override with `LOG_LEVEL` / `LOG_FORMAT`. Invalid values fail startup. - -Every record has a `component`. Ingest records add the broker name; HTTP completion records -add the validated client address, route, status and duration. Query strings and WebSocket -hello payloads are never logged. Expected ingest skips and routine WebSocket lifecycle -events are debug-level. Reverse-proxy access logs (and fail2ban rules on them) are -unaffected. - -### Foreign repeater indication - -Set `nodes.mark_foreign: true` to add `possiblyForeign` to repeater nodes. It is `true` when -the repeater's reported position is outside the union of all configured -`iatas..borderFile` polygons, with `meshmapper.zones` borders replacing them where -MeshMapper has one. IATAs without a border add no area, and airport coordinates aren't used. -Enabling it with no border source, a missing file or invalid geometry fails startup. Border -changes need a restart. - -- Points on an edge (including hole edges) are inside; hole interiors are outside. -- Other roles and missing or invalid positions get no value. A 0/0 advert clears the stored - position. -- It's a hint from the reported position, not proof of origin. Ingest, heard-in IATAs and - route matching are unaffected. -- Computed at read time, so existing nodes need no backfill. The field is omitted when the - feature is off (the default). -- In `nodeUpdate`, it's a boolean when the advert carries a position, `null` when the - position is 0/0 or invalid or the node stops being a repeater, and omitted to keep the - previous value. `lat`/`lng` follow the same rule. - -GeoJSON uses longitude/latitude order. Split antimeridian-crossing borders into -MultiPolygons per [RFC 7946 §3.1.9](https://www.rfc-editor.org/rfc/rfc7946#section-3.1.9); -edges spanning more than 180° are rejected. - ---- - -## WebSocket API - -Connect to `ws://host:8080/ws`. - -On connect the server sends a `hello`: - -```json -{ "v": 1, "type": "hello", "serverTime": 1234567890000, "connectionId": "uuid" } -``` - -The connection closes after 90 seconds of inactivity. Clients should send a -`ping` every 30 seconds. - -### Client → Server messages - -**Subscribe** — add a filter to this connection. Multiple subscriptions are -unioned (OR semantics): an event matches if it satisfies any active -subscription. The server replies with a `subscriptionId` to use for -unsubscribing. - -```json -{ - "v": 1, - "type": "subscribe", - "id": "sub-1", - "scope": { - "iatas": ["YOW", "YYZ"], - "regionIds": ["1"], - "regionSlugs": ["western-canada"], - "payloadTypes": [4, 5], - "routeTypes": [1, 2], - "channelHashes": ["11"], - "observerIds": [""], - "events": ["packetObservation", "channelMessage"] - } -} -``` - -All scope fields are optional; omitted or empty means no filter on that -dimension. `routeTypes` and `observerIds` filter `packetObservation` events -(`observerIds` also `observerStatus`), and `channelHashes` only `channelMessage`. `regionIds` and -`regionSlugs` are both expanded to their member IATAs server-side. - -**Unsubscribe** — remove a specific subscription by ID. - -```json -{ - "v": 1, - "type": "unsubscribe", - "id": "unsub-1", - "subscriptionId": "" -} -``` - -The server replies `{ "v": 1, "type": "unsubscribed", "id": "unsub-1", "subscriptionId": "..." }`. - -**Configure** — connection-wide flags, all default `false`: - -- `resolvePath` adds per-hop node resolution to `packetObservation` events (see below). -- `includeObserverKey` adds `observation.observerPublicKey`. -- `includeRepeats` also streams later hearings of an already-stored observation - over a new path, as `packetObservation` events with `packet.isRepeat: true` - and `observationCount: 0`. - -Unlike `subscribe`, this is not additive: each `configure` sets all three flags -to exactly the values sent, so an omitted flag turns off. - -```json -{ "v": 1, "type": "configure", "id": "cfg-1", "resolvePath": true, "includeRepeats": true } -``` - -The server replies: - -```json -{ "v": 1, "type": "configured", "id": "cfg-1", "resolvePath": true, "includeObserverKey": false, "includeRepeats": true } -``` - -**Ping** - -```json -{ "v": 1, "type": "ping", "id": "ping-1" } -``` - -### Server → Client events - -| Type | Description | -| ------------------- | --------------------------------------------------- | -| `packetObservation` | New observation written to DB | -| `observerStatus` | Observer status update | -| `nodeUpdate` | Node upserted from advert | -| `channelMessage` | Decrypted channel message (scope must include hash) | - -When `resolvePath` is enabled via `configure`, `packetObservation` events -include a `resolvedPath` array: one entry per hop in the packet's path, each -with a `confidence` (`"high"` for exactly one candidate node, `"ambiguous"` -for multiple, `"none"` for zero) and the matching node(s)' id, name, -lat/lng, and public key. Without `resolvePath` enabled, `resolvedPath` is -`null` — this keeps the default event payload small; only opt in if you're -actually rendering path connections (e.g. drawing hops on a map). - -### Backpressure - -The server write buffer per connection is bounded at 256 events. If a client -falls behind, the server drops the oldest queued events and sends a `lagged` -notice: - -```json -{ "v": 1, "type": "lagged", "droppedCount": 12, "since": 1234567890000 } -``` - -Clients should respond by backfilling over REST, then resume streaming: -`/api/v1/packets/backfill?afterObservationId=` and -`/api/v1/messages/backfill?afterId=`. - -### Reconnection - -Subscriptions are not persisted — they exist only for the lifetime of the -connection. On any disconnect the client should reconnect with backoff, re-issue -all subscriptions, and backfill via the same REST endpoints. - -### Connection limits - -- At most `websocket.max_connections_per_ip` (default 5) concurrent connections - per IP. A connection over the cap is accepted and then closed with code - `1013` (try again later) before `hello`. -- At most `websocket.max_connects_per_minute` (default 10) upgrade attempts per - IP; beyond that the handshake gets `429` with `Retry-After: 60`. -- Browsers may only connect from the same host unless their origin is listed in - `websocket.allowed_origins` (`https://*.example.com` covers every subdomain). - -Behind a reverse proxy both limits need `server.trusted_proxies`, or every -visitor counts as the proxy's IP. - ---- - -## REST API - -Base path: `/api/v1` - -All list endpoints support cursor-based pagination via `cursor` and `limit` -query params. See the Swagger UI at `http://localhost:8080/swagger/index.html` -for full parameter documentation. - -### Authentication and limits - -Only `/admin/*` needs `Authorization: Bearer `. All `/api/v1` routes are -rate limited per client IP; see [Reverse proxies and rate limits](#reverse-proxies-and-rate-limits). -List `limit` values are clamped to 1–200. - -### Endpoints - -| Method | Path | Description | -| -------- | --------------------------------------- | --------------------------------------------------------------------------------------------------------------------- | -| `GET` | `/admin/accounts` | List operator accounts (bearer key) | -| `POST` | `/admin/accounts` | Create an operator account (bearer key) | -| `GET` | `/admin/accounts/{id}` | Get an operator account (bearer key) | -| `DELETE` | `/admin/accounts/{id}` | Deactivate an operator account (bearer key) | -| `GET` | `/admin/backup` | Download a private database and saved-config backup (bearer key, opt-in) | -| `GET` | `/admin/config` | Inspect selected running configuration (bearer key) | -| `PUT` | `/admin/config` | Replace runtime CORS origins (bearer key) | -| `GET` | `/brokers` | List MQTT brokers and connection status | -| `GET` | `/channels` | List channels (`hash`, `iata`/`iatas`, `keyKnown`, `pageCursor`, `limit`) | -| `GET` | `/channels/{channelID}` | Get channel detail by integer ID | -| `GET` | `/channels/{channelID}/messages` | List messages for a channel (`since`, `iatas`, `region`/`regionId`, `scope`, `cursor`, `limit`) | -| `GET` | `/iatas` | List all known IATA codes | -| `GET` | `/iatas/{iata}` | Get a single IATA code | -| `GET` | `/iatas/{iata}/border` | Get an IATA's GeoJSON border, if any (204 if not) | -| `GET` | `/messages` | List channel messages (`channelID`, `channelHash`, `since`, `iatas`, `region`/`regionId`, `scope`, `cursor`, `limit`) | -| `GET` | `/messages/backfill` | Messages after `afterId` | -| `GET` | `/nodes` | List nodes (type, IATA/region, name, scope, `pubkey`/`pubkeyPrefix`, capability flags, `neighbors`) | -| `GET` | `/nodes/{nodeId}` | Get node detail | -| `GET` | `/nodes/{nodeId}/neighbors` | List neighboring nodes observed in the mesh | -| `GET` | `/nodes/{nodeId}/observations` | List observations of packets from a node | -| `GET` | `/observers` | List observers (IATA/region, `type`, `broker`, `status=online\|offline`, `name`, `scope`) | -| `GET` | `/observers/{observerId}` | Get observer detail including broker last-seen timestamps | -| `GET` | `/observers/{observerId}/activity` | Heard-activity history (`range`, `interval`, `until`) | -| `GET` | `/observers/{observerId}/adverts` | Adverts heard by observer | -| `GET` | `/observers/{observerId}/telemetry` | Telemetry history (`range`, `interval=1h\|6h\|24h`, `afterId`) | -| `GET` | `/packets` | List packets (payload/route type, IATA/region, scope, `since`/`until`; plural params take CSV) | -| `GET` | `/packets/backfill` | Packets after `afterObservationId` | -| `GET` | `/packets/{packetHash}` | Get packet with all observations | -| `GET` | `/regions` | List all regions (summary) | -| `GET` | `/regions/{regionId}` | Get a single region with IATA list | -| `GET` | `/routes` | List known routes (`iata`, `hopCount`, `cursor`+`cursorId`) | -| `GET` | `/routes/search` | Search routes by source and destination hash | -| `GET` | `/routes/cross` | Search for routes crossing IATA boundaries | -| `GET` | `/routes/{iata}/{pathKey}/observations` | Retained observations matching a saved route (see below) | -| `GET` | `/scopes` | List transport scope names; IATA/region filters use configured regions and MeshMapper catalogues | -| `GET` | `/scopes/{name}` | Get scope detail | -| `GET` | `/stats/clock-drift` | Repeaters and room servers whose clock drifted past the threshold, worst first | -| `GET` | `/stats/node-types` | Node type breakdown | -| `GET` | `/stats/observations` | Hourly observation counts per IATA (last 7 days by default) | -| `GET` | `/stats/observer-comparison` | Compare flood packets reported by two observers | -| `GET` | `/stats/overview` | Network overview over the last 24 rolled hours | -| `GET` | `/stats/paths` | Path-entry and hash-width distributions | -| `GET` | `/stats/payload-breakdown` | Observation counts by payload type (last 24h by default) | -| `GET` | `/stats/radio-presets` | Radio preset usage by IATA | -| `GET` | `/stats/scopes` | Region scopes with hourly packet, node and observer counts (last 7 days by default) | -| `GET` | `/stats/series` | Hourly card metrics and window summary (`since`/`until` required) | -| `GET` | `/stats/signal` | Reception signal distributions and hourly trends | -| `GET` | `/stats/top-advertisers` | Top N nodes by distinct ADVERT packets, split flood/direct (last 24h by default) | -| `GET` | `/stats/top-nodes` | Top N nodes by advert hearings (last 7 days by default) | -| `GET` | `/stats/top-observers` | Top N observers by observation count (last 24h by default) | -| `GET` | `/stats/top-talkers` | Top N companion names by decrypted channel message count (last 24h by default) | -| `GET` | `/traces` | List trace tags (`type=TRACE\|PING`, IATA/region, scope, `since`/`until`, `cursor`+`cursorTag`) | -| `GET` | `/traces/{tag}` | Get full trace detail with resolved routes | - -### Observer activity - -`GET /observers/{observerId}/activity` returns buckets over `[windowStart, windowEnd)` (live -requests include the current partial bucket), plus `generatedAt`, `source` (`raw` or -`hourly`) and `summary`. - -- `recordedPackets` counts stored observations in the window; repeated broker deliveries of - the same packet count once. Unknown payload types show as `-1`. -- Freshness fields (`lastCompleteHour` with its start and end, `latestRecordedAt`) are - measured at `generatedAt`, even for historical `until` requests. -- `until` (epoch ms, within the last 30 days) lines up two observers' charts. -- Hourly mode reads the rollups plus raw rows for hours not rolled yet (at most 24h). - `rolledUntil` ends the rolled range and `rawFrom` starts the raw tail; hours between them - are uncovered. -- Missing records don't prove downtime. The observer's `observationCount` is a legacy - cumulative presence counter (including status and neighbour events), not a packet total. - -### Saved-route evidence - -`GET /routes/{iata}/{pathKey}/observations` returns retained observations matching a saved -route's full path bytes, hash size and hop count within its IATA. `range` defaults to `24h` -(max `720h`) and `limit` to 50 (max 200); page with `nextPageCursor` → `pageCursor`. -Evidence expires with raw packets. - ---- +Fill in `POSTGRES_DSN` and your MQTT broker credentials in `.env`. Every variable is described +in [Configuration](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/docs/configuration.md), +and the fully annotated `config.yaml` lives in beacon-docs as well; the copy here is a working +starter. Migrations run on startup. The API listens on `LISTEN_ADDR` (default `:8080`) and +Swagger is at `http://localhost:8080/swagger/index.html`. + +To run the web frontend against it, see +[Running the full stack locally](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/CONTRIBUTING.md#running-the-full-stack-locally). + +The published image is `ghcr.io/meshcore-beacon/beacon-server`. `latest` tracks stable +releases, `dev` follows the development branch, and each release is also tagged `X.Y.Z` and +`X.Y`. + +### What you see on an empty database + +Path resolution, capability detection and known routes depend on nodes having advertised to a +local observer. On a fresh database every hop shows `"confidence": "none"` and +`supportsMultibytePaths` is `false` until adverts populate `node_short_ids`. That is expected +and fills in as the mesh is observed. + +GRP_TXT packets whose channel key is not known yet are stored hash-only. After adding the key to +`config.yaml`, the next restart decrypts the matching history; look for +`config: backfilled N previously-undecrypted channel message(s)` in the log. + +## Documentation + +In [beacon-docs](https://github.com/MeshCore-Beacon/beacon-docs): + +- [Deploy with Docker](https://github.com/MeshCore-Beacon/beacon-docs#deploy-the-all-in-one-stack) +- [Getting packets in](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/docs/getting-packets-in.md): brokers, the subscriber account, topics +- [Configuration](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/docs/configuration.md): environment variables and `config.yaml` +- [API contract](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/docs/api-contract.md): REST, WebSocket, admin endpoints +- [Reverse proxy and rate limits](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/docs/reverse-proxy.md) +- [Upgrading](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/docs/upgrading.md) +- [Operations](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/docs/operations.md), [backup and export](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/docs/backup-export.md) and [CPU profiling](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/docs/profiling.md) +- [High level design](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/docs/high-level-design.md) +- [Releases and versioning](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/docs/releases.md) + +In this repo: [Historical stats](docs/historical-stats.md) explains the hourly rollups behind +the stats endpoints, for anyone changing them. `docs/swagger.yaml` is the generated OpenAPI +description. + +## Contributing + +[CONTRIBUTING.md](CONTRIBUTING.md) covers code style, tests, and database and API changes. +Branches, commits and the release flow are shared across the Beacon repos and live in +[beacon-docs/CONTRIBUTING.md](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/CONTRIBUTING.md). +Please read the [Code of Conduct](CODE_OF_CONDUCT.md). Security reports go through +[SECURITY.md](SECURITY.md). ## Acknowledgements diff --git a/SECURITY.md b/SECURITY.md index 397b307..12a5e81 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -5,7 +5,7 @@ Please do not report security vulnerabilities through public GitHub issues. Instead, contact the maintainers directly via the MeshCore Canada Discord -server: [MeshCore Canada Discord](https://discord.gg/Gz3KvJx2hf) — reach out to +server: [MeshCore Canada Discord](https://discord.gg/Gz3KvJx2hf); reach out to **dedskelly** directly. Include as much detail as possible: the nature of the issue, steps to reproduce, and any potential impact. diff --git a/SHOULDERS.md b/SHOULDERS.md index ad3b0ec..51a9856 100644 --- a/SHOULDERS.md +++ b/SHOULDERS.md @@ -5,48 +5,48 @@ projects that make this possible: ## Core dependencies -- [meshcore-go](https://github.com/meshcore-go/meshcore-go) — MeshCore packet +- [meshcore-go](https://github.com/meshcore-go/meshcore-go): MeshCore packet decoding -- [paho.mqtt.golang](https://github.com/eclipse/paho.mqtt.golang) — MQTT client -- [pgx](https://github.com/jackc/pgx) — PostgreSQL driver and connection pool -- [sqlc](https://sqlc.dev) — Type-safe SQL code generation -- [chi](https://github.com/go-chi/chi) — HTTP router -- [go-chi/cors](https://github.com/go-chi/cors) — CORS middleware for chi -- [go-chi/httprate](https://github.com/go-chi/httprate) — Per-client HTTP +- [paho.mqtt.golang](https://github.com/eclipse/paho.mqtt.golang): MQTT client +- [pgx](https://github.com/jackc/pgx): PostgreSQL driver and connection pool +- [sqlc](https://sqlc.dev): Type-safe SQL code generation +- [chi](https://github.com/go-chi/chi): HTTP router +- [go-chi/cors](https://github.com/go-chi/cors): CORS middleware for chi +- [go-chi/httprate](https://github.com/go-chi/httprate): Per-client HTTP sliding-window rate limits and retry headers -- [go-redis](https://github.com/redis/go-redis) — Redis client -- [coder/websocket](https://github.com/coder/websocket) — WebSocket +- [go-redis](https://github.com/redis/go-redis): Redis client +- [coder/websocket](https://github.com/coder/websocket): WebSocket implementation -- [swaggo/swag](https://github.com/swaggo/swag) — OpenAPI documentation +- [swaggo/swag](https://github.com/swaggo/swag): OpenAPI documentation generation -- [godotenv](https://github.com/joho/godotenv) — .env file loading -- [yaml.v3](https://github.com/go-yaml/yaml) — YAML config parsing -- [google/uuid](https://github.com/google/uuid) — UUID generation -- [golang.org/x/sync](https://pkg.go.dev/golang.org/x/sync) — singleflight for +- [godotenv](https://github.com/joho/godotenv): .env file loading +- [yaml.v3](https://github.com/go-yaml/yaml): YAML config parsing +- [google/uuid](https://github.com/google/uuid): UUID generation +- [golang.org/x/sync](https://pkg.go.dev/golang.org/x/sync): singleflight for collapsing concurrent cache misses -- [paulmach/orb](https://github.com/paulmach/orb) — GeoJSON border geometry and +- [paulmach/orb](https://github.com/paulmach/orb): GeoJSON border geometry and point-in-polygon checks -- [swaggo/http-swagger](https://github.com/swaggo/http-swagger) — Swagger UI +- [swaggo/http-swagger](https://github.com/swaggo/http-swagger): Swagger UI handler ## Testing -- [go.uber.org/mock](https://github.com/uber-go/mock) — gomock mocks for the +- [go.uber.org/mock](https://github.com/uber-go/mock): gomock mocks for the generated queries -- [miniredis](https://github.com/alicebob/miniredis) — In-memory Redis for cache +- [miniredis](https://github.com/alicebob/miniredis): In-memory Redis for cache tests ## Data -- [OurAirports](https://ourairports.com/data/) — IATA airport geographic data +- [OurAirports](https://ourairports.com/data/): IATA airport geographic data (CC0 public domain) ## Infrastructure -- [PostgreSQL](https://www.postgresql.org) — Primary data store -- [Redis](https://redis.io) — Optional caching layer -- [Caddy](https://caddyserver.com) — Reverse proxy +- [PostgreSQL](https://www.postgresql.org): Primary data store +- [Redis](https://redis.io): Optional caching layer +- [Caddy](https://caddyserver.com): Reverse proxy ## Language -- [Go](https://go.dev) — The language that makes this all possible +- [Go](https://go.dev): The language that makes this all possible diff --git a/config.yaml.example b/config.yaml.example index d68edb8..6044840 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -1,5 +1,6 @@ -# Beacon configuration file -# Copy to config.yaml and adjust as needed. +# Beacon configuration file: local development starter. Copy to config.yaml and adjust. +# The fully annotated reference, with every key and its default, is +# https://github.com/MeshCore-Beacon/beacon-docs/blob/main/app_config/config.yaml.example # Durations are Go strings (30s, 1h, 168h); negative values prevent startup # (observers.delete_after excepted, where nonpositive disables it). @@ -263,7 +264,7 @@ websocket: # CORS configuration. # Controls which origins, methods and headers are allowed for cross-origin requests. -# Defaults to allowing all origins with GET/HEAD/OPTIONS if omitted — appropriate +# Defaults to allowing all origins with GET/HEAD/OPTIONS if omitted, appropriate # for a public read-only API. Restrict allowed_origins if you expose write endpoints. #cors: # allowed_origins: diff --git a/docs/backup-export.md b/docs/backup-export.md deleted file mode 100644 index d7d24da..0000000 --- a/docs/backup-export.md +++ /dev/null @@ -1,136 +0,0 @@ -# Backup and export - -Beacon exports its PostgreSQL database and saved `config.yaml` as a private, versioned -`.tar.gz`, either with the standalone `beacon-backup` command or an opt-in admin download. -`beacon-backup -verify` checks an archive offline. There is no import, scheduling or remote -storage. - -## Archive format 1 - -Exactly three regular files: - -- `manifest.json`: format and exporter version, creation time, payload sizes and SHA-256 - hashes, and the exclusions below. -- `database.sql`: one consistent `pg_dump` snapshot (schema, data, migration journal), - without ownership, ACLs or tablespaces so it restores under any role. -- `config.yaml`: the saved file byte-for-byte, read just before the snapshot. Avoid config - edits during an export. - -**It is sensitive and unencrypted** (keys, message data); keep it private. It is not a full -recovery package: `.env` and environment variables, `borderFile` and TLS files, runtime-only -changes, PostgreSQL roles and cluster settings, Redis and deployment files are not included. - -## Requirements and limits - -- `pg_dump` must be in the **same runtime** as the exporter (the CLI's environment, or the - server's PATH for the download), with a major version matching the database. A client on - the Docker host or in another container doesn't count. -- The Docker image ships the PostgreSQL 16 client. Its Alpine 3.19 base has no 17/18 - packages, so a newer database needs a different runtime, not just `POSTGRES_CLIENT_MAJOR`. -- The temp directory (`TMPDIR`) needs about twice the SQL limit free. A small tmpfs won't do. -- The output filesystem must support hard links (used to publish atomically). Existing - destinations, symlinks included, are never replaced. Files are 0600 and staging dirs 0700; - on Windows, restrict the directory ACLs yourself. - -| Limit | Default | -| --- | --- | -| Timeout | 10 min (`-timeout`) | -| Uncompressed SQL | 1 GiB (`-max-bytes`, up to 1 TiB) | -| Saved YAML / manifest | 1 MiB / 64 KiB | -| Compressed archive (verify) | SQL limit + 1% + 2 MiB | -| Lock wait | 5 s | - -Failures, timeouts and interrupts clean up and publish nothing. A SIGKILL or power loss can -leave a private staging dir: `.beacon-backup-*` next to the CLI output, or -`beacon-download-*` in the server's temp dir, which Beacon sweeps on its next start. - -## CLI export - -```sh -go build ./cmd/beacon-backup -beacon-backup -config /private/config.yaml -output /private/beacon-20260913.tar.gz -``` - -Connection comes only from the standard libpq environment (`PGHOST`, `PGPORT`, -`PGDATABASE`, `PGUSER`, TLS settings); `PGDATABASE` is required. Keep the password in a 0600 -`PGPASSFILE` or a service file. The command ignores `.env` and `POSTGRES_DSN`, runs no -migrations, and never prints `pg_dump` stderr since it can leak private data. If it fails, -check the client version, privileges, connection settings and free space. - -## Download API - -`GET /api/v1/admin/backup` streams a fresh archive. Enable it with `backup.enabled: true` and -the admin bearer key, then restart. Send the key only in the `Authorization` header over -HTTPS; the endpoint takes no query or body. Leave it off on public previews. - -At startup Beacon checks `pg_dump --version` against the server. If that or another -prerequisite fails, it logs why and disables only backup; fix it and restart. - -**Connection.** The download uses `POSTGRES_DSN`, which must be a single-host `postgres://` -URL with an explicit user and database. TLS options, `passfile`, `connect_timeout`, -`application_name`, `target_session_attrs` and `options` are passed through; `pool_*` -options are dropped; supported `PG*` env defaults apply under the URL. Keyword DSNs, -services, multiple hosts, unknown or duplicate options, protocol-negotiation settings and -the `PGSERVICE`, `PGSERVICEFILE`, `PGSSLNEGOTIATION`, `PG{MIN,MAX}PROTOCOLVERSION` and `PGTZ` -env vars are rejected. Use the CLI for those setups. Prefer `sslmode=verify-full` for remote -databases. Credentials go to a 0600 service file in staging, never into process arguments, -logs or errors. - -**Behaviour.** One export at a time. Disconnecting cancels it, and the transfer has a -10-minute write deadline. Responses are `no-store`. A failed export sends no partial archive -and logs a sanitized reason. - -| Status | Meaning | -| --- | --- | -| 400 | Query string or body sent | -| 401 | Missing or invalid key | -| 409 | Another export is running | -| 500 | Export failed | -| 503 | Backup unavailable, no admin key, or shutting down | -| 504 | Timed out | -| 507 | SQL size limit exceeded | - -## Verify - -```sh -beacon-backup -verify /private/beacon-20260913.tar.gz -``` - -Reads the file without extracting, running SQL or connecting to anything, so it needs no -config, `pg_dump` or credentials. `-max-bytes` and `-timeout` apply. It requires exactly the -three members as 0600 regular files with USTAR headers (size-only PAX allowed for SQL over -8 GiB). It rejects duplicates, links, devices, extra metadata, truncated or concatenated -streams, and unknown or duplicate manifest keys. Sizes and SHA-256 hashes must match. - -Exit 0 means structure and checksums passed. Errors don't include payload, manifest values -or the path. The manifest is unsigned, so a pass does **not** prove authenticity, safe SQL or -that the dump restores. - -## Restore - -Only restore trusted bundles, because a dump can run arbitrary SQL. Verify, extract -privately, and load into a **new, empty database** on a compatible PostgreSQL with the -needed extensions: - -```sh -psql -X --set ON_ERROR_STOP=on --single-transaction --file database.sql -``` - -Check the migration journal and some records, review the config, and restore external -secrets and files before pointing a new Beacon at it. Never test an export by restoring over -live data. See [pg_dump](https://www.postgresql.org/docs/16/app-pgdump.html) and -[libpq environment](https://www.postgresql.org/docs/16/libpq-envars.html). - -## Testing - -CI runs export, restore round-trip, offline verify and truncated-archive rejection against -PostgreSQL 16. Locally, with `PG*` pointing at a throwaway server whose role can create -databases: - -```sh -go build -o "$PWD/beacon-backup" ./cmd/beacon-backup -BEACON_BACKUP_TEST_POSTGRES=1 BEACON_BACKUP_TEST_BINARY="$PWD/beacon-backup" \ - go test ./internal/backup -run '^TestExportPostgres$' -v -``` - -The test creates, migrates, restores and drops two randomly named databases. diff --git a/docs/packet-summaries.md b/docs/packet-summaries.md deleted file mode 100644 index 34c7ccf..0000000 --- a/docs/packet-summaries.md +++ /dev/null @@ -1,22 +0,0 @@ -# Packet summaries - -Packet lists, regional lists, reconnect backfill and live `packetObservation` events carry -an optional `summary` string. It comes from the packet's own stored metadata, never from a -node's current name or a fresh decode. - -| Payload | Summary | Source | -|---|---|---| -| ADVERT | Advertised name | Saved `appData.name` / verified decoded advert | -| ACK | `ACK 01020304` | Four-byte acknowledgement checksum | -| TRACE | `TRACE efbeadde` | Trace tag | -| TRACE classified as PING | `PING efbeadde` | Trace tag | - -- References are eight lowercase hex characters in the same byte order as packet detail's - `checksum` / `traceTag`. Zero is valid. Extended ACK retry bytes don't change the checksum. -- They are short references, not unique packet IDs or proof of delivery. No message body, - trace auth code or inferred sender/recipient is included. -- The text is projected from saved JSON in the existing list queries: no extra column, - backfill or per-packet lookup. A stored value of the wrong type or not exactly eight hex - characters omits the field. -- Other payload types have no summary yet. A missing summary doesn't mean the packet is - invalid. diff --git a/env.example b/env.example index f2d930d..c7a152e 100644 --- a/env.example +++ b/env.example @@ -1,3 +1,6 @@ +# Local development starter. Every variable is documented at +# https://github.com/MeshCore-Beacon/beacon-docs/blob/main/docs/configuration.md + LISTEN_ADDR=:8080 # Path to the YAML config (default: config.yaml). @@ -13,7 +16,7 @@ LISTEN_ADDR=:8080 POSTGRES_DSN=postgres://beacon:beacon@localhost:5432/beacon?sslmode=disable -# Redis (optional — leave REDIS_ADDR unset to disable caching) +# Redis (optional; leave REDIS_ADDR unset to disable caching) REDIS_ADDR=localhost:6379 REDIS_PASSWORD= REDIS_DB=0 diff --git a/internal/iatadb/gen/airports.csv b/internal/iatadb/gen/airports.csv index 561e33a..2070812 100644 --- a/internal/iatadb/gen/airports.csv +++ b/internal/iatadb/gen/airports.csv @@ -29945,7 +29945,7 @@ 599256,"GB-1333","heliport","Clifton Dykes",54.644211,-2.69254,,"EU","GB","GB-U-A",,"no",,,,,,, 599308,"GB-1334","closed","RAF Long Kesh",54.490669,-6.103835,115,"EU","GB","GB-NIR","Maze, County Down","no",,,,,,"https://en.wikipedia.org/wiki/RAF_Long_Kesh", 599309,"GB-1335","heliport","Lisburn Heliport",54.49328,-6.097745,118,"EU","GB","GB-NIR","Lisburn, County Down","no",,,,,,, -599921,"GB-1336","small_airport","(Duplicete)East Kirkby Airfield",53.13889,-0.00056,39,"EU","GB","GB-ENG","East Kirkby","no",,,,,"https://lincsaviation.co.uk","https://en.m.wikipedia.org/wiki/Lincolnshire_Aviation_Heritage_Centre?utm_source=chatgpt.com","RAF East Kirkby; Lincolnshire Aviation Heritage Centre; Bomber Command; Avro Lancaster NX611 “Just Jane”; Mosquito HJ711; East Lindsey; WW2 airfield; museum; heritage." +599921,"GB-1336","small_airport","(Duplicete)East Kirkby Airfield",53.13889,-0.00056,39,"EU","GB","GB-ENG","East Kirkby","no",,,,,"https://lincsaviation.co.uk","https://en.m.wikipedia.org/wiki/Lincolnshire_Aviation_Heritage_Centre","RAF East Kirkby; Lincolnshire Aviation Heritage Centre; Bomber Command; Avro Lancaster NX611 “Just Jane”; Mosquito HJ711; East Lindsey; WW2 airfield; museum; heritage." 599951,"GB-1337","closed","RAF Woodhall Spa",53.131667,-0.18444,33,"EU","GB","GB-ENG","East Linsdey, Lincolnshire","no",,,,,"https://www.lincstrust.org.uk/get-involved/top-reserves/woodhall-spa-airfield","https://en.wikipedia.org/wiki/RAF_Woodhall_Spa","Tattershall Thorpe, No. 617 Squadron, Dambusters, Bristol Bloodhound Missile site, Thorpe Camp Visitor Centre, Lincolnshire Wildlife Trust, RAF Coningsby satellite" 600636,"GB-1338","heliport","Chequers Helicopter Landing Ground",51.742163,-0.781822,,"EU","GB","GB-ENG","Ellesborough, Bucks.","no",,,,,,, 602208,"GB-1339","heliport","Uxbridge Business Park Heliport",51.554853,-0.483221,112,"EU","GB","GB-ENG","Uxbridge, Greater London","no",,,,,,, @@ -30972,7 +30972,7 @@ 30801,"HRYI","small_airport","Butare Airport",-2.59583,29.7367,5801,"AF","RW","RW-05","Butare","no","HRYI","BTQ","HRYI",,,"https://en.wikipedia.org/wiki/Butare_Airport", 31612,"HRYN","small_airport","Nemba Airport",-2.336167,30.206658,4905,"AF","RW","RW-02","Nemba","no","HRYN",,"HRYN",,,"https://en.wikipedia.org/wiki/Nemba_Airport", 31613,"HRYO","small_airport","Gabiro Airport",-1.5499999523162842,30.382999420166016,4806,"AF","RW","RW-02","Gabiro","no",,,"HRYO",,,, -3237,"HRYR","large_airport","Kigali International Airport",-1.96863,30.1395,4859,"AF","RW","RW-01","Kigali","yes","HRYR","KGL","HRYR",,"https://www.rac.co.rw/?utm_source=chatgpt.com","https://en.wikipedia.org/wiki/Kigali_International_Airport","Gregoire Kayibanda International Airport, Kanombe International Airport" +3237,"HRYR","large_airport","Kigali International Airport",-1.96863,30.1395,4859,"AF","RW","RW-01","Kigali","yes","HRYR","KGL","HRYR",,"https://www.rac.co.rw/","https://en.wikipedia.org/wiki/Kigali_International_Airport","Gregoire Kayibanda International Airport, Kanombe International Airport" 32208,"HRYU","small_airport","Ruhengeri Airport",-1.496873,29.631343,6102,"AF","RW","RW-03","Ruhengeri","no","HRYU","RHG","HRYU",,,"https://en.wikipedia.org/wiki/Ruhengeri_Airport", 3238,"HRZA","medium_airport","Kamembe Airport",-2.462239980697632,28.907899856567383,5192,"AF","RW","RW-04","Kamembe","yes","HRZA","KME","HRZA",,,"https://en.wikipedia.org/wiki/Kamembe_Airport", 43976,"HSAK","small_airport","Akobo Airport",7.7779,33.0027,1352,"AF","SS","SS-JG","Akobo","no",,,"HJAK",,,"https://en.wikipedia.org/wiki/Akobo_Airport","HSAK, Ahobo"