diff --git a/app_config/apache/analyzer-vhost.snippet.conf b/app_config/apache/analyzer-vhost.snippet.conf index 3d44faf..d6303ba 100644 --- a/app_config/apache/analyzer-vhost.snippet.conf +++ b/app_config/apache/analyzer-vhost.snippet.conf @@ -41,4 +41,4 @@ # # /etc/apache2/beacon-blocklist-ua.conf — one SetEnvIfNoCase per scraper User-Agent signature: # -# SetEnvIfNoCase User-Agent "mesh\.hansimgamr\.net" beacon_block +# SetEnvIfNoCase User-Agent "some-scraper" beacon_block diff --git a/app_config/fail2ban/README.md b/app_config/fail2ban/README.md index 9d7b07b..611b053 100644 --- a/app_config/fail2ban/README.md +++ b/app_config/fail2ban/README.md @@ -39,12 +39,13 @@ docker compose logs --tail 5 app ## 2. fail2ban -Two jails per box, same thresholds: +Three jails per box, same thresholds: | Jail | Counts | Trips at | Why | |---|---|---|---| | `beacon-api-flood` | every `/api/` request | 900 in 60 s | a browser session peaks ~150 in its first 30 s (auto-chained pages), then ~2/min; 15 req/s for a full minute is no browser | | `beacon-api-429` | `/api/` responses with status 429 | 300 in 10 min | only fires on clients that keep hammering through the server's throttle | +| `beacon-api-sustained` | every `/api/` request | 1,000 in 1 h | a paced scraper (the day-0 Pi ran ~135/min for hours) never trips the flood rule; real sessions peak ~300/h | Both use `backend = polling` (the Debian default `systemd` cannot tail files) and a flat 10 min ban. Install: diff --git a/app_config/fail2ban/apache/jail.d/beacon-http.conf b/app_config/fail2ban/apache/jail.d/beacon-http.conf index 1752db2..8e44955 100644 --- a/app_config/fail2ban/apache/jail.d/beacon-http.conf +++ b/app_config/fail2ban/apache/jail.d/beacon-http.conf @@ -29,3 +29,17 @@ maxretry = 300 findtime = 600 bantime = 10m ignoreip = 127.0.0.1/8 ::1 100.64.0.0/10 192.99.151.15 + +# A paced scraper stays under the per-minute flood rule; this counts the hour instead. +# Real sessions peak around 300/h (2026-09-03 observation); a scraper pacing at 2/s does ~7,000. +[beacon-api-sustained] +enabled = true +backend = polling +filter = beacon-api-flood +port = http,https +logpath = /var/log/apache2/analyzer-ssl-access.log +banaction = dummy +maxretry = 1000 +findtime = 1h +bantime = 10m +ignoreip = 127.0.0.1/8 ::1 100.64.0.0/10 192.99.151.15 diff --git a/app_config/fail2ban/caddy/jail.d/beacon-http.conf b/app_config/fail2ban/caddy/jail.d/beacon-http.conf index 2f45414..cf3cf13 100644 --- a/app_config/fail2ban/caddy/jail.d/beacon-http.conf +++ b/app_config/fail2ban/caddy/jail.d/beacon-http.conf @@ -34,3 +34,17 @@ findtime = 10m maxretry = 300 bantime = 10m maxmatches = 5 + +# A paced scraper stays under the per-minute flood rule; this counts the hour instead. +# Real sessions peak around 300/h (2026-09-03 observation); a scraper pacing at 2/s does ~7,000. +[beacon-api-sustained] +enabled = true +backend = polling +filter = beacon-api-flood +logpath = /opt/docker/beacon/data/Caddy/logs/access.json +banaction = dummy +ignoreip = 127.0.0.0/8 ::1 10.0.0.0/8 172.16.0.0/12 192.168.0.0/16 100.64.0.0/10 +findtime = 1h +maxretry = 1000 +bantime = 10m +maxmatches = 5 diff --git a/app_documentation/api_contract.md b/app_documentation/api_contract.md index 57271a7..2388d56 100644 --- a/app_documentation/api_contract.md +++ b/app_documentation/api_contract.md @@ -705,10 +705,13 @@ The node detail includes `iatasHeardIn`, `supportsMultibytePaths`, `supportsMult GET /api/v1/observers?iata=YOW&type=meshcoretomqtt&broker=mqtt1&status=online GET /api/v1/observers/{observerId} GET /api/v1/observers/{observerId}/telemetry?range=24h +GET /api/v1/observers/{observerId}/activity?range=24h&interval=15m GET /api/v1/observers/{observerId}/adverts?limit=50&cursor= ``` -Telemetry response is a time-bucketed array suitable for direct chart consumption: +Telemetry response is a time-bucketed array suitable for direct chart consumption. `airtimeTxSecs` and +`airtimeRxSecs` are seconds of radio time as reported by the observer: cumulative since boot on +`interval=1h` points, and the per-bucket delta on `6h`/`24h`. Divide by the bucket length for a percentage. ```json { @@ -718,8 +721,8 @@ Telemetry response is a time-bucketed array suitable for direct chart consumptio { "t": 1747612800000, "batteryMv": 4180, - "airtimeTxPct": 0.37, - "airtimeRxPct": 1.05, + "airtimeTxSecs": 13.4, + "airtimeRxSecs": 37.8, "noiseFloorDb": -103.2, "uptimeSeconds": 86400, "queueLength": 0, @@ -728,8 +731,8 @@ Telemetry response is a time-bucketed array suitable for direct chart consumptio { "t": 1747613100000, "batteryMv": 4175, - "airtimeTxPct": 0.42, - "airtimeRxPct": 1.12, + "airtimeTxSecs": 15.1, + "airtimeRxSecs": 40.3, "noiseFloorDb": -102.8, "uptimeSeconds": 86700, "queueLength": 1, @@ -739,6 +742,67 @@ Telemetry response is a time-bucketed array suitable for direct chart consumptio } ``` +Activity reports what an observer actually heard, bucketed over a trailing window. `range` is a Go +duration up to `720h` (default `24h`); `interval` is one of `5m`, `15m`, `1h`, `6h`, `24h` +(default `15m`). `range / interval` may not exceed 1000 buckets, sub-hour intervals (`5m`, `15m`) +additionally cap `range` at `48h`, and an unknown observer is a `404`. +Buckets are aligned to the clock in UTC (a `15m` bucket always starts at :00, :15, :30 or :45) and the +window start is rounded up to the next bucket boundary, so bucket starts are stable across requests. +Only non-empty buckets are returned — gaps in `points` mean the observer heard nothing in that bucket, +and the client is expected to render them as zero. + +`radio` echoes the observer's current radio parameters plus the preamble length the airtime maths +assumes; it is `null` when those parameters are unknown, in which case `airtimeMs` is `null` on every +bucket too (airtime cannot be costed without them). `snrAvg`, `snrMin` and `rssiAvg` are `null` for +buckets whose observations carried no usable signal readings. The `payloadTypes` counts cover the same +window and their total matches the sum of `observations` across `points`. On sub-hour intervals only, +observations with an undetermined payload type are counted in `observations` but omitted from the +breakdown; none exist within the retention window in practice. + +`airtimeMs` sums only the observations that could be costed. The migration that introduced airtime +backfills the trailing 7 days, so for the first 30 days after deploy buckets older than 7 days may be +only partially costed, and observations recorded without radio parameters are never costed. + +Intervals of `1h` and up are served from an hourly rollup refreshed on the `background.view_refresh` +interval (1 hour by default), so the most recent hour can lag by up to one refresh. Sub-hour intervals +read live observation rows and are always current. + +```json +{ + "range": "24h", + "interval": "15m", + "radio": { + "freqMhz": 869.525, + "sf": 11, + "bwKhz": 250, + "cr": 5, + "preambleSymbols": 16 + }, + "payloadTypes": [ + { "payloadType": 4, "payloadTypeName": "advert", "count": 128 }, + { "payloadType": 5, "payloadTypeName": "group_text", "count": 41 } + ], + "points": [ + { + "t": 1747612800000, + "observations": 12, + "airtimeMs": 3120.5, + "snrAvg": 6.2, + "snrMin": -4.5, + "rssiAvg": -92.3 + }, + { + "t": 1747613700000, + "observations": 4, + "airtimeMs": 980.2, + "snrAvg": 5.1, + "snrMin": 1.0, + "rssiAvg": -95.0 + } + ] +} +``` + ### Channels ``` diff --git a/app_documentation/high_level_design.md b/app_documentation/high_level_design.md index 02af2db..f2f9361 100644 --- a/app_documentation/high_level_design.md +++ b/app_documentation/high_level_design.md @@ -398,8 +398,8 @@ CREATE TABLE observer_telemetry ( observer_id UUID NOT NULL REFERENCES observers(id) ON DELETE CASCADE, reported_at TIMESTAMPTZ NOT NULL, battery_voltage_mv INT, - airtime_tx_pct REAL, - airtime_rx_pct REAL, + airtime_tx_secs REAL, + airtime_rx_secs REAL, noise_floor_db REAL, uptime_seconds BIGINT, queue_length INT, diff --git a/docker-deployment-type1/data/Caddy/conf.d/blocklist.caddy b/docker-deployment-type1/data/Caddy/conf.d/blocklist.caddy index 6de87d4..4ffc0b6 100644 --- a/docker-deployment-type1/data/Caddy/conf.d/blocklist.caddy +++ b/docker-deployment-type1/data/Caddy/conf.d/blocklist.caddy @@ -2,6 +2,7 @@ # @blocked_ip: one IP or CIDR per remote_ip line. Keep at least one line: 192.0.2.0/24 is TEST-NET # (never routable) and is a safe placeholder. # @blocked_ua: one header_regexp line per scraper User-Agent signature (Go regexp; (?i) = case-insensitive). +# The placeholder line never matches; keep it so the matcher is never empty. # Apply changes with: # docker compose exec caddy caddy validate --config /etc/caddy/Caddyfile # docker compose exec caddy caddy reload --config /etc/caddy/Caddyfile @@ -9,5 +10,5 @@ remote_ip 192.0.2.0/24 } @blocked_ua { - header_regexp User-Agent "(?i)mesh\.hansimgamr\.net" + header_regexp User-Agent "^BLOCKED-UA-PLACEHOLDER-NEVER-MATCHES$" }