diff --git a/CHANGELOG.md b/CHANGELOG.md index 1611b51..627b6f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,34 @@ semantic versioning. ## [Unreleased] +### Added + +- Region-code monitoring and an optional automatic warning to senders whose + channel messages carry no regional flood scope (#279). The bot classifies + every channel message it hears as scoped, global (unscoped) or unknown and + tallies the result per channel per day, which costs no airtime and is on by + default. A new `Settings -> Region Warnings` page in the web viewer shows how + much unscoped traffic the mesh actually carries before any decision to spend + airtime on it. + + Warnings themselves are off by default and start in dry run, where every + decision is logged and consumes the same cooldowns and daily cap it would + when live, so the log is a true preview rather than an upper bound. A sender + must send `min_unscoped_messages` confirmed-unscoped messages, and the + warning is fenced by a per-sender cooldown, a mesh-wide cooldown and a daily + cap, all read from the database so a restart cannot release a burst. Warnings + fire only on positive RF evidence of an unscoped FLOOD; a message the radio + could not classify is never warned about. Channel-delivered warnings are sent + at global scope, because a scoped reply could not reach someone outside the + region. Configure it in `[Region_Warnings]`; retention is governed by + `[Data_Retention] region_warning_retention_days`. + ### Fixed +- Striped and hovered table rows in the web viewer's dark mode no longer render + Bootstrap's light-theme text color on a dark background (about 1.3:1 + contrast). The dark overrides set a background but not a color, so every + `.table-striped` page was affected. - Daily Weather Service forecasts now retry transient Open-Meteo failures at 5, 15, and 30 minutes after the original run (#264). HTTP 429, 500, 502, 503, and 504 responses plus transport failures use one replaceable retry job, diff --git a/config.ini.example b/config.ini.example index dcef21f..2e6709a 100644 --- a/config.ini.example +++ b/config.ini.example @@ -309,6 +309,66 @@ max_response_hops = 7 # flood_scope.general = #west # flood_scope.weather = #sea # flood_scope.#local = * + +[Region_Warnings] +# Regional flood scope ("region code") monitoring, and an optional automatic +# note to senders who never set one. +# +# A MeshCore client with no region configured sends every channel message as a +# plain FLOOD, which every repeater on the mesh rebroadcasts. A client with a +# region sends a TC_FLOOD whose transport code confines it to repeaters holding +# that region key. This section is about the first kind. +# +# Tallying is free and on by default: the bot classifies each channel message it +# hears as scoped, global (unscoped) or unknown and counts it per channel per +# day. The Region Warnings page in the web viewer shows the result. Nothing is +# transmitted unless you also set enabled = true below. +# +# A message is only ever counted as global on positive RF evidence that it was +# an unscoped FLOOD. When the radio did not witness enough to tell, it counts as +# unknown and can never earn a warning. +track_traffic = true + +# Send a warning to senders whose messages are confirmed unscoped. +# This spends airtime automatically. Off by default. +enabled = false + +# Preview mode. While true, warnings are decided and logged exactly as they +# would be sent β€” including consuming the cooldowns and the daily cap β€” but +# nothing is transmitted. Watch the Region Warnings page for a while before +# turning this off. +dry_run = true + +# How the warning reaches the sender: +# dm β€” a direct message to the sender (default; costs the least airtime) +# channel β€” a reply on the channel they posted to, always at global scope, +# because a scoped reply would not reach someone outside your region +delivery = dm + +# Only warn about messages on these channels (comma-separated, # optional). +# Leave empty to consider every channel the bot hears. +# channels = general, public + +# The warning text. {sender} and {channel} are substituted. Keep it short: a DM +# body is 158 bytes and a channel reply is smaller still, and anything longer is +# truncated. For delivery = channel, include @[{sender}] so they see it. +# message = Heads up: your messages have no region code, so they flood the whole mesh. Setting one in your MeshCore app keeps things quiet. Thanks! + +# How many confirmed-unscoped messages a sender must send before they earn a +# warning, so one stray global flood does not cost anyone a DM. The run resets +# after 24 hours of quiet, when the sender posts a scoped message, and on +# restart. +min_unscoped_messages = 3 + +# Do not warn the same sender again within this many hours. 0 disables. +per_sender_cooldown_hours = 168 + +# Minimum gap between warnings to anyone, in minutes. 0 disables. +mesh_cooldown_minutes = 30 + +# Hard ceiling on warnings per local day. 0 means unlimited (not recommended). +max_warnings_per_day = 6 + [Banned_Users] # List of banned sender names (comma-separated). Matching is prefix (starts-with): # "Awful Username" also matches "Awful Username πŸ†". No bot responses in channels or DMs. @@ -1031,6 +1091,10 @@ mesh_connections_retention_days = 7 # grows very slowly and a long window is cheap. The neighbor_links aggregate that # the mesh graph reads is never pruned. neighbor_observations_retention_days = 365 +# +# Regional flood scope tallies and region-warning events (see [Region_Warnings]). +# Both are tiny β€” one tally row per channel per day β€” so a long window is cheap. +region_warning_retention_days = 90 [Path_Command] # Enable or disable the path command enabled = true diff --git a/docs/region-warnings.md b/docs/region-warnings.md new file mode 100644 index 0000000..80defd5 --- /dev/null +++ b/docs/region-warnings.md @@ -0,0 +1,115 @@ +# Region warnings + +MeshCore puts a channel message on the air in one of two ways. + +An ordinary `FLOOD` is rebroadcast by every repeater that hears it, anywhere on the mesh. A `TC_FLOOD` carries a transport code derived from a region key β€” what the apps call a **region code** β€” and only repeaters holding that key pass it on. + +A client with no region configured therefore floods the entire mesh with every message it sends. On a busy mesh that is most of the airtime nobody asked for. + +This feature does two separate things: + +- **Counts** how many of the channel messages your bot hears were unscoped, per channel, per day. Free, on by default, transmits nothing. +- **Warns** the senders, when you explicitly turn that on. Spends airtime automatically, so it is off by default and starts in dry run. + +The Web Viewer page at **Settings β†’ Region Warnings** shows both. + +## How a message is classified + +Each channel message gets one of three verdicts. + +| Verdict | Meaning | Evidence | +| --- | --- | --- | +| `scoped` | A region code was set | The message matched a configured `flood_scopes` entry, or the correlated RF packet was a `TC_FLOOD` carrying a transport code | +| `global` | No region code | The correlated RF packet was an ordinary `FLOOD`, or no scope-eligible packet was heard anywhere in the correlation window | +| `unknown` | The radio could not tell | Anything else | + +Only `global` can earn a warning, and only on positive evidence. If the radio did not witness enough to decide, the message counts as `unknown` and no warning is possible. That matters: absence of correlation is not proof that a sender omitted a region, and this feature answers that ambiguity by staying quiet. + +`unknown` is excluded from the unscoped percentage rather than counted as clean, so a mesh the bot cannot classify reads as "no data" instead of "no problem". + +Messages the radio cached from before the current connection are skipped entirely β€” a reconnect replays them as a burst, and counting them would both distort the tallies and let stale traffic earn someone a warning. + +## Reading the page + +**Channel traffic by scope** breaks the window down per channel. Each bar is unscoped / scoped / couldn't-tell, and the numbers are printed beside it. + +**Daily volume** is the same data per day, plus the headline share. + +A channel sitting at a high unscoped percentage is where a warning would do the most good. A channel dominated by "couldn't tell" means your radio is not hearing enough RF detail to judge, and warnings there will rarely fire. + +## Turning warnings on + +The **What the bot does** control has three positions: + +- **Count only** β€” tallies for the page, nothing transmitted. The default. +- **Dry run** β€” warnings are decided and logged exactly as they would be sent, including consuming the cooldowns and the daily cap, but nothing is transmitted. The log underneath shows precisely what going live would put on the air. +- **Send warnings** β€” transmits. + +Run it in dry run for a few days first. Because dry run spends the same budget, the log is a true preview rather than an upper bound. + +### Delivery + +- **Direct message** (default) β€” the sender alone sees it, and it is the cheaper of the two. +- **Channel reply** β€” sent at **global** scope on purpose. The recipient is by definition outside any region your bot replies under, so a scoped reply would never reach them. Everyone on the channel sees it. + +### Limits + +| Setting | Default | What it does | +| --- | --- | --- | +| `min_unscoped_messages` | 3 | Confirmed-unscoped messages from one sender before they earn a warning. The run resets after 24h of quiet, when the sender posts a scoped message, and on restart. | +| `per_sender_cooldown_hours` | 168 | Do not warn the same sender again within this many hours. 0 disables. | +| `mesh_cooldown_minutes` | 30 | Minimum gap between warnings to anyone. 0 disables. | +| `max_warnings_per_day` | 6 | Hard ceiling per local day. 0 means unlimited. | + +Banned users are never warned, and the bot never warns itself (identified by public key, falling back to `[Bot] bot_name`). `channelpause` silences warnings along with everything else on channels. + +The daily cap counts **attempts**, failures included: its job is to bound how much unprompted activity this feature can produce in a day, and a send that reported failure may still have put something on the air before it did. + +A failed send does not start the mesh cooldown β€” usually nothing was transmitted, so it should not silence the next sender β€” but it does spend that sender's run, so a permanently unreachable contact is not retried on their every message. + +Both cooldowns and the cap read from the database rather than from memory, so restarting the bot does not release a burst of warnings. + +### Message + +`{sender}` and `{channel}` are substituted. Keep it short: a DM body is 158 UTF-8 bytes and a channel reply is smaller still (160 minus your bot's name), and anything longer is truncated. The page shows the byte count live against whichever limit applies. + +For channel delivery, include `@[{sender}]` so the person you are addressing sees it. + +## Configuration + +Everything the page writes lives in `[Region_Warnings]`: + +```ini +[Region_Warnings] +track_traffic = true +enabled = false +dry_run = true +delivery = dm +# channels = general, public +# message = Heads up: your messages have no region code, ... +min_unscoped_messages = 3 +per_sender_cooldown_hours = 168 +mesh_cooldown_minutes = 30 +max_warnings_per_day = 6 +``` + +`channels` is an allowlist; empty means every channel the bot hears. The leading `#` is optional and case does not matter. + +Saving from the Web Viewer queues a hot config reload, so changes take effect without a restart. + +## Storage and retention + +Two small tables: + +- `region_scope_daily` β€” one row per channel per local date holding the three counts. +- `region_warning_events` β€” one row per warning decision that reached the send stage (sent, failed, or withheld by dry run). Suppressions by cooldown or cap are deliberately not rows; they are the common case and would bury the log. + +Both are pruned by `[Data_Retention] region_warning_retention_days` (default 90). + +Set `track_traffic = false` to stop writing tallies. Warnings still work; you just lose the evidence the page is built on. + +## Relationship to `flood_scopes` + +`[Channels] flood_scopes` decides which messages the bot will *reply* to. Region warnings observe every channel message regardless, and the observation runs before that allowlist β€” an unscoped message is exactly what a scoped allowlist drops, so measuring after the gate would blind the monitor to the traffic it exists to measure. + +If you have `flood_scopes` configured with region names and no `*`, unscoped messages get no command replies at all. Region warnings still see them, and can still tell the sender why the bot is ignoring them. diff --git a/docs/web-viewer.md b/docs/web-viewer.md index 6136ac2..03d8319 100644 --- a/docs/web-viewer.md +++ b/docs/web-viewer.md @@ -307,6 +307,10 @@ window; older days stay frozen at the value recorded then. The viewer also provides JSON API endpoints: +- `GET /api/region-warnings` - Region-code settings, per-channel scope tallies, + the daily series, today's warning budget, and recent warning decisions +- `POST /api/region-warnings/settings` - Save `[Region_Warnings]` and queue a + hot config reload - `GET /api/dashboard/summary` - Snapshot-backed dashboard payload, including 30-day sparkline series and change figures, plus `packet_encoding`: 30 days of raw per-payload-type multibyte/total counts for the stacked encoding chart. diff --git a/mkdocs.yml b/mkdocs.yml index a21d984..0e0561a 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -61,6 +61,7 @@ nav: - Check-in API: checkin-api.md - Path Command: path-command-config.md - Config validation: config-validation.md + - Region warnings: region-warnings.md - Web Viewer: web-viewer.md - Command Reference: - Overview: command-reference.md diff --git a/modules/config_schema.py b/modules/config_schema.py index 3d36cf5..f500dc2 100644 --- a/modules/config_schema.py +++ b/modules/config_schema.py @@ -266,6 +266,18 @@ SECTIONS: dict[str, SectionMeta] = { "defer_to_human_greeting": KeyMeta(type="bool"), "levenshtein_distance": KeyMeta(type="int", default="0"), }), + "Region_Warnings": SectionMeta(keys={ + "enabled": KeyMeta(type="bool", default="false"), + "dry_run": KeyMeta(type="bool", default="true"), + "delivery": KeyMeta(default="dm"), + "channels": KeyMeta(), + "message": KeyMeta(), + "min_unscoped_messages": KeyMeta(type="int", default="3"), + "per_sender_cooldown_hours": KeyMeta(type="float", default="168"), + "mesh_cooldown_minutes": KeyMeta(type="float", default="30"), + "max_warnings_per_day": KeyMeta(type="int", default="6"), + "track_traffic": KeyMeta(type="bool", default="true"), + }), "Announcements_Command": SectionMeta(dynamic_keys=True), "Alert_Command": SectionMeta(dynamic_keys=True), } diff --git a/modules/config_validation.py b/modules/config_validation.py index dc324cf..12c6ffa 100644 --- a/modules/config_validation.py +++ b/modules/config_validation.py @@ -71,6 +71,7 @@ CANONICAL_NON_COMMAND_SECTIONS = frozenset({ "DiscordBridge", "TelegramBridge", "DARC_MoWaS_Service", + "Region_Warnings", }) # Sections required for the bot to start (accessed without has_section guards) diff --git a/modules/core.py b/modules/core.py index c1ea460..ea17513 100644 --- a/modules/core.py +++ b/modules/core.py @@ -326,6 +326,14 @@ class MeshCoreBot: self.message_handler = MessageHandler(self) self.command_manager = CommandManager(self) + # Regional flood-scope tallies, and the opt-in warning they can drive. + try: + from .region_warning import RegionWarningMonitor + self.region_warning_monitor = RegionWarningMonitor(self) + except Exception as e: + self.logger.warning(f"Failed to initialize region warning monitor: {e}") + self.region_warning_monitor = None + # Initialize transmission tracker for monitoring TX success try: self.transmission_tracker = TransmissionTracker(self) @@ -963,6 +971,9 @@ class MeshCoreBot: self.channel_manager.max_channels = new_max_channels set_config(new_config) + if getattr(self, 'region_warning_monitor', None): + self.region_warning_monitor.reload_config() + if hasattr(self, 'scheduler'): scheduler_apply_started = True self.scheduler.setup_scheduled_messages() @@ -988,6 +999,8 @@ class MeshCoreBot: ) self.channel_manager.max_channels = old_state["max_channels"] set_config(old_config) + if getattr(self, 'region_warning_monitor', None): + self.region_warning_monitor.reload_config() # setup_scheduled_messages may have stopped the previous # APScheduler before failing. Rebuild it against old config. if scheduler_apply_started and hasattr(self, 'scheduler'): diff --git a/modules/db_manager.py b/modules/db_manager.py index c795328..a7a73a2 100644 --- a/modules/db_manager.py +++ b/modules/db_manager.py @@ -61,6 +61,8 @@ class DBManager: 'observed_paths', # Repeater manager - observed paths from adverts and messages 'neighbor_links', # Zero-hop neighbor discovery - current adjacency 'neighbor_observations', # Zero-hop neighbor discovery - per-cycle history + 'region_scope_daily', # Regional flood scope tallies per channel per day + 'region_warning_events', # Region-code warning decisions } def __init__(self, bot: Any, db_path: str = "meshcore_bot.db"): diff --git a/modules/db_migrations.py b/modules/db_migrations.py index 4c3097a..64b7fdd 100644 --- a/modules/db_migrations.py +++ b/modules/db_migrations.py @@ -800,6 +800,53 @@ def _m0023_observed_paths_zero_hop_signal(cursor: sqlite3.Cursor) -> None: _add_column(cursor, "observed_paths", "snr", "REAL") _add_column(cursor, "observed_paths", "rssi", "REAL") +def _m0024_region_scope_tables(cursor: sqlite3.Cursor) -> None: + """Storage for regional flood-scope observation and the warnings it drives. + + ``region_scope_daily`` is a per-day, per-channel tally of how each channel + message's flood scope was classified. It is written for every channel + message the bot hears, independent of whether warnings are enabled, because + the whole point is letting an operator see how much unscoped traffic there + actually is *before* deciding to spend airtime telling anyone about it. + One row per channel per local date keeps that free. + + ``region_warning_events`` holds one row per warning decision that reached + the send stage β€” actually sent, suppressed by a failed send, or withheld + because the feature is in dry-run. Suppressions by cooldown or daily cap + are deliberately *not* rows: they are the common case and would bury the + log. The cooldown and cap windows are derived from this table rather than + from memory so they survive a restart, which is why dry-run rows are + written too β€” a dry run has to consume the same budget it is previewing. + """ + cursor.executescript( + """ + CREATE TABLE IF NOT EXISTS region_scope_daily ( + date TEXT NOT NULL, + channel TEXT NOT NULL, + scoped_count INTEGER NOT NULL DEFAULT 0, + global_count INTEGER NOT NULL DEFAULT 0, + unknown_count INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (date, channel) + ); + + CREATE TABLE IF NOT EXISTS region_warning_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + created_at TIMESTAMP NOT NULL, + sender_id TEXT NOT NULL, + sender_pubkey TEXT, + channel TEXT, + delivery TEXT NOT NULL, + action TEXT NOT NULL, + detail TEXT + ); + + CREATE INDEX IF NOT EXISTS idx_region_warning_events_created_at + ON region_warning_events(created_at); + CREATE INDEX IF NOT EXISTS idx_region_warning_events_sender + ON region_warning_events(sender_id, created_at); + """ + ) + # --------------------------------------------------------------------------- # Migration registry β€” append new entries here, never remove or reorder. @@ -831,6 +878,7 @@ MIGRATIONS: list[MigrationEntry] = [ (21, "daily_rollup: per-payload-type multibyte split", _m0021_daily_rollup_packet_type_encoding), (22, "neighbor discovery tables", _m0022_neighbor_tables), (23, "observed_paths: snr/rssi for zero-hop adverts", _m0023_observed_paths_zero_hop_signal), + (24, "regional flood scope tallies and warning events", _m0024_region_scope_tables), ] diff --git a/modules/maintenance.py b/modules/maintenance.py index b866315..47f1f37 100644 --- a/modules/maintenance.py +++ b/modules/maintenance.py @@ -159,6 +159,9 @@ class MaintenanceRunner: 'Data_Retention', 'neighbor_observations_retention_days', 365 ) stats_days = get_retention_days('Stats_Command', 'data_retention_days', 7) + region_warning_days = get_retention_days( + 'Data_Retention', 'region_warning_retention_days', 90 + ) try: if hasattr(self.bot, 'web_viewer_integration') and self.bot.web_viewer_integration: @@ -205,6 +208,8 @@ class MaintenanceRunner: # pruned here (losing it would silently drop confirmed direct links). self._cleanup_neighbor_observations(neighbor_observations_days) + self._cleanup_region_scope_history(region_warning_days) + ran_at = _utc_now().isoformat() self._last_retention_stats['ran_at'] = ran_at try: @@ -223,6 +228,33 @@ class MaintenanceRunner: except Exception: pass + def _cleanup_region_scope_history(self, retention_days: int) -> None: + """Prune region-scope tallies and warning events past the retention window. + + Both tables are small β€” one tally row per channel per day, and warning + events are rate-limited by construction β€” so the window is generous. + """ + if retention_days <= 0: + return + db_manager = getattr(self.bot, 'db_manager', None) + if not db_manager or not hasattr(db_manager, 'delete_timestamp_rows_in_chunks'): + return + cutoff_date = (_utc_now() - datetime.timedelta(days=retention_days)).date().isoformat() + try: + db_manager.delete_timestamp_rows_in_chunks( + 'region_scope_daily', 'date', cutoff_date, + progress_label='region_scope_daily retention', + ) + except Exception as e: + self.logger.warning(f"Region scope tally retention failed: {e}") + try: + db_manager.delete_timestamp_rows_in_chunks( + 'region_warning_events', 'created_at', cutoff_date, + progress_label='region_warning_events retention', + ) + except Exception as e: + self.logger.warning(f"Region warning event retention failed: {e}") + def _cleanup_neighbor_observations(self, retention_days: int) -> None: """Prune zero-hop neighbor observation history past the retention window. diff --git a/modules/message_handler.py b/modules/message_handler.py index e41c42b..a8471d8 100644 --- a/modules/message_handler.py +++ b/modules/message_handler.py @@ -17,6 +17,7 @@ from .graph_trace_helper import update_mesh_graph_from_trace_data from .meshcore_payload_decode import channel_hash_for_key, decrypt_group_text from .models import MeshMessage from .neighbors_discovery import upsert_zero_hop_observed_path_via_manager +from .region_warning import VERDICT_GLOBAL, VERDICT_SCOPED, VERDICT_UNKNOWN from .security_utils import sanitize_input, sanitize_name from .utils import ( calculate_packet_hash, @@ -318,6 +319,95 @@ class MessageHandler: return False return int(payload_type) == self._grp_txt_payload_type_int() + def _classify_channel_flood_scope( + self, + *, + reply_scope: str | None, + recent_rf_data: dict[str, Any] | None, + packet_info: dict[str, Any] | None, + scope_rf_data: dict[str, Any] | None, + scope_packet_info: dict[str, Any] | None, + ) -> str: + """Classify a channel message's flood scope as scoped, global, or unknown. + + "Scoped" means the message carried a region code (a TC_FLOOD transport + code), whether or not that code matches one of ours. "Global" means it + was proven to be an ordinary unscoped FLOOD. Anything else is unknown. + + The scoped tests run first and the global test last, so every ambiguity + resolves away from "global". That direction matters: ``global`` is the + verdict that can spend airtime telling someone to fix their config, and + a message whose scope the radio simply did not witness is not evidence + that the sender omitted a region. + """ + if reply_scope: + return VERDICT_SCOPED + + # A correlated scope-eligible row *is* a TC_FLOOD GRP_TXT for this + # message: it carried a transport code, so a region was set even though + # it is not one this bot has keys for. + if ( + scope_rf_data + and rf_data_is_correlated(scope_rf_data) + and self._is_rf_data_scope_eligible(scope_rf_data, scope_packet_info) + ): + return VERDICT_SCOPED + + if recent_rf_data and rf_data_is_correlated(recent_rf_data): + route_type = self._effective_route_type_int(recent_rf_data, packet_info) + if route_type == int(RouteType.TRANSPORT_FLOOD.value): + return VERDICT_SCOPED + + if self._is_confirmed_global_flood( + recent_rf_data, + packet_info, + scoped_traffic_in_window=scope_rf_data is not None, + ): + return VERDICT_GLOBAL + + return VERDICT_UNKNOWN + + async def _observe_flood_scope( + self, + *, + sender_id: str | None, + sender_pubkey: str | None, + channel: str | None, + sender_timestamp: Any, + reply_scope: str | None, + recent_rf_data: dict[str, Any] | None, + packet_info: dict[str, Any] | None, + scope_rf_data: dict[str, Any] | None, + scope_packet_info: dict[str, Any] | None, + ) -> None: + """Hand this channel message's scope verdict to the region-warning monitor. + + Messages the radio cached from before this connection are skipped: on a + reconnect they arrive as a burst of old traffic, and counting them would + both distort the tallies and let a stale message earn someone a warning. + """ + monitor = getattr(self.bot, "region_warning_monitor", None) + if monitor is None: + return + if self._is_old_cached_message(sender_timestamp): + return + try: + verdict = self._classify_channel_flood_scope( + reply_scope=reply_scope, + recent_rf_data=recent_rf_data, + packet_info=packet_info, + scope_rf_data=scope_rf_data, + scope_packet_info=scope_packet_info, + ) + await monitor.observe( + verdict=verdict, + sender_id=sender_id, + sender_pubkey=sender_pubkey, + channel=channel, + ) + except Exception: + self.logger.exception("Flood scope observation failed") + def _is_old_cached_message(self, timestamp: Any) -> bool: """Check if a message timestamp indicates it's from before bot connection. @@ -2742,6 +2832,22 @@ class MessageHandler: "cannot authorise a reply under flood_scopes" ) + # Region-code observation happens here, ahead of the flood_scopes + # allowlist below, because an unscoped message is exactly what that + # allowlist drops β€” running it after the gate would blind the + # monitor to the traffic it exists to measure. + await self._observe_flood_scope( + sender_id=sender_id, + sender_pubkey=payload.get("pubkey_prefix", ""), + channel=channel_name, + sender_timestamp=payload.get("sender_timestamp", 0), + reply_scope=reply_scope, + recent_rf_data=recent_rf_data, + packet_info=packet_info, + scope_rf_data=scope_rf_data, + scope_packet_info=scope_packet_info, + ) + # Allowlist enforcement: when flood_scopes is configured, only reply to # messages whose scope matched an entry. Unscoped FLOOD is allowed only # when '*' (or equivalent) is explicitly listed. diff --git a/modules/region_warning.py b/modules/region_warning.py new file mode 100644 index 0000000..7bac72b --- /dev/null +++ b/modules/region_warning.py @@ -0,0 +1,753 @@ +#!/usr/bin/env python3 +"""Regional flood-scope observation and the optional "set a region code" warning. + +MeshCore puts a channel message on the air in one of two ways. An ordinary +``FLOOD`` is rebroadcast by every repeater that hears it, anywhere on the mesh. +A ``TC_FLOOD`` carries a transport code derived from a region key β€” the "region +code" β€” and only repeaters holding that key pass it on. A client with no region +configured therefore floods the entire mesh with every message it sends, which +is the traffic problem this module is about (issue #279). + +Two things live here: + +* **Observation.** Every channel message the bot hears is classified as + ``scoped``, ``global`` or ``unknown`` and tallied into ``region_scope_daily``. + This costs one upsert per message and no airtime, and it is what lets an + operator see how much unscoped traffic their mesh actually carries *before* + deciding whether telling anyone about it is worth the airtime. + +* **Warning.** When explicitly enabled, a sender whose messages are confirmed + unscoped can be sent a short note asking them to set a region code. This + spends airtime automatically, so it is off by default, starts in dry-run, and + is fenced by a per-sender cooldown, a mesh-wide cooldown and a daily cap. + +The classification only ever warns on *positive* evidence of a global flood. +Absence of evidence classifies as ``unknown`` and is never warned about β€” see +``MessageHandler._classify_channel_flood_scope``, which produces the verdict +this module consumes. + +The pure ``load_settings`` / summary helpers take a ``configparser`` and a +``DBManager`` rather than the bot, because the web viewer runs in a separate +process and has no bot object. +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass, field +from datetime import date, datetime, timedelta +from typing import Any, Optional + +CONFIG_SECTION = "Region_Warnings" + +# Verdicts produced by MessageHandler._classify_channel_flood_scope. +VERDICT_SCOPED = "scoped" +VERDICT_GLOBAL = "global" +VERDICT_UNKNOWN = "unknown" +VERDICTS = (VERDICT_SCOPED, VERDICT_GLOBAL, VERDICT_UNKNOWN) + +_VERDICT_COLUMN = { + VERDICT_SCOPED: "scoped_count", + VERDICT_GLOBAL: "global_count", + VERDICT_UNKNOWN: "unknown_count", +} + +DELIVERY_DM = "dm" +DELIVERY_CHANNEL = "channel" + +ACTION_SENT = "sent" +ACTION_DRY_RUN = "dry_run" +ACTION_FAILED = "failed" + +DEFAULT_MESSAGE = ( + "Heads up: your messages have no region code, so they flood the whole mesh. " + "Setting one in your MeshCore app keeps things quiet. Thanks!" +) + +# Body budget for a DM, matching CommandManager.get_max_message_length. +DM_BODY_LIMIT = 158 + + +@dataclass(frozen=True) +class RegionWarningSettings: + """Resolved ``[Region_Warnings]`` configuration.""" + + enabled: bool = False + dry_run: bool = True + delivery: str = DELIVERY_DM + channels: tuple[str, ...] = () + message: str = DEFAULT_MESSAGE + min_unscoped_messages: int = 3 + per_sender_cooldown_hours: float = 168.0 + mesh_cooldown_minutes: float = 30.0 + max_warnings_per_day: int = 6 + track_traffic: bool = True + + def monitors_channel(self, channel: Optional[str]) -> bool: + """Whether ``channel`` is in the allowlist (an empty allowlist means all).""" + if not self.channels: + return True + return normalize_channel(channel) in self.channels + + +def normalize_channel(channel: Optional[str]) -> str: + """Case-fold a channel name and drop a leading ``#`` so config matches the wire.""" + return (channel or "").strip().lstrip("#").lower() + + +def _get(config: Any, key: str, fallback: str = "") -> str: + try: + if config is not None and config.has_option(CONFIG_SECTION, key): + return (config.get(CONFIG_SECTION, key) or "").strip() + except Exception: + pass + return fallback + + +def _get_bool(config: Any, key: str, fallback: bool) -> bool: + raw = _get(config, key).lower() + if raw in ("1", "true", "yes", "on"): + return True + if raw in ("0", "false", "no", "off"): + return False + return fallback + + +def _get_number(config: Any, key: str, fallback: float, *, minimum: float = 0.0) -> float: + raw = _get(config, key) + if not raw: + return fallback + try: + value = float(raw) + except ValueError: + return fallback + return max(minimum, value) + + +def load_settings(config: Any) -> RegionWarningSettings: + """Read ``[Region_Warnings]`` into a settings object, falling back to defaults. + + Unparseable values fall back rather than raising: this runs on every config + reload in the message path, and a typo should not stop the bot handling mail. + """ + delivery = _get(config, "delivery", DELIVERY_DM).lower() + if delivery not in (DELIVERY_DM, DELIVERY_CHANNEL): + delivery = DELIVERY_DM + + channels = tuple( + normalize_channel(part) + for part in _get(config, "channels").split(",") + if normalize_channel(part) + ) + + message = _get(config, "message") or DEFAULT_MESSAGE + + return RegionWarningSettings( + enabled=_get_bool(config, "enabled", False), + dry_run=_get_bool(config, "dry_run", True), + delivery=delivery, + channels=channels, + message=message, + min_unscoped_messages=int(_get_number(config, "min_unscoped_messages", 3, minimum=1)), + per_sender_cooldown_hours=_get_number(config, "per_sender_cooldown_hours", 168.0), + mesh_cooldown_minutes=_get_number(config, "mesh_cooldown_minutes", 30.0), + max_warnings_per_day=int(_get_number(config, "max_warnings_per_day", 6)), + track_traffic=_get_bool(config, "track_traffic", True), + ) + + +def settings_to_config_values(settings: RegionWarningSettings) -> dict[str, str]: + """Render settings back to INI strings for :mod:`modules.settings_store`.""" + def _num(value: float) -> str: + return str(int(value)) if float(value).is_integer() else str(value) + + return { + "enabled": "true" if settings.enabled else "false", + "dry_run": "true" if settings.dry_run else "false", + "delivery": settings.delivery, + "channels": ", ".join(settings.channels), + "message": settings.message, + "min_unscoped_messages": str(settings.min_unscoped_messages), + "per_sender_cooldown_hours": _num(settings.per_sender_cooldown_hours), + "mesh_cooldown_minutes": _num(settings.mesh_cooldown_minutes), + "max_warnings_per_day": str(settings.max_warnings_per_day), + "track_traffic": "true" if settings.track_traffic else "false", + } + + +def render_message(template: str, sender: Optional[str], channel: Optional[str]) -> str: + """Substitute ``{sender}`` / ``{channel}``, leaving unknown braces untouched.""" + text = template or DEFAULT_MESSAGE + replacements = { + "{sender}": sender or "", + "{channel}": channel or "", + } + for token, value in replacements.items(): + text = text.replace(token, value) + return text.strip() + + +def truncate_to_bytes(text: str, limit: int) -> str: + """Trim ``text`` to ``limit`` UTF-8 bytes without splitting a character.""" + encoded = text.encode("utf-8") + if len(encoded) <= limit: + return text + return encoded[:limit].decode("utf-8", errors="ignore") + + +# --------------------------------------------------------------------------- +# Read helpers β€” shared with the web viewer, which has no bot object +# --------------------------------------------------------------------------- + + +def local_now(config: Any, logger: Any = None) -> datetime: + """Now in the configured ``[Bot] timezone``, as a naive datetime. + + Everything this module stores and compares β€” tallies, the daily cap, both + cooldowns β€” goes through here, so the whole feature agrees on which day it + is even when ``[Bot] timezone`` differs from the host's. Naive, because the + values are written to SQLite and read back by ``date(created_at)``, which + has no notion of offsets. + """ + try: + from modules.utils import get_config_timezone + + tz, _name = get_config_timezone(config, logger) + return datetime.now(tz).replace(tzinfo=None) + except Exception: + return datetime.now() + + +def _local_today(config: Any, logger: Any = None) -> date: + """Today's date in the configured ``[Bot] timezone``.""" + return local_now(config, logger).date() + + +def traffic_summary( + db_manager: Any, + config: Any = None, + days: int = 14, + logger: Any = None, +) -> dict[str, Any]: + """Per-channel scoped/global/unknown tallies over the last ``days`` local days.""" + days = max(1, int(days)) + since = (_local_today(config, logger) - timedelta(days=days - 1)).isoformat() + try: + rows = db_manager.execute_query( + "SELECT channel, " + "SUM(scoped_count) AS scoped, " + "SUM(global_count) AS global_, " + "SUM(unknown_count) AS unknown " + "FROM region_scope_daily WHERE date >= ? " + "GROUP BY channel ORDER BY (SUM(global_count) + SUM(scoped_count) + SUM(unknown_count)) DESC", + (since,), + ) + except Exception: + rows = [] + + channels: list[dict[str, Any]] = [] + totals: dict[str, Any] = {"scoped": 0, "global": 0, "unknown": 0} + for row in rows: + scoped = int(row.get("scoped") or 0) + globally = int(row.get("global_") or 0) + unknown = int(row.get("unknown") or 0) + total = scoped + globally + unknown + if total <= 0: + continue + channels.append({ + "channel": row.get("channel") or "", + "scoped": scoped, + "global": globally, + "unknown": unknown, + "total": total, + # Share of *classified* traffic that was unscoped. Unknown messages + # are excluded from the denominator rather than counted as scoped, + # so a mesh the bot cannot classify reads as "no data", not "clean". + "unscoped_pct": round(100.0 * globally / (scoped + globally), 1) if (scoped + globally) else None, + }) + totals["scoped"] += scoped + totals["global"] += globally + totals["unknown"] += unknown + + classified = totals["scoped"] + totals["global"] + totals["total"] = classified + totals["unknown"] + totals["unscoped_pct"] = round(100.0 * totals["global"] / classified, 1) if classified else None + return {"days": days, "since": since, "channels": channels, "totals": totals} + + +def daily_series( + db_manager: Any, + config: Any = None, + days: int = 14, + logger: Any = None, +) -> list[dict[str, Any]]: + """One row per local date over the window, with zero-filled gaps.""" + days = max(1, int(days)) + today = _local_today(config, logger) + since = (today - timedelta(days=days - 1)).isoformat() + try: + rows = db_manager.execute_query( + "SELECT date, SUM(scoped_count) AS scoped, SUM(global_count) AS global_, " + "SUM(unknown_count) AS unknown FROM region_scope_daily " + "WHERE date >= ? GROUP BY date", + (since,), + ) + except Exception: + rows = [] + by_date = { + row.get("date"): ( + int(row.get("scoped") or 0), + int(row.get("global_") or 0), + int(row.get("unknown") or 0), + ) + for row in rows + } + series = [] + for offset in range(days - 1, -1, -1): + key = (today - timedelta(days=offset)).isoformat() + scoped, globally, unknown = by_date.get(key, (0, 0, 0)) + series.append({ + "date": key, + "scoped": scoped, + "global": globally, + "unknown": unknown, + }) + return series + + +def recent_events(db_manager: Any, limit: int = 50) -> list[dict[str, Any]]: + """Most recent warning decisions, newest first.""" + limit = max(1, min(int(limit), 500)) + try: + return db_manager.execute_query( + "SELECT id, created_at, sender_id, sender_pubkey, channel, delivery, action, detail " + "FROM region_warning_events ORDER BY created_at DESC, id DESC LIMIT ?", + (limit,), + ) + except Exception: + return [] + + +def warning_budget( + db_manager: Any, + settings: RegionWarningSettings, + config: Any = None, + logger: Any = None, +) -> dict[str, Any]: + """How much of the daily cap is spent and when the mesh cooldown lifts. + + The cap counts *attempts*, failures included. Its job is to bound how much + unprompted activity this feature can produce in a day, and a send that + reported failure may still have put something on the air before it did. + """ + today = _local_today(config, logger).isoformat() + used = 0 + last_at: Optional[str] = None + try: + rows = db_manager.execute_query( + "SELECT COUNT(*) AS used FROM region_warning_events " + "WHERE date(created_at) = ?", + (today,), + ) + if rows: + used = int(rows[0].get("used") or 0) + rows = db_manager.execute_query( + "SELECT MAX(created_at) AS last_at FROM region_warning_events WHERE action IN (?, ?)", + (ACTION_SENT, ACTION_DRY_RUN), + ) + if rows: + last_at = rows[0].get("last_at") + except Exception: + pass + + cap = settings.max_warnings_per_day + return { + "date": today, + "used_today": used, + "cap": cap, + "remaining": None if cap <= 0 else max(0, cap - used), + "unlimited": cap <= 0, + "last_warning_at": last_at, + } + + +# --------------------------------------------------------------------------- +# Monitor β€” bot-side +# --------------------------------------------------------------------------- + + +@dataclass +class _SenderState: + """In-memory per-sender counter, rebuilt after a restart.""" + + unscoped_seen: int = 0 + counted_at: float = field(default_factory=time.monotonic) + + +class RegionWarningMonitor: + """Tally flood scopes and, when enabled, warn senders who never set one.""" + + #: How long a sender's unscoped-message run survives without new traffic. + #: A sender who floods three times a year should not accumulate their way + #: to a warning, so the run resets after a quiet spell. + RUN_TTL_SECONDS = 24 * 3600 + + #: Ceiling on tracked senders. A busy mesh sees thousands of names over a + #: long uptime, and an unbounded dict here would be a slow leak in the + #: channel-message path. Expired runs are swept first; if that is not + #: enough, the least recently seen are dropped, which only delays a + #: warning for a sender who had stopped talking anyway. + MAX_TRACKED_SENDERS = 2000 + + def __init__(self, bot: Any) -> None: + self.bot = bot + self.logger = bot.logger + self.settings = load_settings(getattr(bot, "config", None)) + self._senders: dict[str, _SenderState] = {} + # Written-through from the DB on first use so a restart cannot reset the + # mesh-wide cooldown and let a burst of warnings out. + self._last_warning_monotonic: Optional[float] = None + self._last_warning_wall: Optional[datetime] = None + self._budget_loaded = False + if self.settings.enabled: + self.logger.info( + "Region warnings enabled (%s, delivery=%s, cap=%s/day)", + "dry run" if self.settings.dry_run else "live", + self.settings.delivery, + self.settings.max_warnings_per_day or "unlimited", + ) + + # -- configuration ----------------------------------------------------- + + def reload_config(self) -> None: + """Re-read ``[Region_Warnings]`` after a hot config reload.""" + self.settings = load_settings(getattr(self.bot, "config", None)) + + # -- observation ------------------------------------------------------- + + async def observe( + self, + *, + verdict: str, + sender_id: Optional[str], + sender_pubkey: Optional[str], + channel: Optional[str], + ) -> None: + """Record one channel message's scope verdict and warn if it earns one. + + Never raises: this sits on the channel-message path and a bookkeeping + failure must not cost the message. + """ + try: + if verdict not in _VERDICT_COLUMN: + return + if self.settings.track_traffic: + self._record_verdict(verdict, channel) + if verdict != VERDICT_GLOBAL: + if verdict == VERDICT_SCOPED and sender_id: + # A scoped message proves the sender has a region set now, + # so an earlier unscoped run is stale evidence. + self._senders.pop(sender_id, None) + return + await self._consider_warning(sender_id, sender_pubkey, channel) + except Exception: + self.logger.exception("Region warning monitor failed on a channel message") + + def _record_verdict(self, verdict: str, channel: Optional[str]) -> None: + column = _VERDICT_COLUMN[verdict] + today = _local_today(getattr(self.bot, "config", None), self.logger).isoformat() + name = (channel or "").strip() or "(unknown)" + db_manager = getattr(self.bot, "db_manager", None) + if not db_manager: + return + with db_manager.connection() as conn: + # Column name comes from the _VERDICT_COLUMN table, never from input. + conn.execute( + f"INSERT INTO region_scope_daily (date, channel, {column}) VALUES (?, ?, 1) " + f"ON CONFLICT(date, channel) DO UPDATE SET {column} = {column} + 1", + (today, name), + ) + conn.commit() + + # -- warning decision -------------------------------------------------- + + async def _consider_warning( + self, + sender_id: Optional[str], + sender_pubkey: Optional[str], + channel: Optional[str], + ) -> None: + settings = self.settings + if not settings.enabled or not sender_id: + return + if not settings.monitors_channel(channel): + return + + cmd_mgr = getattr(self.bot, "command_manager", None) + if cmd_mgr is None: + return + if cmd_mgr.is_user_banned(sender_id): + return + if self._is_self(sender_id, sender_pubkey): + return + # channelpause silences the bot on channels; a warning is a bot response + # and has no business being the one thing that keeps talking. + if not getattr(self.bot, "channel_responses_enabled", True): + return + + state = self._senders.get(sender_id) + now = time.monotonic() + if state is None or (now - state.counted_at) > self.RUN_TTL_SECONDS: + state = _SenderState() + self._senders[sender_id] = state + self._evict_stale_senders(now) + state.unscoped_seen += 1 + state.counted_at = now + if state.unscoped_seen < settings.min_unscoped_messages: + return + + self._load_budget_once() + + if self._mesh_cooldown_active(now): + return + if self._sender_cooldown_active(sender_id): + return + if self._daily_cap_reached(): + self.logger.debug( + "Region warning for %s withheld: daily cap of %d reached", + sender_id, settings.max_warnings_per_day, + ) + return + + # The run has been spent whether or not the send itself succeeds; not + # resetting it would retry on the sender's very next message. + state.unscoped_seen = 0 + + text = render_message(settings.message, sender_id, channel) + if not text: + return + + if settings.dry_run: + self._record_event(sender_id, sender_pubkey, channel, ACTION_DRY_RUN, text) + self._mark_warning_sent(now) + self.logger.info( + "Region warning (dry run) for %s on %s: %s", sender_id, channel or "?", text + ) + return + + sent, detail = await self._send_warning(sender_id, channel, text) + self._record_event( + sender_id, sender_pubkey, channel, + ACTION_SENT if sent else ACTION_FAILED, + detail, + ) + if sent: + # Only a successful send starts the mesh cooldown; a failed one + # spent no airtime and should not silence a sender who would. + self._mark_warning_sent(now) + + async def _send_warning( + self, sender_id: str, channel: Optional[str], text: str + ) -> tuple[bool, str]: + cmd_mgr = self.bot.command_manager + if self.settings.delivery == DELIVERY_CHANNEL: + if not channel: + return False, "no channel to reply on" + body = truncate_to_bytes(text, self._channel_body_limit()) + # Deliberately global scope: the recipient is by definition not + # inside any region the bot replies under, so a scoped reply would + # never reach them. + ok = await cmd_mgr.send_channel_message( + channel, body, skip_user_rate_limit=True, scope="*" + ) + return ok, body if ok else f"channel send failed: {body}" + + body = truncate_to_bytes(text, DM_BODY_LIMIT) + ok = await cmd_mgr.send_dm(sender_id, body, skip_user_rate_limit=True) + return ok, body if ok else f"DM send failed (contact unknown or radio busy): {body}" + + def _channel_body_limit(self) -> int: + """Channel body budget for a global-scope send from this node.""" + try: + from modules.models import MeshMessage + + probe = MeshMessage(content="", channel="", is_dm=False, reply_scope="") + return int(self.bot.command_manager.get_max_message_length(probe)) + except Exception: + return 130 + + # -- gating ------------------------------------------------------------ + + def _evict_stale_senders(self, now: float) -> None: + """Keep ``_senders`` bounded (see ``MAX_TRACKED_SENDERS``).""" + if len(self._senders) <= self.MAX_TRACKED_SENDERS: + return + for name, state in list(self._senders.items()): + if (now - state.counted_at) > self.RUN_TTL_SECONDS: + del self._senders[name] + overflow = len(self._senders) - self.MAX_TRACKED_SENDERS + if overflow <= 0: + return + oldest = sorted(self._senders.items(), key=lambda kv: kv[1].counted_at) + for name, _state in oldest[:overflow]: + del self._senders[name] + + def _now(self) -> datetime: + return local_now(getattr(self.bot, "config", None), self.logger) + + def _is_self(self, sender_id: str, sender_pubkey: Optional[str]) -> bool: + """Whether this message came from the bot's own node. + + Checked by public key first: the configured name can lag the device's, + and another node is free to call itself whatever it likes. + """ + own_key = self._own_public_key() + prefix = (sender_pubkey or "").strip().lower() + if own_key and prefix: + return own_key.startswith(prefix) or prefix.startswith(own_key) + try: + bot_name = self.bot.config.get("Bot", "bot_name", fallback="") or "" + except Exception: + bot_name = "" + return bool(bot_name) and sender_id.strip().lower() == bot_name.strip().lower() + + def _own_public_key(self) -> str: + """This node's public key as lowercase hex, or "" when unavailable.""" + try: + self_info = getattr(getattr(self.bot, "meshcore", None), "self_info", None) + if self_info is None: + return "" + if isinstance(self_info, dict): + key = self_info.get("public_key", "") + else: + key = getattr(self_info, "public_key", "") + if isinstance(key, (bytes, bytearray)): + return bytes(key).hex() + return str(key or "").strip().lower() + except Exception: + return "" + + def _load_budget_once(self) -> None: + """Seed the mesh cooldown from the DB so a restart cannot bypass it.""" + if self._budget_loaded: + return + self._budget_loaded = True + db_manager = getattr(self.bot, "db_manager", None) + if not db_manager: + return + try: + rows = db_manager.execute_query( + "SELECT MAX(created_at) AS last_at FROM region_warning_events " + "WHERE action IN (?, ?)", + (ACTION_SENT, ACTION_DRY_RUN), + ) + except Exception: + return + raw = rows[0].get("last_at") if rows else None + if not raw: + return + parsed = _parse_timestamp(raw) + if parsed is None: + return + self._last_warning_wall = parsed + elapsed = (self._now() - parsed).total_seconds() + if elapsed >= 0: + self._last_warning_monotonic = time.monotonic() - elapsed + + def _mark_warning_sent(self, now: float) -> None: + self._last_warning_monotonic = now + self._last_warning_wall = self._now() + + def _mesh_cooldown_active(self, now: float) -> bool: + window = self.settings.mesh_cooldown_minutes * 60.0 + if window <= 0 or self._last_warning_monotonic is None: + return False + return (now - self._last_warning_monotonic) < window + + def _sender_cooldown_active(self, sender_id: str) -> bool: + hours = self.settings.per_sender_cooldown_hours + if hours <= 0: + return False + db_manager = getattr(self.bot, "db_manager", None) + if not db_manager: + return False + try: + rows = db_manager.execute_query( + "SELECT MAX(created_at) AS last_at FROM region_warning_events " + "WHERE sender_id = ? AND action IN (?, ?)", + (sender_id, ACTION_SENT, ACTION_DRY_RUN), + ) + except Exception: + return False + raw = rows[0].get("last_at") if rows else None + parsed = _parse_timestamp(raw) if raw else None + if parsed is None: + return False + return (self._now() - parsed) < timedelta(hours=hours) + + def _daily_cap_reached(self) -> bool: + cap = self.settings.max_warnings_per_day + if cap <= 0: + return False + db_manager = getattr(self.bot, "db_manager", None) + if not db_manager: + return False + today = self._now().date().isoformat() + try: + # Attempts, not successes: see warning_budget. + rows = db_manager.execute_query( + "SELECT COUNT(*) AS used FROM region_warning_events " + "WHERE date(created_at) = ?", + (today,), + ) + except Exception: + return False + used = int(rows[0].get("used") or 0) if rows else 0 + return used >= cap + + # -- persistence ------------------------------------------------------- + + def _record_event( + self, + sender_id: str, + sender_pubkey: Optional[str], + channel: Optional[str], + action: str, + detail: str, + ) -> None: + db_manager = getattr(self.bot, "db_manager", None) + if not db_manager: + return + try: + with db_manager.connection() as conn: + conn.execute( + "INSERT INTO region_warning_events " + "(created_at, sender_id, sender_pubkey, channel, delivery, action, detail) " + "VALUES (?, ?, ?, ?, ?, ?, ?)", + ( + local_now(getattr(self.bot, "config", None), self.logger) + .isoformat(sep=" ", timespec="seconds"), + sender_id, + (sender_pubkey or "")[:64] or None, + channel, + self.settings.delivery, + action, + detail[:400] if detail else None, + ), + ) + conn.commit() + except Exception: + self.logger.exception("Failed to record region warning event") + + +def _parse_timestamp(raw: Any) -> Optional[datetime]: + """Parse a stored event timestamp, tolerating ``T`` or space separators.""" + if not raw: + return None + text = str(raw).strip().replace("T", " ") + for fmt in ("%Y-%m-%d %H:%M:%S.%f", "%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M"): + try: + return datetime.strptime(text, fmt) + except ValueError: + continue + return None diff --git a/modules/web_viewer/app.py b/modules/web_viewer/app.py index 9034ad2..084dc76 100644 --- a/modules/web_viewer/app.py +++ b/modules/web_viewer/app.py @@ -44,6 +44,7 @@ from flask import ( ) from flask_socketio import SocketIO, disconnect, emit +from modules import region_warning from modules.database_restore import ( DEFAULT_MAX_RESTORE_BYTES, DatabaseRestoreError, @@ -1281,6 +1282,7 @@ class BotDataViewer: 'contacts', 'plugins_page', 'greeter', + 'region_warnings_page', 'logs', 'multibyte_rollout', 'mesh', @@ -1403,6 +1405,11 @@ class BotDataViewer: """Greeter management page""" return render_template('greeter.html') + @self.app.route('/region-warnings') + def region_warnings_page(): + """Regional flood scope monitoring and warning settings.""" + return render_template('region_warnings.html') + @self.app.route('/feeds') def feeds(): """Feed management page""" @@ -4041,6 +4048,159 @@ class BotDataViewer: if conn: conn.close() + # ── Region warnings (regional flood scope) ─────────────────────────── + + def _region_warning_channel_limit() -> int: + """Channel body budget for a global-scope send, mirroring CommandManager.""" + name = (self.config.get('Bot', 'bot_name', fallback='Bot') or 'Bot').strip() or 'Bot' + return max(130, 160 - len(name.encode('utf-8')) - 2) + + @self.app.route('/api/region-warnings') + def api_region_warnings(): + """Settings, traffic tallies, budget and recent decisions for the page.""" + try: + # Re-read from disk so the page reflects edits made elsewhere. + self.config = self._load_merged_config() + settings = region_warning.load_settings(self.config) + try: + days = max(1, min(int(request.args.get('days', 14)), 90)) + except (TypeError, ValueError): + days = 14 + + known_channels = [] + try: + known_channels = [ + c.get('name') for c in self._get_channels() if c.get('name') + ] + except Exception: + pass + + return jsonify({ + 'settings': region_warning.settings_to_config_values(settings), + 'defaults': region_warning.settings_to_config_values( + region_warning.RegionWarningSettings() + ), + 'default_message': region_warning.DEFAULT_MESSAGE, + 'traffic': region_warning.traffic_summary( + self.db_manager, self.config, days, self.logger + ), + 'series': region_warning.daily_series( + self.db_manager, self.config, days, self.logger + ), + 'budget': region_warning.warning_budget( + self.db_manager, settings, self.config, self.logger + ), + 'events': region_warning.recent_events(self.db_manager, 50), + 'limits': { + 'dm': region_warning.DM_BODY_LIMIT, + 'channel': _region_warning_channel_limit(), + }, + 'known_channels': known_channels, + }) + except Exception: + self.logger.exception("Error building region warning view") + return jsonify({'error': 'Internal error β€” see server logs'}), 500 + + @self.app.route('/api/region-warnings/settings', methods=['POST']) + def api_region_warnings_save(): + """Persist [Region_Warnings] and queue a hot config reload.""" + data = request.get_json(silent=True) or {} + + def _as_bool(key, default): + raw = data.get(key, default) + if isinstance(raw, bool): + return raw + return str(raw).strip().lower() in ('1', 'true', 'yes', 'on') + + def _as_number(key, default, minimum=0.0, maximum=None): + raw = data.get(key, default) + try: + value = float(raw) + except (TypeError, ValueError): + raise ValueError(f'{key} must be a number') + if value < minimum: + raise ValueError(f'{key} must be at least {minimum:g}') + if maximum is not None and value > maximum: + raise ValueError(f'{key} must be at most {maximum:g}') + return value + + try: + delivery = str(data.get('delivery', 'dm')).strip().lower() + if delivery not in (region_warning.DELIVERY_DM, region_warning.DELIVERY_CHANNEL): + raise ValueError('delivery must be "dm" or "channel"') + + message = str(data.get('message') or '').strip() or region_warning.DEFAULT_MESSAGE + if '\n' in message or '\r' in message: + raise ValueError('message must be a single line') + + channels = data.get('channels') + if isinstance(channels, list): + channel_parts = channels + else: + channel_parts = str(channels or '').split(',') + normalized_channels = [] + for part in channel_parts: + name = region_warning.normalize_channel(part) + if name and name not in normalized_channels: + normalized_channels.append(name) + + settings = region_warning.RegionWarningSettings( + enabled=_as_bool('enabled', False), + dry_run=_as_bool('dry_run', True), + delivery=delivery, + channels=tuple(normalized_channels), + message=message, + min_unscoped_messages=int(_as_number('min_unscoped_messages', 3, 1, 100)), + per_sender_cooldown_hours=_as_number('per_sender_cooldown_hours', 168, 0, 8760), + mesh_cooldown_minutes=_as_number('mesh_cooldown_minutes', 30, 0, 10080), + max_warnings_per_day=int(_as_number('max_warnings_per_day', 6, 0, 1000)), + track_traffic=_as_bool('track_traffic', True), + ) + except ValueError as exc: + return jsonify({'success': False, 'error': str(exc)}), 400 + + section = region_warning.CONFIG_SECTION + target_path = ( + self.local_config_path + if section in self._local_sections + else self.config_path + ) + try: + store = get_settings_store(self.config, target_path, self.db_manager) + result = store.write_values( + section, region_warning.settings_to_config_values(settings) + ) + except IniValueError as exc: + return jsonify({'success': False, 'error': str(exc)}), 400 + except Exception: + self.logger.exception("Error saving region warning settings") + return jsonify({'success': False, 'error': 'Internal error β€” see server logs'}), 500 + + backup_path = result.get('backup_path', '') if isinstance(result, dict) else '' + reload_queued = False + try: + with self.db_manager.connection() as conn: + cursor = conn.cursor() + cursor.execute( + "INSERT INTO channel_operations (operation_type, status) " + "VALUES ('config_reload', 'pending')" + ) + conn.commit() + reload_queued = True + except Exception: + self.logger.exception("Failed to queue config reload") + + self.logger.info( + "Region warning settings saved (enabled=%s, dry_run=%s, delivery=%s)", + settings.enabled, settings.dry_run, settings.delivery, + ) + return jsonify({ + 'success': True, + 'backup_path': backup_path, + 'reload_queued': reload_queued, + 'settings': region_warning.settings_to_config_values(settings), + }) + # Feed management API endpoints def _schedule_tz(): from modules.utils import get_config_timezone diff --git a/modules/web_viewer/templates/base.html b/modules/web_viewer/templates/base.html index 9c3933f..20eb951 100644 --- a/modules/web_viewer/templates/base.html +++ b/modules/web_viewer/templates/base.html @@ -156,12 +156,17 @@ color: var(--text-color); } + /* Bootstrap's striping sets a light --bs-table-striped-color on the + cell as well as a background. Overriding only the background left + #212529 text on a #3d3d3d row in dark mode β€” about 1.3:1. */ [data-theme="dark"] .table-striped>tbody>tr:nth-of-type(odd)>td { background-color: var(--bg-tertiary); + color: var(--text-color); } [data-theme="dark"] .table-hover>tbody>tr:hover>td { background-color: var(--bg-tertiary); + color: var(--text-color); } .badge { @@ -441,7 +446,7 @@ {#- Configuration pages live behind one gear so the bar stays about what the mesh is doing, not how the bot is set up. -#} - {% set settings_paths = ['/radio', '/schedule', '/greeter', '/feeds', '/plugins', '/config'] %} + {% set settings_paths = ['/radio', '/schedule', '/greeter', '/feeds', '/region-warnings', '/plugins', '/config'] %}
    • {% endif %} +
  • + + Region Warnings + +
  • Plugins diff --git a/modules/web_viewer/templates/region_warnings.html b/modules/web_viewer/templates/region_warnings.html new file mode 100644 index 0000000..a20d757 --- /dev/null +++ b/modules/web_viewer/templates/region_warnings.html @@ -0,0 +1,786 @@ +{% extends "base.html" %} + +{% block title %}Region Warnings - MeshCore Bot Data Viewer{% endblock %} + +{% block extra_css %} + +{% endblock %} + +{% block content %} +
    +
    +

    Region Warnings

    +

    + A MeshCore client with no region code floods every message to the whole mesh. + This page counts how much of that your bot hears, and can optionally tell those senders. +

    +
    +
    + + +
    +
    +
    +
    +
    +

    Loading…

    +
    Reading current settings.
    +
    +
    +
    +
    +
    +
    + + +
    +
    +
    +
    + Channel traffic by scope +
    + + +
    +
    +
    +
    + No region code + Region code set + Couldn't tell +
    +
    Loading traffic…
    +
    +
    +
    +
    +
    +
    Daily volume
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    + + +
    +
    +
    +
    Settings
    +
    +
    + What the bot does +
    + + + +
    +
    + + +
    +
    + +
    + Who hears it +
    +
    + + +
    +
    +
    + + +
    + Comma-separated. The # is optional. +
    +
    +
    +
    +
    + +
    + Limits +
    +
    + +
    + + msgs +
    +
    Unscoped messages from one sender.
    +
    +
    + +
    + + hours +
    +
    0 disables.
    +
    +
    + +
    + + min +
    +
    Minimum gap between any two warnings.
    +
    +
    + + +
    0 means unlimited.
    +
    +
    +
    + +
    + Message +
    +
    + + +
    + {sender} and {channel} are substituted. +
    + +
    +
    + Preview +
    +
    +
    +
    +
    +
    +
    +
    + + + +
    +
    +
    +
    + + +
    +
    +
    +
    + Recent warning decisions + +
    +
    +
    + + + + + + + + + + + + + +
    WhenSenderChannelOutcomeMessage
    Loading…
    +
    +
    +
    +
    +
    +{% endblock %} + +{% block extra_js %} + +{% endblock %} diff --git a/pyproject.toml b/pyproject.toml index ef5db8e..0c1f113 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -156,6 +156,7 @@ module = [ "modules.security_utils", "modules.commands.base_command", "modules.commands.rain_command", + "modules.region_warning", ] disallow_untyped_defs = true disallow_incomplete_defs = true diff --git a/tests/test_region_warnings_api.py b/tests/test_region_warnings_api.py new file mode 100644 index 0000000..13590a6 --- /dev/null +++ b/tests/test_region_warnings_api.py @@ -0,0 +1,180 @@ +"""Tests for the /region-warnings page and its API.""" + +from __future__ import annotations + +import configparser +from datetime import datetime +from unittest.mock import patch + +import pytest + +from modules import region_warning + + +@pytest.fixture +def viewer(tmp_path): + """BotDataViewer against a temp config + empty DB (migrations create tables).""" + from modules.web_viewer.app import BotDataViewer + + config = configparser.ConfigParser() + config.add_section("Bot") + config.set("Bot", "db_path", str(tmp_path / "meshcore_bot.db")) + config.set("Bot", "bot_name", "TestBot") + config.add_section("Web_Viewer") + for key, value in [ + ("host", "127.0.0.1"), ("port", "8080"), ("enabled", "false"), + ("auto_start", "false"), ("debug", "false"), + ("cors_allowed_origins", "*"), ("web_viewer_password", ""), + ]: + config.set("Web_Viewer", key, value) + + config_path = str(tmp_path / "config.ini") + with open(config_path, "w") as handle: + config.write(handle) + + with patch.object(BotDataViewer, "_start_database_polling"), \ + patch.object(BotDataViewer, "_start_log_tailing"), \ + patch.object(BotDataViewer, "_start_cleanup_scheduler"), \ + patch.object(BotDataViewer, "_start_dashboard_refresher"), \ + patch.object(BotDataViewer, "_setup_socketio_handlers"), \ + patch("modules.web_viewer.app.RepeaterManager"): + instance = BotDataViewer( + db_path=str(tmp_path / "meshcore_bot.db"), config_path=config_path) + instance.app.testing = True + return instance + + +def _seed_tally(viewer, channel="#general", scoped=30, globally=10, unknown=5): + today = datetime.now().date().isoformat() + with viewer.db_manager.connection() as conn: + conn.execute( + "INSERT INTO region_scope_daily " + "(date, channel, scoped_count, global_count, unknown_count) " + "VALUES (?, ?, ?, ?, ?)", + (today, channel, scoped, globally, unknown), + ) + conn.commit() + + +class TestRegionWarningsPage: + def test_page_renders(self, viewer): + resp = viewer.app.test_client().get("/region-warnings") + assert resp.status_code == 200 + assert b"Region Warnings" in resp.data + + def test_page_script_carries_a_csp_nonce(self, viewer): + """The endpoint is in the nonce-hardened set, so its script must be nonced.""" + resp = viewer.app.test_client().get("/region-warnings") + body = resp.data.decode() + assert "