diff --git a/config.ini.example b/config.ini.example index f429874..6b135ab 100644 --- a/config.ini.example +++ b/config.ini.example @@ -35,11 +35,20 @@ reconnect_delay_seconds = 5 # Maximum delay in seconds between reconnect attempts (cap on exponential backoff) reconnect_max_delay_seconds = 60 -# Enable radio debug logging (default: false) -# When true, the meshcore library logs raw protocol data at DEBUG level. -# Useful for diagnosing connection issues, unexpected packet types, or -# protocol-level problems. Generates very verbose output — disable in production. -#radio_debug = false +# Enable verbose meshcore protocol logging (set true only when debugging) +radio_debug = false + +# Radio health probe interval in seconds (clamped to 300-900) +radio_probe_interval_seconds = 300 + +# Consecutive failed probes before zombie state is declared +radio_probe_fail_threshold = 3 + +# Send immediate zombie alert email (uses Notifications SMTP settings) +radio_zombie_alert_enabled = false + +# Optional zombie alert recipients (comma-separated; empty = nightly recipients) +radio_zombie_alert_email = [Bot] # Bot name for identification and logging @@ -176,40 +185,6 @@ service_restart_backoff_seconds = 300 # Max seconds to wait for a channel message send before timing out (default: 30) send_timeout_seconds = 30 -# ── Radio health monitoring (zombie radio detection) ──────────────────────── -# -# A zombie radio is one where the serial/BLE transport is still open and -# incoming RF packets are still received, but the firmware is hung and stops -# ACKing outgoing commands (all sends time out with no_event_received). -# The ONLY recovery is a physical power cycle — disconnect/reconnect does nothing. -# -# Probe interval: how often to send a get_time() health check (seconds). -# Valid range: 300–900 (5–15 minutes). Values outside this range are clamped. -# Default: 300 (5 minutes) -radio_probe_interval_seconds = 300 - -# Number of consecutive failed probes before declaring zombie state. -# At the default 300s interval: threshold 3 = ~15 min before zombie is logged. -# Suggested values by interval: -# 300s (5 min) → threshold 3 (~15 min total) -# 600s (10 min) → threshold 2 (~20 min total) -# 900s (15 min) → threshold 2 (~30 min total) -radio_probe_fail_threshold = 3 - -# Send an immediate alert email when zombie state is confirmed. -# Requires SMTP to be configured in the web viewer Notifications settings. -# false: log CRITICAL only, no email (default) -# true: send alert email in addition to logging -radio_zombie_alert_enabled = false - -# SMTP hosts resolving to private/loopback/reserved ranges are blocked by default (SSRF guard). -# If you intentionally use local SMTP, enable "Allow local SMTP host" in Notifications. -# Alert email recipients for zombie detection (comma-separated addresses). -# If empty, falls back to the nightly maintenance email recipients. -# Set this to a separate on-call address if needed. -# Example: oncall@example.com,admin@example.com -radio_zombie_alert_email = - [Channels] # Channels to monitor (comma-separated) # Bot will only respond to messages on these channels diff --git a/modules/core.py b/modules/core.py index 6246650..a242d3b 100644 --- a/modules/core.py +++ b/modules/core.py @@ -1486,10 +1486,22 @@ long_jokes = false ) if result.type == EventType.ERROR: self._radio_fail_count = getattr(self, '_radio_fail_count', 0) + 1 - threshold = self.config.getint('Bot', 'radio_probe_fail_threshold', fallback=3) - interval = max(300, min(900, self.config.getint( - 'Bot', 'radio_probe_interval_seconds', fallback=300 - ))) + threshold = self.config.getint( + 'Connection', + 'radio_probe_fail_threshold', + fallback=self.config.getint('Bot', 'radio_probe_fail_threshold', fallback=3), + ) + interval = max( + 300, + min( + 900, + self.config.getint( + 'Connection', + 'radio_probe_interval_seconds', + fallback=self.config.getint('Bot', 'radio_probe_interval_seconds', fallback=300), + ), + ), + ) self.logger.warning( f"Radio health probe failed " f"({self._radio_fail_count}/{threshold}): no response to get_time" @@ -1860,9 +1872,17 @@ long_jokes = false if not getattr(self, '_radio_zombie_detected', False): if not hasattr(self, '_last_radio_probe'): self._last_radio_probe = time.time() - probe_interval = max(300, min(900, self.config.getint( - 'Bot', 'radio_probe_interval_seconds', fallback=300 - ))) + probe_interval = max( + 300, + min( + 900, + self.config.getint( + 'Connection', + 'radio_probe_interval_seconds', + fallback=self.config.getint('Bot', 'radio_probe_interval_seconds', fallback=300), + ), + ), + ) if time.time() - self._last_radio_probe >= probe_interval: self._last_radio_probe = time.time() asyncio.create_task(self._probe_radio_health()) diff --git a/modules/scheduler.py b/modules/scheduler.py index 7891161..5b6eb4e 100644 --- a/modules/scheduler.py +++ b/modules/scheduler.py @@ -999,7 +999,12 @@ class MessageScheduler: import ssl as _ssl from email.message import EmailMessage - if not self.bot.config.getboolean('Bot', 'radio_zombie_alert_enabled', fallback=True): + zombie_alert_enabled = self.bot.config.getboolean( + 'Connection', + 'radio_zombie_alert_enabled', + fallback=self.bot.config.getboolean('Bot', 'radio_zombie_alert_enabled', fallback=False), + ) + if not zombie_alert_enabled: return smtp_host = self._get_notif('smtp_host') @@ -1010,7 +1015,11 @@ class MessageScheduler: from_email = self._get_notif('from_email') # Alert recipients: dedicated config key, falls back to nightly recipients - alert_email_cfg = self.bot.config.get('Bot', 'radio_zombie_alert_email', fallback='').strip() + alert_email_cfg = self.bot.config.get( + 'Connection', + 'radio_zombie_alert_email', + fallback=self.bot.config.get('Bot', 'radio_zombie_alert_email', fallback=''), + ).strip() if alert_email_cfg: recipients = [r.strip() for r in alert_email_cfg.split(',') if r.strip()] else: diff --git a/modules/web_viewer/app.py b/modules/web_viewer/app.py index 08166af..856aea9 100644 --- a/modules/web_viewer/app.py +++ b/modules/web_viewer/app.py @@ -1549,10 +1549,18 @@ class BotDataViewer: ini: dict[str, str] = { 'alert_enabled': ( 'true' - if self.config.getboolean('Bot', 'radio_zombie_alert_enabled', fallback=False) + if self.config.getboolean( + 'Connection', + 'radio_zombie_alert_enabled', + fallback=self.config.getboolean('Bot', 'radio_zombie_alert_enabled', fallback=False), + ) else 'false' ), - 'alert_email': self.config.get('Bot', 'radio_zombie_alert_email', fallback=''), + 'alert_email': self.config.get( + 'Connection', + 'radio_zombie_alert_email', + fallback=self.config.get('Bot', 'radio_zombie_alert_email', fallback=''), + ), } return jsonify({'meta': meta, 'config_ini': ini}) @@ -1561,7 +1569,7 @@ class BotDataViewer: """Save zombie alert settings to bot_metadata. If ``write_to_config`` is ``true`` in the request body, the values - are also written back to config.ini under ``[Bot]``. The config + are also written back to config.ini under ``[Connection]``. The config object in memory is updated immediately so the scheduler reads the new values without a restart. """ @@ -1578,15 +1586,15 @@ class BotDataViewer: config_saved = False if write_to_config: try: - if not self.config.has_section('Bot'): - self.config.add_section('Bot') + if not self.config.has_section('Connection'): + self.config.add_section('Connection') if 'alert_enabled' in data: self.config.set( - 'Bot', 'radio_zombie_alert_enabled', + 'Connection', 'radio_zombie_alert_enabled', 'true' if str(data['alert_enabled']).lower() == 'true' else 'false', ) if 'alert_email' in data: - self.config.set('Bot', 'radio_zombie_alert_email', str(data['alert_email'])) + self.config.set('Connection', 'radio_zombie_alert_email', str(data['alert_email'])) with open(self.config_path, 'w') as fh: self.config.write(fh) config_saved = True