diff --git a/README.md b/README.md index 09fb5ad..37673d7 100644 --- a/README.md +++ b/README.md @@ -321,6 +321,9 @@ radio_probe_fail_threshold = 3 # consecutive failures before zombie is dec send_timeout_seconds = 30 # max seconds to wait for a channel message send radio_zombie_alert_enabled = false # send immediate alert email on zombie detection (default: log only) radio_zombie_alert_email = # alert recipient(s); falls back to nightly email if blank +radio_offline_threshold = 3 # consecutive send timeouts before radio-offline state is entered +radio_offline_alert_enabled = true # send alert email when radio-offline state is entered +radio_offline_alert_email = # alert recipient(s); falls back to nightly email if blank ``` ### Keywords diff --git a/modules/core.py b/modules/core.py index 0dae1bd..7f85bb2 100644 --- a/modules/core.py +++ b/modules/core.py @@ -299,6 +299,75 @@ class MeshCoreBot: """ return bool(getattr(self, '_radio_zombie_detected', False)) + @property + def is_radio_offline(self) -> bool: + """True when repeated outbound send timeouts have been detected. + + Distinct from zombie state — the radio may still be forwarding received + packets but is not completing outbound sends. Cleared automatically + when a send succeeds, so no manual intervention is required. + """ + return bool(getattr(self, '_radio_offline', False)) + + def _record_send_failure(self, scheduler: "Any | None" = None) -> None: + """Increment the consecutive-send-failure counter. + + Called by the scheduler when an outbound send times out at the + ``future.result()`` level (i.e. the outer 60-second wall-clock + timeout fired). After ``radio_offline_threshold`` consecutive + failures the bot transitions to radio-offline state, persists it + to the DB for the web viewer banner, and optionally sends an alert + email. + """ + import datetime as _dt + import threading as _threading + + self._send_consecutive_failures: int = ( + getattr(self, '_send_consecutive_failures', 0) + 1 + ) + threshold = self.config.getint('Bot', 'radio_offline_threshold', fallback=3) + if self._send_consecutive_failures >= threshold and not self.is_radio_offline: + self._radio_offline = True + since = _dt.datetime.utcnow().isoformat() + self.logger.critical( + "RADIO OFFLINE: %d consecutive send timeouts (threshold %d). " + "Bot will suppress further outbound sends until one succeeds. " + "Check radio power and connection.", + self._send_consecutive_failures, + threshold, + ) + try: + self.db_manager.set_metadata('bot.radio_offline', 'true') + self.db_manager.set_metadata('bot.radio_offline_since', since) + except Exception: + pass + if scheduler is not None: + _threading.Thread( + target=scheduler.send_radio_offline_alert_email, + args=(self._send_consecutive_failures, threshold), + daemon=True, + ).start() + + def _record_send_success(self) -> None: + """Clear the consecutive-send-failure counter after a successful send.""" + failures = getattr(self, '_send_consecutive_failures', 0) + was_offline = self.is_radio_offline + if failures > 0 or was_offline: + self.logger.info( + "Outbound send succeeded — clearing radio-offline state " + "(was_offline=%s, failure_count=%d)", + was_offline, + failures, + ) + self._send_consecutive_failures = 0 + if was_offline: + self._radio_offline = False + try: + self.db_manager.set_metadata('bot.radio_offline', 'false') + self.db_manager.set_metadata('bot.radio_offline_since', '') + except Exception: + pass + def load_config(self) -> None: """Load configuration from file. diff --git a/modules/scheduler.py b/modules/scheduler.py index e0bf65e..edf3d98 100644 --- a/modules/scheduler.py +++ b/modules/scheduler.py @@ -121,6 +121,13 @@ class MessageScheduler: def send_scheduled_message(self, channel: str, message: str): """Send a scheduled message (synchronous wrapper for schedule library)""" + if self.bot.is_radio_zombie: + self.logger.warning("send_scheduled_message suppressed — radio is in zombie state") + return + if self.bot.is_radio_offline: + self.logger.warning("send_scheduled_message suppressed — radio is offline (repeated send timeouts)") + return + current_time = self.get_current_time() self.logger.info(f"📅 Sending scheduled message at {current_time.strftime('%H:%M:%S')} to {channel}: {message}") @@ -137,8 +144,10 @@ class MessageScheduler: # Wait for completion (with timeout to prevent indefinite blocking) try: future.result(timeout=60) # 60 second timeout + self.bot._record_send_success() except Exception as e: self.logger.error(f"Error sending scheduled message: {type(e).__name__}: {e}", exc_info=True) + self.bot._record_send_failure(scheduler=self) else: # Fallback: create new event loop if main loop not available try: @@ -613,6 +622,10 @@ class MessageScheduler: def send_interval_advert(self): """Send an interval-based advert (synchronous wrapper)""" + if self.bot.is_radio_offline: + self.logger.warning("send_interval_advert suppressed — radio is offline (repeated send timeouts)") + return + current_time = self.get_current_time() self.logger.info(f"📢 Sending interval-based flood advert at {current_time.strftime('%H:%M:%S')}") @@ -629,8 +642,10 @@ class MessageScheduler: # Wait for completion (with timeout to prevent indefinite blocking) try: future.result(timeout=60) # 60 second timeout + self.bot._record_send_success() except Exception as e: self.logger.error(f"Error sending interval advert: {type(e).__name__}: {e}", exc_info=True) + self.bot._record_send_failure(scheduler=self) else: # Fallback: create new event loop if main loop not available try: @@ -1319,6 +1334,124 @@ class MessageScheduler: except Exception as e: self.bot.logger.error(f"Failed to send zombie radio alert email: {e}") + # ── Radio offline alert email ──────────────────────────────────────────── + + def send_radio_offline_alert_email(self, fail_count: int, threshold: int) -> None: + """Send an immediate alert email when the radio-offline state is entered. + + Uses the same SMTP settings as the nightly digest. Recipients are taken + from the ``radio_offline_alert_email`` config key; if that key is empty the + nightly maintenance recipients are used as a fallback. + + Intentionally synchronous — intended to be run in a daemon thread. + """ + import smtplib + import ssl as _ssl + from email.message import EmailMessage + + alert_enabled = self.bot.config.getboolean('Bot', 'radio_offline_alert_enabled', fallback=True) + if not alert_enabled: + return + + smtp_host = self._get_notif('smtp_host') + smtp_security = self._get_notif('smtp_security') or 'starttls' + smtp_user = self._get_notif('smtp_user') + smtp_password = self._get_notif('smtp_password') + from_name = self._get_notif('from_name') or 'MeshCore Bot' + from_email = self._get_notif('from_email') + + alert_email_cfg = self.bot.config.get('Bot', 'radio_offline_alert_email', fallback='').strip() + if alert_email_cfg: + recipients = [r.strip() for r in alert_email_cfg.split(',') if r.strip()] + else: + recipients = [r.strip() for r in self._get_notif('recipients').split(',') if r.strip()] + + if not smtp_host or not from_email or not recipients: + self.bot.logger.warning( + "Radio-offline alert email enabled but SMTP settings incomplete " + f"(host={smtp_host!r}, from={from_email!r}, recipients={recipients}) " + "— alert email not sent" + ) + return + + allow_local = self._get_notif('allow_local_smtp').lower() == 'true' + if not validate_external_url(f'http://{smtp_host}', allow_private=allow_local): + self.bot.logger.error( + "Radio-offline alert email aborted: SMTP host %r resolves to a private or reserved address", + smtp_host, + ) + return + + try: + smtp_port = int(self._get_notif('smtp_port') or (465 if smtp_security == 'ssl' else 587)) + except ValueError: + smtp_port = 587 + + now_utc = datetime.datetime.utcnow() + connection_type = self.bot.config.get('Connection', 'connection_type', fallback='unknown') + serial_port = self.bot.config.get('Connection', 'serial_port', fallback='n/a') + + subject = ( + f'ALERT: MeshCore Bot — Radio Offline ' + f'[{now_utc.strftime("%Y-%m-%d %H:%M UTC")}]' + ) + body = '\n'.join([ + 'MeshCore Bot — Radio Offline Alert', + '=' * 44, + f'Time : {now_utc.strftime("%Y-%m-%d %H:%M:%S UTC")}', + '', + 'RADIO STATUS', + '─' * 30, + f' Connection : {connection_type}', + f' Port / Device : {serial_port}', + f' Failed sends : {fail_count} of {threshold} (threshold)', + '', + 'WHAT THIS MEANS', + '─' * 30, + ' The bot can no longer send outbound messages to the mesh.', + ' Inbound packets from the radio may still be arriving normally.', + ' This is NOT a zombie (firmware lock-up) — the radio is responsive', + ' but outbound sends are timing out.', + '', + 'ACTION REQUIRED', + '─' * 30, + ' Check the radio power supply and physical connection.', + ' Use the dashboard "Clear Offline Flag" button once the issue', + ' is resolved, or restart the bot to auto-probe.', + '', + '─' * 44, + 'Outbound sends will be suppressed until the offline flag is cleared.', + ]) + + try: + msg = EmailMessage() + msg['Subject'] = subject + msg['From'] = f'{from_name} <{from_email}>' + msg['To'] = ', '.join(recipients) + msg.set_content(body) + + context = _ssl.create_default_context() + _smtp_timeout = 30 + + if smtp_security == 'ssl': + with smtplib.SMTP_SSL(smtp_host, smtp_port, context=context, timeout=_smtp_timeout) as s: + if smtp_user and smtp_password: + s.login(smtp_user, smtp_password) + s.send_message(msg) + else: + with smtplib.SMTP(smtp_host, smtp_port, timeout=_smtp_timeout) as s: + if smtp_security == 'starttls': + s.ehlo() + s.starttls(context=context) + s.ehlo() + if smtp_user and smtp_password: + s.login(smtp_user, smtp_password) + s.send_message(msg) + + self.bot.logger.info(f"Radio-offline alert email sent to {recipients}") + except Exception as e: + self.bot.logger.error(f"Failed to send radio-offline alert email: {e}") + # ── Maintenance helpers ────────────────────────────────────────────────── def _get_maint(self, key: str) -> str: diff --git a/modules/web_viewer/app.py b/modules/web_viewer/app.py index d3d206c..31f138b 100644 --- a/modules/web_viewer/app.py +++ b/modules/web_viewer/app.py @@ -281,9 +281,13 @@ class BotDataViewer: try: radio_zombie = self.db_manager.get_metadata('bot.radio_zombie') == 'true' radio_zombie_since = self.db_manager.get_metadata('bot.radio_zombie_since') or None + radio_offline = self.db_manager.get_metadata('bot.radio_offline') == 'true' + radio_offline_since = self.db_manager.get_metadata('bot.radio_offline_since') or None except Exception: radio_zombie = False radio_zombie_since = None + radio_offline = False + radio_offline_since = None return { 'greeter_enabled': greeter_enabled, 'feed_manager_enabled': feed_manager_enabled, @@ -291,6 +295,8 @@ class BotDataViewer: 'version_info': version_info, 'radio_zombie': radio_zombie, 'radio_zombie_since': radio_zombie_since, + 'radio_offline': radio_offline, + 'radio_offline_since': radio_offline_since, } except Exception as e: self.logger.exception("Template context processor failed: %s", e) @@ -301,6 +307,8 @@ class BotDataViewer: 'version_info': version_info, 'radio_zombie': False, 'radio_zombie_since': None, + 'radio_offline': False, + 'radio_offline_since': None, } def _init_databases(self): @@ -1599,6 +1607,24 @@ class BotDataViewer: self.logger.exception("Error clearing zombie state") return jsonify({'success': False, 'error': 'Internal error — see server logs'}), 500 + # ── Radio offline clear ────────────────────────────────────────────── + + @self.app.route('/api/admin/radio-offline-clear', methods=['POST']) + def api_admin_radio_offline_clear() -> "Response": + """Clear the radio-offline flag so the bot resumes outbound sends.""" + try: + self.db_manager.set_metadata('bot.radio_offline', 'false') + self.db_manager.set_metadata('bot.radio_offline_since', '') + bot = getattr(self, 'bot', None) + if bot is not None: + bot._radio_offline = False + bot._send_consecutive_failures = 0 + self.logger.info("Radio-offline state cleared via web UI action") + return jsonify({'success': True, 'message': 'Radio-offline flag cleared; sends will resume'}) + except Exception: + self.logger.exception("Error clearing radio-offline state") + return jsonify({'success': False, 'error': 'Internal error — see server logs'}), 500 + # ── Maintenance status ─────────────────────────────────────────────── @self.app.route('/api/maintenance/backup_now', methods=['POST']) @@ -7252,6 +7278,7 @@ class BotDataViewer: def run(self, host='127.0.0.1', port=8080, debug=False): """Run the modern web viewer""" self.logger.info(f"Starting modern web viewer on {host}:{port}") + self._suppress_werkzeug_headers_error() try: self.socketio.run( self.app, @@ -7264,6 +7291,26 @@ class BotDataViewer: self.logger.error(f"Error running web viewer: {e}") raise + @staticmethod + def _suppress_werkzeug_headers_error() -> None: + """Install a log filter that silences the 'Headers already set' AssertionError. + + Werkzeug's dev server catches this internally and continues serving, but it + logs a full traceback at ERROR level. The underlying cause (concurrent + SocketIO polling requests racing through the WSGI layer) is reduced by the + single-socket-per-page fix, but may still occur occasionally. The filter + downgrades these specific records to DEBUG so they don't alarm operators. + """ + import logging + + class _HeadersAlreadySetFilter(logging.Filter): + def filter(self, record: logging.LogRecord) -> bool: + msg = record.getMessage() + return "Headers already set" not in msg + + for name in ("werkzeug", "werkzeug.serving"): + logging.getLogger(name).addFilter(_HeadersAlreadySetFilter()) + def main(): """Entry point for the meshcore-viewer command""" import argparse diff --git a/modules/web_viewer/integration.py b/modules/web_viewer/integration.py index ed860df..2b7fe07 100644 --- a/modules/web_viewer/integration.py +++ b/modules/web_viewer/integration.py @@ -750,6 +750,15 @@ class WebViewerIntegration: while self.running and self.viewer_process and self.viewer_process.poll() is None: time.sleep(1) + # Process exited unexpectedly — try to restart if we haven't been stopped + if self.running and self.viewer_process and self.viewer_process.poll() is not None: + self.logger.warning( + "Web viewer process exited unexpectedly (code %s) — attempting restart", + self.viewer_process.returncode, + ) + self.restart_viewer() + return + # Process exited - read from log files for error reporting if needed if self.viewer_process and self.viewer_process.returncode != 0: stdout_file.flush() diff --git a/modules/web_viewer/templates/base.html b/modules/web_viewer/templates/base.html index c41e748..f4c7eb7 100644 --- a/modules/web_viewer/templates/base.html +++ b/modules/web_viewer/templates/base.html @@ -512,6 +512,62 @@ {% endif %} + + {% if radio_offline %} + + + {% endif %} +
{% block content %}{% endblock %} diff --git a/tests/test_core.py b/tests/test_core.py index 1f4e3bf..7b8a388 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -481,6 +481,222 @@ class TestProbeRadioHealth: assert mock_warn.called +# --------------------------------------------------------------------------- +# _BotAdminServer — admin HTTP API +# --------------------------------------------------------------------------- + +class TestBotAdminServer: + """Admin HTTP server: /api/admin/reload and /api/admin/health.""" + + def _make_bot_with_admin(self, tmp_path, port=15001): + """Write config with [Admin] enabled and return a bot + token.""" + token = "test-secret-token" + config_file = tmp_path / "config.ini" + db_path = tmp_path / "bot.db" + config_file.write_text( + f"""[Connection] +connection_type = serial +serial_port = /dev/ttyUSB0 +timeout = 30 + +[Bot] +db_path = {db_path.as_posix()} +prefix_bytes = 1 + +[Channels] +monitor_channels = #general + +[Admin] +enabled = true +port = {port} +token = {token} +""", + encoding="utf-8", + ) + bot = MeshCoreBot(config_file=str(config_file)) + return bot, token, port + + def test_admin_server_created_when_enabled(self, tmp_path): + bot, _token, _port = self._make_bot_with_admin(tmp_path) + assert bot._admin_server is not None + + def test_admin_server_none_when_disabled(self, tmp_path): + config_file = tmp_path / "config.ini" + db_path = tmp_path / "bot.db" + _write_config(config_file, db_path) + bot = MeshCoreBot(config_file=str(config_file)) + assert bot._admin_server is None + + def test_admin_server_none_when_token_missing(self, tmp_path): + config_file = tmp_path / "config.ini" + db_path = tmp_path / "bot.db" + config_file.write_text( + f"""[Connection] +connection_type = serial +serial_port = /dev/ttyUSB0 +timeout = 30 + +[Bot] +db_path = {db_path.as_posix()} +prefix_bytes = 1 + +[Channels] +monitor_channels = #general + +[Admin] +enabled = true +port = 15002 +token = +""", + encoding="utf-8", + ) + bot = MeshCoreBot(config_file=str(config_file)) + assert bot._admin_server is None + + def test_reload_endpoint_success(self, tmp_path): + """POST /api/admin/reload returns 200 and success=true when reload succeeds.""" + import time + import urllib.request + + bot, token, port = self._make_bot_with_admin(tmp_path, port=15003) + + with patch.object(bot, "reload_config", return_value=(True, "Config reloaded")): + server = bot._admin_server + server.start() + time.sleep(0.4) + + req = urllib.request.Request( + f"http://127.0.0.1:{port}/api/admin/reload", + method="POST", + headers={"Authorization": f"Bearer {token}"}, + ) + with urllib.request.urlopen(req, timeout=5) as resp: + import json + body = json.loads(resp.read()) + assert body["success"] is True + assert "Config reloaded" in body["message"] + + def test_reload_endpoint_failure(self, tmp_path): + """POST /api/admin/reload returns 409 when reload is rejected.""" + import time + import urllib.request + from urllib.error import HTTPError + + bot, token, port = self._make_bot_with_admin(tmp_path, port=15004) + + with patch.object(bot, "reload_config", return_value=(False, "Radio settings changed")): + server = bot._admin_server + server.start() + time.sleep(0.4) + + req = urllib.request.Request( + f"http://127.0.0.1:{port}/api/admin/reload", + method="POST", + headers={"Authorization": f"Bearer {token}"}, + ) + with pytest.raises(HTTPError) as exc_info: + urllib.request.urlopen(req, timeout=5) + assert exc_info.value.code == 409 + + def test_reload_endpoint_rejects_bad_token(self, tmp_path): + """POST /api/admin/reload returns 401 with wrong token.""" + import time + import urllib.request + from urllib.error import HTTPError + + bot, _token, port = self._make_bot_with_admin(tmp_path, port=15005) + server = bot._admin_server + server.start() + time.sleep(0.4) + + req = urllib.request.Request( + f"http://127.0.0.1:{port}/api/admin/reload", + method="POST", + headers={"Authorization": "Bearer wrong-token"}, + ) + with pytest.raises(HTTPError) as exc_info: + urllib.request.urlopen(req, timeout=5) + assert exc_info.value.code == 401 + + def test_health_endpoint_returns_ok(self, tmp_path): + """GET /api/admin/health returns 200 and status=ok.""" + import time + import urllib.request + + bot, token, port = self._make_bot_with_admin(tmp_path, port=15006) + server = bot._admin_server + server.start() + time.sleep(0.4) + + req = urllib.request.Request( + f"http://127.0.0.1:{port}/api/admin/health", + method="GET", + headers={"Authorization": f"Bearer {token}"}, + ) + with urllib.request.urlopen(req, timeout=5) as resp: + import json + body = json.loads(resp.read()) + assert body["status"] == "ok" + + +# --------------------------------------------------------------------------- +# TestRadioOfflineState +# --------------------------------------------------------------------------- + + +class TestRadioOfflineState: + """Tests for _record_send_failure / _record_send_success / is_radio_offline.""" + + def _make_bot(self, tmp_path: Path) -> "MeshCoreBot": + config_file = tmp_path / "config.ini" + db_path = tmp_path / "bot.db" + _write_config(config_file, db_path) + return MeshCoreBot(config_file=str(config_file)) + + def test_is_radio_offline_defaults_to_false(self, tmp_path): + bot = self._make_bot(tmp_path) + assert bot.is_radio_offline is False + + def test_record_send_failure_increments_counter(self, tmp_path): + bot = self._make_bot(tmp_path) + bot._record_send_failure() + assert bot._send_consecutive_failures == 1 + + def test_record_send_failure_sets_offline_at_threshold(self, tmp_path): + bot = self._make_bot(tmp_path) + # Default threshold is 3 + for _ in range(3): + bot._record_send_failure() + assert bot.is_radio_offline is True + + def test_record_send_success_clears_offline_flag(self, tmp_path): + bot = self._make_bot(tmp_path) + bot._radio_offline = True + bot._send_consecutive_failures = 5 + bot._record_send_success() + assert bot.is_radio_offline is False + assert bot._send_consecutive_failures == 0 + + def test_record_send_success_no_op_when_already_clean(self, tmp_path): + bot = self._make_bot(tmp_path) + bot._record_send_success() # must not raise + assert bot.is_radio_offline is False + + def test_offline_not_set_below_threshold(self, tmp_path): + bot = self._make_bot(tmp_path) + bot._record_send_failure() + bot._record_send_failure() + assert bot.is_radio_offline is False + + def test_custom_threshold_from_config(self, tmp_path): + bot = self._make_bot(tmp_path) + bot.config.set('Bot', 'radio_offline_threshold', '2') + bot._record_send_failure() + assert bot.is_radio_offline is False + bot._record_send_failure() + assert bot.is_radio_offline is True + + # --------------------------------------------------------------------------- # Helper: create a coroutine that returns a fixed value # --------------------------------------------------------------------------- diff --git a/tests/test_scheduler_logic.py b/tests/test_scheduler_logic.py index 4624c73..e2c526f 100644 --- a/tests/test_scheduler_logic.py +++ b/tests/test_scheduler_logic.py @@ -719,6 +719,8 @@ def _make_scheduler(): config.set("Bot", "advert_interval_hours", "0") bot.config = config bot.main_event_loop = None + bot.is_radio_zombie = False + bot.is_radio_offline = False # db_manager.connection() context manager conn_mock = MagicMock() @@ -1030,6 +1032,81 @@ class TestSendScheduledMessageWrapper: mock_loop.close.assert_called_once() mock_send.assert_called_once_with("general", "test message") + def test_suppressed_when_radio_zombie(self): + scheduler = _make_scheduler() + scheduler.bot.is_radio_zombie = True + with patch("asyncio.run_coroutine_threadsafe") as mock_rct: + scheduler.send_scheduled_message("general", "hi") + mock_rct.assert_not_called() + + def test_suppressed_when_radio_offline(self): + scheduler = _make_scheduler() + scheduler.bot.is_radio_offline = True + with patch("asyncio.run_coroutine_threadsafe") as mock_rct: + scheduler.send_scheduled_message("general", "hi") + mock_rct.assert_not_called() + + def test_records_success_on_successful_send(self): + scheduler = _make_scheduler() + mock_loop = Mock() + mock_loop.is_running.return_value = True + scheduler.bot.main_event_loop = mock_loop + fake_future = Mock() + fake_future.result.return_value = None + with patch("asyncio.run_coroutine_threadsafe", return_value=fake_future): + scheduler.send_scheduled_message("general", "hi") + scheduler.bot._record_send_success.assert_called_once() + + def test_records_failure_on_exception(self): + scheduler = _make_scheduler() + mock_loop = Mock() + mock_loop.is_running.return_value = True + scheduler.bot.main_event_loop = mock_loop + fake_future = Mock() + fake_future.result.side_effect = Exception("bang") + with patch("asyncio.run_coroutine_threadsafe", return_value=fake_future): + scheduler.send_scheduled_message("general", "hi") + scheduler.bot._record_send_failure.assert_called_once() + + +# --------------------------------------------------------------------------- +# TestSendIntervalAdvertOfflineGuard +# --------------------------------------------------------------------------- + + +class TestSendIntervalAdvertOfflineGuard: + """Tests for send_interval_advert() radio-offline guard.""" + + def test_suppressed_when_radio_offline(self): + scheduler = _make_scheduler() + scheduler.bot.is_radio_offline = True + with patch("asyncio.run_coroutine_threadsafe") as mock_rct: + scheduler.send_interval_advert() + mock_rct.assert_not_called() + + def test_records_success_on_successful_send(self): + scheduler = _make_scheduler() + mock_loop = Mock() + mock_loop.is_running.return_value = True + scheduler.bot.main_event_loop = mock_loop + fake_future = Mock() + fake_future.result.return_value = None + with patch("asyncio.run_coroutine_threadsafe", return_value=fake_future): + scheduler.send_interval_advert() + scheduler.bot._record_send_success.assert_called_once() + + def test_records_failure_on_exception(self): + from concurrent.futures import TimeoutError as FuturesTimeoutError + scheduler = _make_scheduler() + mock_loop = Mock() + mock_loop.is_running.return_value = True + scheduler.bot.main_event_loop = mock_loop + fake_future = Mock() + fake_future.result.side_effect = FuturesTimeoutError() + with patch("asyncio.run_coroutine_threadsafe", return_value=fake_future): + scheduler.send_interval_advert() + scheduler.bot._record_send_failure.assert_called_once() + # --------------------------------------------------------------------------- # TestRunDataRetention @@ -1242,6 +1319,8 @@ def _make_sched_with_logger(mock_logger): bot.logger = mock_logger bot.config = ConfigParser() bot.config.add_section("Bot") + bot.is_radio_zombie = False # ensure zombie guard does not suppress sends + bot.is_radio_offline = False # ensure offline guard does not suppress sends return MessageScheduler(bot)