feat: radio-offline fail state — suppress sends, auto-restart, banner, and docs

This commit is contained in:
Stacy Olivas
2026-04-09 21:30:58 -07:00
parent 582e56ffb5
commit ac87f70d8e
8 changed files with 612 additions and 0 deletions
+3
View File
@@ -321,6 +321,9 @@ radio_probe_fail_threshold = 3 # consecutive failures before zombie is dec
send_timeout_seconds = 30 # max seconds to wait for a channel message send
radio_zombie_alert_enabled = false # send immediate alert email on zombie detection (default: log only)
radio_zombie_alert_email = # alert recipient(s); falls back to nightly email if blank
radio_offline_threshold = 3 # consecutive send timeouts before radio-offline state is entered
radio_offline_alert_enabled = true # send alert email when radio-offline state is entered
radio_offline_alert_email = # alert recipient(s); falls back to nightly email if blank
```
### Keywords
+69
View File
@@ -299,6 +299,75 @@ class MeshCoreBot:
"""
return bool(getattr(self, '_radio_zombie_detected', False))
@property
def is_radio_offline(self) -> bool:
"""True when repeated outbound send timeouts have been detected.
Distinct from zombie state — the radio may still be forwarding received
packets but is not completing outbound sends. Cleared automatically
when a send succeeds, so no manual intervention is required.
"""
return bool(getattr(self, '_radio_offline', False))
def _record_send_failure(self, scheduler: "Any | None" = None) -> None:
"""Increment the consecutive-send-failure counter.
Called by the scheduler when an outbound send times out at the
``future.result()`` level (i.e. the outer 60-second wall-clock
timeout fired). After ``radio_offline_threshold`` consecutive
failures the bot transitions to radio-offline state, persists it
to the DB for the web viewer banner, and optionally sends an alert
email.
"""
import datetime as _dt
import threading as _threading
self._send_consecutive_failures: int = (
getattr(self, '_send_consecutive_failures', 0) + 1
)
threshold = self.config.getint('Bot', 'radio_offline_threshold', fallback=3)
if self._send_consecutive_failures >= threshold and not self.is_radio_offline:
self._radio_offline = True
since = _dt.datetime.utcnow().isoformat()
self.logger.critical(
"RADIO OFFLINE: %d consecutive send timeouts (threshold %d). "
"Bot will suppress further outbound sends until one succeeds. "
"Check radio power and connection.",
self._send_consecutive_failures,
threshold,
)
try:
self.db_manager.set_metadata('bot.radio_offline', 'true')
self.db_manager.set_metadata('bot.radio_offline_since', since)
except Exception:
pass
if scheduler is not None:
_threading.Thread(
target=scheduler.send_radio_offline_alert_email,
args=(self._send_consecutive_failures, threshold),
daemon=True,
).start()
def _record_send_success(self) -> None:
"""Clear the consecutive-send-failure counter after a successful send."""
failures = getattr(self, '_send_consecutive_failures', 0)
was_offline = self.is_radio_offline
if failures > 0 or was_offline:
self.logger.info(
"Outbound send succeeded — clearing radio-offline state "
"(was_offline=%s, failure_count=%d)",
was_offline,
failures,
)
self._send_consecutive_failures = 0
if was_offline:
self._radio_offline = False
try:
self.db_manager.set_metadata('bot.radio_offline', 'false')
self.db_manager.set_metadata('bot.radio_offline_since', '')
except Exception:
pass
def load_config(self) -> None:
"""Load configuration from file.
+133
View File
@@ -121,6 +121,13 @@ class MessageScheduler:
def send_scheduled_message(self, channel: str, message: str):
"""Send a scheduled message (synchronous wrapper for schedule library)"""
if self.bot.is_radio_zombie:
self.logger.warning("send_scheduled_message suppressed — radio is in zombie state")
return
if self.bot.is_radio_offline:
self.logger.warning("send_scheduled_message suppressed — radio is offline (repeated send timeouts)")
return
current_time = self.get_current_time()
self.logger.info(f"📅 Sending scheduled message at {current_time.strftime('%H:%M:%S')} to {channel}: {message}")
@@ -137,8 +144,10 @@ class MessageScheduler:
# Wait for completion (with timeout to prevent indefinite blocking)
try:
future.result(timeout=60) # 60 second timeout
self.bot._record_send_success()
except Exception as e:
self.logger.error(f"Error sending scheduled message: {type(e).__name__}: {e}", exc_info=True)
self.bot._record_send_failure(scheduler=self)
else:
# Fallback: create new event loop if main loop not available
try:
@@ -613,6 +622,10 @@ class MessageScheduler:
def send_interval_advert(self):
"""Send an interval-based advert (synchronous wrapper)"""
if self.bot.is_radio_offline:
self.logger.warning("send_interval_advert suppressed — radio is offline (repeated send timeouts)")
return
current_time = self.get_current_time()
self.logger.info(f"📢 Sending interval-based flood advert at {current_time.strftime('%H:%M:%S')}")
@@ -629,8 +642,10 @@ class MessageScheduler:
# Wait for completion (with timeout to prevent indefinite blocking)
try:
future.result(timeout=60) # 60 second timeout
self.bot._record_send_success()
except Exception as e:
self.logger.error(f"Error sending interval advert: {type(e).__name__}: {e}", exc_info=True)
self.bot._record_send_failure(scheduler=self)
else:
# Fallback: create new event loop if main loop not available
try:
@@ -1319,6 +1334,124 @@ class MessageScheduler:
except Exception as e:
self.bot.logger.error(f"Failed to send zombie radio alert email: {e}")
# ── Radio offline alert email ────────────────────────────────────────────
def send_radio_offline_alert_email(self, fail_count: int, threshold: int) -> None:
"""Send an immediate alert email when the radio-offline state is entered.
Uses the same SMTP settings as the nightly digest. Recipients are taken
from the ``radio_offline_alert_email`` config key; if that key is empty the
nightly maintenance recipients are used as a fallback.
Intentionally synchronous — intended to be run in a daemon thread.
"""
import smtplib
import ssl as _ssl
from email.message import EmailMessage
alert_enabled = self.bot.config.getboolean('Bot', 'radio_offline_alert_enabled', fallback=True)
if not alert_enabled:
return
smtp_host = self._get_notif('smtp_host')
smtp_security = self._get_notif('smtp_security') or 'starttls'
smtp_user = self._get_notif('smtp_user')
smtp_password = self._get_notif('smtp_password')
from_name = self._get_notif('from_name') or 'MeshCore Bot'
from_email = self._get_notif('from_email')
alert_email_cfg = self.bot.config.get('Bot', 'radio_offline_alert_email', fallback='').strip()
if alert_email_cfg:
recipients = [r.strip() for r in alert_email_cfg.split(',') if r.strip()]
else:
recipients = [r.strip() for r in self._get_notif('recipients').split(',') if r.strip()]
if not smtp_host or not from_email or not recipients:
self.bot.logger.warning(
"Radio-offline alert email enabled but SMTP settings incomplete "
f"(host={smtp_host!r}, from={from_email!r}, recipients={recipients}) "
"— alert email not sent"
)
return
allow_local = self._get_notif('allow_local_smtp').lower() == 'true'
if not validate_external_url(f'http://{smtp_host}', allow_private=allow_local):
self.bot.logger.error(
"Radio-offline alert email aborted: SMTP host %r resolves to a private or reserved address",
smtp_host,
)
return
try:
smtp_port = int(self._get_notif('smtp_port') or (465 if smtp_security == 'ssl' else 587))
except ValueError:
smtp_port = 587
now_utc = datetime.datetime.utcnow()
connection_type = self.bot.config.get('Connection', 'connection_type', fallback='unknown')
serial_port = self.bot.config.get('Connection', 'serial_port', fallback='n/a')
subject = (
f'ALERT: MeshCore Bot — Radio Offline '
f'[{now_utc.strftime("%Y-%m-%d %H:%M UTC")}]'
)
body = '\n'.join([
'MeshCore Bot — Radio Offline Alert',
'=' * 44,
f'Time : {now_utc.strftime("%Y-%m-%d %H:%M:%S UTC")}',
'',
'RADIO STATUS',
'' * 30,
f' Connection : {connection_type}',
f' Port / Device : {serial_port}',
f' Failed sends : {fail_count} of {threshold} (threshold)',
'',
'WHAT THIS MEANS',
'' * 30,
' The bot can no longer send outbound messages to the mesh.',
' Inbound packets from the radio may still be arriving normally.',
' This is NOT a zombie (firmware lock-up) — the radio is responsive',
' but outbound sends are timing out.',
'',
'ACTION REQUIRED',
'' * 30,
' Check the radio power supply and physical connection.',
' Use the dashboard "Clear Offline Flag" button once the issue',
' is resolved, or restart the bot to auto-probe.',
'',
'' * 44,
'Outbound sends will be suppressed until the offline flag is cleared.',
])
try:
msg = EmailMessage()
msg['Subject'] = subject
msg['From'] = f'{from_name} <{from_email}>'
msg['To'] = ', '.join(recipients)
msg.set_content(body)
context = _ssl.create_default_context()
_smtp_timeout = 30
if smtp_security == 'ssl':
with smtplib.SMTP_SSL(smtp_host, smtp_port, context=context, timeout=_smtp_timeout) as s:
if smtp_user and smtp_password:
s.login(smtp_user, smtp_password)
s.send_message(msg)
else:
with smtplib.SMTP(smtp_host, smtp_port, timeout=_smtp_timeout) as s:
if smtp_security == 'starttls':
s.ehlo()
s.starttls(context=context)
s.ehlo()
if smtp_user and smtp_password:
s.login(smtp_user, smtp_password)
s.send_message(msg)
self.bot.logger.info(f"Radio-offline alert email sent to {recipients}")
except Exception as e:
self.bot.logger.error(f"Failed to send radio-offline alert email: {e}")
# ── Maintenance helpers ──────────────────────────────────────────────────
def _get_maint(self, key: str) -> str:
+47
View File
@@ -281,9 +281,13 @@ class BotDataViewer:
try:
radio_zombie = self.db_manager.get_metadata('bot.radio_zombie') == 'true'
radio_zombie_since = self.db_manager.get_metadata('bot.radio_zombie_since') or None
radio_offline = self.db_manager.get_metadata('bot.radio_offline') == 'true'
radio_offline_since = self.db_manager.get_metadata('bot.radio_offline_since') or None
except Exception:
radio_zombie = False
radio_zombie_since = None
radio_offline = False
radio_offline_since = None
return {
'greeter_enabled': greeter_enabled,
'feed_manager_enabled': feed_manager_enabled,
@@ -291,6 +295,8 @@ class BotDataViewer:
'version_info': version_info,
'radio_zombie': radio_zombie,
'radio_zombie_since': radio_zombie_since,
'radio_offline': radio_offline,
'radio_offline_since': radio_offline_since,
}
except Exception as e:
self.logger.exception("Template context processor failed: %s", e)
@@ -301,6 +307,8 @@ class BotDataViewer:
'version_info': version_info,
'radio_zombie': False,
'radio_zombie_since': None,
'radio_offline': False,
'radio_offline_since': None,
}
def _init_databases(self):
@@ -1599,6 +1607,24 @@ class BotDataViewer:
self.logger.exception("Error clearing zombie state")
return jsonify({'success': False, 'error': 'Internal error — see server logs'}), 500
# ── Radio offline clear ──────────────────────────────────────────────
@self.app.route('/api/admin/radio-offline-clear', methods=['POST'])
def api_admin_radio_offline_clear() -> "Response":
"""Clear the radio-offline flag so the bot resumes outbound sends."""
try:
self.db_manager.set_metadata('bot.radio_offline', 'false')
self.db_manager.set_metadata('bot.radio_offline_since', '')
bot = getattr(self, 'bot', None)
if bot is not None:
bot._radio_offline = False
bot._send_consecutive_failures = 0
self.logger.info("Radio-offline state cleared via web UI action")
return jsonify({'success': True, 'message': 'Radio-offline flag cleared; sends will resume'})
except Exception:
self.logger.exception("Error clearing radio-offline state")
return jsonify({'success': False, 'error': 'Internal error — see server logs'}), 500
# ── Maintenance status ───────────────────────────────────────────────
@self.app.route('/api/maintenance/backup_now', methods=['POST'])
@@ -7252,6 +7278,7 @@ class BotDataViewer:
def run(self, host='127.0.0.1', port=8080, debug=False):
"""Run the modern web viewer"""
self.logger.info(f"Starting modern web viewer on {host}:{port}")
self._suppress_werkzeug_headers_error()
try:
self.socketio.run(
self.app,
@@ -7264,6 +7291,26 @@ class BotDataViewer:
self.logger.error(f"Error running web viewer: {e}")
raise
@staticmethod
def _suppress_werkzeug_headers_error() -> None:
"""Install a log filter that silences the 'Headers already set' AssertionError.
Werkzeug's dev server catches this internally and continues serving, but it
logs a full traceback at ERROR level. The underlying cause (concurrent
SocketIO polling requests racing through the WSGI layer) is reduced by the
single-socket-per-page fix, but may still occur occasionally. The filter
downgrades these specific records to DEBUG so they don't alarm operators.
"""
import logging
class _HeadersAlreadySetFilter(logging.Filter):
def filter(self, record: logging.LogRecord) -> bool:
msg = record.getMessage()
return "Headers already set" not in msg
for name in ("werkzeug", "werkzeug.serving"):
logging.getLogger(name).addFilter(_HeadersAlreadySetFilter())
def main():
"""Entry point for the meshcore-viewer command"""
import argparse
+9
View File
@@ -750,6 +750,15 @@ class WebViewerIntegration:
while self.running and self.viewer_process and self.viewer_process.poll() is None:
time.sleep(1)
# Process exited unexpectedly — try to restart if we haven't been stopped
if self.running and self.viewer_process and self.viewer_process.poll() is not None:
self.logger.warning(
"Web viewer process exited unexpectedly (code %s) — attempting restart",
self.viewer_process.returncode,
)
self.restart_viewer()
return
# Process exited - read from log files for error reporting if needed
if self.viewer_process and self.viewer_process.returncode != 0:
stdout_file.flush()
+56
View File
@@ -512,6 +512,62 @@
</script>
{% endif %}
<!-- Radio Offline Banner (shown when repeated send timeouts have entered the offline state) -->
{% if radio_offline %}
<div id="offline-banner" class="alert alert-warning mb-0 rounded-0 border-0 border-bottom border-warning" role="alert" style="border-width: 2px !important;">
<div class="container-fluid">
<div class="d-flex align-items-center justify-content-between flex-wrap gap-2">
<div>
<i class="fas fa-plug me-2"></i>
<strong>Radio Offline</strong> — the bot cannot send outbound messages to the mesh.
Check radio power and connection. Inbound packets may still be arriving normally.
{% if radio_offline_since %}
<span class="ms-3 text-warning-emphasis small opacity-75">
<i class="fas fa-clock me-1"></i>Since: {{ radio_offline_since }}
</span>
{% endif %}
</div>
<div class="d-flex gap-2 align-items-center flex-shrink-0">
<span class="small opacity-75">Once radio is fixed:</span>
<button id="offline-clear-btn" class="btn btn-dark btn-sm fw-semibold"
onclick="radioOfflineClear()" type="button">
<i class="fas fa-check me-1"></i>Clear Offline Flag
</button>
</div>
</div>
</div>
</div>
<script>
function radioOfflineClear() {
var btn = document.getElementById('offline-clear-btn');
btn.disabled = true;
btn.innerHTML = '<i class="fas fa-spinner fa-spin me-1"></i>Clearing\u2026';
fetch('/api/admin/radio-offline-clear', {
method: 'POST',
headers: {'Content-Type': 'application/json'}
})
.then(function(r) { return r.json(); })
.then(function(data) {
var banner = document.getElementById('offline-banner');
if (data.success) {
banner.className = 'alert alert-success mb-0 rounded-0 border-0 border-bottom border-success';
banner.innerHTML = '<div class="container-fluid"><i class="fas fa-check-circle me-2"></i>' +
'<strong>Offline flag cleared.</strong> The bot will resume outbound sends. Refresh to confirm.</div>';
} else {
btn.disabled = false;
btn.innerHTML = '<i class="fas fa-check me-1"></i>Clear Offline Flag';
alert('Error: ' + (data.error || 'Unknown error — check server logs'));
}
})
.catch(function() {
btn.disabled = false;
btn.innerHTML = '<i class="fas fa-check me-1"></i>Clear Offline Flag';
alert('Network error — could not reach the server.');
});
}
</script>
{% endif %}
<!-- Main Content -->
<div class="container-fluid mt-4">
{% block content %}{% endblock %}
+216
View File
@@ -481,6 +481,222 @@ class TestProbeRadioHealth:
assert mock_warn.called
# ---------------------------------------------------------------------------
# _BotAdminServer — admin HTTP API
# ---------------------------------------------------------------------------
class TestBotAdminServer:
"""Admin HTTP server: /api/admin/reload and /api/admin/health."""
def _make_bot_with_admin(self, tmp_path, port=15001):
"""Write config with [Admin] enabled and return a bot + token."""
token = "test-secret-token"
config_file = tmp_path / "config.ini"
db_path = tmp_path / "bot.db"
config_file.write_text(
f"""[Connection]
connection_type = serial
serial_port = /dev/ttyUSB0
timeout = 30
[Bot]
db_path = {db_path.as_posix()}
prefix_bytes = 1
[Channels]
monitor_channels = #general
[Admin]
enabled = true
port = {port}
token = {token}
""",
encoding="utf-8",
)
bot = MeshCoreBot(config_file=str(config_file))
return bot, token, port
def test_admin_server_created_when_enabled(self, tmp_path):
bot, _token, _port = self._make_bot_with_admin(tmp_path)
assert bot._admin_server is not None
def test_admin_server_none_when_disabled(self, tmp_path):
config_file = tmp_path / "config.ini"
db_path = tmp_path / "bot.db"
_write_config(config_file, db_path)
bot = MeshCoreBot(config_file=str(config_file))
assert bot._admin_server is None
def test_admin_server_none_when_token_missing(self, tmp_path):
config_file = tmp_path / "config.ini"
db_path = tmp_path / "bot.db"
config_file.write_text(
f"""[Connection]
connection_type = serial
serial_port = /dev/ttyUSB0
timeout = 30
[Bot]
db_path = {db_path.as_posix()}
prefix_bytes = 1
[Channels]
monitor_channels = #general
[Admin]
enabled = true
port = 15002
token =
""",
encoding="utf-8",
)
bot = MeshCoreBot(config_file=str(config_file))
assert bot._admin_server is None
def test_reload_endpoint_success(self, tmp_path):
"""POST /api/admin/reload returns 200 and success=true when reload succeeds."""
import time
import urllib.request
bot, token, port = self._make_bot_with_admin(tmp_path, port=15003)
with patch.object(bot, "reload_config", return_value=(True, "Config reloaded")):
server = bot._admin_server
server.start()
time.sleep(0.4)
req = urllib.request.Request(
f"http://127.0.0.1:{port}/api/admin/reload",
method="POST",
headers={"Authorization": f"Bearer {token}"},
)
with urllib.request.urlopen(req, timeout=5) as resp:
import json
body = json.loads(resp.read())
assert body["success"] is True
assert "Config reloaded" in body["message"]
def test_reload_endpoint_failure(self, tmp_path):
"""POST /api/admin/reload returns 409 when reload is rejected."""
import time
import urllib.request
from urllib.error import HTTPError
bot, token, port = self._make_bot_with_admin(tmp_path, port=15004)
with patch.object(bot, "reload_config", return_value=(False, "Radio settings changed")):
server = bot._admin_server
server.start()
time.sleep(0.4)
req = urllib.request.Request(
f"http://127.0.0.1:{port}/api/admin/reload",
method="POST",
headers={"Authorization": f"Bearer {token}"},
)
with pytest.raises(HTTPError) as exc_info:
urllib.request.urlopen(req, timeout=5)
assert exc_info.value.code == 409
def test_reload_endpoint_rejects_bad_token(self, tmp_path):
"""POST /api/admin/reload returns 401 with wrong token."""
import time
import urllib.request
from urllib.error import HTTPError
bot, _token, port = self._make_bot_with_admin(tmp_path, port=15005)
server = bot._admin_server
server.start()
time.sleep(0.4)
req = urllib.request.Request(
f"http://127.0.0.1:{port}/api/admin/reload",
method="POST",
headers={"Authorization": "Bearer wrong-token"},
)
with pytest.raises(HTTPError) as exc_info:
urllib.request.urlopen(req, timeout=5)
assert exc_info.value.code == 401
def test_health_endpoint_returns_ok(self, tmp_path):
"""GET /api/admin/health returns 200 and status=ok."""
import time
import urllib.request
bot, token, port = self._make_bot_with_admin(tmp_path, port=15006)
server = bot._admin_server
server.start()
time.sleep(0.4)
req = urllib.request.Request(
f"http://127.0.0.1:{port}/api/admin/health",
method="GET",
headers={"Authorization": f"Bearer {token}"},
)
with urllib.request.urlopen(req, timeout=5) as resp:
import json
body = json.loads(resp.read())
assert body["status"] == "ok"
# ---------------------------------------------------------------------------
# TestRadioOfflineState
# ---------------------------------------------------------------------------
class TestRadioOfflineState:
"""Tests for _record_send_failure / _record_send_success / is_radio_offline."""
def _make_bot(self, tmp_path: Path) -> "MeshCoreBot":
config_file = tmp_path / "config.ini"
db_path = tmp_path / "bot.db"
_write_config(config_file, db_path)
return MeshCoreBot(config_file=str(config_file))
def test_is_radio_offline_defaults_to_false(self, tmp_path):
bot = self._make_bot(tmp_path)
assert bot.is_radio_offline is False
def test_record_send_failure_increments_counter(self, tmp_path):
bot = self._make_bot(tmp_path)
bot._record_send_failure()
assert bot._send_consecutive_failures == 1
def test_record_send_failure_sets_offline_at_threshold(self, tmp_path):
bot = self._make_bot(tmp_path)
# Default threshold is 3
for _ in range(3):
bot._record_send_failure()
assert bot.is_radio_offline is True
def test_record_send_success_clears_offline_flag(self, tmp_path):
bot = self._make_bot(tmp_path)
bot._radio_offline = True
bot._send_consecutive_failures = 5
bot._record_send_success()
assert bot.is_radio_offline is False
assert bot._send_consecutive_failures == 0
def test_record_send_success_no_op_when_already_clean(self, tmp_path):
bot = self._make_bot(tmp_path)
bot._record_send_success() # must not raise
assert bot.is_radio_offline is False
def test_offline_not_set_below_threshold(self, tmp_path):
bot = self._make_bot(tmp_path)
bot._record_send_failure()
bot._record_send_failure()
assert bot.is_radio_offline is False
def test_custom_threshold_from_config(self, tmp_path):
bot = self._make_bot(tmp_path)
bot.config.set('Bot', 'radio_offline_threshold', '2')
bot._record_send_failure()
assert bot.is_radio_offline is False
bot._record_send_failure()
assert bot.is_radio_offline is True
# ---------------------------------------------------------------------------
# Helper: create a coroutine that returns a fixed value
# ---------------------------------------------------------------------------
+79
View File
@@ -719,6 +719,8 @@ def _make_scheduler():
config.set("Bot", "advert_interval_hours", "0")
bot.config = config
bot.main_event_loop = None
bot.is_radio_zombie = False
bot.is_radio_offline = False
# db_manager.connection() context manager
conn_mock = MagicMock()
@@ -1030,6 +1032,81 @@ class TestSendScheduledMessageWrapper:
mock_loop.close.assert_called_once()
mock_send.assert_called_once_with("general", "test message")
def test_suppressed_when_radio_zombie(self):
scheduler = _make_scheduler()
scheduler.bot.is_radio_zombie = True
with patch("asyncio.run_coroutine_threadsafe") as mock_rct:
scheduler.send_scheduled_message("general", "hi")
mock_rct.assert_not_called()
def test_suppressed_when_radio_offline(self):
scheduler = _make_scheduler()
scheduler.bot.is_radio_offline = True
with patch("asyncio.run_coroutine_threadsafe") as mock_rct:
scheduler.send_scheduled_message("general", "hi")
mock_rct.assert_not_called()
def test_records_success_on_successful_send(self):
scheduler = _make_scheduler()
mock_loop = Mock()
mock_loop.is_running.return_value = True
scheduler.bot.main_event_loop = mock_loop
fake_future = Mock()
fake_future.result.return_value = None
with patch("asyncio.run_coroutine_threadsafe", return_value=fake_future):
scheduler.send_scheduled_message("general", "hi")
scheduler.bot._record_send_success.assert_called_once()
def test_records_failure_on_exception(self):
scheduler = _make_scheduler()
mock_loop = Mock()
mock_loop.is_running.return_value = True
scheduler.bot.main_event_loop = mock_loop
fake_future = Mock()
fake_future.result.side_effect = Exception("bang")
with patch("asyncio.run_coroutine_threadsafe", return_value=fake_future):
scheduler.send_scheduled_message("general", "hi")
scheduler.bot._record_send_failure.assert_called_once()
# ---------------------------------------------------------------------------
# TestSendIntervalAdvertOfflineGuard
# ---------------------------------------------------------------------------
class TestSendIntervalAdvertOfflineGuard:
"""Tests for send_interval_advert() radio-offline guard."""
def test_suppressed_when_radio_offline(self):
scheduler = _make_scheduler()
scheduler.bot.is_radio_offline = True
with patch("asyncio.run_coroutine_threadsafe") as mock_rct:
scheduler.send_interval_advert()
mock_rct.assert_not_called()
def test_records_success_on_successful_send(self):
scheduler = _make_scheduler()
mock_loop = Mock()
mock_loop.is_running.return_value = True
scheduler.bot.main_event_loop = mock_loop
fake_future = Mock()
fake_future.result.return_value = None
with patch("asyncio.run_coroutine_threadsafe", return_value=fake_future):
scheduler.send_interval_advert()
scheduler.bot._record_send_success.assert_called_once()
def test_records_failure_on_exception(self):
from concurrent.futures import TimeoutError as FuturesTimeoutError
scheduler = _make_scheduler()
mock_loop = Mock()
mock_loop.is_running.return_value = True
scheduler.bot.main_event_loop = mock_loop
fake_future = Mock()
fake_future.result.side_effect = FuturesTimeoutError()
with patch("asyncio.run_coroutine_threadsafe", return_value=fake_future):
scheduler.send_interval_advert()
scheduler.bot._record_send_failure.assert_called_once()
# ---------------------------------------------------------------------------
# TestRunDataRetention
@@ -1242,6 +1319,8 @@ def _make_sched_with_logger(mock_logger):
bot.logger = mock_logger
bot.config = ConfigParser()
bot.config.add_section("Bot")
bot.is_radio_zombie = False # ensure zombie guard does not suppress sends
bot.is_radio_offline = False # ensure offline guard does not suppress sends
return MessageScheduler(bot)