mirror of
https://github.com/agessaman/meshcore-bot.git
synced 2026-08-27 21:10:13 +00:00
feat: radio-offline fail state — suppress sends, auto-restart, banner, and docs
This commit is contained in:
@@ -321,6 +321,9 @@ radio_probe_fail_threshold = 3 # consecutive failures before zombie is dec
|
||||
send_timeout_seconds = 30 # max seconds to wait for a channel message send
|
||||
radio_zombie_alert_enabled = false # send immediate alert email on zombie detection (default: log only)
|
||||
radio_zombie_alert_email = # alert recipient(s); falls back to nightly email if blank
|
||||
radio_offline_threshold = 3 # consecutive send timeouts before radio-offline state is entered
|
||||
radio_offline_alert_enabled = true # send alert email when radio-offline state is entered
|
||||
radio_offline_alert_email = # alert recipient(s); falls back to nightly email if blank
|
||||
```
|
||||
|
||||
### Keywords
|
||||
|
||||
@@ -299,6 +299,75 @@ class MeshCoreBot:
|
||||
"""
|
||||
return bool(getattr(self, '_radio_zombie_detected', False))
|
||||
|
||||
@property
|
||||
def is_radio_offline(self) -> bool:
|
||||
"""True when repeated outbound send timeouts have been detected.
|
||||
|
||||
Distinct from zombie state — the radio may still be forwarding received
|
||||
packets but is not completing outbound sends. Cleared automatically
|
||||
when a send succeeds, so no manual intervention is required.
|
||||
"""
|
||||
return bool(getattr(self, '_radio_offline', False))
|
||||
|
||||
def _record_send_failure(self, scheduler: "Any | None" = None) -> None:
|
||||
"""Increment the consecutive-send-failure counter.
|
||||
|
||||
Called by the scheduler when an outbound send times out at the
|
||||
``future.result()`` level (i.e. the outer 60-second wall-clock
|
||||
timeout fired). After ``radio_offline_threshold`` consecutive
|
||||
failures the bot transitions to radio-offline state, persists it
|
||||
to the DB for the web viewer banner, and optionally sends an alert
|
||||
email.
|
||||
"""
|
||||
import datetime as _dt
|
||||
import threading as _threading
|
||||
|
||||
self._send_consecutive_failures: int = (
|
||||
getattr(self, '_send_consecutive_failures', 0) + 1
|
||||
)
|
||||
threshold = self.config.getint('Bot', 'radio_offline_threshold', fallback=3)
|
||||
if self._send_consecutive_failures >= threshold and not self.is_radio_offline:
|
||||
self._radio_offline = True
|
||||
since = _dt.datetime.utcnow().isoformat()
|
||||
self.logger.critical(
|
||||
"RADIO OFFLINE: %d consecutive send timeouts (threshold %d). "
|
||||
"Bot will suppress further outbound sends until one succeeds. "
|
||||
"Check radio power and connection.",
|
||||
self._send_consecutive_failures,
|
||||
threshold,
|
||||
)
|
||||
try:
|
||||
self.db_manager.set_metadata('bot.radio_offline', 'true')
|
||||
self.db_manager.set_metadata('bot.radio_offline_since', since)
|
||||
except Exception:
|
||||
pass
|
||||
if scheduler is not None:
|
||||
_threading.Thread(
|
||||
target=scheduler.send_radio_offline_alert_email,
|
||||
args=(self._send_consecutive_failures, threshold),
|
||||
daemon=True,
|
||||
).start()
|
||||
|
||||
def _record_send_success(self) -> None:
|
||||
"""Clear the consecutive-send-failure counter after a successful send."""
|
||||
failures = getattr(self, '_send_consecutive_failures', 0)
|
||||
was_offline = self.is_radio_offline
|
||||
if failures > 0 or was_offline:
|
||||
self.logger.info(
|
||||
"Outbound send succeeded — clearing radio-offline state "
|
||||
"(was_offline=%s, failure_count=%d)",
|
||||
was_offline,
|
||||
failures,
|
||||
)
|
||||
self._send_consecutive_failures = 0
|
||||
if was_offline:
|
||||
self._radio_offline = False
|
||||
try:
|
||||
self.db_manager.set_metadata('bot.radio_offline', 'false')
|
||||
self.db_manager.set_metadata('bot.radio_offline_since', '')
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def load_config(self) -> None:
|
||||
"""Load configuration from file.
|
||||
|
||||
|
||||
@@ -121,6 +121,13 @@ class MessageScheduler:
|
||||
|
||||
def send_scheduled_message(self, channel: str, message: str):
|
||||
"""Send a scheduled message (synchronous wrapper for schedule library)"""
|
||||
if self.bot.is_radio_zombie:
|
||||
self.logger.warning("send_scheduled_message suppressed — radio is in zombie state")
|
||||
return
|
||||
if self.bot.is_radio_offline:
|
||||
self.logger.warning("send_scheduled_message suppressed — radio is offline (repeated send timeouts)")
|
||||
return
|
||||
|
||||
current_time = self.get_current_time()
|
||||
self.logger.info(f"📅 Sending scheduled message at {current_time.strftime('%H:%M:%S')} to {channel}: {message}")
|
||||
|
||||
@@ -137,8 +144,10 @@ class MessageScheduler:
|
||||
# Wait for completion (with timeout to prevent indefinite blocking)
|
||||
try:
|
||||
future.result(timeout=60) # 60 second timeout
|
||||
self.bot._record_send_success()
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error sending scheduled message: {type(e).__name__}: {e}", exc_info=True)
|
||||
self.bot._record_send_failure(scheduler=self)
|
||||
else:
|
||||
# Fallback: create new event loop if main loop not available
|
||||
try:
|
||||
@@ -613,6 +622,10 @@ class MessageScheduler:
|
||||
|
||||
def send_interval_advert(self):
|
||||
"""Send an interval-based advert (synchronous wrapper)"""
|
||||
if self.bot.is_radio_offline:
|
||||
self.logger.warning("send_interval_advert suppressed — radio is offline (repeated send timeouts)")
|
||||
return
|
||||
|
||||
current_time = self.get_current_time()
|
||||
self.logger.info(f"📢 Sending interval-based flood advert at {current_time.strftime('%H:%M:%S')}")
|
||||
|
||||
@@ -629,8 +642,10 @@ class MessageScheduler:
|
||||
# Wait for completion (with timeout to prevent indefinite blocking)
|
||||
try:
|
||||
future.result(timeout=60) # 60 second timeout
|
||||
self.bot._record_send_success()
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error sending interval advert: {type(e).__name__}: {e}", exc_info=True)
|
||||
self.bot._record_send_failure(scheduler=self)
|
||||
else:
|
||||
# Fallback: create new event loop if main loop not available
|
||||
try:
|
||||
@@ -1319,6 +1334,124 @@ class MessageScheduler:
|
||||
except Exception as e:
|
||||
self.bot.logger.error(f"Failed to send zombie radio alert email: {e}")
|
||||
|
||||
# ── Radio offline alert email ────────────────────────────────────────────
|
||||
|
||||
def send_radio_offline_alert_email(self, fail_count: int, threshold: int) -> None:
|
||||
"""Send an immediate alert email when the radio-offline state is entered.
|
||||
|
||||
Uses the same SMTP settings as the nightly digest. Recipients are taken
|
||||
from the ``radio_offline_alert_email`` config key; if that key is empty the
|
||||
nightly maintenance recipients are used as a fallback.
|
||||
|
||||
Intentionally synchronous — intended to be run in a daemon thread.
|
||||
"""
|
||||
import smtplib
|
||||
import ssl as _ssl
|
||||
from email.message import EmailMessage
|
||||
|
||||
alert_enabled = self.bot.config.getboolean('Bot', 'radio_offline_alert_enabled', fallback=True)
|
||||
if not alert_enabled:
|
||||
return
|
||||
|
||||
smtp_host = self._get_notif('smtp_host')
|
||||
smtp_security = self._get_notif('smtp_security') or 'starttls'
|
||||
smtp_user = self._get_notif('smtp_user')
|
||||
smtp_password = self._get_notif('smtp_password')
|
||||
from_name = self._get_notif('from_name') or 'MeshCore Bot'
|
||||
from_email = self._get_notif('from_email')
|
||||
|
||||
alert_email_cfg = self.bot.config.get('Bot', 'radio_offline_alert_email', fallback='').strip()
|
||||
if alert_email_cfg:
|
||||
recipients = [r.strip() for r in alert_email_cfg.split(',') if r.strip()]
|
||||
else:
|
||||
recipients = [r.strip() for r in self._get_notif('recipients').split(',') if r.strip()]
|
||||
|
||||
if not smtp_host or not from_email or not recipients:
|
||||
self.bot.logger.warning(
|
||||
"Radio-offline alert email enabled but SMTP settings incomplete "
|
||||
f"(host={smtp_host!r}, from={from_email!r}, recipients={recipients}) "
|
||||
"— alert email not sent"
|
||||
)
|
||||
return
|
||||
|
||||
allow_local = self._get_notif('allow_local_smtp').lower() == 'true'
|
||||
if not validate_external_url(f'http://{smtp_host}', allow_private=allow_local):
|
||||
self.bot.logger.error(
|
||||
"Radio-offline alert email aborted: SMTP host %r resolves to a private or reserved address",
|
||||
smtp_host,
|
||||
)
|
||||
return
|
||||
|
||||
try:
|
||||
smtp_port = int(self._get_notif('smtp_port') or (465 if smtp_security == 'ssl' else 587))
|
||||
except ValueError:
|
||||
smtp_port = 587
|
||||
|
||||
now_utc = datetime.datetime.utcnow()
|
||||
connection_type = self.bot.config.get('Connection', 'connection_type', fallback='unknown')
|
||||
serial_port = self.bot.config.get('Connection', 'serial_port', fallback='n/a')
|
||||
|
||||
subject = (
|
||||
f'ALERT: MeshCore Bot — Radio Offline '
|
||||
f'[{now_utc.strftime("%Y-%m-%d %H:%M UTC")}]'
|
||||
)
|
||||
body = '\n'.join([
|
||||
'MeshCore Bot — Radio Offline Alert',
|
||||
'=' * 44,
|
||||
f'Time : {now_utc.strftime("%Y-%m-%d %H:%M:%S UTC")}',
|
||||
'',
|
||||
'RADIO STATUS',
|
||||
'─' * 30,
|
||||
f' Connection : {connection_type}',
|
||||
f' Port / Device : {serial_port}',
|
||||
f' Failed sends : {fail_count} of {threshold} (threshold)',
|
||||
'',
|
||||
'WHAT THIS MEANS',
|
||||
'─' * 30,
|
||||
' The bot can no longer send outbound messages to the mesh.',
|
||||
' Inbound packets from the radio may still be arriving normally.',
|
||||
' This is NOT a zombie (firmware lock-up) — the radio is responsive',
|
||||
' but outbound sends are timing out.',
|
||||
'',
|
||||
'ACTION REQUIRED',
|
||||
'─' * 30,
|
||||
' Check the radio power supply and physical connection.',
|
||||
' Use the dashboard "Clear Offline Flag" button once the issue',
|
||||
' is resolved, or restart the bot to auto-probe.',
|
||||
'',
|
||||
'─' * 44,
|
||||
'Outbound sends will be suppressed until the offline flag is cleared.',
|
||||
])
|
||||
|
||||
try:
|
||||
msg = EmailMessage()
|
||||
msg['Subject'] = subject
|
||||
msg['From'] = f'{from_name} <{from_email}>'
|
||||
msg['To'] = ', '.join(recipients)
|
||||
msg.set_content(body)
|
||||
|
||||
context = _ssl.create_default_context()
|
||||
_smtp_timeout = 30
|
||||
|
||||
if smtp_security == 'ssl':
|
||||
with smtplib.SMTP_SSL(smtp_host, smtp_port, context=context, timeout=_smtp_timeout) as s:
|
||||
if smtp_user and smtp_password:
|
||||
s.login(smtp_user, smtp_password)
|
||||
s.send_message(msg)
|
||||
else:
|
||||
with smtplib.SMTP(smtp_host, smtp_port, timeout=_smtp_timeout) as s:
|
||||
if smtp_security == 'starttls':
|
||||
s.ehlo()
|
||||
s.starttls(context=context)
|
||||
s.ehlo()
|
||||
if smtp_user and smtp_password:
|
||||
s.login(smtp_user, smtp_password)
|
||||
s.send_message(msg)
|
||||
|
||||
self.bot.logger.info(f"Radio-offline alert email sent to {recipients}")
|
||||
except Exception as e:
|
||||
self.bot.logger.error(f"Failed to send radio-offline alert email: {e}")
|
||||
|
||||
# ── Maintenance helpers ──────────────────────────────────────────────────
|
||||
|
||||
def _get_maint(self, key: str) -> str:
|
||||
|
||||
@@ -281,9 +281,13 @@ class BotDataViewer:
|
||||
try:
|
||||
radio_zombie = self.db_manager.get_metadata('bot.radio_zombie') == 'true'
|
||||
radio_zombie_since = self.db_manager.get_metadata('bot.radio_zombie_since') or None
|
||||
radio_offline = self.db_manager.get_metadata('bot.radio_offline') == 'true'
|
||||
radio_offline_since = self.db_manager.get_metadata('bot.radio_offline_since') or None
|
||||
except Exception:
|
||||
radio_zombie = False
|
||||
radio_zombie_since = None
|
||||
radio_offline = False
|
||||
radio_offline_since = None
|
||||
return {
|
||||
'greeter_enabled': greeter_enabled,
|
||||
'feed_manager_enabled': feed_manager_enabled,
|
||||
@@ -291,6 +295,8 @@ class BotDataViewer:
|
||||
'version_info': version_info,
|
||||
'radio_zombie': radio_zombie,
|
||||
'radio_zombie_since': radio_zombie_since,
|
||||
'radio_offline': radio_offline,
|
||||
'radio_offline_since': radio_offline_since,
|
||||
}
|
||||
except Exception as e:
|
||||
self.logger.exception("Template context processor failed: %s", e)
|
||||
@@ -301,6 +307,8 @@ class BotDataViewer:
|
||||
'version_info': version_info,
|
||||
'radio_zombie': False,
|
||||
'radio_zombie_since': None,
|
||||
'radio_offline': False,
|
||||
'radio_offline_since': None,
|
||||
}
|
||||
|
||||
def _init_databases(self):
|
||||
@@ -1599,6 +1607,24 @@ class BotDataViewer:
|
||||
self.logger.exception("Error clearing zombie state")
|
||||
return jsonify({'success': False, 'error': 'Internal error — see server logs'}), 500
|
||||
|
||||
# ── Radio offline clear ──────────────────────────────────────────────
|
||||
|
||||
@self.app.route('/api/admin/radio-offline-clear', methods=['POST'])
|
||||
def api_admin_radio_offline_clear() -> "Response":
|
||||
"""Clear the radio-offline flag so the bot resumes outbound sends."""
|
||||
try:
|
||||
self.db_manager.set_metadata('bot.radio_offline', 'false')
|
||||
self.db_manager.set_metadata('bot.radio_offline_since', '')
|
||||
bot = getattr(self, 'bot', None)
|
||||
if bot is not None:
|
||||
bot._radio_offline = False
|
||||
bot._send_consecutive_failures = 0
|
||||
self.logger.info("Radio-offline state cleared via web UI action")
|
||||
return jsonify({'success': True, 'message': 'Radio-offline flag cleared; sends will resume'})
|
||||
except Exception:
|
||||
self.logger.exception("Error clearing radio-offline state")
|
||||
return jsonify({'success': False, 'error': 'Internal error — see server logs'}), 500
|
||||
|
||||
# ── Maintenance status ───────────────────────────────────────────────
|
||||
|
||||
@self.app.route('/api/maintenance/backup_now', methods=['POST'])
|
||||
@@ -7252,6 +7278,7 @@ class BotDataViewer:
|
||||
def run(self, host='127.0.0.1', port=8080, debug=False):
|
||||
"""Run the modern web viewer"""
|
||||
self.logger.info(f"Starting modern web viewer on {host}:{port}")
|
||||
self._suppress_werkzeug_headers_error()
|
||||
try:
|
||||
self.socketio.run(
|
||||
self.app,
|
||||
@@ -7264,6 +7291,26 @@ class BotDataViewer:
|
||||
self.logger.error(f"Error running web viewer: {e}")
|
||||
raise
|
||||
|
||||
@staticmethod
|
||||
def _suppress_werkzeug_headers_error() -> None:
|
||||
"""Install a log filter that silences the 'Headers already set' AssertionError.
|
||||
|
||||
Werkzeug's dev server catches this internally and continues serving, but it
|
||||
logs a full traceback at ERROR level. The underlying cause (concurrent
|
||||
SocketIO polling requests racing through the WSGI layer) is reduced by the
|
||||
single-socket-per-page fix, but may still occur occasionally. The filter
|
||||
downgrades these specific records to DEBUG so they don't alarm operators.
|
||||
"""
|
||||
import logging
|
||||
|
||||
class _HeadersAlreadySetFilter(logging.Filter):
|
||||
def filter(self, record: logging.LogRecord) -> bool:
|
||||
msg = record.getMessage()
|
||||
return "Headers already set" not in msg
|
||||
|
||||
for name in ("werkzeug", "werkzeug.serving"):
|
||||
logging.getLogger(name).addFilter(_HeadersAlreadySetFilter())
|
||||
|
||||
def main():
|
||||
"""Entry point for the meshcore-viewer command"""
|
||||
import argparse
|
||||
|
||||
@@ -750,6 +750,15 @@ class WebViewerIntegration:
|
||||
while self.running and self.viewer_process and self.viewer_process.poll() is None:
|
||||
time.sleep(1)
|
||||
|
||||
# Process exited unexpectedly — try to restart if we haven't been stopped
|
||||
if self.running and self.viewer_process and self.viewer_process.poll() is not None:
|
||||
self.logger.warning(
|
||||
"Web viewer process exited unexpectedly (code %s) — attempting restart",
|
||||
self.viewer_process.returncode,
|
||||
)
|
||||
self.restart_viewer()
|
||||
return
|
||||
|
||||
# Process exited - read from log files for error reporting if needed
|
||||
if self.viewer_process and self.viewer_process.returncode != 0:
|
||||
stdout_file.flush()
|
||||
|
||||
@@ -512,6 +512,62 @@
|
||||
</script>
|
||||
{% endif %}
|
||||
|
||||
<!-- Radio Offline Banner (shown when repeated send timeouts have entered the offline state) -->
|
||||
{% if radio_offline %}
|
||||
<div id="offline-banner" class="alert alert-warning mb-0 rounded-0 border-0 border-bottom border-warning" role="alert" style="border-width: 2px !important;">
|
||||
<div class="container-fluid">
|
||||
<div class="d-flex align-items-center justify-content-between flex-wrap gap-2">
|
||||
<div>
|
||||
<i class="fas fa-plug me-2"></i>
|
||||
<strong>Radio Offline</strong> — the bot cannot send outbound messages to the mesh.
|
||||
Check radio power and connection. Inbound packets may still be arriving normally.
|
||||
{% if radio_offline_since %}
|
||||
<span class="ms-3 text-warning-emphasis small opacity-75">
|
||||
<i class="fas fa-clock me-1"></i>Since: {{ radio_offline_since }}
|
||||
</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
<div class="d-flex gap-2 align-items-center flex-shrink-0">
|
||||
<span class="small opacity-75">Once radio is fixed:</span>
|
||||
<button id="offline-clear-btn" class="btn btn-dark btn-sm fw-semibold"
|
||||
onclick="radioOfflineClear()" type="button">
|
||||
<i class="fas fa-check me-1"></i>Clear Offline Flag
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<script>
|
||||
function radioOfflineClear() {
|
||||
var btn = document.getElementById('offline-clear-btn');
|
||||
btn.disabled = true;
|
||||
btn.innerHTML = '<i class="fas fa-spinner fa-spin me-1"></i>Clearing\u2026';
|
||||
fetch('/api/admin/radio-offline-clear', {
|
||||
method: 'POST',
|
||||
headers: {'Content-Type': 'application/json'}
|
||||
})
|
||||
.then(function(r) { return r.json(); })
|
||||
.then(function(data) {
|
||||
var banner = document.getElementById('offline-banner');
|
||||
if (data.success) {
|
||||
banner.className = 'alert alert-success mb-0 rounded-0 border-0 border-bottom border-success';
|
||||
banner.innerHTML = '<div class="container-fluid"><i class="fas fa-check-circle me-2"></i>' +
|
||||
'<strong>Offline flag cleared.</strong> The bot will resume outbound sends. Refresh to confirm.</div>';
|
||||
} else {
|
||||
btn.disabled = false;
|
||||
btn.innerHTML = '<i class="fas fa-check me-1"></i>Clear Offline Flag';
|
||||
alert('Error: ' + (data.error || 'Unknown error — check server logs'));
|
||||
}
|
||||
})
|
||||
.catch(function() {
|
||||
btn.disabled = false;
|
||||
btn.innerHTML = '<i class="fas fa-check me-1"></i>Clear Offline Flag';
|
||||
alert('Network error — could not reach the server.');
|
||||
});
|
||||
}
|
||||
</script>
|
||||
{% endif %}
|
||||
|
||||
<!-- Main Content -->
|
||||
<div class="container-fluid mt-4">
|
||||
{% block content %}{% endblock %}
|
||||
|
||||
@@ -481,6 +481,222 @@ class TestProbeRadioHealth:
|
||||
assert mock_warn.called
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _BotAdminServer — admin HTTP API
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestBotAdminServer:
|
||||
"""Admin HTTP server: /api/admin/reload and /api/admin/health."""
|
||||
|
||||
def _make_bot_with_admin(self, tmp_path, port=15001):
|
||||
"""Write config with [Admin] enabled and return a bot + token."""
|
||||
token = "test-secret-token"
|
||||
config_file = tmp_path / "config.ini"
|
||||
db_path = tmp_path / "bot.db"
|
||||
config_file.write_text(
|
||||
f"""[Connection]
|
||||
connection_type = serial
|
||||
serial_port = /dev/ttyUSB0
|
||||
timeout = 30
|
||||
|
||||
[Bot]
|
||||
db_path = {db_path.as_posix()}
|
||||
prefix_bytes = 1
|
||||
|
||||
[Channels]
|
||||
monitor_channels = #general
|
||||
|
||||
[Admin]
|
||||
enabled = true
|
||||
port = {port}
|
||||
token = {token}
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
bot = MeshCoreBot(config_file=str(config_file))
|
||||
return bot, token, port
|
||||
|
||||
def test_admin_server_created_when_enabled(self, tmp_path):
|
||||
bot, _token, _port = self._make_bot_with_admin(tmp_path)
|
||||
assert bot._admin_server is not None
|
||||
|
||||
def test_admin_server_none_when_disabled(self, tmp_path):
|
||||
config_file = tmp_path / "config.ini"
|
||||
db_path = tmp_path / "bot.db"
|
||||
_write_config(config_file, db_path)
|
||||
bot = MeshCoreBot(config_file=str(config_file))
|
||||
assert bot._admin_server is None
|
||||
|
||||
def test_admin_server_none_when_token_missing(self, tmp_path):
|
||||
config_file = tmp_path / "config.ini"
|
||||
db_path = tmp_path / "bot.db"
|
||||
config_file.write_text(
|
||||
f"""[Connection]
|
||||
connection_type = serial
|
||||
serial_port = /dev/ttyUSB0
|
||||
timeout = 30
|
||||
|
||||
[Bot]
|
||||
db_path = {db_path.as_posix()}
|
||||
prefix_bytes = 1
|
||||
|
||||
[Channels]
|
||||
monitor_channels = #general
|
||||
|
||||
[Admin]
|
||||
enabled = true
|
||||
port = 15002
|
||||
token =
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
bot = MeshCoreBot(config_file=str(config_file))
|
||||
assert bot._admin_server is None
|
||||
|
||||
def test_reload_endpoint_success(self, tmp_path):
|
||||
"""POST /api/admin/reload returns 200 and success=true when reload succeeds."""
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
bot, token, port = self._make_bot_with_admin(tmp_path, port=15003)
|
||||
|
||||
with patch.object(bot, "reload_config", return_value=(True, "Config reloaded")):
|
||||
server = bot._admin_server
|
||||
server.start()
|
||||
time.sleep(0.4)
|
||||
|
||||
req = urllib.request.Request(
|
||||
f"http://127.0.0.1:{port}/api/admin/reload",
|
||||
method="POST",
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=5) as resp:
|
||||
import json
|
||||
body = json.loads(resp.read())
|
||||
assert body["success"] is True
|
||||
assert "Config reloaded" in body["message"]
|
||||
|
||||
def test_reload_endpoint_failure(self, tmp_path):
|
||||
"""POST /api/admin/reload returns 409 when reload is rejected."""
|
||||
import time
|
||||
import urllib.request
|
||||
from urllib.error import HTTPError
|
||||
|
||||
bot, token, port = self._make_bot_with_admin(tmp_path, port=15004)
|
||||
|
||||
with patch.object(bot, "reload_config", return_value=(False, "Radio settings changed")):
|
||||
server = bot._admin_server
|
||||
server.start()
|
||||
time.sleep(0.4)
|
||||
|
||||
req = urllib.request.Request(
|
||||
f"http://127.0.0.1:{port}/api/admin/reload",
|
||||
method="POST",
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
)
|
||||
with pytest.raises(HTTPError) as exc_info:
|
||||
urllib.request.urlopen(req, timeout=5)
|
||||
assert exc_info.value.code == 409
|
||||
|
||||
def test_reload_endpoint_rejects_bad_token(self, tmp_path):
|
||||
"""POST /api/admin/reload returns 401 with wrong token."""
|
||||
import time
|
||||
import urllib.request
|
||||
from urllib.error import HTTPError
|
||||
|
||||
bot, _token, port = self._make_bot_with_admin(tmp_path, port=15005)
|
||||
server = bot._admin_server
|
||||
server.start()
|
||||
time.sleep(0.4)
|
||||
|
||||
req = urllib.request.Request(
|
||||
f"http://127.0.0.1:{port}/api/admin/reload",
|
||||
method="POST",
|
||||
headers={"Authorization": "Bearer wrong-token"},
|
||||
)
|
||||
with pytest.raises(HTTPError) as exc_info:
|
||||
urllib.request.urlopen(req, timeout=5)
|
||||
assert exc_info.value.code == 401
|
||||
|
||||
def test_health_endpoint_returns_ok(self, tmp_path):
|
||||
"""GET /api/admin/health returns 200 and status=ok."""
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
bot, token, port = self._make_bot_with_admin(tmp_path, port=15006)
|
||||
server = bot._admin_server
|
||||
server.start()
|
||||
time.sleep(0.4)
|
||||
|
||||
req = urllib.request.Request(
|
||||
f"http://127.0.0.1:{port}/api/admin/health",
|
||||
method="GET",
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=5) as resp:
|
||||
import json
|
||||
body = json.loads(resp.read())
|
||||
assert body["status"] == "ok"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TestRadioOfflineState
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestRadioOfflineState:
|
||||
"""Tests for _record_send_failure / _record_send_success / is_radio_offline."""
|
||||
|
||||
def _make_bot(self, tmp_path: Path) -> "MeshCoreBot":
|
||||
config_file = tmp_path / "config.ini"
|
||||
db_path = tmp_path / "bot.db"
|
||||
_write_config(config_file, db_path)
|
||||
return MeshCoreBot(config_file=str(config_file))
|
||||
|
||||
def test_is_radio_offline_defaults_to_false(self, tmp_path):
|
||||
bot = self._make_bot(tmp_path)
|
||||
assert bot.is_radio_offline is False
|
||||
|
||||
def test_record_send_failure_increments_counter(self, tmp_path):
|
||||
bot = self._make_bot(tmp_path)
|
||||
bot._record_send_failure()
|
||||
assert bot._send_consecutive_failures == 1
|
||||
|
||||
def test_record_send_failure_sets_offline_at_threshold(self, tmp_path):
|
||||
bot = self._make_bot(tmp_path)
|
||||
# Default threshold is 3
|
||||
for _ in range(3):
|
||||
bot._record_send_failure()
|
||||
assert bot.is_radio_offline is True
|
||||
|
||||
def test_record_send_success_clears_offline_flag(self, tmp_path):
|
||||
bot = self._make_bot(tmp_path)
|
||||
bot._radio_offline = True
|
||||
bot._send_consecutive_failures = 5
|
||||
bot._record_send_success()
|
||||
assert bot.is_radio_offline is False
|
||||
assert bot._send_consecutive_failures == 0
|
||||
|
||||
def test_record_send_success_no_op_when_already_clean(self, tmp_path):
|
||||
bot = self._make_bot(tmp_path)
|
||||
bot._record_send_success() # must not raise
|
||||
assert bot.is_radio_offline is False
|
||||
|
||||
def test_offline_not_set_below_threshold(self, tmp_path):
|
||||
bot = self._make_bot(tmp_path)
|
||||
bot._record_send_failure()
|
||||
bot._record_send_failure()
|
||||
assert bot.is_radio_offline is False
|
||||
|
||||
def test_custom_threshold_from_config(self, tmp_path):
|
||||
bot = self._make_bot(tmp_path)
|
||||
bot.config.set('Bot', 'radio_offline_threshold', '2')
|
||||
bot._record_send_failure()
|
||||
assert bot.is_radio_offline is False
|
||||
bot._record_send_failure()
|
||||
assert bot.is_radio_offline is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper: create a coroutine that returns a fixed value
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -719,6 +719,8 @@ def _make_scheduler():
|
||||
config.set("Bot", "advert_interval_hours", "0")
|
||||
bot.config = config
|
||||
bot.main_event_loop = None
|
||||
bot.is_radio_zombie = False
|
||||
bot.is_radio_offline = False
|
||||
|
||||
# db_manager.connection() context manager
|
||||
conn_mock = MagicMock()
|
||||
@@ -1030,6 +1032,81 @@ class TestSendScheduledMessageWrapper:
|
||||
mock_loop.close.assert_called_once()
|
||||
mock_send.assert_called_once_with("general", "test message")
|
||||
|
||||
def test_suppressed_when_radio_zombie(self):
|
||||
scheduler = _make_scheduler()
|
||||
scheduler.bot.is_radio_zombie = True
|
||||
with patch("asyncio.run_coroutine_threadsafe") as mock_rct:
|
||||
scheduler.send_scheduled_message("general", "hi")
|
||||
mock_rct.assert_not_called()
|
||||
|
||||
def test_suppressed_when_radio_offline(self):
|
||||
scheduler = _make_scheduler()
|
||||
scheduler.bot.is_radio_offline = True
|
||||
with patch("asyncio.run_coroutine_threadsafe") as mock_rct:
|
||||
scheduler.send_scheduled_message("general", "hi")
|
||||
mock_rct.assert_not_called()
|
||||
|
||||
def test_records_success_on_successful_send(self):
|
||||
scheduler = _make_scheduler()
|
||||
mock_loop = Mock()
|
||||
mock_loop.is_running.return_value = True
|
||||
scheduler.bot.main_event_loop = mock_loop
|
||||
fake_future = Mock()
|
||||
fake_future.result.return_value = None
|
||||
with patch("asyncio.run_coroutine_threadsafe", return_value=fake_future):
|
||||
scheduler.send_scheduled_message("general", "hi")
|
||||
scheduler.bot._record_send_success.assert_called_once()
|
||||
|
||||
def test_records_failure_on_exception(self):
|
||||
scheduler = _make_scheduler()
|
||||
mock_loop = Mock()
|
||||
mock_loop.is_running.return_value = True
|
||||
scheduler.bot.main_event_loop = mock_loop
|
||||
fake_future = Mock()
|
||||
fake_future.result.side_effect = Exception("bang")
|
||||
with patch("asyncio.run_coroutine_threadsafe", return_value=fake_future):
|
||||
scheduler.send_scheduled_message("general", "hi")
|
||||
scheduler.bot._record_send_failure.assert_called_once()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TestSendIntervalAdvertOfflineGuard
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestSendIntervalAdvertOfflineGuard:
|
||||
"""Tests for send_interval_advert() radio-offline guard."""
|
||||
|
||||
def test_suppressed_when_radio_offline(self):
|
||||
scheduler = _make_scheduler()
|
||||
scheduler.bot.is_radio_offline = True
|
||||
with patch("asyncio.run_coroutine_threadsafe") as mock_rct:
|
||||
scheduler.send_interval_advert()
|
||||
mock_rct.assert_not_called()
|
||||
|
||||
def test_records_success_on_successful_send(self):
|
||||
scheduler = _make_scheduler()
|
||||
mock_loop = Mock()
|
||||
mock_loop.is_running.return_value = True
|
||||
scheduler.bot.main_event_loop = mock_loop
|
||||
fake_future = Mock()
|
||||
fake_future.result.return_value = None
|
||||
with patch("asyncio.run_coroutine_threadsafe", return_value=fake_future):
|
||||
scheduler.send_interval_advert()
|
||||
scheduler.bot._record_send_success.assert_called_once()
|
||||
|
||||
def test_records_failure_on_exception(self):
|
||||
from concurrent.futures import TimeoutError as FuturesTimeoutError
|
||||
scheduler = _make_scheduler()
|
||||
mock_loop = Mock()
|
||||
mock_loop.is_running.return_value = True
|
||||
scheduler.bot.main_event_loop = mock_loop
|
||||
fake_future = Mock()
|
||||
fake_future.result.side_effect = FuturesTimeoutError()
|
||||
with patch("asyncio.run_coroutine_threadsafe", return_value=fake_future):
|
||||
scheduler.send_interval_advert()
|
||||
scheduler.bot._record_send_failure.assert_called_once()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TestRunDataRetention
|
||||
@@ -1242,6 +1319,8 @@ def _make_sched_with_logger(mock_logger):
|
||||
bot.logger = mock_logger
|
||||
bot.config = ConfigParser()
|
||||
bot.config.add_section("Bot")
|
||||
bot.is_radio_zombie = False # ensure zombie guard does not suppress sends
|
||||
bot.is_radio_offline = False # ensure offline guard does not suppress sends
|
||||
return MessageScheduler(bot)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user