From 93f73a15a2f71589f45b50df6decd5c70edb0905 Mon Sep 17 00:00:00 2001 From: Stacy Olivas Date: Tue, 17 Mar 2026 17:44:14 -0700 Subject: [PATCH] =?UTF-8?q?feat:=20web=20viewer=20=E2=80=94=20auth,=20cont?= =?UTF-8?q?act=20management,=20live=20streaming,=20config,=20maintenance,?= =?UTF-8?q?=20and=20backup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Auth (BUG-001): - Optional password via web_viewer_password in [Web_Viewer]; /login and /logout; Flask session guard on all routes and SocketIO handlers Contact management and export: - Star contacts of any type; purge-preview + purge inactive contacts - GET /api/export/contacts and /api/export/paths: CSV/JSON with time-range Config tab and maintenance: - /config page: SMTP, log rotation, DB backup settings in bot_metadata - Nightly email digest (uptime, contacts, DB size, log errors); SMTP timeout=30s; pre-rotation log attachment hook - GET /api/maintenance/status: Maintenance Status card DB backup, restore, and purge: - POST /api/maintenance/backup_now; GET /api/maintenance/list_backups; POST /api/maintenance/restore (SQLite magic-byte validation) - POST /api/maintenance/purge: remove rows older than threshold - Scheduled backups: daily/weekly/manual with retention pruning - Config save validates db_backup_dir exists; 400 on missing path Live streaming and realtime monitoring: - Live Activity panel: colour-coded SocketIO feed with pause/clear - capture_channel_message() feeds packet_stream; message_data event - /realtime page: three independent stream panels; [#channel] prefix - /logs page: subscribe_logs/log_line; log-tail thread; level colouring - History replay: last 50/50/200 items on connect - Werkzeug 3.1 WebSocket fix: _apply_werkzeug_websocket_fix() - BUG-029: db_path resolved via config_base = Path(config_path).parent; stored as self._config_base; dead _get_db_path() removed Scroll/filter controls and connected agents: - Scroll-to-top/bottom on Live Activity and all realtime panels - Type-filter checkboxes (Packets/Commands/Messages) with applyFilters() - GET /api/connected_clients: agent count clickable; Bootstrap modal --- modules/web_viewer/__init__.py | 2 +- modules/web_viewer/app.py | 2625 +++++++++++++------- modules/web_viewer/integration.py | 298 +-- modules/web_viewer/templates/base.html | 10 + modules/web_viewer/templates/config.html | 585 +++++ modules/web_viewer/templates/contacts.html | 157 +- modules/web_viewer/templates/index.html | 107 + modules/web_viewer/templates/login.html | 34 + modules/web_viewer/templates/logs.html | 238 ++ modules/web_viewer/templates/radio.html | 130 +- modules/web_viewer/templates/realtime.html | 119 +- tests/test_web_viewer.py | 1621 ++++++++++++ 12 files changed, 4833 insertions(+), 1093 deletions(-) create mode 100644 modules/web_viewer/templates/config.html create mode 100644 modules/web_viewer/templates/login.html create mode 100644 modules/web_viewer/templates/logs.html create mode 100644 tests/test_web_viewer.py diff --git a/modules/web_viewer/__init__.py b/modules/web_viewer/__init__.py index 2f23bd9..9379c42 100644 --- a/modules/web_viewer/__init__.py +++ b/modules/web_viewer/__init__.py @@ -6,6 +6,6 @@ allowing users to visualize bot databases and monitor real-time data. """ from .app import BotDataViewer -from .integration import WebViewerIntegration, BotIntegration +from .integration import BotIntegration, WebViewerIntegration __all__ = ['BotDataViewer', 'WebViewerIntegration', 'BotIntegration'] diff --git a/modules/web_viewer/app.py b/modules/web_viewer/app.py index 69e4e15..0e6adf1 100644 --- a/modules/web_viewer/app.py +++ b/modules/web_viewer/app.py @@ -4,55 +4,101 @@ MeshCore Bot Data Viewer Bot montoring web interface using Flask-SocketIO 5.x """ -import sqlite3 -import json -import time import configparser +import json import logging -import subprocess -import threading -from contextlib import contextmanager, closing -from datetime import datetime, timedelta, date -from flask import Flask, render_template, jsonify, request, send_from_directory, make_response -from flask_socketio import SocketIO, emit, join_room, leave_room, disconnect -from pathlib import Path import os +import sqlite3 +import subprocess import sys -from typing import Dict, Any, Optional, List +import threading +import time +from contextlib import closing, contextmanager, suppress +from datetime import datetime +from pathlib import Path +from typing import Any, Optional + +from flask import ( + Flask, + Response, + jsonify, + make_response, + redirect, + render_template, + request, + send_from_directory, + session, + url_for, +) +from flask_socketio import SocketIO, disconnect, emit + + +def _apply_werkzeug_websocket_fix() -> None: + """Patch SimpleWebSocketWSGI to call start_response after WebSocket teardown. + + python-engineio's SimpleWebSocketWSGI.__call__ handles the WebSocket + session directly on the raw socket and returns [] without ever calling + start_response. Werkzeug then calls write(b"") to flush an empty body, + which triggers ``AssertionError: write() before start_response``. + + The fix calls start_response after the handler returns so that status_set + is not None when write(b"") runs. The subsequent attempt to write HTTP + headers to the already-closed socket raises BrokenPipeError, which Werkzeug + classifies as a dropped connection and silently ignores. + """ + try: + from engineio.async_drivers import _websocket_wsgi # noqa: PLC0415 + _orig_call = _websocket_wsgi.SimpleWebSocketWSGI.__call__ + + def _patched_call(self, environ, start_response): # type: ignore[misc] + result = _orig_call(self, environ, start_response) + try: + start_response('200 OK', [('Content-Length', '0')]) + except Exception: # noqa: BLE001 + pass + return result + + _websocket_wsgi.SimpleWebSocketWSGI.__call__ = _patched_call + except (ImportError, AttributeError): + pass + + +_apply_werkzeug_websocket_fix() # Add the project root to the path so we can import bot components project_root = os.path.join(os.path.dirname(__file__), '..', '..') sys.path.insert(0, project_root) -from modules.db_manager import DBManager from modules.repeater_manager import RepeaterManager -from modules.utils import resolve_path, calculate_distance +from modules.utils import calculate_distance, resolve_path + class BotDataViewer: """Complete web interface using Flask-SocketIO 5.x best practices""" - + def __init__(self, db_path="meshcore_bot.db", repeater_db_path=None, config_path="config.ini"): # Setup comprehensive logging self._setup_logging() - + # Set bot root directory (project root) for path validation # This is the directory containing the modules folder self.bot_root = Path(os.path.join(os.path.dirname(__file__), '..', '..')).resolve() # Resolve relative config path so viewer finds config when started as subprocess (cwd may differ) if not os.path.isabs(config_path): config_path = str(self.bot_root / config_path) - + self.app = Flask( - __name__, + __name__, template_folder=os.path.join(os.path.dirname(__file__), 'templates'), static_folder=os.path.join(os.path.dirname(__file__), 'static'), static_url_path='/static' ) - self.app.config['SECRET_KEY'] = 'meshcore_bot_viewer_secret' - + import secrets as _secrets + self.app.config['SECRET_KEY'] = _secrets.token_hex(32) + # Flask-SocketIO configuration following 5.x best practices self.socketio = SocketIO( - self.app, + self.app, cors_allowed_origins="*", max_http_buffer_size=1000000, # 1MB buffer limit ping_timeout=5, # 5 second ping timeout (Flask-SocketIO 5.x default) @@ -61,23 +107,23 @@ class BotDataViewer: engineio_logger=False, # Disable EngineIO logging async_mode='threading' # Use threading for better stability ) - + self.repeater_db_path = repeater_db_path - + # Connection management using Flask-SocketIO built-ins self.connected_clients = {} # Track client metadata self._clients_lock = threading.Lock() # Thread safety for connected_clients self.max_clients = 10 - + # Database connection pooling with thread safety self._db_connection = None self._db_lock = threading.Lock() self._db_last_used = 0 self._db_timeout = 300 # 5 minutes connection timeout - + # Load configuration self.config = self._load_config(config_path) - + # Use [Bot] db_path when [Web_Viewer] db_path is unset bot_db = self.config.get('Bot', 'db_path', fallback='meshcore_bot.db') if (self.config.has_section('Web_Viewer') and self.config.has_option('Web_Viewer', 'db_path') @@ -86,42 +132,55 @@ class BotDataViewer: else: use_db = bot_db self.db_path = str(resolve_path(use_db, self.bot_root)) - + + # Optional password authentication for web viewer (BUG-001) + self.web_viewer_password = self.config.get('Web_Viewer', 'web_viewer_password', fallback='').strip() + if self.web_viewer_password: + self.logger.info("Web viewer authentication enabled") + else: + self.logger.warning( + "Web viewer has NO authentication. Set web_viewer_password in [Web_Viewer] config " + "or restrict access with host = 127.0.0.1 and firewall rules." + ) + # Version info for footer (tag or branch/commit/date); computed once at startup self._version_info = self._get_version_info() # Setup template context processor for global template variables self._setup_template_context() - + # Initialize databases self._init_databases() - + # Setup routes and SocketIO handlers self._setup_routes() self._setup_socketio_handlers() - + # Start database polling for real-time data self._start_database_polling() - + + # Start log file tailing for /logs page + self._start_log_tailing() + # Start periodic cleanup self._start_cleanup_scheduler() - + self.logger.info("BotDataViewer initialized with Flask-SocketIO 5.x best practices") - + def _setup_logging(self): """Setup comprehensive logging with rotation""" from logging.handlers import RotatingFileHandler - + # Create logs directory if it doesn't exist os.makedirs('logs', exist_ok=True) - + # Get or create logger (don't use basicConfig as it may conflict with existing logging) self.logger = logging.getLogger('modern_web_viewer') self.logger.setLevel(logging.DEBUG) - + # Remove existing handlers to avoid duplicates self.logger.handlers.clear() - + # Create rotating file handler (max 5MB per file, keep 3 backups) file_handler = RotatingFileHandler( 'logs/web_viewer_modern.log', @@ -133,27 +192,27 @@ class BotDataViewer: file_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') file_handler.setFormatter(file_formatter) self.logger.addHandler(file_handler) - + # Create console handler console_handler = logging.StreamHandler() console_handler.setLevel(logging.INFO) console_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') console_handler.setFormatter(console_formatter) self.logger.addHandler(console_handler) - + # Prevent propagation to root logger to avoid duplicate messages self.logger.propagate = False - + self.logger.info("Web viewer logging initialized with rotation (5MB max, 3 backups)") - + def _load_config(self, config_path): """Load configuration from file""" config = configparser.ConfigParser() if os.path.exists(config_path): config.read(config_path) return config - - def _get_version_info(self) -> Dict[str, Optional[str]]: + + def _get_version_info(self) -> dict[str, Optional[str]]: """Get version info for footer: tag if on a tag, else branch, commit hash and date. Checks MESHCORE_BOT_VERSION env (Docker/build), then .version_info, then git. Never raises.""" out = {"tag": None, "branch": None, "commit": None, "date": None} @@ -165,7 +224,7 @@ class BotDataViewer: version_file = self.bot_root / ".version_info" try: if version_file.is_file(): - with open(version_file, "r") as f: + with open(version_file) as f: data = json.load(f) # Installer/tag installs write installer_version (often the tag name) tag = data.get("installer_version") or data.get("tag") @@ -175,7 +234,7 @@ class BotDataViewer: except (OSError, json.JSONDecodeError, KeyError): pass try: - def run(cmd: List[str]) -> Optional[str]: + def run(cmd: list[str]) -> Optional[str]: args = ["git", "-C", str(self.bot_root)] + cmd result = subprocess.run( args, capture_output=True, text=True, timeout=5 @@ -224,16 +283,16 @@ class BotDataViewer: bot_name = (self.config.get('Bot', 'bot_name', fallback='MeshCore Bot') or '').strip() or 'MeshCore Bot' except (configparser.NoSectionError, configparser.NoOptionError): bot_name = 'MeshCore Bot' - return dict( - greeter_enabled=greeter_enabled, - feed_manager_enabled=feed_manager_enabled, - bot_name=bot_name, - version_info=version_info, - ) + return { + 'greeter_enabled': greeter_enabled, + 'feed_manager_enabled': feed_manager_enabled, + 'bot_name': bot_name, + 'version_info': version_info, + } except Exception as e: self.logger.exception("Template context processor failed: %s", e) - return dict(greeter_enabled=False, feed_manager_enabled=False, bot_name='MeshCore Bot', version_info=version_info) - + return {'greeter_enabled': False, 'feed_manager_enabled': False, 'bot_name': 'MeshCore Bot', 'version_info': version_info} + def _get_db_path(self): """Get the database path, falling back to [Bot] db_path if [Web_Viewer] db_path is unset""" # Use [Bot] db_path when [Web_Viewer] db_path is unset @@ -244,7 +303,7 @@ class BotDataViewer: else: use_db = bot_db return str(resolve_path(use_db, self.bot_root)) - + def _init_databases(self): """Initialize database connections""" try: @@ -256,25 +315,25 @@ class BotDataViewer: self.logger = logger self.config = config self.db_manager = db_manager - + # Create DBManager first minimal_bot = MinimalBot(self.logger, self.config) self.db_manager = DBManager(minimal_bot, self.db_path) - + # Now set db_manager on the minimal bot for RepeaterManager minimal_bot.db_manager = self.db_manager - + # Initialize repeater manager for geocoding functionality self.repeater_manager = RepeaterManager(minimal_bot) - + # Initialize mesh graph for path resolution (uses same logic as path command) from modules.mesh_graph import MeshGraph minimal_bot.mesh_graph = MeshGraph(minimal_bot) self.mesh_graph = minimal_bot.mesh_graph - + # Initialize packet_stream table for real-time monitoring self._init_packet_stream_table() - + # Store database paths for direct connection self.db_path = self.db_path self.repeater_db_path = self.repeater_db_path @@ -282,13 +341,13 @@ class BotDataViewer: except Exception as e: self.logger.error(f"Failed to initialize databases: {e}") raise - + def _init_packet_stream_table(self): """Initialize the packet_stream table in the web viewer database (same as [Bot] db_path by default).""" try: with closing(sqlite3.connect(self.db_path, timeout=60)) as conn: cursor = conn.cursor() - + # Create packet_stream table with schema matching the INSERT statements cursor.execute(''' CREATE TABLE IF NOT EXISTS packet_stream ( @@ -298,33 +357,33 @@ class BotDataViewer: type TEXT NOT NULL ) ''') - + # Create index on timestamp for faster queries cursor.execute(''' - CREATE INDEX IF NOT EXISTS idx_packet_stream_timestamp + CREATE INDEX IF NOT EXISTS idx_packet_stream_timestamp ON packet_stream(timestamp) ''') - + # Create index on type for filtering by type cursor.execute(''' - CREATE INDEX IF NOT EXISTS idx_packet_stream_type + CREATE INDEX IF NOT EXISTS idx_packet_stream_type ON packet_stream(type) ''') - + # Enable WAL for better concurrent access (bot + web viewer use same DB) try: cursor.execute('PRAGMA journal_mode=WAL') except sqlite3.OperationalError: pass # Ignore if locked; WAL may already be set - + conn.commit() - + self.logger.info(f"Initialized packet_stream table in {self.db_path}") - + except Exception as e: self.logger.error(f"Failed to initialize packet_stream table: {e}") # Don't raise - allow web viewer to continue even if table init fails - + def _get_db_connection(self): """Get database connection - create new connection for each request to avoid threading issues""" try: @@ -334,7 +393,7 @@ class BotDataViewer: except Exception as e: self.logger.error(f"Failed to create database connection: {e}") raise - + @contextmanager def _with_db_connection(self): """Context manager that yields a configured connection and closes it on exit. @@ -346,23 +405,23 @@ class BotDataViewer: yield conn finally: conn.close() - - def _resolve_path(self, path_input: str) -> Dict[str, Any]: + + def _resolve_path(self, path_input: str) -> dict[str, Any]: """Resolve a hex path to repeater names and locations using the same algorithm as PathCommand. - + This method replicates the path command's logic to ensure consistency between the bot's path command and the web viewer's path resolution. - + Args: path_input: Hex path string (e.g., "7e,01,86" or "7e 01 86") - + Returns: Dictionary with node_ids, repeaters list, and valid flag """ - import re import math + import re from datetime import datetime - + # Check if db_manager is available if not hasattr(self, 'db_manager') or not self.db_manager: return { @@ -371,7 +430,7 @@ class BotDataViewer: 'valid': False, 'error': 'Database manager not initialized' } - + # Parse hex input - same logic as PathCommand._decode_path # Handle both comma/space-separated and continuous hex strings (e.g., "8601a5") prefix_hex_chars = self.config.getint('Bot', 'prefix_bytes', fallback=1) * 2 @@ -392,7 +451,7 @@ class BotDataViewer: if not hex_matches and prefix_hex_chars > 2: hex_pattern = r'[0-9a-fA-F]{2}' hex_matches = re.findall(hex_pattern, path_input) - + if not hex_matches: return { 'node_ids': [], @@ -400,15 +459,15 @@ class BotDataViewer: 'valid': False, 'error': 'No valid hex values found' } - + node_ids = [match.upper() for match in hex_matches] - + # Load all Path_Command config values (same as PathCommand.__init__) # Geographic guessing geographic_guessing_enabled = False bot_latitude = None bot_longitude = None - + try: if self.config.has_section('Bot'): lat = self.config.getfloat('Bot', 'bot_latitude', fallback=None) @@ -419,22 +478,22 @@ class BotDataViewer: geographic_guessing_enabled = True except Exception: pass - + # Path command settings proximity_method = self.config.get('Path_Command', 'proximity_method', fallback='simple') - path_proximity_fallback = self.config.getboolean('Path_Command', 'path_proximity_fallback', fallback=True) + self.config.getboolean('Path_Command', 'path_proximity_fallback', fallback=True) max_proximity_range = self.config.getfloat('Path_Command', 'max_proximity_range', fallback=200.0) max_repeater_age_days = self.config.getint('Path_Command', 'max_repeater_age_days', fallback=14) - + recency_weight = self.config.getfloat('Path_Command', 'recency_weight', fallback=0.4) recency_weight = max(0.0, min(1.0, recency_weight)) proximity_weight = 1.0 - recency_weight - + recency_decay_half_life_hours = self.config.getfloat('Path_Command', 'recency_decay_half_life_hours', fallback=12.0) - + # Check for preset first, then apply individual settings (preset can be overridden) preset = self.config.get('Path_Command', 'path_selection_preset', fallback='balanced').lower() - + # Apply preset defaults, then individual settings override if preset == 'geographic': preset_graph_confidence_threshold = 0.5 @@ -451,10 +510,10 @@ class BotDataViewer: preset_distance_threshold = 30.0 preset_distance_penalty = 0.3 preset_final_hop_weight = 0.25 - + graph_based_validation = self.config.getboolean('Path_Command', 'graph_based_validation', fallback=True) min_edge_observations = self.config.getint('Path_Command', 'min_edge_observations', fallback=3) - + graph_use_bidirectional = self.config.getboolean('Path_Command', 'graph_use_bidirectional', fallback=True) graph_use_hop_position = self.config.getboolean('Path_Command', 'graph_use_hop_position', fallback=True) graph_multi_hop_enabled = self.config.getboolean('Path_Command', 'graph_multi_hop_enabled', fallback=True) @@ -471,7 +530,7 @@ class BotDataViewer: graph_zero_hop_bonus = self.config.getfloat('Path_Command', 'graph_zero_hop_bonus', fallback=0.4) graph_zero_hop_bonus = max(0.0, min(1.0, graph_zero_hop_bonus)) graph_prefer_stored_keys = self.config.getboolean('Path_Command', 'graph_prefer_stored_keys', fallback=True) - + # Final hop proximity settings for graph selection # Defaults based on LoRa ranges: typical < 30km, long up to 200km, very close < 10km graph_final_hop_proximity_enabled = self.config.getboolean('Path_Command', 'graph_final_hop_proximity_enabled', fallback=True) @@ -486,18 +545,18 @@ class BotDataViewer: graph_path_validation_max_bonus = self.config.getfloat('Path_Command', 'graph_path_validation_max_bonus', fallback=0.3) graph_path_validation_max_bonus = max(0.0, min(1.0, graph_path_validation_max_bonus)) graph_path_validation_obs_divisor = self.config.getfloat('Path_Command', 'graph_path_validation_obs_divisor', fallback=50.0) - + star_bias_multiplier = self.config.getfloat('Path_Command', 'star_bias_multiplier', fallback=2.5) star_bias_multiplier = max(1.0, star_bias_multiplier) - + # Helper method to calculate recency scores (same as PathCommand._calculate_recency_weighted_scores) def calculate_recency_weighted_scores(repeaters): scored_repeaters = [] now = datetime.now() - + for repeater in repeaters: most_recent_time = None - + for field in ['last_heard', 'last_advert_timestamp', 'last_seen']: value = repeater.get(field) if value: @@ -510,59 +569,59 @@ class BotDataViewer: most_recent_time = dt except: pass - + if most_recent_time is None: recency_score = 0.1 else: hours_ago = (now - most_recent_time).total_seconds() / 3600.0 recency_score = math.exp(-hours_ago / recency_decay_half_life_hours) recency_score = max(0.0, min(1.0, recency_score)) - + scored_repeaters.append((repeater, recency_score)) - + scored_repeaters.sort(key=lambda x: x[1], reverse=True) return scored_repeaters - + # Helper to get node location (same as PathCommand._get_node_location) def get_node_location(node_id): try: if max_repeater_age_days > 0: - query = ''' - SELECT latitude, longitude FROM complete_contact_tracking + query = f''' + SELECT latitude, longitude FROM complete_contact_tracking WHERE public_key LIKE ? AND latitude IS NOT NULL AND longitude IS NOT NULL AND latitude != 0 AND longitude != 0 AND role IN ('repeater', 'roomserver') AND ( - (last_advert_timestamp IS NOT NULL AND last_advert_timestamp >= datetime('now', '-{} days')) - OR (last_advert_timestamp IS NULL AND last_heard >= datetime('now', '-{} days')) + (last_advert_timestamp IS NOT NULL AND last_advert_timestamp >= datetime('now', '-{max_repeater_age_days} days')) + OR (last_advert_timestamp IS NULL AND last_heard >= datetime('now', '-{max_repeater_age_days} days')) ) ORDER BY is_starred DESC, COALESCE(last_advert_timestamp, last_heard) DESC LIMIT 1 - '''.format(max_repeater_age_days, max_repeater_age_days) + ''' else: query = ''' - SELECT latitude, longitude FROM complete_contact_tracking + SELECT latitude, longitude FROM complete_contact_tracking WHERE public_key LIKE ? AND latitude IS NOT NULL AND longitude IS NOT NULL AND latitude != 0 AND longitude != 0 AND role IN ('repeater', 'roomserver') ORDER BY is_starred DESC, COALESCE(last_advert_timestamp, last_heard) DESC LIMIT 1 ''' - + results = self.db_manager.execute_query(query, (f"{node_id}%",)) if results: return (results[0]['latitude'], results[0]['longitude']) return None except Exception: return None - + # Helper for simple proximity selection (same as PathCommand._select_by_simple_proximity) def select_by_simple_proximity(repeaters_with_location): scored_repeaters = calculate_recency_weighted_scores(repeaters_with_location) min_recency_threshold = 0.01 scored_repeaters = [(r, score) for r, score in scored_repeaters if score >= min_recency_threshold] - + if not scored_repeaters: return None, 0.0 - + if len(scored_repeaters) == 1: repeater, recency_score = scored_repeaters[0] distance = calculate_distance(bot_latitude, bot_longitude, repeater['latitude'], repeater['longitude']) @@ -570,28 +629,28 @@ class BotDataViewer: return None, 0.0 base_confidence = 0.4 + (recency_score * 0.5) return repeater, base_confidence - + combined_scores = [] for repeater, recency_score in scored_repeaters: distance = calculate_distance(bot_latitude, bot_longitude, repeater['latitude'], repeater['longitude']) if max_proximity_range > 0 and distance > max_proximity_range: continue - + normalized_distance = min(distance / 1000.0, 1.0) proximity_score = 1.0 - normalized_distance combined_score = (recency_score * recency_weight) + (proximity_score * proximity_weight) - + if repeater.get('is_starred', False): combined_score *= star_bias_multiplier - + combined_scores.append((combined_score, distance, repeater)) - + if not combined_scores: return None, 0.0 - + combined_scores.sort(key=lambda x: x[0], reverse=True) best_score, best_distance, best_repeater = combined_scores[0] - + if len(combined_scores) == 1: confidence = 0.4 + (best_score * 0.5) else: @@ -605,73 +664,73 @@ class BotDataViewer: confidence = 0.7 else: confidence = 0.5 - + return best_repeater, confidence - + # Helper for path proximity (simplified - for web viewer we'll use simple proximity) def select_by_path_proximity(repeaters_with_location, node_id, path_context, sender_location): scored_repeaters = calculate_recency_weighted_scores(repeaters_with_location) min_recency_threshold = 0.01 recent_repeaters = [r for r, score in scored_repeaters if score >= min_recency_threshold] - + if not recent_repeaters: return None, 0.0 - + current_index = path_context.index(node_id) if node_id in path_context else -1 if current_index == -1: return None, 0.0 - + is_last_repeater = (current_index == len(path_context) - 1) if is_last_repeater and geographic_guessing_enabled and bot_latitude and bot_longitude: bot_location = (bot_latitude, bot_longitude) return select_by_single_proximity(recent_repeaters, bot_location, "bot") - + # For other positions, use simple proximity return select_by_simple_proximity(recent_repeaters) - + # Helper for single proximity (same as PathCommand._select_by_single_proximity) def select_by_single_proximity(repeaters, reference_location, direction): scored_repeaters = calculate_recency_weighted_scores(repeaters) min_recency_threshold = 0.01 scored_repeaters = [(r, score) for r, score in scored_repeaters if score >= min_recency_threshold] - + if not scored_repeaters: return None, 0.0 - + if direction == "bot" or direction == "sender": proximity_weight_local = 1.0 recency_weight_local = 0.0 else: proximity_weight_local = proximity_weight recency_weight_local = recency_weight - + best_repeater = None best_combined_score = 0.0 - + for repeater, recency_score in scored_repeaters: distance = calculate_distance(reference_location[0], reference_location[1], repeater['latitude'], repeater['longitude']) - + if max_proximity_range > 0 and distance > max_proximity_range: continue - + normalized_distance = min(distance / 1000.0, 1.0) proximity_score = 1.0 - normalized_distance combined_score = (recency_score * recency_weight_local) + (proximity_score * proximity_weight_local) - + if repeater.get('is_starred', False): combined_score *= star_bias_multiplier - + if combined_score > best_combined_score: best_combined_score = combined_score best_repeater = repeater - + if best_repeater: confidence = 0.4 + (best_combined_score * 0.5) return best_repeater, confidence - + return None, 0.0 - + # Helper for graph-based selection (same as PathCommand._select_repeater_by_graph) # When path was decoded with 2-byte or 3-byte hops, node_id/path_context have 4 or 6 hex chars; # use path_prefix_hex_chars for candidate matching and normalize to graph_n for edge lookups. @@ -778,7 +837,7 @@ class BotDataViewer: if candidate_to_next_edge and candidate_to_next_edge.get('geographic_distance'): distance = candidate_to_next_edge.get('geographic_distance') max_distance = max(max_distance, distance) - + # Apply penalty if distance exceeds reasonable hop distance if max_distance > graph_max_reasonable_hop_distance_km: excess_distance = max_distance - graph_max_reasonable_hop_distance_km @@ -797,14 +856,14 @@ class BotDataViewer: if bot_latitude is not None and bot_longitude is not None: repeater_lat = repeater.get('latitude') repeater_lon = repeater.get('longitude') - + if repeater_lat is not None and repeater_lon is not None: # Calculate distance to bot distance = calculate_distance( bot_latitude, bot_longitude, repeater_lat, repeater_lon ) - + # Apply max distance threshold if configured if graph_final_hop_max_distance > 0 and distance > graph_final_hop_max_distance: # Beyond max distance - significantly penalize this candidate for final hop @@ -814,7 +873,7 @@ class BotDataViewer: # Use configurable normalization distance (default 500km for more aggressive scoring) normalized_distance = min(distance / graph_final_hop_proximity_normalization_km, 1.0) proximity_score = 1.0 - normalized_distance - + # For final hop, use a higher effective weight to ensure proximity matters more # The configured weight is a minimum; we boost it for very close repeaters effective_weight = graph_final_hop_proximity_weight @@ -824,10 +883,10 @@ class BotDataViewer: elif distance < graph_final_hop_close_threshold_km: # Close - moderate boost effective_weight = min(0.5, graph_final_hop_proximity_weight * 1.5) - + # Combine with graph score using effective weight candidate_score = candidate_score * (1.0 - effective_weight) + proximity_score * effective_weight - + # Path validation bonus: Check if candidate's stored paths match the current path context path_validation_bonus = 0.0 if candidate_public_key and len(path_context) > 1: @@ -841,19 +900,19 @@ class BotDataViewer: LIMIT 10 ''' stored_paths = self.db_manager.execute_query(query, (candidate_public_key,)) - + if stored_paths: # Build the path we're decoding (full path context) decoded_path_hex = ''.join([node.lower() for node in path_context]) # Build the path prefix up to (but not including) the current node # This helps match paths where the candidate appears at the same position path_prefix_up_to_current = ''.join([node.lower() for node in path_context[:current_index]]) - + # Check if any stored path shares common segments with decoded path for stored_path in stored_paths: stored_hex = stored_path.get('path_hex', '').lower() obs_count = stored_path.get('observation_count', 1) - + if stored_hex: # Chunk size: use stored bytes_per_hop (multi-byte path support) n = (stored_path.get('bytes_per_hop') or 1) * 2 @@ -863,7 +922,7 @@ class BotDataViewer: if (len(stored_hex) % n) != 0: stored_nodes = [stored_hex[i:i+2] for i in range(0, len(stored_hex), 2)] decoded_nodes = path_context if path_context else [decoded_path_hex[i:i+n] for i in range(0, len(decoded_path_hex), n)] - + # Count how many nodes appear in both paths (in order) common_segments = 0 min_len = min(len(stored_nodes), len(decoded_nodes)) @@ -872,7 +931,7 @@ class BotDataViewer: common_segments += 1 else: break - + # Also check if stored path starts with the same prefix as the decoded path up to current position # This is important for matching paths where the candidate appears at the same position prefix_match = False @@ -881,7 +940,7 @@ class BotDataViewer: # The stored path has the same prefix, and the candidate appears at the same position # This is a strong indicator of a match prefix_match = True - + # Bonus based on common segments and observation count if common_segments >= 2 or prefix_match: # Stronger bonus for prefix matches (indicates same path structure) @@ -897,56 +956,56 @@ class BotDataViewer: break # Strong match found, no need to check more except Exception: pass - + # Add path validation bonus to graph score candidate_score = min(1.0, candidate_score + path_validation_bonus) - + if repeater.get('is_starred', False): candidate_score *= star_bias_multiplier - + if candidate_score > best_score: best_score = candidate_score best_repeater = repeater best_method = method - + if best_repeater and best_score > 0.0: confidence = min(1.0, best_score) if best_score <= 1.0 else 0.95 + (min(0.05, (best_score - 1.0) / star_bias_multiplier)) return best_repeater, confidence, best_method or 'graph' - + return None, 0.0, None - + # Main resolution logic (same as PathCommand._lookup_repeater_names) repeater_info = {} - + try: for node_id in node_ids: # Query database for matching repeaters if max_repeater_age_days > 0: - query = ''' - SELECT name, public_key, device_type, last_heard, last_heard as last_seen, + query = f''' + SELECT name, public_key, device_type, last_heard, last_heard as last_seen, last_advert_timestamp, latitude, longitude, city, state, country, advert_count, signal_strength, hop_count, role, is_starred - FROM complete_contact_tracking + FROM complete_contact_tracking WHERE public_key LIKE ? AND role IN ('repeater', 'roomserver') AND ( - (last_advert_timestamp IS NOT NULL AND last_advert_timestamp >= datetime('now', '-{} days')) - OR (last_advert_timestamp IS NULL AND last_heard >= datetime('now', '-{} days')) + (last_advert_timestamp IS NOT NULL AND last_advert_timestamp >= datetime('now', '-{max_repeater_age_days} days')) + OR (last_advert_timestamp IS NULL AND last_heard >= datetime('now', '-{max_repeater_age_days} days')) ) ORDER BY COALESCE(last_advert_timestamp, last_heard) DESC - '''.format(max_repeater_age_days, max_repeater_age_days) + ''' else: query = ''' - SELECT name, public_key, device_type, last_heard, last_heard as last_seen, + SELECT name, public_key, device_type, last_heard, last_heard as last_seen, last_advert_timestamp, latitude, longitude, city, state, country, advert_count, signal_strength, hop_count, role, is_starred - FROM complete_contact_tracking + FROM complete_contact_tracking WHERE public_key LIKE ? AND role IN ('repeater', 'roomserver') ORDER BY COALESCE(last_advert_timestamp, last_heard) DESC ''' - + prefix_pattern = f"{node_id}%" results = self.db_manager.execute_query(query, (prefix_pattern,)) - + if results: repeaters_data = [ { @@ -966,11 +1025,11 @@ class BotDataViewer: 'is_starred': bool(row.get('is_starred', 0)) } for row in results ] - + scored_repeaters = calculate_recency_weighted_scores(repeaters_data) min_recency_threshold = 0.01 recent_repeaters = [r for r, score in scored_repeaters if score >= min_recency_threshold] - + if len(recent_repeaters) > 1: # Multiple matches - use graph and geographic selection graph_repeater = None @@ -978,12 +1037,12 @@ class BotDataViewer: selection_method = None geo_repeater = None geo_confidence = 0.0 - + if graph_based_validation and hasattr(self, 'mesh_graph') and self.mesh_graph: graph_repeater, graph_confidence, selection_method = select_repeater_by_graph( recent_repeaters, node_id, node_ids ) - + if geographic_guessing_enabled: if proximity_method == 'path': geo_repeater, geo_confidence = select_by_path_proximity( @@ -991,16 +1050,16 @@ class BotDataViewer: ) else: geo_repeater, geo_confidence = select_by_simple_proximity(recent_repeaters) - + # Combine or choose selected_repeater = None confidence = 0.0 final_method = None - + if graph_geographic_combined and graph_repeater and geo_repeater: graph_pubkey = graph_repeater.get('public_key', '') geo_pubkey = geo_repeater.get('public_key', '') - + if graph_pubkey and geo_pubkey and graph_pubkey == geo_pubkey: combined_confidence = ( graph_confidence * graph_geographic_weight + @@ -1022,7 +1081,7 @@ class BotDataViewer: # For final hop, prefer geographic selection if available and reasonable # The final hop should be close to the bot, so geographic proximity is very important is_final_hop = (node_id == node_ids[-1] if node_ids else False) - + if is_final_hop and geo_repeater and geo_confidence >= 0.6: # For final hop, prefer geographic if it has decent confidence # This ensures we pick the closest repeater for the last hop @@ -1047,7 +1106,7 @@ class BotDataViewer: selected_repeater = graph_repeater confidence = graph_confidence final_method = selection_method or 'graph' - + if selected_repeater and confidence >= 0.5: repeater_info[node_id] = { 'name': selected_repeater['name'], @@ -1102,7 +1161,7 @@ class BotDataViewer: 'valid': False, 'error': str(e) } - + # Format response repeaters_list = [] for node_id in node_ids: @@ -1111,13 +1170,13 @@ class BotDataViewer: 'node_id': node_id, **info }) - + return { 'node_ids': node_ids, 'repeaters': repeaters_list, 'valid': True } - + def _setup_routes(self): """Setup all Flask routes - complete feature parity""" # Log full traceback for 500 errors so service logs show the real cause @@ -1126,47 +1185,298 @@ class BotDataViewer: self.logger.exception("Unhandled exception (500): %s", e) return make_response(("Internal Server Error", 500)) + # Authentication middleware (BUG-001) + _EXEMPT_PATHS = frozenset([ + '/login', '/logout', + '/apple-touch-icon.png', '/favicon-32x32.png', '/favicon-16x16.png', + '/site.webmanifest', '/favicon.ico', + ]) + + @self.app.before_request + def require_auth(): + if not self.web_viewer_password: + return # Auth disabled — no password configured + if request.path in _EXEMPT_PATHS or request.path.startswith('/static/'): + return + if session.get('authenticated'): + return + if request.path.startswith('/api/') or request.is_json: + return make_response(jsonify({'error': 'Authentication required'}), 401) + next_url = request.path + return redirect(url_for('login', next=next_url)) + + @self.app.route('/login', methods=['GET', 'POST']) + def login(): + """Login page for web viewer authentication""" + if not self.web_viewer_password: + return redirect(url_for('index')) + if request.method == 'POST': + password = request.form.get('password', '') + if password == self.web_viewer_password: + session['authenticated'] = True + next_url = request.args.get('next', '/') + if not next_url.startswith('/') or next_url.startswith('//'): + next_url = '/' + return redirect(next_url) + return render_template('login.html', error='Invalid password') + return render_template('login.html') + + @self.app.route('/logout') + def logout(): + """Logout and clear session""" + session.pop('authenticated', None) + return redirect(url_for('login')) + @self.app.route('/') def index(): """Main dashboard""" return render_template('index.html') - + @self.app.route('/realtime') def realtime(): """Real-time monitoring dashboard""" return render_template('realtime.html') - + + @self.app.route('/logs') + def logs(): + """Live log viewer""" + return render_template('logs.html') + @self.app.route('/contacts') def contacts(): """Contacts page - unified contact management and tracking""" return render_template('contacts.html') - + @self.app.route('/cache') def cache(): """Cache management page""" return render_template('cache.html') - - + + @self.app.route('/stats') def stats(): """Statistics page""" return render_template('stats.html') - + @self.app.route('/greeter') def greeter(): """Greeter management page""" return render_template('greeter.html') - + @self.app.route('/feeds') def feeds(): """Feed management page""" return render_template('feeds.html') - + @self.app.route('/radio') def radio(): """Radio settings page""" return render_template('radio.html') - + + @self.app.route('/config') + def config_page(): + """Bot configuration page""" + return render_template('config.html') + + @self.app.route('/api/config/notifications') + def api_config_notifications_get(): + """Return current notification settings from bot_metadata.""" + keys = [ + 'notif.smtp_host', 'notif.smtp_port', 'notif.smtp_security', + 'notif.smtp_user', 'notif.smtp_password', + 'notif.from_name', 'notif.from_email', + 'notif.recipients', 'notif.nightly_enabled', + ] + settings = {} + for k in keys: + val = self.db_manager.get_metadata(k) + short = k.split('.', 1)[1] + settings[short] = val if val is not None else '' + # Provide safe defaults for unset fields + if not settings.get('smtp_port'): + settings['smtp_port'] = '587' + if not settings.get('smtp_security'): + settings['smtp_security'] = 'starttls' + if not settings.get('nightly_enabled'): + settings['nightly_enabled'] = 'false' + return jsonify(settings) + + @self.app.route('/api/config/notifications', methods=['POST']) + def api_config_notifications_post(): + """Save notification settings to bot_metadata.""" + data = request.get_json(silent=True) or {} + allowed = { + 'smtp_host', 'smtp_port', 'smtp_security', + 'smtp_user', 'smtp_password', + 'from_name', 'from_email', + 'recipients', 'nightly_enabled', + } + saved = [] + for field in allowed: + if field in data: + self.db_manager.set_metadata(f'notif.{field}', str(data[field])) + saved.append(field) + self.logger.info(f"Notification settings updated: {', '.join(saved)}") + return jsonify({'success': True, 'saved': saved}) + + @self.app.route('/api/config/notifications/test', methods=['POST']) + def api_config_notifications_test(): + """Send a test email using the saved SMTP settings.""" + import smtplib + import ssl as _ssl + from email.message import EmailMessage + + def _get(key): + return self.db_manager.get_metadata(f'notif.{key}') or '' + + smtp_host = _get('smtp_host') + smtp_port = int(_get('smtp_port') or 587) + smtp_security = _get('smtp_security') or 'starttls' + smtp_user = _get('smtp_user') + smtp_password = _get('smtp_password') + from_name = _get('from_name') or 'MeshCore Bot' + from_email = _get('from_email') + recipients = [r.strip() for r in _get('recipients').split(',') if r.strip()] + + if not smtp_host: + return jsonify({'error': 'SMTP host is not configured'}), 400 + if not from_email: + return jsonify({'error': 'Sender email is not configured'}), 400 + if not recipients: + return jsonify({'error': 'No recipients configured'}), 400 + + try: + msg = EmailMessage() + msg['Subject'] = 'MeshCore Bot — test email' + msg['From'] = f'{from_name} <{from_email}>' + msg['To'] = ', '.join(recipients) + msg.set_content( + 'This is a test email from MeshCore Bot.\n\n' + 'If you received this, your SMTP settings are working correctly.\n' + ) + + context = _ssl.create_default_context() + + if smtp_security == 'ssl': + with smtplib.SMTP_SSL(smtp_host, smtp_port, context=context) as s: + if smtp_user and smtp_password: + s.login(smtp_user, smtp_password) + s.send_message(msg) + else: + with smtplib.SMTP(smtp_host, smtp_port) as s: + if smtp_security == 'starttls': + s.ehlo() + s.starttls(context=context) + s.ehlo() + if smtp_user and smtp_password: + s.login(smtp_user, smtp_password) + s.send_message(msg) + + self.logger.info(f"Test email sent to {recipients}") + return jsonify({'success': True, 'message': f'Test email sent to {", ".join(recipients)}'}) + + except Exception as e: + self.logger.error(f"Test email failed: {e}") + return jsonify({'error': str(e)}), 500 + + # ── Logging config ─────────────────────────────────────────────────── + + @self.app.route('/api/config/logging') + def api_config_logging_get(): + """Return log rotation settings from bot_metadata.""" + keys = ['maint.log_max_bytes', 'maint.log_backup_count'] + settings = {} + for k in keys: + short = k.split('.', 1)[1] + val = self.db_manager.get_metadata(k) + settings[short] = val if val is not None else '' + if not settings.get('log_max_bytes'): + settings['log_max_bytes'] = str(5 * 1024 * 1024) + if not settings.get('log_backup_count'): + settings['log_backup_count'] = '3' + return jsonify(settings) + + @self.app.route('/api/config/logging', methods=['POST']) + def api_config_logging_post(): + """Save log rotation settings to bot_metadata.""" + data = request.get_json(silent=True) or {} + allowed = {'log_max_bytes', 'log_backup_count'} + saved = [] + for field in allowed: + if field in data: + self.db_manager.set_metadata(f'maint.{field}', str(data[field])) + saved.append(field) + self.logger.info(f"Log rotation config updated: {', '.join(saved)}") + return jsonify({'success': True, 'saved': saved}) + + # ── Maintenance config ─────────────────────────────────────────────── + + @self.app.route('/api/config/maintenance') + def api_config_maintenance_get(): + """Return DB backup and email hook settings from bot_metadata.""" + keys = [ + 'maint.db_backup_enabled', 'maint.db_backup_schedule', + 'maint.db_backup_time', 'maint.db_backup_retention_count', + 'maint.db_backup_dir', 'maint.email_attach_log', + ] + settings = {} + for k in keys: + short = k.split('.', 1)[1] + val = self.db_manager.get_metadata(k) + settings[short] = val if val is not None else '' + # Defaults + if not settings.get('db_backup_enabled'): + settings['db_backup_enabled'] = 'false' + if not settings.get('db_backup_schedule'): + settings['db_backup_schedule'] = 'daily' + if not settings.get('db_backup_time'): + settings['db_backup_time'] = '02:00' + if not settings.get('db_backup_retention_count'): + settings['db_backup_retention_count'] = '7' + if not settings.get('db_backup_dir'): + settings['db_backup_dir'] = '/data/backups' + if not settings.get('email_attach_log'): + settings['email_attach_log'] = 'false' + return jsonify(settings) + + @self.app.route('/api/config/maintenance', methods=['POST']) + def api_config_maintenance_post(): + """Save DB backup and email hook settings to bot_metadata.""" + data = request.get_json(silent=True) or {} + allowed = { + 'db_backup_enabled', 'db_backup_schedule', 'db_backup_time', + 'db_backup_retention_count', 'db_backup_dir', 'email_attach_log', + } + saved = [] + for field in allowed: + if field in data: + self.db_manager.set_metadata(f'maint.{field}', str(data[field])) + saved.append(field) + self.logger.info(f"Maintenance config updated: {', '.join(saved)}") + return jsonify({'success': True, 'saved': saved}) + + # ── Maintenance status ─────────────────────────────────────────────── + + @self.app.route('/api/maintenance/status') + def api_maintenance_status(): + """Return last-run times and outcomes for all maintenance jobs.""" + status_keys = [ + 'maint.status.data_retention_ran_at', + 'maint.status.data_retention_outcome', + 'maint.status.nightly_email_ran_at', + 'maint.status.nightly_email_outcome', + 'maint.status.db_backup_ran_at', + 'maint.status.db_backup_outcome', + 'maint.status.db_backup_path', + 'maint.status.log_rotation_applied_at', + ] + result = {} + for k in status_keys: + short = k[len('maint.status.'):] + val = self.db_manager.get_metadata(k) + result[short] = val if val is not None else '' + return jsonify(result) + @self.app.route('/mesh') def mesh(): """Mesh graph visualization page""" @@ -1177,7 +1487,7 @@ class BotDataViewer: 'mesh.html', prefix_hex_chars=prefix_hex_chars ) - + # Favicon routes @self.app.route('/apple-touch-icon.png') def apple_touch_icon(): @@ -1186,7 +1496,7 @@ class BotDataViewer: os.path.join(os.path.dirname(__file__), 'static', 'ico'), 'apple-touch-icon.png' ) - + @self.app.route('/favicon-32x32.png') def favicon_32x32(): """32x32 favicon""" @@ -1194,7 +1504,7 @@ class BotDataViewer: os.path.join(os.path.dirname(__file__), 'static', 'ico'), 'favicon-32x32.png' ) - + @self.app.route('/favicon-16x16.png') def favicon_16x16(): """16x16 favicon""" @@ -1202,7 +1512,7 @@ class BotDataViewer: os.path.join(os.path.dirname(__file__), 'static', 'ico'), 'favicon-16x16.png' ) - + @self.app.route('/site.webmanifest') def site_webmanifest(): """Web manifest file""" @@ -1211,7 +1521,7 @@ class BotDataViewer: 'site.webmanifest', mimetype='application/manifest+json' ) - + @self.app.route('/favicon.ico') def favicon(): """Default favicon""" @@ -1219,18 +1529,18 @@ class BotDataViewer: os.path.join(os.path.dirname(__file__), 'static', 'ico'), 'favicon.ico' ) - - + + # API Routes @self.app.route('/api/health') def api_health(): """Health check endpoint""" # Get bot uptime bot_uptime = self._get_bot_uptime() - + with self._clients_lock: client_count = len(self.connected_clients) - + return jsonify({ 'status': 'healthy', 'connected_clients': client_count, @@ -1239,14 +1549,14 @@ class BotDataViewer: 'bot_uptime': bot_uptime, 'version': 'modern_2.0' }) - + @self.app.route('/api/system-health') def api_system_health(): """Get comprehensive system health status from database""" try: # Read health data from database (consistent with how other data is accessed) health_data = self.db_manager.get_system_health() - + if not health_data: # If no health data in database, return minimal status return jsonify({ @@ -1255,17 +1565,17 @@ class BotDataViewer: 'message': 'Health data not available yet', 'components': {} }) - + # Update timestamp to reflect current time (data may be slightly stale) health_data['timestamp'] = time.time() - + # Recalculate uptime if start_time is available start_time = self.db_manager.get_bot_start_time() if start_time: health_data['uptime_seconds'] = time.time() - start_time - + return jsonify(health_data) - + except Exception as e: self.logger.error(f"Error getting system health: {e}") import traceback @@ -1274,7 +1584,7 @@ class BotDataViewer: 'error': str(e), 'status': 'error' }), 500 - + @self.app.route('/api/stats') def api_stats(): """Get comprehensive database statistics for dashboard""" @@ -1294,9 +1604,37 @@ class BotDataViewer: except Exception as e: self.logger.error(f"Error getting stats: {e}") return jsonify({'error': str(e)}), 500 - - - + + + + @self.app.route('/api/stats/rate_limiters') + def api_rate_limiter_stats(): + """Return current rate limiter statistics from the running bot.""" + try: + bot = getattr(self, 'bot', None) + stats: dict[str, Any] = {} + if bot is None: + return jsonify(stats) + if hasattr(bot, 'rate_limiter') and bot.rate_limiter: + stats['message'] = bot.rate_limiter.get_stats() + if hasattr(bot, 'bot_tx_rate_limiter') and bot.bot_tx_rate_limiter: + stats['tx'] = bot.bot_tx_rate_limiter.get_stats() + if hasattr(bot, 'per_user_rate_limiter') and bot.per_user_rate_limiter: + rl = bot.per_user_rate_limiter + stats['per_user'] = { + 'seconds': rl.seconds, + 'tracked_users': len(rl._last_send), + 'max_entries': rl.max_entries, + } + if hasattr(bot, 'channel_rate_limiter') and bot.channel_rate_limiter: + stats['channels'] = bot.channel_rate_limiter.get_stats() + if hasattr(bot, 'nominatim_rate_limiter') and bot.nominatim_rate_limiter: + stats['nominatim'] = bot.nominatim_rate_limiter.get_stats() + return jsonify(stats) + except Exception as e: + self.logger.error(f"Error getting rate limiter stats: {e}") + return jsonify({'error': str(e)}), 500 + @self.app.route('/api/contacts') def api_contacts(): """Get contact data. Optional query param: since=24h|7d|30d|90d|all (default 30d).""" @@ -1309,7 +1647,7 @@ class BotDataViewer: except Exception as e: self.logger.error(f"Error getting contacts: {e}") return jsonify({'error': str(e)}), 500 - + @self.app.route('/api/cache') def api_cache(): """Get cache data""" @@ -1319,7 +1657,7 @@ class BotDataViewer: except Exception as e: self.logger.error(f"Error getting cache: {e}") return jsonify({'error': str(e)}), 500 - + @self.app.route('/api/database') def api_database(): """Get database information""" @@ -1329,7 +1667,7 @@ class BotDataViewer: except Exception as e: self.logger.error(f"Error getting database info: {e}") return jsonify({'error': str(e)}), 500 - + @self.app.route('/api/optimize-database', methods=['POST']) def api_optimize_database(): """Optimize database using VACUUM, ANALYZE, and REINDEX""" @@ -1339,7 +1677,7 @@ class BotDataViewer: except Exception as e: self.logger.error(f"Error optimizing database: {e}") return jsonify({'success': False, 'error': str(e)}), 500 - + @self.app.route('/api/mesh/nodes') def api_mesh_nodes(): """Get all repeater nodes with locations and metadata. Prefix length from query param or [Bot] prefix_bytes.""" @@ -1352,9 +1690,9 @@ class BotDataViewer: prefix_hex_chars = 2 conn = self._get_db_connection() cursor = conn.cursor() - + query = f''' - SELECT + SELECT public_key, SUBSTR(public_key, 1, {prefix_hex_chars}) as prefix, name, @@ -1366,16 +1704,16 @@ class BotDataViewer: last_advert_timestamp FROM complete_contact_tracking WHERE role IN ('repeater', 'roomserver') - AND latitude IS NOT NULL + AND latitude IS NOT NULL AND longitude IS NOT NULL - AND latitude != 0 + AND latitude != 0 AND longitude != 0 ORDER BY name ''' - + cursor.execute(query) rows = cursor.fetchall() - + nodes = [] for row in rows: nodes.append({ @@ -1389,7 +1727,7 @@ class BotDataViewer: 'last_heard': row['last_heard'], 'last_advert_timestamp': row['last_advert_timestamp'] }) - + return jsonify({'nodes': nodes}) except Exception as e: self.logger.error(f"Error getting mesh nodes: {e}") @@ -1397,7 +1735,7 @@ class BotDataViewer: finally: if conn: conn.close() - + @self.app.route('/api/mesh/edges') def api_mesh_edges(): """Get all graph edges with metadata""" @@ -1408,12 +1746,12 @@ class BotDataViewer: days = request.args.get('days', type=int) min_distance = request.args.get('min_distance', type=float) max_distance = request.args.get('max_distance', type=float) - + conn = self._get_db_connection() cursor = conn.cursor() - + query = ''' - SELECT + SELECT from_prefix, to_prefix, from_public_key, @@ -1427,28 +1765,28 @@ class BotDataViewer: WHERE 1=1 ''' params = [] - + if min_observations is not None: query += ' AND observation_count >= ?' params.append(min_observations) - + if days is not None: query += ' AND last_seen >= datetime("now", "-" || ? || " days")' params.append(days) - + if min_distance is not None: query += ' AND geographic_distance >= ?' params.append(min_distance) - + if max_distance is not None: query += ' AND geographic_distance <= ?' params.append(max_distance) - + query += ' ORDER BY last_seen DESC' - + cursor.execute(query, params) rows = cursor.fetchall() - + edges = [] prefix_hex_chars = 2 # default 1 byte for row in rows: @@ -1465,7 +1803,7 @@ class BotDataViewer: 'avg_hop_position': row['avg_hop_position'], 'geographic_distance': row['geographic_distance'] }) - + return jsonify({'edges': edges, 'prefix_hex_chars': prefix_hex_chars or 2}) except Exception as e: self.logger.error(f"Error getting mesh edges: {e}") @@ -1473,7 +1811,7 @@ class BotDataViewer: finally: if conn: conn.close() - + @self.app.route('/api/mesh/stats') def api_mesh_stats(): """Get graph statistics""" @@ -1481,22 +1819,22 @@ class BotDataViewer: try: conn = self._get_db_connection() cursor = conn.cursor() - + # Get node count cursor.execute(''' SELECT COUNT(*) as count FROM complete_contact_tracking WHERE role IN ('repeater', 'roomserver') - AND latitude IS NOT NULL + AND latitude IS NOT NULL AND longitude IS NOT NULL - AND latitude != 0 + AND latitude != 0 AND longitude != 0 ''') node_count = cursor.fetchone()['count'] - + # Get edge statistics cursor.execute(''' - SELECT + SELECT COUNT(*) as total_edges, SUM(observation_count) as total_observations, AVG(observation_count) as avg_observations, @@ -1509,16 +1847,16 @@ class BotDataViewer: FROM mesh_connections ''') edge_stats = cursor.fetchone() - + # Get most connected nodes cursor.execute(''' - SELECT + SELECT from_prefix as prefix, COUNT(*) as connection_count FROM mesh_connections GROUP BY from_prefix UNION ALL - SELECT + SELECT to_prefix as prefix, COUNT(*) as connection_count FROM mesh_connections @@ -1528,10 +1866,10 @@ class BotDataViewer: for row in cursor.fetchall(): prefix = row['prefix'].lower() connection_counts[prefix] = connection_counts.get(prefix, 0) + row['connection_count'] - + # Get top 10 most connected top_connected = sorted(connection_counts.items(), key=lambda x: x[1], reverse=True)[:10] - + # Get recent edges count (last 24 hours) cursor.execute(''' SELECT COUNT(*) as count @@ -1539,7 +1877,7 @@ class BotDataViewer: WHERE last_seen >= datetime("now", "-1 days") ''') recent_edges = cursor.fetchone()['count'] - + stats = { 'node_count': node_count, 'total_edges': edge_stats['total_edges'] or 0, @@ -1554,12 +1892,12 @@ class BotDataViewer: 'top_connected': [{'prefix': prefix, 'count': count} for prefix, count in top_connected], 'recent_edges_24h': recent_edges } - + return jsonify(stats) except Exception as e: self.logger.error(f"Error getting mesh stats: {e}") return jsonify({'error': str(e)}), 500 - + @self.app.route('/api/mesh/resolve-path', methods=['POST']) def api_resolve_path(): """Resolve a hex path to repeater names and locations using the same algorithm as path command""" @@ -1567,16 +1905,16 @@ class BotDataViewer: data = request.get_json() if not data: return jsonify({'error': 'JSON body required'}), 400 - + path_input = data.get('path', '').strip() if not path_input: return jsonify({'error': 'Path input required'}), 400 - + # Check if db_manager is initialized if not hasattr(self, 'db_manager') or not self.db_manager: self.logger.error("db_manager not initialized") return jsonify({'error': 'Database not initialized'}), 500 - + resolved_path = self._resolve_path(path_input) return jsonify(resolved_path) except Exception as e: @@ -1584,7 +1922,7 @@ class BotDataViewer: error_trace = traceback.format_exc() self.logger.error(f"Error resolving path: {e}\n{error_trace}") return jsonify({'error': str(e), 'traceback': error_trace}), 500 - + @self.app.route('/api/stream_data', methods=['POST']) def api_stream_data(): """API endpoint for receiving real-time data from bot""" @@ -1592,7 +1930,7 @@ class BotDataViewer: data = request.get_json() if not data: return jsonify({'error': 'No data provided'}), 400 - + data_type = data.get('type') if data_type == 'command': self._handle_command_data(data.get('data', {})) @@ -1604,35 +1942,35 @@ class BotDataViewer: self._handle_mesh_node_data(data.get('data', {})) else: return jsonify({'error': 'Invalid data type'}), 400 - + return jsonify({'status': 'success'}) except Exception as e: self.logger.error(f"Error in stream_data endpoint: {e}") return jsonify({'error': str(e)}), 500 - + @self.app.route('/api/recent_commands') def api_recent_commands(): """API endpoint to get recent commands from database""" try: - import sqlite3 import json + import sqlite3 import time - + # Get commands from last 60 minutes cutoff_time = time.time() - (60 * 60) # 60 minutes ago - + with closing(sqlite3.connect(self.db_path, timeout=60)) as conn: cursor = conn.cursor() - + cursor.execute(''' - SELECT data FROM packet_stream + SELECT data FROM packet_stream WHERE type = 'command' AND timestamp > ? ORDER BY timestamp DESC LIMIT 100 ''', (cutoff_time,)) - + rows = cursor.fetchall() - + # Parse and return commands commands = [] for (data_json,) in rows: @@ -1641,13 +1979,116 @@ class BotDataViewer: commands.append(command_data) except Exception as e: self.logger.debug(f"Error parsing command data: {e}") - + return jsonify({'commands': commands}) - + except Exception as e: self.logger.error(f"Error getting recent commands: {e}") return jsonify({'error': str(e)}), 500 - + + # ── Export ────────────────────────────────────────────────────────── + + @self.app.route('/api/export/contacts') + def api_export_contacts(): + """Export contact tracking data as CSV or JSON. + Query params: format=csv|json (default json), since=24h|7d|30d|90d|all (default 30d).""" + import csv + import io + fmt = request.args.get('format', 'json').lower() + since = request.args.get('since', '30d') + if since not in ('24h', '7d', '30d', '90d', 'all'): + since = '30d' + try: + result = self._get_tracking_data(since=since) + contacts = result.get('tracking_data', []) + if fmt == 'csv': + fields = [ + 'user_id', 'username', 'role', 'device_type', + 'latitude', 'longitude', 'city', 'state', 'country', + 'snr', 'hop_count', 'first_heard', 'last_seen', + 'advert_count', 'total_messages', 'distance', 'is_starred', + ] + buf = io.StringIO() + w = csv.DictWriter(buf, fieldnames=fields, extrasaction='ignore') + w.writeheader() + w.writerows(contacts) + return Response( + buf.getvalue(), + mimetype='text/csv', + headers={'Content-Disposition': f'attachment; filename="contacts_{since}.csv"'}, + ) + else: + import json as _json + body = _json.dumps(contacts, indent=2, default=str) + return Response( + body, + mimetype='application/json', + headers={'Content-Disposition': f'attachment; filename="contacts_{since}.json"'}, + ) + except Exception as e: + self.logger.error(f"Error exporting contacts: {e}") + return jsonify({'error': str(e)}), 500 + + @self.app.route('/api/export/paths') + def api_export_paths(): + """Export observed path data as CSV or JSON. + Query params: format=csv|json (default json), since=24h|7d|30d|90d|all (default 30d).""" + import csv + import io + import json as _json + import sqlite3 + fmt = request.args.get('format', 'json').lower() + since = request.args.get('since', '30d') + if since not in ('24h', '7d', '30d', '90d', 'all'): + since = '30d' + try: + days_map = {'24h': 1, '7d': 7, '30d': 30, '90d': 90} + where = ( + f" AND op.last_seen >= datetime('now', '-{days_map[since]} days')" + if since != 'all' else '' + ) + with closing(sqlite3.connect(self.db_path, timeout=60)) as conn: + conn.row_factory = sqlite3.Row + cursor = conn.cursor() + cursor.execute(f""" + SELECT op.public_key, c.name AS contact_name, + op.path_hex, op.path_length, op.observation_count, + op.last_seen, op.from_prefix, op.to_prefix, + op.bytes_per_hop, op.packet_type + FROM observed_paths op + LEFT JOIN complete_contact_tracking c ON op.public_key = c.public_key + WHERE op.packet_type = 'advert' AND op.public_key IS NOT NULL + {where} + ORDER BY op.last_seen DESC + LIMIT 10000 + """) + rows = [dict(r) for r in cursor.fetchall()] + if fmt == 'csv': + fields = [ + 'public_key', 'contact_name', 'path_hex', 'path_length', + 'observation_count', 'last_seen', 'from_prefix', 'to_prefix', + 'bytes_per_hop', 'packet_type', + ] + buf = io.StringIO() + w = csv.DictWriter(buf, fieldnames=fields, extrasaction='ignore') + w.writeheader() + w.writerows(rows) + return Response( + buf.getvalue(), + mimetype='text/csv', + headers={'Content-Disposition': f'attachment; filename="paths_{since}.csv"'}, + ) + else: + body = _json.dumps(rows, indent=2, default=str) + return Response( + body, + mimetype='application/json', + headers={'Content-Disposition': f'attachment; filename="paths_{since}.json"'}, + ) + except Exception as e: + self.logger.error(f"Error exporting paths: {e}") + return jsonify({'error': str(e)}), 500 + @self.app.route('/api/geocode-contact', methods=['POST']) def api_geocode_contact(): """Manually geocode a contact by public_key""" @@ -1656,31 +2097,31 @@ class BotDataViewer: data = request.get_json() if not data or 'public_key' not in data: return jsonify({'error': 'public_key is required'}), 400 - + public_key = data['public_key'] - + # Get contact data from database conn = self._get_db_connection() cursor = conn.cursor() - + cursor.execute(''' SELECT latitude, longitude, name, city, state, country FROM complete_contact_tracking WHERE public_key = ? ''', (public_key,)) - + contact = cursor.fetchone() if not contact: return jsonify({'error': 'Contact not found'}), 404 - + lat = contact['latitude'] lon = contact['longitude'] name = contact['name'] - + # Check if we have valid coordinates if lat is None or lon is None or lat == 0.0 or lon == 0.0: return jsonify({'error': 'Contact does not have valid coordinates'}), 400 - + # Perform geocoding self.logger.info(f"Manual geocoding requested for {name} ({public_key[:16]}...) at coordinates {lat}, {lon}") # sqlite3.Row objects use dictionary-style access with [] @@ -1688,7 +2129,7 @@ class BotDataViewer: current_state = contact['state'] current_country = contact['country'] self.logger.debug(f"Current location data - city: {current_city}, state: {current_state}, country: {current_country}") - + try: location_info = self.repeater_manager._get_full_location_from_coordinates(lat, lon) self.logger.debug(f"Geocoding result for {name}: {location_info}") @@ -1699,10 +2140,10 @@ class BotDataViewer: 'error': f'Geocoding exception: {str(geocode_error)}', 'location': {} }), 500 - + # Check if geocoding returned any useful data has_location_data = location_info.get('city') or location_info.get('state') or location_info.get('country') - + if not has_location_data: self.logger.warning(f"Geocoding returned no location data for {name} at {lat}, {lon}. Result: {location_info}") return jsonify({ @@ -1710,7 +2151,7 @@ class BotDataViewer: 'error': 'Geocoding returned no location data. The coordinates may be invalid or the geocoding service may be unavailable.', 'location': location_info }), 500 - + # Update database with new location data cursor.execute(''' UPDATE complete_contact_tracking @@ -1722,9 +2163,9 @@ class BotDataViewer: location_info.get('country'), public_key )) - + conn.commit() - + # Build success message with what was found found_parts = [] if location_info.get('city'): @@ -1733,56 +2174,48 @@ class BotDataViewer: found_parts.append(f"state: {location_info['state']}") if location_info.get('country'): found_parts.append(f"country: {location_info['country']}") - + success_message = f'Successfully geocoded {name} - Found {", ".join(found_parts)}' self.logger.info(f"Successfully geocoded {name}: {location_info}") - + return jsonify({ 'success': True, 'location': location_info, 'message': success_message }) - + except Exception as e: self.logger.error(f"Error geocoding contact: {e}", exc_info=True) return jsonify({'error': str(e)}), 500 finally: if conn: conn.close() - + @self.app.route('/api/toggle-star-contact', methods=['POST']) def api_toggle_star_contact(): - """Toggle star status for a contact by public_key (only for repeaters and roomservers)""" + """Toggle star status for any contact by public_key.""" conn = None try: data = request.get_json() if not data or 'public_key' not in data: return jsonify({'error': 'public_key is required'}), 400 - + public_key = data['public_key'] - + # Get contact data from database conn = self._get_db_connection() cursor = conn.cursor() - - # Check if contact exists and is a repeater or roomserver + cursor.execute(''' SELECT name, is_starred, role FROM complete_contact_tracking WHERE public_key = ? ''', (public_key,)) - + contact = cursor.fetchone() if not contact: return jsonify({'error': 'Contact not found'}), 404 - - # Only allow starring repeaters and roomservers - # sqlite3.Row objects use dictionary-style access with [] - role = contact['role'] - if role and role.lower() not in ('repeater', 'roomserver'): - return jsonify({'error': 'Only repeaters and roomservers can be starred'}), 400 - + # Toggle star status - # sqlite3.Row objects use dictionary-style access with [] current_starred = contact['is_starred'] new_star_status = 1 if not current_starred else 0 cursor.execute(''' @@ -1790,25 +2223,25 @@ class BotDataViewer: SET is_starred = ? WHERE public_key = ? ''', (new_star_status, public_key)) - + conn.commit() - + action = 'starred' if new_star_status else 'unstarred' self.logger.info(f"Contact {contact['name']} ({public_key[:16]}...) {action}") - + return jsonify({ 'success': True, 'is_starred': bool(new_star_status), 'message': f'Contact {action} successfully' }) - + except Exception as e: self.logger.error(f"Error toggling star status: {e}", exc_info=True) return jsonify({'error': str(e)}), 500 finally: if conn: conn.close() - + @self.app.route('/api/decode-path', methods=['POST']) def api_decode_path(): """Decode path hex string to repeater names (similar to path command). @@ -1843,7 +2276,7 @@ class BotDataViewer: except Exception as e: self.logger.error(f"Error decoding path: {e}", exc_info=True) return jsonify({'error': str(e)}), 500 - + @self.app.route('/api/delete-contact', methods=['POST']) def api_delete_contact(): """Delete a contact from the complete contact tracking database""" @@ -1852,38 +2285,38 @@ class BotDataViewer: data = request.get_json() if not data or 'public_key' not in data: return jsonify({'error': 'public_key is required'}), 400 - + public_key = data['public_key'] - + # Get contact data from database to log what we're deleting conn = self._get_db_connection() cursor = conn.cursor() - + # Check if contact exists cursor.execute(''' SELECT name, role, device_type FROM complete_contact_tracking WHERE public_key = ? ''', (public_key,)) - + contact = cursor.fetchone() if not contact: return jsonify({'error': 'Contact not found'}), 404 - + contact_name = contact['name'] contact_role = contact['role'] contact_device_type = contact['device_type'] - + # Delete from all related tables deleted_counts = {} - + # Delete from complete_contact_tracking cursor.execute('DELETE FROM complete_contact_tracking WHERE public_key = ?', (public_key,)) deleted_counts['complete_contact_tracking'] = cursor.rowcount - + # Delete from daily_stats cursor.execute('DELETE FROM daily_stats WHERE public_key = ?', (public_key,)) deleted_counts['daily_stats'] = cursor.rowcount - + # Delete from repeater_contacts if it exists try: cursor.execute('DELETE FROM repeater_contacts WHERE public_key = ?', (public_key,)) @@ -1891,26 +2324,99 @@ class BotDataViewer: except sqlite3.OperationalError: # Table might not exist, that's okay deleted_counts['repeater_contacts'] = 0 - + conn.commit() - + # Log the deletion self.logger.info(f"Contact deleted: {contact_name} ({public_key[:16]}...) - Role: {contact_role}, Device: {contact_device_type}") self.logger.debug(f"Deleted counts: {deleted_counts}") - + return jsonify({ 'success': True, 'message': f'Contact "{contact_name}" has been deleted successfully', 'deleted_counts': deleted_counts }) - + except Exception as e: self.logger.error(f"Error deleting contact: {e}", exc_info=True) return jsonify({'error': str(e)}), 500 finally: if conn: conn.close() - + + @self.app.route('/api/contacts/purge-preview') + def api_contacts_purge_preview(): + """Return count and sample of contacts not heard within the last N days.""" + days = request.args.get('days', 30, type=int) + if days < 1: + return jsonify({'error': 'days must be >= 1'}), 400 + conn = None + try: + conn = self._get_db_connection() + cursor = conn.cursor() + cursor.execute(''' + SELECT COUNT(*) AS cnt FROM complete_contact_tracking + WHERE last_heard < datetime('now', ? || ' days') + ''', (f'-{days}',)) + count = cursor.fetchone()['cnt'] + cursor.execute(''' + SELECT name, role, last_heard FROM complete_contact_tracking + WHERE last_heard < datetime('now', ? || ' days') + ORDER BY last_heard ASC + LIMIT 5 + ''', (f'-{days}',)) + samples = [dict(r) for r in cursor.fetchall()] + return jsonify({'count': count, 'days': days, 'samples': samples}) + except Exception as e: + self.logger.error(f"Error in purge preview: {e}", exc_info=True) + return jsonify({'error': str(e)}), 500 + finally: + if conn: + conn.close() + + @self.app.route('/api/contacts/purge', methods=['POST']) + def api_contacts_purge(): + """Delete all contacts not heard within the last N days.""" + data = request.get_json(silent=True) or {} + days = data.get('days', 30) + try: + days = int(days) + except (TypeError, ValueError): + return jsonify({'error': 'days must be an integer'}), 400 + if days < 1: + return jsonify({'error': 'days must be >= 1'}), 400 + conn = None + try: + conn = self._get_db_connection() + cursor = conn.cursor() + cutoff = f'-{days} days' + # Collect public_keys to purge so we can cascade + cursor.execute(''' + SELECT public_key FROM complete_contact_tracking + WHERE last_heard < datetime('now', ?) + ''', (cutoff,)) + keys = [r['public_key'] for r in cursor.fetchall()] + if not keys: + return jsonify({'success': True, 'deleted': 0, 'message': 'No contacts matched the threshold'}) + placeholders = ','.join('?' * len(keys)) + cursor.execute(f'DELETE FROM complete_contact_tracking WHERE public_key IN ({placeholders})', keys) + deleted = cursor.rowcount + cursor.execute(f'DELETE FROM daily_stats WHERE public_key IN ({placeholders})', keys) + try: + cursor.execute(f'DELETE FROM repeater_contacts WHERE public_key IN ({placeholders})', keys) + except sqlite3.OperationalError: + pass + conn.commit() + self.logger.info(f"Purged {deleted} contact(s) not heard in {days}+ days") + return jsonify({'success': True, 'deleted': deleted, + 'message': f'Purged {deleted} contact(s) not heard in {days}+ days'}) + except Exception as e: + self.logger.error(f"Error purging contacts: {e}", exc_info=True) + return jsonify({'error': str(e)}), 500 + finally: + if conn: + conn.close() + @self.app.route('/api/greeter') def api_greeter(): """Get greeter data including rollout status, settings, and greeted users""" @@ -1918,7 +2424,7 @@ class BotDataViewer: try: conn = self._get_db_connection() cursor = conn.cursor() - + # Check if greeter tables exist cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='greeter_rollout'") if not cursor.fetchone(): @@ -1929,7 +2435,7 @@ class BotDataViewer: 'greeted_users': [], 'error': 'Greeter tables not found' }) - + # Get active rollout status cursor.execute(''' SELECT id, rollout_started_at, rollout_days, rollout_completed, @@ -1941,21 +2447,21 @@ class BotDataViewer: LIMIT 1 ''') rollout = cursor.fetchone() - + rollout_active = False rollout_data = None time_remaining = None - + if rollout: rollout_id = rollout['id'] started_at_str = rollout['rollout_started_at'] rollout_days = rollout['rollout_days'] end_date_str = rollout['end_date'] current_time_str = rollout['current_time'] - + end_date = datetime.fromisoformat(end_date_str) current_time = datetime.fromisoformat(current_time_str) - + if current_time < end_date: rollout_active = True remaining_seconds = (end_date - current_time).total_seconds() @@ -1972,21 +2478,21 @@ class BotDataViewer: 'days': rollout_days, 'end_date': end_date_str } - + # Get greeter settings from config settings = { 'enabled': self.config.getboolean('Greeter_Command', 'enabled', fallback=False), - 'greeting_message': self.config.get('Greeter_Command', 'greeting_message', + 'greeting_message': self.config.get('Greeter_Command', 'greeting_message', fallback='Welcome to the mesh, {sender}!'), 'rollout_days': self.config.getint('Greeter_Command', 'rollout_days', fallback=7), - 'include_mesh_info': self.config.getboolean('Greeter_Command', 'include_mesh_info', + 'include_mesh_info': self.config.getboolean('Greeter_Command', 'include_mesh_info', fallback=True), 'mesh_info_format': self.config.get('Greeter_Command', 'mesh_info_format', fallback='\n\nMesh Info: {total_contacts} contacts, {repeaters} repeaters'), 'per_channel_greetings': self.config.getboolean('Greeter_Command', 'per_channel_greetings', fallback=False) } - + # Generate sample greeting sample_greeting = settings['greeting_message'].format(sender='SampleUser') if settings['include_mesh_info']: @@ -1997,18 +2503,18 @@ class BotDataViewer: recent_activity_24h=10 ) sample_greeting += sample_mesh_info - + # Check if message_stats table exists for last seen data cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='message_stats'") has_message_stats = cursor.fetchone() is not None - + # Get greeted users - use GROUP BY to ensure only one entry per (sender_id, channel) # This handles any potential duplicates that might exist in the database # We use MIN(greeted_at) to get the earliest (first) greeting time # If per_channel_greetings is False, we'll still show one entry per user (channel will be NULL) # If per_channel_greetings is True, we'll show one entry per user per channel cursor.execute(''' - SELECT sender_id, channel, MIN(greeted_at) as greeted_at, + SELECT sender_id, channel, MIN(greeted_at) as greeted_at, MAX(rollout_marked) as rollout_marked FROM greeted_users GROUP BY sender_id, channel @@ -2017,7 +2523,7 @@ class BotDataViewer: ''') greeted_users_rows = cursor.fetchall() greeted_users = [] - + for row in greeted_users_rows: # Access row data - handle both dict-style (Row) and tuple access try: @@ -2028,10 +2534,10 @@ class BotDataViewer: except (KeyError, IndexError, TypeError) as e: self.logger.error(f"Error accessing row data: {e}, row type: {type(row)}") continue - + sender_id = str(sender_id) if sender_id else '' channel = str(channel_raw) if channel_raw else '(global)' - + # Get last seen timestamp from message_stats if available last_seen = None if has_message_stats: @@ -2042,7 +2548,7 @@ class BotDataViewer: cursor.execute(''' SELECT MAX(timestamp) as last_seen FROM message_stats - WHERE sender_id = ? + WHERE sender_id = ? AND channel = ? AND is_dm = 0 AND channel IS NOT NULL @@ -2052,15 +2558,15 @@ class BotDataViewer: cursor.execute(''' SELECT MAX(timestamp) as last_seen FROM message_stats - WHERE sender_id = ? + WHERE sender_id = ? AND is_dm = 0 AND channel IS NOT NULL ''', (sender_id,)) - + result = cursor.fetchone() if result and result['last_seen']: last_seen = result['last_seen'] - + greeted_users.append({ 'sender_id': sender_id, 'channel': channel, @@ -2068,7 +2574,7 @@ class BotDataViewer: 'rollout_marked': bool(rollout_marked), 'last_seen': last_seen }) - + return jsonify({ 'enabled': settings['enabled'], 'rollout_active': rollout_active, @@ -2079,14 +2585,14 @@ class BotDataViewer: 'greeted_users': greeted_users, 'total_greeted': len(greeted_users) }) - + except Exception as e: self.logger.error(f"Error getting greeter data: {e}", exc_info=True) return jsonify({'error': str(e)}), 500 finally: if conn: conn.close() - + @self.app.route('/api/greeter/end-rollout', methods=['POST']) def api_end_rollout(): """End the active onboarding period""" @@ -2094,7 +2600,7 @@ class BotDataViewer: try: conn = self._get_db_connection() cursor = conn.cursor() - + # Find active rollout cursor.execute(''' SELECT id FROM greeter_rollout @@ -2103,35 +2609,35 @@ class BotDataViewer: LIMIT 1 ''') rollout = cursor.fetchone() - + if not rollout: return jsonify({'success': False, 'error': 'No active rollout found'}), 404 - + rollout_id = rollout['id'] - + # Mark rollout as completed cursor.execute(''' UPDATE greeter_rollout SET rollout_completed = 1 WHERE id = ? ''', (rollout_id,)) - + conn.commit() - + self.logger.info(f"Greeter rollout {rollout_id} ended manually via web viewer") - + return jsonify({ 'success': True, 'message': 'Onboarding period ended successfully' }) - + except Exception as e: self.logger.error(f"Error ending rollout: {e}", exc_info=True) return jsonify({'success': False, 'error': str(e)}), 500 finally: if conn: conn.close() - + @self.app.route('/api/greeter/ungreet', methods=['POST']) def api_ungreet_user(): """Mark a user as ungreeted (remove from greeted_users table)""" @@ -2140,13 +2646,13 @@ class BotDataViewer: data = request.get_json() if not data or 'sender_id' not in data: return jsonify({'error': 'sender_id is required'}), 400 - + sender_id = data['sender_id'] channel = data.get('channel') # Optional - if None, removes global greeting - + conn = self._get_db_connection() cursor = conn.cursor() - + # Check if user exists if channel and channel != '(global)': cursor.execute(''' @@ -2158,10 +2664,10 @@ class BotDataViewer: SELECT id FROM greeted_users WHERE sender_id = ? AND channel IS NULL ''', (sender_id,)) - + if not cursor.fetchone(): return jsonify({'error': 'User not found in greeted users'}), 404 - + # Delete the record if channel and channel != '(global)': cursor.execute(''' @@ -2173,23 +2679,23 @@ class BotDataViewer: DELETE FROM greeted_users WHERE sender_id = ? AND channel IS NULL ''', (sender_id,)) - + conn.commit() - + self.logger.info(f"User {sender_id} marked as ungreeted (channel: {channel or 'global'})") - + return jsonify({ 'success': True, 'message': f'User {sender_id} marked as ungreeted' }) - + except Exception as e: self.logger.error(f"Error ungreeting user: {e}", exc_info=True) return jsonify({'success': False, 'error': str(e)}), 500 finally: if conn: conn.close() - + # Feed management API endpoints @self.app.route('/api/feeds') def api_feeds(): @@ -2200,7 +2706,7 @@ class BotDataViewer: except Exception as e: self.logger.error(f"Error getting feeds: {e}") return jsonify({'error': str(e)}), 500 - + @self.app.route('/api/feeds/') def api_feed_detail(feed_id): """Get detailed information about a specific feed""" @@ -2208,19 +2714,19 @@ class BotDataViewer: feed = self._get_feed_subscription(feed_id) if not feed: return jsonify({'error': 'Feed not found'}), 404 - + # Get activity and errors activity = self._get_feed_activity(feed_id) errors = self._get_feed_errors(feed_id) - + feed['activity'] = activity feed['errors'] = errors - + return jsonify(feed) except Exception as e: self.logger.error(f"Error getting feed detail: {e}") return jsonify({'error': str(e)}), 500 - + @self.app.route('/api/feeds', methods=['POST']) def api_create_feed(): """Create a new feed subscription""" @@ -2228,13 +2734,13 @@ class BotDataViewer: data = request.get_json() if not data: return jsonify({'error': 'No data provided'}), 400 - + feed_id = self._create_feed_subscription(data) return jsonify({'success': True, 'id': feed_id}) except Exception as e: self.logger.error(f"Error creating feed: {e}") return jsonify({'error': str(e)}), 500 - + @self.app.route('/api/feeds/', methods=['PUT']) def api_update_feed(feed_id): """Update an existing feed subscription""" @@ -2242,16 +2748,16 @@ class BotDataViewer: data = request.get_json() if not data: return jsonify({'error': 'No data provided'}), 400 - + success = self._update_feed_subscription(feed_id, data) if not success: return jsonify({'error': 'Feed not found'}), 404 - + return jsonify({'success': True}) except Exception as e: self.logger.error(f"Error updating feed: {e}") return jsonify({'error': str(e)}), 500 - + @self.app.route('/api/feeds/', methods=['DELETE']) def api_delete_feed(feed_id): """Delete a feed subscription""" @@ -2259,23 +2765,23 @@ class BotDataViewer: success = self._delete_feed_subscription(feed_id) if not success: return jsonify({'error': 'Feed not found'}), 404 - + return jsonify({'success': True}) except Exception as e: self.logger.error(f"Error deleting feed: {e}") return jsonify({'error': str(e)}), 500 - + @self.app.route('/api/feeds/default-format', methods=['GET']) def api_get_default_format(): """Get the default output format from config""" try: - default_format = self.config.get('Feed_Manager', 'default_output_format', + default_format = self.config.get('Feed_Manager', 'default_output_format', fallback='{emoji} {body|truncate:100} - {date}\n{link|truncate:50}') return jsonify({'default_format': default_format}) except Exception as e: self.logger.error(f"Error getting default format: {e}") return jsonify({'default_format': '{emoji} {body|truncate:100} - {date}\n{link|truncate:50}'}) - + @self.app.route('/api/feeds/preview', methods=['POST']) def api_preview_feed(): """Preview feed items with custom output format""" @@ -2283,22 +2789,22 @@ class BotDataViewer: data = request.get_json() if not data or 'feed_url' not in data: return jsonify({'error': 'feed_url is required'}), 400 - + feed_url = data['feed_url'] feed_type = data.get('feed_type', 'rss') output_format = data.get('output_format', '') api_config = data.get('api_config', {}) filter_config = data.get('filter_config') sort_config = data.get('sort_config') - + # Get default format from config if not provided if not output_format: - output_format = self.config.get('Feed_Manager', 'default_output_format', + output_format = self.config.get('Feed_Manager', 'default_output_format', fallback='{emoji} {body|truncate:100} - {date}\n{link|truncate:50}') - + # Fetch and format feed items preview_items = self._preview_feed_items(feed_url, feed_type, output_format, api_config, filter_config, sort_config) - + return jsonify({ 'success': True, 'items': preview_items @@ -2306,7 +2812,7 @@ class BotDataViewer: except Exception as e: self.logger.error(f"Error previewing feed: {e}") return jsonify({'error': str(e)}), 500 - + @self.app.route('/api/feeds/test', methods=['POST']) def api_test_feed(): """Test a feed URL and return preview of recent items""" @@ -2314,19 +2820,19 @@ class BotDataViewer: data = request.get_json() if not data or 'url' not in data: return jsonify({'error': 'URL is required'}), 400 - + # This would require feed_manager - for now just validate URL from urllib.parse import urlparse url = data['url'] result = urlparse(url) if not all([result.scheme in ['http', 'https'], result.netloc]): return jsonify({'error': 'Invalid URL format'}), 400 - + return jsonify({'success': True, 'message': 'URL validated (full test requires feed manager)'}) except Exception as e: self.logger.error(f"Error testing feed: {e}") return jsonify({'error': str(e)}), 500 - + @self.app.route('/api/feeds/stats') def api_feed_stats(): """Get aggregate feed statistics""" @@ -2336,7 +2842,7 @@ class BotDataViewer: except Exception as e: self.logger.error(f"Error getting feed stats: {e}") return jsonify({'error': str(e)}), 500 - + @self.app.route('/api/feeds//activity') def api_feed_activity(feed_id): """Get activity log for a specific feed""" @@ -2346,7 +2852,7 @@ class BotDataViewer: except Exception as e: self.logger.error(f"Error getting feed activity: {e}") return jsonify({'error': str(e)}), 500 - + @self.app.route('/api/feeds//errors') def api_feed_errors(feed_id): """Get error history for a specific feed""" @@ -2356,7 +2862,7 @@ class BotDataViewer: except Exception as e: self.logger.error(f"Error getting feed errors: {e}") return jsonify({'error': str(e)}), 500 - + @self.app.route('/api/feeds//refresh', methods=['POST']) def api_refresh_feed(feed_id): """Manually trigger a feed check""" @@ -2367,7 +2873,7 @@ class BotDataViewer: except Exception as e: self.logger.error(f"Error refreshing feed: {e}") return jsonify({'error': str(e)}), 500 - + # Channel management API endpoints @self.app.route('/api/channels') def api_channels(): @@ -2378,7 +2884,7 @@ class BotDataViewer: except Exception as e: self.logger.error(f"Error getting channels: {e}") return jsonify({'error': str(e)}), 500 - + @self.app.route('/api/channels', methods=['POST']) def api_create_channel(): """Create a new channel (hashtag or custom)""" @@ -2386,38 +2892,38 @@ class BotDataViewer: data = request.get_json() if not data or 'name' not in data: return jsonify({'error': 'Channel name is required'}), 400 - + channel_name = data.get('name', '').strip() channel_idx = data.get('channel_idx') channel_key = data.get('channel_key', '').strip() - + if not channel_name: return jsonify({'error': 'Channel name cannot be empty'}), 400 - + # If channel_idx not provided, find the lowest available index if channel_idx is None: channel_idx = self._get_lowest_available_channel_index() if channel_idx is None: max_channels = self.config.getint('Bot', 'max_channels', fallback=40) return jsonify({'error': f'No available channel slots. All {max_channels} channels are in use.'}), 400 - + # Determine if it's a hashtag channel is_hashtag = channel_name.startswith('#') - + # Validate custom channel has key if not is_hashtag and not channel_key: return jsonify({'error': 'Channel key is required for custom channels (channels without # prefix)'}), 400 - + # Validate key format if provided if channel_key: if len(channel_key) != 32: return jsonify({'error': 'Channel key must be exactly 32 hexadecimal characters'}), 400 if not all(c in '0123456789abcdefABCDEF' for c in channel_key): return jsonify({'error': 'Channel key must contain only hexadecimal characters (0-9, a-f, A-F)'}), 400 - + # Try to create channel via bot's channel manager result = self._add_channel_for_web(channel_idx, channel_name, channel_key if not is_hashtag else None) - + if result.get('success'): if result.get('pending'): # Operation is queued, return operation_id for polling @@ -2431,11 +2937,11 @@ class BotDataViewer: return jsonify({'success': True, 'message': 'Channel created successfully'}) else: return jsonify({'error': result.get('error', 'Failed to create channel')}), 500 - + except Exception as e: self.logger.error(f"Error creating channel: {e}") return jsonify({'error': str(e)}), 500 - + @self.app.route('/api/channels/', methods=['DELETE']) def api_delete_channel(channel_idx): """Remove a channel""" @@ -2457,7 +2963,7 @@ class BotDataViewer: except Exception as e: self.logger.error(f"Error deleting channel: {e}") return jsonify({'error': str(e)}), 500 - + @self.app.route('/api/channel-operations/', methods=['GET']) def api_get_operation_status(operation_id): """Get status of a channel operation""" @@ -2470,14 +2976,14 @@ class BotDataViewer: FROM channel_operations WHERE id = ? ''', (operation_id,)) - + result = cursor.fetchone() - + if not result: return jsonify({'error': 'Operation not found'}), 404 - + status, error_msg, result_data, processed_at = result - + return jsonify({ 'operation_id': operation_id, 'status': status, @@ -2491,7 +2997,7 @@ class BotDataViewer: finally: if conn: conn.close() - + @self.app.route('/api/channels/validate', methods=['POST']) def api_validate_channel(): """Validate if a channel exists or can be created""" @@ -2499,11 +3005,11 @@ class BotDataViewer: data = request.get_json() if not data or 'name' not in data: return jsonify({'error': 'Channel name is required'}), 400 - + channel_name = data['name'] # Check if channel exists channel_num = self._get_channel_number(channel_name) - + return jsonify({ 'exists': channel_num is not None, 'channel_num': channel_num @@ -2511,7 +3017,7 @@ class BotDataViewer: except Exception as e: self.logger.error(f"Error validating channel: {e}") return jsonify({'error': str(e)}), 500 - + @self.app.route('/api/channels/', methods=['PUT']) def api_update_channel(channel_idx): """Update channel name or configuration""" @@ -2519,13 +3025,13 @@ class BotDataViewer: data = request.get_json() if not data: return jsonify({'error': 'No data provided'}), 400 - + # This would use channel_manager return jsonify({'success': True, 'message': 'Channel update requires bot connection'}) except Exception as e: self.logger.error(f"Error updating channel: {e}") return jsonify({'error': str(e)}), 500 - + @self.app.route('/api/channels/stats') def api_channel_stats(): """Get channel statistics and usage data""" @@ -2535,7 +3041,7 @@ class BotDataViewer: except Exception as e: self.logger.error(f"Error getting channel stats: {e}") return jsonify({'error': str(e)}), 500 - + @self.app.route('/api/channels//feeds') def api_channel_feeds(channel_idx): """Get all feed subscriptions for a specific channel""" @@ -2545,10 +3051,98 @@ class BotDataViewer: except Exception as e: self.logger.error(f"Error getting channel feeds: {e}") return jsonify({'error': str(e)}), 500 - + + @self.app.route('/api/radio/status') + def api_radio_status(): + """Current radio connection state from bot_metadata.""" + try: + value = self.db_manager.get_metadata('radio_connected') + connected = value == '1' if value is not None else None + return jsonify({'connected': connected, 'status_known': value is not None}) + except Exception as e: + self.logger.error(f"Error getting radio status: {e}") + return jsonify({'error': str(e)}), 500 + + @self.app.route('/api/radio/reboot', methods=['POST']) + def api_radio_reboot(): + """Queue a radio reboot (disconnect + reconnect).""" + try: + with self.db_manager.connection() as conn: + cursor = conn.cursor() + cursor.execute( + "INSERT INTO channel_operations (operation_type, status) VALUES ('radio_reboot', 'pending')" + ) + conn.commit() + op_id = cursor.lastrowid + return jsonify({'success': True, 'operation_id': op_id, 'message': 'Radio reboot queued'}) + except Exception as e: + self.logger.error(f"Error queuing radio reboot: {e}") + return jsonify({'error': str(e)}), 500 + + @self.app.route('/api/radio/connect', methods=['POST']) + def api_radio_connect(): + """Queue radio connect or disconnect. Body: {'action': 'connect'|'disconnect'}""" + try: + data = request.get_json(silent=True) or {} + action = data.get('action', '') + if action not in ('connect', 'disconnect'): + return jsonify({'error': "action must be 'connect' or 'disconnect'"}), 400 + op_type = 'radio_connect' if action == 'connect' else 'radio_disconnect' + with self.db_manager.connection() as conn: + cursor = conn.cursor() + cursor.execute( + "INSERT INTO channel_operations (operation_type, status) VALUES (?, 'pending')", + (op_type,) + ) + conn.commit() + op_id = cursor.lastrowid + return jsonify({'success': True, 'pending': True, 'operation_id': op_id}) + except Exception as e: + self.logger.error(f"Error queuing radio connect/disconnect: {e}") + return jsonify({'error': str(e)}), 500 + + @self.app.route('/api/radio/firmware/config/read', methods=['POST']) + def api_firmware_config_read(): + """Queue a firmware config read (path.hash.mode + custom vars). Poll /api/channel-operations/.""" + try: + with self.db_manager.connection() as conn: + cursor = conn.cursor() + cursor.execute( + "INSERT INTO channel_operations (operation_type, status) VALUES ('firmware_read', 'pending')" + ) + conn.commit() + op_id = cursor.lastrowid + return jsonify({'success': True, 'operation_id': op_id}) + except Exception as e: + self.logger.error(f"Error queuing firmware read: {e}") + return jsonify({'error': str(e)}), 500 + + @self.app.route('/api/radio/firmware/config/write', methods=['POST']) + def api_firmware_config_write(): + """Queue a firmware config write. Body: {path_hash_mode?: int, loop_detect?: str}. + Poll /api/channel-operations/ for result.""" + try: + data = request.get_json(silent=True) or {} + allowed = {'path_hash_mode', 'loop_detect'} + payload = {k: v for k, v in data.items() if k in allowed} + if not payload: + return jsonify({'error': 'No valid fields provided (path_hash_mode, loop_detect)'}), 400 + with self.db_manager.connection() as conn: + cursor = conn.cursor() + cursor.execute( + "INSERT INTO channel_operations (operation_type, payload_data, status) VALUES ('firmware_write', ?, 'pending')", + (json.dumps(payload),) + ) + conn.commit() + op_id = cursor.lastrowid + return jsonify({'success': True, 'operation_id': op_id}) + except Exception as e: + self.logger.error(f"Error queuing firmware write: {e}") + return jsonify({'error': str(e)}), 500 + def _setup_socketio_handlers(self): """Setup SocketIO event handlers using modern patterns""" - + @self.socketio.on('connect') def handle_connect(): """Handle client connection""" @@ -2557,9 +3151,16 @@ class BotDataViewer: if not client_id: self.logger.warning("Connect event received but client_id is None") return False - + + # Reject unauthenticated SocketIO connections when auth is enabled (BUG-001) + if self.web_viewer_password and not session.get('authenticated'): + self.logger.warning(f"Rejected unauthenticated SocketIO connection from {client_id}") + with suppress(Exception): + disconnect() + return False + self.logger.info(f"Client connected: {client_id}") - + with self._clients_lock: # Check client limit if len(self.connected_clients) >= self.max_clients: @@ -2569,22 +3170,23 @@ class BotDataViewer: except Exception as e: self.logger.error(f"Error disconnecting client: {e}") return False - + # Track client self.connected_clients[client_id] = { 'connected_at': time.time(), 'last_activity': time.time(), 'subscribed_commands': False, 'subscribed_packets': False, - 'subscribed_mesh': False + 'subscribed_mesh': False, + 'subscribed_logs': False, } - + # Connection status is shown via the green indicator in the navbar, no toast needed self.logger.info(f"Client {client_id} connected. Total clients: {len(self.connected_clients)}") except Exception as e: self.logger.error(f"Error in handle_connect: {e}", exc_info=True) return False - + @self.socketio.on('disconnect') def handle_disconnect(data=None): """Handle client disconnection""" @@ -2604,10 +3206,10 @@ class BotDataViewer: except Exception as e: # Don't emit errors during disconnect as the connection may be broken self.logger.error(f"Error in handle_disconnect: {e}", exc_info=True) - + @self.socketio.on('subscribe_commands') def handle_subscribe_commands(): - """Handle command stream subscription""" + """Handle command stream subscription — also replays recent history to the new subscriber.""" try: client_id = getattr(request, 'sid', None) with self._clients_lock: @@ -2615,12 +3217,30 @@ class BotDataViewer: self.connected_clients[client_id]['subscribed_commands'] = True emit('status', {'message': 'Subscribed to command stream'}) self.logger.debug(f"Client {client_id} subscribed to commands") + # Replay recent command history so the page isn't blank on load (BUG-023 fix) + try: + with closing(sqlite3.connect(self.db_path, timeout=10, check_same_thread=False)) as _conn: + _conn.row_factory = sqlite3.Row + _cur = _conn.cursor() + _cur.execute( + "SELECT data FROM packet_stream" + " WHERE type = 'command'" + " ORDER BY timestamp DESC LIMIT 50" + ) + rows = list(reversed(_cur.fetchall())) + for row in rows: + try: + emit('command_data', json.loads(row['data'])) + except Exception: + pass + except Exception as e: + self.logger.debug(f"Error replaying command history: {e}") except Exception as e: self.logger.error(f"Error in handle_subscribe_commands: {e}", exc_info=True) - + @self.socketio.on('subscribe_packets') def handle_subscribe_packets(): - """Handle packet stream subscription""" + """Handle packet stream subscription — also replays recent history to the new subscriber.""" try: client_id = getattr(request, 'sid', None) with self._clients_lock: @@ -2628,9 +3248,29 @@ class BotDataViewer: self.connected_clients[client_id]['subscribed_packets'] = True emit('status', {'message': 'Subscribed to packet stream'}) self.logger.debug(f"Client {client_id} subscribed to packets") + # Replay recent packet/command/routing history so the page isn't blank on load + try: + with closing(sqlite3.connect(self.db_path, timeout=10, check_same_thread=False)) as _conn: + _conn.row_factory = sqlite3.Row + _cur = _conn.cursor() + _cur.execute( + "SELECT data, type FROM packet_stream" + " WHERE type IN ('packet','command','routing')" + " ORDER BY timestamp DESC LIMIT 50" + ) + rows = list(reversed(_cur.fetchall())) + for row in rows: + try: + data = json.loads(row['data']) + evt = 'command_data' if row['type'] == 'command' else 'packet_data' + emit(evt, data) + except Exception: + pass + except Exception as e: + self.logger.debug(f"Error replaying packet history: {e}") except Exception as e: self.logger.error(f"Error in handle_subscribe_packets: {e}", exc_info=True) - + @self.socketio.on('subscribe_mesh') def handle_subscribe_mesh(): """Handle mesh graph stream subscription""" @@ -2643,7 +3283,67 @@ class BotDataViewer: self.logger.debug(f"Client {client_id} subscribed to mesh graph") except Exception as e: self.logger.error(f"Error in handle_subscribe_mesh: {e}", exc_info=True) - + + @self.socketio.on('subscribe_messages') + def handle_subscribe_messages(): + """Handle live channel message stream subscription — also replays recent messages.""" + try: + client_id = getattr(request, 'sid', None) + with self._clients_lock: + if client_id and client_id in self.connected_clients: + self.connected_clients[client_id]['subscribed_messages'] = True + emit('status', {'message': 'Subscribed to message stream'}) + self.logger.debug(f"Client {client_id} subscribed to messages") + # Replay recent channel messages so the page isn't blank on load + try: + with closing(sqlite3.connect(self.db_path, timeout=10, check_same_thread=False)) as _conn: + _conn.row_factory = sqlite3.Row + _cur = _conn.cursor() + _cur.execute( + "SELECT data FROM packet_stream" + " WHERE type = 'message'" + " ORDER BY timestamp DESC LIMIT 50" + ) + rows = list(reversed(_cur.fetchall())) + for row in rows: + try: + emit('message_data', json.loads(row['data'])) + except Exception: + pass + except Exception as e: + self.logger.debug(f"Error replaying message history: {e}") + except Exception as e: + self.logger.error(f"Error in handle_subscribe_messages: {e}", exc_info=True) + + @self.socketio.on('subscribe_logs') + def handle_subscribe_logs(): + """Handle live log stream subscription — also sends last 200 log lines to the new subscriber.""" + try: + client_id = getattr(request, 'sid', None) + with self._clients_lock: + if client_id and client_id in self.connected_clients: + self.connected_clients[client_id]['subscribed_logs'] = True + emit('status', {'message': 'Subscribed to log stream'}) + self.logger.debug(f"Client {client_id} subscribed to logs") + # Send recent log history so the page isn't blank on load + log_file = '' + try: + log_file = self.config.get('Logging', 'log_file', fallback='').strip() + if log_file: + log_file = str(resolve_path(log_file, self.bot_root)) + except Exception: + pass + if log_file and os.path.exists(log_file): + try: + with open(log_file, encoding='utf-8', errors='replace') as _fh: + recent_lines = _fh.readlines()[-200:] + for line in recent_lines: + emit('log_line', {'line': line.rstrip()}) + except Exception as e: + self.logger.debug(f"Error reading log history: {e}") + except Exception as e: + self.logger.error(f"Error in handle_subscribe_logs: {e}", exc_info=True) + @self.socketio.on('ping') def handle_ping(): """Handle client ping (modern ping/pong pattern)""" @@ -2655,7 +3355,7 @@ class BotDataViewer: emit('pong') # Server responds with pong (Flask-SocketIO 5.x pattern) except Exception as e: self.logger.error(f"Error in handle_ping: {e}", exc_info=True) - + @self.socketio.on_error_default def default_error_handler(e): """Handle SocketIO errors gracefully""" @@ -2667,7 +3367,7 @@ class BotDataViewer: except Exception as emit_error: # If we can't emit, just log it self.logger.error(f"Error emitting error message: {emit_error}") - + def _handle_command_data(self, command_data): """Handle incoming command data from bot""" try: @@ -2677,13 +3377,13 @@ class BotDataViewer: client_id for client_id, client_info in self.connected_clients.items() if client_info.get('subscribed_commands', False) ] - + if subscribed_clients: self.socketio.emit('command_data', command_data, room=None) self.logger.debug(f"Broadcasted command data to {len(subscribed_clients)} clients") except Exception as e: self.logger.error(f"Error handling command data: {e}") - + def _handle_packet_data(self, packet_data): """Handle incoming packet data from bot""" try: @@ -2693,13 +3393,13 @@ class BotDataViewer: client_id for client_id, client_info in self.connected_clients.items() if client_info.get('subscribed_packets', False) ] - + if subscribed_clients: self.socketio.emit('packet_data', packet_data, room=None) self.logger.debug(f"Broadcasted packet data to {len(subscribed_clients)} clients") except Exception as e: self.logger.error(f"Error handling packet data: {e}") - + def _handle_mesh_edge_data(self, edge_data): """Handle incoming mesh edge data from bot""" try: @@ -2709,13 +3409,13 @@ class BotDataViewer: client_id for client_id, client_info in self.connected_clients.items() if client_info.get('subscribed_mesh', False) ] - + if subscribed_clients: event_type = 'mesh_edge_added' if edge_data.get('is_new', False) else 'mesh_edge_updated' self.socketio.emit(event_type, edge_data, room=None) except Exception as e: self.logger.error(f"Error handling mesh edge data: {e}", exc_info=True) - + def _handle_mesh_node_data(self, node_data): """Handle incoming mesh node data from bot""" try: @@ -2725,27 +3425,103 @@ class BotDataViewer: client_id for client_id, client_info in self.connected_clients.items() if client_info.get('subscribed_mesh', False) ] - + if subscribed_clients: self.socketio.emit('mesh_node_added', node_data, room=None) except Exception as e: self.logger.error(f"Error handling mesh node data: {e}", exc_info=True) - + + def _handle_message_data(self, msg_data): + """Broadcast a captured channel message to subscribed clients.""" + try: + with self._clients_lock: + subscribed_clients = [ + client_id for client_id, client_info in self.connected_clients.items() + if client_info.get('subscribed_messages', False) + ] + if subscribed_clients: + self.socketio.emit('message_data', msg_data, room=None) + except Exception as e: + self.logger.error(f"Error handling message data: {e}") + + def _handle_log_line(self, line: str) -> None: + """Broadcast a log line to clients subscribed to the log stream.""" + try: + with self._clients_lock: + subscribed = [ + cid for cid, info in self.connected_clients.items() + if info.get('subscribed_logs', False) + ] + if subscribed: + self.socketio.emit('log_line', {'line': line.rstrip()}, room=None) + except Exception as e: + self.logger.error(f"Error broadcasting log line: {e}") + + def _start_log_tailing(self) -> None: + """Start a background thread that tails the bot log file and emits SocketIO events.""" + import os + import threading + + log_file = '' + try: + log_file = self.config.get('Logging', 'log_file', fallback='').strip() + if log_file: + log_file = str(resolve_path(log_file, self.bot_root)) + except Exception: + pass + + if not log_file: + self.logger.info("Log tailing disabled: no log_file configured") + return + + def tail_log(): + import time as _time + self.logger.info(f"Log tail thread started: {log_file}") + pos = 0 + # Start at end of file so we only stream new lines + try: + pos = os.path.getsize(log_file) + except OSError: + pass + while True: + try: + if not os.path.exists(log_file): + _time.sleep(2) + continue + current_size = os.path.getsize(log_file) + if current_size < pos: + # File rotated — start from beginning + pos = 0 + if current_size > pos: + with open(log_file, encoding='utf-8', errors='replace') as fh: + fh.seek(pos) + for line in fh: + self._handle_log_line(line) + pos = fh.tell() + except Exception as e: + self.logger.debug(f"Log tail error: {e}") + _time.sleep(1) + + tail_thread = threading.Thread(target=tail_log, daemon=True) + tail_thread.start() + self.logger.info("Log tailing started") + def _start_database_polling(self): """Start background thread to poll database for new data""" import threading - + def poll_database(): - last_timestamp = 0 + import time as _time + last_timestamp = _time.time() - 300 # start 5 min back; subscribe handlers replay full history consecutive_errors = 0 max_consecutive_errors = 10 - + while True: try: - import time - import sqlite3 import json - + import sqlite3 + import time + # Check if database file exists and is accessible db_file = Path(self.db_path) if not db_file.exists(): @@ -2754,37 +3530,37 @@ class BotDataViewer: self.logger.warning(f"Database file does not exist: {self.db_path}") time.sleep(5) continue - + if not os.access(self.db_path, os.R_OK): consecutive_errors += 1 if consecutive_errors == 1 or consecutive_errors % 10 == 0: self.logger.warning(f"Database file is not readable: {self.db_path}") time.sleep(5) continue - + # Connect to database with timeout to prevent hanging try: with closing(sqlite3.connect(self.db_path, timeout=60, check_same_thread=False)) as conn: conn.row_factory = sqlite3.Row cursor = conn.cursor() - + # Get new data since last poll cursor.execute(''' - SELECT timestamp, data, type FROM packet_stream - WHERE timestamp > ? + SELECT timestamp, data, type FROM packet_stream + WHERE timestamp > ? ORDER BY timestamp ASC ''', (last_timestamp,)) - + rows = cursor.fetchall() - + # Process new data for row in rows: try: - timestamp = row[0] + row[0] data_json = row[1] data_type = row[2] data = json.loads(data_json) - + # Broadcast based on type if data_type == 'command': self._handle_command_data(data) @@ -2792,14 +3568,16 @@ class BotDataViewer: self._handle_packet_data(data) elif data_type == 'routing': self._handle_packet_data(data) # Treat routing as packet data - + elif data_type == 'message': + self._handle_message_data(data) + except Exception as e: self.logger.warning(f"Error processing database data: {e}") - + # Update last timestamp if rows: last_timestamp = rows[-1][0] - + # Reset error counter on success consecutive_errors = 0 except sqlite3.OperationalError as conn_error: @@ -2811,14 +3589,14 @@ class BotDataViewer: time.sleep(2) continue raise # Re-raise non-locked OperationalErrors for outer handler to log/backoff - + # Sleep before next poll (back off to reduce lock contention with bot writes) time.sleep(2.0) # Poll every 2s - + except sqlite3.OperationalError as e: consecutive_errors += 1 error_msg = str(e) - + # Provide more diagnostic information on first error or periodic errors if consecutive_errors == 1 or consecutive_errors % 10 == 0: db_file = Path(self.db_path) @@ -2832,7 +3610,7 @@ class BotDataViewer: f" Readable: {readable}\n" f" Writable: {writable}" ) - + # Log at appropriate level based on error frequency if consecutive_errors >= max_consecutive_errors: if consecutive_errors == max_consecutive_errors: @@ -2845,7 +3623,7 @@ class BotDataViewer: else: self.logger.debug(f"Database polling error (attempt {consecutive_errors}): {error_msg}") time.sleep(1) # Wait longer on error - + except Exception as e: consecutive_errors += 1 if consecutive_errors >= max_consecutive_errors: @@ -2855,17 +3633,17 @@ class BotDataViewer: else: self.logger.warning(f"Database polling unexpected error (attempt {consecutive_errors}): {e}") time.sleep(2) - - + + # Start polling thread polling_thread = threading.Thread(target=poll_database, daemon=True) polling_thread.start() self.logger.info("Database polling started") - + def _start_cleanup_scheduler(self): """Start background thread for periodic database cleanup""" import threading - + def cleanup_scheduler(): import time while True: @@ -2874,40 +3652,40 @@ class BotDataViewer: for _ in range(12): # 12 x 5 minutes = 1 hour time.sleep(300) # 5 minutes self._cleanup_stale_clients() - + # Clean up old data every hour (after 12 stale client cleanups) self._cleanup_old_data() - + except Exception as e: self.logger.error(f"Error in cleanup scheduler: {e}", exc_info=True) time.sleep(60) # Sleep on error - + # Start the cleanup thread cleanup_thread = threading.Thread(target=cleanup_scheduler, daemon=True) cleanup_thread.start() self.logger.info("Cleanup scheduler started") - + def _cleanup_stale_clients(self, max_idle_seconds: int = 300): """Remove clients that haven't had activity in max_idle_seconds""" try: current_time = time.time() stale_clients = [] - + with self._clients_lock: for client_id, client_info in self.connected_clients.items(): last_activity = client_info.get('last_activity', 0) if current_time - last_activity > max_idle_seconds: stale_clients.append(client_id) - + for client_id in stale_clients: del self.connected_clients[client_id] - + if stale_clients: self.logger.info(f"Cleaned up {len(stale_clients)} stale client(s)") - + except Exception as e: self.logger.error(f"Error cleaning up stale clients: {e}") - + def _cleanup_old_data(self, days_to_keep: Optional[int] = None): """Clean up old packet stream data to prevent database bloat. Uses [Data_Retention] packet_stream_retention_days when days_to_keep is not provided.""" @@ -2918,27 +3696,25 @@ class BotDataViewer: if days_to_keep is None: days_to_keep = 3 if self.config.has_section('Data_Retention') and self.config.has_option('Data_Retention', 'packet_stream_retention_days'): - try: + with suppress(ValueError, TypeError): days_to_keep = self.config.getint('Data_Retention', 'packet_stream_retention_days') - except (ValueError, TypeError): - pass cutoff_time = time.time() - (days_to_keep * 24 * 60 * 60) - + # Use DEFERRED isolation; longer timeout to wait out bot writes with closing(sqlite3.connect(self.db_path, timeout=60, isolation_level='DEFERRED')) as conn: cursor = conn.cursor() - + # Use WAL mode for better concurrent access (if not already set) try: cursor.execute('PRAGMA journal_mode=WAL') except sqlite3.OperationalError: pass # Ignore if database is locked - WAL may already be set - + # Delete in smaller batches to avoid long locks batch_size = 1000 total_deleted = 0 - + while True: cursor.execute( 'DELETE FROM packet_stream WHERE id IN ' @@ -2947,90 +3723,90 @@ class BotDataViewer: ) deleted_count = cursor.rowcount conn.commit() - + if deleted_count == 0: break total_deleted += deleted_count if deleted_count == batch_size: time.sleep(0.1) - + if total_deleted > 0: self.logger.info(f"Cleaned up {total_deleted} old packet stream entries (older than {days_to_keep} days)") - + except sqlite3.OperationalError as e: self.logger.warning(f"Database busy during cleanup (will retry next cycle): {e}") except Exception as e: self.logger.error(f"Error cleaning up old packet stream data: {e}", exc_info=True) - - def _get_database_stats(self, top_users_window='all', top_commands_window='all', + + def _get_database_stats(self, top_users_window='all', top_commands_window='all', top_paths_window='all', top_channels_window='all'): """Get comprehensive database statistics for dashboard""" conn = None try: conn = self._get_db_connection() cursor = conn.cursor() - + # Get all available tables cursor.execute("SELECT name FROM sqlite_master WHERE type='table'") tables = [row[0] for row in cursor.fetchall()] - + with self._clients_lock: client_count = len(self.connected_clients) - + stats = { 'timestamp': time.time(), 'connected_clients': client_count, 'tables': tables } - + # Contact and tracking statistics if 'complete_contact_tracking' in tables: cursor.execute("SELECT COUNT(*) FROM complete_contact_tracking") stats['total_contacts'] = cursor.fetchone()[0] - + cursor.execute(""" - SELECT COUNT(*) FROM complete_contact_tracking + SELECT COUNT(*) FROM complete_contact_tracking WHERE last_heard > datetime('now', '-24 hours') """) stats['contacts_24h'] = cursor.fetchone()[0] - + cursor.execute(""" - SELECT COUNT(*) FROM complete_contact_tracking + SELECT COUNT(*) FROM complete_contact_tracking WHERE last_heard > datetime('now', '-7 days') """) stats['contacts_7d'] = cursor.fetchone()[0] - + cursor.execute(""" - SELECT COUNT(*) FROM complete_contact_tracking + SELECT COUNT(*) FROM complete_contact_tracking WHERE is_currently_tracked = 1 """) stats['tracked_contacts'] = cursor.fetchone()[0] - + cursor.execute(""" - SELECT AVG(hop_count) FROM complete_contact_tracking + SELECT AVG(hop_count) FROM complete_contact_tracking WHERE hop_count IS NOT NULL """) avg_hops = cursor.fetchone()[0] stats['avg_hop_count'] = round(avg_hops, 1) if avg_hops else 0 - + cursor.execute(""" - SELECT MAX(hop_count) FROM complete_contact_tracking + SELECT MAX(hop_count) FROM complete_contact_tracking WHERE hop_count IS NOT NULL """) stats['max_hop_count'] = cursor.fetchone()[0] or 0 - + cursor.execute(""" - SELECT COUNT(DISTINCT role) FROM complete_contact_tracking + SELECT COUNT(DISTINCT role) FROM complete_contact_tracking WHERE role IS NOT NULL """) stats['unique_roles'] = cursor.fetchone()[0] - + cursor.execute(""" - SELECT COUNT(DISTINCT device_type) FROM complete_contact_tracking + SELECT COUNT(DISTINCT device_type) FROM complete_contact_tracking WHERE device_type IS NOT NULL """) stats['unique_device_types'] = cursor.fetchone()[0] - + # Advertisement statistics using daily tracking table if 'daily_stats' in tables: # Total advertisements (all time) @@ -3039,34 +3815,34 @@ class BotDataViewer: """) total_adverts = cursor.fetchone()[0] stats['total_advertisements'] = total_adverts or 0 - + # 24h advertisements cursor.execute(""" - SELECT SUM(advert_count) FROM daily_stats + SELECT SUM(advert_count) FROM daily_stats WHERE date = date('now') """) stats['advertisements_24h'] = cursor.fetchone()[0] or 0 - + # 7d advertisements (last 7 days, excluding today) cursor.execute(""" - SELECT SUM(advert_count) FROM daily_stats + SELECT SUM(advert_count) FROM daily_stats WHERE date >= date('now', '-7 days') AND date < date('now') """) stats['advertisements_7d'] = cursor.fetchone()[0] or 0 - + # Nodes per day statistics cursor.execute(""" - SELECT COUNT(DISTINCT public_key) FROM daily_stats + SELECT COUNT(DISTINCT public_key) FROM daily_stats WHERE date = date('now') """) stats['nodes_24h'] = cursor.fetchone()[0] or 0 - + cursor.execute(""" - SELECT COUNT(DISTINCT public_key) FROM daily_stats + SELECT COUNT(DISTINCT public_key) FROM daily_stats WHERE date >= date('now', '-6 days') """) stats['nodes_7d'] = cursor.fetchone()[0] or 0 - + cursor.execute(""" SELECT COUNT(DISTINCT public_key) FROM daily_stats """) @@ -3079,69 +3855,69 @@ class BotDataViewer: """) total_adverts = cursor.fetchone()[0] stats['total_advertisements'] = total_adverts or 0 - + cursor.execute(""" - SELECT SUM(advert_count) FROM complete_contact_tracking + SELECT SUM(advert_count) FROM complete_contact_tracking WHERE last_heard > datetime('now', '-24 hours') """) stats['advertisements_24h'] = cursor.fetchone()[0] or 0 - + cursor.execute(""" - SELECT SUM(advert_count) FROM complete_contact_tracking + SELECT SUM(advert_count) FROM complete_contact_tracking WHERE last_heard > datetime('now', '-7 days') """) stats['advertisements_7d'] = cursor.fetchone()[0] or 0 - + # Repeater contacts (if exists) if 'repeater_contacts' in tables: cursor.execute("SELECT COUNT(*) FROM repeater_contacts") stats['repeater_contacts'] = cursor.fetchone()[0] - + cursor.execute("SELECT COUNT(*) FROM repeater_contacts WHERE is_active = 1") stats['active_repeater_contacts'] = cursor.fetchone()[0] - + # Cache statistics cache_tables = [t for t in tables if 'cache' in t] stats['cache_tables'] = cache_tables stats['total_cache_entries'] = 0 stats['active_cache_entries'] = 0 - + for table in cache_tables: cursor.execute(f"SELECT COUNT(*) FROM {table}") count = cursor.fetchone()[0] stats['total_cache_entries'] += count stats[f'{table}_count'] = count - + # Get active entries (not expired) cursor.execute(f"SELECT COUNT(*) FROM {table} WHERE expires_at > datetime('now')") active_count = cursor.fetchone()[0] stats['active_cache_entries'] += active_count stats[f'{table}_active'] = active_count - + # Message and command statistics (if stats tables exist) if 'message_stats' in tables: cursor.execute("SELECT COUNT(*) FROM message_stats") stats['total_messages'] = cursor.fetchone()[0] - + cursor.execute(""" - SELECT COUNT(*) FROM message_stats + SELECT COUNT(*) FROM message_stats WHERE timestamp > strftime('%s', 'now', '-24 hours') """) stats['messages_24h'] = cursor.fetchone()[0] - + cursor.execute(""" - SELECT COUNT(DISTINCT sender_id) FROM message_stats + SELECT COUNT(DISTINCT sender_id) FROM message_stats WHERE timestamp > strftime('%s', 'now', '-24 hours') """) stats['unique_senders_24h'] = cursor.fetchone()[0] - + # Total unique users and channels cursor.execute("SELECT COUNT(DISTINCT sender_id) FROM message_stats") stats['unique_users_total'] = cursor.fetchone()[0] - + cursor.execute("SELECT COUNT(DISTINCT channel) FROM message_stats WHERE channel IS NOT NULL") stats['unique_channels_total'] = cursor.fetchone()[0] - + # Top users (most frequent message senders) - filter by time window if top_users_window == '24h': time_filter = "WHERE timestamp > strftime('%s', 'now', '-24 hours')" @@ -3151,28 +3927,28 @@ class BotDataViewer: time_filter = "WHERE timestamp > strftime('%s', 'now', '-30 days')" else: # 'all' time_filter = "" - + query = f""" - SELECT sender_id, COUNT(*) as count - FROM message_stats + SELECT sender_id, COUNT(*) as count + FROM message_stats {time_filter} - GROUP BY sender_id - ORDER BY count DESC + GROUP BY sender_id + ORDER BY count DESC LIMIT 15 """ cursor.execute(query) stats['top_users'] = [{'user': row[0], 'count': row[1]} for row in cursor.fetchall()] - + if 'command_stats' in tables: cursor.execute("SELECT COUNT(*) FROM command_stats") stats['total_commands'] = cursor.fetchone()[0] - + cursor.execute(""" - SELECT COUNT(*) FROM command_stats + SELECT COUNT(*) FROM command_stats WHERE timestamp > strftime('%s', 'now', '-24 hours') """) stats['commands_24h'] = cursor.fetchone()[0] - + # Top commands - filter by time window if top_commands_window == '24h': time_filter = "WHERE timestamp > strftime('%s', 'now', '-24 hours')" @@ -3182,27 +3958,27 @@ class BotDataViewer: time_filter = "WHERE timestamp > strftime('%s', 'now', '-30 days')" else: # 'all' time_filter = "" - + query = f""" - SELECT command_name, COUNT(*) as count - FROM command_stats + SELECT command_name, COUNT(*) as count + FROM command_stats {time_filter} - GROUP BY command_name - ORDER BY count DESC + GROUP BY command_name + ORDER BY count DESC LIMIT 15 """ cursor.execute(query) stats['top_commands'] = [{'command': row[0], 'count': row[1]} for row in cursor.fetchall()] - + # Bot reply rates (commands that got responses) - calculate for different time windows # 24 hour reply rate cursor.execute(""" - SELECT COUNT(*) FROM command_stats + SELECT COUNT(*) FROM command_stats WHERE timestamp > strftime('%s', 'now', '-24 hours') AND response_sent = 1 """) replied_24h = cursor.fetchone()[0] cursor.execute(""" - SELECT COUNT(*) FROM command_stats + SELECT COUNT(*) FROM command_stats WHERE timestamp > strftime('%s', 'now', '-24 hours') """) total_24h = cursor.fetchone()[0] @@ -3210,15 +3986,15 @@ class BotDataViewer: stats['bot_reply_rate_24h'] = round((replied_24h / total_24h) * 100, 1) else: stats['bot_reply_rate_24h'] = 0 - + # 7 day reply rate cursor.execute(""" - SELECT COUNT(*) FROM command_stats + SELECT COUNT(*) FROM command_stats WHERE timestamp > strftime('%s', 'now', '-7 days') AND response_sent = 1 """) replied_7d = cursor.fetchone()[0] cursor.execute(""" - SELECT COUNT(*) FROM command_stats + SELECT COUNT(*) FROM command_stats WHERE timestamp > strftime('%s', 'now', '-7 days') """) total_7d = cursor.fetchone()[0] @@ -3226,15 +4002,15 @@ class BotDataViewer: stats['bot_reply_rate_7d'] = round((replied_7d / total_7d) * 100, 1) else: stats['bot_reply_rate_7d'] = 0 - + # 30 day reply rate cursor.execute(""" - SELECT COUNT(*) FROM command_stats + SELECT COUNT(*) FROM command_stats WHERE timestamp > strftime('%s', 'now', '-30 days') AND response_sent = 1 """) replied_30d = cursor.fetchone()[0] cursor.execute(""" - SELECT COUNT(*) FROM command_stats + SELECT COUNT(*) FROM command_stats WHERE timestamp > strftime('%s', 'now', '-30 days') """) total_30d = cursor.fetchone()[0] @@ -3242,7 +4018,7 @@ class BotDataViewer: stats['bot_reply_rate_30d'] = round((replied_30d / total_30d) * 100, 1) else: stats['bot_reply_rate_30d'] = 0 - + # Top channels by message count - filter by time window if top_channels_window == '24h': time_filter = "AND timestamp > strftime('%s', 'now', '-24 hours')" @@ -3252,27 +4028,27 @@ class BotDataViewer: time_filter = "AND timestamp > strftime('%s', 'now', '-30 days')" else: # 'all' time_filter = "" - + query = f""" SELECT channel, COUNT(*) as message_count, COUNT(DISTINCT sender_id) as unique_users - FROM message_stats + FROM message_stats WHERE channel IS NOT NULL {time_filter} - GROUP BY channel - ORDER BY message_count DESC + GROUP BY channel + ORDER BY message_count DESC LIMIT 10 """ cursor.execute(query) stats['top_channels'] = [ - {'channel': row[0], 'messages': row[1], 'users': row[2]} + {'channel': row[0], 'messages': row[1], 'users': row[2]} for row in cursor.fetchall() ] - + # Path statistics (if path_stats table exists) if 'path_stats' in tables: cursor.execute(""" SELECT sender_id, path_length, path_string, timestamp - FROM path_stats - ORDER BY path_length DESC + FROM path_stats + ORDER BY path_length DESC LIMIT 1 """) longest_path = cursor.fetchone() @@ -3283,7 +4059,7 @@ class BotDataViewer: 'path_string': longest_path[2], 'timestamp': longest_path[3] } - + # Top paths (longest paths) - filter by time window if top_paths_window == '24h': time_filter = "WHERE timestamp > strftime('%s', 'now', '-24 hours')" @@ -3293,123 +4069,123 @@ class BotDataViewer: time_filter = "WHERE timestamp > strftime('%s', 'now', '-30 days')" else: # 'all' time_filter = "" - + query = f""" SELECT sender_id, path_length, path_string, timestamp - FROM path_stats + FROM path_stats {time_filter} - ORDER BY path_length DESC + ORDER BY path_length DESC LIMIT 5 """ cursor.execute(query) stats['top_paths'] = [ { - 'user': row[0], - 'path_length': row[1], - 'path_string': row[2], + 'user': row[0], + 'path_length': row[1], + 'path_string': row[2], 'timestamp': row[3] - } + } for row in cursor.fetchall() ] - + # Network health metrics if 'complete_contact_tracking' in tables: cursor.execute(""" - SELECT AVG(snr) FROM complete_contact_tracking + SELECT AVG(snr) FROM complete_contact_tracking WHERE snr IS NOT NULL AND last_heard > datetime('now', '-24 hours') """) avg_snr = cursor.fetchone()[0] stats['avg_snr_24h'] = round(avg_snr, 1) if avg_snr else 0 - + cursor.execute(""" - SELECT AVG(signal_strength) FROM complete_contact_tracking + SELECT AVG(signal_strength) FROM complete_contact_tracking WHERE signal_strength IS NOT NULL AND last_heard > datetime('now', '-24 hours') """) avg_signal = cursor.fetchone()[0] stats['avg_signal_strength_24h'] = round(avg_signal, 1) if avg_signal else 0 - + # Geographic distribution - only count currently tracked contacts heard in the last 30 days # Normalize country names to avoid duplicates (e.g., "United States" vs "United States of America") if 'complete_contact_tracking' in tables: cursor.execute(""" - SELECT COUNT(DISTINCT - CASE - WHEN country IN ('United States', 'United States of America', 'US', 'USA') + SELECT COUNT(DISTINCT + CASE + WHEN country IN ('United States', 'United States of America', 'US', 'USA') THEN 'United States' ELSE country END - ) FROM complete_contact_tracking + ) FROM complete_contact_tracking WHERE country IS NOT NULL AND country != '' AND last_heard > datetime('now', '-30 days') AND is_currently_tracked = 1 """) stats['countries'] = cursor.fetchone()[0] - + cursor.execute(""" - SELECT COUNT(DISTINCT state) FROM complete_contact_tracking + SELECT COUNT(DISTINCT state) FROM complete_contact_tracking WHERE state IS NOT NULL AND state != '' AND last_heard > datetime('now', '-30 days') AND is_currently_tracked = 1 """) stats['states'] = cursor.fetchone()[0] - + cursor.execute(""" - SELECT COUNT(DISTINCT city) FROM complete_contact_tracking + SELECT COUNT(DISTINCT city) FROM complete_contact_tracking WHERE city IS NOT NULL AND city != '' AND last_heard > datetime('now', '-30 days') AND is_currently_tracked = 1 """) stats['cities'] = cursor.fetchone()[0] - + return stats - + except Exception as e: self.logger.error(f"Error getting database stats: {e}") return {'error': str(e)} finally: if conn: conn.close() - + def _get_database_info(self): """Get comprehensive database information for database page""" conn = None try: conn = self._get_db_connection() cursor = conn.cursor() - + # Get all tables cursor.execute("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name") table_names = [row[0] for row in cursor.fetchall()] - + # Get table information tables = [] total_records = 0 - + for table_name in table_names: try: # Get record count cursor.execute(f"SELECT COUNT(*) FROM {table_name}") record_count = cursor.fetchone()[0] total_records += record_count - + # Get table size (approximate) cursor.execute(f"PRAGMA table_info({table_name})") columns = cursor.fetchall() - + # Estimate size (rough calculation) estimated_size = record_count * len(columns) * 50 # Rough estimate size_str = f"{estimated_size:,} bytes" if estimated_size < 1024 else f"{estimated_size/1024:.1f} KB" - + # Get table description based on name description = self._get_table_description(table_name) - + tables.append({ 'name': table_name, 'record_count': record_count, 'size': size_str, 'description': description }) - + except Exception as e: self.logger.debug(f"Error getting info for table {table_name}: {e}") tables.append({ @@ -3418,7 +4194,7 @@ class BotDataViewer: 'size': 'Unknown', 'description': 'Error reading table' }) - + # Get database file size import os try: @@ -3431,7 +4207,7 @@ class BotDataViewer: db_size = f"{db_size_bytes/(1024*1024):.1f} MB" except: db_size = "Unknown" - + return { 'total_tables': len(table_names), 'total_records': total_records, @@ -3439,7 +4215,7 @@ class BotDataViewer: 'db_size': db_size, 'tables': tables } - + except Exception as e: self.logger.error(f"Error getting database info: {e}") return { @@ -3452,7 +4228,7 @@ class BotDataViewer: finally: if conn: conn.close() - + def _get_table_description(self, table_name): """Get human-readable description for table""" descriptions = { @@ -3466,32 +4242,32 @@ class BotDataViewer: 'generic_cache': 'General purpose cache storage' } return descriptions.get(table_name, 'Database table') - + def _optimize_database(self): """Optimize database using VACUUM, ANALYZE, and REINDEX""" conn = None try: conn = self._get_db_connection() cursor = conn.cursor() - + # Get initial database size import os initial_size = os.path.getsize(self.db_path) - + # Perform VACUUM to reclaim unused space self.logger.info("Starting database VACUUM...") cursor.execute("VACUUM") vacuum_size = os.path.getsize(self.db_path) vacuum_saved = initial_size - vacuum_size - + # Perform ANALYZE to update table statistics self.logger.info("Starting database ANALYZE...") cursor.execute("ANALYZE") - + # Get all tables for REINDEX cursor.execute("SELECT name FROM sqlite_master WHERE type='table'") tables = [row[0] for row in cursor.fetchall()] - + # Perform REINDEX on all tables self.logger.info("Starting database REINDEX...") reindexed_tables = [] @@ -3502,11 +4278,11 @@ class BotDataViewer: reindexed_tables.append(table) except Exception as e: self.logger.debug(f"Could not reindex table {table}: {e}") - + # Get final database size final_size = os.path.getsize(self.db_path) total_saved = initial_size - final_size - + # Format size information def format_size(size_bytes): if size_bytes < 1024: @@ -3515,7 +4291,7 @@ class BotDataViewer: return f"{size_bytes/1024:.1f} KB" else: return f"{size_bytes/(1024*1024):.1f} MB" - + return { 'success': True, 'vacuum_result': f"VACUUM completed - saved {format_size(vacuum_saved)}", @@ -3527,7 +4303,7 @@ class BotDataViewer: 'tables_processed': len(tables), 'tables_reindexed': len(reindexed_tables) } - + except Exception as e: self.logger.error(f"Error optimizing database: {e}") return { @@ -3537,18 +4313,18 @@ class BotDataViewer: finally: if conn: conn.close() - + def _get_tracking_data(self, since='30d'): """Get contact tracking data. since: 24h, 7d, 30d, 90d, or all (heard in that window).""" conn = None try: conn = self._get_db_connection() cursor = conn.cursor() - + # Get bot location from config bot_lat = self.config.getfloat('Bot', 'bot_latitude', fallback=None) bot_lon = self.config.getfloat('Bot', 'bot_longitude', fallback=None) - + # Filter by last_heard for performance (default: last 30 days) if since == 'all': where_clause = '' @@ -3563,7 +4339,7 @@ class BotDataViewer: else: # 90d where_clause = " WHERE c.last_heard >= datetime('now', '-90 days')" params = () - + # Query with LEFT JOIN to a limited set of paths per contact (max 50 most recent per contact) # to keep GROUP_CONCAT and load time bounded when observed_paths is large. cursor.execute(""" @@ -3573,8 +4349,8 @@ class BotDataViewer: FROM observed_paths WHERE packet_type = 'advert' AND public_key IS NOT NULL ) - SELECT - c.public_key, c.name, c.role, c.device_type, + SELECT + c.public_key, c.name, c.role, c.device_type, c.latitude, c.longitude, c.city, c.state, c.country, c.snr, c.hop_count, c.first_heard, c.last_heard, c.advert_count, c.is_currently_tracked, @@ -3593,7 +4369,7 @@ class BotDataViewer: FROM recent_paths WHERE rn <= 50 ) op ON c.public_key = op.public_key """ + where_clause + """ - GROUP BY c.public_key, c.name, c.role, c.device_type, + GROUP BY c.public_key, c.name, c.role, c.device_type, c.latitude, c.longitude, c.city, c.state, c.country, c.snr, c.hop_count, c.first_heard, c.last_heard, c.advert_count, c.is_currently_tracked, @@ -3601,7 +4377,7 @@ class BotDataViewer: c.out_path, c.out_path_len, c.out_bytes_per_hop ORDER BY c.last_heard DESC """, params) - + tracking = [] for row in cursor.fetchall(): # Parse raw advertisement data if available @@ -3612,13 +4388,13 @@ class BotDataViewer: raw_advert_data_parsed = json.loads(row['raw_advert_data']) except: raw_advert_data_parsed = None - + # Calculate distance if both bot and contact have coordinates distance = None - if (bot_lat is not None and bot_lon is not None and + if (bot_lat is not None and bot_lon is not None and row['latitude'] is not None and row['longitude'] is not None): distance = self._calculate_distance(bot_lat, bot_lon, row['latitude'], row['longitude']) - + # Parse all_paths from concatenated strings all_paths = [] if row['all_paths_hex']: @@ -3627,7 +4403,7 @@ class BotDataViewer: paths_bph = row['all_paths_bytes_per_hop'].split('|||') if row['all_paths_bytes_per_hop'] else [] paths_observations = row['all_paths_observations'].split('|||') if row['all_paths_observations'] else [] paths_last_seen = row['all_paths_last_seen'].split('|||') if row['all_paths_last_seen'] else [] - + for i, path_hex in enumerate(paths_hex): if path_hex: # Skip empty strings bph = None @@ -3645,7 +4421,7 @@ class BotDataViewer: 'observation_count': int(paths_observations[i]) if i < len(paths_observations) and paths_observations[i] else 1, 'last_seen': paths_last_seen[i] if i < len(paths_last_seen) and paths_last_seen[i] else None }) - + tracking.append({ 'user_id': row['public_key'], 'username': row['name'], @@ -3674,7 +4450,7 @@ class BotDataViewer: 'out_bytes_per_hop': row['out_bytes_per_hop'] if row['out_bytes_per_hop'] is not None else None, 'all_paths': all_paths }) - + # Get server statistics for daily tracking using direct database queries server_stats = {} try: @@ -3683,37 +4459,37 @@ class BotDataViewer: if cursor.fetchone(): # 24h: Last 24 hours of advertisements cursor.execute(""" - SELECT SUM(advert_count) FROM daily_stats + SELECT SUM(advert_count) FROM daily_stats WHERE date >= date('now', '-1 day') """) server_stats['advertisements_24h'] = cursor.fetchone()[0] or 0 - + # 7d: Previous 6 days (excluding today) cursor.execute(""" - SELECT SUM(advert_count) FROM daily_stats + SELECT SUM(advert_count) FROM daily_stats WHERE date >= date('now', '-7 days') AND date < date('now') """) server_stats['advertisements_7d'] = cursor.fetchone()[0] or 0 - + # All: Everything cursor.execute(""" SELECT SUM(advert_count) FROM daily_stats """) server_stats['total_advertisements'] = cursor.fetchone()[0] or 0 - + # Nodes per day statistics # Calculate today's unique nodes from complete_contact_tracking # (last_heard in last 24 hours) since daily_stats might not have today's data yet cursor.execute(""" - SELECT COUNT(DISTINCT public_key) FROM complete_contact_tracking + SELECT COUNT(DISTINCT public_key) FROM complete_contact_tracking WHERE last_heard >= datetime('now', '-24 hours') """) server_stats['nodes_24h'] = cursor.fetchone()[0] or 0 - + # Get today's unique nodes by role for the stacked chart cursor.execute(""" SELECT role, COUNT(DISTINCT public_key) as count - FROM complete_contact_tracking + FROM complete_contact_tracking WHERE last_heard >= datetime('now', '-24 hours') AND role IS NOT NULL AND role != '' GROUP BY role @@ -3723,7 +4499,7 @@ class BotDataViewer: role = row[0].lower() if row[0] else 'unknown' count = row[1] today_by_role[role] = count - + server_stats['nodes_24h_by_role'] = { 'companion': today_by_role.get('companion', 0), 'repeater': today_by_role.get('repeater', 0), @@ -3731,56 +4507,56 @@ class BotDataViewer: 'sensor': today_by_role.get('sensor', 0), 'other': sum(v for k, v in today_by_role.items() if k not in ['companion', 'repeater', 'roomserver', 'sensor']) } - + cursor.execute(""" - SELECT COUNT(DISTINCT public_key) FROM daily_stats + SELECT COUNT(DISTINCT public_key) FROM daily_stats WHERE date >= date('now', '-7 days') AND date < date('now') """) server_stats['nodes_7d'] = cursor.fetchone()[0] or 0 - + # Calculate day-over-day and period-over-period comparisons # Today vs 7 days ago (single day comparison) cursor.execute(""" - SELECT COUNT(DISTINCT public_key) FROM daily_stats + SELECT COUNT(DISTINCT public_key) FROM daily_stats WHERE date = date('now', '-7 days') """) result = cursor.fetchone() server_stats['nodes_7d_ago'] = result[0] if result and result[0] else 0 - + # Last 7 days vs previous 7 days (days 8-14 ago) cursor.execute(""" - SELECT COUNT(DISTINCT public_key) FROM daily_stats + SELECT COUNT(DISTINCT public_key) FROM daily_stats WHERE date >= date('now', '-14 days') AND date < date('now', '-7 days') """) result = cursor.fetchone() server_stats['nodes_prev_7d'] = result[0] if result and result[0] else 0 - + # Last 30 days vs previous 30 days (days 31-60 ago) cursor.execute(""" - SELECT COUNT(DISTINCT public_key) FROM daily_stats + SELECT COUNT(DISTINCT public_key) FROM daily_stats WHERE date >= date('now', '-60 days') AND date < date('now', '-30 days') """) result = cursor.fetchone() server_stats['nodes_prev_30d'] = result[0] if result and result[0] else 0 - + # Also get current period totals for comparison cursor.execute(""" - SELECT COUNT(DISTINCT public_key) FROM daily_stats + SELECT COUNT(DISTINCT public_key) FROM daily_stats WHERE date >= date('now', '-7 days') """) server_stats['nodes_7d'] = cursor.fetchone()[0] or 0 - + cursor.execute(""" - SELECT COUNT(DISTINCT public_key) FROM daily_stats + SELECT COUNT(DISTINCT public_key) FROM daily_stats WHERE date >= date('now', '-30 days') """) server_stats['nodes_30d'] = cursor.fetchone()[0] or 0 - + cursor.execute(""" SELECT COUNT(DISTINCT public_key) FROM daily_stats """) server_stats['nodes_all'] = cursor.fetchone()[0] or 0 - + # Get daily unique node counts by role for the last 30 days for the stacked graph # Join daily_stats with complete_contact_tracking to get role information # This gives us accurate historical daily counts by role @@ -3794,18 +4570,18 @@ class BotDataViewer: ORDER BY ds.date ASC, c.role ASC """) daily_data_by_role = cursor.fetchall() - + # Organize data by date and role daily_by_role = {} for row in daily_data_by_role: date_str = row[0] role = (row[1] or 'unknown').lower() count = row[2] - + if date_str not in daily_by_role: daily_by_role[date_str] = {} daily_by_role[date_str][role] = count - + # Convert to array format with all roles for each date server_stats['daily_nodes_30d_by_role'] = [] for date_str in sorted(daily_by_role.keys()): @@ -3818,24 +4594,24 @@ class BotDataViewer: 'sensor': roles_data.get('sensor', 0), 'other': sum(v for k, v in roles_data.items() if k not in ['companion', 'repeater', 'roomserver', 'sensor']) }) - + # Also keep the total count for backward compatibility cursor.execute(""" SELECT date, COUNT(DISTINCT public_key) as daily_count - FROM daily_stats + FROM daily_stats WHERE date >= date('now', '-30 days') AND date <= date('now') GROUP BY date ORDER BY date ASC """) daily_data = cursor.fetchall() server_stats['daily_nodes_30d'] = [ - {'date': row[0], 'count': row[1]} + {'date': row[0], 'count': row[1]} for row in daily_data ] - + except Exception as e: self.logger.debug(f"Could not get server stats: {e}") - + return { 'tracking_data': tracking, 'server_stats': server_stats @@ -3846,48 +4622,48 @@ class BotDataViewer: finally: if conn: conn.close() - + def _calculate_distance(self, lat1, lon1, lat2, lon2): """Calculate distance between two points using Haversine formula""" import math - + # Convert latitude and longitude from degrees to radians lat1, lon1, lat2, lon2 = map(math.radians, [lat1, lon1, lat2, lon2]) - + # Haversine formula dlat = lat2 - lat1 dlon = lon2 - lon1 a = math.sin(dlat/2)**2 + math.cos(lat1) * math.cos(lat2) * math.sin(dlon/2)**2 c = 2 * math.asin(math.sqrt(a)) - + # Radius of earth in kilometers r = 6371 - + return c * r - + def _get_cache_data(self): """Get cache data""" conn = None try: conn = self._get_db_connection() cursor = conn.cursor() - + # Get cache statistics cursor.execute("SELECT COUNT(*) FROM adverts") total_adverts = cursor.fetchone()[0] - + cursor.execute(""" - SELECT COUNT(*) FROM adverts + SELECT COUNT(*) FROM adverts WHERE timestamp > datetime('now', '-1 hour') """) recent_adverts = cursor.fetchone()[0] - + cursor.execute(""" - SELECT COUNT(DISTINCT user_id) FROM adverts + SELECT COUNT(DISTINCT user_id) FROM adverts WHERE timestamp > datetime('now', '-24 hours') """) active_users = cursor.fetchone()[0] - + return { 'total_adverts': total_adverts, 'recent_adverts_1h': recent_adverts, @@ -3900,8 +4676,8 @@ class BotDataViewer: finally: if conn: conn.close() - - + + def _get_feed_subscriptions(self, channel_filter=None): """Get all feed subscriptions, optionally filtered by channel""" import sqlite3 @@ -3910,7 +4686,7 @@ class BotDataViewer: conn = self._get_db_connection() conn.row_factory = sqlite3.Row cursor = conn.cursor() - + if channel_filter: cursor.execute(''' SELECT * FROM feed_subscriptions @@ -3922,7 +4698,7 @@ class BotDataViewer: SELECT * FROM feed_subscriptions ORDER BY id ''') - + rows = cursor.fetchall() feeds = [] for row in rows: @@ -3933,16 +4709,16 @@ class BotDataViewer: WHERE feed_id = ? ''', (feed['id'],)) feed['item_count'] = cursor.fetchone()[0] - + # Get error count cursor.execute(''' SELECT COUNT(*) FROM feed_errors WHERE feed_id = ? AND resolved_at IS NULL ''', (feed['id'],)) feed['error_count'] = cursor.fetchone()[0] - + feeds.append(feed) - + return {'feeds': feeds, 'total': len(feeds)} except Exception as e: self.logger.error(f"Error getting feed subscriptions: {e}") @@ -3950,7 +4726,7 @@ class BotDataViewer: finally: if conn: conn.close() - + def _get_feed_subscription(self, feed_id): """Get a single feed subscription by ID""" import sqlite3 @@ -3968,10 +4744,9 @@ class BotDataViewer: finally: if conn: conn.close() - + def _create_feed_subscription(self, data): """Create a new feed subscription""" - import sqlite3 import json conn = None try: @@ -3983,101 +4758,99 @@ class BotDataViewer: api_config = data.get('api_config') output_format = data.get('output_format') message_send_interval = data.get('message_send_interval_seconds') - + if not all([feed_type, feed_url, channel_name]): raise ValueError("feed_type, feed_url, and channel_name are required") - + conn = self._get_db_connection() cursor = conn.cursor() - + api_config_str = json.dumps(api_config) if api_config else None - + cursor.execute(''' - INSERT INTO feed_subscriptions + INSERT INTO feed_subscriptions (feed_type, feed_url, channel_name, feed_name, check_interval_seconds, api_config, output_format, message_send_interval_seconds) VALUES (?, ?, ?, ?, ?, ?, ?, ?) ''', (feed_type, feed_url, channel_name, feed_name, check_interval, api_config_str, output_format, message_send_interval)) - + conn.commit() return cursor.lastrowid - except Exception as e: + except Exception: if conn: conn.rollback() raise finally: if conn: conn.close() - + def _update_feed_subscription(self, feed_id, data): """Update a feed subscription""" - import sqlite3 import json conn = None try: conn = self._get_db_connection() cursor = conn.cursor() - + updates = [] params = [] - + if 'feed_name' in data: updates.append('feed_name = ?') params.append(data['feed_name']) - + if 'check_interval_seconds' in data: updates.append('check_interval_seconds = ?') params.append(data['check_interval_seconds']) - + if 'enabled' in data: updates.append('enabled = ?') params.append(1 if data['enabled'] else 0) - + if 'api_config' in data: updates.append('api_config = ?') params.append(json.dumps(data['api_config']) if data['api_config'] else None) - + if 'output_format' in data: updates.append('output_format = ?') params.append(data['output_format'] if data['output_format'] else None) - + if 'message_send_interval_seconds' in data: updates.append('message_send_interval_seconds = ?') params.append(float(data['message_send_interval_seconds']) if data['message_send_interval_seconds'] else None) - + if 'filter_config' in data: updates.append('filter_config = ?') params.append(json.dumps(data['filter_config']) if data['filter_config'] else None) - + if 'sort_config' in data: updates.append('sort_config = ?') params.append(json.dumps(data['sort_config']) if data['sort_config'] else None) - + if 'message_send_interval_seconds' in data: updates.append('message_send_interval_seconds = ?') params.append(data['message_send_interval_seconds']) - + if not updates: return True # Nothing to update - + updates.append('updated_at = CURRENT_TIMESTAMP') params.append(feed_id) - + query = f'UPDATE feed_subscriptions SET {", ".join(updates)} WHERE id = ?' cursor.execute(query, params) conn.commit() - + return cursor.rowcount > 0 - except Exception as e: + except Exception: if conn: conn.rollback() raise finally: if conn: conn.close() - + def _delete_feed_subscription(self, feed_id): """Delete a feed subscription""" - import sqlite3 conn = None try: conn = self._get_db_connection() @@ -4085,14 +4858,14 @@ class BotDataViewer: cursor.execute('DELETE FROM feed_subscriptions WHERE id = ?', (feed_id,)) conn.commit() return cursor.rowcount > 0 - except Exception as e: + except Exception: if conn: conn.rollback() raise finally: if conn: conn.close() - + def _get_feed_activity(self, feed_id, limit=50): """Get activity log for a feed""" import sqlite3 @@ -4115,7 +4888,7 @@ class BotDataViewer: finally: if conn: conn.close() - + def _get_feed_errors(self, feed_id, limit=20): """Get error history for a feed""" import sqlite3 @@ -4138,46 +4911,45 @@ class BotDataViewer: finally: if conn: conn.close() - + def _get_feed_statistics(self): """Get aggregate feed statistics""" - import sqlite3 conn = None try: conn = self._get_db_connection() cursor = conn.cursor() - + stats = {} - + # Total subscriptions cursor.execute('SELECT COUNT(*) FROM feed_subscriptions') stats['total_subscriptions'] = cursor.fetchone()[0] - + # Enabled subscriptions cursor.execute('SELECT COUNT(*) FROM feed_subscriptions WHERE enabled = 1') stats['enabled_subscriptions'] = cursor.fetchone()[0] - + # Items processed in last 24h cursor.execute(''' SELECT COUNT(*) FROM feed_activity WHERE processed_at > datetime('now', '-24 hours') ''') stats['items_24h'] = cursor.fetchone()[0] - + # Items processed in last 7d cursor.execute(''' SELECT COUNT(*) FROM feed_activity WHERE processed_at > datetime('now', '-7 days') ''') stats['items_7d'] = cursor.fetchone()[0] - + # Error count cursor.execute(''' SELECT COUNT(*) FROM feed_errors WHERE resolved_at IS NULL ''') stats['active_errors'] = cursor.fetchone()[0] - + # Most active channels cursor.execute(''' SELECT channel_name, COUNT(*) as feed_count @@ -4188,7 +4960,7 @@ class BotDataViewer: LIMIT 10 ''') stats['top_channels'] = [{'channel': row[0], 'count': row[1]} for row in cursor.fetchall()] - + return stats except Exception as e: self.logger.error(f"Error getting feed statistics: {e}") @@ -4196,14 +4968,14 @@ class BotDataViewer: finally: if conn: conn.close() - + def _get_feeds_by_channel(self, channel_idx): """Get all feeds for a specific channel index""" # First get channel name from index # This would require channel_manager access # For now, return empty list return [] - + def _get_channels(self): """Get all configured channels from database plus additional decode-only channels""" import sqlite3 @@ -4222,7 +4994,7 @@ class BotDataViewer: rows = cursor.fetchall() channels = [] existing_names = set() - + for row in rows: name = row['channel_name'] channels.append({ @@ -4263,11 +5035,11 @@ class BotDataViewer: finally: if conn: conn.close() - + def _get_additional_decode_channels(self): """Get additional hashtag channels to decode from config""" channels = set() # Use set for automatic deduplication - + try: # 1. Get channels from decode_hashtag_channels in [Web_Viewer] if self.config and self.config.has_option('Web_Viewer', 'decode_hashtag_channels'): @@ -4280,7 +5052,7 @@ class BotDataViewer: if c.startswith('#'): c = c[1:] channels.add(c) - + # 2. Import channels from [Channels_List] section if self.config and self.config.has_section('Channels_List'): for key in self.config.options('Channels_List'): @@ -4289,49 +5061,48 @@ class BotDataViewer: channel_name = key.split('.')[-1] # Get part after last dot else: channel_name = key - + channel_name = channel_name.strip().lower() if channel_name: channels.add(channel_name) except Exception as e: self.logger.error(f"Error reading decode channels config: {e}") - + return list(channels) - + def _get_channel_number(self, channel_name): """Get channel number from channel name""" # This would use channel_manager # For now, return None return None - + def _get_lowest_available_channel_index(self): """Get the lowest available channel index (0 to max_channels-1)""" try: channels = self._get_channels() used_indices = {c['channel_idx'] for c in channels} - + # Get max_channels from config (default 40) max_channels = self.config.getint('Bot', 'max_channels', fallback=40) - + # Find the lowest available index for i in range(max_channels): if i not in used_indices: return i - + # All channels are used return None except Exception as e: self.logger.error(f"Error getting lowest available channel index: {e}") return None - + def _get_channel_statistics(self): """Get channel statistics""" - import sqlite3 conn = None try: conn = self._get_db_connection() cursor = conn.cursor() - + # Get feed count per channel cursor.execute(''' SELECT channel_name, COUNT(*) as feed_count @@ -4339,12 +5110,12 @@ class BotDataViewer: WHERE enabled = 1 GROUP BY channel_name ''') - + channel_feeds = {row[0]: row[1] for row in cursor.fetchall()} - + # Get max_channels from config (default 40) max_channels = self.config.getint('Bot', 'max_channels', fallback=40) - + return { 'channels_with_feeds': len(channel_feeds), 'channel_feed_counts': channel_feeds, @@ -4356,41 +5127,38 @@ class BotDataViewer: finally: if conn: conn.close() - - def _preview_feed_items(self, feed_url: str, feed_type: str, output_format: str, api_config: dict = None, filter_config: dict = None, sort_config: dict = None) -> List[Dict[str, Any]]: + + def _preview_feed_items(self, feed_url: str, feed_type: str, output_format: str, api_config: dict = None, filter_config: dict = None, sort_config: dict = None) -> list[dict[str, Any]]: """Preview feed items with custom output format (standalone, doesn't require bot)""" + from datetime import datetime, timezone + import feedparser import requests - import html - import re - from datetime import datetime, timezone - + try: items = [] - + if feed_type == 'rss': # Fetch RSS feed response = requests.get(feed_url, timeout=30, headers={'User-Agent': 'MeshCoreBot/1.0 FeedManager'}) response.raise_for_status() parsed = feedparser.parse(response.text) - + # Get items (we'll filter and limit later) for entry in parsed.entries[:20]: # Fetch more items to account for filtering # Parse published date published = None if hasattr(entry, 'published_parsed') and entry.published_parsed: - try: + with suppress(Exception): published = datetime(*entry.published_parsed[:6], tzinfo=timezone.utc) - except Exception: - pass - + items.append({ 'title': entry.get('title', 'Untitled'), 'description': entry.get('description', ''), 'link': entry.get('link', ''), 'published': published }) - + elif feed_type == 'api': # Fetch API feed method = api_config.get('method', 'GET').upper() @@ -4398,13 +5166,13 @@ class BotDataViewer: params = api_config.get('params', {}) body = api_config.get('body') parser_config = api_config.get('response_parser', {}) - + if method == 'POST': response = requests.post(feed_url, headers=headers, params=params, json=body, timeout=30) else: response = requests.get(feed_url, headers=headers, params=params, timeout=30) response.raise_for_status() - + # Try to parse JSON, handle cases where response might be a string try: data = response.json() @@ -4412,15 +5180,15 @@ class BotDataViewer: # If JSON parsing fails, try to get text and see if it's an error message text = response.text raise Exception(f"API returned non-JSON response: {text[:200]}") - + # Check if response is an error message (string) if isinstance(data, str): raise Exception(f"API returned error message: {data[:200]}") - + # Ensure data is a dict or list if not isinstance(data, (dict, list)): raise Exception(f"API response is not a valid JSON object or array: {type(data).__name__} - {str(data)[:200]}") - + # Extract items using parser config items_path = parser_config.get('items_path', '') if items_path: @@ -4440,17 +5208,17 @@ class BotDataViewer: items_data = data.get('items', data.get('data', data.get('results', [data]))) else: items_data = [data] - + # Ensure items_data is a list if not isinstance(items_data, list): items_data = [items_data] - + # Get items (we'll filter and limit later) - id_field = parser_config.get('id_field', 'id') + parser_config.get('id_field', 'id') title_field = parser_config.get('title_field', 'title') description_field = parser_config.get('description_field', 'description') timestamp_field = parser_config.get('timestamp_field', 'created_at') - + # Helper function to get nested values def get_nested_value(data, path, default=''): if not path or not data: @@ -4474,7 +5242,7 @@ class BotDataViewer: if value is None: return default return value if value is not None else default - + for item_data in items_data[:20]: # Fetch more items to account for filtering # Ensure item_data is a dict if not isinstance(item_data, dict): @@ -4485,7 +5253,7 @@ class BotDataViewer: else: # Try to convert to dict or skip continue - + # Parse timestamp if available - support nested paths published = None if timestamp_field: @@ -4514,14 +5282,14 @@ class BotDataViewer: continue except Exception: pass - + # Get description - support nested paths description = '' if description_field: desc_value = get_nested_value(item_data, description_field) if desc_value: description = str(desc_value) - + items.append({ 'title': get_nested_value(item_data, title_field, 'Untitled'), 'description': description, @@ -4529,18 +5297,18 @@ class BotDataViewer: 'published': published, 'raw': item_data # Store raw data for format string access }) - + # Apply sorting if configured if sort_config: items = self._sort_items_preview(items, sort_config) - + # Apply filter if configured if filter_config: items = [item for item in items if self._should_include_item(item, filter_config)] - + # Limit to first 3 items after filtering items = items[:3] - + # Format items using output format formatted_items = [] for item in items: @@ -4549,35 +5317,35 @@ class BotDataViewer: 'original': item, 'formatted': formatted }) - + return formatted_items - + except Exception as e: self.logger.error(f"Error previewing feed: {e}") raise - - def _should_include_item(self, item: Dict[str, Any], filter_config: dict) -> bool: + + def _should_include_item(self, item: dict[str, Any], filter_config: dict) -> bool: """Check if an item should be included based on filter configuration (standalone version for preview)""" import json import re - + if not filter_config: return True - + try: filter_config_dict = json.loads(filter_config) if isinstance(filter_config, str) else filter_config except (json.JSONDecodeError, TypeError): return True - + conditions = filter_config_dict.get('conditions', []) if not conditions: return True - + logic = filter_config_dict.get('logic', 'AND').upper() - + # Get raw data for field access raw_data = item.get('raw', {}) - + # Helper to get nested values def get_nested_value(data, path, default=''): if not path or not data: @@ -4601,27 +5369,27 @@ class BotDataViewer: if value is None: return default return value if value is not None else default - + # Evaluate each condition results = [] for condition in conditions: field_path = condition.get('field') operator = condition.get('operator', 'equals') - + if not field_path: continue - + # Get field value using nested access field_value = get_nested_value(raw_data, field_path, '') if not field_value and field_path.startswith('raw.'): field_value = get_nested_value(raw_data, field_path[4:], '') - + if not field_value: field_value = get_nested_value(item, field_path, '') - + # Convert to string for comparison field_value_str = str(field_value).lower() if field_value is not None else '' - + # Evaluate condition result = False if operator == 'equals': @@ -4658,32 +5426,32 @@ class BotDataViewer: result = compare_value not in field_value_str else: result = True # Default to allowing if operator is unknown - + results.append(result) - + # Apply logic (AND or OR) if logic == 'OR': return any(results) else: # AND (default) return all(results) - + def _parse_microsoft_date(self, date_str: str) -> Optional[datetime]: """Parse Microsoft JSON date format: /Date(timestamp-offset)/""" import re from datetime import timezone - + if not date_str or not isinstance(date_str, str): return None - + # Match /Date(timestamp-offset)/ format match = re.match(r'/Date\((\d+)([+-]\d+)?\)/', date_str) if match: timestamp_ms = int(match.group(1)) offset_str = match.group(2) if match.group(2) else '+0000' - + # Convert milliseconds to seconds timestamp = timestamp_ms / 1000.0 - + # Parse offset (format: +0800 or -0800) try: offset_hours = int(offset_str[:3]) @@ -4691,31 +5459,31 @@ class BotDataViewer: offset_seconds = (offset_hours * 3600) + (offset_mins * 60) if offset_str[0] == '-': offset_seconds = -offset_seconds - + # Create timezone-aware datetime tz = timezone.utc if offset_seconds != 0: from datetime import timedelta tz = timezone(timedelta(seconds=offset_seconds)) - + return datetime.fromtimestamp(timestamp, tz=tz) except (ValueError, IndexError): # Fallback to UTC if offset parsing fails return datetime.fromtimestamp(timestamp, tz=timezone.utc) - + return None - - def _sort_items_preview(self, items: List[Dict[str, Any]], sort_config: dict) -> List[Dict[str, Any]]: + + def _sort_items_preview(self, items: list[dict[str, Any]], sort_config: dict) -> list[dict[str, Any]]: """Sort items based on sort configuration (standalone version for preview)""" if not sort_config or not items: return items - + field_path = sort_config.get('field') order = sort_config.get('order', 'desc').lower() - + if not field_path: return items - + # Helper to get nested values def get_nested_value(data, path, default=''): if not path or not data: @@ -4739,33 +5507,33 @@ class BotDataViewer: if value is None: return default return value if value is not None else default - + def get_sort_value(item): """Get the sort value for an item""" # Try raw data first raw_data = item.get('raw', {}) value = get_nested_value(raw_data, field_path, '') - + if not value and field_path.startswith('raw.'): value = get_nested_value(raw_data, field_path[4:], '') - + if not value: value = get_nested_value(item, field_path, '') - + # Handle Microsoft date format if isinstance(value, str) and value.startswith('/Date('): dt = self._parse_microsoft_date(value) if dt: return dt.timestamp() - + # Handle datetime objects if isinstance(value, datetime): return value.timestamp() - + # Handle numeric values if isinstance(value, (int, float)): return float(value) - + # Handle string timestamps if isinstance(value, str): # Try to parse as ISO format @@ -4774,7 +5542,7 @@ class BotDataViewer: return dt.timestamp() except ValueError: pass - + # Try common date formats for fmt in ['%Y-%m-%dT%H:%M:%S', '%Y-%m-%d %H:%M:%S', '%Y-%m-%d']: try: @@ -4782,10 +5550,10 @@ class BotDataViewer: return dt.timestamp() except ValueError: continue - + # For strings, use lexicographic comparison return str(value) - + # Sort items try: sorted_items = sorted(items, key=get_sort_value, reverse=(order == 'desc')) @@ -4793,17 +5561,17 @@ class BotDataViewer: except Exception as e: self.logger.warning(f"Error sorting items in preview: {e}") return items - - def _format_feed_item(self, item: Dict[str, Any], format_str: str, feed_name: str = '') -> str: + + def _format_feed_item(self, item: dict[str, Any], format_str: str, feed_name: str = '') -> str: """Format a feed item using the output format (standalone version)""" import html import re from datetime import datetime, timezone - + # Extract field values title = item.get('title', 'Untitled') body = item.get('description', '') or item.get('body', '') - + # Clean HTML from body if present if body: body = html.unescape(body) @@ -4821,22 +5589,19 @@ class BotDataViewer: lines = body.split('\n') body = '\n'.join(' '.join(line.split()) for line in lines) # Normalize spaces per line body = body.strip() - + link = item.get('link', '') published = item.get('published') - + # Format timestamp date_str = "" if published: try: - if published.tzinfo: - now = datetime.now(timezone.utc) - else: - now = datetime.now() - + now = datetime.now(timezone.utc) if published.tzinfo else datetime.now() + diff = now - published minutes = int(diff.total_seconds() / 60) - + if minutes < 1: date_str = "now" elif minutes < 60: @@ -4850,7 +5615,7 @@ class BotDataViewer: date_str = f"{days}d ago" except Exception: pass - + # Choose emoji emoji = "📢" feed_name_lower = feed_name.lower() @@ -4860,7 +5625,7 @@ class BotDataViewer: emoji = "⚠️" elif 'info' in feed_name_lower or 'news' in feed_name_lower: emoji = "ℹ️" - + # Build replacements replacements = { 'title': title, @@ -4869,10 +5634,10 @@ class BotDataViewer: 'link': link, 'emoji': emoji } - + # Get raw API data if available (for preview, we don't have raw data, so this will be empty) raw_data = item.get('raw', {}) - + # Helper to get nested values def get_nested_value(data, path, default=''): if not path or not data: @@ -4896,12 +5661,12 @@ class BotDataViewer: if value is None: return default return value if value is not None else default - + # Apply shortening, parsing, and conditional functions def apply_shortening(text: str, function: str) -> str: if not text: return "" - + if function.startswith('truncate:'): try: max_len = int(function.split(':', 1)[1]) @@ -4937,23 +5702,23 @@ class BotDataViewer: # Format: regex:pattern:group or regex:pattern # Need to handle patterns that contain colons, so split from the right remaining = function[6:] # Skip 'regex:' prefix - + # Try to find the last colon that's followed by a number (the group number) # Look for pattern like :N at the end last_colon_idx = remaining.rfind(':') pattern = remaining group_num = None - + if last_colon_idx > 0: # Check if what's after the last colon is a number potential_group = remaining[last_colon_idx + 1:] if potential_group.isdigit(): pattern = remaining[:last_colon_idx] group_num = int(potential_group) - + if not pattern: return text - + # Apply regex match = re.search(pattern, text, re.IGNORECASE | re.DOTALL) if match: @@ -4968,7 +5733,7 @@ class BotDataViewer: else: return match.group(0) return "" # No match found - except (ValueError, IndexError, re.error) as e: + except (ValueError, IndexError, re.error): # Silently fail on regex errors in preview return text elif function.startswith('if_regex:'): @@ -4978,21 +5743,21 @@ class BotDataViewer: parts = function[9:].split(':', 2) # Skip 'if_regex:' prefix, split into [pattern, then, else] if len(parts) < 3: return text - + pattern = parts[0] then_value = parts[1] else_value = parts[2] - + if not pattern: return text - + # Check if pattern matches match = re.search(pattern, text, re.IGNORECASE | re.DOTALL) if match: return then_value else: return else_value - except (ValueError, IndexError, re.error) as e: + except (ValueError, IndexError, re.error): # Silently fail on regex errors in preview return text elif function.startswith('switch:'): @@ -5002,7 +5767,7 @@ class BotDataViewer: parts = function[7:].split(':') # Skip 'switch:' prefix if len(parts) < 2: return text - + # Pairs of value:result, last one is default text_lower = text.lower().strip() for i in range(0, len(parts) - 1, 2): @@ -5011,10 +5776,10 @@ class BotDataViewer: result = parts[i + 1] if text_lower == value: return result - + # Return last part as default if no match return parts[-1] if parts else text - except (ValueError, IndexError) as e: + except (ValueError, IndexError): # Silently fail on switch errors in preview return text elif function.startswith('regex_cond:'): @@ -5023,15 +5788,15 @@ class BotDataViewer: parts = function[11:].split(':', 3) # Skip 'regex_cond:' prefix if len(parts) < 4: return text - + extract_pattern = parts[0] check_pattern = parts[1] then_value = parts[2] else_group = int(parts[3]) if parts[3].isdigit() else 1 - + if not extract_pattern: return text - + # Extract using extract_pattern match = re.search(extract_pattern, text, re.IGNORECASE | re.DOTALL) if match: @@ -5042,20 +5807,20 @@ class BotDataViewer: extracted = extracted.strip() else: extracted = match.group(0).strip() - + # Check if extracted text matches check_pattern (exact match or contains) if check_pattern: # Try exact match first, then substring match if extracted.lower() == check_pattern.lower() or re.search(check_pattern, extracted, re.IGNORECASE): return then_value - + return extracted return "" # No match found - except (ValueError, IndexError, re.error) as e: + except (ValueError, IndexError, re.error): # Silently fail on regex errors in preview return text return text - + # Process format string def replace_placeholder(match): content = match.group(1) @@ -5063,17 +5828,17 @@ class BotDataViewer: field_name, function = content.split('|', 1) field_name = field_name.strip() function = function.strip() - + # Check if it's a raw field access if field_name.startswith('raw.'): value = str(get_nested_value(raw_data, field_name[4:], '')) else: value = replacements.get(field_name, '') - + return apply_shortening(value, function) else: field_name = content.strip() - + # Check if it's a raw field access if field_name.startswith('raw.'): value = get_nested_value(raw_data, field_name[4:], '') @@ -5089,9 +5854,9 @@ class BotDataViewer: return str(value) else: return replacements.get(field_name, '') - + message = re.sub(r'\{([^}]+)\}', replace_placeholder, format_str) - + # Final truncation (130 char limit) max_length = 130 if len(message) > max_length: @@ -5106,9 +5871,9 @@ class BotDataViewer: message = message[:max_length - 3] + "..." else: message = message[:max_length - 3] + "..." - + return message - + def _get_bot_uptime(self): """Get bot uptime in seconds from database""" try: @@ -5120,50 +5885,50 @@ class BotDataViewer: # Fallback: try to get earliest message timestamp conn = self._get_db_connection() cursor = conn.cursor() - + # Try to get earliest message timestamp as fallback cursor.execute(""" - SELECT MIN(timestamp) FROM message_stats + SELECT MIN(timestamp) FROM message_stats WHERE timestamp IS NOT NULL """) result = cursor.fetchone() if result and result[0]: return int(time.time() - result[0]) - + return 0 except Exception as e: self.logger.debug(f"Could not get bot start time from database: {e}") return 0 - + def _add_channel_for_web(self, channel_idx, channel_name, channel_key_hex=None): """ Add a channel by queuing it in the database for the bot to process - + Args: channel_idx: Channel index (0-39) channel_name: Channel name (with or without # prefix) channel_key_hex: Optional hex key for custom channels (32 chars) - + Returns: dict with 'success' and optional 'error' key """ try: conn = self._get_db_connection() cursor = conn.cursor() - + # Insert operation into queue cursor.execute(''' - INSERT INTO channel_operations + INSERT INTO channel_operations (operation_type, channel_idx, channel_name, channel_key_hex, status) VALUES (?, ?, ?, ?, 'pending') ''', ('add', channel_idx, channel_name, channel_key_hex)) - + operation_id = cursor.lastrowid conn.commit() conn.close() - + self.logger.info(f"Queued channel add operation: {channel_name} at index {channel_idx} (operation_id: {operation_id})") - + # Return immediately with operation_id - let frontend poll for status return { 'success': True, @@ -5171,41 +5936,41 @@ class BotDataViewer: 'operation_id': operation_id, 'message': 'Channel operation queued successfully' } - + except Exception as e: self.logger.error(f"Error in _add_channel_for_web: {e}") return { 'success': False, 'error': str(e) } - + def _remove_channel_for_web(self, channel_idx): """ Remove a channel by queuing it in the database for the bot to process - + Args: channel_idx: Channel index to remove - + Returns: dict with 'success' and optional 'error' key """ try: conn = self._get_db_connection() cursor = conn.cursor() - + # Insert operation into queue cursor.execute(''' - INSERT INTO channel_operations + INSERT INTO channel_operations (operation_type, channel_idx, status) VALUES (?, ?, 'pending') ''', ('remove', channel_idx)) - + operation_id = cursor.lastrowid conn.commit() conn.close() - + self.logger.info(f"Queued channel remove operation: index {channel_idx} (operation_id: {operation_id})") - + # Return immediately with operation_id - let frontend poll for status return { 'success': True, @@ -5213,15 +5978,15 @@ class BotDataViewer: 'operation_id': operation_id, 'message': 'Channel operation queued successfully' } - + except Exception as e: self.logger.error(f"Error in _remove_channel_for_web: {e}") return { 'success': False, 'error': str(e) } - - def _decode_path_hex(self, path_hex: str, bytes_per_hop: Optional[int] = None) -> List[Dict[str, Any]]: + + def _decode_path_hex(self, path_hex: str, bytes_per_hop: Optional[int] = None) -> list[dict[str, Any]]: """ Decode hex path string to repeater names using the same sophisticated logic as path command. Returns a list of dictionaries with node_id and repeater info. @@ -5229,8 +5994,8 @@ class BotDataViewer: When the path came from a packet with 2-byte or 3-byte hops, pass bytes_per_hop (2 or 3) so node IDs and graph selection use the correct prefix length. """ - import re import math + import re from datetime import datetime # Parse the path input - use bytes_per_hop when provided (e.g. from packet/contact) @@ -5254,18 +6019,18 @@ class BotDataViewer: if not hex_matches and prefix_hex_chars > 2: hex_pattern = r'[0-9a-fA-F]{2}' hex_matches = re.findall(hex_pattern, path_input) - + if not hex_matches: return [] - + # Convert to uppercase for consistency node_ids = [match.upper() for match in hex_matches] - + # Load Path_Command config values (same as path command) geographic_guessing_enabled = False bot_latitude = None bot_longitude = None - + try: if self.config.has_section('Bot'): lat = self.config.getfloat('Bot', 'bot_latitude', fallback=None) @@ -5276,18 +6041,18 @@ class BotDataViewer: geographic_guessing_enabled = True except Exception: pass - - proximity_method = self.config.get('Path_Command', 'proximity_method', fallback='simple') + + self.config.get('Path_Command', 'proximity_method', fallback='simple') max_proximity_range = self.config.getfloat('Path_Command', 'max_proximity_range', fallback=200.0) max_repeater_age_days = self.config.getint('Path_Command', 'max_repeater_age_days', fallback=14) recency_weight = self.config.getfloat('Path_Command', 'recency_weight', fallback=0.4) recency_weight = max(0.0, min(1.0, recency_weight)) proximity_weight = 1.0 - recency_weight recency_decay_half_life_hours = self.config.getfloat('Path_Command', 'recency_decay_half_life_hours', fallback=12.0) - + # Check for preset first, then apply individual settings (preset can be overridden) preset = self.config.get('Path_Command', 'path_selection_preset', fallback='balanced').lower() - + # Apply preset defaults, then individual settings override if preset == 'geographic': preset_graph_confidence_threshold = 0.5 @@ -5304,7 +6069,7 @@ class BotDataViewer: preset_distance_threshold = 30.0 preset_distance_penalty = 0.3 preset_final_hop_weight = 0.25 - + graph_based_validation = self.config.getboolean('Path_Command', 'graph_based_validation', fallback=True) min_edge_observations = self.config.getint('Path_Command', 'min_edge_observations', fallback=3) graph_use_bidirectional = self.config.getboolean('Path_Command', 'graph_use_bidirectional', fallback=True) @@ -5337,14 +6102,14 @@ class BotDataViewer: graph_path_validation_obs_divisor = self.config.getfloat('Path_Command', 'graph_path_validation_obs_divisor', fallback=50.0) star_bias_multiplier = self.config.getfloat('Path_Command', 'star_bias_multiplier', fallback=2.5) star_bias_multiplier = max(1.0, star_bias_multiplier) - + # Use calculate_distance from utils (already imported) - + # Helper: calculate recency scores def calculate_recency_weighted_scores(repeaters): scored_repeaters = [] now = datetime.now() - + for repeater in repeaters: most_recent_time = None for field in ['last_heard', 'last_advert_timestamp', 'last_seen']: @@ -5359,19 +6124,19 @@ class BotDataViewer: most_recent_time = dt except: pass - + if most_recent_time is None: recency_score = 0.1 else: hours_ago = (now - most_recent_time).total_seconds() / 3600.0 recency_score = math.exp(-hours_ago / recency_decay_half_life_hours) recency_score = max(0.0, min(1.0, recency_score)) - + scored_repeaters.append((repeater, recency_score)) - + scored_repeaters.sort(key=lambda x: x[1], reverse=True) return scored_repeaters - + # Helper: graph-based selection with final hop proximity and path validation # When path was decoded with 2-byte or 3-byte hops, node_id/path_context have 4 or 6 hex chars; # use path_prefix_hex_chars for candidate matching and normalize to graph_n for edge lookups. @@ -5477,7 +6242,7 @@ class BotDataViewer: if candidate_to_next_edge and candidate_to_next_edge.get('geographic_distance'): distance = candidate_to_next_edge.get('geographic_distance') max_distance = max(max_distance, distance) - + # Apply penalty if distance exceeds reasonable hop distance if max_distance > graph_max_reasonable_hop_distance_km: excess_distance = max_distance - graph_max_reasonable_hop_distance_km @@ -5496,10 +6261,10 @@ class BotDataViewer: if bot_latitude is not None and bot_longitude is not None: repeater_lat = repeater.get('latitude') repeater_lon = repeater.get('longitude') - + if repeater_lat is not None and repeater_lon is not None: distance = calculate_distance(bot_latitude, bot_longitude, repeater_lat, repeater_lon) - + if graph_final_hop_max_distance > 0 and distance > graph_final_hop_max_distance: # Beyond max distance - significantly penalize this candidate for final hop candidate_score *= 0.3 # Heavy penalty for distant final hop @@ -5508,7 +6273,7 @@ class BotDataViewer: # Use configurable normalization distance (default 500km for more aggressive scoring) normalized_distance = min(distance / graph_final_hop_proximity_normalization_km, 1.0) proximity_score = 1.0 - normalized_distance - + # For final hop, use a higher effective weight to ensure proximity matters more # The configured weight is a minimum; we boost it for very close repeaters effective_weight = graph_final_hop_proximity_weight @@ -5518,10 +6283,10 @@ class BotDataViewer: elif distance < graph_final_hop_close_threshold_km: # Close - moderate boost effective_weight = min(0.5, graph_final_hop_proximity_weight * 1.5) - + # Combine with graph score using effective weight candidate_score = candidate_score * (1.0 - effective_weight) + proximity_score * effective_weight - + # Path validation bonus: Check if candidate's stored paths match the current path context # This is especially important for prefix collision resolution path_validation_bonus = 0.0 @@ -5535,17 +6300,17 @@ class BotDataViewer: LIMIT 10 ''' stored_paths = self.db_manager.execute_query(query, (candidate_public_key,)) - + if stored_paths: decoded_path_hex = ''.join([node.lower() for node in path_context]) # Build the path prefix up to (but not including) the current node # This helps match paths where the candidate appears at the same position path_prefix_up_to_current = ''.join([node.lower() for node in path_context[:current_index]]) - + for stored_path in stored_paths: stored_hex = stored_path.get('path_hex', '').lower() obs_count = stored_path.get('observation_count', 1) - + if stored_hex: n = (stored_path.get('bytes_per_hop') or 1) * 2 if n <= 0: @@ -5554,7 +6319,7 @@ class BotDataViewer: if (len(stored_hex) % n) != 0: stored_nodes = [stored_hex[i:i+2] for i in range(0, len(stored_hex), 2)] decoded_nodes = path_context if path_context else [decoded_path_hex[i:i+n] for i in range(0, len(decoded_path_hex), n)] - + # Check for exact path match (full path) common_segments = 0 min_len = min(len(stored_nodes), len(decoded_nodes)) @@ -5563,7 +6328,7 @@ class BotDataViewer: common_segments += 1 else: break - + # Also check if stored path starts with the same prefix as the decoded path up to current position # This is important for matching paths where the candidate appears at the same position prefix_match = False @@ -5572,7 +6337,7 @@ class BotDataViewer: # The stored path has the same prefix, and the candidate appears at the same position # This is a strong indicator of a match prefix_match = True - + if common_segments >= 2 or prefix_match: # Stronger bonus for prefix matches (indicates same path structure) if prefix_match and common_segments >= current_index: @@ -5587,32 +6352,32 @@ class BotDataViewer: break # Strong match found, no need to check more except Exception: pass - + candidate_score = min(1.0, candidate_score + path_validation_bonus) - + if repeater.get('is_starred', False): candidate_score *= star_bias_multiplier - + if candidate_score > best_score: best_score = candidate_score best_repeater = repeater best_method = method - + if best_repeater and best_score > 0.0: confidence = min(1.0, best_score) if best_score <= 1.0 else 0.95 + (min(0.05, (best_score - 1.0) / star_bias_multiplier)) return best_repeater, confidence, best_method or 'graph' - + return None, 0.0, None - + # Helper: simple proximity selection def select_by_simple_proximity(repeaters_with_location): scored_repeaters = calculate_recency_weighted_scores(repeaters_with_location) min_recency_threshold = 0.01 scored_repeaters = [(r, score) for r, score in scored_repeaters if score >= min_recency_threshold] - + if not scored_repeaters: return None, 0.0 - + if len(scored_repeaters) == 1: repeater, recency_score = scored_repeaters[0] distance = calculate_distance(bot_latitude, bot_longitude, repeater['latitude'], repeater['longitude']) @@ -5620,28 +6385,28 @@ class BotDataViewer: return None, 0.0 base_confidence = 0.4 + (recency_score * 0.5) return repeater, base_confidence - + combined_scores = [] for repeater, recency_score in scored_repeaters: distance = calculate_distance(bot_latitude, bot_longitude, repeater['latitude'], repeater['longitude']) if max_proximity_range > 0 and distance > max_proximity_range: continue - + normalized_distance = min(distance / 1000.0, 1.0) proximity_score = 1.0 - normalized_distance combined_score = (recency_score * recency_weight) + (proximity_score * proximity_weight) - + if repeater.get('is_starred', False): combined_score *= star_bias_multiplier - + combined_scores.append((combined_score, distance, repeater)) - + if not combined_scores: return None, 0.0 - + combined_scores.sort(key=lambda x: x[0], reverse=True) best_score, best_distance, best_repeater = combined_scores[0] - + if len(combined_scores) == 1: confidence = 0.4 + (best_score * 0.5) else: @@ -5655,40 +6420,40 @@ class BotDataViewer: confidence = 0.7 else: confidence = 0.5 - + return best_repeater, confidence - + # Main decoding logic (same as path command) decoded_path = [] - + try: for node_id in node_ids: # Query database for matching repeaters if max_repeater_age_days > 0: - query = ''' - SELECT name, public_key, device_type, last_heard, last_heard as last_seen, + query = f''' + SELECT name, public_key, device_type, last_heard, last_heard as last_seen, last_advert_timestamp, latitude, longitude, city, state, country, advert_count, signal_strength, hop_count, role, is_starred - FROM complete_contact_tracking + FROM complete_contact_tracking WHERE public_key LIKE ? AND role IN ('repeater', 'roomserver') AND ( - (last_advert_timestamp IS NOT NULL AND last_advert_timestamp >= datetime('now', '-{} days')) - OR (last_advert_timestamp IS NULL AND last_heard >= datetime('now', '-{} days')) + (last_advert_timestamp IS NOT NULL AND last_advert_timestamp >= datetime('now', '-{max_repeater_age_days} days')) + OR (last_advert_timestamp IS NULL AND last_heard >= datetime('now', '-{max_repeater_age_days} days')) ) ORDER BY COALESCE(last_advert_timestamp, last_heard) DESC - '''.format(max_repeater_age_days, max_repeater_age_days) + ''' else: query = ''' - SELECT name, public_key, device_type, last_heard, last_heard as last_seen, + SELECT name, public_key, device_type, last_heard, last_heard as last_seen, last_advert_timestamp, latitude, longitude, city, state, country, advert_count, signal_strength, hop_count, role, is_starred - FROM complete_contact_tracking + FROM complete_contact_tracking WHERE public_key LIKE ? AND role IN ('repeater', 'roomserver') ORDER BY COALESCE(last_advert_timestamp, last_heard) DESC ''' - + results = self.db_manager.execute_query(query, (f"{node_id}%",)) - + if results: repeaters_data = [ { @@ -5708,11 +6473,11 @@ class BotDataViewer: 'is_starred': bool(row.get('is_starred', 0)) } for row in results ] - + scored_repeaters = calculate_recency_weighted_scores(repeaters_data) min_recency_threshold = 0.01 recent_repeaters = [r for r, score in scored_repeaters if score >= min_recency_threshold] - + if len(recent_repeaters) > 1: # Multiple matches - use graph and geographic selection graph_repeater = None @@ -5720,25 +6485,25 @@ class BotDataViewer: selection_method = None geo_repeater = None geo_confidence = 0.0 - + if graph_based_validation and hasattr(self, 'mesh_graph') and self.mesh_graph: graph_repeater, graph_confidence, selection_method = select_repeater_by_graph( recent_repeaters, node_id, node_ids ) - + if geographic_guessing_enabled: repeaters_with_location = [r for r in recent_repeaters if r.get('latitude') and r.get('longitude')] if repeaters_with_location: geo_repeater, geo_confidence = select_by_simple_proximity(repeaters_with_location) - + # Combine or choose selected_repeater = None confidence = 0.0 - + if graph_geographic_combined and graph_repeater and geo_repeater: graph_pubkey = graph_repeater.get('public_key', '') geo_pubkey = geo_repeater.get('public_key', '') - + if graph_pubkey and geo_pubkey and graph_pubkey == geo_pubkey: combined_confidence = ( graph_confidence * graph_geographic_weight + @@ -5757,7 +6522,7 @@ class BotDataViewer: # For final hop, prefer geographic selection if available and reasonable # The final hop should be close to the bot, so geographic proximity is very important is_final_hop = (node_id == node_ids[-1] if node_ids else False) - + if is_final_hop and geo_repeater and geo_confidence >= 0.6: # For final hop, prefer geographic if it has decent confidence # This ensures we pick the closest repeater for the last hop @@ -5777,7 +6542,7 @@ class BotDataViewer: elif graph_repeater: selected_repeater = graph_repeater confidence = graph_confidence - + if selected_repeater and confidence >= 0.5: decoded_path.append({ 'node_id': node_id, @@ -5832,9 +6597,9 @@ class BotDataViewer: except Exception as e: self.logger.error(f"Error decoding path: {e}", exc_info=True) return [] - + return decoded_path - + def run(self, host='127.0.0.1', port=8080, debug=False): """Run the modern web viewer""" self.logger.info(f"Starting modern web viewer on {host}:{port}") @@ -5853,7 +6618,7 @@ class BotDataViewer: def main(): """Entry point for the meshcore-viewer command""" import argparse - + parser = argparse.ArgumentParser(description='MeshCore Bot Data Viewer') parser.add_argument('--host', default='127.0.0.1', help='Host to bind to') parser.add_argument('--port', type=int, default=8080, help='Port to bind to') @@ -5863,9 +6628,9 @@ def main(): default="config.ini", help="Path to configuration file (default: config.ini)", ) - + args = parser.parse_args() - + viewer = BotDataViewer(config_path=args.config) viewer.run(host=args.host, port=args.port, debug=args.debug) diff --git a/modules/web_viewer/integration.py b/modules/web_viewer/integration.py index ba37a27..89701ec 100644 --- a/modules/web_viewer/integration.py +++ b/modules/web_viewer/integration.py @@ -4,21 +4,22 @@ Web Viewer Integration for MeshCore Bot Provides integration between the main bot and the web viewer """ -import threading -import time -import subprocess -import sys import os import re -from contextlib import closing +import subprocess +import sys +import threading +import time +from contextlib import closing, suppress from pathlib import Path from typing import Optional from ..utils import resolve_path + class BotIntegration: """Simple bot integration for web viewer compatibility""" - + # After this many consecutive connection failures, stop sending until cooldown expires CIRCUIT_BREAKER_THRESHOLD = 3 CIRCUIT_BREAKER_COOLDOWN_SEC = 60 @@ -33,33 +34,34 @@ class BotIntegration: self._init_http_session() # Initialize the packet_stream table self._init_packet_stream_table() - + def _init_http_session(self): """Initialize a requests.Session with connection pooling and keep-alive""" try: + import logging + import requests + import urllib3 from requests.adapters import HTTPAdapter from urllib3.util.retry import Retry - import urllib3 - import logging - + # Suppress urllib3 connection pool messages when web viewer is unreachable # Connection refused / Retrying WARNINGs would flood logs during routing bursts urllib3_logger = logging.getLogger('urllib3.connectionpool') urllib3_logger.setLevel(logging.ERROR) - + # Also disable other urllib3 warnings urllib3.disable_warnings(urllib3.exceptions.NotOpenSSLWarning) - + self.http_session = requests.Session() - + # Configure retry strategy retry_strategy = Retry( total=2, backoff_factor=0.1, status_forcelist=[429, 500, 502, 503, 504], ) - + # Mount adapter with connection pooling # pool_block=False allows non-blocking behavior if pool is full adapter = HTTPAdapter( @@ -70,7 +72,7 @@ class BotIntegration: ) self.http_session.mount("http://", adapter) self.http_session.mount("https://", adapter) - + # Set default headers for keep-alive (though urllib3 handles this automatically) self.http_session.headers.update({ 'Connection': 'keep-alive', @@ -81,7 +83,7 @@ class BotIntegration: except Exception as e: self.bot.logger.debug(f"Error initializing HTTP session: {e}") self.http_session = None - + def reset_circuit_breaker(self): """Reset the circuit breaker""" self.circuit_breaker_open = False @@ -110,7 +112,7 @@ class BotIntegration: self.circuit_breaker_failures, self.CIRCUIT_BREAKER_COOLDOWN_SEC, ) - + def _get_web_viewer_db_path(self): """Return resolved database path for web viewer. Uses [Bot] db_path when [Web_Viewer] db_path is unset.""" base_dir = self.bot.bot_root if hasattr(self.bot, 'bot_root') else '.' @@ -119,17 +121,17 @@ class BotIntegration: if raw: return resolve_path(raw, base_dir) return str(Path(self.bot.db_manager.db_path).resolve()) - + def _init_packet_stream_table(self): """Initialize the packet_stream table in the web viewer database (same as [Bot] db_path by default).""" try: import sqlite3 - + db_path = self._get_web_viewer_db_path() - + with closing(sqlite3.connect(str(db_path), timeout=60.0)) as conn: cursor = conn.cursor() - + # Create packet_stream table with schema matching the INSERT statements cursor.execute(''' CREATE TABLE IF NOT EXISTS packet_stream ( @@ -139,34 +141,34 @@ class BotIntegration: type TEXT NOT NULL ) ''') - + # Create index on timestamp for faster queries cursor.execute(''' - CREATE INDEX IF NOT EXISTS idx_packet_stream_timestamp + CREATE INDEX IF NOT EXISTS idx_packet_stream_timestamp ON packet_stream(timestamp) ''') - + # Create index on type for filtering by type cursor.execute(''' - CREATE INDEX IF NOT EXISTS idx_packet_stream_type + CREATE INDEX IF NOT EXISTS idx_packet_stream_type ON packet_stream(type) ''') - + # Enable WAL for better concurrent access (bot + web viewer use same DB) try: cursor.execute('PRAGMA journal_mode=WAL') except sqlite3.OperationalError: pass # Ignore if locked; WAL may already be set - + conn.commit() - + self.bot.logger.info(f"Initialized packet_stream table in {db_path}") - + except Exception as e: self.bot.logger.error(f"Failed to initialize packet_stream table: {e}") # Don't raise - allow bot to continue even if table init fails # The error will be caught when trying to insert data - + def _insert_packet_stream_row(self, data_json: str, row_type: str, log_prefix: str = "packet data"): """Insert one row into packet_stream. Retries on database is locked. Logs and returns on failure.""" import sqlite3 @@ -192,20 +194,20 @@ class BotIntegration: except Exception as e: self.bot.logger.warning(f"Error storing {log_prefix} for web viewer: {e}") return - + def capture_full_packet_data(self, packet_data): """Capture full packet data and store in database for web viewer""" try: import json from datetime import datetime - + # Ensure packet_data is a dict (might be passed as dict already) if not isinstance(packet_data, dict): packet_data = self._make_json_serializable(packet_data) if not isinstance(packet_data, dict): # If still not a dict, wrap it packet_data = {'data': packet_data} - + # Add hops field from path_len if not already present # path_len represents the number of hops (each byte = 1 hop) if 'hops' not in packet_data and 'path_len' in packet_data: @@ -213,43 +215,43 @@ class BotIntegration: elif 'hops' not in packet_data: # If no path_len either, default to 0 hops packet_data['hops'] = 0 - + # Add datetime for frontend display if 'datetime' not in packet_data: packet_data['datetime'] = datetime.now().isoformat() - + # Convert non-serializable objects to strings serializable_data = self._make_json_serializable(packet_data) - + # Store in database for web viewer to read (retries on database is locked) self._insert_packet_stream_row(json.dumps(serializable_data), 'packet', "packet data") - + except Exception as e: self.bot.logger.warning(f"Error storing packet data for web viewer: {e}") - + def capture_command(self, message, command_name, response, success, command_id=None): """Capture command data and store in database for web viewer""" try: import json import time - + # Extract data from message object user = getattr(message, 'sender_id', 'Unknown') channel = getattr(message, 'channel', 'Unknown') user_input = getattr(message, 'content', f'/{command_name}') - + # Get repeat information if transmission tracker is available repeat_count = 0 repeater_prefixes = [] repeater_counts = {} - if (hasattr(self.bot, 'transmission_tracker') and - self.bot.transmission_tracker and + if (hasattr(self.bot, 'transmission_tracker') and + self.bot.transmission_tracker and command_id): repeat_info = self.bot.transmission_tracker.get_repeat_info(command_id=command_id) repeat_count = repeat_info.get('repeat_count', 0) repeater_prefixes = repeat_info.get('repeater_prefixes', []) repeater_counts = repeat_info.get('repeater_counts', {}) - + # Construct command data structure command_data = { 'user': user, @@ -264,30 +266,50 @@ class BotIntegration: 'repeater_counts': repeater_counts, # Count per repeater prefix 'command_id': command_id # Store command_id for later updates } - + # Convert non-serializable objects to strings serializable_data = self._make_json_serializable(command_data) - + # Store in database for web viewer to read (retries on database is locked) self._insert_packet_stream_row(json.dumps(serializable_data), 'command', "command data") - + except Exception as e: self.bot.logger.debug(f"Error storing command data: {e}") - + + def capture_channel_message(self, message) -> None: + """Capture an incoming channel or DM message for the web viewer live monitor.""" + try: + import json + import time + data = { + 'type': 'message', + 'timestamp': time.time(), + 'sender': getattr(message, 'sender_id', ''), + 'channel': getattr(message, 'channel', ''), + 'content': getattr(message, 'content', ''), + 'snr': str(getattr(message, 'snr', '')), + 'hops': getattr(message, 'hops', None), + 'path': getattr(message, 'path', ''), + 'is_dm': bool(getattr(message, 'is_dm', False)), + } + self._insert_packet_stream_row(json.dumps(data), 'message', "channel message") + except Exception as e: + self.bot.logger.debug(f"Error storing channel message for web viewer: {e}") + def capture_packet_routing(self, routing_data): """Capture packet routing data and store in database for web viewer""" try: import json - + # Convert non-serializable objects to strings serializable_data = self._make_json_serializable(routing_data) - + # Store in database for web viewer to read (retries on database is locked) self._insert_packet_stream_row(json.dumps(serializable_data), 'routing', "routing data") - + except Exception as e: self.bot.logger.debug(f"Error storing routing data: {e}") - + def cleanup_old_data(self, days_to_keep: Optional[int] = None): """Clean up old packet stream data to prevent database bloat. Uses [Data_Retention] packet_stream_retention_days when days_to_keep is not provided.""" @@ -298,34 +320,32 @@ class BotIntegration: if days_to_keep is None: days_to_keep = 3 if self.bot.config.has_section('Data_Retention') and self.bot.config.has_option('Data_Retention', 'packet_stream_retention_days'): - try: + with suppress(ValueError, TypeError): days_to_keep = self.bot.config.getint('Data_Retention', 'packet_stream_retention_days') - except (ValueError, TypeError): - pass cutoff_time = time.time() - (days_to_keep * 24 * 60 * 60) - + db_path = self._get_web_viewer_db_path() with closing(sqlite3.connect(str(db_path), timeout=60.0)) as conn: cursor = conn.cursor() - + # Clean up old packet stream data cursor.execute('DELETE FROM packet_stream WHERE timestamp < ?', (cutoff_time,)) deleted_count = cursor.rowcount - + conn.commit() - + if deleted_count > 0: self.bot.logger.info(f"Cleaned up {deleted_count} old packet stream entries (older than {days_to_keep} days)") - + except Exception as e: self.bot.logger.error(f"Error cleaning up old packet stream data: {e}") - + def _make_json_serializable(self, obj, depth=0, max_depth=3): """Convert non-JSON-serializable objects to strings with depth limiting""" if depth > max_depth: return str(obj) - + # Handle basic types first if obj is None or isinstance(obj, (str, int, float, bool)): return obj @@ -345,7 +365,7 @@ class BotIntegration: return str(obj) else: return str(obj) - + def send_mesh_edge_update(self, edge_data): """Send mesh edge update to web viewer via HTTP API""" try: @@ -355,12 +375,12 @@ class BotIntegration: host = self.bot.config.get('Web_Viewer', 'host', fallback='127.0.0.1') port = self.bot.config.getint('Web_Viewer', 'port', fallback=8080) url = f"http://{host}:{port}/api/stream_data" - + payload = { 'type': 'mesh_edge', 'data': edge_data } - + # Use session with connection pooling if available, otherwise fallback to requests.post if self.http_session: try: @@ -377,7 +397,7 @@ class BotIntegration: self._record_web_viewer_result(False) except Exception as e: self.bot.logger.debug(f"Error sending mesh edge update to web viewer: {e}") - + def send_mesh_node_update(self, node_data): """Send mesh node update to web viewer via HTTP API""" try: @@ -388,12 +408,12 @@ class BotIntegration: host = self.bot.config.get('Web_Viewer', 'host', fallback='127.0.0.1') port = self.bot.config.getint('Web_Viewer', 'port', fallback=8080) url = f"http://{host}:{port}/api/stream_data" - + payload = { 'type': 'mesh_node', 'data': node_data } - + try: requests.post(url, json=payload, timeout=0.5) self._record_web_viewer_result(True) @@ -401,55 +421,53 @@ class BotIntegration: self._record_web_viewer_result(False) except Exception as e: self.bot.logger.debug(f"Error sending mesh node update to web viewer: {e}") - + def shutdown(self): """Mark as shutting down and close HTTP session""" self.is_shutting_down = True # Close HTTP session to clean up connections if hasattr(self, 'http_session') and self.http_session: - try: + with suppress(Exception): self.http_session.close() - except Exception: - pass class WebViewerIntegration: """Integration class for starting/stopping the web viewer with the bot""" - + # Whitelist of allowed host bindings for security ALLOWED_HOSTS = ['127.0.0.1', 'localhost', '0.0.0.0'] - + def __init__(self, bot): self.bot = bot self.logger = bot.logger self.viewer_process = None self.viewer_thread = None self.running = False - + # File handles for subprocess stdout/stderr (for proper cleanup) self._viewer_stdout_file = None self._viewer_stderr_file = None - + # Get web viewer settings from config self.enabled = bot.config.getboolean('Web_Viewer', 'enabled', fallback=False) self.host = bot.config.get('Web_Viewer', 'host', fallback='127.0.0.1') self.port = bot.config.getint('Web_Viewer', 'port', fallback=8080) # Web viewer uses 8080 self.debug = bot.config.getboolean('Web_Viewer', 'debug', fallback=False) self.auto_start = bot.config.getboolean('Web_Viewer', 'auto_start', fallback=False) - + # Validate configuration for security self._validate_config() - + # Process monitoring self.restart_count = 0 self.max_restarts = 5 self.last_restart = 0 - + # Initialize bot integration for compatibility self.bot_integration = BotIntegration(bot) - + if self.enabled and self.auto_start: self.start_viewer() - + def _validate_config(self): """Validate web viewer configuration for security""" # Validate host against whitelist @@ -458,13 +476,13 @@ class WebViewerIntegration: f"Invalid host configuration: {self.host}. " f"Allowed hosts: {', '.join(self.ALLOWED_HOSTS)}" ) - + # Validate port range (avoid privileged ports) if not isinstance(self.port, int) or not (1024 <= self.port <= 65535): raise ValueError( f"Port must be between 1024-65535 (non-privileged), got: {self.port}" ) - + # Security warning for network exposure if self.host == '0.0.0.0': self.logger.warning( @@ -475,31 +493,31 @@ class WebViewerIntegration: "For local-only access, use host=127.0.0.1 in config.\n" + "="*70 ) - + def start_viewer(self): """Start the web viewer in a separate thread""" if self.running: self.logger.warning("Web viewer is already running") return - + try: # Start the web viewer self.viewer_thread = threading.Thread(target=self._run_viewer, daemon=True) self.viewer_thread.start() self.running = True self.logger.info(f"Web viewer started on http://{self.host}:{self.port}") - + except Exception as e: self.logger.error(f"Failed to start web viewer: {e}") - + def stop_viewer(self): """Stop the web viewer""" if not self.running and not self.viewer_process: return - + try: self.running = False - + if self.viewer_process and self.viewer_process.poll() is None: self.logger.info("Stopping web viewer...") try: @@ -520,7 +538,7 @@ class WebViewerIntegration: self.logger.warning(f"Error during web viewer shutdown: {e}") finally: self.viewer_process = None - + # Close log file handles if self._viewer_stdout_file: try: @@ -529,7 +547,7 @@ class WebViewerIntegration: self.logger.debug(f"Error closing stdout file: {e}") finally: self._viewer_stdout_file = None - + if self._viewer_stderr_file: try: self._viewer_stderr_file.close() @@ -537,14 +555,13 @@ class WebViewerIntegration: self.logger.debug(f"Error closing stderr file: {e}") finally: self._viewer_stderr_file = None - + if not self.viewer_process: self.logger.info("Web viewer already stopped") - + # Additional cleanup: kill any remaining processes on the port try: - import subprocess - result = subprocess.run(['lsof', '-ti', f':{self.port}'], + result = subprocess.run(['lsof', '-ti', f':{self.port}'], capture_output=True, text=True, timeout=5) if result.returncode == 0 and result.stdout.strip(): pids = result.stdout.strip().split('\n') @@ -552,41 +569,41 @@ class WebViewerIntegration: pid = pid.strip() if not pid: continue - + # Validate PID is numeric only (prevent injection) if not re.match(r'^\d+$', pid): self.logger.warning(f"Invalid PID format: {pid}, skipping") continue - + try: pid_int = int(pid) # Safety check: never kill system PIDs if pid_int < 2: self.logger.warning(f"Refusing to kill system PID: {pid}") continue - + subprocess.run(['kill', '-9', str(pid_int)], timeout=2) self.logger.info(f"Killed remaining process {pid} on port {self.port}") except (ValueError, subprocess.TimeoutExpired) as e: self.logger.warning(f"Failed to kill process {pid}: {e}") except Exception as e: self.logger.debug(f"Port cleanup check failed: {e}") - + except Exception as e: self.logger.error(f"Error stopping web viewer: {e}") - + def _run_viewer(self): """Run the web viewer in a separate process""" stdout_file = None stderr_file = None - + try: # Get the path to the web viewer script viewer_script = Path(__file__).parent / "app.py" # Use same config as bot so viewer finds db_path, Greeter_Command, etc. config_path = getattr(self.bot, 'config_file', 'config.ini') config_path = str(Path(config_path).resolve()) if config_path else 'config.ini' - + # Build command cmd = [ sys.executable, @@ -595,13 +612,13 @@ class WebViewerIntegration: "--host", self.host, "--port", str(self.port) ] - + if self.debug: cmd.append("--debug") - + # Ensure logs directory exists os.makedirs('logs', exist_ok=True) - + # Open log files in write mode to prevent buffer blocking # This fixes the issue where subprocess.PIPE buffers (~64KB) fill up # after ~5 minutes and cause the subprocess to hang. @@ -611,11 +628,11 @@ class WebViewerIntegration: # - Prevents unbounded log file growth stdout_file = open('logs/web_viewer_stdout.log', 'w') stderr_file = open('logs/web_viewer_stderr.log', 'w') - + # Store file handles for proper cleanup self._viewer_stdout_file = stdout_file self._viewer_stderr_file = stderr_file - + # Start the viewer process with log file redirection self.viewer_process = subprocess.Popen( cmd, @@ -623,99 +640,93 @@ class WebViewerIntegration: stderr=stderr_file, text=True ) - + # Give it a moment to start up time.sleep(2) - + # Check if it started successfully if self.viewer_process and self.viewer_process.poll() is not None: # Process failed immediately - read from log files for error reporting stdout_file.flush() stderr_file.flush() - + # Read last few lines from stderr for error reporting try: stderr_file.close() - with open('logs/web_viewer_stderr.log', 'r') as f: + with open('logs/web_viewer_stderr.log') as f: stderr_lines = f.readlines()[-20:] # Last 20 lines stderr = ''.join(stderr_lines) except Exception: stderr = "Could not read stderr log" - + # Read last few lines from stdout for error reporting try: stdout_file.close() - with open('logs/web_viewer_stdout.log', 'r') as f: + with open('logs/web_viewer_stdout.log') as f: stdout_lines = f.readlines()[-20:] # Last 20 lines stdout = ''.join(stdout_lines) except Exception: stdout = "Could not read stdout log" - + self.logger.error(f"Web viewer failed to start. Return code: {self.viewer_process.returncode}") if stderr and stderr.strip(): self.logger.error(f"Web viewer startup error: {stderr}") if stdout and stdout.strip(): self.logger.error(f"Web viewer startup output: {stdout}") - + self.viewer_process = None self._viewer_stdout_file = None self._viewer_stderr_file = None return - + # Web viewer is ready self.logger.info("Web viewer integration ready for data streaming") - + # Monitor the process while self.running and self.viewer_process and self.viewer_process.poll() is None: time.sleep(1) - + # Process exited - read from log files for error reporting if needed if self.viewer_process and self.viewer_process.returncode != 0: stdout_file.flush() stderr_file.flush() - + # Read last few lines from stderr for error reporting try: stderr_file.close() - with open('logs/web_viewer_stderr.log', 'r') as f: + with open('logs/web_viewer_stderr.log') as f: stderr_lines = f.readlines()[-20:] # Last 20 lines stderr = ''.join(stderr_lines) except Exception: stderr = "Could not read stderr log" - + # Close stdout file as well - try: + with suppress(Exception): stdout_file.close() - except Exception: - pass - + self.logger.error(f"Web viewer process exited with code {self.viewer_process.returncode}") if stderr and stderr.strip(): self.logger.error(f"Web viewer stderr: {stderr}") - + self._viewer_stdout_file = None self._viewer_stderr_file = None elif self.viewer_process and self.viewer_process.returncode == 0: self.logger.info("Web viewer process exited normally") - + except Exception as e: self.logger.error(f"Error running web viewer: {e}") # Close file handles on error if stdout_file: - try: + with suppress(Exception): stdout_file.close() - except Exception: - pass if stderr_file: - try: + with suppress(Exception): stderr_file.close() - except Exception: - pass self._viewer_stdout_file = None self._viewer_stderr_file = None finally: self.running = False - + def get_status(self): """Get the current status of the web viewer""" return { @@ -727,37 +738,34 @@ class WebViewerIntegration: 'auto_start': self.auto_start, 'url': f"http://{self.host}:{self.port}" if self.running else None } - + def restart_viewer(self): """Restart the web viewer with rate limiting""" current_time = time.time() - + # Rate limit restarts to prevent restart loops if current_time - self.last_restart < 30: # 30 seconds between restarts self.logger.warning("Restart rate limited - too soon since last restart") return - + if self.restart_count >= self.max_restarts: self.logger.error(f"Maximum restart limit reached ({self.max_restarts}). Web viewer disabled.") self.enabled = False return - + self.restart_count += 1 self.last_restart = current_time - + self.logger.info(f"Restarting web viewer (attempt {self.restart_count}/{self.max_restarts})...") self.stop_viewer() time.sleep(3) # Give it more time to stop - + self.start_viewer() - + def is_viewer_healthy(self): """Check if the web viewer process is healthy""" if not self.viewer_process: return False - + # Check if process is still running - if self.viewer_process.poll() is not None: - return False - - return True + return self.viewer_process.poll() is None diff --git a/modules/web_viewer/templates/base.html b/modules/web_viewer/templates/base.html index 523ab79..bbadedf 100644 --- a/modules/web_viewer/templates/base.html +++ b/modules/web_viewer/templates/base.html @@ -399,6 +399,16 @@ Radio +
  • + + Config + +
    • +
  • + + Logs + +
    • diff --git a/modules/web_viewer/templates/config.html b/modules/web_viewer/templates/config.html new file mode 100644 index 0000000..6c5c05d --- /dev/null +++ b/modules/web_viewer/templates/config.html @@ -0,0 +1,585 @@ +{% extends "base.html" %} + +{% block title %}Configuration - MeshCore Bot{% endblock %} + +{% block content %} +
    +
    +
    +

    + Configuration +

    + + +
    +
    +
    Email & Notifications
    +
    + + +
    +
    +
    +

    + When enabled, a nightly digest is sent summarising maintenance activity + (log rotation, database backup, data retention, error counts). + All fields are stored in the bot database — no config.ini edit required. +

    + +
    + + +
    + SMTP Server +
    +
    +
    + + +
    +
    + + +
    +
    + + +
    +
    + + +
    + Credentials +
    +
    +
    + + +
    +
    + +
    + + +
    +
    + Stored in the bot database. Use an app-specific password where supported. +
    +
    +
    + + +
    + Sender +
    +
    +
    + + +
    +
    + + +
    +
    + + +
    + Recipients +
    +
    +
    + + +
    Separate multiple addresses with commas.
    +
    +
    + + +
    + + + +
    + +
    +
    +
    + + +
    +
    +
    Log Rotation
    +
    +
    +

    + Controls when the log file is rotated and how many backup files are kept. + Changes apply within 60 seconds without a restart. +

    +
    +
    +
    + + +
    Default: 5 242 880 (5 MB). Minimum: 100 KB.
    +
    +
    + + +
    Number of rotated backups to keep (e.g. .log.1, .log.2…).
    +
    +
    +
    + + +
    +
    +
    +
    + + +
    +
    +
    Database Backup
    +
    + + +
    +
    +
    +

    + Creates a consistent SQLite backup using the native backup API. + Old backups beyond the retention count are automatically pruned. +

    +
    +
    +
    + + +
    +
    + + +
    +
    + + +
    +
    +
    +
    + + +
    Absolute path. The directory will be created if it does not exist.
    +
    +
    +
    +
    +
    + + +
    +
    +
    +
    + + +
    +
    +
    +
    + + +
    +
    +
    Maintenance Status
    + +
    +
    + + + + + + + + + + + +
    JobLast ran (UTC)Outcome
    Loading…
    +
    +
    + +
    +
    +
    +{% endblock %} + +{% block extra_js %} + +{% endblock %} diff --git a/modules/web_viewer/templates/contacts.html b/modules/web_viewer/templates/contacts.html index dd168d3..de1c565 100644 --- a/modules/web_viewer/templates/contacts.html +++ b/modules/web_viewer/templates/contacts.html @@ -135,6 +135,38 @@ + +
    + +
      +
    • Contacts
      • +
    • CSV
      • +
    • JSON
      • +
      • +
    • Paths
      • +
    • CSV
      • +
    • JSON
      • +
      • +
    • Time range
      • +
    • +
      + +
      +
      • +
    +
    @@ -187,6 +219,40 @@
    + + + {% endblock %} {% block extra_js %} @@ -480,12 +546,9 @@ class ModernContactsManager { ${contact.advert_count || 0}
    - ${contact.role && (contact.role.toLowerCase() === 'repeater' || contact.role.toLowerCase() === 'roomserver') ? - `` : - '' - } + @@ -1930,11 +1993,93 @@ class ModernContactsManager { }, 5000); } } + + // ── Purge Inactive Contacts ────────────────────────────────────────────── + + async loadPurgePreview(days) { + const previewText = document.getElementById('purge-preview-text'); + const confirmBtn = document.getElementById('confirm-purge-btn'); + const confirmLabel = document.getElementById('confirm-purge-label'); + if (!previewText) return; + previewText.innerHTML = 'Loading...'; + confirmBtn.disabled = true; + try { + const resp = await fetch(`/api/contacts/purge-preview?days=${days}`); + const data = await resp.json(); + if (!resp.ok) throw new Error(data.error || 'Failed to load preview'); + if (data.count === 0) { + previewText.innerHTML = `No contacts found that are older than ${days} days.`; + confirmBtn.disabled = true; + } else { + const sampleNames = (data.samples || []).map(s => `${s.name}`).join(', '); + const more = data.count > 5 ? ` and ${data.count - 5} more` : ''; + previewText.innerHTML = `${data.count} contact(s) will be deleted: ${sampleNames}${more}.`; + confirmLabel.textContent = `Purge ${data.count} contact(s)`; + confirmBtn.disabled = false; + } + } catch (err) { + previewText.innerHTML = `Error: ${err.message}`; + confirmBtn.disabled = true; + } + } + + async executePurge(days) { + const confirmBtn = document.getElementById('confirm-purge-btn'); + const confirmLabel = document.getElementById('confirm-purge-label'); + const originalLabel = confirmLabel.textContent; + confirmBtn.disabled = true; + confirmLabel.textContent = 'Purging...'; + try { + const resp = await fetch('/api/contacts/purge', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ days }) + }); + const data = await resp.json(); + if (!resp.ok) throw new Error(data.error || 'Purge failed'); + bootstrap.Modal.getInstance(document.getElementById('purgeContactsModal')).hide(); + this.showSuccess(data.message || `Purged ${data.deleted} contact(s)`); + await this.loadContactsData(); + } catch (err) { + this.showError('Purge failed: ' + err.message); + confirmLabel.textContent = originalLabel; + confirmBtn.disabled = false; + } + } + + setupPurgeModal() { + const modal = document.getElementById('purgeContactsModal'); + const select = document.getElementById('purge-days-select'); + const confirmBtn = document.getElementById('confirm-purge-btn'); + if (!modal) return; + + modal.addEventListener('show.bs.modal', () => { + this.loadPurgePreview(parseInt(select.value)); + }); + select.addEventListener('change', () => { + this.loadPurgePreview(parseInt(select.value)); + }); + confirmBtn.addEventListener('click', () => { + this.executePurge(parseInt(select.value)); + }); + } +} + +function exportData(dataset, fmt) { + const since = document.getElementById('export-since').value || '30d'; + const url = `/api/export/${dataset}?format=${fmt}&since=${since}`; + const a = document.createElement('a'); + a.href = url; + a.download = ''; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); } // Initialize contacts manager when page loads document.addEventListener('DOMContentLoaded', () => { window.contactsManager = new ModernContactsManager(); + window.contactsManager.setupPurgeModal(); }); diff --git a/modules/web_viewer/templates/index.html b/modules/web_viewer/templates/index.html index 81f89b0..bc97320 100644 --- a/modules/web_viewer/templates/index.html +++ b/modules/web_viewer/templates/index.html @@ -312,6 +312,38 @@
    + +
    +
    +
    +
    + Live Activity + + +
    + 0 + + + + Full Monitor + +
    +
    +
    +
    +
    + Connecting… +
    +
    +
    +
    +
    +
    + {% endblock %} {% block extra_js %} @@ -742,6 +774,81 @@ window.addEventListener('beforeunload', () => { }); + + + + + + + diff --git a/modules/web_viewer/templates/logs.html b/modules/web_viewer/templates/logs.html new file mode 100644 index 0000000..16c8e3f --- /dev/null +++ b/modules/web_viewer/templates/logs.html @@ -0,0 +1,238 @@ +{% extends "base.html" %} + +{% block title %}Live Log Viewer - MeshCore Bot Data Viewer{% endblock %} + +{% block extra_css %} + +{% endblock %} + +{% block content %} +
    +
    +

    + Live Log Viewer +

    + + Real-time Monitor + +
    +
    + +
    +
    +
    +
    Bot Log Stream
    + Connecting… +
    +
    + 0 lines + + + + +
    +
    +
    +
    +
    + Waiting for log data… +
    +
    +
    +
    +{% endblock %} + +{% block extra_js %} + +{% endblock %} diff --git a/modules/web_viewer/templates/radio.html b/modules/web_viewer/templates/radio.html index 9e481bb..4964d0f 100644 --- a/modules/web_viewer/templates/radio.html +++ b/modules/web_viewer/templates/radio.html @@ -6,9 +6,18 @@
    -

    - Radio Settings -

    +
    +

    + Radio Settings +

    +
    + +
    +
    @@ -147,17 +156,20 @@ class RadioManager { constructor() { this.channels = []; this.maxChannels = 40; // Default, will be updated from API + this.radioConnected = null; this.initialize(); } - + async initialize() { await this.loadChannels(); await this.loadStatistics(); + await this.loadRadioStatus(); this.setupEventHandlers(); - + // Auto-refresh every 30 seconds setInterval(() => this.loadChannels(), 30000); setInterval(() => this.loadStatistics(), 60000); + setInterval(() => this.loadRadioStatus(), 15000); } async loadChannels() { @@ -331,6 +343,114 @@ class RadioManager { if (saveBtn) { saveBtn.addEventListener('click', () => this.saveChannel()); } + + // Radio control buttons + const connectToggleBtn = document.getElementById('connectToggleBtn'); + if (connectToggleBtn) { + connectToggleBtn.addEventListener('click', () => this.handleConnectToggle()); + } + } + + async loadRadioStatus() { + try { + const response = await fetch('/api/radio/status'); + const data = await response.json(); + this.radioConnected = data.connected; + this.updateConnectButton(); + } catch (error) { + console.error('Error loading radio status:', error); + } + } + + updateConnectButton() { + const btn = document.getElementById('connectToggleBtn'); + const txt = document.getElementById('connectBtnText'); + if (!btn || !txt) return; + + btn.disabled = false; + if (this.radioConnected === null) { + btn.className = 'btn btn-secondary'; + txt.textContent = 'Status Unknown'; + } else if (this.radioConnected) { + btn.className = 'btn btn-danger'; + txt.textContent = 'Disconnect'; + } else { + btn.className = 'btn btn-success'; + txt.textContent = 'Connect'; + } + } + + async handleConnectToggle() { + const action = this.radioConnected ? 'disconnect' : 'connect'; + const spinner = document.getElementById('connectBtnSpinner'); + const btn = document.getElementById('connectToggleBtn'); + const txt = document.getElementById('connectBtnText'); + + btn.disabled = true; + spinner.style.display = 'inline-block'; + txt.textContent = action === 'connect' ? 'Connecting...' : 'Disconnecting...'; + + try { + const response = await fetch('/api/radio/connect', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ action }) + }); + const data = await response.json(); + + if (response.ok && data.operation_id) { + await this.pollConnectOperation(data.operation_id, action); + } else { + this.showError(data.error || 'Failed to queue operation'); + spinner.style.display = 'none'; + btn.disabled = false; + this.updateConnectButton(); + } + } catch (error) { + this.showError('Error: ' + error.message); + spinner.style.display = 'none'; + btn.disabled = false; + this.updateConnectButton(); + } + } + + async pollConnectOperation(operationId, action, maxWait = 60) { + const startTime = Date.now(); + const spinner = document.getElementById('connectBtnSpinner'); + const txt = document.getElementById('connectBtnText'); + const btn = document.getElementById('connectToggleBtn'); + + for (let attempts = 0; attempts < maxWait; attempts++) { + await new Promise(resolve => setTimeout(resolve, 2000)); + const elapsed = Math.floor((Date.now() - startTime) / 1000); + txt.textContent = `${action === 'connect' ? 'Connecting' : 'Disconnecting'}... (${elapsed}s)`; + + try { + const response = await fetch(`/api/channel-operations/${operationId}`); + const result = await response.json(); + + if (result.status === 'completed') { + spinner.style.display = 'none'; + await this.loadRadioStatus(); + this.showSuccess(`Radio ${action}ed successfully`); + return; + } else if (result.status === 'failed') { + spinner.style.display = 'none'; + btn.disabled = false; + this.updateConnectButton(); + this.showError(result.error_message || `Failed to ${action} radio`); + return; + } + } catch (error) { + console.error('Error polling operation:', error); + } + } + + // Timeout + spinner.style.display = 'none'; + btn.disabled = false; + await this.loadRadioStatus(); + this.showError('Operation timed out — check radio status.'); } handleChannelNameChange(channelName) { diff --git a/modules/web_viewer/templates/realtime.html b/modules/web_viewer/templates/realtime.html index 7cc60ac..e443da7 100644 --- a/modules/web_viewer/templates/realtime.html +++ b/modules/web_viewer/templates/realtime.html @@ -433,10 +433,13 @@ {% block content %}
    -
    -

    +
    +

    Real-time Monitoring

    + + Live Log Viewer +

    @@ -481,6 +484,31 @@
    + +
    +
    +
    +
    +
    Live Channel Messages
    +
    + Active + + +
    +
    +
    +
    +
    + Waiting for channel messages… +
    +
    +
    +
    +
    +
    +