meshcore-bot/modules/web_viewer/integration.py

#!/usr/bin/env python3
"""
Web Viewer Integration for MeshCore Bot
Provides integration between the main bot and the web viewer
"""

import os
import queue
import re
import secrets
import subprocess
import sys
import threading
import time
from contextlib import closing, contextmanager, suppress
from pathlib import Path
from typing import Optional

from ..db_retention import (
    delete_timestamp_rows_in_chunks,
    retention_delete_settings,
)
from ..utils import resolve_path


def normalized_web_viewer_password(config) -> str:
    """Return the effective web viewer password, or '' to disable the login screen.

    Blank values, quoted empties (e.g. INI ``""``), and placeholders ``none`` / ``null`` /
    ``nil`` (case-insensitive) are treated as no password.
    """
    if not config.has_section("Web_Viewer"):
        return ""
    raw = config.get("Web_Viewer", "web_viewer_password", fallback="")
    if raw is None:
        return ""
    s = str(raw).strip()
    if not s:
        return ""
    if len(s) >= 2 and s[0] == s[-1] and s[0] in ('"', "'"):
        s = s[1:-1].strip()
    if not s:
        return ""
    if s.lower() in ("none", "null", "nil"):
        return ""
    return s


class BotIntegration:
    """Simple bot integration for web viewer compatibility"""

    # After this many consecutive connection failures, stop sending until cooldown expires
    CIRCUIT_BREAKER_THRESHOLD = 3
    CIRCUIT_BREAKER_COOLDOWN_SEC = 60

    # How often (seconds) the drain thread flushes the write queue
    DRAIN_INTERVAL = 0.5

    # Bounded packet_stream write queue: producers block up to this long before drop/retry.
    _WRITE_QUEUE_PUT_TIMEOUT_SEC = 5.0
    EDGE_POST_TIMEOUT_SEC = 1.0
    NODE_POST_TIMEOUT_SEC = 0.5
    SQLITE_CONNECT_TIMEOUT_SEC = 60.0
    REQUEUE_PUT_TIMEOUT_SEC = 60.0
    SHUTDOWN_JOIN_TIMEOUT_SEC = 5.0

    def __init__(self, bot):
        self.bot = bot
        self.circuit_breaker_open = False
        self.circuit_breaker_failures = 0
        self.circuit_breaker_last_failure_time = 0.0
        self.is_shutting_down = False
        # Serialize flushes so drain thread, shutdown, and producer retry cannot interleave.
        self._flush_lock = threading.Lock()
        maxsize = 1000
        if self.bot.config.has_section("Web_Viewer"):
            try:
                maxsize = self.bot.config.getint(
                    "Web_Viewer", "packet_stream_write_queue_max", fallback=1000
                )
            except (ValueError, TypeError, OSError):
                maxsize = 1000
        self._write_queue_maxsize = max(1, int(maxsize))
        # Batched write queue: avoids a per-insert sqlite3.connect() round-trip (bounded for RAM).
        self._write_queue: queue.Queue = queue.Queue(maxsize=self._write_queue_maxsize)
        self._drain_stop = threading.Event()
        self._drain_thread: Optional[threading.Thread] = None
        self._load_timeouts_from_config()
        # Initialize HTTP session with connection pooling for efficient reuse
        self._init_http_session()
        # Generate a shared secret for authenticating internal /api/stream_data calls.
        # Stored in DB metadata so the web viewer can validate it.
        self._stream_token = secrets.token_hex(32)
        try:
            self.bot.db_manager.set_metadata('internal.stream_token', self._stream_token)
        except Exception as e:
            self.bot.logger.debug(f"Could not persist stream token: {e}")
        if getattr(self, 'http_session', None):
            self.http_session.headers['X-Stream-Token'] = self._stream_token
        # Start background drain thread after table is confirmed to exist
        self._start_drain_thread()

    def _get_float_config(self, key: str, fallback: float) -> float:
        """Read float from [Web_Viewer] config with sane fallback."""
        try:
            value = self.bot.config.getfloat("Web_Viewer", key, fallback=fallback)
            return value if value > 0 else fallback
        except (ValueError, TypeError, OSError):
            return fallback

    def _load_timeouts_from_config(self) -> None:
        """Load optional BotIntegration timeout settings from config."""
        self.edge_post_timeout_sec = self._get_float_config(
            "edge_post_timeout_sec", self.EDGE_POST_TIMEOUT_SEC
        )
        self.node_post_timeout_sec = self._get_float_config(
            "node_post_timeout_sec", self.NODE_POST_TIMEOUT_SEC
        )
        self.sqlite_connect_timeout_sec = self._get_float_config(
            "sqlite_connect_timeout_sec", self.SQLITE_CONNECT_TIMEOUT_SEC
        )
        self.requeue_put_timeout_sec = self._get_float_config(
            "requeue_put_timeout_sec", self.REQUEUE_PUT_TIMEOUT_SEC
        )
        self.shutdown_join_timeout_sec = self._get_float_config(
            "integration_shutdown_join_timeout_sec",
            self.SHUTDOWN_JOIN_TIMEOUT_SEC,
        )

    def _init_http_session(self):
        """Initialize a requests.Session with connection pooling and keep-alive"""
        try:
            import logging

            import requests
            import urllib3
            from requests.adapters import HTTPAdapter
            from urllib3.util.retry import Retry

            # Suppress urllib3 connection pool messages when web viewer is unreachable
            # Connection refused / Retrying WARNINGs would flood logs during routing bursts
            urllib3_logger = logging.getLogger('urllib3.connectionpool')
            urllib3_logger.setLevel(logging.ERROR)

            # Also disable other urllib3 warnings
            urllib3.disable_warnings(urllib3.exceptions.NotOpenSSLWarning)

            self.http_session = requests.Session()

            # Configure retry strategy
            retry_strategy = Retry(
                total=2,
                backoff_factor=0.1,
                status_forcelist=[429, 500, 502, 503, 504],
            )

            # Mount adapter with connection pooling
            # pool_block=False allows non-blocking behavior if pool is full
            adapter = HTTPAdapter(
                pool_connections=1,  # Single connection pool for web viewer
                pool_maxsize=5,      # Allow up to 5 connections in the pool
                max_retries=retry_strategy,
                pool_block=False     # Don't block if pool is full
            )
            self.http_session.mount("http://", adapter)
            self.http_session.mount("https://", adapter)

            # Set default headers for keep-alive and internal auth
            self.http_session.headers.update({
                'Connection': 'keep-alive',
                'X-Requested-With': 'BotIntegration',  # CSRF bypass for internal calls
            })
        except ImportError:
            # Fallback if requests is not available
            self.http_session = None
        except Exception as e:
            self.bot.logger.debug(f"Error initializing HTTP session: {e}")
            self.http_session = None

    def reset_circuit_breaker(self):
        """Reset the circuit breaker"""
        self.circuit_breaker_open = False
        self.circuit_breaker_failures = 0

    def _should_skip_web_viewer_send(self):
        """Return True if we should skip sending (circuit open and within cooldown)."""
        if not self.circuit_breaker_open:
            return False
        if (time.time() - self.circuit_breaker_last_failure_time) >= self.CIRCUIT_BREAKER_COOLDOWN_SEC:
            self.reset_circuit_breaker()
            return False
        return True

    def _record_web_viewer_result(self, success):
        """Update circuit breaker state after a send attempt."""
        if success:
            self.reset_circuit_breaker()
        else:
            self.circuit_breaker_failures += 1
            self.circuit_breaker_last_failure_time = time.time()
            if self.circuit_breaker_failures >= self.CIRCUIT_BREAKER_THRESHOLD:
                self.circuit_breaker_open = True
                self.bot.logger.debug(
                    "Web viewer unreachable after %d failures; circuit open for %ds",
                    self.circuit_breaker_failures,
                    self.CIRCUIT_BREAKER_COOLDOWN_SEC,
                )

    def _get_web_viewer_db_path(self):
        """Return resolved database path for web viewer. Uses [Bot] db_path when [Web_Viewer] db_path is unset."""
        base_dir = self.bot.bot_root if hasattr(self.bot, 'bot_root') else '.'
        if self.bot.config.has_section('Web_Viewer') and self.bot.config.has_option('Web_Viewer', 'db_path'):
            raw = self.bot.config.get('Web_Viewer', 'db_path', fallback='').strip()
            if raw:
                return resolve_path(raw, base_dir)
        return str(Path(self.bot.db_manager.db_path).resolve())

    # Denormalized packet dimensions written alongside the JSON blob.  Order
    # matters: it is the INSERT column order and the queue tuple order.
    PACKET_DIM_COLUMNS = ("route_type_name", "payload_type_name", "path_len", "bytes_per_hop")

    def _init_packet_stream_table(self):
        """Backward-compatible initializer (now handled by migrations).

        Kept for older call sites and tests that patch this method. Safe to call
        multiple times and safe to ignore failures.

        The table-exists path still runs the ALTERs: this process writes to
        ``[Web_Viewer] db_path``, which in a split-DB install is a file the bot's
        own MigrationRunner never touches.  If the viewer has not started yet
        that DB has an old three-column packet_stream, and skipping the ALTERs
        would leave every insert failing on unknown columns.
        """
        try:
            import sqlite3

            db_path = self._get_web_viewer_db_path()
            with closing(sqlite3.connect(str(db_path), timeout=self.sqlite_connect_timeout_sec)) as conn:
                cur = conn.execute(
                    "SELECT name FROM sqlite_master WHERE type='table' AND name='packet_stream'"
                )
                if cur.fetchone() is not None:
                    self._add_missing_packet_dim_columns(conn)
                    conn.commit()
                    return
                try:
                    foreign_keys = self.bot.config.getboolean("Web_Viewer", "sqlite_foreign_keys", fallback=True)
                    busy_timeout_ms = self.bot.config.getint("Web_Viewer", "sqlite_busy_timeout_ms", fallback=60000)
                    journal_mode = self.bot.config.get("Web_Viewer", "sqlite_journal_mode", fallback="WAL").strip() or "WAL"
                except Exception:
                    foreign_keys = True
                    busy_timeout_ms = 60000
                    journal_mode = "WAL"
                conn.execute(f"PRAGMA foreign_keys={'ON' if foreign_keys else 'OFF'}")
                conn.execute(f"PRAGMA busy_timeout={int(busy_timeout_ms)}")
                try:
                    conn.execute(f"PRAGMA journal_mode={journal_mode}")
                except sqlite3.OperationalError:
                    pass
                conn.execute(
                    """
                    CREATE TABLE IF NOT EXISTS packet_stream (
                        id INTEGER PRIMARY KEY AUTOINCREMENT,
                        timestamp REAL NOT NULL,
                        data TEXT NOT NULL,
                        type TEXT NOT NULL,
                        route_type_name TEXT,
                        payload_type_name TEXT,
                        path_len INTEGER,
                        bytes_per_hop INTEGER
                    )
                    """
                )
                conn.execute(
                    "CREATE INDEX IF NOT EXISTS idx_packet_stream_timestamp ON packet_stream(timestamp)"
                )
                conn.execute(
                    "CREATE INDEX IF NOT EXISTS idx_packet_stream_type ON packet_stream(type)"
                )
                conn.execute(
                    "CREATE INDEX IF NOT EXISTS idx_packet_stream_dims "
                    "ON packet_stream(timestamp, bytes_per_hop, route_type_name, payload_type_name) "
                    "WHERE type = 'packet'"
                )
                conn.execute(
                    "CREATE INDEX IF NOT EXISTS idx_packet_stream_undimensioned "
                    "ON packet_stream(type, id) WHERE route_type_name IS NULL"
                )
                conn.commit()
        except Exception:
            pass

    def _add_missing_packet_dim_columns(self, conn) -> None:
        """ALTER in any denormalized dimension column packet_stream is missing."""
        existing = {row[1] for row in conn.execute("PRAGMA table_info(packet_stream)")}
        types = {"path_len": "INTEGER", "bytes_per_hop": "INTEGER"}
        for column in self.PACKET_DIM_COLUMNS:
            if column not in existing:
                conn.execute(
                    f"ALTER TABLE packet_stream ADD COLUMN {column} {types.get(column, 'TEXT')}"
                )
        if self.PACKET_DIM_COLUMNS[0] not in existing:
            conn.execute(
                "CREATE INDEX IF NOT EXISTS idx_packet_stream_dims "
                "ON packet_stream(timestamp, bytes_per_hop, route_type_name, payload_type_name) "
                "WHERE type = 'packet'"
            )
            conn.execute(
                "CREATE INDEX IF NOT EXISTS idx_packet_stream_undimensioned "
                "ON packet_stream(type, id) WHERE route_type_name IS NULL"
            )

    def _packet_stream_has_dim_columns(self, conn) -> bool:
        """Probe once per process whether the target DB carries the dimension columns.

        The viewer DB may be a separate file this process never migrates, so the
        writer cannot assume the wide schema — it falls back to the three-column
        INSERT rather than failing every flush.
        """
        cached = getattr(self, '_packet_dims_supported', None)
        if cached is not None:
            return cached
        try:
            existing = {row[1] for row in conn.execute("PRAGMA table_info(packet_stream)")}
            supported = all(c in existing for c in self.PACKET_DIM_COLUMNS)
        except Exception:
            supported = False
        self._packet_dims_supported = supported
        if not supported:
            self.bot.logger.info(
                "packet_stream lacks denormalized dimension columns; "
                "writing legacy rows (dashboard packet tiles will backfill from JSON)"
            )
        return supported

    def _start_drain_thread(self) -> None:
        """Start the background thread that flushes the write queue every DRAIN_INTERVAL seconds."""
        self._drain_thread = threading.Thread(
            target=self._drain_loop,
            name="packet-stream-drain",
            daemon=True,
        )
        self._drain_thread.start()

    def _drain_loop(self) -> None:
        """Background loop: flush the write queue every DRAIN_INTERVAL seconds."""
        while not self._drain_stop.is_set():
            self._drain_stop.wait(timeout=self.DRAIN_INTERVAL)
            self._flush_write_queue()

    def _requeue_rows(self, rows: list[tuple]) -> None:
        """Restore rows to the queue after a failed flush (FIFO). Logs if the queue stays full."""
        for i, row in enumerate(rows):
            try:
                self._write_queue.put(row, timeout=self.requeue_put_timeout_sec)
            except queue.Full:
                remaining = len(rows) - i
                self.bot.logger.error(
                    "packet_stream: could not re-queue %d row(s) after flush failure; data may be lost",
                    remaining,
                )
                break

    def _flush_write_queue(self) -> None:
        """Drain all queued rows and insert them in a single batched transaction."""
        import sqlite3

        with self._flush_lock:
            if self._write_queue.empty():
                return
            rows: list[tuple] = []
            while not self._write_queue.empty():
                try:
                    rows.append(self._write_queue.get_nowait())
                except queue.Empty:
                    break
            if not rows:
                return
            db_path = self._get_web_viewer_db_path()
            max_retries = 3
            for attempt in range(max_retries):
                try:
                    with closing(sqlite3.connect(str(db_path), timeout=self.sqlite_connect_timeout_sec)) as conn:
                        if self._packet_stream_has_dim_columns(conn):
                            conn.executemany(
                                'INSERT INTO packet_stream '
                                '(timestamp, data, type, route_type_name, payload_type_name, '
                                'path_len, bytes_per_hop) VALUES (?, ?, ?, ?, ?, ?, ?)',
                                rows,
                            )
                        else:
                            conn.executemany(
                                'INSERT INTO packet_stream (timestamp, data, type) VALUES (?, ?, ?)',
                                [row[:3] for row in rows],
                            )
                        conn.commit()
                    return
                except sqlite3.OperationalError as e:
                    if "locked" in str(e).lower() and attempt < max_retries - 1:
                        time.sleep(0.15 * (attempt + 1))
                        continue
                    self.bot.logger.warning(
                        f"Error flushing packet_stream queue ({len(rows)} rows): {e}"
                    )
                    self._requeue_rows(rows)
                    return
                except Exception as e:
                    self.bot.logger.warning(
                        f"Error flushing packet_stream queue ({len(rows)} rows): {e}"
                    )
                    self._requeue_rows(rows)
                    return

    def _insert_packet_stream_row(
        self,
        data_json: str,
        row_type: str,
        log_prefix: str = "packet data",
        dims: tuple | None = None,
    ):
        """Queue one row for batched insertion into packet_stream by the drain thread.

        ``dims`` carries the denormalized (route_type_name, payload_type_name,
        path_len, bytes_per_hop) values for packet rows; command/message/other
        row types pass None and store NULLs.
        """
        item = (time.time(), data_json, row_type, *(dims or (None, None, None, None)))
        try:
            self._write_queue.put(item, timeout=self._WRITE_QUEUE_PUT_TIMEOUT_SEC)
        except queue.Full:
            try:
                self._flush_write_queue()
            except Exception as e:
                self.bot.logger.debug("packet_stream flush after full queue: %s", e)
            try:
                self._write_queue.put(item, timeout=self._WRITE_QUEUE_PUT_TIMEOUT_SEC)
            except queue.Full:
                self.bot.logger.warning(
                    "packet_stream write queue full (%s/%s items) after flush retry; dropping %s",
                    self._write_queue.qsize(),
                    self._write_queue.maxsize,
                    log_prefix,
                )
        except Exception as e:
            self.bot.logger.warning(f"Error queuing {log_prefix} for web viewer: {e}")

    def capture_full_packet_data(self, packet_data):
        """Capture full packet data and store in database for web viewer"""
        try:
            import json
            from datetime import datetime

            # Ensure packet_data is a dict (might be passed as dict already)
            if not isinstance(packet_data, dict):
                packet_data = self._make_json_serializable(packet_data)
                if not isinstance(packet_data, dict):
                    # If still not a dict, wrap it
                    packet_data = {'data': packet_data}

            # Add hops field from path_len if not already present
            # path_len represents the number of hops (each byte = 1 hop)
            if 'hops' not in packet_data and 'path_len' in packet_data:
                packet_data['hops'] = packet_data.get('path_len', 0)
            elif 'hops' not in packet_data:
                # If no path_len either, default to 0 hops
                packet_data['hops'] = 0

            # Add datetime for frontend display
            if 'datetime' not in packet_data:
                packet_data['datetime'] = datetime.now().isoformat()

            # Convert non-serializable objects to strings
            serializable_data = self._make_json_serializable(packet_data)

            # Store in database for web viewer to read (retries on database is locked)
            self._insert_packet_stream_row(
                json.dumps(serializable_data),
                'packet',
                "packet data",
                dims=self._packet_dims_from_data(serializable_data),
            )

        except Exception as e:
            self.bot.logger.warning(f"Error storing packet data for web viewer: {e}")

    @staticmethod
    def _packet_dims_from_data(packet_data: dict) -> tuple:
        """Pull the denormalized dimension values out of an already-parsed packet.

        Returns (route_type_name, payload_type_name, path_len, bytes_per_hop),
        with None for anything absent or non-coercible — the dashboard reads
        these as "unknown" rather than mis-bucketing the packet.
        """
        def _text(key: str) -> str | None:
            value = packet_data.get(key)
            if value is None or isinstance(value, (dict, list)):
                return None
            text = str(value).strip()
            return text[:32] or None

        def _number(key: str) -> int | None:
            value = packet_data.get(key)
            if value is None or isinstance(value, bool):
                return None
            try:
                return int(value)
            except (TypeError, ValueError):
                return None

        return (
            # Never NULL: route_type_name is the dashboard's "this row has been
            # dimensioned" marker, and a NULL would put the row back in the
            # refresher's backfill queue on every tick.
            _text('route_type_name') or 'UNKNOWN_ROUTE',
            _text('payload_type_name'),
            _number('path_len'),
            _number('bytes_per_hop'),
        )

    @staticmethod
    def _on_air_content(message, fallback: str = "") -> str:
        """Return the on-air body for web-viewer capture.

        Command matching may have stripped @[mentions] from ``message.content``
        (#267). Prefer ``original_content``. Mocks may lack that field
        (MagicMock would invent a non-str attribute), so only honor a real string.
        """
        original = getattr(message, "original_content", None)
        if isinstance(original, str) and original:
            return original
        content = getattr(message, "content", fallback)
        return content if isinstance(content, str) else fallback

    def capture_command(self, message, command_name, response, success, command_id=None):
        """Capture command data and store in database for web viewer"""
        try:
            import json
            import time

            # Extract data from message object
            user = getattr(message, 'sender_id', 'Unknown')
            channel = getattr(message, 'channel', 'Unknown')
            user_input = self._on_air_content(message, fallback=f'/{command_name}')

            # Get repeat information if transmission tracker is available
            repeat_count = 0
            repeater_prefixes = []
            repeater_counts = {}
            reply_packet_hash = None
            if (hasattr(self.bot, 'transmission_tracker') and
                self.bot.transmission_tracker and
                command_id):
                repeat_info = self.bot.transmission_tracker.get_repeat_info(command_id=command_id)
                repeat_count = repeat_info.get('repeat_count', 0)
                repeater_prefixes = repeat_info.get('repeater_prefixes', [])
                repeater_counts = repeat_info.get('repeater_counts', {})
                reply_packet_hash = repeat_info.get('packet_hash')

            # Construct command data structure
            command_data = {
                'user': user,
                'channel': channel,
                'command': command_name,
                'user_input': user_input,
                'response': response,
                'success': success,
                'timestamp': time.time(),
                'repeat_count': repeat_count,
                'repeater_prefixes': repeater_prefixes,
                'repeater_counts': repeater_counts,  # Count per repeater prefix
                'command_id': command_id  # Store command_id for later updates
            }
            # Packet hashes tie the command to the packets the radio heard: the
            # message that triggered it and, once a repeater echoes it, the reply.
            trigger_packet_hash = self._message_packet_hash(message)
            if trigger_packet_hash:
                command_data['trigger_packet_hash'] = trigger_packet_hash
            if reply_packet_hash:
                command_data['reply_packet_hash'] = reply_packet_hash

            # Convert non-serializable objects to strings
            serializable_data = self._make_json_serializable(command_data)

            # Store in database for web viewer to read (retries on database is locked)
            self._insert_packet_stream_row(json.dumps(serializable_data), 'command', "command data")

        except Exception as e:
            self.bot.logger.debug(f"Error storing command data: {e}")

    def capture_channel_message(self, message) -> None:
        """Capture an incoming channel or DM message for the web viewer live monitor."""
        try:
            import json
            import time
            data = {
                'type': 'message',
                'timestamp': time.time(),
                'sender': getattr(message, 'sender_id', ''),
                'channel': getattr(message, 'channel', ''),
                'content': self._on_air_content(message),
                'snr': str(getattr(message, 'snr', '')),
                'hops': getattr(message, 'hops', None),
                'path': getattr(message, 'path', ''),
                'is_dm': bool(getattr(message, 'is_dm', False)),
            }
            packet_hash = self._message_packet_hash(message)
            if packet_hash:
                data['packet_hash'] = packet_hash
            self._insert_packet_stream_row(json.dumps(data), 'message', "channel message")
        except Exception as e:
            self.bot.logger.debug(f"Error storing channel message for web viewer: {e}")

    @staticmethod
    def _message_packet_hash(message) -> Optional[str]:
        """Hash of the RF packet correlated with *message*, or None when there wasn't one."""
        routing_info = getattr(message, 'routing_info', None) or {}
        if not isinstance(routing_info, dict):
            return None
        packet_hash = routing_info.get('packet_hash')
        if packet_hash and str(packet_hash).strip('0'):
            return packet_hash
        return None

    def capture_packet_routing(self, routing_data):
        """Capture packet routing data and store in database for web viewer"""
        try:
            import json

            # Convert non-serializable objects to strings
            serializable_data = self._make_json_serializable(routing_data)

            # Store in database for web viewer to read (retries on database is locked)
            self._insert_packet_stream_row(json.dumps(serializable_data), 'routing', "routing data")

        except Exception as e:
            self.bot.logger.debug(f"Error storing routing data: {e}")

    def cleanup_old_data(self, days_to_keep: Optional[int] = None):
        """Clean up old packet stream data to prevent database bloat.
        Uses [Data_Retention] packet_stream_retention_days when days_to_keep is not provided."""
        try:
            import sqlite3

            if days_to_keep is None:
                days_to_keep = 3
                if self.bot.config.has_section('Data_Retention') and self.bot.config.has_option('Data_Retention', 'packet_stream_retention_days'):
                    with suppress(ValueError, TypeError):
                        days_to_keep = self.bot.config.getint('Data_Retention', 'packet_stream_retention_days')

            cutoff_time = time.time() - (days_to_keep * 24 * 60 * 60)

            db_path = self._get_web_viewer_db_path()
            batch_size, pause_seconds = retention_delete_settings(self.bot.config)

            @contextmanager
            def cleanup_connection():
                with closing(
                    sqlite3.connect(
                        str(db_path),
                        timeout=self.sqlite_connect_timeout_sec,
                    )
                ) as conn:
                    conn.execute(
                        f"PRAGMA busy_timeout={int(self.sqlite_connect_timeout_sec * 1000)}"
                    )
                    yield conn

            deleted_count = delete_timestamp_rows_in_chunks(
                cleanup_connection,
                'packet_stream',
                'timestamp',
                cutoff_time,
                batch_size=batch_size,
                pause_seconds=pause_seconds,
                logger=self.bot.logger,
                progress_label='packet stream',
            )

            if deleted_count > 0:
                self.bot.logger.info(f"Cleaned up {deleted_count} old packet stream entries (older than {days_to_keep} days)")

        except Exception as e:
            self.bot.logger.error(f"Error cleaning up old packet stream data: {e}")

    def _make_json_serializable(self, obj, depth=0, max_depth=3):
        """Convert non-JSON-serializable objects to strings with depth limiting"""
        if depth > max_depth:
            return str(obj)

        # Handle basic types first
        if obj is None or isinstance(obj, (str, int, float, bool)):
            return obj
        elif isinstance(obj, (list, tuple)):
            return [self._make_json_serializable(item, depth + 1) for item in obj]
        elif isinstance(obj, dict):
            return {k: self._make_json_serializable(v, depth + 1) for k, v in obj.items()}
        elif hasattr(obj, 'name'):  # Enum-like objects
            return obj.name
        elif hasattr(obj, 'value'):  # Enum values
            return obj.value
        elif hasattr(obj, '__dict__'):
            # Convert objects to dict, but limit depth
            try:
                return {k: self._make_json_serializable(v, depth + 1) for k, v in obj.__dict__.items()}
            except (RecursionError, RuntimeError):
                return str(obj)
        else:
            return str(obj)

    def send_mesh_edge_update(self, edge_data):
        """Send mesh edge update to web viewer via HTTP API"""
        try:
            if self._should_skip_web_viewer_send():
                return
            # Get web viewer URL from config
            host = self.bot.config.get('Web_Viewer', 'host', fallback='127.0.0.1')
            port = self.bot.config.getint('Web_Viewer', 'port', fallback=8080)
            url = f"http://{host}:{port}/api/stream_data"

            payload = {
                'type': 'mesh_edge',
                'data': edge_data
            }

            # Use session with connection pooling if available, otherwise fallback to requests.post
            headers = {
                'X-Stream-Token': self._stream_token,
                'X-Requested-With': 'BotIntegration',
            }
            if self.http_session:
                try:
                    self.http_session.post(url, json=payload, timeout=self.edge_post_timeout_sec)
                    self._record_web_viewer_result(True)
                except Exception:
                    self._record_web_viewer_result(False)
            else:
                import requests
                try:
                    requests.post(url, json=payload, timeout=self.edge_post_timeout_sec, headers=headers)
                    self._record_web_viewer_result(True)
                except Exception:
                    self._record_web_viewer_result(False)
        except Exception as e:
            self.bot.logger.debug(f"Error sending mesh edge update to web viewer: {e}")

    def send_mesh_node_update(self, node_data):
        """Send mesh node update to web viewer via HTTP API"""
        try:
            if self._should_skip_web_viewer_send():
                return
            import requests

            host = self.bot.config.get('Web_Viewer', 'host', fallback='127.0.0.1')
            port = self.bot.config.getint('Web_Viewer', 'port', fallback=8080)
            url = f"http://{host}:{port}/api/stream_data"

            payload = {
                'type': 'mesh_node',
                'data': node_data
            }

            headers = {
                'X-Stream-Token': self._stream_token,
                'X-Requested-With': 'BotIntegration',
            }
            try:
                requests.post(url, json=payload, timeout=self.node_post_timeout_sec, headers=headers)
                self._record_web_viewer_result(True)
            except Exception:
                self._record_web_viewer_result(False)
        except Exception as e:
            self.bot.logger.debug(f"Error sending mesh node update to web viewer: {e}")

    def shutdown(self):
        """Mark as shutting down, stop drain thread, flush remaining rows, and close HTTP session."""
        self.is_shutting_down = True
        # Stop drain thread and do a final flush of any queued rows
        self._drain_stop.set()
        if self._drain_thread and self._drain_thread.is_alive():
            self._drain_thread.join(timeout=self.shutdown_join_timeout_sec)
        self._flush_write_queue()
        # Close HTTP session to clean up connections
        if hasattr(self, 'http_session') and self.http_session:
            with suppress(Exception):
                self.http_session.close()

class WebViewerIntegration:
    """Integration class for starting/stopping the web viewer with the bot"""

    # Whitelist of allowed host bindings for security
    ALLOWED_HOSTS = ['127.0.0.1', 'localhost', '0.0.0.0']
    VIEWER_STOP_GRACE_TIMEOUT_SEC = 5.0
    VIEWER_STOP_FORCE_TIMEOUT_SEC = 2.0
    PORT_CLEANUP_LSOF_TIMEOUT_SEC = 5.0
    PORT_CLEANUP_KILL_TIMEOUT_SEC = 2.0

    def __init__(self, bot):
        self.bot = bot
        self.logger = bot.logger
        self.viewer_process = None
        self.viewer_thread = None
        self.running = False

        # File handles for subprocess stdout/stderr (for proper cleanup)
        self._viewer_stdout_file = None
        self._viewer_stderr_file = None

        # Get web viewer settings from config
        self.enabled = bot.config.getboolean('Web_Viewer', 'enabled', fallback=False)
        self.host = bot.config.get('Web_Viewer', 'host', fallback='127.0.0.1')
        self.port = bot.config.getint('Web_Viewer', 'port', fallback=8080)  # Web viewer uses 8080
        self.debug = bot.config.getboolean('Web_Viewer', 'debug', fallback=False)
        self.auto_start = bot.config.getboolean('Web_Viewer', 'auto_start', fallback=False)
        self.viewer_stop_grace_timeout_sec = self._get_float_config(
            "viewer_stop_grace_timeout_sec",
            self.VIEWER_STOP_GRACE_TIMEOUT_SEC,
        )
        self.viewer_stop_force_timeout_sec = self._get_float_config(
            "viewer_stop_force_timeout_sec",
            self.VIEWER_STOP_FORCE_TIMEOUT_SEC,
        )
        self.port_cleanup_lsof_timeout_sec = self._get_float_config(
            "port_cleanup_lsof_timeout_sec",
            self.PORT_CLEANUP_LSOF_TIMEOUT_SEC,
        )
        self.port_cleanup_kill_timeout_sec = self._get_float_config(
            "port_cleanup_kill_timeout_sec",
            self.PORT_CLEANUP_KILL_TIMEOUT_SEC,
        )

        # Validate configuration for security
        self._validate_config()

        # Process monitoring
        self.restart_count = 0
        self.max_restarts = 5
        self.last_restart = 0
        # True while stop_viewer() is tearing down; blocks accidental restart during bot shutdown
        self.shutting_down = False

        # Initialize bot integration for compatibility
        self.bot_integration = BotIntegration(bot)

        if self.enabled and self.auto_start:
            self.start_viewer()

    def _get_float_config(self, key: str, fallback: float) -> float:
        """Read float timeout from [Web_Viewer] with positive-value guard."""
        try:
            value = self.bot.config.getfloat("Web_Viewer", key, fallback=fallback)
            return value if value > 0 else fallback
        except (ValueError, TypeError, OSError):
            return fallback

    def _validate_config(self):
        """Validate web viewer configuration for security"""
        # Validate host against whitelist
        if self.host not in self.ALLOWED_HOSTS:
            raise ValueError(
                f"Invalid host configuration: {self.host}. "
                f"Allowed hosts: {', '.join(self.ALLOWED_HOSTS)}"
            )

        # Validate port range (avoid privileged ports)
        if not isinstance(self.port, int) or not (1024 <= self.port <= 65535):
            raise ValueError(
                f"Port must be between 1024-65535 (non-privileged), got: {self.port}"
            )

        # Insecure but allowed: binding to all interfaces without a password.
        # Only warn when the web viewer is configured to run at startup.
        if self.enabled and self.host == "0.0.0.0":
            if not normalized_web_viewer_password(self.bot.config):
                self.logger.error(
                    "Web viewer is configured with host = 0.0.0.0 and no "
                    "web_viewer_password (or password is empty/null); the UI is reachable "
                    "from the network without authentication. Set web_viewer_password or use "
                    "host = 127.0.0.1 for local-only access."
                )

    def _resolve_viewer_db_path(self) -> Path:
        """Return the database path the viewer subprocess will use."""
        config_path = getattr(self.bot, 'config_file', 'config.ini')
        config_base = Path(config_path).resolve().parent if config_path else Path(".").resolve()
        if self.bot.config.has_section('Web_Viewer') and self.bot.config.has_option('Web_Viewer', 'db_path'):
            raw = self.bot.config.get('Web_Viewer', 'db_path', fallback='').strip()
            if raw:
                return Path(resolve_path(raw, config_base))
        return Path(self.bot.db_manager.db_path)

    def _preflight_database_path(self) -> bool:
        """Validate the web viewer DB path before spawning the subprocess."""
        try:
            db_path = self._resolve_viewer_db_path()
        except Exception as e:
            self.logger.error("Web viewer database path could not be resolved: %s", e)
            return False

        parent = db_path.parent if db_path.parent != Path("") else Path(".")
        parent_exists = parent.exists()
        parent_is_dir = parent.is_dir() if parent_exists else False
        parent_writable = os.access(str(parent), os.W_OK) if parent_exists else False
        file_exists = db_path.exists()
        file_readable = os.access(str(db_path), os.R_OK) if file_exists else False
        file_writable = os.access(str(db_path), os.W_OK) if file_exists else False

        self.logger.info("Web viewer database path: %s", db_path)

        if not parent_exists:
            self.logger.error(
                "Web viewer database parent directory does not exist: %s. "
                "Create it, fix [Web_Viewer] db_path, or remove [Web_Viewer] db_path to use [Bot] db_path.",
                parent,
            )
            return False
        if not parent_is_dir:
            self.logger.error(
                "Web viewer database parent path is not a directory: %s. "
                "Fix [Web_Viewer] db_path or remove it to use [Bot] db_path.",
                parent,
            )
            return False
        if not parent_writable:
            self.logger.error(
                "Web viewer database parent directory is not writable: %s. "
                "Fix permissions or choose a writable db_path.",
                parent,
            )
            return False
        if file_exists and (not file_readable or not file_writable):
            self.logger.error(
                "Web viewer database file is not readable and writable: %s "
                "(readable=%s, writable=%s). Fix file permissions.",
                db_path,
                file_readable,
                file_writable,
            )
            return False

        if self.bot.config.has_section('Web_Viewer') and self.bot.config.get('Web_Viewer', 'db_path', fallback='').strip():
            bot_db_path = Path(self.bot.db_manager.db_path)
            if db_path.resolve() != bot_db_path.resolve():
                self.logger.warning(
                    "Web viewer database path differs from bot database: viewer=%s, bot=%s. "
                    "Remove [Web_Viewer] db_path unless you intentionally use a separate viewer database.",
                    db_path,
                    bot_db_path,
                )
        return True

    def start_viewer(self):
        """Start the web viewer in a separate thread"""
        if self.running:
            self.logger.warning("Web viewer is already running")
            return

        if not self._preflight_database_path():
            self.running = False
            return

        # Intentional (re)start after stop_viewer / restart_viewer — allow monitor thread to run
        self.shutting_down = False

        try:
            # Start the web viewer
            self.viewer_thread = threading.Thread(target=self._run_viewer, daemon=True)
            self.viewer_thread.start()
            self.running = True
            self.logger.info(f"Web viewer started on http://{self.host}:{self.port}")

        except Exception as e:
            self.logger.error(f"Failed to start web viewer: {e}")

    def stop_viewer(self):
        """Stop the web viewer"""
        if not self.running and not self.viewer_process:
            return

        try:
            self.shutting_down = True
            self.running = False

            if self.viewer_process and self.viewer_process.poll() is None:
                self.logger.info("Stopping web viewer...")
                try:
                    # First try graceful termination
                    self.viewer_process.terminate()
                    self.viewer_process.wait(timeout=self.viewer_stop_grace_timeout_sec)
                    self.logger.info("Web viewer stopped gracefully")
                except subprocess.TimeoutExpired:
                    self.logger.warning("Web viewer did not stop gracefully, forcing termination")
                    try:
                        self.viewer_process.kill()
                        self.viewer_process.wait(timeout=self.viewer_stop_force_timeout_sec)
                    except subprocess.TimeoutExpired:
                        self.logger.error("Failed to kill web viewer process")
                    except Exception as e:
                        self.logger.warning(f"Error during forced termination: {e}")
                except Exception as e:
                    self.logger.warning(f"Error during web viewer shutdown: {e}")
                finally:
                    self.viewer_process = None

            # Close log file handles
            if self._viewer_stdout_file:
                try:
                    self._viewer_stdout_file.close()
                except Exception as e:
                    self.logger.debug(f"Error closing stdout file: {e}")
                finally:
                    self._viewer_stdout_file = None

            if self._viewer_stderr_file:
                try:
                    self._viewer_stderr_file.close()
                except Exception as e:
                    self.logger.debug(f"Error closing stderr file: {e}")
                finally:
                    self._viewer_stderr_file = None

            if not self.viewer_process:
                self.logger.info("Web viewer already stopped")

            # Additional cleanup: kill any remaining processes on the port
            try:
                result = subprocess.run(['lsof', '-ti', f':{self.port}'],
                                      capture_output=True, text=True, timeout=self.port_cleanup_lsof_timeout_sec)
                if result.returncode == 0 and result.stdout.strip():
                    pids = result.stdout.strip().split('\n')
                    for pid in pids:
                        pid = pid.strip()
                        if not pid:
                            continue

                        # Validate PID is numeric only (prevent injection)
                        if not re.match(r'^\d+$', pid):
                            self.logger.warning(f"Invalid PID format: {pid}, skipping")
                            continue

                        try:
                            pid_int = int(pid)
                            # Safety check: never kill system PIDs
                            if pid_int < 2:
                                self.logger.warning(f"Refusing to kill system PID: {pid}")
                                continue

                            subprocess.run(['kill', '-9', str(pid_int)], timeout=self.port_cleanup_kill_timeout_sec)
                            self.logger.info(f"Killed remaining process {pid} on port {self.port}")
                        except (ValueError, subprocess.TimeoutExpired) as e:
                            self.logger.warning(f"Failed to kill process {pid}: {e}")
            except Exception as e:
                self.logger.debug(f"Port cleanup check failed: {e}")

        except Exception as e:
            self.logger.error(f"Error stopping web viewer: {e}")

    def _viewer_log_dir(self, config_path: str) -> Path | None:
        """Return directory for viewer stdout/stderr files, or None for journal-only.

        Mirrors the main bot: empty [Logging] log_file means console/journal only
        (no files under the install tree). When set, use that file's parent
        (e.g. /var/log/meshcore-bot).
        """
        log_file = ''
        try:
            if self.bot.config.has_section('Logging'):
                log_file = self.bot.config.get('Logging', 'log_file', fallback='').strip()
        except Exception:
            log_file = ''
        if not log_file:
            return None
        config_base = Path(config_path).parent.resolve()
        return Path(resolve_path(log_file, config_base)).parent

    def _run_viewer(self):
        """Run the web viewer in a separate process"""
        stdout_file = None
        stderr_file = None
        stdout_path = None
        stderr_path = None

        try:
            # Get the path to the web viewer script
            viewer_script = Path(__file__).parent / "app.py"
            # Use same config as bot so viewer finds db_path, Greeter_Command, etc.
            config_path = getattr(self.bot, 'config_file', 'config.ini')
            config_path = str(Path(config_path).resolve()) if config_path else 'config.ini'

            # Build command
            cmd = [
                sys.executable,
                str(viewer_script),
                "--config", config_path,
                "--host", self.host,
                "--port", str(self.port)
            ]

            if self.debug:
                cmd.append("--debug")

            # Avoid PIPE deadlock: either inherit stdio (journald under systemd)
            # or redirect to files beside the configured log_file.
            log_dir = self._viewer_log_dir(config_path)
            if log_dir is None:
                # Inherit parent stdout/stderr so systemd/journald captures them.
                # Do not create ./logs under the (often read-only) install tree.
                popen_stdout = None
                popen_stderr = None
            else:
                log_dir.mkdir(parents=True, exist_ok=True)
                stdout_path = log_dir / 'web_viewer_stdout.log'
                stderr_path = log_dir / 'web_viewer_stderr.log'
                # Open in write mode to prevent buffer blocking. Using 'w'
                # (overwrite) instead of 'a' (append) since:
                # - The web viewer already has proper logging to web_viewer.log
                # - stdout/stderr are mainly for immediate debugging
                # - Prevents unbounded log file growth
                stdout_file = open(stdout_path, 'w')
                stderr_file = open(stderr_path, 'w')
                self._viewer_stdout_file = stdout_file
                self._viewer_stderr_file = stderr_file
                popen_stdout = stdout_file
                popen_stderr = stderr_file

            # Start the viewer process
            self.viewer_process = subprocess.Popen(
                cmd,
                stdout=popen_stdout,
                stderr=popen_stderr,
                text=True
            )

            # Give it a moment to start up
            time.sleep(2)

            # Check if it started successfully
            if self.viewer_process and self.viewer_process.poll() is not None:
                # Process failed immediately - read from log files for error reporting
                stderr = ""
                stdout = ""
                if stderr_file is not None and stderr_path is not None:
                    with suppress(Exception):
                        stderr_file.flush()
                        stderr_file.close()
                    try:
                        with open(stderr_path) as f:
                            stderr = ''.join(f.readlines()[-20:])
                    except Exception:
                        stderr = "Could not read stderr log"
                if stdout_file is not None and stdout_path is not None:
                    with suppress(Exception):
                        stdout_file.flush()
                        stdout_file.close()
                    try:
                        with open(stdout_path) as f:
                            stdout = ''.join(f.readlines()[-20:])
                    except Exception:
                        stdout = "Could not read stdout log"

                self.logger.error(f"Web viewer failed to start. Return code: {self.viewer_process.returncode}")
                if stderr and stderr.strip():
                    self.logger.error(f"Web viewer startup error: {stderr}")
                if stdout and stdout.strip():
                    self.logger.error(f"Web viewer startup output: {stdout}")
                elif log_dir is None:
                    self.logger.error(
                        "Web viewer startup output is in the service journal "
                        "(empty [Logging] log_file; stdio inherited)"
                    )

                self.viewer_process = None
                self._viewer_stdout_file = None
                self._viewer_stderr_file = None
                return

            # Web viewer is ready
            self.logger.info("Web viewer integration ready for data streaming")

            # Monitor the process
            while self.running and self.viewer_process and self.viewer_process.poll() is None:
                time.sleep(1)

            # Process exited unexpectedly — try to restart if we haven't been stopped
            if self.running and self.viewer_process and self.viewer_process.poll() is not None:
                if (
                    self.shutting_down
                    or self.bot._shutdown_event.is_set()
                    or not getattr(self.bot, "connected", True)
                ):
                    self.logger.debug(
                        "Web viewer process exited (code %s) during bot shutdown; not restarting",
                        self.viewer_process.returncode,
                    )
                    return
                self.logger.warning(
                    "Web viewer process exited unexpectedly (code %s) — attempting restart",
                    self.viewer_process.returncode,
                )
                self.restart_viewer()
                return

            # Process exited - read from log files for error reporting if needed
            if self.viewer_process and self.viewer_process.returncode != 0:
                stderr = ""
                if stderr_file is not None and stderr_path is not None:
                    with suppress(Exception):
                        stderr_file.flush()
                        stderr_file.close()
                    try:
                        with open(stderr_path) as f:
                            stderr = ''.join(f.readlines()[-20:])
                    except Exception:
                        stderr = "Could not read stderr log"
                if stdout_file is not None:
                    with suppress(Exception):
                        stdout_file.close()

                self.logger.error(f"Web viewer process exited with code {self.viewer_process.returncode}")
                if stderr and stderr.strip():
                    self.logger.error(f"Web viewer stderr: {stderr}")
                elif log_dir is None:
                    self.logger.error(
                        "Web viewer stderr is in the service journal "
                        "(empty [Logging] log_file; stdio inherited)"
                    )

                self._viewer_stdout_file = None
                self._viewer_stderr_file = None
            elif self.viewer_process and self.viewer_process.returncode == 0:
                self.logger.info("Web viewer process exited normally")

        except Exception as e:
            self.logger.error(f"Error running web viewer: {e}")
            # Close file handles on error
            if stdout_file:
                with suppress(Exception):
                    stdout_file.close()
            if stderr_file:
                with suppress(Exception):
                    stderr_file.close()
            self._viewer_stdout_file = None
            self._viewer_stderr_file = None
        finally:
            self.running = False

    def get_status(self):
        """Get the current status of the web viewer"""
        return {
            'enabled': self.enabled,
            'running': self.running,
            'host': self.host,
            'port': self.port,
            'debug': self.debug,
            'auto_start': self.auto_start,
            'url': f"http://{self.host}:{self.port}" if self.running else None
        }

    def restart_viewer(self):
        """Restart the web viewer with rate limiting"""
        if self.shutting_down:
            return
        if getattr(self.bot, "_shutdown_event", None) is not None and self.bot._shutdown_event.is_set():
            return
        if not getattr(self.bot, "connected", True):
            return

        current_time = time.time()

        # Rate limit restarts to prevent restart loops
        if current_time - self.last_restart < 30:  # 30 seconds between restarts
            self.logger.warning("Restart rate limited - too soon since last restart")
            return

        if self.restart_count >= self.max_restarts:
            self.logger.error(f"Maximum restart limit reached ({self.max_restarts}). Web viewer disabled.")
            self.enabled = False
            return

        self.restart_count += 1
        self.last_restart = current_time

        self.logger.info(f"Restarting web viewer (attempt {self.restart_count}/{self.max_restarts})...")
        self.stop_viewer()
        time.sleep(3)  # Give it more time to stop

        self.start_viewer()

    def is_viewer_healthy(self):
        """Check if the web viewer process is healthy"""
        if not self.viewer_process:
            return False

        # Check if process is still running
        return self.viewer_process.poll() is None