diff --git a/config.ini.example b/config.ini.example index 147712a..246e8cd 100644 --- a/config.ini.example +++ b/config.ini.example @@ -1034,9 +1034,9 @@ debug = false # false: Start web viewer manually (recommended) auto_start = false -# Database path for web viewer (same as [Bot] db_path so viewer can show repeater/graph data and packet stream) -# Default: meshcore_bot.db -db_path = meshcore_bot.db +# Optional: database path for web viewer. If unset, the viewer uses [Bot] db_path (recommended). +# Only set this if you use a separate database for the viewer; you will see a startup warning. +# See docs/WEB_VIEWER.md for migrating from a separate database. # Additional hashtag channels to decode in the packet stream # The web viewer can decrypt GroupText messages from hashtag channels diff --git a/modules/core.py b/modules/core.py index e1952b9..7d62ba5 100644 --- a/modules/core.py +++ b/modules/core.py @@ -87,6 +87,19 @@ class MeshCoreBot: except (OSError, sqlite3.Error, AttributeError) as e: self.logger.warning(f"Could not store start time in database: {e}") + # Notify if Web_Viewer uses a different database (split-DB setup) + if self.config.has_section('Web_Viewer') and self.config.has_option('Web_Viewer', 'db_path'): + wv_raw = self.config.get('Web_Viewer', 'db_path').strip() + if wv_raw: + wv_path = Path(resolve_path(wv_raw, self.bot_root)).resolve() + bot_path = Path(self.db_manager.db_path).resolve() + if wv_path != bot_path: + self.logger.warning( + "Web viewer database path differs from bot database: viewer=%s, bot=%s. " + "For shared repeater/graph and packet stream data, set [Web_Viewer] db_path to the same as [Bot] db_path or remove it to use the bot database. See docs/WEB_VIEWER.md (migrating from a separate database).", + wv_path, bot_path + ) + # Initialize web viewer integration (after database manager) try: self.web_viewer_integration = WebViewerIntegration(self) diff --git a/modules/transmission_tracker.py b/modules/transmission_tracker.py index 6c27331..0dcede5 100644 --- a/modules/transmission_tracker.py +++ b/modules/transmission_tracker.py @@ -189,10 +189,14 @@ class TransmissionTracker: if not record.command_id: return - # Get database path - db_path = self.bot.config.get('Web_Viewer', 'db_path', fallback='meshcore_bot.db') + # Get database path (use [Bot] db_path when [Web_Viewer] db_path is unset) base_dir = self.bot.bot_root if hasattr(self.bot, 'bot_root') else '.' - db_path = resolve_path(db_path, base_dir) + if (self.bot.config.has_section('Web_Viewer') and self.bot.config.has_option('Web_Viewer', 'db_path') + and self.bot.config.get('Web_Viewer', 'db_path', fallback='').strip()): + db_path = resolve_path(self.bot.config.get('Web_Viewer', 'db_path').strip(), base_dir) + else: + from pathlib import Path + db_path = Path(self.bot.db_manager.db_path).resolve() with sqlite3.connect(str(db_path), timeout=30.0) as conn: cursor = conn.cursor() diff --git a/modules/web_viewer/app.py b/modules/web_viewer/app.py index 03ba7d5..4cf5870 100644 --- a/modules/web_viewer/app.py +++ b/modules/web_viewer/app.py @@ -76,7 +76,14 @@ class BotDataViewer: # Load configuration self.config = self._load_config(config_path) - self.db_path = self.config.get('Web_Viewer', 'db_path', fallback=db_path) + # Use [Bot] db_path when [Web_Viewer] db_path is unset + bot_db = self.config.get('Bot', 'db_path', fallback='meshcore_bot.db') + if (self.config.has_section('Web_Viewer') and self.config.has_option('Web_Viewer', 'db_path') + and self.config.get('Web_Viewer', 'db_path', fallback='').strip()): + use_db = self.config.get('Web_Viewer', 'db_path').strip() + else: + use_db = bot_db + self.db_path = str(resolve_path(use_db, self.bot_root)) # Setup template context processor for global template variables self._setup_template_context() @@ -202,14 +209,7 @@ class BotDataViewer: """Initialize the packet_stream table in the web viewer database (same as [Bot] db_path by default).""" conn = None try: - # Get database path from config - db_path = self.config.get('Web_Viewer', 'db_path', fallback='meshcore_bot.db') - - # Resolve database path (relative paths resolved from bot root, absolute paths used as-is) - db_path = resolve_path(db_path, self.bot_root) - - # Connect to database and create table if it doesn't exist - with sqlite3.connect(db_path, timeout=30) as conn: + with sqlite3.connect(self.db_path, timeout=30) as conn: cursor = conn.cursor() # Create packet_stream table with schema matching the INSERT statements @@ -236,7 +236,7 @@ class BotDataViewer: conn.commit() - self.logger.info(f"Initialized packet_stream table in {db_path}") + self.logger.info(f"Initialized packet_stream table in {self.db_path}") except Exception as e: self.logger.error(f"Failed to initialize packet_stream table: {e}") @@ -1491,13 +1491,7 @@ class BotDataViewer: # Get commands from last 60 minutes cutoff_time = time.time() - (60 * 60) # 60 minutes ago - # Get database path - db_path = self.config.get('Web_Viewer', 'db_path', fallback='meshcore_bot.db') - - # Resolve database path (relative paths resolved from bot root, absolute paths used as-is) - db_path = resolve_path(db_path, self.bot_root) - - with sqlite3.connect(db_path, timeout=30) as conn: + with sqlite3.connect(self.db_path, timeout=30) as conn: cursor = conn.cursor() cursor.execute(''' @@ -2604,7 +2598,6 @@ class BotDataViewer: last_timestamp = 0 consecutive_errors = 0 max_consecutive_errors = 10 - db_path = None while True: try: @@ -2612,39 +2605,33 @@ class BotDataViewer: import sqlite3 import json - # Get database path (re-resolve on each iteration in case config changed) - db_path = self.config.get('Web_Viewer', 'db_path', fallback='meshcore_bot.db') - - # Resolve database path (relative paths resolved from bot root, absolute paths used as-is) - db_path = resolve_path(db_path, self.bot_root) - # Check if database file exists and is accessible - db_file = Path(db_path) + db_file = Path(self.db_path) if not db_file.exists(): consecutive_errors += 1 if consecutive_errors == 1 or consecutive_errors % 10 == 0: - self.logger.warning(f"Database file does not exist: {db_path}") + self.logger.warning(f"Database file does not exist: {self.db_path}") time.sleep(5) continue - if not os.access(db_path, os.R_OK): + if not os.access(self.db_path, os.R_OK): consecutive_errors += 1 if consecutive_errors == 1 or consecutive_errors % 10 == 0: - self.logger.warning(f"Database file is not readable: {db_path}") + self.logger.warning(f"Database file is not readable: {self.db_path}") time.sleep(5) continue # Connect to database with timeout to prevent hanging # Use check_same_thread=False for thread safety, but be careful try: - conn = sqlite3.connect(db_path, timeout=30, check_same_thread=False) + conn = sqlite3.connect(self.db_path, timeout=30, check_same_thread=False) conn.row_factory = sqlite3.Row except sqlite3.OperationalError as conn_error: error_msg = str(conn_error) if "locked" in error_msg.lower() or "database is locked" in error_msg.lower(): consecutive_errors += 1 if consecutive_errors == 1 or consecutive_errors % 10 == 0: - self.logger.warning(f"Database is locked, waiting: {db_path}") + self.logger.warning(f"Database is locked, waiting: {self.db_path}") time.sleep(2) continue else: @@ -2699,20 +2686,17 @@ class BotDataViewer: # Provide more diagnostic information on first error or periodic errors if consecutive_errors == 1 or consecutive_errors % 10 == 0: - if db_path: - db_file = Path(db_path) - exists = db_file.exists() - readable = os.access(db_path, os.R_OK) if exists else False - writable = os.access(db_path, os.W_OK) if exists else False - self.logger.error( - f"Database polling error (attempt {consecutive_errors}): {error_msg}\n" - f" Path: {db_path}\n" - f" Exists: {exists}\n" - f" Readable: {readable}\n" - f" Writable: {writable}" - ) - else: - self.logger.error(f"Database polling error (attempt {consecutive_errors}): {error_msg}") + db_file = Path(self.db_path) + exists = db_file.exists() + readable = os.access(self.db_path, os.R_OK) if exists else False + writable = os.access(self.db_path, os.W_OK) if exists else False + self.logger.error( + f"Database polling error (attempt {consecutive_errors}): {error_msg}\n" + f" Path: {self.db_path}\n" + f" Exists: {exists}\n" + f" Readable: {readable}\n" + f" Writable: {writable}" + ) # Log at appropriate level based on error frequency if consecutive_errors >= max_consecutive_errors: @@ -2798,14 +2782,8 @@ class BotDataViewer: cutoff_time = time.time() - (days_to_keep * 24 * 60 * 60) - # Get database path - db_path = self.config.get('Web_Viewer', 'db_path', fallback='meshcore_bot.db') - - # Resolve database path (relative paths resolved from bot root, absolute paths used as-is) - db_path = resolve_path(db_path, self.bot_root) - # Use shorter timeout and isolation_level for better concurrency - conn = sqlite3.connect(db_path, timeout=10, isolation_level='DEFERRED') + conn = sqlite3.connect(self.db_path, timeout=10, isolation_level='DEFERRED') cursor = conn.cursor() # Use WAL mode for better concurrent access (if not already set) @@ -3310,11 +3288,8 @@ class BotDataViewer: # Get database file size import os - db_path = self.config.get('Web_Viewer', 'db_path', fallback='meshcore_bot.db') - # Resolve database path (relative paths resolved from bot root, absolute paths used as-is) - db_path = resolve_path(db_path, self.bot_root) try: - db_size_bytes = os.path.getsize(db_path) + db_size_bytes = os.path.getsize(self.db_path) if db_size_bytes < 1024: db_size = f"{db_size_bytes} bytes" elif db_size_bytes < 1024 * 1024: @@ -3368,15 +3343,12 @@ class BotDataViewer: # Get initial database size import os - db_path = self.config.get('Web_Viewer', 'db_path', fallback='meshcore_bot.db') - # Resolve database path (relative paths resolved from bot root, absolute paths used as-is) - db_path = resolve_path(db_path, self.bot_root) - initial_size = os.path.getsize(db_path) + initial_size = os.path.getsize(self.db_path) # Perform VACUUM to reclaim unused space self.logger.info("Starting database VACUUM...") cursor.execute("VACUUM") - vacuum_size = os.path.getsize(db_path) + vacuum_size = os.path.getsize(self.db_path) vacuum_saved = initial_size - vacuum_size # Perform ANALYZE to update table statistics @@ -3399,7 +3371,7 @@ class BotDataViewer: self.logger.debug(f"Could not reindex table {table}: {e}") # Get final database size - final_size = os.path.getsize(db_path) + final_size = os.path.getsize(self.db_path) total_saved = initial_size - final_size # Format size information diff --git a/modules/web_viewer/integration.py b/modules/web_viewer/integration.py index 685eafa..49dc63a 100644 --- a/modules/web_viewer/integration.py +++ b/modules/web_viewer/integration.py @@ -81,17 +81,21 @@ class BotIntegration: self.circuit_breaker_open = False self.circuit_breaker_failures = 0 + def _get_web_viewer_db_path(self): + """Return resolved database path for web viewer. Uses [Bot] db_path when [Web_Viewer] db_path is unset.""" + base_dir = self.bot.bot_root if hasattr(self.bot, 'bot_root') else '.' + if self.bot.config.has_section('Web_Viewer') and self.bot.config.has_option('Web_Viewer', 'db_path'): + raw = self.bot.config.get('Web_Viewer', 'db_path', fallback='').strip() + if raw: + return resolve_path(raw, base_dir) + return Path(self.bot.db_manager.db_path).resolve() + def _init_packet_stream_table(self): """Initialize the packet_stream table in the web viewer database (same as [Bot] db_path by default).""" try: import sqlite3 - # Get database path from config - db_path = self.bot.config.get('Web_Viewer', 'db_path', fallback='meshcore_bot.db') - - # Resolve database path (relative paths resolved from bot root, absolute paths used as-is) - base_dir = self.bot.bot_root if hasattr(self.bot, 'bot_root') else '.' - db_path = resolve_path(db_path, base_dir) + db_path = self._get_web_viewer_db_path() # Connect to database and create table if it doesn't exist conn = sqlite3.connect(str(db_path), timeout=30.0) @@ -225,10 +229,7 @@ class BotIntegration: serializable_data = self._make_json_serializable(command_data) # Store in database for web viewer to read - db_path = self.bot.config.get('Web_Viewer', 'db_path', fallback='meshcore_bot.db') - # Resolve database path (relative paths resolved from bot root, absolute paths used as-is) - base_dir = self.bot.bot_root if hasattr(self.bot, 'bot_root') else '.' - db_path = resolve_path(db_path, base_dir) + db_path = self._get_web_viewer_db_path() conn = sqlite3.connect(str(db_path), timeout=30.0) cursor = conn.cursor() @@ -255,10 +256,7 @@ class BotIntegration: serializable_data = self._make_json_serializable(routing_data) # Store in database for web viewer to read - db_path = self.bot.config.get('Web_Viewer', 'db_path', fallback='meshcore_bot.db') - # Resolve database path (relative paths resolved from bot root, absolute paths used as-is) - base_dir = self.bot.bot_root if hasattr(self.bot, 'bot_root') else '.' - db_path = resolve_path(db_path, base_dir) + db_path = self._get_web_viewer_db_path() conn = sqlite3.connect(str(db_path), timeout=30.0) cursor = conn.cursor() @@ -282,10 +280,7 @@ class BotIntegration: cutoff_time = time.time() - (days_to_keep * 24 * 60 * 60) - db_path = self.bot.config.get('Web_Viewer', 'db_path', fallback='meshcore_bot.db') - # Resolve database path (relative paths resolved from bot root, absolute paths used as-is) - base_dir = self.bot.bot_root if hasattr(self.bot, 'bot_root') else '.' - db_path = resolve_path(db_path, base_dir) + db_path = self._get_web_viewer_db_path() conn = sqlite3.connect(str(db_path), timeout=30.0) cursor = conn.cursor()