From e1f3a7ab0945897ada351efe2869edd98650ec6a Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 7 Feb 2026 21:31:07 -0800 Subject: [PATCH] chore: Update database paths and scoring parameters in configuration - Changed the default database path in config.ini.example and related files from 'bot_data.db' to 'meshcore_bot.db' for consistency across modules. - Adjusted final hop proximity scoring parameters in config.ini.example to reflect updated distance normalization and thresholds, enhancing scoring accuracy for LoRa networks. - Updated documentation to clarify the shared database usage between the bot and web viewer. --- config.ini.example | 20 +++---- docker-setup.sh | 10 ++-- docs/DOCKER.md | 2 +- docs/WEB_VIEWER.md | 27 +++++++++- migrate_webviewer_db.py | 89 +++++++++++++++++++++++++++++++ modules/repeater_manager.py | 4 +- modules/transmission_tracker.py | 2 +- modules/web_viewer/app.py | 14 ++--- modules/web_viewer/integration.py | 12 ++--- 9 files changed, 146 insertions(+), 34 deletions(-) create mode 100644 migrate_webviewer_db.py diff --git a/config.ini.example b/config.ini.example index 9c9c99d..147712a 100644 --- a/config.ini.example +++ b/config.ini.example @@ -795,18 +795,18 @@ graph_final_hop_proximity_weight = 0.25 # Set to 0 to disable distance limiting graph_final_hop_max_distance = 0 -# Distance normalization for final hop proximity scoring (km, default: 500) +# Distance normalization for final hop proximity scoring (km, default: 200) # Closer repeaters get higher proximity scores. Lower values = more aggressive scoring -# 500km means repeaters within 500km get full scoring range, beyond that scores decrease -graph_final_hop_proximity_normalization_km = 500 +# 200km (long LoRa range) means repeaters within 200km get full scoring range, beyond that scores decrease +graph_final_hop_proximity_normalization_km = 200 -# Very close distance threshold for boosted proximity weight (km, default: 50) +# Very close distance threshold for boosted proximity weight (km, default: 10) # Repeaters within this distance get 2x proximity weight (up to 0.6 max) -graph_final_hop_very_close_threshold_km = 50 +graph_final_hop_very_close_threshold_km = 10 -# Close distance threshold for boosted proximity weight (km, default: 100) -# Repeaters within this distance get 1.5x proximity weight (up to 0.5 max) -graph_final_hop_close_threshold_km = 100 +# Close distance threshold for boosted proximity weight (km, default: 30) +# Repeaters within this distance get 1.5x proximity weight (up to 0.5 max). Typical LoRa range. +graph_final_hop_close_threshold_km = 30 # Maximum proximity weight for very close repeaters (0.0-1.0, default: 0.6) # When a repeater is within very_close_threshold_km, proximity weight is boosted up to this value @@ -1034,8 +1034,8 @@ debug = false # false: Start web viewer manually (recommended) auto_start = false -# Database path for web viewer data -# Default: bot_data.db +# Database path for web viewer (same as [Bot] db_path so viewer can show repeater/graph data and packet stream) +# Default: meshcore_bot.db db_path = meshcore_bot.db # Additional hashtag channels to decode in the packet stream diff --git a/docker-setup.sh b/docker-setup.sh index 5015f66..369e1e4 100755 --- a/docker-setup.sh +++ b/docker-setup.sh @@ -74,7 +74,7 @@ echo " ✓ Updated [Bot] db_path" # Ensure we use absolute path to avoid resolution issues update_config "Logging" "log_file" "/data/logs/meshcore_bot.log" # Verify the update worked -VERIFIED_LOG=$(grep "^log_file[[:space:]]*=" "$CONFIG_FILE" 2>/dev/null | sed 's/^log_file[[:space:]]*=[[:space:]]*//' | tr -d ' ' || echo "") +VERIFIED_LOG=$(grep "^log_file[[:space:]]*=" "$CONFIG_FILE" 2>/dev/null | sed 's/^log_file[[:space:]]*=[[:space:]]*//;s/^[[:space:]]*//;s/[[:space:]]*$//' || echo "") if [[ "$VERIFIED_LOG" == "/data/logs/meshcore_bot.log" ]]; then echo " ✓ Updated [Logging] log_file to /data/logs/meshcore_bot.log" ((UPDATED_COUNT++)) @@ -93,7 +93,7 @@ fi # Update Web_Viewer database path (if section exists) if grep -q "^\[Web_Viewer\]" "$CONFIG_FILE"; then - update_config "Web_Viewer" "db_path" "/data/databases/bot_data.db" + update_config "Web_Viewer" "db_path" "/data/databases/meshcore_bot.db" echo " ✓ Updated [Web_Viewer] db_path" ((UPDATED_COUNT++)) fi @@ -101,7 +101,7 @@ fi # Update PacketCapture paths (if section exists) if grep -q "^\[PacketCapture\]" "$CONFIG_FILE"; then # Only update output_file if it's set to a relative path - CURRENT_OUTPUT=$(grep "^output_file[[:space:]]*=" "$CONFIG_FILE" 2>/dev/null | sed 's/^output_file[[:space:]]*=[[:space:]]*//' | tr -d ' ' || echo "") + CURRENT_OUTPUT=$(grep "^output_file[[:space:]]*=" "$CONFIG_FILE" 2>/dev/null | sed 's/^output_file[[:space:]]*=[[:space:]]*//;s/^[[:space:]]*//;s/[[:space:]]*$//' || echo "") if [ -n "$CURRENT_OUTPUT" ] && [[ ! "$CURRENT_OUTPUT" == /* ]]; then # Relative path - update to logs directory update_config "PacketCapture" "output_file" "/data/logs/packets.jsonl" @@ -110,7 +110,7 @@ if grep -q "^\[PacketCapture\]" "$CONFIG_FILE"; then fi # Update private_key_path if it's a relative path - CURRENT_KEY=$(grep "^private_key_path[[:space:]]*=" "$CONFIG_FILE" 2>/dev/null | sed 's/^private_key_path[[:space:]]*=[[:space:]]*//' | tr -d ' ' || echo "") + CURRENT_KEY=$(grep "^private_key_path[[:space:]]*=" "$CONFIG_FILE" 2>/dev/null | sed 's/^private_key_path[[:space:]]*=[[:space:]]*//;s/^[[:space:]]*//;s/[[:space:]]*$//' || echo "") if [ -n "$CURRENT_KEY" ] && [[ ! "$CURRENT_KEY" == /* ]]; then # Relative path - update to config directory (read-only is fine for keys) update_config "PacketCapture" "private_key_path" "/data/config/private_key" @@ -121,7 +121,7 @@ fi # Update MapUploader private_key_path (if section exists) if grep -q "^\[MapUploader\]" "$CONFIG_FILE"; then - CURRENT_KEY=$(grep "^private_key_path[[:space:]]*=" "$CONFIG_FILE" 2>/dev/null | sed 's/^private_key_path[[:space:]]*=[[:space:]]*//' | tr -d ' ' || echo "") + CURRENT_KEY=$(grep "^private_key_path[[:space:]]*=" "$CONFIG_FILE" 2>/dev/null | sed 's/^private_key_path[[:space:]]*=[[:space:]]*//;s/^[[:space:]]*//;s/[[:space:]]*$//' || echo "") if [ -n "$CURRENT_KEY" ] && [[ ! "$CURRENT_KEY" == /* ]]; then # Relative path - update to config directory update_config "MapUploader" "private_key_path" "/data/config/private_key" diff --git a/docs/DOCKER.md b/docs/DOCKER.md index ff0e303..01e21d7 100644 --- a/docs/DOCKER.md +++ b/docs/DOCKER.md @@ -41,7 +41,7 @@ This guide explains how to deploy meshcore-bot using Docker and Docker Compose. db_path = /data/databases/meshcore_bot.db [Web_Viewer] - db_path = /data/databases/bot_data.db + db_path = /data/databases/meshcore_bot.db [Logging] log_file = /data/logs/meshcore_bot.log diff --git a/docs/WEB_VIEWER.md b/docs/WEB_VIEWER.md index 3c85ba3..04996d5 100644 --- a/docs/WEB_VIEWER.md +++ b/docs/WEB_VIEWER.md @@ -118,9 +118,32 @@ curl http://localhost:5000/api/stats ## Database Requirements -The viewer requires access to the main database: -- `meshcore_bot.db` - Main bot database (contains all data including contacts, tracking, cache, and stats) +The viewer uses the same database as the bot by default (`[Bot] db_path`, typically `meshcore_bot.db`). That single file holds repeater contacts, mesh graph, packet stream, and other data so the viewer can show everything. +## Migrating from a separate web viewer database + +If you previously had the web viewer using a **separate** database (e.g. `[Web_Viewer] db_path = bot_data.db`), you can switch to the shared database so the viewer shows repeater/graph data and uses one file. + +1. **Stop the bot and web viewer** so neither has the databases open. + +2. **Optionally preserve packet stream history** from the old viewer DB into the main DB: + - From the project root, run: + ```bash + python3 migrate_webviewer_db.py bot_data.db meshcore_bot.db + ``` + Use your actual paths if they differ (e.g. full paths or different filenames). The script copies the `packet_stream` table from the first file into the second and skips rows that would duplicate IDs. + - If you don’t care about old packet stream data, skip this step; the viewer will create a new `packet_stream` table in the main DB. + +3. **Point the viewer at the main database** in `config.ini`: + ```ini + [Web_Viewer] + db_path = meshcore_bot.db + ``` + (Or the same value as `[Bot] db_path` if you use a different path.) + +4. **Start the bot (and viewer as usual)**. The viewer will now read and write to the same database as the bot. + +You can keep or remove the old `bot_data.db` file after verifying the viewer works with the shared DB. ## Troubleshooting diff --git a/migrate_webviewer_db.py b/migrate_webviewer_db.py new file mode 100644 index 0000000..af1df5e --- /dev/null +++ b/migrate_webviewer_db.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +""" +One-off migration: copy packet_stream from a separate web viewer database +(e.g. bot_data.db) into the main bot database (e.g. meshcore_bot.db). + +Use this when switching from split databases to the shared database so you +don't lose packet stream history. Run with bot and web viewer stopped. + +Usage: + python3 migrate_webviewer_db.py + +Example: + python3 migrate_webviewer_db.py bot_data.db meshcore_bot.db +""" + +import sqlite3 +import sys + + +def main() -> int: + if len(sys.argv) != 3: + print("Usage: python3 migrate_webviewer_db.py ", file=sys.stderr) + print("Example: python3 migrate_webviewer_db.py bot_data.db meshcore_bot.db", file=sys.stderr) + return 1 + + source_path = sys.argv[1] + target_path = sys.argv[2] + + if source_path == target_path: + print("Source and target must be different files.", file=sys.stderr) + return 1 + + try: + conn = sqlite3.connect(target_path, timeout=30.0) + conn.execute("ATTACH DATABASE ? AS src", (source_path,)) + + # Ensure packet_stream exists in target (same schema as web viewer) + conn.execute(""" + CREATE TABLE IF NOT EXISTS packet_stream ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + timestamp REAL NOT NULL, + data TEXT NOT NULL, + type TEXT NOT NULL + ) + """) + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_packet_stream_timestamp ON packet_stream(timestamp)" + ) + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_packet_stream_type ON packet_stream(type)" + ) + + # Check that source has packet_stream + cur = conn.execute( + "SELECT name FROM src.sqlite_master WHERE type='table' AND name='packet_stream'" + ) + if cur.fetchone() is None: + conn.execute("DETACH DATABASE src") + conn.close() + print(f"Source database has no packet_stream table; nothing to migrate.", file=sys.stderr) + return 0 + + # Copy rows from source; skip ids that already exist in target (INSERT OR IGNORE) + before = conn.total_changes + conn.execute( + """ + INSERT OR IGNORE INTO main.packet_stream (id, timestamp, data, type) + SELECT id, timestamp, data, type FROM src.packet_stream + """ + ) + inserted = conn.total_changes - before + conn.commit() + + conn.execute("DETACH DATABASE src") + conn.close() + + print(f"Migrated {inserted} packet_stream row(s) from {source_path} to {target_path}.") + return 0 + + except sqlite3.OperationalError as e: + print(f"Database error: {e}", file=sys.stderr) + return 1 + except OSError as e: + print(f"File error: {e}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/modules/repeater_manager.py b/modules/repeater_manager.py index 93e01ba..84da61e 100644 --- a/modules/repeater_manager.py +++ b/modules/repeater_manager.py @@ -159,7 +159,7 @@ class RepeaterManager: ''') # Create indexes for better performance - with sqlite3.connect(self.db_path) as conn: + with sqlite3.connect(self.db_path, timeout=30.0) as conn: cursor = conn.cursor() cursor.execute('CREATE INDEX IF NOT EXISTS idx_public_key ON repeater_contacts(public_key)') cursor.execute('CREATE INDEX IF NOT EXISTS idx_device_type ON repeater_contacts(device_type)') @@ -214,7 +214,7 @@ class RepeaterManager: """Handle database schema migration for existing installations""" try: # Check if the new location columns exist in repeater_contacts - with sqlite3.connect(self.db_path) as conn: + with sqlite3.connect(self.db_path, timeout=30.0) as conn: cursor = conn.cursor() cursor.execute("PRAGMA table_info(repeater_contacts)") columns = [row[1] for row in cursor.fetchall()] diff --git a/modules/transmission_tracker.py b/modules/transmission_tracker.py index f9a63d9..6c27331 100644 --- a/modules/transmission_tracker.py +++ b/modules/transmission_tracker.py @@ -190,7 +190,7 @@ class TransmissionTracker: return # Get database path - db_path = self.bot.config.get('Web_Viewer', 'db_path', fallback='bot_data.db') + db_path = self.bot.config.get('Web_Viewer', 'db_path', fallback='meshcore_bot.db') base_dir = self.bot.bot_root if hasattr(self.bot, 'bot_root') else '.' db_path = resolve_path(db_path, base_dir) diff --git a/modules/web_viewer/app.py b/modules/web_viewer/app.py index 3bcfe72..03d4974 100644 --- a/modules/web_viewer/app.py +++ b/modules/web_viewer/app.py @@ -196,11 +196,11 @@ class BotDataViewer: raise def _init_packet_stream_table(self): - """Initialize the packet_stream table in bot_data.db""" + """Initialize the packet_stream table in the web viewer database (same as [Bot] db_path by default).""" conn = None try: # Get database path from config - db_path = self.config.get('Web_Viewer', 'db_path', fallback='bot_data.db') + db_path = self.config.get('Web_Viewer', 'db_path', fallback='meshcore_bot.db') # Resolve database path (relative paths resolved from bot root, absolute paths used as-is) db_path = resolve_path(db_path, self.bot_root) @@ -1481,7 +1481,7 @@ class BotDataViewer: cutoff_time = time.time() - (60 * 60) # 60 minutes ago # Get database path - db_path = self.config.get('Web_Viewer', 'db_path', fallback='bot_data.db') + db_path = self.config.get('Web_Viewer', 'db_path', fallback='meshcore_bot.db') # Resolve database path (relative paths resolved from bot root, absolute paths used as-is) db_path = resolve_path(db_path, self.bot_root) @@ -2602,7 +2602,7 @@ class BotDataViewer: import json # Get database path (re-resolve on each iteration in case config changed) - db_path = self.config.get('Web_Viewer', 'db_path', fallback='bot_data.db') + db_path = self.config.get('Web_Viewer', 'db_path', fallback='meshcore_bot.db') # Resolve database path (relative paths resolved from bot root, absolute paths used as-is) db_path = resolve_path(db_path, self.bot_root) @@ -2788,7 +2788,7 @@ class BotDataViewer: cutoff_time = time.time() - (days_to_keep * 24 * 60 * 60) # Get database path - db_path = self.config.get('Web_Viewer', 'db_path', fallback='bot_data.db') + db_path = self.config.get('Web_Viewer', 'db_path', fallback='meshcore_bot.db') # Resolve database path (relative paths resolved from bot root, absolute paths used as-is) db_path = resolve_path(db_path, self.bot_root) @@ -3299,7 +3299,7 @@ class BotDataViewer: # Get database file size import os - db_path = self.config.get('Web_Viewer', 'db_path', fallback='bot_data.db') + db_path = self.config.get('Web_Viewer', 'db_path', fallback='meshcore_bot.db') # Resolve database path (relative paths resolved from bot root, absolute paths used as-is) db_path = resolve_path(db_path, self.bot_root) try: @@ -3357,7 +3357,7 @@ class BotDataViewer: # Get initial database size import os - db_path = self.config.get('Web_Viewer', 'db_path', fallback='bot_data.db') + db_path = self.config.get('Web_Viewer', 'db_path', fallback='meshcore_bot.db') # Resolve database path (relative paths resolved from bot root, absolute paths used as-is) db_path = resolve_path(db_path, self.bot_root) initial_size = os.path.getsize(db_path) diff --git a/modules/web_viewer/integration.py b/modules/web_viewer/integration.py index dd6e2f1..f6e5855 100644 --- a/modules/web_viewer/integration.py +++ b/modules/web_viewer/integration.py @@ -82,12 +82,12 @@ class BotIntegration: self.circuit_breaker_failures = 0 def _init_packet_stream_table(self): - """Initialize the packet_stream table in bot_data.db""" + """Initialize the packet_stream table in the web viewer database (same as [Bot] db_path by default).""" try: import sqlite3 # Get database path from config - db_path = self.bot.config.get('Web_Viewer', 'db_path', fallback='bot_data.db') + db_path = self.bot.config.get('Web_Viewer', 'db_path', fallback='meshcore_bot.db') # Resolve database path (relative paths resolved from bot root, absolute paths used as-is) base_dir = self.bot.bot_root if hasattr(self.bot, 'bot_root') else '.' @@ -160,7 +160,7 @@ class BotIntegration: serializable_data = self._make_json_serializable(packet_data) # Store in database for web viewer to read - db_path = self.bot.config.get('Web_Viewer', 'db_path', fallback='bot_data.db') + db_path = self.bot.config.get('Web_Viewer', 'db_path', fallback='meshcore_bot.db') # Resolve database path (relative paths resolved from bot root, absolute paths used as-is) base_dir = self.bot.bot_root if hasattr(self.bot, 'bot_root') else '.' db_path = resolve_path(db_path, base_dir) @@ -225,7 +225,7 @@ class BotIntegration: serializable_data = self._make_json_serializable(command_data) # Store in database for web viewer to read - db_path = self.bot.config.get('Web_Viewer', 'db_path', fallback='bot_data.db') + db_path = self.bot.config.get('Web_Viewer', 'db_path', fallback='meshcore_bot.db') # Resolve database path (relative paths resolved from bot root, absolute paths used as-is) base_dir = self.bot.bot_root if hasattr(self.bot, 'bot_root') else '.' db_path = resolve_path(db_path, base_dir) @@ -255,7 +255,7 @@ class BotIntegration: serializable_data = self._make_json_serializable(routing_data) # Store in database for web viewer to read - db_path = self.bot.config.get('Web_Viewer', 'db_path', fallback='bot_data.db') + db_path = self.bot.config.get('Web_Viewer', 'db_path', fallback='meshcore_bot.db') # Resolve database path (relative paths resolved from bot root, absolute paths used as-is) base_dir = self.bot.bot_root if hasattr(self.bot, 'bot_root') else '.' db_path = resolve_path(db_path, base_dir) @@ -282,7 +282,7 @@ class BotIntegration: cutoff_time = time.time() - (days_to_keep * 24 * 60 * 60) - db_path = self.bot.config.get('Web_Viewer', 'db_path', fallback='bot_data.db') + db_path = self.bot.config.get('Web_Viewer', 'db_path', fallback='meshcore_bot.db') # Resolve database path (relative paths resolved from bot root, absolute paths used as-is) base_dir = self.bot.bot_root if hasattr(self.bot, 'bot_root') else '.' db_path = resolve_path(db_path, base_dir)