dandri/meshcore-bot
1
0
Fork 0
mirror of https://github.com/agessaman/meshcore-bot.git synced 2026-07-30 07:19:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: Update database paths and scoring parameters in configuration

Browse Source 
- Changed the default database path in config.ini.example and related files from 'bot_data.db' to 'meshcore_bot.db' for consistency across modules.
- Adjusted final hop proximity scoring parameters in config.ini.example to reflect updated distance normalization and thresholds, enhancing scoring accuracy for LoRa networks.
- Updated documentation to clarify the shared database usage between the bot and web viewer.
This commit is contained in:
agessaman
2026-02-07 21:31:07 -08:00
parent d15c58994e
commit e1f3a7ab09
9 changed files with 146 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config.ini.example
+10 -10
View File
@@ -795,18 +795,18 @@ graph_final_hop_proximity_weight = 0.25
# Set to 0 to disable distance limiting
graph_final_hop_max_distance = 0
# Distance normalization for final hop proximity scoring (km, default: 500)
# Distance normalization for final hop proximity scoring (km, default: 200)
# Closer repeaters get higher proximity scores. Lower values = more aggressive scoring
# 500km means repeaters within 500km get full scoring range, beyond that scores decrease
graph_final_hop_proximity_normalization_km = 500
# 200km (long LoRa range) means repeaters within 200km get full scoring range, beyond that scores decrease
graph_final_hop_proximity_normalization_km = 200
# Very close distance threshold for boosted proximity weight (km, default: 50)
# Very close distance threshold for boosted proximity weight (km, default: 10)
# Repeaters within this distance get 2x proximity weight (up to 0.6 max)
graph_final_hop_very_close_threshold_km = 50
graph_final_hop_very_close_threshold_km = 10
# Close distance threshold for boosted proximity weight (km, default: 100)
# Repeaters within this distance get 1.5x proximity weight (up to 0.5 max)
graph_final_hop_close_threshold_km = 100
# Close distance threshold for boosted proximity weight (km, default: 30)
# Repeaters within this distance get 1.5x proximity weight (up to 0.5 max). Typical LoRa range.
graph_final_hop_close_threshold_km = 30
# Maximum proximity weight for very close repeaters (0.0-1.0, default: 0.6)
# When a repeater is within very_close_threshold_km, proximity weight is boosted up to this value
@@ -1034,8 +1034,8 @@ debug = false
# false: Start web viewer manually (recommended)
auto_start = false
# Database path for web viewer data
# Default: bot_data.db
# Database path for web viewer (same as [Bot] db_path so viewer can show repeater/graph data and packet stream)
# Default: meshcore_bot.db
db_path = meshcore_bot.db
# Additional hashtag channels to decode in the packet stream
docker-setup.sh
+5 -5
View File
@@ -74,7 +74,7 @@ echo " ✓ Updated [Bot] db_path"
# Ensure we use absolute path to avoid resolution issues
update_config "Logging" "log_file" "/data/logs/meshcore_bot.log"
# Verify the update worked
VERIFIED_LOG=$(grep "^log_file[[:space:]]*=" "$CONFIG_FILE" 2>/dev/null | sed 's/^log_file[[:space:]]*=[[:space:]]*//' | tr -d ' ' || echo "")
VERIFIED_LOG=$(grep "^log_file[[:space:]]*=" "$CONFIG_FILE" 2>/dev/null | sed 's/^log_file[[:space:]]*=[[:space:]]*//;s/^[[:space:]]*//;s/[[:space:]]*$//' || echo "")
if [[ "$VERIFIED_LOG" == "/data/logs/meshcore_bot.log" ]]; then
echo " ✓ Updated [Logging] log_file to /data/logs/meshcore_bot.log"
((UPDATED_COUNT++))
@@ -93,7 +93,7 @@ fi
# Update Web_Viewer database path (if section exists)
if grep -q "^\[Web_Viewer\]" "$CONFIG_FILE"; then
update_config "Web_Viewer" "db_path" "/data/databases/bot_data.db"
update_config "Web_Viewer" "db_path" "/data/databases/meshcore_bot.db"
echo " ✓ Updated [Web_Viewer] db_path"
((UPDATED_COUNT++))
fi
@@ -101,7 +101,7 @@ fi
# Update PacketCapture paths (if section exists)
if grep -q "^\[PacketCapture\]" "$CONFIG_FILE"; then
# Only update output_file if it's set to a relative path
CURRENT_OUTPUT=$(grep "^output_file[[:space:]]*=" "$CONFIG_FILE" 2>/dev/null | sed 's/^output_file[[:space:]]*=[[:space:]]*//' | tr -d ' ' || echo "")
CURRENT_OUTPUT=$(grep "^output_file[[:space:]]*=" "$CONFIG_FILE" 2>/dev/null | sed 's/^output_file[[:space:]]*=[[:space:]]*//;s/^[[:space:]]*//;s/[[:space:]]*$//' || echo "")
if [ -n "$CURRENT_OUTPUT" ] && [[ ! "$CURRENT_OUTPUT" == /* ]]; then
# Relative path - update to logs directory
update_config "PacketCapture" "output_file" "/data/logs/packets.jsonl"
@@ -110,7 +110,7 @@ if grep -q "^\[PacketCapture\]" "$CONFIG_FILE"; then
fi
# Update private_key_path if it's a relative path
CURRENT_KEY=$(grep "^private_key_path[[:space:]]*=" "$CONFIG_FILE" 2>/dev/null | sed 's/^private_key_path[[:space:]]*=[[:space:]]*//' | tr -d ' ' || echo "")
CURRENT_KEY=$(grep "^private_key_path[[:space:]]*=" "$CONFIG_FILE" 2>/dev/null | sed 's/^private_key_path[[:space:]]*=[[:space:]]*//;s/^[[:space:]]*//;s/[[:space:]]*$//' || echo "")
if [ -n "$CURRENT_KEY" ] && [[ ! "$CURRENT_KEY" == /* ]]; then
# Relative path - update to config directory (read-only is fine for keys)
update_config "PacketCapture" "private_key_path" "/data/config/private_key"
@@ -121,7 +121,7 @@ fi
# Update MapUploader private_key_path (if section exists)
if grep -q "^\[MapUploader\]" "$CONFIG_FILE"; then
CURRENT_KEY=$(grep "^private_key_path[[:space:]]*=" "$CONFIG_FILE" 2>/dev/null | sed 's/^private_key_path[[:space:]]*=[[:space:]]*//' | tr -d ' ' || echo "")
CURRENT_KEY=$(grep "^private_key_path[[:space:]]*=" "$CONFIG_FILE" 2>/dev/null | sed 's/^private_key_path[[:space:]]*=[[:space:]]*//;s/^[[:space:]]*//;s/[[:space:]]*$//' || echo "")
if [ -n "$CURRENT_KEY" ] && [[ ! "$CURRENT_KEY" == /* ]]; then
# Relative path - update to config directory
update_config "MapUploader" "private_key_path" "/data/config/private_key"
docs/DOCKER.md
+1 -1
View File
@@ -41,7 +41,7 @@ This guide explains how to deploy meshcore-bot using Docker and Docker Compose.
db_path = /data/databases/meshcore_bot.db
[Web_Viewer]
db_path = /data/databases/bot_data.db
db_path = /data/databases/meshcore_bot.db
[Logging]
log_file = /data/logs/meshcore_bot.log
docs/WEB_VIEWER.md
+25 -2
View File
@@ -118,9 +118,32 @@ curl http://localhost:5000/api/stats
## Database Requirements
The viewer requires access to the main database:
- `meshcore_bot.db` - Main bot database (contains all data including contacts, tracking, cache, and stats)
The viewer uses the same database as the bot by default (`[Bot] db_path`, typically `meshcore_bot.db`). That single file holds repeater contacts, mesh graph, packet stream, and other data so the viewer can show everything.
## Migrating from a separate web viewer database
If you previously had the web viewer using a **separate** database (e.g. `[Web_Viewer] db_path = bot_data.db`), you can switch to the shared database so the viewer shows repeater/graph data and uses one file.
1. **Stop the bot and web viewer** so neither has the databases open.
2. **Optionally preserve packet stream history** from the old viewer DB into the main DB:
- From the project root, run:
```bash
python3 migrate_webviewer_db.py bot_data.db meshcore_bot.db
```
Use your actual paths if they differ (e.g. full paths or different filenames). The script copies the `packet_stream` table from the first file into the second and skips rows that would duplicate IDs.
- If you don’t care about old packet stream data, skip this step; the viewer will create a new `packet_stream` table in the main DB.
3. **Point the viewer at the main database** in `config.ini`:
```ini
[Web_Viewer]
db_path = meshcore_bot.db
```
(Or the same value as `[Bot] db_path` if you use a different path.)
4. **Start the bot (and viewer as usual)**. The viewer will now read and write to the same database as the bot.
You can keep or remove the old `bot_data.db` file after verifying the viewer works with the shared DB.
## Troubleshooting
migrate_webviewer_db.py
+89
View File
@@ -0,0 +1,89 @@
#!/usr/bin/env python3
"""
One-off migration: copy packet_stream from a separate web viewer database
(e.g. bot_data.db) into the main bot database (e.g. meshcore_bot.db).
Use this when switching from split databases to the shared database so you
don't lose packet stream history. Run with bot and web viewer stopped.
Usage:
python3 migrate_webviewer_db.py <source_db> <target_db>
Example:
python3 migrate_webviewer_db.py bot_data.db meshcore_bot.db
"""
import sqlite3
import sys
def main() -> int:
if len(sys.argv) != 3:
print("Usage: python3 migrate_webviewer_db.py <source_db> <target_db>", file=sys.stderr)
print("Example: python3 migrate_webviewer_db.py bot_data.db meshcore_bot.db", file=sys.stderr)
return 1
source_path = sys.argv[1]
target_path = sys.argv[2]
if source_path == target_path:
print("Source and target must be different files.", file=sys.stderr)
return 1
try:
conn = sqlite3.connect(target_path, timeout=30.0)
conn.execute("ATTACH DATABASE ? AS src", (source_path,))
# Ensure packet_stream exists in target (same schema as web viewer)
conn.execute("""
CREATE TABLE IF NOT EXISTS packet_stream (
id INTEGER PRIMARY KEY AUTOINCREMENT,
timestamp REAL NOT NULL,
data TEXT NOT NULL,
type TEXT NOT NULL
)
""")
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_packet_stream_timestamp ON packet_stream(timestamp)"
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_packet_stream_type ON packet_stream(type)"
)
# Check that source has packet_stream
cur = conn.execute(
"SELECT name FROM src.sqlite_master WHERE type='table' AND name='packet_stream'"
)
if cur.fetchone() is None:
conn.execute("DETACH DATABASE src")
conn.close()
print(f"Source database has no packet_stream table; nothing to migrate.", file=sys.stderr)
return 0
# Copy rows from source; skip ids that already exist in target (INSERT OR IGNORE)
before = conn.total_changes
conn.execute(
"""
INSERT OR IGNORE INTO main.packet_stream (id, timestamp, data, type)
SELECT id, timestamp, data, type FROM src.packet_stream
"""
)
inserted = conn.total_changes - before
conn.commit()
conn.execute("DETACH DATABASE src")
conn.close()
print(f"Migrated {inserted} packet_stream row(s) from {source_path} to {target_path}.")
return 0
except sqlite3.OperationalError as e:
print(f"Database error: {e}", file=sys.stderr)
return 1
except OSError as e:
print(f"File error: {e}", file=sys.stderr)
return 1
if __name__ == "__main__":
sys.exit(main())
modules/repeater_manager.py
+2 -2
View File
@@ -159,7 +159,7 @@ class RepeaterManager:
''')
# Create indexes for better performance
with sqlite3.connect(self.db_path) as conn:
with sqlite3.connect(self.db_path, timeout=30.0) as conn:
cursor = conn.cursor()
cursor.execute('CREATE INDEX IF NOT EXISTS idx_public_key ON repeater_contacts(public_key)')
cursor.execute('CREATE INDEX IF NOT EXISTS idx_device_type ON repeater_contacts(device_type)')
@@ -214,7 +214,7 @@ class RepeaterManager:
"""Handle database schema migration for existing installations"""
try:
# Check if the new location columns exist in repeater_contacts
with sqlite3.connect(self.db_path) as conn:
with sqlite3.connect(self.db_path, timeout=30.0) as conn:
cursor = conn.cursor()
cursor.execute("PRAGMA table_info(repeater_contacts)")
columns = [row[1] for row in cursor.fetchall()]
modules/transmission_tracker.py
+1 -1
View File
@@ -190,7 +190,7 @@ class TransmissionTracker:
return
# Get database path
db_path = self.bot.config.get('Web_Viewer', 'db_path', fallback='bot_data.db')
db_path = self.bot.config.get('Web_Viewer', 'db_path', fallback='meshcore_bot.db')
base_dir = self.bot.bot_root if hasattr(self.bot, 'bot_root') else '.'
db_path = resolve_path(db_path, base_dir)
modules/web_viewer/app.py
+7 -7
View File
@@ -196,11 +196,11 @@ class BotDataViewer:
raise
def _init_packet_stream_table(self):
"""Initialize the packet_stream table in bot_data.db"""
"""Initialize the packet_stream table in the web viewer database (same as [Bot] db_path by default)."""
conn = None
try:
# Get database path from config
db_path = self.config.get('Web_Viewer', 'db_path', fallback='bot_data.db')
db_path = self.config.get('Web_Viewer', 'db_path', fallback='meshcore_bot.db')
# Resolve database path (relative paths resolved from bot root, absolute paths used as-is)
db_path = resolve_path(db_path, self.bot_root)
@@ -1481,7 +1481,7 @@ class BotDataViewer:
cutoff_time = time.time() - (60 * 60) # 60 minutes ago
# Get database path
db_path = self.config.get('Web_Viewer', 'db_path', fallback='bot_data.db')
db_path = self.config.get('Web_Viewer', 'db_path', fallback='meshcore_bot.db')
# Resolve database path (relative paths resolved from bot root, absolute paths used as-is)
db_path = resolve_path(db_path, self.bot_root)
@@ -2602,7 +2602,7 @@ class BotDataViewer:
import json
# Get database path (re-resolve on each iteration in case config changed)
db_path = self.config.get('Web_Viewer', 'db_path', fallback='bot_data.db')
db_path = self.config.get('Web_Viewer', 'db_path', fallback='meshcore_bot.db')
# Resolve database path (relative paths resolved from bot root, absolute paths used as-is)
db_path = resolve_path(db_path, self.bot_root)
@@ -2788,7 +2788,7 @@ class BotDataViewer:
cutoff_time = time.time() - (days_to_keep * 24 * 60 * 60)
# Get database path
db_path = self.config.get('Web_Viewer', 'db_path', fallback='bot_data.db')
db_path = self.config.get('Web_Viewer', 'db_path', fallback='meshcore_bot.db')
# Resolve database path (relative paths resolved from bot root, absolute paths used as-is)
db_path = resolve_path(db_path, self.bot_root)
@@ -3299,7 +3299,7 @@ class BotDataViewer:
# Get database file size
import os
db_path = self.config.get('Web_Viewer', 'db_path', fallback='bot_data.db')
db_path = self.config.get('Web_Viewer', 'db_path', fallback='meshcore_bot.db')
# Resolve database path (relative paths resolved from bot root, absolute paths used as-is)
db_path = resolve_path(db_path, self.bot_root)
try:
@@ -3357,7 +3357,7 @@ class BotDataViewer:
# Get initial database size
import os
db_path = self.config.get('Web_Viewer', 'db_path', fallback='bot_data.db')
db_path = self.config.get('Web_Viewer', 'db_path', fallback='meshcore_bot.db')
# Resolve database path (relative paths resolved from bot root, absolute paths used as-is)
db_path = resolve_path(db_path, self.bot_root)
initial_size = os.path.getsize(db_path)
modules/web_viewer/integration.py
+6 -6
View File
@@ -82,12 +82,12 @@ class BotIntegration:
self.circuit_breaker_failures = 0
def _init_packet_stream_table(self):
"""Initialize the packet_stream table in bot_data.db"""
"""Initialize the packet_stream table in the web viewer database (same as [Bot] db_path by default)."""
try:
import sqlite3
# Get database path from config
db_path = self.bot.config.get('Web_Viewer', 'db_path', fallback='bot_data.db')
db_path = self.bot.config.get('Web_Viewer', 'db_path', fallback='meshcore_bot.db')
# Resolve database path (relative paths resolved from bot root, absolute paths used as-is)
base_dir = self.bot.bot_root if hasattr(self.bot, 'bot_root') else '.'
@@ -160,7 +160,7 @@ class BotIntegration:
serializable_data = self._make_json_serializable(packet_data)
# Store in database for web viewer to read
db_path = self.bot.config.get('Web_Viewer', 'db_path', fallback='bot_data.db')
db_path = self.bot.config.get('Web_Viewer', 'db_path', fallback='meshcore_bot.db')
# Resolve database path (relative paths resolved from bot root, absolute paths used as-is)
base_dir = self.bot.bot_root if hasattr(self.bot, 'bot_root') else '.'
db_path = resolve_path(db_path, base_dir)
@@ -225,7 +225,7 @@ class BotIntegration:
serializable_data = self._make_json_serializable(command_data)
# Store in database for web viewer to read
db_path = self.bot.config.get('Web_Viewer', 'db_path', fallback='bot_data.db')
db_path = self.bot.config.get('Web_Viewer', 'db_path', fallback='meshcore_bot.db')
# Resolve database path (relative paths resolved from bot root, absolute paths used as-is)
base_dir = self.bot.bot_root if hasattr(self.bot, 'bot_root') else '.'
db_path = resolve_path(db_path, base_dir)
@@ -255,7 +255,7 @@ class BotIntegration:
serializable_data = self._make_json_serializable(routing_data)
# Store in database for web viewer to read
db_path = self.bot.config.get('Web_Viewer', 'db_path', fallback='bot_data.db')
db_path = self.bot.config.get('Web_Viewer', 'db_path', fallback='meshcore_bot.db')
# Resolve database path (relative paths resolved from bot root, absolute paths used as-is)
base_dir = self.bot.bot_root if hasattr(self.bot, 'bot_root') else '.'
db_path = resolve_path(db_path, base_dir)
@@ -282,7 +282,7 @@ class BotIntegration:
cutoff_time = time.time() - (days_to_keep * 24 * 60 * 60)
db_path = self.bot.config.get('Web_Viewer', 'db_path', fallback='bot_data.db')
db_path = self.bot.config.get('Web_Viewer', 'db_path', fallback='meshcore_bot.db')
# Resolve database path (relative paths resolved from bot root, absolute paths used as-is)
base_dir = self.bot.bot_root if hasattr(self.bot, 'bot_root') else '.'
db_path = resolve_path(db_path, base_dir)