diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 28d61ab..e376187 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -106,13 +106,40 @@ jobs:
- name: mypy — strict overrides on typed modules
run: mypy modules/ --ignore-missing-imports
+ package:
+ name: Build and smoke-test distributions
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Set up Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: "3.11"
+ cache: pip
+
+ - name: Build wheel and source distribution
+ run: |
+ python -m pip install --upgrade pip build twine
+ python -m build
+ python -m twine check dist/*
+
+ - name: Install and smoke-test wheel outside the source tree
+ run: |
+ python -m venv /tmp/meshcore-wheel-smoke
+ /tmp/meshcore-wheel-smoke/bin/pip install dist/*.whl
+ cd /tmp
+ /tmp/meshcore-wheel-smoke/bin/python "$GITHUB_WORKSPACE/scripts/smoke_test_wheel.py"
+ /tmp/meshcore-wheel-smoke/bin/meshcore-bot --help >/dev/null
+ /tmp/meshcore-wheel-smoke/bin/meshcore-viewer --help >/dev/null
+
test:
name: Tests (Python ${{ matrix.python-version }})
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
- python-version: ["3.11", "3.12", "3.13"]
+ python-version: ["3.10", "3.11", "3.12", "3.13"]
steps:
- uses: actions/checkout@v4
diff --git a/.gitignore b/.gitignore
index 237f2e2..e0ed502 100644
--- a/.gitignore
+++ b/.gitignore
@@ -182,3 +182,6 @@ local/service_plugins/*
!local/service_plugins/.gitkeep
.claude/settings.local.json
+
+# Local engineering audit reports (may contain security-sensitive findings)
+/PROJECT_REVIEW_*.md
diff --git a/com.meshcore.bot.plist b/com.meshcore.bot.plist
index 03e1d82..b5a1e4c 100644
--- a/com.meshcore.bot.plist
+++ b/com.meshcore.bot.plist
@@ -4,11 +4,16 @@
Label
com.meshcore.bot
+
+ UserName
+ __MESHCORE_SERVICE_USER__
ProgramArguments
/usr/local/meshcore-bot/venv/bin/python
/usr/local/meshcore-bot/meshcore_bot.py
+ --config
+ /usr/local/etc/meshcore-bot/config.ini
WorkingDirectory
@@ -47,4 +52,3 @@
1
-
diff --git a/data/randomlines/__init__.py b/data/randomlines/__init__.py
new file mode 100644
index 0000000..2b1fed9
--- /dev/null
+++ b/data/randomlines/__init__.py
@@ -0,0 +1 @@
+"""Bundled default RandomLine response files."""
diff --git a/install-service.sh b/install-service.sh
index 10792ed..3544ab1 100755
--- a/install-service.sh
+++ b/install-service.sh
@@ -17,11 +17,12 @@
#
# Prerequisites:
# - Linux system with systemd OR macOS
-# - Python 3.9+ installed
+# - Python 3.10+ installed
# - sudo access (script will prompt if needed)
# - Run from the meshcore-bot directory
set -e
+umask 077
# Colors for output
RED='\033[0;31m'
@@ -53,6 +54,8 @@ if [[ "$IS_MACOS" == true ]]; then
SERVICE_USER="$(whoami)" # macOS: use current user or _meshcore
SERVICE_GROUP="staff"
INSTALL_DIR="/usr/local/meshcore-bot"
+ CONF_DIR="/usr/local/etc/meshcore-bot"
+ STATE_DIR="/usr/local/var/lib/meshcore-bot"
LOG_DIR="/usr/local/var/log/meshcore-bot"
SERVICE_FILE="com.meshcore.bot.plist"
LAUNCHD_DIR="/Library/LaunchDaemons"
@@ -60,11 +63,15 @@ else
SERVICE_USER="meshcore"
SERVICE_GROUP="meshcore"
INSTALL_DIR="/opt/meshcore-bot"
+ CONF_DIR="/etc/meshcore-bot"
+ STATE_DIR="/var/lib/meshcore-bot"
LOG_DIR="/var/log/meshcore-bot"
SERVICE_FILE="meshcore-bot.service"
SYSTEMD_DIR="/etc/systemd/system"
fi
+CONFIG_FILE="$CONF_DIR/config.ini"
+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# Parse command line arguments (before sudo check so help works)
@@ -222,9 +229,34 @@ fi
# Check if Python 3 is available
if ! command -v python3 &> /dev/null; then
print_error "Python 3 is not installed or not in PATH"
- print_error "Please install Python 3.9 or higher before running this script"
+ print_error "Please install Python 3.10 or higher before running this script"
exit 1
fi
+if ! python3 -c 'import sys; raise SystemExit(sys.version_info < (3, 10))'; then
+ print_error "Python 3.10 or higher is required"
+ print_error "Found: $(python3 --version 2>&1)"
+ exit 1
+fi
+
+# Stop a running legacy service before changing code or taking the SQLite
+# backup used for layout migration. This runs only after sudo re-execution and
+# service-manager validation, so --help and unprivileged discovery stay inert.
+SERVICE_WAS_ACTIVE=false
+if [[ "$IS_MACOS" == true ]]; then
+ if launchctl list "$PLIST_NAME" &>/dev/null; then
+ SERVICE_WAS_ACTIVE=true
+ if ! launchctl unload "$LAUNCHD_DIR/$SERVICE_FILE" 2>/dev/null; then
+ launchctl stop "$PLIST_NAME" 2>/dev/null || true
+ fi
+ if launchctl list "$PLIST_NAME" &>/dev/null; then
+ print_error "The launchd service is still running; refusing an unsafe live database migration"
+ exit 1
+ fi
+ fi
+elif systemctl is-active --quiet "$SERVICE_NAME" 2>/dev/null; then
+ SERVICE_WAS_ACTIVE=true
+ systemctl stop "$SERVICE_NAME"
+fi
print_section "Step 1: Setting Up Service User"
if [[ "$IS_MACOS" == true ]]; then
@@ -291,160 +323,53 @@ else
print_success "Created installation directory: $INSTALL_DIR"
fi
-# Create log directory
-mkdir -p "$LOG_DIR"
+# Create mutable runtime directories separately from executable code.
+mkdir -p "$CONF_DIR" "$STATE_DIR" "$LOG_DIR"
+print_success "Created configuration directory: $CONF_DIR"
+print_success "Created state directory: $STATE_DIR"
print_success "Created log directory: $LOG_DIR"
print_section "Step 3: Copying Bot Files"
if [[ "$UPGRADE_MODE" == true ]]; then
print_info "Upgrading files in $INSTALL_DIR"
- print_info "Only newer files will be copied, preserving existing configuration"
+ print_info "Replacing executable files from the trusted source while preserving explicit runtime state"
else
print_info "Copying bot files to $INSTALL_DIR"
- print_info "Existing files will be updated only if source is newer"
+ print_info "Executable files will exactly match the trusted source"
fi
-# Function to copy files intelligently
+# Synchronize executable code authoritatively. Using --update or a merge-only
+# fallback can preserve a newer/stale Python file written by a previously
+# compromised service account and then cement it as root-owned executable code.
+# Only the explicitly excluded runtime paths survive an upgrade.
copy_files_smart() {
local source_dir="$1"
local dest_dir="$2"
- local files_copied=0
- local files_skipped=0
- local files_updated=0
-
- # Use rsync if available (better for this use case)
- if command -v rsync &> /dev/null; then
- print_info "Using rsync for efficient file copying"
- # Preserve config.ini if it exists
- local preserve_config=""
- if [ -f "$dest_dir/config.ini" ]; then
- preserve_config="--exclude=config.ini"
- print_info "Preserving existing config.ini (not overwriting)"
- fi
-
- # Note: --update flag preserves files in alternatives/ if destination is newer or same
- # This protects user's custom alternative commands while allowing updates to repository files
- if [ -d "$dest_dir/modules/commands/alternatives" ]; then
- print_info "Preserving existing alternative commands (only updating if source is newer)"
- fi
-
- # Preserve install dir's local/ entirely (user custom commands and service plugins)
- # Never overwrite or delete anything under $dest_dir/local/
- if [ -d "$dest_dir/local" ]; then
- print_info "Preserving existing local/ directory (not overwriting)"
- fi
-
- # Exclude patterns
- rsync -a --update --exclude='.git' \
- --exclude='__pycache__' \
- --exclude='*.pyc' \
- --exclude='*.pyo' \
- --exclude='.DS_Store' \
- --exclude='venv' \
- --exclude='*.db' \
- --exclude='*.db-shm' \
- --exclude='*.db-wal' \
- --exclude='*.log' \
- --exclude='backups' \
- --exclude='local/' \
- $preserve_config \
- "$source_dir/" "$dest_dir/" 2>/dev/null || {
- print_warning "rsync had some issues, falling back to manual copy"
- }
- # If install dir has no local/, create minimal structure from source so service has valid layout
- if [ ! -d "$dest_dir/local" ]; then
- print_info "Creating local/ directory structure (first-time install)"
- mkdir -p "$dest_dir/local/commands" "$dest_dir/local/service_plugins"
- [ -f "$source_dir/local/README.md" ] && cp "$source_dir/local/README.md" "$dest_dir/local/" || true
- [ -f "$source_dir/local/__init__.py" ] && cp "$source_dir/local/__init__.py" "$dest_dir/local/" || true
- [ -f "$source_dir/local/commands/.gitkeep" ] && cp "$source_dir/local/commands/.gitkeep" "$dest_dir/local/commands/" || true
- [ -f "$source_dir/local/service_plugins/.gitkeep" ] && cp "$source_dir/local/service_plugins/.gitkeep" "$dest_dir/local/service_plugins/" || true
- fi
- print_success "Files synchronized using rsync"
- return 0
+ if ! command -v rsync &> /dev/null; then
+ print_error "rsync is required for a source-authoritative secure install; install rsync and retry"
+ return 1
fi
-
- # Fallback: manual copy with find
- print_info "Using manual file copy (consider installing rsync for better performance)"
-
- # Preserve alternatives directory if it exists
- if [ -d "$dest_dir/modules/commands/alternatives" ]; then
- print_info "Preserving existing alternative commands (not overwriting)"
+
+ print_info "Using rsync for source-authoritative executable synchronization"
+ if ! rsync -a --delete --exclude='.git' \
+ --exclude='__pycache__' \
+ --exclude='*.pyc' \
+ --exclude='*.pyo' \
+ --exclude='.DS_Store' \
+ --exclude='venv' \
+ --exclude='*.db' \
+ --exclude='*.db-shm' \
+ --exclude='*.db-wal' \
+ --exclude='*.log' \
+ --exclude='backups' \
+ --exclude='local/' \
+ --exclude='config.ini' \
+ "$source_dir/" "$dest_dir/"; then
+ print_error "rsync failed; refusing to continue with a partially synchronized code tree"
+ return 1
fi
-
- # Preserve install dir's local/ entirely - never overwrite when it exists
- if [ -d "$dest_dir/local" ]; then
- print_info "Preserving existing local/ directory (not overwriting)"
- fi
-
- # Copy files, preserving config.ini if it exists
- while IFS= read -r file; do
- local rel_path="${file#$source_dir/}"
- local dest_file="$dest_dir/$rel_path"
- local dest_dir_path
- dest_dir_path="$(dirname "$dest_file")"
-
- # Skip excluded patterns
- [[ "$rel_path" == *".git"* ]] && continue
- [[ "$rel_path" == *"__pycache__"* ]] && continue
- [[ "$rel_path" == *".pyc" ]] && continue
- [[ "$rel_path" == *".pyo" ]] && continue
- [[ "$rel_path" == *".DS_Store"* ]] && continue
- [[ "$rel_path" == *"/venv/"* ]] && continue
- [[ "$rel_path" == *".db" ]] && continue
- [[ "$rel_path" == *".db-shm" ]] && continue
- [[ "$rel_path" == *".db-wal" ]] && continue
- [[ "$rel_path" == *".log" ]] && continue
- [[ "$rel_path" == *"/backups/"* ]] && continue
-
- # Preserve install dir's local/ entirely - skip all files under local/ when dest has local/
- if [[ "$rel_path" == "local/"* ]] && [ -d "$dest_dir/local" ]; then
- files_skipped=$((files_skipped + 1))
- continue
- fi
-
- # Preserve alternatives directory - only update if source file is newer
- if [[ "$rel_path" == "modules/commands/alternatives/"* ]] && [ -d "$dest_dir/modules/commands/alternatives" ]; then
- if [ -f "$dest_file" ]; then
- # File exists in destination - only update if source is newer
- if [ "$file" -nt "$dest_file" ]; then
- # Source is newer, will update below
- :
- else
- # Destination is same or newer - preserve user's version
- files_skipped=$((files_skipped + 1))
- continue
- fi
- fi
- # File doesn't exist in destination, or source is newer - will copy below
- fi
-
- # Create destination directory if needed
- mkdir -p "$dest_dir_path"
-
- # Special handling for config.ini - preserve existing if it exists
- if [[ "$rel_path" == "config.ini" ]] && [ -f "$dest_file" ]; then
- files_skipped=$((files_skipped + 1))
- continue
- fi
-
- # Copy if destination doesn't exist or source is newer
- if [ ! -f "$dest_file" ] || [ "$file" -nt "$dest_file" ]; then
- if cp "$file" "$dest_file" 2>/dev/null; then
- if [ -f "$dest_file" ]; then
- files_updated=$((files_updated + 1))
- else
- files_copied=$((files_copied + 1))
- fi
- else
- print_warning "Could not copy $rel_path"
- fi
- else
- files_skipped=$((files_skipped + 1))
- fi
- done < <(find "$source_dir" -type f 2>/dev/null)
-
- print_success "File sync complete: $files_updated updated, $files_copied new, $files_skipped unchanged"
+
+ print_success "Executable files synchronized authoritatively using rsync"
}
# Copy files using smart copy function
@@ -465,52 +390,63 @@ if command -v git &>/dev/null && [ -d "$SCRIPT_DIR/.git" ]; then
print_success "Wrote version info (${INSTALLER_VER}) to $INSTALL_DIR/.version_info"
fi
-# If no config.ini in install dir, create it from config.ini.example
-if [ ! -f "$INSTALL_DIR/config.ini" ]; then
- if [ -f "$INSTALL_DIR/config.ini.example" ]; then
- cp "$INSTALL_DIR/config.ini.example" "$INSTALL_DIR/config.ini"
- print_success "Created $INSTALL_DIR/config.ini from config.ini.example (no config was present)"
+# Keep configuration out of the root-owned application tree. On upgrade, copy
+# the legacy config once so existing credentials and settings are preserved.
+if [ ! -f "$CONFIG_FILE" ]; then
+ if [ -f "$INSTALL_DIR/config.ini" ] && [ ! -L "$INSTALL_DIR/config.ini" ]; then
+ cp -p "$INSTALL_DIR/config.ini" "$CONFIG_FILE"
+ print_success "Migrated existing configuration to $CONFIG_FILE"
+ elif [ -f "$INSTALL_DIR/config.ini.example" ]; then
+ cp "$INSTALL_DIR/config.ini.example" "$CONFIG_FILE"
+ print_success "Created $CONFIG_FILE from config.ini.example"
elif [ -f "$SCRIPT_DIR/config.ini.example" ]; then
- cp "$SCRIPT_DIR/config.ini.example" "$INSTALL_DIR/config.ini"
- print_success "Created $INSTALL_DIR/config.ini from config.ini.example (no config was present)"
+ cp "$SCRIPT_DIR/config.ini.example" "$CONFIG_FILE"
+ print_success "Created $CONFIG_FILE from config.ini.example"
else
- print_warning "config.ini.example not found. Create $INSTALL_DIR/config.ini manually before starting the bot."
+ print_warning "config.ini.example not found. Create $CONFIG_FILE manually before starting the bot."
fi
fi
-# Create venv and install dependencies before chown so the service user ends up
-# owning a complete, working venv (avoids partial root-owned venv and import errors).
+# Rewrite all relative runtime paths and coherently migrate an existing SQLite
+# database. Absolute custom paths are left untouched; operators can grant an
+# additional systemd path explicitly when they intentionally store state there.
+if [ -f "$CONFIG_FILE" ]; then
+ python3 "$INSTALL_DIR/scripts/migrate_service_layout.py" \
+ --config "$CONFIG_FILE" \
+ --legacy-base "$INSTALL_DIR" \
+ --state-dir "$STATE_DIR" \
+ --log-dir "$LOG_DIR"
+fi
+
+# Remove a source-tree config copied into the application tree only when it is
+# identical to the active service config; otherwise retain it root-only as a
+# migration backup.
+if [ -f "$INSTALL_DIR/config.ini" ] && [ ! -L "$INSTALL_DIR/config.ini" ]; then
+ chmod 0600 "$INSTALL_DIR/config.ini"
+fi
+
+if [ ! -f "$CONFIG_FILE" ]; then
+ # Retain the old diagnostic wording for automation which looks for it.
+ if [ -f "$INSTALL_DIR/config.ini.example" ]; then
+ print_warning "Failed to create $CONFIG_FILE; check directory permissions"
+ fi
+fi
+
+# Build dependencies in a fresh environment. The legacy venv was writable by
+# the service account; reusing it could preserve a malicious .pth/module and
+# turn that persistence into root-owned executable code during hardening.
print_section "Step 4: Setting Up Python Virtual Environment"
-if [ -d "$INSTALL_DIR/venv" ]; then
- print_info "Virtual environment already exists at $INSTALL_DIR/venv"
- print_info "Preserving existing virtual environment"
- if [[ "$UPGRADE_MODE" == true ]]; then
- print_info "Upgrade mode: will update dependencies"
- else
- print_info "Will update dependencies if requirements.txt changed"
- fi
-else
- print_info "Creating an isolated Python environment for the bot"
- print_info "This ensures dependencies don't conflict with system Python packages"
- python3 -m venv "$INSTALL_DIR/venv"
- print_success "Created virtual environment at $INSTALL_DIR/venv"
-fi
-
-# Verify virtual environment looks healthy
-VENV_PYTHON="$INSTALL_DIR/venv/bin/python"
-if [ ! -x "$VENV_PYTHON" ]; then
- print_error "Python virtual environment at $INSTALL_DIR/venv appears to be incomplete or corrupted"
- print_error "Expected Python executable not found at: $VENV_PYTHON"
- print_info "Try removing $INSTALL_DIR/venv and re-running this installer to recreate it:"
- echo " sudo rm -rf $INSTALL_DIR/venv"
- echo " sudo ./install-service.sh"
- exit 1
-fi
+VENV_BUILD="$INSTALL_DIR/.venv-build-$$"
+VENV_OLD="$INSTALL_DIR/.venv-old-$$"
+rm -rf "$VENV_BUILD" "$VENV_OLD"
+print_info "Creating a fresh isolated Python environment"
+python3 -m venv "$VENV_BUILD"
+VENV_BUILD_PYTHON="$VENV_BUILD/bin/python"
# Ensure pip is available and up to date inside the venv
print_info "Ensuring pip is available and up to date in the virtual environment"
-$VENV_PYTHON -m ensurepip --upgrade >/dev/null 2>&1 || true
-$VENV_PYTHON -m pip install --quiet --upgrade pip >/dev/null 2>&1 || true
+$VENV_BUILD_PYTHON -m ensurepip --upgrade >/dev/null 2>&1 || true
+$VENV_BUILD_PYTHON -m pip install --quiet --upgrade pip >/dev/null 2>&1 || true
# Install dependencies in venv using python -m pip (more portable than calling pip directly)
print_info "Installing Python dependencies from requirements.txt"
@@ -519,12 +455,22 @@ if [ ! -f "$INSTALL_DIR/requirements.txt" ]; then
print_error "requirements.txt not found in installation directory"
exit 1
fi
-$VENV_PYTHON -m pip install --quiet -r "$INSTALL_DIR/requirements.txt" || {
+$VENV_BUILD_PYTHON -m pip install --quiet -r "$INSTALL_DIR/requirements.txt" || {
print_error "Failed to install Python dependencies"
print_info "You may need to check your internet connection or Python version"
+ rm -rf "$VENV_BUILD"
exit 1
}
-print_success "Installed all Python dependencies"
+if [ -d "$INSTALL_DIR/venv" ]; then
+ mv "$INSTALL_DIR/venv" "$VENV_OLD"
+fi
+if ! mv "$VENV_BUILD" "$INSTALL_DIR/venv"; then
+ [ -d "$VENV_OLD" ] && mv "$VENV_OLD" "$INSTALL_DIR/venv"
+ print_error "Failed to activate the newly built virtual environment"
+ exit 1
+fi
+rm -rf "$VENV_OLD"
+print_success "Installed all Python dependencies into a fresh virtual environment"
# Optional extras
echo ""
@@ -555,19 +501,37 @@ fi
print_section "Step 5: Setting File Permissions"
print_info "Configuring file ownership and permissions for security"
-print_info "The service user will own all files, with appropriate read/write permissions"
-# Set ownership
-chown -R "$SERVICE_USER:$SERVICE_GROUP" "$INSTALL_DIR"
+print_info "Executable code is root-owned; the service owns only configuration and runtime state"
+# Executable code and the virtual environment must not be writable by the
+# network-facing service account.
+# The service group receives read-only access to any explicitly installed key
+# material while root remains the only account able to modify it.
+CODE_GROUP="$SERVICE_GROUP"
+chown -R "root:$CODE_GROUP" "$INSTALL_DIR"
chown -R "$SERVICE_USER:$SERVICE_GROUP" "$LOG_DIR"
-print_success "Set ownership to $SERVICE_USER:$SERVICE_GROUP"
+chown -R "$SERVICE_USER:$SERVICE_GROUP" "$CONF_DIR" "$STATE_DIR"
+print_success "Separated root-owned code from service-owned runtime state"
-# Set permissions
+# Code is readable/executable but never service-writable. Preserve executable
+# bits created by the virtualenv and source scripts while dropping group/other
+# write access.
chmod 755 "$INSTALL_DIR"
+chmod -R go-w "$INSTALL_DIR"
find "$INSTALL_DIR" -type f -name "*.py" -exec chmod 644 {} \; 2>/dev/null || true
-find "$INSTALL_DIR" -type f -name "*.ini" -exec chmod 644 {} \; 2>/dev/null || true
find "$INSTALL_DIR" -type f -name "*.txt" -exec chmod 644 {} \; 2>/dev/null || true
find "$INSTALL_DIR" -type f -name "*.json" -exec chmod 644 {} \; 2>/dev/null || true
find "$INSTALL_DIR" -type d -exec chmod 755 {} \; 2>/dev/null || true
+find "$INSTALL_DIR" -type f -name "*.ini" -exec chmod 600 {} \; 2>/dev/null || true
+find "$INSTALL_DIR" -type f \( -name ".env" -o -name "*.key" -o -name "*.pem" -o -name "*.p12" -o -name "*.pfx" \) -exec chmod 640 {} \; 2>/dev/null || true
+find "$INSTALL_DIR" -type f \( -name "*.db" -o -name "*.db-wal" -o -name "*.db-shm" -o -name "*.log" -o -name "*.log.*" \) -exec chmod 600 {} \; 2>/dev/null || true
+
+# Credentials, databases, backups, and local-plugin settings are private to the
+# service user. Directories must be writable for SQLite sidecars and atomic
+# config updates; the 0700 boundary prevents local disclosure.
+find "$CONF_DIR" "$STATE_DIR" -type d -exec chmod 700 {} \; 2>/dev/null || true
+find "$CONF_DIR" "$STATE_DIR" -type f -exec chmod 600 {} \; 2>/dev/null || true
+chmod 750 "$LOG_DIR"
+find "$LOG_DIR" -type f -exec chmod 600 {} \; 2>/dev/null || true
# Make main script executable
chmod 755 "$INSTALL_DIR/meshcore_bot.py"
@@ -594,13 +558,16 @@ import re
with open('$SERVICE_FILE', 'r') as f:
content = f.read()
content = content.replace('/usr/local/meshcore-bot', '$INSTALL_DIR')
+content = content.replace('/usr/local/etc/meshcore-bot', '$CONF_DIR')
+content = content.replace('/usr/local/var/lib/meshcore-bot', '$STATE_DIR')
content = content.replace('/usr/local/var/log/meshcore-bot', '$LOG_DIR')
+content = content.replace('__MESHCORE_SERVICE_USER__', '$SERVICE_USER')
with open('$LAUNCHD_DIR/$SERVICE_FILE', 'w') as f:
f.write(content)
"
else
# Fallback to sed (works on both macOS and Linux)
- sed "s|/usr/local/meshcore-bot|$INSTALL_DIR|g; s|/usr/local/var/log/meshcore-bot|$LOG_DIR|g" "$SERVICE_FILE" > "$LAUNCHD_DIR/$SERVICE_FILE"
+ sed "s|/usr/local/meshcore-bot|$INSTALL_DIR|g; s|/usr/local/etc/meshcore-bot|$CONF_DIR|g; s|/usr/local/var/lib/meshcore-bot|$STATE_DIR|g; s|/usr/local/var/log/meshcore-bot|$LOG_DIR|g; s|__MESHCORE_SERVICE_USER__|$SERVICE_USER|g" "$SERVICE_FILE" > "$LAUNCHD_DIR/$SERVICE_FILE"
fi
print_success "Copied and configured plist file to $LAUNCHD_DIR/"
fi
@@ -668,6 +635,15 @@ else
print_info "Note: The service is enabled but not started yet. You'll start it after configuration."
fi
+if [[ "$SERVICE_WAS_ACTIVE" == true ]]; then
+ print_info "Restarting the service because it was running before the upgrade"
+ if [[ "$IS_MACOS" == true ]]; then
+ launchctl load "$LAUNCHD_DIR/$SERVICE_FILE" 2>/dev/null || true
+ else
+ systemctl start "$SERVICE_NAME"
+ fi
+fi
+
if [[ "$UPGRADE_MODE" == true ]]; then
print_section "Upgrade Complete!"
echo ""
@@ -684,7 +660,7 @@ echo -e "${BLUE}📋 Next Steps${NC}"
echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
echo ""
echo -e "${CYAN}1. Configure the bot:${NC}"
-echo -e " ${YELLOW}sudo nano $INSTALL_DIR/config.ini${NC}"
+echo -e " ${YELLOW}sudo nano $CONFIG_FILE${NC}"
echo " Edit the configuration file with your bot settings, API keys, and device information"
echo ""
@@ -744,7 +720,8 @@ echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━
echo -e "${BLUE}📁 Important File Locations${NC}"
echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
echo ""
-echo -e " ${CYAN}Configuration file:${NC} ${YELLOW}$INSTALL_DIR/config.ini${NC}"
+echo -e " ${CYAN}Configuration file:${NC} ${YELLOW}$CONFIG_FILE${NC}"
+echo -e " ${CYAN}State directory:${NC} ${YELLOW}$STATE_DIR${NC}"
echo -e " ${CYAN}Log directory:${NC} ${YELLOW}$LOG_DIR${NC}"
echo -e " ${CYAN}Installation directory:${NC} ${YELLOW}$INSTALL_DIR${NC}"
if [[ "$IS_MACOS" == true ]]; then
@@ -792,4 +769,4 @@ if [[ "$UPGRADE_MODE" == true ]]; then
else
print_success "Installation complete! The bot is ready to configure and start."
fi
-echo ""
\ No newline at end of file
+echo ""
diff --git a/meshcore-bot.service b/meshcore-bot.service
index 5f01c8e..ba0a9a2 100644
--- a/meshcore-bot.service
+++ b/meshcore-bot.service
@@ -9,7 +9,7 @@ Type=simple
User=meshcore
Group=meshcore
WorkingDirectory=/opt/meshcore-bot
-ExecStart=/opt/meshcore-bot/venv/bin/python /opt/meshcore-bot/meshcore_bot.py
+ExecStart=/opt/meshcore-bot/venv/bin/python /opt/meshcore-bot/meshcore_bot.py --config /etc/meshcore-bot/config.ini
ExecReload=/bin/kill -HUP $MAINPID
Restart=always
RestartSec=10
@@ -26,7 +26,15 @@ NoNewPrivileges=true
PrivateTmp=true
ProtectSystem=strict
ProtectHome=true
-ReadWritePaths=/opt/meshcore-bot
+UMask=0077
+ConfigurationDirectory=meshcore-bot
+ConfigurationDirectoryMode=0700
+StateDirectory=meshcore-bot
+StateDirectoryMode=0700
+LogsDirectory=meshcore-bot
+LogsDirectoryMode=0750
+ReadWritePaths=/etc/meshcore-bot
+ReadWritePaths=/var/lib/meshcore-bot
ReadWritePaths=/var/log/meshcore-bot
# Resource limits
diff --git a/meshcore_bot.py b/meshcore_bot.py
index 1789bd7..18cacf7 100644
--- a/meshcore_bot.py
+++ b/meshcore_bot.py
@@ -142,6 +142,29 @@ def main():
except Exception as exc: # noqa: BLE001 - never block startup on the linter itself
print(f"Config validation skipped: {exc}", file=sys.stderr)
+ # A restore requested by the viewer is applied only here, before importing
+ # and constructing MeshCoreBot (which opens the DB and launches writers).
+ # The service manager must restart the bot and viewer as one unit.
+ from modules.database_restore import (
+ DatabaseRestoreError,
+ apply_pending_restores_from_config,
+ )
+
+ try:
+ restore_results = apply_pending_restores_from_config(args.config)
+ except DatabaseRestoreError as exc:
+ print(f"Error: pending database restore was not applied: {exc}", file=sys.stderr)
+ print(
+ "The active database was left unchanged. Correct or remove the pending restore "
+ "file before restarting.",
+ file=sys.stderr,
+ )
+ sys.exit(1)
+ for restore_result in restore_results:
+ recovery = restore_result.recovery_backup_path
+ recovery_note = f"; recovery backup: {recovery}" if recovery else ""
+ print(f"Applied pending database restore: {restore_result.database_path}{recovery_note}")
+
from modules.core import MeshCoreBot
bot = MeshCoreBot(config_file=args.config)
@@ -238,4 +261,3 @@ if __name__ == "__main__":
main()
-
diff --git a/modules/command_manager.py b/modules/command_manager.py
index 54b7dc7..41f9edd 100644
--- a/modules/command_manager.py
+++ b/modules/command_manager.py
@@ -10,6 +10,8 @@ import time
from dataclasses import dataclass
from datetime import datetime
from hashlib import sha256
+from importlib import resources
+from pathlib import Path
from typing import Any
from meshcore import EventType
@@ -955,24 +957,43 @@ class CommandManager:
if not self._is_channel_trigger_allowed(key, message):
return None
- file_path = self.bot.config.get('RandomLine', f'file.{key}', fallback='').strip()
- if not file_path:
+ configured_file_path = self.bot.config.get('RandomLine', f'file.{key}', fallback='').strip()
+ if not configured_file_path:
self.logger.warning(f"RandomLine matched '{key}' but missing config file.{key}")
return None
try:
- validated_path = validate_safe_path(file_path, allow_absolute=True)
+ validated_path = validate_safe_path(configured_file_path, allow_absolute=True)
except ValueError:
validated_path = None
if validated_path is None:
- self.logger.warning(f"RandomLine: unsafe or restricted path rejected for '{key}': {file_path}")
+ self.logger.warning(
+ f"RandomLine: unsafe or restricted path rejected for '{key}': {configured_file_path}"
+ )
return None
file_path = str(validated_path)
# Read usable lines
try:
- with open(file_path, encoding="utf-8") as f:
- lines = [ln.strip() for ln in f.readlines()]
+ if Path(file_path).is_file():
+ with open(file_path, encoding="utf-8") as f:
+ lines = [ln.strip() for ln in f.readlines()]
+ else:
+ # Shipped RandomLine defaults live in package data in wheels.
+ # Only map the documented data/randomlines path; arbitrary
+ # missing custom files must still fail instead of silently
+ # selecting a same-named bundled file.
+ normalized = configured_file_path.replace("\\", "/").lstrip("./")
+ marker = "data/randomlines/"
+ if not normalized.startswith(marker):
+ raise FileNotFoundError(file_path)
+ resource_name = normalized.removeprefix(marker)
+ if not resource_name or "/" in resource_name:
+ raise FileNotFoundError(file_path)
+ bundled = resources.files("data.randomlines").joinpath(resource_name)
+ if not bundled.is_file():
+ raise FileNotFoundError(file_path)
+ lines = [ln.strip() for ln in bundled.read_text(encoding="utf-8").splitlines()]
lines = [ln for ln in lines if ln] # drop blank lines
except Exception as e:
self.logger.error(f"RandomLine error reading {file_path} for '{key}': {e}", exc_info=True)
diff --git a/modules/database_restore.py b/modules/database_restore.py
new file mode 100644
index 0000000..6b413a2
--- /dev/null
+++ b/modules/database_restore.py
@@ -0,0 +1,451 @@
+"""Stage and apply SQLite restores without replacing a live database.
+
+The web viewer runs in a separate process from the bot and both processes open
+short-lived WAL connections. Replacing the database from an HTTP request is
+therefore unsafe: existing descriptors and WAL sidecars can continue referring
+to the old database. This module implements a two-phase restore instead:
+
+* the viewer validates and copies a backup to a sibling pending file; and
+* normal bot startup applies that pending file before any database manager,
+ scheduler, or viewer process is created.
+
+The startup caller is responsible for ensuring the service was restarted as a
+unit (that is, no separately managed viewer process is still using the file).
+"""
+
+from __future__ import annotations
+
+import configparser
+import os
+import shutil
+import sqlite3
+import stat
+import tempfile
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+
+from modules.db_migrations import MIGRATIONS
+
+DEFAULT_MAX_RESTORE_BYTES = 2 * 1024 * 1024 * 1024
+
+# Databases created before numbered migrations do not have ``schema_version``.
+# Requiring bot_metadata plus multiple tables from the original MeshCore schema
+# distinguishes those legitimate backups from an arbitrary SQLite file while
+# still allowing MigrationRunner to upgrade them after startup.
+LEGACY_SCHEMA_TABLES = {
+ "channel_operations",
+ "channels",
+ "feed_activity",
+ "feed_errors",
+ "feed_message_queue",
+ "feed_subscriptions",
+ "generic_cache",
+ "geocoding_cache",
+}
+
+
+class DatabaseRestoreError(RuntimeError):
+ """Raised when a restore cannot be safely staged or applied."""
+
+
+@dataclass(frozen=True)
+class RestoreResult:
+ """Description of a restore applied during startup."""
+
+ database_path: Path
+ recovery_backup_path: Path | None
+
+
+def pending_restore_path(database_path: str | Path) -> Path:
+ """Return the same-filesystem staging path for *database_path*."""
+
+ database = Path(database_path).resolve()
+ return database.with_name(f".{database.name}.restore-pending")
+
+
+def validate_restore_database(database_path: str | Path) -> None:
+ """Fail unless *database_path* is an intact, compatible MeshCore database."""
+
+ path = Path(database_path).resolve()
+ if not path.is_file():
+ raise DatabaseRestoreError(f"Restore database does not exist: {path}")
+
+ try:
+ with path.open("rb") as handle:
+ if handle.read(16) != b"SQLite format 3\x00":
+ raise DatabaseRestoreError("Restore file is not a SQLite database")
+
+ uri = f"{path.as_uri()}?mode=ro"
+ with sqlite3.connect(uri, uri=True, timeout=5.0) as conn:
+ integrity_row = conn.execute("PRAGMA integrity_check(1)").fetchone()
+ if not integrity_row or str(integrity_row[0]).lower() != "ok":
+ detail = str(integrity_row[0]) if integrity_row else "no result"
+ raise DatabaseRestoreError(f"SQLite integrity check failed: {detail}")
+
+ tables = {
+ str(row[0])
+ for row in conn.execute(
+ "SELECT name FROM sqlite_master WHERE type = 'table'"
+ ).fetchall()
+ }
+ if "bot_metadata" not in tables:
+ raise DatabaseRestoreError(
+ "Restore file is not a MeshCore Bot database; "
+ "missing table: bot_metadata"
+ )
+
+ if "schema_version" not in tables:
+ legacy_matches = sorted(LEGACY_SCHEMA_TABLES & tables)
+ if len(legacy_matches) < 2:
+ raise DatabaseRestoreError(
+ "Restore file is not a recognizable legacy MeshCore Bot "
+ "database; numbered migration history is absent and fewer "
+ "than two legacy schema tables were found"
+ )
+ else:
+ known_versions = {
+ int(version) for version, _description, _fn in MIGRATIONS
+ }
+ applied_versions = {
+ int(row[0])
+ for row in conn.execute(
+ "SELECT version FROM schema_version"
+ ).fetchall()
+ if row and row[0] is not None
+ }
+ unknown_versions = sorted(applied_versions - known_versions)
+ if unknown_versions:
+ raise DatabaseRestoreError(
+ "Restore database was created by a newer or incompatible "
+ "version; unknown migration version(s): "
+ f"{unknown_versions}"
+ )
+ except DatabaseRestoreError:
+ raise
+ except (OSError, sqlite3.Error, ValueError) as exc:
+ raise DatabaseRestoreError(f"Could not validate restore database: {exc}") from exc
+
+
+def _safe_database_mode(database_path: Path) -> int:
+ """Preserve owner/group access while never granting access to others."""
+
+ try:
+ current = stat.S_IMODE(database_path.stat().st_mode)
+ except OSError:
+ current = 0
+ return 0o600 | (current & 0o060)
+
+
+def _match_database_ownership(path: Path, database_path: Path) -> None:
+ os.chmod(path, _safe_database_mode(database_path))
+ try:
+ active_stat = database_path.stat()
+ os.chown(path, active_stat.st_uid, active_stat.st_gid)
+ except (AttributeError, FileNotFoundError, PermissionError):
+ # chown is unavailable on Windows and an unprivileged service can only
+ # create files as itself. In both cases the creator is the safe owner.
+ pass
+
+
+def _fsync_directory(directory: Path) -> None:
+ if os.name == "nt":
+ return
+ descriptor = os.open(str(directory), os.O_RDONLY)
+ try:
+ os.fsync(descriptor)
+ finally:
+ os.close(descriptor)
+
+
+def stage_database_restore(
+ source_path: str | Path,
+ database_path: str | Path,
+ *,
+ max_bytes: int = DEFAULT_MAX_RESTORE_BYTES,
+) -> Path:
+ """Validate and stage *source_path* for application on the next bot startup."""
+
+ if max_bytes <= 0:
+ raise DatabaseRestoreError("Restore size limit must be positive")
+
+ source = Path(source_path).resolve()
+ database = Path(database_path).resolve()
+ pending = pending_restore_path(database)
+ if source in {database, pending}:
+ raise DatabaseRestoreError("Restore source must be a separate backup file")
+
+ validate_restore_database(source)
+ database.parent.mkdir(parents=True, exist_ok=True)
+ temp_path: Path | None = None
+ try:
+ descriptor, temp_name = tempfile.mkstemp(
+ dir=database.parent,
+ prefix=f".{database.name}.restore-",
+ suffix=".tmp",
+ )
+ os.close(descriptor)
+ temp_path = Path(temp_name)
+ source_uri = f"{source.as_uri()}?mode=ro"
+ with sqlite3.connect(source_uri, uri=True, timeout=30.0) as source_conn:
+ page_count = int(source_conn.execute("PRAGMA page_count").fetchone()[0])
+ page_size = int(source_conn.execute("PRAGMA page_size").fetchone()[0])
+ if page_count * page_size > max_bytes:
+ raise DatabaseRestoreError(
+ f"Restore database exceeds the {max_bytes}-byte safety limit"
+ )
+ with sqlite3.connect(str(temp_path), timeout=30.0) as destination_conn:
+ # SQLite's online backup API produces a self-contained snapshot
+ # and includes committed WAL content; a raw file copy does not.
+ source_conn.backup(destination_conn)
+ destination_conn.commit()
+
+ if temp_path.stat().st_size > max_bytes:
+ raise DatabaseRestoreError(
+ f"Restore database exceeds the {max_bytes}-byte safety limit"
+ )
+ with temp_path.open("rb") as temp_handle:
+ os.fsync(temp_handle.fileno())
+
+ _match_database_ownership(temp_path, database)
+ validate_restore_database(temp_path)
+ os.replace(temp_path, pending)
+ temp_path = None
+ _fsync_directory(database.parent)
+ return pending
+ except DatabaseRestoreError:
+ raise
+ except OSError as exc:
+ raise DatabaseRestoreError(f"Could not stage restore database: {exc}") from exc
+ finally:
+ if temp_path is not None:
+ try:
+ temp_path.unlink()
+ except FileNotFoundError:
+ pass
+
+
+def _create_recovery_backup(database: Path) -> Path:
+ timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S%fZ")
+ recovery = database.with_name(f"{database.stem}.pre-restore-{timestamp}.db")
+ descriptor, temp_name = tempfile.mkstemp(
+ dir=database.parent,
+ prefix=f".{database.name}.pre-restore-",
+ suffix=".tmp",
+ )
+ os.close(descriptor)
+ temp_path = Path(temp_name)
+ try:
+ with sqlite3.connect(str(database), timeout=30.0) as source_conn:
+ with sqlite3.connect(str(temp_path), timeout=30.0) as destination_conn:
+ source_conn.backup(destination_conn)
+ destination_conn.commit()
+ _match_database_ownership(temp_path, database)
+ validate_restore_database(temp_path)
+ os.replace(temp_path, recovery)
+ _fsync_directory(database.parent)
+ return recovery
+ except Exception:
+ try:
+ temp_path.unlink()
+ except FileNotFoundError:
+ pass
+ raise
+
+
+def _rollback_from_recovery(recovery: Path, database: Path) -> None:
+ """Atomically restore *database* from the already verified recovery copy."""
+
+ descriptor, temp_name = tempfile.mkstemp(
+ dir=database.parent,
+ prefix=f".{database.name}.rollback-",
+ suffix=".tmp",
+ )
+ os.close(descriptor)
+ temp_path = Path(temp_name)
+ try:
+ shutil.copy2(recovery, temp_path)
+ with temp_path.open("rb") as handle:
+ os.fsync(handle.fileno())
+ _match_database_ownership(temp_path, database)
+ for suffix in ("-wal", "-shm"):
+ try:
+ Path(f"{database}{suffix}").unlink()
+ except FileNotFoundError:
+ pass
+ os.replace(temp_path, database)
+ _fsync_directory(database.parent)
+ validate_restore_database(database)
+ finally:
+ try:
+ temp_path.unlink()
+ except FileNotFoundError:
+ pass
+
+
+def apply_pending_database_restore(database_path: str | Path) -> RestoreResult | None:
+ """Apply a staged restore before any process opens *database_path*."""
+
+ database = Path(database_path).resolve()
+ pending = pending_restore_path(database)
+ if not pending.exists():
+ return None
+
+ validate_restore_database(pending)
+ database.parent.mkdir(parents=True, exist_ok=True)
+ recovery: Path | None = None
+ replaced = False
+ try:
+ if database.exists():
+ recovery = _create_recovery_backup(database)
+ active_stat = database.stat()
+ else:
+ active_stat = None
+
+ _match_database_ownership(pending, database)
+ for suffix in ("-wal", "-shm"):
+ sidecar = Path(f"{database}{suffix}")
+ try:
+ sidecar.unlink()
+ except FileNotFoundError:
+ pass
+
+ os.replace(pending, database)
+ replaced = True
+ if active_stat is not None:
+ os.chmod(database, 0o600 | (stat.S_IMODE(active_stat.st_mode) & 0o060))
+ try:
+ os.chown(database, active_stat.st_uid, active_stat.st_gid)
+ except (AttributeError, PermissionError):
+ pass
+ else:
+ os.chmod(database, 0o600)
+ _fsync_directory(database.parent)
+ validate_restore_database(database)
+ return RestoreResult(database_path=database, recovery_backup_path=recovery)
+ except (DatabaseRestoreError, OSError, sqlite3.Error) as exc:
+ if replaced:
+ try:
+ # Preserve the staged candidate for diagnosis/retry before
+ # putting the previous active database back. Without this,
+ # a failure after os.replace consumes one member of a
+ # coordinated Bot/Viewer restore set.
+ os.replace(database, pending)
+ _fsync_directory(database.parent)
+ if recovery is not None:
+ _rollback_from_recovery(recovery, database)
+ except Exception as rollback_exc:
+ raise DatabaseRestoreError(
+ "Pending restore failed after replacement and automatic recovery also "
+ f"failed. Recovery backup: {recovery}. Restore error: {exc}; "
+ f"rollback error: {rollback_exc}"
+ ) from exc
+ if recovery is not None:
+ raise DatabaseRestoreError(
+ "Pending restore failed; the candidate was re-queued and the "
+ f"original database was restored from {recovery}: {exc}"
+ ) from exc
+ raise DatabaseRestoreError(
+ "Pending restore failed; the candidate was re-queued and no active "
+ f"database had existed: {exc}"
+ ) from exc
+ if isinstance(exc, DatabaseRestoreError):
+ raise
+ raise DatabaseRestoreError(f"Could not apply pending database restore: {exc}") from exc
+
+
+def _configured_database_paths(config_path: str | Path) -> list[Path]:
+ path = Path(config_path).resolve()
+ if not path.exists():
+ return []
+
+ config = configparser.ConfigParser()
+ try:
+ loaded = config.read(path, encoding="utf-8")
+ except configparser.Error as exc:
+ raise DatabaseRestoreError(f"Could not read restore configuration: {exc}") from exc
+ if not loaded:
+ return []
+
+ base = path.parent
+ local_dir = Path(config.get("Bot", "local_dir_path", fallback="local"))
+ if not local_dir.is_absolute():
+ local_dir = base / local_dir
+ local_config = local_dir / "config.ini"
+ if local_config.exists():
+ try:
+ config.read(local_config, encoding="utf-8")
+ except configparser.Error as exc:
+ raise DatabaseRestoreError(f"Could not read local restore configuration: {exc}") from exc
+
+ raw_paths = [config.get("Bot", "db_path", fallback="meshcore_bot.db")]
+ viewer_path = config.get("Web_Viewer", "db_path", fallback="").strip()
+ if viewer_path:
+ raw_paths.append(viewer_path)
+
+ resolved: list[Path] = []
+ for raw in raw_paths:
+ if raw == ":memory:":
+ continue
+ candidate = Path(raw)
+ if not candidate.is_absolute():
+ candidate = base / candidate
+ candidate = candidate.resolve()
+ if candidate not in resolved:
+ resolved.append(candidate)
+ return resolved
+
+
+def apply_pending_restores_from_config(config_path: str | Path) -> list[RestoreResult]:
+ """Apply pending bot/viewer restores before normal startup opens either DB."""
+
+ database_paths = _configured_database_paths(config_path)
+ pending_paths = [
+ pending_restore_path(path)
+ for path in database_paths
+ if pending_restore_path(path).exists()
+ ]
+ # Validate the entire set before changing any active database. This avoids
+ # applying the bot restore only to discover that a split viewer restore is
+ # corrupt or incompatible.
+ for pending in pending_paths:
+ validate_restore_database(pending)
+
+ results: list[RestoreResult] = []
+ try:
+ for database_path in database_paths:
+ result = apply_pending_database_restore(database_path)
+ if result is not None:
+ results.append(result)
+ except Exception as exc:
+ rollback_errors: list[str] = []
+ for result in reversed(results):
+ database = result.database_path
+ pending = pending_restore_path(database)
+ try:
+ # Preserve the already-applied candidate so the complete restore
+ # set can be retried after the failure is corrected.
+ for suffix in ("-wal", "-shm"):
+ try:
+ Path(f"{database}{suffix}").unlink()
+ except FileNotFoundError:
+ pass
+ os.replace(database, pending)
+ _fsync_directory(database.parent)
+ validate_restore_database(pending)
+ if result.recovery_backup_path is not None:
+ _rollback_from_recovery(result.recovery_backup_path, database)
+ except Exception as rollback_exc:
+ rollback_errors.append(f"{database}: {rollback_exc}")
+
+ if rollback_errors:
+ raise DatabaseRestoreError(
+ "A configured restore failed and rollback was incomplete. "
+ f"Restore error: {exc}. Rollback error(s): "
+ + "; ".join(rollback_errors)
+ ) from exc
+ raise DatabaseRestoreError(
+ "A configured restore failed; every previously applied database was "
+ f"rolled back and re-queued: {exc}"
+ ) from exc
+ return results
diff --git a/modules/db_migrations.py b/modules/db_migrations.py
index ccd1443..4588509 100644
--- a/modules/db_migrations.py
+++ b/modules/db_migrations.py
@@ -530,6 +530,28 @@ def _m0014_observed_paths_multibyte_covering_index(cursor: sqlite3.Cursor) -> No
)
+def _m0015_channel_operations_claimed_at(cursor: sqlite3.Cursor) -> None:
+ """Record when a queued hardware/config operation is durably claimed.
+
+ A ``processing`` row is deliberately not auto-requeued: after a process
+ crash the device may already have applied the operation, so retrying it
+ automatically could execute a non-idempotent command twice. ``claimed_at``
+ gives operators enough information to diagnose and explicitly resolve such
+ an ambiguous operation.
+ """
+ if _table_exists(cursor, "channel_operations"):
+ _add_column(cursor, "channel_operations", "claimed_at", "TIMESTAMP")
+
+
+def _m0016_channel_operations_claim_owner(cursor: sqlite3.Cursor) -> None:
+ """Persist enough local-process identity to recover only provably dead claims."""
+ if not _table_exists(cursor, "channel_operations"):
+ return
+ _add_column(cursor, "channel_operations", "claim_owner_host", "TEXT")
+ _add_column(cursor, "channel_operations", "claim_owner_pid", "INTEGER")
+ _add_column(cursor, "channel_operations", "claim_owner_boot_id", "TEXT")
+
+
# ---------------------------------------------------------------------------
# Migration registry — append new entries here, never remove or reorder.
# ---------------------------------------------------------------------------
@@ -551,6 +573,8 @@ MIGRATIONS: list[MigrationEntry] = [
(12, "purging_log: add details column", _m0012_purging_log_details_column),
(13, "observed_paths: advert covering index for contacts page", _m0013_observed_paths_advert_covering_index),
(14, "observed_paths: multibyte covering index for mesh graph", _m0014_observed_paths_multibyte_covering_index),
+ (15, "channel_operations: claimed_at", _m0015_channel_operations_claimed_at),
+ (16, "channel_operations: claim owner identity", _m0016_channel_operations_claim_owner),
]
diff --git a/modules/i18n.py b/modules/i18n.py
index 0ac8999..b8a1c22 100644
--- a/modules/i18n.py
+++ b/modules/i18n.py
@@ -5,6 +5,7 @@ Provides translation functionality for bot commands and responses
"""
import json
+from importlib import resources
from pathlib import Path
from typing import Any
@@ -108,12 +109,20 @@ class Translator:
Dictionary of translations, empty dict if file not found
"""
file_path = Path(self.translation_path) / f"{lang}.json"
- if not file_path.exists():
- return {}
-
try:
- with open(file_path, encoding='utf-8') as f:
- return json.load(f)
+ # An explicitly configured filesystem catalog always wins. The
+ # package fallback makes the defaults work from an installed wheel
+ # (where ``translations/`` is not relative to the current cwd).
+ if file_path.is_file():
+ with open(file_path, encoding='utf-8') as f:
+ return json.load(f)
+
+ if not self._uses_bundled_defaults():
+ return {}
+ bundled = resources.files("translations").joinpath(f"{lang}.json")
+ if not bundled.is_file():
+ return {}
+ return json.loads(bundled.read_text(encoding="utf-8"))
except json.JSONDecodeError as e:
print(f"Error parsing translation file {file_path}: {e}")
return {}
@@ -179,7 +188,19 @@ class Translator:
if trans_path.exists():
for file in trans_path.glob('*.json'):
languages.append(file.stem)
- return sorted(languages)
+ if self._uses_bundled_defaults():
+ try:
+ for resource in resources.files("translations").iterdir():
+ if resource.is_file() and resource.name.endswith(".json"):
+ languages.append(resource.name[:-5])
+ except (ModuleNotFoundError, OSError):
+ pass
+ return sorted(set(languages))
+
+ def _uses_bundled_defaults(self) -> bool:
+ """Return whether ``translation_path`` names the shipped catalog."""
+ normalized = str(self.translation_path).replace("\\", "/").rstrip("/")
+ return normalized == "translations"
def get_value(self, key: str) -> Any:
"""
@@ -210,4 +231,3 @@ class Translator:
break
return value
-
diff --git a/modules/scheduler.py b/modules/scheduler.py
index ddad033..ab10a90 100644
--- a/modules/scheduler.py
+++ b/modules/scheduler.py
@@ -9,9 +9,11 @@ import datetime
import hashlib
import json
import os
+import socket
import sqlite3
import threading
import time
+import uuid
from pathlib import Path
from typing import Any, Optional
@@ -26,7 +28,24 @@ from .scheduled_message_cron import (
parse_scheduled_message_value,
)
from .security_utils import validate_external_url
-from .utils import decode_escape_sequences, format_keyword_response_with_placeholders, get_config_timezone
+from .utils import (
+ decode_escape_sequences,
+ format_keyword_response_with_placeholders,
+ get_config_timezone,
+)
+
+_CHANNEL_OPERATION_TYPES = ('add', 'remove')
+_RADIO_OPERATION_TYPES = (
+ 'radio_reboot',
+ 'radio_connect',
+ 'radio_disconnect',
+ 'firmware_read',
+ 'firmware_write',
+ 'radio_params_read',
+ 'radio_params_write',
+ 'radio_advert',
+)
+_CONFIG_OPERATION_TYPES = ('config_reload',)
class MessageScheduler:
@@ -35,6 +54,12 @@ class MessageScheduler:
def __init__(self, bot):
self.bot = bot
self.logger = bot.logger
+ self._claim_owner_host = socket.gethostname()
+ self._claim_owner_pid = os.getpid()
+ # Unique to this scheduler/process start; retained with the PID to make
+ # ownership auditable and prevent one scheduler instance finalizing a
+ # claim made by another instance in the same process.
+ self._claim_owner_boot_id = uuid.uuid4().hex
self.scheduled_messages = {}
self.scheduler_thread = None
self._apscheduler: Optional[BackgroundScheduler] = None
@@ -48,6 +73,10 @@ class MessageScheduler:
self.last_db_backup_run = 0
self.last_log_rotation_check_time = 0
self.maintenance = MaintenanceRunner(bot, get_current_time=self.get_current_time)
+ db_manager = getattr(bot, 'db_manager', None)
+ db_path = getattr(db_manager, 'db_path', None)
+ if db_manager is not None and isinstance(db_path, (str, os.PathLike)):
+ self._recover_interrupted_operations()
def get_current_time(self):
"""Get current time in configured timezone"""
@@ -835,106 +864,237 @@ class MessageScheduler:
raise RuntimeError(f"send_advert failed: {reason}")
self.logger.info("Interval-based flood advert sent successfully")
- async def _process_channel_operations(self):
- """Process pending channel operations from the web viewer"""
+ def _claim_operation(self, operation_types: tuple[str, ...]) -> Optional[dict[str, Any]]:
+ """Atomically claim the oldest pending operation in one serialized group.
+
+ ``BEGIN IMMEDIATE`` prevents two scheduler ticks (or two bot processes)
+ from selecting the same pending row. A group permits only one
+ ``processing`` operation at a time, preserving device-operation order
+ when an earlier command is slow.
+
+ Processing rows are never auto-requeued. If a previous same-host owner
+ is provably dead at startup, its claim becomes ``interrupted`` so later
+ work can proceed without replaying the ambiguous action. Live or
+ unprovable owners continue to block the group conservatively.
+ """
+ if not operation_types:
+ raise ValueError("operation_types must not be empty")
+
+ placeholders = ', '.join('?' for _ in operation_types)
+ with self.bot.db_manager.connection() as conn:
+ conn.row_factory = sqlite3.Row
+ cursor = conn.cursor()
+ cursor.execute('BEGIN IMMEDIATE')
+
+ cursor.execute(
+ f'''SELECT id
+ FROM channel_operations
+ WHERE status = 'processing'
+ AND operation_type IN ({placeholders})
+ LIMIT 1''',
+ operation_types,
+ )
+ if cursor.fetchone() is not None:
+ conn.commit()
+ return None
+
+ cursor.execute(
+ f'''SELECT *
+ FROM channel_operations
+ WHERE status = 'pending'
+ AND operation_type IN ({placeholders})
+ ORDER BY created_at ASC, id ASC
+ LIMIT 1''',
+ operation_types,
+ )
+ row = cursor.fetchone()
+ if row is None:
+ conn.commit()
+ return None
+
+ cursor.execute(
+ '''UPDATE channel_operations
+ SET status = 'processing',
+ claimed_at = CURRENT_TIMESTAMP,
+ claim_owner_host = ?,
+ claim_owner_pid = ?,
+ claim_owner_boot_id = ?,
+ processed_at = NULL,
+ error_message = NULL,
+ result_data = NULL
+ WHERE id = ? AND status = 'pending' ''',
+ (
+ self._claim_owner_host,
+ self._claim_owner_pid,
+ self._claim_owner_boot_id,
+ row['id'],
+ ),
+ )
+ if cursor.rowcount != 1:
+ conn.rollback()
+ return None
+
+ conn.commit()
+ claimed = dict(row)
+ claimed['status'] = 'processing'
+ return claimed
+
+ @staticmethod
+ def _is_local_pid_alive(pid: int) -> Optional[bool]:
+ """Return PID liveness, or ``None`` when it cannot be proven either way."""
+ if not isinstance(pid, int) or pid <= 0:
+ return None
+ try:
+ os.kill(pid, 0)
+ except ProcessLookupError:
+ return False
+ except PermissionError:
+ # The kernel found the process but this user cannot signal it.
+ return True
+ except OSError:
+ return None
+ return True
+
+ def _recover_interrupted_operations(self) -> int:
+ """Resolve only startup claims provably abandoned by a local process.
+
+ Legacy ownerless rows may be interrupted. An owned row is interrupted
+ only when it belongs to this host and its PID is provably dead. Live
+ same-host owners, other hosts, and incomplete/unknown identities stay
+ blocked conservatively. This method is only invoked during scheduler
+ construction, never by polling or a timer.
+ """
+ explanation = (
+ "Bot restarted while this operation was processing; the action may "
+ "already have reached the device. Automatic retry is disabled. "
+ "Verify device state before submitting another operation."
+ )
try:
- # Get pending operations
with self.bot.db_manager.connection() as conn:
conn.row_factory = sqlite3.Row
cursor = conn.cursor()
+ cursor.execute('BEGIN IMMEDIATE')
+ cursor.execute(
+ '''SELECT id, claim_owner_host, claim_owner_pid,
+ claim_owner_boot_id
+ FROM channel_operations
+ WHERE status = 'processing' ''',
+ )
+ rows = cursor.fetchall()
+ recovered = 0
+ blocked = 0
+ for row in rows:
+ owner_host = row['claim_owner_host']
+ owner_pid = row['claim_owner_pid']
+ owner_boot_id = row['claim_owner_boot_id']
+ legacy_ownerless = (
+ owner_host is None
+ and owner_pid is None
+ and owner_boot_id is None
+ )
+ same_host_dead = (
+ owner_host == self._claim_owner_host
+ and self._is_local_pid_alive(owner_pid) is False
+ )
+ if not legacy_ownerless and not same_host_dead:
+ blocked += 1
+ continue
- cursor.execute('''
- SELECT id, operation_type, channel_idx, channel_name, channel_key_hex
- FROM channel_operations
- WHERE status = 'pending'
- AND operation_type IN ('add', 'remove')
- ORDER BY created_at ASC
- LIMIT 10
- ''')
+ cursor.execute(
+ '''UPDATE channel_operations
+ SET status = 'interrupted',
+ processed_at = CURRENT_TIMESTAMP,
+ error_message = ?
+ WHERE id = ? AND status = 'processing' ''',
+ (explanation, row['id']),
+ )
+ recovered += cursor.rowcount
+ conn.commit()
+ except sqlite3.Error as exc:
+ self.logger.exception(
+ "Could not recover interrupted channel/radio operations at startup: %s",
+ exc,
+ )
+ return 0
- operations = cursor.fetchall()
+ if recovered:
+ self.logger.warning(
+ "Marked %s operation(s) interrupted after restart; verify device state before retrying",
+ recovered,
+ )
+ if blocked:
+ self.logger.warning(
+ "Left %s processing operation(s) blocked because their owner is live or cannot be proven dead",
+ blocked,
+ )
+ return recovered
- if not operations:
- return
+ def _finish_claimed_operation(
+ self,
+ op_id: int,
+ *,
+ success: bool,
+ result_payload: Optional[dict[str, Any]] = None,
+ error_message: Optional[str] = None,
+ ) -> None:
+ """Finalize a claim without overwriting externally resolved state."""
+ with self.bot.db_manager.connection() as conn:
+ cursor = conn.cursor()
+ if success:
+ cursor.execute(
+ '''UPDATE channel_operations
+ SET status = 'completed',
+ processed_at = CURRENT_TIMESTAMP,
+ result_data = ?,
+ error_message = NULL
+ WHERE id = ? AND status = 'processing'
+ AND claim_owner_host = ?
+ AND claim_owner_pid = ?
+ AND claim_owner_boot_id = ? ''',
+ (
+ json.dumps(result_payload or {'success': True}),
+ op_id,
+ self._claim_owner_host,
+ self._claim_owner_pid,
+ self._claim_owner_boot_id,
+ ),
+ )
+ else:
+ cursor.execute(
+ '''UPDATE channel_operations
+ SET status = 'failed',
+ processed_at = CURRENT_TIMESTAMP,
+ error_message = ?
+ WHERE id = ? AND status = 'processing'
+ AND claim_owner_host = ?
+ AND claim_owner_pid = ?
+ AND claim_owner_boot_id = ? ''',
+ (
+ error_message or 'Unknown error',
+ op_id,
+ self._claim_owner_host,
+ self._claim_owner_pid,
+ self._claim_owner_boot_id,
+ ),
+ )
+ if cursor.rowcount != 1:
+ self.logger.warning(
+ "Operation %s was no longer processing when finalization was attempted",
+ op_id,
+ )
+ conn.commit()
- self.logger.info(f"Processing {len(operations)} pending channel operation(s)")
-
- for op in operations:
- op_id = op['id']
- op_type = op['operation_type']
- channel_idx = op['channel_idx']
- channel_name = op['channel_name']
- channel_key_hex = op['channel_key_hex']
-
- try:
- success = False
- error_msg = None
-
- if op_type == 'add':
- # Add channel
- if channel_key_hex:
- # Custom channel with key
- channel_secret = bytes.fromhex(channel_key_hex)
- success = await self.bot.channel_manager.add_channel(
- channel_idx, channel_name, channel_secret=channel_secret
- )
- else:
- # Hashtag channel (firmware generates key)
- success = await self.bot.channel_manager.add_channel(
- channel_idx, channel_name
- )
-
- if success:
- self.logger.info(f"Successfully processed channel add operation: {channel_name} at index {channel_idx}")
- else:
- error_msg = "Failed to add channel"
-
- elif op_type == 'remove':
- # Remove channel
- success = await self.bot.channel_manager.remove_channel(channel_idx)
-
- if success:
- self.logger.info(f"Successfully processed channel remove operation: index {channel_idx}")
- else:
- error_msg = "Failed to remove channel"
-
- # Update operation status
- with self.bot.db_manager.connection() as conn:
- cursor = conn.cursor()
- if success:
- cursor.execute('''
- UPDATE channel_operations
- SET status = 'completed',
- processed_at = CURRENT_TIMESTAMP,
- result_data = ?
- WHERE id = ?
- ''', (json.dumps({'success': True}), op_id))
- else:
- cursor.execute('''
- UPDATE channel_operations
- SET status = 'failed',
- processed_at = CURRENT_TIMESTAMP,
- error_message = ?
- WHERE id = ?
- ''', (error_msg or 'Unknown error', op_id))
- conn.commit()
-
- except Exception as e:
- self.logger.error(f"Error processing channel operation {op_id}: {e}")
- # Mark as failed
- try:
- with self.bot.db_manager.connection() as conn:
- cursor = conn.cursor()
- cursor.execute('''
- UPDATE channel_operations
- SET status = 'failed',
- processed_at = CURRENT_TIMESTAMP,
- error_message = ?
- WHERE id = ?
- ''', (str(e), op_id))
- conn.commit()
- except Exception as update_error:
- self.logger.error(f"Error updating operation status: {update_error}")
+ async def _process_channel_operations(self):
+ """Process pending channel operations from the web viewer"""
+ try:
+ # Preserve the previous per-tick batch capacity, but claim only one
+ # row immediately before executing it. The next claim cannot
+ # succeed until the previous row has reached a terminal state.
+ for _ in range(10):
+ op = self._claim_operation(_CHANNEL_OPERATION_TYPES)
+ if not op:
+ return
+ await self._execute_claimed_channel_operation(op)
except Exception as e:
db_path = getattr(self.bot.db_manager, 'db_path', 'unknown')
@@ -950,27 +1110,60 @@ class MessageScheduler:
else:
self.logger.error(f"Database path: {db_path_str}")
+ async def _execute_claimed_channel_operation(self, op: dict[str, Any]) -> None:
+ """Execute and finalize one already-claimed channel operation."""
+ op_id = op['id']
+ op_type = op['operation_type']
+ channel_idx = op['channel_idx']
+ channel_name = op['channel_name']
+ channel_key_hex = op['channel_key_hex']
+ self.logger.info("Processing claimed channel operation %s: %s", op_id, op_type)
+
+ try:
+ success = False
+ error_msg = None
+
+ if op_type == 'add':
+ if channel_key_hex:
+ channel_secret = bytes.fromhex(channel_key_hex)
+ success = await self.bot.channel_manager.add_channel(
+ channel_idx, channel_name, channel_secret=channel_secret
+ )
+ else:
+ success = await self.bot.channel_manager.add_channel(
+ channel_idx, channel_name
+ )
+
+ if success:
+ self.logger.info(f"Successfully processed channel add operation: {channel_name} at index {channel_idx}")
+ else:
+ error_msg = "Failed to add channel"
+
+ elif op_type == 'remove':
+ success = await self.bot.channel_manager.remove_channel(channel_idx)
+
+ if success:
+ self.logger.info(f"Successfully processed channel remove operation: index {channel_idx}")
+ else:
+ error_msg = "Failed to remove channel"
+
+ self._finish_claimed_operation(
+ op_id,
+ success=success,
+ error_message=error_msg,
+ )
+
+ except Exception as e:
+ self.logger.error(f"Error processing channel operation {op_id}: {e}")
+ try:
+ self._finish_claimed_operation(op_id, success=False, error_message=str(e))
+ except Exception as update_error:
+ self.logger.error(f"Error updating operation status: {update_error}")
+
async def _process_radio_operations(self):
"""Process pending radio connect/disconnect/reboot/firmware operations from the web viewer."""
try:
- with self.bot.db_manager.connection() as conn:
- conn.row_factory = sqlite3.Row
- cursor = conn.cursor()
- cursor.execute('''
- SELECT id, operation_type, payload_data
- FROM channel_operations
- WHERE status = 'pending'
- AND operation_type IN (
- 'radio_reboot', 'radio_connect', 'radio_disconnect',
- 'firmware_read', 'firmware_write',
- 'radio_params_read', 'radio_params_write',
- 'radio_advert'
- )
- ORDER BY created_at ASC
- LIMIT 1
- ''')
- op = cursor.fetchone()
-
+ op = self._claim_operation(_RADIO_OPERATION_TYPES)
if not op:
return
@@ -1002,41 +1195,19 @@ class MessageScheduler:
else:
success = False
- with self.bot.db_manager.connection() as conn:
- cursor = conn.cursor()
- if success:
- cursor.execute('''
- UPDATE channel_operations
- SET status = 'completed',
- processed_at = CURRENT_TIMESTAMP,
- result_data = ?
- WHERE id = ?
- ''', (json.dumps(result_payload), op_id))
- else:
- error_msg = result_payload.get('error', 'Radio operation returned False') \
- if isinstance(result_payload, dict) else 'Radio operation returned False'
- cursor.execute('''
- UPDATE channel_operations
- SET status = 'failed',
- processed_at = CURRENT_TIMESTAMP,
- error_message = ?
- WHERE id = ?
- ''', (error_msg, op_id))
- conn.commit()
+ error_msg = result_payload.get('error', 'Radio operation returned False') \
+ if isinstance(result_payload, dict) else 'Radio operation returned False'
+ self._finish_claimed_operation(
+ op_id,
+ success=success,
+ result_payload=result_payload,
+ error_message=error_msg,
+ )
except Exception as e:
self.logger.error(f"Error executing radio operation {op_id}: {e}")
try:
- with self.bot.db_manager.connection() as conn:
- cursor = conn.cursor()
- cursor.execute('''
- UPDATE channel_operations
- SET status = 'failed',
- processed_at = CURRENT_TIMESTAMP,
- error_message = ?
- WHERE id = ?
- ''', (str(e), op_id))
- conn.commit()
+ self._finish_claimed_operation(op_id, success=False, error_message=str(e))
except Exception as update_error:
self.logger.error(f"Error updating radio operation status: {update_error}")
@@ -1052,17 +1223,7 @@ class MessageScheduler:
start/stop is not handled by reload_config and still needs a restart.
"""
try:
- with self.bot.db_manager.connection() as conn:
- conn.row_factory = sqlite3.Row
- cursor = conn.cursor()
- cursor.execute('''
- SELECT id FROM channel_operations
- WHERE status = 'pending' AND operation_type = 'config_reload'
- ORDER BY created_at ASC
- LIMIT 1
- ''')
- op = cursor.fetchone()
-
+ op = self._claim_operation(_CONFIG_OPERATION_TYPES)
if not op:
return
@@ -1075,25 +1236,12 @@ class MessageScheduler:
success, message = False, str(e)
self.logger.exception("Error during config reload")
- with self.bot.db_manager.connection() as conn:
- cursor = conn.cursor()
- if success:
- cursor.execute('''
- UPDATE channel_operations
- SET status = 'completed',
- processed_at = CURRENT_TIMESTAMP,
- result_data = ?
- WHERE id = ?
- ''', (json.dumps({'success': True, 'message': message}), op_id))
- else:
- cursor.execute('''
- UPDATE channel_operations
- SET status = 'failed',
- processed_at = CURRENT_TIMESTAMP,
- error_message = ?
- WHERE id = ?
- ''', (message, op_id))
- conn.commit()
+ self._finish_claimed_operation(
+ op_id,
+ success=success,
+ result_payload={'success': True, 'message': message},
+ error_message=message,
+ )
self.logger.info("Config reload %s: %s", 'succeeded' if success else 'failed', message)
@@ -1645,4 +1793,3 @@ class MessageScheduler:
self.bot.logger.error(f"Failed to send radio-offline alert email: {e}")
# ── Maintenance helpers ──────────────────────────────────────────────────
-
diff --git a/modules/web_viewer/app.py b/modules/web_viewer/app.py
index c6c763c..cc193f6 100644
--- a/modules/web_viewer/app.py
+++ b/modules/web_viewer/app.py
@@ -9,6 +9,7 @@ import json
import logging
import os
import re
+import secrets
import sqlite3
import sys
import threading
@@ -31,6 +32,7 @@ from flask import (
Response,
abort,
current_app,
+ g,
jsonify,
make_response,
redirect,
@@ -42,6 +44,11 @@ from flask import (
)
from flask_socketio import SocketIO, disconnect, emit
+from modules.database_restore import (
+ DEFAULT_MAX_RESTORE_BYTES,
+ DatabaseRestoreError,
+ stage_database_restore,
+)
from modules.ini_writer import update_ini_values
from modules.security_utils import (
VALID_JOURNAL_MODES,
@@ -620,6 +627,11 @@ class BotDataViewer:
'/site.webmanifest', '/favicon.ico',
])
+ @self.app.before_request
+ def create_csp_nonce():
+ """Create a per-response nonce for templates migrated off inline-script CSP."""
+ g.csp_nonce = secrets.token_urlsafe(24)
+
@self.app.before_request
def require_auth():
if not self.web_viewer_password:
@@ -664,10 +676,33 @@ class BotDataViewer:
response.headers['Referrer-Policy'] = 'strict-origin-when-cross-origin'
# Allow CDNs used by templates (base.html, login.html, mesh.html).
# Without these hosts, browsers block external CSS/JS/fonts (not CSRF).
+ # The highest-risk admin screens have migrated their inline handlers
+ # and authorize their remaining template scripts with a per-request
+ # nonce. Other legacy screens retain unsafe-inline until their inline
+ # scripts/handlers are migrated, rather than silently breaking them.
+ nonce_hardened_endpoints = {
+ 'index',
+ 'feeds',
+ 'config_page',
+ 'radio',
+ 'realtime',
+ 'contacts',
+ 'stats',
+ 'plugins_page',
+ 'greeter',
+ 'logs',
+ 'multibyte_rollout',
+ 'mesh',
+ 'api_explorer',
+ }
+ if request.endpoint in nonce_hardened_endpoints:
+ script_source = f"script-src 'self' 'nonce-{g.csp_nonce}' "
+ else:
+ script_source = "script-src 'self' 'unsafe-inline' "
response.headers['Content-Security-Policy'] = (
"default-src 'self'; "
- "script-src 'self' 'unsafe-inline' "
- "https://cdn.jsdelivr.net https://cdnjs.cloudflare.com https://unpkg.com; "
+ + script_source
+ + "https://cdn.jsdelivr.net https://cdnjs.cloudflare.com https://unpkg.com; "
"style-src 'self' 'unsafe-inline' "
"https://cdn.jsdelivr.net https://cdnjs.cloudflare.com https://unpkg.com; "
"img-src 'self' data: https://*.tile.openstreetmap.org "
@@ -1535,10 +1570,11 @@ class BotDataViewer:
@self.app.route('/api/maintenance/restore', methods=['POST'])
def api_maintenance_restore():
- """Restore DB from a backup file.
+ """Stage a verified DB backup for the next full service restart.
Body: {"db_file": "/absolute/path/to/backup.db"}
- The active DB is overwritten; the caller must restart the bot.
+ The active DB is never modified by this request. Normal bot startup
+ applies the sibling pending file before any DB connection is opened.
"""
try:
data = request.get_json(silent=True) or {}
@@ -1574,25 +1610,37 @@ class BotDataViewer:
if not src.exists():
return jsonify({'error': f'File not found: {db_file}'}), 400
- # Validate it is a real SQLite file by checking the magic header
- _SQLITE_MAGIC = b"SQLite format 3\x00"
try:
- with open(str(src), 'rb') as _fh:
- _header = _fh.read(16)
- if _header != _SQLITE_MAGIC:
- raise ValueError("bad magic")
- except Exception:
- return jsonify({'error': f'Not a valid SQLite file: {db_file}'}), 400
- # Copy to active DB path
- import shutil
- shutil.copy2(str(src), self.db_path)
- self.logger.info(f"Database restored from {src} to {self.db_path}")
+ max_restore_bytes = self.config.getint(
+ 'Web_Viewer',
+ 'restore_max_bytes',
+ fallback=DEFAULT_MAX_RESTORE_BYTES,
+ )
+ pending = stage_database_restore(
+ src,
+ self.db_path,
+ max_bytes=max_restore_bytes,
+ )
+ except (DatabaseRestoreError, OSError, ValueError) as exc:
+ self.logger.warning("Database restore staging rejected for %s: %s", src, exc)
+ return jsonify({'error': str(exc)}), 400
+
+ self.logger.warning(
+ "Database restore staged from %s at %s; a full service restart is required",
+ src,
+ pending,
+ )
return jsonify({
'success': True,
- 'restored_from': db_file,
+ 'staged_from': db_file,
+ 'pending_path': str(pending),
'active_db': self.db_path,
- 'warning': 'Restart the bot for the restored database to take effect.',
- })
+ 'requires_restart': True,
+ 'warning': (
+ 'Restore verified and staged. Restart the complete MeshCore Bot service '
+ 'to apply it before any database writer starts.'
+ ),
+ }), 202
except Exception as e:
self.logger.error(f"Error in restore: {e}", exc_info=True)
return jsonify({'error': str(e)}), 500
@@ -3430,7 +3478,7 @@ class BotDataViewer:
conn = self._get_db_connection()
cursor = conn.cursor()
cursor.execute('''
- SELECT status, error_message, result_data, processed_at
+ SELECT status, error_message, result_data, processed_at, claimed_at
FROM channel_operations
WHERE id = ?
''', (operation_id,))
@@ -3440,12 +3488,13 @@ class BotDataViewer:
if not result:
return jsonify({'error': 'Operation not found'}), 404
- status, error_msg, result_data, processed_at = result
+ status, error_msg, result_data, processed_at, claimed_at = result
return jsonify({
'operation_id': operation_id,
'status': status,
'error_message': error_msg,
+ 'claimed_at': claimed_at,
'processed_at': processed_at,
'result_data': json.loads(result_data) if result_data else None
})
diff --git a/modules/web_viewer/static/js/channel_operations.js b/modules/web_viewer/static/js/channel_operations.js
index dd6f0c7..bb15d55 100644
--- a/modules/web_viewer/static/js/channel_operations.js
+++ b/modules/web_viewer/static/js/channel_operations.js
@@ -34,7 +34,8 @@
}
/**
- * Poll GET /api/channel-operations/:id until completed, failed, or timeouts.
+ * Poll GET /api/channel-operations/:id until completed, failed,
+ * interrupted, or timeouts.
* Behavior aligned with legacy radio.html pollOperationStatus.
*
* @param {number} operationId
@@ -48,7 +49,7 @@
* @param {number} [options.maxWaitSeconds=60]
* @param {number} [options.extendedMaxWaitSeconds=120]
* @param {number} [options.checkIntervalMs=1000]
- * @returns {Promise<'completed'|'failed'|'timeout'>}
+ * @returns {Promise<'completed'|'failed'|'interrupted'|'timeout'>}
*/
function pollChannelOperation(operationId, options) {
options = options || {};
@@ -95,12 +96,12 @@
if (result.status === 'completed') {
return 'completed';
}
- if (result.status === 'failed') {
+ if (result.status === 'failed' || result.status === 'interrupted') {
if (typeof options.onFailed === 'function') {
options.onFailed(result.error_message || 'Channel operation failed');
}
resetButton();
- return 'failed';
+ return result.status;
}
var elapsed = Math.floor((Date.now() - startTime) / 1000);
setBtn(
@@ -122,12 +123,12 @@
if (result2.status === 'completed') {
return 'completed';
}
- if (result2.status === 'failed') {
+ if (result2.status === 'failed' || result2.status === 'interrupted') {
if (typeof options.onFailed === 'function') {
options.onFailed(result2.error_message || 'Channel operation failed');
}
resetButton();
- return 'failed';
+ return result2.status;
}
} catch (error) {
console.error('Error checking final status:', error);
@@ -149,12 +150,12 @@
if (result3.status === 'completed') {
return 'completed';
}
- if (result3.status === 'failed') {
+ if (result3.status === 'failed' || result3.status === 'interrupted') {
if (typeof options.onFailed === 'function') {
options.onFailed(result3.error_message || 'Channel operation failed');
}
resetButton();
- return 'failed';
+ return result3.status;
}
var elapsed2 = Math.floor((Date.now() - startTime) / 1000);
setBtn(
diff --git a/modules/web_viewer/templates/api_explorer.html b/modules/web_viewer/templates/api_explorer.html
index 72d5592..810c187 100644
--- a/modules/web_viewer/templates/api_explorer.html
+++ b/modules/web_viewer/templates/api_explorer.html
@@ -227,7 +227,7 @@
{% endblock %}
{% block extra_js %}
-
-
{% endblock %}
-
diff --git a/modules/web_viewer/templates/greeter.html b/modules/web_viewer/templates/greeter.html
index 7c8bc58..9ee39a7 100644
--- a/modules/web_viewer/templates/greeter.html
+++ b/modules/web_viewer/templates/greeter.html
@@ -106,7 +106,7 @@
{% endblock %}
{% block extra_js %}
-
{% endblock %}
-
diff --git a/modules/web_viewer/templates/index.html b/modules/web_viewer/templates/index.html
index 4a64e12..644f8d5 100644
--- a/modules/web_viewer/templates/index.html
+++ b/modules/web_viewer/templates/index.html
@@ -61,8 +61,7 @@
0
+ data-bs-toggle="modal" data-bs-target="#connectedClientsModal">0
Connected Clients
@@ -429,17 +428,17 @@
-