From 9ae9ebb327fd17feafc76d9023c5324bd3f47aa0 Mon Sep 17 00:00:00 2001 From: agessaman Date: Thu, 16 Jul 2026 10:21:50 -0700 Subject: [PATCH] Harden P0 security and release boundaries --- .github/workflows/test.yml | 29 +- .gitignore | 3 + com.meshcore.bot.plist | 6 +- data/randomlines/__init__.py | 1 + install-service.sh | 359 +++++---- meshcore-bot.service | 12 +- meshcore_bot.py | 24 +- modules/command_manager.py | 33 +- modules/database_restore.py | 451 ++++++++++++ modules/db_migrations.py | 24 + modules/i18n.py | 34 +- modules/scheduler.py | 493 ++++++++----- modules/web_viewer/app.py | 91 ++- .../static/js/channel_operations.js | 17 +- .../web_viewer/templates/api_explorer.html | 2 +- modules/web_viewer/templates/base.html | 32 +- modules/web_viewer/templates/cache.html | 88 ++- modules/web_viewer/templates/config.html | 208 ++++-- modules/web_viewer/templates/contacts.html | 226 +++--- modules/web_viewer/templates/feeds.html | 294 +++++--- modules/web_viewer/templates/greeter.html | 193 +++-- modules/web_viewer/templates/index.html | 218 ++++-- modules/web_viewer/templates/logs.html | 17 +- modules/web_viewer/templates/mesh.html | 89 ++- .../templates/multibyte_rollout.html | 130 ++-- modules/web_viewer/templates/plugins.html | 66 +- modules/web_viewer/templates/radio.html | 274 +++++-- modules/web_viewer/templates/realtime.html | 680 ++++++++++-------- modules/web_viewer/templates/stats.html | 127 ++-- pyproject.toml | 13 +- scripts/build-deb.sh | 100 ++- scripts/debian_service_state.sh | 79 ++ scripts/migrate_service_layout.py | 206 ++++++ scripts/smoke_test_wheel.py | 66 ++ tests/test_contacts_template_xss.py | 57 +- tests/test_database_restore.py | 269 +++++++ tests/test_i18n.py | 7 + tests/test_randomline.py | 23 + tests/test_scheduler_operation_claims.py | 373 ++++++++++ tests/test_service_packaging.py | 330 +++++++++ tests/test_web_viewer.py | 56 +- tests/test_web_viewer_xss_hardening.py | 309 ++++++++ translations/__init__.py | 6 + 43 files changed, 4649 insertions(+), 1466 deletions(-) create mode 100644 data/randomlines/__init__.py create mode 100644 modules/database_restore.py create mode 100644 scripts/debian_service_state.sh create mode 100644 scripts/migrate_service_layout.py create mode 100644 scripts/smoke_test_wheel.py create mode 100644 tests/test_database_restore.py create mode 100644 tests/test_scheduler_operation_claims.py create mode 100644 tests/test_service_packaging.py create mode 100644 tests/test_web_viewer_xss_hardening.py create mode 100644 translations/__init__.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 28d61ab..e376187 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -106,13 +106,40 @@ jobs: - name: mypy — strict overrides on typed modules run: mypy modules/ --ignore-missing-imports + package: + name: Build and smoke-test distributions + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: pip + + - name: Build wheel and source distribution + run: | + python -m pip install --upgrade pip build twine + python -m build + python -m twine check dist/* + + - name: Install and smoke-test wheel outside the source tree + run: | + python -m venv /tmp/meshcore-wheel-smoke + /tmp/meshcore-wheel-smoke/bin/pip install dist/*.whl + cd /tmp + /tmp/meshcore-wheel-smoke/bin/python "$GITHUB_WORKSPACE/scripts/smoke_test_wheel.py" + /tmp/meshcore-wheel-smoke/bin/meshcore-bot --help >/dev/null + /tmp/meshcore-wheel-smoke/bin/meshcore-viewer --help >/dev/null + test: name: Tests (Python ${{ matrix.python-version }}) runs-on: ubuntu-latest strategy: fail-fast: false matrix: - python-version: ["3.11", "3.12", "3.13"] + python-version: ["3.10", "3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v4 diff --git a/.gitignore b/.gitignore index 237f2e2..e0ed502 100644 --- a/.gitignore +++ b/.gitignore @@ -182,3 +182,6 @@ local/service_plugins/* !local/service_plugins/.gitkeep .claude/settings.local.json + +# Local engineering audit reports (may contain security-sensitive findings) +/PROJECT_REVIEW_*.md diff --git a/com.meshcore.bot.plist b/com.meshcore.bot.plist index 03e1d82..b5a1e4c 100644 --- a/com.meshcore.bot.plist +++ b/com.meshcore.bot.plist @@ -4,11 +4,16 @@ Label com.meshcore.bot + + UserName + __MESHCORE_SERVICE_USER__ ProgramArguments /usr/local/meshcore-bot/venv/bin/python /usr/local/meshcore-bot/meshcore_bot.py + --config + /usr/local/etc/meshcore-bot/config.ini WorkingDirectory @@ -47,4 +52,3 @@ 1 - diff --git a/data/randomlines/__init__.py b/data/randomlines/__init__.py new file mode 100644 index 0000000..2b1fed9 --- /dev/null +++ b/data/randomlines/__init__.py @@ -0,0 +1 @@ +"""Bundled default RandomLine response files.""" diff --git a/install-service.sh b/install-service.sh index 10792ed..3544ab1 100755 --- a/install-service.sh +++ b/install-service.sh @@ -17,11 +17,12 @@ # # Prerequisites: # - Linux system with systemd OR macOS -# - Python 3.9+ installed +# - Python 3.10+ installed # - sudo access (script will prompt if needed) # - Run from the meshcore-bot directory set -e +umask 077 # Colors for output RED='\033[0;31m' @@ -53,6 +54,8 @@ if [[ "$IS_MACOS" == true ]]; then SERVICE_USER="$(whoami)" # macOS: use current user or _meshcore SERVICE_GROUP="staff" INSTALL_DIR="/usr/local/meshcore-bot" + CONF_DIR="/usr/local/etc/meshcore-bot" + STATE_DIR="/usr/local/var/lib/meshcore-bot" LOG_DIR="/usr/local/var/log/meshcore-bot" SERVICE_FILE="com.meshcore.bot.plist" LAUNCHD_DIR="/Library/LaunchDaemons" @@ -60,11 +63,15 @@ else SERVICE_USER="meshcore" SERVICE_GROUP="meshcore" INSTALL_DIR="/opt/meshcore-bot" + CONF_DIR="/etc/meshcore-bot" + STATE_DIR="/var/lib/meshcore-bot" LOG_DIR="/var/log/meshcore-bot" SERVICE_FILE="meshcore-bot.service" SYSTEMD_DIR="/etc/systemd/system" fi +CONFIG_FILE="$CONF_DIR/config.ini" + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # Parse command line arguments (before sudo check so help works) @@ -222,9 +229,34 @@ fi # Check if Python 3 is available if ! command -v python3 &> /dev/null; then print_error "Python 3 is not installed or not in PATH" - print_error "Please install Python 3.9 or higher before running this script" + print_error "Please install Python 3.10 or higher before running this script" exit 1 fi +if ! python3 -c 'import sys; raise SystemExit(sys.version_info < (3, 10))'; then + print_error "Python 3.10 or higher is required" + print_error "Found: $(python3 --version 2>&1)" + exit 1 +fi + +# Stop a running legacy service before changing code or taking the SQLite +# backup used for layout migration. This runs only after sudo re-execution and +# service-manager validation, so --help and unprivileged discovery stay inert. +SERVICE_WAS_ACTIVE=false +if [[ "$IS_MACOS" == true ]]; then + if launchctl list "$PLIST_NAME" &>/dev/null; then + SERVICE_WAS_ACTIVE=true + if ! launchctl unload "$LAUNCHD_DIR/$SERVICE_FILE" 2>/dev/null; then + launchctl stop "$PLIST_NAME" 2>/dev/null || true + fi + if launchctl list "$PLIST_NAME" &>/dev/null; then + print_error "The launchd service is still running; refusing an unsafe live database migration" + exit 1 + fi + fi +elif systemctl is-active --quiet "$SERVICE_NAME" 2>/dev/null; then + SERVICE_WAS_ACTIVE=true + systemctl stop "$SERVICE_NAME" +fi print_section "Step 1: Setting Up Service User" if [[ "$IS_MACOS" == true ]]; then @@ -291,160 +323,53 @@ else print_success "Created installation directory: $INSTALL_DIR" fi -# Create log directory -mkdir -p "$LOG_DIR" +# Create mutable runtime directories separately from executable code. +mkdir -p "$CONF_DIR" "$STATE_DIR" "$LOG_DIR" +print_success "Created configuration directory: $CONF_DIR" +print_success "Created state directory: $STATE_DIR" print_success "Created log directory: $LOG_DIR" print_section "Step 3: Copying Bot Files" if [[ "$UPGRADE_MODE" == true ]]; then print_info "Upgrading files in $INSTALL_DIR" - print_info "Only newer files will be copied, preserving existing configuration" + print_info "Replacing executable files from the trusted source while preserving explicit runtime state" else print_info "Copying bot files to $INSTALL_DIR" - print_info "Existing files will be updated only if source is newer" + print_info "Executable files will exactly match the trusted source" fi -# Function to copy files intelligently +# Synchronize executable code authoritatively. Using --update or a merge-only +# fallback can preserve a newer/stale Python file written by a previously +# compromised service account and then cement it as root-owned executable code. +# Only the explicitly excluded runtime paths survive an upgrade. copy_files_smart() { local source_dir="$1" local dest_dir="$2" - local files_copied=0 - local files_skipped=0 - local files_updated=0 - - # Use rsync if available (better for this use case) - if command -v rsync &> /dev/null; then - print_info "Using rsync for efficient file copying" - # Preserve config.ini if it exists - local preserve_config="" - if [ -f "$dest_dir/config.ini" ]; then - preserve_config="--exclude=config.ini" - print_info "Preserving existing config.ini (not overwriting)" - fi - - # Note: --update flag preserves files in alternatives/ if destination is newer or same - # This protects user's custom alternative commands while allowing updates to repository files - if [ -d "$dest_dir/modules/commands/alternatives" ]; then - print_info "Preserving existing alternative commands (only updating if source is newer)" - fi - - # Preserve install dir's local/ entirely (user custom commands and service plugins) - # Never overwrite or delete anything under $dest_dir/local/ - if [ -d "$dest_dir/local" ]; then - print_info "Preserving existing local/ directory (not overwriting)" - fi - - # Exclude patterns - rsync -a --update --exclude='.git' \ - --exclude='__pycache__' \ - --exclude='*.pyc' \ - --exclude='*.pyo' \ - --exclude='.DS_Store' \ - --exclude='venv' \ - --exclude='*.db' \ - --exclude='*.db-shm' \ - --exclude='*.db-wal' \ - --exclude='*.log' \ - --exclude='backups' \ - --exclude='local/' \ - $preserve_config \ - "$source_dir/" "$dest_dir/" 2>/dev/null || { - print_warning "rsync had some issues, falling back to manual copy" - } - # If install dir has no local/, create minimal structure from source so service has valid layout - if [ ! -d "$dest_dir/local" ]; then - print_info "Creating local/ directory structure (first-time install)" - mkdir -p "$dest_dir/local/commands" "$dest_dir/local/service_plugins" - [ -f "$source_dir/local/README.md" ] && cp "$source_dir/local/README.md" "$dest_dir/local/" || true - [ -f "$source_dir/local/__init__.py" ] && cp "$source_dir/local/__init__.py" "$dest_dir/local/" || true - [ -f "$source_dir/local/commands/.gitkeep" ] && cp "$source_dir/local/commands/.gitkeep" "$dest_dir/local/commands/" || true - [ -f "$source_dir/local/service_plugins/.gitkeep" ] && cp "$source_dir/local/service_plugins/.gitkeep" "$dest_dir/local/service_plugins/" || true - fi - print_success "Files synchronized using rsync" - return 0 + if ! command -v rsync &> /dev/null; then + print_error "rsync is required for a source-authoritative secure install; install rsync and retry" + return 1 fi - - # Fallback: manual copy with find - print_info "Using manual file copy (consider installing rsync for better performance)" - - # Preserve alternatives directory if it exists - if [ -d "$dest_dir/modules/commands/alternatives" ]; then - print_info "Preserving existing alternative commands (not overwriting)" + + print_info "Using rsync for source-authoritative executable synchronization" + if ! rsync -a --delete --exclude='.git' \ + --exclude='__pycache__' \ + --exclude='*.pyc' \ + --exclude='*.pyo' \ + --exclude='.DS_Store' \ + --exclude='venv' \ + --exclude='*.db' \ + --exclude='*.db-shm' \ + --exclude='*.db-wal' \ + --exclude='*.log' \ + --exclude='backups' \ + --exclude='local/' \ + --exclude='config.ini' \ + "$source_dir/" "$dest_dir/"; then + print_error "rsync failed; refusing to continue with a partially synchronized code tree" + return 1 fi - - # Preserve install dir's local/ entirely - never overwrite when it exists - if [ -d "$dest_dir/local" ]; then - print_info "Preserving existing local/ directory (not overwriting)" - fi - - # Copy files, preserving config.ini if it exists - while IFS= read -r file; do - local rel_path="${file#$source_dir/}" - local dest_file="$dest_dir/$rel_path" - local dest_dir_path - dest_dir_path="$(dirname "$dest_file")" - - # Skip excluded patterns - [[ "$rel_path" == *".git"* ]] && continue - [[ "$rel_path" == *"__pycache__"* ]] && continue - [[ "$rel_path" == *".pyc" ]] && continue - [[ "$rel_path" == *".pyo" ]] && continue - [[ "$rel_path" == *".DS_Store"* ]] && continue - [[ "$rel_path" == *"/venv/"* ]] && continue - [[ "$rel_path" == *".db" ]] && continue - [[ "$rel_path" == *".db-shm" ]] && continue - [[ "$rel_path" == *".db-wal" ]] && continue - [[ "$rel_path" == *".log" ]] && continue - [[ "$rel_path" == *"/backups/"* ]] && continue - - # Preserve install dir's local/ entirely - skip all files under local/ when dest has local/ - if [[ "$rel_path" == "local/"* ]] && [ -d "$dest_dir/local" ]; then - files_skipped=$((files_skipped + 1)) - continue - fi - - # Preserve alternatives directory - only update if source file is newer - if [[ "$rel_path" == "modules/commands/alternatives/"* ]] && [ -d "$dest_dir/modules/commands/alternatives" ]; then - if [ -f "$dest_file" ]; then - # File exists in destination - only update if source is newer - if [ "$file" -nt "$dest_file" ]; then - # Source is newer, will update below - : - else - # Destination is same or newer - preserve user's version - files_skipped=$((files_skipped + 1)) - continue - fi - fi - # File doesn't exist in destination, or source is newer - will copy below - fi - - # Create destination directory if needed - mkdir -p "$dest_dir_path" - - # Special handling for config.ini - preserve existing if it exists - if [[ "$rel_path" == "config.ini" ]] && [ -f "$dest_file" ]; then - files_skipped=$((files_skipped + 1)) - continue - fi - - # Copy if destination doesn't exist or source is newer - if [ ! -f "$dest_file" ] || [ "$file" -nt "$dest_file" ]; then - if cp "$file" "$dest_file" 2>/dev/null; then - if [ -f "$dest_file" ]; then - files_updated=$((files_updated + 1)) - else - files_copied=$((files_copied + 1)) - fi - else - print_warning "Could not copy $rel_path" - fi - else - files_skipped=$((files_skipped + 1)) - fi - done < <(find "$source_dir" -type f 2>/dev/null) - - print_success "File sync complete: $files_updated updated, $files_copied new, $files_skipped unchanged" + + print_success "Executable files synchronized authoritatively using rsync" } # Copy files using smart copy function @@ -465,52 +390,63 @@ if command -v git &>/dev/null && [ -d "$SCRIPT_DIR/.git" ]; then print_success "Wrote version info (${INSTALLER_VER}) to $INSTALL_DIR/.version_info" fi -# If no config.ini in install dir, create it from config.ini.example -if [ ! -f "$INSTALL_DIR/config.ini" ]; then - if [ -f "$INSTALL_DIR/config.ini.example" ]; then - cp "$INSTALL_DIR/config.ini.example" "$INSTALL_DIR/config.ini" - print_success "Created $INSTALL_DIR/config.ini from config.ini.example (no config was present)" +# Keep configuration out of the root-owned application tree. On upgrade, copy +# the legacy config once so existing credentials and settings are preserved. +if [ ! -f "$CONFIG_FILE" ]; then + if [ -f "$INSTALL_DIR/config.ini" ] && [ ! -L "$INSTALL_DIR/config.ini" ]; then + cp -p "$INSTALL_DIR/config.ini" "$CONFIG_FILE" + print_success "Migrated existing configuration to $CONFIG_FILE" + elif [ -f "$INSTALL_DIR/config.ini.example" ]; then + cp "$INSTALL_DIR/config.ini.example" "$CONFIG_FILE" + print_success "Created $CONFIG_FILE from config.ini.example" elif [ -f "$SCRIPT_DIR/config.ini.example" ]; then - cp "$SCRIPT_DIR/config.ini.example" "$INSTALL_DIR/config.ini" - print_success "Created $INSTALL_DIR/config.ini from config.ini.example (no config was present)" + cp "$SCRIPT_DIR/config.ini.example" "$CONFIG_FILE" + print_success "Created $CONFIG_FILE from config.ini.example" else - print_warning "config.ini.example not found. Create $INSTALL_DIR/config.ini manually before starting the bot." + print_warning "config.ini.example not found. Create $CONFIG_FILE manually before starting the bot." fi fi -# Create venv and install dependencies before chown so the service user ends up -# owning a complete, working venv (avoids partial root-owned venv and import errors). +# Rewrite all relative runtime paths and coherently migrate an existing SQLite +# database. Absolute custom paths are left untouched; operators can grant an +# additional systemd path explicitly when they intentionally store state there. +if [ -f "$CONFIG_FILE" ]; then + python3 "$INSTALL_DIR/scripts/migrate_service_layout.py" \ + --config "$CONFIG_FILE" \ + --legacy-base "$INSTALL_DIR" \ + --state-dir "$STATE_DIR" \ + --log-dir "$LOG_DIR" +fi + +# Remove a source-tree config copied into the application tree only when it is +# identical to the active service config; otherwise retain it root-only as a +# migration backup. +if [ -f "$INSTALL_DIR/config.ini" ] && [ ! -L "$INSTALL_DIR/config.ini" ]; then + chmod 0600 "$INSTALL_DIR/config.ini" +fi + +if [ ! -f "$CONFIG_FILE" ]; then + # Retain the old diagnostic wording for automation which looks for it. + if [ -f "$INSTALL_DIR/config.ini.example" ]; then + print_warning "Failed to create $CONFIG_FILE; check directory permissions" + fi +fi + +# Build dependencies in a fresh environment. The legacy venv was writable by +# the service account; reusing it could preserve a malicious .pth/module and +# turn that persistence into root-owned executable code during hardening. print_section "Step 4: Setting Up Python Virtual Environment" -if [ -d "$INSTALL_DIR/venv" ]; then - print_info "Virtual environment already exists at $INSTALL_DIR/venv" - print_info "Preserving existing virtual environment" - if [[ "$UPGRADE_MODE" == true ]]; then - print_info "Upgrade mode: will update dependencies" - else - print_info "Will update dependencies if requirements.txt changed" - fi -else - print_info "Creating an isolated Python environment for the bot" - print_info "This ensures dependencies don't conflict with system Python packages" - python3 -m venv "$INSTALL_DIR/venv" - print_success "Created virtual environment at $INSTALL_DIR/venv" -fi - -# Verify virtual environment looks healthy -VENV_PYTHON="$INSTALL_DIR/venv/bin/python" -if [ ! -x "$VENV_PYTHON" ]; then - print_error "Python virtual environment at $INSTALL_DIR/venv appears to be incomplete or corrupted" - print_error "Expected Python executable not found at: $VENV_PYTHON" - print_info "Try removing $INSTALL_DIR/venv and re-running this installer to recreate it:" - echo " sudo rm -rf $INSTALL_DIR/venv" - echo " sudo ./install-service.sh" - exit 1 -fi +VENV_BUILD="$INSTALL_DIR/.venv-build-$$" +VENV_OLD="$INSTALL_DIR/.venv-old-$$" +rm -rf "$VENV_BUILD" "$VENV_OLD" +print_info "Creating a fresh isolated Python environment" +python3 -m venv "$VENV_BUILD" +VENV_BUILD_PYTHON="$VENV_BUILD/bin/python" # Ensure pip is available and up to date inside the venv print_info "Ensuring pip is available and up to date in the virtual environment" -$VENV_PYTHON -m ensurepip --upgrade >/dev/null 2>&1 || true -$VENV_PYTHON -m pip install --quiet --upgrade pip >/dev/null 2>&1 || true +$VENV_BUILD_PYTHON -m ensurepip --upgrade >/dev/null 2>&1 || true +$VENV_BUILD_PYTHON -m pip install --quiet --upgrade pip >/dev/null 2>&1 || true # Install dependencies in venv using python -m pip (more portable than calling pip directly) print_info "Installing Python dependencies from requirements.txt" @@ -519,12 +455,22 @@ if [ ! -f "$INSTALL_DIR/requirements.txt" ]; then print_error "requirements.txt not found in installation directory" exit 1 fi -$VENV_PYTHON -m pip install --quiet -r "$INSTALL_DIR/requirements.txt" || { +$VENV_BUILD_PYTHON -m pip install --quiet -r "$INSTALL_DIR/requirements.txt" || { print_error "Failed to install Python dependencies" print_info "You may need to check your internet connection or Python version" + rm -rf "$VENV_BUILD" exit 1 } -print_success "Installed all Python dependencies" +if [ -d "$INSTALL_DIR/venv" ]; then + mv "$INSTALL_DIR/venv" "$VENV_OLD" +fi +if ! mv "$VENV_BUILD" "$INSTALL_DIR/venv"; then + [ -d "$VENV_OLD" ] && mv "$VENV_OLD" "$INSTALL_DIR/venv" + print_error "Failed to activate the newly built virtual environment" + exit 1 +fi +rm -rf "$VENV_OLD" +print_success "Installed all Python dependencies into a fresh virtual environment" # Optional extras echo "" @@ -555,19 +501,37 @@ fi print_section "Step 5: Setting File Permissions" print_info "Configuring file ownership and permissions for security" -print_info "The service user will own all files, with appropriate read/write permissions" -# Set ownership -chown -R "$SERVICE_USER:$SERVICE_GROUP" "$INSTALL_DIR" +print_info "Executable code is root-owned; the service owns only configuration and runtime state" +# Executable code and the virtual environment must not be writable by the +# network-facing service account. +# The service group receives read-only access to any explicitly installed key +# material while root remains the only account able to modify it. +CODE_GROUP="$SERVICE_GROUP" +chown -R "root:$CODE_GROUP" "$INSTALL_DIR" chown -R "$SERVICE_USER:$SERVICE_GROUP" "$LOG_DIR" -print_success "Set ownership to $SERVICE_USER:$SERVICE_GROUP" +chown -R "$SERVICE_USER:$SERVICE_GROUP" "$CONF_DIR" "$STATE_DIR" +print_success "Separated root-owned code from service-owned runtime state" -# Set permissions +# Code is readable/executable but never service-writable. Preserve executable +# bits created by the virtualenv and source scripts while dropping group/other +# write access. chmod 755 "$INSTALL_DIR" +chmod -R go-w "$INSTALL_DIR" find "$INSTALL_DIR" -type f -name "*.py" -exec chmod 644 {} \; 2>/dev/null || true -find "$INSTALL_DIR" -type f -name "*.ini" -exec chmod 644 {} \; 2>/dev/null || true find "$INSTALL_DIR" -type f -name "*.txt" -exec chmod 644 {} \; 2>/dev/null || true find "$INSTALL_DIR" -type f -name "*.json" -exec chmod 644 {} \; 2>/dev/null || true find "$INSTALL_DIR" -type d -exec chmod 755 {} \; 2>/dev/null || true +find "$INSTALL_DIR" -type f -name "*.ini" -exec chmod 600 {} \; 2>/dev/null || true +find "$INSTALL_DIR" -type f \( -name ".env" -o -name "*.key" -o -name "*.pem" -o -name "*.p12" -o -name "*.pfx" \) -exec chmod 640 {} \; 2>/dev/null || true +find "$INSTALL_DIR" -type f \( -name "*.db" -o -name "*.db-wal" -o -name "*.db-shm" -o -name "*.log" -o -name "*.log.*" \) -exec chmod 600 {} \; 2>/dev/null || true + +# Credentials, databases, backups, and local-plugin settings are private to the +# service user. Directories must be writable for SQLite sidecars and atomic +# config updates; the 0700 boundary prevents local disclosure. +find "$CONF_DIR" "$STATE_DIR" -type d -exec chmod 700 {} \; 2>/dev/null || true +find "$CONF_DIR" "$STATE_DIR" -type f -exec chmod 600 {} \; 2>/dev/null || true +chmod 750 "$LOG_DIR" +find "$LOG_DIR" -type f -exec chmod 600 {} \; 2>/dev/null || true # Make main script executable chmod 755 "$INSTALL_DIR/meshcore_bot.py" @@ -594,13 +558,16 @@ import re with open('$SERVICE_FILE', 'r') as f: content = f.read() content = content.replace('/usr/local/meshcore-bot', '$INSTALL_DIR') +content = content.replace('/usr/local/etc/meshcore-bot', '$CONF_DIR') +content = content.replace('/usr/local/var/lib/meshcore-bot', '$STATE_DIR') content = content.replace('/usr/local/var/log/meshcore-bot', '$LOG_DIR') +content = content.replace('__MESHCORE_SERVICE_USER__', '$SERVICE_USER') with open('$LAUNCHD_DIR/$SERVICE_FILE', 'w') as f: f.write(content) " else # Fallback to sed (works on both macOS and Linux) - sed "s|/usr/local/meshcore-bot|$INSTALL_DIR|g; s|/usr/local/var/log/meshcore-bot|$LOG_DIR|g" "$SERVICE_FILE" > "$LAUNCHD_DIR/$SERVICE_FILE" + sed "s|/usr/local/meshcore-bot|$INSTALL_DIR|g; s|/usr/local/etc/meshcore-bot|$CONF_DIR|g; s|/usr/local/var/lib/meshcore-bot|$STATE_DIR|g; s|/usr/local/var/log/meshcore-bot|$LOG_DIR|g; s|__MESHCORE_SERVICE_USER__|$SERVICE_USER|g" "$SERVICE_FILE" > "$LAUNCHD_DIR/$SERVICE_FILE" fi print_success "Copied and configured plist file to $LAUNCHD_DIR/" fi @@ -668,6 +635,15 @@ else print_info "Note: The service is enabled but not started yet. You'll start it after configuration." fi +if [[ "$SERVICE_WAS_ACTIVE" == true ]]; then + print_info "Restarting the service because it was running before the upgrade" + if [[ "$IS_MACOS" == true ]]; then + launchctl load "$LAUNCHD_DIR/$SERVICE_FILE" 2>/dev/null || true + else + systemctl start "$SERVICE_NAME" + fi +fi + if [[ "$UPGRADE_MODE" == true ]]; then print_section "Upgrade Complete!" echo "" @@ -684,7 +660,7 @@ echo -e "${BLUE}📋 Next Steps${NC}" echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo "" echo -e "${CYAN}1. Configure the bot:${NC}" -echo -e " ${YELLOW}sudo nano $INSTALL_DIR/config.ini${NC}" +echo -e " ${YELLOW}sudo nano $CONFIG_FILE${NC}" echo " Edit the configuration file with your bot settings, API keys, and device information" echo "" @@ -744,7 +720,8 @@ echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━ echo -e "${BLUE}📁 Important File Locations${NC}" echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo "" -echo -e " ${CYAN}Configuration file:${NC} ${YELLOW}$INSTALL_DIR/config.ini${NC}" +echo -e " ${CYAN}Configuration file:${NC} ${YELLOW}$CONFIG_FILE${NC}" +echo -e " ${CYAN}State directory:${NC} ${YELLOW}$STATE_DIR${NC}" echo -e " ${CYAN}Log directory:${NC} ${YELLOW}$LOG_DIR${NC}" echo -e " ${CYAN}Installation directory:${NC} ${YELLOW}$INSTALL_DIR${NC}" if [[ "$IS_MACOS" == true ]]; then @@ -792,4 +769,4 @@ if [[ "$UPGRADE_MODE" == true ]]; then else print_success "Installation complete! The bot is ready to configure and start." fi -echo "" \ No newline at end of file +echo "" diff --git a/meshcore-bot.service b/meshcore-bot.service index 5f01c8e..ba0a9a2 100644 --- a/meshcore-bot.service +++ b/meshcore-bot.service @@ -9,7 +9,7 @@ Type=simple User=meshcore Group=meshcore WorkingDirectory=/opt/meshcore-bot -ExecStart=/opt/meshcore-bot/venv/bin/python /opt/meshcore-bot/meshcore_bot.py +ExecStart=/opt/meshcore-bot/venv/bin/python /opt/meshcore-bot/meshcore_bot.py --config /etc/meshcore-bot/config.ini ExecReload=/bin/kill -HUP $MAINPID Restart=always RestartSec=10 @@ -26,7 +26,15 @@ NoNewPrivileges=true PrivateTmp=true ProtectSystem=strict ProtectHome=true -ReadWritePaths=/opt/meshcore-bot +UMask=0077 +ConfigurationDirectory=meshcore-bot +ConfigurationDirectoryMode=0700 +StateDirectory=meshcore-bot +StateDirectoryMode=0700 +LogsDirectory=meshcore-bot +LogsDirectoryMode=0750 +ReadWritePaths=/etc/meshcore-bot +ReadWritePaths=/var/lib/meshcore-bot ReadWritePaths=/var/log/meshcore-bot # Resource limits diff --git a/meshcore_bot.py b/meshcore_bot.py index 1789bd7..18cacf7 100644 --- a/meshcore_bot.py +++ b/meshcore_bot.py @@ -142,6 +142,29 @@ def main(): except Exception as exc: # noqa: BLE001 - never block startup on the linter itself print(f"Config validation skipped: {exc}", file=sys.stderr) + # A restore requested by the viewer is applied only here, before importing + # and constructing MeshCoreBot (which opens the DB and launches writers). + # The service manager must restart the bot and viewer as one unit. + from modules.database_restore import ( + DatabaseRestoreError, + apply_pending_restores_from_config, + ) + + try: + restore_results = apply_pending_restores_from_config(args.config) + except DatabaseRestoreError as exc: + print(f"Error: pending database restore was not applied: {exc}", file=sys.stderr) + print( + "The active database was left unchanged. Correct or remove the pending restore " + "file before restarting.", + file=sys.stderr, + ) + sys.exit(1) + for restore_result in restore_results: + recovery = restore_result.recovery_backup_path + recovery_note = f"; recovery backup: {recovery}" if recovery else "" + print(f"Applied pending database restore: {restore_result.database_path}{recovery_note}") + from modules.core import MeshCoreBot bot = MeshCoreBot(config_file=args.config) @@ -238,4 +261,3 @@ if __name__ == "__main__": main() - diff --git a/modules/command_manager.py b/modules/command_manager.py index 54b7dc7..41f9edd 100644 --- a/modules/command_manager.py +++ b/modules/command_manager.py @@ -10,6 +10,8 @@ import time from dataclasses import dataclass from datetime import datetime from hashlib import sha256 +from importlib import resources +from pathlib import Path from typing import Any from meshcore import EventType @@ -955,24 +957,43 @@ class CommandManager: if not self._is_channel_trigger_allowed(key, message): return None - file_path = self.bot.config.get('RandomLine', f'file.{key}', fallback='').strip() - if not file_path: + configured_file_path = self.bot.config.get('RandomLine', f'file.{key}', fallback='').strip() + if not configured_file_path: self.logger.warning(f"RandomLine matched '{key}' but missing config file.{key}") return None try: - validated_path = validate_safe_path(file_path, allow_absolute=True) + validated_path = validate_safe_path(configured_file_path, allow_absolute=True) except ValueError: validated_path = None if validated_path is None: - self.logger.warning(f"RandomLine: unsafe or restricted path rejected for '{key}': {file_path}") + self.logger.warning( + f"RandomLine: unsafe or restricted path rejected for '{key}': {configured_file_path}" + ) return None file_path = str(validated_path) # Read usable lines try: - with open(file_path, encoding="utf-8") as f: - lines = [ln.strip() for ln in f.readlines()] + if Path(file_path).is_file(): + with open(file_path, encoding="utf-8") as f: + lines = [ln.strip() for ln in f.readlines()] + else: + # Shipped RandomLine defaults live in package data in wheels. + # Only map the documented data/randomlines path; arbitrary + # missing custom files must still fail instead of silently + # selecting a same-named bundled file. + normalized = configured_file_path.replace("\\", "/").lstrip("./") + marker = "data/randomlines/" + if not normalized.startswith(marker): + raise FileNotFoundError(file_path) + resource_name = normalized.removeprefix(marker) + if not resource_name or "/" in resource_name: + raise FileNotFoundError(file_path) + bundled = resources.files("data.randomlines").joinpath(resource_name) + if not bundled.is_file(): + raise FileNotFoundError(file_path) + lines = [ln.strip() for ln in bundled.read_text(encoding="utf-8").splitlines()] lines = [ln for ln in lines if ln] # drop blank lines except Exception as e: self.logger.error(f"RandomLine error reading {file_path} for '{key}': {e}", exc_info=True) diff --git a/modules/database_restore.py b/modules/database_restore.py new file mode 100644 index 0000000..6b413a2 --- /dev/null +++ b/modules/database_restore.py @@ -0,0 +1,451 @@ +"""Stage and apply SQLite restores without replacing a live database. + +The web viewer runs in a separate process from the bot and both processes open +short-lived WAL connections. Replacing the database from an HTTP request is +therefore unsafe: existing descriptors and WAL sidecars can continue referring +to the old database. This module implements a two-phase restore instead: + +* the viewer validates and copies a backup to a sibling pending file; and +* normal bot startup applies that pending file before any database manager, + scheduler, or viewer process is created. + +The startup caller is responsible for ensuring the service was restarted as a +unit (that is, no separately managed viewer process is still using the file). +""" + +from __future__ import annotations + +import configparser +import os +import shutil +import sqlite3 +import stat +import tempfile +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path + +from modules.db_migrations import MIGRATIONS + +DEFAULT_MAX_RESTORE_BYTES = 2 * 1024 * 1024 * 1024 + +# Databases created before numbered migrations do not have ``schema_version``. +# Requiring bot_metadata plus multiple tables from the original MeshCore schema +# distinguishes those legitimate backups from an arbitrary SQLite file while +# still allowing MigrationRunner to upgrade them after startup. +LEGACY_SCHEMA_TABLES = { + "channel_operations", + "channels", + "feed_activity", + "feed_errors", + "feed_message_queue", + "feed_subscriptions", + "generic_cache", + "geocoding_cache", +} + + +class DatabaseRestoreError(RuntimeError): + """Raised when a restore cannot be safely staged or applied.""" + + +@dataclass(frozen=True) +class RestoreResult: + """Description of a restore applied during startup.""" + + database_path: Path + recovery_backup_path: Path | None + + +def pending_restore_path(database_path: str | Path) -> Path: + """Return the same-filesystem staging path for *database_path*.""" + + database = Path(database_path).resolve() + return database.with_name(f".{database.name}.restore-pending") + + +def validate_restore_database(database_path: str | Path) -> None: + """Fail unless *database_path* is an intact, compatible MeshCore database.""" + + path = Path(database_path).resolve() + if not path.is_file(): + raise DatabaseRestoreError(f"Restore database does not exist: {path}") + + try: + with path.open("rb") as handle: + if handle.read(16) != b"SQLite format 3\x00": + raise DatabaseRestoreError("Restore file is not a SQLite database") + + uri = f"{path.as_uri()}?mode=ro" + with sqlite3.connect(uri, uri=True, timeout=5.0) as conn: + integrity_row = conn.execute("PRAGMA integrity_check(1)").fetchone() + if not integrity_row or str(integrity_row[0]).lower() != "ok": + detail = str(integrity_row[0]) if integrity_row else "no result" + raise DatabaseRestoreError(f"SQLite integrity check failed: {detail}") + + tables = { + str(row[0]) + for row in conn.execute( + "SELECT name FROM sqlite_master WHERE type = 'table'" + ).fetchall() + } + if "bot_metadata" not in tables: + raise DatabaseRestoreError( + "Restore file is not a MeshCore Bot database; " + "missing table: bot_metadata" + ) + + if "schema_version" not in tables: + legacy_matches = sorted(LEGACY_SCHEMA_TABLES & tables) + if len(legacy_matches) < 2: + raise DatabaseRestoreError( + "Restore file is not a recognizable legacy MeshCore Bot " + "database; numbered migration history is absent and fewer " + "than two legacy schema tables were found" + ) + else: + known_versions = { + int(version) for version, _description, _fn in MIGRATIONS + } + applied_versions = { + int(row[0]) + for row in conn.execute( + "SELECT version FROM schema_version" + ).fetchall() + if row and row[0] is not None + } + unknown_versions = sorted(applied_versions - known_versions) + if unknown_versions: + raise DatabaseRestoreError( + "Restore database was created by a newer or incompatible " + "version; unknown migration version(s): " + f"{unknown_versions}" + ) + except DatabaseRestoreError: + raise + except (OSError, sqlite3.Error, ValueError) as exc: + raise DatabaseRestoreError(f"Could not validate restore database: {exc}") from exc + + +def _safe_database_mode(database_path: Path) -> int: + """Preserve owner/group access while never granting access to others.""" + + try: + current = stat.S_IMODE(database_path.stat().st_mode) + except OSError: + current = 0 + return 0o600 | (current & 0o060) + + +def _match_database_ownership(path: Path, database_path: Path) -> None: + os.chmod(path, _safe_database_mode(database_path)) + try: + active_stat = database_path.stat() + os.chown(path, active_stat.st_uid, active_stat.st_gid) + except (AttributeError, FileNotFoundError, PermissionError): + # chown is unavailable on Windows and an unprivileged service can only + # create files as itself. In both cases the creator is the safe owner. + pass + + +def _fsync_directory(directory: Path) -> None: + if os.name == "nt": + return + descriptor = os.open(str(directory), os.O_RDONLY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def stage_database_restore( + source_path: str | Path, + database_path: str | Path, + *, + max_bytes: int = DEFAULT_MAX_RESTORE_BYTES, +) -> Path: + """Validate and stage *source_path* for application on the next bot startup.""" + + if max_bytes <= 0: + raise DatabaseRestoreError("Restore size limit must be positive") + + source = Path(source_path).resolve() + database = Path(database_path).resolve() + pending = pending_restore_path(database) + if source in {database, pending}: + raise DatabaseRestoreError("Restore source must be a separate backup file") + + validate_restore_database(source) + database.parent.mkdir(parents=True, exist_ok=True) + temp_path: Path | None = None + try: + descriptor, temp_name = tempfile.mkstemp( + dir=database.parent, + prefix=f".{database.name}.restore-", + suffix=".tmp", + ) + os.close(descriptor) + temp_path = Path(temp_name) + source_uri = f"{source.as_uri()}?mode=ro" + with sqlite3.connect(source_uri, uri=True, timeout=30.0) as source_conn: + page_count = int(source_conn.execute("PRAGMA page_count").fetchone()[0]) + page_size = int(source_conn.execute("PRAGMA page_size").fetchone()[0]) + if page_count * page_size > max_bytes: + raise DatabaseRestoreError( + f"Restore database exceeds the {max_bytes}-byte safety limit" + ) + with sqlite3.connect(str(temp_path), timeout=30.0) as destination_conn: + # SQLite's online backup API produces a self-contained snapshot + # and includes committed WAL content; a raw file copy does not. + source_conn.backup(destination_conn) + destination_conn.commit() + + if temp_path.stat().st_size > max_bytes: + raise DatabaseRestoreError( + f"Restore database exceeds the {max_bytes}-byte safety limit" + ) + with temp_path.open("rb") as temp_handle: + os.fsync(temp_handle.fileno()) + + _match_database_ownership(temp_path, database) + validate_restore_database(temp_path) + os.replace(temp_path, pending) + temp_path = None + _fsync_directory(database.parent) + return pending + except DatabaseRestoreError: + raise + except OSError as exc: + raise DatabaseRestoreError(f"Could not stage restore database: {exc}") from exc + finally: + if temp_path is not None: + try: + temp_path.unlink() + except FileNotFoundError: + pass + + +def _create_recovery_backup(database: Path) -> Path: + timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S%fZ") + recovery = database.with_name(f"{database.stem}.pre-restore-{timestamp}.db") + descriptor, temp_name = tempfile.mkstemp( + dir=database.parent, + prefix=f".{database.name}.pre-restore-", + suffix=".tmp", + ) + os.close(descriptor) + temp_path = Path(temp_name) + try: + with sqlite3.connect(str(database), timeout=30.0) as source_conn: + with sqlite3.connect(str(temp_path), timeout=30.0) as destination_conn: + source_conn.backup(destination_conn) + destination_conn.commit() + _match_database_ownership(temp_path, database) + validate_restore_database(temp_path) + os.replace(temp_path, recovery) + _fsync_directory(database.parent) + return recovery + except Exception: + try: + temp_path.unlink() + except FileNotFoundError: + pass + raise + + +def _rollback_from_recovery(recovery: Path, database: Path) -> None: + """Atomically restore *database* from the already verified recovery copy.""" + + descriptor, temp_name = tempfile.mkstemp( + dir=database.parent, + prefix=f".{database.name}.rollback-", + suffix=".tmp", + ) + os.close(descriptor) + temp_path = Path(temp_name) + try: + shutil.copy2(recovery, temp_path) + with temp_path.open("rb") as handle: + os.fsync(handle.fileno()) + _match_database_ownership(temp_path, database) + for suffix in ("-wal", "-shm"): + try: + Path(f"{database}{suffix}").unlink() + except FileNotFoundError: + pass + os.replace(temp_path, database) + _fsync_directory(database.parent) + validate_restore_database(database) + finally: + try: + temp_path.unlink() + except FileNotFoundError: + pass + + +def apply_pending_database_restore(database_path: str | Path) -> RestoreResult | None: + """Apply a staged restore before any process opens *database_path*.""" + + database = Path(database_path).resolve() + pending = pending_restore_path(database) + if not pending.exists(): + return None + + validate_restore_database(pending) + database.parent.mkdir(parents=True, exist_ok=True) + recovery: Path | None = None + replaced = False + try: + if database.exists(): + recovery = _create_recovery_backup(database) + active_stat = database.stat() + else: + active_stat = None + + _match_database_ownership(pending, database) + for suffix in ("-wal", "-shm"): + sidecar = Path(f"{database}{suffix}") + try: + sidecar.unlink() + except FileNotFoundError: + pass + + os.replace(pending, database) + replaced = True + if active_stat is not None: + os.chmod(database, 0o600 | (stat.S_IMODE(active_stat.st_mode) & 0o060)) + try: + os.chown(database, active_stat.st_uid, active_stat.st_gid) + except (AttributeError, PermissionError): + pass + else: + os.chmod(database, 0o600) + _fsync_directory(database.parent) + validate_restore_database(database) + return RestoreResult(database_path=database, recovery_backup_path=recovery) + except (DatabaseRestoreError, OSError, sqlite3.Error) as exc: + if replaced: + try: + # Preserve the staged candidate for diagnosis/retry before + # putting the previous active database back. Without this, + # a failure after os.replace consumes one member of a + # coordinated Bot/Viewer restore set. + os.replace(database, pending) + _fsync_directory(database.parent) + if recovery is not None: + _rollback_from_recovery(recovery, database) + except Exception as rollback_exc: + raise DatabaseRestoreError( + "Pending restore failed after replacement and automatic recovery also " + f"failed. Recovery backup: {recovery}. Restore error: {exc}; " + f"rollback error: {rollback_exc}" + ) from exc + if recovery is not None: + raise DatabaseRestoreError( + "Pending restore failed; the candidate was re-queued and the " + f"original database was restored from {recovery}: {exc}" + ) from exc + raise DatabaseRestoreError( + "Pending restore failed; the candidate was re-queued and no active " + f"database had existed: {exc}" + ) from exc + if isinstance(exc, DatabaseRestoreError): + raise + raise DatabaseRestoreError(f"Could not apply pending database restore: {exc}") from exc + + +def _configured_database_paths(config_path: str | Path) -> list[Path]: + path = Path(config_path).resolve() + if not path.exists(): + return [] + + config = configparser.ConfigParser() + try: + loaded = config.read(path, encoding="utf-8") + except configparser.Error as exc: + raise DatabaseRestoreError(f"Could not read restore configuration: {exc}") from exc + if not loaded: + return [] + + base = path.parent + local_dir = Path(config.get("Bot", "local_dir_path", fallback="local")) + if not local_dir.is_absolute(): + local_dir = base / local_dir + local_config = local_dir / "config.ini" + if local_config.exists(): + try: + config.read(local_config, encoding="utf-8") + except configparser.Error as exc: + raise DatabaseRestoreError(f"Could not read local restore configuration: {exc}") from exc + + raw_paths = [config.get("Bot", "db_path", fallback="meshcore_bot.db")] + viewer_path = config.get("Web_Viewer", "db_path", fallback="").strip() + if viewer_path: + raw_paths.append(viewer_path) + + resolved: list[Path] = [] + for raw in raw_paths: + if raw == ":memory:": + continue + candidate = Path(raw) + if not candidate.is_absolute(): + candidate = base / candidate + candidate = candidate.resolve() + if candidate not in resolved: + resolved.append(candidate) + return resolved + + +def apply_pending_restores_from_config(config_path: str | Path) -> list[RestoreResult]: + """Apply pending bot/viewer restores before normal startup opens either DB.""" + + database_paths = _configured_database_paths(config_path) + pending_paths = [ + pending_restore_path(path) + for path in database_paths + if pending_restore_path(path).exists() + ] + # Validate the entire set before changing any active database. This avoids + # applying the bot restore only to discover that a split viewer restore is + # corrupt or incompatible. + for pending in pending_paths: + validate_restore_database(pending) + + results: list[RestoreResult] = [] + try: + for database_path in database_paths: + result = apply_pending_database_restore(database_path) + if result is not None: + results.append(result) + except Exception as exc: + rollback_errors: list[str] = [] + for result in reversed(results): + database = result.database_path + pending = pending_restore_path(database) + try: + # Preserve the already-applied candidate so the complete restore + # set can be retried after the failure is corrected. + for suffix in ("-wal", "-shm"): + try: + Path(f"{database}{suffix}").unlink() + except FileNotFoundError: + pass + os.replace(database, pending) + _fsync_directory(database.parent) + validate_restore_database(pending) + if result.recovery_backup_path is not None: + _rollback_from_recovery(result.recovery_backup_path, database) + except Exception as rollback_exc: + rollback_errors.append(f"{database}: {rollback_exc}") + + if rollback_errors: + raise DatabaseRestoreError( + "A configured restore failed and rollback was incomplete. " + f"Restore error: {exc}. Rollback error(s): " + + "; ".join(rollback_errors) + ) from exc + raise DatabaseRestoreError( + "A configured restore failed; every previously applied database was " + f"rolled back and re-queued: {exc}" + ) from exc + return results diff --git a/modules/db_migrations.py b/modules/db_migrations.py index ccd1443..4588509 100644 --- a/modules/db_migrations.py +++ b/modules/db_migrations.py @@ -530,6 +530,28 @@ def _m0014_observed_paths_multibyte_covering_index(cursor: sqlite3.Cursor) -> No ) +def _m0015_channel_operations_claimed_at(cursor: sqlite3.Cursor) -> None: + """Record when a queued hardware/config operation is durably claimed. + + A ``processing`` row is deliberately not auto-requeued: after a process + crash the device may already have applied the operation, so retrying it + automatically could execute a non-idempotent command twice. ``claimed_at`` + gives operators enough information to diagnose and explicitly resolve such + an ambiguous operation. + """ + if _table_exists(cursor, "channel_operations"): + _add_column(cursor, "channel_operations", "claimed_at", "TIMESTAMP") + + +def _m0016_channel_operations_claim_owner(cursor: sqlite3.Cursor) -> None: + """Persist enough local-process identity to recover only provably dead claims.""" + if not _table_exists(cursor, "channel_operations"): + return + _add_column(cursor, "channel_operations", "claim_owner_host", "TEXT") + _add_column(cursor, "channel_operations", "claim_owner_pid", "INTEGER") + _add_column(cursor, "channel_operations", "claim_owner_boot_id", "TEXT") + + # --------------------------------------------------------------------------- # Migration registry — append new entries here, never remove or reorder. # --------------------------------------------------------------------------- @@ -551,6 +573,8 @@ MIGRATIONS: list[MigrationEntry] = [ (12, "purging_log: add details column", _m0012_purging_log_details_column), (13, "observed_paths: advert covering index for contacts page", _m0013_observed_paths_advert_covering_index), (14, "observed_paths: multibyte covering index for mesh graph", _m0014_observed_paths_multibyte_covering_index), + (15, "channel_operations: claimed_at", _m0015_channel_operations_claimed_at), + (16, "channel_operations: claim owner identity", _m0016_channel_operations_claim_owner), ] diff --git a/modules/i18n.py b/modules/i18n.py index 0ac8999..b8a1c22 100644 --- a/modules/i18n.py +++ b/modules/i18n.py @@ -5,6 +5,7 @@ Provides translation functionality for bot commands and responses """ import json +from importlib import resources from pathlib import Path from typing import Any @@ -108,12 +109,20 @@ class Translator: Dictionary of translations, empty dict if file not found """ file_path = Path(self.translation_path) / f"{lang}.json" - if not file_path.exists(): - return {} - try: - with open(file_path, encoding='utf-8') as f: - return json.load(f) + # An explicitly configured filesystem catalog always wins. The + # package fallback makes the defaults work from an installed wheel + # (where ``translations/`` is not relative to the current cwd). + if file_path.is_file(): + with open(file_path, encoding='utf-8') as f: + return json.load(f) + + if not self._uses_bundled_defaults(): + return {} + bundled = resources.files("translations").joinpath(f"{lang}.json") + if not bundled.is_file(): + return {} + return json.loads(bundled.read_text(encoding="utf-8")) except json.JSONDecodeError as e: print(f"Error parsing translation file {file_path}: {e}") return {} @@ -179,7 +188,19 @@ class Translator: if trans_path.exists(): for file in trans_path.glob('*.json'): languages.append(file.stem) - return sorted(languages) + if self._uses_bundled_defaults(): + try: + for resource in resources.files("translations").iterdir(): + if resource.is_file() and resource.name.endswith(".json"): + languages.append(resource.name[:-5]) + except (ModuleNotFoundError, OSError): + pass + return sorted(set(languages)) + + def _uses_bundled_defaults(self) -> bool: + """Return whether ``translation_path`` names the shipped catalog.""" + normalized = str(self.translation_path).replace("\\", "/").rstrip("/") + return normalized == "translations" def get_value(self, key: str) -> Any: """ @@ -210,4 +231,3 @@ class Translator: break return value - diff --git a/modules/scheduler.py b/modules/scheduler.py index ddad033..ab10a90 100644 --- a/modules/scheduler.py +++ b/modules/scheduler.py @@ -9,9 +9,11 @@ import datetime import hashlib import json import os +import socket import sqlite3 import threading import time +import uuid from pathlib import Path from typing import Any, Optional @@ -26,7 +28,24 @@ from .scheduled_message_cron import ( parse_scheduled_message_value, ) from .security_utils import validate_external_url -from .utils import decode_escape_sequences, format_keyword_response_with_placeholders, get_config_timezone +from .utils import ( + decode_escape_sequences, + format_keyword_response_with_placeholders, + get_config_timezone, +) + +_CHANNEL_OPERATION_TYPES = ('add', 'remove') +_RADIO_OPERATION_TYPES = ( + 'radio_reboot', + 'radio_connect', + 'radio_disconnect', + 'firmware_read', + 'firmware_write', + 'radio_params_read', + 'radio_params_write', + 'radio_advert', +) +_CONFIG_OPERATION_TYPES = ('config_reload',) class MessageScheduler: @@ -35,6 +54,12 @@ class MessageScheduler: def __init__(self, bot): self.bot = bot self.logger = bot.logger + self._claim_owner_host = socket.gethostname() + self._claim_owner_pid = os.getpid() + # Unique to this scheduler/process start; retained with the PID to make + # ownership auditable and prevent one scheduler instance finalizing a + # claim made by another instance in the same process. + self._claim_owner_boot_id = uuid.uuid4().hex self.scheduled_messages = {} self.scheduler_thread = None self._apscheduler: Optional[BackgroundScheduler] = None @@ -48,6 +73,10 @@ class MessageScheduler: self.last_db_backup_run = 0 self.last_log_rotation_check_time = 0 self.maintenance = MaintenanceRunner(bot, get_current_time=self.get_current_time) + db_manager = getattr(bot, 'db_manager', None) + db_path = getattr(db_manager, 'db_path', None) + if db_manager is not None and isinstance(db_path, (str, os.PathLike)): + self._recover_interrupted_operations() def get_current_time(self): """Get current time in configured timezone""" @@ -835,106 +864,237 @@ class MessageScheduler: raise RuntimeError(f"send_advert failed: {reason}") self.logger.info("Interval-based flood advert sent successfully") - async def _process_channel_operations(self): - """Process pending channel operations from the web viewer""" + def _claim_operation(self, operation_types: tuple[str, ...]) -> Optional[dict[str, Any]]: + """Atomically claim the oldest pending operation in one serialized group. + + ``BEGIN IMMEDIATE`` prevents two scheduler ticks (or two bot processes) + from selecting the same pending row. A group permits only one + ``processing`` operation at a time, preserving device-operation order + when an earlier command is slow. + + Processing rows are never auto-requeued. If a previous same-host owner + is provably dead at startup, its claim becomes ``interrupted`` so later + work can proceed without replaying the ambiguous action. Live or + unprovable owners continue to block the group conservatively. + """ + if not operation_types: + raise ValueError("operation_types must not be empty") + + placeholders = ', '.join('?' for _ in operation_types) + with self.bot.db_manager.connection() as conn: + conn.row_factory = sqlite3.Row + cursor = conn.cursor() + cursor.execute('BEGIN IMMEDIATE') + + cursor.execute( + f'''SELECT id + FROM channel_operations + WHERE status = 'processing' + AND operation_type IN ({placeholders}) + LIMIT 1''', + operation_types, + ) + if cursor.fetchone() is not None: + conn.commit() + return None + + cursor.execute( + f'''SELECT * + FROM channel_operations + WHERE status = 'pending' + AND operation_type IN ({placeholders}) + ORDER BY created_at ASC, id ASC + LIMIT 1''', + operation_types, + ) + row = cursor.fetchone() + if row is None: + conn.commit() + return None + + cursor.execute( + '''UPDATE channel_operations + SET status = 'processing', + claimed_at = CURRENT_TIMESTAMP, + claim_owner_host = ?, + claim_owner_pid = ?, + claim_owner_boot_id = ?, + processed_at = NULL, + error_message = NULL, + result_data = NULL + WHERE id = ? AND status = 'pending' ''', + ( + self._claim_owner_host, + self._claim_owner_pid, + self._claim_owner_boot_id, + row['id'], + ), + ) + if cursor.rowcount != 1: + conn.rollback() + return None + + conn.commit() + claimed = dict(row) + claimed['status'] = 'processing' + return claimed + + @staticmethod + def _is_local_pid_alive(pid: int) -> Optional[bool]: + """Return PID liveness, or ``None`` when it cannot be proven either way.""" + if not isinstance(pid, int) or pid <= 0: + return None + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + # The kernel found the process but this user cannot signal it. + return True + except OSError: + return None + return True + + def _recover_interrupted_operations(self) -> int: + """Resolve only startup claims provably abandoned by a local process. + + Legacy ownerless rows may be interrupted. An owned row is interrupted + only when it belongs to this host and its PID is provably dead. Live + same-host owners, other hosts, and incomplete/unknown identities stay + blocked conservatively. This method is only invoked during scheduler + construction, never by polling or a timer. + """ + explanation = ( + "Bot restarted while this operation was processing; the action may " + "already have reached the device. Automatic retry is disabled. " + "Verify device state before submitting another operation." + ) try: - # Get pending operations with self.bot.db_manager.connection() as conn: conn.row_factory = sqlite3.Row cursor = conn.cursor() + cursor.execute('BEGIN IMMEDIATE') + cursor.execute( + '''SELECT id, claim_owner_host, claim_owner_pid, + claim_owner_boot_id + FROM channel_operations + WHERE status = 'processing' ''', + ) + rows = cursor.fetchall() + recovered = 0 + blocked = 0 + for row in rows: + owner_host = row['claim_owner_host'] + owner_pid = row['claim_owner_pid'] + owner_boot_id = row['claim_owner_boot_id'] + legacy_ownerless = ( + owner_host is None + and owner_pid is None + and owner_boot_id is None + ) + same_host_dead = ( + owner_host == self._claim_owner_host + and self._is_local_pid_alive(owner_pid) is False + ) + if not legacy_ownerless and not same_host_dead: + blocked += 1 + continue - cursor.execute(''' - SELECT id, operation_type, channel_idx, channel_name, channel_key_hex - FROM channel_operations - WHERE status = 'pending' - AND operation_type IN ('add', 'remove') - ORDER BY created_at ASC - LIMIT 10 - ''') + cursor.execute( + '''UPDATE channel_operations + SET status = 'interrupted', + processed_at = CURRENT_TIMESTAMP, + error_message = ? + WHERE id = ? AND status = 'processing' ''', + (explanation, row['id']), + ) + recovered += cursor.rowcount + conn.commit() + except sqlite3.Error as exc: + self.logger.exception( + "Could not recover interrupted channel/radio operations at startup: %s", + exc, + ) + return 0 - operations = cursor.fetchall() + if recovered: + self.logger.warning( + "Marked %s operation(s) interrupted after restart; verify device state before retrying", + recovered, + ) + if blocked: + self.logger.warning( + "Left %s processing operation(s) blocked because their owner is live or cannot be proven dead", + blocked, + ) + return recovered - if not operations: - return + def _finish_claimed_operation( + self, + op_id: int, + *, + success: bool, + result_payload: Optional[dict[str, Any]] = None, + error_message: Optional[str] = None, + ) -> None: + """Finalize a claim without overwriting externally resolved state.""" + with self.bot.db_manager.connection() as conn: + cursor = conn.cursor() + if success: + cursor.execute( + '''UPDATE channel_operations + SET status = 'completed', + processed_at = CURRENT_TIMESTAMP, + result_data = ?, + error_message = NULL + WHERE id = ? AND status = 'processing' + AND claim_owner_host = ? + AND claim_owner_pid = ? + AND claim_owner_boot_id = ? ''', + ( + json.dumps(result_payload or {'success': True}), + op_id, + self._claim_owner_host, + self._claim_owner_pid, + self._claim_owner_boot_id, + ), + ) + else: + cursor.execute( + '''UPDATE channel_operations + SET status = 'failed', + processed_at = CURRENT_TIMESTAMP, + error_message = ? + WHERE id = ? AND status = 'processing' + AND claim_owner_host = ? + AND claim_owner_pid = ? + AND claim_owner_boot_id = ? ''', + ( + error_message or 'Unknown error', + op_id, + self._claim_owner_host, + self._claim_owner_pid, + self._claim_owner_boot_id, + ), + ) + if cursor.rowcount != 1: + self.logger.warning( + "Operation %s was no longer processing when finalization was attempted", + op_id, + ) + conn.commit() - self.logger.info(f"Processing {len(operations)} pending channel operation(s)") - - for op in operations: - op_id = op['id'] - op_type = op['operation_type'] - channel_idx = op['channel_idx'] - channel_name = op['channel_name'] - channel_key_hex = op['channel_key_hex'] - - try: - success = False - error_msg = None - - if op_type == 'add': - # Add channel - if channel_key_hex: - # Custom channel with key - channel_secret = bytes.fromhex(channel_key_hex) - success = await self.bot.channel_manager.add_channel( - channel_idx, channel_name, channel_secret=channel_secret - ) - else: - # Hashtag channel (firmware generates key) - success = await self.bot.channel_manager.add_channel( - channel_idx, channel_name - ) - - if success: - self.logger.info(f"Successfully processed channel add operation: {channel_name} at index {channel_idx}") - else: - error_msg = "Failed to add channel" - - elif op_type == 'remove': - # Remove channel - success = await self.bot.channel_manager.remove_channel(channel_idx) - - if success: - self.logger.info(f"Successfully processed channel remove operation: index {channel_idx}") - else: - error_msg = "Failed to remove channel" - - # Update operation status - with self.bot.db_manager.connection() as conn: - cursor = conn.cursor() - if success: - cursor.execute(''' - UPDATE channel_operations - SET status = 'completed', - processed_at = CURRENT_TIMESTAMP, - result_data = ? - WHERE id = ? - ''', (json.dumps({'success': True}), op_id)) - else: - cursor.execute(''' - UPDATE channel_operations - SET status = 'failed', - processed_at = CURRENT_TIMESTAMP, - error_message = ? - WHERE id = ? - ''', (error_msg or 'Unknown error', op_id)) - conn.commit() - - except Exception as e: - self.logger.error(f"Error processing channel operation {op_id}: {e}") - # Mark as failed - try: - with self.bot.db_manager.connection() as conn: - cursor = conn.cursor() - cursor.execute(''' - UPDATE channel_operations - SET status = 'failed', - processed_at = CURRENT_TIMESTAMP, - error_message = ? - WHERE id = ? - ''', (str(e), op_id)) - conn.commit() - except Exception as update_error: - self.logger.error(f"Error updating operation status: {update_error}") + async def _process_channel_operations(self): + """Process pending channel operations from the web viewer""" + try: + # Preserve the previous per-tick batch capacity, but claim only one + # row immediately before executing it. The next claim cannot + # succeed until the previous row has reached a terminal state. + for _ in range(10): + op = self._claim_operation(_CHANNEL_OPERATION_TYPES) + if not op: + return + await self._execute_claimed_channel_operation(op) except Exception as e: db_path = getattr(self.bot.db_manager, 'db_path', 'unknown') @@ -950,27 +1110,60 @@ class MessageScheduler: else: self.logger.error(f"Database path: {db_path_str}") + async def _execute_claimed_channel_operation(self, op: dict[str, Any]) -> None: + """Execute and finalize one already-claimed channel operation.""" + op_id = op['id'] + op_type = op['operation_type'] + channel_idx = op['channel_idx'] + channel_name = op['channel_name'] + channel_key_hex = op['channel_key_hex'] + self.logger.info("Processing claimed channel operation %s: %s", op_id, op_type) + + try: + success = False + error_msg = None + + if op_type == 'add': + if channel_key_hex: + channel_secret = bytes.fromhex(channel_key_hex) + success = await self.bot.channel_manager.add_channel( + channel_idx, channel_name, channel_secret=channel_secret + ) + else: + success = await self.bot.channel_manager.add_channel( + channel_idx, channel_name + ) + + if success: + self.logger.info(f"Successfully processed channel add operation: {channel_name} at index {channel_idx}") + else: + error_msg = "Failed to add channel" + + elif op_type == 'remove': + success = await self.bot.channel_manager.remove_channel(channel_idx) + + if success: + self.logger.info(f"Successfully processed channel remove operation: index {channel_idx}") + else: + error_msg = "Failed to remove channel" + + self._finish_claimed_operation( + op_id, + success=success, + error_message=error_msg, + ) + + except Exception as e: + self.logger.error(f"Error processing channel operation {op_id}: {e}") + try: + self._finish_claimed_operation(op_id, success=False, error_message=str(e)) + except Exception as update_error: + self.logger.error(f"Error updating operation status: {update_error}") + async def _process_radio_operations(self): """Process pending radio connect/disconnect/reboot/firmware operations from the web viewer.""" try: - with self.bot.db_manager.connection() as conn: - conn.row_factory = sqlite3.Row - cursor = conn.cursor() - cursor.execute(''' - SELECT id, operation_type, payload_data - FROM channel_operations - WHERE status = 'pending' - AND operation_type IN ( - 'radio_reboot', 'radio_connect', 'radio_disconnect', - 'firmware_read', 'firmware_write', - 'radio_params_read', 'radio_params_write', - 'radio_advert' - ) - ORDER BY created_at ASC - LIMIT 1 - ''') - op = cursor.fetchone() - + op = self._claim_operation(_RADIO_OPERATION_TYPES) if not op: return @@ -1002,41 +1195,19 @@ class MessageScheduler: else: success = False - with self.bot.db_manager.connection() as conn: - cursor = conn.cursor() - if success: - cursor.execute(''' - UPDATE channel_operations - SET status = 'completed', - processed_at = CURRENT_TIMESTAMP, - result_data = ? - WHERE id = ? - ''', (json.dumps(result_payload), op_id)) - else: - error_msg = result_payload.get('error', 'Radio operation returned False') \ - if isinstance(result_payload, dict) else 'Radio operation returned False' - cursor.execute(''' - UPDATE channel_operations - SET status = 'failed', - processed_at = CURRENT_TIMESTAMP, - error_message = ? - WHERE id = ? - ''', (error_msg, op_id)) - conn.commit() + error_msg = result_payload.get('error', 'Radio operation returned False') \ + if isinstance(result_payload, dict) else 'Radio operation returned False' + self._finish_claimed_operation( + op_id, + success=success, + result_payload=result_payload, + error_message=error_msg, + ) except Exception as e: self.logger.error(f"Error executing radio operation {op_id}: {e}") try: - with self.bot.db_manager.connection() as conn: - cursor = conn.cursor() - cursor.execute(''' - UPDATE channel_operations - SET status = 'failed', - processed_at = CURRENT_TIMESTAMP, - error_message = ? - WHERE id = ? - ''', (str(e), op_id)) - conn.commit() + self._finish_claimed_operation(op_id, success=False, error_message=str(e)) except Exception as update_error: self.logger.error(f"Error updating radio operation status: {update_error}") @@ -1052,17 +1223,7 @@ class MessageScheduler: start/stop is not handled by reload_config and still needs a restart. """ try: - with self.bot.db_manager.connection() as conn: - conn.row_factory = sqlite3.Row - cursor = conn.cursor() - cursor.execute(''' - SELECT id FROM channel_operations - WHERE status = 'pending' AND operation_type = 'config_reload' - ORDER BY created_at ASC - LIMIT 1 - ''') - op = cursor.fetchone() - + op = self._claim_operation(_CONFIG_OPERATION_TYPES) if not op: return @@ -1075,25 +1236,12 @@ class MessageScheduler: success, message = False, str(e) self.logger.exception("Error during config reload") - with self.bot.db_manager.connection() as conn: - cursor = conn.cursor() - if success: - cursor.execute(''' - UPDATE channel_operations - SET status = 'completed', - processed_at = CURRENT_TIMESTAMP, - result_data = ? - WHERE id = ? - ''', (json.dumps({'success': True, 'message': message}), op_id)) - else: - cursor.execute(''' - UPDATE channel_operations - SET status = 'failed', - processed_at = CURRENT_TIMESTAMP, - error_message = ? - WHERE id = ? - ''', (message, op_id)) - conn.commit() + self._finish_claimed_operation( + op_id, + success=success, + result_payload={'success': True, 'message': message}, + error_message=message, + ) self.logger.info("Config reload %s: %s", 'succeeded' if success else 'failed', message) @@ -1645,4 +1793,3 @@ class MessageScheduler: self.bot.logger.error(f"Failed to send radio-offline alert email: {e}") # ── Maintenance helpers ────────────────────────────────────────────────── - diff --git a/modules/web_viewer/app.py b/modules/web_viewer/app.py index c6c763c..cc193f6 100644 --- a/modules/web_viewer/app.py +++ b/modules/web_viewer/app.py @@ -9,6 +9,7 @@ import json import logging import os import re +import secrets import sqlite3 import sys import threading @@ -31,6 +32,7 @@ from flask import ( Response, abort, current_app, + g, jsonify, make_response, redirect, @@ -42,6 +44,11 @@ from flask import ( ) from flask_socketio import SocketIO, disconnect, emit +from modules.database_restore import ( + DEFAULT_MAX_RESTORE_BYTES, + DatabaseRestoreError, + stage_database_restore, +) from modules.ini_writer import update_ini_values from modules.security_utils import ( VALID_JOURNAL_MODES, @@ -620,6 +627,11 @@ class BotDataViewer: '/site.webmanifest', '/favicon.ico', ]) + @self.app.before_request + def create_csp_nonce(): + """Create a per-response nonce for templates migrated off inline-script CSP.""" + g.csp_nonce = secrets.token_urlsafe(24) + @self.app.before_request def require_auth(): if not self.web_viewer_password: @@ -664,10 +676,33 @@ class BotDataViewer: response.headers['Referrer-Policy'] = 'strict-origin-when-cross-origin' # Allow CDNs used by templates (base.html, login.html, mesh.html). # Without these hosts, browsers block external CSS/JS/fonts (not CSRF). + # The highest-risk admin screens have migrated their inline handlers + # and authorize their remaining template scripts with a per-request + # nonce. Other legacy screens retain unsafe-inline until their inline + # scripts/handlers are migrated, rather than silently breaking them. + nonce_hardened_endpoints = { + 'index', + 'feeds', + 'config_page', + 'radio', + 'realtime', + 'contacts', + 'stats', + 'plugins_page', + 'greeter', + 'logs', + 'multibyte_rollout', + 'mesh', + 'api_explorer', + } + if request.endpoint in nonce_hardened_endpoints: + script_source = f"script-src 'self' 'nonce-{g.csp_nonce}' " + else: + script_source = "script-src 'self' 'unsafe-inline' " response.headers['Content-Security-Policy'] = ( "default-src 'self'; " - "script-src 'self' 'unsafe-inline' " - "https://cdn.jsdelivr.net https://cdnjs.cloudflare.com https://unpkg.com; " + + script_source + + "https://cdn.jsdelivr.net https://cdnjs.cloudflare.com https://unpkg.com; " "style-src 'self' 'unsafe-inline' " "https://cdn.jsdelivr.net https://cdnjs.cloudflare.com https://unpkg.com; " "img-src 'self' data: https://*.tile.openstreetmap.org " @@ -1535,10 +1570,11 @@ class BotDataViewer: @self.app.route('/api/maintenance/restore', methods=['POST']) def api_maintenance_restore(): - """Restore DB from a backup file. + """Stage a verified DB backup for the next full service restart. Body: {"db_file": "/absolute/path/to/backup.db"} - The active DB is overwritten; the caller must restart the bot. + The active DB is never modified by this request. Normal bot startup + applies the sibling pending file before any DB connection is opened. """ try: data = request.get_json(silent=True) or {} @@ -1574,25 +1610,37 @@ class BotDataViewer: if not src.exists(): return jsonify({'error': f'File not found: {db_file}'}), 400 - # Validate it is a real SQLite file by checking the magic header - _SQLITE_MAGIC = b"SQLite format 3\x00" try: - with open(str(src), 'rb') as _fh: - _header = _fh.read(16) - if _header != _SQLITE_MAGIC: - raise ValueError("bad magic") - except Exception: - return jsonify({'error': f'Not a valid SQLite file: {db_file}'}), 400 - # Copy to active DB path - import shutil - shutil.copy2(str(src), self.db_path) - self.logger.info(f"Database restored from {src} to {self.db_path}") + max_restore_bytes = self.config.getint( + 'Web_Viewer', + 'restore_max_bytes', + fallback=DEFAULT_MAX_RESTORE_BYTES, + ) + pending = stage_database_restore( + src, + self.db_path, + max_bytes=max_restore_bytes, + ) + except (DatabaseRestoreError, OSError, ValueError) as exc: + self.logger.warning("Database restore staging rejected for %s: %s", src, exc) + return jsonify({'error': str(exc)}), 400 + + self.logger.warning( + "Database restore staged from %s at %s; a full service restart is required", + src, + pending, + ) return jsonify({ 'success': True, - 'restored_from': db_file, + 'staged_from': db_file, + 'pending_path': str(pending), 'active_db': self.db_path, - 'warning': 'Restart the bot for the restored database to take effect.', - }) + 'requires_restart': True, + 'warning': ( + 'Restore verified and staged. Restart the complete MeshCore Bot service ' + 'to apply it before any database writer starts.' + ), + }), 202 except Exception as e: self.logger.error(f"Error in restore: {e}", exc_info=True) return jsonify({'error': str(e)}), 500 @@ -3430,7 +3478,7 @@ class BotDataViewer: conn = self._get_db_connection() cursor = conn.cursor() cursor.execute(''' - SELECT status, error_message, result_data, processed_at + SELECT status, error_message, result_data, processed_at, claimed_at FROM channel_operations WHERE id = ? ''', (operation_id,)) @@ -3440,12 +3488,13 @@ class BotDataViewer: if not result: return jsonify({'error': 'Operation not found'}), 404 - status, error_msg, result_data, processed_at = result + status, error_msg, result_data, processed_at, claimed_at = result return jsonify({ 'operation_id': operation_id, 'status': status, 'error_message': error_msg, + 'claimed_at': claimed_at, 'processed_at': processed_at, 'result_data': json.loads(result_data) if result_data else None }) diff --git a/modules/web_viewer/static/js/channel_operations.js b/modules/web_viewer/static/js/channel_operations.js index dd6f0c7..bb15d55 100644 --- a/modules/web_viewer/static/js/channel_operations.js +++ b/modules/web_viewer/static/js/channel_operations.js @@ -34,7 +34,8 @@ } /** - * Poll GET /api/channel-operations/:id until completed, failed, or timeouts. + * Poll GET /api/channel-operations/:id until completed, failed, + * interrupted, or timeouts. * Behavior aligned with legacy radio.html pollOperationStatus. * * @param {number} operationId @@ -48,7 +49,7 @@ * @param {number} [options.maxWaitSeconds=60] * @param {number} [options.extendedMaxWaitSeconds=120] * @param {number} [options.checkIntervalMs=1000] - * @returns {Promise<'completed'|'failed'|'timeout'>} + * @returns {Promise<'completed'|'failed'|'interrupted'|'timeout'>} */ function pollChannelOperation(operationId, options) { options = options || {}; @@ -95,12 +96,12 @@ if (result.status === 'completed') { return 'completed'; } - if (result.status === 'failed') { + if (result.status === 'failed' || result.status === 'interrupted') { if (typeof options.onFailed === 'function') { options.onFailed(result.error_message || 'Channel operation failed'); } resetButton(); - return 'failed'; + return result.status; } var elapsed = Math.floor((Date.now() - startTime) / 1000); setBtn( @@ -122,12 +123,12 @@ if (result2.status === 'completed') { return 'completed'; } - if (result2.status === 'failed') { + if (result2.status === 'failed' || result2.status === 'interrupted') { if (typeof options.onFailed === 'function') { options.onFailed(result2.error_message || 'Channel operation failed'); } resetButton(); - return 'failed'; + return result2.status; } } catch (error) { console.error('Error checking final status:', error); @@ -149,12 +150,12 @@ if (result3.status === 'completed') { return 'completed'; } - if (result3.status === 'failed') { + if (result3.status === 'failed' || result3.status === 'interrupted') { if (typeof options.onFailed === 'function') { options.onFailed(result3.error_message || 'Channel operation failed'); } resetButton(); - return 'failed'; + return result3.status; } var elapsed2 = Math.floor((Date.now() - startTime) / 1000); setBtn( diff --git a/modules/web_viewer/templates/api_explorer.html b/modules/web_viewer/templates/api_explorer.html index 72d5592..810c187 100644 --- a/modules/web_viewer/templates/api_explorer.html +++ b/modules/web_viewer/templates/api_explorer.html @@ -227,7 +227,7 @@ {% endblock %} {% block extra_js %} - - {% endblock %} - diff --git a/modules/web_viewer/templates/greeter.html b/modules/web_viewer/templates/greeter.html index 7c8bc58..9ee39a7 100644 --- a/modules/web_viewer/templates/greeter.html +++ b/modules/web_viewer/templates/greeter.html @@ -106,7 +106,7 @@ {% endblock %} {% block extra_js %} - {% endblock %} - diff --git a/modules/web_viewer/templates/index.html b/modules/web_viewer/templates/index.html index 4a64e12..644f8d5 100644 --- a/modules/web_viewer/templates/index.html +++ b/modules/web_viewer/templates/index.html @@ -61,8 +61,7 @@

0 + data-bs-toggle="modal" data-bs-target="#connectedClientsModal">0

Connected Clients @@ -429,17 +428,17 @@ - - 0 - - @@ -494,7 +493,7 @@ {% endblock %} {% block extra_js %} - - {% endblock %} diff --git a/modules/web_viewer/templates/mesh.html b/modules/web_viewer/templates/mesh.html index 10432b6..7fcdf21 100644 --- a/modules/web_viewer/templates/mesh.html +++ b/modules/web_viewer/templates/mesh.html @@ -173,10 +173,10 @@
- -
@@ -184,19 +184,19 @@ - - - - -
@@ -214,12 +214,12 @@
- + 1
- @@ -231,7 +231,7 @@
- @@ -243,7 +243,7 @@
- @@ -252,14 +252,14 @@
- +
- +
@@ -619,7 +619,7 @@ - {% endblock %} diff --git a/modules/web_viewer/templates/stats.html b/modules/web_viewer/templates/stats.html index ad88653..6523892 100644 --- a/modules/web_viewer/templates/stats.html +++ b/modules/web_viewer/templates/stats.html @@ -149,7 +149,7 @@ {% endblock %} {% block extra_js %} -"), + ("path", "' onmouseover='window.pathOwned=1"), + ("error", ""), + ], +) +@pytest.mark.parametrize("template_name", ["radio.html", "realtime.html"]) +def test_radio_and_realtime_payload_classes_never_reach_an_html_parser( + template_name: str, + payload_class: str, + payload: str, +) -> None: + """The representative payload is documentation for each trust class. + + These screens receive the values asynchronously, so the regression guard + is static: there must be no API-to-parser sink capable of interpreting any + payload as markup, regardless of its exact spelling. + """ + assert payload_class + assert payload + source = _source(template_name) + for parser_sink in ( + ".innerHTML", + ".outerHTML", + "insertAdjacentHTML", + "document.write", + ".onclick", + ): + assert parser_sink not in source, (template_name, payload_class, parser_sink) + + +def test_radio_dynamic_channel_feed_key_and_error_values_use_dom_text_nodes() -> None: + source = _source("radio.html") + for safe_sink in ( + "cell.textContent = String(value ?? '')", + "link.textContent = String(feed.feed_name || feed.feed_url || 'Unnamed feed')", + "code.textContent = String(channelKey)", + "messageNode.textContent = String(message ?? '')", + "viewButton.addEventListener('click'", + "deleteButton.addEventListener('click'", + ): + assert safe_sink in source + + +def test_all_direct_radio_pollers_treat_interrupted_as_terminal() -> None: + source = _source("radio.html") + + assert source.count("result.status === 'interrupted'") == 2 + assert "data.status === 'interrupted'" in source + + +def test_realtime_mesh_user_message_path_and_error_values_use_dom_text_nodes() -> None: + source = _source("realtime.html") + for safe_sink in ( + "element.textContent = String(text)", + "data.channel\n )", + "makeElement('strong', '', data.sender || '?')", + "entry.append(header, makeElement('div', 'text-break', bodyRaw))", + "makeElement('strong', 'text-primary me-2', data.user || 'Unknown')", + "makeElement('div', 'response-text', data.response || 'No response')", + "document.createTextNode(` ${String(pathDisplay)}", + "document.getElementById('decoder-error-message').textContent = decodeError", + ): + assert safe_sink in source diff --git a/translations/__init__.py b/translations/__init__.py new file mode 100644 index 0000000..3f95068 --- /dev/null +++ b/translations/__init__.py @@ -0,0 +1,6 @@ +"""Bundled MeshCore Bot translation catalogs. + +The JSON files in this package are runtime data. Keeping them in an importable +package lets installed wheels load the defaults through ``importlib.resources`` +without depending on the process working directory. +"""