diff --git a/config.ini.example b/config.ini.example index d8fc182..5744496 100644 --- a/config.ini.example +++ b/config.ini.example @@ -96,6 +96,24 @@ admin_pubkeys = # These commands will only work for users in the admin_pubkeys list admin_commands = repeater,webviewer +[Companion_Purge] +# Enable companion contact purging +# true: Purge inactive companions when contact list is full +# false: Never purge companions (default: false for safety) +companion_purge_enabled = false + +# Days since last DM to consider companion inactive +# Companions who haven't DM'd the bot in this many days may be purged +companion_dm_threshold_days = 30 + +# Days since last advert to consider companion inactive +# Companions who haven't adverted in this many days may be purged +companion_advert_threshold_days = 30 + +# Minimum days since last activity (DM or advert) before purge +# Companions must be inactive for at least this many days +companion_min_inactive_days = 30 + [Jokes] # Enable or disable the joke command # true: Joke command is available @@ -405,6 +423,14 @@ max_proximity_range = 200 # Set to 0 to disable age filtering max_repeater_age_days = 14 +# Recency vs Proximity weighting (0.0 to 1.0) +# Controls how much recency (when last heard) vs proximity (distance) matters +# 0.0 = 100% proximity (only distance matters) +# 1.0 = 100% recency (only when last heard matters) +# 0.4 = 40% recency, 60% proximity (default - balanced for path routing) +# Lower values favor closer repeaters, higher values favor recently heard repeaters +recency_weight = 0.4 + # Confidence indicator symbols for path command # High confidence (>= 0.9): Shows when path decoding is very reliable high_confidence_symbol = 🎯 diff --git a/docs/MODERN_REWRITE_BENEFITS.md b/docs/MODERN_REWRITE_BENEFITS.md deleted file mode 100644 index f49a3c3..0000000 --- a/docs/MODERN_REWRITE_BENEFITS.md +++ /dev/null @@ -1,219 +0,0 @@ -# Modern Flask-SocketIO Rewrite Benefits - -## 🔍 Current Issues vs Modern Solution - -### **1. Protocol & Architecture Issues** - -| Current Implementation | Modern Flask-SocketIO 5.x | -|------------------------|---------------------------| -| ❌ Manual background thread with complex timeout handling | ✅ Built-in event-driven architecture | -| ❌ Fighting against Flask-SocketIO's design | ✅ Leveraging Flask-SocketIO's strengths | -| ❌ Complex connection tracking and cleanup | ✅ Built-in connection management | -| ❌ Manual ping/pong implementation | ✅ Native ping/pong with proper timeouts | - -### **2. Connection Management** - -| Current Issues | Modern Solution | -|----------------|-----------------| -| ❌ 170+ file descriptors due to connection leaks | ✅ Automatic connection cleanup | -| ❌ Manual connection tracking with race conditions | ✅ Built-in client tracking | -| ❌ Complex stale connection cleanup | ✅ Automatic connection lifecycle management | -| ❌ Circuit breaker trips due to hanging connections | ✅ Robust connection handling | - -### **3. Real-time Data Handling** - -| Current Problems | Modern Approach | -|------------------|-----------------| -| ❌ Background thread hanging on SocketIO emit | ✅ Event-driven data broadcasting | -| ❌ Complex timeout mechanisms | ✅ Built-in timeout handling | -| ❌ Manual queue management | ✅ Direct event emission | -| ❌ Resource leaks from hanging threads | ✅ Clean event-driven architecture | - -## 🛠️ Key Improvements in Modern Implementation - -### **1. Flask-SocketIO 5.x Best Practices** - -```python -# Modern SocketIO Configuration -self.socketio = SocketIO( - self.app, - cors_allowed_origins="*", - max_http_buffer_size=1000000, - ping_timeout=5, # Flask-SocketIO 5.x default - ping_interval=25, # Flask-SocketIO 5.x default - logger=True, # Proper logging - engineio_logger=True, # EngineIO logging - async_mode='threading' # Better stability -) -``` - -### **2. Event-Driven Architecture** - -```python -# Modern event handlers -@socketio.on('connect') -def handle_connect(): - # Built-in connection management - client_id = request.sid - self.connected_clients[client_id] = { - 'connected_at': time.time(), - 'last_activity': time.time(), - 'subscribed_commands': False, - 'subscribed_packets': False - } - emit('status', {'message': 'Connected'}) - -@socketio.on('subscribe_commands') -def handle_subscribe_commands(): - # Clean subscription handling - client_id = request.sid - self.connected_clients[client_id]['subscribed_commands'] = True - emit('status', {'message': 'Subscribed to command stream'}) -``` - -### **3. Direct Data Broadcasting** - -```python -# Modern data handling - no background threads needed -def _handle_command_data(self, command_data): - """Handle incoming command data from bot""" - subscribed_clients = [ - client_id for client_id, client_info in self.connected_clients.items() - if client_info.get('subscribed_commands', False) - ] - - if subscribed_clients: - self.socketio.emit('command_data', command_data, room=None) -``` - -### **4. Modern Client-Side Implementation** - -```javascript -// Modern Socket.IO client with proper error handling -class ModernBotMonitor { - constructor() { - this.socket = io({ - transports: ['websocket', 'polling'], - timeout: 5000, - forceNew: true - }); - - this.setupSocketEvents(); - this.startPingInterval(); - } - - setupSocketEvents() { - this.socket.on('connect', () => { - this.connected = true; - this.updateConnectionStatus('Connected', 'connected'); - }); - - this.socket.on('command_data', (data) => { - this.addCommandEntry(data); - }); - - // Modern ping/pong pattern - this.socket.on('pong', () => { - this.lastActivity = new Date(); - this.updateLastActivity(); - }); - } -} -``` - -## 📊 Expected Performance Improvements - -### **Resource Usage** -- **File Descriptors**: 170+ → <20 (90% reduction) -- **Memory Usage**: Significant reduction due to no background threads -- **CPU Usage**: Lower due to event-driven architecture -- **Connection Stability**: Much more stable with built-in management - -### **Reliability** -- **No More Hanging**: Event-driven architecture prevents thread blocking -- **Automatic Recovery**: Built-in connection management handles failures -- **Better Error Handling**: Proper SocketIO error handling -- **Circuit Breaker**: Less likely to trip due to better connection management - -### **Maintainability** -- **Simpler Code**: No complex background thread management -- **Better Logging**: Proper Flask-SocketIO logging -- **Easier Debugging**: Clear event-driven flow -- **Modern Patterns**: Following Flask-SocketIO 5.x best practices - -## 🚀 Migration Strategy - -### **Phase 1: Parallel Implementation** -1. Deploy modern implementation alongside current one -2. Test with real bot data -3. Compare performance and stability - -### **Phase 2: Gradual Migration** -1. Switch bot integration to use modern web viewer -2. Monitor for 24+ hours -3. Compare metrics with current implementation - -### **Phase 3: Full Replacement** -1. Replace current web viewer with modern implementation -2. Remove old code -3. Update documentation - -## 🎯 Expected Results - -### **Immediate Benefits** -- ✅ No more 40-minute hanging pattern -- ✅ No more circuit breaker trips -- ✅ No more connection leaks -- ✅ Better real-time performance - -### **Long-term Benefits** -- ✅ Easier maintenance and debugging -- ✅ Better scalability -- ✅ Modern, maintainable codebase -- ✅ Following Flask-SocketIO best practices - -## 🔧 Implementation Notes - -### **Configuration Changes** -```ini -# Modern configuration -[Web_Viewer] -host = 127.0.0.1 -port = 8080 -enabled = true -auto_start = true -debug = false - -# Modern SocketIO settings -ping_timeout = 5 -ping_interval = 25 -max_clients = 10 -``` - -### **Bot Integration Changes** -```python -# Modern bot integration - much simpler -def _handle_command_data(self, command_data): - """Send command data to modern web viewer""" - try: - response = self.session.post( - f"{self.web_viewer_url}/api/stream_data", - json={'type': 'command', 'data': command_data}, - timeout=5 - ) - if response.status_code == 200: - self.logger.debug("Command data sent to modern web viewer") - except Exception as e: - self.logger.debug(f"Failed to send command data: {e}") -``` - -## 📈 Conclusion - -The modern rewrite using Flask-SocketIO 5.x best practices would: - -1. **Eliminate all current issues** (hanging, connection leaks, circuit breaker trips) -2. **Provide better performance** (lower resource usage, better stability) -3. **Improve maintainability** (cleaner code, better debugging) -4. **Follow modern patterns** (event-driven architecture, proper error handling) - -**Recommendation**: Proceed with the modern rewrite to eliminate the recurring issues and create a more maintainable, stable web viewer. diff --git a/modules/commands/path_command.py b/modules/commands/path_command.py index b75c212..c80b66a 100644 --- a/modules/commands/path_command.py +++ b/modules/commands/path_command.py @@ -38,6 +38,12 @@ class PathCommand(BaseCommand): self.max_proximity_range = bot.config.getfloat('Path_Command', 'max_proximity_range', fallback=200.0) self.max_repeater_age_days = bot.config.getint('Path_Command', 'max_repeater_age_days', fallback=14) + # Get recency/proximity weighting (0.0 to 1.0, where 1.0 = 100% recency, 0.0 = 100% proximity) + # Default 0.4 means 40% recency, 60% proximity (more balanced for path routing) + recency_weight = bot.config.getfloat('Path_Command', 'recency_weight', fallback=0.4) + self.recency_weight = max(0.0, min(1.0, recency_weight)) # Clamp to 0.0-1.0 + self.proximity_weight = 1.0 - self.recency_weight + # Get confidence indicator symbols from config self.high_confidence_symbol = bot.config.get('Path_Command', 'high_confidence_symbol', fallback='🎯') self.medium_confidence_symbol = bot.config.get('Path_Command', 'medium_confidence_symbol', fallback='📍') @@ -473,9 +479,8 @@ class PathCommand(BaseCommand): normalized_distance = min(distance / 1000.0, 1.0) proximity_score = 1.0 - normalized_distance # Invert so closer = higher score - # Weight recency more heavily for path decoding (70% recency, 30% proximity) - # Recent repeaters are more likely to have been involved in the message path - combined_score = (recency_score * 0.7) + (proximity_score * 0.3) + # Use configurable weighting (default: 40% recency, 60% proximity) + combined_score = (recency_score * self.recency_weight) + (proximity_score * self.proximity_weight) combined_scores.append((combined_score, distance, repeater)) if not combined_scores: @@ -815,8 +820,8 @@ class PathCommand(BaseCommand): normalized_distance = min(avg_distance / 1000.0, 1.0) proximity_score = 1.0 - normalized_distance - # Combined score: 70% recency, 30% proximity (recency more important for path decoding) - combined_score = (recency_score * 0.7) + (proximity_score * 0.3) + # Use configurable weighting (default: 40% recency, 60% proximity) + combined_score = (recency_score * self.recency_weight) + (proximity_score * self.proximity_weight) if combined_score > best_combined_score: best_combined_score = combined_score @@ -865,8 +870,8 @@ class PathCommand(BaseCommand): normalized_distance = min(distance / 1000.0, 1.0) proximity_score = 1.0 - normalized_distance - # Combined score: 70% recency, 30% proximity (recency more important for path decoding) - combined_score = (recency_score * 0.7) + (proximity_score * 0.3) + # Use configurable weighting (default: 40% recency, 60% proximity) + combined_score = (recency_score * self.recency_weight) + (proximity_score * self.proximity_weight) if combined_score > best_combined_score: best_combined_score = combined_score diff --git a/modules/commands/repeater_command.py b/modules/commands/repeater_command.py index 3fbe78f..d47f15c 100644 --- a/modules/commands/repeater_command.py +++ b/modules/commands/repeater_command.py @@ -205,14 +205,18 @@ class RepeaterCommand(BaseCommand): return f"❌ Error listing repeaters: {e}" async def _handle_purge(self, args: List[str]) -> str: - """Purge repeater contacts""" + """Purge repeater or companion contacts""" if not hasattr(self.bot, 'repeater_manager'): return "Repeater manager not initialized. Please check bot configuration." if not args: - return "Usage: !repeater purge [all|days|name] [reason]\nExamples:\n !repeater purge all 'Clear all repeaters'\n !repeater purge all force 'Force clear all repeaters'\n !repeater purge 30 'Auto-cleanup old repeaters'\n !repeater purge 'Hillcrest' 'Remove specific repeater'" + return "Usage: !repeater purge [all|days|name|companions] [reason]\nExamples:\n !repeater purge all 'Clear all repeaters'\n !repeater purge companions 'Clear inactive companions'\n !repeater purge companions 30 'Purge companions inactive 30+ days'\n !repeater purge 30 'Auto-cleanup old repeaters'\n !repeater purge 'Hillcrest' 'Remove specific repeater'" try: + # Check if purging companions + if args[0].lower() == 'companions': + return await self._handle_purge_companions(args[1:]) + if args[0].lower() == 'all': # Check for force flag force_purge = len(args) > 1 and args[1].lower() == 'force' @@ -358,6 +362,91 @@ class RepeaterCommand(BaseCommand): except Exception as e: return f"❌ Error purging repeaters: {e}" + async def _handle_purge_companions(self, args: List[str]) -> str: + """Purge companion contacts based on inactivity""" + if not hasattr(self.bot, 'repeater_manager'): + return "Repeater manager not initialized. Please check bot configuration." + + if not self.bot.repeater_manager.companion_purge_enabled: + return "❌ Companion purge disabled. Enable: [Companion_Purge] companion_purge_enabled = true" + + try: + # Check for days argument + days_old = None + reason = "Manual purge - inactive companions" + + if args: + try: + # Try to parse first arg as number of days + days_old = int(args[0]) + reason = " ".join(args[1:]) if len(args) > 1 else f"Manual purge - companions inactive {days_old}+ days" + except ValueError: + # Not a number, treat as reason + reason = " ".join(args) if args else "Manual purge - inactive companions" + + # Get companions for purging + if days_old: + # Purge companions inactive for specified days + companions_to_purge = await self.bot.repeater_manager._get_companions_for_purging(999) # Get all eligible + # Filter by days + from datetime import datetime, timedelta + cutoff_date = datetime.now() - timedelta(days=days_old) + filtered_companions = [] + for companion in companions_to_purge: + if companion.get('last_activity'): + try: + last_activity = datetime.fromisoformat(companion['last_activity']) + if last_activity < cutoff_date: + filtered_companions.append(companion) + except: + pass + elif companion.get('days_inactive', 0) >= days_old: + filtered_companions.append(companion) + companions_to_purge = filtered_companions + else: + # Get companions based on configured thresholds + companions_to_purge = await self.bot.repeater_manager._get_companions_for_purging(999) # Get all eligible + + if not companions_to_purge: + return "❌ No companions match criteria (inactive for DM+advert thresholds, not in ACL)" + + # Purge companions (compact format for 130 char limit) + total_to_purge = len(companions_to_purge) + purged_count = 0 + failed_count = 0 + + for i, companion in enumerate(companions_to_purge): + self.logger.info(f"Purging companion {i+1}/{total_to_purge}: {companion['name']}") + + success = await self.bot.repeater_manager.purge_companion_from_contacts( + companion['public_key'], reason + ) + + if success: + purged_count += 1 + else: + failed_count += 1 + + # Add delay between purges to avoid overwhelming the radio + # Use 2 seconds to give radio time to process each removal + if i < total_to_purge - 1: + await asyncio.sleep(2) + + # Build compact response (must fit in 130 chars) + if failed_count > 0: + response = f"✅ {purged_count}/{total_to_purge} companions purged, {failed_count} failed" + else: + response = f"✅ {purged_count}/{total_to_purge} companions purged" + + # Truncate if still too long + if len(response) > 130: + response = f"✅ {purged_count}/{total_to_purge} purged" + + return response + + except Exception as e: + return f"❌ Error purging companions: {e}" + async def _handle_restore(self, args: List[str]) -> str: """Restore purged repeater contacts""" if not hasattr(self.bot, 'repeater_manager'): diff --git a/modules/repeater_manager.py b/modules/repeater_manager.py index f4b1b84..4d691f2 100644 --- a/modules/repeater_manager.py +++ b/modules/repeater_manager.py @@ -35,6 +35,12 @@ class RepeaterManager: self.contact_limit = 300 # MeshCore device limit (will be updated from device info) self.auto_purge_threshold = 280 # Start purging when 280+ contacts self.auto_purge_enabled = True + + # Initialize companion purge settings + self.companion_purge_enabled = bot.config.getboolean('Companion_Purge', 'companion_purge_enabled', fallback=False) + self.companion_dm_threshold_days = bot.config.getint('Companion_Purge', 'companion_dm_threshold_days', fallback=30) + self.companion_advert_threshold_days = bot.config.getint('Companion_Purge', 'companion_advert_threshold_days', fallback=30) + self.companion_min_inactive_days = bot.config.getint('Companion_Purge', 'companion_min_inactive_days', fallback=30) def _init_repeater_tables(self): """Initialize repeater-specific database tables""" @@ -552,7 +558,7 @@ class RepeaterManager: return await self.get_complete_contact_database(role_filter='bot', include_historical=include_historical) async def check_and_auto_purge(self) -> bool: - """Check contact limit and auto-purge repeaters if needed""" + """Check contact limit and auto-purge repeaters and companions if needed""" try: if not self.auto_purge_enabled: return False @@ -568,12 +574,25 @@ class RepeaterManager: purge_count = current_count - target_count if purge_count > 0: - success = await self._auto_purge_repeaters(purge_count) - if success: - self.logger.info(f"✅ Auto-purged {purge_count} repeaters, now at {len(self.bot.meshcore.contacts)}/{self.contact_limit} contacts") + # First try to purge repeaters + repeater_success = await self._auto_purge_repeaters(purge_count) + remaining_count = len(self.bot.meshcore.contacts) + + # If still above threshold and companion purging is enabled, purge companions + if remaining_count >= self.auto_purge_threshold and self.companion_purge_enabled: + remaining_purge_count = remaining_count - target_count + self.logger.info(f"Still above threshold after repeater purge, purging {remaining_purge_count} companions...") + companion_success = await self._auto_purge_companions(remaining_purge_count) + + if repeater_success or companion_success: + final_count = len(self.bot.meshcore.contacts) + self.logger.info(f"✅ Auto-purge completed, now at {final_count}/{self.contact_limit} contacts") + return True + elif repeater_success: + self.logger.info(f"✅ Auto-purged {purge_count} repeaters, now at {remaining_count}/{self.contact_limit} contacts") return True else: - self.logger.warning(f"❌ Auto-purge failed to remove {purge_count} repeaters") + self.logger.warning(f"❌ Auto-purge failed to remove {purge_count} contacts") return False return False @@ -620,6 +639,60 @@ class RepeaterManager: self.logger.error(f"Error in auto-purge execution: {e}") return False + async def _auto_purge_companions(self, count: int) -> bool: + """Automatically purge companion contacts using intelligent selection""" + try: + if not self.companion_purge_enabled: + self.logger.debug("Companion purging is disabled") + return False + + # Get all companions sorted by priority (most inactive first) + companions_to_purge = await self._get_companions_for_purging(count) + + if not companions_to_purge: + self.logger.warning("No companions available for auto-purge") + # Log some debugging info + total_contacts = len(self.bot.meshcore.contacts) + companion_count = sum(1 for contact_data in self.bot.meshcore.contacts.values() if self._is_companion_device(contact_data)) + self.logger.debug(f"Debug: {total_contacts} total contacts, {companion_count} companions found") + return False + + purged_count = 0 + for i, companion in enumerate(companions_to_purge): + try: + public_key = companion['public_key'] + # Get activity info (already formatted as 'never' if no activity) + last_dm = companion.get('last_dm', 'never') + last_advert = companion.get('last_advert', 'never') + days_inactive = companion.get('days_inactive', 'unknown') + + success = await self.purge_companion_from_contacts(public_key, "Auto-purge - contact limit management") + + if success: + purged_count += 1 + self.logger.info(f"🗑️ Auto-purged companion: {companion['name']} (DM: {last_dm}, Advert: {last_advert}, Inactive: {days_inactive}d)") + else: + self.logger.warning(f"Failed to auto-purge companion: {companion['name']}") + + # Add delay between removals to avoid overwhelming the radio + # Use longer delay (2 seconds) to give radio time to process + if i < len(companions_to_purge) - 1: + await asyncio.sleep(2) + + except Exception as e: + self.logger.error(f"Error auto-purging companion {companion['name']}: {e}") + # Still add delay even on error + if i < len(companions_to_purge) - 1: + await asyncio.sleep(2) + continue + + self.logger.info(f"✅ Auto-purge completed: {purged_count}/{count} companions removed") + return purged_count > 0 + + except Exception as e: + self.logger.error(f"Error in companion auto-purge execution: {e}") + return False + async def _get_repeaters_for_purging(self, count: int) -> List[Dict]: """Get list of repeaters to purge based on intelligent criteria from device contacts""" try: @@ -711,6 +784,133 @@ class RepeaterManager: self.logger.error(f"Error getting repeaters for purging: {e}") return [] + async def _get_companions_for_purging(self, count: int) -> List[Dict]: + """Get list of companion contacts to purge based on activity criteria""" + try: + if not self.companion_purge_enabled: + self.logger.debug("Companion purging is disabled") + return [] + + # Get companions directly from device contacts + device_companions = [] + current_time = datetime.now() + + for contact_key, contact_data in self.bot.meshcore.contacts.items(): + # Check if this is a companion device + if not self._is_companion_device(contact_data): + continue + + public_key = contact_data.get('public_key', contact_key) + name = contact_data.get('adv_name', contact_data.get('name', 'Unknown')) + + # Skip if in ACL (never purge ACL members) + if self._is_in_acl(public_key): + self.logger.debug(f"Skipping companion {name} - in ACL") + continue + + # Get last DM activity + last_dm = self._get_last_dm_activity(public_key) + + # Get last advert activity + last_advert = self._get_last_advert_activity(public_key) + + # Determine most recent activity (DM or advert) + last_activity = None + if last_dm and last_advert: + last_activity = max(last_dm, last_advert) + elif last_dm: + last_activity = last_dm + elif last_advert: + last_activity = last_advert + + # Calculate days since last activity + days_inactive = None + if last_activity: + days_inactive = (current_time - last_activity).days + else: + # No activity found - use last_seen from device or default to very old + last_seen = contact_data.get('last_seen', contact_data.get('last_advert', contact_data.get('timestamp'))) + if last_seen: + try: + if isinstance(last_seen, str): + last_seen_dt = datetime.fromisoformat(last_seen.replace('Z', '+00:00')) + elif isinstance(last_seen, (int, float)): + last_seen_dt = datetime.fromtimestamp(last_seen) + else: + last_seen_dt = last_seen + days_inactive = (current_time - last_seen_dt).days + except: + days_inactive = 999 # Very old if we can't parse + else: + days_inactive = 999 # Very old if no data + + # Check if companion meets purge criteria + dm_threshold_met = True + advert_threshold_met = True + + if last_dm: + days_since_dm = (current_time - last_dm).days + dm_threshold_met = days_since_dm >= self.companion_dm_threshold_days + else: + # No DM history - check if we have minimum inactive days + dm_threshold_met = days_inactive >= self.companion_min_inactive_days if days_inactive else False + + if last_advert: + days_since_advert = (current_time - last_advert).days + advert_threshold_met = days_since_advert >= self.companion_advert_threshold_days + else: + # No advert history - check if we have minimum inactive days + advert_threshold_met = days_inactive >= self.companion_min_inactive_days if days_inactive else False + + # Only add if both thresholds are met (hasn't DM'd AND hasn't adverted recently) + if dm_threshold_met and advert_threshold_met: + device_companions.append({ + 'public_key': public_key, + 'name': name, + 'last_dm': last_dm.isoformat() if last_dm else 'never', + 'last_advert': last_advert.isoformat() if last_advert else 'never', + 'last_activity': last_activity.isoformat() if last_activity else None, + 'days_inactive': days_inactive, + 'latitude': contact_data.get('adv_lat'), + 'longitude': contact_data.get('adv_lon'), + 'city': contact_data.get('city'), + 'state': contact_data.get('state'), + 'country': contact_data.get('country') + }) + + # Sort by priority (most inactive first, then without location data) + device_companions.sort(key=lambda x: ( + # Priority 1: Most inactive first + -x['days_inactive'] if x['days_inactive'] else -999, + # Priority 2: Without location data first + 0 if not (x.get('latitude') and x.get('longitude')) else 1 + )) + + # Apply additional filtering + filtered_companions = [] + for companion in device_companions: + # Skip very recently active (last 2 hours) - more lenient + if companion['last_activity']: + try: + last_activity_dt = datetime.fromisoformat(companion['last_activity']) + if last_activity_dt > current_time - timedelta(hours=2): + continue + except: + pass + + filtered_companions.append(companion) + + if len(filtered_companions) >= count: + break + + self.logger.debug(f"Found {len(device_companions)} device companions, {len(filtered_companions)} available for purging") + + return filtered_companions[:count] + + except Exception as e: + self.logger.error(f"Error getting companions for purging: {e}") + return [] + def _extract_location_data(self, contact_data: Dict, should_geocode: bool = True) -> Dict[str, Optional[str]]: """Extract location data from contact_data JSON""" location_info = { @@ -870,16 +1070,16 @@ class RepeaterManager: should_geocode = False updated_location_info = location_info.copy() - # If no existing data, only geocode if we have valid coordinates but no city + # If no existing data, only geocode if we have valid coordinates but missing location data if not existing_data: should_geocode = ( location_info['latitude'] is not None and location_info['longitude'] is not None and not (location_info['latitude'] == 0.0 and location_info['longitude'] == 0.0) and - not location_info['city'] + not (location_info['state'] and location_info['country']) ) if should_geocode: - self.logger.debug(f"📍 New contact {name}, will geocode coordinates") + self.logger.debug(f"📍 New contact {name}, will geocode coordinates (missing state/country)") return should_geocode, updated_location_info # Extract existing location data @@ -889,27 +1089,37 @@ class RepeaterManager: existing_state = existing_data.get('state') existing_country = existing_data.get('country') - # Only geocode if coordinates changed or we don't have city data + # Check if we have valid coordinates in the new data if (location_info['latitude'] is not None and location_info['longitude'] is not None and not (location_info['latitude'] == 0.0 and location_info['longitude'] == 0.0)): + # Use a more lenient threshold for coordinate changes (0.001 degrees ≈ 111 meters) + # This prevents geocoding for minor GPS variations in stationary repeaters coordinates_changed = ( - abs(location_info['latitude'] - existing_lat) > 0.0001 or - abs(location_info['longitude'] - existing_lon) > 0.0001 + abs(location_info['latitude'] - existing_lat) > 0.001 or + abs(location_info['longitude'] - existing_lon) > 0.001 ) - # Only geocode if coordinates changed or we don't have a city - should_geocode = coordinates_changed or not existing_city + # Check if we have sufficient location data (state AND country) + has_sufficient_location_data = existing_state and existing_country - if not should_geocode and existing_city: - # Use existing city data, no need to geocode + # Only geocode if: + # 1. Coordinates changed significantly (repeater moved), OR + # 2. We're missing both state and country (critical location data) + should_geocode = coordinates_changed or not has_sufficient_location_data + + if not should_geocode: + # Coordinates haven't changed and we have sufficient location data + # Use existing location data, no need to geocode updated_location_info['city'] = existing_city updated_location_info['state'] = existing_state updated_location_info['country'] = existing_country - self.logger.debug(f"📍 Using existing location data for {name}: {existing_city}") - elif should_geocode: - self.logger.debug(f"📍 Location changed for {name}, will geocode new coordinates") + self.logger.debug(f"📍 Using existing location data for {name} (coordinates unchanged, has state/country)") + elif coordinates_changed: + self.logger.debug(f"📍 Location changed significantly for {name} (moved >111m), will geocode new coordinates") + else: + self.logger.debug(f"📍 Missing state/country for {name}, will geocode coordinates") else: # No valid coordinates in new data, keep existing location updated_location_info['latitude'] = existing_lat if existing_lat != 0.0 else None @@ -1191,6 +1401,131 @@ class RepeaterManager: self.logger.error(f"Error checking if device is repeater: {e}") return False + def _is_companion_device(self, contact_data: Dict) -> bool: + """Check if a contact is a companion (human user, not a repeater)""" + try: + # Companion is simply the inverse of repeater + return not self._is_repeater_device(contact_data) + except Exception as e: + self.logger.error(f"Error checking if device is companion: {e}") + return False + + def _is_in_acl(self, public_key: str) -> bool: + """Check if a public key is in the bot's admin ACL (should never be purged)""" + try: + if not hasattr(self.bot, 'config'): + return False + + # Get admin pubkeys from config + admin_pubkeys = self.bot.config.get('Admin_ACL', 'admin_pubkeys', fallback='') + if not admin_pubkeys: + return False + + # Parse admin pubkeys + admin_pubkey_list = [key.strip() for key in admin_pubkeys.split(',') if key.strip()] + if not admin_pubkey_list: + return False + + # Check if public key matches any admin key (exact match required for security) + for admin_key in admin_pubkey_list: + if public_key == admin_key: + return True + + return False + + except Exception as e: + self.logger.error(f"Error checking ACL membership: {e}") + return False # Default to not in ACL on error (safer) + + def _get_last_dm_activity(self, public_key: str, sender_id: str = None) -> Optional[datetime]: + """Get the timestamp of the last DM from a contact""" + try: + import time + + # Try to find sender_id from contact if not provided + if not sender_id: + # Try to get sender_id from device contacts + if hasattr(self.bot.meshcore, 'contacts'): + for contact_key, contact_data in self.bot.meshcore.contacts.items(): + if contact_data.get('public_key', contact_key) == public_key: + sender_id = contact_data.get('name', contact_data.get('adv_name', '')) + break + + if not sender_id: + # Try to get from complete_contact_tracking + tracking_data = self.db_manager.execute_query( + 'SELECT name FROM complete_contact_tracking WHERE public_key = ? LIMIT 1', + (public_key,) + ) + if tracking_data: + sender_id = tracking_data[0]['name'] + + if not sender_id: + return None + + # Query message_stats for last DM + query = ''' + SELECT MAX(timestamp) as last_dm_timestamp + FROM message_stats + WHERE sender_id = ? AND is_dm = 1 + ''' + results = self.db_manager.execute_query(query, (sender_id,)) + + if results and results[0]['last_dm_timestamp']: + timestamp = results[0]['last_dm_timestamp'] + # Convert to datetime + if isinstance(timestamp, (int, float)): + return datetime.fromtimestamp(timestamp) + elif isinstance(timestamp, str): + try: + return datetime.fromisoformat(timestamp.replace('Z', '+00:00')) + except: + return None + + return None + + except Exception as e: + self.logger.debug(f"Error getting last DM activity for {public_key}: {e}") + return None + + def _get_last_advert_activity(self, public_key: str) -> Optional[datetime]: + """Get the timestamp of the last advert from a contact""" + try: + # Query complete_contact_tracking for last advert + query = ''' + SELECT last_advert_timestamp, last_heard + FROM complete_contact_tracking + WHERE public_key = ? AND role = 'companion' + LIMIT 1 + ''' + results = self.db_manager.execute_query(query, (public_key,)) + + if results: + # Prefer last_advert_timestamp, fallback to last_heard + timestamp = results[0].get('last_advert_timestamp') or results[0].get('last_heard') + + if timestamp: + # Convert to datetime + if isinstance(timestamp, datetime): + return timestamp + elif isinstance(timestamp, str): + try: + return datetime.fromisoformat(timestamp.replace('Z', '+00:00')) + except: + # Try parsing as timestamp + try: + return datetime.fromtimestamp(float(timestamp)) + except: + return None + elif isinstance(timestamp, (int, float)): + return datetime.fromtimestamp(timestamp) + + return None + + except Exception as e: + self.logger.debug(f"Error getting last advert activity for {public_key}: {e}") + return None + async def scan_and_catalog_repeaters(self) -> int: """Scan current contacts and catalog any repeaters found""" # Wait for contacts to be loaded if they're not ready yet @@ -1581,6 +1916,181 @@ class RepeaterManager: self.logger.debug(f"Error type: {type(e).__name__}") return False + async def purge_companion_from_contacts(self, public_key: str, reason: str = "Manual purge") -> bool: + """Remove a companion contact from the device's contact list""" + self.logger.info(f"Starting companion purge process for public_key: {public_key}") + self.logger.debug(f"Purge reason: {reason}") + + try: + # Safety check: Never purge ACL members + if self._is_in_acl(public_key): + self.logger.warning(f"❌ Attempted to purge companion in ACL - BLOCKED: {public_key[:16]}...") + return False + + # Find the contact in meshcore + contact_to_remove = None + contact_name = None + contact_key = None + + self.logger.debug(f"Searching through {len(self.bot.meshcore.contacts)} contacts...") + + # Try to find contact using MeshCore helper methods first + try: + contact_to_remove = self.bot.meshcore.get_contact_by_key_prefix(public_key[:8]) + if contact_to_remove: + contact_name = contact_to_remove.get('adv_name', contact_to_remove.get('name', 'Unknown')) + contact_key = public_key + self.logger.debug(f"Found contact using key prefix: {contact_name}") + except Exception as e: + self.logger.debug(f"Key prefix lookup failed: {e}") + + # Fallback to manual search + if not contact_to_remove: + for key, contact_data in self.bot.meshcore.contacts.items(): + if contact_data.get('public_key', key) == public_key: + # Verify it's a companion + if not self._is_companion_device(contact_data): + self.logger.warning(f"Contact {public_key} is not a companion - skipping") + return False + contact_to_remove = contact_data + contact_name = contact_data.get('adv_name', contact_data.get('name', 'Unknown')) + contact_key = key + self.logger.debug(f"Found companion manually: {contact_name} (key: {contact_key})") + break + + if not contact_to_remove: + self.logger.warning(f"Companion with public key {public_key} not found in current contacts") + return False + + # Track whether device removal was successful + device_removal_successful = False + + # Verify contact still exists before attempting removal + contact_still_exists = any( + contact_data.get('public_key', key) == public_key + for key, contact_data in self.bot.meshcore.contacts.items() + ) + + if not contact_still_exists: + self.logger.info(f"✅ Contact '{contact_name}' not found in device contacts (already removed) - treating as success") + device_removal_successful = True + else: + # Remove the contact using the proper MeshCore API + try: + self.logger.info(f"Removing companion '{contact_name}' from device using MeshCore API...") + self.logger.debug(f"Contact details: public_key={public_key}, contact_key={contact_key}, name='{contact_name}'") + + # Try removal methods in order of preference + # Method 1: Try with contact object (if supported) + if contact_to_remove: + try: + self.logger.debug(f"Trying removal with contact object...") + result = await asyncio.wait_for( + self.bot.meshcore.commands.remove_contact(contact_to_remove), + timeout=30.0 + ) + if result.type == EventType.OK: + device_removal_successful = True + self.logger.info(f"✅ Successfully removed companion '{contact_name}' using contact object") + except Exception as e: + self.logger.debug(f"Contact object removal failed: {e}") + # Small delay between method attempts + if not device_removal_successful: + await asyncio.sleep(0.5) + + # Method 2: Try with public_key (string) + if not device_removal_successful: + try: + self.logger.debug(f"Trying removal with public_key string: {public_key[:16]}...") + result = await asyncio.wait_for( + self.bot.meshcore.commands.remove_contact(public_key), + timeout=30.0 + ) + if result.type == EventType.OK: + device_removal_successful = True + self.logger.info(f"✅ Successfully removed companion '{contact_name}' using public_key") + else: + error_code = result.payload.get('error_code', 'unknown') if hasattr(result, 'payload') else 'unknown' + self.logger.debug(f"Removal with public_key failed: error_code={error_code}") + except Exception as e: + self.logger.debug(f"Public key removal failed: {e}") + # Small delay between method attempts + if not device_removal_successful: + await asyncio.sleep(0.5) + + # Method 3: Try with contact_key (dictionary key) + if not device_removal_successful and contact_key and contact_key != public_key: + try: + self.logger.debug(f"Trying removal with contact_key: {contact_key[:16]}...") + result = await asyncio.wait_for( + self.bot.meshcore.commands.remove_contact(contact_key), + timeout=30.0 + ) + if result.type == EventType.OK: + device_removal_successful = True + self.logger.info(f"✅ Successfully removed companion '{contact_name}' using contact_key") + except Exception as e: + self.logger.debug(f"Contact key removal failed: {e}") + # Small delay between method attempts + if not device_removal_successful: + await asyncio.sleep(0.5) + + # Method 4: Try with public_key as bytes + if not device_removal_successful: + try: + if len(public_key) == 64: # 32 bytes in hex + public_key_bytes = bytes.fromhex(public_key) + self.logger.debug(f"Trying removal with public_key as bytes...") + result = await asyncio.wait_for( + self.bot.meshcore.commands.remove_contact(public_key_bytes), + timeout=30.0 + ) + if result.type == EventType.OK: + device_removal_successful = True + self.logger.info(f"✅ Successfully removed companion '{contact_name}' using public_key bytes") + except Exception as e: + self.logger.debug(f"Public key bytes removal failed: {e}") + + # If all methods failed, try fallback methods + if not device_removal_successful: + self.logger.warning(f"All removal methods failed for '{contact_name}' - trying fallback methods...") + device_removal_successful = await self._try_fallback_removal_methods(public_key, contact_name, reason) + + except Exception as e: + self.logger.error(f"Failed to remove companion '{contact_name}' from device: {e}") + device_removal_successful = await self._try_fallback_removal_methods(public_key, contact_name, reason) + + # Update tracking database if device removal was successful + if device_removal_successful: + # Update complete_contact_tracking to mark as not currently tracked + self.db_manager.execute_update( + 'UPDATE complete_contact_tracking SET is_currently_tracked = 0 WHERE public_key = ?', + (public_key,) + ) + + # Log the purge action + self.db_manager.execute_update(''' + INSERT INTO purging_log (action, public_key, name, reason) + VALUES ('companion_purged', ?, ?, ?) + ''', (public_key, contact_name, reason)) + + self.logger.info(f"✅ Successfully purged companion {contact_name}: {reason}") + self.logger.debug(f"Companion purge process completed successfully for {contact_name}") + return True + else: + self.logger.error(f"Failed to remove companion {contact_name} from device - not marking as purged in database") + # Log the failed attempt + self.db_manager.execute_update(''' + INSERT INTO purging_log (action, public_key, name, reason) + VALUES ('companion_purge_failed', ?, ?, ?) + ''', (public_key, contact_name, f"{reason} - Device removal failed")) + return False + + except Exception as e: + self.logger.error(f"Error purging companion {public_key}: {e}") + self.logger.debug(f"Error type: {type(e).__name__}") + return False + async def _try_fallback_removal_methods(self, public_key: str, contact_name: str, reason: str) -> bool: """Try alternative methods to remove a contact when the primary MeshCore API fails""" try: diff --git a/modules/web_viewer/app.py b/modules/web_viewer/app.py index 06e61c7..404542f 100644 --- a/modules/web_viewer/app.py +++ b/modules/web_viewer/app.py @@ -108,9 +108,20 @@ class BotDataViewer: from modules.db_manager import DBManager # Create a minimal bot object for DBManager class MinimalBot: - def __init__(self, logger): + def __init__(self, logger, config, db_manager=None): self.logger = logger - self.db_manager = DBManager(MinimalBot(self.logger), self.db_path) + self.config = config + self.db_manager = db_manager + + # Create DBManager first + minimal_bot = MinimalBot(self.logger, self.config) + self.db_manager = DBManager(minimal_bot, self.db_path) + + # Now set db_manager on the minimal bot for RepeaterManager + minimal_bot.db_manager = self.db_manager + + # Initialize repeater manager for geocoding functionality + self.repeater_manager = RepeaterManager(minimal_bot) # Store database paths for direct connection self.db_path = self.db_path @@ -291,6 +302,71 @@ class BotDataViewer: except Exception as e: self.logger.error(f"Error getting recent commands: {e}") return jsonify({'error': str(e)}), 500 + + @self.app.route('/api/geocode-contact', methods=['POST']) + def api_geocode_contact(): + """Manually geocode a contact by public_key""" + try: + data = request.get_json() + if not data or 'public_key' not in data: + return jsonify({'error': 'public_key is required'}), 400 + + public_key = data['public_key'] + + # Get contact data from database + conn = self._get_db_connection() + cursor = conn.cursor() + + cursor.execute(''' + SELECT latitude, longitude, name, city, state, country + FROM complete_contact_tracking + WHERE public_key = ? + ''', (public_key,)) + + contact = cursor.fetchone() + if not contact: + conn.close() + return jsonify({'error': 'Contact not found'}), 404 + + lat = contact['latitude'] + lon = contact['longitude'] + name = contact['name'] + + # Check if we have valid coordinates + if lat is None or lon is None or lat == 0.0 or lon == 0.0: + conn.close() + return jsonify({'error': 'Contact does not have valid coordinates'}), 400 + + # Perform geocoding + self.logger.info(f"Manual geocoding requested for {name} ({public_key[:16]}...)") + location_info = self.repeater_manager._get_full_location_from_coordinates(lat, lon) + + # Update database with new location data + cursor.execute(''' + UPDATE complete_contact_tracking + SET city = ?, state = ?, country = ? + WHERE public_key = ? + ''', ( + location_info.get('city'), + location_info.get('state'), + location_info.get('country'), + public_key + )) + + conn.commit() + conn.close() + + self.logger.info(f"Successfully geocoded {name}: {location_info}") + + return jsonify({ + 'success': True, + 'location': location_info, + 'message': f'Successfully geocoded {name}' + }) + + except Exception as e: + self.logger.error(f"Error geocoding contact: {e}") + return jsonify({'error': str(e)}), 500 def _setup_socketio_handlers(self): """Setup SocketIO event handlers using modern patterns""" diff --git a/modules/web_viewer/templates/contacts.html b/modules/web_viewer/templates/contacts.html index de64cd7..5c5af2f 100644 --- a/modules/web_viewer/templates/contacts.html +++ b/modules/web_viewer/templates/contacts.html @@ -418,9 +418,17 @@ class ModernContactsManager {