diff --git a/modules/commands/stats_command.py b/modules/commands/stats_command.py index a6d97a7..3c122bb 100644 --- a/modules/commands/stats_command.py +++ b/modules/commands/stats_command.py @@ -22,12 +22,12 @@ class StatsCommand(BaseCommand): # Plugin metadata name = "stats" keywords = ['stats'] - description = "Show statistics for past 24 hours. Use 'stats messages', 'stats channels', or 'stats paths' for specific stats." + description = "Show statistics for past 24 hours. Use 'stats messages', 'stats channels', 'stats paths', or 'stats adverts' for specific stats." category = "analytics" # Documentation short_description = "Show bot usage statistics for past 24 hours" - usage = "stats [messages|channels|paths]" + usage = "stats [messages|channels|paths|adverts]" examples = ["stats", "stats channels"] parameters = [ {"name": "type", "description": "messages, channels, or paths (optional)"} @@ -349,7 +349,11 @@ class StatsCommand(BaseCommand): elif subcommand in ['channels', 'channel']: response = await self._get_channel_leaderboard() elif subcommand in ['paths', 'path']: - response = await self._get_path_leaderboard() + response = await self._get_path_leaderboard(message) + elif subcommand in ['adverts', 'advert', 'advertisements', 'advertisement']: + # Check for verbose/hash option + show_hashes = len(parts) > 2 and parts[2].lower() in ['hash', 'hashes', 'verbose', 'verb'] + response = await self._get_adverts_leaderboard(message, show_hashes=show_hashes) else: response = self.translate('commands.stats.unknown_subcommand', subcommand=subcommand) else: @@ -521,7 +525,7 @@ class StatsCommand(BaseCommand): self.logger.error(f"Error getting channel leaderboard: {e}") return self.translate('commands.stats.error_channels', error=str(e)) - async def _get_path_leaderboard(self) -> str: + async def _get_path_leaderboard(self, message: Optional[MeshMessage] = None) -> str: """Get leaderboard for longest paths seen. Returns: @@ -554,7 +558,7 @@ class StatsCommand(BaseCommand): # Build compact response with length checking response = "" - max_length = 130 # Safe length for mesh network + max_length = self.get_max_message_length(message) if message else 130 if longest_paths: for i, (sender, path_len, path_str) in enumerate(longest_paths, 1): @@ -577,6 +581,176 @@ class StatsCommand(BaseCommand): self.logger.error(f"Error getting path leaderboard: {e}") return self.translate('commands.stats.error_paths', error=str(e)) + async def _get_adverts_leaderboard(self, message: Optional[MeshMessage] = None, show_hashes: bool = False) -> str: + """Get leaderboard for nodes with most unique advert packets in last 24 hours. + + Args: + message: Optional message for dynamic length calculation. + show_hashes: If True, include packet hashes in output. + + Returns: + str: Formatted leaderboard string. + """ + try: + with sqlite3.connect(self.bot.db_manager.db_path) as conn: + cursor = conn.cursor() + + # Check if daily_stats table exists + cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='daily_stats'") + has_daily_stats = cursor.fetchone() is not None + + if has_daily_stats: + if show_hashes: + # Query with packet hashes for validation + # First get the top nodes, then get their hashes + cursor.execute(''' + SELECT + c.public_key, + c.name, + COALESCE( + (SELECT advert_count FROM daily_stats + WHERE date = DATE(c.last_advert_timestamp) + AND public_key = c.public_key), 0 + ) as unique_adverts + FROM complete_contact_tracking c + WHERE c.last_advert_timestamp >= datetime('now', '-24 hours') + GROUP BY c.public_key, c.name + HAVING unique_adverts > 0 + ORDER BY unique_adverts DESC + LIMIT 20 + ''') + top_nodes = cursor.fetchall() + + # Now get packet hashes for each node + top_adverts = [] + for public_key, name, count in top_nodes: + # Get packet hashes for this node in the last 24 hours + cursor.execute(''' + SELECT packet_hash + FROM unique_advert_packets + WHERE public_key = ? + AND first_seen >= datetime('now', '-24 hours') + ORDER BY first_seen + ''', (public_key,)) + hash_rows = cursor.fetchall() + packet_hashes = ', '.join([row[0] for row in hash_rows]) if hash_rows else None + top_adverts.append((public_key, name, count, packet_hashes)) + else: + # Query nodes that advertised in the last 24 hours + # Get advert_count from daily_stats for the day of last_advert_timestamp + # This gives us the count of unique adverts for that day + cursor.execute(''' + SELECT + c.public_key, + c.name, + COALESCE( + (SELECT advert_count FROM daily_stats + WHERE date = DATE(c.last_advert_timestamp) + AND public_key = c.public_key), 0 + ) as unique_adverts + FROM complete_contact_tracking c + WHERE c.last_advert_timestamp >= datetime('now', '-24 hours') + GROUP BY c.public_key, c.name + HAVING unique_adverts > 0 + ORDER BY unique_adverts DESC + LIMIT 20 + ''') + top_adverts = cursor.fetchall() + else: + # Fallback: use advert_count from complete_contact_tracking + # This is less accurate but works if daily_stats doesn't exist + if show_hashes: + # Get nodes first + cursor.execute(''' + SELECT public_key, name, advert_count as unique_adverts + FROM complete_contact_tracking + WHERE last_advert_timestamp >= datetime('now', '-24 hours') + ORDER BY advert_count DESC + LIMIT 20 + ''') + top_nodes = cursor.fetchall() + + # Get packet hashes for each node + top_adverts = [] + for public_key, name, count in top_nodes: + cursor.execute(''' + SELECT packet_hash + FROM unique_advert_packets + WHERE public_key = ? + AND first_seen >= datetime('now', '-24 hours') + ORDER BY first_seen + ''', (public_key,)) + hash_rows = cursor.fetchall() + packet_hashes = ', '.join([row[0] for row in hash_rows]) if hash_rows else None + top_adverts.append((public_key, name, count, packet_hashes)) + else: + cursor.execute(''' + SELECT public_key, name, advert_count as unique_adverts + FROM complete_contact_tracking + WHERE last_advert_timestamp >= datetime('now', '-24 hours') + ORDER BY advert_count DESC + LIMIT 20 + ''') + top_adverts = cursor.fetchall() + + # Build compact response with length checking + response = self.translate('commands.stats.adverts.header') + "\n" + max_length = self.get_max_message_length(message) if message else 130 + + if top_adverts: + for i, row in enumerate(top_adverts, 1): + if show_hashes: + public_key, name, count, packet_hashes = row + else: + public_key, name, count = row + packet_hashes = None + + # Truncate name if needed + display_name = name[:15] + "..." if len(name) > 18 else name + # Format: "1. NodeName: 42 adverts" + advert_text = self.translate('commands.stats.adverts.advert_singular') if count == 1 else self.translate('commands.stats.adverts.advert_plural') + + if show_hashes and packet_hashes: + # Format with packet hashes: "1. NodeName: 42 adverts\n Hashes: abc123, def456, ..." + main_line = self.translate('commands.stats.adverts.format', + rank=i, + name=display_name, + count=count, + advert_text=advert_text) + + # Split packet hashes and format them + hash_list = [h.strip() for h in packet_hashes.split(',') if h.strip()] + # Show first few hashes (truncate if too many) + if len(hash_list) > 10: + hash_display = ', '.join(hash_list[:10]) + f" ... ({len(hash_list)} total)" + else: + hash_display = ', '.join(hash_list) + + new_line = f"{main_line}\n {self.translate('commands.stats.adverts.hashes_label', hashes=hash_display)}" + else: + new_line = self.translate('commands.stats.adverts.format', + rank=i, + name=display_name, + count=count, + advert_text=advert_text) + "\n" + + # Check if adding this line would exceed the limit + if len(response + new_line.rstrip('\n')) > max_length: + break + + response += new_line + + # Remove trailing newline + response = response.rstrip('\n') + else: + response += self.translate('commands.stats.adverts.none') + + return response + + except Exception as e: + self.logger.error(f"Error getting adverts leaderboard: {e}") + return self.translate('commands.stats.error_adverts', error=str(e)) + def cleanup_old_stats(self, days_to_keep: int = 7) -> None: """Clean up old stats data to prevent database bloat. diff --git a/modules/db_manager.py b/modules/db_manager.py index b629774..c8b6af5 100644 --- a/modules/db_manager.py +++ b/modules/db_manager.py @@ -30,6 +30,7 @@ class DBManager: 'repeater_contacts', 'complete_contact_tracking', # Repeater manager 'daily_stats', # Repeater manager + 'unique_advert_packets', # Repeater manager - unique packet tracking 'purging_log', # Repeater manager } diff --git a/modules/repeater_manager.py b/modules/repeater_manager.py index 283706e..db8fe04 100644 --- a/modules/repeater_manager.py +++ b/modules/repeater_manager.py @@ -107,6 +107,17 @@ class RepeaterManager: UNIQUE(date, public_key) ''') + # Create unique_advert_packets table for tracking unique packet hashes + # This allows us to count unique advert packets (deduplicate by packet_hash) + self.db_manager.create_table('unique_advert_packets', ''' + id INTEGER PRIMARY KEY AUTOINCREMENT, + date DATE NOT NULL, + public_key TEXT NOT NULL, + packet_hash TEXT NOT NULL, + first_seen TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + UNIQUE(date, public_key, packet_hash) + ''') + # Create purging_log table for audit trail self.db_manager.create_table('purging_log', ''' id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -132,6 +143,10 @@ class RepeaterManager: cursor.execute('CREATE INDEX IF NOT EXISTS idx_complete_currently_tracked ON complete_contact_tracking(is_currently_tracked)') cursor.execute('CREATE INDEX IF NOT EXISTS idx_complete_location ON complete_contact_tracking(latitude, longitude)') cursor.execute('CREATE INDEX IF NOT EXISTS idx_complete_role_tracked ON complete_contact_tracking(role, is_currently_tracked)') + + # Indexes for unique_advert_packets table + cursor.execute('CREATE INDEX IF NOT EXISTS idx_unique_advert_date_pubkey ON unique_advert_packets(date, public_key)') + cursor.execute('CREATE INDEX IF NOT EXISTS idx_unique_advert_hash ON unique_advert_packets(packet_hash)') conn.commit() self.logger.info("Repeater contacts database initialized successfully") @@ -296,9 +311,9 @@ class RepeaterManager: # Update the currently_tracked flag based on device contact list await self._update_currently_tracked_status(public_key) - # Track daily advertisement statistics + # Track daily advertisement statistics (with packet_hash for unique tracking) await self._track_daily_advertisement(public_key, name, role, device_type_str, - location_info, signal_strength, snr, hop_count, current_time) + location_info, signal_strength, snr, hop_count, current_time, packet_hash=packet_hash) return True @@ -308,39 +323,90 @@ class RepeaterManager: async def _track_daily_advertisement(self, public_key: str, name: str, role: str, device_type: str, location_info: Dict, signal_strength: float, snr: float, - hop_count: int, timestamp: datetime): - """Track daily advertisement statistics for accurate time-based reporting""" + hop_count: int, timestamp: datetime, packet_hash: Optional[str] = None): + """Track daily advertisement statistics for accurate time-based reporting. + + Args: + public_key: The public key of the node + name: The name of the node + role: The role of the node + device_type: The device type string + location_info: Location information dictionary + signal_strength: Signal strength (RSSI) + snr: Signal-to-noise ratio + hop_count: Number of hops + timestamp: Timestamp of the advert + packet_hash: Optional packet hash for unique packet tracking + """ try: from datetime import date # Get today's date today = date.today() - # Check if we already have an entry for this contact today - existing_daily = self.db_manager.execute_query( - 'SELECT id, advert_count, first_advert_time FROM daily_stats WHERE date = ? AND public_key = ?', - (today, public_key) - ) - - if existing_daily: - # Update existing daily entry - daily_advert_count = existing_daily[0]['advert_count'] + 1 - self.db_manager.execute_update(''' - UPDATE daily_stats - SET advert_count = ?, last_advert_time = ? - WHERE date = ? AND public_key = ? - ''', (daily_advert_count, timestamp, today, public_key)) - - self.logger.debug(f"Updated daily stats for {name}: {daily_advert_count} adverts today") + # Track unique packet hash if provided (for deduplication) + is_unique_packet = False + if packet_hash and packet_hash != "0000000000000000": + try: + # Check if we've already seen this packet hash today + existing_packet = self.db_manager.execute_query( + 'SELECT id FROM unique_advert_packets WHERE date = ? AND public_key = ? AND packet_hash = ?', + (today, public_key, packet_hash) + ) + + if not existing_packet: + # This is a new unique packet - insert it + self.db_manager.execute_update(''' + INSERT INTO unique_advert_packets + (date, public_key, packet_hash, first_seen) + VALUES (?, ?, ?, ?) + ''', (today, public_key, packet_hash, timestamp)) + is_unique_packet = True + self.logger.debug(f"New unique advert packet for {name}: {packet_hash[:8]}...") + else: + # We've already seen this packet hash today - don't count it again + self.logger.debug(f"Duplicate advert packet for {name}: {packet_hash[:8]}... (already counted)") + except Exception as e: + self.logger.debug(f"Error tracking unique packet hash: {e}") + # Fall through to count it anyway if unique tracking fails + is_unique_packet = True else: - # Insert new daily entry - self.db_manager.execute_update(''' - INSERT INTO daily_stats - (date, public_key, advert_count, first_advert_time, last_advert_time) - VALUES (?, ?, ?, ?, ?) - ''', (today, public_key, 1, timestamp, timestamp)) + # No packet hash provided, count it as unique (can't deduplicate) + is_unique_packet = True + + # Only increment count if this is a unique packet + if is_unique_packet: + # Check if we already have an entry for this contact today + existing_daily = self.db_manager.execute_query( + 'SELECT id, advert_count, first_advert_time FROM daily_stats WHERE date = ? AND public_key = ?', + (today, public_key) + ) - self.logger.debug(f"Added daily stats for {name}: first advert today") + if existing_daily: + # Update existing daily entry - count unique packets only + # Count distinct packet hashes for today from unique_advert_packets table + unique_count = self.db_manager.execute_query( + 'SELECT COUNT(*) FROM unique_advert_packets WHERE date = ? AND public_key = ?', + (today, public_key) + ) + daily_advert_count = unique_count[0]['COUNT(*)'] if unique_count else existing_daily[0]['advert_count'] + 1 + + self.db_manager.execute_update(''' + UPDATE daily_stats + SET advert_count = ?, last_advert_time = ? + WHERE date = ? AND public_key = ? + ''', (daily_advert_count, timestamp, today, public_key)) + + self.logger.debug(f"Updated daily stats for {name}: {daily_advert_count} unique adverts today") + else: + # Insert new daily entry + self.db_manager.execute_update(''' + INSERT INTO daily_stats + (date, public_key, advert_count, first_advert_time, last_advert_time) + VALUES (?, ?, ?, ?, ?) + ''', (today, public_key, 1, timestamp, timestamp)) + + self.logger.debug(f"Added daily stats for {name}: first unique advert today") except Exception as e: self.logger.error(f"Error tracking daily advertisement: {e}") diff --git a/translations/en.json b/translations/en.json index f53c12c..c18c3da 100644 --- a/translations/en.json +++ b/translations/en.json @@ -151,8 +151,8 @@ "team_not_found": "Team/League '{team}' not found. Try: seahawks, mariners, sounders, kraken, storm, chiefs, lfc, mlb, nfl, mls, wnba, epl, etc." }, "stats": { - "description": "Show statistics for past 24 hours. Use 'stats messages', 'stats channels', or 'stats paths' for specific stats.", - "help": "Show 24-hour bot statistics. Commands: 'stats' (basic), 'stats messages' (bot users), 'stats channels' (channel activity), 'stats paths' (longest paths)", + "description": "Show statistics for past 24 hours. Use 'stats messages', 'stats channels', 'stats paths', or 'stats adverts' for specific stats.", + "help": "Show 24-hour bot statistics. Commands: 'stats' (basic), 'stats messages' (bot users), 'stats channels' (channel activity), 'stats paths' (longest paths), 'stats adverts' (top advert nodes)", "subcommands": [ { "name": "messages", @@ -165,10 +165,14 @@ { "name": "paths", "description": "Show longest routing paths" + }, + { + "name": "adverts", + "description": "Show top nodes by unique advert packets" } ], "disabled": "Stats command is disabled", - "unknown_subcommand": "Unknown: {subcommand}. Use 'stats', 'stats messages', 'stats channels', or 'stats paths'", + "unknown_subcommand": "Unknown: {subcommand}. Use 'stats', 'stats messages', 'stats channels', 'stats paths', or 'stats adverts'", "error": "Error getting stats: {error}", "error_bot_users": "Error getting bot user stats: {error}", "error_channels": "Error getting channel stats: {error}", @@ -196,7 +200,16 @@ "paths": { "none": "No path data", "format": "{rank} {sender} {path}" - } + }, + "adverts": { + "header": "Top Advert Nodes (24h):", + "none": "No advert activity", + "advert_singular": "advert", + "advert_plural": "adverts", + "format": "{rank}. {name}: {count} {advert_text}", + "hashes_label": "Hashes: {hashes}" + }, + "error_adverts": "Error getting advert stats: {error}" }, "solarforecast": { "description": "Get solar panel production forecast (usage: sf [panel_size] [azimuth, 0=south] [angle])",