diff --git a/config.ini.example b/config.ini.example index 638ed83..88d0ba6 100644 --- a/config.ini.example +++ b/config.ini.example @@ -90,6 +90,12 @@ bot_longitude = -74.0060 # Set to a lower value if you want to limit channel fetching for performance max_channels = 12 +# Channel refresh interval in seconds +# How often to refresh channel list from device to prevent stale data in database +# Default: 3600 (1 hour). Set to 0 to disable periodic refresh (channels only refreshed on startup) +# This ensures the database stays in sync if channels are changed on the device directly +channel_refresh_interval_seconds = 3600 + # Interval-based advertising settings # Send periodic flood adverts at specified intervals # 0: Disabled (default) @@ -254,73 +260,6 @@ colored_output = true # Options: DEBUG, INFO, WARNING, ERROR, CRITICAL meshcore_log_level = INFO -[Greeter_Command] -# Enable or disable the greeter command -# true: Bot will greet users on their first public channel message -# false: Greeter is disabled -enabled = false - -# Channels where greetings should occur (comma-separated) -# IMPORTANT: Leave commented out (or omit entirely) to use global monitor_channels (default behavior) -# If uncommented with empty value (channels = ), command will be DM-only -# Comma-separated list to restrict to specific channels (only greeter command works there) -# Example: channels = general,welcome,newbies -# If not specified, uses the channels from [Channels] monitor_channels setting -# channels = - -# Greeting message template (default for all channels) -# Available fields: {sender} - the user's name/ID -# For multi-part greetings, separate messages with pipe (|) -# Example (single): "Welcome to the mesh, @[{sender}]!" -# Example (multi-part): "Welcome to the mesh, @[{sender}]!|This is a great place to chat.|Use !help for commands." -greeting_message = Welcome to the mesh, @[{sender}]! - -# Channel-specific greeting messages (optional) -# Format: channel_name:greeting_message,channel_name2:greeting_message2 -# If a channel has a specific greeting, it will be used instead of the default greeting_message -# Example: Public:Welcome to Public channel, @[{sender}]!|general:Welcome to general, @[{sender}]! -# Multi-part greetings are supported per channel using pipe (|) separator -# Leave empty to use greeting_message for all channels -channel_greetings = - -# Per-channel greetings (tracking behavior) -# false: Greet each user only once globally (default - user gets one greeting total) -# true: Greet each user once per channel (user can be greeted on each channel separately) -# Note: This controls tracking, not the greeting message itself. Use channel_greetings for different messages. -per_channel_greetings = false - -# Include mesh network information in greeting -# true: Add mesh statistics to greeting (total contacts, repeaters, etc.) -# false: Only send the greeting message -include_mesh_info = true - -# Mesh info format template -# Available fields: {total_contacts}, {repeaters}, {companions}, {recent_activity_24h} -# Example: "\n\nMesh Info: {total_contacts} contacts, {repeaters} repeaters" -# Note: Mesh info is appended to the last greeting message part -mesh_info_format = \n\nMesh Info: {total_contacts} contacts, {repeaters} repeaters, {recent_activity_24h} active in last 24h - -# Rollout period in days -# When greeter is first enabled on an active mesh, this sets how many days -# to listen and mark all active users as already greeted before beginning -# to greet new users. This prevents greeting everyone on an established mesh. -# Set to 0 to disable rollout (will greet all new users immediately) -# Note: Use auto_backfill to mark historical users and shorten/eliminate rollout period -rollout_days = 7 - -# Auto-backfill from historical message_stats data -# true: Automatically mark all users who have posted on public channels in the past -# false: Only mark users during rollout period (default) -# This allows shortening or eliminating the rollout period by using existing data -auto_backfill = false - -# Backfill lookback period in days -# Number of days to look back when auto-backfilling (0 = all time) -# Only used if auto_backfill = true -# Example: 30 = only mark users who posted in last 30 days -# Example: 0 = mark all users who have ever posted (all time) -backfill_lookback_days = 30 - [Custom_Syntax] # Custom syntax patterns for special message formats # Format: pattern = "response_format" @@ -372,6 +311,55 @@ repeater_prefix_api_url = # Recommended: 1-6 hours (data doesn't change frequently) repeater_prefix_cache_hours = 1 +[Feed_Manager] +# Enable feed manager functionality +# true: Feed manager is enabled and will poll feeds +# false: Feed manager is disabled +feed_manager_enabled = false + +# Default check interval in seconds (5 minutes) +# How often to check feeds for new items +# Individual feeds can override this with their own interval +default_check_interval_seconds = 300 + +# Maximum number of items to process per feed per check +# Prevents overwhelming channels with too many items at once +max_items_per_check = 10 + +# Default output format for feed messages +# Placeholders: {title}, {body}, {date}, {link}, {emoji} +# Shortening functions: {field|truncate:N}, {field|word_wrap:N}, {field|first_words:N} +# Example: "{emoji} {body|truncate:100} - {date}\n{link|truncate:50}" +# Default uses body instead of title for main content +default_output_format = {emoji} {body|truncate:100} - {date}\n{link|truncate:50} + +# Default message send interval in seconds +# How long to wait between sending queued messages from the same feed +# Prevents rate limiting by spacing out message sends +# Individual feeds can override this with their own interval +default_message_send_interval_seconds = 2.0 + +# Request timeout in seconds +# Maximum time to wait for feed requests +feed_request_timeout = 30 + +# User-Agent string for HTTP requests +# Identifies the bot when fetching feeds +feed_user_agent = MeshCoreBot/1.0 FeedManager + +# Rate limiting: minimum seconds between requests to same domain +# Prevents overwhelming feed sources +feed_rate_limit_seconds = 5 + +# Maximum message length (mesh limit is 130) +# Messages longer than this will be truncated +max_message_length = 130 + +# Enable/disable feed command +# true: Feed command is available (admin only) +# false: Feed command is disabled +feed_command_enabled = true + [Prefix_Command] # Enable or disable repeater geolocation in prefix command # true: Show city names with repeaters when location data is available @@ -655,4 +643,95 @@ debug = false # Auto-start web viewer with bot # true: Start web viewer automatically when bot starts # false: Start web viewer manually -auto_start = false \ No newline at end of file +auto_start = false + +[Greeter_Command] +# Enable or disable the greeter command +# true: Bot will greet users on their first public channel message +# false: Greeter is disabled +enabled = false + +# Channels where greetings should occur (comma-separated) +# IMPORTANT: Leave commented out (or omit entirely) to use global monitor_channels (default behavior) +# If uncommented with empty value (channels = ), command will be DM-only +# Comma-separated list to restrict to specific channels (only greeter command works there) +# Example: channels = general,welcome,newbies +# If not specified, uses the channels from [Channels] monitor_channels setting +# channels = + +# Greeting message template (default for all channels) +# Available fields: {sender} - the user's name/ID +# For multi-part greetings, separate messages with pipe (|) +# Example (single): "Welcome to the mesh, @[{sender}]!" +# Example (multi-part): "Welcome to the mesh, @[{sender}]!|This is a great place to chat.|Use !help for commands." +greeting_message = Welcome to the mesh, @[{sender}]! + +# Channel-specific greeting messages (optional) +# Format: channel_name:greeting_message,channel_name2:greeting_message2 +# If a channel has a specific greeting, it will be used instead of the default greeting_message +# Example: Public:Welcome to Public channel, @[{sender}]!|general:Welcome to general, @[{sender}]! +# Multi-part greetings are supported per channel using pipe (|) separator +# Leave empty to use greeting_message for all channels +channel_greetings = + +# Per-channel greetings (tracking behavior) +# false: Greet each user only once globally (default - user gets one greeting total) +# true: Greet each user once per channel (user can be greeted on each channel separately) +# Note: This controls tracking, not the greeting message itself. Use channel_greetings for different messages. +per_channel_greetings = false + +# Include mesh network information in greeting +# true: Add mesh statistics to greeting (total contacts, repeaters, etc.) +# false: Only send the greeting message +include_mesh_info = true + +# Mesh info format template +# Available fields: {total_contacts}, {repeaters}, {companions}, {recent_activity_24h} +# Example: "\n\nMesh Info: {total_contacts} contacts, {repeaters} repeaters" +# Note: Mesh info is appended to the last greeting message part +mesh_info_format = \n\nMesh Info: {total_contacts} contacts, {repeaters} repeaters, {recent_activity_24h} active in last 24h + +# Rollout period in days +# When greeter is first enabled on an active mesh, this sets how many days +# to listen and mark all active users as already greeted before beginning +# to greet new users. This prevents greeting everyone on an established mesh. +# Set to 0 to disable rollout (will greet all new users immediately) +# Note: Use auto_backfill to mark historical users and shorten/eliminate rollout period +rollout_days = 7 + +# Auto-backfill from historical message_stats data +# true: Automatically mark all users who have posted on public channels in the past +# false: Only mark users during rollout period (default) +# This allows shortening or eliminating the rollout period by using existing data +auto_backfill = false + +# Backfill lookback period in days +# Number of days to look back when auto-backfilling (0 = all time) +# Only used if auto_backfill = true +# Example: 30 = only mark users who posted in last 30 days +# Example: 0 = mark all users who have ever posted (all time) +backfill_lookback_days = 30 + +# Dead air delay in seconds +# Wait this many seconds before sending a greeting to a new user +# This allows time for other users to greet the new user first +# Set to 0 to send greetings immediately (default behavior) +# Example: 30 = wait 30 seconds before greeting +dead_air_delay_seconds = 0 + +# Defer to human greeting +# If enabled and dead_air_delay_seconds > 0, the bot will not send a greeting +# if another user mentions the new user's name within the dead air delay period +# true: Check for human greetings and defer if found +# false: Always send bot greeting after delay (default) +# Note: Only effective when dead_air_delay_seconds > 0 +defer_to_human_greeting = false + +# Levenshtein distance for name matching +# Maximum edit distance allowed when checking if a user has been greeted before +# Prevents duplicate greetings for users who make small changes to their name +# Set to 0 to disable fuzzy matching (exact name match only, default) +# Example: 2 = "John" and "Jon" (distance 1) or "John" and "Jhon" (distance 1) would match +# Example: 3 = "Alice" and "Alicia" (distance 2) would match +# Recommended: 1-3 for most use cases +levenshtein_distance = 0 \ No newline at end of file diff --git a/docs/FEEDS.md b/docs/FEEDS.md new file mode 100644 index 0000000..4b7c12f --- /dev/null +++ b/docs/FEEDS.md @@ -0,0 +1,413 @@ +# Feed Management + +The Feed Management system allows the bot to subscribe to RSS feeds and REST APIs, automatically polling for new content and posting updates to specified mesh channels. + +## Overview + +The feed manager supports two feed types: +- **RSS Feeds**: Standard RSS/Atom feeds +- **API Feeds**: REST API endpoints returning JSON data + +Both feed types support: +- Configurable polling intervals +- Custom message formatting +- Item filtering +- Sorting +- Automatic deduplication +- Rate limiting + +## Configuration + +### Global Settings + +Configure feed manager behavior in `config.ini`: + +```ini +[Feed_Manager] +# Enable/disable feed manager +feed_manager_enabled = true + +# Default check interval (seconds) +default_check_interval_seconds = 300 + +# Maximum items to process per check +max_items_per_check = 10 + +# HTTP request timeout (seconds) +feed_request_timeout = 30 + +# User agent for HTTP requests +feed_user_agent = MeshCoreBot/1.0 FeedManager + +# Rate limit between requests to same domain (seconds) +feed_rate_limit_seconds = 5.0 + +# Maximum message length (characters) +max_message_length = 130 + +# Default output format +default_output_format = {emoji} {body|truncate:100} - {date}\n{link|truncate:50} + +# Default interval between sending queued messages (seconds) +default_message_send_interval_seconds = 2.0 +``` + +## RSS Feed Configuration + +The web interface provides separate input fields for each configuration option. Below are examples showing the values to enter in each field. + +### Basic RSS Feed + +**Feed Type:** `rss` +**Feed URL:** `https://example.com/rss.xml` +**Channel:** `#alerts` +**Feed Name (Optional):** `Example RSS Feed` +**Check Interval (seconds):** `300` +**Output Format:** (leave empty to use default) +**Message Send Interval (seconds):** `2.0` +**Filter Configuration:** (leave empty) +**Sort Configuration:** (leave empty) + +### RSS Feed with Custom Format + +**Feed Type:** `rss` +**Feed URL:** `https://example.com/rss.xml` +**Channel:** `#alerts` +**Feed Name (Optional):** `Emergency Alerts` +**Check Interval (seconds):** `60` +**Output Format:** +``` +{emoji} {title|truncate:80} +{body|truncate:100} +{date} +``` +**Message Send Interval (seconds):** `2.0` +**Filter Configuration:** (leave empty) +**Sort Configuration:** (leave empty) + +## API Feed Configuration + +### Basic API Feed + +**Feed Type:** `api` +**Feed URL:** `https://api.example.com/alerts` +**Channel:** `#alerts` +**Feed Name (Optional):** `API Alerts` +**Check Interval (seconds):** `300` +**Output Format:** (leave empty to use default) +**Message Send Interval (seconds):** `2.0` +**API Configuration (JSON):** +```json +{ + "method": "GET", + "headers": {}, + "params": { + "api_key": "your-api-key" + }, + "response_parser": { + "items_path": "data.alerts", + "id_field": "id", + "title_field": "title", + "description_field": "description", + "timestamp_field": "created_at" + } +} +``` +**Filter Configuration:** (leave empty) +**Sort Configuration:** (leave empty) + +### WSDOT Highway Alerts Example + +**Feed Type:** `api` +**Feed URL:** `https://wsdot.wa.gov/Traffic/api/HighwayAlerts/HighwayAlertsREST.svc/GetAlertsAsJson` +**Channel:** `#traffic` +**Feed Name (Optional):** `WSDOT Highway Alerts` +**Check Interval (seconds):** `300` +**Output Format:** +``` +{emoji} [{raw.Priority|switch:highest:🔴:high:🟠:medium:🟡:âšĒ}] {title|truncate:80} +{raw.EventCategory} | {raw.Region} | {raw.EventStatus} +{body|truncate:70} +``` +**Message Send Interval (seconds):** `2.0` +**API Configuration (JSON):** +```json +{ + "method": "GET", + "headers": {}, + "params": { + "AccessCode": "your-access-code" + }, + "response_parser": { + "items_path": "", + "id_field": "AlertID", + "title_field": "HeadlineDescription", + "description_field": "ExtendedDescription", + "timestamp_field": "LastUpdatedTime" + } +} +``` +**Filter Configuration (JSON):** +```json +{ + "conditions": [ + { + "field": "raw.EventCategory", + "operator": "in", + "values": ["Alert", "Closure"] + } + ], + "logic": "OR" +} +``` +**Sort Configuration (JSON):** +```json +{ + "field": "raw.LastUpdatedTime", + "order": "desc" +} +``` + +## Output Format + +The output format string controls how feed items are formatted before sending to channels. + +### Placeholders + +- `{title}` - Item title +- `{body}` - Item description/body text +- `{date}` - Relative time (e.g., "5m ago", "2h 30m ago") +- `{link}` - Item URL +- `{emoji}` - Auto-selected emoji based on feed name (đŸ“ĸ, 🚨, ⚠ī¸, ℹī¸) +- `{raw.field}` - Access raw API data fields (API feeds only) +- `{raw.nested.field}` - Access nested API fields (e.g., `{raw.StartRoadwayLocation.RoadName}`) + +### Shortening Functions + +Apply functions to placeholders using the pipe operator: + +- `{field|truncate:N}` - Truncate to N characters +- `{field|word_wrap:N}` - Wrap at N characters, breaking at word boundaries +- `{field|first_words:N}` - Take first N words + +**Examples:** +``` +{title|truncate:60} +{body|word_wrap:100} +{body|first_words:20} +``` + +### Regex Extraction + +Extract specific content using regex patterns: + +- `{field|regex:pattern}` - Extract using regex (uses first capture group) +- `{field|regex:pattern:group}` - Extract specific capture group (0 = whole match, 1 = first group, etc.) + +**Examples:** +``` +{body|regex:Temperature:\s*([^\n]+):1} +{body|regex:Conditions:\s*([^\n]+):1} +``` + +### Conditional Formatting + +- `{field|if_regex:pattern:then:else}` - If pattern matches, return "then", else return "else" +- `{field|switch:value1:result1:value2:result2:...:default}` - Multi-value conditional + +**Examples:** +``` +{raw.Priority|switch:highest:🔴:high:🟠:medium:🟡:âšĒ} +{body|if_regex:No restrictions:👍:Restrictions apply} +``` + +### Extract and Check + +- `{field|regex_cond:extract_pattern:check_pattern:then:group}` - Extract text, check if it matches pattern, return "then" if match, else return extracted text + +**Example:** +``` +{body|regex_cond:Northbound\s*\n([^\n]+):No restrictions:👍:1} +``` + +## Filter Configuration + +Filter configuration determines which items are sent to channels. + +### Filter Structure + +```json +{ + "conditions": [ + { + "field": "raw.Priority", + "operator": "in", + "values": ["highest", "high"] + }, + { + "field": "raw.EventStatus", + "operator": "equals", + "value": "open" + } + ], + "logic": "AND" +} +``` + +### Operators + +- `equals` - Exact match +- `not_equals` - Not equal +- `in` - Value in list +- `not_in` - Value not in list +- `matches` - Regex match +- `not_matches` - Regex does not match +- `contains` - String contains value +- `not_contains` - String does not contain value + +### Logic + +- `AND` - All conditions must match (default) +- `OR` - Any condition matches + +### Field Paths + +For API feeds, use `raw.field` or `raw.nested.field` to access API response fields: +- `raw.Priority` +- `raw.EventStatus` +- `raw.StartRoadwayLocation.RoadName` + +## Sort Configuration + +Sort items before processing: + +```json +{ + "field": "raw.LastUpdatedTime", + "order": "desc" +} +``` + +### Sort Options + +- `field` - Field path to sort by (e.g., `raw.LastUpdatedTime`, `raw.Priority`, `published`) +- `order` - `asc` (ascending) or `desc` (descending) + +### Date Format Support + +The sort function supports: +- ISO format dates +- Microsoft JSON date format: `/Date(timestamp-offset)/` (e.g., WSDOT API) +- Unix timestamps +- Common date string formats + +## Message Queuing + +Messages are queued and sent at configured intervals to prevent rate limiting: + +- `message_send_interval_seconds` - Time between sending messages from the same feed (default: 2.0 seconds) +- Messages are automatically queued and processed in order +- Each feed maintains its own send interval + +## Deduplication + +The system automatically prevents duplicate posts: + +- Items are tracked by ID in the database +- Previously processed items are skipped +- Works correctly even when sorting changes item order +- Database-backed deduplication ensures reliability across restarts + +## Rate Limiting + +The feed manager implements rate limiting: + +- Per-domain rate limiting (default: 5 seconds between requests to same domain) +- Configurable via `feed_rate_limit_seconds` +- Prevents overwhelming feed sources + +## Web Interface + +The feed management system includes a web interface accessible at `/feeds`: + +- View all feed subscriptions +- Add/edit/delete feeds +- Preview output format with live feed data +- View feed statistics and activity +- Monitor errors + +## Command Interface + +Feeds can be managed via mesh commands. The feed command requires admin access and must be sent as a direct message (DM) to the bot. The command is enabled by default. + +**Command Format:** `feed [arguments]` (DM only) + +### Available Commands + +- `feed subscribe [name] [api_config]` - Subscribe to a feed +- `feed unsubscribe [channel]` - Unsubscribe from a feed (by ID or URL) +- `feed list [channel]` - List all feed subscriptions (optionally filtered by channel) +- `feed status ` - Show detailed status for a feed +- `feed enable ` - Enable a feed subscription +- `feed disable ` - Disable a feed subscription +- `feed update [interval_seconds]` - Update feed settings +- `feed test ` - Test/validate a feed URL + +### Examples + +``` +feed subscribe rss https://alerts.example.com/rss emergency "Emergency Alerts" +feed subscribe api https://api.example.com/alerts emergency "API Alerts" '{"headers": {"Authorization": "Bearer TOKEN"}}' +feed list +feed list #alerts +feed status 1 +feed enable 1 +feed disable 1 +feed unsubscribe 1 +feed update 1 60 +``` + +**Note:** The feed command requires admin access. API feeds require JSON configuration as the last argument when subscribing. + +## Best Practices + +1. **Check Intervals**: Set appropriate intervals based on feed update frequency (60-300 seconds typical) + +2. **Message Formatting**: Keep messages under 130 characters for mesh compatibility + +3. **Filtering**: Use filters to reduce noise and only send relevant items + +4. **Rate Limiting**: Respect feed source rate limits by configuring appropriate intervals + +5. **Error Handling**: Monitor feed errors in the web interface and adjust configuration as needed + +6. **Testing**: Use the preview feature in the web interface to test output formats before enabling feeds + +## Troubleshooting + +### Feeds Not Polling + +- Verify `feed_manager_enabled = true` in config +- Check that feeds are enabled in the database +- Review bot logs for errors +- Ensure bot is connected to mesh network + +### Items Not Appearing + +- Check filter configuration - items may be filtered out +- Verify output format is correct +- Check feed activity log in web interface +- Review error log for parsing issues + +### Duplicate Messages + +- Deduplication is automatic - check if item IDs are changing +- Verify `last_item_id` is being updated correctly +- Check database for processed items + +### Rate Limiting Issues + +- Increase `feed_rate_limit_seconds` in config +- Increase `message_send_interval_seconds` for specific feeds +- Reduce `check_interval_seconds` to poll less frequently + diff --git a/modules/channel_manager.py b/modules/channel_manager.py index ef0d3bb..a9c7323 100644 --- a/modules/channel_manager.py +++ b/modules/channel_manager.py @@ -7,6 +7,7 @@ Handles efficient concurrent channel fetching with caching import asyncio import sys import os +import hashlib from typing import Dict, Any, List, Optional from meshcore import EventType @@ -104,8 +105,70 @@ class ChannelManager: # Update the bot's meshcore channels for compatibility self.bot.meshcore.channels = self._channels_cache + # Store channels in database for web viewer access + self._store_channels_in_db(valid_channels) + return valid_channels + def _store_channels_in_db(self, channels: List[Dict[str, Any]]): + """Store channel information in database for web viewer access (full refresh - clears all first)""" + try: + import sqlite3 + db_path = self.bot.db_manager.db_path + + with sqlite3.connect(db_path) as conn: + cursor = conn.cursor() + + # Clear existing channels (full refresh) + cursor.execute('DELETE FROM channels') + + # Insert all channels + for channel in channels: + self._insert_channel_in_db(cursor, channel) + + conn.commit() + self.logger.debug(f"Stored {len(channels)} channels in database (full refresh)") + except Exception as e: + self.logger.warning(f"Failed to store channels in database: {e}") + + def _store_single_channel_in_db(self, channel: Dict[str, Any]): + """Store or update a single channel in database (without clearing others)""" + try: + import sqlite3 + db_path = self.bot.db_manager.db_path + + with sqlite3.connect(db_path) as conn: + cursor = conn.cursor() + self._insert_channel_in_db(cursor, channel) + conn.commit() + self.logger.debug(f"Stored/updated channel {channel.get('channel_idx')} in database") + except Exception as e: + self.logger.warning(f"Failed to store single channel in database: {e}") + + def _insert_channel_in_db(self, cursor, channel: Dict[str, Any]): + """Helper method to insert/update a single channel in database""" + channel_idx = channel.get('channel_idx') + channel_name = channel.get('channel_name', '') + channel_key_hex = channel.get('channel_key_hex', '') + + # Determine channel type based on key derivation + # If key matches hashtag derivation, it's a hashtag channel + channel_type = 'hashtag' # Default assumption + if channel_name and channel_key_hex: + # Check if key matches hashtag derivation + expected_key = self.generate_hashtag_key(channel_name) + if expected_key.hex() == channel_key_hex: + channel_type = 'hashtag' + else: + channel_type = 'custom' + + if channel_name: # Only store non-empty channels + cursor.execute(''' + INSERT OR REPLACE INTO channels + (channel_idx, channel_name, channel_type, channel_key_hex, last_updated) + VALUES (?, ?, ?, ?, CURRENT_TIMESTAMP) + ''', (channel_idx, channel_name, channel_type, channel_key_hex)) + async def _fetch_single_channel(self, channel_idx: int) -> Optional[Dict[str, Any]]: """ Fetch a single channel with error handling @@ -293,3 +356,354 @@ class ChannelManager: """Invalidate the channels cache""" self._cache_valid = False self.logger.debug("Channels cache invalidated") + + @staticmethod + def generate_hashtag_key(channel_name: str) -> bytes: + """ + Generate a hashtag channel key from the channel name + + The key is the first 16 bytes of the SHA256 hash of the channel name + (including the # symbol), converted to lowercase. + + Args: + channel_name: The channel name (e.g., "#general" or "general") + + Returns: + 16-byte key for the hashtag channel + """ + # Ensure channel name starts with # and is lowercase + if not channel_name.startswith('#'): + channel_name = '#' + channel_name + channel_name_lower = channel_name.lower() + + # Compute SHA256 hash + hash_obj = hashlib.sha256(channel_name_lower.encode('utf-8')) + hash_bytes = hash_obj.digest() + + # Take first 16 bytes + return hash_bytes[:16] + + async def add_hashtag_channel(self, channel_idx: int, channel_name: str) -> bool: + """ + Add or update a hashtag channel on the radio + + Hashtag channels use publicly derivable keys based on the channel name. + The firmware automatically generates the key when the channel name starts with #. + + Args: + channel_idx: The channel index (0-39) + channel_name: The name of the channel (with or without # prefix) + + Returns: + True if successful, False otherwise + """ + # Ensure channel name has # prefix for consistency + if not channel_name.startswith('#'): + channel_name = '#' + channel_name + + self.logger.info(f"Adding hashtag channel {channel_idx}: {channel_name}") + + # Use the simplified add_channel method - firmware will auto-generate key + return await self.add_channel(channel_idx, channel_name) + + async def add_channel(self, channel_idx: int, channel_name: str, channel_secret: Optional[bytes] = None, channel_secret_hex: Optional[str] = None) -> bool: + """ + Add or update a channel on the radio + + For hashtag channels (name starts with #), the firmware automatically generates the key. + For custom channels, provide either channel_secret (bytes) or channel_secret_hex (hex string). + + Args: + channel_idx: The channel index (0-39) + channel_name: The name of the channel + channel_secret: Optional 16-byte encryption key for custom channels + channel_secret_hex: Optional hex string (32 chars) for the encryption key. Takes precedence over channel_secret. + + Returns: + True if successful, False otherwise + """ + if not self.bot.connected or not self.bot.meshcore: + self.logger.error("Not connected to MeshCore node") + return False + + if channel_idx < 0 or channel_idx >= self.max_channels: + self.logger.error(f"Channel index {channel_idx} out of range (0-{self.max_channels-1})") + return False + + try: + # Check if this is a hashtag channel (firmware auto-generates key) + is_hashtag = channel_name.startswith('#') + + # For custom channels, validate and prepare the key + if not is_hashtag: + if channel_secret_hex: + # Validate hex string + if len(channel_secret_hex) != 32: + self.logger.error(f"Channel secret hex must be exactly 32 characters (16 bytes), got {len(channel_secret_hex)}") + return False + try: + channel_secret = bytes.fromhex(channel_secret_hex) + except ValueError as e: + self.logger.error(f"Invalid hex string for channel secret: {e}") + return False + elif channel_secret is None: + self.logger.error("Custom channel requires a channel key (channel_secret or channel_secret_hex)") + return False + elif len(channel_secret) != 16: + self.logger.error(f"Channel secret must be exactly 16 bytes, got {len(channel_secret)}") + return False + + self.logger.info(f"Adding custom channel {channel_idx}: {channel_name} (key: {channel_secret.hex()[:8]}...)") + else: + self.logger.info(f"Adding hashtag channel {channel_idx}: {channel_name} (firmware will auto-generate key)") + + # Use meshcore.commands.set_channel API directly + if hasattr(self.bot.meshcore, 'commands') and hasattr(self.bot.meshcore.commands, 'set_channel'): + # For hashtag channels, just pass the name (firmware generates key) + if is_hashtag: + res = await self.bot.meshcore.commands.set_channel(channel_idx, channel_name) + else: + # For custom channels, we need to pass the key + # Check if set_channel accepts a key parameter + # Try with key as third parameter + try: + res = await self.bot.meshcore.commands.set_channel(channel_idx, channel_name, channel_secret) + except TypeError: + # If that doesn't work, try with hex string + try: + res = await self.bot.meshcore.commands.set_channel(channel_idx, channel_name, channel_secret_hex or channel_secret.hex()) + except TypeError: + # Fallback to CLI method if API doesn't support key parameter + self.logger.warning("meshcore.commands.set_channel doesn't accept key parameter, using CLI fallback") + return await self._add_channel_via_cli(channel_idx, channel_name, channel_secret.hex() if channel_secret else channel_secret_hex) + + # Check for errors + if hasattr(res, 'type') and res.type == EventType.ERROR: + self.logger.error(f"Failed to set channel {channel_idx}: {res.payload if hasattr(res, 'payload') else 'Unknown error'}") + return False + + # Fetch the channel back to get the generated key and verify + res = await self.bot.meshcore.commands.get_channel(channel_idx) + + if hasattr(res, 'type') and res.type == EventType.ERROR: + self.logger.error(f"Failed to get channel {channel_idx} after setting: {res.payload if hasattr(res, 'payload') else 'Unknown error'}") + return False + + # Extract channel info from response + if hasattr(res, 'payload'): + channel_info = res.payload + else: + # Fallback: try to get from event subscription + channel_info = await self._fetch_single_channel(channel_idx) + if not channel_info: + self.logger.error(f"Could not retrieve channel {channel_idx} after setting") + return False + + # Verify channel was set correctly + if channel_info.get('channel_name') != channel_name: + self.logger.error(f"Channel name mismatch: expected {channel_name}, got {channel_info.get('channel_name')}") + return False + + # For custom channels, verify the key matches + if not is_hashtag: + channel_secret_from_device = channel_info.get('channel_secret', b'') + if isinstance(channel_secret_from_device, bytes) and channel_secret_from_device != channel_secret: + self.logger.error(f"Channel key mismatch for custom channel {channel_idx}") + return False + + # Update cache and database + channel_info['channel_key_hex'] = channel_info.get('channel_secret', b'').hex() if isinstance(channel_info.get('channel_secret'), bytes) else '' + self._channels_cache[channel_idx] = channel_info + self._store_single_channel_in_db(channel_info) + + self.logger.info(f"Successfully added channel {channel_idx}: {channel_name}") + return True + else: + # Fallback to CLI method if commands API not available + self.logger.warning("meshcore.commands.set_channel not available, using CLI fallback") + channel_secret_hex = channel_secret.hex() if channel_secret else channel_secret_hex + if is_hashtag: + # For hashtag, generate key ourselves as fallback + channel_secret = self.generate_hashtag_key(channel_name) + channel_secret_hex = channel_secret.hex() + return await self._add_channel_via_cli(channel_idx, channel_name, channel_secret_hex) + + except Exception as e: + self.logger.error(f"Error adding channel {channel_idx}: {e}") + import traceback + self.logger.debug(traceback.format_exc()) + return False + + async def _add_channel_via_cli(self, channel_idx: int, channel_name: str, channel_secret_hex: str) -> bool: + """ + Fallback method to add channel using CLI wrapper (for older meshcore versions) + + Args: + channel_idx: The channel index + channel_name: The channel name + channel_secret_hex: The channel key as hex string + + Returns: + True if successful, False otherwise + """ + try: + # Subscribe to channel info events to confirm the channel was set + channel_set = False + event_received = asyncio.Event() + + async def on_channel_info(event): + nonlocal channel_set + if event.payload.get('channel_idx') == channel_idx: + payload = event.payload + if payload.get('channel_name') == channel_name: + channel_set = True + event_received.set() + + subscription = self.bot.meshcore.subscribe(EventType.CHANNEL_INFO, on_channel_info) + + try: + from meshcore_cli.meshcore_cli import next_cmd + + # Suppress raw JSON output + with open(os.devnull, 'w') as devnull: + old_stdout = sys.stdout + sys.stdout = devnull + try: + await next_cmd( + self.bot.meshcore, + ["set_channel", str(channel_idx), channel_name, channel_secret_hex] + ) + finally: + sys.stdout = old_stdout + + # Wait for confirmation with timeout + try: + await asyncio.wait_for(event_received.wait(), timeout=self._fetch_timeout * 2) + except asyncio.TimeoutError: + self.logger.warning(f"Timeout waiting for channel {channel_idx} set confirmation") + await asyncio.sleep(0.5) + result = await self._fetch_single_channel(channel_idx) + if result and result.get('channel_name') == channel_name: + channel_set = True + + if channel_set: + # Update cache + result = await self._fetch_single_channel(channel_idx) + if result: + self._channels_cache[channel_idx] = result + self._store_single_channel_in_db(result) + self.logger.info(f"Successfully added channel {channel_idx}: {channel_name}") + return True + else: + self.logger.warning(f"Channel {channel_idx} was set but could not be verified") + return False + else: + self.logger.error(f"Failed to set channel {channel_idx}") + return False + + finally: + self.bot.meshcore.unsubscribe(subscription) + + except Exception as e: + self.logger.error(f"Error in CLI fallback for channel {channel_idx}: {e}") + return False + + async def remove_channel(self, channel_idx: int) -> bool: + """ + Remove a channel from the radio by clearing it + + Args: + channel_idx: The channel index to remove + + Returns: + True if successful, False otherwise + """ + if not self.bot.connected or not self.bot.meshcore: + self.logger.error("Not connected to MeshCore node") + return False + + if channel_idx < 0 or channel_idx >= self.max_channels: + self.logger.error(f"Channel index {channel_idx} out of range (0-{self.max_channels-1})") + return False + + try: + self.logger.info(f"Removing channel {channel_idx}") + + # Create all-zero channel secret (16 bytes) to clear the channel + empty_secret = b'\x00' * 16 + empty_secret_hex = empty_secret.hex() + + # Subscribe to channel info events to confirm the channel was cleared + channel_cleared = False + event_received = asyncio.Event() + + async def on_channel_info(event): + nonlocal channel_cleared + if event.payload.get('channel_idx') == channel_idx: + payload = event.payload + event_secret = payload.get('channel_secret', b'') + # Check if the channel was cleared (all zeros or empty name) + if isinstance(event_secret, bytes) and event_secret == empty_secret: + channel_cleared = True + event_received.set() + elif not payload.get('channel_name') or payload.get('channel_name') == '': + channel_cleared = True + event_received.set() + + subscription = self.bot.meshcore.subscribe(EventType.CHANNEL_INFO, on_channel_info) + + try: + from meshcore_cli.meshcore_cli import next_cmd + + # Suppress raw JSON output + with open(os.devnull, 'w') as devnull: + old_stdout = sys.stdout + sys.stdout = devnull + try: + # Clear the channel by setting it with empty name and all-zero secret + # Format: set_channel "" + await next_cmd( + self.bot.meshcore, + ["set_channel", str(channel_idx), "", empty_secret_hex] + ) + finally: + sys.stdout = old_stdout + + # Wait for confirmation with timeout + try: + await asyncio.wait_for(event_received.wait(), timeout=self._fetch_timeout * 2) + except asyncio.TimeoutError: + self.logger.warning(f"Timeout waiting for channel {channel_idx} removal confirmation") + # Still try to verify by fetching the channel + await asyncio.sleep(0.5) + result = await self._fetch_single_channel(channel_idx) + if not result or not result.get('channel_name'): + channel_cleared = True + + if channel_cleared: + # Remove from cache + if channel_idx in self._channels_cache: + del self._channels_cache[channel_idx] + # Update database - remove the channel + try: + import sqlite3 + db_path = self.bot.db_manager.db_path + with sqlite3.connect(db_path) as conn: + cursor = conn.cursor() + cursor.execute('DELETE FROM channels WHERE channel_idx = ?', (channel_idx,)) + conn.commit() + except Exception as e: + self.logger.warning(f"Failed to remove channel from database: {e}") + self.logger.info(f"Successfully removed channel {channel_idx}") + return True + else: + self.logger.error(f"Failed to remove channel {channel_idx}") + return False + + finally: + # Unsubscribe + self.bot.meshcore.unsubscribe(subscription) + + except Exception as e: + self.logger.error(f"Error removing channel {channel_idx}: {e}") + return False \ No newline at end of file diff --git a/modules/command_manager.py b/modules/command_manager.py index d89b839..6218346 100644 --- a/modules/command_manager.py +++ b/modules/command_manager.py @@ -205,7 +205,24 @@ class CommandManager: async def handle_advert_command(self, message: MeshMessage): """Handle the advert command from DM""" - await self.commands['advert'].execute(message) + command = self.commands['advert'] + success = await command.execute(message) + + # Small delay to ensure send_response has completed + await asyncio.sleep(0.1) + + # Determine if a response was sent + response_sent = False + if hasattr(command, 'last_response') and command.last_response: + response_sent = True + elif hasattr(self, '_last_response') and self._last_response: + response_sent = True + + # Record command execution in stats database + if 'stats' in self.commands: + stats_command = self.commands['stats'] + if stats_command: + stats_command.record_command(message, 'advert', response_sent) async def send_dm(self, recipient_id: str, content: str) -> bool: """Send a direct message using meshcore-cli command""" @@ -528,6 +545,7 @@ class CommandManager: # Check if command can execute (cooldown, DM requirements, etc.) if not command.can_execute_now(message): + response_sent = False # For DM-only commands in public channels, only show error if channel is allowed # (i.e., channel is in monitor_channels or command's allowed_channels) # This prevents prompting users in channels where the command shouldn't work at all @@ -536,10 +554,12 @@ class CommandManager: if command.is_channel_allowed(message): error_msg = command.translate('errors.dm_only', command=command_name) await self.send_response(message, error_msg) + response_sent = True # Otherwise, silently ignore (channel not configured for this command) elif command.requires_admin_access(): error_msg = command.translate('errors.access_denied', command=command_name) await self.send_response(message, error_msg) + response_sent = True elif hasattr(command, 'get_remaining_cooldown') and callable(command.get_remaining_cooldown): # Check if it's the per-user version (takes user_id parameter) import inspect @@ -552,6 +572,14 @@ class CommandManager: if remaining > 0: error_msg = command.translate('errors.cooldown', command=command_name, seconds=remaining) await self.send_response(message, error_msg) + response_sent = True + + # Record command execution in stats database (even if it failed checks) + if 'stats' in self.commands: + stats_command = self.commands['stats'] + if stats_command: + stats_command.record_command(message, command_name, response_sent) + return try: @@ -567,22 +595,33 @@ class CommandManager: # Execute the command success = await command.execute(message) - # Capture command data for web viewer (with small delay to ensure response is set) + # Small delay to ensure send_response has completed + await asyncio.sleep(0.1) + + # Determine if a response was sent by checking response tracking + response_sent = False + response = None + if hasattr(command, 'last_response') and command.last_response: + response = command.last_response + response_sent = True + elif hasattr(self, '_last_response') and self._last_response: + response = self._last_response + response_sent = True + + # Record command execution in stats database + if 'stats' in self.commands: + stats_command = self.commands['stats'] + if stats_command: + stats_command.record_command(message, command_name, response_sent) + + # Capture command data for web viewer if (hasattr(self.bot, 'web_viewer_integration') and self.bot.web_viewer_integration and self.bot.web_viewer_integration.bot_integration): try: - # Small delay to ensure send_response has completed - await asyncio.sleep(0.1) - - # Get the response that was sent (if any) - # Prioritize command.last_response (full response) over _last_response (may be split) - # This ensures commands like path that split messages still show full response in webviewer - response = "Command executed" # Default response - if hasattr(command, 'last_response') and command.last_response: - response = command.last_response - elif hasattr(self, '_last_response') and self._last_response: - response = self._last_response + # Use the response we found, or default + if response is None: + response = "Command executed" self.bot.web_viewer_integration.bot_integration.capture_command( message, command_name, response, success if success is not None else True @@ -596,6 +635,12 @@ class CommandManager: error_msg = command.translate('errors.execution_error', command=command_name, error=str(e)) await self.send_response(message, error_msg) + # Record command execution in stats database (error response was sent) + if 'stats' in self.commands: + stats_command = self.commands['stats'] + if stats_command: + stats_command.record_command(message, command_name, True) # Error message counts as response + # Capture failed command for web viewer if (hasattr(self.bot, 'web_viewer_integration') and self.bot.web_viewer_integration and diff --git a/modules/commands/feed_command.py b/modules/commands/feed_command.py new file mode 100644 index 0000000..c563c11 --- /dev/null +++ b/modules/commands/feed_command.py @@ -0,0 +1,417 @@ +#!/usr/bin/env python3 +""" +Feed command for the MeshCore Bot +Handles RSS and API feed subscription management +""" + +import json +import re +from typing import Optional, List, Dict, Any +from urllib.parse import urlparse +from .base_command import BaseCommand +from ..models import MeshMessage + + +class FeedCommand(BaseCommand): + """Handles feed subscription management""" + + # Plugin metadata + name = "feed" + keywords = ['feed', 'feeds', 'rss', 'subscription', 'subscriptions'] + description = "Manage RSS and API feed subscriptions (usage: feed subscribe rss [name])" + category = "admin" + requires_dm = True + cooldown_seconds = 2 + + def __init__(self, bot): + super().__init__(bot) + self.db_path = bot.db_manager.db_path + + def can_execute(self, message: MeshMessage) -> bool: + """Check if this command can be executed (admin only)""" + if not self.requires_admin_access(): + return False + return super().can_execute(message) + + def requires_admin_access(self) -> bool: + """Feed command requires admin access""" + return True + + async def execute(self, message: MeshMessage) -> bool: + """Execute the feed command""" + content = message.content.strip() + + # Handle exclamation prefix + if content.startswith('!'): + content = content[1:].strip() + + # Parse command + parts = content.split() + if len(parts) < 2: + return await self.send_response(message, self.get_help_text()) + + subcommand = parts[1].lower() + + if subcommand == 'subscribe': + return await self._handle_subscribe(message, parts[2:]) + elif subcommand == 'unsubscribe': + return await self._handle_unsubscribe(message, parts[2:]) + elif subcommand == 'list': + return await self._handle_list(message, parts[2:]) + elif subcommand == 'status': + return await self._handle_status(message, parts[2:]) + elif subcommand == 'test': + return await self._handle_test(message, parts[2:]) + elif subcommand == 'enable': + return await self._handle_enable_disable(message, parts[2:], True) + elif subcommand == 'disable': + return await self._handle_enable_disable(message, parts[2:], False) + elif subcommand == 'update': + return await self._handle_update(message, parts[2:]) + else: + return await self.send_response(message, self.get_help_text()) + + def get_help_text(self) -> str: + """Get help text for feed command""" + return """Feed Command Usage: +feed subscribe [name] +feed unsubscribe +feed list [channel] +feed status +feed test +feed enable +feed disable +feed update [interval_seconds] + +Examples: +feed subscribe rss https://alerts.example.com/rss emergency "Emergency Alerts" +feed subscribe api https://api.example.com/alerts emergency "API Alerts" '{"headers": {"Authorization": "Bearer TOKEN"}}' +feed list +feed status 1""" + + async def _handle_subscribe(self, message: MeshMessage, args: List[str]) -> bool: + """Handle feed subscribe command""" + if len(args) < 3: + return await self.send_response(message, "Usage: feed subscribe [name] [api_config]") + + feed_type = args[0].lower() + if feed_type not in ['rss', 'api']: + return await self.send_response(message, "Feed type must be 'rss' or 'api'") + + feed_url = args[1] + channel_name = args[2] + feed_name = args[3] if len(args) > 3 else None + api_config = args[4] if len(args) > 4 and feed_type == 'api' else None + + # Validate URL + if not self._validate_url(feed_url): + return await self.send_response(message, "Invalid URL format") + + # Validate channel exists + channel_num = self.bot.channel_manager.get_channel_number(channel_name) + if channel_num is None: + return await self.send_response(message, f"Channel '{channel_name}' not found. Create it first or use a valid channel name.") + + # Parse API config if provided + api_config_json = None + if feed_type == 'api' and api_config: + try: + api_config_json = json.loads(api_config) + except json.JSONDecodeError: + return await self.send_response(message, "Invalid API config JSON") + + # Create subscription + try: + feed_id = self._create_subscription( + feed_type=feed_type, + feed_url=feed_url, + channel_name=channel_name, + feed_name=feed_name, + api_config=api_config_json + ) + + response = f"Subscribed to {feed_type.upper()} feed" + if feed_name: + response += f" '{feed_name}'" + response += f" -> channel: {channel_name} (ID: {feed_id})" + return await self.send_response(message, response) + + except Exception as e: + self.logger.error(f"Error creating subscription: {e}") + return await self.send_response(message, f"Error creating subscription: {str(e)}") + + async def _handle_unsubscribe(self, message: MeshMessage, args: List[str]) -> bool: + """Handle feed unsubscribe command""" + if len(args) < 1: + return await self.send_response(message, "Usage: feed unsubscribe [channel]") + + identifier = args[0] + channel_name = args[1] if len(args) > 1 else None + + try: + # Try as ID first + try: + feed_id = int(identifier) + success = self._delete_subscription_by_id(feed_id) + except ValueError: + # Try as URL + if channel_name: + success = self._delete_subscription_by_url(identifier, channel_name) + else: + return await self.send_response(message, "Channel name required when using URL") + + if success: + return await self.send_response(message, f"Unsubscribed from feed (ID: {identifier})") + else: + return await self.send_response(message, "Feed subscription not found") + + except Exception as e: + self.logger.error(f"Error unsubscribing: {e}") + return await self.send_response(message, f"Error unsubscribing: {str(e)}") + + async def _handle_list(self, message: MeshMessage, args: List[str]) -> bool: + """Handle feed list command""" + channel_filter = args[0] if args else None + + try: + feeds = self._get_subscriptions(channel_filter) + + if not feeds: + response = "No feed subscriptions" + if channel_filter: + response += f" for channel '{channel_filter}'" + return await self.send_response(message, response) + + response = f"Feed Subscriptions ({len(feeds)}):\n" + for feed in feeds[:10]: # Limit to 10 for mesh message + status = "enabled" if feed['enabled'] else "disabled" + name = feed.get('feed_name') or feed['feed_url'][:30] + response += f"{feed['id']}. {name} ({feed['feed_type']}) -> {feed['channel_name']} [{status}]\n" + + if len(feeds) > 10: + response += f"({len(feeds) - 10} more...)" + + return await self.send_response(message, response) + + except Exception as e: + self.logger.error(f"Error listing feeds: {e}") + return await self.send_response(message, f"Error listing feeds: {str(e)}") + + async def _handle_status(self, message: MeshMessage, args: List[str]) -> bool: + """Handle feed status command""" + if not args: + return await self.send_response(message, "Usage: feed status ") + + try: + feed_id = int(args[0]) + feed = self._get_subscription_by_id(feed_id) + + if not feed: + return await self.send_response(message, f"Feed subscription {feed_id} not found") + + status = "enabled" if feed['enabled'] else "disabled" + last_check = feed.get('last_check_time') or "Never" + last_item = feed.get('last_item_id') or "None" + + response = f"Feed {feed_id} Status:\n" + response += f"Name: {feed.get('feed_name') or 'N/A'}\n" + response += f"Type: {feed['feed_type']}\n" + response += f"URL: {feed['feed_url']}\n" + response += f"Channel: {feed['channel_name']}\n" + response += f"Status: {status}\n" + response += f"Interval: {feed.get('check_interval_seconds', 300)}s\n" + response += f"Last check: {last_check}\n" + response += f"Last item: {last_item[:30] if last_item != 'None' else 'None'}" + + return await self.send_response(message, response) + + except ValueError: + return await self.send_response(message, "Invalid feed ID") + except Exception as e: + self.logger.error(f"Error getting feed status: {e}") + return await self.send_response(message, f"Error getting feed status: {str(e)}") + + async def _handle_test(self, message: MeshMessage, args: List[str]) -> bool: + """Handle feed test command""" + if not args: + return await self.send_response(message, "Usage: feed test ") + + feed_url = args[0] + + if not self._validate_url(feed_url): + return await self.send_response(message, "Invalid URL format") + + # Test would require feed_manager to be available + # For now, just validate URL + return await self.send_response(message, f"URL validated: {feed_url}\n(Full test requires feed manager)") + + async def _handle_enable_disable(self, message: MeshMessage, args: List[str], enable: bool) -> bool: + """Handle enable/disable command""" + if not args: + return await self.send_response(message, f"Usage: feed {'enable' if enable else 'disable'} ") + + try: + feed_id = int(args[0]) + success = self._set_subscription_enabled(feed_id, enable) + + if success: + status = "enabled" if enable else "disabled" + return await self.send_response(message, f"Feed {feed_id} {status}") + else: + return await self.send_response(message, f"Feed subscription {feed_id} not found") + + except ValueError: + return await self.send_response(message, "Invalid feed ID") + except Exception as e: + self.logger.error(f"Error setting feed status: {e}") + return await self.send_response(message, f"Error: {str(e)}") + + async def _handle_update(self, message: MeshMessage, args: List[str]) -> bool: + """Handle update command""" + if not args: + return await self.send_response(message, "Usage: feed update [interval_seconds]") + + try: + feed_id = int(args[0]) + interval = int(args[1]) if len(args) > 1 else None + + success = self._update_subscription(feed_id, interval) + + if success: + response = f"Feed {feed_id} updated" + if interval: + response += f" (interval: {interval}s)" + return await self.send_response(message, response) + else: + return await self.send_response(message, f"Feed subscription {feed_id} not found") + + except ValueError: + return await self.send_response(message, "Invalid feed ID or interval") + except Exception as e: + self.logger.error(f"Error updating feed: {e}") + return await self.send_response(message, f"Error: {str(e)}") + + def _validate_url(self, url: str) -> bool: + """Validate URL format""" + try: + result = urlparse(url) + return all([result.scheme in ['http', 'https'], result.netloc]) + except Exception: + return False + + def _create_subscription(self, feed_type: str, feed_url: str, channel_name: str, + feed_name: Optional[str] = None, api_config: Optional[Dict] = None) -> int: + """Create a new feed subscription""" + import sqlite3 + + with sqlite3.connect(self.db_path) as conn: + cursor = conn.cursor() + + # Get default check interval + default_interval = self.bot.config.getint('Feed_Manager', 'default_check_interval_seconds', fallback=300) + + api_config_str = json.dumps(api_config) if api_config else None + + cursor.execute(''' + INSERT INTO feed_subscriptions + (feed_type, feed_url, channel_name, feed_name, check_interval_seconds, api_config) + VALUES (?, ?, ?, ?, ?, ?) + ''', (feed_type, feed_url, channel_name, feed_name, default_interval, api_config_str)) + + conn.commit() + return cursor.lastrowid + + def _delete_subscription_by_id(self, feed_id: int) -> bool: + """Delete subscription by ID""" + import sqlite3 + + with sqlite3.connect(self.db_path) as conn: + cursor = conn.cursor() + cursor.execute('DELETE FROM feed_subscriptions WHERE id = ?', (feed_id,)) + conn.commit() + return cursor.rowcount > 0 + + def _delete_subscription_by_url(self, feed_url: str, channel_name: str) -> bool: + """Delete subscription by URL and channel""" + import sqlite3 + + with sqlite3.connect(self.db_path) as conn: + cursor = conn.cursor() + cursor.execute(''' + DELETE FROM feed_subscriptions + WHERE feed_url = ? AND channel_name = ? + ''', (feed_url, channel_name)) + conn.commit() + return cursor.rowcount > 0 + + def _get_subscriptions(self, channel_filter: Optional[str] = None) -> List[Dict]: + """Get all subscriptions, optionally filtered by channel""" + import sqlite3 + + with sqlite3.connect(self.db_path) as conn: + conn.row_factory = sqlite3.Row + cursor = conn.cursor() + + if channel_filter: + cursor.execute(''' + SELECT * FROM feed_subscriptions + WHERE channel_name = ? + ORDER BY id + ''', (channel_filter,)) + else: + cursor.execute(''' + SELECT * FROM feed_subscriptions + ORDER BY id + ''') + + rows = cursor.fetchall() + return [dict(row) for row in rows] + + def _get_subscription_by_id(self, feed_id: int) -> Optional[Dict]: + """Get subscription by ID""" + import sqlite3 + + with sqlite3.connect(self.db_path) as conn: + conn.row_factory = sqlite3.Row + cursor = conn.cursor() + cursor.execute('SELECT * FROM feed_subscriptions WHERE id = ?', (feed_id,)) + row = cursor.fetchone() + return dict(row) if row else None + + def _set_subscription_enabled(self, feed_id: int, enabled: bool) -> bool: + """Enable or disable a subscription""" + import sqlite3 + + with sqlite3.connect(self.db_path) as conn: + cursor = conn.cursor() + cursor.execute(''' + UPDATE feed_subscriptions + SET enabled = ?, updated_at = CURRENT_TIMESTAMP + WHERE id = ? + ''', (1 if enabled else 0, feed_id)) + conn.commit() + return cursor.rowcount > 0 + + def _update_subscription(self, feed_id: int, interval: Optional[int] = None) -> bool: + """Update subscription settings""" + import sqlite3 + + with sqlite3.connect(self.db_path) as conn: + cursor = conn.cursor() + + if interval: + cursor.execute(''' + UPDATE feed_subscriptions + SET check_interval_seconds = ?, updated_at = CURRENT_TIMESTAMP + WHERE id = ? + ''', (interval, feed_id)) + else: + cursor.execute(''' + UPDATE feed_subscriptions + SET updated_at = CURRENT_TIMESTAMP + WHERE id = ? + ''', (feed_id,)) + + conn.commit() + return cursor.rowcount > 0 + diff --git a/modules/commands/greeter_command.py b/modules/commands/greeter_command.py index 83fecdf..6a5ab63 100644 --- a/modules/commands/greeter_command.py +++ b/modules/commands/greeter_command.py @@ -6,6 +6,7 @@ Greets users on their first public channel message with mesh information import sqlite3 import time +import asyncio from datetime import datetime, timedelta from typing import Optional, Dict, Any, List from .base_command import BaseCommand @@ -26,6 +27,9 @@ class GreeterCommand(BaseCommand): self._init_greeter_tables() self._load_config() + # Track pending greetings (for dead air delay) + self.pending_greetings = {} # key: (sender_id, channel), value: asyncio.Task + # Auto-backfill if enabled if self.enabled and self.auto_backfill: self.logger.info("Auto-backfill enabled - backfilling greeted users from historical data") @@ -166,6 +170,14 @@ class GreeterCommand(BaseCommand): self.greeting_parts = [part.strip() for part in self.greeting_message.split('|') if part.strip()] else: self.greeting_parts = [self.greeting_message] + + # Dead air delay settings + self.dead_air_delay_seconds = self.get_config_value('Greeter_Command', 'dead_air_delay_seconds', + fallback=0, value_type='int') + self.defer_to_human_greeting = self.get_config_value('Greeter_Command', 'defer_to_human_greeting', + fallback=False, value_type='bool') + self.levenshtein_distance = self.get_config_value('Greeter_Command', 'levenshtein_distance', + fallback=0, value_type='int') def _init_greeter_tables(self): """Initialize database tables for greeter tracking""" @@ -508,9 +520,83 @@ class GreeterCommand(BaseCommand): self.logger.error(f"Error starting rollout: {e}") return False + def _levenshtein_distance(self, s1: str, s2: str) -> int: + """ + Calculate Levenshtein distance between two strings + + Args: + s1: First string + s2: Second string + + Returns: + Levenshtein distance (number of edits needed) + """ + if len(s1) < len(s2): + return self._levenshtein_distance(s2, s1) + + if len(s2) == 0: + return len(s1) + + previous_row = range(len(s2) + 1) + for i, c1 in enumerate(s1): + current_row = [i + 1] + for j, c2 in enumerate(s2): + insertions = previous_row[j + 1] + 1 + deletions = current_row[j] + 1 + substitutions = previous_row[j] + (c1 != c2) + current_row.append(min(insertions, deletions, substitutions)) + previous_row = current_row + + return previous_row[-1] + + def _find_similar_greeted_user(self, sender_id: str, channel: str) -> Optional[str]: + """ + Find if a user with a similar name (within Levenshtein distance) has been greeted + + Args: + sender_id: The user's ID to check + channel: The channel name (used only if per_channel_greetings is True) + + Returns: + The greeted sender_id if a similar one is found, None otherwise + """ + if self.levenshtein_distance <= 0: + return None + + try: + with sqlite3.connect(self.bot.db_manager.db_path) as conn: + cursor = conn.cursor() + + if self.per_channel_greetings: + # Per-channel mode: check greeted users on this specific channel + cursor.execute(''' + SELECT DISTINCT sender_id FROM greeted_users + WHERE channel = ? + ''', (channel,)) + else: + # Global mode: check all greeted users (channel = NULL) + cursor.execute(''' + SELECT DISTINCT sender_id FROM greeted_users + WHERE channel IS NULL + ''') + + greeted_users = cursor.fetchall() + + # Check each greeted user for similarity + for (greeted_id,) in greeted_users: + distance = self._levenshtein_distance(sender_id.lower(), greeted_id.lower()) + if distance <= self.levenshtein_distance: + self.logger.debug(f"Found similar user: {greeted_id} (distance: {distance} from {sender_id})") + return greeted_id + + return None + except Exception as e: + self.logger.error(f"Error checking for similar greeted users: {e}") + return None + def has_been_greeted(self, sender_id: str, channel: str) -> bool: """ - Check if a user has been greeted + Check if a user has been greeted (with optional Levenshtein distance matching) Args: sender_id: The user's ID @@ -536,7 +622,17 @@ class GreeterCommand(BaseCommand): WHERE sender_id = ? AND channel IS NULL ''', (sender_id,)) - return cursor.fetchone() is not None + if cursor.fetchone() is not None: + return True + + # If exact match not found and Levenshtein distance is enabled, check for similar names + if self.levenshtein_distance > 0: + similar_user = self._find_similar_greeted_user(sender_id, channel) + if similar_user: + self.logger.info(f"User {sender_id} matches previously greeted user {similar_user} (Levenshtein distance enabled)") + return True + + return False except Exception as e: self.logger.error(f"Error checking if user has been greeted: {e}") return False @@ -587,11 +683,11 @@ class GreeterCommand(BaseCommand): count = cursor.fetchone()[0] if count > 1: # Duplicates exist - clean them up, keeping the earliest (first) greeting - cursor.execute(''' - SELECT id FROM greeted_users - WHERE sender_id = ? AND channel = ? + cursor.execute(''' + SELECT id FROM greeted_users + WHERE sender_id = ? AND channel = ? ORDER BY greeted_at ASC - ''', (sender_id, channel)) + ''', (sender_id, channel)) all_ids = [row[0] for row in cursor.fetchall()] if len(all_ids) > 1: # Delete all but the first (earliest) @@ -639,11 +735,11 @@ class GreeterCommand(BaseCommand): count = cursor.fetchone()[0] if count > 1: # Duplicates exist - clean them up, keeping the earliest (first) greeting - cursor.execute(''' - SELECT id FROM greeted_users - WHERE sender_id = ? AND channel IS NULL + cursor.execute(''' + SELECT id FROM greeted_users + WHERE sender_id = ? AND channel IS NULL ORDER BY greeted_at ASC - ''', (sender_id,)) + ''', (sender_id,)) all_ids = [row[0] for row in cursor.fetchall()] if len(all_ids) > 1: # Delete all but the first (earliest) @@ -956,6 +1052,182 @@ class GreeterCommand(BaseCommand): self.logger.error(traceback.format_exc()) return False + def _check_human_greeting(self, new_user_id: str, channel: str, since_timestamp: int) -> bool: + """ + Check if a human has greeted the new user by mentioning their name in a message + + Args: + new_user_id: The new user's ID to check for + channel: The channel to check + since_timestamp: Only check messages after this timestamp + + Returns: + True if a human (not the new user) has mentioned the new user's name + """ + if not self.defer_to_human_greeting: + return False + + try: + with sqlite3.connect(self.bot.db_manager.db_path) as conn: + cursor = conn.cursor() + + # Check if message_stats table exists + cursor.execute(''' + SELECT name FROM sqlite_master + WHERE type='table' AND name='message_stats' + ''') + if not cursor.fetchone(): + return False + + # Get recent messages from this channel since the new user posted + cursor.execute(''' + SELECT sender_id, content + FROM message_stats + WHERE channel = ? + AND timestamp >= ? + AND is_dm = 0 + AND sender_id != ? + ORDER BY timestamp DESC + ''', (channel, since_timestamp, new_user_id)) + + messages = cursor.fetchall() + + # Check if any message contains the new user's name + new_user_id_lower = new_user_id.lower() + for sender_id, content in messages: + if content and new_user_id_lower in content.lower(): + # Also check with Levenshtein distance if enabled + if self.levenshtein_distance > 0: + # Check if any word in the message is within Levenshtein distance + words = content.lower().split() + for word in words: + # Remove common punctuation + word = word.strip('.,!?;:()[]{}@') + distance = self._levenshtein_distance(new_user_id_lower, word) + if distance <= self.levenshtein_distance: + self.logger.info(f"Human greeting detected: {sender_id} mentioned {new_user_id} in channel {channel}") + return True + else: + # Simple substring match + self.logger.info(f"Human greeting detected: {sender_id} mentioned {new_user_id} in channel {channel}") + return True + + return False + except Exception as e: + self.logger.error(f"Error checking for human greeting: {e}") + return False + + def _cancel_pending_greeting(self, sender_id: str, channel: str): + """Cancel a pending greeting if it exists""" + key = (sender_id, channel) + if key in self.pending_greetings: + task = self.pending_greetings[key] + if not task.done(): + task.cancel() + self.logger.info(f"Cancelled pending greeting for {sender_id} on {channel}") + del self.pending_greetings[key] + + async def _send_delayed_greeting(self, message: MeshMessage): + """ + Send a greeting after the dead air delay, checking for human greetings during the delay + + Args: + message: The original message that triggered the greeting + """ + key = (message.sender_id, message.channel) + original_timestamp = message.timestamp or int(time.time()) + + try: + # Wait for the dead air delay + if self.dead_air_delay_seconds > 0: + self.logger.debug(f"Waiting {self.dead_air_delay_seconds} seconds before greeting {message.sender_id} on {message.channel}") + await asyncio.sleep(self.dead_air_delay_seconds) + + # Check if greeting was cancelled (user was already greeted or human responded) + if key not in self.pending_greetings: + self.logger.debug(f"Greeting for {message.sender_id} on {message.channel} was cancelled") + return + + # Check if we should still greet (user might have been greeted by another process) + if self.has_been_greeted(message.sender_id, message.channel): + self.logger.debug(f"User {message.sender_id} already greeted on {message.channel} - skipping") + if key in self.pending_greetings: + del self.pending_greetings[key] + return + + # If defer to human greeting is enabled, check if a human has greeted the user + # Check messages from the original timestamp onwards (during the delay period) + if self.defer_to_human_greeting and self.dead_air_delay_seconds > 0: + if self._check_human_greeting(message.sender_id, message.channel, original_timestamp): + self.logger.info(f"Deferring to human greeting for {message.sender_id} on {message.channel}") + # Mark as greeted so we don't greet them later + self.mark_as_greeted(message.sender_id, message.channel) + if key in self.pending_greetings: + del self.pending_greetings[key] + return + + # Send the greeting + await self._send_greeting(message) + + # Clean up + if key in self.pending_greetings: + del self.pending_greetings[key] + + except asyncio.CancelledError: + self.logger.debug(f"Delayed greeting for {message.sender_id} on {message.channel} was cancelled") + # Clean up on cancellation + if key in self.pending_greetings: + del self.pending_greetings[key] + except Exception as e: + self.logger.error(f"Error in delayed greeting for {message.sender_id}: {e}") + if key in self.pending_greetings: + del self.pending_greetings[key] + + async def _send_greeting(self, message: MeshMessage) -> bool: + """ + Actually send the greeting message (extracted from execute for reuse) + + Args: + message: The message that triggered the greeting + + Returns: + True if greeting was sent successfully + """ + try: + # Format greeting parts (may be single or multi-part) + # Pass channel name for channel-specific greetings + greeting_parts = await self._format_greeting_parts(message.sender_id, message.channel) + + # Send greeting(s) + mode_str = "per-channel" if self.per_channel_greetings else "global" + self.logger.info(f"Greeting {message.sender_id} on channel {message.channel} ({mode_str} mode, {len(greeting_parts)} part(s))") + + # Log database verification + total_greeted = self.get_greeted_users_count() + self.logger.debug(f"Database verification: {total_greeted} total user(s) marked as greeted") + + # Send all greeting parts + success = True + for i, greeting_part in enumerate(greeting_parts): + if i > 0: + # Wait for bot TX rate limiter between multi-part messages + # This ensures we respect the bot's rate limiting configuration + await self.bot.bot_tx_rate_limiter.wait_for_tx() + # Additional delay to ensure proper spacing (use configured rate limit) + rate_limit = self.bot.config.getfloat('Bot', 'bot_tx_rate_limit_seconds', fallback=1.0) + # Use a conservative sleep time to avoid rate limiting + sleep_time = max(rate_limit + 0.5, 1.0) # At least 1 second, or rate_limit + 0.5 seconds + await asyncio.sleep(sleep_time) + + result = await self.send_response(message, greeting_part) + if not result: + success = False + + return success + except Exception as e: + self.logger.error(f"Error sending greeting: {e}") + return False + def should_execute(self, message: MeshMessage) -> bool: """ Check if greeter should execute for this message @@ -1013,7 +1285,7 @@ class GreeterCommand(BaseCommand): if not self.should_execute(message): return False - # Mark as greeted BEFORE getting mesh info (to prevent duplicate greetings) + # Mark as greeted BEFORE scheduling greeting (to prevent duplicate greetings) # This ensures we don't greet the same user twice even if there's a delay # mark_as_greeted uses atomic INSERT OR IGNORE to handle race conditions marked = self.mark_as_greeted(message.sender_id, message.channel) @@ -1061,42 +1333,71 @@ class GreeterCommand(BaseCommand): # If check fails, proceed anyway (better to greet than miss a greeting) self.logger.debug(f"Could not verify greeting timestamp (proceeding anyway): {e}") - # Format greeting parts (may be single or multi-part) - # Pass channel name for channel-specific greetings - greeting_parts = await self._format_greeting_parts(message.sender_id, message.channel) - - # Send greeting(s) - mode_str = "per-channel" if self.per_channel_greetings else "global" - self.logger.info(f"Greeting {message.sender_id} on channel {message.channel} ({mode_str} mode, {len(greeting_parts)} part(s))") - - # Log database verification - total_greeted = self.get_greeted_users_count() - self.logger.debug(f"Database verification: {total_greeted} total user(s) marked as greeted") - - # Send all greeting parts - success = True - for i, greeting_part in enumerate(greeting_parts): - if i > 0: - # Wait for bot TX rate limiter between multi-part messages - # This ensures we respect the bot's rate limiting configuration - await self.bot.bot_tx_rate_limiter.wait_for_tx() - # Additional delay to ensure proper spacing (use configured rate limit) - import asyncio - rate_limit = self.bot.config.getfloat('Bot', 'bot_tx_rate_limit_seconds', fallback=1.0) - # Use a conservative sleep time to avoid rate limiting - sleep_time = max(rate_limit + 0.5, 1.0) # At least 1 second, or rate_limit + 0.5 seconds - await asyncio.sleep(sleep_time) + # Check if dead air delay is enabled + if self.dead_air_delay_seconds > 0: + # Schedule delayed greeting + key = (message.sender_id, message.channel) - result = await self.send_response(message, greeting_part) - if not result: - success = False - - return success + # Cancel any existing pending greeting for this user/channel + if key in self.pending_greetings: + self._cancel_pending_greeting(message.sender_id, message.channel) + + # Schedule new delayed greeting + task = asyncio.create_task(self._send_delayed_greeting(message)) + self.pending_greetings[key] = task + self.logger.info(f"Scheduled delayed greeting for {message.sender_id} on {message.channel} (delay: {self.dead_air_delay_seconds}s)") + return True + else: + # Send greeting immediately (original behavior) + return await self._send_greeting(message) except Exception as e: self.logger.error(f"Error executing greeter command: {e}") return False + def check_message_for_human_greeting(self, message: MeshMessage): + """ + Check if an incoming message should cancel a pending greeting + Called from message handler when new messages arrive + + Args: + message: The incoming message to check + """ + if not self.defer_to_human_greeting or not self.dead_air_delay_seconds > 0: + return + + if message.is_dm or not message.channel: + return + + # Check all pending greetings for this channel + keys_to_cancel = [] + for (sender_id, channel), task in list(self.pending_greetings.items()): + if channel == message.channel and sender_id != message.sender_id: + # Check if this message mentions the pending user + if message.content and sender_id.lower() in message.content.lower(): + # Also check with Levenshtein distance if enabled + should_cancel = False + if self.levenshtein_distance > 0: + words = message.content.lower().split() + for word in words: + word = word.strip('.,!?;:()[]{}@') + distance = self._levenshtein_distance(sender_id.lower(), word) + if distance <= self.levenshtein_distance: + should_cancel = True + break + else: + should_cancel = True + + if should_cancel: + self.logger.info(f"Human greeting detected in real-time: {message.sender_id} mentioned {sender_id} - cancelling pending greeting") + keys_to_cancel.append((sender_id, channel)) + + # Cancel the pending greetings + for key in keys_to_cancel: + self._cancel_pending_greeting(key[0], key[1]) + # Mark as greeted so we don't greet them later + self.mark_as_greeted(key[0], key[1]) + def get_help_text(self) -> str: mode = "per-channel" if self.per_channel_greetings else "global (once total)" return f"Greeter automatically welcomes new users on public channels ({mode} mode). Configure in [Greeter_Command] section." diff --git a/modules/core.py b/modules/core.py index 6d4ef80..d6f35c1 100644 --- a/modules/core.py +++ b/modules/core.py @@ -35,6 +35,7 @@ from .db_manager import DBManager from .i18n import Translator from .solar_conditions import set_config from .web_viewer.integration import WebViewerIntegration +from .feed_manager import FeedManager class MeshCoreBot: @@ -125,6 +126,15 @@ class MeshCoreBot: self.scheduler = MessageScheduler(self) + # Initialize feed manager + self.logger.info("Initializing feed manager") + try: + self.feed_manager = FeedManager(self) + self.logger.info("Feed manager initialized successfully") + except Exception as e: + self.logger.warning(f"Failed to initialize feed manager: {e}") + self.feed_manager = None + # Initialize repeater manager self.logger.info("Initializing repeater manager") try: @@ -743,6 +753,10 @@ use_zulu_time = false # Setup scheduled messages self.scheduler.setup_scheduled_messages() + # Initialize feed manager (if enabled) + if self.feed_manager: + await self.feed_manager.initialize() + # Start scheduler thread self.scheduler.start() @@ -794,6 +808,10 @@ use_zulu_time = false self.connected = False + # Stop feed manager + if self.feed_manager: + await self.feed_manager.stop() + # Stop web viewer with proper shutdown sequence if self.web_viewer_integration: # Web viewer has simpler shutdown diff --git a/modules/db_manager.py b/modules/db_manager.py index 57db433..5e65ed3 100644 --- a/modules/db_manager.py +++ b/modules/db_manager.py @@ -59,12 +59,135 @@ class DBManager: ) ''') + # Create feed_subscriptions table for RSS/API feed subscriptions + cursor.execute(''' + CREATE TABLE IF NOT EXISTS feed_subscriptions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + feed_type TEXT NOT NULL, + feed_url TEXT NOT NULL, + channel_name TEXT NOT NULL, + feed_name TEXT, + last_item_id TEXT, + last_check_time TIMESTAMP, + check_interval_seconds INTEGER DEFAULT 300, + enabled BOOLEAN DEFAULT 1, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + api_config TEXT, + rss_config TEXT, + output_format TEXT, + message_send_interval_seconds REAL DEFAULT 2.0, + UNIQUE(feed_url, channel_name) + ) + ''') + + # Add new columns if they don't exist (for existing databases) + try: + cursor.execute('ALTER TABLE feed_subscriptions ADD COLUMN output_format TEXT') + except sqlite3.OperationalError: + pass # Column already exists + try: + cursor.execute('ALTER TABLE feed_subscriptions ADD COLUMN message_send_interval_seconds REAL DEFAULT 2.0') + except sqlite3.OperationalError: + pass # Column already exists + try: + cursor.execute('ALTER TABLE feed_subscriptions ADD COLUMN filter_config TEXT') + except sqlite3.OperationalError: + pass # Column already exists + try: + cursor.execute('ALTER TABLE feed_subscriptions ADD COLUMN sort_config TEXT') + except sqlite3.OperationalError: + pass # Column already exists + + # Create feed_activity table for tracking processed items + cursor.execute(''' + CREATE TABLE IF NOT EXISTS feed_activity ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + feed_id INTEGER NOT NULL, + item_id TEXT NOT NULL, + item_title TEXT, + processed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + message_sent BOOLEAN DEFAULT 1, + FOREIGN KEY (feed_id) REFERENCES feed_subscriptions(id) ON DELETE CASCADE + ) + ''') + + # Create feed_errors table for tracking feed errors + cursor.execute(''' + CREATE TABLE IF NOT EXISTS feed_errors ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + feed_id INTEGER NOT NULL, + error_type TEXT NOT NULL, + error_message TEXT, + occurred_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + resolved_at TIMESTAMP, + FOREIGN KEY (feed_id) REFERENCES feed_subscriptions(id) ON DELETE CASCADE + ) + ''') + + # Create channels table for storing channel information + cursor.execute(''' + CREATE TABLE IF NOT EXISTS channels ( + channel_idx INTEGER PRIMARY KEY, + channel_name TEXT NOT NULL, + channel_type TEXT, + channel_key_hex TEXT, + last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + UNIQUE(channel_idx) + ) + ''') + + # Create channel_operations queue table for web viewer -> bot communication + cursor.execute(''' + CREATE TABLE IF NOT EXISTS channel_operations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + operation_type TEXT NOT NULL, + channel_idx INTEGER, + channel_name TEXT, + channel_key_hex TEXT, + status TEXT DEFAULT 'pending', + error_message TEXT, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + processed_at TIMESTAMP, + result_data TEXT + ) + ''') + + # Create feed_message_queue table for queuing feed messages + cursor.execute(''' + CREATE TABLE IF NOT EXISTS feed_message_queue ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + feed_id INTEGER NOT NULL, + channel_name TEXT NOT NULL, + message TEXT NOT NULL, + item_id TEXT, + item_title TEXT, + priority INTEGER DEFAULT 0, + queued_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + sent_at TIMESTAMP, + FOREIGN KEY (feed_id) REFERENCES feed_subscriptions(id) ON DELETE CASCADE + ) + ''') + # Create indexes for better performance cursor.execute('CREATE INDEX IF NOT EXISTS idx_geocoding_query ON geocoding_cache(query)') cursor.execute('CREATE INDEX IF NOT EXISTS idx_geocoding_expires ON geocoding_cache(expires_at)') cursor.execute('CREATE INDEX IF NOT EXISTS idx_generic_key ON generic_cache(cache_key)') cursor.execute('CREATE INDEX IF NOT EXISTS idx_generic_type ON generic_cache(cache_type)') cursor.execute('CREATE INDEX IF NOT EXISTS idx_generic_expires ON generic_cache(expires_at)') + cursor.execute('CREATE INDEX IF NOT EXISTS idx_feed_subscriptions_enabled ON feed_subscriptions(enabled)') + cursor.execute('CREATE INDEX IF NOT EXISTS idx_feed_subscriptions_type ON feed_subscriptions(feed_type)') + cursor.execute('CREATE INDEX IF NOT EXISTS idx_feed_subscriptions_last_check ON feed_subscriptions(last_check_time)') + cursor.execute('CREATE INDEX IF NOT EXISTS idx_feed_activity_feed_id ON feed_activity(feed_id)') + cursor.execute('CREATE INDEX IF NOT EXISTS idx_feed_activity_processed_at ON feed_activity(processed_at)') + cursor.execute('CREATE INDEX IF NOT EXISTS idx_feed_errors_feed_id ON feed_errors(feed_id)') + cursor.execute('CREATE INDEX IF NOT EXISTS idx_feed_errors_occurred_at ON feed_errors(occurred_at)') + cursor.execute('CREATE INDEX IF NOT EXISTS idx_feed_errors_resolved ON feed_errors(resolved_at)') + cursor.execute('CREATE INDEX IF NOT EXISTS idx_channels_name ON channels(channel_name)') + cursor.execute('CREATE INDEX IF NOT EXISTS idx_channel_ops_status ON channel_operations(status, created_at)') + cursor.execute('CREATE INDEX IF NOT EXISTS idx_feed_message_queue_feed_id ON feed_message_queue(feed_id)') + cursor.execute('CREATE INDEX IF NOT EXISTS idx_feed_message_queue_sent ON feed_message_queue(sent_at)') + cursor.execute('CREATE INDEX IF NOT EXISTS idx_feed_message_queue_priority ON feed_message_queue(priority DESC, queued_at ASC)') conn.commit() self.logger.info("Database manager initialized successfully") diff --git a/modules/feed_manager.py b/modules/feed_manager.py new file mode 100644 index 0000000..fcf02ed --- /dev/null +++ b/modules/feed_manager.py @@ -0,0 +1,1252 @@ +#!/usr/bin/env python3 +""" +Feed Manager for RSS and API feed subscriptions +Handles polling feeds and sending updates to channels +""" + +import asyncio +import aiohttp +import json +import time +import hashlib +import html +import re +from datetime import datetime, timezone +from typing import Dict, List, Optional, Any, Tuple +import sqlite3 +import feedparser +from urllib.parse import urlparse + + +class FeedManager: + """Manages RSS and API feed subscriptions""" + + def __init__(self, bot): + self.bot = bot + self.logger = bot.logger + self.db_path = bot.db_manager.db_path + + # Configuration + self.enabled = bot.config.getboolean('Feed_Manager', 'feed_manager_enabled', fallback=False) + self.default_check_interval = bot.config.getint('Feed_Manager', 'default_check_interval_seconds', fallback=300) + self.max_items_per_check = bot.config.getint('Feed_Manager', 'max_items_per_check', fallback=10) + self.request_timeout = bot.config.getint('Feed_Manager', 'feed_request_timeout', fallback=30) + self.user_agent = bot.config.get('Feed_Manager', 'feed_user_agent', fallback='MeshCoreBot/1.0 FeedManager') + self.rate_limit_seconds = bot.config.getfloat('Feed_Manager', 'feed_rate_limit_seconds', fallback=5.0) + self.max_message_length = bot.config.getint('Feed_Manager', 'max_message_length', fallback=130) + self.default_output_format = bot.config.get('Feed_Manager', 'default_output_format', fallback='{emoji} {body|truncate:100} - {date}\n{link|truncate:50}') + self.default_send_interval = bot.config.getfloat('Feed_Manager', 'default_message_send_interval_seconds', fallback=2.0) + + # Rate limiting per domain + self._domain_last_request: Dict[str, float] = {} + + # HTTP session + self.session: Optional[aiohttp.ClientSession] = None + + # Semaphore to limit concurrent requests + self._request_semaphore = asyncio.Semaphore(5) + + self.logger.info("FeedManager initialized") + + async def initialize(self): + """Initialize the feed manager (create HTTP session)""" + if not self.enabled: + self.logger.info("FeedManager is disabled in config") + return + + # Don't create session here - create it lazily when needed + # This avoids issues with using sessions across different event loops + # The session will be created in the same event loop where it's used + self.logger.info("FeedManager initialized (session will be created on first use)") + + async def stop(self): + """Stop the feed manager (close HTTP session)""" + if self.session and not self.session.closed: + await self.session.close() + self.session = None + self.logger.info("FeedManager stopped") + + async def poll_all_feeds(self): + """Poll all enabled feeds that are due for checking""" + if not self.enabled: + return + + try: + # Get all enabled feeds + feeds = self._get_enabled_feeds() + + if not feeds: + return + + # Filter feeds that are due for checking + current_time = time.time() + feeds_to_check = [] + + for feed in feeds: + last_check = feed.get('last_check_time') + if last_check: + try: + # Parse timestamp - handle both ISO format and SQLite format + if isinstance(last_check, str): + # Try ISO format first (with timezone) + try: + last_check_dt = datetime.fromisoformat(last_check.replace('Z', '+00:00')) + except ValueError: + # Try SQLite format (YYYY-MM-DD HH:MM:SS) - treat as UTC + try: + last_check_dt = datetime.strptime(last_check, '%Y-%m-%d %H:%M:%S') + last_check_dt = last_check_dt.replace(tzinfo=timezone.utc) + except ValueError: + # Try with microseconds + try: + last_check_dt = datetime.strptime(last_check, '%Y-%m-%d %H:%M:%S.%f') + last_check_dt = last_check_dt.replace(tzinfo=timezone.utc) + except ValueError: + raise ValueError(f"Unknown timestamp format: {last_check}") + else: + last_check_dt = datetime.fromtimestamp(last_check, tz=timezone.utc) + + # Convert to timestamp + if last_check_dt.tzinfo: + last_check_ts = last_check_dt.timestamp() + else: + # Assume UTC if no timezone + last_check_ts = last_check_dt.replace(tzinfo=timezone.utc).timestamp() + except Exception as e: + self.logger.debug(f"Error parsing last_check_time for feed {feed['id']}: {e}") + last_check_ts = 0 + else: + last_check_ts = 0 + + interval = feed.get('check_interval_seconds', self.default_check_interval) + + if current_time - last_check_ts >= interval: + feeds_to_check.append(feed) + + if not feeds_to_check: + self.logger.debug("No feeds due for checking at this time") + return + + self.logger.info(f"Polling {len(feeds_to_check)} feed(s) that are due for checking") + + # Poll feeds in parallel (with semaphore limit) + tasks = [self.poll_feed(feed) for feed in feeds_to_check] + await asyncio.gather(*tasks, return_exceptions=True) + + except Exception as e: + self.logger.error(f"Error in poll_all_feeds: {e}") + + async def _ensure_session(self): + """Ensure HTTP session exists in the current event loop""" + if self.session is None or self.session.closed: + # Create session in the current event loop context + self.session = aiohttp.ClientSession( + headers={'User-Agent': self.user_agent} + ) + self.logger.debug("Created FeedManager HTTP session in current event loop") + + async def poll_feed(self, feed: Dict[str, Any]): + """Poll a single feed and process new items""" + # Ensure session exists in current event loop + await self._ensure_session() + + feed_id = feed['id'] + feed_type = feed['feed_type'] + feed_url = feed['feed_url'] + channel_name = feed['channel_name'] + + try: + self.logger.debug(f"Polling {feed_type} feed {feed_id}: {feed_url}") + + # Rate limit per domain + domain = urlparse(feed_url).netloc + await self._wait_for_rate_limit(domain) + + # Fetch feed data + if feed_type == 'rss': + new_items = await self.process_rss_feed(feed) + elif feed_type == 'api': + new_items = await self.process_api_feed(feed) + else: + self.logger.warning(f"Unknown feed type: {feed_type}") + return + + # Process new items + if new_items: + self.logger.info(f"Found {len(new_items)} new items for feed {feed_id}") + filtered_count = 0 + for item in new_items[:self.max_items_per_check]: + # Check if item passes filter conditions + if self._should_send_item(feed, item): + await self._send_feed_item(feed, item) + else: + filtered_count += 1 + self.logger.debug(f"Filtered out item: {item.get('title', 'Untitled')[:50]}") + + if filtered_count > 0: + self.logger.debug(f"Filtered out {filtered_count} items for feed {feed_id}") + else: + self.logger.debug(f"No new items found for feed {feed_id}") + + # Always update last check time, even if no new items + self._update_feed_last_check(feed_id) + + except Exception as e: + self.logger.error(f"Error polling feed {feed_id}: {e}") + self._record_feed_error(feed_id, 'network', str(e)) + + async def process_rss_feed(self, feed: Dict[str, Any]) -> List[Dict[str, Any]]: + """Process an RSS feed and return new items""" + feed_url = feed['feed_url'] + last_item_id = feed.get('last_item_id') + + try: + # Fetch RSS feed - use aiohttp's timeout directly + # Create timeout object in the current async context + timeout = aiohttp.ClientTimeout(total=self.request_timeout) + + async with self._request_semaphore: + try: + async with self.session.get(feed_url, timeout=timeout) as response: + if response.status != 200: + raise Exception(f"HTTP {response.status}") + content = await response.text() + except (asyncio.TimeoutError, aiohttp.ServerTimeoutError): + raise Exception(f"Request timeout after {self.request_timeout} seconds") + + # Parse RSS feed + parsed = feedparser.parse(content) + + if parsed.bozo: + self.logger.warning(f"RSS feed parsing warning: {parsed.bozo_exception}") + + # Extract items - collect ALL items first (don't break early if sorting is configured) + all_items = [] + for entry in parsed.entries: + # Get item ID (prefer guid, then link, then hash of title+link) + item_id = entry.get('id') or entry.get('guid') or entry.get('link') + if not item_id: + # Generate ID from title and link + item_id = hashlib.md5( + f"{entry.get('title', '')}{entry.get('link', '')}".encode() + ).hexdigest() + + # Parse published date + published = None + if hasattr(entry, 'published_parsed') and entry.published_parsed: + try: + published = datetime(*entry.published_parsed[:6], tzinfo=timezone.utc) + except Exception: + pass + + all_items.append({ + 'id': item_id, + 'title': entry.get('title', 'Untitled'), + 'link': entry.get('link', ''), + 'description': entry.get('description', ''), + 'published': published + }) + + # Apply sorting if configured (before filtering, so we can properly track the last item) + sort_config_str = feed.get('sort_config') + if sort_config_str: + try: + sort_config = json.loads(sort_config_str) if isinstance(sort_config_str, str) else sort_config_str + all_items = self._sort_items(all_items, sort_config) + except (json.JSONDecodeError, TypeError, Exception) as e: + self.logger.warning(f"Error applying sort config for feed {feed['id']}: {e}") + + # Reverse to get oldest first (if no sort config) + if not sort_config_str: + all_items.reverse() + + # Now filter out items that have already been processed + # Check against both last_item_id and the feed_activity table for robust deduplication + items = [] + processed_item_ids = set() + + # Get all previously processed item IDs from feed_activity table + if last_item_id: + processed_item_ids.add(last_item_id) + + # Query database for all processed item IDs for this feed + try: + with sqlite3.connect(self.db_path) as conn: + cursor = conn.cursor() + cursor.execute(''' + SELECT DISTINCT item_id FROM feed_activity + WHERE feed_id = ? + ''', (feed['id'],)) + for row in cursor.fetchall(): + processed_item_ids.add(row[0]) + except Exception as e: + self.logger.debug(f"Error querying processed items for feed {feed['id']}: {e}") + + # Filter out already processed items + for item in all_items: + if item['id'] not in processed_item_ids: + items.append(item) + else: + self.logger.debug(f"Skipping already processed item {item['id']} for feed {feed['id']}") + + # Update last_item_id if we have new items (use the last item from the sorted list) + if items: + # Use the last item from the original sorted list (all_items), not the filtered list + # This ensures we track the most recent item even if it was already processed + self._update_feed_last_item_id(feed['id'], all_items[-1]['id']) + + return items + + except Exception as e: + self.logger.error(f"Error processing RSS feed: {e}") + raise + + async def process_api_feed(self, feed: Dict[str, Any]) -> List[Dict[str, Any]]: + """Process an API feed and return new items""" + feed_url = feed['feed_url'] + api_config_str = feed.get('api_config', '{}') + last_item_id = feed.get('last_item_id') + + try: + # Parse API config + api_config = json.loads(api_config_str) if api_config_str else {} + + method = api_config.get('method', 'GET').upper() + headers = api_config.get('headers', {}) + params = api_config.get('params', {}) + body = api_config.get('body') + parser_config = api_config.get('response_parser', {}) + + # Make HTTP request - use aiohttp's timeout directly + # Create timeout object in the current async context + timeout = aiohttp.ClientTimeout(total=self.request_timeout) + + async with self._request_semaphore: + try: + if method == 'POST': + async with self.session.post(feed_url, headers=headers, params=params, json=body, timeout=timeout) as response: + if response.status != 200: + raise Exception(f"HTTP {response.status}") + data = await response.json() + else: + async with self.session.get(feed_url, headers=headers, params=params, timeout=timeout) as response: + if response.status != 200: + raise Exception(f"HTTP {response.status}") + data = await response.json() + except (asyncio.TimeoutError, aiohttp.ServerTimeoutError): + raise Exception(f"Request timeout after {self.request_timeout} seconds") + + # Extract items using parser config + items_path = parser_config.get('items_path', '') + if items_path: + # Navigate JSON path + parts = items_path.split('.') + items_data = data + for part in parts: + items_data = items_data.get(part, []) + else: + # Assume data is a list + items_data = data if isinstance(data, list) else [data] + + # Extract items + id_field = parser_config.get('id_field', 'id') + title_field = parser_config.get('title_field', 'title') + description_field = parser_config.get('description_field', 'description') # New: allow custom description field + timestamp_field = parser_config.get('timestamp_field', 'created_at') + + # Collect ALL items first (don't break early, as sorting may reorder them) + all_items = [] + for item_data in items_data: + item_id = str(self._get_nested_value(item_data, id_field, '')) + if not item_id: + continue + + # Parse timestamp if available - support nested paths + published = None + if timestamp_field: + ts_value = self._get_nested_value(item_data, timestamp_field) + if ts_value: + try: + if isinstance(ts_value, (int, float)): + published = datetime.fromtimestamp(ts_value, tz=timezone.utc) + elif isinstance(ts_value, str): + # Try Microsoft date format first + if ts_value.startswith('/Date('): + published = self._parse_microsoft_date(ts_value) + else: + # Try ISO format + try: + published = datetime.fromisoformat(ts_value.replace('Z', '+00:00')) + except ValueError: + # Try common formats + for fmt in ['%Y-%m-%dT%H:%M:%S', '%Y-%m-%d %H:%M:%S', '%Y-%m-%d']: + try: + published = datetime.strptime(ts_value, fmt) + if published.tzinfo is None: + published = published.replace(tzinfo=timezone.utc) + break + except ValueError: + continue + except Exception: + pass + + # Get description - support nested paths + description = '' + if description_field: + desc_value = self._get_nested_value(item_data, description_field) + if desc_value: + description = str(desc_value) + + all_items.append({ + 'id': item_id, + 'title': self._get_nested_value(item_data, title_field, 'Untitled'), + 'link': item_data.get('link', ''), + 'description': description, + 'published': published, + 'raw': item_data # Store full raw response for field access + }) + + # Apply sorting if configured (before filtering, so we can properly track the last item) + sort_config_str = feed.get('sort_config') + if sort_config_str: + try: + sort_config = json.loads(sort_config_str) if isinstance(sort_config_str, str) else sort_config_str + all_items = self._sort_items(all_items, sort_config) + except (json.JSONDecodeError, TypeError, Exception) as e: + self.logger.warning(f"Error applying sort config for feed {feed['id']}: {e}") + + # Reverse to get oldest first (if no sort config) + if not sort_config_str: + all_items.reverse() + + # Now filter out items that have already been processed + # Check against both last_item_id and the feed_activity table for robust deduplication + items = [] + processed_item_ids = set() + + # Get all previously processed item IDs from feed_activity table + if last_item_id: + processed_item_ids.add(last_item_id) + + # Query database for all processed item IDs for this feed + try: + with sqlite3.connect(self.db_path) as conn: + cursor = conn.cursor() + cursor.execute(''' + SELECT DISTINCT item_id FROM feed_activity + WHERE feed_id = ? + ''', (feed['id'],)) + for row in cursor.fetchall(): + processed_item_ids.add(row[0]) + except Exception as e: + self.logger.debug(f"Error querying processed items for feed {feed['id']}: {e}") + + # Filter out already processed items + for item in all_items: + if item['id'] not in processed_item_ids: + items.append(item) + else: + self.logger.debug(f"Skipping already processed item {item['id']} for feed {feed['id']}") + + # Update last_item_id if we have new items (use the last item from the sorted list) + if items: + # Use the last item from the original sorted list (all_items), not the filtered list + # This ensures we track the most recent item even if it was already processed + self._update_feed_last_item_id(feed['id'], all_items[-1]['id']) + + return items + + except Exception as e: + self.logger.error(f"Error processing API feed: {e}") + raise + + def _format_timestamp(self, published: Optional[datetime]) -> str: + """Format a timestamp as a relative time string""" + if not published: + return "" + + try: + if published.tzinfo: + now = datetime.now(timezone.utc) + else: + now = datetime.now() + + diff = now - published + minutes = int(diff.total_seconds() / 60) + + if minutes < 1: + return "now" + elif minutes < 60: + return f"{minutes}m ago" + elif minutes < 1440: + hours = minutes // 60 + mins = minutes % 60 + return f"{hours}h {mins}m ago" + else: + days = minutes // 1440 + return f"{days}d ago" + except Exception: + return "" + + def _apply_shortening(self, text: str, function: str) -> str: + """Apply a shortening, parsing, or conditional function to text + + Supported functions: + - truncate:N - truncate to N characters + - word_wrap:N - wrap at N characters, breaking at word boundaries + - first_words:N - take first N words + - regex:pattern - extract using regex pattern (uses first capture group, or whole match) + - regex:pattern:group - extract specific capture group (0 = whole match, 1 = first group, etc.) + - if_regex:pattern:then:else - if pattern matches, return "then", else return "else" + """ + if not text: + return "" + + if function.startswith('truncate:'): + try: + max_len = int(function.split(':', 1)[1]) + if len(text) <= max_len: + return text + return text[:max_len] + "..." + except (ValueError, IndexError): + return text + + elif function.startswith('word_wrap:'): + try: + max_len = int(function.split(':', 1)[1]) + if len(text) <= max_len: + return text + # Find last space before max_len + truncated = text[:max_len] + last_space = truncated.rfind(' ') + if last_space > max_len * 0.7: # Only use word boundary if it's not too short + return truncated[:last_space] + "..." + return truncated + "..." + except (ValueError, IndexError): + return text + + elif function.startswith('first_words:'): + try: + num_words = int(function.split(':', 1)[1]) + words = text.split() + if len(words) <= num_words: + return text + return ' '.join(words[:num_words]) + "..." + except (ValueError, IndexError): + return text + + elif function.startswith('regex:'): + try: + # Parse regex pattern and optional group number + # Format: regex:pattern:group or regex:pattern + # Need to handle patterns that contain colons, so split from the right + remaining = function[6:] # Skip 'regex:' prefix + + # Try to find the last colon that's followed by a number (the group number) + # Look for pattern like :N at the end + last_colon_idx = remaining.rfind(':') + pattern = remaining + group_num = None + + if last_colon_idx > 0: + # Check if what's after the last colon is a number + potential_group = remaining[last_colon_idx + 1:] + if potential_group.isdigit(): + pattern = remaining[:last_colon_idx] + group_num = int(potential_group) + + if not pattern: + return text + + # Apply regex + match = re.search(pattern, text, re.IGNORECASE | re.DOTALL) + if match: + if group_num is not None: + # Use specified group (0 = whole match, 1 = first group, etc.) + if 0 <= group_num <= len(match.groups()): + return match.group(group_num) if group_num > 0 else match.group(0) + else: + # Use first capture group if available, otherwise whole match + if match.groups(): + return match.group(1) + else: + return match.group(0) + return "" # No match found + except (ValueError, IndexError, re.error) as e: + self.logger.debug(f"Error applying regex function: {e}") + return text + + elif function.startswith('if_regex:'): + try: + # Parse: if_regex:pattern:then:else + # Split by ':' but need to handle regex patterns that contain ':' + # Use a smarter split that respects the structure + parts = function[9:].split(':', 2) # Skip 'if_regex:' prefix, split into [pattern, then, else] + if len(parts) < 3: + return text + + pattern = parts[0] + then_value = parts[1] + else_value = parts[2] + + if not pattern: + return text + + # Check if pattern matches + match = re.search(pattern, text, re.IGNORECASE | re.DOTALL) + if match: + return then_value + else: + return else_value + except (ValueError, IndexError, re.error) as e: + self.logger.debug(f"Error applying if_regex function: {e}") + return text + + elif function.startswith('switch:'): + try: + # Parse: switch:value1:result1:value2:result2:...:default + # Example: switch:highest:🔴:high:🟠:medium:🟡:low:âšĒ:âšĒ + # This checks if text exactly matches value1, returns result1, etc., or default + parts = function[7:].split(':') # Skip 'switch:' prefix + if len(parts) < 2: + return text + + # Pairs of value:result, last one is default + text_lower = text.lower().strip() + for i in range(0, len(parts) - 1, 2): + if i + 1 < len(parts): + value = parts[i].lower() + result = parts[i + 1] + if text_lower == value: + return result + + # Return last part as default if no match + return parts[-1] if parts else text + except (ValueError, IndexError) as e: + self.logger.debug(f"Error applying switch function: {e}") + return text + + elif function.startswith('regex_cond:'): + try: + # Parse: regex_cond:extract_pattern:check_pattern:then:group + # This extracts text using extract_pattern, then checks if it matches check_pattern + # If check_pattern matches, return "then", else return the extracted text + # Example: regex_cond:Northbound\s*\n([^\n]+):No restrictions:👍:1 + # This extracts text after "Northbound\n" up to next newline, checks if it's "No restrictions", + # if yes returns "👍", else returns the extracted text + parts = function[11:].split(':', 3) # Skip 'regex_cond:' prefix + if len(parts) < 4: + return text + + extract_pattern = parts[0] + check_pattern = parts[1] + then_value = parts[2] + else_group = int(parts[3]) if parts[3].isdigit() else 1 + + if not extract_pattern: + return text + + # Extract using extract_pattern + match = re.search(extract_pattern, text, re.IGNORECASE | re.DOTALL) + if match: + # Get the captured group + if match.groups(): + extracted = match.group(else_group) if else_group <= len(match.groups()) else match.group(1) + # Strip whitespace from extracted text + extracted = extracted.strip() + else: + extracted = match.group(0).strip() + + # Check if extracted text matches check_pattern (exact match or contains) + if check_pattern: + # Try exact match first, then substring match + if extracted.lower() == check_pattern.lower() or re.search(check_pattern, extracted, re.IGNORECASE): + return then_value + + return extracted + return "" # No match found + except (ValueError, IndexError, re.error) as e: + self.logger.debug(f"Error applying regex_cond function: {e}") + return text + + return text + + def _get_nested_value(self, data: Any, path: str, default: Any = '') -> Any: + """Get a nested value from a dict/list using dot notation (e.g., 'raw.Priority' or 'raw.StartRoadwayLocation.RoadName')""" + if not path or not data: + return default + + parts = path.split('.') + value = data + + for part in parts: + if isinstance(value, dict): + value = value.get(part) + elif isinstance(value, list): + try: + idx = int(part) + if 0 <= idx < len(value): + value = value[idx] + else: + return default + except (ValueError, TypeError): + return default + else: + return default + + if value is None: + return default + + return value if value is not None else default + + def _parse_microsoft_date(self, date_str: str) -> Optional[datetime]: + """Parse Microsoft JSON date format: /Date(timestamp-offset)/""" + if not date_str or not isinstance(date_str, str): + return None + + # Match /Date(timestamp-offset)/ format + match = re.match(r'/Date\((\d+)([+-]\d+)?\)/', date_str) + if match: + timestamp_ms = int(match.group(1)) + offset_str = match.group(2) if match.group(2) else '+0000' + + # Convert milliseconds to seconds + timestamp = timestamp_ms / 1000.0 + + # Parse offset (format: +0800 or -0800) + try: + offset_hours = int(offset_str[:3]) + offset_mins = int(offset_str[3:5]) + offset_seconds = (offset_hours * 3600) + (offset_mins * 60) + if offset_str[0] == '-': + offset_seconds = -offset_seconds + + # Create timezone-aware datetime + tz = timezone.utc + if offset_seconds != 0: + from datetime import timedelta + tz = timezone(timedelta(seconds=offset_seconds)) + + return datetime.fromtimestamp(timestamp, tz=tz) + except (ValueError, IndexError): + # Fallback to UTC if offset parsing fails + return datetime.fromtimestamp(timestamp, tz=timezone.utc) + + return None + + def _sort_items(self, items: List[Dict[str, Any]], sort_config: dict) -> List[Dict[str, Any]]: + """Sort items based on sort configuration + + Sort config format: + { + "field": "raw.LastUpdatedTime", # Field path to sort by + "order": "desc" # "asc" or "desc" + } + """ + if not sort_config or not items: + return items + + field_path = sort_config.get('field') + order = sort_config.get('order', 'desc').lower() + + if not field_path: + return items + + def get_sort_value(item): + """Get the sort value for an item""" + # Try raw data first + raw_data = item.get('raw', {}) + value = self._get_nested_value(raw_data, field_path, '') + + if not value and field_path.startswith('raw.'): + value = self._get_nested_value(raw_data, field_path[4:], '') + + if not value: + value = self._get_nested_value(item, field_path, '') + + # Handle Microsoft date format + if isinstance(value, str) and value.startswith('/Date('): + dt = self._parse_microsoft_date(value) + if dt: + return dt.timestamp() + + # Handle datetime objects + if isinstance(value, datetime): + return value.timestamp() + + # Handle numeric values + if isinstance(value, (int, float)): + return float(value) + + # Handle string timestamps + if isinstance(value, str): + # Try to parse as ISO format + try: + dt = datetime.fromisoformat(value.replace('Z', '+00:00')) + return dt.timestamp() + except ValueError: + pass + + # Try common date formats + for fmt in ['%Y-%m-%dT%H:%M:%S', '%Y-%m-%d %H:%M:%S', '%Y-%m-%d']: + try: + dt = datetime.strptime(value, fmt) + return dt.timestamp() + except ValueError: + continue + + # For strings, use lexicographic comparison + return str(value) + + # Sort items + try: + sorted_items = sorted(items, key=get_sort_value, reverse=(order == 'desc')) + return sorted_items + except Exception as e: + self.logger.warning(f"Error sorting items: {e}") + return items + + def format_message(self, item: Dict[str, Any], feed: Dict[str, Any]) -> str: + """Format a feed item as a message for the mesh using configurable format with placeholders + + Supported placeholders: + - {title} - item title + - {body} - item description/body + - {date} - relative time (e.g., "5m ago") + - {link} - item link URL + - {emoji} - emoji based on feed type + - {raw.field} - access any field from raw API response (e.g., {raw.Priority}, {raw.StartRoadwayLocation.RoadName}) + + Supported shortening functions: + - {field|truncate:N} - truncate to N characters + - {field|word_wrap:N} - wrap at N characters + - {field|first_words:N} - take first N words + - {field|regex:pattern} - extract using regex (first group or whole match) + - {field|regex:pattern:group} - extract specific capture group + - {field|if_regex:pattern:then:else} - if pattern matches, return "then", else "else" + - {field|switch:value1:result1:value2:result2:...:default} - exact match switch (e.g., switch:highest:🔴:high:🟠:medium:🟡:âšĒ) + - {field|regex_cond:extract_pattern:check_pattern:then:group} - extract text, check if it matches check_pattern, return "then" if match else extracted text + """ + + # Get format string from feed config or use default + format_str = feed.get('output_format') or self.default_output_format + + # Extract field values + title = item.get('title', 'Untitled') + body = item.get('description', '') or item.get('body', '') + # Clean HTML from body if present + if body: + import html + body = html.unescape(body) + # Convert line break tags to newlines before stripping other HTML + # Handle
,
,
,
, etc. + body = re.sub(r'', '\n', body, flags=re.IGNORECASE) + # Convert paragraph tags to newlines (with spacing) + body = re.sub(r'

', '\n\n', body, flags=re.IGNORECASE) + body = re.sub(r']*>', '', body, flags=re.IGNORECASE) + # Remove remaining HTML tags + body = re.sub(r'<[^>]+>', '', body) + # Clean up whitespace (preserve intentional line breaks) + # Replace multiple newlines with double newline, then normalize spaces within lines + body = re.sub(r'\n\s*\n\s*\n+', '\n\n', body) # Multiple newlines -> double newline + lines = body.split('\n') + body = '\n'.join(' '.join(line.split()) for line in lines) # Normalize spaces per line + body = body.strip() + + link = item.get('link', '') + published = item.get('published') + date_str = self._format_timestamp(published) + + # Choose emoji based on feed type or content + emoji = "đŸ“ĸ" + feed_name = feed.get('feed_name', '').lower() + if 'emergency' in feed_name or 'alert' in feed_name: + emoji = "🚨" + elif 'warning' in feed_name: + emoji = "⚠ī¸" + elif 'info' in feed_name or 'news' in feed_name: + emoji = "ℹī¸" + + # Build replacement dictionary + replacements = { + 'title': title, + 'body': body, + 'date': date_str, + 'link': link, + 'emoji': emoji + } + + # Get raw API data if available + raw_data = item.get('raw', {}) + + # Process format string with placeholders and functions + # Pattern: {field|function} or {field} or {raw.field.path} + def replace_placeholder(match): + full_match = match.group(0) + content = match.group(1) # Content inside {} + + if '|' in content: + field_name, function = content.split('|', 1) + field_name = field_name.strip() + function = function.strip() + + # Check if it's a raw field access + if field_name.startswith('raw.'): + value = str(self._get_nested_value(raw_data, field_name[4:], '')) + else: + value = replacements.get(field_name, '') + + return self._apply_shortening(value, function) + else: + field_name = content.strip() + + # Check if it's a raw field access + if field_name.startswith('raw.'): + value = self._get_nested_value(raw_data, field_name[4:], '') + # Convert to string, handling None and complex types + if value is None: + return '' + elif isinstance(value, (dict, list)): + # For complex types, convert to JSON string + try: + return json.dumps(value) + except Exception: + return str(value) + else: + return str(value) + else: + return replacements.get(field_name, '') + + # Replace all placeholders + message = re.sub(r'\{([^}]+)\}', replace_placeholder, format_str) + + # Final truncation if message is too long + if len(message) > self.max_message_length: + # Try to preserve structure by truncating at newline if possible + lines = message.split('\n') + if len(lines) > 1: + # Truncate last line + total_length = sum(len(line) + 1 for line in lines[:-1]) # +1 for newline + remaining = self.max_message_length - total_length - 3 # -3 for "..." + if remaining > 20: + lines[-1] = lines[-1][:remaining] + "..." + message = '\n'.join(lines) + else: + # Just truncate everything + message = message[:self.max_message_length - 3] + "..." + else: + message = message[:self.max_message_length - 3] + "..." + + return message + + def _queue_feed_message(self, feed: Dict[str, Any], item: Dict[str, Any], message: str): + """Queue a feed message for later sending""" + try: + with sqlite3.connect(self.db_path) as conn: + cursor = conn.cursor() + cursor.execute(''' + INSERT INTO feed_message_queue + (feed_id, channel_name, message, item_id, item_title, priority) + VALUES (?, ?, ?, ?, ?, 0) + ''', ( + feed['id'], + feed['channel_name'], + message, + item.get('id', ''), + item.get('title', '')[:200] # Limit title length + )) + conn.commit() + self.logger.debug(f"Queued feed message for {feed['channel_name']}: {item.get('title', '')[:50]}") + except Exception as e: + self.logger.error(f"Error queuing feed message: {e}") + self._record_feed_error(feed['id'], 'queue', str(e)) + + def _should_send_item(self, feed: Dict[str, Any], item: Dict[str, Any]) -> bool: + """Check if an item should be sent based on filter configuration + + Filter config format: + { + "conditions": [ + {"field": "Priority", "operator": "in", "values": ["highest", "high"]}, + {"field": "EventStatus", "operator": "equals", "value": "open"}, + {"field": "EventCategory", "operator": "not_equals", "value": "Maintenance"}, + {"field": "raw.Priority", "operator": "matches", "pattern": "^(highest|high)$"} + ], + "logic": "AND" # or "OR" + } + + Supported operators: + - equals: exact match + - not_equals: not exact match + - in: value is in list + - not_in: value is not in list + - matches: regex match + - not_matches: regex doesn't match + - contains: substring match + - not_contains: substring doesn't match + """ + filter_config_str = feed.get('filter_config') + if not filter_config_str: + # No filter configured, send all items + return True + + try: + filter_config = json.loads(filter_config_str) if isinstance(filter_config_str, str) else filter_config_str + except (json.JSONDecodeError, TypeError): + self.logger.warning(f"Invalid filter_config for feed {feed['id']}, sending all items") + return True + + conditions = filter_config.get('conditions', []) + if not conditions: + # Empty conditions, send all items + return True + + logic = filter_config.get('logic', 'AND').upper() + + # Get raw data for field access + raw_data = item.get('raw', {}) + + # Evaluate each condition + results = [] + for condition in conditions: + field_path = condition.get('field') + operator = condition.get('operator', 'equals') + + if not field_path: + # Invalid condition, skip it + continue + + # Get field value using nested access + field_value = self._get_nested_value(raw_data, field_path, '') + if not field_value and field_path.startswith('raw.'): + # Try without 'raw.' prefix + field_value = self._get_nested_value(raw_data, field_path[4:], '') + + # If still not found, try top-level item fields + if not field_value: + field_value = self._get_nested_value(item, field_path, '') + + # Convert to string for comparison + field_value_str = str(field_value).lower() if field_value is not None else '' + + # Evaluate condition + result = False + if operator == 'equals': + compare_value = str(condition.get('value', '')).lower() + result = field_value_str == compare_value + elif operator == 'not_equals': + compare_value = str(condition.get('value', '')).lower() + result = field_value_str != compare_value + elif operator == 'in': + values = [str(v).lower() for v in condition.get('values', [])] + result = field_value_str in values + elif operator == 'not_in': + values = [str(v).lower() for v in condition.get('values', [])] + result = field_value_str not in values + elif operator == 'matches': + pattern = condition.get('pattern', '') + if pattern: + try: + result = bool(re.search(pattern, str(field_value), re.IGNORECASE)) + except re.error: + result = False + elif operator == 'not_matches': + pattern = condition.get('pattern', '') + if pattern: + try: + result = not bool(re.search(pattern, str(field_value), re.IGNORECASE)) + except re.error: + result = True + elif operator == 'contains': + compare_value = str(condition.get('value', '')).lower() + result = compare_value in field_value_str + elif operator == 'not_contains': + compare_value = str(condition.get('value', '')).lower() + result = compare_value not in field_value_str + else: + self.logger.warning(f"Unknown filter operator: {operator}") + result = True # Default to allowing if operator is unknown + + results.append(result) + + # Apply logic (AND or OR) + if logic == 'OR': + return any(results) + else: # AND (default) + return all(results) + + async def _send_feed_item(self, feed: Dict[str, Any], item: Dict[str, Any]): + """Queue a feed item message instead of sending immediately""" + try: + message = self.format_message(item, feed) + # Queue the message instead of sending immediately + self._queue_feed_message(feed, item, message) + except Exception as e: + self.logger.error(f"Error processing feed item: {e}") + self._record_feed_error(feed['id'], 'other', str(e)) + + async def _wait_for_rate_limit(self, domain: str): + """Wait if needed to respect rate limits""" + if domain in self._domain_last_request: + last_request = self._domain_last_request[domain] + elapsed = time.time() - last_request + if elapsed < self.rate_limit_seconds: + wait_time = self.rate_limit_seconds - elapsed + await asyncio.sleep(wait_time) + + self._domain_last_request[domain] = time.time() + + def _get_enabled_feeds(self) -> List[Dict[str, Any]]: + """Get all enabled feed subscriptions from database""" + try: + with sqlite3.connect(self.db_path) as conn: + conn.row_factory = sqlite3.Row + cursor = conn.cursor() + cursor.execute(''' + SELECT * FROM feed_subscriptions + WHERE enabled = 1 + ORDER BY last_check_time ASC NULLS FIRST + ''') + rows = cursor.fetchall() + return [dict(row) for row in rows] + except Exception as e: + self.logger.error(f"Error getting enabled feeds: {e}") + return [] + + def _update_feed_last_check(self, feed_id: int): + """Update the last check time for a feed""" + try: + from datetime import datetime, timezone + # Use Python's datetime to ensure proper timezone handling + # Store in ISO format with timezone for JavaScript compatibility + now = datetime.now(timezone.utc) + now_str = now.isoformat() # ISO format: 2025-12-05T12:34:56.789+00:00 + + with sqlite3.connect(self.db_path) as conn: + cursor = conn.cursor() + cursor.execute(''' + UPDATE feed_subscriptions + SET last_check_time = ?, + updated_at = ? + WHERE id = ? + ''', (now_str, now_str, feed_id)) + conn.commit() + self.logger.debug(f"Updated last_check_time for feed {feed_id} to {now_str}") + except Exception as e: + self.logger.error(f"Error updating feed last check: {e}") + + def _update_feed_last_item_id(self, feed_id: int, item_id: str): + """Update the last processed item ID for a feed""" + try: + with sqlite3.connect(self.db_path) as conn: + cursor = conn.cursor() + cursor.execute(''' + UPDATE feed_subscriptions + SET last_item_id = ?, + updated_at = CURRENT_TIMESTAMP + WHERE id = ? + ''', (item_id, feed_id)) + conn.commit() + except Exception as e: + self.logger.error(f"Error updating feed last item ID: {e}") + + def _record_feed_activity(self, feed_id: int, item_id: str, item_title: str): + """Record that a feed item was processed""" + try: + with sqlite3.connect(self.db_path) as conn: + cursor = conn.cursor() + cursor.execute(''' + INSERT INTO feed_activity (feed_id, item_id, item_title, message_sent) + VALUES (?, ?, ?, 1) + ''', (feed_id, item_id, item_title[:200])) # Limit title length + conn.commit() + except Exception as e: + self.logger.error(f"Error recording feed activity: {e}") + + def _record_feed_error(self, feed_id: int, error_type: str, error_message: str): + """Record a feed error""" + try: + with sqlite3.connect(self.db_path) as conn: + cursor = conn.cursor() + cursor.execute(''' + INSERT INTO feed_errors (feed_id, error_type, error_message) + VALUES (?, ?, ?) + ''', (feed_id, error_type, error_message[:500])) # Limit message length + conn.commit() + except Exception as e: + self.logger.error(f"Error recording feed error: {e}") + + async def process_message_queue(self): + """Process queued feed messages and send them at configured intervals""" + try: + # Get all unsent messages, ordered by priority and queue time + with sqlite3.connect(self.db_path) as conn: + conn.row_factory = sqlite3.Row + cursor = conn.cursor() + cursor.execute(''' + SELECT q.id, q.feed_id, q.channel_name, q.message, q.item_id, q.item_title, + f.message_send_interval_seconds + FROM feed_message_queue q + JOIN feed_subscriptions f ON q.feed_id = f.id + WHERE q.sent_at IS NULL + ORDER BY q.priority DESC, q.queued_at ASC + LIMIT 100 + ''') + messages = cursor.fetchall() + + if not messages: + return + + # Group messages by feed to respect per-feed send intervals + feed_last_send: Dict[int, float] = {} + + for msg in messages: + feed_id = msg['feed_id'] + channel_name = msg['channel_name'] + message_text = msg['message'] + queue_id = msg['id'] + item_id = msg['item_id'] + item_title = msg['item_title'] + + # Get send interval for this feed (default if not set) + send_interval = msg['message_send_interval_seconds'] or self.default_send_interval + + # Check if we need to wait before sending this feed's message + if feed_id in feed_last_send: + elapsed = time.time() - feed_last_send[feed_id] + if elapsed < send_interval: + wait_time = send_interval - elapsed + await asyncio.sleep(wait_time) + + # Send the message + try: + success = await self.bot.command_manager.send_channel_message(channel_name, message_text) + + if success: + # Mark as sent + with sqlite3.connect(self.db_path) as conn: + cursor = conn.cursor() + cursor.execute(''' + UPDATE feed_message_queue + SET sent_at = CURRENT_TIMESTAMP + WHERE id = ? + ''', (queue_id,)) + conn.commit() + + # Record activity + self._record_feed_activity(feed_id, item_id, item_title) + self.logger.debug(f"Sent queued feed message to {channel_name}: {item_title[:50]}") + feed_last_send[feed_id] = time.time() + else: + self.logger.warning(f"Failed to send queued feed message to channel {channel_name}") + self._record_feed_error(feed_id, 'channel', f"Failed to send to channel {channel_name}") + # Don't mark as sent, will retry later + + except Exception as e: + self.logger.error(f"Error sending queued feed message: {e}") + self._record_feed_error(feed_id, 'other', str(e)) + # Don't mark as sent, will retry later + + except Exception as e: + self.logger.error(f"Error processing message queue: {e}") + diff --git a/modules/message_handler.py b/modules/message_handler.py index 2aed5ab..c6c302c 100644 --- a/modules/message_handler.py +++ b/modules/message_handler.py @@ -1562,9 +1562,29 @@ class MessageHandler: # This allows greeter to work on its own configured channels even if not in monitor_channels if 'greeter' in self.bot.command_manager.commands: greeter_command = self.bot.command_manager.commands['greeter'] + # First, check if this message should cancel a pending greeting (human greeting detection) + if greeter_command: + greeter_command.check_message_for_human_greeting(message) + # Then check if we should greet this user if greeter_command and greeter_command.should_execute(message): try: - await greeter_command.execute(message) + success = await greeter_command.execute(message) + + # Small delay to ensure send_response has completed + await asyncio.sleep(0.1) + + # Determine if a response was sent + response_sent = False + if hasattr(greeter_command, 'last_response') and greeter_command.last_response: + response_sent = True + elif hasattr(self.bot.command_manager, '_last_response') and self.bot.command_manager._last_response: + response_sent = True + + # Record command execution in stats database + if 'stats' in self.bot.command_manager.commands: + stats_command = self.bot.command_manager.commands['stats'] + if stats_command: + stats_command.record_command(message, 'greeter', response_sent) except Exception as e: self.logger.error(f"Error executing greeter command: {e}") @@ -1593,15 +1613,6 @@ class MessageHandler: else: self.logger.info(f"Keyword '{keyword}' matched, responding") - # Record command execution in stats database - if 'stats' in self.bot.command_manager.commands: - stats_command = self.bot.command_manager.commands['stats'] - if stats_command: - stats_command.record_command(message, keyword, response is not None) - - # Note: Command data capture is handled in command_manager.py after execution - # to avoid duplicate messages to web viewer - # Track if this is a help response if keyword == 'help': help_response_sent = True @@ -1611,9 +1622,21 @@ class MessageHandler: plugin_command_with_response_matched = True # Skip commands that handle their own responses (response is None) + # These will be recorded when they execute via execute_commands if response is None: continue + # Record command execution in stats database for keyword-matched commands with responses + # Commands without responses (response is None) are recorded in execute_commands to avoid double-counting + if 'stats' in self.bot.command_manager.commands: + stats_command = self.bot.command_manager.commands['stats'] + if stats_command: + # response is not None here, so we know a response will be sent + stats_command.record_command(message, keyword, True) + + # Note: Command data capture is handled in command_manager.py after execution + # to avoid duplicate messages to web viewer + # Send response if message.is_dm: await self.bot.command_manager.send_dm(message.sender_id, response) diff --git a/modules/scheduler.py b/modules/scheduler.py index f066762..e452a87 100644 --- a/modules/scheduler.py +++ b/modules/scheduler.py @@ -9,6 +9,8 @@ import threading import schedule import datetime import pytz +import sqlite3 +import json from typing import Dict, Tuple @@ -121,6 +123,7 @@ class MessageScheduler: """Run the scheduler in a separate thread""" self.logger.info("Scheduler thread started") last_log_time = 0 + last_feed_poll_time = 0 while self.bot.connected: current_time = self.get_current_time() @@ -138,6 +141,83 @@ class MessageScheduler: # Check for interval-based advertising self.check_interval_advertising() + # Poll feeds every minute (but feeds themselves control their check intervals) + if time.time() - last_feed_poll_time >= 60: # Every 60 seconds + if (hasattr(self.bot, 'feed_manager') and self.bot.feed_manager and + hasattr(self.bot.feed_manager, 'enabled') and self.bot.feed_manager.enabled and + hasattr(self.bot, 'connected') and self.bot.connected): + # Run feed polling in async context + import asyncio + try: + loop = asyncio.get_event_loop() + except RuntimeError: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + # Schedule feed polling + try: + loop.run_until_complete(self.bot.feed_manager.poll_all_feeds()) + self.logger.debug("Feed polling cycle completed") + except Exception as e: + self.logger.error(f"Error in feed polling cycle: {e}") + last_feed_poll_time = time.time() + + # Periodically refresh channels from device (every hour) to prevent stale data + if not hasattr(self, 'last_channel_refresh_time'): + # Initialize to current time so it doesn't run immediately on startup + self.last_channel_refresh_time = time.time() + + channel_refresh_interval = self.bot.config.getint('Bot', 'channel_refresh_interval_seconds', fallback=3600) # Default 1 hour + if time.time() - self.last_channel_refresh_time >= channel_refresh_interval: + if (hasattr(self.bot, 'channel_manager') and self.bot.channel_manager and + hasattr(self.bot, 'connected') and self.bot.connected): + # Refresh channels from device + import asyncio + try: + loop = asyncio.get_event_loop() + except RuntimeError: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + # Fetch channels and update database + self.bot.logger.debug("Periodic channel refresh: fetching channels from device") + loop.run_until_complete(self.bot.channel_manager.fetch_all_channels(force_refresh=True)) + self.last_channel_refresh_time = time.time() + + # Process pending channel operations from web viewer (every 5 seconds) + if not hasattr(self, 'last_channel_ops_check_time'): + self.last_channel_ops_check_time = 0 + + if time.time() - self.last_channel_ops_check_time >= 5: # Every 5 seconds + if (hasattr(self.bot, 'channel_manager') and self.bot.channel_manager and + hasattr(self.bot, 'connected') and self.bot.connected): + import asyncio + try: + loop = asyncio.get_event_loop() + except RuntimeError: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + loop.run_until_complete(self._process_channel_operations()) + self.last_channel_ops_check_time = time.time() + + # Process feed message queue (every 2 seconds) + if not hasattr(self, 'last_message_queue_check_time'): + self.last_message_queue_check_time = 0 + + if time.time() - self.last_message_queue_check_time >= 2: # Every 2 seconds + if (hasattr(self.bot, 'feed_manager') and self.bot.feed_manager and + hasattr(self.bot, 'connected') and self.bot.connected): + import asyncio + try: + loop = asyncio.get_event_loop() + except RuntimeError: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + loop.run_until_complete(self.bot.feed_manager.process_message_queue()) + self.last_message_queue_check_time = time.time() + schedule.run_pending() time.sleep(1) @@ -194,3 +274,108 @@ class MessageScheduler: self.logger.info("Interval-based flood advert sent successfully") except Exception as e: self.logger.error(f"Error sending interval-based advert: {e}") + + async def _process_channel_operations(self): + """Process pending channel operations from the web viewer""" + try: + db_path = self.bot.db_manager.db_path + + # Get pending operations + with sqlite3.connect(db_path) as conn: + conn.row_factory = sqlite3.Row + cursor = conn.cursor() + + cursor.execute(''' + SELECT id, operation_type, channel_idx, channel_name, channel_key_hex + FROM channel_operations + WHERE status = 'pending' + ORDER BY created_at ASC + LIMIT 10 + ''') + + operations = cursor.fetchall() + + if not operations: + return + + self.logger.info(f"Processing {len(operations)} pending channel operation(s)") + + for op in operations: + op_id = op['id'] + op_type = op['operation_type'] + channel_idx = op['channel_idx'] + channel_name = op['channel_name'] + channel_key_hex = op['channel_key_hex'] + + try: + success = False + error_msg = None + + if op_type == 'add': + # Add channel + if channel_key_hex: + # Custom channel with key + channel_secret = bytes.fromhex(channel_key_hex) + success = await self.bot.channel_manager.add_channel( + channel_idx, channel_name, channel_secret=channel_secret + ) + else: + # Hashtag channel (firmware generates key) + success = await self.bot.channel_manager.add_channel( + channel_idx, channel_name + ) + + if success: + self.logger.info(f"Successfully processed channel add operation: {channel_name} at index {channel_idx}") + else: + error_msg = "Failed to add channel" + + elif op_type == 'remove': + # Remove channel + success = await self.bot.channel_manager.remove_channel(channel_idx) + + if success: + self.logger.info(f"Successfully processed channel remove operation: index {channel_idx}") + else: + error_msg = "Failed to remove channel" + + # Update operation status + with sqlite3.connect(db_path) as conn: + cursor = conn.cursor() + if success: + cursor.execute(''' + UPDATE channel_operations + SET status = 'completed', + processed_at = CURRENT_TIMESTAMP, + result_data = ? + WHERE id = ? + ''', (json.dumps({'success': True}), op_id)) + else: + cursor.execute(''' + UPDATE channel_operations + SET status = 'failed', + processed_at = CURRENT_TIMESTAMP, + error_message = ? + WHERE id = ? + ''', (error_msg or 'Unknown error', op_id)) + conn.commit() + + except Exception as e: + self.logger.error(f"Error processing channel operation {op_id}: {e}") + # Mark as failed + try: + with sqlite3.connect(db_path) as conn: + cursor = conn.cursor() + cursor.execute(''' + UPDATE channel_operations + SET status = 'failed', + processed_at = CURRENT_TIMESTAMP, + error_message = ? + WHERE id = ? + ''', (str(e), op_id)) + conn.commit() + except Exception as update_error: + self.logger.error(f"Error updating operation status: {update_error}") + + except Exception as e: + self.logger.error(f"Error in _process_channel_operations: {e}") diff --git a/modules/web_viewer/app.py b/modules/web_viewer/app.py index e96d4aa..8b3e4c0 100644 --- a/modules/web_viewer/app.py +++ b/modules/web_viewer/app.py @@ -63,6 +63,9 @@ class BotDataViewer: # Load configuration self.config = self._load_config(config_path) + # Setup template context processor for global template variables + self._setup_template_context() + # Initialize databases self._init_databases() @@ -116,6 +119,25 @@ class BotDataViewer: config.read(config_path) return config + def _setup_template_context(self): + """Setup template context processor to inject global variables""" + @self.app.context_processor + def inject_template_vars(): + """Inject variables available to all templates""" + # Check if greeter is enabled, defaulting to False if section doesn't exist + try: + greeter_enabled = self.config.getboolean('Greeter_Command', 'enabled', fallback=False) + except (configparser.NoSectionError, configparser.NoOptionError): + greeter_enabled = False + + # Check if feed manager is enabled, defaulting to False if section doesn't exist + try: + feed_manager_enabled = self.config.getboolean('Feed_Manager', 'feed_manager_enabled', fallback=False) + except (configparser.NoSectionError, configparser.NoOptionError): + feed_manager_enabled = False + + return dict(greeter_enabled=greeter_enabled, feed_manager_enabled=feed_manager_enabled) + def _init_databases(self): """Initialize database connections""" try: @@ -151,12 +173,13 @@ class BotDataViewer: def _init_packet_stream_table(self): """Initialize the packet_stream table in bot_data.db""" + conn = None try: # Get database path from config db_path = self.config.get('Database', 'path', fallback='bot_data.db') # Connect to database and create table if it doesn't exist - conn = sqlite3.connect(db_path) + conn = sqlite3.connect(db_path, timeout=30) cursor = conn.cursor() # Create packet_stream table with schema matching the INSERT statements @@ -182,13 +205,18 @@ class BotDataViewer: ''') conn.commit() - conn.close() self.logger.info(f"Initialized packet_stream table in {db_path}") except Exception as e: self.logger.error(f"Failed to initialize packet_stream table: {e}") # Don't raise - allow web viewer to continue even if table init fails + finally: + if conn: + try: + conn.close() + except Exception as e: + self.logger.debug(f"Error closing init connection: {e}") def _get_db_connection(self): """Get database connection - create new connection for each request to avoid threading issues""" @@ -234,6 +262,16 @@ class BotDataViewer: """Greeter management page""" return render_template('greeter.html') + @self.app.route('/feeds') + def feeds(): + """Feed management page""" + return render_template('feeds.html') + + @self.app.route('/radio') + def radio(): + """Radio settings page""" + return render_template('radio.html') + # API Routes @self.app.route('/api/health') @@ -338,6 +376,7 @@ class BotDataViewer: @self.app.route('/api/recent_commands') def api_recent_commands(): """API endpoint to get recent commands from database""" + conn = None try: import sqlite3 import json @@ -349,7 +388,7 @@ class BotDataViewer: # Get database path db_path = self.config.get('Database', 'path', fallback='bot_data.db') - conn = sqlite3.connect(db_path) + conn = sqlite3.connect(db_path, timeout=30) cursor = conn.cursor() cursor.execute(''' @@ -360,7 +399,6 @@ class BotDataViewer: ''', (cutoff_time,)) rows = cursor.fetchall() - conn.close() # Parse and return commands commands = [] @@ -376,6 +414,12 @@ class BotDataViewer: except Exception as e: self.logger.error(f"Error getting recent commands: {e}") return jsonify({'error': str(e)}), 500 + finally: + if conn: + try: + conn.close() + except Exception as e: + self.logger.debug(f"Error closing recent_commands connection: {e}") @self.app.route('/api/geocode-contact', methods=['POST']) def api_geocode_contact(): @@ -876,6 +920,358 @@ class BotDataViewer: except Exception as e: self.logger.error(f"Error ungreeting user: {e}", exc_info=True) return jsonify({'success': False, 'error': str(e)}), 500 + + # Feed management API endpoints + @self.app.route('/api/feeds') + def api_feeds(): + """Get all feed subscriptions with statistics""" + try: + feeds = self._get_feed_subscriptions() + return jsonify(feeds) + except Exception as e: + self.logger.error(f"Error getting feeds: {e}") + return jsonify({'error': str(e)}), 500 + + @self.app.route('/api/feeds/') + def api_feed_detail(feed_id): + """Get detailed information about a specific feed""" + try: + feed = self._get_feed_subscription(feed_id) + if not feed: + return jsonify({'error': 'Feed not found'}), 404 + + # Get activity and errors + activity = self._get_feed_activity(feed_id) + errors = self._get_feed_errors(feed_id) + + feed['activity'] = activity + feed['errors'] = errors + + return jsonify(feed) + except Exception as e: + self.logger.error(f"Error getting feed detail: {e}") + return jsonify({'error': str(e)}), 500 + + @self.app.route('/api/feeds', methods=['POST']) + def api_create_feed(): + """Create a new feed subscription""" + try: + data = request.get_json() + if not data: + return jsonify({'error': 'No data provided'}), 400 + + feed_id = self._create_feed_subscription(data) + return jsonify({'success': True, 'id': feed_id}) + except Exception as e: + self.logger.error(f"Error creating feed: {e}") + return jsonify({'error': str(e)}), 500 + + @self.app.route('/api/feeds/', methods=['PUT']) + def api_update_feed(feed_id): + """Update an existing feed subscription""" + try: + data = request.get_json() + if not data: + return jsonify({'error': 'No data provided'}), 400 + + success = self._update_feed_subscription(feed_id, data) + if not success: + return jsonify({'error': 'Feed not found'}), 404 + + return jsonify({'success': True}) + except Exception as e: + self.logger.error(f"Error updating feed: {e}") + return jsonify({'error': str(e)}), 500 + + @self.app.route('/api/feeds/', methods=['DELETE']) + def api_delete_feed(feed_id): + """Delete a feed subscription""" + try: + success = self._delete_feed_subscription(feed_id) + if not success: + return jsonify({'error': 'Feed not found'}), 404 + + return jsonify({'success': True}) + except Exception as e: + self.logger.error(f"Error deleting feed: {e}") + return jsonify({'error': str(e)}), 500 + + @self.app.route('/api/feeds/default-format', methods=['GET']) + def api_get_default_format(): + """Get the default output format from config""" + try: + default_format = self.config.get('Feed_Manager', 'default_output_format', + fallback='{emoji} {body|truncate:100} - {date}\n{link|truncate:50}') + return jsonify({'default_format': default_format}) + except Exception as e: + self.logger.error(f"Error getting default format: {e}") + return jsonify({'default_format': '{emoji} {body|truncate:100} - {date}\n{link|truncate:50}'}) + + @self.app.route('/api/feeds/preview', methods=['POST']) + def api_preview_feed(): + """Preview feed items with custom output format""" + try: + data = request.get_json() + if not data or 'feed_url' not in data: + return jsonify({'error': 'feed_url is required'}), 400 + + feed_url = data['feed_url'] + feed_type = data.get('feed_type', 'rss') + output_format = data.get('output_format', '') + api_config = data.get('api_config', {}) + filter_config = data.get('filter_config') + sort_config = data.get('sort_config') + + # Get default format from config if not provided + if not output_format: + output_format = self.config.get('Feed_Manager', 'default_output_format', + fallback='{emoji} {body|truncate:100} - {date}\n{link|truncate:50}') + + # Fetch and format feed items + preview_items = self._preview_feed_items(feed_url, feed_type, output_format, api_config, filter_config, sort_config) + + return jsonify({ + 'success': True, + 'items': preview_items + }) + except Exception as e: + self.logger.error(f"Error previewing feed: {e}") + return jsonify({'error': str(e)}), 500 + + @self.app.route('/api/feeds/test', methods=['POST']) + def api_test_feed(): + """Test a feed URL and return preview of recent items""" + try: + data = request.get_json() + if not data or 'url' not in data: + return jsonify({'error': 'URL is required'}), 400 + + # This would require feed_manager - for now just validate URL + from urllib.parse import urlparse + url = data['url'] + result = urlparse(url) + if not all([result.scheme in ['http', 'https'], result.netloc]): + return jsonify({'error': 'Invalid URL format'}), 400 + + return jsonify({'success': True, 'message': 'URL validated (full test requires feed manager)'}) + except Exception as e: + self.logger.error(f"Error testing feed: {e}") + return jsonify({'error': str(e)}), 500 + + @self.app.route('/api/feeds/stats') + def api_feed_stats(): + """Get aggregate feed statistics""" + try: + stats = self._get_feed_statistics() + return jsonify(stats) + except Exception as e: + self.logger.error(f"Error getting feed stats: {e}") + return jsonify({'error': str(e)}), 500 + + @self.app.route('/api/feeds//activity') + def api_feed_activity(feed_id): + """Get activity log for a specific feed""" + try: + activity = self._get_feed_activity(feed_id, limit=50) + return jsonify({'activity': activity}) + except Exception as e: + self.logger.error(f"Error getting feed activity: {e}") + return jsonify({'error': str(e)}), 500 + + @self.app.route('/api/feeds//errors') + def api_feed_errors(feed_id): + """Get error history for a specific feed""" + try: + errors = self._get_feed_errors(feed_id, limit=20) + return jsonify({'errors': errors}) + except Exception as e: + self.logger.error(f"Error getting feed errors: {e}") + return jsonify({'error': str(e)}), 500 + + @self.app.route('/api/feeds//refresh', methods=['POST']) + def api_refresh_feed(feed_id): + """Manually trigger a feed check""" + try: + # This would trigger feed_manager to poll this feed immediately + # For now, just acknowledge the request + return jsonify({'success': True, 'message': 'Feed refresh queued'}) + except Exception as e: + self.logger.error(f"Error refreshing feed: {e}") + return jsonify({'error': str(e)}), 500 + + # Channel management API endpoints + @self.app.route('/api/channels') + def api_channels(): + """Get all configured channels""" + try: + channels = self._get_channels() + return jsonify({'channels': channels}) + except Exception as e: + self.logger.error(f"Error getting channels: {e}") + return jsonify({'error': str(e)}), 500 + + @self.app.route('/api/channels', methods=['POST']) + def api_create_channel(): + """Create a new channel (hashtag or custom)""" + try: + data = request.get_json() + if not data or 'name' not in data: + return jsonify({'error': 'Channel name is required'}), 400 + + channel_name = data.get('name', '').strip() + channel_idx = data.get('channel_idx') + channel_key = data.get('channel_key', '').strip() + + if not channel_name: + return jsonify({'error': 'Channel name cannot be empty'}), 400 + + # If channel_idx not provided, find the lowest available index + if channel_idx is None: + channel_idx = self._get_lowest_available_channel_index() + if channel_idx is None: + return jsonify({'error': 'No available channel slots. All 40 channels are in use.'}), 400 + + # Determine if it's a hashtag channel + is_hashtag = channel_name.startswith('#') + + # Validate custom channel has key + if not is_hashtag and not channel_key: + return jsonify({'error': 'Channel key is required for custom channels (channels without # prefix)'}), 400 + + # Validate key format if provided + if channel_key: + if len(channel_key) != 32: + return jsonify({'error': 'Channel key must be exactly 32 hexadecimal characters'}), 400 + if not all(c in '0123456789abcdefABCDEF' for c in channel_key): + return jsonify({'error': 'Channel key must contain only hexadecimal characters (0-9, a-f, A-F)'}), 400 + + # Try to create channel via bot's channel manager + result = self._add_channel_for_web(channel_idx, channel_name, channel_key if not is_hashtag else None) + + if result.get('success'): + if result.get('pending'): + # Operation is queued, return operation_id for polling + return jsonify({ + 'success': True, + 'pending': True, + 'operation_id': result.get('operation_id'), + 'message': result.get('message', 'Channel operation queued') + }) + else: + return jsonify({'success': True, 'message': 'Channel created successfully'}) + else: + return jsonify({'error': result.get('error', 'Failed to create channel')}), 500 + + except Exception as e: + self.logger.error(f"Error creating channel: {e}") + return jsonify({'error': str(e)}), 500 + + @self.app.route('/api/channels/', methods=['DELETE']) + def api_delete_channel(channel_idx): + """Remove a channel""" + try: + result = self._remove_channel_for_web(channel_idx) + if result.get('success'): + if result.get('pending'): + # Operation is queued, return operation_id for polling + return jsonify({ + 'success': True, + 'pending': True, + 'operation_id': result.get('operation_id'), + 'message': result.get('message', 'Channel operation queued') + }) + else: + return jsonify({'success': True, 'message': 'Channel deleted successfully'}) + else: + return jsonify({'error': result.get('error', 'Failed to delete channel')}), 500 + except Exception as e: + self.logger.error(f"Error deleting channel: {e}") + return jsonify({'error': str(e)}), 500 + + @self.app.route('/api/channel-operations/', methods=['GET']) + def api_get_operation_status(operation_id): + """Get status of a channel operation""" + try: + conn = self._get_db_connection() + cursor = conn.cursor() + cursor.execute(''' + SELECT status, error_message, result_data, processed_at + FROM channel_operations + WHERE id = ? + ''', (operation_id,)) + + result = cursor.fetchone() + conn.close() + + if not result: + return jsonify({'error': 'Operation not found'}), 404 + + status, error_msg, result_data, processed_at = result + + return jsonify({ + 'operation_id': operation_id, + 'status': status, + 'error_message': error_msg, + 'processed_at': processed_at, + 'result_data': json.loads(result_data) if result_data else None + }) + except Exception as e: + self.logger.error(f"Error getting operation status: {e}") + return jsonify({'error': str(e)}), 500 + + @self.app.route('/api/channels/validate', methods=['POST']) + def api_validate_channel(): + """Validate if a channel exists or can be created""" + try: + data = request.get_json() + if not data or 'name' not in data: + return jsonify({'error': 'Channel name is required'}), 400 + + channel_name = data['name'] + # Check if channel exists + channel_num = self._get_channel_number(channel_name) + + return jsonify({ + 'exists': channel_num is not None, + 'channel_num': channel_num + }) + except Exception as e: + self.logger.error(f"Error validating channel: {e}") + return jsonify({'error': str(e)}), 500 + + @self.app.route('/api/channels/', methods=['PUT']) + def api_update_channel(channel_idx): + """Update channel name or configuration""" + try: + data = request.get_json() + if not data: + return jsonify({'error': 'No data provided'}), 400 + + # This would use channel_manager + return jsonify({'success': True, 'message': 'Channel update requires bot connection'}) + except Exception as e: + self.logger.error(f"Error updating channel: {e}") + return jsonify({'error': str(e)}), 500 + + @self.app.route('/api/channels/stats') + def api_channel_stats(): + """Get channel statistics and usage data""" + try: + stats = self._get_channel_statistics() + return jsonify(stats) + except Exception as e: + self.logger.error(f"Error getting channel stats: {e}") + return jsonify({'error': str(e)}), 500 + + @self.app.route('/api/channels//feeds') + def api_channel_feeds(channel_idx): + """Get all feed subscriptions for a specific channel""" + try: + feeds = self._get_feeds_by_channel(channel_idx) + return jsonify({'feeds': feeds}) + except Exception as e: + self.logger.error(f"Error getting channel feeds: {e}") + return jsonify({'error': str(e)}), 500 def _setup_socketio_handlers(self): """Setup SocketIO event handlers using modern patterns""" @@ -979,7 +1375,11 @@ class BotDataViewer: def poll_database(): last_timestamp = 0 + consecutive_errors = 0 + max_consecutive_errors = 10 + while True: + conn = None try: import time import sqlite3 @@ -988,8 +1388,8 @@ class BotDataViewer: # Get database path db_path = self.config.get('Database', 'path', fallback='bot_data.db') - # Connect to database - conn = sqlite3.connect(db_path) + # Connect to database with timeout to prevent hanging + conn = sqlite3.connect(db_path, timeout=30) cursor = conn.cursor() # Get new data since last poll @@ -1000,7 +1400,6 @@ class BotDataViewer: ''', (last_timestamp,)) rows = cursor.fetchall() - conn.close() # Process new data for timestamp, data_json, data_type in rows: @@ -1016,18 +1415,50 @@ class BotDataViewer: self._handle_packet_data(data) # Treat routing as packet data except Exception as e: - self.logger.debug(f"Error processing database data: {e}") + self.logger.warning(f"Error processing database data: {e}") # Update last timestamp if rows: last_timestamp = rows[-1][0] + # Reset error counter on success + consecutive_errors = 0 + # Sleep before next poll time.sleep(0.5) # Poll every 500ms + except sqlite3.OperationalError as e: + consecutive_errors += 1 + error_msg = str(e) + + # Log at appropriate level based on error frequency + if consecutive_errors >= max_consecutive_errors: + self.logger.error(f"Database polling persistent error (attempt {consecutive_errors}): {error_msg}") + # Exponential backoff for persistent errors + time.sleep(min(60, 2 ** min(consecutive_errors - max_consecutive_errors, 5))) + elif consecutive_errors > 3: + self.logger.warning(f"Database polling error (attempt {consecutive_errors}): {error_msg}") + time.sleep(5) # Wait longer on repeated errors + else: + self.logger.debug(f"Database polling error (attempt {consecutive_errors}): {error_msg}") + time.sleep(1) # Wait longer on error + except Exception as e: - self.logger.debug(f"Database polling error: {e}") - time.sleep(1) # Wait longer on error + consecutive_errors += 1 + if consecutive_errors >= max_consecutive_errors: + self.logger.error(f"Database polling unexpected error (attempt {consecutive_errors}): {e}", exc_info=True) + time.sleep(min(60, 2 ** min(consecutive_errors - max_consecutive_errors, 5))) + else: + self.logger.warning(f"Database polling unexpected error (attempt {consecutive_errors}): {e}") + time.sleep(2) + + finally: + # Always close connection, even on error + if conn: + try: + conn.close() + except Exception as e: + self.logger.debug(f"Error closing database connection: {e}") # Start polling thread polling_thread = threading.Thread(target=poll_database, daemon=True) @@ -1059,6 +1490,7 @@ class BotDataViewer: def _cleanup_old_data(self, days_to_keep: int = 7): """Clean up old packet stream data to prevent database bloat""" + conn = None try: import sqlite3 import time @@ -1068,7 +1500,8 @@ class BotDataViewer: # Get database path db_path = self.config.get('Database', 'path', fallback='bot_data.db') - conn = sqlite3.connect(db_path) + # Use timeout to prevent hanging + conn = sqlite3.connect(db_path, timeout=30) cursor = conn.cursor() # Clean up old packet stream data @@ -1076,13 +1509,18 @@ class BotDataViewer: deleted_count = cursor.rowcount conn.commit() - conn.close() if deleted_count > 0: self.logger.info(f"Cleaned up {deleted_count} old packet stream entries (older than {days_to_keep} days)") except Exception as e: self.logger.error(f"Error cleaning up old packet stream data: {e}") + finally: + if conn: + try: + conn.close() + except Exception as e: + self.logger.debug(f"Error closing cleanup connection: {e}") def _get_database_stats(self, top_users_window='all', top_commands_window='all', top_paths_window='all', top_channels_window='all'): @@ -1902,6 +2340,1148 @@ class BotDataViewer: return {'error': str(e)} + def _get_feed_subscriptions(self, channel_filter=None): + """Get all feed subscriptions, optionally filtered by channel""" + import sqlite3 + conn = None + try: + conn = self._get_db_connection() + conn.row_factory = sqlite3.Row + cursor = conn.cursor() + + if channel_filter: + cursor.execute(''' + SELECT * FROM feed_subscriptions + WHERE channel_name = ? + ORDER BY id + ''', (channel_filter,)) + else: + cursor.execute(''' + SELECT * FROM feed_subscriptions + ORDER BY id + ''') + + rows = cursor.fetchall() + feeds = [] + for row in rows: + feed = dict(row) + # Get feed count for this channel + cursor.execute(''' + SELECT COUNT(*) FROM feed_activity + WHERE feed_id = ? + ''', (feed['id'],)) + feed['item_count'] = cursor.fetchone()[0] + + # Get error count + cursor.execute(''' + SELECT COUNT(*) FROM feed_errors + WHERE feed_id = ? AND resolved_at IS NULL + ''', (feed['id'],)) + feed['error_count'] = cursor.fetchone()[0] + + feeds.append(feed) + + return {'feeds': feeds, 'total': len(feeds)} + except Exception as e: + self.logger.error(f"Error getting feed subscriptions: {e}") + return {'feeds': [], 'total': 0, 'error': str(e)} + finally: + if conn: + conn.close() + + def _get_feed_subscription(self, feed_id): + """Get a single feed subscription by ID""" + import sqlite3 + conn = None + try: + conn = self._get_db_connection() + conn.row_factory = sqlite3.Row + cursor = conn.cursor() + cursor.execute('SELECT * FROM feed_subscriptions WHERE id = ?', (feed_id,)) + row = cursor.fetchone() + return dict(row) if row else None + except Exception as e: + self.logger.error(f"Error getting feed subscription: {e}") + return None + finally: + if conn: + conn.close() + + def _create_feed_subscription(self, data): + """Create a new feed subscription""" + import sqlite3 + import json + conn = None + try: + feed_type = data.get('feed_type') + feed_url = data.get('feed_url') + channel_name = data.get('channel_name') + feed_name = data.get('feed_name') + check_interval = data.get('check_interval_seconds', 300) + api_config = data.get('api_config') + output_format = data.get('output_format') + message_send_interval = data.get('message_send_interval_seconds') + + if not all([feed_type, feed_url, channel_name]): + raise ValueError("feed_type, feed_url, and channel_name are required") + + conn = self._get_db_connection() + cursor = conn.cursor() + + api_config_str = json.dumps(api_config) if api_config else None + + cursor.execute(''' + INSERT INTO feed_subscriptions + (feed_type, feed_url, channel_name, feed_name, check_interval_seconds, api_config, output_format, message_send_interval_seconds) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ''', (feed_type, feed_url, channel_name, feed_name, check_interval, api_config_str, output_format, message_send_interval)) + + conn.commit() + return cursor.lastrowid + except Exception as e: + if conn: + conn.rollback() + raise + finally: + if conn: + conn.close() + + def _update_feed_subscription(self, feed_id, data): + """Update a feed subscription""" + import sqlite3 + import json + conn = None + try: + conn = self._get_db_connection() + cursor = conn.cursor() + + updates = [] + params = [] + + if 'feed_name' in data: + updates.append('feed_name = ?') + params.append(data['feed_name']) + + if 'check_interval_seconds' in data: + updates.append('check_interval_seconds = ?') + params.append(data['check_interval_seconds']) + + if 'enabled' in data: + updates.append('enabled = ?') + params.append(1 if data['enabled'] else 0) + + if 'api_config' in data: + updates.append('api_config = ?') + params.append(json.dumps(data['api_config']) if data['api_config'] else None) + + if 'output_format' in data: + updates.append('output_format = ?') + params.append(data['output_format'] if data['output_format'] else None) + + if 'message_send_interval_seconds' in data: + updates.append('message_send_interval_seconds = ?') + params.append(float(data['message_send_interval_seconds']) if data['message_send_interval_seconds'] else None) + + if 'filter_config' in data: + updates.append('filter_config = ?') + params.append(json.dumps(data['filter_config']) if data['filter_config'] else None) + + if 'sort_config' in data: + updates.append('sort_config = ?') + params.append(json.dumps(data['sort_config']) if data['sort_config'] else None) + + if 'message_send_interval_seconds' in data: + updates.append('message_send_interval_seconds = ?') + params.append(data['message_send_interval_seconds']) + + if not updates: + return True # Nothing to update + + updates.append('updated_at = CURRENT_TIMESTAMP') + params.append(feed_id) + + query = f'UPDATE feed_subscriptions SET {", ".join(updates)} WHERE id = ?' + cursor.execute(query, params) + conn.commit() + + return cursor.rowcount > 0 + except Exception as e: + if conn: + conn.rollback() + raise + finally: + if conn: + conn.close() + + def _delete_feed_subscription(self, feed_id): + """Delete a feed subscription""" + import sqlite3 + conn = None + try: + conn = self._get_db_connection() + cursor = conn.cursor() + cursor.execute('DELETE FROM feed_subscriptions WHERE id = ?', (feed_id,)) + conn.commit() + return cursor.rowcount > 0 + except Exception as e: + if conn: + conn.rollback() + raise + finally: + if conn: + conn.close() + + def _get_feed_activity(self, feed_id, limit=50): + """Get activity log for a feed""" + import sqlite3 + conn = None + try: + conn = self._get_db_connection() + conn.row_factory = sqlite3.Row + cursor = conn.cursor() + cursor.execute(''' + SELECT * FROM feed_activity + WHERE feed_id = ? + ORDER BY processed_at DESC + LIMIT ? + ''', (feed_id, limit)) + rows = cursor.fetchall() + return [dict(row) for row in rows] + except Exception as e: + self.logger.error(f"Error getting feed activity: {e}") + return [] + finally: + if conn: + conn.close() + + def _get_feed_errors(self, feed_id, limit=20): + """Get error history for a feed""" + import sqlite3 + conn = None + try: + conn = self._get_db_connection() + conn.row_factory = sqlite3.Row + cursor = conn.cursor() + cursor.execute(''' + SELECT * FROM feed_errors + WHERE feed_id = ? + ORDER BY occurred_at DESC + LIMIT ? + ''', (feed_id, limit)) + rows = cursor.fetchall() + return [dict(row) for row in rows] + except Exception as e: + self.logger.error(f"Error getting feed errors: {e}") + return [] + finally: + if conn: + conn.close() + + def _get_feed_statistics(self): + """Get aggregate feed statistics""" + import sqlite3 + conn = None + try: + conn = self._get_db_connection() + cursor = conn.cursor() + + stats = {} + + # Total subscriptions + cursor.execute('SELECT COUNT(*) FROM feed_subscriptions') + stats['total_subscriptions'] = cursor.fetchone()[0] + + # Enabled subscriptions + cursor.execute('SELECT COUNT(*) FROM feed_subscriptions WHERE enabled = 1') + stats['enabled_subscriptions'] = cursor.fetchone()[0] + + # Items processed in last 24h + cursor.execute(''' + SELECT COUNT(*) FROM feed_activity + WHERE processed_at > datetime('now', '-24 hours') + ''') + stats['items_24h'] = cursor.fetchone()[0] + + # Items processed in last 7d + cursor.execute(''' + SELECT COUNT(*) FROM feed_activity + WHERE processed_at > datetime('now', '-7 days') + ''') + stats['items_7d'] = cursor.fetchone()[0] + + # Error count + cursor.execute(''' + SELECT COUNT(*) FROM feed_errors + WHERE resolved_at IS NULL + ''') + stats['active_errors'] = cursor.fetchone()[0] + + # Most active channels + cursor.execute(''' + SELECT channel_name, COUNT(*) as feed_count + FROM feed_subscriptions + WHERE enabled = 1 + GROUP BY channel_name + ORDER BY feed_count DESC + LIMIT 10 + ''') + stats['top_channels'] = [{'channel': row[0], 'count': row[1]} for row in cursor.fetchall()] + + return stats + except Exception as e: + self.logger.error(f"Error getting feed statistics: {e}") + return {'error': str(e)} + finally: + if conn: + conn.close() + + def _get_feeds_by_channel(self, channel_idx): + """Get all feeds for a specific channel index""" + # First get channel name from index + # This would require channel_manager access + # For now, return empty list + return [] + + def _get_channels(self): + """Get all configured channels from database""" + import sqlite3 + conn = None + try: + conn = self._get_db_connection() + conn.row_factory = sqlite3.Row + cursor = conn.cursor() + + cursor.execute(''' + SELECT channel_idx, channel_name, channel_type, channel_key_hex, last_updated + FROM channels + ORDER BY channel_idx + ''') + + rows = cursor.fetchall() + channels = [] + for row in rows: + channels.append({ + 'channel_idx': row['channel_idx'], + 'index': row['channel_idx'], # Alias for compatibility + 'name': row['channel_name'], + 'channel_name': row['channel_name'], # Alias for compatibility + 'type': row['channel_type'] or 'hashtag', + 'key_hex': row['channel_key_hex'], + 'last_updated': row['last_updated'] + }) + + return channels + except Exception as e: + self.logger.error(f"Error getting channels: {e}") + return [] + finally: + if conn: + conn.close() + + def _get_channel_number(self, channel_name): + """Get channel number from channel name""" + # This would use channel_manager + # For now, return None + return None + + def _get_lowest_available_channel_index(self): + """Get the lowest available channel index (0-39)""" + try: + channels = self._get_channels() + used_indices = {c['channel_idx'] for c in channels} + + # Find the lowest available index + for i in range(40): + if i not in used_indices: + return i + + # All channels are used + return None + except Exception as e: + self.logger.error(f"Error getting lowest available channel index: {e}") + return None + + def _get_channel_statistics(self): + """Get channel statistics""" + import sqlite3 + conn = None + try: + conn = self._get_db_connection() + cursor = conn.cursor() + + # Get feed count per channel + cursor.execute(''' + SELECT channel_name, COUNT(*) as feed_count + FROM feed_subscriptions + WHERE enabled = 1 + GROUP BY channel_name + ''') + + channel_feeds = {row[0]: row[1] for row in cursor.fetchall()} + + return { + 'channels_with_feeds': len(channel_feeds), + 'channel_feed_counts': channel_feeds + } + except Exception as e: + self.logger.error(f"Error getting channel statistics: {e}") + return {'error': str(e)} + finally: + if conn: + conn.close() + + def _preview_feed_items(self, feed_url: str, feed_type: str, output_format: str, api_config: dict = None, filter_config: dict = None, sort_config: dict = None) -> List[Dict[str, Any]]: + """Preview feed items with custom output format (standalone, doesn't require bot)""" + import feedparser + import requests + import html + import re + from datetime import datetime, timezone + + try: + items = [] + + if feed_type == 'rss': + # Fetch RSS feed + response = requests.get(feed_url, timeout=30, headers={'User-Agent': 'MeshCoreBot/1.0 FeedManager'}) + response.raise_for_status() + parsed = feedparser.parse(response.text) + + # Get items (we'll filter and limit later) + for entry in parsed.entries[:20]: # Fetch more items to account for filtering + # Parse published date + published = None + if hasattr(entry, 'published_parsed') and entry.published_parsed: + try: + published = datetime(*entry.published_parsed[:6], tzinfo=timezone.utc) + except Exception: + pass + + items.append({ + 'title': entry.get('title', 'Untitled'), + 'description': entry.get('description', ''), + 'link': entry.get('link', ''), + 'published': published + }) + + elif feed_type == 'api': + # Fetch API feed + method = api_config.get('method', 'GET').upper() + headers = api_config.get('headers', {}) + params = api_config.get('params', {}) + body = api_config.get('body') + parser_config = api_config.get('response_parser', {}) + + if method == 'POST': + response = requests.post(feed_url, headers=headers, params=params, json=body, timeout=30) + else: + response = requests.get(feed_url, headers=headers, params=params, timeout=30) + response.raise_for_status() + + # Try to parse JSON, handle cases where response might be a string + try: + data = response.json() + except ValueError: + # If JSON parsing fails, try to get text and see if it's an error message + text = response.text + raise Exception(f"API returned non-JSON response: {text[:200]}") + + # Check if response is an error message (string) + if isinstance(data, str): + raise Exception(f"API returned error message: {data[:200]}") + + # Ensure data is a dict or list + if not isinstance(data, (dict, list)): + raise Exception(f"API response is not a valid JSON object or array: {type(data).__name__} - {str(data)[:200]}") + + # Extract items using parser config + items_path = parser_config.get('items_path', '') + if items_path: + parts = items_path.split('.') + items_data = data + for part in parts: + if isinstance(items_data, dict): + items_data = items_data.get(part, []) + else: + raise Exception(f"Cannot navigate path '{items_path}': expected dict at '{part}', got {type(items_data).__name__}") + else: + # If no items_path, data should be a list or we wrap it + if isinstance(data, list): + items_data = data + elif isinstance(data, dict): + # If it's a dict, try to find common array fields + items_data = data.get('items', data.get('data', data.get('results', [data]))) + else: + items_data = [data] + + # Ensure items_data is a list + if not isinstance(items_data, list): + items_data = [items_data] + + # Get items (we'll filter and limit later) + id_field = parser_config.get('id_field', 'id') + title_field = parser_config.get('title_field', 'title') + description_field = parser_config.get('description_field', 'description') + timestamp_field = parser_config.get('timestamp_field', 'created_at') + + # Helper function to get nested values + def get_nested_value(data, path, default=''): + if not path or not data: + return default + parts = path.split('.') + value = data + for part in parts: + if isinstance(value, dict): + value = value.get(part) + elif isinstance(value, list): + try: + idx = int(part) + if 0 <= idx < len(value): + value = value[idx] + else: + return default + except (ValueError, TypeError): + return default + else: + return default + if value is None: + return default + return value if value is not None else default + + for item_data in items_data[:20]: # Fetch more items to account for filtering + # Ensure item_data is a dict + if not isinstance(item_data, dict): + # If it's not a dict, try to convert or skip + if isinstance(item_data, str): + # If it's a string, create a simple dict + item_data = {'title': item_data, 'description': item_data} + else: + # Try to convert to dict or skip + continue + + # Parse timestamp if available - support nested paths + published = None + if timestamp_field: + ts_value = get_nested_value(item_data, timestamp_field) + if ts_value: + try: + if isinstance(ts_value, (int, float)): + published = datetime.fromtimestamp(ts_value, tz=timezone.utc) + elif isinstance(ts_value, str): + # Try Microsoft date format first + if ts_value.startswith('/Date('): + published = self._parse_microsoft_date(ts_value) + else: + # Try ISO format + try: + published = datetime.fromisoformat(ts_value.replace('Z', '+00:00')) + except ValueError: + # Try common formats + for fmt in ['%Y-%m-%dT%H:%M:%S', '%Y-%m-%d %H:%M:%S', '%Y-%m-%d']: + try: + published = datetime.strptime(ts_value, fmt) + if published.tzinfo is None: + published = published.replace(tzinfo=timezone.utc) + break + except ValueError: + continue + except Exception: + pass + + # Get description - support nested paths + description = '' + if description_field: + desc_value = get_nested_value(item_data, description_field) + if desc_value: + description = str(desc_value) + + items.append({ + 'title': get_nested_value(item_data, title_field, 'Untitled'), + 'description': description, + 'link': item_data.get('link', '') if isinstance(item_data, dict) else '', + 'published': published, + 'raw': item_data # Store raw data for format string access + }) + + # Apply sorting if configured + if sort_config: + items = self._sort_items_preview(items, sort_config) + + # Apply filter if configured + if filter_config: + items = [item for item in items if self._should_include_item(item, filter_config)] + + # Limit to first 3 items after filtering + items = items[:3] + + # Format items using output format + formatted_items = [] + for item in items: + formatted = self._format_feed_item(item, output_format, feed_name='') + formatted_items.append({ + 'original': item, + 'formatted': formatted + }) + + return formatted_items + + except Exception as e: + self.logger.error(f"Error previewing feed: {e}") + raise + + def _should_include_item(self, item: Dict[str, Any], filter_config: dict) -> bool: + """Check if an item should be included based on filter configuration (standalone version for preview)""" + import json + import re + + if not filter_config: + return True + + try: + filter_config_dict = json.loads(filter_config) if isinstance(filter_config, str) else filter_config + except (json.JSONDecodeError, TypeError): + return True + + conditions = filter_config_dict.get('conditions', []) + if not conditions: + return True + + logic = filter_config_dict.get('logic', 'AND').upper() + + # Get raw data for field access + raw_data = item.get('raw', {}) + + # Helper to get nested values + def get_nested_value(data, path, default=''): + if not path or not data: + return default + parts = path.split('.') + value = data + for part in parts: + if isinstance(value, dict): + value = value.get(part) + elif isinstance(value, list): + try: + idx = int(part) + if 0 <= idx < len(value): + value = value[idx] + else: + return default + except (ValueError, TypeError): + return default + else: + return default + if value is None: + return default + return value if value is not None else default + + # Evaluate each condition + results = [] + for condition in conditions: + field_path = condition.get('field') + operator = condition.get('operator', 'equals') + + if not field_path: + continue + + # Get field value using nested access + field_value = get_nested_value(raw_data, field_path, '') + if not field_value and field_path.startswith('raw.'): + field_value = get_nested_value(raw_data, field_path[4:], '') + + if not field_value: + field_value = get_nested_value(item, field_path, '') + + # Convert to string for comparison + field_value_str = str(field_value).lower() if field_value is not None else '' + + # Evaluate condition + result = False + if operator == 'equals': + compare_value = str(condition.get('value', '')).lower() + result = field_value_str == compare_value + elif operator == 'not_equals': + compare_value = str(condition.get('value', '')).lower() + result = field_value_str != compare_value + elif operator == 'in': + values = [str(v).lower() for v in condition.get('values', [])] + result = field_value_str in values + elif operator == 'not_in': + values = [str(v).lower() for v in condition.get('values', [])] + result = field_value_str not in values + elif operator == 'matches': + pattern = condition.get('pattern', '') + if pattern: + try: + result = bool(re.search(pattern, str(field_value), re.IGNORECASE)) + except re.error: + result = False + elif operator == 'not_matches': + pattern = condition.get('pattern', '') + if pattern: + try: + result = not bool(re.search(pattern, str(field_value), re.IGNORECASE)) + except re.error: + result = True + elif operator == 'contains': + compare_value = str(condition.get('value', '')).lower() + result = compare_value in field_value_str + elif operator == 'not_contains': + compare_value = str(condition.get('value', '')).lower() + result = compare_value not in field_value_str + else: + result = True # Default to allowing if operator is unknown + + results.append(result) + + # Apply logic (AND or OR) + if logic == 'OR': + return any(results) + else: # AND (default) + return all(results) + + def _parse_microsoft_date(self, date_str: str) -> Optional[datetime]: + """Parse Microsoft JSON date format: /Date(timestamp-offset)/""" + import re + from datetime import timezone + + if not date_str or not isinstance(date_str, str): + return None + + # Match /Date(timestamp-offset)/ format + match = re.match(r'/Date\((\d+)([+-]\d+)?\)/', date_str) + if match: + timestamp_ms = int(match.group(1)) + offset_str = match.group(2) if match.group(2) else '+0000' + + # Convert milliseconds to seconds + timestamp = timestamp_ms / 1000.0 + + # Parse offset (format: +0800 or -0800) + try: + offset_hours = int(offset_str[:3]) + offset_mins = int(offset_str[3:5]) + offset_seconds = (offset_hours * 3600) + (offset_mins * 60) + if offset_str[0] == '-': + offset_seconds = -offset_seconds + + # Create timezone-aware datetime + tz = timezone.utc + if offset_seconds != 0: + from datetime import timedelta + tz = timezone(timedelta(seconds=offset_seconds)) + + return datetime.fromtimestamp(timestamp, tz=tz) + except (ValueError, IndexError): + # Fallback to UTC if offset parsing fails + return datetime.fromtimestamp(timestamp, tz=timezone.utc) + + return None + + def _sort_items_preview(self, items: List[Dict[str, Any]], sort_config: dict) -> List[Dict[str, Any]]: + """Sort items based on sort configuration (standalone version for preview)""" + if not sort_config or not items: + return items + + field_path = sort_config.get('field') + order = sort_config.get('order', 'desc').lower() + + if not field_path: + return items + + # Helper to get nested values + def get_nested_value(data, path, default=''): + if not path or not data: + return default + parts = path.split('.') + value = data + for part in parts: + if isinstance(value, dict): + value = value.get(part) + elif isinstance(value, list): + try: + idx = int(part) + if 0 <= idx < len(value): + value = value[idx] + else: + return default + except (ValueError, TypeError): + return default + else: + return default + if value is None: + return default + return value if value is not None else default + + def get_sort_value(item): + """Get the sort value for an item""" + # Try raw data first + raw_data = item.get('raw', {}) + value = get_nested_value(raw_data, field_path, '') + + if not value and field_path.startswith('raw.'): + value = get_nested_value(raw_data, field_path[4:], '') + + if not value: + value = get_nested_value(item, field_path, '') + + # Handle Microsoft date format + if isinstance(value, str) and value.startswith('/Date('): + dt = self._parse_microsoft_date(value) + if dt: + return dt.timestamp() + + # Handle datetime objects + if isinstance(value, datetime): + return value.timestamp() + + # Handle numeric values + if isinstance(value, (int, float)): + return float(value) + + # Handle string timestamps + if isinstance(value, str): + # Try to parse as ISO format + try: + dt = datetime.fromisoformat(value.replace('Z', '+00:00')) + return dt.timestamp() + except ValueError: + pass + + # Try common date formats + for fmt in ['%Y-%m-%dT%H:%M:%S', '%Y-%m-%d %H:%M:%S', '%Y-%m-%d']: + try: + dt = datetime.strptime(value, fmt) + return dt.timestamp() + except ValueError: + continue + + # For strings, use lexicographic comparison + return str(value) + + # Sort items + try: + sorted_items = sorted(items, key=get_sort_value, reverse=(order == 'desc')) + return sorted_items + except Exception as e: + self.logger.warning(f"Error sorting items in preview: {e}") + return items + + def _format_feed_item(self, item: Dict[str, Any], format_str: str, feed_name: str = '') -> str: + """Format a feed item using the output format (standalone version)""" + import html + import re + from datetime import datetime, timezone + + # Extract field values + title = item.get('title', 'Untitled') + body = item.get('description', '') or item.get('body', '') + + # Clean HTML from body if present + if body: + body = html.unescape(body) + # Convert line break tags to newlines before stripping other HTML + # Handle
,
,
,
, etc. + body = re.sub(r'', '\n', body, flags=re.IGNORECASE) + # Convert paragraph tags to newlines (with spacing) + body = re.sub(r'

', '\n\n', body, flags=re.IGNORECASE) + body = re.sub(r']*>', '', body, flags=re.IGNORECASE) + # Remove remaining HTML tags + body = re.sub(r'<[^>]+>', '', body) + # Clean up whitespace (preserve intentional line breaks) + # Replace multiple newlines with double newline, then normalize spaces within lines + body = re.sub(r'\n\s*\n\s*\n+', '\n\n', body) # Multiple newlines -> double newline + lines = body.split('\n') + body = '\n'.join(' '.join(line.split()) for line in lines) # Normalize spaces per line + body = body.strip() + + link = item.get('link', '') + published = item.get('published') + + # Format timestamp + date_str = "" + if published: + try: + if published.tzinfo: + now = datetime.now(timezone.utc) + else: + now = datetime.now() + + diff = now - published + minutes = int(diff.total_seconds() / 60) + + if minutes < 1: + date_str = "now" + elif minutes < 60: + date_str = f"{minutes}m ago" + elif minutes < 1440: + hours = minutes // 60 + mins = minutes % 60 + date_str = f"{hours}h {mins}m ago" + else: + days = minutes // 1440 + date_str = f"{days}d ago" + except Exception: + pass + + # Choose emoji + emoji = "đŸ“ĸ" + feed_name_lower = feed_name.lower() + if 'emergency' in feed_name_lower or 'alert' in feed_name_lower: + emoji = "🚨" + elif 'warning' in feed_name_lower: + emoji = "⚠ī¸" + elif 'info' in feed_name_lower or 'news' in feed_name_lower: + emoji = "ℹī¸" + + # Build replacements + replacements = { + 'title': title, + 'body': body, + 'date': date_str, + 'link': link, + 'emoji': emoji + } + + # Get raw API data if available (for preview, we don't have raw data, so this will be empty) + raw_data = item.get('raw', {}) + + # Helper to get nested values + def get_nested_value(data, path, default=''): + if not path or not data: + return default + parts = path.split('.') + value = data + for part in parts: + if isinstance(value, dict): + value = value.get(part) + elif isinstance(value, list): + try: + idx = int(part) + if 0 <= idx < len(value): + value = value[idx] + else: + return default + except (ValueError, TypeError): + return default + else: + return default + if value is None: + return default + return value if value is not None else default + + # Apply shortening, parsing, and conditional functions + def apply_shortening(text: str, function: str) -> str: + if not text: + return "" + + if function.startswith('truncate:'): + try: + max_len = int(function.split(':', 1)[1]) + if len(text) <= max_len: + return text + return text[:max_len] + "..." + except (ValueError, IndexError): + return text + elif function.startswith('word_wrap:'): + try: + max_len = int(function.split(':', 1)[1]) + if len(text) <= max_len: + return text + truncated = text[:max_len] + last_space = truncated.rfind(' ') + if last_space > max_len * 0.7: + return truncated[:last_space] + "..." + return truncated + "..." + except (ValueError, IndexError): + return text + elif function.startswith('first_words:'): + try: + num_words = int(function.split(':', 1)[1]) + words = text.split() + if len(words) <= num_words: + return text + return ' '.join(words[:num_words]) + "..." + except (ValueError, IndexError): + return text + elif function.startswith('regex:'): + try: + # Parse regex pattern and optional group number + # Format: regex:pattern:group or regex:pattern + # Need to handle patterns that contain colons, so split from the right + remaining = function[6:] # Skip 'regex:' prefix + + # Try to find the last colon that's followed by a number (the group number) + # Look for pattern like :N at the end + last_colon_idx = remaining.rfind(':') + pattern = remaining + group_num = None + + if last_colon_idx > 0: + # Check if what's after the last colon is a number + potential_group = remaining[last_colon_idx + 1:] + if potential_group.isdigit(): + pattern = remaining[:last_colon_idx] + group_num = int(potential_group) + + if not pattern: + return text + + # Apply regex + match = re.search(pattern, text, re.IGNORECASE | re.DOTALL) + if match: + if group_num is not None: + # Use specified group (0 = whole match, 1 = first group, etc.) + if 0 <= group_num <= len(match.groups()): + return match.group(group_num) if group_num > 0 else match.group(0) + else: + # Use first capture group if available, otherwise whole match + if match.groups(): + return match.group(1) + else: + return match.group(0) + return "" # No match found + except (ValueError, IndexError, re.error) as e: + # Silently fail on regex errors in preview + return text + elif function.startswith('if_regex:'): + try: + # Parse: if_regex:pattern:then:else + # Split by ':' but need to handle regex patterns that contain ':' + parts = function[9:].split(':', 2) # Skip 'if_regex:' prefix, split into [pattern, then, else] + if len(parts) < 3: + return text + + pattern = parts[0] + then_value = parts[1] + else_value = parts[2] + + if not pattern: + return text + + # Check if pattern matches + match = re.search(pattern, text, re.IGNORECASE | re.DOTALL) + if match: + return then_value + else: + return else_value + except (ValueError, IndexError, re.error) as e: + # Silently fail on regex errors in preview + return text + elif function.startswith('switch:'): + try: + # Parse: switch:value1:result1:value2:result2:...:default + # Example: switch:highest:🔴:high:🟠:medium:🟡:low:âšĒ:âšĒ + parts = function[7:].split(':') # Skip 'switch:' prefix + if len(parts) < 2: + return text + + # Pairs of value:result, last one is default + text_lower = text.lower().strip() + for i in range(0, len(parts) - 1, 2): + if i + 1 < len(parts): + value = parts[i].lower() + result = parts[i + 1] + if text_lower == value: + return result + + # Return last part as default if no match + return parts[-1] if parts else text + except (ValueError, IndexError) as e: + # Silently fail on switch errors in preview + return text + elif function.startswith('regex_cond:'): + try: + # Parse: regex_cond:extract_pattern:check_pattern:then:group + parts = function[11:].split(':', 3) # Skip 'regex_cond:' prefix + if len(parts) < 4: + return text + + extract_pattern = parts[0] + check_pattern = parts[1] + then_value = parts[2] + else_group = int(parts[3]) if parts[3].isdigit() else 1 + + if not extract_pattern: + return text + + # Extract using extract_pattern + match = re.search(extract_pattern, text, re.IGNORECASE | re.DOTALL) + if match: + # Get the captured group + if match.groups(): + extracted = match.group(else_group) if else_group <= len(match.groups()) else match.group(1) + # Strip whitespace from extracted text + extracted = extracted.strip() + else: + extracted = match.group(0).strip() + + # Check if extracted text matches check_pattern (exact match or contains) + if check_pattern: + # Try exact match first, then substring match + if extracted.lower() == check_pattern.lower() or re.search(check_pattern, extracted, re.IGNORECASE): + return then_value + + return extracted + return "" # No match found + except (ValueError, IndexError, re.error) as e: + # Silently fail on regex errors in preview + return text + return text + + # Process format string + def replace_placeholder(match): + content = match.group(1) + if '|' in content: + field_name, function = content.split('|', 1) + field_name = field_name.strip() + function = function.strip() + + # Check if it's a raw field access + if field_name.startswith('raw.'): + value = str(get_nested_value(raw_data, field_name[4:], '')) + else: + value = replacements.get(field_name, '') + + return apply_shortening(value, function) + else: + field_name = content.strip() + + # Check if it's a raw field access + if field_name.startswith('raw.'): + value = get_nested_value(raw_data, field_name[4:], '') + if value is None: + return '' + elif isinstance(value, (dict, list)): + try: + import json + return json.dumps(value) + except Exception: + return str(value) + else: + return str(value) + else: + return replacements.get(field_name, '') + + message = re.sub(r'\{([^}]+)\}', replace_placeholder, format_str) + + # Final truncation (130 char limit) + max_length = 130 + if len(message) > max_length: + lines = message.split('\n') + if len(lines) > 1: + total_length = sum(len(line) + 1 for line in lines[:-1]) + remaining = max_length - total_length - 3 + if remaining > 20: + lines[-1] = lines[-1][:remaining] + "..." + message = '\n'.join(lines) + else: + message = message[:max_length - 3] + "..." + else: + message = message[:max_length - 3] + "..." + + return message + def _get_bot_uptime(self): """Get bot uptime in seconds from database""" try: @@ -1928,6 +3508,92 @@ class BotDataViewer: self.logger.debug(f"Could not get bot start time from database: {e}") return 0 + def _add_channel_for_web(self, channel_idx, channel_name, channel_key_hex=None): + """ + Add a channel by queuing it in the database for the bot to process + + Args: + channel_idx: Channel index (0-39) + channel_name: Channel name (with or without # prefix) + channel_key_hex: Optional hex key for custom channels (32 chars) + + Returns: + dict with 'success' and optional 'error' key + """ + try: + conn = self._get_db_connection() + cursor = conn.cursor() + + # Insert operation into queue + cursor.execute(''' + INSERT INTO channel_operations + (operation_type, channel_idx, channel_name, channel_key_hex, status) + VALUES (?, ?, ?, ?, 'pending') + ''', ('add', channel_idx, channel_name, channel_key_hex)) + + operation_id = cursor.lastrowid + conn.commit() + conn.close() + + self.logger.info(f"Queued channel add operation: {channel_name} at index {channel_idx} (operation_id: {operation_id})") + + # Return immediately with operation_id - let frontend poll for status + return { + 'success': True, + 'pending': True, + 'operation_id': operation_id, + 'message': 'Channel operation queued successfully' + } + + except Exception as e: + self.logger.error(f"Error in _add_channel_for_web: {e}") + return { + 'success': False, + 'error': str(e) + } + + def _remove_channel_for_web(self, channel_idx): + """ + Remove a channel by queuing it in the database for the bot to process + + Args: + channel_idx: Channel index to remove + + Returns: + dict with 'success' and optional 'error' key + """ + try: + conn = self._get_db_connection() + cursor = conn.cursor() + + # Insert operation into queue + cursor.execute(''' + INSERT INTO channel_operations + (operation_type, channel_idx, status) + VALUES (?, ?, 'pending') + ''', ('remove', channel_idx)) + + operation_id = cursor.lastrowid + conn.commit() + conn.close() + + self.logger.info(f"Queued channel remove operation: index {channel_idx} (operation_id: {operation_id})") + + # Return immediately with operation_id - let frontend poll for status + return { + 'success': True, + 'pending': True, + 'operation_id': operation_id, + 'message': 'Channel operation queued successfully' + } + + except Exception as e: + self.logger.error(f"Error in _remove_channel_for_web: {e}") + return { + 'success': False, + 'error': str(e) + } + def run(self, host='127.0.0.1', port=8080, debug=False): """Run the modern web viewer""" self.logger.info(f"Starting modern web viewer on {host}:{port}") diff --git a/modules/web_viewer/templates/base.html b/modules/web_viewer/templates/base.html index f11409a..16559f0 100644 --- a/modules/web_viewer/templates/base.html +++ b/modules/web_viewer/templates/base.html @@ -146,11 +146,25 @@ Cache + {% if greeter_enabled %}
  • Greeter
    • + {% endif %} + {% if feed_manager_enabled %} +
  • + + Feeds + +
    • + {% endif %} +
  • + + Radio + +
    • diff --git a/modules/web_viewer/templates/feeds.html b/modules/web_viewer/templates/feeds.html new file mode 100644 index 0000000..0c2b73b --- /dev/null +++ b/modules/web_viewer/templates/feeds.html @@ -0,0 +1,848 @@ +{% extends "base.html" %} + +{% block title %}Feed Management - MeshCore Bot{% endblock %} + +{% block content %} +
    +
    +
    +

    + Feed Management +

    + + +
    +
    +
    +
    +
    Total Subscriptions
    +

    -

    +
    +
    +
    +
    +
    +
    +
    Active Feeds
    +

    -

    +
    +
    +
    +
    +
    +
    +
    Items (24h)
    +

    -

    +
    +
    +
    +
    +
    +
    +
    Active Errors
    +

    -

    +
    +
    +
    +
    + + +
    +
    +
    Feed Subscriptions
    + +
    +
    +
    + + + + + + + + + + + + + + + + + + + +
    IDNameTypeChannelStatusLast CheckItemsErrorsActions
    Loading...
    +
    +
    +
    +
    +
    +
    + + + + + + +{% endblock %} + +{% block extra_js %} + +{% endblock %} + diff --git a/modules/web_viewer/templates/radio.html b/modules/web_viewer/templates/radio.html new file mode 100644 index 0000000..805ccc8 --- /dev/null +++ b/modules/web_viewer/templates/radio.html @@ -0,0 +1,718 @@ +{% extends "base.html" %} + +{% block title %}Radio Settings - MeshCore Bot{% endblock %} + +{% block content %} +
    +
    +
    +

    + Radio Settings +

    + + +
    +
    +
    +
    +
    Total Channels
    +

    -

    +
    +
    +
    +
    +
    +
    +
    Available Slots
    +

    -

    +
    +
    +
    +
    +
    +
    +
    Channels with Feeds
    +

    -

    +
    +
    +
    +
    +
    +
    +
    Empty Channels
    +

    -

    +
    +
    +
    +
    + + +
    +
    +
    Channel Management
    + +
    +
    +
    + + + + + + + + + + + + + + + + +
    IndexNameTypeFeed CountStatusActions
    Loading...
    +
    +
    +
    +
    +
    +
    + + + + + + +{% endblock %} + +{% block extra_js %} + +{% endblock %} + diff --git a/requirements.txt b/requirements.txt index 2086743..d5bf6a3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,3 +18,4 @@ retry-requests>=1.0.0 flask>=2.3.0 flask-socketio>=5.3.0 meshcore-cli +feedparser>=6.0.10