From c151fba7a54bca222888faa87bda13c76d056367 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 6 Dec 2025 10:18:12 -0800 Subject: [PATCH] Enhance configuration and database management for feed and channel operations. Add channel refresh interval setting to config.ini.example, implement feed manager functionality with new database tables for feed subscriptions and activity tracking, and improve greeter command logic with human greeting detection and Levenshtein distance matching for user greetings. Update web viewer to include feed management pages and integrate global template variables for better user experience. --- config.ini.example | 215 ++- docs/FEEDS.md | 413 ++++++ modules/channel_manager.py | 414 ++++++ modules/command_manager.py | 71 +- modules/commands/feed_command.py | 417 ++++++ modules/commands/greeter_command.py | 383 ++++- modules/core.py | 18 + modules/db_manager.py | 123 ++ modules/feed_manager.py | 1252 +++++++++++++++++ modules/message_handler.py | 43 +- modules/scheduler.py | 185 +++ modules/web_viewer/app.py | 1690 ++++++++++++++++++++++- modules/web_viewer/templates/base.html | 14 + modules/web_viewer/templates/feeds.html | 848 ++++++++++++ modules/web_viewer/templates/radio.html | 718 ++++++++++ requirements.txt | 1 + 16 files changed, 6661 insertions(+), 144 deletions(-) create mode 100644 docs/FEEDS.md create mode 100644 modules/commands/feed_command.py create mode 100644 modules/feed_manager.py create mode 100644 modules/web_viewer/templates/feeds.html create mode 100644 modules/web_viewer/templates/radio.html diff --git a/config.ini.example b/config.ini.example index 638ed83..88d0ba6 100644 --- a/config.ini.example +++ b/config.ini.example @@ -90,6 +90,12 @@ bot_longitude = -74.0060 # Set to a lower value if you want to limit channel fetching for performance max_channels = 12 +# Channel refresh interval in seconds +# How often to refresh channel list from device to prevent stale data in database +# Default: 3600 (1 hour). Set to 0 to disable periodic refresh (channels only refreshed on startup) +# This ensures the database stays in sync if channels are changed on the device directly +channel_refresh_interval_seconds = 3600 + # Interval-based advertising settings # Send periodic flood adverts at specified intervals # 0: Disabled (default) @@ -254,73 +260,6 @@ colored_output = true # Options: DEBUG, INFO, WARNING, ERROR, CRITICAL meshcore_log_level = INFO -[Greeter_Command] -# Enable or disable the greeter command -# true: Bot will greet users on their first public channel message -# false: Greeter is disabled -enabled = false - -# Channels where greetings should occur (comma-separated) -# IMPORTANT: Leave commented out (or omit entirely) to use global monitor_channels (default behavior) -# If uncommented with empty value (channels = ), command will be DM-only -# Comma-separated list to restrict to specific channels (only greeter command works there) -# Example: channels = general,welcome,newbies -# If not specified, uses the channels from [Channels] monitor_channels setting -# channels = - -# Greeting message template (default for all channels) -# Available fields: {sender} - the user's name/ID -# For multi-part greetings, separate messages with pipe (|) -# Example (single): "Welcome to the mesh, @[{sender}]!" -# Example (multi-part): "Welcome to the mesh, @[{sender}]!|This is a great place to chat.|Use !help for commands." -greeting_message = Welcome to the mesh, @[{sender}]! - -# Channel-specific greeting messages (optional) -# Format: channel_name:greeting_message,channel_name2:greeting_message2 -# If a channel has a specific greeting, it will be used instead of the default greeting_message -# Example: Public:Welcome to Public channel, @[{sender}]!|general:Welcome to general, @[{sender}]! -# Multi-part greetings are supported per channel using pipe (|) separator -# Leave empty to use greeting_message for all channels -channel_greetings = - -# Per-channel greetings (tracking behavior) -# false: Greet each user only once globally (default - user gets one greeting total) -# true: Greet each user once per channel (user can be greeted on each channel separately) -# Note: This controls tracking, not the greeting message itself. Use channel_greetings for different messages. -per_channel_greetings = false - -# Include mesh network information in greeting -# true: Add mesh statistics to greeting (total contacts, repeaters, etc.) -# false: Only send the greeting message -include_mesh_info = true - -# Mesh info format template -# Available fields: {total_contacts}, {repeaters}, {companions}, {recent_activity_24h} -# Example: "\n\nMesh Info: {total_contacts} contacts, {repeaters} repeaters" -# Note: Mesh info is appended to the last greeting message part -mesh_info_format = \n\nMesh Info: {total_contacts} contacts, {repeaters} repeaters, {recent_activity_24h} active in last 24h - -# Rollout period in days -# When greeter is first enabled on an active mesh, this sets how many days -# to listen and mark all active users as already greeted before beginning -# to greet new users. This prevents greeting everyone on an established mesh. -# Set to 0 to disable rollout (will greet all new users immediately) -# Note: Use auto_backfill to mark historical users and shorten/eliminate rollout period -rollout_days = 7 - -# Auto-backfill from historical message_stats data -# true: Automatically mark all users who have posted on public channels in the past -# false: Only mark users during rollout period (default) -# This allows shortening or eliminating the rollout period by using existing data -auto_backfill = false - -# Backfill lookback period in days -# Number of days to look back when auto-backfilling (0 = all time) -# Only used if auto_backfill = true -# Example: 30 = only mark users who posted in last 30 days -# Example: 0 = mark all users who have ever posted (all time) -backfill_lookback_days = 30 - [Custom_Syntax] # Custom syntax patterns for special message formats # Format: pattern = "response_format" @@ -372,6 +311,55 @@ repeater_prefix_api_url = # Recommended: 1-6 hours (data doesn't change frequently) repeater_prefix_cache_hours = 1 +[Feed_Manager] +# Enable feed manager functionality +# true: Feed manager is enabled and will poll feeds +# false: Feed manager is disabled +feed_manager_enabled = false + +# Default check interval in seconds (5 minutes) +# How often to check feeds for new items +# Individual feeds can override this with their own interval +default_check_interval_seconds = 300 + +# Maximum number of items to process per feed per check +# Prevents overwhelming channels with too many items at once +max_items_per_check = 10 + +# Default output format for feed messages +# Placeholders: {title}, {body}, {date}, {link}, {emoji} +# Shortening functions: {field|truncate:N}, {field|word_wrap:N}, {field|first_words:N} +# Example: "{emoji} {body|truncate:100} - {date}\n{link|truncate:50}" +# Default uses body instead of title for main content +default_output_format = {emoji} {body|truncate:100} - {date}\n{link|truncate:50} + +# Default message send interval in seconds +# How long to wait between sending queued messages from the same feed +# Prevents rate limiting by spacing out message sends +# Individual feeds can override this with their own interval +default_message_send_interval_seconds = 2.0 + +# Request timeout in seconds +# Maximum time to wait for feed requests +feed_request_timeout = 30 + +# User-Agent string for HTTP requests +# Identifies the bot when fetching feeds +feed_user_agent = MeshCoreBot/1.0 FeedManager + +# Rate limiting: minimum seconds between requests to same domain +# Prevents overwhelming feed sources +feed_rate_limit_seconds = 5 + +# Maximum message length (mesh limit is 130) +# Messages longer than this will be truncated +max_message_length = 130 + +# Enable/disable feed command +# true: Feed command is available (admin only) +# false: Feed command is disabled +feed_command_enabled = true + [Prefix_Command] # Enable or disable repeater geolocation in prefix command # true: Show city names with repeaters when location data is available @@ -655,4 +643,95 @@ debug = false # Auto-start web viewer with bot # true: Start web viewer automatically when bot starts # false: Start web viewer manually -auto_start = false \ No newline at end of file +auto_start = false + +[Greeter_Command] +# Enable or disable the greeter command +# true: Bot will greet users on their first public channel message +# false: Greeter is disabled +enabled = false + +# Channels where greetings should occur (comma-separated) +# IMPORTANT: Leave commented out (or omit entirely) to use global monitor_channels (default behavior) +# If uncommented with empty value (channels = ), command will be DM-only +# Comma-separated list to restrict to specific channels (only greeter command works there) +# Example: channels = general,welcome,newbies +# If not specified, uses the channels from [Channels] monitor_channels setting +# channels = + +# Greeting message template (default for all channels) +# Available fields: {sender} - the user's name/ID +# For multi-part greetings, separate messages with pipe (|) +# Example (single): "Welcome to the mesh, @[{sender}]!" +# Example (multi-part): "Welcome to the mesh, @[{sender}]!|This is a great place to chat.|Use !help for commands." +greeting_message = Welcome to the mesh, @[{sender}]! + +# Channel-specific greeting messages (optional) +# Format: channel_name:greeting_message,channel_name2:greeting_message2 +# If a channel has a specific greeting, it will be used instead of the default greeting_message +# Example: Public:Welcome to Public channel, @[{sender}]!|general:Welcome to general, @[{sender}]! +# Multi-part greetings are supported per channel using pipe (|) separator +# Leave empty to use greeting_message for all channels +channel_greetings = + +# Per-channel greetings (tracking behavior) +# false: Greet each user only once globally (default - user gets one greeting total) +# true: Greet each user once per channel (user can be greeted on each channel separately) +# Note: This controls tracking, not the greeting message itself. Use channel_greetings for different messages. +per_channel_greetings = false + +# Include mesh network information in greeting +# true: Add mesh statistics to greeting (total contacts, repeaters, etc.) +# false: Only send the greeting message +include_mesh_info = true + +# Mesh info format template +# Available fields: {total_contacts}, {repeaters}, {companions}, {recent_activity_24h} +# Example: "\n\nMesh Info: {total_contacts} contacts, {repeaters} repeaters" +# Note: Mesh info is appended to the last greeting message part +mesh_info_format = \n\nMesh Info: {total_contacts} contacts, {repeaters} repeaters, {recent_activity_24h} active in last 24h + +# Rollout period in days +# When greeter is first enabled on an active mesh, this sets how many days +# to listen and mark all active users as already greeted before beginning +# to greet new users. This prevents greeting everyone on an established mesh. +# Set to 0 to disable rollout (will greet all new users immediately) +# Note: Use auto_backfill to mark historical users and shorten/eliminate rollout period +rollout_days = 7 + +# Auto-backfill from historical message_stats data +# true: Automatically mark all users who have posted on public channels in the past +# false: Only mark users during rollout period (default) +# This allows shortening or eliminating the rollout period by using existing data +auto_backfill = false + +# Backfill lookback period in days +# Number of days to look back when auto-backfilling (0 = all time) +# Only used if auto_backfill = true +# Example: 30 = only mark users who posted in last 30 days +# Example: 0 = mark all users who have ever posted (all time) +backfill_lookback_days = 30 + +# Dead air delay in seconds +# Wait this many seconds before sending a greeting to a new user +# This allows time for other users to greet the new user first +# Set to 0 to send greetings immediately (default behavior) +# Example: 30 = wait 30 seconds before greeting +dead_air_delay_seconds = 0 + +# Defer to human greeting +# If enabled and dead_air_delay_seconds > 0, the bot will not send a greeting +# if another user mentions the new user's name within the dead air delay period +# true: Check for human greetings and defer if found +# false: Always send bot greeting after delay (default) +# Note: Only effective when dead_air_delay_seconds > 0 +defer_to_human_greeting = false + +# Levenshtein distance for name matching +# Maximum edit distance allowed when checking if a user has been greeted before +# Prevents duplicate greetings for users who make small changes to their name +# Set to 0 to disable fuzzy matching (exact name match only, default) +# Example: 2 = "John" and "Jon" (distance 1) or "John" and "Jhon" (distance 1) would match +# Example: 3 = "Alice" and "Alicia" (distance 2) would match +# Recommended: 1-3 for most use cases +levenshtein_distance = 0 \ No newline at end of file diff --git a/docs/FEEDS.md b/docs/FEEDS.md new file mode 100644 index 0000000..4b7c12f --- /dev/null +++ b/docs/FEEDS.md @@ -0,0 +1,413 @@ +# Feed Management + +The Feed Management system allows the bot to subscribe to RSS feeds and REST APIs, automatically polling for new content and posting updates to specified mesh channels. + +## Overview + +The feed manager supports two feed types: +- **RSS Feeds**: Standard RSS/Atom feeds +- **API Feeds**: REST API endpoints returning JSON data + +Both feed types support: +- Configurable polling intervals +- Custom message formatting +- Item filtering +- Sorting +- Automatic deduplication +- Rate limiting + +## Configuration + +### Global Settings + +Configure feed manager behavior in `config.ini`: + +```ini +[Feed_Manager] +# Enable/disable feed manager +feed_manager_enabled = true + +# Default check interval (seconds) +default_check_interval_seconds = 300 + +# Maximum items to process per check +max_items_per_check = 10 + +# HTTP request timeout (seconds) +feed_request_timeout = 30 + +# User agent for HTTP requests +feed_user_agent = MeshCoreBot/1.0 FeedManager + +# Rate limit between requests to same domain (seconds) +feed_rate_limit_seconds = 5.0 + +# Maximum message length (characters) +max_message_length = 130 + +# Default output format +default_output_format = {emoji} {body|truncate:100} - {date}\n{link|truncate:50} + +# Default interval between sending queued messages (seconds) +default_message_send_interval_seconds = 2.0 +``` + +## RSS Feed Configuration + +The web interface provides separate input fields for each configuration option. Below are examples showing the values to enter in each field. + +### Basic RSS Feed + +**Feed Type:** `rss` +**Feed URL:** `https://example.com/rss.xml` +**Channel:** `#alerts` +**Feed Name (Optional):** `Example RSS Feed` +**Check Interval (seconds):** `300` +**Output Format:** (leave empty to use default) +**Message Send Interval (seconds):** `2.0` +**Filter Configuration:** (leave empty) +**Sort Configuration:** (leave empty) + +### RSS Feed with Custom Format + +**Feed Type:** `rss` +**Feed URL:** `https://example.com/rss.xml` +**Channel:** `#alerts` +**Feed Name (Optional):** `Emergency Alerts` +**Check Interval (seconds):** `60` +**Output Format:** +``` +{emoji} {title|truncate:80} +{body|truncate:100} +{date} +``` +**Message Send Interval (seconds):** `2.0` +**Filter Configuration:** (leave empty) +**Sort Configuration:** (leave empty) + +## API Feed Configuration + +### Basic API Feed + +**Feed Type:** `api` +**Feed URL:** `https://api.example.com/alerts` +**Channel:** `#alerts` +**Feed Name (Optional):** `API Alerts` +**Check Interval (seconds):** `300` +**Output Format:** (leave empty to use default) +**Message Send Interval (seconds):** `2.0` +**API Configuration (JSON):** +```json +{ + "method": "GET", + "headers": {}, + "params": { + "api_key": "your-api-key" + }, + "response_parser": { + "items_path": "data.alerts", + "id_field": "id", + "title_field": "title", + "description_field": "description", + "timestamp_field": "created_at" + } +} +``` +**Filter Configuration:** (leave empty) +**Sort Configuration:** (leave empty) + +### WSDOT Highway Alerts Example + +**Feed Type:** `api` +**Feed URL:** `https://wsdot.wa.gov/Traffic/api/HighwayAlerts/HighwayAlertsREST.svc/GetAlertsAsJson` +**Channel:** `#traffic` +**Feed Name (Optional):** `WSDOT Highway Alerts` +**Check Interval (seconds):** `300` +**Output Format:** +``` +{emoji} [{raw.Priority|switch:highest:🔴:high:🟠:medium:🟡:âšĒ}] {title|truncate:80} +{raw.EventCategory} | {raw.Region} | {raw.EventStatus} +{body|truncate:70} +``` +**Message Send Interval (seconds):** `2.0` +**API Configuration (JSON):** +```json +{ + "method": "GET", + "headers": {}, + "params": { + "AccessCode": "your-access-code" + }, + "response_parser": { + "items_path": "", + "id_field": "AlertID", + "title_field": "HeadlineDescription", + "description_field": "ExtendedDescription", + "timestamp_field": "LastUpdatedTime" + } +} +``` +**Filter Configuration (JSON):** +```json +{ + "conditions": [ + { + "field": "raw.EventCategory", + "operator": "in", + "values": ["Alert", "Closure"] + } + ], + "logic": "OR" +} +``` +**Sort Configuration (JSON):** +```json +{ + "field": "raw.LastUpdatedTime", + "order": "desc" +} +``` + +## Output Format + +The output format string controls how feed items are formatted before sending to channels. + +### Placeholders + +- `{title}` - Item title +- `{body}` - Item description/body text +- `{date}` - Relative time (e.g., "5m ago", "2h 30m ago") +- `{link}` - Item URL +- `{emoji}` - Auto-selected emoji based on feed name (đŸ“ĸ, 🚨, ⚠ī¸, ℹī¸) +- `{raw.field}` - Access raw API data fields (API feeds only) +- `{raw.nested.field}` - Access nested API fields (e.g., `{raw.StartRoadwayLocation.RoadName}`) + +### Shortening Functions + +Apply functions to placeholders using the pipe operator: + +- `{field|truncate:N}` - Truncate to N characters +- `{field|word_wrap:N}` - Wrap at N characters, breaking at word boundaries +- `{field|first_words:N}` - Take first N words + +**Examples:** +``` +{title|truncate:60} +{body|word_wrap:100} +{body|first_words:20} +``` + +### Regex Extraction + +Extract specific content using regex patterns: + +- `{field|regex:pattern}` - Extract using regex (uses first capture group) +- `{field|regex:pattern:group}` - Extract specific capture group (0 = whole match, 1 = first group, etc.) + +**Examples:** +``` +{body|regex:Temperature:\s*([^\n]+):1} +{body|regex:Conditions:\s*([^\n]+):1} +``` + +### Conditional Formatting + +- `{field|if_regex:pattern:then:else}` - If pattern matches, return "then", else return "else" +- `{field|switch:value1:result1:value2:result2:...:default}` - Multi-value conditional + +**Examples:** +``` +{raw.Priority|switch:highest:🔴:high:🟠:medium:🟡:âšĒ} +{body|if_regex:No restrictions:👍:Restrictions apply} +``` + +### Extract and Check + +- `{field|regex_cond:extract_pattern:check_pattern:then:group}` - Extract text, check if it matches pattern, return "then" if match, else return extracted text + +**Example:** +``` +{body|regex_cond:Northbound\s*\n([^\n]+):No restrictions:👍:1} +``` + +## Filter Configuration + +Filter configuration determines which items are sent to channels. + +### Filter Structure + +```json +{ + "conditions": [ + { + "field": "raw.Priority", + "operator": "in", + "values": ["highest", "high"] + }, + { + "field": "raw.EventStatus", + "operator": "equals", + "value": "open" + } + ], + "logic": "AND" +} +``` + +### Operators + +- `equals` - Exact match +- `not_equals` - Not equal +- `in` - Value in list +- `not_in` - Value not in list +- `matches` - Regex match +- `not_matches` - Regex does not match +- `contains` - String contains value +- `not_contains` - String does not contain value + +### Logic + +- `AND` - All conditions must match (default) +- `OR` - Any condition matches + +### Field Paths + +For API feeds, use `raw.field` or `raw.nested.field` to access API response fields: +- `raw.Priority` +- `raw.EventStatus` +- `raw.StartRoadwayLocation.RoadName` + +## Sort Configuration + +Sort items before processing: + +```json +{ + "field": "raw.LastUpdatedTime", + "order": "desc" +} +``` + +### Sort Options + +- `field` - Field path to sort by (e.g., `raw.LastUpdatedTime`, `raw.Priority`, `published`) +- `order` - `asc` (ascending) or `desc` (descending) + +### Date Format Support + +The sort function supports: +- ISO format dates +- Microsoft JSON date format: `/Date(timestamp-offset)/` (e.g., WSDOT API) +- Unix timestamps +- Common date string formats + +## Message Queuing + +Messages are queued and sent at configured intervals to prevent rate limiting: + +- `message_send_interval_seconds` - Time between sending messages from the same feed (default: 2.0 seconds) +- Messages are automatically queued and processed in order +- Each feed maintains its own send interval + +## Deduplication + +The system automatically prevents duplicate posts: + +- Items are tracked by ID in the database +- Previously processed items are skipped +- Works correctly even when sorting changes item order +- Database-backed deduplication ensures reliability across restarts + +## Rate Limiting + +The feed manager implements rate limiting: + +- Per-domain rate limiting (default: 5 seconds between requests to same domain) +- Configurable via `feed_rate_limit_seconds` +- Prevents overwhelming feed sources + +## Web Interface + +The feed management system includes a web interface accessible at `/feeds`: + +- View all feed subscriptions +- Add/edit/delete feeds +- Preview output format with live feed data +- View feed statistics and activity +- Monitor errors + +## Command Interface + +Feeds can be managed via mesh commands. The feed command requires admin access and must be sent as a direct message (DM) to the bot. The command is enabled by default. + +**Command Format:** `feed [arguments]` (DM only) + +### Available Commands + +- `feed subscribe [name] [api_config]` - Subscribe to a feed +- `feed unsubscribe [channel]` - Unsubscribe from a feed (by ID or URL) +- `feed list [channel]` - List all feed subscriptions (optionally filtered by channel) +- `feed status ` - Show detailed status for a feed +- `feed enable ` - Enable a feed subscription +- `feed disable ` - Disable a feed subscription +- `feed update [interval_seconds]` - Update feed settings +- `feed test ` - Test/validate a feed URL + +### Examples + +``` +feed subscribe rss https://alerts.example.com/rss emergency "Emergency Alerts" +feed subscribe api https://api.example.com/alerts emergency "API Alerts" '{"headers": {"Authorization": "Bearer TOKEN"}}' +feed list +feed list #alerts +feed status 1 +feed enable 1 +feed disable 1 +feed unsubscribe 1 +feed update 1 60 +``` + +**Note:** The feed command requires admin access. API feeds require JSON configuration as the last argument when subscribing. + +## Best Practices + +1. **Check Intervals**: Set appropriate intervals based on feed update frequency (60-300 seconds typical) + +2. **Message Formatting**: Keep messages under 130 characters for mesh compatibility + +3. **Filtering**: Use filters to reduce noise and only send relevant items + +4. **Rate Limiting**: Respect feed source rate limits by configuring appropriate intervals + +5. **Error Handling**: Monitor feed errors in the web interface and adjust configuration as needed + +6. **Testing**: Use the preview feature in the web interface to test output formats before enabling feeds + +## Troubleshooting + +### Feeds Not Polling + +- Verify `feed_manager_enabled = true` in config +- Check that feeds are enabled in the database +- Review bot logs for errors +- Ensure bot is connected to mesh network + +### Items Not Appearing + +- Check filter configuration - items may be filtered out +- Verify output format is correct +- Check feed activity log in web interface +- Review error log for parsing issues + +### Duplicate Messages + +- Deduplication is automatic - check if item IDs are changing +- Verify `last_item_id` is being updated correctly +- Check database for processed items + +### Rate Limiting Issues + +- Increase `feed_rate_limit_seconds` in config +- Increase `message_send_interval_seconds` for specific feeds +- Reduce `check_interval_seconds` to poll less frequently + diff --git a/modules/channel_manager.py b/modules/channel_manager.py index ef0d3bb..a9c7323 100644 --- a/modules/channel_manager.py +++ b/modules/channel_manager.py @@ -7,6 +7,7 @@ Handles efficient concurrent channel fetching with caching import asyncio import sys import os +import hashlib from typing import Dict, Any, List, Optional from meshcore import EventType @@ -104,8 +105,70 @@ class ChannelManager: # Update the bot's meshcore channels for compatibility self.bot.meshcore.channels = self._channels_cache + # Store channels in database for web viewer access + self._store_channels_in_db(valid_channels) + return valid_channels + def _store_channels_in_db(self, channels: List[Dict[str, Any]]): + """Store channel information in database for web viewer access (full refresh - clears all first)""" + try: + import sqlite3 + db_path = self.bot.db_manager.db_path + + with sqlite3.connect(db_path) as conn: + cursor = conn.cursor() + + # Clear existing channels (full refresh) + cursor.execute('DELETE FROM channels') + + # Insert all channels + for channel in channels: + self._insert_channel_in_db(cursor, channel) + + conn.commit() + self.logger.debug(f"Stored {len(channels)} channels in database (full refresh)") + except Exception as e: + self.logger.warning(f"Failed to store channels in database: {e}") + + def _store_single_channel_in_db(self, channel: Dict[str, Any]): + """Store or update a single channel in database (without clearing others)""" + try: + import sqlite3 + db_path = self.bot.db_manager.db_path + + with sqlite3.connect(db_path) as conn: + cursor = conn.cursor() + self._insert_channel_in_db(cursor, channel) + conn.commit() + self.logger.debug(f"Stored/updated channel {channel.get('channel_idx')} in database") + except Exception as e: + self.logger.warning(f"Failed to store single channel in database: {e}") + + def _insert_channel_in_db(self, cursor, channel: Dict[str, Any]): + """Helper method to insert/update a single channel in database""" + channel_idx = channel.get('channel_idx') + channel_name = channel.get('channel_name', '') + channel_key_hex = channel.get('channel_key_hex', '') + + # Determine channel type based on key derivation + # If key matches hashtag derivation, it's a hashtag channel + channel_type = 'hashtag' # Default assumption + if channel_name and channel_key_hex: + # Check if key matches hashtag derivation + expected_key = self.generate_hashtag_key(channel_name) + if expected_key.hex() == channel_key_hex: + channel_type = 'hashtag' + else: + channel_type = 'custom' + + if channel_name: # Only store non-empty channels + cursor.execute(''' + INSERT OR REPLACE INTO channels + (channel_idx, channel_name, channel_type, channel_key_hex, last_updated) + VALUES (?, ?, ?, ?, CURRENT_TIMESTAMP) + ''', (channel_idx, channel_name, channel_type, channel_key_hex)) + async def _fetch_single_channel(self, channel_idx: int) -> Optional[Dict[str, Any]]: """ Fetch a single channel with error handling @@ -293,3 +356,354 @@ class ChannelManager: """Invalidate the channels cache""" self._cache_valid = False self.logger.debug("Channels cache invalidated") + + @staticmethod + def generate_hashtag_key(channel_name: str) -> bytes: + """ + Generate a hashtag channel key from the channel name + + The key is the first 16 bytes of the SHA256 hash of the channel name + (including the # symbol), converted to lowercase. + + Args: + channel_name: The channel name (e.g., "#general" or "general") + + Returns: + 16-byte key for the hashtag channel + """ + # Ensure channel name starts with # and is lowercase + if not channel_name.startswith('#'): + channel_name = '#' + channel_name + channel_name_lower = channel_name.lower() + + # Compute SHA256 hash + hash_obj = hashlib.sha256(channel_name_lower.encode('utf-8')) + hash_bytes = hash_obj.digest() + + # Take first 16 bytes + return hash_bytes[:16] + + async def add_hashtag_channel(self, channel_idx: int, channel_name: str) -> bool: + """ + Add or update a hashtag channel on the radio + + Hashtag channels use publicly derivable keys based on the channel name. + The firmware automatically generates the key when the channel name starts with #. + + Args: + channel_idx: The channel index (0-39) + channel_name: The name of the channel (with or without # prefix) + + Returns: + True if successful, False otherwise + """ + # Ensure channel name has # prefix for consistency + if not channel_name.startswith('#'): + channel_name = '#' + channel_name + + self.logger.info(f"Adding hashtag channel {channel_idx}: {channel_name}") + + # Use the simplified add_channel method - firmware will auto-generate key + return await self.add_channel(channel_idx, channel_name) + + async def add_channel(self, channel_idx: int, channel_name: str, channel_secret: Optional[bytes] = None, channel_secret_hex: Optional[str] = None) -> bool: + """ + Add or update a channel on the radio + + For hashtag channels (name starts with #), the firmware automatically generates the key. + For custom channels, provide either channel_secret (bytes) or channel_secret_hex (hex string). + + Args: + channel_idx: The channel index (0-39) + channel_name: The name of the channel + channel_secret: Optional 16-byte encryption key for custom channels + channel_secret_hex: Optional hex string (32 chars) for the encryption key. Takes precedence over channel_secret. + + Returns: + True if successful, False otherwise + """ + if not self.bot.connected or not self.bot.meshcore: + self.logger.error("Not connected to MeshCore node") + return False + + if channel_idx < 0 or channel_idx >= self.max_channels: + self.logger.error(f"Channel index {channel_idx} out of range (0-{self.max_channels-1})") + return False + + try: + # Check if this is a hashtag channel (firmware auto-generates key) + is_hashtag = channel_name.startswith('#') + + # For custom channels, validate and prepare the key + if not is_hashtag: + if channel_secret_hex: + # Validate hex string + if len(channel_secret_hex) != 32: + self.logger.error(f"Channel secret hex must be exactly 32 characters (16 bytes), got {len(channel_secret_hex)}") + return False + try: + channel_secret = bytes.fromhex(channel_secret_hex) + except ValueError as e: + self.logger.error(f"Invalid hex string for channel secret: {e}") + return False + elif channel_secret is None: + self.logger.error("Custom channel requires a channel key (channel_secret or channel_secret_hex)") + return False + elif len(channel_secret) != 16: + self.logger.error(f"Channel secret must be exactly 16 bytes, got {len(channel_secret)}") + return False + + self.logger.info(f"Adding custom channel {channel_idx}: {channel_name} (key: {channel_secret.hex()[:8]}...)") + else: + self.logger.info(f"Adding hashtag channel {channel_idx}: {channel_name} (firmware will auto-generate key)") + + # Use meshcore.commands.set_channel API directly + if hasattr(self.bot.meshcore, 'commands') and hasattr(self.bot.meshcore.commands, 'set_channel'): + # For hashtag channels, just pass the name (firmware generates key) + if is_hashtag: + res = await self.bot.meshcore.commands.set_channel(channel_idx, channel_name) + else: + # For custom channels, we need to pass the key + # Check if set_channel accepts a key parameter + # Try with key as third parameter + try: + res = await self.bot.meshcore.commands.set_channel(channel_idx, channel_name, channel_secret) + except TypeError: + # If that doesn't work, try with hex string + try: + res = await self.bot.meshcore.commands.set_channel(channel_idx, channel_name, channel_secret_hex or channel_secret.hex()) + except TypeError: + # Fallback to CLI method if API doesn't support key parameter + self.logger.warning("meshcore.commands.set_channel doesn't accept key parameter, using CLI fallback") + return await self._add_channel_via_cli(channel_idx, channel_name, channel_secret.hex() if channel_secret else channel_secret_hex) + + # Check for errors + if hasattr(res, 'type') and res.type == EventType.ERROR: + self.logger.error(f"Failed to set channel {channel_idx}: {res.payload if hasattr(res, 'payload') else 'Unknown error'}") + return False + + # Fetch the channel back to get the generated key and verify + res = await self.bot.meshcore.commands.get_channel(channel_idx) + + if hasattr(res, 'type') and res.type == EventType.ERROR: + self.logger.error(f"Failed to get channel {channel_idx} after setting: {res.payload if hasattr(res, 'payload') else 'Unknown error'}") + return False + + # Extract channel info from response + if hasattr(res, 'payload'): + channel_info = res.payload + else: + # Fallback: try to get from event subscription + channel_info = await self._fetch_single_channel(channel_idx) + if not channel_info: + self.logger.error(f"Could not retrieve channel {channel_idx} after setting") + return False + + # Verify channel was set correctly + if channel_info.get('channel_name') != channel_name: + self.logger.error(f"Channel name mismatch: expected {channel_name}, got {channel_info.get('channel_name')}") + return False + + # For custom channels, verify the key matches + if not is_hashtag: + channel_secret_from_device = channel_info.get('channel_secret', b'') + if isinstance(channel_secret_from_device, bytes) and channel_secret_from_device != channel_secret: + self.logger.error(f"Channel key mismatch for custom channel {channel_idx}") + return False + + # Update cache and database + channel_info['channel_key_hex'] = channel_info.get('channel_secret', b'').hex() if isinstance(channel_info.get('channel_secret'), bytes) else '' + self._channels_cache[channel_idx] = channel_info + self._store_single_channel_in_db(channel_info) + + self.logger.info(f"Successfully added channel {channel_idx}: {channel_name}") + return True + else: + # Fallback to CLI method if commands API not available + self.logger.warning("meshcore.commands.set_channel not available, using CLI fallback") + channel_secret_hex = channel_secret.hex() if channel_secret else channel_secret_hex + if is_hashtag: + # For hashtag, generate key ourselves as fallback + channel_secret = self.generate_hashtag_key(channel_name) + channel_secret_hex = channel_secret.hex() + return await self._add_channel_via_cli(channel_idx, channel_name, channel_secret_hex) + + except Exception as e: + self.logger.error(f"Error adding channel {channel_idx}: {e}") + import traceback + self.logger.debug(traceback.format_exc()) + return False + + async def _add_channel_via_cli(self, channel_idx: int, channel_name: str, channel_secret_hex: str) -> bool: + """ + Fallback method to add channel using CLI wrapper (for older meshcore versions) + + Args: + channel_idx: The channel index + channel_name: The channel name + channel_secret_hex: The channel key as hex string + + Returns: + True if successful, False otherwise + """ + try: + # Subscribe to channel info events to confirm the channel was set + channel_set = False + event_received = asyncio.Event() + + async def on_channel_info(event): + nonlocal channel_set + if event.payload.get('channel_idx') == channel_idx: + payload = event.payload + if payload.get('channel_name') == channel_name: + channel_set = True + event_received.set() + + subscription = self.bot.meshcore.subscribe(EventType.CHANNEL_INFO, on_channel_info) + + try: + from meshcore_cli.meshcore_cli import next_cmd + + # Suppress raw JSON output + with open(os.devnull, 'w') as devnull: + old_stdout = sys.stdout + sys.stdout = devnull + try: + await next_cmd( + self.bot.meshcore, + ["set_channel", str(channel_idx), channel_name, channel_secret_hex] + ) + finally: + sys.stdout = old_stdout + + # Wait for confirmation with timeout + try: + await asyncio.wait_for(event_received.wait(), timeout=self._fetch_timeout * 2) + except asyncio.TimeoutError: + self.logger.warning(f"Timeout waiting for channel {channel_idx} set confirmation") + await asyncio.sleep(0.5) + result = await self._fetch_single_channel(channel_idx) + if result and result.get('channel_name') == channel_name: + channel_set = True + + if channel_set: + # Update cache + result = await self._fetch_single_channel(channel_idx) + if result: + self._channels_cache[channel_idx] = result + self._store_single_channel_in_db(result) + self.logger.info(f"Successfully added channel {channel_idx}: {channel_name}") + return True + else: + self.logger.warning(f"Channel {channel_idx} was set but could not be verified") + return False + else: + self.logger.error(f"Failed to set channel {channel_idx}") + return False + + finally: + self.bot.meshcore.unsubscribe(subscription) + + except Exception as e: + self.logger.error(f"Error in CLI fallback for channel {channel_idx}: {e}") + return False + + async def remove_channel(self, channel_idx: int) -> bool: + """ + Remove a channel from the radio by clearing it + + Args: + channel_idx: The channel index to remove + + Returns: + True if successful, False otherwise + """ + if not self.bot.connected or not self.bot.meshcore: + self.logger.error("Not connected to MeshCore node") + return False + + if channel_idx < 0 or channel_idx >= self.max_channels: + self.logger.error(f"Channel index {channel_idx} out of range (0-{self.max_channels-1})") + return False + + try: + self.logger.info(f"Removing channel {channel_idx}") + + # Create all-zero channel secret (16 bytes) to clear the channel + empty_secret = b'\x00' * 16 + empty_secret_hex = empty_secret.hex() + + # Subscribe to channel info events to confirm the channel was cleared + channel_cleared = False + event_received = asyncio.Event() + + async def on_channel_info(event): + nonlocal channel_cleared + if event.payload.get('channel_idx') == channel_idx: + payload = event.payload + event_secret = payload.get('channel_secret', b'') + # Check if the channel was cleared (all zeros or empty name) + if isinstance(event_secret, bytes) and event_secret == empty_secret: + channel_cleared = True + event_received.set() + elif not payload.get('channel_name') or payload.get('channel_name') == '': + channel_cleared = True + event_received.set() + + subscription = self.bot.meshcore.subscribe(EventType.CHANNEL_INFO, on_channel_info) + + try: + from meshcore_cli.meshcore_cli import next_cmd + + # Suppress raw JSON output + with open(os.devnull, 'w') as devnull: + old_stdout = sys.stdout + sys.stdout = devnull + try: + # Clear the channel by setting it with empty name and all-zero secret + # Format: set_channel "" + await next_cmd( + self.bot.meshcore, + ["set_channel", str(channel_idx), "", empty_secret_hex] + ) + finally: + sys.stdout = old_stdout + + # Wait for confirmation with timeout + try: + await asyncio.wait_for(event_received.wait(), timeout=self._fetch_timeout * 2) + except asyncio.TimeoutError: + self.logger.warning(f"Timeout waiting for channel {channel_idx} removal confirmation") + # Still try to verify by fetching the channel + await asyncio.sleep(0.5) + result = await self._fetch_single_channel(channel_idx) + if not result or not result.get('channel_name'): + channel_cleared = True + + if channel_cleared: + # Remove from cache + if channel_idx in self._channels_cache: + del self._channels_cache[channel_idx] + # Update database - remove the channel + try: + import sqlite3 + db_path = self.bot.db_manager.db_path + with sqlite3.connect(db_path) as conn: + cursor = conn.cursor() + cursor.execute('DELETE FROM channels WHERE channel_idx = ?', (channel_idx,)) + conn.commit() + except Exception as e: + self.logger.warning(f"Failed to remove channel from database: {e}") + self.logger.info(f"Successfully removed channel {channel_idx}") + return True + else: + self.logger.error(f"Failed to remove channel {channel_idx}") + return False + + finally: + # Unsubscribe + self.bot.meshcore.unsubscribe(subscription) + + except Exception as e: + self.logger.error(f"Error removing channel {channel_idx}: {e}") + return False \ No newline at end of file diff --git a/modules/command_manager.py b/modules/command_manager.py index d89b839..6218346 100644 --- a/modules/command_manager.py +++ b/modules/command_manager.py @@ -205,7 +205,24 @@ class CommandManager: async def handle_advert_command(self, message: MeshMessage): """Handle the advert command from DM""" - await self.commands['advert'].execute(message) + command = self.commands['advert'] + success = await command.execute(message) + + # Small delay to ensure send_response has completed + await asyncio.sleep(0.1) + + # Determine if a response was sent + response_sent = False + if hasattr(command, 'last_response') and command.last_response: + response_sent = True + elif hasattr(self, '_last_response') and self._last_response: + response_sent = True + + # Record command execution in stats database + if 'stats' in self.commands: + stats_command = self.commands['stats'] + if stats_command: + stats_command.record_command(message, 'advert', response_sent) async def send_dm(self, recipient_id: str, content: str) -> bool: """Send a direct message using meshcore-cli command""" @@ -528,6 +545,7 @@ class CommandManager: # Check if command can execute (cooldown, DM requirements, etc.) if not command.can_execute_now(message): + response_sent = False # For DM-only commands in public channels, only show error if channel is allowed # (i.e., channel is in monitor_channels or command's allowed_channels) # This prevents prompting users in channels where the command shouldn't work at all @@ -536,10 +554,12 @@ class CommandManager: if command.is_channel_allowed(message): error_msg = command.translate('errors.dm_only', command=command_name) await self.send_response(message, error_msg) + response_sent = True # Otherwise, silently ignore (channel not configured for this command) elif command.requires_admin_access(): error_msg = command.translate('errors.access_denied', command=command_name) await self.send_response(message, error_msg) + response_sent = True elif hasattr(command, 'get_remaining_cooldown') and callable(command.get_remaining_cooldown): # Check if it's the per-user version (takes user_id parameter) import inspect @@ -552,6 +572,14 @@ class CommandManager: if remaining > 0: error_msg = command.translate('errors.cooldown', command=command_name, seconds=remaining) await self.send_response(message, error_msg) + response_sent = True + + # Record command execution in stats database (even if it failed checks) + if 'stats' in self.commands: + stats_command = self.commands['stats'] + if stats_command: + stats_command.record_command(message, command_name, response_sent) + return try: @@ -567,22 +595,33 @@ class CommandManager: # Execute the command success = await command.execute(message) - # Capture command data for web viewer (with small delay to ensure response is set) + # Small delay to ensure send_response has completed + await asyncio.sleep(0.1) + + # Determine if a response was sent by checking response tracking + response_sent = False + response = None + if hasattr(command, 'last_response') and command.last_response: + response = command.last_response + response_sent = True + elif hasattr(self, '_last_response') and self._last_response: + response = self._last_response + response_sent = True + + # Record command execution in stats database + if 'stats' in self.commands: + stats_command = self.commands['stats'] + if stats_command: + stats_command.record_command(message, command_name, response_sent) + + # Capture command data for web viewer if (hasattr(self.bot, 'web_viewer_integration') and self.bot.web_viewer_integration and self.bot.web_viewer_integration.bot_integration): try: - # Small delay to ensure send_response has completed - await asyncio.sleep(0.1) - - # Get the response that was sent (if any) - # Prioritize command.last_response (full response) over _last_response (may be split) - # This ensures commands like path that split messages still show full response in webviewer - response = "Command executed" # Default response - if hasattr(command, 'last_response') and command.last_response: - response = command.last_response - elif hasattr(self, '_last_response') and self._last_response: - response = self._last_response + # Use the response we found, or default + if response is None: + response = "Command executed" self.bot.web_viewer_integration.bot_integration.capture_command( message, command_name, response, success if success is not None else True @@ -596,6 +635,12 @@ class CommandManager: error_msg = command.translate('errors.execution_error', command=command_name, error=str(e)) await self.send_response(message, error_msg) + # Record command execution in stats database (error response was sent) + if 'stats' in self.commands: + stats_command = self.commands['stats'] + if stats_command: + stats_command.record_command(message, command_name, True) # Error message counts as response + # Capture failed command for web viewer if (hasattr(self.bot, 'web_viewer_integration') and self.bot.web_viewer_integration and diff --git a/modules/commands/feed_command.py b/modules/commands/feed_command.py new file mode 100644 index 0000000..c563c11 --- /dev/null +++ b/modules/commands/feed_command.py @@ -0,0 +1,417 @@ +#!/usr/bin/env python3 +""" +Feed command for the MeshCore Bot +Handles RSS and API feed subscription management +""" + +import json +import re +from typing import Optional, List, Dict, Any +from urllib.parse import urlparse +from .base_command import BaseCommand +from ..models import MeshMessage + + +class FeedCommand(BaseCommand): + """Handles feed subscription management""" + + # Plugin metadata + name = "feed" + keywords = ['feed', 'feeds', 'rss', 'subscription', 'subscriptions'] + description = "Manage RSS and API feed subscriptions (usage: feed subscribe rss [name])" + category = "admin" + requires_dm = True + cooldown_seconds = 2 + + def __init__(self, bot): + super().__init__(bot) + self.db_path = bot.db_manager.db_path + + def can_execute(self, message: MeshMessage) -> bool: + """Check if this command can be executed (admin only)""" + if not self.requires_admin_access(): + return False + return super().can_execute(message) + + def requires_admin_access(self) -> bool: + """Feed command requires admin access""" + return True + + async def execute(self, message: MeshMessage) -> bool: + """Execute the feed command""" + content = message.content.strip() + + # Handle exclamation prefix + if content.startswith('!'): + content = content[1:].strip() + + # Parse command + parts = content.split() + if len(parts) < 2: + return await self.send_response(message, self.get_help_text()) + + subcommand = parts[1].lower() + + if subcommand == 'subscribe': + return await self._handle_subscribe(message, parts[2:]) + elif subcommand == 'unsubscribe': + return await self._handle_unsubscribe(message, parts[2:]) + elif subcommand == 'list': + return await self._handle_list(message, parts[2:]) + elif subcommand == 'status': + return await self._handle_status(message, parts[2:]) + elif subcommand == 'test': + return await self._handle_test(message, parts[2:]) + elif subcommand == 'enable': + return await self._handle_enable_disable(message, parts[2:], True) + elif subcommand == 'disable': + return await self._handle_enable_disable(message, parts[2:], False) + elif subcommand == 'update': + return await self._handle_update(message, parts[2:]) + else: + return await self.send_response(message, self.get_help_text()) + + def get_help_text(self) -> str: + """Get help text for feed command""" + return """Feed Command Usage: +feed subscribe [name] +feed unsubscribe +feed list [channel] +feed status +feed test +feed enable +feed disable +feed update [interval_seconds] + +Examples: +feed subscribe rss https://alerts.example.com/rss emergency "Emergency Alerts" +feed subscribe api https://api.example.com/alerts emergency "API Alerts" '{"headers": {"Authorization": "Bearer TOKEN"}}' +feed list +feed status 1""" + + async def _handle_subscribe(self, message: MeshMessage, args: List[str]) -> bool: + """Handle feed subscribe command""" + if len(args) < 3: + return await self.send_response(message, "Usage: feed subscribe [name] [api_config]") + + feed_type = args[0].lower() + if feed_type not in ['rss', 'api']: + return await self.send_response(message, "Feed type must be 'rss' or 'api'") + + feed_url = args[1] + channel_name = args[2] + feed_name = args[3] if len(args) > 3 else None + api_config = args[4] if len(args) > 4 and feed_type == 'api' else None + + # Validate URL + if not self._validate_url(feed_url): + return await self.send_response(message, "Invalid URL format") + + # Validate channel exists + channel_num = self.bot.channel_manager.get_channel_number(channel_name) + if channel_num is None: + return await self.send_response(message, f"Channel '{channel_name}' not found. Create it first or use a valid channel name.") + + # Parse API config if provided + api_config_json = None + if feed_type == 'api' and api_config: + try: + api_config_json = json.loads(api_config) + except json.JSONDecodeError: + return await self.send_response(message, "Invalid API config JSON") + + # Create subscription + try: + feed_id = self._create_subscription( + feed_type=feed_type, + feed_url=feed_url, + channel_name=channel_name, + feed_name=feed_name, + api_config=api_config_json + ) + + response = f"Subscribed to {feed_type.upper()} feed" + if feed_name: + response += f" '{feed_name}'" + response += f" -> channel: {channel_name} (ID: {feed_id})" + return await self.send_response(message, response) + + except Exception as e: + self.logger.error(f"Error creating subscription: {e}") + return await self.send_response(message, f"Error creating subscription: {str(e)}") + + async def _handle_unsubscribe(self, message: MeshMessage, args: List[str]) -> bool: + """Handle feed unsubscribe command""" + if len(args) < 1: + return await self.send_response(message, "Usage: feed unsubscribe [channel]") + + identifier = args[0] + channel_name = args[1] if len(args) > 1 else None + + try: + # Try as ID first + try: + feed_id = int(identifier) + success = self._delete_subscription_by_id(feed_id) + except ValueError: + # Try as URL + if channel_name: + success = self._delete_subscription_by_url(identifier, channel_name) + else: + return await self.send_response(message, "Channel name required when using URL") + + if success: + return await self.send_response(message, f"Unsubscribed from feed (ID: {identifier})") + else: + return await self.send_response(message, "Feed subscription not found") + + except Exception as e: + self.logger.error(f"Error unsubscribing: {e}") + return await self.send_response(message, f"Error unsubscribing: {str(e)}") + + async def _handle_list(self, message: MeshMessage, args: List[str]) -> bool: + """Handle feed list command""" + channel_filter = args[0] if args else None + + try: + feeds = self._get_subscriptions(channel_filter) + + if not feeds: + response = "No feed subscriptions" + if channel_filter: + response += f" for channel '{channel_filter}'" + return await self.send_response(message, response) + + response = f"Feed Subscriptions ({len(feeds)}):\n" + for feed in feeds[:10]: # Limit to 10 for mesh message + status = "enabled" if feed['enabled'] else "disabled" + name = feed.get('feed_name') or feed['feed_url'][:30] + response += f"{feed['id']}. {name} ({feed['feed_type']}) -> {feed['channel_name']} [{status}]\n" + + if len(feeds) > 10: + response += f"({len(feeds) - 10} more...)" + + return await self.send_response(message, response) + + except Exception as e: + self.logger.error(f"Error listing feeds: {e}") + return await self.send_response(message, f"Error listing feeds: {str(e)}") + + async def _handle_status(self, message: MeshMessage, args: List[str]) -> bool: + """Handle feed status command""" + if not args: + return await self.send_response(message, "Usage: feed status ") + + try: + feed_id = int(args[0]) + feed = self._get_subscription_by_id(feed_id) + + if not feed: + return await self.send_response(message, f"Feed subscription {feed_id} not found") + + status = "enabled" if feed['enabled'] else "disabled" + last_check = feed.get('last_check_time') or "Never" + last_item = feed.get('last_item_id') or "None" + + response = f"Feed {feed_id} Status:\n" + response += f"Name: {feed.get('feed_name') or 'N/A'}\n" + response += f"Type: {feed['feed_type']}\n" + response += f"URL: {feed['feed_url']}\n" + response += f"Channel: {feed['channel_name']}\n" + response += f"Status: {status}\n" + response += f"Interval: {feed.get('check_interval_seconds', 300)}s\n" + response += f"Last check: {last_check}\n" + response += f"Last item: {last_item[:30] if last_item != 'None' else 'None'}" + + return await self.send_response(message, response) + + except ValueError: + return await self.send_response(message, "Invalid feed ID") + except Exception as e: + self.logger.error(f"Error getting feed status: {e}") + return await self.send_response(message, f"Error getting feed status: {str(e)}") + + async def _handle_test(self, message: MeshMessage, args: List[str]) -> bool: + """Handle feed test command""" + if not args: + return await self.send_response(message, "Usage: feed test ") + + feed_url = args[0] + + if not self._validate_url(feed_url): + return await self.send_response(message, "Invalid URL format") + + # Test would require feed_manager to be available + # For now, just validate URL + return await self.send_response(message, f"URL validated: {feed_url}\n(Full test requires feed manager)") + + async def _handle_enable_disable(self, message: MeshMessage, args: List[str], enable: bool) -> bool: + """Handle enable/disable command""" + if not args: + return await self.send_response(message, f"Usage: feed {'enable' if enable else 'disable'} ") + + try: + feed_id = int(args[0]) + success = self._set_subscription_enabled(feed_id, enable) + + if success: + status = "enabled" if enable else "disabled" + return await self.send_response(message, f"Feed {feed_id} {status}") + else: + return await self.send_response(message, f"Feed subscription {feed_id} not found") + + except ValueError: + return await self.send_response(message, "Invalid feed ID") + except Exception as e: + self.logger.error(f"Error setting feed status: {e}") + return await self.send_response(message, f"Error: {str(e)}") + + async def _handle_update(self, message: MeshMessage, args: List[str]) -> bool: + """Handle update command""" + if not args: + return await self.send_response(message, "Usage: feed update [interval_seconds]") + + try: + feed_id = int(args[0]) + interval = int(args[1]) if len(args) > 1 else None + + success = self._update_subscription(feed_id, interval) + + if success: + response = f"Feed {feed_id} updated" + if interval: + response += f" (interval: {interval}s)" + return await self.send_response(message, response) + else: + return await self.send_response(message, f"Feed subscription {feed_id} not found") + + except ValueError: + return await self.send_response(message, "Invalid feed ID or interval") + except Exception as e: + self.logger.error(f"Error updating feed: {e}") + return await self.send_response(message, f"Error: {str(e)}") + + def _validate_url(self, url: str) -> bool: + """Validate URL format""" + try: + result = urlparse(url) + return all([result.scheme in ['http', 'https'], result.netloc]) + except Exception: + return False + + def _create_subscription(self, feed_type: str, feed_url: str, channel_name: str, + feed_name: Optional[str] = None, api_config: Optional[Dict] = None) -> int: + """Create a new feed subscription""" + import sqlite3 + + with sqlite3.connect(self.db_path) as conn: + cursor = conn.cursor() + + # Get default check interval + default_interval = self.bot.config.getint('Feed_Manager', 'default_check_interval_seconds', fallback=300) + + api_config_str = json.dumps(api_config) if api_config else None + + cursor.execute(''' + INSERT INTO feed_subscriptions + (feed_type, feed_url, channel_name, feed_name, check_interval_seconds, api_config) + VALUES (?, ?, ?, ?, ?, ?) + ''', (feed_type, feed_url, channel_name, feed_name, default_interval, api_config_str)) + + conn.commit() + return cursor.lastrowid + + def _delete_subscription_by_id(self, feed_id: int) -> bool: + """Delete subscription by ID""" + import sqlite3 + + with sqlite3.connect(self.db_path) as conn: + cursor = conn.cursor() + cursor.execute('DELETE FROM feed_subscriptions WHERE id = ?', (feed_id,)) + conn.commit() + return cursor.rowcount > 0 + + def _delete_subscription_by_url(self, feed_url: str, channel_name: str) -> bool: + """Delete subscription by URL and channel""" + import sqlite3 + + with sqlite3.connect(self.db_path) as conn: + cursor = conn.cursor() + cursor.execute(''' + DELETE FROM feed_subscriptions + WHERE feed_url = ? AND channel_name = ? + ''', (feed_url, channel_name)) + conn.commit() + return cursor.rowcount > 0 + + def _get_subscriptions(self, channel_filter: Optional[str] = None) -> List[Dict]: + """Get all subscriptions, optionally filtered by channel""" + import sqlite3 + + with sqlite3.connect(self.db_path) as conn: + conn.row_factory = sqlite3.Row + cursor = conn.cursor() + + if channel_filter: + cursor.execute(''' + SELECT * FROM feed_subscriptions + WHERE channel_name = ? + ORDER BY id + ''', (channel_filter,)) + else: + cursor.execute(''' + SELECT * FROM feed_subscriptions + ORDER BY id + ''') + + rows = cursor.fetchall() + return [dict(row) for row in rows] + + def _get_subscription_by_id(self, feed_id: int) -> Optional[Dict]: + """Get subscription by ID""" + import sqlite3 + + with sqlite3.connect(self.db_path) as conn: + conn.row_factory = sqlite3.Row + cursor = conn.cursor() + cursor.execute('SELECT * FROM feed_subscriptions WHERE id = ?', (feed_id,)) + row = cursor.fetchone() + return dict(row) if row else None + + def _set_subscription_enabled(self, feed_id: int, enabled: bool) -> bool: + """Enable or disable a subscription""" + import sqlite3 + + with sqlite3.connect(self.db_path) as conn: + cursor = conn.cursor() + cursor.execute(''' + UPDATE feed_subscriptions + SET enabled = ?, updated_at = CURRENT_TIMESTAMP + WHERE id = ? + ''', (1 if enabled else 0, feed_id)) + conn.commit() + return cursor.rowcount > 0 + + def _update_subscription(self, feed_id: int, interval: Optional[int] = None) -> bool: + """Update subscription settings""" + import sqlite3 + + with sqlite3.connect(self.db_path) as conn: + cursor = conn.cursor() + + if interval: + cursor.execute(''' + UPDATE feed_subscriptions + SET check_interval_seconds = ?, updated_at = CURRENT_TIMESTAMP + WHERE id = ? + ''', (interval, feed_id)) + else: + cursor.execute(''' + UPDATE feed_subscriptions + SET updated_at = CURRENT_TIMESTAMP + WHERE id = ? + ''', (feed_id,)) + + conn.commit() + return cursor.rowcount > 0 + diff --git a/modules/commands/greeter_command.py b/modules/commands/greeter_command.py index 83fecdf..6a5ab63 100644 --- a/modules/commands/greeter_command.py +++ b/modules/commands/greeter_command.py @@ -6,6 +6,7 @@ Greets users on their first public channel message with mesh information import sqlite3 import time +import asyncio from datetime import datetime, timedelta from typing import Optional, Dict, Any, List from .base_command import BaseCommand @@ -26,6 +27,9 @@ class GreeterCommand(BaseCommand): self._init_greeter_tables() self._load_config() + # Track pending greetings (for dead air delay) + self.pending_greetings = {} # key: (sender_id, channel), value: asyncio.Task + # Auto-backfill if enabled if self.enabled and self.auto_backfill: self.logger.info("Auto-backfill enabled - backfilling greeted users from historical data") @@ -166,6 +170,14 @@ class GreeterCommand(BaseCommand): self.greeting_parts = [part.strip() for part in self.greeting_message.split('|') if part.strip()] else: self.greeting_parts = [self.greeting_message] + + # Dead air delay settings + self.dead_air_delay_seconds = self.get_config_value('Greeter_Command', 'dead_air_delay_seconds', + fallback=0, value_type='int') + self.defer_to_human_greeting = self.get_config_value('Greeter_Command', 'defer_to_human_greeting', + fallback=False, value_type='bool') + self.levenshtein_distance = self.get_config_value('Greeter_Command', 'levenshtein_distance', + fallback=0, value_type='int') def _init_greeter_tables(self): """Initialize database tables for greeter tracking""" @@ -508,9 +520,83 @@ class GreeterCommand(BaseCommand): self.logger.error(f"Error starting rollout: {e}") return False + def _levenshtein_distance(self, s1: str, s2: str) -> int: + """ + Calculate Levenshtein distance between two strings + + Args: + s1: First string + s2: Second string + + Returns: + Levenshtein distance (number of edits needed) + """ + if len(s1) < len(s2): + return self._levenshtein_distance(s2, s1) + + if len(s2) == 0: + return len(s1) + + previous_row = range(len(s2) + 1) + for i, c1 in enumerate(s1): + current_row = [i + 1] + for j, c2 in enumerate(s2): + insertions = previous_row[j + 1] + 1 + deletions = current_row[j] + 1 + substitutions = previous_row[j] + (c1 != c2) + current_row.append(min(insertions, deletions, substitutions)) + previous_row = current_row + + return previous_row[-1] + + def _find_similar_greeted_user(self, sender_id: str, channel: str) -> Optional[str]: + """ + Find if a user with a similar name (within Levenshtein distance) has been greeted + + Args: + sender_id: The user's ID to check + channel: The channel name (used only if per_channel_greetings is True) + + Returns: + The greeted sender_id if a similar one is found, None otherwise + """ + if self.levenshtein_distance <= 0: + return None + + try: + with sqlite3.connect(self.bot.db_manager.db_path) as conn: + cursor = conn.cursor() + + if self.per_channel_greetings: + # Per-channel mode: check greeted users on this specific channel + cursor.execute(''' + SELECT DISTINCT sender_id FROM greeted_users + WHERE channel = ? + ''', (channel,)) + else: + # Global mode: check all greeted users (channel = NULL) + cursor.execute(''' + SELECT DISTINCT sender_id FROM greeted_users + WHERE channel IS NULL + ''') + + greeted_users = cursor.fetchall() + + # Check each greeted user for similarity + for (greeted_id,) in greeted_users: + distance = self._levenshtein_distance(sender_id.lower(), greeted_id.lower()) + if distance <= self.levenshtein_distance: + self.logger.debug(f"Found similar user: {greeted_id} (distance: {distance} from {sender_id})") + return greeted_id + + return None + except Exception as e: + self.logger.error(f"Error checking for similar greeted users: {e}") + return None + def has_been_greeted(self, sender_id: str, channel: str) -> bool: """ - Check if a user has been greeted + Check if a user has been greeted (with optional Levenshtein distance matching) Args: sender_id: The user's ID @@ -536,7 +622,17 @@ class GreeterCommand(BaseCommand): WHERE sender_id = ? AND channel IS NULL ''', (sender_id,)) - return cursor.fetchone() is not None + if cursor.fetchone() is not None: + return True + + # If exact match not found and Levenshtein distance is enabled, check for similar names + if self.levenshtein_distance > 0: + similar_user = self._find_similar_greeted_user(sender_id, channel) + if similar_user: + self.logger.info(f"User {sender_id} matches previously greeted user {similar_user} (Levenshtein distance enabled)") + return True + + return False except Exception as e: self.logger.error(f"Error checking if user has been greeted: {e}") return False @@ -587,11 +683,11 @@ class GreeterCommand(BaseCommand): count = cursor.fetchone()[0] if count > 1: # Duplicates exist - clean them up, keeping the earliest (first) greeting - cursor.execute(''' - SELECT id FROM greeted_users - WHERE sender_id = ? AND channel = ? + cursor.execute(''' + SELECT id FROM greeted_users + WHERE sender_id = ? AND channel = ? ORDER BY greeted_at ASC - ''', (sender_id, channel)) + ''', (sender_id, channel)) all_ids = [row[0] for row in cursor.fetchall()] if len(all_ids) > 1: # Delete all but the first (earliest) @@ -639,11 +735,11 @@ class GreeterCommand(BaseCommand): count = cursor.fetchone()[0] if count > 1: # Duplicates exist - clean them up, keeping the earliest (first) greeting - cursor.execute(''' - SELECT id FROM greeted_users - WHERE sender_id = ? AND channel IS NULL + cursor.execute(''' + SELECT id FROM greeted_users + WHERE sender_id = ? AND channel IS NULL ORDER BY greeted_at ASC - ''', (sender_id,)) + ''', (sender_id,)) all_ids = [row[0] for row in cursor.fetchall()] if len(all_ids) > 1: # Delete all but the first (earliest) @@ -956,6 +1052,182 @@ class GreeterCommand(BaseCommand): self.logger.error(traceback.format_exc()) return False + def _check_human_greeting(self, new_user_id: str, channel: str, since_timestamp: int) -> bool: + """ + Check if a human has greeted the new user by mentioning their name in a message + + Args: + new_user_id: The new user's ID to check for + channel: The channel to check + since_timestamp: Only check messages after this timestamp + + Returns: + True if a human (not the new user) has mentioned the new user's name + """ + if not self.defer_to_human_greeting: + return False + + try: + with sqlite3.connect(self.bot.db_manager.db_path) as conn: + cursor = conn.cursor() + + # Check if message_stats table exists + cursor.execute(''' + SELECT name FROM sqlite_master + WHERE type='table' AND name='message_stats' + ''') + if not cursor.fetchone(): + return False + + # Get recent messages from this channel since the new user posted + cursor.execute(''' + SELECT sender_id, content + FROM message_stats + WHERE channel = ? + AND timestamp >= ? + AND is_dm = 0 + AND sender_id != ? + ORDER BY timestamp DESC + ''', (channel, since_timestamp, new_user_id)) + + messages = cursor.fetchall() + + # Check if any message contains the new user's name + new_user_id_lower = new_user_id.lower() + for sender_id, content in messages: + if content and new_user_id_lower in content.lower(): + # Also check with Levenshtein distance if enabled + if self.levenshtein_distance > 0: + # Check if any word in the message is within Levenshtein distance + words = content.lower().split() + for word in words: + # Remove common punctuation + word = word.strip('.,!?;:()[]{}@') + distance = self._levenshtein_distance(new_user_id_lower, word) + if distance <= self.levenshtein_distance: + self.logger.info(f"Human greeting detected: {sender_id} mentioned {new_user_id} in channel {channel}") + return True + else: + # Simple substring match + self.logger.info(f"Human greeting detected: {sender_id} mentioned {new_user_id} in channel {channel}") + return True + + return False + except Exception as e: + self.logger.error(f"Error checking for human greeting: {e}") + return False + + def _cancel_pending_greeting(self, sender_id: str, channel: str): + """Cancel a pending greeting if it exists""" + key = (sender_id, channel) + if key in self.pending_greetings: + task = self.pending_greetings[key] + if not task.done(): + task.cancel() + self.logger.info(f"Cancelled pending greeting for {sender_id} on {channel}") + del self.pending_greetings[key] + + async def _send_delayed_greeting(self, message: MeshMessage): + """ + Send a greeting after the dead air delay, checking for human greetings during the delay + + Args: + message: The original message that triggered the greeting + """ + key = (message.sender_id, message.channel) + original_timestamp = message.timestamp or int(time.time()) + + try: + # Wait for the dead air delay + if self.dead_air_delay_seconds > 0: + self.logger.debug(f"Waiting {self.dead_air_delay_seconds} seconds before greeting {message.sender_id} on {message.channel}") + await asyncio.sleep(self.dead_air_delay_seconds) + + # Check if greeting was cancelled (user was already greeted or human responded) + if key not in self.pending_greetings: + self.logger.debug(f"Greeting for {message.sender_id} on {message.channel} was cancelled") + return + + # Check if we should still greet (user might have been greeted by another process) + if self.has_been_greeted(message.sender_id, message.channel): + self.logger.debug(f"User {message.sender_id} already greeted on {message.channel} - skipping") + if key in self.pending_greetings: + del self.pending_greetings[key] + return + + # If defer to human greeting is enabled, check if a human has greeted the user + # Check messages from the original timestamp onwards (during the delay period) + if self.defer_to_human_greeting and self.dead_air_delay_seconds > 0: + if self._check_human_greeting(message.sender_id, message.channel, original_timestamp): + self.logger.info(f"Deferring to human greeting for {message.sender_id} on {message.channel}") + # Mark as greeted so we don't greet them later + self.mark_as_greeted(message.sender_id, message.channel) + if key in self.pending_greetings: + del self.pending_greetings[key] + return + + # Send the greeting + await self._send_greeting(message) + + # Clean up + if key in self.pending_greetings: + del self.pending_greetings[key] + + except asyncio.CancelledError: + self.logger.debug(f"Delayed greeting for {message.sender_id} on {message.channel} was cancelled") + # Clean up on cancellation + if key in self.pending_greetings: + del self.pending_greetings[key] + except Exception as e: + self.logger.error(f"Error in delayed greeting for {message.sender_id}: {e}") + if key in self.pending_greetings: + del self.pending_greetings[key] + + async def _send_greeting(self, message: MeshMessage) -> bool: + """ + Actually send the greeting message (extracted from execute for reuse) + + Args: + message: The message that triggered the greeting + + Returns: + True if greeting was sent successfully + """ + try: + # Format greeting parts (may be single or multi-part) + # Pass channel name for channel-specific greetings + greeting_parts = await self._format_greeting_parts(message.sender_id, message.channel) + + # Send greeting(s) + mode_str = "per-channel" if self.per_channel_greetings else "global" + self.logger.info(f"Greeting {message.sender_id} on channel {message.channel} ({mode_str} mode, {len(greeting_parts)} part(s))") + + # Log database verification + total_greeted = self.get_greeted_users_count() + self.logger.debug(f"Database verification: {total_greeted} total user(s) marked as greeted") + + # Send all greeting parts + success = True + for i, greeting_part in enumerate(greeting_parts): + if i > 0: + # Wait for bot TX rate limiter between multi-part messages + # This ensures we respect the bot's rate limiting configuration + await self.bot.bot_tx_rate_limiter.wait_for_tx() + # Additional delay to ensure proper spacing (use configured rate limit) + rate_limit = self.bot.config.getfloat('Bot', 'bot_tx_rate_limit_seconds', fallback=1.0) + # Use a conservative sleep time to avoid rate limiting + sleep_time = max(rate_limit + 0.5, 1.0) # At least 1 second, or rate_limit + 0.5 seconds + await asyncio.sleep(sleep_time) + + result = await self.send_response(message, greeting_part) + if not result: + success = False + + return success + except Exception as e: + self.logger.error(f"Error sending greeting: {e}") + return False + def should_execute(self, message: MeshMessage) -> bool: """ Check if greeter should execute for this message @@ -1013,7 +1285,7 @@ class GreeterCommand(BaseCommand): if not self.should_execute(message): return False - # Mark as greeted BEFORE getting mesh info (to prevent duplicate greetings) + # Mark as greeted BEFORE scheduling greeting (to prevent duplicate greetings) # This ensures we don't greet the same user twice even if there's a delay # mark_as_greeted uses atomic INSERT OR IGNORE to handle race conditions marked = self.mark_as_greeted(message.sender_id, message.channel) @@ -1061,42 +1333,71 @@ class GreeterCommand(BaseCommand): # If check fails, proceed anyway (better to greet than miss a greeting) self.logger.debug(f"Could not verify greeting timestamp (proceeding anyway): {e}") - # Format greeting parts (may be single or multi-part) - # Pass channel name for channel-specific greetings - greeting_parts = await self._format_greeting_parts(message.sender_id, message.channel) - - # Send greeting(s) - mode_str = "per-channel" if self.per_channel_greetings else "global" - self.logger.info(f"Greeting {message.sender_id} on channel {message.channel} ({mode_str} mode, {len(greeting_parts)} part(s))") - - # Log database verification - total_greeted = self.get_greeted_users_count() - self.logger.debug(f"Database verification: {total_greeted} total user(s) marked as greeted") - - # Send all greeting parts - success = True - for i, greeting_part in enumerate(greeting_parts): - if i > 0: - # Wait for bot TX rate limiter between multi-part messages - # This ensures we respect the bot's rate limiting configuration - await self.bot.bot_tx_rate_limiter.wait_for_tx() - # Additional delay to ensure proper spacing (use configured rate limit) - import asyncio - rate_limit = self.bot.config.getfloat('Bot', 'bot_tx_rate_limit_seconds', fallback=1.0) - # Use a conservative sleep time to avoid rate limiting - sleep_time = max(rate_limit + 0.5, 1.0) # At least 1 second, or rate_limit + 0.5 seconds - await asyncio.sleep(sleep_time) + # Check if dead air delay is enabled + if self.dead_air_delay_seconds > 0: + # Schedule delayed greeting + key = (message.sender_id, message.channel) - result = await self.send_response(message, greeting_part) - if not result: - success = False - - return success + # Cancel any existing pending greeting for this user/channel + if key in self.pending_greetings: + self._cancel_pending_greeting(message.sender_id, message.channel) + + # Schedule new delayed greeting + task = asyncio.create_task(self._send_delayed_greeting(message)) + self.pending_greetings[key] = task + self.logger.info(f"Scheduled delayed greeting for {message.sender_id} on {message.channel} (delay: {self.dead_air_delay_seconds}s)") + return True + else: + # Send greeting immediately (original behavior) + return await self._send_greeting(message) except Exception as e: self.logger.error(f"Error executing greeter command: {e}") return False + def check_message_for_human_greeting(self, message: MeshMessage): + """ + Check if an incoming message should cancel a pending greeting + Called from message handler when new messages arrive + + Args: + message: The incoming message to check + """ + if not self.defer_to_human_greeting or not self.dead_air_delay_seconds > 0: + return + + if message.is_dm or not message.channel: + return + + # Check all pending greetings for this channel + keys_to_cancel = [] + for (sender_id, channel), task in list(self.pending_greetings.items()): + if channel == message.channel and sender_id != message.sender_id: + # Check if this message mentions the pending user + if message.content and sender_id.lower() in message.content.lower(): + # Also check with Levenshtein distance if enabled + should_cancel = False + if self.levenshtein_distance > 0: + words = message.content.lower().split() + for word in words: + word = word.strip('.,!?;:()[]{}@') + distance = self._levenshtein_distance(sender_id.lower(), word) + if distance <= self.levenshtein_distance: + should_cancel = True + break + else: + should_cancel = True + + if should_cancel: + self.logger.info(f"Human greeting detected in real-time: {message.sender_id} mentioned {sender_id} - cancelling pending greeting") + keys_to_cancel.append((sender_id, channel)) + + # Cancel the pending greetings + for key in keys_to_cancel: + self._cancel_pending_greeting(key[0], key[1]) + # Mark as greeted so we don't greet them later + self.mark_as_greeted(key[0], key[1]) + def get_help_text(self) -> str: mode = "per-channel" if self.per_channel_greetings else "global (once total)" return f"Greeter automatically welcomes new users on public channels ({mode} mode). Configure in [Greeter_Command] section." diff --git a/modules/core.py b/modules/core.py index 6d4ef80..d6f35c1 100644 --- a/modules/core.py +++ b/modules/core.py @@ -35,6 +35,7 @@ from .db_manager import DBManager from .i18n import Translator from .solar_conditions import set_config from .web_viewer.integration import WebViewerIntegration +from .feed_manager import FeedManager class MeshCoreBot: @@ -125,6 +126,15 @@ class MeshCoreBot: self.scheduler = MessageScheduler(self) + # Initialize feed manager + self.logger.info("Initializing feed manager") + try: + self.feed_manager = FeedManager(self) + self.logger.info("Feed manager initialized successfully") + except Exception as e: + self.logger.warning(f"Failed to initialize feed manager: {e}") + self.feed_manager = None + # Initialize repeater manager self.logger.info("Initializing repeater manager") try: @@ -743,6 +753,10 @@ use_zulu_time = false # Setup scheduled messages self.scheduler.setup_scheduled_messages() + # Initialize feed manager (if enabled) + if self.feed_manager: + await self.feed_manager.initialize() + # Start scheduler thread self.scheduler.start() @@ -794,6 +808,10 @@ use_zulu_time = false self.connected = False + # Stop feed manager + if self.feed_manager: + await self.feed_manager.stop() + # Stop web viewer with proper shutdown sequence if self.web_viewer_integration: # Web viewer has simpler shutdown diff --git a/modules/db_manager.py b/modules/db_manager.py index 57db433..5e65ed3 100644 --- a/modules/db_manager.py +++ b/modules/db_manager.py @@ -59,12 +59,135 @@ class DBManager: ) ''') + # Create feed_subscriptions table for RSS/API feed subscriptions + cursor.execute(''' + CREATE TABLE IF NOT EXISTS feed_subscriptions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + feed_type TEXT NOT NULL, + feed_url TEXT NOT NULL, + channel_name TEXT NOT NULL, + feed_name TEXT, + last_item_id TEXT, + last_check_time TIMESTAMP, + check_interval_seconds INTEGER DEFAULT 300, + enabled BOOLEAN DEFAULT 1, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + api_config TEXT, + rss_config TEXT, + output_format TEXT, + message_send_interval_seconds REAL DEFAULT 2.0, + UNIQUE(feed_url, channel_name) + ) + ''') + + # Add new columns if they don't exist (for existing databases) + try: + cursor.execute('ALTER TABLE feed_subscriptions ADD COLUMN output_format TEXT') + except sqlite3.OperationalError: + pass # Column already exists + try: + cursor.execute('ALTER TABLE feed_subscriptions ADD COLUMN message_send_interval_seconds REAL DEFAULT 2.0') + except sqlite3.OperationalError: + pass # Column already exists + try: + cursor.execute('ALTER TABLE feed_subscriptions ADD COLUMN filter_config TEXT') + except sqlite3.OperationalError: + pass # Column already exists + try: + cursor.execute('ALTER TABLE feed_subscriptions ADD COLUMN sort_config TEXT') + except sqlite3.OperationalError: + pass # Column already exists + + # Create feed_activity table for tracking processed items + cursor.execute(''' + CREATE TABLE IF NOT EXISTS feed_activity ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + feed_id INTEGER NOT NULL, + item_id TEXT NOT NULL, + item_title TEXT, + processed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + message_sent BOOLEAN DEFAULT 1, + FOREIGN KEY (feed_id) REFERENCES feed_subscriptions(id) ON DELETE CASCADE + ) + ''') + + # Create feed_errors table for tracking feed errors + cursor.execute(''' + CREATE TABLE IF NOT EXISTS feed_errors ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + feed_id INTEGER NOT NULL, + error_type TEXT NOT NULL, + error_message TEXT, + occurred_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + resolved_at TIMESTAMP, + FOREIGN KEY (feed_id) REFERENCES feed_subscriptions(id) ON DELETE CASCADE + ) + ''') + + # Create channels table for storing channel information + cursor.execute(''' + CREATE TABLE IF NOT EXISTS channels ( + channel_idx INTEGER PRIMARY KEY, + channel_name TEXT NOT NULL, + channel_type TEXT, + channel_key_hex TEXT, + last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + UNIQUE(channel_idx) + ) + ''') + + # Create channel_operations queue table for web viewer -> bot communication + cursor.execute(''' + CREATE TABLE IF NOT EXISTS channel_operations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + operation_type TEXT NOT NULL, + channel_idx INTEGER, + channel_name TEXT, + channel_key_hex TEXT, + status TEXT DEFAULT 'pending', + error_message TEXT, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + processed_at TIMESTAMP, + result_data TEXT + ) + ''') + + # Create feed_message_queue table for queuing feed messages + cursor.execute(''' + CREATE TABLE IF NOT EXISTS feed_message_queue ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + feed_id INTEGER NOT NULL, + channel_name TEXT NOT NULL, + message TEXT NOT NULL, + item_id TEXT, + item_title TEXT, + priority INTEGER DEFAULT 0, + queued_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + sent_at TIMESTAMP, + FOREIGN KEY (feed_id) REFERENCES feed_subscriptions(id) ON DELETE CASCADE + ) + ''') + # Create indexes for better performance cursor.execute('CREATE INDEX IF NOT EXISTS idx_geocoding_query ON geocoding_cache(query)') cursor.execute('CREATE INDEX IF NOT EXISTS idx_geocoding_expires ON geocoding_cache(expires_at)') cursor.execute('CREATE INDEX IF NOT EXISTS idx_generic_key ON generic_cache(cache_key)') cursor.execute('CREATE INDEX IF NOT EXISTS idx_generic_type ON generic_cache(cache_type)') cursor.execute('CREATE INDEX IF NOT EXISTS idx_generic_expires ON generic_cache(expires_at)') + cursor.execute('CREATE INDEX IF NOT EXISTS idx_feed_subscriptions_enabled ON feed_subscriptions(enabled)') + cursor.execute('CREATE INDEX IF NOT EXISTS idx_feed_subscriptions_type ON feed_subscriptions(feed_type)') + cursor.execute('CREATE INDEX IF NOT EXISTS idx_feed_subscriptions_last_check ON feed_subscriptions(last_check_time)') + cursor.execute('CREATE INDEX IF NOT EXISTS idx_feed_activity_feed_id ON feed_activity(feed_id)') + cursor.execute('CREATE INDEX IF NOT EXISTS idx_feed_activity_processed_at ON feed_activity(processed_at)') + cursor.execute('CREATE INDEX IF NOT EXISTS idx_feed_errors_feed_id ON feed_errors(feed_id)') + cursor.execute('CREATE INDEX IF NOT EXISTS idx_feed_errors_occurred_at ON feed_errors(occurred_at)') + cursor.execute('CREATE INDEX IF NOT EXISTS idx_feed_errors_resolved ON feed_errors(resolved_at)') + cursor.execute('CREATE INDEX IF NOT EXISTS idx_channels_name ON channels(channel_name)') + cursor.execute('CREATE INDEX IF NOT EXISTS idx_channel_ops_status ON channel_operations(status, created_at)') + cursor.execute('CREATE INDEX IF NOT EXISTS idx_feed_message_queue_feed_id ON feed_message_queue(feed_id)') + cursor.execute('CREATE INDEX IF NOT EXISTS idx_feed_message_queue_sent ON feed_message_queue(sent_at)') + cursor.execute('CREATE INDEX IF NOT EXISTS idx_feed_message_queue_priority ON feed_message_queue(priority DESC, queued_at ASC)') conn.commit() self.logger.info("Database manager initialized successfully") diff --git a/modules/feed_manager.py b/modules/feed_manager.py new file mode 100644 index 0000000..fcf02ed --- /dev/null +++ b/modules/feed_manager.py @@ -0,0 +1,1252 @@ +#!/usr/bin/env python3 +""" +Feed Manager for RSS and API feed subscriptions +Handles polling feeds and sending updates to channels +""" + +import asyncio +import aiohttp +import json +import time +import hashlib +import html +import re +from datetime import datetime, timezone +from typing import Dict, List, Optional, Any, Tuple +import sqlite3 +import feedparser +from urllib.parse import urlparse + + +class FeedManager: + """Manages RSS and API feed subscriptions""" + + def __init__(self, bot): + self.bot = bot + self.logger = bot.logger + self.db_path = bot.db_manager.db_path + + # Configuration + self.enabled = bot.config.getboolean('Feed_Manager', 'feed_manager_enabled', fallback=False) + self.default_check_interval = bot.config.getint('Feed_Manager', 'default_check_interval_seconds', fallback=300) + self.max_items_per_check = bot.config.getint('Feed_Manager', 'max_items_per_check', fallback=10) + self.request_timeout = bot.config.getint('Feed_Manager', 'feed_request_timeout', fallback=30) + self.user_agent = bot.config.get('Feed_Manager', 'feed_user_agent', fallback='MeshCoreBot/1.0 FeedManager') + self.rate_limit_seconds = bot.config.getfloat('Feed_Manager', 'feed_rate_limit_seconds', fallback=5.0) + self.max_message_length = bot.config.getint('Feed_Manager', 'max_message_length', fallback=130) + self.default_output_format = bot.config.get('Feed_Manager', 'default_output_format', fallback='{emoji} {body|truncate:100} - {date}\n{link|truncate:50}') + self.default_send_interval = bot.config.getfloat('Feed_Manager', 'default_message_send_interval_seconds', fallback=2.0) + + # Rate limiting per domain + self._domain_last_request: Dict[str, float] = {} + + # HTTP session + self.session: Optional[aiohttp.ClientSession] = None + + # Semaphore to limit concurrent requests + self._request_semaphore = asyncio.Semaphore(5) + + self.logger.info("FeedManager initialized") + + async def initialize(self): + """Initialize the feed manager (create HTTP session)""" + if not self.enabled: + self.logger.info("FeedManager is disabled in config") + return + + # Don't create session here - create it lazily when needed + # This avoids issues with using sessions across different event loops + # The session will be created in the same event loop where it's used + self.logger.info("FeedManager initialized (session will be created on first use)") + + async def stop(self): + """Stop the feed manager (close HTTP session)""" + if self.session and not self.session.closed: + await self.session.close() + self.session = None + self.logger.info("FeedManager stopped") + + async def poll_all_feeds(self): + """Poll all enabled feeds that are due for checking""" + if not self.enabled: + return + + try: + # Get all enabled feeds + feeds = self._get_enabled_feeds() + + if not feeds: + return + + # Filter feeds that are due for checking + current_time = time.time() + feeds_to_check = [] + + for feed in feeds: + last_check = feed.get('last_check_time') + if last_check: + try: + # Parse timestamp - handle both ISO format and SQLite format + if isinstance(last_check, str): + # Try ISO format first (with timezone) + try: + last_check_dt = datetime.fromisoformat(last_check.replace('Z', '+00:00')) + except ValueError: + # Try SQLite format (YYYY-MM-DD HH:MM:SS) - treat as UTC + try: + last_check_dt = datetime.strptime(last_check, '%Y-%m-%d %H:%M:%S') + last_check_dt = last_check_dt.replace(tzinfo=timezone.utc) + except ValueError: + # Try with microseconds + try: + last_check_dt = datetime.strptime(last_check, '%Y-%m-%d %H:%M:%S.%f') + last_check_dt = last_check_dt.replace(tzinfo=timezone.utc) + except ValueError: + raise ValueError(f"Unknown timestamp format: {last_check}") + else: + last_check_dt = datetime.fromtimestamp(last_check, tz=timezone.utc) + + # Convert to timestamp + if last_check_dt.tzinfo: + last_check_ts = last_check_dt.timestamp() + else: + # Assume UTC if no timezone + last_check_ts = last_check_dt.replace(tzinfo=timezone.utc).timestamp() + except Exception as e: + self.logger.debug(f"Error parsing last_check_time for feed {feed['id']}: {e}") + last_check_ts = 0 + else: + last_check_ts = 0 + + interval = feed.get('check_interval_seconds', self.default_check_interval) + + if current_time - last_check_ts >= interval: + feeds_to_check.append(feed) + + if not feeds_to_check: + self.logger.debug("No feeds due for checking at this time") + return + + self.logger.info(f"Polling {len(feeds_to_check)} feed(s) that are due for checking") + + # Poll feeds in parallel (with semaphore limit) + tasks = [self.poll_feed(feed) for feed in feeds_to_check] + await asyncio.gather(*tasks, return_exceptions=True) + + except Exception as e: + self.logger.error(f"Error in poll_all_feeds: {e}") + + async def _ensure_session(self): + """Ensure HTTP session exists in the current event loop""" + if self.session is None or self.session.closed: + # Create session in the current event loop context + self.session = aiohttp.ClientSession( + headers={'User-Agent': self.user_agent} + ) + self.logger.debug("Created FeedManager HTTP session in current event loop") + + async def poll_feed(self, feed: Dict[str, Any]): + """Poll a single feed and process new items""" + # Ensure session exists in current event loop + await self._ensure_session() + + feed_id = feed['id'] + feed_type = feed['feed_type'] + feed_url = feed['feed_url'] + channel_name = feed['channel_name'] + + try: + self.logger.debug(f"Polling {feed_type} feed {feed_id}: {feed_url}") + + # Rate limit per domain + domain = urlparse(feed_url).netloc + await self._wait_for_rate_limit(domain) + + # Fetch feed data + if feed_type == 'rss': + new_items = await self.process_rss_feed(feed) + elif feed_type == 'api': + new_items = await self.process_api_feed(feed) + else: + self.logger.warning(f"Unknown feed type: {feed_type}") + return + + # Process new items + if new_items: + self.logger.info(f"Found {len(new_items)} new items for feed {feed_id}") + filtered_count = 0 + for item in new_items[:self.max_items_per_check]: + # Check if item passes filter conditions + if self._should_send_item(feed, item): + await self._send_feed_item(feed, item) + else: + filtered_count += 1 + self.logger.debug(f"Filtered out item: {item.get('title', 'Untitled')[:50]}") + + if filtered_count > 0: + self.logger.debug(f"Filtered out {filtered_count} items for feed {feed_id}") + else: + self.logger.debug(f"No new items found for feed {feed_id}") + + # Always update last check time, even if no new items + self._update_feed_last_check(feed_id) + + except Exception as e: + self.logger.error(f"Error polling feed {feed_id}: {e}") + self._record_feed_error(feed_id, 'network', str(e)) + + async def process_rss_feed(self, feed: Dict[str, Any]) -> List[Dict[str, Any]]: + """Process an RSS feed and return new items""" + feed_url = feed['feed_url'] + last_item_id = feed.get('last_item_id') + + try: + # Fetch RSS feed - use aiohttp's timeout directly + # Create timeout object in the current async context + timeout = aiohttp.ClientTimeout(total=self.request_timeout) + + async with self._request_semaphore: + try: + async with self.session.get(feed_url, timeout=timeout) as response: + if response.status != 200: + raise Exception(f"HTTP {response.status}") + content = await response.text() + except (asyncio.TimeoutError, aiohttp.ServerTimeoutError): + raise Exception(f"Request timeout after {self.request_timeout} seconds") + + # Parse RSS feed + parsed = feedparser.parse(content) + + if parsed.bozo: + self.logger.warning(f"RSS feed parsing warning: {parsed.bozo_exception}") + + # Extract items - collect ALL items first (don't break early if sorting is configured) + all_items = [] + for entry in parsed.entries: + # Get item ID (prefer guid, then link, then hash of title+link) + item_id = entry.get('id') or entry.get('guid') or entry.get('link') + if not item_id: + # Generate ID from title and link + item_id = hashlib.md5( + f"{entry.get('title', '')}{entry.get('link', '')}".encode() + ).hexdigest() + + # Parse published date + published = None + if hasattr(entry, 'published_parsed') and entry.published_parsed: + try: + published = datetime(*entry.published_parsed[:6], tzinfo=timezone.utc) + except Exception: + pass + + all_items.append({ + 'id': item_id, + 'title': entry.get('title', 'Untitled'), + 'link': entry.get('link', ''), + 'description': entry.get('description', ''), + 'published': published + }) + + # Apply sorting if configured (before filtering, so we can properly track the last item) + sort_config_str = feed.get('sort_config') + if sort_config_str: + try: + sort_config = json.loads(sort_config_str) if isinstance(sort_config_str, str) else sort_config_str + all_items = self._sort_items(all_items, sort_config) + except (json.JSONDecodeError, TypeError, Exception) as e: + self.logger.warning(f"Error applying sort config for feed {feed['id']}: {e}") + + # Reverse to get oldest first (if no sort config) + if not sort_config_str: + all_items.reverse() + + # Now filter out items that have already been processed + # Check against both last_item_id and the feed_activity table for robust deduplication + items = [] + processed_item_ids = set() + + # Get all previously processed item IDs from feed_activity table + if last_item_id: + processed_item_ids.add(last_item_id) + + # Query database for all processed item IDs for this feed + try: + with sqlite3.connect(self.db_path) as conn: + cursor = conn.cursor() + cursor.execute(''' + SELECT DISTINCT item_id FROM feed_activity + WHERE feed_id = ? + ''', (feed['id'],)) + for row in cursor.fetchall(): + processed_item_ids.add(row[0]) + except Exception as e: + self.logger.debug(f"Error querying processed items for feed {feed['id']}: {e}") + + # Filter out already processed items + for item in all_items: + if item['id'] not in processed_item_ids: + items.append(item) + else: + self.logger.debug(f"Skipping already processed item {item['id']} for feed {feed['id']}") + + # Update last_item_id if we have new items (use the last item from the sorted list) + if items: + # Use the last item from the original sorted list (all_items), not the filtered list + # This ensures we track the most recent item even if it was already processed + self._update_feed_last_item_id(feed['id'], all_items[-1]['id']) + + return items + + except Exception as e: + self.logger.error(f"Error processing RSS feed: {e}") + raise + + async def process_api_feed(self, feed: Dict[str, Any]) -> List[Dict[str, Any]]: + """Process an API feed and return new items""" + feed_url = feed['feed_url'] + api_config_str = feed.get('api_config', '{}') + last_item_id = feed.get('last_item_id') + + try: + # Parse API config + api_config = json.loads(api_config_str) if api_config_str else {} + + method = api_config.get('method', 'GET').upper() + headers = api_config.get('headers', {}) + params = api_config.get('params', {}) + body = api_config.get('body') + parser_config = api_config.get('response_parser', {}) + + # Make HTTP request - use aiohttp's timeout directly + # Create timeout object in the current async context + timeout = aiohttp.ClientTimeout(total=self.request_timeout) + + async with self._request_semaphore: + try: + if method == 'POST': + async with self.session.post(feed_url, headers=headers, params=params, json=body, timeout=timeout) as response: + if response.status != 200: + raise Exception(f"HTTP {response.status}") + data = await response.json() + else: + async with self.session.get(feed_url, headers=headers, params=params, timeout=timeout) as response: + if response.status != 200: + raise Exception(f"HTTP {response.status}") + data = await response.json() + except (asyncio.TimeoutError, aiohttp.ServerTimeoutError): + raise Exception(f"Request timeout after {self.request_timeout} seconds") + + # Extract items using parser config + items_path = parser_config.get('items_path', '') + if items_path: + # Navigate JSON path + parts = items_path.split('.') + items_data = data + for part in parts: + items_data = items_data.get(part, []) + else: + # Assume data is a list + items_data = data if isinstance(data, list) else [data] + + # Extract items + id_field = parser_config.get('id_field', 'id') + title_field = parser_config.get('title_field', 'title') + description_field = parser_config.get('description_field', 'description') # New: allow custom description field + timestamp_field = parser_config.get('timestamp_field', 'created_at') + + # Collect ALL items first (don't break early, as sorting may reorder them) + all_items = [] + for item_data in items_data: + item_id = str(self._get_nested_value(item_data, id_field, '')) + if not item_id: + continue + + # Parse timestamp if available - support nested paths + published = None + if timestamp_field: + ts_value = self._get_nested_value(item_data, timestamp_field) + if ts_value: + try: + if isinstance(ts_value, (int, float)): + published = datetime.fromtimestamp(ts_value, tz=timezone.utc) + elif isinstance(ts_value, str): + # Try Microsoft date format first + if ts_value.startswith('/Date('): + published = self._parse_microsoft_date(ts_value) + else: + # Try ISO format + try: + published = datetime.fromisoformat(ts_value.replace('Z', '+00:00')) + except ValueError: + # Try common formats + for fmt in ['%Y-%m-%dT%H:%M:%S', '%Y-%m-%d %H:%M:%S', '%Y-%m-%d']: + try: + published = datetime.strptime(ts_value, fmt) + if published.tzinfo is None: + published = published.replace(tzinfo=timezone.utc) + break + except ValueError: + continue + except Exception: + pass + + # Get description - support nested paths + description = '' + if description_field: + desc_value = self._get_nested_value(item_data, description_field) + if desc_value: + description = str(desc_value) + + all_items.append({ + 'id': item_id, + 'title': self._get_nested_value(item_data, title_field, 'Untitled'), + 'link': item_data.get('link', ''), + 'description': description, + 'published': published, + 'raw': item_data # Store full raw response for field access + }) + + # Apply sorting if configured (before filtering, so we can properly track the last item) + sort_config_str = feed.get('sort_config') + if sort_config_str: + try: + sort_config = json.loads(sort_config_str) if isinstance(sort_config_str, str) else sort_config_str + all_items = self._sort_items(all_items, sort_config) + except (json.JSONDecodeError, TypeError, Exception) as e: + self.logger.warning(f"Error applying sort config for feed {feed['id']}: {e}") + + # Reverse to get oldest first (if no sort config) + if not sort_config_str: + all_items.reverse() + + # Now filter out items that have already been processed + # Check against both last_item_id and the feed_activity table for robust deduplication + items = [] + processed_item_ids = set() + + # Get all previously processed item IDs from feed_activity table + if last_item_id: + processed_item_ids.add(last_item_id) + + # Query database for all processed item IDs for this feed + try: + with sqlite3.connect(self.db_path) as conn: + cursor = conn.cursor() + cursor.execute(''' + SELECT DISTINCT item_id FROM feed_activity + WHERE feed_id = ? + ''', (feed['id'],)) + for row in cursor.fetchall(): + processed_item_ids.add(row[0]) + except Exception as e: + self.logger.debug(f"Error querying processed items for feed {feed['id']}: {e}") + + # Filter out already processed items + for item in all_items: + if item['id'] not in processed_item_ids: + items.append(item) + else: + self.logger.debug(f"Skipping already processed item {item['id']} for feed {feed['id']}") + + # Update last_item_id if we have new items (use the last item from the sorted list) + if items: + # Use the last item from the original sorted list (all_items), not the filtered list + # This ensures we track the most recent item even if it was already processed + self._update_feed_last_item_id(feed['id'], all_items[-1]['id']) + + return items + + except Exception as e: + self.logger.error(f"Error processing API feed: {e}") + raise + + def _format_timestamp(self, published: Optional[datetime]) -> str: + """Format a timestamp as a relative time string""" + if not published: + return "" + + try: + if published.tzinfo: + now = datetime.now(timezone.utc) + else: + now = datetime.now() + + diff = now - published + minutes = int(diff.total_seconds() / 60) + + if minutes < 1: + return "now" + elif minutes < 60: + return f"{minutes}m ago" + elif minutes < 1440: + hours = minutes // 60 + mins = minutes % 60 + return f"{hours}h {mins}m ago" + else: + days = minutes // 1440 + return f"{days}d ago" + except Exception: + return "" + + def _apply_shortening(self, text: str, function: str) -> str: + """Apply a shortening, parsing, or conditional function to text + + Supported functions: + - truncate:N - truncate to N characters + - word_wrap:N - wrap at N characters, breaking at word boundaries + - first_words:N - take first N words + - regex:pattern - extract using regex pattern (uses first capture group, or whole match) + - regex:pattern:group - extract specific capture group (0 = whole match, 1 = first group, etc.) + - if_regex:pattern:then:else - if pattern matches, return "then", else return "else" + """ + if not text: + return "" + + if function.startswith('truncate:'): + try: + max_len = int(function.split(':', 1)[1]) + if len(text) <= max_len: + return text + return text[:max_len] + "..." + except (ValueError, IndexError): + return text + + elif function.startswith('word_wrap:'): + try: + max_len = int(function.split(':', 1)[1]) + if len(text) <= max_len: + return text + # Find last space before max_len + truncated = text[:max_len] + last_space = truncated.rfind(' ') + if last_space > max_len * 0.7: # Only use word boundary if it's not too short + return truncated[:last_space] + "..." + return truncated + "..." + except (ValueError, IndexError): + return text + + elif function.startswith('first_words:'): + try: + num_words = int(function.split(':', 1)[1]) + words = text.split() + if len(words) <= num_words: + return text + return ' '.join(words[:num_words]) + "..." + except (ValueError, IndexError): + return text + + elif function.startswith('regex:'): + try: + # Parse regex pattern and optional group number + # Format: regex:pattern:group or regex:pattern + # Need to handle patterns that contain colons, so split from the right + remaining = function[6:] # Skip 'regex:' prefix + + # Try to find the last colon that's followed by a number (the group number) + # Look for pattern like :N at the end + last_colon_idx = remaining.rfind(':') + pattern = remaining + group_num = None + + if last_colon_idx > 0: + # Check if what's after the last colon is a number + potential_group = remaining[last_colon_idx + 1:] + if potential_group.isdigit(): + pattern = remaining[:last_colon_idx] + group_num = int(potential_group) + + if not pattern: + return text + + # Apply regex + match = re.search(pattern, text, re.IGNORECASE | re.DOTALL) + if match: + if group_num is not None: + # Use specified group (0 = whole match, 1 = first group, etc.) + if 0 <= group_num <= len(match.groups()): + return match.group(group_num) if group_num > 0 else match.group(0) + else: + # Use first capture group if available, otherwise whole match + if match.groups(): + return match.group(1) + else: + return match.group(0) + return "" # No match found + except (ValueError, IndexError, re.error) as e: + self.logger.debug(f"Error applying regex function: {e}") + return text + + elif function.startswith('if_regex:'): + try: + # Parse: if_regex:pattern:then:else + # Split by ':' but need to handle regex patterns that contain ':' + # Use a smarter split that respects the structure + parts = function[9:].split(':', 2) # Skip 'if_regex:' prefix, split into [pattern, then, else] + if len(parts) < 3: + return text + + pattern = parts[0] + then_value = parts[1] + else_value = parts[2] + + if not pattern: + return text + + # Check if pattern matches + match = re.search(pattern, text, re.IGNORECASE | re.DOTALL) + if match: + return then_value + else: + return else_value + except (ValueError, IndexError, re.error) as e: + self.logger.debug(f"Error applying if_regex function: {e}") + return text + + elif function.startswith('switch:'): + try: + # Parse: switch:value1:result1:value2:result2:...:default + # Example: switch:highest:🔴:high:🟠:medium:🟡:low:âšĒ:âšĒ + # This checks if text exactly matches value1, returns result1, etc., or default + parts = function[7:].split(':') # Skip 'switch:' prefix + if len(parts) < 2: + return text + + # Pairs of value:result, last one is default + text_lower = text.lower().strip() + for i in range(0, len(parts) - 1, 2): + if i + 1 < len(parts): + value = parts[i].lower() + result = parts[i + 1] + if text_lower == value: + return result + + # Return last part as default if no match + return parts[-1] if parts else text + except (ValueError, IndexError) as e: + self.logger.debug(f"Error applying switch function: {e}") + return text + + elif function.startswith('regex_cond:'): + try: + # Parse: regex_cond:extract_pattern:check_pattern:then:group + # This extracts text using extract_pattern, then checks if it matches check_pattern + # If check_pattern matches, return "then", else return the extracted text + # Example: regex_cond:Northbound\s*\n([^\n]+):No restrictions:👍:1 + # This extracts text after "Northbound\n" up to next newline, checks if it's "No restrictions", + # if yes returns "👍", else returns the extracted text + parts = function[11:].split(':', 3) # Skip 'regex_cond:' prefix + if len(parts) < 4: + return text + + extract_pattern = parts[0] + check_pattern = parts[1] + then_value = parts[2] + else_group = int(parts[3]) if parts[3].isdigit() else 1 + + if not extract_pattern: + return text + + # Extract using extract_pattern + match = re.search(extract_pattern, text, re.IGNORECASE | re.DOTALL) + if match: + # Get the captured group + if match.groups(): + extracted = match.group(else_group) if else_group <= len(match.groups()) else match.group(1) + # Strip whitespace from extracted text + extracted = extracted.strip() + else: + extracted = match.group(0).strip() + + # Check if extracted text matches check_pattern (exact match or contains) + if check_pattern: + # Try exact match first, then substring match + if extracted.lower() == check_pattern.lower() or re.search(check_pattern, extracted, re.IGNORECASE): + return then_value + + return extracted + return "" # No match found + except (ValueError, IndexError, re.error) as e: + self.logger.debug(f"Error applying regex_cond function: {e}") + return text + + return text + + def _get_nested_value(self, data: Any, path: str, default: Any = '') -> Any: + """Get a nested value from a dict/list using dot notation (e.g., 'raw.Priority' or 'raw.StartRoadwayLocation.RoadName')""" + if not path or not data: + return default + + parts = path.split('.') + value = data + + for part in parts: + if isinstance(value, dict): + value = value.get(part) + elif isinstance(value, list): + try: + idx = int(part) + if 0 <= idx < len(value): + value = value[idx] + else: + return default + except (ValueError, TypeError): + return default + else: + return default + + if value is None: + return default + + return value if value is not None else default + + def _parse_microsoft_date(self, date_str: str) -> Optional[datetime]: + """Parse Microsoft JSON date format: /Date(timestamp-offset)/""" + if not date_str or not isinstance(date_str, str): + return None + + # Match /Date(timestamp-offset)/ format + match = re.match(r'/Date\((\d+)([+-]\d+)?\)/', date_str) + if match: + timestamp_ms = int(match.group(1)) + offset_str = match.group(2) if match.group(2) else '+0000' + + # Convert milliseconds to seconds + timestamp = timestamp_ms / 1000.0 + + # Parse offset (format: +0800 or -0800) + try: + offset_hours = int(offset_str[:3]) + offset_mins = int(offset_str[3:5]) + offset_seconds = (offset_hours * 3600) + (offset_mins * 60) + if offset_str[0] == '-': + offset_seconds = -offset_seconds + + # Create timezone-aware datetime + tz = timezone.utc + if offset_seconds != 0: + from datetime import timedelta + tz = timezone(timedelta(seconds=offset_seconds)) + + return datetime.fromtimestamp(timestamp, tz=tz) + except (ValueError, IndexError): + # Fallback to UTC if offset parsing fails + return datetime.fromtimestamp(timestamp, tz=timezone.utc) + + return None + + def _sort_items(self, items: List[Dict[str, Any]], sort_config: dict) -> List[Dict[str, Any]]: + """Sort items based on sort configuration + + Sort config format: + { + "field": "raw.LastUpdatedTime", # Field path to sort by + "order": "desc" # "asc" or "desc" + } + """ + if not sort_config or not items: + return items + + field_path = sort_config.get('field') + order = sort_config.get('order', 'desc').lower() + + if not field_path: + return items + + def get_sort_value(item): + """Get the sort value for an item""" + # Try raw data first + raw_data = item.get('raw', {}) + value = self._get_nested_value(raw_data, field_path, '') + + if not value and field_path.startswith('raw.'): + value = self._get_nested_value(raw_data, field_path[4:], '') + + if not value: + value = self._get_nested_value(item, field_path, '') + + # Handle Microsoft date format + if isinstance(value, str) and value.startswith('/Date('): + dt = self._parse_microsoft_date(value) + if dt: + return dt.timestamp() + + # Handle datetime objects + if isinstance(value, datetime): + return value.timestamp() + + # Handle numeric values + if isinstance(value, (int, float)): + return float(value) + + # Handle string timestamps + if isinstance(value, str): + # Try to parse as ISO format + try: + dt = datetime.fromisoformat(value.replace('Z', '+00:00')) + return dt.timestamp() + except ValueError: + pass + + # Try common date formats + for fmt in ['%Y-%m-%dT%H:%M:%S', '%Y-%m-%d %H:%M:%S', '%Y-%m-%d']: + try: + dt = datetime.strptime(value, fmt) + return dt.timestamp() + except ValueError: + continue + + # For strings, use lexicographic comparison + return str(value) + + # Sort items + try: + sorted_items = sorted(items, key=get_sort_value, reverse=(order == 'desc')) + return sorted_items + except Exception as e: + self.logger.warning(f"Error sorting items: {e}") + return items + + def format_message(self, item: Dict[str, Any], feed: Dict[str, Any]) -> str: + """Format a feed item as a message for the mesh using configurable format with placeholders + + Supported placeholders: + - {title} - item title + - {body} - item description/body + - {date} - relative time (e.g., "5m ago") + - {link} - item link URL + - {emoji} - emoji based on feed type + - {raw.field} - access any field from raw API response (e.g., {raw.Priority}, {raw.StartRoadwayLocation.RoadName}) + + Supported shortening functions: + - {field|truncate:N} - truncate to N characters + - {field|word_wrap:N} - wrap at N characters + - {field|first_words:N} - take first N words + - {field|regex:pattern} - extract using regex (first group or whole match) + - {field|regex:pattern:group} - extract specific capture group + - {field|if_regex:pattern:then:else} - if pattern matches, return "then", else "else" + - {field|switch:value1:result1:value2:result2:...:default} - exact match switch (e.g., switch:highest:🔴:high:🟠:medium:🟡:âšĒ) + - {field|regex_cond:extract_pattern:check_pattern:then:group} - extract text, check if it matches check_pattern, return "then" if match else extracted text + """ + + # Get format string from feed config or use default + format_str = feed.get('output_format') or self.default_output_format + + # Extract field values + title = item.get('title', 'Untitled') + body = item.get('description', '') or item.get('body', '') + # Clean HTML from body if present + if body: + import html + body = html.unescape(body) + # Convert line break tags to newlines before stripping other HTML + # Handle
,
,
,
, etc. + body = re.sub(r'', '\n', body, flags=re.IGNORECASE) + # Convert paragraph tags to newlines (with spacing) + body = re.sub(r'

', '\n\n', body, flags=re.IGNORECASE) + body = re.sub(r']*>', '', body, flags=re.IGNORECASE) + # Remove remaining HTML tags + body = re.sub(r'<[^>]+>', '', body) + # Clean up whitespace (preserve intentional line breaks) + # Replace multiple newlines with double newline, then normalize spaces within lines + body = re.sub(r'\n\s*\n\s*\n+', '\n\n', body) # Multiple newlines -> double newline + lines = body.split('\n') + body = '\n'.join(' '.join(line.split()) for line in lines) # Normalize spaces per line + body = body.strip() + + link = item.get('link', '') + published = item.get('published') + date_str = self._format_timestamp(published) + + # Choose emoji based on feed type or content + emoji = "đŸ“ĸ" + feed_name = feed.get('feed_name', '').lower() + if 'emergency' in feed_name or 'alert' in feed_name: + emoji = "🚨" + elif 'warning' in feed_name: + emoji = "⚠ī¸" + elif 'info' in feed_name or 'news' in feed_name: + emoji = "ℹī¸" + + # Build replacement dictionary + replacements = { + 'title': title, + 'body': body, + 'date': date_str, + 'link': link, + 'emoji': emoji + } + + # Get raw API data if available + raw_data = item.get('raw', {}) + + # Process format string with placeholders and functions + # Pattern: {field|function} or {field} or {raw.field.path} + def replace_placeholder(match): + full_match = match.group(0) + content = match.group(1) # Content inside {} + + if '|' in content: + field_name, function = content.split('|', 1) + field_name = field_name.strip() + function = function.strip() + + # Check if it's a raw field access + if field_name.startswith('raw.'): + value = str(self._get_nested_value(raw_data, field_name[4:], '')) + else: + value = replacements.get(field_name, '') + + return self._apply_shortening(value, function) + else: + field_name = content.strip() + + # Check if it's a raw field access + if field_name.startswith('raw.'): + value = self._get_nested_value(raw_data, field_name[4:], '') + # Convert to string, handling None and complex types + if value is None: + return '' + elif isinstance(value, (dict, list)): + # For complex types, convert to JSON string + try: + return json.dumps(value) + except Exception: + return str(value) + else: + return str(value) + else: + return replacements.get(field_name, '') + + # Replace all placeholders + message = re.sub(r'\{([^}]+)\}', replace_placeholder, format_str) + + # Final truncation if message is too long + if len(message) > self.max_message_length: + # Try to preserve structure by truncating at newline if possible + lines = message.split('\n') + if len(lines) > 1: + # Truncate last line + total_length = sum(len(line) + 1 for line in lines[:-1]) # +1 for newline + remaining = self.max_message_length - total_length - 3 # -3 for "..." + if remaining > 20: + lines[-1] = lines[-1][:remaining] + "..." + message = '\n'.join(lines) + else: + # Just truncate everything + message = message[:self.max_message_length - 3] + "..." + else: + message = message[:self.max_message_length - 3] + "..." + + return message + + def _queue_feed_message(self, feed: Dict[str, Any], item: Dict[str, Any], message: str): + """Queue a feed message for later sending""" + try: + with sqlite3.connect(self.db_path) as conn: + cursor = conn.cursor() + cursor.execute(''' + INSERT INTO feed_message_queue + (feed_id, channel_name, message, item_id, item_title, priority) + VALUES (?, ?, ?, ?, ?, 0) + ''', ( + feed['id'], + feed['channel_name'], + message, + item.get('id', ''), + item.get('title', '')[:200] # Limit title length + )) + conn.commit() + self.logger.debug(f"Queued feed message for {feed['channel_name']}: {item.get('title', '')[:50]}") + except Exception as e: + self.logger.error(f"Error queuing feed message: {e}") + self._record_feed_error(feed['id'], 'queue', str(e)) + + def _should_send_item(self, feed: Dict[str, Any], item: Dict[str, Any]) -> bool: + """Check if an item should be sent based on filter configuration + + Filter config format: + { + "conditions": [ + {"field": "Priority", "operator": "in", "values": ["highest", "high"]}, + {"field": "EventStatus", "operator": "equals", "value": "open"}, + {"field": "EventCategory", "operator": "not_equals", "value": "Maintenance"}, + {"field": "raw.Priority", "operator": "matches", "pattern": "^(highest|high)$"} + ], + "logic": "AND" # or "OR" + } + + Supported operators: + - equals: exact match + - not_equals: not exact match + - in: value is in list + - not_in: value is not in list + - matches: regex match + - not_matches: regex doesn't match + - contains: substring match + - not_contains: substring doesn't match + """ + filter_config_str = feed.get('filter_config') + if not filter_config_str: + # No filter configured, send all items + return True + + try: + filter_config = json.loads(filter_config_str) if isinstance(filter_config_str, str) else filter_config_str + except (json.JSONDecodeError, TypeError): + self.logger.warning(f"Invalid filter_config for feed {feed['id']}, sending all items") + return True + + conditions = filter_config.get('conditions', []) + if not conditions: + # Empty conditions, send all items + return True + + logic = filter_config.get('logic', 'AND').upper() + + # Get raw data for field access + raw_data = item.get('raw', {}) + + # Evaluate each condition + results = [] + for condition in conditions: + field_path = condition.get('field') + operator = condition.get('operator', 'equals') + + if not field_path: + # Invalid condition, skip it + continue + + # Get field value using nested access + field_value = self._get_nested_value(raw_data, field_path, '') + if not field_value and field_path.startswith('raw.'): + # Try without 'raw.' prefix + field_value = self._get_nested_value(raw_data, field_path[4:], '') + + # If still not found, try top-level item fields + if not field_value: + field_value = self._get_nested_value(item, field_path, '') + + # Convert to string for comparison + field_value_str = str(field_value).lower() if field_value is not None else '' + + # Evaluate condition + result = False + if operator == 'equals': + compare_value = str(condition.get('value', '')).lower() + result = field_value_str == compare_value + elif operator == 'not_equals': + compare_value = str(condition.get('value', '')).lower() + result = field_value_str != compare_value + elif operator == 'in': + values = [str(v).lower() for v in condition.get('values', [])] + result = field_value_str in values + elif operator == 'not_in': + values = [str(v).lower() for v in condition.get('values', [])] + result = field_value_str not in values + elif operator == 'matches': + pattern = condition.get('pattern', '') + if pattern: + try: + result = bool(re.search(pattern, str(field_value), re.IGNORECASE)) + except re.error: + result = False + elif operator == 'not_matches': + pattern = condition.get('pattern', '') + if pattern: + try: + result = not bool(re.search(pattern, str(field_value), re.IGNORECASE)) + except re.error: + result = True + elif operator == 'contains': + compare_value = str(condition.get('value', '')).lower() + result = compare_value in field_value_str + elif operator == 'not_contains': + compare_value = str(condition.get('value', '')).lower() + result = compare_value not in field_value_str + else: + self.logger.warning(f"Unknown filter operator: {operator}") + result = True # Default to allowing if operator is unknown + + results.append(result) + + # Apply logic (AND or OR) + if logic == 'OR': + return any(results) + else: # AND (default) + return all(results) + + async def _send_feed_item(self, feed: Dict[str, Any], item: Dict[str, Any]): + """Queue a feed item message instead of sending immediately""" + try: + message = self.format_message(item, feed) + # Queue the message instead of sending immediately + self._queue_feed_message(feed, item, message) + except Exception as e: + self.logger.error(f"Error processing feed item: {e}") + self._record_feed_error(feed['id'], 'other', str(e)) + + async def _wait_for_rate_limit(self, domain: str): + """Wait if needed to respect rate limits""" + if domain in self._domain_last_request: + last_request = self._domain_last_request[domain] + elapsed = time.time() - last_request + if elapsed < self.rate_limit_seconds: + wait_time = self.rate_limit_seconds - elapsed + await asyncio.sleep(wait_time) + + self._domain_last_request[domain] = time.time() + + def _get_enabled_feeds(self) -> List[Dict[str, Any]]: + """Get all enabled feed subscriptions from database""" + try: + with sqlite3.connect(self.db_path) as conn: + conn.row_factory = sqlite3.Row + cursor = conn.cursor() + cursor.execute(''' + SELECT * FROM feed_subscriptions + WHERE enabled = 1 + ORDER BY last_check_time ASC NULLS FIRST + ''') + rows = cursor.fetchall() + return [dict(row) for row in rows] + except Exception as e: + self.logger.error(f"Error getting enabled feeds: {e}") + return [] + + def _update_feed_last_check(self, feed_id: int): + """Update the last check time for a feed""" + try: + from datetime import datetime, timezone + # Use Python's datetime to ensure proper timezone handling + # Store in ISO format with timezone for JavaScript compatibility + now = datetime.now(timezone.utc) + now_str = now.isoformat() # ISO format: 2025-12-05T12:34:56.789+00:00 + + with sqlite3.connect(self.db_path) as conn: + cursor = conn.cursor() + cursor.execute(''' + UPDATE feed_subscriptions + SET last_check_time = ?, + updated_at = ? + WHERE id = ? + ''', (now_str, now_str, feed_id)) + conn.commit() + self.logger.debug(f"Updated last_check_time for feed {feed_id} to {now_str}") + except Exception as e: + self.logger.error(f"Error updating feed last check: {e}") + + def _update_feed_last_item_id(self, feed_id: int, item_id: str): + """Update the last processed item ID for a feed""" + try: + with sqlite3.connect(self.db_path) as conn: + cursor = conn.cursor() + cursor.execute(''' + UPDATE feed_subscriptions + SET last_item_id = ?, + updated_at = CURRENT_TIMESTAMP + WHERE id = ? + ''', (item_id, feed_id)) + conn.commit() + except Exception as e: + self.logger.error(f"Error updating feed last item ID: {e}") + + def _record_feed_activity(self, feed_id: int, item_id: str, item_title: str): + """Record that a feed item was processed""" + try: + with sqlite3.connect(self.db_path) as conn: + cursor = conn.cursor() + cursor.execute(''' + INSERT INTO feed_activity (feed_id, item_id, item_title, message_sent) + VALUES (?, ?, ?, 1) + ''', (feed_id, item_id, item_title[:200])) # Limit title length + conn.commit() + except Exception as e: + self.logger.error(f"Error recording feed activity: {e}") + + def _record_feed_error(self, feed_id: int, error_type: str, error_message: str): + """Record a feed error""" + try: + with sqlite3.connect(self.db_path) as conn: + cursor = conn.cursor() + cursor.execute(''' + INSERT INTO feed_errors (feed_id, error_type, error_message) + VALUES (?, ?, ?) + ''', (feed_id, error_type, error_message[:500])) # Limit message length + conn.commit() + except Exception as e: + self.logger.error(f"Error recording feed error: {e}") + + async def process_message_queue(self): + """Process queued feed messages and send them at configured intervals""" + try: + # Get all unsent messages, ordered by priority and queue time + with sqlite3.connect(self.db_path) as conn: + conn.row_factory = sqlite3.Row + cursor = conn.cursor() + cursor.execute(''' + SELECT q.id, q.feed_id, q.channel_name, q.message, q.item_id, q.item_title, + f.message_send_interval_seconds + FROM feed_message_queue q + JOIN feed_subscriptions f ON q.feed_id = f.id + WHERE q.sent_at IS NULL + ORDER BY q.priority DESC, q.queued_at ASC + LIMIT 100 + ''') + messages = cursor.fetchall() + + if not messages: + return + + # Group messages by feed to respect per-feed send intervals + feed_last_send: Dict[int, float] = {} + + for msg in messages: + feed_id = msg['feed_id'] + channel_name = msg['channel_name'] + message_text = msg['message'] + queue_id = msg['id'] + item_id = msg['item_id'] + item_title = msg['item_title'] + + # Get send interval for this feed (default if not set) + send_interval = msg['message_send_interval_seconds'] or self.default_send_interval + + # Check if we need to wait before sending this feed's message + if feed_id in feed_last_send: + elapsed = time.time() - feed_last_send[feed_id] + if elapsed < send_interval: + wait_time = send_interval - elapsed + await asyncio.sleep(wait_time) + + # Send the message + try: + success = await self.bot.command_manager.send_channel_message(channel_name, message_text) + + if success: + # Mark as sent + with sqlite3.connect(self.db_path) as conn: + cursor = conn.cursor() + cursor.execute(''' + UPDATE feed_message_queue + SET sent_at = CURRENT_TIMESTAMP + WHERE id = ? + ''', (queue_id,)) + conn.commit() + + # Record activity + self._record_feed_activity(feed_id, item_id, item_title) + self.logger.debug(f"Sent queued feed message to {channel_name}: {item_title[:50]}") + feed_last_send[feed_id] = time.time() + else: + self.logger.warning(f"Failed to send queued feed message to channel {channel_name}") + self._record_feed_error(feed_id, 'channel', f"Failed to send to channel {channel_name}") + # Don't mark as sent, will retry later + + except Exception as e: + self.logger.error(f"Error sending queued feed message: {e}") + self._record_feed_error(feed_id, 'other', str(e)) + # Don't mark as sent, will retry later + + except Exception as e: + self.logger.error(f"Error processing message queue: {e}") + diff --git a/modules/message_handler.py b/modules/message_handler.py index 2aed5ab..c6c302c 100644 --- a/modules/message_handler.py +++ b/modules/message_handler.py @@ -1562,9 +1562,29 @@ class MessageHandler: # This allows greeter to work on its own configured channels even if not in monitor_channels if 'greeter' in self.bot.command_manager.commands: greeter_command = self.bot.command_manager.commands['greeter'] + # First, check if this message should cancel a pending greeting (human greeting detection) + if greeter_command: + greeter_command.check_message_for_human_greeting(message) + # Then check if we should greet this user if greeter_command and greeter_command.should_execute(message): try: - await greeter_command.execute(message) + success = await greeter_command.execute(message) + + # Small delay to ensure send_response has completed + await asyncio.sleep(0.1) + + # Determine if a response was sent + response_sent = False + if hasattr(greeter_command, 'last_response') and greeter_command.last_response: + response_sent = True + elif hasattr(self.bot.command_manager, '_last_response') and self.bot.command_manager._last_response: + response_sent = True + + # Record command execution in stats database + if 'stats' in self.bot.command_manager.commands: + stats_command = self.bot.command_manager.commands['stats'] + if stats_command: + stats_command.record_command(message, 'greeter', response_sent) except Exception as e: self.logger.error(f"Error executing greeter command: {e}") @@ -1593,15 +1613,6 @@ class MessageHandler: else: self.logger.info(f"Keyword '{keyword}' matched, responding") - # Record command execution in stats database - if 'stats' in self.bot.command_manager.commands: - stats_command = self.bot.command_manager.commands['stats'] - if stats_command: - stats_command.record_command(message, keyword, response is not None) - - # Note: Command data capture is handled in command_manager.py after execution - # to avoid duplicate messages to web viewer - # Track if this is a help response if keyword == 'help': help_response_sent = True @@ -1611,9 +1622,21 @@ class MessageHandler: plugin_command_with_response_matched = True # Skip commands that handle their own responses (response is None) + # These will be recorded when they execute via execute_commands if response is None: continue + # Record command execution in stats database for keyword-matched commands with responses + # Commands without responses (response is None) are recorded in execute_commands to avoid double-counting + if 'stats' in self.bot.command_manager.commands: + stats_command = self.bot.command_manager.commands['stats'] + if stats_command: + # response is not None here, so we know a response will be sent + stats_command.record_command(message, keyword, True) + + # Note: Command data capture is handled in command_manager.py after execution + # to avoid duplicate messages to web viewer + # Send response if message.is_dm: await self.bot.command_manager.send_dm(message.sender_id, response) diff --git a/modules/scheduler.py b/modules/scheduler.py index f066762..e452a87 100644 --- a/modules/scheduler.py +++ b/modules/scheduler.py @@ -9,6 +9,8 @@ import threading import schedule import datetime import pytz +import sqlite3 +import json from typing import Dict, Tuple @@ -121,6 +123,7 @@ class MessageScheduler: """Run the scheduler in a separate thread""" self.logger.info("Scheduler thread started") last_log_time = 0 + last_feed_poll_time = 0 while self.bot.connected: current_time = self.get_current_time() @@ -138,6 +141,83 @@ class MessageScheduler: # Check for interval-based advertising self.check_interval_advertising() + # Poll feeds every minute (but feeds themselves control their check intervals) + if time.time() - last_feed_poll_time >= 60: # Every 60 seconds + if (hasattr(self.bot, 'feed_manager') and self.bot.feed_manager and + hasattr(self.bot.feed_manager, 'enabled') and self.bot.feed_manager.enabled and + hasattr(self.bot, 'connected') and self.bot.connected): + # Run feed polling in async context + import asyncio + try: + loop = asyncio.get_event_loop() + except RuntimeError: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + # Schedule feed polling + try: + loop.run_until_complete(self.bot.feed_manager.poll_all_feeds()) + self.logger.debug("Feed polling cycle completed") + except Exception as e: + self.logger.error(f"Error in feed polling cycle: {e}") + last_feed_poll_time = time.time() + + # Periodically refresh channels from device (every hour) to prevent stale data + if not hasattr(self, 'last_channel_refresh_time'): + # Initialize to current time so it doesn't run immediately on startup + self.last_channel_refresh_time = time.time() + + channel_refresh_interval = self.bot.config.getint('Bot', 'channel_refresh_interval_seconds', fallback=3600) # Default 1 hour + if time.time() - self.last_channel_refresh_time >= channel_refresh_interval: + if (hasattr(self.bot, 'channel_manager') and self.bot.channel_manager and + hasattr(self.bot, 'connected') and self.bot.connected): + # Refresh channels from device + import asyncio + try: + loop = asyncio.get_event_loop() + except RuntimeError: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + # Fetch channels and update database + self.bot.logger.debug("Periodic channel refresh: fetching channels from device") + loop.run_until_complete(self.bot.channel_manager.fetch_all_channels(force_refresh=True)) + self.last_channel_refresh_time = time.time() + + # Process pending channel operations from web viewer (every 5 seconds) + if not hasattr(self, 'last_channel_ops_check_time'): + self.last_channel_ops_check_time = 0 + + if time.time() - self.last_channel_ops_check_time >= 5: # Every 5 seconds + if (hasattr(self.bot, 'channel_manager') and self.bot.channel_manager and + hasattr(self.bot, 'connected') and self.bot.connected): + import asyncio + try: + loop = asyncio.get_event_loop() + except RuntimeError: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + loop.run_until_complete(self._process_channel_operations()) + self.last_channel_ops_check_time = time.time() + + # Process feed message queue (every 2 seconds) + if not hasattr(self, 'last_message_queue_check_time'): + self.last_message_queue_check_time = 0 + + if time.time() - self.last_message_queue_check_time >= 2: # Every 2 seconds + if (hasattr(self.bot, 'feed_manager') and self.bot.feed_manager and + hasattr(self.bot, 'connected') and self.bot.connected): + import asyncio + try: + loop = asyncio.get_event_loop() + except RuntimeError: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + loop.run_until_complete(self.bot.feed_manager.process_message_queue()) + self.last_message_queue_check_time = time.time() + schedule.run_pending() time.sleep(1) @@ -194,3 +274,108 @@ class MessageScheduler: self.logger.info("Interval-based flood advert sent successfully") except Exception as e: self.logger.error(f"Error sending interval-based advert: {e}") + + async def _process_channel_operations(self): + """Process pending channel operations from the web viewer""" + try: + db_path = self.bot.db_manager.db_path + + # Get pending operations + with sqlite3.connect(db_path) as conn: + conn.row_factory = sqlite3.Row + cursor = conn.cursor() + + cursor.execute(''' + SELECT id, operation_type, channel_idx, channel_name, channel_key_hex + FROM channel_operations + WHERE status = 'pending' + ORDER BY created_at ASC + LIMIT 10 + ''') + + operations = cursor.fetchall() + + if not operations: + return + + self.logger.info(f"Processing {len(operations)} pending channel operation(s)") + + for op in operations: + op_id = op['id'] + op_type = op['operation_type'] + channel_idx = op['channel_idx'] + channel_name = op['channel_name'] + channel_key_hex = op['channel_key_hex'] + + try: + success = False + error_msg = None + + if op_type == 'add': + # Add channel + if channel_key_hex: + # Custom channel with key + channel_secret = bytes.fromhex(channel_key_hex) + success = await self.bot.channel_manager.add_channel( + channel_idx, channel_name, channel_secret=channel_secret + ) + else: + # Hashtag channel (firmware generates key) + success = await self.bot.channel_manager.add_channel( + channel_idx, channel_name + ) + + if success: + self.logger.info(f"Successfully processed channel add operation: {channel_name} at index {channel_idx}") + else: + error_msg = "Failed to add channel" + + elif op_type == 'remove': + # Remove channel + success = await self.bot.channel_manager.remove_channel(channel_idx) + + if success: + self.logger.info(f"Successfully processed channel remove operation: index {channel_idx}") + else: + error_msg = "Failed to remove channel" + + # Update operation status + with sqlite3.connect(db_path) as conn: + cursor = conn.cursor() + if success: + cursor.execute(''' + UPDATE channel_operations + SET status = 'completed', + processed_at = CURRENT_TIMESTAMP, + result_data = ? + WHERE id = ? + ''', (json.dumps({'success': True}), op_id)) + else: + cursor.execute(''' + UPDATE channel_operations + SET status = 'failed', + processed_at = CURRENT_TIMESTAMP, + error_message = ? + WHERE id = ? + ''', (error_msg or 'Unknown error', op_id)) + conn.commit() + + except Exception as e: + self.logger.error(f"Error processing channel operation {op_id}: {e}") + # Mark as failed + try: + with sqlite3.connect(db_path) as conn: + cursor = conn.cursor() + cursor.execute(''' + UPDATE channel_operations + SET status = 'failed', + processed_at = CURRENT_TIMESTAMP, + error_message = ? + WHERE id = ? + ''', (str(e), op_id)) + conn.commit() + except Exception as update_error: + self.logger.error(f"Error updating operation status: {update_error}") + + except Exception as e: + self.logger.error(f"Error in _process_channel_operations: {e}") diff --git a/modules/web_viewer/app.py b/modules/web_viewer/app.py index e96d4aa..8b3e4c0 100644 --- a/modules/web_viewer/app.py +++ b/modules/web_viewer/app.py @@ -63,6 +63,9 @@ class BotDataViewer: # Load configuration self.config = self._load_config(config_path) + # Setup template context processor for global template variables + self._setup_template_context() + # Initialize databases self._init_databases() @@ -116,6 +119,25 @@ class BotDataViewer: config.read(config_path) return config + def _setup_template_context(self): + """Setup template context processor to inject global variables""" + @self.app.context_processor + def inject_template_vars(): + """Inject variables available to all templates""" + # Check if greeter is enabled, defaulting to False if section doesn't exist + try: + greeter_enabled = self.config.getboolean('Greeter_Command', 'enabled', fallback=False) + except (configparser.NoSectionError, configparser.NoOptionError): + greeter_enabled = False + + # Check if feed manager is enabled, defaulting to False if section doesn't exist + try: + feed_manager_enabled = self.config.getboolean('Feed_Manager', 'feed_manager_enabled', fallback=False) + except (configparser.NoSectionError, configparser.NoOptionError): + feed_manager_enabled = False + + return dict(greeter_enabled=greeter_enabled, feed_manager_enabled=feed_manager_enabled) + def _init_databases(self): """Initialize database connections""" try: @@ -151,12 +173,13 @@ class BotDataViewer: def _init_packet_stream_table(self): """Initialize the packet_stream table in bot_data.db""" + conn = None try: # Get database path from config db_path = self.config.get('Database', 'path', fallback='bot_data.db') # Connect to database and create table if it doesn't exist - conn = sqlite3.connect(db_path) + conn = sqlite3.connect(db_path, timeout=30) cursor = conn.cursor() # Create packet_stream table with schema matching the INSERT statements @@ -182,13 +205,18 @@ class BotDataViewer: ''') conn.commit() - conn.close() self.logger.info(f"Initialized packet_stream table in {db_path}") except Exception as e: self.logger.error(f"Failed to initialize packet_stream table: {e}") # Don't raise - allow web viewer to continue even if table init fails + finally: + if conn: + try: + conn.close() + except Exception as e: + self.logger.debug(f"Error closing init connection: {e}") def _get_db_connection(self): """Get database connection - create new connection for each request to avoid threading issues""" @@ -234,6 +262,16 @@ class BotDataViewer: """Greeter management page""" return render_template('greeter.html') + @self.app.route('/feeds') + def feeds(): + """Feed management page""" + return render_template('feeds.html') + + @self.app.route('/radio') + def radio(): + """Radio settings page""" + return render_template('radio.html') + # API Routes @self.app.route('/api/health') @@ -338,6 +376,7 @@ class BotDataViewer: @self.app.route('/api/recent_commands') def api_recent_commands(): """API endpoint to get recent commands from database""" + conn = None try: import sqlite3 import json @@ -349,7 +388,7 @@ class BotDataViewer: # Get database path db_path = self.config.get('Database', 'path', fallback='bot_data.db') - conn = sqlite3.connect(db_path) + conn = sqlite3.connect(db_path, timeout=30) cursor = conn.cursor() cursor.execute(''' @@ -360,7 +399,6 @@ class BotDataViewer: ''', (cutoff_time,)) rows = cursor.fetchall() - conn.close() # Parse and return commands commands = [] @@ -376,6 +414,12 @@ class BotDataViewer: except Exception as e: self.logger.error(f"Error getting recent commands: {e}") return jsonify({'error': str(e)}), 500 + finally: + if conn: + try: + conn.close() + except Exception as e: + self.logger.debug(f"Error closing recent_commands connection: {e}") @self.app.route('/api/geocode-contact', methods=['POST']) def api_geocode_contact(): @@ -876,6 +920,358 @@ class BotDataViewer: except Exception as e: self.logger.error(f"Error ungreeting user: {e}", exc_info=True) return jsonify({'success': False, 'error': str(e)}), 500 + + # Feed management API endpoints + @self.app.route('/api/feeds') + def api_feeds(): + """Get all feed subscriptions with statistics""" + try: + feeds = self._get_feed_subscriptions() + return jsonify(feeds) + except Exception as e: + self.logger.error(f"Error getting feeds: {e}") + return jsonify({'error': str(e)}), 500 + + @self.app.route('/api/feeds/') + def api_feed_detail(feed_id): + """Get detailed information about a specific feed""" + try: + feed = self._get_feed_subscription(feed_id) + if not feed: + return jsonify({'error': 'Feed not found'}), 404 + + # Get activity and errors + activity = self._get_feed_activity(feed_id) + errors = self._get_feed_errors(feed_id) + + feed['activity'] = activity + feed['errors'] = errors + + return jsonify(feed) + except Exception as e: + self.logger.error(f"Error getting feed detail: {e}") + return jsonify({'error': str(e)}), 500 + + @self.app.route('/api/feeds', methods=['POST']) + def api_create_feed(): + """Create a new feed subscription""" + try: + data = request.get_json() + if not data: + return jsonify({'error': 'No data provided'}), 400 + + feed_id = self._create_feed_subscription(data) + return jsonify({'success': True, 'id': feed_id}) + except Exception as e: + self.logger.error(f"Error creating feed: {e}") + return jsonify({'error': str(e)}), 500 + + @self.app.route('/api/feeds/', methods=['PUT']) + def api_update_feed(feed_id): + """Update an existing feed subscription""" + try: + data = request.get_json() + if not data: + return jsonify({'error': 'No data provided'}), 400 + + success = self._update_feed_subscription(feed_id, data) + if not success: + return jsonify({'error': 'Feed not found'}), 404 + + return jsonify({'success': True}) + except Exception as e: + self.logger.error(f"Error updating feed: {e}") + return jsonify({'error': str(e)}), 500 + + @self.app.route('/api/feeds/', methods=['DELETE']) + def api_delete_feed(feed_id): + """Delete a feed subscription""" + try: + success = self._delete_feed_subscription(feed_id) + if not success: + return jsonify({'error': 'Feed not found'}), 404 + + return jsonify({'success': True}) + except Exception as e: + self.logger.error(f"Error deleting feed: {e}") + return jsonify({'error': str(e)}), 500 + + @self.app.route('/api/feeds/default-format', methods=['GET']) + def api_get_default_format(): + """Get the default output format from config""" + try: + default_format = self.config.get('Feed_Manager', 'default_output_format', + fallback='{emoji} {body|truncate:100} - {date}\n{link|truncate:50}') + return jsonify({'default_format': default_format}) + except Exception as e: + self.logger.error(f"Error getting default format: {e}") + return jsonify({'default_format': '{emoji} {body|truncate:100} - {date}\n{link|truncate:50}'}) + + @self.app.route('/api/feeds/preview', methods=['POST']) + def api_preview_feed(): + """Preview feed items with custom output format""" + try: + data = request.get_json() + if not data or 'feed_url' not in data: + return jsonify({'error': 'feed_url is required'}), 400 + + feed_url = data['feed_url'] + feed_type = data.get('feed_type', 'rss') + output_format = data.get('output_format', '') + api_config = data.get('api_config', {}) + filter_config = data.get('filter_config') + sort_config = data.get('sort_config') + + # Get default format from config if not provided + if not output_format: + output_format = self.config.get('Feed_Manager', 'default_output_format', + fallback='{emoji} {body|truncate:100} - {date}\n{link|truncate:50}') + + # Fetch and format feed items + preview_items = self._preview_feed_items(feed_url, feed_type, output_format, api_config, filter_config, sort_config) + + return jsonify({ + 'success': True, + 'items': preview_items + }) + except Exception as e: + self.logger.error(f"Error previewing feed: {e}") + return jsonify({'error': str(e)}), 500 + + @self.app.route('/api/feeds/test', methods=['POST']) + def api_test_feed(): + """Test a feed URL and return preview of recent items""" + try: + data = request.get_json() + if not data or 'url' not in data: + return jsonify({'error': 'URL is required'}), 400 + + # This would require feed_manager - for now just validate URL + from urllib.parse import urlparse + url = data['url'] + result = urlparse(url) + if not all([result.scheme in ['http', 'https'], result.netloc]): + return jsonify({'error': 'Invalid URL format'}), 400 + + return jsonify({'success': True, 'message': 'URL validated (full test requires feed manager)'}) + except Exception as e: + self.logger.error(f"Error testing feed: {e}") + return jsonify({'error': str(e)}), 500 + + @self.app.route('/api/feeds/stats') + def api_feed_stats(): + """Get aggregate feed statistics""" + try: + stats = self._get_feed_statistics() + return jsonify(stats) + except Exception as e: + self.logger.error(f"Error getting feed stats: {e}") + return jsonify({'error': str(e)}), 500 + + @self.app.route('/api/feeds//activity') + def api_feed_activity(feed_id): + """Get activity log for a specific feed""" + try: + activity = self._get_feed_activity(feed_id, limit=50) + return jsonify({'activity': activity}) + except Exception as e: + self.logger.error(f"Error getting feed activity: {e}") + return jsonify({'error': str(e)}), 500 + + @self.app.route('/api/feeds//errors') + def api_feed_errors(feed_id): + """Get error history for a specific feed""" + try: + errors = self._get_feed_errors(feed_id, limit=20) + return jsonify({'errors': errors}) + except Exception as e: + self.logger.error(f"Error getting feed errors: {e}") + return jsonify({'error': str(e)}), 500 + + @self.app.route('/api/feeds//refresh', methods=['POST']) + def api_refresh_feed(feed_id): + """Manually trigger a feed check""" + try: + # This would trigger feed_manager to poll this feed immediately + # For now, just acknowledge the request + return jsonify({'success': True, 'message': 'Feed refresh queued'}) + except Exception as e: + self.logger.error(f"Error refreshing feed: {e}") + return jsonify({'error': str(e)}), 500 + + # Channel management API endpoints + @self.app.route('/api/channels') + def api_channels(): + """Get all configured channels""" + try: + channels = self._get_channels() + return jsonify({'channels': channels}) + except Exception as e: + self.logger.error(f"Error getting channels: {e}") + return jsonify({'error': str(e)}), 500 + + @self.app.route('/api/channels', methods=['POST']) + def api_create_channel(): + """Create a new channel (hashtag or custom)""" + try: + data = request.get_json() + if not data or 'name' not in data: + return jsonify({'error': 'Channel name is required'}), 400 + + channel_name = data.get('name', '').strip() + channel_idx = data.get('channel_idx') + channel_key = data.get('channel_key', '').strip() + + if not channel_name: + return jsonify({'error': 'Channel name cannot be empty'}), 400 + + # If channel_idx not provided, find the lowest available index + if channel_idx is None: + channel_idx = self._get_lowest_available_channel_index() + if channel_idx is None: + return jsonify({'error': 'No available channel slots. All 40 channels are in use.'}), 400 + + # Determine if it's a hashtag channel + is_hashtag = channel_name.startswith('#') + + # Validate custom channel has key + if not is_hashtag and not channel_key: + return jsonify({'error': 'Channel key is required for custom channels (channels without # prefix)'}), 400 + + # Validate key format if provided + if channel_key: + if len(channel_key) != 32: + return jsonify({'error': 'Channel key must be exactly 32 hexadecimal characters'}), 400 + if not all(c in '0123456789abcdefABCDEF' for c in channel_key): + return jsonify({'error': 'Channel key must contain only hexadecimal characters (0-9, a-f, A-F)'}), 400 + + # Try to create channel via bot's channel manager + result = self._add_channel_for_web(channel_idx, channel_name, channel_key if not is_hashtag else None) + + if result.get('success'): + if result.get('pending'): + # Operation is queued, return operation_id for polling + return jsonify({ + 'success': True, + 'pending': True, + 'operation_id': result.get('operation_id'), + 'message': result.get('message', 'Channel operation queued') + }) + else: + return jsonify({'success': True, 'message': 'Channel created successfully'}) + else: + return jsonify({'error': result.get('error', 'Failed to create channel')}), 500 + + except Exception as e: + self.logger.error(f"Error creating channel: {e}") + return jsonify({'error': str(e)}), 500 + + @self.app.route('/api/channels/', methods=['DELETE']) + def api_delete_channel(channel_idx): + """Remove a channel""" + try: + result = self._remove_channel_for_web(channel_idx) + if result.get('success'): + if result.get('pending'): + # Operation is queued, return operation_id for polling + return jsonify({ + 'success': True, + 'pending': True, + 'operation_id': result.get('operation_id'), + 'message': result.get('message', 'Channel operation queued') + }) + else: + return jsonify({'success': True, 'message': 'Channel deleted successfully'}) + else: + return jsonify({'error': result.get('error', 'Failed to delete channel')}), 500 + except Exception as e: + self.logger.error(f"Error deleting channel: {e}") + return jsonify({'error': str(e)}), 500 + + @self.app.route('/api/channel-operations/', methods=['GET']) + def api_get_operation_status(operation_id): + """Get status of a channel operation""" + try: + conn = self._get_db_connection() + cursor = conn.cursor() + cursor.execute(''' + SELECT status, error_message, result_data, processed_at + FROM channel_operations + WHERE id = ? + ''', (operation_id,)) + + result = cursor.fetchone() + conn.close() + + if not result: + return jsonify({'error': 'Operation not found'}), 404 + + status, error_msg, result_data, processed_at = result + + return jsonify({ + 'operation_id': operation_id, + 'status': status, + 'error_message': error_msg, + 'processed_at': processed_at, + 'result_data': json.loads(result_data) if result_data else None + }) + except Exception as e: + self.logger.error(f"Error getting operation status: {e}") + return jsonify({'error': str(e)}), 500 + + @self.app.route('/api/channels/validate', methods=['POST']) + def api_validate_channel(): + """Validate if a channel exists or can be created""" + try: + data = request.get_json() + if not data or 'name' not in data: + return jsonify({'error': 'Channel name is required'}), 400 + + channel_name = data['name'] + # Check if channel exists + channel_num = self._get_channel_number(channel_name) + + return jsonify({ + 'exists': channel_num is not None, + 'channel_num': channel_num + }) + except Exception as e: + self.logger.error(f"Error validating channel: {e}") + return jsonify({'error': str(e)}), 500 + + @self.app.route('/api/channels/', methods=['PUT']) + def api_update_channel(channel_idx): + """Update channel name or configuration""" + try: + data = request.get_json() + if not data: + return jsonify({'error': 'No data provided'}), 400 + + # This would use channel_manager + return jsonify({'success': True, 'message': 'Channel update requires bot connection'}) + except Exception as e: + self.logger.error(f"Error updating channel: {e}") + return jsonify({'error': str(e)}), 500 + + @self.app.route('/api/channels/stats') + def api_channel_stats(): + """Get channel statistics and usage data""" + try: + stats = self._get_channel_statistics() + return jsonify(stats) + except Exception as e: + self.logger.error(f"Error getting channel stats: {e}") + return jsonify({'error': str(e)}), 500 + + @self.app.route('/api/channels//feeds') + def api_channel_feeds(channel_idx): + """Get all feed subscriptions for a specific channel""" + try: + feeds = self._get_feeds_by_channel(channel_idx) + return jsonify({'feeds': feeds}) + except Exception as e: + self.logger.error(f"Error getting channel feeds: {e}") + return jsonify({'error': str(e)}), 500 def _setup_socketio_handlers(self): """Setup SocketIO event handlers using modern patterns""" @@ -979,7 +1375,11 @@ class BotDataViewer: def poll_database(): last_timestamp = 0 + consecutive_errors = 0 + max_consecutive_errors = 10 + while True: + conn = None try: import time import sqlite3 @@ -988,8 +1388,8 @@ class BotDataViewer: # Get database path db_path = self.config.get('Database', 'path', fallback='bot_data.db') - # Connect to database - conn = sqlite3.connect(db_path) + # Connect to database with timeout to prevent hanging + conn = sqlite3.connect(db_path, timeout=30) cursor = conn.cursor() # Get new data since last poll @@ -1000,7 +1400,6 @@ class BotDataViewer: ''', (last_timestamp,)) rows = cursor.fetchall() - conn.close() # Process new data for timestamp, data_json, data_type in rows: @@ -1016,18 +1415,50 @@ class BotDataViewer: self._handle_packet_data(data) # Treat routing as packet data except Exception as e: - self.logger.debug(f"Error processing database data: {e}") + self.logger.warning(f"Error processing database data: {e}") # Update last timestamp if rows: last_timestamp = rows[-1][0] + # Reset error counter on success + consecutive_errors = 0 + # Sleep before next poll time.sleep(0.5) # Poll every 500ms + except sqlite3.OperationalError as e: + consecutive_errors += 1 + error_msg = str(e) + + # Log at appropriate level based on error frequency + if consecutive_errors >= max_consecutive_errors: + self.logger.error(f"Database polling persistent error (attempt {consecutive_errors}): {error_msg}") + # Exponential backoff for persistent errors + time.sleep(min(60, 2 ** min(consecutive_errors - max_consecutive_errors, 5))) + elif consecutive_errors > 3: + self.logger.warning(f"Database polling error (attempt {consecutive_errors}): {error_msg}") + time.sleep(5) # Wait longer on repeated errors + else: + self.logger.debug(f"Database polling error (attempt {consecutive_errors}): {error_msg}") + time.sleep(1) # Wait longer on error + except Exception as e: - self.logger.debug(f"Database polling error: {e}") - time.sleep(1) # Wait longer on error + consecutive_errors += 1 + if consecutive_errors >= max_consecutive_errors: + self.logger.error(f"Database polling unexpected error (attempt {consecutive_errors}): {e}", exc_info=True) + time.sleep(min(60, 2 ** min(consecutive_errors - max_consecutive_errors, 5))) + else: + self.logger.warning(f"Database polling unexpected error (attempt {consecutive_errors}): {e}") + time.sleep(2) + + finally: + # Always close connection, even on error + if conn: + try: + conn.close() + except Exception as e: + self.logger.debug(f"Error closing database connection: {e}") # Start polling thread polling_thread = threading.Thread(target=poll_database, daemon=True) @@ -1059,6 +1490,7 @@ class BotDataViewer: def _cleanup_old_data(self, days_to_keep: int = 7): """Clean up old packet stream data to prevent database bloat""" + conn = None try: import sqlite3 import time @@ -1068,7 +1500,8 @@ class BotDataViewer: # Get database path db_path = self.config.get('Database', 'path', fallback='bot_data.db') - conn = sqlite3.connect(db_path) + # Use timeout to prevent hanging + conn = sqlite3.connect(db_path, timeout=30) cursor = conn.cursor() # Clean up old packet stream data @@ -1076,13 +1509,18 @@ class BotDataViewer: deleted_count = cursor.rowcount conn.commit() - conn.close() if deleted_count > 0: self.logger.info(f"Cleaned up {deleted_count} old packet stream entries (older than {days_to_keep} days)") except Exception as e: self.logger.error(f"Error cleaning up old packet stream data: {e}") + finally: + if conn: + try: + conn.close() + except Exception as e: + self.logger.debug(f"Error closing cleanup connection: {e}") def _get_database_stats(self, top_users_window='all', top_commands_window='all', top_paths_window='all', top_channels_window='all'): @@ -1902,6 +2340,1148 @@ class BotDataViewer: return {'error': str(e)} + def _get_feed_subscriptions(self, channel_filter=None): + """Get all feed subscriptions, optionally filtered by channel""" + import sqlite3 + conn = None + try: + conn = self._get_db_connection() + conn.row_factory = sqlite3.Row + cursor = conn.cursor() + + if channel_filter: + cursor.execute(''' + SELECT * FROM feed_subscriptions + WHERE channel_name = ? + ORDER BY id + ''', (channel_filter,)) + else: + cursor.execute(''' + SELECT * FROM feed_subscriptions + ORDER BY id + ''') + + rows = cursor.fetchall() + feeds = [] + for row in rows: + feed = dict(row) + # Get feed count for this channel + cursor.execute(''' + SELECT COUNT(*) FROM feed_activity + WHERE feed_id = ? + ''', (feed['id'],)) + feed['item_count'] = cursor.fetchone()[0] + + # Get error count + cursor.execute(''' + SELECT COUNT(*) FROM feed_errors + WHERE feed_id = ? AND resolved_at IS NULL + ''', (feed['id'],)) + feed['error_count'] = cursor.fetchone()[0] + + feeds.append(feed) + + return {'feeds': feeds, 'total': len(feeds)} + except Exception as e: + self.logger.error(f"Error getting feed subscriptions: {e}") + return {'feeds': [], 'total': 0, 'error': str(e)} + finally: + if conn: + conn.close() + + def _get_feed_subscription(self, feed_id): + """Get a single feed subscription by ID""" + import sqlite3 + conn = None + try: + conn = self._get_db_connection() + conn.row_factory = sqlite3.Row + cursor = conn.cursor() + cursor.execute('SELECT * FROM feed_subscriptions WHERE id = ?', (feed_id,)) + row = cursor.fetchone() + return dict(row) if row else None + except Exception as e: + self.logger.error(f"Error getting feed subscription: {e}") + return None + finally: + if conn: + conn.close() + + def _create_feed_subscription(self, data): + """Create a new feed subscription""" + import sqlite3 + import json + conn = None + try: + feed_type = data.get('feed_type') + feed_url = data.get('feed_url') + channel_name = data.get('channel_name') + feed_name = data.get('feed_name') + check_interval = data.get('check_interval_seconds', 300) + api_config = data.get('api_config') + output_format = data.get('output_format') + message_send_interval = data.get('message_send_interval_seconds') + + if not all([feed_type, feed_url, channel_name]): + raise ValueError("feed_type, feed_url, and channel_name are required") + + conn = self._get_db_connection() + cursor = conn.cursor() + + api_config_str = json.dumps(api_config) if api_config else None + + cursor.execute(''' + INSERT INTO feed_subscriptions + (feed_type, feed_url, channel_name, feed_name, check_interval_seconds, api_config, output_format, message_send_interval_seconds) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ''', (feed_type, feed_url, channel_name, feed_name, check_interval, api_config_str, output_format, message_send_interval)) + + conn.commit() + return cursor.lastrowid + except Exception as e: + if conn: + conn.rollback() + raise + finally: + if conn: + conn.close() + + def _update_feed_subscription(self, feed_id, data): + """Update a feed subscription""" + import sqlite3 + import json + conn = None + try: + conn = self._get_db_connection() + cursor = conn.cursor() + + updates = [] + params = [] + + if 'feed_name' in data: + updates.append('feed_name = ?') + params.append(data['feed_name']) + + if 'check_interval_seconds' in data: + updates.append('check_interval_seconds = ?') + params.append(data['check_interval_seconds']) + + if 'enabled' in data: + updates.append('enabled = ?') + params.append(1 if data['enabled'] else 0) + + if 'api_config' in data: + updates.append('api_config = ?') + params.append(json.dumps(data['api_config']) if data['api_config'] else None) + + if 'output_format' in data: + updates.append('output_format = ?') + params.append(data['output_format'] if data['output_format'] else None) + + if 'message_send_interval_seconds' in data: + updates.append('message_send_interval_seconds = ?') + params.append(float(data['message_send_interval_seconds']) if data['message_send_interval_seconds'] else None) + + if 'filter_config' in data: + updates.append('filter_config = ?') + params.append(json.dumps(data['filter_config']) if data['filter_config'] else None) + + if 'sort_config' in data: + updates.append('sort_config = ?') + params.append(json.dumps(data['sort_config']) if data['sort_config'] else None) + + if 'message_send_interval_seconds' in data: + updates.append('message_send_interval_seconds = ?') + params.append(data['message_send_interval_seconds']) + + if not updates: + return True # Nothing to update + + updates.append('updated_at = CURRENT_TIMESTAMP') + params.append(feed_id) + + query = f'UPDATE feed_subscriptions SET {", ".join(updates)} WHERE id = ?' + cursor.execute(query, params) + conn.commit() + + return cursor.rowcount > 0 + except Exception as e: + if conn: + conn.rollback() + raise + finally: + if conn: + conn.close() + + def _delete_feed_subscription(self, feed_id): + """Delete a feed subscription""" + import sqlite3 + conn = None + try: + conn = self._get_db_connection() + cursor = conn.cursor() + cursor.execute('DELETE FROM feed_subscriptions WHERE id = ?', (feed_id,)) + conn.commit() + return cursor.rowcount > 0 + except Exception as e: + if conn: + conn.rollback() + raise + finally: + if conn: + conn.close() + + def _get_feed_activity(self, feed_id, limit=50): + """Get activity log for a feed""" + import sqlite3 + conn = None + try: + conn = self._get_db_connection() + conn.row_factory = sqlite3.Row + cursor = conn.cursor() + cursor.execute(''' + SELECT * FROM feed_activity + WHERE feed_id = ? + ORDER BY processed_at DESC + LIMIT ? + ''', (feed_id, limit)) + rows = cursor.fetchall() + return [dict(row) for row in rows] + except Exception as e: + self.logger.error(f"Error getting feed activity: {e}") + return [] + finally: + if conn: + conn.close() + + def _get_feed_errors(self, feed_id, limit=20): + """Get error history for a feed""" + import sqlite3 + conn = None + try: + conn = self._get_db_connection() + conn.row_factory = sqlite3.Row + cursor = conn.cursor() + cursor.execute(''' + SELECT * FROM feed_errors + WHERE feed_id = ? + ORDER BY occurred_at DESC + LIMIT ? + ''', (feed_id, limit)) + rows = cursor.fetchall() + return [dict(row) for row in rows] + except Exception as e: + self.logger.error(f"Error getting feed errors: {e}") + return [] + finally: + if conn: + conn.close() + + def _get_feed_statistics(self): + """Get aggregate feed statistics""" + import sqlite3 + conn = None + try: + conn = self._get_db_connection() + cursor = conn.cursor() + + stats = {} + + # Total subscriptions + cursor.execute('SELECT COUNT(*) FROM feed_subscriptions') + stats['total_subscriptions'] = cursor.fetchone()[0] + + # Enabled subscriptions + cursor.execute('SELECT COUNT(*) FROM feed_subscriptions WHERE enabled = 1') + stats['enabled_subscriptions'] = cursor.fetchone()[0] + + # Items processed in last 24h + cursor.execute(''' + SELECT COUNT(*) FROM feed_activity + WHERE processed_at > datetime('now', '-24 hours') + ''') + stats['items_24h'] = cursor.fetchone()[0] + + # Items processed in last 7d + cursor.execute(''' + SELECT COUNT(*) FROM feed_activity + WHERE processed_at > datetime('now', '-7 days') + ''') + stats['items_7d'] = cursor.fetchone()[0] + + # Error count + cursor.execute(''' + SELECT COUNT(*) FROM feed_errors + WHERE resolved_at IS NULL + ''') + stats['active_errors'] = cursor.fetchone()[0] + + # Most active channels + cursor.execute(''' + SELECT channel_name, COUNT(*) as feed_count + FROM feed_subscriptions + WHERE enabled = 1 + GROUP BY channel_name + ORDER BY feed_count DESC + LIMIT 10 + ''') + stats['top_channels'] = [{'channel': row[0], 'count': row[1]} for row in cursor.fetchall()] + + return stats + except Exception as e: + self.logger.error(f"Error getting feed statistics: {e}") + return {'error': str(e)} + finally: + if conn: + conn.close() + + def _get_feeds_by_channel(self, channel_idx): + """Get all feeds for a specific channel index""" + # First get channel name from index + # This would require channel_manager access + # For now, return empty list + return [] + + def _get_channels(self): + """Get all configured channels from database""" + import sqlite3 + conn = None + try: + conn = self._get_db_connection() + conn.row_factory = sqlite3.Row + cursor = conn.cursor() + + cursor.execute(''' + SELECT channel_idx, channel_name, channel_type, channel_key_hex, last_updated + FROM channels + ORDER BY channel_idx + ''') + + rows = cursor.fetchall() + channels = [] + for row in rows: + channels.append({ + 'channel_idx': row['channel_idx'], + 'index': row['channel_idx'], # Alias for compatibility + 'name': row['channel_name'], + 'channel_name': row['channel_name'], # Alias for compatibility + 'type': row['channel_type'] or 'hashtag', + 'key_hex': row['channel_key_hex'], + 'last_updated': row['last_updated'] + }) + + return channels + except Exception as e: + self.logger.error(f"Error getting channels: {e}") + return [] + finally: + if conn: + conn.close() + + def _get_channel_number(self, channel_name): + """Get channel number from channel name""" + # This would use channel_manager + # For now, return None + return None + + def _get_lowest_available_channel_index(self): + """Get the lowest available channel index (0-39)""" + try: + channels = self._get_channels() + used_indices = {c['channel_idx'] for c in channels} + + # Find the lowest available index + for i in range(40): + if i not in used_indices: + return i + + # All channels are used + return None + except Exception as e: + self.logger.error(f"Error getting lowest available channel index: {e}") + return None + + def _get_channel_statistics(self): + """Get channel statistics""" + import sqlite3 + conn = None + try: + conn = self._get_db_connection() + cursor = conn.cursor() + + # Get feed count per channel + cursor.execute(''' + SELECT channel_name, COUNT(*) as feed_count + FROM feed_subscriptions + WHERE enabled = 1 + GROUP BY channel_name + ''') + + channel_feeds = {row[0]: row[1] for row in cursor.fetchall()} + + return { + 'channels_with_feeds': len(channel_feeds), + 'channel_feed_counts': channel_feeds + } + except Exception as e: + self.logger.error(f"Error getting channel statistics: {e}") + return {'error': str(e)} + finally: + if conn: + conn.close() + + def _preview_feed_items(self, feed_url: str, feed_type: str, output_format: str, api_config: dict = None, filter_config: dict = None, sort_config: dict = None) -> List[Dict[str, Any]]: + """Preview feed items with custom output format (standalone, doesn't require bot)""" + import feedparser + import requests + import html + import re + from datetime import datetime, timezone + + try: + items = [] + + if feed_type == 'rss': + # Fetch RSS feed + response = requests.get(feed_url, timeout=30, headers={'User-Agent': 'MeshCoreBot/1.0 FeedManager'}) + response.raise_for_status() + parsed = feedparser.parse(response.text) + + # Get items (we'll filter and limit later) + for entry in parsed.entries[:20]: # Fetch more items to account for filtering + # Parse published date + published = None + if hasattr(entry, 'published_parsed') and entry.published_parsed: + try: + published = datetime(*entry.published_parsed[:6], tzinfo=timezone.utc) + except Exception: + pass + + items.append({ + 'title': entry.get('title', 'Untitled'), + 'description': entry.get('description', ''), + 'link': entry.get('link', ''), + 'published': published + }) + + elif feed_type == 'api': + # Fetch API feed + method = api_config.get('method', 'GET').upper() + headers = api_config.get('headers', {}) + params = api_config.get('params', {}) + body = api_config.get('body') + parser_config = api_config.get('response_parser', {}) + + if method == 'POST': + response = requests.post(feed_url, headers=headers, params=params, json=body, timeout=30) + else: + response = requests.get(feed_url, headers=headers, params=params, timeout=30) + response.raise_for_status() + + # Try to parse JSON, handle cases where response might be a string + try: + data = response.json() + except ValueError: + # If JSON parsing fails, try to get text and see if it's an error message + text = response.text + raise Exception(f"API returned non-JSON response: {text[:200]}") + + # Check if response is an error message (string) + if isinstance(data, str): + raise Exception(f"API returned error message: {data[:200]}") + + # Ensure data is a dict or list + if not isinstance(data, (dict, list)): + raise Exception(f"API response is not a valid JSON object or array: {type(data).__name__} - {str(data)[:200]}") + + # Extract items using parser config + items_path = parser_config.get('items_path', '') + if items_path: + parts = items_path.split('.') + items_data = data + for part in parts: + if isinstance(items_data, dict): + items_data = items_data.get(part, []) + else: + raise Exception(f"Cannot navigate path '{items_path}': expected dict at '{part}', got {type(items_data).__name__}") + else: + # If no items_path, data should be a list or we wrap it + if isinstance(data, list): + items_data = data + elif isinstance(data, dict): + # If it's a dict, try to find common array fields + items_data = data.get('items', data.get('data', data.get('results', [data]))) + else: + items_data = [data] + + # Ensure items_data is a list + if not isinstance(items_data, list): + items_data = [items_data] + + # Get items (we'll filter and limit later) + id_field = parser_config.get('id_field', 'id') + title_field = parser_config.get('title_field', 'title') + description_field = parser_config.get('description_field', 'description') + timestamp_field = parser_config.get('timestamp_field', 'created_at') + + # Helper function to get nested values + def get_nested_value(data, path, default=''): + if not path or not data: + return default + parts = path.split('.') + value = data + for part in parts: + if isinstance(value, dict): + value = value.get(part) + elif isinstance(value, list): + try: + idx = int(part) + if 0 <= idx < len(value): + value = value[idx] + else: + return default + except (ValueError, TypeError): + return default + else: + return default + if value is None: + return default + return value if value is not None else default + + for item_data in items_data[:20]: # Fetch more items to account for filtering + # Ensure item_data is a dict + if not isinstance(item_data, dict): + # If it's not a dict, try to convert or skip + if isinstance(item_data, str): + # If it's a string, create a simple dict + item_data = {'title': item_data, 'description': item_data} + else: + # Try to convert to dict or skip + continue + + # Parse timestamp if available - support nested paths + published = None + if timestamp_field: + ts_value = get_nested_value(item_data, timestamp_field) + if ts_value: + try: + if isinstance(ts_value, (int, float)): + published = datetime.fromtimestamp(ts_value, tz=timezone.utc) + elif isinstance(ts_value, str): + # Try Microsoft date format first + if ts_value.startswith('/Date('): + published = self._parse_microsoft_date(ts_value) + else: + # Try ISO format + try: + published = datetime.fromisoformat(ts_value.replace('Z', '+00:00')) + except ValueError: + # Try common formats + for fmt in ['%Y-%m-%dT%H:%M:%S', '%Y-%m-%d %H:%M:%S', '%Y-%m-%d']: + try: + published = datetime.strptime(ts_value, fmt) + if published.tzinfo is None: + published = published.replace(tzinfo=timezone.utc) + break + except ValueError: + continue + except Exception: + pass + + # Get description - support nested paths + description = '' + if description_field: + desc_value = get_nested_value(item_data, description_field) + if desc_value: + description = str(desc_value) + + items.append({ + 'title': get_nested_value(item_data, title_field, 'Untitled'), + 'description': description, + 'link': item_data.get('link', '') if isinstance(item_data, dict) else '', + 'published': published, + 'raw': item_data # Store raw data for format string access + }) + + # Apply sorting if configured + if sort_config: + items = self._sort_items_preview(items, sort_config) + + # Apply filter if configured + if filter_config: + items = [item for item in items if self._should_include_item(item, filter_config)] + + # Limit to first 3 items after filtering + items = items[:3] + + # Format items using output format + formatted_items = [] + for item in items: + formatted = self._format_feed_item(item, output_format, feed_name='') + formatted_items.append({ + 'original': item, + 'formatted': formatted + }) + + return formatted_items + + except Exception as e: + self.logger.error(f"Error previewing feed: {e}") + raise + + def _should_include_item(self, item: Dict[str, Any], filter_config: dict) -> bool: + """Check if an item should be included based on filter configuration (standalone version for preview)""" + import json + import re + + if not filter_config: + return True + + try: + filter_config_dict = json.loads(filter_config) if isinstance(filter_config, str) else filter_config + except (json.JSONDecodeError, TypeError): + return True + + conditions = filter_config_dict.get('conditions', []) + if not conditions: + return True + + logic = filter_config_dict.get('logic', 'AND').upper() + + # Get raw data for field access + raw_data = item.get('raw', {}) + + # Helper to get nested values + def get_nested_value(data, path, default=''): + if not path or not data: + return default + parts = path.split('.') + value = data + for part in parts: + if isinstance(value, dict): + value = value.get(part) + elif isinstance(value, list): + try: + idx = int(part) + if 0 <= idx < len(value): + value = value[idx] + else: + return default + except (ValueError, TypeError): + return default + else: + return default + if value is None: + return default + return value if value is not None else default + + # Evaluate each condition + results = [] + for condition in conditions: + field_path = condition.get('field') + operator = condition.get('operator', 'equals') + + if not field_path: + continue + + # Get field value using nested access + field_value = get_nested_value(raw_data, field_path, '') + if not field_value and field_path.startswith('raw.'): + field_value = get_nested_value(raw_data, field_path[4:], '') + + if not field_value: + field_value = get_nested_value(item, field_path, '') + + # Convert to string for comparison + field_value_str = str(field_value).lower() if field_value is not None else '' + + # Evaluate condition + result = False + if operator == 'equals': + compare_value = str(condition.get('value', '')).lower() + result = field_value_str == compare_value + elif operator == 'not_equals': + compare_value = str(condition.get('value', '')).lower() + result = field_value_str != compare_value + elif operator == 'in': + values = [str(v).lower() for v in condition.get('values', [])] + result = field_value_str in values + elif operator == 'not_in': + values = [str(v).lower() for v in condition.get('values', [])] + result = field_value_str not in values + elif operator == 'matches': + pattern = condition.get('pattern', '') + if pattern: + try: + result = bool(re.search(pattern, str(field_value), re.IGNORECASE)) + except re.error: + result = False + elif operator == 'not_matches': + pattern = condition.get('pattern', '') + if pattern: + try: + result = not bool(re.search(pattern, str(field_value), re.IGNORECASE)) + except re.error: + result = True + elif operator == 'contains': + compare_value = str(condition.get('value', '')).lower() + result = compare_value in field_value_str + elif operator == 'not_contains': + compare_value = str(condition.get('value', '')).lower() + result = compare_value not in field_value_str + else: + result = True # Default to allowing if operator is unknown + + results.append(result) + + # Apply logic (AND or OR) + if logic == 'OR': + return any(results) + else: # AND (default) + return all(results) + + def _parse_microsoft_date(self, date_str: str) -> Optional[datetime]: + """Parse Microsoft JSON date format: /Date(timestamp-offset)/""" + import re + from datetime import timezone + + if not date_str or not isinstance(date_str, str): + return None + + # Match /Date(timestamp-offset)/ format + match = re.match(r'/Date\((\d+)([+-]\d+)?\)/', date_str) + if match: + timestamp_ms = int(match.group(1)) + offset_str = match.group(2) if match.group(2) else '+0000' + + # Convert milliseconds to seconds + timestamp = timestamp_ms / 1000.0 + + # Parse offset (format: +0800 or -0800) + try: + offset_hours = int(offset_str[:3]) + offset_mins = int(offset_str[3:5]) + offset_seconds = (offset_hours * 3600) + (offset_mins * 60) + if offset_str[0] == '-': + offset_seconds = -offset_seconds + + # Create timezone-aware datetime + tz = timezone.utc + if offset_seconds != 0: + from datetime import timedelta + tz = timezone(timedelta(seconds=offset_seconds)) + + return datetime.fromtimestamp(timestamp, tz=tz) + except (ValueError, IndexError): + # Fallback to UTC if offset parsing fails + return datetime.fromtimestamp(timestamp, tz=timezone.utc) + + return None + + def _sort_items_preview(self, items: List[Dict[str, Any]], sort_config: dict) -> List[Dict[str, Any]]: + """Sort items based on sort configuration (standalone version for preview)""" + if not sort_config or not items: + return items + + field_path = sort_config.get('field') + order = sort_config.get('order', 'desc').lower() + + if not field_path: + return items + + # Helper to get nested values + def get_nested_value(data, path, default=''): + if not path or not data: + return default + parts = path.split('.') + value = data + for part in parts: + if isinstance(value, dict): + value = value.get(part) + elif isinstance(value, list): + try: + idx = int(part) + if 0 <= idx < len(value): + value = value[idx] + else: + return default + except (ValueError, TypeError): + return default + else: + return default + if value is None: + return default + return value if value is not None else default + + def get_sort_value(item): + """Get the sort value for an item""" + # Try raw data first + raw_data = item.get('raw', {}) + value = get_nested_value(raw_data, field_path, '') + + if not value and field_path.startswith('raw.'): + value = get_nested_value(raw_data, field_path[4:], '') + + if not value: + value = get_nested_value(item, field_path, '') + + # Handle Microsoft date format + if isinstance(value, str) and value.startswith('/Date('): + dt = self._parse_microsoft_date(value) + if dt: + return dt.timestamp() + + # Handle datetime objects + if isinstance(value, datetime): + return value.timestamp() + + # Handle numeric values + if isinstance(value, (int, float)): + return float(value) + + # Handle string timestamps + if isinstance(value, str): + # Try to parse as ISO format + try: + dt = datetime.fromisoformat(value.replace('Z', '+00:00')) + return dt.timestamp() + except ValueError: + pass + + # Try common date formats + for fmt in ['%Y-%m-%dT%H:%M:%S', '%Y-%m-%d %H:%M:%S', '%Y-%m-%d']: + try: + dt = datetime.strptime(value, fmt) + return dt.timestamp() + except ValueError: + continue + + # For strings, use lexicographic comparison + return str(value) + + # Sort items + try: + sorted_items = sorted(items, key=get_sort_value, reverse=(order == 'desc')) + return sorted_items + except Exception as e: + self.logger.warning(f"Error sorting items in preview: {e}") + return items + + def _format_feed_item(self, item: Dict[str, Any], format_str: str, feed_name: str = '') -> str: + """Format a feed item using the output format (standalone version)""" + import html + import re + from datetime import datetime, timezone + + # Extract field values + title = item.get('title', 'Untitled') + body = item.get('description', '') or item.get('body', '') + + # Clean HTML from body if present + if body: + body = html.unescape(body) + # Convert line break tags to newlines before stripping other HTML + # Handle
,
,
,
, etc. + body = re.sub(r'', '\n', body, flags=re.IGNORECASE) + # Convert paragraph tags to newlines (with spacing) + body = re.sub(r'

', '\n\n', body, flags=re.IGNORECASE) + body = re.sub(r']*>', '', body, flags=re.IGNORECASE) + # Remove remaining HTML tags + body = re.sub(r'<[^>]+>', '', body) + # Clean up whitespace (preserve intentional line breaks) + # Replace multiple newlines with double newline, then normalize spaces within lines + body = re.sub(r'\n\s*\n\s*\n+', '\n\n', body) # Multiple newlines -> double newline + lines = body.split('\n') + body = '\n'.join(' '.join(line.split()) for line in lines) # Normalize spaces per line + body = body.strip() + + link = item.get('link', '') + published = item.get('published') + + # Format timestamp + date_str = "" + if published: + try: + if published.tzinfo: + now = datetime.now(timezone.utc) + else: + now = datetime.now() + + diff = now - published + minutes = int(diff.total_seconds() / 60) + + if minutes < 1: + date_str = "now" + elif minutes < 60: + date_str = f"{minutes}m ago" + elif minutes < 1440: + hours = minutes // 60 + mins = minutes % 60 + date_str = f"{hours}h {mins}m ago" + else: + days = minutes // 1440 + date_str = f"{days}d ago" + except Exception: + pass + + # Choose emoji + emoji = "đŸ“ĸ" + feed_name_lower = feed_name.lower() + if 'emergency' in feed_name_lower or 'alert' in feed_name_lower: + emoji = "🚨" + elif 'warning' in feed_name_lower: + emoji = "⚠ī¸" + elif 'info' in feed_name_lower or 'news' in feed_name_lower: + emoji = "ℹī¸" + + # Build replacements + replacements = { + 'title': title, + 'body': body, + 'date': date_str, + 'link': link, + 'emoji': emoji + } + + # Get raw API data if available (for preview, we don't have raw data, so this will be empty) + raw_data = item.get('raw', {}) + + # Helper to get nested values + def get_nested_value(data, path, default=''): + if not path or not data: + return default + parts = path.split('.') + value = data + for part in parts: + if isinstance(value, dict): + value = value.get(part) + elif isinstance(value, list): + try: + idx = int(part) + if 0 <= idx < len(value): + value = value[idx] + else: + return default + except (ValueError, TypeError): + return default + else: + return default + if value is None: + return default + return value if value is not None else default + + # Apply shortening, parsing, and conditional functions + def apply_shortening(text: str, function: str) -> str: + if not text: + return "" + + if function.startswith('truncate:'): + try: + max_len = int(function.split(':', 1)[1]) + if len(text) <= max_len: + return text + return text[:max_len] + "..." + except (ValueError, IndexError): + return text + elif function.startswith('word_wrap:'): + try: + max_len = int(function.split(':', 1)[1]) + if len(text) <= max_len: + return text + truncated = text[:max_len] + last_space = truncated.rfind(' ') + if last_space > max_len * 0.7: + return truncated[:last_space] + "..." + return truncated + "..." + except (ValueError, IndexError): + return text + elif function.startswith('first_words:'): + try: + num_words = int(function.split(':', 1)[1]) + words = text.split() + if len(words) <= num_words: + return text + return ' '.join(words[:num_words]) + "..." + except (ValueError, IndexError): + return text + elif function.startswith('regex:'): + try: + # Parse regex pattern and optional group number + # Format: regex:pattern:group or regex:pattern + # Need to handle patterns that contain colons, so split from the right + remaining = function[6:] # Skip 'regex:' prefix + + # Try to find the last colon that's followed by a number (the group number) + # Look for pattern like :N at the end + last_colon_idx = remaining.rfind(':') + pattern = remaining + group_num = None + + if last_colon_idx > 0: + # Check if what's after the last colon is a number + potential_group = remaining[last_colon_idx + 1:] + if potential_group.isdigit(): + pattern = remaining[:last_colon_idx] + group_num = int(potential_group) + + if not pattern: + return text + + # Apply regex + match = re.search(pattern, text, re.IGNORECASE | re.DOTALL) + if match: + if group_num is not None: + # Use specified group (0 = whole match, 1 = first group, etc.) + if 0 <= group_num <= len(match.groups()): + return match.group(group_num) if group_num > 0 else match.group(0) + else: + # Use first capture group if available, otherwise whole match + if match.groups(): + return match.group(1) + else: + return match.group(0) + return "" # No match found + except (ValueError, IndexError, re.error) as e: + # Silently fail on regex errors in preview + return text + elif function.startswith('if_regex:'): + try: + # Parse: if_regex:pattern:then:else + # Split by ':' but need to handle regex patterns that contain ':' + parts = function[9:].split(':', 2) # Skip 'if_regex:' prefix, split into [pattern, then, else] + if len(parts) < 3: + return text + + pattern = parts[0] + then_value = parts[1] + else_value = parts[2] + + if not pattern: + return text + + # Check if pattern matches + match = re.search(pattern, text, re.IGNORECASE | re.DOTALL) + if match: + return then_value + else: + return else_value + except (ValueError, IndexError, re.error) as e: + # Silently fail on regex errors in preview + return text + elif function.startswith('switch:'): + try: + # Parse: switch:value1:result1:value2:result2:...:default + # Example: switch:highest:🔴:high:🟠:medium:🟡:low:âšĒ:âšĒ + parts = function[7:].split(':') # Skip 'switch:' prefix + if len(parts) < 2: + return text + + # Pairs of value:result, last one is default + text_lower = text.lower().strip() + for i in range(0, len(parts) - 1, 2): + if i + 1 < len(parts): + value = parts[i].lower() + result = parts[i + 1] + if text_lower == value: + return result + + # Return last part as default if no match + return parts[-1] if parts else text + except (ValueError, IndexError) as e: + # Silently fail on switch errors in preview + return text + elif function.startswith('regex_cond:'): + try: + # Parse: regex_cond:extract_pattern:check_pattern:then:group + parts = function[11:].split(':', 3) # Skip 'regex_cond:' prefix + if len(parts) < 4: + return text + + extract_pattern = parts[0] + check_pattern = parts[1] + then_value = parts[2] + else_group = int(parts[3]) if parts[3].isdigit() else 1 + + if not extract_pattern: + return text + + # Extract using extract_pattern + match = re.search(extract_pattern, text, re.IGNORECASE | re.DOTALL) + if match: + # Get the captured group + if match.groups(): + extracted = match.group(else_group) if else_group <= len(match.groups()) else match.group(1) + # Strip whitespace from extracted text + extracted = extracted.strip() + else: + extracted = match.group(0).strip() + + # Check if extracted text matches check_pattern (exact match or contains) + if check_pattern: + # Try exact match first, then substring match + if extracted.lower() == check_pattern.lower() or re.search(check_pattern, extracted, re.IGNORECASE): + return then_value + + return extracted + return "" # No match found + except (ValueError, IndexError, re.error) as e: + # Silently fail on regex errors in preview + return text + return text + + # Process format string + def replace_placeholder(match): + content = match.group(1) + if '|' in content: + field_name, function = content.split('|', 1) + field_name = field_name.strip() + function = function.strip() + + # Check if it's a raw field access + if field_name.startswith('raw.'): + value = str(get_nested_value(raw_data, field_name[4:], '')) + else: + value = replacements.get(field_name, '') + + return apply_shortening(value, function) + else: + field_name = content.strip() + + # Check if it's a raw field access + if field_name.startswith('raw.'): + value = get_nested_value(raw_data, field_name[4:], '') + if value is None: + return '' + elif isinstance(value, (dict, list)): + try: + import json + return json.dumps(value) + except Exception: + return str(value) + else: + return str(value) + else: + return replacements.get(field_name, '') + + message = re.sub(r'\{([^}]+)\}', replace_placeholder, format_str) + + # Final truncation (130 char limit) + max_length = 130 + if len(message) > max_length: + lines = message.split('\n') + if len(lines) > 1: + total_length = sum(len(line) + 1 for line in lines[:-1]) + remaining = max_length - total_length - 3 + if remaining > 20: + lines[-1] = lines[-1][:remaining] + "..." + message = '\n'.join(lines) + else: + message = message[:max_length - 3] + "..." + else: + message = message[:max_length - 3] + "..." + + return message + def _get_bot_uptime(self): """Get bot uptime in seconds from database""" try: @@ -1928,6 +3508,92 @@ class BotDataViewer: self.logger.debug(f"Could not get bot start time from database: {e}") return 0 + def _add_channel_for_web(self, channel_idx, channel_name, channel_key_hex=None): + """ + Add a channel by queuing it in the database for the bot to process + + Args: + channel_idx: Channel index (0-39) + channel_name: Channel name (with or without # prefix) + channel_key_hex: Optional hex key for custom channels (32 chars) + + Returns: + dict with 'success' and optional 'error' key + """ + try: + conn = self._get_db_connection() + cursor = conn.cursor() + + # Insert operation into queue + cursor.execute(''' + INSERT INTO channel_operations + (operation_type, channel_idx, channel_name, channel_key_hex, status) + VALUES (?, ?, ?, ?, 'pending') + ''', ('add', channel_idx, channel_name, channel_key_hex)) + + operation_id = cursor.lastrowid + conn.commit() + conn.close() + + self.logger.info(f"Queued channel add operation: {channel_name} at index {channel_idx} (operation_id: {operation_id})") + + # Return immediately with operation_id - let frontend poll for status + return { + 'success': True, + 'pending': True, + 'operation_id': operation_id, + 'message': 'Channel operation queued successfully' + } + + except Exception as e: + self.logger.error(f"Error in _add_channel_for_web: {e}") + return { + 'success': False, + 'error': str(e) + } + + def _remove_channel_for_web(self, channel_idx): + """ + Remove a channel by queuing it in the database for the bot to process + + Args: + channel_idx: Channel index to remove + + Returns: + dict with 'success' and optional 'error' key + """ + try: + conn = self._get_db_connection() + cursor = conn.cursor() + + # Insert operation into queue + cursor.execute(''' + INSERT INTO channel_operations + (operation_type, channel_idx, status) + VALUES (?, ?, 'pending') + ''', ('remove', channel_idx)) + + operation_id = cursor.lastrowid + conn.commit() + conn.close() + + self.logger.info(f"Queued channel remove operation: index {channel_idx} (operation_id: {operation_id})") + + # Return immediately with operation_id - let frontend poll for status + return { + 'success': True, + 'pending': True, + 'operation_id': operation_id, + 'message': 'Channel operation queued successfully' + } + + except Exception as e: + self.logger.error(f"Error in _remove_channel_for_web: {e}") + return { + 'success': False, + 'error': str(e) + } + def run(self, host='127.0.0.1', port=8080, debug=False): """Run the modern web viewer""" self.logger.info(f"Starting modern web viewer on {host}:{port}") diff --git a/modules/web_viewer/templates/base.html b/modules/web_viewer/templates/base.html index f11409a..16559f0 100644 --- a/modules/web_viewer/templates/base.html +++ b/modules/web_viewer/templates/base.html @@ -146,11 +146,25 @@ Cache + {% if greeter_enabled %}
  • Greeter
    • + {% endif %} + {% if feed_manager_enabled %} +
  • + + Feeds + +
    • + {% endif %} +
  • + + Radio + +
    • diff --git a/modules/web_viewer/templates/feeds.html b/modules/web_viewer/templates/feeds.html new file mode 100644 index 0000000..0c2b73b --- /dev/null +++ b/modules/web_viewer/templates/feeds.html @@ -0,0 +1,848 @@ +{% extends "base.html" %} + +{% block title %}Feed Management - MeshCore Bot{% endblock %} + +{% block content %} +
    +
    +
    +

    + Feed Management +

    + + +
    +
    +
    +
    +
    Total Subscriptions
    +

    -

    +
    +
    +
    +
    +
    +
    +
    Active Feeds
    +

    -

    +
    +
    +
    +
    +
    +
    +
    Items (24h)
    +

    -

    +
    +
    +
    +
    +
    +
    +
    Active Errors
    +

    -

    +
    +
    +
    +
    + + +
    +
    +
    Feed Subscriptions
    + +
    +
    +
    + + + + + + + + + + + + + + + + + + + +
    IDNameTypeChannelStatusLast CheckItemsErrorsActions
    Loading...
    +
    +
    +
    +
    +
    +
    + + + + + + +{% endblock %} + +{% block extra_js %} + +{% endblock %} + diff --git a/modules/web_viewer/templates/radio.html b/modules/web_viewer/templates/radio.html new file mode 100644 index 0000000..805ccc8 --- /dev/null +++ b/modules/web_viewer/templates/radio.html @@ -0,0 +1,718 @@ +{% extends "base.html" %} + +{% block title %}Radio Settings - MeshCore Bot{% endblock %} + +{% block content %} +
    +
    +
    +

    + Radio Settings +

    + + +
    +
    +
    +
    +
    Total Channels
    +

    -

    +
    +
    +
    +
    +
    +
    +
    Available Slots
    +

    -

    +
    +
    +
    +
    +
    +
    +
    Channels with Feeds
    +

    -

    +
    +
    +
    +
    +
    +
    +
    Empty Channels
    +

    -

    +
    +
    +
    +
    + + +
    +
    +
    Channel Management
    + +
    +
    +
    + + + + + + + + + + + + + + + + +
    IndexNameTypeFeed CountStatusActions
    Loading...
    +
    +
    +
    +
    +
    +
    + + + + + + +{% endblock %} + +{% block extra_js %} + +{% endblock %} + diff --git a/requirements.txt b/requirements.txt index 2086743..d5bf6a3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,3 +18,4 @@ retry-requests>=1.0.0 flask>=2.3.0 flask-socketio>=5.3.0 meshcore-cli +feedparser>=6.0.10