From 92f063b4bc19852666552e302f93d1d79444f7df Mon Sep 17 00:00:00 2001 From: agessaman Date: Wed, 26 Nov 2025 14:34:45 -0800 Subject: [PATCH] Add Nominatim rate limiting and caching enhancements across commands. Introduce NominatimRateLimiter for API compliance, update geocoding methods to utilize rate-limited calls, and extend caching duration for geocoding results to 30 days. Refactor command implementations to improve efficiency and prevent duplicate API requests. --- .../commands/alternatives/wx_international.py | 17 ++- modules/commands/aqi_command.py | 39 ++--- modules/commands/prefix_command.py | 7 +- modules/commands/solarforecast_command.py | 46 +++--- modules/commands/wx_command.py | 48 +++++-- modules/core.py | 6 +- modules/db_manager.py | 8 +- modules/rate_limiter.py | 42 ++++++ modules/repeater_manager.py | 45 +++--- modules/utils.py | 135 ++++++++++++++++++ 10 files changed, 293 insertions(+), 100 deletions(-) diff --git a/modules/commands/alternatives/wx_international.py b/modules/commands/alternatives/wx_international.py index b2bbc77..ca09a9a 100644 --- a/modules/commands/alternatives/wx_international.py +++ b/modules/commands/alternatives/wx_international.py @@ -8,6 +8,7 @@ import re import requests from datetime import datetime, timedelta from geopy.geocoders import Nominatim +from ...utils import rate_limited_nominatim_geocode_sync, rate_limited_nominatim_reverse_sync, get_nominatim_geocoder from ..base_command import BaseCommand from ...models import MeshMessage @@ -53,8 +54,8 @@ class GlobalWxCommand(BaseCommand): self.logger.warning(f"Invalid precipitation_unit '{self.precipitation_unit}', using 'inch'") self.precipitation_unit = 'inch' - # Initialize geocoder - self.geolocator = Nominatim(user_agent="meshcore-bot") + # Initialize geocoder (will use rate-limited helpers for actual calls) + self.geolocator = get_nominatim_geocoder() # Get database manager for geocoding cache self.db_manager = bot.db_manager @@ -241,7 +242,9 @@ class GlobalWxCommand(BaseCommand): self.logger.debug(f"Using cached geocoding for {location}") # Get address details with reverse geocoding try: - reverse_location = self.geolocator.reverse(f"{cached_lat}, {cached_lon}") + reverse_location = rate_limited_nominatim_reverse_sync( + self.bot, f"{cached_lat}, {cached_lon}", timeout=10 + ) if reverse_location: address_info = reverse_location.raw.get('address', {}) # Store the full geocode result for display name @@ -254,11 +257,15 @@ class GlobalWxCommand(BaseCommand): geocode_result = None # Strategy 1: Try as-is - geocode_result = self.geolocator.geocode(location) + geocode_result = rate_limited_nominatim_geocode_sync( + self.bot, location, timeout=10 + ) # Strategy 2: If no result and no country specified, try with default country if not geocode_result and ',' not in location: - geocode_result = self.geolocator.geocode(f"{location}, {self.default_country}") + geocode_result = rate_limited_nominatim_geocode_sync( + self.bot, f"{location}, {self.default_country}", timeout=10 + ) if not geocode_result: return None, None, None, None diff --git a/modules/commands/aqi_command.py b/modules/commands/aqi_command.py index c1c79ad..09d5988 100644 --- a/modules/commands/aqi_command.py +++ b/modules/commands/aqi_command.py @@ -10,6 +10,7 @@ import requests_cache from retry_requests import retry from datetime import datetime from geopy.geocoders import Nominatim +from ..utils import rate_limited_nominatim_geocode_sync, rate_limited_nominatim_reverse_sync, get_nominatim_geocoder from .base_command import BaseCommand from ..models import MeshMessage @@ -41,8 +42,8 @@ class AqiCommand(BaseCommand): # Get timezone from config self.timezone = self.bot.config.get('Bot', 'timezone', fallback='America/Los_Angeles') - # Initialize geocoder - self.geolocator = Nominatim(user_agent="meshcore-bot") + # Initialize geocoder (will use rate-limited helpers for actual calls) + self.geolocator = get_nominatim_geocoder() # Get database manager for geocoding cache self.db_manager = bot.db_manager @@ -384,7 +385,7 @@ class AqiCommand(BaseCommand): mapped_location = zip_code_mappings[zip_code] self.logger.debug(f"Using specific mapping for ZIP {zip_code}: {mapped_location}") try: - result = self.geolocator.geocode(mapped_location) + result = rate_limited_nominatim_geocode_sync(self.bot, mapped_location, timeout=10) if result and result.address: location_result = result self.logger.debug(f"Found mapped location for ZIP {zip_code}: {result.address}") @@ -397,10 +398,10 @@ class AqiCommand(BaseCommand): try: if isinstance(query, dict): # Use structured query - result = self.geolocator.geocode(query=query) + result = rate_limited_nominatim_geocode_sync(self.bot, query, timeout=10) else: # Use text-based query - result = self.geolocator.geocode(query) + result = rate_limited_nominatim_geocode_sync(self.bot, query, timeout=10) if result and result.address: # Check if it's a US location @@ -425,7 +426,7 @@ class AqiCommand(BaseCommand): # Get detailed address info via reverse geocoding try: - reverse_location = self.geolocator.reverse(f"{lat}, {lon}") + reverse_location = rate_limited_nominatim_reverse_sync(self.bot, f"{lat}, {lon}", timeout=10) if reverse_location: address_info = reverse_location.raw.get('address', {}) else: @@ -608,7 +609,7 @@ class AqiCommand(BaseCommand): self.logger.debug(f"Using cached geocoding for {city}") # Still need to do reverse geocoding for address details try: - reverse_location = self.geolocator.reverse(f"{cached_lat}, {cached_lon}") + reverse_location = rate_limited_nominatim_reverse_sync(self.bot, f"{cached_lat}, {cached_lon}", timeout=10) if reverse_location: return cached_lat, cached_lon, reverse_location.raw.get('address', {}) except: @@ -629,14 +630,14 @@ class AqiCommand(BaseCommand): if is_country: # Handle international cities - location = self.geolocator.geocode(f"{city_name}, {state_or_country}") + location = rate_limited_nominatim_geocode_sync(self.bot, f"{city_name}, {state_or_country}", timeout=10) if location: # Cache the result self.db_manager.cache_geocoding(f"{city_name}, {state_or_country}", location.latitude, location.longitude) # Use reverse geocoding to get detailed address info try: - reverse_location = self.geolocator.reverse(f"{location.latitude}, {location.longitude}") + reverse_location = rate_limited_nominatim_reverse_sync(self.bot, f"{location.latitude}, {location.longitude}", timeout=10) if reverse_location: return location.latitude, location.longitude, reverse_location.raw.get('address', {}) except: @@ -644,14 +645,14 @@ class AqiCommand(BaseCommand): return location.latitude, location.longitude, location.raw.get('address', {}) else: # Handle US city, state format - location = self.geolocator.geocode(f"{city_name}, {state_or_country}, USA") + location = rate_limited_nominatim_geocode_sync(self.bot, f"{city_name}, {state_or_country}, USA", timeout=10) if location: # Cache the result self.db_manager.cache_geocoding(f"{city_name}, {state_or_country}, USA", location.latitude, location.longitude) # Use reverse geocoding to get detailed address info try: - reverse_location = self.geolocator.reverse(f"{location.latitude}, {location.longitude}") + reverse_location = rate_limited_nominatim_reverse_sync(self.bot, f"{location.latitude}, {location.longitude}", timeout=10) if reverse_location: return location.latitude, location.longitude, reverse_location.raw.get('address', {}) except: @@ -676,14 +677,14 @@ class AqiCommand(BaseCommand): # If it's a major city with multiple locations, try the major ones first if city.lower() in major_city_mappings: for major_city_query in major_city_mappings[city.lower()]: - location = self.geolocator.geocode(major_city_query) + location = rate_limited_nominatim_geocode_sync(self.bot, major_city_query, timeout=10) if location: # Cache the result self.db_manager.cache_geocoding(major_city_query, location.latitude, location.longitude) # Use reverse geocoding to get detailed address info try: - reverse_location = self.geolocator.reverse(f"{location.latitude}, {location.longitude}") + reverse_location = rate_limited_nominatim_reverse_sync(self.bot, f"{location.latitude}, {location.longitude}", timeout=10) if reverse_location: return location.latitude, location.longitude, reverse_location.raw.get('address', {}) except: @@ -691,14 +692,14 @@ class AqiCommand(BaseCommand): return location.latitude, location.longitude, location.raw.get('address', {}) # First try with default state - location = self.geolocator.geocode(f"{city}, {self.default_state}, USA") + location = rate_limited_nominatim_geocode_sync(self.bot, f"{city}, {self.default_state}, USA", timeout=10) if location: # Cache the result self.db_manager.cache_geocoding(f"{city}, {self.default_state}, USA", location.latitude, location.longitude) # Use reverse geocoding to get detailed address info try: - reverse_location = self.geolocator.reverse(f"{location.latitude}, {location.longitude}") + reverse_location = rate_limited_nominatim_reverse_sync(self.bot, f"{location.latitude}, {location.longitude}", timeout=10) if reverse_location: return location.latitude, location.longitude, reverse_location.raw.get('address', {}) except: @@ -708,14 +709,14 @@ class AqiCommand(BaseCommand): # Try neighborhood-specific queries for major cities neighborhood_queries = self.get_neighborhood_queries(city) for query in neighborhood_queries: - location = self.geolocator.geocode(query) + location = rate_limited_nominatim_geocode_sync(self.bot, query, timeout=10) if location: # Cache the result self.db_manager.cache_geocoding(query, location.latitude, location.longitude) # Use reverse geocoding to get detailed address info try: - reverse_location = self.geolocator.reverse(f"{location.latitude}, {location.longitude}") + reverse_location = rate_limited_nominatim_reverse_sync(self.bot, f"{location.latitude}, {location.longitude}", timeout=10) if reverse_location: return location.latitude, location.longitude, reverse_location.raw.get('address', {}) except: @@ -723,14 +724,14 @@ class AqiCommand(BaseCommand): return location.latitude, location.longitude, location.raw.get('address', {}) # Try without state as final fallback - location = self.geolocator.geocode(f"{city}, USA") + location = rate_limited_nominatim_geocode_sync(self.bot, f"{city}, USA", timeout=10) if location: # Cache the result self.db_manager.cache_geocoding(f"{city}, USA", location.latitude, location.longitude) # Use reverse geocoding to get detailed address info try: - reverse_location = self.geolocator.reverse(f"{location.latitude}, {location.longitude}") + reverse_location = rate_limited_nominatim_reverse_sync(self.bot, f"{location.latitude}, {location.longitude}", timeout=10) if reverse_location: return location.latitude, location.longitude, reverse_location.raw.get('address', {}) except: diff --git a/modules/commands/prefix_command.py b/modules/commands/prefix_command.py index fd414d7..8612b40 100644 --- a/modules/commands/prefix_command.py +++ b/modules/commands/prefix_command.py @@ -393,9 +393,10 @@ class PrefixCommand(BaseCommand): location_str = abbreviate_location(city, 20) else: # Fallback to basic geocoding - from geopy.geocoders import Nominatim - geolocator = Nominatim(user_agent="meshcore-bot") - location = geolocator.reverse(f"{row['latitude']}, {row['longitude']}") + from ..utils import rate_limited_nominatim_reverse_sync + location = rate_limited_nominatim_reverse_sync( + self.bot, f"{row['latitude']}, {row['longitude']}", timeout=10 + ) if location: address = location.raw.get('address', {}) # Try neighborhood first, then city, then town, etc. diff --git a/modules/commands/solarforecast_command.py b/modules/commands/solarforecast_command.py index c694784..a2ffe2f 100644 --- a/modules/commands/solarforecast_command.py +++ b/modules/commands/solarforecast_command.py @@ -10,6 +10,7 @@ import time import hashlib import pytz from geopy.geocoders import Nominatim +from ..utils import rate_limited_nominatim_geocode, rate_limited_nominatim_reverse, get_nominatim_geocoder from datetime import datetime, timedelta, timezone from typing import Optional, Tuple, Dict from .base_command import BaseCommand @@ -50,8 +51,8 @@ class SolarforecastCommand(BaseCommand): # Get default state from config for city disambiguation self.default_state = self.bot.config.get('Weather', 'default_state', fallback='WA') - # Initialize geocoder - self.geolocator = Nominatim(user_agent="meshcore-bot") + # Initialize geocoder (will use rate-limited helpers for actual calls) + self.geolocator = get_nominatim_geocoder() # Get database manager for geocoding cache self.db_manager = bot.db_manager @@ -352,10 +353,8 @@ class SolarforecastCommand(BaseCommand): if cached_lat and cached_lon: return cached_lat, cached_lon - loop = asyncio.get_event_loop() - location = await loop.run_in_executor( - None, - lambda: self.geolocator.geocode(f"{zipcode}, USA", timeout=self.url_timeout) + location = await rate_limited_nominatim_geocode( + self.bot, f"{zipcode}, USA", timeout=self.url_timeout ) if location: self.db_manager.cache_geocoding(cache_query, location.latitude, location.longitude) @@ -391,9 +390,8 @@ class SolarforecastCommand(BaseCommand): if cached_lat and cached_lon: return cached_lat, cached_lon - location = await loop.run_in_executor( - None, - lambda q=major_city_query: self.geolocator.geocode(q, timeout=self.url_timeout) + location = await rate_limited_nominatim_geocode( + self.bot, major_city_query, timeout=self.url_timeout ) if location: self.db_manager.cache_geocoding(major_city_query, location.latitude, location.longitude) @@ -406,9 +404,8 @@ class SolarforecastCommand(BaseCommand): if cached_lat and cached_lon: return cached_lat, cached_lon - location = await loop.run_in_executor( - None, - lambda: self.geolocator.geocode(state_query, timeout=self.url_timeout) + location = await rate_limited_nominatim_geocode( + self.bot, state_query, timeout=self.url_timeout ) if location: self.db_manager.cache_geocoding(state_query, location.latitude, location.longitude) @@ -420,27 +417,24 @@ class SolarforecastCommand(BaseCommand): if cached_lat and cached_lon: return cached_lat, cached_lon - location = await loop.run_in_executor( - None, - lambda: self.geolocator.geocode(cache_query, timeout=self.url_timeout) + location = await rate_limited_nominatim_geocode( + self.bot, cache_query, timeout=self.url_timeout ) if location: self.db_manager.cache_geocoding(cache_query, location.latitude, location.longitude) return location.latitude, location.longitude # Try without state - location = await loop.run_in_executor( - None, - lambda: self.geolocator.geocode(f"{city_clean}, USA", timeout=self.url_timeout) + location = await rate_limited_nominatim_geocode( + self.bot, f"{city_clean}, USA", timeout=self.url_timeout ) if location: self.db_manager.cache_geocoding(f"{city_clean}, USA", location.latitude, location.longitude) return location.latitude, location.longitude # Try international - location = await loop.run_in_executor( - None, - lambda: self.geolocator.geocode(city_clean, timeout=self.url_timeout) + location = await rate_limited_nominatim_geocode( + self.bot, city_clean, timeout=self.url_timeout ) if location: self.db_manager.cache_geocoding(city_clean, location.latitude, location.longitude) @@ -480,9 +474,8 @@ class SolarforecastCommand(BaseCommand): # For coordinates, always do reverse geocoding if location_type == "coordinates": - location = await loop.run_in_executor( - None, - lambda: self.geolocator.reverse(f"{lat}, {lon}", timeout=self.url_timeout) + location = await rate_limited_nominatim_reverse( + self.bot, f"{lat}, {lon}", timeout=self.url_timeout ) if location and location.raw: address = location.raw.get('address', {}) @@ -499,9 +492,8 @@ class SolarforecastCommand(BaseCommand): # For city/zipcode, use original if it worked, or reverse geocode if location_type in ["city", "zipcode"]: # Try reverse geocoding to get confirmed city name - location = await loop.run_in_executor( - None, - lambda: self.geolocator.reverse(f"{lat}, {lon}", timeout=self.url_timeout) + location = await rate_limited_nominatim_reverse( + self.bot, f"{lat}, {lon}", timeout=self.url_timeout ) if location and location.raw: address = location.raw.get('address', {}) diff --git a/modules/commands/wx_command.py b/modules/commands/wx_command.py index 69e884e..6a841a7 100644 --- a/modules/commands/wx_command.py +++ b/modules/commands/wx_command.py @@ -10,6 +10,7 @@ import requests import xml.dom.minidom from datetime import datetime, timedelta from geopy.geocoders import Nominatim +from ..utils import rate_limited_nominatim_geocode_sync, rate_limited_nominatim_reverse_sync, get_nominatim_geocoder import maidenhead as mh from .base_command import BaseCommand from ..models import MeshMessage @@ -44,8 +45,9 @@ class WxCommand(BaseCommand): # Get default state from config for city disambiguation self.default_state = self.bot.config.get('Weather', 'default_state', fallback='WA') - # Initialize geocoder - self.geolocator = Nominatim(user_agent="meshcore-bot") + # Initialize geocoder (will use rate-limited helpers for actual calls) + # Keep geolocator for backwards compatibility, but prefer rate-limited helpers + self.geolocator = get_nominatim_geocoder() # Get database manager for geocoding cache self.db_manager = bot.db_manager @@ -318,8 +320,10 @@ class WxCommand(BaseCommand): def zipcode_to_lat_lon(self, zipcode: str) -> tuple: """Convert zipcode to latitude and longitude""" try: - # Use Nominatim to geocode the zipcode - location = self.geolocator.geocode(f"{zipcode}, USA") + # Use rate-limited Nominatim to geocode the zipcode + location = rate_limited_nominatim_geocode_sync( + self.bot, f"{zipcode}, USA", timeout=10 + ) if location: return location.latitude, location.longitude else: @@ -338,7 +342,9 @@ class WxCommand(BaseCommand): self.logger.debug(f"Using cached geocoding for {city}") # Still need to do reverse geocoding for address details try: - reverse_location = self.geolocator.reverse(f"{cached_lat}, {cached_lon}") + reverse_location = rate_limited_nominatim_reverse_sync( + self.bot, f"{cached_lat}, {cached_lon}", timeout=10 + ) if reverse_location: return cached_lat, cached_lon, reverse_location.raw.get('address', {}) except: @@ -354,14 +360,18 @@ class WxCommand(BaseCommand): state = city_parts[1] # Try the specific city, state combination first - location = self.geolocator.geocode(f"{city_name}, {state}, USA") + location = rate_limited_nominatim_geocode_sync( + self.bot, f"{city_name}, {state}, USA", timeout=10 + ) if location: # Cache the result self.db_manager.cache_geocoding(f"{city_name}, {state}, USA", location.latitude, location.longitude) # Use reverse geocoding to get detailed address info try: - reverse_location = self.geolocator.reverse(f"{location.latitude}, {location.longitude}") + reverse_location = rate_limited_nominatim_reverse_sync( + self.bot, f"{location.latitude}, {location.longitude}", timeout=10 + ) if reverse_location: return location.latitude, location.longitude, reverse_location.raw.get('address', {}) except: @@ -386,14 +396,18 @@ class WxCommand(BaseCommand): # If it's a major city with multiple locations, try the major ones first if city.lower() in major_city_mappings: for major_city_query in major_city_mappings[city.lower()]: - location = self.geolocator.geocode(major_city_query) + location = rate_limited_nominatim_geocode_sync( + self.bot, major_city_query, timeout=10 + ) if location: # Cache the result self.db_manager.cache_geocoding(major_city_query, location.latitude, location.longitude) # Use reverse geocoding to get detailed address info try: - reverse_location = self.geolocator.reverse(f"{location.latitude}, {location.longitude}") + reverse_location = rate_limited_nominatim_reverse_sync( + self.bot, f"{location.latitude}, {location.longitude}", timeout=10 + ) if reverse_location: return location.latitude, location.longitude, reverse_location.raw.get('address', {}) except: @@ -401,14 +415,18 @@ class WxCommand(BaseCommand): return location.latitude, location.longitude, location.raw.get('address', {}) # First try with default state - location = self.geolocator.geocode(f"{city}, {self.default_state}, USA") + location = rate_limited_nominatim_geocode_sync( + self.bot, f"{city}, {self.default_state}, USA", timeout=10 + ) if location: # Cache the result self.db_manager.cache_geocoding(f"{city}, {self.default_state}, USA", location.latitude, location.longitude) # Use reverse geocoding to get detailed address info try: - reverse_location = self.geolocator.reverse(f"{location.latitude}, {location.longitude}") + reverse_location = rate_limited_nominatim_reverse_sync( + self.bot, f"{location.latitude}, {location.longitude}", timeout=10 + ) if reverse_location: return location.latitude, location.longitude, reverse_location.raw.get('address', {}) except: @@ -416,14 +434,18 @@ class WxCommand(BaseCommand): return location.latitude, location.longitude, location.raw.get('address', {}) else: # Try without state as fallback - location = self.geolocator.geocode(f"{city}, USA") + location = rate_limited_nominatim_geocode_sync( + self.bot, f"{city}, USA", timeout=10 + ) if location: # Cache the result self.db_manager.cache_geocoding(f"{city}, USA", location.latitude, location.longitude) # Use reverse geocoding to get detailed address info try: - reverse_location = self.geolocator.reverse(f"{location.latitude}, {location.longitude}") + reverse_location = rate_limited_nominatim_reverse_sync( + self.bot, f"{location.latitude}, {location.longitude}", timeout=10 + ) if reverse_location: return location.latitude, location.longitude, reverse_location.raw.get('address', {}) except: diff --git a/modules/core.py b/modules/core.py index 2c92a02..6d4ef80 100644 --- a/modules/core.py +++ b/modules/core.py @@ -25,7 +25,7 @@ from meshcore import EventType from meshcore_cli.meshcore_cli import send_cmd, send_chan_msg # Import our modules -from .rate_limiter import RateLimiter, BotTxRateLimiter +from .rate_limiter import RateLimiter, BotTxRateLimiter, NominatimRateLimiter from .message_handler import MessageHandler from .command_manager import CommandManager from .channel_manager import ChannelManager @@ -90,6 +90,10 @@ class MeshCoreBot: self.bot_tx_rate_limiter = BotTxRateLimiter( self.config.getfloat('Bot', 'bot_tx_rate_limit_seconds', fallback=1.0) ) + # Nominatim rate limiter: 1.1 seconds between requests (Nominatim policy: max 1 req/sec) + self.nominatim_rate_limiter = NominatimRateLimiter( + self.config.getfloat('Bot', 'nominatim_rate_limit_seconds', fallback=1.1) + ) self.tx_delay_ms = self.config.getint('Bot', 'tx_delay_ms', fallback=250) # Initialize translator for localization BEFORE CommandManager diff --git a/modules/db_manager.py b/modules/db_manager.py index 0625726..57db433 100644 --- a/modules/db_manager.py +++ b/modules/db_manager.py @@ -91,8 +91,8 @@ class DBManager: self.logger.error(f"Error getting cached geocoding: {e}") return None, None - def cache_geocoding(self, query: str, latitude: float, longitude: float, cache_hours: int = 24): - """Cache geocoding result for future use""" + def cache_geocoding(self, query: str, latitude: float, longitude: float, cache_hours: int = 720): + """Cache geocoding result for future use (default: 30 days)""" try: with sqlite3.connect(self.db_path) as conn: cursor = conn.cursor() @@ -148,8 +148,8 @@ class DBManager: return None return None - def cache_json(self, cache_key: str, cache_value: Dict, cache_type: str, cache_hours: int = 24): - """Cache a JSON value for future use""" + def cache_json(self, cache_key: str, cache_value: Dict, cache_type: str, cache_hours: int = 720): + """Cache a JSON value for future use (default: 30 days for geolocation)""" try: json_str = json.dumps(cache_value) self.cache_value(cache_key, json_str, cache_type, cache_hours) diff --git a/modules/rate_limiter.py b/modules/rate_limiter.py index d6144cd..f48d98a 100644 --- a/modules/rate_limiter.py +++ b/modules/rate_limiter.py @@ -55,3 +55,45 @@ class BotTxRateLimiter: wait_time = self.time_until_next_tx() if wait_time > 0: await asyncio.sleep(wait_time + 0.05) # Small buffer + + +class NominatimRateLimiter: + """Rate limiting for Nominatim geocoding API requests + + Nominatim policy: Maximum 1 request per second + We'll be conservative and use 1.1 seconds to ensure compliance + """ + + def __init__(self, seconds: float = 1.1): + self.seconds = seconds + self.last_request = 0 + self._lock = None # Will be set to asyncio.Lock if needed + + def can_request(self) -> bool: + """Check if we can make a Nominatim request""" + return time.time() - self.last_request >= self.seconds + + def time_until_next(self) -> float: + """Get time until next allowed request""" + elapsed = time.time() - self.last_request + return max(0, self.seconds - elapsed) + + def record_request(self): + """Record that we made a Nominatim request""" + self.last_request = time.time() + + async def wait_for_request(self): + """Wait until we can make a Nominatim request (async)""" + import asyncio + while not self.can_request(): + wait_time = self.time_until_next() + if wait_time > 0: + await asyncio.sleep(wait_time + 0.05) # Small buffer + + def wait_for_request_sync(self): + """Wait until we can make a Nominatim request (synchronous)""" + import time as time_module + while not self.can_request(): + wait_time = self.time_until_next() + if wait_time > 0: + time_module.sleep(wait_time + 0.05) # Small buffer diff --git a/modules/repeater_manager.py b/modules/repeater_manager.py index 23377ff..1852bd6 100644 --- a/modules/repeater_manager.py +++ b/modules/repeater_manager.py @@ -12,6 +12,7 @@ from datetime import datetime, timedelta from typing import Dict, List, Optional, Tuple from pathlib import Path from meshcore import EventType +from .utils import rate_limited_nominatim_reverse_sync @@ -1276,13 +1277,10 @@ class RepeaterManager: return existing_data.get('state'), existing_data.get('country') try: - from geopy.geocoders import Nominatim - - # Initialize geocoder with proper timeout - geolocator = Nominatim(user_agent="meshcore-bot", timeout=10) - - # Perform reverse geocoding - location = geolocator.reverse(f"{latitude}, {longitude}") + # Use rate-limited Nominatim reverse geocoding + location = rate_limited_nominatim_reverse_sync( + self.bot, f"{latitude}, {longitude}", timeout=10 + ) if location: address = location.raw.get('address', {}) @@ -1322,13 +1320,10 @@ class RepeaterManager: return existing_data.get('city') try: - from geopy.geocoders import Nominatim - - # Initialize geocoder with proper timeout - geolocator = Nominatim(user_agent="meshcore-bot", timeout=10) - - # Perform reverse geocoding - location = geolocator.reverse(f"{latitude}, {longitude}") + # Use rate-limited Nominatim reverse geocoding + location = rate_limited_nominatim_reverse_sync( + self.bot, f"{latitude}, {longitude}", timeout=10 + ) if location: address = location.raw.get('address', {}) @@ -1400,17 +1395,11 @@ class RepeaterManager: self.logger.debug(f"Using cached location data for {latitude}, {longitude}") return cached_result - from geopy.geocoders import Nominatim - - # Initialize geocoder with proper user agent and timeout - geolocator = Nominatim( - user_agent="meshcore-bot-geolocation-update", - timeout=10 # 10 second timeout - ) - - # Perform reverse geocoding + # Use rate-limited Nominatim reverse geocoding self.logger.debug(f"Calling Nominatim reverse geocoding for {latitude}, {longitude}") - location = geolocator.reverse(f"{latitude}, {longitude}") + location = rate_limited_nominatim_reverse_sync( + self.bot, f"{latitude}, {longitude}", timeout=10 + ) if location: address = location.raw.get('address', {}) @@ -1451,8 +1440,8 @@ class RepeaterManager: else: self.logger.warning(f"Geocoding API returned no location for {latitude}, {longitude}") - # Cache the result for 24 hours to avoid duplicate API calls - self.db_manager.cache_json(cache_key, location_info, "geolocation", cache_hours=24) + # Cache the result for 30 days - geolocation data is very stable + self.db_manager.cache_json(cache_key, location_info, "geolocation", cache_hours=720) return location_info @@ -3473,8 +3462,8 @@ class RepeaterManager: """Get cached geocoding result for a query""" return self.db_manager.get_cached_geocoding(query) - def cache_geocoding(self, query: str, latitude: float, longitude: float, cache_hours: int = 24): - """Cache geocoding result for future use""" + def cache_geocoding(self, query: str, latitude: float, longitude: float, cache_hours: int = 720): + """Cache geocoding result for future use (default: 30 days)""" self.db_manager.cache_geocoding(query, latitude, longitude, cache_hours) def cleanup_geocoding_cache(self): diff --git a/modules/utils.py b/modules/utils.py index dd7f97f..a79c8bf 100644 --- a/modules/utils.py +++ b/modules/utils.py @@ -363,3 +363,138 @@ def calculate_distance(lat1: float, lon1: float, lat2: float, lon2: float) -> fl # Earth's radius in kilometers earth_radius = 6371.0 return earth_radius * c + + +def get_nominatim_geocoder(user_agent: str = "meshcore-bot", timeout: int = 10): + """ + Get a Nominatim geocoder instance with proper User-Agent. + + Args: + user_agent: User-Agent string for Nominatim (required by their policy) + timeout: Request timeout in seconds + + Returns: + Nominatim geocoder instance + """ + from geopy.geocoders import Nominatim + return Nominatim(user_agent=user_agent, timeout=timeout) + + +async def rate_limited_nominatim_geocode(bot, query: str, timeout: int = 10): + """ + Perform rate-limited Nominatim geocoding (forward geocoding). + + Args: + bot: Bot instance (must have nominatim_rate_limiter attribute) + query: Location query string + timeout: Request timeout in seconds + + Returns: + Geocoding result or None + """ + if not hasattr(bot, 'nominatim_rate_limiter'): + # Fallback if rate limiter not initialized + geolocator = get_nominatim_geocoder(timeout=timeout) + return geolocator.geocode(query, timeout=timeout) + + # Wait for rate limiter + await bot.nominatim_rate_limiter.wait_for_request() + + # Make the request + geolocator = get_nominatim_geocoder(timeout=timeout) + result = geolocator.geocode(query, timeout=timeout) + + # Record the request + bot.nominatim_rate_limiter.record_request() + + return result + + +async def rate_limited_nominatim_reverse(bot, coordinates: str, timeout: int = 10): + """ + Perform rate-limited Nominatim reverse geocoding. + + Args: + bot: Bot instance (must have nominatim_rate_limiter attribute) + coordinates: Coordinates string in format "lat, lon" + timeout: Request timeout in seconds + + Returns: + Reverse geocoding result or None + """ + if not hasattr(bot, 'nominatim_rate_limiter'): + # Fallback if rate limiter not initialized + geolocator = get_nominatim_geocoder(timeout=timeout) + return geolocator.reverse(coordinates, timeout=timeout) + + # Wait for rate limiter + await bot.nominatim_rate_limiter.wait_for_request() + + # Make the request + geolocator = get_nominatim_geocoder(timeout=timeout) + result = geolocator.reverse(coordinates, timeout=timeout) + + # Record the request + bot.nominatim_rate_limiter.record_request() + + return result + + +def rate_limited_nominatim_geocode_sync(bot, query: str, timeout: int = 10): + """ + Perform rate-limited Nominatim geocoding (synchronous version). + + Args: + bot: Bot instance (must have nominatim_rate_limiter attribute) + query: Location query string + timeout: Request timeout in seconds + + Returns: + Geocoding result or None + """ + if not hasattr(bot, 'nominatim_rate_limiter'): + # Fallback if rate limiter not initialized + geolocator = get_nominatim_geocoder(timeout=timeout) + return geolocator.geocode(query, timeout=timeout) + + # Wait for rate limiter + bot.nominatim_rate_limiter.wait_for_request_sync() + + # Make the request + geolocator = get_nominatim_geocoder(timeout=timeout) + result = geolocator.geocode(query, timeout=timeout) + + # Record the request + bot.nominatim_rate_limiter.record_request() + + return result + + +def rate_limited_nominatim_reverse_sync(bot, coordinates: str, timeout: int = 10): + """ + Perform rate-limited Nominatim reverse geocoding (synchronous version). + + Args: + bot: Bot instance (must have nominatim_rate_limiter attribute) + coordinates: Coordinates string in format "lat, lon" + timeout: Request timeout in seconds + + Returns: + Reverse geocoding result or None + """ + if not hasattr(bot, 'nominatim_rate_limiter'): + # Fallback if rate limiter not initialized + geolocator = get_nominatim_geocoder(timeout=timeout) + return geolocator.reverse(coordinates, timeout=timeout) + + # Wait for rate limiter + bot.nominatim_rate_limiter.wait_for_request_sync() + + # Make the request + geolocator = get_nominatim_geocoder(timeout=timeout) + result = geolocator.reverse(coordinates, timeout=timeout) + + # Record the request + bot.nominatim_rate_limiter.record_request() + + return result