From 4982ba482d54dcd96c009e0795a015147241b022 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sun, 1 Mar 2026 20:42:06 -0800 Subject: [PATCH] Enhance mesh graph handling for multi-resolution prefix support - Updated `MeshGraph` to support multi-resolution storage of edges, allowing prefixes of 2, 4, or 6 hex chars without truncation. - Implemented prefix matching logic to ensure distinct links are maintained and accurately retrieved based on prefix queries. - Refactored methods in `MessageHandler` and `PathCommand` to accommodate variable prefix lengths during graph lookups. - Enhanced tests to validate prefix match functionality and edge management in the mesh graph. --- config.ini.example | 3 + docs/path-command-config.md | 4 + modules/commands/path_command.py | 96 +++++----- modules/mesh_graph.py | 142 ++++++++++----- modules/message_handler.py | 8 +- modules/web_viewer/app.py | 199 ++++++++++++--------- modules/web_viewer/templates/contacts.html | 16 +- tests/unit/test_mesh_graph.py | 68 ++++++- 8 files changed, 363 insertions(+), 173 deletions(-) diff --git a/config.ini.example b/config.ini.example index c581795..ee0c1ec 100644 --- a/config.ini.example +++ b/config.ini.example @@ -155,6 +155,9 @@ respond_to_dms = true # channel_keywords = # Set a custom prefix length for the public keys to identify repeaters +# 1 = 2 hex chars (e.g. 7E), 2 = 4 hex chars (e.g. 7E42). Also used as the mesh +# graph key length; if the mesh often uses 2-byte paths, set to 2 to avoid +# conflating distinct links (see Path Command / path-command-config.md). prefix_bytes = 1 # Flood scope for channel messages (MeshCore regions; optional) diff --git a/docs/path-command-config.md b/docs/path-command-config.md index 5691281..274901f 100644 --- a/docs/path-command-config.md +++ b/docs/path-command-config.md @@ -52,6 +52,10 @@ path_selection_preset = balanced ## Graph-Based Selection +**Prefix length and graph conflation** + +The graph stores edges using the bot’s prefix length ([Bot] `prefix_bytes`). Paths from packets can be 1-, 2-, or 3-byte encoded (per sender); when we record edges we normalize to the bot’s prefix. If the bot uses **prefix_bytes=1** (2 hex chars) and the mesh often uses 2-byte paths, **distinct links can be merged**: e.g. 7E42→8611 and 7E99→86FF both become a single edge (7e, 86). That can overcount observations and make path resolution ambiguous when several repeaters share the same short prefix. **Recommendation:** set `prefix_bytes` to match the mesh (e.g. 2 if most traffic is 2-byte) so the graph keeps finer resolution and the mesh viewer shows one node per prefix instead of collapsing many repeaters into one. + **`graph_based_validation`** (boolean) - Enable graph-based path validation using observed mesh connections - Default: `true` diff --git a/modules/commands/path_command.py b/modules/commands/path_command.py index 07da28e..7f67519 100644 --- a/modules/commands/path_command.py +++ b/modules/commands/path_command.py @@ -437,8 +437,9 @@ class PathCommand(BaseCommand): # Try graph-based selection if enabled if self.graph_based_validation and hasattr(self.bot, 'mesh_graph') and self.bot.mesh_graph: + path_prefix_hex_chars = len(node_id) graph_repeater, graph_confidence, selection_method = self._select_repeater_by_graph( - recent_repeaters, node_id, node_ids + recent_repeaters, node_id, node_ids, path_prefix_hex_chars=path_prefix_hex_chars ) # Get geographic proximity selection @@ -1281,17 +1282,24 @@ class PathCommand(BaseCommand): return None, 0.0 - def _select_repeater_by_graph(self, repeaters: List[Dict[str, Any]], node_id: str, - path_context: List[str]) -> Tuple[Optional[Dict[str, Any]], float, str]: + def _select_repeater_by_graph(self, repeaters: List[Dict[str, Any]], node_id: str, + path_context: List[str], + path_prefix_hex_chars: Optional[int] = None) -> Tuple[Optional[Dict[str, Any]], float, str]: """Select repeater based on graph evidence. - + Uses enhanced direct-edge validation and multi-hop path inference. - + When the path was decoded with two-byte or three-byte hops, pass path_prefix_hex_chars + (e.g. 4 or 6) so candidate matching uses the full node_id; graph lookups are normalized + to the configured prefix length. + Args: repeaters: List of repeaters to choose from - node_id: The current node ID being processed + node_id: The current node ID being processed (2, 4, or 6 hex chars depending on path) path_context: Full path for context - + path_prefix_hex_chars: Optional. When the path is multi-byte (2 or 3 bytes per hop), + pass hex chars per node (4 or 6). Used for candidate_prefix; graph lookups use + bot.prefix_hex_chars. When None, uses bot.prefix_hex_chars for both. + Returns: Tuple of (selected_repeater, confidence_score, method_name) confidence_score: 0.0 to 1.0, where 1.0 is very confident @@ -1299,63 +1307,71 @@ class PathCommand(BaseCommand): """ if not self.graph_based_validation or not hasattr(self.bot, 'mesh_graph') or not self.bot.mesh_graph: return None, 0.0, None - + mesh_graph = self.bot.mesh_graph + graph_n = getattr(self.bot, 'prefix_hex_chars', 2) + if graph_n <= 0: + graph_n = 2 + # When path has longer node IDs (multi-byte), use that for candidate prefix; normalize for graph + prefix_n = path_prefix_hex_chars if path_prefix_hex_chars is not None and path_prefix_hex_chars >= 2 else graph_n # Find current node position in path try: current_index = path_context.index(node_id) if node_id in path_context else -1 - except: + except Exception: current_index = -1 - + if current_index == -1: return None, 0.0, None - - # Get previous and next node IDs + + # Get previous and next node IDs; normalize to graph prefix length for edge lookups prev_node_id = path_context[current_index - 1] if current_index > 0 else None next_node_id = path_context[current_index + 1] if current_index < len(path_context) - 1 else None - + prev_norm = (prev_node_id[:graph_n].lower() if prev_node_id and len(prev_node_id) > graph_n else (prev_node_id.lower() if prev_node_id else None)) + next_norm = (next_node_id[:graph_n].lower() if next_node_id and len(next_node_id) > graph_n else (next_node_id.lower() if next_node_id else None)) + # Score each candidate based on enhanced graph evidence best_repeater = None best_score = 0.0 best_method = None - + for repeater in repeaters: pk = repeater.get('public_key') or '' - candidate_prefix = self.bot.key_prefix(pk).lower() if pk else None + candidate_prefix = (pk[:prefix_n].lower() if pk else None) candidate_public_key = repeater.get('public_key', '').lower() if repeater.get('public_key') else None if not candidate_prefix: continue - - # First attempt: Enhanced direct-edge validation + candidate_norm = candidate_prefix[:graph_n].lower() if len(candidate_prefix) > graph_n else candidate_prefix + + # First attempt: Enhanced direct-edge validation (use normalized IDs for graph) graph_score = mesh_graph.get_candidate_score( - candidate_prefix, prev_node_id, next_node_id, self.min_edge_observations, + candidate_norm, prev_norm, next_norm, self.min_edge_observations, hop_position=current_index if self.graph_use_hop_position else None, use_bidirectional=self.graph_use_bidirectional, use_hop_position=self.graph_use_hop_position ) - + # Check if edges have stored public keys that match this candidate # This indicates high confidence in the edge and should be prioritized stored_key_bonus = 0.0 if self.graph_prefer_stored_keys and candidate_public_key: # Check edge from previous node to candidate - if prev_node_id: - prev_to_candidate_edge = mesh_graph.get_edge(prev_node_id, candidate_prefix) + if prev_norm: + prev_to_candidate_edge = mesh_graph.get_edge(prev_norm, candidate_norm) if prev_to_candidate_edge: stored_to_key = prev_to_candidate_edge.get('to_public_key', '').lower() if prev_to_candidate_edge.get('to_public_key') else None if stored_to_key and stored_to_key == candidate_public_key: stored_key_bonus = max(stored_key_bonus, 0.4) # Strong bonus for matching stored key - self.logger.debug(f"Found stored public key match for {repeater.get('name', 'unknown')} in edge {prev_node_id}->{candidate_prefix}") - + self.logger.debug(f"Found stored public key match for {repeater.get('name', 'unknown')} in edge {prev_norm}->{candidate_norm}") + # Check edge from candidate to next node - if next_node_id: - candidate_to_next_edge = mesh_graph.get_edge(candidate_prefix, next_node_id) + if next_norm: + candidate_to_next_edge = mesh_graph.get_edge(candidate_norm, next_norm) if candidate_to_next_edge: stored_from_key = candidate_to_next_edge.get('from_public_key', '').lower() if candidate_to_next_edge.get('from_public_key') else None if stored_from_key and stored_from_key == candidate_public_key: stored_key_bonus = max(stored_key_bonus, 0.4) # Strong bonus for matching stored key - self.logger.debug(f"Found stored public key match for {repeater.get('name', 'unknown')} in edge {candidate_prefix}->{next_node_id}") + self.logger.debug(f"Found stored public key match for {repeater.get('name', 'unknown')} in edge {candidate_norm}->{next_norm}") # Zero-hop bonus: If this repeater has been heard directly by the bot (zero-hop advert), # it's strong evidence it's close and should be preferred, even for intermediate hops. @@ -1449,16 +1465,16 @@ class PathCommand(BaseCommand): # Second attempt: Multi-hop inference if direct edges have low confidence multi_hop_score = 0.0 - if self.graph_multi_hop_enabled and graph_score_with_bonus < 0.6 and prev_node_id and next_node_id: + if self.graph_multi_hop_enabled and graph_score_with_bonus < 0.6 and prev_norm and next_norm: # Try to find intermediate nodes that connect prev to next intermediate_candidates = mesh_graph.find_intermediate_nodes( - prev_node_id, next_node_id, self.min_edge_observations, + prev_norm, next_norm, self.min_edge_observations, max_hops=self.graph_multi_hop_max_hops ) - + # Check if our candidate appears in the intermediate nodes list for intermediate_prefix, intermediate_score in intermediate_candidates: - if intermediate_prefix == candidate_prefix: + if intermediate_prefix == candidate_norm: multi_hop_score = intermediate_score break @@ -1468,16 +1484,16 @@ class PathCommand(BaseCommand): # Apply distance penalty for intermediate hops (prevents selecting very distant repeaters) # This is especially important when graph has strong evidence for long-distance links - if self.graph_distance_penalty_enabled and next_node_id is not None: # Not final hop + if self.graph_distance_penalty_enabled and next_norm is not None: # Not final hop repeater_lat = repeater.get('latitude') repeater_lon = repeater.get('longitude') - + if repeater_lat is not None and repeater_lon is not None: max_distance = 0.0 - + # Check distance from previous node to candidate (use stored edge distance if available) - if prev_node_id: - prev_to_candidate_edge = mesh_graph.get_edge(prev_node_id, candidate_prefix) + if prev_norm: + prev_to_candidate_edge = mesh_graph.get_edge(prev_norm, candidate_norm) if prev_to_candidate_edge and prev_to_candidate_edge.get('geographic_distance'): # Use stored geographic distance from edge (most accurate) distance = prev_to_candidate_edge.get('geographic_distance') @@ -1488,10 +1504,10 @@ class PathCommand(BaseCommand): # Note: This is a limitation - we'd need to track previous selections # For now, we'll rely on edge distances which are stored when paths are observed pass - + # Check distance from candidate to next node (use stored edge distance if available) - if next_node_id: - candidate_to_next_edge = mesh_graph.get_edge(candidate_prefix, next_node_id) + if next_norm: + candidate_to_next_edge = mesh_graph.get_edge(candidate_norm, next_norm) if candidate_to_next_edge and candidate_to_next_edge.get('geographic_distance'): distance = candidate_to_next_edge.get('geographic_distance') max_distance = max(max_distance, distance) @@ -1513,8 +1529,8 @@ class PathCommand(BaseCommand): small_penalty = (max_distance - self.graph_max_reasonable_hop_distance_km * 0.8) / (self.graph_max_reasonable_hop_distance_km * 0.2) * self.graph_distance_penalty_strength * 0.5 candidate_score = candidate_score * (1.0 - small_penalty) - # For final hop (next_node_id is None), add bot location proximity bonus - if next_node_id is None and self.graph_final_hop_proximity_enabled: + # For final hop (next_norm is None), add bot location proximity bonus + if next_norm is None and self.graph_final_hop_proximity_enabled: if self.bot_latitude is not None and self.bot_longitude is not None: repeater_lat = repeater.get('latitude') repeater_lon = repeater.get('longitude') diff --git a/modules/mesh_graph.py b/modules/mesh_graph.py index d52c6a7..790a635 100644 --- a/modules/mesh_graph.py +++ b/modules/mesh_graph.py @@ -3,6 +3,14 @@ Mesh Graph Module Tracks observed connections between repeaters to improve path guessing accuracy. Persists graph state across bot restarts for development scenarios. + +Multi-resolution storage: + Edges are stored at the resolution observed (2, 4, or 6 hex chars per node). No + truncation on write; lookup supports prefix matching so get_edge(7e, 86) can + return stored (7e42, 8611). Distinct links (e.g. 7e42→8611 and 7e99→86ff) stay + separate. When multiple edges match a short-prefix query, we return one edge by + tie-break (most specific, then observation_count, then last_seen); we never sum + observation counts across candidates. """ import sqlite3 @@ -74,6 +82,54 @@ class MeshGraph: except (TypeError, ValueError): return 2 + def _valid_prefix_length(self, prefix: str) -> bool: + """Return True if prefix has 2, 4, or 6 hex chars (after stripping).""" + if not prefix or not isinstance(prefix, str): + return False + s = prefix.strip().lower() + if len(s) not in (2, 4, 6): + return False + return all(c in '0123456789abcdef' for c in s) + + def _prefix_match(self, a: str, b: str) -> bool: + """Return True if a and b match: exact or one is a prefix of the other (after lowercasing).""" + if not a or not b: + return False + a, b = a.lower().strip(), b.lower().strip() + return a == b or a.startswith(b) or b.startswith(a) + + def _get_edge_by_prefix_match(self, from_q: str, to_q: str) -> Optional[Dict]: + """Return the best matching edge for a prefix query. Single edge only; never merge counts. + Tie-break: exact key if present, else longest combined prefix length, then max + observation_count, then most recent last_seen. + """ + from_q = from_q.lower().strip() + to_q = to_q.lower().strip() + candidates = [] + for edge_key, edge in self.edges.items(): + from_p, to_p = edge_key + if self._prefix_match(from_p, from_q) and self._prefix_match(to_p, to_q): + candidates.append(edge) + if not candidates: + return None + # Prefer exact match + for e in candidates: + if e['from_prefix'] == from_q and e['to_prefix'] == to_q: + return e + # Tie-break: longest combined prefix length, then max observation_count, then most recent last_seen + def key(e): + spec = len(e['from_prefix']) + len(e['to_prefix']) + obs = e.get('observation_count', 0) + last = e.get('last_seen') + if isinstance(last, str): + try: + last = datetime.fromisoformat(last.replace('Z', '+00:00')) + except ValueError: + last = datetime.min + last_ts = last if isinstance(last, datetime) else datetime.min + return (spec, obs, last_ts) + return max(candidates, key=key) + def _load_from_database(self): """Load graph edges from database on startup.""" try: @@ -159,9 +215,13 @@ class MeshGraph: prefix_bytes: int = 1): """Add or update an edge in the graph. + Prefixes are stored at the resolution provided (2, 4, or 6 hex chars). + No truncation; the same physical link can be recorded at different + resolutions and will create/update the matching edge. + Args: - from_prefix: Source node prefix (2 hex chars, or 4 when prefix_bytes=2). - to_prefix: Destination node prefix (2 hex chars, or 4 when prefix_bytes=2). + from_prefix: Source node prefix (2, 4, or 6 hex chars depending on path encoding). + to_prefix: Destination node prefix (2, 4, or 6 hex chars depending on path encoding). from_public_key: Full public key of source node (optional). to_public_key: Full public key of destination node (optional). hop_position: Position in path where this edge was observed (optional). @@ -175,9 +235,11 @@ class MeshGraph: if not self.capture_enabled: return - # Normalize prefixes to lowercase - from_prefix = from_prefix.lower()[:self._prefix_len()] - to_prefix = to_prefix.lower()[:self._prefix_len()] + # Normalize to lowercase; validate length (2, 4, or 6 hex chars). No truncation. + from_prefix = from_prefix.lower().strip() + to_prefix = to_prefix.lower().strip() + if not self._valid_prefix_length(from_prefix) or not self._valid_prefix_length(to_prefix): + return # Intern public key strings so repeated identical keys share one object in RAM if from_public_key: @@ -813,74 +875,70 @@ class MeshGraph: self._flush_pending_updates_sync() def has_edge(self, from_prefix: str, to_prefix: str) -> bool: - """Check if an edge exists in the graph. - + """Check if an edge exists in the graph (exact or prefix match). + Args: from_prefix: Source node prefix. to_prefix: Destination node prefix. - + Returns: bool: True if edge exists. """ - from_prefix = from_prefix.lower()[:self._prefix_len()] - to_prefix = to_prefix.lower()[:self._prefix_len()] - return (from_prefix, to_prefix) in self.edges - + return self.get_edge(from_prefix, to_prefix) is not None + def get_edge(self, from_prefix: str, to_prefix: str) -> Optional[Dict]: - """Get edge data if it exists. - + """Get edge data if it exists (exact key first, then prefix match). + Args: from_prefix: Source node prefix. to_prefix: Destination node prefix. - + Returns: Dict with edge data or None if not found. """ - from_prefix = from_prefix.lower()[:self._prefix_len()] - to_prefix = to_prefix.lower()[:self._prefix_len()] - return self.edges.get((from_prefix, to_prefix)) + from_norm = from_prefix.lower().strip() if from_prefix else "" + to_norm = to_prefix.lower().strip() if to_prefix else "" + if not from_norm or not to_norm: + return None + # Exact key first + exact = self.edges.get((from_norm, to_norm)) + if exact is not None: + return exact + return self._get_edge_by_prefix_match(from_norm, to_norm) def get_outgoing_edges(self, prefix: str) -> List[Dict]: - """Get all edges originating from a node. - - Uses the adjacency index for O(1) lookup instead of a full scan. + """Get all edges originating from a node (prefix match: returns edges where from_prefix matches prefix). Args: - prefix: Node prefix. + prefix: Node prefix (2, 4, or 6 hex chars). Returns: List of edge dictionaries. """ - prefix = prefix.lower()[:self._prefix_len()] - to_prefixes = self._outgoing_index.get(prefix) - if not to_prefixes: + prefix = prefix.lower().strip() if prefix else "" + if not prefix: return [] result = [] - for to_prefix in to_prefixes: - edge = self.edges.get((prefix, to_prefix)) - if edge is not None: + for edge in self.edges.values(): + if self._prefix_match(edge['from_prefix'], prefix): result.append(edge) return result def get_incoming_edges(self, prefix: str) -> List[Dict]: - """Get all edges ending at a node. - - Uses the adjacency index for O(1) lookup instead of a full scan. + """Get all edges ending at a node (prefix match: returns edges where to_prefix matches prefix). Args: - prefix: Node prefix. + prefix: Node prefix (2, 4, or 6 hex chars). Returns: List of edge dictionaries. """ - prefix = prefix.lower()[:self._prefix_len()] - from_prefixes = self._incoming_index.get(prefix) - if not from_prefixes: + prefix = prefix.lower().strip() if prefix else "" + if not prefix: return [] result = [] - for from_prefix in from_prefixes: - edge = self.edges.get((from_prefix, prefix)) - if edge is not None: + for edge in self.edges.values(): + if self._prefix_match(edge['to_prefix'], prefix): result.append(edge) return result @@ -1077,9 +1135,11 @@ class MeshGraph: List of (candidate_prefix, score) tuples sorted by score (highest first). Score is 0.0-1.0 based on path strength. """ - from_prefix = from_prefix.lower()[:self._prefix_len()] - to_prefix = to_prefix.lower()[:self._prefix_len()] - + from_prefix = from_prefix.lower().strip() if from_prefix else "" + to_prefix = to_prefix.lower().strip() if to_prefix else "" + if not from_prefix or not to_prefix: + return [] + candidates: Dict[str, float] = {} # Try 2-hop paths first: from_prefix -> intermediate -> to_prefix diff --git a/modules/message_handler.py b/modules/message_handler.py index a9f0970..7655ab1 100644 --- a/modules/message_handler.py +++ b/modules/message_handler.py @@ -1790,9 +1790,13 @@ class MessageHandler: def _update_mesh_graph(self, path_nodes: List[str], packet_info: Dict[str, Any]): """Update mesh graph with edges from a message path. - + + path_nodes may be 2, 4, or 6 hex chars per node depending on the packet's + bytes_per_hop (sender setting). add_edge stores at the resolution provided; + no truncation, so distinct links (e.g. 7e42→8611 and 7e99→86ff) stay separate. + Args: - path_nodes: List of node prefixes in path order. + path_nodes: List of node prefixes in path order (length per node from packet's bytes_per_hop). packet_info: Packet information dictionary with routing data. """ if not path_nodes or len(path_nodes) < 2: diff --git a/modules/web_viewer/app.py b/modules/web_viewer/app.py index 12a4f27..63a9378 100644 --- a/modules/web_viewer/app.py +++ b/modules/web_viewer/app.py @@ -673,56 +673,64 @@ class BotDataViewer: return None, 0.0 # Helper for graph-based selection (same as PathCommand._select_repeater_by_graph) + # When path was decoded with 2-byte or 3-byte hops, node_id/path_context have 4 or 6 hex chars; + # use path_prefix_hex_chars for candidate matching and normalize to graph_n for edge lookups. def select_repeater_by_graph(repeaters, node_id, path_context): if not graph_based_validation or not hasattr(self, 'mesh_graph') or not self.mesh_graph: return None, 0.0, None - + mesh_graph = self.mesh_graph - + graph_n = prefix_hex_chars # graph is keyed by config prefix length + path_prefix_hex_chars = len(node_id) if node_id else graph_n + prefix_n = path_prefix_hex_chars if path_prefix_hex_chars >= 2 else graph_n + try: current_index = path_context.index(node_id) if node_id in path_context else -1 - except: + except Exception: current_index = -1 - + if current_index == -1: return None, 0.0, None - + prev_node_id = path_context[current_index - 1] if current_index > 0 else None next_node_id = path_context[current_index + 1] if current_index < len(path_context) - 1 else None - + prev_norm = (prev_node_id[:graph_n].lower() if prev_node_id and len(prev_node_id) > graph_n else (prev_node_id.lower() if prev_node_id else None)) + next_norm = (next_node_id[:graph_n].lower() if next_node_id and len(next_node_id) > graph_n else (next_node_id.lower() if next_node_id else None)) + best_repeater = None best_score = 0.0 best_method = None - + for repeater in repeaters: - candidate_prefix = repeater.get('public_key', '')[:prefix_hex_chars].lower() if repeater.get('public_key') else None + candidate_prefix = repeater.get('public_key', '')[:prefix_n].lower() if repeater.get('public_key') else None candidate_public_key = repeater.get('public_key', '').lower() if repeater.get('public_key') else None if not candidate_prefix: continue - + candidate_norm = candidate_prefix[:graph_n].lower() if len(candidate_prefix) > graph_n else candidate_prefix + graph_score = mesh_graph.get_candidate_score( - candidate_prefix, prev_node_id, next_node_id, min_edge_observations, + candidate_norm, prev_norm, next_norm, min_edge_observations, hop_position=current_index if graph_use_hop_position else None, use_bidirectional=graph_use_bidirectional, use_hop_position=graph_use_hop_position ) - + stored_key_bonus = 0.0 if graph_prefer_stored_keys and candidate_public_key: - if prev_node_id: - prev_to_candidate_edge = mesh_graph.get_edge(prev_node_id, candidate_prefix) + if prev_norm: + prev_to_candidate_edge = mesh_graph.get_edge(prev_norm, candidate_norm) if prev_to_candidate_edge: stored_to_key = prev_to_candidate_edge.get('to_public_key', '').lower() if prev_to_candidate_edge.get('to_public_key') else None if stored_to_key and stored_to_key == candidate_public_key: stored_key_bonus = max(stored_key_bonus, 0.4) - - if next_node_id: - candidate_to_next_edge = mesh_graph.get_edge(candidate_prefix, next_node_id) + + if next_norm: + candidate_to_next_edge = mesh_graph.get_edge(candidate_norm, next_norm) if candidate_to_next_edge: stored_from_key = candidate_to_next_edge.get('from_public_key', '').lower() if candidate_to_next_edge.get('from_public_key') else None if stored_from_key and stored_from_key == candidate_public_key: stored_key_bonus = max(stored_key_bonus, 0.4) - + # Zero-hop bonus: If this repeater has been heard directly by the bot (zero-hop advert), # it's strong evidence it's close and should be preferred, even for intermediate hops zero_hop_bonus = 0.0 @@ -730,43 +738,43 @@ class BotDataViewer: if hop_count is not None and hop_count == 0: # This repeater has been heard directly - strong evidence it's close to bot zero_hop_bonus = graph_zero_hop_bonus - + graph_score_with_bonus = min(1.0, graph_score + stored_key_bonus + zero_hop_bonus) - + multi_hop_score = 0.0 - if graph_multi_hop_enabled and graph_score_with_bonus < 0.6 and prev_node_id and next_node_id: + if graph_multi_hop_enabled and graph_score_with_bonus < 0.6 and prev_norm and next_norm: intermediate_candidates = mesh_graph.find_intermediate_nodes( - prev_node_id, next_node_id, min_edge_observations, + prev_norm, next_norm, min_edge_observations, max_hops=graph_multi_hop_max_hops ) - + for intermediate_prefix, intermediate_score in intermediate_candidates: - if intermediate_prefix == candidate_prefix: + if intermediate_prefix == candidate_norm: multi_hop_score = intermediate_score break - + candidate_score = max(graph_score_with_bonus, multi_hop_score) method = 'graph_multihop' if multi_hop_score > graph_score_with_bonus else 'graph' - + # Apply distance penalty for intermediate hops (prevents selecting very distant repeaters) # This is especially important when graph has strong evidence for long-distance links - if graph_distance_penalty_enabled and next_node_id is not None: # Not final hop + if graph_distance_penalty_enabled and next_norm is not None: # Not final hop repeater_lat = repeater.get('latitude') repeater_lon = repeater.get('longitude') - + if repeater_lat is not None and repeater_lon is not None: max_distance = 0.0 - + # Check distance from previous node to candidate (use stored edge distance if available) - if prev_node_id: - prev_to_candidate_edge = mesh_graph.get_edge(prev_node_id, candidate_prefix) + if prev_norm: + prev_to_candidate_edge = mesh_graph.get_edge(prev_norm, candidate_norm) if prev_to_candidate_edge and prev_to_candidate_edge.get('geographic_distance'): distance = prev_to_candidate_edge.get('geographic_distance') max_distance = max(max_distance, distance) - + # Check distance from candidate to next node (use stored edge distance if available) - if next_node_id: - candidate_to_next_edge = mesh_graph.get_edge(candidate_prefix, next_node_id) + if next_norm: + candidate_to_next_edge = mesh_graph.get_edge(candidate_norm, next_norm) if candidate_to_next_edge and candidate_to_next_edge.get('geographic_distance'): distance = candidate_to_next_edge.get('geographic_distance') max_distance = max(max_distance, distance) @@ -782,10 +790,10 @@ class BotDataViewer: if max_distance > graph_max_reasonable_hop_distance_km * 0.8: small_penalty = (max_distance - graph_max_reasonable_hop_distance_km * 0.8) / (graph_max_reasonable_hop_distance_km * 0.2) * graph_distance_penalty_strength * 0.5 candidate_score = candidate_score * (1.0 - small_penalty) - - # For final hop (next_node_id is None), add bot location proximity bonus + + # For final hop (next_norm is None), add bot location proximity bonus # This is critical for final hop selection - the last repeater before the bot should be close - if next_node_id is None and graph_final_hop_proximity_enabled: + if next_norm is None and graph_final_hop_proximity_enabled: if bot_latitude is not None and bot_longitude is not None: repeater_lat = repeater.get('latitude') repeater_lon = repeater.get('longitude') @@ -1795,24 +1803,35 @@ class BotDataViewer: @self.app.route('/api/decode-path', methods=['POST']) def api_decode_path(): - """Decode path hex string to repeater names (similar to path command)""" + """Decode path hex string to repeater names (similar to path command). + Optional bytes_per_hop (1, 2, or 3): use when path came from a packet with multi-byte hops + so decoding and graph selection use the correct prefix length.""" try: data = request.get_json() if not data or 'path_hex' not in data: return jsonify({'error': 'path_hex is required'}), 400 - + path_hex = data['path_hex'] if not path_hex: return jsonify({'error': 'path_hex cannot be empty'}), 400 - - # Decode the path - decoded_path = self._decode_path_hex(path_hex) - + + bytes_per_hop = data.get('bytes_per_hop') + if bytes_per_hop is not None: + try: + bytes_per_hop = int(bytes_per_hop) + if bytes_per_hop not in (1, 2, 3): + bytes_per_hop = None + except (TypeError, ValueError): + bytes_per_hop = None + + # Decode the path (use bytes_per_hop when provided, e.g. from packet/contact) + decoded_path = self._decode_path_hex(path_hex, bytes_per_hop=bytes_per_hop) + return jsonify({ 'success': True, 'path': decoded_path }) - + except Exception as e: self.logger.error(f"Error decoding path: {e}", exc_info=True) return jsonify({'error': str(e)}), 500 @@ -3624,7 +3643,7 @@ class BotDataViewer: 'is_starred': bool(row['is_starred'] if row['is_starred'] is not None else 0), 'out_path': row['out_path'] if row['out_path'] is not None else '', 'out_path_len': row['out_path_len'] if row['out_path_len'] is not None else -1, - 'out_bytes_per_hop': row['out_bytes_per_hop'] if row.get('out_bytes_per_hop') is not None else None, + 'out_bytes_per_hop': row['out_bytes_per_hop'] if row['out_bytes_per_hop'] is not None else None, 'all_paths': all_paths }) @@ -5174,17 +5193,23 @@ class BotDataViewer: 'error': str(e) } - def _decode_path_hex(self, path_hex: str) -> List[Dict[str, Any]]: + def _decode_path_hex(self, path_hex: str, bytes_per_hop: Optional[int] = None) -> List[Dict[str, Any]]: """ Decode hex path string to repeater names using the same sophisticated logic as path command. Returns a list of dictionaries with node_id and repeater info. + + When the path came from a packet with 2-byte or 3-byte hops, pass bytes_per_hop (2 or 3) + so node IDs and graph selection use the correct prefix length. """ import re import math from datetime import datetime - - # Parse the path input - handle various formats - prefix_hex_chars = self.config.getint('Bot', 'prefix_bytes', fallback=1) * 2 + + # Parse the path input - use bytes_per_hop when provided (e.g. from packet/contact) + if bytes_per_hop is not None and bytes_per_hop in (1, 2, 3): + prefix_hex_chars = bytes_per_hop * 2 + else: + prefix_hex_chars = self.config.getint('Bot', 'prefix_bytes', fallback=1) * 2 if prefix_hex_chars <= 0: prefix_hex_chars = 2 path_input_clean = path_hex.replace(' ', '').replace(',', '').replace(':', '') @@ -5320,56 +5345,64 @@ class BotDataViewer: return scored_repeaters # Helper: graph-based selection with final hop proximity and path validation + # When path was decoded with 2-byte or 3-byte hops, node_id/path_context have 4 or 6 hex chars; + # use path_prefix_hex_chars for candidate matching and normalize to graph_n for edge lookups. def select_repeater_by_graph(repeaters, node_id, path_context): if not graph_based_validation or not hasattr(self, 'mesh_graph') or not self.mesh_graph: return None, 0.0, None - + mesh_graph = self.mesh_graph - + graph_n = prefix_hex_chars + path_prefix_hex_chars = len(node_id) if node_id else graph_n + prefix_n = path_prefix_hex_chars if path_prefix_hex_chars >= 2 else graph_n + try: current_index = path_context.index(node_id) if node_id in path_context else -1 - except: + except Exception: current_index = -1 - + if current_index == -1: return None, 0.0, None - + prev_node_id = path_context[current_index - 1] if current_index > 0 else None next_node_id = path_context[current_index + 1] if current_index < len(path_context) - 1 else None - + prev_norm = (prev_node_id[:graph_n].lower() if prev_node_id and len(prev_node_id) > graph_n else (prev_node_id.lower() if prev_node_id else None)) + next_norm = (next_node_id[:graph_n].lower() if next_node_id and len(next_node_id) > graph_n else (next_node_id.lower() if next_node_id else None)) + best_repeater = None best_score = 0.0 best_method = None - + for repeater in repeaters: - candidate_prefix = repeater.get('public_key', '')[:prefix_hex_chars].lower() if repeater.get('public_key') else None + candidate_prefix = repeater.get('public_key', '')[:prefix_n].lower() if repeater.get('public_key') else None candidate_public_key = repeater.get('public_key', '').lower() if repeater.get('public_key') else None if not candidate_prefix: continue - + candidate_norm = candidate_prefix[:graph_n].lower() if len(candidate_prefix) > graph_n else candidate_prefix + graph_score = mesh_graph.get_candidate_score( - candidate_prefix, prev_node_id, next_node_id, min_edge_observations, + candidate_norm, prev_norm, next_norm, min_edge_observations, hop_position=current_index if graph_use_hop_position else None, use_bidirectional=graph_use_bidirectional, use_hop_position=graph_use_hop_position ) - + stored_key_bonus = 0.0 if graph_prefer_stored_keys and candidate_public_key: - if prev_node_id: - prev_to_candidate_edge = mesh_graph.get_edge(prev_node_id, candidate_prefix) + if prev_norm: + prev_to_candidate_edge = mesh_graph.get_edge(prev_norm, candidate_norm) if prev_to_candidate_edge: stored_to_key = prev_to_candidate_edge.get('to_public_key', '').lower() if prev_to_candidate_edge.get('to_public_key') else None if stored_to_key and stored_to_key == candidate_public_key: stored_key_bonus = max(stored_key_bonus, 0.4) - - if next_node_id: - candidate_to_next_edge = mesh_graph.get_edge(candidate_prefix, next_node_id) + + if next_norm: + candidate_to_next_edge = mesh_graph.get_edge(candidate_norm, next_norm) if candidate_to_next_edge: stored_from_key = candidate_to_next_edge.get('from_public_key', '').lower() if candidate_to_next_edge.get('from_public_key') else None if stored_from_key and stored_from_key == candidate_public_key: stored_key_bonus = max(stored_key_bonus, 0.4) - + # Zero-hop bonus: If this repeater has been heard directly by the bot (zero-hop advert), # it's strong evidence it's close and should be preferred, even for intermediate hops zero_hop_bonus = 0.0 @@ -5377,42 +5410,42 @@ class BotDataViewer: if hop_count is not None and hop_count == 0: # This repeater has been heard directly - strong evidence it's close to bot zero_hop_bonus = graph_zero_hop_bonus - + graph_score_with_bonus = min(1.0, graph_score + stored_key_bonus + zero_hop_bonus) - + multi_hop_score = 0.0 - if graph_multi_hop_enabled and graph_score_with_bonus < 0.6 and prev_node_id and next_node_id: + if graph_multi_hop_enabled and graph_score_with_bonus < 0.6 and prev_norm and next_norm: intermediate_candidates = mesh_graph.find_intermediate_nodes( - prev_node_id, next_node_id, min_edge_observations, + prev_norm, next_norm, min_edge_observations, max_hops=graph_multi_hop_max_hops ) for intermediate_prefix, intermediate_score in intermediate_candidates: - if intermediate_prefix == candidate_prefix: + if intermediate_prefix == candidate_norm: multi_hop_score = intermediate_score break - + candidate_score = max(graph_score_with_bonus, multi_hop_score) method = 'graph_multihop' if multi_hop_score > graph_score_with_bonus else 'graph' - + # Apply distance penalty for intermediate hops (prevents selecting very distant repeaters) # This is especially important when graph has strong evidence for long-distance links - if graph_distance_penalty_enabled and next_node_id is not None: # Not final hop + if graph_distance_penalty_enabled and next_norm is not None: # Not final hop repeater_lat = repeater.get('latitude') repeater_lon = repeater.get('longitude') - + if repeater_lat is not None and repeater_lon is not None: max_distance = 0.0 - + # Check distance from previous node to candidate (use stored edge distance if available) - if prev_node_id: - prev_to_candidate_edge = mesh_graph.get_edge(prev_node_id, candidate_prefix) + if prev_norm: + prev_to_candidate_edge = mesh_graph.get_edge(prev_norm, candidate_norm) if prev_to_candidate_edge and prev_to_candidate_edge.get('geographic_distance'): distance = prev_to_candidate_edge.get('geographic_distance') max_distance = max(max_distance, distance) - + # Check distance from candidate to next node (use stored edge distance if available) - if next_node_id: - candidate_to_next_edge = mesh_graph.get_edge(candidate_prefix, next_node_id) + if next_norm: + candidate_to_next_edge = mesh_graph.get_edge(candidate_norm, next_norm) if candidate_to_next_edge and candidate_to_next_edge.get('geographic_distance'): distance = candidate_to_next_edge.get('geographic_distance') max_distance = max(max_distance, distance) @@ -5428,10 +5461,10 @@ class BotDataViewer: if max_distance > graph_max_reasonable_hop_distance_km * 0.8: small_penalty = (max_distance - graph_max_reasonable_hop_distance_km * 0.8) / (graph_max_reasonable_hop_distance_km * 0.2) * graph_distance_penalty_strength * 0.5 candidate_score = candidate_score * (1.0 - small_penalty) - - # For final hop (next_node_id is None), add bot location proximity bonus + + # For final hop (next_norm is None), add bot location proximity bonus # This is critical for final hop selection - the last repeater before the bot should be close - if next_node_id is None and graph_final_hop_proximity_enabled: + if next_norm is None and graph_final_hop_proximity_enabled: if bot_latitude is not None and bot_longitude is not None: repeater_lat = repeater.get('latitude') repeater_lon = repeater.get('longitude') diff --git a/modules/web_viewer/templates/contacts.html b/modules/web_viewer/templates/contacts.html index 341d711..0d4d518 100644 --- a/modules/web_viewer/templates/contacts.html +++ b/modules/web_viewer/templates/contacts.html @@ -1324,25 +1324,29 @@ class ModernContactsManager { async decodePathInModal(pathHex, pathIndex) { const decodedDiv = document.getElementById(`decoded-path-${pathIndex}`); const button = document.querySelector(`#path-item-${pathIndex} button`); - + if (!decodedDiv || !button) return; - + // Show loading state const originalHtml = button.innerHTML; button.disabled = true; button.innerHTML = ' Decoding...'; decodedDiv.style.display = 'block'; decodedDiv.innerHTML = '
Decoding path...
'; - + try { + // Use contact's out_bytes_per_hop when path came from a packet with multi-byte hops + const bytesPerHop = this.currentModalContact?.out_bytes_per_hop || null; + const body = { path_hex: pathHex }; + if (bytesPerHop != null && [1, 2, 3].includes(Number(bytesPerHop))) { + body.bytes_per_hop = Number(bytesPerHop); + } const response = await fetch('/api/decode-path', { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - path_hex: pathHex - }) + body: JSON.stringify(body) }); const data = await response.json(); diff --git a/tests/unit/test_mesh_graph.py b/tests/unit/test_mesh_graph.py index 8d1db68..3091fce 100644 --- a/tests/unit/test_mesh_graph.py +++ b/tests/unit/test_mesh_graph.py @@ -138,10 +138,76 @@ class TestMeshGraphEdgeManagement: assert mesh_graph.get_edge("01", "7E") is not None +@pytest.mark.unit +class TestMeshGraphMultiResolution: + """Test multi-resolution storage and prefix-match lookup.""" + + def test_prefix_match_lookup(self, mesh_graph): + """Store (7e42, 8611); get_edge(7e, 86) returns it via prefix match; get_edge(7e42, 8611) exact.""" + mesh_graph.add_edge("7e42", "8611") + # Short-prefix query returns the stored edge via prefix match + edge_short = mesh_graph.get_edge("7e", "86") + assert edge_short is not None + assert edge_short["from_prefix"] == "7e42" + assert edge_short["to_prefix"] == "8611" + # Exact query returns the same edge + edge_exact = mesh_graph.get_edge("7e42", "8611") + assert edge_exact is not None + assert edge_exact["from_prefix"] == "7e42" + assert edge_exact["to_prefix"] == "8611" + assert mesh_graph.has_edge("7e", "86") is True + assert mesh_graph.has_edge("7e42", "8611") is True + + def test_prefix_match_best_of_two(self, mesh_graph): + """Two edges (7e42, 8611) and (7e99, 86ff); get_edge(7e, 86) returns one by tie-break (e.g. observation_count).""" + mesh_graph.add_edge("7e42", "8611") + for _ in range(4): + mesh_graph.add_edge("7e42", "8611") # 5 observations + mesh_graph.add_edge("7e99", "86ff") # 1 observation + edge = mesh_graph.get_edge("7e", "86") + assert edge is not None + # Tie-break: same specificity (4+4), then higher observation_count wins + assert edge["from_prefix"] == "7e42" + assert edge["to_prefix"] == "8611" + assert edge["observation_count"] == 5 + # Exact lookups still work + assert mesh_graph.get_edge("7e42", "8611")["observation_count"] == 5 + assert mesh_graph.get_edge("7e99", "86ff")["observation_count"] == 1 + + def test_get_outgoing_edges_prefix_match(self, mesh_graph): + """get_outgoing_edges(7e) returns edges from 7e42 and 7e99 when both exist.""" + mesh_graph.add_edge("7e42", "8611") + mesh_graph.add_edge("7e99", "86ff") + outgoing = mesh_graph.get_outgoing_edges("7e") + assert len(outgoing) == 2 + from_prefs = {e["from_prefix"] for e in outgoing} + assert from_prefs == {"7e42", "7e99"} + + def test_get_incoming_edges_prefix_match(self, mesh_graph): + """get_incoming_edges(86) returns edges to 8611 and 86ff when both exist.""" + mesh_graph.add_edge("7e42", "8611") + mesh_graph.add_edge("7e99", "86ff") + incoming = mesh_graph.get_incoming_edges("86") + assert len(incoming) == 2 + to_prefs = {e["to_prefix"] for e in incoming} + assert to_prefs == {"8611", "86ff"} + + def test_add_edge_rejects_invalid_length(self, mesh_graph): + """add_edge rejects prefixes that are not 2, 4, or 6 hex chars.""" + mesh_graph.add_edge("7", "86") # 1 char + assert mesh_graph.get_edge("7", "86") is None + assert len(mesh_graph.edges) == 0 + mesh_graph.add_edge("7e4", "8611") # 3 chars + assert len(mesh_graph.edges) == 0 + mesh_graph.add_edge("7e42", "8611") # 4 chars valid + assert mesh_graph.get_edge("7e42", "8611") is not None + assert len(mesh_graph.edges) == 1 + + @pytest.mark.unit class TestMeshGraphPathValidation: """Test path segment and full path validation.""" - + def test_validate_path_segment_valid(self, mesh_graph): """Test validation with sufficient observations.""" from_prefix = "01"