fix(neighbors): update zero-hop neighbor discovery and data handling

This commit enhances the handling of zero-hop neighbors in the dashboard and database. It ensures that the **One-hop neighbours** section accurately reflects radios heard directly (MeshCore hop count 0) rather than originators of relayed adverts. The `observed_paths` table now includes nullable `snr` and `rssi` columns for zero-hop advert rows, allowing for better signal reporting. Additionally, a one-time backfill process copies recent zero-hop ADVERTs from the `packet_stream` to `observed_paths`. Documentation and tests have been updated to reflect these changes.
This commit is contained in:
agessaman
2026-08-12 18:01:28 -07:00
parent 2fea97b21d
commit a594b72a85
12 changed files with 728 additions and 118 deletions
+15
View File
@@ -4,6 +4,21 @@ All notable changes to this project are documented here. The format loosely foll
[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the project adheres to
semantic versioning.
## [Unreleased]
### Fixed
- Dashboard **One-hop neighbours** now lists radios this node heard directly
(MeshCore hop count 0: empty RF path), not originators of 1-hop relayed
adverts. Empty-path adverts are stored in `observed_paths` with SNR/RSSI;
neighbor-discover cycles refresh SNR on those rows. A one-time backfill
copies recent zero-hop ADVERTs out of `packet_stream`.
### Added
- Migration 23: nullable `snr` / `rssi` columns on `observed_paths` for
zero-hop advert rows.
## [1.0.0] — 2026-08-07
v1.0.0 marks the first stable release. It adds zero-hop neighbor discovery, a
+15 -10
View File
@@ -148,7 +148,8 @@ proxy_set_header X-Forwarded-Proto $scheme;
`RESPONSE`, `REQ`, `PATH`, `TXT_MSG`, `ANON_REQ`, `GRP_DATA`, `ADVERT`, and
`Other` for the rest). Its y-axis is the tallest bar rounded up to the next
5%, so it rescales as the mesh changes
- Busiest repeaters, and **one-hop neighbours** (24-hour or 7-day window)
- Busiest repeaters, and **one-hop neighbours** (radios heard directly; 24-hour
or 7-day window)
The live packet feed lives on the **Real-time** page rather than here; the
dashboard reads a single snapshot per poll and holds no streaming
@@ -178,15 +179,19 @@ packets withheld is printed beneath the chart, and percentages stay shares of
the full series so that hiding the tail cannot inflate the remaining bars. The
node series is shown in full.
**Neighbour signal is reported only where two sources agree.**
`complete_contact_tracking.hop_count` is not a reliable direct-neighbour marker:
on a representative database it claims 800 zero-hop contacts while only 68 have
any one-hop path to corroborate it, and the SNR stored against them clusters in
a ~1.5 dB band with RSSI near -45 dBm — one strong local link recorded against
every node whose traffic arrived through it. Neighbour membership therefore
comes from path evidence, and SNR/RSSI appear only when the stored hop count
agrees; otherwise the row reads "no signal reading". A relayed packet's SNR
measures the last hop into this radio, never the link to whoever sent it.
**Neighbour membership is direct RF (MeshCore hop count 0).**
An empty path means this radio heard the originator on the air. A path whose
byte length equals `bytes_per_hop` already contains one hop hash — that
originator is one repeater away, not a neighbour. The dashboard lists empty-path
adverts in `observed_paths` plus in-window rows from `neighbor_links` (zero-hop
node-discover). `complete_contact_tracking.hop_count` is not used for
membership: on a representative database it claims 800 zero-hop contacts while
only a few dozen have any empty-path advert to corroborate it, and the SNR
stored against them clusters in a ~1.5 dB band with RSSI near -45 dBm — one
strong local link recorded against every node whose traffic arrived through it.
SNR/RSSI on the panel come from the zero-hop path row or from discover (SNR
only). A relayed packet's SNR measures the last hop into this radio, never the
link to whoever sent it.
- **Bot**: messages, commands, reply rate, and unique users, plus the top
commands/users/channels and longest paths
- Live activity feed
+14
View File
@@ -788,6 +788,19 @@ def _m0022_neighbor_tables(cursor: sqlite3.Cursor) -> None:
)
def _m0023_observed_paths_zero_hop_signal(cursor: sqlite3.Cursor) -> None:
"""SNR/RSSI on observed_paths for direct (zero-hop) advert rows.
Empty-path adverts are a confirmed last-hop-is-originator measurement, so
the figure belongs on the path row rather than on complete_contact_tracking
(whose hop_count over-claims zero-hop). Multi-hop rows leave these NULL.
"""
if not _table_exists(cursor, "observed_paths"):
return
_add_column(cursor, "observed_paths", "snr", "REAL")
_add_column(cursor, "observed_paths", "rssi", "REAL")
# ---------------------------------------------------------------------------
# Migration registry — append new entries here, never remove or reorder.
# ---------------------------------------------------------------------------
@@ -817,6 +830,7 @@ MIGRATIONS: list[MigrationEntry] = [
(20, "mesh_connections: table-specific last_seen index", _m0020_mesh_connections_last_seen_index),
(21, "daily_rollup: per-payload-type multibyte split", _m0021_daily_rollup_packet_type_encoding),
(22, "neighbor discovery tables", _m0022_neighbor_tables),
(23, "observed_paths: snr/rssi for zero-hop adverts", _m0023_observed_paths_zero_hop_signal),
]
+22 -2
View File
@@ -15,6 +15,7 @@ from typing import Any, TypedDict
from .enums import AdvertFlags, DeviceRole, PayloadType, PayloadVersion, RouteType
from .graph_trace_helper import update_mesh_graph_from_trace_data
from .models import MeshMessage
from .neighbors_discovery import upsert_zero_hop_observed_path_via_manager
from .security_utils import sanitize_input, sanitize_name
from .utils import (
calculate_packet_hash,
@@ -770,8 +771,24 @@ class MessageHandler:
):
self._update_mesh_graph_from_advert(advert_data, out_path, path_byte_length, packet_info)
# Store complete path in observed_paths table
if out_path and out_path_len > 0:
# Store complete path in observed_paths table. Empty-path (direct RF)
# adverts are stored too — those are true one-hop neighbours.
if out_path_len == 0 and advert_data.get("public_key"):
upsert_zero_hop_observed_path_via_manager(
getattr(self.bot, "db_manager", None),
advert_data["public_key"],
self.logger,
snr=signal_info.get("snr") if signal_info else None,
rssi=(
signal_info.get("rssi", signal_info.get("signal_strength"))
if signal_info
else None
),
bytes_per_hop=packet_info.get("bytes_per_hop", 1) or 1,
packet_hash=packet_hash,
update_rssi=True,
)
elif out_path and out_path_len > 0:
self._store_observed_path(
advert_data,
out_path,
@@ -1137,6 +1154,9 @@ class MessageHandler:
# (header + path_len + path + payload, without RF wrapper)
decoded_packet["raw_packet_hex"] = extracted_payload if extracted_payload else raw_hex
decoded_packet["packet_hash"] = packet_hash
decoded_packet["snr"] = snr_value
if "rssi" in payload:
decoded_packet["rssi"] = payload.get("rssi")
self.bot.web_viewer_integration.bot_integration.capture_full_packet_data(decoded_packet)
# Process ADVERT packets for contact tracking (regardless of path length)
+129
View File
@@ -92,6 +92,11 @@ LIBRARY_MSG_SENT_TIMEOUT = 15.0
# before pinning a contact to zero-hop and restores afterwards.
CONTACT_NO_PATH = -1
# Empty-path advert rows in observed_paths: both endpoints are the originator.
# 3-byte prefixes match the mesh graph's neighbor-evidence width.
ZERO_HOP_PATH_HEX = ""
ZERO_HOP_PREFIX_HEX_CHARS = 6
def clamp_interval_hours(hours: int) -> int:
"""Clamp to the firmware's 12-336h band, falling back to the 24h default."""
@@ -573,6 +578,122 @@ async def fetch_self_scopes(meshcore: Any, cfg: NeighborsConfig,
return str((getattr(result, "payload", None) or {}).get("scope_name", "") or "").strip()
def _observed_paths_has_signal_columns(cursor: Any) -> bool:
cols = {row[1] for row in cursor.execute("PRAGMA table_info(observed_paths)")}
return "snr" in cols and "rssi" in cols
def upsert_zero_hop_observed_path(
cursor: Any,
public_key: str,
*,
snr: Optional[float] = None,
rssi: Optional[float] = None,
bytes_per_hop: int = 1,
packet_hash: Optional[str] = None,
last_seen: Optional[str] = None,
update_rssi: bool = True,
) -> None:
"""Insert or refresh a direct-RF (empty path) advert row in observed_paths.
Discover responses carry SNR only, so ``update_rssi=False`` leaves a
previously stored RSSI from a zero-path advert in place. A later reception
with no measurement must not NULL out a stored figure either: COALESCE
keeps the existing column when the new value is None.
"""
key = (public_key or "").strip().lower()
if len(key) < 2:
return
tables = {
row[0]
for row in cursor.execute(
"SELECT name FROM sqlite_master WHERE type='table' AND name='observed_paths'"
)
}
if "observed_paths" not in tables:
return
prefix = key[:ZERO_HOP_PREFIX_HEX_CHARS]
stamp = last_seen or datetime.now().isoformat()
stored_hash = packet_hash if (packet_hash and packet_hash != "0000000000000000") else None
has_signal = _observed_paths_has_signal_columns(cursor)
existing = cursor.execute(
"""
SELECT id, observation_count FROM observed_paths
WHERE public_key = ? AND path_hex = ? AND packet_type = 'advert'
""",
(key, ZERO_HOP_PATH_HEX),
).fetchone()
if existing:
path_id = existing["id"] if not isinstance(existing, tuple) else existing[0]
count = (existing["observation_count"] if not isinstance(existing, tuple) else existing[1]) or 1
if has_signal:
cursor.execute(
"""
UPDATE observed_paths
SET observation_count = ?,
last_seen = ?,
snr = COALESCE(?, snr),
rssi = CASE WHEN ? THEN COALESCE(?, rssi) ELSE rssi END
WHERE id = ?
""",
(count + 1, stamp, snr, 1 if update_rssi else 0, rssi, path_id),
)
else:
cursor.execute(
"""
UPDATE observed_paths
SET observation_count = ?, last_seen = ?
WHERE id = ?
""",
(count + 1, stamp, path_id),
)
return
if has_signal:
cursor.execute(
"""
INSERT INTO observed_paths
(public_key, packet_hash, from_prefix, to_prefix, path_hex, path_length,
bytes_per_hop, packet_type, first_seen, last_seen, observation_count,
snr, rssi)
VALUES (?, ?, ?, ?, ?, 0, ?, 'advert', ?, ?, 1, ?, ?)
""",
(key, stored_hash, prefix, prefix, ZERO_HOP_PATH_HEX, bytes_per_hop,
stamp, stamp, snr, rssi if update_rssi else None),
)
else:
cursor.execute(
"""
INSERT INTO observed_paths
(public_key, packet_hash, from_prefix, to_prefix, path_hex, path_length,
bytes_per_hop, packet_type, first_seen, last_seen, observation_count)
VALUES (?, ?, ?, ?, ?, 0, ?, 'advert', ?, ?, 1)
""",
(key, stored_hash, prefix, prefix, ZERO_HOP_PATH_HEX, bytes_per_hop,
stamp, stamp),
)
def upsert_zero_hop_observed_path_via_manager(
db_manager: Any,
public_key: str,
logger: logging.Logger,
**kwargs: Any,
) -> None:
"""Open a connection, upsert one zero-hop row, and commit."""
if db_manager is None or not hasattr(db_manager, "connection"):
return
try:
with db_manager.connection() as conn:
upsert_zero_hop_observed_path(conn.cursor(), public_key, **kwargs)
conn.commit()
except Exception as exc:
logger.debug(f"Could not store zero-hop observed path: {exc}")
def record_neighbors(
db_manager: Any,
self_pubkey: str,
@@ -638,6 +759,14 @@ def record_neighbors(
(self_key, entry.pubkey.lower(), stamp, stamp,
entry.snr, entry.snr, entry.snr, entry.status, entry.scopes or ""),
)
# Discover is a confirmed direct RF reception: keep the dashboard
# neighbour list in sync. SNR only — leave RSSI to zero-path adverts.
upsert_zero_hop_observed_path(
cursor,
entry.pubkey,
snr=entry.snr,
update_rssi=False,
)
written += 1
conn.commit()
except Exception as exc:
+310 -55
View File
@@ -38,12 +38,13 @@ import socket
import sqlite3
import time
from contextlib import suppress
from datetime import datetime, timedelta
from datetime import datetime, timedelta, timezone
from typing import Any, Callable
SCHEMA_REV = 1
LEASE_KEY = "dashboard.snapshot_lease"
ZERO_HOP_BACKFILL_KEY = "dashboard.zero_hop_path_backfill"
# Source-presence bitmask stored on each daily_rollup row. A missing table is
# genuinely possible: message_stats/command_stats/path_stats are created by
@@ -179,11 +180,10 @@ TOP_KINDS = ("users", "commands", "channels", "paths", "repeaters", "neighbors")
# link last exercised a month ago says nothing about whether it works today.
NEIGHBOR_WINDOWS = ("24h", "7d")
# A path is one hop when its byte length equals the per-hop encoding width.
# path_length is measured in BYTES, and with 2- or 3-byte hop encoding a 3-hop
# path is 6 or 9 bytes long — reading the raw value as a hop count inflates
# every multibyte path by 2-3x.
ONE_HOP_PATH = "op.bytes_per_hop > 0 AND op.path_length = op.bytes_per_hop"
# Direct RF neighbour: MeshCore hop count 0 (empty path). path_length is a
# BYTE count, so a 1-hop relayed path is path_length = bytes_per_hop — that is
# not a neighbour. Empty-path rows are the ones this radio heard itself.
ZERO_HOP_PATH = "op.path_length = 0"
# Roles the firmware reports as an unmapped enum ordinal. They are real
# contacts, so they belong in the mix — just not as sixteen singleton slices.
@@ -213,6 +213,30 @@ def _table_exists(conn: sqlite3.Connection, table: str) -> bool:
return row is not None
def _column_exists(conn: sqlite3.Connection, table: str, column: str) -> bool:
if not table.isidentifier():
return False
return any(row[1] == column for row in conn.execute(f'PRAGMA table_info("{table}")'))
def _parse_aware_utc(value: Any) -> datetime | None:
"""Parse neighbor_links.last_seen (UTC ISO, with or without offset)."""
if not value:
return None
try:
parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
except (TypeError, ValueError):
return None
if parsed.tzinfo is None:
return parsed.replace(tzinfo=timezone.utc)
return parsed.astimezone(timezone.utc)
def _neighbor_window_cutoff_utc(window: str) -> datetime:
days = 1 if window == "24h" else 7
return datetime.now(timezone.utc) - timedelta(days=days)
def _top_n_with_other(entries: list[list[Any]], limit: int = MIX_ROWS) -> list[list[Any]]:
"""Keep the largest *limit* categories and roll the tail into "Other".
@@ -301,6 +325,7 @@ class DashboardStatsService:
# retention: recomputing a day whose raw rows were just pruned would
# overwrite a real value with a zero.
self.trailing_days = 3
self._zero_hop_backfilled = False
# -- source availability -------------------------------------------------
@@ -370,6 +395,150 @@ class DashboardStatsService:
self.logger.debug(f"packet_stream dimension backfill unavailable: {exc}")
return 0
@staticmethod
def _advert_identity_from_packet_json(
data: dict[str, Any],
) -> tuple[str | None, float | None, float | None]:
"""Recover originator key and RF signal from a packet_stream JSON blob."""
pk = data.get("advert_public_key")
if not (isinstance(pk, str) and len(pk) >= 64):
hex_str = str(data.get("payload_hex") or "").replace("0x", "").replace(" ", "")
pk = hex_str[:64] if len(hex_str) >= 64 else None
if isinstance(pk, str):
pk = pk.lower()
if len(pk) != 64 or any(char not in "0123456789abcdef" for char in pk):
pk = None
else:
pk = None
def _num(value: Any) -> float | None:
if value is None or value == "" or value == "Unknown":
return None
try:
return float(value)
except (TypeError, ValueError):
return None
return pk, _num(data.get("snr", data.get("SNR"))), _num(data.get("rssi", data.get("RSSI")))
def backfill_zero_hop_from_packet_stream(self, conn: sqlite3.Connection) -> int:
"""One-time copy of zero-hop ADVERT packets into observed_paths.
Empty-path adverts were never stored; packet_stream still has ~3 days of
them with hop count and payload_hex. Idempotent on the advert unique
index. SNR is taken from the JSON when present, else neighbor_links.
"""
if self._zero_hop_backfilled:
return 0
if _table_exists(conn, "bot_metadata"):
row = conn.execute(
"SELECT value FROM bot_metadata WHERE key = ?",
(ZERO_HOP_BACKFILL_KEY,),
).fetchone()
if row:
self._zero_hop_backfilled = True
return 0
if not _table_exists(conn, "packet_stream") or not _table_exists(conn, "observed_paths"):
self._zero_hop_backfilled = True
return 0
from modules.neighbors_discovery import upsert_zero_hop_observed_path
try:
rows = conn.execute(
"""
SELECT timestamp, data, path_len, bytes_per_hop, payload_type_name
FROM packet_stream
WHERE type = 'packet'
AND (
payload_type_name = 'ADVERT'
OR json_extract(data, '$.payload_type_name') = 'ADVERT'
)
AND COALESCE(
path_len,
CAST(json_extract(data, '$.path_len') AS INTEGER)
) = 0
"""
).fetchall()
except sqlite3.OperationalError as exc:
self.logger.debug(f"zero-hop packet_stream backfill unavailable: {exc}")
return 0
discover_snr: dict[str, float] = {}
if _table_exists(conn, "neighbor_links"):
for row in conn.execute(
"SELECT neighbor_public_key, last_snr FROM neighbor_links WHERE last_snr IS NOT NULL"
):
key = (row["neighbor_public_key"] or "").lower()
if key:
discover_snr[key] = float(row["last_snr"])
latest: dict[str, dict[str, Any]] = {}
existing_keys = {
(row[0] or "").lower()
for row in conn.execute(
"""
SELECT public_key FROM observed_paths
WHERE packet_type = 'advert' AND path_length = 0 AND public_key IS NOT NULL
"""
)
}
for row in rows:
try:
data = json.loads(row["data"] or "{}")
except (TypeError, ValueError):
continue
if not isinstance(data, dict):
continue
pk, snr, rssi = self._advert_identity_from_packet_json(data)
if not pk or pk in existing_keys:
continue
ts = row["timestamp"]
try:
ts_f = float(ts)
except (TypeError, ValueError):
continue
prev = latest.get(pk)
if prev is not None and ts_f <= prev["ts"]:
continue
if snr is None:
snr = discover_snr.get(pk)
bph = row["bytes_per_hop"] or data.get("bytes_per_hop") or 1
try:
bph = int(bph)
except (TypeError, ValueError):
bph = 1
latest[pk] = {"ts": ts_f, "snr": snr, "rssi": rssi, "bytes_per_hop": bph}
cursor = conn.cursor()
for pk, info in latest.items():
upsert_zero_hop_observed_path(
cursor,
pk,
snr=info["snr"],
rssi=info["rssi"],
bytes_per_hop=info["bytes_per_hop"] or 1,
last_seen=datetime.fromtimestamp(info["ts"]).isoformat(),
update_rssi=info["rssi"] is not None,
)
if _table_exists(conn, "bot_metadata"):
conn.execute(
"""
INSERT INTO bot_metadata (key, value, updated_at)
VALUES (?, '1', CURRENT_TIMESTAMP)
ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = CURRENT_TIMESTAMP
""",
(ZERO_HOP_BACKFILL_KEY,),
)
self._zero_hop_backfilled = True
if latest:
self.logger.info(
"Backfilled %d zero-hop neighbour path(s) from packet_stream",
len(latest),
)
return len(latest)
def packet_coverage(self, conn: sqlite3.Connection) -> dict[str, Any]:
"""Actual time span covered by packet rows carrying denormalized dims.
@@ -862,7 +1031,7 @@ class DashboardStatsService:
mesh["hops"] = self._hops_distribution(conn, sources)
if sources & SOURCE_OBSERVED_PATHS:
if sources & SOURCE_OBSERVED_PATHS or _table_exists(conn, "neighbor_links"):
mesh["neighbors"] = {
window: self._count_one_hop_nodes(conn, window)
for window in NEIGHBOR_WINDOWS
@@ -1030,69 +1199,154 @@ class DashboardStatsService:
counts[int(hops)] = counts.get(int(hops), 0) + (1 if per_row else (row["n"] or 0))
return [[hop, count] for hop, count in sorted(counts.items())]
def _count_one_hop_nodes(self, conn: sqlite3.Connection, window: str) -> int:
return conn.execute(
f"""
SELECT COUNT(DISTINCT op.public_key) FROM observed_paths op
WHERE op.packet_type = 'advert' AND {ONE_HOP_PATH}
AND op.last_seen > datetime('now','localtime', ?)
""", # noqa: S608 - ONE_HOP_PATH is a module constant, not input
(self._window_offset(window),),
).fetchone()[0] or 0
@staticmethod
def _window_offset(window: str) -> str:
return {"24h": "-1 days", "7d": "-7 days", "30d": "-30 days"}.get(window, "-7 days")
def _one_hop_rows(self, conn: sqlite3.Connection, window: str, limit: int) -> list[dict]:
"""Nodes whose advert reached this radio in a single hop, weakest first.
def _count_one_hop_nodes(self, conn: sqlite3.Connection, window: str) -> int:
return len(self._direct_neighbor_keys(conn, window))
Membership comes from path evidence, not from
``complete_contact_tracking.hop_count``. That column claims 800 zero-hop
contacts, but only 68 of them have any one-hop path to corroborate it,
their stored SNR piles up in a 1.5 dB band (655 of 800 between 11.25 and
12.75), and their RSSI clusters around -45 dBm the signature of one
strong local link being recorded against every node whose traffic came
through it, not of hundreds of separate radios.
def _direct_neighbor_keys(self, conn: sqlite3.Connection, window: str) -> set[str]:
"""Public keys heard directly in *window*: empty-path adverts plus discover."""
keys: set[str] = set()
if _table_exists(conn, "observed_paths"):
for row in conn.execute(
f"""
SELECT DISTINCT op.public_key FROM observed_paths op
WHERE op.packet_type = 'advert' AND {ZERO_HOP_PATH}
AND op.last_seen > datetime('now','localtime', ?)
""", # noqa: S608 - ZERO_HOP_PATH is a module constant, not input
(self._window_offset(window),),
):
if row[0]:
keys.add(str(row[0]).lower())
keys.update(self._neighbor_link_keys_in_window(conn, window))
return keys
Signal is therefore reported only where the path evidence and the stored
hop_count agree; everything else lists as unknown rather than being given
a number that belongs to somebody else's link.
def _neighbor_link_keys_in_window(
self, conn: sqlite3.Connection, window: str
) -> dict[str, dict[str, Any]]:
"""Discover-confirmed neighbours whose last_seen falls in *window*.
neighbor_links.last_seen is UTC ISO; do not compare it with
datetime('now','localtime', ).
"""
rows = conn.execute(
f"""
SELECT op.public_key,
MAX(op.last_seen) AS last_seen,
c.name, c.role, c.snr, c.signal_strength, c.hop_count
FROM observed_paths op
LEFT JOIN complete_contact_tracking c ON c.public_key = op.public_key
WHERE op.packet_type = 'advert' AND {ONE_HOP_PATH}
AND op.last_seen > datetime('now','localtime', ?)
GROUP BY op.public_key
""", # noqa: S608 - ONE_HOP_PATH is a module constant, not input
(self._window_offset(window),),
).fetchall()
found: dict[str, dict[str, Any]] = {}
if not _table_exists(conn, "neighbor_links"):
return found
cutoff = _neighbor_window_cutoff_utc(window)
for row in conn.execute(
"""
SELECT neighbor_public_key, last_snr, last_seen
FROM neighbor_links
"""
):
key = (row["neighbor_public_key"] or "").lower()
if not key:
continue
seen = _parse_aware_utc(row["last_seen"])
if seen is None or seen < cutoff:
continue
found[key] = {
"snr": row["last_snr"],
"last_seen": row["last_seen"],
}
return found
measured: list[dict[str, Any]] = []
unmeasured: list[dict[str, Any]] = []
for row in rows:
corroborated = row["hop_count"] == 0 and row["snr"] is not None
item = {
"name": row["name"] or (row["public_key"] or "")[:12],
def _one_hop_rows(self, conn: sqlite3.Connection, window: str, limit: int) -> list[dict]:
"""Nodes this radio heard directly, weakest measured link first.
Membership is empty-path advert evidence (MeshCore hop count 0) union
in-window ``neighbor_links``. ``complete_contact_tracking.hop_count``
is not used for membership it over-claims zero-hop.
Signal prefers the path-row measurement, then discover SNR, then the
contact row only when hop_count is 0.
"""
has_path_snr = _table_exists(conn, "observed_paths") and _column_exists(
conn, "observed_paths", "snr"
)
snr_select = "op.snr AS path_snr, op.rssi AS path_rssi," if has_path_snr else (
"NULL AS path_snr, NULL AS path_rssi,"
)
path_rows = []
if _table_exists(conn, "observed_paths"):
path_rows = conn.execute(
f"""
SELECT op.public_key,
MAX(op.last_seen) AS last_seen,
{snr_select}
c.name, c.role, c.snr, c.signal_strength, c.hop_count
FROM observed_paths op
LEFT JOIN complete_contact_tracking c ON c.public_key = op.public_key
WHERE op.packet_type = 'advert' AND {ZERO_HOP_PATH}
AND op.last_seen > datetime('now','localtime', ?)
GROUP BY op.public_key
""", # noqa: S608 - ZERO_HOP_PATH / snr_select are module-controlled
(self._window_offset(window),),
).fetchall()
discover = self._neighbor_link_keys_in_window(conn, window)
by_key: dict[str, dict[str, Any]] = {}
def _contact_name(row, key: str) -> str:
return row["name"] or key[:12]
for row in path_rows:
key = (row["public_key"] or "").lower()
if not key:
continue
snr = row["path_snr"]
rssi = row["path_rssi"]
link = discover.pop(key, None)
if snr is None and link is not None:
snr = link.get("snr")
if snr is None and row["hop_count"] == 0:
snr = row["snr"]
rssi = row["signal_strength"] if rssi is None else rssi
corroborated = snr is not None
by_key[key] = {
"name": _contact_name(row, key),
"public_key": row["public_key"],
"role": normalize_role(row["role"]),
"snr": round(float(row["snr"]), 1) if corroborated else None,
"snr": round(float(snr), 1) if corroborated else None,
"rssi": (
round(float(row["signal_strength"]))
if corroborated and row["signal_strength"] is not None
else None
round(float(rssi)) if corroborated and rssi is not None else None
),
"signal_corroborated": corroborated,
"last_seen": row["last_seen"],
}
(measured if corroborated else unmeasured).append(item)
# Weakest measured links first — those are the ones worth acting on.
# Discover-only neighbours (answered node-discover, no stored empty-path advert).
if discover:
contacts = {}
if _table_exists(conn, "complete_contact_tracking"):
placeholders = ",".join("?" * len(discover))
for row in conn.execute(
f"""
SELECT public_key, name, role FROM complete_contact_tracking
WHERE lower(public_key) IN ({placeholders})
""", # noqa: S608 - placeholders match bound keys
tuple(discover),
):
contacts[(row["public_key"] or "").lower()] = row
for key, link in discover.items():
contact = contacts.get(key)
snr = link.get("snr")
corroborated = snr is not None
by_key[key] = {
"name": (contact["name"] if contact else None) or key[:12],
"public_key": (contact["public_key"] if contact else key),
"role": normalize_role(contact["role"] if contact else None),
"snr": round(float(snr), 1) if corroborated else None,
"rssi": None,
"signal_corroborated": corroborated,
"last_seen": link.get("last_seen"),
}
items = list(by_key.values())
measured = [item for item in items if item["signal_corroborated"]]
unmeasured = [item for item in items if not item["signal_corroborated"]]
measured.sort(key=lambda item: item["snr"])
unmeasured.sort(key=lambda item: item["last_seen"] or "", reverse=True)
return (measured + unmeasured)[:limit]
@@ -1251,6 +1505,7 @@ class DashboardStatsService:
conn.execute("BEGIN IMMEDIATE")
try:
backfilled_packets = self.backfill_packet_dims(conn)
self.backfill_zero_hop_from_packet_stream(conn)
conn.commit()
except Exception:
conn.rollback()
@@ -1481,7 +1736,7 @@ class DashboardStatsService:
if window not in NEIGHBOR_WINDOWS:
window = NEIGHBOR_WINDOWS[0]
retention = self.adverts_retention_days
if _table_exists(conn, "observed_paths"):
if _table_exists(conn, "observed_paths") or _table_exists(conn, "neighbor_links"):
items = self._one_hop_rows(conn, window, limit)
total = self._count_one_hop_nodes(conn, window)
elif kind == "repeaters":
+13 -14
View File
@@ -1046,11 +1046,10 @@
}
/**
* One one-hop neighbour: name, SNR bar on a fixed scale, values.
* One direct neighbour: name, SNR bar on a fixed scale, values.
*
* Signal is shown only where the stored hop count corroborates the path
* evidence. For the rest the row says "no signal reading" rather than
* borrowing a number measured on somebody else's link.
* Signal is shown when a zero-hop advert or neighbor-discover measurement
* is stored for this node. Otherwise the row says "no signal reading".
*/
function neighborRow(item) {
const row = document.createElement('div');
@@ -1084,8 +1083,7 @@
} else {
track.classList.add('neighbor-row__track--unknown');
const unknown = makeTextElement('span', 'no signal reading', 'neighbor-row__rssi');
unknown.title = 'Heard one hop away, but no corroborated direct-reception '
+ 'measurement is stored for this node.';
unknown.title = 'Heard directly, but no SNR/RSSI measurement is stored for this node.';
values.appendChild(unknown);
}
@@ -1167,7 +1165,7 @@
select: 'neighbors-window',
container: 'neighbors-list',
limit: 10,
empty: 'No one-hop adverts heard in this window.',
empty: 'No direct neighbours heard in this window.',
render: (item) => neighborRow(item),
onLoad: (data) => {
setText('neighbors-count', formatNumber(data.total));
@@ -1241,13 +1239,14 @@
'Change compares the last complete calendar day against the day before it — ' +
'not a rolling 24 hours. The headline number above it is a rolling 24-hour count.',
'neighbors-info':
'Nodes whose advert reached this radio in a single hop, newest evidence first, ' +
'with the weakest measured links promoted to the top. Membership comes from the ' +
'observed path length divided by its bytes-per-hop encoding, not from the stored ' +
'hop count — that field claims far more direct neighbours than the path evidence ' +
'supports. SNR is shown only where both agree; a relayed packet\'s SNR measures ' +
'the last hop into this radio rather than the link to whoever sent it, so the ' +
'rest are left blank instead of borrowing another link\'s number.',
'Radios this node heard directly (MeshCore hop count 0: an empty RF path). ' +
'A path that already contains one hop hash is a relayed packet, not a neighbour. ' +
'Membership comes from those empty-path adverts plus confirmed zero-hop ' +
'neighbor-discover responses, not from the stored hop count — that field claims ' +
'far more direct neighbours than the path evidence supports. SNR/RSSI are the ' +
'measurement from that direct reception (or from discover, which reports SNR ' +
'only). A relayed packet\'s SNR measures the last hop into this radio rather ' +
'than the link to whoever sent it.',
'hops-info':
'Two distributions on one axis. Nodes: the fewest hops any of a node\'s adverts ' +
'took to reach this radio, over 7 days. Flood packets: how far each flood packet ' +
+1 -1
View File
@@ -257,7 +257,7 @@
<div class="card-body">
<div class="d-flex align-items-baseline gap-2 flex-wrap mb-1">
<span class="stat-tile__value" id="neighbors-count"></span>
<span class="dashboard-note">nodes reached this radio in a single hop</span>
<span class="dashboard-note">heard directly by this radio</span>
</div>
<p class="dashboard-note mb-3" id="neighbors-shown"></p>
<div id="neighbors-list"><div class="dashboard-empty">Loading…</div></div>
+99 -36
View File
@@ -784,12 +784,9 @@ class TestDerivedWindows:
class TestOneHopNeighbours:
"""Neighbour membership comes from path evidence, not the stored hop_count.
"""Neighbour membership is MeshCore hop count 0 (empty path), not 1-hop relays.
On the live database hop_count claims 800 zero-hop contacts while only 68
have any one-hop path to corroborate it, and their stored SNR piles up in a
1.5 dB band one strong local link recorded against every node whose
traffic came through it.
hop_count on contacts over-claims zero-hop and is not used for membership.
"""
def _seed_contact(self, conn, pk, name, role, hop_count, snr, rssi):
@@ -802,52 +799,71 @@ class TestOneHopNeighbours:
(pk, name, role, hop_count, snr, rssi),
)
def _seed_path(self, conn, pk, path_length, bytes_per_hop, age="-1 hours"):
def _seed_path(self, conn, pk, path_length, bytes_per_hop, age="-1 hours",
snr=None, rssi=None):
conn.execute(
"""
INSERT INTO observed_paths
(public_key, from_prefix, to_prefix, path_hex, path_length,
bytes_per_hop, packet_type, last_seen)
VALUES (?, 'aa', 'bb', ?, ?, ?, 'advert', datetime('now','localtime', ?))
bytes_per_hop, packet_type, last_seen, snr, rssi)
VALUES (?, 'aa', 'bb', ?, ?, ?, 'advert', datetime('now','localtime', ?), ?, ?)
""",
(pk, "ab" * path_length, path_length, bytes_per_hop, age),
(pk, "ab" * path_length, path_length, bytes_per_hop, age, snr, rssi),
)
def _seed_neighbor_link(self, conn, pk, snr, age_hours=1):
seen = datetime.now(timezone.utc) - timedelta(hours=age_hours)
conn.execute(
"""
INSERT INTO neighbor_links
(self_public_key, neighbor_public_key, last_seen, last_snr, snr_sum, snr_count)
VALUES (?, ?, ?, ?, ?, 1)
""",
("ff" * 32, pk, seen.isoformat(), snr, snr),
)
def _top(self, viewer, window="24h", limit=10):
with closing(viewer._dashboard_connection()) as conn:
return viewer.dashboard_stats.read_top(conn, "neighbors", window, limit)
def test_multibyte_paths_are_not_read_as_extra_hops(self, viewer):
"""path_length is bytes: 3 bytes at 3 bytes/hop is ONE hop, not three."""
def test_empty_path_is_a_direct_neighbour(self, viewer):
"""path_length = 0 is MeshCore hop count 0 — heard directly."""
with sqlite3.connect(viewer.db_path) as conn:
self._seed_contact(conn, _pk(1), "direct", "repeater", 4, 12.0, -45.0)
self._seed_path(conn, _pk(1), path_length=0, bytes_per_hop=2, snr=5.0, rssi=-80.0)
self._seed_contact(conn, _pk(2), "one-hop-relay", "repeater", 0, 6.0, -60.0)
self._seed_path(conn, _pk(2), path_length=1, bytes_per_hop=1)
self._seed_contact(conn, _pk(3), "multibyte-1hop", "repeater", 0, 7.0, -50.0)
self._seed_path(conn, _pk(3), path_length=3, bytes_per_hop=3)
items = self._top(viewer)["items"]
assert [item["name"] for item in items] == ["direct"]
assert items[0]["signal_corroborated"] is True
assert items[0]["snr"] == 5.0
assert items[0]["rssi"] == -80
def test_relayed_one_hop_paths_are_not_neighbours(self, viewer):
"""path_length == bytes_per_hop is one encoded hop, not a direct neighbour."""
with sqlite3.connect(viewer.db_path) as conn:
self._seed_contact(conn, _pk(1), "onebyte-1hop", "repeater", 0, 5.0, -70.0)
self._seed_path(conn, _pk(1), path_length=1, bytes_per_hop=1)
self._seed_contact(conn, _pk(2), "multibyte-1hop", "repeater", 0, 6.0, -60.0)
self._seed_path(conn, _pk(2), path_length=3, bytes_per_hop=3)
self._seed_contact(conn, _pk(3), "multibyte-2hop", "repeater", 0, 7.0, -50.0)
self._seed_path(conn, _pk(3), path_length=6, bytes_per_hop=3)
assert self._top(viewer)["items"] == []
assert self._top(viewer)["total"] == 0
names = {item["name"] for item in self._top(viewer)["items"]}
assert names == {"onebyte-1hop", "multibyte-1hop"}
def test_uncorroborated_signal_is_withheld(self, viewer):
"""Path evidence without a matching hop_count gets no SNR figure."""
def test_path_snr_is_shown_without_hop_count_agreement(self, viewer):
"""The path-row measurement belongs to this link; hop_count is ignored."""
with sqlite3.connect(viewer.db_path) as conn:
self._seed_contact(conn, _pk(1), "agrees", "repeater", 0, 4.0, -80.0)
self._seed_path(conn, _pk(1), 1, 1)
# hop_count says 4 hops but a one-hop path exists: the stored signal
# belongs to some other link, so it must not be shown.
self._seed_contact(conn, _pk(2), "disagrees", "repeater", 4, 12.0, -45.0)
self._seed_path(conn, _pk(2), 1, 1)
self._seed_contact(conn, _pk(1), "direct", "repeater", 4, 12.0, -45.0)
self._seed_path(conn, _pk(1), 0, 1, snr=4.0, rssi=-80.0)
items = {item["name"]: item for item in self._top(viewer)["items"]}
assert items["agrees"]["signal_corroborated"] is True
assert items["agrees"]["snr"] == 4.0
assert items["disagrees"]["signal_corroborated"] is False
assert items["disagrees"]["snr"] is None
assert items["disagrees"]["rssi"] is None
item = self._top(viewer)["items"][0]
assert item["signal_corroborated"] is True
assert item["snr"] == 4.0
assert item["rssi"] == -80
def test_contacts_with_no_one_hop_path_are_excluded(self, viewer):
def test_contacts_with_no_zero_hop_path_are_excluded(self, viewer):
"""hop_count = 0 alone does not make something a neighbour."""
with sqlite3.connect(viewer.db_path) as conn:
self._seed_contact(conn, _pk(1), "claims-direct", "repeater", 0, 12.0, -45.0)
@@ -858,10 +874,10 @@ class TestOneHopNeighbours:
def test_weakest_measured_links_are_promoted(self, viewer):
with sqlite3.connect(viewer.db_path) as conn:
for i, snr in enumerate([9.0, -8.0, 2.0]):
self._seed_contact(conn, _pk(i), f"m{i}", "repeater", 0, snr, -70.0)
self._seed_path(conn, _pk(i), 1, 1)
self._seed_contact(conn, _pk(i), f"m{i}", "repeater", 0, None, None)
self._seed_path(conn, _pk(i), 0, 1, snr=snr, rssi=-70.0)
self._seed_contact(conn, _pk(9), "unmeasured", "repeater", 3, None, None)
self._seed_path(conn, _pk(9), 1, 1)
self._seed_path(conn, _pk(9), 0, 1)
items = self._top(viewer)["items"]
assert [i["name"] for i in items[:3]] == ["m1", "m2", "m0"]
@@ -870,13 +886,28 @@ class TestOneHopNeighbours:
def test_window_bounds_membership(self, viewer):
with sqlite3.connect(viewer.db_path) as conn:
self._seed_contact(conn, _pk(1), "today", "repeater", 0, 5.0, -70.0)
self._seed_path(conn, _pk(1), 1, 1, age="-2 hours")
self._seed_path(conn, _pk(1), 0, 1, age="-2 hours", snr=5.0)
self._seed_contact(conn, _pk(2), "last-week", "repeater", 0, 5.0, -70.0)
self._seed_path(conn, _pk(2), 1, 1, age="-4 days")
self._seed_path(conn, _pk(2), 0, 1, age="-4 days", snr=5.0)
assert {i["name"] for i in self._top(viewer, "24h")["items"]} == {"today"}
assert {i["name"] for i in self._top(viewer, "7d")["items"]} == {"today", "last-week"}
def test_discover_links_are_unioned(self, viewer):
"""A repeater that answered node-discover is a neighbour even without an advert path."""
with sqlite3.connect(viewer.db_path) as conn:
self._seed_contact(conn, _pk(1), "heard-advert", "repeater", 0, None, None)
self._seed_path(conn, _pk(1), 0, 1, snr=3.0, rssi=-90.0)
self._seed_contact(conn, _pk(2), "discover-only", "repeater", 1, None, None)
self._seed_neighbor_link(conn, _pk(2), snr=8.5, age_hours=2)
items = {item["name"]: item for item in self._top(viewer)["items"]}
assert set(items) == {"heard-advert", "discover-only"}
assert items["heard-advert"]["snr"] == 3.0
assert items["discover-only"]["snr"] == 8.5
assert items["discover-only"]["rssi"] is None
assert self._top(viewer)["total"] == 2
def test_windows_are_capped_below_retention(self, viewer):
"""observed_paths keeps 90 days; a month-old link says nothing about today."""
options = viewer.dashboard_stats.derive_windows()["windows"]["neighbors"]
@@ -903,6 +934,38 @@ class TestOneHopNeighbours:
assert payload["items"] == []
assert payload["total"] == 0
def test_packet_stream_backfill_inserts_zero_hop_adverts(self, viewer):
pk = _pk(7)
payload_hex = pk + "00" * 70
packet = {
"payload_type_name": "ADVERT",
"path_len": 0,
"bytes_per_hop": 2,
"payload_hex": payload_hex,
"snr": 6.5,
"rssi": -72,
}
with sqlite3.connect(viewer.db_path) as conn:
conn.execute(
"""
INSERT INTO packet_stream
(timestamp, data, type, route_type_name, payload_type_name, path_len, bytes_per_hop)
VALUES (?, ?, 'packet', 'FLOOD', 'ADVERT', 0, 2)
""",
(time.time() - 3600, json.dumps(packet)),
)
self._seed_contact(conn, pk, "from-log", "repeater", 2, None, None)
_refresh(viewer)
items = {item["name"]: item for item in self._top(viewer)["items"]}
assert "from-log" in items
assert items["from-log"]["snr"] == 6.5
assert items["from-log"]["rssi"] == -72
# Second refresh must not duplicate or clear the marker.
_refresh(viewer)
assert self._top(viewer)["total"] == 1
class TestHopConventions:
"""The two path tables measure paths in different units. Pin both.
+16
View File
@@ -468,3 +468,19 @@ class TestNeighborTables:
runner.run()
assert "neighbor_links" in DBManager.ALLOWED_TABLES
assert "neighbor_observations" in DBManager.ALLOWED_TABLES
class TestObservedPathsZeroHopSignal:
def test_snr_rssi_columns_added(self, runner, conn):
runner.run()
cursor = conn.cursor()
assert _column_exists(cursor, "observed_paths", "snr") is True
assert _column_exists(cursor, "observed_paths", "rssi") is True
def test_migration_is_idempotent(self, conn, logger):
MigrationRunner(conn, logger).run()
MigrationRunner(conn, logger).run()
applied = conn.execute(
"SELECT COUNT(*) FROM schema_version WHERE version = 23"
).fetchone()[0]
assert applied == 1
+40
View File
@@ -2109,3 +2109,43 @@ class TestHandleNewContactAutoManage:
assert any("New repeater discovered" in msg for msg in log_messages)
mesh.commands.add_contact.assert_not_called()
rm.add_companion_from_contact_data.assert_not_called()
class TestZeroHopObservedPathWriter:
def test_store_observed_path_skips_empty_and_one_byte_paths(self, handler, bot):
bot.db_manager = Mock()
handler._store_observed_path({"public_key": "aa" * 32}, "", 0, "advert")
handler._store_observed_path({"public_key": "aa" * 32}, "ab", 1, "advert")
bot.db_manager.execute_query.assert_not_called()
bot.db_manager.execute_update.assert_not_called()
async def test_zero_hop_advert_upserts_direct_neighbour_row(self, handler, bot):
bot.db_manager = Mock()
bot.repeater_manager = Mock()
bot.repeater_manager.track_contact_advertisement = AsyncMock(
return_value=Mock(ok=True)
)
pk = "ab" * 32
with patch(
"modules.message_handler.upsert_zero_hop_observed_path_via_manager"
) as upsert:
await handler._process_advertisement_packet(
{
"payload_type_name": "ADVERT",
"sender_id": pk,
"bytes_per_hop": 2,
"path_byte_length": 0,
"routing_info": {
"path_hex": "",
"path_length": 0,
"packet_hash": "1111111111111111",
},
},
{"snr": 5.5, "rssi": -77},
)
upsert.assert_called_once()
_args, kwargs = upsert.call_args
assert _args[1] == pk
assert kwargs["snr"] == 5.5
assert kwargs["rssi"] == -77
assert kwargs["update_rssi"] is True
+54
View File
@@ -809,3 +809,57 @@ def test_record_neighbors_reports_zero_on_database_error(tmp_path):
[nb.NeighborEntry(pubkey=KEY_A, snr=1.0, heard_at=0.0)], LOGGER,
)
assert written == 0
def _zero_hop_rows(db):
with db.connection() as conn:
return [dict(r) for r in conn.execute(
"SELECT * FROM observed_paths WHERE path_length = 0"
)]
def test_record_neighbors_upserts_zero_hop_observed_path(db):
entries = [nb.NeighborEntry(pubkey=KEY_A, snr=7.5, heard_at=0.0,
status=nb.STATUS_RESPONDED)]
assert nb.record_neighbors(db, SELF_KEY, entries, LOGGER) == 1
rows = _zero_hop_rows(db)
assert len(rows) == 1
assert rows[0]["public_key"] == KEY_A
assert rows[0]["path_hex"] == ""
assert rows[0]["path_length"] == 0
assert rows[0]["snr"] == 7.5
assert rows[0]["rssi"] is None
def test_upsert_zero_hop_discover_does_not_clear_rssi(db):
with db.connection() as conn:
nb.upsert_zero_hop_observed_path(
conn.cursor(), KEY_A, snr=4.0, rssi=-80.0, update_rssi=True
)
conn.commit()
nb.upsert_zero_hop_observed_path(
conn.cursor(), KEY_A, snr=9.0, update_rssi=False
)
conn.commit()
rows = _zero_hop_rows(db)
assert len(rows) == 1
assert rows[0]["snr"] == 9.0
assert rows[0]["rssi"] == -80.0
assert rows[0]["observation_count"] == 2
def test_upsert_zero_hop_preserves_signal_when_refresh_has_none(db):
"""A later empty-path advert without RF figures must not NULL the row."""
with db.connection() as conn:
nb.upsert_zero_hop_observed_path(
conn.cursor(), KEY_A, snr=4.0, rssi=-80.0, update_rssi=True
)
conn.commit()
nb.upsert_zero_hop_observed_path(
conn.cursor(), KEY_A, snr=None, rssi=None, update_rssi=True
)
conn.commit()
rows = _zero_hop_rows(db)
assert rows[0]["snr"] == 4.0
assert rows[0]["rssi"] == -80.0
assert rows[0]["observation_count"] == 2