diff --git a/CHANGELOG.md b/CHANGELOG.md index 924db0f..78cfcdd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,10 +39,20 @@ semantic versioning. traffic is, next to how it is routed. Category lists roll their tail into "Other" rather than truncating, so the bars still sum to the total printed beside them. -- Hop-distance chart derived as `path_length / bytes_per_hop`. `path_length` is - a byte count, so with 2- or 3-byte hop encoding — about 95% of adverts on the - live mesh — reading it directly as a hop count overstates distance two- to - threefold. This replaces both the raw path-length chart and the chart built +- Hop-distance chart carrying two distributions: nodes by their closest advert + path, and arriving flood packets by how far they had already travelled. One + counts nodes and the other packets, so both are drawn as a share of their own + total with raw counts in the tooltip. On the live mesh nodes peak at 2-3 hops + and fall away quickly while flood traffic peaks at 5 with a much longer tail — + a nearby neighbourhood absorbing flood from well beyond it. + + Note that the two source tables measure paths in **different units**: + `observed_paths.path_length` is a byte count, so hops are + `path_length / bytes_per_hop` (a 3-hop multibyte path is 6 or 9), while + `packet_stream.path_len` is already a hop count with its byte length kept + separately as `path_byte_length`. Applying either rule to the other table + silently rescales an axis; both are pinned by tests. This replaces the + earlier raw path-length chart, which read bytes as hops, and the chart built on the untrustworthy stored hop count. - New `[Web_Viewer]` settings: `dashboard_snapshot_enabled`, `dashboard_snapshot_interval_seconds`, `dashboard_snapshot_history_days`, and diff --git a/docs/web-viewer.md b/docs/web-viewer.md index 37f1743..b1b13db 100644 --- a/docs/web-viewer.md +++ b/docs/web-viewer.md @@ -148,10 +148,21 @@ proxy_set_header X-Forwarded-Proto $scheme; Two measurement notes for the mesh charts: -**Hops are derived, not read.** `observed_paths.path_length` is a *byte* count, -and with 2- or 3-byte hop encoding a three-hop path is six or nine bytes long. -The dashboard divides by `bytes_per_hop`; charting the raw value would overstate -distance two- to threefold on a mesh that is ~95% multibyte. +**Hops are derived, and the two path tables disagree on units.** +`observed_paths.path_length` is a *byte* count, so hops are +`path_length / bytes_per_hop` — with 2- or 3-byte encoding a three-hop path is +six or nine bytes long, and charting the raw value would overstate distance two- +to threefold on a mesh that is ~95% multibyte. `packet_stream.path_len`, by +contrast, is *already a hop count*, with its byte length carried separately as +`path_byte_length`. Applying either table's rule to the other silently rescales +the axis, so both conventions are pinned by tests. + +The hops chart shows both distributions: nodes by their closest advert path (7 +days) and arriving flood packets by distance travelled (whatever +`packet_stream` retains, typically 3 days). One counts nodes and the other +packets, so each is drawn as a share of its own total. Flood packets carry no +sender identity, which is why they cannot be reduced to a shortest path per +node the way adverts can. **Neighbour signal is reported only where two sources agree.** `complete_contact_tracking.hop_count` is not a reliable direct-neighbour marker: diff --git a/modules/web_viewer/dashboard_stats.py b/modules/web_viewer/dashboard_stats.py index 3862316..2fdb88f 100644 --- a/modules/web_viewer/dashboard_stats.py +++ b/modules/web_viewer/dashboard_stats.py @@ -87,6 +87,9 @@ SUMMARY_SERIES_POINTS = 30 # Categories to show in a role/payload mix before the tail is rolled into "Other". MIX_ROWS = 8 +# Hop counts beyond this are corrupt path data rather than real distance. +MAX_PLOTTED_HOPS = 32 + # Metrics that are already a ratio: a period "total" has to be the mean of the # daily values, not their sum — adding percentages together means nothing. RATIO_METRICS = frozenset({"multibyte_share"}) @@ -733,8 +736,9 @@ class DashboardStatsService: mesh["role_mix"] = self._mix(conn, "role") + mesh["hops"] = self._hops_distribution(conn, sources) + if sources & SOURCE_OBSERVED_PATHS: - mesh["hops_histogram"] = self._hops_histogram(conn) mesh["neighbors"] = { window: self._count_one_hop_nodes(conn, window) for window in NEIGHBOR_WINDOWS @@ -795,33 +799,73 @@ class DashboardStatsService: buckets[key] = buckets.get(key, 0) + (count or 0) return _top_n_with_other([[name, count] for name, count in buckets.items()]) - def _hops_histogram(self, conn: sqlite3.Connection) -> list[list[int]]: - """Nodes by how many hops away their closest observed advert path was. + def _hops_distribution(self, conn: sqlite3.Connection, sources: int) -> dict[str, Any]: + """Two views of distance: where nodes are, and where flood traffic comes from. - Derived from ``path_length / bytes_per_hop`` rather than read from - ``complete_contact_tracking.hop_count``. Two reasons: path_length is a - byte count, so with multibyte encoding the raw value overstates hops by - 2-3x; and the stored hop_count disagrees with the path evidence badly - enough at the low end to be unusable (see _one_hop_rows). + Beware that the two tables count paths in different units, and the + conversion is not symmetric: + + * ``observed_paths.path_length`` is a BYTE count, so hops are + ``path_length / bytes_per_hop`` — a 3-hop multibyte path is 6 or 9. + * ``packet_stream.path_len`` is already a HOP count, with the byte + length carried separately as ``path_byte_length``. + + Dividing the second by bytes_per_hop, or failing to divide the first, + silently rescales a whole axis. Verified against live rows in + tests/test_dashboard_stats.py::TestHopConventions. + + The two series also cover different spans — adverts over 7 days, + packets over whatever packet_stream retains (typically 3) — so each is + labelled with its own window rather than being presented as one period. """ - rows = conn.execute( - """ - SELECT MIN(path_length / bytes_per_hop) AS hops - FROM observed_paths - WHERE packet_type = 'advert' AND bytes_per_hop > 0 - AND last_seen >= datetime('now','localtime','-7 days') - GROUP BY public_key - """ - ).fetchall() + distribution: dict[str, Any] = {"nodes": [], "flood_packets": []} + + if sources & SOURCE_OBSERVED_PATHS: + distribution["nodes"] = self._bucket_hops( + conn.execute( + """ + SELECT MIN(path_length / bytes_per_hop) AS hops, COUNT(*) AS n + FROM observed_paths + WHERE packet_type = 'advert' AND bytes_per_hop > 0 + AND last_seen >= datetime('now','localtime','-7 days') + GROUP BY public_key + """ + ), + per_row=True, + ) + + if sources & SOURCE_PACKET_STREAM: + # path_len is already hops here — do NOT divide by bytes_per_hop. + distribution["flood_packets"] = self._bucket_hops( + conn.execute( + """ + SELECT path_len AS hops, COUNT(*) AS n FROM packet_stream + WHERE type = 'packet' AND route_type_name LIKE '%FLOOD' + AND path_len IS NOT NULL + GROUP BY path_len + """ + ) + ) + + # Pad both onto one contiguous axis so the bars line up. + edges = [hop for series in distribution.values() for hop, _ in series] + if edges: + low, high = min(edges), max(edges) + for key, series in distribution.items(): + counts = dict(series) + distribution[key] = [[hop, counts.get(hop, 0)] for hop in range(low, high + 1)] + return distribution + + @staticmethod + def _bucket_hops(rows, per_row: bool = False) -> list[list[int]]: + """Fold (hops, n) rows into a hop -> count mapping, dropping absurd hops.""" counts: dict[int, int] = {} for row in rows: hops = row["hops"] - if hops is None or not 0 <= hops <= 32: + if hops is None or not 0 <= hops <= MAX_PLOTTED_HOPS: continue - counts[int(hops)] = counts.get(int(hops), 0) + 1 - if not counts: - return [] - return [[hop, counts.get(hop, 0)] for hop in range(min(counts), max(counts) + 1)] + counts[int(hops)] = counts.get(int(hops), 0) + (1 if per_row else (row["n"] or 0)) + return [[hop, count] for hop, count in sorted(counts.items())] def _count_one_hop_nodes(self, conn: sqlite3.Connection, window: str) -> int: return conn.execute( diff --git a/modules/web_viewer/static/js/dashboard.js b/modules/web_viewer/static/js/dashboard.js index 26ec1a3..ece14f2 100644 --- a/modules/web_viewer/static/js/dashboard.js +++ b/modules/web_viewer/static/js/dashboard.js @@ -367,7 +367,7 @@ // Routing, encoding, distributions this.renderRouteMix(mesh.route_mix, coverage.packets_window_label); this.renderMix('payload-mix', mesh.payload_mix); - this.renderHopsHistogram(mesh.hops_histogram); + this.renderHopsHistogram(mesh.hops, coverage.packets_window_label); this.renderDoughnut('contactsEncodingChart', 'contacts-encoding-summary', (mesh.encoding || {}).contacts_7d, 'contacts'); this.renderDoughnut('packetsEncodingChart', 'packets-encoding-summary', @@ -501,39 +501,78 @@ ); } - renderHopsHistogram(points) { + /** + * Hop distance, two ways: where nodes sit, and where flood traffic + * comes from. + * + * The series count different things — 2.8k nodes against 74k packets — + * so plotting raw counts together would flatten the node series into + * the axis. Both are shown as a share of their own total, which is what + * makes the two shapes comparable; absolute counts stay in the tooltip. + */ + renderHopsHistogram(hops, packetWindowLabel) { const canvasId = 'hopsHistogramChart'; const canvas = el(canvasId); if (!canvas || typeof Chart === 'undefined') return; - const bins = Array.isArray(points) ? points : []; + const nodes = Array.isArray((hops || {}).nodes) ? hops.nodes : []; + const flood = Array.isArray((hops || {}).flood_packets) ? hops.flood_packets : []; const colors = themeColors(); if (this.charts[canvasId]) { this.charts[canvasId].destroy(); delete this.charts[canvasId]; } - if (bins.length === 0) return; + if (nodes.length === 0 && flood.length === 0) return; + + const labels = (nodes.length ? nodes : flood).map((b) => b[0]); + const share = (series) => { + const total = series.reduce((sum, b) => sum + b[1], 0); + return { total, values: series.map((b) => (total ? (b[1] / total) * 100 : 0)) }; + }; + const nodeShare = share(nodes); + const floodShare = share(flood); + + const datasets = []; + if (nodes.length) { + datasets.push({ + label: 'Nodes (adverts, 7d)', + data: nodeShare.values, + counts: nodes.map((b) => b[1]), + unit: 'nodes', + backgroundColor: COLOR.accent, + borderRadius: 3, + }); + } + if (flood.length) { + datasets.push({ + label: 'Flood packets (' + (packetWindowLabel || 'retained') + ')', + data: floodShare.values, + counts: flood.map((b) => b[1]), + unit: 'packets', + backgroundColor: COLOR.flood, + borderRadius: 3, + }); + } this.charts[canvasId] = new Chart(canvas.getContext('2d'), { type: 'bar', - data: { - labels: bins.map((b) => b[0]), - datasets: [{ - label: 'Nodes', - data: bins.map((b) => b[1]), - backgroundColor: COLOR.accent, - borderRadius: 3, - }], - }, + data: { labels, datasets }, options: noAnimation({ responsive: true, maintainAspectRatio: false, plugins: { - legend: { display: false }, + legend: { + position: 'bottom', + labels: { color: colors.muted, boxWidth: 12, font: { size: 10 } }, + }, tooltip: { callbacks: { title: (items) => items[0].label + ' hops away', - label: (ctx) => formatNumber(ctx.parsed.y) + ' nodes', + label: (ctx) => { + const count = ctx.dataset.counts[ctx.dataIndex]; + return ctx.dataset.label + ': ' + formatNumber(count) + ' ' + + ctx.dataset.unit + ' (' + ctx.parsed.y.toFixed(1) + '%)'; + }, }, }, }, @@ -546,7 +585,13 @@ }, y: { beginAtZero: true, - ticks: { color: colors.muted, font: { size: 10 }, precision: 0 }, + title: { display: true, text: '% of series', color: colors.muted, + font: { size: 10 } }, + ticks: { + color: colors.muted, + font: { size: 10 }, + callback: (v) => v + '%', + }, grid: { color: colors.grid }, }, }, @@ -1091,10 +1136,13 @@ 'the last hop into this radio rather than the link to whoever sent it, so the ' + 'rest are left blank instead of borrowing another link\'s number.', 'hops-info': - 'Nodes by the fewest hops any of their adverts took to reach this radio, over the ' + - 'last 7 days. Hops are derived as path length divided by bytes per hop: path ' + - 'length is a byte count, so with 2- or 3-byte hop encoding the raw value would ' + - 'overstate the distance by two to three times.', + 'Two distributions on one axis. Nodes: the fewest hops any of a node\'s adverts ' + + 'took to reach this radio, over 7 days. Flood packets: how far each flood packet ' + + 'had already travelled when it arrived, over whatever the packet stream retains ' + + '(typically 3 days). Because one counts nodes and the other counts packets, both ' + + 'are drawn as a share of their own total; the tooltip gives the raw counts. ' + + 'Flood packets carry no sender identity, so they cannot be reduced to a shortest ' + + 'path per node the way adverts can.', }; function initTooltips() { diff --git a/modules/web_viewer/templates/index.html b/modules/web_viewer/templates/index.html index 14bae29..148ba47 100644 --- a/modules/web_viewer/templates/index.html +++ b/modules/web_viewer/templates/index.html @@ -139,10 +139,12 @@

- Nodes by their closest advert path, last 7 days + Nodes by their closest advert path, and how far arriving + flood packets had travelled

- +
diff --git a/tests/test_dashboard_stats.py b/tests/test_dashboard_stats.py index e2ca4a8..2726c83 100644 --- a/tests/test_dashboard_stats.py +++ b/tests/test_dashboard_stats.py @@ -699,7 +699,7 @@ class TestOneHopNeighbours: assert "device_mix" not in mesh # same field as role assert "hop_histogram" not in mesh # replaced by path-derived hops assert "path_len_histogram" not in mesh # was byte length labelled as hops - assert dict(mesh["hops_histogram"]) == {2: 1, 3: 0, 4: 1} + assert dict(mesh["hops"]["nodes"]) == {2: 1, 3: 0, 4: 1} assert mesh["neighbors"] == {"24h": 0, "7d": 0} def test_no_neighbours_degrades_cleanly(self, viewer): @@ -708,6 +708,89 @@ class TestOneHopNeighbours: assert payload["total"] == 0 +class TestHopConventions: + """The two path tables measure paths in different units. Pin both. + + observed_paths.path_length is a BYTE count; packet_stream.path_len is + already a HOP count (its byte length lives in path_byte_length). Applying + either table's rule to the other silently rescales a whole axis, and the + only visible symptom is a chart that looks slightly wrong. + """ + + def _hops(self, viewer) -> dict: + with closing(viewer._dashboard_connection()) as conn: + return viewer.dashboard_stats._hops_distribution( + conn, viewer.dashboard_stats.detect_sources(conn) + ) + + def test_advert_path_length_is_divided_by_bytes_per_hop(self, viewer): + with sqlite3.connect(viewer.db_path) as conn: + for i, (length, bph) in enumerate([(9, 3), (4, 2), (3, 1)]): # 3, 2 and 3 hops + conn.execute( + """ + INSERT INTO observed_paths (public_key, from_prefix, to_prefix, path_hex, + path_length, bytes_per_hop, packet_type, last_seen) + VALUES (?, 'aa', 'bb', 'ab', ?, ?, 'advert', datetime('now','localtime')) + """, + (_pk(i), length, bph), + ) + assert dict(self._hops(viewer)["nodes"]) == {2: 1, 3: 2} + + def test_packet_path_len_is_used_as_hops_directly(self, viewer): + """17 hops at 3 bytes each is 17 on the axis, not 5.""" + with sqlite3.connect(viewer.db_path) as conn: + conn.execute( + "INSERT INTO packet_stream (timestamp, data, type, route_type_name, path_len, " + "bytes_per_hop) VALUES (?, '{}', 'packet', 'FLOOD', 17, 3)", + (time.time(),), + ) + assert dict(self._hops(viewer)["flood_packets"]) == {17: 1} + + def test_direct_packets_are_excluded_from_the_flood_series(self, viewer): + with sqlite3.connect(viewer.db_path) as conn: + conn.executemany( + "INSERT INTO packet_stream (timestamp, data, type, route_type_name, path_len, " + "bytes_per_hop) VALUES (?, '{}', 'packet', ?, ?, 1)", + [ + (time.time(), "FLOOD", 2), + (time.time(), "TRANSPORT_FLOOD", 2), + (time.time(), "DIRECT", 2), + ], + ) + assert dict(self._hops(viewer)["flood_packets"]) == {2: 2} + + def test_series_share_one_contiguous_axis(self, viewer): + """Bars must line up, so both series are padded onto a common range.""" + with sqlite3.connect(viewer.db_path) as conn: + conn.execute( + """ + INSERT INTO observed_paths (public_key, from_prefix, to_prefix, path_hex, + path_length, bytes_per_hop, packet_type, last_seen) + VALUES (?, 'aa', 'bb', 'ab', 1, 1, 'advert', datetime('now','localtime')) + """, + (_pk(1),), + ) + conn.execute( + "INSERT INTO packet_stream (timestamp, data, type, route_type_name, path_len, " + "bytes_per_hop) VALUES (?, '{}', 'packet', 'FLOOD', 4, 1)", + (time.time(),), + ) + hops = self._hops(viewer) + assert [h for h, _ in hops["nodes"]] == [1, 2, 3, 4] + assert [h for h, _ in hops["flood_packets"]] == [1, 2, 3, 4] + assert dict(hops["nodes"]) == {1: 1, 2: 0, 3: 0, 4: 0} + assert dict(hops["flood_packets"]) == {1: 0, 2: 0, 3: 0, 4: 1} + + def test_absurd_hop_counts_are_dropped(self, viewer): + with sqlite3.connect(viewer.db_path) as conn: + conn.executemany( + "INSERT INTO packet_stream (timestamp, data, type, route_type_name, path_len, " + "bytes_per_hop) VALUES (?, '{}', 'packet', 'FLOOD', ?, 1)", + [(time.time(), 3), (time.time(), 900), (time.time(), -1)], + ) + assert dict(self._hops(viewer)["flood_packets"]) == {3: 1} + + class TestCategoryMixes: def test_tail_is_rolled_into_other_not_dropped(self, viewer):