From 7ea34a2672e358b2d9d63a037814ec0dfefeb190 Mon Sep 17 00:00:00 2001 From: agessaman Date: Wed, 29 Jul 2026 23:16:51 -0700 Subject: [PATCH] feat(web-viewer): hide flood hop buckets under 0.1% of the series MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The flood tail decays over roughly twenty hops in bars under a pixel tall. Buckets holding less than 0.1% of the series are no longer drawn, which on the live mesh takes the axis from 64 buckets to 44. What is left out is reported, not dropped: a line under the chart reads "1,467 further flood packets (0.9%) sit in 20 hop buckets below 0.1% each, and are not drawn." An unannounced cut would be the same lie as a silently truncated category list. Two details that matter for honesty: Percentages still divide by the whole series, never by the drawn subset, so removing the tail cannot inflate the bars that remain. The smallest surviving bar on live data is 0.118%, which is what it was before. A withheld bucket inside the axis is null rather than zero. Zero would claim no packets travelled that far, which is a different and false statement; null draws nothing and says nothing. Padding gaps stay zero, because there it is true. The threshold applies to the flood series only — the node series is small enough to draw in full — and the underlying computation still covers the whole 64-hop protocol range. --- CHANGELOG.md | 11 ++++- docs/web-viewer.md | 6 +++ modules/web_viewer/dashboard_stats.py | 49 ++++++++++++++++++++-- modules/web_viewer/static/js/dashboard.js | 36 +++++++++++----- modules/web_viewer/templates/index.html | 1 + tests/test_dashboard_stats.py | 50 +++++++++++++++++++++++ 6 files changed, 136 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 908945f..d2e243d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,8 +55,15 @@ semantic versioning. earlier raw path-length chart, which read bytes as hops, and the chart built on the untrustworthy stored hop count. - The chart plots the full protocol range: a 64-byte path is 64 hops at one - byte per hop. The old dashboard's `BETWEEN 0 AND 32` filters, carried forward + Hop buckets holding under 0.1% of the flood series are not drawn — the tail + decays for around twenty hops in bars under a pixel tall — and the amount + withheld is stated beneath the chart (1,467 packets, 0.9%, on the live mesh, + taking the axis from 64 buckets to 44). Percentages remain shares of the full + series rather than of the drawn subset, so hiding the tail cannot inflate the + bars that remain. The node series is never thresholded. + + The chart still computes the full protocol range: a 64-byte path is 64 hops + at one byte per hop. The old dashboard's `BETWEEN 0 AND 32` filters, carried forward at first, discarded 5,654 flood packets arriving from as far as 63 hops, and because that limit applies after the per-node minimum it would erase a node whose closest path was longer than 32 hops rather than plotting it at the far diff --git a/docs/web-viewer.md b/docs/web-viewer.md index 66f6bba..893d088 100644 --- a/docs/web-viewer.md +++ b/docs/web-viewer.md @@ -168,6 +168,12 @@ packets, so each is drawn as a share of its own total. Flood packets carry no sender identity, which is why they cannot be reduced to a shortest path per node the way adverts can. +Flood hop buckets holding under 0.1% of the series are omitted, because that +tail decays over roughly twenty hops in bars thinner than a pixel. The number of +packets withheld is printed beneath the chart, and percentages stay shares of +the full series so that hiding the tail cannot inflate the remaining bars. The +node series is shown in full. + **Neighbour signal is reported only where two sources agree.** `complete_contact_tracking.hop_count` is not a reliable direct-neighbour marker: on a representative database it claims 800 zero-hop contacts while only 68 have diff --git a/modules/web_viewer/dashboard_stats.py b/modules/web_viewer/dashboard_stats.py index 7a3faa6..8f481e9 100644 --- a/modules/web_viewer/dashboard_stats.py +++ b/modules/web_viewer/dashboard_stats.py @@ -99,6 +99,12 @@ MIX_ROWS = 8 # at the far end; one such node exists in the live history. MAX_PLOTTED_HOPS = 64 +# Flood hop buckets carrying less than this share of the series are not drawn. +# The tail decays for ~20 hops in bars under a pixel tall, which reads as noise; +# the amount withheld is reported alongside the chart rather than dropped +# quietly. +FLOOD_MIN_SHARE_PCT = 0.1 + # Metrics that are already a ratio: a period "total" has to be the mean of the # daily values, not their sum — adding percentages together means nothing. RATIO_METRICS = frozenset({"multibyte_share"}) @@ -856,17 +862,52 @@ class DashboardStatsService: ) ) + totals = {key: sum(count for _, count in series) for key, series in distribution.items()} + + # The flood tail is a long thin decay — on the live mesh it runs to 63 + # hops in bars under a pixel tall. Hide the buckets carrying less than + # FLOOD_MIN_SHARE_PCT of the series, but count what was hidden: an + # unannounced cut is the same lie as a truncated category list. + flood_total = totals["flood_packets"] + hidden_packets = 0 + hidden_buckets = 0 + if flood_total: + kept = [] + for hop, count in distribution["flood_packets"]: + if count and (count / flood_total) * 100 < FLOOD_MIN_SHARE_PCT: + hidden_packets += count + hidden_buckets += 1 + kept.append([hop, None]) + else: + kept.append([hop, count]) + distribution["flood_packets"] = kept + # Pad both onto one contiguous axis so the bars line up, bounded by the - # hops that actually carry something: an axis that runs on past the last - # observation spends its width on nothing. + # hops that actually show something: an axis that runs on past the last + # drawn bar spends its width on nothing. populated = [ - hop for series in distribution.values() for hop, count in series if count + hop + for series in distribution.values() + for hop, count in series + if count ] if populated: low, high = min(populated), max(populated) for key, series in distribution.items(): counts = dict(series) - distribution[key] = [[hop, counts.get(hop, 0)] for hop in range(low, high + 1)] + # Gaps pad with 0 ("no packets at this hop"), which is a + # different claim from the None used above for "withheld". + distribution[key] = [ + [hop, counts.get(hop, 0)] for hop in range(low, high + 1) + ] + + distribution["totals"] = totals + distribution["flood_hidden"] = { + "packets": hidden_packets, + "buckets": hidden_buckets, + "share_pct": round(hidden_packets / flood_total * 100, 2) if flood_total else 0, + "threshold_pct": FLOOD_MIN_SHARE_PCT, + } return distribution @staticmethod diff --git a/modules/web_viewer/static/js/dashboard.js b/modules/web_viewer/static/js/dashboard.js index bc187a8..d8e3b62 100644 --- a/modules/web_viewer/static/js/dashboard.js +++ b/modules/web_viewer/static/js/dashboard.js @@ -522,15 +522,25 @@ this.charts[canvasId].destroy(); delete this.charts[canvasId]; } + const withheld = (hops || {}).flood_hidden || {}; + setText('hops-hidden-note', withheld.packets + ? formatNumber(withheld.packets) + ' further flood packets (' + + withheld.share_pct + '%) sit in ' + withheld.buckets + + ' hop buckets below ' + withheld.threshold_pct + '% each, and are not drawn.' + : ''); + if (nodes.length === 0 && flood.length === 0) return; const labels = (nodes.length ? nodes : flood).map((b) => b[0]); - const share = (series) => { - const total = series.reduce((sum, b) => sum + b[1], 0); - return { total, values: series.map((b) => (total ? (b[1] / total) * 100 : 0)) }; - }; - const nodeShare = share(nodes); - const floodShare = share(flood); + const totals = (hops || {}).totals || {}; + // Percentages are shares of the whole series, not of the bars that + // survived the display threshold — otherwise hiding the tail would + // silently inflate every remaining bar. + const share = (series, total) => series.map( + (b) => (b[1] === null || !total ? null : (b[1] / total) * 100) + ); + const nodeShare = share(nodes, totals.nodes); + const floodShare = share(flood, totals.flood_packets); // A path can be 64 hops at one byte per hop, so the axis may carry // 64 grouped categories. Rounded corners and default bar padding @@ -540,7 +550,7 @@ if (nodes.length) { datasets.push({ label: 'Nodes (adverts, 7d)', - data: nodeShare.values, + data: nodeShare, counts: nodes.map((b) => b[1]), unit: 'nodes', backgroundColor: COLOR.accent, @@ -552,7 +562,7 @@ if (flood.length) { datasets.push({ label: 'Flood packets (' + (packetWindowLabel || 'retained') + ')', - data: floodShare.values, + data: floodShare, counts: flood.map((b) => b[1]), unit: 'packets', backgroundColor: COLOR.flood, @@ -578,8 +588,9 @@ title: (items) => items[0].label + ' hops away', label: (ctx) => { const count = ctx.dataset.counts[ctx.dataIndex]; + if (count === null || count === undefined) return null; return ctx.dataset.label + ': ' + formatNumber(count) + ' ' + - ctx.dataset.unit + ' (' + ctx.parsed.y.toFixed(1) + '%)'; + ctx.dataset.unit + ' (' + ctx.parsed.y.toFixed(2) + '%)'; }, }, }, @@ -1072,8 +1083,11 @@ 'had already travelled when it arrived, over whatever the packet stream retains ' + '(typically 3 days). Because one counts nodes and the other counts packets, both ' + 'are drawn as a share of their own total; the tooltip gives the raw counts. ' + - 'Flood packets carry no sender identity, so they cannot be reduced to a shortest ' + - 'path per node the way adverts can.', + 'Hop buckets holding under 0.1% of the flood series are omitted — that tail ' + + 'decays for around twenty hops in bars under a pixel tall — and the amount left ' + + 'out is stated beneath the chart. Percentages stay shares of the full series, ' + + 'not of the drawn subset. Flood packets carry no sender identity, so they cannot ' + + 'be reduced to a shortest path per node the way adverts can.', }; function initTooltips() { diff --git a/modules/web_viewer/templates/index.html b/modules/web_viewer/templates/index.html index fbc430a..2db37c0 100644 --- a/modules/web_viewer/templates/index.html +++ b/modules/web_viewer/templates/index.html @@ -146,6 +146,7 @@ +

diff --git a/tests/test_dashboard_stats.py b/tests/test_dashboard_stats.py index c8ff545..6927406 100644 --- a/tests/test_dashboard_stats.py +++ b/tests/test_dashboard_stats.py @@ -824,6 +824,56 @@ class TestHopConventions: ) assert dict(self._hops(viewer)["nodes"]) == {48: 1} + def test_negligible_flood_buckets_are_withheld_and_counted(self, viewer): + """The long thin tail is hidden, but never silently.""" + with sqlite3.connect(viewer.db_path) as conn: + conn.executemany( + "INSERT INTO packet_stream (timestamp, data, type, route_type_name, path_len, " + "bytes_per_hop) VALUES (?, '{}', 'packet', 'FLOOD', ?, 1)", + # 2000 at hop 3, then single packets far out: each is 0.05% of + # 2002, comfortably under the 0.1% display threshold. + [(time.time(), 3) for _ in range(2000)] + [(time.time(), 40), (time.time(), 55)], + ) + hops = self._hops(viewer) + drawn = dict(hops["flood_packets"]) + + assert drawn[3] == 2000 + assert 40 not in drawn and 55 not in drawn, "withheld buckets fall outside the axis" + assert hops["flood_hidden"] == { + "packets": 2, + "buckets": 2, + "share_pct": 0.1, + "threshold_pct": 0.1, + } + # Percentages must divide by the whole series, not the drawn subset. + assert hops["totals"]["flood_packets"] == 2002 + + def test_withheld_bucket_inside_the_axis_is_null_not_zero(self, viewer): + """Null says "not shown"; zero would claim no packets travelled that far.""" + with sqlite3.connect(viewer.db_path) as conn: + conn.executemany( + "INSERT INTO packet_stream (timestamp, data, type, route_type_name, path_len, " + "bytes_per_hop) VALUES (?, '{}', 'packet', 'FLOOD', ?, 1)", + [(time.time(), 2) for _ in range(1000)] + + [(time.time(), 5)] # 0.09% — withheld + + [(time.time(), 9) for _ in range(100)], # keeps the axis open past it + ) + drawn = dict(self._hops(viewer)["flood_packets"]) + assert drawn[5] is None, "a withheld bucket inside the range must be null" + assert drawn[6] == 0, "a genuinely empty bucket stays zero" + + def test_a_single_bucket_is_never_withheld(self, viewer): + """One bucket holding everything is 100%, not below threshold.""" + with sqlite3.connect(viewer.db_path) as conn: + conn.execute( + "INSERT INTO packet_stream (timestamp, data, type, route_type_name, path_len, " + "bytes_per_hop) VALUES (?, '{}', 'packet', 'FLOOD', 7, 1)", + (time.time(),), + ) + hops = self._hops(viewer) + assert dict(hops["flood_packets"]) == {7: 1} + assert hops["flood_hidden"]["packets"] == 0 + def test_axis_stops_at_the_last_populated_hop(self, viewer): """An axis running past the last observation spends its width on nothing.""" with sqlite3.connect(viewer.db_path) as conn: