From 0296c2e93ee5c90c8fa015415ebb9420d63a5af6 Mon Sep 17 00:00:00 2001 From: agessaman Date: Wed, 29 Jul 2026 22:34:18 -0700 Subject: [PATCH] feat(web-viewer): show payload-type mix beside the routing mix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Moving the hop histogram out into its own card left the routing mix as a 26px bar alone in a col-lg-5, next to a full-height chart. Fill it with something that belongs there: what the traffic is, beside how it is routed, over exactly the same packets — the totals agree because both read the dimensioned rows. This also gives payload_type_name a reason to exist. Migration 0019 added the column, integration.py writes it at capture time, the refresher backfills it and an index covers it, and until now nothing read it. Category lists now roll their tail into "Other" instead of being truncated at eight rows in the client. Silently dropping the tail left bars that no longer summed to the total printed beside them; on the live database that would have hidden 1,015 packets across four payload types. --- CHANGELOG.md | 4 +++ modules/web_viewer/dashboard_stats.py | 32 +++++++++++++++++- modules/web_viewer/static/js/dashboard.js | 6 +++- modules/web_viewer/templates/index.html | 3 ++ tests/test_dashboard_stats.py | 40 +++++++++++++++++++++++ 5 files changed, 83 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f8cc62..924db0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,10 @@ semantic versioning. therefore shown only where the path evidence and the stored hop count agree; the rest read "no signal reading" rather than borrowing another link's measurement. +- Payload-type mix beside the routing mix, over the same packets — what the + traffic is, next to how it is routed. Category lists roll their tail into + "Other" rather than truncating, so the bars still sum to the total printed + beside them. - Hop-distance chart derived as `path_length / bytes_per_hop`. `path_length` is a byte count, so with 2- or 3-byte hop encoding — about 95% of adverts on the live mesh — reading it directly as a hop count overstates distance two- to diff --git a/modules/web_viewer/dashboard_stats.py b/modules/web_viewer/dashboard_stats.py index c1489a2..3862316 100644 --- a/modules/web_viewer/dashboard_stats.py +++ b/modules/web_viewer/dashboard_stats.py @@ -84,6 +84,9 @@ SERIES_METRICS: dict[str, str] = { SUMMARY_SERIES_POINTS = 30 +# Categories to show in a role/payload mix before the tail is rolled into "Other". +MIX_ROWS = 8 + # Metrics that are already a ratio: a period "total" has to be the mean of the # daily values, not their sum — adding percentages together means nothing. RATIO_METRICS = frozenset({"multibyte_share"}) @@ -128,6 +131,19 @@ def _table_exists(conn: sqlite3.Connection, table: str) -> bool: return row is not None +def _top_n_with_other(entries: list[list[Any]], limit: int = MIX_ROWS) -> list[list[Any]]: + """Keep the largest *limit* categories and roll the tail into "Other". + + Truncating the list instead would silently drop counts, leaving a chart whose + bars no longer add up to the total printed beside it. + """ + ranked = sorted(entries, key=lambda entry: -(entry[1] or 0)) + if len(ranked) <= limit: + return ranked + tail = sum(entry[1] or 0 for entry in ranked[limit:]) + return [*ranked[:limit], ["Other", tail]] + + def _change_pct(current: float | None, previous: float | None) -> float | None: """Percent change, or None when the baseline cannot support one. @@ -740,6 +756,20 @@ class DashboardStatsService: "multibyte": row[2] or 0, "total": row[3] or 0, } + # Same population as route_mix — what the traffic is, beside how it + # is routed. + mesh["payload_mix"] = _top_n_with_other( + [ + [name or "Unknown", count] + for name, count in conn.execute( + """ + SELECT payload_type_name, COUNT(*) FROM packet_stream + WHERE type = 'packet' AND route_type_name IS NOT NULL + GROUP BY payload_type_name ORDER BY COUNT(*) DESC + """ + ) + ] + ) if self.multibyte_contacts_fn is not None: try: @@ -763,7 +793,7 @@ class DashboardStatsService: ): key = normalize_role(value) if column == "role" else (value or "Unknown") buckets[key] = buckets.get(key, 0) + (count or 0) - return [[name, count] for name, count in sorted(buckets.items(), key=lambda kv: -kv[1])] + return _top_n_with_other([[name, count] for name, count in buckets.items()]) def _hops_histogram(self, conn: sqlite3.Connection) -> list[list[int]]: """Nodes by how many hops away their closest observed advert path was. diff --git a/modules/web_viewer/static/js/dashboard.js b/modules/web_viewer/static/js/dashboard.js index bcdb9ef..26ec1a3 100644 --- a/modules/web_viewer/static/js/dashboard.js +++ b/modules/web_viewer/static/js/dashboard.js @@ -366,6 +366,7 @@ // Routing, encoding, distributions this.renderRouteMix(mesh.route_mix, coverage.packets_window_label); + this.renderMix('payload-mix', mesh.payload_mix); this.renderHopsHistogram(mesh.hops_histogram); this.renderDoughnut('contactsEncodingChart', 'contacts-encoding-summary', (mesh.encoding || {}).contacts_7d, 'contacts'); @@ -693,9 +694,12 @@ container.replaceChildren(makeTextElement('div', 'No data.', 'dashboard-empty')); return; } + // No slicing here: the server already rolls the tail into "Other", + // so dropping rows now would leave bars that no longer sum to the + // total shown beside them. const max = Math.max(...list.map((entry) => Number(entry[1]) || 0), 1); container.replaceChildren(); - list.slice(0, 8).forEach((entry, index) => { + list.forEach((entry, index) => { const [name, count] = entry; const row = document.createElement('div'); row.className = 'mix-row'; diff --git a/modules/web_viewer/templates/index.html b/modules/web_viewer/templates/index.html index 813e26c..14bae29 100644 --- a/modules/web_viewer/templates/index.html +++ b/modules/web_viewer/templates/index.html @@ -121,6 +121,9 @@

+
+

Payload types over the same packets

+
Loading…
diff --git a/tests/test_dashboard_stats.py b/tests/test_dashboard_stats.py index 44bfa23..e2ca4a8 100644 --- a/tests/test_dashboard_stats.py +++ b/tests/test_dashboard_stats.py @@ -708,6 +708,46 @@ class TestOneHopNeighbours: assert payload["total"] == 0 +class TestCategoryMixes: + + def test_tail_is_rolled_into_other_not_dropped(self, viewer): + """Truncating would leave bars that no longer sum to the printed total.""" + with sqlite3.connect(viewer.db_path) as conn: + conn.executemany( + "INSERT INTO packet_stream (timestamp, data, type, route_type_name, " + "payload_type_name) VALUES (?, '{}', 'packet', 'FLOOD', ?)", + [ + (time.time(), f"TYPE{i}") + for i in range(12) + for _ in range(12 - i) # descending counts, 12 distinct types + ], + ) + _refresh(viewer) + with viewer.app.test_client() as client: + mix = client.get("/api/dashboard/summary").get_json()["mesh"]["payload_mix"] + + assert len(mix) == 9, mix # 8 named categories plus Other + assert mix[-1][0] == "Other" + assert sum(count for _, count in mix) == sum(range(1, 13)) + + def test_payload_mix_matches_the_route_mix_population(self, viewer): + """Both describe the dimensioned packets, so their totals must agree.""" + with sqlite3.connect(viewer.db_path) as conn: + conn.executemany( + "INSERT INTO packet_stream (timestamp, data, type, route_type_name, " + "payload_type_name) VALUES (?, '{}', 'packet', ?, ?)", + [ + (time.time(), "FLOOD", "GRP_TXT"), + (time.time(), "TRANSPORT_FLOOD", "ADVERT"), + (time.time(), "DIRECT", "ACK"), + ], + ) + _refresh(viewer) + with viewer.app.test_client() as client: + mesh = client.get("/api/dashboard/summary").get_json()["mesh"] + assert sum(mesh["route_mix"].values()) == sum(c for _, c in mesh["payload_mix"]) == 3 + + class TestRoleBucketing: @pytest.mark.parametrize(