From 75fda424c25ba8ffe8df377c8c744c640fdf25a7 Mon Sep 17 00:00:00 2001 From: agessaman Date: Fri, 31 Jul 2026 21:23:51 -0700 Subject: [PATCH] feat(web-viewer): implement per-payload-type multibyte share tracking - Added a new feature to track and display the multibyte share of packets by payload type in the dashboard. - Introduced a new database migration to store per-payload-type multibyte encoding data in the daily rollup. - Updated the dashboard to visualize the multibyte share as a stacked bar chart, reflecting the share of each day's packets that took a multibyte path. - Enhanced the API to provide raw counts for each payload type, ensuring accurate representation in the dashboard. - Adjusted the frontend to maintain consistent color coding for payload types and improve the overall user experience. - Updated tests to validate the new multibyte share functionality and ensure data integrity. --- docs/web-viewer.md | 56 +++++- modules/db_migrations.py | 18 ++ modules/web_viewer/dashboard_stats.py | 165 +++++++++++++++- modules/web_viewer/static/js/dashboard.js | 231 +++++++++++++++++++--- modules/web_viewer/templates/index.html | 9 +- tests/test_dashboard_stats.py | 183 +++++++++++++++++ 6 files changed, 616 insertions(+), 46 deletions(-) diff --git a/docs/web-viewer.md b/docs/web-viewer.md index 893d088..dca2e2b 100644 --- a/docs/web-viewer.md +++ b/docs/web-viewer.md @@ -143,7 +143,11 @@ proxy_set_header X-Forwarded-Proto $scheme; coverage — the count tiles carry a 30-day sparkline and a change chip - Routing mix (flood vs direct), a hop-distance chart, and the role mix - Path encoding: multibyte share among contacts and among incoming packets, - plus a 30-day multibyte adoption trend + plus a 30-day stacked bar whose height is the share of each day's packets that + took a multibyte path, split by the payload type carrying them (`GRP_TXT`, + `RESPONSE`, `REQ`, `PATH`, `TXT_MSG`, `ANON_REQ`, `GRP_DATA`, `ADVERT`, and + `Other` for the rest). Its y-axis is the tallest bar rounded up to the next + 5%, so it rescales as the mesh changes - Busiest repeaters, and **one-hop neighbours** (24-hour or 7-day window) The live packet feed lives on the **Real-time** page rather than here; the @@ -206,8 +210,42 @@ Two consequences worth knowing: from each source's configured retention, so the list cannot offer "30 days" against a table pruned at 7. -The **multibyte share** trend is accumulated forward, not derived. `observed_paths` -is deduplicated with a lifetime `observation_count` and a `last_seen` that is +The **multibyte share** trends are accumulated forward, not derived. + +The per-payload-type figures come from `packet_stream`, which is pruned within +days while the chart spans thirty, so each day's split is written into +`daily_rollup.packet_type_encoding` as that day is rolled up and cannot be +recomputed afterwards. Enabling the feature therefore starts an empty chart that +fills in over the following month. Packets whose denormalized dimensions have +not been backfilled yet are excluded from both sides of the ratio rather than +counted as single-byte — counting them would invent a dip in whichever type the +backfill has not reached. + +**The chart measures the day, and the API serves per-type adoption.** A bar's +height is the share of that day's packets that went multibyte, and its segments +are each type's multibyte packets over that same day-wide denominator — so the +segments sum to the bar and the bar equals the figure the packet doughnut +reports for its own window. Every payload type is counted, with the uncharted +tail (`ACK`, `TRACE`, unmapped ordinals — about 0.8% of live traffic) summed into +`Other`; omitting it would leave bar heights a share of the charted types rather +than of the day. + +The `multibyte_share_*` metrics answer the different question "how much of *this +type* went multibyte?", each a ratio over its own denominator. Eight such ratios +share no denominator and cannot be stacked, which is why the dashboard payload +carries the raw counts in `packet_encoding` rather than the eight percentage +series; the tooltip quotes both readings per segment. + +**Two different advert shares exist, and they do not agree.** The charted +`multibyte_share_advert` counts advert *packets* off the packet stream, the same +way as every other line. The older `multibyte_share` counts a day's adverts +against a classification of the *node* that sent them — one multibyte path ever +observed marks that node multibyte for every advert it sends. Neither is wrong; +they answer different questions, and the gap between them is roughly the set of +nodes that can do multibyte but mostly do not. + +That older share is also frozen for a different reason. `observed_paths` is +deduplicated with a lifetime `observation_count` and a `last_seen` that is bumped on every re-observation, so historical per-day shares cannot be reconstructed from it — nearly half the observation volume would be attributed to the wrong day. Each refresh recomputes today plus a three-day trailing @@ -265,11 +303,17 @@ window; older days stay frozen at the value recorded then. The viewer also provides JSON API endpoints: - `GET /api/dashboard/summary` - Snapshot-backed dashboard payload, including - 30-day sparkline series and change figures. Sends a strong `ETag`; poll with - `If-None-Match` to get a bodyless `304` while the snapshot is unchanged. + 30-day sparkline series and change figures, plus `packet_encoding`: 30 days of + raw per-payload-type multibyte/total counts for the stacked encoding chart. + Sends a strong `ETag`; poll with `If-None-Match` to get a bodyless `304` while + the snapshot is unchanged. - `GET /api/dashboard/series?metric=&days=` - Full-history points for one metric. `metric` is one of `messages`, `commands`, `adverts`, `nodes`, - `new_nodes`, `packets`, `multibyte_share`. + `new_nodes`, `packets`, `multibyte_share` (adverts), or the per-payload-type + packet shares `multibyte_share_grp_txt`, `multibyte_share_response`, + `multibyte_share_req`, `multibyte_share_path`, `multibyte_share_txt_msg`, + `multibyte_share_anon_req`, `multibyte_share_grp_data`, + `multibyte_share_advert`. - `GET /api/dashboard/top?kind=&window=&limit=` - One leaderboard. `kind` is one of `users`, `commands`, `channels`, `paths`, `repeaters`. The response carries `window_label`, `retention_days`, and diff --git a/modules/db_migrations.py b/modules/db_migrations.py index 57912d1..b67bce6 100644 --- a/modules/db_migrations.py +++ b/modules/db_migrations.py @@ -711,6 +711,23 @@ def _m0020_mesh_connections_last_seen_index(cursor: sqlite3.Cursor) -> None: ) +def _m0021_daily_rollup_packet_type_encoding(cursor: sqlite3.Cursor) -> None: + """Record the per-payload-type multibyte split on each rollup day. + + ``packet_stream`` is pruned at three days, so the dashboard's 30-day + encoding trend cannot be recomputed from it after the fact — the split has + to be written as each day is rolled up, the same accumulate-forward shape + the advert share already uses. + + One JSON column rather than a count pair per type: the payload-type + vocabulary belongs to the firmware, not to us, so a type appearing on the + mesh should not need a schema migration to be charted. + """ + if not _table_exists(cursor, "daily_rollup"): + return + _add_column(cursor, "daily_rollup", "packet_type_encoding", "TEXT") + + # --------------------------------------------------------------------------- # Migration registry — append new entries here, never remove or reorder. # --------------------------------------------------------------------------- @@ -738,6 +755,7 @@ MIGRATIONS: list[MigrationEntry] = [ (18, "dashboard rollup and snapshot tables", _m0018_dashboard_rollup_tables), (19, "packet_stream: denormalized packet dimensions", _m0019_packet_stream_denorm_dims), (20, "mesh_connections: table-specific last_seen index", _m0020_mesh_connections_last_seen_index), + (21, "daily_rollup: per-payload-type multibyte split", _m0021_daily_rollup_packet_type_encoding), ] diff --git a/modules/web_viewer/dashboard_stats.py b/modules/web_viewer/dashboard_stats.py index 8f481e9..27d1d31 100644 --- a/modules/web_viewer/dashboard_stats.py +++ b/modules/web_viewer/dashboard_stats.py @@ -66,8 +66,50 @@ SOURCE_NAMES = { SOURCE_OBSERVED_PATHS: "observed_paths", } +# Payload types charted on the packet encoding trend, in the fixed order the +# chart assigns its categorical colours in. A colour belongs to a type and not +# to its current rank, so a quiet day for one type must never repaint the other +# seven lines — which means this order is part of the contract with the client, +# not a display detail, and new types are appended rather than inserted. +PACKET_ENCODING_TYPES = ( + "GRP_TXT", "RESPONSE", "REQ", "PATH", "TXT_MSG", "ANON_REQ", "GRP_DATA", "ADVERT", +) + +# Everything else the firmware emits — ACK, TRACE, MULTIPART, unmapped ordinals +# like 'Type11' — folded into one bucket rather than dropped. On the live mesh +# that is 0.8% of traffic, and dropping it would leave the chart's bar heights a +# share of the charted types instead of a share of the day, disagreeing with the +# multibyte doughnut sitting on the same card. Charting each separately instead +# would spend the palette's remaining separation on traffic nobody watches. +OTHER_PAYLOAD_TYPE = "OTHER" + +# Storage/chart order. The named types keep their colour slots and the residual +# bucket sits last, drawn in a neutral so it does not read as a ninth category. +PACKET_ENCODING_BUCKETS = (*PACKET_ENCODING_TYPES, OTHER_PAYLOAD_TYPE) + + +def packet_share_metric(payload_type: str) -> str: + """Series-metric name carrying one payload type's multibyte share.""" + return f"multibyte_share_{payload_type.lower()}" + + +def _packet_share_sql(payload_type: str) -> str: + """Percent of that type's classified packets that used 2- or 3-byte hops. + + Reads the JSON written by ``_packet_encoding_by_type``. A day with no + stored split, or none for this type, yields NULL — a gap, not a zero. + """ + multibyte = f"""json_extract(packet_type_encoding, '$."{payload_type}".mb')""" + total = f"""json_extract(packet_type_encoding, '$."{payload_type}".total')""" + return ( + f"CASE WHEN COALESCE({total}, 0) > 0 " + f"THEN ROUND(COALESCE({multibyte}, 0) * 100.0 / {total}, 1) END" + ) + + # Series exposed by /api/dashboard/series and folded into the summary payload. -# multibyte_share is a ratio the UI plots on a 0-100 axis; the rest are counts. +# The multibyte shares are ratios the UI plots on a 0-100 axis; the rest are +# counts. SERIES_METRICS: dict[str, str] = { "messages": "messages_total", "commands": "commands_total", @@ -80,8 +122,25 @@ SERIES_METRICS: dict[str, str] = { "THEN ROUND(adverts_from_multibyte * 100.0 / " "(adverts_from_multibyte + adverts_from_singlebyte), 1) END" ), + **{ + packet_share_metric(payload_type): _packet_share_sql(payload_type) + for payload_type in PACKET_ENCODING_TYPES + }, } +PACKET_SHARE_METRICS = frozenset( + packet_share_metric(payload_type) for payload_type in PACKET_ENCODING_TYPES +) + +# Metrics folded into the summary payload's sparkline block. The per-payload- +# type shares stay out of it: the dashboard chart stacks raw counts from +# ``packet_encoding`` instead, and eight independent percentages cannot be +# restacked into a composition — the shared denominator is gone. They remain on +# /api/dashboard/series for anyone plotting one type on its own. +SUMMARY_METRICS = tuple( + name for name in SERIES_METRICS if name not in PACKET_SHARE_METRICS +) + SUMMARY_SERIES_POINTS = 30 # Categories to show in a role/payload mix before the tail is rolled into "Other". @@ -107,7 +166,12 @@ FLOOD_MIN_SHARE_PCT = 0.1 # Metrics that are already a ratio: a period "total" has to be the mean of the # daily values, not their sum — adding percentages together means nothing. -RATIO_METRICS = frozenset({"multibyte_share"}) +RATIO_METRICS = frozenset( + { + "multibyte_share", + *(packet_share_metric(payload_type) for payload_type in PACKET_ENCODING_TYPES), + } +) TOP_KINDS = ("users", "commands", "channels", "paths", "repeaters", "neighbors") @@ -440,8 +504,52 @@ class DashboardStatsService: "packets_flood": row[1] or 0, "packets_direct": row[2] or 0, "packets_multibyte": row[3] or 0, + "packet_type_encoding": self._packet_encoding_by_type(conn, start, end), } + def _packet_encoding_by_type( + self, conn: sqlite3.Connection, start: float, end: float + ) -> str | None: + """One day's multibyte split per payload type, as JSON. + + Written now because it cannot be recovered later: packet_stream is + pruned at three days while the chart shows thirty. + + Covers *every* payload type, with the uncharted tail summed into + ``OTHER``, because the totals here are the chart's denominator: bar + heights are each type's multibyte packets over the day's whole traffic, + so leaving a type out of the file would quietly inflate every bar. + + Packets whose dimensions the refresher has not backfilled yet are + excluded from both sides rather than counted as single-byte. On a + count that would merely undershoot; on a ratio it invents a dip in + whichever type the backfill has not reached, which reads as a real + change in the mesh. + """ + counts: dict[str, dict[str, int]] = {} + for name, total, multibyte in conn.execute( + """ + SELECT payload_type_name, COUNT(*), + SUM(CASE WHEN bytes_per_hop IN (2, 3) THEN 1 ELSE 0 END) + FROM packet_stream + WHERE type = 'packet' AND timestamp >= ? AND timestamp < ? + AND bytes_per_hop IS NOT NULL + GROUP BY payload_type_name + """, + (start, end), + ): + # A NULL type is dimensioned but unnamed, which is still a packet + # the day's traffic contains — it belongs in the residual bucket, + # not thrown away. + key = name if name in PACKET_ENCODING_TYPES else OTHER_PAYLOAD_TYPE + bucket = counts.setdefault(key, {"mb": 0, "total": 0}) + bucket["mb"] += multibyte or 0 + bucket["total"] += total or 0 + # Nothing found means "cannot say", not "the mesh was silent": a day + # whose rows were pruned ahead of the retention window would otherwise + # overwrite the split recorded for it while the rows still existed. + return json.dumps(counts, separators=(",", ":")) if counts else None + def _advert_metrics( self, conn: sqlite3.Connection, @@ -578,6 +686,7 @@ class DashboardStatsService: "snr_sum", "snr_count", "rssi_sum", "rssi_count", "hops_sum", "hops_count", "packets_total", "packets_flood", "packets_direct", "packets_multibyte", + "packet_type_encoding", "adverts_total", "nodes_active", "nodes_new", "adverts_from_multibyte", "adverts_from_singlebyte", "contacts_known", "contacts_tracked", @@ -1020,7 +1129,7 @@ class DashboardStatsService: def _series_from_rollup( self, conn: sqlite3.Connection, points: int = SUMMARY_SERIES_POINTS ) -> dict[str, list[dict[str, Any]]]: - expressions = ", ".join(f"{sql} AS {name}" for name, sql in SERIES_METRICS.items()) + expressions = ", ".join(f"{SERIES_METRICS[name]} AS {name}" for name in SUMMARY_METRICS) rows = conn.execute( f""" SELECT date, is_final, {expressions} @@ -1028,9 +1137,9 @@ class DashboardStatsService: """, (points,), ).fetchall() - series: dict[str, list[dict[str, Any]]] = {name: [] for name in SERIES_METRICS} + series: dict[str, list[dict[str, Any]]] = {name: [] for name in SUMMARY_METRICS} for row in reversed(rows): - for name in SERIES_METRICS: + for name in SUMMARY_METRICS: series[name].append( { "date": row["date"], @@ -1046,7 +1155,7 @@ class DashboardStatsService: Explicitly calendar days, not a rolling 24 hours: quoting one against the other is the classic dashboard lie, so the UI says which it is. """ - expressions = ", ".join(f"{sql} AS {name}" for name, sql in SERIES_METRICS.items()) + expressions = ", ".join(f"{SERIES_METRICS[name]} AS {name}" for name in SUMMARY_METRICS) rows = conn.execute( f""" SELECT date, {expressions} FROM daily_rollup @@ -1056,7 +1165,48 @@ class DashboardStatsService: if len(rows) < 2: return {} current, previous = rows[0], rows[1] - return {name: _change_pct(current[name], previous[name]) for name in SERIES_METRICS} + return {name: _change_pct(current[name], previous[name]) for name in SUMMARY_METRICS} + + def _packet_encoding_from_rollup( + self, conn: sqlite3.Connection, points: int = SUMMARY_SERIES_POINTS + ) -> list[dict[str, Any]]: + """Per-day multibyte/total counts per payload type, oldest first. + + The chart stacks raw counts rather than the ratio series because a + composition needs one shared denominator — each day's multibyte total — + and that cannot be rebuilt from eight percentages taken over eight + different denominators. Sending the counts also lets the tooltip quote + both readings: a type's slice of the day's multibyte traffic, and how + much of that type's own traffic was multibyte. + """ + rows = conn.execute( + "SELECT date, is_final, packet_type_encoding FROM daily_rollup " + "ORDER BY date DESC LIMIT ?", + (points,), + ).fetchall() + series: list[dict[str, Any]] = [] + for row in reversed(rows): + try: + stored = json.loads(row["packet_type_encoding"] or "{}") + except (TypeError, ValueError): + self.logger.debug(f"Unreadable packet encoding split for {row['date']}") + stored = {} + series.append( + { + "date": row["date"], + "complete": bool(row["is_final"]), + # Keyed by name: Flask sorts JSON object keys, so the client + # cannot read a colour slot off position and looks each + # bucket up instead. OTHER is included — it is part of the + # denominator the client divides by. + "types": { + name: stored[name] + for name in PACKET_ENCODING_BUCKETS + if isinstance(stored.get(name), dict) + }, + } + ) + return series def build_snapshot(self, conn: sqlite3.Connection) -> dict[str, Any]: """Assemble the whole current-state payload. Read-only.""" @@ -1076,6 +1226,7 @@ class DashboardStatsService: } payload["series"] = self._series_from_rollup(conn) payload["deltas"] = self._deltas_from_rollup(conn) + payload["packet_encoding"] = self._packet_encoding_from_rollup(conn) return payload # -- refresh orchestration ---------------------------------------------- diff --git a/modules/web_viewer/static/js/dashboard.js b/modules/web_viewer/static/js/dashboard.js index d8e3b62..63bf314 100644 --- a/modules/web_viewer/static/js/dashboard.js +++ b/modules/web_viewer/static/js/dashboard.js @@ -33,6 +33,41 @@ '#d63384', '#198754', '#0dcaf0', '#adb5bd', ]; + // Payload buckets on the encoding chart. Must stay in the same order as + // PACKET_ENCODING_BUCKETS server-side: the slot index picks the colour, so a + // type keeps its colour even on a day when it has no traffic and drops out + // of the chart. Colour by rank instead and one quiet day repaints the lot. + // Append new types; inserting one would recolour everything after it. + const PACKET_ENCODING_TYPES = [ + 'GRP_TXT', 'RESPONSE', 'REQ', 'PATH', 'TXT_MSG', 'ANON_REQ', 'GRP_DATA', 'ADVERT', + // The uncharted tail (ACK, TRACE, unmapped ordinals). Present so the + // bars measure the whole day's traffic, not just the named types. + 'OTHER', + ]; + + // Eight categorical slots, each mode stepped for its own card surface + // (#ffffff light, #2d2d2d dark) rather than flipped from the other. Both + // columns are validated for lightness band, chroma floor, colour-vision + // separation (worst adjacent pair ΔE 9.1 light / 8.4 dark) and + // normal-vision separation (19.6 / 19.3). Several sit under 3:1 against + // the surface, which is why the legend is not optional here — identity has + // to survive without the colour. Eight is the ceiling for hues: the ninth + // slot is deliberately a neutral, because "everything else" is not a + // category and must not compete with the ones that are. + const SERIES_PALETTE = { + light: ['#2a78d6', '#eb6834', '#1baf7a', '#eda100', '#e87ba4', '#008300', '#4a3aa7', '#e34948', '#a8a29e'], + dark: ['#3987e5', '#d95926', '#199e70', '#c98500', '#d55181', '#008300', '#9085e9', '#e66767', '#87817c'], + }; + + // Segments thinner than this fraction of the y-axis range are drawn without + // the hairline that separates stacked bands — below it the border is + // thicker than the band and erases the colour instead of framing it. + const MIN_SEGMENT_FRACTION_FOR_GAP = 0.01; + + // The y-axis tops out at the tallest bar rounded up to the next multiple of + // this, so a 24% peak gets a 25% axis and a 26% peak gets 30%. + const AXIS_STEP_PCT = 5; + function readBoot() { const el = document.getElementById('dashboard-boot'); if (!el) return {}; @@ -129,6 +164,10 @@ muted: (cs.getPropertyValue('--text-muted') || '#6c757d').trim(), grid: isDark() ? 'rgba(255,255,255,0.08)' : 'rgba(0,0,0,0.06)', empty: (cs.getPropertyValue('--bg-tertiary') || '#dee2e6').trim(), + // The card the chart sits on. Stacked segments are separated by a + // hairline of the surface itself rather than a drawn border, so the + // gap reads as space instead of as an extra category. + surface: (cs.getPropertyValue('--card-bg') || '#ffffff').trim(), }; } @@ -373,7 +412,7 @@ this.renderDoughnut('packetsEncodingChart', 'packets-encoding-summary', (mesh.encoding || {}).packets, 'packets'); setText('packets-window-label', coverage.packets_window_label || 'no data'); - this.renderMultibyteTrend(series.multibyte_share); + this.renderPacketEncodingTrend(data.packet_encoding); this.renderMix('role-mix', mesh.role_mix); this.renderSourceNotes(coverage); @@ -697,58 +736,185 @@ }); } - renderMultibyteTrend(points) { + /** + * One bar per day. The bar's full height is the share of that day's + * packets that took a multibyte path, and it is divided by the payload + * type that carried them — so a day where 24% of traffic went multibyte + * draws a bar 24% tall, split eight ways. + * + * Every segment is a fraction of the same denominator, the day's whole + * traffic. That is what makes the stack legitimate: each type's *own* + * multibyte share is a ratio over its own packet count, and eight such + * ratios share no denominator, so stacking those would sum to whatever + * it happened to sum to. The per-type adoption figure is not lost — + * the tooltip quotes both readings. + */ + renderPacketEncodingTrend(days) { const canvasId = 'multibyteTrendChart'; const canvas = el(canvasId); if (!canvas || typeof Chart === 'undefined') return; - const list = Array.isArray(points) ? points : []; + const list = Array.isArray(days) ? days : []; const colors = themeColors(); + const palette = SERIES_PALETTE[isDark() ? 'dark' : 'light']; if (this.charts[canvasId]) { this.charts[canvasId].destroy(); delete this.charts[canvasId]; } + + const counts = (day, name) => (day.types || {})[name] || null; + // The denominator is every packet the day classified, including the + // OTHER bucket and every single-byte packet — not just the + // multibyte ones, which is what makes bar height a share of traffic. + const dayPackets = list.map((day) => Object.values(day.types || {}).reduce( + (sum, entry) => sum + ((entry && entry.total) || 0), 0 + )); + const dayMultibyte = list.map((day) => Object.values(day.types || {}).reduce( + (sum, entry) => sum + ((entry && entry.mb) || 0), 0 + )); + const segment = (day, name, i) => { + const entry = counts(day, name); + if (!entry || !dayPackets[i]) return null; + return (entry.mb / dayPackets[i]) * 100; + }; + const barHeights = list.map((day, i) => PACKET_ENCODING_TYPES.reduce( + (sum, name) => sum + (segment(day, name, i) || 0), 0 + )); + + // Round the tallest bar up to the next step so the bars fill the + // plot: a 24% peak against a 0-100 axis wastes three quarters of the + // card and flattens every difference worth seeing. + const peak = barHeights.length ? Math.max(...barHeights) : 0; + const axisMax = Math.min(100, Math.max( + AXIS_STEP_PCT, + // Nudge down before rounding so a peak landing exactly on a step + // keeps that step rather than jumping to the next one. + Math.ceil((peak - 1e-9) / AXIS_STEP_PCT) * AXIS_STEP_PCT + )); + + const datasets = []; + PACKET_ENCODING_TYPES.forEach((name, index) => { + // A type that carried no multibyte traffic all window would take + // a legend slot to describe an invisible segment. + if (!list.some((day, i) => segment(day, name, i) > 0)) return; + const color = palette[index % palette.length]; + datasets.push({ + label: name === 'OTHER' ? 'Other' : name, + data: list.map((day, i) => segment(day, name, i)), + // Raw figures ride along so the tooltip can quote packet + // counts and each type's own adoption without a second fetch. + rawCounts: list.map((day) => counts(day, name)), + backgroundColor: color, + // A hairline of the card colour between segments; without it + // two adjacent hues read as one band. Dropped on segments + // too thin to separate — GRP_DATA runs well under 1% of a + // day, and a border on a 1px band erases the colour + // entirely, turning a real category into a surface-coloured + // gap. Measured against the axis, not the value, because + // the axis is what decides how many pixels a percent buys. + borderColor: colors.surface, + borderWidth: (ctx) => ( + (ctx.raw ?? 0) / axisMax >= MIN_SEGMENT_FRACTION_FOR_GAP + ? { top: 1, bottom: 1 } + : 0 + ), + borderSkipped: false, + // Now that the tooltip describes one segment, something has + // to say which. Chart.js would answer by saturating the + // fill to a hue outside the validated palette; outline it + // instead, so identity keeps its colour and the cue reads as + // a pointer. It also gives the sub-pixel slices a mark big + // enough to see once they are the ones being described. + hoverBackgroundColor: color, + hoverBorderColor: colors.text, + hoverBorderWidth: 1, + }); + }); + const empty = el('multibyte-trend-empty'); - const hasData = list.some((p) => p.value !== null && p.value !== undefined); + const hasData = datasets.length > 0 && barHeights.some((height) => height > 0); if (empty) empty.style.display = hasData ? 'none' : ''; canvas.style.display = hasData ? '' : 'none'; if (!hasData) return; this.charts[canvasId] = new Chart(canvas.getContext('2d'), { - type: 'line', - data: { - labels: list.map((p) => p.date), - datasets: [{ - label: 'Multibyte share', - data: list.map((p) => p.value), - borderColor: COLOR.multibyte, - backgroundColor: COLOR.multibyte + '33', - fill: true, - tension: 0.4, - pointRadius: 0, - pointHoverRadius: 4, - spanGaps: false, - }], - }, + type: 'bar', + data: { labels: list.map((day) => day.date), datasets }, options: noAnimation({ responsive: true, maintainAspectRatio: false, + // One segment at a time. 'nearest' rather than 'point' + // because intersect would make the thin slices unhittable — + // GRP_DATA is under a pixel tall on most days, so requiring + // the cursor to land inside it hides the very rows a reader + // most needs the tooltip for. This way the column picks the + // segment whose band the cursor is closest to. + interaction: { mode: 'nearest', intersect: false, axis: 'xy' }, plugins: { - legend: { display: false }, + legend: { + position: 'bottom', + labels: { + color: colors.muted, + boxWidth: 10, + boxHeight: 10, + padding: 8, + font: { size: 10 }, + usePointStyle: true, + pointStyle: 'rect', + }, + }, tooltip: { + filter: (ctx) => ctx.parsed.y !== null && ctx.parsed.y > 0, callbacks: { - label: (ctx) => (ctx.parsed.y === null - ? 'no data for this day' - : ctx.parsed.y.toFixed(1) + '% of adverts'), + // Room to spell both readings out now that only + // the hovered type is described. + label: (ctx) => { + const entry = (ctx.dataset.rawCounts || [])[ctx.dataIndex]; + const name = ctx.dataset.label; + const slice = ctx.parsed.y.toFixed(1) + '% of the day\'s packets'; + if (!entry) return [' ' + name, ' ' + slice]; + const adoption = entry.total + ? (entry.mb / entry.total * 100).toFixed(1) + + '% of all ' + name + : 'no ' + name + ' traffic'; + return [ + ' ' + name + ' — ' + slice, + ' ' + formatNumber(entry.mb) + ' of ' + + formatNumber(entry.total) + ' packets · ' + adoption, + ]; + }, + // The bar's own height, so the segment reads + // against the whole it is part of. + footer: (items) => { + if (!items.length) return ''; + const i = items[0].dataIndex; + return 'Whole bar: ' + barHeights[i].toFixed(1) + + '% multibyte (' + formatNumber(dayMultibyte[i]) + + ' of ' + formatNumber(dayPackets[i]) + ')'; + }, }, }, }, scales: { - x: { ticks: { color: colors.muted, font: { size: 10 }, maxTicksLimit: 8 }, grid: { display: false } }, + x: { + stacked: true, + ticks: { color: colors.muted, font: { size: 10 }, maxTicksLimit: 8 }, + grid: { display: false }, + }, y: { + stacked: true, min: 0, - max: 100, - ticks: { color: colors.muted, font: { size: 10 }, callback: (v) => v + '%' }, + max: axisMax, + ticks: { + // Gridlines on the same step the ceiling is + // rounded to, so every label is a round number + // and the top one is the ceiling itself. + stepSize: axisMax <= 40 ? AXIS_STEP_PCT : undefined, + maxTicksLimit: 8, + color: colors.muted, + font: { size: 10 }, + callback: (v) => v + '%', + }, grid: { color: colors.grid }, }, }, @@ -1062,10 +1228,15 @@ 'them in that window on a path with multibyte hops; or, for repeaters and room ' + 'servers, their public key prefix matches a hop prefix from a multibyte advert path.', 'multibyte-trend-info': - 'Share of daily adverts from nodes classified as multibyte capable. ' + - 'Classification is only knowable as of now, so only the last few days are ' + - 'recomputed and older days are frozen at the value recorded then. Days before ' + - 'the rollup existed have no value and render as a gap, not as zero.', + 'Each bar\'s height is the share of that day\'s packets that arrived on a path ' + + 'with 2- or 3-byte hop hashes, divided by the payload type that carried them. A ' + + 'segment is that type\'s share of the whole day\'s traffic, not of its own ' + + 'packets — hover for both figures. The axis tops out at the tallest bar rounded ' + + 'up to the next 5%, so it is not comparable between screenshots taken on ' + + 'different days. Counted from the packet stream as each day is rolled up: the ' + + 'stream itself is pruned within days, so the chart accumulates forward and days ' + + 'before it existed are blank rather than zero. Packets whose encoding has not ' + + 'been classified yet are left out rather than counted as single-byte.', 'delta-info': 'Change compares the last complete calendar day against the day before it — ' + 'not a rolling 24 hours. The headline number above it is a rolling 24-hour count.', diff --git a/modules/web_viewer/templates/index.html b/modules/web_viewer/templates/index.html index 2db37c0..4a1d393 100644 --- a/modules/web_viewer/templates/index.html +++ b/modules/web_viewer/templates/index.html @@ -199,7 +199,7 @@
- Multibyte share of adverts + Multibyte Share of Packets
+

+ Share of each day's packets that took a multibyte path, by payload type +

+ aria-label="Daily multibyte packets by payload type, last 30 days">
diff --git a/tests/test_dashboard_stats.py b/tests/test_dashboard_stats.py index 6927406..7d2a712 100644 --- a/tests/test_dashboard_stats.py +++ b/tests/test_dashboard_stats.py @@ -569,6 +569,189 @@ class TestPacketDimensions: assert coverage["packets_with_dims"] == 2 +class TestPacketEncodingTrend: + """The per-payload-type multibyte share charted on the dashboard. + + packet_stream is pruned within days while the chart spans thirty, so each + day's split is written as that day is rolled up and can never be recovered + afterwards — every case here is about what gets frozen into the row. + """ + + @staticmethod + def _insert(conn, payload_type, bytes_per_hop, count, *, age_seconds=60.0): + conn.executemany( + "INSERT INTO packet_stream (timestamp, data, type, route_type_name, " + "payload_type_name, bytes_per_hop) VALUES (?, '{}', 'packet', 'FLOOD', ?, ?)", + [(time.time() - age_seconds, payload_type, bytes_per_hop)] * count, + ) + + @staticmethod + def _stored_split(viewer) -> dict: + row = _rollup(viewer, local_date_str()) + return json.loads(row["packet_type_encoding"]) + + def test_split_is_recorded_per_payload_type(self, viewer): + with sqlite3.connect(viewer.db_path) as conn: + self._insert(conn, "GRP_TXT", 3, 3) + self._insert(conn, "GRP_TXT", 1, 1) + self._insert(conn, "TXT_MSG", 2, 1) + self._insert(conn, "TXT_MSG", 1, 3) + _refresh(viewer) + + assert self._stored_split(viewer) == { + "GRP_TXT": {"mb": 3, "total": 4}, + "TXT_MSG": {"mb": 1, "total": 4}, + } + + def test_series_reports_one_share_per_type(self, viewer): + with sqlite3.connect(viewer.db_path) as conn: + self._insert(conn, "REQ", 2, 1) + self._insert(conn, "REQ", 1, 3) + _refresh(viewer) + + with viewer.app.test_client() as client: + payload = client.get( + "/api/dashboard/series?metric=multibyte_share_req&days=30" + ).get_json() + assert payload["is_ratio"] is True + assert payload["points"][-1] == { + "date": local_date_str(), + "value": 25.0, + "complete": False, + } + + def test_undimensioned_packets_leave_the_ratio_alone(self, viewer): + """They are not evidence of single-byte routing, only of pending work. + + Counting them on the denominator invents a dip in whichever type the + backfill has not reached yet, which reads as a change in the mesh. + """ + with sqlite3.connect(viewer.db_path) as conn: + self._insert(conn, "PATH", 2, 1) + self._insert(conn, "PATH", 1, 1) + conn.execute( + "INSERT INTO packet_stream (timestamp, data, type, payload_type_name) " + "VALUES (?, '{}', 'packet', 'PATH')", + (time.time(),), + ) + _refresh(viewer) + + assert self._stored_split(viewer)["PATH"] == {"mb": 1, "total": 2} + + def test_untracked_types_roll_into_other(self, viewer): + """The tail is summed, not dropped: it is part of the denominator. + + The chart has eight colour slots and ACK, TRACE and unmapped ordinals + like 'Type11' are not among them — but bar heights are a share of the + day's whole traffic, so discarding those packets would inflate every + bar rather than simply omitting a category. + """ + with sqlite3.connect(viewer.db_path) as conn: + self._insert(conn, "ACK", 2, 3) + self._insert(conn, "Type11", 1, 4) + self._insert(conn, "ANON_REQ", 2, 1) + _refresh(viewer) + + assert self._stored_split(viewer) == { + "ANON_REQ": {"mb": 1, "total": 1}, + "OTHER": {"mb": 3, "total": 7}, + } + + def test_bar_height_matches_the_days_multibyte_share(self, viewer): + """Segments over the day-wide denominator must sum to the real share.""" + with sqlite3.connect(viewer.db_path) as conn: + self._insert(conn, "GRP_TXT", 2, 2) # multibyte + self._insert(conn, "GRP_TXT", 1, 6) # single-byte + self._insert(conn, "ACK", 3, 1) # multibyte, rolls into OTHER + self._insert(conn, "ACK", 1, 1) + _refresh(viewer) + + split = self._stored_split(viewer) + packets = sum(entry["total"] for entry in split.values()) + multibyte = sum(entry["mb"] for entry in split.values()) + assert (packets, multibyte) == (10, 3) + # What the client draws: each segment over the day's whole traffic. + assert multibyte / packets * 100 == 30.0 + + def test_a_type_with_no_traffic_reads_as_a_gap(self, viewer): + with sqlite3.connect(viewer.db_path) as conn: + self._insert(conn, "GRP_DATA", 2, 1) + _refresh(viewer) + + with viewer.app.test_client() as client: + payload = client.get( + "/api/dashboard/series?metric=multibyte_share_response&days=30" + ).get_json() + assert payload["points"], "the rollup rows exist; only the value is absent" + assert all(point["value"] is None for point in payload["points"]) + + def test_an_early_prune_does_not_erase_a_recorded_day(self, viewer): + """A day still inside the retention window can lose its rows anyway. + + Recomputing it as an empty split would overwrite the only copy of that + day's share, so an empty result has to mean "cannot say" instead. + """ + with sqlite3.connect(viewer.db_path) as conn: + self._insert(conn, "GRP_TXT", 2, 2) + _refresh(viewer) + recorded = self._stored_split(viewer) + + with sqlite3.connect(viewer.db_path) as conn: + conn.execute("DELETE FROM packet_stream") + _refresh(viewer) + + assert self._stored_split(viewer) == recorded + + def test_summary_carries_raw_counts_for_the_stack(self, viewer): + """The stacked bars need a shared denominator, so counts are sent. + + Eight percentages each taken over their own type's traffic cannot be + restacked into a composition — the day's multibyte total is not + recoverable from them. + """ + with sqlite3.connect(viewer.db_path) as conn: + self._insert(conn, "GRP_TXT", 3, 3) + self._insert(conn, "GRP_TXT", 1, 1) + self._insert(conn, "TXT_MSG", 2, 1) + _refresh(viewer) + + with viewer.app.test_client() as client: + payload = client.get("/api/dashboard/summary").get_json() + today = [day for day in payload["packet_encoding"] if day["date"] == local_date_str()] + assert len(today) == 1 + assert today[0]["types"] == { + "GRP_TXT": {"mb": 3, "total": 4}, + "TXT_MSG": {"mb": 1, "total": 1}, + } + + def test_summary_omits_the_per_type_share_series(self, viewer): + """Shipping both forms would send the same thirty days twice.""" + with sqlite3.connect(viewer.db_path) as conn: + self._insert(conn, "REQ", 2, 1) + _refresh(viewer) + + with viewer.app.test_client() as client: + payload = client.get("/api/dashboard/summary").get_json() + assert not [name for name in payload["series"] if name.startswith("multibyte_share_")] + assert "multibyte_share" in payload["series"], "the advert share still sparklines" + # Still addressable one at a time for anything plotting a single type. + with viewer.app.test_client() as client: + single = client.get("/api/dashboard/series?metric=multibyte_share_req").get_json() + assert single["is_ratio"] is True + + def test_days_with_no_split_survive_as_empty_slots(self, viewer): + """The x-axis is every rollup day; a blank one must not shift the rest.""" + with sqlite3.connect(viewer.db_path) as conn: + self._insert(conn, "PATH", 2, 1) + _refresh(viewer) + + with viewer.app.test_client() as client: + payload = client.get("/api/dashboard/summary").get_json() + dates = [day["date"] for day in payload["packet_encoding"]] + assert dates == sorted(dates), "oldest first, so bars read left to right" + assert any(day["types"] == {} for day in payload["packet_encoding"]) + + class TestDerivedWindows: @pytest.mark.parametrize(