diff --git a/modules/web_viewer/app.py b/modules/web_viewer/app.py index 3b7464a..c677bd2 100644 --- a/modules/web_viewer/app.py +++ b/modules/web_viewer/app.py @@ -4106,6 +4106,53 @@ class BotDataViewer: """) stats['contacts_7d'] = cursor.fetchone()[0] + # Contacts heard in 7d with multibyte path evidence. Scope observed_paths to 7d so + # the pie chart matches "last 7 days" (lifetime paths + stale out_bytes_per_hop + # otherwise inflated the percentage). + stats['contacts_7d_multibyte_path'] = 0 + multibyte_chunks: set[str] = set() + mb_advert_pks: set[str] = set() + if 'observed_paths' in tables: + try: + multibyte_chunks = self._collect_multibyte_hop_chunks( + cursor, recent_days=7 + ) + # Use date() — julianday(iso8601) often returns NULL for Python isoformat() strings + cursor.execute( + """ + SELECT DISTINCT public_key FROM observed_paths + WHERE packet_type = 'advert' AND public_key IS NOT NULL + AND bytes_per_hop IN (2, 3) + AND date(last_seen) >= date('now', '-7 days') + """ + ) + mb_advert_pks = { + row["public_key"] for row in cursor.fetchall() if row["public_key"] + } + except Exception as e: + self.logger.debug(f"Could not load multibyte path sets for 7d stats: {e}") + try: + cursor.execute( + """ + SELECT public_key, role, out_bytes_per_hop + FROM complete_contact_tracking + WHERE last_heard > datetime('now', '-7 days') + """ + ) + mb_7d = 0 + for row in cursor.fetchall(): + if self._contact_has_multibyte_path_evidence( + row["public_key"], + row["role"], + row["out_bytes_per_hop"], + mb_advert_pks, + multibyte_chunks, + ): + mb_7d += 1 + stats['contacts_7d_multibyte_path'] = mb_7d + except Exception as e: + self.logger.debug(f"Could not compute contacts_7d_multibyte_path: {e}") + cursor.execute(""" SELECT COUNT(*) FROM complete_contact_tracking WHERE is_currently_tracked = 1 @@ -4137,6 +4184,37 @@ class BotDataViewer: """) stats['unique_device_types'] = cursor.fetchone()[0] + # Incoming packets (packet_stream): multibyte path share, last 7 days (decoded bytes_per_hop) + stats['incoming_packets_7d'] = 0 + stats['incoming_packets_7d_multibyte_path'] = 0 + if 'packet_stream' in tables: + try: + cutoff_ts = time.time() - 7 * 86400 + cursor.execute( + """ + SELECT COUNT(*) FROM packet_stream + WHERE type = ? AND timestamp > ? + """, + ("packet", cutoff_ts), + ) + stats['incoming_packets_7d'] = cursor.fetchone()[0] or 0 + mb_pk = 0 + try: + cursor.execute( + """ + SELECT COUNT(*) FROM packet_stream + WHERE type = ? AND timestamp > ? + AND CAST(json_extract(data, '$.bytes_per_hop') AS INTEGER) IN (2, 3) + """, + ("packet", cutoff_ts), + ) + mb_pk = cursor.fetchone()[0] or 0 + except sqlite3.OperationalError: + mb_pk = self._count_multibyte_packets_from_stream_json(cursor, cutoff_ts) + stats['incoming_packets_7d_multibyte_path'] = mb_pk + except Exception as e: + self.logger.debug(f"Could not compute incoming_packets_7d multibyte stats: {e}") + # Advertisement statistics using daily tracking table if 'daily_stats' in tables: # Total advertisements (all time) @@ -4660,6 +4738,181 @@ class BotDataViewer: if conn: conn.close() + @staticmethod + def _chunks_from_multibyte_path_hex(path_hex: str, bytes_per_hop: int) -> list[str]: + """Split path hex into per-hop segments for 2- or 3-byte hop encoding.""" + if not path_hex or bytes_per_hop not in (2, 3): + return [] + step = bytes_per_hop * 2 + out: list[str] = [] + for i in range(0, len(path_hex), step): + seg = path_hex[i : i + step] + if len(seg) == step: + out.append(seg.lower()) + return out + + def _count_multibyte_packets_from_stream_json(self, cursor, cutoff_ts: float) -> int: + """Count packet_stream rows (type=packet) since cutoff with bytes_per_hop in (2, 3). JSON parse fallback.""" + import json + + n = 0 + try: + cursor.execute( + """ + SELECT data FROM packet_stream + WHERE type = ? AND timestamp > ? + """, + ("packet", cutoff_ts), + ) + for row in cursor.fetchall(): + raw = row["data"] + if not raw: + continue + try: + d = json.loads(raw) + except (json.JSONDecodeError, TypeError): + continue + bph = d.get("bytes_per_hop") + try: + bph_i = int(bph) if bph is not None else None + except (TypeError, ValueError): + bph_i = None + if bph_i in (2, 3): + n += 1 + except Exception as e: + self.logger.debug(f"packet_stream JSON scan for multibyte: {e}") + return n + + def _collect_multibyte_hop_chunks( + self, cursor, recent_days: Optional[int] = None + ) -> set[str]: + """Hop prefixes from multibyte paths in observed_paths (for repeater/room pubkey matching). + + If ``recent_days`` is set (e.g. 7), only paths whose ``last_seen`` falls within that + window are used. Default (None) keeps full history — used by the contacts API badge. + Dashboard 7d stats pass ``recent_days=7`` so percentages match the chart title. + """ + chunks: set[str] = set() + try: + extra = "" + if recent_days is not None: + d = max(1, min(int(recent_days), 366)) + extra = f" AND date(last_seen) >= date('now', '-{d} days')" + cursor.execute( + f""" + SELECT path_hex, bytes_per_hop FROM observed_paths + WHERE bytes_per_hop IN (2, 3) AND path_hex IS NOT NULL AND length(path_hex) > 0 + {extra} + """ + ) + for row in cursor.fetchall(): + ph = row["path_hex"] + bph = row["bytes_per_hop"] + try: + bph_i = int(bph) if bph is not None else 0 + except (TypeError, ValueError): + bph_i = 0 + for c in self._chunks_from_multibyte_path_hex(ph, bph_i): + if len(c) in (4, 6): + chunks.add(c) + except Exception as e: + self.logger.debug(f"Could not load multibyte hop chunks: {e}") + return chunks + + def _compute_path_encoding_badge( + self, + row: Any, + all_paths: list[dict[str, Any]], + multibyte_hop_chunks: set[str], + ) -> Optional[str]: + """Return 'multibyte', 'one_byte', or None for contacts path-encoding badge.""" + pk = row["public_key"] or "" + role = (row["role"] or "").lower() + obph_raw = row["out_bytes_per_hop"] + obph: Optional[int] + try: + obph = int(obph_raw) if obph_raw is not None else None + except (TypeError, ValueError): + obph = None + if obph is not None and obph not in (1, 2, 3): + obph = None + + out_path_len = row["out_path_len"] + if out_path_len is None: + out_path_len = -1 + try: + out_path_len = int(out_path_len) + except (TypeError, ValueError): + out_path_len = -1 + + advert_count = row["advert_count"] or 0 + + def norm_bph(b: Any) -> int: + if b is None: + return 1 + try: + i = int(b) + return i if i in (1, 2, 3) else 1 + except (TypeError, ValueError): + return 1 + + # Multibyte evidence + if obph in (2, 3): + return "multibyte" + for p in all_paths: + if norm_bph(p.get("bytes_per_hop")) in (2, 3): + return "multibyte" + if role in ("repeater", "roomserver") and pk: + pk_low = pk.lower() + for chunk in multibyte_hop_chunks: + if pk_low.startswith(chunk): + return "multibyte" + + # One-byte: positive signal and no multibyte observation + has_signal = bool( + advert_count > 0 or len(all_paths) > 0 or out_path_len >= 0 + ) + if not has_signal: + return None + + if obph is not None and obph != 1: + return None + for p in all_paths: + if norm_bph(p.get("bytes_per_hop")) != 1: + return None + + return "one_byte" + + def _contact_has_multibyte_path_evidence( + self, + public_key: str, + role: Optional[str], + out_bytes_per_hop: Any, + multibyte_advert_public_keys: set[str], + multibyte_hop_chunks: set[str], + ) -> bool: + """Multibyte detection for dashboard 7d stats (observed_paths scoped by date in SQL).""" + pk = public_key or "" + role_l = (role or "").lower() + obph: Optional[int] + try: + obph = int(out_bytes_per_hop) if out_bytes_per_hop is not None else None + except (TypeError, ValueError): + obph = None + if obph is not None and obph not in (1, 2, 3): + obph = None + + if obph in (2, 3): + return True + if pk and pk in multibyte_advert_public_keys: + return True + if role_l in ("repeater", "roomserver") and pk: + pk_low = pk.lower() + for chunk in multibyte_hop_chunks: + if pk_low.startswith(chunk): + return True + return False + def _get_tracking_data(self, since='30d'): """Get contact tracking data. since: 24h, 7d, 30d, 90d, or all (heard in that window).""" conn = None @@ -4724,8 +4977,11 @@ class BotDataViewer: ORDER BY c.last_heard DESC """, params) + main_rows = cursor.fetchall() + multibyte_hop_chunks = self._collect_multibyte_hop_chunks(cursor) + tracking = [] - for row in cursor.fetchall(): + for row in main_rows: # Parse raw advertisement data if available raw_advert_data_parsed = None if row['raw_advert_data']: @@ -4768,6 +5024,10 @@ class BotDataViewer: 'last_seen': paths_last_seen[i] if i < len(paths_last_seen) and paths_last_seen[i] else None }) + path_encoding_badge = self._compute_path_encoding_badge( + row, all_paths, multibyte_hop_chunks + ) + tracking.append({ 'user_id': row['public_key'], 'username': row['name'], @@ -4794,7 +5054,8 @@ class BotDataViewer: 'out_path': row['out_path'] if row['out_path'] is not None else '', 'out_path_len': row['out_path_len'] if row['out_path_len'] is not None else -1, 'out_bytes_per_hop': row['out_bytes_per_hop'] if row['out_bytes_per_hop'] is not None else None, - 'all_paths': all_paths + 'all_paths': all_paths, + 'path_encoding_badge': path_encoding_badge, }) # Get server statistics for daily tracking using direct database queries diff --git a/modules/web_viewer/templates/contacts.html b/modules/web_viewer/templates/contacts.html index 28f4154..87a71a5 100644 --- a/modules/web_viewer/templates/contacts.html +++ b/modules/web_viewer/templates/contacts.html @@ -177,6 +177,47 @@ .contacts-mobile-sort-row .flex-grow-1 { min-width: 0; } + + /* Path encoding (bytes/hop) — neutral, non-judgmental pairing */ + .path-encoding-badge { + font-size: 0.65rem; + font-weight: 500; + line-height: 1.2; + padding: 0.2em 0.45em; + } + /* Multibyte: cool cyan (distinct from warm 1-byte) */ + .path-encoding-badge-multibyte { + background-color: #cffafe; + color: #0c4a6e; + border: 1px solid #0ea5e9; + } + /* 1-byte: warm stone (hue-shifted from gray so it reads clearly vs cyan) */ + .path-encoding-badge-onebyte { + background-color: #faf7f2; + color: #44403c; + border: 1px solid #a8a29e; + } + /* Dark mode: same cool vs warm split, higher saturation on borders */ + [data-theme="dark"] .path-encoding-badge-multibyte { + background-color: #134e6a; + color: #bae6fd; + border-color: #38bdf8; + } + [data-theme="dark"] .path-encoding-badge-onebyte { + background-color: #3f3a36; + color: #f5f0eb; + border-color: #a8a29e; + } + + .tooltip.multibyte-capable-hint .tooltip-inner { + max-width: min(22rem, 92vw); + text-align: left; + } + .path-encoding-info-btn:hover, + .path-encoding-info-btn:focus { + text-decoration: none; + color: var(--bs-info, #0dcaf0) !important; + } {% endblock %} @@ -421,7 +462,16 @@ SNR - Hops + + Hops + + First Heard @@ -840,7 +890,10 @@ class ModernContactsManager {
${this.formatTimestamp(contact.first_heard)} · ${this.formatTimeAgo(contact.last_seen)}
+
${this.formatHops(contact)}
+ ${this.formatPathEncodingBadge(contact)} +
${contact.advert_count || 0} adverts @@ -904,7 +957,7 @@ class ModernContactsManager { ${this.formatLocation(contact)} ${this.formatDistance(contact)} ${this.formatSignal(contact)} - ${this.formatHops(contact)} +
${this.formatHops(contact)}${this.formatPathEncodingBadge(contact)}
${this.formatTimestamp(contact.first_heard)} ${this.formatTimeAgo(contact.last_seen)} ${contact.advert_count || 0} @@ -1446,6 +1499,18 @@ class ModernContactsManager { return `${hopCount}`; } } + + formatPathEncodingBadge(contact) { + const v = contact.path_encoding_badge; + if (!v) return ''; + if (v === 'multibyte') { + return 'Multibyte'; + } + if (v === 'one_byte') { + return '1-byte only'; + } + return ''; + } setupPathTooltips() { // Create a global custom tooltip element if it doesn't exist @@ -2463,12 +2528,32 @@ function exportData(dataset, fmt) { document.body.removeChild(a); } +function initContactsMultibyteBadgeTooltip() { + if (typeof bootstrap === 'undefined' || !bootstrap.Tooltip) return; + const listTip = + 'Multibyte vs 1-byte only: multibyte if stored path encoding (out bytes per hop) is 2 or 3, ' + + 'any loaded advert path shows multibyte hops (2–3 bytes per hop), ' + + 'or (repeaters / room servers) their public key prefix matches a hop on a multibyte path. ' + + 'Loaded paths can include more history than the 7-day dashboard chart.'; + const el = document.getElementById('contacts-list-multibyte-badge-info'); + if (el) { + const existing = bootstrap.Tooltip.getInstance(el); + if (existing) existing.dispose(); + new bootstrap.Tooltip(el, { + title: listTip, + html: false, + customClass: 'multibyte-capable-hint', + }); + } +} + // Initialize contacts manager when page loads document.addEventListener('DOMContentLoaded', () => { window.contactsManager = new ModernContactsManager(); window.contactsManager.setupPurgeModal(); - }); - + initContactsMultibyteBadgeTooltip(); +}); + +{% endblock %} + {% block content %}
@@ -60,50 +91,130 @@
- -
-
-
-
- Active Contacts + +
+
+
+
+
+ Active Contacts +
+
+
+
+

0

+ 24h +
+
+

0

+ 7d +
+
+

0

+ All +
+
+
-
-
-
-

0

- 24h -
-
-

0

- 7d -
-
-

0

- All +
+
+ Devices & cache +
+
+
+
+

0

+ Device types +
+
+

0

+ Roles +
+
+

0

+ Active cache +
- -
-
-
- Network Health +
+
+
+
+ Network Health +
+
+
+
+

0

+ Avg Hops +
+
+

0

+ Max Hops +
+
+

0

+ Tracked +
+
+
-
-
-
-

0

- Avg Hops +
+
+ Geographic Coverage +
+
+
+
+

0

+ Countries +
+
+

0

+ States +
+
+

0

+ Cities +
-
-

0

- Max Hops -
-
-

0

- Tracked +
+
+
+
+
+
+
+
+ Path encoding (7d) +
+
+
+
+

+ Contacts (last 7 days) + +

+
+ +
+

+
+
+

Incoming packets (last 7 days)

+
+ +
+

+
@@ -111,60 +222,9 @@
- +
-
-
- Devices & cache -
-
-
-
-

0

- Device types -
-
-

0

- Roles -
-
-

0

- Active cache entries -
-
-
-
-
-
- - -
-
-
-
- Geographic Coverage -
-
-
-
-

0

- Countries -
-
-

0

- States -
-
-

0

- Cities -
-
-
-
-
- -
Channel Statistics @@ -440,6 +500,8 @@ class ModernDashboard { this.botStartTime = null; this.updateInterval = null; this.uptimeInterval = null; + this.pathEncodingPieChart = null; + this.incomingPacketsPieChart = null; this.initializeDashboard(); } @@ -583,8 +645,195 @@ class ModernDashboard { this.updateElement('bot-reply-rate-7d', data.bot_reply_rate_7d !== undefined ? `${data.bot_reply_rate_7d}%` : '0%'); this.updateElement('bot-reply-rate-30d', data.bot_reply_rate_30d !== undefined ? `${data.bot_reply_rate_30d}%` : '0%'); + this.updatePathEncodingPieChart(data); + this.updateIncomingPacketsPieChart(data); + // Analytics are loaded separately with time window selectors } + + updatePathEncodingPieChart(data) { + const canvas = document.getElementById('pathEncodingPieChart'); + const summaryEl = document.getElementById('path-encoding-pie-summary'); + if (!canvas || typeof Chart === 'undefined') { + return; + } + const total = Number(data.contacts_7d) || 0; + let mb = data.contacts_7d_multibyte_path; + mb = Number(mb); + if (!Number.isFinite(mb)) { + mb = 0; + } + mb = Math.max(0, Math.min(mb, total)); + const other = Math.max(0, total - mb); + + if (this.pathEncodingPieChart) { + this.pathEncodingPieChart.destroy(); + this.pathEncodingPieChart = null; + } + + const cs = getComputedStyle(document.documentElement); + const legendColor = (cs.getPropertyValue('--text-muted') || '#6c757d').trim(); + const mbColor = '#0891b2'; + const otherColor = '#a8a29e'; + const emptyColor = (cs.getPropertyValue('--bg-tertiary') || '#dee2e6').trim(); + + if (total === 0) { + if (summaryEl) { + summaryEl.textContent = 'No contacts in the last 7 days.'; + } + this.pathEncodingPieChart = new Chart(canvas.getContext('2d'), { + type: 'doughnut', + data: { + labels: ['—'], + datasets: [{ + data: [1], + backgroundColor: [emptyColor], + borderWidth: 0, + }], + }, + options: { + responsive: true, + maintainAspectRatio: true, + plugins: { legend: { display: false }, tooltip: { enabled: false } }, + cutout: '55%', + }, + }); + return; + } + + const pct = Math.round((mb / total) * 1000) / 10; + if (summaryEl) { + summaryEl.textContent = `${pct}% multibyte (${mb} of ${total})`; + } + + this.pathEncodingPieChart = new Chart(canvas.getContext('2d'), { + type: 'doughnut', + data: { + labels: ['Multibyte paths', 'Other'], + datasets: [{ + data: [mb, other], + backgroundColor: [mbColor, otherColor], + borderWidth: 0, + }], + }, + options: { + responsive: true, + maintainAspectRatio: true, + plugins: { + legend: { + position: 'bottom', + labels: { + color: legendColor, + boxWidth: 12, + font: { size: 11 }, + }, + }, + tooltip: { + callbacks: { + label: (ctx) => { + const n = ctx.raw; + const p = total ? Math.round((n / total) * 1000) / 10 : 0; + return ` ${n} (${p}%)`; + }, + }, + }, + }, + cutout: '55%', + }, + }); + } + + updateIncomingPacketsPieChart(data) { + const canvas = document.getElementById('incomingPacketsPieChart'); + const summaryEl = document.getElementById('incoming-packets-pie-summary'); + if (!canvas || typeof Chart === 'undefined') { + return; + } + const total = Number(data.incoming_packets_7d) || 0; + let mb = data.incoming_packets_7d_multibyte_path; + mb = Number(mb); + if (!Number.isFinite(mb)) { + mb = 0; + } + mb = Math.max(0, Math.min(mb, total)); + const other = Math.max(0, total - mb); + + if (this.incomingPacketsPieChart) { + this.incomingPacketsPieChart.destroy(); + this.incomingPacketsPieChart = null; + } + + const cs = getComputedStyle(document.documentElement); + const legendColor = (cs.getPropertyValue('--text-muted') || '#6c757d').trim(); + const mbColor = '#0891b2'; + const otherColor = '#a8a29e'; + const emptyColor = (cs.getPropertyValue('--bg-tertiary') || '#dee2e6').trim(); + + if (total === 0) { + if (summaryEl) { + summaryEl.textContent = 'No incoming packets in the last 7 days.'; + } + this.incomingPacketsPieChart = new Chart(canvas.getContext('2d'), { + type: 'doughnut', + data: { + labels: ['—'], + datasets: [{ + data: [1], + backgroundColor: [emptyColor], + borderWidth: 0, + }], + }, + options: { + responsive: true, + maintainAspectRatio: true, + plugins: { legend: { display: false }, tooltip: { enabled: false } }, + cutout: '55%', + }, + }); + return; + } + + const pct = Math.round((mb / total) * 1000) / 10; + if (summaryEl) { + summaryEl.textContent = `${pct}% multibyte (${mb} of ${total})`; + } + + this.incomingPacketsPieChart = new Chart(canvas.getContext('2d'), { + type: 'doughnut', + data: { + labels: ['Multibyte paths', 'Other'], + datasets: [{ + data: [mb, other], + backgroundColor: [mbColor, otherColor], + borderWidth: 0, + }], + }, + options: { + responsive: true, + maintainAspectRatio: true, + plugins: { + legend: { + position: 'bottom', + labels: { + color: legendColor, + boxWidth: 12, + font: { size: 11 }, + }, + }, + tooltip: { + callbacks: { + label: (ctx) => { + const n = ctx.raw; + const p = total ? Math.round((n / total) * 1000) / 10 : 0; + return ` ${n} (${p}%)`; + }, + }, + }, + }, + cutout: '55%', + }, + }); + } updateElement(elementId, value) { const element = document.getElementById(elementId); @@ -822,6 +1071,14 @@ class ModernDashboard { if (this.uptimeInterval) { clearInterval(this.uptimeInterval); } + if (this.pathEncodingPieChart) { + this.pathEncodingPieChart.destroy(); + this.pathEncodingPieChart = null; + } + if (this.incomingPacketsPieChart) { + this.incomingPacketsPieChart.destroy(); + this.incomingPacketsPieChart = null; + } } async loadConnectedClients() { @@ -884,10 +1141,30 @@ class ModernDashboard { } } +function initMultibyteCapableTooltips() { + if (typeof bootstrap === 'undefined' || !bootstrap.Tooltip) return; + const dashboardTip = + 'Contacts heard in the last 7 days are counted as multibyte capable if any apply: ' + + 'stored path encoding (out bytes per hop) is 2 or 3; ' + + 'we saw an advert from them in that window on observed paths with multibyte hops (2–3 bytes per hop); ' + + 'or, for repeaters and room servers, their public key prefix matches a hop prefix from a multibyte advert path in that window.'; + const el = document.getElementById('contacts-multibyte-chart-info'); + if (el) { + const existing = bootstrap.Tooltip.getInstance(el); + if (existing) existing.dispose(); + new bootstrap.Tooltip(el, { + title: dashboardTip, + html: false, + customClass: 'multibyte-capable-hint', + }); + } +} + // Initialize dashboard when page loads document.addEventListener('DOMContentLoaded', () => { console.log('DOM loaded, initializing dashboard...'); window.dashboard = new ModernDashboard(); + initMultibyteCapableTooltips(); }); // Clean up when page is unloaded