Merge branch 'areas-meshguide-sync'

This commit is contained in:
dborup
2026-07-28 06:31:30 +02:00
7 changed files with 316 additions and 8 deletions
+79
View File
@@ -17,6 +17,7 @@ import (
"github.com/meshcore-analyzer/dbschema"
"github.com/meshcore-analyzer/geofilter"
regionutil "github.com/meshcore-analyzer/regions"
_ "modernc.org/sqlite"
)
@@ -1668,6 +1669,84 @@ func (db *DB) GetAllObserverNeighbors() ([]AllObserverNeighborsEntry, error) {
return result, nil
}
// UnknownScopeEntry is one region-scope name that turned up in an
// observer's reported neighbor scope list but isn't part of the
// deployment's configured hashRegions -- a scope the mesh is actually
// using that CoreScope doesn't know about yet. dborup asked for this
// after seeing how cluttered a real observer's scope list can get (e.g.
// "*,#eu,#dk,#dk-sjl,#nordic,...").
type UnknownScopeEntry struct {
Scope string `json:"scope"`
// Count is the number of DISTINCT neighbors (by display name, falling
// back to pubkey) that reported this scope -- not a raw row count,
// which would double-count a neighbor seen by multiple observers.
Count int `json:"count"`
Examples []string `json:"examples"`
}
// computeUnknownScopes finds scope names present in observer-reported
// neighbor scope lists that aren't in the configured hashRegions set.
// "*" (the catch-all wildcard some firmware includes) is excluded, it's
// not a real region. Pure function over already-fetched
// AllObserverNeighborsEntry rows -- no DB access, so callers can reuse
// data they already have (handleAllObserverNeighbors does exactly that,
// no second query needed).
func computeUnknownScopes(entries []AllObserverNeighborsEntry, configuredHashRegions []string) []UnknownScopeEntry {
configured := make(map[string]bool)
for _, r := range regionutil.NormalizeNames(configuredHashRegions) {
configured[r] = true
}
neighborsByScope := make(map[string]map[string]bool)
for _, e := range entries {
if e.Scopes == nil || *e.Scopes == "" {
continue
}
label := e.NeighborPubkey
if e.NeighborName != nil && *e.NeighborName != "" {
label = *e.NeighborName
}
for _, raw := range strings.Split(*e.Scopes, ",") {
s := strings.TrimSpace(raw)
if s == "" || s == "*" {
continue
}
if !strings.HasPrefix(s, "#") {
s = "#" + s
}
if configured[s] {
continue
}
if neighborsByScope[s] == nil {
neighborsByScope[s] = make(map[string]bool)
}
neighborsByScope[s][label] = true
}
}
const maxExamples = 5
result := make([]UnknownScopeEntry, 0, len(neighborsByScope))
for scope, labelSet := range neighborsByScope {
labels := make([]string, 0, len(labelSet))
for l := range labelSet {
labels = append(labels, l)
}
sort.Strings(labels)
examples := labels
if len(examples) > maxExamples {
examples = examples[:maxExamples]
}
result = append(result, UnknownScopeEntry{Scope: scope, Count: len(labels), Examples: examples})
}
sort.Slice(result, func(i, j int) bool {
if result[i].Count != result[j].Count {
return result[i].Count > result[j].Count
}
return result[i].Scope < result[j].Scope
})
return result
}
// NeighborMetricPoint is one time-series sample of an observer<->neighbor
// direct-RF link (#1865 follow-up: the /neighbors report's snr and
// heard_secs_ago fields, previously dropped). Mirrors MetricsSample's
@@ -201,3 +201,41 @@ func TestHandleAllObserverNeighbors_ExcludesBlacklistedObserver(t *testing.T) {
t.Fatalf("expected only normal-obs's row, got %+v", body.Neighbors)
}
}
// dborup: "kan vi have en panel med scopes vi ikke kender på corescope
// som observer neighbors har fundet" -- unknownScopes surfaces
// region-scope names seen in reported neighbor scope lists that aren't
// part of the deployment's configured hashRegions.
func TestHandleAllObserverNeighbors_UnknownScopes(t *testing.T) {
srv, router := setupTestServer(t)
srv.cfg.HashRegions = []string{"dk"}
if _, err := srv.db.conn.Exec(`INSERT INTO observer_neighbors (observer_id, neighbor_pubkey, scopes, status, reported_at) VALUES
('obs1', ?, '*,#dk,#dk-storkbh', 'responded', '2026-07-28T14:00:00Z')`,
"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"); err != nil {
t.Fatalf("seed observer_neighbors: %v", err)
}
req := httptest.NewRequest("GET", "/api/observers/neighbors", nil)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
if w.Code != 200 {
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
}
var body struct {
UnknownScopes []struct {
Scope string `json:"scope"`
Count int `json:"count"`
} `json:"unknownScopes"`
}
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
t.Fatalf("decode: %v body=%s", err, w.Body.String())
}
if len(body.UnknownScopes) != 1 {
t.Fatalf("expected 1 unknown scope (#dk-storkbh; #dk is configured, * is the wildcard), got %+v", body.UnknownScopes)
}
if body.UnknownScopes[0].Scope != "#dk-storkbh" || body.UnknownScopes[0].Count != 1 {
t.Errorf("UnknownScopes[0] = %+v, want {#dk-storkbh 1}", body.UnknownScopes[0])
}
}
+12 -2
View File
@@ -139,7 +139,7 @@ func routeDescriptions() map[string]routeMeta {
"GET /api/observers/{id}/neighbors": {Summary: "Get an observer's direct (zero-hop) neighbors", Description: "Ground truth from the observer's own /neighbors firmware report (#1865) -- distinct from the packet-path-inferred neighbor graph. Empty `neighbors` (never null) and an empty `reportedAt` mean the observer has never sent a /neighbors report: opt-in firmware, unavailable on non-PSRAM hardware -- absence is normal, not a fault. Each entry's `scopes` is null unless the neighbor's OTA scope query responded (status=\"responded\"); `name`/`role` are null when the pubkey doesn't resolve to a known node. `seenViaPackets` cross-references the packet-path-inferred neighbor_edges graph: false means this firmware-confirmed neighbor has never had a resolved packet path between it and the observer, a diagnostic signal (possible coverage gap or packet loss), not itself a fault.", Tag: "observers"},
"GET /api/observers/{id}/neighbors/{pubkey}/metrics": {Summary: "Get SNR history for one observer<->neighbor direct-RF link", Description: "Raw (unaggregated) history of the snr/heard_secs_ago fields the observer's own /neighbors report carries per neighbor -- report volume per pair is inherently low so, unlike /api/observers/{id}/metrics, there is no resolution/downsampling. Defaults to the last 30 days.", Tag: "observers", QueryParams: []paramMeta{{Name: "since", Description: "RFC3339 lower bound (default: 30 days ago)", Type: "string"}, {Name: "until", Description: "RFC3339 upper bound (default: none)", Type: "string"}}},
"GET /api/observers/metrics/summary": {Summary: "Observer metrics summary", Description: "Aggregate metrics across all observers.", Tag: "observers"},
"GET /api/observers/neighbors": {Summary: "Every observer's reported direct neighbors, network-wide", Description: "Flattens /api/observers/{id}/neighbors across ALL observers into one list -- Tools > Observer Neighbors. Same per-entry semantics (scopes null unless the OTA scope query responded, seenViaPackets cross-references the packet-derived neighbor_edges graph, observer/neighbor name null when unresolved). Blacklisted observers (config.json observerBlacklist) are excluded. Empty list (not an error) when no observer has ever sent a /neighbors report.", Tag: "observers",
"GET /api/observers/neighbors": {Summary: "Every observer's reported direct neighbors, network-wide", Description: "Flattens /api/observers/{id}/neighbors across ALL observers into one list -- Tools > Observer Neighbors. Same per-entry semantics (scopes null unless the OTA scope query responded, seenViaPackets cross-references the packet-derived neighbor_edges graph, observer/neighbor name null when unresolved). Blacklisted observers (config.json observerBlacklist) are excluded. Empty list (not an error) when no observer has ever sent a /neighbors report. unknownScopes surfaces region-scope names that turned up in a reported neighbor scope list but aren't part of this deployment's configured hashRegions -- scopes the mesh is using that CoreScope doesn't know about yet.", Tag: "observers",
Response: schemaRef("AllObserverNeighborsResponse")},
// Misc
@@ -527,11 +527,21 @@ func componentSchemas() map[string]interface{} {
"reportedAt": str("RFC3339 timestamp of the /neighbors report this row came from."),
},
},
"UnknownScopeEntry": map[string]interface{}{
"type": "object",
"description": "A region-scope name seen in a reported neighbor scope list that isn't part of this deployment's configured hashRegions.",
"properties": map[string]interface{}{
"scope": str("The scope name (e.g. \"#dk-storkbh\"), always #-prefixed."),
"count": map[string]interface{}{"type": "integer", "description": "Number of distinct neighbors (by display name, falling back to pubkey) that reported this scope."},
"examples": map[string]interface{}{"type": "array", "items": map[string]interface{}{"type": "string"}, "description": "Up to 5 example neighbor display names/pubkeys that reported this scope."},
},
},
"AllObserverNeighborsResponse": map[string]interface{}{
"type": "object",
"description": "Every observer's reported direct neighbors, network-wide (Tools > Observer Neighbors). Empty (not an error) when no observer has ever sent a /neighbors report.",
"properties": map[string]interface{}{
"neighbors": map[string]interface{}{"type": "array", "items": schemaRef("AllObserverNeighborsEntry")},
"neighbors": map[string]interface{}{"type": "array", "items": schemaRef("AllObserverNeighborsEntry")},
"unknownScopes": map[string]interface{}{"type": "array", "items": schemaRef("UnknownScopeEntry"), "description": "Region-scope names observed in the wild that aren't part of this deployment's configured hashRegions, ranked by how many distinct neighbors reported them."},
},
},
}
+12 -2
View File
@@ -3326,7 +3326,10 @@ func (s *Server) handleObserverNeighbors(w http.ResponseWriter, r *http.Request)
// into each observer individually). Same fields/semantics as
// handleObserverNeighbors, just not scoped to one observer -- including
// the empty-list-not-error convention for a network with no /neighbors
// data reported yet.
// data reported yet. Also surfaces unknownScopes -- region-scope names
// that turned up in a reported neighbor scope list but aren't part of
// this deployment's configured hashRegions, computed from the SAME
// entries (no second query).
func (s *Server) handleAllObserverNeighbors(w http.ResponseWriter, r *http.Request) {
entries, err := s.db.GetAllObserverNeighbors()
if err != nil {
@@ -3342,7 +3345,14 @@ func (s *Server) handleAllObserverNeighbors(w http.ResponseWriter, r *http.Reque
}
entries = filtered
}
writeJSON(w, map[string]interface{}{"neighbors": entries})
var hashRegions []string
if s.cfg != nil {
hashRegions = s.cfg.HashRegions
}
writeJSON(w, map[string]interface{}{
"neighbors": entries,
"unknownScopes": computeUnknownScopes(entries, hashRegions),
})
}
// handleObserverNeighborMetrics serves the SNR/heard_secs_ago history for
+119
View File
@@ -0,0 +1,119 @@
package main
import "testing"
func strPtr(s string) *string { return &s }
// TestComputeUnknownScopes covers the core signal: a scope not in the
// configured hashRegions list, reported by multiple distinct neighbors,
// counted once per distinct neighbor (not once per row) and capped to 5
// examples.
func TestComputeUnknownScopes(t *testing.T) {
entries := []AllObserverNeighborsEntry{
{NeighborPubkey: "pk1", NeighborName: strPtr("Neighbor One"), Scopes: strPtr("*,#eu,#dk,#dk-storkbh")},
{NeighborPubkey: "pk2", NeighborName: strPtr("Neighbor Two"), Scopes: strPtr("#dk,#dk-storkbh")},
// Same neighbor reported again via a different observer -- must
// count once, not twice.
{NeighborPubkey: "pk1", NeighborName: strPtr("Neighbor One"), Scopes: strPtr("#dk-storkbh")},
// #dk and #eu are configured -- must not appear as unknown.
{NeighborPubkey: "pk3", NeighborName: strPtr("Neighbor Three"), Scopes: strPtr("#dk,#eu")},
}
configured := []string{"dk", "eu"} // regionutil.Normalize adds the leading #
got := computeUnknownScopes(entries, configured)
if len(got) != 1 {
t.Fatalf("got %d unknown scopes, want 1 (#dk-storkbh): %+v", len(got), got)
}
entry := got[0]
if entry.Scope != "#dk-storkbh" {
t.Errorf("Scope = %q, want #dk-storkbh", entry.Scope)
}
if entry.Count != 2 {
t.Errorf("Count = %d, want 2 (Neighbor One + Neighbor Two, deduplicated across pk1's two rows)", entry.Count)
}
if len(entry.Examples) != 2 {
t.Fatalf("got %d examples, want 2: %+v", len(entry.Examples), entry.Examples)
}
// Examples are sorted for determinism.
if entry.Examples[0] != "Neighbor One" || entry.Examples[1] != "Neighbor Two" {
t.Errorf("Examples = %+v, want [Neighbor One, Neighbor Two]", entry.Examples)
}
}
// TestComputeUnknownScopes_WildcardExcluded confirms "*" never appears as
// an unknown scope -- it's firmware's catch-all marker, not a real region.
func TestComputeUnknownScopes_WildcardExcluded(t *testing.T) {
entries := []AllObserverNeighborsEntry{
{NeighborPubkey: "pk1", Scopes: strPtr("*")},
}
got := computeUnknownScopes(entries, nil)
if len(got) != 0 {
t.Errorf("got %+v, want empty -- '*' must never surface as an unknown scope", got)
}
}
// TestComputeUnknownScopes_MissingHashPrefixNormalized confirms a scope
// entry without a leading '#' (shouldn't happen in practice, but the
// ingestor stores whatever firmware sends) still gets compared correctly
// against the #-prefixed configured set.
func TestComputeUnknownScopes_MissingHashPrefixNormalized(t *testing.T) {
entries := []AllObserverNeighborsEntry{
{NeighborPubkey: "pk1", Scopes: strPtr("dk-storkbh")}, // no leading #
}
got := computeUnknownScopes(entries, nil)
if len(got) != 1 || got[0].Scope != "#dk-storkbh" {
t.Fatalf("got %+v, want a single #dk-storkbh entry (# added)", got)
}
}
// TestComputeUnknownScopes_NilOrEmptyScopesSkipped confirms rows with no
// scope data (nil Scopes, or a nil/blank string) don't panic or produce
// spurious entries -- this is the "timeout" / "no reply" case.
func TestComputeUnknownScopes_NilOrEmptyScopesSkipped(t *testing.T) {
entries := []AllObserverNeighborsEntry{
{NeighborPubkey: "pk1", Scopes: nil},
{NeighborPubkey: "pk2", Scopes: strPtr("")},
}
got := computeUnknownScopes(entries, nil)
if len(got) != 0 {
t.Errorf("got %+v, want empty", got)
}
}
// TestComputeUnknownScopes_CapsExamplesAtFive confirms a widely-reported
// unknown scope doesn't balloon the response -- Examples caps at 5, but
// Count still reflects the true total.
func TestComputeUnknownScopes_CapsExamplesAtFive(t *testing.T) {
var entries []AllObserverNeighborsEntry
names := []string{"A", "B", "C", "D", "E", "F", "G"}
for i, n := range names {
entries = append(entries, AllObserverNeighborsEntry{
NeighborPubkey: "pk" + string(rune('0'+i)),
NeighborName: strPtr(n),
Scopes: strPtr("#widespread"),
})
}
got := computeUnknownScopes(entries, nil)
if len(got) != 1 {
t.Fatalf("got %d entries, want 1", len(got))
}
if got[0].Count != 7 {
t.Errorf("Count = %d, want 7 (true total, not capped)", got[0].Count)
}
if len(got[0].Examples) != 5 {
t.Errorf("got %d examples, want 5 (capped)", len(got[0].Examples))
}
}
// TestComputeUnknownScopes_FallsBackToPubkeyWhenNameUnresolved confirms an
// unresolved neighbor's pubkey is used as its display label (and dedupe
// key) instead of a nil-pointer panic.
func TestComputeUnknownScopes_FallsBackToPubkeyWhenNameUnresolved(t *testing.T) {
entries := []AllObserverNeighborsEntry{
{NeighborPubkey: "deadbeef00", NeighborName: nil, Scopes: strPtr("#unlisted")},
}
got := computeUnknownScopes(entries, nil)
if len(got) != 1 || len(got[0].Examples) != 1 || got[0].Examples[0] != "deadbeef00" {
t.Fatalf("got %+v, want a single entry with example 'deadbeef00'", got)
}
}
+34
View File
@@ -10,6 +10,7 @@
var container = null;
var allRows = [];
var unknownScopes = [];
var filterText = '';
var sortState = { col: 'observer', dir: 'asc' };
@@ -20,6 +21,7 @@
function init(app) {
container = app;
allRows = [];
unknownScopes = [];
filterText = '';
sortState = { col: 'observer', dir: 'asc' };
@@ -27,6 +29,7 @@
'<div class="tools-landing" style="max-width:1100px">' +
'<h2>Observer Neighbors</h2>' +
'<p class="help-text">Every observer\'s firmware-reported direct (zero-hop) neighbors, network-wide. Ground truth from each observer\'s own /neighbors report -- distinct from the packet-path-inferred neighbor graph. Click a column header to sort.</p>' +
'<div id="obs-nb-unknown-scopes-wrap"></div>' +
'<div style="margin:12px 0"><input type="text" id="obs-nb-filter" class="input" placeholder="Filter by observer or neighbor name…" style="max-width:320px"></div>' +
'<div id="obs-nb-status" class="text-muted" style="font-size:12px;margin-bottom:8px"></div>' +
'<div id="obs-nb-table-wrap" class="table-fluid-wrap"></div>' +
@@ -46,6 +49,7 @@
function destroy() {
container = null;
allRows = [];
unknownScopes = [];
}
function load() {
@@ -59,6 +63,8 @@
})
.then(function (data) {
allRows = (data && Array.isArray(data.neighbors)) ? data.neighbors : [];
unknownScopes = (data && Array.isArray(data.unknownScopes)) ? data.unknownScopes : [];
renderUnknownScopes();
renderTable();
})
.catch(function (e) {
@@ -67,6 +73,34 @@
});
}
// "Scopes CoreScope doesn't know about yet" -- region-scope names seen
// in reported neighbor scope lists that aren't part of this
// deployment's configured hashRegions (dborup: "kan vi have en panel
// med scopes vi ikke kender på corescope som observer neighbors har
// fundet"). Computed server-side (computeUnknownScopes, db.go) from the
// same rows this page already fetches -- no second request.
function renderUnknownScopes() {
var wrap = document.getElementById('obs-nb-unknown-scopes-wrap');
if (!wrap) return;
if (unknownScopes.length === 0) {
wrap.innerHTML = '';
return;
}
var rows = unknownScopes.map(function (u) {
return '<tr>' +
'<td><code>' + escapeHtml(u.scope) + '</code></td>' +
'<td style="text-align:right">' + u.count.toLocaleString() + '</td>' +
'<td class="text-muted" style="font-size:0.85em">' + (u.examples || []).map(escapeHtml).join(', ') + '</td>' +
'</tr>';
}).join('');
wrap.innerHTML =
'<div class="analytics-card" style="margin:12px 0">' +
'<h3 style="margin:0 0 4px">Scopes CoreScope Doesn\'t Know About Yet (' + unknownScopes.length.toLocaleString() + ')</h3>' +
'<p class="text-muted" style="margin:0 0 8px;font-size:0.85em">Region-scope names reported in the wild by neighbors\' OTA scope query, but not part of this deployment\'s configured regions. Might be worth adding to config -- or just neighboring mesh communities using their own naming.</p>' +
'<table class="data-table"><thead><tr><th>Scope</th><th style="text-align:right">Seen By</th><th>Example Neighbors</th></tr></thead><tbody>' + rows + '</tbody></table>' +
'</div>';
}
function sortValue(row, col) {
switch (col) {
case 'observer': return (row.observerName || row.observerId || '').toLowerCase();
+22 -4
View File
@@ -32,7 +32,7 @@ function makeRow(overrides) {
}, overrides);
}
function createSandbox(rows) {
function createSandbox(rows, unknownScopesFixture) {
const docStore = {};
const listeners = {};
function fakeEl(id) {
@@ -60,7 +60,7 @@ function createSandbox(rows) {
querySelector: () => null,
},
location: { hash: '#/tools/observer-neighbors' },
fetch: () => Promise.resolve({ ok: true, json: () => Promise.resolve({ neighbors: rows }) }),
fetch: () => Promise.resolve({ ok: true, json: () => Promise.resolve({ neighbors: rows, unknownScopes: unknownScopesFixture || [] }) }),
URLSearchParams: URLSearchParams,
registerPage: function () {},
timeAgo: (iso) => 'TIME_AGO(' + iso + ')',
@@ -84,8 +84,8 @@ function createSandbox(rows) {
// That's fine for these tests: we only need to observe what renderTable()
// assigns to '#obs-nb-table-wrap'.innerHTML and '#obs-nb-status'.textContent
// after fetch resolves, and that async load() runs inside init().
function initWith(rows) {
const sb = createSandbox(rows);
function initWith(rows, unknownScopesFixture) {
const sb = createSandbox(rows, unknownScopesFixture);
const container = { innerHTML: '' };
sb.window.ObserverNeighborsTool.init(container);
return sb;
@@ -142,6 +142,24 @@ function waitForLoad() {
assert.ok(status.textContent.includes('2 of 2 neighbor pairs'), `got: ${status.textContent}`);
});
await test('renders the Unknown Scopes panel with scope, count, and example neighbors', async () => {
const sb = initWith([makeRow()], [
{ scope: '#dk-storkbh', count: 3, examples: ['Neighbor A', 'Neighbor B', 'Neighbor C'] },
]);
await waitForLoad();
const wrap = sb.__docStore['obs-nb-unknown-scopes-wrap'];
assert.ok(wrap.innerHTML.includes('Scopes CoreScope Doesn\'t Know About Yet (1)'), `got: ${wrap.innerHTML}`);
assert.ok(wrap.innerHTML.includes('#dk-storkbh'), 'expected the unknown scope name');
assert.ok(wrap.innerHTML.includes('Neighbor A, Neighbor B, Neighbor C'), 'expected the example neighbors joined');
});
await test('Unknown Scopes panel renders nothing when there are no unknown scopes', async () => {
const sb = initWith([makeRow()], []);
await waitForLoad();
const wrap = sb.__docStore['obs-nb-unknown-scopes-wrap'];
assert.strictEqual(wrap.innerHTML, '', 'panel should be empty, not an empty-state message -- absence of unknown scopes is the normal case');
});
await test('sortValue: observer/neighbor fall back to id/pubkey when unresolved, lowercased', () => {
const sb = createSandbox([]);
const sv = sb.window.ObserverNeighborsTool.sortValue;