From 8da5f8312e71fbb9ad275fdc8e4f30b03f166d68 Mon Sep 17 00:00:00 2001 From: dborup Date: Tue, 21 Jul 2026 16:40:58 +0200 Subject: [PATCH] feat: add Scope Adoption by Area to the Scopes tab MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New geographic view, independent of the raw hashRegion-code-based Region Utilization: buckets every positioned node by its configured area (AreaKeyForPoint) and tallies how many have any default_scope at all, and how many specifically match the area's own linked region. Surfaces gaps Region Utilization can't see, since that only knows about region strings that already appeared in traffic — a real area with real nodes that never produced a single scoped message is invisible there but shows up here as 0% adoption. Co-Authored-By: Claude Sonnet 5 --- cmd/server/config.go | 21 ++++++++-- cmd/server/db.go | 85 +++++++++++++++++++++++++++++++++++++++ cmd/server/db_test.go | 56 ++++++++++++++++++++++++++ cmd/server/routes.go | 8 ++++ cmd/server/routes_test.go | 49 ++++++++++++++++++++++ cmd/server/types.go | 27 +++++++++++++ public/analytics.js | 38 +++++++++++++++++ 7 files changed, 281 insertions(+), 3 deletions(-) diff --git a/cmd/server/config.go b/cmd/server/config.go index 0f713b90..839247a3 100644 --- a/cmd/server/config.go +++ b/cmd/server/config.go @@ -37,11 +37,25 @@ type AreaEntry struct { // nested areas overlap (e.g. a point inside both "Odense by" and "Fyn"). // Returns ok=false for (0,0)/no-fix points or when no area matches. func AreaForPoint(lat, lon float64, areas map[string]AreaEntry) (label string, ok bool) { + _, label, ok = areaMatchForPoint(lat, lon, areas) + return label, ok +} + +// AreaKeyForPoint is AreaForPoint but returns the area's config key (e.g. +// "ODE") instead of its display label — for callers that need to look up +// other fields on the matched AreaEntry (e.g. RegionScope), not just show +// the name. +func AreaKeyForPoint(lat, lon float64, areas map[string]AreaEntry) (key string, ok bool) { + key, _, ok = areaMatchForPoint(lat, lon, areas) + return key, ok +} + +func areaMatchForPoint(lat, lon float64, areas map[string]AreaEntry) (key, label string, ok bool) { if lat == 0 && lon == 0 { - return "", false + return "", "", false } bestSpan := math.MaxFloat64 - for _, a := range areas { + for k, a := range areas { gf := &geofilter.Config{Polygon: a.Polygon, LatMin: a.LatMin, LatMax: a.LatMax, LonMin: a.LonMin, LonMax: a.LonMax} if !geofilter.PassesFilter(lat, lon, gf) { continue @@ -49,11 +63,12 @@ func AreaForPoint(lat, lon float64, areas map[string]AreaEntry) (label string, o span := areaSpan(a) if span < bestSpan { bestSpan = span + key = k label = a.Label ok = true } } - return label, ok + return key, label, ok } // areaSpan approximates an area's size as its bounding-box extent in diff --git a/cmd/server/db.go b/cmd/server/db.go index 49cc1bed..3f837352 100644 --- a/cmd/server/db.go +++ b/cmd/server/db.go @@ -2239,6 +2239,91 @@ func (db *DB) GetNodesByDefaultScope() (map[string][]RepeaterRef, error) { return result, rows.Err() } +// nodeAreaScopeInput is one node's position + default_scope — the raw +// input to computeScopeAdoptionByArea. DefaultScope is "" when unset or +// when this DB predates #899 (no default_scope column at all). +type nodeAreaScopeInput struct { + Lat, Lon float64 + DefaultScope string +} + +// GetNodesForScopeAdoption returns every node with a real GPS fix (0,0 +// excluded, same convention as geofilter.PassesFilter) and its +// default_scope, for computeScopeAdoptionByArea to bucket by configured +// area. Unlike GetNodesByDefaultScope, this includes nodes with NO scope +// too — the whole point is measuring adoption, not just listing who has one. +func (db *DB) GetNodesForScopeAdoption() ([]nodeAreaScopeInput, error) { + query := "SELECT lat, lon" + if db.hasDefaultScope { + query += ", default_scope" + } + query += " FROM nodes WHERE lat IS NOT NULL AND lon IS NOT NULL AND lat != 0 AND lon != 0" + rows, err := db.conn.Query(query) + if err != nil { + return nil, fmt.Errorf("nodes for scope adoption query: %w", err) + } + defer rows.Close() + var out []nodeAreaScopeInput + for rows.Next() { + var lat, lon float64 + var scope sql.NullString + var scanErr error + if db.hasDefaultScope { + scanErr = rows.Scan(&lat, &lon, &scope) + } else { + scanErr = rows.Scan(&lat, &lon) + } + if scanErr != nil { + continue + } + out = append(out, nodeAreaScopeInput{Lat: lat, Lon: lon, DefaultScope: scope.String}) + } + return out, rows.Err() +} + +// computeScopeAdoptionByArea buckets nodes by their most specific +// configured area (AreaKeyForPoint) and tallies, per area: how many nodes +// sit there at all, how many have ANY default_scope configured, and (when +// the area itself has a RegionScope link) how many of those specifically +// match the area's own region — i.e. does this geographic community +// actually use the scope the area is nominally tied to, or something else +// entirely (or nothing at all). A node outside every configured area is +// excluded, same as the area-badge features above. +func computeScopeAdoptionByArea(nodes []nodeAreaScopeInput, areas map[string]AreaEntry) []AreaScopeAdoption { + counts := make(map[string]*AreaScopeAdoption) + for _, n := range nodes { + key, ok := AreaKeyForPoint(n.Lat, n.Lon, areas) + if !ok { + continue + } + c, exists := counts[key] + if !exists { + a := areas[key] + c = &AreaScopeAdoption{AreaKey: key, Label: a.Label, RegionScope: a.RegionScope} + counts[key] = c + } + c.TotalNodes++ + scope := strings.ToLower(strings.TrimPrefix(n.DefaultScope, "#")) + if scope != "" { + c.NodesWithAnyScope++ + if c.RegionScope != "" && scope == strings.ToLower(c.RegionScope) { + c.NodesMatchingArea++ + } + } + } + result := make([]AreaScopeAdoption, 0, len(counts)) + for _, c := range counts { + result = append(result, *c) + } + sort.Slice(result, func(i, j int) bool { + if result[i].TotalNodes != result[j].TotalNodes { + return result[i].TotalNodes > result[j].TotalNodes + } + return result[i].Label < result[j].Label + }) + return result +} + // QueryMultiNodePackets returns transmissions referencing any of the given pubkeys. func (db *DB) QueryMultiNodePackets(pubkeys []string, limit, offset int, order, since, until string) (*PacketResult, error) { if len(pubkeys) == 0 { diff --git a/cmd/server/db_test.go b/cmd/server/db_test.go index 5ac18c08..7bc73753 100644 --- a/cmd/server/db_test.go +++ b/cmd/server/db_test.go @@ -2327,3 +2327,59 @@ func TestLoadIndexesRelayHopsFromResolvedPath(t *testing.T) { t.Errorf("relay byNode entry has wrong hash: %s", store.byNode[relayPubkey][0].Hash) } } + +func TestComputeScopeAdoptionByArea(t *testing.T) { + f := func(v float64) *float64 { return &v } + areas := map[string]AreaEntry{ + "ODE": {Label: "Odense by", RegionScope: "dk-fyn-odense", LatMin: f(55.32), LatMax: f(55.45), LonMin: f(10.3), LonMax: f(10.5)}, + "GOT": {Label: "Göteborg, SE", LatMin: f(57.35), LatMax: f(57.90), LonMin: f(11.85), LonMax: f(12.85)}, // no RegionScope link + } + + nodes := []nodeAreaScopeInput{ + {Lat: 55.4047, Lon: 10.3810, DefaultScope: "#dk-fyn-odense"}, // Odense, matches area's own region + {Lat: 55.40, Lon: 10.40, DefaultScope: "#dk-aarhus"}, // Odense, but a DIFFERENT region + {Lat: 55.41, Lon: 10.41, DefaultScope: ""}, // Odense, no scope at all + {Lat: 57.68, Lon: 11.97, DefaultScope: "#dk-aarhus"}, // Göteborg, has a scope, but area has no RegionScope link + {Lat: 57.70, Lon: 11.98, DefaultScope: ""}, // Göteborg, no scope + {Lat: 0, Lon: 0, DefaultScope: "#dk-aarhus"}, // no-fix, must be excluded entirely + {Lat: 51.0, Lon: 4.0, DefaultScope: "#belgium"}, // outside every configured area, excluded + } + + got := computeScopeAdoptionByArea(nodes, areas) + if len(got) != 2 { + t.Fatalf("got %d areas, want 2 (ODE and GOT) -- result: %+v", len(got), got) + } + byKey := map[string]AreaScopeAdoption{} + for _, a := range got { + byKey[a.AreaKey] = a + } + + ode := byKey["ODE"] + if ode.TotalNodes != 3 { + t.Errorf("ODE.TotalNodes = %d, want 3", ode.TotalNodes) + } + if ode.NodesWithAnyScope != 2 { + t.Errorf("ODE.NodesWithAnyScope = %d, want 2 (one has no scope at all)", ode.NodesWithAnyScope) + } + if ode.NodesMatchingArea != 1 { + t.Errorf("ODE.NodesMatchingArea = %d, want 1 (only the dk-fyn-odense one matches, the dk-aarhus one doesn't)", ode.NodesMatchingArea) + } + + got2 := byKey["GOT"] + if got2.TotalNodes != 2 { + t.Errorf("GOT.TotalNodes = %d, want 2", got2.TotalNodes) + } + if got2.NodesWithAnyScope != 1 { + t.Errorf("GOT.NodesWithAnyScope = %d, want 1", got2.NodesWithAnyScope) + } + if got2.NodesMatchingArea != 0 { + t.Errorf("GOT.NodesMatchingArea = %d, want 0 (area has no RegionScope link to match against)", got2.NodesMatchingArea) + } +} + +func TestComputeScopeAdoptionByArea_Empty(t *testing.T) { + got := computeScopeAdoptionByArea(nil, map[string]AreaEntry{"DK": {Label: "Danmark"}}) + if len(got) != 0 { + t.Errorf("expected no areas with 0 nodes, got %+v", got) + } +} diff --git a/cmd/server/routes.go b/cmd/server/routes.go index 38e618f2..34c5d598 100644 --- a/cmd/server/routes.go +++ b/cmd/server/routes.go @@ -3744,6 +3744,14 @@ func (s *Server) handleScopeStats(w http.ResponseWriter, r *http.Request) { log.Printf("WARN GetChannelMessageScopeStats: %v", err) } + if s.cfg != nil && len(s.cfg.Areas) > 0 { + if nodes, err := s.db.GetNodesForScopeAdoption(); err == nil { + resp.ScopeAdoptionByArea = computeScopeAdoptionByArea(nodes, s.cfg.Areas) + } else { + log.Printf("WARN GetNodesForScopeAdoption: %v", err) + } + } + if adoption, err := s.db.GetChannelScopeAdoption(window); err == nil { resp.ChannelScopeAdoption = adoption } else { diff --git a/cmd/server/routes_test.go b/cmd/server/routes_test.go index e6619976..c5dba4e0 100644 --- a/cmd/server/routes_test.go +++ b/cmd/server/routes_test.go @@ -4786,6 +4786,55 @@ func TestHandleScopeStats_OriginatingNodesByRegion(t *testing.T) { } } +func TestHandleScopeStats_ScopeAdoptionByArea(t *testing.T) { + srv, _ := setupTestServer(t) + if _, err := srv.db.conn.Exec(`ALTER TABLE transmissions ADD COLUMN scope_name TEXT DEFAULT NULL`); err != nil { + t.Fatalf("add scope_name column: %v", err) + } + srv.db.hasScopeName = true + if !srv.db.hasDefaultScope { + if _, err := srv.db.conn.Exec(`ALTER TABLE nodes ADD COLUMN default_scope TEXT DEFAULT NULL`); err != nil { + t.Fatalf("add default_scope column: %v", err) + } + srv.db.hasDefaultScope = true + } + f := func(v float64) *float64 { return &v } + srv.cfg.Areas = map[string]AreaEntry{ + "ODE": {Label: "Odense by", RegionScope: "dk-fyn-odense", LatMin: f(55.32), LatMax: f(55.45), LonMin: f(10.3), LonMax: f(10.5)}, + } + + insertNode := func(pk, defaultScope string, lat, lon float64) { + if _, err := srv.db.conn.Exec( + `INSERT INTO nodes (public_key, name, role, default_scope, lat, lon) VALUES (?, ?, 'repeater', ?, ?, ?)`, + pk, pk, defaultScope, lat, lon, + ); err != nil { + t.Fatalf("seed node %s: %v", pk, err) + } + } + insertNode("odematch01", "#dk-fyn-odense", 55.4047, 10.3810) + insertNode("odewrong01", "#dk-aarhus", 55.40, 10.40) + insertNode("odenoscope1", "", 55.41, 10.41) + + req := httptest.NewRequest("GET", "/api/scope-stats?window=24h", nil) + w := httptest.NewRecorder() + srv.handleScopeStats(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body: %s", w.Code, w.Body.String()) + } + var resp ScopeStatsResponse + if err := json.NewDecoder(w.Body).Decode(&resp); err != nil { + t.Fatalf("decode: %v", err) + } + if len(resp.ScopeAdoptionByArea) != 1 { + t.Fatalf("scopeAdoptionByArea = %+v, want 1 entry (ODE)", resp.ScopeAdoptionByArea) + } + ode := resp.ScopeAdoptionByArea[0] + if ode.AreaKey != "ODE" || ode.TotalNodes != 3 || ode.NodesWithAnyScope != 2 || ode.NodesMatchingArea != 1 { + t.Errorf("ScopeAdoptionByArea[0] = %+v, want AreaKey=ODE TotalNodes=3 NodesWithAnyScope=2 NodesMatchingArea=1", ode) + } +} + func TestHandleScopeStatsInvalidWindow(t *testing.T) { srv, _ := setupTestServer(t) if _, err := srv.db.conn.Exec(`ALTER TABLE transmissions ADD COLUMN scope_name TEXT DEFAULT NULL`); err != nil { diff --git a/cmd/server/types.go b/cmd/server/types.go index ac1ac5bf..25e1a3ae 100644 --- a/cmd/server/types.go +++ b/cmd/server/types.go @@ -191,6 +191,33 @@ type ScopeStatsResponse struct { // HourlyActivityByRegion is window-scoped like Summary/TimeSeries // above — see ScopeHourlyActivity doc. HourlyActivityByRegion []ScopeHourlyActivity `json:"hourlyActivityByRegion,omitempty"` + // ScopeAdoptionByArea buckets every positioned node by its configured + // geographic area (config.Areas, AreaKeyForPoint) and tallies scope + // adoption within that area — independent of whether the raw + // hashRegion codes above are "used" at all. Surfaces gaps like "34 + // real nodes here, 0 have ever configured a scope" that + // UnusedRegions/RepeatersByRegion can't see, since those only know + // about region strings that already appeared in traffic. All-time, + // like the other Regions-tab sections. Omitted when no areas are + // configured. + ScopeAdoptionByArea []AreaScopeAdoption `json:"scopeAdoptionByArea,omitempty"` +} + +// AreaScopeAdoption is one configured area's node count and scope adoption +// — see ScopeStatsResponse.ScopeAdoptionByArea and computeScopeAdoptionByArea. +type AreaScopeAdoption struct { + AreaKey string `json:"areaKey"` + Label string `json:"label"` + RegionScope string `json:"regionScope,omitempty"` + TotalNodes int `json:"totalNodes"` + // NodesWithAnyScope is how many of TotalNodes have ANY default_scope + // configured, regardless of which region it is. + NodesWithAnyScope int `json:"nodesWithAnyScope"` + // NodesMatchingArea is the subset of NodesWithAnyScope whose scope + // matches this area's own RegionScope specifically. Only meaningful + // when RegionScope is set — 0 otherwise (not the same as "0 of them + // match", there's simply nothing configured to match against). + NodesMatchingArea int `json:"nodesMatchingArea,omitempty"` } type RepeaterRef struct { diff --git a/public/analytics.js b/public/analytics.js index 9e5e18d0..23039e03 100644 --- a/public/analytics.js +++ b/public/analytics.js @@ -4556,6 +4556,7 @@ function destroy() { _stopRolesRefresh(); _stopScopesRefresh(); _stopForeignTraf '' + '' + '
' + + '
' + '
' + '
' + '
' + @@ -5064,6 +5065,43 @@ function destroy() { _stopRolesRefresh(); _stopScopesRefresh(); _stopForeignTraf } } + // Scope Adoption by Area: buckets every positioned node by its + // configured geographic area and asks "does this real, physical + // community actually use the region-scope system at all" — + // independent of which raw hashRegion codes have ever appeared in + // traffic. Region Utilization below only knows about region strings + // that already showed up in a message; a real area with real nodes + // that has NEVER produced a single region-scoped message is + // invisible there. This section catches exactly that gap. + var areaAdoptEl = document.getElementById('scopes-area-adoption'); + if (areaAdoptEl) { + var byArea = d.scopeAdoptionByArea || []; + if (byArea.length > 0) { + var areaRows = byArea.map(function(a) { + var withScope = a.nodesWithAnyScope.toLocaleString() + ' (' + pct(a.nodesWithAnyScope, a.totalNodes) + ')'; + var matching = a.regionScope + ? a.nodesMatchingArea.toLocaleString() + ' (' + pct(a.nodesMatchingArea, a.totalNodes) + ')' + + ' of #' + esc(a.regionScope) + '' + : 'no region linked to this area'; + return '' + esc(a.label) + '' + + '' + a.totalNodes.toLocaleString() + '' + + '' + withScope + '' + + '' + matching + ''; + }).join(''); + setSectionHtml(areaAdoptEl, detailsSection( + 'Scope Adoption by Area (' + byArea.length.toLocaleString() + ' areas)', + 'All-time — every configured area with at least one positioned node. Shows whether that geographic community actually runs the region-scope system, and whether it runs the specific region this area is linked to.', + '' + + '' + + '' + areaRows + '' + + '
AreaNodesWith Any ScopeMatching Area’s Own Region
', + 'scope-adoption-by-area' + )); + } else { + areaAdoptEl.innerHTML = ''; + } + } + // Region utilization: how much of the configured hashRegions list // has never actually matched anything — all-time (not window-scoped), // so it doesn't fluctuate with the 1h/24h/7d selector above. Only