feat(topology): serve bounded route links and identify decoded Public traffic

This commit is contained in:
n30nex
2026-10-04 01:03:08 -04:00
parent 53f2467864
commit 84ac3c8781
19 changed files with 486 additions and 0 deletions
+15
View File
@@ -1853,3 +1853,18 @@ JOIN node_short_ids si ON si.iata = ANY(@iatas::bpchar[]) AND
JOIN nodes n ON n.id=si.node_id
WHERE substring(n.public_key FROM 1 FOR octet_length(h.hash))=h.hash
ORDER BY si.iata, h.hash, n.id;
-- name: GetTopologyLinks :many
-- Reduce route history to unique adjacent pairs before transferring it to a browser.
WITH pairs AS (
SELECT h.node_id AS a, r.node_ids[(h.ord+1)::int] AS b
FROM known_routes r CROSS JOIN LATERAL unnest(r.node_ids) WITH ORDINALITY h(node_id,ord)
WHERE r.last_seen >= @since::timestamptz AND r.last_seen < @until::timestamptz
AND (COALESCE(cardinality(@iatas::bpchar[]),0)=0 OR r.iata=ANY(@iatas::bpchar[]))
AND h.ord < cardinality(r.node_ids)
)
SELECT LEAST(a,b)::uuid AS from_id, GREATEST(a,b)::uuid AS to_id
FROM pairs WHERE a IS NOT NULL AND b IS NOT NULL AND a<>b
GROUP BY LEAST(a,b), GREATEST(a,b)
ORDER BY LEAST(a,b), GREATEST(a,b)
LIMIT @link_limit::int;
+15
View File
@@ -1137,6 +1137,21 @@ func (mr *MockQuerierMockRecorder) GetTopNodes(ctx, arg any) *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTopNodes", reflect.TypeOf((*MockQuerier)(nil).GetTopNodes), ctx, arg)
}
// GetTopologyLinks mocks base method.
func (m *MockQuerier) GetTopologyLinks(ctx context.Context, arg db.GetTopologyLinksParams) ([]db.GetTopologyLinksRow, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "GetTopologyLinks", ctx, arg)
ret0, _ := ret[0].([]db.GetTopologyLinksRow)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// GetTopologyLinks indicates an expected call of GetTopologyLinks.
func (mr *MockQuerierMockRecorder) GetTopologyLinks(ctx, arg any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTopologyLinks", reflect.TypeOf((*MockQuerier)(nil).GetTopologyLinks), ctx, arg)
}
// GetTransportScopeByName mocks base method.
func (m *MockQuerier) GetTransportScopeByName(ctx context.Context, name string) (int32, error) {
m.ctrl.T.Helper()
+2
View File
@@ -156,6 +156,8 @@ type Querier interface {
// Nodes by ADVERT hearings since the given hour. Live names win over the rolled snapshot;
// node_id is NULL once the node row is gone. iata is a representative one.
GetTopNodes(ctx context.Context, arg GetTopNodesParams) ([]GetTopNodesRow, error)
// Reduce route history to unique adjacent pairs before transferring it to a browser.
GetTopologyLinks(ctx context.Context, arg GetTopologyLinksParams) ([]GetTopologyLinksRow, error)
GetTransportScopeByName(ctx context.Context, name string) (int32, error)
GetTransportScopes(ctx context.Context) ([]GetTransportScopesRow, error)
// ============================================================
+53
View File
@@ -2237,6 +2237,59 @@ func (q *Queries) GetTopNodes(ctx context.Context, arg GetTopNodesParams) ([]Get
return items, nil
}
const getTopologyLinks = `-- name: GetTopologyLinks :many
WITH pairs AS (
SELECT h.node_id AS a, r.node_ids[(h.ord+1)::int] AS b
FROM known_routes r CROSS JOIN LATERAL unnest(r.node_ids) WITH ORDINALITY h(node_id,ord)
WHERE r.last_seen >= $2::timestamptz AND r.last_seen < $3::timestamptz
AND (COALESCE(cardinality($4::bpchar[]),0)=0 OR r.iata=ANY($4::bpchar[]))
AND h.ord < cardinality(r.node_ids)
)
SELECT LEAST(a,b)::uuid AS from_id, GREATEST(a,b)::uuid AS to_id
FROM pairs WHERE a IS NOT NULL AND b IS NOT NULL AND a<>b
GROUP BY LEAST(a,b), GREATEST(a,b)
ORDER BY LEAST(a,b), GREATEST(a,b)
LIMIT $1::int
`
type GetTopologyLinksParams struct {
LinkLimit int32 `json:"link_limit"`
Since pgtype.Timestamptz `json:"since"`
Until pgtype.Timestamptz `json:"until"`
Iatas []string `json:"iatas"`
}
type GetTopologyLinksRow struct {
FromID uuid.UUID `json:"from_id"`
ToID uuid.UUID `json:"to_id"`
}
// Reduce route history to unique adjacent pairs before transferring it to a browser.
func (q *Queries) GetTopologyLinks(ctx context.Context, arg GetTopologyLinksParams) ([]GetTopologyLinksRow, error) {
rows, err := q.db.Query(ctx, getTopologyLinks,
arg.LinkLimit,
arg.Since,
arg.Until,
arg.Iatas,
)
if err != nil {
return nil, err
}
defer rows.Close()
items := []GetTopologyLinksRow{}
for rows.Next() {
var i GetTopologyLinksRow
if err := rows.Scan(&i.FromID, &i.ToID); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const getTransportScopeByName = `-- name: GetTransportScopeByName :one
SELECT id FROM transport_scopes WHERE name = $1
`
+25
View File
@@ -0,0 +1,25 @@
// Copyright 2026 Beacon Contributors
// SPDX-License-Identifier: AGPL-3.0-or-later
package db
import (
"context"
sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc"
"github.com/MeshCore-Beacon/beacon-server/internal/api"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgtype"
"time"
)
func (s *Store) GetTopologyLinks(ctx context.Context, iatas []string, since, until time.Time) (*api.TopologyLinks, error) {
const limit = 100000
rows, err := s.q.GetTopologyLinks(ctx, sqlc.GetTopologyLinksParams{Iatas: iatas, Since: pgtype.Timestamptz{Time: since, Valid: true}, Until: pgtype.Timestamptz{Time: until, Valid: true}, LinkLimit: limit + 1})
if err != nil {
return nil, err
}
result := &api.TopologyLinks{Links: make([][2]uuid.UUID, 0, min(len(rows), limit)), Capped: len(rows) > limit, Since: since.UnixMilli(), Until: until.UnixMilli()}
for _, r := range rows[:min(len(rows), limit)] {
result.Links = append(result.Links, [2]uuid.UUID{r.FromID, r.ToID})
}
return result, nil
}
+54
View File
@@ -0,0 +1,54 @@
// Copyright 2026 Beacon Contributors
// SPDX-License-Identifier: AGPL-3.0-or-later
package db
import (
"context"
sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc"
"github.com/google/uuid"
"testing"
"time"
)
func TestTopologyLinksPostgres(t *testing.T) {
ctx, pool := schemaPool(t)
if err := RunMigrations(ctx, pool); err != nil {
t.Fatal(err)
}
tx, err := pool.Begin(ctx)
if err != nil {
t.Fatal(err)
}
defer tx.Rollback(context.Background())
if _, err = tx.Exec(ctx, "CREATE TEMP TABLE known_routes (LIKE public.known_routes INCLUDING ALL) ON COMMIT DROP"); err != nil {
t.Fatal(err)
}
ids := []uuid.UUID{uuid.New(), uuid.New(), uuid.New()}
now := time.Now().UTC().Truncate(time.Minute)
// Thousands of duplicate/reversed routes transfer only their unique adjacent pairs.
_, err = tx.Exec(ctx, `INSERT INTO known_routes (path_key,node_ids,hash_prefix,iata,hop_count,first_seen,last_seen)
SELECT decode(md5(i::text),'hex'),CASE WHEN i%2=0 THEN $1::uuid[] ELSE $2::uuid[] END,ARRAY['\xaa'::bytea,'\xbb'::bytea],
'YKF',2,$3,$3 FROM generate_series(1,1000) i`, ids[:2], []uuid.UUID{ids[1], ids[0]}, now)
if err != nil {
t.Fatal(err)
}
_, err = tx.Exec(ctx, `INSERT INTO known_routes(path_key,node_ids,hash_prefix,iata,hop_count,first_seen,last_seen) VALUES
(decode(md5('other'),'hex'),$1,ARRAY['\xbb'::bytea,'\xcc'::bytea],'YOW',2,$3,$3),
(decode(md5('gap'),'hex'),$2,ARRAY['\xaa'::bytea,'\x00'::bytea,'\xcc'::bytea],'YKF',3,$3,$3),
(decode(md5('old'),'hex'),$1,ARRAY['\xbb'::bytea,'\xcc'::bytea],'YKF',2,$3-interval '25 hours',$3-interval '25 hours')`, ids[1:], []*uuid.UUID{&ids[0], nil, &ids[2]}, now)
if err != nil {
t.Fatal(err)
}
store := &Store{q: sqlc.New(tx)}
result, err := store.GetTopologyLinks(ctx, []string{"YKF"}, now.Add(-24*time.Hour), now.Add(time.Minute))
if err != nil || result.Capped || len(result.Links) != 1 {
t.Fatalf("scoped: %+v %v", result, err)
}
all, err := store.GetTopologyLinks(ctx, nil, now.Add(-24*time.Hour), now.Add(time.Minute))
if err != nil || len(all.Links) != 2 {
t.Fatalf("all regions: %+v %v", all, err)
}
if result.Since != now.Add(-24*time.Hour).UnixMilli() {
t.Fatal("window changed")
}
}
+69
View File
@@ -2179,6 +2179,52 @@ const docTemplate = `{
}
}
},
"/routes/topology": {
"get": {
"description": "Returns up to 100000 undirected node-ID pairs, deduplicated across known routes. Minute-aligned window includes the current minute. Existing rate limits apply; no per-node lookup or route pagination is needed. Capped is true when more links exist.",
"produces": [
"application/json"
],
"tags": [
"Routes"
],
"summary": "Unique adjacent route links for a topology snapshot",
"parameters": [
{
"type": "string",
"description": "History window: 15m (default), 1h, or 24h",
"name": "window",
"in": "query"
},
{
"type": "string",
"description": "Comma-separated reception IATA codes",
"name": "iatas",
"in": "query"
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.TopologyLinks"
}
},
"400": {
"description": "Bad Request",
"schema": {
"$ref": "#/definitions/internal_api_handlers.APIError"
}
},
"503": {
"description": "Service Unavailable",
"schema": {
"$ref": "#/definitions/internal_api_handlers.APIError"
}
}
}
}
},
"/routes/{iata}/{pathKey}/observations": {
"get": {
"description": "Matches one complete path representation in the saved IATA. Defaults to the latest processed prefixes; hashSize/pathBytes pin a prior response. Excludes TRACE and unclassified reports; widths are never combined. Pinned prefixes must match the saved node chain or its current metadata. Byte matches do not prove hop identities or delivery. The route counter can outlive raw evidence. New cursors pin path, route, window and full timestamp precision; legacy unpinned cursors remain supported.",
@@ -5642,6 +5688,29 @@ const docTemplate = `{
}
}
},
"github_com_MeshCore-Beacon_beacon-server_internal_api.TopologyLinks": {
"type": "object",
"properties": {
"capped": {
"type": "boolean"
},
"links": {
"type": "array",
"items": {
"type": "array",
"items": {
"type": "string"
}
}
},
"since": {
"type": "integer"
},
"until": {
"type": "integer"
}
}
},
"github_com_MeshCore-Beacon_beacon-server_internal_api.TraceDetail": {
"type": "object",
"properties": {
+69
View File
@@ -2177,6 +2177,52 @@
}
}
},
"/routes/topology": {
"get": {
"description": "Returns up to 100000 undirected node-ID pairs, deduplicated across known routes. Minute-aligned window includes the current minute. Existing rate limits apply; no per-node lookup or route pagination is needed. Capped is true when more links exist.",
"produces": [
"application/json"
],
"tags": [
"Routes"
],
"summary": "Unique adjacent route links for a topology snapshot",
"parameters": [
{
"type": "string",
"description": "History window: 15m (default), 1h, or 24h",
"name": "window",
"in": "query"
},
{
"type": "string",
"description": "Comma-separated reception IATA codes",
"name": "iatas",
"in": "query"
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.TopologyLinks"
}
},
"400": {
"description": "Bad Request",
"schema": {
"$ref": "#/definitions/internal_api_handlers.APIError"
}
},
"503": {
"description": "Service Unavailable",
"schema": {
"$ref": "#/definitions/internal_api_handlers.APIError"
}
}
}
}
},
"/routes/{iata}/{pathKey}/observations": {
"get": {
"description": "Matches one complete path representation in the saved IATA. Defaults to the latest processed prefixes; hashSize/pathBytes pin a prior response. Excludes TRACE and unclassified reports; widths are never combined. Pinned prefixes must match the saved node chain or its current metadata. Byte matches do not prove hop identities or delivery. The route counter can outlive raw evidence. New cursors pin path, route, window and full timestamp precision; legacy unpinned cursors remain supported.",
@@ -5640,6 +5686,29 @@
}
}
},
"github_com_MeshCore-Beacon_beacon-server_internal_api.TopologyLinks": {
"type": "object",
"properties": {
"capped": {
"type": "boolean"
},
"links": {
"type": "array",
"items": {
"type": "array",
"items": {
"type": "string"
}
}
},
"since": {
"type": "integer"
},
"until": {
"type": "integer"
}
}
},
"github_com_MeshCore-Beacon_beacon-server_internal_api.TraceDetail": {
"type": "object",
"properties": {
+48
View File
@@ -1515,6 +1515,21 @@ definitions:
senderName:
type: string
type: object
github_com_MeshCore-Beacon_beacon-server_internal_api.TopologyLinks:
properties:
capped:
type: boolean
links:
items:
items:
type: string
type: array
type: array
since:
type: integer
until:
type: integer
type: object
github_com_MeshCore-Beacon_beacon-server_internal_api.TraceDetail:
properties:
packets:
@@ -3233,6 +3248,39 @@ paths:
summary: Search known routes by source and destination hash
tags:
- Routes
/routes/topology:
get:
description: Returns up to 100000 undirected node-ID pairs, deduplicated across
known routes. Minute-aligned window includes the current minute. Existing
rate limits apply; no per-node lookup or route pagination is needed. Capped
is true when more links exist.
parameters:
- description: 'History window: 15m (default), 1h, or 24h'
in: query
name: window
type: string
- description: Comma-separated reception IATA codes
in: query
name: iatas
type: string
produces:
- application/json
responses:
"200":
description: OK
schema:
$ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.TopologyLinks'
"400":
description: Bad Request
schema:
$ref: '#/definitions/internal_api_handlers.APIError'
"503":
description: Service Unavailable
schema:
$ref: '#/definitions/internal_api_handlers.APIError'
summary: Unique adjacent route links for a topology snapshot
tags:
- Routes
/scope-catalogues:
get:
description: Discovered sources only. Counts describe the source region, not
+1
View File
@@ -20,6 +20,7 @@ import (
func RoutesRouter(reader api.Reader) http.Handler {
r := chi.NewRouter()
r.Get("/", listKnownRoutes(reader))
r.Get("/topology", getTopologyLinks(reader))
r.Get("/cross", searchCrossIATARoutes(reader))
r.Get("/search", searchKnownRoutes(reader))
r.Get("/{iata}/{pathKey}/observations", getRouteEvidence(reader))
@@ -417,3 +417,7 @@ func (s stubReader) GetRouteEvidence(ctx context.Context, iata, key string, q ap
}
return nil, nil
}
func (s stubReader) GetTopologyLinks(_ context.Context, _ []string, _ time.Time, _ time.Time) (*api.TopologyLinks, error) {
return &api.TopologyLinks{}, nil
}
+44
View File
@@ -0,0 +1,44 @@
// Copyright 2026 Beacon Contributors
// SPDX-License-Identifier: AGPL-3.0-or-later
package handlers
import (
"context"
"github.com/MeshCore-Beacon/beacon-server/internal/api"
"net/http"
"time"
)
// getTopologyLinks godoc
// @Summary Unique adjacent route links for a topology snapshot
// @Description Returns up to 100000 undirected node-ID pairs, deduplicated across known routes. Minute-aligned window includes the current minute. Existing rate limits apply; no per-node lookup or route pagination is needed. Capped is true when more links exist.
// @Tags Routes
// @Produce json
// @Param window query string false "History window: 15m (default), 1h, or 24h"
// @Param iatas query string false "Comma-separated reception IATA codes"
// @Success 200 {object} api.TopologyLinks
// @Failure 400 {object} handlers.APIError
// @Failure 503 {object} handlers.APIError
// @Router /routes/topology [get]
func getTopologyLinks(reader api.Reader) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
window := r.URL.Query().Get("window")
if window == "" {
window = "15m"
}
duration, ok := map[string]time.Duration{"15m": 15 * time.Minute, "1h": time.Hour, "24h": 24 * time.Hour}[window]
if !ok {
respondError(w, http.StatusBadRequest, "window must be 15m, 1h or 24h")
return
}
until := time.Now().UTC().Truncate(time.Minute).Add(time.Minute)
ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second)
defer cancel()
links, err := reader.GetTopologyLinks(ctx, parseIATAs(r), until.Add(-duration), until)
if err != nil {
respondError(w, http.StatusServiceUnavailable, "route snapshot unavailable; try again shortly")
return
}
respond(w, http.StatusOK, links)
}
}
+22
View File
@@ -0,0 +1,22 @@
// Copyright 2026 Beacon Contributors
// SPDX-License-Identifier: AGPL-3.0-or-later
package handlers
import (
"net/http/httptest"
"testing"
)
func TestTopologyWindowValidation(t *testing.T) {
for _, window := range []string{"", "15m", "1h", "24h", "72h", "0", "garbage"} {
w := httptest.NewRecorder()
RoutesRouter(stubReader{}).ServeHTTP(w, httptest.NewRequest("GET", "/topology?window="+window, nil))
want := 200
if window == "72h" || window == "0" || window == "garbage" {
want = 400
}
if w.Code != want {
t.Fatalf("%s: %d", window, w.Code)
}
}
}
+2
View File
@@ -224,6 +224,8 @@ type Reader interface {
// GetTraceByTag returns all packets for a given trace tag with resolved routes.
GetTraceByTag(ctx context.Context, tag string) (*TraceDetail, error)
GetTopologyLinks(ctx context.Context, iatas []string, since, until time.Time) (*TopologyLinks, error)
// ListKnownRoutes returns known routes filtered by IATA and optional hop count.
// A nonzero cursorID breaks ties within the cursor's millisecond.
ListKnownRoutes(ctx context.Context, iata string, hopCount int32, cursor time.Time, cursorID int64, limit int32) ([]KnownRoute, error)
+14
View File
@@ -0,0 +1,14 @@
// Copyright 2026 Beacon Contributors
// SPDX-License-Identifier: AGPL-3.0-or-later
package api
import "github.com/google/uuid"
// TopologyLinks is a bounded set of undirected, adjacent known-route segments.
// It does not imply radio reachability or join across missing path entries.
type TopologyLinks struct {
Links [][2]uuid.UUID `json:"links"`
Capped bool `json:"capped"`
Since int64 `json:"since"`
Until int64 `json:"until"`
}
+4
View File
@@ -446,3 +446,7 @@ func TestCachedReader_InvalidateObserver(t *testing.T) {
func (s *stubReader) GetRouteEvidence(_ context.Context, _, _ string, _ api.RouteEvidenceQuery) (*api.RouteEvidence, error) {
return nil, nil
}
func (s *stubReader) GetTopologyLinks(_ context.Context, _ []string, _ time.Time, _ time.Time) (*api.TopologyLinks, error) {
return &api.TopologyLinks{}, nil
}
+9
View File
@@ -378,3 +378,12 @@ func (cr *CachedReader) SearchCrossIATARoutes(ctx context.Context, fromHash, fro
func (cr *CachedReader) ListTraceTags(ctx context.Context, iatas []string, scope, traceType string, since, until time.Time, cursor time.Time, cursorTag string, limit int32) ([]api.TraceTagSummary, error) {
return cr.inner.ListTraceTags(ctx, iatas, scope, traceType, since, until, cursor, cursorTag, limit)
}
func (cr *CachedReader) GetTopologyLinks(ctx context.Context, iatas []string, since, until time.Time) (*api.TopologyLinks, error) {
regions := append([]string{}, iatas...)
sort.Strings(regions)
key := fmt.Sprintf("beacon:topology:%s:%d:%d", strings.Join(regions, ","), since.Unix(), until.Unix())
return getOrSet(ctx, cr.c, key, 30*time.Second, func(ctx context.Context) (*api.TopologyLinks, error) {
return cr.inner.GetTopologyLinks(ctx, iatas, since, until)
})
}
+34
View File
@@ -5,6 +5,7 @@ package ingest
import (
"context"
"encoding/hex"
"encoding/json"
"testing"
"time"
@@ -73,3 +74,36 @@ func TestChannelMessageScopeLive(t *testing.T) {
})
}
}
func TestChannelPublicMarkerUsesDecryptionKey(t *testing.T) {
for _, public := range []bool{false, true} {
w, base := newTestWorker()
w.db = &scopeMessageDB{stubDB: base, next: &InsertedChannelMessage{ID: 1}}
key := make([]byte, 16)
if public {
key, _ = hex.DecodeString("8b3387e9c5cdea6ac9e5edbaa115cd72")
}
w.keys = &mapKeys{entries: map[byte][]keystore.Entry{0x11: {{Key: key, Name: "Public"}}}}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
go w.hub.Run()
client := w.hub.NewClient()
defer w.hub.Remove(client)
w.hub.AddScope(client, "public-marker", hub.Scope{Events: []hub.EventType{hub.EventChannelMessage, hub.EventObserverStatus}})
waitForSummarySubscriber(t, ctx, w.hub, client)
packet := &meshcore.Packet{Header: meshcore.MakeHeader(meshcore.RouteTypeFlood, meshcore.PayloadTypeGrpTxt, 0), Payload: encryptedGroupText(t, 0x11, key, "Fixture", "test")}
w.handlePayloadTypeSideEffects(ctx, packet, "YKF", []byte{1}, RadioSettings{}, nil, nil, nil, 0)
select {
case event := <-client.Send:
var got channelMessageEvent
if err := json.Unmarshal(event.Payload, &got); err != nil {
t.Fatal(err)
}
if got.IsPublic != public {
t.Fatal("channel name/hash used instead of actual decryption key")
}
case <-ctx.Done():
t.Fatal("no channel event")
}
}
}
+2
View File
@@ -48,6 +48,7 @@ type InsertedChannelMessage struct {
// channelMessageEvent is the JSON payload for a channelMessage WS event.
type channelMessageEvent struct {
IsPublic bool `json:"isPublic"` // True only when decryption used the documented default Public key, never inferred from a one-byte channel hash.
ID int64 `json:"id"`
ChannelID int `json:"channelId"`
ChannelHash string `json:"channelHash"` // hex-encoded single byte
@@ -223,6 +224,7 @@ func (w *Worker) handlePayloadTypeSideEffects(ctx context.Context, packet *meshc
if result.Message != nil {
evt := channelMessageEvent{
IsPublic: hex.EncodeToString(result.Entry.Key) == "8b3387e9c5cdea6ac9e5edbaa115cd72",
ID: result.Message.ID,
Scope: result.Message.Scope,
ScopeStatus: result.Message.ScopeStatus,