feat(scopes): import cached regional MeshMapper catalogues

This commit is contained in:
n30nex authored and MrAlders0n committed 2026-09-30 07:19:03 -04:00
1 parent 831148ff5f
commit c18e059e0d
25 files changed
+1260 -49

No files matched your search

+49
View File
@@ -0,0 +1,49 @@
// Copyright 2026 Beacon Contributors
// SPDX-License-Identifier: AGPL-3.0-or-later
package db
import (
"context"
"errors"
sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc"
"github.com/MeshCore-Beacon/beacon-server/internal/meshmapper"
"github.com/MeshCore-Beacon/beacon-server/internal/scopestore"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgtype"
)
func (s *Store) GetScopeCatalogue(ctx context.Context, iata, url string) (*meshmapper.Cache, error) {
row, err := s.q.GetScopeCatalogue(ctx, sqlc.GetScopeCatalogueParams{Iata: iata, Url: url})
if errors.Is(err, pgx.ErrNoRows) {
return nil, nil
}
if err != nil {
return nil, err
}
cache := &meshmapper.Cache{Payload: row.Payload, CheckedAt: row.CheckedAt.Time, AttemptedAt: row.AttemptedAt.Time, NextAttempt: row.NextAttempt.Time, LastError: row.LastError}
if row.Etag != nil {
cache.ETag = *row.Etag
}
return cache, nil
}
func (s *Store) SaveScopeCatalogue(ctx context.Context, iata, url string, cache meshmapper.Cache, entries []scopestore.Entry) error {
params := sqlc.SaveScopeCatalogueParams{
Iata: iata, Url: url, Payload: cache.Payload, LastError: cache.LastError,
CheckedAt: pgtype.Timestamptz{Time: cache.CheckedAt, Valid: !cache.CheckedAt.IsZero()},
AttemptedAt: pgtype.Timestamptz{Time: cache.AttemptedAt, Valid: true},
NextAttempt: pgtype.Timestamptz{Time: cache.NextAttempt, Valid: true},
Names: []string{}, Keys: [][]byte{}, Fingerprints: [][]byte{},
}
if !cache.CheckedAt.IsZero() {
params.Etag = &cache.ETag
}
for _, entry := range entries {
params.Names = append(params.Names, entry.Name)
params.Keys = append(params.Keys, entry.TransportKey)
params.Fingerprints = append(params.Fingerprints, entry.KeyFingerprint)
}
return s.q.SaveScopeCatalogue(ctx, params)
}
+139
View File
@@ -0,0 +1,139 @@
// Copyright 2026 Beacon Contributors
// SPDX-License-Identifier: AGPL-3.0-or-later
package db
import (
"context"
"os"
"testing"
"time"
sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc"
"github.com/MeshCore-Beacon/beacon-server/internal/config"
"github.com/MeshCore-Beacon/beacon-server/internal/meshmapper"
"github.com/MeshCore-Beacon/beacon-server/internal/scopestore"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
)
func TestMeshMapperCataloguePostgres(t *testing.T) {
dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN")
if dsn == "" {
t.Skip("set BEACON_TEST_POSTGRES_DSN")
}
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()
pool, err := pgxpool.New(ctx, dsn)
if err != nil {
t.Fatal(err)
}
if err := RunMigrations(ctx, pool); err != nil {
pool.Close()
t.Fatal(err)
}
pool.Close()
conn, err := pgx.Connect(ctx, dsn)
if err != nil {
t.Fatal(err)
}
defer conn.Close(context.Background())
tx, err := conn.Begin(ctx)
if err != nil {
t.Fatal(err)
}
defer tx.Rollback(context.Background())
for _, table := range []string{"transport_scopes", "meshmapper_scope_catalogues"} {
if _, err := tx.Exec(ctx, "CREATE TEMP TABLE "+table+" (LIKE public."+table+" INCLUDING ALL) ON COMMIT DROP"); err != nil {
t.Fatal(err)
}
}
store := &Store{q: sqlc.New(tx)}
manual := scopestore.FromName("manual")
manual.TransportKey[0] ^= 1
if err := store.UpsertTransportScope(ctx, manual.Name, "Operator label", manual.TransportKey, manual.KeyFingerprint); err != nil {
t.Fatal(err)
}
url := "https://yow.meshmapper.net/get_scopes.php"
if row, err := store.GetScopeCatalogue(ctx, "YOW", url); err != nil || row != nil {
t.Fatal(row, err)
}
now := time.Now().UTC().Truncate(time.Second)
payload := []byte(`{"generated_at":"2026-09-27T16:58:49Z","region":"YOW","zones":["YOW"],"repeaters":5,"scoped":3,"scopes":[{"name":"yow","repeaters":0,"default":0,"monitored":true,"wardriving":false}]}`)
update := meshmapper.Cache{Payload: payload, ETag: `"v1"`, CheckedAt: now, AttemptedAt: now, NextAttempt: now.Add(time.Hour)}
if err := store.SaveScopeCatalogue(ctx, "YOW", url, update, []scopestore.Entry{scopestore.FromName("manual"), scopestore.FromName("yow")}); err != nil {
t.Fatal(err)
}
manualRows, err := store.GetTransportScopes(ctx)
if err != nil || len(manualRows) != 1 || manualRows[0].TransportKey[0] != manual.TransportKey[0] {
t.Fatal("manual key changed", manualRows, err)
}
var label string
var imported bool
if err := tx.QueryRow(ctx, "SELECT display_name,imported_only FROM transport_scopes WHERE name='#manual'").Scan(&label, &imported); err != nil || label != "Operator label" || imported {
t.Fatal(label, imported, err)
}
row, err := store.GetScopeCatalogue(ctx, "YOW", url)
if err != nil || row.ETag != `"v1"` || !row.CheckedAt.Equal(now) {
t.Fatal(row, err)
}
scopes := scopestore.New()
_, err = meshmapper.New(ctx, config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": url}}, store, scopes, manualRows)
if err != nil || len(scopes.Entries()) != 2 {
t.Fatal("restart did not restore imported membership", err)
}
// A conditional success preserves the document, advances checked time and clears a failure.
checked := now.Add(time.Hour)
if err := store.SaveScopeCatalogue(ctx, "YOW", url, meshmapper.Cache{ETag: `"v1"`, CheckedAt: checked, AttemptedAt: checked, NextAttempt: checked.Add(time.Hour)}, nil); err != nil {
t.Fatal(err)
}
if err := store.SaveScopeCatalogue(ctx, "YOW", url, meshmapper.Cache{AttemptedAt: checked.Add(time.Minute), NextAttempt: checked.Add(2 * time.Hour), LastError: "HTTP 429"}, nil); err != nil {
t.Fatal(err)
}
row, err = store.GetScopeCatalogue(ctx, "YOW", url)
if err != nil || !row.CheckedAt.Equal(checked) || row.ETag != `"v1"` || row.LastError != "HTTP 429" || len(row.Payload) == 0 {
t.Fatal("failed refresh lost durable snapshot", row, err)
}
// Empty membership deactivates matching but retains the old scope identity for historical rows.
empty := []byte(`{"generated_at":"2026-09-27T17:58:49Z","region":"YOW","zones":["YOW"],"repeaters":5,"scoped":0,"scopes":[]}`)
update.Payload = empty
update.ETag = ""
if err := store.SaveScopeCatalogue(ctx, "YOW", url, update, nil); err != nil {
t.Fatal(err)
}
_, err = meshmapper.New(ctx, config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": url}}, store, scopes, manualRows)
if err != nil || len(scopes.Entries()) != 1 {
t.Fatal("removed imported membership remained active", err)
}
if _, err := store.GetTransportScopeByName(ctx, "#yow"); err != nil {
t.Fatal("deleted historical identity", err)
}
row, _ = store.GetScopeCatalogue(ctx, "YOW", url)
if row.ETag != "" {
t.Fatal("200 without ETag retained obsolete validator")
}
// Reconfiguring a known imported name as manual promotes it without losing its identity.
yow := scopestore.FromName("yow")
if err := store.UpsertTransportScope(ctx, yow.Name, "Local", yow.TransportKey, yow.KeyFingerprint); err != nil {
t.Fatal(err)
}
manualRows, err = store.GetTransportScopes(ctx)
if err != nil || len(manualRows) != 2 {
t.Fatal("manual promotion failed", err)
}
// A failed cache write also rolls back the identity insertion in the same statement.
if _, err := tx.Exec(ctx, "ALTER TABLE meshmapper_scope_catalogues ADD CHECK(last_error <> 'force-failure'); SAVEPOINT rejected"); err != nil {
t.Fatal(err)
}
update.LastError = "force-failure"
if err := store.SaveScopeCatalogue(ctx, "YOW", url, update, []scopestore.Entry{scopestore.FromName("uncommitted")}); err == nil {
t.Fatal("expected constraint failure")
}
if _, err := tx.Exec(ctx, "ROLLBACK TO SAVEPOINT rejected"); err != nil {
t.Fatal(err)
}
var count int
if err := tx.QueryRow(ctx, "SELECT count(*) FROM transport_scopes WHERE name='#uncommitted'").Scan(&count); err != nil || count != 0 {
t.Fatal("partially committed catalogue", count, err)
}
}
+17
View File
@@ -0,0 +1,17 @@
-- Copyright 2026 Beacon Contributors
-- SPDX-License-Identifier: AGPL-3.0-or-later
-- Preserve historical identities while distinguishing import-only matching keys.
ALTER TABLE transport_scopes ADD COLUMN IF NOT EXISTS imported_only BOOLEAN NOT NULL DEFAULT FALSE;
CREATE TABLE IF NOT EXISTS meshmapper_scope_catalogues (
iata TEXT NOT NULL,
url TEXT NOT NULL,
payload JSONB,
etag TEXT,
checked_at TIMESTAMPTZ,
attempted_at TIMESTAMPTZ NOT NULL,
next_attempt TIMESTAMPTZ NOT NULL,
last_error TEXT NOT NULL DEFAULT '',
PRIMARY KEY (iata, url)
);
+27
View File
@@ -0,0 +1,27 @@
-- Copyright 2026 Beacon Contributors
-- SPDX-License-Identifier: AGPL-3.0-or-later
-- name: GetScopeCatalogue :one
SELECT * FROM meshmapper_scope_catalogues WHERE iata = $1 AND url = $2;
-- name: SaveScopeCatalogue :exec
-- One statement commits the validated snapshot and its lookup identities together.
-- Empty arrays insert nothing. NULL payload/checked_at retain last-known-good data
-- after an error or 304. Imported names never replace existing manual metadata.
WITH inserted AS (
INSERT INTO transport_scopes (name, transport_key, key_fingerprint, imported_only)
SELECT entry.name, entry.key, entry.fingerprint, TRUE
FROM (SELECT unnest(@names::text[]) AS name, unnest(@keys::bytea[]) AS key,
unnest(@fingerprints::bytea[]) AS fingerprint) AS entry
ON CONFLICT (name) DO NOTHING
)
INSERT INTO meshmapper_scope_catalogues (iata, url, payload, etag, checked_at, attempted_at, next_attempt, last_error)
VALUES (@iata, @url, sqlc.narg(payload)::jsonb, sqlc.narg(etag)::text,
sqlc.narg(checked_at)::timestamptz, @attempted_at, @next_attempt, @last_error)
ON CONFLICT (iata, url) DO UPDATE SET
payload = COALESCE(EXCLUDED.payload, meshmapper_scope_catalogues.payload),
etag = COALESCE(EXCLUDED.etag, meshmapper_scope_catalogues.etag),
checked_at = COALESCE(EXCLUDED.checked_at, meshmapper_scope_catalogues.checked_at),
attempted_at = EXCLUDED.attempted_at,
next_attempt = EXCLUDED.next_attempt,
last_error = EXCLUDED.last_error;
+3 -2
View File
@@ -49,10 +49,11 @@ VALUES ($1, $2, $3, $4)
ON CONFLICT (name) DO UPDATE SET
display_name = EXCLUDED.display_name,
transport_key = EXCLUDED.transport_key,
key_fingerprint = EXCLUDED.key_fingerprint;
key_fingerprint = EXCLUDED.key_fingerprint,
imported_only = FALSE;
-- name: GetTransportScopes :many
SELECT name, transport_key, key_fingerprint FROM transport_scopes ORDER BY name;
SELECT name, transport_key, key_fingerprint FROM transport_scopes WHERE NOT imported_only ORDER BY name;
-- name: GetTransportScopeByName :one
SELECT id FROM transport_scopes WHERE name = $1;
+94
View File
@@ -0,0 +1,94 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.31.1
// source: meshmapper.sql
package db
import (
"context"
"github.com/jackc/pgx/v5/pgtype"
)
const getScopeCatalogue = `-- name: GetScopeCatalogue :one
SELECT iata, url, payload, etag, checked_at, attempted_at, next_attempt, last_error FROM meshmapper_scope_catalogues WHERE iata = $1 AND url = $2
`
type GetScopeCatalogueParams struct {
Iata string `json:"iata"`
Url string `json:"url"`
}
// Copyright 2026 Beacon Contributors
// SPDX-License-Identifier: AGPL-3.0-or-later
func (q *Queries) GetScopeCatalogue(ctx context.Context, arg GetScopeCatalogueParams) (MeshmapperScopeCatalogue, error) {
row := q.db.QueryRow(ctx, getScopeCatalogue, arg.Iata, arg.Url)
var i MeshmapperScopeCatalogue
err := row.Scan(
&i.Iata,
&i.Url,
&i.Payload,
&i.Etag,
&i.CheckedAt,
&i.AttemptedAt,
&i.NextAttempt,
&i.LastError,
)
return i, err
}
const saveScopeCatalogue = `-- name: SaveScopeCatalogue :exec
WITH inserted AS (
INSERT INTO transport_scopes (name, transport_key, key_fingerprint, imported_only)
SELECT entry.name, entry.key, entry.fingerprint, TRUE
FROM (SELECT unnest($9::text[]) AS name, unnest($10::bytea[]) AS key,
unnest($11::bytea[]) AS fingerprint) AS entry
ON CONFLICT (name) DO NOTHING
)
INSERT INTO meshmapper_scope_catalogues (iata, url, payload, etag, checked_at, attempted_at, next_attempt, last_error)
VALUES ($1, $2, $3::jsonb, $4::text,
$5::timestamptz, $6, $7, $8)
ON CONFLICT (iata, url) DO UPDATE SET
payload = COALESCE(EXCLUDED.payload, meshmapper_scope_catalogues.payload),
etag = COALESCE(EXCLUDED.etag, meshmapper_scope_catalogues.etag),
checked_at = COALESCE(EXCLUDED.checked_at, meshmapper_scope_catalogues.checked_at),
attempted_at = EXCLUDED.attempted_at,
next_attempt = EXCLUDED.next_attempt,
last_error = EXCLUDED.last_error
`
type SaveScopeCatalogueParams struct {
Iata string `json:"iata"`
Url string `json:"url"`
Payload []byte `json:"payload"`
Etag *string `json:"etag"`
CheckedAt pgtype.Timestamptz `json:"checked_at"`
AttemptedAt pgtype.Timestamptz `json:"attempted_at"`
NextAttempt pgtype.Timestamptz `json:"next_attempt"`
LastError string `json:"last_error"`
Names []string `json:"names"`
Keys [][]byte `json:"keys"`
Fingerprints [][]byte `json:"fingerprints"`
}
// One statement commits the validated snapshot and its lookup identities together.
// Empty arrays insert nothing. NULL payload/checked_at retain last-known-good data
// after an error or 304. Imported names never replace existing manual metadata.
func (q *Queries) SaveScopeCatalogue(ctx context.Context, arg SaveScopeCatalogueParams) error {
_, err := q.db.Exec(ctx, saveScopeCatalogue,
arg.Iata,
arg.Url,
arg.Payload,
arg.Etag,
arg.CheckedAt,
arg.AttemptedAt,
arg.NextAttempt,
arg.LastError,
arg.Names,
arg.Keys,
arg.Fingerprints,
)
return err
}
+29
View File
@@ -714,6 +714,21 @@ func (mr *MockQuerierMockRecorder) GetScopeByName(ctx, name any) *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetScopeByName", reflect.TypeOf((*MockQuerier)(nil).GetScopeByName), ctx, name)
}
// GetScopeCatalogue mocks base method.
func (m *MockQuerier) GetScopeCatalogue(ctx context.Context, arg db.GetScopeCatalogueParams) (db.MeshmapperScopeCatalogue, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "GetScopeCatalogue", ctx, arg)
ret0, _ := ret[0].(db.MeshmapperScopeCatalogue)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// GetScopeCatalogue indicates an expected call of GetScopeCatalogue.
func (mr *MockQuerierMockRecorder) GetScopeCatalogue(ctx, arg any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetScopeCatalogue", reflect.TypeOf((*MockQuerier)(nil).GetScopeCatalogue), ctx, arg)
}
// GetScopeNames mocks base method.
func (m *MockQuerier) GetScopeNames(ctx context.Context) ([]string, error) {
m.ctrl.T.Helper()
@@ -1542,6 +1557,20 @@ func (mr *MockQuerierMockRecorder) ResolvePathHashesP4(ctx, arg any) *gomock.Cal
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResolvePathHashesP4", reflect.TypeOf((*MockQuerier)(nil).ResolvePathHashesP4), ctx, arg)
}
// SaveScopeCatalogue mocks base method.
func (m *MockQuerier) SaveScopeCatalogue(ctx context.Context, arg db.SaveScopeCatalogueParams) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "SaveScopeCatalogue", ctx, arg)
ret0, _ := ret[0].(error)
return ret0
}
// SaveScopeCatalogue indicates an expected call of SaveScopeCatalogue.
func (mr *MockQuerierMockRecorder) SaveScopeCatalogue(ctx, arg any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveScopeCatalogue", reflect.TypeOf((*MockQuerier)(nil).SaveScopeCatalogue), ctx, arg)
}
// SearchKnownRoutes mocks base method.
func (m *MockQuerier) SearchKnownRoutes(ctx context.Context, arg db.SearchKnownRoutesParams) ([]db.SearchKnownRoutesRow, error) {
m.ctrl.T.Helper()
+12
View File
@@ -234,6 +234,17 @@ type KnownRoute struct {
LastReconfirmedAt pgtype.Timestamptz `json:"last_reconfirmed_at"`
}
type MeshmapperScopeCatalogue struct {
Iata string `json:"iata"`
Url string `json:"url"`
Payload []byte `json:"payload"`
Etag *string `json:"etag"`
CheckedAt pgtype.Timestamptz `json:"checked_at"`
AttemptedAt pgtype.Timestamptz `json:"attempted_at"`
NextAttempt pgtype.Timestamptz `json:"next_attempt"`
LastError string `json:"last_error"`
}
type MvHourlyIataStat struct {
Iata string `json:"iata"`
Hour pgtype.Timestamptz `json:"hour"`
@@ -528,4 +539,5 @@ type TransportScope struct {
TransportKey []byte `json:"transport_key"`
KeyFingerprint []byte `json:"key_fingerprint"`
CreatedAt pgtype.Timestamptz `json:"created_at"`
ImportedOnly bool `json:"imported_only"`
}
+7
View File
@@ -89,6 +89,9 @@ type Querier interface {
GetRegionIATAs(ctx context.Context, regionID int32) ([]string, error)
GetRouteEvidenceRoute(ctx context.Context, arg GetRouteEvidenceRouteParams) (KnownRoute, error)
GetScopeByName(ctx context.Context, name string) (GetScopeByNameRow, error)
// Copyright 2026 Beacon Contributors
// SPDX-License-Identifier: AGPL-3.0-or-later
GetScopeCatalogue(ctx context.Context, arg GetScopeCatalogueParams) (MeshmapperScopeCatalogue, error)
GetScopeNames(ctx context.Context) ([]string, error)
// Aggregate matching observations once, separately from node memberships to avoid
// a cross-join. Empty IATAs keep the original global counts, including associations
@@ -250,6 +253,10 @@ type Querier interface {
ResolvePathHashesP2(ctx context.Context, arg ResolvePathHashesP2Params) ([]ResolvePathHashesP2Row, error)
ResolvePathHashesP3(ctx context.Context, arg ResolvePathHashesP3Params) ([]ResolvePathHashesP3Row, error)
ResolvePathHashesP4(ctx context.Context, arg ResolvePathHashesP4Params) ([]ResolvePathHashesP4Row, error)
// One statement commits the validated snapshot and its lookup identities together.
// Empty arrays insert nothing. NULL payload/checked_at retain last-known-good data
// after an error or 304. Imported names never replace existing manual metadata.
SaveScopeCatalogue(ctx context.Context, arg SaveScopeCatalogueParams) error
// Returns known routes containing a subsequence from source to destination hash prefix.
// Verifies source appears before destination in the route.
SearchKnownRoutes(ctx context.Context, arg SearchKnownRoutesParams) ([]SearchKnownRoutesRow, error)
+3 -2
View File
@@ -2051,7 +2051,7 @@ func (q *Queries) GetTransportScopeByName(ctx context.Context, name string) (int
}
const getTransportScopes = `-- name: GetTransportScopes :many
SELECT name, transport_key, key_fingerprint FROM transport_scopes ORDER BY name
SELECT name, transport_key, key_fingerprint FROM transport_scopes WHERE NOT imported_only ORDER BY name
`
type GetTransportScopesRow struct {
@@ -5228,7 +5228,8 @@ VALUES ($1, $2, $3, $4)
ON CONFLICT (name) DO UPDATE SET
display_name = EXCLUDED.display_name,
transport_key = EXCLUDED.transport_key,
key_fingerprint = EXCLUDED.key_fingerprint
key_fingerprint = EXCLUDED.key_fingerprint,
imported_only = FALSE
`
type UpsertTransportScopeParams struct {