dandri/beacon-server
1
0
Fork 0
mirror of https://github.com/MeshCore-Beacon/beacon-server.git synced 2026-10-20 07:31:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge upstream MeshMapper discovery into the 2.0 experiment

Browse Source
This commit is contained in:
n30nex
2026-10-01 20:33:12 -04:00
parent 835d1cdd48 6cd03e73da
commit 7891ccd687
70 changed files with 4186 additions and 795 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+24 -3
View File
@@ -192,6 +192,19 @@ even when the same bearer-authenticated request works with curl.
### Config file (`config.yaml`)
```yaml
# MeshMapper integrations (optional, no API key). Import transport scopes and
# IATA border outlines from MeshMapper instead of maintaining scopes: and
# iatas.*.borderFile by hand. Both cover every known IATA that MeshMapper lists;
# no sources to configure. See config.yaml.example for limits and behaviour.
#meshmapper:
# scopes:
# enabled: true
# zones:
# enabled: true
# import_groups: true # MeshMapper zone groups become regions
# channels:
# enabled: true # import each region's public hashtag channels
# Optional IATA overrides — auto-created on first packet arrival,
# only needed if you want to customise display name or coordinates.
# borderFile points to a GeoJSON Feature (Polygon or MultiPolygon) for the
@@ -202,7 +215,7 @@ iatas:
name: Vancouver International
lat: 49.1967
lng: -123.1815
borderFile: borders/yvr.geojson # optional
borderFile: borders/yvr.geojson # optional; or use meshmapper.zones
# Super-regions grouping multiple IATAs.
regions:
@@ -218,9 +231,12 @@ regions:
channel_keys:
# Hashtag channels: Beacon derives the PSK from the tag name automatically.
# secret = SHA256("#tag")[:16], channel_hash = SHA256(secret)[0]
# Tag names should be provided without the # prefix.
# Tag names should be provided without the # prefix. Plain names show under
# every region; {name, region: <slug>} limits a channel to one region.
hashtags:
- meshcore
- name: vancouver-mesh
region: western-canada
# Explicit keys: channel hash (hex) and key (hex), with optional display name.
# The public MeshCore channel key is included in config.yaml.example.
@@ -231,9 +247,14 @@ channel_keys:
# Regional transport scopes for matching TRANSPORT_FLOOD packets.
# Plain names have # prepended automatically (e.g. "bc" → "#bc").
# region (required) is a configured region slug; region-filtered scope lists
# and scope stats show the scope under that region's IATAs. Matching stays global.
# Optional when meshmapper.scopes covers your regions.
scopes:
- name: bc
region: western-canada
- name: "#west"
region: western-canada
# Observer telemetry storage settings.
telemetry:
@@ -500,7 +521,7 @@ Not yet implemented — see the Authentication section above.
| `GET` | `/routes` | List known routes (all hops high confidence) |
| `GET` | `/routes/search` | Search routes by source and destination hash |
| `GET` | `/routes/cross` | Search for routes crossing IATA boundaries |
| `GET` | `/scopes` | List transport scopes |
| `GET` | `/scopes` | List transport scope names; IATA/region filters use configured regions and MeshMapper catalogues |
| `GET` | `/scopes/{name}` | Get scope detail |
| `GET` | `/stats/observations` | Hourly observation time series (last 7 days by default) |
| `GET` | `/stats/overview` | Network overview stats |
cmd/beacon/main.go
+50 -4
View File
@@ -45,7 +45,7 @@ import (
var version = "dev"
// @title MeshCore Beacon API
// @version 1.6.0
// @version 2.0.0
// @description MeshCore network observation backend. Ingests LoRa packets from MQTT brokers, stores in PostgreSQL, and streams live events via WebSocket.
// @description REST requests share a configurable per-client rate limit (default 300/minute and a 300-request one-second burst cap). Exceeded limits return HTTP 429 with error.code=rate_limited and a Retry-After header in seconds. CORS preflights and WebSocket upgrades do not consume this API budget.
// @description WebSocket upgrade attempts at /ws have a configurable per-client limit (default 10/minute, including failed handshakes). Rate exhaustion returns HTTP 429 with Retry-After before upgrade. An accepted socket exceeding the concurrent cap closes with code 1013 before hello; established connections remain open.
@@ -196,11 +196,22 @@ func main() {
os.Exit(1)
}
scopes.Load(scopeEntries)
scopes.SetManualMembers(cfg.ManualScopeMembers())
slog.Info(fmt.Sprintf("loaded %d transport scopes", len(scopeEntries)), "component", "startup")
directory := meshmapper.NewDirectory(store)
if cfg.MeshMapper.Scopes.Enabled || cfg.MeshMapper.Zones.Enabled || cfg.MeshMapper.Channels.Enabled {
restoreCtx, cancelRestore := context.WithTimeout(ctx, 10*time.Second)
err = directory.Restore(restoreCtx)
cancelRestore()
if err != nil {
slog.Error("failed to restore MeshMapper zone lists", "component", "startup", "error", err)
os.Exit(1)
}
}
var scopeImporter *meshmapper.Importer
if cfg.MeshMapper.Scopes.Enabled {
restoreCtx, cancelRestore := context.WithTimeout(ctx, 10*time.Second)
scopeImporter, err = meshmapper.New(restoreCtx, cfg.MeshMapper.Scopes, store, scopes, scopeEntries)
scopeImporter, err = meshmapper.New(restoreCtx, cfg.MeshMapper.Scopes, store, directory, scopes, scopeEntries)
cancelRestore()
if err != nil {
slog.Error("failed to restore MeshMapper scope catalogues", "component", "startup", "error", err)
@@ -209,9 +220,18 @@ func main() {
}
// Always restored, so disabling the import prunes it and file borders return.
zones := meshmapper.NewZones(cfg.MeshMapper.Zones, cfg.RegionIATAs(), store)
zones := meshmapper.NewZones(cfg.MeshMapper.Zones, store, directory)
var detailed []string
for iata, d := range cfg.IATAs {
if d.Name != "" || d.Lat != nil || d.Lng != nil {
detailed = append(detailed, iata)
}
}
zones.SetConfiguredIATAs(detailed)
if cr, ok := reader.(*cache.CachedReader); ok {
zones.OnChange(cr.InvalidateIATABorder)
zones.OnRegionsChange(cr.InvalidateRegions)
zones.OnIATAsChange(cr.InvalidateIATAs)
}
if localBorders != nil {
zones.OnUpdate(func(imported map[string]json.RawMessage) {
@@ -238,7 +258,8 @@ func main() {
entries := make(map[string][]keystore.Entry)
// Hashtag-derived channels: secret = SHA256("#tag")[:16], hash = SHA256(secret)[0]
for _, tag := range cfg.ChannelKeys.Hashtags {
for _, hashtag := range cfg.ChannelKeys.Hashtags {
tag := hashtag.Name
secret, channelHash, fingerprint := keystore.DeriveHashtagKey(tag)
hashHex := fmt.Sprintf("%02x", channelHash)
entry := keystore.Entry{
@@ -273,6 +294,27 @@ func main() {
keys := keystore.NewMapKeyStore(entries)
// Restored before the boot backfill so saved MeshMapper keys decrypt history too.
restoreCtx, cancelRestore = context.WithTimeout(ctx, 10*time.Second)
channelImporter, err := meshmapper.NewChannels(restoreCtx, cfg.MeshMapper.Channels, store, directory, keys)
cancelRestore()
if err != nil {
slog.Error("failed to restore MeshMapper channels", "component", "startup", "error", err)
os.Exit(1)
}
channelImporter.OnNewKeys(func(hashes [][]byte) {
// Off the refresh tick: a backfill can outlast it, and history isn't urgent.
go func() {
backfillCtx, cancel := context.WithTimeout(ctx, 10*time.Minute)
defer cancel()
if n, err := ingest.BackfillChannelHashes(backfillCtx, store, keys, hashes); err != nil {
slog.Error("MeshMapper channel backfill failed", "component", "meshmapper.channels", "error", err)
} else if n > 0 {
slog.Info(fmt.Sprintf("backfilled %d channel message(s) for imported keys", n), "component", "meshmapper.channels")
}
}()
})
// ── Backfill channel messages ────────────────────────────────────────────
// Packets whose channel key wasn't yet configured at ingest time were stored as
// hash-only channels and never decrypted. Retry them now against the keystore we just
@@ -354,6 +396,9 @@ func main() {
if cfg.MeshMapper.Zones.Enabled {
tasks = append(tasks, background.Task{Name: "meshmapper.zones", Interval: meshmapper.PollInterval, Run: zones.Refresh})
}
if cfg.MeshMapper.Channels.Enabled {
tasks = append(tasks, background.Task{Name: "meshmapper.channels", Interval: meshmapper.PollInterval, Run: channelImporter.Refresh})
}
profiles := configureProfiling(ctx, pool)
defer profiles.Stop()
for i := range tasks {
@@ -372,6 +417,7 @@ func main() {
MaxConnectsPerMinute: resolved.MaxConnectsPerMinute,
WSAllowedOrigins: cfg.WebSocket.AllowedOrigins,
CORS: cfg.CORS, Server: cfg.Server, Auth: cfg.Auth, RateLimit: resolved.RateLimit,
Scopes: scopes,
AdminRoutes: map[string]http.Handler{
"/accounts": handlers.AccountsRouter(store),
"/backup": handlers.BackupRouter(backupOpts, ctx),
config.yaml.example
+72 -26
View File
@@ -41,13 +41,72 @@ ratelimit:
# A 429 returns error.code=rate_limited and Retry-After (1 or 60 seconds).
burst: 300
# MeshMapper integrations (optional, no API key). Import transport scopes and
# IATA border outlines from MeshMapper instead of maintaining scopes: and
# iatas.*.borderFile by hand.
#
# Scope-name discovery. Manual scopes remain authoritative.
# Every known IATA, including ones auto-created from traffic, is looked up in its
# country's zone list (shared with zones below) and its site's get_scopes.php
# catalogue is imported; IATAs MeshMapper doesn't list are skipped. No sources
# to configure; a legacy sources: list is ignored with a warning.
# Catalogues covering more than one IATA are rejected.
# 64 names per catalogue, 64 KiB per response; no page scraping.
# Catalogues and ETags persist in PostgreSQL. Failed refreshes retain known names.
# Refresh success/failure/freshness is logged under component=meshmapper.scopes.
# Turning scopes off deactivates imported matching keys on restart;
# historical scope identities and recorded evidence are retained.
# Region-filtered scope lists and scope stats show imported names only under the
# IATAs whose catalogue lists them; IATAs without a catalogue show manual scopes only.
#meshmapper:
# scopes:
# enabled: false
# refresh_interval: 1h # 1h-24h (MeshMapper allows one get_scopes.php per region per 55m); one source per 15s tick
#
# Region boundaries (Zones API). Every known IATA, including ones auto-created
# from traffic, is looked up in its country's zone list, then its get_geojson.php
# outline is imported; new IATAs are picked up while running. An imported boundary
# overrides that IATA's borderFile on the border map and for nodes.mark_foreign.
# A missing or failed boundary keeps the last good import, or the borderFile when
# there is none. Disabling drops every import on restart. Logged under
# component=meshmapper.zones.
# Each country's zone list is fetched at most once a day (MeshMapper allows one
# get_zones.php per country per 23.5h) and saved, so restarts don't refetch it.
# Failed or rate-limited requests also wait out the limit, or Retry-After if longer.
# Listed IATAs also take their display name and approximate location from the
# zone list, unless iatas: below sets a name or lat/lng for them.
# import_groups turns MeshMapper zone groups (e.g. ONQC: YOW, YQB, YUL) into
# regions, from the zone lists already fetched. A group is imported when at least
# one member is a known IATA, with its full member list (members Beacon hasn't
# heard yet are added as IATAs); slug = lowercased group code, centered on the
# members' average location, listed after the regions below. A region below with
# the same slug wins.
# Imported regions are removed when the group disappears or stops qualifying,
# and all of them when import_groups is turned off. Requires zones.enabled.
# zones:
# enabled: false
# import_groups: false
# refresh_interval: 24h # 24h-168h (one get_geojson.php per region per 23.5h); one request per 15s tick, ETag-conditional
#
# Public hashtag channels (get_channels.php). Every known IATA that MeshMapper
# lists is looked up on its site, the same way as scopes. Each key and hash is
# checked against the name before use; a list with any mismatch is rejected and
# the last good one kept. Imported channels decrypt like configured ones (history
# is backfilled when a key first appears) and are listed under that IATA when
# /channels is filtered by region. Saved in PostgreSQL across restarts; failures
# wait out the 23.5h limit. Turning it off removes the region listing on restart.
# Logged under component=meshmapper.channels.
# channels:
# enabled: false
# refresh_interval: 24h # 24h-168h (one get_channels.php per region per 23.5h)
iatas:
YVR:
name: Vancouver International
lat: 49.1967
lng: -123.1815
# borderFile: borders/yvr.geojson # optional GeoJSON Feature (Polygon/MultiPolygon)
# # (meshmapper.zones overrides it when enabled)
# # (or use meshmapper.zones above)
# # for the region border map; path is relative to
# # this file's directory; validated at boot.
YYJ:
@@ -71,6 +130,8 @@ iatas:
lat: 43.6777
lng: -79.6248
# Hand-written regions; optional with meshmapper.zones.import_groups. Seeding
# sets each region's members to exactly this list.
regions:
- slug: western-canada
name: Western Canada
@@ -93,8 +154,12 @@ channel_keys:
# secret = SHA256("#tag")[:16], channel_hash = SHA256(secret)[0]
# Hashtag names should be provided without the # prefix.
# Beacon prepends # automatically before deriving the key.
# A plain name is listed under every region; add region: <slug> (a configured
# region) to list it only there when /channels is filtered by region.
hashtags:
- meshcore
# - name: ottawa-mesh
# region: eastern-canada
# Explicit keys: provide the channel hash (hex) and key (hex) directly.
# An optional name can be set for display purposes.
@@ -102,12 +167,18 @@ channel_keys:
"11":
key: "8b3387e9c5cdea6ac9e5edbaa115cd72"
name: "Public"
# region: eastern-canada # optional, as for hashtags
# Regional transport scopes for matching TRANSPORT_FLOOD packets.
# Plain names have # prepended automatically (e.g. "bc" → "#bc").
# region (required) is a configured region slug; region-filtered scope lists
# and scope stats show the scope under that region's IATAs. Matching stays global.
# Optional when meshmapper.scopes covers your regions.
scopes:
- name: bc
region: western-canada
- name: "#west"
region: western-canada
telemetry:
retention: 672h # 4 weeks
@@ -216,31 +287,6 @@ cache:
# allow_countries: [CA, US]
# allow_continents: [NA]
# Optional public MeshMapper scope-name discovery. Manual scopes remain authoritative.
# No API key is needed. Sources must belong to a configured region's IATAs.
# Group endpoints are not supported: their names cannot be attributed to each IATA.
# At most 16 sources, 64 names per source, 64 KiB per response; no page scraping.
# Catalogues and ETags persist in PostgreSQL. Failed refreshes retain known names.
# Refresh success/failure/freshness is logged under component=meshmapper.scopes.
# Removing/turning off a source deactivates its imported matching keys on restart;
# historical scope identities and recorded evidence are retained.
#meshmapper:
# scopes:
# enabled: false
# refresh_interval: 1h # 5m-24h; one source checked per 15s tick
# sources:
# YOW: https://yow.meshmapper.net/get_scopes.php
#
# Optional MeshMapper region boundaries (Zones API). Every IATA in a configured
# region is looked up in its country's zone list, then its get_geojson.php outline
# is imported. An imported boundary overrides that IATA's borderFile on the border
# map and for nodes.mark_foreign. A missing or failed boundary keeps the last good
# import, or the borderFile when there is none. Disabling, or removing an IATA from
# every region, drops its import on restart. Logged under component=meshmapper.zones.
# zones:
# enabled: false
# refresh_interval: 24h # 1h-168h; one request per 15s tick, ETag-conditional
# Background task intervals.
# Shorter intervals are useful during initial deployment to confirm data is
# flowing. Back off to 1h or more once stable.
db/channel_cursor_integration_test.go
+13 -7
View File
@@ -57,13 +57,14 @@ func TestChannelCursorPostgres(t *testing.T) {
}
}
applyBaseline(t, ctx, tx)
exec(`INSERT INTO channels(id,channel_hash,key_fingerprint,last_seen) VALUES
(1,'\xaa','\x01','2026-09-08 12:00:00+00'),(2,'\xaa','\x02','2026-09-08 12:00:00+00'),
(3,'\xaa','\x03','2026-09-08 12:00:00+00'),(4,'\xbb','\x04','2026-09-08 11:59:59+00'),
(5,'\xcc','\x05','2026-09-08 12:00:00.000321+00'),
(6,'\xcc','\x06','2026-09-08 12:00:00.000111+00'),(7,'\xcc','\x07','2026-09-08 12:00:00.000111+00');
INSERT INTO channel_iatas(channel_hash,iata,last_heard) VALUES
('\xaa','YOW',now()),('\xbb','YOW',now()),('\xcc','YYZ',now());`)
// NULL key_known is reported as false, so keyKnown=false must include it.
exec(`INSERT INTO channels(id,channel_hash,key_fingerprint,last_seen,key_known) VALUES
(1,'\xaa','\x01','2026-09-08 12:00:00+00',true),(2,'\xaa','\x02','2026-09-08 12:00:00+00',false),
(3,'\xaa','\x03','2026-09-08 12:00:00+00',NULL),(4,'\xbb','\x04','2026-09-08 11:59:59+00',true),
(5,'\xcc','\x05','2026-09-08 12:00:00.000321+00',true),
(6,'\xcc','\x06','2026-09-08 12:00:00.000111+00',false),(7,'\xcc','\x07','2026-09-08 12:00:00.000111+00',true);
INSERT INTO meshmapper_channel_members(iata,key_fingerprint) VALUES
('YOW','\x01'),('YOW','\x02'),('YOW','\x03'),('YOW','\x04'),('YYZ','\x05'),('YYZ','\x06'),('YYZ','\x07');`)
queries := &channelCursorQueries{DBTX: tx}
router := handlers.ChannelsRouter(&Store{q: sqlc.New(queries)})
// Decode the wire contract so the same regression runs against the old server.
@@ -97,6 +98,11 @@ INSERT INTO channel_iatas(channel_hash,iata,last_heard) VALUES
{"hash and region", "hash=cc&iata=yyz", []int{5, 7, 6}},
{"no regional match", "hash=aa&iata=yyz", nil},
{"missing region", "iata=ZZZ", nil},
{"key known", "keyKnown=true", []int{5, 7, 1, 4}},
{"key unknown", "keyKnown=false", []int{6, 3, 2}},
{"key known in region", "keyKnown=true&iata=yow", []int{1, 4}},
{"key unknown in region", "keyKnown=false&iata=yow", []int{3, 2}},
{"key unknown hash", "keyKnown=false&hash=cc", []int{6}},
} {
t.Run(tc.name, func(t *testing.T) {
params, _ := url.ParseQuery(tc.query)
db/channels.go
+28 -3
View File
@@ -65,6 +65,31 @@ func (s *Store) ListUndecryptedGroupTextPackets(ctx context.Context) ([]ingest.U
return packets, nil
}
// ListUndecryptedGroupTextPacketsByHash limits the boot backfill to channels that just gained a key.
func (s *Store) ListUndecryptedGroupTextPacketsByHash(ctx context.Context, hashes [][]byte) ([]ingest.UndecryptedPacket, error) {
rows, err := s.q.ListUndecryptedGroupTextPacketsByHash(ctx, hashes)
if err != nil {
return nil, err
}
packets := make([]ingest.UndecryptedPacket, 0, len(rows))
for _, v := range rows {
packets = append(packets, ingest.UndecryptedPacket{PacketHash: v.PacketHash, RawPayload: v.RawPayload})
}
return packets, nil
}
// SetChannelConfigScopes replaces every configured channel's region placement;
// an empty region means Beacon-wide.
func (s *Store) SetChannelConfigScopes(ctx context.Context, fingerprints [][]byte, regions []string) error {
if err := s.q.DeleteChannelConfigScopes(ctx); err != nil {
return err
}
if len(fingerprints) == 0 {
return nil
}
return s.q.AddChannelConfigScopes(ctx, sqlc.AddChannelConfigScopesParams{Fingerprints: fingerprints, Regions: regions})
}
func (s *Store) UpsertChannelIATA(ctx context.Context, channelHash []byte, iata string, heardAt time.Time) error {
return s.q.UpsertChannelIATA(ctx, sqlc.UpsertChannelIATAParams{
ChannelHash: channelHash,
@@ -77,7 +102,7 @@ func (s *Store) DeleteOldChannelIATAs(ctx context.Context, cutoff time.Time) err
return s.q.DeleteOldChannelIATAs(ctx, pgtype.Timestamptz{Time: cutoff, Valid: true})
}
func (s *Store) ListChannels(ctx context.Context, limit int32, hash []byte, iatas []string, cursor int64, pageCursor *api.ChannelCursor) (api.ChannelPage, error) {
func (s *Store) ListChannels(ctx context.Context, limit int32, hash []byte, iatas []string, keyKnown *bool, cursor int64, pageCursor *api.ChannelCursor) (api.ChannelPage, error) {
var cursorTS pgtype.Timestamptz
if cursor > 0 {
cursorTS = pgtype.Timestamptz{Time: time.UnixMilli(cursor), Valid: true}
@@ -86,12 +111,12 @@ func (s *Store) ListChannels(ctx context.Context, limit int32, hash []byte, iata
var err error
if pageCursor != nil {
rows, err = s.q.ListChannelsAfter(ctx, sqlc.ListChannelsAfterParams{
ChannelHash: hash, Iatas: iatas, PageLimit: limit + 1,
ChannelHash: hash, Iatas: iatas, KeyKnown: keyKnown, PageLimit: limit + 1,
CursorTs: pgtype.Timestamptz{Time: pageCursor.LastSeen, Valid: true}, CursorID: pageCursor.ID,
})
} else {
rows, err = s.q.ListChannels(ctx, sqlc.ListChannelsParams{
ChannelHash: hash, Iatas: iatas, CursorTs: cursorTS, PageLimit: limit + 1,
ChannelHash: hash, Iatas: iatas, KeyKnown: keyKnown, CursorTs: cursorTS, PageLimit: limit + 1,
})
}
if err != nil {
db/channels_test.go
+30 -4
View File
@@ -11,6 +11,7 @@ import (
sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc"
mockdb "github.com/MeshCore-Beacon/beacon-server/db/sqlc/mock"
"github.com/MeshCore-Beacon/beacon-server/internal/api"
"github.com/jackc/pgx/v5/pgtype"
"go.uber.org/mock/gomock"
)
@@ -29,7 +30,7 @@ func TestListChannels_Empty(t *testing.T) {
Return([]sqlc.Channel{}, nil)
store := &Store{q: mock}
page, err := store.ListChannels(context.Background(), 10, nil, nil, 0, nil)
page, err := store.ListChannels(context.Background(), 10, nil, nil, nil, 0, nil)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -71,7 +72,7 @@ func TestListChannels_Pagination(t *testing.T) {
Return(rows, nil)
store := &Store{q: mock}
page, err := store.ListChannels(context.Background(), 2, nil, nil, 0, nil)
page, err := store.ListChannels(context.Background(), 2, nil, nil, nil, 0, nil)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -95,7 +96,7 @@ func TestListChannels_DBError(t *testing.T) {
Return(nil, errors.New("db error"))
store := &Store{q: mock}
_, err := store.ListChannels(context.Background(), 10, nil, nil, 0, nil)
_, err := store.ListChannels(context.Background(), 10, nil, nil, nil, 0, nil)
if err == nil {
t.Fatal("expected error, got nil")
}
@@ -115,12 +116,37 @@ func TestListChannels_IATAFilter(t *testing.T) {
Return([]sqlc.Channel{}, nil)
store := &Store{q: mock}
_, err := store.ListChannels(context.Background(), 10, nil, []string{"YOW", "YYZ"}, 0, nil)
_, err := store.ListChannels(context.Background(), 10, nil, []string{"YOW", "YYZ"}, nil, 0, nil)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
func TestListChannels_KeyKnownFilter(t *testing.T) {
ctrl := gomock.NewController(t)
mock := mockdb.NewMockQuerier(ctrl)
keyKnown := false
at := time.UnixMilli(1700000000000)
mock.EXPECT().
ListChannels(gomock.Any(), sqlc.ListChannelsParams{KeyKnown: &keyKnown, PageLimit: 11}).
Return([]sqlc.Channel{}, nil)
mock.EXPECT().
ListChannelsAfter(gomock.Any(), sqlc.ListChannelsAfterParams{
KeyKnown: &keyKnown, PageLimit: 11,
CursorTs: pgtype.Timestamptz{Time: at, Valid: true}, CursorID: 7,
}).
Return([]sqlc.Channel{}, nil)
store := &Store{q: mock}
if _, err := store.ListChannels(context.Background(), 10, nil, nil, &keyKnown, 0, nil); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if _, err := store.ListChannels(context.Background(), 10, nil, nil, &keyKnown, 0, &api.ChannelCursor{LastSeen: at, ID: 7}); err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
func TestGetChannel_Basic(t *testing.T) {
ctrl := gomock.NewController(t)
mock := mockdb.NewMockQuerier(ctrl)
db/config.go
+9 -5
View File
@@ -166,9 +166,13 @@ func (s *Store) GetRegionBySlug(ctx context.Context, slug string) (*api.Region,
return &result, nil
}
func (s *Store) UpsertRegionIATA(ctx context.Context, regionID int32, iata string) error {
return s.q.UpsertRegionIATA(ctx, sqlc.UpsertRegionIATAParams{
RegionID: regionID,
Iata: iata,
})
// SetRegionIATAs makes iatas the region's exact member list.
func (s *Store) SetRegionIATAs(ctx context.Context, regionID int32, iatas []string) error {
if iatas == nil {
iatas = []string{}
}
if err := s.q.DeleteRegionIATAsNotIn(ctx, sqlc.DeleteRegionIATAsNotInParams{RegionID: regionID, Keep: iatas}); err != nil {
return err
}
return s.q.AddRegionIATAs(ctx, sqlc.AddRegionIATAsParams{RegionID: regionID, Iatas: iatas})
}
db/meshmapper.go
+165
View File
@@ -29,6 +29,23 @@ func (s *Store) GetScopeCatalogue(ctx context.Context, iata, url string) (*meshm
return cache, nil
}
func (s *Store) ListScopeCatalogues(ctx context.Context) ([]meshmapper.Catalogue, error) {
rows, err := s.q.ListScopeCatalogues(ctx)
if err != nil {
return nil, err
}
out := make([]meshmapper.Catalogue, 0, len(rows))
for _, row := range rows {
c := meshmapper.Catalogue{IATA: row.Iata, URL: row.Url, Cache: meshmapper.Cache{Payload: row.Payload, CheckedAt: row.CheckedAt.Time,
AttemptedAt: row.AttemptedAt.Time, NextAttempt: row.NextAttempt.Time, LastError: row.LastError}}
if row.Etag != nil {
c.ETag = *row.Etag
}
out = append(out, c)
}
return out, nil
}
func (s *Store) SaveScopeCatalogue(ctx context.Context, iata, url string, cache meshmapper.Cache, entries []scopestore.Entry) error {
params := sqlc.SaveScopeCatalogueParams{
Iata: iata, Url: url, Payload: cache.Payload, LastError: cache.LastError,
@@ -48,6 +65,18 @@ func (s *Store) SaveScopeCatalogue(ctx context.Context, iata, url string, cache
return s.q.SaveScopeCatalogue(ctx, params)
}
func (s *Store) ListKnownIATAs(ctx context.Context) ([]string, error) {
rows, err := s.q.ListIATAs(ctx)
if err != nil {
return nil, err
}
iatas := make([]string, 0, len(rows))
for _, row := range rows {
iatas = append(iatas, row.Iata)
}
return iatas, nil
}
func (s *Store) PruneZoneBoundaries(ctx context.Context, keep []string) ([]string, error) {
if keep == nil {
keep = []string{}
@@ -84,3 +113,139 @@ func (s *Store) SaveZoneBoundary(ctx context.Context, b meshmapper.Boundary) err
}
return s.q.SaveZoneBoundary(ctx, params)
}
func (s *Store) ListZoneLists(ctx context.Context) ([]meshmapper.ZoneList, error) {
rows, err := s.q.ListZoneLists(ctx)
if err != nil {
return nil, err
}
out := make([]meshmapper.ZoneList, 0, len(rows))
for _, row := range rows {
l := meshmapper.ZoneList{Country: row.Country, Payload: row.Payload, FetchedAt: row.FetchedAt.Time,
AttemptedAt: row.AttemptedAt.Time, NextAttempt: row.NextAttempt.Time, LastError: row.LastError}
if row.Etag != nil {
l.ETag = *row.Etag
}
out = append(out, l)
}
return out, nil
}
func (s *Store) SaveZoneList(ctx context.Context, l meshmapper.ZoneList) error {
params := sqlc.SaveZoneListParams{
Country: l.Country, Payload: l.Payload, LastError: l.LastError,
FetchedAt: pgtype.Timestamptz{Time: l.FetchedAt, Valid: !l.FetchedAt.IsZero()},
AttemptedAt: pgtype.Timestamptz{Time: l.AttemptedAt, Valid: true},
NextAttempt: pgtype.Timestamptz{Time: l.NextAttempt, Valid: true},
}
if !l.FetchedAt.IsZero() {
params.Etag = &l.ETag
}
return s.q.SaveZoneList(ctx, params)
}
func (s *Store) ListRegionState(ctx context.Context) ([]meshmapper.RegionState, error) {
rows, err := s.q.ListRegionState(ctx)
if err != nil {
return nil, err
}
out := make([]meshmapper.RegionState, 0, len(rows))
for _, row := range rows {
out = append(out, meshmapper.RegionState{Slug: row.Slug, Name: row.Name, DisplayOrder: int(row.DisplayOrder),
Imported: row.Imported, IATAs: row.Iatas, CenterLat: row.CenterLat, CenterLng: row.CenterLng})
}
return out, nil
}
// SaveImportedRegion reports false when a hand-written region owns the slug.
func (s *Store) SaveImportedRegion(ctx context.Context, r meshmapper.RegionState) (bool, error) {
order := int32(r.DisplayOrder)
id, err := s.q.UpsertImportedRegion(ctx, sqlc.UpsertImportedRegionParams{Slug: r.Slug, Name: r.Name, DisplayOrder: &order,
CenterLat: r.CenterLat, CenterLng: r.CenterLng})
if errors.Is(err, pgx.ErrNoRows) {
return false, nil
}
if err != nil {
return false, err
}
// region_iatas references iata_codes, so members Beacon hasn't heard yet are created first, as Seed does.
if err := s.q.AddIATAs(ctx, r.IATAs); err != nil {
return false, err
}
return true, s.SetRegionIATAs(ctx, id, r.IATAs)
}
func (s *Store) PruneImportedRegions(ctx context.Context, keep []string) ([]string, error) {
if keep == nil {
keep = []string{}
}
return s.q.PruneImportedRegions(ctx, keep)
}
func (s *Store) ListChannelCatalogues(ctx context.Context) ([]meshmapper.Catalogue, error) {
rows, err := s.q.ListChannelCatalogues(ctx)
if err != nil {
return nil, err
}
out := make([]meshmapper.Catalogue, 0, len(rows))
for _, row := range rows {
c := meshmapper.Catalogue{IATA: row.Iata, URL: row.Url, Cache: meshmapper.Cache{Payload: row.Payload, CheckedAt: row.CheckedAt.Time,
AttemptedAt: row.AttemptedAt.Time, NextAttempt: row.NextAttempt.Time, LastError: row.LastError}}
if row.Etag != nil {
c.ETag = *row.Etag
}
out = append(out, c)
}
return out, nil
}
// SaveChannelCatalogue records a fetch; a new payload also replaces the IATA's
// channel members, which are written first so a failure never outlives its snapshot.
func (s *Store) SaveChannelCatalogue(ctx context.Context, iata, url string, cache meshmapper.Cache, fingerprints [][]byte) error {
if cache.Payload != nil {
if err := s.SetChannelMembers(ctx, iata, fingerprints); err != nil {
return err
}
}
params := sqlc.SaveChannelCatalogueParams{
Iata: iata, Url: url, Payload: cache.Payload, LastError: cache.LastError,
CheckedAt: pgtype.Timestamptz{Time: cache.CheckedAt, Valid: !cache.CheckedAt.IsZero()},
AttemptedAt: pgtype.Timestamptz{Time: cache.AttemptedAt, Valid: true},
NextAttempt: pgtype.Timestamptz{Time: cache.NextAttempt, Valid: true},
}
if !cache.CheckedAt.IsZero() {
params.Etag = &cache.ETag
}
return s.q.SaveChannelCatalogue(ctx, params)
}
// SetChannelMembers makes fingerprints the IATA's exact MeshMapper channel list.
func (s *Store) SetChannelMembers(ctx context.Context, iata string, fingerprints [][]byte) error {
if fingerprints == nil {
fingerprints = [][]byte{}
}
if err := s.q.DeleteChannelMembersNotIn(ctx, sqlc.DeleteChannelMembersNotInParams{Iata: iata, Keep: fingerprints}); err != nil {
return err
}
return s.q.AddChannelMembers(ctx, sqlc.AddChannelMembersParams{Iata: iata, Fingerprints: fingerprints})
}
func (s *Store) ClearChannelMembers(ctx context.Context) error {
return s.q.DeleteAllChannelMembers(ctx)
}
func (s *Store) ListIATADetails(ctx context.Context) ([]meshmapper.IATADetails, error) {
rows, err := s.q.ListIATAs(ctx)
if err != nil {
return nil, err
}
out := make([]meshmapper.IATADetails, 0, len(rows))
for _, row := range rows {
d := meshmapper.IATADetails{IATA: row.Iata, Lat: row.ApproxLat, Lng: row.ApproxLng}
if row.DisplayName != nil {
d.Name = *row.DisplayName
}
out = append(out, d)
}
return out, nil
}
db/meshmapper_integration_test.go
+232 -4
View File
@@ -6,6 +6,7 @@ package db
import (
"context"
"os"
"slices"
"strings"
"testing"
"time"
@@ -44,7 +45,7 @@ func TestMeshMapperCataloguePostgres(t *testing.T) {
t.Fatal(err)
}
defer tx.Rollback(context.Background())
for _, table := range []string{"transport_scopes", "meshmapper_scope_catalogues"} {
for _, table := range []string{"iata_codes", "transport_scopes", "meshmapper_scope_catalogues"} {
if _, err := tx.Exec(ctx, "CREATE TEMP TABLE "+table+" (LIKE public."+table+" INCLUDING ALL) ON COMMIT DROP"); err != nil {
t.Fatal(err)
}
@@ -55,6 +56,9 @@ func TestMeshMapperCataloguePostgres(t *testing.T) {
if err := store.UpsertTransportScope(ctx, manual.Name, "Operator label", manual.TransportKey, manual.KeyFingerprint); err != nil {
t.Fatal(err)
}
if err := store.UpsertIATA(ctx, "YOW"); err != nil {
t.Fatal(err)
}
url := "https://yow.meshmapper.net/get_scopes.php"
if row, err := store.GetScopeCatalogue(ctx, "YOW", url); err != nil || row != nil {
t.Fatal(row, err)
@@ -79,7 +83,7 @@ func TestMeshMapperCataloguePostgres(t *testing.T) {
t.Fatal(row, err)
}
scopes := scopestore.New()
_, err = meshmapper.New(ctx, config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": url}}, store, scopes, manualRows)
_, err = meshmapper.New(ctx, config.MeshMapperScopesConfig{Enabled: true}, store, meshmapper.NewDirectory(store), scopes, manualRows)
if err != nil || len(scopes.Entries()) != 2 {
t.Fatal("restart did not restore imported membership", err)
}
@@ -102,7 +106,7 @@ func TestMeshMapperCataloguePostgres(t *testing.T) {
if err := store.SaveScopeCatalogue(ctx, "YOW", url, update, nil); err != nil {
t.Fatal(err)
}
_, err = meshmapper.New(ctx, config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": url}}, store, scopes, manualRows)
_, err = meshmapper.New(ctx, config.MeshMapperScopesConfig{Enabled: true}, store, meshmapper.NewDirectory(store), scopes, manualRows)
if err != nil || len(scopes.Entries()) != 1 {
t.Fatal("removed imported membership remained active", err)
}
@@ -165,7 +169,7 @@ func TestMeshMapperZoneBoundariesPostgres(t *testing.T) {
t.Fatal(err)
}
defer tx.Rollback(context.Background())
for _, table := range []string{"iata_codes", "meshmapper_zone_boundaries"} {
for _, table := range []string{"iata_codes", "meshmapper_zone_boundaries", "meshmapper_zone_lists"} {
if _, err := tx.Exec(ctx, "CREATE TEMP TABLE "+table+" (LIKE public."+table+" INCLUDING ALL) ON COMMIT DROP"); err != nil {
t.Fatal(err)
}
@@ -222,4 +226,228 @@ func TestMeshMapperZoneBoundariesPostgres(t *testing.T) {
if !strings.Contains(border("YOW"), `"file"`) {
t.Fatal("manual border did not return after pruning")
}
list := meshmapper.ZoneList{Country: "CA", Payload: []byte(`{"country":"CA","zones":[]}`), ETag: `"z1"`,
FetchedAt: now, AttemptedAt: now, NextAttempt: now.Add(24 * time.Hour)}
if err := store.SaveZoneList(ctx, list); err != nil {
t.Fatal(err)
}
later := now.Add(25 * time.Hour)
if err := store.SaveZoneList(ctx, meshmapper.ZoneList{Country: "CA", AttemptedAt: later, NextAttempt: later.Add(48 * time.Hour), LastError: "HTTP 429"}); err != nil {
t.Fatal(err)
}
lists, err := store.ListZoneLists(ctx)
if err != nil || len(lists) != 1 || len(lists[0].Payload) == 0 || lists[0].ETag != `"z1"` || !lists[0].FetchedAt.Equal(now) ||
lists[0].LastError != "HTTP 429" || !lists[0].NextAttempt.Equal(later.Add(48*time.Hour)) {
t.Fatal("failed fetch lost the last good zone list", lists, err)
}
}
func TestMeshMapperRegionsPostgres(t *testing.T) {
dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN")
if dsn == "" {
t.Skip("set BEACON_TEST_POSTGRES_DSN")
}
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()
pool, err := pgxpool.New(ctx, dsn)
if err != nil {
t.Fatal(err)
}
if err := RunMigrations(ctx, pool); err != nil {
pool.Close()
t.Fatal(err)
}
pool.Close()
conn, err := pgx.Connect(ctx, dsn)
if err != nil {
t.Fatal(err)
}
defer conn.Close(context.Background())
tx, err := conn.Begin(ctx)
if err != nil {
t.Fatal(err)
}
defer tx.Rollback(context.Background())
// Real tables inside the rolled-back tx: TEMP copies drop region_iatas' foreign key to iata_codes.
store := &Store{q: sqlc.New(tx)}
state := func() map[string]meshmapper.RegionState {
t.Helper()
rows, err := store.ListRegionState(ctx)
if err != nil {
t.Fatal(err)
}
out := map[string]meshmapper.RegionState{}
for _, r := range rows {
out[r.Slug] = r
}
return out
}
for _, iata := range []string{"YOW", "YUL"} { // Seed creates config region IATAs first
if err := store.UpsertIATA(ctx, iata); err != nil {
t.Fatal(err)
}
}
hand, err := store.UpsertRegion(ctx, "east", "East", "", 3, nil, nil, nil)
if err != nil {
t.Fatal(err)
}
if err := store.SetRegionIATAs(ctx, hand, []string{"YOW", "YUL"}); err != nil {
t.Fatal(err)
}
if err := store.SetRegionIATAs(ctx, hand, []string{"YOW"}); err != nil {
t.Fatal(err)
}
if got := state()["east"]; got.Imported || got.DisplayOrder != 3 || strings.Join(got.IATAs, ",") != "YOW" {
t.Fatal("configured members not replaced", got)
}
lat, lng := 45.5, -75.0
group := meshmapper.RegionState{Slug: "onqc", Name: "Corridor", DisplayOrder: 4, IATAs: []string{"YOW", "YQB"}, CenterLat: &lat, CenterLng: &lng}
if saved, err := store.SaveImportedRegion(ctx, group); err != nil || !saved {
t.Fatal("members Beacon hasn't heard must be created first", saved, err)
}
if got := state()["onqc"]; got.CenterLat == nil || *got.CenterLat != lat || *got.CenterLng != lng {
t.Fatal("center not saved", got)
}
details, err := store.ListIATADetails(ctx)
if err != nil || !slices.ContainsFunc(details, func(d meshmapper.IATADetails) bool { return d.IATA == "YQB" }) {
t.Fatal("member IATA not created", err)
}
group.IATAs = []string{"YQB", "YUL"}
if saved, err := store.SaveImportedRegion(ctx, group); err != nil || !saved {
t.Fatal(saved, err)
}
if got := state()["onqc"]; !got.Imported || strings.Join(got.IATAs, ",") != "YQB,YUL" {
t.Fatal("imported region not updated", got)
}
if saved, err := store.SaveImportedRegion(ctx, meshmapper.RegionState{Slug: "east", Name: "Group", IATAs: []string{"YYZ"}}); err != nil || saved {
t.Fatal("imported group replaced a configured region", saved, err)
}
if got := state()["east"]; got.Name != "East" || strings.Join(got.IATAs, ",") != "YOW" {
t.Fatal("configured region changed by a clash", got)
}
if _, err := store.UpsertRegion(ctx, "onqc", "Mine", "", 1, nil, nil, nil); err != nil {
t.Fatal(err)
}
if state()["onqc"].Imported {
t.Fatal("config did not take over an imported slug")
}
if _, err := store.SaveImportedRegion(ctx, meshmapper.RegionState{Slug: "golm", Name: "Lakes", IATAs: []string{"YYZ"}}); err != nil {
t.Fatal(err)
}
if err := store.UpsertIATADetails(ctx, "YQB", "Quebec City", &lat, &lng); err != nil {
t.Fatal(err)
}
if details, _ := store.ListIATADetails(ctx); !slices.ContainsFunc(details, func(d meshmapper.IATADetails) bool {
return d.IATA == "YQB" && d.Name == "Quebec City" && d.Lat != nil && *d.Lat == lat
}) {
t.Fatal("IATA details not saved")
}
if pruned, err := store.PruneImportedRegions(ctx, nil); err != nil || strings.Join(pruned, ",") != "golm" {
t.Fatal("prune touched configured regions", pruned, err)
}
}
func TestChannelRegionScopingPostgres(t *testing.T) {
dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN")
if dsn == "" {
t.Skip("set BEACON_TEST_POSTGRES_DSN")
}
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()
pool, err := pgxpool.New(ctx, dsn)
if err != nil {
t.Fatal(err)
}
if err := RunMigrations(ctx, pool); err != nil {
pool.Close()
t.Fatal(err)
}
pool.Close()
conn, err := pgx.Connect(ctx, dsn)
if err != nil {
t.Fatal(err)
}
defer conn.Close(context.Background())
tx, err := conn.Begin(ctx)
if err != nil {
t.Fatal(err)
}
defer tx.Rollback(context.Background())
for _, table := range []string{"regions", "region_iatas", "channels", "channel_iatas", "channel_config_scopes",
"meshmapper_channel_members", "meshmapper_channel_catalogues"} {
if _, err := tx.Exec(ctx, "CREATE TEMP TABLE "+table+" (LIKE public."+table+" INCLUDING ALL) ON COMMIT DROP"); err != nil {
t.Fatal(err)
}
}
store := &Store{q: sqlc.New(tx)}
for slug, iata := range map[string]string{"east": "YOW", "west": "YYZ"} {
id, err := store.UpsertRegion(ctx, slug, slug, "", 0, nil, nil, nil)
if err != nil {
t.Fatal(err)
}
if err := store.SetRegionIATAs(ctx, id, []string{iata}); err != nil {
t.Fatal(err)
}
}
fp := func(b byte) []byte { return []byte{b, b, b, b, b, b, b, b} }
for name, b := range map[string]byte{"#mm-yow": 1, "#config-east": 2, "#global": 3, "#toronto": 4} {
if _, err := store.UpsertChannel(ctx, []byte{b}, fp(b), name, name[1:]); err != nil {
t.Fatal(err)
}
}
if err := store.UpsertChannelIATA(ctx, []byte{4}, "YOW", time.Now()); err != nil {
t.Fatal(err)
}
if err := store.SetChannelConfigScopes(ctx, [][]byte{fp(2), fp(3)}, []string{"east", ""}); err != nil {
t.Fatal(err)
}
now := time.Now().UTC()
for iata, b := range map[string]byte{"YOW": 1, "YYZ": 4} {
update := meshmapper.Cache{Payload: []byte(`{}`), CheckedAt: now, AttemptedAt: now, NextAttempt: now.Add(24 * time.Hour)}
if err := store.SaveChannelCatalogue(ctx, iata, "https://x.meshmapper.net/get_channels.php", update, [][]byte{fp(b)}); err != nil {
t.Fatal(err)
}
}
names := func(iatas ...string) string {
t.Helper()
page, err := store.ListChannels(ctx, 50, nil, iatas, nil, 0, nil)
if err != nil {
t.Fatal(err)
}
var out []string
for _, c := range page.Items {
out = append(out, *c.Name)
}
slices.Sort(out)
return strings.Join(out, ",")
}
if got := names("YOW"); got != "#config-east,#global,#mm-yow" {
t.Fatal("YOW channels:", got)
}
if got := names("YYZ"); got != "#global,#toronto" {
t.Fatal("YYZ channels:", got)
}
if got := names(); got != "#config-east,#global,#mm-yow,#toronto" {
t.Fatal("unfiltered channels:", got)
}
// A 304 keeps membership; a new payload replaces it.
if err := store.SaveChannelCatalogue(ctx, "YOW", "https://x.meshmapper.net/get_channels.php", meshmapper.Cache{AttemptedAt: now, NextAttempt: now}, nil); err != nil {
t.Fatal(err)
}
if got := names("YOW"); got != "#config-east,#global,#mm-yow" {
t.Fatal("304 dropped membership:", got)
}
if err := store.SaveChannelCatalogue(ctx, "YOW", "https://x.meshmapper.net/get_channels.php", meshmapper.Cache{Payload: []byte(`{}`), AttemptedAt: now, NextAttempt: now}, [][]byte{}); err != nil {
t.Fatal(err)
}
if got := names("YOW"); got != "#config-east,#global" {
t.Fatal("empty list kept membership:", got)
}
cats, err := store.ListChannelCatalogues(ctx)
if err != nil || len(cats) != 2 {
t.Fatal(cats, err)
}
if err := store.ClearChannelMembers(ctx); err != nil || names("YYZ") != "#global" {
t.Fatal("members not cleared", err)
}
}
db/migrations/001_baseline.sql
+46 -1
View File
@@ -564,6 +564,11 @@ CREATE TABLE analytics_top_talkers_by_iata (
last_sent timestamp with time zone
);
CREATE TABLE channel_config_scopes (
key_fingerprint bytea NOT NULL,
region_slug text
);
CREATE TABLE channel_iatas (
channel_hash bytea NOT NULL,
iata character(3) NOT NULL,
@@ -643,6 +648,22 @@ ALTER TABLE known_routes ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY (
CACHE 1
);
CREATE TABLE meshmapper_channel_catalogues (
iata text NOT NULL,
url text NOT NULL,
payload jsonb,
etag text,
checked_at timestamp with time zone,
attempted_at timestamp with time zone NOT NULL,
next_attempt timestamp with time zone NOT NULL,
last_error text DEFAULT ''::text NOT NULL
);
CREATE TABLE meshmapper_channel_members (
iata text NOT NULL,
key_fingerprint bytea NOT NULL
);
CREATE TABLE meshmapper_scope_catalogues (
iata text NOT NULL,
url text NOT NULL,
@@ -665,6 +686,16 @@ CREATE TABLE meshmapper_zone_boundaries (
last_error text DEFAULT ''::text NOT NULL
);
CREATE TABLE meshmapper_zone_lists (
country text NOT NULL,
payload jsonb,
etag text,
fetched_at timestamp with time zone,
attempted_at timestamp with time zone NOT NULL,
next_attempt timestamp with time zone NOT NULL,
last_error text DEFAULT ''::text NOT NULL
);
CREATE MATERIALIZED VIEW mv_hourly_iata_stats AS
WITH combined AS (
SELECT iata, hour, observation_count, unique_packets FROM analytics_live_hourly_iata_stats
@@ -911,7 +942,8 @@ CREATE TABLE regions (
center_lng double precision,
zoom_level integer DEFAULT 8,
created_at timestamp with time zone DEFAULT now() NOT NULL,
updated_at timestamp with time zone DEFAULT now() NOT NULL
updated_at timestamp with time zone DEFAULT now() NOT NULL,
imported boolean DEFAULT false NOT NULL
);
CREATE SEQUENCE regions_id_seq
@@ -1016,12 +1048,21 @@ ALTER TABLE ONLY iata_codes
ALTER TABLE ONLY known_routes
ADD CONSTRAINT known_routes_pkey PRIMARY KEY (iata, path_key);
ALTER TABLE ONLY meshmapper_channel_catalogues
ADD CONSTRAINT meshmapper_channel_catalogues_pkey PRIMARY KEY (iata, url);
ALTER TABLE ONLY meshmapper_channel_members
ADD CONSTRAINT meshmapper_channel_members_pkey PRIMARY KEY (iata, key_fingerprint);
ALTER TABLE ONLY meshmapper_scope_catalogues
ADD CONSTRAINT meshmapper_scope_catalogues_pkey PRIMARY KEY (iata, url);
ALTER TABLE ONLY meshmapper_zone_boundaries
ADD CONSTRAINT meshmapper_zone_boundaries_pkey PRIMARY KEY (iata);
ALTER TABLE ONLY meshmapper_zone_lists
ADD CONSTRAINT meshmapper_zone_lists_pkey PRIMARY KEY (country);
ALTER TABLE ONLY node_iatas
ADD CONSTRAINT node_iatas_pkey PRIMARY KEY (node_id, iata);
@@ -1128,6 +1169,10 @@ CREATE INDEX idx_channel_messages_channel ON channel_messages USING btree (chann
CREATE INDEX idx_channel_messages_sent_brin ON channel_messages USING brin (sent_at);
CREATE INDEX idx_channel_config_scopes_fingerprint ON channel_config_scopes USING btree (key_fingerprint);
CREATE INDEX idx_meshmapper_channel_members_fingerprint ON meshmapper_channel_members USING btree (key_fingerprint);
CREATE INDEX idx_channels_hash ON channels USING btree (channel_hash);
CREATE UNIQUE INDEX idx_channels_hash_no_key ON channels USING btree (channel_hash) WHERE (key_fingerprint IS NULL);
db/paths_integration_test.go
+1 -1
View File
@@ -115,7 +115,7 @@ SELECT int4send(n),'00000000-0000-0000-0000-000000000001',heard_at,iata,payload_
}
}
w := httptest.NewRecorder()
handlers.StatsRouter(store).ServeHTTP(w, httptest.NewRequest("GET", fmt.Sprintf("/paths?since=%d&until=%d&iatas=YVR", since.UnixMilli()+123, until.UnixMilli()+123), nil).WithContext(ctx))
handlers.StatsRouter(store, nil).ServeHTTP(w, httptest.NewRequest("GET", fmt.Sprintf("/paths?since=%d&until=%d&iatas=YVR", since.UnixMilli()+123, until.UnixMilli()+123), nil).WithContext(ctx))
var response api.PathStats
if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil || w.Code != 200 || response.Receptions != 14 || response.Hashed != 5 {
t.Fatalf("HTTP %d: %s (%v)", w.Code, w.Body.String(), err)
db/queries/queries.sql
+116 -27
View File
@@ -10,6 +10,11 @@ INSERT INTO iata_codes (iata)
VALUES ($1)
ON CONFLICT (iata) DO NOTHING;
-- name: AddIATAs :exec
INSERT INTO iata_codes (iata)
SELECT unnest(@iatas::bpchar[])
ON CONFLICT (iata) DO NOTHING;
-- name: GetIATA :one
SELECT * FROM iata_codes WHERE iata = $1;
@@ -64,21 +69,6 @@ SELECT id FROM transport_scopes WHERE name = $1;
-- name: GetScopeNames :many
SELECT name FROM transport_scopes ORDER BY name;
-- name: GetScopesByIATAs :many
SELECT
ts.name,
COUNT(DISTINCT os.observer_id) AS observer_count,
COUNT(DISTINCT n.id) AS node_count,
COUNT(DISTINCT po.iata) AS iata_count
FROM transport_scopes ts
LEFT JOIN observer_scopes os ON os.scope_id = ts.id
LEFT JOIN observers o ON o.id = os.observer_id
LEFT JOIN packet_observations po ON po.observer_id = o.id
LEFT JOIN nodes n ON n.default_scope_id = ts.id
WHERE (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR po.iata = ANY($1::bpchar[]))
GROUP BY ts.name
ORDER BY ts.name;
-- name: GetScopeByName :one
SELECT
ts.name,
@@ -878,6 +868,19 @@ RETURNING id;
SELECT packet_hash, raw_payload FROM packets
WHERE payload_type = 5 AND decrypted IS NOT TRUE;
-- name: ListUndecryptedGroupTextPacketsByHash :many
-- Like ListUndecryptedGroupTextPackets, limited to channels that just gained a key.
SELECT packet_hash, raw_payload FROM packets
WHERE payload_type = 5 AND decrypted IS NOT TRUE AND channel_hash = ANY(@hashes::bytea[]);
-- name: DeleteChannelConfigScopes :exec
DELETE FROM channel_config_scopes;
-- name: AddChannelConfigScopes :exec
-- An empty region places the channel Beacon-wide.
INSERT INTO channel_config_scopes (key_fingerprint, region_slug)
SELECT unnest(@fingerprints::bytea[]), NULLIF(unnest(@regions::text[]), '');
-- name: UpsertChannelIATA :exec
-- Refreshes at most hourly so repeat hears don't churn the row.
INSERT INTO channel_iatas (channel_hash, iata, last_heard)
@@ -895,15 +898,20 @@ ON CONFLICT (trace_tag, iata) DO UPDATE SET
WHERE EXCLUDED.last_heard > trace_iatas.last_heard + INTERVAL '1 hour';
-- name: ListChannels :many
-- Channels ordered by last seen, optionally filtered by hash and/or IATAs
-- (membership via channel_iatas). NULL hash / empty array skip those filters.
-- Channels ordered by last seen, optionally filtered by hash and/or IATAs.
-- A channel belongs to an IATA when MeshMapper lists it there or config scopes it
-- to a region containing it (or Beacon-wide). NULL hash / empty array / NULL key_known skip those filters.
-- Pass cursor=0 to start from the beginning (cursor is last_seen epoch ms).
SELECT c.* FROM channels c
WHERE (@channel_hash::bytea IS NULL OR c.channel_hash = @channel_hash)
AND (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 OR c.channel_hash IN (
SELECT ci.channel_hash FROM channel_iatas ci
WHERE ci.iata = ANY(@iatas::bpchar[])
))
AND (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0
OR EXISTS (SELECT 1 FROM meshmapper_channel_members m
WHERE m.key_fingerprint = c.key_fingerprint AND m.iata::bpchar = ANY(@iatas::bpchar[]))
OR EXISTS (SELECT 1 FROM channel_config_scopes s
WHERE s.key_fingerprint = c.key_fingerprint AND (s.region_slug IS NULL OR s.region_slug IN (
SELECT r.slug FROM regions r JOIN region_iatas ri ON ri.region_id = r.id
WHERE ri.iata = ANY(@iatas::bpchar[])))))
AND (sqlc.narg(key_known)::boolean IS NULL OR COALESCE(c.key_known, false) = sqlc.narg(key_known))
AND (@cursor_ts::timestamptz IS NULL OR c.last_seen < @cursor_ts)
ORDER BY c.last_seen DESC, c.id DESC
LIMIT @page_limit;
@@ -914,9 +922,14 @@ LIMIT @page_limit;
SELECT c.* FROM channels c
WHERE (c.last_seen, c.id) < (@cursor_ts::timestamptz, @cursor_id::integer)
AND (@channel_hash::bytea IS NULL OR c.channel_hash = @channel_hash)
AND (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 OR c.channel_hash IN (
SELECT ci.channel_hash FROM channel_iatas ci WHERE ci.iata = ANY(@iatas::bpchar[])
))
AND (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0
OR EXISTS (SELECT 1 FROM meshmapper_channel_members m
WHERE m.key_fingerprint = c.key_fingerprint AND m.iata::bpchar = ANY(@iatas::bpchar[]))
OR EXISTS (SELECT 1 FROM channel_config_scopes s
WHERE s.key_fingerprint = c.key_fingerprint AND (s.region_slug IS NULL OR s.region_slug IN (
SELECT r.slug FROM regions r JOIN region_iatas ri ON ri.region_id = r.id
WHERE ri.iata = ANY(@iatas::bpchar[])))))
AND (sqlc.narg(key_known)::boolean IS NULL OR COALESCE(c.key_known, false) = sqlc.narg(key_known))
ORDER BY c.last_seen DESC, c.id DESC
LIMIT @page_limit;
@@ -1215,14 +1228,43 @@ ON CONFLICT (slug) DO UPDATE SET
center_lat = EXCLUDED.center_lat,
center_lng = EXCLUDED.center_lng,
zoom_level = EXCLUDED.zoom_level,
imported = FALSE, -- config owns the slug from now on
updated_at = NOW()
RETURNING id;
-- name: UpsertRegionIATA :exec
-- name: DeleteRegionIATAsNotIn :exec
DELETE FROM region_iatas WHERE region_id = @region_id AND NOT (iata = ANY(@keep::bpchar[]));
-- name: AddRegionIATAs :exec
INSERT INTO region_iatas (region_id, iata)
VALUES ($1, $2)
SELECT @region_id, unnest(@iatas::bpchar[])
ON CONFLICT (region_id, iata) DO NOTHING;
-- name: ListRegionState :many
-- Every region with its members, for reconciling imported MeshMapper groups.
SELECT r.slug, r.name, COALESCE(r.display_order, 0)::int AS display_order, r.imported, r.center_lat, r.center_lng,
COALESCE(array_agg(ri.iata::text ORDER BY ri.iata) FILTER (WHERE ri.iata IS NOT NULL), '{}')::text[] AS iatas
FROM regions r
LEFT JOIN region_iatas ri ON ri.region_id = r.id
GROUP BY r.id
ORDER BY r.slug;
-- name: UpsertImportedRegion :one
-- A hand-written region owns its slug: the WHERE turns a clash into no row.
INSERT INTO regions (slug, name, display_order, center_lat, center_lng, zoom_level, imported, updated_at)
VALUES (@slug, @name, @display_order, sqlc.narg(center_lat), sqlc.narg(center_lng), NULL, TRUE, NOW())
ON CONFLICT (slug) DO UPDATE SET
name = EXCLUDED.name,
display_order = EXCLUDED.display_order,
center_lat = EXCLUDED.center_lat,
center_lng = EXCLUDED.center_lng,
updated_at = NOW()
WHERE regions.imported
RETURNING id;
-- name: PruneImportedRegions :many
DELETE FROM regions WHERE imported AND NOT (slug = ANY(@keep::text[])) RETURNING slug;
-- ============================================================
-- TRACES
-- ============================================================
@@ -1275,7 +1317,8 @@ ORDER BY t.last_heard_at DESC;
-- name: UpsertKnownRoute :exec
-- Route identity is path_key, an md5 of node_ids computed by the caller.
-- Keep the latest processed representation without changing the node-chain identity.
-- On conflict, observation_count and last_seen are bumped and hash_prefix follows the
-- latest hearing, so evidence matches the hash width the route uses now.
INSERT INTO known_routes (path_key, node_ids, hash_prefix, iata, hop_count)
VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (iata, path_key) DO UPDATE SET
@@ -1592,6 +1635,9 @@ SELECT (SELECT heard_at FROM latest)::timestamptz AS latest_recorded_at,
-- name: GetScopeCatalogue :one
SELECT * FROM meshmapper_scope_catalogues WHERE iata = $1 AND url = $2;
-- name: ListScopeCatalogues :many
SELECT * FROM meshmapper_scope_catalogues ORDER BY iata, attempted_at;
-- name: SaveScopeCatalogue :exec
-- One statement commits the validated snapshot and its lookup identities together.
-- Empty arrays insert nothing. NULL payload/checked_at retain last-known-good data
@@ -1614,6 +1660,33 @@ ON CONFLICT (iata, url) DO UPDATE SET
next_attempt = EXCLUDED.next_attempt,
last_error = EXCLUDED.last_error;
-- name: ListChannelCatalogues :many
SELECT * FROM meshmapper_channel_catalogues ORDER BY iata, attempted_at;
-- name: SaveChannelCatalogue :exec
-- NULL payload/etag/checked_at retain the last good list after an error or 304.
INSERT INTO meshmapper_channel_catalogues (iata, url, payload, etag, checked_at, attempted_at, next_attempt, last_error)
VALUES (@iata, @url, sqlc.narg(payload)::jsonb, sqlc.narg(etag)::text,
sqlc.narg(checked_at)::timestamptz, @attempted_at, @next_attempt, @last_error)
ON CONFLICT (iata, url) DO UPDATE SET
payload = COALESCE(EXCLUDED.payload, meshmapper_channel_catalogues.payload),
etag = COALESCE(EXCLUDED.etag, meshmapper_channel_catalogues.etag),
checked_at = COALESCE(EXCLUDED.checked_at, meshmapper_channel_catalogues.checked_at),
attempted_at = EXCLUDED.attempted_at,
next_attempt = EXCLUDED.next_attempt,
last_error = EXCLUDED.last_error;
-- name: DeleteChannelMembersNotIn :exec
DELETE FROM meshmapper_channel_members WHERE iata = @iata AND NOT (key_fingerprint = ANY(@keep::bytea[]));
-- name: AddChannelMembers :exec
INSERT INTO meshmapper_channel_members (iata, key_fingerprint)
SELECT @iata, unnest(@fingerprints::bytea[])
ON CONFLICT (iata, key_fingerprint) DO NOTHING;
-- name: DeleteAllChannelMembers :exec
DELETE FROM meshmapper_channel_members;
-- name: ListZoneBoundaries :many
SELECT * FROM meshmapper_zone_boundaries ORDER BY iata;
@@ -1631,6 +1704,22 @@ ON CONFLICT (iata) DO UPDATE SET
next_attempt = EXCLUDED.next_attempt,
last_error = EXCLUDED.last_error;
-- name: ListZoneLists :many
SELECT * FROM meshmapper_zone_lists ORDER BY country;
-- name: SaveZoneList :exec
-- NULL payload/etag/fetched_at retain the last good list after an error or 304.
INSERT INTO meshmapper_zone_lists (country, payload, etag, fetched_at, attempted_at, next_attempt, last_error)
VALUES (@country, sqlc.narg(payload)::jsonb, sqlc.narg(etag)::text,
sqlc.narg(fetched_at)::timestamptz, @attempted_at, @next_attempt, @last_error)
ON CONFLICT (country) DO UPDATE SET
payload = COALESCE(EXCLUDED.payload, meshmapper_zone_lists.payload),
etag = COALESCE(EXCLUDED.etag, meshmapper_zone_lists.etag),
fetched_at = COALESCE(EXCLUDED.fetched_at, meshmapper_zone_lists.fetched_at),
attempted_at = EXCLUDED.attempted_at,
next_attempt = EXCLUDED.next_attempt,
last_error = EXCLUDED.last_error;
-- name: PruneZoneBoundaries :many
-- Drops imports for IATAs no longer configured, so their manual border returns.
DELETE FROM meshmapper_zone_boundaries WHERE NOT (iata = ANY(@keep::text[])) RETURNING iata;
db/route_evidence_integration_test.go
+28
View File
@@ -234,3 +234,31 @@ ANALYZE packet_observations`); err != nil {
t.Fatalf("0/1-hop rows entered the route index: %v %v", indexed, err)
}
}
func TestRouteEvidenceFollowsPrefixWidthPostgres(t *testing.T) {
ctx, tx := retentionTx(t)
applyBaseline(t, ctx, tx)
if _, err := tx.Exec(ctx, `INSERT INTO observers(id,public_key) VALUES ('00000000-0000-0000-0000-000000000001','\x01');
INSERT INTO packets(packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,first_heard_at,last_heard_at)
VALUES (int4send(1),4,0,1,'\x00','\x00',NOW(),NOW());
INSERT INTO packet_observations(id,packet_hash,observer_id,iata,heard_at,hash_size,hop_count,path_bytes,payload_type,path_length_byte)
VALUES (1,int4send(1),'00000000-0000-0000-0000-000000000001','YOW',NOW(),2,2,'\xaa11bb22',4,(1<<6)|2)`); err != nil {
t.Fatal(err)
}
store := &Store{q: sqlc.New(tx)}
ids := []uuid.UUID{uuid.New(), uuid.New()}
if err := store.UpsertKnownRoute(ctx, ids, [][]byte{{0xaa}, {0xbb}}, "YOW", 2); err != nil {
t.Fatal(err)
}
if err := store.UpsertKnownRoute(ctx, ids, [][]byte{{0xaa, 0x11}, {0xbb, 0x22}}, "YOW", 2); err != nil {
t.Fatal(err)
}
now := time.Now().UTC()
got, err := store.GetRouteEvidence(ctx, "YOW", hex.EncodeToString(routePathKey(ids)), api.RouteEvidenceQuery{Since: now.Add(-time.Hour), Until: now.Add(time.Minute), Limit: 10})
if err != nil {
t.Fatal(err)
}
if got.HashSize != 2 || got.PathBytes != "aa11bb22" || len(got.Items) != 1 || got.Route.ObservationCount != 2 {
t.Fatalf("evidence kept the first prefix width: %+v", got)
}
}
db/scopes.go
-18
View File
@@ -49,24 +49,6 @@ func (s *Store) GetScopeNames(ctx context.Context) ([]string, error) {
return s.q.GetScopeNames(ctx)
}
// GetScopesByIATAs returns scope summaries filtered by the given IATA codes.
func (s *Store) GetScopesByIATAs(ctx context.Context, iatas []string) ([]api.ScopeSummary, error) {
rows, err := s.q.GetScopesByIATAs(ctx, iatas)
if err != nil {
return nil, err
}
items := make([]api.ScopeSummary, 0, len(rows))
for _, r := range rows {
items = append(items, api.ScopeSummary{
Name: r.Name,
ObserverCount: r.ObserverCount,
NodeCount: r.NodeCount,
IATACount: r.IataCount,
})
}
return items, nil
}
// GetScopeByName returns full detail for a single scope by its normalized name.
func (s *Store) GetScopeByName(ctx context.Context, name string) (*api.ScopeDetail, error) {
row, err := s.q.GetScopeByName(ctx, name)
db/scopes_test.go
-23
View File
@@ -43,29 +43,6 @@ func TestGetTransportScopes(t *testing.T) {
}
}
func TestGetScopesByIATAs(t *testing.T) {
ctrl := gomock.NewController(t)
mock := mockdb.NewMockQuerier(ctrl)
mock.EXPECT().
GetScopesByIATAs(gomock.Any(), []string{"YVR", "YYJ"}).
Return([]sqlc.GetScopesByIATAsRow{
{Name: "default", ObserverCount: 3, NodeCount: 10, IataCount: 2},
}, nil)
store := &Store{q: mock}
items, err := store.GetScopesByIATAs(context.Background(), []string{"YVR", "YYJ"})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(items) != 1 {
t.Fatalf("expected 1 item, got %d", len(items))
}
if items[0].IATACount != 2 {
t.Errorf("expected IATACount 2, got %d", items[0].IATACount)
}
}
func TestGetScopeByName(t *testing.T) {
ctrl := gomock.NewController(t)
mock := mockdb.NewMockQuerier(ctrl)
db/signal_integration_test.go
+1 -1
View File
@@ -138,7 +138,7 @@ SELECT int4send(n),'00000000-0000-0000-0000-000000000001',heard_at,iata,snr,rssi
}
w := httptest.NewRecorder()
request := httptest.NewRequest("GET", fmt.Sprintf("/signal?since=%d&until=%d&iatas=YVR", since.UnixMilli()+123, until.UnixMilli()+123), nil).WithContext(ctx)
handlers.StatsRouter(store).ServeHTTP(w, request)
handlers.StatsRouter(store, nil).ServeHTTP(w, request)
var response api.SignalStats
if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil || w.Code != 200 || response.Receptions != 13 || response.SNR.Samples != 5 {
t.Fatalf("PostgreSQL HTTP response: status=%d body=%s error=%v", w.Code, w.Body.String(), err)
db/sqlc/mock/querier.go
+245 -29
View File
@@ -44,6 +44,62 @@ func (m *MockQuerier) EXPECT() *MockQuerierMockRecorder {
return m.recorder
}
// AddChannelConfigScopes mocks base method.
func (m *MockQuerier) AddChannelConfigScopes(ctx context.Context, arg db.AddChannelConfigScopesParams) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "AddChannelConfigScopes", ctx, arg)
ret0, _ := ret[0].(error)
return ret0
}
// AddChannelConfigScopes indicates an expected call of AddChannelConfigScopes.
func (mr *MockQuerierMockRecorder) AddChannelConfigScopes(ctx, arg any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddChannelConfigScopes", reflect.TypeOf((*MockQuerier)(nil).AddChannelConfigScopes), ctx, arg)
}
// AddChannelMembers mocks base method.
func (m *MockQuerier) AddChannelMembers(ctx context.Context, arg db.AddChannelMembersParams) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "AddChannelMembers", ctx, arg)
ret0, _ := ret[0].(error)
return ret0
}
// AddChannelMembers indicates an expected call of AddChannelMembers.
func (mr *MockQuerierMockRecorder) AddChannelMembers(ctx, arg any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddChannelMembers", reflect.TypeOf((*MockQuerier)(nil).AddChannelMembers), ctx, arg)
}
// AddIATAs mocks base method.
func (m *MockQuerier) AddIATAs(ctx context.Context, iatas []string) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "AddIATAs", ctx, iatas)
ret0, _ := ret[0].(error)
return ret0
}
// AddIATAs indicates an expected call of AddIATAs.
func (mr *MockQuerierMockRecorder) AddIATAs(ctx, iatas any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddIATAs", reflect.TypeOf((*MockQuerier)(nil).AddIATAs), ctx, iatas)
}
// AddRegionIATAs mocks base method.
func (m *MockQuerier) AddRegionIATAs(ctx context.Context, arg db.AddRegionIATAsParams) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "AddRegionIATAs", ctx, arg)
ret0, _ := ret[0].(error)
return ret0
}
// AddRegionIATAs indicates an expected call of AddRegionIATAs.
func (mr *MockQuerierMockRecorder) AddRegionIATAs(ctx, arg any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddRegionIATAs", reflect.TypeOf((*MockQuerier)(nil).AddRegionIATAs), ctx, arg)
}
// AmbiguousPrefixes mocks base method.
func (m *MockQuerier) AmbiguousPrefixes(ctx context.Context) ([]db.AmbiguousPrefixesRow, error) {
m.ctrl.T.Helper()
@@ -89,6 +145,48 @@ func (mr *MockQuerierMockRecorder) DeactivateAccount(ctx, id any) *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeactivateAccount", reflect.TypeOf((*MockQuerier)(nil).DeactivateAccount), ctx, id)
}
// DeleteAllChannelMembers mocks base method.
func (m *MockQuerier) DeleteAllChannelMembers(ctx context.Context) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "DeleteAllChannelMembers", ctx)
ret0, _ := ret[0].(error)
return ret0
}
// DeleteAllChannelMembers indicates an expected call of DeleteAllChannelMembers.
func (mr *MockQuerierMockRecorder) DeleteAllChannelMembers(ctx any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAllChannelMembers", reflect.TypeOf((*MockQuerier)(nil).DeleteAllChannelMembers), ctx)
}
// DeleteChannelConfigScopes mocks base method.
func (m *MockQuerier) DeleteChannelConfigScopes(ctx context.Context) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "DeleteChannelConfigScopes", ctx)
ret0, _ := ret[0].(error)
return ret0
}
// DeleteChannelConfigScopes indicates an expected call of DeleteChannelConfigScopes.
func (mr *MockQuerierMockRecorder) DeleteChannelConfigScopes(ctx any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteChannelConfigScopes", reflect.TypeOf((*MockQuerier)(nil).DeleteChannelConfigScopes), ctx)
}
// DeleteChannelMembersNotIn mocks base method.
func (m *MockQuerier) DeleteChannelMembersNotIn(ctx context.Context, arg db.DeleteChannelMembersNotInParams) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "DeleteChannelMembersNotIn", ctx, arg)
ret0, _ := ret[0].(error)
return ret0
}
// DeleteChannelMembersNotIn indicates an expected call of DeleteChannelMembersNotIn.
func (mr *MockQuerierMockRecorder) DeleteChannelMembersNotIn(ctx, arg any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteChannelMembersNotIn", reflect.TypeOf((*MockQuerier)(nil).DeleteChannelMembersNotIn), ctx, arg)
}
// DeleteOldChannelIATAs mocks base method.
func (m *MockQuerier) DeleteOldChannelIATAs(ctx context.Context, lastHeard pgtype.Timestamptz) error {
m.ctrl.T.Helper()
@@ -190,6 +288,20 @@ func (mr *MockQuerierMockRecorder) DeleteOldTraceIATAs(ctx, lastHeard any) *gomo
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteOldTraceIATAs", reflect.TypeOf((*MockQuerier)(nil).DeleteOldTraceIATAs), ctx, lastHeard)
}
// DeleteRegionIATAsNotIn mocks base method.
func (m *MockQuerier) DeleteRegionIATAsNotIn(ctx context.Context, arg db.DeleteRegionIATAsNotInParams) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "DeleteRegionIATAsNotIn", ctx, arg)
ret0, _ := ret[0].(error)
return ret0
}
// DeleteRegionIATAsNotIn indicates an expected call of DeleteRegionIATAsNotIn.
func (mr *MockQuerierMockRecorder) DeleteRegionIATAsNotIn(ctx, arg any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteRegionIATAsNotIn", reflect.TypeOf((*MockQuerier)(nil).DeleteRegionIATAsNotIn), ctx, arg)
}
// GetAccount mocks base method.
func (m *MockQuerier) GetAccount(ctx context.Context, id uuid.UUID) (db.Account, error) {
m.ctrl.T.Helper()
@@ -760,21 +872,6 @@ func (mr *MockQuerierMockRecorder) GetScopeStats(ctx, iatas any) *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetScopeStats", reflect.TypeOf((*MockQuerier)(nil).GetScopeStats), ctx, iatas)
}
// GetScopesByIATAs mocks base method.
func (m *MockQuerier) GetScopesByIATAs(ctx context.Context, dollar_1 []string) ([]db.GetScopesByIATAsRow, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "GetScopesByIATAs", ctx, dollar_1)
ret0, _ := ret[0].([]db.GetScopesByIATAsRow)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// GetScopesByIATAs indicates an expected call of GetScopesByIATAs.
func (mr *MockQuerierMockRecorder) GetScopesByIATAs(ctx, dollar_1 any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetScopesByIATAs", reflect.TypeOf((*MockQuerier)(nil).GetScopesByIATAs), ctx, dollar_1)
}
// GetSignalStats mocks base method.
func (m *MockQuerier) GetSignalStats(ctx context.Context, arg db.GetSignalStatsParams) ([]db.GetSignalStatsRow, error) {
m.ctrl.T.Helper()
@@ -1014,6 +1111,21 @@ func (mr *MockQuerierMockRecorder) ListAllChannelMessages(ctx, arg any) *gomock.
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListAllChannelMessages", reflect.TypeOf((*MockQuerier)(nil).ListAllChannelMessages), ctx, arg)
}
// ListChannelCatalogues mocks base method.
func (m *MockQuerier) ListChannelCatalogues(ctx context.Context) ([]db.MeshmapperChannelCatalogue, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ListChannelCatalogues", ctx)
ret0, _ := ret[0].([]db.MeshmapperChannelCatalogue)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// ListChannelCatalogues indicates an expected call of ListChannelCatalogues.
func (mr *MockQuerierMockRecorder) ListChannelCatalogues(ctx any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListChannelCatalogues", reflect.TypeOf((*MockQuerier)(nil).ListChannelCatalogues), ctx)
}
// ListChannelMessages mocks base method.
func (m *MockQuerier) ListChannelMessages(ctx context.Context, arg db.ListChannelMessagesParams) ([]db.ListChannelMessagesRow, error) {
m.ctrl.T.Helper()
@@ -1239,6 +1351,21 @@ func (mr *MockQuerierMockRecorder) ListPacketsByIATAs(ctx, arg any) *gomock.Call
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListPacketsByIATAs", reflect.TypeOf((*MockQuerier)(nil).ListPacketsByIATAs), ctx, arg)
}
// ListRegionState mocks base method.
func (m *MockQuerier) ListRegionState(ctx context.Context) ([]db.ListRegionStateRow, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ListRegionState", ctx)
ret0, _ := ret[0].([]db.ListRegionStateRow)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// ListRegionState indicates an expected call of ListRegionState.
func (mr *MockQuerierMockRecorder) ListRegionState(ctx any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListRegionState", reflect.TypeOf((*MockQuerier)(nil).ListRegionState), ctx)
}
// ListRegions mocks base method.
func (m *MockQuerier) ListRegions(ctx context.Context) ([]db.ListRegionsRow, error) {
m.ctrl.T.Helper()
@@ -1269,6 +1396,21 @@ func (mr *MockQuerierMockRecorder) ListRouteEvidence(ctx, arg any) *gomock.Call
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListRouteEvidence", reflect.TypeOf((*MockQuerier)(nil).ListRouteEvidence), ctx, arg)
}
// ListScopeCatalogues mocks base method.
func (m *MockQuerier) ListScopeCatalogues(ctx context.Context) ([]db.MeshmapperScopeCatalogue, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ListScopeCatalogues", ctx)
ret0, _ := ret[0].([]db.MeshmapperScopeCatalogue)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// ListScopeCatalogues indicates an expected call of ListScopeCatalogues.
func (mr *MockQuerierMockRecorder) ListScopeCatalogues(ctx any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListScopeCatalogues", reflect.TypeOf((*MockQuerier)(nil).ListScopeCatalogues), ctx)
}
// ListTraceTags mocks base method.
func (m *MockQuerier) ListTraceTags(ctx context.Context, arg db.ListTraceTagsParams) ([]db.ListTraceTagsRow, error) {
m.ctrl.T.Helper()
@@ -1299,6 +1441,21 @@ func (mr *MockQuerierMockRecorder) ListUndecryptedGroupTextPackets(ctx any) *gom
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListUndecryptedGroupTextPackets", reflect.TypeOf((*MockQuerier)(nil).ListUndecryptedGroupTextPackets), ctx)
}
// ListUndecryptedGroupTextPacketsByHash mocks base method.
func (m *MockQuerier) ListUndecryptedGroupTextPacketsByHash(ctx context.Context, hashes [][]byte) ([]db.ListUndecryptedGroupTextPacketsByHashRow, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ListUndecryptedGroupTextPacketsByHash", ctx, hashes)
ret0, _ := ret[0].([]db.ListUndecryptedGroupTextPacketsByHashRow)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// ListUndecryptedGroupTextPacketsByHash indicates an expected call of ListUndecryptedGroupTextPacketsByHash.
func (mr *MockQuerierMockRecorder) ListUndecryptedGroupTextPacketsByHash(ctx, hashes any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListUndecryptedGroupTextPacketsByHash", reflect.TypeOf((*MockQuerier)(nil).ListUndecryptedGroupTextPacketsByHash), ctx, hashes)
}
// ListZoneBoundaries mocks base method.
func (m *MockQuerier) ListZoneBoundaries(ctx context.Context) ([]db.MeshmapperZoneBoundary, error) {
m.ctrl.T.Helper()
@@ -1314,6 +1471,36 @@ func (mr *MockQuerierMockRecorder) ListZoneBoundaries(ctx any) *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListZoneBoundaries", reflect.TypeOf((*MockQuerier)(nil).ListZoneBoundaries), ctx)
}
// ListZoneLists mocks base method.
func (m *MockQuerier) ListZoneLists(ctx context.Context) ([]db.MeshmapperZoneList, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ListZoneLists", ctx)
ret0, _ := ret[0].([]db.MeshmapperZoneList)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// ListZoneLists indicates an expected call of ListZoneLists.
func (mr *MockQuerierMockRecorder) ListZoneLists(ctx any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListZoneLists", reflect.TypeOf((*MockQuerier)(nil).ListZoneLists), ctx)
}
// PruneImportedRegions mocks base method.
func (m *MockQuerier) PruneImportedRegions(ctx context.Context, keep []string) ([]string, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "PruneImportedRegions", ctx, keep)
ret0, _ := ret[0].([]string)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// PruneImportedRegions indicates an expected call of PruneImportedRegions.
func (mr *MockQuerierMockRecorder) PruneImportedRegions(ctx, keep any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PruneImportedRegions", reflect.TypeOf((*MockQuerier)(nil).PruneImportedRegions), ctx, keep)
}
// PruneZoneBoundaries mocks base method.
func (m *MockQuerier) PruneZoneBoundaries(ctx context.Context, keep []string) ([]string, error) {
m.ctrl.T.Helper()
@@ -1588,6 +1775,20 @@ func (mr *MockQuerierMockRecorder) ResolvePathHashesP4(ctx, arg any) *gomock.Cal
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResolvePathHashesP4", reflect.TypeOf((*MockQuerier)(nil).ResolvePathHashesP4), ctx, arg)
}
// SaveChannelCatalogue mocks base method.
func (m *MockQuerier) SaveChannelCatalogue(ctx context.Context, arg db.SaveChannelCatalogueParams) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "SaveChannelCatalogue", ctx, arg)
ret0, _ := ret[0].(error)
return ret0
}
// SaveChannelCatalogue indicates an expected call of SaveChannelCatalogue.
func (mr *MockQuerierMockRecorder) SaveChannelCatalogue(ctx, arg any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveChannelCatalogue", reflect.TypeOf((*MockQuerier)(nil).SaveChannelCatalogue), ctx, arg)
}
// SaveScopeCatalogue mocks base method.
func (m *MockQuerier) SaveScopeCatalogue(ctx context.Context, arg db.SaveScopeCatalogueParams) error {
m.ctrl.T.Helper()
@@ -1616,6 +1817,20 @@ func (mr *MockQuerierMockRecorder) SaveZoneBoundary(ctx, arg any) *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveZoneBoundary", reflect.TypeOf((*MockQuerier)(nil).SaveZoneBoundary), ctx, arg)
}
// SaveZoneList mocks base method.
func (m *MockQuerier) SaveZoneList(ctx context.Context, arg db.SaveZoneListParams) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "SaveZoneList", ctx, arg)
ret0, _ := ret[0].(error)
return ret0
}
// SaveZoneList indicates an expected call of SaveZoneList.
func (mr *MockQuerierMockRecorder) SaveZoneList(ctx, arg any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveZoneList", reflect.TypeOf((*MockQuerier)(nil).SaveZoneList), ctx, arg)
}
// SearchKnownRoutes mocks base method.
func (m *MockQuerier) SearchKnownRoutes(ctx context.Context, arg db.SearchKnownRoutesParams) ([]db.SearchKnownRoutesRow, error) {
m.ctrl.T.Helper()
@@ -1844,6 +2059,21 @@ func (mr *MockQuerierMockRecorder) UpsertIATADetails(ctx, arg any) *gomock.Call
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertIATADetails", reflect.TypeOf((*MockQuerier)(nil).UpsertIATADetails), ctx, arg)
}
// UpsertImportedRegion mocks base method.
func (m *MockQuerier) UpsertImportedRegion(ctx context.Context, arg db.UpsertImportedRegionParams) (int32, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "UpsertImportedRegion", ctx, arg)
ret0, _ := ret[0].(int32)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// UpsertImportedRegion indicates an expected call of UpsertImportedRegion.
func (mr *MockQuerierMockRecorder) UpsertImportedRegion(ctx, arg any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertImportedRegion", reflect.TypeOf((*MockQuerier)(nil).UpsertImportedRegion), ctx, arg)
}
// UpsertKnownRoute mocks base method.
func (m *MockQuerier) UpsertKnownRoute(ctx context.Context, arg db.UpsertKnownRouteParams) error {
m.ctrl.T.Helper()
@@ -1988,20 +2218,6 @@ func (mr *MockQuerierMockRecorder) UpsertRegion(ctx, arg any) *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertRegion", reflect.TypeOf((*MockQuerier)(nil).UpsertRegion), ctx, arg)
}
// UpsertRegionIATA mocks base method.
func (m *MockQuerier) UpsertRegionIATA(ctx context.Context, arg db.UpsertRegionIATAParams) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "UpsertRegionIATA", ctx, arg)
ret0, _ := ret[0].(error)
return ret0
}
// UpsertRegionIATA indicates an expected call of UpsertRegionIATA.
func (mr *MockQuerierMockRecorder) UpsertRegionIATA(ctx, arg any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertRegionIATA", reflect.TypeOf((*MockQuerier)(nil).UpsertRegionIATA), ctx, arg)
}
// UpsertTraceIATA mocks base method.
func (m *MockQuerier) UpsertTraceIATA(ctx context.Context, arg db.UpsertTraceIATAParams) error {
m.ctrl.T.Helper()
db/sqlc/models.go
+32
View File
@@ -188,6 +188,11 @@ type Channel struct {
MessageCount *int64 `json:"message_count"`
}
type ChannelConfigScope struct {
KeyFingerprint []byte `json:"key_fingerprint"`
RegionSlug *string `json:"region_slug"`
}
type ChannelIata struct {
ChannelHash []byte `json:"channel_hash"`
Iata string `json:"iata"`
@@ -234,6 +239,22 @@ type KnownRoute struct {
LastReconfirmedAt pgtype.Timestamptz `json:"last_reconfirmed_at"`
}
type MeshmapperChannelCatalogue struct {
Iata string `json:"iata"`
Url string `json:"url"`
Payload []byte `json:"payload"`
Etag *string `json:"etag"`
CheckedAt pgtype.Timestamptz `json:"checked_at"`
AttemptedAt pgtype.Timestamptz `json:"attempted_at"`
NextAttempt pgtype.Timestamptz `json:"next_attempt"`
LastError string `json:"last_error"`
}
type MeshmapperChannelMember struct {
Iata string `json:"iata"`
KeyFingerprint []byte `json:"key_fingerprint"`
}
type MeshmapperScopeCatalogue struct {
Iata string `json:"iata"`
Url string `json:"url"`
@@ -256,6 +277,16 @@ type MeshmapperZoneBoundary struct {
LastError string `json:"last_error"`
}
type MeshmapperZoneList struct {
Country string `json:"country"`
Payload []byte `json:"payload"`
Etag *string `json:"etag"`
FetchedAt pgtype.Timestamptz `json:"fetched_at"`
AttemptedAt pgtype.Timestamptz `json:"attempted_at"`
NextAttempt pgtype.Timestamptz `json:"next_attempt"`
LastError string `json:"last_error"`
}
type MvHourlyIataStat struct {
Iata string `json:"iata"`
Hour pgtype.Timestamptz `json:"hour"`
@@ -529,6 +560,7 @@ type Region struct {
ZoomLevel *int32 `json:"zoom_level"`
CreatedAt pgtype.Timestamptz `json:"created_at"`
UpdatedAt pgtype.Timestamptz `json:"updated_at"`
Imported bool `json:"imported"`
}
type RegionIata struct {
db/sqlc/querier.go
+28 -5
View File
@@ -13,12 +13,20 @@ import (
)
type Querier interface {
// An empty region places the channel Beacon-wide.
AddChannelConfigScopes(ctx context.Context, arg AddChannelConfigScopesParams) error
AddChannelMembers(ctx context.Context, arg AddChannelMembersParams) error
AddIATAs(ctx context.Context, iatas []string) error
AddRegionIATAs(ctx context.Context, arg AddRegionIATAsParams) error
// Hop prefixes that match >1 node in an IATA, per width. Computed once per reconfirm run.
AmbiguousPrefixes(ctx context.Context) ([]AmbiguousPrefixesRow, error)
CreateAccount(ctx context.Context, name string) (Account, error)
// Lock the current row before deciding the outcome, including when another
// deactivation commits while this statement is waiting for its row lock.
DeactivateAccount(ctx context.Context, id uuid.UUID) (DeactivateAccountRow, error)
DeleteAllChannelMembers(ctx context.Context) error
DeleteChannelConfigScopes(ctx context.Context) error
DeleteChannelMembersNotIn(ctx context.Context, arg DeleteChannelMembersNotInParams) error
// Keeps the channel IATA filter in step with packet retention.
DeleteOldChannelIATAs(ctx context.Context, lastHeard pgtype.Timestamptz) error
// Deletes nodes not seen since the given cutoff. node_iatas and node_neighbors cascade-
@@ -41,6 +49,7 @@ type Querier interface {
DeleteOldTelemetry(ctx context.Context, reportedAt pgtype.Timestamptz) error
// Keeps the trace IATA filter in step with packet retention.
DeleteOldTraceIATAs(ctx context.Context, lastHeard pgtype.Timestamptz) error
DeleteRegionIATAsNotIn(ctx context.Context, arg DeleteRegionIATAsNotInParams) error
GetAccount(ctx context.Context, id uuid.UUID) (Account, error)
GetChannelByID(ctx context.Context, id int32) (Channel, error)
// Returns neighbors of a node that are in a different IATA.
@@ -97,7 +106,6 @@ type Querier interface {
// a cross-join. Empty IATAs keep the original global counts, including associations
// whose observations have expired; the filtered aggregates are empty in that case.
GetScopeStats(ctx context.Context, iatas []string) ([]GetScopeStatsRow, error)
GetScopesByIATAs(ctx context.Context, dollar_1 []string) ([]GetScopesByIATAsRow, error)
// Read compact hourly snapshots, never observations on an HTTP request.
// Weight averages by sample counts instead of averaging regional/hourly means.
GetSignalStats(ctx context.Context, arg GetSignalStatsParams) ([]GetSignalStatsRow, error)
@@ -143,6 +151,7 @@ type Querier interface {
// Pass empty string for iata or scope to skip those filters.
// Pass cursor=0 to start from the beginning.
ListAllChannelMessages(ctx context.Context, arg ListAllChannelMessagesParams) ([]ListAllChannelMessagesRow, error)
ListChannelCatalogues(ctx context.Context) ([]MeshmapperChannelCatalogue, error)
// Returns messages for a channel identified by integer ID.
// Pass a zero/null timestamp for since to return all messages up to limit.
// Pass empty string for iata to skip IATA filtering.
@@ -153,8 +162,9 @@ type Querier interface {
// Pass empty string for iata or scope to skip those filters.
// Pass cursor=0 to start from the beginning.
ListChannelMessagesByHash(ctx context.Context, arg ListChannelMessagesByHashParams) ([]ListChannelMessagesByHashRow, error)
// Channels ordered by last seen, optionally filtered by hash and/or IATAs
// (membership via channel_iatas). NULL hash / empty array skip those filters.
// Channels ordered by last seen, optionally filtered by hash and/or IATAs.
// A channel belongs to an IATA when MeshMapper lists it there or config scopes it
// to a region containing it (or Beacon-wide). NULL hash / empty array / NULL key_known skip those filters.
// Pass cursor=0 to start from the beginning (cursor is last_seen epoch ms).
ListChannels(ctx context.Context, arg ListChannelsParams) ([]Channel, error)
// Keep the non-null tuple boundary separate from the legacy optional cursor so
@@ -200,6 +210,8 @@ type Querier interface {
// A site that filled scan_depth still has unread history below its floor.
// The newest such floor is the point above which every site is covered.
ListPacketsByIATAs(ctx context.Context, arg ListPacketsByIATAsParams) ([]ListPacketsByIATAsRow, error)
// Every region with its members, for reconciling imported MeshMapper groups.
ListRegionState(ctx context.Context) ([]ListRegionStateRow, error)
// ============================================================
// REGIONS
// ============================================================
@@ -208,6 +220,7 @@ type Querier interface {
// Full-byte equality is required even when the compact digest matches. TRACE path bytes
// carry readings; unclassified legacy observations cannot be safely called ordinary paths.
ListRouteEvidence(ctx context.Context, arg ListRouteEvidenceParams) ([]ListRouteEvidenceRow, error)
ListScopeCatalogues(ctx context.Context) ([]MeshmapperScopeCatalogue, error)
// ============================================================
// TRACES
// ============================================================
@@ -220,7 +233,11 @@ type Querier interface {
// to the config after they'd already been ingested -- see
// internal/ingest.BackfillChannelMessages.
ListUndecryptedGroupTextPackets(ctx context.Context) ([]ListUndecryptedGroupTextPacketsRow, error)
// Like ListUndecryptedGroupTextPackets, limited to channels that just gained a key.
ListUndecryptedGroupTextPacketsByHash(ctx context.Context, hashes [][]byte) ([]ListUndecryptedGroupTextPacketsByHashRow, error)
ListZoneBoundaries(ctx context.Context) ([]MeshmapperZoneBoundary, error)
ListZoneLists(ctx context.Context) ([]MeshmapperZoneList, error)
PruneImportedRegions(ctx context.Context, keep []string) ([]string, error)
// Drops imports for IATAs no longer configured, so their manual border returns.
PruneZoneBoundaries(ctx context.Context, keep []string) ([]string, error)
// Delete node_neighbors where the neighbor has departed from node_short_ids
@@ -259,12 +276,16 @@ type Querier interface {
ResolvePathHashesP2(ctx context.Context, arg ResolvePathHashesP2Params) ([]ResolvePathHashesP2Row, error)
ResolvePathHashesP3(ctx context.Context, arg ResolvePathHashesP3Params) ([]ResolvePathHashesP3Row, error)
ResolvePathHashesP4(ctx context.Context, arg ResolvePathHashesP4Params) ([]ResolvePathHashesP4Row, error)
// NULL payload/etag/checked_at retain the last good list after an error or 304.
SaveChannelCatalogue(ctx context.Context, arg SaveChannelCatalogueParams) error
// One statement commits the validated snapshot and its lookup identities together.
// Empty arrays insert nothing. NULL payload/checked_at retain last-known-good data
// after an error or 304. Imported names never replace existing manual metadata.
SaveScopeCatalogue(ctx context.Context, arg SaveScopeCatalogueParams) error
// NULL feature/etag/checked_at retain the last good boundary after an error or 304.
SaveZoneBoundary(ctx context.Context, arg SaveZoneBoundaryParams) error
// NULL payload/etag/fetched_at retain the last good list after an error or 304.
SaveZoneList(ctx context.Context, arg SaveZoneListParams) error
// Returns known routes containing a subsequence from source to destination hash prefix.
// Verifies source appears before destination in the route.
SearchKnownRoutes(ctx context.Context, arg SearchKnownRoutesParams) ([]SearchKnownRoutesRow, error)
@@ -302,11 +323,14 @@ type Querier interface {
// bbox already computed -- see internal/config/border.go.
UpsertIATABorder(ctx context.Context, arg UpsertIATABorderParams) error
UpsertIATADetails(ctx context.Context, arg UpsertIATADetailsParams) error
// A hand-written region owns its slug: the WHERE turns a clash into no row.
UpsertImportedRegion(ctx context.Context, arg UpsertImportedRegionParams) (int32, error)
// ============================================================
// ROUTES
// ============================================================
// Route identity is path_key, an md5 of node_ids computed by the caller.
// Keep the latest processed representation without changing the node-chain identity.
// On conflict, observation_count and last_seen are bumped and hash_prefix follows the
// latest hearing, so evidence matches the hash width the route uses now.
UpsertKnownRoute(ctx context.Context, arg UpsertKnownRouteParams) error
// ============================================================
// NODES
@@ -343,7 +367,6 @@ type Querier interface {
// ============================================================
UpsertPacket(ctx context.Context, arg UpsertPacketParams) (UpsertPacketRow, error)
UpsertRegion(ctx context.Context, arg UpsertRegionParams) (int32, error)
UpsertRegionIATA(ctx context.Context, arg UpsertRegionIATAParams) error
// Refreshes at most hourly so repeat hears don't churn the row.
UpsertTraceIATA(ctx context.Context, arg UpsertTraceIATAParams) error
// ============================================================
db/sqlc/queries.sql.go
+446 -77
View File
@@ -13,6 +13,65 @@ import (
"github.com/jackc/pgx/v5/pgtype"
)
const addChannelConfigScopes = `-- name: AddChannelConfigScopes :exec
INSERT INTO channel_config_scopes (key_fingerprint, region_slug)
SELECT unnest($1::bytea[]), NULLIF(unnest($2::text[]), '')
`
type AddChannelConfigScopesParams struct {
Fingerprints [][]byte `json:"fingerprints"`
Regions []string `json:"regions"`
}
// An empty region places the channel Beacon-wide.
func (q *Queries) AddChannelConfigScopes(ctx context.Context, arg AddChannelConfigScopesParams) error {
_, err := q.db.Exec(ctx, addChannelConfigScopes, arg.Fingerprints, arg.Regions)
return err
}
const addChannelMembers = `-- name: AddChannelMembers :exec
INSERT INTO meshmapper_channel_members (iata, key_fingerprint)
SELECT $1, unnest($2::bytea[])
ON CONFLICT (iata, key_fingerprint) DO NOTHING
`
type AddChannelMembersParams struct {
Iata string `json:"iata"`
Fingerprints [][]byte `json:"fingerprints"`
}
func (q *Queries) AddChannelMembers(ctx context.Context, arg AddChannelMembersParams) error {
_, err := q.db.Exec(ctx, addChannelMembers, arg.Iata, arg.Fingerprints)
return err
}
const addIATAs = `-- name: AddIATAs :exec
INSERT INTO iata_codes (iata)
SELECT unnest($1::bpchar[])
ON CONFLICT (iata) DO NOTHING
`
func (q *Queries) AddIATAs(ctx context.Context, iatas []string) error {
_, err := q.db.Exec(ctx, addIATAs, iatas)
return err
}
const addRegionIATAs = `-- name: AddRegionIATAs :exec
INSERT INTO region_iatas (region_id, iata)
SELECT $1, unnest($2::bpchar[])
ON CONFLICT (region_id, iata) DO NOTHING
`
type AddRegionIATAsParams struct {
RegionID int32 `json:"region_id"`
Iatas []string `json:"iatas"`
}
func (q *Queries) AddRegionIATAs(ctx context.Context, arg AddRegionIATAsParams) error {
_, err := q.db.Exec(ctx, addRegionIATAs, arg.RegionID, arg.Iatas)
return err
}
const ambiguousPrefixes = `-- name: AmbiguousPrefixes :many
SELECT iata::text AS iata, 1::int AS len, prefix_1 AS prefix FROM node_short_ids GROUP BY iata, prefix_1 HAVING COUNT(*) > 1
UNION ALL
@@ -94,6 +153,38 @@ func (q *Queries) DeactivateAccount(ctx context.Context, id uuid.UUID) (Deactiva
return i, err
}
const deleteAllChannelMembers = `-- name: DeleteAllChannelMembers :exec
DELETE FROM meshmapper_channel_members
`
func (q *Queries) DeleteAllChannelMembers(ctx context.Context) error {
_, err := q.db.Exec(ctx, deleteAllChannelMembers)
return err
}
const deleteChannelConfigScopes = `-- name: DeleteChannelConfigScopes :exec
DELETE FROM channel_config_scopes
`
func (q *Queries) DeleteChannelConfigScopes(ctx context.Context) error {
_, err := q.db.Exec(ctx, deleteChannelConfigScopes)
return err
}
const deleteChannelMembersNotIn = `-- name: DeleteChannelMembersNotIn :exec
DELETE FROM meshmapper_channel_members WHERE iata = $1 AND NOT (key_fingerprint = ANY($2::bytea[]))
`
type DeleteChannelMembersNotInParams struct {
Iata string `json:"iata"`
Keep [][]byte `json:"keep"`
}
func (q *Queries) DeleteChannelMembersNotIn(ctx context.Context, arg DeleteChannelMembersNotInParams) error {
_, err := q.db.Exec(ctx, deleteChannelMembersNotIn, arg.Iata, arg.Keep)
return err
}
const deleteOldChannelIATAs = `-- name: DeleteOldChannelIATAs :exec
DELETE FROM channel_iatas WHERE last_heard < $1
`
@@ -235,6 +326,20 @@ func (q *Queries) DeleteOldTraceIATAs(ctx context.Context, lastHeard pgtype.Time
return err
}
const deleteRegionIATAsNotIn = `-- name: DeleteRegionIATAsNotIn :exec
DELETE FROM region_iatas WHERE region_id = $1 AND NOT (iata = ANY($2::bpchar[]))
`
type DeleteRegionIATAsNotInParams struct {
RegionID int32 `json:"region_id"`
Keep []string `json:"keep"`
}
func (q *Queries) DeleteRegionIATAsNotIn(ctx context.Context, arg DeleteRegionIATAsNotInParams) error {
_, err := q.db.Exec(ctx, deleteRegionIATAsNotIn, arg.RegionID, arg.Keep)
return err
}
const getAccount = `-- name: GetAccount :one
SELECT id, name, created_at, deactivated_at FROM accounts WHERE id = $1
`
@@ -1596,54 +1701,6 @@ func (q *Queries) GetScopeStats(ctx context.Context, iatas []string) ([]GetScope
return items, nil
}
const getScopesByIATAs = `-- name: GetScopesByIATAs :many
SELECT
ts.name,
COUNT(DISTINCT os.observer_id) AS observer_count,
COUNT(DISTINCT n.id) AS node_count,
COUNT(DISTINCT po.iata) AS iata_count
FROM transport_scopes ts
LEFT JOIN observer_scopes os ON os.scope_id = ts.id
LEFT JOIN observers o ON o.id = os.observer_id
LEFT JOIN packet_observations po ON po.observer_id = o.id
LEFT JOIN nodes n ON n.default_scope_id = ts.id
WHERE (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR po.iata = ANY($1::bpchar[]))
GROUP BY ts.name
ORDER BY ts.name
`
type GetScopesByIATAsRow struct {
Name string `json:"name"`
ObserverCount int64 `json:"observer_count"`
NodeCount int64 `json:"node_count"`
IataCount int64 `json:"iata_count"`
}
func (q *Queries) GetScopesByIATAs(ctx context.Context, dollar_1 []string) ([]GetScopesByIATAsRow, error) {
rows, err := q.db.Query(ctx, getScopesByIATAs, dollar_1)
if err != nil {
return nil, err
}
defer rows.Close()
items := []GetScopesByIATAsRow{}
for rows.Next() {
var i GetScopesByIATAsRow
if err := rows.Scan(
&i.Name,
&i.ObserverCount,
&i.NodeCount,
&i.IataCount,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const getStatsClockDrift = `-- name: GetStatsClockDrift :many
WITH page AS (
SELECT
@@ -2370,6 +2427,39 @@ func (q *Queries) ListAllChannelMessages(ctx context.Context, arg ListAllChannel
return items, nil
}
const listChannelCatalogues = `-- name: ListChannelCatalogues :many
SELECT iata, url, payload, etag, checked_at, attempted_at, next_attempt, last_error FROM meshmapper_channel_catalogues ORDER BY iata, attempted_at
`
func (q *Queries) ListChannelCatalogues(ctx context.Context) ([]MeshmapperChannelCatalogue, error) {
rows, err := q.db.Query(ctx, listChannelCatalogues)
if err != nil {
return nil, err
}
defer rows.Close()
items := []MeshmapperChannelCatalogue{}
for rows.Next() {
var i MeshmapperChannelCatalogue
if err := rows.Scan(
&i.Iata,
&i.Url,
&i.Payload,
&i.Etag,
&i.CheckedAt,
&i.AttemptedAt,
&i.NextAttempt,
&i.LastError,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const listChannelMessages = `-- name: ListChannelMessages :many
SELECT DISTINCT ON (cm.id) cm.id, cm.channel_id, cm.packet_hash, cm.sender_name, cm.sender_pubkey, cm.content, cm.sent_at, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, ts.name AS scope_name, p.transport_codes_present,
(SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = cm.packet_hash) AS observation_count
@@ -2541,29 +2631,36 @@ func (q *Queries) ListChannelMessagesByHash(ctx context.Context, arg ListChannel
const listChannels = `-- name: ListChannels :many
SELECT c.id, c.channel_hash, c.key_fingerprint, c.name, c.hashtag, c.is_hashtag, c.is_public, c.key_known, c.first_seen, c.last_seen, c.message_count FROM channels c
WHERE ($1::bytea IS NULL OR c.channel_hash = $1)
AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR c.channel_hash IN (
SELECT ci.channel_hash FROM channel_iatas ci
WHERE ci.iata = ANY($2::bpchar[])
))
AND ($3::timestamptz IS NULL OR c.last_seen < $3)
AND (COALESCE(cardinality($2::bpchar[]), 0) = 0
OR EXISTS (SELECT 1 FROM meshmapper_channel_members m
WHERE m.key_fingerprint = c.key_fingerprint AND m.iata::bpchar = ANY($2::bpchar[]))
OR EXISTS (SELECT 1 FROM channel_config_scopes s
WHERE s.key_fingerprint = c.key_fingerprint AND (s.region_slug IS NULL OR s.region_slug IN (
SELECT r.slug FROM regions r JOIN region_iatas ri ON ri.region_id = r.id
WHERE ri.iata = ANY($2::bpchar[])))))
AND ($3::boolean IS NULL OR COALESCE(c.key_known, false) = $3)
AND ($4::timestamptz IS NULL OR c.last_seen < $4)
ORDER BY c.last_seen DESC, c.id DESC
LIMIT $4
LIMIT $5
`
type ListChannelsParams struct {
ChannelHash []byte `json:"channel_hash"`
Iatas []string `json:"iatas"`
KeyKnown *bool `json:"key_known"`
CursorTs pgtype.Timestamptz `json:"cursor_ts"`
PageLimit int32 `json:"page_limit"`
}
// Channels ordered by last seen, optionally filtered by hash and/or IATAs
// (membership via channel_iatas). NULL hash / empty array skip those filters.
// Channels ordered by last seen, optionally filtered by hash and/or IATAs.
// A channel belongs to an IATA when MeshMapper lists it there or config scopes it
// to a region containing it (or Beacon-wide). NULL hash / empty array / NULL key_known skip those filters.
// Pass cursor=0 to start from the beginning (cursor is last_seen epoch ms).
func (q *Queries) ListChannels(ctx context.Context, arg ListChannelsParams) ([]Channel, error) {
rows, err := q.db.Query(ctx, listChannels,
arg.ChannelHash,
arg.Iatas,
arg.KeyKnown,
arg.CursorTs,
arg.PageLimit,
)
@@ -2601,11 +2698,16 @@ const listChannelsAfter = `-- name: ListChannelsAfter :many
SELECT c.id, c.channel_hash, c.key_fingerprint, c.name, c.hashtag, c.is_hashtag, c.is_public, c.key_known, c.first_seen, c.last_seen, c.message_count FROM channels c
WHERE (c.last_seen, c.id) < ($1::timestamptz, $2::integer)
AND ($3::bytea IS NULL OR c.channel_hash = $3)
AND (COALESCE(cardinality($4::bpchar[]), 0) = 0 OR c.channel_hash IN (
SELECT ci.channel_hash FROM channel_iatas ci WHERE ci.iata = ANY($4::bpchar[])
))
AND (COALESCE(cardinality($4::bpchar[]), 0) = 0
OR EXISTS (SELECT 1 FROM meshmapper_channel_members m
WHERE m.key_fingerprint = c.key_fingerprint AND m.iata::bpchar = ANY($4::bpchar[]))
OR EXISTS (SELECT 1 FROM channel_config_scopes s
WHERE s.key_fingerprint = c.key_fingerprint AND (s.region_slug IS NULL OR s.region_slug IN (
SELECT r.slug FROM regions r JOIN region_iatas ri ON ri.region_id = r.id
WHERE ri.iata = ANY($4::bpchar[])))))
AND ($5::boolean IS NULL OR COALESCE(c.key_known, false) = $5)
ORDER BY c.last_seen DESC, c.id DESC
LIMIT $5
LIMIT $6
`
type ListChannelsAfterParams struct {
@@ -2613,6 +2715,7 @@ type ListChannelsAfterParams struct {
CursorID int32 `json:"cursor_id"`
ChannelHash []byte `json:"channel_hash"`
Iatas []string `json:"iatas"`
KeyKnown *bool `json:"key_known"`
PageLimit int32 `json:"page_limit"`
}
@@ -2624,6 +2727,7 @@ func (q *Queries) ListChannelsAfter(ctx context.Context, arg ListChannelsAfterPa
arg.CursorID,
arg.ChannelHash,
arg.Iatas,
arg.KeyKnown,
arg.PageLimit,
)
if err != nil {
@@ -3735,6 +3839,54 @@ func (q *Queries) ListPacketsByIATAs(ctx context.Context, arg ListPacketsByIATAs
return items, nil
}
const listRegionState = `-- name: ListRegionState :many
SELECT r.slug, r.name, COALESCE(r.display_order, 0)::int AS display_order, r.imported, r.center_lat, r.center_lng,
COALESCE(array_agg(ri.iata::text ORDER BY ri.iata) FILTER (WHERE ri.iata IS NOT NULL), '{}')::text[] AS iatas
FROM regions r
LEFT JOIN region_iatas ri ON ri.region_id = r.id
GROUP BY r.id
ORDER BY r.slug
`
type ListRegionStateRow struct {
Slug string `json:"slug"`
Name string `json:"name"`
DisplayOrder int32 `json:"display_order"`
Imported bool `json:"imported"`
CenterLat *float64 `json:"center_lat"`
CenterLng *float64 `json:"center_lng"`
Iatas []string `json:"iatas"`
}
// Every region with its members, for reconciling imported MeshMapper groups.
func (q *Queries) ListRegionState(ctx context.Context) ([]ListRegionStateRow, error) {
rows, err := q.db.Query(ctx, listRegionState)
if err != nil {
return nil, err
}
defer rows.Close()
items := []ListRegionStateRow{}
for rows.Next() {
var i ListRegionStateRow
if err := rows.Scan(
&i.Slug,
&i.Name,
&i.DisplayOrder,
&i.Imported,
&i.CenterLat,
&i.CenterLng,
&i.Iatas,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const listRegions = `-- name: ListRegions :many
SELECT id, slug, name
@@ -3771,6 +3923,39 @@ func (q *Queries) ListRegions(ctx context.Context) ([]ListRegionsRow, error) {
return items, nil
}
const listScopeCatalogues = `-- name: ListScopeCatalogues :many
SELECT iata, url, payload, etag, checked_at, attempted_at, next_attempt, last_error FROM meshmapper_scope_catalogues ORDER BY iata, attempted_at
`
func (q *Queries) ListScopeCatalogues(ctx context.Context) ([]MeshmapperScopeCatalogue, error) {
rows, err := q.db.Query(ctx, listScopeCatalogues)
if err != nil {
return nil, err
}
defer rows.Close()
items := []MeshmapperScopeCatalogue{}
for rows.Next() {
var i MeshmapperScopeCatalogue
if err := rows.Scan(
&i.Iata,
&i.Url,
&i.Payload,
&i.Etag,
&i.CheckedAt,
&i.AttemptedAt,
&i.NextAttempt,
&i.LastError,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const listTraceTags = `-- name: ListTraceTags :many
WITH tags AS (
@@ -3908,6 +4093,37 @@ func (q *Queries) ListUndecryptedGroupTextPackets(ctx context.Context) ([]ListUn
return items, nil
}
const listUndecryptedGroupTextPacketsByHash = `-- name: ListUndecryptedGroupTextPacketsByHash :many
SELECT packet_hash, raw_payload FROM packets
WHERE payload_type = 5 AND decrypted IS NOT TRUE AND channel_hash = ANY($1::bytea[])
`
type ListUndecryptedGroupTextPacketsByHashRow struct {
PacketHash []byte `json:"packet_hash"`
RawPayload []byte `json:"raw_payload"`
}
// Like ListUndecryptedGroupTextPackets, limited to channels that just gained a key.
func (q *Queries) ListUndecryptedGroupTextPacketsByHash(ctx context.Context, hashes [][]byte) ([]ListUndecryptedGroupTextPacketsByHashRow, error) {
rows, err := q.db.Query(ctx, listUndecryptedGroupTextPacketsByHash, hashes)
if err != nil {
return nil, err
}
defer rows.Close()
items := []ListUndecryptedGroupTextPacketsByHashRow{}
for rows.Next() {
var i ListUndecryptedGroupTextPacketsByHashRow
if err := rows.Scan(&i.PacketHash, &i.RawPayload); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const listZoneBoundaries = `-- name: ListZoneBoundaries :many
SELECT iata, url, feature, etag, checked_at, attempted_at, next_attempt, last_error FROM meshmapper_zone_boundaries ORDER BY iata
`
@@ -3941,6 +4157,62 @@ func (q *Queries) ListZoneBoundaries(ctx context.Context) ([]MeshmapperZoneBound
return items, nil
}
const listZoneLists = `-- name: ListZoneLists :many
SELECT country, payload, etag, fetched_at, attempted_at, next_attempt, last_error FROM meshmapper_zone_lists ORDER BY country
`
func (q *Queries) ListZoneLists(ctx context.Context) ([]MeshmapperZoneList, error) {
rows, err := q.db.Query(ctx, listZoneLists)
if err != nil {
return nil, err
}
defer rows.Close()
items := []MeshmapperZoneList{}
for rows.Next() {
var i MeshmapperZoneList
if err := rows.Scan(
&i.Country,
&i.Payload,
&i.Etag,
&i.FetchedAt,
&i.AttemptedAt,
&i.NextAttempt,
&i.LastError,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const pruneImportedRegions = `-- name: PruneImportedRegions :many
DELETE FROM regions WHERE imported AND NOT (slug = ANY($1::text[])) RETURNING slug
`
func (q *Queries) PruneImportedRegions(ctx context.Context, keep []string) ([]string, error) {
rows, err := q.db.Query(ctx, pruneImportedRegions, keep)
if err != nil {
return nil, err
}
defer rows.Close()
items := []string{}
for rows.Next() {
var slug string
if err := rows.Scan(&slug); err != nil {
return nil, err
}
items = append(items, slug)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const pruneZoneBoundaries = `-- name: PruneZoneBoundaries :many
DELETE FROM meshmapper_zone_boundaries WHERE NOT (iata = ANY($1::text[])) RETURNING iata
`
@@ -4459,6 +4731,45 @@ func (q *Queries) ResolvePathHashesP4(ctx context.Context, arg ResolvePathHashes
return items, nil
}
const saveChannelCatalogue = `-- name: SaveChannelCatalogue :exec
INSERT INTO meshmapper_channel_catalogues (iata, url, payload, etag, checked_at, attempted_at, next_attempt, last_error)
VALUES ($1, $2, $3::jsonb, $4::text,
$5::timestamptz, $6, $7, $8)
ON CONFLICT (iata, url) DO UPDATE SET
payload = COALESCE(EXCLUDED.payload, meshmapper_channel_catalogues.payload),
etag = COALESCE(EXCLUDED.etag, meshmapper_channel_catalogues.etag),
checked_at = COALESCE(EXCLUDED.checked_at, meshmapper_channel_catalogues.checked_at),
attempted_at = EXCLUDED.attempted_at,
next_attempt = EXCLUDED.next_attempt,
last_error = EXCLUDED.last_error
`
type SaveChannelCatalogueParams struct {
Iata string `json:"iata"`
Url string `json:"url"`
Payload []byte `json:"payload"`
Etag *string `json:"etag"`
CheckedAt pgtype.Timestamptz `json:"checked_at"`
AttemptedAt pgtype.Timestamptz `json:"attempted_at"`
NextAttempt pgtype.Timestamptz `json:"next_attempt"`
LastError string `json:"last_error"`
}
// NULL payload/etag/checked_at retain the last good list after an error or 304.
func (q *Queries) SaveChannelCatalogue(ctx context.Context, arg SaveChannelCatalogueParams) error {
_, err := q.db.Exec(ctx, saveChannelCatalogue,
arg.Iata,
arg.Url,
arg.Payload,
arg.Etag,
arg.CheckedAt,
arg.AttemptedAt,
arg.NextAttempt,
arg.LastError,
)
return err
}
const saveScopeCatalogue = `-- name: SaveScopeCatalogue :exec
WITH inserted AS (
INSERT INTO transport_scopes (name, transport_key, key_fingerprint, imported_only)
@@ -4553,6 +4864,43 @@ func (q *Queries) SaveZoneBoundary(ctx context.Context, arg SaveZoneBoundaryPara
return err
}
const saveZoneList = `-- name: SaveZoneList :exec
INSERT INTO meshmapper_zone_lists (country, payload, etag, fetched_at, attempted_at, next_attempt, last_error)
VALUES ($1, $2::jsonb, $3::text,
$4::timestamptz, $5, $6, $7)
ON CONFLICT (country) DO UPDATE SET
payload = COALESCE(EXCLUDED.payload, meshmapper_zone_lists.payload),
etag = COALESCE(EXCLUDED.etag, meshmapper_zone_lists.etag),
fetched_at = COALESCE(EXCLUDED.fetched_at, meshmapper_zone_lists.fetched_at),
attempted_at = EXCLUDED.attempted_at,
next_attempt = EXCLUDED.next_attempt,
last_error = EXCLUDED.last_error
`
type SaveZoneListParams struct {
Country string `json:"country"`
Payload []byte `json:"payload"`
Etag *string `json:"etag"`
FetchedAt pgtype.Timestamptz `json:"fetched_at"`
AttemptedAt pgtype.Timestamptz `json:"attempted_at"`
NextAttempt pgtype.Timestamptz `json:"next_attempt"`
LastError string `json:"last_error"`
}
// NULL payload/etag/fetched_at retain the last good list after an error or 304.
func (q *Queries) SaveZoneList(ctx context.Context, arg SaveZoneListParams) error {
_, err := q.db.Exec(ctx, saveZoneList,
arg.Country,
arg.Payload,
arg.Etag,
arg.FetchedAt,
arg.AttemptedAt,
arg.NextAttempt,
arg.LastError,
)
return err
}
const searchKnownRoutes = `-- name: SearchKnownRoutes :many
SELECT id, node_ids, hash_prefix, iata, hop_count, first_seen, last_seen, observation_count
FROM known_routes
@@ -4956,6 +5304,41 @@ func (q *Queries) UpsertIATADetails(ctx context.Context, arg UpsertIATADetailsPa
return err
}
const upsertImportedRegion = `-- name: UpsertImportedRegion :one
INSERT INTO regions (slug, name, display_order, center_lat, center_lng, zoom_level, imported, updated_at)
VALUES ($1, $2, $3, $4, $5, NULL, TRUE, NOW())
ON CONFLICT (slug) DO UPDATE SET
name = EXCLUDED.name,
display_order = EXCLUDED.display_order,
center_lat = EXCLUDED.center_lat,
center_lng = EXCLUDED.center_lng,
updated_at = NOW()
WHERE regions.imported
RETURNING id
`
type UpsertImportedRegionParams struct {
Slug string `json:"slug"`
Name string `json:"name"`
DisplayOrder *int32 `json:"display_order"`
CenterLat *float64 `json:"center_lat"`
CenterLng *float64 `json:"center_lng"`
}
// A hand-written region owns its slug: the WHERE turns a clash into no row.
func (q *Queries) UpsertImportedRegion(ctx context.Context, arg UpsertImportedRegionParams) (int32, error) {
row := q.db.QueryRow(ctx, upsertImportedRegion,
arg.Slug,
arg.Name,
arg.DisplayOrder,
arg.CenterLat,
arg.CenterLng,
)
var id int32
err := row.Scan(&id)
return id, err
}
const upsertKnownRoute = `-- name: UpsertKnownRoute :exec
INSERT INTO known_routes (path_key, node_ids, hash_prefix, iata, hop_count)
@@ -4978,7 +5361,8 @@ type UpsertKnownRouteParams struct {
// ROUTES
// ============================================================
// Route identity is path_key, an md5 of node_ids computed by the caller.
// Keep the latest processed representation without changing the node-chain identity.
// On conflict, observation_count and last_seen are bumped and hash_prefix follows the
// latest hearing, so evidence matches the hash width the route uses now.
func (q *Queries) UpsertKnownRoute(ctx context.Context, arg UpsertKnownRouteParams) error {
_, err := q.db.Exec(ctx, upsertKnownRoute,
arg.PathKey,
@@ -5341,6 +5725,7 @@ ON CONFLICT (slug) DO UPDATE SET
center_lat = EXCLUDED.center_lat,
center_lng = EXCLUDED.center_lng,
zoom_level = EXCLUDED.zoom_level,
imported = FALSE, -- config owns the slug from now on
updated_at = NOW()
RETURNING id
`
@@ -5370,22 +5755,6 @@ func (q *Queries) UpsertRegion(ctx context.Context, arg UpsertRegionParams) (int
return id, err
}
const upsertRegionIATA = `-- name: UpsertRegionIATA :exec
INSERT INTO region_iatas (region_id, iata)
VALUES ($1, $2)
ON CONFLICT (region_id, iata) DO NOTHING
`
type UpsertRegionIATAParams struct {
RegionID int32 `json:"region_id"`
Iata string `json:"iata"`
}
func (q *Queries) UpsertRegionIATA(ctx context.Context, arg UpsertRegionIATAParams) error {
_, err := q.db.Exec(ctx, upsertRegionIATA, arg.RegionID, arg.Iata)
return err
}
const upsertTraceIATA = `-- name: UpsertTraceIATA :exec
INSERT INTO trace_iatas (trace_tag, iata, last_heard)
VALUES ($1, $2, $3)
docs/docs.go
+44 -29
View File
@@ -597,16 +597,22 @@ const docTemplate = `{
},
{
"type": "string",
"description": "Filter by IATA code",
"description": "Filter by IATA code: channels MeshMapper lists there, config channels scoped to a region containing it, and Beacon-wide config channels",
"name": "iata",
"in": "query"
},
{
"type": "string",
"description": "Filter by IATA code(s), comma-separated e.g. YOW or YOW,YYZ",
"description": "Filter by IATA code(s), comma-separated e.g. YOW or YOW,YYZ; same membership rule as iata",
"name": "iatas",
"in": "query"
},
{
"type": "boolean",
"description": "Only channels Beacon can (true) or cannot (false) decrypt; omit for all",
"name": "keyKnown",
"in": "query"
},
{
"type": "integer",
"description": "last_seen epoch ms of last item for pagination; 0 starts from the beginning",
@@ -2233,7 +2239,7 @@ const docTemplate = `{
},
"/scope-catalogues": {
"get": {
"description": "Configured sources only. Counts describe the source region, not individual nodes or links. No upstream request is made. An empty list means no configured importer; checkedAt=0 means no successful check. lastError or a past freshUntil marks stale metadata. Manual Beacon scopes and packet evidence are separate.",
"description": "Discovered sources only. Counts describe the source region, not individual nodes or links. No upstream request is made. An empty list means no configured importer; checkedAt=0 means no successful check. lastError or a past freshUntil marks stale metadata. Manual Beacon scopes and packet evidence are separate.",
"produces": [
"application/json"
],
@@ -2247,7 +2253,7 @@ const docTemplate = `{
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.Catalogue"
"$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.PublicScopeCatalogue"
}
}
}
@@ -2256,7 +2262,7 @@ const docTemplate = `{
},
"/scopes": {
"get": {
"description": "The unfiltered list includes stored imported names after an importer is disabled or a source is removed. Those historical identities are retained indefinitely; listing them does not establish observed traffic or current source membership.",
"description": "Without filters, lists every stored scope name, including imported names retained after an importer is disabled or a source is removed. With IATA or region filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; observed traffic does not add scopes.",
"produces": [
"application/json"
],
@@ -2288,7 +2294,16 @@ const docTemplate = `{
"200": {
"description": "OK",
"schema": {
"type": "object"
"type": "array",
"items": {
"type": "string"
}
}
},
"400": {
"description": "Bad Request",
"schema": {
"$ref": "#/definitions/internal_api_handlers.APIError"
}
},
"500": {
@@ -2817,7 +2832,7 @@ const docTemplate = `{
},
"/stats/scopes": {
"get": {
"description": "Counts each packet, observer and node once per scope. IATA filters use retained observations for packets/observers and node IATA memberships for nodes. Without filters, returns global totals. Scopes with zero matching counts remain listed; an empty region returns an empty array.",
"description": "Counts each packet, observer and node once per scope. IATA filters use retained observations for packets/observers and node IATA memberships for nodes. Without filters, returns global totals for every stored scope. With filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; those remain listed with zero counts. An empty region returns an empty array.",
"produces": [
"application/json"
],
@@ -5465,7 +5480,27 @@ const docTemplate = `{
}
}
},
"github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.Catalogue": {
"github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.CatalogueScope": {
"type": "object",
"properties": {
"default": {
"type": "integer"
},
"monitored": {
"type": "boolean"
},
"name": {
"type": "string"
},
"repeaters": {
"type": "integer"
},
"wardriving": {
"type": "boolean"
}
}
},
"github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.PublicScopeCatalogue": {
"type": "object",
"properties": {
"checkedAt": {
@@ -5500,26 +5535,6 @@ const docTemplate = `{
}
}
},
"github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.CatalogueScope": {
"type": "object",
"properties": {
"default": {
"type": "integer"
},
"monitored": {
"type": "boolean"
},
"name": {
"type": "string"
},
"repeaters": {
"type": "integer"
},
"wardriving": {
"type": "boolean"
}
}
},
"internal_api_handlers.APIError": {
"type": "object",
"properties": {
@@ -5595,7 +5610,7 @@ const docTemplate = `{
// SwaggerInfo holds exported Swagger Info so clients can modify it
var SwaggerInfo = &swag.Spec{
Version: "1.6.0",
Version: "2.0.0",
Host: "localhost:8080",
BasePath: "/api/v1",
Schemes: []string{"http", "https"},
docs/swagger.json
+44 -29
View File
@@ -15,7 +15,7 @@
"license": {
"name": "AGPL-3-or-later"
},
"version": "1.6.0"
"version": "2.0.0"
},
"host": "localhost:8080",
"basePath": "/api/v1",
@@ -595,16 +595,22 @@
},
{
"type": "string",
"description": "Filter by IATA code",
"description": "Filter by IATA code: channels MeshMapper lists there, config channels scoped to a region containing it, and Beacon-wide config channels",
"name": "iata",
"in": "query"
},
{
"type": "string",
"description": "Filter by IATA code(s), comma-separated e.g. YOW or YOW,YYZ",
"description": "Filter by IATA code(s), comma-separated e.g. YOW or YOW,YYZ; same membership rule as iata",
"name": "iatas",
"in": "query"
},
{
"type": "boolean",
"description": "Only channels Beacon can (true) or cannot (false) decrypt; omit for all",
"name": "keyKnown",
"in": "query"
},
{
"type": "integer",
"description": "last_seen epoch ms of last item for pagination; 0 starts from the beginning",
@@ -2231,7 +2237,7 @@
},
"/scope-catalogues": {
"get": {
"description": "Configured sources only. Counts describe the source region, not individual nodes or links. No upstream request is made. An empty list means no configured importer; checkedAt=0 means no successful check. lastError or a past freshUntil marks stale metadata. Manual Beacon scopes and packet evidence are separate.",
"description": "Discovered sources only. Counts describe the source region, not individual nodes or links. No upstream request is made. An empty list means no configured importer; checkedAt=0 means no successful check. lastError or a past freshUntil marks stale metadata. Manual Beacon scopes and packet evidence are separate.",
"produces": [
"application/json"
],
@@ -2245,7 +2251,7 @@
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.Catalogue"
"$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.PublicScopeCatalogue"
}
}
}
@@ -2254,7 +2260,7 @@
},
"/scopes": {
"get": {
"description": "The unfiltered list includes stored imported names after an importer is disabled or a source is removed. Those historical identities are retained indefinitely; listing them does not establish observed traffic or current source membership.",
"description": "Without filters, lists every stored scope name, including imported names retained after an importer is disabled or a source is removed. With IATA or region filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; observed traffic does not add scopes.",
"produces": [
"application/json"
],
@@ -2286,7 +2292,16 @@
"200": {
"description": "OK",
"schema": {
"type": "object"
"type": "array",
"items": {
"type": "string"
}
}
},
"400": {
"description": "Bad Request",
"schema": {
"$ref": "#/definitions/internal_api_handlers.APIError"
}
},
"500": {
@@ -2815,7 +2830,7 @@
},
"/stats/scopes": {
"get": {
"description": "Counts each packet, observer and node once per scope. IATA filters use retained observations for packets/observers and node IATA memberships for nodes. Without filters, returns global totals. Scopes with zero matching counts remain listed; an empty region returns an empty array.",
"description": "Counts each packet, observer and node once per scope. IATA filters use retained observations for packets/observers and node IATA memberships for nodes. Without filters, returns global totals for every stored scope. With filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; those remain listed with zero counts. An empty region returns an empty array.",
"produces": [
"application/json"
],
@@ -5463,7 +5478,27 @@
}
}
},
"github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.Catalogue": {
"github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.CatalogueScope": {
"type": "object",
"properties": {
"default": {
"type": "integer"
},
"monitored": {
"type": "boolean"
},
"name": {
"type": "string"
},
"repeaters": {
"type": "integer"
},
"wardriving": {
"type": "boolean"
}
}
},
"github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.PublicScopeCatalogue": {
"type": "object",
"properties": {
"checkedAt": {
@@ -5498,26 +5533,6 @@
}
}
},
"github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.CatalogueScope": {
"type": "object",
"properties": {
"default": {
"type": "integer"
},
"monitored": {
"type": "boolean"
},
"name": {
"type": "string"
},
"repeaters": {
"type": "integer"
},
"wardriving": {
"type": "boolean"
}
}
},
"internal_api_handlers.APIError": {
"type": "object",
"properties": {
docs/swagger.yaml
+42 -26
View File
@@ -1492,7 +1492,20 @@ definitions:
requires_restart:
type: boolean
type: object
github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.Catalogue:
github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.CatalogueScope:
properties:
default:
type: integer
monitored:
type: boolean
name:
type: string
repeaters:
type: integer
wardriving:
type: boolean
type: object
github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.PublicScopeCatalogue:
properties:
checkedAt:
type: integer
@@ -1515,19 +1528,6 @@ definitions:
url:
type: string
type: object
github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.CatalogueScope:
properties:
default:
type: integer
monitored:
type: boolean
name:
type: string
repeaters:
type: integer
wardriving:
type: boolean
type: object
internal_api_handlers.APIError:
properties:
code:
@@ -1557,7 +1557,7 @@ info:
name: AGPL-3-or-later
termsOfService: https://github.com/MeshCore-Beacon/beacon-server
title: MeshCore Beacon API
version: 1.6.0
version: 2.0.0
paths:
/admin/accounts:
get:
@@ -1939,14 +1939,21 @@ paths:
in: query
name: hash
type: string
- description: Filter by IATA code
- description: 'Filter by IATA code: channels MeshMapper lists there, config
channels scoped to a region containing it, and Beacon-wide config channels'
in: query
name: iata
type: string
- description: Filter by IATA code(s), comma-separated e.g. YOW or YOW,YYZ
- description: Filter by IATA code(s), comma-separated e.g. YOW or YOW,YYZ;
same membership rule as iata
in: query
name: iatas
type: string
- description: Only channels Beacon can (true) or cannot (false) decrypt; omit
for all
in: query
name: keyKnown
type: boolean
- description: last_seen epoch ms of last item for pagination; 0 starts from
the beginning
in: query
@@ -3064,7 +3071,7 @@ paths:
- Routes
/scope-catalogues:
get:
description: Configured sources only. Counts describe the source region, not
description: Discovered sources only. Counts describe the source region, not
individual nodes or links. No upstream request is made. An empty list means
no configured importer; checkedAt=0 means no successful check. lastError or
a past freshUntil marks stale metadata. Manual Beacon scopes and packet evidence
@@ -3076,17 +3083,18 @@ paths:
description: OK
schema:
items:
$ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.Catalogue'
$ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.PublicScopeCatalogue'
type: array
summary: Cached MeshMapper regional scope catalogues
tags:
- Scopes
/scopes:
get:
description: The unfiltered list includes stored imported names after an importer
is disabled or a source is removed. Those historical identities are retained
indefinitely; listing them does not establish observed traffic or current
source membership.
description: Without filters, lists every stored scope name, including imported
names retained after an importer is disabled or a source is removed. With
IATA or region filters, lists only manual scopes configured for a matching
region and imported scopes whose current MeshMapper catalogue includes a matching
IATA; observed traffic does not add scopes.
parameters:
- description: Filter by IATA code(s), comma-separated
in: query
@@ -3106,7 +3114,13 @@ paths:
"200":
description: OK
schema:
type: object
items:
type: string
type: array
"400":
description: Bad Request
schema:
$ref: '#/definitions/internal_api_handlers.APIError'
"500":
description: Internal Server Error
schema:
@@ -3470,8 +3484,10 @@ paths:
get:
description: Counts each packet, observer and node once per scope. IATA filters
use retained observations for packets/observers and node IATA memberships
for nodes. Without filters, returns global totals. Scopes with zero matching
counts remain listed; an empty region returns an empty array.
for nodes. Without filters, returns global totals for every stored scope.
With filters, lists only manual scopes configured for a matching region and
imported scopes whose current MeshMapper catalogue includes a matching IATA;
those remain listed with zero counts. An empty region returns an empty array.
parameters:
- description: Comma-separated IATA codes
in: query
internal/api/handlers/channels.go
+16 -3
View File
@@ -35,8 +35,9 @@ func ChannelsRouter(reader api.Reader) http.Handler {
// @Tags Channels
// @Produce json
// @Param hash query string false "Single-byte channel hash (hex)"
// @Param iata query string false "Filter by IATA code"
// @Param iatas query string false "Filter by IATA code(s), comma-separated e.g. YOW or YOW,YYZ"
// @Param iata query string false "Filter by IATA code: channels MeshMapper lists there, config channels scoped to a region containing it, and Beacon-wide config channels"
// @Param iatas query string false "Filter by IATA code(s), comma-separated e.g. YOW or YOW,YYZ; same membership rule as iata"
// @Param keyKnown query bool false "Only channels Beacon can (true) or cannot (false) decrypt; omit for all"
// @Param cursor query int false "last_seen epoch ms of last item for pagination; 0 starts from the beginning"
// @Param pageCursor query string false "Opaque nextPageCursor from a previous response; preserves timestamp ties and precision. Cannot be combined with a positive cursor; cursor=0 is allowed."
// @Param limit query int false "Max results (default 50); must be positive, values above 200 are clamped" minimum(1) maximum(200)
@@ -86,7 +87,19 @@ func listChannels(reader api.Reader) http.HandlerFunc {
}
hashHex = h
}
channels, err := reader.ListChannels(r.Context(), limit, hashHex, iatas, cursor, pageCursor)
var keyKnown *bool
if q := r.URL.Query(); q.Has("keyKnown") {
switch q.Get("keyKnown") {
case "true":
keyKnown = new(true)
case "false":
keyKnown = new(false)
default:
respondError(w, http.StatusBadRequest, "keyKnown must be true or false")
return
}
}
channels, err := reader.ListChannels(r.Context(), limit, hashHex, iatas, keyKnown, cursor, pageCursor)
if err != nil {
respondError(w, http.StatusInternalServerError, "internal server error")
return
internal/api/handlers/channels_test.go
+52 -3
View File
@@ -8,6 +8,7 @@ import (
"net/http"
"net/http/httptest"
"reflect"
"strings"
"testing"
"time"
@@ -39,7 +40,7 @@ func TestListChannels_InvalidCursor(t *testing.T) {
func TestListChannels_PageCursor(t *testing.T) {
called := false
r := ChannelsRouter(stubReader{listChannels: func(_ context.Context, limit int32, hash []byte, iatas []string, legacy int64, cursor *api.ChannelCursor) (api.ChannelPage, error) {
r := ChannelsRouter(stubReader{listChannels: func(_ context.Context, limit int32, hash []byte, iatas []string, _ *bool, legacy int64, cursor *api.ChannelCursor) (api.ChannelPage, error) {
called = true
if limit != 2 || legacy != 0 || cursor == nil || cursor.ID != 9 || !cursor.LastSeen.Equal(time.UnixMicro(1700000000000123)) ||
!reflect.DeepEqual(hash, []byte{0xaa}) || !reflect.DeepEqual(iatas, []string{"YOW"}) {
@@ -145,7 +146,7 @@ func TestListChannelMessages_InvalidCursor(t *testing.T) {
func TestListChannels_OK(t *testing.T) {
r := chi.NewRouter()
r.Get("/channels", listChannels(stubReader{
listChannels: func(_ context.Context, _ int32, _ []byte, _ []string, _ int64, _ *api.ChannelCursor) (api.ChannelPage, error) {
listChannels: func(_ context.Context, _ int32, _ []byte, _ []string, _ *bool, _ int64, _ *api.ChannelCursor) (api.ChannelPage, error) {
return api.ChannelPage{Page: api.Page[api.ChannelSummary]{Items: []api.ChannelSummary{{ID: 1, ChannelHash: "ab"}}}}, nil
},
}))
@@ -172,7 +173,7 @@ func TestListChannels_IATAParsing(t *testing.T) {
var got []string
r := chi.NewRouter()
r.Get("/channels", listChannels(stubReader{
listChannels: func(_ context.Context, _ int32, _ []byte, iatas []string, _ int64, _ *api.ChannelCursor) (api.ChannelPage, error) {
listChannels: func(_ context.Context, _ int32, _ []byte, iatas []string, _ *bool, _ int64, _ *api.ChannelCursor) (api.ChannelPage, error) {
got = iatas
return api.ChannelPage{}, nil
},
@@ -190,6 +191,54 @@ func TestListChannels_IATAParsing(t *testing.T) {
}
}
func TestListChannels_KeyKnownParsing(t *testing.T) {
yes, no := true, false
cases := []struct {
name string
query string
code int
want *bool
}{
{"true", "?keyKnown=true", http.StatusOK, &yes},
{"false", "?keyKnown=false", http.StatusOK, &no},
{"absent", "", http.StatusOK, nil},
{"uppercase", "?keyKnown=TRUE", http.StatusBadRequest, nil},
{"numeric", "?keyKnown=1", http.StatusBadRequest, nil},
{"empty", "?keyKnown=", http.StatusBadRequest, nil},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
var got *bool
called := false
r := chi.NewRouter()
r.Get("/channels", listChannels(stubReader{
listChannels: func(_ context.Context, _ int32, _ []byte, _ []string, keyKnown *bool, _ int64, _ *api.ChannelCursor) (api.ChannelPage, error) {
called = true
got = keyKnown
return api.ChannelPage{}, nil
},
}))
w := httptest.NewRecorder()
r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/channels"+tc.query, nil))
if w.Code != tc.code {
t.Fatalf("expected %d, got %d", tc.code, w.Code)
}
if tc.code != http.StatusOK {
if called {
t.Error("reader called for invalid keyKnown")
}
if !strings.Contains(w.Body.String(), "keyKnown must be true or false") {
t.Errorf("unexpected error body %s", w.Body.String())
}
return
}
if !reflect.DeepEqual(got, tc.want) {
t.Errorf("expected keyKnown %v, got %v", tc.want, got)
}
})
}
}
func TestGetChannel_OK(t *testing.T) {
r := chi.NewRouter()
r.Get("/channels/{channelID}", getChannel(stubReader{
internal/api/handlers/limits_test.go
+2 -2
View File
@@ -19,7 +19,7 @@ func TestListLimits(t *testing.T) {
var got int32
var calls int
reader := stubReader{
listChannels: func(ctx context.Context, limit int32, hash []byte, iatas []string, cursor int64, pageCursor *api.ChannelCursor) (api.ChannelPage, error) {
listChannels: func(ctx context.Context, limit int32, hash []byte, iatas []string, keyKnown *bool, cursor int64, pageCursor *api.ChannelCursor) (api.ChannelPage, error) {
calls++
got = limit
return api.ChannelPage{}, nil
@@ -113,7 +113,7 @@ func TestListLimits(t *testing.T) {
router.Mount("/messages", MessagesRouter(reader))
router.Mount("/routes", RoutesRouter(reader))
router.Mount("/traces", TracesRouter(reader))
router.Mount("/stats", StatsRouter(reader))
router.Mount("/stats", StatsRouter(reader, nil))
for _, endpoint := range []struct {
path string
defaultLimit int32
internal/api/handlers/observer_comparison_test.go
+3 -3
View File
@@ -28,7 +28,7 @@ func TestObserverComparisonRejectsInvalidQuery(t *testing.T) {
} {
t.Run(query, func(t *testing.T) {
w := httptest.NewRecorder()
StatsRouter(nil).ServeHTTP(w, httptest.NewRequest("GET", "/observer-comparison?"+query, nil))
StatsRouter(nil, nil).ServeHTTP(w, httptest.NewRequest("GET", "/observer-comparison?"+query, nil))
if w.Code != 400 {
t.Fatalf("status %d; wanted validation error", w.Code)
}
@@ -42,7 +42,7 @@ func TestObserverComparisonHTTP(t *testing.T) {
query := "observerA=" + a.String() + "&observerB=" + b.String() + "&since=0&until=1000"
for _, suffix := range []string{"&since=2", "&observerA=" + b.String(), "&until=253402300800000", "&observerB=00000000-0000-0000-0000-000000000000"} {
w := httptest.NewRecorder()
StatsRouter(nil).ServeHTTP(w, httptest.NewRequest("GET", "/observer-comparison?"+query+suffix, nil))
StatsRouter(nil, nil).ServeHTTP(w, httptest.NewRequest("GET", "/observer-comparison?"+query+suffix, nil))
if w.Code != 400 {
t.Fatalf("invalid query returned %d", w.Code)
}
@@ -80,7 +80,7 @@ func TestObserverComparisonHTTP(t *testing.T) {
},
}
w := httptest.NewRecorder()
StatsRouter(reader).ServeHTTP(w, httptest.NewRequest("GET", "/observer-comparison?"+query+tc.suffix, nil))
StatsRouter(reader, nil).ServeHTTP(w, httptest.NewRequest("GET", "/observer-comparison?"+query+tc.suffix, nil))
if w.Code != tc.status {
t.Fatalf("status = %d, body = %s", w.Code, w.Body.String())
}
internal/api/handlers/paths_test.go
+3 -3
View File
@@ -28,7 +28,7 @@ func (s pathReader) GetPathStats(ctx context.Context, since, until time.Time, ia
func TestPathsRejectInvalidWindow(t *testing.T) {
for _, query := range []string{"", "since=0", "since=x&until=1", "since=-1&until=1", "since=1&until=1", "since=2&until=1", "since=0&until=2592000001", "since=0&since=0&until=1", "since=0&until=1&until=2", "since=253402300799998&until=253402300800000"} {
w := httptest.NewRecorder()
StatsRouter(nil).ServeHTTP(w, httptest.NewRequest("GET", "/paths?"+query, nil))
StatsRouter(nil, nil).ServeHTTP(w, httptest.NewRequest("GET", "/paths?"+query, nil))
if w.Code != 400 {
t.Fatalf("query %q: %d %s", query, w.Code, w.Body.String())
}
@@ -66,7 +66,7 @@ func TestPathsFiltersErrorsAndCancellation(t *testing.T) {
return &api.PathStats{Until: until.UnixMilli(), Receptions: 4, Hashed: 1, Empty: 1, Trace: 1, Unclassified: 1}, tc.err
}}
w := httptest.NewRecorder()
StatsRouter(reader).ServeHTTP(w, httptest.NewRequest("GET", "/paths?since=0&until=2592000000"+tc.query, nil))
StatsRouter(reader, nil).ServeHTTP(w, httptest.NewRequest("GET", "/paths?since=0&until=2592000000"+tc.query, nil))
if w.Code != tc.status || strings.Contains(w.Body.String(), "private detail") {
t.Fatalf("%d %s", w.Code, w.Body.String())
}
@@ -86,7 +86,7 @@ func TestPathsFiltersErrorsAndCancellation(t *testing.T) {
return nil, ctx.Err()
}}
w := httptest.NewRecorder()
StatsRouter(r).ServeHTTP(w, httptest.NewRequest("GET", "/paths?since=0&until=1", nil).WithContext(ctx))
StatsRouter(r, nil).ServeHTTP(w, httptest.NewRequest("GET", "/paths?since=0&until=1", nil).WithContext(ctx))
if !called || w.Body.Len() != 0 {
t.Fatal("cancellation did not reach reader")
}
internal/api/handlers/scope_catalogues.go
+4 -4
View File
@@ -11,19 +11,19 @@ import (
// ScopeCatalogueSource supplies immutable, already-cached public metadata.
type ScopeCatalogueSource interface {
Catalogues() []meshmapper.Catalogue
Catalogues() []meshmapper.PublicScopeCatalogue
}
// ScopeCatalogues godoc
// @Summary Cached MeshMapper regional scope catalogues
// @Description Configured sources only. Counts describe the source region, not individual nodes or links. No upstream request is made. An empty list means no configured importer; checkedAt=0 means no successful check. lastError or a past freshUntil marks stale metadata. Manual Beacon scopes and packet evidence are separate.
// @Description Discovered sources only. Counts describe the source region, not individual nodes or links. No upstream request is made. An empty list means no configured importer; checkedAt=0 means no successful check. lastError or a past freshUntil marks stale metadata. Manual Beacon scopes and packet evidence are separate.
// @Tags Scopes
// @Produce json
// @Success 200 {array} meshmapper.Catalogue
// @Success 200 {array} meshmapper.PublicScopeCatalogue
// @Router /scope-catalogues [get]
func ScopeCatalogues(source ScopeCatalogueSource) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
items := []meshmapper.Catalogue{}
items := []meshmapper.PublicScopeCatalogue{}
if source != nil {
items = source.Catalogues()
}
internal/api/handlers/scope_stats_test.go
+41 -3
View File
@@ -13,6 +13,7 @@ import (
"testing"
"github.com/MeshCore-Beacon/beacon-server/internal/api"
"github.com/MeshCore-Beacon/beacon-server/internal/scopestore"
)
func TestGetStatsScopes_RegionErrors(t *testing.T) {
@@ -27,7 +28,7 @@ func TestGetStatsScopes_RegionErrors(t *testing.T) {
},
}
w := httptest.NewRecorder()
getStatsScopes(reader)(w, httptest.NewRequest(http.MethodGet, "/stats/scopes?"+query, nil))
getStatsScopes(reader, nil)(w, httptest.NewRequest(http.MethodGet, "/stats/scopes?"+query, nil))
if w.Code != http.StatusBadRequest {
t.Fatalf("got status %d, want 400", w.Code)
}
@@ -79,7 +80,7 @@ func TestGetStatsScopes_Filters(t *testing.T) {
},
}
w := httptest.NewRecorder()
getStatsScopes(reader)(w, httptest.NewRequest(http.MethodGet, "/stats/scopes?"+tc.query, nil))
getStatsScopes(reader, testScopes(map[string][]string{"YVR": {"#test"}, "YYJ": {"#test"}}))(w, httptest.NewRequest(http.MethodGet, "/stats/scopes?"+tc.query, nil))
if w.Code != http.StatusOK {
t.Fatalf("status = %d: %s", w.Code, w.Body.String())
}
@@ -105,8 +106,45 @@ func TestGetStatsScopes_ReaderError(t *testing.T) {
w := httptest.NewRecorder()
getStatsScopes(stubReader{getScopeStats: func(context.Context, []string) ([]api.ScopeStats, error) {
return nil, errors.New("database unavailable")
}})(w, httptest.NewRequest(http.MethodGet, "/stats/scopes?iata=YVR", nil))
}}, nil)(w, httptest.NewRequest(http.MethodGet, "/stats/scopes?iata=YVR", nil))
if w.Code != http.StatusInternalServerError {
t.Fatalf("status = %d, want 500", w.Code)
}
}
func testScopes(byIATA map[string][]string) *scopestore.ScopeStore {
s := scopestore.New()
s.SetManualMembers(byIATA)
return s
}
func TestGetStatsScopes_FiltersToRegionMembers(t *testing.T) {
reader := stubReader{getScopeStats: func(context.Context, []string) ([]api.ScopeStats, error) {
return []api.ScopeStats{{Name: "#elsewhere", PacketCount: 9}, {Name: "#on"}, {Name: "#yow", PacketCount: 4}}, nil
}}
members := testScopes(map[string][]string{"YOW": {"#yow", "#on"}})
for _, tc := range []struct {
query string
members ScopeMembership
want []string
}{
{"", members, []string{"#elsewhere", "#on", "#yow"}},
{"iata=YOW", members, []string{"#on", "#yow"}},
{"iata=YUL", members, []string{}},
{"iata=YOW", nil, []string{}},
} {
w := httptest.NewRecorder()
getStatsScopes(reader, tc.members)(w, httptest.NewRequest(http.MethodGet, "/stats/scopes?"+tc.query, nil))
var rows []api.ScopeStats
if err := json.Unmarshal(w.Body.Bytes(), &rows); err != nil || rows == nil {
t.Fatalf("%q: body %s, err %v", tc.query, w.Body.String(), err)
}
got := []string{}
for _, row := range rows {
got = append(got, row.Name)
}
if !slices.Equal(got, tc.want) {
t.Fatalf("%q: got %v, want %v", tc.query, got, tc.want)
}
}
}
internal/api/handlers/scopes.go
+23 -15
View File
@@ -10,13 +10,25 @@ import (
"github.com/go-chi/chi/v5"
)
// ScopeMembership lists scopes by configured region and imported catalogue membership.
type ScopeMembership interface {
NamesForIATAs(iatas []string) []string // sorted
}
func scopeNamesFor(scopes ScopeMembership, iatas []string) []string {
if scopes == nil {
return []string{}
}
return scopes.NamesForIATAs(iatas)
}
// ScopesRouter mounts all /scopes routes onto a subrouter.
//
// GET /scopes → listScopes
// GET /scopes/{name} → getScope
func ScopesRouter(reader api.Reader) http.Handler {
func ScopesRouter(reader api.Reader, scopes ScopeMembership) http.Handler {
r := chi.NewRouter()
r.Get("/", listScopes(reader))
r.Get("/", listScopes(reader, scopes))
r.Get("/{name}", getScope(reader))
return r
}
@@ -24,27 +36,28 @@ func ScopesRouter(reader api.Reader) http.Handler {
// listScopes godoc
//
// @Summary List transport scopes
// @Description The unfiltered list includes stored imported names after an importer is disabled or a source is removed. Those historical identities are retained indefinitely; listing them does not establish observed traffic or current source membership.
// @Description Without filters, lists every stored scope name, including imported names retained after an importer is disabled or a source is removed. With IATA or region filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; observed traffic does not add scopes.
// @Tags Scopes
// @Produce json
// @Param iatas query string false "Filter by IATA code(s), comma-separated"
// @Param region query string false "Filter by region slug"
// @Param regionId query int false "Filter by region ID"
// @Success 200 {object} object
// @Success 200 {array} string
// @Failure 400 {object} handlers.APIError
// @Failure 500 {object} handlers.APIError
// @Router /scopes [get]
func listScopes(reader api.Reader) http.HandlerFunc {
func listScopes(reader api.Reader, scopes ScopeMembership) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
iatas := parseIATAs(r)
if regionIDStr := r.URL.Query().Get("regionId"); regionIDStr != "" || r.URL.Query().Get("region") != "" {
regionIATAs, err := resolveRegionIATAs(r.Context(), r.URL.Query().Get("regionId"), r.URL.Query().Get("region"), reader)
regionID, region := r.URL.Query().Get("regionId"), r.URL.Query().Get("region")
if regionID != "" || region != "" {
regionIATAs, err := resolveRegionIATAs(r.Context(), regionID, region, reader)
if err != nil {
respondError(w, http.StatusBadRequest, err.Error())
return
}
iatas = append(iatas, regionIATAs...)
}
if len(iatas) == 0 {
} else if len(iatas) == 0 {
names, err := reader.GetScopeNames(r.Context())
if err != nil {
respondError(w, http.StatusInternalServerError, "internal server error")
@@ -53,12 +66,7 @@ func listScopes(reader api.Reader) http.HandlerFunc {
respond(w, http.StatusOK, names)
return
}
scopes, err := reader.GetScopesByIATAs(r.Context(), iatas)
if err != nil {
respondError(w, http.StatusInternalServerError, "internal server error")
return
}
respond(w, http.StatusOK, scopes)
respond(w, http.StatusOK, scopeNamesFor(scopes, iatas))
}
}
internal/api/handlers/scopes_test.go
+40 -1
View File
@@ -5,8 +5,10 @@ package handlers
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"slices"
"testing"
"github.com/MeshCore-Beacon/beacon-server/internal/api"
@@ -19,7 +21,7 @@ func TestListScopes_NoIATAs_OK(t *testing.T) {
getScopeNames: func(_ context.Context) ([]string, error) {
return []string{"#bc", "#west"}, nil
},
}))
}, nil))
req := httptest.NewRequest(http.MethodGet, "/scopes", nil)
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
@@ -42,3 +44,40 @@ func TestGetScope_OK(t *testing.T) {
t.Errorf("expected 200, got %d", w.Code)
}
}
func TestListScopes_FiltersByMembership(t *testing.T) {
reader := stubReader{
getScopeNames: func(context.Context) ([]string, error) {
t.Fatal("filtered list must not fall back to every stored name")
return nil, nil
},
getRegionBySlug: func(_ context.Context, slug string) (*api.Region, error) {
if slug == "empty" {
return &api.Region{}, nil
}
return &api.Region{IATAs: []string{"YOW", "YGK"}}, nil
},
}
members := testScopes(map[string][]string{"YOW": {"#on", "#yow"}, "YGK": {"#on"}, "YUL": {"#qc"}})
for _, tc := range []struct {
query string
members ScopeMembership
want []string
}{
{"iatas=yow", members, []string{"#on", "#yow"}},
{"iatas=YUL,YYZ", members, []string{"#qc"}},
{"region=ottawa", members, []string{"#on", "#yow"}},
{"region=empty", members, []string{}},
{"iatas=YOW", nil, []string{}},
} {
w := httptest.NewRecorder()
listScopes(reader, tc.members)(w, httptest.NewRequest(http.MethodGet, "/scopes?"+tc.query, nil))
var got []string
if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil || w.Code != http.StatusOK || got == nil {
t.Fatalf("%q: status %d body %s", tc.query, w.Code, w.Body.String())
}
if !slices.Equal(got, tc.want) {
t.Fatalf("%q: got %v, want %v", tc.query, got, tc.want)
}
}
}
internal/api/handlers/signal_test.go
+3 -3
View File
@@ -28,7 +28,7 @@ func (s signalReader) GetSignalStats(ctx context.Context, since, until time.Time
func TestSignalRejectsInvalidWindow(t *testing.T) {
for _, query := range []string{"", "since=0", "until=1", "since=x&until=1", "since=-1&until=1", "since=0&until=0", "since=2&until=1", "since=0&until=2592000001", "since=0&since=0&until=1", "since=0&until=1&until=2", "since=253402300799998&until=253402300800000"} {
w := httptest.NewRecorder()
StatsRouter(nil).ServeHTTP(w, httptest.NewRequest("GET", "/signal?"+query, nil))
StatsRouter(nil, nil).ServeHTTP(w, httptest.NewRequest("GET", "/signal?"+query, nil))
if w.Code != 400 {
t.Fatalf("query %q: %d %s", query, w.Code, w.Body.String())
}
@@ -70,7 +70,7 @@ func TestSignalFiltersAndErrors(t *testing.T) {
return &api.SignalStats{Until: until.UnixMilli(), Receptions: 5, Hourly: []api.SignalHour{}}, tc.err
}}
w := httptest.NewRecorder()
StatsRouter(reader).ServeHTTP(w, httptest.NewRequest("GET", "/signal?since=0&until=2592000000"+tc.query, nil))
StatsRouter(reader, nil).ServeHTTP(w, httptest.NewRequest("GET", "/signal?since=0&until=2592000000"+tc.query, nil))
if w.Code != tc.status || strings.Contains(w.Body.String(), "private detail") {
t.Fatalf("%d %s", w.Code, w.Body.String())
}
@@ -93,7 +93,7 @@ func TestSignalCancellationReachesReader(t *testing.T) {
return nil, ctx.Err()
}}
w := httptest.NewRecorder()
StatsRouter(reader).ServeHTTP(w, httptest.NewRequest("GET", "/signal?since=0&until=1", nil).WithContext(ctx))
StatsRouter(reader, nil).ServeHTTP(w, httptest.NewRequest("GET", "/signal?since=0&until=1", nil).WithContext(ctx))
if !called || w.Body.Len() != 0 {
t.Fatalf("cancellation: called=%v body=%s", called, w.Body.String())
}
internal/api/handlers/stats.go
+12 -4
View File
@@ -6,6 +6,7 @@ package handlers
import (
"log/slog"
"net/http"
"slices"
"strconv"
"time"
@@ -26,7 +27,7 @@ import (
// GET /stats/scopes → GetStatsScopes
//
// All endpoints accept an optional iata= filter (case-insensitive).
func StatsRouter(reader api.Reader) http.Handler {
func StatsRouter(reader api.Reader, scopes ScopeMembership) http.Handler {
r := chi.NewRouter()
r.Get("/overview", getStatsOverview(reader))
r.Get("/observations", getStatsObservations(reader))
@@ -40,7 +41,7 @@ func StatsRouter(reader api.Reader) http.Handler {
r.Get("/clock-drift", getStatsClockDrift(reader))
r.Get("/top-talkers", getStatsTopTalkers(reader))
r.Get("/radio-presets", getStatsRadioPresets(reader))
r.Get("/scopes", getStatsScopes(reader))
r.Get("/scopes", getStatsScopes(reader, scopes))
r.Get("/node-types", getStatsNodeTypes(reader))
return r
}
@@ -417,7 +418,7 @@ func getStatsRadioPresets(reader api.Reader) http.HandlerFunc {
// getStatsScopes godoc
//
// @Summary Scope statistics
// @Description Counts each packet, observer and node once per scope. IATA filters use retained observations for packets/observers and node IATA memberships for nodes. Without filters, returns global totals. Scopes with zero matching counts remain listed; an empty region returns an empty array.
// @Description Counts each packet, observer and node once per scope. IATA filters use retained observations for packets/observers and node IATA memberships for nodes. Without filters, returns global totals for every stored scope. With filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; those remain listed with zero counts. An empty region returns an empty array.
// @Tags Stats
// @Produce json
// @Param iatas query string false "Comma-separated IATA codes"
@@ -428,7 +429,7 @@ func getStatsRadioPresets(reader api.Reader) http.HandlerFunc {
// @Failure 400 {object} handlers.APIError
// @Failure 500 {object} handlers.APIError
// @Router /stats/scopes [get]
func getStatsScopes(reader api.Reader) http.HandlerFunc {
func getStatsScopes(reader api.Reader, scopes ScopeMembership) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
iatas := parseIATAs(r)
if regionID := r.URL.Query().Get("regionId"); regionID != "" || r.URL.Query().Get("region") != "" {
@@ -448,6 +449,13 @@ func getStatsScopes(reader api.Reader) http.HandlerFunc {
respondError(w, http.StatusInternalServerError, "internal server error")
return
}
if len(iatas) > 0 {
members := scopeNamesFor(scopes, iatas)
stats = slices.DeleteFunc(slices.Clone(stats), func(s api.ScopeStats) bool {
_, found := slices.BinarySearch(members, s.Name)
return !found
})
}
respond(w, http.StatusOK, stats)
}
}
internal/api/handlers/stats_test.go
+1 -1
View File
@@ -285,7 +285,7 @@ func TestGetStatsScopes_OK(t *testing.T) {
getScopeStats: func(_ context.Context, _ []string) ([]api.ScopeStats, error) {
return []api.ScopeStats{{Name: "#bc", PacketCount: 100}}, nil
},
}))
}, nil))
req := httptest.NewRequest(http.MethodGet, "/stats/scopes", nil)
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
internal/api/handlers/stub_reader_test.go
+3 -11
View File
@@ -24,7 +24,7 @@ type stubReader struct {
listRegions func(ctx context.Context) ([]api.RegionSummary, error)
getRegion func(ctx context.Context, regionID int32) (*api.Region, error)
getRegionBySlug func(ctx context.Context, slug string) (*api.Region, error)
listChannels func(ctx context.Context, limit int32, hash []byte, iatas []string, cursor int64, pageCursor *api.ChannelCursor) (api.ChannelPage, error)
listChannels func(ctx context.Context, limit int32, hash []byte, iatas []string, keyKnown *bool, cursor int64, pageCursor *api.ChannelCursor) (api.ChannelPage, error)
getChannel func(ctx context.Context, channelID int32) (*api.Channel, error)
listChannelMessages func(ctx context.Context, channelID *int32, since time.Time, limit int32, iatas []string, scope string, cursor int64) (api.Page[api.ChannelMessage], error)
listChannelMessagesByHash func(ctx context.Context, hash []byte, since time.Time, limit int32, iatas []string, scope string, cursor int64) (api.Page[api.ChannelMessage], error)
@@ -55,7 +55,6 @@ type stubReader struct {
getScopeStats func(ctx context.Context, iatas []string) ([]api.ScopeStats, error)
getStatsNodeTypes func(ctx context.Context, iatas []string) ([]api.NodeTypeCount, error)
getScopeNames func(ctx context.Context) ([]string, error)
getScopesByIATAs func(ctx context.Context, iatas []string) ([]api.ScopeSummary, error)
getScopeByName func(ctx context.Context, name string) (*api.ScopeDetail, error)
listTraceTags func(ctx context.Context, iatas []string, scope, traceType string, since, until time.Time, cursor time.Time, limit int32) ([]api.TraceTagSummary, error)
getTraceByTag func(ctx context.Context, tag string) (*api.TraceDetail, error)
@@ -124,9 +123,9 @@ func (s stubReader) GetRegionBySlug(ctx context.Context, slug string) (*api.Regi
return nil, nil
}
func (s stubReader) ListChannels(ctx context.Context, limit int32, hash []byte, iatas []string, cursor int64, pageCursor *api.ChannelCursor) (api.ChannelPage, error) {
func (s stubReader) ListChannels(ctx context.Context, limit int32, hash []byte, iatas []string, keyKnown *bool, cursor int64, pageCursor *api.ChannelCursor) (api.ChannelPage, error) {
if s.listChannels != nil {
return s.listChannels(ctx, limit, hash, iatas, cursor, pageCursor)
return s.listChannels(ctx, limit, hash, iatas, keyKnown, cursor, pageCursor)
}
return api.ChannelPage{}, nil
}
@@ -341,13 +340,6 @@ func (s stubReader) GetScopeNames(ctx context.Context) ([]string, error) {
return nil, nil
}
func (s stubReader) GetScopesByIATAs(ctx context.Context, iatas []string) ([]api.ScopeSummary, error) {
if s.getScopesByIATAs != nil {
return s.getScopesByIATAs(ctx, iatas)
}
return nil, nil
}
func (s stubReader) GetScopeByName(ctx context.Context, name string) (*api.ScopeDetail, error) {
if s.getScopeByName != nil {
return s.getScopeByName(ctx, name)
internal/api/reader.go
+2 -6
View File
@@ -53,11 +53,11 @@ type Reader interface {
// ListChannels returns a paginated list of channels ordered by last seen.
// Includes both hashtag-derived and explicit key channels.
// Pass nil hash to skip hash filtering. Pass empty iatas to return all channels;
// IATAs must be uppercase.
// IATAs must be uppercase. Nil keyKnown skips the decryption-key filter.
// cursor is last_seen epoch ms of the last item; pass 0 to start from the beginning.
// pageCursor is the precise timestamp/ID boundary; nil preserves the legacy numeric cursor.
// When non-nil, pageCursor takes precedence over cursor.
ListChannels(ctx context.Context, limit int32, hash []byte, iatas []string, cursor int64, pageCursor *ChannelCursor) (ChannelPage, error)
ListChannels(ctx context.Context, limit int32, hash []byte, iatas []string, keyKnown *bool, cursor int64, pageCursor *ChannelCursor) (ChannelPage, error)
// GetChannel returns full detail for a single channel by its integer ID.
// Returns nil, pgx.ErrNoRows if the channel is not found.
@@ -210,10 +210,6 @@ type Reader interface {
// Use when no geographic filter is applied — returns names only for a lightweight response.
GetScopeNames(ctx context.Context) ([]string, error)
// GetScopesByIATAs returns scope summaries filtered by the given IATA codes,
// including observer, node and IATA counts. Expands region/regionId to IATAs automatically.
GetScopesByIATAs(ctx context.Context, iatas []string) ([]ScopeSummary, error)
// GetScopeByName returns full detail for a single scope by its normalized name (e.g. "#bc"),
// including packet count, observer count, node count, and the list of IATAs it is active in.
// Returns nil if the scope is not found.
internal/api/router/options.go
+1 -1
View File
@@ -7,7 +7,6 @@ import (
"net/http"
"github.com/MeshCore-Beacon/beacon-server/internal/api/handlers"
"github.com/MeshCore-Beacon/beacon-server/internal/config"
)
@@ -22,6 +21,7 @@ type Options struct {
Server config.ServerConfig
Auth config.AuthConfig
RateLimit config.ResolvedRateLimitConfig
Scopes handlers.ScopeMembership // region-filtered scope lists; nil lists none
// AdminRoutes mounts operator-only subrouters at literal paths such as
// "/accounts". Each feature owns its dependencies, methods and handlers.
// The router applies authentication to every path and method in this group.
internal/api/router/router.go
+2 -2
View File
@@ -107,9 +107,9 @@ func New(h *hub.Hub, reader api.Reader, workers []*ingest.Worker, opts Options)
r.Mount("/iatas", handlers.IATAsRouter(reader))
r.Mount("/regions", handlers.RegionsRouter(reader))
r.Mount("/routes", handlers.RoutesRouter(reader))
r.Mount("/scopes", handlers.ScopesRouter(reader))
r.Mount("/scopes", handlers.ScopesRouter(reader, opts.Scopes))
r.Mount("/stats", handlers.StatsRouter(reader, opts.Scopes))
r.Get("/scope-catalogues", handlers.ScopeCatalogues(opts.ScopeCatalogues))
r.Mount("/stats", handlers.StatsRouter(reader))
r.Mount("/traces", handlers.TracesRouter(reader))
})
internal/api/scopes.go
-8
View File
@@ -3,14 +3,6 @@
package api
// ScopeSummary is the minimal scope representation used in filtered list responses.
type ScopeSummary struct {
Name string `json:"name"`
ObserverCount int64 `json:"observerCount"`
NodeCount int64 `json:"nodeCount"`
IATACount int64 `json:"iataCount"`
}
// ScopeDetail is the full scope representation including packet count and IATA list.
type ScopeDetail struct {
Name string `json:"name"`
internal/cache/cache_test.go
Vendored
+2 -6
View File
@@ -76,10 +76,6 @@ func (s *stubReader) GetScopeNames(_ context.Context) ([]string, error) { return
func (s *stubReader) GetScopeStats(_ context.Context, _ []string) ([]api.ScopeStats, error) {
return nil, nil
}
func (s *stubReader) GetScopesByIATAs(_ context.Context, _ []string) ([]api.ScopeSummary, error) {
return nil, nil
}
func (s *stubReader) GetScopeByName(_ context.Context, _ string) (*api.ScopeDetail, error) {
return nil, nil
}
@@ -168,7 +164,7 @@ func (s *stubReader) GetCrossIATANeighbors(_ context.Context, _ uuid.UUID, _ str
return nil, nil
}
func (s *stubReader) ListChannels(_ context.Context, _ int32, _ []byte, _ []string, _ int64, _ *api.ChannelCursor) (api.ChannelPage, error) {
func (s *stubReader) ListChannels(_ context.Context, _ int32, _ []byte, _ []string, _ *bool, _ int64, _ *api.ChannelCursor) (api.ChannelPage, error) {
return api.ChannelPage{}, nil
}
@@ -386,7 +382,7 @@ func TestCachedReader_IATASortingForStableKey(t *testing.T) {
func TestInvalidateScopeNamesClearsDerivedKeys(t *testing.T) {
c, mr := newTestClient(t)
cr := &CachedReader{inner: &stubReader{}, c: c, ttl: CacheTTLs{}}
for _, k := range []string{keyScopeNames, keyScopesByIATAsPrefix + "YOW", keyScopesByIATAsPrefix + "YOW,YVR", keyScopeByNamePrefix + "#yow"} {
for _, k := range []string{keyScopeNames, keyScopeByNamePrefix + "#yow"} {
mr.Set(k, "x")
}
mr.Set("beacon:iatas", "keep")
internal/cache/reader.go
Vendored
+14 -15
View File
@@ -25,7 +25,6 @@ const (
keyRegionSlugPrefix = "beacon:region:slug:"
keyScopeNames = "beacon:scope:names"
keyScopeStats = "beacon:scope:stats"
keyScopesByIATAsPrefix = "beacon:scopes:iatas:"
keyScopeByNamePrefix = "beacon:scope:name:"
keyStatsOverviewPrefix = "beacon:stats:overview:"
keyStatsObservationsPrefix = "beacon:stats:observations:"
@@ -83,7 +82,6 @@ func NewCachedReader(inner api.Reader, c *Client, ttl CacheTTLs) api.Reader {
// InvalidateScopeNames makes newly committed catalogue names visible to every scope read.
func (cr *CachedReader) InvalidateScopeNames(ctx context.Context) {
cr.c.del(ctx, keyScopeNames)
cr.c.delPrefix(ctx, keyScopesByIATAsPrefix)
cr.c.delPrefix(ctx, keyScopeByNamePrefix)
}
@@ -92,6 +90,18 @@ func (cr *CachedReader) InvalidateIATABorder(ctx context.Context, iata string) {
cr.c.del(ctx, keyIATABorderPrefix+iata)
}
// InvalidateIATAs makes imported MeshMapper names and locations visible to every IATA read.
func (cr *CachedReader) InvalidateIATAs(ctx context.Context) {
cr.c.del(ctx, keyIATAs)
cr.c.delPrefix(ctx, keyIATAPrefix)
}
// InvalidateRegions makes imported MeshMapper regions visible to every region read.
func (cr *CachedReader) InvalidateRegions(ctx context.Context) {
cr.c.del(ctx, keyRegions)
cr.c.delPrefix(ctx, keyRegionPrefix)
}
// InvalidateNode removes the cached entries for a node by UUID.
// Should be called from the ingest path after a node upsert.
func (cr *CachedReader) InvalidateNode(ctx context.Context, nodeID uuid.UUID) {
@@ -174,17 +184,6 @@ func (cr *CachedReader) GetScopeStats(ctx context.Context, iatas []string) ([]ap
})
}
// GetScopesByIATAs implements [api.Reader].
func (cr *CachedReader) GetScopesByIATAs(ctx context.Context, iatas []string) ([]api.ScopeSummary, error) {
sorted := make([]string, len(iatas))
copy(sorted, iatas)
sort.Strings(sorted)
key := keyScopesByIATAsPrefix + strings.Join(sorted, ",")
return getOrSet(ctx, cr.c, key, cr.ttl.Reference, func() ([]api.ScopeSummary, error) {
return cr.inner.GetScopesByIATAs(ctx, iatas)
})
}
// GetScopeByName implements [api.Reader].
func (cr *CachedReader) GetScopeByName(ctx context.Context, name string) (*api.ScopeDetail, error) {
return getOrSet(ctx, cr.c, keyScopeByNamePrefix+name, cr.ttl.Reference, func() (*api.ScopeDetail, error) {
@@ -420,8 +419,8 @@ func (cr *CachedReader) GetCrossIATANeighbors(ctx context.Context, nodeID uuid.U
}
// ListChannels implements [api.Reader].
func (cr *CachedReader) ListChannels(ctx context.Context, limit int32, hash []byte, iatas []string, cursor int64, pageCursor *api.ChannelCursor) (api.ChannelPage, error) {
return cr.inner.ListChannels(ctx, limit, hash, iatas, cursor, pageCursor)
func (cr *CachedReader) ListChannels(ctx context.Context, limit int32, hash []byte, iatas []string, keyKnown *bool, cursor int64, pageCursor *api.ChannelCursor) (api.ChannelPage, error) {
return cr.inner.ListChannels(ctx, limit, hash, iatas, keyKnown, cursor, pageCursor)
}
// ListChannelMessages implements [api.Reader].
internal/config/channels.go
+66
View File
@@ -0,0 +1,66 @@
// Copyright 2026 Beacon Contributors
// SPDX-License-Identifier: AGPL-3.0-or-later
package config
import (
"encoding/hex"
"fmt"
"strings"
"github.com/MeshCore-Beacon/beacon-server/internal/keystore"
"gopkg.in/yaml.v3"
)
// HashtagConfig is a hashtag channel; a plain string in YAML is a Beacon-wide one.
type HashtagConfig struct {
Name string `yaml:"name"`
Region string `yaml:"region"` // optional region slug
}
func (h *HashtagConfig) UnmarshalYAML(node *yaml.Node) error {
if node.Kind == yaml.ScalarNode {
return node.Decode(&h.Name)
}
type plain HashtagConfig
return node.Decode((*plain)(h))
}
// ChannelScope places a configured channel under a region, or Beacon-wide when Region is empty.
type ChannelScope struct {
Fingerprint []byte
Region string
}
func (c *Config) validateChannelKeys() error {
for i, h := range c.ChannelKeys.Hashtags {
if strings.TrimSpace(h.Name) == "" {
return fmt.Errorf("channel_keys.hashtags[%d] needs a name", i)
}
if h.Region != "" && c.region(h.Region) == nil {
return fmt.Errorf("channel #%s region %q is not a configured region", h.Name, h.Region)
}
}
for hash, k := range c.ChannelKeys.Keys {
if k.Region != "" && c.region(k.Region) == nil {
return fmt.Errorf("channel key %s region %q is not a configured region", hash, k.Region)
}
}
return nil
}
// ChannelScopes lists every configured channel's region placement. Keys that
// aren't valid hex are skipped; startup already warns about them.
func (c *Config) ChannelScopes() []ChannelScope {
var scopes []ChannelScope
for _, h := range c.ChannelKeys.Hashtags {
_, _, fingerprint := keystore.DeriveHashtagKey(h.Name)
scopes = append(scopes, ChannelScope{Fingerprint: fingerprint, Region: h.Region})
}
for _, k := range c.ChannelKeys.Keys {
if key, err := hex.DecodeString(k.Key); err == nil {
scopes = append(scopes, ChannelScope{Fingerprint: keystore.Fingerprint(key), Region: k.Region})
}
}
return scopes
}
internal/config/channels_test.go
+88
View File
@@ -0,0 +1,88 @@
// Copyright 2026 Beacon Contributors
// SPDX-License-Identifier: AGPL-3.0-or-later
package config
import (
"bytes"
"os"
"path/filepath"
"strings"
"testing"
"github.com/MeshCore-Beacon/beacon-server/internal/keystore"
)
func loadText(t *testing.T, text string) (*Config, error) {
t.Helper()
p := filepath.Join(t.TempDir(), "config.yaml")
if err := os.WriteFile(p, []byte(text), 0600); err != nil {
t.Fatal(err)
}
return Load(p)
}
func TestChannelKeysRegionScoping(t *testing.T) {
text := `regions:
- slug: east
iatas: [YOW]
channel_keys:
hashtags:
- meshcore
- name: ottawa-mesh
region: east
keys:
"11":
key: "8b3387e9c5cdea6ac9e5edbaa115cd72"
name: Public
`
cfg, err := loadText(t, text)
if err != nil {
t.Fatal(err)
}
if got := cfg.ChannelKeys.Hashtags; len(got) != 2 || got[0].Name != "meshcore" || got[0].Region != "" || got[1].Name != "ottawa-mesh" || got[1].Region != "east" {
t.Fatalf("hashtags parsed as %+v", got)
}
scopes := cfg.ChannelScopes()
if len(scopes) != 3 {
t.Fatalf("scopes: %+v", scopes)
}
_, _, ottawa := keystore.DeriveHashtagKey("ottawa-mesh")
found := false
for _, s := range scopes {
if bytes.Equal(s.Fingerprint, ottawa) {
found = s.Region == "east"
}
}
if !found {
t.Fatal("region-scoped hashtag missing its region", scopes)
}
for name, bad := range map[string]string{
"unknown hashtag region": strings.Replace(text, "region: east", "region: west", 1),
"empty hashtag": strings.Replace(text, "- meshcore", `- ""`, 1),
"unknown key region": text + " region: west\n",
} {
if _, err := loadText(t, bad); err == nil {
t.Errorf("%s accepted", name)
}
}
}
func TestMeshMapperChannelsConfig(t *testing.T) {
valid := "meshmapper:\n channels:\n enabled: true\n"
for text, bad := range map[string]bool{
valid: false,
valid + " refresh_interval: 24h\n": false,
valid + " refresh_interval: 168h\n": false,
valid + " refresh_interval: 23h\n": true,
valid + " refresh_interval: 169h\n": true,
} {
if _, err := loadText(t, text); (err != nil) != bad {
t.Errorf("%q: error=%v", text, err)
}
}
if (MeshMapperChannelsConfig{}).Interval() != MinChannelsRefresh {
t.Fatal("wrong default")
}
}
internal/config/config.go
+13 -5
View File
@@ -221,7 +221,8 @@ type CacheTTLsConfig struct {
// Name can be provided with or without the # or $ prefix.
// Beacon normalizes plain names by prepending #.
type ScopeConfig struct {
Name string `yaml:"name"` // e.g. "bc", "#west", "$private"
Name string `yaml:"name"` // e.g. "bc", "#west", "$private"
Region string `yaml:"region"` // configured region slug; lists the scope under that region's IATAs
}
// TelemetryConfig controls observer telemetry storage behaviour.
@@ -319,9 +320,9 @@ func (d *duration) UnmarshalYAML(value *yaml.Node) error {
// channel_hash = SHA256(secret)[0]. Explicit keys are provided as hex strings
// keyed by the channel hash hex (e.g. "11" for 0x11).
type ChannelKeysConfig struct {
// Hashtags is a list of hashtag names (without the # prefix).
// Hashtags lists hashtag names (without the # prefix), each optionally scoped to a region.
// Beacon derives the PSK and channel hash automatically.
Hashtags []string `yaml:"hashtags"`
Hashtags []HashtagConfig `yaml:"hashtags"`
// Keys maps channel hash hex → explicit key config.
Keys map[string]ExplicitKeyConfig `yaml:"keys"`
@@ -329,8 +330,9 @@ type ChannelKeysConfig struct {
// ExplicitKeyConfig holds an explicit channel key and optional display name.
type ExplicitKeyConfig struct {
Key string `yaml:"key"` // hex-encoded key bytes
Name string `yaml:"name"` // optional display name
Key string `yaml:"key"` // hex-encoded key bytes
Name string `yaml:"name"` // optional display name
Region string `yaml:"region"` // optional region slug; empty lists the channel Beacon-wide
}
// IATAConfig holds optional overrides for a known IATA code.
@@ -413,6 +415,12 @@ func Load(path string) (*Config, error) {
if err := cfg.validateMeshMapper(); err != nil {
return nil, err
}
if err := cfg.validateChannelKeys(); err != nil {
return nil, err
}
if err := cfg.validateScopes(); err != nil {
return nil, err
}
if cfg.Nodes.MarkForeign && !cfg.MeshMapper.Zones.Enabled && !cfg.hasBorderFile() {
return nil, fmt.Errorf("nodes.mark_foreign requires an iatas.*.borderFile or meshmapper.zones")
}
internal/config/meshmapper.go
+40 -47
View File
@@ -5,23 +5,41 @@ package config
import (
"fmt"
"net/url"
"regexp"
"slices"
"log/slog"
"time"
)
// MaxScopeSources bounds both network refresh work and imported matching keys.
const MaxScopeSources = 16
// MeshMapper rate-limits get_zones/get_geojson to one request per target per
// 23.5h and get_scopes to one per region per 55m; minimums round up.
const (
MinZonesRefresh = 24 * time.Hour
MinScopesRefresh = time.Hour
MinChannelsRefresh = 24 * time.Hour // get_channels.php: one per region per 23.5h
)
type MeshMapperConfig struct {
Scopes MeshMapperScopesConfig `yaml:"scopes"`
Zones MeshMapperZonesConfig `yaml:"zones"`
Scopes MeshMapperScopesConfig `yaml:"scopes"`
Zones MeshMapperZonesConfig `yaml:"zones"`
Channels MeshMapperChannelsConfig `yaml:"channels"`
}
// MeshMapperZonesConfig imports each region IATA's published boundary, overriding borderFile.
// MeshMapperChannelsConfig imports each known IATA's public hashtag channels.
type MeshMapperChannelsConfig struct {
Enabled bool `yaml:"enabled"`
RefreshInterval duration `yaml:"refresh_interval"`
}
func (c MeshMapperChannelsConfig) Interval() time.Duration {
if c.RefreshInterval.Duration == 0 {
return MinChannelsRefresh
}
return c.RefreshInterval.Duration
}
// MeshMapperZonesConfig imports each known IATA's published boundary, overriding borderFile.
type MeshMapperZonesConfig struct {
Enabled bool `yaml:"enabled"`
ImportGroups bool `yaml:"import_groups"` // zone groups become regions
RefreshInterval duration `yaml:"refresh_interval"`
}
@@ -32,20 +50,11 @@ func (c MeshMapperZonesConfig) Interval() time.Duration {
return c.RefreshInterval.Duration
}
// RegionIATAs returns the sorted, unique IATAs of every configured region.
func (c *Config) RegionIATAs() []string {
var iatas []string
for _, region := range c.Regions {
iatas = append(iatas, region.IATAs...)
}
slices.Sort(iatas)
return slices.Compact(iatas)
}
// MeshMapperScopesConfig augments, but never replaces, the manual scopes list.
// Sources are discovered per known IATA from the zone list.
type MeshMapperScopesConfig struct {
Enabled bool `yaml:"enabled"`
Sources map[string]string `yaml:"sources"` // configured IATA -> published regional endpoint
Sources map[string]string `yaml:"sources"` // deprecated: ignored, kept to warn
RefreshInterval duration `yaml:"refresh_interval"`
}
@@ -56,43 +65,27 @@ func (c MeshMapperScopesConfig) Interval() time.Duration {
return c.RefreshInterval.Duration
}
var meshMapperHost = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*\.meshmapper\.net$`)
func (c *Config) validateMeshMapper() error {
if z := c.MeshMapper.Zones; z.ImportGroups && !z.Enabled {
return fmt.Errorf("meshmapper.zones.import_groups requires meshmapper.zones.enabled")
}
if z := c.MeshMapper.Zones; z.Enabled {
// The Zones API asks clients not to poll more than once an hour.
if z.Interval() < time.Hour || z.Interval() > 7*24*time.Hour {
return fmt.Errorf("meshmapper.zones.refresh_interval must be between 1h and 168h")
}
if len(c.RegionIATAs()) == 0 {
return fmt.Errorf("meshmapper.zones needs at least one configured region IATA")
if z.Interval() < MinZonesRefresh || z.Interval() > 7*24*time.Hour {
return fmt.Errorf("meshmapper.zones.refresh_interval must be between 24h and 168h")
}
}
if ch := c.MeshMapper.Channels; ch.Enabled && (ch.Interval() < MinChannelsRefresh || ch.Interval() > 7*24*time.Hour) {
return fmt.Errorf("meshmapper.channels.refresh_interval must be between 24h and 168h")
}
s := c.MeshMapper.Scopes
if !s.Enabled {
return nil
}
if len(s.Sources) == 0 || len(s.Sources) > MaxScopeSources {
return fmt.Errorf("meshmapper.scopes.sources must contain 1-%d regional endpoints", MaxScopeSources)
if s.Interval() < MinScopesRefresh || s.Interval() > 24*time.Hour {
return fmt.Errorf("meshmapper.scopes.refresh_interval must be between 1h and 24h")
}
if s.Interval() < 5*time.Minute || s.Interval() > 24*time.Hour {
return fmt.Errorf("meshmapper.scopes.refresh_interval must be between 5m and 24h")
}
configured := map[string]bool{}
for _, region := range c.Regions {
for _, iata := range region.IATAs {
configured[iata] = true
}
}
for iata, endpoint := range s.Sources {
if !configured[iata] {
return fmt.Errorf("meshmapper.scopes source %q must belong to a configured region", iata)
}
u, err := url.Parse(endpoint)
if err != nil || u.Scheme != "https" || u.User != nil || !meshMapperHost.MatchString(u.Host) ||
u.Path != "/get_scopes.php" || u.RawPath != "" || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" {
return fmt.Errorf("meshmapper.scopes source %q must be a published https://<region>.meshmapper.net/get_scopes.php endpoint without credentials or parameters", iata)
}
if len(s.Sources) > 0 {
slog.Warn("meshmapper.scopes.sources is ignored; sources are discovered from the MeshMapper zone list", "component", "config")
}
return nil
}
internal/config/meshmapper_test.go
+16 -21
View File
@@ -12,19 +12,18 @@ import (
)
func TestMeshMapperConfig(t *testing.T) {
valid := "regions:\n - slug: ottawa\n iatas: [YOW]\nmeshmapper:\n scopes:\n enabled: true\n sources:\n YOW: https://yow.meshmapper.net/get_scopes.php\n"
valid := "meshmapper:\n scopes:\n enabled: true\n"
for _, tc := range []struct {
name, text string
bad bool
}{
{"valid", valid, false},
{"unknown region", strings.Replace(valid, "iatas: [YOW]", "iatas: [YVR]", 1), true},
{"http", strings.Replace(valid, "https:", "http:", 1), true},
{"private host", strings.Replace(valid, "yow.meshmapper.net", "127.0.0.1", 1), true},
{"credentials", strings.Replace(valid, "https://", "https://secret@", 1), true},
{"query", strings.Replace(valid, "get_scopes.php", "get_scopes.php?key=secret", 1), true},
{"non API", strings.Replace(valid, "get_scopes.php", "index.php", 1), true},
{"daily", valid + " refresh_interval: 24h\n", false},
{"legacy sources ignored", valid + " sources:\n yow: http://127.0.0.1/\n", false},
{"hourly", valid + " refresh_interval: 1h\n", false},
{"under the rate limit", valid + " refresh_interval: 55m\n", true},
{"fast", valid + " refresh_interval: 1s\n", true},
{"slow", valid + " refresh_interval: 25h\n", true},
{"negative", valid + " refresh_interval: -1h\n", true},
{"disabled", strings.Replace(valid, "enabled: true", "enabled: false", 1), false},
} {
@@ -33,18 +32,14 @@ func TestMeshMapperConfig(t *testing.T) {
if err := os.WriteFile(p, []byte(tc.text), 0600); err != nil {
t.Fatal(err)
}
cfg, err := Load(p)
_, err := Load(p)
if (err != nil) != tc.bad {
t.Fatalf("error=%v", err)
}
if err == nil && cfg.MeshMapper.Scopes.Interval() != time.Hour {
t.Fatal("wrong default")
}
})
}
cfg := Config{MeshMapper: MeshMapperConfig{Scopes: MeshMapperScopesConfig{Enabled: true}}}
if err := cfg.validateMeshMapper(); err == nil {
t.Fatal("empty sources accepted")
if (MeshMapperScopesConfig{}).Interval() != time.Hour {
t.Fatal("wrong default")
}
}
@@ -56,10 +51,14 @@ func TestMeshMapperZonesConfig(t *testing.T) {
interval time.Duration
}{
{"default", valid, false, 24 * time.Hour},
{"hourly", valid + " refresh_interval: 1h\n", false, time.Hour},
{"too often", valid + " refresh_interval: 59m\n", true, 0},
{"daily", valid + " refresh_interval: 24h\n", false, 24 * time.Hour},
{"weekly", valid + " refresh_interval: 168h\n", false, 168 * time.Hour},
{"under the rate limit", valid + " refresh_interval: 23h30m\n", true, 0},
{"hourly", valid + " refresh_interval: 1h\n", true, 0},
{"too rare", valid + " refresh_interval: 169h\n", true, 0},
{"no regions", "meshmapper:\n zones:\n enabled: true\n", true, 0},
{"no regions", "meshmapper:\n zones:\n enabled: true\n", false, 24 * time.Hour},
{"groups", valid + " import_groups: true\n", false, 24 * time.Hour},
{"groups without zones", "meshmapper:\n zones:\n import_groups: true\n", true, 0},
{"disabled", "meshmapper:\n zones:\n enabled: false\n", false, 24 * time.Hour},
} {
t.Run(tc.name, func(t *testing.T) {
@@ -76,8 +75,4 @@ func TestMeshMapperZonesConfig(t *testing.T) {
}
})
}
cfg := &Config{Regions: []RegionConfig{{IATAs: []string{"YYZ", "YOW"}}, {IATAs: []string{"YOW"}}}}
if got := strings.Join(cfg.RegionIATAs(), ","); got != "YOW,YYZ" {
t.Fatal(got)
}
}
internal/config/scopes.go
+52
View File
@@ -0,0 +1,52 @@
// Copyright 2026 Beacon Contributors
// SPDX-License-Identifier: AGPL-3.0-or-later
package config
import (
"fmt"
"strings"
"github.com/MeshCore-Beacon/beacon-server/internal/scopestore"
)
func (c *Config) validateScopes() error {
for i, s := range c.Scopes {
if strings.TrimSpace(s.Name) == "" {
return fmt.Errorf("scopes[%d].name is required", i)
}
if s.Region == "" {
return fmt.Errorf("scope %q needs a region", s.Name)
}
if c.region(s.Region) == nil {
return fmt.Errorf("scope %q region %q is not a configured region", s.Name, s.Region)
}
}
return nil
}
func (c *Config) region(slug string) *RegionConfig {
for i := range c.Regions {
if c.Regions[i].Slug == slug {
return &c.Regions[i]
}
}
return nil
}
// ManualScopeMembers maps each region IATA to the normalized names of its configured scopes.
func (c *Config) ManualScopeMembers() map[string][]string {
members := map[string][]string{}
for _, s := range c.Scopes {
region := c.region(s.Region)
if region == nil {
continue
}
name := scopestore.FromName(s.Name).Name
for _, iata := range region.IATAs {
iata = strings.ToUpper(iata)
members[iata] = append(members[iata], name)
}
}
return members
}
internal/config/scopes_test.go
+43
View File
@@ -0,0 +1,43 @@
// Copyright 2026 Beacon Contributors
// SPDX-License-Identifier: AGPL-3.0-or-later
package config
import (
"os"
"path/filepath"
"reflect"
"strings"
"testing"
)
func TestScopesConfig(t *testing.T) {
valid := "regions:\n - slug: ottawa\n iatas: [YOW, YGK]\n - slug: bc\n iatas: [YVR]\nscopes:\n - name: on\n region: ottawa\n - name: \"#west\"\n region: bc\n - name: \"$ottawa\"\n region: ottawa\n"
for _, tc := range []struct {
name, text string
bad bool
}{
{"valid", valid, false},
{"missing region", strings.Replace(valid, " region: bc\n", "", 1), true},
{"unknown region", strings.Replace(valid, "region: bc", "region: prairies", 1), true},
{"empty name", strings.Replace(valid, "name: on", "name: \"\"", 1), true},
} {
t.Run(tc.name, func(t *testing.T) {
p := filepath.Join(t.TempDir(), "config.yaml")
if err := os.WriteFile(p, []byte(tc.text), 0600); err != nil {
t.Fatal(err)
}
cfg, err := Load(p)
if (err != nil) != tc.bad {
t.Fatalf("error=%v", err)
}
if tc.bad {
return
}
want := map[string][]string{"YOW": {"#on", "$ottawa"}, "YGK": {"#on", "$ottawa"}, "YVR": {"#west"}}
if got := cfg.ManualScopeMembers(); !reflect.DeepEqual(got, want) {
t.Fatalf("ManualScopeMembers = %v, want %v", got, want)
}
})
}
}
internal/config/seed.go
+15 -4
View File
@@ -19,8 +19,9 @@ type Seeder interface {
UpsertIATADetails(ctx context.Context, iata string, name string, lat, lng *float64) error
UpsertIATABorder(ctx context.Context, iata string, border json.RawMessage) error
UpsertRegion(ctx context.Context, slug, name, description string, displayOrder int, centerLat, centerLng *float64, zoomLevel *int) (int32, error)
UpsertRegionIATA(ctx context.Context, regionID int32, iata string) error
SetRegionIATAs(ctx context.Context, regionID int32, iatas []string) error
UpsertTransportScope(ctx context.Context, name, displayName string, transportKey, keyFingerprint []byte) error
SetChannelConfigScopes(ctx context.Context, fingerprints [][]byte, regions []string) error
}
// Seed applies config-defined regions, IATA overrides to the database.
@@ -56,10 +57,20 @@ func Seed(ctx context.Context, cfg *Config, db Seeder) error {
if err := db.UpsertIATA(ctx, iata); err != nil {
return err
}
if err := db.UpsertRegionIATA(ctx, id, iata); err != nil {
return err
}
}
// Config owns the member list, including when it takes over an imported slug.
if err := db.SetRegionIATAs(ctx, id, r.IATAs); err != nil {
return err
}
}
// Channel region placement
var fingerprints [][]byte
var regions []string
for _, scope := range cfg.ChannelScopes() {
fingerprints, regions = append(fingerprints, scope.Fingerprint), append(regions, scope.Region)
}
if err := db.SetChannelConfigScopes(ctx, fingerprints, regions); err != nil {
return err
}
// Transport Codes
for _, s := range cfg.Scopes {
internal/config/seed_test.go
+28 -8
View File
@@ -15,12 +15,13 @@ import (
)
type stubSeeder struct {
regions map[string]int32
iatas []string
regionIATAs map[int32][]string
scopes []string
borders map[string]json.RawMessage
upsertErr error
regions map[string]int32
iatas []string
regionIATAs map[int32][]string
channelRegions []string
scopes []string
borders map[string]json.RawMessage
upsertErr error
}
func newStubSeeder() *stubSeeder {
@@ -52,8 +53,13 @@ func (s *stubSeeder) UpsertRegion(_ context.Context, slug, _, _ string, _ int, _
return id, s.upsertErr
}
func (s *stubSeeder) UpsertRegionIATA(_ context.Context, regionID int32, iata string) error {
s.regionIATAs[regionID] = append(s.regionIATAs[regionID], iata)
func (s *stubSeeder) SetRegionIATAs(_ context.Context, regionID int32, iatas []string) error {
s.regionIATAs[regionID] = iatas
return s.upsertErr
}
func (s *stubSeeder) SetChannelConfigScopes(_ context.Context, fingerprints [][]byte, regions []string) error {
s.channelRegions = regions
return s.upsertErr
}
@@ -256,3 +262,17 @@ func TestDeriveScopeKey_DifferentInputs(t *testing.T) {
t.Error("expected different keys for different inputs")
}
}
func TestSeed_ChannelScopes(t *testing.T) {
cfg := &Config{
Regions: []RegionConfig{{Slug: "bc", IATAs: []string{"YVR"}}},
ChannelKeys: ChannelKeysConfig{Hashtags: []HashtagConfig{{Name: "meshcore"}, {Name: "vancouver", Region: "bc"}}},
}
db := newStubSeeder()
if err := Seed(context.Background(), cfg, db); err != nil {
t.Fatal(err)
}
if len(db.channelRegions) != 2 || db.channelRegions[0] != "" || db.channelRegions[1] != "bc" {
t.Fatalf("channel regions = %q", db.channelRegions)
}
}
internal/ingest/backfill.go
+14 -1
View File
@@ -107,6 +107,19 @@ func BackfillChannelMessages(ctx context.Context, db DB, keys ChannelKeyStore) (
if err != nil {
return 0, err
}
return backfill(ctx, db, keys, packets), nil
}
// BackfillChannelHashes retries only the channels whose keys were just imported at runtime.
func BackfillChannelHashes(ctx context.Context, db DB, keys ChannelKeyStore, hashes [][]byte) (int, error) {
packets, err := db.ListUndecryptedGroupTextPacketsByHash(ctx, hashes)
if err != nil {
return 0, err
}
return backfill(ctx, db, keys, packets), nil
}
func backfill(ctx context.Context, db DB, keys ChannelKeyStore, packets []UndecryptedPacket) int {
decrypted := 0
for _, p := range packets {
result, err := DecryptGroupText(ctx, db, keys, p.PacketHash, p.RawPayload)
@@ -118,5 +131,5 @@ func BackfillChannelMessages(ctx context.Context, db DB, keys ChannelKeyStore) (
decrypted++
}
}
return decrypted, nil
return decrypted
}
internal/ingest/backfill_test.go
+17
View File
@@ -135,6 +135,23 @@ func TestBackfillChannelMessages_DecryptsAndCounts(t *testing.T) {
}
}
func TestBackfillChannelHashes_OnlyScansNewHashes(t *testing.T) {
psk := make([]byte, 16)
db := &stubDB{
insertChannelMessageResult: true,
undecryptedByHash: []UndecryptedPacket{{PacketHash: []byte{0x01}, RawPayload: encryptedGroupText(t, 0x44, psk, "ded", "hi")}},
undecryptedPackets: []UndecryptedPacket{{PacketHash: []byte{0x02}, RawPayload: encryptedGroupText(t, 0x44, psk, "x", "y")}},
}
keys := &mapKeys{entries: map[byte][]keystore.Entry{0x44: {{Key: psk, Fingerprint: []byte{0xAA}}}}}
n, err := BackfillChannelHashes(context.Background(), db, keys, [][]byte{{0x44}})
if err != nil || n != 1 {
t.Fatalf("n=%d err=%v", n, err)
}
if len(db.backfillHashes) != 1 || db.backfillHashes[0][0] != 0x44 {
t.Fatalf("scanned hashes %x", db.backfillHashes)
}
}
func TestBackfillChannelMessages_NoUndecryptedPackets(t *testing.T) {
db := &stubDB{}
keys := &mapKeys{}
internal/ingest/ingest.go
+3
View File
@@ -174,6 +174,9 @@ type DB interface {
// used by BackfillChannelMessages to retry them against the current keystore at boot.
ListUndecryptedGroupTextPackets(ctx context.Context) ([]UndecryptedPacket, error)
// ListUndecryptedGroupTextPacketsByHash limits that scan to channels that just gained a key.
ListUndecryptedGroupTextPacketsByHash(ctx context.Context, hashes [][]byte) ([]UndecryptedPacket, error)
// UpsertChannelIATA upserts a channel_iatas row.
UpsertChannelIATA(ctx context.Context, channelHash []byte, iata string, heardAt time.Time) error
internal/ingest/ingest_test.go
+7
View File
@@ -184,6 +184,8 @@ type stubDB struct {
observationInserted bool
insertChannelMessageResult bool // configurable return for InsertChannelMessage; default false
undecryptedPackets []UndecryptedPacket
undecryptedByHash []UndecryptedPacket
backfillHashes [][]byte
dbHook func() // runs in UpsertChannelIATA, the first DB call after InsertObservation
}
@@ -279,6 +281,11 @@ func (s *stubDB) ListUndecryptedGroupTextPackets(_ context.Context) ([]Undecrypt
return s.undecryptedPackets, nil
}
func (s *stubDB) ListUndecryptedGroupTextPacketsByHash(_ context.Context, hashes [][]byte) ([]UndecryptedPacket, error) {
s.backfillHashes = hashes
return s.undecryptedByHash, nil
}
func (s *stubDB) UpsertChannelIATA(_ context.Context, _ []byte, _ string, _ time.Time) error {
if s.dbHook != nil {
s.dbHook()
internal/keystore/keystore.go
+37 -8
View File
@@ -2,14 +2,15 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
// Package keystore provides channel key lookup for the ingest pipeline.
// Keys are loaded from config at startup and never written at runtime.
// Future: add a DB-backed fallback that checks the channel_keys table.
// Config keys are fixed at startup; MeshMapper imports are swapped in at runtime.
package keystore
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"slices"
"sync/atomic"
)
// Entry holds a resolved channel key along with its metadata.
@@ -23,23 +24,51 @@ type Entry struct {
// MapKeyStore maps channel hash hex → list of known key entries.
// Multiple entries per hash are supported to handle 1-byte hash collisions.
type MapKeyStore struct {
entries map[string][]Entry // channel hash hex → entries
config map[string][]Entry
imported map[string][]Entry
entries atomic.Pointer[map[string][]Entry] // config + imported, read lock-free by ingest
}
// NewMapKeyStore creates a keystore from a pre-built map of entries.
// Use BuildFromConfig to construct entries from a config.ChannelKeysConfig.
// NewMapKeyStore creates a keystore from the configured entries.
func NewMapKeyStore(entries map[string][]Entry) *MapKeyStore {
store := &MapKeyStore{entries: make(map[string][]Entry)}
store := &MapKeyStore{config: make(map[string][]Entry), imported: map[string][]Entry{}}
for hash, list := range entries {
store.entries[hash] = append(store.entries[hash], list...)
store.config[hash] = append(store.config[hash], list...)
}
store.entries.Store(&store.config)
return store
}
// GetKey returns all known key entries for the given channel hash byte.
// Returns nil if no keys are known for this hash.
func (s *MapKeyStore) GetKey(channelHash []byte) []Entry {
return s.entries[hex.EncodeToString(channelHash)]
return (*s.entries.Load())[hex.EncodeToString(channelHash)]
}
// SetImported replaces the imported keys and returns the channel hashes of keys
// that weren't known before. Config keys win; one writer at a time.
func (s *MapKeyStore) SetImported(entries []Entry) [][]byte {
merged := make(map[string][]Entry, len(s.config))
for hash, list := range s.config {
merged[hash] = list
}
imported := map[string][]Entry{}
var added [][]byte
for _, e := range entries {
hash := sha256.Sum256(e.Key)
hashHex := hex.EncodeToString(hash[:1])
if EntryExists(merged[hashHex], e) {
continue
}
merged[hashHex] = append(slices.Clone(merged[hashHex]), e)
imported[hashHex] = append(imported[hashHex], e)
if !EntryExists(s.imported[hashHex], e) {
added = append(added, hash[:1])
}
}
s.imported = imported
s.entries.Store(&merged)
return added
}
// DeriveHashtagKey derives the channel secret and hash for a hashtag name.
internal/keystore/keystore_test.go
+30
View File
@@ -153,3 +153,33 @@ func TestEntryExists_Empty(t *testing.T) {
t.Error("expected false for empty slice")
}
}
func TestSetImported_ConfigWinsAndNewHashesReported(t *testing.T) {
secret, hash, fp := DeriveHashtagKey("meshcore")
hashHex := hex.EncodeToString([]byte{hash})
configured := Entry{Key: secret, Fingerprint: fp, Hashtag: "meshcore", Name: "Configured"}
store := NewMapKeyStore(map[string][]Entry{hashHex: {configured}})
dupSecret, _, dupFP := DeriveHashtagKey("meshcore")
newSecret, newHash, newFP := DeriveHashtagKey("ottawa-mesh")
added := store.SetImported([]Entry{
{Key: dupSecret, Fingerprint: dupFP, Hashtag: "meshcore", Name: "#meshcore"},
{Key: newSecret, Fingerprint: newFP, Hashtag: "ottawa-mesh", Name: "#ottawa-mesh"},
})
if len(added) != 1 || added[0][0] != newHash {
t.Fatalf("new hashes = %x", added)
}
if got := store.GetKey([]byte{hash}); len(got) != 1 || got[0].Name != "Configured" {
t.Fatalf("config entry replaced: %+v", got)
}
if got := store.GetKey([]byte{newHash}); len(got) != 1 || got[0].Hashtag != "ottawa-mesh" {
t.Fatalf("imported entry missing: %+v", got)
}
if added = store.SetImported([]Entry{{Key: newSecret, Fingerprint: newFP, Hashtag: "ottawa-mesh"}}); len(added) != 0 {
t.Fatal("unchanged import reported new hashes", added)
}
store.SetImported(nil)
if store.GetKey([]byte{newHash}) != nil || len(store.GetKey([]byte{hash})) != 1 {
t.Fatal("clearing imports touched config keys or kept the import")
}
}
internal/meshmapper/catalogues.go
+7 -7
View File
@@ -19,7 +19,7 @@ type CatalogueScope struct {
Wardriving bool `json:"wardriving"`
}
type Catalogue struct {
type PublicScopeCatalogue struct {
IATA string `json:"iata"`
URL string `json:"url"`
GeneratedAt int64 `json:"generatedAt"`
@@ -31,21 +31,21 @@ type Catalogue struct {
Scopes []CatalogueScope `json:"scopes"`
}
// Catalogues returns an immutable snapshot of configured sources. No request
// Catalogues returns an immutable snapshot of discovered sources. No request
// triggers an upstream fetch or reads ingestion tables. Callers must not mutate it.
func (i *Importer) Catalogues() []Catalogue {
func (i *Importer) Catalogues() []PublicScopeCatalogue {
if i != nil {
if snapshot := i.catalogues.Load(); snapshot != nil {
return snapshot.([]Catalogue)
return snapshot.([]PublicScopeCatalogue)
}
}
return []Catalogue{}
return []PublicScopeCatalogue{}
}
func (i *Importer) publishCatalogues() {
result := make([]Catalogue, 0, len(i.sources))
result := make([]PublicScopeCatalogue, 0, len(i.sources))
for _, s := range i.sources {
c := Catalogue{IATA: s.iata, URL: s.url, LastError: s.cache.LastError, Scopes: []CatalogueScope{}}
c := PublicScopeCatalogue{IATA: s.iata, URL: s.url, LastError: s.cache.LastError, Scopes: []CatalogueScope{}}
if !s.cache.CheckedAt.IsZero() {
c.CheckedAt = s.cache.CheckedAt.UnixMilli()
c.FreshUntil = s.cache.CheckedAt.Add(i.interval + 5*time.Minute).UnixMilli()
internal/meshmapper/catalogues_test.go
+29 -6
View File
@@ -27,19 +27,21 @@ func TestPublicCataloguesFreshnessAndIsolation(t *testing.T) {
}
}))
defer server.Close()
cfg := config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": server.URL}}
store := &memoryStore{rows: map[string]Cache{}}
imp, err := New(context.Background(), cfg, store, scopestore.New(), nil)
cfg := config.MeshMapperScopesConfig{Enabled: true}
store := newMemoryStore("YOW")
imp, err := New(context.Background(), cfg, store, NewDirectory(newZoneListMemory()), scopestore.New(), nil)
if err != nil {
t.Fatal(err)
}
if got := imp.Catalogues(); len(got) != 1 || got[0].CheckedAt != 0 || len(got[0].Scopes) != 0 {
t.Fatal(got)
}
imp.sources[0].url = server.URL
now := time.Now()
if err := imp.refresh(context.Background(), &imp.sources[0], now); err != nil {
t.Fatal(err)
}
imp.publishCatalogues()
old := imp.Catalogues()
if old[0].Scopes[0].Name != "#yow" || old[0].Scopes[1].Name != "#YOW" || len(old[0].Scopes) != 3 || old[0].Repeaters != 5 || old[0].Scoped != 3 || old[0].FreshUntil <= old[0].CheckedAt {
t.Fatal(old)
@@ -48,6 +50,7 @@ func TestPublicCataloguesFreshnessAndIsolation(t *testing.T) {
if err := imp.refresh(context.Background(), &imp.sources[0], now.Add(time.Minute)); err != nil {
t.Fatal(err)
}
imp.publishCatalogues()
current := imp.Catalogues()[0]
if current.LastError != "HTTP 503" || current.CheckedAt != old[0].CheckedAt || current.GeneratedAt != old[0].GeneratedAt || len(current.Scopes) != 3 || old[0].LastError != "" {
t.Fatal(current, old)
@@ -68,13 +71,33 @@ func TestPublicCataloguesFreshnessAndIsolation(t *testing.T) {
if calls != 2 {
t.Fatal("reading public metadata performed upstream requests", calls)
}
disabled, err := New(context.Background(), config.MeshMapperScopesConfig{}, store, scopestore.New(), nil)
disabled, err := New(context.Background(), config.MeshMapperScopesConfig{}, store, NewDirectory(newZoneListMemory()), scopestore.New(), nil)
if err != nil || len(disabled.Catalogues()) != 0 {
t.Fatal("disabled importer exposed old membership")
}
store.rows["YOW"+server.URL] = Cache{Payload: []byte(`{"scopes":"invalid"}`)}
restored, err := New(context.Background(), cfg, store, scopestore.New(), nil)
store.rows[sourceKey{"YOW", server.URL}] = Cache{Payload: []byte(`{"scopes":"invalid"}`)}
restored, err := New(context.Background(), cfg, store, NewDirectory(newZoneListMemory()), scopestore.New(), nil)
if err != nil || len(restored.Catalogues()[0].Scopes) != 0 || restored.Catalogues()[0].LastError == "" {
t.Fatal("invalid saved metadata leaked")
}
}
func TestPublicCataloguesIncludeDiscoveredAndUnlistedRegions(t *testing.T) {
ctx := context.Background()
store := newMemoryStore("YOW")
dir := NewDirectory(newZoneListMemory())
dir.lists["CA"] = &zoneList{zones: map[string]zoneEntry{}, fetchedAt: time.Now()}
imp, err := New(ctx, config.MeshMapperScopesConfig{Enabled: true}, store, dir, scopestore.New(), nil)
if err != nil {
t.Fatal(err)
}
before := imp.Catalogues()
store.iatas = append(store.iatas, "YKF")
if err := imp.Refresh(ctx); err != nil {
t.Fatal(err)
}
after := imp.Catalogues()
if len(before) != 1 || before[0].LastError != "" || len(after) != 2 || after[1].IATA != "YKF" || after[1].LastError != "not listed" {
t.Fatal(before, after)
}
}
internal/meshmapper/channels.go
+307
View File
@@ -0,0 +1,307 @@
// Copyright 2026 Beacon Contributors
// SPDX-License-Identifier: AGPL-3.0-or-later
package meshmapper
import (
"bytes"
"context"
"encoding/hex"
"encoding/json"
"fmt"
"log/slog"
"net/http"
"strings"
"time"
"unicode"
"unicode/utf8"
"github.com/MeshCore-Beacon/beacon-server/internal/config"
"github.com/MeshCore-Beacon/beacon-server/internal/iatadb"
"github.com/MeshCore-Beacon/beacon-server/internal/keystore"
)
const (
MaxChannels = 256 // per region
MaxChannelBody = 64 << 10
channelRetry = config.MinChannelsRefresh // get_channels.php allows one request per region per 23.5h, failed or not
)
type ChannelStore interface {
ListKnownIATAs(ctx context.Context) ([]string, error)
ListChannelCatalogues(ctx context.Context) ([]Catalogue, error)
// SaveChannelCatalogue replaces the IATA's members when the cache carries a new payload.
SaveChannelCatalogue(ctx context.Context, iata, url string, cache Cache, fingerprints [][]byte) error
ClearChannelMembers(ctx context.Context) error
}
// ChannelKeys receives every imported key and reports the channel hashes it hadn't seen.
type ChannelKeys interface {
SetImported(entries []keystore.Entry) [][]byte
}
type channelSource struct {
iata, country, url string
cache Cache
entries []keystore.Entry
}
// Channels is owned by one background task.
type Channels struct {
store ChannelStore
keys ChannelKeys
dir *Directory
channelsURL func(site string) (string, bool)
client *http.Client
interval time.Duration
seen map[string]bool
sources []channelSource
onNewKeys func(hashes [][]byte)
}
// NewChannels restores saved channel lists into keys without HTTP requests.
// Disabled, it clears MeshMapper's region membership so config scoping alone applies.
func NewChannels(ctx context.Context, cfg config.MeshMapperChannelsConfig, store ChannelStore, dir *Directory, keys ChannelKeys) (*Channels, error) {
c := &Channels{store: store, keys: keys, dir: dir, channelsURL: channelsEndpoint, interval: cfg.Interval(), seen: map[string]bool{},
client: &http.Client{
Timeout: requestTimeout,
CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
}}
if !cfg.Enabled {
if err := store.ClearChannelMembers(ctx); err != nil {
return nil, fmt.Errorf("clear MeshMapper channel members: %w", err)
}
return c, nil
}
saved, err := store.ListChannelCatalogues(ctx)
if err != nil {
return nil, fmt.Errorf("restore channel catalogues: %w", err)
}
latest := map[string]Catalogue{}
for _, cat := range saved {
if old, ok := latest[cat.IATA]; !ok || cat.AttemptedAt.After(old.AttemptedAt) {
latest[cat.IATA] = cat
}
}
iatas, err := store.ListKnownIATAs(ctx)
if err != nil {
return nil, fmt.Errorf("list IATAs for MeshMapper channels: %w", err)
}
c.track(iatas, latest)
c.publish()
return c, nil
}
// OnNewKeys is told the channel hashes an import added, for backfill.
func (c *Channels) OnNewKeys(fn func(hashes [][]byte)) { c.onNewKeys = fn }
func (c *Channels) track(iatas []string, saved map[string]Catalogue) {
for _, iata := range iatas {
if c.seen[iata] {
continue
}
c.seen[iata] = true
country := iatadb.CountryFor(iata)
if country == "" {
continue
}
s := channelSource{iata: iata, country: country}
if cat, ok := saved[iata]; ok {
s.url, s.cache = cat.URL, cat.Cache
if len(cat.Payload) > 0 {
var err error
if s.entries, err = decodeChannels(cat.Payload, iata); err != nil {
s.entries, s.cache.Payload, s.cache.ETag = nil, nil, ""
s.cache.LastError = "invalid saved catalogue; awaiting refresh"
}
}
c.log(s, "restored")
}
c.sources = append(c.sources, s)
}
}
// Refresh makes at most one request. The scheduler serializes calls every 15s.
func (c *Channels) Refresh(ctx context.Context) (err error) {
parent := ctx
defer func() {
if parent.Err() != nil {
err = nil
}
}()
ctx, cancel := context.WithTimeout(ctx, refreshTimeout)
defer cancel()
now := time.Now().UTC()
iatas, err := c.store.ListKnownIATAs(ctx)
if err != nil {
return fmt.Errorf("list IATAs for MeshMapper channels: %w", err)
}
c.track(iatas, nil)
for n := range c.sources {
s := &c.sources[n]
if now.Before(s.cache.NextAttempt) {
continue
}
zones, fetched, err := c.dir.List(ctx, s.country, now)
if err != nil || fetched {
return err
}
if zones == nil {
continue
}
entry, listed := zones[s.iata]
endpoint, valid := "", false
if listed {
endpoint, valid = c.channelsURL(entry.url)
}
if !valid {
// No request to make; channels already imported stay until a list replaces them.
s.cache.NextAttempt, s.cache.LastError = now.Add(c.interval), "not listed"
if listed {
s.cache.LastError = "invalid site URL"
}
c.log(*s, "skipped")
continue
}
if endpoint != s.url {
s.url, s.cache.ETag = endpoint, ""
}
return c.refresh(ctx, s, now)
}
return nil
}
func (c *Channels) refresh(ctx context.Context, s *channelSource, now time.Time) error {
update := Cache{AttemptedAt: now, NextAttempt: now.Add(c.interval)}
etag := ""
if len(s.cache.Payload) > 0 {
etag = s.cache.ETag
}
status, body, header, err := get(ctx, c.client, "Beacon-MeshMapper-Channels/1", s.url, etag, MaxChannelBody)
if err != nil {
return err
}
var entries []keystore.Entry
var retryAt time.Time
switch status {
case http.StatusOK:
if entries, err = decodeChannels(body, s.iata); err != nil {
update.LastError = "invalid response"
} else {
update.Payload = body
}
case http.StatusNotModified:
if len(s.cache.Payload) == 0 {
update.LastError = "304 without cached catalogue"
}
default:
update.LastError = statusProblem(status, header, now, &retryAt)
}
if update.LastError == "" {
etag := header.Get("ETag")
if len(etag) > 256 || strings.ContainsAny(etag, "\r\n") {
update.LastError, update.Payload, entries = "invalid ETag", nil, nil
} else {
if status == http.StatusNotModified && etag == "" {
etag = s.cache.ETag
}
update.ETag, update.CheckedAt = etag, now
}
}
if update.LastError != "" {
update.NextAttempt = now.Add(min(c.interval, channelRetry))
if retryAt.After(update.NextAttempt) {
update.NextAttempt = retryAt
}
}
var fingerprints [][]byte
for _, e := range entries {
fingerprints = append(fingerprints, e.Fingerprint)
}
if update.Payload != nil && fingerprints == nil {
fingerprints = [][]byte{}
}
if err := c.store.SaveChannelCatalogue(ctx, s.iata, s.url, update, fingerprints); err != nil {
s.cache.NextAttempt = update.NextAttempt // never retry every tick during a DB outage
return fmt.Errorf("persist channel catalogue %s: %w", s.iata, err)
}
if update.Payload != nil {
s.cache.Payload, s.entries = update.Payload, entries
}
if !update.CheckedAt.IsZero() {
s.cache.CheckedAt, s.cache.ETag = update.CheckedAt, update.ETag
}
s.cache.AttemptedAt, s.cache.NextAttempt, s.cache.LastError = now, update.NextAttempt, update.LastError
if update.Payload != nil {
c.publish()
}
c.log(*s, "checked")
return nil
}
func (c *Channels) publish() {
var all []keystore.Entry
for _, s := range c.sources {
all = append(all, s.entries...)
}
if added := c.keys.SetImported(all); len(added) > 0 && c.onNewKeys != nil {
c.onNewKeys(added)
}
}
func (c *Channels) log(s channelSource, action string) {
level := slog.LevelInfo
if s.cache.LastError != "" && s.cache.LastError != "not listed" {
level = slog.LevelWarn
}
slog.Log(context.Background(), level, "MeshMapper channels "+action, "component", "meshmapper.channels", "iata", s.iata, "source", s.url,
"channels", len(s.entries), "checked_at", s.cache.CheckedAt, "next_attempt", s.cache.NextAttempt, "last_error", s.cache.LastError)
}
func channelsEndpoint(site string) (string, bool) { return siteEndpoint(site, "get_channels.php") }
// decodeChannels accepts a single region's list only when every key and hash is
// exactly what Beacon derives from the name, so a bad entry can't plant a wrong key.
func decodeChannels(body []byte, iata string) ([]keystore.Entry, error) {
var document struct {
GeneratedAt time.Time `json:"generated_at"`
Region string `json:"region"`
Zones []string `json:"zones"`
Count *int `json:"count"`
Channels *[]struct {
Name string `json:"name"`
Key string `json:"key"`
Hash string `json:"hash"`
Zones []string `json:"zones"`
} `json:"channels"`
}
if len(body) > MaxChannelBody || !utf8.Valid(body) {
return nil, fmt.Errorf("invalid channel list size/encoding")
}
if err := json.Unmarshal(body, &document); err != nil {
return nil, err
}
if document.GeneratedAt.IsZero() || document.Region != iata || len(document.Zones) != 1 || document.Zones[0] != iata ||
document.Channels == nil || document.Count == nil || *document.Count != len(*document.Channels) || len(*document.Channels) > MaxChannels {
return nil, fmt.Errorf("invalid regional channel list")
}
entries := make([]keystore.Entry, 0, len(*document.Channels))
seen := map[string]bool{}
for _, ch := range *document.Channels {
tag, ok := strings.CutPrefix(ch.Name, "#")
if !ok || tag == "" || len(ch.Name) > 64 || strings.ToLower(ch.Name) != ch.Name || seen[ch.Name] ||
strings.ContainsFunc(tag, func(r rune) bool { return unicode.IsSpace(r) || unicode.IsControl(r) || r == '#' }) ||
len(ch.Zones) != 1 || ch.Zones[0] != iata {
return nil, fmt.Errorf("invalid channel entry")
}
secret, hash, fingerprint := keystore.DeriveHashtagKey(tag)
key, err := hex.DecodeString(ch.Key)
if err != nil || !bytes.Equal(key, secret) || !strings.EqualFold(ch.Hash, hex.EncodeToString([]byte{hash})) {
return nil, fmt.Errorf("channel %s key does not match its name", ch.Name)
}
seen[ch.Name] = true
entries = append(entries, keystore.Entry{Key: secret, Fingerprint: fingerprint, Hashtag: tag, Name: ch.Name})
}
return entries, nil
}
internal/meshmapper/channels_test.go
+234
View File
@@ -0,0 +1,234 @@
// Copyright 2026 Beacon Contributors
// SPDX-License-Identifier: AGPL-3.0-or-later
package meshmapper
import (
"bytes"
"context"
"encoding/hex"
"encoding/json"
"net/http"
"net/http/httptest"
"slices"
"strings"
"testing"
"time"
"github.com/MeshCore-Beacon/beacon-server/internal/config"
"github.com/MeshCore-Beacon/beacon-server/internal/keystore"
)
func channelList(iata string, names ...string) []byte {
rows := []map[string]any{}
for _, name := range names {
secret, hash, _ := keystore.DeriveHashtagKey(strings.TrimPrefix(name, "#"))
rows = append(rows, map[string]any{"name": name, "key": hex.EncodeToString(secret), "hash": hex.EncodeToString([]byte{hash}), "zones": []string{iata}})
}
b, _ := json.Marshal(map[string]any{"generated_at": "2026-09-30T18:00:00Z", "region": iata, "zones": []string{iata}, "count": len(rows), "channels": rows})
return b
}
type channelMemoryStore struct {
iatas []string
rows map[sourceKey]Cache
members map[string][][]byte
fail bool
}
func newChannelMemoryStore(iatas ...string) *channelMemoryStore {
return &channelMemoryStore{iatas: iatas, rows: map[sourceKey]Cache{}, members: map[string][][]byte{}}
}
func (s *channelMemoryStore) ListKnownIATAs(context.Context) ([]string, error) {
return slices.Clone(s.iatas), nil
}
func (s *channelMemoryStore) ListChannelCatalogues(context.Context) ([]Catalogue, error) {
var out []Catalogue
for key, c := range s.rows {
out = append(out, Catalogue{IATA: key.iata, URL: key.url, Cache: c})
}
return out, nil
}
func (s *channelMemoryStore) SaveChannelCatalogue(_ context.Context, iata, url string, next Cache, fingerprints [][]byte) error {
if s.fail {
return context.DeadlineExceeded
}
if next.Payload != nil {
s.members[iata] = fingerprints
}
old := s.rows[sourceKey{iata, url}]
if next.Payload == nil {
next.Payload = old.Payload
}
if next.CheckedAt.IsZero() {
next.CheckedAt, next.ETag = old.CheckedAt, old.ETag
}
s.rows[sourceKey{iata, url}] = next
return nil
}
func (s *channelMemoryStore) ClearChannelMembers(context.Context) error {
s.members = map[string][][]byte{}
return nil
}
type channelSite struct {
*httptest.Server
list, body string
status int
retryAfter string
listCalls, channels int
}
func newChannelSite(t *testing.T) *channelSite {
f := &channelSite{status: 200, body: string(channelList("YOW", "#ottawa-mesh"))}
f.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/get_zones.php":
f.listCalls++
_, _ = w.Write([]byte(f.list))
case "/get_channels.php":
f.channels++
w.Header().Set("ETag", `"c1"`)
w.Header().Set("Retry-After", f.retryAfter)
w.WriteHeader(f.status)
if f.status == 200 {
_, _ = w.Write([]byte(f.body))
}
default:
t.Errorf("unexpected path %s", r.URL.Path)
}
}))
f.list = `{"country":"CA","zones":[{"code":"YOW","url":"` + f.URL + `/","has_boundary":true,"group":null}]}`
t.Cleanup(f.Close)
return f
}
type channelHarness struct {
c *Channels
keys *keystore.MapKeyStore
added [][]byte
}
func newChannelHarness(t *testing.T, f *channelSite, store *channelMemoryStore, lists *zoneListMemory, enabled bool) *channelHarness {
t.Helper()
dir := NewDirectory(lists)
dir.listURL = f.URL + "/get_zones.php"
if err := dir.Restore(context.Background()); err != nil {
t.Fatal(err)
}
h := &channelHarness{keys: keystore.NewMapKeyStore(nil)}
c, err := NewChannels(context.Background(), config.MeshMapperChannelsConfig{Enabled: enabled}, store, dir, h.keys)
if err != nil {
t.Fatal(err)
}
c.channelsURL = func(site string) (string, bool) { return site + "get_channels.php", strings.HasPrefix(site, f.URL) }
c.OnNewKeys(func(hashes [][]byte) { h.added = append(h.added, hashes...) })
h.c = c
return h
}
func (h *channelHarness) refresh(t *testing.T, n int) {
t.Helper()
for range n {
if err := h.c.Refresh(context.Background()); err != nil {
t.Fatal(err)
}
}
}
func TestChannelsImportKeysAndMembers(t *testing.T) {
f := newChannelSite(t)
store := newChannelMemoryStore("YOW")
h := newChannelHarness(t, f, store, newZoneListMemory(), true)
h.refresh(t, 2)
secret, hash, fingerprint := keystore.DeriveHashtagKey("ottawa-mesh")
got := h.keys.GetKey([]byte{hash})
if len(got) != 1 || !bytes.Equal(got[0].Key, secret) || got[0].Name != "#ottawa-mesh" || got[0].Hashtag != "ottawa-mesh" {
t.Fatalf("key not imported: %+v", got)
}
if len(store.members["YOW"]) != 1 || !bytes.Equal(store.members["YOW"][0], fingerprint) {
t.Fatal("YOW membership not saved", store.members)
}
if len(h.added) != 1 || h.added[0][0] != hash {
t.Fatal("new key not reported for backfill", h.added)
}
if b := h.c.sources[0].cache; b.NextAttempt.Sub(b.AttemptedAt) != 24*time.Hour {
t.Fatal("refresh interval", b.NextAttempt.Sub(b.AttemptedAt))
}
}
func TestChannelsRejectMismatchKeepsLastGood(t *testing.T) {
f := newChannelSite(t)
store := newChannelMemoryStore("YOW")
h := newChannelHarness(t, f, store, newZoneListMemory(), true)
h.refresh(t, 2)
good := string(channelList("YOW", "#ottawa-mesh"))
secret, hash, _ := keystore.DeriveHashtagKey("ottawa-mesh")
for name, body := range map[string]string{
"planted key": strings.Replace(good, hex.EncodeToString(secret), strings.Repeat("0", 32), 1),
"wrong key": strings.Replace(string(channelList("YOW", "#other")), `"#other"`, `"#planted"`, 1),
"wrong hash": strings.Replace(good, `"hash":"`+hex.EncodeToString([]byte{hash})+`"`, `"hash":"zz"`, 1),
"other region": string(channelList("YYZ", "#ottawa-mesh")),
"bad count": strings.Replace(good, `"count":1`, `"count":2`, 1),
"uppercase": string(channelList("YOW", "#Ottawa")),
"group zones": strings.Replace(good, `"zones":["YOW"]}`, `"zones":["YOW","YYZ"]}`, 1),
"missing count": strings.Replace(good, `"count":1,`, ``, 1),
} {
f.body = body
h.c.sources[0].cache.NextAttempt = time.Time{}
h.refresh(t, 1)
s := h.c.sources[0]
if s.cache.LastError != "invalid response" || len(h.keys.GetKey([]byte{hash})) != 1 || len(store.members["YOW"]) != 1 {
t.Fatalf("%s: accepted or dropped last good list: %q", name, s.cache.LastError)
}
if s.cache.NextAttempt.Sub(s.cache.AttemptedAt) != 24*time.Hour {
t.Fatalf("%s: retried before the rate limit", name)
}
}
}
func TestChannelsRateLimitAndFailuresWait(t *testing.T) {
for _, tc := range []struct {
status int
retryAfter string
want time.Duration
}{{503, "", 24 * time.Hour}, {429, "60", 24 * time.Hour}, {429, "172800", 48 * time.Hour}} {
f := newChannelSite(t)
f.status, f.retryAfter = tc.status, tc.retryAfter
h := newChannelHarness(t, f, newChannelMemoryStore("YOW"), newZoneListMemory(), true)
h.refresh(t, 3)
s := h.c.sources[0]
if f.channels != 1 || s.cache.NextAttempt.Sub(s.cache.AttemptedAt) != tc.want {
t.Fatalf("HTTP %d Retry-After %q: calls=%d wait=%v", tc.status, tc.retryAfter, f.channels, s.cache.NextAttempt.Sub(s.cache.AttemptedAt))
}
}
}
func TestChannelsRestoreWithoutRequests(t *testing.T) {
f := newChannelSite(t)
store, lists := newChannelMemoryStore("YOW"), newZoneListMemory()
newChannelHarness(t, f, store, lists, true).refresh(t, 2)
restarted := newChannelHarness(t, f, store, lists, true)
_, hash, _ := keystore.DeriveHashtagKey("ottawa-mesh")
if len(restarted.keys.GetKey([]byte{hash})) != 1 {
t.Fatal("saved channels not restored")
}
restarted.refresh(t, 2)
if f.listCalls != 1 || f.channels != 1 {
t.Fatal("restart made requests", f.listCalls, f.channels)
}
}
func TestChannelsDisabledClearsMembers(t *testing.T) {
f := newChannelSite(t)
store := newChannelMemoryStore("YOW")
store.members["YOW"] = [][]byte{{1}}
h := newChannelHarness(t, f, store, newZoneListMemory(), false)
if len(store.members) != 0 || len(h.c.sources) != 0 {
t.Fatal("disabled import kept membership or sources")
}
}
internal/meshmapper/directory.go
+248
View File
@@ -0,0 +1,248 @@
// Copyright 2026 Beacon Contributors
// SPDX-License-Identifier: AGPL-3.0-or-later
package meshmapper
import (
"cmp"
"context"
"encoding/json"
"fmt"
"log/slog"
"net/http"
"net/url"
"regexp"
"slices"
"strings"
"sync"
"time"
"unicode"
)
type zoneEntry struct {
url string
hasBoundary bool
name string // short display name, e.g. "Nanaimo"
lat, lon *float64 // nil when missing or out of range
}
type zoneGroup struct {
name string
members []string
}
type zoneList struct {
zones map[string]zoneEntry
groups map[string]zoneGroup
etag string
fetchedAt, nextAttempt time.Time
}
// ZoneList is one country's saved get_zones.php response and fetch state.
type ZoneList struct {
Country string
Payload json.RawMessage
ETag string
FetchedAt, AttemptedAt, NextAttempt time.Time
LastError string
}
type DirectoryStore interface {
ListZoneLists(ctx context.Context) ([]ZoneList, error)
SaveZoneList(ctx context.Context, l ZoneList) error
}
// Directory caches MeshMapper's per-country zone lists, shared by the zones and scopes imports.
type Directory struct {
mu sync.Mutex
store DirectoryStore
client *http.Client
listURL string
lists map[string]*zoneList
version int // bumped whenever a list is replaced
}
func NewDirectory(store DirectoryStore) *Directory {
return &Directory{store: store, listURL: ZonesURL, lists: map[string]*zoneList{}, client: &http.Client{
Timeout: requestTimeout,
CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
}}
}
// Restore loads saved lists so a restart doesn't spend each country's daily request.
func (d *Directory) Restore(ctx context.Context) error {
saved, err := d.store.ListZoneLists(ctx)
if err != nil {
return fmt.Errorf("restore MeshMapper zone lists: %w", err)
}
d.mu.Lock()
defer d.mu.Unlock()
for _, l := range saved {
list := &zoneList{nextAttempt: l.NextAttempt}
if zones, groups, err := decodeZones(l.Payload, l.Country); err == nil {
list.zones, list.groups, list.etag, list.fetchedAt = zones, groups, l.ETag, l.FetchedAt
d.version++
}
d.lists[l.Country] = list
}
return nil
}
// List returns the country's fresh zone list. fetched reports that this call
// spent the caller's one request; nil zones without fetched means back off.
func (d *Directory) List(ctx context.Context, country string, now time.Time) (zones map[string]zoneEntry, fetched bool, err error) {
d.mu.Lock()
defer d.mu.Unlock()
list := d.lists[country]
if list != nil && list.zones != nil && now.Sub(list.fetchedAt) < zoneListFresh {
return list.zones, false, nil
}
if list != nil && now.Before(list.nextAttempt) {
return nil, false, nil
}
return nil, true, d.fetch(ctx, country, now)
}
func (d *Directory) fetch(ctx context.Context, country string, now time.Time) error {
list := d.lists[country]
if list == nil {
list = &zoneList{}
d.lists[country] = list
}
endpoint := d.listURL + "?country=" + url.QueryEscape(country)
etag := ""
if list.zones != nil {
etag = list.etag
}
status, body, header, err := get(ctx, d.client, "Beacon-MeshMapper-Zones/1", endpoint, etag, MaxZoneList)
if err != nil {
return err
}
problem := ""
var payload json.RawMessage
var retryAt time.Time
switch status {
case http.StatusOK:
zones, groups, decodeErr := decodeZones(body, country)
if decodeErr != nil {
problem = "invalid response"
} else {
list.zones, list.groups, list.etag = zones, groups, header.Get("ETag")
payload = body
d.version++
}
case http.StatusNotModified:
if list.zones == nil {
problem = "304 without cached list"
}
default:
problem = statusProblem(status, header, now, &retryAt)
}
// Every request counts against the country's 23.5h allowance, failed or not.
list.nextAttempt = now.Add(zoneListFresh)
if retryAt.After(list.nextAttempt) {
list.nextAttempt = retryAt
}
saved := ZoneList{Country: country, Payload: payload, AttemptedAt: now, NextAttempt: list.nextAttempt, LastError: problem}
if problem == "" {
list.fetchedAt = now
saved.FetchedAt, saved.ETag = now, list.etag
}
level := slog.LevelInfo
if problem != "" {
level = slog.LevelWarn
}
slog.Log(ctx, level, "MeshMapper zone list checked", "component", "meshmapper.zones", "country", country,
"zones", len(list.zones), "groups", len(list.groups), "next_attempt", list.nextAttempt, "last_error", problem)
if err := d.store.SaveZoneList(ctx, saved); err != nil {
return fmt.Errorf("persist MeshMapper zone list %s: %w", country, err)
}
return nil
}
// snapshot merges every loaded list's zones and groups by code. complete is false until
// each country has a list, so a cold start can't look like every group disappearing.
func (d *Directory) snapshot(countries []string) (merged map[string]zoneGroup, zones map[string]zoneEntry, version int, complete bool) {
d.mu.Lock()
defer d.mu.Unlock()
merged, zones = map[string]zoneGroup{}, map[string]zoneEntry{}
for _, country := range countries {
list := d.lists[country]
if list == nil || list.zones == nil {
return nil, nil, d.version, false
}
for code, z := range list.zones {
zones[code] = z
}
for code, g := range list.groups {
if have, ok := merged[code]; ok {
g.members = append(slices.Clone(have.members), g.members...)
slices.Sort(g.members)
g.members = slices.Compact(g.members)
g.name = have.name
}
merged[code] = g
}
}
return merged, zones, d.version, true
}
var (
groupCode = regexp.MustCompile(`^[A-Z0-9]{2,16}$`)
memberIATA = regexp.MustCompile(`^[A-Z]{3}$`)
)
func decodeZones(body []byte, country string) (map[string]zoneEntry, map[string]zoneGroup, error) {
var document struct {
Country string `json:"country"`
Zones *[]struct {
Code string `json:"code"`
Name string `json:"name"`
ShortName string `json:"short_name"`
Lat *float64 `json:"lat"`
Lon *float64 `json:"lon"`
URL string `json:"url"`
HasBoundary bool `json:"has_boundary"`
} `json:"zones"`
Groups []struct {
Code string `json:"code"`
Name string `json:"name"`
Members []string `json:"members"`
} `json:"groups"`
}
if err := json.Unmarshal(body, &document); err != nil {
return nil, nil, err
}
if !strings.EqualFold(document.Country, country) || document.Zones == nil {
return nil, nil, fmt.Errorf("invalid zone list")
}
zones := make(map[string]zoneEntry, len(*document.Zones))
for _, zone := range *document.Zones {
entry := zoneEntry{url: zone.URL, hasBoundary: zone.HasBoundary, name: cmp.Or(zone.ShortName, zone.Name)}
if entry.name = strings.TrimSpace(entry.name); len(entry.name) > 128 || strings.ContainsFunc(entry.name, unicode.IsControl) {
entry.name = ""
}
if zone.Lat != nil && zone.Lon != nil && *zone.Lat >= -90 && *zone.Lat <= 90 && *zone.Lon >= -180 && *zone.Lon <= 180 {
entry.lat, entry.lon = zone.Lat, zone.Lon
}
zones[strings.ToUpper(zone.Code)] = entry
}
groups := map[string]zoneGroup{}
for _, g := range document.Groups {
code, name := strings.ToUpper(g.Code), strings.TrimSpace(g.Name)
if !groupCode.MatchString(code) || name == "" || len(name) > 128 || strings.ContainsFunc(name, unicode.IsControl) {
continue // a bad group never blocks the zone list
}
var members []string
for _, m := range g.Members {
if m = strings.ToUpper(m); memberIATA.MatchString(m) {
members = append(members, m)
}
}
slices.Sort(members)
if members = slices.Compact(members); len(members) > 0 {
groups[code] = zoneGroup{name: name, members: members}
}
}
return zones, groups, nil
}
internal/meshmapper/groups.go
+201
View File
@@ -0,0 +1,201 @@
// Copyright 2026 Beacon Contributors
// SPDX-License-Identifier: AGPL-3.0-or-later
package meshmapper
import (
"cmp"
"context"
"fmt"
"log/slog"
"slices"
"strings"
)
// RegionState is a stored region; imported ones are also what SaveImportedRegion writes.
type RegionState struct {
Slug, Name string
DisplayOrder int
Imported bool
IATAs []string
CenterLat, CenterLng *float64
}
// IATADetails is an IATA's display name and approximate location.
type IATADetails struct {
IATA string
Name string
Lat, Lng *float64
}
// SetConfiguredIATAs names the IATAs whose config entry sets a name or location; those win.
func (z *Zones) SetConfiguredIATAs(iatas []string) {
z.configured = map[string]bool{}
for _, iata := range iatas {
z.configured[iata] = true
}
}
// OnIATAsChange is told whenever imported IATA names or locations change.
func (z *Zones) OnIATAsChange(fn func(ctx context.Context)) { z.onIATAs = fn }
// OnRegionsChange is told whenever the imported regions change, for cache invalidation.
func (z *Zones) OnRegionsChange(fn func(ctx context.Context)) { z.onRegions = fn }
// pruneGroups drops every imported region when group import is off.
func (z *Zones) pruneGroups(ctx context.Context) error {
pruned, err := z.store.PruneImportedRegions(ctx, nil)
if err != nil {
return fmt.Errorf("prune MeshMapper regions: %w", err)
}
z.regionsRemoved(pruned)
if len(pruned) > 0 {
z.regionsChanged(ctx)
}
return nil
}
// syncDirectory applies the loaded zone lists to IATA details and, when enabled,
// imported regions. It only runs once every tracked country has a list, and again
// when a list or the IATA set changes.
func (z *Zones) syncDirectory(ctx context.Context, iatas []string) error {
var countries []string
for _, r := range z.regions {
countries = append(countries, r.country)
}
slices.Sort(countries)
groups, zones, version, complete := z.dir.snapshot(slices.Compact(countries))
if !complete || (z.groupsSynced && version == z.groupsVersion && len(iatas) == z.groupsKnown) {
return nil
}
if err := z.syncIATADetails(ctx, zones); err != nil {
return err
}
if z.importGroups {
if err := z.syncGroups(ctx, iatas, groups, zones); err != nil {
return err
}
}
z.groupsSynced, z.groupsVersion, z.groupsKnown = true, version, len(iatas)
return nil
}
// syncIATADetails names and locates each listed IATA unless its config entry does.
func (z *Zones) syncIATADetails(ctx context.Context, zones map[string]zoneEntry) error {
stored, err := z.store.ListIATADetails(ctx)
if err != nil {
return fmt.Errorf("list IATAs for MeshMapper details: %w", err)
}
changed := false
for _, cur := range stored {
zone, ok := zones[cur.IATA]
if !ok || z.configured[cur.IATA] || zone.name == "" && zone.lat == nil {
continue
}
name := cmp.Or(zone.name, cur.Name)
lat, lng := cur.Lat, cur.Lng
if zone.lat != nil {
lat, lng = zone.lat, zone.lon
}
if name == cur.Name && sameFloat(lat, cur.Lat) && sameFloat(lng, cur.Lng) {
continue
}
if err := z.store.UpsertIATADetails(ctx, cur.IATA, name, lat, lng); err != nil {
return fmt.Errorf("save MeshMapper details for %s: %w", cur.IATA, err)
}
changed = true
}
if changed && z.onIATAs != nil {
z.onIATAs(ctx)
}
return nil
}
func (z *Zones) syncGroups(ctx context.Context, iatas []string, groups map[string]zoneGroup, zones map[string]zoneEntry) error {
known := make(map[string]bool, len(iatas))
for _, iata := range iatas {
known[iata] = true
}
state, err := z.store.ListRegionState(ctx)
if err != nil {
return fmt.Errorf("list regions for MeshMapper groups: %w", err)
}
stored, handOrder := map[string]RegionState{}, 0
for _, r := range state {
stored[r.Slug] = r
if !r.Imported {
handOrder = max(handOrder, r.DisplayOrder)
}
}
var want []RegionState
for code, g := range groups {
if slices.ContainsFunc(g.members, func(m string) bool { return known[m] }) {
lat, lng := center(g.members, zones)
want = append(want, RegionState{Slug: strings.ToLower(code), Name: g.name, Imported: true, IATAs: g.members, CenterLat: lat, CenterLng: lng})
}
}
slices.SortFunc(want, func(a, b RegionState) int { return cmp.Or(cmp.Compare(a.Name, b.Name), cmp.Compare(a.Slug, b.Slug)) })
keep, changed := []string{}, false
for n, r := range want {
r.DisplayOrder = handOrder + 1 + n
if cur, ok := stored[r.Slug]; ok && !cur.Imported {
slog.Info("MeshMapper group skipped: a configured region uses its slug", "component", "meshmapper.zones", "slug", r.Slug)
continue
} else if ok && cur.Name == r.Name && cur.DisplayOrder == r.DisplayOrder && slices.Equal(cur.IATAs, r.IATAs) &&
sameFloat(cur.CenterLat, r.CenterLat) && sameFloat(cur.CenterLng, r.CenterLng) {
keep = append(keep, r.Slug)
continue
}
saved, err := z.store.SaveImportedRegion(ctx, r)
if err != nil {
return fmt.Errorf("save MeshMapper region %s: %w", r.Slug, err)
}
if !saved {
slog.Info("MeshMapper group skipped: a configured region uses its slug", "component", "meshmapper.zones", "slug", r.Slug)
continue
}
keep, changed = append(keep, r.Slug), true
slog.Info("MeshMapper region imported", "component", "meshmapper.zones", "slug", r.Slug, "name", r.Name, "iatas", r.IATAs)
}
pruned, err := z.store.PruneImportedRegions(ctx, keep)
if err != nil {
return fmt.Errorf("prune MeshMapper regions: %w", err)
}
z.regionsRemoved(pruned)
if changed || len(pruned) > 0 {
z.regionsChanged(ctx)
}
return nil
}
// center averages the members' zone locations; nil when none has one.
func center(members []string, zones map[string]zoneEntry) (*float64, *float64) {
var lat, lng float64
n := 0
for _, m := range members {
if z := zones[m]; z.lat != nil {
lat, lng, n = lat+*z.lat, lng+*z.lon, n+1
}
}
if n == 0 {
return nil, nil
}
lat, lng = lat/float64(n), lng/float64(n)
return &lat, &lng
}
func sameFloat(a, b *float64) bool {
return a == nil && b == nil || a != nil && b != nil && *a == *b
}
func (z *Zones) regionsRemoved(slugs []string) {
for _, slug := range slugs {
slog.Info("MeshMapper region removed", "component", "meshmapper.zones", "slug", slug)
}
}
func (z *Zones) regionsChanged(ctx context.Context) {
if z.onRegions != nil {
z.onRegions(ctx)
}
}
internal/meshmapper/groups_test.go
+184
View File
@@ -0,0 +1,184 @@
// Copyright 2026 Beacon Contributors
// SPDX-License-Identifier: AGPL-3.0-or-later
package meshmapper
import (
"context"
"slices"
"strings"
"testing"
"time"
"github.com/MeshCore-Beacon/beacon-server/internal/config"
)
const groupedList = `{"country":"CA","zones":[` +
`{"code":"YOW","name":"Ottawa, CA","short_name":"Ottawa","lat":45,"lon":-76,"url":"URL/","has_boundary":true,"group":"ONQC"},` +
`{"code":"YUL","name":"Montreal, CA","lat":46,"lon":-74,"url":"URL/","has_boundary":true,"group":"ONQC"},` +
`{"code":"YQB","name":"Quebec City, CA","url":"URL/","has_boundary":true,"group":"ONQC"}],"groups":[` +
`{"code":"ONQC","name":"Ottawa-Quebec Corridor","members":["YUL","YOW","YQB"],"url":"https://onqc.meshmapper.net/"},` +
`{"code":"GOLM","name":"Great Ontario Lake Mesh","members":["YYZ","YKF"],"url":"https://golm.meshmapper.net/"}]}`
type groupHarness struct {
z *Zones
f *fakeMeshMapper
store *zoneMemoryStore
invalidated int
}
func newGroupHarness(t *testing.T, f *fakeMeshMapper, store *zoneMemoryStore, importGroups bool) *groupHarness {
t.Helper()
if store.iatas == nil {
store.iatas = []string{"YOW"}
}
if store.rows == nil {
store.rows = map[string]Boundary{}
}
h := &groupHarness{f: f, store: store}
dir := NewDirectory(newZoneListMemory())
dir.listURL = f.URL + "/get_zones.php"
h.z = NewZones(config.MeshMapperZonesConfig{Enabled: true, ImportGroups: importGroups}, store, dir)
h.z.boundsURL = func(site string) (string, bool) { return site + "get_geojson.php", true }
h.z.OnRegionsChange(func(context.Context) { h.invalidated++ })
if err := h.z.Restore(context.Background()); err != nil {
t.Fatal(err)
}
return h
}
func groupedFake(t *testing.T) *fakeMeshMapper {
f := newFakeMeshMapper(t)
f.list = strings.ReplaceAll(groupedList, "URL", f.URL)
return f
}
// refetch makes the country list and YOW due, then runs the fetch and the sync tick.
func (h *groupHarness) refetch(t *testing.T) {
t.Helper()
if list := h.z.dir.lists["CA"]; list != nil {
list.fetchedAt, list.nextAttempt = time.Time{}, time.Time{}
}
for _, r := range h.z.regions {
r.b.NextAttempt = time.Time{}
}
for range 2 {
if err := h.z.Refresh(context.Background()); err != nil {
t.Fatal(err)
}
}
}
func TestGroupImportedWhenAMemberIsKnown(t *testing.T) {
h := newGroupHarness(t, groupedFake(t), &zoneMemoryStore{}, true)
h.refetch(t)
got, ok := h.store.regions["onqc"]
if !ok || got.Name != "Ottawa-Quebec Corridor" || !slices.Equal(got.IATAs, []string{"YOW", "YQB", "YUL"}) || got.DisplayOrder != 1 {
t.Fatalf("group not imported with full members: %+v", h.store.regions)
}
if _, ok := h.store.regions["golm"]; ok {
t.Fatal("group with no known member imported")
}
if got.CenterLat == nil || *got.CenterLat != 45.5 || *got.CenterLng != -75 {
t.Fatalf("center should average the located members: %v, %v", got.CenterLat, got.CenterLng)
}
if !slices.Contains(h.store.iatas, "YQB") || !slices.Contains(h.store.iatas, "YUL") {
t.Fatal("members not created as IATAs", h.store.iatas)
}
if h.invalidated != 1 {
t.Fatal("region caches not invalidated", h.invalidated)
}
h.refetch(t)
if h.invalidated != 1 {
t.Fatal("unchanged groups invalidated again", h.invalidated)
}
}
func TestGroupSlugClashKeepsConfiguredRegion(t *testing.T) {
hand := RegionState{Slug: "onqc", Name: "Mine", DisplayOrder: 4, IATAs: []string{"YOW"}}
store := &zoneMemoryStore{regions: map[string]RegionState{"onqc": hand}}
h := newGroupHarness(t, groupedFake(t), store, true)
h.refetch(t)
if got := store.regions["onqc"]; got.Imported || got.Name != "Mine" || !slices.Equal(got.IATAs, []string{"YOW"}) || h.invalidated != 0 {
t.Fatalf("configured region replaced: %+v", got)
}
}
func TestGroupDroppedRemovesRegion(t *testing.T) {
h := newGroupHarness(t, groupedFake(t), &zoneMemoryStore{}, true)
h.refetch(t)
h.f.list = strings.ReplaceAll(`{"country":"CA","zones":[{"code":"YOW","url":"URL/","has_boundary":true,"group":null}],"groups":[]}`, "URL", h.f.URL)
h.refetch(t)
if len(h.store.regions) != 0 || h.invalidated != 2 {
t.Fatalf("dropped group kept: %+v invalidated=%d", h.store.regions, h.invalidated)
}
}
func TestGroupImportOffRemovesImportedRegions(t *testing.T) {
hand := RegionState{Slug: "east", Name: "East", IATAs: []string{"YOW"}}
imported := RegionState{Slug: "onqc", Name: "Ottawa-Quebec Corridor", Imported: true, IATAs: []string{"YOW"}}
store := &zoneMemoryStore{regions: map[string]RegionState{"east": hand, "onqc": imported}}
h := newGroupHarness(t, groupedFake(t), store, false)
if _, ok := store.regions["onqc"]; ok || h.invalidated != 1 {
t.Fatal("imported region kept with import_groups off")
}
if _, ok := store.regions["east"]; !ok {
t.Fatal("configured region removed")
}
h.refetch(t)
if len(store.regions) != 1 {
t.Fatal("groups imported with import_groups off", store.regions)
}
}
func TestGroupFailedFetchKeepsGroups(t *testing.T) {
h := newGroupHarness(t, groupedFake(t), &zoneMemoryStore{}, true)
h.refetch(t)
h.f.listStatus = 500
h.z.groupsSynced = false // force a reconcile against the retained list
h.refetch(t)
if _, ok := h.store.regions["onqc"]; !ok {
t.Fatal("failed fetch dropped imported region")
}
}
func TestGroupColdStartWaitsForEveryList(t *testing.T) {
old := RegionState{Slug: "old", Name: "Old", Imported: true, IATAs: []string{"YOW"}}
store := &zoneMemoryStore{regions: map[string]RegionState{"old": old}}
f := groupedFake(t)
f.listStatus = 500
h := newGroupHarness(t, f, store, true)
h.refetch(t)
if _, ok := store.regions["old"]; !ok {
t.Fatal("imported regions pruned before any list loaded")
}
}
func TestZoneListNamesAndLocatesIATAs(t *testing.T) {
h := newGroupHarness(t, groupedFake(t), &zoneMemoryStore{iatas: []string{"YOW", "YUL", "YQB", "YYZ"}}, false)
h.z.SetConfiguredIATAs([]string{"YUL"})
changed := 0
h.z.OnIATAsChange(func(context.Context) { changed++ })
h.refetch(t)
yow := h.store.details["YOW"]
if yow.Name != "Ottawa" || yow.Lat == nil || *yow.Lat != 45 || *yow.Lng != -76 {
t.Fatalf("YOW not named and located from short_name: %+v", yow)
}
if yqb := h.store.details["YQB"]; yqb.Name != "Quebec City, CA" || yqb.Lat != nil {
t.Fatalf("YQB should fall back to name and stay unlocated: %+v", yqb)
}
if _, ok := h.store.details["YUL"]; ok {
t.Fatal("configured IATA overwritten")
}
if _, ok := h.store.details["YYZ"]; ok {
t.Fatal("unlisted IATA written")
}
if changed != 1 || h.store.writes != 2 {
t.Fatal("writes/invalidations", h.store.writes, changed)
}
h.z.groupsSynced = false
h.refetch(t)
if h.store.writes != 2 || changed != 1 {
t.Fatal("unchanged details rewritten", h.store.writes, changed)
}
}
internal/meshmapper/scopes.go
+99 -43
View File
@@ -20,15 +20,16 @@ import (
"unicode/utf8"
"github.com/MeshCore-Beacon/beacon-server/internal/config"
"github.com/MeshCore-Beacon/beacon-server/internal/iatadb"
"github.com/MeshCore-Beacon/beacon-server/internal/scopestore"
)
const (
MaxScopes = 64 // per source; at most 16 configured sources (1024 imported identities)
MaxScopes = 64 // per source
MaxBody = 64 << 10
PollInterval = 15 * time.Second // one source per tick, at most four requests/minute
failureRetry = 5 * time.Minute // transient failures shouldn't cost a whole refresh interval
failureRetry = config.MinScopesRefresh // get_scopes.php allows one request per region per 55m
)
// Cache stores source provenance and freshness separately from packet evidence.
@@ -39,28 +40,37 @@ type Cache struct {
LastError string
}
// Catalogue is one saved source snapshot.
type Catalogue struct {
IATA, URL string
Cache
}
type Store interface {
GetScopeCatalogue(context.Context, string, string) (*Cache, error)
ListKnownIATAs(context.Context) ([]string, error)
ListScopeCatalogues(context.Context) ([]Catalogue, error)
SaveScopeCatalogue(context.Context, string, string, Cache, []scopestore.Entry) error
}
type source struct {
iata, url string
cache Cache
entries []scopestore.Entry
generated time.Time
iata, country, url string
cache Cache
entries []scopestore.Entry
generated time.Time
}
// Importer is owned by one background task; ScopeStore synchronizes its consumers.
type Importer struct {
catalogues atomic.Value // immutable []Catalogue, refreshed off the ingest path
catalogues atomic.Value // immutable []PublicScopeCatalogue, off the ingest path
store Store
scopes *scopestore.ScopeStore
manual []scopestore.Entry
dir *Directory
scopesURL func(site string) (string, bool)
seen map[string]bool
sources []source
interval time.Duration
client *http.Client
retryAfter time.Time
onChange func(context.Context)
}
@@ -68,66 +78,111 @@ type Importer struct {
func (i *Importer) SetCacheInvalidator(fn func(context.Context)) { i.onChange = fn }
// New restores validated snapshots before ingestion, without making HTTP requests.
func New(ctx context.Context, cfg config.MeshMapperScopesConfig, store Store, scopes *scopestore.ScopeStore, manual []scopestore.Entry) (*Importer, error) {
i := &Importer{store: store, scopes: scopes, manual: manual, interval: cfg.Interval(), client: &http.Client{
Timeout: 10 * time.Second,
CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
}}
func New(ctx context.Context, cfg config.MeshMapperScopesConfig, store Store, dir *Directory, scopes *scopestore.ScopeStore, manual []scopestore.Entry) (*Importer, error) {
i := &Importer{store: store, scopes: scopes, manual: manual, dir: dir, scopesURL: scopesEndpoint, seen: map[string]bool{},
interval: cfg.Interval(), client: &http.Client{
Timeout: requestTimeout,
CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
}}
if !cfg.Enabled {
return i, nil
}
keys := make([]string, 0, len(cfg.Sources))
for key := range cfg.Sources {
keys = append(keys, key)
saved, err := store.ListScopeCatalogues(ctx)
if err != nil {
return nil, fmt.Errorf("restore scope catalogues: %w", err)
}
slices.Sort(keys)
for _, key := range keys {
s := source{iata: key, url: cfg.Sources[key]}
cached, err := store.GetScopeCatalogue(ctx, s.iata, s.url)
if err != nil {
return nil, fmt.Errorf("restore scope catalogue %s: %w", key, err)
// A site that moved leaves its old row behind; the latest attempt wins.
latest := map[string]Catalogue{}
for _, c := range saved {
if old, ok := latest[c.IATA]; !ok || c.AttemptedAt.After(old.AttemptedAt) {
latest[c.IATA] = c
}
if cached != nil {
s.cache = *cached
if len(cached.Payload) > 0 {
s.entries, s.generated, err = decode(cached.Payload, key)
}
iatas, err := store.ListKnownIATAs(ctx)
if err != nil {
return nil, fmt.Errorf("list IATAs for MeshMapper scopes: %w", err)
}
i.track(iatas, latest)
i.publish()
i.publishCatalogues()
return i, nil
}
// track adds a source for each IATA not seen before.
func (i *Importer) track(iatas []string, saved map[string]Catalogue) {
for _, iata := range iatas {
if i.seen[iata] {
continue
}
i.seen[iata] = true
country := iatadb.CountryFor(iata)
if country == "" {
continue
}
s := source{iata: iata, country: country}
if c, ok := saved[iata]; ok {
s.url, s.cache = c.URL, c.Cache
if len(c.Payload) > 0 {
var err error
s.entries, s.generated, err = decode(c.Payload, iata)
if err != nil {
s.entries = nil
s.cache.Payload, s.cache.ETag = nil, ""
s.cache.LastError = "invalid saved catalogue; awaiting refresh"
}
}
if cached.LastError == "HTTP 429" && cached.NextAttempt.After(i.retryAfter) {
i.retryAfter = cached.NextAttempt
}
i.log(s, "restored")
}
i.sources = append(i.sources, s)
i.log(s, "restored")
}
i.publish()
i.publishCatalogues()
return i, nil
}
// Refresh checks only one due source. The scheduler serializes calls every 15s.
func (i *Importer) Refresh(ctx context.Context) (err error) {
defer i.publishCatalogues()
parent := ctx
defer func() {
if parent.Err() != nil {
err = nil
}
}()
ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
ctx, cancel := context.WithTimeout(ctx, refreshTimeout)
defer cancel()
now := time.Now().UTC()
if now.Before(i.retryAfter) {
return nil
iatas, err := i.store.ListKnownIATAs(ctx)
if err != nil {
return fmt.Errorf("list IATAs for MeshMapper scopes: %w", err)
}
i.track(iatas, nil)
for n := range i.sources {
s := &i.sources[n]
if now.Before(s.cache.NextAttempt) {
continue
}
zones, fetched, err := i.dir.List(ctx, s.country, now)
if err != nil || fetched {
return err
}
if zones == nil {
continue
}
entry, listed := zones[s.iata]
endpoint, valid := "", false
if listed {
endpoint, valid = i.scopesURL(entry.url)
}
if !valid {
// No request to make; names already imported stay until a catalogue replaces them.
s.cache.NextAttempt, s.cache.LastError = now.Add(i.interval), "not listed"
if listed {
s.cache.LastError = "invalid site URL"
}
i.log(*s, "skipped")
continue
}
if endpoint != s.url {
s.url, s.cache.ETag = endpoint, ""
}
return i.refresh(ctx, s, now)
}
return nil
@@ -195,15 +250,12 @@ func (i *Importer) refresh(ctx context.Context, s *source, now time.Time) error
}
}
if update.LastError != "" {
// Transient failures shouldn't cost a whole refresh interval, but never beat a Retry-After.
// Failures retry before the full interval, but never sooner than the rate limit or a Retry-After.
update.NextAttempt = now.Add(min(i.interval, failureRetry))
if retryAfter.After(update.NextAttempt) {
update.NextAttempt = retryAfter
}
}
if update.LastError == "HTTP 429" {
i.retryAfter = update.NextAttempt
}
if err := i.store.SaveScopeCatalogue(ctx, s.iata, s.url, update, entries); err != nil {
// Avoid a retry every scheduler tick during a DB outage; never publish uncommitted keys.
s.cache.NextAttempt = update.NextAttempt
@@ -227,7 +279,6 @@ func (i *Importer) refresh(ctx context.Context, s *source, now time.Time) error
i.onChange(ctx)
}
}
i.publishCatalogues()
i.log(*s, "checked")
return nil
}
@@ -237,8 +288,10 @@ func (i *Importer) publish() {
for _, e := range i.manual {
byName[e.Name] = e
}
members := make(map[string][]string, len(i.sources))
for _, s := range i.sources {
for _, candidate := range s.entries {
members[s.iata] = append(members[s.iata], candidate.Name)
entry, exists := byName[candidate.Name]
if exists && entry.IATAs == nil {
continue
@@ -260,17 +313,20 @@ func (i *Importer) publish() {
entries = append(entries, byName[name])
}
i.scopes.Load(entries)
i.scopes.SetCatalogueMembers(members)
}
func (i *Importer) log(s source, action string) {
level := slog.LevelInfo
if s.cache.LastError != "" {
if s.cache.LastError != "" && s.cache.LastError != "not listed" {
level = slog.LevelWarn
}
slog.Log(context.Background(), level, "MeshMapper scopes "+action, "component", "meshmapper.scopes", "iata", s.iata, "source", s.url,
"names", len(s.entries), "generated_at", s.generated, "checked_at", s.cache.CheckedAt, "next_attempt", s.cache.NextAttempt, "last_error", s.cache.LastError)
}
func scopesEndpoint(site string) (string, bool) { return siteEndpoint(site, "get_scopes.php") }
func decode(body []byte, iata string) ([]scopestore.Entry, time.Time, error) {
var document struct {
GeneratedAt time.Time `json:"generated_at"`
internal/meshmapper/scopes_test.go
+158 -67
View File
@@ -11,6 +11,7 @@ import (
"net/http"
"net/http/httptest"
"reflect"
"slices"
"strings"
"sync/atomic"
"testing"
@@ -29,24 +30,36 @@ func catalogue(names ...string) []byte {
return b
}
type sourceKey struct{ iata, url string }
type memoryStore struct {
rows map[string]Cache
iatas []string
rows map[sourceKey]Cache
fail bool
saved []scopestore.Entry
}
func (s *memoryStore) GetScopeCatalogue(_ context.Context, iata, url string) (*Cache, error) {
row, ok := s.rows[iata+url]
if !ok {
return nil, nil
}
return &row, nil
func newMemoryStore(iatas ...string) *memoryStore {
return &memoryStore{iatas: iatas, rows: map[sourceKey]Cache{}}
}
func (s *memoryStore) ListKnownIATAs(context.Context) ([]string, error) {
return slices.Clone(s.iatas), nil
}
func (s *memoryStore) ListScopeCatalogues(context.Context) ([]Catalogue, error) {
var out []Catalogue
for key, c := range s.rows {
out = append(out, Catalogue{IATA: key.iata, URL: key.url, Cache: c})
}
return out, nil
}
func (s *memoryStore) SaveScopeCatalogue(_ context.Context, iata, url string, next Cache, entries []scopestore.Entry) error {
if s.fail {
return errors.New("offline")
}
old := s.rows[iata+url]
old := s.rows[sourceKey{iata, url}]
if next.Payload == nil {
next.Payload = old.Payload
}
@@ -54,11 +67,25 @@ func (s *memoryStore) SaveScopeCatalogue(_ context.Context, iata, url string, ne
next.CheckedAt = old.CheckedAt
next.ETag = old.ETag
}
s.rows[iata+url] = next
s.rows[sourceKey{iata, url}] = next
s.saved = append(s.saved, entries...)
return nil
}
// newImporter restores an importer for YOW whose source points at url.
func newImporter(t *testing.T, cfg config.MeshMapperScopesConfig, store *memoryStore, url string, scopes *scopestore.ScopeStore, manual ...scopestore.Entry) *Importer {
t.Helper()
imp, err := New(context.Background(), cfg, store, NewDirectory(newZoneListMemory()), scopes, manual)
if err != nil {
t.Fatal(err)
}
if len(imp.sources) != 1 {
t.Fatal("expected one source", imp.sources)
}
imp.sources[0].url = url
return imp
}
func names(store *scopestore.ScopeStore) []string {
out := []string{}
for _, e := range store.Entries() {
@@ -84,15 +111,12 @@ func TestCatalogueRefreshFallbackAndRestart(t *testing.T) {
}
}))
defer server.Close()
cfg := config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": server.URL}}
store := &memoryStore{rows: map[string]Cache{}}
cfg := config.MeshMapperScopesConfig{Enabled: true}
store := newMemoryStore("YOW")
scopes := scopestore.New()
manual := scopestore.FromName("manual")
manual.TransportKey = []byte("manual wins")
imp, err := New(ctx, cfg, store, scopes, []scopestore.Entry{manual})
if err != nil {
t.Fatal(err)
}
imp := newImporter(t, cfg, store, server.URL, scopes, manual)
now := time.Now().UTC()
check := func() {
t.Helper()
@@ -111,6 +135,9 @@ func TestCatalogueRefreshFallbackAndRestart(t *testing.T) {
if !reflect.DeepEqual(scopes.Entries()[1].IATAs, []string{"YOW"}) {
t.Fatal("import not bound to region")
}
if got := scopes.NamesForIATAs([]string{"YOW"}); !reflect.DeepEqual(got, []string{"#manual", "#yow"}) {
t.Fatal("catalogue membership must include manually overridden names", got)
}
generated := imp.sources[0].generated
status = 304
check()
@@ -127,14 +154,11 @@ func TestCatalogueRefreshFallbackAndRestart(t *testing.T) {
}
status, after = 429, "7200"
check()
if !imp.retryAfter.Equal(now.Add(2 * time.Hour)) {
if !imp.sources[0].cache.NextAttempt.Equal(now.Add(2 * time.Hour)) {
t.Fatal("Retry-After ignored")
}
restored := scopestore.New()
restart, err := New(ctx, cfg, store, restored, []scopestore.Entry{manual})
if err != nil {
t.Fatal(err)
}
restart := newImporter(t, cfg, store, server.URL, restored, manual)
if !reflect.DeepEqual(names(restored), names(scopes)) {
t.Fatal("outage restart lost cached names")
}
@@ -143,7 +167,7 @@ func TestCatalogueRefreshFallbackAndRestart(t *testing.T) {
t.Fatal(err)
}
if calls != before {
t.Fatal("restart ignored shared rate-limit cooldown")
t.Fatal("restart retried before Retry-After")
}
status, after, body = 200, "", catalogue()
check()
@@ -212,11 +236,7 @@ func TestNoRedirectAndResponseBounds(t *testing.T) {
}
}))
defer server.Close()
store := &memoryStore{rows: map[string]Cache{}}
imp, err := New(context.Background(), config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": server.URL}}, store, scopestore.New(), nil)
if err != nil {
t.Fatal(err)
}
imp := newImporter(t, config.MeshMapperScopesConfig{Enabled: true}, newMemoryStore("YOW"), server.URL, scopestore.New())
imp.client.Timeout = 20 * time.Millisecond
for _, value := range []string{"redirect", "oversize", "timeout", "304"} {
mode.Store(value)
@@ -234,22 +254,22 @@ func TestNoRedirectAndResponseBounds(t *testing.T) {
func TestImportedSourcesOverlapWithoutGlobalMembership(t *testing.T) {
ctx := context.Background()
store := &memoryStore{rows: map[string]Cache{}}
store := newMemoryStore("YOW", "YVR")
for _, iata := range []string{"YOW", "YVR"} {
store.rows[iata+iata] = Cache{Payload: []byte(strings.ReplaceAll(string(catalogue("can")), "YOW", iata))}
store.rows[sourceKey{iata, "old-" + iata}] = Cache{Payload: []byte(`{}`)}
store.rows[sourceKey{iata, iata}] = Cache{Payload: []byte(strings.ReplaceAll(string(catalogue("can")), "YOW", iata)), AttemptedAt: time.Now()}
}
scopes := scopestore.New()
_, err := New(ctx, config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": "YOW", "YVR": "YVR"}}, store, scopes, nil)
_, err := New(ctx, config.MeshMapperScopesConfig{Enabled: true}, store, NewDirectory(newZoneListMemory()), scopes, nil)
if err != nil || len(scopes.Entries()) != 1 || !reflect.DeepEqual(scopes.Entries()[0].IATAs, []string{"YOW", "YVR"}) {
t.Fatal(scopes.Entries(), err)
t.Fatal("latest saved catalogue per IATA not restored", scopes.Entries(), err)
}
_, err = New(ctx, config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": "changed-source"}}, store, scopes, nil)
if err != nil || len(scopes.Entries()) != 0 {
t.Fatal("removed/changed source remained active", err)
if got := scopes.NamesForIATAs([]string{"YVR"}); !reflect.DeepEqual(got, []string{"#can"}) {
t.Fatal(got)
}
// Disabled mode never calls the store, even if URLs remain configured.
_, err = New(ctx, config.MeshMapperScopesConfig{Sources: map[string]string{"YOW": "YOW"}}, nil, scopes, nil)
if err != nil {
// Disabled mode never calls the store.
disabled := scopestore.New()
if _, err = New(ctx, config.MeshMapperScopesConfig{}, nil, NewDirectory(newZoneListMemory()), disabled, nil); err != nil || len(disabled.Entries()) != 0 {
t.Fatal(err)
}
}
@@ -262,41 +282,39 @@ func TestInvalidSavedCatalogueDoesNotBlockStartupOrRefresh(t *testing.T) {
_, _ = w.Write(catalogue("yow"))
}))
defer server.Close()
store := &memoryStore{rows: map[string]Cache{"YOW" + server.URL: {Payload: []byte(`{"invalid":true}`), ETag: `"old"`}}}
store := newMemoryStore("YOW")
store.rows[sourceKey{"YOW", server.URL}] = Cache{Payload: []byte(`{"invalid":true}`), ETag: `"old"`}
scopes := scopestore.New()
imp, err := New(context.Background(), config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": server.URL}}, store, scopes, []scopestore.Entry{scopestore.FromName("manual")})
if err != nil || !reflect.DeepEqual(names(scopes), []string{"#manual"}) {
t.Fatal("invalid optional source blocked manual startup", err)
imp := newImporter(t, config.MeshMapperScopesConfig{Enabled: true}, store, server.URL, scopes, scopestore.FromName("manual"))
if !reflect.DeepEqual(names(scopes), []string{"#manual"}) {
t.Fatal("invalid optional source blocked manual startup")
}
if err = imp.Refresh(context.Background()); err != nil || !reflect.DeepEqual(names(scopes), []string{"#manual", "#yow"}) {
if err := imp.refresh(context.Background(), &imp.sources[0], time.Now()); err != nil || !reflect.DeepEqual(names(scopes), []string{"#manual", "#yow"}) {
t.Fatal("source did not recover on refresh", err, names(scopes))
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
imp.sources[0].cache.NextAttempt = time.Time{}
if err = imp.Refresh(ctx); err != nil {
if err := imp.Refresh(ctx); err != nil {
t.Fatalf("normal shutdown is a task failure: %v", err)
}
}
func TestRetryAfterShorterThanIntervalIsHonoured(t *testing.T) {
ctx := context.Background()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Retry-After", "1800")
w.WriteHeader(http.StatusTooManyRequests)
}))
defer server.Close()
cfg := config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": server.URL}}
imp, err := New(ctx, cfg, &memoryStore{rows: map[string]Cache{}}, scopestore.New(), nil)
if err != nil {
t.Fatal(err)
}
now := time.Now().UTC()
if err := imp.refresh(ctx, &imp.sources[0], now); err != nil {
t.Fatal(err)
}
if got := imp.sources[0].cache.NextAttempt.Sub(now); got != 30*time.Minute {
t.Fatalf("next attempt after 429 in %v, want %v (server Retry-After must not be overridden by the 1h interval)", got, 30*time.Minute)
func TestRetryAfterNeverBeatsTheRateLimit(t *testing.T) {
for retryAfter, want := range map[string]time.Duration{"1800": time.Hour, "7200": 2 * time.Hour} {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Retry-After", retryAfter)
w.WriteHeader(http.StatusTooManyRequests)
}))
imp := newImporter(t, config.MeshMapperScopesConfig{Enabled: true}, newMemoryStore("YOW"), server.URL, scopestore.New())
now := time.Now().UTC()
if err := imp.refresh(context.Background(), &imp.sources[0], now); err != nil {
t.Fatal(err)
}
server.Close()
if got := imp.sources[0].cache.NextAttempt.Sub(now); got != want {
t.Fatalf("Retry-After %ss: next attempt in %v, want %v", retryAfter, got, want)
}
}
}
@@ -304,16 +322,89 @@ func TestFailedRefreshRetriesSoon(t *testing.T) {
ctx := context.Background()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(500) }))
defer server.Close()
cfg := config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": server.URL}}
imp, err := New(ctx, cfg, &memoryStore{rows: map[string]Cache{}}, scopestore.New(), nil)
if err != nil {
t.Fatal(err)
}
imp := newImporter(t, config.MeshMapperScopesConfig{Enabled: true}, newMemoryStore("YOW"), server.URL, scopestore.New())
now := time.Now().UTC()
if err := imp.refresh(ctx, &imp.sources[0], now); err != nil {
t.Fatal(err)
}
if got := imp.sources[0].cache.NextAttempt.Sub(now); got != failureRetry {
t.Fatalf("next attempt after failure in %v, want %v", got, failureRetry)
if got := imp.sources[0].cache.NextAttempt.Sub(now); got != time.Hour {
t.Fatalf("next attempt after failure in %v, want 1h", got)
}
}
// fakeSites serves a CA zone list and YOW's scope catalogue.
type fakeSites struct {
*httptest.Server
list string
listCalls, scopeHits int
}
func newFakeSites(t *testing.T) *fakeSites {
f := &fakeSites{}
f.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/get_zones.php":
f.listCalls++
_, _ = w.Write([]byte(f.list))
case "/get_scopes.php":
f.scopeHits++
_, _ = w.Write(catalogue("yow"))
default:
t.Errorf("unexpected path %s", r.URL.Path)
}
}))
f.list = `{"country":"CA","zones":[{"code":"YOW","url":"` + f.URL + `/","has_boundary":true,"group":null}]}`
t.Cleanup(f.Close)
return f
}
func newDiscoveringImporter(t *testing.T, f *fakeSites, store *memoryStore, scopes *scopestore.ScopeStore) *Importer {
t.Helper()
dir := NewDirectory(newZoneListMemory())
dir.listURL = f.URL + "/get_zones.php"
imp, err := New(context.Background(), config.MeshMapperScopesConfig{Enabled: true}, store, dir, scopes, nil)
if err != nil {
t.Fatal(err)
}
imp.scopesURL = func(site string) (string, bool) { return site + "get_scopes.php", strings.HasPrefix(site, f.URL) }
return imp
}
func refreshN(t *testing.T, imp *Importer, n int) {
t.Helper()
for range n {
if err := imp.Refresh(context.Background()); err != nil {
t.Fatal(err)
}
}
}
func TestScopesDiscoverSourcesFromZoneList(t *testing.T) {
f := newFakeSites(t)
store := newMemoryStore()
scopes := scopestore.New()
imp := newDiscoveringImporter(t, f, store, scopes)
refreshN(t, imp, 2)
if f.listCalls != 0 || f.scopeHits != 0 {
t.Fatal("requested with no IATAs", f.listCalls, f.scopeHits)
}
store.iatas = []string{"YOW", "ZZZ"}
refreshN(t, imp, 3)
if f.listCalls != 1 || f.scopeHits != 1 || !reflect.DeepEqual(names(scopes), []string{"#yow"}) {
t.Fatal("source not discovered", f.listCalls, f.scopeHits, names(scopes))
}
if _, ok := store.rows[sourceKey{"YOW", f.URL + "/get_scopes.php"}]; !ok {
t.Fatal("catalogue not saved under the discovered URL", store.rows)
}
}
func TestScopesSkipUnlistedIATAs(t *testing.T) {
f := newFakeSites(t)
f.list = `{"country":"CA","zones":[]}`
imp := newDiscoveringImporter(t, f, newMemoryStore("YOW"), scopestore.New())
refreshN(t, imp, 3)
s := imp.sources[0]
if f.scopeHits != 0 || s.cache.LastError != "not listed" || s.cache.NextAttempt.Sub(time.Now()) < 59*time.Minute {
t.Fatalf("unlisted IATA requested or retried early: hits=%d %+v", f.scopeHits, s.cache)
}
}
internal/meshmapper/zones.go
+92 -136
View File
@@ -26,8 +26,12 @@ const (
MaxZoneList = 1 << 20
MaxBoundaryBody = 8 << 20 // outlines are never simplified
zoneListFresh = time.Hour // the API's own cache lifetime
zoneFailureRetry = time.Hour // fair use: no more than one poll an hour
// MeshMapper asks clients to allow 60-120s; a request given up on may still use the day's call.
requestTimeout = 60 * time.Second
refreshTimeout = requestTimeout + 30*time.Second // room for the DB write after a slow request
zoneListFresh = config.MinZonesRefresh // get_zones.php allows one request per country per 23.5h
zoneFailureRetry = config.MinZonesRefresh // failed requests count too
)
var zoneSite = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*\.meshmapper\.net$`)
@@ -42,9 +46,15 @@ type Boundary struct {
}
type ZoneStore interface {
ListKnownIATAs(ctx context.Context) ([]string, error)
PruneZoneBoundaries(ctx context.Context, keep []string) ([]string, error)
ListZoneBoundaries(ctx context.Context) ([]Boundary, error)
SaveZoneBoundary(ctx context.Context, b Boundary) error
ListRegionState(ctx context.Context) ([]RegionState, error)
ListIATADetails(ctx context.Context) ([]IATADetails, error)
UpsertIATADetails(ctx context.Context, iata, name string, lat, lng *float64) error
SaveImportedRegion(ctx context.Context, r RegionState) (bool, error)
PruneImportedRegions(ctx context.Context, keep []string) ([]string, error)
}
type zoneRegion struct {
@@ -52,37 +62,31 @@ type zoneRegion struct {
b Boundary
}
type zoneEntry struct {
url string
hasBoundary bool
}
type zoneList struct {
zones map[string]zoneEntry
etag string
fetchedAt, nextAttempt time.Time
}
// Zones is owned by one background task; listeners are wired before Restore.
type Zones struct {
store ZoneStore
enabled bool
iatas []string
interval time.Duration
client *http.Client
listURL string
boundsURL func(site string) (string, bool)
regions []*zoneRegion
lists map[string]*zoneList
retryAfter time.Time
onChange func(ctx context.Context, iata string)
onUpdate func(imported map[string]json.RawMessage)
store ZoneStore
enabled bool
seen map[string]bool
interval time.Duration
client *http.Client
dir *Directory
boundsURL func(site string) (string, bool)
regions []*zoneRegion
onChange func(ctx context.Context, iata string)
onUpdate func(imported map[string]json.RawMessage)
importGroups bool
groupsSynced bool
groupsVersion, groupsKnown int
onRegions func(ctx context.Context)
configured map[string]bool
onIATAs func(ctx context.Context)
}
func NewZones(cfg config.MeshMapperZonesConfig, iatas []string, store ZoneStore) *Zones {
return &Zones{store: store, enabled: cfg.Enabled, iatas: iatas, interval: cfg.Interval(), listURL: ZonesURL,
boundsURL: boundaryEndpoint, lists: map[string]*zoneList{}, client: &http.Client{
Timeout: 30 * time.Second,
func NewZones(cfg config.MeshMapperZonesConfig, store ZoneStore, dir *Directory) *Zones {
return &Zones{store: store, enabled: cfg.Enabled, importGroups: cfg.Enabled && cfg.ImportGroups, seen: map[string]bool{}, interval: cfg.Interval(), dir: dir,
boundsURL: boundaryEndpoint, client: &http.Client{
Timeout: requestTimeout,
CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
}}
}
@@ -93,12 +97,22 @@ func (z *Zones) OnChange(fn func(ctx context.Context, iata string)) { z.onChange
// OnUpdate receives every imported boundary whenever the set changes.
func (z *Zones) OnUpdate(fn func(imported map[string]json.RawMessage)) { z.onUpdate = fn }
// Restore prunes imports for unconfigured IATAs (all of them when disabled) and
// loads saved boundaries, without making HTTP requests.
// Restore prunes imports for unknown IATAs (all of them when disabled), and
// imported regions when group import is off, then loads saved boundaries,
// without making HTTP requests.
func (z *Zones) Restore(ctx context.Context) error {
if !z.importGroups {
if err := z.pruneGroups(ctx); err != nil {
return err
}
}
iatas, err := z.store.ListKnownIATAs(ctx)
if err != nil {
return fmt.Errorf("list IATAs for MeshMapper boundaries: %w", err)
}
keep := []string{}
if z.enabled {
keep = z.iatas
keep = iatas
}
pruned, err := z.store.PruneZoneBoundaries(ctx, keep)
if err != nil {
@@ -119,24 +133,32 @@ func (z *Zones) Restore(ctx context.Context) error {
for _, b := range saved {
byIATA[b.IATA] = b
}
for _, iata := range z.iatas {
z.track(iatas, byIATA)
z.publish()
return nil
}
// track adds a region for each IATA not seen before.
func (z *Zones) track(iatas []string, saved map[string]Boundary) {
for _, iata := range iatas {
if z.seen[iata] {
continue
}
z.seen[iata] = true
country := iatadb.CountryFor(iata)
if country == "" {
slog.Warn("MeshMapper boundary skipped: IATA has no known country", "component", "meshmapper.zones", "iata", iata)
continue
}
b, ok := byIATA[iata]
b, ok := saved[iata]
if !ok {
b = Boundary{IATA: iata}
}
if b.LastError == "HTTP 429" && b.NextAttempt.After(z.retryAfter) {
z.retryAfter = b.NextAttempt
}
z.regions = append(z.regions, &zoneRegion{country: country, b: b})
z.log(b, "restored")
if ok {
z.log(b, "restored")
}
}
z.publish()
return nil
}
// Refresh makes at most one request. The scheduler serializes calls every 15s.
@@ -147,77 +169,38 @@ func (z *Zones) Refresh(ctx context.Context) (err error) {
err = nil
}
}()
ctx, cancel := context.WithTimeout(ctx, time.Minute)
ctx, cancel := context.WithTimeout(ctx, refreshTimeout)
defer cancel()
now := time.Now().UTC()
if now.Before(z.retryAfter) {
return nil
// IATAs are created from traffic, so pick up new ones every tick.
iatas, err := z.store.ListKnownIATAs(ctx)
if err != nil {
return fmt.Errorf("list IATAs for MeshMapper boundaries: %w", err)
}
z.track(iatas, nil)
if err := z.syncDirectory(ctx, iatas); err != nil {
return err
}
for _, r := range z.regions {
if now.Before(r.b.NextAttempt) {
continue
}
list := z.lists[r.country]
if list == nil || now.Sub(list.fetchedAt) >= zoneListFresh {
if list != nil && now.Before(list.nextAttempt) {
continue
}
return z.fetchList(ctx, r.country, now)
zones, fetched, err := z.dir.List(ctx, r.country, now)
if err != nil || fetched {
return err
}
return z.refresh(ctx, r, list, now)
if zones == nil {
continue
}
return z.refresh(ctx, r, zones, now)
}
return nil
}
func (z *Zones) fetchList(ctx context.Context, country string, now time.Time) error {
list := z.lists[country]
if list == nil {
list = &zoneList{}
z.lists[country] = list
}
endpoint := z.listURL + "?country=" + url.QueryEscape(country)
etag := ""
if list.zones != nil {
etag = list.etag
}
status, body, header, err := z.get(ctx, endpoint, etag, MaxZoneList)
if err != nil {
return err
}
problem := ""
switch status {
case http.StatusOK:
zones, decodeErr := decodeZones(body, country)
if decodeErr != nil {
problem = "invalid response"
} else {
list.zones, list.etag = zones, header.Get("ETag")
}
case http.StatusNotModified:
if list.zones == nil {
problem = "304 without cached list"
}
default:
problem = z.statusProblem(status, header, now, &list.nextAttempt)
}
if problem == "" {
list.fetchedAt = now
} else if list.nextAttempt.Before(now.Add(zoneFailureRetry)) {
list.nextAttempt = now.Add(zoneFailureRetry)
}
level := slog.LevelInfo
if problem != "" {
level = slog.LevelWarn
}
slog.Log(ctx, level, "MeshMapper zone list checked", "component", "meshmapper.zones", "country", country,
"zones", len(list.zones), "last_error", problem)
return nil
}
func (z *Zones) refresh(ctx context.Context, r *zoneRegion, list *zoneList, now time.Time) error {
func (z *Zones) refresh(ctx context.Context, r *zoneRegion, zones map[string]zoneEntry, now time.Time) error {
update := Boundary{IATA: r.b.IATA, URL: r.b.URL, AttemptedAt: now, NextAttempt: now.Add(z.interval)}
var retryAt time.Time
entry, listed := list.zones[r.b.IATA]
entry, listed := zones[r.b.IATA]
endpoint, valid := "", false
if listed {
update.URL = entry.url
@@ -235,7 +218,7 @@ func (z *Zones) refresh(ctx context.Context, r *zoneRegion, list *zoneList, now
if r.b.Feature != nil {
etag = r.b.ETag
}
status, body, header, err := z.get(ctx, endpoint, etag, MaxBoundaryBody)
status, body, header, err := get(ctx, z.client, "Beacon-MeshMapper-Zones/1", endpoint, etag, MaxBoundaryBody)
if err != nil {
return err
}
@@ -255,7 +238,7 @@ func (z *Zones) refresh(ctx context.Context, r *zoneRegion, list *zoneList, now
update.LastError = "304 without cached boundary"
}
default:
update.LastError = z.statusProblem(status, header, now, &retryAt)
update.LastError = statusProblem(status, header, now, &retryAt)
}
if update.LastError == "" {
etag := header.Get("ETag")
@@ -296,17 +279,17 @@ func (z *Zones) refresh(ctx context.Context, r *zoneRegion, list *zoneList, now
}
// get returns transport errors only when ctx ended; other failures become status 0.
func (z *Zones) get(ctx context.Context, endpoint, etag string, limit int64) (int, []byte, http.Header, error) {
func get(ctx context.Context, client *http.Client, agent, endpoint, etag string, limit int64) (int, []byte, http.Header, error) {
request, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
if err != nil {
return 0, nil, nil, err
}
request.Header.Set("Accept", "application/json")
request.Header.Set("User-Agent", "Beacon-MeshMapper-Zones/1")
request.Header.Set("User-Agent", agent)
if etag != "" {
request.Header.Set("If-None-Match", etag)
}
response, err := z.client.Do(request)
response, err := client.Do(request)
if err != nil {
if ctx.Err() != nil {
return 0, nil, nil, ctx.Err()
@@ -324,8 +307,8 @@ func (z *Zones) get(ctx context.Context, endpoint, etag string, limit int64) (in
return response.StatusCode, body, response.Header, nil
}
// statusProblem describes a failed response and honours Retry-After on 429/503.
func (z *Zones) statusProblem(status int, header http.Header, now time.Time, retryAt *time.Time) string {
// statusProblem describes a failed response and reads Retry-After on 429/503.
func statusProblem(status int, header http.Header, now time.Time, retryAt *time.Time) string {
if status == 0 {
return "request failed" // don't persist untrusted error text
}
@@ -333,13 +316,6 @@ func (z *Zones) statusProblem(status int, header http.Header, now time.Time, ret
if seconds, err := strconv.ParseInt(header.Get("Retry-After"), 10, 64); err == nil && seconds > 0 && seconds <= int64((1<<63-1)/time.Second) {
*retryAt = now.Add(time.Duration(seconds) * time.Second)
}
// The rate limit is shared by both endpoints, so pause every request.
if status == http.StatusTooManyRequests {
z.retryAfter = *retryAt
if retryAt.IsZero() {
z.retryAfter = now.Add(time.Minute)
}
}
}
return fmt.Sprintf("HTTP %d", status)
}
@@ -372,36 +348,16 @@ func (z *Zones) log(b Boundary, action string) {
"source", b.URL, "imported", b.Feature != nil, "checked_at", b.CheckedAt, "next_attempt", b.NextAttempt, "last_error", b.LastError)
}
// boundaryEndpoint accepts only a published region site root.
func boundaryEndpoint(site string) (string, bool) {
func boundaryEndpoint(site string) (string, bool) { return siteEndpoint(site, "get_geojson.php") }
// siteEndpoint accepts only a published region site root.
func siteEndpoint(site, file string) (string, bool) {
u, err := url.Parse(site)
if err != nil || u.Scheme != "https" || u.User != nil || !zoneSite.MatchString(u.Host) ||
(u.Path != "" && u.Path != "/") || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" {
return "", false
}
return "https://" + u.Host + "/get_geojson.php", true
}
func decodeZones(body []byte, country string) (map[string]zoneEntry, error) {
var document struct {
Country string `json:"country"`
Zones *[]struct {
Code string `json:"code"`
URL string `json:"url"`
HasBoundary bool `json:"has_boundary"`
} `json:"zones"`
}
if err := json.Unmarshal(body, &document); err != nil {
return nil, err
}
if !strings.EqualFold(document.Country, country) || document.Zones == nil {
return nil, fmt.Errorf("invalid zone list")
}
zones := make(map[string]zoneEntry, len(*document.Zones))
for _, zone := range *document.Zones {
zones[strings.ToUpper(zone.Code)] = zoneEntry{url: zone.URL, hasBoundary: zone.HasBoundary}
}
return zones, nil
return "https://" + u.Host + "/" + file, true
}
// decodeBoundary returns nil when MeshMapper has no usable outline for the region.
internal/meshmapper/zones_test.go
+189 -26
View File
@@ -15,6 +15,8 @@ import (
"time"
"github.com/MeshCore-Beacon/beacon-server/internal/config"
"github.com/MeshCore-Beacon/beacon-server/internal/keystore"
"github.com/MeshCore-Beacon/beacon-server/internal/scopestore"
)
const square = `{"type":"Polygon","coordinates":[[[-76,45],[-75,45],[-75,46],[-76,46],[-76,45]]]}`
@@ -25,9 +27,73 @@ func boundaryBody(code, geometry string) string {
}
type zoneMemoryStore struct {
rows map[string]Boundary
pruned []string
fail bool
iatas []string
rows map[string]Boundary
pruned []string
fail bool
regions map[string]RegionState
lists *zoneListMemory
details map[string]IATADetails
writes int
}
func (s *zoneMemoryStore) ListIATADetails(context.Context) ([]IATADetails, error) {
var out []IATADetails
for _, iata := range s.iatas {
d := s.details[iata]
d.IATA = iata
out = append(out, d)
}
return out, nil
}
func (s *zoneMemoryStore) UpsertIATADetails(_ context.Context, iata, name string, lat, lng *float64) error {
if s.details == nil {
s.details = map[string]IATADetails{}
}
s.details[iata] = IATADetails{IATA: iata, Name: name, Lat: lat, Lng: lng}
s.writes++
return nil
}
func (s *zoneMemoryStore) ListRegionState(context.Context) ([]RegionState, error) {
var out []RegionState
for _, r := range s.regions {
out = append(out, r)
}
return out, nil
}
func (s *zoneMemoryStore) SaveImportedRegion(_ context.Context, r RegionState) (bool, error) {
if s.regions == nil {
s.regions = map[string]RegionState{}
}
if cur, ok := s.regions[r.Slug]; ok && !cur.Imported {
return false, nil
}
r.Imported = true
s.regions[r.Slug] = r
for _, m := range r.IATAs { // members become known IATAs, as AddIATAs does
if !slices.Contains(s.iatas, m) {
s.iatas = append(s.iatas, m)
}
}
return true, nil
}
func (s *zoneMemoryStore) PruneImportedRegions(_ context.Context, keep []string) ([]string, error) {
var removed []string
for slug, r := range s.regions {
if r.Imported && !slices.Contains(keep, slug) {
delete(s.regions, slug)
removed = append(removed, slug)
}
}
return removed, nil
}
func (s *zoneMemoryStore) ListKnownIATAs(context.Context) ([]string, error) {
return slices.Clone(s.iatas), nil
}
func (s *zoneMemoryStore) PruneZoneBoundaries(_ context.Context, keep []string) ([]string, error) {
@@ -65,6 +131,30 @@ func (s *zoneMemoryStore) SaveZoneBoundary(_ context.Context, b Boundary) error
return nil
}
type zoneListMemory struct{ rows map[string]ZoneList }
func newZoneListMemory() *zoneListMemory { return &zoneListMemory{rows: map[string]ZoneList{}} }
func (s *zoneListMemory) ListZoneLists(context.Context) ([]ZoneList, error) {
var out []ZoneList
for _, l := range s.rows {
out = append(out, l)
}
return out, nil
}
func (s *zoneListMemory) SaveZoneList(_ context.Context, l ZoneList) error {
old := s.rows[l.Country]
if l.Payload == nil {
l.Payload = old.Payload
}
if l.FetchedAt.IsZero() {
l.FetchedAt, l.ETag = old.FetchedAt, old.ETag
}
s.rows[l.Country] = l
return nil
}
type fakeMeshMapper struct {
*httptest.Server
list, boundary string
@@ -113,8 +203,18 @@ type zoneHarness struct {
func newZoneHarness(t *testing.T, f *fakeMeshMapper, store *zoneMemoryStore, enabled bool) *zoneHarness {
t.Helper()
h := &zoneHarness{store: store}
h.z = NewZones(config.MeshMapperZonesConfig{Enabled: enabled}, []string{"YOW"}, store)
h.z.listURL = f.URL + "/get_zones.php"
if store.iatas == nil {
store.iatas = []string{"YOW"}
}
if store.lists == nil {
store.lists = newZoneListMemory()
}
dir := NewDirectory(store.lists)
dir.listURL = f.URL + "/get_zones.php"
if err := dir.Restore(context.Background()); err != nil {
t.Fatal(err)
}
h.z = NewZones(config.MeshMapperZonesConfig{Enabled: enabled}, store, dir)
h.z.boundsURL = func(site string) (string, bool) { return site + "get_geojson.php", strings.HasPrefix(site, f.URL) }
h.z.OnChange(func(_ context.Context, iata string) { h.changed = append(h.changed, iata) })
h.z.OnUpdate(func(imported map[string]json.RawMessage) { h.imported = imported })
@@ -175,11 +275,11 @@ func TestZonesKeepLastGoodBoundary(t *testing.T) {
wantRetry time.Duration
}{
{"null geometry", boundaryBody("YOW", "null"), 200, "no boundary", 24 * time.Hour},
{"not found", "", 404, "HTTP 404", time.Hour},
{"unavailable", "", 503, "HTTP 503", time.Hour},
{"truncated", boundaryBody("YOW", square)[:80], 200, "invalid response", time.Hour},
{"other region", boundaryBody("YVR", square), 200, "invalid response", time.Hour},
{"oversized", `{"type":"FeatureCollection","features":[],"pad":"` + strings.Repeat("x", MaxBoundaryBody) + `"}`, 200, "invalid response", time.Hour},
{"not found", "", 404, "HTTP 404", 24 * time.Hour},
{"unavailable", "", 503, "HTTP 503", 24 * time.Hour},
{"truncated", boundaryBody("YOW", square)[:80], 200, "invalid response", 24 * time.Hour},
{"other region", boundaryBody("YVR", square), 200, "invalid response", 24 * time.Hour},
{"oversized", `{"type":"FeatureCollection","features":[],"pad":"` + strings.Repeat("x", MaxBoundaryBody) + `"}`, 200, "invalid response", 24 * time.Hour},
} {
t.Run(tc.name, func(t *testing.T) {
f := newFakeMeshMapper(t)
@@ -231,29 +331,44 @@ func TestZonesListFailureBacksOff(t *testing.T) {
t.Fatal("failed list retried early", f.listCalls, f.calls)
}
f.listStatus, f.list = 200, `{"country":"US","zones":[]}`
h.z.lists["CA"].nextAttempt, h.z.lists["CA"].fetchedAt = time.Time{}, time.Time{}
h.z.dir.lists["CA"].nextAttempt, h.z.dir.lists["CA"].fetchedAt = time.Time{}, time.Time{}
_ = h.z.Refresh(context.Background())
if h.z.lists["CA"].zones != nil {
if h.z.dir.lists["CA"].zones != nil {
t.Fatal("list for another country accepted")
}
}
func TestZonesRateLimitPausesEveryRequest(t *testing.T) {
func TestZonesRateLimitWaitsOnlyForThatRegion(t *testing.T) {
for retryAfter, want := range map[string]time.Duration{"120": 24 * time.Hour, "172800": 48 * time.Hour} {
f := newFakeMeshMapper(t)
h := newZoneHarness(t, f, &zoneMemoryStore{rows: map[string]Boundary{}}, true)
f.status, f.retryAfter = 429, retryAfter
b := h.tick(t)
if b.LastError != "HTTP 429" || b.NextAttempt.Sub(b.AttemptedAt) != want {
t.Fatalf("Retry-After %ss: next attempt in %v, want %v", retryAfter, b.NextAttempt.Sub(b.AttemptedAt), want)
}
restarted := newZoneHarness(t, f, h.store, true)
if err := restarted.z.Refresh(context.Background()); err != nil || f.calls != 1 {
t.Fatal("requested again before the rate limit allows", err, f.calls)
}
}
}
func TestZoneListSurvivesRestart(t *testing.T) {
f := newFakeMeshMapper(t)
h := newZoneHarness(t, f, &zoneMemoryStore{rows: map[string]Boundary{}}, true)
f.status, f.retryAfter = 429, "120"
b := h.tick(t)
if b.LastError != "HTTP 429" || b.NextAttempt.Sub(b.AttemptedAt) != time.Hour || h.z.retryAfter.Sub(b.AttemptedAt) != 2*time.Minute {
t.Fatalf("429 not honoured: %+v retryAfter=%v", b, h.z.retryAfter)
store := &zoneMemoryStore{rows: map[string]Boundary{}}
h := newZoneHarness(t, f, store, true)
h.tick(t)
if f.listCalls != 1 {
t.Fatal("list not fetched", f.listCalls)
}
h.z.regions[0].b.NextAttempt = time.Time{}
_ = h.z.Refresh(context.Background())
if f.calls != 1 {
t.Fatal("requested during Retry-After")
if got := h.z.dir.lists["CA"].nextAttempt.Sub(h.z.dir.lists["CA"].fetchedAt); got != 24*time.Hour {
t.Fatal("zone list refetched sooner than the rate limit allows", got)
}
restarted := newZoneHarness(t, f, h.store, true)
if !restarted.z.retryAfter.Equal(b.NextAttempt) {
t.Fatal("persisted 429 not restored")
restarted := newZoneHarness(t, f, store, true)
restarted.tick(t)
if f.listCalls != 1 || f.calls != 2 {
t.Fatal("restart refetched the zone list", f.listCalls, f.calls)
}
}
@@ -266,7 +381,7 @@ func TestZonesRestoreAndPrune(t *testing.T) {
restarted := newZoneHarness(t, f, store, true)
if !slices.Equal(store.pruned, []string{"OLD"}) || !slices.Equal(restarted.changed, []string{"OLD"}) {
t.Fatal("unconfigured IATA not pruned", store.pruned, restarted.changed)
t.Fatal("unknown IATA not pruned", store.pruned, restarted.changed)
}
if restarted.imported["YOW"] == nil || f.calls != 1 {
t.Fatal("saved boundary not restored without a request")
@@ -281,6 +396,29 @@ func TestZonesRestoreAndPrune(t *testing.T) {
}
}
func TestZonesPickUpNewIATAs(t *testing.T) {
f := newFakeMeshMapper(t)
store := &zoneMemoryStore{iatas: []string{}, rows: map[string]Boundary{}}
h := newZoneHarness(t, f, store, true)
for range 2 {
if err := h.z.Refresh(context.Background()); err != nil {
t.Fatal(err)
}
}
if f.listCalls != 0 || f.calls != 0 {
t.Fatal("requested with no IATAs", f.listCalls, f.calls)
}
store.iatas = []string{"YOW", "ZZZ"}
for range 2 {
if err := h.z.Refresh(context.Background()); err != nil {
t.Fatal(err)
}
}
if len(h.z.regions) != 1 || h.imported["YOW"] == nil || !slices.Equal(h.changed, []string{"YOW"}) {
t.Fatal("new IATA not imported", len(h.z.regions), h.changed)
}
}
func TestZonesPersistFailureWaitsForNextAttempt(t *testing.T) {
f := newFakeMeshMapper(t)
h := newZoneHarness(t, f, &zoneMemoryStore{rows: map[string]Boundary{}, fail: true}, true)
@@ -329,3 +467,28 @@ func TestDecodeBoundaryRejectsUnusableOutlines(t *testing.T) {
t.Fatal("case-insensitive code rejected", err)
}
}
func TestClientsAllowMeshMapperTimeout(t *testing.T) {
ctx := context.Background()
scopes, err := New(ctx, config.MeshMapperScopesConfig{}, nil, NewDirectory(nil), scopestore.New(), nil)
if err != nil {
t.Fatal(err)
}
channels, err := NewChannels(ctx, config.MeshMapperChannelsConfig{}, newChannelMemoryStore(), NewDirectory(nil), keystore.NewMapKeyStore(nil))
if err != nil {
t.Fatal(err)
}
for name, c := range map[string]*http.Client{
"directory": NewDirectory(nil).client,
"zones": NewZones(config.MeshMapperZonesConfig{}, nil, nil).client,
"scopes": scopes.client,
"channels": channels.client,
} {
if c.Timeout < 60*time.Second {
t.Errorf("%s client gives up after %v; MeshMapper asks for 60-120s", name, c.Timeout)
}
}
if refreshTimeout <= requestTimeout {
t.Fatal("a refresh must outlast its request")
}
}
internal/scopestore/scopestore.go
+30
View File
@@ -7,6 +7,7 @@ package scopestore
import (
"crypto/sha256"
"slices"
"strings"
"sync"
)
@@ -34,6 +35,8 @@ func FromName(name string) Entry {
type ScopeStore struct {
mu sync.RWMutex
entries []Entry
// IATA -> scope names, for region-filtered listings only; matching uses entries.
manual, catalogue map[string][]string
}
// New creates an empty ScopeStore.
@@ -54,3 +57,30 @@ func (s *ScopeStore) Entries() []Entry {
defer s.mu.RUnlock()
return s.entries
}
// SetManualMembers publishes configured region membership of manual scopes.
func (s *ScopeStore) SetManualMembers(byIATA map[string][]string) {
s.mu.Lock()
defer s.mu.Unlock()
s.manual = byIATA
}
// SetCatalogueMembers publishes imported catalogue membership, including names a manual scope overrides.
func (s *ScopeStore) SetCatalogueMembers(byIATA map[string][]string) {
s.mu.Lock()
defer s.mu.Unlock()
s.catalogue = byIATA
}
// NamesForIATAs returns the sorted names of scopes that belong to any of the given IATAs.
func (s *ScopeStore) NamesForIATAs(iatas []string) []string {
s.mu.RLock()
defer s.mu.RUnlock()
names := []string{}
for _, iata := range iatas {
names = append(names, s.manual[iata]...)
names = append(names, s.catalogue[iata]...)
}
slices.Sort(names)
return slices.Compact(names)
}
internal/scopestore/scopestore_test.go
+19
View File
@@ -4,6 +4,7 @@
package scopestore
import (
"slices"
"testing"
)
@@ -54,3 +55,21 @@ func TestEntriesRetainsImmutableSnapshotWithoutAllocation(t *testing.T) {
t.Fatal("replacement mutated the previous reader snapshot")
}
}
func TestNamesForIATAs(t *testing.T) {
s := New()
if got := s.NamesForIATAs([]string{"YOW"}); got == nil || len(got) != 0 {
t.Fatalf("empty store = %#v, want empty non-nil slice", got)
}
s.SetManualMembers(map[string][]string{"YOW": {"#ottawa", "#on"}, "YVR": {"#bc"}})
s.SetCatalogueMembers(map[string][]string{"YOW": {"#on", "#gatineau"}, "YUL": {"#qc"}})
got := s.NamesForIATAs([]string{"YOW", "YUL", "YYZ"})
want := []string{"#gatineau", "#on", "#ottawa", "#qc"}
if !slices.Equal(got, want) {
t.Fatalf("NamesForIATAs = %v, want %v", got, want)
}
s.SetCatalogueMembers(nil)
if got := s.NamesForIATAs([]string{"YUL"}); len(got) != 0 {
t.Fatalf("cleared catalogue still lists %v", got)
}
}