diff --git a/README.md b/README.md index dfcec34..855b9b8 100644 --- a/README.md +++ b/README.md @@ -192,6 +192,19 @@ even when the same bearer-authenticated request works with curl. ### Config file (`config.yaml`) ```yaml +# MeshMapper integrations (optional, no API key). Import transport scopes and +# IATA border outlines from MeshMapper instead of maintaining scopes: and +# iatas.*.borderFile by hand. Both cover every known IATA that MeshMapper lists; +# no sources to configure. See config.yaml.example for limits and behaviour. +#meshmapper: +# scopes: +# enabled: true +# zones: +# enabled: true +# import_groups: true # MeshMapper zone groups become regions +# channels: +# enabled: true # import each region's public hashtag channels + # Optional IATA overrides — auto-created on first packet arrival, # only needed if you want to customise display name or coordinates. # borderFile points to a GeoJSON Feature (Polygon or MultiPolygon) for the @@ -202,7 +215,7 @@ iatas: name: Vancouver International lat: 49.1967 lng: -123.1815 - borderFile: borders/yvr.geojson # optional + borderFile: borders/yvr.geojson # optional; or use meshmapper.zones # Super-regions grouping multiple IATAs. regions: @@ -218,9 +231,12 @@ regions: channel_keys: # Hashtag channels: Beacon derives the PSK from the tag name automatically. # secret = SHA256("#tag")[:16], channel_hash = SHA256(secret)[0] - # Tag names should be provided without the # prefix. + # Tag names should be provided without the # prefix. Plain names show under + # every region; {name, region: } limits a channel to one region. hashtags: - meshcore + - name: vancouver-mesh + region: western-canada # Explicit keys: channel hash (hex) and key (hex), with optional display name. # The public MeshCore channel key is included in config.yaml.example. @@ -231,9 +247,14 @@ channel_keys: # Regional transport scopes for matching TRANSPORT_FLOOD packets. # Plain names have # prepended automatically (e.g. "bc" → "#bc"). +# region (required) is a configured region slug; region-filtered scope lists +# and scope stats show the scope under that region's IATAs. Matching stays global. +# Optional when meshmapper.scopes covers your regions. scopes: - name: bc + region: western-canada - name: "#west" + region: western-canada # Observer telemetry storage settings. telemetry: @@ -500,7 +521,7 @@ Not yet implemented — see the Authentication section above. | `GET` | `/routes` | List known routes (all hops high confidence) | | `GET` | `/routes/search` | Search routes by source and destination hash | | `GET` | `/routes/cross` | Search for routes crossing IATA boundaries | -| `GET` | `/scopes` | List transport scopes | +| `GET` | `/scopes` | List transport scope names; IATA/region filters use configured regions and MeshMapper catalogues | | `GET` | `/scopes/{name}` | Get scope detail | | `GET` | `/stats/observations` | Hourly observation time series (last 7 days by default) | | `GET` | `/stats/overview` | Network overview stats | diff --git a/cmd/beacon/main.go b/cmd/beacon/main.go index daafc06..1b728cf 100644 --- a/cmd/beacon/main.go +++ b/cmd/beacon/main.go @@ -45,7 +45,7 @@ import ( var version = "dev" // @title MeshCore Beacon API -// @version 1.6.0 +// @version 2.0.0 // @description MeshCore network observation backend. Ingests LoRa packets from MQTT brokers, stores in PostgreSQL, and streams live events via WebSocket. // @description REST requests share a configurable per-client rate limit (default 300/minute and a 300-request one-second burst cap). Exceeded limits return HTTP 429 with error.code=rate_limited and a Retry-After header in seconds. CORS preflights and WebSocket upgrades do not consume this API budget. // @description WebSocket upgrade attempts at /ws have a configurable per-client limit (default 10/minute, including failed handshakes). Rate exhaustion returns HTTP 429 with Retry-After before upgrade. An accepted socket exceeding the concurrent cap closes with code 1013 before hello; established connections remain open. @@ -196,11 +196,22 @@ func main() { os.Exit(1) } scopes.Load(scopeEntries) + scopes.SetManualMembers(cfg.ManualScopeMembers()) slog.Info(fmt.Sprintf("loaded %d transport scopes", len(scopeEntries)), "component", "startup") + directory := meshmapper.NewDirectory(store) + if cfg.MeshMapper.Scopes.Enabled || cfg.MeshMapper.Zones.Enabled || cfg.MeshMapper.Channels.Enabled { + restoreCtx, cancelRestore := context.WithTimeout(ctx, 10*time.Second) + err = directory.Restore(restoreCtx) + cancelRestore() + if err != nil { + slog.Error("failed to restore MeshMapper zone lists", "component", "startup", "error", err) + os.Exit(1) + } + } var scopeImporter *meshmapper.Importer if cfg.MeshMapper.Scopes.Enabled { restoreCtx, cancelRestore := context.WithTimeout(ctx, 10*time.Second) - scopeImporter, err = meshmapper.New(restoreCtx, cfg.MeshMapper.Scopes, store, scopes, scopeEntries) + scopeImporter, err = meshmapper.New(restoreCtx, cfg.MeshMapper.Scopes, store, directory, scopes, scopeEntries) cancelRestore() if err != nil { slog.Error("failed to restore MeshMapper scope catalogues", "component", "startup", "error", err) @@ -209,9 +220,18 @@ func main() { } // Always restored, so disabling the import prunes it and file borders return. - zones := meshmapper.NewZones(cfg.MeshMapper.Zones, cfg.RegionIATAs(), store) + zones := meshmapper.NewZones(cfg.MeshMapper.Zones, store, directory) + var detailed []string + for iata, d := range cfg.IATAs { + if d.Name != "" || d.Lat != nil || d.Lng != nil { + detailed = append(detailed, iata) + } + } + zones.SetConfiguredIATAs(detailed) if cr, ok := reader.(*cache.CachedReader); ok { zones.OnChange(cr.InvalidateIATABorder) + zones.OnRegionsChange(cr.InvalidateRegions) + zones.OnIATAsChange(cr.InvalidateIATAs) } if localBorders != nil { zones.OnUpdate(func(imported map[string]json.RawMessage) { @@ -238,7 +258,8 @@ func main() { entries := make(map[string][]keystore.Entry) // Hashtag-derived channels: secret = SHA256("#tag")[:16], hash = SHA256(secret)[0] - for _, tag := range cfg.ChannelKeys.Hashtags { + for _, hashtag := range cfg.ChannelKeys.Hashtags { + tag := hashtag.Name secret, channelHash, fingerprint := keystore.DeriveHashtagKey(tag) hashHex := fmt.Sprintf("%02x", channelHash) entry := keystore.Entry{ @@ -273,6 +294,27 @@ func main() { keys := keystore.NewMapKeyStore(entries) + // Restored before the boot backfill so saved MeshMapper keys decrypt history too. + restoreCtx, cancelRestore = context.WithTimeout(ctx, 10*time.Second) + channelImporter, err := meshmapper.NewChannels(restoreCtx, cfg.MeshMapper.Channels, store, directory, keys) + cancelRestore() + if err != nil { + slog.Error("failed to restore MeshMapper channels", "component", "startup", "error", err) + os.Exit(1) + } + channelImporter.OnNewKeys(func(hashes [][]byte) { + // Off the refresh tick: a backfill can outlast it, and history isn't urgent. + go func() { + backfillCtx, cancel := context.WithTimeout(ctx, 10*time.Minute) + defer cancel() + if n, err := ingest.BackfillChannelHashes(backfillCtx, store, keys, hashes); err != nil { + slog.Error("MeshMapper channel backfill failed", "component", "meshmapper.channels", "error", err) + } else if n > 0 { + slog.Info(fmt.Sprintf("backfilled %d channel message(s) for imported keys", n), "component", "meshmapper.channels") + } + }() + }) + // ── Backfill channel messages ──────────────────────────────────────────── // Packets whose channel key wasn't yet configured at ingest time were stored as // hash-only channels and never decrypted. Retry them now against the keystore we just @@ -354,6 +396,9 @@ func main() { if cfg.MeshMapper.Zones.Enabled { tasks = append(tasks, background.Task{Name: "meshmapper.zones", Interval: meshmapper.PollInterval, Run: zones.Refresh}) } + if cfg.MeshMapper.Channels.Enabled { + tasks = append(tasks, background.Task{Name: "meshmapper.channels", Interval: meshmapper.PollInterval, Run: channelImporter.Refresh}) + } profiles := configureProfiling(ctx, pool) defer profiles.Stop() for i := range tasks { @@ -372,6 +417,7 @@ func main() { MaxConnectsPerMinute: resolved.MaxConnectsPerMinute, WSAllowedOrigins: cfg.WebSocket.AllowedOrigins, CORS: cfg.CORS, Server: cfg.Server, Auth: cfg.Auth, RateLimit: resolved.RateLimit, + Scopes: scopes, AdminRoutes: map[string]http.Handler{ "/accounts": handlers.AccountsRouter(store), "/backup": handlers.BackupRouter(backupOpts, ctx), diff --git a/config.yaml.example b/config.yaml.example index 0c6eb6e..ae3a35a 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -41,13 +41,72 @@ ratelimit: # A 429 returns error.code=rate_limited and Retry-After (1 or 60 seconds). burst: 300 +# MeshMapper integrations (optional, no API key). Import transport scopes and +# IATA border outlines from MeshMapper instead of maintaining scopes: and +# iatas.*.borderFile by hand. +# +# Scope-name discovery. Manual scopes remain authoritative. +# Every known IATA, including ones auto-created from traffic, is looked up in its +# country's zone list (shared with zones below) and its site's get_scopes.php +# catalogue is imported; IATAs MeshMapper doesn't list are skipped. No sources +# to configure; a legacy sources: list is ignored with a warning. +# Catalogues covering more than one IATA are rejected. +# 64 names per catalogue, 64 KiB per response; no page scraping. +# Catalogues and ETags persist in PostgreSQL. Failed refreshes retain known names. +# Refresh success/failure/freshness is logged under component=meshmapper.scopes. +# Turning scopes off deactivates imported matching keys on restart; +# historical scope identities and recorded evidence are retained. +# Region-filtered scope lists and scope stats show imported names only under the +# IATAs whose catalogue lists them; IATAs without a catalogue show manual scopes only. +#meshmapper: +# scopes: +# enabled: false +# refresh_interval: 1h # 1h-24h (MeshMapper allows one get_scopes.php per region per 55m); one source per 15s tick +# +# Region boundaries (Zones API). Every known IATA, including ones auto-created +# from traffic, is looked up in its country's zone list, then its get_geojson.php +# outline is imported; new IATAs are picked up while running. An imported boundary +# overrides that IATA's borderFile on the border map and for nodes.mark_foreign. +# A missing or failed boundary keeps the last good import, or the borderFile when +# there is none. Disabling drops every import on restart. Logged under +# component=meshmapper.zones. +# Each country's zone list is fetched at most once a day (MeshMapper allows one +# get_zones.php per country per 23.5h) and saved, so restarts don't refetch it. +# Failed or rate-limited requests also wait out the limit, or Retry-After if longer. +# Listed IATAs also take their display name and approximate location from the +# zone list, unless iatas: below sets a name or lat/lng for them. +# import_groups turns MeshMapper zone groups (e.g. ONQC: YOW, YQB, YUL) into +# regions, from the zone lists already fetched. A group is imported when at least +# one member is a known IATA, with its full member list (members Beacon hasn't +# heard yet are added as IATAs); slug = lowercased group code, centered on the +# members' average location, listed after the regions below. A region below with +# the same slug wins. +# Imported regions are removed when the group disappears or stops qualifying, +# and all of them when import_groups is turned off. Requires zones.enabled. +# zones: +# enabled: false +# import_groups: false +# refresh_interval: 24h # 24h-168h (one get_geojson.php per region per 23.5h); one request per 15s tick, ETag-conditional +# +# Public hashtag channels (get_channels.php). Every known IATA that MeshMapper +# lists is looked up on its site, the same way as scopes. Each key and hash is +# checked against the name before use; a list with any mismatch is rejected and +# the last good one kept. Imported channels decrypt like configured ones (history +# is backfilled when a key first appears) and are listed under that IATA when +# /channels is filtered by region. Saved in PostgreSQL across restarts; failures +# wait out the 23.5h limit. Turning it off removes the region listing on restart. +# Logged under component=meshmapper.channels. +# channels: +# enabled: false +# refresh_interval: 24h # 24h-168h (one get_channels.php per region per 23.5h) + iatas: YVR: name: Vancouver International lat: 49.1967 lng: -123.1815 # borderFile: borders/yvr.geojson # optional GeoJSON Feature (Polygon/MultiPolygon) - # # (meshmapper.zones overrides it when enabled) + # # (or use meshmapper.zones above) # # for the region border map; path is relative to # # this file's directory; validated at boot. YYJ: @@ -71,6 +130,8 @@ iatas: lat: 43.6777 lng: -79.6248 +# Hand-written regions; optional with meshmapper.zones.import_groups. Seeding +# sets each region's members to exactly this list. regions: - slug: western-canada name: Western Canada @@ -93,8 +154,12 @@ channel_keys: # secret = SHA256("#tag")[:16], channel_hash = SHA256(secret)[0] # Hashtag names should be provided without the # prefix. # Beacon prepends # automatically before deriving the key. + # A plain name is listed under every region; add region: (a configured + # region) to list it only there when /channels is filtered by region. hashtags: - meshcore + # - name: ottawa-mesh + # region: eastern-canada # Explicit keys: provide the channel hash (hex) and key (hex) directly. # An optional name can be set for display purposes. @@ -102,12 +167,18 @@ channel_keys: "11": key: "8b3387e9c5cdea6ac9e5edbaa115cd72" name: "Public" + # region: eastern-canada # optional, as for hashtags # Regional transport scopes for matching TRANSPORT_FLOOD packets. # Plain names have # prepended automatically (e.g. "bc" → "#bc"). +# region (required) is a configured region slug; region-filtered scope lists +# and scope stats show the scope under that region's IATAs. Matching stays global. +# Optional when meshmapper.scopes covers your regions. scopes: - name: bc + region: western-canada - name: "#west" + region: western-canada telemetry: retention: 672h # 4 weeks @@ -216,31 +287,6 @@ cache: # allow_countries: [CA, US] # allow_continents: [NA] -# Optional public MeshMapper scope-name discovery. Manual scopes remain authoritative. -# No API key is needed. Sources must belong to a configured region's IATAs. -# Group endpoints are not supported: their names cannot be attributed to each IATA. -# At most 16 sources, 64 names per source, 64 KiB per response; no page scraping. -# Catalogues and ETags persist in PostgreSQL. Failed refreshes retain known names. -# Refresh success/failure/freshness is logged under component=meshmapper.scopes. -# Removing/turning off a source deactivates its imported matching keys on restart; -# historical scope identities and recorded evidence are retained. -#meshmapper: -# scopes: -# enabled: false -# refresh_interval: 1h # 5m-24h; one source checked per 15s tick -# sources: -# YOW: https://yow.meshmapper.net/get_scopes.php -# -# Optional MeshMapper region boundaries (Zones API). Every IATA in a configured -# region is looked up in its country's zone list, then its get_geojson.php outline -# is imported. An imported boundary overrides that IATA's borderFile on the border -# map and for nodes.mark_foreign. A missing or failed boundary keeps the last good -# import, or the borderFile when there is none. Disabling, or removing an IATA from -# every region, drops its import on restart. Logged under component=meshmapper.zones. -# zones: -# enabled: false -# refresh_interval: 24h # 1h-168h; one request per 15s tick, ETag-conditional - # Background task intervals. # Shorter intervals are useful during initial deployment to confirm data is # flowing. Back off to 1h or more once stable. diff --git a/db/channel_cursor_integration_test.go b/db/channel_cursor_integration_test.go index d748f2a..a25eb9f 100644 --- a/db/channel_cursor_integration_test.go +++ b/db/channel_cursor_integration_test.go @@ -57,13 +57,14 @@ func TestChannelCursorPostgres(t *testing.T) { } } applyBaseline(t, ctx, tx) - exec(`INSERT INTO channels(id,channel_hash,key_fingerprint,last_seen) VALUES - (1,'\xaa','\x01','2026-09-08 12:00:00+00'),(2,'\xaa','\x02','2026-09-08 12:00:00+00'), - (3,'\xaa','\x03','2026-09-08 12:00:00+00'),(4,'\xbb','\x04','2026-09-08 11:59:59+00'), - (5,'\xcc','\x05','2026-09-08 12:00:00.000321+00'), - (6,'\xcc','\x06','2026-09-08 12:00:00.000111+00'),(7,'\xcc','\x07','2026-09-08 12:00:00.000111+00'); -INSERT INTO channel_iatas(channel_hash,iata,last_heard) VALUES - ('\xaa','YOW',now()),('\xbb','YOW',now()),('\xcc','YYZ',now());`) + // NULL key_known is reported as false, so keyKnown=false must include it. + exec(`INSERT INTO channels(id,channel_hash,key_fingerprint,last_seen,key_known) VALUES + (1,'\xaa','\x01','2026-09-08 12:00:00+00',true),(2,'\xaa','\x02','2026-09-08 12:00:00+00',false), + (3,'\xaa','\x03','2026-09-08 12:00:00+00',NULL),(4,'\xbb','\x04','2026-09-08 11:59:59+00',true), + (5,'\xcc','\x05','2026-09-08 12:00:00.000321+00',true), + (6,'\xcc','\x06','2026-09-08 12:00:00.000111+00',false),(7,'\xcc','\x07','2026-09-08 12:00:00.000111+00',true); +INSERT INTO meshmapper_channel_members(iata,key_fingerprint) VALUES + ('YOW','\x01'),('YOW','\x02'),('YOW','\x03'),('YOW','\x04'),('YYZ','\x05'),('YYZ','\x06'),('YYZ','\x07');`) queries := &channelCursorQueries{DBTX: tx} router := handlers.ChannelsRouter(&Store{q: sqlc.New(queries)}) // Decode the wire contract so the same regression runs against the old server. @@ -97,6 +98,11 @@ INSERT INTO channel_iatas(channel_hash,iata,last_heard) VALUES {"hash and region", "hash=cc&iata=yyz", []int{5, 7, 6}}, {"no regional match", "hash=aa&iata=yyz", nil}, {"missing region", "iata=ZZZ", nil}, + {"key known", "keyKnown=true", []int{5, 7, 1, 4}}, + {"key unknown", "keyKnown=false", []int{6, 3, 2}}, + {"key known in region", "keyKnown=true&iata=yow", []int{1, 4}}, + {"key unknown in region", "keyKnown=false&iata=yow", []int{3, 2}}, + {"key unknown hash", "keyKnown=false&hash=cc", []int{6}}, } { t.Run(tc.name, func(t *testing.T) { params, _ := url.ParseQuery(tc.query) diff --git a/db/channels.go b/db/channels.go index 9082e5a..3e07032 100644 --- a/db/channels.go +++ b/db/channels.go @@ -65,6 +65,31 @@ func (s *Store) ListUndecryptedGroupTextPackets(ctx context.Context) ([]ingest.U return packets, nil } +// ListUndecryptedGroupTextPacketsByHash limits the boot backfill to channels that just gained a key. +func (s *Store) ListUndecryptedGroupTextPacketsByHash(ctx context.Context, hashes [][]byte) ([]ingest.UndecryptedPacket, error) { + rows, err := s.q.ListUndecryptedGroupTextPacketsByHash(ctx, hashes) + if err != nil { + return nil, err + } + packets := make([]ingest.UndecryptedPacket, 0, len(rows)) + for _, v := range rows { + packets = append(packets, ingest.UndecryptedPacket{PacketHash: v.PacketHash, RawPayload: v.RawPayload}) + } + return packets, nil +} + +// SetChannelConfigScopes replaces every configured channel's region placement; +// an empty region means Beacon-wide. +func (s *Store) SetChannelConfigScopes(ctx context.Context, fingerprints [][]byte, regions []string) error { + if err := s.q.DeleteChannelConfigScopes(ctx); err != nil { + return err + } + if len(fingerprints) == 0 { + return nil + } + return s.q.AddChannelConfigScopes(ctx, sqlc.AddChannelConfigScopesParams{Fingerprints: fingerprints, Regions: regions}) +} + func (s *Store) UpsertChannelIATA(ctx context.Context, channelHash []byte, iata string, heardAt time.Time) error { return s.q.UpsertChannelIATA(ctx, sqlc.UpsertChannelIATAParams{ ChannelHash: channelHash, @@ -77,7 +102,7 @@ func (s *Store) DeleteOldChannelIATAs(ctx context.Context, cutoff time.Time) err return s.q.DeleteOldChannelIATAs(ctx, pgtype.Timestamptz{Time: cutoff, Valid: true}) } -func (s *Store) ListChannels(ctx context.Context, limit int32, hash []byte, iatas []string, cursor int64, pageCursor *api.ChannelCursor) (api.ChannelPage, error) { +func (s *Store) ListChannels(ctx context.Context, limit int32, hash []byte, iatas []string, keyKnown *bool, cursor int64, pageCursor *api.ChannelCursor) (api.ChannelPage, error) { var cursorTS pgtype.Timestamptz if cursor > 0 { cursorTS = pgtype.Timestamptz{Time: time.UnixMilli(cursor), Valid: true} @@ -86,12 +111,12 @@ func (s *Store) ListChannels(ctx context.Context, limit int32, hash []byte, iata var err error if pageCursor != nil { rows, err = s.q.ListChannelsAfter(ctx, sqlc.ListChannelsAfterParams{ - ChannelHash: hash, Iatas: iatas, PageLimit: limit + 1, + ChannelHash: hash, Iatas: iatas, KeyKnown: keyKnown, PageLimit: limit + 1, CursorTs: pgtype.Timestamptz{Time: pageCursor.LastSeen, Valid: true}, CursorID: pageCursor.ID, }) } else { rows, err = s.q.ListChannels(ctx, sqlc.ListChannelsParams{ - ChannelHash: hash, Iatas: iatas, CursorTs: cursorTS, PageLimit: limit + 1, + ChannelHash: hash, Iatas: iatas, KeyKnown: keyKnown, CursorTs: cursorTS, PageLimit: limit + 1, }) } if err != nil { diff --git a/db/channels_test.go b/db/channels_test.go index 0f51c63..9ba475c 100644 --- a/db/channels_test.go +++ b/db/channels_test.go @@ -11,6 +11,7 @@ import ( sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" mockdb "github.com/MeshCore-Beacon/beacon-server/db/sqlc/mock" + "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/jackc/pgx/v5/pgtype" "go.uber.org/mock/gomock" ) @@ -29,7 +30,7 @@ func TestListChannels_Empty(t *testing.T) { Return([]sqlc.Channel{}, nil) store := &Store{q: mock} - page, err := store.ListChannels(context.Background(), 10, nil, nil, 0, nil) + page, err := store.ListChannels(context.Background(), 10, nil, nil, nil, 0, nil) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -71,7 +72,7 @@ func TestListChannels_Pagination(t *testing.T) { Return(rows, nil) store := &Store{q: mock} - page, err := store.ListChannels(context.Background(), 2, nil, nil, 0, nil) + page, err := store.ListChannels(context.Background(), 2, nil, nil, nil, 0, nil) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -95,7 +96,7 @@ func TestListChannels_DBError(t *testing.T) { Return(nil, errors.New("db error")) store := &Store{q: mock} - _, err := store.ListChannels(context.Background(), 10, nil, nil, 0, nil) + _, err := store.ListChannels(context.Background(), 10, nil, nil, nil, 0, nil) if err == nil { t.Fatal("expected error, got nil") } @@ -115,12 +116,37 @@ func TestListChannels_IATAFilter(t *testing.T) { Return([]sqlc.Channel{}, nil) store := &Store{q: mock} - _, err := store.ListChannels(context.Background(), 10, nil, []string{"YOW", "YYZ"}, 0, nil) + _, err := store.ListChannels(context.Background(), 10, nil, []string{"YOW", "YYZ"}, nil, 0, nil) if err != nil { t.Fatalf("unexpected error: %v", err) } } +func TestListChannels_KeyKnownFilter(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + keyKnown := false + at := time.UnixMilli(1700000000000) + + mock.EXPECT(). + ListChannels(gomock.Any(), sqlc.ListChannelsParams{KeyKnown: &keyKnown, PageLimit: 11}). + Return([]sqlc.Channel{}, nil) + mock.EXPECT(). + ListChannelsAfter(gomock.Any(), sqlc.ListChannelsAfterParams{ + KeyKnown: &keyKnown, PageLimit: 11, + CursorTs: pgtype.Timestamptz{Time: at, Valid: true}, CursorID: 7, + }). + Return([]sqlc.Channel{}, nil) + + store := &Store{q: mock} + if _, err := store.ListChannels(context.Background(), 10, nil, nil, &keyKnown, 0, nil); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if _, err := store.ListChannels(context.Background(), 10, nil, nil, &keyKnown, 0, &api.ChannelCursor{LastSeen: at, ID: 7}); err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + func TestGetChannel_Basic(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) diff --git a/db/config.go b/db/config.go index ac15b6f..3836a33 100644 --- a/db/config.go +++ b/db/config.go @@ -166,9 +166,13 @@ func (s *Store) GetRegionBySlug(ctx context.Context, slug string) (*api.Region, return &result, nil } -func (s *Store) UpsertRegionIATA(ctx context.Context, regionID int32, iata string) error { - return s.q.UpsertRegionIATA(ctx, sqlc.UpsertRegionIATAParams{ - RegionID: regionID, - Iata: iata, - }) +// SetRegionIATAs makes iatas the region's exact member list. +func (s *Store) SetRegionIATAs(ctx context.Context, regionID int32, iatas []string) error { + if iatas == nil { + iatas = []string{} + } + if err := s.q.DeleteRegionIATAsNotIn(ctx, sqlc.DeleteRegionIATAsNotInParams{RegionID: regionID, Keep: iatas}); err != nil { + return err + } + return s.q.AddRegionIATAs(ctx, sqlc.AddRegionIATAsParams{RegionID: regionID, Iatas: iatas}) } diff --git a/db/meshmapper.go b/db/meshmapper.go index 0e2bc28..8ce90d1 100644 --- a/db/meshmapper.go +++ b/db/meshmapper.go @@ -29,6 +29,23 @@ func (s *Store) GetScopeCatalogue(ctx context.Context, iata, url string) (*meshm return cache, nil } +func (s *Store) ListScopeCatalogues(ctx context.Context) ([]meshmapper.Catalogue, error) { + rows, err := s.q.ListScopeCatalogues(ctx) + if err != nil { + return nil, err + } + out := make([]meshmapper.Catalogue, 0, len(rows)) + for _, row := range rows { + c := meshmapper.Catalogue{IATA: row.Iata, URL: row.Url, Cache: meshmapper.Cache{Payload: row.Payload, CheckedAt: row.CheckedAt.Time, + AttemptedAt: row.AttemptedAt.Time, NextAttempt: row.NextAttempt.Time, LastError: row.LastError}} + if row.Etag != nil { + c.ETag = *row.Etag + } + out = append(out, c) + } + return out, nil +} + func (s *Store) SaveScopeCatalogue(ctx context.Context, iata, url string, cache meshmapper.Cache, entries []scopestore.Entry) error { params := sqlc.SaveScopeCatalogueParams{ Iata: iata, Url: url, Payload: cache.Payload, LastError: cache.LastError, @@ -48,6 +65,18 @@ func (s *Store) SaveScopeCatalogue(ctx context.Context, iata, url string, cache return s.q.SaveScopeCatalogue(ctx, params) } +func (s *Store) ListKnownIATAs(ctx context.Context) ([]string, error) { + rows, err := s.q.ListIATAs(ctx) + if err != nil { + return nil, err + } + iatas := make([]string, 0, len(rows)) + for _, row := range rows { + iatas = append(iatas, row.Iata) + } + return iatas, nil +} + func (s *Store) PruneZoneBoundaries(ctx context.Context, keep []string) ([]string, error) { if keep == nil { keep = []string{} @@ -84,3 +113,139 @@ func (s *Store) SaveZoneBoundary(ctx context.Context, b meshmapper.Boundary) err } return s.q.SaveZoneBoundary(ctx, params) } + +func (s *Store) ListZoneLists(ctx context.Context) ([]meshmapper.ZoneList, error) { + rows, err := s.q.ListZoneLists(ctx) + if err != nil { + return nil, err + } + out := make([]meshmapper.ZoneList, 0, len(rows)) + for _, row := range rows { + l := meshmapper.ZoneList{Country: row.Country, Payload: row.Payload, FetchedAt: row.FetchedAt.Time, + AttemptedAt: row.AttemptedAt.Time, NextAttempt: row.NextAttempt.Time, LastError: row.LastError} + if row.Etag != nil { + l.ETag = *row.Etag + } + out = append(out, l) + } + return out, nil +} + +func (s *Store) SaveZoneList(ctx context.Context, l meshmapper.ZoneList) error { + params := sqlc.SaveZoneListParams{ + Country: l.Country, Payload: l.Payload, LastError: l.LastError, + FetchedAt: pgtype.Timestamptz{Time: l.FetchedAt, Valid: !l.FetchedAt.IsZero()}, + AttemptedAt: pgtype.Timestamptz{Time: l.AttemptedAt, Valid: true}, + NextAttempt: pgtype.Timestamptz{Time: l.NextAttempt, Valid: true}, + } + if !l.FetchedAt.IsZero() { + params.Etag = &l.ETag + } + return s.q.SaveZoneList(ctx, params) +} + +func (s *Store) ListRegionState(ctx context.Context) ([]meshmapper.RegionState, error) { + rows, err := s.q.ListRegionState(ctx) + if err != nil { + return nil, err + } + out := make([]meshmapper.RegionState, 0, len(rows)) + for _, row := range rows { + out = append(out, meshmapper.RegionState{Slug: row.Slug, Name: row.Name, DisplayOrder: int(row.DisplayOrder), + Imported: row.Imported, IATAs: row.Iatas, CenterLat: row.CenterLat, CenterLng: row.CenterLng}) + } + return out, nil +} + +// SaveImportedRegion reports false when a hand-written region owns the slug. +func (s *Store) SaveImportedRegion(ctx context.Context, r meshmapper.RegionState) (bool, error) { + order := int32(r.DisplayOrder) + id, err := s.q.UpsertImportedRegion(ctx, sqlc.UpsertImportedRegionParams{Slug: r.Slug, Name: r.Name, DisplayOrder: &order, + CenterLat: r.CenterLat, CenterLng: r.CenterLng}) + if errors.Is(err, pgx.ErrNoRows) { + return false, nil + } + if err != nil { + return false, err + } + // region_iatas references iata_codes, so members Beacon hasn't heard yet are created first, as Seed does. + if err := s.q.AddIATAs(ctx, r.IATAs); err != nil { + return false, err + } + return true, s.SetRegionIATAs(ctx, id, r.IATAs) +} + +func (s *Store) PruneImportedRegions(ctx context.Context, keep []string) ([]string, error) { + if keep == nil { + keep = []string{} + } + return s.q.PruneImportedRegions(ctx, keep) +} + +func (s *Store) ListChannelCatalogues(ctx context.Context) ([]meshmapper.Catalogue, error) { + rows, err := s.q.ListChannelCatalogues(ctx) + if err != nil { + return nil, err + } + out := make([]meshmapper.Catalogue, 0, len(rows)) + for _, row := range rows { + c := meshmapper.Catalogue{IATA: row.Iata, URL: row.Url, Cache: meshmapper.Cache{Payload: row.Payload, CheckedAt: row.CheckedAt.Time, + AttemptedAt: row.AttemptedAt.Time, NextAttempt: row.NextAttempt.Time, LastError: row.LastError}} + if row.Etag != nil { + c.ETag = *row.Etag + } + out = append(out, c) + } + return out, nil +} + +// SaveChannelCatalogue records a fetch; a new payload also replaces the IATA's +// channel members, which are written first so a failure never outlives its snapshot. +func (s *Store) SaveChannelCatalogue(ctx context.Context, iata, url string, cache meshmapper.Cache, fingerprints [][]byte) error { + if cache.Payload != nil { + if err := s.SetChannelMembers(ctx, iata, fingerprints); err != nil { + return err + } + } + params := sqlc.SaveChannelCatalogueParams{ + Iata: iata, Url: url, Payload: cache.Payload, LastError: cache.LastError, + CheckedAt: pgtype.Timestamptz{Time: cache.CheckedAt, Valid: !cache.CheckedAt.IsZero()}, + AttemptedAt: pgtype.Timestamptz{Time: cache.AttemptedAt, Valid: true}, + NextAttempt: pgtype.Timestamptz{Time: cache.NextAttempt, Valid: true}, + } + if !cache.CheckedAt.IsZero() { + params.Etag = &cache.ETag + } + return s.q.SaveChannelCatalogue(ctx, params) +} + +// SetChannelMembers makes fingerprints the IATA's exact MeshMapper channel list. +func (s *Store) SetChannelMembers(ctx context.Context, iata string, fingerprints [][]byte) error { + if fingerprints == nil { + fingerprints = [][]byte{} + } + if err := s.q.DeleteChannelMembersNotIn(ctx, sqlc.DeleteChannelMembersNotInParams{Iata: iata, Keep: fingerprints}); err != nil { + return err + } + return s.q.AddChannelMembers(ctx, sqlc.AddChannelMembersParams{Iata: iata, Fingerprints: fingerprints}) +} + +func (s *Store) ClearChannelMembers(ctx context.Context) error { + return s.q.DeleteAllChannelMembers(ctx) +} + +func (s *Store) ListIATADetails(ctx context.Context) ([]meshmapper.IATADetails, error) { + rows, err := s.q.ListIATAs(ctx) + if err != nil { + return nil, err + } + out := make([]meshmapper.IATADetails, 0, len(rows)) + for _, row := range rows { + d := meshmapper.IATADetails{IATA: row.Iata, Lat: row.ApproxLat, Lng: row.ApproxLng} + if row.DisplayName != nil { + d.Name = *row.DisplayName + } + out = append(out, d) + } + return out, nil +} diff --git a/db/meshmapper_integration_test.go b/db/meshmapper_integration_test.go index 7d8c2a1..5b9af83 100644 --- a/db/meshmapper_integration_test.go +++ b/db/meshmapper_integration_test.go @@ -6,6 +6,7 @@ package db import ( "context" "os" + "slices" "strings" "testing" "time" @@ -44,7 +45,7 @@ func TestMeshMapperCataloguePostgres(t *testing.T) { t.Fatal(err) } defer tx.Rollback(context.Background()) - for _, table := range []string{"transport_scopes", "meshmapper_scope_catalogues"} { + for _, table := range []string{"iata_codes", "transport_scopes", "meshmapper_scope_catalogues"} { if _, err := tx.Exec(ctx, "CREATE TEMP TABLE "+table+" (LIKE public."+table+" INCLUDING ALL) ON COMMIT DROP"); err != nil { t.Fatal(err) } @@ -55,6 +56,9 @@ func TestMeshMapperCataloguePostgres(t *testing.T) { if err := store.UpsertTransportScope(ctx, manual.Name, "Operator label", manual.TransportKey, manual.KeyFingerprint); err != nil { t.Fatal(err) } + if err := store.UpsertIATA(ctx, "YOW"); err != nil { + t.Fatal(err) + } url := "https://yow.meshmapper.net/get_scopes.php" if row, err := store.GetScopeCatalogue(ctx, "YOW", url); err != nil || row != nil { t.Fatal(row, err) @@ -79,7 +83,7 @@ func TestMeshMapperCataloguePostgres(t *testing.T) { t.Fatal(row, err) } scopes := scopestore.New() - _, err = meshmapper.New(ctx, config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": url}}, store, scopes, manualRows) + _, err = meshmapper.New(ctx, config.MeshMapperScopesConfig{Enabled: true}, store, meshmapper.NewDirectory(store), scopes, manualRows) if err != nil || len(scopes.Entries()) != 2 { t.Fatal("restart did not restore imported membership", err) } @@ -102,7 +106,7 @@ func TestMeshMapperCataloguePostgres(t *testing.T) { if err := store.SaveScopeCatalogue(ctx, "YOW", url, update, nil); err != nil { t.Fatal(err) } - _, err = meshmapper.New(ctx, config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": url}}, store, scopes, manualRows) + _, err = meshmapper.New(ctx, config.MeshMapperScopesConfig{Enabled: true}, store, meshmapper.NewDirectory(store), scopes, manualRows) if err != nil || len(scopes.Entries()) != 1 { t.Fatal("removed imported membership remained active", err) } @@ -165,7 +169,7 @@ func TestMeshMapperZoneBoundariesPostgres(t *testing.T) { t.Fatal(err) } defer tx.Rollback(context.Background()) - for _, table := range []string{"iata_codes", "meshmapper_zone_boundaries"} { + for _, table := range []string{"iata_codes", "meshmapper_zone_boundaries", "meshmapper_zone_lists"} { if _, err := tx.Exec(ctx, "CREATE TEMP TABLE "+table+" (LIKE public."+table+" INCLUDING ALL) ON COMMIT DROP"); err != nil { t.Fatal(err) } @@ -222,4 +226,228 @@ func TestMeshMapperZoneBoundariesPostgres(t *testing.T) { if !strings.Contains(border("YOW"), `"file"`) { t.Fatal("manual border did not return after pruning") } + list := meshmapper.ZoneList{Country: "CA", Payload: []byte(`{"country":"CA","zones":[]}`), ETag: `"z1"`, + FetchedAt: now, AttemptedAt: now, NextAttempt: now.Add(24 * time.Hour)} + if err := store.SaveZoneList(ctx, list); err != nil { + t.Fatal(err) + } + later := now.Add(25 * time.Hour) + if err := store.SaveZoneList(ctx, meshmapper.ZoneList{Country: "CA", AttemptedAt: later, NextAttempt: later.Add(48 * time.Hour), LastError: "HTTP 429"}); err != nil { + t.Fatal(err) + } + lists, err := store.ListZoneLists(ctx) + if err != nil || len(lists) != 1 || len(lists[0].Payload) == 0 || lists[0].ETag != `"z1"` || !lists[0].FetchedAt.Equal(now) || + lists[0].LastError != "HTTP 429" || !lists[0].NextAttempt.Equal(later.Add(48*time.Hour)) { + t.Fatal("failed fetch lost the last good zone list", lists, err) + } +} + +func TestMeshMapperRegionsPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + pool, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatal(err) + } + if err := RunMigrations(ctx, pool); err != nil { + pool.Close() + t.Fatal(err) + } + pool.Close() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + // Real tables inside the rolled-back tx: TEMP copies drop region_iatas' foreign key to iata_codes. + store := &Store{q: sqlc.New(tx)} + state := func() map[string]meshmapper.RegionState { + t.Helper() + rows, err := store.ListRegionState(ctx) + if err != nil { + t.Fatal(err) + } + out := map[string]meshmapper.RegionState{} + for _, r := range rows { + out[r.Slug] = r + } + return out + } + for _, iata := range []string{"YOW", "YUL"} { // Seed creates config region IATAs first + if err := store.UpsertIATA(ctx, iata); err != nil { + t.Fatal(err) + } + } + hand, err := store.UpsertRegion(ctx, "east", "East", "", 3, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + if err := store.SetRegionIATAs(ctx, hand, []string{"YOW", "YUL"}); err != nil { + t.Fatal(err) + } + if err := store.SetRegionIATAs(ctx, hand, []string{"YOW"}); err != nil { + t.Fatal(err) + } + if got := state()["east"]; got.Imported || got.DisplayOrder != 3 || strings.Join(got.IATAs, ",") != "YOW" { + t.Fatal("configured members not replaced", got) + } + lat, lng := 45.5, -75.0 + group := meshmapper.RegionState{Slug: "onqc", Name: "Corridor", DisplayOrder: 4, IATAs: []string{"YOW", "YQB"}, CenterLat: &lat, CenterLng: &lng} + if saved, err := store.SaveImportedRegion(ctx, group); err != nil || !saved { + t.Fatal("members Beacon hasn't heard must be created first", saved, err) + } + if got := state()["onqc"]; got.CenterLat == nil || *got.CenterLat != lat || *got.CenterLng != lng { + t.Fatal("center not saved", got) + } + details, err := store.ListIATADetails(ctx) + if err != nil || !slices.ContainsFunc(details, func(d meshmapper.IATADetails) bool { return d.IATA == "YQB" }) { + t.Fatal("member IATA not created", err) + } + group.IATAs = []string{"YQB", "YUL"} + if saved, err := store.SaveImportedRegion(ctx, group); err != nil || !saved { + t.Fatal(saved, err) + } + if got := state()["onqc"]; !got.Imported || strings.Join(got.IATAs, ",") != "YQB,YUL" { + t.Fatal("imported region not updated", got) + } + if saved, err := store.SaveImportedRegion(ctx, meshmapper.RegionState{Slug: "east", Name: "Group", IATAs: []string{"YYZ"}}); err != nil || saved { + t.Fatal("imported group replaced a configured region", saved, err) + } + if got := state()["east"]; got.Name != "East" || strings.Join(got.IATAs, ",") != "YOW" { + t.Fatal("configured region changed by a clash", got) + } + if _, err := store.UpsertRegion(ctx, "onqc", "Mine", "", 1, nil, nil, nil); err != nil { + t.Fatal(err) + } + if state()["onqc"].Imported { + t.Fatal("config did not take over an imported slug") + } + if _, err := store.SaveImportedRegion(ctx, meshmapper.RegionState{Slug: "golm", Name: "Lakes", IATAs: []string{"YYZ"}}); err != nil { + t.Fatal(err) + } + if err := store.UpsertIATADetails(ctx, "YQB", "Quebec City", &lat, &lng); err != nil { + t.Fatal(err) + } + if details, _ := store.ListIATADetails(ctx); !slices.ContainsFunc(details, func(d meshmapper.IATADetails) bool { + return d.IATA == "YQB" && d.Name == "Quebec City" && d.Lat != nil && *d.Lat == lat + }) { + t.Fatal("IATA details not saved") + } + if pruned, err := store.PruneImportedRegions(ctx, nil); err != nil || strings.Join(pruned, ",") != "golm" { + t.Fatal("prune touched configured regions", pruned, err) + } +} + +func TestChannelRegionScopingPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + pool, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatal(err) + } + if err := RunMigrations(ctx, pool); err != nil { + pool.Close() + t.Fatal(err) + } + pool.Close() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + for _, table := range []string{"regions", "region_iatas", "channels", "channel_iatas", "channel_config_scopes", + "meshmapper_channel_members", "meshmapper_channel_catalogues"} { + if _, err := tx.Exec(ctx, "CREATE TEMP TABLE "+table+" (LIKE public."+table+" INCLUDING ALL) ON COMMIT DROP"); err != nil { + t.Fatal(err) + } + } + store := &Store{q: sqlc.New(tx)} + for slug, iata := range map[string]string{"east": "YOW", "west": "YYZ"} { + id, err := store.UpsertRegion(ctx, slug, slug, "", 0, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + if err := store.SetRegionIATAs(ctx, id, []string{iata}); err != nil { + t.Fatal(err) + } + } + fp := func(b byte) []byte { return []byte{b, b, b, b, b, b, b, b} } + for name, b := range map[string]byte{"#mm-yow": 1, "#config-east": 2, "#global": 3, "#toronto": 4} { + if _, err := store.UpsertChannel(ctx, []byte{b}, fp(b), name, name[1:]); err != nil { + t.Fatal(err) + } + } + if err := store.UpsertChannelIATA(ctx, []byte{4}, "YOW", time.Now()); err != nil { + t.Fatal(err) + } + if err := store.SetChannelConfigScopes(ctx, [][]byte{fp(2), fp(3)}, []string{"east", ""}); err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + for iata, b := range map[string]byte{"YOW": 1, "YYZ": 4} { + update := meshmapper.Cache{Payload: []byte(`{}`), CheckedAt: now, AttemptedAt: now, NextAttempt: now.Add(24 * time.Hour)} + if err := store.SaveChannelCatalogue(ctx, iata, "https://x.meshmapper.net/get_channels.php", update, [][]byte{fp(b)}); err != nil { + t.Fatal(err) + } + } + names := func(iatas ...string) string { + t.Helper() + page, err := store.ListChannels(ctx, 50, nil, iatas, nil, 0, nil) + if err != nil { + t.Fatal(err) + } + var out []string + for _, c := range page.Items { + out = append(out, *c.Name) + } + slices.Sort(out) + return strings.Join(out, ",") + } + if got := names("YOW"); got != "#config-east,#global,#mm-yow" { + t.Fatal("YOW channels:", got) + } + if got := names("YYZ"); got != "#global,#toronto" { + t.Fatal("YYZ channels:", got) + } + if got := names(); got != "#config-east,#global,#mm-yow,#toronto" { + t.Fatal("unfiltered channels:", got) + } + // A 304 keeps membership; a new payload replaces it. + if err := store.SaveChannelCatalogue(ctx, "YOW", "https://x.meshmapper.net/get_channels.php", meshmapper.Cache{AttemptedAt: now, NextAttempt: now}, nil); err != nil { + t.Fatal(err) + } + if got := names("YOW"); got != "#config-east,#global,#mm-yow" { + t.Fatal("304 dropped membership:", got) + } + if err := store.SaveChannelCatalogue(ctx, "YOW", "https://x.meshmapper.net/get_channels.php", meshmapper.Cache{Payload: []byte(`{}`), AttemptedAt: now, NextAttempt: now}, [][]byte{}); err != nil { + t.Fatal(err) + } + if got := names("YOW"); got != "#config-east,#global" { + t.Fatal("empty list kept membership:", got) + } + cats, err := store.ListChannelCatalogues(ctx) + if err != nil || len(cats) != 2 { + t.Fatal(cats, err) + } + if err := store.ClearChannelMembers(ctx); err != nil || names("YYZ") != "#global" { + t.Fatal("members not cleared", err) + } } diff --git a/db/migrations/001_baseline.sql b/db/migrations/001_baseline.sql index b858627..06e4335 100644 --- a/db/migrations/001_baseline.sql +++ b/db/migrations/001_baseline.sql @@ -564,6 +564,11 @@ CREATE TABLE analytics_top_talkers_by_iata ( last_sent timestamp with time zone ); +CREATE TABLE channel_config_scopes ( + key_fingerprint bytea NOT NULL, + region_slug text +); + CREATE TABLE channel_iatas ( channel_hash bytea NOT NULL, iata character(3) NOT NULL, @@ -643,6 +648,22 @@ ALTER TABLE known_routes ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY ( CACHE 1 ); +CREATE TABLE meshmapper_channel_catalogues ( + iata text NOT NULL, + url text NOT NULL, + payload jsonb, + etag text, + checked_at timestamp with time zone, + attempted_at timestamp with time zone NOT NULL, + next_attempt timestamp with time zone NOT NULL, + last_error text DEFAULT ''::text NOT NULL +); + +CREATE TABLE meshmapper_channel_members ( + iata text NOT NULL, + key_fingerprint bytea NOT NULL +); + CREATE TABLE meshmapper_scope_catalogues ( iata text NOT NULL, url text NOT NULL, @@ -665,6 +686,16 @@ CREATE TABLE meshmapper_zone_boundaries ( last_error text DEFAULT ''::text NOT NULL ); +CREATE TABLE meshmapper_zone_lists ( + country text NOT NULL, + payload jsonb, + etag text, + fetched_at timestamp with time zone, + attempted_at timestamp with time zone NOT NULL, + next_attempt timestamp with time zone NOT NULL, + last_error text DEFAULT ''::text NOT NULL +); + CREATE MATERIALIZED VIEW mv_hourly_iata_stats AS WITH combined AS ( SELECT iata, hour, observation_count, unique_packets FROM analytics_live_hourly_iata_stats @@ -911,7 +942,8 @@ CREATE TABLE regions ( center_lng double precision, zoom_level integer DEFAULT 8, created_at timestamp with time zone DEFAULT now() NOT NULL, - updated_at timestamp with time zone DEFAULT now() NOT NULL + updated_at timestamp with time zone DEFAULT now() NOT NULL, + imported boolean DEFAULT false NOT NULL ); CREATE SEQUENCE regions_id_seq @@ -1016,12 +1048,21 @@ ALTER TABLE ONLY iata_codes ALTER TABLE ONLY known_routes ADD CONSTRAINT known_routes_pkey PRIMARY KEY (iata, path_key); +ALTER TABLE ONLY meshmapper_channel_catalogues + ADD CONSTRAINT meshmapper_channel_catalogues_pkey PRIMARY KEY (iata, url); + +ALTER TABLE ONLY meshmapper_channel_members + ADD CONSTRAINT meshmapper_channel_members_pkey PRIMARY KEY (iata, key_fingerprint); + ALTER TABLE ONLY meshmapper_scope_catalogues ADD CONSTRAINT meshmapper_scope_catalogues_pkey PRIMARY KEY (iata, url); ALTER TABLE ONLY meshmapper_zone_boundaries ADD CONSTRAINT meshmapper_zone_boundaries_pkey PRIMARY KEY (iata); +ALTER TABLE ONLY meshmapper_zone_lists + ADD CONSTRAINT meshmapper_zone_lists_pkey PRIMARY KEY (country); + ALTER TABLE ONLY node_iatas ADD CONSTRAINT node_iatas_pkey PRIMARY KEY (node_id, iata); @@ -1128,6 +1169,10 @@ CREATE INDEX idx_channel_messages_channel ON channel_messages USING btree (chann CREATE INDEX idx_channel_messages_sent_brin ON channel_messages USING brin (sent_at); +CREATE INDEX idx_channel_config_scopes_fingerprint ON channel_config_scopes USING btree (key_fingerprint); + +CREATE INDEX idx_meshmapper_channel_members_fingerprint ON meshmapper_channel_members USING btree (key_fingerprint); + CREATE INDEX idx_channels_hash ON channels USING btree (channel_hash); CREATE UNIQUE INDEX idx_channels_hash_no_key ON channels USING btree (channel_hash) WHERE (key_fingerprint IS NULL); diff --git a/db/paths_integration_test.go b/db/paths_integration_test.go index 76a9662..a49f8cc 100644 --- a/db/paths_integration_test.go +++ b/db/paths_integration_test.go @@ -115,7 +115,7 @@ SELECT int4send(n),'00000000-0000-0000-0000-000000000001',heard_at,iata,payload_ } } w := httptest.NewRecorder() - handlers.StatsRouter(store).ServeHTTP(w, httptest.NewRequest("GET", fmt.Sprintf("/paths?since=%d&until=%d&iatas=YVR", since.UnixMilli()+123, until.UnixMilli()+123), nil).WithContext(ctx)) + handlers.StatsRouter(store, nil).ServeHTTP(w, httptest.NewRequest("GET", fmt.Sprintf("/paths?since=%d&until=%d&iatas=YVR", since.UnixMilli()+123, until.UnixMilli()+123), nil).WithContext(ctx)) var response api.PathStats if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil || w.Code != 200 || response.Receptions != 14 || response.Hashed != 5 { t.Fatalf("HTTP %d: %s (%v)", w.Code, w.Body.String(), err) diff --git a/db/queries/queries.sql b/db/queries/queries.sql index fd38c20..65e2ec3 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -10,6 +10,11 @@ INSERT INTO iata_codes (iata) VALUES ($1) ON CONFLICT (iata) DO NOTHING; +-- name: AddIATAs :exec +INSERT INTO iata_codes (iata) +SELECT unnest(@iatas::bpchar[]) +ON CONFLICT (iata) DO NOTHING; + -- name: GetIATA :one SELECT * FROM iata_codes WHERE iata = $1; @@ -64,21 +69,6 @@ SELECT id FROM transport_scopes WHERE name = $1; -- name: GetScopeNames :many SELECT name FROM transport_scopes ORDER BY name; --- name: GetScopesByIATAs :many -SELECT - ts.name, - COUNT(DISTINCT os.observer_id) AS observer_count, - COUNT(DISTINCT n.id) AS node_count, - COUNT(DISTINCT po.iata) AS iata_count -FROM transport_scopes ts -LEFT JOIN observer_scopes os ON os.scope_id = ts.id -LEFT JOIN observers o ON o.id = os.observer_id -LEFT JOIN packet_observations po ON po.observer_id = o.id -LEFT JOIN nodes n ON n.default_scope_id = ts.id -WHERE (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR po.iata = ANY($1::bpchar[])) -GROUP BY ts.name -ORDER BY ts.name; - -- name: GetScopeByName :one SELECT ts.name, @@ -878,6 +868,19 @@ RETURNING id; SELECT packet_hash, raw_payload FROM packets WHERE payload_type = 5 AND decrypted IS NOT TRUE; +-- name: ListUndecryptedGroupTextPacketsByHash :many +-- Like ListUndecryptedGroupTextPackets, limited to channels that just gained a key. +SELECT packet_hash, raw_payload FROM packets +WHERE payload_type = 5 AND decrypted IS NOT TRUE AND channel_hash = ANY(@hashes::bytea[]); + +-- name: DeleteChannelConfigScopes :exec +DELETE FROM channel_config_scopes; + +-- name: AddChannelConfigScopes :exec +-- An empty region places the channel Beacon-wide. +INSERT INTO channel_config_scopes (key_fingerprint, region_slug) +SELECT unnest(@fingerprints::bytea[]), NULLIF(unnest(@regions::text[]), ''); + -- name: UpsertChannelIATA :exec -- Refreshes at most hourly so repeat hears don't churn the row. INSERT INTO channel_iatas (channel_hash, iata, last_heard) @@ -895,15 +898,20 @@ ON CONFLICT (trace_tag, iata) DO UPDATE SET WHERE EXCLUDED.last_heard > trace_iatas.last_heard + INTERVAL '1 hour'; -- name: ListChannels :many --- Channels ordered by last seen, optionally filtered by hash and/or IATAs --- (membership via channel_iatas). NULL hash / empty array skip those filters. +-- Channels ordered by last seen, optionally filtered by hash and/or IATAs. +-- A channel belongs to an IATA when MeshMapper lists it there or config scopes it +-- to a region containing it (or Beacon-wide). NULL hash / empty array / NULL key_known skip those filters. -- Pass cursor=0 to start from the beginning (cursor is last_seen epoch ms). SELECT c.* FROM channels c WHERE (@channel_hash::bytea IS NULL OR c.channel_hash = @channel_hash) - AND (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 OR c.channel_hash IN ( - SELECT ci.channel_hash FROM channel_iatas ci - WHERE ci.iata = ANY(@iatas::bpchar[]) - )) + AND (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 + OR EXISTS (SELECT 1 FROM meshmapper_channel_members m + WHERE m.key_fingerprint = c.key_fingerprint AND m.iata::bpchar = ANY(@iatas::bpchar[])) + OR EXISTS (SELECT 1 FROM channel_config_scopes s + WHERE s.key_fingerprint = c.key_fingerprint AND (s.region_slug IS NULL OR s.region_slug IN ( + SELECT r.slug FROM regions r JOIN region_iatas ri ON ri.region_id = r.id + WHERE ri.iata = ANY(@iatas::bpchar[]))))) + AND (sqlc.narg(key_known)::boolean IS NULL OR COALESCE(c.key_known, false) = sqlc.narg(key_known)) AND (@cursor_ts::timestamptz IS NULL OR c.last_seen < @cursor_ts) ORDER BY c.last_seen DESC, c.id DESC LIMIT @page_limit; @@ -914,9 +922,14 @@ LIMIT @page_limit; SELECT c.* FROM channels c WHERE (c.last_seen, c.id) < (@cursor_ts::timestamptz, @cursor_id::integer) AND (@channel_hash::bytea IS NULL OR c.channel_hash = @channel_hash) - AND (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 OR c.channel_hash IN ( - SELECT ci.channel_hash FROM channel_iatas ci WHERE ci.iata = ANY(@iatas::bpchar[]) - )) + AND (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 + OR EXISTS (SELECT 1 FROM meshmapper_channel_members m + WHERE m.key_fingerprint = c.key_fingerprint AND m.iata::bpchar = ANY(@iatas::bpchar[])) + OR EXISTS (SELECT 1 FROM channel_config_scopes s + WHERE s.key_fingerprint = c.key_fingerprint AND (s.region_slug IS NULL OR s.region_slug IN ( + SELECT r.slug FROM regions r JOIN region_iatas ri ON ri.region_id = r.id + WHERE ri.iata = ANY(@iatas::bpchar[]))))) + AND (sqlc.narg(key_known)::boolean IS NULL OR COALESCE(c.key_known, false) = sqlc.narg(key_known)) ORDER BY c.last_seen DESC, c.id DESC LIMIT @page_limit; @@ -1215,14 +1228,43 @@ ON CONFLICT (slug) DO UPDATE SET center_lat = EXCLUDED.center_lat, center_lng = EXCLUDED.center_lng, zoom_level = EXCLUDED.zoom_level, + imported = FALSE, -- config owns the slug from now on updated_at = NOW() RETURNING id; --- name: UpsertRegionIATA :exec +-- name: DeleteRegionIATAsNotIn :exec +DELETE FROM region_iatas WHERE region_id = @region_id AND NOT (iata = ANY(@keep::bpchar[])); + +-- name: AddRegionIATAs :exec INSERT INTO region_iatas (region_id, iata) -VALUES ($1, $2) +SELECT @region_id, unnest(@iatas::bpchar[]) ON CONFLICT (region_id, iata) DO NOTHING; +-- name: ListRegionState :many +-- Every region with its members, for reconciling imported MeshMapper groups. +SELECT r.slug, r.name, COALESCE(r.display_order, 0)::int AS display_order, r.imported, r.center_lat, r.center_lng, + COALESCE(array_agg(ri.iata::text ORDER BY ri.iata) FILTER (WHERE ri.iata IS NOT NULL), '{}')::text[] AS iatas +FROM regions r +LEFT JOIN region_iatas ri ON ri.region_id = r.id +GROUP BY r.id +ORDER BY r.slug; + +-- name: UpsertImportedRegion :one +-- A hand-written region owns its slug: the WHERE turns a clash into no row. +INSERT INTO regions (slug, name, display_order, center_lat, center_lng, zoom_level, imported, updated_at) +VALUES (@slug, @name, @display_order, sqlc.narg(center_lat), sqlc.narg(center_lng), NULL, TRUE, NOW()) +ON CONFLICT (slug) DO UPDATE SET + name = EXCLUDED.name, + display_order = EXCLUDED.display_order, + center_lat = EXCLUDED.center_lat, + center_lng = EXCLUDED.center_lng, + updated_at = NOW() +WHERE regions.imported +RETURNING id; + +-- name: PruneImportedRegions :many +DELETE FROM regions WHERE imported AND NOT (slug = ANY(@keep::text[])) RETURNING slug; + -- ============================================================ -- TRACES -- ============================================================ @@ -1275,7 +1317,8 @@ ORDER BY t.last_heard_at DESC; -- name: UpsertKnownRoute :exec -- Route identity is path_key, an md5 of node_ids computed by the caller. --- Keep the latest processed representation without changing the node-chain identity. +-- On conflict, observation_count and last_seen are bumped and hash_prefix follows the +-- latest hearing, so evidence matches the hash width the route uses now. INSERT INTO known_routes (path_key, node_ids, hash_prefix, iata, hop_count) VALUES ($1, $2, $3, $4, $5) ON CONFLICT (iata, path_key) DO UPDATE SET @@ -1592,6 +1635,9 @@ SELECT (SELECT heard_at FROM latest)::timestamptz AS latest_recorded_at, -- name: GetScopeCatalogue :one SELECT * FROM meshmapper_scope_catalogues WHERE iata = $1 AND url = $2; +-- name: ListScopeCatalogues :many +SELECT * FROM meshmapper_scope_catalogues ORDER BY iata, attempted_at; + -- name: SaveScopeCatalogue :exec -- One statement commits the validated snapshot and its lookup identities together. -- Empty arrays insert nothing. NULL payload/checked_at retain last-known-good data @@ -1614,6 +1660,33 @@ ON CONFLICT (iata, url) DO UPDATE SET next_attempt = EXCLUDED.next_attempt, last_error = EXCLUDED.last_error; +-- name: ListChannelCatalogues :many +SELECT * FROM meshmapper_channel_catalogues ORDER BY iata, attempted_at; + +-- name: SaveChannelCatalogue :exec +-- NULL payload/etag/checked_at retain the last good list after an error or 304. +INSERT INTO meshmapper_channel_catalogues (iata, url, payload, etag, checked_at, attempted_at, next_attempt, last_error) +VALUES (@iata, @url, sqlc.narg(payload)::jsonb, sqlc.narg(etag)::text, + sqlc.narg(checked_at)::timestamptz, @attempted_at, @next_attempt, @last_error) +ON CONFLICT (iata, url) DO UPDATE SET + payload = COALESCE(EXCLUDED.payload, meshmapper_channel_catalogues.payload), + etag = COALESCE(EXCLUDED.etag, meshmapper_channel_catalogues.etag), + checked_at = COALESCE(EXCLUDED.checked_at, meshmapper_channel_catalogues.checked_at), + attempted_at = EXCLUDED.attempted_at, + next_attempt = EXCLUDED.next_attempt, + last_error = EXCLUDED.last_error; + +-- name: DeleteChannelMembersNotIn :exec +DELETE FROM meshmapper_channel_members WHERE iata = @iata AND NOT (key_fingerprint = ANY(@keep::bytea[])); + +-- name: AddChannelMembers :exec +INSERT INTO meshmapper_channel_members (iata, key_fingerprint) +SELECT @iata, unnest(@fingerprints::bytea[]) +ON CONFLICT (iata, key_fingerprint) DO NOTHING; + +-- name: DeleteAllChannelMembers :exec +DELETE FROM meshmapper_channel_members; + -- name: ListZoneBoundaries :many SELECT * FROM meshmapper_zone_boundaries ORDER BY iata; @@ -1631,6 +1704,22 @@ ON CONFLICT (iata) DO UPDATE SET next_attempt = EXCLUDED.next_attempt, last_error = EXCLUDED.last_error; +-- name: ListZoneLists :many +SELECT * FROM meshmapper_zone_lists ORDER BY country; + +-- name: SaveZoneList :exec +-- NULL payload/etag/fetched_at retain the last good list after an error or 304. +INSERT INTO meshmapper_zone_lists (country, payload, etag, fetched_at, attempted_at, next_attempt, last_error) +VALUES (@country, sqlc.narg(payload)::jsonb, sqlc.narg(etag)::text, + sqlc.narg(fetched_at)::timestamptz, @attempted_at, @next_attempt, @last_error) +ON CONFLICT (country) DO UPDATE SET + payload = COALESCE(EXCLUDED.payload, meshmapper_zone_lists.payload), + etag = COALESCE(EXCLUDED.etag, meshmapper_zone_lists.etag), + fetched_at = COALESCE(EXCLUDED.fetched_at, meshmapper_zone_lists.fetched_at), + attempted_at = EXCLUDED.attempted_at, + next_attempt = EXCLUDED.next_attempt, + last_error = EXCLUDED.last_error; + -- name: PruneZoneBoundaries :many -- Drops imports for IATAs no longer configured, so their manual border returns. DELETE FROM meshmapper_zone_boundaries WHERE NOT (iata = ANY(@keep::text[])) RETURNING iata; diff --git a/db/route_evidence_integration_test.go b/db/route_evidence_integration_test.go index 53e70c0..23abc34 100644 --- a/db/route_evidence_integration_test.go +++ b/db/route_evidence_integration_test.go @@ -234,3 +234,31 @@ ANALYZE packet_observations`); err != nil { t.Fatalf("0/1-hop rows entered the route index: %v %v", indexed, err) } } + +func TestRouteEvidenceFollowsPrefixWidthPostgres(t *testing.T) { + ctx, tx := retentionTx(t) + applyBaseline(t, ctx, tx) + if _, err := tx.Exec(ctx, `INSERT INTO observers(id,public_key) VALUES ('00000000-0000-0000-0000-000000000001','\x01'); +INSERT INTO packets(packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,first_heard_at,last_heard_at) +VALUES (int4send(1),4,0,1,'\x00','\x00',NOW(),NOW()); +INSERT INTO packet_observations(id,packet_hash,observer_id,iata,heard_at,hash_size,hop_count,path_bytes,payload_type,path_length_byte) +VALUES (1,int4send(1),'00000000-0000-0000-0000-000000000001','YOW',NOW(),2,2,'\xaa11bb22',4,(1<<6)|2)`); err != nil { + t.Fatal(err) + } + store := &Store{q: sqlc.New(tx)} + ids := []uuid.UUID{uuid.New(), uuid.New()} + if err := store.UpsertKnownRoute(ctx, ids, [][]byte{{0xaa}, {0xbb}}, "YOW", 2); err != nil { + t.Fatal(err) + } + if err := store.UpsertKnownRoute(ctx, ids, [][]byte{{0xaa, 0x11}, {0xbb, 0x22}}, "YOW", 2); err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + got, err := store.GetRouteEvidence(ctx, "YOW", hex.EncodeToString(routePathKey(ids)), api.RouteEvidenceQuery{Since: now.Add(-time.Hour), Until: now.Add(time.Minute), Limit: 10}) + if err != nil { + t.Fatal(err) + } + if got.HashSize != 2 || got.PathBytes != "aa11bb22" || len(got.Items) != 1 || got.Route.ObservationCount != 2 { + t.Fatalf("evidence kept the first prefix width: %+v", got) + } +} diff --git a/db/scopes.go b/db/scopes.go index af5cf9a..a9b1cb2 100644 --- a/db/scopes.go +++ b/db/scopes.go @@ -49,24 +49,6 @@ func (s *Store) GetScopeNames(ctx context.Context) ([]string, error) { return s.q.GetScopeNames(ctx) } -// GetScopesByIATAs returns scope summaries filtered by the given IATA codes. -func (s *Store) GetScopesByIATAs(ctx context.Context, iatas []string) ([]api.ScopeSummary, error) { - rows, err := s.q.GetScopesByIATAs(ctx, iatas) - if err != nil { - return nil, err - } - items := make([]api.ScopeSummary, 0, len(rows)) - for _, r := range rows { - items = append(items, api.ScopeSummary{ - Name: r.Name, - ObserverCount: r.ObserverCount, - NodeCount: r.NodeCount, - IATACount: r.IataCount, - }) - } - return items, nil -} - // GetScopeByName returns full detail for a single scope by its normalized name. func (s *Store) GetScopeByName(ctx context.Context, name string) (*api.ScopeDetail, error) { row, err := s.q.GetScopeByName(ctx, name) diff --git a/db/scopes_test.go b/db/scopes_test.go index 890e93a..c581dee 100644 --- a/db/scopes_test.go +++ b/db/scopes_test.go @@ -43,29 +43,6 @@ func TestGetTransportScopes(t *testing.T) { } } -func TestGetScopesByIATAs(t *testing.T) { - ctrl := gomock.NewController(t) - mock := mockdb.NewMockQuerier(ctrl) - - mock.EXPECT(). - GetScopesByIATAs(gomock.Any(), []string{"YVR", "YYJ"}). - Return([]sqlc.GetScopesByIATAsRow{ - {Name: "default", ObserverCount: 3, NodeCount: 10, IataCount: 2}, - }, nil) - - store := &Store{q: mock} - items, err := store.GetScopesByIATAs(context.Background(), []string{"YVR", "YYJ"}) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if len(items) != 1 { - t.Fatalf("expected 1 item, got %d", len(items)) - } - if items[0].IATACount != 2 { - t.Errorf("expected IATACount 2, got %d", items[0].IATACount) - } -} - func TestGetScopeByName(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) diff --git a/db/signal_integration_test.go b/db/signal_integration_test.go index b41af9f..73bc825 100644 --- a/db/signal_integration_test.go +++ b/db/signal_integration_test.go @@ -138,7 +138,7 @@ SELECT int4send(n),'00000000-0000-0000-0000-000000000001',heard_at,iata,snr,rssi } w := httptest.NewRecorder() request := httptest.NewRequest("GET", fmt.Sprintf("/signal?since=%d&until=%d&iatas=YVR", since.UnixMilli()+123, until.UnixMilli()+123), nil).WithContext(ctx) - handlers.StatsRouter(store).ServeHTTP(w, request) + handlers.StatsRouter(store, nil).ServeHTTP(w, request) var response api.SignalStats if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil || w.Code != 200 || response.Receptions != 13 || response.SNR.Samples != 5 { t.Fatalf("PostgreSQL HTTP response: status=%d body=%s error=%v", w.Code, w.Body.String(), err) diff --git a/db/sqlc/mock/querier.go b/db/sqlc/mock/querier.go index f9914cf..0626fda 100644 --- a/db/sqlc/mock/querier.go +++ b/db/sqlc/mock/querier.go @@ -44,6 +44,62 @@ func (m *MockQuerier) EXPECT() *MockQuerierMockRecorder { return m.recorder } +// AddChannelConfigScopes mocks base method. +func (m *MockQuerier) AddChannelConfigScopes(ctx context.Context, arg db.AddChannelConfigScopesParams) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddChannelConfigScopes", ctx, arg) + ret0, _ := ret[0].(error) + return ret0 +} + +// AddChannelConfigScopes indicates an expected call of AddChannelConfigScopes. +func (mr *MockQuerierMockRecorder) AddChannelConfigScopes(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddChannelConfigScopes", reflect.TypeOf((*MockQuerier)(nil).AddChannelConfigScopes), ctx, arg) +} + +// AddChannelMembers mocks base method. +func (m *MockQuerier) AddChannelMembers(ctx context.Context, arg db.AddChannelMembersParams) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddChannelMembers", ctx, arg) + ret0, _ := ret[0].(error) + return ret0 +} + +// AddChannelMembers indicates an expected call of AddChannelMembers. +func (mr *MockQuerierMockRecorder) AddChannelMembers(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddChannelMembers", reflect.TypeOf((*MockQuerier)(nil).AddChannelMembers), ctx, arg) +} + +// AddIATAs mocks base method. +func (m *MockQuerier) AddIATAs(ctx context.Context, iatas []string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddIATAs", ctx, iatas) + ret0, _ := ret[0].(error) + return ret0 +} + +// AddIATAs indicates an expected call of AddIATAs. +func (mr *MockQuerierMockRecorder) AddIATAs(ctx, iatas any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddIATAs", reflect.TypeOf((*MockQuerier)(nil).AddIATAs), ctx, iatas) +} + +// AddRegionIATAs mocks base method. +func (m *MockQuerier) AddRegionIATAs(ctx context.Context, arg db.AddRegionIATAsParams) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddRegionIATAs", ctx, arg) + ret0, _ := ret[0].(error) + return ret0 +} + +// AddRegionIATAs indicates an expected call of AddRegionIATAs. +func (mr *MockQuerierMockRecorder) AddRegionIATAs(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddRegionIATAs", reflect.TypeOf((*MockQuerier)(nil).AddRegionIATAs), ctx, arg) +} + // AmbiguousPrefixes mocks base method. func (m *MockQuerier) AmbiguousPrefixes(ctx context.Context) ([]db.AmbiguousPrefixesRow, error) { m.ctrl.T.Helper() @@ -89,6 +145,48 @@ func (mr *MockQuerierMockRecorder) DeactivateAccount(ctx, id any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeactivateAccount", reflect.TypeOf((*MockQuerier)(nil).DeactivateAccount), ctx, id) } +// DeleteAllChannelMembers mocks base method. +func (m *MockQuerier) DeleteAllChannelMembers(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteAllChannelMembers", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteAllChannelMembers indicates an expected call of DeleteAllChannelMembers. +func (mr *MockQuerierMockRecorder) DeleteAllChannelMembers(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAllChannelMembers", reflect.TypeOf((*MockQuerier)(nil).DeleteAllChannelMembers), ctx) +} + +// DeleteChannelConfigScopes mocks base method. +func (m *MockQuerier) DeleteChannelConfigScopes(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteChannelConfigScopes", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteChannelConfigScopes indicates an expected call of DeleteChannelConfigScopes. +func (mr *MockQuerierMockRecorder) DeleteChannelConfigScopes(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteChannelConfigScopes", reflect.TypeOf((*MockQuerier)(nil).DeleteChannelConfigScopes), ctx) +} + +// DeleteChannelMembersNotIn mocks base method. +func (m *MockQuerier) DeleteChannelMembersNotIn(ctx context.Context, arg db.DeleteChannelMembersNotInParams) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteChannelMembersNotIn", ctx, arg) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteChannelMembersNotIn indicates an expected call of DeleteChannelMembersNotIn. +func (mr *MockQuerierMockRecorder) DeleteChannelMembersNotIn(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteChannelMembersNotIn", reflect.TypeOf((*MockQuerier)(nil).DeleteChannelMembersNotIn), ctx, arg) +} + // DeleteOldChannelIATAs mocks base method. func (m *MockQuerier) DeleteOldChannelIATAs(ctx context.Context, lastHeard pgtype.Timestamptz) error { m.ctrl.T.Helper() @@ -190,6 +288,20 @@ func (mr *MockQuerierMockRecorder) DeleteOldTraceIATAs(ctx, lastHeard any) *gomo return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteOldTraceIATAs", reflect.TypeOf((*MockQuerier)(nil).DeleteOldTraceIATAs), ctx, lastHeard) } +// DeleteRegionIATAsNotIn mocks base method. +func (m *MockQuerier) DeleteRegionIATAsNotIn(ctx context.Context, arg db.DeleteRegionIATAsNotInParams) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteRegionIATAsNotIn", ctx, arg) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteRegionIATAsNotIn indicates an expected call of DeleteRegionIATAsNotIn. +func (mr *MockQuerierMockRecorder) DeleteRegionIATAsNotIn(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteRegionIATAsNotIn", reflect.TypeOf((*MockQuerier)(nil).DeleteRegionIATAsNotIn), ctx, arg) +} + // GetAccount mocks base method. func (m *MockQuerier) GetAccount(ctx context.Context, id uuid.UUID) (db.Account, error) { m.ctrl.T.Helper() @@ -760,21 +872,6 @@ func (mr *MockQuerierMockRecorder) GetScopeStats(ctx, iatas any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetScopeStats", reflect.TypeOf((*MockQuerier)(nil).GetScopeStats), ctx, iatas) } -// GetScopesByIATAs mocks base method. -func (m *MockQuerier) GetScopesByIATAs(ctx context.Context, dollar_1 []string) ([]db.GetScopesByIATAsRow, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetScopesByIATAs", ctx, dollar_1) - ret0, _ := ret[0].([]db.GetScopesByIATAsRow) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetScopesByIATAs indicates an expected call of GetScopesByIATAs. -func (mr *MockQuerierMockRecorder) GetScopesByIATAs(ctx, dollar_1 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetScopesByIATAs", reflect.TypeOf((*MockQuerier)(nil).GetScopesByIATAs), ctx, dollar_1) -} - // GetSignalStats mocks base method. func (m *MockQuerier) GetSignalStats(ctx context.Context, arg db.GetSignalStatsParams) ([]db.GetSignalStatsRow, error) { m.ctrl.T.Helper() @@ -1014,6 +1111,21 @@ func (mr *MockQuerierMockRecorder) ListAllChannelMessages(ctx, arg any) *gomock. return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListAllChannelMessages", reflect.TypeOf((*MockQuerier)(nil).ListAllChannelMessages), ctx, arg) } +// ListChannelCatalogues mocks base method. +func (m *MockQuerier) ListChannelCatalogues(ctx context.Context) ([]db.MeshmapperChannelCatalogue, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListChannelCatalogues", ctx) + ret0, _ := ret[0].([]db.MeshmapperChannelCatalogue) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListChannelCatalogues indicates an expected call of ListChannelCatalogues. +func (mr *MockQuerierMockRecorder) ListChannelCatalogues(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListChannelCatalogues", reflect.TypeOf((*MockQuerier)(nil).ListChannelCatalogues), ctx) +} + // ListChannelMessages mocks base method. func (m *MockQuerier) ListChannelMessages(ctx context.Context, arg db.ListChannelMessagesParams) ([]db.ListChannelMessagesRow, error) { m.ctrl.T.Helper() @@ -1239,6 +1351,21 @@ func (mr *MockQuerierMockRecorder) ListPacketsByIATAs(ctx, arg any) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListPacketsByIATAs", reflect.TypeOf((*MockQuerier)(nil).ListPacketsByIATAs), ctx, arg) } +// ListRegionState mocks base method. +func (m *MockQuerier) ListRegionState(ctx context.Context) ([]db.ListRegionStateRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListRegionState", ctx) + ret0, _ := ret[0].([]db.ListRegionStateRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListRegionState indicates an expected call of ListRegionState. +func (mr *MockQuerierMockRecorder) ListRegionState(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListRegionState", reflect.TypeOf((*MockQuerier)(nil).ListRegionState), ctx) +} + // ListRegions mocks base method. func (m *MockQuerier) ListRegions(ctx context.Context) ([]db.ListRegionsRow, error) { m.ctrl.T.Helper() @@ -1269,6 +1396,21 @@ func (mr *MockQuerierMockRecorder) ListRouteEvidence(ctx, arg any) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListRouteEvidence", reflect.TypeOf((*MockQuerier)(nil).ListRouteEvidence), ctx, arg) } +// ListScopeCatalogues mocks base method. +func (m *MockQuerier) ListScopeCatalogues(ctx context.Context) ([]db.MeshmapperScopeCatalogue, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListScopeCatalogues", ctx) + ret0, _ := ret[0].([]db.MeshmapperScopeCatalogue) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListScopeCatalogues indicates an expected call of ListScopeCatalogues. +func (mr *MockQuerierMockRecorder) ListScopeCatalogues(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListScopeCatalogues", reflect.TypeOf((*MockQuerier)(nil).ListScopeCatalogues), ctx) +} + // ListTraceTags mocks base method. func (m *MockQuerier) ListTraceTags(ctx context.Context, arg db.ListTraceTagsParams) ([]db.ListTraceTagsRow, error) { m.ctrl.T.Helper() @@ -1299,6 +1441,21 @@ func (mr *MockQuerierMockRecorder) ListUndecryptedGroupTextPackets(ctx any) *gom return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListUndecryptedGroupTextPackets", reflect.TypeOf((*MockQuerier)(nil).ListUndecryptedGroupTextPackets), ctx) } +// ListUndecryptedGroupTextPacketsByHash mocks base method. +func (m *MockQuerier) ListUndecryptedGroupTextPacketsByHash(ctx context.Context, hashes [][]byte) ([]db.ListUndecryptedGroupTextPacketsByHashRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListUndecryptedGroupTextPacketsByHash", ctx, hashes) + ret0, _ := ret[0].([]db.ListUndecryptedGroupTextPacketsByHashRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListUndecryptedGroupTextPacketsByHash indicates an expected call of ListUndecryptedGroupTextPacketsByHash. +func (mr *MockQuerierMockRecorder) ListUndecryptedGroupTextPacketsByHash(ctx, hashes any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListUndecryptedGroupTextPacketsByHash", reflect.TypeOf((*MockQuerier)(nil).ListUndecryptedGroupTextPacketsByHash), ctx, hashes) +} + // ListZoneBoundaries mocks base method. func (m *MockQuerier) ListZoneBoundaries(ctx context.Context) ([]db.MeshmapperZoneBoundary, error) { m.ctrl.T.Helper() @@ -1314,6 +1471,36 @@ func (mr *MockQuerierMockRecorder) ListZoneBoundaries(ctx any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListZoneBoundaries", reflect.TypeOf((*MockQuerier)(nil).ListZoneBoundaries), ctx) } +// ListZoneLists mocks base method. +func (m *MockQuerier) ListZoneLists(ctx context.Context) ([]db.MeshmapperZoneList, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListZoneLists", ctx) + ret0, _ := ret[0].([]db.MeshmapperZoneList) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListZoneLists indicates an expected call of ListZoneLists. +func (mr *MockQuerierMockRecorder) ListZoneLists(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListZoneLists", reflect.TypeOf((*MockQuerier)(nil).ListZoneLists), ctx) +} + +// PruneImportedRegions mocks base method. +func (m *MockQuerier) PruneImportedRegions(ctx context.Context, keep []string) ([]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PruneImportedRegions", ctx, keep) + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// PruneImportedRegions indicates an expected call of PruneImportedRegions. +func (mr *MockQuerierMockRecorder) PruneImportedRegions(ctx, keep any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PruneImportedRegions", reflect.TypeOf((*MockQuerier)(nil).PruneImportedRegions), ctx, keep) +} + // PruneZoneBoundaries mocks base method. func (m *MockQuerier) PruneZoneBoundaries(ctx context.Context, keep []string) ([]string, error) { m.ctrl.T.Helper() @@ -1588,6 +1775,20 @@ func (mr *MockQuerierMockRecorder) ResolvePathHashesP4(ctx, arg any) *gomock.Cal return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResolvePathHashesP4", reflect.TypeOf((*MockQuerier)(nil).ResolvePathHashesP4), ctx, arg) } +// SaveChannelCatalogue mocks base method. +func (m *MockQuerier) SaveChannelCatalogue(ctx context.Context, arg db.SaveChannelCatalogueParams) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SaveChannelCatalogue", ctx, arg) + ret0, _ := ret[0].(error) + return ret0 +} + +// SaveChannelCatalogue indicates an expected call of SaveChannelCatalogue. +func (mr *MockQuerierMockRecorder) SaveChannelCatalogue(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveChannelCatalogue", reflect.TypeOf((*MockQuerier)(nil).SaveChannelCatalogue), ctx, arg) +} + // SaveScopeCatalogue mocks base method. func (m *MockQuerier) SaveScopeCatalogue(ctx context.Context, arg db.SaveScopeCatalogueParams) error { m.ctrl.T.Helper() @@ -1616,6 +1817,20 @@ func (mr *MockQuerierMockRecorder) SaveZoneBoundary(ctx, arg any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveZoneBoundary", reflect.TypeOf((*MockQuerier)(nil).SaveZoneBoundary), ctx, arg) } +// SaveZoneList mocks base method. +func (m *MockQuerier) SaveZoneList(ctx context.Context, arg db.SaveZoneListParams) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SaveZoneList", ctx, arg) + ret0, _ := ret[0].(error) + return ret0 +} + +// SaveZoneList indicates an expected call of SaveZoneList. +func (mr *MockQuerierMockRecorder) SaveZoneList(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveZoneList", reflect.TypeOf((*MockQuerier)(nil).SaveZoneList), ctx, arg) +} + // SearchKnownRoutes mocks base method. func (m *MockQuerier) SearchKnownRoutes(ctx context.Context, arg db.SearchKnownRoutesParams) ([]db.SearchKnownRoutesRow, error) { m.ctrl.T.Helper() @@ -1844,6 +2059,21 @@ func (mr *MockQuerierMockRecorder) UpsertIATADetails(ctx, arg any) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertIATADetails", reflect.TypeOf((*MockQuerier)(nil).UpsertIATADetails), ctx, arg) } +// UpsertImportedRegion mocks base method. +func (m *MockQuerier) UpsertImportedRegion(ctx context.Context, arg db.UpsertImportedRegionParams) (int32, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpsertImportedRegion", ctx, arg) + ret0, _ := ret[0].(int32) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// UpsertImportedRegion indicates an expected call of UpsertImportedRegion. +func (mr *MockQuerierMockRecorder) UpsertImportedRegion(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertImportedRegion", reflect.TypeOf((*MockQuerier)(nil).UpsertImportedRegion), ctx, arg) +} + // UpsertKnownRoute mocks base method. func (m *MockQuerier) UpsertKnownRoute(ctx context.Context, arg db.UpsertKnownRouteParams) error { m.ctrl.T.Helper() @@ -1988,20 +2218,6 @@ func (mr *MockQuerierMockRecorder) UpsertRegion(ctx, arg any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertRegion", reflect.TypeOf((*MockQuerier)(nil).UpsertRegion), ctx, arg) } -// UpsertRegionIATA mocks base method. -func (m *MockQuerier) UpsertRegionIATA(ctx context.Context, arg db.UpsertRegionIATAParams) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UpsertRegionIATA", ctx, arg) - ret0, _ := ret[0].(error) - return ret0 -} - -// UpsertRegionIATA indicates an expected call of UpsertRegionIATA. -func (mr *MockQuerierMockRecorder) UpsertRegionIATA(ctx, arg any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertRegionIATA", reflect.TypeOf((*MockQuerier)(nil).UpsertRegionIATA), ctx, arg) -} - // UpsertTraceIATA mocks base method. func (m *MockQuerier) UpsertTraceIATA(ctx context.Context, arg db.UpsertTraceIATAParams) error { m.ctrl.T.Helper() diff --git a/db/sqlc/models.go b/db/sqlc/models.go index bf81a1d..e08581c 100644 --- a/db/sqlc/models.go +++ b/db/sqlc/models.go @@ -188,6 +188,11 @@ type Channel struct { MessageCount *int64 `json:"message_count"` } +type ChannelConfigScope struct { + KeyFingerprint []byte `json:"key_fingerprint"` + RegionSlug *string `json:"region_slug"` +} + type ChannelIata struct { ChannelHash []byte `json:"channel_hash"` Iata string `json:"iata"` @@ -234,6 +239,22 @@ type KnownRoute struct { LastReconfirmedAt pgtype.Timestamptz `json:"last_reconfirmed_at"` } +type MeshmapperChannelCatalogue struct { + Iata string `json:"iata"` + Url string `json:"url"` + Payload []byte `json:"payload"` + Etag *string `json:"etag"` + CheckedAt pgtype.Timestamptz `json:"checked_at"` + AttemptedAt pgtype.Timestamptz `json:"attempted_at"` + NextAttempt pgtype.Timestamptz `json:"next_attempt"` + LastError string `json:"last_error"` +} + +type MeshmapperChannelMember struct { + Iata string `json:"iata"` + KeyFingerprint []byte `json:"key_fingerprint"` +} + type MeshmapperScopeCatalogue struct { Iata string `json:"iata"` Url string `json:"url"` @@ -256,6 +277,16 @@ type MeshmapperZoneBoundary struct { LastError string `json:"last_error"` } +type MeshmapperZoneList struct { + Country string `json:"country"` + Payload []byte `json:"payload"` + Etag *string `json:"etag"` + FetchedAt pgtype.Timestamptz `json:"fetched_at"` + AttemptedAt pgtype.Timestamptz `json:"attempted_at"` + NextAttempt pgtype.Timestamptz `json:"next_attempt"` + LastError string `json:"last_error"` +} + type MvHourlyIataStat struct { Iata string `json:"iata"` Hour pgtype.Timestamptz `json:"hour"` @@ -529,6 +560,7 @@ type Region struct { ZoomLevel *int32 `json:"zoom_level"` CreatedAt pgtype.Timestamptz `json:"created_at"` UpdatedAt pgtype.Timestamptz `json:"updated_at"` + Imported bool `json:"imported"` } type RegionIata struct { diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index ae84cc0..c2adf52 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -13,12 +13,20 @@ import ( ) type Querier interface { + // An empty region places the channel Beacon-wide. + AddChannelConfigScopes(ctx context.Context, arg AddChannelConfigScopesParams) error + AddChannelMembers(ctx context.Context, arg AddChannelMembersParams) error + AddIATAs(ctx context.Context, iatas []string) error + AddRegionIATAs(ctx context.Context, arg AddRegionIATAsParams) error // Hop prefixes that match >1 node in an IATA, per width. Computed once per reconfirm run. AmbiguousPrefixes(ctx context.Context) ([]AmbiguousPrefixesRow, error) CreateAccount(ctx context.Context, name string) (Account, error) // Lock the current row before deciding the outcome, including when another // deactivation commits while this statement is waiting for its row lock. DeactivateAccount(ctx context.Context, id uuid.UUID) (DeactivateAccountRow, error) + DeleteAllChannelMembers(ctx context.Context) error + DeleteChannelConfigScopes(ctx context.Context) error + DeleteChannelMembersNotIn(ctx context.Context, arg DeleteChannelMembersNotInParams) error // Keeps the channel IATA filter in step with packet retention. DeleteOldChannelIATAs(ctx context.Context, lastHeard pgtype.Timestamptz) error // Deletes nodes not seen since the given cutoff. node_iatas and node_neighbors cascade- @@ -41,6 +49,7 @@ type Querier interface { DeleteOldTelemetry(ctx context.Context, reportedAt pgtype.Timestamptz) error // Keeps the trace IATA filter in step with packet retention. DeleteOldTraceIATAs(ctx context.Context, lastHeard pgtype.Timestamptz) error + DeleteRegionIATAsNotIn(ctx context.Context, arg DeleteRegionIATAsNotInParams) error GetAccount(ctx context.Context, id uuid.UUID) (Account, error) GetChannelByID(ctx context.Context, id int32) (Channel, error) // Returns neighbors of a node that are in a different IATA. @@ -97,7 +106,6 @@ type Querier interface { // a cross-join. Empty IATAs keep the original global counts, including associations // whose observations have expired; the filtered aggregates are empty in that case. GetScopeStats(ctx context.Context, iatas []string) ([]GetScopeStatsRow, error) - GetScopesByIATAs(ctx context.Context, dollar_1 []string) ([]GetScopesByIATAsRow, error) // Read compact hourly snapshots, never observations on an HTTP request. // Weight averages by sample counts instead of averaging regional/hourly means. GetSignalStats(ctx context.Context, arg GetSignalStatsParams) ([]GetSignalStatsRow, error) @@ -143,6 +151,7 @@ type Querier interface { // Pass empty string for iata or scope to skip those filters. // Pass cursor=0 to start from the beginning. ListAllChannelMessages(ctx context.Context, arg ListAllChannelMessagesParams) ([]ListAllChannelMessagesRow, error) + ListChannelCatalogues(ctx context.Context) ([]MeshmapperChannelCatalogue, error) // Returns messages for a channel identified by integer ID. // Pass a zero/null timestamp for since to return all messages up to limit. // Pass empty string for iata to skip IATA filtering. @@ -153,8 +162,9 @@ type Querier interface { // Pass empty string for iata or scope to skip those filters. // Pass cursor=0 to start from the beginning. ListChannelMessagesByHash(ctx context.Context, arg ListChannelMessagesByHashParams) ([]ListChannelMessagesByHashRow, error) - // Channels ordered by last seen, optionally filtered by hash and/or IATAs - // (membership via channel_iatas). NULL hash / empty array skip those filters. + // Channels ordered by last seen, optionally filtered by hash and/or IATAs. + // A channel belongs to an IATA when MeshMapper lists it there or config scopes it + // to a region containing it (or Beacon-wide). NULL hash / empty array / NULL key_known skip those filters. // Pass cursor=0 to start from the beginning (cursor is last_seen epoch ms). ListChannels(ctx context.Context, arg ListChannelsParams) ([]Channel, error) // Keep the non-null tuple boundary separate from the legacy optional cursor so @@ -200,6 +210,8 @@ type Querier interface { // A site that filled scan_depth still has unread history below its floor. // The newest such floor is the point above which every site is covered. ListPacketsByIATAs(ctx context.Context, arg ListPacketsByIATAsParams) ([]ListPacketsByIATAsRow, error) + // Every region with its members, for reconciling imported MeshMapper groups. + ListRegionState(ctx context.Context) ([]ListRegionStateRow, error) // ============================================================ // REGIONS // ============================================================ @@ -208,6 +220,7 @@ type Querier interface { // Full-byte equality is required even when the compact digest matches. TRACE path bytes // carry readings; unclassified legacy observations cannot be safely called ordinary paths. ListRouteEvidence(ctx context.Context, arg ListRouteEvidenceParams) ([]ListRouteEvidenceRow, error) + ListScopeCatalogues(ctx context.Context) ([]MeshmapperScopeCatalogue, error) // ============================================================ // TRACES // ============================================================ @@ -220,7 +233,11 @@ type Querier interface { // to the config after they'd already been ingested -- see // internal/ingest.BackfillChannelMessages. ListUndecryptedGroupTextPackets(ctx context.Context) ([]ListUndecryptedGroupTextPacketsRow, error) + // Like ListUndecryptedGroupTextPackets, limited to channels that just gained a key. + ListUndecryptedGroupTextPacketsByHash(ctx context.Context, hashes [][]byte) ([]ListUndecryptedGroupTextPacketsByHashRow, error) ListZoneBoundaries(ctx context.Context) ([]MeshmapperZoneBoundary, error) + ListZoneLists(ctx context.Context) ([]MeshmapperZoneList, error) + PruneImportedRegions(ctx context.Context, keep []string) ([]string, error) // Drops imports for IATAs no longer configured, so their manual border returns. PruneZoneBoundaries(ctx context.Context, keep []string) ([]string, error) // Delete node_neighbors where the neighbor has departed from node_short_ids @@ -259,12 +276,16 @@ type Querier interface { ResolvePathHashesP2(ctx context.Context, arg ResolvePathHashesP2Params) ([]ResolvePathHashesP2Row, error) ResolvePathHashesP3(ctx context.Context, arg ResolvePathHashesP3Params) ([]ResolvePathHashesP3Row, error) ResolvePathHashesP4(ctx context.Context, arg ResolvePathHashesP4Params) ([]ResolvePathHashesP4Row, error) + // NULL payload/etag/checked_at retain the last good list after an error or 304. + SaveChannelCatalogue(ctx context.Context, arg SaveChannelCatalogueParams) error // One statement commits the validated snapshot and its lookup identities together. // Empty arrays insert nothing. NULL payload/checked_at retain last-known-good data // after an error or 304. Imported names never replace existing manual metadata. SaveScopeCatalogue(ctx context.Context, arg SaveScopeCatalogueParams) error // NULL feature/etag/checked_at retain the last good boundary after an error or 304. SaveZoneBoundary(ctx context.Context, arg SaveZoneBoundaryParams) error + // NULL payload/etag/fetched_at retain the last good list after an error or 304. + SaveZoneList(ctx context.Context, arg SaveZoneListParams) error // Returns known routes containing a subsequence from source to destination hash prefix. // Verifies source appears before destination in the route. SearchKnownRoutes(ctx context.Context, arg SearchKnownRoutesParams) ([]SearchKnownRoutesRow, error) @@ -302,11 +323,14 @@ type Querier interface { // bbox already computed -- see internal/config/border.go. UpsertIATABorder(ctx context.Context, arg UpsertIATABorderParams) error UpsertIATADetails(ctx context.Context, arg UpsertIATADetailsParams) error + // A hand-written region owns its slug: the WHERE turns a clash into no row. + UpsertImportedRegion(ctx context.Context, arg UpsertImportedRegionParams) (int32, error) // ============================================================ // ROUTES // ============================================================ // Route identity is path_key, an md5 of node_ids computed by the caller. - // Keep the latest processed representation without changing the node-chain identity. + // On conflict, observation_count and last_seen are bumped and hash_prefix follows the + // latest hearing, so evidence matches the hash width the route uses now. UpsertKnownRoute(ctx context.Context, arg UpsertKnownRouteParams) error // ============================================================ // NODES @@ -343,7 +367,6 @@ type Querier interface { // ============================================================ UpsertPacket(ctx context.Context, arg UpsertPacketParams) (UpsertPacketRow, error) UpsertRegion(ctx context.Context, arg UpsertRegionParams) (int32, error) - UpsertRegionIATA(ctx context.Context, arg UpsertRegionIATAParams) error // Refreshes at most hourly so repeat hears don't churn the row. UpsertTraceIATA(ctx context.Context, arg UpsertTraceIATAParams) error // ============================================================ diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index be72f7a..30db7f5 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -13,6 +13,65 @@ import ( "github.com/jackc/pgx/v5/pgtype" ) +const addChannelConfigScopes = `-- name: AddChannelConfigScopes :exec +INSERT INTO channel_config_scopes (key_fingerprint, region_slug) +SELECT unnest($1::bytea[]), NULLIF(unnest($2::text[]), '') +` + +type AddChannelConfigScopesParams struct { + Fingerprints [][]byte `json:"fingerprints"` + Regions []string `json:"regions"` +} + +// An empty region places the channel Beacon-wide. +func (q *Queries) AddChannelConfigScopes(ctx context.Context, arg AddChannelConfigScopesParams) error { + _, err := q.db.Exec(ctx, addChannelConfigScopes, arg.Fingerprints, arg.Regions) + return err +} + +const addChannelMembers = `-- name: AddChannelMembers :exec +INSERT INTO meshmapper_channel_members (iata, key_fingerprint) +SELECT $1, unnest($2::bytea[]) +ON CONFLICT (iata, key_fingerprint) DO NOTHING +` + +type AddChannelMembersParams struct { + Iata string `json:"iata"` + Fingerprints [][]byte `json:"fingerprints"` +} + +func (q *Queries) AddChannelMembers(ctx context.Context, arg AddChannelMembersParams) error { + _, err := q.db.Exec(ctx, addChannelMembers, arg.Iata, arg.Fingerprints) + return err +} + +const addIATAs = `-- name: AddIATAs :exec +INSERT INTO iata_codes (iata) +SELECT unnest($1::bpchar[]) +ON CONFLICT (iata) DO NOTHING +` + +func (q *Queries) AddIATAs(ctx context.Context, iatas []string) error { + _, err := q.db.Exec(ctx, addIATAs, iatas) + return err +} + +const addRegionIATAs = `-- name: AddRegionIATAs :exec +INSERT INTO region_iatas (region_id, iata) +SELECT $1, unnest($2::bpchar[]) +ON CONFLICT (region_id, iata) DO NOTHING +` + +type AddRegionIATAsParams struct { + RegionID int32 `json:"region_id"` + Iatas []string `json:"iatas"` +} + +func (q *Queries) AddRegionIATAs(ctx context.Context, arg AddRegionIATAsParams) error { + _, err := q.db.Exec(ctx, addRegionIATAs, arg.RegionID, arg.Iatas) + return err +} + const ambiguousPrefixes = `-- name: AmbiguousPrefixes :many SELECT iata::text AS iata, 1::int AS len, prefix_1 AS prefix FROM node_short_ids GROUP BY iata, prefix_1 HAVING COUNT(*) > 1 UNION ALL @@ -94,6 +153,38 @@ func (q *Queries) DeactivateAccount(ctx context.Context, id uuid.UUID) (Deactiva return i, err } +const deleteAllChannelMembers = `-- name: DeleteAllChannelMembers :exec +DELETE FROM meshmapper_channel_members +` + +func (q *Queries) DeleteAllChannelMembers(ctx context.Context) error { + _, err := q.db.Exec(ctx, deleteAllChannelMembers) + return err +} + +const deleteChannelConfigScopes = `-- name: DeleteChannelConfigScopes :exec +DELETE FROM channel_config_scopes +` + +func (q *Queries) DeleteChannelConfigScopes(ctx context.Context) error { + _, err := q.db.Exec(ctx, deleteChannelConfigScopes) + return err +} + +const deleteChannelMembersNotIn = `-- name: DeleteChannelMembersNotIn :exec +DELETE FROM meshmapper_channel_members WHERE iata = $1 AND NOT (key_fingerprint = ANY($2::bytea[])) +` + +type DeleteChannelMembersNotInParams struct { + Iata string `json:"iata"` + Keep [][]byte `json:"keep"` +} + +func (q *Queries) DeleteChannelMembersNotIn(ctx context.Context, arg DeleteChannelMembersNotInParams) error { + _, err := q.db.Exec(ctx, deleteChannelMembersNotIn, arg.Iata, arg.Keep) + return err +} + const deleteOldChannelIATAs = `-- name: DeleteOldChannelIATAs :exec DELETE FROM channel_iatas WHERE last_heard < $1 ` @@ -235,6 +326,20 @@ func (q *Queries) DeleteOldTraceIATAs(ctx context.Context, lastHeard pgtype.Time return err } +const deleteRegionIATAsNotIn = `-- name: DeleteRegionIATAsNotIn :exec +DELETE FROM region_iatas WHERE region_id = $1 AND NOT (iata = ANY($2::bpchar[])) +` + +type DeleteRegionIATAsNotInParams struct { + RegionID int32 `json:"region_id"` + Keep []string `json:"keep"` +} + +func (q *Queries) DeleteRegionIATAsNotIn(ctx context.Context, arg DeleteRegionIATAsNotInParams) error { + _, err := q.db.Exec(ctx, deleteRegionIATAsNotIn, arg.RegionID, arg.Keep) + return err +} + const getAccount = `-- name: GetAccount :one SELECT id, name, created_at, deactivated_at FROM accounts WHERE id = $1 ` @@ -1596,54 +1701,6 @@ func (q *Queries) GetScopeStats(ctx context.Context, iatas []string) ([]GetScope return items, nil } -const getScopesByIATAs = `-- name: GetScopesByIATAs :many -SELECT - ts.name, - COUNT(DISTINCT os.observer_id) AS observer_count, - COUNT(DISTINCT n.id) AS node_count, - COUNT(DISTINCT po.iata) AS iata_count -FROM transport_scopes ts -LEFT JOIN observer_scopes os ON os.scope_id = ts.id -LEFT JOIN observers o ON o.id = os.observer_id -LEFT JOIN packet_observations po ON po.observer_id = o.id -LEFT JOIN nodes n ON n.default_scope_id = ts.id -WHERE (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR po.iata = ANY($1::bpchar[])) -GROUP BY ts.name -ORDER BY ts.name -` - -type GetScopesByIATAsRow struct { - Name string `json:"name"` - ObserverCount int64 `json:"observer_count"` - NodeCount int64 `json:"node_count"` - IataCount int64 `json:"iata_count"` -} - -func (q *Queries) GetScopesByIATAs(ctx context.Context, dollar_1 []string) ([]GetScopesByIATAsRow, error) { - rows, err := q.db.Query(ctx, getScopesByIATAs, dollar_1) - if err != nil { - return nil, err - } - defer rows.Close() - items := []GetScopesByIATAsRow{} - for rows.Next() { - var i GetScopesByIATAsRow - if err := rows.Scan( - &i.Name, - &i.ObserverCount, - &i.NodeCount, - &i.IataCount, - ); err != nil { - return nil, err - } - items = append(items, i) - } - if err := rows.Err(); err != nil { - return nil, err - } - return items, nil -} - const getStatsClockDrift = `-- name: GetStatsClockDrift :many WITH page AS ( SELECT @@ -2370,6 +2427,39 @@ func (q *Queries) ListAllChannelMessages(ctx context.Context, arg ListAllChannel return items, nil } +const listChannelCatalogues = `-- name: ListChannelCatalogues :many +SELECT iata, url, payload, etag, checked_at, attempted_at, next_attempt, last_error FROM meshmapper_channel_catalogues ORDER BY iata, attempted_at +` + +func (q *Queries) ListChannelCatalogues(ctx context.Context) ([]MeshmapperChannelCatalogue, error) { + rows, err := q.db.Query(ctx, listChannelCatalogues) + if err != nil { + return nil, err + } + defer rows.Close() + items := []MeshmapperChannelCatalogue{} + for rows.Next() { + var i MeshmapperChannelCatalogue + if err := rows.Scan( + &i.Iata, + &i.Url, + &i.Payload, + &i.Etag, + &i.CheckedAt, + &i.AttemptedAt, + &i.NextAttempt, + &i.LastError, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const listChannelMessages = `-- name: ListChannelMessages :many SELECT DISTINCT ON (cm.id) cm.id, cm.channel_id, cm.packet_hash, cm.sender_name, cm.sender_pubkey, cm.content, cm.sent_at, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, ts.name AS scope_name, p.transport_codes_present, (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = cm.packet_hash) AS observation_count @@ -2541,29 +2631,36 @@ func (q *Queries) ListChannelMessagesByHash(ctx context.Context, arg ListChannel const listChannels = `-- name: ListChannels :many SELECT c.id, c.channel_hash, c.key_fingerprint, c.name, c.hashtag, c.is_hashtag, c.is_public, c.key_known, c.first_seen, c.last_seen, c.message_count FROM channels c WHERE ($1::bytea IS NULL OR c.channel_hash = $1) - AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR c.channel_hash IN ( - SELECT ci.channel_hash FROM channel_iatas ci - WHERE ci.iata = ANY($2::bpchar[]) - )) - AND ($3::timestamptz IS NULL OR c.last_seen < $3) + AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 + OR EXISTS (SELECT 1 FROM meshmapper_channel_members m + WHERE m.key_fingerprint = c.key_fingerprint AND m.iata::bpchar = ANY($2::bpchar[])) + OR EXISTS (SELECT 1 FROM channel_config_scopes s + WHERE s.key_fingerprint = c.key_fingerprint AND (s.region_slug IS NULL OR s.region_slug IN ( + SELECT r.slug FROM regions r JOIN region_iatas ri ON ri.region_id = r.id + WHERE ri.iata = ANY($2::bpchar[]))))) + AND ($3::boolean IS NULL OR COALESCE(c.key_known, false) = $3) + AND ($4::timestamptz IS NULL OR c.last_seen < $4) ORDER BY c.last_seen DESC, c.id DESC -LIMIT $4 +LIMIT $5 ` type ListChannelsParams struct { ChannelHash []byte `json:"channel_hash"` Iatas []string `json:"iatas"` + KeyKnown *bool `json:"key_known"` CursorTs pgtype.Timestamptz `json:"cursor_ts"` PageLimit int32 `json:"page_limit"` } -// Channels ordered by last seen, optionally filtered by hash and/or IATAs -// (membership via channel_iatas). NULL hash / empty array skip those filters. +// Channels ordered by last seen, optionally filtered by hash and/or IATAs. +// A channel belongs to an IATA when MeshMapper lists it there or config scopes it +// to a region containing it (or Beacon-wide). NULL hash / empty array / NULL key_known skip those filters. // Pass cursor=0 to start from the beginning (cursor is last_seen epoch ms). func (q *Queries) ListChannels(ctx context.Context, arg ListChannelsParams) ([]Channel, error) { rows, err := q.db.Query(ctx, listChannels, arg.ChannelHash, arg.Iatas, + arg.KeyKnown, arg.CursorTs, arg.PageLimit, ) @@ -2601,11 +2698,16 @@ const listChannelsAfter = `-- name: ListChannelsAfter :many SELECT c.id, c.channel_hash, c.key_fingerprint, c.name, c.hashtag, c.is_hashtag, c.is_public, c.key_known, c.first_seen, c.last_seen, c.message_count FROM channels c WHERE (c.last_seen, c.id) < ($1::timestamptz, $2::integer) AND ($3::bytea IS NULL OR c.channel_hash = $3) - AND (COALESCE(cardinality($4::bpchar[]), 0) = 0 OR c.channel_hash IN ( - SELECT ci.channel_hash FROM channel_iatas ci WHERE ci.iata = ANY($4::bpchar[]) - )) + AND (COALESCE(cardinality($4::bpchar[]), 0) = 0 + OR EXISTS (SELECT 1 FROM meshmapper_channel_members m + WHERE m.key_fingerprint = c.key_fingerprint AND m.iata::bpchar = ANY($4::bpchar[])) + OR EXISTS (SELECT 1 FROM channel_config_scopes s + WHERE s.key_fingerprint = c.key_fingerprint AND (s.region_slug IS NULL OR s.region_slug IN ( + SELECT r.slug FROM regions r JOIN region_iatas ri ON ri.region_id = r.id + WHERE ri.iata = ANY($4::bpchar[]))))) + AND ($5::boolean IS NULL OR COALESCE(c.key_known, false) = $5) ORDER BY c.last_seen DESC, c.id DESC -LIMIT $5 +LIMIT $6 ` type ListChannelsAfterParams struct { @@ -2613,6 +2715,7 @@ type ListChannelsAfterParams struct { CursorID int32 `json:"cursor_id"` ChannelHash []byte `json:"channel_hash"` Iatas []string `json:"iatas"` + KeyKnown *bool `json:"key_known"` PageLimit int32 `json:"page_limit"` } @@ -2624,6 +2727,7 @@ func (q *Queries) ListChannelsAfter(ctx context.Context, arg ListChannelsAfterPa arg.CursorID, arg.ChannelHash, arg.Iatas, + arg.KeyKnown, arg.PageLimit, ) if err != nil { @@ -3735,6 +3839,54 @@ func (q *Queries) ListPacketsByIATAs(ctx context.Context, arg ListPacketsByIATAs return items, nil } +const listRegionState = `-- name: ListRegionState :many +SELECT r.slug, r.name, COALESCE(r.display_order, 0)::int AS display_order, r.imported, r.center_lat, r.center_lng, + COALESCE(array_agg(ri.iata::text ORDER BY ri.iata) FILTER (WHERE ri.iata IS NOT NULL), '{}')::text[] AS iatas +FROM regions r +LEFT JOIN region_iatas ri ON ri.region_id = r.id +GROUP BY r.id +ORDER BY r.slug +` + +type ListRegionStateRow struct { + Slug string `json:"slug"` + Name string `json:"name"` + DisplayOrder int32 `json:"display_order"` + Imported bool `json:"imported"` + CenterLat *float64 `json:"center_lat"` + CenterLng *float64 `json:"center_lng"` + Iatas []string `json:"iatas"` +} + +// Every region with its members, for reconciling imported MeshMapper groups. +func (q *Queries) ListRegionState(ctx context.Context) ([]ListRegionStateRow, error) { + rows, err := q.db.Query(ctx, listRegionState) + if err != nil { + return nil, err + } + defer rows.Close() + items := []ListRegionStateRow{} + for rows.Next() { + var i ListRegionStateRow + if err := rows.Scan( + &i.Slug, + &i.Name, + &i.DisplayOrder, + &i.Imported, + &i.CenterLat, + &i.CenterLng, + &i.Iatas, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const listRegions = `-- name: ListRegions :many SELECT id, slug, name @@ -3771,6 +3923,39 @@ func (q *Queries) ListRegions(ctx context.Context) ([]ListRegionsRow, error) { return items, nil } +const listScopeCatalogues = `-- name: ListScopeCatalogues :many +SELECT iata, url, payload, etag, checked_at, attempted_at, next_attempt, last_error FROM meshmapper_scope_catalogues ORDER BY iata, attempted_at +` + +func (q *Queries) ListScopeCatalogues(ctx context.Context) ([]MeshmapperScopeCatalogue, error) { + rows, err := q.db.Query(ctx, listScopeCatalogues) + if err != nil { + return nil, err + } + defer rows.Close() + items := []MeshmapperScopeCatalogue{} + for rows.Next() { + var i MeshmapperScopeCatalogue + if err := rows.Scan( + &i.Iata, + &i.Url, + &i.Payload, + &i.Etag, + &i.CheckedAt, + &i.AttemptedAt, + &i.NextAttempt, + &i.LastError, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const listTraceTags = `-- name: ListTraceTags :many WITH tags AS ( @@ -3908,6 +4093,37 @@ func (q *Queries) ListUndecryptedGroupTextPackets(ctx context.Context) ([]ListUn return items, nil } +const listUndecryptedGroupTextPacketsByHash = `-- name: ListUndecryptedGroupTextPacketsByHash :many +SELECT packet_hash, raw_payload FROM packets +WHERE payload_type = 5 AND decrypted IS NOT TRUE AND channel_hash = ANY($1::bytea[]) +` + +type ListUndecryptedGroupTextPacketsByHashRow struct { + PacketHash []byte `json:"packet_hash"` + RawPayload []byte `json:"raw_payload"` +} + +// Like ListUndecryptedGroupTextPackets, limited to channels that just gained a key. +func (q *Queries) ListUndecryptedGroupTextPacketsByHash(ctx context.Context, hashes [][]byte) ([]ListUndecryptedGroupTextPacketsByHashRow, error) { + rows, err := q.db.Query(ctx, listUndecryptedGroupTextPacketsByHash, hashes) + if err != nil { + return nil, err + } + defer rows.Close() + items := []ListUndecryptedGroupTextPacketsByHashRow{} + for rows.Next() { + var i ListUndecryptedGroupTextPacketsByHashRow + if err := rows.Scan(&i.PacketHash, &i.RawPayload); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const listZoneBoundaries = `-- name: ListZoneBoundaries :many SELECT iata, url, feature, etag, checked_at, attempted_at, next_attempt, last_error FROM meshmapper_zone_boundaries ORDER BY iata ` @@ -3941,6 +4157,62 @@ func (q *Queries) ListZoneBoundaries(ctx context.Context) ([]MeshmapperZoneBound return items, nil } +const listZoneLists = `-- name: ListZoneLists :many +SELECT country, payload, etag, fetched_at, attempted_at, next_attempt, last_error FROM meshmapper_zone_lists ORDER BY country +` + +func (q *Queries) ListZoneLists(ctx context.Context) ([]MeshmapperZoneList, error) { + rows, err := q.db.Query(ctx, listZoneLists) + if err != nil { + return nil, err + } + defer rows.Close() + items := []MeshmapperZoneList{} + for rows.Next() { + var i MeshmapperZoneList + if err := rows.Scan( + &i.Country, + &i.Payload, + &i.Etag, + &i.FetchedAt, + &i.AttemptedAt, + &i.NextAttempt, + &i.LastError, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const pruneImportedRegions = `-- name: PruneImportedRegions :many +DELETE FROM regions WHERE imported AND NOT (slug = ANY($1::text[])) RETURNING slug +` + +func (q *Queries) PruneImportedRegions(ctx context.Context, keep []string) ([]string, error) { + rows, err := q.db.Query(ctx, pruneImportedRegions, keep) + if err != nil { + return nil, err + } + defer rows.Close() + items := []string{} + for rows.Next() { + var slug string + if err := rows.Scan(&slug); err != nil { + return nil, err + } + items = append(items, slug) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const pruneZoneBoundaries = `-- name: PruneZoneBoundaries :many DELETE FROM meshmapper_zone_boundaries WHERE NOT (iata = ANY($1::text[])) RETURNING iata ` @@ -4459,6 +4731,45 @@ func (q *Queries) ResolvePathHashesP4(ctx context.Context, arg ResolvePathHashes return items, nil } +const saveChannelCatalogue = `-- name: SaveChannelCatalogue :exec +INSERT INTO meshmapper_channel_catalogues (iata, url, payload, etag, checked_at, attempted_at, next_attempt, last_error) +VALUES ($1, $2, $3::jsonb, $4::text, + $5::timestamptz, $6, $7, $8) +ON CONFLICT (iata, url) DO UPDATE SET + payload = COALESCE(EXCLUDED.payload, meshmapper_channel_catalogues.payload), + etag = COALESCE(EXCLUDED.etag, meshmapper_channel_catalogues.etag), + checked_at = COALESCE(EXCLUDED.checked_at, meshmapper_channel_catalogues.checked_at), + attempted_at = EXCLUDED.attempted_at, + next_attempt = EXCLUDED.next_attempt, + last_error = EXCLUDED.last_error +` + +type SaveChannelCatalogueParams struct { + Iata string `json:"iata"` + Url string `json:"url"` + Payload []byte `json:"payload"` + Etag *string `json:"etag"` + CheckedAt pgtype.Timestamptz `json:"checked_at"` + AttemptedAt pgtype.Timestamptz `json:"attempted_at"` + NextAttempt pgtype.Timestamptz `json:"next_attempt"` + LastError string `json:"last_error"` +} + +// NULL payload/etag/checked_at retain the last good list after an error or 304. +func (q *Queries) SaveChannelCatalogue(ctx context.Context, arg SaveChannelCatalogueParams) error { + _, err := q.db.Exec(ctx, saveChannelCatalogue, + arg.Iata, + arg.Url, + arg.Payload, + arg.Etag, + arg.CheckedAt, + arg.AttemptedAt, + arg.NextAttempt, + arg.LastError, + ) + return err +} + const saveScopeCatalogue = `-- name: SaveScopeCatalogue :exec WITH inserted AS ( INSERT INTO transport_scopes (name, transport_key, key_fingerprint, imported_only) @@ -4553,6 +4864,43 @@ func (q *Queries) SaveZoneBoundary(ctx context.Context, arg SaveZoneBoundaryPara return err } +const saveZoneList = `-- name: SaveZoneList :exec +INSERT INTO meshmapper_zone_lists (country, payload, etag, fetched_at, attempted_at, next_attempt, last_error) +VALUES ($1, $2::jsonb, $3::text, + $4::timestamptz, $5, $6, $7) +ON CONFLICT (country) DO UPDATE SET + payload = COALESCE(EXCLUDED.payload, meshmapper_zone_lists.payload), + etag = COALESCE(EXCLUDED.etag, meshmapper_zone_lists.etag), + fetched_at = COALESCE(EXCLUDED.fetched_at, meshmapper_zone_lists.fetched_at), + attempted_at = EXCLUDED.attempted_at, + next_attempt = EXCLUDED.next_attempt, + last_error = EXCLUDED.last_error +` + +type SaveZoneListParams struct { + Country string `json:"country"` + Payload []byte `json:"payload"` + Etag *string `json:"etag"` + FetchedAt pgtype.Timestamptz `json:"fetched_at"` + AttemptedAt pgtype.Timestamptz `json:"attempted_at"` + NextAttempt pgtype.Timestamptz `json:"next_attempt"` + LastError string `json:"last_error"` +} + +// NULL payload/etag/fetched_at retain the last good list after an error or 304. +func (q *Queries) SaveZoneList(ctx context.Context, arg SaveZoneListParams) error { + _, err := q.db.Exec(ctx, saveZoneList, + arg.Country, + arg.Payload, + arg.Etag, + arg.FetchedAt, + arg.AttemptedAt, + arg.NextAttempt, + arg.LastError, + ) + return err +} + const searchKnownRoutes = `-- name: SearchKnownRoutes :many SELECT id, node_ids, hash_prefix, iata, hop_count, first_seen, last_seen, observation_count FROM known_routes @@ -4956,6 +5304,41 @@ func (q *Queries) UpsertIATADetails(ctx context.Context, arg UpsertIATADetailsPa return err } +const upsertImportedRegion = `-- name: UpsertImportedRegion :one +INSERT INTO regions (slug, name, display_order, center_lat, center_lng, zoom_level, imported, updated_at) +VALUES ($1, $2, $3, $4, $5, NULL, TRUE, NOW()) +ON CONFLICT (slug) DO UPDATE SET + name = EXCLUDED.name, + display_order = EXCLUDED.display_order, + center_lat = EXCLUDED.center_lat, + center_lng = EXCLUDED.center_lng, + updated_at = NOW() +WHERE regions.imported +RETURNING id +` + +type UpsertImportedRegionParams struct { + Slug string `json:"slug"` + Name string `json:"name"` + DisplayOrder *int32 `json:"display_order"` + CenterLat *float64 `json:"center_lat"` + CenterLng *float64 `json:"center_lng"` +} + +// A hand-written region owns its slug: the WHERE turns a clash into no row. +func (q *Queries) UpsertImportedRegion(ctx context.Context, arg UpsertImportedRegionParams) (int32, error) { + row := q.db.QueryRow(ctx, upsertImportedRegion, + arg.Slug, + arg.Name, + arg.DisplayOrder, + arg.CenterLat, + arg.CenterLng, + ) + var id int32 + err := row.Scan(&id) + return id, err +} + const upsertKnownRoute = `-- name: UpsertKnownRoute :exec INSERT INTO known_routes (path_key, node_ids, hash_prefix, iata, hop_count) @@ -4978,7 +5361,8 @@ type UpsertKnownRouteParams struct { // ROUTES // ============================================================ // Route identity is path_key, an md5 of node_ids computed by the caller. -// Keep the latest processed representation without changing the node-chain identity. +// On conflict, observation_count and last_seen are bumped and hash_prefix follows the +// latest hearing, so evidence matches the hash width the route uses now. func (q *Queries) UpsertKnownRoute(ctx context.Context, arg UpsertKnownRouteParams) error { _, err := q.db.Exec(ctx, upsertKnownRoute, arg.PathKey, @@ -5341,6 +5725,7 @@ ON CONFLICT (slug) DO UPDATE SET center_lat = EXCLUDED.center_lat, center_lng = EXCLUDED.center_lng, zoom_level = EXCLUDED.zoom_level, + imported = FALSE, -- config owns the slug from now on updated_at = NOW() RETURNING id ` @@ -5370,22 +5755,6 @@ func (q *Queries) UpsertRegion(ctx context.Context, arg UpsertRegionParams) (int return id, err } -const upsertRegionIATA = `-- name: UpsertRegionIATA :exec -INSERT INTO region_iatas (region_id, iata) -VALUES ($1, $2) -ON CONFLICT (region_id, iata) DO NOTHING -` - -type UpsertRegionIATAParams struct { - RegionID int32 `json:"region_id"` - Iata string `json:"iata"` -} - -func (q *Queries) UpsertRegionIATA(ctx context.Context, arg UpsertRegionIATAParams) error { - _, err := q.db.Exec(ctx, upsertRegionIATA, arg.RegionID, arg.Iata) - return err -} - const upsertTraceIATA = `-- name: UpsertTraceIATA :exec INSERT INTO trace_iatas (trace_tag, iata, last_heard) VALUES ($1, $2, $3) diff --git a/docs/docs.go b/docs/docs.go index 81b8afc..c045486 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -597,16 +597,22 @@ const docTemplate = `{ }, { "type": "string", - "description": "Filter by IATA code", + "description": "Filter by IATA code: channels MeshMapper lists there, config channels scoped to a region containing it, and Beacon-wide config channels", "name": "iata", "in": "query" }, { "type": "string", - "description": "Filter by IATA code(s), comma-separated e.g. YOW or YOW,YYZ", + "description": "Filter by IATA code(s), comma-separated e.g. YOW or YOW,YYZ; same membership rule as iata", "name": "iatas", "in": "query" }, + { + "type": "boolean", + "description": "Only channels Beacon can (true) or cannot (false) decrypt; omit for all", + "name": "keyKnown", + "in": "query" + }, { "type": "integer", "description": "last_seen epoch ms of last item for pagination; 0 starts from the beginning", @@ -2233,7 +2239,7 @@ const docTemplate = `{ }, "/scope-catalogues": { "get": { - "description": "Configured sources only. Counts describe the source region, not individual nodes or links. No upstream request is made. An empty list means no configured importer; checkedAt=0 means no successful check. lastError or a past freshUntil marks stale metadata. Manual Beacon scopes and packet evidence are separate.", + "description": "Discovered sources only. Counts describe the source region, not individual nodes or links. No upstream request is made. An empty list means no configured importer; checkedAt=0 means no successful check. lastError or a past freshUntil marks stale metadata. Manual Beacon scopes and packet evidence are separate.", "produces": [ "application/json" ], @@ -2247,7 +2253,7 @@ const docTemplate = `{ "schema": { "type": "array", "items": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.Catalogue" + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.PublicScopeCatalogue" } } } @@ -2256,7 +2262,7 @@ const docTemplate = `{ }, "/scopes": { "get": { - "description": "The unfiltered list includes stored imported names after an importer is disabled or a source is removed. Those historical identities are retained indefinitely; listing them does not establish observed traffic or current source membership.", + "description": "Without filters, lists every stored scope name, including imported names retained after an importer is disabled or a source is removed. With IATA or region filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; observed traffic does not add scopes.", "produces": [ "application/json" ], @@ -2288,7 +2294,16 @@ const docTemplate = `{ "200": { "description": "OK", "schema": { - "type": "object" + "type": "array", + "items": { + "type": "string" + } + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" } }, "500": { @@ -2817,7 +2832,7 @@ const docTemplate = `{ }, "/stats/scopes": { "get": { - "description": "Counts each packet, observer and node once per scope. IATA filters use retained observations for packets/observers and node IATA memberships for nodes. Without filters, returns global totals. Scopes with zero matching counts remain listed; an empty region returns an empty array.", + "description": "Counts each packet, observer and node once per scope. IATA filters use retained observations for packets/observers and node IATA memberships for nodes. Without filters, returns global totals for every stored scope. With filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; those remain listed with zero counts. An empty region returns an empty array.", "produces": [ "application/json" ], @@ -5465,7 +5480,27 @@ const docTemplate = `{ } } }, - "github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.Catalogue": { + "github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.CatalogueScope": { + "type": "object", + "properties": { + "default": { + "type": "integer" + }, + "monitored": { + "type": "boolean" + }, + "name": { + "type": "string" + }, + "repeaters": { + "type": "integer" + }, + "wardriving": { + "type": "boolean" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.PublicScopeCatalogue": { "type": "object", "properties": { "checkedAt": { @@ -5500,26 +5535,6 @@ const docTemplate = `{ } } }, - "github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.CatalogueScope": { - "type": "object", - "properties": { - "default": { - "type": "integer" - }, - "monitored": { - "type": "boolean" - }, - "name": { - "type": "string" - }, - "repeaters": { - "type": "integer" - }, - "wardriving": { - "type": "boolean" - } - } - }, "internal_api_handlers.APIError": { "type": "object", "properties": { @@ -5595,7 +5610,7 @@ const docTemplate = `{ // SwaggerInfo holds exported Swagger Info so clients can modify it var SwaggerInfo = &swag.Spec{ - Version: "1.6.0", + Version: "2.0.0", Host: "localhost:8080", BasePath: "/api/v1", Schemes: []string{"http", "https"}, diff --git a/docs/swagger.json b/docs/swagger.json index ca0e504..4f3ff4b 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -15,7 +15,7 @@ "license": { "name": "AGPL-3-or-later" }, - "version": "1.6.0" + "version": "2.0.0" }, "host": "localhost:8080", "basePath": "/api/v1", @@ -595,16 +595,22 @@ }, { "type": "string", - "description": "Filter by IATA code", + "description": "Filter by IATA code: channels MeshMapper lists there, config channels scoped to a region containing it, and Beacon-wide config channels", "name": "iata", "in": "query" }, { "type": "string", - "description": "Filter by IATA code(s), comma-separated e.g. YOW or YOW,YYZ", + "description": "Filter by IATA code(s), comma-separated e.g. YOW or YOW,YYZ; same membership rule as iata", "name": "iatas", "in": "query" }, + { + "type": "boolean", + "description": "Only channels Beacon can (true) or cannot (false) decrypt; omit for all", + "name": "keyKnown", + "in": "query" + }, { "type": "integer", "description": "last_seen epoch ms of last item for pagination; 0 starts from the beginning", @@ -2231,7 +2237,7 @@ }, "/scope-catalogues": { "get": { - "description": "Configured sources only. Counts describe the source region, not individual nodes or links. No upstream request is made. An empty list means no configured importer; checkedAt=0 means no successful check. lastError or a past freshUntil marks stale metadata. Manual Beacon scopes and packet evidence are separate.", + "description": "Discovered sources only. Counts describe the source region, not individual nodes or links. No upstream request is made. An empty list means no configured importer; checkedAt=0 means no successful check. lastError or a past freshUntil marks stale metadata. Manual Beacon scopes and packet evidence are separate.", "produces": [ "application/json" ], @@ -2245,7 +2251,7 @@ "schema": { "type": "array", "items": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.Catalogue" + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.PublicScopeCatalogue" } } } @@ -2254,7 +2260,7 @@ }, "/scopes": { "get": { - "description": "The unfiltered list includes stored imported names after an importer is disabled or a source is removed. Those historical identities are retained indefinitely; listing them does not establish observed traffic or current source membership.", + "description": "Without filters, lists every stored scope name, including imported names retained after an importer is disabled or a source is removed. With IATA or region filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; observed traffic does not add scopes.", "produces": [ "application/json" ], @@ -2286,7 +2292,16 @@ "200": { "description": "OK", "schema": { - "type": "object" + "type": "array", + "items": { + "type": "string" + } + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" } }, "500": { @@ -2815,7 +2830,7 @@ }, "/stats/scopes": { "get": { - "description": "Counts each packet, observer and node once per scope. IATA filters use retained observations for packets/observers and node IATA memberships for nodes. Without filters, returns global totals. Scopes with zero matching counts remain listed; an empty region returns an empty array.", + "description": "Counts each packet, observer and node once per scope. IATA filters use retained observations for packets/observers and node IATA memberships for nodes. Without filters, returns global totals for every stored scope. With filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; those remain listed with zero counts. An empty region returns an empty array.", "produces": [ "application/json" ], @@ -5463,7 +5478,27 @@ } } }, - "github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.Catalogue": { + "github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.CatalogueScope": { + "type": "object", + "properties": { + "default": { + "type": "integer" + }, + "monitored": { + "type": "boolean" + }, + "name": { + "type": "string" + }, + "repeaters": { + "type": "integer" + }, + "wardriving": { + "type": "boolean" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.PublicScopeCatalogue": { "type": "object", "properties": { "checkedAt": { @@ -5498,26 +5533,6 @@ } } }, - "github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.CatalogueScope": { - "type": "object", - "properties": { - "default": { - "type": "integer" - }, - "monitored": { - "type": "boolean" - }, - "name": { - "type": "string" - }, - "repeaters": { - "type": "integer" - }, - "wardriving": { - "type": "boolean" - } - } - }, "internal_api_handlers.APIError": { "type": "object", "properties": { diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 796cc77..fe15e08 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -1492,7 +1492,20 @@ definitions: requires_restart: type: boolean type: object - github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.Catalogue: + github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.CatalogueScope: + properties: + default: + type: integer + monitored: + type: boolean + name: + type: string + repeaters: + type: integer + wardriving: + type: boolean + type: object + github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.PublicScopeCatalogue: properties: checkedAt: type: integer @@ -1515,19 +1528,6 @@ definitions: url: type: string type: object - github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.CatalogueScope: - properties: - default: - type: integer - monitored: - type: boolean - name: - type: string - repeaters: - type: integer - wardriving: - type: boolean - type: object internal_api_handlers.APIError: properties: code: @@ -1557,7 +1557,7 @@ info: name: AGPL-3-or-later termsOfService: https://github.com/MeshCore-Beacon/beacon-server title: MeshCore Beacon API - version: 1.6.0 + version: 2.0.0 paths: /admin/accounts: get: @@ -1939,14 +1939,21 @@ paths: in: query name: hash type: string - - description: Filter by IATA code + - description: 'Filter by IATA code: channels MeshMapper lists there, config + channels scoped to a region containing it, and Beacon-wide config channels' in: query name: iata type: string - - description: Filter by IATA code(s), comma-separated e.g. YOW or YOW,YYZ + - description: Filter by IATA code(s), comma-separated e.g. YOW or YOW,YYZ; + same membership rule as iata in: query name: iatas type: string + - description: Only channels Beacon can (true) or cannot (false) decrypt; omit + for all + in: query + name: keyKnown + type: boolean - description: last_seen epoch ms of last item for pagination; 0 starts from the beginning in: query @@ -3064,7 +3071,7 @@ paths: - Routes /scope-catalogues: get: - description: Configured sources only. Counts describe the source region, not + description: Discovered sources only. Counts describe the source region, not individual nodes or links. No upstream request is made. An empty list means no configured importer; checkedAt=0 means no successful check. lastError or a past freshUntil marks stale metadata. Manual Beacon scopes and packet evidence @@ -3076,17 +3083,18 @@ paths: description: OK schema: items: - $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.Catalogue' + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.PublicScopeCatalogue' type: array summary: Cached MeshMapper regional scope catalogues tags: - Scopes /scopes: get: - description: The unfiltered list includes stored imported names after an importer - is disabled or a source is removed. Those historical identities are retained - indefinitely; listing them does not establish observed traffic or current - source membership. + description: Without filters, lists every stored scope name, including imported + names retained after an importer is disabled or a source is removed. With + IATA or region filters, lists only manual scopes configured for a matching + region and imported scopes whose current MeshMapper catalogue includes a matching + IATA; observed traffic does not add scopes. parameters: - description: Filter by IATA code(s), comma-separated in: query @@ -3106,7 +3114,13 @@ paths: "200": description: OK schema: - type: object + items: + type: string + type: array + "400": + description: Bad Request + schema: + $ref: '#/definitions/internal_api_handlers.APIError' "500": description: Internal Server Error schema: @@ -3470,8 +3484,10 @@ paths: get: description: Counts each packet, observer and node once per scope. IATA filters use retained observations for packets/observers and node IATA memberships - for nodes. Without filters, returns global totals. Scopes with zero matching - counts remain listed; an empty region returns an empty array. + for nodes. Without filters, returns global totals for every stored scope. + With filters, lists only manual scopes configured for a matching region and + imported scopes whose current MeshMapper catalogue includes a matching IATA; + those remain listed with zero counts. An empty region returns an empty array. parameters: - description: Comma-separated IATA codes in: query diff --git a/internal/api/handlers/channels.go b/internal/api/handlers/channels.go index 81ee792..9bf80d5 100644 --- a/internal/api/handlers/channels.go +++ b/internal/api/handlers/channels.go @@ -35,8 +35,9 @@ func ChannelsRouter(reader api.Reader) http.Handler { // @Tags Channels // @Produce json // @Param hash query string false "Single-byte channel hash (hex)" -// @Param iata query string false "Filter by IATA code" -// @Param iatas query string false "Filter by IATA code(s), comma-separated e.g. YOW or YOW,YYZ" +// @Param iata query string false "Filter by IATA code: channels MeshMapper lists there, config channels scoped to a region containing it, and Beacon-wide config channels" +// @Param iatas query string false "Filter by IATA code(s), comma-separated e.g. YOW or YOW,YYZ; same membership rule as iata" +// @Param keyKnown query bool false "Only channels Beacon can (true) or cannot (false) decrypt; omit for all" // @Param cursor query int false "last_seen epoch ms of last item for pagination; 0 starts from the beginning" // @Param pageCursor query string false "Opaque nextPageCursor from a previous response; preserves timestamp ties and precision. Cannot be combined with a positive cursor; cursor=0 is allowed." // @Param limit query int false "Max results (default 50); must be positive, values above 200 are clamped" minimum(1) maximum(200) @@ -86,7 +87,19 @@ func listChannels(reader api.Reader) http.HandlerFunc { } hashHex = h } - channels, err := reader.ListChannels(r.Context(), limit, hashHex, iatas, cursor, pageCursor) + var keyKnown *bool + if q := r.URL.Query(); q.Has("keyKnown") { + switch q.Get("keyKnown") { + case "true": + keyKnown = new(true) + case "false": + keyKnown = new(false) + default: + respondError(w, http.StatusBadRequest, "keyKnown must be true or false") + return + } + } + channels, err := reader.ListChannels(r.Context(), limit, hashHex, iatas, keyKnown, cursor, pageCursor) if err != nil { respondError(w, http.StatusInternalServerError, "internal server error") return diff --git a/internal/api/handlers/channels_test.go b/internal/api/handlers/channels_test.go index 8b964d7..00bc5d7 100644 --- a/internal/api/handlers/channels_test.go +++ b/internal/api/handlers/channels_test.go @@ -8,6 +8,7 @@ import ( "net/http" "net/http/httptest" "reflect" + "strings" "testing" "time" @@ -39,7 +40,7 @@ func TestListChannels_InvalidCursor(t *testing.T) { func TestListChannels_PageCursor(t *testing.T) { called := false - r := ChannelsRouter(stubReader{listChannels: func(_ context.Context, limit int32, hash []byte, iatas []string, legacy int64, cursor *api.ChannelCursor) (api.ChannelPage, error) { + r := ChannelsRouter(stubReader{listChannels: func(_ context.Context, limit int32, hash []byte, iatas []string, _ *bool, legacy int64, cursor *api.ChannelCursor) (api.ChannelPage, error) { called = true if limit != 2 || legacy != 0 || cursor == nil || cursor.ID != 9 || !cursor.LastSeen.Equal(time.UnixMicro(1700000000000123)) || !reflect.DeepEqual(hash, []byte{0xaa}) || !reflect.DeepEqual(iatas, []string{"YOW"}) { @@ -145,7 +146,7 @@ func TestListChannelMessages_InvalidCursor(t *testing.T) { func TestListChannels_OK(t *testing.T) { r := chi.NewRouter() r.Get("/channels", listChannels(stubReader{ - listChannels: func(_ context.Context, _ int32, _ []byte, _ []string, _ int64, _ *api.ChannelCursor) (api.ChannelPage, error) { + listChannels: func(_ context.Context, _ int32, _ []byte, _ []string, _ *bool, _ int64, _ *api.ChannelCursor) (api.ChannelPage, error) { return api.ChannelPage{Page: api.Page[api.ChannelSummary]{Items: []api.ChannelSummary{{ID: 1, ChannelHash: "ab"}}}}, nil }, })) @@ -172,7 +173,7 @@ func TestListChannels_IATAParsing(t *testing.T) { var got []string r := chi.NewRouter() r.Get("/channels", listChannels(stubReader{ - listChannels: func(_ context.Context, _ int32, _ []byte, iatas []string, _ int64, _ *api.ChannelCursor) (api.ChannelPage, error) { + listChannels: func(_ context.Context, _ int32, _ []byte, iatas []string, _ *bool, _ int64, _ *api.ChannelCursor) (api.ChannelPage, error) { got = iatas return api.ChannelPage{}, nil }, @@ -190,6 +191,54 @@ func TestListChannels_IATAParsing(t *testing.T) { } } +func TestListChannels_KeyKnownParsing(t *testing.T) { + yes, no := true, false + cases := []struct { + name string + query string + code int + want *bool + }{ + {"true", "?keyKnown=true", http.StatusOK, &yes}, + {"false", "?keyKnown=false", http.StatusOK, &no}, + {"absent", "", http.StatusOK, nil}, + {"uppercase", "?keyKnown=TRUE", http.StatusBadRequest, nil}, + {"numeric", "?keyKnown=1", http.StatusBadRequest, nil}, + {"empty", "?keyKnown=", http.StatusBadRequest, nil}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var got *bool + called := false + r := chi.NewRouter() + r.Get("/channels", listChannels(stubReader{ + listChannels: func(_ context.Context, _ int32, _ []byte, _ []string, keyKnown *bool, _ int64, _ *api.ChannelCursor) (api.ChannelPage, error) { + called = true + got = keyKnown + return api.ChannelPage{}, nil + }, + })) + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/channels"+tc.query, nil)) + if w.Code != tc.code { + t.Fatalf("expected %d, got %d", tc.code, w.Code) + } + if tc.code != http.StatusOK { + if called { + t.Error("reader called for invalid keyKnown") + } + if !strings.Contains(w.Body.String(), "keyKnown must be true or false") { + t.Errorf("unexpected error body %s", w.Body.String()) + } + return + } + if !reflect.DeepEqual(got, tc.want) { + t.Errorf("expected keyKnown %v, got %v", tc.want, got) + } + }) + } +} + func TestGetChannel_OK(t *testing.T) { r := chi.NewRouter() r.Get("/channels/{channelID}", getChannel(stubReader{ diff --git a/internal/api/handlers/limits_test.go b/internal/api/handlers/limits_test.go index 991a207..3d7e3a2 100644 --- a/internal/api/handlers/limits_test.go +++ b/internal/api/handlers/limits_test.go @@ -19,7 +19,7 @@ func TestListLimits(t *testing.T) { var got int32 var calls int reader := stubReader{ - listChannels: func(ctx context.Context, limit int32, hash []byte, iatas []string, cursor int64, pageCursor *api.ChannelCursor) (api.ChannelPage, error) { + listChannels: func(ctx context.Context, limit int32, hash []byte, iatas []string, keyKnown *bool, cursor int64, pageCursor *api.ChannelCursor) (api.ChannelPage, error) { calls++ got = limit return api.ChannelPage{}, nil @@ -113,7 +113,7 @@ func TestListLimits(t *testing.T) { router.Mount("/messages", MessagesRouter(reader)) router.Mount("/routes", RoutesRouter(reader)) router.Mount("/traces", TracesRouter(reader)) - router.Mount("/stats", StatsRouter(reader)) + router.Mount("/stats", StatsRouter(reader, nil)) for _, endpoint := range []struct { path string defaultLimit int32 diff --git a/internal/api/handlers/observer_comparison_test.go b/internal/api/handlers/observer_comparison_test.go index 3a46da7..0b11e6d 100644 --- a/internal/api/handlers/observer_comparison_test.go +++ b/internal/api/handlers/observer_comparison_test.go @@ -28,7 +28,7 @@ func TestObserverComparisonRejectsInvalidQuery(t *testing.T) { } { t.Run(query, func(t *testing.T) { w := httptest.NewRecorder() - StatsRouter(nil).ServeHTTP(w, httptest.NewRequest("GET", "/observer-comparison?"+query, nil)) + StatsRouter(nil, nil).ServeHTTP(w, httptest.NewRequest("GET", "/observer-comparison?"+query, nil)) if w.Code != 400 { t.Fatalf("status %d; wanted validation error", w.Code) } @@ -42,7 +42,7 @@ func TestObserverComparisonHTTP(t *testing.T) { query := "observerA=" + a.String() + "&observerB=" + b.String() + "&since=0&until=1000" for _, suffix := range []string{"&since=2", "&observerA=" + b.String(), "&until=253402300800000", "&observerB=00000000-0000-0000-0000-000000000000"} { w := httptest.NewRecorder() - StatsRouter(nil).ServeHTTP(w, httptest.NewRequest("GET", "/observer-comparison?"+query+suffix, nil)) + StatsRouter(nil, nil).ServeHTTP(w, httptest.NewRequest("GET", "/observer-comparison?"+query+suffix, nil)) if w.Code != 400 { t.Fatalf("invalid query returned %d", w.Code) } @@ -80,7 +80,7 @@ func TestObserverComparisonHTTP(t *testing.T) { }, } w := httptest.NewRecorder() - StatsRouter(reader).ServeHTTP(w, httptest.NewRequest("GET", "/observer-comparison?"+query+tc.suffix, nil)) + StatsRouter(reader, nil).ServeHTTP(w, httptest.NewRequest("GET", "/observer-comparison?"+query+tc.suffix, nil)) if w.Code != tc.status { t.Fatalf("status = %d, body = %s", w.Code, w.Body.String()) } diff --git a/internal/api/handlers/paths_test.go b/internal/api/handlers/paths_test.go index 6447346..190fe69 100644 --- a/internal/api/handlers/paths_test.go +++ b/internal/api/handlers/paths_test.go @@ -28,7 +28,7 @@ func (s pathReader) GetPathStats(ctx context.Context, since, until time.Time, ia func TestPathsRejectInvalidWindow(t *testing.T) { for _, query := range []string{"", "since=0", "since=x&until=1", "since=-1&until=1", "since=1&until=1", "since=2&until=1", "since=0&until=2592000001", "since=0&since=0&until=1", "since=0&until=1&until=2", "since=253402300799998&until=253402300800000"} { w := httptest.NewRecorder() - StatsRouter(nil).ServeHTTP(w, httptest.NewRequest("GET", "/paths?"+query, nil)) + StatsRouter(nil, nil).ServeHTTP(w, httptest.NewRequest("GET", "/paths?"+query, nil)) if w.Code != 400 { t.Fatalf("query %q: %d %s", query, w.Code, w.Body.String()) } @@ -66,7 +66,7 @@ func TestPathsFiltersErrorsAndCancellation(t *testing.T) { return &api.PathStats{Until: until.UnixMilli(), Receptions: 4, Hashed: 1, Empty: 1, Trace: 1, Unclassified: 1}, tc.err }} w := httptest.NewRecorder() - StatsRouter(reader).ServeHTTP(w, httptest.NewRequest("GET", "/paths?since=0&until=2592000000"+tc.query, nil)) + StatsRouter(reader, nil).ServeHTTP(w, httptest.NewRequest("GET", "/paths?since=0&until=2592000000"+tc.query, nil)) if w.Code != tc.status || strings.Contains(w.Body.String(), "private detail") { t.Fatalf("%d %s", w.Code, w.Body.String()) } @@ -86,7 +86,7 @@ func TestPathsFiltersErrorsAndCancellation(t *testing.T) { return nil, ctx.Err() }} w := httptest.NewRecorder() - StatsRouter(r).ServeHTTP(w, httptest.NewRequest("GET", "/paths?since=0&until=1", nil).WithContext(ctx)) + StatsRouter(r, nil).ServeHTTP(w, httptest.NewRequest("GET", "/paths?since=0&until=1", nil).WithContext(ctx)) if !called || w.Body.Len() != 0 { t.Fatal("cancellation did not reach reader") } diff --git a/internal/api/handlers/scope_catalogues.go b/internal/api/handlers/scope_catalogues.go index 86ec772..886a874 100644 --- a/internal/api/handlers/scope_catalogues.go +++ b/internal/api/handlers/scope_catalogues.go @@ -11,19 +11,19 @@ import ( // ScopeCatalogueSource supplies immutable, already-cached public metadata. type ScopeCatalogueSource interface { - Catalogues() []meshmapper.Catalogue + Catalogues() []meshmapper.PublicScopeCatalogue } // ScopeCatalogues godoc // @Summary Cached MeshMapper regional scope catalogues -// @Description Configured sources only. Counts describe the source region, not individual nodes or links. No upstream request is made. An empty list means no configured importer; checkedAt=0 means no successful check. lastError or a past freshUntil marks stale metadata. Manual Beacon scopes and packet evidence are separate. +// @Description Discovered sources only. Counts describe the source region, not individual nodes or links. No upstream request is made. An empty list means no configured importer; checkedAt=0 means no successful check. lastError or a past freshUntil marks stale metadata. Manual Beacon scopes and packet evidence are separate. // @Tags Scopes // @Produce json -// @Success 200 {array} meshmapper.Catalogue +// @Success 200 {array} meshmapper.PublicScopeCatalogue // @Router /scope-catalogues [get] func ScopeCatalogues(source ScopeCatalogueSource) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - items := []meshmapper.Catalogue{} + items := []meshmapper.PublicScopeCatalogue{} if source != nil { items = source.Catalogues() } diff --git a/internal/api/handlers/scope_stats_test.go b/internal/api/handlers/scope_stats_test.go index d63ba21..111681d 100644 --- a/internal/api/handlers/scope_stats_test.go +++ b/internal/api/handlers/scope_stats_test.go @@ -13,6 +13,7 @@ import ( "testing" "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" ) func TestGetStatsScopes_RegionErrors(t *testing.T) { @@ -27,7 +28,7 @@ func TestGetStatsScopes_RegionErrors(t *testing.T) { }, } w := httptest.NewRecorder() - getStatsScopes(reader)(w, httptest.NewRequest(http.MethodGet, "/stats/scopes?"+query, nil)) + getStatsScopes(reader, nil)(w, httptest.NewRequest(http.MethodGet, "/stats/scopes?"+query, nil)) if w.Code != http.StatusBadRequest { t.Fatalf("got status %d, want 400", w.Code) } @@ -79,7 +80,7 @@ func TestGetStatsScopes_Filters(t *testing.T) { }, } w := httptest.NewRecorder() - getStatsScopes(reader)(w, httptest.NewRequest(http.MethodGet, "/stats/scopes?"+tc.query, nil)) + getStatsScopes(reader, testScopes(map[string][]string{"YVR": {"#test"}, "YYJ": {"#test"}}))(w, httptest.NewRequest(http.MethodGet, "/stats/scopes?"+tc.query, nil)) if w.Code != http.StatusOK { t.Fatalf("status = %d: %s", w.Code, w.Body.String()) } @@ -105,8 +106,45 @@ func TestGetStatsScopes_ReaderError(t *testing.T) { w := httptest.NewRecorder() getStatsScopes(stubReader{getScopeStats: func(context.Context, []string) ([]api.ScopeStats, error) { return nil, errors.New("database unavailable") - }})(w, httptest.NewRequest(http.MethodGet, "/stats/scopes?iata=YVR", nil)) + }}, nil)(w, httptest.NewRequest(http.MethodGet, "/stats/scopes?iata=YVR", nil)) if w.Code != http.StatusInternalServerError { t.Fatalf("status = %d, want 500", w.Code) } } + +func testScopes(byIATA map[string][]string) *scopestore.ScopeStore { + s := scopestore.New() + s.SetManualMembers(byIATA) + return s +} + +func TestGetStatsScopes_FiltersToRegionMembers(t *testing.T) { + reader := stubReader{getScopeStats: func(context.Context, []string) ([]api.ScopeStats, error) { + return []api.ScopeStats{{Name: "#elsewhere", PacketCount: 9}, {Name: "#on"}, {Name: "#yow", PacketCount: 4}}, nil + }} + members := testScopes(map[string][]string{"YOW": {"#yow", "#on"}}) + for _, tc := range []struct { + query string + members ScopeMembership + want []string + }{ + {"", members, []string{"#elsewhere", "#on", "#yow"}}, + {"iata=YOW", members, []string{"#on", "#yow"}}, + {"iata=YUL", members, []string{}}, + {"iata=YOW", nil, []string{}}, + } { + w := httptest.NewRecorder() + getStatsScopes(reader, tc.members)(w, httptest.NewRequest(http.MethodGet, "/stats/scopes?"+tc.query, nil)) + var rows []api.ScopeStats + if err := json.Unmarshal(w.Body.Bytes(), &rows); err != nil || rows == nil { + t.Fatalf("%q: body %s, err %v", tc.query, w.Body.String(), err) + } + got := []string{} + for _, row := range rows { + got = append(got, row.Name) + } + if !slices.Equal(got, tc.want) { + t.Fatalf("%q: got %v, want %v", tc.query, got, tc.want) + } + } +} diff --git a/internal/api/handlers/scopes.go b/internal/api/handlers/scopes.go index c86de6f..41cca54 100644 --- a/internal/api/handlers/scopes.go +++ b/internal/api/handlers/scopes.go @@ -10,13 +10,25 @@ import ( "github.com/go-chi/chi/v5" ) +// ScopeMembership lists scopes by configured region and imported catalogue membership. +type ScopeMembership interface { + NamesForIATAs(iatas []string) []string // sorted +} + +func scopeNamesFor(scopes ScopeMembership, iatas []string) []string { + if scopes == nil { + return []string{} + } + return scopes.NamesForIATAs(iatas) +} + // ScopesRouter mounts all /scopes routes onto a subrouter. // // GET /scopes → listScopes // GET /scopes/{name} → getScope -func ScopesRouter(reader api.Reader) http.Handler { +func ScopesRouter(reader api.Reader, scopes ScopeMembership) http.Handler { r := chi.NewRouter() - r.Get("/", listScopes(reader)) + r.Get("/", listScopes(reader, scopes)) r.Get("/{name}", getScope(reader)) return r } @@ -24,27 +36,28 @@ func ScopesRouter(reader api.Reader) http.Handler { // listScopes godoc // // @Summary List transport scopes -// @Description The unfiltered list includes stored imported names after an importer is disabled or a source is removed. Those historical identities are retained indefinitely; listing them does not establish observed traffic or current source membership. +// @Description Without filters, lists every stored scope name, including imported names retained after an importer is disabled or a source is removed. With IATA or region filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; observed traffic does not add scopes. // @Tags Scopes // @Produce json // @Param iatas query string false "Filter by IATA code(s), comma-separated" // @Param region query string false "Filter by region slug" // @Param regionId query int false "Filter by region ID" -// @Success 200 {object} object +// @Success 200 {array} string +// @Failure 400 {object} handlers.APIError // @Failure 500 {object} handlers.APIError // @Router /scopes [get] -func listScopes(reader api.Reader) http.HandlerFunc { +func listScopes(reader api.Reader, scopes ScopeMembership) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { iatas := parseIATAs(r) - if regionIDStr := r.URL.Query().Get("regionId"); regionIDStr != "" || r.URL.Query().Get("region") != "" { - regionIATAs, err := resolveRegionIATAs(r.Context(), r.URL.Query().Get("regionId"), r.URL.Query().Get("region"), reader) + regionID, region := r.URL.Query().Get("regionId"), r.URL.Query().Get("region") + if regionID != "" || region != "" { + regionIATAs, err := resolveRegionIATAs(r.Context(), regionID, region, reader) if err != nil { respondError(w, http.StatusBadRequest, err.Error()) return } iatas = append(iatas, regionIATAs...) - } - if len(iatas) == 0 { + } else if len(iatas) == 0 { names, err := reader.GetScopeNames(r.Context()) if err != nil { respondError(w, http.StatusInternalServerError, "internal server error") @@ -53,12 +66,7 @@ func listScopes(reader api.Reader) http.HandlerFunc { respond(w, http.StatusOK, names) return } - scopes, err := reader.GetScopesByIATAs(r.Context(), iatas) - if err != nil { - respondError(w, http.StatusInternalServerError, "internal server error") - return - } - respond(w, http.StatusOK, scopes) + respond(w, http.StatusOK, scopeNamesFor(scopes, iatas)) } } diff --git a/internal/api/handlers/scopes_test.go b/internal/api/handlers/scopes_test.go index 29abb39..f83262e 100644 --- a/internal/api/handlers/scopes_test.go +++ b/internal/api/handlers/scopes_test.go @@ -5,8 +5,10 @@ package handlers import ( "context" + "encoding/json" "net/http" "net/http/httptest" + "slices" "testing" "github.com/MeshCore-Beacon/beacon-server/internal/api" @@ -19,7 +21,7 @@ func TestListScopes_NoIATAs_OK(t *testing.T) { getScopeNames: func(_ context.Context) ([]string, error) { return []string{"#bc", "#west"}, nil }, - })) + }, nil)) req := httptest.NewRequest(http.MethodGet, "/scopes", nil) w := httptest.NewRecorder() r.ServeHTTP(w, req) @@ -42,3 +44,40 @@ func TestGetScope_OK(t *testing.T) { t.Errorf("expected 200, got %d", w.Code) } } + +func TestListScopes_FiltersByMembership(t *testing.T) { + reader := stubReader{ + getScopeNames: func(context.Context) ([]string, error) { + t.Fatal("filtered list must not fall back to every stored name") + return nil, nil + }, + getRegionBySlug: func(_ context.Context, slug string) (*api.Region, error) { + if slug == "empty" { + return &api.Region{}, nil + } + return &api.Region{IATAs: []string{"YOW", "YGK"}}, nil + }, + } + members := testScopes(map[string][]string{"YOW": {"#on", "#yow"}, "YGK": {"#on"}, "YUL": {"#qc"}}) + for _, tc := range []struct { + query string + members ScopeMembership + want []string + }{ + {"iatas=yow", members, []string{"#on", "#yow"}}, + {"iatas=YUL,YYZ", members, []string{"#qc"}}, + {"region=ottawa", members, []string{"#on", "#yow"}}, + {"region=empty", members, []string{}}, + {"iatas=YOW", nil, []string{}}, + } { + w := httptest.NewRecorder() + listScopes(reader, tc.members)(w, httptest.NewRequest(http.MethodGet, "/scopes?"+tc.query, nil)) + var got []string + if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil || w.Code != http.StatusOK || got == nil { + t.Fatalf("%q: status %d body %s", tc.query, w.Code, w.Body.String()) + } + if !slices.Equal(got, tc.want) { + t.Fatalf("%q: got %v, want %v", tc.query, got, tc.want) + } + } +} diff --git a/internal/api/handlers/signal_test.go b/internal/api/handlers/signal_test.go index 405db1c..f9146fa 100644 --- a/internal/api/handlers/signal_test.go +++ b/internal/api/handlers/signal_test.go @@ -28,7 +28,7 @@ func (s signalReader) GetSignalStats(ctx context.Context, since, until time.Time func TestSignalRejectsInvalidWindow(t *testing.T) { for _, query := range []string{"", "since=0", "until=1", "since=x&until=1", "since=-1&until=1", "since=0&until=0", "since=2&until=1", "since=0&until=2592000001", "since=0&since=0&until=1", "since=0&until=1&until=2", "since=253402300799998&until=253402300800000"} { w := httptest.NewRecorder() - StatsRouter(nil).ServeHTTP(w, httptest.NewRequest("GET", "/signal?"+query, nil)) + StatsRouter(nil, nil).ServeHTTP(w, httptest.NewRequest("GET", "/signal?"+query, nil)) if w.Code != 400 { t.Fatalf("query %q: %d %s", query, w.Code, w.Body.String()) } @@ -70,7 +70,7 @@ func TestSignalFiltersAndErrors(t *testing.T) { return &api.SignalStats{Until: until.UnixMilli(), Receptions: 5, Hourly: []api.SignalHour{}}, tc.err }} w := httptest.NewRecorder() - StatsRouter(reader).ServeHTTP(w, httptest.NewRequest("GET", "/signal?since=0&until=2592000000"+tc.query, nil)) + StatsRouter(reader, nil).ServeHTTP(w, httptest.NewRequest("GET", "/signal?since=0&until=2592000000"+tc.query, nil)) if w.Code != tc.status || strings.Contains(w.Body.String(), "private detail") { t.Fatalf("%d %s", w.Code, w.Body.String()) } @@ -93,7 +93,7 @@ func TestSignalCancellationReachesReader(t *testing.T) { return nil, ctx.Err() }} w := httptest.NewRecorder() - StatsRouter(reader).ServeHTTP(w, httptest.NewRequest("GET", "/signal?since=0&until=1", nil).WithContext(ctx)) + StatsRouter(reader, nil).ServeHTTP(w, httptest.NewRequest("GET", "/signal?since=0&until=1", nil).WithContext(ctx)) if !called || w.Body.Len() != 0 { t.Fatalf("cancellation: called=%v body=%s", called, w.Body.String()) } diff --git a/internal/api/handlers/stats.go b/internal/api/handlers/stats.go index 1309efd..ca06423 100644 --- a/internal/api/handlers/stats.go +++ b/internal/api/handlers/stats.go @@ -6,6 +6,7 @@ package handlers import ( "log/slog" "net/http" + "slices" "strconv" "time" @@ -26,7 +27,7 @@ import ( // GET /stats/scopes → GetStatsScopes // // All endpoints accept an optional iata= filter (case-insensitive). -func StatsRouter(reader api.Reader) http.Handler { +func StatsRouter(reader api.Reader, scopes ScopeMembership) http.Handler { r := chi.NewRouter() r.Get("/overview", getStatsOverview(reader)) r.Get("/observations", getStatsObservations(reader)) @@ -40,7 +41,7 @@ func StatsRouter(reader api.Reader) http.Handler { r.Get("/clock-drift", getStatsClockDrift(reader)) r.Get("/top-talkers", getStatsTopTalkers(reader)) r.Get("/radio-presets", getStatsRadioPresets(reader)) - r.Get("/scopes", getStatsScopes(reader)) + r.Get("/scopes", getStatsScopes(reader, scopes)) r.Get("/node-types", getStatsNodeTypes(reader)) return r } @@ -417,7 +418,7 @@ func getStatsRadioPresets(reader api.Reader) http.HandlerFunc { // getStatsScopes godoc // // @Summary Scope statistics -// @Description Counts each packet, observer and node once per scope. IATA filters use retained observations for packets/observers and node IATA memberships for nodes. Without filters, returns global totals. Scopes with zero matching counts remain listed; an empty region returns an empty array. +// @Description Counts each packet, observer and node once per scope. IATA filters use retained observations for packets/observers and node IATA memberships for nodes. Without filters, returns global totals for every stored scope. With filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; those remain listed with zero counts. An empty region returns an empty array. // @Tags Stats // @Produce json // @Param iatas query string false "Comma-separated IATA codes" @@ -428,7 +429,7 @@ func getStatsRadioPresets(reader api.Reader) http.HandlerFunc { // @Failure 400 {object} handlers.APIError // @Failure 500 {object} handlers.APIError // @Router /stats/scopes [get] -func getStatsScopes(reader api.Reader) http.HandlerFunc { +func getStatsScopes(reader api.Reader, scopes ScopeMembership) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { iatas := parseIATAs(r) if regionID := r.URL.Query().Get("regionId"); regionID != "" || r.URL.Query().Get("region") != "" { @@ -448,6 +449,13 @@ func getStatsScopes(reader api.Reader) http.HandlerFunc { respondError(w, http.StatusInternalServerError, "internal server error") return } + if len(iatas) > 0 { + members := scopeNamesFor(scopes, iatas) + stats = slices.DeleteFunc(slices.Clone(stats), func(s api.ScopeStats) bool { + _, found := slices.BinarySearch(members, s.Name) + return !found + }) + } respond(w, http.StatusOK, stats) } } diff --git a/internal/api/handlers/stats_test.go b/internal/api/handlers/stats_test.go index 8225fc8..86d9d9b 100644 --- a/internal/api/handlers/stats_test.go +++ b/internal/api/handlers/stats_test.go @@ -285,7 +285,7 @@ func TestGetStatsScopes_OK(t *testing.T) { getScopeStats: func(_ context.Context, _ []string) ([]api.ScopeStats, error) { return []api.ScopeStats{{Name: "#bc", PacketCount: 100}}, nil }, - })) + }, nil)) req := httptest.NewRequest(http.MethodGet, "/stats/scopes", nil) w := httptest.NewRecorder() r.ServeHTTP(w, req) diff --git a/internal/api/handlers/stub_reader_test.go b/internal/api/handlers/stub_reader_test.go index f9f885a..3d3626c 100644 --- a/internal/api/handlers/stub_reader_test.go +++ b/internal/api/handlers/stub_reader_test.go @@ -24,7 +24,7 @@ type stubReader struct { listRegions func(ctx context.Context) ([]api.RegionSummary, error) getRegion func(ctx context.Context, regionID int32) (*api.Region, error) getRegionBySlug func(ctx context.Context, slug string) (*api.Region, error) - listChannels func(ctx context.Context, limit int32, hash []byte, iatas []string, cursor int64, pageCursor *api.ChannelCursor) (api.ChannelPage, error) + listChannels func(ctx context.Context, limit int32, hash []byte, iatas []string, keyKnown *bool, cursor int64, pageCursor *api.ChannelCursor) (api.ChannelPage, error) getChannel func(ctx context.Context, channelID int32) (*api.Channel, error) listChannelMessages func(ctx context.Context, channelID *int32, since time.Time, limit int32, iatas []string, scope string, cursor int64) (api.Page[api.ChannelMessage], error) listChannelMessagesByHash func(ctx context.Context, hash []byte, since time.Time, limit int32, iatas []string, scope string, cursor int64) (api.Page[api.ChannelMessage], error) @@ -55,7 +55,6 @@ type stubReader struct { getScopeStats func(ctx context.Context, iatas []string) ([]api.ScopeStats, error) getStatsNodeTypes func(ctx context.Context, iatas []string) ([]api.NodeTypeCount, error) getScopeNames func(ctx context.Context) ([]string, error) - getScopesByIATAs func(ctx context.Context, iatas []string) ([]api.ScopeSummary, error) getScopeByName func(ctx context.Context, name string) (*api.ScopeDetail, error) listTraceTags func(ctx context.Context, iatas []string, scope, traceType string, since, until time.Time, cursor time.Time, limit int32) ([]api.TraceTagSummary, error) getTraceByTag func(ctx context.Context, tag string) (*api.TraceDetail, error) @@ -124,9 +123,9 @@ func (s stubReader) GetRegionBySlug(ctx context.Context, slug string) (*api.Regi return nil, nil } -func (s stubReader) ListChannels(ctx context.Context, limit int32, hash []byte, iatas []string, cursor int64, pageCursor *api.ChannelCursor) (api.ChannelPage, error) { +func (s stubReader) ListChannels(ctx context.Context, limit int32, hash []byte, iatas []string, keyKnown *bool, cursor int64, pageCursor *api.ChannelCursor) (api.ChannelPage, error) { if s.listChannels != nil { - return s.listChannels(ctx, limit, hash, iatas, cursor, pageCursor) + return s.listChannels(ctx, limit, hash, iatas, keyKnown, cursor, pageCursor) } return api.ChannelPage{}, nil } @@ -341,13 +340,6 @@ func (s stubReader) GetScopeNames(ctx context.Context) ([]string, error) { return nil, nil } -func (s stubReader) GetScopesByIATAs(ctx context.Context, iatas []string) ([]api.ScopeSummary, error) { - if s.getScopesByIATAs != nil { - return s.getScopesByIATAs(ctx, iatas) - } - return nil, nil -} - func (s stubReader) GetScopeByName(ctx context.Context, name string) (*api.ScopeDetail, error) { if s.getScopeByName != nil { return s.getScopeByName(ctx, name) diff --git a/internal/api/reader.go b/internal/api/reader.go index d46ec2d..ba6c7cd 100644 --- a/internal/api/reader.go +++ b/internal/api/reader.go @@ -53,11 +53,11 @@ type Reader interface { // ListChannels returns a paginated list of channels ordered by last seen. // Includes both hashtag-derived and explicit key channels. // Pass nil hash to skip hash filtering. Pass empty iatas to return all channels; - // IATAs must be uppercase. + // IATAs must be uppercase. Nil keyKnown skips the decryption-key filter. // cursor is last_seen epoch ms of the last item; pass 0 to start from the beginning. // pageCursor is the precise timestamp/ID boundary; nil preserves the legacy numeric cursor. // When non-nil, pageCursor takes precedence over cursor. - ListChannels(ctx context.Context, limit int32, hash []byte, iatas []string, cursor int64, pageCursor *ChannelCursor) (ChannelPage, error) + ListChannels(ctx context.Context, limit int32, hash []byte, iatas []string, keyKnown *bool, cursor int64, pageCursor *ChannelCursor) (ChannelPage, error) // GetChannel returns full detail for a single channel by its integer ID. // Returns nil, pgx.ErrNoRows if the channel is not found. @@ -210,10 +210,6 @@ type Reader interface { // Use when no geographic filter is applied — returns names only for a lightweight response. GetScopeNames(ctx context.Context) ([]string, error) - // GetScopesByIATAs returns scope summaries filtered by the given IATA codes, - // including observer, node and IATA counts. Expands region/regionId to IATAs automatically. - GetScopesByIATAs(ctx context.Context, iatas []string) ([]ScopeSummary, error) - // GetScopeByName returns full detail for a single scope by its normalized name (e.g. "#bc"), // including packet count, observer count, node count, and the list of IATAs it is active in. // Returns nil if the scope is not found. diff --git a/internal/api/router/options.go b/internal/api/router/options.go index f033bd9..e84205e 100644 --- a/internal/api/router/options.go +++ b/internal/api/router/options.go @@ -7,7 +7,6 @@ import ( "net/http" "github.com/MeshCore-Beacon/beacon-server/internal/api/handlers" - "github.com/MeshCore-Beacon/beacon-server/internal/config" ) @@ -22,6 +21,7 @@ type Options struct { Server config.ServerConfig Auth config.AuthConfig RateLimit config.ResolvedRateLimitConfig + Scopes handlers.ScopeMembership // region-filtered scope lists; nil lists none // AdminRoutes mounts operator-only subrouters at literal paths such as // "/accounts". Each feature owns its dependencies, methods and handlers. // The router applies authentication to every path and method in this group. diff --git a/internal/api/router/router.go b/internal/api/router/router.go index 95ba3cc..14c393e 100644 --- a/internal/api/router/router.go +++ b/internal/api/router/router.go @@ -107,9 +107,9 @@ func New(h *hub.Hub, reader api.Reader, workers []*ingest.Worker, opts Options) r.Mount("/iatas", handlers.IATAsRouter(reader)) r.Mount("/regions", handlers.RegionsRouter(reader)) r.Mount("/routes", handlers.RoutesRouter(reader)) - r.Mount("/scopes", handlers.ScopesRouter(reader)) + r.Mount("/scopes", handlers.ScopesRouter(reader, opts.Scopes)) + r.Mount("/stats", handlers.StatsRouter(reader, opts.Scopes)) r.Get("/scope-catalogues", handlers.ScopeCatalogues(opts.ScopeCatalogues)) - r.Mount("/stats", handlers.StatsRouter(reader)) r.Mount("/traces", handlers.TracesRouter(reader)) }) diff --git a/internal/api/scopes.go b/internal/api/scopes.go index c4b23c1..58a1fc2 100644 --- a/internal/api/scopes.go +++ b/internal/api/scopes.go @@ -3,14 +3,6 @@ package api -// ScopeSummary is the minimal scope representation used in filtered list responses. -type ScopeSummary struct { - Name string `json:"name"` - ObserverCount int64 `json:"observerCount"` - NodeCount int64 `json:"nodeCount"` - IATACount int64 `json:"iataCount"` -} - // ScopeDetail is the full scope representation including packet count and IATA list. type ScopeDetail struct { Name string `json:"name"` diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go index 147b9da..467cfcf 100644 --- a/internal/cache/cache_test.go +++ b/internal/cache/cache_test.go @@ -76,10 +76,6 @@ func (s *stubReader) GetScopeNames(_ context.Context) ([]string, error) { return func (s *stubReader) GetScopeStats(_ context.Context, _ []string) ([]api.ScopeStats, error) { return nil, nil } -func (s *stubReader) GetScopesByIATAs(_ context.Context, _ []string) ([]api.ScopeSummary, error) { - return nil, nil -} - func (s *stubReader) GetScopeByName(_ context.Context, _ string) (*api.ScopeDetail, error) { return nil, nil } @@ -168,7 +164,7 @@ func (s *stubReader) GetCrossIATANeighbors(_ context.Context, _ uuid.UUID, _ str return nil, nil } -func (s *stubReader) ListChannels(_ context.Context, _ int32, _ []byte, _ []string, _ int64, _ *api.ChannelCursor) (api.ChannelPage, error) { +func (s *stubReader) ListChannels(_ context.Context, _ int32, _ []byte, _ []string, _ *bool, _ int64, _ *api.ChannelCursor) (api.ChannelPage, error) { return api.ChannelPage{}, nil } @@ -386,7 +382,7 @@ func TestCachedReader_IATASortingForStableKey(t *testing.T) { func TestInvalidateScopeNamesClearsDerivedKeys(t *testing.T) { c, mr := newTestClient(t) cr := &CachedReader{inner: &stubReader{}, c: c, ttl: CacheTTLs{}} - for _, k := range []string{keyScopeNames, keyScopesByIATAsPrefix + "YOW", keyScopesByIATAsPrefix + "YOW,YVR", keyScopeByNamePrefix + "#yow"} { + for _, k := range []string{keyScopeNames, keyScopeByNamePrefix + "#yow"} { mr.Set(k, "x") } mr.Set("beacon:iatas", "keep") diff --git a/internal/cache/reader.go b/internal/cache/reader.go index 2273447..e845e62 100644 --- a/internal/cache/reader.go +++ b/internal/cache/reader.go @@ -25,7 +25,6 @@ const ( keyRegionSlugPrefix = "beacon:region:slug:" keyScopeNames = "beacon:scope:names" keyScopeStats = "beacon:scope:stats" - keyScopesByIATAsPrefix = "beacon:scopes:iatas:" keyScopeByNamePrefix = "beacon:scope:name:" keyStatsOverviewPrefix = "beacon:stats:overview:" keyStatsObservationsPrefix = "beacon:stats:observations:" @@ -83,7 +82,6 @@ func NewCachedReader(inner api.Reader, c *Client, ttl CacheTTLs) api.Reader { // InvalidateScopeNames makes newly committed catalogue names visible to every scope read. func (cr *CachedReader) InvalidateScopeNames(ctx context.Context) { cr.c.del(ctx, keyScopeNames) - cr.c.delPrefix(ctx, keyScopesByIATAsPrefix) cr.c.delPrefix(ctx, keyScopeByNamePrefix) } @@ -92,6 +90,18 @@ func (cr *CachedReader) InvalidateIATABorder(ctx context.Context, iata string) { cr.c.del(ctx, keyIATABorderPrefix+iata) } +// InvalidateIATAs makes imported MeshMapper names and locations visible to every IATA read. +func (cr *CachedReader) InvalidateIATAs(ctx context.Context) { + cr.c.del(ctx, keyIATAs) + cr.c.delPrefix(ctx, keyIATAPrefix) +} + +// InvalidateRegions makes imported MeshMapper regions visible to every region read. +func (cr *CachedReader) InvalidateRegions(ctx context.Context) { + cr.c.del(ctx, keyRegions) + cr.c.delPrefix(ctx, keyRegionPrefix) +} + // InvalidateNode removes the cached entries for a node by UUID. // Should be called from the ingest path after a node upsert. func (cr *CachedReader) InvalidateNode(ctx context.Context, nodeID uuid.UUID) { @@ -174,17 +184,6 @@ func (cr *CachedReader) GetScopeStats(ctx context.Context, iatas []string) ([]ap }) } -// GetScopesByIATAs implements [api.Reader]. -func (cr *CachedReader) GetScopesByIATAs(ctx context.Context, iatas []string) ([]api.ScopeSummary, error) { - sorted := make([]string, len(iatas)) - copy(sorted, iatas) - sort.Strings(sorted) - key := keyScopesByIATAsPrefix + strings.Join(sorted, ",") - return getOrSet(ctx, cr.c, key, cr.ttl.Reference, func() ([]api.ScopeSummary, error) { - return cr.inner.GetScopesByIATAs(ctx, iatas) - }) -} - // GetScopeByName implements [api.Reader]. func (cr *CachedReader) GetScopeByName(ctx context.Context, name string) (*api.ScopeDetail, error) { return getOrSet(ctx, cr.c, keyScopeByNamePrefix+name, cr.ttl.Reference, func() (*api.ScopeDetail, error) { @@ -420,8 +419,8 @@ func (cr *CachedReader) GetCrossIATANeighbors(ctx context.Context, nodeID uuid.U } // ListChannels implements [api.Reader]. -func (cr *CachedReader) ListChannels(ctx context.Context, limit int32, hash []byte, iatas []string, cursor int64, pageCursor *api.ChannelCursor) (api.ChannelPage, error) { - return cr.inner.ListChannels(ctx, limit, hash, iatas, cursor, pageCursor) +func (cr *CachedReader) ListChannels(ctx context.Context, limit int32, hash []byte, iatas []string, keyKnown *bool, cursor int64, pageCursor *api.ChannelCursor) (api.ChannelPage, error) { + return cr.inner.ListChannels(ctx, limit, hash, iatas, keyKnown, cursor, pageCursor) } // ListChannelMessages implements [api.Reader]. diff --git a/internal/config/channels.go b/internal/config/channels.go new file mode 100644 index 0000000..09216c3 --- /dev/null +++ b/internal/config/channels.go @@ -0,0 +1,66 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package config + +import ( + "encoding/hex" + "fmt" + "strings" + + "github.com/MeshCore-Beacon/beacon-server/internal/keystore" + "gopkg.in/yaml.v3" +) + +// HashtagConfig is a hashtag channel; a plain string in YAML is a Beacon-wide one. +type HashtagConfig struct { + Name string `yaml:"name"` + Region string `yaml:"region"` // optional region slug +} + +func (h *HashtagConfig) UnmarshalYAML(node *yaml.Node) error { + if node.Kind == yaml.ScalarNode { + return node.Decode(&h.Name) + } + type plain HashtagConfig + return node.Decode((*plain)(h)) +} + +// ChannelScope places a configured channel under a region, or Beacon-wide when Region is empty. +type ChannelScope struct { + Fingerprint []byte + Region string +} + +func (c *Config) validateChannelKeys() error { + for i, h := range c.ChannelKeys.Hashtags { + if strings.TrimSpace(h.Name) == "" { + return fmt.Errorf("channel_keys.hashtags[%d] needs a name", i) + } + if h.Region != "" && c.region(h.Region) == nil { + return fmt.Errorf("channel #%s region %q is not a configured region", h.Name, h.Region) + } + } + for hash, k := range c.ChannelKeys.Keys { + if k.Region != "" && c.region(k.Region) == nil { + return fmt.Errorf("channel key %s region %q is not a configured region", hash, k.Region) + } + } + return nil +} + +// ChannelScopes lists every configured channel's region placement. Keys that +// aren't valid hex are skipped; startup already warns about them. +func (c *Config) ChannelScopes() []ChannelScope { + var scopes []ChannelScope + for _, h := range c.ChannelKeys.Hashtags { + _, _, fingerprint := keystore.DeriveHashtagKey(h.Name) + scopes = append(scopes, ChannelScope{Fingerprint: fingerprint, Region: h.Region}) + } + for _, k := range c.ChannelKeys.Keys { + if key, err := hex.DecodeString(k.Key); err == nil { + scopes = append(scopes, ChannelScope{Fingerprint: keystore.Fingerprint(key), Region: k.Region}) + } + } + return scopes +} diff --git a/internal/config/channels_test.go b/internal/config/channels_test.go new file mode 100644 index 0000000..4865670 --- /dev/null +++ b/internal/config/channels_test.go @@ -0,0 +1,88 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package config + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/keystore" +) + +func loadText(t *testing.T, text string) (*Config, error) { + t.Helper() + p := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(p, []byte(text), 0600); err != nil { + t.Fatal(err) + } + return Load(p) +} + +func TestChannelKeysRegionScoping(t *testing.T) { + text := `regions: + - slug: east + iatas: [YOW] +channel_keys: + hashtags: + - meshcore + - name: ottawa-mesh + region: east + keys: + "11": + key: "8b3387e9c5cdea6ac9e5edbaa115cd72" + name: Public +` + cfg, err := loadText(t, text) + if err != nil { + t.Fatal(err) + } + if got := cfg.ChannelKeys.Hashtags; len(got) != 2 || got[0].Name != "meshcore" || got[0].Region != "" || got[1].Name != "ottawa-mesh" || got[1].Region != "east" { + t.Fatalf("hashtags parsed as %+v", got) + } + scopes := cfg.ChannelScopes() + if len(scopes) != 3 { + t.Fatalf("scopes: %+v", scopes) + } + _, _, ottawa := keystore.DeriveHashtagKey("ottawa-mesh") + found := false + for _, s := range scopes { + if bytes.Equal(s.Fingerprint, ottawa) { + found = s.Region == "east" + } + } + if !found { + t.Fatal("region-scoped hashtag missing its region", scopes) + } + + for name, bad := range map[string]string{ + "unknown hashtag region": strings.Replace(text, "region: east", "region: west", 1), + "empty hashtag": strings.Replace(text, "- meshcore", `- ""`, 1), + "unknown key region": text + " region: west\n", + } { + if _, err := loadText(t, bad); err == nil { + t.Errorf("%s accepted", name) + } + } +} + +func TestMeshMapperChannelsConfig(t *testing.T) { + valid := "meshmapper:\n channels:\n enabled: true\n" + for text, bad := range map[string]bool{ + valid: false, + valid + " refresh_interval: 24h\n": false, + valid + " refresh_interval: 168h\n": false, + valid + " refresh_interval: 23h\n": true, + valid + " refresh_interval: 169h\n": true, + } { + if _, err := loadText(t, text); (err != nil) != bad { + t.Errorf("%q: error=%v", text, err) + } + } + if (MeshMapperChannelsConfig{}).Interval() != MinChannelsRefresh { + t.Fatal("wrong default") + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 85df469..65abd77 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -221,7 +221,8 @@ type CacheTTLsConfig struct { // Name can be provided with or without the # or $ prefix. // Beacon normalizes plain names by prepending #. type ScopeConfig struct { - Name string `yaml:"name"` // e.g. "bc", "#west", "$private" + Name string `yaml:"name"` // e.g. "bc", "#west", "$private" + Region string `yaml:"region"` // configured region slug; lists the scope under that region's IATAs } // TelemetryConfig controls observer telemetry storage behaviour. @@ -319,9 +320,9 @@ func (d *duration) UnmarshalYAML(value *yaml.Node) error { // channel_hash = SHA256(secret)[0]. Explicit keys are provided as hex strings // keyed by the channel hash hex (e.g. "11" for 0x11). type ChannelKeysConfig struct { - // Hashtags is a list of hashtag names (without the # prefix). + // Hashtags lists hashtag names (without the # prefix), each optionally scoped to a region. // Beacon derives the PSK and channel hash automatically. - Hashtags []string `yaml:"hashtags"` + Hashtags []HashtagConfig `yaml:"hashtags"` // Keys maps channel hash hex → explicit key config. Keys map[string]ExplicitKeyConfig `yaml:"keys"` @@ -329,8 +330,9 @@ type ChannelKeysConfig struct { // ExplicitKeyConfig holds an explicit channel key and optional display name. type ExplicitKeyConfig struct { - Key string `yaml:"key"` // hex-encoded key bytes - Name string `yaml:"name"` // optional display name + Key string `yaml:"key"` // hex-encoded key bytes + Name string `yaml:"name"` // optional display name + Region string `yaml:"region"` // optional region slug; empty lists the channel Beacon-wide } // IATAConfig holds optional overrides for a known IATA code. @@ -413,6 +415,12 @@ func Load(path string) (*Config, error) { if err := cfg.validateMeshMapper(); err != nil { return nil, err } + if err := cfg.validateChannelKeys(); err != nil { + return nil, err + } + if err := cfg.validateScopes(); err != nil { + return nil, err + } if cfg.Nodes.MarkForeign && !cfg.MeshMapper.Zones.Enabled && !cfg.hasBorderFile() { return nil, fmt.Errorf("nodes.mark_foreign requires an iatas.*.borderFile or meshmapper.zones") } diff --git a/internal/config/meshmapper.go b/internal/config/meshmapper.go index 31bec78..8f8a0c7 100644 --- a/internal/config/meshmapper.go +++ b/internal/config/meshmapper.go @@ -5,23 +5,41 @@ package config import ( "fmt" - "net/url" - "regexp" - "slices" + "log/slog" "time" ) -// MaxScopeSources bounds both network refresh work and imported matching keys. -const MaxScopeSources = 16 +// MeshMapper rate-limits get_zones/get_geojson to one request per target per +// 23.5h and get_scopes to one per region per 55m; minimums round up. +const ( + MinZonesRefresh = 24 * time.Hour + MinScopesRefresh = time.Hour + MinChannelsRefresh = 24 * time.Hour // get_channels.php: one per region per 23.5h +) type MeshMapperConfig struct { - Scopes MeshMapperScopesConfig `yaml:"scopes"` - Zones MeshMapperZonesConfig `yaml:"zones"` + Scopes MeshMapperScopesConfig `yaml:"scopes"` + Zones MeshMapperZonesConfig `yaml:"zones"` + Channels MeshMapperChannelsConfig `yaml:"channels"` } -// MeshMapperZonesConfig imports each region IATA's published boundary, overriding borderFile. +// MeshMapperChannelsConfig imports each known IATA's public hashtag channels. +type MeshMapperChannelsConfig struct { + Enabled bool `yaml:"enabled"` + RefreshInterval duration `yaml:"refresh_interval"` +} + +func (c MeshMapperChannelsConfig) Interval() time.Duration { + if c.RefreshInterval.Duration == 0 { + return MinChannelsRefresh + } + return c.RefreshInterval.Duration +} + +// MeshMapperZonesConfig imports each known IATA's published boundary, overriding borderFile. type MeshMapperZonesConfig struct { Enabled bool `yaml:"enabled"` + ImportGroups bool `yaml:"import_groups"` // zone groups become regions RefreshInterval duration `yaml:"refresh_interval"` } @@ -32,20 +50,11 @@ func (c MeshMapperZonesConfig) Interval() time.Duration { return c.RefreshInterval.Duration } -// RegionIATAs returns the sorted, unique IATAs of every configured region. -func (c *Config) RegionIATAs() []string { - var iatas []string - for _, region := range c.Regions { - iatas = append(iatas, region.IATAs...) - } - slices.Sort(iatas) - return slices.Compact(iatas) -} - // MeshMapperScopesConfig augments, but never replaces, the manual scopes list. +// Sources are discovered per known IATA from the zone list. type MeshMapperScopesConfig struct { Enabled bool `yaml:"enabled"` - Sources map[string]string `yaml:"sources"` // configured IATA -> published regional endpoint + Sources map[string]string `yaml:"sources"` // deprecated: ignored, kept to warn RefreshInterval duration `yaml:"refresh_interval"` } @@ -56,43 +65,27 @@ func (c MeshMapperScopesConfig) Interval() time.Duration { return c.RefreshInterval.Duration } -var meshMapperHost = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*\.meshmapper\.net$`) - func (c *Config) validateMeshMapper() error { + if z := c.MeshMapper.Zones; z.ImportGroups && !z.Enabled { + return fmt.Errorf("meshmapper.zones.import_groups requires meshmapper.zones.enabled") + } if z := c.MeshMapper.Zones; z.Enabled { - // The Zones API asks clients not to poll more than once an hour. - if z.Interval() < time.Hour || z.Interval() > 7*24*time.Hour { - return fmt.Errorf("meshmapper.zones.refresh_interval must be between 1h and 168h") - } - if len(c.RegionIATAs()) == 0 { - return fmt.Errorf("meshmapper.zones needs at least one configured region IATA") + if z.Interval() < MinZonesRefresh || z.Interval() > 7*24*time.Hour { + return fmt.Errorf("meshmapper.zones.refresh_interval must be between 24h and 168h") } } + if ch := c.MeshMapper.Channels; ch.Enabled && (ch.Interval() < MinChannelsRefresh || ch.Interval() > 7*24*time.Hour) { + return fmt.Errorf("meshmapper.channels.refresh_interval must be between 24h and 168h") + } s := c.MeshMapper.Scopes if !s.Enabled { return nil } - if len(s.Sources) == 0 || len(s.Sources) > MaxScopeSources { - return fmt.Errorf("meshmapper.scopes.sources must contain 1-%d regional endpoints", MaxScopeSources) + if s.Interval() < MinScopesRefresh || s.Interval() > 24*time.Hour { + return fmt.Errorf("meshmapper.scopes.refresh_interval must be between 1h and 24h") } - if s.Interval() < 5*time.Minute || s.Interval() > 24*time.Hour { - return fmt.Errorf("meshmapper.scopes.refresh_interval must be between 5m and 24h") - } - configured := map[string]bool{} - for _, region := range c.Regions { - for _, iata := range region.IATAs { - configured[iata] = true - } - } - for iata, endpoint := range s.Sources { - if !configured[iata] { - return fmt.Errorf("meshmapper.scopes source %q must belong to a configured region", iata) - } - u, err := url.Parse(endpoint) - if err != nil || u.Scheme != "https" || u.User != nil || !meshMapperHost.MatchString(u.Host) || - u.Path != "/get_scopes.php" || u.RawPath != "" || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" { - return fmt.Errorf("meshmapper.scopes source %q must be a published https://.meshmapper.net/get_scopes.php endpoint without credentials or parameters", iata) - } + if len(s.Sources) > 0 { + slog.Warn("meshmapper.scopes.sources is ignored; sources are discovered from the MeshMapper zone list", "component", "config") } return nil } diff --git a/internal/config/meshmapper_test.go b/internal/config/meshmapper_test.go index d4e343b..5127959 100644 --- a/internal/config/meshmapper_test.go +++ b/internal/config/meshmapper_test.go @@ -12,19 +12,18 @@ import ( ) func TestMeshMapperConfig(t *testing.T) { - valid := "regions:\n - slug: ottawa\n iatas: [YOW]\nmeshmapper:\n scopes:\n enabled: true\n sources:\n YOW: https://yow.meshmapper.net/get_scopes.php\n" + valid := "meshmapper:\n scopes:\n enabled: true\n" for _, tc := range []struct { name, text string bad bool }{ {"valid", valid, false}, - {"unknown region", strings.Replace(valid, "iatas: [YOW]", "iatas: [YVR]", 1), true}, - {"http", strings.Replace(valid, "https:", "http:", 1), true}, - {"private host", strings.Replace(valid, "yow.meshmapper.net", "127.0.0.1", 1), true}, - {"credentials", strings.Replace(valid, "https://", "https://secret@", 1), true}, - {"query", strings.Replace(valid, "get_scopes.php", "get_scopes.php?key=secret", 1), true}, - {"non API", strings.Replace(valid, "get_scopes.php", "index.php", 1), true}, + {"daily", valid + " refresh_interval: 24h\n", false}, + {"legacy sources ignored", valid + " sources:\n yow: http://127.0.0.1/\n", false}, + {"hourly", valid + " refresh_interval: 1h\n", false}, + {"under the rate limit", valid + " refresh_interval: 55m\n", true}, {"fast", valid + " refresh_interval: 1s\n", true}, + {"slow", valid + " refresh_interval: 25h\n", true}, {"negative", valid + " refresh_interval: -1h\n", true}, {"disabled", strings.Replace(valid, "enabled: true", "enabled: false", 1), false}, } { @@ -33,18 +32,14 @@ func TestMeshMapperConfig(t *testing.T) { if err := os.WriteFile(p, []byte(tc.text), 0600); err != nil { t.Fatal(err) } - cfg, err := Load(p) + _, err := Load(p) if (err != nil) != tc.bad { t.Fatalf("error=%v", err) } - if err == nil && cfg.MeshMapper.Scopes.Interval() != time.Hour { - t.Fatal("wrong default") - } }) } - cfg := Config{MeshMapper: MeshMapperConfig{Scopes: MeshMapperScopesConfig{Enabled: true}}} - if err := cfg.validateMeshMapper(); err == nil { - t.Fatal("empty sources accepted") + if (MeshMapperScopesConfig{}).Interval() != time.Hour { + t.Fatal("wrong default") } } @@ -56,10 +51,14 @@ func TestMeshMapperZonesConfig(t *testing.T) { interval time.Duration }{ {"default", valid, false, 24 * time.Hour}, - {"hourly", valid + " refresh_interval: 1h\n", false, time.Hour}, - {"too often", valid + " refresh_interval: 59m\n", true, 0}, + {"daily", valid + " refresh_interval: 24h\n", false, 24 * time.Hour}, + {"weekly", valid + " refresh_interval: 168h\n", false, 168 * time.Hour}, + {"under the rate limit", valid + " refresh_interval: 23h30m\n", true, 0}, + {"hourly", valid + " refresh_interval: 1h\n", true, 0}, {"too rare", valid + " refresh_interval: 169h\n", true, 0}, - {"no regions", "meshmapper:\n zones:\n enabled: true\n", true, 0}, + {"no regions", "meshmapper:\n zones:\n enabled: true\n", false, 24 * time.Hour}, + {"groups", valid + " import_groups: true\n", false, 24 * time.Hour}, + {"groups without zones", "meshmapper:\n zones:\n import_groups: true\n", true, 0}, {"disabled", "meshmapper:\n zones:\n enabled: false\n", false, 24 * time.Hour}, } { t.Run(tc.name, func(t *testing.T) { @@ -76,8 +75,4 @@ func TestMeshMapperZonesConfig(t *testing.T) { } }) } - cfg := &Config{Regions: []RegionConfig{{IATAs: []string{"YYZ", "YOW"}}, {IATAs: []string{"YOW"}}}} - if got := strings.Join(cfg.RegionIATAs(), ","); got != "YOW,YYZ" { - t.Fatal(got) - } } diff --git a/internal/config/scopes.go b/internal/config/scopes.go new file mode 100644 index 0000000..87c05a5 --- /dev/null +++ b/internal/config/scopes.go @@ -0,0 +1,52 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package config + +import ( + "fmt" + "strings" + + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" +) + +func (c *Config) validateScopes() error { + for i, s := range c.Scopes { + if strings.TrimSpace(s.Name) == "" { + return fmt.Errorf("scopes[%d].name is required", i) + } + if s.Region == "" { + return fmt.Errorf("scope %q needs a region", s.Name) + } + if c.region(s.Region) == nil { + return fmt.Errorf("scope %q region %q is not a configured region", s.Name, s.Region) + } + } + return nil +} + +func (c *Config) region(slug string) *RegionConfig { + for i := range c.Regions { + if c.Regions[i].Slug == slug { + return &c.Regions[i] + } + } + return nil +} + +// ManualScopeMembers maps each region IATA to the normalized names of its configured scopes. +func (c *Config) ManualScopeMembers() map[string][]string { + members := map[string][]string{} + for _, s := range c.Scopes { + region := c.region(s.Region) + if region == nil { + continue + } + name := scopestore.FromName(s.Name).Name + for _, iata := range region.IATAs { + iata = strings.ToUpper(iata) + members[iata] = append(members[iata], name) + } + } + return members +} diff --git a/internal/config/scopes_test.go b/internal/config/scopes_test.go new file mode 100644 index 0000000..6be61e4 --- /dev/null +++ b/internal/config/scopes_test.go @@ -0,0 +1,43 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package config + +import ( + "os" + "path/filepath" + "reflect" + "strings" + "testing" +) + +func TestScopesConfig(t *testing.T) { + valid := "regions:\n - slug: ottawa\n iatas: [YOW, YGK]\n - slug: bc\n iatas: [YVR]\nscopes:\n - name: on\n region: ottawa\n - name: \"#west\"\n region: bc\n - name: \"$ottawa\"\n region: ottawa\n" + for _, tc := range []struct { + name, text string + bad bool + }{ + {"valid", valid, false}, + {"missing region", strings.Replace(valid, " region: bc\n", "", 1), true}, + {"unknown region", strings.Replace(valid, "region: bc", "region: prairies", 1), true}, + {"empty name", strings.Replace(valid, "name: on", "name: \"\"", 1), true}, + } { + t.Run(tc.name, func(t *testing.T) { + p := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(p, []byte(tc.text), 0600); err != nil { + t.Fatal(err) + } + cfg, err := Load(p) + if (err != nil) != tc.bad { + t.Fatalf("error=%v", err) + } + if tc.bad { + return + } + want := map[string][]string{"YOW": {"#on", "$ottawa"}, "YGK": {"#on", "$ottawa"}, "YVR": {"#west"}} + if got := cfg.ManualScopeMembers(); !reflect.DeepEqual(got, want) { + t.Fatalf("ManualScopeMembers = %v, want %v", got, want) + } + }) + } +} diff --git a/internal/config/seed.go b/internal/config/seed.go index a419573..e08f259 100644 --- a/internal/config/seed.go +++ b/internal/config/seed.go @@ -19,8 +19,9 @@ type Seeder interface { UpsertIATADetails(ctx context.Context, iata string, name string, lat, lng *float64) error UpsertIATABorder(ctx context.Context, iata string, border json.RawMessage) error UpsertRegion(ctx context.Context, slug, name, description string, displayOrder int, centerLat, centerLng *float64, zoomLevel *int) (int32, error) - UpsertRegionIATA(ctx context.Context, regionID int32, iata string) error + SetRegionIATAs(ctx context.Context, regionID int32, iatas []string) error UpsertTransportScope(ctx context.Context, name, displayName string, transportKey, keyFingerprint []byte) error + SetChannelConfigScopes(ctx context.Context, fingerprints [][]byte, regions []string) error } // Seed applies config-defined regions, IATA overrides to the database. @@ -56,10 +57,20 @@ func Seed(ctx context.Context, cfg *Config, db Seeder) error { if err := db.UpsertIATA(ctx, iata); err != nil { return err } - if err := db.UpsertRegionIATA(ctx, id, iata); err != nil { - return err - } } + // Config owns the member list, including when it takes over an imported slug. + if err := db.SetRegionIATAs(ctx, id, r.IATAs); err != nil { + return err + } + } + // Channel region placement + var fingerprints [][]byte + var regions []string + for _, scope := range cfg.ChannelScopes() { + fingerprints, regions = append(fingerprints, scope.Fingerprint), append(regions, scope.Region) + } + if err := db.SetChannelConfigScopes(ctx, fingerprints, regions); err != nil { + return err } // Transport Codes for _, s := range cfg.Scopes { diff --git a/internal/config/seed_test.go b/internal/config/seed_test.go index 04c8e63..c09aa11 100644 --- a/internal/config/seed_test.go +++ b/internal/config/seed_test.go @@ -15,12 +15,13 @@ import ( ) type stubSeeder struct { - regions map[string]int32 - iatas []string - regionIATAs map[int32][]string - scopes []string - borders map[string]json.RawMessage - upsertErr error + regions map[string]int32 + iatas []string + regionIATAs map[int32][]string + channelRegions []string + scopes []string + borders map[string]json.RawMessage + upsertErr error } func newStubSeeder() *stubSeeder { @@ -52,8 +53,13 @@ func (s *stubSeeder) UpsertRegion(_ context.Context, slug, _, _ string, _ int, _ return id, s.upsertErr } -func (s *stubSeeder) UpsertRegionIATA(_ context.Context, regionID int32, iata string) error { - s.regionIATAs[regionID] = append(s.regionIATAs[regionID], iata) +func (s *stubSeeder) SetRegionIATAs(_ context.Context, regionID int32, iatas []string) error { + s.regionIATAs[regionID] = iatas + return s.upsertErr +} + +func (s *stubSeeder) SetChannelConfigScopes(_ context.Context, fingerprints [][]byte, regions []string) error { + s.channelRegions = regions return s.upsertErr } @@ -256,3 +262,17 @@ func TestDeriveScopeKey_DifferentInputs(t *testing.T) { t.Error("expected different keys for different inputs") } } + +func TestSeed_ChannelScopes(t *testing.T) { + cfg := &Config{ + Regions: []RegionConfig{{Slug: "bc", IATAs: []string{"YVR"}}}, + ChannelKeys: ChannelKeysConfig{Hashtags: []HashtagConfig{{Name: "meshcore"}, {Name: "vancouver", Region: "bc"}}}, + } + db := newStubSeeder() + if err := Seed(context.Background(), cfg, db); err != nil { + t.Fatal(err) + } + if len(db.channelRegions) != 2 || db.channelRegions[0] != "" || db.channelRegions[1] != "bc" { + t.Fatalf("channel regions = %q", db.channelRegions) + } +} diff --git a/internal/ingest/backfill.go b/internal/ingest/backfill.go index ecb7ebb..1a41283 100644 --- a/internal/ingest/backfill.go +++ b/internal/ingest/backfill.go @@ -107,6 +107,19 @@ func BackfillChannelMessages(ctx context.Context, db DB, keys ChannelKeyStore) ( if err != nil { return 0, err } + return backfill(ctx, db, keys, packets), nil +} + +// BackfillChannelHashes retries only the channels whose keys were just imported at runtime. +func BackfillChannelHashes(ctx context.Context, db DB, keys ChannelKeyStore, hashes [][]byte) (int, error) { + packets, err := db.ListUndecryptedGroupTextPacketsByHash(ctx, hashes) + if err != nil { + return 0, err + } + return backfill(ctx, db, keys, packets), nil +} + +func backfill(ctx context.Context, db DB, keys ChannelKeyStore, packets []UndecryptedPacket) int { decrypted := 0 for _, p := range packets { result, err := DecryptGroupText(ctx, db, keys, p.PacketHash, p.RawPayload) @@ -118,5 +131,5 @@ func BackfillChannelMessages(ctx context.Context, db DB, keys ChannelKeyStore) ( decrypted++ } } - return decrypted, nil + return decrypted } diff --git a/internal/ingest/backfill_test.go b/internal/ingest/backfill_test.go index 1bbeeef..c50d7a6 100644 --- a/internal/ingest/backfill_test.go +++ b/internal/ingest/backfill_test.go @@ -135,6 +135,23 @@ func TestBackfillChannelMessages_DecryptsAndCounts(t *testing.T) { } } +func TestBackfillChannelHashes_OnlyScansNewHashes(t *testing.T) { + psk := make([]byte, 16) + db := &stubDB{ + insertChannelMessageResult: true, + undecryptedByHash: []UndecryptedPacket{{PacketHash: []byte{0x01}, RawPayload: encryptedGroupText(t, 0x44, psk, "ded", "hi")}}, + undecryptedPackets: []UndecryptedPacket{{PacketHash: []byte{0x02}, RawPayload: encryptedGroupText(t, 0x44, psk, "x", "y")}}, + } + keys := &mapKeys{entries: map[byte][]keystore.Entry{0x44: {{Key: psk, Fingerprint: []byte{0xAA}}}}} + n, err := BackfillChannelHashes(context.Background(), db, keys, [][]byte{{0x44}}) + if err != nil || n != 1 { + t.Fatalf("n=%d err=%v", n, err) + } + if len(db.backfillHashes) != 1 || db.backfillHashes[0][0] != 0x44 { + t.Fatalf("scanned hashes %x", db.backfillHashes) + } +} + func TestBackfillChannelMessages_NoUndecryptedPackets(t *testing.T) { db := &stubDB{} keys := &mapKeys{} diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index 59b9b4e..c9d0dc7 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -174,6 +174,9 @@ type DB interface { // used by BackfillChannelMessages to retry them against the current keystore at boot. ListUndecryptedGroupTextPackets(ctx context.Context) ([]UndecryptedPacket, error) + // ListUndecryptedGroupTextPacketsByHash limits that scan to channels that just gained a key. + ListUndecryptedGroupTextPacketsByHash(ctx context.Context, hashes [][]byte) ([]UndecryptedPacket, error) + // UpsertChannelIATA upserts a channel_iatas row. UpsertChannelIATA(ctx context.Context, channelHash []byte, iata string, heardAt time.Time) error diff --git a/internal/ingest/ingest_test.go b/internal/ingest/ingest_test.go index aa8f892..14acd2a 100644 --- a/internal/ingest/ingest_test.go +++ b/internal/ingest/ingest_test.go @@ -184,6 +184,8 @@ type stubDB struct { observationInserted bool insertChannelMessageResult bool // configurable return for InsertChannelMessage; default false undecryptedPackets []UndecryptedPacket + undecryptedByHash []UndecryptedPacket + backfillHashes [][]byte dbHook func() // runs in UpsertChannelIATA, the first DB call after InsertObservation } @@ -279,6 +281,11 @@ func (s *stubDB) ListUndecryptedGroupTextPackets(_ context.Context) ([]Undecrypt return s.undecryptedPackets, nil } +func (s *stubDB) ListUndecryptedGroupTextPacketsByHash(_ context.Context, hashes [][]byte) ([]UndecryptedPacket, error) { + s.backfillHashes = hashes + return s.undecryptedByHash, nil +} + func (s *stubDB) UpsertChannelIATA(_ context.Context, _ []byte, _ string, _ time.Time) error { if s.dbHook != nil { s.dbHook() diff --git a/internal/keystore/keystore.go b/internal/keystore/keystore.go index 9211993..0485442 100644 --- a/internal/keystore/keystore.go +++ b/internal/keystore/keystore.go @@ -2,14 +2,15 @@ // SPDX-License-Identifier: AGPL-3.0-or-later // Package keystore provides channel key lookup for the ingest pipeline. -// Keys are loaded from config at startup and never written at runtime. -// Future: add a DB-backed fallback that checks the channel_keys table. +// Config keys are fixed at startup; MeshMapper imports are swapped in at runtime. package keystore import ( "bytes" "crypto/sha256" "encoding/hex" + "slices" + "sync/atomic" ) // Entry holds a resolved channel key along with its metadata. @@ -23,23 +24,51 @@ type Entry struct { // MapKeyStore maps channel hash hex → list of known key entries. // Multiple entries per hash are supported to handle 1-byte hash collisions. type MapKeyStore struct { - entries map[string][]Entry // channel hash hex → entries + config map[string][]Entry + imported map[string][]Entry + entries atomic.Pointer[map[string][]Entry] // config + imported, read lock-free by ingest } -// NewMapKeyStore creates a keystore from a pre-built map of entries. -// Use BuildFromConfig to construct entries from a config.ChannelKeysConfig. +// NewMapKeyStore creates a keystore from the configured entries. func NewMapKeyStore(entries map[string][]Entry) *MapKeyStore { - store := &MapKeyStore{entries: make(map[string][]Entry)} + store := &MapKeyStore{config: make(map[string][]Entry), imported: map[string][]Entry{}} for hash, list := range entries { - store.entries[hash] = append(store.entries[hash], list...) + store.config[hash] = append(store.config[hash], list...) } + store.entries.Store(&store.config) return store } // GetKey returns all known key entries for the given channel hash byte. // Returns nil if no keys are known for this hash. func (s *MapKeyStore) GetKey(channelHash []byte) []Entry { - return s.entries[hex.EncodeToString(channelHash)] + return (*s.entries.Load())[hex.EncodeToString(channelHash)] +} + +// SetImported replaces the imported keys and returns the channel hashes of keys +// that weren't known before. Config keys win; one writer at a time. +func (s *MapKeyStore) SetImported(entries []Entry) [][]byte { + merged := make(map[string][]Entry, len(s.config)) + for hash, list := range s.config { + merged[hash] = list + } + imported := map[string][]Entry{} + var added [][]byte + for _, e := range entries { + hash := sha256.Sum256(e.Key) + hashHex := hex.EncodeToString(hash[:1]) + if EntryExists(merged[hashHex], e) { + continue + } + merged[hashHex] = append(slices.Clone(merged[hashHex]), e) + imported[hashHex] = append(imported[hashHex], e) + if !EntryExists(s.imported[hashHex], e) { + added = append(added, hash[:1]) + } + } + s.imported = imported + s.entries.Store(&merged) + return added } // DeriveHashtagKey derives the channel secret and hash for a hashtag name. diff --git a/internal/keystore/keystore_test.go b/internal/keystore/keystore_test.go index bed3102..2060c76 100644 --- a/internal/keystore/keystore_test.go +++ b/internal/keystore/keystore_test.go @@ -153,3 +153,33 @@ func TestEntryExists_Empty(t *testing.T) { t.Error("expected false for empty slice") } } + +func TestSetImported_ConfigWinsAndNewHashesReported(t *testing.T) { + secret, hash, fp := DeriveHashtagKey("meshcore") + hashHex := hex.EncodeToString([]byte{hash}) + configured := Entry{Key: secret, Fingerprint: fp, Hashtag: "meshcore", Name: "Configured"} + store := NewMapKeyStore(map[string][]Entry{hashHex: {configured}}) + + dupSecret, _, dupFP := DeriveHashtagKey("meshcore") + newSecret, newHash, newFP := DeriveHashtagKey("ottawa-mesh") + added := store.SetImported([]Entry{ + {Key: dupSecret, Fingerprint: dupFP, Hashtag: "meshcore", Name: "#meshcore"}, + {Key: newSecret, Fingerprint: newFP, Hashtag: "ottawa-mesh", Name: "#ottawa-mesh"}, + }) + if len(added) != 1 || added[0][0] != newHash { + t.Fatalf("new hashes = %x", added) + } + if got := store.GetKey([]byte{hash}); len(got) != 1 || got[0].Name != "Configured" { + t.Fatalf("config entry replaced: %+v", got) + } + if got := store.GetKey([]byte{newHash}); len(got) != 1 || got[0].Hashtag != "ottawa-mesh" { + t.Fatalf("imported entry missing: %+v", got) + } + if added = store.SetImported([]Entry{{Key: newSecret, Fingerprint: newFP, Hashtag: "ottawa-mesh"}}); len(added) != 0 { + t.Fatal("unchanged import reported new hashes", added) + } + store.SetImported(nil) + if store.GetKey([]byte{newHash}) != nil || len(store.GetKey([]byte{hash})) != 1 { + t.Fatal("clearing imports touched config keys or kept the import") + } +} diff --git a/internal/meshmapper/catalogues.go b/internal/meshmapper/catalogues.go index 530aafc..7655229 100644 --- a/internal/meshmapper/catalogues.go +++ b/internal/meshmapper/catalogues.go @@ -19,7 +19,7 @@ type CatalogueScope struct { Wardriving bool `json:"wardriving"` } -type Catalogue struct { +type PublicScopeCatalogue struct { IATA string `json:"iata"` URL string `json:"url"` GeneratedAt int64 `json:"generatedAt"` @@ -31,21 +31,21 @@ type Catalogue struct { Scopes []CatalogueScope `json:"scopes"` } -// Catalogues returns an immutable snapshot of configured sources. No request +// Catalogues returns an immutable snapshot of discovered sources. No request // triggers an upstream fetch or reads ingestion tables. Callers must not mutate it. -func (i *Importer) Catalogues() []Catalogue { +func (i *Importer) Catalogues() []PublicScopeCatalogue { if i != nil { if snapshot := i.catalogues.Load(); snapshot != nil { - return snapshot.([]Catalogue) + return snapshot.([]PublicScopeCatalogue) } } - return []Catalogue{} + return []PublicScopeCatalogue{} } func (i *Importer) publishCatalogues() { - result := make([]Catalogue, 0, len(i.sources)) + result := make([]PublicScopeCatalogue, 0, len(i.sources)) for _, s := range i.sources { - c := Catalogue{IATA: s.iata, URL: s.url, LastError: s.cache.LastError, Scopes: []CatalogueScope{}} + c := PublicScopeCatalogue{IATA: s.iata, URL: s.url, LastError: s.cache.LastError, Scopes: []CatalogueScope{}} if !s.cache.CheckedAt.IsZero() { c.CheckedAt = s.cache.CheckedAt.UnixMilli() c.FreshUntil = s.cache.CheckedAt.Add(i.interval + 5*time.Minute).UnixMilli() diff --git a/internal/meshmapper/catalogues_test.go b/internal/meshmapper/catalogues_test.go index 01f6cb8..baefa72 100644 --- a/internal/meshmapper/catalogues_test.go +++ b/internal/meshmapper/catalogues_test.go @@ -27,19 +27,21 @@ func TestPublicCataloguesFreshnessAndIsolation(t *testing.T) { } })) defer server.Close() - cfg := config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": server.URL}} - store := &memoryStore{rows: map[string]Cache{}} - imp, err := New(context.Background(), cfg, store, scopestore.New(), nil) + cfg := config.MeshMapperScopesConfig{Enabled: true} + store := newMemoryStore("YOW") + imp, err := New(context.Background(), cfg, store, NewDirectory(newZoneListMemory()), scopestore.New(), nil) if err != nil { t.Fatal(err) } if got := imp.Catalogues(); len(got) != 1 || got[0].CheckedAt != 0 || len(got[0].Scopes) != 0 { t.Fatal(got) } + imp.sources[0].url = server.URL now := time.Now() if err := imp.refresh(context.Background(), &imp.sources[0], now); err != nil { t.Fatal(err) } + imp.publishCatalogues() old := imp.Catalogues() if old[0].Scopes[0].Name != "#yow" || old[0].Scopes[1].Name != "#YOW" || len(old[0].Scopes) != 3 || old[0].Repeaters != 5 || old[0].Scoped != 3 || old[0].FreshUntil <= old[0].CheckedAt { t.Fatal(old) @@ -48,6 +50,7 @@ func TestPublicCataloguesFreshnessAndIsolation(t *testing.T) { if err := imp.refresh(context.Background(), &imp.sources[0], now.Add(time.Minute)); err != nil { t.Fatal(err) } + imp.publishCatalogues() current := imp.Catalogues()[0] if current.LastError != "HTTP 503" || current.CheckedAt != old[0].CheckedAt || current.GeneratedAt != old[0].GeneratedAt || len(current.Scopes) != 3 || old[0].LastError != "" { t.Fatal(current, old) @@ -68,13 +71,33 @@ func TestPublicCataloguesFreshnessAndIsolation(t *testing.T) { if calls != 2 { t.Fatal("reading public metadata performed upstream requests", calls) } - disabled, err := New(context.Background(), config.MeshMapperScopesConfig{}, store, scopestore.New(), nil) + disabled, err := New(context.Background(), config.MeshMapperScopesConfig{}, store, NewDirectory(newZoneListMemory()), scopestore.New(), nil) if err != nil || len(disabled.Catalogues()) != 0 { t.Fatal("disabled importer exposed old membership") } - store.rows["YOW"+server.URL] = Cache{Payload: []byte(`{"scopes":"invalid"}`)} - restored, err := New(context.Background(), cfg, store, scopestore.New(), nil) + store.rows[sourceKey{"YOW", server.URL}] = Cache{Payload: []byte(`{"scopes":"invalid"}`)} + restored, err := New(context.Background(), cfg, store, NewDirectory(newZoneListMemory()), scopestore.New(), nil) if err != nil || len(restored.Catalogues()[0].Scopes) != 0 || restored.Catalogues()[0].LastError == "" { t.Fatal("invalid saved metadata leaked") } } + +func TestPublicCataloguesIncludeDiscoveredAndUnlistedRegions(t *testing.T) { + ctx := context.Background() + store := newMemoryStore("YOW") + dir := NewDirectory(newZoneListMemory()) + dir.lists["CA"] = &zoneList{zones: map[string]zoneEntry{}, fetchedAt: time.Now()} + imp, err := New(ctx, config.MeshMapperScopesConfig{Enabled: true}, store, dir, scopestore.New(), nil) + if err != nil { + t.Fatal(err) + } + before := imp.Catalogues() + store.iatas = append(store.iatas, "YKF") + if err := imp.Refresh(ctx); err != nil { + t.Fatal(err) + } + after := imp.Catalogues() + if len(before) != 1 || before[0].LastError != "" || len(after) != 2 || after[1].IATA != "YKF" || after[1].LastError != "not listed" { + t.Fatal(before, after) + } +} diff --git a/internal/meshmapper/channels.go b/internal/meshmapper/channels.go new file mode 100644 index 0000000..4ead380 --- /dev/null +++ b/internal/meshmapper/channels.go @@ -0,0 +1,307 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package meshmapper + +import ( + "bytes" + "context" + "encoding/hex" + "encoding/json" + "fmt" + "log/slog" + "net/http" + "strings" + "time" + "unicode" + "unicode/utf8" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" + "github.com/MeshCore-Beacon/beacon-server/internal/iatadb" + "github.com/MeshCore-Beacon/beacon-server/internal/keystore" +) + +const ( + MaxChannels = 256 // per region + MaxChannelBody = 64 << 10 + + channelRetry = config.MinChannelsRefresh // get_channels.php allows one request per region per 23.5h, failed or not +) + +type ChannelStore interface { + ListKnownIATAs(ctx context.Context) ([]string, error) + ListChannelCatalogues(ctx context.Context) ([]Catalogue, error) + // SaveChannelCatalogue replaces the IATA's members when the cache carries a new payload. + SaveChannelCatalogue(ctx context.Context, iata, url string, cache Cache, fingerprints [][]byte) error + ClearChannelMembers(ctx context.Context) error +} + +// ChannelKeys receives every imported key and reports the channel hashes it hadn't seen. +type ChannelKeys interface { + SetImported(entries []keystore.Entry) [][]byte +} + +type channelSource struct { + iata, country, url string + cache Cache + entries []keystore.Entry +} + +// Channels is owned by one background task. +type Channels struct { + store ChannelStore + keys ChannelKeys + dir *Directory + channelsURL func(site string) (string, bool) + client *http.Client + interval time.Duration + seen map[string]bool + sources []channelSource + onNewKeys func(hashes [][]byte) +} + +// NewChannels restores saved channel lists into keys without HTTP requests. +// Disabled, it clears MeshMapper's region membership so config scoping alone applies. +func NewChannels(ctx context.Context, cfg config.MeshMapperChannelsConfig, store ChannelStore, dir *Directory, keys ChannelKeys) (*Channels, error) { + c := &Channels{store: store, keys: keys, dir: dir, channelsURL: channelsEndpoint, interval: cfg.Interval(), seen: map[string]bool{}, + client: &http.Client{ + Timeout: requestTimeout, + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, + }} + if !cfg.Enabled { + if err := store.ClearChannelMembers(ctx); err != nil { + return nil, fmt.Errorf("clear MeshMapper channel members: %w", err) + } + return c, nil + } + saved, err := store.ListChannelCatalogues(ctx) + if err != nil { + return nil, fmt.Errorf("restore channel catalogues: %w", err) + } + latest := map[string]Catalogue{} + for _, cat := range saved { + if old, ok := latest[cat.IATA]; !ok || cat.AttemptedAt.After(old.AttemptedAt) { + latest[cat.IATA] = cat + } + } + iatas, err := store.ListKnownIATAs(ctx) + if err != nil { + return nil, fmt.Errorf("list IATAs for MeshMapper channels: %w", err) + } + c.track(iatas, latest) + c.publish() + return c, nil +} + +// OnNewKeys is told the channel hashes an import added, for backfill. +func (c *Channels) OnNewKeys(fn func(hashes [][]byte)) { c.onNewKeys = fn } + +func (c *Channels) track(iatas []string, saved map[string]Catalogue) { + for _, iata := range iatas { + if c.seen[iata] { + continue + } + c.seen[iata] = true + country := iatadb.CountryFor(iata) + if country == "" { + continue + } + s := channelSource{iata: iata, country: country} + if cat, ok := saved[iata]; ok { + s.url, s.cache = cat.URL, cat.Cache + if len(cat.Payload) > 0 { + var err error + if s.entries, err = decodeChannels(cat.Payload, iata); err != nil { + s.entries, s.cache.Payload, s.cache.ETag = nil, nil, "" + s.cache.LastError = "invalid saved catalogue; awaiting refresh" + } + } + c.log(s, "restored") + } + c.sources = append(c.sources, s) + } +} + +// Refresh makes at most one request. The scheduler serializes calls every 15s. +func (c *Channels) Refresh(ctx context.Context) (err error) { + parent := ctx + defer func() { + if parent.Err() != nil { + err = nil + } + }() + ctx, cancel := context.WithTimeout(ctx, refreshTimeout) + defer cancel() + now := time.Now().UTC() + iatas, err := c.store.ListKnownIATAs(ctx) + if err != nil { + return fmt.Errorf("list IATAs for MeshMapper channels: %w", err) + } + c.track(iatas, nil) + for n := range c.sources { + s := &c.sources[n] + if now.Before(s.cache.NextAttempt) { + continue + } + zones, fetched, err := c.dir.List(ctx, s.country, now) + if err != nil || fetched { + return err + } + if zones == nil { + continue + } + entry, listed := zones[s.iata] + endpoint, valid := "", false + if listed { + endpoint, valid = c.channelsURL(entry.url) + } + if !valid { + // No request to make; channels already imported stay until a list replaces them. + s.cache.NextAttempt, s.cache.LastError = now.Add(c.interval), "not listed" + if listed { + s.cache.LastError = "invalid site URL" + } + c.log(*s, "skipped") + continue + } + if endpoint != s.url { + s.url, s.cache.ETag = endpoint, "" + } + return c.refresh(ctx, s, now) + } + return nil +} + +func (c *Channels) refresh(ctx context.Context, s *channelSource, now time.Time) error { + update := Cache{AttemptedAt: now, NextAttempt: now.Add(c.interval)} + etag := "" + if len(s.cache.Payload) > 0 { + etag = s.cache.ETag + } + status, body, header, err := get(ctx, c.client, "Beacon-MeshMapper-Channels/1", s.url, etag, MaxChannelBody) + if err != nil { + return err + } + var entries []keystore.Entry + var retryAt time.Time + switch status { + case http.StatusOK: + if entries, err = decodeChannels(body, s.iata); err != nil { + update.LastError = "invalid response" + } else { + update.Payload = body + } + case http.StatusNotModified: + if len(s.cache.Payload) == 0 { + update.LastError = "304 without cached catalogue" + } + default: + update.LastError = statusProblem(status, header, now, &retryAt) + } + if update.LastError == "" { + etag := header.Get("ETag") + if len(etag) > 256 || strings.ContainsAny(etag, "\r\n") { + update.LastError, update.Payload, entries = "invalid ETag", nil, nil + } else { + if status == http.StatusNotModified && etag == "" { + etag = s.cache.ETag + } + update.ETag, update.CheckedAt = etag, now + } + } + if update.LastError != "" { + update.NextAttempt = now.Add(min(c.interval, channelRetry)) + if retryAt.After(update.NextAttempt) { + update.NextAttempt = retryAt + } + } + var fingerprints [][]byte + for _, e := range entries { + fingerprints = append(fingerprints, e.Fingerprint) + } + if update.Payload != nil && fingerprints == nil { + fingerprints = [][]byte{} + } + if err := c.store.SaveChannelCatalogue(ctx, s.iata, s.url, update, fingerprints); err != nil { + s.cache.NextAttempt = update.NextAttempt // never retry every tick during a DB outage + return fmt.Errorf("persist channel catalogue %s: %w", s.iata, err) + } + if update.Payload != nil { + s.cache.Payload, s.entries = update.Payload, entries + } + if !update.CheckedAt.IsZero() { + s.cache.CheckedAt, s.cache.ETag = update.CheckedAt, update.ETag + } + s.cache.AttemptedAt, s.cache.NextAttempt, s.cache.LastError = now, update.NextAttempt, update.LastError + if update.Payload != nil { + c.publish() + } + c.log(*s, "checked") + return nil +} + +func (c *Channels) publish() { + var all []keystore.Entry + for _, s := range c.sources { + all = append(all, s.entries...) + } + if added := c.keys.SetImported(all); len(added) > 0 && c.onNewKeys != nil { + c.onNewKeys(added) + } +} + +func (c *Channels) log(s channelSource, action string) { + level := slog.LevelInfo + if s.cache.LastError != "" && s.cache.LastError != "not listed" { + level = slog.LevelWarn + } + slog.Log(context.Background(), level, "MeshMapper channels "+action, "component", "meshmapper.channels", "iata", s.iata, "source", s.url, + "channels", len(s.entries), "checked_at", s.cache.CheckedAt, "next_attempt", s.cache.NextAttempt, "last_error", s.cache.LastError) +} + +func channelsEndpoint(site string) (string, bool) { return siteEndpoint(site, "get_channels.php") } + +// decodeChannels accepts a single region's list only when every key and hash is +// exactly what Beacon derives from the name, so a bad entry can't plant a wrong key. +func decodeChannels(body []byte, iata string) ([]keystore.Entry, error) { + var document struct { + GeneratedAt time.Time `json:"generated_at"` + Region string `json:"region"` + Zones []string `json:"zones"` + Count *int `json:"count"` + Channels *[]struct { + Name string `json:"name"` + Key string `json:"key"` + Hash string `json:"hash"` + Zones []string `json:"zones"` + } `json:"channels"` + } + if len(body) > MaxChannelBody || !utf8.Valid(body) { + return nil, fmt.Errorf("invalid channel list size/encoding") + } + if err := json.Unmarshal(body, &document); err != nil { + return nil, err + } + if document.GeneratedAt.IsZero() || document.Region != iata || len(document.Zones) != 1 || document.Zones[0] != iata || + document.Channels == nil || document.Count == nil || *document.Count != len(*document.Channels) || len(*document.Channels) > MaxChannels { + return nil, fmt.Errorf("invalid regional channel list") + } + entries := make([]keystore.Entry, 0, len(*document.Channels)) + seen := map[string]bool{} + for _, ch := range *document.Channels { + tag, ok := strings.CutPrefix(ch.Name, "#") + if !ok || tag == "" || len(ch.Name) > 64 || strings.ToLower(ch.Name) != ch.Name || seen[ch.Name] || + strings.ContainsFunc(tag, func(r rune) bool { return unicode.IsSpace(r) || unicode.IsControl(r) || r == '#' }) || + len(ch.Zones) != 1 || ch.Zones[0] != iata { + return nil, fmt.Errorf("invalid channel entry") + } + secret, hash, fingerprint := keystore.DeriveHashtagKey(tag) + key, err := hex.DecodeString(ch.Key) + if err != nil || !bytes.Equal(key, secret) || !strings.EqualFold(ch.Hash, hex.EncodeToString([]byte{hash})) { + return nil, fmt.Errorf("channel %s key does not match its name", ch.Name) + } + seen[ch.Name] = true + entries = append(entries, keystore.Entry{Key: secret, Fingerprint: fingerprint, Hashtag: tag, Name: ch.Name}) + } + return entries, nil +} diff --git a/internal/meshmapper/channels_test.go b/internal/meshmapper/channels_test.go new file mode 100644 index 0000000..5a3def5 --- /dev/null +++ b/internal/meshmapper/channels_test.go @@ -0,0 +1,234 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package meshmapper + +import ( + "bytes" + "context" + "encoding/hex" + "encoding/json" + "net/http" + "net/http/httptest" + "slices" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" + "github.com/MeshCore-Beacon/beacon-server/internal/keystore" +) + +func channelList(iata string, names ...string) []byte { + rows := []map[string]any{} + for _, name := range names { + secret, hash, _ := keystore.DeriveHashtagKey(strings.TrimPrefix(name, "#")) + rows = append(rows, map[string]any{"name": name, "key": hex.EncodeToString(secret), "hash": hex.EncodeToString([]byte{hash}), "zones": []string{iata}}) + } + b, _ := json.Marshal(map[string]any{"generated_at": "2026-09-30T18:00:00Z", "region": iata, "zones": []string{iata}, "count": len(rows), "channels": rows}) + return b +} + +type channelMemoryStore struct { + iatas []string + rows map[sourceKey]Cache + members map[string][][]byte + fail bool +} + +func newChannelMemoryStore(iatas ...string) *channelMemoryStore { + return &channelMemoryStore{iatas: iatas, rows: map[sourceKey]Cache{}, members: map[string][][]byte{}} +} + +func (s *channelMemoryStore) ListKnownIATAs(context.Context) ([]string, error) { + return slices.Clone(s.iatas), nil +} + +func (s *channelMemoryStore) ListChannelCatalogues(context.Context) ([]Catalogue, error) { + var out []Catalogue + for key, c := range s.rows { + out = append(out, Catalogue{IATA: key.iata, URL: key.url, Cache: c}) + } + return out, nil +} + +func (s *channelMemoryStore) SaveChannelCatalogue(_ context.Context, iata, url string, next Cache, fingerprints [][]byte) error { + if s.fail { + return context.DeadlineExceeded + } + if next.Payload != nil { + s.members[iata] = fingerprints + } + old := s.rows[sourceKey{iata, url}] + if next.Payload == nil { + next.Payload = old.Payload + } + if next.CheckedAt.IsZero() { + next.CheckedAt, next.ETag = old.CheckedAt, old.ETag + } + s.rows[sourceKey{iata, url}] = next + return nil +} + +func (s *channelMemoryStore) ClearChannelMembers(context.Context) error { + s.members = map[string][][]byte{} + return nil +} + +type channelSite struct { + *httptest.Server + list, body string + status int + retryAfter string + listCalls, channels int +} + +func newChannelSite(t *testing.T) *channelSite { + f := &channelSite{status: 200, body: string(channelList("YOW", "#ottawa-mesh"))} + f.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/get_zones.php": + f.listCalls++ + _, _ = w.Write([]byte(f.list)) + case "/get_channels.php": + f.channels++ + w.Header().Set("ETag", `"c1"`) + w.Header().Set("Retry-After", f.retryAfter) + w.WriteHeader(f.status) + if f.status == 200 { + _, _ = w.Write([]byte(f.body)) + } + default: + t.Errorf("unexpected path %s", r.URL.Path) + } + })) + f.list = `{"country":"CA","zones":[{"code":"YOW","url":"` + f.URL + `/","has_boundary":true,"group":null}]}` + t.Cleanup(f.Close) + return f +} + +type channelHarness struct { + c *Channels + keys *keystore.MapKeyStore + added [][]byte +} + +func newChannelHarness(t *testing.T, f *channelSite, store *channelMemoryStore, lists *zoneListMemory, enabled bool) *channelHarness { + t.Helper() + dir := NewDirectory(lists) + dir.listURL = f.URL + "/get_zones.php" + if err := dir.Restore(context.Background()); err != nil { + t.Fatal(err) + } + h := &channelHarness{keys: keystore.NewMapKeyStore(nil)} + c, err := NewChannels(context.Background(), config.MeshMapperChannelsConfig{Enabled: enabled}, store, dir, h.keys) + if err != nil { + t.Fatal(err) + } + c.channelsURL = func(site string) (string, bool) { return site + "get_channels.php", strings.HasPrefix(site, f.URL) } + c.OnNewKeys(func(hashes [][]byte) { h.added = append(h.added, hashes...) }) + h.c = c + return h +} + +func (h *channelHarness) refresh(t *testing.T, n int) { + t.Helper() + for range n { + if err := h.c.Refresh(context.Background()); err != nil { + t.Fatal(err) + } + } +} + +func TestChannelsImportKeysAndMembers(t *testing.T) { + f := newChannelSite(t) + store := newChannelMemoryStore("YOW") + h := newChannelHarness(t, f, store, newZoneListMemory(), true) + h.refresh(t, 2) + secret, hash, fingerprint := keystore.DeriveHashtagKey("ottawa-mesh") + got := h.keys.GetKey([]byte{hash}) + if len(got) != 1 || !bytes.Equal(got[0].Key, secret) || got[0].Name != "#ottawa-mesh" || got[0].Hashtag != "ottawa-mesh" { + t.Fatalf("key not imported: %+v", got) + } + if len(store.members["YOW"]) != 1 || !bytes.Equal(store.members["YOW"][0], fingerprint) { + t.Fatal("YOW membership not saved", store.members) + } + if len(h.added) != 1 || h.added[0][0] != hash { + t.Fatal("new key not reported for backfill", h.added) + } + if b := h.c.sources[0].cache; b.NextAttempt.Sub(b.AttemptedAt) != 24*time.Hour { + t.Fatal("refresh interval", b.NextAttempt.Sub(b.AttemptedAt)) + } +} + +func TestChannelsRejectMismatchKeepsLastGood(t *testing.T) { + f := newChannelSite(t) + store := newChannelMemoryStore("YOW") + h := newChannelHarness(t, f, store, newZoneListMemory(), true) + h.refresh(t, 2) + good := string(channelList("YOW", "#ottawa-mesh")) + secret, hash, _ := keystore.DeriveHashtagKey("ottawa-mesh") + for name, body := range map[string]string{ + "planted key": strings.Replace(good, hex.EncodeToString(secret), strings.Repeat("0", 32), 1), + "wrong key": strings.Replace(string(channelList("YOW", "#other")), `"#other"`, `"#planted"`, 1), + "wrong hash": strings.Replace(good, `"hash":"`+hex.EncodeToString([]byte{hash})+`"`, `"hash":"zz"`, 1), + "other region": string(channelList("YYZ", "#ottawa-mesh")), + "bad count": strings.Replace(good, `"count":1`, `"count":2`, 1), + "uppercase": string(channelList("YOW", "#Ottawa")), + "group zones": strings.Replace(good, `"zones":["YOW"]}`, `"zones":["YOW","YYZ"]}`, 1), + "missing count": strings.Replace(good, `"count":1,`, ``, 1), + } { + f.body = body + h.c.sources[0].cache.NextAttempt = time.Time{} + h.refresh(t, 1) + s := h.c.sources[0] + if s.cache.LastError != "invalid response" || len(h.keys.GetKey([]byte{hash})) != 1 || len(store.members["YOW"]) != 1 { + t.Fatalf("%s: accepted or dropped last good list: %q", name, s.cache.LastError) + } + if s.cache.NextAttempt.Sub(s.cache.AttemptedAt) != 24*time.Hour { + t.Fatalf("%s: retried before the rate limit", name) + } + } +} + +func TestChannelsRateLimitAndFailuresWait(t *testing.T) { + for _, tc := range []struct { + status int + retryAfter string + want time.Duration + }{{503, "", 24 * time.Hour}, {429, "60", 24 * time.Hour}, {429, "172800", 48 * time.Hour}} { + f := newChannelSite(t) + f.status, f.retryAfter = tc.status, tc.retryAfter + h := newChannelHarness(t, f, newChannelMemoryStore("YOW"), newZoneListMemory(), true) + h.refresh(t, 3) + s := h.c.sources[0] + if f.channels != 1 || s.cache.NextAttempt.Sub(s.cache.AttemptedAt) != tc.want { + t.Fatalf("HTTP %d Retry-After %q: calls=%d wait=%v", tc.status, tc.retryAfter, f.channels, s.cache.NextAttempt.Sub(s.cache.AttemptedAt)) + } + } +} + +func TestChannelsRestoreWithoutRequests(t *testing.T) { + f := newChannelSite(t) + store, lists := newChannelMemoryStore("YOW"), newZoneListMemory() + newChannelHarness(t, f, store, lists, true).refresh(t, 2) + restarted := newChannelHarness(t, f, store, lists, true) + _, hash, _ := keystore.DeriveHashtagKey("ottawa-mesh") + if len(restarted.keys.GetKey([]byte{hash})) != 1 { + t.Fatal("saved channels not restored") + } + restarted.refresh(t, 2) + if f.listCalls != 1 || f.channels != 1 { + t.Fatal("restart made requests", f.listCalls, f.channels) + } +} + +func TestChannelsDisabledClearsMembers(t *testing.T) { + f := newChannelSite(t) + store := newChannelMemoryStore("YOW") + store.members["YOW"] = [][]byte{{1}} + h := newChannelHarness(t, f, store, newZoneListMemory(), false) + if len(store.members) != 0 || len(h.c.sources) != 0 { + t.Fatal("disabled import kept membership or sources") + } +} diff --git a/internal/meshmapper/directory.go b/internal/meshmapper/directory.go new file mode 100644 index 0000000..20f7503 --- /dev/null +++ b/internal/meshmapper/directory.go @@ -0,0 +1,248 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package meshmapper + +import ( + "cmp" + "context" + "encoding/json" + "fmt" + "log/slog" + "net/http" + "net/url" + "regexp" + "slices" + "strings" + "sync" + "time" + "unicode" +) + +type zoneEntry struct { + url string + hasBoundary bool + name string // short display name, e.g. "Nanaimo" + lat, lon *float64 // nil when missing or out of range +} + +type zoneGroup struct { + name string + members []string +} + +type zoneList struct { + zones map[string]zoneEntry + groups map[string]zoneGroup + etag string + fetchedAt, nextAttempt time.Time +} + +// ZoneList is one country's saved get_zones.php response and fetch state. +type ZoneList struct { + Country string + Payload json.RawMessage + ETag string + FetchedAt, AttemptedAt, NextAttempt time.Time + LastError string +} + +type DirectoryStore interface { + ListZoneLists(ctx context.Context) ([]ZoneList, error) + SaveZoneList(ctx context.Context, l ZoneList) error +} + +// Directory caches MeshMapper's per-country zone lists, shared by the zones and scopes imports. +type Directory struct { + mu sync.Mutex + store DirectoryStore + client *http.Client + listURL string + lists map[string]*zoneList + version int // bumped whenever a list is replaced +} + +func NewDirectory(store DirectoryStore) *Directory { + return &Directory{store: store, listURL: ZonesURL, lists: map[string]*zoneList{}, client: &http.Client{ + Timeout: requestTimeout, + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, + }} +} + +// Restore loads saved lists so a restart doesn't spend each country's daily request. +func (d *Directory) Restore(ctx context.Context) error { + saved, err := d.store.ListZoneLists(ctx) + if err != nil { + return fmt.Errorf("restore MeshMapper zone lists: %w", err) + } + d.mu.Lock() + defer d.mu.Unlock() + for _, l := range saved { + list := &zoneList{nextAttempt: l.NextAttempt} + if zones, groups, err := decodeZones(l.Payload, l.Country); err == nil { + list.zones, list.groups, list.etag, list.fetchedAt = zones, groups, l.ETag, l.FetchedAt + d.version++ + } + d.lists[l.Country] = list + } + return nil +} + +// List returns the country's fresh zone list. fetched reports that this call +// spent the caller's one request; nil zones without fetched means back off. +func (d *Directory) List(ctx context.Context, country string, now time.Time) (zones map[string]zoneEntry, fetched bool, err error) { + d.mu.Lock() + defer d.mu.Unlock() + list := d.lists[country] + if list != nil && list.zones != nil && now.Sub(list.fetchedAt) < zoneListFresh { + return list.zones, false, nil + } + if list != nil && now.Before(list.nextAttempt) { + return nil, false, nil + } + return nil, true, d.fetch(ctx, country, now) +} + +func (d *Directory) fetch(ctx context.Context, country string, now time.Time) error { + list := d.lists[country] + if list == nil { + list = &zoneList{} + d.lists[country] = list + } + endpoint := d.listURL + "?country=" + url.QueryEscape(country) + etag := "" + if list.zones != nil { + etag = list.etag + } + status, body, header, err := get(ctx, d.client, "Beacon-MeshMapper-Zones/1", endpoint, etag, MaxZoneList) + if err != nil { + return err + } + problem := "" + var payload json.RawMessage + var retryAt time.Time + switch status { + case http.StatusOK: + zones, groups, decodeErr := decodeZones(body, country) + if decodeErr != nil { + problem = "invalid response" + } else { + list.zones, list.groups, list.etag = zones, groups, header.Get("ETag") + payload = body + d.version++ + } + case http.StatusNotModified: + if list.zones == nil { + problem = "304 without cached list" + } + default: + problem = statusProblem(status, header, now, &retryAt) + } + // Every request counts against the country's 23.5h allowance, failed or not. + list.nextAttempt = now.Add(zoneListFresh) + if retryAt.After(list.nextAttempt) { + list.nextAttempt = retryAt + } + saved := ZoneList{Country: country, Payload: payload, AttemptedAt: now, NextAttempt: list.nextAttempt, LastError: problem} + if problem == "" { + list.fetchedAt = now + saved.FetchedAt, saved.ETag = now, list.etag + } + level := slog.LevelInfo + if problem != "" { + level = slog.LevelWarn + } + slog.Log(ctx, level, "MeshMapper zone list checked", "component", "meshmapper.zones", "country", country, + "zones", len(list.zones), "groups", len(list.groups), "next_attempt", list.nextAttempt, "last_error", problem) + if err := d.store.SaveZoneList(ctx, saved); err != nil { + return fmt.Errorf("persist MeshMapper zone list %s: %w", country, err) + } + return nil +} + +// snapshot merges every loaded list's zones and groups by code. complete is false until +// each country has a list, so a cold start can't look like every group disappearing. +func (d *Directory) snapshot(countries []string) (merged map[string]zoneGroup, zones map[string]zoneEntry, version int, complete bool) { + d.mu.Lock() + defer d.mu.Unlock() + merged, zones = map[string]zoneGroup{}, map[string]zoneEntry{} + for _, country := range countries { + list := d.lists[country] + if list == nil || list.zones == nil { + return nil, nil, d.version, false + } + for code, z := range list.zones { + zones[code] = z + } + for code, g := range list.groups { + if have, ok := merged[code]; ok { + g.members = append(slices.Clone(have.members), g.members...) + slices.Sort(g.members) + g.members = slices.Compact(g.members) + g.name = have.name + } + merged[code] = g + } + } + return merged, zones, d.version, true +} + +var ( + groupCode = regexp.MustCompile(`^[A-Z0-9]{2,16}$`) + memberIATA = regexp.MustCompile(`^[A-Z]{3}$`) +) + +func decodeZones(body []byte, country string) (map[string]zoneEntry, map[string]zoneGroup, error) { + var document struct { + Country string `json:"country"` + Zones *[]struct { + Code string `json:"code"` + Name string `json:"name"` + ShortName string `json:"short_name"` + Lat *float64 `json:"lat"` + Lon *float64 `json:"lon"` + URL string `json:"url"` + HasBoundary bool `json:"has_boundary"` + } `json:"zones"` + Groups []struct { + Code string `json:"code"` + Name string `json:"name"` + Members []string `json:"members"` + } `json:"groups"` + } + if err := json.Unmarshal(body, &document); err != nil { + return nil, nil, err + } + if !strings.EqualFold(document.Country, country) || document.Zones == nil { + return nil, nil, fmt.Errorf("invalid zone list") + } + zones := make(map[string]zoneEntry, len(*document.Zones)) + for _, zone := range *document.Zones { + entry := zoneEntry{url: zone.URL, hasBoundary: zone.HasBoundary, name: cmp.Or(zone.ShortName, zone.Name)} + if entry.name = strings.TrimSpace(entry.name); len(entry.name) > 128 || strings.ContainsFunc(entry.name, unicode.IsControl) { + entry.name = "" + } + if zone.Lat != nil && zone.Lon != nil && *zone.Lat >= -90 && *zone.Lat <= 90 && *zone.Lon >= -180 && *zone.Lon <= 180 { + entry.lat, entry.lon = zone.Lat, zone.Lon + } + zones[strings.ToUpper(zone.Code)] = entry + } + groups := map[string]zoneGroup{} + for _, g := range document.Groups { + code, name := strings.ToUpper(g.Code), strings.TrimSpace(g.Name) + if !groupCode.MatchString(code) || name == "" || len(name) > 128 || strings.ContainsFunc(name, unicode.IsControl) { + continue // a bad group never blocks the zone list + } + var members []string + for _, m := range g.Members { + if m = strings.ToUpper(m); memberIATA.MatchString(m) { + members = append(members, m) + } + } + slices.Sort(members) + if members = slices.Compact(members); len(members) > 0 { + groups[code] = zoneGroup{name: name, members: members} + } + } + return zones, groups, nil +} diff --git a/internal/meshmapper/groups.go b/internal/meshmapper/groups.go new file mode 100644 index 0000000..588e1ee --- /dev/null +++ b/internal/meshmapper/groups.go @@ -0,0 +1,201 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package meshmapper + +import ( + "cmp" + "context" + "fmt" + "log/slog" + "slices" + "strings" +) + +// RegionState is a stored region; imported ones are also what SaveImportedRegion writes. +type RegionState struct { + Slug, Name string + DisplayOrder int + Imported bool + IATAs []string + CenterLat, CenterLng *float64 +} + +// IATADetails is an IATA's display name and approximate location. +type IATADetails struct { + IATA string + Name string + Lat, Lng *float64 +} + +// SetConfiguredIATAs names the IATAs whose config entry sets a name or location; those win. +func (z *Zones) SetConfiguredIATAs(iatas []string) { + z.configured = map[string]bool{} + for _, iata := range iatas { + z.configured[iata] = true + } +} + +// OnIATAsChange is told whenever imported IATA names or locations change. +func (z *Zones) OnIATAsChange(fn func(ctx context.Context)) { z.onIATAs = fn } + +// OnRegionsChange is told whenever the imported regions change, for cache invalidation. +func (z *Zones) OnRegionsChange(fn func(ctx context.Context)) { z.onRegions = fn } + +// pruneGroups drops every imported region when group import is off. +func (z *Zones) pruneGroups(ctx context.Context) error { + pruned, err := z.store.PruneImportedRegions(ctx, nil) + if err != nil { + return fmt.Errorf("prune MeshMapper regions: %w", err) + } + z.regionsRemoved(pruned) + if len(pruned) > 0 { + z.regionsChanged(ctx) + } + return nil +} + +// syncDirectory applies the loaded zone lists to IATA details and, when enabled, +// imported regions. It only runs once every tracked country has a list, and again +// when a list or the IATA set changes. +func (z *Zones) syncDirectory(ctx context.Context, iatas []string) error { + var countries []string + for _, r := range z.regions { + countries = append(countries, r.country) + } + slices.Sort(countries) + groups, zones, version, complete := z.dir.snapshot(slices.Compact(countries)) + if !complete || (z.groupsSynced && version == z.groupsVersion && len(iatas) == z.groupsKnown) { + return nil + } + if err := z.syncIATADetails(ctx, zones); err != nil { + return err + } + if z.importGroups { + if err := z.syncGroups(ctx, iatas, groups, zones); err != nil { + return err + } + } + z.groupsSynced, z.groupsVersion, z.groupsKnown = true, version, len(iatas) + return nil +} + +// syncIATADetails names and locates each listed IATA unless its config entry does. +func (z *Zones) syncIATADetails(ctx context.Context, zones map[string]zoneEntry) error { + stored, err := z.store.ListIATADetails(ctx) + if err != nil { + return fmt.Errorf("list IATAs for MeshMapper details: %w", err) + } + changed := false + for _, cur := range stored { + zone, ok := zones[cur.IATA] + if !ok || z.configured[cur.IATA] || zone.name == "" && zone.lat == nil { + continue + } + name := cmp.Or(zone.name, cur.Name) + lat, lng := cur.Lat, cur.Lng + if zone.lat != nil { + lat, lng = zone.lat, zone.lon + } + if name == cur.Name && sameFloat(lat, cur.Lat) && sameFloat(lng, cur.Lng) { + continue + } + if err := z.store.UpsertIATADetails(ctx, cur.IATA, name, lat, lng); err != nil { + return fmt.Errorf("save MeshMapper details for %s: %w", cur.IATA, err) + } + changed = true + } + if changed && z.onIATAs != nil { + z.onIATAs(ctx) + } + return nil +} + +func (z *Zones) syncGroups(ctx context.Context, iatas []string, groups map[string]zoneGroup, zones map[string]zoneEntry) error { + known := make(map[string]bool, len(iatas)) + for _, iata := range iatas { + known[iata] = true + } + state, err := z.store.ListRegionState(ctx) + if err != nil { + return fmt.Errorf("list regions for MeshMapper groups: %w", err) + } + stored, handOrder := map[string]RegionState{}, 0 + for _, r := range state { + stored[r.Slug] = r + if !r.Imported { + handOrder = max(handOrder, r.DisplayOrder) + } + } + var want []RegionState + for code, g := range groups { + if slices.ContainsFunc(g.members, func(m string) bool { return known[m] }) { + lat, lng := center(g.members, zones) + want = append(want, RegionState{Slug: strings.ToLower(code), Name: g.name, Imported: true, IATAs: g.members, CenterLat: lat, CenterLng: lng}) + } + } + slices.SortFunc(want, func(a, b RegionState) int { return cmp.Or(cmp.Compare(a.Name, b.Name), cmp.Compare(a.Slug, b.Slug)) }) + keep, changed := []string{}, false + for n, r := range want { + r.DisplayOrder = handOrder + 1 + n + if cur, ok := stored[r.Slug]; ok && !cur.Imported { + slog.Info("MeshMapper group skipped: a configured region uses its slug", "component", "meshmapper.zones", "slug", r.Slug) + continue + } else if ok && cur.Name == r.Name && cur.DisplayOrder == r.DisplayOrder && slices.Equal(cur.IATAs, r.IATAs) && + sameFloat(cur.CenterLat, r.CenterLat) && sameFloat(cur.CenterLng, r.CenterLng) { + keep = append(keep, r.Slug) + continue + } + saved, err := z.store.SaveImportedRegion(ctx, r) + if err != nil { + return fmt.Errorf("save MeshMapper region %s: %w", r.Slug, err) + } + if !saved { + slog.Info("MeshMapper group skipped: a configured region uses its slug", "component", "meshmapper.zones", "slug", r.Slug) + continue + } + keep, changed = append(keep, r.Slug), true + slog.Info("MeshMapper region imported", "component", "meshmapper.zones", "slug", r.Slug, "name", r.Name, "iatas", r.IATAs) + } + pruned, err := z.store.PruneImportedRegions(ctx, keep) + if err != nil { + return fmt.Errorf("prune MeshMapper regions: %w", err) + } + z.regionsRemoved(pruned) + if changed || len(pruned) > 0 { + z.regionsChanged(ctx) + } + return nil +} + +// center averages the members' zone locations; nil when none has one. +func center(members []string, zones map[string]zoneEntry) (*float64, *float64) { + var lat, lng float64 + n := 0 + for _, m := range members { + if z := zones[m]; z.lat != nil { + lat, lng, n = lat+*z.lat, lng+*z.lon, n+1 + } + } + if n == 0 { + return nil, nil + } + lat, lng = lat/float64(n), lng/float64(n) + return &lat, &lng +} + +func sameFloat(a, b *float64) bool { + return a == nil && b == nil || a != nil && b != nil && *a == *b +} + +func (z *Zones) regionsRemoved(slugs []string) { + for _, slug := range slugs { + slog.Info("MeshMapper region removed", "component", "meshmapper.zones", "slug", slug) + } +} + +func (z *Zones) regionsChanged(ctx context.Context) { + if z.onRegions != nil { + z.onRegions(ctx) + } +} diff --git a/internal/meshmapper/groups_test.go b/internal/meshmapper/groups_test.go new file mode 100644 index 0000000..4caefa3 --- /dev/null +++ b/internal/meshmapper/groups_test.go @@ -0,0 +1,184 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package meshmapper + +import ( + "context" + "slices" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" +) + +const groupedList = `{"country":"CA","zones":[` + + `{"code":"YOW","name":"Ottawa, CA","short_name":"Ottawa","lat":45,"lon":-76,"url":"URL/","has_boundary":true,"group":"ONQC"},` + + `{"code":"YUL","name":"Montreal, CA","lat":46,"lon":-74,"url":"URL/","has_boundary":true,"group":"ONQC"},` + + `{"code":"YQB","name":"Quebec City, CA","url":"URL/","has_boundary":true,"group":"ONQC"}],"groups":[` + + `{"code":"ONQC","name":"Ottawa-Quebec Corridor","members":["YUL","YOW","YQB"],"url":"https://onqc.meshmapper.net/"},` + + `{"code":"GOLM","name":"Great Ontario Lake Mesh","members":["YYZ","YKF"],"url":"https://golm.meshmapper.net/"}]}` + +type groupHarness struct { + z *Zones + f *fakeMeshMapper + store *zoneMemoryStore + invalidated int +} + +func newGroupHarness(t *testing.T, f *fakeMeshMapper, store *zoneMemoryStore, importGroups bool) *groupHarness { + t.Helper() + if store.iatas == nil { + store.iatas = []string{"YOW"} + } + if store.rows == nil { + store.rows = map[string]Boundary{} + } + h := &groupHarness{f: f, store: store} + dir := NewDirectory(newZoneListMemory()) + dir.listURL = f.URL + "/get_zones.php" + h.z = NewZones(config.MeshMapperZonesConfig{Enabled: true, ImportGroups: importGroups}, store, dir) + h.z.boundsURL = func(site string) (string, bool) { return site + "get_geojson.php", true } + h.z.OnRegionsChange(func(context.Context) { h.invalidated++ }) + if err := h.z.Restore(context.Background()); err != nil { + t.Fatal(err) + } + return h +} + +func groupedFake(t *testing.T) *fakeMeshMapper { + f := newFakeMeshMapper(t) + f.list = strings.ReplaceAll(groupedList, "URL", f.URL) + return f +} + +// refetch makes the country list and YOW due, then runs the fetch and the sync tick. +func (h *groupHarness) refetch(t *testing.T) { + t.Helper() + if list := h.z.dir.lists["CA"]; list != nil { + list.fetchedAt, list.nextAttempt = time.Time{}, time.Time{} + } + for _, r := range h.z.regions { + r.b.NextAttempt = time.Time{} + } + for range 2 { + if err := h.z.Refresh(context.Background()); err != nil { + t.Fatal(err) + } + } +} + +func TestGroupImportedWhenAMemberIsKnown(t *testing.T) { + h := newGroupHarness(t, groupedFake(t), &zoneMemoryStore{}, true) + h.refetch(t) + got, ok := h.store.regions["onqc"] + if !ok || got.Name != "Ottawa-Quebec Corridor" || !slices.Equal(got.IATAs, []string{"YOW", "YQB", "YUL"}) || got.DisplayOrder != 1 { + t.Fatalf("group not imported with full members: %+v", h.store.regions) + } + if _, ok := h.store.regions["golm"]; ok { + t.Fatal("group with no known member imported") + } + if got.CenterLat == nil || *got.CenterLat != 45.5 || *got.CenterLng != -75 { + t.Fatalf("center should average the located members: %v, %v", got.CenterLat, got.CenterLng) + } + if !slices.Contains(h.store.iatas, "YQB") || !slices.Contains(h.store.iatas, "YUL") { + t.Fatal("members not created as IATAs", h.store.iatas) + } + if h.invalidated != 1 { + t.Fatal("region caches not invalidated", h.invalidated) + } + h.refetch(t) + if h.invalidated != 1 { + t.Fatal("unchanged groups invalidated again", h.invalidated) + } +} + +func TestGroupSlugClashKeepsConfiguredRegion(t *testing.T) { + hand := RegionState{Slug: "onqc", Name: "Mine", DisplayOrder: 4, IATAs: []string{"YOW"}} + store := &zoneMemoryStore{regions: map[string]RegionState{"onqc": hand}} + h := newGroupHarness(t, groupedFake(t), store, true) + h.refetch(t) + if got := store.regions["onqc"]; got.Imported || got.Name != "Mine" || !slices.Equal(got.IATAs, []string{"YOW"}) || h.invalidated != 0 { + t.Fatalf("configured region replaced: %+v", got) + } +} + +func TestGroupDroppedRemovesRegion(t *testing.T) { + h := newGroupHarness(t, groupedFake(t), &zoneMemoryStore{}, true) + h.refetch(t) + h.f.list = strings.ReplaceAll(`{"country":"CA","zones":[{"code":"YOW","url":"URL/","has_boundary":true,"group":null}],"groups":[]}`, "URL", h.f.URL) + h.refetch(t) + if len(h.store.regions) != 0 || h.invalidated != 2 { + t.Fatalf("dropped group kept: %+v invalidated=%d", h.store.regions, h.invalidated) + } +} + +func TestGroupImportOffRemovesImportedRegions(t *testing.T) { + hand := RegionState{Slug: "east", Name: "East", IATAs: []string{"YOW"}} + imported := RegionState{Slug: "onqc", Name: "Ottawa-Quebec Corridor", Imported: true, IATAs: []string{"YOW"}} + store := &zoneMemoryStore{regions: map[string]RegionState{"east": hand, "onqc": imported}} + h := newGroupHarness(t, groupedFake(t), store, false) + if _, ok := store.regions["onqc"]; ok || h.invalidated != 1 { + t.Fatal("imported region kept with import_groups off") + } + if _, ok := store.regions["east"]; !ok { + t.Fatal("configured region removed") + } + h.refetch(t) + if len(store.regions) != 1 { + t.Fatal("groups imported with import_groups off", store.regions) + } +} + +func TestGroupFailedFetchKeepsGroups(t *testing.T) { + h := newGroupHarness(t, groupedFake(t), &zoneMemoryStore{}, true) + h.refetch(t) + h.f.listStatus = 500 + h.z.groupsSynced = false // force a reconcile against the retained list + h.refetch(t) + if _, ok := h.store.regions["onqc"]; !ok { + t.Fatal("failed fetch dropped imported region") + } +} + +func TestGroupColdStartWaitsForEveryList(t *testing.T) { + old := RegionState{Slug: "old", Name: "Old", Imported: true, IATAs: []string{"YOW"}} + store := &zoneMemoryStore{regions: map[string]RegionState{"old": old}} + f := groupedFake(t) + f.listStatus = 500 + h := newGroupHarness(t, f, store, true) + h.refetch(t) + if _, ok := store.regions["old"]; !ok { + t.Fatal("imported regions pruned before any list loaded") + } +} + +func TestZoneListNamesAndLocatesIATAs(t *testing.T) { + h := newGroupHarness(t, groupedFake(t), &zoneMemoryStore{iatas: []string{"YOW", "YUL", "YQB", "YYZ"}}, false) + h.z.SetConfiguredIATAs([]string{"YUL"}) + changed := 0 + h.z.OnIATAsChange(func(context.Context) { changed++ }) + h.refetch(t) + yow := h.store.details["YOW"] + if yow.Name != "Ottawa" || yow.Lat == nil || *yow.Lat != 45 || *yow.Lng != -76 { + t.Fatalf("YOW not named and located from short_name: %+v", yow) + } + if yqb := h.store.details["YQB"]; yqb.Name != "Quebec City, CA" || yqb.Lat != nil { + t.Fatalf("YQB should fall back to name and stay unlocated: %+v", yqb) + } + if _, ok := h.store.details["YUL"]; ok { + t.Fatal("configured IATA overwritten") + } + if _, ok := h.store.details["YYZ"]; ok { + t.Fatal("unlisted IATA written") + } + if changed != 1 || h.store.writes != 2 { + t.Fatal("writes/invalidations", h.store.writes, changed) + } + h.z.groupsSynced = false + h.refetch(t) + if h.store.writes != 2 || changed != 1 { + t.Fatal("unchanged details rewritten", h.store.writes, changed) + } +} diff --git a/internal/meshmapper/scopes.go b/internal/meshmapper/scopes.go index 3f210be..330686b 100644 --- a/internal/meshmapper/scopes.go +++ b/internal/meshmapper/scopes.go @@ -20,15 +20,16 @@ import ( "unicode/utf8" "github.com/MeshCore-Beacon/beacon-server/internal/config" + "github.com/MeshCore-Beacon/beacon-server/internal/iatadb" "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" ) const ( - MaxScopes = 64 // per source; at most 16 configured sources (1024 imported identities) + MaxScopes = 64 // per source MaxBody = 64 << 10 PollInterval = 15 * time.Second // one source per tick, at most four requests/minute - failureRetry = 5 * time.Minute // transient failures shouldn't cost a whole refresh interval + failureRetry = config.MinScopesRefresh // get_scopes.php allows one request per region per 55m ) // Cache stores source provenance and freshness separately from packet evidence. @@ -39,28 +40,37 @@ type Cache struct { LastError string } +// Catalogue is one saved source snapshot. +type Catalogue struct { + IATA, URL string + Cache +} + type Store interface { - GetScopeCatalogue(context.Context, string, string) (*Cache, error) + ListKnownIATAs(context.Context) ([]string, error) + ListScopeCatalogues(context.Context) ([]Catalogue, error) SaveScopeCatalogue(context.Context, string, string, Cache, []scopestore.Entry) error } type source struct { - iata, url string - cache Cache - entries []scopestore.Entry - generated time.Time + iata, country, url string + cache Cache + entries []scopestore.Entry + generated time.Time } // Importer is owned by one background task; ScopeStore synchronizes its consumers. type Importer struct { - catalogues atomic.Value // immutable []Catalogue, refreshed off the ingest path + catalogues atomic.Value // immutable []PublicScopeCatalogue, off the ingest path store Store scopes *scopestore.ScopeStore manual []scopestore.Entry + dir *Directory + scopesURL func(site string) (string, bool) + seen map[string]bool sources []source interval time.Duration client *http.Client - retryAfter time.Time onChange func(context.Context) } @@ -68,66 +78,111 @@ type Importer struct { func (i *Importer) SetCacheInvalidator(fn func(context.Context)) { i.onChange = fn } // New restores validated snapshots before ingestion, without making HTTP requests. -func New(ctx context.Context, cfg config.MeshMapperScopesConfig, store Store, scopes *scopestore.ScopeStore, manual []scopestore.Entry) (*Importer, error) { - i := &Importer{store: store, scopes: scopes, manual: manual, interval: cfg.Interval(), client: &http.Client{ - Timeout: 10 * time.Second, - CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, - }} +func New(ctx context.Context, cfg config.MeshMapperScopesConfig, store Store, dir *Directory, scopes *scopestore.ScopeStore, manual []scopestore.Entry) (*Importer, error) { + i := &Importer{store: store, scopes: scopes, manual: manual, dir: dir, scopesURL: scopesEndpoint, seen: map[string]bool{}, + interval: cfg.Interval(), client: &http.Client{ + Timeout: requestTimeout, + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, + }} if !cfg.Enabled { return i, nil } - keys := make([]string, 0, len(cfg.Sources)) - for key := range cfg.Sources { - keys = append(keys, key) + saved, err := store.ListScopeCatalogues(ctx) + if err != nil { + return nil, fmt.Errorf("restore scope catalogues: %w", err) } - slices.Sort(keys) - for _, key := range keys { - s := source{iata: key, url: cfg.Sources[key]} - cached, err := store.GetScopeCatalogue(ctx, s.iata, s.url) - if err != nil { - return nil, fmt.Errorf("restore scope catalogue %s: %w", key, err) + // A site that moved leaves its old row behind; the latest attempt wins. + latest := map[string]Catalogue{} + for _, c := range saved { + if old, ok := latest[c.IATA]; !ok || c.AttemptedAt.After(old.AttemptedAt) { + latest[c.IATA] = c } - if cached != nil { - s.cache = *cached - if len(cached.Payload) > 0 { - s.entries, s.generated, err = decode(cached.Payload, key) + } + iatas, err := store.ListKnownIATAs(ctx) + if err != nil { + return nil, fmt.Errorf("list IATAs for MeshMapper scopes: %w", err) + } + i.track(iatas, latest) + i.publish() + i.publishCatalogues() + return i, nil +} + +// track adds a source for each IATA not seen before. +func (i *Importer) track(iatas []string, saved map[string]Catalogue) { + for _, iata := range iatas { + if i.seen[iata] { + continue + } + i.seen[iata] = true + country := iatadb.CountryFor(iata) + if country == "" { + continue + } + s := source{iata: iata, country: country} + if c, ok := saved[iata]; ok { + s.url, s.cache = c.URL, c.Cache + if len(c.Payload) > 0 { + var err error + s.entries, s.generated, err = decode(c.Payload, iata) if err != nil { s.entries = nil s.cache.Payload, s.cache.ETag = nil, "" s.cache.LastError = "invalid saved catalogue; awaiting refresh" } } - if cached.LastError == "HTTP 429" && cached.NextAttempt.After(i.retryAfter) { - i.retryAfter = cached.NextAttempt - } + i.log(s, "restored") } i.sources = append(i.sources, s) - i.log(s, "restored") } - i.publish() - i.publishCatalogues() - return i, nil } // Refresh checks only one due source. The scheduler serializes calls every 15s. func (i *Importer) Refresh(ctx context.Context) (err error) { + defer i.publishCatalogues() parent := ctx defer func() { if parent.Err() != nil { err = nil } }() - ctx, cancel := context.WithTimeout(ctx, 15*time.Second) + ctx, cancel := context.WithTimeout(ctx, refreshTimeout) defer cancel() now := time.Now().UTC() - if now.Before(i.retryAfter) { - return nil + iatas, err := i.store.ListKnownIATAs(ctx) + if err != nil { + return fmt.Errorf("list IATAs for MeshMapper scopes: %w", err) } + i.track(iatas, nil) for n := range i.sources { s := &i.sources[n] if now.Before(s.cache.NextAttempt) { continue } + zones, fetched, err := i.dir.List(ctx, s.country, now) + if err != nil || fetched { + return err + } + if zones == nil { + continue + } + entry, listed := zones[s.iata] + endpoint, valid := "", false + if listed { + endpoint, valid = i.scopesURL(entry.url) + } + if !valid { + // No request to make; names already imported stay until a catalogue replaces them. + s.cache.NextAttempt, s.cache.LastError = now.Add(i.interval), "not listed" + if listed { + s.cache.LastError = "invalid site URL" + } + i.log(*s, "skipped") + continue + } + if endpoint != s.url { + s.url, s.cache.ETag = endpoint, "" + } return i.refresh(ctx, s, now) } return nil @@ -195,15 +250,12 @@ func (i *Importer) refresh(ctx context.Context, s *source, now time.Time) error } } if update.LastError != "" { - // Transient failures shouldn't cost a whole refresh interval, but never beat a Retry-After. + // Failures retry before the full interval, but never sooner than the rate limit or a Retry-After. update.NextAttempt = now.Add(min(i.interval, failureRetry)) if retryAfter.After(update.NextAttempt) { update.NextAttempt = retryAfter } } - if update.LastError == "HTTP 429" { - i.retryAfter = update.NextAttempt - } if err := i.store.SaveScopeCatalogue(ctx, s.iata, s.url, update, entries); err != nil { // Avoid a retry every scheduler tick during a DB outage; never publish uncommitted keys. s.cache.NextAttempt = update.NextAttempt @@ -227,7 +279,6 @@ func (i *Importer) refresh(ctx context.Context, s *source, now time.Time) error i.onChange(ctx) } } - i.publishCatalogues() i.log(*s, "checked") return nil } @@ -237,8 +288,10 @@ func (i *Importer) publish() { for _, e := range i.manual { byName[e.Name] = e } + members := make(map[string][]string, len(i.sources)) for _, s := range i.sources { for _, candidate := range s.entries { + members[s.iata] = append(members[s.iata], candidate.Name) entry, exists := byName[candidate.Name] if exists && entry.IATAs == nil { continue @@ -260,17 +313,20 @@ func (i *Importer) publish() { entries = append(entries, byName[name]) } i.scopes.Load(entries) + i.scopes.SetCatalogueMembers(members) } func (i *Importer) log(s source, action string) { level := slog.LevelInfo - if s.cache.LastError != "" { + if s.cache.LastError != "" && s.cache.LastError != "not listed" { level = slog.LevelWarn } slog.Log(context.Background(), level, "MeshMapper scopes "+action, "component", "meshmapper.scopes", "iata", s.iata, "source", s.url, "names", len(s.entries), "generated_at", s.generated, "checked_at", s.cache.CheckedAt, "next_attempt", s.cache.NextAttempt, "last_error", s.cache.LastError) } +func scopesEndpoint(site string) (string, bool) { return siteEndpoint(site, "get_scopes.php") } + func decode(body []byte, iata string) ([]scopestore.Entry, time.Time, error) { var document struct { GeneratedAt time.Time `json:"generated_at"` diff --git a/internal/meshmapper/scopes_test.go b/internal/meshmapper/scopes_test.go index b8e1b24..48104f5 100644 --- a/internal/meshmapper/scopes_test.go +++ b/internal/meshmapper/scopes_test.go @@ -11,6 +11,7 @@ import ( "net/http" "net/http/httptest" "reflect" + "slices" "strings" "sync/atomic" "testing" @@ -29,24 +30,36 @@ func catalogue(names ...string) []byte { return b } +type sourceKey struct{ iata, url string } + type memoryStore struct { - rows map[string]Cache + iatas []string + rows map[sourceKey]Cache fail bool saved []scopestore.Entry } -func (s *memoryStore) GetScopeCatalogue(_ context.Context, iata, url string) (*Cache, error) { - row, ok := s.rows[iata+url] - if !ok { - return nil, nil - } - return &row, nil +func newMemoryStore(iatas ...string) *memoryStore { + return &memoryStore{iatas: iatas, rows: map[sourceKey]Cache{}} } + +func (s *memoryStore) ListKnownIATAs(context.Context) ([]string, error) { + return slices.Clone(s.iatas), nil +} + +func (s *memoryStore) ListScopeCatalogues(context.Context) ([]Catalogue, error) { + var out []Catalogue + for key, c := range s.rows { + out = append(out, Catalogue{IATA: key.iata, URL: key.url, Cache: c}) + } + return out, nil +} + func (s *memoryStore) SaveScopeCatalogue(_ context.Context, iata, url string, next Cache, entries []scopestore.Entry) error { if s.fail { return errors.New("offline") } - old := s.rows[iata+url] + old := s.rows[sourceKey{iata, url}] if next.Payload == nil { next.Payload = old.Payload } @@ -54,11 +67,25 @@ func (s *memoryStore) SaveScopeCatalogue(_ context.Context, iata, url string, ne next.CheckedAt = old.CheckedAt next.ETag = old.ETag } - s.rows[iata+url] = next + s.rows[sourceKey{iata, url}] = next s.saved = append(s.saved, entries...) return nil } +// newImporter restores an importer for YOW whose source points at url. +func newImporter(t *testing.T, cfg config.MeshMapperScopesConfig, store *memoryStore, url string, scopes *scopestore.ScopeStore, manual ...scopestore.Entry) *Importer { + t.Helper() + imp, err := New(context.Background(), cfg, store, NewDirectory(newZoneListMemory()), scopes, manual) + if err != nil { + t.Fatal(err) + } + if len(imp.sources) != 1 { + t.Fatal("expected one source", imp.sources) + } + imp.sources[0].url = url + return imp +} + func names(store *scopestore.ScopeStore) []string { out := []string{} for _, e := range store.Entries() { @@ -84,15 +111,12 @@ func TestCatalogueRefreshFallbackAndRestart(t *testing.T) { } })) defer server.Close() - cfg := config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": server.URL}} - store := &memoryStore{rows: map[string]Cache{}} + cfg := config.MeshMapperScopesConfig{Enabled: true} + store := newMemoryStore("YOW") scopes := scopestore.New() manual := scopestore.FromName("manual") manual.TransportKey = []byte("manual wins") - imp, err := New(ctx, cfg, store, scopes, []scopestore.Entry{manual}) - if err != nil { - t.Fatal(err) - } + imp := newImporter(t, cfg, store, server.URL, scopes, manual) now := time.Now().UTC() check := func() { t.Helper() @@ -111,6 +135,9 @@ func TestCatalogueRefreshFallbackAndRestart(t *testing.T) { if !reflect.DeepEqual(scopes.Entries()[1].IATAs, []string{"YOW"}) { t.Fatal("import not bound to region") } + if got := scopes.NamesForIATAs([]string{"YOW"}); !reflect.DeepEqual(got, []string{"#manual", "#yow"}) { + t.Fatal("catalogue membership must include manually overridden names", got) + } generated := imp.sources[0].generated status = 304 check() @@ -127,14 +154,11 @@ func TestCatalogueRefreshFallbackAndRestart(t *testing.T) { } status, after = 429, "7200" check() - if !imp.retryAfter.Equal(now.Add(2 * time.Hour)) { + if !imp.sources[0].cache.NextAttempt.Equal(now.Add(2 * time.Hour)) { t.Fatal("Retry-After ignored") } restored := scopestore.New() - restart, err := New(ctx, cfg, store, restored, []scopestore.Entry{manual}) - if err != nil { - t.Fatal(err) - } + restart := newImporter(t, cfg, store, server.URL, restored, manual) if !reflect.DeepEqual(names(restored), names(scopes)) { t.Fatal("outage restart lost cached names") } @@ -143,7 +167,7 @@ func TestCatalogueRefreshFallbackAndRestart(t *testing.T) { t.Fatal(err) } if calls != before { - t.Fatal("restart ignored shared rate-limit cooldown") + t.Fatal("restart retried before Retry-After") } status, after, body = 200, "", catalogue() check() @@ -212,11 +236,7 @@ func TestNoRedirectAndResponseBounds(t *testing.T) { } })) defer server.Close() - store := &memoryStore{rows: map[string]Cache{}} - imp, err := New(context.Background(), config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": server.URL}}, store, scopestore.New(), nil) - if err != nil { - t.Fatal(err) - } + imp := newImporter(t, config.MeshMapperScopesConfig{Enabled: true}, newMemoryStore("YOW"), server.URL, scopestore.New()) imp.client.Timeout = 20 * time.Millisecond for _, value := range []string{"redirect", "oversize", "timeout", "304"} { mode.Store(value) @@ -234,22 +254,22 @@ func TestNoRedirectAndResponseBounds(t *testing.T) { func TestImportedSourcesOverlapWithoutGlobalMembership(t *testing.T) { ctx := context.Background() - store := &memoryStore{rows: map[string]Cache{}} + store := newMemoryStore("YOW", "YVR") for _, iata := range []string{"YOW", "YVR"} { - store.rows[iata+iata] = Cache{Payload: []byte(strings.ReplaceAll(string(catalogue("can")), "YOW", iata))} + store.rows[sourceKey{iata, "old-" + iata}] = Cache{Payload: []byte(`{}`)} + store.rows[sourceKey{iata, iata}] = Cache{Payload: []byte(strings.ReplaceAll(string(catalogue("can")), "YOW", iata)), AttemptedAt: time.Now()} } scopes := scopestore.New() - _, err := New(ctx, config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": "YOW", "YVR": "YVR"}}, store, scopes, nil) + _, err := New(ctx, config.MeshMapperScopesConfig{Enabled: true}, store, NewDirectory(newZoneListMemory()), scopes, nil) if err != nil || len(scopes.Entries()) != 1 || !reflect.DeepEqual(scopes.Entries()[0].IATAs, []string{"YOW", "YVR"}) { - t.Fatal(scopes.Entries(), err) + t.Fatal("latest saved catalogue per IATA not restored", scopes.Entries(), err) } - _, err = New(ctx, config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": "changed-source"}}, store, scopes, nil) - if err != nil || len(scopes.Entries()) != 0 { - t.Fatal("removed/changed source remained active", err) + if got := scopes.NamesForIATAs([]string{"YVR"}); !reflect.DeepEqual(got, []string{"#can"}) { + t.Fatal(got) } - // Disabled mode never calls the store, even if URLs remain configured. - _, err = New(ctx, config.MeshMapperScopesConfig{Sources: map[string]string{"YOW": "YOW"}}, nil, scopes, nil) - if err != nil { + // Disabled mode never calls the store. + disabled := scopestore.New() + if _, err = New(ctx, config.MeshMapperScopesConfig{}, nil, NewDirectory(newZoneListMemory()), disabled, nil); err != nil || len(disabled.Entries()) != 0 { t.Fatal(err) } } @@ -262,41 +282,39 @@ func TestInvalidSavedCatalogueDoesNotBlockStartupOrRefresh(t *testing.T) { _, _ = w.Write(catalogue("yow")) })) defer server.Close() - store := &memoryStore{rows: map[string]Cache{"YOW" + server.URL: {Payload: []byte(`{"invalid":true}`), ETag: `"old"`}}} + store := newMemoryStore("YOW") + store.rows[sourceKey{"YOW", server.URL}] = Cache{Payload: []byte(`{"invalid":true}`), ETag: `"old"`} scopes := scopestore.New() - imp, err := New(context.Background(), config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": server.URL}}, store, scopes, []scopestore.Entry{scopestore.FromName("manual")}) - if err != nil || !reflect.DeepEqual(names(scopes), []string{"#manual"}) { - t.Fatal("invalid optional source blocked manual startup", err) + imp := newImporter(t, config.MeshMapperScopesConfig{Enabled: true}, store, server.URL, scopes, scopestore.FromName("manual")) + if !reflect.DeepEqual(names(scopes), []string{"#manual"}) { + t.Fatal("invalid optional source blocked manual startup") } - if err = imp.Refresh(context.Background()); err != nil || !reflect.DeepEqual(names(scopes), []string{"#manual", "#yow"}) { + if err := imp.refresh(context.Background(), &imp.sources[0], time.Now()); err != nil || !reflect.DeepEqual(names(scopes), []string{"#manual", "#yow"}) { t.Fatal("source did not recover on refresh", err, names(scopes)) } ctx, cancel := context.WithCancel(context.Background()) cancel() imp.sources[0].cache.NextAttempt = time.Time{} - if err = imp.Refresh(ctx); err != nil { + if err := imp.Refresh(ctx); err != nil { t.Fatalf("normal shutdown is a task failure: %v", err) } } -func TestRetryAfterShorterThanIntervalIsHonoured(t *testing.T) { - ctx := context.Background() - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.Header().Set("Retry-After", "1800") - w.WriteHeader(http.StatusTooManyRequests) - })) - defer server.Close() - cfg := config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": server.URL}} - imp, err := New(ctx, cfg, &memoryStore{rows: map[string]Cache{}}, scopestore.New(), nil) - if err != nil { - t.Fatal(err) - } - now := time.Now().UTC() - if err := imp.refresh(ctx, &imp.sources[0], now); err != nil { - t.Fatal(err) - } - if got := imp.sources[0].cache.NextAttempt.Sub(now); got != 30*time.Minute { - t.Fatalf("next attempt after 429 in %v, want %v (server Retry-After must not be overridden by the 1h interval)", got, 30*time.Minute) +func TestRetryAfterNeverBeatsTheRateLimit(t *testing.T) { + for retryAfter, want := range map[string]time.Duration{"1800": time.Hour, "7200": 2 * time.Hour} { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Retry-After", retryAfter) + w.WriteHeader(http.StatusTooManyRequests) + })) + imp := newImporter(t, config.MeshMapperScopesConfig{Enabled: true}, newMemoryStore("YOW"), server.URL, scopestore.New()) + now := time.Now().UTC() + if err := imp.refresh(context.Background(), &imp.sources[0], now); err != nil { + t.Fatal(err) + } + server.Close() + if got := imp.sources[0].cache.NextAttempt.Sub(now); got != want { + t.Fatalf("Retry-After %ss: next attempt in %v, want %v", retryAfter, got, want) + } } } @@ -304,16 +322,89 @@ func TestFailedRefreshRetriesSoon(t *testing.T) { ctx := context.Background() server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(500) })) defer server.Close() - cfg := config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": server.URL}} - imp, err := New(ctx, cfg, &memoryStore{rows: map[string]Cache{}}, scopestore.New(), nil) - if err != nil { - t.Fatal(err) - } + imp := newImporter(t, config.MeshMapperScopesConfig{Enabled: true}, newMemoryStore("YOW"), server.URL, scopestore.New()) now := time.Now().UTC() if err := imp.refresh(ctx, &imp.sources[0], now); err != nil { t.Fatal(err) } - if got := imp.sources[0].cache.NextAttempt.Sub(now); got != failureRetry { - t.Fatalf("next attempt after failure in %v, want %v", got, failureRetry) + if got := imp.sources[0].cache.NextAttempt.Sub(now); got != time.Hour { + t.Fatalf("next attempt after failure in %v, want 1h", got) + } +} + +// fakeSites serves a CA zone list and YOW's scope catalogue. +type fakeSites struct { + *httptest.Server + list string + listCalls, scopeHits int +} + +func newFakeSites(t *testing.T) *fakeSites { + f := &fakeSites{} + f.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/get_zones.php": + f.listCalls++ + _, _ = w.Write([]byte(f.list)) + case "/get_scopes.php": + f.scopeHits++ + _, _ = w.Write(catalogue("yow")) + default: + t.Errorf("unexpected path %s", r.URL.Path) + } + })) + f.list = `{"country":"CA","zones":[{"code":"YOW","url":"` + f.URL + `/","has_boundary":true,"group":null}]}` + t.Cleanup(f.Close) + return f +} + +func newDiscoveringImporter(t *testing.T, f *fakeSites, store *memoryStore, scopes *scopestore.ScopeStore) *Importer { + t.Helper() + dir := NewDirectory(newZoneListMemory()) + dir.listURL = f.URL + "/get_zones.php" + imp, err := New(context.Background(), config.MeshMapperScopesConfig{Enabled: true}, store, dir, scopes, nil) + if err != nil { + t.Fatal(err) + } + imp.scopesURL = func(site string) (string, bool) { return site + "get_scopes.php", strings.HasPrefix(site, f.URL) } + return imp +} + +func refreshN(t *testing.T, imp *Importer, n int) { + t.Helper() + for range n { + if err := imp.Refresh(context.Background()); err != nil { + t.Fatal(err) + } + } +} + +func TestScopesDiscoverSourcesFromZoneList(t *testing.T) { + f := newFakeSites(t) + store := newMemoryStore() + scopes := scopestore.New() + imp := newDiscoveringImporter(t, f, store, scopes) + refreshN(t, imp, 2) + if f.listCalls != 0 || f.scopeHits != 0 { + t.Fatal("requested with no IATAs", f.listCalls, f.scopeHits) + } + store.iatas = []string{"YOW", "ZZZ"} + refreshN(t, imp, 3) + if f.listCalls != 1 || f.scopeHits != 1 || !reflect.DeepEqual(names(scopes), []string{"#yow"}) { + t.Fatal("source not discovered", f.listCalls, f.scopeHits, names(scopes)) + } + if _, ok := store.rows[sourceKey{"YOW", f.URL + "/get_scopes.php"}]; !ok { + t.Fatal("catalogue not saved under the discovered URL", store.rows) + } +} + +func TestScopesSkipUnlistedIATAs(t *testing.T) { + f := newFakeSites(t) + f.list = `{"country":"CA","zones":[]}` + imp := newDiscoveringImporter(t, f, newMemoryStore("YOW"), scopestore.New()) + refreshN(t, imp, 3) + s := imp.sources[0] + if f.scopeHits != 0 || s.cache.LastError != "not listed" || s.cache.NextAttempt.Sub(time.Now()) < 59*time.Minute { + t.Fatalf("unlisted IATA requested or retried early: hits=%d %+v", f.scopeHits, s.cache) } } diff --git a/internal/meshmapper/zones.go b/internal/meshmapper/zones.go index 2d86661..1310b71 100644 --- a/internal/meshmapper/zones.go +++ b/internal/meshmapper/zones.go @@ -26,8 +26,12 @@ const ( MaxZoneList = 1 << 20 MaxBoundaryBody = 8 << 20 // outlines are never simplified - zoneListFresh = time.Hour // the API's own cache lifetime - zoneFailureRetry = time.Hour // fair use: no more than one poll an hour + // MeshMapper asks clients to allow 60-120s; a request given up on may still use the day's call. + requestTimeout = 60 * time.Second + refreshTimeout = requestTimeout + 30*time.Second // room for the DB write after a slow request + + zoneListFresh = config.MinZonesRefresh // get_zones.php allows one request per country per 23.5h + zoneFailureRetry = config.MinZonesRefresh // failed requests count too ) var zoneSite = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*\.meshmapper\.net$`) @@ -42,9 +46,15 @@ type Boundary struct { } type ZoneStore interface { + ListKnownIATAs(ctx context.Context) ([]string, error) PruneZoneBoundaries(ctx context.Context, keep []string) ([]string, error) ListZoneBoundaries(ctx context.Context) ([]Boundary, error) SaveZoneBoundary(ctx context.Context, b Boundary) error + ListRegionState(ctx context.Context) ([]RegionState, error) + ListIATADetails(ctx context.Context) ([]IATADetails, error) + UpsertIATADetails(ctx context.Context, iata, name string, lat, lng *float64) error + SaveImportedRegion(ctx context.Context, r RegionState) (bool, error) + PruneImportedRegions(ctx context.Context, keep []string) ([]string, error) } type zoneRegion struct { @@ -52,37 +62,31 @@ type zoneRegion struct { b Boundary } -type zoneEntry struct { - url string - hasBoundary bool -} - -type zoneList struct { - zones map[string]zoneEntry - etag string - fetchedAt, nextAttempt time.Time -} - // Zones is owned by one background task; listeners are wired before Restore. type Zones struct { - store ZoneStore - enabled bool - iatas []string - interval time.Duration - client *http.Client - listURL string - boundsURL func(site string) (string, bool) - regions []*zoneRegion - lists map[string]*zoneList - retryAfter time.Time - onChange func(ctx context.Context, iata string) - onUpdate func(imported map[string]json.RawMessage) + store ZoneStore + enabled bool + seen map[string]bool + interval time.Duration + client *http.Client + dir *Directory + boundsURL func(site string) (string, bool) + regions []*zoneRegion + onChange func(ctx context.Context, iata string) + onUpdate func(imported map[string]json.RawMessage) + + importGroups bool + groupsSynced bool + groupsVersion, groupsKnown int + onRegions func(ctx context.Context) + configured map[string]bool + onIATAs func(ctx context.Context) } -func NewZones(cfg config.MeshMapperZonesConfig, iatas []string, store ZoneStore) *Zones { - return &Zones{store: store, enabled: cfg.Enabled, iatas: iatas, interval: cfg.Interval(), listURL: ZonesURL, - boundsURL: boundaryEndpoint, lists: map[string]*zoneList{}, client: &http.Client{ - Timeout: 30 * time.Second, +func NewZones(cfg config.MeshMapperZonesConfig, store ZoneStore, dir *Directory) *Zones { + return &Zones{store: store, enabled: cfg.Enabled, importGroups: cfg.Enabled && cfg.ImportGroups, seen: map[string]bool{}, interval: cfg.Interval(), dir: dir, + boundsURL: boundaryEndpoint, client: &http.Client{ + Timeout: requestTimeout, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, }} } @@ -93,12 +97,22 @@ func (z *Zones) OnChange(fn func(ctx context.Context, iata string)) { z.onChange // OnUpdate receives every imported boundary whenever the set changes. func (z *Zones) OnUpdate(fn func(imported map[string]json.RawMessage)) { z.onUpdate = fn } -// Restore prunes imports for unconfigured IATAs (all of them when disabled) and -// loads saved boundaries, without making HTTP requests. +// Restore prunes imports for unknown IATAs (all of them when disabled), and +// imported regions when group import is off, then loads saved boundaries, +// without making HTTP requests. func (z *Zones) Restore(ctx context.Context) error { + if !z.importGroups { + if err := z.pruneGroups(ctx); err != nil { + return err + } + } + iatas, err := z.store.ListKnownIATAs(ctx) + if err != nil { + return fmt.Errorf("list IATAs for MeshMapper boundaries: %w", err) + } keep := []string{} if z.enabled { - keep = z.iatas + keep = iatas } pruned, err := z.store.PruneZoneBoundaries(ctx, keep) if err != nil { @@ -119,24 +133,32 @@ func (z *Zones) Restore(ctx context.Context) error { for _, b := range saved { byIATA[b.IATA] = b } - for _, iata := range z.iatas { + z.track(iatas, byIATA) + z.publish() + return nil +} + +// track adds a region for each IATA not seen before. +func (z *Zones) track(iatas []string, saved map[string]Boundary) { + for _, iata := range iatas { + if z.seen[iata] { + continue + } + z.seen[iata] = true country := iatadb.CountryFor(iata) if country == "" { slog.Warn("MeshMapper boundary skipped: IATA has no known country", "component", "meshmapper.zones", "iata", iata) continue } - b, ok := byIATA[iata] + b, ok := saved[iata] if !ok { b = Boundary{IATA: iata} } - if b.LastError == "HTTP 429" && b.NextAttempt.After(z.retryAfter) { - z.retryAfter = b.NextAttempt - } z.regions = append(z.regions, &zoneRegion{country: country, b: b}) - z.log(b, "restored") + if ok { + z.log(b, "restored") + } } - z.publish() - return nil } // Refresh makes at most one request. The scheduler serializes calls every 15s. @@ -147,77 +169,38 @@ func (z *Zones) Refresh(ctx context.Context) (err error) { err = nil } }() - ctx, cancel := context.WithTimeout(ctx, time.Minute) + ctx, cancel := context.WithTimeout(ctx, refreshTimeout) defer cancel() now := time.Now().UTC() - if now.Before(z.retryAfter) { - return nil + // IATAs are created from traffic, so pick up new ones every tick. + iatas, err := z.store.ListKnownIATAs(ctx) + if err != nil { + return fmt.Errorf("list IATAs for MeshMapper boundaries: %w", err) + } + z.track(iatas, nil) + if err := z.syncDirectory(ctx, iatas); err != nil { + return err } for _, r := range z.regions { if now.Before(r.b.NextAttempt) { continue } - list := z.lists[r.country] - if list == nil || now.Sub(list.fetchedAt) >= zoneListFresh { - if list != nil && now.Before(list.nextAttempt) { - continue - } - return z.fetchList(ctx, r.country, now) + zones, fetched, err := z.dir.List(ctx, r.country, now) + if err != nil || fetched { + return err } - return z.refresh(ctx, r, list, now) + if zones == nil { + continue + } + return z.refresh(ctx, r, zones, now) } return nil } -func (z *Zones) fetchList(ctx context.Context, country string, now time.Time) error { - list := z.lists[country] - if list == nil { - list = &zoneList{} - z.lists[country] = list - } - endpoint := z.listURL + "?country=" + url.QueryEscape(country) - etag := "" - if list.zones != nil { - etag = list.etag - } - status, body, header, err := z.get(ctx, endpoint, etag, MaxZoneList) - if err != nil { - return err - } - problem := "" - switch status { - case http.StatusOK: - zones, decodeErr := decodeZones(body, country) - if decodeErr != nil { - problem = "invalid response" - } else { - list.zones, list.etag = zones, header.Get("ETag") - } - case http.StatusNotModified: - if list.zones == nil { - problem = "304 without cached list" - } - default: - problem = z.statusProblem(status, header, now, &list.nextAttempt) - } - if problem == "" { - list.fetchedAt = now - } else if list.nextAttempt.Before(now.Add(zoneFailureRetry)) { - list.nextAttempt = now.Add(zoneFailureRetry) - } - level := slog.LevelInfo - if problem != "" { - level = slog.LevelWarn - } - slog.Log(ctx, level, "MeshMapper zone list checked", "component", "meshmapper.zones", "country", country, - "zones", len(list.zones), "last_error", problem) - return nil -} - -func (z *Zones) refresh(ctx context.Context, r *zoneRegion, list *zoneList, now time.Time) error { +func (z *Zones) refresh(ctx context.Context, r *zoneRegion, zones map[string]zoneEntry, now time.Time) error { update := Boundary{IATA: r.b.IATA, URL: r.b.URL, AttemptedAt: now, NextAttempt: now.Add(z.interval)} var retryAt time.Time - entry, listed := list.zones[r.b.IATA] + entry, listed := zones[r.b.IATA] endpoint, valid := "", false if listed { update.URL = entry.url @@ -235,7 +218,7 @@ func (z *Zones) refresh(ctx context.Context, r *zoneRegion, list *zoneList, now if r.b.Feature != nil { etag = r.b.ETag } - status, body, header, err := z.get(ctx, endpoint, etag, MaxBoundaryBody) + status, body, header, err := get(ctx, z.client, "Beacon-MeshMapper-Zones/1", endpoint, etag, MaxBoundaryBody) if err != nil { return err } @@ -255,7 +238,7 @@ func (z *Zones) refresh(ctx context.Context, r *zoneRegion, list *zoneList, now update.LastError = "304 without cached boundary" } default: - update.LastError = z.statusProblem(status, header, now, &retryAt) + update.LastError = statusProblem(status, header, now, &retryAt) } if update.LastError == "" { etag := header.Get("ETag") @@ -296,17 +279,17 @@ func (z *Zones) refresh(ctx context.Context, r *zoneRegion, list *zoneList, now } // get returns transport errors only when ctx ended; other failures become status 0. -func (z *Zones) get(ctx context.Context, endpoint, etag string, limit int64) (int, []byte, http.Header, error) { +func get(ctx context.Context, client *http.Client, agent, endpoint, etag string, limit int64) (int, []byte, http.Header, error) { request, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) if err != nil { return 0, nil, nil, err } request.Header.Set("Accept", "application/json") - request.Header.Set("User-Agent", "Beacon-MeshMapper-Zones/1") + request.Header.Set("User-Agent", agent) if etag != "" { request.Header.Set("If-None-Match", etag) } - response, err := z.client.Do(request) + response, err := client.Do(request) if err != nil { if ctx.Err() != nil { return 0, nil, nil, ctx.Err() @@ -324,8 +307,8 @@ func (z *Zones) get(ctx context.Context, endpoint, etag string, limit int64) (in return response.StatusCode, body, response.Header, nil } -// statusProblem describes a failed response and honours Retry-After on 429/503. -func (z *Zones) statusProblem(status int, header http.Header, now time.Time, retryAt *time.Time) string { +// statusProblem describes a failed response and reads Retry-After on 429/503. +func statusProblem(status int, header http.Header, now time.Time, retryAt *time.Time) string { if status == 0 { return "request failed" // don't persist untrusted error text } @@ -333,13 +316,6 @@ func (z *Zones) statusProblem(status int, header http.Header, now time.Time, ret if seconds, err := strconv.ParseInt(header.Get("Retry-After"), 10, 64); err == nil && seconds > 0 && seconds <= int64((1<<63-1)/time.Second) { *retryAt = now.Add(time.Duration(seconds) * time.Second) } - // The rate limit is shared by both endpoints, so pause every request. - if status == http.StatusTooManyRequests { - z.retryAfter = *retryAt - if retryAt.IsZero() { - z.retryAfter = now.Add(time.Minute) - } - } } return fmt.Sprintf("HTTP %d", status) } @@ -372,36 +348,16 @@ func (z *Zones) log(b Boundary, action string) { "source", b.URL, "imported", b.Feature != nil, "checked_at", b.CheckedAt, "next_attempt", b.NextAttempt, "last_error", b.LastError) } -// boundaryEndpoint accepts only a published region site root. -func boundaryEndpoint(site string) (string, bool) { +func boundaryEndpoint(site string) (string, bool) { return siteEndpoint(site, "get_geojson.php") } + +// siteEndpoint accepts only a published region site root. +func siteEndpoint(site, file string) (string, bool) { u, err := url.Parse(site) if err != nil || u.Scheme != "https" || u.User != nil || !zoneSite.MatchString(u.Host) || (u.Path != "" && u.Path != "/") || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" { return "", false } - return "https://" + u.Host + "/get_geojson.php", true -} - -func decodeZones(body []byte, country string) (map[string]zoneEntry, error) { - var document struct { - Country string `json:"country"` - Zones *[]struct { - Code string `json:"code"` - URL string `json:"url"` - HasBoundary bool `json:"has_boundary"` - } `json:"zones"` - } - if err := json.Unmarshal(body, &document); err != nil { - return nil, err - } - if !strings.EqualFold(document.Country, country) || document.Zones == nil { - return nil, fmt.Errorf("invalid zone list") - } - zones := make(map[string]zoneEntry, len(*document.Zones)) - for _, zone := range *document.Zones { - zones[strings.ToUpper(zone.Code)] = zoneEntry{url: zone.URL, hasBoundary: zone.HasBoundary} - } - return zones, nil + return "https://" + u.Host + "/" + file, true } // decodeBoundary returns nil when MeshMapper has no usable outline for the region. diff --git a/internal/meshmapper/zones_test.go b/internal/meshmapper/zones_test.go index af62cc8..bdf4589 100644 --- a/internal/meshmapper/zones_test.go +++ b/internal/meshmapper/zones_test.go @@ -15,6 +15,8 @@ import ( "time" "github.com/MeshCore-Beacon/beacon-server/internal/config" + "github.com/MeshCore-Beacon/beacon-server/internal/keystore" + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" ) const square = `{"type":"Polygon","coordinates":[[[-76,45],[-75,45],[-75,46],[-76,46],[-76,45]]]}` @@ -25,9 +27,73 @@ func boundaryBody(code, geometry string) string { } type zoneMemoryStore struct { - rows map[string]Boundary - pruned []string - fail bool + iatas []string + rows map[string]Boundary + pruned []string + fail bool + regions map[string]RegionState + lists *zoneListMemory + details map[string]IATADetails + writes int +} + +func (s *zoneMemoryStore) ListIATADetails(context.Context) ([]IATADetails, error) { + var out []IATADetails + for _, iata := range s.iatas { + d := s.details[iata] + d.IATA = iata + out = append(out, d) + } + return out, nil +} + +func (s *zoneMemoryStore) UpsertIATADetails(_ context.Context, iata, name string, lat, lng *float64) error { + if s.details == nil { + s.details = map[string]IATADetails{} + } + s.details[iata] = IATADetails{IATA: iata, Name: name, Lat: lat, Lng: lng} + s.writes++ + return nil +} + +func (s *zoneMemoryStore) ListRegionState(context.Context) ([]RegionState, error) { + var out []RegionState + for _, r := range s.regions { + out = append(out, r) + } + return out, nil +} + +func (s *zoneMemoryStore) SaveImportedRegion(_ context.Context, r RegionState) (bool, error) { + if s.regions == nil { + s.regions = map[string]RegionState{} + } + if cur, ok := s.regions[r.Slug]; ok && !cur.Imported { + return false, nil + } + r.Imported = true + s.regions[r.Slug] = r + for _, m := range r.IATAs { // members become known IATAs, as AddIATAs does + if !slices.Contains(s.iatas, m) { + s.iatas = append(s.iatas, m) + } + } + return true, nil +} + +func (s *zoneMemoryStore) PruneImportedRegions(_ context.Context, keep []string) ([]string, error) { + var removed []string + for slug, r := range s.regions { + if r.Imported && !slices.Contains(keep, slug) { + delete(s.regions, slug) + removed = append(removed, slug) + } + } + return removed, nil +} + +func (s *zoneMemoryStore) ListKnownIATAs(context.Context) ([]string, error) { + return slices.Clone(s.iatas), nil } func (s *zoneMemoryStore) PruneZoneBoundaries(_ context.Context, keep []string) ([]string, error) { @@ -65,6 +131,30 @@ func (s *zoneMemoryStore) SaveZoneBoundary(_ context.Context, b Boundary) error return nil } +type zoneListMemory struct{ rows map[string]ZoneList } + +func newZoneListMemory() *zoneListMemory { return &zoneListMemory{rows: map[string]ZoneList{}} } + +func (s *zoneListMemory) ListZoneLists(context.Context) ([]ZoneList, error) { + var out []ZoneList + for _, l := range s.rows { + out = append(out, l) + } + return out, nil +} + +func (s *zoneListMemory) SaveZoneList(_ context.Context, l ZoneList) error { + old := s.rows[l.Country] + if l.Payload == nil { + l.Payload = old.Payload + } + if l.FetchedAt.IsZero() { + l.FetchedAt, l.ETag = old.FetchedAt, old.ETag + } + s.rows[l.Country] = l + return nil +} + type fakeMeshMapper struct { *httptest.Server list, boundary string @@ -113,8 +203,18 @@ type zoneHarness struct { func newZoneHarness(t *testing.T, f *fakeMeshMapper, store *zoneMemoryStore, enabled bool) *zoneHarness { t.Helper() h := &zoneHarness{store: store} - h.z = NewZones(config.MeshMapperZonesConfig{Enabled: enabled}, []string{"YOW"}, store) - h.z.listURL = f.URL + "/get_zones.php" + if store.iatas == nil { + store.iatas = []string{"YOW"} + } + if store.lists == nil { + store.lists = newZoneListMemory() + } + dir := NewDirectory(store.lists) + dir.listURL = f.URL + "/get_zones.php" + if err := dir.Restore(context.Background()); err != nil { + t.Fatal(err) + } + h.z = NewZones(config.MeshMapperZonesConfig{Enabled: enabled}, store, dir) h.z.boundsURL = func(site string) (string, bool) { return site + "get_geojson.php", strings.HasPrefix(site, f.URL) } h.z.OnChange(func(_ context.Context, iata string) { h.changed = append(h.changed, iata) }) h.z.OnUpdate(func(imported map[string]json.RawMessage) { h.imported = imported }) @@ -175,11 +275,11 @@ func TestZonesKeepLastGoodBoundary(t *testing.T) { wantRetry time.Duration }{ {"null geometry", boundaryBody("YOW", "null"), 200, "no boundary", 24 * time.Hour}, - {"not found", "", 404, "HTTP 404", time.Hour}, - {"unavailable", "", 503, "HTTP 503", time.Hour}, - {"truncated", boundaryBody("YOW", square)[:80], 200, "invalid response", time.Hour}, - {"other region", boundaryBody("YVR", square), 200, "invalid response", time.Hour}, - {"oversized", `{"type":"FeatureCollection","features":[],"pad":"` + strings.Repeat("x", MaxBoundaryBody) + `"}`, 200, "invalid response", time.Hour}, + {"not found", "", 404, "HTTP 404", 24 * time.Hour}, + {"unavailable", "", 503, "HTTP 503", 24 * time.Hour}, + {"truncated", boundaryBody("YOW", square)[:80], 200, "invalid response", 24 * time.Hour}, + {"other region", boundaryBody("YVR", square), 200, "invalid response", 24 * time.Hour}, + {"oversized", `{"type":"FeatureCollection","features":[],"pad":"` + strings.Repeat("x", MaxBoundaryBody) + `"}`, 200, "invalid response", 24 * time.Hour}, } { t.Run(tc.name, func(t *testing.T) { f := newFakeMeshMapper(t) @@ -231,29 +331,44 @@ func TestZonesListFailureBacksOff(t *testing.T) { t.Fatal("failed list retried early", f.listCalls, f.calls) } f.listStatus, f.list = 200, `{"country":"US","zones":[]}` - h.z.lists["CA"].nextAttempt, h.z.lists["CA"].fetchedAt = time.Time{}, time.Time{} + h.z.dir.lists["CA"].nextAttempt, h.z.dir.lists["CA"].fetchedAt = time.Time{}, time.Time{} _ = h.z.Refresh(context.Background()) - if h.z.lists["CA"].zones != nil { + if h.z.dir.lists["CA"].zones != nil { t.Fatal("list for another country accepted") } } -func TestZonesRateLimitPausesEveryRequest(t *testing.T) { +func TestZonesRateLimitWaitsOnlyForThatRegion(t *testing.T) { + for retryAfter, want := range map[string]time.Duration{"120": 24 * time.Hour, "172800": 48 * time.Hour} { + f := newFakeMeshMapper(t) + h := newZoneHarness(t, f, &zoneMemoryStore{rows: map[string]Boundary{}}, true) + f.status, f.retryAfter = 429, retryAfter + b := h.tick(t) + if b.LastError != "HTTP 429" || b.NextAttempt.Sub(b.AttemptedAt) != want { + t.Fatalf("Retry-After %ss: next attempt in %v, want %v", retryAfter, b.NextAttempt.Sub(b.AttemptedAt), want) + } + restarted := newZoneHarness(t, f, h.store, true) + if err := restarted.z.Refresh(context.Background()); err != nil || f.calls != 1 { + t.Fatal("requested again before the rate limit allows", err, f.calls) + } + } +} + +func TestZoneListSurvivesRestart(t *testing.T) { f := newFakeMeshMapper(t) - h := newZoneHarness(t, f, &zoneMemoryStore{rows: map[string]Boundary{}}, true) - f.status, f.retryAfter = 429, "120" - b := h.tick(t) - if b.LastError != "HTTP 429" || b.NextAttempt.Sub(b.AttemptedAt) != time.Hour || h.z.retryAfter.Sub(b.AttemptedAt) != 2*time.Minute { - t.Fatalf("429 not honoured: %+v retryAfter=%v", b, h.z.retryAfter) + store := &zoneMemoryStore{rows: map[string]Boundary{}} + h := newZoneHarness(t, f, store, true) + h.tick(t) + if f.listCalls != 1 { + t.Fatal("list not fetched", f.listCalls) } - h.z.regions[0].b.NextAttempt = time.Time{} - _ = h.z.Refresh(context.Background()) - if f.calls != 1 { - t.Fatal("requested during Retry-After") + if got := h.z.dir.lists["CA"].nextAttempt.Sub(h.z.dir.lists["CA"].fetchedAt); got != 24*time.Hour { + t.Fatal("zone list refetched sooner than the rate limit allows", got) } - restarted := newZoneHarness(t, f, h.store, true) - if !restarted.z.retryAfter.Equal(b.NextAttempt) { - t.Fatal("persisted 429 not restored") + restarted := newZoneHarness(t, f, store, true) + restarted.tick(t) + if f.listCalls != 1 || f.calls != 2 { + t.Fatal("restart refetched the zone list", f.listCalls, f.calls) } } @@ -266,7 +381,7 @@ func TestZonesRestoreAndPrune(t *testing.T) { restarted := newZoneHarness(t, f, store, true) if !slices.Equal(store.pruned, []string{"OLD"}) || !slices.Equal(restarted.changed, []string{"OLD"}) { - t.Fatal("unconfigured IATA not pruned", store.pruned, restarted.changed) + t.Fatal("unknown IATA not pruned", store.pruned, restarted.changed) } if restarted.imported["YOW"] == nil || f.calls != 1 { t.Fatal("saved boundary not restored without a request") @@ -281,6 +396,29 @@ func TestZonesRestoreAndPrune(t *testing.T) { } } +func TestZonesPickUpNewIATAs(t *testing.T) { + f := newFakeMeshMapper(t) + store := &zoneMemoryStore{iatas: []string{}, rows: map[string]Boundary{}} + h := newZoneHarness(t, f, store, true) + for range 2 { + if err := h.z.Refresh(context.Background()); err != nil { + t.Fatal(err) + } + } + if f.listCalls != 0 || f.calls != 0 { + t.Fatal("requested with no IATAs", f.listCalls, f.calls) + } + store.iatas = []string{"YOW", "ZZZ"} + for range 2 { + if err := h.z.Refresh(context.Background()); err != nil { + t.Fatal(err) + } + } + if len(h.z.regions) != 1 || h.imported["YOW"] == nil || !slices.Equal(h.changed, []string{"YOW"}) { + t.Fatal("new IATA not imported", len(h.z.regions), h.changed) + } +} + func TestZonesPersistFailureWaitsForNextAttempt(t *testing.T) { f := newFakeMeshMapper(t) h := newZoneHarness(t, f, &zoneMemoryStore{rows: map[string]Boundary{}, fail: true}, true) @@ -329,3 +467,28 @@ func TestDecodeBoundaryRejectsUnusableOutlines(t *testing.T) { t.Fatal("case-insensitive code rejected", err) } } + +func TestClientsAllowMeshMapperTimeout(t *testing.T) { + ctx := context.Background() + scopes, err := New(ctx, config.MeshMapperScopesConfig{}, nil, NewDirectory(nil), scopestore.New(), nil) + if err != nil { + t.Fatal(err) + } + channels, err := NewChannels(ctx, config.MeshMapperChannelsConfig{}, newChannelMemoryStore(), NewDirectory(nil), keystore.NewMapKeyStore(nil)) + if err != nil { + t.Fatal(err) + } + for name, c := range map[string]*http.Client{ + "directory": NewDirectory(nil).client, + "zones": NewZones(config.MeshMapperZonesConfig{}, nil, nil).client, + "scopes": scopes.client, + "channels": channels.client, + } { + if c.Timeout < 60*time.Second { + t.Errorf("%s client gives up after %v; MeshMapper asks for 60-120s", name, c.Timeout) + } + } + if refreshTimeout <= requestTimeout { + t.Fatal("a refresh must outlast its request") + } +} diff --git a/internal/scopestore/scopestore.go b/internal/scopestore/scopestore.go index da5c80f..55d6cd0 100644 --- a/internal/scopestore/scopestore.go +++ b/internal/scopestore/scopestore.go @@ -7,6 +7,7 @@ package scopestore import ( "crypto/sha256" + "slices" "strings" "sync" ) @@ -34,6 +35,8 @@ func FromName(name string) Entry { type ScopeStore struct { mu sync.RWMutex entries []Entry + // IATA -> scope names, for region-filtered listings only; matching uses entries. + manual, catalogue map[string][]string } // New creates an empty ScopeStore. @@ -54,3 +57,30 @@ func (s *ScopeStore) Entries() []Entry { defer s.mu.RUnlock() return s.entries } + +// SetManualMembers publishes configured region membership of manual scopes. +func (s *ScopeStore) SetManualMembers(byIATA map[string][]string) { + s.mu.Lock() + defer s.mu.Unlock() + s.manual = byIATA +} + +// SetCatalogueMembers publishes imported catalogue membership, including names a manual scope overrides. +func (s *ScopeStore) SetCatalogueMembers(byIATA map[string][]string) { + s.mu.Lock() + defer s.mu.Unlock() + s.catalogue = byIATA +} + +// NamesForIATAs returns the sorted names of scopes that belong to any of the given IATAs. +func (s *ScopeStore) NamesForIATAs(iatas []string) []string { + s.mu.RLock() + defer s.mu.RUnlock() + names := []string{} + for _, iata := range iatas { + names = append(names, s.manual[iata]...) + names = append(names, s.catalogue[iata]...) + } + slices.Sort(names) + return slices.Compact(names) +} diff --git a/internal/scopestore/scopestore_test.go b/internal/scopestore/scopestore_test.go index 3e2f359..08eef4c 100644 --- a/internal/scopestore/scopestore_test.go +++ b/internal/scopestore/scopestore_test.go @@ -4,6 +4,7 @@ package scopestore import ( + "slices" "testing" ) @@ -54,3 +55,21 @@ func TestEntriesRetainsImmutableSnapshotWithoutAllocation(t *testing.T) { t.Fatal("replacement mutated the previous reader snapshot") } } + +func TestNamesForIATAs(t *testing.T) { + s := New() + if got := s.NamesForIATAs([]string{"YOW"}); got == nil || len(got) != 0 { + t.Fatalf("empty store = %#v, want empty non-nil slice", got) + } + s.SetManualMembers(map[string][]string{"YOW": {"#ottawa", "#on"}, "YVR": {"#bc"}}) + s.SetCatalogueMembers(map[string][]string{"YOW": {"#on", "#gatineau"}, "YUL": {"#qc"}}) + got := s.NamesForIATAs([]string{"YOW", "YUL", "YYZ"}) + want := []string{"#gatineau", "#on", "#ottawa", "#qc"} + if !slices.Equal(got, want) { + t.Fatalf("NamesForIATAs = %v, want %v", got, want) + } + s.SetCatalogueMembers(nil) + if got := s.NamesForIATAs([]string{"YUL"}); len(got) != 0 { + t.Fatalf("cleared catalogue still lists %v", got) + } +}