fix(meshmapper): respect MeshMapper's per-endpoint rate limits

db/sqlc/mock/querier.go

@@ -1357,6 +1357,21 @@ func (mr *MockQuerierMockRecorder) ListZoneBoundaries(ctx any) *gomock.Call {
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListZoneBoundaries", reflect.TypeOf((*MockQuerier)(nil).ListZoneBoundaries), ctx)
 }
 
+// ListZoneLists mocks base method.
+func (m *MockQuerier) ListZoneLists(ctx context.Context) ([]db.MeshmapperZoneList, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "ListZoneLists", ctx)
+	ret0, _ := ret[0].([]db.MeshmapperZoneList)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// ListZoneLists indicates an expected call of ListZoneLists.
+func (mr *MockQuerierMockRecorder) ListZoneLists(ctx any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListZoneLists", reflect.TypeOf((*MockQuerier)(nil).ListZoneLists), ctx)
+}
+
 // PruneImportedRegions mocks base method.
 func (m *MockQuerier) PruneImportedRegions(ctx context.Context, keep []string) ([]string, error) {
 	m.ctrl.T.Helper()
@@ -1674,6 +1689,20 @@ func (mr *MockQuerierMockRecorder) SaveZoneBoundary(ctx, arg any) *gomock.Call {
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveZoneBoundary", reflect.TypeOf((*MockQuerier)(nil).SaveZoneBoundary), ctx, arg)
 }
 
+// SaveZoneList mocks base method.
+func (m *MockQuerier) SaveZoneList(ctx context.Context, arg db.SaveZoneListParams) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "SaveZoneList", ctx, arg)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// SaveZoneList indicates an expected call of SaveZoneList.
+func (mr *MockQuerierMockRecorder) SaveZoneList(ctx, arg any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveZoneList", reflect.TypeOf((*MockQuerier)(nil).SaveZoneList), ctx, arg)
+}
+
 // SearchKnownRoutes mocks base method.
 func (m *MockQuerier) SearchKnownRoutes(ctx context.Context, arg db.SearchKnownRoutesParams) ([]db.SearchKnownRoutesRow, error) {
 	m.ctrl.T.Helper()

db/sqlc/models.go

@@ -256,6 +256,16 @@ type MeshmapperZoneBoundary struct {
 	LastError   string             `json:"last_error"`
 }
 
+type MeshmapperZoneList struct {
+	Country     string             `json:"country"`
+	Payload     []byte             `json:"payload"`
+	Etag        *string            `json:"etag"`
+	FetchedAt   pgtype.Timestamptz `json:"fetched_at"`
+	AttemptedAt pgtype.Timestamptz `json:"attempted_at"`
+	NextAttempt pgtype.Timestamptz `json:"next_attempt"`
+	LastError   string             `json:"last_error"`
+}
+
 type MvHourlyIataStat struct {
 	Iata             string             `json:"iata"`
 	Hour             pgtype.Timestamptz `json:"hour"`

db/sqlc/querier.go

@@ -225,6 +225,7 @@ type Querier interface {
 	// internal/ingest.BackfillChannelMessages.
 	ListUndecryptedGroupTextPackets(ctx context.Context) ([]ListUndecryptedGroupTextPacketsRow, error)
 	ListZoneBoundaries(ctx context.Context) ([]MeshmapperZoneBoundary, error)
+	ListZoneLists(ctx context.Context) ([]MeshmapperZoneList, error)
 	PruneImportedRegions(ctx context.Context, keep []string) ([]string, error)
 	// Drops imports for IATAs no longer configured, so their manual border returns.
 	PruneZoneBoundaries(ctx context.Context, keep []string) ([]string, error)
@@ -270,6 +271,8 @@ type Querier interface {
 	SaveScopeCatalogue(ctx context.Context, arg SaveScopeCatalogueParams) error
 	// NULL feature/etag/checked_at retain the last good boundary after an error or 304.
 	SaveZoneBoundary(ctx context.Context, arg SaveZoneBoundaryParams) error
+	// NULL payload/etag/fetched_at retain the last good list after an error or 304.
+	SaveZoneList(ctx context.Context, arg SaveZoneListParams) error
 	// Returns known routes containing a subsequence from source to destination hash prefix.
 	// Verifies source appears before destination in the route.
 	SearchKnownRoutes(ctx context.Context, arg SearchKnownRoutesParams) ([]SearchKnownRoutesRow, error)

db/sqlc/queries.sql.go

@@ -4000,6 +4000,38 @@ func (q *Queries) ListZoneBoundaries(ctx context.Context) ([]MeshmapperZoneBound
 	return items, nil
 }
 
+const listZoneLists = `-- name: ListZoneLists :many
+SELECT country, payload, etag, fetched_at, attempted_at, next_attempt, last_error FROM meshmapper_zone_lists ORDER BY country
+`
+
+func (q *Queries) ListZoneLists(ctx context.Context) ([]MeshmapperZoneList, error) {
+	rows, err := q.db.Query(ctx, listZoneLists)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	items := []MeshmapperZoneList{}
+	for rows.Next() {
+		var i MeshmapperZoneList
+		if err := rows.Scan(
+			&i.Country,
+			&i.Payload,
+			&i.Etag,
+			&i.FetchedAt,
+			&i.AttemptedAt,
+			&i.NextAttempt,
+			&i.LastError,
+		); err != nil {
+			return nil, err
+		}
+		items = append(items, i)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}
+
 const pruneImportedRegions = `-- name: PruneImportedRegions :many
 DELETE FROM regions WHERE imported AND NOT (slug = ANY($1::text[])) RETURNING slug
 `
@@ -4636,6 +4668,43 @@ func (q *Queries) SaveZoneBoundary(ctx context.Context, arg SaveZoneBoundaryPara
 	return err
 }
 
+const saveZoneList = `-- name: SaveZoneList :exec
+INSERT INTO meshmapper_zone_lists (country, payload, etag, fetched_at, attempted_at, next_attempt, last_error)
+VALUES ($1, $2::jsonb, $3::text,
+    $4::timestamptz, $5, $6, $7)
+ON CONFLICT (country) DO UPDATE SET
+    payload = COALESCE(EXCLUDED.payload, meshmapper_zone_lists.payload),
+    etag = COALESCE(EXCLUDED.etag, meshmapper_zone_lists.etag),
+    fetched_at = COALESCE(EXCLUDED.fetched_at, meshmapper_zone_lists.fetched_at),
+    attempted_at = EXCLUDED.attempted_at,
+    next_attempt = EXCLUDED.next_attempt,
+    last_error = EXCLUDED.last_error
+`
+
+type SaveZoneListParams struct {
+	Country     string             `json:"country"`
+	Payload     []byte             `json:"payload"`
+	Etag        *string            `json:"etag"`
+	FetchedAt   pgtype.Timestamptz `json:"fetched_at"`
+	AttemptedAt pgtype.Timestamptz `json:"attempted_at"`
+	NextAttempt pgtype.Timestamptz `json:"next_attempt"`
+	LastError   string             `json:"last_error"`
+}
+
+// NULL payload/etag/fetched_at retain the last good list after an error or 304.
+func (q *Queries) SaveZoneList(ctx context.Context, arg SaveZoneListParams) error {
+	_, err := q.db.Exec(ctx, saveZoneList,
+		arg.Country,
+		arg.Payload,
+		arg.Etag,
+		arg.FetchedAt,
+		arg.AttemptedAt,
+		arg.NextAttempt,
+		arg.LastError,
+	)
+	return err
+}
+
 const searchKnownRoutes = `-- name: SearchKnownRoutes :many
 SELECT id, node_ids, hash_prefix, iata, hop_count, first_seen, last_seen, observation_count
 FROM known_routes