diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dee0b98..6e9c7ab 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,7 +58,7 @@ jobs: - name: Verify stats and endpoint queries against PostgreSQL 16 env: BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable - run: go test ./db -run '^Test(Signal|Paths|PacketSummaries|PacketEndpointsResolveLive|ObserverMetrics|RouteEvidence|RouteEvidenceIndex|AnalyticsRetention|AnalyticsRetentionConcurrent|DeleteOldPacketsBatches|MeshMapperCatalogue|ChannelMessageScopes|NodeLocationReset|RunMigrations[A-Za-z]*|Reconfirm[A-Za-z]*|BigintCursors|RouteSearchHopOrder|StatsNodeTypesSplit|ListKnownRoutes|ListKnownRoutesMatchesLegacy|KnownRoutesCursorTies|KnownRoutesUnanalyzedRegion|KnownRoutesHopStatementsStayCustom|KnownRoutePaginationTies|RouteRetentionCutoffsExact|DeleteExpiredRoutes|DeleteGraceRoutesSlice|GraceWatermarkHeldByLockedRow)Postgres$' -count=1 -v + run: go test ./db -run '^Test(Signal|Paths|PacketSummaries|PacketEndpointsResolveLive|ObserverMetrics|RouteEvidence|RouteEvidenceIndex|AnalyticsRetention|AnalyticsRetentionConcurrent|DeleteOldPacketsBatches|MeshMapperCatalogue|ChannelMessageScopes|NodeLocationReset|RunMigrations[A-Za-z]*|KnownRoutesIndexSet|Reconfirm[A-Za-z]*|BigintCursors|RouteSearchHopOrder|StatsNodeTypesSplit|ListKnownRoutes|ListKnownRoutesMatchesLegacy|KnownRoutesCursorTies|KnownRoutesUnanalyzedRegion|KnownRoutesHopStatementsStayCustom|KnownRoutePaginationTies|RouteRetentionCutoffsExact|DeleteExpiredRoutes|DeleteGraceRoutesSlice|GraceWatermarkHeldByLockedRow)Postgres$' -count=1 -v - name: Verify route retention task against PostgreSQL 16 env: diff --git a/db/migrate_integration_test.go b/db/migrate_integration_test.go index 58054ee..a7fdd89 100644 --- a/db/migrate_integration_test.go +++ b/db/migrate_integration_test.go @@ -180,3 +180,21 @@ func TestRunMigrationsHopIndexPrebuiltPostgres(t *testing.T) { t.Fatalf("006 not recorded: %v", err) } } + +// The migrations leave known_routes with exactly the indexes some statement reads, all usable. +func TestKnownRoutesIndexSetPostgres(t *testing.T) { + ctx, pool := schemaPool(t) + if err := RunMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + var got, bad []string + if err := pool.QueryRow(ctx, `SELECT coalesce(array_agg(indexrelid::regclass::text ORDER BY indexrelid::regclass::text), '{}'), + coalesce(array_agg(indexrelid::regclass::text) FILTER (WHERE NOT (indisvalid AND indisready)), '{}') +FROM pg_index WHERE indrelid = 'known_routes'::regclass`).Scan(&got, &bad); err != nil { + t.Fatal(err) + } + want := []string{"idx_known_routes_hop_iata", "idx_known_routes_iata_keyset", "idx_known_routes_keyset", "idx_known_routes_node_ids", "known_routes_pkey"} + if strings.Join(got, ",") != strings.Join(want, ",") || len(bad) != 0 { + t.Fatalf("known_routes indexes = %v, want %v; not valid and ready: %v", got, want, bad) + } +} diff --git a/db/migrations/007_drop_known_routes_last_seen.sql b/db/migrations/007_drop_known_routes_last_seen.sql new file mode 100644 index 0000000..d06a0a7 --- /dev/null +++ b/db/migrations/007_drop_known_routes_last_seen.sql @@ -0,0 +1,5 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- No statement reads (last_seen DESC); retention walks idx_known_routes_keyset. +DROP INDEX CONCURRENTLY IF EXISTS idx_known_routes_last_seen; diff --git a/db/migrations/008_drop_known_routes_hop_count.sql b/db/migrations/008_drop_known_routes_hop_count.sql new file mode 100644 index 0000000..818db36 --- /dev/null +++ b/db/migrations/008_drop_known_routes_hop_count.sql @@ -0,0 +1,5 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- No statement reads (iata, hop_count); hop filters use idx_known_routes_hop_iata. +DROP INDEX CONCURRENTLY IF EXISTS idx_known_routes_hop_count; diff --git a/db/migrations/009_drop_known_routes_reconfirm.sql b/db/migrations/009_drop_known_routes_reconfirm.sql new file mode 100644 index 0000000..46f59b2 --- /dev/null +++ b/db/migrations/009_drop_known_routes_reconfirm.sql @@ -0,0 +1,5 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- No statement reads (last_reconfirmed_at); reconfirm no longer scans by it. +DROP INDEX CONCURRENTLY IF EXISTS idx_known_routes_reconfirm; diff --git a/db/queries/queries.sql b/db/queries/queries.sql index 12eefd9..75d7c2d 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -775,7 +775,8 @@ WHERE last_seen < $1 -- name: DeleteExpiredRoutes :one -- One batch past retention, keyset after the per-run cursor (the first call passes -infinity, 0). -- The truncated bound (ms ceiling of the cutoff) drives idx_known_routes_keyset; the exact --- residual decides membership. + interval 0 keeps it off idx_known_routes_last_seen. +-- residual decides membership. + interval 0 keeps the exact cutoff a Filter, so the planner +-- cannot switch to a last_seen index on deployments that still have one. WITH expired AS ( SELECT r.iata, r.path_key, date_trunc('milliseconds', r.last_seen, 'UTC') AS ts, r.id FROM known_routes r diff --git a/db/routes_statements_integration_test.go b/db/routes_statements_integration_test.go index 79ece99..b04648f 100644 --- a/db/routes_statements_integration_test.go +++ b/db/routes_statements_integration_test.go @@ -82,7 +82,7 @@ func TestKnownRoutesCursorTiesPostgres(t *testing.T) { } // An unanalyzed region pages from iata_keyset, never a Sort over the region. Built on the -// end-state index set (a later change drops the rest), where the (hop_count, iata) index must not win. +// end-state index set (the baseline less what migrations 004-009 drop), where the (hop_count, iata) index must not win. func TestKnownRoutesUnanalyzedRegionPostgres(t *testing.T) { ctx, tx := retentionTx(t) applyBaseline(t, ctx, tx) diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index 03137e6..a7fcb77 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -29,7 +29,8 @@ type Querier interface { DeleteChannelMembersNotIn(ctx context.Context, arg DeleteChannelMembersNotInParams) error // One batch past retention, keyset after the per-run cursor (the first call passes -infinity, 0). // The truncated bound (ms ceiling of the cutoff) drives idx_known_routes_keyset; the exact - // residual decides membership. + interval 0 keeps it off idx_known_routes_last_seen. + // residual decides membership. + interval 0 keeps the exact cutoff a Filter, so the planner + // cannot switch to a last_seen index on deployments that still have one. DeleteExpiredRoutes(ctx context.Context, arg DeleteExpiredRoutesParams) (DeleteExpiredRoutesRow, error) // One batch of rarely-seen routes after the cursor, inside a window of at most 6 h below the grace // cutoff: candidates without locks, then lock SKIP LOCKED through the pkey (re-checked on the newest diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 8892dd2..4e9ec02 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -195,7 +195,8 @@ type DeleteExpiredRoutesRow struct { // One batch past retention, keyset after the per-run cursor (the first call passes -infinity, 0). // The truncated bound (ms ceiling of the cutoff) drives idx_known_routes_keyset; the exact -// residual decides membership. + interval 0 keeps it off idx_known_routes_last_seen. +// residual decides membership. + interval 0 keeps the exact cutoff a Filter, so the planner +// cannot switch to a last_seen index on deployments that still have one. func (q *Queries) DeleteExpiredRoutes(ctx context.Context, arg DeleteExpiredRoutesParams) (DeleteExpiredRoutesRow, error) { row := q.db.QueryRow(ctx, deleteExpiredRoutes, arg.AfterTs,