From 94b4b40f9b47c507735fa1ece8eef7af9928ab2d Mon Sep 17 00:00:00 2001 From: MrAlders0n <55921894+MrAlders0n@users.noreply.github.com> Date: Mon, 19 Oct 2026 13:45:39 -0400 Subject: [PATCH] planguard: plan and result regression checks, fixture tier in CI, read-only mode (#240) * feat(planguard): read sqlc statement text and load a strict plan corpus * fix(planguard): fail on unextracted statements, keep fragments mode-neutral * feat(planguard): plan assertions, scoped sort exemptions and narrow retired-index verdicts * fix(planguard): an errored run always fails * fix(planguard): keep subplan conds off their parent scan; fail on unanalyzed budgets * feat(planguard): read-only plan runner, reports and the run command * fix(planguard): validate identifiers in emitted scripts; prove read-only runs * feat(planguard): end-state corpus, planted regressions and the generated cached-plan policy * fix(planguard): derive observer cases from data; mark legacy generic plans * fix(planguard): bound the dense-region route resolvers; test policy refusals * fix(planguard): re-measure the dense-region route cases; cover every policy refusal * feat(planguard): synthetic fixture built through the migrations, checked in CI * ci(planguard): build the fixture once; guard fixture data for identifiers * fix(planguard): hash every fixture generator input; harden the fixture guard * feat(planguard): snapshot diff over every column, named result classes and cursor walks * fix(planguard): bound the relaxed diff classes; derive the cursor tie from data * fix(planguard): refuse multi-statement text in the corpus runner * feat: read-only server mode and an HTTP page diff between two builds * fix(planguard): refuse escape strings and psql meta-commands in corpus text * fix: storediff follows each build's own cursor; prove read-only gates via the log * fix(planguard): match the PostgreSQL lexer on high-bit bytes; check every emitted line * feat(admin): daily read-only plan check on the maintenance pool, its metric and report * feat(planguard): daily prod check behind planguard.daily_check * fix(planguard): cancel fresh runs at shutdown; expose a never-ran check * ci(planguard): pin the fixture server and collation; refresh baselines * fix(planguard): calibrate route budgets on one dataset; skip entries whose index is absent --- .github/workflows/ci.yml | 31 +- .gitignore | 1 + cmd/beacon/main.go | 223 +- cmd/beacon/planguard_test.go | 76 + cmd/beacon/readonly.go | 39 + cmd/beacon/readonly_integration_test.go | 411 ++ cmd/beacon/readonly_test.go | 31 + cmd/planguard/main.go | 613 +++ cmd/planguard/main_test.go | 107 + config.yaml.example | 8 + ...k.maint.delete_old_packets.dml.custom.json | 18 + ...int.delete_old_packets.victims.custom.json | 5 + .../fixture/retention.expired.dml.custom.json | 18 + .../retention.expired.select.custom.json | 4 + ...retention.expired.select.final.custom.json | 4 + .../fixture/retention.grace.cand.custom.json | 4 + .../fixture/retention.grace.dml.custom.json | 28 + .../retention.grace.fullpass.custom.json | 4 + .../routes.byiata.R01_SEA_first_l50.auto.json | 4 + ...outes.byiata.R01_SEA_first_l50.custom.json | 4 + ...utes.byiata.R01_SEA_first_l50.generic.json | 4 + ...routes.byiata.R02_SEA_first_l200.auto.json | 4 + ...utes.byiata.R02_SEA_first_l200.custom.json | 4 + ...tes.byiata.R02_SEA_first_l200.generic.json | 4 + .../routes.byiata.R03_SEA_page2_l50.auto.json | 4 + ...outes.byiata.R03_SEA_page2_l50.custom.json | 4 + ...utes.byiata.R03_SEA_page2_l50.generic.json | 4 + .../routes.byiata.R04_SEA_deep_l200.auto.json | 4 + ...outes.byiata.R04_SEA_deep_l200.custom.json | 4 + ...utes.byiata.R04_SEA_deep_l200.generic.json | 4 + ...routes.byiata.R05_BWE_first_l200.auto.json | 4 + ...utes.byiata.R05_BWE_first_l200.custom.json | 4 + ...tes.byiata.R05_BWE_first_l200.generic.json | 4 + .../routes.byiata.R06_MAD_first_l50.auto.json | 4 + ...outes.byiata.R06_MAD_first_l50.custom.json | 4 + ...utes.byiata.R06_MAD_first_l50.generic.json | 4 + ...routes.byiata.R07_CLE_first_l200.auto.json | 4 + ...utes.byiata.R07_CLE_first_l200.custom.json | 4 + ...tes.byiata.R07_CLE_first_l200.generic.json | 4 + .../routes.byiata.R08_NEW_first_l50.auto.json | 4 + ...outes.byiata.R08_NEW_first_l50.custom.json | 4 + ...utes.byiata.R08_NEW_first_l50.generic.json | 4 + .../routes.byiata.R09_NEW_page2_l50.auto.json | 4 + ...outes.byiata.R09_NEW_page2_l50.custom.json | 4 + ...utes.byiata.R09_NEW_page2_l50.generic.json | 4 + .../routes.byiata.R10_NEW_deep_l200.auto.json | 4 + ...outes.byiata.R10_NEW_deep_l200.custom.json | 4 + ...utes.byiata.R10_NEW_deep_l200.generic.json | 4 + .../routes.byiata.R11_ZZZ_l50.auto.json | 4 + .../routes.byiata.R11_ZZZ_l50.custom.json | 4 + .../routes.byiata.R11_ZZZ_l50.generic.json | 4 + .../routes.byiata.R12_WRE_l50.auto.json | 4 + .../routes.byiata.R12_WRE_l50.custom.json | 4 + .../routes.byiata.R12_WRE_l50.generic.json | 4 + .../routes.byiata.R13_SEA_tie_l200.auto.json | 4 + ...routes.byiata.R13_SEA_tie_l200.custom.json | 4 + ...outes.byiata.R13_SEA_tie_l200.generic.json | 4 + .../routes.byiata.R14_BWE_deep_l200.auto.json | 4 + ...outes.byiata.R14_BWE_deep_l200.custom.json | 4 + ...utes.byiata.R14_BWE_deep_l200.generic.json | 4 + ...atahop.H01_SEA_hop6_dense_l200.custom.json | 4 + ...atahop.H03_SEA_hop25_rare_l200.custom.json | 5 + ...atahop.H04_SEA_hop32_vrare_l50.custom.json | 5 + ...tahop.H05_SEA_hop1_absent_l200.custom.json | 5 + ...atahop.H06_SEA_hop6_page2_l200.custom.json | 4 + ...atahop.H07_BWE_hop5_dense_l200.custom.json | 4 + ...iatahop.H08_BWE_hop22_rare_l50.custom.json | 5 + ...iatahop.H10_MAD_hop16_rare_l50.custom.json | 5 + .../fixture/routes.cross.x_0355.custom.json | 6 + .../fixture/routes.cross.x_0867.custom.json | 8 + .../fixture/routes.cross.x_1410.custom.json | 8 + .../fixture/routes.cross.x_3d67.custom.json | 8 + .../fixture/routes.cross.x_5731.custom.json | 8 + .../fixture/routes.cross.x_f877.custom.json | 8 + .../routes.cross.x_sea_rare.custom.json | 8 + .../routes.cross.x_sea_top1.custom.json | 8 + .../routes.global.G01_first_l50.auto.json | 4 + .../routes.global.G01_first_l50.custom.json | 4 + .../routes.global.G01_first_l50.generic.json | 4 + .../routes.global.G02_page2_l50.auto.json | 4 + .../routes.global.G02_page2_l50.custom.json | 4 + .../routes.global.G02_page2_l50.generic.json | 4 + ...tes.global.G03_deep_curonly_l200.auto.json | 4 + ...s.global.G03_deep_curonly_l200.custom.json | 4 + ....global.G03_deep_curonly_l200.generic.json | 4 + .../routes.global.G04_tie_l50.auto.json | 4 + .../routes.global.G04_tie_l50.custom.json | 4 + .../routes.global.G04_tie_l50.generic.json | 4 + ....globalhop.GH01_hop6_dense_l50.custom.json | 4 + ...globalhop.GH02_hop6_dense_l200.custom.json | 4 + ...utes.globalhop.GH03_hop15_l200.custom.json | 4 + ...utes.globalhop.GH04_hop20_l200.custom.json | 4 + ...utes.globalhop.GH05_hop25_l200.custom.json | 5 + ...outes.globalhop.GH06_hop32_l50.custom.json | 5 + ...lobalhop.GH07_hop42_single_l50.custom.json | 5 + ...globalhop.GH08_hop1_absent_l50.custom.json | 5 + ...alhop.GH09_hop1_absent_cur_l50.custom.json | 5 + ....globalhop.GH10_hop6_deep_l200.custom.json | 4 + .../routes.search.s_1410_3d67_RTM.custom.json | 6 + .../routes.search.s_1410_3d67_all.custom.json | 6 + ...tes.search.s_14_3d_RTM_AMS_EIN.custom.json | 6 + .../routes.search.s_14_3d_all.custom.json | 6 + .../routes.search.s_5731_0355_all.custom.json | 6 + .../routes.search.s_f877_0867_all.custom.json | 6 + ...routes.search.s_sea_lowco_rare.custom.json | 6 + .../routes.search.s_sea_pop_rare.custom.json | 6 + .../routes.search.s_sea_pop_top2.custom.json | 8 + .../fixture/s7.pa.all625_7d_today.auto.json | 11 + .../fixture/s7.pa.all625_7d_today.custom.json | 6 + .../fixture/s7.ta.all625_7d_today.auto.json | 21 + .../fixture/s7.ta.all625_7d_today.custom.json | 18 + .../fixture/s7.tn.all625_7d_today.auto.json | 10 + .../fixture/s7.tn.all625_7d_today.custom.json | 8 + .../fixture/sm.chan.after.custom.json | 5 + .../fixture/sm.chan.first.custom.json | 5 + .../baseline/fixture/sm.chan.sea.custom.json | 10 + .../fixture/sm.drift.SEA_t0_500.custom.json | 12 + .../baseline/fixture/sm.drift.all.custom.json | 10 + .../fixture/sm.drift.bogus5000.custom.json | 11 + .../fixture/sm.node.top_sea.auto.json | 19 + .../fixture/sm.node.top_sea.custom.json | 19 + .../fixture/sm.node.top_sea.generic.json | 19 + .../fixture/sm.node.unknown.auto.json | 19 + .../fixture/sm.node.unknown.custom.json | 19 + .../fixture/sm.node.unknown.generic.json | 19 + .../fixture/sm.ntypes.SEA.custom.json | 9 + .../fixture/sm.ntypes.all.custom.json | 5 + .../fixture/sm.ntypes.bogus5000.custom.json | 9 + .../fixture/sm.ntypes.list50.custom.json | 9 + .../fixture/sm.write.touch_channel.auto.json | 4 + .../fixture/sm.write.upsert_channel.auto.json | 4 + .../sm.write.upsert_node_iata.auto.json | 4 + db/planguard/corpus.yaml | 4574 +++++++++++++++++ db/planguard/diff/planted_readmodels.yaml | 138 + db/planguard/embed.go | 14 + db/planguard/fixture/10_reference.sql | 109 + db/planguard/fixture/20_routes.sql | 82 + db/planguard/fixture/30_packets.sql | 51 + db/planguard/fixture/40_observations.sql | 72 + db/planguard/fixture/50_channel_messages.sql | 37 + db/planguard/fixture/60_load.sql | 34 + db/planguard/fixture/70_rollups.sql | 70 + db/planguard/fixture/80_statistics.sql | 38 + db/planguard/fixture/90_stale_region.sql | 34 + db/planguard/fixture/source.sql | 34 + db/planguard/fixture/testdata/cm_channel.csv | 364 ++ .../fixture/testdata/node_iatas_iata.csv | 584 +++ db/planguard/fixture/testdata/nodes_types.csv | 8 + .../fixture/testdata/packets_hour.csv | 73 + .../fixture/testdata/packets_life.csv | 102 + .../fixture/testdata/packets_obscount.csv | 210 + .../fixture/testdata/packets_scope.csv | 418 ++ .../fixture/testdata/packets_types.csv | 65 + db/planguard/fixture/testdata/po_hop.csv | 65 + db/planguard/fixture/testdata/po_iata.csv | 562 ++ db/planguard/fixture/testdata/po_observer.csv | 3559 +++++++++++++ .../fixture/testdata/po_sites_per_packet.csv | 44 + .../fixture/testdata/po_whole_second.csv | 2 + .../fixture/testdata/routes_age_obs.csv | 509 ++ .../fixture/testdata/routes_iata_hop.csv | 1454 ++++++ db/planguard/fixture/testdata/routes_obs.csv | 51 + .../fixture/testdata/routes_regions.csv | 408 ++ .../fixture/testdata/routes_relay.csv | 2196 ++++++++ db/planguard/overlay/packet_sites_walk.sql | 46 + .../sql/derived/DeleteExpiredRoutesSelect.sql | 13 + .../derived/DeleteGraceRoutesSliceCand.sql | 16 + .../sql/derived/DeleteOldPacketsVictims.sql | 10 + db/planguard/sql/oracle/OracleBySites.sql | 47 + .../sql/pending/AddChannelMessageIATA.sql | 9 + .../FillChannelMessageIATAsByMessages.sql | 11 + db/planguard/sql/pending/FilterKnownSites.sql | 10 + .../sql/pending/GetObserverTelemetrySince.sql | 11 + .../sql/pending/GetPathStatsProbe.sql | 25 + db/planguard/sql/pending/GetStatsSeries.sql | 94 + .../pending/GetStatsTopAdvertisersProbe.sql | 39 + db/planguard/sql/pending/GetTopNodesProbe.sql | 29 + .../sql/pending/InsertObservation.sql | 24 + .../pending/ListAllChannelMessagesByIATAs.sql | 24 + .../ListAllChannelMessagesByIATAsScoped.sql | 24 + .../ListAllChannelMessagesByIATAsSince.sql | 27 + .../pending/ListAllChannelMessagesPlain.sql | 14 + .../ListAllChannelMessagesPlainScoped.sql | 14 + .../pending/ListAllChannelMessagesSince.sql | 16 + .../pending/ListChannelMessagesByIATAs.sql | 24 + .../ListChannelMessagesByIATAsScoped.sql | 24 + .../sql/pending/ListChannelMessagesPlain.sql | 14 + .../ListChannelMessagesPlainScoped.sql | 18 + .../pending/ListChannelMessagesSinceWalk.sql | 18 + db/planguard/sql/pending/ListNodes.sql | 42 + .../sql/pending/ListObserverAdverts.sql | 23 + db/planguard/sql/pending/ListObservers.sql | 41 + .../sql/pending/ListPacketsByScopes.sql | 48 + .../sql/pending/ListPacketsByScopesTypes.sql | 49 + .../sql/pending/ListPacketsBySite.sql | 44 + .../sql/pending/ListPacketsBySites.sql | 54 + .../sql/pending/ListPacketsBySitesFirst.sql | 50 + .../sql/pending/ListPacketsBySitesScoped.sql | 59 + .../pending/ListPacketsBySitesScopedFirst.sql | 54 + .../sql/pending/ListPacketsBySitesSince.sql | 58 + .../pending/ListPacketsBySitesSinceFirst.sql | 53 + .../sql/pending/ListPacketsBySitesTyped.sql | 56 + .../pending/ListPacketsBySitesTypedFirst.sql | 52 + .../sql/pending/ListPacketsByTypes.sql | 48 + db/planguard/sql/pending/ListPacketsPage.sql | 41 + .../sql/pending/ListPacketsWindow.sql | 55 + .../sql/pending/ListPacketsWindowScoped.sql | 61 + db/planguard/sql/pending/ListTraceTags.sql | 28 + .../sql/pending/SweepPacketSitesFronts.sql | 37 + .../sql/planted/DeleteOldRoutesSelect.sql | 12 + .../planted/DeleteOldRoutesSelectNoLock.sql | 11 + .../sql/planted/DeleteOldRoutes_202acc6.sql | 15 + .../sql/planted/ListKnownRoutes_202acc6.sql | 31 + .../planted/ListPacketsBySitesReadmodels.sql | 57 + db/planguard/storediff/smoke.yaml | 45 + db/planpolicy/cached_gen.go | 9 +- db/planpolicy/planpolicy_test.go | 3 +- db/planrouter_test.go | 27 +- db/pools_integration_test.go | 73 + db/sqltext.go | 11 + docs/docs.go | 144 + docs/swagger.json | 144 + docs/swagger.yaml | 107 + internal/api/admin.go | 33 +- internal/api/handlers/db_admin.go | 64 +- internal/api/handlers/db_admin_test.go | 147 +- internal/background/planguard.go | 162 + internal/background/planguard_test.go | 322 ++ internal/config/config.go | 26 +- internal/config/config_test.go | 60 + internal/metrics/metrics.go | 38 +- internal/metrics/metrics_test.go | 31 + internal/planguard/assert.go | 380 ++ internal/planguard/assert_test.go | 415 ++ internal/planguard/casefile.go | 151 + internal/planguard/corpus.go | 781 +++ internal/planguard/corpus_integration_test.go | 112 + internal/planguard/corpus_repo_test.go | 262 + internal/planguard/corpus_test.go | 248 + internal/planguard/diff.go | 1355 +++++ internal/planguard/diff_integration_test.go | 360 ++ internal/planguard/diff_report.go | 143 + internal/planguard/diff_test.go | 592 +++ internal/planguard/dml_internal_test.go | 29 + internal/planguard/emit.go | 344 ++ internal/planguard/emit_test.go | 370 ++ internal/planguard/export_test.go | 10 + internal/planguard/fixture.go | 409 ++ .../planguard/fixture_integration_test.go | 317 ++ internal/planguard/fixture_internal_test.go | 161 + internal/planguard/genpolicy.go | 134 + internal/planguard/genpolicy_test.go | 130 + internal/planguard/helpers_test.go | 35 + .../planguard/multistatement_internal_test.go | 101 + internal/planguard/multistatement_test.go | 253 + internal/planguard/plan.go | 207 + internal/planguard/plan_test.go | 101 + internal/planguard/prod.go | 100 + internal/planguard/prod_integration_test.go | 388 ++ internal/planguard/prod_internal_test.go | 85 + internal/planguard/report.go | 269 + internal/planguard/report_test.go | 110 + internal/planguard/runner.go | 822 +++ internal/planguard/runner_integration_test.go | 314 ++ internal/planguard/session.go | 51 + internal/planguard/sqltext.go | 510 ++ internal/planguard/sqltext_test.go | 159 + internal/planguard/storediff.go | 888 ++++ internal/planguard/storediff_test.go | 431 ++ .../corpus/allow_sort_above_limit.yaml | 10 + .../corpus/bad_cached_no_admission.yaml | 7 + .../corpus/bad_cached_unknown_profile.yaml | 9 + .../corpus/bad_default_sort_exempt.yaml | 9 + .../testdata/corpus/bad_derived_edited.yaml | 10 + .../testdata/corpus/bad_duplicate_id.yaml | 8 + .../testdata/corpus/bad_entry_id.yaml | 5 + .../testdata/corpus/bad_index_cond_digit.yaml | 12 + .../testdata/corpus/bad_index_cond_param.yaml | 13 + .../testdata/corpus/bad_index_cond_quote.yaml | 12 + .../testdata/corpus/bad_inherit_cycle.yaml | 8 + .../corpus/bad_inline_resolver_schema.yaml | 7 + .../testdata/corpus/bad_missing_sqlc.yaml | 7 + .../testdata/corpus/bad_param_key.yaml | 7 + .../testdata/corpus/bad_resolver_ref.yaml | 9 + .../testdata/corpus/bad_resolver_schema.yaml | 9 + .../corpus/bad_scan_pattern_literal.yaml | 15 + .../corpus/bad_select_cte_writes.yaml | 7 + .../testdata/corpus/bad_select_writes.yaml | 7 + .../testdata/corpus/bad_sort_exempt.yaml | 10 + .../testdata/corpus/bad_unknown_inherit.yaml | 7 + .../testdata/corpus/bad_unknown_mode.yaml | 7 + .../corpus/bad_unknown_mode_override.yaml | 10 + .../testdata/corpus/bad_unknown_stmt.yaml | 7 + .../planguard/testdata/corpus/derived.yaml | 13 + .../testdata/corpus/genpolicy_hop.yaml | 12 + .../testdata/corpus/genpolicy_hop_skip.yaml | 20 + .../corpus/genpolicy_legacy_pending.yaml | 13 + .../corpus/genpolicy_refuse_degrade_to.yaml | 15 + .../genpolicy_refuse_entry_known_issue.yaml | 15 + .../corpus/genpolicy_refuse_expect_fail.yaml | 15 + ...enpolicy_refuse_mode_known_issue_auto.yaml | 15 + ...olicy_refuse_mode_known_issue_generic.yaml | 15 + .../corpus/genpolicy_refuse_no_auto.yaml | 15 + .../corpus/genpolicy_refuse_no_entries.yaml | 8 + .../corpus/genpolicy_refuse_no_generic.yaml | 15 + .../corpus/genpolicy_refuse_no_sqlc.yaml | 10 + .../corpus/genpolicy_refuse_skip_auto.yaml | 15 + .../corpus/genpolicy_region_no_stale.yaml | 12 + internal/planguard/testdata/corpus/good.yaml | 90 + .../planguard/testdata/corpus/runner.yaml | 55 + .../sql/derived/DeleteExpiredRoutesSelect.sql | 10 + .../DeleteExpiredRoutesSelectEdited.sql | 10 + .../testdata/corpus/sql/harness/H.sql | 3 + .../testdata/corpus/sql/harness/WriteCTE.sql | 4 + .../testdata/corpus/unknown_entry.yaml | 7 + .../testdata/corpus/unknown_top.yaml | 8 + .../testdata/plans/adverts_past_end.json | 100 + .../testdata/plans/bysites_nolimit.json | 328 ++ .../planguard/testdata/plans/bysites_ok.json | 333 ++ .../planguard/testdata/plans/cmi_seqscan.json | 220 + .../plans/globalhop_rare_retired.json | 57 + .../testdata/plans/packets_page.json | 239 + .../testdata/plans/routes_byiata_custom.json | 38 + .../testdata/plans/routes_byiata_generic.json | 38 + .../testdata/plans/routes_byiata_mid.json | 38 + .../planguard/testdata/prodcorpus/corpus.yaml | 46 + internal/planguard/walk.go | 248 + internal/planguard/walk_test.go | 294 ++ 327 files changed, 35447 insertions(+), 79 deletions(-) create mode 100644 cmd/beacon/planguard_test.go create mode 100644 cmd/beacon/readonly.go create mode 100644 cmd/beacon/readonly_integration_test.go create mode 100644 cmd/beacon/readonly_test.go create mode 100644 cmd/planguard/main.go create mode 100644 cmd/planguard/main_test.go create mode 100644 db/planguard/baseline/fixture/pk.maint.delete_old_packets.dml.custom.json create mode 100644 db/planguard/baseline/fixture/pk.maint.delete_old_packets.victims.custom.json create mode 100644 db/planguard/baseline/fixture/retention.expired.dml.custom.json create mode 100644 db/planguard/baseline/fixture/retention.expired.select.custom.json create mode 100644 db/planguard/baseline/fixture/retention.expired.select.final.custom.json create mode 100644 db/planguard/baseline/fixture/retention.grace.cand.custom.json create mode 100644 db/planguard/baseline/fixture/retention.grace.dml.custom.json create mode 100644 db/planguard/baseline/fixture/retention.grace.fullpass.custom.json create mode 100644 db/planguard/baseline/fixture/routes.byiata.R01_SEA_first_l50.auto.json create mode 100644 db/planguard/baseline/fixture/routes.byiata.R01_SEA_first_l50.custom.json create mode 100644 db/planguard/baseline/fixture/routes.byiata.R01_SEA_first_l50.generic.json create mode 100644 db/planguard/baseline/fixture/routes.byiata.R02_SEA_first_l200.auto.json create mode 100644 db/planguard/baseline/fixture/routes.byiata.R02_SEA_first_l200.custom.json create mode 100644 db/planguard/baseline/fixture/routes.byiata.R02_SEA_first_l200.generic.json create mode 100644 db/planguard/baseline/fixture/routes.byiata.R03_SEA_page2_l50.auto.json create mode 100644 db/planguard/baseline/fixture/routes.byiata.R03_SEA_page2_l50.custom.json create mode 100644 db/planguard/baseline/fixture/routes.byiata.R03_SEA_page2_l50.generic.json create mode 100644 db/planguard/baseline/fixture/routes.byiata.R04_SEA_deep_l200.auto.json create mode 100644 db/planguard/baseline/fixture/routes.byiata.R04_SEA_deep_l200.custom.json create mode 100644 db/planguard/baseline/fixture/routes.byiata.R04_SEA_deep_l200.generic.json create mode 100644 db/planguard/baseline/fixture/routes.byiata.R05_BWE_first_l200.auto.json create mode 100644 db/planguard/baseline/fixture/routes.byiata.R05_BWE_first_l200.custom.json create mode 100644 db/planguard/baseline/fixture/routes.byiata.R05_BWE_first_l200.generic.json create mode 100644 db/planguard/baseline/fixture/routes.byiata.R06_MAD_first_l50.auto.json create mode 100644 db/planguard/baseline/fixture/routes.byiata.R06_MAD_first_l50.custom.json create mode 100644 db/planguard/baseline/fixture/routes.byiata.R06_MAD_first_l50.generic.json create mode 100644 db/planguard/baseline/fixture/routes.byiata.R07_CLE_first_l200.auto.json create mode 100644 db/planguard/baseline/fixture/routes.byiata.R07_CLE_first_l200.custom.json create mode 100644 db/planguard/baseline/fixture/routes.byiata.R07_CLE_first_l200.generic.json create mode 100644 db/planguard/baseline/fixture/routes.byiata.R08_NEW_first_l50.auto.json create mode 100644 db/planguard/baseline/fixture/routes.byiata.R08_NEW_first_l50.custom.json create mode 100644 db/planguard/baseline/fixture/routes.byiata.R08_NEW_first_l50.generic.json create mode 100644 db/planguard/baseline/fixture/routes.byiata.R09_NEW_page2_l50.auto.json create mode 100644 db/planguard/baseline/fixture/routes.byiata.R09_NEW_page2_l50.custom.json create mode 100644 db/planguard/baseline/fixture/routes.byiata.R09_NEW_page2_l50.generic.json create mode 100644 db/planguard/baseline/fixture/routes.byiata.R10_NEW_deep_l200.auto.json create mode 100644 db/planguard/baseline/fixture/routes.byiata.R10_NEW_deep_l200.custom.json create mode 100644 db/planguard/baseline/fixture/routes.byiata.R10_NEW_deep_l200.generic.json create mode 100644 db/planguard/baseline/fixture/routes.byiata.R11_ZZZ_l50.auto.json create mode 100644 db/planguard/baseline/fixture/routes.byiata.R11_ZZZ_l50.custom.json create mode 100644 db/planguard/baseline/fixture/routes.byiata.R11_ZZZ_l50.generic.json create mode 100644 db/planguard/baseline/fixture/routes.byiata.R12_WRE_l50.auto.json create mode 100644 db/planguard/baseline/fixture/routes.byiata.R12_WRE_l50.custom.json create mode 100644 db/planguard/baseline/fixture/routes.byiata.R12_WRE_l50.generic.json create mode 100644 db/planguard/baseline/fixture/routes.byiata.R13_SEA_tie_l200.auto.json create mode 100644 db/planguard/baseline/fixture/routes.byiata.R13_SEA_tie_l200.custom.json create mode 100644 db/planguard/baseline/fixture/routes.byiata.R13_SEA_tie_l200.generic.json create mode 100644 db/planguard/baseline/fixture/routes.byiata.R14_BWE_deep_l200.auto.json create mode 100644 db/planguard/baseline/fixture/routes.byiata.R14_BWE_deep_l200.custom.json create mode 100644 db/planguard/baseline/fixture/routes.byiata.R14_BWE_deep_l200.generic.json create mode 100644 db/planguard/baseline/fixture/routes.byiatahop.H01_SEA_hop6_dense_l200.custom.json create mode 100644 db/planguard/baseline/fixture/routes.byiatahop.H03_SEA_hop25_rare_l200.custom.json create mode 100644 db/planguard/baseline/fixture/routes.byiatahop.H04_SEA_hop32_vrare_l50.custom.json create mode 100644 db/planguard/baseline/fixture/routes.byiatahop.H05_SEA_hop1_absent_l200.custom.json create mode 100644 db/planguard/baseline/fixture/routes.byiatahop.H06_SEA_hop6_page2_l200.custom.json create mode 100644 db/planguard/baseline/fixture/routes.byiatahop.H07_BWE_hop5_dense_l200.custom.json create mode 100644 db/planguard/baseline/fixture/routes.byiatahop.H08_BWE_hop22_rare_l50.custom.json create mode 100644 db/planguard/baseline/fixture/routes.byiatahop.H10_MAD_hop16_rare_l50.custom.json create mode 100644 db/planguard/baseline/fixture/routes.cross.x_0355.custom.json create mode 100644 db/planguard/baseline/fixture/routes.cross.x_0867.custom.json create mode 100644 db/planguard/baseline/fixture/routes.cross.x_1410.custom.json create mode 100644 db/planguard/baseline/fixture/routes.cross.x_3d67.custom.json create mode 100644 db/planguard/baseline/fixture/routes.cross.x_5731.custom.json create mode 100644 db/planguard/baseline/fixture/routes.cross.x_f877.custom.json create mode 100644 db/planguard/baseline/fixture/routes.cross.x_sea_rare.custom.json create mode 100644 db/planguard/baseline/fixture/routes.cross.x_sea_top1.custom.json create mode 100644 db/planguard/baseline/fixture/routes.global.G01_first_l50.auto.json create mode 100644 db/planguard/baseline/fixture/routes.global.G01_first_l50.custom.json create mode 100644 db/planguard/baseline/fixture/routes.global.G01_first_l50.generic.json create mode 100644 db/planguard/baseline/fixture/routes.global.G02_page2_l50.auto.json create mode 100644 db/planguard/baseline/fixture/routes.global.G02_page2_l50.custom.json create mode 100644 db/planguard/baseline/fixture/routes.global.G02_page2_l50.generic.json create mode 100644 db/planguard/baseline/fixture/routes.global.G03_deep_curonly_l200.auto.json create mode 100644 db/planguard/baseline/fixture/routes.global.G03_deep_curonly_l200.custom.json create mode 100644 db/planguard/baseline/fixture/routes.global.G03_deep_curonly_l200.generic.json create mode 100644 db/planguard/baseline/fixture/routes.global.G04_tie_l50.auto.json create mode 100644 db/planguard/baseline/fixture/routes.global.G04_tie_l50.custom.json create mode 100644 db/planguard/baseline/fixture/routes.global.G04_tie_l50.generic.json create mode 100644 db/planguard/baseline/fixture/routes.globalhop.GH01_hop6_dense_l50.custom.json create mode 100644 db/planguard/baseline/fixture/routes.globalhop.GH02_hop6_dense_l200.custom.json create mode 100644 db/planguard/baseline/fixture/routes.globalhop.GH03_hop15_l200.custom.json create mode 100644 db/planguard/baseline/fixture/routes.globalhop.GH04_hop20_l200.custom.json create mode 100644 db/planguard/baseline/fixture/routes.globalhop.GH05_hop25_l200.custom.json create mode 100644 db/planguard/baseline/fixture/routes.globalhop.GH06_hop32_l50.custom.json create mode 100644 db/planguard/baseline/fixture/routes.globalhop.GH07_hop42_single_l50.custom.json create mode 100644 db/planguard/baseline/fixture/routes.globalhop.GH08_hop1_absent_l50.custom.json create mode 100644 db/planguard/baseline/fixture/routes.globalhop.GH09_hop1_absent_cur_l50.custom.json create mode 100644 db/planguard/baseline/fixture/routes.globalhop.GH10_hop6_deep_l200.custom.json create mode 100644 db/planguard/baseline/fixture/routes.search.s_1410_3d67_RTM.custom.json create mode 100644 db/planguard/baseline/fixture/routes.search.s_1410_3d67_all.custom.json create mode 100644 db/planguard/baseline/fixture/routes.search.s_14_3d_RTM_AMS_EIN.custom.json create mode 100644 db/planguard/baseline/fixture/routes.search.s_14_3d_all.custom.json create mode 100644 db/planguard/baseline/fixture/routes.search.s_5731_0355_all.custom.json create mode 100644 db/planguard/baseline/fixture/routes.search.s_f877_0867_all.custom.json create mode 100644 db/planguard/baseline/fixture/routes.search.s_sea_lowco_rare.custom.json create mode 100644 db/planguard/baseline/fixture/routes.search.s_sea_pop_rare.custom.json create mode 100644 db/planguard/baseline/fixture/routes.search.s_sea_pop_top2.custom.json create mode 100644 db/planguard/baseline/fixture/s7.pa.all625_7d_today.auto.json create mode 100644 db/planguard/baseline/fixture/s7.pa.all625_7d_today.custom.json create mode 100644 db/planguard/baseline/fixture/s7.ta.all625_7d_today.auto.json create mode 100644 db/planguard/baseline/fixture/s7.ta.all625_7d_today.custom.json create mode 100644 db/planguard/baseline/fixture/s7.tn.all625_7d_today.auto.json create mode 100644 db/planguard/baseline/fixture/s7.tn.all625_7d_today.custom.json create mode 100644 db/planguard/baseline/fixture/sm.chan.after.custom.json create mode 100644 db/planguard/baseline/fixture/sm.chan.first.custom.json create mode 100644 db/planguard/baseline/fixture/sm.chan.sea.custom.json create mode 100644 db/planguard/baseline/fixture/sm.drift.SEA_t0_500.custom.json create mode 100644 db/planguard/baseline/fixture/sm.drift.all.custom.json create mode 100644 db/planguard/baseline/fixture/sm.drift.bogus5000.custom.json create mode 100644 db/planguard/baseline/fixture/sm.node.top_sea.auto.json create mode 100644 db/planguard/baseline/fixture/sm.node.top_sea.custom.json create mode 100644 db/planguard/baseline/fixture/sm.node.top_sea.generic.json create mode 100644 db/planguard/baseline/fixture/sm.node.unknown.auto.json create mode 100644 db/planguard/baseline/fixture/sm.node.unknown.custom.json create mode 100644 db/planguard/baseline/fixture/sm.node.unknown.generic.json create mode 100644 db/planguard/baseline/fixture/sm.ntypes.SEA.custom.json create mode 100644 db/planguard/baseline/fixture/sm.ntypes.all.custom.json create mode 100644 db/planguard/baseline/fixture/sm.ntypes.bogus5000.custom.json create mode 100644 db/planguard/baseline/fixture/sm.ntypes.list50.custom.json create mode 100644 db/planguard/baseline/fixture/sm.write.touch_channel.auto.json create mode 100644 db/planguard/baseline/fixture/sm.write.upsert_channel.auto.json create mode 100644 db/planguard/baseline/fixture/sm.write.upsert_node_iata.auto.json create mode 100644 db/planguard/corpus.yaml create mode 100644 db/planguard/diff/planted_readmodels.yaml create mode 100644 db/planguard/embed.go create mode 100644 db/planguard/fixture/10_reference.sql create mode 100644 db/planguard/fixture/20_routes.sql create mode 100644 db/planguard/fixture/30_packets.sql create mode 100644 db/planguard/fixture/40_observations.sql create mode 100644 db/planguard/fixture/50_channel_messages.sql create mode 100644 db/planguard/fixture/60_load.sql create mode 100644 db/planguard/fixture/70_rollups.sql create mode 100644 db/planguard/fixture/80_statistics.sql create mode 100644 db/planguard/fixture/90_stale_region.sql create mode 100644 db/planguard/fixture/source.sql create mode 100644 db/planguard/fixture/testdata/cm_channel.csv create mode 100644 db/planguard/fixture/testdata/node_iatas_iata.csv create mode 100644 db/planguard/fixture/testdata/nodes_types.csv create mode 100644 db/planguard/fixture/testdata/packets_hour.csv create mode 100644 db/planguard/fixture/testdata/packets_life.csv create mode 100644 db/planguard/fixture/testdata/packets_obscount.csv create mode 100644 db/planguard/fixture/testdata/packets_scope.csv create mode 100644 db/planguard/fixture/testdata/packets_types.csv create mode 100644 db/planguard/fixture/testdata/po_hop.csv create mode 100644 db/planguard/fixture/testdata/po_iata.csv create mode 100644 db/planguard/fixture/testdata/po_observer.csv create mode 100644 db/planguard/fixture/testdata/po_sites_per_packet.csv create mode 100644 db/planguard/fixture/testdata/po_whole_second.csv create mode 100644 db/planguard/fixture/testdata/routes_age_obs.csv create mode 100644 db/planguard/fixture/testdata/routes_iata_hop.csv create mode 100644 db/planguard/fixture/testdata/routes_obs.csv create mode 100644 db/planguard/fixture/testdata/routes_regions.csv create mode 100644 db/planguard/fixture/testdata/routes_relay.csv create mode 100644 db/planguard/overlay/packet_sites_walk.sql create mode 100644 db/planguard/sql/derived/DeleteExpiredRoutesSelect.sql create mode 100644 db/planguard/sql/derived/DeleteGraceRoutesSliceCand.sql create mode 100644 db/planguard/sql/derived/DeleteOldPacketsVictims.sql create mode 100644 db/planguard/sql/oracle/OracleBySites.sql create mode 100644 db/planguard/sql/pending/AddChannelMessageIATA.sql create mode 100644 db/planguard/sql/pending/FillChannelMessageIATAsByMessages.sql create mode 100644 db/planguard/sql/pending/FilterKnownSites.sql create mode 100644 db/planguard/sql/pending/GetObserverTelemetrySince.sql create mode 100644 db/planguard/sql/pending/GetPathStatsProbe.sql create mode 100644 db/planguard/sql/pending/GetStatsSeries.sql create mode 100644 db/planguard/sql/pending/GetStatsTopAdvertisersProbe.sql create mode 100644 db/planguard/sql/pending/GetTopNodesProbe.sql create mode 100644 db/planguard/sql/pending/InsertObservation.sql create mode 100644 db/planguard/sql/pending/ListAllChannelMessagesByIATAs.sql create mode 100644 db/planguard/sql/pending/ListAllChannelMessagesByIATAsScoped.sql create mode 100644 db/planguard/sql/pending/ListAllChannelMessagesByIATAsSince.sql create mode 100644 db/planguard/sql/pending/ListAllChannelMessagesPlain.sql create mode 100644 db/planguard/sql/pending/ListAllChannelMessagesPlainScoped.sql create mode 100644 db/planguard/sql/pending/ListAllChannelMessagesSince.sql create mode 100644 db/planguard/sql/pending/ListChannelMessagesByIATAs.sql create mode 100644 db/planguard/sql/pending/ListChannelMessagesByIATAsScoped.sql create mode 100644 db/planguard/sql/pending/ListChannelMessagesPlain.sql create mode 100644 db/planguard/sql/pending/ListChannelMessagesPlainScoped.sql create mode 100644 db/planguard/sql/pending/ListChannelMessagesSinceWalk.sql create mode 100644 db/planguard/sql/pending/ListNodes.sql create mode 100644 db/planguard/sql/pending/ListObserverAdverts.sql create mode 100644 db/planguard/sql/pending/ListObservers.sql create mode 100644 db/planguard/sql/pending/ListPacketsByScopes.sql create mode 100644 db/planguard/sql/pending/ListPacketsByScopesTypes.sql create mode 100644 db/planguard/sql/pending/ListPacketsBySite.sql create mode 100644 db/planguard/sql/pending/ListPacketsBySites.sql create mode 100644 db/planguard/sql/pending/ListPacketsBySitesFirst.sql create mode 100644 db/planguard/sql/pending/ListPacketsBySitesScoped.sql create mode 100644 db/planguard/sql/pending/ListPacketsBySitesScopedFirst.sql create mode 100644 db/planguard/sql/pending/ListPacketsBySitesSince.sql create mode 100644 db/planguard/sql/pending/ListPacketsBySitesSinceFirst.sql create mode 100644 db/planguard/sql/pending/ListPacketsBySitesTyped.sql create mode 100644 db/planguard/sql/pending/ListPacketsBySitesTypedFirst.sql create mode 100644 db/planguard/sql/pending/ListPacketsByTypes.sql create mode 100644 db/planguard/sql/pending/ListPacketsPage.sql create mode 100644 db/planguard/sql/pending/ListPacketsWindow.sql create mode 100644 db/planguard/sql/pending/ListPacketsWindowScoped.sql create mode 100644 db/planguard/sql/pending/ListTraceTags.sql create mode 100644 db/planguard/sql/pending/SweepPacketSitesFronts.sql create mode 100644 db/planguard/sql/planted/DeleteOldRoutesSelect.sql create mode 100644 db/planguard/sql/planted/DeleteOldRoutesSelectNoLock.sql create mode 100644 db/planguard/sql/planted/DeleteOldRoutes_202acc6.sql create mode 100644 db/planguard/sql/planted/ListKnownRoutes_202acc6.sql create mode 100644 db/planguard/sql/planted/ListPacketsBySitesReadmodels.sql create mode 100644 db/planguard/storediff/smoke.yaml create mode 100644 db/sqltext.go create mode 100644 internal/background/planguard.go create mode 100644 internal/background/planguard_test.go create mode 100644 internal/planguard/assert.go create mode 100644 internal/planguard/assert_test.go create mode 100644 internal/planguard/casefile.go create mode 100644 internal/planguard/corpus.go create mode 100644 internal/planguard/corpus_integration_test.go create mode 100644 internal/planguard/corpus_repo_test.go create mode 100644 internal/planguard/corpus_test.go create mode 100644 internal/planguard/diff.go create mode 100644 internal/planguard/diff_integration_test.go create mode 100644 internal/planguard/diff_report.go create mode 100644 internal/planguard/diff_test.go create mode 100644 internal/planguard/dml_internal_test.go create mode 100644 internal/planguard/emit.go create mode 100644 internal/planguard/emit_test.go create mode 100644 internal/planguard/export_test.go create mode 100644 internal/planguard/fixture.go create mode 100644 internal/planguard/fixture_integration_test.go create mode 100644 internal/planguard/fixture_internal_test.go create mode 100644 internal/planguard/genpolicy.go create mode 100644 internal/planguard/genpolicy_test.go create mode 100644 internal/planguard/helpers_test.go create mode 100644 internal/planguard/multistatement_internal_test.go create mode 100644 internal/planguard/multistatement_test.go create mode 100644 internal/planguard/plan.go create mode 100644 internal/planguard/plan_test.go create mode 100644 internal/planguard/prod.go create mode 100644 internal/planguard/prod_integration_test.go create mode 100644 internal/planguard/prod_internal_test.go create mode 100644 internal/planguard/report.go create mode 100644 internal/planguard/report_test.go create mode 100644 internal/planguard/runner.go create mode 100644 internal/planguard/runner_integration_test.go create mode 100644 internal/planguard/session.go create mode 100644 internal/planguard/sqltext.go create mode 100644 internal/planguard/sqltext_test.go create mode 100644 internal/planguard/storediff.go create mode 100644 internal/planguard/storediff_test.go create mode 100644 internal/planguard/testdata/corpus/allow_sort_above_limit.yaml create mode 100644 internal/planguard/testdata/corpus/bad_cached_no_admission.yaml create mode 100644 internal/planguard/testdata/corpus/bad_cached_unknown_profile.yaml create mode 100644 internal/planguard/testdata/corpus/bad_default_sort_exempt.yaml create mode 100644 internal/planguard/testdata/corpus/bad_derived_edited.yaml create mode 100644 internal/planguard/testdata/corpus/bad_duplicate_id.yaml create mode 100644 internal/planguard/testdata/corpus/bad_entry_id.yaml create mode 100644 internal/planguard/testdata/corpus/bad_index_cond_digit.yaml create mode 100644 internal/planguard/testdata/corpus/bad_index_cond_param.yaml create mode 100644 internal/planguard/testdata/corpus/bad_index_cond_quote.yaml create mode 100644 internal/planguard/testdata/corpus/bad_inherit_cycle.yaml create mode 100644 internal/planguard/testdata/corpus/bad_inline_resolver_schema.yaml create mode 100644 internal/planguard/testdata/corpus/bad_missing_sqlc.yaml create mode 100644 internal/planguard/testdata/corpus/bad_param_key.yaml create mode 100644 internal/planguard/testdata/corpus/bad_resolver_ref.yaml create mode 100644 internal/planguard/testdata/corpus/bad_resolver_schema.yaml create mode 100644 internal/planguard/testdata/corpus/bad_scan_pattern_literal.yaml create mode 100644 internal/planguard/testdata/corpus/bad_select_cte_writes.yaml create mode 100644 internal/planguard/testdata/corpus/bad_select_writes.yaml create mode 100644 internal/planguard/testdata/corpus/bad_sort_exempt.yaml create mode 100644 internal/planguard/testdata/corpus/bad_unknown_inherit.yaml create mode 100644 internal/planguard/testdata/corpus/bad_unknown_mode.yaml create mode 100644 internal/planguard/testdata/corpus/bad_unknown_mode_override.yaml create mode 100644 internal/planguard/testdata/corpus/bad_unknown_stmt.yaml create mode 100644 internal/planguard/testdata/corpus/derived.yaml create mode 100644 internal/planguard/testdata/corpus/genpolicy_hop.yaml create mode 100644 internal/planguard/testdata/corpus/genpolicy_hop_skip.yaml create mode 100644 internal/planguard/testdata/corpus/genpolicy_legacy_pending.yaml create mode 100644 internal/planguard/testdata/corpus/genpolicy_refuse_degrade_to.yaml create mode 100644 internal/planguard/testdata/corpus/genpolicy_refuse_entry_known_issue.yaml create mode 100644 internal/planguard/testdata/corpus/genpolicy_refuse_expect_fail.yaml create mode 100644 internal/planguard/testdata/corpus/genpolicy_refuse_mode_known_issue_auto.yaml create mode 100644 internal/planguard/testdata/corpus/genpolicy_refuse_mode_known_issue_generic.yaml create mode 100644 internal/planguard/testdata/corpus/genpolicy_refuse_no_auto.yaml create mode 100644 internal/planguard/testdata/corpus/genpolicy_refuse_no_entries.yaml create mode 100644 internal/planguard/testdata/corpus/genpolicy_refuse_no_generic.yaml create mode 100644 internal/planguard/testdata/corpus/genpolicy_refuse_no_sqlc.yaml create mode 100644 internal/planguard/testdata/corpus/genpolicy_refuse_skip_auto.yaml create mode 100644 internal/planguard/testdata/corpus/genpolicy_region_no_stale.yaml create mode 100644 internal/planguard/testdata/corpus/good.yaml create mode 100644 internal/planguard/testdata/corpus/runner.yaml create mode 100644 internal/planguard/testdata/corpus/sql/derived/DeleteExpiredRoutesSelect.sql create mode 100644 internal/planguard/testdata/corpus/sql/derived/DeleteExpiredRoutesSelectEdited.sql create mode 100644 internal/planguard/testdata/corpus/sql/harness/H.sql create mode 100644 internal/planguard/testdata/corpus/sql/harness/WriteCTE.sql create mode 100644 internal/planguard/testdata/corpus/unknown_entry.yaml create mode 100644 internal/planguard/testdata/corpus/unknown_top.yaml create mode 100644 internal/planguard/testdata/plans/adverts_past_end.json create mode 100644 internal/planguard/testdata/plans/bysites_nolimit.json create mode 100644 internal/planguard/testdata/plans/bysites_ok.json create mode 100644 internal/planguard/testdata/plans/cmi_seqscan.json create mode 100644 internal/planguard/testdata/plans/globalhop_rare_retired.json create mode 100644 internal/planguard/testdata/plans/packets_page.json create mode 100644 internal/planguard/testdata/plans/routes_byiata_custom.json create mode 100644 internal/planguard/testdata/plans/routes_byiata_generic.json create mode 100644 internal/planguard/testdata/plans/routes_byiata_mid.json create mode 100644 internal/planguard/testdata/prodcorpus/corpus.yaml create mode 100644 internal/planguard/walk.go create mode 100644 internal/planguard/walk_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b0c45cd..b54010a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,6 +22,17 @@ jobs: --health-interval 5s --health-timeout 5s --health-retries 10 + # The fixture tier's baselines were written on PostgreSQL 16.15 under C collation: keep both fixed here. + planguard-postgres: + image: postgres:16.15-alpine + env: + POSTGRES_PASSWORD: planguard-ci-only + POSTGRES_DB: planguard + POSTGRES_INITDB_ARGS: --locale=C --encoding=UTF8 + ports: + - 5433:5432 + options: >- + --health-cmd pg_isready --health-interval 5s --health-timeout 5s --health-retries 10 steps: - uses: actions/checkout@v4 @@ -50,21 +61,35 @@ jobs: - name: Verify scope refresh concurrency run: go test -race ./internal/meshmapper ./internal/scopestore && go test -race ./db -run '^TestChannelIDCache' - - name: Verify backup startup against PostgreSQL 16 + - name: Verify backup startup and the read-only boot against PostgreSQL 16 env: BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable - run: go test ./cmd/beacon -run '^TestBackupConfigurationPostgres$' -count=1 -v + run: go test ./cmd/beacon -run '^Test(BackupConfiguration|ReadOnlyBootWritesNothing)Postgres$' -count=1 -v - name: Verify stats and endpoint queries against PostgreSQL 16 env: BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable - run: go test ./db -run '^Test(Signal|Paths|PacketSummaries|PacketEndpointsResolveLive|ObserverMetrics|RouteEvidence|RouteEvidenceIndex|AnalyticsRetention|AnalyticsRetentionConcurrent|DeleteOldPacketsBatches|MeshMapperCatalogue|ChannelMessageScopes|NodeLocationReset|RunMigrations[A-Za-z]*|KnownRoutesIndexSet|Reconfirm[A-Za-z]*|BigintCursors|RouteSearchHopOrder|StatsNodeTypesSplit|ListKnownRoutes|ListKnownRoutesMatchesLegacy|KnownRoutesCursorTies|KnownRoutesUnanalyzedRegion|KnownRoutesHopStatementsStayCustom|KnownRoutePaginationTies|RouteRetentionCutoffsExact|DeleteExpiredRoutes|DeleteGraceRoutesSlice|GraceWatermarkHeldByLockedRow|OpenPoolsSettings|PlanRouterClassCUnnamed|APIStoreHopStatementsUnnamed|PgStatStatementsMigrationGuarded|DBStatements|DBStatementsNoExtension|PgStatStatementsMigrationNoCreatableSchema|IndexHealthBaselineOnReindex|UnusedIndexes|PgActivity|RollHourSetsWorkMem|SmallTableIndexSet|NodeReadersAfterIndexDrops|ObserverChannelReadersAfterIndexDrops|UpsertChannelWriteThrough|UpsertChannelHashOnlyWriteThrough)Postgres$' -count=1 -v + run: go test ./db -run '^Test(Signal|Paths|PacketSummaries|PacketEndpointsResolveLive|ObserverMetrics|RouteEvidence|RouteEvidenceIndex|AnalyticsRetention|AnalyticsRetentionConcurrent|DeleteOldPacketsBatches|MeshMapperCatalogue|ChannelMessageScopes|NodeLocationReset|RunMigrations[A-Za-z]*|KnownRoutesIndexSet|Reconfirm[A-Za-z]*|BigintCursors|RouteSearchHopOrder|StatsNodeTypesSplit|ListKnownRoutes|ListKnownRoutesMatchesLegacy|KnownRoutesCursorTies|KnownRoutesUnanalyzedRegion|KnownRoutesHopStatementsStayCustom|KnownRoutePaginationTies|RouteRetentionCutoffsExact|DeleteExpiredRoutes|DeleteGraceRoutesSlice|GraceWatermarkHeldByLockedRow|OpenPoolsSettings|PlanRouterClassCUnnamed|APIStoreHopStatementsUnnamed|PgStatStatementsMigrationGuarded|DBStatements|DBStatementsNoExtension|PgStatStatementsMigrationNoCreatableSchema|IndexHealthBaselineOnReindex|UnusedIndexes|PgActivity|RollHourSetsWorkMem|SmallTableIndexSet|NodeReadersAfterIndexDrops|ObserverChannelReadersAfterIndexDrops|UpsertChannelWriteThrough|UpsertChannelHashOnlyWriteThrough|ReadOnlyPoolsRejectWrites)Postgres$' -count=1 -v - name: Verify route retention task against PostgreSQL 16 env: BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable run: go test ./internal/background -run '^TestRouteRetentionTaskPostgres$' -count=1 -v + # Builds the fixture once; the planguard tests below reuse it (PLANGUARD_FIXTURE_DSN) after checking that its + # marker records a build of the same files. + - name: Planguard fixture tier (plain EXPLAIN on a migrated, skewed fixture) + run: go run ./cmd/planguard run --tier fixture --dsn 'postgres://postgres:planguard-ci-only@127.0.0.1:5433/planguard?sslmode=disable' --out "$RUNNER_TEMP/planguard" + + - name: Verify planguard against PostgreSQL 16 + env: + BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable + PLANGUARD_FIXTURE_DSN: postgres://postgres:planguard-ci-only@127.0.0.1:5433/planguard?sslmode=disable + run: go test ./internal/planguard -timeout 20m -run '^Test(RunnerReadOnly|RunnerRecoversAfterError|UnresolvedParamsSkip|GenericModeShowsParams|CorpusStatementsPrepareClean|BuildFixtureThroughMigrations|BuildFixtureRefusesRealDatabase|BuildFixtureRebuildsStaleInputs|RunDiffSnapshot|RunDiffFindsReadmodelsSkip|StatementScannerMatches|PlanguardProdTier|PlanguardProdCancel|RequiresIndexSkips)Postgres$' -count=1 -v + + - name: Plan policy up to date + run: go run ./cmd/planguard gen-policy && git diff --exit-code db/planpolicy + - name: Verify backup command against PostgreSQL 16 run: | CGO_ENABLED=0 go build -o "$RUNNER_TEMP/beacon-backup" ./cmd/beacon-backup diff --git a/.gitignore b/.gitignore index 4677c15..0257e5f 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ roadmap.md binaries/ cover.out +planguard-out/ diff --git a/cmd/beacon/main.go b/cmd/beacon/main.go index 6efebc1..1ca09a8 100644 --- a/cmd/beacon/main.go +++ b/cmd/beacon/main.go @@ -18,6 +18,7 @@ import ( "time" "github.com/MeshCore-Beacon/beacon-server/db" + pgdata "github.com/MeshCore-Beacon/beacon-server/db/planguard" "github.com/MeshCore-Beacon/beacon-server/db/planpolicy" "github.com/MeshCore-Beacon/beacon-server/docs" "github.com/MeshCore-Beacon/beacon-server/internal/api" @@ -34,10 +35,13 @@ import ( "github.com/MeshCore-Beacon/beacon-server/internal/keystore" "github.com/MeshCore-Beacon/beacon-server/internal/logging" "github.com/MeshCore-Beacon/beacon-server/internal/meshmapper" + "github.com/MeshCore-Beacon/beacon-server/internal/metrics" + "github.com/MeshCore-Beacon/beacon-server/internal/planguard" "github.com/MeshCore-Beacon/beacon-server/internal/presence" "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" "github.com/joho/godotenv" ) @@ -139,7 +143,13 @@ func main() { dsn := getEnv("POSTGRES_DSN") layout := resolved.APIPlanLayout - pools, err := db.OpenPools(ctx, dsn, layout) + mode := bootModeFor(resolved.ReadOnly) + var tune []func(string, *pgxpool.Config) + if resolved.ReadOnly { + slog.Warn(readOnlyWarning, "component", "startup") + tune = append(tune, readOnlyTune) + } + pools, err := db.OpenPools(ctx, dsn, layout, tune...) if err != nil { // Parse errors can embed the complete DSN, including its password. slog.Error("invalid PostgreSQL connection configuration; check POSTGRES_DSN", "component", "startup") @@ -149,9 +159,11 @@ func main() { slog.Info("database pools open", "component", "startup", "layout", layout, "connections", pools.Total()) backupOpts := configureBackup(ctx, cfg, pools.Maint, dsn, configPath) - if err := db.RunMigrations(ctx, pools.Maint); err != nil { - slog.Error("migrations failed", "component", "startup", "error", err) - os.Exit(1) + if mode.Migrate { + if err := db.RunMigrations(ctx, pools.Maint); err != nil { + slog.Error("migrations failed", "component", "startup", "error", err) + os.Exit(1) + } } // Three stores: ingest writes, background and operator work, and API reads. The API store @@ -172,7 +184,9 @@ func main() { // ── Presence write coalescing ──────────────────────────────────────────── // Ingest writes go through the coalescer; reads use the API store. coalescer := presence.New(ingestStore, resolved.PresenceFlushInterval, resolved.PresencePacketTTL) - go coalescer.Run(ctx) + if mode.Coalescer { + go coalescer.Run(ctx) + } // ── Redis cache layer ──────────────────────────────────────────────────── var reader api.Reader = apiStore @@ -201,9 +215,11 @@ func main() { } // ── Seed config data ───────────────────────────────────────────────────── - if err := config.Seed(ctx, cfg, maintStore); err != nil { - slog.Error("failed to seed config", "component", "startup", "error", err) - os.Exit(1) + if mode.Seed { + if err := config.Seed(ctx, cfg, maintStore); err != nil { + slog.Error("failed to seed config", "component", "startup", "error", err) + os.Exit(1) + } } // ── Build transport scope keystore ─────────────────────────────────────── @@ -217,7 +233,7 @@ func main() { scopes.SetManualMembers(cfg.ManualScopeMembers()) slog.Info(fmt.Sprintf("loaded %d transport scopes", len(scopeEntries)), "component", "startup") directory := meshmapper.NewDirectory(maintStore, cfg.MeshMapper.APIKey) - if mm := cfg.MeshMapper; mm.Scopes.Enabled || mm.Zones.Enabled || mm.Channels.Enabled || mm.Repeaters.Enabled { + if mm := cfg.MeshMapper; mode.MeshMapper && (mm.Scopes.Enabled || mm.Zones.Enabled || mm.Channels.Enabled || mm.Repeaters.Enabled) { if cfg.MeshMapper.APIKey == "" { slog.Warn("MeshMapper integration unconfigured: set MESHMAPPER_API_KEY or meshmapper.api_key; retaining cached data", "component", "meshmapper") } @@ -230,7 +246,7 @@ func main() { } } var scopeImporter *meshmapper.Importer - if cfg.MeshMapper.Scopes.Enabled { + if mode.MeshMapper && cfg.MeshMapper.Scopes.Enabled { restoreCtx, cancelRestore := context.WithTimeout(ctx, 10*time.Second) scopeImporter, err = meshmapper.New(restoreCtx, cfg.MeshMapper.Scopes, maintStore, directory, scopes, scopeEntries) cancelRestore() @@ -264,12 +280,15 @@ func main() { localBorders.Store(local) }) } - restoreCtx, cancelRestore := context.WithTimeout(ctx, 10*time.Second) - err = zones.Restore(restoreCtx) - cancelRestore() - if err != nil { - slog.Error("failed to restore MeshMapper boundaries", "component", "startup", "error", err) - os.Exit(1) + // Restore prunes boundaries a disabled import left behind, so a read-only server skips it. + if mode.MeshMapper { + restoreCtx, cancelRestore := context.WithTimeout(ctx, 10*time.Second) + err = zones.Restore(restoreCtx) + cancelRestore() + if err != nil { + slog.Error("failed to restore MeshMapper boundaries", "component", "startup", "error", err) + os.Exit(1) + } } if localBorders != nil && !localBorders.Ready() { slog.Warn("nodes.mark_foreign has no boundaries yet; waiting for MeshMapper", "component", "startup") @@ -315,47 +334,56 @@ func main() { keys := keystore.NewMapKeyStore(entries) - // Restored before the boot backfill so saved MeshMapper keys decrypt history too. - restoreCtx, cancelRestore = context.WithTimeout(ctx, 10*time.Second) - channelImporter, err := meshmapper.NewChannels(restoreCtx, cfg.MeshMapper.Channels, maintStore, directory, keys) - cancelRestore() - if err != nil { - slog.Error("failed to restore MeshMapper channels", "component", "startup", "error", err) - os.Exit(1) + // Restored before the boot backfill so saved MeshMapper keys decrypt history too. Both + // constructors may clear rows when their import is disabled, so a read-only server skips them. + var channelImporter *meshmapper.Channels + var repeaterImporter *meshmapper.Repeaters + if mode.MeshMapper { + restoreCtx, cancelRestore := context.WithTimeout(ctx, 10*time.Second) + channelImporter, err = meshmapper.NewChannels(restoreCtx, cfg.MeshMapper.Channels, maintStore, directory, keys) + cancelRestore() + if err != nil { + slog.Error("failed to restore MeshMapper channels", "component", "startup", "error", err) + os.Exit(1) + } + var onRepeatersChange func(context.Context) + if cr, ok := reader.(*cache.CachedReader); ok { + onRepeatersChange = cr.InvalidateNodeDetails + } + restoreCtx, cancelRestore = context.WithTimeout(ctx, 10*time.Second) + repeaterImporter, err = meshmapper.NewRepeaters(restoreCtx, cfg.MeshMapper.Repeaters, maintStore, directory, onRepeatersChange) + cancelRestore() + if err != nil { + slog.Error("failed to restore MeshMapper repeaters", "component", "startup", "error", err) + os.Exit(1) + } } - var onRepeatersChange func(context.Context) - if cr, ok := reader.(*cache.CachedReader); ok { - onRepeatersChange = cr.InvalidateNodeDetails + if channelImporter != nil && mode.Backfill { + channelImporter.OnNewKeys(func(hashes [][]byte) { + // Off the refresh tick: a backfill can outlast it, and history isn't urgent. + go func() { + backfillCtx, cancel := context.WithTimeout(ctx, 10*time.Minute) + defer cancel() + if n, err := ingest.BackfillChannelHashes(backfillCtx, maintStore, keys, hashes); err != nil { + slog.Error("MeshMapper channel backfill failed", "component", "meshmapper.channels", "error", err) + } else if n > 0 { + slog.Info(fmt.Sprintf("backfilled %d channel message(s) for imported keys", n), "component", "meshmapper.channels") + } + }() + }) } - restoreCtx, cancelRestore = context.WithTimeout(ctx, 10*time.Second) - repeaterImporter, err := meshmapper.NewRepeaters(restoreCtx, cfg.MeshMapper.Repeaters, maintStore, directory, onRepeatersChange) - cancelRestore() - if err != nil { - slog.Error("failed to restore MeshMapper repeaters", "component", "startup", "error", err) - os.Exit(1) - } - channelImporter.OnNewKeys(func(hashes [][]byte) { - // Off the refresh tick: a backfill can outlast it, and history isn't urgent. - go func() { - backfillCtx, cancel := context.WithTimeout(ctx, 10*time.Minute) - defer cancel() - if n, err := ingest.BackfillChannelHashes(backfillCtx, maintStore, keys, hashes); err != nil { - slog.Error("MeshMapper channel backfill failed", "component", "meshmapper.channels", "error", err) - } else if n > 0 { - slog.Info(fmt.Sprintf("backfilled %d channel message(s) for imported keys", n), "component", "meshmapper.channels") - } - }() - }) // ── Backfill channel messages ──────────────────────────────────────────── // Packets whose channel key wasn't yet configured at ingest time were stored as // hash-only channels and never decrypted. Retry them now against the keystore we just // built, so adding a channel key to the config surfaces its history on the next boot // instead of leaving it stranded in the DB indefinitely. - if n, err := ingest.BackfillChannelMessages(ctx, maintStore, keys); err != nil { - slog.Error("config: channel message backfill failed", "component", "startup", "error", err) - } else if n > 0 { - slog.Info(fmt.Sprintf("config: backfilled %d previously-undecrypted channel message(s)", n), "component", "startup") + if mode.Backfill { + if n, err := ingest.BackfillChannelMessages(ctx, maintStore, keys); err != nil { + slog.Error("config: channel message backfill failed", "component", "startup", "error", err) + } else if n > 0 { + slog.Info(fmt.Sprintf("config: backfilled %d previously-undecrypted channel message(s)", n), "component", "startup") + } } // ── Build geographic ingest filter ─────────────────────────────────────────────────────────── @@ -407,15 +435,19 @@ func main() { } var ingestWorkers sync.WaitGroup - ingestWorkers.Go(func() { broker1.Start(ctx) }) - ingestWorkers.Go(func() { broker2.Start(ctx) }) + if mode.Ingest { + ingestWorkers.Go(func() { broker1.Start(ctx) }) + ingestWorkers.Go(func() { broker2.Start(ctx) }) + } rollup := background.NewRollup(maintStore, resolved.PacketRetention, resolved.RollupRetention) - go func() { - if err := rollup.CatchUp(ctx); err != nil { - slog.Error("analytics rollup catch-up failed", "component", "background", "error", err) - } - }() + if mode.Jobs { + go func() { + if err := rollup.CatchUp(ctx); err != nil { + slog.Error("analytics rollup catch-up failed", "component", "background", "error", err) + } + }() + } tasks := []background.Task{ rollup.Task(), background.ViewRefreshTask(maintStore, resolved.ViewRefreshInterval), @@ -441,22 +473,28 @@ func main() { if scopeImporter != nil { tasks = append(tasks, background.Task{Name: "meshmapper.scopes", Interval: meshmapper.PollInterval, Run: scopeImporter.Refresh}) } - if cfg.MeshMapper.Zones.Enabled { + if mode.MeshMapper && cfg.MeshMapper.Zones.Enabled { tasks = append(tasks, background.Task{Name: "meshmapper.zones", Interval: meshmapper.PollInterval, Run: zones.Refresh}) } - if cfg.MeshMapper.Channels.Enabled { + if channelImporter != nil && cfg.MeshMapper.Channels.Enabled { tasks = append(tasks, background.Task{Name: "meshmapper.channels", Interval: meshmapper.PollInterval, Run: channelImporter.Refresh}) } - if cfg.MeshMapper.Repeaters.Enabled { + if repeaterImporter != nil && cfg.MeshMapper.Repeaters.Enabled { tasks = append(tasks, background.Task{Name: "meshmapper.repeaters", Interval: meshmapper.PollInterval, Run: repeaterImporter.Refresh}) } + pgSource, pgTask := planguardWiring(ctx, resolved.PlanguardDailyCheck, mode.Jobs, pools.Maint) + if pgTask != nil { + tasks = append(tasks, *pgTask) + } profiles := configureProfiling(ctx, pools.Maint) defer profiles.Stop() for i := range tasks { tasks[i].Run = profiles.WrapTask(tasks[i].Name, tasks[i].Run) } - scheduler := background.New(tasks) - go scheduler.Start(ctx) + if mode.Jobs { + scheduler := background.New(tasks) + go scheduler.Start(ctx) + } // ── HTTP server ────────────────────────────────────────────────────────── // Wrap after wiring cache invalidators and cleanup callbacks to the actual @@ -474,7 +512,7 @@ func main() { AdminRoutes: map[string]http.Handler{ "/accounts": handlers.AccountsRouter(maintStore), "/backup": handlers.BackupRouter(backupOpts, ctx), - "/db": handlers.DBAdminRouter(maintStore), + "/db": handlers.DBAdminRouter(maintStore, pgSource), }, Guards: guard, APIPlanLayout: string(layout), @@ -520,7 +558,9 @@ func main() { } } ingestWorkers.Wait() - coalescer.Flush(shutdownCtx) + if mode.Coalescer { + coalescer.Flush(shutdownCtx) + } } // proxyLimitWarning explains why per-IP limits go site-wide behind an unlisted reverse proxy. @@ -545,3 +585,60 @@ func getEnv(key string) string { } return v } + +// planguardWiring builds the daily read-only plan check on the maintenance pool, behind planguard.daily_check. +// ctx is the server's root context: its end cancels any run in flight. +// Off, /planguard answers that the check is disabled. On, the check runs with the other jobs (config validation +// refuses it on a read-only server); a corpus that does not load disables it but never stops the server, and the +// source stays a nil interface, so /planguard is 404. +func planguardWiring(ctx context.Context, enabled, jobs bool, maint *pgxpool.Pool) (handlers.PlanguardSource, *background.Task) { + if !enabled { + return disabledPlanguard{}, nil + } + // On, the last-run timestamp is exported from boot (0 until a run succeeds), so a check that never succeeds, + // even one whose corpus does not load, still trips the staleness alert. + metrics.EnablePlanguard() + // Defensive: configuration refuses planguard.daily_check together with server.read_only, the only mode + // without jobs, so this branch is not reached today. It keeps a jobless server from ever running the check. + if !jobs { + return nil, nil + } + checker, err := planguard.NewProdChecker(maint, db.SQLCSources, pgdata.Files) + if err != nil { + slog.Error("plan check disabled", "component", "startup", "error", err) + return nil, nil + } + pg := background.NewPlanguard(ctx, checker.Run) + task := pg.Task(24 * time.Hour) + return planguardSource{pg}, &task +} + +// planguardSource serves the plan check's reports in their API shape. +type planguardSource struct{ pg *background.Planguard } + +func (s planguardSource) Last() (api.PlanguardReport, error) { + rep, ok := s.pg.Last() + if !ok { + return api.PlanguardReport{}, api.ErrPlanguardNotRun + } + return rep.API(), nil +} + +func (s planguardSource) RunNow(ctx context.Context) (api.PlanguardReport, error) { + rep, err := s.pg.RunNow(ctx) + if err != nil { + return api.PlanguardReport{}, err + } + return rep.API(), nil +} + +// disabledPlanguard answers /planguard when planguard.daily_check is off. +type disabledPlanguard struct{} + +func (disabledPlanguard) Last() (api.PlanguardReport, error) { + return api.PlanguardReport{}, api.ErrPlanguardDisabled +} + +func (disabledPlanguard) RunNow(context.Context) (api.PlanguardReport, error) { + return api.PlanguardReport{}, api.ErrPlanguardDisabled +} diff --git a/cmd/beacon/planguard_test.go b/cmd/beacon/planguard_test.go new file mode 100644 index 0000000..77d39e5 --- /dev/null +++ b/cmd/beacon/planguard_test.go @@ -0,0 +1,76 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "context" + "encoding/json" + "net/http/httptest" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/api/handlers" + "github.com/MeshCore-Beacon/beacon-server/internal/config" + "github.com/MeshCore-Beacon/beacon-server/internal/metrics" + "github.com/prometheus/client_golang/prometheus/testutil" +) + +// lastRunSeries counts the beacon_planguard_last_run_timestamp_seconds series the registry exports. +func lastRunSeries(t *testing.T) int { + t.Helper() + n, err := testutil.GatherAndCount(metrics.Registry, "beacon_planguard_last_run_timestamp_seconds") + if err != nil { + t.Fatal(err) + } + return n +} + +func TestPlanguardWiring(t *testing.T) { + get := func(t *testing.T, src handlers.PlanguardSource, target string) (int, string) { + t.Helper() + w := httptest.NewRecorder() + handlers.DBAdminRouter(nil, src).ServeHTTP(w, httptest.NewRequest("GET", target, nil)) + var body map[string]handlers.APIError + _ = json.Unmarshal(w.Body.Bytes(), &body) + return w.Code, body["error"].Message + } + + // The default config leaves the check off: no task, and /planguard says so, fresh or not. + if config.Resolve(&config.Config{}).PlanguardDailyCheck { + t.Fatal("planguard.daily_check is on by default") + } + src, task := planguardWiring(context.Background(), false, true, nil) + if task != nil { + t.Errorf("disabled check scheduled a task %q", task.Name) + } + for _, target := range []string{"/planguard", "/planguard?fresh=true"} { + if code, msg := get(t, src, target); code != 404 || msg != "planguard daily check is disabled" { + t.Errorf("disabled %s: %d %q", target, code, msg) + } + } + + if n := lastRunSeries(t); n != 0 { + t.Errorf("disabled check exports %d last-run series, want 0", n) + } + + // On: a daily task on the hourly tick, and 404 "has not run yet" until its first run. + src, task = planguardWiring(context.Background(), true, true, nil) + if task == nil || task.Name != "planguard" || task.Interval.Hours() != 1 { + t.Fatalf("enabled check: task %+v", task) + } + if code, msg := get(t, src, "/planguard"); code != 404 || msg != "planguard has not run yet" { + t.Errorf("enabled before the first run: %d %q", code, msg) + } + if n := lastRunSeries(t); n != 1 { + t.Errorf("enabled check exports %d last-run series before its first run, want 1 (at 0)", n) + } + + // On without jobs: no task and no route. + src, task = planguardWiring(context.Background(), true, false, nil) + if task != nil || src != nil { + t.Errorf("enabled without jobs: task %v source %v", task, src) + } + if code, _ := get(t, src, "/planguard"); code != 404 { + t.Errorf("enabled without jobs: %d", code) + } +} diff --git a/cmd/beacon/readonly.go b/cmd/beacon/readonly.go new file mode 100644 index 0000000..593a12a --- /dev/null +++ b/cmd/beacon/readonly.go @@ -0,0 +1,39 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import "github.com/jackc/pgx/v5/pgxpool" + +// readOnlyWarning is logged once at boot when server.read_only is set. +const readOnlyWarning = "server.read_only: no migrations, ingest, background jobs or MeshMapper imports; every database session is read-only" + +// bootMode says which boot steps that write may run. Each field gates one group of call sites in main. +type bootMode struct { + // Migrate runs the schema migrations. + Migrate bool + // Seed writes the configured IATAs, regions and scopes. + Seed bool + // Backfill decrypts stored channel messages at boot and for newly imported keys. + Backfill bool + // Ingest starts the MQTT ingest workers. + Ingest bool + // Coalescer runs and flushes the presence write coalescer. + Coalescer bool + // Jobs runs the rollup catch-up and the background scheduler. + Jobs bool + // MeshMapper restores and runs the MeshMapper importers; their restores can prune or clear rows. + MeshMapper bool +} + +// bootModeFor returns every step for a normal server and none for a read-only one. +func bootModeFor(readOnly bool) bootMode { + on := !readOnly + return bootMode{Migrate: on, Seed: on, Backfill: on, Ingest: on, Coalescer: on, Jobs: on, MeshMapper: on} +} + +// readOnlyTune makes every session of a pool read-only, so a write fails with SQLSTATE 25006 +// instead of changing the database. It is passed to db.OpenPools when server.read_only is set. +func readOnlyTune(_ string, cfg *pgxpool.Config) { + cfg.ConnConfig.RuntimeParams["default_transaction_read_only"] = "on" +} diff --git a/cmd/beacon/readonly_integration_test.go b/cmd/beacon/readonly_integration_test.go new file mode 100644 index 0000000..e9875a8 --- /dev/null +++ b/cmd/beacon/readonly_integration_test.go @@ -0,0 +1,411 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +//go:build unix + +package main + +import ( + "bytes" + "context" + "crypto/rand" + "fmt" + "io" + "maps" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "slices" + "strings" + "sync" + "syscall" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/db" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +const readOnlyBootAPIKey = "read-only-boot-test-key" + +// readOnlyBootConfig makes a normal boot write: Seed upserts the IATA, region and hashtag +// channel, the disabled MeshMapper imports clear their rows, and the jobs run at once. +const readOnlyBootConfig = `server: + read_only: %t +ratelimit: + enabled: false +auth: + api_key: ` + readOnlyBootAPIKey + ` +iatas: + SEA: + name: Seattle + lat: 47.45 + lng: -122.31 +regions: + - name: Pacific Northwest + slug: pnw + iatas: [SEA] +channel_keys: + hashtags: + - readonlytest +` + +// readOnlyRequests is the request mix a read-only server serves. GETs must not fail with a +// server error; writes must fail and never succeed. +var readOnlyRequests = []struct { + method, path, body string + write bool +}{ + {"GET", "/api/v1/info", "", false}, + {"GET", "/api/v1/iatas", "", false}, + {"GET", "/api/v1/iatas/SEA", "", false}, + {"GET", "/api/v1/regions", "", false}, + {"GET", "/api/v1/nodes?limit=5", "", false}, + {"GET", "/api/v1/observers?limit=5", "", false}, + {"GET", "/api/v1/observers/directory", "", false}, + {"GET", "/api/v1/packets?limit=5", "", false}, + {"GET", "/api/v1/routes?limit=5", "", false}, + {"GET", "/api/v1/routes?iata=SEA&limit=5", "", false}, + {"GET", "/api/v1/channels", "", false}, + {"GET", "/api/v1/messages?limit=5", "", false}, + {"GET", "/api/v1/scopes", "", false}, + {"GET", "/api/v1/brokers", "", false}, + {"GET", "/api/v1/traces", "", false}, + {"GET", "/api/v1/stats/overview", "", false}, + {"GET", "/api/v1/stats/node-types", "", false}, + {"GET", "/api/v1/stats/top-nodes", "", false}, + {"GET", "/api/v1/admin/accounts", "", false}, + {"POST", "/api/v1/admin/accounts", `{"name":"read-only boot"}`, true}, +} + +// tableStat is a table's live row count and its insert, update and delete counters. +type tableStat struct{ rows, ins, upd, del int64 } + +// A read-only server boots against a migrated database, serves a request mix and shuts down +// without changing any table: no row count and no insert, update or delete counter moves. +// The same boot without read_only does move them, so the check can see a write. +func TestReadOnlyBootWritesNothingPostgres(t *testing.T) { + if os.Getenv("BEACON_TEST_BOOT_MAIN") == "1" { + // The parent holds stdin open; when it dies, even by SIGKILL, stdin closes and the server goes with it. + go func() { + _, _ = io.Copy(io.Discard, os.Stdin) + os.Exit(3) + }() + main() + return + } + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN") + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + admin, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + name := "beacon_read_only_" + strings.ToLower(rand.Text()) + ident := pgx.Identifier{name}.Sanitize() + _, err = admin.Exec(ctx, "CREATE DATABASE "+ident) + admin.Close(context.Background()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if admin, err := pgx.Connect(ctx, dsn); err == nil { + _, _ = admin.Exec(ctx, "DROP DATABASE "+ident+" WITH (FORCE)") + admin.Close(ctx) + } + }) + cfg, err := pgxpool.ParseConfig(dsn) + if err != nil { + t.Fatal(err) + } + cfg.ConnConfig.Database = name + pool, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + t.Fatal(err) + } + if err := db.RunMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + for _, q := range []string{ + "INSERT INTO iata_codes(iata) VALUES ('YVR')", + "INSERT INTO observers(id, public_key) VALUES ('00000000-0000-0000-0000-000000000001', '\\x01')", + "INSERT INTO nodes(id, public_key, name, node_type) VALUES ('00000000-0000-0000-0000-0000000000a1', '\\xaa11', 'Alice', 2)", + } { + if _, err := pool.Exec(ctx, q); err != nil { + t.Fatalf("%s: %v", q, err) + } + } + pool.Close() + statsConn := func() *pgx.Conn { + c, err := pgx.ConnectConfig(ctx, cfg.ConnConfig.Copy()) + if err != nil { + t.Fatal(err) + } + return c + } + + // The control boot writes; it also leaves the database as a server would after a normal start. + before := snapshotTables(t, ctx, statsConn(), name) + bootAndServe(t, ctx, poolDSN(t, dsn, name), false) + after := snapshotTables(t, ctx, statsConn(), name) + if maps.Equal(before, after) { + t.Fatal("a normal boot changed no table, so the check cannot see a write") + } + + before = after + log := bootAndServe(t, ctx, poolDSN(t, dsn, name), true) + after = snapshotTables(t, ctx, statsConn(), name) + for _, table := range slices.Sorted(maps.Keys(after)) { + if before[table] != after[table] { + t.Errorf("%s changed under read_only: before %+v, after %+v", table, before[table], after[table]) + } + } + if len(before) != len(after) { + t.Errorf("table set changed under read_only: %d tables before, %d after", len(before), len(after)) + } + if !strings.Contains(log, readOnlyWarning) { + t.Errorf("boot log lacks the read-only warning:\n%s", log) + } + // A gated step that still ran would have hit a read-only session and logged it. The only refused write is + // the request mix's admin POST. + refused := 0 + for _, line := range strings.Split(log, "\n") { + if !strings.Contains(line, "25006") && !strings.Contains(line, "read-only transaction") { + continue + } + if strings.Contains(line, `msg="account request failed"`) { + refused++ + continue + } + t.Errorf("read-only server refused a write outside the request mix: %s", line) + } + if refused != 1 { + t.Errorf("the admin POST logged %d refused writes, want 1:\n%s", refused, log) + } +} + +// poolDSN points dsn at the named database. +func poolDSN(t *testing.T, dsn, name string) string { + t.Helper() + cfg, err := pgx.ParseConfig(dsn) + if err != nil { + t.Fatal(err) + } + return fmt.Sprintf("postgres://%s@%s/%s?sslmode=disable", userInfo(cfg), net.JoinHostPort(cfg.Host, fmt.Sprint(cfg.Port)), name) +} + +func userInfo(cfg *pgx.ConnConfig) string { + if cfg.Password == "" { + return cfg.User + } + return cfg.User + ":" + cfg.Password +} + +// bootAndServe runs the server in a child process, sends the request mix once it is ready, +// stops it with SIGTERM and returns its log. +func bootAndServe(t *testing.T, ctx context.Context, dsn string, readOnly bool) string { + t.Helper() + dir := t.TempDir() + path := filepath.Join(dir, "config.yaml") + if err := os.WriteFile(path, []byte(fmt.Sprintf(readOnlyBootConfig, readOnly)), 0600); err != nil { + t.Fatal(err) + } + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + addr := l.Addr().String() + l.Close() + + cmd := exec.Command(os.Args[0], "-test.run=^TestReadOnlyBootWritesNothingPostgres$") + cmd.Dir = dir + cmd.Env = append(os.Environ(), "BEACON_TEST_BOOT_MAIN=1", "CONFIG_PATH="+path, "LISTEN_ADDR="+addr, + "POSTGRES_DSN="+dsn, "BEACON_API_KEY="+readOnlyBootAPIKey, "MESHMAPPER_API_KEY=", "REDIS_ADDR=", + "LOG_FORMAT=text", "LOG_LEVEL=info") + var out lockedBuffer + cmd.Stdout, cmd.Stderr = &out, &out + // Its own process group, so cleanup reaches anything it started. + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + stdin, err := cmd.StdinPipe() + if err != nil { + t.Fatal(err) + } + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { done <- cmd.Wait() }() + stopped := false + defer func() { + if !stopped { + _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + <-done + } + stdin.Close() + }() + + base := "http://" + addr + client := &http.Client{Timeout: 10 * time.Second} + ready := time.Now().Add(60 * time.Second) + for { + resp, err := client.Get(base + "/api/v1/info") + if err == nil { + resp.Body.Close() + if resp.StatusCode == http.StatusOK { + break + } + } + select { + case err := <-done: + stopped = true + t.Fatalf("server exited before it was ready (read_only=%t): %v\n%s", readOnly, err, out.String()) + default: + } + if time.Now().After(ready) { + t.Fatalf("server not ready (read_only=%t):\n%s", readOnly, out.String()) + } + time.Sleep(100 * time.Millisecond) + } + + if readOnly { + for _, r := range readOnlyRequests { + req, err := http.NewRequestWithContext(ctx, r.method, base+r.path, strings.NewReader(r.body)) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Authorization", "Bearer "+readOnlyBootAPIKey) + if r.body != "" { + req.Header.Set("Content-Type", "application/json") + } + resp, err := client.Do(req) + if err != nil { + t.Fatalf("%s %s: %v", r.method, r.path, err) + } + body, _ := io.ReadAll(resp.Body) + resp.Body.Close() + switch { + case r.write && resp.StatusCode < 300: + t.Errorf("%s %s succeeded under read_only: %d %s", r.method, r.path, resp.StatusCode, body) + case !r.write && resp.StatusCode >= 500: + t.Errorf("%s %s: %d %s", r.method, r.path, resp.StatusCode, body) + } + } + } else { + // Give the boot-time jobs a moment to write before the control stops. + time.Sleep(2 * time.Second) + } + + if err := cmd.Process.Signal(syscall.SIGTERM); err != nil { + t.Fatal(err) + } + select { + case err := <-done: + stopped = true + if err != nil { + t.Fatalf("server exit (read_only=%t): %v\n%s", readOnly, err, out.String()) + } + case <-time.After(60 * time.Second): + t.Fatalf("server did not stop (read_only=%t):\n%s", readOnly, out.String()) + } + return out.String() +} + +// snapshotTables reads every user table's row count and tuple counters once the server's +// sessions have ended; a backend flushes its statistics as it exits. +func snapshotTables(t *testing.T, ctx context.Context, conn *pgx.Conn, database string) map[string]tableStat { + t.Helper() + defer conn.Close(context.Background()) + waitSessionsGone(t, ctx, conn, database) + // Statistics arrive as backends flush them; read until two snapshots in a row agree. + prev := readTables(t, ctx, conn) + for deadline := time.Now().Add(30 * time.Second); ; { + time.Sleep(300 * time.Millisecond) + cur := readTables(t, ctx, conn) + if maps.Equal(prev, cur) { + return cur + } + if time.Now().After(deadline) { + t.Fatal("table statistics did not settle") + } + prev = cur + } +} + +func waitSessionsGone(t *testing.T, ctx context.Context, conn *pgx.Conn, database string) { + t.Helper() + deadline := time.Now().Add(30 * time.Second) + for { + var n int + if err := conn.QueryRow(ctx, "SELECT count(*) FROM pg_stat_activity WHERE datname = $1 AND application_name LIKE 'beacon-%'", database).Scan(&n); err != nil { + t.Fatal(err) + } + if n == 0 { + break + } + if time.Now().After(deadline) { + t.Fatalf("%d server sessions still open", n) + } + time.Sleep(100 * time.Millisecond) + } +} + +// readTables reads every user table's row count and tuple counters. +func readTables(t *testing.T, ctx context.Context, conn *pgx.Conn) map[string]tableStat { + t.Helper() + if _, err := conn.Exec(ctx, "SELECT pg_stat_clear_snapshot()"); err != nil { + t.Fatal(err) + } + rows, err := conn.Query(ctx, "SELECT schemaname, relname, n_tup_ins, n_tup_upd, n_tup_del FROM pg_stat_user_tables") + if err != nil { + t.Fatal(err) + } + stats := map[string]tableStat{} + for rows.Next() { + var schema, rel string + var s tableStat + if err := rows.Scan(&schema, &rel, &s.ins, &s.upd, &s.del); err != nil { + t.Fatal(err) + } + stats[pgx.Identifier{schema, rel}.Sanitize()] = s + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + if len(stats) == 0 { + t.Fatal("no user tables") + } + for table, s := range stats { + if err := conn.QueryRow(ctx, "SELECT count(*) FROM "+table).Scan(&s.rows); err != nil { + t.Fatal(err) + } + stats[table] = s + } + return stats +} + +// lockedBuffer collects a child's output from two pipes. +type lockedBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (b *lockedBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Write(p) +} + +func (b *lockedBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.String() +} diff --git a/cmd/beacon/readonly_test.go b/cmd/beacon/readonly_test.go new file mode 100644 index 0000000..7654556 --- /dev/null +++ b/cmd/beacon/readonly_test.go @@ -0,0 +1,31 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "testing" + + "github.com/jackc/pgx/v5/pgxpool" +) + +func TestBootModeReadOnly(t *testing.T) { + if got, want := bootModeFor(false), (bootMode{true, true, true, true, true, true, true}); got != want { + t.Errorf("bootModeFor(false) = %+v, want %+v", got, want) + } + if got := bootModeFor(true); got != (bootMode{}) { + t.Errorf("bootModeFor(true) = %+v, want every step off", got) + } + + cfg, err := pgxpool.ParseConfig("postgres://beacon@127.0.0.1:5432/beacon?application_name=beacon-maint") + if err != nil { + t.Fatal(err) + } + readOnlyTune("maint", cfg) + if got := cfg.ConnConfig.RuntimeParams["default_transaction_read_only"]; got != "on" { + t.Errorf("default_transaction_read_only = %q, want on", got) + } + if got := cfg.ConnConfig.RuntimeParams["application_name"]; got != "beacon-maint" { + t.Errorf("application_name = %q, want it kept", got) + } +} diff --git a/cmd/planguard/main.go b/cmd/planguard/main.go new file mode 100644 index 0000000..c31fac2 --- /dev/null +++ b/cmd/planguard/main.go @@ -0,0 +1,613 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Command planguard checks that the planner keeps choosing the plans Beacon's statements were designed for. +// +// planguard run --tier fixture|local|prod [--corpus db/planguard/corpus.yaml] [--dsn $PLANGUARD_DSN] +// [--out planguard-out] [--only RE] [--skip RE] [--modes m,..] [--include-pending] [--search-path P] +// [--baseline check|write|off] [--emit resolvers|plans --params FILE] [--plans FILE --params FILE] +// [--fidelity-against results.jsonl] +// planguard gen-policy [--corpus FILE] [--out FILE] (defaults and db/sqlc are under the module root) +// planguard diff --cases db/planguard/diff/NAME.yaml [--old-ref origin/dev] [--walk 1,2,3,web] [--dsn $PLANGUARD_DSN] +// [--out planguard-out] [--files db/planguard] +// planguard storediff --old URL --new URL --cases db/planguard/storediff/NAME.yaml [--walk 1,2,3,web] [--out DIR] +// +// diff runs each batch of a case file in one READ ONLY snapshot and compares every pair's full results. old: +// statements are read from db/sqlc at --old-ref through git (the checkout is never switched), new: from the working +// tree's db/sqlc, file: from --files. +// +// storediff walks each case of a case file page by page on two running servers (normally both read-only on one +// database) and compares the whole JSON of every page. +// +// --tier fixture first builds the synthetic fixture on --dsn through the migrations (or finds it built from the same +// files); it refuses a database that holds data and was not built that way, and one whose collation is not C +// (create it with CREATE DATABASE TEMPLATE template0 LC_COLLATE 'C' LC_CTYPE 'C' ENCODING 'UTF8'). +// +// Exit status: 0 when no run fails, 1 when one does (or the run cannot be made), 2 on usage errors. +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + "net/http" + "net/url" + "os" + "os/signal" + "path/filepath" + "regexp" + "strconv" + "strings" + "syscall" + "time" + + "github.com/MeshCore-Beacon/beacon-server/db/planpolicy" + "github.com/MeshCore-Beacon/beacon-server/internal/planguard" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" +) + +const usage = "usage: planguard run --tier fixture|local|prod [flags] (planguard run -h for the flags)\n" + + " planguard gen-policy [--corpus FILE] [--out FILE] (paths default to the module root's db/)\n" + + " planguard diff --cases FILE [--old-ref REF] [--walk 1,2,3,web] [--dsn DSN] [--out DIR] [--files DIR]\n" + + " planguard storediff --old URL --new URL --cases FILE [--walk 1,2,3,web] [--out DIR]" + +func main() { + os.Exit(run(os.Args[1:], os.Stdout, os.Stderr)) +} + +func run(args []string, stdout, stderr io.Writer) int { + if len(args) == 0 { + fmt.Fprintln(stderr, usage) + return 2 + } + switch args[0] { + case "run": + return runCmd(args[1:], stdout, stderr) + case "gen-policy": + return genPolicyCmd(args[1:], stdout, stderr) + case "diff": + return diffCmd(args[1:], stderr) + case "storediff": + return storeDiffCmd(args[1:], stderr) + case "-h", "-help", "--help", "help": + fmt.Fprintln(stderr, usage) + return 2 + } + fmt.Fprintf(stderr, "planguard: unknown subcommand %q\n%s\n", args[0], usage) + return 2 +} + +// genPolicyCmd writes db/planpolicy/cached_gen.go from the corpus: the cached statements whose entries prove +// every shape and mode of their admission profile. Statement texts come from db/sqlc of the module that holds the +// working directory (found by walking up to go.mod), so it runs from any directory of the checkout. +func genPolicyCmd(args []string, stdout, stderr io.Writer) int { + fs := flag.NewFlagSet("gen-policy", flag.ContinueOnError) + fs.SetOutput(stderr) + corpus := fs.String("corpus", "", "corpus file (default /db/planguard/corpus.yaml)") + out := fs.String("out", "", "generated Go file (default /db/planpolicy/cached_gen.go)") + if err := fs.Parse(args); err != nil { + return 2 + } + if fs.NArg() > 0 { + fmt.Fprintf(stderr, "planguard gen-policy: unexpected argument %q\n", fs.Arg(0)) + return 2 + } + root, err := moduleRoot() + if err != nil { + fmt.Fprintln(stderr, "planguard gen-policy:", err) + return 1 + } + if *corpus == "" { + *corpus = filepath.Join(root, "db", "planguard", "corpus.yaml") + } + if *out == "" { + *out = filepath.Join(root, "db", "planpolicy", "cached_gen.go") + } + sqlc, err := planguard.LoadStatements(filepath.Join(root, "db", "sqlc")) + if err != nil { + fmt.Fprintln(stderr, "planguard:", err) + return 1 + } + c, err := planguard.LoadCorpus(os.DirFS(filepath.Dir(*corpus)), filepath.Base(*corpus), sqlc) + if err != nil { + fmt.Fprintln(stderr, "planguard:", err) + return 1 + } + src, err := planguard.GenPolicy(c, planpolicy.Legacy) + if err != nil { + fmt.Fprintln(stderr, "planguard gen-policy:", err) + return 1 + } + if old, err := os.ReadFile(*out); err == nil && string(old) == string(src) { + fmt.Fprintf(stdout, "%s is up to date\n", *out) + return 0 + } + if err := os.WriteFile(*out, src, 0o644); err != nil { + fmt.Fprintln(stderr, "planguard:", err) + return 1 + } + fmt.Fprintf(stdout, "wrote %s\n", *out) + return 0 +} + +// diffCmd compares old and new statement results on one snapshot per batch. +func diffCmd(args []string, stderr io.Writer) int { + fs := flag.NewFlagSet("diff", flag.ContinueOnError) + fs.SetOutput(stderr) + cases := fs.String("cases", "", "diff case file, e.g. db/planguard/diff/planted_readmodels.yaml (required)") + oldRef := fs.String("old-ref", "", "git ref whose db/sqlc gives the old: statements (required when the cases use old:)") + walk := fs.String("walk", "", "comma list of page sizes to walk the pairs that have a walk at; web = each walk's web_limit") + dsn := fs.String("dsn", os.Getenv("PLANGUARD_DSN"), "PostgreSQL DSN (default $PLANGUARD_DSN)") + out := fs.String("out", "planguard-out", "output directory") + files := fs.String("files", "", "directory file: statements are relative to (default /db/planguard)") + if err := fs.Parse(args); err != nil { + return 2 + } + usageErr := func(format string, a ...any) int { + fmt.Fprintf(stderr, "planguard diff: "+format+"\n", a...) + return 2 + } + if fs.NArg() > 0 { + return usageErr("unexpected argument %q", fs.Arg(0)) + } + if *cases == "" { + return usageErr("--cases is required") + } + limits, err := parseWalk(*walk) + if err != nil { + return usageErr("--walk: %v", err) + } + if *dsn == "" { + return usageErr("--dsn (or $PLANGUARD_DSN) is required") + } + root, err := moduleRoot() + if err != nil { + fmt.Fprintln(stderr, "planguard diff:", err) + return 1 + } + if *files == "" { + *files = filepath.Join(root, "db", "planguard") + } + raw, err := os.ReadFile(*cases) + if err != nil { + fmt.Fprintln(stderr, "planguard diff:", err) + return 1 + } + fsys := os.DirFS(*files) + spec, err := planguard.ParseDiffSpec(raw, *cases, fsys) + if err != nil { + fmt.Fprintln(stderr, "planguard diff:", err) + return 1 + } + needOld := false + for _, b := range spec.Batches { + for _, src := range b.Statements { + needOld = needOld || strings.HasPrefix(src, "old:") + } + } + var oldSQL map[string]string + if needOld { + if *oldRef == "" { + return usageErr("--old-ref is required: %s uses old: statements", *cases) + } + if oldSQL, err = planguard.LoadStatementsAtRef(root, *oldRef); err != nil { + fmt.Fprintln(stderr, "planguard diff:", err) + return 1 + } + } + newSQL, err := planguard.LoadStatements(filepath.Join(root, "db", "sqlc")) + if err != nil { + fmt.Fprintln(stderr, "planguard diff:", err) + return 1 + } + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + pc, err := pgconn.Connect(ctx, *dsn) + if err != nil { + fmt.Fprintln(stderr, "planguard diff: connect:", err) + return 1 + } + defer pc.Close(context.Background()) + rep, runErr := planguard.RunDiff(ctx, planguard.NewSession(pc), spec, oldSQL, newSQL, fsys, *out, limits) + code := 0 + if rep != nil { + for _, b := range rep.Batches { + for _, p := range b.Pairs { + fmt.Fprintln(stderr, p.Line()) + } + } + fmt.Fprintf(stderr, "report: %s\n", filepath.Join(*out, "diff_report.md")) + if n := len(rep.Failed()); n > 0 { + fmt.Fprintf(stderr, "%d pairs missed their expectation\n", n) + code = 1 + } + if n := len(rep.Skipped()); n > 0 { + fmt.Fprintf(stderr, "%d pairs were skipped: their {sql: …} params found no value on this snapshot, so they prove nothing\n", n) + code = 1 + } + } + if runErr != nil { + fmt.Fprintln(stderr, "planguard diff:", runErr) + code = 1 + } + return code +} + +// storeDiffCmd compares the pages of two servers' API. +func storeDiffCmd(args []string, stderr io.Writer) int { + fs := flag.NewFlagSet("storediff", flag.ContinueOnError) + fs.SetOutput(stderr) + oldBase := fs.String("old", "", "base URL of the old server, e.g. http://127.0.0.1:8481 (required)") + newBase := fs.String("new", "", "base URL of the new server, e.g. http://127.0.0.1:8482 (required)") + cases := fs.String("cases", "", "storediff case file, e.g. db/planguard/storediff/smoke.yaml (required)") + walk := fs.String("walk", "web", "comma list of page sizes; web = each case's web_limit") + out := fs.String("out", "", "directory for storediff_report.md (default: no file)") + if err := fs.Parse(args); err != nil { + return 2 + } + usageErr := func(format string, a ...any) int { + fmt.Fprintf(stderr, "planguard storediff: "+format+"\n", a...) + return 2 + } + if fs.NArg() > 0 { + return usageErr("unexpected argument %q", fs.Arg(0)) + } + if *oldBase == "" || *newBase == "" || *cases == "" { + return usageErr("--old, --new and --cases are required") + } + for _, base := range []string{*oldBase, *newBase} { + if u, err := url.Parse(base); err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { + return usageErr("%q is not an http(s) base URL", base) + } + } + if _, err := parseWalk(*walk); err != nil { + return usageErr("--walk: %v", err) + } + spec, err := planguard.LoadStoreDiffSpec(*cases) + if err != nil { + fmt.Fprintln(stderr, "planguard storediff:", err) + return 1 + } + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + hc := &http.Client{Timeout: 2 * time.Minute} + rep, runErr := planguard.RunStoreDiff(ctx, hc, *oldBase, *newBase, spec, strings.Split(*walk, ",")) + code := 0 + if rep != nil { + for _, c := range rep.Cases { + fmt.Fprintln(stderr, c.Line()) + } + if *out != "" { + path := filepath.Join(*out, "storediff_report.md") + if err := os.MkdirAll(*out, 0o755); err != nil { + fmt.Fprintln(stderr, "planguard storediff:", err) + code = 1 + } else if err := os.WriteFile(path, []byte(rep.Markdown()), 0o644); err != nil { + fmt.Fprintln(stderr, "planguard storediff:", err) + code = 1 + } else { + fmt.Fprintf(stderr, "report: %s\n", path) + } + } + if n := len(rep.Failed()); n > 0 { + fmt.Fprintf(stderr, "%d results differ\n", n) + code = 1 + } + if n := len(rep.Skipped()); n > 0 { + fmt.Fprintf(stderr, "%d results were skipped: a path var found no value, so they prove nothing\n", n) + code = 1 + } + } + if runErr != nil { + fmt.Fprintln(stderr, "planguard storediff:", runErr) + code = 1 + } + return code +} + +// parseWalk reads "1,2,3,web" (web → 0, each walk's web_limit). +func parseWalk(s string) ([]int, error) { + if s == "" { + return nil, nil + } + var out []int + for _, part := range strings.Split(s, ",") { + part = strings.TrimSpace(part) + if part == "web" { + out = append(out, 0) + continue + } + n, err := strconv.Atoi(part) + if err != nil || n < 1 { + return nil, fmt.Errorf("%q is not a page size ≥ 1 or web", part) + } + out = append(out, n) + } + return out, nil +} + +// moduleRoot returns the nearest directory at or above the working directory that holds go.mod. +func moduleRoot() (string, error) { + dir, err := os.Getwd() + if err != nil { + return "", err + } + for { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + return dir, nil + } + parent := filepath.Dir(dir) + if parent == dir { + return "", errors.New("no go.mod at or above the working directory; run it inside the beacon-server checkout") + } + dir = parent + } +} + +type runFlags struct { + tier, corpus, dsn, out, only, skip, modes, searchPath, baseline, emit, params, plans, fidelity string + includePending bool +} + +func usageErr(stderr io.Writer, format string, a ...any) int { + fmt.Fprintf(stderr, "planguard run: "+format+"\n", a...) + return 2 +} + +func runCmd(args []string, stdout, stderr io.Writer) int { + var f runFlags + fs := flag.NewFlagSet("run", flag.ContinueOnError) + fs.SetOutput(stderr) + fs.StringVar(&f.tier, "tier", "", "fixture | local | prod (required)") + fs.StringVar(&f.corpus, "corpus", "db/planguard/corpus.yaml", "corpus file; baselines live in its baseline/ directory") + fs.StringVar(&f.dsn, "dsn", os.Getenv("PLANGUARD_DSN"), "PostgreSQL DSN (default $PLANGUARD_DSN)") + fs.StringVar(&f.out, "out", "planguard-out", "output directory") + fs.StringVar(&f.only, "only", "", "regexp: run only entries whose id matches") + fs.StringVar(&f.skip, "skip", "", "regexp: leave out entries whose id matches") + fs.StringVar(&f.modes, "modes", "", "comma list of plan modes overriding every entry's") + fs.BoolVar(&f.includePending, "include-pending", false, "run pending statements too (local overlays only)") + fs.StringVar(&f.searchPath, "search-path", "", "override every group's search_path") + fs.StringVar(&f.baseline, "baseline", "", "check | write | off (default check; prod is always off)") + fs.StringVar(&f.emit, "emit", "", "resolvers | plans: write a read-only psql script to stdout instead of running") + fs.StringVar(&f.params, "params", "", "psql output of the --emit resolvers script") + fs.StringVar(&f.plans, "plans", "", "psql output of the --emit plans script: check it instead of running") + fs.StringVar(&f.fidelity, "fidelity-against", "", "results..jsonl of another run to compare drivers with") + if err := fs.Parse(args); err != nil { + return 2 + } + if fs.NArg() > 0 { + return usageErr(stderr, "unexpected argument %q", fs.Arg(0)) + } + o, code := f.options(stderr) + if code != 0 { + return code + } + + sqlc, err := planguard.LoadStatements("db/sqlc") + if err != nil { + fmt.Fprintln(stderr, "planguard:", err) + return 1 + } + c, err := planguard.LoadCorpus(os.DirFS(filepath.Dir(f.corpus)), filepath.Base(f.corpus), sqlc) + if err != nil { + fmt.Fprintln(stderr, "planguard:", err) + return 1 + } + if f.params != "" { + if o.Params, err = readResolverOutput(f.params); err != nil { + fmt.Fprintln(stderr, "planguard:", err) + return 1 + } + } + + switch { + case f.emit == "resolvers": + return emit(stderr, func(nonce string) error { return planguard.EmitResolvers(stdout, c, o, nonce) }) + case f.emit == "plans": + return emit(stderr, func(nonce string) error { return planguard.EmitPlans(stdout, c, o, o.Params, nonce) }) + case f.plans != "": + raw, err := os.ReadFile(f.plans) + if err == nil { + var nonce string + if nonce, err = planguard.ScriptNonce(raw); err == nil { + var plans map[string][]byte + if plans, err = planguard.ParsePlanOutput(strings.NewReader(string(raw)), nonce); err == nil { + var rep *planguard.Report + rep, err = planguard.CheckPlans(c, o, plans) + if err == nil { + for _, r := range rep.Results { + fmt.Fprintln(stderr, r.Line()) + } + return finish(rep, nil, f, stderr) + } + } + } + } + fmt.Fprintf(stderr, "planguard: %s: %v\n", f.plans, err) + return 1 + } + + if f.dsn == "" { + return usageErr(stderr, "--dsn (or $PLANGUARD_DSN) is required to run") + } + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + if f.tier == "fixture" { + if err := buildFixture(ctx, f, stderr); err != nil { + fmt.Fprintln(stderr, "planguard:", err) + return 1 + } + } + pc, err := pgconn.Connect(ctx, f.dsn) + if err != nil { + fmt.Fprintln(stderr, "planguard: connect:", err) + return 1 + } + defer pc.Close(context.Background()) + o.Progress = func(r planguard.Result) { fmt.Fprintln(stderr, r.Line()) } + rep, err := planguard.Run(ctx, c, planguard.NewSession(pc), o) + if rep == nil { + fmt.Fprintln(stderr, "planguard:", err) + return 1 + } + return finish(rep, err, f, stderr) +} + +// buildFixture builds the fixture tier's database on --dsn from the fixture/ directory beside the corpus, or finds +// it built from the same files. +func buildFixture(ctx context.Context, f runFlags, stderr io.Writer) error { + pool, err := pgxpool.New(ctx, f.dsn) + if err != nil { + return fmt.Errorf("connect: %w", err) + } + defer pool.Close() + t0 := time.Now() + if err := planguard.BuildFixture(ctx, pool, os.DirFS(filepath.Dir(f.corpus))); err != nil { + return err + } + fmt.Fprintf(stderr, "fixture ready in %.1f s\n", time.Since(t0).Seconds()) + return nil +} + +// options checks the flags and builds the run options; code is 2 on a usage error. +func (f runFlags) options(stderr io.Writer) (planguard.RunOptions, int) { + o := planguard.RunOptions{Tier: f.tier, IncludePending: f.includePending, SearchPath: f.searchPath} + switch f.tier { + case "": + return o, usageErr(stderr, "--tier is required (fixture, local or prod)") + case "fixture", "local", "prod": + default: + return o, usageErr(stderr, "unknown --tier %q (want fixture, local or prod)", f.tier) + } + switch f.baseline { + case "": + o.Baseline = planguard.BaselineCheck + case "check", "write", "off": + o.Baseline = planguard.BaselineMode(f.baseline) + default: + return o, usageErr(stderr, "unknown --baseline %q (want check, write or off)", f.baseline) + } + if f.tier == "prod" { + o.Baseline = planguard.BaselineOff + } + o.BaselineDir = filepath.Join(filepath.Dir(f.corpus), "baseline", f.tier) + var err error + if f.only != "" { + if o.Only, err = regexp.Compile(f.only); err != nil { + return o, usageErr(stderr, "--only: %v", err) + } + } + if f.skip != "" { + if o.Skip, err = regexp.Compile(f.skip); err != nil { + return o, usageErr(stderr, "--skip: %v", err) + } + } + if f.modes != "" { + for _, m := range strings.Split(f.modes, ",") { + switch m = strings.TrimSpace(m); m { + case "custom", "generic", "auto": + o.Modes = append(o.Modes, m) + default: + return o, usageErr(stderr, "--modes: unknown mode %q (want custom, generic or auto)", m) + } + } + } + if (f.emit != "" || f.plans != "") && f.tier != "prod" { + return o, usageErr(stderr, "--emit and --plans are the prod tier's psql path; use them with --tier prod") + } + switch f.emit { + case "", "resolvers": + case "plans": + if f.params == "" { + return o, usageErr(stderr, "--emit plans needs --params (the psql output of --emit resolvers)") + } + default: + return o, usageErr(stderr, "unknown --emit %q (want resolvers or plans)", f.emit) + } + if f.plans != "" { + if f.emit != "" { + return o, usageErr(stderr, "--plans and --emit are exclusive") + } + if f.params == "" { + return o, usageErr(stderr, "--plans needs --params (the psql output of --emit resolvers)") + } + } + return o, 0 +} + +func emit(stderr io.Writer, write func(nonce string) error) int { + nonce, err := planguard.NewNonce() + if err == nil { + err = write(nonce) + } + if err != nil { + fmt.Fprintln(stderr, "planguard:", err) + return 1 + } + return 0 +} + +func readResolverOutput(path string) (map[string]*string, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + nonce, err := planguard.ScriptNonce(raw) + if err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + res, err := planguard.ParseResolverOutput(strings.NewReader(string(raw)), nonce) + if err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + return res, nil +} + +// finish writes the report files, prints the totals and the fidelity, and returns the exit status. +func finish(rep *planguard.Report, runErr error, f runFlags, stderr io.Writer) int { + code := 0 + if err := rep.WriteFiles(f.out); err != nil { + fmt.Fprintln(stderr, "planguard: write report:", err) + code = 1 + } + fmt.Fprintf(stderr, "%d runs in %.1f s: %s\n", len(rep.Results), rep.DurationMs/1000, rep.CountsString()) + if f.fidelity != "" { + other, err := planguard.ReadResults(f.fidelity) + if err != nil { + fmt.Fprintln(stderr, "planguard: fidelity:", err) + code = 1 + } else { + var otherIx []string + if meta, err := planguard.ReadReportMeta(f.fidelity); err == nil { + otherIx = meta.Indexes + } else if errors.Is(err, os.ErrNotExist) { + fmt.Fprintf(stderr, "planguard: fidelity: no report file beside %s; its index set is unknown, so nothing is excluded for it\n", f.fidelity) + } else { + fmt.Fprintln(stderr, "planguard: fidelity:", err) + code = 1 + } + fr := planguard.Fidelity(rep.Results, other, rep.Indexes, otherIx) + pct := 0.0 + if fr.Total > 0 { + pct = 100 * float64(fr.Match) / float64(fr.Total) + } + fmt.Fprintf(stderr, "fidelity: %d of %d driver sets match (%.1f %%), %d excluded for index-set differences\n", fr.Match, fr.Total, pct, fr.Excluded) + for _, m := range fr.Mismatches { + fmt.Fprintln(stderr, " differs:", m) + } + } + } + if runErr != nil { + fmt.Fprintln(stderr, "planguard: run stopped:", runErr) + code = 1 + } + if n := len(rep.Failures()); n > 0 { + fmt.Fprintf(stderr, "%d failing runs\n", n) + code = 1 + } + return code +} diff --git a/cmd/planguard/main_test.go b/cmd/planguard/main_test.go new file mode 100644 index 0000000..f5e2c8f --- /dev/null +++ b/cmd/planguard/main_test.go @@ -0,0 +1,107 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "bytes" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestCLIUsage(t *testing.T) { + cases := []struct { + name string + args []string + want string // in stderr + }{ + {"no args", nil, "usage"}, + {"unknown subcommand", []string{"frobnicate"}, "frobnicate"}, + {"run without tier", []string{"run"}, "--tier"}, + {"run with unknown tier", []string{"run", "--tier", "staging"}, "staging"}, + {"bad baseline", []string{"run", "--tier", "local", "--baseline", "maybe"}, "--baseline"}, + {"bad only", []string{"run", "--tier", "local", "--only", "("}, "--only"}, + {"emit plans without params", []string{"run", "--tier", "prod", "--emit", "plans"}, "--params"}, + {"emit on local", []string{"run", "--tier", "local", "--emit", "resolvers"}, "--tier prod"}, + {"plans on fixture", []string{"run", "--tier", "fixture", "--plans", "p.out", "--params", "r.out"}, "--tier prod"}, + {"unknown flag", []string{"run", "--tier", "local", "--nope"}, "nope"}, + {"gen-policy unknown flag", []string{"gen-policy", "--nope"}, "nope"}, + {"gen-policy argument", []string{"gen-policy", "extra"}, "extra"}, + {"diff without cases", []string{"diff", "--dsn", "x"}, "--cases"}, + {"diff bad walk", []string{"diff", "--cases", "c.yaml", "--walk", "1,0"}, "--walk"}, + {"diff bad walk word", []string{"diff", "--cases", "c.yaml", "--walk", "web,all"}, "all"}, + {"diff unknown flag", []string{"diff", "--nope"}, "nope"}, + {"storediff without new", []string{"storediff", "--old", "http://a", "--cases", "c.yaml"}, "--new"}, + {"storediff bad url", []string{"storediff", "--old", "a:8481", "--new", "http://b", "--cases", "c.yaml"}, "a:8481"}, + {"storediff bad walk", []string{"storediff", "--old", "http://a", "--new", "http://b", "--cases", "c.yaml", "--walk", "0"}, "--walk"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var out, errb bytes.Buffer + if code := run(tc.args, &out, &errb); code != 2 { + t.Errorf("exit %d, want 2 (stderr %q)", code, errb.String()) + } + if !strings.Contains(errb.String(), tc.want) { + t.Errorf("stderr %q, want it to name %q", errb.String(), tc.want) + } + }) + } +} + +// gen-policy finds the module root from a subdirectory. It writes to a temporary file, which must equal the +// committed cached_gen.go; the committed file is never written. +func TestGenPolicyFromSubdirectory(t *testing.T) { + want, err := os.ReadFile("../../db/planpolicy/cached_gen.go") + if err != nil { + t.Fatal(err) + } + out := filepath.Join(t.TempDir(), "cached_gen.go") + t.Chdir("../../internal") + var stdout, errb bytes.Buffer + if code := run([]string{"gen-policy", "--out", out}, &stdout, &errb); code != 0 { + t.Fatalf("exit %d: %s", code, errb.String()) + } + got, err := os.ReadFile(out) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, want) { + t.Errorf("generated policy differs from the committed db/planpolicy/cached_gen.go:\n%s", got) + } +} + +// storediff exits 0 when every page matches and 1 when one differs, and writes its report under --out. +func TestStoreDiffCLI(t *testing.T) { + serve := func(name string) *httptest.Server { + s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + fmt.Fprintf(w, `{"items":[{"id":1,"name":%q}],"hasMore":false}`, name) + })) + t.Cleanup(s.Close) + return s + } + dir := t.TempDir() + cases := filepath.Join(dir, "cases.yaml") + if err := os.WriteFile(cases, []byte("cases:\n - id: things\n path: /api/v1/things\n"), 0o600); err != nil { + t.Fatal(err) + } + a, b, c := serve("x"), serve("x"), serve("y") + var out, errb bytes.Buffer + if code := run([]string{"storediff", "--old", a.URL, "--new", b.URL, "--cases", cases, "--walk", "1,web", "--out", dir}, &out, &errb); code != 0 { + t.Fatalf("same servers: exit %d, stderr %s", code, errb.String()) + } + report, err := os.ReadFile(filepath.Join(dir, "storediff_report.md")) + if err != nil || !strings.Contains(string(report), "| things | 50 | 1 | identical |") { + t.Errorf("report %q (%v)", report, err) + } + errb.Reset() + if code := run([]string{"storediff", "--old", a.URL, "--new", c.URL, "--cases", cases}, &out, &errb); code != 1 || + !strings.Contains(errb.String(), `$.items[0].name: old "x", new "y"`) { + t.Errorf("different servers: exit %d, stderr %s", code, errb.String()) + } +} diff --git a/config.yaml.example b/config.yaml.example index db60a06..ef3b84e 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -37,6 +37,9 @@ server: # Address of the admin listener serving Prometheus /metrics, e.g. 127.0.0.1:9090. Off when empty. # It has no authentication: bind it to a private interface. admin_listen: "" + # read_only: true boots a server that never writes: no migrations, no ingest, no background jobs, no + # MeshMapper imports, and every database session is read-only. Used to compare two builds on one database. + read_only: false # Applies to /api/v1/* only. Both counters are local to this Beacon instance. # Set trusted_proxies correctly before using a reverse proxy, or clients share @@ -69,6 +72,11 @@ ratelimit: # enforce: [] # budget: shadow # the per-client DB-time budget only records; enforcement is not available +# Daily read-only plan regression check (GET /api/v1/admin/db/planguard, beacon_planguard_* metrics). +# Enable on large deployments; plan expectations assume production-sized tables. +#planguard: +# daily_check: false + # MeshMapper integrations (optional, require a regional or grouped-region API key). # Your local MeshMapper regional or grouped-region admin can generate the key. # A grouped-region key covers multiple regions. diff --git a/db/planguard/baseline/fixture/pk.maint.delete_old_packets.dml.custom.json b/db/planguard/baseline/fixture/pk.maint.delete_old_packets.dml.custom.json new file mode 100644 index 0000000..4647c08 --- /dev/null +++ b/db/planguard/baseline/fixture/pk.maint.delete_old_packets.dml.custom.json @@ -0,0 +1,18 @@ +[ + "Aggregate partial=Simple", + " Limit [InitPlan]", + " LockRows", + " Incremental Sort", + " Index Scan on packets vp using idx_packets_last_heard scan=Backward", + " ModifyTable on packets dp [InitPlan]", + " Nested Loop", + " CTE Scan", + " Index Scan on packets dp using packets_pkey scan=Forward", + " ModifyTable on analytics_raw_state [InitPlan]", + " Aggregate partial=Simple [InitPlan]", + " CTE Scan", + " CTE Scan [InitPlan]", + " Result", + " Seq Scan on analytics_raw_state", + " CTE Scan" +] diff --git a/db/planguard/baseline/fixture/pk.maint.delete_old_packets.victims.custom.json b/db/planguard/baseline/fixture/pk.maint.delete_old_packets.victims.custom.json new file mode 100644 index 0000000..f38b4d4 --- /dev/null +++ b/db/planguard/baseline/fixture/pk.maint.delete_old_packets.victims.custom.json @@ -0,0 +1,5 @@ +[ + "Limit", + " Incremental Sort", + " Index Scan on packets vp using idx_packets_last_heard scan=Backward" +] diff --git a/db/planguard/baseline/fixture/retention.expired.dml.custom.json b/db/planguard/baseline/fixture/retention.expired.dml.custom.json new file mode 100644 index 0000000..64ed38a --- /dev/null +++ b/db/planguard/baseline/fixture/retention.expired.dml.custom.json @@ -0,0 +1,18 @@ +[ + "Result", + " Limit [InitPlan]", + " LockRows", + " Index Scan on known_routes r using idx_known_routes_keyset scan=Backward", + " ModifyTable on known_routes kr [InitPlan]", + " Nested Loop", + " CTE Scan", + " Index Scan on known_routes kr using known_routes_pkey scan=Forward", + " Aggregate partial=Simple [InitPlan]", + " CTE Scan", + " Limit [InitPlan]", + " Sort", + " CTE Scan", + " Limit [InitPlan]", + " Sort", + " CTE Scan" +] diff --git a/db/planguard/baseline/fixture/retention.expired.select.custom.json b/db/planguard/baseline/fixture/retention.expired.select.custom.json new file mode 100644 index 0000000..46005dc --- /dev/null +++ b/db/planguard/baseline/fixture/retention.expired.select.custom.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes r using idx_known_routes_keyset scan=Backward" +] diff --git a/db/planguard/baseline/fixture/retention.expired.select.final.custom.json b/db/planguard/baseline/fixture/retention.expired.select.final.custom.json new file mode 100644 index 0000000..46005dc --- /dev/null +++ b/db/planguard/baseline/fixture/retention.expired.select.final.custom.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes r using idx_known_routes_keyset scan=Backward" +] diff --git a/db/planguard/baseline/fixture/retention.grace.cand.custom.json b/db/planguard/baseline/fixture/retention.grace.cand.custom.json new file mode 100644 index 0000000..46005dc --- /dev/null +++ b/db/planguard/baseline/fixture/retention.grace.cand.custom.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes r using idx_known_routes_keyset scan=Backward" +] diff --git a/db/planguard/baseline/fixture/retention.grace.dml.custom.json b/db/planguard/baseline/fixture/retention.grace.dml.custom.json new file mode 100644 index 0000000..6647549 --- /dev/null +++ b/db/planguard/baseline/fixture/retention.grace.dml.custom.json @@ -0,0 +1,28 @@ +[ + "Result", + " Limit [InitPlan]", + " Index Scan on known_routes r using idx_known_routes_keyset scan=Backward", + " LockRows [InitPlan]", + " Nested Loop", + " CTE Scan", + " Index Scan on known_routes r_1 using known_routes_pkey scan=Forward", + " ModifyTable on known_routes kr [InitPlan]", + " Nested Loop", + " CTE Scan", + " Index Scan on known_routes kr using known_routes_pkey scan=Forward", + " Aggregate partial=Simple [InitPlan]", + " CTE Scan", + " Aggregate partial=Simple [InitPlan]", + " CTE Scan", + " Limit [InitPlan]", + " Sort", + " CTE Scan", + " Limit [InitPlan]", + " Sort", + " CTE Scan", + " Aggregate partial=Simple [InitPlan]", + " Hash Join (Anti)", + " CTE Scan", + " Hash", + " CTE Scan" +] diff --git a/db/planguard/baseline/fixture/retention.grace.fullpass.custom.json b/db/planguard/baseline/fixture/retention.grace.fullpass.custom.json new file mode 100644 index 0000000..46005dc --- /dev/null +++ b/db/planguard/baseline/fixture/retention.grace.fullpass.custom.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes r using idx_known_routes_keyset scan=Backward" +] diff --git a/db/planguard/baseline/fixture/routes.byiata.R01_SEA_first_l50.auto.json b/db/planguard/baseline/fixture/routes.byiata.R01_SEA_first_l50.auto.json new file mode 100644 index 0000000..7272862 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiata.R01_SEA_first_l50.auto.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_iata_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiata.R01_SEA_first_l50.custom.json b/db/planguard/baseline/fixture/routes.byiata.R01_SEA_first_l50.custom.json new file mode 100644 index 0000000..7272862 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiata.R01_SEA_first_l50.custom.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_iata_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiata.R01_SEA_first_l50.generic.json b/db/planguard/baseline/fixture/routes.byiata.R01_SEA_first_l50.generic.json new file mode 100644 index 0000000..7272862 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiata.R01_SEA_first_l50.generic.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_iata_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiata.R02_SEA_first_l200.auto.json b/db/planguard/baseline/fixture/routes.byiata.R02_SEA_first_l200.auto.json new file mode 100644 index 0000000..7272862 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiata.R02_SEA_first_l200.auto.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_iata_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiata.R02_SEA_first_l200.custom.json b/db/planguard/baseline/fixture/routes.byiata.R02_SEA_first_l200.custom.json new file mode 100644 index 0000000..7272862 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiata.R02_SEA_first_l200.custom.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_iata_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiata.R02_SEA_first_l200.generic.json b/db/planguard/baseline/fixture/routes.byiata.R02_SEA_first_l200.generic.json new file mode 100644 index 0000000..7272862 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiata.R02_SEA_first_l200.generic.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_iata_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiata.R03_SEA_page2_l50.auto.json b/db/planguard/baseline/fixture/routes.byiata.R03_SEA_page2_l50.auto.json new file mode 100644 index 0000000..7272862 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiata.R03_SEA_page2_l50.auto.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_iata_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiata.R03_SEA_page2_l50.custom.json b/db/planguard/baseline/fixture/routes.byiata.R03_SEA_page2_l50.custom.json new file mode 100644 index 0000000..7272862 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiata.R03_SEA_page2_l50.custom.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_iata_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiata.R03_SEA_page2_l50.generic.json b/db/planguard/baseline/fixture/routes.byiata.R03_SEA_page2_l50.generic.json new file mode 100644 index 0000000..7272862 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiata.R03_SEA_page2_l50.generic.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_iata_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiata.R04_SEA_deep_l200.auto.json b/db/planguard/baseline/fixture/routes.byiata.R04_SEA_deep_l200.auto.json new file mode 100644 index 0000000..7272862 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiata.R04_SEA_deep_l200.auto.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_iata_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiata.R04_SEA_deep_l200.custom.json b/db/planguard/baseline/fixture/routes.byiata.R04_SEA_deep_l200.custom.json new file mode 100644 index 0000000..7272862 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiata.R04_SEA_deep_l200.custom.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_iata_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiata.R04_SEA_deep_l200.generic.json b/db/planguard/baseline/fixture/routes.byiata.R04_SEA_deep_l200.generic.json new file mode 100644 index 0000000..7272862 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiata.R04_SEA_deep_l200.generic.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_iata_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiata.R05_BWE_first_l200.auto.json b/db/planguard/baseline/fixture/routes.byiata.R05_BWE_first_l200.auto.json new file mode 100644 index 0000000..7272862 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiata.R05_BWE_first_l200.auto.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_iata_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiata.R05_BWE_first_l200.custom.json b/db/planguard/baseline/fixture/routes.byiata.R05_BWE_first_l200.custom.json new file mode 100644 index 0000000..7272862 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiata.R05_BWE_first_l200.custom.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_iata_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiata.R05_BWE_first_l200.generic.json b/db/planguard/baseline/fixture/routes.byiata.R05_BWE_first_l200.generic.json new file mode 100644 index 0000000..7272862 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiata.R05_BWE_first_l200.generic.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_iata_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiata.R06_MAD_first_l50.auto.json b/db/planguard/baseline/fixture/routes.byiata.R06_MAD_first_l50.auto.json new file mode 100644 index 0000000..7272862 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiata.R06_MAD_first_l50.auto.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_iata_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiata.R06_MAD_first_l50.custom.json b/db/planguard/baseline/fixture/routes.byiata.R06_MAD_first_l50.custom.json new file mode 100644 index 0000000..7272862 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiata.R06_MAD_first_l50.custom.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_iata_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiata.R06_MAD_first_l50.generic.json b/db/planguard/baseline/fixture/routes.byiata.R06_MAD_first_l50.generic.json new file mode 100644 index 0000000..7272862 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiata.R06_MAD_first_l50.generic.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_iata_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiata.R07_CLE_first_l200.auto.json b/db/planguard/baseline/fixture/routes.byiata.R07_CLE_first_l200.auto.json new file mode 100644 index 0000000..7272862 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiata.R07_CLE_first_l200.auto.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_iata_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiata.R07_CLE_first_l200.custom.json b/db/planguard/baseline/fixture/routes.byiata.R07_CLE_first_l200.custom.json new file mode 100644 index 0000000..7272862 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiata.R07_CLE_first_l200.custom.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_iata_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiata.R07_CLE_first_l200.generic.json b/db/planguard/baseline/fixture/routes.byiata.R07_CLE_first_l200.generic.json new file mode 100644 index 0000000..7272862 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiata.R07_CLE_first_l200.generic.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_iata_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiata.R08_NEW_first_l50.auto.json b/db/planguard/baseline/fixture/routes.byiata.R08_NEW_first_l50.auto.json new file mode 100644 index 0000000..7272862 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiata.R08_NEW_first_l50.auto.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_iata_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiata.R08_NEW_first_l50.custom.json b/db/planguard/baseline/fixture/routes.byiata.R08_NEW_first_l50.custom.json new file mode 100644 index 0000000..7272862 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiata.R08_NEW_first_l50.custom.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_iata_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiata.R08_NEW_first_l50.generic.json b/db/planguard/baseline/fixture/routes.byiata.R08_NEW_first_l50.generic.json new file mode 100644 index 0000000..7272862 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiata.R08_NEW_first_l50.generic.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_iata_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiata.R09_NEW_page2_l50.auto.json b/db/planguard/baseline/fixture/routes.byiata.R09_NEW_page2_l50.auto.json new file mode 100644 index 0000000..7272862 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiata.R09_NEW_page2_l50.auto.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_iata_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiata.R09_NEW_page2_l50.custom.json b/db/planguard/baseline/fixture/routes.byiata.R09_NEW_page2_l50.custom.json new file mode 100644 index 0000000..7272862 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiata.R09_NEW_page2_l50.custom.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_iata_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiata.R09_NEW_page2_l50.generic.json b/db/planguard/baseline/fixture/routes.byiata.R09_NEW_page2_l50.generic.json new file mode 100644 index 0000000..7272862 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiata.R09_NEW_page2_l50.generic.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_iata_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiata.R10_NEW_deep_l200.auto.json b/db/planguard/baseline/fixture/routes.byiata.R10_NEW_deep_l200.auto.json new file mode 100644 index 0000000..7272862 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiata.R10_NEW_deep_l200.auto.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_iata_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiata.R10_NEW_deep_l200.custom.json b/db/planguard/baseline/fixture/routes.byiata.R10_NEW_deep_l200.custom.json new file mode 100644 index 0000000..7272862 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiata.R10_NEW_deep_l200.custom.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_iata_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiata.R10_NEW_deep_l200.generic.json b/db/planguard/baseline/fixture/routes.byiata.R10_NEW_deep_l200.generic.json new file mode 100644 index 0000000..7272862 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiata.R10_NEW_deep_l200.generic.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_iata_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiata.R11_ZZZ_l50.auto.json b/db/planguard/baseline/fixture/routes.byiata.R11_ZZZ_l50.auto.json new file mode 100644 index 0000000..7272862 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiata.R11_ZZZ_l50.auto.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_iata_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiata.R11_ZZZ_l50.custom.json b/db/planguard/baseline/fixture/routes.byiata.R11_ZZZ_l50.custom.json new file mode 100644 index 0000000..7272862 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiata.R11_ZZZ_l50.custom.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_iata_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiata.R11_ZZZ_l50.generic.json b/db/planguard/baseline/fixture/routes.byiata.R11_ZZZ_l50.generic.json new file mode 100644 index 0000000..7272862 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiata.R11_ZZZ_l50.generic.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_iata_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiata.R12_WRE_l50.auto.json b/db/planguard/baseline/fixture/routes.byiata.R12_WRE_l50.auto.json new file mode 100644 index 0000000..7272862 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiata.R12_WRE_l50.auto.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_iata_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiata.R12_WRE_l50.custom.json b/db/planguard/baseline/fixture/routes.byiata.R12_WRE_l50.custom.json new file mode 100644 index 0000000..7272862 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiata.R12_WRE_l50.custom.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_iata_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiata.R12_WRE_l50.generic.json b/db/planguard/baseline/fixture/routes.byiata.R12_WRE_l50.generic.json new file mode 100644 index 0000000..7272862 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiata.R12_WRE_l50.generic.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_iata_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiata.R13_SEA_tie_l200.auto.json b/db/planguard/baseline/fixture/routes.byiata.R13_SEA_tie_l200.auto.json new file mode 100644 index 0000000..7272862 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiata.R13_SEA_tie_l200.auto.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_iata_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiata.R13_SEA_tie_l200.custom.json b/db/planguard/baseline/fixture/routes.byiata.R13_SEA_tie_l200.custom.json new file mode 100644 index 0000000..7272862 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiata.R13_SEA_tie_l200.custom.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_iata_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiata.R13_SEA_tie_l200.generic.json b/db/planguard/baseline/fixture/routes.byiata.R13_SEA_tie_l200.generic.json new file mode 100644 index 0000000..7272862 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiata.R13_SEA_tie_l200.generic.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_iata_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiata.R14_BWE_deep_l200.auto.json b/db/planguard/baseline/fixture/routes.byiata.R14_BWE_deep_l200.auto.json new file mode 100644 index 0000000..7272862 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiata.R14_BWE_deep_l200.auto.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_iata_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiata.R14_BWE_deep_l200.custom.json b/db/planguard/baseline/fixture/routes.byiata.R14_BWE_deep_l200.custom.json new file mode 100644 index 0000000..7272862 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiata.R14_BWE_deep_l200.custom.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_iata_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiata.R14_BWE_deep_l200.generic.json b/db/planguard/baseline/fixture/routes.byiata.R14_BWE_deep_l200.generic.json new file mode 100644 index 0000000..7272862 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiata.R14_BWE_deep_l200.generic.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_iata_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiatahop.H01_SEA_hop6_dense_l200.custom.json b/db/planguard/baseline/fixture/routes.byiatahop.H01_SEA_hop6_dense_l200.custom.json new file mode 100644 index 0000000..7272862 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiatahop.H01_SEA_hop6_dense_l200.custom.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_iata_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiatahop.H03_SEA_hop25_rare_l200.custom.json b/db/planguard/baseline/fixture/routes.byiatahop.H03_SEA_hop25_rare_l200.custom.json new file mode 100644 index 0000000..c639fd5 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiatahop.H03_SEA_hop25_rare_l200.custom.json @@ -0,0 +1,5 @@ +[ + "Limit", + " Sort", + " Index Scan on known_routes using idx_known_routes_hop_iata scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiatahop.H04_SEA_hop32_vrare_l50.custom.json b/db/planguard/baseline/fixture/routes.byiatahop.H04_SEA_hop32_vrare_l50.custom.json new file mode 100644 index 0000000..c639fd5 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiatahop.H04_SEA_hop32_vrare_l50.custom.json @@ -0,0 +1,5 @@ +[ + "Limit", + " Sort", + " Index Scan on known_routes using idx_known_routes_hop_iata scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiatahop.H05_SEA_hop1_absent_l200.custom.json b/db/planguard/baseline/fixture/routes.byiatahop.H05_SEA_hop1_absent_l200.custom.json new file mode 100644 index 0000000..c639fd5 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiatahop.H05_SEA_hop1_absent_l200.custom.json @@ -0,0 +1,5 @@ +[ + "Limit", + " Sort", + " Index Scan on known_routes using idx_known_routes_hop_iata scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiatahop.H06_SEA_hop6_page2_l200.custom.json b/db/planguard/baseline/fixture/routes.byiatahop.H06_SEA_hop6_page2_l200.custom.json new file mode 100644 index 0000000..7272862 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiatahop.H06_SEA_hop6_page2_l200.custom.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_iata_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiatahop.H07_BWE_hop5_dense_l200.custom.json b/db/planguard/baseline/fixture/routes.byiatahop.H07_BWE_hop5_dense_l200.custom.json new file mode 100644 index 0000000..7272862 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiatahop.H07_BWE_hop5_dense_l200.custom.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_iata_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiatahop.H08_BWE_hop22_rare_l50.custom.json b/db/planguard/baseline/fixture/routes.byiatahop.H08_BWE_hop22_rare_l50.custom.json new file mode 100644 index 0000000..c639fd5 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiatahop.H08_BWE_hop22_rare_l50.custom.json @@ -0,0 +1,5 @@ +[ + "Limit", + " Sort", + " Index Scan on known_routes using idx_known_routes_hop_iata scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.byiatahop.H10_MAD_hop16_rare_l50.custom.json b/db/planguard/baseline/fixture/routes.byiatahop.H10_MAD_hop16_rare_l50.custom.json new file mode 100644 index 0000000..c639fd5 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.byiatahop.H10_MAD_hop16_rare_l50.custom.json @@ -0,0 +1,5 @@ +[ + "Limit", + " Sort", + " Index Scan on known_routes using idx_known_routes_hop_iata scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.cross.x_0355.custom.json b/db/planguard/baseline/fixture/routes.cross.x_0355.custom.json new file mode 100644 index 0000000..7442eea --- /dev/null +++ b/db/planguard/baseline/fixture/routes.cross.x_0355.custom.json @@ -0,0 +1,6 @@ +[ + "Limit", + " Sort", + " Bitmap Heap Scan on known_routes kr", + " Bitmap Index Scan using idx_known_routes_node_ids" +] diff --git a/db/planguard/baseline/fixture/routes.cross.x_0867.custom.json b/db/planguard/baseline/fixture/routes.cross.x_0867.custom.json new file mode 100644 index 0000000..cf06dfe --- /dev/null +++ b/db/planguard/baseline/fixture/routes.cross.x_0867.custom.json @@ -0,0 +1,8 @@ +[ + "Limit", + " Sort", + " Bitmap Heap Scan on known_routes kr", + " BitmapAnd", + " Bitmap Index Scan using idx_known_routes_node_ids", + " Bitmap Index Scan using known_routes_pkey" +] diff --git a/db/planguard/baseline/fixture/routes.cross.x_1410.custom.json b/db/planguard/baseline/fixture/routes.cross.x_1410.custom.json new file mode 100644 index 0000000..cf06dfe --- /dev/null +++ b/db/planguard/baseline/fixture/routes.cross.x_1410.custom.json @@ -0,0 +1,8 @@ +[ + "Limit", + " Sort", + " Bitmap Heap Scan on known_routes kr", + " BitmapAnd", + " Bitmap Index Scan using idx_known_routes_node_ids", + " Bitmap Index Scan using known_routes_pkey" +] diff --git a/db/planguard/baseline/fixture/routes.cross.x_3d67.custom.json b/db/planguard/baseline/fixture/routes.cross.x_3d67.custom.json new file mode 100644 index 0000000..cf06dfe --- /dev/null +++ b/db/planguard/baseline/fixture/routes.cross.x_3d67.custom.json @@ -0,0 +1,8 @@ +[ + "Limit", + " Sort", + " Bitmap Heap Scan on known_routes kr", + " BitmapAnd", + " Bitmap Index Scan using idx_known_routes_node_ids", + " Bitmap Index Scan using known_routes_pkey" +] diff --git a/db/planguard/baseline/fixture/routes.cross.x_5731.custom.json b/db/planguard/baseline/fixture/routes.cross.x_5731.custom.json new file mode 100644 index 0000000..cf06dfe --- /dev/null +++ b/db/planguard/baseline/fixture/routes.cross.x_5731.custom.json @@ -0,0 +1,8 @@ +[ + "Limit", + " Sort", + " Bitmap Heap Scan on known_routes kr", + " BitmapAnd", + " Bitmap Index Scan using idx_known_routes_node_ids", + " Bitmap Index Scan using known_routes_pkey" +] diff --git a/db/planguard/baseline/fixture/routes.cross.x_f877.custom.json b/db/planguard/baseline/fixture/routes.cross.x_f877.custom.json new file mode 100644 index 0000000..cf06dfe --- /dev/null +++ b/db/planguard/baseline/fixture/routes.cross.x_f877.custom.json @@ -0,0 +1,8 @@ +[ + "Limit", + " Sort", + " Bitmap Heap Scan on known_routes kr", + " BitmapAnd", + " Bitmap Index Scan using idx_known_routes_node_ids", + " Bitmap Index Scan using known_routes_pkey" +] diff --git a/db/planguard/baseline/fixture/routes.cross.x_sea_rare.custom.json b/db/planguard/baseline/fixture/routes.cross.x_sea_rare.custom.json new file mode 100644 index 0000000..cf06dfe --- /dev/null +++ b/db/planguard/baseline/fixture/routes.cross.x_sea_rare.custom.json @@ -0,0 +1,8 @@ +[ + "Limit", + " Sort", + " Bitmap Heap Scan on known_routes kr", + " BitmapAnd", + " Bitmap Index Scan using idx_known_routes_node_ids", + " Bitmap Index Scan using known_routes_pkey" +] diff --git a/db/planguard/baseline/fixture/routes.cross.x_sea_top1.custom.json b/db/planguard/baseline/fixture/routes.cross.x_sea_top1.custom.json new file mode 100644 index 0000000..cf06dfe --- /dev/null +++ b/db/planguard/baseline/fixture/routes.cross.x_sea_top1.custom.json @@ -0,0 +1,8 @@ +[ + "Limit", + " Sort", + " Bitmap Heap Scan on known_routes kr", + " BitmapAnd", + " Bitmap Index Scan using idx_known_routes_node_ids", + " Bitmap Index Scan using known_routes_pkey" +] diff --git a/db/planguard/baseline/fixture/routes.global.G01_first_l50.auto.json b/db/planguard/baseline/fixture/routes.global.G01_first_l50.auto.json new file mode 100644 index 0000000..f7e9aec --- /dev/null +++ b/db/planguard/baseline/fixture/routes.global.G01_first_l50.auto.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.global.G01_first_l50.custom.json b/db/planguard/baseline/fixture/routes.global.G01_first_l50.custom.json new file mode 100644 index 0000000..f7e9aec --- /dev/null +++ b/db/planguard/baseline/fixture/routes.global.G01_first_l50.custom.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.global.G01_first_l50.generic.json b/db/planguard/baseline/fixture/routes.global.G01_first_l50.generic.json new file mode 100644 index 0000000..f7e9aec --- /dev/null +++ b/db/planguard/baseline/fixture/routes.global.G01_first_l50.generic.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.global.G02_page2_l50.auto.json b/db/planguard/baseline/fixture/routes.global.G02_page2_l50.auto.json new file mode 100644 index 0000000..f7e9aec --- /dev/null +++ b/db/planguard/baseline/fixture/routes.global.G02_page2_l50.auto.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.global.G02_page2_l50.custom.json b/db/planguard/baseline/fixture/routes.global.G02_page2_l50.custom.json new file mode 100644 index 0000000..f7e9aec --- /dev/null +++ b/db/planguard/baseline/fixture/routes.global.G02_page2_l50.custom.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.global.G02_page2_l50.generic.json b/db/planguard/baseline/fixture/routes.global.G02_page2_l50.generic.json new file mode 100644 index 0000000..f7e9aec --- /dev/null +++ b/db/planguard/baseline/fixture/routes.global.G02_page2_l50.generic.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.global.G03_deep_curonly_l200.auto.json b/db/planguard/baseline/fixture/routes.global.G03_deep_curonly_l200.auto.json new file mode 100644 index 0000000..f7e9aec --- /dev/null +++ b/db/planguard/baseline/fixture/routes.global.G03_deep_curonly_l200.auto.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.global.G03_deep_curonly_l200.custom.json b/db/planguard/baseline/fixture/routes.global.G03_deep_curonly_l200.custom.json new file mode 100644 index 0000000..f7e9aec --- /dev/null +++ b/db/planguard/baseline/fixture/routes.global.G03_deep_curonly_l200.custom.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.global.G03_deep_curonly_l200.generic.json b/db/planguard/baseline/fixture/routes.global.G03_deep_curonly_l200.generic.json new file mode 100644 index 0000000..f7e9aec --- /dev/null +++ b/db/planguard/baseline/fixture/routes.global.G03_deep_curonly_l200.generic.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.global.G04_tie_l50.auto.json b/db/planguard/baseline/fixture/routes.global.G04_tie_l50.auto.json new file mode 100644 index 0000000..f7e9aec --- /dev/null +++ b/db/planguard/baseline/fixture/routes.global.G04_tie_l50.auto.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.global.G04_tie_l50.custom.json b/db/planguard/baseline/fixture/routes.global.G04_tie_l50.custom.json new file mode 100644 index 0000000..f7e9aec --- /dev/null +++ b/db/planguard/baseline/fixture/routes.global.G04_tie_l50.custom.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.global.G04_tie_l50.generic.json b/db/planguard/baseline/fixture/routes.global.G04_tie_l50.generic.json new file mode 100644 index 0000000..f7e9aec --- /dev/null +++ b/db/planguard/baseline/fixture/routes.global.G04_tie_l50.generic.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.globalhop.GH01_hop6_dense_l50.custom.json b/db/planguard/baseline/fixture/routes.globalhop.GH01_hop6_dense_l50.custom.json new file mode 100644 index 0000000..f7e9aec --- /dev/null +++ b/db/planguard/baseline/fixture/routes.globalhop.GH01_hop6_dense_l50.custom.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.globalhop.GH02_hop6_dense_l200.custom.json b/db/planguard/baseline/fixture/routes.globalhop.GH02_hop6_dense_l200.custom.json new file mode 100644 index 0000000..f7e9aec --- /dev/null +++ b/db/planguard/baseline/fixture/routes.globalhop.GH02_hop6_dense_l200.custom.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.globalhop.GH03_hop15_l200.custom.json b/db/planguard/baseline/fixture/routes.globalhop.GH03_hop15_l200.custom.json new file mode 100644 index 0000000..f7e9aec --- /dev/null +++ b/db/planguard/baseline/fixture/routes.globalhop.GH03_hop15_l200.custom.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.globalhop.GH04_hop20_l200.custom.json b/db/planguard/baseline/fixture/routes.globalhop.GH04_hop20_l200.custom.json new file mode 100644 index 0000000..f7e9aec --- /dev/null +++ b/db/planguard/baseline/fixture/routes.globalhop.GH04_hop20_l200.custom.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.globalhop.GH05_hop25_l200.custom.json b/db/planguard/baseline/fixture/routes.globalhop.GH05_hop25_l200.custom.json new file mode 100644 index 0000000..c639fd5 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.globalhop.GH05_hop25_l200.custom.json @@ -0,0 +1,5 @@ +[ + "Limit", + " Sort", + " Index Scan on known_routes using idx_known_routes_hop_iata scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.globalhop.GH06_hop32_l50.custom.json b/db/planguard/baseline/fixture/routes.globalhop.GH06_hop32_l50.custom.json new file mode 100644 index 0000000..c639fd5 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.globalhop.GH06_hop32_l50.custom.json @@ -0,0 +1,5 @@ +[ + "Limit", + " Sort", + " Index Scan on known_routes using idx_known_routes_hop_iata scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.globalhop.GH07_hop42_single_l50.custom.json b/db/planguard/baseline/fixture/routes.globalhop.GH07_hop42_single_l50.custom.json new file mode 100644 index 0000000..c639fd5 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.globalhop.GH07_hop42_single_l50.custom.json @@ -0,0 +1,5 @@ +[ + "Limit", + " Sort", + " Index Scan on known_routes using idx_known_routes_hop_iata scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.globalhop.GH08_hop1_absent_l50.custom.json b/db/planguard/baseline/fixture/routes.globalhop.GH08_hop1_absent_l50.custom.json new file mode 100644 index 0000000..c639fd5 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.globalhop.GH08_hop1_absent_l50.custom.json @@ -0,0 +1,5 @@ +[ + "Limit", + " Sort", + " Index Scan on known_routes using idx_known_routes_hop_iata scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.globalhop.GH09_hop1_absent_cur_l50.custom.json b/db/planguard/baseline/fixture/routes.globalhop.GH09_hop1_absent_cur_l50.custom.json new file mode 100644 index 0000000..c639fd5 --- /dev/null +++ b/db/planguard/baseline/fixture/routes.globalhop.GH09_hop1_absent_cur_l50.custom.json @@ -0,0 +1,5 @@ +[ + "Limit", + " Sort", + " Index Scan on known_routes using idx_known_routes_hop_iata scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.globalhop.GH10_hop6_deep_l200.custom.json b/db/planguard/baseline/fixture/routes.globalhop.GH10_hop6_deep_l200.custom.json new file mode 100644 index 0000000..f7e9aec --- /dev/null +++ b/db/planguard/baseline/fixture/routes.globalhop.GH10_hop6_deep_l200.custom.json @@ -0,0 +1,4 @@ +[ + "Limit", + " Index Scan on known_routes using idx_known_routes_keyset scan=Forward" +] diff --git a/db/planguard/baseline/fixture/routes.search.s_1410_3d67_RTM.custom.json b/db/planguard/baseline/fixture/routes.search.s_1410_3d67_RTM.custom.json new file mode 100644 index 0000000..7442eea --- /dev/null +++ b/db/planguard/baseline/fixture/routes.search.s_1410_3d67_RTM.custom.json @@ -0,0 +1,6 @@ +[ + "Limit", + " Sort", + " Bitmap Heap Scan on known_routes kr", + " Bitmap Index Scan using idx_known_routes_node_ids" +] diff --git a/db/planguard/baseline/fixture/routes.search.s_1410_3d67_all.custom.json b/db/planguard/baseline/fixture/routes.search.s_1410_3d67_all.custom.json new file mode 100644 index 0000000..7442eea --- /dev/null +++ b/db/planguard/baseline/fixture/routes.search.s_1410_3d67_all.custom.json @@ -0,0 +1,6 @@ +[ + "Limit", + " Sort", + " Bitmap Heap Scan on known_routes kr", + " Bitmap Index Scan using idx_known_routes_node_ids" +] diff --git a/db/planguard/baseline/fixture/routes.search.s_14_3d_RTM_AMS_EIN.custom.json b/db/planguard/baseline/fixture/routes.search.s_14_3d_RTM_AMS_EIN.custom.json new file mode 100644 index 0000000..7442eea --- /dev/null +++ b/db/planguard/baseline/fixture/routes.search.s_14_3d_RTM_AMS_EIN.custom.json @@ -0,0 +1,6 @@ +[ + "Limit", + " Sort", + " Bitmap Heap Scan on known_routes kr", + " Bitmap Index Scan using idx_known_routes_node_ids" +] diff --git a/db/planguard/baseline/fixture/routes.search.s_14_3d_all.custom.json b/db/planguard/baseline/fixture/routes.search.s_14_3d_all.custom.json new file mode 100644 index 0000000..7442eea --- /dev/null +++ b/db/planguard/baseline/fixture/routes.search.s_14_3d_all.custom.json @@ -0,0 +1,6 @@ +[ + "Limit", + " Sort", + " Bitmap Heap Scan on known_routes kr", + " Bitmap Index Scan using idx_known_routes_node_ids" +] diff --git a/db/planguard/baseline/fixture/routes.search.s_5731_0355_all.custom.json b/db/planguard/baseline/fixture/routes.search.s_5731_0355_all.custom.json new file mode 100644 index 0000000..7442eea --- /dev/null +++ b/db/planguard/baseline/fixture/routes.search.s_5731_0355_all.custom.json @@ -0,0 +1,6 @@ +[ + "Limit", + " Sort", + " Bitmap Heap Scan on known_routes kr", + " Bitmap Index Scan using idx_known_routes_node_ids" +] diff --git a/db/planguard/baseline/fixture/routes.search.s_f877_0867_all.custom.json b/db/planguard/baseline/fixture/routes.search.s_f877_0867_all.custom.json new file mode 100644 index 0000000..7442eea --- /dev/null +++ b/db/planguard/baseline/fixture/routes.search.s_f877_0867_all.custom.json @@ -0,0 +1,6 @@ +[ + "Limit", + " Sort", + " Bitmap Heap Scan on known_routes kr", + " Bitmap Index Scan using idx_known_routes_node_ids" +] diff --git a/db/planguard/baseline/fixture/routes.search.s_sea_lowco_rare.custom.json b/db/planguard/baseline/fixture/routes.search.s_sea_lowco_rare.custom.json new file mode 100644 index 0000000..7442eea --- /dev/null +++ b/db/planguard/baseline/fixture/routes.search.s_sea_lowco_rare.custom.json @@ -0,0 +1,6 @@ +[ + "Limit", + " Sort", + " Bitmap Heap Scan on known_routes kr", + " Bitmap Index Scan using idx_known_routes_node_ids" +] diff --git a/db/planguard/baseline/fixture/routes.search.s_sea_pop_rare.custom.json b/db/planguard/baseline/fixture/routes.search.s_sea_pop_rare.custom.json new file mode 100644 index 0000000..7442eea --- /dev/null +++ b/db/planguard/baseline/fixture/routes.search.s_sea_pop_rare.custom.json @@ -0,0 +1,6 @@ +[ + "Limit", + " Sort", + " Bitmap Heap Scan on known_routes kr", + " Bitmap Index Scan using idx_known_routes_node_ids" +] diff --git a/db/planguard/baseline/fixture/routes.search.s_sea_pop_top2.custom.json b/db/planguard/baseline/fixture/routes.search.s_sea_pop_top2.custom.json new file mode 100644 index 0000000..cf06dfe --- /dev/null +++ b/db/planguard/baseline/fixture/routes.search.s_sea_pop_top2.custom.json @@ -0,0 +1,8 @@ +[ + "Limit", + " Sort", + " Bitmap Heap Scan on known_routes kr", + " BitmapAnd", + " Bitmap Index Scan using idx_known_routes_node_ids", + " Bitmap Index Scan using known_routes_pkey" +] diff --git a/db/planguard/baseline/fixture/s7.pa.all625_7d_today.auto.json b/db/planguard/baseline/fixture/s7.pa.all625_7d_today.auto.json new file mode 100644 index 0000000..2065b79 --- /dev/null +++ b/db/planguard/baseline/fixture/s7.pa.all625_7d_today.auto.json @@ -0,0 +1,11 @@ +[ + "Sort", + " Aggregate partial=Simple {Hashed}", + " Append", + " Subquery Scan", + " Result [Subquery]", + " Index Scan on analytics_hourly_paths using analytics_hourly_paths_pkey scan=Forward", + " Subquery Scan", + " Result [Subquery]", + " Index Scan on analytics_hourly_paths analytics_hourly_paths_1 using analytics_hourly_paths_pkey scan=Forward" +] diff --git a/db/planguard/baseline/fixture/s7.pa.all625_7d_today.custom.json b/db/planguard/baseline/fixture/s7.pa.all625_7d_today.custom.json new file mode 100644 index 0000000..903ad6a --- /dev/null +++ b/db/planguard/baseline/fixture/s7.pa.all625_7d_today.custom.json @@ -0,0 +1,6 @@ +[ + "Sort", + " Aggregate partial=Simple {Hashed}", + " Subquery Scan", + " Index Scan on analytics_hourly_paths using analytics_hourly_paths_pkey scan=Forward [Subquery]" +] diff --git a/db/planguard/baseline/fixture/s7.ta.all625_7d_today.auto.json b/db/planguard/baseline/fixture/s7.ta.all625_7d_today.auto.json new file mode 100644 index 0000000..ab3f35b --- /dev/null +++ b/db/planguard/baseline/fixture/s7.ta.all625_7d_today.auto.json @@ -0,0 +1,21 @@ +[ + "Sort", + " Limit [InitPlan]", + " Sort", + " Aggregate partial=Simple {Sorted}", + " Sort", + " Index Scan on analytics_hourly_advert_sets using analytics_hourly_advert_sets_pkey scan=Forward", + " Hash Join (Left)", + " Nested Loop (Left)", + " CTE Scan", + " Index Scan on nodes n using nodes_public_key_key scan=Forward", + " Hash", + " Subquery Scan", + " Aggregate partial=Simple [Subquery] {Sorted}", + " Sort", + " Hash Join (Semi)", + " Gather", + " Parallel Index Scan on analytics_hourly_advert_hearings h_1 using analytics_hourly_advert_hearings_pkey scan=Forward", + " Hash", + " CTE Scan" +] diff --git a/db/planguard/baseline/fixture/s7.ta.all625_7d_today.custom.json b/db/planguard/baseline/fixture/s7.ta.all625_7d_today.custom.json new file mode 100644 index 0000000..bd93640 --- /dev/null +++ b/db/planguard/baseline/fixture/s7.ta.all625_7d_today.custom.json @@ -0,0 +1,18 @@ +[ + "Sort", + " Limit [InitPlan]", + " Sort", + " Aggregate partial=Simple {Hashed}", + " Index Scan on analytics_hourly_advert_sets using analytics_hourly_advert_sets_pkey scan=Forward", + " Merge Join (Left)", + " Sort", + " Nested Loop (Left)", + " CTE Scan", + " Index Scan on nodes n using nodes_public_key_key scan=Forward", + " Aggregate partial=Simple {Sorted}", + " Sort", + " Hash Join (Semi)", + " Index Scan on analytics_hourly_advert_hearings h using analytics_hourly_advert_hearings_pkey scan=Forward", + " Hash", + " CTE Scan" +] diff --git a/db/planguard/baseline/fixture/s7.tn.all625_7d_today.auto.json b/db/planguard/baseline/fixture/s7.tn.all625_7d_today.auto.json new file mode 100644 index 0000000..cadf59d --- /dev/null +++ b/db/planguard/baseline/fixture/s7.tn.all625_7d_today.auto.json @@ -0,0 +1,10 @@ +[ + "Nested Loop (Left)", + " Limit", + " Sort", + " Aggregate partial=Finalize {Hashed}", + " Gather", + " Aggregate partial=Partial {Hashed}", + " Parallel Index Scan on analytics_hourly_advert_hearings using analytics_hourly_advert_hearings_pkey scan=Forward", + " Index Scan on nodes n using nodes_public_key_key scan=Forward" +] diff --git a/db/planguard/baseline/fixture/s7.tn.all625_7d_today.custom.json b/db/planguard/baseline/fixture/s7.tn.all625_7d_today.custom.json new file mode 100644 index 0000000..8f67a90 --- /dev/null +++ b/db/planguard/baseline/fixture/s7.tn.all625_7d_today.custom.json @@ -0,0 +1,8 @@ +[ + "Nested Loop (Left)", + " Limit", + " Sort", + " Aggregate partial=Simple {Hashed}", + " Index Scan on analytics_hourly_advert_hearings using analytics_hourly_advert_hearings_pkey scan=Forward", + " Index Scan on nodes n using nodes_public_key_key scan=Forward" +] diff --git a/db/planguard/baseline/fixture/sm.chan.after.custom.json b/db/planguard/baseline/fixture/sm.chan.after.custom.json new file mode 100644 index 0000000..27797f0 --- /dev/null +++ b/db/planguard/baseline/fixture/sm.chan.after.custom.json @@ -0,0 +1,5 @@ +[ + "Limit", + " Sort", + " Seq Scan on channels c" +] diff --git a/db/planguard/baseline/fixture/sm.chan.first.custom.json b/db/planguard/baseline/fixture/sm.chan.first.custom.json new file mode 100644 index 0000000..27797f0 --- /dev/null +++ b/db/planguard/baseline/fixture/sm.chan.first.custom.json @@ -0,0 +1,5 @@ +[ + "Limit", + " Sort", + " Seq Scan on channels c" +] diff --git a/db/planguard/baseline/fixture/sm.chan.sea.custom.json b/db/planguard/baseline/fixture/sm.chan.sea.custom.json new file mode 100644 index 0000000..81a9682 --- /dev/null +++ b/db/planguard/baseline/fixture/sm.chan.sea.custom.json @@ -0,0 +1,10 @@ +[ + "Limit", + " Sort", + " Seq Scan on channels c", + " Seq Scan on meshmapper_channel_members m [SubPlan]", + " Seq Scan on channel_config_scopes s [SubPlan]", + " Nested Loop [SubPlan]", + " Seq Scan on regions r", + " Seq Scan on region_iatas ri" +] diff --git a/db/planguard/baseline/fixture/sm.drift.SEA_t0_500.custom.json b/db/planguard/baseline/fixture/sm.drift.SEA_t0_500.custom.json new file mode 100644 index 0000000..2939058 --- /dev/null +++ b/db/planguard/baseline/fixture/sm.drift.SEA_t0_500.custom.json @@ -0,0 +1,12 @@ +[ + "Sort", + " Subquery Scan", + " Limit [Subquery]", + " Sort", + " Seq Scan on nodes n_1", + " Bitmap Heap Scan on node_iatas [SubPlan]", + " Bitmap Index Scan using idx_node_iatas_iata", + " Aggregate partial=Simple [SubPlan]", + " Sort", + " Index Scan on node_iatas ni using node_iatas_pkey scan=Forward" +] diff --git a/db/planguard/baseline/fixture/sm.drift.all.custom.json b/db/planguard/baseline/fixture/sm.drift.all.custom.json new file mode 100644 index 0000000..c78b8de --- /dev/null +++ b/db/planguard/baseline/fixture/sm.drift.all.custom.json @@ -0,0 +1,10 @@ +[ + "Sort", + " Subquery Scan", + " Limit [Subquery]", + " Sort", + " Seq Scan on nodes n_1", + " Aggregate partial=Simple [SubPlan]", + " Sort", + " Index Scan on node_iatas ni using node_iatas_pkey scan=Forward" +] diff --git a/db/planguard/baseline/fixture/sm.drift.bogus5000.custom.json b/db/planguard/baseline/fixture/sm.drift.bogus5000.custom.json new file mode 100644 index 0000000..0743b83 --- /dev/null +++ b/db/planguard/baseline/fixture/sm.drift.bogus5000.custom.json @@ -0,0 +1,11 @@ +[ + "Sort", + " Subquery Scan", + " Limit [Subquery]", + " Sort", + " Seq Scan on nodes n_1", + " Seq Scan on node_iatas [SubPlan]", + " Aggregate partial=Simple [SubPlan]", + " Sort", + " Index Scan on node_iatas ni using node_iatas_pkey scan=Forward" +] diff --git a/db/planguard/baseline/fixture/sm.node.top_sea.auto.json b/db/planguard/baseline/fixture/sm.node.top_sea.auto.json new file mode 100644 index 0000000..7590f01 --- /dev/null +++ b/db/planguard/baseline/fixture/sm.node.top_sea.auto.json @@ -0,0 +1,19 @@ +[ + "Nested Loop (Left)", + " Index Scan on nodes n using nodes_pkey scan=Forward", + " Index Scan on transport_scopes ts using transport_scopes_pkey scan=Forward", + " Index Only Scan on observers o using observers_public_key_key scan=Forward [SubPlan]", + " Limit [SubPlan]", + " Index Scan on observers o_1 using observers_public_key_key scan=Forward", + " Aggregate partial=Simple [SubPlan]", + " Sort", + " Index Scan on node_iatas ni using node_iatas_pkey scan=Forward", + " Aggregate partial=Simple [SubPlan]", + " Sort", + " Seq Scan on node_neighbors nn", + " Limit [SubPlan]", + " Sort", + " Nested Loop", + " Seq Scan on meshmapper_repeaters m", + " Seq Scan on meshmapper_repeater_sources s" +] diff --git a/db/planguard/baseline/fixture/sm.node.top_sea.custom.json b/db/planguard/baseline/fixture/sm.node.top_sea.custom.json new file mode 100644 index 0000000..7590f01 --- /dev/null +++ b/db/planguard/baseline/fixture/sm.node.top_sea.custom.json @@ -0,0 +1,19 @@ +[ + "Nested Loop (Left)", + " Index Scan on nodes n using nodes_pkey scan=Forward", + " Index Scan on transport_scopes ts using transport_scopes_pkey scan=Forward", + " Index Only Scan on observers o using observers_public_key_key scan=Forward [SubPlan]", + " Limit [SubPlan]", + " Index Scan on observers o_1 using observers_public_key_key scan=Forward", + " Aggregate partial=Simple [SubPlan]", + " Sort", + " Index Scan on node_iatas ni using node_iatas_pkey scan=Forward", + " Aggregate partial=Simple [SubPlan]", + " Sort", + " Seq Scan on node_neighbors nn", + " Limit [SubPlan]", + " Sort", + " Nested Loop", + " Seq Scan on meshmapper_repeaters m", + " Seq Scan on meshmapper_repeater_sources s" +] diff --git a/db/planguard/baseline/fixture/sm.node.top_sea.generic.json b/db/planguard/baseline/fixture/sm.node.top_sea.generic.json new file mode 100644 index 0000000..7590f01 --- /dev/null +++ b/db/planguard/baseline/fixture/sm.node.top_sea.generic.json @@ -0,0 +1,19 @@ +[ + "Nested Loop (Left)", + " Index Scan on nodes n using nodes_pkey scan=Forward", + " Index Scan on transport_scopes ts using transport_scopes_pkey scan=Forward", + " Index Only Scan on observers o using observers_public_key_key scan=Forward [SubPlan]", + " Limit [SubPlan]", + " Index Scan on observers o_1 using observers_public_key_key scan=Forward", + " Aggregate partial=Simple [SubPlan]", + " Sort", + " Index Scan on node_iatas ni using node_iatas_pkey scan=Forward", + " Aggregate partial=Simple [SubPlan]", + " Sort", + " Seq Scan on node_neighbors nn", + " Limit [SubPlan]", + " Sort", + " Nested Loop", + " Seq Scan on meshmapper_repeaters m", + " Seq Scan on meshmapper_repeater_sources s" +] diff --git a/db/planguard/baseline/fixture/sm.node.unknown.auto.json b/db/planguard/baseline/fixture/sm.node.unknown.auto.json new file mode 100644 index 0000000..7590f01 --- /dev/null +++ b/db/planguard/baseline/fixture/sm.node.unknown.auto.json @@ -0,0 +1,19 @@ +[ + "Nested Loop (Left)", + " Index Scan on nodes n using nodes_pkey scan=Forward", + " Index Scan on transport_scopes ts using transport_scopes_pkey scan=Forward", + " Index Only Scan on observers o using observers_public_key_key scan=Forward [SubPlan]", + " Limit [SubPlan]", + " Index Scan on observers o_1 using observers_public_key_key scan=Forward", + " Aggregate partial=Simple [SubPlan]", + " Sort", + " Index Scan on node_iatas ni using node_iatas_pkey scan=Forward", + " Aggregate partial=Simple [SubPlan]", + " Sort", + " Seq Scan on node_neighbors nn", + " Limit [SubPlan]", + " Sort", + " Nested Loop", + " Seq Scan on meshmapper_repeaters m", + " Seq Scan on meshmapper_repeater_sources s" +] diff --git a/db/planguard/baseline/fixture/sm.node.unknown.custom.json b/db/planguard/baseline/fixture/sm.node.unknown.custom.json new file mode 100644 index 0000000..7590f01 --- /dev/null +++ b/db/planguard/baseline/fixture/sm.node.unknown.custom.json @@ -0,0 +1,19 @@ +[ + "Nested Loop (Left)", + " Index Scan on nodes n using nodes_pkey scan=Forward", + " Index Scan on transport_scopes ts using transport_scopes_pkey scan=Forward", + " Index Only Scan on observers o using observers_public_key_key scan=Forward [SubPlan]", + " Limit [SubPlan]", + " Index Scan on observers o_1 using observers_public_key_key scan=Forward", + " Aggregate partial=Simple [SubPlan]", + " Sort", + " Index Scan on node_iatas ni using node_iatas_pkey scan=Forward", + " Aggregate partial=Simple [SubPlan]", + " Sort", + " Seq Scan on node_neighbors nn", + " Limit [SubPlan]", + " Sort", + " Nested Loop", + " Seq Scan on meshmapper_repeaters m", + " Seq Scan on meshmapper_repeater_sources s" +] diff --git a/db/planguard/baseline/fixture/sm.node.unknown.generic.json b/db/planguard/baseline/fixture/sm.node.unknown.generic.json new file mode 100644 index 0000000..7590f01 --- /dev/null +++ b/db/planguard/baseline/fixture/sm.node.unknown.generic.json @@ -0,0 +1,19 @@ +[ + "Nested Loop (Left)", + " Index Scan on nodes n using nodes_pkey scan=Forward", + " Index Scan on transport_scopes ts using transport_scopes_pkey scan=Forward", + " Index Only Scan on observers o using observers_public_key_key scan=Forward [SubPlan]", + " Limit [SubPlan]", + " Index Scan on observers o_1 using observers_public_key_key scan=Forward", + " Aggregate partial=Simple [SubPlan]", + " Sort", + " Index Scan on node_iatas ni using node_iatas_pkey scan=Forward", + " Aggregate partial=Simple [SubPlan]", + " Sort", + " Seq Scan on node_neighbors nn", + " Limit [SubPlan]", + " Sort", + " Nested Loop", + " Seq Scan on meshmapper_repeaters m", + " Seq Scan on meshmapper_repeater_sources s" +] diff --git a/db/planguard/baseline/fixture/sm.ntypes.SEA.custom.json b/db/planguard/baseline/fixture/sm.ntypes.SEA.custom.json new file mode 100644 index 0000000..6624333 --- /dev/null +++ b/db/planguard/baseline/fixture/sm.ntypes.SEA.custom.json @@ -0,0 +1,9 @@ +[ + "Sort", + " Aggregate partial=Simple {Hashed}", + " Hash Join (Semi)", + " Seq Scan on nodes n", + " Hash", + " Bitmap Heap Scan on node_iatas ni", + " Bitmap Index Scan using idx_node_iatas_iata" +] diff --git a/db/planguard/baseline/fixture/sm.ntypes.all.custom.json b/db/planguard/baseline/fixture/sm.ntypes.all.custom.json new file mode 100644 index 0000000..9d830eb --- /dev/null +++ b/db/planguard/baseline/fixture/sm.ntypes.all.custom.json @@ -0,0 +1,5 @@ +[ + "Sort", + " Aggregate partial=Simple {Hashed}", + " Seq Scan on nodes n" +] diff --git a/db/planguard/baseline/fixture/sm.ntypes.bogus5000.custom.json b/db/planguard/baseline/fixture/sm.ntypes.bogus5000.custom.json new file mode 100644 index 0000000..3916a0f --- /dev/null +++ b/db/planguard/baseline/fixture/sm.ntypes.bogus5000.custom.json @@ -0,0 +1,9 @@ +[ + "Sort", + " Aggregate partial=Simple {Hashed}", + " Hash Join", + " Seq Scan on nodes n", + " Hash", + " Aggregate partial=Simple {Hashed}", + " Seq Scan on node_iatas ni" +] diff --git a/db/planguard/baseline/fixture/sm.ntypes.list50.custom.json b/db/planguard/baseline/fixture/sm.ntypes.list50.custom.json new file mode 100644 index 0000000..1a1dae3 --- /dev/null +++ b/db/planguard/baseline/fixture/sm.ntypes.list50.custom.json @@ -0,0 +1,9 @@ +[ + "Sort", + " Aggregate partial=Simple {Hashed}", + " Hash Join", + " Seq Scan on nodes n", + " Hash", + " Aggregate partial=Simple {Hashed}", + " Index Scan on node_iatas ni using idx_node_iatas_iata scan=Forward" +] diff --git a/db/planguard/baseline/fixture/sm.write.touch_channel.auto.json b/db/planguard/baseline/fixture/sm.write.touch_channel.auto.json new file mode 100644 index 0000000..0b5ea3a --- /dev/null +++ b/db/planguard/baseline/fixture/sm.write.touch_channel.auto.json @@ -0,0 +1,4 @@ +[ + "ModifyTable on channels", + " Index Scan on channels using channels_pkey scan=Forward" +] diff --git a/db/planguard/baseline/fixture/sm.write.upsert_channel.auto.json b/db/planguard/baseline/fixture/sm.write.upsert_channel.auto.json new file mode 100644 index 0000000..219dbf6 --- /dev/null +++ b/db/planguard/baseline/fixture/sm.write.upsert_channel.auto.json @@ -0,0 +1,4 @@ +[ + "ModifyTable on channels", + " Result" +] diff --git a/db/planguard/baseline/fixture/sm.write.upsert_node_iata.auto.json b/db/planguard/baseline/fixture/sm.write.upsert_node_iata.auto.json new file mode 100644 index 0000000..1c4cb23 --- /dev/null +++ b/db/planguard/baseline/fixture/sm.write.upsert_node_iata.auto.json @@ -0,0 +1,4 @@ +[ + "ModifyTable on node_iatas", + " Result" +] diff --git a/db/planguard/corpus.yaml b/db/planguard/corpus.yaml new file mode 100644 index 0000000..7dc44d7 --- /dev/null +++ b/db/planguard/corpus.yaml @@ -0,0 +1,4574 @@ +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +# planguard corpus: which statements are planned, with which parameters, and what each plan must look like. +# +# Tiers (one corpus, three databases): +# fixture CI. A synthetic, skewed fixture built through the migrations; plain EXPLAIN only. Checks each plan's +# shape (drivers, Index Cond fragments, forbidden nodes, the sort rule) and its signature against +# baseline/fixture. Budgets are not checked, because nothing is executed. +# local The reference local copy of production. EXPLAIN (ANALYZE, BUFFERS) after the configured executions; +# checks shapes, budgets and baseline/local. This is the release gate. +# prod Production, read-only plain EXPLAIN on the maint pool (custom and generic modes). Checks shapes only; +# statistics drift is what it watches, so it keeps no baseline. +# +# Plan class (statement `plan:`) -> the modes its entries run when they name none: +# cached custom, generic, auto admitted to the generic-plan class (planpolicy.Cached, written by gen-policy); +# must pass every layout, and its entries' shapes must cover its admission profile +# custom custom sent unnamed, so no layout ever plans it generic +# maint custom the maint pool forces custom plans +# ingest auto the ingest pool keeps automatic plan caching +# legacy custom, generic, auto planpolicy.Legacy: runs on api_legacy with automatic plan caching +# harness custom, generic, auto planted regressions and oracles; never served +# +# Verdicts: a failing check is FALSE_POSITIVE (a real regression), a planted entry (expect_fail) that passes is +# MISSED; both fail the run, as does ERROR. RETIRED_INDEX, KNOWN_ISSUE and SKIP are reported only. +# +# A plan change updates the baseline in the same diff: run with --baseline write, review the signature diff and +# commit it with the change that caused it. + +defaults: + big_tables: [known_routes, packet_observations, packets, packet_sites] + executions: 6 + timeout_s: 60 + forbid_sort_above_limit: true + forbid: + - {node: Seq Scan, relations: [big], reason: page statements never scan a big table} + sort_exempt: [latest_observation] + # Indexes the end state drops. A plan that fails only because it uses one of them is RETIRED_INDEX, not a failure. + retired_indexes: [idx_known_routes_iata_last_seen, idx_known_routes_last_seen, idx_known_routes_hop_count, idx_known_routes_reconfirm, + idx_packets_last_heard, idx_packets_payload_type, idx_packets_route_type, idx_packets_scope, + idx_observations_packet, idx_observations_observer_id, idx_channel_messages_channel, idx_channel_messages_sent_brin] + +tiers: + fixture: + search_path: public + # The local copy's planner settings, so fixture plans match the reference copy's. + settings: {effective_cache_size: 5GB, random_page_cost: "1.1", work_mem: 16MB} + local: + search_path: public + prod: {} # no search_path: the maint pool's own + +groups: + # Pending statements plan against an overlay schema on the local copy, which a later change builds. + packets_global: {search_path: {local: "t_s2b, public"}} + packet_sites: {search_path: {local: "t_s2b, public"}} + channel_message_iatas: {search_path: {local: "t_s2b, public"}} + stats_adverts: {search_path: {local: "t_s2b, public"}} + +statements: + # Merged statements, text from db/sqlc. + ListKnownRoutesByIATA: {sqlc: ListKnownRoutesByIATA, plan: cached, admission: region_page} + ListKnownRoutesGlobal: {sqlc: ListKnownRoutesGlobal, plan: cached, admission: global_page} + ListKnownRoutesByIATAHop: {sqlc: ListKnownRoutesByIATAHop, plan: custom} + ListKnownRoutesGlobalHop: {sqlc: ListKnownRoutesGlobalHop, plan: custom} + SearchKnownRoutesByNodes: {sqlc: SearchKnownRoutesByNodes, plan: custom} + GetKnownRoutesByNodes: {sqlc: GetKnownRoutesByNodes, plan: custom} + GetNodeByID: {sqlc: GetNodeByID, plan: cached, admission: by_id} + ListChannels: {sqlc: ListChannels, plan: custom} + ListChannelsAfter: {sqlc: ListChannelsAfter, plan: custom} + GetStatsClockDrift: {sqlc: GetStatsClockDrift, plan: custom} + GetStatsNodeTypesAll: {sqlc: GetStatsNodeTypesAll, plan: custom} + GetStatsNodeTypesByIATAs: {sqlc: GetStatsNodeTypesByIATAs, plan: custom} + DeleteOldPackets: {sqlc: DeleteOldPackets, kind: dml, plan: maint} + DeleteExpiredRoutes: {sqlc: DeleteExpiredRoutes, kind: dml, plan: maint} + DeleteGraceRoutesSlice: {sqlc: DeleteGraceRoutesSlice, kind: dml, plan: maint} + UpsertChannel: {sqlc: UpsertChannel, kind: dml, plan: ingest} + UpsertNodeIATA: {sqlc: UpsertNodeIATA, kind: dml, plan: ingest} + TouchChannel: {sqlc: TouchChannel, kind: dml, plan: ingest} + TouchChannelHashOnly: {sqlc: TouchChannelHashOnly, kind: dml, plan: ingest} + # Read-only parts of maint DML, cut from today's sqlc text (lock clause removed); a drift test keeps them in step. + DeleteExpiredRoutesSelect: {file: sql/derived/DeleteExpiredRoutesSelect.sql, derived_from: DeleteExpiredRoutes, plan: maint} + DeleteGraceRoutesSliceCand: {file: sql/derived/DeleteGraceRoutesSliceCand.sql, derived_from: DeleteGraceRoutesSlice, plan: maint} + DeleteOldPacketsVictims: {file: sql/derived/DeleteOldPacketsVictims.sql, derived_from: DeleteOldPackets, plan: maint} + # Legacy readers (planpolicy.Legacy): they run on api_legacy with automatic plan caching. + GetPathStats: {sqlc: GetPathStats, plan: legacy} + GetTopNodes: {sqlc: GetTopNodes, plan: legacy} + GetStatsTopAdvertisers: {sqlc: GetStatsTopAdvertisers, plan: legacy} + ListChannelMessages: {sqlc: ListChannelMessages, plan: legacy} + GetStatsSeries: {sqlc: GetStatsSeries, plan: legacy} + ListObserverAdverts: {sqlc: ListObserverAdverts, plan: legacy} + # Planted regressions and harness texts: frozen, never served. + ListKnownRoutes_202acc6: {file: sql/planted/ListKnownRoutes_202acc6.sql, plan: harness} + DeleteOldRoutes_202acc6: {file: sql/planted/DeleteOldRoutes_202acc6.sql, kind: dml, plan: harness} + DeleteOldRoutesSelect: {file: sql/planted/DeleteOldRoutesSelect.sql, kind: dml, plan: harness} + DeleteOldRoutesSelectNoLock: {file: sql/planted/DeleteOldRoutesSelectNoLock.sql, plan: harness} + ListPacketsBySitesReadmodels: {file: sql/planted/ListPacketsBySitesReadmodels.sql, group: packet_sites, plan: harness, pending: packet_sites} + OracleBySites: {file: sql/oracle/OracleBySites.sql, plan: harness} + # Not merged yet: frozen end-state texts. They SKIP until their feature lands, which switches them to sqlc:, + # drops pending and commits their baselines in the same diff. + ListPacketsPage: {file: sql/pending/ListPacketsPage.sql, group: packets_global, plan: cached, admission: global_page, pending: packets_global} + ListPacketsByTypes: {file: sql/pending/ListPacketsByTypes.sql, group: packets_global, plan: cached, admission: global_page, pending: packets_global} + ListPacketsByScopes: {file: sql/pending/ListPacketsByScopes.sql, group: packets_global, plan: custom, pending: packets_global} + ListPacketsByScopesTypes: {file: sql/pending/ListPacketsByScopesTypes.sql, group: packets_global, plan: custom, pending: packets_global} + ListPacketsWindow: {file: sql/pending/ListPacketsWindow.sql, group: packets_global, plan: custom, pending: packets_global} + ListPacketsWindowScoped: {file: sql/pending/ListPacketsWindowScoped.sql, group: packets_global, plan: custom, pending: packets_global} + ListNodes_end: {file: sql/pending/ListNodes.sql, group: packets_global, plan: custom, pending: packets_global} + ListObservers_end: {file: sql/pending/ListObservers.sql, group: packets_global, plan: custom, pending: packets_global} + ListPacketsBySite: {file: sql/pending/ListPacketsBySite.sql, group: packet_sites, plan: cached, admission: region_page, pending: packet_sites} + ListPacketsBySites: {file: sql/pending/ListPacketsBySites.sql, group: packet_sites, plan: cached, admission: region_list, pending: packet_sites} + ListPacketsBySitesFirst: {file: sql/pending/ListPacketsBySitesFirst.sql, group: packet_sites, plan: cached, admission: region_list, pending: packet_sites} + ListPacketsBySitesTyped: {file: sql/pending/ListPacketsBySitesTyped.sql, group: packet_sites, plan: cached, admission: region_list, pending: packet_sites} + ListPacketsBySitesTypedFirst: {file: sql/pending/ListPacketsBySitesTypedFirst.sql, group: packet_sites, plan: cached, admission: region_list, pending: packet_sites} + ListPacketsBySitesSince: {file: sql/pending/ListPacketsBySitesSince.sql, group: packet_sites, plan: custom, pending: packet_sites} + ListPacketsBySitesSinceFirst: {file: sql/pending/ListPacketsBySitesSinceFirst.sql, group: packet_sites, plan: custom, pending: packet_sites} + ListPacketsBySitesScoped: {file: sql/pending/ListPacketsBySitesScoped.sql, group: packet_sites, plan: custom, pending: packet_sites} + ListPacketsBySitesScopedFirst: {file: sql/pending/ListPacketsBySitesScopedFirst.sql, group: packet_sites, plan: custom, pending: packet_sites} + FilterKnownSites: {file: sql/pending/FilterKnownSites.sql, group: packet_sites, plan: custom, pending: packet_sites} + SweepPacketSitesFronts: {file: sql/pending/SweepPacketSitesFronts.sql, group: packet_sites, plan: maint, pending: packet_sites} + InsertObservation_end: {file: sql/pending/InsertObservation.sql, kind: dml, group: packet_sites, plan: ingest, pending: packet_sites} + ListChannelMessagesByIATAs: {file: sql/pending/ListChannelMessagesByIATAs.sql, group: channel_message_iatas, plan: cached, admission: region_list, pending: channel_message_iatas} + ListChannelMessagesByIATAsScoped: {file: sql/pending/ListChannelMessagesByIATAsScoped.sql, group: channel_message_iatas, plan: cached, admission: region_list, pending: channel_message_iatas} + ListChannelMessagesPlain: {file: sql/pending/ListChannelMessagesPlain.sql, group: channel_message_iatas, plan: cached, admission: global_page, pending: channel_message_iatas} + ListAllChannelMessagesByIATAs: {file: sql/pending/ListAllChannelMessagesByIATAs.sql, group: channel_message_iatas, plan: cached, admission: region_list, pending: channel_message_iatas} + ListAllChannelMessagesPlain: {file: sql/pending/ListAllChannelMessagesPlain.sql, group: channel_message_iatas, plan: cached, admission: global_page, pending: channel_message_iatas} + ListChannelMessagesPlainScoped: {file: sql/pending/ListChannelMessagesPlainScoped.sql, group: channel_message_iatas, plan: custom, pending: channel_message_iatas} + ListAllChannelMessagesByIATAsScoped: {file: sql/pending/ListAllChannelMessagesByIATAsScoped.sql, group: channel_message_iatas, plan: custom, pending: channel_message_iatas} + ListAllChannelMessagesPlainScoped: {file: sql/pending/ListAllChannelMessagesPlainScoped.sql, group: channel_message_iatas, plan: custom, pending: channel_message_iatas} + ListAllChannelMessagesSince: {file: sql/pending/ListAllChannelMessagesSince.sql, group: channel_message_iatas, plan: custom, pending: channel_message_iatas} + ListAllChannelMessagesByIATAsSince: {file: sql/pending/ListAllChannelMessagesByIATAsSince.sql, group: channel_message_iatas, plan: custom, pending: channel_message_iatas} + ListChannelMessagesSinceWalk: {file: sql/pending/ListChannelMessagesSinceWalk.sql, group: channel_message_iatas, plan: custom, pending: channel_message_iatas} + AddChannelMessageIATA: {file: sql/pending/AddChannelMessageIATA.sql, kind: dml, group: channel_message_iatas, plan: ingest, pending: channel_message_iatas} + FillChannelMessageIATAsByMessages: {file: sql/pending/FillChannelMessageIATAsByMessages.sql, kind: dml, group: channel_message_iatas, plan: maint, pending: channel_message_iatas} + ListObserverAdverts_end: {file: sql/pending/ListObserverAdverts.sql, group: stats_adverts, plan: cached, admission: by_id, pending: stats_adverts} + GetObserverTelemetrySince: {file: sql/pending/GetObserverTelemetrySince.sql, group: stats_adverts, plan: cached, admission: by_id, pending: stats_adverts} + GetStatsSeries_end: {file: sql/pending/GetStatsSeries.sql, group: stats_adverts, plan: custom, pending: stats_adverts} + GetTopNodesProbe: {file: sql/pending/GetTopNodesProbe.sql, group: stats_adverts, plan: custom, pending: stats_adverts} + GetStatsTopAdvertisersProbe: {file: sql/pending/GetStatsTopAdvertisersProbe.sql, group: stats_adverts, plan: custom, pending: stats_adverts} + GetPathStatsProbe: {file: sql/pending/GetPathStatsProbe.sql, group: stats_adverts, plan: custom, pending: stats_adverts} + ListTraceTags_end: {file: sql/pending/ListTraceTags.sql, group: stats_adverts, plan: custom, pending: stats_adverts} + +sort_exempt_sets: + hop_group: + - {relation: known_routes, index: idx_known_routes_hop_iata, index_cond: "hop_count =", reason: "a hop page sorts one (hop, region) group, bounded by the group size"} + latest_observation: + - {relation: packet_observations, index_cond: "packet_hash = p.packet_hash", reason: "the latest-observation LATERAL sorts one packet's observations (a few hundred at most)"} + per_site_message_ranges: + - {relation: packets, index: idx_packets_scope_last_heard, index_cond: "scope_id =", reason: "a scope's message page walks one scope's packets newest first and stops at the page"} + route_search_top_n: + - {relation: known_routes, index_cond: "node_ids &&", reason: "route search and cross sort their GIN matches for the top 500, as today"} + - {relation: known_routes, index: idx_known_routes_iata_keyset, index_cond: "iata = ANY", reason: "the same top 500 when the region list is the narrower path"} + sites_recheck: + - {relation: packet_sites, index_cond: "packet_hash = ps.packet_hash", reason: "the cross-site re-check probes one packet's sites; the per-site ranges each have their own Limit"} + window_branches: + - {relation: packets, index: idx_packets_heard_inverted, reason: "branch 2 reads only the inverted rows (partial index)"} + - {relation: packets, index: idx_packets_type_last_heard, index_cond: "payload_type = ANY", reason: "branch 1's type range is bounded by last_heard_at >= since"} + +# The shapes a cached statement's entries must cover before gen-policy admits it. A statement with a hop parameter +# must also cover rare_hop and absent_hop. +admission_profiles: + region_page: [dense, sparse, empty_region, stale_region, page1, deep, tie] + global_page: [page1, deep, tie] + region_list: [dense, sparse, empty_region, stale_region, page1, deep, tie, giant_list] + by_id: [hit, miss] + +resolvers: + adv_busiest: "SELECT observer_id FROM packet_observations WHERE payload_type = 4 GROUP BY 1 ORDER BY count(*) DESC, observer_id LIMIT 1" + adv_busiest_mid: "SELECT percentile_disc(0.5) WITHIN GROUP (ORDER BY id) FROM packet_observations WHERE payload_type = 4 AND observer_id = (SELECT observer_id FROM packet_observations WHERE payload_type = 4 GROUP BY 1 ORDER BY count(*) DESC, observer_id LIMIT 1)" + adv_dense1: "SELECT observer_id FROM packet_observations GROUP BY 1 HAVING count(*) >= 500 ORDER BY count(*) FILTER (WHERE payload_type = 4)::float8 / count(*) DESC, observer_id LIMIT 1" + adv_dense2: "SELECT observer_id FROM packet_observations GROUP BY 1 HAVING count(*) >= 500 ORDER BY count(*) FILTER (WHERE payload_type = 4)::float8 / count(*) DESC, observer_id OFFSET 1 LIMIT 1" + adv_mid: "SELECT observer_id FROM (SELECT observer_id, row_number() OVER (ORDER BY count(*), observer_id) rn, count(*) OVER () n FROM packet_observations WHERE payload_type = 4 GROUP BY observer_id) s WHERE rn = (n + 1) / 2" + adv_quiet: "SELECT observer_id FROM packet_observations GROUP BY 1 HAVING count(*) FILTER (WHERE payload_type = 4) > 0 ORDER BY count(*) FILTER (WHERE payload_type = 4), count(*) DESC, observer_id LIMIT 1" + adv_sparse: "SELECT observer_id FROM packet_observations GROUP BY 1 HAVING count(*) >= 500 AND count(*) FILTER (WHERE payload_type = 4) > 0 ORDER BY count(*) FILTER (WHERE payload_type = 4)::float8 / count(*), observer_id LIMIT 1" + adv_topobs: "SELECT observer_id FROM packet_observations GROUP BY 1 ORDER BY count(*) DESC, observer_id LIMIT 1" + adv_topobs_mid: "SELECT percentile_disc(0.5) WITHIN GROUP (ORDER BY id) FROM packet_observations WHERE payload_type = 4 AND observer_id = (SELECT observer_id FROM packet_observations GROUP BY 1 ORDER BY count(*) DESC, observer_id LIMIT 1)" + adv_zero: "SELECT observer_id FROM packet_observations GROUP BY 1 HAVING count(*) FILTER (WHERE payload_type = 4) = 0 ORDER BY count(*) DESC, observer_id LIMIT 1" + allbogus: "SELECT array_agg(v ORDER BY v) FROM (SELECT name v FROM transport_scopes UNION ALL SELECT '#zz-bogus-' || g FROM generate_series(1, 5000) g) s" + bogus5000: "SELECT array_agg('#zz-bogus-' || g ORDER BY g) FROM generate_series(1, 5000) g" + bwe_deep_id: "SELECT id FROM known_routes WHERE iata >= 'BWE' AND iata <= 'BWE' ORDER BY iata, date_trunc('milliseconds', last_seen, 'UTC') DESC, id DESC OFFSET 5000 LIMIT 1" + bwe_deep_ts: "SELECT date_trunc('milliseconds', last_seen, 'UTC') FROM known_routes WHERE iata >= 'BWE' AND iata <= 'BWE' ORDER BY iata, date_trunc('milliseconds', last_seen, 'UTC') DESC, id DESC OFFSET 5000 LIMIT 1" + cm_newest_hash: "SELECT packet_hash FROM channel_messages ORDER BY id DESC LIMIT 1" + cmi_all_sites: "SELECT array_agg(DISTINCT iata ORDER BY iata) FROM channel_message_iatas" + cmi_bogus5000: "SELECT array_agg(c ORDER BY c) FROM (SELECT c FROM (SELECT chr(65 + a) || chr(65 + b) || chr(65 + d) c FROM generate_series(0, 25) a, generate_series(0, 25) b, generate_series(0, 25) d) g WHERE c NOT IN (SELECT DISTINCT iata FROM channel_message_iatas) AND c NOT IN (SELECT iata FROM iata_codes) ORDER BY c LIMIT 5000) s" + g50_id: "SELECT id FROM known_routes ORDER BY date_trunc('milliseconds', last_seen, 'UTC') DESC, id DESC OFFSET 49 LIMIT 1" + g50_ts: "SELECT date_trunc('milliseconds', last_seen, 'UTC') FROM known_routes ORDER BY date_trunc('milliseconds', last_seen, 'UTC') DESC, id DESC OFFSET 49 LIMIT 1" + global_tie_id: "SELECT max(id) FROM known_routes WHERE date_trunc('milliseconds', last_seen, 'UTC') = (SELECT ts FROM (SELECT date_trunc('milliseconds', last_seen, 'UTC') ts FROM known_routes ORDER BY date_trunc('milliseconds', last_seen, 'UTC') DESC, id DESC LIMIT 20000) s GROUP BY ts HAVING count(*) > 1 ORDER BY ts DESC LIMIT 1)" + global_tie_ts: "SELECT ts FROM (SELECT date_trunc('milliseconds', last_seen, 'UTC') ts FROM known_routes ORDER BY date_trunc('milliseconds', last_seen, 'UTC') DESC, id DESC LIMIT 20000) s GROUP BY ts HAVING count(*) > 1 ORDER BY ts DESC LIMIT 1" + new50_id: "SELECT id FROM known_routes WHERE iata = 'NEW' ORDER BY iata, date_trunc('milliseconds', last_seen, 'UTC') DESC, id DESC OFFSET 49 LIMIT 1" + new50_ts: "SELECT date_trunc('milliseconds', last_seen, 'UTC') FROM known_routes WHERE iata = 'NEW' ORDER BY iata, date_trunc('milliseconds', last_seen, 'UTC') DESC, id DESC OFFSET 49 LIMIT 1" + nl12_judge_hash: "SELECT packet_hash FROM packet_sites WHERE iata = ANY('{AMS,EIN,ENS,GLZ,GRQ,LEY,LWR,MST,QAR,RTM,UDE,UTC}') AND site_heard_at <= timestamptz '2026-10-18 15:40:39.570763+00' ORDER BY site_heard_at DESC, packet_hash DESC LIMIT 1" + nl12_judge_ts: "SELECT site_heard_at FROM packet_sites WHERE iata = ANY('{AMS,EIN,ENS,GLZ,GRQ,LEY,LWR,MST,QAR,RTM,UDE,UTC}') AND site_heard_at <= timestamptz '2026-10-18 15:40:39.570763+00' ORDER BY site_heard_at DESC, packet_hash DESC LIMIT 1" + nodes_p500_id: "SELECT id FROM nodes ORDER BY last_seen DESC, id DESC OFFSET 500 LIMIT 1" + nodes_p500_ts: "SELECT last_seen FROM nodes ORDER BY last_seen DESC, id DESC OFFSET 500 LIMIT 1" + obs_newest_hash: "SELECT packet_hash FROM packet_observations ORDER BY id DESC LIMIT 1" + obs_newest_observer: "SELECT observer_id FROM packet_observations ORDER BY id DESC LIMIT 1" + pk_deep_hash: "SELECT packet_hash FROM packets ORDER BY last_heard_at DESC, packet_hash DESC OFFSET 100000 LIMIT 1" + pk_deep_ts: "SELECT last_heard_at FROM packets ORDER BY last_heard_at DESC, packet_hash DESC OFFSET 100000 LIMIT 1" + ps_all_sites: "SELECT array_agg(DISTINCT iata ORDER BY iata) FROM packet_sites WHERE iata <> 'NEW'" + ps_bogus5000: "SELECT array_agg(c ORDER BY c) FROM (SELECT c FROM (SELECT chr(65 + a) || chr(65 + b) || chr(65 + d) c FROM generate_series(0, 25) a, generate_series(0, 25) b, generate_series(0, 25) d) g WHERE c NOT IN (SELECT DISTINCT iata FROM packet_sites) AND c NOT IN (SELECT iata FROM iata_codes) ORDER BY c LIMIT 5000) s" + ps_kept: "SELECT array_agg(DISTINCT v ORDER BY v) FROM (SELECT ri.iata::text v FROM region_iatas ri JOIN regions r ON r.id = ri.region_id WHERE r.slug IN ('nl', 'pnw', 'gbr', 'cz', 'swe') UNION ALL SELECT unnest('{STR,MHG,FRA,CGN,DTM,ESS,DUS,BWE,RIX,YOW,MAD,DSM,ZZZ}'::text[])) s" + ps_list_cz: "SELECT array_agg(ri.iata::text ORDER BY ri.iata) FROM region_iatas ri JOIN regions r ON r.id = ri.region_id WHERE r.slug = 'cz'" + ps_list_dash8: "SELECT '{#pnw,#wa,#w-wa,#thurston,#pierce,#oly,#rainier,#tumwater}'::text[]" + ps_list_gbr: "SELECT array_agg(ri.iata::text ORDER BY ri.iata) FROM region_iatas ri JOIN regions r ON r.id = ri.region_id WHERE r.slug = 'gbr'" + ps_list_nl: "SELECT array_agg(ri.iata::text ORDER BY ri.iata) FROM region_iatas ri JOIN regions r ON r.id = ri.region_id WHERE r.slug = 'nl'" + ps_list_pathfinder7: "SELECT '{STR,MHG,FRA,CGN,DTM,ESS,DUS}'::text[]" + ps_list_pnw: "SELECT array_agg(ri.iata::text ORDER BY ri.iata) FROM region_iatas ri JOIN regions r ON r.id = ri.region_id WHERE r.slug = 'pnw'" + ps_list_scopes1258: "SELECT array_agg(name ORDER BY name) FROM (SELECT name FROM transport_scopes ORDER BY name LIMIT 1258) s" + ps_list_scopes_sea41: "SELECT array_agg(name ORDER BY name) FROM (SELECT name FROM transport_scopes ORDER BY name LIMIT 41) s" + ps_pnw_tie_hash: "SELECT packet_hash FROM packet_sites WHERE iata = ANY('{BLI,CVO,KLS,LMT,MFR,MMV,ONP,PAE,PDX,RDM,SEA,SLE,TTD}') AND site_heard_at = (SELECT site_heard_at FROM packet_sites WHERE iata = ANY('{BLI,CVO,KLS,LMT,MFR,MMV,ONP,PAE,PDX,RDM,SEA,SLE,TTD}') AND site_heard_at > timestamptz '2026-10-18 06:00+00' GROUP BY 1 HAVING count(DISTINCT packet_hash) > 1 ORDER BY 1 DESC LIMIT 1) ORDER BY packet_hash DESC LIMIT 1" + ps_pnw_tie_ts: "SELECT site_heard_at FROM packet_sites WHERE iata = ANY('{BLI,CVO,KLS,LMT,MFR,MMV,ONP,PAE,PDX,RDM,SEA,SLE,TTD}') AND site_heard_at > timestamptz '2026-10-18 06:00+00' GROUP BY 1 HAVING count(DISTINCT packet_hash) > 1 ORDER BY 1 DESC LIMIT 1" + ps_sea_tie_hash: "SELECT packet_hash FROM packet_sites WHERE iata = 'SEA' AND site_heard_at = (SELECT site_heard_at FROM packet_sites WHERE iata = 'SEA' AND site_heard_at > timestamptz '2026-10-18 12:00+00' GROUP BY 1 HAVING count(*) > 1 ORDER BY 1 DESC LIMIT 1) ORDER BY packet_hash DESC LIMIT 1" + ps_sea_tie_ts: "SELECT site_heard_at FROM packet_sites WHERE iata = 'SEA' AND site_heard_at > timestamptz '2026-10-18 12:00+00' GROUP BY 1 HAVING count(*) > 1 ORDER BY 1 DESC LIMIT 1" + s_1410_3d67_RTM_f: "WITH cand AS (SELECT h.hx, ns.iata, ns.node_id FROM (VALUES ('1410'), ('3d67')) h(hx) JOIN node_short_ids ns ON (length(h.hx) = 2 AND ns.prefix_1 = decode(h.hx, 'hex')) OR (length(h.hx) = 4 AND ns.prefix_2 = decode(h.hx, 'hex')) JOIN nodes n ON n.id = ns.node_id AND n.node_type IN (2, 3) WHERE ns.iata IN (SELECT iata FROM iata_codes)) SELECT array_agg(c.node_id ORDER BY c.iata, c.node_id) FROM cand c WHERE c.hx = '1410' AND c.iata = ANY('{RTM}'::bpchar[])" + s_1410_3d67_RTM_i: "WITH cand AS (SELECT h.hx, ns.iata, ns.node_id FROM (VALUES ('1410'), ('3d67')) h(hx) JOIN node_short_ids ns ON (length(h.hx) = 2 AND ns.prefix_1 = decode(h.hx, 'hex')) OR (length(h.hx) = 4 AND ns.prefix_2 = decode(h.hx, 'hex')) JOIN nodes n ON n.id = ns.node_id AND n.node_type IN (2, 3) WHERE ns.iata IN (SELECT iata FROM iata_codes)) SELECT array_agg(DISTINCT a.iata ORDER BY a.iata) FROM cand a JOIN cand b ON b.iata = a.iata AND b.hx = '3d67' WHERE a.hx = '1410' AND a.iata = ANY('{RTM}'::bpchar[])" + s_1410_3d67_RTM_t: "WITH cand AS (SELECT h.hx, ns.iata, ns.node_id FROM (VALUES ('1410'), ('3d67')) h(hx) JOIN node_short_ids ns ON (length(h.hx) = 2 AND ns.prefix_1 = decode(h.hx, 'hex')) OR (length(h.hx) = 4 AND ns.prefix_2 = decode(h.hx, 'hex')) JOIN nodes n ON n.id = ns.node_id AND n.node_type IN (2, 3) WHERE ns.iata IN (SELECT iata FROM iata_codes)) SELECT array_agg(c.node_id ORDER BY c.iata, c.node_id) FROM cand c WHERE c.hx = '3d67' AND c.iata = ANY('{RTM}'::bpchar[])" + s_1410_3d67_all_f: "WITH cand AS (SELECT h.hx, ns.iata, ns.node_id FROM (VALUES ('1410'), ('3d67')) h(hx) JOIN node_short_ids ns ON (length(h.hx) = 2 AND ns.prefix_1 = decode(h.hx, 'hex')) OR (length(h.hx) = 4 AND ns.prefix_2 = decode(h.hx, 'hex')) JOIN nodes n ON n.id = ns.node_id AND n.node_type IN (2, 3) WHERE ns.iata IN (SELECT iata FROM iata_codes)) SELECT array_agg(c.node_id ORDER BY c.iata, c.node_id) FROM cand c WHERE c.hx = '1410'" + s_1410_3d67_all_i: "WITH cand AS (SELECT h.hx, ns.iata, ns.node_id FROM (VALUES ('1410'), ('3d67')) h(hx) JOIN node_short_ids ns ON (length(h.hx) = 2 AND ns.prefix_1 = decode(h.hx, 'hex')) OR (length(h.hx) = 4 AND ns.prefix_2 = decode(h.hx, 'hex')) JOIN nodes n ON n.id = ns.node_id AND n.node_type IN (2, 3) WHERE ns.iata IN (SELECT iata FROM iata_codes)) SELECT array_agg(DISTINCT a.iata ORDER BY a.iata) FROM cand a JOIN cand b ON b.iata = a.iata AND b.hx = '3d67' WHERE a.hx = '1410'" + s_1410_3d67_all_t: "WITH cand AS (SELECT h.hx, ns.iata, ns.node_id FROM (VALUES ('1410'), ('3d67')) h(hx) JOIN node_short_ids ns ON (length(h.hx) = 2 AND ns.prefix_1 = decode(h.hx, 'hex')) OR (length(h.hx) = 4 AND ns.prefix_2 = decode(h.hx, 'hex')) JOIN nodes n ON n.id = ns.node_id AND n.node_type IN (2, 3) WHERE ns.iata IN (SELECT iata FROM iata_codes)) SELECT array_agg(c.node_id ORDER BY c.iata, c.node_id) FROM cand c WHERE c.hx = '3d67'" + s_14_3d_RTM_AMS_EIN_f: "WITH cand AS (SELECT h.hx, ns.iata, ns.node_id FROM (VALUES ('14'), ('3d')) h(hx) JOIN node_short_ids ns ON (length(h.hx) = 2 AND ns.prefix_1 = decode(h.hx, 'hex')) OR (length(h.hx) = 4 AND ns.prefix_2 = decode(h.hx, 'hex')) JOIN nodes n ON n.id = ns.node_id AND n.node_type IN (2, 3) WHERE ns.iata IN (SELECT iata FROM iata_codes)) SELECT array_agg(c.node_id ORDER BY c.iata, c.node_id) FROM cand c WHERE c.hx = '14' AND c.iata = ANY('{RTM,AMS,EIN}'::bpchar[])" + s_14_3d_RTM_AMS_EIN_i: "WITH cand AS (SELECT h.hx, ns.iata, ns.node_id FROM (VALUES ('14'), ('3d')) h(hx) JOIN node_short_ids ns ON (length(h.hx) = 2 AND ns.prefix_1 = decode(h.hx, 'hex')) OR (length(h.hx) = 4 AND ns.prefix_2 = decode(h.hx, 'hex')) JOIN nodes n ON n.id = ns.node_id AND n.node_type IN (2, 3) WHERE ns.iata IN (SELECT iata FROM iata_codes)) SELECT array_agg(DISTINCT a.iata ORDER BY a.iata) FROM cand a JOIN cand b ON b.iata = a.iata AND b.hx = '3d' WHERE a.hx = '14' AND a.iata = ANY('{RTM,AMS,EIN}'::bpchar[])" + s_14_3d_RTM_AMS_EIN_t: "WITH cand AS (SELECT h.hx, ns.iata, ns.node_id FROM (VALUES ('14'), ('3d')) h(hx) JOIN node_short_ids ns ON (length(h.hx) = 2 AND ns.prefix_1 = decode(h.hx, 'hex')) OR (length(h.hx) = 4 AND ns.prefix_2 = decode(h.hx, 'hex')) JOIN nodes n ON n.id = ns.node_id AND n.node_type IN (2, 3) WHERE ns.iata IN (SELECT iata FROM iata_codes)) SELECT array_agg(c.node_id ORDER BY c.iata, c.node_id) FROM cand c WHERE c.hx = '3d' AND c.iata = ANY('{RTM,AMS,EIN}'::bpchar[])" + s_14_3d_all_f: "WITH cand AS (SELECT h.hx, ns.iata, ns.node_id FROM (VALUES ('14'), ('3d')) h(hx) JOIN node_short_ids ns ON (length(h.hx) = 2 AND ns.prefix_1 = decode(h.hx, 'hex')) OR (length(h.hx) = 4 AND ns.prefix_2 = decode(h.hx, 'hex')) JOIN nodes n ON n.id = ns.node_id AND n.node_type IN (2, 3) WHERE ns.iata IN (SELECT iata FROM iata_codes)) SELECT array_agg(c.node_id ORDER BY c.iata, c.node_id) FROM cand c WHERE c.hx = '14'" + s_14_3d_all_i: "WITH cand AS (SELECT h.hx, ns.iata, ns.node_id FROM (VALUES ('14'), ('3d')) h(hx) JOIN node_short_ids ns ON (length(h.hx) = 2 AND ns.prefix_1 = decode(h.hx, 'hex')) OR (length(h.hx) = 4 AND ns.prefix_2 = decode(h.hx, 'hex')) JOIN nodes n ON n.id = ns.node_id AND n.node_type IN (2, 3) WHERE ns.iata IN (SELECT iata FROM iata_codes)) SELECT array_agg(DISTINCT a.iata ORDER BY a.iata) FROM cand a JOIN cand b ON b.iata = a.iata AND b.hx = '3d' WHERE a.hx = '14'" + s_14_3d_all_t: "WITH cand AS (SELECT h.hx, ns.iata, ns.node_id FROM (VALUES ('14'), ('3d')) h(hx) JOIN node_short_ids ns ON (length(h.hx) = 2 AND ns.prefix_1 = decode(h.hx, 'hex')) OR (length(h.hx) = 4 AND ns.prefix_2 = decode(h.hx, 'hex')) JOIN nodes n ON n.id = ns.node_id AND n.node_type IN (2, 3) WHERE ns.iata IN (SELECT iata FROM iata_codes)) SELECT array_agg(c.node_id ORDER BY c.iata, c.node_id) FROM cand c WHERE c.hx = '3d'" + s_5731_0355_all_f: "WITH cand AS (SELECT h.hx, ns.iata, ns.node_id FROM (VALUES ('5731'), ('0355')) h(hx) JOIN node_short_ids ns ON (length(h.hx) = 2 AND ns.prefix_1 = decode(h.hx, 'hex')) OR (length(h.hx) = 4 AND ns.prefix_2 = decode(h.hx, 'hex')) JOIN nodes n ON n.id = ns.node_id AND n.node_type IN (2, 3) WHERE ns.iata IN (SELECT iata FROM iata_codes)) SELECT array_agg(c.node_id ORDER BY c.iata, c.node_id) FROM cand c WHERE c.hx = '5731'" + s_5731_0355_all_i: "WITH cand AS (SELECT h.hx, ns.iata, ns.node_id FROM (VALUES ('5731'), ('0355')) h(hx) JOIN node_short_ids ns ON (length(h.hx) = 2 AND ns.prefix_1 = decode(h.hx, 'hex')) OR (length(h.hx) = 4 AND ns.prefix_2 = decode(h.hx, 'hex')) JOIN nodes n ON n.id = ns.node_id AND n.node_type IN (2, 3) WHERE ns.iata IN (SELECT iata FROM iata_codes)) SELECT array_agg(DISTINCT a.iata ORDER BY a.iata) FROM cand a JOIN cand b ON b.iata = a.iata AND b.hx = '0355' WHERE a.hx = '5731'" + s_5731_0355_all_t: "WITH cand AS (SELECT h.hx, ns.iata, ns.node_id FROM (VALUES ('5731'), ('0355')) h(hx) JOIN node_short_ids ns ON (length(h.hx) = 2 AND ns.prefix_1 = decode(h.hx, 'hex')) OR (length(h.hx) = 4 AND ns.prefix_2 = decode(h.hx, 'hex')) JOIN nodes n ON n.id = ns.node_id AND n.node_type IN (2, 3) WHERE ns.iata IN (SELECT iata FROM iata_codes)) SELECT array_agg(c.node_id ORDER BY c.iata, c.node_id) FROM cand c WHERE c.hx = '0355'" + s_f877_0867_all_f: "WITH cand AS (SELECT h.hx, ns.iata, ns.node_id FROM (VALUES ('f877'), ('0867')) h(hx) JOIN node_short_ids ns ON (length(h.hx) = 2 AND ns.prefix_1 = decode(h.hx, 'hex')) OR (length(h.hx) = 4 AND ns.prefix_2 = decode(h.hx, 'hex')) JOIN nodes n ON n.id = ns.node_id AND n.node_type IN (2, 3) WHERE ns.iata IN (SELECT iata FROM iata_codes)) SELECT array_agg(c.node_id ORDER BY c.iata, c.node_id) FROM cand c WHERE c.hx = 'f877'" + s_f877_0867_all_i: "WITH cand AS (SELECT h.hx, ns.iata, ns.node_id FROM (VALUES ('f877'), ('0867')) h(hx) JOIN node_short_ids ns ON (length(h.hx) = 2 AND ns.prefix_1 = decode(h.hx, 'hex')) OR (length(h.hx) = 4 AND ns.prefix_2 = decode(h.hx, 'hex')) JOIN nodes n ON n.id = ns.node_id AND n.node_type IN (2, 3) WHERE ns.iata IN (SELECT iata FROM iata_codes)) SELECT array_agg(DISTINCT a.iata ORDER BY a.iata) FROM cand a JOIN cand b ON b.iata = a.iata AND b.hx = '0867' WHERE a.hx = 'f877'" + s_f877_0867_all_t: "WITH cand AS (SELECT h.hx, ns.iata, ns.node_id FROM (VALUES ('f877'), ('0867')) h(hx) JOIN node_short_ids ns ON (length(h.hx) = 2 AND ns.prefix_1 = decode(h.hx, 'hex')) OR (length(h.hx) = 4 AND ns.prefix_2 = decode(h.hx, 'hex')) JOIN nodes n ON n.id = ns.node_id AND n.node_type IN (2, 3) WHERE ns.iata IN (SELECT iata FROM iata_codes)) SELECT array_agg(c.node_id ORDER BY c.iata, c.node_id) FROM cand c WHERE c.hx = '0867'" + # The SEA relay cases read only the newest 50,000 SEA routes through the region keyset index, so they stay + # far below the prod tier's 20 s cap (0.1-0.4 s each on the reference copy). The rare relay is also held to + # 20-80 SEA routes in total, counted among at most 500 of its routes through the node_ids GIN index for the 50 + # rarest recent candidates only. + s_sea_pop_f: "SELECT ARRAY[p.n] FROM (SELECT u.n FROM (SELECT kr.node_ids FROM known_routes kr WHERE kr.iata >= 'SEA' AND kr.iata <= 'SEA' ORDER BY kr.iata, date_trunc('milliseconds', kr.last_seen, 'UTC') DESC, kr.id DESC LIMIT 50000) r, unnest(r.node_ids) u(n) WHERE u.n IN (SELECT ns.node_id FROM node_short_ids ns JOIN nodes n ON n.id = ns.node_id AND n.node_type IN (2, 3) WHERE ns.iata = 'SEA') GROUP BY u.n ORDER BY count(*) DESC, u.n OFFSET 0 LIMIT 1) p" + s_sea_pop_t: "SELECT ARRAY[p.n] FROM (SELECT u.n FROM (SELECT kr.node_ids FROM known_routes kr WHERE kr.iata >= 'SEA' AND kr.iata <= 'SEA' ORDER BY kr.iata, date_trunc('milliseconds', kr.last_seen, 'UTC') DESC, kr.id DESC LIMIT 50000) r, unnest(r.node_ids) u(n) WHERE u.n IN (SELECT ns.node_id FROM node_short_ids ns JOIN nodes n ON n.id = ns.node_id AND n.node_type IN (2, 3) WHERE ns.iata = 'SEA') GROUP BY u.n ORDER BY count(*) DESC, u.n OFFSET 1 LIMIT 1) p" + s_sea_prefix_2525: "SELECT array_agg(ns.node_id ORDER BY ns.node_id) FROM node_short_ids ns JOIN nodes n ON n.id = ns.node_id AND n.node_type IN (2, 3) WHERE ns.iata = 'SEA' AND ns.prefix_2 = '\\x2525'" + s_sea_prefix_fade: "SELECT array_agg(ns.node_id ORDER BY ns.node_id) FROM node_short_ids ns JOIN nodes n ON n.id = ns.node_id AND n.node_type IN (2, 3) WHERE ns.iata = 'SEA' AND ns.prefix_2 = '\\xfade'" + s_sea_rare_t: "WITH recent AS MATERIALIZED (SELECT u.n, count(*) c FROM (SELECT kr.node_ids FROM known_routes kr WHERE kr.iata >= 'SEA' AND kr.iata <= 'SEA' ORDER BY kr.iata, date_trunc('milliseconds', kr.last_seen, 'UTC') DESC, kr.id DESC LIMIT 50000) r, unnest(r.node_ids) u(n) GROUP BY u.n HAVING count(*) >= 2), cand AS MATERIALIZED (SELECT c.n, c.c FROM recent c JOIN node_short_ids ns ON ns.node_id = c.n AND ns.iata = 'SEA' WHERE (SELECT count(*) FROM node_short_ids x JOIN nodes nn ON nn.id = x.node_id AND nn.node_type IN (2, 3) WHERE x.iata = 'SEA' AND x.prefix_2 = ns.prefix_2) = 1 ORDER BY c.c, c.n LIMIT 50) SELECT ARRAY[cand.n] FROM cand WHERE (SELECT count(*) FILTER (WHERE t.iata = 'SEA') FROM (SELECT k.iata FROM known_routes k WHERE k.node_ids @> ARRAY[cand.n] LIMIT 500) t) BETWEEN 20 AND 80 ORDER BY cand.c, cand.n LIMIT 1" + sea50_id: "SELECT id FROM known_routes WHERE iata = 'SEA' ORDER BY iata, date_trunc('milliseconds', last_seen, 'UTC') DESC, id DESC OFFSET 49 LIMIT 1" + sea50_ts: "SELECT date_trunc('milliseconds', last_seen, 'UTC') FROM known_routes WHERE iata = 'SEA' ORDER BY iata, date_trunc('milliseconds', last_seen, 'UTC') DESC, id DESC OFFSET 49 LIMIT 1" + sea_i: "SELECT '{SEA}'::bpchar[]" + sea_tie_id: "SELECT max(id) FROM known_routes WHERE iata >= 'SEA' AND iata <= 'SEA' AND date_trunc('milliseconds', last_seen, 'UTC') = (SELECT ts FROM (SELECT date_trunc('milliseconds', last_seen, 'UTC') ts FROM known_routes WHERE iata >= 'SEA' AND iata <= 'SEA' ORDER BY iata, date_trunc('milliseconds', last_seen, 'UTC') DESC, id DESC LIMIT 20000) s GROUP BY ts HAVING count(*) > 1 ORDER BY ts DESC LIMIT 1)" + sea_tie_ts: "SELECT ts FROM (SELECT date_trunc('milliseconds', last_seen, 'UTC') ts FROM known_routes WHERE iata >= 'SEA' AND iata <= 'SEA' ORDER BY iata, date_trunc('milliseconds', last_seen, 'UTC') DESC, id DESC LIMIT 20000) s GROUP BY ts HAVING count(*) > 1 ORDER BY ts DESC LIMIT 1" + seah6_id: "SELECT id FROM known_routes WHERE iata = 'SEA' AND hop_count = 6 ORDER BY iata, date_trunc('milliseconds', last_seen, 'UTC') DESC, id DESC OFFSET 199 LIMIT 1" + seah6_ts: "SELECT date_trunc('milliseconds', last_seen, 'UTC') FROM known_routes WHERE iata = 'SEA' AND hop_count = 6 ORDER BY iata, date_trunc('milliseconds', last_seen, 'UTC') DESC, id DESC OFFSET 199 LIMIT 1" + small_bogus5000: "SELECT array_agg(c ORDER BY c) FROM (SELECT c FROM (SELECT chr(65 + a) || chr(65 + b) || chr(65 + d) c FROM generate_series(0, 25) a, generate_series(0, 25) b, generate_series(0, 25) d) g WHERE c NOT IN (SELECT DISTINCT iata FROM node_iatas) AND c NOT IN (SELECT iata FROM iata_codes) ORDER BY c LIMIT 5000) s" + small_list50: "SELECT array_agg(iata ORDER BY n DESC, iata) FROM (SELECT iata, count(*) n FROM node_iatas GROUP BY iata ORDER BY 2 DESC, 1 LIMIT 50) s" + small_top_sea_node: "SELECT ni.node_id FROM node_iatas ni WHERE ni.iata = 'SEA' ORDER BY (SELECT count(*) FROM node_iatas x WHERE x.node_id = ni.node_id) DESC, ni.node_id LIMIT 1" + tel_top: "SELECT observer_id FROM observer_telemetry GROUP BY 1 ORDER BY count(*) DESC, observer_id LIMIT 1" + tel_top_mid: "SELECT percentile_disc(0.5) WITHIN GROUP (ORDER BY id) FROM observer_telemetry WHERE observer_id = (SELECT observer_id FROM observer_telemetry GROUP BY 1 ORDER BY count(*) DESC, observer_id LIMIT 1)" + trc_tie_tag: "SELECT max(trace_tag) FROM trace_tags WHERE date_trunc('milliseconds', last_heard_at, 'UTC') = (SELECT ts FROM (SELECT date_trunc('milliseconds', last_heard_at, 'UTC') ts FROM trace_tags) s GROUP BY ts HAVING count(*) > 1 ORDER BY ts DESC LIMIT 1)" + trc_tie_ts: "SELECT ts FROM (SELECT date_trunc('milliseconds', last_heard_at, 'UTC') ts FROM trace_tags) s GROUP BY ts HAVING count(*) > 1 ORDER BY ts DESC LIMIT 1" + ustie_hash: "SELECT packet_hash FROM packets WHERE last_heard_at = (SELECT last_heard_at FROM packets GROUP BY last_heard_at HAVING count(*) > 1 ORDER BY last_heard_at DESC LIMIT 1) ORDER BY packet_hash DESC LIMIT 1" + ustie_ts: "SELECT last_heard_at FROM packets GROUP BY last_heard_at HAVING count(*) > 1 ORDER BY last_heard_at DESC LIMIT 1" + x_0355_i: "WITH cand AS (SELECT h.hx, ns.iata, ns.node_id FROM (VALUES ('0355')) h(hx) JOIN node_short_ids ns ON (length(h.hx) = 2 AND ns.prefix_1 = decode(h.hx, 'hex')) OR (length(h.hx) = 4 AND ns.prefix_2 = decode(h.hx, 'hex')) JOIN nodes n ON n.id = ns.node_id AND n.node_type IN (2, 3) WHERE ns.iata IN (SELECT iata FROM iata_codes)) SELECT array_agg(u.iata ORDER BY u.iata) FROM (SELECT iata, min(node_id::text)::uuid node_id FROM cand GROUP BY iata HAVING count(*) = 1) u HAVING count(*) <= 50" + x_0355_n: "WITH cand AS (SELECT h.hx, ns.iata, ns.node_id FROM (VALUES ('0355')) h(hx) JOIN node_short_ids ns ON (length(h.hx) = 2 AND ns.prefix_1 = decode(h.hx, 'hex')) OR (length(h.hx) = 4 AND ns.prefix_2 = decode(h.hx, 'hex')) JOIN nodes n ON n.id = ns.node_id AND n.node_type IN (2, 3) WHERE ns.iata IN (SELECT iata FROM iata_codes)) SELECT array_agg(u.node_id ORDER BY u.iata) FROM (SELECT iata, min(node_id::text)::uuid node_id FROM cand GROUP BY iata HAVING count(*) = 1) u HAVING count(*) <= 50" + x_0867_i: "WITH cand AS (SELECT h.hx, ns.iata, ns.node_id FROM (VALUES ('0867')) h(hx) JOIN node_short_ids ns ON (length(h.hx) = 2 AND ns.prefix_1 = decode(h.hx, 'hex')) OR (length(h.hx) = 4 AND ns.prefix_2 = decode(h.hx, 'hex')) JOIN nodes n ON n.id = ns.node_id AND n.node_type IN (2, 3) WHERE ns.iata IN (SELECT iata FROM iata_codes)) SELECT array_agg(u.iata ORDER BY u.iata) FROM (SELECT iata, min(node_id::text)::uuid node_id FROM cand GROUP BY iata HAVING count(*) = 1) u HAVING count(*) <= 50" + x_0867_n: "WITH cand AS (SELECT h.hx, ns.iata, ns.node_id FROM (VALUES ('0867')) h(hx) JOIN node_short_ids ns ON (length(h.hx) = 2 AND ns.prefix_1 = decode(h.hx, 'hex')) OR (length(h.hx) = 4 AND ns.prefix_2 = decode(h.hx, 'hex')) JOIN nodes n ON n.id = ns.node_id AND n.node_type IN (2, 3) WHERE ns.iata IN (SELECT iata FROM iata_codes)) SELECT array_agg(u.node_id ORDER BY u.iata) FROM (SELECT iata, min(node_id::text)::uuid node_id FROM cand GROUP BY iata HAVING count(*) = 1) u HAVING count(*) <= 50" + x_1410_i: "WITH cand AS (SELECT h.hx, ns.iata, ns.node_id FROM (VALUES ('1410')) h(hx) JOIN node_short_ids ns ON (length(h.hx) = 2 AND ns.prefix_1 = decode(h.hx, 'hex')) OR (length(h.hx) = 4 AND ns.prefix_2 = decode(h.hx, 'hex')) JOIN nodes n ON n.id = ns.node_id AND n.node_type IN (2, 3) WHERE ns.iata IN (SELECT iata FROM iata_codes)) SELECT array_agg(u.iata ORDER BY u.iata) FROM (SELECT iata, min(node_id::text)::uuid node_id FROM cand GROUP BY iata HAVING count(*) = 1) u HAVING count(*) <= 50" + x_1410_n: "WITH cand AS (SELECT h.hx, ns.iata, ns.node_id FROM (VALUES ('1410')) h(hx) JOIN node_short_ids ns ON (length(h.hx) = 2 AND ns.prefix_1 = decode(h.hx, 'hex')) OR (length(h.hx) = 4 AND ns.prefix_2 = decode(h.hx, 'hex')) JOIN nodes n ON n.id = ns.node_id AND n.node_type IN (2, 3) WHERE ns.iata IN (SELECT iata FROM iata_codes)) SELECT array_agg(u.node_id ORDER BY u.iata) FROM (SELECT iata, min(node_id::text)::uuid node_id FROM cand GROUP BY iata HAVING count(*) = 1) u HAVING count(*) <= 50" + x_3d67_i: "WITH cand AS (SELECT h.hx, ns.iata, ns.node_id FROM (VALUES ('3d67')) h(hx) JOIN node_short_ids ns ON (length(h.hx) = 2 AND ns.prefix_1 = decode(h.hx, 'hex')) OR (length(h.hx) = 4 AND ns.prefix_2 = decode(h.hx, 'hex')) JOIN nodes n ON n.id = ns.node_id AND n.node_type IN (2, 3) WHERE ns.iata IN (SELECT iata FROM iata_codes)) SELECT array_agg(u.iata ORDER BY u.iata) FROM (SELECT iata, min(node_id::text)::uuid node_id FROM cand GROUP BY iata HAVING count(*) = 1) u HAVING count(*) <= 50" + x_3d67_n: "WITH cand AS (SELECT h.hx, ns.iata, ns.node_id FROM (VALUES ('3d67')) h(hx) JOIN node_short_ids ns ON (length(h.hx) = 2 AND ns.prefix_1 = decode(h.hx, 'hex')) OR (length(h.hx) = 4 AND ns.prefix_2 = decode(h.hx, 'hex')) JOIN nodes n ON n.id = ns.node_id AND n.node_type IN (2, 3) WHERE ns.iata IN (SELECT iata FROM iata_codes)) SELECT array_agg(u.node_id ORDER BY u.iata) FROM (SELECT iata, min(node_id::text)::uuid node_id FROM cand GROUP BY iata HAVING count(*) = 1) u HAVING count(*) <= 50" + x_5731_i: "WITH cand AS (SELECT h.hx, ns.iata, ns.node_id FROM (VALUES ('5731')) h(hx) JOIN node_short_ids ns ON (length(h.hx) = 2 AND ns.prefix_1 = decode(h.hx, 'hex')) OR (length(h.hx) = 4 AND ns.prefix_2 = decode(h.hx, 'hex')) JOIN nodes n ON n.id = ns.node_id AND n.node_type IN (2, 3) WHERE ns.iata IN (SELECT iata FROM iata_codes)) SELECT array_agg(u.iata ORDER BY u.iata) FROM (SELECT iata, min(node_id::text)::uuid node_id FROM cand GROUP BY iata HAVING count(*) = 1) u HAVING count(*) <= 50" + x_5731_n: "WITH cand AS (SELECT h.hx, ns.iata, ns.node_id FROM (VALUES ('5731')) h(hx) JOIN node_short_ids ns ON (length(h.hx) = 2 AND ns.prefix_1 = decode(h.hx, 'hex')) OR (length(h.hx) = 4 AND ns.prefix_2 = decode(h.hx, 'hex')) JOIN nodes n ON n.id = ns.node_id AND n.node_type IN (2, 3) WHERE ns.iata IN (SELECT iata FROM iata_codes)) SELECT array_agg(u.node_id ORDER BY u.iata) FROM (SELECT iata, min(node_id::text)::uuid node_id FROM cand GROUP BY iata HAVING count(*) = 1) u HAVING count(*) <= 50" + x_f877_i: "WITH cand AS (SELECT h.hx, ns.iata, ns.node_id FROM (VALUES ('f877')) h(hx) JOIN node_short_ids ns ON (length(h.hx) = 2 AND ns.prefix_1 = decode(h.hx, 'hex')) OR (length(h.hx) = 4 AND ns.prefix_2 = decode(h.hx, 'hex')) JOIN nodes n ON n.id = ns.node_id AND n.node_type IN (2, 3) WHERE ns.iata IN (SELECT iata FROM iata_codes)) SELECT array_agg(u.iata ORDER BY u.iata) FROM (SELECT iata, min(node_id::text)::uuid node_id FROM cand GROUP BY iata HAVING count(*) = 1) u HAVING count(*) <= 50" + x_f877_n: "WITH cand AS (SELECT h.hx, ns.iata, ns.node_id FROM (VALUES ('f877')) h(hx) JOIN node_short_ids ns ON (length(h.hx) = 2 AND ns.prefix_1 = decode(h.hx, 'hex')) OR (length(h.hx) = 4 AND ns.prefix_2 = decode(h.hx, 'hex')) JOIN nodes n ON n.id = ns.node_id AND n.node_type IN (2, 3) WHERE ns.iata IN (SELECT iata FROM iata_codes)) SELECT array_agg(u.node_id ORDER BY u.iata) FROM (SELECT iata, min(node_id::text)::uuid node_id FROM cand GROUP BY iata HAVING count(*) = 1) u HAVING count(*) <= 50" + +entries: + + # ---- known_routes: region and global pages, hop pages, route search and cross ---- + # Budgets: no-hop pages <= 300, empty region <= 10, absent or single-row global hop <= 10, otherwise 1.5x the + # measured buffers of the shape. + - id: routes.byiata.R01_SEA_first_l50 + stmt: ListKnownRoutesByIATA + shape: "dense region (SEA), page 1, limit 50" + shapes: [dense, page1] + params: [SEA, null, 0, 50] + expect: + driver: [{relation: known_routes, index: idx_known_routes_iata_keyset, index_cond: ["iata >=", "id) < ROW("]}] + budget: {buffers: 80, planning_ms: 10} + note: "measured: 53 B on the full local copy (max over modes)" + - id: routes.byiata.R02_SEA_first_l200 + stmt: ListKnownRoutesByIATA + shape: "dense region, page 1, limit 200" + shapes: [dense, page1] + params: [SEA, null, 0, 200] + expect: + driver: [{relation: known_routes, index: idx_known_routes_iata_keyset, index_cond: ["iata >=", "id) < ROW("]}] + budget: {buffers: 288, planning_ms: 10} + note: "measured: 192 B on the full local copy (max over modes)" + - id: routes.byiata.R03_SEA_page2_l50 + stmt: ListKnownRoutesByIATA + shape: "dense region, page 2 (cursor + cursorId)" + shapes: [dense, cursor] + params: [SEA, {r: sea50_ts}, {r: sea50_id}, 50] + expect: + driver: [{relation: known_routes, index: idx_known_routes_iata_keyset, index_cond: ["iata >=", "id) < ROW("]}] + budget: {buffers: 69, planning_ms: 10} + note: "measured: 46 B on the full local copy (max over modes)" + - id: routes.byiata.R04_SEA_deep_l200 + stmt: ListKnownRoutesByIATA + shape: "dense region, deep cursor-only (legacy, 2026-10-05)" + shapes: [dense, deep] + params: [SEA, "2026-10-05 19:00:00+00", 0, 200] + expect: + driver: [{relation: known_routes, index: idx_known_routes_iata_keyset, index_cond: ["iata >=", "id) < ROW("]}] + budget: {buffers: 305, planning_ms: 10} + note: "measured: 203 B on the full local copy (max over modes)" + - id: routes.byiata.R05_BWE_first_l200 + stmt: ListKnownRoutesByIATA + shape: "BWE (prod 55 s plan today), page 1" + shapes: [dense, page1] + params: [BWE, null, 0, 200] + expect: + driver: [{relation: known_routes, index: idx_known_routes_iata_keyset, index_cond: ["iata >=", "id) < ROW("]}] + budget: {buffers: 306, planning_ms: 10} + note: "measured: 204 B on the full local copy (max over modes)" + - id: routes.byiata.R06_MAD_first_l50 + stmt: ListKnownRoutesByIATA + shape: "sparse region (MAD), page 1" + shapes: [sparse, page1] + params: [MAD, null, 0, 50] + expect: + driver: [{relation: known_routes, index: idx_known_routes_iata_keyset, index_cond: ["iata >=", "id) < ROW("]}] + budget: {buffers: 78, planning_ms: 10} + note: "measured: 52 B on the full local copy (max over modes)" + - id: routes.byiata.R07_CLE_first_l200 + stmt: ListKnownRoutesByIATA + shape: "small region (CLE), page 1, limit 200" + shapes: [sparse, page1] + params: [CLE, null, 0, 200] + expect: + driver: [{relation: known_routes, index: idx_known_routes_iata_keyset, index_cond: ["iata >=", "id) < ROW("]}] + budget: {buffers: 288, planning_ms: 10} + note: "measured: 192 B on the full local copy (max over modes)" + - id: routes.byiata.R08_NEW_first_l50 + stmt: ListKnownRoutesByIATA + shape: "stale-statistics region (NEW, 104,886 routes, est. 1 row), page 1" + shapes: [stale_region, page1] + params: [NEW, null, 0, 50] + expect: + driver: [{relation: known_routes, index: idx_known_routes_iata_keyset, index_cond: ["iata >=", "id) < ROW("]}] + budget: {buffers: 10, planning_ms: 10} + note: "measured: 4 B on the full local copy (max over modes)" + - id: routes.byiata.R09_NEW_page2_l50 + stmt: ListKnownRoutesByIATA + shape: "stale region, page 2" + shapes: [stale_region, cursor] + params: [NEW, {r: new50_ts}, {r: new50_id}, 50] + expect: + driver: [{relation: known_routes, index: idx_known_routes_iata_keyset, index_cond: ["iata >=", "id) < ROW("]}] + budget: {buffers: 300, planning_ms: 10} + note: "measured: 55 / 55 (custom / generic)" + - id: routes.byiata.R10_NEW_deep_l200 + stmt: ListKnownRoutesByIATA + shape: "stale region, deep cursor-only" + shapes: [stale_region, deep] + params: [NEW, "2026-10-05 19:00:00+00", 0, 200] + expect: + driver: [{relation: known_routes, index: idx_known_routes_iata_keyset, index_cond: ["iata >=", "id) < ROW("]}] + budget: {buffers: 10, planning_ms: 10} + note: "measured: 4 B on the full local copy (max over modes)" + - id: routes.byiata.R11_ZZZ_l50 + stmt: ListKnownRoutesByIATA + shape: "not an IATA code (ZZZ): empty region" + shapes: [empty_region, page1] + params: [ZZZ, null, 0, 50] + expect: + driver: [{relation: known_routes, index: idx_known_routes_iata_keyset, index_cond: ["iata >=", "id) < ROW("]}] + budget: {buffers: 10, planning_ms: 10} + note: "measured: 4 B on the full local copy (max over modes)" + - id: routes.byiata.R12_WRE_l50 + stmt: ListKnownRoutesByIATA + shape: "region with no routes (WRE): empty region" + shapes: [empty_region, page1] + params: [WRE, null, 0, 50] + expect: + driver: [{relation: known_routes, index: idx_known_routes_iata_keyset, index_cond: ["iata >=", "id) < ROW("]}] + budget: {buffers: 10, planning_ms: 10} + note: "measured: 4 B on the full local copy (max over modes)" + - id: routes.byiata.R13_SEA_tie_l200 + stmt: ListKnownRoutesByIATA + shape: "dense region, cursor inside a tied millisecond (cursorId = top id of the tie)" + shapes: [dense, tie] + params: [SEA, {r: sea_tie_ts}, {r: sea_tie_id}, 200] + expect: + driver: [{relation: known_routes, index: idx_known_routes_iata_keyset, index_cond: ["iata >=", "id) < ROW("]}] + budget: {buffers: 299, planning_ms: 10} + note: "measured: 199 B on the full local copy (max over modes)" + - id: routes.byiata.R14_BWE_deep_l200 + stmt: ListKnownRoutesByIATA + shape: "BWE cursor + cursorId 5,000 deep" + shapes: [dense, deep] + params: [BWE, {r: bwe_deep_ts}, {r: bwe_deep_id}, 200] + expect: + driver: [{relation: known_routes, index: idx_known_routes_iata_keyset, index_cond: ["iata >=", "id) < ROW("]}] + budget: {buffers: 308, planning_ms: 60} + note: "measured: 205 B on the full local copy (max over modes)" + - id: routes.global.G01_first_l50 + stmt: ListKnownRoutesGlobal + shape: "no region, page 1, limit 50" + shapes: [page1] + params: [null, 0, 50] + expect: + driver: [{relation: known_routes, index: idx_known_routes_keyset, index_cond: ["id) < ROW("]}] + budget: {buffers: 80, planning_ms: 10} + note: "measured: 53 B on the full local copy (max over modes)" + - id: routes.global.G02_page2_l50 + stmt: ListKnownRoutesGlobal + shape: "no region, page 2 (cursor + cursorId)" + shapes: [cursor] + params: [{r: g50_ts}, {r: g50_id}, 50] + expect: + driver: [{relation: known_routes, index: idx_known_routes_keyset, index_cond: ["id) < ROW("]}] + budget: {buffers: 80, planning_ms: 10} + note: "measured: 53 B on the full local copy (max over modes)" + - id: routes.global.G03_deep_curonly_l200 + stmt: ListKnownRoutesGlobal + shape: "no region, deep cursor-only" + shapes: [deep] + params: ["2026-10-05 19:00:00+00", 0, 200] + expect: + driver: [{relation: known_routes, index: idx_known_routes_keyset, index_cond: ["id) < ROW("]}] + budget: {buffers: 303, planning_ms: 10} + note: "measured: 202 B on the full local copy (max over modes)" + - id: routes.global.G04_tie_l50 + stmt: ListKnownRoutesGlobal + shape: "no region, cursor inside a tied millisecond" + shapes: [tie] + params: [{r: global_tie_ts}, {r: global_tie_id}, 50] + expect: + driver: [{relation: known_routes, index: idx_known_routes_keyset, index_cond: ["id) < ROW("]}] + budget: {buffers: 80, planning_ms: 10} + note: "measured: 53 B on the full local copy (max over modes)" + - id: routes.byiatahop.H01_SEA_hop6_dense_l200 + stmt: ListKnownRoutesByIATAHop + shape: "region + dense hop (SEA hop 6, 263k)" + params: [SEA, 6, null, 0, 200] + expect: + driver: [{relation: known_routes, index: idx_known_routes_iata_keyset, index_cond: ["iata =", "id) < ROW("]}] + budget: {buffers: 2136, planning_ms: 10} + note: "measured: 1424 B on the full local copy (max over modes)" + # Not on the fixture: at 1/50 the (region, hop) group is small enough that sorting it is cheaper than walking + # the region's keyset; at full size the walk wins. The same holds for H09. + - id: routes.byiatahop.H02_SEA_hop15_mid_l200 + stmt: ListKnownRoutesByIATAHop + shape: "region + mid hop (SEA hop 15, 39k)" + params: [SEA, 15, null, 0, 200] + tiers: [local, prod] + expect: + driver: [{relation: known_routes, index: idx_known_routes_iata_keyset, index_cond: ["iata =", "id) < ROW("]}] + budget: {buffers: 11499, planning_ms: 10} + note: "measured: 7666 B on the full local copy (max over modes)" + - id: routes.byiatahop.H03_SEA_hop25_rare_l200 + stmt: ListKnownRoutesByIATAHop + shape: "region + rare hop (SEA hop 25, 393 rows)" + params: [SEA, 25, null, 0, 200] + expect: + driver: [{relation: known_routes, index: idx_known_routes_hop_iata, index_cond: ["hop_count =", "iata ="]}] + sort_exempt: [hop_group] + budget: {buffers: 543, planning_ms: 10} + note: "measured: 362 B on the full local copy (max over modes)" + - id: routes.byiatahop.H04_SEA_hop32_vrare_l50 + stmt: ListKnownRoutesByIATAHop + shape: "region + very rare hop (SEA hop 32, 18 rows)" + params: [SEA, 32, null, 0, 50] + expect: + driver: [{relation: known_routes, index: idx_known_routes_hop_iata, index_cond: ["hop_count =", "iata ="]}] + sort_exempt: [hop_group] + budget: {buffers: 32, planning_ms: 10} + note: "measured: 21 B on the full local copy (max over modes)" + - id: routes.byiatahop.H05_SEA_hop1_absent_l200 + stmt: ListKnownRoutesByIATAHop + shape: "region + absent hop (SEA hop 1)" + params: [SEA, 1, null, 0, 200] + expect: + driver: [{relation: known_routes, index: idx_known_routes_hop_iata, index_cond: ["hop_count =", "iata ="]}] + sort_exempt: [hop_group] + budget: {buffers: 10, planning_ms: 10} + note: "measured: 3 B on the full local copy (max over modes)" + - id: routes.byiatahop.H06_SEA_hop6_page2_l200 + stmt: ListKnownRoutesByIATAHop + shape: "region + dense hop, page 2" + params: [SEA, 6, {r: seah6_ts}, {r: seah6_id}, 200] + expect: + driver: [{relation: known_routes, index: idx_known_routes_iata_keyset, index_cond: ["iata =", "id) < ROW("]}] + budget: {buffers: 1520, planning_ms: 10} + note: "measured: 1013 B on the full local copy (max over modes)" + - id: routes.byiatahop.H07_BWE_hop5_dense_l200 + stmt: ListKnownRoutesByIATAHop + shape: "BWE + dense hop 5" + params: [BWE, 5, null, 0, 200] + expect: + driver: [{relation: known_routes, index: idx_known_routes_iata_keyset, index_cond: ["iata =", "id) < ROW("]}] + budget: {buffers: 1620, planning_ms: 10} + note: "measured: 1080 B on the full local copy (max over modes)" + - id: routes.byiatahop.H08_BWE_hop22_rare_l50 + stmt: ListKnownRoutesByIATAHop + shape: "BWE + rare hop 22 (6 rows)" + params: [BWE, 22, null, 0, 50] + expect: + driver: [{relation: known_routes, index: idx_known_routes_hop_iata, index_cond: ["hop_count =", "iata ="]}] + sort_exempt: [hop_group] + budget: {buffers: 14, planning_ms: 10} + note: "measured: 9 B on the full local copy (max over modes)" + - id: routes.byiatahop.H09_MAD_hop4_dense_l50 + stmt: ListKnownRoutesByIATAHop + shape: "sparse region + dense hop (MAD hop 4)" + params: [MAD, 4, null, 0, 50] + tiers: [local, prod] + expect: + driver: [{relation: known_routes, index: idx_known_routes_iata_keyset, index_cond: ["iata =", "id) < ROW("]}] + budget: {buffers: 207, planning_ms: 10} + note: "measured: 138 B on the full local copy (max over modes)" + - id: routes.byiatahop.H10_MAD_hop16_rare_l50 + stmt: ListKnownRoutesByIATAHop + shape: "sparse region + rare hop (MAD hop 16, 1 row)" + params: [MAD, 16, null, 0, 50] + expect: + driver: [{relation: known_routes, index: idx_known_routes_hop_iata, index_cond: ["hop_count =", "iata ="]}] + sort_exempt: [hop_group] + budget: {buffers: 10, planning_ms: 10} + note: "measured: 4 B on the full local copy (max over modes)" + # H11-H14 are not on the fixture: the region is estimated at one row, so the hop index (and a sort) and the + # region's keyset (and a filter) cost the same to within the index geometry, which at 1/50 tips to the keyset. + - id: routes.byiatahop.H11_NEW_hop4_dense_l200 + stmt: ListKnownRoutesByIATAHop + shape: "stale region + dense hop (NEW hop 4, 29k)" + params: [NEW, 4, null, 0, 200] + tiers: [local, prod] + expect: + driver: [{relation: known_routes, index: idx_known_routes_hop_iata, index_cond: ["hop_count =", "iata ="]}] + sort_exempt: [hop_group] + budget: {buffers: 10, planning_ms: 10} + note: "measured: 3 B on the full local copy (max over modes)" + - id: routes.byiatahop.H12_NEW_hop14_rare_l50 + stmt: ListKnownRoutesByIATAHop + shape: "stale region + rare hop (NEW hop 14, 53 rows)" + params: [NEW, 14, null, 0, 50] + tiers: [local, prod] + expect: + driver: [{relation: known_routes, index: idx_known_routes_hop_iata, index_cond: ["hop_count =", "iata ="]}] + sort_exempt: [hop_group] + budget: {buffers: 10, planning_ms: 10} + note: "measured: 3 B on the full local copy (max over modes)" + - id: routes.byiatahop.H13_NEW_hop1_absent_l50 + stmt: ListKnownRoutesByIATAHop + shape: "stale region + absent hop (NEW hop 1)" + params: [NEW, 1, null, 0, 50] + tiers: [local, prod] + expect: + driver: [{relation: known_routes, index: idx_known_routes_hop_iata, index_cond: ["hop_count =", "iata ="]}] + sort_exempt: [hop_group] + budget: {buffers: 10, planning_ms: 10} + note: "measured: 3 B on the full local copy (max over modes)" + - id: routes.byiatahop.H14_ZZZ_hop6_l50 + stmt: ListKnownRoutesByIATAHop + shape: "empty region + hop (ZZZ hop 6)" + params: [ZZZ, 6, null, 0, 50] + tiers: [local, prod] + expect: + driver: [{relation: known_routes, index: idx_known_routes_hop_iata, index_cond: ["hop_count =", "iata ="]}] + sort_exempt: [hop_group] + budget: {buffers: 10, planning_ms: 10} + note: "measured: 3 B on the full local copy (max over modes)" + - id: routes.globalhop.GH01_hop6_dense_l50 + stmt: ListKnownRoutesGlobalHop + shape: "no region + dense hop 6, limit 50" + params: [6, null, 0, 50] + expect: + driver: [{relation: known_routes, index: idx_known_routes_keyset, index_cond: ["id) < ROW("]}] + budget: {buffers: 588, planning_ms: 10} + note: "measured: 392 B on the full local copy (max over modes)" + - id: routes.globalhop.GH02_hop6_dense_l200 + stmt: ListKnownRoutesGlobalHop + shape: "no region + dense hop 6, limit 200" + params: [6, null, 0, 200] + expect: + driver: [{relation: known_routes, index: idx_known_routes_keyset, index_cond: ["id) < ROW("]}] + budget: {buffers: 2316, planning_ms: 10} + note: "measured: 1544 B on the full local copy (max over modes)" + - id: routes.globalhop.GH03_hop15_l200 + stmt: ListKnownRoutesGlobalHop + shape: "no region + mid hop 15" + params: [15, null, 0, 200] + expect: + driver: [{relation: known_routes, index: idx_known_routes_keyset, index_cond: ["id) < ROW("]}] + budget: {buffers: 26771, planning_ms: 10} + note: "measured: 17847 B on the full local copy (max over modes)" + - id: routes.globalhop.GH04_hop20_l200 + stmt: ListKnownRoutesGlobalHop + shape: "no region + hop 20 (keyset + Filter, same as today)" + params: [20, null, 0, 200] + expect: + driver: [{relation: known_routes, index: idx_known_routes_keyset, index_cond: ["id) < ROW("]}] + budget: {buffers: 60861, planning_ms: 10} + note: "measured: 40574 B on the full local copy (max over modes)" + - id: routes.globalhop.GH05_hop25_l200 + stmt: ListKnownRoutesGlobalHop + shape: "no region + rare hop 25" + params: [25, null, 0, 200] + requires_indexes: [idx_known_routes_hop_iata] + expect: + driver: [{relation: known_routes, index: idx_known_routes_hop_iata, index_cond: ["hop_count ="]}] + sort_exempt: [hop_group] + budget: {buffers: 15002, planning_ms: 10} + note: "measured: 10001 B on the full local copy with the end-state hop index (this copy lacks it)" + - id: routes.globalhop.GH06_hop32_l50 + stmt: ListKnownRoutesGlobalHop + shape: "no region + very rare hop 32" + params: [32, null, 0, 50] + requires_indexes: [idx_known_routes_hop_iata] + expect: + driver: [{relation: known_routes, index: idx_known_routes_hop_iata, index_cond: ["hop_count ="]}] + sort_exempt: [hop_group] + budget: {buffers: 3633, planning_ms: 10} + note: "measured: 2422 B on the full local copy with the end-state hop index (this copy lacks it)" + - id: routes.globalhop.GH07_hop42_single_l50 + stmt: ListKnownRoutesGlobalHop + shape: "no region + single-row hop 42" + params: [42, null, 0, 50] + requires_indexes: [idx_known_routes_hop_iata] + expect: + driver: [{relation: known_routes, index: idx_known_routes_hop_iata, index_cond: ["hop_count ="]}] + sort_exempt: [hop_group] + budget: {buffers: 10, planning_ms: 10} + note: "passes within 10 B on the full local copy with the end-state hop index (this copy lacks it)" + - id: routes.globalhop.GH08_hop1_absent_l50 + stmt: ListKnownRoutesGlobalHop + shape: "no region + absent hop 1" + params: [1, null, 0, 50] + requires_indexes: [idx_known_routes_hop_iata] + expect: + driver: [{relation: known_routes, index: idx_known_routes_hop_iata, index_cond: ["hop_count ="]}] + sort_exempt: [hop_group] + budget: {buffers: 10, planning_ms: 10} + note: "passes within 10 B on the full local copy with the end-state hop index (this copy lacks it)" + - id: routes.globalhop.GH09_hop1_absent_cur_l50 + stmt: ListKnownRoutesGlobalHop + shape: "no region + absent hop 1, with cursor + cursorId" + params: [1, {r: g50_ts}, {r: g50_id}, 50] + requires_indexes: [idx_known_routes_hop_iata] + expect: + driver: [{relation: known_routes, index: idx_known_routes_hop_iata, index_cond: ["hop_count ="]}] + sort_exempt: [hop_group] + budget: {buffers: 10, planning_ms: 10} + note: "passes within 10 B on the full local copy with the end-state hop index (this copy lacks it)" + - id: routes.globalhop.GH10_hop6_deep_l200 + stmt: ListKnownRoutesGlobalHop + shape: "no region + dense hop 6, deep cursor-only" + params: [6, "2026-10-05 19:00:00+00", 0, 200] + expect: + driver: [{relation: known_routes, index: idx_known_routes_keyset, index_cond: ["id) < ROW("]}] + budget: {buffers: 1605, planning_ms: 10} + note: "measured: 1070 B on the full local copy (max over modes)" + - id: routes.search.s_14_3d_all + stmt: SearchKnownRoutesByNodes + shape: "route search s_14_3d_all" + params: [{r: s_14_3d_all_f}, {r: s_14_3d_all_t}, {r: s_14_3d_all_i}, 500] + expect: + driver: [{relation: known_routes, alias: kr, nodes: ["Bitmap Heap Scan", "Index Scan"]}] + forbid: [{node: "Incremental Sort", reason: "hop-ordered walk of a hop index: hop_count + 0 must keep it out"}] + sort_exempt: [route_search_top_n] + budget: {buffers: 24276, planning_ms: 10} + note: "measured: 16184 B on the full local copy (max over modes)" + - id: routes.search.s_1410_3d67_all + stmt: SearchKnownRoutesByNodes + shape: "route search s_1410_3d67_all" + params: [{r: s_1410_3d67_all_f}, {r: s_1410_3d67_all_t}, {r: s_1410_3d67_all_i}, 500] + expect: + driver: [{relation: known_routes, alias: kr, nodes: ["Bitmap Heap Scan", "Index Scan"]}] + forbid: [{node: "Incremental Sort", reason: "hop-ordered walk of a hop index: hop_count + 0 must keep it out"}] + sort_exempt: [route_search_top_n] + budget: {buffers: 221, planning_ms: 10} + note: "measured: 147 B on the full local copy (max over modes)" + - id: routes.search.s_1410_3d67_RTM + stmt: SearchKnownRoutesByNodes + shape: "route search s_1410_3d67_RTM" + params: [{r: s_1410_3d67_RTM_f}, {r: s_1410_3d67_RTM_t}, {r: s_1410_3d67_RTM_i}, 500] + expect: + driver: [{relation: known_routes, alias: kr, nodes: ["Bitmap Heap Scan", "Index Scan"]}] + forbid: [{node: "Incremental Sort", reason: "hop-ordered walk of a hop index: hop_count + 0 must keep it out"}] + sort_exempt: [route_search_top_n] + budget: {buffers: 221, planning_ms: 10} + note: "measured: 147 B on the full local copy (max over modes)" + - id: routes.search.s_14_3d_RTM_AMS_EIN + stmt: SearchKnownRoutesByNodes + shape: "route search s_14_3d_RTM_AMS_EIN" + params: [{r: s_14_3d_RTM_AMS_EIN_f}, {r: s_14_3d_RTM_AMS_EIN_t}, {r: s_14_3d_RTM_AMS_EIN_i}, 500] + expect: + driver: [{relation: known_routes, alias: kr, nodes: ["Bitmap Heap Scan", "Index Scan"]}] + forbid: [{node: "Incremental Sort", reason: "hop-ordered walk of a hop index: hop_count + 0 must keep it out"}] + sort_exempt: [route_search_top_n] + budget: {buffers: 677, planning_ms: 10} + note: "measured: 451 B on the full local copy (max over modes)" + - id: routes.search.s_5731_0355_all + stmt: SearchKnownRoutesByNodes + shape: "route search s_5731_0355_all" + params: [{r: s_5731_0355_all_f}, {r: s_5731_0355_all_t}, {r: s_5731_0355_all_i}, 500] + expect: + driver: [{relation: known_routes, alias: kr, nodes: ["Bitmap Heap Scan", "Index Scan"]}] + forbid: [{node: "Incremental Sort", reason: "hop-ordered walk of a hop index: hop_count + 0 must keep it out"}] + sort_exempt: [route_search_top_n] + budget: {buffers: 7227, planning_ms: 10} + note: "measured: 4818 B on the full local copy (max over modes)" + - id: routes.search.s_f877_0867_all + stmt: SearchKnownRoutesByNodes + shape: "route search s_f877_0867_all" + params: [{r: s_f877_0867_all_f}, {r: s_f877_0867_all_t}, {r: s_f877_0867_all_i}, 500] + expect: + driver: [{relation: known_routes, alias: kr, nodes: ["Bitmap Heap Scan", "Index Scan"]}] + forbid: [{node: "Incremental Sort", reason: "hop-ordered walk of a hop index: hop_count + 0 must keep it out"}] + sort_exempt: [route_search_top_n] + budget: {buffers: 986, planning_ms: 10} + note: "measured: 657 B on the full local copy (max over modes)" + - id: routes.search.s_sea_lowco_rare + stmt: SearchKnownRoutesByNodes + shape: "route search, SEA: prefix fade to prefix 2525 (few routes)" + params: [{r: s_sea_prefix_fade}, {r: s_sea_prefix_2525}, {r: sea_i}, 500] + expect: + driver: [{relation: known_routes, alias: kr, nodes: ["Bitmap Heap Scan", "Index Scan"]}] + forbid: [{node: "Incremental Sort", reason: "hop-ordered walk of a hop index: hop_count + 0 must keep it out"}] + sort_exempt: [route_search_top_n] + budget: {buffers: 14519, planning_ms: 10} + note: "measured: 9679 B on the full local copy (max over modes)" + - id: routes.search.s_sea_pop_top2 + stmt: SearchKnownRoutesByNodes + shape: "route search, SEA: the relay on the most of the newest 50,000 SEA routes to the second" + params: [{r: s_sea_pop_f}, {r: s_sea_pop_t}, {r: sea_i}, 500] + expect: + driver: [{relation: known_routes, alias: kr, nodes: ["Bitmap Heap Scan", "Index Scan"]}] + forbid: [{node: "Incremental Sort", reason: "hop-ordered walk of a hop index: hop_count + 0 must keep it out"}] + sort_exempt: [route_search_top_n] + budget: {buffers: 75092, planning_ms: 10} + note: "measured: 50061 B on the full local copy (max over modes)" + - id: routes.search.s_sea_pop_rare + stmt: SearchKnownRoutesByNodes + shape: "route search, SEA: the most-routed relay to a rare relay (fewest of the newest 50,000 SEA routes)" + params: [{r: s_sea_pop_f}, {r: s_sea_rare_t}, {r: sea_i}, 500] + expect: + driver: [{relation: known_routes, alias: kr, nodes: ["Bitmap Heap Scan", "Index Scan"]}] + forbid: [{node: "Incremental Sort", reason: "hop-ordered walk of a hop index: hop_count + 0 must keep it out"}] + sort_exempt: [route_search_top_n] + budget: {buffers: 110, planning_ms: 10} + note: "measured: 73 B on the full local copy (max over modes)" + - id: routes.cross.x_0355 + stmt: GetKnownRoutesByNodes + shape: "route cross x_0355" + params: [{r: x_0355_n}, {r: x_0355_i}, 500] + expect: + driver: [{relation: known_routes, alias: kr, nodes: ["Bitmap Heap Scan", "Index Scan"]}] + forbid: [{node: "Incremental Sort", reason: "hop-ordered walk of a hop index: hop_count + 0 must keep it out"}] + sort_exempt: [route_search_top_n] + budget: {buffers: 57584, planning_ms: 10} + note: "measured: 38389 B on the full local copy (max over modes)" + - id: routes.cross.x_0867 + stmt: GetKnownRoutesByNodes + shape: "route cross x_0867" + params: [{r: x_0867_n}, {r: x_0867_i}, 500] + expect: + driver: [{relation: known_routes, alias: kr, nodes: ["Bitmap Heap Scan", "Index Scan"]}] + forbid: [{node: "Incremental Sort", reason: "hop-ordered walk of a hop index: hop_count + 0 must keep it out"}] + sort_exempt: [route_search_top_n] + budget: {buffers: 21257, planning_ms: 10} + note: "measured: 14171 B on the full local copy (max over modes)" + - id: routes.cross.x_1410 + stmt: GetKnownRoutesByNodes + shape: "route cross x_1410" + params: [{r: x_1410_n}, {r: x_1410_i}, 500] + expect: + driver: [{relation: known_routes, alias: kr, nodes: ["Bitmap Heap Scan", "Index Scan"]}] + forbid: [{node: "Incremental Sort", reason: "hop-ordered walk of a hop index: hop_count + 0 must keep it out"}] + sort_exempt: [route_search_top_n] + budget: {buffers: 4511, planning_ms: 10} + note: "measured: 3007 B on the full local copy (max over modes)" + - id: routes.cross.x_3d67 + stmt: GetKnownRoutesByNodes + shape: "route cross x_3d67" + params: [{r: x_3d67_n}, {r: x_3d67_i}, 500] + expect: + driver: [{relation: known_routes, alias: kr, nodes: ["Bitmap Heap Scan", "Index Scan"]}] + forbid: [{node: "Incremental Sort", reason: "hop-ordered walk of a hop index: hop_count + 0 must keep it out"}] + sort_exempt: [route_search_top_n] + budget: {buffers: 182, planning_ms: 10} + note: "measured: 121 B on the full local copy (max over modes)" + - id: routes.cross.x_5731 + stmt: GetKnownRoutesByNodes + shape: "route cross x_5731" + params: [{r: x_5731_n}, {r: x_5731_i}, 500] + expect: + driver: [{relation: known_routes, alias: kr, nodes: ["Bitmap Heap Scan", "Index Scan"]}] + forbid: [{node: "Incremental Sort", reason: "hop-ordered walk of a hop index: hop_count + 0 must keep it out"}] + sort_exempt: [route_search_top_n] + budget: {buffers: 11622, planning_ms: 10} + note: "measured: 7748 B on the full local copy (max over modes)" + - id: routes.cross.x_f877 + stmt: GetKnownRoutesByNodes + shape: "route cross x_f877" + params: [{r: x_f877_n}, {r: x_f877_i}, 500] + expect: + driver: [{relation: known_routes, alias: kr, nodes: ["Bitmap Heap Scan", "Index Scan"]}] + forbid: [{node: "Incremental Sort", reason: "hop-ordered walk of a hop index: hop_count + 0 must keep it out"}] + sort_exempt: [route_search_top_n] + budget: {buffers: 41414, planning_ms: 10} + note: "measured: 27609 B on the full local copy (max over modes)" + - id: routes.cross.x_sea_rare + stmt: GetKnownRoutesByNodes + shape: "route cross, SEA: a rare relay (fewest of the newest 50,000 SEA routes)" + params: [{r: s_sea_rare_t}, {r: sea_i}, 500] + expect: + driver: [{relation: known_routes, alias: kr, nodes: ["Bitmap Heap Scan", "Index Scan"]}] + forbid: [{node: "Incremental Sort", reason: "hop-ordered walk of a hop index: hop_count + 0 must keep it out"}] + sort_exempt: [route_search_top_n] + budget: {buffers: 221, planning_ms: 10} + note: "measured: 147 B on the full local copy (max over modes)" + - id: routes.cross.x_sea_top1 + stmt: GetKnownRoutesByNodes + shape: "route cross, SEA: the relay on the most of the newest 50,000 SEA routes" + params: [{r: s_sea_pop_f}, {r: sea_i}, 500] + expect: + driver: [{relation: known_routes, alias: kr, nodes: ["Bitmap Heap Scan", "Index Scan"]}] + forbid: [{node: "Incremental Sort", reason: "hop-ordered walk of a hop index: hop_count + 0 must keep it out"}] + sort_exempt: [route_search_top_n] + budget: {buffers: 122249, planning_ms: 10} + note: "measured: 81499 B on the full local copy (max over modes)" + + # ---- route retention: keyset walk with a cursor, grace slices in bounded 6 h windows (maint) ---- + - id: retention.expired.select + stmt: DeleteExpiredRoutesSelect + shape: "retention half, prod-like cutoff R = T-14 d, first batch of 10,000" + params: ["-infinity", 0, "2026-10-04 19:00:00+00", "2026-10-04 19:00:00+00", 10000] + expect: + driver: [{relation: known_routes, index: idx_known_routes_keyset, index_cond: ["id) > ROW(", "::text) <"]}] + budget: {buffers: 12536, planning_ms: 10} + note: "measured: 8357 B on the full local copy (max over modes)" + - id: retention.expired.dml + stmt: DeleteExpiredRoutes + shape: "the full DELETE statement (plain EXPLAIN: shape only)" + params: ["-infinity", 0, "2026-10-04 19:00:00+00", "2026-10-04 19:00:00+00", 10000] + expect: + driver: [{relation: known_routes, alias: r, index: idx_known_routes_keyset, index_cond: ["ROW(date_trunc"]}] + - id: retention.grace.cand + stmt: DeleteGraceRoutesSliceCand + shape: "grace slice from G-1h (window end = LEAST(G, after + 6 h) = G), G = T-7 d, min_observations 3, batch 10,000" + params: ["2026-10-11 19:00:00+00", "2026-10-11 18:00:00+00", 0, "2026-10-11 19:00:00+00", 3, 10000] + expect: + driver: [{relation: known_routes, index: idx_known_routes_keyset, index_cond: ["id) > ROW(", "::text) < LEAST("]}] + budget: {buffers: 31608, planning_ms: 10} + note: "measured: 21072 B on the full local copy (max over modes)" + - id: retention.grace.dml + stmt: DeleteGraceRoutesSlice + shape: "the full grace statement (plain EXPLAIN)" + params: ["2026-10-11 19:00:00+00", "2026-10-11 18:00:00+00", 0, "2026-10-11 19:00:00+00", 3, 10000] + expect: + driver: [{relation: known_routes, alias: r, index: idx_known_routes_keyset, index_cond: ["ROW(date_trunc"]}] + - id: retention.grace.fullpass + stmt: DeleteGraceRoutesSliceCand + shape: "full pass from R (first deploy / policy change / weekly): first bounded window [R, R + 6 h)" + params: ["2026-10-11 19:00:00+00", "2026-10-04 19:00:00+00", 0, "2026-10-11 19:00:00+00", 3, 10000] + executions: 1 + expect: + driver: [{relation: known_routes, index: idx_known_routes_keyset, index_cond: ["ROW(date_trunc", "LEAST("]}] + budget: {buffers: 70488, planning_ms: 10} + note: "measured: 46992 B on the full local copy (max over modes)" + - id: retention.expired.select.final + stmt: DeleteExpiredRoutesSelect + shape: "retention half, last batch of a run (nothing left past R)" + params: ["-infinity", 0, "2026-10-04 17:00:00+00", "2026-10-04 17:00:00+00", 10000] + expect: + driver: [{relation: known_routes, index: idx_known_routes_keyset, index_cond: ["ROW(date_trunc"]}] + budget: {buffers: 10, planning_ms: 10} + + # ---- channel messages through channel_message_iatas (pending) ---- + note: "measured: 3 B on the full local copy (max over modes)" + - id: cmi2.P01 + stmt: ListChannelMessagesByIATAs + shape: "ch27 YUL p1 lim50" + params: ["{YUL}", 27, 9223372036854775807, 51] + expect: + driver: [{scope: "CTE hits", relation: channel_message_iatas, alias: i, index: idx_cmi_channel_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["channel_id =", "message_id <"]}] + forbid: [{node: "Seq Scan", relations: [channel_messages, channel_message_iatas]}] + sort_exempt: [per_site_message_ranges] + budget: {buffers: 600, planning_ms: 10} + note: "measured: 574 / 574 B (custom / generic), idx ,idx_cmi_channel_site; measured budget <=600" + - id: cmi2.P02 + stmt: ListChannelMessagesByIATAs + shape: "ch27 YUL cursor page (2)" + params: ["{YUL}", 27, 17936665, 51] + expect: + driver: [{scope: "CTE hits", relation: channel_message_iatas, alias: i, index: idx_cmi_channel_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["channel_id =", "message_id <"]}] + forbid: [{node: "Seq Scan", relations: [channel_messages, channel_message_iatas]}] + sort_exempt: [per_site_message_ranges] + budget: {buffers: 600, planning_ms: 10} + note: "measured: 565 / 565 B (custom / generic), idx ,idx_cmi_channel_site; measured budget <=600" + - id: cmi2.P03 + stmt: ListChannelMessagesByIATAs + shape: "ch27 YUL deep cursor" + params: ["{YUL}", 27, 15018736, 51] + expect: + driver: [{scope: "CTE hits", relation: channel_message_iatas, alias: i, index: idx_cmi_channel_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["channel_id =", "message_id <"]}] + forbid: [{node: "Seq Scan", relations: [channel_messages, channel_message_iatas]}] + sort_exempt: [per_site_message_ranges] + budget: {buffers: 600, planning_ms: 10} + note: "measured: 499 / 499 B (custom / generic), idx ,idx_cmi_channel_site; measured budget <=600" + - id: cmi2.P04 + stmt: ListChannelMessagesByIATAs + shape: "zero-hit ch14 GDW" + params: ["{GDW}", 14, 9223372036854775807, 51] + expect: + driver: [{scope: "CTE hits", relation: channel_message_iatas, alias: i, index: idx_cmi_channel_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["channel_id =", "message_id <"]}] + forbid: [{node: "Seq Scan", relations: [channel_messages, channel_message_iatas]}] + sort_exempt: [per_site_message_ranges] + budget: {buffers: 10, planning_ms: 10} + note: "measured: 3 / 3 B (custom / generic), idx ,idx_cmi_channel_site; measured budget <=10" + - id: cmi2.P05 + stmt: ListChannelMessagesByIATAs + shape: "zero-hit ch27 BLK (valid code)" + params: ["{BLK}", 27, 9223372036854775807, 51] + expect: + driver: [{scope: "CTE hits", relation: channel_message_iatas, alias: i, index: idx_cmi_channel_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["channel_id =", "message_id <"]}] + forbid: [{node: "Seq Scan", relations: [channel_messages, channel_message_iatas]}] + sort_exempt: [per_site_message_ranges] + budget: {buffers: 10, planning_ms: 10} + note: "measured: 3 / 3 B (custom / generic), idx ,idx_cmi_channel_site; measured budget <=10" + - id: cmi2.P06 + stmt: ListChannelMessagesByIATAs + shape: "zero-hit ch27 ZZZ" + params: ["{ZZZ}", 27, 9223372036854775807, 51] + expect: + driver: [{scope: "CTE hits", relation: channel_message_iatas, alias: i, index: idx_cmi_channel_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["channel_id =", "message_id <"]}] + forbid: [{node: "Seq Scan", relations: [channel_messages, channel_message_iatas]}] + sort_exempt: [per_site_message_ranges] + budget: {buffers: 10, planning_ms: 10} + note: "measured: 3 / 3 B (custom / generic), idx ,idx_cmi_channel_site; measured budget <=10" + - id: cmi2.P07 + stmt: ListChannelMessagesByIATAs + shape: "ch27 UK22 p1 lim50" + params: ["{BHX,BLK,BOH,BRS,CAX,CEG,DSA,EMA,EXT,HUY,LCY,LON,LPL,MAN,MME,NQT,NQY,NWI,ORM,PLH,PME,VLY}", 27, 9223372036854775807, 51] + expect: + driver: [{scope: "CTE hits", relation: channel_message_iatas, alias: i, index: idx_cmi_channel_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["channel_id =", "message_id <"]}] + forbid: [{node: "Seq Scan", relations: [channel_messages, channel_message_iatas]}] + sort_exempt: [per_site_message_ranges] + budget: {buffers: 1500, planning_ms: 10} + note: "measured: 646 / 646 B (custom / generic), idx ,idx_cmi_channel_site; measured budget <=1500" + - id: cmi2.P08 + stmt: ListChannelMessagesByIATAs + shape: "ch27 UK22 deep cursor" + params: ["{BHX,BLK,BOH,BRS,CAX,CEG,DSA,EMA,EXT,HUY,LCY,LON,LPL,MAN,MME,NQT,NQY,NWI,ORM,PLH,PME,VLY}", 27, 15018736, 51] + expect: + driver: [{scope: "CTE hits", relation: channel_message_iatas, alias: i, index: idx_cmi_channel_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["channel_id =", "message_id <"]}] + forbid: [{node: "Seq Scan", relations: [channel_messages, channel_message_iatas]}] + sort_exempt: [per_site_message_ranges] + budget: {buffers: 1500, planning_ms: 10} + note: "measured: 640 / 640 B (custom / generic), idx ,idx_cmi_channel_site; measured budget <=1500" + - id: cmi2.P09 + stmt: ListChannelMessagesByIATAs + shape: "ch5 SEA lim200" + params: ["{SEA}", 5, 9223372036854775807, 201] + expect: + driver: [{scope: "CTE hits", relation: channel_message_iatas, alias: i, index: idx_cmi_channel_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["channel_id =", "message_id <"]}] + forbid: [{node: "Seq Scan", relations: [channel_messages, channel_message_iatas]}] + sort_exempt: [per_site_message_ranges] + budget: {buffers: 3333, planning_ms: 10} + note: "measured: 2222 / 2222 B (custom / generic), idx ,idx_cmi_channel_site; 1.5x measured" + - id: cmi2.P10 + stmt: ListChannelMessagesByIATAs + shape: "ch5 ALL625 list lim50" + params: ["{AAB,AAH,AAL,AAR,ABC,ABE,ABI,ABQ,ABX,ABZ,ACH,ACY,ADL,AGS,AHN,AIZ,AKL,ALB,ALC,ALF,ALH,ALL,AMA,AMS,ANC,ANE,ANR,AOH,APF,ATH,ATL,ATW,AUH,AUS,AVL,AVP,AXN,AZO,BBH,BCN,BDL,BEG,BER,BES,BFD,BFL,BGM,BGN,BGR,BHD,BHM,BHS,BHX,BHZ,BJP,BLA,BLE,BLI,BLK,BLL,BNA,BNE,BOD,BOH,BOI,BOJ,BOS,BQN,BRE,BRN,BRQ,BRS,BRU,BTS,BUD,BUF,BUY,BVE,BWE,BWI,BWT,BYG,BYU,BZG,CAE,CAK,CAX,CBG,CBR,CDG,CEB,CEG,CEZ,CFR,CGI,CGN,CHA,CHC,CLE,CLT,CMH,CMI,COA,CON,COS,CPH,CPQ,CPT,CRL,CRW,CRX,CSM,CVG,CVO,CWA,CWB,CZW,DAB,DAL,DAY,DEC,DEN,DET,DFW,DIQ,DLE,DLH,DND,DNQ,DPO,DRO,DRS,DSA,DSM,DTM,DTW,DUB,DUD,DUS,DVN,DXB,EAT,EAU,EDI,EGE,EIN,ELM,ELP,EMA,EME,ENS,EPH,ERF,ESS,EUG,EVV,EWN,EXT,FAO,FAR,FAT,FKB,FLO,FLR,FMM,FNB,FNL,FNT,FPR,FRA,FRD,FWA,GAS,GBR,GCI,GDN,GDW,GEG,GEX,GGG,GHV,GJT,GLA,GLZ,GOA,GOJ,GOR,GOT,GPT,GPZ,GRQ,GRR,GRU,GRZ,GSO,GSP,GTI,GTP,GTR,GTW,GUF,GUL,GVX,HAD,HAJ,HAM,HDB,HEL,HLG,HLZ,HOH,HOU,HRA,HTS,HUY,HVN,HYR,IAD,IAS,IDA,IKT,ILM,ILZ,IND,INV,IQQ,ITA,IVC,JAN,JAX,JBQ,JCL,JER,JKG,JTC,JYV,KEL,KJK,KLR,KLS,KLU,KLV,KOI,KOK,KPS,KRK,KSC,KSD,KSF,KTN,KTW,KUF,KUN,KZG,KZN,LAA,LAS,LAX,LBA,LBB,LBC,LCH,LCJ,LCQ,LCY,LEH,LEJ,LEX,LEY,LFT,LGA,LGG,LGU,LIL,LIN,LIS,LIT,LJU,LKV,LLA,LMT,LNS,LNZ,LOI,LON,LOZ,LPI,LPL,LRF,LRH,LRT,LRU,LSE,LTN,LUG,LUX,LUZ,LWB,LWK,LWR,LYS,MAC,MAD,MAF,MAN,MBS,MCE,MCI,MCO,MDT,MEL,MEM,MFE,MFR,MGL,MGM,MHG,MIA,MIM,MKA,MKE,MLB,MLU,MME,MMH,MMV,MMX,MNN,MOT,MOW,MPL,MQL,MRO,MRY,MSN,MSP,MST,MSV,MSY,MTJ,MUC,MUE,MVH,MXP,NAP,NCE,NHZ,NIT,NOA,NOC,NOD,NPE,NPL,NQT,NQY,NSN,NTE,NTL,NTU,NUE,NUW,NWI,NYC,NYO,OAK,OBF,OBN,OCF,ODE,OGG,OKC,OLO,OMA,ONP,ONT,OOL,OPO,ORB,ORD,ORL,ORM,ORY,OSD,OSI,OSL,OSR,OTP,OUL,OXF,OXR,PAD,PAE,PAH,PAR,PDK,PDV,PDX,PED,PER,PFB,PGD,PGV,PHF,PHL,PHX,PIA,PIH,PIK,PIT,PKB,PLH,PLO,PLQ,PLR,PME,PMI,PMR,PNS,POA,POZ,PPM,PQQ,PRB,PRC,PRG,PRV,PSC,PSF,PSH,PSK,PSP,PUB,PUW,PVD,QAR,QDV,RAO,RBG,RDD,RDM,RDO,RDU,RDZ,RFD,RIC,RIX,RJK,RKD,RKS,RKV,RLG,RMO,RMU,RNB,RNN,RNO,ROA,ROC,ROK,ROT,ROV,RST,RSW,RTM,RUT,RYG,RZE,SAN,SAO,SAT,SBA,SBN,SBP,SBY,SCE,SCL,SCN,SDF,SDL,SDR,SEA,SFO,SGD,SGU,SIR,SJC,SJO,SJT,SJU,SJY,SKE,SLC,SLE,SMF,SMN,SNN,SOD,SOF,SPM,SQL,SQQ,SRQ,SSZ,STL,STN,STO,STR,STS,SVC,SVG,SVX,SWE,SWH,SXB,SYD,SYR,SZZ,TAT,TGD,TGN,TJM,TKU,TLH,TLL,TLS,TMP,TPA,TRD,TRF,TRG,TRI,TRN,TRO,TRS,TSR,TSV,TTD,TUF,TUL,TWB,TWF,TXK,TYR,TYS,UDE,UHE,UIP,UKK,UME,UTC,VAG,VAP,VBY,VCP,VIE,VIS,VJI,VLC,VLY,VNO,VOD,VOG,VPZ,VRZ,VST,VXO,WAW,WBG,WGA,WIC,WIE,WLG,WOL,WRE,WRO,WRT,WVN,WYA,XCM,XMS,XPH,XYZ,YBL,YCD,YEG,YGK,YHM,YHU,YHZ,YKA,YKF,YKM,YLK,YML,YOW,YPQ,YQA,YQB,YQF,YQL,YQQ,YQS,YQT,YQY,YRQ,YSE,YSJ,YTF,YTR,YUL,YVE,YVR,YWG,YWS,YXU,YXX,YYB,YYC,YYJ,YYY,YYZ,ZBE,ZBM,ZNO,ZPH}", 5, 9223372036854775807, 51] + expect: + driver: [{scope: "CTE hits", relation: channel_message_iatas, alias: i, index: idx_cmi_channel_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["channel_id =", "message_id <"]}] + forbid: [{node: "Seq Scan", relations: [channel_messages, channel_message_iatas]}] + sort_exempt: [per_site_message_ranges] + budget: {buffers: 3770, planning_ms: 10} + note: "measured: 2513 / 2513 B (custom / generic), idx ,idx_cmi_channel_site; 1.5x measured" + - id: cmi2.P11 + stmt: ListChannelMessagesByIATAsScoped + shape: "ch27 UK22 scope #fr lim50" + params: ["{BHX,BLK,BOH,BRS,CAX,CEG,DSA,EMA,EXT,HUY,LCY,LON,LPL,MAN,MME,NQT,NQY,NWI,ORM,PLH,PME,VLY}", 27, 175, 9223372036854775807, 51] + expect: + driver: [{scope: "CTE hits", relation: channel_message_iatas, alias: i, index: idx_cmi_channel_site_scope, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["channel_id =", "message_id <"]}] + forbid: [{node: "Seq Scan", relations: [channel_messages, channel_message_iatas]}] + sort_exempt: [per_site_message_ranges] + budget: {buffers: 1500, planning_ms: 10} + note: "measured: 66 / 66 B (custom / generic), idx ,idx_cmi_channel_site_scope; measured budget <=1500" + - id: cmi2.P12 + stmt: ListAllChannelMessagesByIATAs + shape: "/messages?iatas=YUL lim50" + params: ["{YUL}", 9223372036854775807, 51] + expect: + driver: [{scope: "CTE hits", relation: channel_message_iatas, alias: i, index: idx_cmi_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["message_id <"]}] + forbid: [{node: "Seq Scan", relations: [channel_messages, channel_message_iatas]}] + sort_exempt: [per_site_message_ranges] + budget: {buffers: 600, planning_ms: 10} + note: "measured: 569 / 569 B (custom / generic), idx ,idx_cmi_site; measured budget <=600" + - id: cmi2.P13 + stmt: ListAllChannelMessagesByIATAs + shape: "/messages?iatas=UK22 lim50" + params: ["{BHX,BLK,BOH,BRS,CAX,CEG,DSA,EMA,EXT,HUY,LCY,LON,LPL,MAN,MME,NQT,NQY,NWI,ORM,PLH,PME,VLY}", 9223372036854775807, 51] + expect: + driver: [{scope: "CTE hits", relation: channel_message_iatas, alias: i, index: idx_cmi_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["message_id <"]}] + forbid: [{node: "Seq Scan", relations: [channel_messages, channel_message_iatas]}] + sort_exempt: [per_site_message_ranges] + budget: {buffers: 1500, planning_ms: 10} + note: "measured: 652 / 652 B (custom / generic), idx ,idx_cmi_site; measured budget <=1500" + - id: cmi2.P14 + stmt: ListChannelMessagesPlain + shape: "plain ch27 lim50" + params: [27, 9223372036854775807, 51] + expect: + driver: [{relation: channel_messages, alias: cm, index: idx_channel_messages_channel_id, index_cond: ["channel_id =", "id <"]}] + forbid: [{node: "Seq Scan", relations: [channel_messages, channel_message_iatas]}] + sort_exempt: [per_site_message_ranges] + budget: {buffers: 1000, planning_ms: 10} + note: "measured: 413 / 422 B (custom / generic), idx ,idx_channel_messages_channel_id,idx_observations_packet; measured budget <=1000" + - id: cmi2.P15 + stmt: ListChannelMessagesPlain + shape: "plain ch27 mid cursor lim50" + params: [27, 16415687, 51] + expect: + driver: [{relation: channel_messages, alias: cm, index: idx_channel_messages_channel_id, index_cond: ["channel_id =", "id <"]}] + forbid: [{node: "Seq Scan", relations: [channel_messages, channel_message_iatas]}] + sort_exempt: [per_site_message_ranges] + budget: {buffers: 1000, planning_ms: 10} + note: "measured: 411 / 411 B (custom / generic), idx ,idx_channel_messages_channel_id,idx_observations_packet; measured budget <=1000" + - id: cmi2.P20 + stmt: ListAllChannelMessagesByIATAsSince + shape: "/messages since-10.2h + PNW13 lim200" + params: ["{BLI,CVO,KLS,LMT,MFR,MMV,ONP,PAE,PDX,RDM,SEA,SLE,TTD}", "2026-10-18 06:48+00", null, 9223372036854775807, 201] + expect: + driver: [{scope: "CTE hits", relation: channel_message_iatas, alias: i, index: idx_cmi_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["message_id <"]}] + forbid: [{node: "Seq Scan", relations: [channel_messages, channel_message_iatas]}] + sort_exempt: [per_site_message_ranges] + budget: {buffers: 42152, planning_ms: 10} + note: "measured: 28101 / 168613 B (custom / generic), idx ,idx_cmi_site; 1.5x measured" + - id: cmi2.P21 + stmt: ListAllChannelMessagesSince + shape: "/messages since-10.2h cursor lim200 (pathfinder)" + params: ["2026-10-18 06:48+00", null, 18200000, 201] + expect: + driver: [{relation: channel_messages, alias: cm, nodes: ["Index Scan", "Index Only Scan", "Bitmap Heap Scan"]}] + forbid: [{node: "Seq Scan", relations: [channel_messages, channel_message_iatas]}] + sort_exempt: [per_site_message_ranges] + budget: {buffers: 2573, planning_ms: 10} + note: "measured: 1715 / 3132 B (custom / generic), idx ,idx_observations_packet; 1.5x measured" + - id: cmi2.P22 + stmt: ListAllChannelMessagesSince + shape: "/messages since-10.2h past the end (last page)" + params: ["2026-10-18 06:48+00", null, 18100000, 201] + expect: + driver: [{relation: channel_messages, alias: cm, nodes: ["Index Scan", "Index Only Scan", "Bitmap Heap Scan"]}] + forbid: [{node: "Seq Scan", relations: [channel_messages, channel_message_iatas]}] + sort_exempt: [per_site_message_ranges] + budget: {buffers: 2549, planning_ms: 10} + note: "measured: 1699 / 2997 B (custom / generic), idx ,idx_observations_packet; 1.5x measured" + - id: cmi2.P23 + stmt: ListAllChannelMessagesPlain + shape: "/messages plain lim100" + params: [9223372036854775807, 101] + expect: + driver: [{relation: channel_messages, alias: cm, nodes: ["Index Scan", "Index Only Scan", "Bitmap Heap Scan"]}] + forbid: [{node: "Seq Scan", relations: [channel_messages, channel_message_iatas]}] + sort_exempt: [per_site_message_ranges] + budget: {buffers: 1000, planning_ms: 10} + note: "measured: 877 / 2174 B (custom / generic), idx ,idx_observations_packet; measured budget <=1000" + - id: cmi2.P24 + stmt: ListAllChannelMessagesPlainScoped + shape: "/messages scope #fr lim20" + params: [175, 9223372036854775807, 21] + expect: + driver: [{relation: channel_messages, alias: cm, nodes: ["Index Scan", "Index Only Scan", "Bitmap Heap Scan"]}] + forbid: [{node: "Seq Scan", relations: [channel_messages, channel_message_iatas]}] + sort_exempt: [per_site_message_ranges] + budget: {buffers: 83595, planning_ms: 10} + known_issue: "/messages?scope= has no bounded path until it reads channel_message_iatas by scope" + note: "measured: 55730 / 50294 B (custom / generic), idx ,cm_packet_hash_key,idx_observations_packet,idx_packets_scope; 1.5x measured" + - id: cmi2.P25 + stmt: ListAllChannelMessagesPlainScoped + shape: "/messages scope #rheinland (0 msgs)" + params: [5, 9223372036854775807, 51] + expect: + driver: [{relation: channel_messages, alias: cm, nodes: ["Index Scan", "Index Only Scan", "Bitmap Heap Scan"]}] + forbid: [{node: "Seq Scan", relations: [channel_messages, channel_message_iatas]}] + sort_exempt: [per_site_message_ranges] + budget: {buffers: 180, planning_ms: 10} + note: "measured: 120 / 120 B (custom / generic), idx ,cm_packet_hash_key,idx_observations_packet,idx_packets_scope; 1.5x measured" + - id: cmi2.P26 + stmt: ListChannelMessagesSinceWalk + shape: "SinceWalk ch5 YUL since-3d lim50" + params: [5, "2026-10-15 17:00+00", "{YUL}", null, 9223372036854775807, 51] + expect: + driver: [{relation: channel_messages, alias: cm, index: idx_channel_messages_channel_id, index_cond: ["channel_id =", "id <"]}] + forbid: [{node: "Seq Scan", relations: [channel_messages, channel_message_iatas]}] + sort_exempt: [per_site_message_ranges] + budget: {buffers: 63578, planning_ms: 10} + note: "measured: 42385 / 192982 B (custom / generic), idx ,idx_channel_messages_channel_id,idx_cmi_site,idx_observations_packet; 1.5x measured" + - id: cmi2.P27 + stmt: ListChannelMessagesSinceWalk + shape: "SinceWalk ch27 since-10.2h lim50" + params: [27, "2026-10-18 06:48+00", "{}", null, 9223372036854775807, 51] + expect: + driver: [{relation: channel_messages, alias: cm, index: idx_channel_messages_channel_id, index_cond: ["channel_id =", "id <"]}] + forbid: [{node: "Seq Scan", relations: [channel_messages, channel_message_iatas]}] + sort_exempt: [per_site_message_ranges] + budget: {buffers: 620, planning_ms: 10} + note: "measured: 413 / 572 B (custom / generic), idx ,idx_channel_messages_channel_id,idx_observations_packet; 1.5x measured" + - id: cmi2.P32 + stmt: ListAllChannelMessagesPlainScoped + shape: "/messages scope #nl lim50" + params: [50, 9223372036854775807, 51] + expect: + driver: [{relation: channel_messages, alias: cm, nodes: ["Index Scan", "Index Only Scan", "Bitmap Heap Scan"]}] + forbid: [{node: "Seq Scan", relations: [channel_messages, channel_message_iatas]}] + sort_exempt: [per_site_message_ranges] + budget: {buffers: 28782, planning_ms: 10} + known_issue: "/messages?scope= has no bounded path until it reads channel_message_iatas by scope" + note: "measured: 19188 / 97204 B (custom / generic), idx idx_observations_packet,idx_packets_scope; 1.5x measured" + - id: cmi2.P16_cmi + stmt: ListChannelMessagesPlainScoped + shape: "PlainScoped ch27 #fr lim50" + params: [27, 175, 9223372036854775807, 51] + expect: + driver: [{scope: "CTE hits", relation: channel_message_iatas, alias: i, index: idx_cmi_channel_scope, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["channel_id =", "scope_id =", "message_id <"]}] + forbid: [{node: "Seq Scan", relations: [channel_messages, channel_message_iatas]}] + sort_exempt: [per_site_message_ranges] + budget: {buffers: 1077, planning_ms: 10} + note: "measured: 718 / 718 B (custom / generic), idx channels_pkey,cm_pkey,idx_cmi_channel_scope,packets_pkey,transport_scopes_pkey; 1.5x measured" + - id: cmi2.P17_cmi + stmt: ListChannelMessagesPlainScoped + shape: "PlainScoped ch27 #bx (1 msg, deep)" + params: [27, 65, 9223372036854775807, 51] + expect: + driver: [{scope: "CTE hits", relation: channel_message_iatas, alias: i, index: idx_cmi_channel_scope, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["channel_id =", "scope_id =", "message_id <"]}] + forbid: [{node: "Seq Scan", relations: [channel_messages, channel_message_iatas]}] + sort_exempt: [per_site_message_ranges] + budget: {buffers: 50, planning_ms: 10} + note: "measured: 18 / 18 B (custom / generic), idx channels_pkey,cm_pkey,idx_cmi_channel_scope,packets_pkey,transport_scopes_pkey; 1.5x measured" + - id: cmi2.P18_cmi + stmt: ListChannelMessagesPlainScoped + shape: "PlainScoped ch27 #rheinland (0 msgs)" + params: [27, 5, 9223372036854775807, 51] + expect: + driver: [{scope: "CTE hits", relation: channel_message_iatas, alias: i, index: idx_cmi_channel_scope, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["channel_id =", "scope_id =", "message_id <"]}] + forbid: [{node: "Seq Scan", relations: [channel_messages, channel_message_iatas]}] + sort_exempt: [per_site_message_ranges] + budget: {buffers: 50, planning_ms: 10} + note: "measured: 3 / 3 B (custom / generic), idx channels_pkey,cm_pkey,idx_cmi_channel_scope,packets_pkey,transport_scopes_pkey; 1.5x measured" + - id: cmi2.P19_cmi + stmt: ListChannelMessagesPlainScoped + shape: "PlainScoped ch5 #rheinland (0 msgs)" + params: [5, 5, 9223372036854775807, 51] + expect: + driver: [{scope: "CTE hits", relation: channel_message_iatas, alias: i, index: idx_cmi_channel_scope, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["channel_id =", "scope_id =", "message_id <"]}] + forbid: [{node: "Seq Scan", relations: [channel_messages, channel_message_iatas]}] + sort_exempt: [per_site_message_ranges] + budget: {buffers: 50, planning_ms: 10} + note: "measured: 3 / 3 B (custom / generic), idx channels_pkey,cm_pkey,idx_cmi_channel_scope,packets_pkey,transport_scopes_pkey; 1.5x measured" + - id: cmi2.P28_cmi + stmt: ListChannelMessagesPlainScoped + shape: "PlainScoped ch23 #de (5 msgs)" + params: [23, 2, 9223372036854775807, 51] + expect: + driver: [{scope: "CTE hits", relation: channel_message_iatas, alias: i, index: idx_cmi_channel_scope, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["channel_id =", "scope_id =", "message_id <"]}] + forbid: [{node: "Seq Scan", relations: [channel_messages, channel_message_iatas]}] + sort_exempt: [per_site_message_ranges] + budget: {buffers: 111, planning_ms: 10} + note: "measured: 74 / 74 B (custom / generic), idx channels_pkey,cm_pkey,idx_cmi_channel_scope,packets_pkey,transport_scopes_pkey; 1.5x measured" + - id: cmi2.P29_cmi + stmt: ListChannelMessagesPlainScoped + shape: "PlainScoped ch5 #de lim50" + params: [5, 2, 9223372036854775807, 51] + expect: + driver: [{scope: "CTE hits", relation: channel_message_iatas, alias: i, index: idx_cmi_channel_scope, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["channel_id =", "scope_id =", "message_id <"]}] + forbid: [{node: "Seq Scan", relations: [channel_messages, channel_message_iatas]}] + sort_exempt: [per_site_message_ranges] + budget: {buffers: 1079, planning_ms: 10} + note: "measured: 719 / 719 B (custom / generic), idx channels_pkey,cm_pkey,idx_cmi_channel_scope,packets_pkey,transport_scopes_pkey; 1.5x measured" + - id: cmi2.P30_cmi + stmt: ListChannelMessagesPlainScoped + shape: "PlainScoped ch23 #nl (0 msgs)" + params: [23, 50, 9223372036854775807, 51] + expect: + driver: [{scope: "CTE hits", relation: channel_message_iatas, alias: i, index: idx_cmi_channel_scope, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["channel_id =", "scope_id =", "message_id <"]}] + forbid: [{node: "Seq Scan", relations: [channel_messages, channel_message_iatas]}] + sort_exempt: [per_site_message_ranges] + budget: {buffers: 50, planning_ms: 10} + note: "measured: 3 / 3 B (custom / generic), idx channels_pkey,cm_pkey,idx_cmi_channel_scope,packets_pkey,transport_scopes_pkey; 1.5x measured" + - id: cmi2.P31_cmi + stmt: ListChannelMessagesPlainScoped + shape: "PlainScoped ch27 #nl lim200" + params: [27, 50, 9223372036854775807, 201] + expect: + driver: [{scope: "CTE hits", relation: channel_message_iatas, alias: i, index: idx_cmi_channel_scope, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["channel_id =", "scope_id =", "message_id <"]}] + forbid: [{node: "Seq Scan", relations: [channel_messages, channel_message_iatas]}] + sort_exempt: [per_site_message_ranges] + budget: {buffers: 2459, planning_ms: 10} + note: "measured: 1639 / 2821 B (custom / generic), idx channels_pkey,cm_pkey,idx_cmi_channel_scope,packets_pkey,transport_scopes_pkey; 1.5x measured" + - id: cmi2.P33_cmi + stmt: ListChannelMessagesPlainScoped + shape: "PlainScoped ch5 #bx (0 msgs)" + params: [5, 65, 9223372036854775807, 51] + expect: + driver: [{scope: "CTE hits", relation: channel_message_iatas, alias: i, index: idx_cmi_channel_scope, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["channel_id =", "scope_id =", "message_id <"]}] + forbid: [{node: "Seq Scan", relations: [channel_messages, channel_message_iatas]}] + sort_exempt: [per_site_message_ranges] + budget: {buffers: 50, planning_ms: 10} + note: "measured: 3 / 3 B (custom / generic), idx channels_pkey,cm_pkey,idx_cmi_channel_scope,packets_pkey,transport_scopes_pkey; 1.5x measured" + - id: cmi2.P34_cmi + stmt: ListChannelMessagesPlainScoped + shape: "PlainScoped ch5 #de deep cursor" + params: [5, 2, 14994622, 51] + expect: + driver: [{scope: "CTE hits", relation: channel_message_iatas, alias: i, index: idx_cmi_channel_scope, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["channel_id =", "scope_id =", "message_id <"]}] + forbid: [{node: "Seq Scan", relations: [channel_messages, channel_message_iatas]}] + sort_exempt: [per_site_message_ranges] + budget: {buffers: 1077, planning_ms: 10} + note: "measured: 718 / 718 B (custom / generic), idx channels_pkey,cm_pkey,idx_cmi_channel_scope,packets_pkey,transport_scopes_pkey; 1.5x measured" + - id: cmi2.v_ch27_yul_cursor_3e9 + stmt: ListChannelMessagesByIATAs + shape: "channel endpoint, cursor 3e9 (>= 2^31)" + params: ["{YUL}", 27, 3000000000, 51] + expect: + driver: [{scope: "CTE hits", relation: channel_message_iatas, alias: i, index: idx_cmi_channel_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["channel_id =", "message_id <"]}] + forbid: [{node: "Seq Scan", relations: [channel_messages, channel_message_iatas]}] + sort_exempt: [per_site_message_ranges] + budget: {buffers: 600, planning_ms: 10} + note: "no measured run; measured budget <=600" + - id: cmi2.v_all_yul_cursor_3e9 + stmt: ListAllChannelMessagesByIATAs + shape: "/messages?iatas=YUL, cursor 3e9 (>= 2^31; today: 500)" + params: ["{YUL}", 3000000000, 51] + expect: + driver: [{scope: "CTE hits", relation: channel_message_iatas, alias: i, index: idx_cmi_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["message_id <"]}] + forbid: [{node: "Seq Scan", relations: [channel_messages, channel_message_iatas]}] + sort_exempt: [per_site_message_ranges] + budget: {buffers: 600, planning_ms: 10} + note: "no measured run; measured budget <=600" + - id: cmi2.v_all_plain_cursor_3e9 + stmt: ListAllChannelMessagesPlain + shape: "/messages plain, cursor 3e9 (>= 2^31; today: 500)" + params: [3000000000, 101] + expect: + driver: [{relation: channel_messages, alias: cm, nodes: ["Index Scan", "Index Only Scan", "Bitmap Heap Scan"]}] + forbid: [{node: "Seq Scan", relations: [channel_messages, channel_message_iatas]}] + sort_exempt: [per_site_message_ranges] + budget: {buffers: 1000, planning_ms: 10} + note: "no measured run; measured budget <=1000" + - id: cmi2.v_all_since_cursor_3e9 + stmt: ListAllChannelMessagesSince + shape: "/messages since T0-10.2h, cursor 3e9" + params: ["2026-10-18 06:48+00", null, 3000000000, 201] + expect: + driver: [{relation: channel_messages, alias: cm, nodes: ["Index Scan", "Index Only Scan", "Bitmap Heap Scan"]}] + forbid: [{node: "Seq Scan", relations: [channel_messages, channel_message_iatas]}] + sort_exempt: [per_site_message_ranges] + budget: {buffers: 25000, planning_ms: 10} + note: "no measured run; class budget" + - id: cmi2.v_ch27_bogus5000 + stmt: ListChannelMessagesByIATAs + shape: "channel 27 + 5,000 bogus IATAs" + params: [{r: cmi_bogus5000}, 27, 9223372036854775807, 51] + expect: + driver: [{scope: "CTE hits", relation: channel_message_iatas, alias: i, index: idx_cmi_channel_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["channel_id =", "message_id <"]}] + forbid: [{node: "Seq Scan", relations: [channel_messages, channel_message_iatas]}] + sort_exempt: [per_site_message_ranges] + budget: {buffers: 25000, planning_ms: 60} + note: "no measured run; measured budget <=25000" + - id: cmi2.v_all_bogus5000 + stmt: ListAllChannelMessagesByIATAs + shape: "/messages + 5,000 bogus IATAs" + params: [{r: cmi_bogus5000}, 9223372036854775807, 51] + expect: + driver: [{scope: "CTE hits", relation: channel_message_iatas, alias: i, index: idx_cmi_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["message_id <"]}] + forbid: [{node: "Seq Scan", relations: [channel_messages, channel_message_iatas]}] + sort_exempt: [per_site_message_ranges] + budget: {buffers: 25000, planning_ms: 60} + note: "no measured run; measured budget <=25000" + - id: cmi2.v_ch27_allcmi + stmt: ListChannelMessagesByIATAs + shape: "channel 27 + every site with a message" + params: [{r: cmi_all_sites}, 27, 9223372036854775807, 51] + expect: + driver: [{scope: "CTE hits", relation: channel_message_iatas, alias: i, index: idx_cmi_channel_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["channel_id =", "message_id <"]}] + forbid: [{node: "Seq Scan", relations: [channel_messages, channel_message_iatas]}] + sort_exempt: [per_site_message_ranges] + budget: {buffers: 25000, planning_ms: 10} + note: "no measured run; measured budget <=25000" + - id: cmi2.v_all_allcmi + stmt: ListAllChannelMessagesByIATAs + shape: "/messages + every site with a message" + params: [{r: cmi_all_sites}, 9223372036854775807, 51] + expect: + driver: [{scope: "CTE hits", relation: channel_message_iatas, alias: i, index: idx_cmi_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["message_id <"]}] + forbid: [{node: "Seq Scan", relations: [channel_messages, channel_message_iatas]}] + sort_exempt: [per_site_message_ranges] + budget: {buffers: 25000, planning_ms: 10} + note: "no measured run; measured budget <=25000" + - id: cmi2.v_ch27_new_zerohit_scoped + stmt: ListChannelMessagesByIATAsScoped + shape: "channel 27 + YUL + a scope with no messages there (#rheinland)" + params: ["{YUL}", 27, 5, 9223372036854775807, 51] + expect: + driver: [{scope: "CTE hits", relation: channel_message_iatas, alias: i, index: idx_cmi_channel_site_scope, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["channel_id =", "message_id <"]}] + forbid: [{node: "Seq Scan", relations: [channel_messages, channel_message_iatas]}] + sort_exempt: [per_site_message_ranges] + budget: {buffers: 10, planning_ms: 10} + note: "no measured run; measured budget <=10" + - id: cmi2.ingest.add_iata + stmt: AddChannelMessageIATA + shape: "AddChannelMessageIATA for the newest message hash (plain EXPLAIN; ingest pool, auto)" + params: [YUL, {r: cm_newest_hash}] + expect: + driver: [{relation: channel_messages, alias: cm, index: channel_messages_packet_hash_key, index_cond: ["packet_hash ="]}] + - id: cmi2.maint.fill_by_messages + stmt: FillChannelMessageIATAsByMessages + shape: "backfill / reconcile message part over one 10K-message id range (plain EXPLAIN; maint)" + params: [17000000, 17010000] + expect: + driver: [{relation: channel_messages, alias: cm, index: channel_messages_pkey, index_cond: ["id >"]}] + + # ---- stats, adverts, traces and telemetry (legacy readers and pending rewrites) ---- + - id: s7.tn.LIS_1d + stmt: GetTopNodesProbe + shape: "tn probe LIS_1d" + params: [{raw: "'2026-10-17 17:00+00'::timestamptz"}, {raw: "ARRAY['LIS']::bpchar[]"}, {raw: "10"}] + expect: + budget: {buffers: 251, planning_ms: 10} + note: "measured: 167 / 167 B (custom / generic); budget: 1.5x measured" + - id: s7.tn.LIS_7d + stmt: GetTopNodesProbe + shape: "tn probe LIS_7d" + params: [{raw: "'2026-10-11 17:00+00'::timestamptz"}, {raw: "ARRAY['LIS']::bpchar[]"}, {raw: "10"}] + expect: + budget: {buffers: 1262, planning_ms: 10} + note: "measured: 841 / 841 B (custom / generic); budget: 1.5x measured" + - id: s7.tn.LIS_14d + stmt: GetTopNodesProbe + shape: "tn probe LIS_14d" + params: [{raw: "'2026-10-04 17:00+00'::timestamptz"}, {raw: "ARRAY['LIS']::bpchar[]"}, {raw: "10"}] + expect: + budget: {buffers: 1262, planning_ms: 10} + note: "measured: 841 / 841 B (custom / generic); budget: 1.5x measured" + - id: s7.tn.SEA_1d + stmt: GetTopNodesProbe + shape: "tn probe SEA_1d" + params: [{raw: "'2026-10-17 17:00+00'::timestamptz"}, {raw: "ARRAY['SEA']::bpchar[]"}, {raw: "10"}] + expect: + budget: {buffers: 351, planning_ms: 10} + note: "measured: 234 / 234 B (custom / generic); budget: 1.5x measured" + - id: s7.tn.SEA_7d + stmt: GetTopNodesProbe + shape: "tn probe SEA_7d" + params: [{raw: "'2026-10-11 17:00+00'::timestamptz"}, {raw: "ARRAY['SEA']::bpchar[]"}, {raw: "10"}] + expect: + budget: {buffers: 1916, planning_ms: 10} + note: "measured: 1277 / 1277 B (custom / generic); budget: 1.5x measured" + - id: s7.tn.SEA_14d + stmt: GetTopNodesProbe + shape: "tn probe SEA_14d" + params: [{raw: "'2026-10-04 17:00+00'::timestamptz"}, {raw: "ARRAY['SEA']::bpchar[]"}, {raw: "10"}] + expect: + budget: {buffers: 1916, planning_ms: 10} + note: "measured: 1277 / 1277 B (custom / generic); budget: 1.5x measured" + - id: s7.tn.pt3_1d + stmt: GetTopNodesProbe + shape: "tn probe pt3_1d" + params: [{raw: "'2026-10-17 17:00+00'::timestamptz"}, {raw: "ARRAY['FAO','LIS','OPO']::bpchar[]"}, {raw: "10"}] + expect: + budget: {buffers: 557, planning_ms: 10} + note: "measured: 371 / 371 B (custom / generic); budget: 1.5x measured" + - id: s7.tn.pt3_7d + stmt: GetTopNodesProbe + shape: "tn probe pt3_7d" + params: [{raw: "'2026-10-11 17:00+00'::timestamptz"}, {raw: "ARRAY['FAO','LIS','OPO']::bpchar[]"}, {raw: "10"}] + expect: + budget: {buffers: 3089, planning_ms: 10} + note: "measured: 2059 / 2059 B (custom / generic); budget: 1.5x measured" + - id: s7.tn.pt3_14d + stmt: GetTopNodesProbe + shape: "tn probe pt3_14d" + params: [{raw: "'2026-10-04 17:00+00'::timestamptz"}, {raw: "ARRAY['FAO','LIS','OPO']::bpchar[]"}, {raw: "10"}] + expect: + budget: {buffers: 3089, planning_ms: 10} + note: "measured: 2059 / 2059 B (custom / generic); budget: 1.5x measured" + - id: s7.tn.nrw6_1d + stmt: GetTopNodesProbe + shape: "tn probe nrw6_1d" + params: [{raw: "'2026-10-17 17:00+00'::timestamptz"}, {raw: "ARRAY['AAH','CGN','DTM','DUS','ESS','PAD']::bpchar[]"}, {raw: "10"}] + expect: + budget: {buffers: 1295, planning_ms: 10} + note: "measured: 863 / 863 B (custom / generic); budget: 1.5x measured" + - id: s7.tn.nrw6_7d + stmt: GetTopNodesProbe + shape: "tn probe nrw6_7d" + params: [{raw: "'2026-10-11 17:00+00'::timestamptz"}, {raw: "ARRAY['AAH','CGN','DTM','DUS','ESS','PAD']::bpchar[]"}, {raw: "10"}] + expect: + budget: {buffers: 8049, planning_ms: 10} + note: "measured: 5366 / 5366 B (custom / generic); budget: 1.5x measured" + - id: s7.tn.nrw6_14d + stmt: GetTopNodesProbe + shape: "tn probe nrw6_14d" + params: [{raw: "'2026-10-04 17:00+00'::timestamptz"}, {raw: "ARRAY['AAH','CGN','DTM','DUS','ESS','PAD']::bpchar[]"}, {raw: "10"}] + expect: + budget: {buffers: 8049, planning_ms: 10} + note: "measured: 5366 / 5366 B (custom / generic); budget: 1.5x measured" + - id: s7.tn.co10_1d + stmt: GetTopNodesProbe + shape: "tn probe co10_1d" + params: [{raw: "'2026-10-17 17:00+00'::timestamptz"}, {raw: "ARRAY['CEZ','COS','DEN','DRO','EGE','FNL','GJT','LAA','MTJ','PUB']::bpchar[]"}, {raw: "10"}] + expect: + budget: {buffers: 1703, planning_ms: 10} + note: "measured: 1135 / 1135 B (custom / generic); budget: 1.5x measured" + - id: s7.tn.co10_7d + stmt: GetTopNodesProbe + shape: "tn probe co10_7d" + params: [{raw: "'2026-10-11 17:00+00'::timestamptz"}, {raw: "ARRAY['CEZ','COS','DEN','DRO','EGE','FNL','GJT','LAA','MTJ','PUB']::bpchar[]"}, {raw: "10"}] + expect: + budget: {buffers: 10346, planning_ms: 10} + note: "measured: 6897 / 6897 B (custom / generic); budget: 1.5x measured" + - id: s7.tn.co10_14d + stmt: GetTopNodesProbe + shape: "tn probe co10_14d" + params: [{raw: "'2026-10-04 17:00+00'::timestamptz"}, {raw: "ARRAY['CEZ','COS','DEN','DRO','EGE','FNL','GJT','LAA','MTJ','PUB']::bpchar[]"}, {raw: "10"}] + expect: + budget: {buffers: 10346, planning_ms: 10} + note: "measured: 6897 / 6897 B (custom / generic); budget: 1.5x measured" + - id: s7.tn.swe19_1d + stmt: GetTopNodesProbe + shape: "tn probe swe19_1d" + params: [{raw: "'2026-10-17 17:00+00'::timestamptz"}, {raw: "ARRAY['BLE','GOT','GVX','HAD','JKG','KLR','KSD','LLA','LPI','MMX','NYO','ORB','OSD','RNB','SDL','STO','VBY','VST','VXO']::bpchar[]"}, {raw: "10"}] + expect: + budget: {buffers: 3122, planning_ms: 10} + note: "measured: 2081 / 2081 B (custom / generic); budget: 1.5x measured" + - id: s7.tn.swe19_7d + stmt: GetTopNodesProbe + shape: "tn probe swe19_7d" + params: [{raw: "'2026-10-11 17:00+00'::timestamptz"}, {raw: "ARRAY['BLE','GOT','GVX','HAD','JKG','KLR','KSD','LLA','LPI','MMX','NYO','ORB','OSD','RNB','SDL','STO','VBY','VST','VXO']::bpchar[]"}, {raw: "10"}] + expect: + budget: {buffers: 19352, planning_ms: 10} + note: "measured: 12901 / 12901 B (custom / generic); budget: 1.5x measured" + - id: s7.tn.swe19_14d + stmt: GetTopNodesProbe + shape: "tn probe swe19_14d" + params: [{raw: "'2026-10-04 17:00+00'::timestamptz"}, {raw: "ARRAY['BLE','GOT','GVX','HAD','JKG','KLR','KSD','LLA','LPI','MMX','NYO','ORB','OSD','RNB','SDL','STO','VBY','VST','VXO']::bpchar[]"}, {raw: "10"}] + expect: + budget: {buffers: 19352, planning_ms: 10} + note: "measured: 12901 / 12901 B (custom / generic); budget: 1.5x measured" + - id: s7.tn.gbr22_1d + stmt: GetTopNodesProbe + shape: "tn probe gbr22_1d" + params: [{raw: "'2026-10-17 17:00+00'::timestamptz"}, {raw: "ARRAY['BHX','BLK','BOH','BRS','CAX','CEG','DSA','EMA','EXT','HUY','LCY','LON','LPL','MAN','MME','NQT','NQY','NWI','ORM','PLH','PME','VLY']::bpchar[]"}, {raw: "10"}] + expect: + budget: {buffers: 3546, planning_ms: 10} + note: "measured: 2364 / 2364 B (custom / generic); budget: 1.5x measured" + - id: s7.tn.gbr22_7d + stmt: GetTopNodesProbe + shape: "tn probe gbr22_7d" + params: [{raw: "'2026-10-11 17:00+00'::timestamptz"}, {raw: "ARRAY['BHX','BLK','BOH','BRS','CAX','CEG','DSA','EMA','EXT','HUY','LCY','LON','LPL','MAN','MME','NQT','NQY','NWI','ORM','PLH','PME','VLY']::bpchar[]"}, {raw: "10"}] + expect: + budget: {buffers: 21869, planning_ms: 10} + note: "measured: 14579 / 14579 B (custom / generic); budget: 1.5x measured" + - id: s7.tn.gbr22_14d + stmt: GetTopNodesProbe + shape: "tn probe gbr22_14d" + params: [{raw: "'2026-10-04 17:00+00'::timestamptz"}, {raw: "ARRAY['BHX','BLK','BOH','BRS','CAX','CEG','DSA','EMA','EXT','HUY','LCY','LON','LPL','MAN','MME','NQT','NQY','NWI','ORM','PLH','PME','VLY']::bpchar[]"}, {raw: "10"}] + expect: + budget: {buffers: 21869, planning_ms: 10} + note: "measured: 14579 / 14579 B (custom / generic); budget: 1.5x measured" + - id: s7.tn.top50_7d + stmt: GetTopNodesProbe + shape: "tn probe top50_7d" + params: [{raw: "'2026-10-11 17:00+00'::timestamptz"}, {raw: "ARRAY['SEA','KRK','PDX','BNA','RDU','ATL','TYS','AVL','CHA','TRI','POZ','STR','GSP','PRG','PAD','KTW','FRA','RDO','BRQ','BRN','BWE','GSO','AMS','AHN','MOW','PSK','CGN','YVR','ORD','RST','LEJ','SPM','DTM','PAE','BER','SJC','LIN','DUS','LOZ','BOS','VOD','VIE','TRN','PED','WAW','WRO','BLI','HRA','NUE','IDA']::bpchar[]"}, {raw: "10"}] + expect: + budget: {buffers: 71594, planning_ms: 10} + note: "measured: 47729 / 47729 B (custom / generic); budget: 1.5x measured" + - id: s7.ta.LIS_1d + stmt: GetStatsTopAdvertisersProbe + shape: "ta probe LIS_1d" + params: [{raw: "'flood'"}, {raw: "'2026-10-17 17:00+00'::timestamptz"}, {raw: "ARRAY['LIS']::bpchar[]"}, {raw: "10"}] + expect: + budget: {buffers: 3897, planning_ms: 10} + note: "measured: 2598 / 6096 B (custom / generic); budget: 1.5x measured" + - id: s7.ta.LIS_7d + stmt: GetStatsTopAdvertisersProbe + shape: "ta probe LIS_7d" + params: [{raw: "'flood'"}, {raw: "'2026-10-11 17:00+00'::timestamptz"}, {raw: "ARRAY['LIS']::bpchar[]"}, {raw: "10"}] + expect: + budget: {buffers: 7752, planning_ms: 10} + note: "measured: 5168 / 41844 B (custom / generic); budget: 1.5x measured" + - id: s7.ta.LIS_14d + stmt: GetStatsTopAdvertisersProbe + shape: "ta probe LIS_14d" + params: [{raw: "'flood'"}, {raw: "'2026-10-04 17:00+00'::timestamptz"}, {raw: "ARRAY['LIS']::bpchar[]"}, {raw: "10"}] + expect: + budget: {buffers: 7752, planning_ms: 10} + note: "measured: 5168 / 41833 B (custom / generic); budget: 1.5x measured" + - id: s7.ta.SEA_1d + stmt: GetStatsTopAdvertisersProbe + shape: "ta probe SEA_1d" + params: [{raw: "'flood'"}, {raw: "'2026-10-17 17:00+00'::timestamptz"}, {raw: "ARRAY['SEA']::bpchar[]"}, {raw: "10"}] + expect: + budget: {buffers: 5954, planning_ms: 10} + note: "measured: 3969 / 6165 B (custom / generic); budget: 1.5x measured" + - id: s7.ta.SEA_7d + stmt: GetStatsTopAdvertisersProbe + shape: "ta probe SEA_7d" + params: [{raw: "'flood'"}, {raw: "'2026-10-11 17:00+00'::timestamptz"}, {raw: "ARRAY['SEA']::bpchar[]"}, {raw: "10"}] + expect: + budget: {buffers: 21107, planning_ms: 10} + note: "measured: 14071 / 41908 B (custom / generic); budget: 1.5x measured" + - id: s7.ta.SEA_14d + stmt: GetStatsTopAdvertisersProbe + shape: "ta probe SEA_14d" + params: [{raw: "'flood'"}, {raw: "'2026-10-04 17:00+00'::timestamptz"}, {raw: "ARRAY['SEA']::bpchar[]"}, {raw: "10"}] + expect: + budget: {buffers: 21107, planning_ms: 10} + note: "measured: 14071 / 42320 B (custom / generic); budget: 1.5x measured" + - id: s7.ta.pt3_1d + stmt: GetStatsTopAdvertisersProbe + shape: "ta probe pt3_1d" + params: [{raw: "'flood'"}, {raw: "'2026-10-17 17:00+00'::timestamptz"}, {raw: "ARRAY['FAO','LIS','OPO']::bpchar[]"}, {raw: "10"}] + expect: + budget: {buffers: 4295, planning_ms: 10} + note: "measured: 2863 / 6394 B (custom / generic); budget: 1.5x measured" + - id: s7.ta.pt3_7d + stmt: GetStatsTopAdvertisersProbe + shape: "ta probe pt3_7d" + params: [{raw: "'flood'"}, {raw: "'2026-10-11 17:00+00'::timestamptz"}, {raw: "ARRAY['FAO','LIS','OPO']::bpchar[]"}, {raw: "10"}] + expect: + budget: {buffers: 9854, planning_ms: 10} + note: "measured: 6569 / 43165 B (custom / generic); budget: 1.5x measured" + - id: s7.ta.pt3_14d + stmt: GetStatsTopAdvertisersProbe + shape: "ta probe pt3_14d" + params: [{raw: "'flood'"}, {raw: "'2026-10-04 17:00+00'::timestamptz"}, {raw: "ARRAY['FAO','LIS','OPO']::bpchar[]"}, {raw: "10"}] + expect: + budget: {buffers: 9854, planning_ms: 10} + note: "measured: 6569 / 43027 B (custom / generic); budget: 1.5x measured" + - id: s7.ta.nrw6_1d + stmt: GetStatsTopAdvertisersProbe + shape: "ta probe nrw6_1d" + params: [{raw: "'flood'"}, {raw: "'2026-10-17 17:00+00'::timestamptz"}, {raw: "ARRAY['AAH','CGN','DTM','DUS','ESS','PAD']::bpchar[]"}, {raw: "10"}] + expect: + budget: {buffers: 6906, planning_ms: 10} + note: "measured: 4604 / 6919 B (custom / generic); budget: 1.5x measured" + - id: s7.ta.nrw6_7d + stmt: GetStatsTopAdvertisersProbe + shape: "ta probe nrw6_7d" + params: [{raw: "'flood'"}, {raw: "'2026-10-11 17:00+00'::timestamptz"}, {raw: "ARRAY['AAH','CGN','DTM','DUS','ESS','PAD']::bpchar[]"}, {raw: "10"}] + expect: + budget: {buffers: 27884, planning_ms: 10} + note: "measured: 18589 / 46508 B (custom / generic); budget: 1.5x measured" + - id: s7.ta.nrw6_14d + stmt: GetStatsTopAdvertisersProbe + shape: "ta probe nrw6_14d" + params: [{raw: "'flood'"}, {raw: "'2026-10-04 17:00+00'::timestamptz"}, {raw: "ARRAY['AAH','CGN','DTM','DUS','ESS','PAD']::bpchar[]"}, {raw: "10"}] + expect: + budget: {buffers: 27884, planning_ms: 10} + note: "measured: 18589 / 46438 B (custom / generic); budget: 1.5x measured" + - id: s7.ta.co10_1d + stmt: GetStatsTopAdvertisersProbe + shape: "ta probe co10_1d" + params: [{raw: "'flood'"}, {raw: "'2026-10-17 17:00+00'::timestamptz"}, {raw: "ARRAY['CEZ','COS','DEN','DRO','EGE','FNL','GJT','LAA','MTJ','PUB']::bpchar[]"}, {raw: "10"}] + expect: + budget: {buffers: 6503, planning_ms: 10} + note: "measured: 4335 / 7208 B (custom / generic); budget: 1.5x measured" + - id: s7.ta.co10_7d + stmt: GetStatsTopAdvertisersProbe + shape: "ta probe co10_7d" + params: [{raw: "'flood'"}, {raw: "'2026-10-11 17:00+00'::timestamptz"}, {raw: "ARRAY['CEZ','COS','DEN','DRO','EGE','FNL','GJT','LAA','MTJ','PUB']::bpchar[]"}, {raw: "10"}] + expect: + budget: {buffers: 24174, planning_ms: 10} + note: "measured: 16116 / 48133 B (custom / generic); budget: 1.5x measured" + - id: s7.ta.co10_14d + stmt: GetStatsTopAdvertisersProbe + shape: "ta probe co10_14d" + params: [{raw: "'flood'"}, {raw: "'2026-10-04 17:00+00'::timestamptz"}, {raw: "ARRAY['CEZ','COS','DEN','DRO','EGE','FNL','GJT','LAA','MTJ','PUB']::bpchar[]"}, {raw: "10"}] + expect: + budget: {buffers: 24174, planning_ms: 10} + note: "measured: 16116 / 48112 B (custom / generic); budget: 1.5x measured" + - id: s7.ta.swe19_1d + stmt: GetStatsTopAdvertisersProbe + shape: "ta probe swe19_1d" + params: [{raw: "'flood'"}, {raw: "'2026-10-17 17:00+00'::timestamptz"}, {raw: "ARRAY['BLE','GOT','GVX','HAD','JKG','KLR','KSD','LLA','LPI','MMX','NYO','ORB','OSD','RNB','SDL','STO','VBY','VST','VXO']::bpchar[]"}, {raw: "10"}] + expect: + budget: {buffers: 8664, planning_ms: 10} + note: "measured: 5776 / 8200 B (custom / generic); budget: 1.5x measured" + - id: s7.ta.swe19_7d + stmt: GetStatsTopAdvertisersProbe + shape: "ta probe swe19_7d" + params: [{raw: "'flood'"}, {raw: "'2026-10-11 17:00+00'::timestamptz"}, {raw: "ARRAY['BLE','GOT','GVX','HAD','JKG','KLR','KSD','LLA','LPI','MMX','NYO','ORB','OSD','RNB','SDL','STO','VBY','VST','VXO']::bpchar[]"}, {raw: "10"}] + expect: + budget: {buffers: 38816, planning_ms: 10} + note: "measured: 25877 / 54141 B (custom / generic); budget: 1.5x measured" + - id: s7.ta.swe19_14d + stmt: GetStatsTopAdvertisersProbe + shape: "ta probe swe19_14d" + params: [{raw: "'flood'"}, {raw: "'2026-10-04 17:00+00'::timestamptz"}, {raw: "ARRAY['BLE','GOT','GVX','HAD','JKG','KLR','KSD','LLA','LPI','MMX','NYO','ORB','OSD','RNB','SDL','STO','VBY','VST','VXO']::bpchar[]"}, {raw: "10"}] + expect: + budget: {buffers: 38816, planning_ms: 10} + note: "measured: 25877 / 54048 B (custom / generic); budget: 1.5x measured" + - id: s7.ta.gbr22_1d + stmt: GetStatsTopAdvertisersProbe + shape: "ta probe gbr22_1d" + params: [{raw: "'flood'"}, {raw: "'2026-10-17 17:00+00'::timestamptz"}, {raw: "ARRAY['BHX','BLK','BOH','BRS','CAX','CEG','DSA','EMA','EXT','HUY','LCY','LON','LPL','MAN','MME','NQT','NQY','NWI','ORM','PLH','PME','VLY']::bpchar[]"}, {raw: "10"}] + expect: + budget: {buffers: 9231, planning_ms: 10} + note: "measured: 6154 / 8496 B (custom / generic); budget: 1.5x measured" + - id: s7.ta.gbr22_7d + stmt: GetStatsTopAdvertisersProbe + shape: "ta probe gbr22_7d" + params: [{raw: "'flood'"}, {raw: "'2026-10-11 17:00+00'::timestamptz"}, {raw: "ARRAY['BHX','BLK','BOH','BRS','CAX','CEG','DSA','EMA','EXT','HUY','LCY','LON','LPL','MAN','MME','NQT','NQY','NWI','ORM','PLH','PME','VLY']::bpchar[]"}, {raw: "10"}] + expect: + budget: {buffers: 41507, planning_ms: 10} + note: "measured: 27671 / 55750 B (custom / generic); budget: 1.5x measured" + - id: s7.ta.gbr22_14d + stmt: GetStatsTopAdvertisersProbe + shape: "ta probe gbr22_14d" + params: [{raw: "'flood'"}, {raw: "'2026-10-04 17:00+00'::timestamptz"}, {raw: "ARRAY['BHX','BLK','BOH','BRS','CAX','CEG','DSA','EMA','EXT','HUY','LCY','LON','LPL','MAN','MME','NQT','NQY','NWI','ORM','PLH','PME','VLY']::bpchar[]"}, {raw: "10"}] + expect: + budget: {buffers: 41507, planning_ms: 10} + note: "measured: 27671 / 55769 B (custom / generic); budget: 1.5x measured" + - id: s7.ta.top50_7d + stmt: GetStatsTopAdvertisersProbe + shape: "ta probe top50_7d" + params: [{raw: "'flood'"}, {raw: "'2026-10-11 17:00+00'::timestamptz"}, {raw: "ARRAY['SEA','KRK','PDX','BNA','RDU','ATL','TYS','AVL','CHA','TRI','POZ','STR','GSP','PRG','PAD','KTW','FRA','RDO','BRQ','BRN','BWE','GSO','AMS','AHN','MOW','PSK','CGN','YVR','ORD','RST','LEJ','SPM','DTM','PAE','BER','SJC','LIN','DUS','LOZ','BOS','VOD','VIE','TRN','PED','WAW','WRO','BLI','HRA','NUE','IDA']::bpchar[]"}, {raw: "10"}] + expect: + budget: {buffers: 95826, planning_ms: 10} + note: "measured: 63884 / 88853 B (custom / generic); budget: 1.5x measured" + - id: s7.pa.LIS_1d + stmt: GetPathStatsProbe + shape: "pa probe LIS_1d" + params: [{raw: "'2026-10-17 17:00+00'::timestamptz"}, {raw: "'2026-10-18 17:00+00'::timestamptz"}, {raw: "ARRAY['LIS']::bpchar[]"}] + expect: + budget: {buffers: 176, planning_ms: 10} + note: "measured: 117 / 117 B (custom / generic); budget: 1.5x measured" + - id: s7.pa.LIS_7d + stmt: GetPathStatsProbe + shape: "pa probe LIS_7d" + params: [{raw: "'2026-10-11 17:00+00'::timestamptz"}, {raw: "'2026-10-18 17:00+00'::timestamptz"}, {raw: "ARRAY['LIS']::bpchar[]"}] + expect: + budget: {buffers: 1167, planning_ms: 10} + note: "measured: 778 / 778 B (custom / generic); budget: 1.5x measured" + - id: s7.pa.LIS_14d + stmt: GetPathStatsProbe + shape: "pa probe LIS_14d" + params: [{raw: "'2026-10-04 17:00+00'::timestamptz"}, {raw: "'2026-10-18 17:00+00'::timestamptz"}, {raw: "ARRAY['LIS']::bpchar[]"}] + expect: + budget: {buffers: 1167, planning_ms: 10} + note: "measured: 778 / 778 B (custom / generic); budget: 1.5x measured" + - id: s7.pa.SEA_1d + stmt: GetPathStatsProbe + shape: "pa probe SEA_1d" + params: [{raw: "'2026-10-17 17:00+00'::timestamptz"}, {raw: "'2026-10-18 17:00+00'::timestamptz"}, {raw: "ARRAY['SEA']::bpchar[]"}] + expect: + budget: {buffers: 198, planning_ms: 10} + note: "measured: 132 / 132 B (custom / generic); budget: 1.5x measured" + - id: s7.pa.SEA_7d + stmt: GetPathStatsProbe + shape: "pa probe SEA_7d" + params: [{raw: "'2026-10-11 17:00+00'::timestamptz"}, {raw: "'2026-10-18 17:00+00'::timestamptz"}, {raw: "ARRAY['SEA']::bpchar[]"}] + expect: + budget: {buffers: 1347, planning_ms: 10} + note: "measured: 898 / 898 B (custom / generic); budget: 1.5x measured" + - id: s7.pa.SEA_14d + stmt: GetPathStatsProbe + shape: "pa probe SEA_14d" + params: [{raw: "'2026-10-04 17:00+00'::timestamptz"}, {raw: "'2026-10-18 17:00+00'::timestamptz"}, {raw: "ARRAY['SEA']::bpchar[]"}] + expect: + budget: {buffers: 1347, planning_ms: 10} + note: "measured: 898 / 898 B (custom / generic); budget: 1.5x measured" + - id: s7.pa.pt3_1d + stmt: GetPathStatsProbe + shape: "pa probe pt3_1d" + params: [{raw: "'2026-10-17 17:00+00'::timestamptz"}, {raw: "'2026-10-18 17:00+00'::timestamptz"}, {raw: "ARRAY['FAO','LIS','OPO']::bpchar[]"}] + expect: + budget: {buffers: 444, planning_ms: 10} + note: "measured: 296 / 296 B (custom / generic); budget: 1.5x measured" + - id: s7.pa.pt3_7d + stmt: GetPathStatsProbe + shape: "pa probe pt3_7d" + params: [{raw: "'2026-10-11 17:00+00'::timestamptz"}, {raw: "'2026-10-18 17:00+00'::timestamptz"}, {raw: "ARRAY['FAO','LIS','OPO']::bpchar[]"}] + expect: + budget: {buffers: 2988, planning_ms: 10} + note: "measured: 1992 / 1992 B (custom / generic); budget: 1.5x measured" + - id: s7.pa.pt3_14d + stmt: GetPathStatsProbe + shape: "pa probe pt3_14d" + params: [{raw: "'2026-10-04 17:00+00'::timestamptz"}, {raw: "'2026-10-18 17:00+00'::timestamptz"}, {raw: "ARRAY['FAO','LIS','OPO']::bpchar[]"}] + expect: + budget: {buffers: 2988, planning_ms: 10} + note: "measured: 1992 / 1992 B (custom / generic); budget: 1.5x measured" + - id: s7.pa.nrw6_1d + stmt: GetPathStatsProbe + shape: "pa probe nrw6_1d" + params: [{raw: "'2026-10-17 17:00+00'::timestamptz"}, {raw: "'2026-10-18 17:00+00'::timestamptz"}, {raw: "ARRAY['AAH','CGN','DTM','DUS','ESS','PAD']::bpchar[]"}] + expect: + budget: {buffers: 1049, planning_ms: 10} + note: "measured: 699 / 699 B (custom / generic); budget: 1.5x measured" + - id: s7.pa.nrw6_7d + stmt: GetPathStatsProbe + shape: "pa probe nrw6_7d" + params: [{raw: "'2026-10-11 17:00+00'::timestamptz"}, {raw: "'2026-10-18 17:00+00'::timestamptz"}, {raw: "ARRAY['AAH','CGN','DTM','DUS','ESS','PAD']::bpchar[]"}] + expect: + budget: {buffers: 7317, planning_ms: 10} + note: "measured: 4878 / 4878 B (custom / generic); budget: 1.5x measured" + - id: s7.pa.nrw6_14d + stmt: GetPathStatsProbe + shape: "pa probe nrw6_14d" + params: [{raw: "'2026-10-04 17:00+00'::timestamptz"}, {raw: "'2026-10-18 17:00+00'::timestamptz"}, {raw: "ARRAY['AAH','CGN','DTM','DUS','ESS','PAD']::bpchar[]"}] + expect: + budget: {buffers: 7317, planning_ms: 10} + note: "measured: 4878 / 4878 B (custom / generic); budget: 1.5x measured" + - id: s7.pa.co10_1d + stmt: GetPathStatsProbe + shape: "pa probe co10_1d" + params: [{raw: "'2026-10-17 17:00+00'::timestamptz"}, {raw: "'2026-10-18 17:00+00'::timestamptz"}, {raw: "ARRAY['CEZ','COS','DEN','DRO','EGE','FNL','GJT','LAA','MTJ','PUB']::bpchar[]"}] + expect: + budget: {buffers: 1460, planning_ms: 10} + note: "measured: 973 / 973 B (custom / generic); budget: 1.5x measured" + - id: s7.pa.co10_7d + stmt: GetPathStatsProbe + shape: "pa probe co10_7d" + params: [{raw: "'2026-10-11 17:00+00'::timestamptz"}, {raw: "'2026-10-18 17:00+00'::timestamptz"}, {raw: "ARRAY['CEZ','COS','DEN','DRO','EGE','FNL','GJT','LAA','MTJ','PUB']::bpchar[]"}] + expect: + budget: {buffers: 10052, planning_ms: 10} + note: "measured: 6701 / 6701 B (custom / generic); budget: 1.5x measured" + - id: s7.pa.co10_14d + stmt: GetPathStatsProbe + shape: "pa probe co10_14d" + params: [{raw: "'2026-10-04 17:00+00'::timestamptz"}, {raw: "'2026-10-18 17:00+00'::timestamptz"}, {raw: "ARRAY['CEZ','COS','DEN','DRO','EGE','FNL','GJT','LAA','MTJ','PUB']::bpchar[]"}] + expect: + budget: {buffers: 10052, planning_ms: 10} + note: "measured: 6701 / 6701 B (custom / generic); budget: 1.5x measured" + - id: s7.pa.swe19_1d + stmt: GetPathStatsProbe + shape: "pa probe swe19_1d" + params: [{raw: "'2026-10-17 17:00+00'::timestamptz"}, {raw: "'2026-10-18 17:00+00'::timestamptz"}, {raw: "ARRAY['BLE','GOT','GVX','HAD','JKG','KLR','KSD','LLA','LPI','MMX','NYO','ORB','OSD','RNB','SDL','STO','VBY','VST','VXO']::bpchar[]"}] + expect: + budget: {buffers: 2681, planning_ms: 10} + note: "measured: 1787 / 1787 B (custom / generic); budget: 1.5x measured" + - id: s7.pa.swe19_7d + stmt: GetPathStatsProbe + shape: "pa probe swe19_7d" + params: [{raw: "'2026-10-11 17:00+00'::timestamptz"}, {raw: "'2026-10-18 17:00+00'::timestamptz"}, {raw: "ARRAY['BLE','GOT','GVX','HAD','JKG','KLR','KSD','LLA','LPI','MMX','NYO','ORB','OSD','RNB','SDL','STO','VBY','VST','VXO']::bpchar[]"}] + expect: + budget: {buffers: 18702, planning_ms: 10} + note: "measured: 12468 / 12468 B (custom / generic); budget: 1.5x measured" + - id: s7.pa.swe19_14d + stmt: GetPathStatsProbe + shape: "pa probe swe19_14d" + params: [{raw: "'2026-10-04 17:00+00'::timestamptz"}, {raw: "'2026-10-18 17:00+00'::timestamptz"}, {raw: "ARRAY['BLE','GOT','GVX','HAD','JKG','KLR','KSD','LLA','LPI','MMX','NYO','ORB','OSD','RNB','SDL','STO','VBY','VST','VXO']::bpchar[]"}] + expect: + budget: {buffers: 18702, planning_ms: 10} + note: "measured: 12468 / 12468 B (custom / generic); budget: 1.5x measured" + - id: s7.pa.gbr22_1d + stmt: GetPathStatsProbe + shape: "pa probe gbr22_1d" + params: [{raw: "'2026-10-17 17:00+00'::timestamptz"}, {raw: "'2026-10-18 17:00+00'::timestamptz"}, {raw: "ARRAY['BHX','BLK','BOH','BRS','CAX','CEG','DSA','EMA','EXT','HUY','LCY','LON','LPL','MAN','MME','NQT','NQY','NWI','ORM','PLH','PME','VLY']::bpchar[]"}] + expect: + budget: {buffers: 3206, planning_ms: 10} + note: "measured: 2137 / 2137 B (custom / generic); budget: 1.5x measured" + - id: s7.pa.gbr22_7d + stmt: GetPathStatsProbe + shape: "pa probe gbr22_7d" + params: [{raw: "'2026-10-11 17:00+00'::timestamptz"}, {raw: "'2026-10-18 17:00+00'::timestamptz"}, {raw: "ARRAY['BHX','BLK','BOH','BRS','CAX','CEG','DSA','EMA','EXT','HUY','LCY','LON','LPL','MAN','MME','NQT','NQY','NWI','ORM','PLH','PME','VLY']::bpchar[]"}] + expect: + budget: {buffers: 22536, planning_ms: 10} + note: "measured: 15024 / 15024 B (custom / generic); budget: 1.5x measured" + - id: s7.pa.gbr22_14d + stmt: GetPathStatsProbe + shape: "pa probe gbr22_14d" + params: [{raw: "'2026-10-04 17:00+00'::timestamptz"}, {raw: "'2026-10-18 17:00+00'::timestamptz"}, {raw: "ARRAY['BHX','BLK','BOH','BRS','CAX','CEG','DSA','EMA','EXT','HUY','LCY','LON','LPL','MAN','MME','NQT','NQY','NWI','ORM','PLH','PME','VLY']::bpchar[]"}] + expect: + budget: {buffers: 22536, planning_ms: 10} + note: "measured: 15024 / 15024 B (custom / generic); budget: 1.5x measured" + - id: s7.pa.top50_7d + stmt: GetPathStatsProbe + shape: "pa probe top50_7d" + params: [{raw: "'2026-10-11 17:00+00'::timestamptz"}, {raw: "'2026-10-18 17:00+00'::timestamptz"}, {raw: "ARRAY['SEA','KRK','PDX','BNA','RDU','ATL','TYS','AVL','CHA','TRI','POZ','STR','GSP','PRG','PAD','KTW','FRA','RDO','BRQ','BRN','BWE','GSO','AMS','AHN','MOW','PSK','CGN','YVR','ORD','RST','LEJ','SPM','DTM','PAE','BER','SJC','LIN','DUS','LOZ','BOS','VOD','VIE','TRN','PED','WAW','WRO','BLI','HRA','NUE','IDA']::bpchar[]"}] + expect: + budget: {buffers: 60750, planning_ms: 10} + note: "measured: 40500 / 40500 B (custom / generic); budget: 1.5x measured" + - id: s7.tn.all625_7d_today + stmt: GetTopNodes + shape: "tn all625_7d: > 32 distinct IATAs -> today's statement" + params: [{raw: "'2026-10-11 17:00+00'::timestamptz"}, {raw: "ARRAY['AAB','AAH','AAL','AAR','ABC','ABE','ABI','ABQ','ABX','ABZ','ACH','ACY','ADL','AGS','AHN','AIZ','AKL','ALB','ALC','ALF','ALH','ALL','AMA','AMS','ANC','ANE','ANR','AOH','APF','ATH','ATL','ATW','AUH','AUS','AVL','AVP','AXN','AZO','BBH','BCN','BDL','BEG','BER','BES','BFD','BFL','BGM','BGN','BGR','BHD','BHM','BHS','BHX','BHZ','BJP','BLA','BLE','BLI','BLK','BLL','BNA','BNE','BOD','BOH','BOI','BOJ','BOS','BQN','BRE','BRN','BRQ','BRS','BRU','BTS','BUD','BUF','BUY','BVE','BWE','BWI','BWT','BYG','BYU','BZG','CAE','CAK','CAX','CBG','CBR','CDG','CEB','CEG','CEZ','CFR','CGI','CGN','CHA','CHC','CLE','CLT','CMH','CMI','COA','CON','COS','CPH','CPQ','CPT','CRL','CRW','CRX','CSM','CVG','CVO','CWA','CWB','CZW','DAB','DAL','DAY','DEC','DEN','DET','DFW','DIQ','DLE','DLH','DND','DNQ','DPO','DRO','DRS','DSA','DSM','DTM','DTW','DUB','DUD','DUS','DVN','DXB','EAT','EAU','EDI','EGE','EIN','ELM','ELP','EMA','EME','ENS','EPH','ERF','ESS','EUG','EVV','EWN','EXT','FAO','FAR','FAT','FKB','FLO','FLR','FMM','FNB','FNL','FNT','FPR','FRA','FRD','FWA','GAS','GBR','GCI','GDN','GDW','GEG','GEX','GGG','GHV','GJT','GLA','GLZ','GOA','GOJ','GOR','GOT','GPT','GPZ','GRQ','GRR','GRU','GRZ','GSO','GSP','GTI','GTP','GTR','GTW','GUF','GUL','GVX','HAD','HAJ','HAM','HDB','HEL','HLG','HLZ','HOH','HOU','HRA','HTS','HUY','HVN','HYR','IAD','IAS','IDA','IKT','ILM','ILZ','IND','INV','IQQ','ITA','IVC','JAN','JAX','JBQ','JCL','JER','JKG','JTC','JYV','KEL','KJK','KLR','KLS','KLU','KLV','KOI','KOK','KPS','KRK','KSC','KSD','KSF','KTN','KTW','KUF','KUN','KZG','KZN','LAA','LAS','LAX','LBA','LBB','LBC','LCH','LCJ','LCQ','LCY','LEH','LEJ','LEX','LEY','LFT','LGA','LGG','LGU','LIL','LIN','LIS','LIT','LJU','LKV','LLA','LMT','LNS','LNZ','LOI','LON','LOZ','LPI','LPL','LRF','LRH','LRT','LRU','LSE','LTN','LUG','LUX','LUZ','LWB','LWK','LWR','LYS','MAC','MAD','MAF','MAN','MBS','MCE','MCI','MCO','MDT','MEL','MEM','MFE','MFR','MGL','MGM','MHG','MIA','MIM','MKA','MKE','MLB','MLU','MME','MMH','MMV','MMX','MNN','MOT','MOW','MPL','MQL','MRO','MRY','MSN','MSP','MST','MSV','MSY','MTJ','MUC','MUE','MVH','MXP','NAP','NCE','NHZ','NIT','NOA','NOC','NOD','NPE','NPL','NQT','NQY','NSN','NTE','NTL','NTU','NUE','NUW','NWI','NYC','NYO','OAK','OBF','OBN','OCF','ODE','OGG','OKC','OLO','OMA','ONP','ONT','OOL','OPO','ORB','ORD','ORL','ORM','ORY','OSD','OSI','OSL','OSR','OTP','OUL','OXF','OXR','PAD','PAE','PAH','PAR','PDK','PDV','PDX','PED','PER','PFB','PGD','PGV','PHF','PHL','PHX','PIA','PIH','PIK','PIT','PKB','PLH','PLO','PLQ','PLR','PME','PMI','PMR','PNS','POA','POZ','PPM','PQQ','PRB','PRC','PRG','PRV','PSC','PSF','PSH','PSK','PSP','PUB','PUW','PVD','QAR','QDV','RAO','RBG','RDD','RDM','RDO','RDU','RDZ','RFD','RIC','RIX','RJK','RKD','RKS','RKV','RLG','RMO','RMU','RNB','RNN','RNO','ROA','ROC','ROK','ROT','ROV','RST','RSW','RTM','RUT','RYG','RZE','SAN','SAO','SAT','SBA','SBN','SBP','SBY','SCE','SCL','SCN','SDF','SDL','SDR','SEA','SFO','SGD','SGU','SIR','SJC','SJO','SJT','SJU','SJY','SKE','SLC','SLE','SMF','SMN','SNN','SOD','SOF','SPM','SQL','SQQ','SRQ','SSZ','STL','STN','STO','STR','STS','SVC','SVG','SVX','SWE','SWH','SXB','SYD','SYR','SZZ','TAT','TGD','TGN','TJM','TKU','TLH','TLL','TLS','TMP','TPA','TRD','TRF','TRG','TRI','TRN','TRO','TRS','TSR','TSV','TTD','TUF','TUL','TWB','TWF','TXK','TYR','TYS','UDE','UHE','UIP','UKK','UME','UTC','VAG','VAP','VBY','VCP','VIE','VIS','VJI','VLC','VLY','VNO','VOD','VOG','VPZ','VRZ','VST','VXO','WAW','WBG','WGA','WIC','WIE','WLG','WOL','WRE','WRO','WRT','WVN','WYA','XCM','XMS','XPH','XYZ','YBL','YCD','YEG','YGK','YHM','YHU','YHZ','YKA','YKF','YKM','YLK','YML','YOW','YPQ','YQA','YQB','YQF','YQL','YQQ','YQS','YQT','YQY','YRQ','YSE','YSJ','YTF','YTR','YUL','YVE','YVR','YWG','YWS','YXU','YXX','YYB','YYC','YYJ','YYY','YYZ','ZBE','ZBM','ZNO','ZPH']::bpchar[]"}, {raw: "10"}] + modes: [custom, auto] + mode: {auto: {known_issue: "legacy statement; generic plan over budget until the stats rewrite replaces it"}} + expect: + budget: {buffers: 43416, planning_ms: 10} + note: "measured: 28944 / 62065 B (custom / generic); budget: 1.5x measured" + - id: s7.ta.all625_7d_today + stmt: GetStatsTopAdvertisers + shape: "ta all625_7d: > 32 distinct IATAs -> today's statement" + params: [{raw: "'flood'"}, {raw: "'2026-10-11 17:00+00'::timestamptz"}, {raw: "ARRAY['AAB','AAH','AAL','AAR','ABC','ABE','ABI','ABQ','ABX','ABZ','ACH','ACY','ADL','AGS','AHN','AIZ','AKL','ALB','ALC','ALF','ALH','ALL','AMA','AMS','ANC','ANE','ANR','AOH','APF','ATH','ATL','ATW','AUH','AUS','AVL','AVP','AXN','AZO','BBH','BCN','BDL','BEG','BER','BES','BFD','BFL','BGM','BGN','BGR','BHD','BHM','BHS','BHX','BHZ','BJP','BLA','BLE','BLI','BLK','BLL','BNA','BNE','BOD','BOH','BOI','BOJ','BOS','BQN','BRE','BRN','BRQ','BRS','BRU','BTS','BUD','BUF','BUY','BVE','BWE','BWI','BWT','BYG','BYU','BZG','CAE','CAK','CAX','CBG','CBR','CDG','CEB','CEG','CEZ','CFR','CGI','CGN','CHA','CHC','CLE','CLT','CMH','CMI','COA','CON','COS','CPH','CPQ','CPT','CRL','CRW','CRX','CSM','CVG','CVO','CWA','CWB','CZW','DAB','DAL','DAY','DEC','DEN','DET','DFW','DIQ','DLE','DLH','DND','DNQ','DPO','DRO','DRS','DSA','DSM','DTM','DTW','DUB','DUD','DUS','DVN','DXB','EAT','EAU','EDI','EGE','EIN','ELM','ELP','EMA','EME','ENS','EPH','ERF','ESS','EUG','EVV','EWN','EXT','FAO','FAR','FAT','FKB','FLO','FLR','FMM','FNB','FNL','FNT','FPR','FRA','FRD','FWA','GAS','GBR','GCI','GDN','GDW','GEG','GEX','GGG','GHV','GJT','GLA','GLZ','GOA','GOJ','GOR','GOT','GPT','GPZ','GRQ','GRR','GRU','GRZ','GSO','GSP','GTI','GTP','GTR','GTW','GUF','GUL','GVX','HAD','HAJ','HAM','HDB','HEL','HLG','HLZ','HOH','HOU','HRA','HTS','HUY','HVN','HYR','IAD','IAS','IDA','IKT','ILM','ILZ','IND','INV','IQQ','ITA','IVC','JAN','JAX','JBQ','JCL','JER','JKG','JTC','JYV','KEL','KJK','KLR','KLS','KLU','KLV','KOI','KOK','KPS','KRK','KSC','KSD','KSF','KTN','KTW','KUF','KUN','KZG','KZN','LAA','LAS','LAX','LBA','LBB','LBC','LCH','LCJ','LCQ','LCY','LEH','LEJ','LEX','LEY','LFT','LGA','LGG','LGU','LIL','LIN','LIS','LIT','LJU','LKV','LLA','LMT','LNS','LNZ','LOI','LON','LOZ','LPI','LPL','LRF','LRH','LRT','LRU','LSE','LTN','LUG','LUX','LUZ','LWB','LWK','LWR','LYS','MAC','MAD','MAF','MAN','MBS','MCE','MCI','MCO','MDT','MEL','MEM','MFE','MFR','MGL','MGM','MHG','MIA','MIM','MKA','MKE','MLB','MLU','MME','MMH','MMV','MMX','MNN','MOT','MOW','MPL','MQL','MRO','MRY','MSN','MSP','MST','MSV','MSY','MTJ','MUC','MUE','MVH','MXP','NAP','NCE','NHZ','NIT','NOA','NOC','NOD','NPE','NPL','NQT','NQY','NSN','NTE','NTL','NTU','NUE','NUW','NWI','NYC','NYO','OAK','OBF','OBN','OCF','ODE','OGG','OKC','OLO','OMA','ONP','ONT','OOL','OPO','ORB','ORD','ORL','ORM','ORY','OSD','OSI','OSL','OSR','OTP','OUL','OXF','OXR','PAD','PAE','PAH','PAR','PDK','PDV','PDX','PED','PER','PFB','PGD','PGV','PHF','PHL','PHX','PIA','PIH','PIK','PIT','PKB','PLH','PLO','PLQ','PLR','PME','PMI','PMR','PNS','POA','POZ','PPM','PQQ','PRB','PRC','PRG','PRV','PSC','PSF','PSH','PSK','PSP','PUB','PUW','PVD','QAR','QDV','RAO','RBG','RDD','RDM','RDO','RDU','RDZ','RFD','RIC','RIX','RJK','RKD','RKS','RKV','RLG','RMO','RMU','RNB','RNN','RNO','ROA','ROC','ROK','ROT','ROV','RST','RSW','RTM','RUT','RYG','RZE','SAN','SAO','SAT','SBA','SBN','SBP','SBY','SCE','SCL','SCN','SDF','SDL','SDR','SEA','SFO','SGD','SGU','SIR','SJC','SJO','SJT','SJU','SJY','SKE','SLC','SLE','SMF','SMN','SNN','SOD','SOF','SPM','SQL','SQQ','SRQ','SSZ','STL','STN','STO','STR','STS','SVC','SVG','SVX','SWE','SWH','SXB','SYD','SYR','SZZ','TAT','TGD','TGN','TJM','TKU','TLH','TLL','TLS','TMP','TPA','TRD','TRF','TRG','TRI','TRN','TRO','TRS','TSR','TSV','TTD','TUF','TUL','TWB','TWF','TXK','TYR','TYS','UDE','UHE','UIP','UKK','UME','UTC','VAG','VAP','VBY','VCP','VIE','VIS','VJI','VLC','VLY','VNO','VOD','VOG','VPZ','VRZ','VST','VXO','WAW','WBG','WGA','WIC','WIE','WLG','WOL','WRE','WRO','WRT','WVN','WYA','XCM','XMS','XPH','XYZ','YBL','YCD','YEG','YGK','YHM','YHU','YHZ','YKA','YKF','YKM','YLK','YML','YOW','YPQ','YQA','YQB','YQF','YQL','YQQ','YQS','YQT','YQY','YRQ','YSE','YSJ','YTF','YTR','YUL','YVE','YVR','YWG','YWS','YXU','YXX','YYB','YYC','YYJ','YYY','YYZ','ZBE','ZBM','ZNO','ZPH']::bpchar[]"}, {raw: "10"}] + modes: [custom, auto] + mode: {auto: {known_issue: "legacy statement; generic plan over budget until the stats rewrite replaces it"}} + expect: + budget: {buffers: 67419, planning_ms: 10} + note: "measured: 44946 / 103054 B (custom / generic); budget: 1.5x measured" + - id: s7.pa.all625_7d_today + stmt: GetPathStats + shape: "pa all625_7d: > 32 distinct IATAs -> today's statement" + params: [{raw: "'2026-10-11 17:00+00'::timestamptz"}, {raw: "'2026-10-18 17:00+00'::timestamptz"}, {raw: "ARRAY['AAB','AAH','AAL','AAR','ABC','ABE','ABI','ABQ','ABX','ABZ','ACH','ACY','ADL','AGS','AHN','AIZ','AKL','ALB','ALC','ALF','ALH','ALL','AMA','AMS','ANC','ANE','ANR','AOH','APF','ATH','ATL','ATW','AUH','AUS','AVL','AVP','AXN','AZO','BBH','BCN','BDL','BEG','BER','BES','BFD','BFL','BGM','BGN','BGR','BHD','BHM','BHS','BHX','BHZ','BJP','BLA','BLE','BLI','BLK','BLL','BNA','BNE','BOD','BOH','BOI','BOJ','BOS','BQN','BRE','BRN','BRQ','BRS','BRU','BTS','BUD','BUF','BUY','BVE','BWE','BWI','BWT','BYG','BYU','BZG','CAE','CAK','CAX','CBG','CBR','CDG','CEB','CEG','CEZ','CFR','CGI','CGN','CHA','CHC','CLE','CLT','CMH','CMI','COA','CON','COS','CPH','CPQ','CPT','CRL','CRW','CRX','CSM','CVG','CVO','CWA','CWB','CZW','DAB','DAL','DAY','DEC','DEN','DET','DFW','DIQ','DLE','DLH','DND','DNQ','DPO','DRO','DRS','DSA','DSM','DTM','DTW','DUB','DUD','DUS','DVN','DXB','EAT','EAU','EDI','EGE','EIN','ELM','ELP','EMA','EME','ENS','EPH','ERF','ESS','EUG','EVV','EWN','EXT','FAO','FAR','FAT','FKB','FLO','FLR','FMM','FNB','FNL','FNT','FPR','FRA','FRD','FWA','GAS','GBR','GCI','GDN','GDW','GEG','GEX','GGG','GHV','GJT','GLA','GLZ','GOA','GOJ','GOR','GOT','GPT','GPZ','GRQ','GRR','GRU','GRZ','GSO','GSP','GTI','GTP','GTR','GTW','GUF','GUL','GVX','HAD','HAJ','HAM','HDB','HEL','HLG','HLZ','HOH','HOU','HRA','HTS','HUY','HVN','HYR','IAD','IAS','IDA','IKT','ILM','ILZ','IND','INV','IQQ','ITA','IVC','JAN','JAX','JBQ','JCL','JER','JKG','JTC','JYV','KEL','KJK','KLR','KLS','KLU','KLV','KOI','KOK','KPS','KRK','KSC','KSD','KSF','KTN','KTW','KUF','KUN','KZG','KZN','LAA','LAS','LAX','LBA','LBB','LBC','LCH','LCJ','LCQ','LCY','LEH','LEJ','LEX','LEY','LFT','LGA','LGG','LGU','LIL','LIN','LIS','LIT','LJU','LKV','LLA','LMT','LNS','LNZ','LOI','LON','LOZ','LPI','LPL','LRF','LRH','LRT','LRU','LSE','LTN','LUG','LUX','LUZ','LWB','LWK','LWR','LYS','MAC','MAD','MAF','MAN','MBS','MCE','MCI','MCO','MDT','MEL','MEM','MFE','MFR','MGL','MGM','MHG','MIA','MIM','MKA','MKE','MLB','MLU','MME','MMH','MMV','MMX','MNN','MOT','MOW','MPL','MQL','MRO','MRY','MSN','MSP','MST','MSV','MSY','MTJ','MUC','MUE','MVH','MXP','NAP','NCE','NHZ','NIT','NOA','NOC','NOD','NPE','NPL','NQT','NQY','NSN','NTE','NTL','NTU','NUE','NUW','NWI','NYC','NYO','OAK','OBF','OBN','OCF','ODE','OGG','OKC','OLO','OMA','ONP','ONT','OOL','OPO','ORB','ORD','ORL','ORM','ORY','OSD','OSI','OSL','OSR','OTP','OUL','OXF','OXR','PAD','PAE','PAH','PAR','PDK','PDV','PDX','PED','PER','PFB','PGD','PGV','PHF','PHL','PHX','PIA','PIH','PIK','PIT','PKB','PLH','PLO','PLQ','PLR','PME','PMI','PMR','PNS','POA','POZ','PPM','PQQ','PRB','PRC','PRG','PRV','PSC','PSF','PSH','PSK','PSP','PUB','PUW','PVD','QAR','QDV','RAO','RBG','RDD','RDM','RDO','RDU','RDZ','RFD','RIC','RIX','RJK','RKD','RKS','RKV','RLG','RMO','RMU','RNB','RNN','RNO','ROA','ROC','ROK','ROT','ROV','RST','RSW','RTM','RUT','RYG','RZE','SAN','SAO','SAT','SBA','SBN','SBP','SBY','SCE','SCL','SCN','SDF','SDL','SDR','SEA','SFO','SGD','SGU','SIR','SJC','SJO','SJT','SJU','SJY','SKE','SLC','SLE','SMF','SMN','SNN','SOD','SOF','SPM','SQL','SQQ','SRQ','SSZ','STL','STN','STO','STR','STS','SVC','SVG','SVX','SWE','SWH','SXB','SYD','SYR','SZZ','TAT','TGD','TGN','TJM','TKU','TLH','TLL','TLS','TMP','TPA','TRD','TRF','TRG','TRI','TRN','TRO','TRS','TSR','TSV','TTD','TUF','TUL','TWB','TWF','TXK','TYR','TYS','UDE','UHE','UIP','UKK','UME','UTC','VAG','VAP','VBY','VCP','VIE','VIS','VJI','VLC','VLY','VNO','VOD','VOG','VPZ','VRZ','VST','VXO','WAW','WBG','WGA','WIC','WIE','WLG','WOL','WRE','WRO','WRT','WVN','WYA','XCM','XMS','XPH','XYZ','YBL','YCD','YEG','YGK','YHM','YHU','YHZ','YKA','YKF','YKM','YLK','YML','YOW','YPQ','YQA','YQB','YQF','YQL','YQQ','YQS','YQT','YQY','YRQ','YSE','YSJ','YTF','YTR','YUL','YVE','YVR','YWG','YWS','YXU','YXX','YYB','YYC','YYJ','YYY','YYZ','ZBE','ZBM','ZNO','ZPH']::bpchar[]"}] + modes: [custom, auto] + mode: {auto: {known_issue: "legacy statement; generic plan over budget until the stats rewrite replaces it"}} + expect: + budget: {buffers: 32652, planning_ms: 10} + note: "measured: 21768 / - B (custom / generic); budget: 1.5x measured" + - id: s7.se.LIS_1d + stmt: GetStatsSeries_end + shape: "series se2 LIS_1d" + params: [{raw: "'2026-10-17 17:00+00'::timestamptz"}, {raw: "'2026-10-18 17:00+00'::timestamptz"}, {raw: "ARRAY['LIS']::bpchar[]"}] + expect: + budget: {buffers: 1520, planning_ms: 10} + note: "measured: 1013 / 1923 B (custom / generic); budget: 1.5x measured" + - id: s7.se.LIS_7d + stmt: GetStatsSeries_end + shape: "series se2 LIS_7d" + params: [{raw: "'2026-10-11 17:00+00'::timestamptz"}, {raw: "'2026-10-18 17:00+00'::timestamptz"}, {raw: "ARRAY['LIS']::bpchar[]"}] + expect: + budget: {buffers: 4578, planning_ms: 10} + note: "measured: 3052 / 14463 B (custom / generic); budget: 1.5x measured" + - id: s7.se.LIS_14d + stmt: GetStatsSeries_end + shape: "series se2 LIS_14d" + params: [{raw: "'2026-10-04 17:00+00'::timestamptz"}, {raw: "'2026-10-18 17:00+00'::timestamptz"}, {raw: "ARRAY['LIS']::bpchar[]"}] + expect: + budget: {buffers: 7602, planning_ms: 10} + note: "measured: 5068 / 16479 B (custom / generic); budget: 1.5x measured" + - id: s7.se.pt3_1d + stmt: GetStatsSeries_end + shape: "series se2 pt3_1d" + params: [{raw: "'2026-10-17 17:00+00'::timestamptz"}, {raw: "'2026-10-18 17:00+00'::timestamptz"}, {raw: "ARRAY['FAO','LIS','OPO']::bpchar[]"}] + expect: + budget: {buffers: 2630, planning_ms: 10} + note: "measured: 1753 / 2543 B (custom / generic); budget: 1.5x measured" + - id: s7.se.pt3_7d + stmt: GetStatsSeries_end + shape: "series se2 pt3_7d" + params: [{raw: "'2026-10-11 17:00+00'::timestamptz"}, {raw: "'2026-10-18 17:00+00'::timestamptz"}, {raw: "ARRAY['FAO','LIS','OPO']::bpchar[]"}] + expect: + budget: {buffers: 11180, planning_ms: 10} + note: "measured: 7453 / 18864 B (custom / generic); budget: 1.5x measured" + - id: s7.se.pt3_14d + stmt: GetStatsSeries_end + shape: "series se2 pt3_14d" + params: [{raw: "'2026-10-04 17:00+00'::timestamptz"}, {raw: "'2026-10-18 17:00+00'::timestamptz"}, {raw: "ARRAY['FAO','LIS','OPO']::bpchar[]"}] + expect: + budget: {buffers: 20252, planning_ms: 10} + note: "measured: 13501 / 24912 B (custom / generic); budget: 1.5x measured" + - id: s7.se.co10_1d + stmt: GetStatsSeries_end + shape: "series se2 co10_1d" + params: [{raw: "'2026-10-17 17:00+00'::timestamptz"}, {raw: "'2026-10-18 17:00+00'::timestamptz"}, {raw: "ARRAY['CEZ','COS','DEN','DRO','EGE','FNL','GJT','LAA','MTJ','PUB']::bpchar[]"}] + expect: + budget: {buffers: 7008, planning_ms: 10} + note: "measured: 4672 / 5005 B (custom / generic); budget: 1.5x measured" + - id: s7.se.co10_7d + stmt: GetStatsSeries_end + shape: "series se2 co10_7d" + params: [{raw: "'2026-10-11 17:00+00'::timestamptz"}, {raw: "'2026-10-18 17:00+00'::timestamptz"}, {raw: "ARRAY['CEZ','COS','DEN','DRO','EGE','FNL','GJT','LAA','MTJ','PUB']::bpchar[]"}] + expect: + budget: {buffers: 36020, planning_ms: 10} + note: "measured: 24013 / 35424 B (custom / generic); budget: 1.5x measured" + - id: s7.se.co10_14d + stmt: GetStatsSeries_end + shape: "series se2 co10_14d" + params: [{raw: "'2026-10-04 17:00+00'::timestamptz"}, {raw: "'2026-10-18 17:00+00'::timestamptz"}, {raw: "ARRAY['CEZ','COS','DEN','DRO','EGE','FNL','GJT','LAA','MTJ','PUB']::bpchar[]"}] + expect: + budget: {buffers: 66260, planning_ms: 10} + note: "measured: 44173 / 55584 B (custom / generic); budget: 1.5x measured" + - id: s7.se.gbr22_1d + stmt: GetStatsSeries_end + shape: "series se2 gbr22_1d" + params: [{raw: "'2026-10-17 17:00+00'::timestamptz"}, {raw: "'2026-10-18 17:00+00'::timestamptz"}, {raw: "ARRAY['BHX','BLK','BOH','BRS','CAX','CEG','DSA','EMA','EXT','HUY','LCY','LON','LPL','MAN','MME','NQT','NQY','NWI','ORM','PLH','PME','VLY']::bpchar[]"}] + expect: + budget: {buffers: 109523, planning_ms: 10} + note: "measured: 73015 / 9472 B (custom / generic); budget: 1.5x measured" + - id: s7.se.gbr22_7d + stmt: GetStatsSeries_end + shape: "series se2 gbr22_7d" + params: [{raw: "'2026-10-11 17:00+00'::timestamptz"}, {raw: "'2026-10-18 17:00+00'::timestamptz"}, {raw: "ARRAY['BHX','BLK','BOH','BRS','CAX','CEG','DSA','EMA','EXT','HUY','LCY','LON','LPL','MAN','MME','NQT','NQY','NWI','ORM','PLH','PME','VLY']::bpchar[]"}] + expect: + budget: {buffers: 85506, planning_ms: 10} + note: "measured: 57004 / 68415 B (custom / generic); budget: 1.5x measured" + - id: s7.se.gbr22_14d + stmt: GetStatsSeries_end + shape: "series se2 gbr22_14d" + params: [{raw: "'2026-10-04 17:00+00'::timestamptz"}, {raw: "'2026-10-18 17:00+00'::timestamptz"}, {raw: "ARRAY['BHX','BLK','BOH','BRS','CAX','CEG','DSA','EMA','EXT','HUY','LCY','LON','LPL','MAN','MME','NQT','NQY','NWI','ORM','PLH','PME','VLY']::bpchar[]"}] + expect: + budget: {buffers: 152034, planning_ms: 10} + note: "measured: 101356 / 112767 B (custom / generic); budget: 1.5x measured" + - id: s7.se.SEA_1d + stmt: GetStatsSeries_end + shape: "series se2 SEA_1d" + params: [{raw: "'2026-10-17 17:00+00'::timestamptz"}, {raw: "'2026-10-18 17:00+00'::timestamptz"}, {raw: "ARRAY['SEA']::bpchar[]"}] + expect: + budget: {buffers: 4595, planning_ms: 10} + note: "measured: 3063 / 2993 B (custom / generic); budget: 1.5x measured" + - id: s7.se.SEA_7d + stmt: GetStatsSeries_end + shape: "series se2 SEA_7d" + params: [{raw: "'2026-10-11 17:00+00'::timestamptz"}, {raw: "'2026-10-18 17:00+00'::timestamptz"}, {raw: "ARRAY['SEA']::bpchar[]"}] + expect: + budget: {buffers: 16221, planning_ms: 10} + note: "measured: 10814 / 22225 B (custom / generic); budget: 1.5x measured" + - id: s7.se.SEA_14d + stmt: GetStatsSeries_end + shape: "series se2 SEA_14d" + params: [{raw: "'2026-10-04 17:00+00'::timestamptz"}, {raw: "'2026-10-18 17:00+00'::timestamptz"}, {raw: "ARRAY['SEA']::bpchar[]"}] + expect: + budget: {buffers: 19245, planning_ms: 10} + note: "measured: 12830 / 24241 B (custom / generic); budget: 1.5x measured" + - id: s7.se.all_1d + stmt: GetStatsSeries_end + shape: "series se2 all_1d" + params: [{raw: "'2026-10-17 17:00+00'::timestamptz"}, {raw: "'2026-10-18 17:00+00'::timestamptz"}, {raw: "NULL::bpchar[]"}] + expect: + budget: {buffers: 130127, planning_ms: 10} + note: "measured: 86751 / 86751 B (custom / generic); budget: 1.5x measured" + - id: s7.se.all_7d + stmt: GetStatsSeries_end + shape: "series se2 all_7d" + params: [{raw: "'2026-10-11 17:00+00'::timestamptz"}, {raw: "'2026-10-18 17:00+00'::timestamptz"}, {raw: "NULL::bpchar[]"}] + expect: + budget: {buffers: 960902, planning_ms: 10} + note: "measured: 640601 / 640601 B (custom / generic); budget: 1.5x measured" + - id: s7.trc.first_l200 + stmt: ListTraceTags_end + shape: "traces keyset first_l200" + params: [{raw: "NULL::bpchar[]"}, {raw: "''"}, {raw: "NULL::timestamptz"}, {raw: "NULL::timestamptz"}, {raw: "NULL::timestamptz"}, {raw: "200"}, {raw: "''"}, {raw: "NULL::bytea"}] + expect: + driver: [{relation: trace_tags, alias: t, index: idx_trace_tags_keyset}] + budget: {buffers: 876, planning_ms: 10} + note: "measured: 584 / 584 B (custom / generic); budget: 1.5x measured" + - id: s7.trc.cursor_l200 + stmt: ListTraceTags_end + shape: "traces keyset cursor_l200" + params: [{raw: "NULL::bpchar[]"}, {raw: "''"}, {raw: "NULL::timestamptz"}, {raw: "NULL::timestamptz"}, {raw: "'2026-10-17 06:17:50.081+00'::timestamptz"}, {raw: "200"}, {raw: "''"}, {raw: "NULL::bytea"}] + expect: + driver: [{relation: trace_tags, alias: t, index: idx_trace_tags_keyset}] + budget: {buffers: 873, planning_ms: 10} + note: "measured: 582 / 582 B (custom / generic); budget: 1.5x measured" + - id: s7.trc.tie_tag_l200 + stmt: ListTraceTags_end + shape: "traces keyset, cursor inside a tied millisecond, limit 200" + params: [{raw: "NULL::bpchar[]"}, {raw: "''"}, {raw: "NULL::timestamptz"}, {raw: "NULL::timestamptz"}, {r: trc_tie_ts}, {raw: "200"}, {raw: "''"}, {r: trc_tie_tag}] + expect: + driver: [{relation: trace_tags, alias: t, index: idx_trace_tags_keyset}] + budget: {buffers: 885, planning_ms: 10} + note: "measured: 590 / 590 B (custom / generic); budget: 1.5x measured" + - id: s7.trc.AUS_cursor_l200 + stmt: ListTraceTags_end + shape: "traces keyset AUS_cursor_l200" + params: [{raw: "ARRAY['AUS']::bpchar[]"}, {raw: "''"}, {raw: "NULL::timestamptz"}, {raw: "NULL::timestamptz"}, {raw: "'2026-10-17 06:17:50.081+00'::timestamptz"}, {raw: "200"}, {raw: "''"}, {raw: "NULL::bytea"}] + expect: + driver: [{relation: trace_tags, alias: t, index: idx_trace_tags_keyset}] + budget: {buffers: 2586, planning_ms: 10} + note: "measured: 1724 / 3249 B (custom / generic); budget: 1.5x measured" + - id: s7.trc.cursor_l1 + stmt: ListTraceTags_end + shape: "traces keyset cursor_l1" + params: [{raw: "NULL::bpchar[]"}, {raw: "''"}, {raw: "NULL::timestamptz"}, {raw: "NULL::timestamptz"}, {raw: "'2026-10-17 06:17:50.081+00'::timestamptz"}, {raw: "1"}, {raw: "''"}, {raw: "NULL::bytea"}] + expect: + driver: [{relation: trace_tags, alias: t, index: idx_trace_tags_keyset}] + budget: {buffers: 50, planning_ms: 10} + note: "measured: 7 / 7 B (custom / generic); budget: 1.5x measured" + - id: s7.tel.top_168h_a0 + stmt: GetObserverTelemetrySince + shape: "telemetry since-form, the observer with the most telemetry, 7 d, page 1" + params: [{r: tel_top}, {raw: "'2026-10-11 17:00+00'::timestamptz"}, {raw: "0"}] + expect: + driver: [{relation: observer_telemetry, index: idx_telemetry_observer_recent, nodes: ["Index Scan", "Index Only Scan", "Bitmap Heap Scan"]}] + budget: {buffers: 600, planning_ms: 10} + note: "measured: - / - B (custom / generic); budget: runtime-guards 2.6 (29-394 B) -> 600" + - id: s7.tel.top_168h_a5921868 + stmt: GetObserverTelemetrySince + shape: "telemetry since-form, the observer with the most telemetry, 7 d, cursor at its median row" + params: [{r: tel_top}, {raw: "'2026-10-11 17:00+00'::timestamptz"}, {r: tel_top_mid}] + expect: + driver: [{relation: observer_telemetry, index: idx_telemetry_observer_recent, nodes: ["Index Scan", "Index Only Scan", "Bitmap Heap Scan"]}] + budget: {buffers: 600, planning_ms: 10} + note: "measured: - / - B (custom / generic); budget: runtime-guards 2.6 (29-394 B) -> 600" + - id: s7.adv.busy1_c0_d16 + stmt: ListObserverAdverts_end + shape: "adverts, the observer with the most adverts, page 1" + params: [{r: adv_busiest}, {raw: "0"}, {raw: "51"}] + expect: + driver: [{relation: packet_observations, alias: po, index: idx_observations_adverts, index_cond: ["observer_id =", "id >"]}] + budget: {buffers: 520, planning_ms: 10} + note: "measured: 462 / 462 B (custom / generic); budget: gate <= 10 B/row + 10" + - id: s7.adv.busy1_median + stmt: ListObserverAdverts_end + shape: "adverts, the observer with the most adverts, cursor at its median advert" + params: [{r: adv_busiest}, {r: adv_busiest_mid}, {raw: "51"}] + expect: + driver: [{relation: packet_observations, alias: po, index: idx_observations_adverts, index_cond: ["observer_id =", "id >"]}] + budget: {buffers: 520, planning_ms: 10} + note: "measured: 463 / 463 B (custom / generic); budget: gate <= 10 B/row + 10" + - id: s7.adv.busy1_median_l201 + stmt: ListObserverAdverts_end + shape: "adverts, the observer with the most adverts, cursor at its median advert, limit 201" + params: [{r: adv_busiest}, {r: adv_busiest_mid}, {raw: "201"}] + expect: + driver: [{relation: packet_observations, alias: po, index: idx_observations_adverts, index_cond: ["observer_id =", "id >"]}] + budget: {buffers: 2720, planning_ms: 10} + note: "measured: 1813 / 1813 B (custom / generic); budget: gate <= 10 B/row + 10; 1.5x measured for limit 201" + - id: s7.adv.topobs_c0_d6 + stmt: ListObserverAdverts_end + shape: "adverts, the observer with the most observations, page 1" + params: [{r: adv_topobs}, {raw: "0"}, {raw: "51"}] + expect: + driver: [{relation: packet_observations, alias: po, index: idx_observations_adverts, index_cond: ["observer_id =", "id >"]}] + budget: {buffers: 520, planning_ms: 10} + note: "measured: 453 / 453 B (custom / generic); budget: gate <= 10 B/row + 10" + - id: s7.adv.topobs_median + stmt: ListObserverAdverts_end + shape: "adverts, the observer with the most observations, cursor at its median advert" + params: [{r: adv_topobs}, {r: adv_topobs_mid}, {raw: "51"}] + expect: + driver: [{relation: packet_observations, alias: po, index: idx_observations_adverts, index_cond: ["observer_id =", "id >"]}] + budget: {buffers: 520, planning_ms: 10} + note: "measured: 462 / 462 B (custom / generic); budget: gate <= 10 B/row + 10" + - id: s7.adv.d51_c0 + stmt: ListObserverAdverts_end + shape: "adverts, the observer with the highest advert share (>= 500 observations), page 1" + params: [{r: adv_dense1}, {raw: "0"}, {raw: "51"}] + expect: + driver: [{relation: packet_observations, alias: po, index: idx_observations_adverts, index_cond: ["observer_id =", "id >"]}] + budget: {buffers: 520, planning_ms: 10} + note: "measured: 463 / 463 B (custom / generic); budget: gate <= 10 B/row + 10" + - id: s7.adv.d48_c0 + stmt: ListObserverAdverts_end + shape: "adverts, the observer with the second highest advert share, page 1" + params: [{r: adv_dense2}, {raw: "0"}, {raw: "51"}] + expect: + driver: [{relation: packet_observations, alias: po, index: idx_observations_adverts, index_cond: ["observer_id =", "id >"]}] + budget: {buffers: 520, planning_ms: 10} + note: "measured: 462 / 462 B (custom / generic); budget: gate <= 10 B/row + 10" + - id: s7.adv.d3_c0 + stmt: ListObserverAdverts_end + shape: "adverts, the observer with the lowest non-zero advert share, page 1" + params: [{r: adv_sparse}, {raw: "0"}, {raw: "51"}] + expect: + driver: [{relation: packet_observations, alias: po, index: idx_observations_adverts, index_cond: ["observer_id =", "id >"]}] + budget: {buffers: 520, planning_ms: 10} + note: "measured: 462 / 462 B (custom / generic); budget: gate <= 10 B/row + 10" + - id: s7.adv.mid1_c0 + stmt: ListObserverAdverts_end + shape: "adverts, the observer with the median advert count, page 1" + params: [{r: adv_mid}, {raw: "0"}, {raw: "51"}] + expect: + driver: [{relation: packet_observations, alias: po, index: idx_observations_adverts, index_cond: ["observer_id =", "id >"]}] + budget: {buffers: 520, planning_ms: 10} + note: "measured: 454 / 454 B (custom / generic); budget: gate <= 10 B/row + 10" + - id: s7.adv.quiet1_c0 + stmt: ListObserverAdverts_end + shape: "adverts, the observer with the fewest adverts, page 1" + params: [{r: adv_quiet}, {raw: "0"}, {raw: "51"}] + expect: + driver: [{relation: packet_observations, alias: po, index: idx_observations_adverts, index_cond: ["observer_id =", "id >"]}] + budget: {buffers: 520, planning_ms: 10} + note: "measured: 12 / 12 B (custom / generic); budget: gate <= 10 B/row + 10" + - id: s7.adv.zeroadv_big + stmt: ListObserverAdverts_end + shape: "adverts, the busiest observer with no adverts" + params: [{r: adv_zero}, {raw: "0"}, {raw: "51"}] + expect: + driver: [{relation: packet_observations, alias: po, index: idx_observations_adverts, index_cond: ["observer_id =", "id >"]}] + budget: {buffers: 520, planning_ms: 10} + note: "measured: 3 / 3 B (custom / generic); budget: gate <= 10 B/row + 10" + - id: s7.adv.unknown + stmt: ListObserverAdverts_end + shape: "adverts, an unknown observer id" + params: [{raw: "'00000000-0000-0000-0000-00000000000b'::uuid"}, {raw: "0"}, {raw: "51"}] + expect: + driver: [{relation: packet_observations, alias: po, index: idx_observations_adverts, index_cond: ["observer_id =", "id >"]}] + budget: {buffers: 520, planning_ms: 10} + note: "measured: 3 / 3 B (custom / generic); budget: gate <= 10 B/row + 10" + - id: s7.adv.busy1_cursor_3e9 + stmt: ListObserverAdverts_end + shape: "adverts, the observer with the most adverts, cursor 3e9 (>= 2^31): empty page, no error" + params: [{r: adv_busiest}, {raw: "3000000000"}, {raw: "51"}] + expect: + driver: [{relation: packet_observations, alias: po, index_cond: ["id >"], any_index: [idx_observations_adverts, packet_observations_pkey]}] + budget: {buffers: 520, planning_ms: 10} + note: "measured: - / - B (custom / generic); budget: gate <= 10 B/row + 10" + + # ---- small tables: nodes, observers, channels, node types, clock drift ---- + - id: sm.nodes.global + stmt: ListNodes_end + shape: "ListNodes global" + params: [{raw: "0"}, {raw: "NULL::bpchar[]"}, {raw: "'any'"}, {raw: "'any'"}, {raw: "NULL::bytea"}, {raw: "''"}, {raw: "'infinity'::timestamptz"}, {raw: "50"}, {raw: "''"}, {raw: "false"}, {raw: "''"}, {raw: "'00000000-0000-0000-0000-000000000000'::uuid"}] + expect: + budget: {buffers: 3030, planning_ms: 10} + note: "measured: 2020 / 40650 B (custom / generic); budget: 1.5x measured" + - id: sm.nodes.type2 + stmt: ListNodes_end + shape: "ListNodes type2" + params: [{raw: "2"}, {raw: "NULL::bpchar[]"}, {raw: "'any'"}, {raw: "'any'"}, {raw: "NULL::bytea"}, {raw: "''"}, {raw: "'infinity'::timestamptz"}, {raw: "50"}, {raw: "''"}, {raw: "false"}, {raw: "''"}, {raw: "'00000000-0000-0000-0000-000000000000'::uuid"}] + expect: + budget: {buffers: 3132, planning_ms: 10} + note: "measured: 2088 / 29731 B (custom / generic); budget: 1.5x measured" + - id: sm.nodes.type1 + stmt: ListNodes_end + shape: "ListNodes type1" + params: [{raw: "1"}, {raw: "NULL::bpchar[]"}, {raw: "'any'"}, {raw: "'any'"}, {raw: "NULL::bytea"}, {raw: "''"}, {raw: "'infinity'::timestamptz"}, {raw: "50"}, {raw: "''"}, {raw: "false"}, {raw: "''"}, {raw: "'00000000-0000-0000-0000-000000000000'::uuid"}] + expect: + budget: {buffers: 2874, planning_ms: 10} + note: "measured: 1916 / 11712 B (custom / generic); budget: 1.5x measured" + - id: sm.nodes.type3 + stmt: ListNodes_end + shape: "ListNodes type3" + params: [{raw: "3"}, {raw: "NULL::bpchar[]"}, {raw: "'any'"}, {raw: "'any'"}, {raw: "NULL::bytea"}, {raw: "''"}, {raw: "'infinity'::timestamptz"}, {raw: "50"}, {raw: "''"}, {raw: "false"}, {raw: "''"}, {raw: "'00000000-0000-0000-0000-000000000000'::uuid"}] + expect: + budget: {buffers: 3446, planning_ms: 10} + note: "measured: 2297 / 3676 B (custom / generic); budget: 1.5x measured" + - id: sm.nodes.type2_deep_cursor + stmt: ListNodes_end + shape: "ListNodes type2_deep_cursor" + params: [{raw: "2"}, {raw: "NULL::bpchar[]"}, {raw: "'any'"}, {raw: "'any'"}, {raw: "NULL::bytea"}, {raw: "''"}, {raw: "'2026-10-10 00:00+00'::timestamptz"}, {raw: "50"}, {raw: "''"}, {raw: "false"}, {raw: "''"}, {raw: "'00000000-0000-0000-0000-000000000000'::uuid"}] + expect: + budget: {buffers: 3209, planning_ms: 10} + note: "measured: 2139 / 3734 B (custom / generic); budget: 1.5x measured" + - id: sm.nodes.global_deep_cursor200 + stmt: ListNodes_end + shape: "ListNodes global_deep_cursor200" + params: [{raw: "0"}, {raw: "NULL::bpchar[]"}, {raw: "'any'"}, {raw: "'any'"}, {raw: "NULL::bytea"}, {raw: "''"}, {raw: "'2026-10-10 00:00+00'::timestamptz"}, {raw: "200"}, {raw: "''"}, {raw: "false"}, {raw: "''"}, {raw: "'00000000-0000-0000-0000-000000000000'::uuid"}] + expect: + budget: {buffers: 5829, planning_ms: 10} + note: "measured: 3886 / 8415 B (custom / generic); budget: 1.5x measured" + - id: sm.nodes.SEA + stmt: ListNodes_end + shape: "ListNodes SEA" + params: [{raw: "0"}, {raw: "'{SEA}'::bpchar[]"}, {raw: "'any'"}, {raw: "'any'"}, {raw: "NULL::bytea"}, {raw: "''"}, {raw: "'infinity'::timestamptz"}, {raw: "50"}, {raw: "''"}, {raw: "false"}, {raw: "''"}, {raw: "'00000000-0000-0000-0000-000000000000'::uuid"}] + expect: + budget: {buffers: 8009, planning_ms: 10} + note: "measured: 5339 / 9253 B (custom / generic); budget: 1.5x measured" + - id: sm.nodes.SEA_type2 + stmt: ListNodes_end + shape: "ListNodes SEA_type2" + params: [{raw: "2"}, {raw: "'{SEA}'::bpchar[]"}, {raw: "'any'"}, {raw: "'any'"}, {raw: "NULL::bytea"}, {raw: "''"}, {raw: "'infinity'::timestamptz"}, {raw: "50"}, {raw: "''"}, {raw: "false"}, {raw: "''"}, {raw: "'00000000-0000-0000-0000-000000000000'::uuid"}] + expect: + budget: {buffers: 7974, planning_ms: 10} + note: "measured: 5316 / 9017 B (custom / generic); budget: 1.5x measured" + - id: sm.nodes.name_a + stmt: ListNodes_end + shape: "ListNodes name_a" + params: [{raw: "0"}, {raw: "NULL::bpchar[]"}, {raw: "'any'"}, {raw: "'any'"}, {raw: "NULL::bytea"}, {raw: "'a'"}, {raw: "'infinity'::timestamptz"}, {raw: "50"}, {raw: "''"}, {raw: "false"}, {raw: "''"}, {raw: "'00000000-0000-0000-0000-000000000000'::uuid"}] + expect: + budget: {buffers: 2876, planning_ms: 10} + note: "measured: 1917 / 23166 B (custom / generic); budget: 1.5x measured" + - id: sm.nodes.multibyte_true + stmt: ListNodes_end + shape: "ListNodes multibyte_true" + params: [{raw: "0"}, {raw: "NULL::bpchar[]"}, {raw: "'true'"}, {raw: "'any'"}, {raw: "NULL::bytea"}, {raw: "''"}, {raw: "'infinity'::timestamptz"}, {raw: "50"}, {raw: "''"}, {raw: "false"}, {raw: "''"}, {raw: "'00000000-0000-0000-0000-000000000000'::uuid"}] + expect: + budget: {buffers: 3413, planning_ms: 10} + note: "measured: 2275 / 28864 B (custom / generic); budget: 1.5x measured" + - id: sm.nodes.traces_true + stmt: ListNodes_end + shape: "ListNodes traces_true" + params: [{raw: "0"}, {raw: "NULL::bpchar[]"}, {raw: "'any'"}, {raw: "'true'"}, {raw: "NULL::bytea"}, {raw: "''"}, {raw: "'infinity'::timestamptz"}, {raw: "50"}, {raw: "''"}, {raw: "false"}, {raw: "''"}, {raw: "'00000000-0000-0000-0000-000000000000'::uuid"}] + expect: + budget: {buffers: 3728, planning_ms: 10} + note: "measured: 2485 / 19023 B (custom / generic); budget: 1.5x measured" + - id: sm.nodes.pubkey_prefix + stmt: ListNodes_end + shape: "ListNodes pubkey_prefix" + params: [{raw: "0"}, {raw: "NULL::bpchar[]"}, {raw: "'any'"}, {raw: "'any'"}, {raw: "NULL::bytea"}, {raw: "''"}, {raw: "'infinity'::timestamptz"}, {raw: "50"}, {raw: "''"}, {raw: "false"}, {raw: "'1a'"}, {raw: "'00000000-0000-0000-0000-000000000000'::uuid"}] + expect: + budget: {buffers: 3458, planning_ms: 10} + note: "measured: 2305 / 2490 B (custom / generic); budget: 1.5x measured" + - id: sm.nodes.type9_empty + stmt: ListNodes_end + shape: "ListNodes type9_empty" + params: [{raw: "9"}, {raw: "NULL::bpchar[]"}, {raw: "'any'"}, {raw: "'any'"}, {raw: "NULL::bytea"}, {raw: "''"}, {raw: "'infinity'::timestamptz"}, {raw: "50"}, {raw: "''"}, {raw: "false"}, {raw: "''"}, {raw: "'00000000-0000-0000-0000-000000000000'::uuid"}] + expect: + budget: {buffers: 2189, planning_ms: 10} + note: "measured: 1459 / 1467 B (custom / generic); budget: 1.5x measured" + - id: sm.drift.all + stmt: GetStatsClockDrift + shape: "GetStatsClockDrift all" + params: [{raw: "60::int"}, {raw: "NULL::bpchar[]"}, {raw: "50"}] + expect: + budget: {buffers: 2852, planning_ms: 10} + note: "measured: 1901 / 1908 B (custom / generic); budget: 1.5x measured" + - id: sm.drift.SEA_t0_500 + stmt: GetStatsClockDrift + shape: "GetStatsClockDrift SEA_t0_500" + params: [{raw: "0::int"}, {raw: "'{SEA}'::bpchar[]"}, {raw: "500"}] + expect: + budget: {buffers: 18795, planning_ms: 10} + note: "measured: 12530 / 15018 B (custom / generic); budget: 1.5x measured" + - id: sm.ntypes.all + stmt: GetStatsNodeTypesAll + shape: "GetStatsNodeTypes, no filter -> All" + params: [] + expect: + budget: {buffers: 2189, planning_ms: 10} + note: "measured 1,459 B on the end-state index set; budget 1.5x" + - id: sm.ntypes.SEA + stmt: GetStatsNodeTypesByIATAs + shape: "GetStatsNodeTypes SEA -> ByIATAs (measured: today's text)" + params: [{raw: "'{SEA}'::bpchar[]"}] + expect: + budget: {buffers: 5904, planning_ms: 10} + note: "measured: 3936 / 5263 B (custom / generic); budget: 1.5x measured" + - id: sm.trc.first_l200 + stmt: ListTraceTags_end + shape: "ListTraceTags first_l200 (perf_small)" + params: [{raw: "NULL::bpchar[]"}, {raw: "''"}, {raw: "NULL::timestamptz"}, {raw: "NULL::timestamptz"}, {raw: "NULL::timestamptz"}, {raw: "200"}, {raw: "''"}, {raw: "NULL::bytea"}] + expect: + budget: {buffers: 876, planning_ms: 10} + note: "measured: 584 / 584 B (custom / generic); budget: 1.5x measured" + - id: sm.trc.cursor_l200 + stmt: ListTraceTags_end + shape: "ListTraceTags cursor_l200 (perf_small)" + params: [{raw: "NULL::bpchar[]"}, {raw: "''"}, {raw: "NULL::timestamptz"}, {raw: "NULL::timestamptz"}, {raw: "'2026-10-17 06:17:50.081+00'::timestamptz"}, {raw: "200"}, {raw: "''"}, {raw: "NULL::bytea"}] + expect: + budget: {buffers: 873, planning_ms: 10} + note: "measured: 582 / 582 B (custom / generic); budget: 1.5x measured" + - id: sm.trc.tie_tag_l200 + stmt: ListTraceTags_end + shape: "ListTraceTags, cursor inside a tied millisecond, limit 200" + params: [{raw: "NULL::bpchar[]"}, {raw: "''"}, {raw: "NULL::timestamptz"}, {raw: "NULL::timestamptz"}, {r: trc_tie_ts}, {raw: "200"}, {raw: "''"}, {r: trc_tie_tag}] + expect: + budget: {buffers: 885, planning_ms: 10} + note: "measured: 590 / 590 B (custom / generic); budget: 1.5x measured" + - id: sm.trc.AUS_cursor_l200 + stmt: ListTraceTags_end + shape: "ListTraceTags AUS_cursor_l200 (perf_small)" + params: [{raw: "ARRAY['AUS']::bpchar[]"}, {raw: "''"}, {raw: "NULL::timestamptz"}, {raw: "NULL::timestamptz"}, {raw: "'2026-10-17 06:17:50.081+00'::timestamptz"}, {raw: "200"}, {raw: "''"}, {raw: "NULL::bytea"}] + expect: + budget: {buffers: 2586, planning_ms: 10} + note: "measured: 1724 / 3249 B (custom / generic); budget: 1.5x measured" + - id: sm.trc.cursor_l1 + stmt: ListTraceTags_end + shape: "ListTraceTags cursor_l1 (perf_small)" + params: [{raw: "NULL::bpchar[]"}, {raw: "''"}, {raw: "NULL::timestamptz"}, {raw: "NULL::timestamptz"}, {raw: "'2026-10-17 06:17:50.081+00'::timestamptz"}, {raw: "1"}, {raw: "''"}, {raw: "NULL::bytea"}] + expect: + budget: {buffers: 50, planning_ms: 10} + note: "measured: 7 / 7 B (custom / generic); budget: 1.5x measured" + - id: sm.nodes.first_l501 + stmt: ListNodes_end + shape: "ListNodes first_l501 (perf_small)" + params: [{raw: "0"}, {raw: "NULL::bpchar[]"}, {raw: "'any'"}, {raw: "'any'"}, {raw: "NULL::bytea"}, {raw: "''"}, {raw: "'infinity'::timestamptz"}, {raw: "501"}, {raw: "''"}, {raw: "false"}, {raw: "''"}, {raw: "'00000000-0000-0000-0000-000000000000'::uuid"}] + expect: + budget: {buffers: 13329, planning_ms: 10} + note: "measured: 8886 / 48171 B (custom / generic); budget: 1.5x measured" + - id: sm.nodes.first_l201_neighbors_SEA + stmt: ListNodes_end + shape: "ListNodes first_l201_neighbors_SEA (perf_small)" + params: [{raw: "0"}, {raw: "ARRAY['SEA']::bpchar[]"}, {raw: "'any'"}, {raw: "'any'"}, {raw: "NULL::bytea"}, {raw: "''"}, {raw: "'infinity'::timestamptz"}, {raw: "201"}, {raw: "''"}, {raw: "false"}, {raw: "''"}, {raw: "'00000000-0000-0000-0000-000000000000'::uuid"}] + expect: + budget: {buffers: 15000, planning_ms: 10} + note: "measured: - / - B (custom / generic); budget: class budget" + - id: sm.nodes.legacy_cursor_l501 + stmt: ListNodes_end + shape: "ListNodes legacy_cursor_l501 (perf_small)" + params: [{raw: "0"}, {raw: "NULL::bpchar[]"}, {raw: "'any'"}, {raw: "'any'"}, {raw: "NULL::bytea"}, {raw: "''"}, {raw: "'2026-10-18 12:00:00.123+00'::timestamptz"}, {raw: "501"}, {raw: "''"}, {raw: "false"}, {raw: "''"}, {raw: "'00000000-0000-0000-0000-000000000000'::uuid"}] + expect: + budget: {buffers: 15000, planning_ms: 10} + note: "measured: - / - B (custom / generic); budget: class budget" + - id: sm.nodes.pagecursor_l501 + stmt: ListNodes_end + shape: "ListNodes, page cursor at the 501st node, limit 501" + params: [{raw: "0"}, {raw: "NULL::bpchar[]"}, {raw: "'any'"}, {raw: "'any'"}, {raw: "NULL::bytea"}, {raw: "''"}, {r: nodes_p500_ts}, {raw: "501"}, {raw: "''"}, {raw: "false"}, {raw: "''"}, {r: nodes_p500_id}] + expect: + budget: {buffers: 14388, planning_ms: 10} + note: "measured: 9592 / 48827 B (custom / generic); budget: 1.5x measured" + - id: sm.obsv.first_l201 + stmt: ListObservers_end + shape: "ListObservers first_l201 (perf_small)" + params: [{raw: "NULL::bpchar[]"}, {raw: "''"}, {raw: "''"}, {raw: "''"}, {raw: "''"}, {raw: "'infinity'::timestamptz"}, {raw: "201"}, {raw: "''"}, {raw: "'00000000-0000-0000-0000-000000000000'::uuid"}] + expect: + budget: {buffers: 1310, planning_ms: 10} + note: "measured 873 B at fillfactor 50; budget 1.5x" + - id: sm.obsv.legacy_l201 + stmt: ListObservers_end + shape: "ListObservers legacy_l201 (perf_small)" + params: [{raw: "NULL::bpchar[]"}, {raw: "''"}, {raw: "''"}, {raw: "''"}, {raw: "''"}, {raw: "'2026-10-18 17:28:26.053+00'::timestamptz"}, {raw: "201"}, {raw: "''"}, {raw: "'00000000-0000-0000-0000-000000000000'::uuid"}] + expect: + budget: {buffers: 1310, planning_ms: 10} + note: "measured 873 B at fillfactor 50; budget 1.5x" + - id: sm.trc.since1d_l200 + stmt: ListTraceTags_end + shape: "traces with a first_heard window" + params: [{raw: "NULL::bpchar[]"}, {raw: "''"}, {raw: "'2026-10-17 17:00+00'::timestamptz"}, {raw: "NULL::timestamptz"}, {raw: "NULL::timestamptz"}, {raw: "200"}, {raw: "''"}, {raw: "NULL::bytea"}] + expect: + budget: {buffers: 30000, planning_ms: 10} + note: "measured: - / - B (custom / generic); budget: class budget" + - id: sm.ntypes.bogus5000 + stmt: GetStatsNodeTypesByIATAs + shape: "5,000 bogus IATAs" + params: [{r: small_bogus5000}] + expect: + budget: {buffers: 25000, planning_ms: 60} + note: "measured: - / - B (custom / generic); budget: class budget" + - id: sm.ntypes.list50 + stmt: GetStatsNodeTypesByIATAs + shape: "50-IATA list" + params: [{r: small_list50}] + expect: + budget: {buffers: 191898, planning_ms: 10} + note: "measured: 127932 B on the full local copy. The API pools run with parallel workers off by design, so the planner takes an Index Scan of the region rows (127,932 B, 46 ms) instead of a parallel bitmap scan (5,908 B, 23 ms); accepted" + - id: sm.nodes.bogus5000 + stmt: ListNodes_end + shape: "ListNodes, 5,000 bogus IATAs" + params: [{raw: "0"}, {r: small_bogus5000}, {raw: "'any'"}, {raw: "'any'"}, {raw: "NULL::bytea"}, {raw: "''"}, {raw: "'infinity'::timestamptz"}, {raw: "51"}, {raw: "''"}, {raw: "false"}, {raw: "''"}, {raw: "'00000000-0000-0000-0000-000000000000'::uuid"}] + expect: + budget: {buffers: 15000, planning_ms: 60} + note: "measured: - / - B (custom / generic); budget: class budget" + - id: sm.drift.bogus5000 + stmt: GetStatsClockDrift + shape: "clock drift, 5,000 bogus IATAs" + params: [{raw: "0::int"}, {r: small_bogus5000}, {raw: "50"}] + expect: + budget: {buffers: 25000, planning_ms: 60} + note: "measured: - / - B (custom / generic); budget: class budget" + - id: sm.node.top_sea + stmt: GetNodeByID + shape: "GetNodeByID, the SEA node with the most node_iatas rows" + shapes: [hit] + params: [{r: small_top_sea_node}] + expect: + budget: {buffers: 1000, planning_ms: 10} + note: "measured: - / - B (custom / generic); budget: class budget" + - id: sm.node.unknown + stmt: GetNodeByID + shape: "GetNodeByID, unknown id" + shapes: [miss] + params: [{raw: "'00000000-0000-0000-0000-000000000000'::uuid"}] + expect: + budget: {buffers: 1000, planning_ms: 10} + note: "measured: - / - B (custom / generic); budget: class budget" + - id: sm.chan.first + stmt: ListChannels + shape: "ListChannels page 1 (idx_channels_last_seen_id dropped: sorts 729 rows)" + params: [{raw: "NULL::bytea"}, {raw: "NULL::bpchar[]"}, {raw: "NULL::boolean"}, {raw: "NULL::timestamptz"}, {raw: "51"}] + expect: + budget: {buffers: 1000, planning_ms: 10} + note: "measured: - / - B (custom / generic); budget: class budget" + - id: sm.chan.sea + stmt: ListChannels + shape: "ListChannels, iatas {SEA}" + params: [{raw: "NULL::bytea"}, {raw: "ARRAY['SEA']::bpchar[]"}, {raw: "NULL::boolean"}, {raw: "NULL::timestamptz"}, {raw: "51"}] + expect: + budget: {buffers: 1000, planning_ms: 10} + note: "measured: - / - B (custom / generic); budget: class budget" + - id: sm.chan.after + stmt: ListChannelsAfter + shape: "ListChannelsAfter, cursor 5 h back" + params: [{raw: "'2026-10-18 12:00+00'::timestamptz"}, {raw: "2147483647"}, {raw: "NULL::bytea"}, {raw: "NULL::bpchar[]"}, {raw: "NULL::boolean"}, {raw: "51"}] + expect: + budget: {buffers: 1000, planning_ms: 10} + note: "measured: - / - B (custom / generic); budget: class budget" + - id: sm.write.touch_channel + stmt: TouchChannel + shape: "TouchChannel write-through (plain EXPLAIN)" + params: [null, 5, "\\x01", "\\x0102030405060708"] + expect: + # Either single-row unique probe: on freshly built indexes the (channel_hash, key_fingerprint) unique index + # can cost less than the primary key. + driver: [{relation: channels, any_index: [channels_pkey, channels_channel_hash_key_fingerprint_key]}] + # Not on the fixture: which of its two unique probes the planner takes flips with how fresh the indexes are, so + # its plan signature is not stable there. On freshly built indexes the one-page unique partial index on + # channel_hash costs a hair less than the primary key; both are single-row unique lookups. + - id: sm.write.touch_channel_hash_only + stmt: TouchChannelHashOnly + shape: "TouchChannelHashOnly write-through (plain EXPLAIN)" + params: [5, "\\x01"] + tiers: [local, prod] + expect: + driver: [{relation: channels, any_index: [channels_pkey, idx_channels_hash_no_key]}] + - id: sm.write.upsert_channel + stmt: UpsertChannel + shape: "UpsertChannel fallback for an unknown key (plain EXPLAIN; ingest)" + params: ["\\x0102", null, x, null, false] + - id: sm.write.upsert_node_iata + stmt: UpsertNodeIATA + shape: "UpsertNodeIATA on the swapped node_iatas index set (plain EXPLAIN; ingest)" + params: ["00000000-0000-0000-0000-000000000000", SEA] + + # ---- packets by site through packet_sites (pending) ---- + - id: ps.t.S1_SEA_p1 + stmt: ListPacketsBySite + shape: "SEA p1" + params: [SEA, infinity, "\\x", 51] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 1128, planning_ms: 10} + note: "measured: full 752 / 752 B (custom / generic), page CTE 6" + - id: ps.t.S1_SEA_1d + stmt: ListPacketsBySite + shape: "SEA 1d" + params: [SEA, "2026-10-17 17:00+00", "\\x", 51] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 1728, planning_ms: 10} + note: "measured: full 1152 / 1152 B (custom / generic), page CTE 5" + - id: ps.t.S1_SEA_2_5d + stmt: ListPacketsBySite + shape: "SEA 2.5d" + params: [SEA, "2026-10-16 05:00+00", "\\x", 51] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 1745, planning_ms: 10} + note: "measured: full 1163 / 1157 B (custom / generic), page CTE 5" + - id: ps.t.S1_YOW_p1 + stmt: ListPacketsBySite + shape: "YOW p1" + params: [YOW, infinity, "\\x", 51] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 686, planning_ms: 10} + note: "measured: full 457 / 457 B (custom / generic), page CTE 5" + - id: ps.t.S1_RIX_p1 + stmt: ListPacketsBySite + shape: "RIX p1" + params: [RIX, infinity, "\\x", 51] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 1122, planning_ms: 10} + note: "measured: full 748 / 748 B (custom / generic), page CTE 5" + - id: ps.t.S1_DSM_rare + stmt: ListPacketsBySite + shape: "DSM rare" + params: [DSM, infinity, "\\x", 51] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 72, planning_ms: 10} + note: "measured: full 48 / 48 B (custom / generic), page CTE 5" + - id: ps.t.S1_ZZZ_empty + stmt: ListPacketsBySite + shape: "ZZZ empty" + params: [ZZZ, infinity, "\\x", 51] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 50, planning_ms: 10} + note: "measured: full 4 / 4 B (custom / generic), page CTE 4" + - id: ps.t.S1_SEA_l200 + stmt: ListPacketsBySite + shape: "SEA l200" + params: [SEA, infinity, "\\x", 201] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 5313, planning_ms: 10} + note: "measured: full 3542 / 3542 B (custom / generic), page CTE 7" + - id: ps.t.S2_BRE_WVN + stmt: ListPacketsBySitesFirst + shape: "BRE,WVN" + params: ["{BRE,WVN}", infinity, "\\x", 51] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 741, planning_ms: 10} + note: "measured: full 494 / 494 B (custom / generic), page CTE 9" + - id: ps.t.S2_EUG_PDX_SEA_SLE + stmt: ListPacketsBySitesFirst + shape: "EUG,PDX,SEA,SLE" + params: ["{EUG,PDX,SEA,SLE}", infinity, "\\x", 51] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 1197, planning_ms: 10} + note: "measured: full 798 / 798 B (custom / generic), page CTE 19" + - id: ps.t.S2_pnw13_l50 + stmt: ListPacketsBySitesFirst + shape: "pnw13 l50" + params: ["{BLI,CVO,KLS,LMT,MFR,MMV,ONP,PAE,PDX,RDM,SEA,SLE,TTD}", infinity, "\\x", 51] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 1259, planning_ms: 10} + note: "measured: full 839 / 839 B (custom / generic), page CTE 56" + - id: ps.t.S2_pnw13_l100 + stmt: ListPacketsBySitesFirst + shape: "pnw13 l100" + params: ["{BLI,CVO,KLS,LMT,MFR,MMV,ONP,PAE,PDX,RDM,SEA,SLE,TTD}", infinity, "\\x", 101] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 2300, planning_ms: 10} + note: "measured: full 1533 / 1533 B (custom / generic), page CTE 62" + - id: ps.t.S2_pnw13_l100_1d + stmt: ListPacketsBySites + shape: "pnw13 l100 1d" + params: ["{BLI,CVO,KLS,LMT,MFR,MMV,ONP,PAE,PDX,RDM,SEA,SLE,TTD}", "2026-10-17 17:00+00", "\\x", 101] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}, {scope: "CTE page", relation: packet_sites, alias: x, index: packet_sites_pkey, nodes: ["Index Only Scan"], index_cond: ["packet_hash = ps.packet_hash"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 9977, planning_ms: 10} + note: "measured: full 6651 / 6651 B (custom / generic), page CTE 4413" + - id: ps.t.S2_pnw13_l100_2_5d + stmt: ListPacketsBySites + shape: "pnw13 l100 2.5d" + params: ["{BLI,CVO,KLS,LMT,MFR,MMV,ONP,PAE,PDX,RDM,SEA,SLE,TTD}", "2026-10-16 05:00+00", "\\x", 101] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}, {scope: "CTE page", relation: packet_sites, alias: x, index: packet_sites_pkey, nodes: ["Index Only Scan"], index_cond: ["packet_hash = ps.packet_hash"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 9297, planning_ms: 10} + note: "measured: full 6198 / 6198 B (custom / generic), page CTE 3982" + - id: ps.t.S2_pnw13_l200 + stmt: ListPacketsBySitesFirst + shape: "pnw13 l200" + params: ["{BLI,CVO,KLS,LMT,MFR,MMV,ONP,PAE,PDX,RDM,SEA,SLE,TTD}", infinity, "\\x", 201] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 5190, planning_ms: 10} + note: "measured: full 3460 / 3460 B (custom / generic), page CTE 70" + - id: ps.t.S2_nl12_l50 + stmt: ListPacketsBySitesFirst + shape: "nl12 l50" + params: ["{AMS,EIN,ENS,GLZ,GRQ,LEY,LWR,MST,QAR,RTM,UDE,UTC}", infinity, "\\x", 51] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 1364, planning_ms: 10} + note: "measured: full 909 / 909 B (custom / generic), page CTE 53" + - id: ps.t.S2_nl12_judge_cursor_l1 + stmt: ListPacketsBySites + shape: "nl12, cursor at the newest row before a fixed time, limit 1" + params: ["{AMS,EIN,ENS,GLZ,GRQ,LEY,LWR,MST,QAR,RTM,UDE,UTC}", {r: nl12_judge_ts}, {r: nl12_judge_hash}, 2] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}, {scope: "CTE page", relation: packet_sites, alias: x, index: packet_sites_pkey, nodes: ["Index Only Scan"], index_cond: ["packet_hash = ps.packet_hash"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 275, planning_ms: 10} + note: "measured: full 183 / 180 B (custom / generic), page CTE 161" + - id: ps.t.S2_gbr22_l50 + stmt: ListPacketsBySitesFirst + shape: "gbr22 l50" + params: ["{BHX,BLK,BOH,BRS,CAX,CEG,DSA,EMA,EXT,HUY,LCY,LON,LPL,MAN,MME,NQT,NQY,NWI,ORM,PLH,PME,VLY}", infinity, "\\x", 51] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 1386, planning_ms: 10} + note: "measured: full 924 / 924 B (custom / generic), page CTE 91" + - id: ps.t.S2_gbr22_l100_1d + stmt: ListPacketsBySites + shape: "gbr22 l100 1d" + params: ["{BHX,BLK,BOH,BRS,CAX,CEG,DSA,EMA,EXT,HUY,LCY,LON,LPL,MAN,MME,NQT,NQY,NWI,ORM,PLH,PME,VLY}", "2026-10-17 17:00+00", "\\x", 101] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}, {scope: "CTE page", relation: packet_sites, alias: x, index: packet_sites_pkey, nodes: ["Index Only Scan"], index_cond: ["packet_hash = ps.packet_hash"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 9629, planning_ms: 10} + note: "measured: full 6419 / 6419 B (custom / generic), page CTE 4743" + - id: ps.t.S3_STR_pt9 + stmt: ListPacketsBySitesTypedFirst + shape: "STR pt9" + params: ["{STR}", "{9}", "{0,1,2,3}", infinity, "\\x", 51] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site_type, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["payload_type = pt.t", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 884, planning_ms: 10} + note: "measured: full 469 / 589 B (custom / generic), page CTE 21" + - id: ps.t.S3_pf7_pt2_rt2_3_l100 + stmt: ListPacketsBySitesTypedFirst + shape: "pf7 pt2 rt2,3 l100" + params: ["{STR,MHG,FRA,CGN,DTM,ESS,DUS}", "{2}", "{2,3}", infinity, "\\x", 101] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site_type, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["payload_type = pt.t", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 1800, planning_ms: 10} + note: "measured: full 948 / 1200 B (custom / generic), page CTE 67" + - id: ps.t.S3_pf7_l100_1d + stmt: ListPacketsBySitesTyped + shape: "pf7 l100 1d" + params: ["{STR,MHG,FRA,CGN,DTM,ESS,DUS}", "{2}", "{2,3}", "2026-10-17 17:00+00", "\\x", 101] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site_type, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["payload_type = pt.t", "ROW(site_heard_at, packet_hash) <"]}, {scope: "CTE page", relation: packet_sites, alias: x, index: packet_sites_pkey, nodes: ["Index Only Scan"], index_cond: ["packet_hash = ps.packet_hash"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 4805, planning_ms: 10} + note: "measured: full 2996 / 3203 B (custom / generic), page CTE 2039" + - id: ps.t.S3_SEA_pt13 + stmt: ListPacketsBySitesTypedFirst + shape: "SEA pt13" + params: ["{SEA}", "{13}", "{0,1,2,3}", infinity, "\\x", 51] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site_type, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["payload_type = pt.t", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 921, planning_ms: 10} + note: "measured: full 524 / 614 B (custom / generic), page CTE 26" + - id: ps.t.S3_SEA_rt3 + stmt: ListPacketsBySitesTypedFirst + shape: "SEA rt3" + params: ["{SEA}", "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{3}", infinity, "\\x", 51] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site_type, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["payload_type = pt.t", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 1007, planning_ms: 10} + note: "measured: full 584 / 671 B (custom / generic), page CTE 100" + - id: ps.t.S3_pnw13_rt0_l200 + stmt: ListPacketsBySitesTypedFirst + shape: "pnw13 rt0 l200" + params: ["{BLI,CVO,KLS,LMT,MFR,MMV,ONP,PAE,PDX,RDM,SEA,SLE,TTD}", "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0}", infinity, "\\x", 201] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site_type, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["payload_type = pt.t", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 6645, planning_ms: 10} + note: "measured: full 3980 / 4430 B (custom / generic), page CTE 1051" + - id: ps.t.S3_pnw13_rt2_l50 + stmt: ListPacketsBySitesTypedFirst + shape: "pnw13 rt2 l50" + params: ["{BLI,CVO,KLS,LMT,MFR,MMV,ONP,PAE,PDX,RDM,SEA,SLE,TTD}", "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{2}", infinity, "\\x", 51] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site_type, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["payload_type = pt.t", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 2327, planning_ms: 10} + note: "measured: full 1466 / 1551 B (custom / generic), page CTE 967" + - id: ps.t.S4_AMS_nl + stmt: ListPacketsBySitesScopedFirst + shape: "AMS #nl" + params: ["{AMS}", infinity, "\\x", 51, "-infinity", infinity, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}", "{#nl}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site_scope, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 15000, planning_ms: 10} + note: "no matching measured run" + - id: ps.t.S4_SEA_GRR + stmt: ListPacketsBySitesScopedFirst + shape: "SEA #GRR" + params: ["{SEA}", infinity, "\\x", 51, "-infinity", infinity, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}", "{#GRR}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site_scope, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 15000, planning_ms: 10} + note: "no matching measured run" + - id: ps.t.S4_SEA_dash8_pt4 + stmt: ListPacketsBySitesScopedFirst + shape: "SEA dash8 pt4" + params: ["{SEA}", infinity, "\\x", 51, "-infinity", infinity, "{4}", "{0,1,2,3}", {r: ps_list_dash8}] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site_scope, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 15000, planning_ms: 10} + note: "no matching measured run" + - id: ps.t.S4_SEA_1258 + stmt: ListPacketsBySitesScopedFirst + shape: "SEA 1258" + params: ["{SEA}", infinity, "\\x", 51, "-infinity", infinity, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}", {r: ps_list_scopes1258}] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site_scope, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 15000, planning_ms: 10} + note: "no matching measured run" + - id: ps.t.S4_pnw13_1258 + stmt: ListPacketsBySitesScopedFirst + shape: "pnw13 1258" + params: ["{BLI,CVO,KLS,LMT,MFR,MMV,ONP,PAE,PDX,RDM,SEA,SLE,TTD}", infinity, "\\x", 51, "-infinity", infinity, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}", {r: ps_list_scopes1258}] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site_scope, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 75377, planning_ms: 10} + note: "no matching measured run; budget corrected from the end-state measurement (was 15000)" + - id: ps.t.S4_nl12_nl_1d + stmt: ListPacketsBySitesScoped + shape: "nl12 #nl 1d" + params: ["{AMS,EIN,ENS,GLZ,GRQ,LEY,LWR,MST,QAR,RTM,UDE,UTC}", "2026-10-17 17:00+00", "\\x", 51, "-infinity", infinity, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}", "{#nl}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site_scope, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(site_heard_at, packet_hash) <"]}, {scope: "CTE page", relation: packet_sites, alias: x, index: packet_sites_pkey, nodes: ["Index Only Scan"], index_cond: ["packet_hash = ps.packet_hash"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 15000, planning_ms: 10} + note: "no matching measured run" + - id: ps.t.S2_pnw13_l50_1d + stmt: ListPacketsBySites + shape: "pnw13 l50 1d" + params: ["{BLI,CVO,KLS,LMT,MFR,MMV,ONP,PAE,PDX,RDM,SEA,SLE,TTD}", "2026-10-17 17:00+00", "\\x", 51] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}, {scope: "CTE page", relation: packet_sites, alias: x, index: packet_sites_pkey, nodes: ["Index Only Scan"], index_cond: ["packet_hash = ps.packet_hash"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 5165, planning_ms: 10} + note: "measured: full 3443 / 3443 B (custom / generic), page CTE 2341" + - id: ps.t.S2_nl12_l50_2_5d + stmt: ListPacketsBySites + shape: "nl12 l50 2.5d" + params: ["{AMS,EIN,ENS,GLZ,GRQ,LEY,LWR,MST,QAR,RTM,UDE,UTC}", "2026-10-16 05:00+00", "\\x", 51] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}, {scope: "CTE page", relation: packet_sites, alias: x, index: packet_sites_pkey, nodes: ["Index Only Scan"], index_cond: ["packet_hash = ps.packet_hash"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 4181, planning_ms: 10} + note: "measured: full 2787 / 2787 B (custom / generic), page CTE 1971" + - id: ps.t.S2_gbr22_l50_2_5d + stmt: ListPacketsBySites + shape: "gbr22 l50 2.5d" + params: ["{BHX,BLK,BOH,BRS,CAX,CEG,DSA,EMA,EXT,HUY,LCY,LON,LPL,MAN,MME,NQT,NQY,NWI,ORM,PLH,PME,VLY}", "2026-10-16 05:00+00", "\\x", 51] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}, {scope: "CTE page", relation: packet_sites, alias: x, index: packet_sites_pkey, nodes: ["Index Only Scan"], index_cond: ["packet_hash = ps.packet_hash"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 4776, planning_ms: 10} + note: "measured: full 3184 / 3184 B (custom / generic), page CTE 2310" + - id: ps.t.S2_all561_l50 + stmt: ListPacketsBySitesFirst + shape: "all561 l50" + params: [{r: ps_all_sites}, infinity, "\\x", 51] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 4865, planning_ms: 10} + note: "measured: full 3243 / 3243 B (custom / generic), page CTE 2496" + - id: ps.t.S2_all561_l50_1d + stmt: ListPacketsBySites + shape: "all561 l50 1d" + params: [{r: ps_all_sites}, "2026-10-17 17:00+00", "\\x", 51] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}, {scope: "CTE page", relation: packet_sites, alias: x, index: packet_sites_pkey, nodes: ["Index Only Scan"], index_cond: ["packet_hash = ps.packet_hash"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 135714, planning_ms: 10} + note: "measured: full 90476 / 90476 B (custom / generic), page CTE 89703" + - id: ps.t.S3_pf7_l100_2_5d + stmt: ListPacketsBySitesTyped + shape: "pf7 l100 2.5d" + params: ["{STR,MHG,FRA,CGN,DTM,ESS,DUS}", "{2}", "{2,3}", "2026-10-16 05:00+00", "\\x", 101] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site_type, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["payload_type = pt.t", "ROW(site_heard_at, packet_hash) <"]}, {scope: "CTE page", relation: packet_sites, alias: x, index: packet_sites_pkey, nodes: ["Index Only Scan"], index_cond: ["packet_hash = ps.packet_hash"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 3417, planning_ms: 10} + note: "measured: full 2011 / 2278 B (custom / generic), page CTE 1157" + - id: ps.t.S3_pnw13_rt0_l200_1d + stmt: ListPacketsBySitesTyped + shape: "pnw13 rt0 l200 1d" + params: ["{BLI,CVO,KLS,LMT,MFR,MMV,ONP,PAE,PDX,RDM,SEA,SLE,TTD}", "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0}", "2026-10-17 17:00+00", "\\x", 201] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site_type, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["payload_type = pt.t", "ROW(site_heard_at, packet_hash) <"]}, {scope: "CTE page", relation: packet_sites, alias: x, index: packet_sites_pkey, nodes: ["Index Only Scan"], index_cond: ["packet_hash = ps.packet_hash"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 27417, planning_ms: 10} + note: "measured: full 17792 / 18278 B (custom / generic), page CTE 15136" + - id: ps.t.S3_pnw13_rt2_l50_2_5d + stmt: ListPacketsBySitesTyped + shape: "pnw13 rt2 l50 2.5d" + params: ["{BLI,CVO,KLS,LMT,MFR,MMV,ONP,PAE,PDX,RDM,SEA,SLE,TTD}", "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{2}", "2026-10-16 05:00+00", "\\x", 51] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site_type, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["payload_type = pt.t", "ROW(site_heard_at, packet_hash) <"]}, {scope: "CTE page", relation: packet_sites, alias: x, index: packet_sites_pkey, nodes: ["Index Only Scan"], index_cond: ["packet_hash = ps.packet_hash"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 10610, planning_ms: 10} + note: "measured: full 6992 / 7073 B (custom / generic), page CTE 6489" + - id: ps.t.S3_STR_pt9_1d + stmt: ListPacketsBySitesTypedFirst + shape: "STR pt9 1d" + params: ["{STR}", "{9}", "{0,1,2,3}", "2026-10-17 17:00+00", "\\x", 51] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site_type, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["payload_type = pt.t", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 30000, planning_ms: 10} + note: "no matching measured run" + - id: ps.t.S4_pnw13_1258_1d + stmt: ListPacketsBySitesScoped + shape: "pnw13 1258 1d" + params: ["{BLI,CVO,KLS,LMT,MFR,MMV,ONP,PAE,PDX,RDM,SEA,SLE,TTD}", "2026-10-17 17:00+00", "\\x", 51, "-infinity", infinity, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}", {r: ps_list_scopes1258}] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site_scope, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(site_heard_at, packet_hash) <"]}, {scope: "CTE page", relation: packet_sites, alias: x, index: packet_sites_pkey, nodes: ["Index Only Scan"], index_cond: ["packet_hash = ps.packet_hash"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 85917, planning_ms: 10} + note: "no matching measured run; budget corrected from the end-state measurement (was 15000)" + - id: ps.t.S4_SEA_dash8_pt4_1d + stmt: ListPacketsBySitesScopedFirst + shape: "SEA dash8 pt4 1d" + params: ["{SEA}", "2026-10-17 17:00+00", "\\x", 51, "-infinity", infinity, "{4}", "{0,1,2,3}", {r: ps_list_dash8}] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site_scope, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 15000, planning_ms: 10} + note: "no matching measured run" + - id: ps.t.S4_AMS_nl_2_5d + stmt: ListPacketsBySitesScopedFirst + shape: "AMS #nl 2.5d" + params: ["{AMS}", "2026-10-16 05:00+00", "\\x", 51, "-infinity", infinity, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}", "{#nl}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site_scope, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 15000, planning_ms: 10} + note: "no matching measured run" + - id: ps.f.SEA_since_T0_3d_L50 + stmt: ListPacketsBySitesSinceFirst + shape: "fixer: SEA since T0-3d L50" + params: ["{SEA}", infinity, "\\x", 51, "2026-10-15 17:00+00", infinity, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 1125, planning_ms: 10} + note: "measured: full 750 / 846 B (custom / generic), page CTE 4" + - id: ps.f.SEA_since_0_L50 + stmt: ListPacketsBySitesSinceFirst + shape: "fixer: SEA since=0 L50" + params: ["{SEA}", infinity, "\\x", 51, "1970-01-01 00:00+00", infinity, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 1125, planning_ms: 10} + note: "measured: full 750 / 846 B (custom / generic), page CTE 4" + - id: ps.f.SEA_since_T0_10min_L50 + stmt: ListPacketsBySitesSinceFirst + shape: "fixer: SEA since T0-10min L50" + params: ["{SEA}", infinity, "\\x", 51, "2026-10-18 16:50+00", infinity, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 1269, planning_ms: 10} + note: "measured: full 846 / 846 B (custom / generic), page CTE 4" + - id: ps.f.MAD_since_T0_3d_L50 + stmt: ListPacketsBySitesSinceFirst + shape: "fixer: MAD since T0-3d L50" + params: ["{MAD}", infinity, "\\x", 51, "2026-10-15 17:00+00", infinity, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 770, planning_ms: 10} + note: "measured: full 513 / 645 B (custom / generic), page CTE 4" + - id: ps.f.DSM_since_T0_1d_L50 + stmt: ListPacketsBySitesSinceFirst + shape: "fixer: DSM since T0-1d L50" + params: ["{DSM}", infinity, "\\x", 51, "2026-10-17 17:00+00", infinity, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 50, planning_ms: 10} + note: "measured: full 4 / 4 B (custom / generic), page CTE 4" + - id: ps.f.ZZZ_since_T0_1d_L50 + stmt: ListPacketsBySitesSinceFirst + shape: "fixer: ZZZ since T0-1d L50" + params: ["{ZZZ}", infinity, "\\x", 51, "2026-10-17 17:00+00", infinity, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 50, planning_ms: 10} + note: "measured: full 3 / 3 B (custom / generic), page CTE 3" + - id: ps.f.pnw_since_T0_1d_L50 + stmt: ListPacketsBySitesSinceFirst + shape: "fixer: pnw since T0-1d L50" + params: [{r: ps_list_pnw}, infinity, "\\x", 51, "2026-10-17 17:00+00", infinity, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 1241, planning_ms: 10} + note: "measured: full 827 / 920 B (custom / generic), page CTE 44" + - id: ps.f.pnw_until_only_T0_L50 + stmt: ListPacketsBySitesSinceFirst + shape: "fixer: pnw until-only T0 L50" + params: [{r: ps_list_pnw}, infinity, "\\x", 51, "-infinity", "2026-10-18 17:00+00", "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 1239, planning_ms: 10} + note: "measured: full 826 / 919 B (custom / generic), page CTE 43" + - id: ps.f.pnw_until_only_T0_2d_L50 + stmt: ListPacketsBySitesSinceFirst + shape: "fixer: pnw until-only T0-2d L50" + params: [{r: ps_list_pnw}, infinity, "\\x", 51, "-infinity", "2026-10-16 17:00+00", "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 4959, planning_ms: 10} + note: "measured: full 3306 / 3366 B (custom / generic), page CTE 882" + - id: ps.f.pnw_wide_past_T0_3d_T0_1d_L50 + stmt: ListPacketsBySitesSinceFirst + shape: "fixer: pnw wide-past T0-3d..T0-1d L50" + params: [{r: ps_list_pnw}, infinity, "\\x", 51, "2026-10-15 17:00+00", "2026-10-17 17:00+00", "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 4631, planning_ms: 10} + note: "measured: full 3087 / 3144 B (custom / generic), page CTE 721" + - id: ps.f.pnw_pf_75min_14_00_15_15_L200 + stmt: ListPacketsBySitesSinceFirst + shape: "fixer: pnw pf 75min 14:00-15:15 L200" + params: [{r: ps_list_pnw}, infinity, "\\x", 201, "2026-10-18 14:00+00", "2026-10-18 15:15+00", "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 7916, planning_ms: 10} + note: "measured: full 5277 / 5277 B (custom / generic), page CTE 541" + - id: ps.f.pnw_pf_12_5h_rt0_L200 + stmt: ListPacketsBySitesSinceFirst + shape: "fixer: pnw pf 12.5h rt0 L200" + params: [{r: ps_list_pnw}, infinity, "\\x", 201, "2026-10-18 03:30+00", "2026-10-18 16:00+00", "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 7388, planning_ms: 10} + note: "measured: full 4925 / 4925 B (custom / generic), page CTE 1715" + - id: ps.f.pnw_pf_12_5h_no_type_L200 + stmt: ListPacketsBySitesSinceFirst + shape: "fixer: pnw pf 12.5h no type L200" + params: [{r: ps_list_pnw}, infinity, "\\x", 201, "2026-10-18 03:30+00", "2026-10-18 16:00+00", "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 7332, planning_ms: 10} + note: "measured: full 4888 / 4888 B (custom / generic), page CTE 101" + - id: ps.f.pnw_narrow_deep_10_16_17_00_75min_L200 + stmt: ListPacketsBySitesSinceFirst + shape: "fixer: pnw narrow deep 10-16 17:00+75min L200" + params: [{r: ps_list_pnw}, infinity, "\\x", 201, "2026-10-16 17:00+00", "2026-10-16 18:15+00", "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 10584, planning_ms: 10} + note: "measured: full 7056 / 7056 B (custom / generic), page CTE 2270" + - id: ps.f.pnw_pt4_since_T0_1d_L50 + stmt: ListPacketsBySitesSinceFirst + shape: "fixer: pnw pt4 since T0-1d L50" + params: [{r: ps_list_pnw}, infinity, "\\x", 51, "2026-10-17 17:00+00", infinity, "{4}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 1535, planning_ms: 10} + note: "measured: full 1023 / 1023 B (custom / generic), page CTE 77" + - id: ps.f.nl_since_T0_1d_L50 + stmt: ListPacketsBySitesSinceFirst + shape: "fixer: nl since T0-1d L50" + params: [{r: ps_list_nl}, infinity, "\\x", 51, "2026-10-17 17:00+00", infinity, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 1344, planning_ms: 10} + note: "measured: full 896 / 953 B (custom / generic), page CTE 40" + - id: ps.f.gbr_wide_past_T0_3d_T0_1d_L50 + stmt: ListPacketsBySitesSinceFirst + shape: "fixer: gbr wide-past T0-3d..T0-1d L50" + params: [{r: ps_list_gbr}, infinity, "\\x", 51, "2026-10-15 17:00+00", "2026-10-17 17:00+00", "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 2460, planning_ms: 10} + note: "measured: full 1640 / 1730 B (custom / generic), page CTE 692" + - id: ps.f.cz_until_only_T0_2d_L50 + stmt: ListPacketsBySitesSinceFirst + shape: "fixer: cz until-only T0-2d L50" + params: [{r: ps_list_cz}, infinity, "\\x", 51, "-infinity", "2026-10-16 17:00+00", "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 3185, planning_ms: 10} + note: "measured: full 2123 / 2216 B (custom / generic), page CTE 1177" + - id: ps.f.pf7_pt2_rt2_3_12_5h_L100 + stmt: ListPacketsBySitesSinceFirst + shape: "fixer: pf7 pt2 rt2,3 12.5h L100" + params: [{r: ps_list_pathfinder7}, infinity, "\\x", 101, "2026-10-18 03:30+00", "2026-10-18 16:00+00", "{2}", "{2,3}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 3264, planning_ms: 10} + note: "measured: full 2176 / 2176 B (custom / generic), page CTE 1035" + - id: ps.f.deep_pnw_since_T0_3d_cur_10_17_05_00_L50 + stmt: ListPacketsBySitesSince + shape: "fixer: deep pnw since T0-3d cur 10-17 05:00 L50" + params: [{r: ps_list_pnw}, "2026-10-17 05:00+00", "\\x", 51, "2026-10-15 17:00+00", infinity, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}, {scope: "CTE page", relation: packet_sites, alias: x, index: packet_sites_pkey, nodes: ["Index Only Scan"], index_cond: ["packet_hash = ps.packet_hash"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 5426, planning_ms: 10} + note: "measured: full 3617 / 3719 B (custom / generic), page CTE 2207" + - id: ps.f.deep_pnw_wide_past_cur_10_16_12_00_L50 + stmt: ListPacketsBySitesSince + shape: "fixer: deep pnw wide-past cur 10-16 12:00 L50" + params: [{r: ps_list_pnw}, "2026-10-16 12:00+00", "\\x", 51, "2026-10-15 17:00+00", "2026-10-17 17:00+00", "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}, {scope: "CTE page", relation: packet_sites, alias: x, index: packet_sites_pkey, nodes: ["Index Only Scan"], index_cond: ["packet_hash = ps.packet_hash"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 5156, planning_ms: 10} + note: "measured: full 3437 / 3437 B (custom / generic), page CTE 2121" + - id: ps.f.deep_pnw_75min_cur_15_00_L200 + stmt: ListPacketsBySitesSince + shape: "fixer: deep pnw 75min cur 15:00 L200" + params: [{r: ps_list_pnw}, "2026-10-18 15:00+00", "\\x", 201, "2026-10-18 14:00+00", "2026-10-18 15:15+00", "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}, {scope: "CTE page", relation: packet_sites, alias: x, index: packet_sites_pkey, nodes: ["Index Only Scan"], index_cond: ["packet_hash = ps.packet_hash"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 14528, planning_ms: 10} + note: "measured: full 9685 / 9685 B (custom / generic), page CTE 5560" + - id: ps.f.deep_SEA_since_T0_3d_cur_10_16_05_00_L50 + stmt: ListPacketsBySitesSinceFirst + shape: "fixer: deep SEA since T0-3d cur 10-16 05:00 L50" + params: ["{SEA}", "2026-10-16 05:00+00", "\\x", 51, "2026-10-15 17:00+00", infinity, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 1907, planning_ms: 10} + note: "measured: full 1271 / 1271 B (custom / generic), page CTE 5" + - id: ps.f.deep_nl_until_only_cur_10_16_04_00_L100 + stmt: ListPacketsBySitesSince + shape: "fixer: deep nl until-only cur 10-16 04:00 L100" + params: [{r: ps_list_nl}, "2026-10-16 04:00+00", "\\x", 101, "-infinity", "2026-10-18 17:00+00", "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}, {scope: "CTE page", relation: packet_sites, alias: x, index: packet_sites_pkey, nodes: ["Index Only Scan"], index_cond: ["packet_hash = ps.packet_hash"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 8612, planning_ms: 10} + note: "measured: full 5741 / 5741 B (custom / generic), page CTE 3706" + - id: ps.f.deep_gbr_since_T0_2d_cur_10_17_03_00_L100 + stmt: ListPacketsBySitesSince + shape: "fixer: deep gbr since T0-2d cur 10-17 03:00 L100" + params: [{r: ps_list_gbr}, "2026-10-17 03:00+00", "\\x", 101, "2026-10-16 17:00+00", infinity, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}, {scope: "CTE page", relation: packet_sites, alias: x, index: packet_sites_pkey, nodes: ["Index Only Scan"], index_cond: ["packet_hash = ps.packet_hash"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 10170, planning_ms: 10} + note: "measured: full 6780 / 6780 B (custom / generic), page CTE 4678" + - id: ps.f.sc_nl_nl_pt4_2_1d_L50 + stmt: ListPacketsBySitesScopedFirst + shape: "fixer: sc nl #nl pt4,2 1d L50" + params: [{r: ps_list_nl}, infinity, "\\x", 51, "2026-10-16 00:00+00", "2026-10-17 00:00+00", "{4,2}", "{0,1,2,3}", "{#nl}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site_scope, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 3888, planning_ms: 10} + note: "measured: full 2592 / 2592 B (custom / generic), page CTE 895" + - id: ps.f.sc_pnw_dash8_pt4_since_2_7d_L50 + stmt: ListPacketsBySitesScopedFirst + shape: "fixer: sc pnw dash8 pt4 since 2.7d L50" + params: [{r: ps_list_pnw}, infinity, "\\x", 51, "2026-10-16 00:00+00", infinity, "{4}", "{0,1,2,3}", {r: ps_list_dash8}] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site_scope, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 1959, planning_ms: 10} + note: "measured: full 1306 / 1306 B (custom / generic), page CTE 344" + - id: ps.f.sc_pnw_sea41_since_T0_1d_cur_10_18_06_00_L50 + stmt: ListPacketsBySitesScoped + shape: "fixer: sc pnw sea41 since T0-1d cur 10-18 06:00 L50" + params: [{r: ps_list_pnw}, "2026-10-18 06:00+00", "\\x", 51, "2026-10-17 17:00+00", infinity, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}", {r: ps_list_scopes_sea41}] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site_scope, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(site_heard_at, packet_hash) <"]}, {scope: "CTE page", relation: packet_sites, alias: x, index: packet_sites_pkey, nodes: ["Index Only Scan"], index_cond: ["packet_hash = ps.packet_hash"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 7677, planning_ms: 10} + note: "measured: full 5118 / 5118 B (custom / generic), page CTE 4091" + - id: ps.f.sc_SEA_GRR_since_T0_3d_L50 + stmt: ListPacketsBySitesScopedFirst + shape: "fixer: sc SEA #GRR since T0-3d L50" + params: ["{SEA}", infinity, "\\x", 51, "2026-10-15 17:00+00", infinity, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}", "{#GRR}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site_scope, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 50, planning_ms: 10} + note: "measured: full 6 / 6 B (custom / generic), page CTE 6" + - id: ps.f.pnw_since_T0_3d_cur_10_17_05_00_L50_recheck + stmt: ListPacketsBySitesSince + shape: "fixer: pnw since T0-3d cur 10-17 05:00 L50 [recheck]" + params: [{r: ps_list_pnw}, "2026-10-17 05:00+00", "\\x", 51, "2026-10-15 17:00+00", infinity, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}, {scope: "CTE page", relation: packet_sites, alias: x, index: packet_sites_pkey, nodes: ["Index Only Scan"], index_cond: ["packet_hash = ps.packet_hash"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 5426, planning_ms: 10} + note: "measured: full 3617 / 3719 B (custom / generic), page CTE 2207" + - id: ps.f.pnw_75min_cur_15_00_L200_recheck + stmt: ListPacketsBySitesSince + shape: "fixer: pnw 75min cur 15:00 L200 [recheck]" + params: [{r: ps_list_pnw}, "2026-10-18 15:00+00", "\\x", 201, "2026-10-18 14:00+00", "2026-10-18 15:15+00", "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}, {scope: "CTE page", relation: packet_sites, alias: x, index: packet_sites_pkey, nodes: ["Index Only Scan"], index_cond: ["packet_hash = ps.packet_hash"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 14528, planning_ms: 10} + note: "measured: full 9685 / 9685 B (custom / generic), page CTE 5560" + - id: ps.f.kept81_75min_14_00_15_15_L200 + stmt: ListPacketsBySitesSinceFirst + shape: "fixer: kept81 75min 14:00-15:15 L200" + params: [{r: ps_kept}, infinity, "\\x", 201, "2026-10-18 14:00+00", "2026-10-18 15:15+00", "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 11486, planning_ms: 10} + note: "measured: full 7657 / 7657 B (custom / generic), page CTE 1552" + - id: ps.f.kept81_since_T0_10min_L50 + stmt: ListPacketsBySitesSinceFirst + shape: "fixer: kept81 since T0-10min L50" + params: [{r: ps_kept}, infinity, "\\x", 51, "2026-10-18 16:50+00", infinity, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 8237, planning_ms: 10} + note: "measured: full 5491 / 5491 B (custom / generic), page CTE 4680" + - id: ps.f.kept81_wide_past_T0_3d_T0_1d_L50 + stmt: ListPacketsBySitesSinceFirst + shape: "fixer: kept81 wide-past T0-3d..T0-1d L50" + params: [{r: ps_kept}, infinity, "\\x", 51, "2026-10-15 17:00+00", "2026-10-17 17:00+00", "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 9746, planning_ms: 10} + note: "measured: full 6497 / 6542 B (custom / generic), page CTE 4298" + - id: ps.f.kept81_empty_match_since_T0_1h_L50 + stmt: ListPacketsBySitesSinceFirst + shape: "fixer: kept81 empty-match since T0+1h L50" + params: [{r: ps_kept}, infinity, "\\x", 51, "2026-10-18 18:00+00", infinity, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 20489, planning_ms: 10} + note: "measured: full 13659 / 13659 B (custom / generic), page CTE 13659" + - id: ps.f.kept81_deep_wide_past_cur_10_16_12_00_L50 + stmt: ListPacketsBySitesSince + shape: "fixer: kept81 deep wide-past cur 10-16 12:00 L50" + params: [{r: ps_kept}, "2026-10-16 12:00+00", "\\x", 51, "2026-10-15 17:00+00", "2026-10-17 17:00+00", "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}, {scope: "CTE page", relation: packet_sites, alias: x, index: packet_sites_pkey, nodes: ["Index Only Scan"], index_cond: ["packet_hash = ps.packet_hash"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 19031, planning_ms: 10} + note: "measured: full 12687 / 12687 B (custom / generic), page CTE 11746" + - id: ps.f.S_nl_nl_pt15_0_rows_first + stmt: ListPacketsBySitesScopedFirst + shape: "fixer: S nl #nl pt15 (0 rows) first" + params: [{r: ps_list_nl}, infinity, "\\x", 51, "-infinity", infinity, "{15}", "{0,1,2,3}", "{#nl}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site_scope, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 2111, planning_ms: 10} + note: "measured: full 1407 / 1407 B (custom / generic), page CTE 1407" + - id: ps.f.S_nl_nl_pt4_1d + stmt: ListPacketsBySitesScoped + shape: "fixer: S nl #nl pt4 1d" + params: [{r: ps_list_nl}, "2026-10-17 17:00+00", "\\x", 51, "-infinity", infinity, "{4}", "{0,1,2,3}", "{#nl}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site_scope, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(site_heard_at, packet_hash) <"]}, {scope: "CTE page", relation: packet_sites, alias: x, index: packet_sites_pkey, nodes: ["Index Only Scan"], index_cond: ["packet_hash = ps.packet_hash"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 5642, planning_ms: 10} + note: "measured: full 3761 / 3774 B (custom / generic), page CTE 2030" + - id: ps.f.S_AMS_nl_first + stmt: ListPacketsBySitesScopedFirst + shape: "fixer: S AMS #nl first" + params: ["{AMS}", infinity, "\\x", 51, "-infinity", infinity, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}", "{#nl}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site_scope, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 3068, planning_ms: 10} + note: "measured: full 2045 / 2045 B (custom / generic), page CTE 7" + - id: ps.f.S_AMS_nl_2_5d + stmt: ListPacketsBySitesScopedFirst + shape: "fixer: S AMS #nl 2.5d" + params: ["{AMS}", "2026-10-16 05:00+00", "\\x", 51, "-infinity", infinity, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}", "{#nl}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site_scope, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 2582, planning_ms: 10} + note: "measured: full 1721 / 1721 B (custom / generic), page CTE 8" + - id: ps.f.S_SEA_GRR_0_rows + stmt: ListPacketsBySitesScopedFirst + shape: "fixer: S SEA #GRR (0 rows)" + params: ["{SEA}", infinity, "\\x", 51, "-infinity", infinity, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}", "{#GRR}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site_scope, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 50, planning_ms: 10} + note: "measured: full 6 / 6 B (custom / generic), page CTE 6" + - id: ps.f.S_SEA_dash8_pt4_1d + stmt: ListPacketsBySitesScopedFirst + shape: "fixer: S SEA dash8 pt4 1d" + params: ["{SEA}", "2026-10-17 17:00+00", "\\x", 51, "-infinity", infinity, "{4}", "{0,1,2,3}", {r: ps_list_dash8}] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site_scope, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 1250, planning_ms: 10} + note: "measured: full 833 / 833 B (custom / generic), page CTE 48" + - id: ps.f.S_pnw_sea41_first + stmt: ListPacketsBySitesScopedFirst + shape: "fixer: S pnw sea41 first" + params: [{r: ps_list_pnw}, infinity, "\\x", 51, "-infinity", infinity, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}", {r: ps_list_scopes_sea41}] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site_scope, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 4025, planning_ms: 10} + note: "measured: full 2683 / 2781 B (custom / generic), page CTE 1626" + - id: ps.f.S_pnw_sea41_10_17_00_00 + stmt: ListPacketsBySitesScoped + shape: "fixer: S pnw sea41 10-17 00:00" + params: [{r: ps_list_pnw}, "2026-10-17 00:00+00", "\\x", 51, "-infinity", infinity, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}", {r: ps_list_scopes_sea41}] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site_scope, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(site_heard_at, packet_hash) <"]}, {scope: "CTE page", relation: packet_sites, alias: x, index: packet_sites_pkey, nodes: ["Index Only Scan"], index_cond: ["packet_hash = ps.packet_hash"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 7767, planning_ms: 10} + note: "measured: full 5178 / 5178 B (custom / generic), page CTE 4188" + - id: ps.v.bogus5000_first + stmt: ListPacketsBySitesFirst + shape: "5,000 bogus IATAs, page 1" + params: [{r: ps_bogus5000}, infinity, "\\x", 51] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 25000, planning_ms: 60} + note: "no matching measured run" + - id: ps.v.bogus5000_1d + stmt: ListPacketsBySites + shape: "5,000 bogus IATAs, legacy cursor 1 d" + params: [{r: ps_bogus5000}, "2026-10-17 17:00:00+00", "\\x", 51] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}, {scope: "CTE page", relation: packet_sites, alias: x, index: packet_sites_pkey, nodes: ["Index Only Scan"], index_cond: ["packet_hash = ps.packet_hash"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 25000, planning_ms: 60} + note: "no matching measured run; budget corrected from the end-state measurement (was 25000)" + - id: ps.v.bogus5000_typed_pt4 + stmt: ListPacketsBySitesTypedFirst + shape: "5,000 bogus IATAs, payloadTypes=4" + params: [{r: ps_bogus5000}, "{4}", "{0,1,2,3}", infinity, "\\x", 51] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site_type, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["payload_type = pt.t", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 25000, planning_ms: 60} + known_issue: "the page statements must be pre-filtered unknown IATAs (FilterKnownSites): unfiltered, 5,000 bogus IATAs x types cost 4x today" + note: "no matching measured run" + - id: ps.v.bogus5000_since1d + stmt: ListPacketsBySitesSinceFirst + shape: "5,000 bogus IATAs, since T0-1d" + params: [{r: ps_bogus5000}, infinity, "\\x", 51, "2026-10-17 17:00:00+00", infinity, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 25000, planning_ms: 60} + note: "no matching measured run" + - id: ps.v.bogus5000_scoped + stmt: ListPacketsBySitesScopedFirst + shape: "5,000 bogus IATAs x dash8 scopes" + params: [{r: ps_bogus5000}, infinity, "\\x", 51, "-infinity", infinity, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}", {r: ps_list_dash8}] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site_scope, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 25000, planning_ms: 60} + known_issue: "the page statements must be pre-filtered unknown IATAs (FilterKnownSites): unfiltered, 5,000 bogus IATAs x scopes cost 6x today" + note: "no matching measured run" + - id: ps.v.prefilter_bogus5000 + stmt: FilterKnownSites + shape: "pre-filter on 5,000 bogus IATAs (0 survive: Go returns an empty page, no page statement)" + params: [{r: ps_bogus5000}] + expect: + driver: [{relation: packet_sites, alias: ps, nodes: ["Index Only Scan"], index_cond: ["iata ="]}] + sort_exempt: [sites_recheck] + budget: {buffers: 30005, planning_ms: 60} + note: "no matching measured run; budget corrected from the end-state measurement (was 30005)" + - id: ps.v.prefilter_all561 + stmt: FilterKnownSites + shape: "pre-filter on all 561 sites (561 survive)" + params: [{r: ps_all_sites}] + expect: + driver: [{relation: packet_sites, alias: ps, nodes: ["Index Only Scan"], index_cond: ["iata ="]}] + sort_exempt: [sites_recheck] + budget: {buffers: 3599, planning_ms: 10} + note: "no matching measured run; budget corrected from the end-state measurement (was 30005)" + - id: ps.v.all561_typed_pt4 + stmt: ListPacketsBySitesTypedFirst + shape: "all 561 sites, payloadTypes=4, page 1" + params: [{r: ps_all_sites}, "{4}", "{0,1,2,3}", infinity, "\\x", 51] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site_type, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["payload_type = pt.t", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 60000, planning_ms: 10} + note: "no matching measured run" + - id: ps.v.all561_typed_pt4_1d + stmt: ListPacketsBySitesTyped + shape: "all 561 sites, payloadTypes=4, cursor 1 d" + params: [{r: ps_all_sites}, "{4}", "{0,1,2,3}", "2026-10-17 17:00:00+00", "\\x", 51] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site_type, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["payload_type = pt.t", "ROW(site_heard_at, packet_hash) <"]}, {scope: "CTE page", relation: packet_sites, alias: x, index: packet_sites_pkey, nodes: ["Index Only Scan"], index_cond: ["packet_hash = ps.packet_hash"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 281606, planning_ms: 10} + note: "no matching measured run; budget corrected from the end-state measurement (was 140000)" + - id: ps.v.all561_since75 + stmt: ListPacketsBySitesSinceFirst + shape: "all 561 sites, 75 min window, limit 200" + params: [{r: ps_all_sites}, infinity, "\\x", 201, "2026-10-18 14:00:00+00", "2026-10-18 15:15:00+00", "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 60000, planning_ms: 10} + note: "no matching measured run" + - id: ps.v.all561_since_cur + stmt: ListPacketsBySitesSince + shape: "all 561 sites, since T0-1d, cursor 10-18 06:00" + params: [{r: ps_all_sites}, "2026-10-18 06:00:00+00", "\\x", 51, "2026-10-17 17:00:00+00", infinity, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}, {scope: "CTE page", relation: packet_sites, alias: x, index: packet_sites_pkey, nodes: ["Index Only Scan"], index_cond: ["packet_hash = ps.packet_hash"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 140000, planning_ms: 10} + note: "no matching measured run" + - id: ps.v.all561_future + stmt: ListPacketsBySitesSinceFirst + shape: "all 561 sites, future since (0 rows: walks every range; documented degrade)" + params: [{r: ps_all_sites}, infinity, "\\x", 51, "2026-10-18 18:00:00+00", infinity, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 140000, planning_ms: 10} + note: "no matching measured run" + - id: ps.v.all561_scoped_dash8 + stmt: ListPacketsBySitesScopedFirst + shape: "all 561 sites x dash8 scopes" + params: [{r: ps_all_sites}, infinity, "\\x", 51, "-infinity", infinity, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}", {r: ps_list_dash8}] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site_scope, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 60000, planning_ms: 10} + note: "no matching measured run" + - id: ps.v.new_first + stmt: ListPacketsBySite + shape: "unanalyzed region NEW (21,255 rows), page 1" + params: [NEW, infinity, "\\x", 51] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 3000, planning_ms: 10} + note: "no matching measured run" + - id: ps.v.new_1d + stmt: ListPacketsBySite + shape: "unanalyzed region, legacy cursor 1 d" + params: [NEW, "2026-10-17 17:00:00+00", "\\x", 51] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 3000, planning_ms: 10} + note: "no matching measured run" + - id: ps.v.new_sea_1d + stmt: ListPacketsBySites + shape: "unanalyzed + dense region, cursor 1 d" + params: ["{NEW,SEA}", "2026-10-17 17:00:00+00", "\\x", 51] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}, {scope: "CTE page", relation: packet_sites, alias: x, index: packet_sites_pkey, nodes: ["Index Only Scan"], index_cond: ["packet_hash = ps.packet_hash"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 15000, planning_ms: 10} + note: "no matching measured run" + - id: ps.v.new_typed_pt4 + stmt: ListPacketsBySitesTypedFirst + shape: "unanalyzed region, payloadTypes=4" + params: ["{NEW}", "{4}", "{0,1,2,3}", infinity, "\\x", 51] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site_type, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["payload_type = pt.t", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 30000, planning_ms: 10} + note: "no matching measured run" + - id: ps.v.new_since1d + stmt: ListPacketsBySitesSinceFirst + shape: "unanalyzed region, since T0-1d" + params: ["{NEW}", infinity, "\\x", 51, "2026-10-17 17:00:00+00", infinity, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 15000, planning_ms: 10} + note: "no matching measured run" + - id: ps.v.new_scoped + stmt: ListPacketsBySitesScopedFirst + shape: "unanalyzed region x dash8 scopes" + params: ["{NEW}", infinity, "\\x", 51, "-infinity", infinity, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}", {r: ps_list_dash8}] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site_scope, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 15000, planning_ms: 10} + note: "no matching measured run" + - id: ps.v.sea_tie + stmt: ListPacketsBySite + shape: "dense region, pageCursor inside a tied site_heard_at" + params: [SEA, {r: ps_sea_tie_ts}, {r: ps_sea_tie_hash}, 51] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 3000, planning_ms: 10} + note: "no matching measured run" + - id: ps.v.pnw_tie + stmt: ListPacketsBySites + shape: "pnw 13, pageCursor inside a tied site_heard_at, limit 100" + params: ["{BLI,CVO,KLS,LMT,MFR,MMV,ONP,PAE,PDX,RDM,SEA,SLE,TTD}", {r: ps_pnw_tie_ts}, {r: ps_pnw_tie_hash}, 101] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}, {scope: "CTE page", relation: packet_sites, alias: x, index: packet_sites_pkey, nodes: ["Index Only Scan"], index_cond: ["packet_hash = ps.packet_hash"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 15000, planning_ms: 10} + note: "no matching measured run" + - id: ps.v.pnw_tie_since + stmt: ListPacketsBySitesSince + shape: "pnw 13, since T0-1d, pageCursor in a tie" + params: ["{BLI,CVO,KLS,LMT,MFR,MMV,ONP,PAE,PDX,RDM,SEA,SLE,TTD}", {r: ps_pnw_tie_ts}, {r: ps_pnw_tie_hash}, 51, "2026-10-17 17:00:00+00", infinity, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}, {scope: "CTE page", relation: packet_sites, alias: x, index: packet_sites_pkey, nodes: ["Index Only Scan"], index_cond: ["packet_hash = ps.packet_hash"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 15000, planning_ms: 10} + note: "no matching measured run" + - id: ps.v.pnw_edge + stmt: ListPacketsBySites + shape: "pnw 13, cursor at the retention edge (10-15 17:30), limit 100" + params: ["{BLI,CVO,KLS,LMT,MFR,MMV,ONP,PAE,PDX,RDM,SEA,SLE,TTD}", "2026-10-15 17:30:00+00", "\\x", 101] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}, {scope: "CTE page", relation: packet_sites, alias: x, index: packet_sites_pkey, nodes: ["Index Only Scan"], index_cond: ["packet_hash = ps.packet_hash"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 15000, planning_ms: 10} + note: "no matching measured run" + - id: ps.v.sea_edge + stmt: ListPacketsBySite + shape: "dense region, cursor at the retention edge" + params: [SEA, "2026-10-15 17:30:00+00", "\\x", 51] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 3000, planning_ms: 10} + note: "no matching measured run" + - id: ps.v.mad_sparse + stmt: ListPacketsBySite + shape: "sparse region (MAD), page 1" + params: [MAD, infinity, "\\x", 51] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 3000, planning_ms: 10} + note: "no matching measured run" + - id: ps.v.zzz_sites + stmt: ListPacketsBySitesFirst + shape: "two empty regions" + params: ["{ZZZ,QQQ}", infinity, "\\x", 51] + expect: + driver: [{scope: "CTE page", relation: packet_sites, alias: ps, index: idx_packet_sites_site, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["iata =", "ROW(site_heard_at, packet_hash) <"]}] + sort_exempt: [sites_recheck] + budget: {buffers: 6000, planning_ms: 10} + note: "no matching measured run" + - id: ps.maint.sweeper + stmt: SweepPacketSitesFronts + shape: "front sweeper, lookback 2 minutes; run with enable_bitmapscan = off, enable_seqscan = off via PGOPTIONS" + params: ["2 minutes"] + expect: + driver: [{relation: packet_sites, alias: p, nodes: ["Index Scan", "Index Only Scan"]}] + require: [{node: "Index Scan", relations: [idx_packet_sites_site]}, {node: "Index Only Scan", relations: [idx_packet_sites_site_type]}, {node: "Index Only Scan", relations: [idx_packet_sites_site_scope]}] + forbid: [{node: "Bitmap Heap Scan", reason: "a bitmap plan sets no LP_DEAD hints"}] + budget: {buffers: 20000, planning_ms: 20} + note: "maint pool; the copy is static at T0, so now() - 2 min finds no rows: this pins the plan shape, not the cost" + - id: ps.ingest.insert_observation + stmt: InsertObservation_end + shape: "InsertObservation with packet_sites maintenance, the newest stored observation re-sent (plain EXPLAIN)" + params: [{r: obs_newest_hash}, {r: obs_newest_observer}, BDL, "2026-10-18T16:59:59.77791+00:00", "64", "2", "0", "\\x", "0", "0", "0", "910.525", "7", "62.5", "5", broker-1, "1", "305.664"] + expect: + driver: [{relation: packets, index: packets_pkey, index_cond: ["packet_hash ="]}] + + # ---- global packet pages (pending) ---- + - id: pk.e_at_tied + stmt: ListPacketsPage + shape: "[pt - rt - scopes 0 since - until - cursor_ms 1792342797721 limit 50]" + params: ["2026-10-18 16:59:57.721+00", "\\x", 51] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_last_heard_hash, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 2090, planning_ms: 10} + note: "no measured case; page-CTE budget 50" + - id: pk.e_c1 + stmt: ListPacketsPage + shape: "cursor 1ms [pt - rt - scopes 0 since - until - cursor_ms 1 limit 50]" + params: ["1970-01-01 00:00:00.001+00", "\\x", 51] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_last_heard_hash, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 2090, planning_ms: 10} + note: "no measured case; page-CTE budget 50" + - id: pk.e_c9999 + stmt: ListPacketsPage + shape: "cursor year 9999 [pt - rt - scopes 0 since - until - cursor_ms 253402300799999 limit 50]" + params: ["9999-12-31 23:59:59.999008+00", "\\x", 51] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_last_heard_hash, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 2090, planning_ms: 10} + note: "no measured case; page-CTE budget 50" + - id: pk.e_cneg + stmt: ListPacketsPage + shape: "cursor<0 [pt - rt - scopes 0 since - until - cursor_ms -5 limit 50]" + params: [infinity, "\\x", 51] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_last_heard_hash, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 2090, planning_ms: 10} + note: "no measured case; page-CTE budget 50" + - id: pk.e_deep + stmt: ListPacketsPage + shape: "[pt - rt - scopes 0 since - until - cursor_ms 1792170000000 limit 50]" + params: ["2026-10-16 17:00:00+00", "\\x", 51] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_last_heard_hash, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 2090, planning_ms: 10} + note: "measured page CTE 4 / 4 B (custom / generic), access Index Only Scan:idx_packets_last_heard_hash; page-CTE budget 50" + - id: pk.e_deep200 + stmt: ListPacketsPage + shape: "[pt - rt - scopes 0 since - until - cursor_ms 1792170000000 limit 200]" + params: ["2026-10-16 17:00:00+00", "\\x", 201] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_last_heard_hash, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 8090, planning_ms: 10} + note: "no measured case; page-CTE budget 50" + - id: pk.e_l1 + stmt: ListPacketsPage + shape: "[pt - rt - scopes 0 since - until - cursor_ms 0 limit 1]" + params: [infinity, "\\x", 2] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_last_heard_hash, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 130, planning_ms: 10} + note: "no measured case; page-CTE budget 50" + - id: pk.e_l200 + stmt: ListPacketsPage + shape: "[pt - rt - scopes 0 since - until - cursor_ms 0 limit 200]" + params: [infinity, "\\x", 201] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_last_heard_hash, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 8090, planning_ms: 10} + note: "measured page CTE 4 / 4 B (custom / generic), access Index Only Scan:idx_packets_last_heard_hash; page-CTE budget 50" + - id: pk.e_l50 + stmt: ListPacketsPage + shape: "[pt - rt - scopes 0 since - until - cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_last_heard_hash, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 2090, planning_ms: 10} + note: "measured page CTE 4 / 4 B (custom / generic), access Index Only Scan:idx_packets_last_heard_hash; page-CTE budget 50" + - id: pk.e_tail + stmt: ListPacketsPage + shape: "tail [pt - rt - scopes 0 since - until - cursor_ms 1792083600050 limit 50]" + params: ["2026-10-15 17:00:00.05+00", "\\x", 51] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_last_heard_hash, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 2090, planning_ms: 10} + note: "no measured case; page-CTE budget 50" + - id: pk.e_tiedms_l1 + stmt: ListPacketsPage + shape: "tied ms on top [pt - rt - scopes 0 since - until - cursor_ms 1792342797722 limit 1]" + params: ["2026-10-18 16:59:57.722+00", "\\x", 2] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_last_heard_hash, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 130, planning_ms: 10} + note: "no measured case; page-CTE budget 50" + - id: pk.e_tiedms_l2 + stmt: ListPacketsPage + shape: "[pt - rt - scopes 0 since - until - cursor_ms 1792342797722 limit 2]" + params: ["2026-10-18 16:59:57.722+00", "\\x", 3] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_last_heard_hash, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 170, planning_ms: 10} + note: "no measured case; page-CTE budget 50" + - id: pk.e_ustie_l1 + stmt: ListPacketsPage + shape: "us tie [pt - rt - scopes 0 since - until - cursor_ms 1792341411653 limit 1]" + params: ["2026-10-18 16:36:51.653+00", "\\x", 2] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_last_heard_hash, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 130, planning_ms: 10} + note: "no measured case; page-CTE budget 50" + - id: pk.e_ustie_l2 + stmt: ListPacketsPage + shape: "[pt - rt - scopes 0 since - until - cursor_ms 1792341411653 limit 2]" + params: ["2026-10-18 16:36:51.653+00", "\\x", 3] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_last_heard_hash, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 170, planning_ms: 10} + note: "no measured case; page-CTE budget 50" + - id: pk.inv_all_5m + stmt: ListPacketsWindow + shape: "inverted pt11 rt2 [pt - rt - scopes 0 since 2026-10-18 10:19:12.796+00 until 2026-10-18 10:24:12.796+00 cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}", "2026-10-18 10:19:12.796+00", "2026-10-18 10:24:12.796+00"] + expect: + driver: [{scope: "CTE page", relation: packets, nodes: ["Index Only Scan", "Index Scan", "Bitmap Heap Scan"], index_cond: ["last_heard_at >="]}] + require: [{node: "Index Only Scan", relations: [idx_packets_heard_inverted]}] + sort_exempt: [window_branches] + budget: {buffers: 451638, planning_ms: 10} + note: "measured at server defaults page CTE 299732 / 299732 B (custom / generic), access IOS:idx_packets_heard_inverted, IS:idx_packets_last_heard_hash; page-CTE budget 449598" + - id: pk.inv_pt1_10m + stmt: ListPacketsWindow + shape: "inverted pt1 rt2 [pt {1} rt - scopes 0 since 2026-10-18 05:59:27.384+00 until 2026-10-18 06:09:27.384+00 cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, "{1}", "{0,1,2,3}", "2026-10-18 05:59:27.384+00", "2026-10-18 06:09:27.384+00"] + expect: + driver: [{scope: "CTE page", relation: packets, nodes: ["Index Only Scan", "Index Scan", "Bitmap Heap Scan"], index_cond: ["last_heard_at >="]}] + require: [{node: "Index Only Scan", relations: [idx_packets_heard_inverted]}] + sort_exempt: [window_branches] + budget: {buffers: 724541, planning_ms: 10} + note: "measured at server defaults page CTE 481667 / 481285 B (custom / generic), access IOS:idx_packets_heard_inverted, IS:idx_packets_last_heard_hash; page-CTE budget 722501" + - id: pk.inv_pt4rt2_1h + stmt: ListPacketsWindow + shape: "inverted pt4 rt2, deep [pt {4} rt {2} scopes 0 since 2026-10-16 20:36:52.971+00 until 2026-10-16 21:36:52.971+00 cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, "{4}", "{2}", "2026-10-16 20:36:52.971+00", "2026-10-16 21:36:52.971+00"] + expect: + driver: [{scope: "CTE page", relation: packets, nodes: ["Index Only Scan", "Index Scan", "Bitmap Heap Scan"], index_cond: ["last_heard_at >="]}] + require: [{node: "Index Only Scan", relations: [idx_packets_heard_inverted]}] + sort_exempt: [window_branches] + budget: {buffers: 2802257, planning_ms: 10} + note: "measured at server defaults page CTE 1866811 / 1866811 B (custom / generic), access IOS:idx_packets_heard_inverted, IS:idx_packets_last_heard_hash; page-CTE budget 2800217" + - id: pk.inv_rt0_32m + stmt: ListPacketsWindow + shape: "inverted pt5 rt0 row k="]}] + require: [{node: "Index Only Scan", relations: [idx_packets_heard_inverted]}] + sort_exempt: [window_branches] + budget: {buffers: 1525854, planning_ms: 10} + note: "measured at server defaults page CTE 1011876 / 950986 B (custom / generic), access IOS:idx_packets_heard_inverted, IS:idx_packets_last_heard_hash; page-CTE budget 1517814" + - id: pk.inv_rt0_cur + stmt: ListPacketsWindow + shape: "cursor = since: only the inverted row [pt - rt {0} scopes 0 since 2026-10-17 17:23:53.466+00 until 2026-10-17 17:55:53.466+00 cursor_ms 1792257833466 limit 200]" + params: ["2026-10-17 17:23:53.466+00", "\\x", 201, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0}", "2026-10-17 17:23:53.466+00", "2026-10-17 17:55:53.466+00"] + expect: + driver: [{scope: "CTE page", relation: packets, nodes: ["Index Only Scan", "Index Scan", "Bitmap Heap Scan"], index_cond: ["last_heard_at >="]}] + require: [{node: "Index Only Scan", relations: [idx_packets_heard_inverted]}] + sort_exempt: [window_branches] + budget: {buffers: 8090, planning_ms: 10} + note: "measured at server defaults page CTE 5 / 5 B (custom / generic), access IOS:idx_packets_heard_inverted, IS:idx_packets_last_heard_hash; page-CTE budget 50" + - id: pk.inv_scoped_none + stmt: ListPacketsWindowScoped + shape: "scoped window at an inverted ms (row has no scope) [pt - rt - scopes 1 since 2026-10-17 06:10:14.61+00 until 2026-10-17 07:10:14.61+00 cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}", {sql: "SELECT array_agg(DISTINCT v) FROM unnest('{#nl}'::text[]) v"}, "2026-10-17 06:10:14.61+00", "2026-10-17 07:10:14.61+00"] + expect: + driver: [{scope: "CTE page", relation: packets, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id ="]}] + require: [{node: "Index Only Scan", relations: [idx_packets_heard_inverted]}] + sort_exempt: [window_branches] + budget: {buffers: 2186, planning_ms: 10} + note: "measured at server defaults page CTE 97 / 97 B (custom / generic), access IOS:idx_packets_heard_inverted, IOS:idx_packets_scope_last_heard; page-CTE budget 146" + - id: pk.inv_since_only + stmt: ListPacketsWindow + shape: "inverted pt11 rt2, since only [pt {11} rt - scopes 0 since 2026-10-17 06:58:12.751+00 until - cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, "{11}", "{0,1,2,3}", "2026-10-17 06:58:12.751+00", infinity] + expect: + driver: [{scope: "CTE page", relation: packets, nodes: ["Index Only Scan", "Index Scan", "Bitmap Heap Scan"], index_cond: ["last_heard_at >="]}] + require: [{node: "Index Only Scan", relations: [idx_packets_heard_inverted]}] + sort_exempt: [window_branches] + budget: {buffers: 2364, planning_ms: 10} + note: "measured at server defaults page CTE 216 / 216 B (custom / generic), access IOS:idx_packets_heard_inverted, IS:idx_packets_last_heard_hash; page-CTE budget 324" + - id: pk.s_all + stmt: ListPacketsByScopes + shape: "[pt - rt - scopes 1491 since - until - cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, {sql: "SELECT array_agg(DISTINCT name) FROM transport_scopes"}] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_scope_last_heard, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 27040, planning_ms: 10} + note: "measured page CTE 4572 / 7554 B (custom / generic), access Index Only Scan:idx_packets_scope_last_heard; page-CTE budget 25000" + - id: pk.s_all_pt14 + stmt: ListPacketsByScopesTypes + shape: "rare type all scopes [pt {14} rt - scopes 1491 since - until - cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, {sql: "SELECT array_agg(DISTINCT name) FROM transport_scopes"}, "{14}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_scope_last_heard, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 27040, planning_ms: 10} + note: "measured page CTE 5935 / 8917 B (custom / generic), access Index Only Scan:idx_packets_scope_last_heard; page-CTE budget 25000" + - id: pk.s_all_pt4 + stmt: ListPacketsByScopesTypes + shape: "[pt {4} rt - scopes 1491 since - until - cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, {sql: "SELECT array_agg(DISTINCT name) FROM transport_scopes"}, "{4}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_scope_last_heard, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 27040, planning_ms: 10} + note: "measured page CTE 4944 / 7926 B (custom / generic), access Index Only Scan:idx_packets_scope_last_heard; page-CTE budget 25000" + - id: pk.s_all_pt9 + stmt: ListPacketsByScopesTypes + shape: "[pt {9} rt - scopes 1491 since - until - cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, {sql: "SELECT array_agg(DISTINCT name) FROM transport_scopes"}, "{9}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_scope_last_heard, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 27040, planning_ms: 10} + note: "measured page CTE 5935 / 8917 B (custom / generic), access Index Only Scan:idx_packets_scope_last_heard; page-CTE budget 25000" + - id: pk.s_all_rt3 + stmt: ListPacketsByScopesTypes + shape: "[pt - rt {3} scopes 1491 since - until - cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, {sql: "SELECT array_agg(DISTINCT name) FROM transport_scopes"}, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{3}"] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_scope_last_heard, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 27040, planning_ms: 10} + note: "measured page CTE 5935 / 8917 B (custom / generic), access Index Only Scan:idx_packets_scope_last_heard; page-CTE budget 25000" + - id: pk.s_big0 + stmt: ListPacketsByScopes + shape: "real 1258 list [pt - rt - scopes 1258 since - until - cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, {sql: "SELECT array_agg(DISTINCT name) FROM (SELECT name FROM transport_scopes ORDER BY name LIMIT 1258) s"}] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_scope_last_heard, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 27040, planning_ms: 10} + note: "measured page CTE 3866 / 6380 B (custom / generic), access Index Only Scan:idx_packets_scope_last_heard; page-CTE budget 25000" + - id: pk.s_big0_pt12rt2 + stmt: ListPacketsByScopesTypes + shape: "[pt {12} rt {2} scopes 1258 since - until - cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, {sql: "SELECT array_agg(DISTINCT name) FROM (SELECT name FROM transport_scopes ORDER BY name LIMIT 1258) s"}, "{12}", "{2}"] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_scope_last_heard, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 27040, planning_ms: 10} + note: "measured page CTE 8611 / 7724 B (custom / generic), access Index Only Scan:idx_packets_scope_last_heard; page-CTE budget 25000" + - id: pk.s_big0_pt14 + stmt: ListPacketsByScopesTypes + shape: "real list + rare type [pt {14} rt - scopes 1258 since - until - cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, {sql: "SELECT array_agg(DISTINCT name) FROM (SELECT name FROM transport_scopes ORDER BY name LIMIT 1258) s"}, "{14}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_scope_last_heard, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 27040, planning_ms: 10} + note: "measured page CTE 8611 / 7724 B (custom / generic), access Index Only Scan:idx_packets_scope_last_heard; page-CTE budget 25000" + - id: pk.s_big1 + stmt: ListPacketsByScopes + shape: "real 1365 list [pt - rt - scopes 1365 since - until - cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, {sql: "SELECT array_agg(DISTINCT name) FROM (SELECT name FROM transport_scopes ORDER BY name LIMIT 1365) s"}] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_scope_last_heard, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 27040, planning_ms: 10} + note: "measured page CTE 4190 / 6919 B (custom / generic), access Index Only Scan:idx_packets_scope_last_heard; page-CTE budget 25000" + - id: pk.s_case + stmt: ListPacketsByScopes + shape: "[pt - rt - scopes 3 since - until - cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, {sql: "SELECT array_agg(DISTINCT v) FROM unnest('{#PNW,#SEA,#sea}'::text[]) v"}] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_scope_last_heard, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 7040, planning_ms: 10} + note: "no measured case; page-CTE budget 5000" + - id: pk.s_dash + stmt: ListPacketsByScopes + shape: "[pt - rt - scopes 8 since - until - cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, {sql: "SELECT array_agg(DISTINCT v) FROM unnest('{#pnw,#wa,#w-wa,#thurston,#pierce,#oly,#rainier,#tumwater}'::text[]) v"}] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_scope_last_heard, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 7040, planning_ms: 10} + note: "no measured case; page-CTE budget 5000" + - id: pk.s_dash_pt14 + stmt: ListPacketsByScopesTypes + shape: "[pt {14} rt - scopes 8 since - until - cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, {sql: "SELECT array_agg(DISTINCT v) FROM unnest('{#pnw,#wa,#w-wa,#thurston,#pierce,#oly,#rainier,#tumwater}'::text[]) v"}, "{14}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_scope_last_heard, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 7040, planning_ms: 10} + note: "measured page CTE 52 / 50 B (custom / generic), access Index Only Scan:idx_packets_scope_last_heard; page-CTE budget 5000" + - id: pk.s_dash_pt4 + stmt: ListPacketsByScopesTypes + shape: "real dashboard [pt {4} rt - scopes 8 since - until - cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, {sql: "SELECT array_agg(DISTINCT v) FROM unnest('{#pnw,#wa,#w-wa,#thurston,#pierce,#oly,#rainier,#tumwater}'::text[]) v"}, "{4}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_scope_last_heard, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 7040, planning_ms: 10} + note: "measured page CTE 54 / 49 B (custom / generic), access Index Only Scan:idx_packets_scope_last_heard; page-CTE budget 5000" + - id: pk.s_dash_pt4_cur + stmt: ListPacketsByScopesTypes + shape: "[pt {4} rt - scopes 8 since - until - cursor_ms 1792256400000 limit 50]" + params: ["2026-10-17 17:00:00+00", "\\x", 51, {sql: "SELECT array_agg(DISTINCT v) FROM unnest('{#pnw,#wa,#w-wa,#thurston,#pierce,#oly,#rainier,#tumwater}'::text[]) v"}, "{4}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_scope_last_heard, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 7040, planning_ms: 10} + note: "measured page CTE 50 / 48 B (custom / generic), access Index Only Scan:idx_packets_scope_last_heard; page-CTE budget 5000" + - id: pk.s_dash_rt1 + stmt: ListPacketsByScopesTypes + shape: "[pt - rt {1} scopes 8 since - until - cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, {sql: "SELECT array_agg(DISTINCT v) FROM unnest('{#pnw,#wa,#w-wa,#thurston,#pierce,#oly,#rainier,#tumwater}'::text[]) v"}, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{1}"] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_scope_last_heard, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 7040, planning_ms: 10} + note: "no measured case; page-CTE budget 5000" + - id: pk.s_dups + stmt: ListPacketsByScopes + shape: "[pt - rt - scopes 5 since - until - cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, {sql: "SELECT array_agg(DISTINCT v) FROM unnest('{#pnw,#pnw,#wa,#wa,#wa}'::text[]) v"}] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_scope_last_heard, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 7040, planning_ms: 10} + note: "no measured case; page-CTE budget 5000" + - id: pk.s_emptyslice + stmt: ListPacketsPage + shape: "scopes=, (empty slice) [pt - rt - scopes 0 since - until - cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_last_heard_hash, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 2090, planning_ms: 10} + note: "no measured case; page-CTE budget 50" + - id: pk.s_grr + stmt: ListPacketsByScopes + shape: "0 packets [pt - rt - scopes 1 since - until - cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, {sql: "SELECT array_agg(DISTINCT v) FROM unnest('{#GRR}'::text[]) v"}] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_scope_last_heard, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 7040, planning_ms: 10} + note: "measured page CTE 6 / 6 B (custom / generic), access Index Only Scan:idx_packets_scope_last_heard; page-CTE budget 5000" + - id: pk.s_nl + stmt: ListPacketsByScopes + shape: "[pt - rt - scopes 1 since - until - cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, {sql: "SELECT array_agg(DISTINCT v) FROM unnest('{#nl}'::text[]) v"}] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_scope_last_heard, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 7040, planning_ms: 10} + note: "measured page CTE 7 / 7 B (custom / generic), access Index Only Scan:idx_packets_scope_last_heard; page-CTE budget 5000" + - id: pk.s_nl_deep200 + stmt: ListPacketsByScopes + shape: "[pt - rt - scopes 1 since - until - cursor_ms 1792256400000 limit 200]" + params: ["2026-10-17 17:00:00+00", "\\x", 201, {sql: "SELECT array_agg(DISTINCT v) FROM unnest('{#nl}'::text[]) v"}] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_scope_last_heard, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 13040, planning_ms: 10} + note: "no measured case; page-CTE budget 5000" + - id: pk.s_nl_l1_tie + stmt: ListPacketsByScopes + shape: "[pt - rt - scopes 1 since - until - cursor_ms 0 limit 1]" + params: [infinity, "\\x", 2, {sql: "SELECT array_agg(DISTINCT v) FROM unnest('{#nl}'::text[]) v"}] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_scope_last_heard, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 5080, planning_ms: 10} + note: "no measured case; page-CTE budget 5000" + - id: pk.s_nl_pt14 + stmt: ListPacketsByScopesTypes + shape: "[pt {14} rt - scopes 1 since - until - cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, {sql: "SELECT array_agg(DISTINCT v) FROM unnest('{#nl}'::text[]) v"}, "{14}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_scope_last_heard, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 7040, planning_ms: 10} + note: "measured page CTE 161 / 161 B (custom / generic), access Index Only Scan:idx_packets_scope_last_heard; page-CTE budget 5000" + - id: pk.s_nl_pt4rt1 + stmt: ListPacketsByScopesTypes + shape: "[pt {4} rt {1} scopes 1 since - until - cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, {sql: "SELECT array_agg(DISTINCT v) FROM unnest('{#nl}'::text[]) v"}, "{4}", "{1}"] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_scope_last_heard, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 7040, planning_ms: 10} + note: "no measured case; page-CTE budget 5000" + - id: pk.s_nl_rt3 + stmt: ListPacketsByScopesTypes + shape: "[pt - rt {3} scopes 1 since - until - cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, {sql: "SELECT array_agg(DISTINCT v) FROM unnest('{#nl}'::text[]) v"}, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{3}"] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_scope_last_heard, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 7040, planning_ms: 10} + note: "measured page CTE 161 / 161 B (custom / generic), access Index Only Scan:idx_packets_scope_last_heard; page-CTE budget 5000" + - id: pk.s_unknown + stmt: ListPacketsByScopes + shape: "[pt - rt - scopes 1 since - until - cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, {sql: "SELECT array_agg(DISTINCT v) FROM unnest('{#nope}'::text[]) v"}] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_scope_last_heard, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 7040, planning_ms: 10} + note: "measured page CTE 2 / 2 B (custom / generic), access Index Only Scan:idx_packets_scope_last_heard; page-CTE budget 5000" + - id: pk.s_ws + stmt: ListPacketsByScopes + shape: "singular scope untrimmed [pt - rt - scopes 1 since - until - cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, {sql: "SELECT array_agg(DISTINCT v) FROM unnest('{\" #pnw\"}'::text[]) v"}] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_scope_last_heard, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 7040, planning_ms: 10} + note: "no measured case; page-CTE budget 5000" + - id: pk.t_all16_rt + stmt: ListPacketsByTypes + shape: "explicit full domain [pt {0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15} rt {0,1,2,3} scopes 0 since - until - cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_type_last_heard, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["payload_type =", "ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 3040, planning_ms: 10} + note: "no measured case; page-CTE budget 1000" + - id: pk.t_dupmix + stmt: ListPacketsByTypes + shape: "dup + cursor [pt {4,4,4,13,13} rt {1,1,2} scopes 0 since - until - cursor_ms 1792324800000 limit 200]" + params: ["2026-10-18 12:00:00+00", "\\x", 201, "{4,4,4,13,13}", "{1,1,2}"] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_type_last_heard, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["payload_type =", "ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 9040, planning_ms: 10} + note: "no measured case; page-CTE budget 1000" + - id: pk.t_pt054 + stmt: ListPacketsByTypes + shape: "[pt {0,5,4} rt - scopes 0 since - until - cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, "{0,5,4}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_type_last_heard, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["payload_type =", "ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 3040, planning_ms: 10} + note: "no measured case; page-CTE budget 1000" + - id: pk.t_pt12_rt2 + stmt: ListPacketsByTypes + shape: "rare combo [pt {12} rt {2} scopes 0 since - until - cursor_ms 0 limit 200]" + params: [infinity, "\\x", 201, "{12}", "{2}"] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_type_last_heard, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["payload_type =", "ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 8340, planning_ms: 10} + note: "measured page CTE 5 / 5 B (custom / generic), access Index Only Scan:idx_packets_type_last_heard; page-CTE budget 300" + - id: pk.t_pt13_l200 + stmt: ListPacketsByTypes + shape: "[pt {13} rt - scopes 0 since - until - cursor_ms 0 limit 200]" + params: [infinity, "\\x", 201, "{13}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_type_last_heard, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["payload_type =", "ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 8340, planning_ms: 10} + note: "measured page CTE 17 / 17 B (custom / generic), access Index Only Scan:idx_packets_type_last_heard; page-CTE budget 300" + - id: pk.t_pt14 + stmt: ListPacketsByTypes + shape: "rare [pt {14} rt - scopes 0 since - until - cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, "{14}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_type_last_heard, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["payload_type =", "ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 2340, planning_ms: 10} + note: "measured page CTE 15 / 15 B (custom / generic), access Index Only Scan:idx_packets_type_last_heard; page-CTE budget 300" + - id: pk.t_pt2_rt23 + stmt: ListPacketsByTypes + shape: "[pt {2} rt {2,3} scopes 0 since - until - cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, "{2}", "{2,3}"] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_type_last_heard, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["payload_type =", "ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 2340, planning_ms: 10} + note: "no measured case; page-CTE budget 300" + - id: pk.t_pt4 + stmt: ListPacketsByTypes + shape: "[pt {4} rt - scopes 0 since - until - cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, "{4}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_type_last_heard, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["payload_type =", "ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 2340, planning_ms: 10} + note: "measured page CTE 20 / 14 B (custom / generic), access Index Only Scan:idx_packets_type_last_heard; page-CTE budget 300" + - id: pk.t_pt44 + stmt: ListPacketsByTypes + shape: "dup [pt {4,4} rt - scopes 0 since - until - cursor_ms 0 limit 200]" + params: [infinity, "\\x", 201, "{4,4}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_type_last_heard, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["payload_type =", "ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 9040, planning_ms: 10} + note: "no measured case; page-CTE budget 1000" + - id: pk.t_pt5_ustie_l1 + stmt: ListPacketsByTypes + shape: "[pt {5} rt {0,1} scopes 0 since - until - cursor_ms 1792341411653 limit 1]" + params: ["2026-10-18 16:36:51.653+00", "\\x", 2, "{5}", "{0,1}"] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_type_last_heard, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["payload_type =", "ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 380, planning_ms: 10} + note: "no measured case; page-CTE budget 300" + - id: pk.t_pt5_ustie_l2 + stmt: ListPacketsByTypes + shape: "[pt {5} rt {0,1} scopes 0 since - until - cursor_ms 1792341411653 limit 2]" + params: ["2026-10-18 16:36:51.653+00", "\\x", 3, "{5}", "{0,1}"] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_type_last_heard, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["payload_type =", "ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 420, planning_ms: 10} + note: "no measured case; page-CTE budget 300" + - id: pk.t_pt99_4 + stmt: ListPacketsByTypes + shape: "[pt {99,4} rt - scopes 0 since - until - cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, "{4}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_type_last_heard, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["payload_type =", "ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 2340, planning_ms: 10} + note: "no measured case; page-CTE budget 300" + - id: pk.t_ptempty + stmt: ListPacketsPage + shape: "payloadTypes=, (empty slice) [pt {} rt - scopes 0 since - until - cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_last_heard_hash, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 2090, planning_ms: 10} + note: "no measured case; page-CTE budget 50" + - id: pk.t_ptempty_rt3 + stmt: ListPacketsByTypes + shape: "[pt {} rt {3} scopes 0 since - until - cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{3}"] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_type_last_heard, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["payload_type =", "ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 2340, planning_ms: 10} + note: "no measured case; page-CTE budget 300" + - id: pk.t_rt00 + stmt: ListPacketsByTypes + shape: "dup [pt - rt {0,0} scopes 0 since - until - cursor_ms 0 limit 200]" + params: [infinity, "\\x", 201, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,0}"] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_type_last_heard, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["payload_type =", "ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 9040, planning_ms: 10} + note: "no measured case; page-CTE budget 1000" + - id: pk.t_rt3_l200 + stmt: ListPacketsByTypes + shape: "[pt - rt {3} scopes 0 since - until - cursor_ms 0 limit 200]" + params: [infinity, "\\x", 201, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{3}"] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_type_last_heard, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["payload_type =", "ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 8340, planning_ms: 10} + note: "measured page CTE 66 / 66 B (custom / generic), access Index Only Scan:idx_packets_type_last_heard; page-CTE budget 300" + - id: pk.t_rt37 + stmt: ListPacketsByTypes + shape: "[pt - rt {3,7} scopes 0 since - until - cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{3}"] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_type_last_heard, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["payload_type =", "ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 2340, planning_ms: 10} + note: "no measured case; page-CTE budget 300" + - id: pk.t_web13 + stmt: ListPacketsByTypes + shape: "[pt {0,1,2,3,4,5,6,7,8,9,10,11,15} rt - scopes 0 since - until - cursor_ms 1792317600000 limit 50]" + params: ["2026-10-18 10:00:00+00", "\\x", 51, "{0,1,2,3,4,5,6,7,8,9,10,11,15}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_type_last_heard, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["payload_type =", "ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 3040, planning_ms: 10} + note: "measured page CTE 175 / 175 B (custom / generic), access Index Only Scan:idx_packets_type_last_heard; page-CTE budget 1000" + - id: pk.w_epoch + stmt: ListPacketsWindow + shape: "extreme [pt - rt - scopes 0 since 1970-01-01 00:00:00+00 until 9999-12-31 23:59:59.999008+00 cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}", "1970-01-01 00:00:00+00", "9999-12-31 23:59:59.999008+00"] + expect: + driver: [{scope: "CTE page", relation: packets, nodes: ["Index Only Scan", "Index Scan", "Bitmap Heap Scan"], index_cond: ["last_heard_at >="]}] + require: [{node: "Index Only Scan", relations: [idx_packets_heard_inverted]}] + sort_exempt: [window_branches] + budget: {buffers: 2124, planning_ms: 10} + note: "measured at server defaults page CTE 56 / 56 B (custom / generic), access IOS:idx_packets_heard_inverted, IS:idx_packets_last_heard_hash; page-CTE budget 84" + - id: pk.w_eq + stmt: ListPacketsWindow + shape: "tiny [pt - rt - scopes 0 since 2026-10-18 12:00:00.021+00 until 2026-10-18 12:00:00.022+00 cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}", "2026-10-18 12:00:00.021+00", "2026-10-18 12:00:00.022+00"] + expect: + driver: [{scope: "CTE page", relation: packets, nodes: ["Index Only Scan", "Index Scan", "Bitmap Heap Scan"], index_cond: ["last_heard_at >="]}] + require: [{node: "Index Only Scan", relations: [idx_packets_heard_inverted]}] + sort_exempt: [window_branches] + budget: {buffers: 38483, planning_ms: 10} + note: "measured at server defaults page CTE 24295 / 237399 B (custom / generic), access Bitmap Heap Scan:packets, Bitmap IS:idx_packets_type_last_heard, IOS:idx_packets_heard_inverted; page-CTE budget 36443" + - id: pk.w_long + stmt: ListPacketsWindow + shape: "long-lived [pt - rt - scopes 0 since 2026-10-02 00:00:00+00 until 2026-10-12 00:00:00+00 cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}", "2026-10-02 00:00:00+00", "2026-10-12 00:00:00+00"] + expect: + driver: [{scope: "CTE page", relation: packets, nodes: ["Index Only Scan", "Index Scan", "Bitmap Heap Scan"], index_cond: ["last_heard_at >="]}] + require: [{node: "Index Only Scan", relations: [idx_packets_heard_inverted]}] + sort_exempt: [window_branches] + budget: {buffers: 128466, planning_ms: 10} + note: "measured at server defaults page CTE 84284 / 84284 B (custom / generic), access IOS:idx_packets_heard_inverted, IS:idx_packets_last_heard_hash; page-CTE budget 126426" + - id: pk.w_negsince + stmt: ListPacketsWindow + shape: "[pt - rt - scopes 0 since 1969-12-31 23:59:59+00 until - cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}", "1969-12-31 23:59:59+00", infinity] + expect: + driver: [{scope: "CTE page", relation: packets, nodes: ["Index Only Scan", "Index Scan", "Bitmap Heap Scan"], index_cond: ["last_heard_at >="]}] + require: [{node: "Index Only Scan", relations: [idx_packets_heard_inverted]}] + sort_exempt: [window_branches] + budget: {buffers: 2124, planning_ms: 10} + note: "measured at server defaults page CTE 56 / 56 B (custom / generic), access IOS:idx_packets_heard_inverted, IS:idx_packets_last_heard_hash; page-CTE budget 84" + - id: pk.w_pf_32m + stmt: ListPacketsWindow + shape: "[pt - rt {0} scopes 0 since 2026-10-18 16:28:00+00 until 2026-10-18 17:00:00+00 cursor_ms 0 limit 200]" + params: [infinity, "\\x", 201, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0}", "2026-10-18 16:28:00+00", "2026-10-18 17:00:00+00"] + expect: + driver: [{scope: "CTE page", relation: packets, nodes: ["Index Only Scan", "Index Scan", "Bitmap Heap Scan"], index_cond: ["last_heard_at >="]}] + require: [{node: "Index Only Scan", relations: [idx_packets_heard_inverted]}] + sort_exempt: [window_branches] + budget: {buffers: 12239, planning_ms: 10} + note: "measured at server defaults page CTE 2799 / 1831 B (custom / generic), access IOS:idx_packets_heard_inverted, IS:idx_packets_type_last_heard; page-CTE budget 4199" + - id: pk.w_pf_32m_real + stmt: ListPacketsWindow + shape: "pathfinder 32 min (log shape) [pt - rt {0} scopes 0 since 2026-10-18 16:28:00+00 until 2026-10-18 17:00:00+00 cursor_ms 0 limit 200]" + params: [infinity, "\\x", 201, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0}", "2026-10-18 16:28:00+00", "2026-10-18 17:00:00+00"] + expect: + driver: [{scope: "CTE page", relation: packets, nodes: ["Index Only Scan", "Index Scan", "Bitmap Heap Scan"], index_cond: ["last_heard_at >="]}] + require: [{node: "Index Only Scan", relations: [idx_packets_heard_inverted]}] + sort_exempt: [window_branches] + budget: {buffers: 12239, planning_ms: 10} + note: "measured at server defaults page CTE 2799 / 1831 B (custom / generic), access IOS:idx_packets_heard_inverted, IS:idx_packets_type_last_heard; page-CTE budget 4199" + - id: pk.w_pf_5m + stmt: ListPacketsWindow + shape: "5-min window, < 3x201 matches [pt - rt {0} scopes 0 since 2026-10-18 16:55:00+00 until 2026-10-18 17:00:00+00 cursor_ms 0 limit 200]" + params: [infinity, "\\x", 201, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0}", "2026-10-18 16:55:00+00", "2026-10-18 17:00:00+00"] + expect: + driver: [{scope: "CTE page", relation: packets, nodes: ["Index Only Scan", "Index Scan", "Bitmap Heap Scan"], index_cond: ["last_heard_at >="]}] + require: [{node: "Index Only Scan", relations: [idx_packets_heard_inverted]}] + sort_exempt: [window_branches] + budget: {buffers: 8705, planning_ms: 10} + note: "measured at server defaults page CTE 443 / 1959 B (custom / generic), access IOS:idx_packets_heard_inverted, IS:idx_packets_type_last_heard; page-CTE budget 665" + - id: pk.w_pf_5m_b + stmt: ListPacketsWindow + shape: "pathfinder 5 min mid-day [pt - rt {0} scopes 0 since 2026-10-18 12:00:00+00 until 2026-10-18 12:05:00+00 cursor_ms 0 limit 200]" + params: [infinity, "\\x", 201, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0}", "2026-10-18 12:00:00+00", "2026-10-18 12:05:00+00"] + expect: + driver: [{scope: "CTE page", relation: packets, nodes: ["Index Only Scan", "Index Scan", "Bitmap Heap Scan"], index_cond: ["last_heard_at >="]}] + require: [{node: "Index Only Scan", relations: [idx_packets_heard_inverted]}] + sort_exempt: [window_branches] + budget: {buffers: 31496, planning_ms: 10} + note: "measured at server defaults page CTE 15637 / 234030 B (custom / generic), access Bitmap Heap Scan:packets, Bitmap IS:idx_packets_type_last_heard, IOS:idx_packets_heard_inverted; page-CTE budget 23456" + - id: pk.w_pf_6h + stmt: ListPacketsWindow + shape: "pathfinder real [pt - rt {0} scopes 0 since 2026-10-18 10:30:00+00 until 2026-10-18 17:00:00+00 cursor_ms 0 limit 200]" + params: [infinity, "\\x", 201, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0}", "2026-10-18 10:30:00+00", "2026-10-18 17:00:00+00"] + expect: + driver: [{scope: "CTE page", relation: packets, nodes: ["Index Only Scan", "Index Scan", "Bitmap Heap Scan"], index_cond: ["last_heard_at >="]}] + require: [{node: "Index Only Scan", relations: [idx_packets_heard_inverted]}] + sort_exempt: [window_branches] + budget: {buffers: 10787, planning_ms: 10} + note: "measured at server defaults page CTE 1831 / 1831 B (custom / generic), access IOS:idx_packets_heard_inverted, IS:idx_packets_last_heard_hash; page-CTE budget 2747" + - id: pk.w_pf_6h_2ndlast + stmt: ListPacketsWindow + shape: "2nd-last page [pt - rt {0} scopes 0 since 2026-10-18 10:30:00+00 until 2026-10-18 17:00:00+00 cursor_ms 1792319629827 limit 200]" + params: ["2026-10-18 10:33:49.827+00", "\\x", 201, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0}", "2026-10-18 10:30:00+00", "2026-10-18 17:00:00+00"] + expect: + driver: [{scope: "CTE page", relation: packets, nodes: ["Index Only Scan", "Index Scan", "Bitmap Heap Scan"], index_cond: ["last_heard_at >="]}] + require: [{node: "Index Only Scan", relations: [idx_packets_heard_inverted]}] + sort_exempt: [window_branches] + budget: {buffers: 10758, planning_ms: 10} + note: "measured at server defaults page CTE 1812 / 1812 B (custom / generic), access IOS:idx_packets_heard_inverted, IS:idx_packets_last_heard_hash; page-CTE budget 2718" + - id: pk.w_pf_6h_cur + stmt: ListPacketsWindow + shape: "[pt - rt {0} scopes 0 since 2026-10-18 10:30:00+00 until 2026-10-18 17:00:00+00 cursor_ms 1792339200000 limit 200]" + params: ["2026-10-18 16:00:00+00", "\\x", 201, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0}", "2026-10-18 10:30:00+00", "2026-10-18 17:00:00+00"] + expect: + driver: [{scope: "CTE page", relation: packets, nodes: ["Index Only Scan", "Index Scan", "Bitmap Heap Scan"], index_cond: ["last_heard_at >="]}] + require: [{node: "Index Only Scan", relations: [idx_packets_heard_inverted]}] + sort_exempt: [window_branches] + budget: {buffers: 10391, planning_ms: 10} + note: "measured at server defaults page CTE 1567 / 1567 B (custom / generic), access IOS:idx_packets_heard_inverted, IS:idx_packets_last_heard_hash; page-CTE budget 2351" + - id: pk.w_pf_6h_last + stmt: ListPacketsWindow + shape: "last page of 6.5h chain [pt - rt {0} scopes 0 since 2026-10-18 10:30:00+00 until 2026-10-18 17:00:00+00 cursor_ms 1792319480893 limit 200]" + params: ["2026-10-18 10:31:20.893+00", "\\x", 201, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0}", "2026-10-18 10:30:00+00", "2026-10-18 17:00:00+00"] + expect: + driver: [{scope: "CTE page", relation: packets, nodes: ["Index Only Scan", "Index Scan", "Bitmap Heap Scan"], index_cond: ["last_heard_at >="]}] + require: [{node: "Index Only Scan", relations: [idx_packets_heard_inverted]}] + sort_exempt: [window_branches] + budget: {buffers: 9408, planning_ms: 10} + note: "measured at server defaults page CTE 912 / 912 B (custom / generic), access IOS:idx_packets_heard_inverted, IS:idx_packets_last_heard_hash; page-CTE budget 1368" + - id: pk.w_pt14_3d + stmt: ListPacketsWindow + shape: "rare type window [pt {14} rt - scopes 0 since 2026-10-15 17:00:00+00 until 2026-10-18 17:00:00+00 cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, "{14}", "{0,1,2,3}", "2026-10-15 17:00:00+00", "2026-10-18 17:00:00+00"] + expect: + driver: [{scope: "CTE page", relation: packets, nodes: ["Index Only Scan", "Index Scan", "Bitmap Heap Scan"], index_cond: ["last_heard_at >="]}] + require: [{node: "Index Only Scan", relations: [idx_packets_heard_inverted]}] + sort_exempt: [window_branches] + budget: {buffers: 3171, planning_ms: 10} + note: "measured at server defaults page CTE 754 / 233316 B (custom / generic), access IOS:idx_packets_heard_inverted, IS:idx_packets_type_last_heard; page-CTE budget 1131" + - id: pk.w_pt4rt1_tied + stmt: ListPacketsWindow + shape: "[pt {4} rt {1} scopes 0 since 2026-10-18 00:00:00+00 until - cursor_ms 1792342797722 limit 50]" + params: ["2026-10-18 16:59:57.722+00", "\\x", 51, "{4}", "{1}", "2026-10-18 00:00:00+00", infinity] + expect: + driver: [{scope: "CTE page", relation: packets, nodes: ["Index Only Scan", "Index Scan", "Bitmap Heap Scan"], index_cond: ["last_heard_at >="]}] + require: [{node: "Index Only Scan", relations: [idx_packets_heard_inverted]}] + sort_exempt: [window_branches] + budget: {buffers: 3192, planning_ms: 10} + note: "measured at server defaults page CTE 768 / 768 B (custom / generic), access IOS:idx_packets_heard_inverted, IS:idx_packets_last_heard_hash; page-CTE budget 1152" + - id: pk.w_since75 + stmt: ListPacketsWindow + shape: "[pt - rt - scopes 0 since 2026-10-18 15:45:00+00 until - cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}", "2026-10-18 15:45:00+00", infinity] + expect: + driver: [{scope: "CTE page", relation: packets, nodes: ["Index Only Scan", "Index Scan", "Bitmap Heap Scan"], index_cond: ["last_heard_at >="]}] + require: [{node: "Index Only Scan", relations: [idx_packets_heard_inverted]}] + sort_exempt: [window_branches] + budget: {buffers: 2126, planning_ms: 10} + note: "measured at server defaults page CTE 57 / 57 B (custom / generic), access IOS:idx_packets_heard_inverted, IS:idx_packets_last_heard_hash; page-CTE budget 86" + - id: pk.w_since75_pt4 + stmt: ListPacketsWindow + shape: "[pt {4} rt - scopes 0 since 2026-10-18 15:45:00+00 until - cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, "{4}", "{0,1,2,3}", "2026-10-18 15:45:00+00", infinity] + expect: + driver: [{scope: "CTE page", relation: packets, nodes: ["Index Only Scan", "Index Scan", "Bitmap Heap Scan"], index_cond: ["last_heard_at >="]}] + require: [{node: "Index Only Scan", relations: [idx_packets_heard_inverted]}] + sort_exempt: [window_branches] + budget: {buffers: 12818, planning_ms: 10} + note: "measured at server defaults page CTE 7185 / 369 B (custom / generic), access Bitmap Heap Scan:packets, Bitmap IS:idx_packets_type_last_heard, IOS:idx_packets_heard_inverted; page-CTE budget 10778" + - id: pk.w_until_deep + stmt: ListPacketsWindow + shape: "[pt - rt - scopes 0 since - until 2026-10-16 17:00:00+00 cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}", "-infinity", "2026-10-16 17:00:00+00"] + expect: + driver: [{scope: "CTE page", relation: packets, nodes: ["Index Only Scan", "Index Scan", "Bitmap Heap Scan"], index_cond: ["last_heard_at >="]}] + require: [{node: "Index Only Scan", relations: [idx_packets_heard_inverted]}] + sort_exempt: [window_branches] + budget: {buffers: 72077, planning_ms: 10} + note: "measured at server defaults page CTE 46691 / 46691 B (custom / generic), access IOS:idx_packets_heard_inverted, IS:idx_packets_last_heard_hash; page-CTE budget 70037" + - id: pk.w_until_only_rt0 + stmt: ListPacketsWindow + shape: "until only (zero traffic) [pt - rt {0} scopes 0 since - until 2026-10-18 06:00:00+00 cursor_ms 0 limit 200]" + params: [infinity, "\\x", 201, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0}", "-infinity", "2026-10-18 06:00:00+00"] + expect: + driver: [{scope: "CTE page", relation: packets, nodes: ["Index Only Scan", "Index Scan", "Bitmap Heap Scan"], index_cond: ["last_heard_at >="]}] + require: [{node: "Index Only Scan", relations: [idx_packets_heard_inverted]}] + sort_exempt: [window_branches] + budget: {buffers: 703970, planning_ms: 10} + note: "measured at server defaults page CTE 463953 / 463953 B (custom / generic), access IOS:idx_packets_heard_inverted, IS:idx_packets_last_heard_hash; page-CTE budget 695930" + - id: pk.ws_all_1d + stmt: ListPacketsWindowScoped + shape: "[pt - rt - scopes 1491 since 2026-10-17 17:00:00+00 until 2026-10-18 17:00:00+00 cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}", {sql: "SELECT array_agg(DISTINCT name) FROM transport_scopes"}, "2026-10-17 17:00:00+00", "2026-10-18 17:00:00+00"] + expect: + driver: [{scope: "CTE page", relation: packets, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id ="]}] + require: [{node: "Index Only Scan", relations: [idx_packets_heard_inverted]}] + sort_exempt: [window_branches] + budget: {buffers: 8864, planning_ms: 10} + note: "measured at server defaults page CTE 4549 / 7534 B (custom / generic), access IOS:idx_packets_heard_inverted, IOS:idx_packets_scope_last_heard; page-CTE budget 6824" + - id: pk.ws_all_until_deep + stmt: ListPacketsWindowScoped + shape: "[pt - rt - scopes 1491 since - until 2026-10-16 17:00:00+00 cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}", {sql: "SELECT array_agg(DISTINCT name) FROM transport_scopes"}, "-infinity", "2026-10-16 17:00:00+00"] + expect: + driver: [{scope: "CTE page", relation: packets, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id ="]}] + require: [{node: "Index Only Scan", relations: [idx_packets_heard_inverted]}] + sort_exempt: [window_branches] + budget: {buffers: 10241, planning_ms: 10} + note: "measured at server defaults page CTE 5467 / 8449 B (custom / generic), access IOS:idx_packets_heard_inverted, IOS:idx_packets_scope_last_heard; page-CTE budget 8201" + - id: pk.ws_chain_dash + stmt: ListPacketsWindowScoped + shape: "dashboard scopes + pt4 1 day [pt {4} rt - scopes 8 since 2026-10-17 17:00:00+00 until 2026-10-18 17:00:00+00 cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, "{4}", "{0,1,2,3}", {sql: "SELECT array_agg(DISTINCT v) FROM unnest('{#pnw,#wa,#w-wa,#thurston,#pierce,#oly,#rainier,#tumwater}'::text[]) v"}, "2026-10-17 17:00:00+00", "2026-10-18 17:00:00+00"] + expect: + driver: [{scope: "CTE page", relation: packets, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id ="]}] + require: [{node: "Index Only Scan", relations: [idx_packets_heard_inverted]}] + sort_exempt: [window_branches] + budget: {buffers: 2114, planning_ms: 10} + note: "measured at server defaults page CTE 49 / 49 B (custom / generic), access Bitmap IS:transport_scopes_name_key, IOS:idx_packets_heard_inverted, IOS:idx_packets_scope_last_heard; page-CTE budget 74" + - id: pk.ws_chain_nl_1h + stmt: ListPacketsWindowScoped + shape: "scoped window chain [pt - rt - scopes 1 since 2026-10-18 15:00:00+00 until 2026-10-18 16:00:00+00 cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}", {sql: "SELECT array_agg(DISTINCT v) FROM unnest('{#nl}'::text[]) v"}, "2026-10-18 15:00:00+00", "2026-10-18 16:00:00+00"] + expect: + driver: [{scope: "CTE page", relation: packets, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id ="]}] + require: [{node: "Index Only Scan", relations: [idx_packets_heard_inverted]}] + sort_exempt: [window_branches] + budget: {buffers: 2090, planning_ms: 10} + note: "measured at server defaults page CTE 11 / 11 B (custom / generic), access IOS:idx_packets_heard_inverted, IOS:idx_packets_scope_last_heard; page-CTE budget 50" + - id: pk.ws_chain_sea4d + stmt: ListPacketsWindowScoped + shape: "scoped window chain, limit 2 [pt - rt - scopes 1 since 2026-10-14 17:00:00+00 until 2026-10-18 17:00:00+00 cursor_ms 0 limit 2]" + params: [infinity, "\\x", 3, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}", {sql: "SELECT array_agg(DISTINCT v) FROM unnest('{#SEA}'::text[]) v"}, "2026-10-14 17:00:00+00", "2026-10-18 17:00:00+00"] + expect: + driver: [{scope: "CTE page", relation: packets, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id ="]}] + require: [{node: "Index Only Scan", relations: [idx_packets_heard_inverted]}] + sort_exempt: [window_branches] + budget: {buffers: 170, planning_ms: 10} + note: "measured at server defaults page CTE 9 / 9 B (custom / generic), access IOS:idx_packets_heard_inverted, IOS:idx_packets_scope_last_heard; page-CTE budget 50" + - id: pk.ws_dash_pt4_cur + stmt: ListPacketsWindowScoped + shape: "[pt {4} rt - scopes 8 since 2026-10-17 17:00:00+00 until - cursor_ms 1792303200000 limit 50]" + params: ["2026-10-18 06:00:00+00", "\\x", 51, "{4}", "{0,1,2,3}", {sql: "SELECT array_agg(DISTINCT v) FROM unnest('{#pnw,#wa,#w-wa,#thurston,#pierce,#oly,#rainier,#tumwater}'::text[]) v"}, "2026-10-17 17:00:00+00", infinity] + expect: + driver: [{scope: "CTE page", relation: packets, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id ="]}] + require: [{node: "Index Only Scan", relations: [idx_packets_heard_inverted]}] + sort_exempt: [window_branches] + budget: {buffers: 2114, planning_ms: 10} + note: "measured at server defaults page CTE 49 / 49 B (custom / generic), access Bitmap IS:transport_scopes_name_key, IOS:idx_packets_heard_inverted, IOS:idx_packets_scope_last_heard; page-CTE budget 74" + - id: pk.ws_emptyslice + stmt: ListPacketsWindow + shape: "scopes=, with window [pt - rt - scopes 0 since 2026-10-18 16:00:00+00 until - cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}", "2026-10-18 16:00:00+00", infinity] + expect: + driver: [{scope: "CTE page", relation: packets, nodes: ["Index Only Scan", "Index Scan", "Bitmap Heap Scan"], index_cond: ["last_heard_at >="]}] + require: [{node: "Index Only Scan", relations: [idx_packets_heard_inverted]}] + sort_exempt: [window_branches] + budget: {buffers: 2126, planning_ms: 10} + note: "measured at server defaults page CTE 57 / 57 B (custom / generic), access IOS:idx_packets_heard_inverted, IS:idx_packets_last_heard_hash; page-CTE budget 86" + - id: pk.ws_nl_pt4 + stmt: ListPacketsWindowScoped + shape: "[pt {4} rt - scopes 1 since 2026-10-17 17:00:00+00 until 2026-10-18 17:00:00+00 cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, "{4}", "{0,1,2,3}", {sql: "SELECT array_agg(DISTINCT v) FROM unnest('{#nl}'::text[]) v"}, "2026-10-17 17:00:00+00", "2026-10-18 17:00:00+00"] + expect: + driver: [{scope: "CTE page", relation: packets, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id ="]}] + require: [{node: "Index Only Scan", relations: [idx_packets_heard_inverted]}] + sort_exempt: [window_branches] + budget: {buffers: 2090, planning_ms: 10} + note: "measured at server defaults page CTE 18 / 18 B (custom / generic), access IOS:idx_packets_heard_inverted, IOS:idx_packets_scope_last_heard; page-CTE budget 50" + - id: pk.ws_sea_1d + stmt: ListPacketsWindowScoped + shape: "[pt - rt - scopes 1 since 2026-10-17 17:00:00+00 until 2026-10-18 17:00:00+00 cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}", {sql: "SELECT array_agg(DISTINCT v) FROM unnest('{#sea}'::text[]) v"}, "2026-10-17 17:00:00+00", "2026-10-18 17:00:00+00"] + expect: + driver: [{scope: "CTE page", relation: packets, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id ="]}] + require: [{node: "Index Only Scan", relations: [idx_packets_heard_inverted]}] + sort_exempt: [window_branches] + budget: {buffers: 2090, planning_ms: 10} + note: "measured at server defaults page CTE 9 / 9 B (custom / generic), access IOS:idx_packets_heard_inverted, IOS:idx_packets_scope_last_heard; page-CTE budget 50" + - id: pk.ws_SEA_4d + stmt: ListPacketsWindowScoped + shape: "[pt - rt - scopes 1 since 2026-10-14 17:00:00+00 until 2026-10-18 17:00:00+00 cursor_ms 0 limit 200]" + params: [infinity, "\\x", 201, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}", {sql: "SELECT array_agg(DISTINCT v) FROM unnest('{#SEA}'::text[]) v"}, "2026-10-14 17:00:00+00", "2026-10-18 17:00:00+00"] + expect: + driver: [{scope: "CTE page", relation: packets, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id ="]}] + require: [{node: "Index Only Scan", relations: [idx_packets_heard_inverted]}] + sort_exempt: [window_branches] + budget: {buffers: 8090, planning_ms: 10} + note: "measured at server defaults page CTE 10 / 10 B (custom / generic), access IOS:idx_packets_heard_inverted, IOS:idx_packets_scope_last_heard; page-CTE budget 50" + - id: pk.ws_unknown + stmt: ListPacketsWindowScoped + shape: "[pt - rt - scopes 1 since 2026-10-17 17:00:00+00 until - cursor_ms 0 limit 50]" + params: [infinity, "\\x", 51, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}", {sql: "SELECT array_agg(DISTINCT v) FROM unnest('{#nope}'::text[]) v"}, "2026-10-17 17:00:00+00", infinity] + expect: + driver: [{scope: "CTE page", relation: packets, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id ="]}] + require: [{node: "Index Only Scan", relations: [idx_packets_heard_inverted]}] + sort_exempt: [window_branches] + budget: {buffers: 2090, planning_ms: 10} + note: "measured at server defaults page CTE 4 / 4 B (custom / generic), access IOS:idx_packets_heard_inverted, IOS:idx_packets_scope_last_heard; page-CTE budget 50" + - id: pk.s_bogus5000 + stmt: ListPacketsByScopes + shape: "5,000 bogus scope names [pt - rt - scopes 5000 since - until - cursor_ms - limit 50]" + params: [infinity, "\\x", 51, {r: bogus5000}] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_scope_last_heard, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 27040, planning_ms: 60} + note: "no measured case; page-CTE budget 25000" + - id: pk.s_bogus5000_pt4 + stmt: ListPacketsByScopesTypes + shape: "5,000 bogus names + payloadTypes=4 [pt - rt - scopes 5000 since - until - cursor_ms - limit 50]" + params: [infinity, "\\x", 51, {r: bogus5000}, "{4}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_scope_last_heard, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 27040, planning_ms: 60} + note: "no measured case; page-CTE budget 25000" + - id: pk.s_allbogus6491 + stmt: ListPacketsByScopes + shape: "all 1,491 real names + 5,000 bogus [pt - rt - scopes 6491 since - until - cursor_ms - limit 50]" + params: [infinity, "\\x", 51, {r: allbogus}] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_scope_last_heard, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id =", "ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 27040, planning_ms: 60} + note: "no measured case; page-CTE budget 25000" + - id: pk.ws_bogus5000_1d + stmt: ListPacketsWindowScoped + shape: "5,000 bogus names, 1-day window [pt - rt - scopes 5000 since 2026-10-17 17:00:00+00 until 2026-10-18 17:00:00+00 cursor_ms - limit 50]" + params: [infinity, "\\x", 51, "{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}", "{0,1,2,3}", {r: bogus5000}, "2026-10-17 17:00:00+00", "2026-10-18 17:00:00+00"] + expect: + driver: [{scope: "CTE page", relation: packets, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["scope_id ="]}] + require: [{node: "Index Only Scan", relations: [idx_packets_heard_inverted]}] + sort_exempt: [window_branches] + budget: {buffers: 27040, planning_ms: 60} + note: "no measured case; page-CTE budget 25000" + - id: pk.e_pc_ustie_l1 + stmt: ListPacketsPage + shape: "pageCursor = the newest microsecond-tie pair's top row, limit 1 [pt - rt - scopes 0 since - until - cursor_ms - limit 1]" + params: [{r: ustie_ts}, {r: ustie_hash}, 2] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_last_heard_hash, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 130, planning_ms: 10} + note: "no measured case; page-CTE budget 50" + - id: pk.e_pc_ustie_l2 + stmt: ListPacketsPage + shape: "same, limit 2 [pt - rt - scopes 0 since - until - cursor_ms - limit 2]" + params: [{r: ustie_ts}, {r: ustie_hash}, 3] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_last_heard_hash, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 170, planning_ms: 10} + note: "no measured case; page-CTE budget 50" + - id: pk.t_pt4_pc_deep + stmt: ListPacketsByTypes + shape: "payloadTypes=4, pageCursor 100,000 packets deep (prototype) [pt - rt - scopes 0 since - until - cursor_ms - limit 200]" + params: [{r: pk_deep_ts}, {r: pk_deep_hash}, 201, "{4}", "{0,1,2,3}"] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_type_last_heard, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["payload_type =", "ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 8340, planning_ms: 10} + note: "no measured case; page-CTE budget 300" + - id: pk.e_pc_deep + stmt: ListPacketsPage + shape: "pageCursor 100,000 packets deep (prototype) [pt - rt - scopes 0 since - until - cursor_ms - limit 200]" + params: [{r: pk_deep_ts}, {r: pk_deep_hash}, 201] + expect: + driver: [{scope: "CTE page", relation: packets, index: idx_packets_last_heard_hash, nodes: ["Index Only Scan", "Index Scan"], index_cond: ["ROW(last_heard_at, packet_hash) <"]}] + budget: {buffers: 8090, planning_ms: 10} + note: "no measured case; page-CTE budget 50" + - id: pk.maint.delete_old_packets.victims + stmt: DeleteOldPacketsVictims + shape: "DeleteOldPackets victims, cutoff T0-3d+30min, batch 10,000 (measured 3.4: Index Scan Backward on idx_packets_last_heard_hash, no Sort)" + params: ["2026-10-15 17:30:00+00", 10000] + expect: + driver: [{relation: packets, alias: vp, index: idx_packets_last_heard_hash, index_cond: ["last_heard_at <"]}] + forbid: [{node: Sort, reason: "the (last_heard_at, packet_hash) order comes from the index"}] + budget: {buffers: 30000, planning_ms: 10} + - id: pk.maint.delete_old_packets.dml + stmt: DeleteOldPackets + shape: "the full DeleteOldPackets statement (plain EXPLAIN)" + params: ["2026-10-15 17:30:00+00", 10000] + expect: + driver: [{relation: packets, alias: vp, index: idx_packets_last_heard_hash, index_cond: ["last_heard_at <"]}] + + # ---- planted regressions: earlier statements that must be reported ---- + # 1. ListKnownRoutes as of 202acc6, judged by the end-state entry's expectations. + - id: planted1.routes.today.bwe.page1 + stmt: ListKnownRoutes_202acc6 + shape: "today's ListKnownRoutes, BWE page 1" + params: [BWE, 0, null, 200, 0] + tiers: [fixture, local, prod] + modes: [custom, generic] + executions: 1 + inherit: routes.byiata.R05_BWE_first_l200 + expect_fail: true + - id: planted1.routes.today.bwe.cursor + stmt: ListKnownRoutes_202acc6 + shape: "today's ListKnownRoutes, BWE cursor+cursorId" + params: [BWE, 0, {r: bwe_deep_ts}, 200, {r: bwe_deep_id}] + tiers: [fixture, local, prod] + modes: [custom, generic] + executions: 1 + inherit: routes.byiata.R14_BWE_deep_l200 + expect_fail: true + - id: planted1.routes.today.global.cursor + stmt: ListKnownRoutes_202acc6 + shape: "today's ListKnownRoutes, global cursor+cursorId" + params: ["", 0, {r: g50_ts}, 50, {r: g50_id}] + tiers: [fixture, local, prod] + modes: [custom, generic] + executions: 1 + inherit: routes.global.G02_page2_l50 + expect_fail: true + + # 2. DeleteOldRoutes as of 202acc6 at production-like cutoffs. + - id: planted2.retention.today.select + stmt: DeleteOldRoutesSelect + shape: "today's expired CTE (FOR UPDATE SKIP LOCKED), plain EXPLAIN" + params: ["2026-10-04 19:00:00+00", "2026-10-11 19:00:00+00", 3, 10000] + tiers: [fixture, local, prod] + modes: [custom, generic] + inherit: retention.expired.dml + expect_fail: true + - id: planted2.retention.today.select_nolock + stmt: DeleteOldRoutesSelectNoLock + shape: "today's expired CTE without the lock, EXPLAIN ANALYZE" + params: ["2026-10-04 19:00:00+00", "2026-10-11 19:00:00+00", 3, 10000] + tiers: [local] + executions: 1 + inherit: retention.expired.select + expect_fail: true + - id: planted2.retention.today.final_nolock + stmt: DeleteOldRoutesSelectNoLock + shape: "today's expired CTE, last batch of a run (no victims left: the hourly 23 s scan)" + params: ["2026-10-04 17:00:00+00", "2026-10-11 17:00:00+00", 3, 10000] + tiers: [local] + executions: 1 + timeout_s: 120 + inherit: retention.expired.select.final + expect_fail: true + - id: planted2.retention.today.dml + stmt: DeleteOldRoutes_202acc6 + shape: "today's full DELETE, plain EXPLAIN" + params: ["2026-10-04 19:00:00+00", "2026-10-11 19:00:00+00", 3, 10000] + tiers: [fixture, local, prod] + inherit: retention.expired.dml + expect_fail: true + + # 4. Today's ListChannelMessages, channel 27 + GDW, against the channel_message_iatas entry's budget. + - id: planted4.cmi.today.ch27.gdw + stmt: ListChannelMessages + shape: "today's ListChannelMessages, ch 27 + GDW" + params: [27, null, [GDW], "", 0, 51] + tiers: [local] + modes: [custom, generic] + executions: 1 + expect: + budget: {buffers: 600, planning_ms: 10} + expect_fail: true + + # Today's series and adverts readers against the end-state budgets. + - id: planted.extra.series.today.lis7d + stmt: GetStatsSeries + shape: "today's GetStatsSeries, LIS 7 d" + params: ["2026-10-11 17:00:00+00", "2026-10-18 17:00:00+00", [LIS]] + tiers: [local] + modes: [custom] + executions: 1 + inherit: s7.se.LIS_7d + expect_fail: true + - id: planted.extra.adverts.today.busiest + stmt: ListObserverAdverts + shape: "today's ListObserverAdverts, busiest observer" + params: [{r: adv_busiest}, 0, 51] + tiers: [local] + executions: 1 + inherit: s7.adv.busy1_c0_d16 + expect_fail: true + - id: planted.extra.adverts.today.zero + stmt: ListObserverAdverts + shape: "today's ListObserverAdverts, observer with no adverts" + params: [{r: adv_zero}, 0, 51] + tiers: [local] + executions: 1 + inherit: s7.adv.zeroadv_big + expect_fail: true + + # The hop statements must stay class custom: each forced generic plan below must break its entry's budget + # (DETECTED). A MISSED here means the shape no longer needs custom plans. + - id: cproof.byiatahop.H03_SEA_hop25_rare_l200.generic + stmt: ListKnownRoutesByIATAHop + shape: "forced generic plan of the class C ByIATAHop, rare/absent hop (measured generic: 918579 B)" + params: [SEA, 25, null, 0, 200] + tiers: [local] + modes: [generic] + executions: 1 + expect: + driver: [{relation: known_routes, index: idx_known_routes_hop_iata}] + sort_exempt: [hop_group] + budget: {buffers: 1500, planning_ms: 10} + expect_fail: true + - id: cproof.byiatahop.H04_SEA_hop32_vrare_l50.generic + stmt: ListKnownRoutesByIATAHop + shape: "forced generic plan of the class C ByIATAHop, rare/absent hop (measured generic: 1943228 B)" + params: [SEA, 32, null, 0, 50] + tiers: [local] + modes: [generic] + executions: 1 + expect: + driver: [{relation: known_routes, index: idx_known_routes_hop_iata}] + sort_exempt: [hop_group] + budget: {buffers: 1500, planning_ms: 10} + expect_fail: true + - id: cproof.byiatahop.H05_SEA_hop1_absent_l200.generic + stmt: ListKnownRoutesByIATAHop + shape: "forced generic plan of the class C ByIATAHop, rare/absent hop (measured generic: 1943228 B)" + params: [SEA, 1, null, 0, 200] + tiers: [local] + modes: [generic] + executions: 1 + expect: + driver: [{relation: known_routes, index: idx_known_routes_hop_iata}] + sort_exempt: [hop_group] + budget: {buffers: 1500, planning_ms: 10} + expect_fail: true + - id: cproof.byiatahop.H08_BWE_hop22_rare_l50.generic + stmt: ListKnownRoutesByIATAHop + shape: "forced generic plan of the class C ByIATAHop, rare/absent hop (measured generic: 317949 B)" + params: [BWE, 22, null, 0, 50] + tiers: [local] + modes: [generic] + executions: 1 + expect: + driver: [{relation: known_routes, index: idx_known_routes_hop_iata}] + sort_exempt: [hop_group] + budget: {buffers: 1500, planning_ms: 10} + expect_fail: true + - id: cproof.byiatahop.H10_MAD_hop16_rare_l50.generic + stmt: ListKnownRoutesByIATAHop + shape: "forced generic plan of the class C ByIATAHop, rare/absent hop (measured generic: 21525 B)" + params: [MAD, 16, null, 0, 50] + tiers: [local] + modes: [generic] + executions: 1 + expect: + driver: [{relation: known_routes, index: idx_known_routes_hop_iata}] + sort_exempt: [hop_group] + budget: {buffers: 1500, planning_ms: 10} + expect_fail: true + - id: cproof.byiatahop.H12_NEW_hop14_rare_l50.generic + stmt: ListKnownRoutesByIATAHop + shape: "forced generic plan of the class C ByIATAHop, rare/absent hop (measured generic: 86472 B)" + params: [NEW, 14, null, 0, 50] + tiers: [local] + modes: [generic] + executions: 1 + expect: + driver: [{relation: known_routes, index: idx_known_routes_hop_iata}] + sort_exempt: [hop_group] + budget: {buffers: 1500, planning_ms: 10} + expect_fail: true + - id: cproof.byiatahop.H13_NEW_hop1_absent_l50.generic + stmt: ListKnownRoutesByIATAHop + shape: "forced generic plan of the class C ByIATAHop, rare/absent hop (measured generic: 106477 B)" + params: [NEW, 1, null, 0, 50] + tiers: [local] + modes: [generic] + executions: 1 + expect: + driver: [{relation: known_routes, index: idx_known_routes_hop_iata}] + sort_exempt: [hop_group] + budget: {buffers: 1500, planning_ms: 10} + expect_fail: true + - id: cproof.globalhop.GH05_hop25_l200.generic + stmt: ListKnownRoutesGlobalHop + shape: "forced generic plan of the class C GlobalHop, rare/absent hop (measured generic: 714617 B)" + params: [25, null, 0, 200] + tiers: [local] + modes: [generic] + executions: 1 + expect: + driver: [{relation: known_routes, index: idx_known_routes_hop_iata}] + sort_exempt: [hop_group] + budget: {buffers: 1700, planning_ms: 10} + expect_fail: true + - id: cproof.globalhop.GH06_hop32_l50.generic + stmt: ListKnownRoutesGlobalHop + shape: "forced generic plan of the class C GlobalHop, rare/absent hop (measured generic: 619733 B)" + params: [32, null, 0, 50] + tiers: [local] + modes: [generic] + executions: 1 + expect: + driver: [{relation: known_routes, index: idx_known_routes_hop_iata}] + sort_exempt: [hop_group] + budget: {buffers: 1700, planning_ms: 10} + expect_fail: true + - id: cproof.globalhop.GH07_hop42_single_l50.generic + stmt: ListKnownRoutesGlobalHop + shape: "forced generic plan of the class C GlobalHop, rare/absent hop (measured generic: 3869764 B)" + params: [42, null, 0, 50] + tiers: [local] + modes: [generic] + executions: 1 + expect: + driver: [{relation: known_routes, index: idx_known_routes_hop_iata}] + sort_exempt: [hop_group] + budget: {buffers: 1700, planning_ms: 10} + expect_fail: true + - id: cproof.globalhop.GH08_hop1_absent_l50.generic + stmt: ListKnownRoutesGlobalHop + shape: "forced generic plan of the class C GlobalHop, rare/absent hop (measured generic: 3869764 B)" + params: [1, null, 0, 50] + tiers: [local] + modes: [generic] + executions: 1 + expect: + driver: [{relation: known_routes, index: idx_known_routes_hop_iata}] + sort_exempt: [hop_group] + budget: {buffers: 1700, planning_ms: 10} + expect_fail: true diff --git a/db/planguard/diff/planted_readmodels.yaml b/db/planguard/diff/planted_readmodels.yaml new file mode 100644 index 0000000..c451938 --- /dev/null +++ b/db/planguard/diff/planted_readmodels.yaml @@ -0,0 +1,138 @@ +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +# Planted regression: ListPacketsBySites as first proposed (rm: a per-site LIMIT before the cross-site re-check) +# loses rows; the end-state statement (end: the re-check inside each per-site range) does not. +# +# Needs the overlay overlay/packet_sites_walk.sql (schema t_s2b, the 12 sites below). Run it with walks: +# planguard diff --old-ref origin/dev --cases db/planguard/diff/planted_readmodels.yaml --walk 1,2,3,web --dsn ... +# then DROP SCHEMA t_s2b CASCADE. Without --walk, nl.walk compares the first page only, where rm is usually right. +# +# The legacy pairs compare today's ListPacketsByIATAs (millisecond cursor, a saturating per-site scan, the latest +# observer picked without a tie-break) with the packet_sites statements at the same cursor: the named classes cover +# exactly those differences, and nothing else may differ. +# +# The legacy judge pairs and nl.walk.judge use fixed times from a 2026-10-18 snapshot: they mean something only on +# a copy that holds data around those times (elsewhere they compare near-empty pages). nl.l1.pagecursor.tie derives +# its cursor from the data instead, and is skipped (which fails the run) on a snapshot without a microsecond tie. +batches: + - name: planted_readmodels + search_path: "t_s2b, public" + statements: + old: "old:ListPacketsByIATAs" + rm: "file:sql/planted/ListPacketsBySitesReadmodels.sql" + end: "file:sql/pending/ListPacketsBySites.sql" + oracle: "file:sql/oracle/OracleBySites.sql" + pairs: + # Every page boundary from the first page: ref = the oracle (packet_observations, independent of + # packet_sites) fetched as one page, cands walked as the client walks them. + - id: nl.walk + ref: {stmt: oracle, params: [&nl "{AMS,EIN,ENS,GLZ,GRQ,LEY,LWR,MST,QAR,RTM,UDE,UTC}", infinity, '\x', 2]} + cands: + - {stmt: rm, params: [*nl, infinity, '\x', 2]} + - {stmt: end, params: [*nl, infinity, '\x', 2]} + key: [site_heard_at] + walk: + limit_param: 4 + fetch_extra: 1 + cursor: [{param: 2, column: site_heard_at}, {param: 3, column: packet_hash}] + first: [infinity, '\x'] + web_limit: 50 + oracle: true + expect: {rm: different, end: same} + # The same walk from just above the judge times below, so every cursor around them is a page boundary. + - id: nl.walk.judge + ref: {stmt: oracle, params: [*nl, "2026-10-18 15:41:00+00", '\x', 2]} + cands: + - {stmt: rm, params: [*nl, "2026-10-18 15:41:00+00", '\x', 2]} + - {stmt: end, params: [*nl, "2026-10-18 15:41:00+00", '\x', 2]} + key: [site_heard_at] + walk: + limit_param: 4 + fetch_extra: 1 + cursor: [{param: 2, column: site_heard_at}, {param: 3, column: packet_hash}] + first: ["2026-10-18 15:41:00+00", '\x'] + web_limit: 50 + oracle: true + expect: {rm: different, end: same} + + # Today's statement at a millisecond cursor, with the server's parameters for limit 1 (scan depth + # (limit+1)*8, page limit+1). A cursor (t, '\x') is "below t", the same rows as the legacy cursor. + - id: nl.l1.legacy.judge1 + ref: {stmt: old, params: [*nl, "2026-10-18 15:40:39.570+00", null, null, null, null, null, 16, 2]} + cands: + - {stmt: rm, params: [*nl, "2026-10-18 15:40:39.57+00", '\x', 2]} + - {stmt: end, params: [*nl, "2026-10-18 15:40:39.57+00", '\x', 2]} + key: [site_heard_at] + allowed: [tie_order, saturated_suffix, latest_observer_tie] + omit: &latest [latest_observer_id, latest_observer_name, latest_observer_iata, latest_observer_path_length_byte, + latest_observer_hash_size, latest_observer_hop_count, latest_observer_path_bytes] + saturated: {flag: scan_saturated, floor: scan_floor} + expect: {rm: different, end: same} + - id: nl.l1.legacy.judge2 + ref: {stmt: old, params: [*nl, "2026-10-18 15:24:51.737+00", null, null, null, null, null, 16, 2]} + cands: + - {stmt: end, params: [*nl, "2026-10-18 15:24:51.737+00", '\x', 2]} + key: [site_heard_at] + allowed: [tie_order, saturated_suffix, latest_observer_tie] + omit: *latest + saturated: {flag: scan_saturated, floor: scan_floor} + expect: {end: same} + + # The page cursor (exact microseconds and hash) against the legacy millisecond cursor at a real tie: the + # newest key two packets share to the microsecond. The cand's cursor is (that key, the larger hash), so its + # page leads with the other tied packet, which the legacy page (below the key's millisecond) leaves out. The + # three params are read from the overlay in the batch's snapshot; no tie, and the pair is skipped. + - id: nl.l1.pagecursor.tie + ref: + stmt: old + params: + - *nl + - sql: >- + WITH k AS (SELECT packet_hash, max(site_heard_at) AS k FROM packet_sites + WHERE iata = ANY('{AMS,EIN,ENS,GLZ,GRQ,LEY,LWR,MST,QAR,RTM,UDE,UTC}'::bpchar[]) + GROUP BY packet_hash), + t AS (SELECT k FROM k GROUP BY k HAVING count(*) >= 2 ORDER BY k DESC LIMIT 1) + SELECT date_trunc('milliseconds', k) FROM t + - null + - null + - null + - null + - null + - 16 + - 2 + cands: + - stmt: end + params: + - *nl + - sql: >- + WITH k AS (SELECT packet_hash, max(site_heard_at) AS k FROM packet_sites + WHERE iata = ANY('{AMS,EIN,ENS,GLZ,GRQ,LEY,LWR,MST,QAR,RTM,UDE,UTC}'::bpchar[]) + GROUP BY packet_hash), + t AS (SELECT k FROM k GROUP BY k HAVING count(*) >= 2 ORDER BY k DESC LIMIT 1) + SELECT k FROM t + - sql: >- + WITH k AS (SELECT packet_hash, max(site_heard_at) AS k FROM packet_sites + WHERE iata = ANY('{AMS,EIN,ENS,GLZ,GRQ,LEY,LWR,MST,QAR,RTM,UDE,UTC}'::bpchar[]) + GROUP BY packet_hash), + t AS (SELECT k, (array_agg(packet_hash ORDER BY packet_hash DESC))[1] AS h FROM k + GROUP BY k HAVING count(*) >= 2 ORDER BY k DESC LIMIT 1) + SELECT h FROM t + - 2 + key: [site_heard_at] + allowed: [pagecursor_tie_rows, tie_order, saturated_suffix, latest_observer_tie] + require: [pagecursor_tie_rows] + tie_boundary: 2 + tie_cursor: {ts: 2, hash: 3, hash_column: packet_hash} + omit: *latest + saturated: {flag: scan_saturated, floor: scan_floor} + expect: {end: allowed} + + # Control: a shape where rm is right, so a diff that flags everything would show here. + - id: nl.l50.first + ref: {stmt: oracle, params: [*nl, infinity, '\x', 51]} + cands: + - {stmt: rm, params: [*nl, infinity, '\x', 51]} + - {stmt: end, params: [*nl, infinity, '\x', 51]} + key: [site_heard_at] + expect: {rm: same, end: same} diff --git a/db/planguard/embed.go b/db/planguard/embed.go new file mode 100644 index 0000000..3a006d8 --- /dev/null +++ b/db/planguard/embed.go @@ -0,0 +1,14 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package planguard holds the planguard corpus and its frozen statement texts, so a binary can check plans +// without a checkout. Import it as pgdata; the engine is internal/planguard. +package planguard + +import "embed" + +// Files holds corpus.yaml, the sql/ texts it names, the fixture generator (fixture/: its SQL steps and the +// aggregate CSVs they read) and the committed plan signatures (baseline//). +// +//go:embed corpus.yaml sql fixture baseline +var Files embed.FS diff --git a/db/planguard/fixture/10_reference.sql b/db/planguard/fixture/10_reference.sql new file mode 100644 index 0000000..d3a896e --- /dev/null +++ b/db/planguard/fixture/10_reference.sql @@ -0,0 +1,109 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- Fixture step 10: sampling helpers and the reference tables (regions, scopes, channels, observers, nodes and the +-- regions each node is heard in). Every identifier is synthetic: ids and keys are digests of a rank, names are +-- scope_, channel_, observer_ and node_. The fixture's clock stops at 2026-10-18 19:00 UTC (routes) +-- and 2026-10-18 17:00 UTC (packets), as the copy the distributions were read from. +SELECT setseed(0.42); + +-- mkcum(src, value, weight, name) builds planguard_src.(v, lo): value with the cumulative weight below it, as a +-- fraction. A step draws from it with +-- (SELECT array_agg(v ORDER BY lo) FROM planguard_src.)[width_bucket(r, (SELECT array_agg(lo ORDER BY lo) FROM planguard_src.))] +-- for a uniform r in [0, 1): both arrays are computed once per statement and width_bucket searches them. +CREATE FUNCTION planguard_src.mkcum(src text, valcol text, wcol text, tgt text) RETURNS void LANGUAGE plpgsql AS $$ +BEGIN + EXECUTE format('CREATE TABLE planguard_src.%I AS SELECT %s AS v, ' + 'coalesce(sum(%s) OVER (ORDER BY %s ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), 0)::float8 / sum(%s) OVER () AS lo ' + 'FROM %s WHERE %s > 0', tgt, valcol, wcol, valcol, wcol, src, wcol); +END $$; + +-- ---------- regions ---------- +-- Every code the distributions name, plus NEW (the stale region, loaded after ANALYZE in step 90) and WRE (a +-- region that never has routes). +INSERT INTO iata_codes (iata, display_name, added_at) +SELECT iata, 'region ' || iata, timestamptz '2026-01-01 00:00+00' +FROM (SELECT iata FROM planguard_src.routes_regions UNION SELECT iata FROM planguard_src.routes_iata_hop + UNION SELECT iata FROM planguard_src.po_iata UNION SELECT iata FROM planguard_src.node_iatas_iata + UNION SELECT home_iata FROM planguard_src.po_observer + UNION SELECT home_iata FROM planguard_src.cm_channel WHERE home_iata IS NOT NULL + UNION SELECT 'NEW' UNION SELECT 'WRE') s +ORDER BY iata; + +-- ---------- transport scopes: one per rank of packets_scope ---------- +INSERT INTO transport_scopes (id, name, display_name, transport_key, key_fingerprint, created_at) +SELECT k, 'scope_' || k, 'Scope ' || k, decode(md5('scope-key' || k), 'hex'), substr(decode(md5('scope-fp' || k), 'hex'), 1, 8), + timestamptz '2026-01-01 00:00+00' +FROM generate_series(1, (SELECT greatest(count(*), 1491) FROM planguard_src.packets_scope)) k; + +-- ---------- channels: one per rank of cm_channel, then hash-only channels (a third keyed) up to 729 ---------- +CREATE TABLE planguard_src.channel_rank AS +SELECT rank, keyed, row_number() OVER (PARTITION BY keyed ORDER BY rank) - 1 AS nth +FROM (SELECT k AS rank, (k <= (SELECT count(*) FROM planguard_src.cm_channel) OR k % 3 = 0) AS keyed FROM generate_series(1, 729) k) s; +-- Staged in planguard_src.ch; step 60 inserts them with their message counts. A keyless channel is known by its +-- one-byte hash alone, so those hashes are distinct. +CREATE TABLE planguard_src.ch AS +SELECT r.rank AS id, CASE WHEN r.keyed THEN substr(decode(md5('channel-hash' || r.rank), 'hex'), 1, 1) + ELSE set_byte('\x00'::bytea, 0, r.nth::int) END AS channel_hash, + CASE WHEN r.keyed THEN substr(decode(md5('channel-fp' || r.rank), 'hex'), 1, 8) END AS key_fingerprint, + CASE WHEN r.keyed THEN 'channel_' || r.rank END AS name, + CASE WHEN r.keyed AND r.rank % 2 = 0 THEN '#channel_' || r.rank END AS hashtag, + r.keyed AND r.rank % 2 = 0 AS is_hashtag, r.rank = 1 AS is_public, r.keyed AS key_known, + timestamptz '2026-09-01 00:00+00' + (r.rank % 40) * interval '1 day' AS first_seen, + timestamptz '2026-10-18 17:00+00' - (r.rank * r.rank % 9973) * interval '1 minute' - (random() * 59) * interval '1 s' AS last_seen +FROM planguard_src.channel_rank r; + +-- ---------- observers: one per rank of po_observer, at its home site ---------- +INSERT INTO observers (id, public_key, display_name, observer_type, first_seen, last_seen, observation_count, last_iata, last_iata_at) +SELECT md5('observer' || o.rank)::uuid, decode(md5('observer-key' || o.rank) || md5('observer-key2' || o.rank), 'hex'), + 'observer_' || o.rank, CASE WHEN o.rank % 7 = 0 THEN 'room' ELSE 'repeater' END, + timestamptz '2026-09-01 00:00+00', timestamptz '2026-10-18 17:00+00' - (o.rank % 600) * interval '1 minute', + o.n, o.home_iata, timestamptz '2026-10-18 16:00+00' +FROM planguard_src.po_observer o; + +-- ---------- nodes: 1/20 of the copy's, by type ---------- +CREATE TABLE planguard_src.node_rank AS +SELECT row_number() OVER (ORDER BY t.node_type, g) AS rank, t.node_type, + g <= round(t.n_drift60 / 20.0) AS drift60, g <= round(t.n_drift / 20.0) AS has_drift +FROM planguard_src.nodes_types t CROSS JOIN LATERAL generate_series(1, greatest(1, round(t.n / 20.0)::int)) g; +-- Relays whose keys start with the hash prefixes the route-search cases look up: two per prefix, each heard at its +-- home region, and the prefixes searched as a pair share a region (1410 + 3d67 in RTM, 5731 + 0355 in PRG, +-- f877 + 0867 in DEN). +CREATE TABLE planguard_src.prefix_relay AS +SELECT hx, home, row_number() OVER (ORDER BY hx, home) AS k +FROM (VALUES ('1410', 'RTM'), ('1410', 'AMS'), ('3d67', 'RTM'), ('3d67', 'EIN'), ('5731', 'PRG'), ('5731', 'SEA'), + ('0355', 'PRG'), ('0355', 'BWE'), ('f877', 'DEN'), ('f877', 'SYD'), ('0867', 'DEN'), ('0867', 'MAD'), + ('fade', 'SEA'), ('fade', 'PDX'), ('2525', 'SEA'), ('2525', 'BER')) p(hx, home); +INSERT INTO nodes (id, public_key, node_type, name, last_advert_at, first_seen, last_seen, device_clock_drift_seconds) +SELECT md5('node' || n.rank)::uuid, + CASE WHEN pr.hx IS NOT NULL THEN decode(pr.hx || substr(md5('node-key' || n.rank) || md5('node-key2' || n.rank), 5), 'hex') + ELSE decode(md5('node-key' || n.rank) || md5('node-key2' || n.rank), 'hex') END, + n.node_type, 'node_' || n.rank, + timestamptz '2026-10-18 17:00+00' - (random() * 72) * interval '1 hour', + timestamptz '2026-09-01 00:00+00' + (random() * 40) * interval '1 day', + timestamptz '2026-10-18 19:00+00' - (random() * random() * 30) * interval '1 day', + CASE WHEN n.drift60 THEN (61 + floor(random() * 3600)::int) * (CASE WHEN random() < 0.5 THEN -1 ELSE 1 END) + WHEN n.has_drift THEN floor(random() * 121)::int - 60 END +FROM planguard_src.node_rank n +LEFT JOIN planguard_src.prefix_relay pr + ON n.node_type = 2 AND pr.k = n.rank - (SELECT min(rank) FROM planguard_src.node_rank WHERE node_type = 2) + 1; + +-- node_iatas: 1/20 of the copy's nodes per region, at least 8 nodes in every region that has routes; a node is +-- heard in about five regions. The planted prefix relays are heard at their home region. +CREATE TABLE planguard_src.region_nodes AS +SELECT i.iata, greatest(round(coalesce(ni.n, 0) / 20.0)::int, CASE WHEN rr.iata IS NOT NULL THEN 8 ELSE 1 END) AS n +FROM iata_codes i +LEFT JOIN planguard_src.node_iatas_iata ni ON ni.iata = i.iata +LEFT JOIN (SELECT iata FROM planguard_src.routes_regions UNION SELECT iata FROM planguard_src.routes_iata_hop UNION SELECT 'NEW') rr ON rr.iata = i.iata; +INSERT INTO node_iatas (node_id, iata, first_heard, last_heard, observation_count) +SELECT DISTINCT ON (s.id, s.iata) s.id, s.iata, timestamptz '2026-09-01 00:00+00' + (random() * 40) * interval '1 day', + timestamptz '2026-10-18 17:00+00' - (random() * random() * 14) * interval '1 day', 1 + floor(random() * 500)::int +FROM ( + SELECT x.iata, x.id FROM planguard_src.region_nodes r + CROSS JOIN LATERAL (SELECT r.iata, n.id FROM nodes n ORDER BY md5(r.iata || n.id::text) LIMIT r.n) x + UNION ALL + SELECT pr.home, md5('node' || (pr.k + (SELECT min(rank) FROM planguard_src.node_rank WHERE node_type = 2) - 1))::uuid + FROM planguard_src.prefix_relay pr +) s ORDER BY s.id, s.iata; +INSERT INTO node_short_ids (node_id, iata, prefix_4) +SELECT ni.node_id, ni.iata, substr(n.public_key, 1, 4) FROM node_iatas ni JOIN nodes n ON n.id = ni.node_id; diff --git a/db/planguard/fixture/20_routes.sql b/db/planguard/fixture/20_routes.sql new file mode 100644 index 0000000..a2e77fb --- /dev/null +++ b/db/planguard/fixture/20_routes.sql @@ -0,0 +1,82 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- Fixture step 20: known_routes at 1/50. +-- * Region sizes: routes_regions (production's region skew). Regions with a (region, hop) histogram in +-- routes_iata_hop take their hop counts from it, each non-empty cell keeping at least one route, so rare and +-- single-route hops stay present; the other regions draw hops from the pooled histogram. +-- * Age and observation_count: routes_age_obs / routes_obs, from the copy's clock (2026-10-18 19:00 UTC); +-- 3 % of last_seen values are whole seconds, so cursor ties exist. +-- * Relays: drawn from the region's relay nodes (node_iatas, types 2 and 3) with the densest region's +-- popularity curve (routes_relay) folded onto the pool, so a few relays are on many routes and most on few. +-- * WRE gets no routes; NEW is loaded after ANALYZE (step 90). +SELECT setseed(0.43); + +SELECT planguard_src.mkcum('(SELECT hop_count, sum(n) n FROM planguard_src.routes_iata_hop GROUP BY 1) s', 'hop_count', 'n', 'c_routes_hop'); +SELECT planguard_src.mkcum('(SELECT age_h, sum(n) n FROM planguard_src.routes_age_obs GROUP BY 1) s', 'age_h', 'n', 'c_routes_age'); +SELECT planguard_src.mkcum('(SELECT obs, n FROM planguard_src.routes_obs WHERE ge3) s', 'obs', 'n', 'c_routes_obs_ge3'); +SELECT planguard_src.mkcum('(SELECT obs, n FROM planguard_src.routes_obs WHERE NOT ge3) s', 'obs', 'n', 'c_routes_obs_lt3'); +SELECT planguard_src.mkcum('planguard_src.routes_relay', 'rank', 'n', 'c_routes_relay'); +CREATE TABLE planguard_src.routes_age_p AS +SELECT age_h, sum(n) FILTER (WHERE obs_ge3)::float8 / sum(n) AS p FROM planguard_src.routes_age_obs GROUP BY 1; + +-- Relay pools: idx 1..k per region, in a fixed pseudo-random order. +CREATE TABLE planguard_src.region_relays AS +SELECT ni.iata, row_number() OVER (PARTITION BY ni.iata ORDER BY md5(ni.iata || n.id::text)) AS idx, n.id, n.public_key +FROM node_iatas ni JOIN nodes n ON n.id = ni.node_id AND n.node_type IN (2, 3); +CREATE TABLE planguard_src.region_relay_k AS SELECT iata, count(*)::int AS k FROM planguard_src.region_relays GROUP BY 1; +CREATE UNIQUE INDEX ON planguard_src.region_relays (iata, idx); + +-- One row per route: region and hop count. +CREATE TABLE planguard_src.kr_cells AS +SELECT h.iata, h.hop_count AS hop, greatest(1, round(h.n / 50.0))::int AS n +FROM planguard_src.routes_iata_hop h WHERE h.n > 0 AND h.iata NOT IN ('WRE', 'NEW'); +CREATE TABLE planguard_src.kr0 AS +SELECT row_number() OVER (ORDER BY c.iata, c.hop, g) AS rid, c.iata, c.hop +FROM planguard_src.kr_cells c CROSS JOIN LATERAL generate_series(1, c.n) g; +INSERT INTO planguard_src.kr0 +SELECT (SELECT max(rid) FROM planguard_src.kr0) + row_number() OVER (ORDER BY r.iata, g), r.iata, + (SELECT array_agg(v ORDER BY lo) FROM planguard_src.c_routes_hop)[width_bucket(random(), (SELECT array_agg(lo ORDER BY lo) FROM planguard_src.c_routes_hop))] +FROM planguard_src.routes_regions r CROSS JOIN LATERAL generate_series(1, greatest(1, round(r.prod_rows / 50.0))::int) g +WHERE r.iata NOT IN ('WRE', 'NEW') AND r.iata NOT IN (SELECT iata FROM planguard_src.routes_iata_hop); + +-- Ages, observation counts, sightings. The draws are stored first, so each is made once per route. +CREATE TABLE planguard_src.kra AS +SELECT k.*, (SELECT array_agg(v ORDER BY lo) FROM planguard_src.c_routes_age)[width_bucket(random(), (SELECT array_agg(lo ORDER BY lo) FROM planguard_src.c_routes_age))] AS age_h, + random() AS r_obs, random() AS r_sub, random() AS r_tie, random() AS r_o2, random() AS r_fs, random() AS r_hs +FROM planguard_src.kr0 k; +CREATE TABLE planguard_src.kr1 AS +SELECT a.rid, a.iata, a.hop, a.age_h, + CASE WHEN a.age_h >= 168 OR a.r_obs < coalesce(ap.p, 1) + THEN (SELECT array_agg(v ORDER BY lo) FROM planguard_src.c_routes_obs_ge3)[width_bucket(a.r_o2, (SELECT array_agg(lo ORDER BY lo) FROM planguard_src.c_routes_obs_ge3))] + ELSE (SELECT array_agg(v ORDER BY lo) FROM planguard_src.c_routes_obs_lt3)[width_bucket(a.r_o2, (SELECT array_agg(lo ORDER BY lo) FROM planguard_src.c_routes_obs_lt3))] END AS obs, + CASE WHEN a.r_tie < 0.03 THEN date_trunc('second', timestamptz '2026-10-18 19:00+00' - (a.age_h + a.r_sub) * interval '1 hour') + ELSE timestamptz '2026-10-18 19:00+00' - (a.age_h + a.r_sub) * interval '1 hour' END AS ls, + a.r_fs, a.r_hs +FROM planguard_src.kra a +LEFT JOIN planguard_src.routes_age_p ap ON ap.age_h = a.age_h; + +-- Relays per hop, folded from the densest region's popularity rank onto the region's pool. +CREATE TABLE planguard_src.kr_slots AS +SELECT s.rid, s.j, rr.id, rr.public_key +FROM (SELECT k.rid, g.j, k.iata, + 1 + floor((width_bucket(random(), (SELECT array_agg(lo ORDER BY lo) FROM planguard_src.c_routes_relay)) - 1)::float8 + * pk.k / (SELECT count(*) FROM planguard_src.c_routes_relay))::int AS idx + FROM planguard_src.kr0 k JOIN planguard_src.region_relay_k pk ON pk.iata = k.iata + CROSS JOIN LATERAL generate_series(1, k.hop) g(j)) s +JOIN planguard_src.region_relays rr ON rr.iata = s.iata AND rr.idx = s.idx; +CREATE TABLE planguard_src.kr_paths AS +SELECT s.rid, array_agg(s.id ORDER BY s.j) AS node_ids, + array_agg(substr(s.public_key, 1, CASE WHEN k.r_hs < 0.006 THEN 1 WHEN k.r_hs < 0.76 THEN 2 ELSE 3 END) ORDER BY s.j) AS hash_prefix +FROM planguard_src.kr_slots s JOIN planguard_src.kr1 k ON k.rid = s.rid GROUP BY s.rid; + +-- Ids follow first sighting; the heap is in (iata, path_key) order. +INSERT INTO known_routes (id, path_key, node_ids, hash_prefix, iata, hop_count, first_seen, last_seen, observation_count, last_reconfirmed_at) +SELECT 1000 + row_number() OVER (ORDER BY fs, pk), pk, p.node_ids, p.hash_prefix, k.iata, k.hop, fs, k.ls, k.obs, k.ls +FROM (SELECT kr1.*, decode(md5('route' || kr1.rid), 'hex') AS pk, + greatest(timestamptz '2026-09-01 00:00+00', least(kr1.ls, kr1.ls - (kr1.r_fs * kr1.r_fs * 10) * interval '1 day')) AS fs + FROM planguard_src.kr1) k +JOIN planguard_src.kr_paths p ON p.rid = k.rid +ORDER BY k.iata, k.pk; +SELECT setval('known_routes_new_id_seq', (SELECT max(id) FROM known_routes)); +DROP TABLE planguard_src.kra, planguard_src.kr_slots, planguard_src.kr_paths; diff --git a/db/planguard/fixture/30_packets.sql b/db/planguard/fixture/30_packets.sql new file mode 100644 index 0000000..9e27e89 --- /dev/null +++ b/db/planguard/fixture/30_packets.sql @@ -0,0 +1,51 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- Fixture step 30: packets at 1/20 over the 72 hours before 2026-10-18 17:00 UTC: hour, (payload, route) type, +-- the channel / origin / trace / scope shares of each type, the scope skew, observation counts and life times as +-- in the testdata. 2 % of last_heard_at values are whole seconds. r_q is the packet's quantile of observation +-- count; step 40 draws its site count at the same quantile, so the most heard packets reach the most sites. The packets are staged in planguard_src.pk: +-- steps 40 and 50 correct them, and step 60 inserts them once, so the table holds no dead rows. +SELECT setseed(0.44); + +SELECT planguard_src.mkcum('planguard_src.packets_hour', 'age_h', 'n', 'c_packets_hour'); +SELECT planguard_src.mkcum('(SELECT payload_type * 4 + route_type AS k, n FROM planguard_src.packets_types) s', 'k', 'n', 'c_packets_types'); +SELECT planguard_src.mkcum('planguard_src.packets_scope', 'rank', 'n', 'c_packets_scope'); +SELECT planguard_src.mkcum('planguard_src.packets_obscount', 'obs', 'n', 'c_packets_obscount'); + +CREATE TABLE planguard_src.pk0 AS +SELECT i, r_q, + (SELECT array_agg(v ORDER BY lo) FROM planguard_src.c_packets_hour)[width_bucket(random(), (SELECT array_agg(lo ORDER BY lo) FROM planguard_src.c_packets_hour))] AS age_h, + random() AS r_sub, + (SELECT array_agg(v ORDER BY lo) FROM planguard_src.c_packets_types)[width_bucket(random(), (SELECT array_agg(lo ORDER BY lo) FROM planguard_src.c_packets_types))] AS k, + floor(random() * 100)::int AS life_p, random() AS r_life, random() AS r_ch, random() AS r_or, random() AS r_tr, random() AS r_sc, + (SELECT array_agg(v ORDER BY lo) FROM planguard_src.c_packets_obscount)[width_bucket(r_q, (SELECT array_agg(lo ORDER BY lo) FROM planguard_src.c_packets_obscount))] AS obs, + (SELECT array_agg(v ORDER BY lo) FROM planguard_src.c_packets_scope)[width_bucket(random(), (SELECT array_agg(lo ORDER BY lo) FROM planguard_src.c_packets_scope))] AS scope_rank, + 1 + floor(random() * (SELECT count(*) FROM nodes))::int AS origin_k, + 1 + floor(random() * 729)::int AS channel_k, + random() AS r_tie +FROM (SELECT i, random() AS r_q FROM generate_series(1, (SELECT round(sum(n) / 20.0)::int FROM planguard_src.packets_hour)) i) g; + +CREATE TABLE planguard_src.node_keys AS SELECT row_number() OVER (ORDER BY id) AS k, public_key FROM nodes; +CREATE UNIQUE INDEX ON planguard_src.node_keys (k); + +CREATE TABLE planguard_src.pk AS +SELECT substr(decode(md5('packet' || p.i), 'hex'), 1, 8) AS packet_hash, (p.k / 4)::smallint AS payload_type, + 0::smallint AS payload_version, (p.k % 4)::smallint AS route_type, + p.r_sc < t.n_scoped::float8 / t.n AS transport_codes_present, + CASE WHEN p.r_sc < t.n_scoped::float8 / t.n THEN p.scope_rank END AS scope_id, + CASE WHEN p.r_or < t.n_origin::float8 / t.n THEN nk.public_key END AS origin_pubkey, + decode(md5('raw' || p.i) || md5('raw2' || p.i) || md5('raw3' || p.i), 'hex') AS raw_payload, + CASE WHEN p.r_ch < t.n_channel::float8 / t.n THEN c.channel_hash END AS channel_hash, + CASE WHEN p.r_tr < t.n_trace::float8 / t.n THEN substr(decode(md5('trace' || p.i), 'hex'), 1, 4) END AS trace_tag, + p.lh - make_interval(secs => greatest(0, l0.life_s + (l1.life_s - l0.life_s) * p.r_life)) AS first_heard_at, + p.lh AS last_heard_at, p.obs::bigint AS observation_count, p.r_q +FROM (SELECT pk0.*, CASE WHEN pk0.r_tie < 0.02 THEN date_trunc('second', pk0.t) ELSE pk0.t END AS lh + FROM (SELECT pk0.*, timestamptz '2026-10-18 17:00+00' - (pk0.age_h + pk0.r_sub) * interval '1 hour' AS t + FROM planguard_src.pk0) pk0) p +JOIN planguard_src.packets_types t ON t.payload_type * 4 + t.route_type = p.k +JOIN planguard_src.packets_life l0 ON l0.p = p.life_p +JOIN planguard_src.packets_life l1 ON l1.p = p.life_p + 1 +JOIN planguard_src.node_keys nk ON nk.k = p.origin_k +JOIN planguard_src.ch c ON c.id = p.channel_k; +CREATE UNIQUE INDEX ON planguard_src.pk (packet_hash); diff --git a/db/planguard/fixture/40_observations.sql b/db/planguard/fixture/40_observations.sql new file mode 100644 index 0000000..43090eb --- /dev/null +++ b/db/planguard/fixture/40_observations.sql @@ -0,0 +1,72 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- Fixture step 40: packet_observations, about observation_count rows per packet. +-- * Sites: each packet is heard at n_sites distinct sites (po_sites_per_packet at the packet's observation-count +-- quantile, at most its observation count), drawn by site volume (po_iata); its observations go round them. +-- * Observers: each observer hears at its home site only (po_observer), so a packet's observers at a site are +-- that site's, drawn by volume; an observer hears a packet once, so a small site caps the packet's count there. +-- * Times: the packet's last observation is at last_heard_at, the rest within its life, 1 % before +-- first_heard_at (late reports); po_whole_second percent of heard_at values are whole seconds. +-- Then each staged packet's observation_count and last_heard_at are set from its observations. The observations are +-- staged in planguard_src.po; step 60 inserts them. +SELECT setseed(0.45); + +SELECT planguard_src.mkcum('planguard_src.po_iata', 'iata', 'n', 'c_po_iata'); +SELECT planguard_src.mkcum('planguard_src.po_sites_per_packet', 'n_sites', 'packets', 'c_po_sites'); +SELECT planguard_src.mkcum('planguard_src.po_hop', 'hop_count', 'n', 'c_po_hop'); +-- Observers by site, in volume order: idx 1..cnt and the cumulative volume within the site. +CREATE TABLE planguard_src.site_observers AS +SELECT o.home_iata AS iata, row_number() OVER (PARTITION BY o.home_iata ORDER BY o.rank) AS idx, + count(*) OVER (PARTITION BY o.home_iata) AS cnt, md5('observer' || o.rank)::uuid AS observer_id, + sum(o.n) OVER (PARTITION BY o.home_iata ORDER BY o.rank ROWS UNBOUNDED PRECEDING)::float8 / sum(o.n) OVER (PARTITION BY o.home_iata) AS hi +FROM planguard_src.po_observer o WHERE o.n > 0; +CREATE INDEX ON planguard_src.site_observers (iata, hi); +CREATE UNIQUE INDEX ON planguard_src.site_observers (iata, idx); + +-- Each packet's sites: n_sites distinct ones, drawn by volume without replacement (the first draws that differ). +CREATE TABLE planguard_src.pk_sites AS +SELECT d.packet_hash, (array_agg(d.iata ORDER BY d.first_draw))[1:max(d.n_sites)] AS sites +FROM (SELECT x.packet_hash, x.n_sites, x.iata, min(x.g) AS first_draw + FROM (SELECT p.packet_hash, p.n_sites, g, + (SELECT array_agg(v ORDER BY lo) FROM planguard_src.c_po_iata)[width_bucket(random(), (SELECT array_agg(lo ORDER BY lo) FROM planguard_src.c_po_iata))] AS iata + FROM (SELECT packet_hash, greatest(1, least(observation_count::int, + (SELECT array_agg(v ORDER BY lo) FROM planguard_src.c_po_sites)[width_bucket(r_q, (SELECT array_agg(lo ORDER BY lo) FROM planguard_src.c_po_sites))])) AS n_sites + FROM planguard_src.pk) p + CROSS JOIN LATERAL generate_series(1, 3 * p.n_sites + 2) g) x + GROUP BY x.packet_hash, x.n_sites, x.iata) d +GROUP BY d.packet_hash; + +-- Observations: the packet's observation_count goes round its sites; at each site the packet is heard by distinct +-- observers, from one drawn by volume onwards in volume order, at most every observer of the site once. +CREATE TABLE planguard_src.po1 AS +SELECT q.packet_hash, q.payload_type, q.iata, so.observer_id, q.si = 1 AND t.t = 0 AS newest, random() AS r_t, random() AS r_skew, + q.first_heard_at, q.last_heard_at, + (SELECT array_agg(v ORDER BY lo) FROM planguard_src.c_po_hop)[width_bucket(random(), (SELECT array_agg(lo ORDER BY lo) FROM planguard_src.c_po_hop))] AS hop +FROM (SELECT p.packet_hash, p.payload_type, p.first_heard_at, p.last_heard_at, si.si, s.sites[si.si] AS iata, + (p.observation_count::int - si.si) / cardinality(s.sites) + 1 AS quota, st.idx AS start, st.cnt + FROM planguard_src.pk p JOIN planguard_src.pk_sites s ON s.packet_hash = p.packet_hash + CROSS JOIN LATERAL (SELECT g AS si, random() AS r + FROM generate_series(1, least(cardinality(s.sites), greatest(1, p.observation_count::int))) g) si + CROSS JOIN LATERAL (SELECT so.idx, so.cnt FROM planguard_src.site_observers so + WHERE so.iata = s.sites[si.si] AND so.hi > si.r ORDER BY so.hi LIMIT 1) st) q +CROSS JOIN LATERAL generate_series(0, least(q.quota, q.cnt) - 1) t(t) +JOIN planguard_src.site_observers so ON so.iata = q.iata AND so.idx = 1 + (q.start - 1 + t.t) % q.cnt; + +CREATE TABLE planguard_src.po AS +SELECT row_number() OVER (ORDER BY heard_at, packet_hash, observer_id) AS id, packet_hash, observer_id, iata, heard_at, + hop AS path_length_byte, 1::smallint AS hash_size, hop AS hop_count, + substr(decode(md5(packet_hash::text || observer_id::text) || md5('path' || packet_hash::text), 'hex'), 1, hop) AS path_bytes, + (-60 - floor(random() * 60))::smallint AS rssi, (random() * 20 - 8)::real AS snr, payload_type +FROM (SELECT y.*, CASE WHEN random() * 100 < (SELECT whole_second_pct FROM planguard_src.po_whole_second LIMIT 1) + THEN date_trunc('second', y.t) ELSE y.t END AS heard_at + FROM (SELECT po1.*, CASE WHEN po1.newest THEN po1.last_heard_at + WHEN po1.r_skew < 0.01 THEN po1.first_heard_at - po1.r_t * interval '30 min' + ELSE po1.first_heard_at + (po1.last_heard_at - po1.first_heard_at) * po1.r_t END AS t + FROM planguard_src.po1) y) x +ORDER BY heard_at, packet_hash, observer_id; +DROP TABLE planguard_src.po1; + +UPDATE planguard_src.pk p SET observation_count = c.n, last_heard_at = greatest(p.last_heard_at, c.mx) +FROM (SELECT packet_hash, count(*) n, max(heard_at) mx FROM planguard_src.po GROUP BY 1) c +WHERE c.packet_hash = p.packet_hash AND (p.observation_count <> c.n OR c.mx > p.last_heard_at); diff --git a/db/planguard/fixture/50_channel_messages.sql b/db/planguard/fixture/50_channel_messages.sql new file mode 100644 index 0000000..a0e6f4f --- /dev/null +++ b/db/planguard/fixture/50_channel_messages.sql @@ -0,0 +1,37 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- Fixture step 50: channel messages at 1/20, staged in planguard_src.cm. A message is a channel packet; its +-- channel is drawn by volume (cm_channel) among the channels whose home site is one of the packet's sites, with +-- the testdata's home share, else among all channels. The packet then carries its channel's hash. +SELECT setseed(0.46); + +SELECT planguard_src.mkcum('planguard_src.cm_channel', 'rank', 'n', 'c_cm_channel'); +CREATE TABLE planguard_src.site_channels AS +SELECT home_iata AS iata, rank, + sum(n) OVER (PARTITION BY home_iata ORDER BY rank ROWS UNBOUNDED PRECEDING)::float8 / sum(n) OVER (PARTITION BY home_iata) AS hi +FROM planguard_src.cm_channel WHERE home_iata IS NOT NULL AND n > 0; +CREATE INDEX ON planguard_src.site_channels (iata, hi); + +CREATE TABLE planguard_src.cm0 AS +SELECT p.packet_hash, p.first_heard_at, s.sites, random() AS r_home, random() AS r_ch, random() AS r_site +FROM planguard_src.pk p JOIN planguard_src.pk_sites s ON s.packet_hash = p.packet_hash +WHERE p.channel_hash IS NOT NULL +ORDER BY md5(p.packet_hash::text) +LIMIT (SELECT round(sum(n) / 20.0) FROM planguard_src.cm_channel); + +CREATE TABLE planguard_src.cm AS +SELECT 1000 + row_number() OVER (ORDER BY c.first_heard_at, c.packet_hash) AS id, + coalesce(sc.rank, (SELECT array_agg(v ORDER BY lo) FROM planguard_src.c_cm_channel)[width_bucket(c.r_ch, (SELECT array_agg(lo ORDER BY lo) FROM planguard_src.c_cm_channel))]) AS channel_id, + c.packet_hash, 'user_' || floor(random() * 500)::int AS sender_name, c.first_heard_at - random() * interval '5 s' AS sent_at +FROM planguard_src.cm0 c +LEFT JOIN LATERAL ( + SELECT so.rank FROM planguard_src.site_channels so + WHERE c.r_home < (SELECT sum(home_n)::float8 / sum(n) FROM planguard_src.cm_channel WHERE home_n IS NOT NULL) + AND so.iata = c.sites[1 + floor(c.r_site * cardinality(c.sites))::int] AND so.hi > c.r_ch + ORDER BY so.hi LIMIT 1) sc ON true; +DROP TABLE planguard_src.cm0; + +UPDATE planguard_src.pk p SET channel_hash = ch.channel_hash +FROM planguard_src.cm m JOIN planguard_src.ch ch ON ch.id = m.channel_id +WHERE m.packet_hash = p.packet_hash; diff --git a/db/planguard/fixture/60_load.sql b/db/planguard/fixture/60_load.sql new file mode 100644 index 0000000..ba8bce0 --- /dev/null +++ b/db/planguard/fixture/60_load.sql @@ -0,0 +1,34 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- Fixture step 60: insert the staged channels, packets, observations and messages once each, in time order, then +-- drop the staging tables. Every row references rows built in this build, so where the role may, the foreign-key +-- triggers are switched off for the load (session_replication_role = replica) and on again after it. +DO $$ BEGIN SET LOCAL session_replication_role = replica; EXCEPTION WHEN insufficient_privilege THEN NULL; END $$; +INSERT INTO channels (id, channel_hash, key_fingerprint, name, hashtag, is_hashtag, is_public, key_known, first_seen, last_seen, message_count) +SELECT ch.id, ch.channel_hash, ch.key_fingerprint, ch.name, ch.hashtag, ch.is_hashtag, ch.is_public, ch.key_known, ch.first_seen, + ch.last_seen, coalesce(m.n, 0) +FROM planguard_src.ch LEFT JOIN (SELECT channel_id, count(*) n FROM planguard_src.cm GROUP BY 1) m ON m.channel_id = ch.id +ORDER BY ch.id; +SELECT setval('channels_id_seq', (SELECT max(id) FROM channels)); + +INSERT INTO packets (packet_hash, payload_type, payload_version, route_type, transport_codes_present, scope_id, origin_pubkey, + raw_payload, raw_header, parsed_payload, decrypted, channel_hash, trace_tag, first_heard_at, last_heard_at, + observation_count) +SELECT packet_hash, payload_type, payload_version, route_type, transport_codes_present, scope_id, origin_pubkey, raw_payload, + '\x01', NULL, false, channel_hash, trace_tag, first_heard_at, last_heard_at, observation_count +FROM planguard_src.pk ORDER BY last_heard_at, packet_hash; + +INSERT INTO packet_observations (id, packet_hash, observer_id, iata, heard_at, path_length_byte, hash_size, hop_count, path_bytes, + rssi, snr, payload_type) +SELECT id, packet_hash, observer_id, iata, heard_at, path_length_byte, hash_size, hop_count, path_bytes, rssi, snr, payload_type +FROM planguard_src.po ORDER BY id; +SELECT setval('packet_observations_id_seq', (SELECT max(id) FROM packet_observations)); + +INSERT INTO channel_messages (id, channel_id, packet_hash, sender_name, content, sent_at) +SELECT id, channel_id, packet_hash, sender_name, 'message ' || id, sent_at FROM planguard_src.cm ORDER BY id; +SELECT setval('channel_messages_id_seq', (SELECT max(id) FROM channel_messages)); +DO $$ BEGIN SET LOCAL session_replication_role = origin; EXCEPTION WHEN insufficient_privilege THEN NULL; END $$; + +DROP TABLE planguard_src.pk0, planguard_src.pk, planguard_src.po, planguard_src.cm, planguard_src.pk_sites, planguard_src.kr0, + planguard_src.kr1; diff --git a/db/planguard/fixture/70_rollups.sql b/db/planguard/fixture/70_rollups.sql new file mode 100644 index 0000000..44683db --- /dev/null +++ b/db/planguard/fixture/70_rollups.sql @@ -0,0 +1,70 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- Fixture step 70: hourly rollups for 381 complete hours (2026-10-02 19:00 to 2026-10-18 15:00 UTC). Each rollup +-- hour holds the aggregates of the fixture hour at the same offset in the 72-hour packet window, so every hour has +-- a day's shape. observer_identity keeps the six busiest observers per (hour, region), as a real copy holds about +-- that many. Each aggregate is taken once per fixture hour, then copied to the rollup hours it stands for. +INSERT INTO analytics_rollup_hours (hour, status, rolled_at) +SELECT h, 'complete', h + interval '1 hour 5 min' +FROM generate_series(timestamptz '2026-10-02 19:00+00', timestamptz '2026-10-18 15:00+00', interval '1 hour') h; +CREATE TABLE planguard_src.hmap AS +SELECT hour, timestamptz '2026-10-15 17:00+00' + + ((extract(epoch FROM hour - timestamptz '2026-10-02 19:00+00') / 3600)::int % 72) * interval '1 hour' AS src +FROM analytics_rollup_hours; +CREATE TABLE planguard_src.po_src AS +SELECT date_trunc('hour', heard_at) AS src, iata, observer_id, payload_type, hop_count, snr, rssi, heard_at, packet_hash +FROM packet_observations; +CREATE TABLE planguard_src.pkh AS +SELECT p.packet_hash, date_trunc('hour', p.first_heard_at) AS src, p.scope_id, p.payload_type, p.route_type, p.origin_pubkey, s.iatas +FROM packets p +JOIN (SELECT packet_hash, array_agg(DISTINCT iata ORDER BY iata) AS iatas FROM packet_observations GROUP BY 1) s ON s.packet_hash = p.packet_hash; + +INSERT INTO analytics_hourly_iata_observations (hour, iata, observation_count) +SELECT m.hour, a.iata, a.n +FROM (SELECT src, iata, count(*) n FROM planguard_src.po_src GROUP BY 1, 2) a JOIN planguard_src.hmap m ON m.src = a.src +ORDER BY 1, 2; +INSERT INTO analytics_hourly_packet_sets (hour, iatas, packets) +SELECT m.hour, a.iatas, a.n +FROM (SELECT src, iatas, count(*) n FROM planguard_src.pkh GROUP BY 1, 2) a JOIN planguard_src.hmap m ON m.src = a.src +ORDER BY 1, 2; +INSERT INTO analytics_hourly_scope_sets (hour, iatas, scope_id, packets) +SELECT m.hour, a.iatas, a.scope_id, a.n +FROM (SELECT src, iatas, scope_id, count(*) n FROM planguard_src.pkh WHERE scope_id IS NOT NULL GROUP BY 1, 2, 3) a +JOIN planguard_src.hmap m ON m.src = a.src +ORDER BY 1, 2, 3; +INSERT INTO analytics_hourly_observer_identity (hour, iata, observer_id, observation_count) +SELECT m.hour, a.iata, a.observer_id, a.n +FROM (SELECT src, iata, observer_id, n FROM ( + SELECT src, iata, observer_id, count(*) n, row_number() OVER (PARTITION BY src, iata ORDER BY count(*) DESC, observer_id) rk + FROM planguard_src.po_src GROUP BY 1, 2, 3) r WHERE rk <= 6) a +JOIN planguard_src.hmap m ON m.src = a.src +ORDER BY 1, 2, 3; +INSERT INTO analytics_hourly_paths (hour, iata, category, hash_bytes, entries, receptions) +SELECT m.hour, a.iata, c.category, 1, a.entries, a.n +FROM (SELECT src, iata, least(hop_count, 8) AS entries, count(*) n FROM planguard_src.po_src GROUP BY 1, 2, 3) a +JOIN planguard_src.hmap m ON m.src = a.src CROSS JOIN (VALUES (0), (1)) c(category) +ORDER BY 1, 2, 3, 5; +INSERT INTO analytics_hourly_signal (hour, iata, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum) +SELECT m.hour, a.iata, k.kind, a.snr_bin, a.rssi_bin, a.n, a.n, a.snr_sum, a.n, a.rssi_sum +FROM (SELECT src, iata, floor(snr / 10)::int AS snr_bin, floor(rssi / 30)::int AS rssi_bin, count(*) n, sum(snr) snr_sum, sum(rssi) rssi_sum + FROM planguard_src.po_src GROUP BY 1, 2, 3, 4) a +JOIN planguard_src.hmap m ON m.src = a.src CROSS JOIN (VALUES (1), (3)) k(kind) +ORDER BY 1, 2, 3, 4, 5; +-- Adverts: hearings per (hour, region, origin) and distinct advert packets per (hour, origin, region set). +INSERT INTO analytics_hourly_advert_hearings (hour, iata, origin_pubkey, observations, last_heard, name, node_type) +SELECT m.hour, a.iata, a.origin_pubkey, a.n, a.last_heard - a.src + m.hour, n.name, n.node_type +FROM (SELECT po.src, po.iata, p.origin_pubkey, count(*) n, max(po.heard_at) last_heard + FROM planguard_src.po_src po JOIN packets p ON p.packet_hash = po.packet_hash + WHERE p.payload_type = 4 AND p.origin_pubkey IS NOT NULL GROUP BY 1, 2, 3) a +JOIN planguard_src.hmap m ON m.src = a.src +LEFT JOIN nodes n ON n.public_key = a.origin_pubkey +ORDER BY 1, 2, 3; +INSERT INTO analytics_hourly_advert_sets (hour, origin_pubkey, iatas, advert_packets, flood_packets, direct_packets) +SELECT m.hour, a.origin_pubkey, a.iatas, a.n, a.flood, a.direct +FROM (SELECT src, origin_pubkey, iatas, count(*) n, count(*) FILTER (WHERE route_type IN (0, 1)) flood, + count(*) FILTER (WHERE route_type IN (2, 3)) direct + FROM planguard_src.pkh WHERE payload_type = 4 AND origin_pubkey IS NOT NULL GROUP BY 1, 2, 3) a +JOIN planguard_src.hmap m ON m.src = a.src +ORDER BY 1, 2, 3; +DROP TABLE planguard_src.po_src, planguard_src.pkh; diff --git a/db/planguard/fixture/80_statistics.sql b/db/planguard/fixture/80_statistics.sql new file mode 100644 index 0000000..1e951da --- /dev/null +++ b/db/planguard/fixture/80_statistics.sql @@ -0,0 +1,38 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- Fixture step 80: ANALYZE samples 300 x the largest statistics target of a table's columns, at random, so on a +-- table above 60,000 rows two builds would get different statistics and could plan differently. One column per +-- such table that no statement filters, joins or sorts on gets a target large enough to cover the table, so +-- ANALYZE reads every row; every other column keeps default_statistics_target = 200, and its statistics come out +-- the same on every build. +ALTER TABLE known_routes ALTER COLUMN path_key SET STATISTICS 10000; +-- The reference copy's known_routes.iata statistics list every region it holds as a most common value, so a +-- region they miss (a new one, or one loaded after ANALYZE) is estimated at one row. 200 entries cannot hold the +-- fixture's 400-odd regions; 1000 can, which keeps that behaviour. +ALTER TABLE known_routes ALTER COLUMN iata SET STATISTICS 1000; +ALTER TABLE packets ALTER COLUMN raw_payload SET STATISTICS 10000; +ALTER TABLE packet_observations ALTER COLUMN airtime_ms SET STATISTICS 10000; +ALTER TABLE analytics_hourly_iata_observations ALTER COLUMN observation_count SET STATISTICS 10000; +ALTER TABLE analytics_hourly_packet_sets ALTER COLUMN packets SET STATISTICS 10000; +ALTER TABLE analytics_hourly_observer_identity ALTER COLUMN display_name SET STATISTICS 10000; +ALTER TABLE analytics_hourly_paths ALTER COLUMN receptions SET STATISTICS 10000; +ALTER TABLE analytics_hourly_signal ALTER COLUMN rssi_sum SET STATISTICS 10000; +ALTER TABLE analytics_hourly_advert_hearings ALTER COLUMN name SET STATISTICS 10000; +ALTER TABLE analytics_hourly_advert_sets ALTER COLUMN direct_packets SET STATISTICS 10000; +ALTER TABLE analytics_hourly_scope_sets ALTER COLUMN packets SET STATISTICS 10000; + +-- Fail the build when a table outgrows that: more than 3,000,000 rows (10000 x 300), or above 60,000 rows with +-- no such column. +DO $$ +DECLARE r record; +BEGIN + FOR r IN SELECT c.relname, (xpath('/row/n/text()', query_to_xml(format('SELECT count(*) AS n FROM public.%I', c.relname), false, true, '')))[1]::text::bigint AS n, + (SELECT max(a.attstattarget) FROM pg_attribute a WHERE a.attrelid = c.oid AND a.attnum > 0 AND NOT a.attisdropped) AS tgt + FROM pg_class c WHERE c.relnamespace = 'public'::regnamespace AND c.relkind = 'r' LOOP + IF r.n > 300 * greatest(r.tgt, 200) THEN + RAISE EXCEPTION 'fixture: % holds % rows, more than ANALYZE reads at its largest statistics target %; give it a full-sample column in 80_statistics.sql', + r.relname, r.n, greatest(r.tgt, 200); + END IF; + END LOOP; +END $$; diff --git a/db/planguard/fixture/90_stale_region.sql b/db/planguard/fixture/90_stale_region.sql new file mode 100644 index 0000000..9fd301d --- /dev/null +++ b/db/planguard/fixture/90_stale_region.sql @@ -0,0 +1,34 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- Fixture step 90 (after ANALYZE): the stale region. NEW gets about 2,100 routes (1/50 of a region that grew +-- after the last ANALYZE) that the statistics have never seen, so the planner still estimates it at one row. +-- Autovacuum is off on known_routes, so nothing refreshes them. Hops: 4 is dense (28 %), 14 has a single +-- route, 1 has none; the rest follow the pooled histogram. Ages spread over the last 14 days. +ALTER TABLE known_routes SET (autovacuum_enabled = false); +SELECT setseed(0.49); + +SELECT planguard_src.mkcum('(SELECT hop_count, sum(n) n FROM planguard_src.routes_iata_hop WHERE hop_count NOT IN (1, 4, 14) GROUP BY 1) s', + 'hop_count', 'n', 'c_new_hop'); +CREATE TABLE planguard_src.new0 AS +SELECT g AS rid, + CASE WHEN g = 1 THEN 14 WHEN random() < 0.28 THEN 4 + ELSE (SELECT array_agg(v ORDER BY lo) FROM planguard_src.c_new_hop)[width_bucket(random(), (SELECT array_agg(lo ORDER BY lo) FROM planguard_src.c_new_hop))] END AS hop, + timestamptz '2026-10-18 19:00+00' - random() * interval '14 days' AS ls, 1 + floor(random() * random() * 40)::int AS obs, random() AS r_hs +FROM generate_series(1, 2100) g; +CREATE TABLE planguard_src.new_paths AS +SELECT s.rid, array_agg(rr.id ORDER BY s.j) AS node_ids, + array_agg(substr(rr.public_key, 1, CASE WHEN s.r_hs < 0.006 THEN 1 WHEN s.r_hs < 0.76 THEN 2 ELSE 3 END) ORDER BY s.j) AS hash_prefix +FROM (SELECT n.rid, n.r_hs, g.j, + 1 + floor((width_bucket(random(), (SELECT array_agg(lo ORDER BY lo) FROM planguard_src.c_routes_relay)) - 1)::float8 + * (SELECT k FROM planguard_src.region_relay_k WHERE iata = 'NEW') / (SELECT count(*) FROM planguard_src.c_routes_relay))::int AS idx + FROM planguard_src.new0 n CROSS JOIN LATERAL generate_series(1, n.hop) g(j)) s +JOIN planguard_src.region_relays rr ON rr.iata = 'NEW' AND rr.idx = s.idx +GROUP BY s.rid; +INSERT INTO known_routes (id, path_key, node_ids, hash_prefix, iata, hop_count, first_seen, last_seen, observation_count, last_reconfirmed_at) +SELECT (SELECT max(id) FROM known_routes) + row_number() OVER (ORDER BY n.ls, n.rid), decode(md5('new-route' || n.rid), 'hex'), + p.node_ids, p.hash_prefix, 'NEW', n.hop, n.ls - interval '2 days', n.ls, n.obs, n.ls +FROM planguard_src.new0 n JOIN planguard_src.new_paths p ON p.rid = n.rid +ORDER BY decode(md5('new-route' || n.rid), 'hex'); +SELECT setval('known_routes_new_id_seq', (SELECT max(id) FROM known_routes)); +DROP TABLE planguard_src.new0, planguard_src.new_paths; diff --git a/db/planguard/fixture/source.sql b/db/planguard/fixture/source.sql new file mode 100644 index 0000000..42082b3 --- /dev/null +++ b/db/planguard/fixture/source.sql @@ -0,0 +1,34 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- The tables testdata/.csv loads into (schema planguard_src). The CSVs hold aggregate distributions of a +-- production-sized copy only: counts by region, hour, type, rank and percentile. They hold no rows, keys, names +-- or identifiers; observers, scopes and channels appear as ranks. +SET search_path = planguard_src; + +-- known_routes +CREATE TABLE routes_regions (iata char(3) NOT NULL, prod_rows bigint NOT NULL); -- routes per region +CREATE TABLE routes_iata_hop (iata char(3) NOT NULL, hop_count int NOT NULL, n bigint NOT NULL); +CREATE TABLE routes_age_obs (age_h int NOT NULL, obs_ge3 boolean NOT NULL, n bigint NOT NULL); +CREATE TABLE routes_obs (obs bigint NOT NULL, ge3 boolean NOT NULL, n bigint NOT NULL); +CREATE TABLE routes_relay (rank int NOT NULL, n bigint NOT NULL); -- relay popularity, densest region +-- packets +CREATE TABLE packets_hour (age_h int NOT NULL, n bigint NOT NULL); +CREATE TABLE packets_types (payload_type smallint NOT NULL, route_type smallint NOT NULL, n bigint NOT NULL, + n_channel bigint NOT NULL, n_origin bigint NOT NULL, n_trace bigint NOT NULL, n_scoped bigint NOT NULL); +CREATE TABLE packets_scope (rank int NOT NULL, n bigint NOT NULL); +CREATE TABLE packets_obscount (obs int NOT NULL, n bigint NOT NULL); +CREATE TABLE packets_life (p int NOT NULL, life_s float8 NOT NULL); +-- packet_observations +CREATE TABLE po_iata (iata char(3) NOT NULL, n bigint NOT NULL); +CREATE TABLE po_sites_per_packet (n_sites int NOT NULL, packets bigint NOT NULL); +CREATE TABLE po_observer (rank int NOT NULL, n bigint NOT NULL, home_iata char(3) NOT NULL); -- each observer hears at its home site +CREATE TABLE po_hop (hop_count smallint NOT NULL, n bigint NOT NULL); +CREATE TABLE po_whole_second (whole_second_pct numeric NOT NULL); +-- channel messages by channel rank, with the site that hears each channel most +CREATE TABLE cm_channel (rank int NOT NULL, n bigint NOT NULL, home_iata char(3), home_n bigint); +-- nodes +CREATE TABLE nodes_types (node_type smallint NOT NULL, n bigint NOT NULL, n_drift60 bigint NOT NULL, n_drift bigint NOT NULL); +CREATE TABLE node_iatas_iata (iata char(3) NOT NULL, n bigint NOT NULL); + +RESET search_path; diff --git a/db/planguard/fixture/testdata/cm_channel.csv b/db/planguard/fixture/testdata/cm_channel.csv new file mode 100644 index 0000000..c856adc --- /dev/null +++ b/db/planguard/fixture/testdata/cm_channel.csv @@ -0,0 +1,364 @@ +rank,n,home_iata,home_n +1,50465,DTM,4546 +2,50125,SEA,2577 +3,35554,DEN,1331 +4,15597,KRK,2040 +5,8688,PAD,1794 +6,3693,WLG,498 +7,2690,MOW,2394 +8,2415,ABE,230 +9,1366,HUY,220 +10,1355,ALB,1317 +11,1089,AMS,772 +12,1002,YUL,801 +13,822,ALB,534 +14,819,HUY,690 +15,733,BTS,690 +16,658,SEA,650 +17,649,DTM,304 +18,608,JKG,608 +19,536,BUD,530 +20,533,HUY,427 +21,520,VIE,452 +22,452,BER,439 +23,398,YYC,297 +24,387,WAW,183 +25,344,DND,310 +26,332,LYS,229 +27,311,TMP,311 +28,311,BER,308 +29,283,UDE,273 +30,270,MEL,265 +31,256,PSK,51 +32,254,FAT,250 +33,245,ERF,242 +34,235,ABX,234 +35,213,LEJ,208 +36,211,BER,211 +37,209,MEL,206 +38,201,HAJ,197 +39,201,NWI,160 +40,192,PDX,187 +41,189,ALB,187 +42,186,CAX,186 +43,183,STR,182 +44,182,BDL,181 +45,182,SCN,177 +46,175,RDO,50 +47,157,JAX,53 +48,150,TLS,147 +49,149,VIE,146 +50,146,LCY,47 +51,144,BWE,144 +52,140,BRQ,138 +53,139,EIN,139 +54,139,YTR,139 +55,138,LRT,95 +56,131,PRB,128 +57,130,CHA,123 +58,129,SAN,129 +59,127,POZ,116 +60,125,STR,36 +61,125,BOI,121 +62,125,SEA,119 +63,123,SAN,122 +64,118,FAT,82 +65,116,YVR,110 +66,114,ERF,112 +67,112,LYS,106 +68,112,HAM,101 +69,107,HEL,94 +70,101,SAN,101 +71,100,YYC,99 +72,100,ABX,98 +73,99,LAX,98 +74,96,RDU,93 +75,96,SLE,93 +76,96,SEA,96 +77,95,EDI,74 +78,92,RMO,91 +79,91,DUB,86 +80,88,ALH,84 +81,87,DRS,87 +82,86,MMX,86 +83,86,NTL,80 +84,85,BUF,85 +85,83,EUG,75 +86,82,LAX,81 +87,81,ORD,81 +88,81,BHX,74 +89,81,BRN,78 +90,80,TSV,80 +91,79,BOI,76 +92,77,ERF,77 +93,76,MCE,76 +94,73,LYS,31 +95,73,NTL,55 +96,73,BOS,67 +97,71,RIX,71 +98,70,PRG,67 +99,68,PHX,56 +100,67,BEG,62 +101,66,BDL,63 +102,66,BZG,22 +103,66,ANR,60 +104,65,SEA,64 +105,65,KLR,65 +106,62,BUD,62 +107,62,SCE,61 +108,62,LAX,61 +109,62,KTW,59 +110,62,AUS,18 +111,61,PDX,57 +112,61,TRI,60 +113,60,BOS,58 +114,60,YTR,60 +115,59,SZZ,59 +116,59,LNZ,52 +117,58,PRG,41 +118,57,KLU,56 +119,55,SCE,55 +120,55,CHA,54 +121,52,YVR,17 +122,52,KEL,36 +123,52,GRQ,52 +124,48,YYC,48 +125,48,HAM,48 +126,46,TWF,46 +127,46,ERF,42 +128,45,OAK,37 +129,45,SCL,42 +130,44,BRQ,42 +131,43,CEG,43 +132,43,VIE,43 +133,42,DRS,42 +134,42,YKA,42 +135,41,OMA,41 +136,40,YYZ,24 +137,40,YYC,40 +138,40,TRG,16 +139,40,LAX,40 +140,39,EAT,37 +141,38,MSN,38 +142,37,BTS,16 +143,37,FMM,34 +144,35,YYC,35 +145,35,PDX,31 +146,35,ODE,35 +147,34,OMA,34 +148,33,TLL,32 +149,33,BTS,33 +150,32,SBA,32 +151,31,BHM,24 +152,31,LAX,31 +153,31,BOD,17 +154,31,BDL,12 +155,30,BOS,20 +156,30,SEA,29 +157,30,TGN,30 +158,29,TSR,28 +159,29,SLC,29 +160,29,EUG,29 +161,29,KLU,25 +162,28,PDX,28 +163,28,BNA,28 +164,28,PRG,28 +165,28,AHN,22 +166,28,ROA,13 +167,28,CON,28 +168,27,ABE,6 +169,27,LCJ,25 +170,27,PRG,27 +171,27,LOZ,26 +172,27,FAT,27 +173,26,LEJ,24 +174,26,CHA,25 +175,26,BNE,22 +176,26,LEJ,26 +177,25,YOW,25 +178,24,OMA,24 +179,24,LEJ,24 +180,24,SEA,23 +181,24,PSC,22 +182,24,CHA,24 +183,24,ABX,23 +184,23,MOW,15 +185,23,LYS,23 +186,23,BHD,23 +187,22,RDU,19 +188,22,SCE,22 +189,22,KRK,20 +190,22,PRG,10 +191,22,NOD,20 +192,22,SAN,22 +193,22,RDU,22 +194,22,DFW,22 +195,21,SCE,21 +196,20,FRA,20 +197,20,NTE,19 +198,20,FWA,7 +199,19,KSC,15 +200,19,CBR,19 +201,19,LAX,19 +202,19,CFR,19 +203,19,LAX,19 +204,18,NCE,18 +205,18,AAR,14 +206,18,BRQ,18 +207,18,YKF,15 +208,18,GPT,18 +209,17,PDX,15 +210,16,ATL,15 +211,16,FNL,16 +212,16,ELP,15 +213,15,SEA,12 +214,15,LAX,8 +215,15,CVG,11 +216,15,TWF,15 +217,15,LAX,15 +218,14,YYZ,13 +219,14,SOF,11 +220,14,NTL,14 +221,14,SZZ,14 +222,13,GRR,13 +223,13,AMS,9 +224,13,PSK,13 +225,13,ENS,13 +226,13,YKF,13 +227,13,PUW,13 +228,12,SCE,12 +229,12,SJO,12 +230,12,YCD,10 +231,12,LEY,9 +232,12,OSL,12 +233,12,KRK,12 +234,12,MDT,11 +235,12,MEL,12 +236,12,YTR,12 +237,12,SEA,9 +238,11,SCE,11 +239,11,MST,5 +240,11,PDX,10 +241,11,AAR,11 +242,11,BRQ,11 +243,11,KUF,11 +244,11,PSK,10 +245,11,PDX,10 +246,11,LIN,8 +247,10,CHA,10 +248,10,DRS,10 +249,10,EUG,10 +250,10,LAX,10 +251,10,MEL,7 +252,10,PGV,10 +253,9,BLI,9 +254,9,DTM,8 +255,9,SYD,8 +256,8,BHS,8 +257,8,MKE,8 +258,8,HEL,8 +259,8,BHX,6 +260,8,OSL,8 +261,8,CVG,8 +262,8,BHD,8 +263,8,PGV,8 +264,8,OSL,8 +265,8,FRA,8 +266,7,LIN,7 +267,7,AHN,7 +268,7,PDX,5 +269,7,LAX,7 +270,7,RDU,7 +271,7,EAT,7 +272,6,YYC,6 +273,6,PSK,6 +274,6,MSY,3 +275,6,BHX,5 +276,6,HAM,4 +277,6,GEX,4 +278,6,MCO,6 +279,6,EUG,4 +280,6,ABX,6 +281,6,TMP,6 +282,6,BHS,6 +283,6,DET,6 +284,6,LAX,6 +285,6,BOI,6 +286,5,BYU,4 +287,5,SGU,2 +288,5,EAT,5 +289,5,EUG,4 +290,5,BLI,5 +291,5,EUG,5 +292,5,IDA,5 +293,5,BRN,5 +294,5,WAW,5 +295,5,YEG,5 +296,5,BDL,5 +297,5,CPH,4 +298,5,GSO,5 +299,5,TRI,5 +300,4,BLI,4 +301,4,GEG,3 +302,4,MMX,4 +303,4,IDA,4 +304,4,BWE,4 +305,4,YGK,4 +306,4,SJO,4 +307,4,DRS,4 +308,4,PNS,4 +309,3,BCN,3 +310,3,EAT,3 +311,3,LAX,3 +312,3,ABQ,2 +313,3,ONP,2 +314,3,FRA,3 +315,3,BOI,2 +316,3,GPT,3 +317,3,KUF,3 +318,3,BHX,2 +319,3,TSR,3 +320,3,NOC,3 +321,3,OMA,3 +322,3,OMA,3 +323,3,OSL,3 +324,3,HOU,3 +325,3,BUF,3 +326,3,GSP,3 +327,3,AUS,3 +328,3,ALB,3 +329,2,ONP,2 +330,2,YYJ,2 +331,2,AKL,1 +332,2,SZZ,2 +333,2,AHN,1 +334,2,BLI,2 +335,2,JKG,2 +336,2,BLI,2 +337,2,BRQ,2 +338,2,CEG,2 +339,2,BHS,2 +340,2,AKL,1 +341,1,DAB,1 +342,1,GRR,1 +343,1,CVO,1 +344,1,CVO,1 +345,1,AVL,1 +346,1,NOC,1 +347,1,FPR,1 +348,1,BLI,1 +349,1,BYU,1 +350,1,YQB,1 +351,1,AHN,1 +352,1,CVO,1 +353,1,ROA,1 +354,1,ELP,1 +355,1,KRK,1 +356,1,MKE,1 +357,1,CPH,1 +358,1,LAX,1 +359,1,LAX,1 +360,1,BRQ,1 +361,1,BLI,1 +362,1,GPT,1 +363,1,ANE,1 diff --git a/db/planguard/fixture/testdata/node_iatas_iata.csv b/db/planguard/fixture/testdata/node_iatas_iata.csv new file mode 100644 index 0000000..3c21abf --- /dev/null +++ b/db/planguard/fixture/testdata/node_iatas_iata.csv @@ -0,0 +1,584 @@ +iata,n +SEA,4203 +BRN,3543 +FRA,3423 +STR,3174 +PDX,3073 +FMM,2938 +AVL,2864 +TRI,2843 +RDU,2812 +BWE,2794 +YVR,2771 +PAE,2770 +CHA,2697 +PAD,2695 +BLI,2689 +ATL,2674 +YYJ,2625 +NUE,2622 +TYS,2622 +GSP,2615 +KLS,2611 +TTD,2532 +PSK,2507 +PSC,2505 +EAT,2453 +GSO,2434 +GEG,2433 +AHN,2431 +RDO,2387 +LOZ,2384 +MUC,2382 +NUW,2381 +AMS,2365 +HOH,2347 +DTM,2313 +BNA,2296 +SLE,2294 +CGN,2233 +YKM,2203 +KRK,2159 +BHX,2154 +CRW,2128 +ACH,2070 +CLT,2033 +VIE,2022 +BRQ,1984 +DUS,1919 +POZ,1913 +EUG,1905 +LEJ,1905 +WAW,1905 +FRD,1878 +SPM,1873 +HAJ,1854 +EPH,1848 +KSF,1834 +RDM,1746 +UTC,1693 +ROA,1686 +ESS,1668 +EIN,1659 +CVO,1634 +SXB,1610 +MST,1564 +PUW,1554 +FLO,1552 +KZG,1552 +RTM,1552 +LCJ,1547 +WRO,1519 +CBG,1508 +HAM,1497 +BYU,1493 +LCY,1477 +LON,1472 +BRS,1471 +LEY,1468 +MFR,1442 +GTP,1432 +DRS,1419 +AUS,1382 +LUX,1377 +RST,1376 +BDL,1363 +LGG,1361 +OXF,1359 +ENS,1352 +MAN,1350 +ERF,1335 +BER,1331 +PRG,1322 +STN,1306 +LYS,1302 +BHM,1301 +PED,1295 +GLZ,1291 +VOD,1291 +HRA,1290 +RIC,1286 +BOH,1277 +BOS,1277 +CVG,1277 +FKB,1268 +QAR,1265 +CEG,1255 +LEX,1244 +MMX,1225 +DAY,1220 +SCN,1217 +ANR,1208 +UHE,1196 +CON,1183 +SAN,1173 +LAX,1170 +ALB,1169 +CPH,1162 +AAR,1159 +CAE,1159 +ONT,1156 +OXR,1147 +SJC,1137 +KLR,1135 +SFO,1131 +OAK,1129 +OSR,1121 +AAH,1120 +LIN,1120 +PRB,1119 +SQL,1115 +SBA,1110 +MCE,1101 +MOW,1101 +COA,1094 +IAD,1089 +SBP,1081 +GRZ,1075 +DLE,1072 +GRQ,1062 +MSY,1060 +FAT,1057 +OBF,1057 +GDN,1051 +TRN,1051 +LWR,1049 +CZW,1027 +AGS,1020 +STS,1019 +IND,1006 +CAX,994 +WIE,989 +UDE,980 +NCE,965 +LMT,958 +ODE,947 +BHD,942 +KLU,940 +LIS,933 +GPT,926 +MHG,925 +BRU,923 +TAT,920 +MMV,918 +SOF,906 +KTN,902 +PNS,882 +MEL,880 +BHS,879 +SYD,878 +RBG,862 +VIS,857 +IDA,850 +SCE,842 +PIH,835 +BUD,831 +MDT,828 +PAH,826 +PKB,823 +BLL,822 +WOL,821 +GEX,812 +TGN,812 +BZG,811 +HUY,809 +JAN,809 +MGL,805 +NTU,805 +CBR,801 +MIM,786 +LNZ,785 +ABX,775 +GUL,774 +NYC,771 +NTL,768 +BTS,765 +GUF,764 +NOC,757 +SGD,754 +DEN,749 +MRY,749 +ALL,743 +RIX,727 +SNN,727 +PGV,723 +NWI,721 +LGU,716 +WRT,716 +ONP,709 +ZNO,709 +GCI,702 +SAT,697 +FNL,693 +LUZ,688 +SQQ,682 +BOI,679 +LTN,678 +SZZ,677 +ORD,674 +OPO,673 +TWF,672 +DFW,667 +MME,665 +VNO,661 +FNB,656 +COS,637 +AKL,634 +TXK,632 +CRL,623 +SLC,622 +HDB,616 +WRE,616 +HAD,613 +KSC,612 +TRO,612 +EXT,605 +BEG,603 +GOA,603 +TSR,603 +KUN,598 +DSA,597 +DUB,588 +LRT,583 +NTE,578 +ACY,576 +APF,575 +MKE,569 +PUB,568 +KTW,562 +EMA,561 +NPL,561 +BLA,559 +TRG,552 +KEL,549 +MSN,547 +EGE,534 +HEL,530 +WGA,530 +HOU,524 +CMH,519 +WLG,516 +PDV,514 +NOA,512 +TLL,509 +TPA,506 +PIT,503 +ATW,489 +YYZ,487 +PQQ,484 +MVH,471 +SRQ,470 +MRO,466 +KJK,464 +MPL,463 +YCD,454 +TLS,448 +YUL,445 +STO,444 +OTP,439 +BUF,437 +PAR,437 +PLH,437 +EDI,434 +EVV,433 +HTS,433 +ROT,429 +GLA,428 +OSI,428 +PSP,427 +YOW,423 +AAL,421 +BNE,419 +CLE,419 +DND,417 +BES,415 +VLY,410 +LNS,409 +SDF,407 +TYR,404 +ORY,403 +MAD,396 +PGD,392 +OSL,391 +OMA,390 +OOL,390 +TRF,389 +RYG,388 +RZE,386 +MCO,385 +LBC,378 +MLB,375 +INV,365 +ABE,364 +FPR,360 +BFL,358 +EWN,358 +YHU,349 +YXX,347 +PPM,343 +GRR,342 +PVD,341 +YTR,338 +AOH,333 +GHV,332 +UIP,330 +ANE,323 +ABQ,320 +BRE,320 +SKE,320 +BWI,317 +FLR,316 +ORL,315 +DAB,313 +LRU,308 +TWB,304 +ROC,298 +ZBM,298 +TMP,295 +RMO,292 +GRU,289 +SVC,289 +NSN,288 +ROV,285 +VCP,285 +SOD,283 +FWA,282 +ORM,281 +YQB,281 +BJP,279 +WVN,279 +PIA,277 +QDV,276 +CPQ,275 +SSZ,275 +PMR,270 +ELP,269 +PHX,264 +ADL,263 +IAS,250 +BCN,247 +GOT,246 +YKF,242 +LRH,239 +YRQ,232 +SAO,227 +PER,225 +AAB,224 +TRS,216 +BOD,215 +RKS,214 +KSD,211 +MNN,211 +SGU,207 +LPL,206 +MLU,200 +PLO,199 +GTR,196 +KUF,195 +MSP,190 +DET,189 +ORB,189 +LJU,188 +RNB,187 +NAP,186 +GAS,182 +LIL,179 +YEG,177 +CWA,176 +LAS,172 +MBS,168 +RNO,167 +SMF,165 +CFR,154 +BUY,148 +HLG,145 +PHL,145 +GOJ,142 +YYC,142 +MCI,141 +RKD,139 +SYR,137 +PSF,130 +RMU,130 +YGK,129 +ALC,122 +CAK,122 +PDK,119 +TUL,117 +VLC,117 +PRC,116 +WBG,112 +BLE,110 +TKU,105 +AZO,104 +CHC,104 +WYA,104 +ITA,101 +LBA,98 +LFT,97 +CPT,94 +JAX,91 +XYZ,91 +GTI,89 +UME,85 +LPI,83 +JTC,80 +PLQ,80 +SVX,80 +OGG,78 +RUT,78 +VOG,77 +ATH,75 +MUE,75 +LAA,73 +DUD,66 +JKG,64 +MIA,64 +FAR,62 +RJK,62 +VXO,62 +SJO,59 +KZN,57 +MFE,57 +SJT,56 +YKA,53 +YVE,53 +SJY,52 +SCL,51 +RLG,50 +ABI,49 +CWB,49 +IVC,49 +KOK,49 +LCH,48 +TSV,46 +ROK,45 +LRF,44 +AMA,41 +ALH,40 +GOR,40 +RAO,40 +TJM,40 +TRD,40 +FNT,38 +YSE,38 +MQL,37 +SIR,37 +SJU,37 +SMN,37 +LBB,36 +RKV,35 +BGR,33 +GJT,33 +YYY,32 +BYG,30 +YHM,30 +YQL,29 +YWG,29 +BWT,28 +SDR,28 +YML,28 +CEB,27 +RFD,26 +DVN,22 +VAG,21 +BBH,20 +DIQ,19 +NYO,19 +RDD,19 +YXU,19 +GDW,18 +MOT,18 +LCQ,17 +NOD,17 +YLK,17 +DPO,16 +DXB,16 +VST,15 +DNQ,14 +HVN,14 +MEM,13 +ABC,12 +EAU,12 +GPZ,12 +OKC,12 +SDL,12 +STL,12 +ELM,11 +ALF,10 +IQQ,10 +LWB,10 +AIZ,9 +LSE,9 +OSD,9 +PFB,9 +PMI,9 +RDZ,9 +VRZ,9 +YQY,9 +YHZ,8 +DRO,7 +MMH,7 +SBN,7 +TLH,7 +VAP,7 +AUH,6 +PHF,6 +XMS,6 +YQA,6 +YQQ,6 +ANC,5 +BVE,5 +CSM,5 +EME,5 +OUL,5 +SBY,5 +SVG,5 +VBY,5 +YSJ,5 +YTF,5 +BQN,4 +DSM,4 +MSV,4 +TGD,4 +YQS,4 +CMI,3 +DLH,3 +JBQ,3 +LOI,3 +OCF,3 +PSH,3 +UKK,3 +AVP,2 +AXN,2 +BFD,2 +BGM,2 +CDG,2 +DTW,2 +HYR,2 +LEH,2 +LGA,2 +MAF,2 +NHZ,2 +RSW,2 +YQF,2 +BOJ,1 +CGI,1 +CRX,1 +DAL,1 +DEC,1 +GBR,1 +GGG,1 +IKT,1 +JER,1 +JYV,1 +LIT,1 +MAC,1 +MGM,1 +PLR,1 +POA,1 +SWE,1 +SWH,1 +XCM,1 +YBL,1 +YYB,1 diff --git a/db/planguard/fixture/testdata/nodes_types.csv b/db/planguard/fixture/testdata/nodes_types.csv new file mode 100644 index 0000000..e6d8f3e --- /dev/null +++ b/db/planguard/fixture/testdata/nodes_types.csv @@ -0,0 +1,8 @@ +node_type,n,n_drift60,n_drift +0,1,0,1 +1,19818,4328,19814 +2,45822,27477,45822 +3,2985,1595,2985 +4,94,60,94 +5,1,0,1 +13,2,1,2 diff --git a/db/planguard/fixture/testdata/packets_hour.csv b/db/planguard/fixture/testdata/packets_hour.csv new file mode 100644 index 0000000..28e340e --- /dev/null +++ b/db/planguard/fixture/testdata/packets_hour.csv @@ -0,0 +1,73 @@ +age_h,n +0,47819 +1,47923 +2,47123 +3,46727 +4,46645 +5,44138 +6,43817 +7,41704 +8,40916 +9,39673 +10,39103 +11,38409 +12,38854 +13,38783 +14,38417 +15,39893 +16,42511 +17,42549 +18,43504 +19,46795 +20,46620 +21,49957 +22,50312 +23,48380 +24,46706 +25,45285 +26,46848 +27,46812 +28,45931 +29,42695 +30,42737 +31,41138 +32,40590 +33,39984 +34,38409 +35,38512 +36,37527 +37,37464 +38,37016 +39,38935 +40,39156 +41,39661 +42,44352 +43,45473 +44,45166 +45,45158 +46,46101 +47,47574 +48,45698 +49,45524 +50,43492 +51,43498 +52,40167 +53,39978 +54,39208 +55,38390 +56,36916 +57,36237 +58,36160 +59,38001 +60,35828 +61,35120 +62,36842 +63,37968 +64,38207 +65,39820 +66,42055 +67,44608 +68,45429 +69,47404 +70,47617 +71,44689 diff --git a/db/planguard/fixture/testdata/packets_life.csv b/db/planguard/fixture/testdata/packets_life.csv new file mode 100644 index 0000000..4dc8ebc --- /dev/null +++ b/db/planguard/fixture/testdata/packets_life.csv @@ -0,0 +1,102 @@ +p,life_s +0,-0.000622 +1,0.000000 +2,0.000000 +3,0.000000 +4,0.000000 +5,0.000000 +6,0.000000 +7,0.000000 +8,0.000000 +9,0.000000 +10,0.000000 +11,0.000000 +12,0.000000 +13,0.000000 +14,0.000000 +15,0.000000 +16,0.000000 +17,0.000000 +18,0.002718 +19,0.003499 +20,0.004565 +21,0.005764 +22,0.007167 +23,0.008875 +24,0.010879 +25,0.013252 +26,0.015906 +27,0.018954 +28,0.022242 +29,0.025699 +30,0.029256 +31,0.032854 +32,0.036614 +33,0.040572 +34,0.044725 +35,0.049133 +36,0.054041 +37,0.059773 +38,0.066834 +39,0.075345 +40,0.085434 +41,0.096087 +42,0.107574 +43,0.124665 +44,0.148193 +45,0.179271 +46,0.216855 +47,0.264202 +48,0.319583 +49,0.388519 +50,0.461696 +51,0.536017 +52,0.621437 +53,0.715665 +54,0.819139 +55,0.937521 +56,1.068236 +57,1.231052 +58,1.426089 +59,1.685540 +60,1.996238 +61,2.338973 +62,2.708982 +63,3.150733 +64,3.648131 +65,4.158584 +66,4.670691 +67,5.173542 +68,5.712973 +69,6.275049 +70,6.858699 +71,7.472982 +72,8.114002 +73,8.802321 +74,9.539922 +75,10.311899 +76,11.138005 +77,12.036378 +78,13.012939 +79,14.063799 +80,15.189421 +81,16.431947 +82,17.807235 +83,19.331766 +84,21.002007 +85,22.891016 +86,24.986567 +87,27.298242 +88,29.974254 +89,33.041156 +90,36.792441 +91,41.924791 +92,50.008342 +93,66.200844 +94,102.018279 +95,159.390332 +96,224.100594 +97,285.059189 +98,667.661740 +99,5855.340815 +100,1373016.616798 diff --git a/db/planguard/fixture/testdata/packets_obscount.csv b/db/planguard/fixture/testdata/packets_obscount.csv new file mode 100644 index 0000000..89f1d81 --- /dev/null +++ b/db/planguard/fixture/testdata/packets_obscount.csv @@ -0,0 +1,210 @@ +obs,n +1,1538610 +2,319641 +3,132196 +4,75424 +5,52295 +6,40847 +7,38711 +8,39010 +9,47145 +10,49834 +11,46574 +12,37277 +13,32428 +14,28555 +15,28953 +16,28826 +17,27286 +18,24300 +19,21700 +20,19715 +21,18107 +22,15789 +23,13264 +24,12005 +25,11138 +26,10463 +27,10122 +28,10043 +29,10661 +30,13316 +31,18338 +32,19347 +33,10301 +34,11090 +35,11663 +36,11571 +37,11064 +38,10962 +39,10058 +40,8148 +41,6118 +42,5157 +43,4575 +44,4352 +45,4282 +46,4224 +47,3914 +48,3422 +49,3049 +50,2624 +51,2270 +52,1983 +53,1953 +54,1846 +55,1933 +56,1947 +57,1968 +58,2108 +59,2087 +60,1971 +61,1952 +62,1993 +63,1960 +64,1901 +65,1830 +66,1700 +67,1598 +68,1499 +69,1442 +70,1511 +71,1514 +72,1395 +73,1589 +74,1631 +75,1652 +76,1722 +77,1833 +78,1952 +79,2045 +80,2144 +81,2308 +82,2286 +83,2342 +84,2339 +85,2257 +86,2312 +87,2277 +88,2314 +89,2322 +90,2363 +91,2454 +92,2376 +93,2487 +94,2506 +95,2692 +96,2654 +97,2788 +98,2840 +99,2799 +100,2769 +101,2784 +102,2680 +103,2469 +104,2332 +105,2196 +106,1866 +107,1707 +108,1359 +109,1128 +110,879 +111,777 +112,599 +113,485 +114,439 +115,324 +116,275 +117,283 +118,198 +119,184 +120,158 +121,124 +122,129 +123,113 +124,82 +125,93 +126,125 +127,103 +128,97 +129,113 +130,122 +131,118 +132,98 +133,105 +134,119 +135,105 +136,107 +137,75 +138,100 +139,78 +140,88 +141,66 +142,55 +143,46 +144,40 +145,26 +146,24 +147,25 +148,22 +149,17 +150,13 +151,17 +152,17 +153,16 +154,5 +155,7 +156,6 +157,12 +158,5 +159,7 +160,6 +161,2 +162,6 +163,3 +164,6 +165,4 +166,3 +167,5 +168,3 +169,4 +170,2 +171,1 +172,3 +173,2 +174,5 +176,5 +177,2 +178,2 +179,1 +182,2 +184,2 +185,1 +187,2 +188,1 +189,1 +191,1 +194,1 +195,1 +198,1 +199,1 +207,2 +208,1 +212,1 +214,1 +215,2 +216,1 +218,1 +220,3 +221,2 +224,1 +225,1 +226,2 +227,2 +229,3 +230,1 +231,1 +233,1 +235,1 +237,1 +239,1 diff --git a/db/planguard/fixture/testdata/packets_scope.csv b/db/planguard/fixture/testdata/packets_scope.csv new file mode 100644 index 0000000..ad2c19e --- /dev/null +++ b/db/planguard/fixture/testdata/packets_scope.csv @@ -0,0 +1,418 @@ +rank,n +1,21566 +2,19543 +3,10349 +4,9592 +5,8353 +6,6383 +7,5610 +8,5383 +9,5280 +10,5084 +11,4917 +12,4481 +13,3600 +14,3205 +15,3092 +16,2351 +17,2341 +18,2200 +19,2022 +20,1957 +21,1940 +22,1939 +23,1927 +24,1922 +25,1867 +26,1838 +27,1721 +28,1714 +29,1576 +30,1414 +31,1379 +32,1331 +33,1321 +34,1258 +35,1239 +36,1178 +37,1177 +38,1097 +39,1076 +40,1025 +41,1003 +42,986 +43,980 +44,977 +45,932 +46,829 +47,827 +48,825 +49,810 +50,788 +51,762 +52,725 +53,723 +54,718 +55,696 +56,695 +57,686 +58,686 +59,618 +60,613 +61,589 +62,580 +63,535 +64,531 +65,527 +66,520 +67,510 +68,500 +69,485 +70,470 +71,470 +72,467 +73,467 +74,452 +75,451 +76,394 +77,388 +78,386 +79,371 +80,368 +81,367 +82,345 +83,340 +84,328 +85,327 +86,318 +87,304 +88,292 +89,288 +90,282 +91,276 +92,274 +93,252 +94,247 +95,247 +96,246 +97,245 +98,244 +99,241 +100,231 +101,231 +102,228 +103,225 +104,224 +105,218 +106,217 +107,216 +108,215 +109,214 +110,213 +111,213 +112,210 +113,197 +114,196 +115,193 +116,193 +117,184 +118,178 +119,173 +120,172 +121,170 +122,169 +123,166 +124,165 +125,165 +126,157 +127,155 +128,155 +129,154 +130,151 +131,150 +132,148 +133,148 +134,146 +135,143 +136,142 +137,139 +138,138 +139,136 +140,132 +141,132 +142,131 +143,128 +144,127 +145,123 +146,121 +147,119 +148,116 +149,107 +150,105 +151,102 +152,99 +153,95 +154,94 +155,94 +156,92 +157,90 +158,85 +159,83 +160,80 +161,79 +162,77 +163,76 +164,75 +165,75 +166,74 +167,72 +168,71 +169,71 +170,69 +171,69 +172,69 +173,68 +174,67 +175,67 +176,67 +177,66 +178,64 +179,64 +180,64 +181,63 +182,61 +183,58 +184,58 +185,58 +186,57 +187,57 +188,54 +189,53 +190,53 +191,53 +192,53 +193,52 +194,51 +195,50 +196,50 +197,49 +198,49 +199,48 +200,46 +201,46 +202,46 +203,45 +204,45 +205,44 +206,44 +207,44 +208,43 +209,43 +210,40 +211,40 +212,37 +213,37 +214,35 +215,34 +216,34 +217,34 +218,34 +219,33 +220,32 +221,31 +222,31 +223,31 +224,30 +225,30 +226,30 +227,30 +228,29 +229,29 +230,28 +231,28 +232,28 +233,28 +234,27 +235,27 +236,25 +237,25 +238,24 +239,24 +240,24 +241,24 +242,23 +243,23 +244,23 +245,22 +246,22 +247,21 +248,21 +249,20 +250,20 +251,20 +252,20 +253,19 +254,19 +255,19 +256,18 +257,17 +258,17 +259,17 +260,16 +261,16 +262,16 +263,15 +264,15 +265,15 +266,14 +267,13 +268,13 +269,12 +270,12 +271,11 +272,11 +273,11 +274,11 +275,11 +276,10 +277,10 +278,10 +279,10 +280,10 +281,9 +282,9 +283,9 +284,9 +285,9 +286,8 +287,8 +288,8 +289,8 +290,8 +291,8 +292,7 +293,7 +294,7 +295,7 +296,7 +297,6 +298,6 +299,6 +300,6 +301,6 +302,6 +303,6 +304,6 +305,6 +306,6 +307,5 +308,5 +309,5 +310,5 +311,5 +312,5 +313,5 +314,5 +315,5 +316,5 +317,4 +318,4 +319,4 +320,4 +321,4 +322,4 +323,4 +324,4 +325,4 +326,4 +327,4 +328,3 +329,3 +330,3 +331,3 +332,3 +333,3 +334,3 +335,3 +336,3 +337,3 +338,3 +339,3 +340,3 +341,3 +342,3 +343,3 +344,3 +345,3 +346,3 +347,3 +348,2 +349,2 +350,2 +351,2 +352,2 +353,2 +354,2 +355,2 +356,2 +357,2 +358,2 +359,2 +360,2 +361,2 +362,2 +363,2 +364,2 +365,2 +366,2 +367,1 +368,1 +369,1 +370,1 +371,1 +372,1 +373,1 +374,1 +375,1 +376,1 +377,1 +378,1 +379,1 +380,1 +381,1 +382,1 +383,1 +384,1 +385,1 +386,1 +387,1 +388,1 +389,1 +390,1 +391,1 +392,1 +393,1 +394,1 +395,1 +396,1 +397,1 +398,1 +399,1 +400,1 +401,1 +402,1 +403,1 +404,1 +405,1 +406,1 +407,1 +408,1 +409,1 +410,1 +411,1 +412,1 +413,1 +414,1 +415,1 +416,1 +417,1 diff --git a/db/planguard/fixture/testdata/packets_types.csv b/db/planguard/fixture/testdata/packets_types.csv new file mode 100644 index 0000000..3f66881 --- /dev/null +++ b/db/planguard/fixture/testdata/packets_types.csv @@ -0,0 +1,65 @@ +payload_type,route_type,n,n_channel,n_origin,n_trace,n_scoped +0,0,56902,0,0,0,33625 +0,1,216649,0,0,0,0 +0,2,305800,0,0,0,0 +0,3,273,0,0,0,0 +1,0,13603,0,0,0,11080 +1,1,56577,0,0,0,0 +1,2,336219,0,0,0,0 +1,3,214,0,0,0,0 +2,0,25969,0,0,0,19887 +2,1,71395,0,0,0,0 +2,2,107283,0,0,0,0 +2,3,302,0,0,0,25 +3,0,1321,0,0,0,780 +3,1,11485,0,0,0,0 +3,2,16660,0,0,0,0 +3,3,209,0,0,0,1 +4,0,33400,0,32402,0,24051 +4,1,218006,0,207896,0,0 +4,2,213324,0,212920,0,0 +4,3,279,0,21,0,0 +5,0,133652,133649,0,0,91763 +5,1,265744,265742,0,0,0 +5,2,268,267,0,0,0 +5,3,252,252,0,0,0 +6,0,814,0,0,0,150 +6,1,38692,0,0,0,0 +6,2,32987,0,0,0,0 +6,3,191,0,0,0,0 +7,0,11229,0,11149,0,7825 +7,1,77425,0,77165,0,0 +7,2,75763,0,75702,0,0 +7,3,256,0,155,0,0 +8,0,17103,0,0,0,13261 +8,1,73285,0,0,0,0 +8,2,33702,0,0,0,0 +8,3,182,0,0,0,0 +9,0,276,0,0,262,0 +9,1,696,0,0,641,0 +9,2,114280,0,0,114267,0 +9,3,202,0,0,184,0 +10,0,263,0,0,0,0 +10,1,267,0,0,0,0 +10,2,6202,0,0,0,0 +10,3,174,0,0,0,0 +11,0,290,0,0,0,0 +11,1,344,0,0,0,0 +11,2,470256,0,0,0,0 +11,3,265,0,0,0,0 +12,0,236,0,0,0,0 +12,1,915,0,0,0,0 +12,2,157,0,0,0,0 +12,3,245,0,0,0,0 +13,0,245,0,0,0,0 +13,1,316,0,0,0,0 +13,2,204,0,0,0,0 +13,3,192,0,0,0,0 +14,0,163,0,0,0,0 +14,1,219,0,0,0,0 +14,2,185,0,0,0,0 +14,3,168,0,0,0,0 +15,0,218,0,0,0,0 +15,1,163,0,0,0,0 +15,2,3661,0,0,0,0 +15,3,441,0,0,0,0 diff --git a/db/planguard/fixture/testdata/po_hop.csv b/db/planguard/fixture/testdata/po_hop.csv new file mode 100644 index 0000000..4ced3da --- /dev/null +++ b/db/planguard/fixture/testdata/po_hop.csv @@ -0,0 +1,65 @@ +hop_count,n +0,118900 +1,52904 +2,82862 +3,117223 +4,128955 +5,126966 +6,115964 +7,103687 +8,92240 +9,71647 +10,60424 +11,51601 +12,44264 +13,37774 +14,32232 +15,28084 +16,24295 +17,20971 +18,18141 +19,15857 +20,14067 +21,12756 +22,9989 +23,8665 +24,7776 +25,7081 +26,6317 +27,5794 +28,5308 +29,4691 +30,4304 +31,3980 +32,3722 +33,2695 +34,2489 +35,2284 +36,2114 +37,2030 +38,1728 +39,1585 +40,1397 +41,1362 +42,1248 +43,1127 +44,1158 +45,1001 +46,935 +47,875 +48,818 +49,758 +50,724 +51,638 +52,637 +53,586 +54,615 +55,518 +56,416 +57,398 +58,372 +59,373 +60,320 +61,305 +62,274 +63,217 diff --git a/db/planguard/fixture/testdata/po_iata.csv b/db/planguard/fixture/testdata/po_iata.csv new file mode 100644 index 0000000..f6a66b4 --- /dev/null +++ b/db/planguard/fixture/testdata/po_iata.csv @@ -0,0 +1,562 @@ +iata,n +SEA,2738907 +SYD,560564 +AMS,548213 +PDX,543852 +STR,507198 +RTM,472387 +BNA,441241 +MOW,392317 +BWE,383973 +LAX,361268 +SJC,341707 +BER,330543 +RDU,328351 +PRG,326288 +PAE,326094 +ORD,320066 +PAD,311706 +MEL,290742 +ATL,287105 +BRQ,286567 +DEN,280926 +FRA,277125 +CGN,264737 +SAN,259926 +BOS,257066 +SCE,254507 +ENS,254020 +DUS,246949 +BDL,242958 +DTM,242745 +AUS,240515 +TYS,230295 +CHA,226027 +MSN,221502 +CVG,215917 +YVR,215893 +YKF,201449 +BLI,192961 +IDA,188472 +AKL,187066 +EIN,186458 +NUE,185360 +BRN,185211 +LYS,184925 +OAK,184080 +UTC,181104 +SCN,167262 +WRO,163106 +SPM,157977 +TRI,157577 +BOI,153227 +HAM,152605 +VNO,152449 +RIX,150697 +MUC,147463 +FNL,147211 +KRK,147187 +SLC,142417 +SFO,140835 +PAR,139670 +POZ,136395 +KLS,134277 +PSC,133364 +MDT,128666 +GRQ,126655 +GEG,125675 +KTW,125246 +LEY,120244 +UDE,118727 +WAW,117286 +MMX,116818 +FMM,115799 +ABQ,115745 +ALB,109429 +BRU,107831 +VIE,107676 +DAY,107125 +YYZ,105932 +YYC,105759 +IAD,105684 +LEJ,104738 +PIT,104403 +BRS,101542 +LIS,101163 +HAJ,100443 +TLL,100400 +SOF,97899 +PRB,95365 +YUL,94907 +LIN,94012 +KSF,93349 +RDM,92501 +AVL,91662 +SMF,90704 +YOW,88738 +LUX,87588 +CBR,87108 +LGG,85145 +PED,83699 +HRA,82839 +HOH,81632 +MSY,78837 +GLZ,76833 +MAN,76145 +KZG,74743 +GDN,74475 +AAR,72999 +CPH,71900 +PSK,71868 +BTS,71155 +OMA,70080 +LON,70009 +YEG,67929 +ANR,66702 +NTE,66057 +ODE,65653 +GSP,65361 +YKM,63539 +EAT,63042 +DFW,62944 +MST,62419 +ERF,61878 +MRO,61820 +BNE,59667 +WOL,59336 +BUF,58721 +VOD,58417 +WLG,57818 +BHS,57483 +MCE,57471 +WRE,57061 +ESS,56295 +KUN,55621 +SLE,54624 +UHE,54315 +MKE,53955 +BYU,53797 +LWR,53207 +BOH,52806 +SGU,51182 +BHD,50865 +NUW,48262 +HAD,48179 +OSL,47513 +ACH,47330 +NWI,46721 +LCY,46011 +RDO,45886 +NPL,45744 +GSO,45669 +CON,45521 +YYJ,45232 +SBA,44942 +FAT,44272 +TRN,43563 +QAR,43558 +BHX,43153 +PUW,42701 +AAH,41735 +FRD,41721 +COA,41560 +DRS,41556 +KLU,40616 +SAT,39629 +TTD,39531 +COS,37280 +DET,36408 +OXR,35786 +GRR,35584 +CEG,35570 +HEL,35260 +PPM,35136 +GRZ,35092 +YTR,34866 +LRU,34693 +EMA,33970 +JKG,33415 +LCJ,33385 +NYC,33088 +FKB,33047 +EPH,32786 +YQT,32587 +ADL,31516 +CLE,31464 +SJO,30938 +LEX,30273 +ABE,30189 +YCD,30097 +BUD,30066 +HUY,29406 +BCN,29168 +BHM,29087 +OSR,28983 +GEX,28979 +ELP,28861 +RIC,28231 +PIH,27389 +SQL,27359 +NTL,27318 +TRG,27206 +NCE,26749 +TGN,26507 +TRF,26363 +BLL,26225 +WIE,26111 +GPT,26074 +YQB,24881 +AHN,24743 +PNS,24608 +OXF,24587 +MLB,24398 +GOT,24158 +TWF,23878 +LOZ,23716 +EUG,23703 +CAX,23269 +TPA,23106 +VIS,22670 +NSN,22625 +CRW,22534 +STN,22127 +ROA,21665 +STS,21458 +ROV,21443 +KLR,21144 +CBG,20996 +CLT,20628 +SZZ,20531 +TSR,20513 +FLO,20480 +HOU,20165 +PUB,20131 +LGU,20081 +EDI,19810 +AGS,19788 +ONT,19727 +RNO,19646 +IND,19274 +PHL,19228 +WRT,18984 +SBP,18893 +BLA,18029 +OBF,17959 +ABX,17536 +MSP,17392 +MAD,17308 +GLA,17286 +SXB,17196 +LTN,17134 +MIM,16849 +SGD,16794 +MCO,16687 +ORY,16486 +CPT,16291 +RST,16173 +DRO,15913 +LAS,15813 +ALH,15321 +MME,14900 +BLE,14872 +GHV,14719 +EGE,14424 +DLE,14324 +KTN,14226 +RYG,13995 +TLS,13702 +DAB,13697 +BOD,13691 +FNB,13650 +WGA,13595 +ROT,13552 +ORB,13401 +ACY,13323 +CAE,13043 +KEL,13041 +MPL,12940 +ORL,12618 +AAL,12597 +LRT,12562 +BZG,12481 +ATW,12343 +OPO,12115 +PDV,12028 +PMR,11640 +PSP,11513 +MGL,11491 +CVO,11446 +MHG,11192 +SQQ,11030 +QDV,10935 +SVC,10755 +FPR,10695 +GRU,10640 +BEG,10619 +ALL,10593 +LUZ,10454 +DUB,10379 +SOD,10312 +CMH,10258 +OOL,10044 +ZNO,9840 +YWG,9805 +NOC,9765 +CRL,9490 +PER,9302 +OTP,9122 +SNN,9020 +EXT,9018 +MFR,9000 +STO,8974 +PAH,8811 +HLG,8353 +SRQ,8343 +OSI,8287 +KUF,8279 +SJU,8213 +LNZ,8034 +TRD,7945 +RMU,7687 +INV,7677 +TAT,7640 +DND,7550 +IAS,7463 +FWA,7259 +TMP,7137 +GOJ,6838 +HDB,6782 +KSD,6753 +SYR,6729 +SKE,6645 +PQQ,6607 +GUL,6570 +VCP,6481 +PLQ,6478 +BRE,6432 +VLC,6349 +RKS,6119 +PGD,6009 +PVD,5894 +WVN,5717 +MVH,5671 +TSV,5554 +MCI,5338 +GUF,5323 +LPI,5275 +LBC,5273 +TJM,5221 +GCI,5050 +YHU,5027 +TRO,4904 +JAN,4859 +CFR,4851 +TYR,4779 +ZBM,4733 +SVX,4715 +RMO,4637 +ROC,4628 +ANE,4531 +TUL,4463 +GTI,4454 +WBG,4444 +LFT,4431 +CPQ,4418 +TXK,4405 +YQL,4386 +FAR,4358 +KSC,4289 +ALC,4172 +CHC,4161 +CWA,4117 +APF,4071 +BJP,3982 +GTP,3960 +GOA,3945 +PHX,3909 +PRC,3700 +KJK,3654 +MMV,3639 +UME,3636 +YXX,3629 +BGR,3606 +KZN,3533 +LRF,3505 +LIL,3417 +MLU,3377 +NTU,3303 +LJU,3212 +HTS,3077 +ABI,3040 +YRQ,2986 +TKU,2969 +LSE,2957 +NAP,2945 +LBA,2879 +RKV,2839 +SSZ,2834 +RKD,2815 +PGV,2808 +DSA,2774 +LNS,2737 +YQY,2720 +SCL,2652 +AZO,2650 +RAO,2613 +MFE,2526 +AAB,2481 +RBG,2387 +DUD,2260 +PKB,2248 +BES,2213 +EVV,2157 +SAO,2078 +JAX,2028 +RNB,1999 +SDF,1996 +PIA,1981 +TWB,1968 +SJT,1894 +ATH,1874 +CAK,1831 +VST,1745 +CZW,1727 +FLR,1667 +KOK,1662 +SJY,1622 +PSF,1609 +RZE,1609 +YSE,1591 +YHZ,1464 +GDW,1444 +PDK,1416 +BWT,1372 +NYO,1349 +BUY,1347 +CWB,1318 +RUT,1300 +MUE,1294 +MOT,1250 +ITA,1232 +BYG,1192 +ROK,1188 +RFD,1145 +MBS,1120 +BWI,1049 +YVE,1020 +NOD,1004 +RLG,953 +ORM,936 +RDZ,930 +JTC,926 +MIA,909 +VXO,909 +AOH,903 +LAA,884 +SDR,825 +YGK,821 +SIR,798 +PLO,792 +GJT,765 +TLH,758 +RDD,706 +OGG,703 +YKA,703 +BBH,700 +EWN,692 +VAP,674 +YML,618 +GTR,586 +DIQ,579 +YTF,575 +MNN,535 +VLY,497 +MQL,490 +UKK,448 +LMT,426 +PLR,423 +YYY,422 +SMN,369 +OKC,359 +LBB,347 +LCQ,326 +BFL,320 +GOR,320 +AMA,315 +YXU,305 +STL,303 +IQQ,299 +DNQ,286 +CEB,275 +EME,275 +DPO,272 +OUL,261 +XCM,253 +SDL,246 +YQA,242 +MEM,232 +FNT,230 +PFB,224 +YLK,216 +AIZ,201 +SVG,195 +PSH,188 +ABC,186 +LOI,180 +WYA,167 +ONP,156 +VAG,150 +MMH,147 +OSD,137 +VRZ,135 +MSV,127 +LWB,104 +YQF,98 +BFD,96 +PMI,90 +POA,87 +DXB,86 +ELM,80 +GPZ,80 +LCH,74 +YQQ,73 +BGM,71 +PHF,64 +ANC,62 +ALF,56 +HYR,50 +RSW,50 +EAU,48 +SBN,46 +JER,43 +HVN,40 +TGD,40 +YQS,40 +AXN,39 +MAC,35 +SBY,29 +CMI,25 +RJK,25 +BOJ,19 +DEC,18 +YHM,17 +NOA,12 +VBY,11 +NHZ,8 +LUG,7 +XMS,7 +BHZ,6 +DVN,6 +LPL,6 +MGM,6 +DSM,5 +MRY,3 +YYB,3 +IKT,2 +LGA,2 +VOG,2 +XYZ,2 +CGI,1 +LKV,1 +LRH,1 +YPQ,1 diff --git a/db/planguard/fixture/testdata/po_observer.csv b/db/planguard/fixture/testdata/po_observer.csv new file mode 100644 index 0000000..e27b860 --- /dev/null +++ b/db/planguard/fixture/testdata/po_observer.csv @@ -0,0 +1,3559 @@ +rank,n,home_iata +1,55365,PAE +2,55146,PAE +3,54953,SEA +4,54734,PAE +5,54543,SEA +6,52957,SEA +7,52273,SEA +8,52123,SEA +9,51999,SEA +10,51646,SEA +11,51193,PAE +12,51168,SEA +13,50984,SEA +14,50947,SEA +15,50830,SEA +16,50811,BLI +17,50618,SEA +18,50547,SEA +19,50540,SEA +20,50520,PDX +21,50301,SEA +22,50246,SEA +23,50200,PDX +24,50043,AUS +25,50004,BLI +26,49958,SEA +27,49927,SEA +28,49823,SEA +29,49805,SEA +30,49468,SEA +31,49466,SEA +32,49438,SEA +33,49380,SEA +34,49234,SEA +35,49125,SEA +36,48884,SEA +37,48683,SEA +38,48621,SEA +39,48394,PAE +40,48262,NUW +41,47774,SJC +42,47626,KLS +43,47134,SEA +44,46953,SEA +45,46750,PDX +46,46457,PDX +47,46380,SEA +48,46252,KLS +49,46217,SEA +50,46032,SEA +51,45750,SJC +52,45518,PDX +53,45232,YYJ +54,45163,SEA +55,44749,SEA +56,44699,SEA +57,44659,SEA +58,44365,SEA +59,43903,SEA +60,43823,SEA +61,43537,SEA +62,43488,YVR +63,43452,SEA +64,43436,PDX +65,43431,PDX +66,42995,SEA +67,42860,PDX +68,41935,PDX +69,41721,FRD +70,41624,YVR +71,41141,YKM +72,40399,KLS +73,40030,SEA +74,39809,GEG +75,39759,EAT +76,39667,GEG +77,39127,PDX +78,39115,BLI +79,38823,GEG +80,38222,PSC +81,37418,TTD +82,37303,SEA +83,37191,PDX +84,37096,PRG +85,36665,SLE +86,36549,SEA +87,36363,FRA +88,35575,SCE +89,35455,SEA +90,34795,YVR +91,34794,SEA +92,34694,BWE +93,34353,SEA +94,33714,SCE +95,33664,BLI +96,33531,COA +97,33230,AKL +98,33127,SCE +99,32876,PSC +100,32786,EPH +101,32698,SCE +102,32609,BRN +103,32587,YQT +104,32422,PAE +105,32359,AMS +106,32233,MCE +107,31969,PRG +108,31834,RDM +109,31764,SCE +110,31667,SEA +111,31597,SCE +112,31511,SYD +113,31407,RDM +114,31169,SYD +115,30974,GLZ +116,30959,SEA +117,30916,SFO +118,30511,BRQ +119,30402,VOD +120,30044,BRQ +121,29956,SYD +122,29802,BRN +123,29787,PSC +124,29585,SEA +125,29518,SYD +126,29450,PUW +127,29441,PRG +128,29438,PRG +129,29434,BRQ +130,29342,BRQ +131,29299,HRA +132,29260,RDM +133,29221,SYD +134,29029,PRG +135,29023,PED +136,28983,OSR +137,28979,PRG +138,28846,AKL +139,28840,PAE +140,28675,SJC +141,28620,BRN +142,28562,HRA +143,28558,RTM +144,28556,WRE +145,28531,PRG +146,28509,SCE +147,28505,WRE +148,28377,PRG +149,28334,DUS +150,28272,BRQ +151,28182,PRB +152,28160,PDX +153,28128,BRQ +154,28110,PED +155,28068,ATL +156,28015,VOD +157,27916,RTM +158,27909,UHE +159,27895,PRG +160,27824,PRG +161,27803,HOH +162,27733,SEA +163,27732,AVL +164,27703,PRG +165,27685,SEA +166,27490,EIN +167,27397,AKL +168,27359,SQL +169,27301,CHA +170,27248,AKL +171,27200,SYD +172,27189,CGN +173,27165,TRG +174,27155,PSK +175,26985,SYD +176,26952,BRQ +177,26931,FMM +178,26903,BRQ +179,26866,RDU +180,26858,AKL +181,26632,BRQ +182,26627,MDT +183,26566,PED +184,26540,HOH +185,26520,SJC +186,26454,CGN +187,26406,UHE +188,26403,DUS +189,26220,STR +190,26168,SYD +191,26160,DUS +192,26141,FRA +193,26123,MOW +194,26111,WIE +195,26088,HOH +196,26080,CGN +197,26064,NUE +198,25966,SYD +199,25958,MDT +200,25894,PAD +201,25885,FRA +202,25876,STR +203,25842,FRA +204,25819,DTM +205,25797,SYD +206,25779,TRI +207,25734,RTM +208,25723,SEA +209,25718,SYD +210,25645,AKL +211,25625,CHA +212,25580,UTC +213,25536,DTM +214,25330,TRI +215,25239,FRA +216,25238,MCE +217,25211,ATL +218,25159,TRI +219,25073,OAK +220,25004,MOW +221,24978,HRA +222,24968,DTM +223,24959,NUE +224,24896,ACH +225,24880,CGN +226,24849,CHA +227,24844,CGN +228,24821,MOW +229,24810,SJC +230,24774,OAK +231,24743,AHN +232,24712,SFO +233,24650,AMS +234,24601,CHA +235,24587,OXF +236,24526,LAX +237,24470,MOW +238,24381,MUC +239,24341,RTM +240,24330,SEA +241,24314,STR +242,24283,SEA +243,24250,RTM +244,24219,SYD +245,24143,RTM +246,24104,KZG +247,24091,MOW +248,24072,BNA +249,24034,AMS +250,24029,TRI +251,24025,SJC +252,24013,LCY +253,23876,TYS +254,23860,BRN +255,23788,BNA +256,23743,AMS +257,23735,TYS +258,23723,BNA +259,23716,LOZ +260,23668,OAK +261,23606,ATL +262,23590,SFO +263,23559,CHA +264,23511,BNA +265,23494,TYS +266,23485,BNA +267,23480,MOW +268,23472,PRB +269,23449,DTM +270,23438,STR +271,23437,BRN +272,23426,LEY +273,23408,GSP +274,23392,AMS +275,23354,STR +276,23335,BNA +277,23331,BHX +278,23331,SEA +279,23331,RTM +280,23311,BRS +281,23309,PRB +282,23299,BNA +283,23283,EAT +284,23257,TYS +285,23213,SYD +286,23196,FRA +287,23193,UTC +288,23182,SYD +289,23176,RDU +290,23132,STR +291,23130,NUE +292,23080,AVL +293,23075,STR +294,23061,UTC +295,23042,YKF +296,23035,SYD +297,23013,UTC +298,22961,SJC +299,22940,BNA +300,22938,SYD +301,22902,SJC +302,22878,CHA +303,22794,LON +304,22670,VIS +305,22644,SJC +306,22578,OAK +307,22534,CRW +308,22507,VIE +309,22502,TYS +310,22481,BNA +311,22475,SFO +312,22440,TYS +313,22406,ACH +314,22401,BRS +315,22398,YKM +316,22393,AMS +317,22380,AMS +318,22348,PSK +319,22304,TYS +320,22281,OAK +321,22277,RTM +322,22271,MOW +323,22230,LIS +324,22202,DUS +325,22155,FAT +326,22138,PIT +327,22138,ATL +328,22132,MRO +329,22127,STN +330,22114,BNA +331,22096,MOW +332,22089,BOH +333,22056,BNA +334,22053,LAX +335,22049,AMS +336,22007,STR +337,22006,RDU +338,21981,PSK +339,21976,EIN +340,21953,FMM +341,21898,TRI +342,21888,BNA +343,21853,BNA +344,21851,RTM +345,21846,MST +346,21844,QAR +347,21805,AMS +348,21793,MOW +349,21781,DTM +350,21774,BWE +351,21756,BHS +352,21747,FAT +353,21714,QAR +354,21621,AMS +355,21539,STR +356,21526,ROA +357,21508,MDT +358,21481,DTM +359,21459,CGN +360,21458,STS +361,21418,MOW +362,21406,BRS +363,21374,SAN +364,21366,AMS +365,21363,UTC +366,21359,BRS +367,21320,CGN +368,21319,IAD +369,21318,DTM +370,21310,SFO +371,21292,BNA +372,21270,BWE +373,21262,DUS +374,21191,BWE +375,21169,MOW +376,21136,GSO +377,21135,RTM +378,21121,PAD +379,21098,OAK +380,21058,LCY +381,21039,RTM +382,20998,SJC +383,20995,BNA +384,20979,STR +385,20954,UDE +386,20940,KSF +387,20926,STR +388,20920,CBG +389,20913,TYS +390,20903,STR +391,20901,SYD +392,20898,AMS +393,20885,BNA +394,20871,SYD +395,20863,SYD +396,20857,LGG +397,20855,PAD +398,20842,BNA +399,20794,MUC +400,20791,AMS +401,20733,LAX +402,20731,ATL +403,20722,LAX +404,20708,BNA +405,20688,KSF +406,20666,SYD +407,20664,MOW +408,20659,DUS +409,20628,MST +410,20625,BWE +411,20611,RIX +412,20560,FRA +413,20555,RDU +414,20524,AMS +415,20518,CHA +416,20515,IAD +417,20497,AMS +418,20483,KZG +419,20445,NUE +420,20445,IAD +421,20440,MAN +422,20402,PRB +423,20400,LON +424,20390,BWE +425,20383,OAK +426,20304,ALB +427,20293,BWE +428,20285,UTC +429,20284,PAD +430,20283,STR +431,20243,ATL +432,20240,SYD +433,20227,LGG +434,20225,BWE +435,20204,OAK +436,20183,GSO +437,20153,PIH +438,20095,SJC +439,20092,AVL +440,20022,DTM +441,19992,VIE +442,19942,UDE +443,19915,PAD +444,19889,WOL +445,19886,LAX +446,19873,CHA +447,19854,SAN +448,19853,CHA +449,19837,FMM +450,19822,BHX +451,19809,ATL +452,19784,SAN +453,19754,LAX +454,19727,ONT +455,19723,LON +456,19715,SAN +457,19708,ENS +458,19641,DUS +459,19640,WOL +460,19623,RTM +461,19623,BWE +462,19604,BNA +463,19595,PSC +464,19593,BYU +465,19585,NUE +466,19583,SEA +467,19582,AMS +468,19558,SEA +469,19513,RTM +470,19510,LAX +471,19483,LAX +472,19459,MDT +473,19421,BWE +474,19392,EIN +475,19376,TYS +476,19369,CGN +477,19367,BLI +478,19365,AVL +479,19358,BOS +480,19313,LAX +481,19255,CBR +482,19191,UDE +483,19183,ENS +484,19171,LAX +485,19128,LAX +486,19123,SAN +487,19118,AMS +488,19082,MEL +489,19006,STR +490,18984,WRT +491,18981,MEL +492,18972,EIN +493,18950,FNL +494,18939,LAX +495,18895,AMS +496,18893,CBR +497,18876,SAN +498,18851,ATL +499,18848,EIN +500,18848,BWE +501,18843,ENS +502,18838,KTW +503,18787,SAN +504,18780,MOW +505,18773,SAN +506,18767,SAN +507,18754,BWE +508,18748,ATL +509,18746,WOL +510,18740,EIN +511,18718,BRQ +512,18688,RDU +513,18684,BRU +514,18650,UDE +515,18605,SEA +516,18572,LAX +517,18565,EIN +518,18563,RIC +519,18545,UTC +520,18545,NWI +521,18541,CBR +522,18525,ATL +523,18498,CBR +524,18491,CBR +525,18477,RDU +526,18476,RTM +527,18472,KSF +528,18437,BRU +529,18382,YKF +530,18353,FRA +531,18222,PAD +532,18189,SAN +533,18079,BOS +534,18076,LAX +535,18055,SPM +536,18041,DEN +537,18029,BLA +538,17997,LAX +539,17968,LWR +540,17959,OBF +541,17959,SLE +542,17946,ENS +543,17939,LWR +544,17938,TRI +545,17898,SCE +546,17897,PAD +547,17857,CEG +548,17852,MAN +549,17846,ESS +550,17834,LEY +551,17816,EMA +552,17813,LAX +553,17812,SFO +554,17812,SEA +555,17783,LYS +556,17780,SBA +557,17763,OXR +558,17754,MRO +559,17690,AMS +560,17668,WLG +561,17656,LGG +562,17580,SPM +563,17539,LIN +564,17532,BOS +565,17529,ABX +566,17497,SBA +567,17451,RTM +568,17438,IDA +569,17435,RDU +570,17431,GSP +571,17408,FMM +572,17381,MUC +573,17351,AKL +574,17351,ENS +575,17324,ENS +576,17308,BRU +577,17298,WLG +578,17260,KSF +579,17247,SAN +580,17243,LYS +581,17206,NUE +582,17185,ENS +583,17159,ENS +584,17158,SAN +585,17153,AMS +586,17143,CGN +587,17134,LTN +588,17130,BER +589,17111,MEL +590,17094,AMS +591,17091,MDT +592,17066,ALB +593,17043,RTM +594,17040,SYD +595,17021,ATL +596,17012,RDU +597,17006,SPM +598,16977,GEX +599,16976,SEA +600,16969,CHA +601,16968,BRN +602,16968,TGN +603,16941,FRA +604,16936,ENS +605,16920,IDA +606,16899,STR +607,16849,MIM +608,16839,DUS +609,16835,BWE +610,16818,AMS +611,16811,MOW +612,16792,BTS +613,16786,BHS +614,16740,BER +615,16718,RDU +616,16705,ENS +617,16687,MCO +618,16681,LEY +619,16672,NWI +620,16648,BOS +621,16635,LAX +622,16634,BHS +623,16627,UDE +624,16625,LEY +625,16489,YKF +626,16486,BDL +627,16453,MEL +628,16386,MEL +629,16382,LIN +630,16346,BDL +631,16343,LYS +632,16335,AUS +633,16270,KRK +634,16242,BRN +635,16186,WLG +636,16166,SCN +637,16163,SBP +638,16154,EMA +639,16151,YVR +640,16128,IDA +641,16101,YKF +642,16097,BDL +643,16089,SPM +644,16080,GRQ +645,16079,AMS +646,16047,RDU +647,16034,DEN +648,16023,LEX +649,16018,RDU +650,16016,DUS +651,16012,BTS +652,15981,IAD +653,15928,MOW +654,15923,KRK +655,15922,KSF +656,15919,BOS +657,15913,DRO +658,15899,PAD +659,15873,ALB +660,15871,MEL +661,15849,DTM +662,15842,AAH +663,15833,MAN +664,15827,SPM +665,15777,PAD +666,15774,DUS +667,15771,BRU +668,15762,NTL +669,15754,FRA +670,15748,GRQ +671,15737,LWR +672,15706,KRK +673,15703,SLC +674,15672,CON +675,15648,BWE +676,15639,BRU +677,15613,BER +678,15597,BOS +679,15569,BDL +680,15560,SPM +681,15560,AMS +682,15554,BTS +683,15546,BDL +684,15542,LGG +685,15526,YKF +686,15515,MUC +687,15509,GRQ +688,15498,NPL +689,15494,YKF +690,15487,ENS +691,15458,ABQ +692,15448,BWE +693,15394,STR +694,15367,BOH +695,15350,BOH +696,15348,PAD +697,15323,YKF +698,15322,EUG +699,15288,RST +700,15244,MOW +701,15236,PAD +702,15227,YKF +703,15209,ALB +704,15182,ALB +705,15168,DUS +706,15135,PAD +707,15132,VNO +708,15119,KRK +709,15101,YKF +710,15065,OXR +711,15039,RDU +712,15034,PDX +713,15024,ABQ +714,15019,RTM +715,15018,BOS +716,15004,WRO +717,14993,SCN +718,14987,DEN +719,14985,FMM +720,14974,LYS +721,14966,UTC +722,14950,BOI +723,14899,BER +724,14886,MUC +725,14878,BYU +726,14834,LIN +727,14734,RTM +728,14670,FMM +729,14669,BOS +730,14662,GSP +731,14634,PAD +732,14624,SJC +733,14609,RTM +734,14608,LEJ +735,14603,ABQ +736,14601,BDL +737,14593,AAH +738,14591,GRQ +739,14587,BER +740,14544,MEL +741,14535,BOI +742,14486,SPM +743,14479,BOS +744,14461,NPL +745,14460,AAR +746,14424,EGE +747,14364,ATL +748,14358,KZG +749,14358,BER +750,14347,IDA +751,14343,ABQ +752,14324,DLE +753,14288,HOU +754,14269,TRN +755,14226,KTN +756,14224,BDL +757,14223,DEN +758,14215,WAW +759,14200,BOS +760,14191,BOI +761,14190,IDA +762,14172,COS +763,14160,NPL +764,14157,ENS +765,14155,LEY +766,14121,DEN +767,14111,PIT +768,14099,KTW +769,14098,PIT +770,14077,BDL +771,14061,DUS +772,14048,KLU +773,14026,IDA +774,14006,DEN +775,13997,IDA +776,13989,BDL +777,13963,ANR +778,13893,DEN +779,13886,RDU +780,13874,POZ +781,13810,TRN +782,13784,KLU +783,13769,ANR +784,13767,RDO +785,13726,BER +786,13718,DTM +787,13665,LIN +788,13637,BWE +789,13635,VIE +790,13621,LIN +791,13617,KRK +792,13599,YKF +793,13595,WGA +794,13593,MOW +795,13566,DEN +796,13552,ROT +797,13525,IDA +798,13521,MDT +799,13521,DEN +800,13497,RDO +801,13473,KRK +802,13465,FNL +803,13464,DEN +804,13440,CAX +805,13431,BER +806,13419,FNL +807,13411,PAR +808,13398,TRI +809,13383,KTW +810,13372,HAJ +811,13359,IAD +812,13348,KRK +813,13313,LEJ +814,13307,PIT +815,13301,GRQ +816,13272,SOF +817,13270,WRO +818,13262,IDA +819,13258,CON +820,13251,LIN +821,13246,GLZ +822,13243,BER +823,13172,FNL +824,13133,TYS +825,13131,BDL +826,13124,BOS +827,13118,SCN +828,13117,BOI +829,13105,KRK +830,13092,ENS +831,13072,BER +832,13065,BRS +833,13061,AAR +834,13052,ALB +835,13034,FNL +836,13027,MAN +837,13015,RIX +838,12986,BER +839,12983,MRO +840,12980,WAW +841,12974,DEN +842,12965,RIX +843,12940,MPL +844,12895,DTM +845,12884,PSC +846,12872,SPM +847,12850,DEN +848,12844,ABQ +849,12838,MLB +850,12834,RDU +851,12833,WAW +852,12823,IDA +853,12810,DEN +854,12797,BWE +855,12784,KLU +856,12782,MMX +857,12751,STR +858,12742,LYS +859,12710,LEJ +860,12699,WRO +861,12695,LYS +862,12690,SCN +863,12687,STR +864,12671,KTW +865,12662,LYS +866,12657,FNL +867,12647,BWE +868,12618,ORL +869,12610,ATL +870,12587,BOI +871,12566,LYS +872,12557,BOI +873,12539,BOI +874,12492,IAD +875,12465,AUS +876,12464,COS +877,12451,LEJ +878,12443,ABQ +879,12414,BOI +880,12395,GRZ +881,12384,RTM +882,12378,LGU +883,12378,LUX +884,12371,BDL +885,12347,SAN +886,12342,ZNO +887,12335,CGN +888,12323,SOF +889,12315,CPH +890,12309,VIE +891,12304,DEN +892,12299,BER +893,12293,BOI +894,12244,CVG +895,12234,DEN +896,12216,STR +897,12213,KTW +898,12197,KZG +899,12197,LUX +900,12187,LEJ +901,12163,RIX +902,12146,POZ +903,12133,HAJ +904,12132,BOS +905,12101,TWF +906,12086,MMX +907,12081,YKF +908,12074,BWE +909,12049,ALB +910,12044,YKF +911,12043,EDI +912,12000,GEX +913,11984,ABQ +914,11976,FNL +915,11974,FNL +916,11971,SLC +917,11968,DAY +918,11957,AMS +919,11949,WAW +920,11927,FKB +921,11907,PIT +922,11896,CPH +923,11893,LEY +924,11886,LCJ +925,11883,ENS +926,11869,CGN +927,11865,BDL +928,11848,BOI +929,11838,BER +930,11836,UDE +931,11830,WAW +932,11823,MST +933,11822,STR +934,11817,PAD +935,11816,SPM +936,11815,WAW +937,11804,GRQ +938,11790,FNL +939,11777,TWF +940,11754,SCN +941,11754,LAX +942,11750,SLC +943,11746,MOW +944,11731,AAR +945,11722,BDL +946,11715,GRQ +947,11705,MUC +948,11694,GLZ +949,11690,ESS +950,11686,BER +951,11681,NUE +952,11676,BDL +953,11665,GRZ +954,11665,BTS +955,11641,LYS +956,11640,PMR +957,11634,LRU +958,11631,BRQ +959,11621,PAR +960,11619,MMX +961,11619,BER +962,11606,VIE +963,11598,CLT +964,11593,SLC +965,11587,KTW +966,11581,BER +967,11578,LRU +968,11560,MLB +969,11523,NSN +970,11513,PUB +971,11494,CAE +972,11493,DEN +973,11491,MGL +974,11490,POZ +975,11463,SLC +976,11463,ESS +977,11446,CVO +978,11436,UDE +979,11413,SLC +980,11412,VNO +981,11376,LIS +982,11351,LAX +983,11327,LEY +984,11319,DAY +985,11312,CGN +986,11291,POZ +987,11288,MEL +988,11282,MMX +989,11277,LRU +990,11256,SJC +991,11235,FRA +992,11209,CEG +993,11209,ODE +994,11205,PSP +995,11205,NTE +996,11192,MHG +997,11150,NTL +998,11130,BTS +999,11126,STR +1000,11118,NYC +1001,11106,LCJ +1002,11102,NSN +1003,11095,SMF +1004,11092,SMF +1005,11083,KTW +1006,11075,VNO +1007,11074,MEL +1008,11053,ERF +1009,11051,WAW +1010,11032,GRZ +1011,11020,LEX +1012,11015,SLC +1013,11006,DEN +1014,11005,PPM +1015,11004,POZ +1016,10999,NUE +1017,10984,CPH +1018,10968,MMX +1019,10962,LIS +1020,10961,BUD +1021,10948,SOF +1022,10904,CPH +1023,10861,SMF +1024,10848,AAR +1025,10846,ODE +1026,10823,VNO +1027,10819,HAJ +1028,10815,GLZ +1029,10813,HUY +1030,10808,RTM +1031,10799,MEL +1032,10785,MSN +1033,10774,NWI +1034,10772,WRO +1035,10768,SLC +1036,10763,LIS +1037,10755,SVC +1038,10742,RTM +1039,10728,YYC +1040,10717,DEN +1041,10714,TLS +1042,10713,LEJ +1043,10711,LIS +1044,10697,AAH +1045,10696,MEL +1046,10695,FPR +1047,10668,POZ +1048,10664,MEL +1049,10658,PIT +1050,10643,COS +1051,10638,SOF +1052,10619,BEG +1053,10608,BER +1054,10593,ALL +1055,10592,LYS +1056,10584,SQQ +1057,10579,ERF +1058,10575,AUS +1059,10571,KTW +1060,10569,RTM +1061,10561,LUX +1062,10560,KTW +1063,10546,MMX +1064,10545,PAR +1065,10500,YEG +1066,10491,VNO +1067,10483,CVG +1068,10474,VNO +1069,10454,LUZ +1070,10431,HUY +1071,10431,BER +1072,10417,MEL +1073,10361,RIX +1074,10357,LCJ +1075,10339,MEL +1076,10338,AUS +1077,10335,MOW +1078,10327,YYC +1079,10320,WRO +1080,10317,RIX +1081,10295,MEL +1082,10268,SOF +1083,10259,HAM +1084,10241,KTW +1085,10212,MSN +1086,10200,AUS +1087,10188,PAD +1088,10165,HAJ +1089,10159,KUN +1090,10151,LYS +1091,10143,CVG +1092,10108,BHD +1093,10099,MEL +1094,10097,LUX +1095,10094,VNO +1096,10088,BOI +1097,10075,VNO +1098,10071,TPA +1099,10070,ODE +1100,10059,MEL +1101,10055,POZ +1102,10055,ENS +1103,10025,MSN +1104,10012,HAM +1105,10011,GLZ +1106,10003,MEL +1107,10000,RIX +1108,9999,LUX +1109,9998,SCN +1110,9971,WRO +1111,9950,IDA +1112,9945,HAJ +1113,9942,BWE +1114,9939,SEA +1115,9933,YYC +1116,9922,RTM +1117,9921,SLC +1118,9911,KRK +1119,9898,TLL +1120,9875,PPM +1121,9873,ENS +1122,9854,VNO +1123,9852,MEL +1124,9846,SMF +1125,9846,KUN +1126,9830,BOS +1127,9814,IDA +1128,9813,RIX +1129,9801,BUD +1130,9792,CVG +1131,9776,VNO +1132,9775,EIN +1133,9765,NOC +1134,9758,RIX +1135,9757,EIN +1136,9745,SPM +1137,9745,KUN +1138,9719,FRA +1139,9707,WAW +1140,9700,ELP +1141,9699,WRO +1142,9695,PAD +1143,9695,TRN +1144,9690,YVR +1145,9683,AUS +1146,9678,RDU +1147,9669,KUN +1148,9665,ATL +1149,9665,SBA +1150,9650,ELP +1151,9642,IDA +1152,9641,CVG +1153,9641,VNO +1154,9629,KUN +1155,9585,YYZ +1156,9576,EIN +1157,9566,TSR +1158,9548,ABQ +1159,9546,RIX +1160,9539,TGN +1161,9532,WRO +1162,9521,CON +1163,9521,LUX +1164,9511,ELP +1165,9510,FKB +1166,9507,POZ +1167,9498,ABQ +1168,9493,YOW +1169,9484,BDL +1170,9483,SMF +1171,9471,RDU +1172,9457,AMS +1173,9441,LUX +1174,9436,WRO +1175,9426,ODE +1176,9413,AAR +1177,9408,CRL +1178,9408,MEL +1179,9392,DAY +1180,9380,STR +1181,9375,RIC +1182,9360,YYZ +1183,9358,AUS +1184,9358,WRO +1185,9340,OSL +1186,9308,BHD +1187,9290,FRA +1188,9287,SLC +1189,9276,TLL +1190,9271,RIX +1191,9243,LYS +1192,9240,GHV +1193,9232,FNL +1194,9228,HAM +1195,9199,HAM +1196,9169,LGG +1197,9161,BYU +1198,9159,DEN +1199,9137,BOS +1200,9114,SOF +1201,9112,RIX +1202,9096,PAR +1203,9071,CVG +1204,9059,FNL +1205,9048,AUS +1206,9044,BLL +1207,9035,LIS +1208,9020,SNN +1209,9015,PAD +1210,9003,CVG +1211,8988,VNO +1212,8959,BER +1213,8952,SAN +1214,8951,MRO +1215,8943,AUS +1216,8940,BLL +1217,8939,MUC +1218,8917,OTP +1219,8909,CPH +1220,8907,HAJ +1221,8876,PAR +1222,8863,YKF +1223,8848,SCN +1224,8822,PAR +1225,8819,BHD +1226,8817,GDN +1227,8810,MSN +1228,8805,FKB +1229,8767,RDO +1230,8749,DAY +1231,8705,ANR +1232,8680,AUS +1233,8655,GDN +1234,8655,CVG +1235,8640,CVG +1236,8593,TLL +1237,8579,BOS +1238,8578,DAY +1239,8576,SLC +1240,8570,CVG +1241,8569,SEA +1242,8563,CVG +1243,8559,SCE +1244,8557,CLE +1245,8552,BUF +1246,8545,BER +1247,8528,CPH +1248,8500,BUF +1249,8493,CVG +1250,8484,HAM +1251,8468,YYZ +1252,8455,YYZ +1253,8454,BDL +1254,8444,ORY +1255,8440,TYS +1256,8438,TLL +1257,8437,SJO +1258,8430,YYZ +1259,8423,MSN +1260,8422,TLL +1261,8411,SGD +1262,8406,YYZ +1263,8402,LUX +1264,8400,TSR +1265,8400,SCN +1266,8385,TLL +1267,8384,MSN +1268,8383,SGD +1269,8381,EUG +1270,8374,YYZ +1271,8361,SCN +1272,8361,TRF +1273,8360,GDN +1274,8354,MSN +1275,8347,ANR +1276,8339,BHD +1277,8319,AUS +1278,8315,ORD +1279,8314,MMX +1280,8310,KRK +1281,8308,CVG +1282,8302,MUC +1283,8283,CVG +1284,8266,MSN +1285,8262,SJO +1286,8250,LYS +1287,8248,AUS +1288,8242,AUS +1289,8241,BLL +1290,8230,SYD +1291,8217,LIS +1292,8209,SEA +1293,8205,LEJ +1294,8205,SJC +1295,8200,CVG +1296,8199,MSN +1297,8190,GDN +1298,8173,YYZ +1299,8163,MSY +1300,8162,YTR +1301,8155,LYS +1302,8154,PUB +1303,8151,SMF +1304,8137,BOI +1305,8130,BER +1306,8119,DAY +1307,8105,CVG +1308,8090,PIT +1309,8090,MSN +1310,8089,AUS +1311,8087,CGN +1312,8080,TLL +1313,8063,BHD +1314,8060,DAY +1315,8046,STR +1316,8042,ORY +1317,8022,GDN +1318,8007,PPM +1319,8004,HAM +1320,7992,DAY +1321,7983,WRO +1322,7982,ABE +1323,7978,GDN +1324,7971,PAR +1325,7961,FRA +1326,7955,MUC +1327,7953,STR +1328,7944,BUF +1329,7943,DRS +1330,7939,CVG +1331,7939,VNO +1332,7933,YVR +1333,7933,MSN +1334,7909,GDN +1335,7899,BER +1336,7898,MSN +1337,7888,GRQ +1338,7871,CVG +1339,7865,AUS +1340,7857,WRO +1341,7852,GRR +1342,7850,UTC +1343,7845,CVG +1344,7843,MSN +1345,7831,TLL +1346,7814,NCE +1347,7802,MEL +1348,7801,YEG +1349,7789,BUF +1350,7775,OSL +1351,7773,MMX +1352,7769,DET +1353,7767,EDI +1354,7762,YVR +1355,7761,VNO +1356,7756,MEL +1357,7750,STR +1358,7741,MSN +1359,7738,ERF +1360,7731,EIN +1361,7727,BER +1362,7692,GDN +1363,7689,BOS +1364,7683,AAR +1365,7682,GRR +1366,7677,INV +1367,7655,ORD +1368,7645,AMS +1369,7640,TAT +1370,7631,YCD +1371,7629,ESS +1372,7616,CVG +1373,7606,BER +1374,7591,HAJ +1375,7550,LYS +1376,7550,DND +1377,7550,BWE +1378,7512,BOS +1379,7505,SXB +1380,7503,ODE +1381,7502,LIS +1382,7498,MSN +1383,7497,PAH +1384,7488,BYU +1385,7486,AUS +1386,7482,NUE +1387,7475,GRQ +1388,7475,MSN +1389,7473,MSN +1390,7457,ORD +1391,7452,SAN +1392,7444,MMX +1393,7443,BUF +1394,7441,CVG +1395,7425,KRK +1396,7414,LNZ +1397,7405,CLE +1398,7388,ANR +1399,7384,STR +1400,7376,GEG +1401,7356,MEL +1402,7354,YYC +1403,7340,PNS +1404,7304,LEY +1405,7292,ORD +1406,7287,SMF +1407,7285,AUS +1408,7278,ODE +1409,7268,MSN +1410,7265,BER +1411,7263,BDL +1412,7263,AMS +1413,7260,PAR +1414,7249,HAM +1415,7244,CVG +1416,7236,PIH +1417,7224,POZ +1418,7220,PAR +1419,7203,BER +1420,7195,GLA +1421,7181,VIE +1422,7176,NCE +1423,7175,DEN +1424,7174,STR +1425,7159,SCN +1426,7159,DAY +1427,7129,NTE +1428,7117,FNB +1429,7111,ERF +1430,7099,MAN +1431,7092,MMX +1432,7091,AGS +1433,7090,IAS +1434,7084,HAM +1435,7077,BOS +1436,7064,WRO +1437,7062,YEG +1438,7060,RYG +1439,7042,CVG +1440,7017,TLL +1441,7015,PAD +1442,6998,ATL +1443,6994,EXT +1444,6985,ORD +1445,6979,SCN +1446,6975,PAD +1447,6965,STR +1448,6958,YTR +1449,6957,ORD +1450,6947,VIE +1451,6945,ORD +1452,6940,DRS +1453,6935,RYG +1454,6919,HAM +1455,6906,CPH +1456,6897,RIX +1457,6896,TLL +1458,6895,HAM +1459,6886,BER +1460,6880,DRS +1461,6876,GRQ +1462,6870,MKE +1463,6868,RIX +1464,6864,NUE +1465,6857,OSL +1466,6850,MSN +1467,6827,ORD +1468,6825,TYS +1469,6809,KLR +1470,6804,OSL +1471,6799,MSN +1472,6798,YEG +1473,6793,YVR +1474,6782,SOF +1475,6775,RTM +1476,6772,ATL +1477,6738,PAR +1478,6720,OMA +1479,6718,BNE +1480,6714,ORD +1481,6711,MSN +1482,6705,OMA +1483,6698,SOF +1484,6697,BNE +1485,6689,AUS +1486,6687,PAR +1487,6681,POZ +1488,6679,NCE +1489,6671,GDN +1490,6666,WLG +1491,6660,GPT +1492,6645,SKE +1493,6636,ORD +1494,6634,FLO +1495,6633,MKE +1496,6627,PUW +1497,6624,RDU +1498,6621,PUW +1499,6607,PQQ +1500,6606,WRO +1501,6600,COA +1502,6594,YCD +1503,6576,YTR +1504,6573,KUN +1505,6562,ORD +1506,6558,SCN +1507,6558,MSN +1508,6555,LON +1509,6552,TRF +1510,6550,TRF +1511,6528,YYC +1512,6441,PAR +1513,6440,MKE +1514,6433,ORD +1515,6433,SLC +1516,6429,OSL +1517,6422,ORD +1518,6421,SOF +1519,6420,BDL +1520,6417,RDU +1521,6408,PAR +1522,6407,ORD +1523,6389,YYC +1524,6388,CEG +1525,6387,RNO +1526,6381,DEN +1527,6378,ORD +1528,6375,YVR +1529,6373,ORD +1530,6364,FNB +1531,6359,MSN +1532,6354,MSY +1533,6349,BNE +1534,6344,SEA +1535,6334,TRO +1536,6332,MKE +1537,6330,ORD +1538,6329,DAY +1539,6321,SMF +1540,6317,HAJ +1541,6306,YEG +1542,6306,ORD +1543,6301,ORD +1544,6296,PAR +1545,6295,TLL +1546,6290,ORD +1547,6286,YYZ +1548,6282,OSL +1549,6275,BDL +1550,6267,MAD +1551,6262,BCN +1552,6254,ORD +1553,6249,PPM +1554,6240,TLL +1555,6238,SCN +1556,6226,BHD +1557,6222,YEG +1558,6216,PAR +1559,6208,RDU +1560,6204,STR +1561,6203,YVR +1562,6195,ORD +1563,6189,WRO +1564,6180,HAM +1565,6178,MKE +1566,6175,ORD +1567,6172,MOW +1568,6154,ORD +1569,6150,ORD +1570,6139,YYC +1571,6136,BRU +1572,6136,POZ +1573,6132,OMA +1574,6122,YCD +1575,6121,STR +1576,6120,POZ +1577,6119,RKS +1578,6117,SOF +1579,6111,ORD +1580,6109,STR +1581,6103,ERF +1582,6092,BRN +1583,6088,YVR +1584,6081,ORD +1585,6078,ORD +1586,6069,PAR +1587,6054,MSY +1588,6051,ORD +1589,6047,ORD +1590,6031,ORD +1591,6030,SAT +1592,6023,YVR +1593,6021,YVR +1594,6009,SCN +1595,6007,ORD +1596,6001,YVR +1597,5998,WRO +1598,5994,OMA +1599,5982,SYD +1600,5973,BNE +1601,5972,LEJ +1602,5969,YYC +1603,5967,GLA +1604,5961,DEN +1605,5958,YYC +1606,5946,ORD +1607,5939,PNS +1608,5913,GPT +1609,5894,PVD +1610,5877,SLC +1611,5870,YOW +1612,5870,AGS +1613,5867,ORD +1614,5865,NUE +1615,5863,BOS +1616,5862,DRS +1617,5861,BNE +1618,5860,FNL +1619,5846,LAX +1620,5839,YYZ +1621,5813,BHM +1622,5807,RDU +1623,5801,PHL +1624,5777,YUL +1625,5769,CLE +1626,5762,MOW +1627,5756,ROV +1628,5754,MSN +1629,5754,DEN +1630,5752,ORD +1631,5749,YCD +1632,5736,RDO +1633,5734,MKE +1634,5729,DTM +1635,5721,SCN +1636,5718,ORD +1637,5685,ORD +1638,5680,PNS +1639,5671,ORD +1640,5671,MVH +1641,5668,OPO +1642,5655,BDL +1643,5655,DET +1644,5645,CGN +1645,5629,MSY +1646,5627,ERF +1647,5611,PHL +1648,5605,AGS +1649,5604,MKE +1650,5599,PDV +1651,5592,HDB +1652,5574,YVR +1653,5570,AAL +1654,5550,ORD +1655,5534,GRQ +1656,5533,HUY +1657,5530,DET +1658,5527,SGU +1659,5524,DTM +1660,5522,ESS +1661,5511,YUL +1662,5507,BNE +1663,5501,BCN +1664,5495,POZ +1665,5494,NUE +1666,5485,CGN +1667,5482,TRN +1668,5479,GHV +1669,5479,HEL +1670,5474,IDA +1671,5442,BNE +1672,5441,SMF +1673,5440,MSY +1674,5426,LGU +1675,5405,AUS +1676,5393,YOW +1677,5392,DFW +1678,5382,RDU +1679,5366,VIE +1680,5365,ERF +1681,5359,MSN +1682,5358,YUL +1683,5345,OMA +1684,5330,MEL +1685,5323,GUF +1686,5322,BNE +1687,5319,YYZ +1688,5310,ORD +1689,5305,YYC +1690,5305,ORD +1691,5298,YYC +1692,5291,YVR +1693,5287,BER +1694,5285,CAX +1695,5278,ROV +1696,5276,YUL +1697,5273,GPT +1698,5271,DUB +1699,5267,LAS +1700,5255,OMA +1701,5253,SMF +1702,5243,ROV +1703,5239,OMA +1704,5233,MUC +1705,5221,TJM +1706,5212,NYC +1707,5197,EIN +1708,5197,YEG +1709,5189,LIS +1710,5183,MSY +1711,5174,ORD +1712,5166,ROV +1713,5135,LBC +1714,5125,PAD +1715,5121,RDU +1716,5108,YOW +1717,5098,DUB +1718,5097,MSN +1719,5093,GRU +1720,5087,OOL +1721,5080,NCE +1722,5052,BRN +1723,5045,SJO +1724,5027,YHU +1725,5026,BCN +1726,5008,BWE +1727,5008,ANR +1728,5007,NYC +1729,4997,YEG +1730,4962,BWE +1731,4957,OOL +1732,4955,MEL +1733,4941,LUX +1734,4938,MFR +1735,4934,NYC +1736,4931,MSY +1737,4928,ABE +1738,4921,SAT +1739,4907,ORD +1740,4907,ODE +1741,4899,BNE +1742,4899,PDX +1743,4895,HAD +1744,4889,HAJ +1745,4883,STR +1746,4881,HAM +1747,4862,AUS +1748,4855,VNO +1749,4853,ERF +1750,4850,FLO +1751,4841,MSN +1752,4833,BNA +1753,4814,STR +1754,4793,BUF +1755,4788,POZ +1756,4777,YUL +1757,4774,KRK +1758,4773,SAT +1759,4771,YYZ +1760,4760,FLO +1761,4758,YUL +1762,4745,IDA +1763,4744,MSY +1764,4735,DFW +1765,4734,ORD +1766,4733,QDV +1767,4731,ZBM +1768,4728,PDX +1769,4728,VIE +1770,4726,RTM +1771,4725,NYC +1772,4718,LIN +1773,4715,YWG +1774,4702,TRF +1775,4700,YUL +1776,4698,BUF +1777,4695,OMA +1778,4693,YOW +1779,4678,YUL +1780,4677,YEG +1781,4669,MSY +1782,4667,YUL +1783,4639,CPT +1784,4637,RMO +1785,4618,YOW +1786,4607,SAT +1787,4606,YUL +1788,4600,HAD +1789,4600,YQB +1790,4590,PIT +1791,4586,HAD +1792,4580,STO +1793,4579,SGU +1794,4578,RDU +1795,4578,OMA +1796,4566,YUL +1797,4556,SGU +1798,4554,AUS +1799,4544,CAX +1800,4542,ORD +1801,4542,STR +1802,4539,BUF +1803,4538,BOS +1804,4535,BNE +1805,4534,YUL +1806,4526,NTE +1807,4523,BER +1808,4503,PAD +1809,4499,ANR +1810,4495,GSP +1811,4494,NTE +1812,4494,DAB +1813,4494,YUL +1814,4481,YYC +1815,4474,HAM +1816,4469,BCN +1817,4462,WAW +1818,4453,WBG +1819,4451,ROC +1820,4449,HAM +1821,4447,TPA +1822,4446,SAT +1823,4445,SLC +1824,4439,YQB +1825,4435,GPT +1826,4424,BRU +1827,4423,ABE +1828,4420,DAB +1829,4416,WAW +1830,4411,SOF +1831,4392,YYC +1832,4381,ANR +1833,4370,HAD +1834,4337,BHM +1835,4332,TXK +1836,4322,SJO +1837,4319,PHL +1838,4316,YOW +1839,4311,STR +1840,4309,PAD +1841,4302,DET +1842,4295,YUL +1843,4294,DAB +1844,4289,SOD +1845,4289,KSC +1846,4288,OSI +1847,4284,SGU +1848,4283,LIS +1849,4281,SAT +1850,4280,SCN +1851,4274,HAD +1852,4271,HAD +1853,4269,KUF +1854,4269,YEG +1855,4265,YUL +1856,4253,GOJ +1857,4252,YTR +1858,4251,YTR +1859,4251,HAJ +1860,4251,NUE +1861,4250,BNA +1862,4236,FLO +1863,4226,SOD +1864,4223,WAW +1865,4212,SAT +1866,4196,ORD +1867,4189,YOW +1868,4180,BHM +1869,4175,YOW +1870,4170,BCN +1871,4157,DTM +1872,4156,HAM +1873,4156,MSY +1874,4154,SGU +1875,4148,MUC +1876,4138,TLL +1877,4128,RNO +1878,4124,GLA +1879,4119,SCN +1880,4119,RDO +1881,4113,NTE +1882,4111,MSN +1883,4100,YEG +1884,4095,YYC +1885,4095,YYC +1886,4089,HAM +1887,4085,SEA +1888,4082,YYZ +1889,4072,MST +1890,4064,CLT +1891,4062,MFR +1892,4052,STR +1893,4050,RNO +1894,4046,TRI +1895,4031,LAS +1896,4029,MME +1897,4026,RTM +1898,4026,OSL +1899,4021,OAK +1900,4021,BUF +1901,4015,BHM +1902,4013,MDT +1903,4013,LEJ +1904,4011,SPM +1905,4010,KUF +1906,4001,YCD +1907,3990,WRO +1908,3988,MME +1909,3987,SRQ +1910,3985,CVG +1911,3984,OSI +1912,3982,BJP +1913,3979,NTE +1914,3974,YUL +1915,3971,BOI +1916,3971,HAD +1917,3971,CVG +1918,3960,GTP +1919,3931,AMS +1920,3930,WAW +1921,3926,RDU +1922,3924,PDV +1923,3905,PHX +1924,3899,YUL +1925,3898,OMA +1926,3891,GOA +1927,3889,MSY +1928,3882,HAD +1929,3881,NTE +1930,3871,JKG +1931,3869,AUS +1932,3860,ORD +1933,3858,HAD +1934,3858,DFW +1935,3854,HAJ +1936,3837,YQB +1937,3837,MMX +1938,3837,BER +1939,3836,SGU +1940,3818,HAM +1941,3803,SPM +1942,3801,HEL +1943,3798,LRT +1944,3788,ALH +1945,3782,ORD +1946,3780,BUD +1947,3776,LRT +1948,3774,BZG +1949,3769,YUL +1950,3763,YKF +1951,3746,BLE +1952,3741,ALH +1953,3734,ABE +1954,3731,DAY +1955,3730,DRS +1956,3728,BRU +1957,3714,SXB +1958,3690,HAD +1959,3689,BHM +1960,3685,SGU +1961,3681,DET +1962,3680,ORD +1963,3676,MSY +1964,3673,AUS +1965,3673,CLE +1966,3663,OMA +1967,3639,MMV +1968,3631,DAY +1969,3617,MSY +1970,3615,SGU +1971,3612,DRS +1972,3606,HAM +1973,3603,SJU +1974,3601,KZG +1975,3600,YQL +1976,3600,BZG +1977,3594,SCN +1978,3573,GPT +1979,3567,ADL +1980,3566,ADL +1981,3563,SCN +1982,3559,DFW +1983,3553,SGU +1984,3553,ABE +1985,3544,GSO +1986,3542,TPA +1987,3537,YYZ +1988,3535,OMA +1989,3528,SAN +1990,3522,SMF +1991,3521,DFW +1992,3501,ORD +1993,3495,PHL +1994,3490,YOW +1995,3488,OMA +1996,3487,JAN +1997,3480,CON +1998,3469,DFW +1999,3466,DFW +2000,3462,NTE +2001,3453,BRU +2002,3449,MSY +2003,3438,YOW +2004,3433,BLE +2005,3425,AAL +2006,3422,CPQ +2007,3412,NTE +2008,3403,AAL +2009,3402,DEN +2010,3386,SGU +2011,3383,OPO +2012,3377,YOW +2013,3375,KEL +2014,3374,CLT +2015,3369,MUC +2016,3363,HEL +2017,3358,YOW +2018,3356,YYZ +2019,3349,YOW +2020,3340,MME +2021,3337,BHM +2022,3336,CPT +2023,3329,BHM +2024,3324,YUL +2025,3318,MSN +2026,3318,HEL +2027,3312,SCN +2028,3309,MKE +2029,3307,HAM +2030,3301,PIT +2031,3288,YWG +2032,3287,RDU +2033,3276,JKG +2034,3274,MME +2035,3272,SGU +2036,3269,YOW +2037,3262,DFW +2038,3250,VCP +2039,3246,YTR +2040,3245,DFW +2041,3243,FRA +2042,3241,YQB +2043,3240,GSP +2044,3236,ODE +2045,3229,QDV +2046,3221,VNO +2047,3213,ADL +2048,3212,PRC +2049,3212,LJU +2050,3202,MSN +2051,3195,YOW +2052,3191,UTC +2053,3190,VCP +2054,3189,YXX +2055,3185,MEL +2056,3174,PAR +2057,3167,HEL +2058,3165,BUD +2059,3161,SJU +2060,3161,HEL +2061,3160,ALH +2062,3152,SAT +2063,3146,SXB +2064,3138,BNA +2065,3120,ALH +2066,3116,KLR +2067,3116,GRR +2068,3115,CLE +2069,3109,ADL +2070,3088,STR +2071,3084,HAM +2072,3075,MSY +2073,3066,RMU +2074,3066,YQB +2075,3059,MLU +2076,3051,MSP +2077,3048,GRR +2078,3043,BRU +2079,3032,WRO +2080,3023,ADL +2081,3022,GRR +2082,3017,LRT +2083,3013,MUC +2084,3002,MST +2085,2996,YOW +2086,2991,KEL +2087,2987,GRR +2088,2986,YRQ +2089,2980,LEJ +2090,2977,SJO +2091,2973,QDV +2092,2960,CON +2093,2958,OXR +2094,2957,LSE +2095,2947,DET +2096,2947,POZ +2097,2937,MKE +2098,2934,DFW +2099,2924,KLR +2100,2915,CPT +2101,2914,DFW +2102,2909,POZ +2103,2905,DET +2104,2903,KLR +2105,2903,OPO +2106,2903,HAD +2107,2899,GRU +2108,2895,LEJ +2109,2894,MSN +2110,2890,ADL +2111,2883,HAM +2112,2881,HAM +2113,2879,LBA +2114,2865,HAM +2115,2865,ABE +2116,2863,BZG +2117,2858,ORD +2118,2849,PAD +2119,2839,RKV +2120,2838,ACY +2121,2835,PER +2122,2835,ACY +2123,2834,SSZ +2124,2827,SZZ +2125,2820,STR +2126,2818,GRR +2127,2814,ATL +2128,2784,APF +2129,2774,DSA +2130,2772,AMS +2131,2771,NTE +2132,2771,SXB +2133,2763,PAR +2134,2759,HEL +2135,2754,DAY +2136,2753,DFW +2137,2752,YOW +2138,2724,CFR +2139,2720,YQY +2140,2713,SBP +2141,2712,YOW +2142,2711,HAM +2143,2708,NTE +2144,2708,GRR +2145,2707,NTE +2146,2706,YQB +2147,2679,ADL +2148,2676,DFW +2149,2672,FAR +2150,2668,PDX +2151,2647,STR +2152,2633,KSD +2153,2629,HUY +2154,2621,FNL +2155,2611,HLG +2156,2599,CPT +2157,2590,MAD +2158,2589,ABI +2159,2588,SZZ +2160,2588,YYC +2161,2585,GOJ +2162,2584,AAR +2163,2579,JKG +2164,2579,PNS +2165,2564,HAM +2166,2560,HAM +2167,2548,HAM +2168,2547,TSR +2169,2542,KJK +2170,2539,JKG +2171,2537,TMP +2172,2534,GOT +2173,2531,RMU +2174,2530,ACY +2175,2530,BRE +2176,2525,RDU +2177,2523,HEL +2178,2519,YOW +2179,2516,JKG +2180,2505,PDV +2181,2504,ORD +2182,2504,DAY +2183,2495,GOT +2184,2486,CVG +2185,2481,AAB +2186,2474,RDU +2187,2467,DAY +2188,2466,KEL +2189,2457,NTE +2190,2453,CPT +2191,2450,MMX +2192,2445,BNA +2193,2426,SGU +2194,2403,SGU +2195,2397,VIE +2196,2392,ORB +2197,2387,RBG +2198,2384,MSY +2199,2378,GRU +2200,2376,DUS +2201,2374,PAR +2202,2364,BNA +2203,2362,BLE +2204,2352,SMF +2205,2351,PLQ +2206,2351,BOS +2207,2351,WAW +2208,2349,KSD +2209,2321,AAR +2210,2319,MAD +2211,2310,BER +2212,2302,ANE +2213,2297,HLG +2214,2291,ORB +2215,2289,SVX +2216,2288,ATW +2217,2287,BLE +2218,2282,OMA +2219,2277,LGU +2220,2276,ADL +2221,2273,SZZ +2222,2256,LEJ +2223,2237,CVG +2224,2231,ERF +2225,2222,DAY +2226,2220,ADL +2227,2213,BES +2228,2211,PGD +2229,2210,BHS +2230,2195,ORD +2231,2191,MSN +2232,2184,ORD +2233,2181,GDN +2234,2181,PER +2235,2168,TLS +2236,2156,YOW +2237,2155,ORD +2238,2151,DAY +2239,2148,BGR +2240,2146,SRQ +2241,2145,FKB +2242,2137,PGD +2243,2125,NTE +2244,2113,KEL +2245,2113,TTD +2246,2110,LAS +2247,2109,CMH +2248,2101,ORB +2249,2098,YOW +2250,2093,SCL +2251,2090,RMU +2252,2090,ABE +2253,2085,DFW +2254,2083,MMX +2255,2080,PAD +2256,2079,NAP +2257,2078,SAO +2258,2075,ESS +2259,2075,CVG +2260,2072,ACY +2261,2066,CMH +2262,2065,PIT +2263,2061,GOT +2264,2052,LEJ +2265,2050,BER +2266,2029,DFW +2267,2021,NTE +2268,2018,JKG +2269,2010,EXT +2270,2006,DUS +2271,2001,MUC +2272,2000,TPA +2273,1999,RNB +2274,1983,ORD +2275,1981,CMH +2276,1974,KLR +2277,1965,BZG +2278,1959,YOW +2279,1954,SZZ +2280,1953,PER +2281,1950,MMX +2282,1948,BOD +2283,1937,TWB +2284,1936,UME +2285,1933,MFE +2286,1931,BCN +2287,1930,SZZ +2288,1914,JKG +2289,1895,SJO +2290,1887,MAN +2291,1885,TPA +2292,1882,HAJ +2293,1875,JKG +2294,1874,YUL +2295,1874,ATH +2296,1872,TSV +2297,1871,IND +2298,1857,MAD +2299,1853,DFW +2300,1848,GTI +2301,1832,SCN +2302,1830,SEA +2303,1828,LRF +2304,1823,TSV +2305,1819,TYR +2306,1818,GOT +2307,1816,ORB +2308,1816,GTI +2309,1811,SYR +2310,1804,NTE +2311,1804,BYU +2312,1801,TSV +2313,1797,SOD +2314,1792,MMX +2315,1791,KLR +2316,1787,ANE +2317,1786,ADL +2318,1765,ATW +2319,1756,HEL +2320,1751,CAK +2321,1750,YWG +2322,1745,JKG +2323,1741,TRD +2324,1740,FWA +2325,1735,HLG +2326,1733,SVX +2327,1732,WRO +2328,1728,ATW +2329,1728,BRE +2330,1727,CZW +2331,1717,MSP +2332,1713,WRO +2333,1712,JKG +2334,1711,NTE +2335,1710,BOD +2336,1710,HLG +2337,1707,POZ +2338,1703,MSP +2339,1702,JKG +2340,1700,SLC +2341,1697,BRE +2342,1678,FWA +2343,1677,LRF +2344,1677,IND +2345,1668,BRN +2346,1667,FLR +2347,1666,HAM +2348,1664,IND +2349,1661,BCN +2350,1657,BOD +2351,1652,CMH +2352,1652,RNO +2353,1648,ORB +2354,1639,BLE +2355,1637,PLQ +2356,1635,CVG +2357,1627,KLR +2358,1625,HTS +2359,1625,NPL +2360,1623,ORB +2361,1620,SZZ +2362,1616,LIL +2363,1616,ALC +2364,1615,BNE +2365,1611,ALC +2366,1609,RZE +2367,1609,PSF +2368,1603,MMX +2369,1598,BOS +2370,1594,AMS +2371,1591,YSE +2372,1591,TRD +2373,1570,LAS +2374,1569,HEL +2375,1568,TRD +2376,1568,LPI +2377,1567,PNS +2378,1563,LWR +2379,1555,MMX +2380,1553,ORD +2381,1552,TRD +2382,1550,LGG +2383,1549,CAE +2384,1547,RAO +2385,1544,GSP +2386,1540,KEL +2387,1540,IND +2388,1538,DFW +2389,1535,FWA +2390,1534,MAD +2391,1530,ORB +2392,1528,RNO +2393,1525,IND +2394,1520,BOD +2395,1516,SAT +2396,1516,SRQ +2397,1515,LAS +2398,1514,IND +2399,1512,ALH +2400,1511,SAT +2401,1501,YYC +2402,1501,HAM +2403,1500,DFW +2404,1497,STO +2405,1496,WAW +2406,1493,TRD +2407,1490,MSP +2408,1489,YYZ +2409,1484,TKU +2410,1480,GOT +2411,1480,LYS +2412,1476,GOT +2413,1475,JKG +2414,1475,LPI +2415,1474,ADL +2416,1470,CWA +2417,1464,DUD +2418,1463,YHZ +2419,1459,LEX +2420,1458,GOT +2421,1456,PER +2422,1454,BER +2423,1452,HTS +2424,1451,GOT +2425,1449,SJU +2426,1448,MSP +2427,1444,NTE +2428,1432,IND +2429,1430,FWA +2430,1429,NYC +2431,1429,COA +2432,1423,SCN +2433,1421,YTR +2434,1416,PDK +2435,1414,MAD +2436,1410,GOT +2437,1396,RDU +2438,1394,TKU +2439,1393,AVL +2440,1388,DFW +2441,1387,KSD +2442,1386,BGR +2443,1383,PGV +2444,1380,PKB +2445,1379,RTM +2446,1374,YYC +2447,1368,YYZ +2448,1366,LEX +2449,1363,SYR +2450,1360,SZZ +2451,1356,PLQ +2452,1350,JKG +2453,1349,SGU +2454,1346,MSP +2455,1341,STO +2456,1340,ORD +2457,1333,PAR +2458,1323,BOS +2459,1321,MAD +2460,1319,JAN +2461,1318,CWB +2462,1315,BOS +2463,1314,PAH +2464,1313,NYO +2465,1306,FRA +2466,1303,PGD +2467,1297,JKG +2468,1294,MUE +2469,1287,PAD +2470,1287,AZO +2471,1287,APF +2472,1286,VLC +2473,1278,VLC +2474,1277,MCI +2475,1273,BOD +2476,1266,GOT +2477,1261,MSY +2478,1260,BOD +2479,1257,BWT +2480,1251,BOD +2481,1248,GOT +2482,1246,IDA +2483,1239,OMA +2484,1232,ATW +2485,1232,ITA +2486,1228,LIL +2487,1220,ACY +2488,1217,BOS +2489,1217,CMH +2490,1216,MSY +2491,1214,YUL +2492,1213,YUL +2493,1212,ERF +2494,1202,VLC +2495,1202,BOD +2496,1201,HOH +2497,1199,CGN +2498,1197,DFW +2499,1196,LFT +2500,1192,BYG +2501,1190,HDB +2502,1188,ROK +2503,1187,BLE +2504,1184,LNS +2505,1181,GOT +2506,1176,IND +2507,1174,YUL +2508,1171,MSP +2509,1160,WVN +2510,1154,MSP +2511,1149,AGS +2512,1146,WVN +2513,1145,BRU +2514,1145,RFD +2515,1143,HAD +2516,1139,SZZ +2517,1135,CFR +2518,1133,PLQ +2519,1132,MSP +2520,1126,LFT +2521,1122,BOD +2522,1121,PNS +2523,1117,GOT +2524,1115,SCN +2525,1114,SZZ +2526,1100,VLC +2527,1097,LFT +2528,1097,ORD +2529,1097,ADL +2530,1096,YQB +2531,1094,NUE +2532,1092,CLT +2533,1086,HEL +2534,1083,BNA +2535,1082,RNO +2536,1076,CWA +2537,1071,BWE +2538,1070,ENS +2539,1066,SCE +2540,1066,RAO +2541,1062,PIA +2542,1062,LAS +2543,1055,NTU +2544,1052,AMS +2545,1051,BUD +2546,1051,ACY +2547,1050,BDL +2548,1046,MCI +2549,1046,CWA +2550,1044,KZN +2551,1034,HAM +2552,1024,SZZ +2553,1023,SPM +2554,1020,MMX +2555,1020,YVE +2556,1009,LFT +2557,1008,VIE +2558,1007,HEL +2559,1001,SZZ +2560,999,SDF +2561,993,PGV +2562,992,NTU +2563,990,BER +2564,989,AMS +2565,983,IND +2566,983,WVN +2567,982,MSP +2568,982,CPH +2569,976,KZN +2570,976,TYR +2571,975,AZO +2572,967,DET +2573,964,PGD +2574,963,GOT +2575,961,TUL +2576,960,POZ +2577,958,ATW +2578,958,WVN +2579,953,RLG +2580,953,YYC +2581,949,NOD +2582,947,DFW +2583,946,KZN +2584,942,ORD +2585,940,HEL +2586,940,ALC +2587,938,EVV +2588,936,ORM +2589,930,RDZ +2590,930,YYC +2591,929,IDA +2592,928,MUC +2593,926,JTC +2594,918,GRR +2595,917,JKG +2596,917,YYC +2597,908,IND +2598,903,PIA +2599,901,MEL +2600,898,CHC +2601,897,HOU +2602,896,MIA +2603,895,LIS +2604,890,FAR +2605,889,UME +2606,885,ATW +2607,884,LAA +2608,882,CFR +2609,880,AAR +2610,873,BYU +2611,869,CHC +2612,868,PKB +2613,866,NAP +2614,862,MCI +2615,860,SJY +2616,851,LEY +2617,842,TUL +2618,839,TMP +2619,837,WRO +2620,825,SDR +2621,823,ATW +2622,822,BRN +2623,820,TLS +2624,819,MKE +2625,819,YQB +2626,819,RNO +2627,819,FWA +2628,815,GRR +2629,815,SZZ +2630,812,CPQ +2631,810,GOT +2632,808,DEN +2633,804,HOU +2634,804,ATW +2635,802,KOK +2636,801,DFW +2637,798,CHC +2638,798,RKD +2639,796,DUD +2640,796,SYR +2641,796,FAR +2642,795,RKD +2643,794,HEL +2644,792,PLO +2645,791,YQB +2646,788,GTI +2647,786,YQL +2648,786,ODE +2649,785,CHC +2650,783,CHC +2651,783,MCI +2652,779,ACY +2653,774,TMP +2654,769,LRT +2655,767,STO +2656,764,BOS +2657,761,CLE +2658,760,VNO +2659,760,MSP +2660,758,TLH +2661,755,BOS +2662,753,AOH +2663,753,IND +2664,748,BOD +2665,743,YUL +2666,741,MOT +2667,739,BWE +2668,737,ADL +2669,736,JAX +2670,736,GCI +2671,734,IND +2672,733,PER +2673,728,KOK +2674,722,YUL +2675,721,TYR +2676,718,RKD +2677,718,JKG +2678,716,TMP +2679,715,MST +2680,713,SCN +2681,712,DRS +2682,709,GDW +2683,709,TUL +2684,708,EVV +2685,707,SCL +2686,704,BRE +2687,703,TMP +2688,703,YKA +2689,703,OGG +2690,698,POZ +2691,697,HAD +2692,696,SYR +2693,692,KJK +2694,690,HOU +2695,689,MKE +2696,686,BUD +2697,678,DFW +2698,676,GCI +2699,676,HAJ +2700,675,SVX +2701,674,VAP +2702,673,POZ +2703,671,GOT +2704,670,GSO +2705,668,NTU +2706,664,CLE +2707,664,GDW +2708,663,VLC +2709,661,LRT +2710,661,HOU +2711,660,FKB +2712,644,HOU +2713,643,MSY +2714,642,MSN +2715,637,ATW +2716,635,HAM +2717,634,YYZ +2718,632,HAJ +2719,630,TYR +2720,628,HOU +2721,623,TMP +2722,620,LNZ +2723,620,TMP +2724,619,SJY +2725,619,BUD +2726,618,YML +2727,615,PAD +2728,614,NWI +2729,614,PDX +2730,613,OMA +2731,613,ANR +2732,610,HAM +2733,609,GCI +2734,608,MKE +2735,607,IND +2736,604,CMH +2737,603,AAH +2738,602,ATL +2739,602,ORD +2740,599,RDD +2741,598,CON +2742,598,UME +2743,594,MBS +2744,594,TLL +2745,593,MFE +2746,591,MKE +2747,585,TUL +2748,580,GCI +2749,579,DIQ +2750,579,ORD +2751,578,VST +2752,575,GSP +2753,575,WVN +2754,573,LIL +2755,573,MSP +2756,572,IAD +2757,571,ABE +2758,567,KZN +2759,567,TPA +2760,566,LCY +2761,566,BWE +2762,564,HOU +2763,562,BUY +2764,562,BNE +2765,561,IND +2766,561,SYR +2767,560,GCI +2768,560,SZZ +2769,559,MKE +2770,558,STR +2771,555,SYR +2772,554,CMH +2773,552,MSP +2774,551,IAD +2775,550,CLE +2776,549,VXO +2777,549,NYC +2778,548,HAM +2779,546,SJT +2780,542,MCI +2781,534,TYR +2782,534,MNN +2783,531,ATW +2784,530,DET +2785,528,BWI +2786,524,GCI +2787,524,LON +2788,523,JAX +2789,521,YUL +2790,521,BWI +2791,517,BUY +2792,517,TPA +2793,513,BOS +2794,513,TUL +2795,510,GCI +2796,508,SIR +2797,507,EVV +2798,505,YYC +2799,504,RKD +2800,502,GJT +2801,500,DET +2802,500,CLT +2803,499,DTM +2804,498,DFW +2805,497,VLY +2806,495,ATW +2807,490,MQL +2808,489,DAB +2809,489,JAX +2810,488,AKL +2811,487,MKE +2812,486,PRC +2813,480,DET +2814,480,ORD +2815,480,YGK +2816,477,VLC +2817,476,CPH +2818,473,RST +2819,471,SYR +2820,469,BBH +2821,466,DFW +2822,459,EME +2823,458,AMS +2824,457,SOF +2825,456,TUL +2826,454,LYS +2827,454,SGU +2828,452,IND +2829,451,IND +2830,450,MCI +2831,448,UKK +2832,448,JKG +2833,446,SQQ +2834,445,PUB +2835,444,MOW +2836,443,HOU +2837,442,ANE +2838,440,YXX +2839,439,EIN +2840,431,SJC +2841,431,GCI +2842,428,AMS +2843,426,LMT +2844,424,VST +2845,424,GCI +2846,423,PLR +2847,413,GOT +2848,411,LNS +2849,409,SDF +2850,405,LEX +2851,401,NTU +2852,401,DET +2853,400,HAD +2854,399,DFW +2855,397,TUL +2856,397,SYR +2857,392,ODE +2858,391,OMA +2859,387,BHM +2860,385,CLE +2861,384,PSK +2862,383,EWN +2863,381,MDT +2864,381,LNS +2865,377,HAM +2866,375,DFW +2867,374,LPI +2868,371,VST +2869,370,HAD +2870,370,YYY +2871,369,IAS +2872,369,SMN +2873,367,LCY +2874,366,ORD +2875,366,PNS +2876,354,JKG +2877,354,LPI +2878,352,RUT +2879,352,MUC +2880,352,GTR +2881,351,FAT +2882,351,ALB +2883,351,MMX +2884,351,KEL +2885,349,DFW +2886,349,PAR +2887,349,OKC +2888,347,CPT +2889,343,GRR +2890,343,VLC +2891,341,WVN +2892,337,DFW +2893,337,RUT +2894,333,MST +2895,328,ABI +2896,326,LCQ +2897,325,MBS +2898,323,MOT +2899,322,BUF +2900,320,RUT +2901,320,GOR +2902,318,WVN +2903,318,MLU +2904,316,SZZ +2905,316,HAM +2906,314,VST +2907,314,YGK +2908,314,SCN +2909,309,EWN +2910,308,PSP +2911,303,BDL +2912,301,CWA +2913,295,LEJ +2914,294,SDF +2915,294,SDF +2916,292,SJT +2917,292,ORD +2918,292,SJT +2919,291,IND +2920,291,RUT +2921,290,SIR +2922,289,YTF +2923,286,YTF +2924,286,DNQ +2925,286,YQB +2926,285,SJT +2927,282,JKG +2928,281,LNS +2929,279,BZG +2930,277,TMP +2931,275,CEB +2932,274,BFL +2933,273,DET +2934,272,DPO +2935,272,CLE +2936,269,HAD +2937,269,MME +2938,266,LRT +2939,264,RST +2940,263,SLC +2941,263,GJT +2942,262,KJK +2943,262,TLL +2944,261,OUL +2945,261,MMX +2946,259,SJT +2947,256,SEA +2948,253,XCM +2949,251,DRS +2950,250,OMA +2951,249,LPI +2952,247,AUS +2953,247,PGV +2954,246,JKG +2955,244,BNA +2956,243,LPI +2957,241,NUE +2958,241,GRU +2959,239,SLC +2960,238,LBB +2961,236,CVG +2962,235,LPI +2963,234,GTR +2964,233,YKF +2965,231,BBH +2966,229,LAS +2967,224,PFB +2968,223,SOF +2969,221,MSP +2970,220,GPT +2971,219,HEL +2972,218,CWA +2973,218,JKG +2974,217,LPI +2975,215,ORD +2976,213,UME +2977,209,PDX +2978,204,STO +2979,204,MCI +2980,204,LRU +2981,204,OTP +2982,201,JKG +2983,200,VXO +2984,199,BLE +2985,198,DET +2986,198,TRF +2987,196,AZO +2988,195,DFW +2989,195,SVG +2990,193,YXU +2991,192,AZO +2992,191,RDU +2993,188,PSH +2994,187,ORD +2995,187,LPI +2996,186,MOT +2997,185,RDU +2998,185,LPI +2999,185,PGV +3000,184,BNE +3001,184,STO +3002,184,CPQ +3003,182,JAX +3004,181,IAD +3005,181,ATL +3006,181,YKF +3007,180,LOI +3008,180,LRT +3009,178,HOU +3010,178,LYS +3011,178,LYS +3012,178,SAT +3013,177,ALB +3014,177,STL +3015,177,TRN +3016,176,LNS +3017,175,AMA +3018,171,MSY +3019,169,CLE +3020,168,FNT +3021,162,SEA +3022,162,LNS +3023,161,MEM +3024,161,OPO +3025,161,JKG +3026,161,RDU +3027,160,HOU +3028,160,AMS +3029,159,BUY +3030,159,ORD +3031,158,DET +3032,158,BDL +3033,156,KSD +3034,156,KRK +3035,156,ONP +3036,155,YOW +3037,150,VAG +3038,150,AOH +3039,150,ATW +3040,149,IQQ +3041,148,LEY +3042,147,AIZ +3043,147,MMH +3044,146,IND +3045,145,RDU +3046,144,PER +3047,144,STO +3048,141,HEL +3049,140,IND +3050,139,RIC +3051,139,ROA +3052,139,MKE +3053,138,PIT +3054,135,VRZ +3055,134,FNB +3056,134,GRQ +3057,133,SJT +3058,133,YUL +3059,133,ATL +3060,132,SEA +3061,132,KOK +3062,130,MBS +3063,130,YLK +3064,130,TRN +3065,129,IND +3066,129,LPI +3067,129,LNS +3068,127,MSV +3069,126,STL +3070,126,IAD +3071,124,LBC +3072,123,ABI +3073,122,LGG +3074,122,KEL +3075,122,YQA +3076,121,HOU +3077,120,YQA +3078,120,GSO +3079,119,BUF +3080,119,RIC +3081,118,DFW +3082,118,SJY +3083,116,NWI +3084,116,SEA +3085,116,CEG +3086,115,SEA +3087,115,BRE +3088,115,BWT +3089,113,GRR +3090,112,YXU +3091,111,PDX +3092,110,SOF +3093,109,BUY +3094,109,IND +3095,109,LBB +3096,109,NTU +3097,109,SDL +3098,108,STO +3099,108,GRR +3100,108,NYC +3101,108,ABC +3102,107,RDD +3103,104,LWB +3104,103,SPM +3105,103,ROC +3106,103,SGU +3107,102,ORD +3108,101,AAL +3109,99,CLE +3110,98,YQF +3111,98,JAX +3112,98,STO +3113,97,BHS +3114,97,BCN +3115,97,KSD +3116,96,MSY +3117,96,VXO +3118,96,DFW +3119,96,KSD +3120,96,BFD +3121,92,BOS +3122,91,GLZ +3123,91,TKU +3124,91,UDE +3125,90,IAD +3126,90,PMI +3127,88,SRQ +3128,87,AMA +3129,87,DFW +3130,87,SJT +3131,87,BOS +3132,87,POA +3133,86,IND +3134,86,YLK +3135,84,SEA +3136,83,LRT +3137,82,SYR +3138,82,MSP +3139,82,CRL +3140,82,HOU +3141,81,RST +3142,81,KJK +3143,80,YVR +3144,80,GPZ +3145,80,ELM +3146,80,CAK +3147,80,SDL +3148,79,MCI +3149,79,TYR +3150,78,VNO +3151,78,NTE +3152,78,ABC +3153,78,NTU +3154,78,HAJ +3155,77,TPA +3156,77,KJK +3157,76,CBG +3158,76,LEJ +3159,75,SEA +3160,75,MDT +3161,75,HEL +3162,75,OSD +3163,75,CMH +3164,75,HEL +3165,74,LCH +3166,73,TXK +3167,73,DFW +3168,73,IND +3169,73,YQQ +3170,73,AGS +3171,73,GOT +3172,73,IND +3173,72,YUL +3174,72,ROC +3175,72,BDL +3176,72,MCI +3177,71,IND +3178,71,MEM +3179,71,BGM +3180,71,MSN +3181,70,ESS +3182,69,ALB +3183,69,SOF +3184,68,AAL +3185,68,IND +3186,67,KSF +3187,67,GOT +3188,67,RST +3189,66,DET +3190,64,PHF +3191,63,ENS +3192,63,KEL +3193,62,OSD +3194,62,ANC +3195,61,ORD +3196,61,BDL +3197,60,SXB +3198,60,YOW +3199,58,GOT +3200,58,TSV +3201,57,SEA +3202,57,ALB +3203,57,CGN +3204,57,FWA +3205,56,MBS +3206,56,VST +3207,56,ALF +3208,55,NOD +3209,55,CFR +3210,55,SDL +3211,55,CFR +3212,54,AIZ +3213,54,GRR +3214,54,DFW +3215,54,GOA +3216,54,IND +3217,53,DXB +3218,53,JAN +3219,52,YWG +3220,52,YYY +3221,51,STO +3222,51,UTC +3223,50,HYR +3224,50,DFW +3225,50,LUX +3226,50,RSW +3227,48,LPI +3228,48,EAU +3229,48,MOW +3230,47,SOF +3231,46,GOT +3232,46,DUS +3233,46,BFL +3234,46,BGR +3235,46,ADL +3236,46,DET +3237,45,DRS +3238,44,IND +3239,44,ATW +3240,44,CVG +3241,44,MSN +3242,44,DFW +3243,43,ABE +3244,43,WRO +3245,43,JER +3246,43,OMA +3247,43,IAD +3248,42,WVN +3249,42,LYS +3250,41,TRG +3251,40,ORD +3252,40,FRA +3253,40,HVN +3254,40,BRU +3255,40,TGD +3256,39,KRK +3257,39,BDL +3258,39,GOT +3259,39,AXN +3260,38,MOW +3261,38,BRN +3262,38,MSY +3263,38,MUC +3264,38,CLE +3265,37,SJC +3266,37,IND +3267,37,YQS +3268,37,VXO +3269,37,DEN +3270,36,AMA +3271,36,HAM +3272,36,GDW +3273,36,CVG +3274,36,NYO +3275,35,GDW +3276,35,TLL +3277,35,KSD +3278,35,MAC +3279,34,VCP +3280,34,FNB +3281,33,ORD +3282,33,MDT +3283,33,DXB +3284,31,FNT +3285,31,IND +3286,31,TWB +3287,31,FRA +3288,31,FNT +3289,31,IND +3290,30,AAL +3291,30,CON +3292,30,ALB +3293,30,BCN +3294,29,BRE +3295,29,SBY +3296,29,ANR +3297,29,LAS +3298,28,FAT +3299,28,ACH +3300,28,CHC +3301,28,BOS +3302,27,TMP +3303,27,DFW +3304,27,YGK +3305,27,IND +3306,27,SBN +3307,27,HEL +3308,27,RIC +3309,26,BGR +3310,26,MKE +3311,26,VXO +3312,25,RJK +3313,25,CMI +3314,25,SJY +3315,24,ORD +3316,23,MEL +3317,23,MCI +3318,23,GOT +3319,23,NTE +3320,22,LGG +3321,22,POZ +3322,22,BOS +3323,21,TMP +3324,21,BRU +3325,20,BOS +3326,20,TYR +3327,20,GRU +3328,19,LCJ +3329,19,BLE +3330,19,PUB +3331,19,BOJ +3332,18,SVX +3333,18,STR +3334,18,DEC +3335,18,SBN +3336,18,AAR +3337,18,RDU +3338,18,SEA +3339,17,SBP +3340,17,LCJ +3341,17,YHM +3342,17,AMA +3343,16,PIA +3344,16,FRA +3345,16,GSO +3346,16,PAD +3347,16,IDA +3348,15,MOW +3349,15,MBS +3350,15,BCN +3351,15,IND +3352,15,OSI +3353,15,BDL +3354,14,OMA +3355,14,FMM +3356,14,MSN +3357,14,EXT +3358,13,SFO +3359,13,MIA +3360,13,YUL +3361,13,AMS +3362,13,LNS +3363,13,BER +3364,13,PNS +3365,13,WAW +3366,13,MUC +3367,12,NOA +3368,12,AUS +3369,12,ATL +3370,12,SYD +3371,12,WAW +3372,12,DRS +3373,11,VBY +3374,11,LBC +3375,11,LPI +3376,10,WVN +3377,10,DUB +3378,10,KRK +3379,10,CVG +3380,10,MMX +3381,10,ORD +3382,10,OKC +3383,10,RTM +3384,10,MSP +3385,9,LON +3386,9,BNA +3387,9,LRT +3388,8,FAT +3389,8,NHZ +3390,8,MEL +3391,8,IAD +3392,8,RIC +3393,8,NYC +3394,7,XMS +3395,7,SEA +3396,7,SEA +3397,7,ABX +3398,7,SFO +3399,7,LCY +3400,7,KEL +3401,6,BER +3402,6,FRA +3403,6,ERF +3404,6,MAD +3405,6,LAX +3406,6,DVN +3407,6,LPL +3408,6,CLE +3409,6,CGN +3410,6,BCN +3411,6,UTC +3412,6,GSP +3413,6,MGM +3414,6,BHZ +3415,6,CVG +3416,6,SZZ +3417,6,NTE +3418,6,SEA +3419,5,HOU +3420,5,BDL +3421,5,ALB +3422,5,GRU +3423,5,DSM +3424,5,MAN +3425,5,ALC +3426,5,LEJ +3427,5,BRN +3428,4,BNA +3429,4,CVG +3430,4,SCN +3431,4,PDX +3432,4,GRU +3433,4,VCP +3434,4,IAS +3435,4,PHX +3436,4,ORD +3437,3,MEL +3438,3,VCP +3439,3,KEL +3440,3,LON +3441,3,BER +3442,3,LEJ +3443,3,EVV +3444,3,BNE +3445,3,BUD +3446,3,RTM +3447,3,BRN +3448,3,AKL +3449,3,BDL +3450,3,YQS +3451,3,LYS +3452,3,AMS +3453,3,LBC +3454,3,PUW +3455,3,YYB +3456,3,ATL +3457,3,SJC +3458,3,WAW +3459,3,LFT +3460,2,CON +3461,2,MUC +3462,2,SZZ +3463,2,IKT +3464,2,BER +3465,2,VST +3466,2,MEL +3467,2,SCL +3468,2,MSN +3469,2,ATW +3470,2,LEJ +3471,2,MAN +3472,2,DEN +3473,2,PRG +3474,2,PRG +3475,2,PRC +3476,2,ORD +3477,2,GEX +3478,2,BTS +3479,2,LEJ +3480,2,CPT +3481,2,ALB +3482,2,LRT +3483,2,BRU +3484,2,FRA +3485,2,GTI +3486,2,SZZ +3487,2,IAD +3488,2,VOG +3489,2,KRK +3490,2,GLZ +3491,2,ALB +3492,2,CGN +3493,2,PRG +3494,2,BHD +3495,2,FRA +3496,2,SAT +3497,2,AUS +3498,2,LIN +3499,1,FMM +3500,1,SDL +3501,1,KEL +3502,1,BOS +3503,1,DEN +3504,1,YPQ +3505,1,PLQ +3506,1,JKG +3507,1,NTE +3508,1,SDL +3509,1,DUS +3510,1,ATW +3511,1,MOW +3512,1,CGN +3513,1,ZBM +3514,1,MNN +3515,1,FNB +3516,1,LUX +3517,1,YVR +3518,1,SPM +3519,1,EVV +3520,1,VXO +3521,1,LON +3522,1,COS +3523,1,DEN +3524,1,PAR +3525,1,OMA +3526,1,LRH +3527,1,CHA +3528,1,SEA +3529,1,MUC +3530,1,OTP +3531,1,LKV +3532,1,LRT +3533,1,SEA +3534,1,MSN +3535,1,SEA +3536,1,BER +3537,1,CGI +3538,1,FNL +3539,1,FNL +3540,1,CWA +3541,1,DEN +3542,1,ALB +3543,1,CLE +3544,1,IND +3545,1,PNS +3546,1,SEA +3547,1,YHZ +3548,1,WRO +3549,1,STR +3550,1,SCN +3551,1,DUS +3552,1,PNS +3553,1,CGN +3554,1,SBN +3555,1,MSN +3556,1,BDL +3557,1,ZBM +3558,1,XYZ diff --git a/db/planguard/fixture/testdata/po_sites_per_packet.csv b/db/planguard/fixture/testdata/po_sites_per_packet.csv new file mode 100644 index 0000000..db3d7a4 --- /dev/null +++ b/db/planguard/fixture/testdata/po_sites_per_packet.csv @@ -0,0 +1,44 @@ +n_sites,packets +1,2180711 +2,207420 +3,121586 +4,117839 +5,81600 +6,57253 +7,61762 +8,30569 +9,25871 +10,22554 +11,23833 +12,24830 +13,17044 +14,9737 +15,12325 +16,15143 +17,15138 +18,10259 +19,4936 +20,3433 +21,1654 +22,1027 +23,709 +24,448 +25,326 +26,219 +27,120 +28,85 +29,69 +30,40 +31,33 +32,19 +33,11 +34,12 +35,15 +36,7 +37,1 +38,8 +39,3 +40,2 +41,5 +45,1 +57,1 diff --git a/db/planguard/fixture/testdata/po_whole_second.csv b/db/planguard/fixture/testdata/po_whole_second.csv new file mode 100644 index 0000000..c57cbbe --- /dev/null +++ b/db/planguard/fixture/testdata/po_whole_second.csv @@ -0,0 +1,2 @@ +whole_second_pct +19.0 diff --git a/db/planguard/fixture/testdata/routes_age_obs.csv b/db/planguard/fixture/testdata/routes_age_obs.csv new file mode 100644 index 0000000..d529695 --- /dev/null +++ b/db/planguard/fixture/testdata/routes_age_obs.csv @@ -0,0 +1,509 @@ +age_h,obs_ge3,n +0,f,29894 +0,t,29670 +1,f,38390 +1,t,35466 +2,f,39508 +2,t,34848 +3,f,42119 +3,t,36955 +4,f,39017 +4,t,37227 +5,f,38387 +5,t,32684 +6,f,41202 +6,t,33711 +7,f,39407 +7,t,30247 +8,f,42312 +8,t,31729 +9,f,38955 +9,t,30421 +10,f,35662 +10,t,25368 +11,f,35941 +11,t,25648 +12,f,32776 +12,t,24140 +13,f,27606 +13,t,24729 +14,f,26890 +14,t,26620 +15,f,25260 +15,t,22875 +16,f,25431 +16,t,22460 +17,f,24379 +17,t,20389 +18,f,27704 +18,t,22691 +19,f,30026 +19,t,23239 +20,f,35770 +20,t,25051 +21,f,34870 +21,t,24811 +22,f,45193 +22,t,23839 +23,f,53129 +23,t,27082 +24,f,44760 +24,t,27406 +25,f,42286 +25,t,25463 +26,f,38992 +26,t,23289 +27,f,45883 +27,t,25912 +28,f,37964 +28,t,23444 +29,f,45124 +29,t,24758 +30,f,44440 +30,t,22948 +31,f,40084 +31,t,22508 +32,f,40086 +32,t,20363 +33,f,34943 +33,t,19836 +34,f,35969 +34,t,19725 +35,f,42756 +35,t,21464 +36,f,34059 +36,t,19884 +37,f,27531 +37,t,17698 +38,f,30301 +38,t,20830 +39,f,25968 +39,t,18721 +40,f,22562 +40,t,15109 +41,f,24086 +41,t,16958 +42,f,25695 +42,t,17999 +43,f,27945 +43,t,17746 +44,f,29879 +44,t,18212 +45,f,37445 +45,t,20503 +46,f,38670 +46,t,20792 +47,f,40768 +47,t,21519 +48,f,40722 +48,t,19612 +49,f,39518 +49,t,19961 +50,f,38720 +50,t,21066 +51,f,37635 +51,t,19922 +52,f,40295 +52,t,20615 +53,f,39419 +53,t,21009 +54,f,34559 +54,t,19629 +55,f,34514 +55,t,20274 +56,f,32273 +56,t,18238 +57,f,30205 +57,t,16450 +58,f,32808 +58,t,16584 +59,f,32839 +59,t,16460 +60,f,32932 +60,t,18447 +61,f,31643 +61,t,18431 +62,f,28914 +62,t,17165 +63,f,25014 +63,t,16694 +64,f,25340 +64,t,15520 +65,f,25062 +65,t,15539 +66,f,25367 +66,t,15792 +67,f,28521 +67,t,17937 +68,f,29846 +68,t,15442 +69,f,35276 +69,t,17506 +70,f,41760 +70,t,19618 +71,f,44115 +71,t,20431 +72,f,43629 +72,t,19147 +73,f,39015 +73,t,18964 +74,f,37799 +74,t,20505 +75,f,34063 +75,t,16569 +76,f,37796 +76,t,18005 +77,f,31496 +77,t,16070 +78,f,29548 +78,t,15515 +79,f,29689 +79,t,14754 +80,f,30877 +80,t,14758 +81,f,25110 +81,t,13181 +82,f,28055 +82,t,14363 +83,f,31796 +83,t,16529 +84,f,28166 +84,t,15135 +85,f,25973 +85,t,14757 +86,f,24735 +86,t,14279 +87,f,20855 +87,t,11982 +88,f,21522 +88,t,10641 +89,f,21372 +89,t,10562 +90,f,23120 +90,t,12429 +91,f,21076 +91,t,10468 +92,f,26599 +92,t,13182 +93,f,34613 +93,t,17303 +94,f,36482 +94,t,16774 +95,f,37902 +95,t,13867 +96,f,38635 +96,t,9266 +97,f,37964 +97,t,14456 +98,f,35231 +98,t,18694 +99,f,33863 +99,t,18349 +100,f,29440 +100,t,15831 +101,f,29483 +101,t,16615 +102,f,30979 +102,t,14783 +103,f,30302 +103,t,14911 +104,f,25481 +104,t,13583 +105,f,26020 +105,t,13199 +106,f,28424 +106,t,14506 +107,f,30717 +107,t,16955 +108,f,28611 +108,t,15147 +109,f,28355 +109,t,19056 +110,f,25080 +110,t,14491 +111,f,23449 +111,t,14805 +112,f,22470 +112,t,15990 +113,f,22936 +113,t,15487 +114,f,23089 +114,t,14916 +115,f,24840 +115,t,15275 +116,f,28797 +116,t,16235 +117,f,31689 +117,t,14181 +118,f,44600 +118,t,10127 +119,f,50233 +119,t,11484 +120,f,54860 +120,t,11038 +121,f,40871 +121,t,9780 +122,f,42603 +122,t,9982 +123,f,40920 +123,t,9873 +124,f,36715 +124,t,9575 +125,f,28219 +125,t,14328 +126,f,29216 +126,t,14603 +127,f,27001 +127,t,13295 +128,f,28620 +128,t,16007 +129,f,27190 +129,t,13420 +130,f,30805 +130,t,14478 +131,f,27903 +131,t,14474 +132,f,31031 +132,t,16046 +133,f,26115 +133,t,15651 +134,f,24655 +134,t,14417 +135,f,23066 +135,t,14474 +136,f,22865 +136,t,14484 +137,f,20780 +137,t,13294 +138,f,23070 +138,t,14899 +139,f,23359 +139,t,13050 +140,f,26743 +140,t,15035 +141,f,34530 +141,t,18964 +142,f,31055 +142,t,16560 +143,f,36496 +143,t,18565 +144,f,49197 +144,t,24281 +145,f,42673 +145,t,19574 +146,f,37306 +146,t,20533 +147,f,39660 +147,t,21886 +148,f,38655 +148,t,22808 +149,f,30228 +149,t,14925 +150,f,30873 +150,t,15256 +151,f,29434 +151,t,14232 +152,f,31118 +152,t,16149 +153,f,29178 +153,t,15619 +154,f,27961 +154,t,12965 +155,f,32302 +155,t,14801 +156,f,30664 +156,t,14744 +157,f,24432 +157,t,13681 +158,f,23832 +158,t,14094 +159,f,22027 +159,t,13354 +160,f,20994 +160,t,12907 +161,f,19210 +161,t,11830 +162,f,21189 +162,t,13880 +163,f,19175 +163,t,11261 +164,f,24826 +164,t,15096 +165,f,30319 +165,t,17178 +166,f,32732 +166,t,16867 +167,f,34919 +167,t,18607 +168,f,22582 +168,t,16848 +169,f,5133 +169,t,17084 +170,t,17658 +171,t,17025 +172,t,16492 +173,t,15563 +174,t,13566 +175,t,16219 +176,t,13153 +177,t,14284 +178,t,15712 +179,t,15247 +180,t,14824 +181,t,12397 +182,t,11093 +183,t,11857 +184,t,12776 +185,t,11728 +186,t,13075 +187,t,11325 +188,t,12784 +189,t,13766 +190,t,15578 +191,t,15626 +192,t,14827 +193,t,16415 +194,t,17178 +195,t,16576 +196,t,13817 +197,t,13203 +198,t,14916 +199,t,13969 +200,t,12724 +201,t,12184 +202,t,12575 +203,t,13127 +204,t,13259 +205,t,13230 +206,t,11035 +207,t,9104 +208,t,14843 +209,t,11239 +210,t,11810 +211,t,11796 +212,t,13375 +213,t,12671 +214,t,13780 +215,t,16595 +216,t,13823 +217,t,14351 +218,t,13085 +219,t,14787 +220,t,14075 +221,t,14780 +222,t,11876 +223,t,11223 +224,t,10199 +225,t,11214 +226,t,11192 +227,t,12864 +228,t,12627 +229,t,13421 +230,t,10629 +231,t,12308 +232,t,12596 +233,t,11258 +234,t,11353 +235,t,10626 +236,t,13104 +237,t,13440 +238,t,16048 +239,t,15524 +240,t,15445 +241,t,14006 +242,t,12937 +243,t,13491 +244,t,16001 +245,t,12705 +246,t,12666 +247,t,10128 +248,t,11492 +249,t,9805 +250,t,10671 +251,t,11684 +252,t,14252 +253,t,12758 +254,t,12270 +255,t,10465 +256,t,14373 +257,t,14162 +258,t,11051 +259,t,11304 +260,t,11200 +261,t,13513 +262,t,15084 +263,t,12355 +264,t,12017 +265,t,11239 +266,t,12493 +267,t,11881 +268,t,13876 +269,t,13262 +270,t,12926 +271,t,12923 +272,t,12369 +273,t,11331 +274,t,11090 +275,t,11658 +276,t,11759 +277,t,11796 +278,t,12182 +279,t,11336 +280,t,13730 +281,t,10604 +282,t,11250 +283,t,11449 +284,t,12309 +285,t,11326 +286,t,11990 +287,t,11939 +288,t,11921 +289,t,12584 +290,t,13268 +291,t,13497 +292,t,11296 +293,t,11494 +294,t,10792 +295,t,8508 +296,t,9511 +297,t,8182 +298,t,9114 +299,t,11407 +300,t,10227 +301,t,10142 +302,t,11486 +303,t,10155 +304,t,8212 +305,t,9831 +306,t,10238 +307,t,8940 +308,t,8619 +309,t,9736 +310,t,11331 +311,t,9523 +312,t,10965 +313,t,11647 +314,t,9941 +315,t,8905 +316,t,7935 +317,t,9373 +318,t,7210 +319,t,8199 +320,t,7432 +321,t,7603 +322,t,6170 +323,t,7862 +324,t,7418 +325,t,6589 +326,t,6383 +327,t,5959 +328,t,6478 +329,t,8989 +330,t,7498 +331,t,6269 +332,t,8541 +333,t,7875 +334,t,7606 +335,t,9121 +336,t,6720 +337,t,1657 diff --git a/db/planguard/fixture/testdata/routes_iata_hop.csv b/db/planguard/fixture/testdata/routes_iata_hop.csv new file mode 100644 index 0000000..ae6d48b --- /dev/null +++ b/db/planguard/fixture/testdata/routes_iata_hop.csv @@ -0,0 +1,1454 @@ +iata,hop_count,n +ABE,2,350 +ABE,3,1819 +ABE,4,3964 +ABE,5,4654 +ABE,6,3824 +ABE,7,2415 +ABE,8,1233 +ABE,9,541 +ABE,10,274 +ABE,11,130 +ABE,12,79 +ABE,13,36 +ABE,14,17 +ABE,15,11 +ABE,16,5 +ABE,17,7 +ABE,18,1 +ABE,19,1 +ABE,21,1 +ABE,22,1 +ACH,2,142 +ACH,3,602 +ACH,4,1220 +ACH,5,1801 +ACH,6,2398 +ACH,7,2585 +ACH,8,2499 +ACH,9,2040 +ACH,10,1535 +ACH,11,1062 +ACH,12,724 +ACH,13,559 +ACH,14,407 +ACH,15,287 +ACH,16,196 +ACH,17,147 +ACH,18,103 +ACH,19,69 +ACH,20,55 +ACH,21,33 +ACH,22,18 +ACH,23,10 +ACH,24,8 +ACH,25,9 +ACH,26,3 +ACH,27,4 +AMS,2,3452 +AMS,3,20772 +AMS,4,64532 +AMS,5,127105 +AMS,6,183549 +AMS,7,211336 +AMS,8,206446 +AMS,9,157965 +AMS,10,113761 +AMS,11,77461 +AMS,12,52747 +AMS,13,36337 +AMS,14,25144 +AMS,15,17832 +AMS,16,12507 +AMS,17,8990 +AMS,18,6299 +AMS,19,4500 +AMS,20,3215 +AMS,21,2175 +AMS,22,1328 +AMS,23,977 +AMS,24,632 +AMS,25,498 +AMS,26,314 +AMS,27,229 +AMS,28,156 +AMS,29,132 +AMS,30,83 +AMS,31,47 +AMS,32,34 +ANR,2,530 +ANR,3,3247 +ANR,4,10140 +ANR,5,20420 +ANR,6,24118 +ANR,7,18029 +ANR,8,10677 +ANR,9,5697 +ANR,10,2861 +ANR,11,1355 +ANR,12,628 +ANR,13,280 +ANR,14,130 +ANR,15,43 +ANR,16,14 +ANR,17,7 +ANR,18,4 +ANR,19,3 +BHM,2,369 +BHM,3,2258 +BHM,4,4644 +BHM,5,5158 +BHM,6,3571 +BHM,7,2054 +BHM,8,1046 +BHM,9,338 +BHM,10,212 +BHM,11,112 +BHM,12,37 +BHM,13,18 +BHM,14,21 +BHM,15,9 +BHM,16,5 +BHM,17,2 +BHM,18,1 +BHM,19,2 +BHM,20,4 +BHM,21,1 +BHX,2,37 +BHX,3,100 +BHX,4,234 +BHX,5,424 +BHX,6,724 +BHX,7,1040 +BHX,8,1217 +BHX,9,1253 +BHX,10,1017 +BHX,11,776 +BHX,12,579 +BHX,13,351 +BHX,14,220 +BHX,15,142 +BHX,16,86 +BHX,17,80 +BHX,18,40 +BHX,19,28 +BHX,20,26 +BHX,21,17 +BHX,22,8 +BHX,23,3 +BHX,24,2 +BHX,25,6 +BHX,26,5 +BHX,27,5 +BHX,30,1 +BRQ,2,1104 +BRQ,3,8357 +BRQ,4,21682 +BRQ,5,39038 +BRQ,6,61406 +BRQ,7,76780 +BRQ,8,73855 +BRQ,9,52110 +BRQ,10,39714 +BRQ,11,32747 +BRQ,12,27832 +BRQ,13,24129 +BRQ,14,20979 +BRQ,15,17812 +BRQ,16,15040 +BRQ,17,12796 +BRQ,18,10782 +BRQ,19,8714 +BRQ,20,7542 +BRQ,21,6439 +BRQ,22,5396 +BRQ,23,4428 +BRQ,24,3700 +BRQ,25,3073 +BRQ,26,2688 +BRQ,27,2297 +BRQ,28,1814 +BRQ,29,1444 +BRQ,30,1235 +BRQ,31,1036 +BRQ,32,875 +BRS,2,337 +BRS,3,975 +BRS,4,1244 +BRS,5,1123 +BRS,6,1323 +BRS,7,1599 +BRS,8,1920 +BRS,9,1327 +BRS,10,1014 +BRS,11,679 +BRS,12,422 +BRS,13,231 +BRS,14,170 +BRS,15,98 +BRS,16,24 +BRS,17,10 +BRS,18,1 +BRS,19,1 +BRS,20,5 +BRS,21,4 +BWE,2,2633 +BWE,3,17513 +BWE,4,41313 +BWE,5,58337 +BWE,6,55163 +BWE,7,45091 +BWE,8,31133 +BWE,9,19908 +BWE,10,14374 +BWE,11,10875 +BWE,12,7329 +BWE,13,4629 +BWE,14,2391 +BWE,15,1224 +BWE,16,608 +BWE,17,296 +BWE,18,151 +BWE,19,101 +BWE,20,58 +BWE,21,67 +BWE,22,6 +BWE,23,2 +BYU,2,240 +BYU,3,1498 +BYU,4,3790 +BYU,5,5337 +BYU,6,5480 +BYU,7,4480 +BYU,8,3200 +BYU,9,1885 +BYU,10,1076 +BYU,11,526 +BYU,12,304 +BYU,13,180 +BYU,14,108 +BYU,15,67 +BYU,16,54 +BYU,17,38 +BYU,18,26 +BYU,19,15 +BYU,20,14 +BYU,21,9 +CAX,2,215 +CAX,3,945 +CAX,4,1903 +CAX,5,2357 +CAX,6,2734 +CAX,7,2845 +CAX,8,2182 +CAX,9,1204 +CAX,10,757 +CAX,11,559 +CAX,12,346 +CAX,13,206 +CAX,14,123 +CAX,15,74 +CAX,16,37 +CAX,17,22 +CAX,18,5 +CAX,19,5 +CAX,20,5 +CEG,2,55 +CEG,3,241 +CEG,4,518 +CEG,5,584 +CEG,6,504 +CEG,7,407 +CEG,8,369 +CEG,9,261 +CEG,10,237 +CEG,11,174 +CEG,12,138 +CEG,13,102 +CEG,14,89 +CEG,15,66 +CEG,16,40 +CEG,17,19 +CEG,18,8 +CEG,19,4 +CEG,20,1 +CEG,21,2 +CGN,2,2900 +CGN,3,22628 +CGN,4,66615 +CGN,5,105203 +CGN,6,118167 +CGN,7,110212 +CGN,8,93500 +CGN,9,63186 +CGN,10,46899 +CGN,11,33386 +CGN,12,23188 +CGN,13,16155 +CGN,14,11046 +CGN,15,7524 +CGN,16,5214 +CGN,17,3311 +CGN,18,2153 +CGN,19,1322 +CGN,20,876 +CGN,21,624 +CGN,22,389 +CGN,23,241 +CGN,24,146 +CGN,25,87 +CGN,26,49 +CGN,27,29 +CGN,28,24 +CGN,29,22 +CGN,30,28 +CGN,31,10 +CGN,32,15 +CLE,2,151 +CLE,3,540 +CLE,4,839 +CLE,5,866 +CLE,6,614 +CLE,7,371 +CLE,8,173 +CLE,9,56 +CLE,10,31 +CLE,11,18 +CLE,12,8 +CLE,13,2 +CLE,14,1 +DEN,2,2114 +DEN,3,17664 +DEN,4,55619 +DEN,5,87250 +DEN,6,100100 +DEN,7,97461 +DEN,8,82599 +DEN,9,56469 +DEN,10,41195 +DEN,11,30130 +DEN,12,22577 +DEN,13,17228 +DEN,14,13100 +DEN,15,10178 +DEN,16,7712 +DEN,17,5771 +DEN,18,4381 +DEN,19,3489 +DEN,20,2712 +DEN,21,2101 +DEN,22,1576 +DEN,23,1164 +DEN,24,937 +DEN,25,819 +DEN,26,607 +DEN,27,420 +DEN,28,289 +DEN,29,207 +DEN,30,164 +DEN,31,139 +DEN,32,103 +DRS,2,834 +DRS,3,3763 +DRS,4,5402 +DRS,5,5736 +DRS,6,5997 +DRS,7,5089 +DRS,8,2771 +DRS,9,1114 +DRS,10,491 +DRS,11,237 +DRS,12,109 +DRS,13,52 +DRS,14,13 +DRS,15,9 +DRS,16,10 +DRS,17,3 +DUS,2,1878 +DUS,3,15820 +DUS,4,48015 +DUS,5,75330 +DUS,6,80590 +DUS,7,71623 +DUS,8,58991 +DUS,9,41493 +DUS,10,29200 +DUS,11,20627 +DUS,12,14240 +DUS,13,9405 +DUS,14,6282 +DUS,15,4117 +DUS,16,2699 +DUS,17,1704 +DUS,18,1189 +DUS,19,765 +DUS,20,475 +DUS,21,306 +DUS,22,177 +DUS,23,81 +DUS,24,43 +DUS,25,34 +DUS,26,15 +DUS,27,6 +DUS,28,11 +DUS,29,10 +DUS,30,11 +DUS,31,2 +DUS,32,1 +ELP,2,66 +ELP,3,172 +ELP,4,300 +ELP,5,779 +ELP,6,1128 +ELP,7,1446 +ELP,8,1482 +ELP,9,981 +ELP,10,669 +ELP,11,392 +ELP,12,277 +ELP,13,151 +ELP,14,63 +ELP,15,33 +ELP,16,20 +ELP,17,11 +ELP,18,4 +ELP,20,1 +ELP,21,1 +FLO,2,187 +FLO,3,1080 +FLO,4,2632 +FLO,5,3966 +FLO,6,4818 +FLO,7,4777 +FLO,8,4444 +FLO,9,2488 +FLO,10,1982 +FLO,11,875 +FLO,12,315 +FLO,13,223 +FLO,14,180 +FLO,15,139 +FLO,16,89 +FLO,17,59 +FLO,18,41 +FLO,19,24 +FLO,20,30 +FLO,21,15 +FLO,22,4 +FLO,23,4 +FLO,24,3 +FLO,25,6 +FLO,26,9 +FLO,27,1 +FRD,2,8 +FRD,3,36 +FRD,4,225 +FRD,5,595 +FRD,6,897 +FRD,7,843 +FRD,8,691 +FRD,9,414 +FRD,10,276 +FRD,11,206 +FRD,12,134 +FRD,13,107 +FRD,14,61 +FRD,15,51 +FRD,16,33 +FRD,17,25 +FRD,18,23 +FRD,19,13 +FRD,20,8 +FRD,21,7 +FRD,22,2 +FRD,23,2 +FRD,24,1 +FRD,26,1 +FRD,28,1 +FRD,30,1 +FRD,31,1 +GLA,2,89 +GLA,3,442 +GLA,4,1291 +GLA,5,1749 +GLA,6,1738 +GLA,7,1498 +GLA,8,1414 +GLA,9,1192 +GLA,10,834 +GLA,11,403 +GLA,12,212 +GLA,13,101 +GLA,14,43 +GLA,15,33 +GLA,16,14 +GLA,17,2 +GRR,2,158 +GRR,3,783 +GRR,4,1697 +GRR,5,2018 +GRR,6,1582 +GRR,7,1059 +GRR,8,490 +GRR,9,153 +GRR,10,76 +GRR,11,47 +GRR,12,12 +GRR,13,2 +HAD,2,229 +HAD,3,929 +HAD,4,1973 +HAD,5,3156 +HAD,6,5063 +HAD,7,6344 +HAD,8,5064 +HAD,9,2548 +HAD,10,1061 +HAD,11,419 +HAD,12,190 +HAD,13,112 +HAD,14,36 +HAD,15,10 +HAD,16,7 +HAD,17,1 +HAD,18,1 +HAD,19,1 +HEL,2,324 +HEL,3,1433 +HEL,4,3479 +HEL,5,4713 +HEL,6,4191 +HEL,7,3393 +HEL,8,3392 +HEL,9,3476 +HEL,10,3020 +HEL,11,1984 +HEL,12,968 +HEL,13,373 +HEL,14,100 +HEL,15,34 +HEL,16,12 +HEL,17,8 +HEL,18,3 +HEL,19,3 +HEL,20,1 +HEL,21,1 +HUY,2,257 +HUY,3,818 +HUY,4,1302 +HUY,5,1800 +HUY,6,2208 +HUY,7,2235 +HUY,8,2060 +HUY,9,1668 +HUY,10,1160 +HUY,11,617 +HUY,12,265 +HUY,13,162 +HUY,14,106 +HUY,15,59 +HUY,16,30 +HUY,17,17 +HUY,18,9 +HUY,19,5 +HUY,20,3 +HUY,21,2 +JKG,2,210 +JKG,3,885 +JKG,4,1775 +JKG,5,1857 +JKG,6,1183 +JKG,7,530 +JKG,8,183 +JKG,9,51 +JKG,10,15 +JKG,11,2 +JKG,12,1 +KLS,2,221 +KLS,3,1501 +KLS,4,5091 +KLS,5,9814 +KLS,6,12860 +KLS,7,13596 +KLS,8,12559 +KLS,9,9411 +KLS,10,7370 +KLS,11,5913 +KLS,12,4673 +KLS,13,3444 +KLS,14,2556 +KLS,15,1860 +KLS,16,1467 +KLS,17,1065 +KLS,18,841 +KLS,19,666 +KLS,20,524 +KLS,21,423 +KLS,22,102 +KLS,23,84 +KLS,24,63 +KLS,25,40 +KLS,26,33 +KLS,27,22 +KLS,28,17 +KLS,29,19 +KLS,30,5 +KLS,31,6 +KLS,32,14 +LIS,2,783 +LIS,3,3303 +LIS,4,6146 +LIS,5,9003 +LIS,6,11541 +LIS,7,11989 +LIS,8,11460 +LIS,9,9018 +LIS,10,7696 +LIS,11,6592 +LIS,12,5415 +LIS,13,4460 +LIS,14,3663 +LIS,15,2805 +LIS,16,2141 +LIS,17,1434 +LIS,18,1170 +LIS,19,820 +LIS,20,681 +LIS,21,473 +LIS,22,348 +LIS,23,294 +LIS,24,190 +LIS,25,147 +LIS,26,123 +LIS,27,107 +LIS,28,78 +LIS,29,52 +LIS,30,48 +LIS,31,42 +LIS,32,41 +LOZ,2,19 +LOZ,3,85 +LOZ,4,273 +LOZ,5,972 +LOZ,6,2181 +LOZ,7,3423 +LOZ,8,4531 +LOZ,9,4351 +LOZ,10,3559 +LOZ,11,2482 +LOZ,12,1861 +LOZ,13,1341 +LOZ,14,955 +LOZ,15,717 +LOZ,16,525 +LOZ,17,404 +LOZ,18,318 +LOZ,19,254 +LOZ,20,203 +LOZ,21,168 +LOZ,22,90 +LOZ,23,65 +LOZ,24,65 +LOZ,25,56 +LOZ,26,42 +LOZ,27,30 +LOZ,28,29 +LOZ,29,24 +LOZ,30,12 +LOZ,31,10 +LOZ,32,11 +LRU,2,328 +LRU,3,1463 +LRU,4,2627 +LRU,5,4883 +LRU,6,6070 +LRU,7,4584 +LRU,8,2638 +LRU,9,1164 +LRU,10,582 +LRU,11,255 +LRU,12,169 +LRU,13,110 +LRU,14,51 +LRU,15,21 +LRU,16,14 +LRU,17,5 +LRU,18,1 +LRU,19,1 +MAD,2,799 +MAD,3,4457 +MAD,4,6721 +MAD,5,4850 +MAD,6,2306 +MAD,7,1039 +MAD,8,533 +MAD,9,278 +MAD,10,122 +MAD,11,49 +MAD,12,22 +MAD,13,7 +MAD,14,10 +MAD,15,6 +MAD,16,1 +MAN,2,225 +MAN,3,822 +MAN,4,1575 +MAN,5,2184 +MAN,6,1996 +MAN,7,1831 +MAN,8,1789 +MAN,9,1728 +MAN,10,1629 +MAN,11,1437 +MAN,12,1234 +MAN,13,911 +MAN,14,711 +MAN,15,531 +MAN,16,416 +MAN,17,302 +MAN,18,189 +MAN,19,138 +MAN,20,87 +MAN,21,49 +MAN,22,23 +MAN,23,16 +MAN,24,5 +MAN,25,4 +MAN,26,2 +MAN,32,1 +MCE,2,384 +MCE,3,2008 +MCE,4,7236 +MCE,5,13857 +MCE,6,15221 +MCE,7,12843 +MCE,8,10008 +MCE,9,6445 +MCE,10,4982 +MCE,11,3960 +MCE,12,3145 +MCE,13,2687 +MCE,14,2195 +MCE,15,1822 +MCE,16,1405 +MCE,17,1110 +MCE,18,884 +MCE,19,735 +MCE,20,638 +MCE,21,514 +MCE,22,329 +MCE,23,252 +MCE,24,203 +MCE,25,136 +MCE,26,115 +MCE,27,87 +MCE,28,80 +MCE,29,70 +MCE,30,45 +MCE,31,29 +MCE,32,33 +MDT,2,249 +MDT,3,830 +MDT,4,1491 +MDT,5,2654 +MDT,6,4073 +MDT,7,4600 +MDT,8,4253 +MDT,9,3374 +MDT,10,2170 +MDT,11,1237 +MDT,12,804 +MDT,13,482 +MDT,14,318 +MDT,15,153 +MDT,16,84 +MDT,17,71 +MDT,18,52 +MDT,19,24 +MDT,20,10 +MDT,21,4 +MDT,22,5 +MDT,23,2 +MLB,2,48 +MLB,3,246 +MLB,4,683 +MLB,5,1192 +MLB,6,1079 +MLB,7,602 +MLB,8,352 +MLB,9,227 +MLB,10,183 +MLB,11,169 +MLB,12,111 +MLB,13,49 +MLB,14,14 +MLB,15,11 +MLB,16,6 +MLB,17,3 +MLB,18,1 +MLB,19,2 +MLB,20,1 +MOW,2,4418 +MOW,3,33970 +MOW,4,92858 +MOW,5,128545 +MOW,6,119199 +MOW,7,89127 +MOW,8,62012 +MOW,9,37524 +MOW,10,23799 +MOW,11,15478 +MOW,12,10203 +MOW,13,6802 +MOW,14,4595 +MOW,15,2920 +MOW,16,2035 +MOW,17,1475 +MOW,18,1067 +MOW,19,829 +MOW,20,645 +MOW,21,456 +MOW,22,362 +MOW,23,238 +MOW,24,169 +MOW,25,128 +MOW,26,106 +MOW,27,69 +MOW,28,42 +MOW,29,25 +MOW,30,10 +MOW,31,5 +MOW,32,8 +MSP,2,518 +MSP,3,2536 +MSP,4,5444 +MSP,5,6929 +MSP,6,5431 +MSP,7,2987 +MSP,8,1336 +MSP,9,525 +MSP,10,280 +MSP,11,151 +MSP,12,87 +MSP,13,35 +MSP,14,14 +MSP,15,16 +MSP,16,4 +NCE,2,79 +NCE,3,307 +NCE,4,1040 +NCE,5,1947 +NCE,6,2520 +NCE,7,2719 +NCE,8,2450 +NCE,9,1813 +NCE,10,1279 +NCE,11,783 +NCE,12,479 +NCE,13,244 +NCE,14,118 +NCE,15,95 +NCE,16,68 +NCE,17,31 +NCE,18,26 +NCE,19,9 +NCE,20,1 +NCE,21,5 +NCE,22,3 +NCE,23,1 +NCE,24,1 +NCE,25,3 +NCE,26,1 +NCE,28,3 +NCE,30,1 +NPL,2,98 +NPL,3,431 +NPL,4,1398 +NPL,5,3156 +NPL,6,4850 +NPL,7,5198 +NPL,8,4121 +NPL,9,2437 +NPL,10,2174 +NPL,11,2226 +NPL,12,1758 +NPL,13,1114 +NPL,14,560 +NPL,15,259 +NPL,16,128 +NPL,17,85 +NPL,18,26 +NPL,19,18 +NPL,20,12 +NPL,21,13 +NPL,22,8 +NPL,23,6 +NPL,24,7 +NPL,25,2 +NPL,26,1 +NPL,32,1 +NSN,2,283 +NSN,3,1598 +NSN,4,2779 +NSN,5,2958 +NSN,6,3247 +NSN,7,3595 +NSN,8,3208 +NSN,9,2051 +NSN,10,1265 +NSN,11,733 +NSN,12,404 +NSN,13,263 +NSN,14,145 +NSN,15,88 +NSN,16,65 +NSN,17,32 +NSN,18,8 +NSN,19,12 +NSN,20,7 +NSN,21,2 +NSN,22,4 +NSN,23,5 +NSN,24,1 +NSN,26,1 +NSN,27,1 +NSN,28,1 +NUW,2,185 +NUW,3,1176 +NUW,4,2732 +NUW,5,3374 +NUW,6,2976 +NUW,7,2298 +NUW,8,1792 +NUW,9,1080 +NUW,10,711 +NUW,11,557 +NUW,12,422 +NUW,13,344 +NUW,14,244 +NUW,15,149 +NUW,16,168 +NUW,17,116 +NUW,18,60 +NUW,19,70 +NUW,20,44 +NUW,21,50 +NUW,22,7 +NUW,23,4 +NUW,24,1 +NUW,25,1 +NUW,26,2 +NUW,28,1 +NUW,31,1 +NYC,2,173 +NYC,3,679 +NYC,4,1604 +NYC,5,2601 +NYC,6,3347 +NYC,7,3612 +NYC,8,3244 +NYC,9,2041 +NYC,10,1542 +NYC,11,1214 +NYC,12,887 +NYC,13,635 +NYC,14,445 +NYC,15,293 +NYC,16,185 +NYC,17,110 +NYC,18,105 +NYC,19,68 +NYC,20,32 +NYC,21,26 +NYC,22,17 +NYC,23,13 +NYC,24,9 +NYC,25,5 +NYC,26,9 +NYC,27,5 +NYC,28,2 +NYC,30,1 +NYC,31,4 +OSL,2,474 +OSL,3,2727 +OSL,4,6234 +OSL,5,7121 +OSL,6,6115 +OSL,7,3559 +OSL,8,2169 +OSL,9,1331 +OSL,10,760 +OSL,11,403 +OSL,12,222 +OSL,13,98 +OSL,14,43 +OSL,15,23 +OSL,16,9 +OSL,17,9 +OSL,18,17 +OSL,19,7 +OSL,20,2 +OSR,2,19 +OSR,3,156 +OSR,4,422 +OSR,5,1122 +OSR,6,2393 +OSR,7,3608 +OSR,8,4066 +OSR,9,2536 +OSR,10,1975 +OSR,11,1547 +OSR,12,1297 +OSR,13,1128 +OSR,14,1016 +OSR,15,843 +OSR,16,705 +OSR,17,648 +OSR,18,537 +OSR,19,469 +OSR,20,374 +OSR,21,309 +OSR,22,282 +OSR,23,193 +OSR,24,184 +OSR,25,157 +OSR,26,116 +OSR,27,108 +OSR,28,77 +OSR,29,58 +OSR,30,72 +OSR,31,48 +OSR,32,40 +PHL,2,362 +PHL,3,2077 +PHL,4,4112 +PHL,5,4147 +PHL,6,2560 +PHL,7,1365 +PHL,8,571 +PHL,9,274 +PHL,10,104 +PHL,11,51 +PHL,12,21 +PHL,13,3 +PHL,19,1 +PIT,2,213 +PIT,3,689 +PIT,4,1474 +PIT,5,2294 +PIT,6,2350 +PIT,7,1780 +PIT,8,1171 +PIT,9,527 +PIT,10,254 +PIT,11,117 +PIT,12,65 +PIT,13,43 +PIT,14,35 +PIT,15,9 +PIT,16,8 +PIT,17,5 +PIT,18,5 +PIT,20,1 +PIT,21,2 +PPM,2,213 +PPM,3,1174 +PPM,4,1811 +PPM,5,1755 +PPM,6,1547 +PPM,7,1413 +PPM,8,1362 +PPM,9,1192 +PPM,10,918 +PPM,11,774 +PPM,12,521 +PPM,13,271 +PPM,14,115 +PPM,15,53 +PPM,16,20 +PPM,17,6 +PPM,18,7 +PPM,19,3 +PPM,20,1 +PRG,2,2926 +PRG,3,20493 +PRG,4,59190 +PRG,5,93934 +PRG,6,107421 +PRG,7,110493 +PRG,8,107781 +PRG,9,88971 +PRG,10,74326 +PRG,11,61666 +PRG,12,51885 +PRG,13,43919 +PRG,14,36252 +PRG,15,29694 +PRG,16,24119 +PRG,17,19815 +PRG,18,16634 +PRG,19,13687 +PRG,20,11213 +PRG,21,9039 +PRG,22,7266 +PRG,23,5945 +PRG,24,4985 +PRG,25,4085 +PRG,26,3385 +PRG,27,2794 +PRG,28,2313 +PRG,29,1975 +PRG,30,1679 +PRG,31,1381 +PRG,32,1174 +PSC,2,334 +PSC,3,1527 +PSC,4,4174 +PSC,5,8762 +PSC,6,14199 +PSC,7,17203 +PSC,8,16354 +PSC,9,11726 +PSC,10,9045 +PSC,11,6946 +PSC,12,5018 +PSC,13,3625 +PSC,14,2589 +PSC,15,1926 +PSC,16,1339 +PSC,17,808 +PSC,18,547 +PSC,19,413 +PSC,20,321 +PSC,21,248 +PSC,22,36 +PSC,23,26 +PSC,24,22 +PSC,25,18 +PSC,26,11 +PSC,27,5 +PSC,28,1 +PSC,30,1 +PSC,31,3 +PSC,32,2 +PUB,2,37 +PUB,3,160 +PUB,4,607 +PUB,5,1790 +PUB,6,3173 +PUB,7,3855 +PUB,8,3475 +PUB,9,2312 +PUB,10,1525 +PUB,11,1055 +PUB,12,669 +PUB,13,496 +PUB,14,383 +PUB,15,269 +PUB,16,222 +PUB,17,182 +PUB,18,131 +PUB,19,88 +PUB,20,52 +PUB,21,49 +PUB,22,44 +PUB,23,28 +PUB,24,25 +PUB,25,18 +PUB,26,12 +PUB,27,6 +PUB,28,9 +PUB,29,6 +PUB,30,3 +PUB,31,4 +PUW,2,68 +PUW,3,264 +PUW,4,819 +PUW,5,1616 +PUW,6,2170 +PUW,7,2255 +PUW,8,1847 +PUW,9,1111 +PUW,10,736 +PUW,11,448 +PUW,12,342 +PUW,13,224 +PUW,14,158 +PUW,15,121 +PUW,16,80 +PUW,17,68 +PUW,18,30 +PUW,19,22 +PUW,20,9 +PUW,21,2 +RTM,2,2070 +RTM,3,13934 +RTM,4,43198 +RTM,5,76793 +RTM,6,92533 +RTM,7,90882 +RTM,8,80603 +RTM,9,60539 +RTM,10,47368 +RTM,11,36730 +RTM,12,28443 +RTM,13,21288 +RTM,14,15796 +RTM,15,11453 +RTM,16,8354 +RTM,17,5647 +RTM,18,4061 +RTM,19,2985 +RTM,20,2092 +RTM,21,1549 +RTM,22,814 +RTM,23,582 +RTM,24,413 +RTM,25,280 +RTM,26,203 +RTM,27,93 +RTM,28,65 +RTM,29,45 +RTM,30,39 +RTM,31,41 +RTM,32,17 +SBP,2,67 +SBP,3,269 +SBP,4,720 +SBP,5,1793 +SBP,6,3859 +SBP,7,4550 +SBP,8,3777 +SBP,9,2685 +SBP,10,2058 +SBP,11,1517 +SBP,12,1181 +SBP,13,937 +SBP,14,764 +SBP,15,704 +SBP,16,553 +SBP,17,458 +SBP,18,373 +SBP,19,319 +SBP,20,280 +SBP,21,234 +SBP,22,154 +SBP,23,149 +SBP,24,125 +SBP,25,81 +SBP,26,59 +SBP,27,49 +SBP,28,49 +SBP,29,44 +SBP,30,27 +SBP,31,24 +SBP,32,10 +SCE,2,237 +SCE,3,1139 +SCE,4,2503 +SCE,5,3562 +SCE,6,3971 +SCE,7,3269 +SCE,8,2276 +SCE,9,1189 +SCE,10,659 +SCE,11,351 +SCE,12,168 +SCE,13,117 +SCE,14,44 +SCE,15,30 +SCE,16,23 +SCE,17,13 +SCE,18,6 +SCE,19,3 +SCE,20,2 +SCE,22,3 +SCE,23,6 +SCE,24,1 +SEA,2,8241 +SEA,3,68138 +SEA,4,179363 +SEA,5,252561 +SEA,6,263117 +SEA,7,232624 +SEA,8,197867 +SEA,9,149681 +SEA,10,120278 +SEA,11,96610 +SEA,12,77165 +SEA,13,61708 +SEA,14,49306 +SEA,15,39242 +SEA,16,31221 +SEA,17,23822 +SEA,18,19297 +SEA,19,15575 +SEA,20,13001 +SEA,21,11685 +SEA,22,1046 +SEA,23,810 +SEA,24,568 +SEA,25,393 +SEA,26,282 +SEA,27,229 +SEA,28,180 +SEA,29,105 +SEA,30,63 +SEA,31,39 +SEA,32,18 +SGU,2,620 +SGU,3,2879 +SGU,4,4183 +SGU,5,1997 +SGU,6,540 +SGU,7,146 +SGU,8,51 +SGU,9,21 +SGU,10,7 +SGU,11,4 +SJO,2,205 +SJO,3,885 +SJO,4,1533 +SJO,5,1335 +SJO,6,750 +SJO,7,371 +SJO,8,129 +SJO,9,43 +SJO,10,16 +SJO,11,1 +SJO,12,1 +SMF,2,534 +SMF,3,2545 +SMF,4,4076 +SMF,5,3095 +SMF,6,1416 +SMF,7,557 +SMF,8,225 +SMF,9,71 +SMF,10,38 +SMF,11,15 +SMF,12,4 +SOF,2,1642 +SOF,3,8299 +SOF,4,11912 +SOF,5,12253 +SOF,6,13054 +SOF,7,12739 +SOF,8,10691 +SOF,9,6685 +SOF,10,4597 +SOF,11,3651 +SOF,12,2456 +SOF,13,1649 +SOF,14,1134 +SOF,15,722 +SOF,16,425 +SOF,17,246 +SOF,18,157 +SOF,19,83 +SOF,20,47 +SOF,21,14 +SOF,22,4 +SOF,23,5 +SOF,24,3 +SYD,2,4482 +SYD,3,39944 +SYD,4,84331 +SYD,5,86079 +SYD,6,75547 +SYD,7,65408 +SYD,8,60837 +SYD,9,52944 +SYD,10,43708 +SYD,11,34125 +SYD,12,26583 +SYD,13,20926 +SYD,14,15173 +SYD,15,11302 +SYD,16,9076 +SYD,17,6867 +SYD,18,5361 +SYD,19,4091 +SYD,20,3147 +SYD,21,2538 +SYD,22,1530 +SYD,23,1042 +SYD,24,822 +SYD,25,584 +SYD,26,421 +SYD,27,352 +SYD,28,304 +SYD,29,248 +SYD,30,158 +SYD,31,143 +SYD,32,145 +SYD,42,1 +SYD,43,1 +SYD,46,1 +SYD,53,1 +SYD,63,1 +TGN,2,204 +TGN,3,1439 +TGN,4,4323 +TGN,5,6113 +TGN,6,5671 +TGN,7,4272 +TGN,8,3002 +TGN,9,1828 +TGN,10,1365 +TGN,11,981 +TGN,12,733 +TGN,13,553 +TGN,14,420 +TGN,15,274 +TGN,16,177 +TGN,17,133 +TGN,18,78 +TGN,19,62 +TGN,20,53 +TGN,21,33 +TGN,22,32 +TGN,23,15 +TGN,24,11 +TGN,25,13 +TGN,26,14 +TGN,27,6 +TGN,28,7 +TGN,29,6 +TGN,30,2 +TGN,31,2 +TGN,32,3 +TWF,2,331 +TWF,3,1764 +TWF,4,4307 +TWF,5,5972 +TWF,6,5121 +TWF,7,3044 +TWF,8,1488 +TWF,9,623 +TWF,10,262 +TWF,11,106 +TWF,12,83 +TWF,13,34 +TWF,14,14 +TWF,15,15 +TWF,16,15 +TWF,17,8 +TWF,18,4 +TWF,19,1 +YOW,2,1622 +YOW,3,13108 +YOW,4,29218 +YOW,5,28671 +YOW,6,16977 +YOW,7,7984 +YOW,8,3663 +YOW,9,1721 +YOW,10,931 +YOW,11,501 +YOW,12,276 +YOW,13,116 +YOW,14,53 +YOW,15,18 +YOW,16,18 +YOW,17,9 diff --git a/db/planguard/fixture/testdata/routes_obs.csv b/db/planguard/fixture/testdata/routes_obs.csv new file mode 100644 index 0000000..1265680 --- /dev/null +++ b/db/planguard/fixture/testdata/routes_obs.csv @@ -0,0 +1,51 @@ +obs,ge3,n +1,f,2180026 +2,f,3276578 +3,t,1598216 +4,t,1439827 +5,t,376803 +6,t,512051 +7,t,188358 +8,t,220349 +9,t,99959 +10,t,97137 +11,t,50291 +12,t,76557 +13,t,27270 +14,t,40627 +15,t,26394 +16,t,29545 +17,t,14914 +18,t,24921 +19,t,11768 +20,t,17870 +21,t,10983 +22,t,13347 +23,t,7830 +24,t,12435 +25,t,6244 +26,t,8856 +27,t,6300 +28,t,7642 +29,t,4791 +30,t,7162 +31,t,4204 +32,t,5839 +33,t,4203 +34,t,4734 +35,t,3333 +36,t,4852 +37,t,2832 +38,t,3861 +39,t,2939 +40,t,3578 +41,t,2319 +42,t,3376 +43,t,2148 +44,t,2737 +45,t,2067 +46,t,2299 +47,t,1715 +48,t,2379 +49,t,1566 +50,t,74045 diff --git a/db/planguard/fixture/testdata/routes_regions.csv b/db/planguard/fixture/testdata/routes_regions.csv new file mode 100644 index 0000000..016b3e4 --- /dev/null +++ b/db/planguard/fixture/testdata/routes_regions.csv @@ -0,0 +1,408 @@ +iata,prod_rows +SEA,1947382 +AMS,1379665 +PRG,1037310 +CGN,730538 +RTM,658832 +MOW,655058 +SYD,644275 +DEN,643736 +BRQ,607613 +DUS,490619 +PDX,459888 +BER,429696 +ATL,424844 +FNL,421070 +LAX,396809 +BNA,393035 +SJC,371469 +BOS,369312 +CHA,368234 +TYS,364999 +IDA,355834 +RDU,348825 +BDL,347208 +ORD,342894 +DTM,333190 +MEL,322946 +SAN,311085 +LYS,308929 +FRA,305155 +BWE,302459 +EIN,296528 +OAK,293293 +TRI,288980 +SFO,270110 +KTW,263641 +CVG,256093 +AUS,249623 +MSY,241536 +STR,239918 +KRK,239379 +PAE,239379 +BRN,235066 +AVL,233988 +VOD,226979 +PAD,225901 +BTS,210265 +YVR,207570 +MSN,204874 +AKL,200561 +ENS,197865 +UTC,193013 +YYZ,193013 +SPM,178995 +DAY,176300 +BHS,173604 +SLC,172526 +VIE,170369 +POZ,169291 +PAR,167134 +SCN,160664 +PED,159047 +HRA,158508 +MMX,151499 +OMA,147186 +NUE,140177 +PRB,140177 +LEY,139638 +GDN,137481 +GRQ,132629 +LIN,131551 +BRU,128855 +VNO,127777 +WOL,127777 +WRO,126159 +RIX,124003 +ALB,123464 +BUF,119690 +YUL,118611 +BHD,114837 +ODE,112142 +PSC,107828 +CPH,106750 +KLS,105672 +LEJ,105133 +LIS,105133 +YOW,104054 +MCE,97585 +BOI,96506 +SOF,96506 +ESS,95428 +ANR,94889 +COA,94889 +TRN,90576 +DFW,88958 +RDO,88958 +PSK,88419 +HAM,85184 +BLI,84645 +WAW,84645 +MUC,80871 +KSF,79254 +AAR,76019 +BNE,74941 +NTL,74402 +SQL,74402 +HOH,72784 +AHN,70088 +KLU,67393 +BLL,66854 +GSP,66854 +LGG,65775 +YCD,64697 +YEG,63619 +TRG,63080 +YKF,62540 +FAT,62001 +GEG,62001 +CBR,61462 +HAJ,59306 +QAR,59306 +COS,58227 +GSO,58227 +UDE,55532 +GEX,54993 +GPT,54453 +SAT,54453 +PIH,53375 +ABQ,52836 +TLL,52836 +ADL,50679 +LUX,49601 +OXR,49601 +CRW,49062 +YKM,49062 +KZG,48523 +NTE,48523 +SLE,46905 +ROV,44749 +BUD,44210 +EGE,44210 +RDM,43671 +EAT,41514 +EUG,41514 +MST,40975 +WLG,39357 +YYJ,38279 +UHE,35044 +YYC,34505 +AAH,33966 +TGN,33966 +IAD,33427 +OSL,32888 +TTD,32888 +FLO,31270 +KUN,31270 +HEL,30731 +BEG,30192 +DRS,30192 +ERF,30192 +BYU,29653 +CON,29653 +HAD,29114 +ONT,29114 +NPL,28575 +SBA,28575 +LOZ,28035 +VIS,26957 +SBP,26418 +MRO,25879 +OSR,25879 +MFR,25340 +NYC,25340 +YTR,25340 +ABE,24801 +GRZ,24801 +LRU,24801 +LWR,24801 +NSN,24801 +LCJ,24261 +MDT,24261 +FMM,23722 +MAD,23183 +PNS,23183 +TWF,23183 +MKE,22644 +SCE,22644 +AAL,22105 +MAN,22105 +ORY,22105 +PUB,21566 +TSR,21027 +MSP,20487 +EPH,19409 +NCE,19409 +FKB,18870 +GLA,18870 +GLZ,18331 +PHL,17253 +BCN,16713 +BHM,19893 +ACH,18601 +NUW,18421 +CAX,16529 +HUY,14831 +PPM,13127 +SMF,12550 +BRS,12459 +PUW,12311 +PIT,11118 +SGU,10520 +BHX,8397 +GRR,8082 +ELP,7935 +JKG,6706 +SJO,5283 +MLB,4982 +FRD,4652 +CEG,3806 +CLE,3648 +T00,3600 +T01,3600 +T02,3600 +T03,3600 +T04,3600 +T05,3600 +T06,3600 +T07,3600 +T08,3600 +T09,3600 +T10,3600 +T11,3600 +T12,3600 +T13,3600 +T14,3599 +T15,3599 +T16,3599 +T17,3599 +T18,3599 +T19,3599 +T20,3599 +T21,3599 +T22,3599 +T23,3599 +T24,3599 +T25,3599 +T26,3599 +T27,3599 +T28,3599 +T29,3599 +T30,3599 +T31,3599 +T32,3599 +T33,3599 +T34,3599 +T35,3599 +T36,3599 +T37,3599 +T38,3599 +T39,3599 +T40,3599 +T41,3599 +T42,3598 +T43,3598 +T44,3598 +T45,3598 +T46,3598 +T47,3598 +T48,3598 +T49,3598 +T50,3598 +T51,3598 +T52,3598 +T53,3598 +T54,3598 +T55,3598 +T56,3598 +T57,3598 +T58,3598 +T59,3598 +T60,3598 +T61,3598 +T62,3598 +T63,3598 +T64,3598 +T65,3598 +T66,3598 +T67,3598 +T68,3598 +T69,3598 +T70,3597 +T71,3597 +T72,3597 +T73,3597 +T74,3597 +T75,3597 +T76,3597 +T77,3597 +T78,3597 +T79,3597 +T80,3597 +T81,3597 +T82,3597 +T83,3597 +T84,3597 +T85,3597 +T86,3597 +T87,3597 +T88,3597 +T89,3597 +T90,3597 +T91,3597 +T92,3597 +T93,3597 +T94,3597 +T95,3597 +T96,3597 +T97,3597 +T98,3596 +T99,3596 +U00,3596 +U01,3596 +U02,3596 +U03,3596 +U04,3596 +U05,3596 +U06,3596 +U07,3596 +U08,3596 +U09,3596 +U10,3596 +U11,3596 +U12,3596 +U13,3596 +U14,3596 +U15,3596 +U16,3596 +U17,3596 +U18,3596 +U19,3596 +U20,3596 +U21,3596 +U22,3596 +U23,3596 +U24,3596 +U25,3596 +U26,3595 +U27,3595 +U28,3595 +U29,3595 +U30,3595 +U31,3595 +U32,3595 +U33,3595 +U34,3595 +U35,3595 +U36,3595 +U37,3595 +U38,3595 +U39,3595 +U40,3595 +U41,3595 +U42,3595 +U43,3595 +U44,3595 +U45,3595 +U46,3595 +U47,3595 +U48,3595 +U49,3595 +U50,3595 +U51,3595 +U52,3595 +U53,3594 +U54,3594 +U55,3594 +U56,3594 +U57,3594 +U58,3594 +U59,3594 +U60,3594 +U61,3594 +U62,3594 +U63,3594 +U64,3594 +U65,3594 +U66,3594 +U67,3594 +U68,3594 +U69,3594 +U70,3594 +U71,3594 +U72,3594 +U73,3594 +U74,3594 +U75,3594 +U76,3594 +U77,3594 +U78,3594 +U79,3594 +U80,3594 +U81,3593 +U82,3593 +U83,3593 +U84,3593 +U85,3593 +U86,3593 +U87,3593 diff --git a/db/planguard/fixture/testdata/routes_relay.csv b/db/planguard/fixture/testdata/routes_relay.csv new file mode 100644 index 0000000..04b8afb --- /dev/null +++ b/db/planguard/fixture/testdata/routes_relay.csv @@ -0,0 +1,2196 @@ +rank,n +1,537581 +2,453351 +3,428531 +4,412938 +5,271796 +6,210506 +7,184318 +8,169045 +9,160033 +10,145443 +11,140763 +12,138944 +13,132819 +14,130307 +15,126950 +16,125112 +17,121711 +18,111945 +19,110279 +20,109038 +21,100720 +22,98663 +23,96738 +24,95035 +25,94420 +26,93603 +27,89352 +28,88285 +29,87241 +30,85167 +31,83450 +32,79882 +33,78148 +34,77579 +35,77428 +36,72617 +37,72538 +38,72423 +39,71020 +40,69056 +41,62917 +42,62653 +43,62112 +44,62038 +45,61511 +46,60528 +47,60193 +48,59744 +49,58772 +50,58657 +51,58205 +52,58140 +53,57637 +54,56228 +55,55251 +56,54965 +57,54771 +58,54705 +59,54276 +60,54256 +61,54213 +62,53792 +63,53685 +64,51626 +65,51426 +66,49994 +67,49542 +68,48541 +69,48114 +70,47561 +71,46093 +72,45725 +73,45214 +74,45011 +75,43914 +76,43820 +77,43805 +78,42843 +79,42220 +80,41640 +81,40905 +82,40755 +83,40623 +84,40030 +85,39791 +86,39082 +87,39039 +88,38513 +89,38308 +90,38221 +91,38065 +92,37735 +93,37457 +94,36670 +95,36466 +96,34526 +97,34489 +98,34116 +99,33928 +100,33756 +101,33616 +102,33468 +103,33384 +104,33113 +105,33103 +106,33086 +107,32585 +108,32431 +109,32424 +110,32408 +111,31689 +112,31381 +113,31114 +114,31071 +115,30726 +116,30637 +117,30551 +118,30516 +119,30291 +120,30260 +121,30222 +122,30050 +123,30013 +124,29770 +125,29518 +126,29480 +127,29460 +128,29236 +129,29230 +130,28950 +131,28666 +132,28458 +133,28365 +134,28341 +135,28287 +136,28173 +137,28090 +138,27762 +139,27094 +140,27054 +141,26736 +142,26079 +143,25401 +144,25353 +145,25332 +146,25236 +147,25232 +148,24981 +149,24842 +150,24694 +151,24349 +152,23696 +153,23549 +154,23532 +155,23361 +156,23337 +157,23026 +158,22978 +159,22966 +160,22960 +161,22656 +162,22606 +163,22557 +164,22527 +165,22492 +166,22419 +167,22304 +168,22154 +169,22109 +170,21943 +171,21939 +172,21834 +173,21810 +174,21699 +175,20992 +176,20967 +177,20924 +178,20571 +179,20416 +180,20396 +181,20382 +182,20301 +183,20189 +184,19903 +185,19756 +186,19706 +187,19679 +188,19668 +189,19521 +190,19082 +191,19051 +192,18922 +193,18762 +194,18718 +195,18543 +196,18150 +197,18129 +198,18058 +199,18036 +200,17865 +201,17840 +202,17722 +203,17471 +204,17428 +205,17387 +206,17385 +207,17348 +208,17337 +209,17225 +210,17194 +211,17180 +212,17038 +213,17038 +214,16946 +215,16908 +216,16869 +217,16838 +218,16714 +219,16676 +220,16673 +221,16596 +222,16572 +223,16424 +224,16384 +225,16362 +226,16341 +227,16222 +228,16142 +229,16099 +230,16058 +231,16057 +232,15987 +233,15978 +234,15926 +235,15621 +236,15581 +237,15441 +238,15393 +239,15377 +240,15306 +241,15190 +242,15162 +243,15014 +244,14763 +245,14691 +246,14600 +247,14549 +248,14535 +249,14532 +250,14511 +251,14406 +252,14397 +253,14354 +254,14327 +255,14252 +256,14239 +257,14147 +258,14117 +259,14102 +260,14011 +261,13996 +262,13751 +263,13744 +264,13598 +265,13570 +266,13345 +267,13273 +268,13124 +269,12997 +270,12982 +271,12948 +272,12936 +273,12928 +274,12817 +275,12771 +276,12717 +277,12669 +278,12592 +279,12524 +280,12503 +281,12371 +282,12369 +283,12367 +284,12314 +285,12300 +286,11942 +287,11938 +288,11858 +289,11847 +290,11833 +291,11830 +292,11809 +293,11683 +294,11612 +295,11546 +296,11528 +297,11491 +298,11420 +299,11402 +300,11328 +301,11309 +302,11307 +303,11304 +304,11302 +305,11216 +306,11072 +307,11016 +308,10968 +309,10754 +310,10752 +311,10729 +312,10654 +313,10573 +314,10549 +315,10548 +316,10506 +317,10496 +318,10465 +319,10452 +320,10445 +321,10359 +322,10359 +323,10332 +324,10294 +325,10286 +326,10243 +327,10163 +328,10069 +329,10038 +330,10035 +331,10029 +332,9959 +333,9933 +334,9932 +335,9888 +336,9849 +337,9841 +338,9813 +339,9806 +340,9780 +341,9769 +342,9723 +343,9672 +344,9644 +345,9629 +346,9596 +347,9574 +348,9535 +349,9509 +350,9459 +351,9438 +352,9401 +353,9306 +354,9288 +355,9281 +356,9166 +357,9158 +358,9136 +359,9105 +360,9046 +361,9005 +362,8972 +363,8964 +364,8905 +365,8905 +366,8885 +367,8870 +368,8841 +369,8788 +370,8720 +371,8656 +372,8654 +373,8650 +374,8603 +375,8582 +376,8565 +377,8447 +378,8387 +379,8379 +380,8377 +381,8359 +382,8271 +383,8264 +384,8202 +385,8201 +386,8172 +387,8163 +388,8143 +389,8126 +390,8098 +391,8079 +392,8079 +393,8023 +394,8008 +395,7962 +396,7957 +397,7950 +398,7925 +399,7916 +400,7902 +401,7865 +402,7834 +403,7816 +404,7778 +405,7751 +406,7748 +407,7715 +408,7705 +409,7694 +410,7617 +411,7605 +412,7600 +413,7580 +414,7548 +415,7526 +416,7525 +417,7521 +418,7494 +419,7491 +420,7479 +421,7474 +422,7392 +423,7349 +424,7318 +425,7266 +426,7145 +427,7115 +428,7026 +429,7019 +430,6965 +431,6963 +432,6963 +433,6951 +434,6915 +435,6904 +436,6898 +437,6887 +438,6855 +439,6769 +440,6657 +441,6653 +442,6620 +443,6601 +444,6570 +445,6563 +446,6501 +447,6488 +448,6469 +449,6468 +450,6460 +451,6361 +452,6333 +453,6328 +454,6313 +455,6311 +456,6306 +457,6289 +458,6268 +459,6244 +460,6239 +461,6230 +462,6168 +463,6167 +464,6106 +465,6105 +466,6087 +467,6080 +468,6029 +469,6027 +470,6011 +471,6004 +472,5989 +473,5980 +474,5915 +475,5907 +476,5883 +477,5806 +478,5783 +479,5759 +480,5750 +481,5668 +482,5663 +483,5613 +484,5605 +485,5602 +486,5585 +487,5518 +488,5481 +489,5477 +490,5474 +491,5450 +492,5396 +493,5387 +494,5363 +495,5363 +496,5360 +497,5347 +498,5337 +499,5300 +500,5265 +501,5263 +502,5200 +503,5173 +504,5154 +505,5141 +506,5141 +507,5135 +508,5086 +509,5085 +510,5082 +511,5034 +512,5024 +513,5022 +514,4970 +515,4889 +516,4865 +517,4850 +518,4850 +519,4841 +520,4833 +521,4827 +522,4817 +523,4815 +524,4765 +525,4756 +526,4653 +527,4633 +528,4622 +529,4580 +530,4571 +531,4570 +532,4559 +533,4546 +534,4537 +535,4533 +536,4512 +537,4503 +538,4492 +539,4459 +540,4456 +541,4449 +542,4425 +543,4399 +544,4398 +545,4393 +546,4359 +547,4346 +548,4336 +549,4317 +550,4286 +551,4268 +552,4267 +553,4266 +554,4247 +555,4223 +556,4220 +557,4201 +558,4198 +559,4122 +560,4105 +561,4088 +562,4049 +563,4043 +564,3982 +565,3981 +566,3969 +567,3957 +568,3956 +569,3954 +570,3949 +571,3940 +572,3923 +573,3923 +574,3897 +575,3895 +576,3895 +577,3887 +578,3867 +579,3823 +580,3820 +581,3812 +582,3800 +583,3785 +584,3763 +585,3759 +586,3757 +587,3757 +588,3744 +589,3713 +590,3710 +591,3704 +592,3692 +593,3685 +594,3673 +595,3667 +596,3666 +597,3659 +598,3654 +599,3638 +600,3632 +601,3595 +602,3594 +603,3593 +604,3593 +605,3528 +606,3524 +607,3518 +608,3513 +609,3461 +610,3453 +611,3448 +612,3447 +613,3437 +614,3432 +615,3424 +616,3412 +617,3398 +618,3398 +619,3376 +620,3356 +621,3322 +622,3292 +623,3267 +624,3239 +625,3210 +626,3199 +627,3189 +628,3183 +629,3172 +630,3170 +631,3169 +632,3151 +633,3147 +634,3101 +635,3065 +636,3035 +637,3011 +638,3009 +639,3007 +640,2999 +641,2977 +642,2951 +643,2951 +644,2935 +645,2929 +646,2924 +647,2898 +648,2895 +649,2883 +650,2881 +651,2862 +652,2855 +653,2843 +654,2841 +655,2834 +656,2814 +657,2810 +658,2809 +659,2805 +660,2800 +661,2799 +662,2796 +663,2772 +664,2768 +665,2762 +666,2731 +667,2713 +668,2699 +669,2687 +670,2686 +671,2681 +672,2665 +673,2663 +674,2658 +675,2650 +676,2643 +677,2609 +678,2605 +679,2596 +680,2595 +681,2593 +682,2587 +683,2548 +684,2541 +685,2513 +686,2495 +687,2488 +688,2485 +689,2483 +690,2481 +691,2473 +692,2469 +693,2466 +694,2461 +695,2459 +696,2455 +697,2452 +698,2437 +699,2434 +700,2425 +701,2412 +702,2395 +703,2384 +704,2368 +705,2364 +706,2359 +707,2353 +708,2335 +709,2332 +710,2329 +711,2317 +712,2308 +713,2301 +714,2297 +715,2296 +716,2294 +717,2292 +718,2289 +719,2287 +720,2277 +721,2266 +722,2266 +723,2259 +724,2243 +725,2241 +726,2238 +727,2233 +728,2225 +729,2220 +730,2218 +731,2213 +732,2204 +733,2194 +734,2186 +735,2164 +736,2162 +737,2161 +738,2151 +739,2142 +740,2141 +741,2139 +742,2123 +743,2122 +744,2115 +745,2107 +746,2094 +747,2093 +748,2087 +749,2086 +750,2062 +751,2053 +752,2034 +753,2029 +754,2028 +755,2027 +756,2026 +757,2020 +758,2019 +759,2018 +760,2014 +761,2013 +762,2001 +763,1999 +764,1997 +765,1996 +766,1991 +767,1990 +768,1963 +769,1952 +770,1934 +771,1931 +772,1924 +773,1917 +774,1910 +775,1906 +776,1898 +777,1897 +778,1896 +779,1889 +780,1882 +781,1874 +782,1873 +783,1865 +784,1848 +785,1838 +786,1833 +787,1830 +788,1827 +789,1814 +790,1780 +791,1772 +792,1772 +793,1770 +794,1769 +795,1761 +796,1760 +797,1756 +798,1752 +799,1745 +800,1731 +801,1729 +802,1722 +803,1714 +804,1708 +805,1703 +806,1682 +807,1681 +808,1679 +809,1675 +810,1671 +811,1664 +812,1660 +813,1657 +814,1657 +815,1652 +816,1651 +817,1648 +818,1639 +819,1632 +820,1625 +821,1621 +822,1611 +823,1608 +824,1603 +825,1603 +826,1600 +827,1583 +828,1580 +829,1571 +830,1568 +831,1566 +832,1548 +833,1545 +834,1535 +835,1521 +836,1511 +837,1487 +838,1487 +839,1486 +840,1484 +841,1477 +842,1466 +843,1466 +844,1463 +845,1446 +846,1444 +847,1444 +848,1443 +849,1442 +850,1439 +851,1429 +852,1429 +853,1415 +854,1408 +855,1407 +856,1407 +857,1407 +858,1397 +859,1397 +860,1386 +861,1386 +862,1385 +863,1384 +864,1373 +865,1373 +866,1369 +867,1366 +868,1366 +869,1365 +870,1362 +871,1362 +872,1360 +873,1354 +874,1351 +875,1351 +876,1348 +877,1346 +878,1331 +879,1327 +880,1327 +881,1325 +882,1324 +883,1324 +884,1323 +885,1313 +886,1298 +887,1297 +888,1290 +889,1289 +890,1287 +891,1283 +892,1276 +893,1275 +894,1269 +895,1267 +896,1264 +897,1262 +898,1259 +899,1259 +900,1253 +901,1235 +902,1231 +903,1223 +904,1213 +905,1208 +906,1196 +907,1194 +908,1190 +909,1189 +910,1189 +911,1186 +912,1185 +913,1181 +914,1180 +915,1171 +916,1165 +917,1164 +918,1164 +919,1158 +920,1156 +921,1156 +922,1145 +923,1142 +924,1142 +925,1140 +926,1135 +927,1128 +928,1125 +929,1121 +930,1121 +931,1118 +932,1115 +933,1113 +934,1101 +935,1097 +936,1088 +937,1087 +938,1085 +939,1079 +940,1077 +941,1077 +942,1075 +943,1065 +944,1057 +945,1053 +946,1053 +947,1048 +948,1048 +949,1043 +950,1041 +951,1038 +952,1033 +953,1031 +954,1030 +955,1026 +956,1021 +957,1021 +958,1017 +959,1016 +960,1009 +961,998 +962,993 +963,990 +964,988 +965,984 +966,984 +967,983 +968,981 +969,979 +970,977 +971,969 +972,969 +973,969 +974,967 +975,966 +976,964 +977,960 +978,951 +979,948 +980,948 +981,947 +982,946 +983,931 +984,929 +985,929 +986,928 +987,928 +988,920 +989,915 +990,910 +991,909 +992,904 +993,897 +994,892 +995,887 +996,885 +997,884 +998,883 +999,882 +1000,878 +1001,874 +1002,871 +1003,870 +1004,866 +1005,865 +1006,859 +1007,853 +1008,852 +1009,851 +1010,849 +1011,848 +1012,844 +1013,840 +1014,839 +1015,839 +1016,837 +1017,837 +1018,834 +1019,827 +1020,826 +1021,824 +1022,821 +1023,819 +1024,814 +1025,814 +1026,813 +1027,802 +1028,798 +1029,794 +1030,794 +1031,793 +1032,791 +1033,791 +1034,790 +1035,785 +1036,785 +1037,785 +1038,784 +1039,784 +1040,784 +1041,782 +1042,779 +1043,779 +1044,778 +1045,777 +1046,775 +1047,772 +1048,767 +1049,764 +1050,763 +1051,761 +1052,760 +1053,756 +1054,753 +1055,749 +1056,747 +1057,746 +1058,746 +1059,742 +1060,742 +1061,739 +1062,738 +1063,737 +1064,737 +1065,728 +1066,722 +1067,719 +1068,716 +1069,716 +1070,709 +1071,706 +1072,705 +1073,704 +1074,703 +1075,698 +1076,698 +1077,694 +1078,686 +1079,686 +1080,682 +1081,681 +1082,680 +1083,679 +1084,676 +1085,671 +1086,670 +1087,669 +1088,665 +1089,665 +1090,663 +1091,661 +1092,657 +1093,656 +1094,654 +1095,654 +1096,651 +1097,647 +1098,644 +1099,641 +1100,630 +1101,623 +1102,622 +1103,621 +1104,611 +1105,609 +1106,606 +1107,594 +1108,593 +1109,593 +1110,591 +1111,590 +1112,583 +1113,582 +1114,578 +1115,575 +1116,573 +1117,572 +1118,571 +1119,568 +1120,568 +1121,566 +1122,564 +1123,563 +1124,561 +1125,560 +1126,559 +1127,558 +1128,557 +1129,554 +1130,553 +1131,553 +1132,551 +1133,549 +1134,546 +1135,546 +1136,541 +1137,536 +1138,529 +1139,529 +1140,528 +1141,526 +1142,521 +1143,520 +1144,518 +1145,517 +1146,516 +1147,515 +1148,512 +1149,511 +1150,511 +1151,510 +1152,509 +1153,507 +1154,506 +1155,506 +1156,506 +1157,505 +1158,504 +1159,503 +1160,502 +1161,501 +1162,500 +1163,495 +1164,494 +1165,488 +1166,486 +1167,484 +1168,483 +1169,482 +1170,480 +1171,477 +1172,476 +1173,475 +1174,472 +1175,472 +1176,469 +1177,466 +1178,462 +1179,456 +1180,455 +1181,453 +1182,452 +1183,447 +1184,445 +1185,444 +1186,444 +1187,443 +1188,443 +1189,442 +1190,440 +1191,436 +1192,432 +1193,432 +1194,432 +1195,430 +1196,429 +1197,428 +1198,427 +1199,426 +1200,426 +1201,424 +1202,424 +1203,423 +1204,422 +1205,420 +1206,419 +1207,419 +1208,416 +1209,414 +1210,413 +1211,410 +1212,408 +1213,408 +1214,408 +1215,404 +1216,404 +1217,401 +1218,400 +1219,400 +1220,400 +1221,399 +1222,392 +1223,391 +1224,390 +1225,390 +1226,389 +1227,388 +1228,387 +1229,382 +1230,381 +1231,380 +1232,379 +1233,377 +1234,372 +1235,372 +1236,371 +1237,371 +1238,369 +1239,367 +1240,366 +1241,364 +1242,364 +1243,362 +1244,361 +1245,359 +1246,359 +1247,357 +1248,355 +1249,354 +1250,354 +1251,349 +1252,349 +1253,347 +1254,346 +1255,346 +1256,346 +1257,346 +1258,345 +1259,342 +1260,340 +1261,340 +1262,336 +1263,335 +1264,335 +1265,333 +1266,332 +1267,332 +1268,330 +1269,330 +1270,329 +1271,325 +1272,325 +1273,324 +1274,322 +1275,322 +1276,322 +1277,321 +1278,321 +1279,320 +1280,319 +1281,319 +1282,315 +1283,315 +1284,314 +1285,313 +1286,312 +1287,311 +1288,311 +1289,311 +1290,309 +1291,309 +1292,309 +1293,309 +1294,307 +1295,307 +1296,306 +1297,302 +1298,302 +1299,302 +1300,301 +1301,301 +1302,301 +1303,301 +1304,297 +1305,297 +1306,296 +1307,296 +1308,295 +1309,294 +1310,292 +1311,292 +1312,290 +1313,289 +1314,289 +1315,288 +1316,288 +1317,287 +1318,286 +1319,285 +1320,280 +1321,279 +1322,278 +1323,278 +1324,275 +1325,274 +1326,274 +1327,273 +1328,273 +1329,272 +1330,271 +1331,270 +1332,267 +1333,266 +1334,265 +1335,265 +1336,265 +1337,264 +1338,264 +1339,263 +1340,263 +1341,263 +1342,262 +1343,261 +1344,260 +1345,258 +1346,258 +1347,256 +1348,256 +1349,255 +1350,255 +1351,253 +1352,253 +1353,252 +1354,252 +1355,251 +1356,249 +1357,247 +1358,247 +1359,247 +1360,245 +1361,244 +1362,243 +1363,242 +1364,242 +1365,241 +1366,241 +1367,240 +1368,240 +1369,239 +1370,237 +1371,237 +1372,237 +1373,236 +1374,236 +1375,234 +1376,232 +1377,229 +1378,229 +1379,229 +1380,227 +1381,227 +1382,226 +1383,225 +1384,225 +1385,224 +1386,224 +1387,224 +1388,222 +1389,221 +1390,220 +1391,220 +1392,220 +1393,219 +1394,219 +1395,218 +1396,216 +1397,215 +1398,213 +1399,212 +1400,212 +1401,211 +1402,210 +1403,208 +1404,208 +1405,208 +1406,207 +1407,207 +1408,207 +1409,207 +1410,207 +1411,206 +1412,206 +1413,205 +1414,204 +1415,203 +1416,202 +1417,202 +1418,200 +1419,200 +1420,199 +1421,199 +1422,198 +1423,196 +1424,196 +1425,195 +1426,194 +1427,194 +1428,193 +1429,193 +1430,193 +1431,192 +1432,192 +1433,192 +1434,192 +1435,191 +1436,190 +1437,190 +1438,189 +1439,189 +1440,188 +1441,187 +1442,187 +1443,186 +1444,185 +1445,185 +1446,185 +1447,184 +1448,184 +1449,184 +1450,183 +1451,183 +1452,182 +1453,181 +1454,180 +1455,179 +1456,179 +1457,178 +1458,178 +1459,178 +1460,177 +1461,176 +1462,176 +1463,175 +1464,175 +1465,174 +1466,173 +1467,171 +1468,170 +1469,170 +1470,170 +1471,168 +1472,168 +1473,168 +1474,166 +1475,164 +1476,163 +1477,163 +1478,162 +1479,162 +1480,161 +1481,161 +1482,160 +1483,160 +1484,160 +1485,159 +1486,158 +1487,157 +1488,157 +1489,157 +1490,156 +1491,155 +1492,155 +1493,154 +1494,153 +1495,153 +1496,152 +1497,152 +1498,152 +1499,151 +1500,149 +1501,148 +1502,148 +1503,147 +1504,147 +1505,147 +1506,145 +1507,145 +1508,144 +1509,143 +1510,142 +1511,141 +1512,139 +1513,139 +1514,138 +1515,138 +1516,137 +1517,137 +1518,137 +1519,136 +1520,136 +1521,135 +1522,135 +1523,135 +1524,134 +1525,134 +1526,134 +1527,134 +1528,133 +1529,130 +1530,129 +1531,128 +1532,127 +1533,126 +1534,125 +1535,125 +1536,124 +1537,124 +1538,124 +1539,124 +1540,122 +1541,122 +1542,122 +1543,122 +1544,120 +1545,119 +1546,119 +1547,118 +1548,118 +1549,118 +1550,117 +1551,117 +1552,116 +1553,116 +1554,115 +1555,113 +1556,113 +1557,113 +1558,113 +1559,113 +1560,113 +1561,113 +1562,111 +1563,111 +1564,110 +1565,110 +1566,109 +1567,108 +1568,107 +1569,107 +1570,107 +1571,107 +1572,106 +1573,106 +1574,106 +1575,106 +1576,106 +1577,104 +1578,104 +1579,102 +1580,102 +1581,101 +1582,100 +1583,100 +1584,100 +1585,99 +1586,99 +1587,98 +1588,98 +1589,97 +1590,97 +1591,97 +1592,97 +1593,97 +1594,97 +1595,96 +1596,96 +1597,96 +1598,95 +1599,94 +1600,94 +1601,94 +1602,93 +1603,93 +1604,92 +1605,92 +1606,92 +1607,92 +1608,91 +1609,91 +1610,90 +1611,90 +1612,90 +1613,90 +1614,90 +1615,88 +1616,87 +1617,87 +1618,86 +1619,85 +1620,85 +1621,85 +1622,85 +1623,84 +1624,84 +1625,84 +1626,83 +1627,83 +1628,83 +1629,83 +1630,82 +1631,81 +1632,81 +1633,81 +1634,81 +1635,80 +1636,79 +1637,78 +1638,77 +1639,77 +1640,77 +1641,77 +1642,77 +1643,77 +1644,77 +1645,77 +1646,77 +1647,76 +1648,76 +1649,75 +1650,75 +1651,75 +1652,74 +1653,74 +1654,74 +1655,73 +1656,73 +1657,73 +1658,73 +1659,72 +1660,72 +1661,72 +1662,72 +1663,72 +1664,71 +1665,71 +1666,69 +1667,69 +1668,69 +1669,69 +1670,68 +1671,68 +1672,67 +1673,67 +1674,66 +1675,66 +1676,64 +1677,64 +1678,64 +1679,64 +1680,63 +1681,63 +1682,62 +1683,62 +1684,62 +1685,62 +1686,62 +1687,62 +1688,62 +1689,62 +1690,61 +1691,61 +1692,60 +1693,59 +1694,59 +1695,59 +1696,59 +1697,58 +1698,58 +1699,57 +1700,57 +1701,57 +1702,57 +1703,56 +1704,56 +1705,55 +1706,55 +1707,55 +1708,54 +1709,54 +1710,54 +1711,54 +1712,54 +1713,54 +1714,54 +1715,53 +1716,53 +1717,53 +1718,52 +1719,52 +1720,52 +1721,52 +1722,51 +1723,50 +1724,50 +1725,50 +1726,50 +1727,49 +1728,49 +1729,49 +1730,49 +1731,49 +1732,49 +1733,48 +1734,48 +1735,48 +1736,48 +1737,48 +1738,48 +1739,47 +1740,47 +1741,47 +1742,46 +1743,46 +1744,46 +1745,46 +1746,46 +1747,45 +1748,45 +1749,45 +1750,45 +1751,45 +1752,45 +1753,44 +1754,44 +1755,44 +1756,44 +1757,44 +1758,43 +1759,43 +1760,43 +1761,43 +1762,43 +1763,43 +1764,42 +1765,42 +1766,42 +1767,42 +1768,42 +1769,41 +1770,41 +1771,41 +1772,40 +1773,40 +1774,39 +1775,39 +1776,39 +1777,39 +1778,38 +1779,38 +1780,38 +1781,38 +1782,37 +1783,37 +1784,37 +1785,37 +1786,36 +1787,36 +1788,36 +1789,35 +1790,35 +1791,35 +1792,35 +1793,35 +1794,35 +1795,34 +1796,34 +1797,34 +1798,34 +1799,33 +1800,33 +1801,33 +1802,33 +1803,32 +1804,32 +1805,32 +1806,31 +1807,31 +1808,31 +1809,31 +1810,31 +1811,31 +1812,30 +1813,30 +1814,30 +1815,30 +1816,30 +1817,30 +1818,30 +1819,30 +1820,30 +1821,29 +1822,29 +1823,29 +1824,28 +1825,28 +1826,28 +1827,28 +1828,28 +1829,28 +1830,27 +1831,27 +1832,27 +1833,26 +1834,26 +1835,26 +1836,26 +1837,26 +1838,26 +1839,26 +1840,26 +1841,26 +1842,25 +1843,25 +1844,25 +1845,25 +1846,25 +1847,25 +1848,24 +1849,24 +1850,24 +1851,24 +1852,24 +1853,24 +1854,24 +1855,23 +1856,23 +1857,23 +1858,23 +1859,23 +1860,22 +1861,22 +1862,22 +1863,22 +1864,22 +1865,22 +1866,21 +1867,21 +1868,21 +1869,21 +1870,21 +1871,21 +1872,20 +1873,20 +1874,20 +1875,20 +1876,20 +1877,19 +1878,19 +1879,19 +1880,19 +1881,19 +1882,18 +1883,18 +1884,18 +1885,18 +1886,18 +1887,18 +1888,18 +1889,18 +1890,18 +1891,18 +1892,17 +1893,17 +1894,17 +1895,17 +1896,17 +1897,17 +1898,16 +1899,16 +1900,16 +1901,16 +1902,16 +1903,15 +1904,15 +1905,15 +1906,15 +1907,15 +1908,15 +1909,15 +1910,15 +1911,15 +1912,15 +1913,15 +1914,15 +1915,15 +1916,15 +1917,14 +1918,14 +1919,14 +1920,14 +1921,14 +1922,14 +1923,14 +1924,14 +1925,14 +1926,14 +1927,14 +1928,14 +1929,13 +1930,13 +1931,13 +1932,13 +1933,13 +1934,13 +1935,13 +1936,13 +1937,13 +1938,13 +1939,12 +1940,12 +1941,12 +1942,12 +1943,12 +1944,12 +1945,12 +1946,12 +1947,12 +1948,12 +1949,12 +1950,12 +1951,12 +1952,11 +1953,11 +1954,11 +1955,11 +1956,11 +1957,11 +1958,11 +1959,11 +1960,11 +1961,11 +1962,11 +1963,11 +1964,11 +1965,11 +1966,11 +1967,11 +1968,11 +1969,11 +1970,10 +1971,10 +1972,10 +1973,10 +1974,10 +1975,10 +1976,10 +1977,10 +1978,10 +1979,9 +1980,9 +1981,9 +1982,9 +1983,9 +1984,9 +1985,9 +1986,8 +1987,8 +1988,8 +1989,8 +1990,8 +1991,8 +1992,8 +1993,8 +1994,8 +1995,8 +1996,8 +1997,8 +1998,8 +1999,7 +2000,7 +2001,7 +2002,7 +2003,7 +2004,7 +2005,7 +2006,7 +2007,6 +2008,6 +2009,6 +2010,6 +2011,6 +2012,6 +2013,6 +2014,6 +2015,6 +2016,6 +2017,6 +2018,6 +2019,6 +2020,6 +2021,6 +2022,6 +2023,6 +2024,6 +2025,6 +2026,5 +2027,5 +2028,5 +2029,5 +2030,5 +2031,5 +2032,5 +2033,5 +2034,5 +2035,5 +2036,5 +2037,5 +2038,5 +2039,5 +2040,5 +2041,5 +2042,5 +2043,5 +2044,5 +2045,5 +2046,5 +2047,5 +2048,5 +2049,4 +2050,4 +2051,4 +2052,4 +2053,4 +2054,4 +2055,4 +2056,4 +2057,4 +2058,4 +2059,4 +2060,4 +2061,4 +2062,4 +2063,4 +2064,4 +2065,4 +2066,4 +2067,4 +2068,4 +2069,4 +2070,4 +2071,4 +2072,4 +2073,4 +2074,4 +2075,4 +2076,4 +2077,4 +2078,4 +2079,4 +2080,4 +2081,4 +2082,4 +2083,3 +2084,3 +2085,3 +2086,3 +2087,3 +2088,3 +2089,3 +2090,3 +2091,3 +2092,3 +2093,3 +2094,3 +2095,3 +2096,3 +2097,3 +2098,3 +2099,3 +2100,3 +2101,3 +2102,3 +2103,3 +2104,3 +2105,3 +2106,3 +2107,2 +2108,2 +2109,2 +2110,2 +2111,2 +2112,2 +2113,2 +2114,2 +2115,2 +2116,2 +2117,2 +2118,2 +2119,2 +2120,2 +2121,2 +2122,2 +2123,2 +2124,2 +2125,2 +2126,2 +2127,2 +2128,2 +2129,2 +2130,2 +2131,2 +2132,2 +2133,2 +2134,2 +2135,2 +2136,2 +2137,2 +2138,2 +2139,2 +2140,2 +2141,2 +2142,2 +2143,2 +2144,2 +2145,2 +2146,2 +2147,2 +2148,2 +2149,2 +2150,2 +2151,2 +2152,1 +2153,1 +2154,1 +2155,1 +2156,1 +2157,1 +2158,1 +2159,1 +2160,1 +2161,1 +2162,1 +2163,1 +2164,1 +2165,1 +2166,1 +2167,1 +2168,1 +2169,1 +2170,1 +2171,1 +2172,1 +2173,1 +2174,1 +2175,1 +2176,1 +2177,1 +2178,1 +2179,1 +2180,1 +2181,1 +2182,1 +2183,1 +2184,1 +2185,1 +2186,1 +2187,1 +2188,1 +2189,1 +2190,1 +2191,1 +2192,1 +2193,1 +2194,1 +2195,1 diff --git a/db/planguard/overlay/packet_sites_walk.sql b/db/planguard/overlay/packet_sites_walk.sql new file mode 100644 index 0000000..f16d255 --- /dev/null +++ b/db/planguard/overlay/packet_sites_walk.sql @@ -0,0 +1,46 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- Overlay for diff/planted_readmodels.yaml: the packet_sites table in its merged form, built in the schema t_s2b +-- from public.packet_observations and public.packets for the 12 sites the walk requests, and nothing else. Any +-- statement whose site list is a subset of these 12 sees the rows a full packet_sites would give it. +-- +-- It reads public and writes only inside t_s2b (about 394K rows and 75 MB on a full copy). Remove it with +-- DROP SCHEMA t_s2b CASCADE; +-- Run: psql "$DSN" -v ON_ERROR_STOP=1 -f db/planguard/overlay/packet_sites_walk.sql +-- +-- UNLOGGED and without foreign keys (an overlay only; the planner treats it like a logged table). Autovacuum is off +-- so the statistics stay as built. Rows go in ingest order (first observation id); site_heard_at is the packet's +-- newest observation at that site. +SET maintenance_work_mem = '512MB'; +SET work_mem = '256MB'; +SET jit = off; +CREATE SCHEMA IF NOT EXISTS t_s2b; +DROP TABLE IF EXISTS t_s2b.packet_sites; +CREATE UNLOGGED TABLE t_s2b.packet_sites ( + packet_hash bytea NOT NULL, + iata character(3) NOT NULL, + site_heard_at timestamptz NOT NULL, + first_heard_at timestamptz NOT NULL, + payload_type smallint NOT NULL CHECK (payload_type BETWEEN 0 AND 15), + route_type smallint NOT NULL CHECK (route_type BETWEEN 0 AND 3), + scope_id integer +) WITH (autovacuum_vacuum_scale_factor = 0.05, autovacuum_vacuum_cost_limit = 1000, autovacuum_enabled = false); +ALTER TABLE t_s2b.packet_sites ALTER COLUMN iata SET STATISTICS 500; +INSERT INTO t_s2b.packet_sites +SELECT po.packet_hash, po.iata, max(po.heard_at), p.first_heard_at, p.payload_type, p.route_type, p.scope_id +FROM public.packet_observations po JOIN public.packets p ON p.packet_hash = po.packet_hash +WHERE po.iata = ANY('{AMS,EIN,ENS,GLZ,GRQ,LEY,LWR,MST,QAR,RTM,UDE,UTC}'::bpchar[]) +GROUP BY po.packet_hash, po.iata, p.first_heard_at, p.payload_type, p.route_type, p.scope_id +ORDER BY min(po.id); +ALTER TABLE t_s2b.packet_sites ADD CONSTRAINT packet_sites_pkey PRIMARY KEY (packet_hash, iata) INCLUDE (site_heard_at); +CREATE INDEX idx_packet_sites_site ON t_s2b.packet_sites (iata, site_heard_at DESC, packet_hash DESC) + INCLUDE (payload_type, route_type, first_heard_at); +CREATE INDEX idx_packet_sites_site_type ON t_s2b.packet_sites (iata, payload_type, route_type, site_heard_at DESC, packet_hash DESC); +CREATE INDEX idx_packet_sites_site_scope ON t_s2b.packet_sites (iata, scope_id, site_heard_at DESC, packet_hash DESC) + INCLUDE (payload_type, route_type, first_heard_at) + WHERE scope_id IS NOT NULL; +VACUUM ANALYZE t_s2b.packet_sites; +SELECT count(*) AS rows, count(DISTINCT packet_hash) AS packets, count(DISTINCT iata) AS sites, + pg_size_pretty(pg_total_relation_size('t_s2b.packet_sites')) AS size +FROM t_s2b.packet_sites; diff --git a/db/planguard/sql/derived/DeleteExpiredRoutesSelect.sql b/db/planguard/sql/derived/DeleteExpiredRoutesSelect.sql new file mode 100644 index 0000000..56961c6 --- /dev/null +++ b/db/planguard/sql/derived/DeleteExpiredRoutesSelect.sql @@ -0,0 +1,13 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: DeleteExpiredRoutesSelect +-- The read-only part of DeleteExpiredRoutes: its expired CTE without the lock clause. +-- Parameters as in DeleteExpiredRoutes: $1 after_ts, $2 after_id, $3 cutoff (ms ceiling), $4 cutoff, $5 limit. +SELECT r.iata, r.path_key, date_trunc('milliseconds', r.last_seen, 'UTC') AS ts, r.id + FROM known_routes r + WHERE (date_trunc('milliseconds', r.last_seen, 'UTC'), r.id) > ($1::timestamptz, $2::bigint) + AND date_trunc('milliseconds', r.last_seen, 'UTC') < $3::timestamptz + AND r.last_seen + interval '0' < $4::timestamptz + ORDER BY date_trunc('milliseconds', r.last_seen, 'UTC'), r.id + LIMIT $5 diff --git a/db/planguard/sql/derived/DeleteGraceRoutesSliceCand.sql b/db/planguard/sql/derived/DeleteGraceRoutesSliceCand.sql new file mode 100644 index 0000000..9cdfc76 --- /dev/null +++ b/db/planguard/sql/derived/DeleteGraceRoutesSliceCand.sql @@ -0,0 +1,16 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: DeleteGraceRoutesSliceCand +-- The read-only part of DeleteGraceRoutesSlice: its cand CTE (it holds no lock clause). +-- Parameters as in DeleteGraceRoutesSlice: $1 grace cutoff (ms ceiling), $2 after_ts, $3 after_id, $4 grace cutoff, +-- $5 min_observations, $6 limit. +SELECT r.iata, r.path_key, date_trunc('milliseconds', r.last_seen, 'UTC') AS ts, r.id + FROM known_routes r + WHERE (date_trunc('milliseconds', r.last_seen, 'UTC'), r.id) > ($2::timestamptz, $3::bigint) + AND date_trunc('milliseconds', r.last_seen, 'UTC') + < LEAST($1::timestamptz, $2::timestamptz + interval '6 hours') + AND r.last_seen + interval '0' < $4::timestamptz + AND r.observation_count < $5::bigint + ORDER BY date_trunc('milliseconds', r.last_seen, 'UTC'), r.id + LIMIT $6 diff --git a/db/planguard/sql/derived/DeleteOldPacketsVictims.sql b/db/planguard/sql/derived/DeleteOldPacketsVictims.sql new file mode 100644 index 0000000..7f2ccfe --- /dev/null +++ b/db/planguard/sql/derived/DeleteOldPacketsVictims.sql @@ -0,0 +1,10 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: DeleteOldPacketsVictims +-- The read-only part of DeleteOldPackets: its victims CTE without the lock clause. +-- Parameters as in DeleteOldPackets: $1 cutoff, $2 limit. +SELECT vp.packet_hash, vp.last_heard_at FROM packets vp + WHERE vp.last_heard_at < $1::timestamptz + ORDER BY vp.last_heard_at, vp.packet_hash + LIMIT $2::integer diff --git a/db/planguard/sql/oracle/OracleBySites.sql b/db/planguard/sql/oracle/OracleBySites.sql new file mode 100644 index 0000000..f556d2d --- /dev/null +++ b/db/planguard/sql/oracle/OracleBySites.sql @@ -0,0 +1,47 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: OracleBySites +-- The exact (max heard_at at the requested sites, hash) order, read from packet_observations; it +-- does not use packet_sites, so it is the truth a packet_sites page is compared with. +WITH page AS MATERIALIZED ( + SELECT po.packet_hash, max(po.heard_at) AS k + FROM packet_observations po + WHERE po.iata = ANY($1::bpchar[]) + GROUP BY po.packet_hash + HAVING (max(po.heard_at), po.packet_hash) < ($2::timestamptz, $3::bytea) + ORDER BY max(po.heard_at) DESC, po.packet_hash DESC + LIMIT $4) +SELECT p.packet_hash, p.payload_type, + COALESCE(CASE + WHEN p.payload_type = 4 AND jsonb_typeof(p.parsed_payload #> '{appData,name}') = 'string' + THEN p.parsed_payload #>> '{appData,name}' + WHEN p.payload_type = 3 AND p.parsed_payload ->> 'type' = 'ACK' + AND jsonb_typeof(p.parsed_payload -> 'checksum') = 'string' + AND p.parsed_payload ->> 'checksum' ~ '^[0-9a-fA-F]{8}$' + THEN 'ACK ' || lower(p.parsed_payload ->> 'checksum') + WHEN p.payload_type = 9 AND p.parsed_payload ->> 'type' IN ('TRACE', 'PING') + AND jsonb_typeof(p.parsed_payload -> 'traceTag') = 'string' + AND p.parsed_payload ->> 'traceTag' ~ '^[0-9a-fA-F]{8}$' + THEN (p.parsed_payload ->> 'type') || ' ' || lower(p.parsed_payload ->> 'traceTag') + END, '')::text AS summary, + p.route_type, + p.first_heard_at, p.last_heard_at, p.scope_id, ts.name AS scope_name, + pg.k AS site_heard_at, + p.observation_count AS observation_count, + po.observer_id AS latest_observer_id, o.display_name AS latest_observer_name, + COALESCE(po.iata, ''::bpchar) AS latest_observer_iata, + COALESCE(po.path_length_byte, 0::smallint) AS latest_observer_path_length_byte, + COALESCE(po.hash_size, 0::smallint) AS latest_observer_hash_size, + COALESCE(po.hop_count, 0::smallint) AS latest_observer_hop_count, + po.path_bytes AS latest_observer_path_bytes, p.raw_payload, p.origin_pubkey +FROM page pg +JOIN packets p ON p.packet_hash = pg.packet_hash +LEFT JOIN LATERAL ( + SELECT observer_id, iata, path_length_byte, hash_size, hop_count, path_bytes + FROM packet_observations WHERE packet_hash = p.packet_hash + ORDER BY heard_at DESC, id DESC + LIMIT 1) po ON true +LEFT JOIN observers o ON o.id = po.observer_id +LEFT JOIN transport_scopes ts ON ts.id = p.scope_id +ORDER BY pg.k DESC, pg.packet_hash DESC diff --git a/db/planguard/sql/pending/AddChannelMessageIATA.sql b/db/planguard/sql/pending/AddChannelMessageIATA.sql new file mode 100644 index 0000000..e30fe3b --- /dev/null +++ b/db/planguard/sql/pending/AddChannelMessageIATA.sql @@ -0,0 +1,9 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: AddChannelMessageIATA :exec +INSERT INTO channel_message_iatas (message_id, iata, channel_id, scope_id) +SELECT cm.id, $1::bpchar, cm.channel_id, p.scope_id +FROM channel_messages cm JOIN packets p ON p.packet_hash = cm.packet_hash +WHERE cm.packet_hash = $2::bytea +ON CONFLICT (message_id, iata) DO NOTHING diff --git a/db/planguard/sql/pending/FillChannelMessageIATAsByMessages.sql b/db/planguard/sql/pending/FillChannelMessageIATAsByMessages.sql new file mode 100644 index 0000000..21911ae --- /dev/null +++ b/db/planguard/sql/pending/FillChannelMessageIATAsByMessages.sql @@ -0,0 +1,11 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: FillChannelMessageIATAsByMessages :execrows +INSERT INTO channel_message_iatas (message_id, iata, channel_id, scope_id) +SELECT DISTINCT cm.id, po.iata, cm.channel_id, p.scope_id +FROM channel_messages cm +JOIN packets p USING (packet_hash) +JOIN packet_observations po USING (packet_hash) +WHERE cm.id > $1::bigint AND cm.id <= $2::bigint +ON CONFLICT DO NOTHING diff --git a/db/planguard/sql/pending/FilterKnownSites.sql b/db/planguard/sql/pending/FilterKnownSites.sql new file mode 100644 index 0000000..bf60b1d --- /dev/null +++ b/db/planguard/sql/pending/FilterKnownSites.sql @@ -0,0 +1,10 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: FilterKnownSites :one +-- Keeps only the requested IATAs that exist in packet_sites itself (never a static list). Lossless: an IATA +-- with no packet_sites row contributes no row to any packet_sites statement. Go runs it before the typed, +-- scoped and since statements and returns an empty page when nothing survives. +SELECT COALESCE(array_agg(u ORDER BY u), '{}')::bpchar[] AS iatas +FROM unnest($1::bpchar[]) u +WHERE EXISTS (SELECT 1 FROM packet_sites ps WHERE ps.iata = u) diff --git a/db/planguard/sql/pending/GetObserverTelemetrySince.sql b/db/planguard/sql/pending/GetObserverTelemetrySince.sql new file mode 100644 index 0000000..1e0b88d --- /dev/null +++ b/db/planguard/sql/pending/GetObserverTelemetrySince.sql @@ -0,0 +1,11 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: GetObserverTelemetrySince :many +SELECT id, reported_at, battery_voltage_mv, airtime_tx_secs, airtime_rx_secs, + noise_floor_db, uptime_seconds, queue_length, debug_flags, receive_errors +FROM observer_telemetry +WHERE observer_id = $1 + AND reported_at >= $2::timestamptz + AND id > $3::bigint +ORDER BY reported_at ASC diff --git a/db/planguard/sql/pending/GetPathStatsProbe.sql b/db/planguard/sql/pending/GetPathStatsProbe.sql new file mode 100644 index 0000000..dd2d1ca --- /dev/null +++ b/db/planguard/sql/pending/GetPathStatsProbe.sql @@ -0,0 +1,25 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: GetPathStatsProbe +WITH readings AS ( + SELECT iata, hour, category, hash_bytes, entries, receptions FROM analytics_hourly_paths + WHERE hour >= $1::timestamptz AND hour < $2::timestamptz + AND COALESCE(cardinality($3::bpchar[]), 0) = 0 + UNION ALL + SELECT p.iata, p.hour, p.category, p.hash_bytes, p.entries, p.receptions + FROM generate_series(GREATEST($1::timestamptz, (SELECT min(hour) FROM analytics_hourly_paths)), $2::timestamptz, interval '1 hour') g(h) + CROSS JOIN (SELECT DISTINCT unnest($3::bpchar[]) AS iata) i + CROSS JOIN LATERAL (SELECT x.iata, x.hour, x.category, x.hash_bytes, x.entries, x.receptions + FROM analytics_hourly_paths x WHERE x.hour = g.h AND x.iata = i.iata OFFSET 0) p + WHERE g.h < $2::timestamptz AND cardinality($3::bpchar[]) > 0 +) +SELECT (grouping(hour) = 0)::boolean AS is_hourly, + COALESCE(hour, 'epoch'::timestamptz)::timestamptz AS hour, + category, hash_bytes, COALESCE(entries, 0)::integer AS entries, + sum(receptions)::bigint AS receptions, + -- Non-routed categories store entries = 0, so an hour of only empty paths gives 0. + COALESCE(max(entries) FILTER (WHERE receptions > 0), 0)::integer AS max_entries +FROM readings +GROUP BY GROUPING SETS ((category, hash_bytes, entries), (hour, category, hash_bytes)) +ORDER BY is_hourly, hour, category, hash_bytes, entries diff --git a/db/planguard/sql/pending/GetStatsSeries.sql b/db/planguard/sql/pending/GetStatsSeries.sql new file mode 100644 index 0000000..d06ca9b --- /dev/null +++ b/db/planguard/sql/pending/GetStatsSeries.sql @@ -0,0 +1,94 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: GetStatsSeries :many +WITH hours AS ( + SELECT g.h::timestamptz AS hour, COALESCE(r.status, 'missing')::text AS status + FROM generate_series($1::timestamptz, $2::timestamptz, INTERVAL '1 hour') g(h) + LEFT JOIN analytics_rollup_hours r ON r.hour = g.h + WHERE g.h < $2::timestamptz +), complete AS ( + SELECT hour FROM hours WHERE status = 'complete' +), obs AS ( + SELECT o.hour, o.iata, o.observation_count FROM analytics_hourly_iata_observations o JOIN complete c ON c.hour = o.hour + WHERE COALESCE(cardinality($3::bpchar[]), 0) = 0 + UNION ALL + SELECT o.hour, o.iata, o.observation_count FROM analytics_hourly_iata_observations o JOIN complete c ON c.hour = o.hour + WHERE cardinality($3::bpchar[]) > 0 AND o.iata = ANY($3::bpchar[]) +), obs_agg AS ( + SELECT hour, sum(observation_count)::bigint AS observations, count(DISTINCT iata)::bigint AS active_iatas + FROM obs GROUP BY GROUPING SETS ((), (hour)) +), pkt_rows AS MATERIALIZED ( + SELECT s.hour, s.packets FROM analytics_hourly_packet_sets s + WHERE cardinality($3::bpchar[]) > 0 AND s.iatas && $3::bpchar[] + AND s.hour >= $1::timestamptz AND s.hour < $2::timestamptz +), pkt AS ( + SELECT s.hour, s.packets FROM analytics_hourly_packet_sets s JOIN complete c ON c.hour = s.hour + WHERE COALESCE(cardinality($3::bpchar[]), 0) = 0 + UNION ALL + SELECT s.hour, s.packets FROM pkt_rows s JOIN complete c ON c.hour = s.hour +), pkt_agg AS ( + SELECT hour, sum(packets)::bigint AS unique_packets FROM pkt GROUP BY GROUPING SETS ((), (hour)) +), obsr AS ( + SELECT i.hour, i.observer_id FROM analytics_hourly_observer_identity i JOIN complete c ON c.hour = i.hour + WHERE COALESCE(cardinality($3::bpchar[]), 0) = 0 + UNION ALL + SELECT i.hour, i.observer_id FROM analytics_hourly_observer_identity i JOIN complete c ON c.hour = i.hour + WHERE cardinality($3::bpchar[]) > 0 AND i.iata = ANY($3::bpchar[]) +), obsr_agg AS ( + SELECT hour, count(DISTINCT observer_id)::bigint AS active_observers FROM obsr GROUP BY GROUPING SETS ((), (hour)) +), scp_rows AS MATERIALIZED ( + SELECT s.hour, s.scope_id, s.packets FROM analytics_hourly_scope_sets s + WHERE cardinality($3::bpchar[]) > 0 AND s.iatas && $3::bpchar[] + AND s.hour >= $1::timestamptz AND s.hour < $2::timestamptz +), scp AS ( + SELECT s.hour, s.scope_id, s.packets FROM analytics_hourly_scope_sets s JOIN complete c ON c.hour = s.hour + WHERE COALESCE(cardinality($3::bpchar[]), 0) = 0 + UNION ALL + SELECT s.hour, s.scope_id, s.packets FROM scp_rows s JOIN complete c ON c.hour = s.hour +), scp_agg AS ( + SELECT hour, sum(packets)::bigint AS scoped_packets, count(DISTINCT scope_id)::bigint AS active_scopes + FROM scp GROUP BY GROUPING SETS ((), (hour)) +), pth AS ( + SELECT p.hour, p.entries FROM analytics_hourly_paths p JOIN complete c ON c.hour = p.hour + WHERE p.category = 0 AND COALESCE(cardinality($3::bpchar[]), 0) = 0 + UNION ALL + SELECT p.hour, p.entries FROM analytics_hourly_paths p JOIN complete c ON c.hour = p.hour + WHERE p.category = 0 AND cardinality($3::bpchar[]) > 0 AND p.iata = ANY($3::bpchar[]) +), pth_agg AS ( + SELECT hour, max(entries)::integer AS max_path_entries FROM pth GROUP BY GROUPING SETS ((), (hour)) +), sig AS ( + SELECT s.hour, s.snr_sum, s.snr_samples, s.rssi_sum, s.rssi_samples FROM analytics_hourly_signal s JOIN complete c ON c.hour = s.hour + WHERE s.kind = 3 AND COALESCE(cardinality($3::bpchar[]), 0) = 0 + UNION ALL + SELECT s.hour, s.snr_sum, s.snr_samples, s.rssi_sum, s.rssi_samples FROM analytics_hourly_signal s JOIN complete c ON c.hour = s.hour + WHERE s.kind = 3 AND cardinality($3::bpchar[]) > 0 AND s.iata = ANY($3::bpchar[]) +), sig_agg AS ( + SELECT hour, sum(snr_sum)::double precision AS snr_sum, sum(snr_samples)::bigint AS snr_samples, + sum(rssi_sum)::double precision AS rssi_sum, sum(rssi_samples)::bigint AS rssi_samples + FROM sig GROUP BY GROUPING SETS ((), (hour)) +), keys AS ( + SELECT hour, status FROM hours + UNION ALL + SELECT NULL::timestamptz, 'summary'::text +) +SELECT k.hour, k.status, + COALESCE(o.observations, 0)::bigint AS observations, + COALESCE(p.unique_packets, 0)::bigint AS unique_packets, + COALESCE(ob.active_observers, 0)::bigint AS active_observers, + COALESCE(o.active_iatas, 0)::bigint AS active_iatas, + COALESCE(s.scoped_packets, 0)::bigint AS scoped_packets, + COALESCE(s.active_scopes, 0)::bigint AS active_scopes, + COALESCE(pa.max_path_entries, 0)::integer AS max_path_entries, + COALESCE(sg.snr_sum, 0)::double precision AS snr_sum, + COALESCE(sg.snr_samples, 0)::bigint AS snr_samples, + COALESCE(sg.rssi_sum, 0)::double precision AS rssi_sum, + COALESCE(sg.rssi_samples, 0)::bigint AS rssi_samples +FROM keys k +LEFT JOIN obs_agg o ON o.hour IS NOT DISTINCT FROM k.hour +LEFT JOIN pkt_agg p ON p.hour IS NOT DISTINCT FROM k.hour +LEFT JOIN obsr_agg ob ON ob.hour IS NOT DISTINCT FROM k.hour +LEFT JOIN scp_agg s ON s.hour IS NOT DISTINCT FROM k.hour +LEFT JOIN pth_agg pa ON pa.hour IS NOT DISTINCT FROM k.hour +LEFT JOIN sig_agg sg ON sg.hour IS NOT DISTINCT FROM k.hour +ORDER BY k.hour NULLS LAST diff --git a/db/planguard/sql/pending/GetStatsTopAdvertisersProbe.sql b/db/planguard/sql/pending/GetStatsTopAdvertisersProbe.sql new file mode 100644 index 0000000..a93768d --- /dev/null +++ b/db/planguard/sql/pending/GetStatsTopAdvertisersProbe.sql @@ -0,0 +1,39 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: GetStatsTopAdvertisersProbe +WITH ranked AS ( + SELECT origin_pubkey, SUM(advert_packets)::bigint AS advert_count, + SUM(flood_packets)::bigint AS flood_advert_count, SUM(direct_packets)::bigint AS direct_advert_count + FROM analytics_hourly_advert_sets + WHERE hour >= $2::timestamptz + AND (COALESCE(cardinality($3::bpchar[]), 0) = 0 OR iatas && $3::bpchar[]) + GROUP BY origin_pubkey + ORDER BY CASE WHEN $1::text = 'flood' THEN SUM(flood_packets) ELSE SUM(direct_packets) END DESC, + advert_count DESC, origin_pubkey + LIMIT $4 +), heard_rows AS ( + SELECT h.origin_pubkey, h.last_heard, h.iata, h.name, h.node_type + FROM analytics_hourly_advert_hearings h + WHERE h.hour >= $2::timestamptz AND COALESCE(cardinality($3::bpchar[]), 0) = 0 + AND h.origin_pubkey IN (SELECT origin_pubkey FROM ranked) + UNION ALL + SELECT h.origin_pubkey, h.last_heard, h.iata, h.name, h.node_type + FROM generate_series(GREATEST($2::timestamptz, (SELECT min(hour) FROM analytics_hourly_advert_hearings)), date_trunc('hour', now(), 'UTC'), interval '1 hour') g(h) + CROSS JOIN (SELECT DISTINCT unnest($3::bpchar[]) AS iata) i + CROSS JOIN LATERAL (SELECT x.origin_pubkey, x.last_heard, x.iata, x.name, x.node_type + FROM analytics_hourly_advert_hearings x WHERE x.hour = g.h AND x.iata = i.iata OFFSET 0) h + WHERE cardinality($3::bpchar[]) > 0 AND h.origin_pubkey IN (SELECT origin_pubkey FROM ranked) +), heard AS ( + SELECT origin_pubkey, MAX(last_heard) AS last_heard, MAX(iata) AS iata, MAX(name) AS name, MAX(node_type) AS node_type + FROM heard_rows GROUP BY origin_pubkey +) +SELECT n.id AS node_id, encode(r.origin_pubkey, 'hex') AS public_key, + COALESCE(n.name, h.name, '')::text AS name, COALESCE(n.node_type, h.node_type, 0)::smallint AS node_type, + r.advert_count, r.flood_advert_count, r.direct_advert_count, + h.last_heard::timestamptz AS last_heard, COALESCE(h.iata, '')::bpchar AS iata +FROM ranked r +LEFT JOIN heard h ON h.origin_pubkey = r.origin_pubkey +LEFT JOIN nodes n ON n.public_key = r.origin_pubkey +ORDER BY CASE WHEN $1::text = 'flood' THEN r.flood_advert_count ELSE r.direct_advert_count END DESC, + r.advert_count DESC, r.origin_pubkey diff --git a/db/planguard/sql/pending/GetTopNodesProbe.sql b/db/planguard/sql/pending/GetTopNodesProbe.sql new file mode 100644 index 0000000..12c30b6 --- /dev/null +++ b/db/planguard/sql/pending/GetTopNodesProbe.sql @@ -0,0 +1,29 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: GetTopNodesProbe +WITH src AS ( + SELECT origin_pubkey, observations, last_heard, iata, name, node_type + FROM analytics_hourly_advert_hearings + WHERE hour >= $1::timestamptz AND COALESCE(cardinality($2::bpchar[]), 0) = 0 + UNION ALL + SELECT a.origin_pubkey, a.observations, a.last_heard, a.iata, a.name, a.node_type + FROM generate_series(GREATEST($1::timestamptz, (SELECT min(hour) FROM analytics_hourly_advert_hearings)), date_trunc('hour', now(), 'UTC'), interval '1 hour') g(h) + CROSS JOIN (SELECT DISTINCT unnest($2::bpchar[]) AS iata) i + CROSS JOIN LATERAL (SELECT x.origin_pubkey, x.observations, x.last_heard, x.iata, x.name, x.node_type + FROM analytics_hourly_advert_hearings x WHERE x.hour = g.h AND x.iata = i.iata OFFSET 0) a + WHERE cardinality($2::bpchar[]) > 0 +), ranked AS ( + SELECT origin_pubkey, SUM(observations)::bigint AS observation_count, MAX(last_heard) AS last_heard, + MAX(iata) AS iata, MAX(name) AS name, MAX(node_type) AS node_type + FROM src + GROUP BY origin_pubkey + ORDER BY observation_count DESC, origin_pubkey + LIMIT $3 +) +SELECT n.id AS node_id, encode(r.origin_pubkey, 'hex') AS public_key, + COALESCE(n.name, r.name, '')::text AS name, COALESCE(n.node_type, r.node_type, 0)::smallint AS node_type, + r.iata::bpchar AS iata, r.observation_count, r.last_heard::timestamptz AS last_heard +FROM ranked r +LEFT JOIN nodes n ON n.public_key = r.origin_pubkey +ORDER BY r.observation_count DESC, r.origin_pubkey diff --git a/db/planguard/sql/pending/InsertObservation.sql b/db/planguard/sql/pending/InsertObservation.sql new file mode 100644 index 0000000..97ef5c2 --- /dev/null +++ b/db/planguard/sql/pending/InsertObservation.sql @@ -0,0 +1,24 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: InsertObservation :one +WITH ins AS ( + INSERT INTO packet_observations ( + packet_hash, observer_id, iata, heard_at, path_length_byte, hash_size, hop_count, path_bytes, rssi, snr, + propagation_time_ms, radio_freq_mhz, spread_factor, bandwidth_khz, coding_rate, source_broker, payload_type, airtime_ms + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18) + ON CONFLICT (packet_hash, observer_id) DO NOTHING + RETURNING packet_hash, iata, heard_at +), bump AS ( + UPDATE packets p SET observation_count = p.observation_count + 1 FROM ins WHERE p.packet_hash = ins.packet_hash + RETURNING p.packet_hash, p.observation_count, p.payload_type, p.route_type, p.scope_id, p.first_heard_at +), site AS ( + INSERT INTO packet_sites (packet_hash, iata, site_heard_at, first_heard_at, payload_type, route_type, scope_id) + SELECT b.packet_hash, i.iata, i.heard_at, b.first_heard_at, b.payload_type, b.route_type, b.scope_id + FROM ins i JOIN bump b USING (packet_hash) + ON CONFLICT (packet_hash, iata) DO UPDATE SET site_heard_at = EXCLUDED.site_heard_at + WHERE packet_sites.site_heard_at < EXCLUDED.site_heard_at +) +SELECT EXISTS (SELECT 1 FROM ins) AS inserted, + COALESCE((SELECT b.observation_count FROM bump b), + (SELECT pk.observation_count FROM packets pk WHERE pk.packet_hash = $1), 0)::bigint AS observation_count diff --git a/db/planguard/sql/pending/ListAllChannelMessagesByIATAs.sql b/db/planguard/sql/pending/ListAllChannelMessagesByIATAs.sql new file mode 100644 index 0000000..67f195c --- /dev/null +++ b/db/planguard/sql/pending/ListAllChannelMessagesByIATAs.sql @@ -0,0 +1,24 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: ListAllChannelMessagesByIATAs :many +WITH hits AS MATERIALIZED ( + SELECT m.message_id + FROM unnest($1::bpchar[]) AS req(iata) + CROSS JOIN LATERAL ( + SELECT i.message_id FROM channel_message_iatas i + WHERE i.iata = req.iata + AND i.message_id < $2::bigint + ORDER BY i.message_id DESC + LIMIT $3) m + GROUP BY m.message_id + ORDER BY m.message_id DESC + LIMIT $3) +SELECT cm.id, cm.channel_id, cm.packet_hash, cm.sender_name, cm.sender_pubkey, cm.content, cm.sent_at, encode(cm.packet_hash, 'hex') AS packet_hash_hex, c.channel_hash, ts.name AS scope_name, + p.transport_codes_present, p.observation_count AS observation_count +FROM hits h +JOIN channel_messages cm ON cm.id = h.message_id +JOIN channels c ON c.id = cm.channel_id +JOIN packets p ON p.packet_hash = cm.packet_hash +LEFT JOIN transport_scopes ts ON ts.id = p.scope_id +ORDER BY cm.id DESC diff --git a/db/planguard/sql/pending/ListAllChannelMessagesByIATAsScoped.sql b/db/planguard/sql/pending/ListAllChannelMessagesByIATAsScoped.sql new file mode 100644 index 0000000..7d33c25 --- /dev/null +++ b/db/planguard/sql/pending/ListAllChannelMessagesByIATAsScoped.sql @@ -0,0 +1,24 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: ListAllChannelMessagesByIATAsScoped :many +WITH hits AS MATERIALIZED ( + SELECT m.message_id + FROM unnest($1::bpchar[]) AS req(iata) + CROSS JOIN LATERAL ( + SELECT i.message_id FROM channel_message_iatas i + WHERE i.iata = req.iata AND i.scope_id = $2::int + AND i.message_id < $3::bigint + ORDER BY i.message_id DESC + LIMIT $4) m + GROUP BY m.message_id + ORDER BY m.message_id DESC + LIMIT $4) +SELECT cm.id, cm.channel_id, cm.packet_hash, cm.sender_name, cm.sender_pubkey, cm.content, cm.sent_at, encode(cm.packet_hash, 'hex') AS packet_hash_hex, c.channel_hash, ts.name AS scope_name, + p.transport_codes_present, p.observation_count AS observation_count +FROM hits h +JOIN channel_messages cm ON cm.id = h.message_id +JOIN channels c ON c.id = cm.channel_id +JOIN packets p ON p.packet_hash = cm.packet_hash +LEFT JOIN transport_scopes ts ON ts.id = p.scope_id +ORDER BY cm.id DESC diff --git a/db/planguard/sql/pending/ListAllChannelMessagesByIATAsSince.sql b/db/planguard/sql/pending/ListAllChannelMessagesByIATAsSince.sql new file mode 100644 index 0000000..1828053 --- /dev/null +++ b/db/planguard/sql/pending/ListAllChannelMessagesByIATAsSince.sql @@ -0,0 +1,27 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: ListAllChannelMessagesByIATAsSince :many +WITH hits AS MATERIALIZED ( + SELECT m.message_id + FROM unnest($1::bpchar[]) AS req(iata) + CROSS JOIN LATERAL ( + SELECT i.message_id FROM channel_message_iatas i + JOIN channel_messages cm ON cm.id = i.message_id + WHERE i.iata = req.iata + AND i.message_id < $4::bigint + AND cm.sent_at >= $2::timestamptz + AND ($3::int IS NULL OR i.scope_id = $3::int) + ORDER BY i.message_id DESC + LIMIT $5) m + GROUP BY m.message_id + ORDER BY m.message_id DESC + LIMIT $5) +SELECT cm.id, cm.channel_id, cm.packet_hash, cm.sender_name, cm.sender_pubkey, cm.content, cm.sent_at, encode(cm.packet_hash, 'hex') AS packet_hash_hex, c.channel_hash, ts.name AS scope_name, + p.transport_codes_present, p.observation_count AS observation_count +FROM hits h +JOIN channel_messages cm ON cm.id = h.message_id +JOIN channels c ON c.id = cm.channel_id +JOIN packets p ON p.packet_hash = cm.packet_hash +LEFT JOIN transport_scopes ts ON ts.id = p.scope_id +ORDER BY cm.id DESC diff --git a/db/planguard/sql/pending/ListAllChannelMessagesPlain.sql b/db/planguard/sql/pending/ListAllChannelMessagesPlain.sql new file mode 100644 index 0000000..e506a24 --- /dev/null +++ b/db/planguard/sql/pending/ListAllChannelMessagesPlain.sql @@ -0,0 +1,14 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: ListAllChannelMessagesPlain :many +SELECT cm.id, cm.channel_id, cm.packet_hash, cm.sender_name, cm.sender_pubkey, cm.content, cm.sent_at, encode(cm.packet_hash, 'hex') AS packet_hash_hex, c.channel_hash, ts.name AS scope_name, + p.transport_codes_present, p.observation_count AS observation_count +FROM channel_messages cm +JOIN channels c ON c.id = cm.channel_id +JOIN packets p ON p.packet_hash = cm.packet_hash +LEFT JOIN transport_scopes ts ON ts.id = p.scope_id +WHERE cm.id < $1::bigint + AND EXISTS (SELECT 1 FROM packet_observations po WHERE po.packet_hash = cm.packet_hash) +ORDER BY cm.id DESC +LIMIT $2 diff --git a/db/planguard/sql/pending/ListAllChannelMessagesPlainScoped.sql b/db/planguard/sql/pending/ListAllChannelMessagesPlainScoped.sql new file mode 100644 index 0000000..59b31ea --- /dev/null +++ b/db/planguard/sql/pending/ListAllChannelMessagesPlainScoped.sql @@ -0,0 +1,14 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: ListAllChannelMessagesPlainScoped :many +SELECT cm.id, cm.channel_id, cm.packet_hash, cm.sender_name, cm.sender_pubkey, cm.content, cm.sent_at, encode(cm.packet_hash, 'hex') AS packet_hash_hex, c.channel_hash, ts.name AS scope_name, + p.transport_codes_present, p.observation_count AS observation_count +FROM channel_messages cm +JOIN channels c ON c.id = cm.channel_id +JOIN packets p ON p.packet_hash = cm.packet_hash +LEFT JOIN transport_scopes ts ON ts.id = p.scope_id +WHERE p.scope_id = $1::int AND cm.id < $2::bigint + AND EXISTS (SELECT 1 FROM packet_observations po WHERE po.packet_hash = cm.packet_hash) +ORDER BY cm.id DESC +LIMIT $3 diff --git a/db/planguard/sql/pending/ListAllChannelMessagesSince.sql b/db/planguard/sql/pending/ListAllChannelMessagesSince.sql new file mode 100644 index 0000000..36b18fe --- /dev/null +++ b/db/planguard/sql/pending/ListAllChannelMessagesSince.sql @@ -0,0 +1,16 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: ListAllChannelMessagesSince :many +SELECT cm.id, cm.channel_id, cm.packet_hash, cm.sender_name, cm.sender_pubkey, cm.content, cm.sent_at, encode(cm.packet_hash, 'hex') AS packet_hash_hex, c.channel_hash, ts.name AS scope_name, + p.transport_codes_present, p.observation_count AS observation_count +FROM channel_messages cm +JOIN channels c ON c.id = cm.channel_id +JOIN packets p ON p.packet_hash = cm.packet_hash +LEFT JOIN transport_scopes ts ON ts.id = p.scope_id +WHERE cm.id < $3::bigint + AND cm.sent_at >= $1::timestamptz + AND ($2::int IS NULL OR p.scope_id = $2::int) + AND EXISTS (SELECT 1 FROM packet_observations po WHERE po.packet_hash = cm.packet_hash) +ORDER BY cm.id DESC +LIMIT $4 diff --git a/db/planguard/sql/pending/ListChannelMessagesByIATAs.sql b/db/planguard/sql/pending/ListChannelMessagesByIATAs.sql new file mode 100644 index 0000000..a09437a --- /dev/null +++ b/db/planguard/sql/pending/ListChannelMessagesByIATAs.sql @@ -0,0 +1,24 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: ListChannelMessagesByIATAs :many +WITH hits AS MATERIALIZED ( + SELECT m.message_id + FROM unnest($1::bpchar[]) AS req(iata) + CROSS JOIN LATERAL ( + SELECT i.message_id FROM channel_message_iatas i + WHERE i.channel_id = $2::int AND i.iata = req.iata + AND i.message_id < $3::bigint + ORDER BY i.message_id DESC + LIMIT $4) m + GROUP BY m.message_id + ORDER BY m.message_id DESC + LIMIT $4) +SELECT cm.id, cm.channel_id, cm.packet_hash, cm.sender_name, cm.sender_pubkey, cm.content, cm.sent_at, encode(cm.packet_hash, 'hex') AS packet_hash_hex, c.channel_hash, ts.name AS scope_name, + p.transport_codes_present, p.observation_count AS observation_count +FROM hits h +JOIN channel_messages cm ON cm.id = h.message_id +JOIN channels c ON c.id = cm.channel_id +JOIN packets p ON p.packet_hash = cm.packet_hash +LEFT JOIN transport_scopes ts ON ts.id = p.scope_id +ORDER BY cm.id DESC diff --git a/db/planguard/sql/pending/ListChannelMessagesByIATAsScoped.sql b/db/planguard/sql/pending/ListChannelMessagesByIATAsScoped.sql new file mode 100644 index 0000000..6d8fb9d --- /dev/null +++ b/db/planguard/sql/pending/ListChannelMessagesByIATAsScoped.sql @@ -0,0 +1,24 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: ListChannelMessagesByIATAsScoped :many +WITH hits AS MATERIALIZED ( + SELECT m.message_id + FROM unnest($1::bpchar[]) AS req(iata) + CROSS JOIN LATERAL ( + SELECT i.message_id FROM channel_message_iatas i + WHERE i.channel_id = $2::int AND i.iata = req.iata AND i.scope_id = $3::int + AND i.message_id < $4::bigint + ORDER BY i.message_id DESC + LIMIT $5) m + GROUP BY m.message_id + ORDER BY m.message_id DESC + LIMIT $5) +SELECT cm.id, cm.channel_id, cm.packet_hash, cm.sender_name, cm.sender_pubkey, cm.content, cm.sent_at, encode(cm.packet_hash, 'hex') AS packet_hash_hex, c.channel_hash, ts.name AS scope_name, + p.transport_codes_present, p.observation_count AS observation_count +FROM hits h +JOIN channel_messages cm ON cm.id = h.message_id +JOIN channels c ON c.id = cm.channel_id +JOIN packets p ON p.packet_hash = cm.packet_hash +LEFT JOIN transport_scopes ts ON ts.id = p.scope_id +ORDER BY cm.id DESC diff --git a/db/planguard/sql/pending/ListChannelMessagesPlain.sql b/db/planguard/sql/pending/ListChannelMessagesPlain.sql new file mode 100644 index 0000000..58b0346 --- /dev/null +++ b/db/planguard/sql/pending/ListChannelMessagesPlain.sql @@ -0,0 +1,14 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: ListChannelMessagesPlain :many +SELECT cm.id, cm.channel_id, cm.packet_hash, cm.sender_name, cm.sender_pubkey, cm.content, cm.sent_at, encode(cm.packet_hash, 'hex') AS packet_hash_hex, c.channel_hash, ts.name AS scope_name, + p.transport_codes_present, p.observation_count AS observation_count +FROM channel_messages cm +JOIN channels c ON c.id = cm.channel_id +JOIN packets p ON p.packet_hash = cm.packet_hash +LEFT JOIN transport_scopes ts ON ts.id = p.scope_id +WHERE cm.channel_id = $1::int AND cm.id < $2::bigint + AND EXISTS (SELECT 1 FROM packet_observations po WHERE po.packet_hash = cm.packet_hash) +ORDER BY cm.id DESC +LIMIT $3 diff --git a/db/planguard/sql/pending/ListChannelMessagesPlainScoped.sql b/db/planguard/sql/pending/ListChannelMessagesPlainScoped.sql new file mode 100644 index 0000000..d696441 --- /dev/null +++ b/db/planguard/sql/pending/ListChannelMessagesPlainScoped.sql @@ -0,0 +1,18 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: ListChannelMessagesPlainScoped :many +WITH hits AS MATERIALIZED ( + SELECT DISTINCT i.message_id + FROM channel_message_iatas i + WHERE i.channel_id = $1::int AND i.scope_id = $2::int AND i.message_id < $3::bigint + ORDER BY i.message_id DESC + LIMIT $4) +SELECT cm.id, cm.channel_id, cm.packet_hash, cm.sender_name, cm.sender_pubkey, cm.content, cm.sent_at, encode(cm.packet_hash, 'hex') AS packet_hash_hex, c.channel_hash, ts.name AS scope_name, + p.transport_codes_present, p.observation_count AS observation_count +FROM hits h +JOIN channel_messages cm ON cm.id = h.message_id +JOIN channels c ON c.id = cm.channel_id +JOIN packets p ON p.packet_hash = cm.packet_hash +LEFT JOIN transport_scopes ts ON ts.id = p.scope_id +ORDER BY cm.id DESC diff --git a/db/planguard/sql/pending/ListChannelMessagesSinceWalk.sql b/db/planguard/sql/pending/ListChannelMessagesSinceWalk.sql new file mode 100644 index 0000000..30f02b0 --- /dev/null +++ b/db/planguard/sql/pending/ListChannelMessagesSinceWalk.sql @@ -0,0 +1,18 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: ListChannelMessagesSinceWalk :many +SELECT cm.id, cm.channel_id, cm.packet_hash, cm.sender_name, cm.sender_pubkey, cm.content, cm.sent_at, encode(cm.packet_hash, 'hex') AS packet_hash_hex, c.channel_hash, ts.name AS scope_name, + p.transport_codes_present, p.observation_count AS observation_count +FROM channel_messages cm +JOIN channels c ON c.id = cm.channel_id +JOIN packets p ON p.packet_hash = cm.packet_hash +LEFT JOIN transport_scopes ts ON ts.id = p.scope_id +WHERE cm.channel_id = $1::int AND cm.id < $5::bigint + AND cm.sent_at >= $2::timestamptz + AND EXISTS (SELECT 1 FROM packet_observations po WHERE po.packet_hash = cm.packet_hash) + AND (COALESCE(cardinality($3::bpchar[]), 0) = 0 + OR EXISTS (SELECT 1 FROM channel_message_iatas i WHERE i.message_id = cm.id AND i.iata = ANY($3::bpchar[]))) + AND ($4::int IS NULL OR p.scope_id = $4::int) +ORDER BY cm.id DESC +LIMIT $6 diff --git a/db/planguard/sql/pending/ListNodes.sql b/db/planguard/sql/pending/ListNodes.sql new file mode 100644 index 0000000..1d0e462 --- /dev/null +++ b/db/planguard/sql/pending/ListNodes.sql @@ -0,0 +1,42 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: ListNodes :many +WITH page AS ( +SELECT n.id, n.public_key, n.node_type, n.name, n.latitude, n.longitude, n.last_seen, + n.radio_freq_mhz, n.radio_sf, n.radio_bw_khz, ts.name AS default_scope_name +FROM nodes n +LEFT JOIN transport_scopes ts ON ts.id = n.default_scope_id +WHERE + ($1 = 0 OR n.node_type = $1) + AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR n.id IN (SELECT node_id FROM node_iatas WHERE iata = ANY($2::bpchar[]))) + AND ( + $3::text = 'any' + OR ($3::text = 'true' AND n.supports_multibyte_paths = TRUE) + OR ($3::text = 'false' AND n.supports_multibyte_paths = FALSE) + ) + AND ( + $4::text = 'any' + OR ($4::text = 'true' AND n.supports_multibyte_traces = TRUE) + OR ($4::text = 'false' AND n.supports_multibyte_traces = FALSE) + ) + AND ($5::bytea IS NULL OR n.public_key = $5) + AND ($6 = '' OR n.name ILIKE '%' || $6 || '%') + AND (n.last_seen, n.id) < ($7::timestamptz, $12::uuid) + AND ($9::text = '' OR ts.name = $9::text) + AND ($11::text = '' OR encode(n.public_key, 'hex') ILIKE $11 || '%') +ORDER BY n.last_seen DESC, n.id DESC +LIMIT $8 +) +SELECT n.id, n.public_key, n.node_type, n.name, n.latitude, n.longitude, n.last_seen, + n.radio_freq_mhz, n.radio_sf, n.radio_bw_khz, n.default_scope_name, + (SELECT json_agg(json_build_object('iata', ni.iata, 'lastHeard', (extract(epoch from ni.last_heard) * 1000)::bigint) ORDER BY ni.last_heard DESC) + FROM node_iatas ni WHERE ni.node_id = n.id) AS iatas, + EXISTS (SELECT 1 FROM observers o WHERE o.public_key = n.public_key) AS is_observer, + (SELECT o.id FROM observers o WHERE o.public_key = n.public_key LIMIT 1) AS observer_id, + (SELECT COUNT(DISTINCT nn.neighbor_id) FROM node_neighbors nn WHERE nn.node_id = n.id)::bigint AS known_neighbor_count, + (CASE WHEN $10::bool THEN + (SELECT COALESCE(array_agg(DISTINCT nn.neighbor_id), '{}'::uuid[]) FROM node_neighbors nn WHERE nn.node_id = n.id) + ELSE NULL END)::uuid[] AS neighbor_ids +FROM page n +ORDER BY n.last_seen DESC, n.id DESC diff --git a/db/planguard/sql/pending/ListObserverAdverts.sql b/db/planguard/sql/pending/ListObserverAdverts.sql new file mode 100644 index 0000000..791b0a9 --- /dev/null +++ b/db/planguard/sql/pending/ListObserverAdverts.sql @@ -0,0 +1,23 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: ListObserverAdverts :many +SELECT + po.id, + encode(po.packet_hash, 'hex') AS packet_hash_hex, + p.payload_type, + po.iata, + po.heard_at, + po.rssi, + po.snr, + po.hop_count, + n.name AS node_name, + COALESCE(encode(p.origin_pubkey, 'hex'), '')::text AS node_public_key +FROM packet_observations po +JOIN packets p ON p.packet_hash = po.packet_hash +LEFT JOIN nodes n ON n.public_key = p.origin_pubkey +WHERE po.observer_id = $1 + AND po.payload_type = 4 + AND po.id > $2::bigint +ORDER BY po.id ASC +LIMIT $3 diff --git a/db/planguard/sql/pending/ListObservers.sql b/db/planguard/sql/pending/ListObservers.sql new file mode 100644 index 0000000..7dc6bfe --- /dev/null +++ b/db/planguard/sql/pending/ListObservers.sql @@ -0,0 +1,41 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: ListObservers :many +SELECT + o.id, + o.display_name, + o.observer_type, + o.last_status_at, + o.radio_freq_mhz, + o.radio_sf, + o.radio_bw_khz, + array_remove(array_agg(DISTINCT ts.name ORDER BY ts.name), NULL)::text[] AS scopes, +COALESCE(CASE + WHEN GREATEST(COALESCE(o.last_status_at, o.last_seen), o.last_seen) > NOW() - INTERVAL '5 minutes' THEN 'online' + ELSE 'offline' +END, 'offline')::text AS status, +COALESCE(o.last_iata, '')::text AS iata, +o.last_seen +FROM observers o +LEFT JOIN observer_brokers ob ON ob.observer_id = o.id +LEFT JOIN observer_scopes os ON os.observer_id = o.id +LEFT JOIN transport_scopes ts ON ts.id = os.scope_id +WHERE + (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR o.last_iata = ANY($1::bpchar[])) + AND ($2 = '' OR o.observer_type = $2) + AND ($3 = '' OR ob.broker_name = $3) + AND ($4 = '' OR CASE + WHEN GREATEST(COALESCE(o.last_status_at, o.last_seen), o.last_seen) > NOW() - INTERVAL '5 minutes' THEN 'online' + ELSE 'offline' + END = $4) + AND ($5 = '' OR o.display_name ILIKE '%' || $5 || '%') + AND (o.last_seen, o.id) < ($6::timestamptz, $9::uuid) + AND ($8::text = '' OR EXISTS ( + SELECT 1 FROM observer_scopes os2 + JOIN transport_scopes ts2 ON ts2.id = os2.scope_id + WHERE os2.observer_id = o.id AND ts2.name = $8::text + )) +GROUP BY o.id +ORDER BY o.last_seen DESC, o.id DESC +LIMIT $7 diff --git a/db/planguard/sql/pending/ListPacketsByScopes.sql b/db/planguard/sql/pending/ListPacketsByScopes.sql new file mode 100644 index 0000000..85424b4 --- /dev/null +++ b/db/planguard/sql/pending/ListPacketsByScopes.sql @@ -0,0 +1,48 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: ListPacketsByScopes :many +WITH page AS MATERIALIZED ( + SELECT x.packet_hash, x.last_heard_at AS k + FROM transport_scopes s + CROSS JOIN LATERAL ( + SELECT p.packet_hash, p.last_heard_at FROM packets p + WHERE p.scope_id = s.id + AND (p.last_heard_at, p.packet_hash) < ($1::timestamptz, $2::bytea) + ORDER BY p.last_heard_at DESC, p.packet_hash DESC + LIMIT $3) x + WHERE s.name = ANY($4::text[]) + ORDER BY x.last_heard_at DESC, x.packet_hash DESC + LIMIT $3) +SELECT p.packet_hash, p.payload_type, + COALESCE(CASE + WHEN p.payload_type = 4 AND jsonb_typeof(p.parsed_payload #> '{appData,name}') = 'string' + THEN p.parsed_payload #>> '{appData,name}' + WHEN p.payload_type = 3 AND p.parsed_payload ->> 'type' = 'ACK' + AND jsonb_typeof(p.parsed_payload -> 'checksum') = 'string' + AND p.parsed_payload ->> 'checksum' ~ '^[0-9a-fA-F]{8}$' + THEN 'ACK ' || lower(p.parsed_payload ->> 'checksum') + WHEN p.payload_type = 9 AND p.parsed_payload ->> 'type' IN ('TRACE', 'PING') + AND jsonb_typeof(p.parsed_payload -> 'traceTag') = 'string' + AND p.parsed_payload ->> 'traceTag' ~ '^[0-9a-fA-F]{8}$' + THEN (p.parsed_payload ->> 'type') || ' ' || lower(p.parsed_payload ->> 'traceTag') + END, '')::text AS summary, + p.route_type, + p.first_heard_at, p.last_heard_at, p.scope_id, ts.name AS scope_name, + p.observation_count AS observation_count, + po.observer_id AS latest_observer_id, o.display_name AS latest_observer_name, + COALESCE(po.iata, ''::bpchar) AS latest_observer_iata, + COALESCE(po.path_length_byte, 0::smallint) AS latest_observer_path_length_byte, + COALESCE(po.hash_size, 0::smallint) AS latest_observer_hash_size, + COALESCE(po.hop_count, 0::smallint) AS latest_observer_hop_count, + po.path_bytes AS latest_observer_path_bytes, p.raw_payload, p.origin_pubkey +FROM page pg +JOIN packets p ON p.packet_hash = pg.packet_hash +LEFT JOIN LATERAL ( + SELECT observer_id, iata, path_length_byte, hash_size, hop_count, path_bytes + FROM packet_observations WHERE packet_hash = p.packet_hash + ORDER BY heard_at DESC, id DESC + LIMIT 1) po ON true +LEFT JOIN observers o ON o.id = po.observer_id +LEFT JOIN transport_scopes ts ON ts.id = p.scope_id +ORDER BY pg.k DESC, pg.packet_hash DESC diff --git a/db/planguard/sql/pending/ListPacketsByScopesTypes.sql b/db/planguard/sql/pending/ListPacketsByScopesTypes.sql new file mode 100644 index 0000000..a44c867 --- /dev/null +++ b/db/planguard/sql/pending/ListPacketsByScopesTypes.sql @@ -0,0 +1,49 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: ListPacketsByScopesTypes :many +WITH page AS MATERIALIZED ( + SELECT x.packet_hash, x.last_heard_at AS k + FROM transport_scopes s + CROSS JOIN LATERAL ( + SELECT p.packet_hash, p.last_heard_at FROM packets p + WHERE p.scope_id = s.id + AND (p.last_heard_at, p.packet_hash) < ($1::timestamptz, $2::bytea) + AND p.payload_type = ANY($5::smallint[]) AND p.route_type = ANY($6::smallint[]) + ORDER BY p.last_heard_at DESC, p.packet_hash DESC + LIMIT $3) x + WHERE s.name = ANY($4::text[]) + ORDER BY x.last_heard_at DESC, x.packet_hash DESC + LIMIT $3) +SELECT p.packet_hash, p.payload_type, + COALESCE(CASE + WHEN p.payload_type = 4 AND jsonb_typeof(p.parsed_payload #> '{appData,name}') = 'string' + THEN p.parsed_payload #>> '{appData,name}' + WHEN p.payload_type = 3 AND p.parsed_payload ->> 'type' = 'ACK' + AND jsonb_typeof(p.parsed_payload -> 'checksum') = 'string' + AND p.parsed_payload ->> 'checksum' ~ '^[0-9a-fA-F]{8}$' + THEN 'ACK ' || lower(p.parsed_payload ->> 'checksum') + WHEN p.payload_type = 9 AND p.parsed_payload ->> 'type' IN ('TRACE', 'PING') + AND jsonb_typeof(p.parsed_payload -> 'traceTag') = 'string' + AND p.parsed_payload ->> 'traceTag' ~ '^[0-9a-fA-F]{8}$' + THEN (p.parsed_payload ->> 'type') || ' ' || lower(p.parsed_payload ->> 'traceTag') + END, '')::text AS summary, + p.route_type, + p.first_heard_at, p.last_heard_at, p.scope_id, ts.name AS scope_name, + p.observation_count AS observation_count, + po.observer_id AS latest_observer_id, o.display_name AS latest_observer_name, + COALESCE(po.iata, ''::bpchar) AS latest_observer_iata, + COALESCE(po.path_length_byte, 0::smallint) AS latest_observer_path_length_byte, + COALESCE(po.hash_size, 0::smallint) AS latest_observer_hash_size, + COALESCE(po.hop_count, 0::smallint) AS latest_observer_hop_count, + po.path_bytes AS latest_observer_path_bytes, p.raw_payload, p.origin_pubkey +FROM page pg +JOIN packets p ON p.packet_hash = pg.packet_hash +LEFT JOIN LATERAL ( + SELECT observer_id, iata, path_length_byte, hash_size, hop_count, path_bytes + FROM packet_observations WHERE packet_hash = p.packet_hash + ORDER BY heard_at DESC, id DESC + LIMIT 1) po ON true +LEFT JOIN observers o ON o.id = po.observer_id +LEFT JOIN transport_scopes ts ON ts.id = p.scope_id +ORDER BY pg.k DESC, pg.packet_hash DESC diff --git a/db/planguard/sql/pending/ListPacketsBySite.sql b/db/planguard/sql/pending/ListPacketsBySite.sql new file mode 100644 index 0000000..b5a9bba --- /dev/null +++ b/db/planguard/sql/pending/ListPacketsBySite.sql @@ -0,0 +1,44 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: ListPacketsBySite :many +WITH page AS MATERIALIZED ( + SELECT ps.packet_hash, ps.site_heard_at AS k + FROM packet_sites ps + WHERE ps.iata = $1::bpchar + AND (ps.site_heard_at, ps.packet_hash) < ($2::timestamptz, $3::bytea) + ORDER BY ps.site_heard_at DESC, ps.packet_hash DESC + LIMIT $4) +SELECT p.packet_hash, p.payload_type, + COALESCE(CASE + WHEN p.payload_type = 4 AND jsonb_typeof(p.parsed_payload #> '{appData,name}') = 'string' + THEN p.parsed_payload #>> '{appData,name}' + WHEN p.payload_type = 3 AND p.parsed_payload ->> 'type' = 'ACK' + AND jsonb_typeof(p.parsed_payload -> 'checksum') = 'string' + AND p.parsed_payload ->> 'checksum' ~ '^[0-9a-fA-F]{8}$' + THEN 'ACK ' || lower(p.parsed_payload ->> 'checksum') + WHEN p.payload_type = 9 AND p.parsed_payload ->> 'type' IN ('TRACE', 'PING') + AND jsonb_typeof(p.parsed_payload -> 'traceTag') = 'string' + AND p.parsed_payload ->> 'traceTag' ~ '^[0-9a-fA-F]{8}$' + THEN (p.parsed_payload ->> 'type') || ' ' || lower(p.parsed_payload ->> 'traceTag') + END, '')::text AS summary, + p.route_type, + p.first_heard_at, p.last_heard_at, p.scope_id, ts.name AS scope_name, + pg.k AS site_heard_at, + p.observation_count AS observation_count, + po.observer_id AS latest_observer_id, o.display_name AS latest_observer_name, + COALESCE(po.iata, ''::bpchar) AS latest_observer_iata, + COALESCE(po.path_length_byte, 0::smallint) AS latest_observer_path_length_byte, + COALESCE(po.hash_size, 0::smallint) AS latest_observer_hash_size, + COALESCE(po.hop_count, 0::smallint) AS latest_observer_hop_count, + po.path_bytes AS latest_observer_path_bytes, p.raw_payload, p.origin_pubkey +FROM page pg +JOIN packets p ON p.packet_hash = pg.packet_hash +LEFT JOIN LATERAL ( + SELECT observer_id, iata, path_length_byte, hash_size, hop_count, path_bytes + FROM packet_observations WHERE packet_hash = p.packet_hash + ORDER BY heard_at DESC, id DESC + LIMIT 1) po ON true +LEFT JOIN observers o ON o.id = po.observer_id +LEFT JOIN transport_scopes ts ON ts.id = p.scope_id +ORDER BY pg.k DESC, pg.packet_hash DESC diff --git a/db/planguard/sql/pending/ListPacketsBySites.sql b/db/planguard/sql/pending/ListPacketsBySites.sql new file mode 100644 index 0000000..13ee1b6 --- /dev/null +++ b/db/planguard/sql/pending/ListPacketsBySites.sql @@ -0,0 +1,54 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: ListPacketsBySites :many +WITH page AS MATERIALIZED ( + SELECT s.packet_hash, max(s.site_heard_at) AS k + FROM unnest($1::bpchar[]) AS req(iata) + CROSS JOIN LATERAL ( + SELECT ps.packet_hash, ps.site_heard_at + FROM packet_sites ps + WHERE ps.iata = req.iata + AND (ps.site_heard_at, ps.packet_hash) < ($2::timestamptz, $3::bytea) + AND NOT EXISTS (SELECT 1 FROM packet_sites x + WHERE x.packet_hash = ps.packet_hash + AND x.iata = ANY($1::bpchar[]) + AND (x.site_heard_at, x.packet_hash) >= ($2::timestamptz, $3::bytea) OFFSET 0) + ORDER BY ps.site_heard_at DESC, ps.packet_hash DESC + LIMIT $4) s + GROUP BY s.packet_hash + ORDER BY max(s.site_heard_at) DESC, s.packet_hash DESC + LIMIT $4) +SELECT p.packet_hash, p.payload_type, + COALESCE(CASE + WHEN p.payload_type = 4 AND jsonb_typeof(p.parsed_payload #> '{appData,name}') = 'string' + THEN p.parsed_payload #>> '{appData,name}' + WHEN p.payload_type = 3 AND p.parsed_payload ->> 'type' = 'ACK' + AND jsonb_typeof(p.parsed_payload -> 'checksum') = 'string' + AND p.parsed_payload ->> 'checksum' ~ '^[0-9a-fA-F]{8}$' + THEN 'ACK ' || lower(p.parsed_payload ->> 'checksum') + WHEN p.payload_type = 9 AND p.parsed_payload ->> 'type' IN ('TRACE', 'PING') + AND jsonb_typeof(p.parsed_payload -> 'traceTag') = 'string' + AND p.parsed_payload ->> 'traceTag' ~ '^[0-9a-fA-F]{8}$' + THEN (p.parsed_payload ->> 'type') || ' ' || lower(p.parsed_payload ->> 'traceTag') + END, '')::text AS summary, + p.route_type, + p.first_heard_at, p.last_heard_at, p.scope_id, ts.name AS scope_name, + pg.k AS site_heard_at, + p.observation_count AS observation_count, + po.observer_id AS latest_observer_id, o.display_name AS latest_observer_name, + COALESCE(po.iata, ''::bpchar) AS latest_observer_iata, + COALESCE(po.path_length_byte, 0::smallint) AS latest_observer_path_length_byte, + COALESCE(po.hash_size, 0::smallint) AS latest_observer_hash_size, + COALESCE(po.hop_count, 0::smallint) AS latest_observer_hop_count, + po.path_bytes AS latest_observer_path_bytes, p.raw_payload, p.origin_pubkey +FROM page pg +JOIN packets p ON p.packet_hash = pg.packet_hash +LEFT JOIN LATERAL ( + SELECT observer_id, iata, path_length_byte, hash_size, hop_count, path_bytes + FROM packet_observations WHERE packet_hash = p.packet_hash + ORDER BY heard_at DESC, id DESC + LIMIT 1) po ON true +LEFT JOIN observers o ON o.id = po.observer_id +LEFT JOIN transport_scopes ts ON ts.id = p.scope_id +ORDER BY pg.k DESC, pg.packet_hash DESC diff --git a/db/planguard/sql/pending/ListPacketsBySitesFirst.sql b/db/planguard/sql/pending/ListPacketsBySitesFirst.sql new file mode 100644 index 0000000..0f5bb43 --- /dev/null +++ b/db/planguard/sql/pending/ListPacketsBySitesFirst.sql @@ -0,0 +1,50 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: ListPacketsBySitesFirst :many +WITH page AS MATERIALIZED ( + SELECT s.packet_hash, max(s.site_heard_at) AS k + FROM unnest($1::bpchar[]) AS req(iata) + CROSS JOIN LATERAL ( + SELECT ps.packet_hash, ps.site_heard_at + FROM packet_sites ps + WHERE ps.iata = req.iata + AND (ps.site_heard_at, ps.packet_hash) < ($2::timestamptz, $3::bytea) + ORDER BY ps.site_heard_at DESC, ps.packet_hash DESC + LIMIT $4) s + GROUP BY s.packet_hash + ORDER BY max(s.site_heard_at) DESC, s.packet_hash DESC + LIMIT $4) +SELECT p.packet_hash, p.payload_type, + COALESCE(CASE + WHEN p.payload_type = 4 AND jsonb_typeof(p.parsed_payload #> '{appData,name}') = 'string' + THEN p.parsed_payload #>> '{appData,name}' + WHEN p.payload_type = 3 AND p.parsed_payload ->> 'type' = 'ACK' + AND jsonb_typeof(p.parsed_payload -> 'checksum') = 'string' + AND p.parsed_payload ->> 'checksum' ~ '^[0-9a-fA-F]{8}$' + THEN 'ACK ' || lower(p.parsed_payload ->> 'checksum') + WHEN p.payload_type = 9 AND p.parsed_payload ->> 'type' IN ('TRACE', 'PING') + AND jsonb_typeof(p.parsed_payload -> 'traceTag') = 'string' + AND p.parsed_payload ->> 'traceTag' ~ '^[0-9a-fA-F]{8}$' + THEN (p.parsed_payload ->> 'type') || ' ' || lower(p.parsed_payload ->> 'traceTag') + END, '')::text AS summary, + p.route_type, + p.first_heard_at, p.last_heard_at, p.scope_id, ts.name AS scope_name, + pg.k AS site_heard_at, + p.observation_count AS observation_count, + po.observer_id AS latest_observer_id, o.display_name AS latest_observer_name, + COALESCE(po.iata, ''::bpchar) AS latest_observer_iata, + COALESCE(po.path_length_byte, 0::smallint) AS latest_observer_path_length_byte, + COALESCE(po.hash_size, 0::smallint) AS latest_observer_hash_size, + COALESCE(po.hop_count, 0::smallint) AS latest_observer_hop_count, + po.path_bytes AS latest_observer_path_bytes, p.raw_payload, p.origin_pubkey +FROM page pg +JOIN packets p ON p.packet_hash = pg.packet_hash +LEFT JOIN LATERAL ( + SELECT observer_id, iata, path_length_byte, hash_size, hop_count, path_bytes + FROM packet_observations WHERE packet_hash = p.packet_hash + ORDER BY heard_at DESC, id DESC + LIMIT 1) po ON true +LEFT JOIN observers o ON o.id = po.observer_id +LEFT JOIN transport_scopes ts ON ts.id = p.scope_id +ORDER BY pg.k DESC, pg.packet_hash DESC diff --git a/db/planguard/sql/pending/ListPacketsBySitesScoped.sql b/db/planguard/sql/pending/ListPacketsBySitesScoped.sql new file mode 100644 index 0000000..49ab109 --- /dev/null +++ b/db/planguard/sql/pending/ListPacketsBySitesScoped.sql @@ -0,0 +1,59 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: ListPacketsBySitesScoped :many +-- params: $1 iatas, $2 cursor_ts, $3 cursor_hash, $4 limit, $5 since (-infinity), $6 until (infinity), $7 payload_types int[], $8 route_types int[], $9 scope_names +WITH page AS MATERIALIZED ( + SELECT s.packet_hash, max(s.site_heard_at) AS k + FROM unnest($1::bpchar[]) AS req(iata) + CROSS JOIN (SELECT id FROM transport_scopes WHERE name = ANY($9::text[])) sc + CROSS JOIN LATERAL ( + SELECT ps.packet_hash, ps.site_heard_at + FROM packet_sites ps + WHERE ps.iata = req.iata AND ps.scope_id = sc.id + AND (ps.site_heard_at, ps.packet_hash) < ($2::timestamptz, $3::bytea) + AND ps.first_heard_at >= $5::timestamptz AND ps.first_heard_at <= $6::timestamptz + AND (ps.payload_type + 0) = ANY($7::int[]) AND (ps.route_type + 0) = ANY($8::int[]) + AND NOT EXISTS (SELECT 1 FROM packet_sites x + WHERE x.packet_hash = ps.packet_hash + AND x.iata = ANY($1::bpchar[]) + AND (x.site_heard_at, x.packet_hash) >= ($2::timestamptz, $3::bytea) + OFFSET 0) + ORDER BY ps.site_heard_at DESC, ps.packet_hash DESC + LIMIT $4) s + GROUP BY s.packet_hash + ORDER BY max(s.site_heard_at) DESC, s.packet_hash DESC + LIMIT $4) +SELECT p.packet_hash, p.payload_type, + COALESCE(CASE + WHEN p.payload_type = 4 AND jsonb_typeof(p.parsed_payload #> '{appData,name}') = 'string' + THEN p.parsed_payload #>> '{appData,name}' + WHEN p.payload_type = 3 AND p.parsed_payload ->> 'type' = 'ACK' + AND jsonb_typeof(p.parsed_payload -> 'checksum') = 'string' + AND p.parsed_payload ->> 'checksum' ~ '^[0-9a-fA-F]{8}$' + THEN 'ACK ' || lower(p.parsed_payload ->> 'checksum') + WHEN p.payload_type = 9 AND p.parsed_payload ->> 'type' IN ('TRACE', 'PING') + AND jsonb_typeof(p.parsed_payload -> 'traceTag') = 'string' + AND p.parsed_payload ->> 'traceTag' ~ '^[0-9a-fA-F]{8}$' + THEN (p.parsed_payload ->> 'type') || ' ' || lower(p.parsed_payload ->> 'traceTag') + END, '')::text AS summary, + p.route_type, + p.first_heard_at, p.last_heard_at, p.scope_id, ts.name AS scope_name, + pg.k AS site_heard_at, + p.observation_count AS observation_count, + po.observer_id AS latest_observer_id, o.display_name AS latest_observer_name, + COALESCE(po.iata, ''::bpchar) AS latest_observer_iata, + COALESCE(po.path_length_byte, 0::smallint) AS latest_observer_path_length_byte, + COALESCE(po.hash_size, 0::smallint) AS latest_observer_hash_size, + COALESCE(po.hop_count, 0::smallint) AS latest_observer_hop_count, + po.path_bytes AS latest_observer_path_bytes, p.raw_payload, p.origin_pubkey +FROM page pg +JOIN packets p ON p.packet_hash = pg.packet_hash +LEFT JOIN LATERAL ( + SELECT observer_id, iata, path_length_byte, hash_size, hop_count, path_bytes + FROM packet_observations WHERE packet_hash = p.packet_hash + ORDER BY heard_at DESC, id DESC + LIMIT 1) po ON true +LEFT JOIN observers o ON o.id = po.observer_id +LEFT JOIN transport_scopes ts ON ts.id = p.scope_id +ORDER BY pg.k DESC, pg.packet_hash DESC diff --git a/db/planguard/sql/pending/ListPacketsBySitesScopedFirst.sql b/db/planguard/sql/pending/ListPacketsBySitesScopedFirst.sql new file mode 100644 index 0000000..312aa77 --- /dev/null +++ b/db/planguard/sql/pending/ListPacketsBySitesScopedFirst.sql @@ -0,0 +1,54 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: ListPacketsBySitesScopedFirst :many +-- params: $1 iatas, $2 cursor_ts, $3 cursor_hash, $4 limit, $5 since (-infinity), $6 until (infinity), $7 payload_types int[], $8 route_types int[], $9 scope_names +WITH page AS MATERIALIZED ( + SELECT s.packet_hash, max(s.site_heard_at) AS k + FROM unnest($1::bpchar[]) AS req(iata) + CROSS JOIN (SELECT id FROM transport_scopes WHERE name = ANY($9::text[])) sc + CROSS JOIN LATERAL ( + SELECT ps.packet_hash, ps.site_heard_at + FROM packet_sites ps + WHERE ps.iata = req.iata AND ps.scope_id = sc.id + AND (ps.site_heard_at, ps.packet_hash) < ($2::timestamptz, $3::bytea) + AND ps.first_heard_at >= $5::timestamptz AND ps.first_heard_at <= $6::timestamptz + AND (ps.payload_type + 0) = ANY($7::int[]) AND (ps.route_type + 0) = ANY($8::int[]) + ORDER BY ps.site_heard_at DESC, ps.packet_hash DESC + LIMIT $4) s + GROUP BY s.packet_hash + ORDER BY max(s.site_heard_at) DESC, s.packet_hash DESC + LIMIT $4) +SELECT p.packet_hash, p.payload_type, + COALESCE(CASE + WHEN p.payload_type = 4 AND jsonb_typeof(p.parsed_payload #> '{appData,name}') = 'string' + THEN p.parsed_payload #>> '{appData,name}' + WHEN p.payload_type = 3 AND p.parsed_payload ->> 'type' = 'ACK' + AND jsonb_typeof(p.parsed_payload -> 'checksum') = 'string' + AND p.parsed_payload ->> 'checksum' ~ '^[0-9a-fA-F]{8}$' + THEN 'ACK ' || lower(p.parsed_payload ->> 'checksum') + WHEN p.payload_type = 9 AND p.parsed_payload ->> 'type' IN ('TRACE', 'PING') + AND jsonb_typeof(p.parsed_payload -> 'traceTag') = 'string' + AND p.parsed_payload ->> 'traceTag' ~ '^[0-9a-fA-F]{8}$' + THEN (p.parsed_payload ->> 'type') || ' ' || lower(p.parsed_payload ->> 'traceTag') + END, '')::text AS summary, + p.route_type, + p.first_heard_at, p.last_heard_at, p.scope_id, ts.name AS scope_name, + pg.k AS site_heard_at, + p.observation_count AS observation_count, + po.observer_id AS latest_observer_id, o.display_name AS latest_observer_name, + COALESCE(po.iata, ''::bpchar) AS latest_observer_iata, + COALESCE(po.path_length_byte, 0::smallint) AS latest_observer_path_length_byte, + COALESCE(po.hash_size, 0::smallint) AS latest_observer_hash_size, + COALESCE(po.hop_count, 0::smallint) AS latest_observer_hop_count, + po.path_bytes AS latest_observer_path_bytes, p.raw_payload, p.origin_pubkey +FROM page pg +JOIN packets p ON p.packet_hash = pg.packet_hash +LEFT JOIN LATERAL ( + SELECT observer_id, iata, path_length_byte, hash_size, hop_count, path_bytes + FROM packet_observations WHERE packet_hash = p.packet_hash + ORDER BY heard_at DESC, id DESC + LIMIT 1) po ON true +LEFT JOIN observers o ON o.id = po.observer_id +LEFT JOIN transport_scopes ts ON ts.id = p.scope_id +ORDER BY pg.k DESC, pg.packet_hash DESC diff --git a/db/planguard/sql/pending/ListPacketsBySitesSince.sql b/db/planguard/sql/pending/ListPacketsBySitesSince.sql new file mode 100644 index 0000000..4b0368e --- /dev/null +++ b/db/planguard/sql/pending/ListPacketsBySitesSince.sql @@ -0,0 +1,58 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: ListPacketsBySitesSince :many +-- params: $1 iatas, $2 cursor_ts, $3 cursor_hash, $4 limit, $5 since (-infinity), $6 until (infinity), $7 payload_types int[], $8 route_types int[] +WITH page AS MATERIALIZED ( + SELECT s.packet_hash, max(s.site_heard_at) AS k + FROM unnest($1::bpchar[]) AS req(iata) + CROSS JOIN LATERAL ( + SELECT ps.packet_hash, ps.site_heard_at + FROM packet_sites ps + WHERE ps.iata = req.iata + AND (ps.site_heard_at, ps.packet_hash) < ($2::timestamptz, $3::bytea) + AND ps.first_heard_at >= $5::timestamptz AND ps.first_heard_at <= $6::timestamptz + AND (ps.payload_type + 0) = ANY($7::int[]) AND (ps.route_type + 0) = ANY($8::int[]) + AND NOT EXISTS (SELECT 1 FROM packet_sites x + WHERE x.packet_hash = ps.packet_hash + AND x.iata = ANY($1::bpchar[]) + AND (x.site_heard_at, x.packet_hash) >= ($2::timestamptz, $3::bytea) + OFFSET 0) + ORDER BY ps.site_heard_at DESC, ps.packet_hash DESC + LIMIT $4) s + GROUP BY s.packet_hash + ORDER BY max(s.site_heard_at) DESC, s.packet_hash DESC + LIMIT $4) +SELECT p.packet_hash, p.payload_type, + COALESCE(CASE + WHEN p.payload_type = 4 AND jsonb_typeof(p.parsed_payload #> '{appData,name}') = 'string' + THEN p.parsed_payload #>> '{appData,name}' + WHEN p.payload_type = 3 AND p.parsed_payload ->> 'type' = 'ACK' + AND jsonb_typeof(p.parsed_payload -> 'checksum') = 'string' + AND p.parsed_payload ->> 'checksum' ~ '^[0-9a-fA-F]{8}$' + THEN 'ACK ' || lower(p.parsed_payload ->> 'checksum') + WHEN p.payload_type = 9 AND p.parsed_payload ->> 'type' IN ('TRACE', 'PING') + AND jsonb_typeof(p.parsed_payload -> 'traceTag') = 'string' + AND p.parsed_payload ->> 'traceTag' ~ '^[0-9a-fA-F]{8}$' + THEN (p.parsed_payload ->> 'type') || ' ' || lower(p.parsed_payload ->> 'traceTag') + END, '')::text AS summary, + p.route_type, + p.first_heard_at, p.last_heard_at, p.scope_id, ts.name AS scope_name, + pg.k AS site_heard_at, + p.observation_count AS observation_count, + po.observer_id AS latest_observer_id, o.display_name AS latest_observer_name, + COALESCE(po.iata, ''::bpchar) AS latest_observer_iata, + COALESCE(po.path_length_byte, 0::smallint) AS latest_observer_path_length_byte, + COALESCE(po.hash_size, 0::smallint) AS latest_observer_hash_size, + COALESCE(po.hop_count, 0::smallint) AS latest_observer_hop_count, + po.path_bytes AS latest_observer_path_bytes, p.raw_payload, p.origin_pubkey +FROM page pg +JOIN packets p ON p.packet_hash = pg.packet_hash +LEFT JOIN LATERAL ( + SELECT observer_id, iata, path_length_byte, hash_size, hop_count, path_bytes + FROM packet_observations WHERE packet_hash = p.packet_hash + ORDER BY heard_at DESC, id DESC + LIMIT 1) po ON true +LEFT JOIN observers o ON o.id = po.observer_id +LEFT JOIN transport_scopes ts ON ts.id = p.scope_id +ORDER BY pg.k DESC, pg.packet_hash DESC diff --git a/db/planguard/sql/pending/ListPacketsBySitesSinceFirst.sql b/db/planguard/sql/pending/ListPacketsBySitesSinceFirst.sql new file mode 100644 index 0000000..1f34d51 --- /dev/null +++ b/db/planguard/sql/pending/ListPacketsBySitesSinceFirst.sql @@ -0,0 +1,53 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: ListPacketsBySitesSinceFirst :many +-- params: $1 iatas, $2 cursor_ts, $3 cursor_hash, $4 limit, $5 since (-infinity), $6 until (infinity), $7 payload_types int[], $8 route_types int[] +WITH page AS MATERIALIZED ( + SELECT s.packet_hash, max(s.site_heard_at) AS k + FROM unnest($1::bpchar[]) AS req(iata) + CROSS JOIN LATERAL ( + SELECT ps.packet_hash, ps.site_heard_at + FROM packet_sites ps + WHERE ps.iata = req.iata + AND (ps.site_heard_at, ps.packet_hash) < ($2::timestamptz, $3::bytea) + AND ps.first_heard_at >= $5::timestamptz AND ps.first_heard_at <= $6::timestamptz + AND (ps.payload_type + 0) = ANY($7::int[]) AND (ps.route_type + 0) = ANY($8::int[]) + ORDER BY ps.site_heard_at DESC, ps.packet_hash DESC + LIMIT $4) s + GROUP BY s.packet_hash + ORDER BY max(s.site_heard_at) DESC, s.packet_hash DESC + LIMIT $4) +SELECT p.packet_hash, p.payload_type, + COALESCE(CASE + WHEN p.payload_type = 4 AND jsonb_typeof(p.parsed_payload #> '{appData,name}') = 'string' + THEN p.parsed_payload #>> '{appData,name}' + WHEN p.payload_type = 3 AND p.parsed_payload ->> 'type' = 'ACK' + AND jsonb_typeof(p.parsed_payload -> 'checksum') = 'string' + AND p.parsed_payload ->> 'checksum' ~ '^[0-9a-fA-F]{8}$' + THEN 'ACK ' || lower(p.parsed_payload ->> 'checksum') + WHEN p.payload_type = 9 AND p.parsed_payload ->> 'type' IN ('TRACE', 'PING') + AND jsonb_typeof(p.parsed_payload -> 'traceTag') = 'string' + AND p.parsed_payload ->> 'traceTag' ~ '^[0-9a-fA-F]{8}$' + THEN (p.parsed_payload ->> 'type') || ' ' || lower(p.parsed_payload ->> 'traceTag') + END, '')::text AS summary, + p.route_type, + p.first_heard_at, p.last_heard_at, p.scope_id, ts.name AS scope_name, + pg.k AS site_heard_at, + p.observation_count AS observation_count, + po.observer_id AS latest_observer_id, o.display_name AS latest_observer_name, + COALESCE(po.iata, ''::bpchar) AS latest_observer_iata, + COALESCE(po.path_length_byte, 0::smallint) AS latest_observer_path_length_byte, + COALESCE(po.hash_size, 0::smallint) AS latest_observer_hash_size, + COALESCE(po.hop_count, 0::smallint) AS latest_observer_hop_count, + po.path_bytes AS latest_observer_path_bytes, p.raw_payload, p.origin_pubkey +FROM page pg +JOIN packets p ON p.packet_hash = pg.packet_hash +LEFT JOIN LATERAL ( + SELECT observer_id, iata, path_length_byte, hash_size, hop_count, path_bytes + FROM packet_observations WHERE packet_hash = p.packet_hash + ORDER BY heard_at DESC, id DESC + LIMIT 1) po ON true +LEFT JOIN observers o ON o.id = po.observer_id +LEFT JOIN transport_scopes ts ON ts.id = p.scope_id +ORDER BY pg.k DESC, pg.packet_hash DESC diff --git a/db/planguard/sql/pending/ListPacketsBySitesTyped.sql b/db/planguard/sql/pending/ListPacketsBySitesTyped.sql new file mode 100644 index 0000000..51637c0 --- /dev/null +++ b/db/planguard/sql/pending/ListPacketsBySitesTyped.sql @@ -0,0 +1,56 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: ListPacketsBySitesTyped :many +WITH page AS MATERIALIZED ( + SELECT s.packet_hash, max(s.site_heard_at) AS k + FROM unnest($1::bpchar[]) AS req(iata) + CROSS JOIN unnest($2::smallint[]) AS pt(t) + CROSS JOIN unnest($3::smallint[]) AS rt(t) + CROSS JOIN LATERAL ( + SELECT ps.packet_hash, ps.site_heard_at + FROM packet_sites ps + WHERE ps.iata = req.iata AND ps.payload_type = pt.t AND ps.route_type = rt.t + AND (ps.site_heard_at, ps.packet_hash) < ($4::timestamptz, $5::bytea) + AND NOT EXISTS (SELECT 1 FROM packet_sites x + WHERE x.packet_hash = ps.packet_hash + AND x.iata = ANY($1::bpchar[]) + AND (x.site_heard_at, x.packet_hash) >= ($4::timestamptz, $5::bytea) OFFSET 0) + ORDER BY ps.site_heard_at DESC, ps.packet_hash DESC + LIMIT $6) s + GROUP BY s.packet_hash + ORDER BY max(s.site_heard_at) DESC, s.packet_hash DESC + LIMIT $6) +SELECT p.packet_hash, p.payload_type, + COALESCE(CASE + WHEN p.payload_type = 4 AND jsonb_typeof(p.parsed_payload #> '{appData,name}') = 'string' + THEN p.parsed_payload #>> '{appData,name}' + WHEN p.payload_type = 3 AND p.parsed_payload ->> 'type' = 'ACK' + AND jsonb_typeof(p.parsed_payload -> 'checksum') = 'string' + AND p.parsed_payload ->> 'checksum' ~ '^[0-9a-fA-F]{8}$' + THEN 'ACK ' || lower(p.parsed_payload ->> 'checksum') + WHEN p.payload_type = 9 AND p.parsed_payload ->> 'type' IN ('TRACE', 'PING') + AND jsonb_typeof(p.parsed_payload -> 'traceTag') = 'string' + AND p.parsed_payload ->> 'traceTag' ~ '^[0-9a-fA-F]{8}$' + THEN (p.parsed_payload ->> 'type') || ' ' || lower(p.parsed_payload ->> 'traceTag') + END, '')::text AS summary, + p.route_type, + p.first_heard_at, p.last_heard_at, p.scope_id, ts.name AS scope_name, + pg.k AS site_heard_at, + p.observation_count AS observation_count, + po.observer_id AS latest_observer_id, o.display_name AS latest_observer_name, + COALESCE(po.iata, ''::bpchar) AS latest_observer_iata, + COALESCE(po.path_length_byte, 0::smallint) AS latest_observer_path_length_byte, + COALESCE(po.hash_size, 0::smallint) AS latest_observer_hash_size, + COALESCE(po.hop_count, 0::smallint) AS latest_observer_hop_count, + po.path_bytes AS latest_observer_path_bytes, p.raw_payload, p.origin_pubkey +FROM page pg +JOIN packets p ON p.packet_hash = pg.packet_hash +LEFT JOIN LATERAL ( + SELECT observer_id, iata, path_length_byte, hash_size, hop_count, path_bytes + FROM packet_observations WHERE packet_hash = p.packet_hash + ORDER BY heard_at DESC, id DESC + LIMIT 1) po ON true +LEFT JOIN observers o ON o.id = po.observer_id +LEFT JOIN transport_scopes ts ON ts.id = p.scope_id +ORDER BY pg.k DESC, pg.packet_hash DESC diff --git a/db/planguard/sql/pending/ListPacketsBySitesTypedFirst.sql b/db/planguard/sql/pending/ListPacketsBySitesTypedFirst.sql new file mode 100644 index 0000000..7833e9c --- /dev/null +++ b/db/planguard/sql/pending/ListPacketsBySitesTypedFirst.sql @@ -0,0 +1,52 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: ListPacketsBySitesTypedFirst :many +WITH page AS MATERIALIZED ( + SELECT s.packet_hash, max(s.site_heard_at) AS k + FROM unnest($1::bpchar[]) AS req(iata) + CROSS JOIN unnest($2::smallint[]) AS pt(t) + CROSS JOIN unnest($3::smallint[]) AS rt(t) + CROSS JOIN LATERAL ( + SELECT ps.packet_hash, ps.site_heard_at + FROM packet_sites ps + WHERE ps.iata = req.iata AND ps.payload_type = pt.t AND ps.route_type = rt.t + AND (ps.site_heard_at, ps.packet_hash) < ($4::timestamptz, $5::bytea) + ORDER BY ps.site_heard_at DESC, ps.packet_hash DESC + LIMIT $6) s + GROUP BY s.packet_hash + ORDER BY max(s.site_heard_at) DESC, s.packet_hash DESC + LIMIT $6) +SELECT p.packet_hash, p.payload_type, + COALESCE(CASE + WHEN p.payload_type = 4 AND jsonb_typeof(p.parsed_payload #> '{appData,name}') = 'string' + THEN p.parsed_payload #>> '{appData,name}' + WHEN p.payload_type = 3 AND p.parsed_payload ->> 'type' = 'ACK' + AND jsonb_typeof(p.parsed_payload -> 'checksum') = 'string' + AND p.parsed_payload ->> 'checksum' ~ '^[0-9a-fA-F]{8}$' + THEN 'ACK ' || lower(p.parsed_payload ->> 'checksum') + WHEN p.payload_type = 9 AND p.parsed_payload ->> 'type' IN ('TRACE', 'PING') + AND jsonb_typeof(p.parsed_payload -> 'traceTag') = 'string' + AND p.parsed_payload ->> 'traceTag' ~ '^[0-9a-fA-F]{8}$' + THEN (p.parsed_payload ->> 'type') || ' ' || lower(p.parsed_payload ->> 'traceTag') + END, '')::text AS summary, + p.route_type, + p.first_heard_at, p.last_heard_at, p.scope_id, ts.name AS scope_name, + pg.k AS site_heard_at, + p.observation_count AS observation_count, + po.observer_id AS latest_observer_id, o.display_name AS latest_observer_name, + COALESCE(po.iata, ''::bpchar) AS latest_observer_iata, + COALESCE(po.path_length_byte, 0::smallint) AS latest_observer_path_length_byte, + COALESCE(po.hash_size, 0::smallint) AS latest_observer_hash_size, + COALESCE(po.hop_count, 0::smallint) AS latest_observer_hop_count, + po.path_bytes AS latest_observer_path_bytes, p.raw_payload, p.origin_pubkey +FROM page pg +JOIN packets p ON p.packet_hash = pg.packet_hash +LEFT JOIN LATERAL ( + SELECT observer_id, iata, path_length_byte, hash_size, hop_count, path_bytes + FROM packet_observations WHERE packet_hash = p.packet_hash + ORDER BY heard_at DESC, id DESC + LIMIT 1) po ON true +LEFT JOIN observers o ON o.id = po.observer_id +LEFT JOIN transport_scopes ts ON ts.id = p.scope_id +ORDER BY pg.k DESC, pg.packet_hash DESC diff --git a/db/planguard/sql/pending/ListPacketsByTypes.sql b/db/planguard/sql/pending/ListPacketsByTypes.sql new file mode 100644 index 0000000..cf64167 --- /dev/null +++ b/db/planguard/sql/pending/ListPacketsByTypes.sql @@ -0,0 +1,48 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: ListPacketsByTypes :many +WITH page AS MATERIALIZED ( + SELECT x.packet_hash, x.last_heard_at AS k + FROM (SELECT DISTINCT unnest($4::smallint[])) AS pt(t) + CROSS JOIN (SELECT DISTINCT unnest($5::smallint[])) AS rt(t) + CROSS JOIN LATERAL ( + SELECT p.packet_hash, p.last_heard_at FROM packets p + WHERE p.payload_type = pt.t AND p.route_type = rt.t + AND (p.last_heard_at, p.packet_hash) < ($1::timestamptz, $2::bytea) + ORDER BY p.last_heard_at DESC, p.packet_hash DESC + LIMIT $3) x + ORDER BY x.last_heard_at DESC, x.packet_hash DESC + LIMIT $3) +SELECT p.packet_hash, p.payload_type, + COALESCE(CASE + WHEN p.payload_type = 4 AND jsonb_typeof(p.parsed_payload #> '{appData,name}') = 'string' + THEN p.parsed_payload #>> '{appData,name}' + WHEN p.payload_type = 3 AND p.parsed_payload ->> 'type' = 'ACK' + AND jsonb_typeof(p.parsed_payload -> 'checksum') = 'string' + AND p.parsed_payload ->> 'checksum' ~ '^[0-9a-fA-F]{8}$' + THEN 'ACK ' || lower(p.parsed_payload ->> 'checksum') + WHEN p.payload_type = 9 AND p.parsed_payload ->> 'type' IN ('TRACE', 'PING') + AND jsonb_typeof(p.parsed_payload -> 'traceTag') = 'string' + AND p.parsed_payload ->> 'traceTag' ~ '^[0-9a-fA-F]{8}$' + THEN (p.parsed_payload ->> 'type') || ' ' || lower(p.parsed_payload ->> 'traceTag') + END, '')::text AS summary, + p.route_type, + p.first_heard_at, p.last_heard_at, p.scope_id, ts.name AS scope_name, + p.observation_count AS observation_count, + po.observer_id AS latest_observer_id, o.display_name AS latest_observer_name, + COALESCE(po.iata, ''::bpchar) AS latest_observer_iata, + COALESCE(po.path_length_byte, 0::smallint) AS latest_observer_path_length_byte, + COALESCE(po.hash_size, 0::smallint) AS latest_observer_hash_size, + COALESCE(po.hop_count, 0::smallint) AS latest_observer_hop_count, + po.path_bytes AS latest_observer_path_bytes, p.raw_payload, p.origin_pubkey +FROM page pg +JOIN packets p ON p.packet_hash = pg.packet_hash +LEFT JOIN LATERAL ( + SELECT observer_id, iata, path_length_byte, hash_size, hop_count, path_bytes + FROM packet_observations WHERE packet_hash = p.packet_hash + ORDER BY heard_at DESC, id DESC + LIMIT 1) po ON true +LEFT JOIN observers o ON o.id = po.observer_id +LEFT JOIN transport_scopes ts ON ts.id = p.scope_id +ORDER BY pg.k DESC, pg.packet_hash DESC diff --git a/db/planguard/sql/pending/ListPacketsPage.sql b/db/planguard/sql/pending/ListPacketsPage.sql new file mode 100644 index 0000000..1921717 --- /dev/null +++ b/db/planguard/sql/pending/ListPacketsPage.sql @@ -0,0 +1,41 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: ListPacketsPage :many +WITH page AS MATERIALIZED ( + SELECT p.packet_hash, p.last_heard_at AS k FROM packets p + WHERE (p.last_heard_at, p.packet_hash) < ($1::timestamptz, $2::bytea) + ORDER BY p.last_heard_at DESC, p.packet_hash DESC + LIMIT $3) +SELECT p.packet_hash, p.payload_type, + COALESCE(CASE + WHEN p.payload_type = 4 AND jsonb_typeof(p.parsed_payload #> '{appData,name}') = 'string' + THEN p.parsed_payload #>> '{appData,name}' + WHEN p.payload_type = 3 AND p.parsed_payload ->> 'type' = 'ACK' + AND jsonb_typeof(p.parsed_payload -> 'checksum') = 'string' + AND p.parsed_payload ->> 'checksum' ~ '^[0-9a-fA-F]{8}$' + THEN 'ACK ' || lower(p.parsed_payload ->> 'checksum') + WHEN p.payload_type = 9 AND p.parsed_payload ->> 'type' IN ('TRACE', 'PING') + AND jsonb_typeof(p.parsed_payload -> 'traceTag') = 'string' + AND p.parsed_payload ->> 'traceTag' ~ '^[0-9a-fA-F]{8}$' + THEN (p.parsed_payload ->> 'type') || ' ' || lower(p.parsed_payload ->> 'traceTag') + END, '')::text AS summary, + p.route_type, + p.first_heard_at, p.last_heard_at, p.scope_id, ts.name AS scope_name, + p.observation_count AS observation_count, + po.observer_id AS latest_observer_id, o.display_name AS latest_observer_name, + COALESCE(po.iata, ''::bpchar) AS latest_observer_iata, + COALESCE(po.path_length_byte, 0::smallint) AS latest_observer_path_length_byte, + COALESCE(po.hash_size, 0::smallint) AS latest_observer_hash_size, + COALESCE(po.hop_count, 0::smallint) AS latest_observer_hop_count, + po.path_bytes AS latest_observer_path_bytes, p.raw_payload, p.origin_pubkey +FROM page pg +JOIN packets p ON p.packet_hash = pg.packet_hash +LEFT JOIN LATERAL ( + SELECT observer_id, iata, path_length_byte, hash_size, hop_count, path_bytes + FROM packet_observations WHERE packet_hash = p.packet_hash + ORDER BY heard_at DESC, id DESC + LIMIT 1) po ON true +LEFT JOIN observers o ON o.id = po.observer_id +LEFT JOIN transport_scopes ts ON ts.id = p.scope_id +ORDER BY pg.k DESC, pg.packet_hash DESC diff --git a/db/planguard/sql/pending/ListPacketsWindow.sql b/db/planguard/sql/pending/ListPacketsWindow.sql new file mode 100644 index 0000000..e1d42d8 --- /dev/null +++ b/db/planguard/sql/pending/ListPacketsWindow.sql @@ -0,0 +1,55 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: ListPacketsWindow :many +WITH page AS MATERIALIZED ( + SELECT u.packet_hash, u.k FROM ( + (SELECT p.packet_hash, p.last_heard_at AS k FROM packets p + WHERE p.last_heard_at >= $6::timestamptz + AND (p.last_heard_at, p.packet_hash) < ($1::timestamptz, $2::bytea) + AND (p.first_heard_at >= $6::timestamptz AND p.first_heard_at <= $7::timestamptz) IS TRUE + AND p.payload_type = ANY($4::smallint[]) AND p.route_type = ANY($5::smallint[]) + ORDER BY p.last_heard_at DESC, p.packet_hash DESC + LIMIT $3) + UNION ALL + (SELECT p.packet_hash, p.last_heard_at AS k FROM packets p + WHERE (p.first_heard_at > p.last_heard_at) IS TRUE + AND (p.first_heard_at >= $6::timestamptz AND p.first_heard_at <= $7::timestamptz + AND p.last_heard_at < $6::timestamptz + AND (p.last_heard_at, p.packet_hash) < ($1::timestamptz, $2::bytea) + AND p.payload_type = ANY($4::smallint[]) AND p.route_type = ANY($5::smallint[])) IS TRUE) + ) u + ORDER BY u.k DESC, u.packet_hash DESC + LIMIT $3) +SELECT p.packet_hash, p.payload_type, + COALESCE(CASE + WHEN p.payload_type = 4 AND jsonb_typeof(p.parsed_payload #> '{appData,name}') = 'string' + THEN p.parsed_payload #>> '{appData,name}' + WHEN p.payload_type = 3 AND p.parsed_payload ->> 'type' = 'ACK' + AND jsonb_typeof(p.parsed_payload -> 'checksum') = 'string' + AND p.parsed_payload ->> 'checksum' ~ '^[0-9a-fA-F]{8}$' + THEN 'ACK ' || lower(p.parsed_payload ->> 'checksum') + WHEN p.payload_type = 9 AND p.parsed_payload ->> 'type' IN ('TRACE', 'PING') + AND jsonb_typeof(p.parsed_payload -> 'traceTag') = 'string' + AND p.parsed_payload ->> 'traceTag' ~ '^[0-9a-fA-F]{8}$' + THEN (p.parsed_payload ->> 'type') || ' ' || lower(p.parsed_payload ->> 'traceTag') + END, '')::text AS summary, + p.route_type, + p.first_heard_at, p.last_heard_at, p.scope_id, ts.name AS scope_name, + p.observation_count AS observation_count, + po.observer_id AS latest_observer_id, o.display_name AS latest_observer_name, + COALESCE(po.iata, ''::bpchar) AS latest_observer_iata, + COALESCE(po.path_length_byte, 0::smallint) AS latest_observer_path_length_byte, + COALESCE(po.hash_size, 0::smallint) AS latest_observer_hash_size, + COALESCE(po.hop_count, 0::smallint) AS latest_observer_hop_count, + po.path_bytes AS latest_observer_path_bytes, p.raw_payload, p.origin_pubkey +FROM page pg +JOIN packets p ON p.packet_hash = pg.packet_hash +LEFT JOIN LATERAL ( + SELECT observer_id, iata, path_length_byte, hash_size, hop_count, path_bytes + FROM packet_observations WHERE packet_hash = p.packet_hash + ORDER BY heard_at DESC, id DESC + LIMIT 1) po ON true +LEFT JOIN observers o ON o.id = po.observer_id +LEFT JOIN transport_scopes ts ON ts.id = p.scope_id +ORDER BY pg.k DESC, pg.packet_hash DESC diff --git a/db/planguard/sql/pending/ListPacketsWindowScoped.sql b/db/planguard/sql/pending/ListPacketsWindowScoped.sql new file mode 100644 index 0000000..e0463cb --- /dev/null +++ b/db/planguard/sql/pending/ListPacketsWindowScoped.sql @@ -0,0 +1,61 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: ListPacketsWindowScoped :many +WITH page AS MATERIALIZED ( + SELECT u.packet_hash, u.k FROM ( + (SELECT x.packet_hash, x.last_heard_at AS k + FROM transport_scopes s + CROSS JOIN LATERAL ( + SELECT p.packet_hash, p.last_heard_at FROM packets p + WHERE p.scope_id = s.id + AND p.last_heard_at >= $7::timestamptz + AND (p.last_heard_at, p.packet_hash) < ($1::timestamptz, $2::bytea) + AND (p.first_heard_at >= $7::timestamptz AND p.first_heard_at <= $8::timestamptz) IS TRUE + AND p.payload_type = ANY($4::smallint[]) AND p.route_type = ANY($5::smallint[]) + ORDER BY p.last_heard_at DESC, p.packet_hash DESC + LIMIT $3) x + WHERE s.name = ANY($6::text[])) + UNION ALL + (SELECT p.packet_hash, p.last_heard_at AS k FROM packets p + WHERE (p.first_heard_at > p.last_heard_at) IS TRUE + AND (p.first_heard_at >= $7::timestamptz AND p.first_heard_at <= $8::timestamptz + AND p.last_heard_at < $7::timestamptz + AND (p.last_heard_at, p.packet_hash) < ($1::timestamptz, $2::bytea) + AND p.payload_type = ANY($4::smallint[]) AND p.route_type = ANY($5::smallint[]) + AND p.scope_id = ANY(ARRAY(SELECT id FROM transport_scopes WHERE name = ANY($6::text[])))) IS TRUE) + ) u + ORDER BY u.k DESC, u.packet_hash DESC + LIMIT $3) +SELECT p.packet_hash, p.payload_type, + COALESCE(CASE + WHEN p.payload_type = 4 AND jsonb_typeof(p.parsed_payload #> '{appData,name}') = 'string' + THEN p.parsed_payload #>> '{appData,name}' + WHEN p.payload_type = 3 AND p.parsed_payload ->> 'type' = 'ACK' + AND jsonb_typeof(p.parsed_payload -> 'checksum') = 'string' + AND p.parsed_payload ->> 'checksum' ~ '^[0-9a-fA-F]{8}$' + THEN 'ACK ' || lower(p.parsed_payload ->> 'checksum') + WHEN p.payload_type = 9 AND p.parsed_payload ->> 'type' IN ('TRACE', 'PING') + AND jsonb_typeof(p.parsed_payload -> 'traceTag') = 'string' + AND p.parsed_payload ->> 'traceTag' ~ '^[0-9a-fA-F]{8}$' + THEN (p.parsed_payload ->> 'type') || ' ' || lower(p.parsed_payload ->> 'traceTag') + END, '')::text AS summary, + p.route_type, + p.first_heard_at, p.last_heard_at, p.scope_id, ts.name AS scope_name, + p.observation_count AS observation_count, + po.observer_id AS latest_observer_id, o.display_name AS latest_observer_name, + COALESCE(po.iata, ''::bpchar) AS latest_observer_iata, + COALESCE(po.path_length_byte, 0::smallint) AS latest_observer_path_length_byte, + COALESCE(po.hash_size, 0::smallint) AS latest_observer_hash_size, + COALESCE(po.hop_count, 0::smallint) AS latest_observer_hop_count, + po.path_bytes AS latest_observer_path_bytes, p.raw_payload, p.origin_pubkey +FROM page pg +JOIN packets p ON p.packet_hash = pg.packet_hash +LEFT JOIN LATERAL ( + SELECT observer_id, iata, path_length_byte, hash_size, hop_count, path_bytes + FROM packet_observations WHERE packet_hash = p.packet_hash + ORDER BY heard_at DESC, id DESC + LIMIT 1) po ON true +LEFT JOIN observers o ON o.id = po.observer_id +LEFT JOIN transport_scopes ts ON ts.id = p.scope_id +ORDER BY pg.k DESC, pg.packet_hash DESC diff --git a/db/planguard/sql/pending/ListTraceTags.sql b/db/planguard/sql/pending/ListTraceTags.sql new file mode 100644 index 0000000..e773179 --- /dev/null +++ b/db/planguard/sql/pending/ListTraceTags.sql @@ -0,0 +1,28 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: ListTraceTags :many + +SELECT + encode(t.trace_tag, 'hex') AS trace_tag, + t.first_heard_at, + t.last_heard_at, + t.packet_count, + (SELECT COUNT(*) + FROM trace_iatas ti + WHERE ti.trace_tag = t.trace_tag + AND (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR ti.iata = ANY($1::bpchar[]))) AS iata_count, + COALESCE(t.trace_type, '')::text AS trace_type, + t.best_payload +FROM trace_tags t +WHERE (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR EXISTS ( + SELECT 1 FROM trace_iatas ti WHERE ti.trace_tag = t.trace_tag AND ti.iata = ANY($1::bpchar[]))) + AND ($2::text = '' OR t.scope_id = (SELECT id FROM transport_scopes WHERE name = $2)) + AND ($3::timestamptz IS NULL OR t.first_heard_at >= $3) + AND ($4::timestamptz IS NULL OR t.first_heard_at <= $4) + -- Keyset on the millisecond clients see, tie-broken by tag; matches idx_trace_tags_keyset. + AND (date_trunc('milliseconds', t.last_heard_at, 'UTC'), t.trace_tag) + < (COALESCE($5::timestamptz, 'infinity'::timestamptz), COALESCE($8::bytea, ''::bytea)) + AND ($7::text = '' OR t.trace_type = $7) +ORDER BY date_trunc('milliseconds', t.last_heard_at, 'UTC') DESC, t.trace_tag DESC +LIMIT $6 diff --git a/db/planguard/sql/pending/SweepPacketSitesFronts.sql b/db/planguard/sql/pending/SweepPacketSitesFronts.sql new file mode 100644 index 0000000..ddd6ea4 --- /dev/null +++ b/db/planguard/sql/pending/SweepPacketSitesFronts.sql @@ -0,0 +1,37 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: SweepPacketSitesFronts :one +-- maint, every 30 s; run with SET LOCAL enable_bitmapscan = off, enable_seqscan = off. +-- Read-only. Visits, through each time-ordered packet_sites index, every entry whose key was written in the last +-- @lookback (2 minutes). An index scan that reaches an entry whose heap tuple is dead to every snapshot sets the +-- entry's LP_DEAD hint; when that leaf later fills, the inserting backend deletes hinted entries instead of splitting +-- (nbtree simple deletion). Keeps the three indexes at ~2x a fresh build instead of 4-9x. Changes no row. +-- The ORDER BY ... LIMIT keeps every lateral a per-front index range scan (never flattened into a join). +-- The cap is far above any front's 2-minute volume (SEA: ~150 entries) and only bounds the gauge. +WITH site AS MATERIALIZED ( + SELECT ps.iata, ps.payload_type, ps.route_type, ps.scope_id + FROM iata_codes c + CROSS JOIN LATERAL ( + SELECT p.iata, p.payload_type, p.route_type, p.scope_id + FROM packet_sites p + WHERE p.iata = c.iata AND p.site_heard_at > now() - $1::interval + ORDER BY p.site_heard_at DESC LIMIT 100000) ps -- idx_packet_sites_site, Index Scan +), typ AS ( + SELECT count(*) AS n + FROM (SELECT DISTINCT iata, payload_type, route_type FROM site) f + CROSS JOIN LATERAL ( + SELECT 1 FROM packet_sites p + WHERE p.iata = f.iata AND p.payload_type = f.payload_type AND p.route_type = f.route_type + AND p.site_heard_at > now() - $1::interval + ORDER BY p.site_heard_at DESC LIMIT 100000) x -- idx_packet_sites_site_type, Index Only Scan +), scp AS ( + SELECT count(*) AS n + FROM (SELECT DISTINCT iata, scope_id FROM site WHERE scope_id IS NOT NULL) f + CROSS JOIN LATERAL ( + SELECT 1 FROM packet_sites p + WHERE p.iata = f.iata AND p.scope_id = f.scope_id AND p.scope_id IS NOT NULL + AND p.site_heard_at > now() - $1::interval + ORDER BY p.site_heard_at DESC LIMIT 100000) x -- idx_packet_sites_site_scope, Index Only Scan +) +SELECT (SELECT count(*) FROM site) AS site_live, (SELECT n FROM typ) AS type_live, (SELECT n FROM scp) AS scope_live; diff --git a/db/planguard/sql/planted/DeleteOldRoutesSelect.sql b/db/planguard/sql/planted/DeleteOldRoutesSelect.sql new file mode 100644 index 0000000..599e63a --- /dev/null +++ b/db/planguard/sql/planted/DeleteOldRoutesSelect.sql @@ -0,0 +1,12 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: DeleteOldRoutesSelect +-- The read-only part of DeleteOldRoutes as of 202acc6, with its row lock (plain EXPLAIN only). + SELECT r.iata, r.path_key + FROM known_routes r + WHERE r.last_seen < GREATEST($1::timestamptz, $2::timestamptz) + AND (r.last_seen < $1 OR + (r.observation_count < $3 AND r.last_seen < $2)) + LIMIT $4 + FOR UPDATE OF r SKIP LOCKED diff --git a/db/planguard/sql/planted/DeleteOldRoutesSelectNoLock.sql b/db/planguard/sql/planted/DeleteOldRoutesSelectNoLock.sql new file mode 100644 index 0000000..d8f2998 --- /dev/null +++ b/db/planguard/sql/planted/DeleteOldRoutesSelectNoLock.sql @@ -0,0 +1,11 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: DeleteOldRoutesSelectNoLock +-- The read-only part of DeleteOldRoutes as of 202acc6, without its row lock (for EXPLAIN ANALYZE). + SELECT r.iata, r.path_key + FROM known_routes r + WHERE r.last_seen < GREATEST($1::timestamptz, $2::timestamptz) + AND (r.last_seen < $1 OR + (r.observation_count < $3 AND r.last_seen < $2)) + LIMIT $4 diff --git a/db/planguard/sql/planted/DeleteOldRoutes_202acc6.sql b/db/planguard/sql/planted/DeleteOldRoutes_202acc6.sql new file mode 100644 index 0000000..806163e --- /dev/null +++ b/db/planguard/sql/planted/DeleteOldRoutes_202acc6.sql @@ -0,0 +1,15 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: DeleteOldRoutes :execrows +WITH expired AS ( + SELECT r.iata, r.path_key + FROM known_routes r + WHERE r.last_seen < GREATEST($1::timestamptz, $2::timestamptz) + AND (r.last_seen < $1 OR + (r.observation_count < $3 AND r.last_seen < $2)) + LIMIT $4 + FOR UPDATE OF r SKIP LOCKED +) +DELETE FROM known_routes kr USING expired e +WHERE kr.iata = e.iata AND kr.path_key = e.path_key diff --git a/db/planguard/sql/planted/ListKnownRoutes_202acc6.sql b/db/planguard/sql/planted/ListKnownRoutes_202acc6.sql new file mode 100644 index 0000000..01f7622 --- /dev/null +++ b/db/planguard/sql/planted/ListKnownRoutes_202acc6.sql @@ -0,0 +1,31 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: ListKnownRoutes :many +SELECT r.id, r.node_ids, r.hash_prefix, r.iata, r.hop_count, r.first_seen, r.last_seen, r.observation_count +FROM (( +SELECT id, node_ids, hash_prefix, iata, hop_count, first_seen, last_seen, observation_count +FROM known_routes +WHERE $1 = '' + AND ($2 = 0 OR hop_count = $2) + AND ($3::timestamptz IS NULL + OR date_trunc('milliseconds', last_seen, 'UTC') < $3 + OR ($5::bigint > 0 AND date_trunc('milliseconds', last_seen, 'UTC') = $3 AND id < $5)) +ORDER BY date_trunc('milliseconds', last_seen, 'UTC') DESC, id DESC +LIMIT $4 +) +UNION ALL +( +SELECT id, node_ids, hash_prefix, iata, hop_count, first_seen, last_seen, observation_count +FROM known_routes +WHERE $1 <> '' AND iata::text = $1 + AND iata >= $1::bpchar AND iata <= $1::bpchar + AND ($2 = 0 OR hop_count = $2) + AND ($3::timestamptz IS NULL + OR date_trunc('milliseconds', last_seen, 'UTC') < $3 + OR ($5::bigint > 0 AND date_trunc('milliseconds', last_seen, 'UTC') = $3 AND id < $5)) +ORDER BY iata, date_trunc('milliseconds', last_seen, 'UTC') DESC, id DESC +LIMIT $4 +)) r +ORDER BY date_trunc('milliseconds', r.last_seen, 'UTC') DESC, r.id DESC +LIMIT $4 diff --git a/db/planguard/sql/planted/ListPacketsBySitesReadmodels.sql b/db/planguard/sql/planted/ListPacketsBySitesReadmodels.sql new file mode 100644 index 0000000..3ccc53a --- /dev/null +++ b/db/planguard/sql/planted/ListPacketsBySitesReadmodels.sql @@ -0,0 +1,57 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: ListPacketsBySitesReadmodels :many +-- ListPacketsBySites as first proposed: a per-site LIMIT before the cross-site re-check (loses rows). +WITH page AS MATERIALIZED ( + SELECT s.packet_hash, max(s.site_heard_at) AS k + FROM unnest($1::bpchar[]) AS req(iata) + CROSS JOIN LATERAL ( + SELECT ps.packet_hash, ps.site_heard_at + FROM packet_sites ps + WHERE ps.iata = req.iata + AND (ps.site_heard_at, ps.packet_hash) < ($2::timestamptz, $3::bytea) + ORDER BY ps.site_heard_at DESC, ps.packet_hash DESC + LIMIT $4 + ) s + GROUP BY s.packet_hash + HAVING NOT EXISTS ( + SELECT 1 FROM packet_sites x + WHERE x.packet_hash = s.packet_hash AND x.iata = ANY($1::bpchar[]) + AND (x.site_heard_at, x.packet_hash) >= ($2::timestamptz, $3::bytea)) + ORDER BY max(s.site_heard_at) DESC, s.packet_hash DESC + LIMIT $4 +) +SELECT p.packet_hash, p.payload_type, + COALESCE(CASE + WHEN p.payload_type = 4 AND jsonb_typeof(p.parsed_payload #> '{appData,name}') = 'string' + THEN p.parsed_payload #>> '{appData,name}' + WHEN p.payload_type = 3 AND p.parsed_payload ->> 'type' = 'ACK' + AND jsonb_typeof(p.parsed_payload -> 'checksum') = 'string' + AND p.parsed_payload ->> 'checksum' ~ '^[0-9a-fA-F]{8}$' + THEN 'ACK ' || lower(p.parsed_payload ->> 'checksum') + WHEN p.payload_type = 9 AND p.parsed_payload ->> 'type' IN ('TRACE', 'PING') + AND jsonb_typeof(p.parsed_payload -> 'traceTag') = 'string' + AND p.parsed_payload ->> 'traceTag' ~ '^[0-9a-fA-F]{8}$' + THEN (p.parsed_payload ->> 'type') || ' ' || lower(p.parsed_payload ->> 'traceTag') + END, '')::text AS summary, + p.route_type, + p.first_heard_at, p.last_heard_at, p.scope_id, ts.name AS scope_name, + pg.k AS site_heard_at, + p.observation_count AS observation_count, + po.observer_id AS latest_observer_id, o.display_name AS latest_observer_name, + COALESCE(po.iata, ''::bpchar) AS latest_observer_iata, + COALESCE(po.path_length_byte, 0::smallint) AS latest_observer_path_length_byte, + COALESCE(po.hash_size, 0::smallint) AS latest_observer_hash_size, + COALESCE(po.hop_count, 0::smallint) AS latest_observer_hop_count, + po.path_bytes AS latest_observer_path_bytes, p.raw_payload, p.origin_pubkey +FROM page pg +JOIN packets p ON p.packet_hash = pg.packet_hash +LEFT JOIN LATERAL ( + SELECT observer_id, iata, path_length_byte, hash_size, hop_count, path_bytes + FROM packet_observations WHERE packet_hash = p.packet_hash + ORDER BY heard_at DESC, id DESC + LIMIT 1) po ON true +LEFT JOIN observers o ON o.id = po.observer_id +LEFT JOIN transport_scopes ts ON ts.id = p.scope_id +ORDER BY pg.k DESC, pg.packet_hash DESC diff --git a/db/planguard/storediff/smoke.yaml b/db/planguard/storediff/smoke.yaml new file mode 100644 index 0000000..18e52e9 --- /dev/null +++ b/db/planguard/storediff/smoke.yaml @@ -0,0 +1,45 @@ +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +# Smoke cases: two read-only servers on one database, walked page by page and compared as whole JSON. +# +# planguard storediff --old http://127.0.0.1:8481 --new http://127.0.0.1:8482 \ +# --cases db/planguard/storediff/smoke.yaml --walk 1,2,3,web +# +# /routes answers a bare array: it has more while a page is full, and the next page starts at the last route's +# lastSeen and id. +cases: + - id: routes_sea + path: /api/v1/routes + query: {iata: SEA} + follow: + - {param: cursor, from: last.lastSeen} + - {param: cursorId, from: last.id} + - id: routes_global + path: /api/v1/routes + follow: + - {param: cursor, from: last.lastSeen} + - {param: cursorId, from: last.id} + - id: nodes + path: /api/v1/nodes + follow: + - {param: cursor, from: nextCursor} + - id: observers + path: /api/v1/observers + follow: + - {param: cursor, from: nextCursor} + - id: packets + path: /api/v1/packets + follow: + - {param: cursor, from: nextCursor} + - id: channel_messages + # The channel heard most recently: /channels is ordered by last seen. + path: /api/v1/channels/{channel}/messages + vars: + channel: {path: /api/v1/channels, query: {limit: "1"}, from: first.id} + follow: + - {param: cursor, from: nextCursor} + - id: messages + path: /api/v1/messages + follow: + - {param: cursor, from: nextCursor} diff --git a/db/planpolicy/cached_gen.go b/db/planpolicy/cached_gen.go index 32f2070..b97291d 100644 --- a/db/planpolicy/cached_gen.go +++ b/db/planpolicy/cached_gen.go @@ -5,6 +5,9 @@ package planpolicy -// Cached lists class G statements: those admitted to the generic-plan class. It is empty, so -// split has no effect yet. -var Cached = map[string]bool{} +// Cached lists class G statements: those admitted to the generic-plan class. +var Cached = map[string]bool{ + "GetNodeByID": true, + "ListKnownRoutesByIATA": true, + "ListKnownRoutesGlobal": true, +} diff --git a/db/planpolicy/planpolicy_test.go b/db/planpolicy/planpolicy_test.go index e30d386..333bdd4 100644 --- a/db/planpolicy/planpolicy_test.go +++ b/db/planpolicy/planpolicy_test.go @@ -25,7 +25,8 @@ func TestPlanPolicyNamesExist(t *testing.T) { } } if len(planpolicy.Legacy) != 10 || planpolicy.ClassOf("ListPackets") != planpolicy.ClassLegacy || - planpolicy.ClassOf("GetNodeByID") != planpolicy.ClassC || planpolicy.ClassOf("") != planpolicy.ClassC { + planpolicy.ClassOf("ListKnownRoutesGlobalHop") != planpolicy.ClassC || planpolicy.ClassOf("GetNodeByID") != planpolicy.ClassG || + planpolicy.ClassOf("") != planpolicy.ClassC { t.Fatal("ClassOf") } } diff --git a/db/planrouter_test.go b/db/planrouter_test.go index a15df14..f520e19 100644 --- a/db/planrouter_test.go +++ b/db/planrouter_test.go @@ -323,8 +323,8 @@ func TestPlanRouterRoutesByClass(t *testing.T) { // A class C statement never reaches api_cached, and nothing does outside split. for _, layout := range []planpolicy.Layout{planpolicy.CustomAll, planpolicy.Split, planpolicy.Single} { - f := newRouterFixture(layout, nil) // the real policy: Cached is empty, so every non-Legacy name is C - for _, stmt := range []string{"ListPackets", "GetNodeByID", "ListKnownRoutesByIATAHop", "ListKnownRoutesGlobalHop", ""} { + f := newRouterFixture(layout, nil) // the real policy + for _, stmt := range []string{"ListPackets", "ListChannels", "ListKnownRoutesByIATAHop", "ListKnownRoutesGlobalHop", ""} { sql := "SELECT 1" if stmt != "" { sql = "-- name: " + stmt + " :one\nSELECT 1" @@ -359,6 +359,29 @@ func TestPlanRouterRoutesByClass(t *testing.T) { } } + // The real policy admits GetNodeByID (class G): under split it runs named on api_cached, with no mode argument. + f := newRouterFixture(planpolicy.Split, nil) + sql := "-- name: GetNodeByID :one\nSELECT $1" + for _, method := range routerMethods { + if err := runVia(context.Background(), f.r, method, sql, 7); err != nil { + t.Fatal(err) + } + } + calls, _, _ := f.cached.snapshot() + if len(calls) != len(routerMethods) { + t.Fatalf("split: api_cached ran %d GetNodeByID statements, want %d", len(calls), len(routerMethods)) + } + for _, c := range calls { + if c.sql != sql || !reflect.DeepEqual(c.args, []any{7}) { + t.Errorf("split: api_cached call %q %v, want %q [7]", c.sql, c.args, sql) + } + } + for _, p := range []*fakePool{f.custom, f.legacy} { + if calls, _, _ := p.snapshot(); len(calls) != 0 { + t.Errorf("split: %s ran %d statements, want 0", p.n, len(calls)) + } + } + // Through NewPlanRouter with real, unconnected pools: no layout dereferences a pool it did not open. cfg, err := pgxpool.ParseConfig("postgres://nobody@127.0.0.1:1/none?sslmode=disable&connect_timeout=1") if err != nil { diff --git a/db/pools_integration_test.go b/db/pools_integration_test.go index 2cf8fd6..4db40f4 100644 --- a/db/pools_integration_test.go +++ b/db/pools_integration_test.go @@ -5,6 +5,8 @@ package db import ( "context" + "crypto/rand" + "errors" "io" "net/http/httptest" "os" @@ -15,6 +17,8 @@ import ( "github.com/MeshCore-Beacon/beacon-server/db/planpolicy" "github.com/MeshCore-Beacon/beacon-server/internal/metrics" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" "github.com/jackc/pgx/v5/pgxpool" ) @@ -143,3 +147,72 @@ func scrapeMetrics(t *testing.T) string { } return string(b) } + +// A tune that sets default_transaction_read_only on every pool makes each pool refuse a write +// with SQLSTATE 25006 while reads still work. +func TestReadOnlyPoolsRejectWritesPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + // Own throwaway database, so the table exists and nothing else sees the attempt. + admin, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer admin.Close(context.Background()) + name := "beacon_read_only_pools_" + strings.ToLower(rand.Text()) + ident := pgx.Identifier{name}.Sanitize() + if _, err := admin.Exec(ctx, "CREATE DATABASE "+ident); err != nil { + t.Fatal(err) + } + defer admin.Exec(context.Background(), "DROP DATABASE "+ident+" WITH (FORCE)") + cfg, err := pgxpool.ParseConfig(dsn) + if err != nil { + t.Fatal(err) + } + cfg.ConnConfig.Database = name + migrated, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + t.Fatal(err) + } + err = RunMigrations(ctx, migrated) + migrated.Close() + if err != nil { + t.Fatal(err) + } + + tuned := map[string]bool{} + tune := func(pool string, c *pgxpool.Config) { + tuned[pool] = true + c.ConnConfig.Database = name + c.ConnConfig.RuntimeParams["default_transaction_read_only"] = "on" + } + p, err := OpenPools(ctx, dsn, planpolicy.CustomAll, tune) + if err != nil { + t.Fatal(err) + } + defer p.Close() + pools := map[string]*pgxpool.Pool{PoolAPILegacy: p.APILegacy, PoolAPICustom: p.APICustom, PoolIngest: p.Ingest, PoolMaint: p.Maint} + for name, pool := range pools { + if !tuned[name] { + t.Errorf("%s: tune not called", name) + } + _, err := pool.Exec(ctx, "INSERT INTO background_state(key, value) VALUES ($1, '{}')", "read-only-"+name) + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) || pgErr.Code != "25006" { + t.Errorf("%s: INSERT error = %v, want SQLSTATE 25006", name, err) + } + var n int + if err := pool.QueryRow(ctx, "SELECT count(*) FROM background_state").Scan(&n); err != nil { + t.Errorf("%s: SELECT: %v", name, err) + } else if n != 0 { + t.Errorf("%s: background_state has %d rows, want 0", name, n) + } + } + if len(tuned) != len(pools) { + t.Errorf("tune called for %v, want the %d custom_all pools", tuned, len(pools)) + } +} diff --git a/db/sqltext.go b/db/sqltext.go new file mode 100644 index 0000000..0cdf4d7 --- /dev/null +++ b/db/sqltext.go @@ -0,0 +1,11 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import "embed" + +// SQLCSources holds the generated sqlc files, so a binary can read every statement's text without a checkout. +// +//go:embed sqlc/*.sql.go +var SQLCSources embed.FS diff --git a/docs/docs.go b/docs/docs.go index 7df0c72..52a6287 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -557,6 +557,75 @@ const docTemplate = `{ } } }, + "/admin/db/planguard": { + "get": { + "security": [ + { + "AdminKey": [] + } + ], + "description": "Plans every corpus statement in custom and generic mode against this database, read-only and without executing it (plain EXPLAIN, 5 s per plan, 1 s lock timeout), and checks each plan against the corpus. Off unless planguard.daily_check is set (404 \"planguard daily check is disabled\"); when on, the check runs about an hour after boot and then daily, and until its first run this returns 404. fresh=true runs it now, shared with any run in progress and cancelled at server shutdown; within 5 minutes of the last run's end it starts nothing and returns the last successful report (started_at gives its age), or 500 when no run has succeeded yet. A read-only server cannot enable the check (configuration refuses planguard.daily_check with server.read_only), so there this returns 404 \"planguard daily check is disabled\".", + "produces": [ + "application/json" + ], + "tags": [ + "Admin" + ], + "summary": "Last daily plan regression check", + "parameters": [ + { + "type": "boolean", + "description": "Run the check now instead of returning the last report", + "name": "fresh", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PlanguardReport" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "404": { + "description": "Not Found", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + } + }, "/admin/db/statements": { "get": { "security": [ @@ -5566,6 +5635,81 @@ const docTemplate = `{ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.PlanguardReport": { + "type": "object", + "properties": { + "clean": { + "description": "no FALSE_POSITIVE, MISSED or ERROR", + "type": "boolean" + }, + "counts": { + "description": "verdict → runs", + "type": "object", + "additionalProperties": { + "type": "integer" + } + }, + "duration_ms": { + "type": "number" + }, + "failures": { + "description": "never null", + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PlanguardRun" + } + }, + "runs": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PlanguardRun" + } + }, + "started_at": { + "type": "string" + }, + "tier": { + "type": "string" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.PlanguardRun": { + "type": "object", + "properties": { + "drivers": { + "description": "relation:node:index of each driving scan", + "type": "array", + "items": { + "type": "string" + } + }, + "entry": { + "type": "string" + }, + "failures": { + "description": "the plan rules the run broke", + "type": "array", + "items": { + "type": "string" + } + }, + "mode": { + "description": "custom or generic", + "type": "string" + }, + "skip": { + "description": "why the run was not made", + "type": "string" + }, + "stmt": { + "type": "string" + }, + "verdict": { + "description": "ok, DETECTED, MISSED, FALSE_POSITIVE, RETIRED_INDEX, KNOWN_ISSUE, SKIP or ERROR", + "type": "string" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.RadioPreset": { "type": "object", "properties": { diff --git a/docs/swagger.json b/docs/swagger.json index ad46f5a..8bfac5c 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -555,6 +555,75 @@ } } }, + "/admin/db/planguard": { + "get": { + "security": [ + { + "AdminKey": [] + } + ], + "description": "Plans every corpus statement in custom and generic mode against this database, read-only and without executing it (plain EXPLAIN, 5 s per plan, 1 s lock timeout), and checks each plan against the corpus. Off unless planguard.daily_check is set (404 \"planguard daily check is disabled\"); when on, the check runs about an hour after boot and then daily, and until its first run this returns 404. fresh=true runs it now, shared with any run in progress and cancelled at server shutdown; within 5 minutes of the last run's end it starts nothing and returns the last successful report (started_at gives its age), or 500 when no run has succeeded yet. A read-only server cannot enable the check (configuration refuses planguard.daily_check with server.read_only), so there this returns 404 \"planguard daily check is disabled\".", + "produces": [ + "application/json" + ], + "tags": [ + "Admin" + ], + "summary": "Last daily plan regression check", + "parameters": [ + { + "type": "boolean", + "description": "Run the check now instead of returning the last report", + "name": "fresh", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PlanguardReport" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "404": { + "description": "Not Found", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + } + }, "/admin/db/statements": { "get": { "security": [ @@ -5564,6 +5633,81 @@ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.PlanguardReport": { + "type": "object", + "properties": { + "clean": { + "description": "no FALSE_POSITIVE, MISSED or ERROR", + "type": "boolean" + }, + "counts": { + "description": "verdict → runs", + "type": "object", + "additionalProperties": { + "type": "integer" + } + }, + "duration_ms": { + "type": "number" + }, + "failures": { + "description": "never null", + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PlanguardRun" + } + }, + "runs": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PlanguardRun" + } + }, + "started_at": { + "type": "string" + }, + "tier": { + "type": "string" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.PlanguardRun": { + "type": "object", + "properties": { + "drivers": { + "description": "relation:node:index of each driving scan", + "type": "array", + "items": { + "type": "string" + } + }, + "entry": { + "type": "string" + }, + "failures": { + "description": "the plan rules the run broke", + "type": "array", + "items": { + "type": "string" + } + }, + "mode": { + "description": "custom or generic", + "type": "string" + }, + "skip": { + "description": "why the run was not made", + "type": "string" + }, + "stmt": { + "type": "string" + }, + "verdict": { + "description": "ok, DETECTED, MISSED, FALSE_POSITIVE, RETIRED_INDEX, KNOWN_ISSUE, SKIP or ERROR", + "type": "string" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.RadioPreset": { "type": "object", "properties": { diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 831a527..312edce 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -1331,6 +1331,59 @@ definitions: payloadTypeName: type: string type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.PlanguardReport: + properties: + clean: + description: no FALSE_POSITIVE, MISSED or ERROR + type: boolean + counts: + additionalProperties: + type: integer + description: verdict → runs + type: object + duration_ms: + type: number + failures: + description: never null + items: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PlanguardRun' + type: array + runs: + items: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PlanguardRun' + type: array + started_at: + type: string + tier: + type: string + type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.PlanguardRun: + properties: + drivers: + description: relation:node:index of each driving scan + items: + type: string + type: array + entry: + type: string + failures: + description: the plan rules the run broke + items: + type: string + type: array + mode: + description: custom or generic + type: string + skip: + description: why the run was not made + type: string + stmt: + type: string + verdict: + description: ok, DETECTED, MISSED, FALSE_POSITIVE, RETIRED_INDEX, KNOWN_ISSUE, + SKIP or ERROR + type: string + type: object github_com_MeshCore-Beacon_beacon-server_internal_api.RadioPreset: properties: count: @@ -2251,6 +2304,60 @@ paths: summary: Update runtime CORS origins and enforced guard families tags: - Admin + /admin/db/planguard: + get: + description: Plans every corpus statement in custom and generic mode against + this database, read-only and without executing it (plain EXPLAIN, 5 s per + plan, 1 s lock timeout), and checks each plan against the corpus. Off unless + planguard.daily_check is set (404 "planguard daily check is disabled"); when + on, the check runs about an hour after boot and then daily, and until its + first run this returns 404. fresh=true runs it now, shared with any run in + progress and cancelled at server shutdown; within 5 minutes of the last run's + end it starts nothing and returns the last successful report (started_at gives + its age), or 500 when no run has succeeded yet. A read-only server cannot + enable the check (configuration refuses planguard.daily_check with server.read_only), + so there this returns 404 "planguard daily check is disabled". + parameters: + - description: Run the check now instead of returning the last report + in: query + name: fresh + type: boolean + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PlanguardReport' + "400": + description: Bad Request + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "401": + description: Unauthorized + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "404": + description: Not Found + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "500": + description: Internal Server Error + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + security: + - AdminKey: [] + summary: Last daily plan regression check + tags: + - Admin /admin/db/statements: get: description: Sums pg_stat_statements for this database by sqlc statement name, diff --git a/internal/api/admin.go b/internal/api/admin.go index aedc084..fb62d63 100644 --- a/internal/api/admin.go +++ b/internal/api/admin.go @@ -3,7 +3,10 @@ package api -import "errors" +import ( + "errors" + "time" +) // AdminConfig is a whitelist of selected nonsecret running settings. // It is not a serialization of the full file or environment configuration. @@ -92,3 +95,31 @@ type DBStatement struct { type DBStatementsResponse struct { Items []DBStatement `json:"items"` } + +// ErrPlanguardNotRun means the daily plan check has not finished a run since the server started. +var ErrPlanguardNotRun = errors.New("planguard has not run yet") + +// ErrPlanguardDisabled means planguard.daily_check is off. +var ErrPlanguardDisabled = errors.New("planguard daily check is disabled") + +// PlanguardRun is one corpus entry planned in one plan mode. +type PlanguardRun struct { + Entry string `json:"entry"` + Stmt string `json:"stmt"` + Mode string `json:"mode"` // custom or generic + Verdict string `json:"verdict"` // ok, DETECTED, MISSED, FALSE_POSITIVE, RETIRED_INDEX, KNOWN_ISSUE, SKIP or ERROR + Failures []string `json:"failures,omitempty"` // the plan rules the run broke + Drivers []string `json:"drivers,omitempty"` // relation:node:index of each driving scan + Skip string `json:"skip,omitempty"` // why the run was not made +} + +// PlanguardReport is one read-only plan regression check of the corpus against this database. +type PlanguardReport struct { + Tier string `json:"tier"` + StartedAt time.Time `json:"started_at"` + DurationMs float64 `json:"duration_ms"` + Clean bool `json:"clean"` // no FALSE_POSITIVE, MISSED or ERROR + Counts map[string]int `json:"counts"` // verdict → runs + Failures []PlanguardRun `json:"failures"` // never null + Runs []PlanguardRun `json:"runs"` +} diff --git a/internal/api/handlers/db_admin.go b/internal/api/handlers/db_admin.go index eb254bd..0066f95 100644 --- a/internal/api/handlers/db_admin.go +++ b/internal/api/handlers/db_admin.go @@ -8,6 +8,7 @@ import ( "errors" "log/slog" "net/http" + "strconv" "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/go-chi/chi/v5" @@ -18,13 +19,72 @@ type DBStatementsReader interface { DBStatements(context.Context) ([]api.DBStatement, error) } -// DBAdminRouter serves database diagnostics under the admin router. -func DBAdminRouter(reader DBStatementsReader) http.Handler { +// PlanguardSource holds the daily plan regression check. +type PlanguardSource interface { + Last() (api.PlanguardReport, error) // api.ErrPlanguardNotRun before the first run; api.ErrPlanguardDisabled when off + RunNow(ctx context.Context) (api.PlanguardReport, error) +} + +// DBAdminRouter serves database diagnostics under the admin router. With pg nil, /planguard is not served. +func DBAdminRouter(reader DBStatementsReader, pg PlanguardSource) http.Handler { r := chi.NewRouter() r.Get("/statements", getDBStatements(reader)) + if pg != nil { + r.Get("/planguard", getPlanguard(pg)) + } return r } +// getPlanguard godoc +// +// @Summary Last daily plan regression check +// @Description Plans every corpus statement in custom and generic mode against this database, read-only and without executing it (plain EXPLAIN, 5 s per plan, 1 s lock timeout), and checks each plan against the corpus. Off unless planguard.daily_check is set (404 "planguard daily check is disabled"); when on, the check runs about an hour after boot and then daily, and until its first run this returns 404. fresh=true runs it now, shared with any run in progress and cancelled at server shutdown; within 5 minutes of the last run's end it starts nothing and returns the last successful report (started_at gives its age), or 500 when no run has succeeded yet. A read-only server cannot enable the check (configuration refuses planguard.daily_check with server.read_only), so there this returns 404 "planguard daily check is disabled". +// @Tags Admin +// @Produce json +// @Security AdminKey +// @Param fresh query bool false "Run the check now instead of returning the last report" +// @Success 200 {object} api.PlanguardReport +// @Failure 400,401,404,500 {object} map[string]APIError +// @Router /admin/db/planguard [get] +func getPlanguard(pg PlanguardSource) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + fresh := false + if v := r.URL.Query().Get("fresh"); v != "" { + b, err := strconv.ParseBool(v) + if err != nil { + respondError(w, http.StatusBadRequest, "invalid fresh value") + return + } + fresh = b + } + if fresh { + rep, err := pg.RunNow(r.Context()) + if errors.Is(err, api.ErrPlanguardDisabled) { + respondError(w, http.StatusNotFound, err.Error()) + return + } + if err != nil { + slog.Error("api: plan check", "component", "api", "error", err) + respondError(w, http.StatusInternalServerError, "cannot run the plan check") + return + } + respond(w, http.StatusOK, rep) + return + } + rep, err := pg.Last() + if errors.Is(err, api.ErrPlanguardNotRun) || errors.Is(err, api.ErrPlanguardDisabled) { + respondError(w, http.StatusNotFound, err.Error()) + return + } + if err != nil { + slog.Error("api: read plan check", "component", "api", "error", err) + respondError(w, http.StatusInternalServerError, "cannot read the plan check") + return + } + respond(w, http.StatusOK, rep) + } +} + // getDBStatements godoc // // @Summary Statement statistics by sqlc name diff --git a/internal/api/handlers/db_admin_test.go b/internal/api/handlers/db_admin_test.go index e4b925f..f38d784 100644 --- a/internal/api/handlers/db_admin_test.go +++ b/internal/api/handlers/db_admin_test.go @@ -10,6 +10,7 @@ import ( "net/http/httptest" "strings" "testing" + "time" "github.com/MeshCore-Beacon/beacon-server/internal/api" ) @@ -26,7 +27,7 @@ func (s stubStatements) DBStatements(context.Context) ([]api.DBStatement, error) func serveStatements(t *testing.T, reader DBStatementsReader) *httptest.ResponseRecorder { t.Helper() w := httptest.NewRecorder() - DBAdminRouter(reader).ServeHTTP(w, httptest.NewRequest("GET", "/statements", nil)) + DBAdminRouter(reader, nil).ServeHTTP(w, httptest.NewRequest("GET", "/statements", nil)) return w } @@ -87,3 +88,147 @@ func TestDBAdminStatementsErrors(t *testing.T) { } } } + +type stubPlanguard struct { + last api.PlanguardReport + lastErr error + fresh api.PlanguardReport + runErr error + runs int +} + +func (s *stubPlanguard) Last() (api.PlanguardReport, error) { return s.last, s.lastErr } + +func (s *stubPlanguard) RunNow(context.Context) (api.PlanguardReport, error) { + s.runs++ + return s.fresh, s.runErr +} + +func servePlanguard(t *testing.T, pg PlanguardSource, target string) *httptest.ResponseRecorder { + t.Helper() + w := httptest.NewRecorder() + DBAdminRouter(stubStatements{}, pg).ServeHTTP(w, httptest.NewRequest("GET", target, nil)) + return w +} + +func errorBody(t *testing.T, w *httptest.ResponseRecorder) APIError { + t.Helper() + var body map[string]APIError + if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil { + t.Fatalf("%v: %s", err, w.Body) + } + return body["error"] +} + +func TestDBAdminPlanguard(t *testing.T) { + report := api.PlanguardReport{ + Tier: "prod", + StartedAt: time.Date(2026, 10, 19, 1, 2, 3, 0, time.UTC), + DurationMs: 1234.5, + Clean: false, + Counts: map[string]int{"ok": 1, "MISSED": 1}, + Failures: []api.PlanguardRun{{Entry: "b", Stmt: "S", Mode: "generic", Verdict: "MISSED", Failures: []string{"driver: x"}}}, + Runs: []api.PlanguardRun{ + {Entry: "a", Stmt: "S", Mode: "custom", Verdict: "ok", Drivers: []string{"known_routes:Index Scan:ix"}}, + {Entry: "b", Stmt: "S", Mode: "generic", Verdict: "MISSED", Failures: []string{"driver: x"}}, + }, + } + + t.Run("not run yet", func(t *testing.T) { + pg := &stubPlanguard{lastErr: api.ErrPlanguardNotRun} + w := servePlanguard(t, pg, "/planguard") + if e := errorBody(t, w); w.Code != 404 || e.Code != "not_found" || e.Message != "planguard has not run yet" { + t.Fatalf("got %d %s", w.Code, w.Body) + } + if pg.runs != 0 { + t.Fatalf("RunNow called %d times", pg.runs) + } + }) + + t.Run("last report", func(t *testing.T) { + pg := &stubPlanguard{last: report} + w := servePlanguard(t, pg, "/planguard") + if w.Code != 200 || pg.runs != 0 { + t.Fatalf("got %d (runs %d) %s", w.Code, pg.runs, w.Body) + } + var raw map[string]any + if err := json.Unmarshal(w.Body.Bytes(), &raw); err != nil { + t.Fatal(err) + } + for _, key := range []string{"tier", "started_at", "duration_ms", "clean", "counts", "failures", "runs"} { + if _, ok := raw[key]; !ok { + t.Errorf("report lacks %q: %s", key, w.Body) + } + } + if raw["tier"] != "prod" || raw["clean"] != false || raw["started_at"] != "2026-10-19T01:02:03Z" { + t.Errorf("report = %s", w.Body) + } + run := raw["runs"].([]any)[0].(map[string]any) + for _, key := range []string{"entry", "stmt", "mode", "verdict", "drivers"} { + if _, ok := run[key]; !ok { + t.Errorf("run lacks %q: %v", key, run) + } + } + if _, ok := run["failures"]; ok { + t.Errorf("a run without failures carries the key: %v", run) + } + }) + + t.Run("empty failures is an array", func(t *testing.T) { + clean := api.PlanguardReport{Tier: "prod", Clean: true, Counts: map[string]int{}, Failures: []api.PlanguardRun{}, Runs: []api.PlanguardRun{}} + w := servePlanguard(t, &stubPlanguard{last: clean}, "/planguard") + if w.Code != 200 || !strings.Contains(w.Body.String(), `"failures":[]`) { + t.Fatalf("got %d %s", w.Code, w.Body) + } + }) + + t.Run("fresh", func(t *testing.T) { + fresh := report + fresh.DurationMs = 99 + pg := &stubPlanguard{lastErr: api.ErrPlanguardNotRun, fresh: fresh} + w := servePlanguard(t, pg, "/planguard?fresh=true") + if w.Code != 200 || pg.runs != 1 || !strings.Contains(w.Body.String(), `"duration_ms":99`) { + t.Fatalf("got %d (runs %d) %s", w.Code, pg.runs, w.Body) + } + }) + + t.Run("fresh fails", func(t *testing.T) { + pg := &stubPlanguard{last: report, runErr: errors.New("PRIVATE_CANARY")} + w := servePlanguard(t, pg, "/planguard?fresh=true") + if e := errorBody(t, w); w.Code != 500 || e.Message != "cannot run the plan check" || strings.Contains(w.Body.String(), "PRIVATE_CANARY") { + t.Fatalf("got %d %s", w.Code, w.Body) + } + }) + + t.Run("bad fresh", func(t *testing.T) { + pg := &stubPlanguard{last: report} + w := servePlanguard(t, pg, "/planguard?fresh=maybe") + if w.Code != 400 || pg.runs != 0 { + t.Fatalf("got %d (runs %d) %s", w.Code, pg.runs, w.Body) + } + }) + + t.Run("statements still served", func(t *testing.T) { + w := servePlanguard(t, &stubPlanguard{last: report}, "/statements") + if w.Code != 200 || strings.TrimSpace(w.Body.String()) != `{"items":[]}` { + t.Fatalf("got %d %s", w.Code, w.Body) + } + }) + + t.Run("disabled", func(t *testing.T) { + for _, target := range []string{"/planguard", "/planguard?fresh=true"} { + pg := &stubPlanguard{lastErr: api.ErrPlanguardDisabled, runErr: api.ErrPlanguardDisabled} + w := servePlanguard(t, pg, target) + if e := errorBody(t, w); w.Code != 404 || e.Code != "not_found" || e.Message != "planguard daily check is disabled" { + t.Fatalf("%s: got %d %s", target, w.Code, w.Body) + } + } + }) + + t.Run("no source", func(t *testing.T) { + w := servePlanguard(t, nil, "/planguard?fresh=true") + if w.Code != 404 { + t.Fatalf("got %d %s", w.Code, w.Body) + } + }) +} diff --git a/internal/background/planguard.go b/internal/background/planguard.go new file mode 100644 index 0000000..c10806c --- /dev/null +++ b/internal/background/planguard.go @@ -0,0 +1,162 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package background + +import ( + "context" + "errors" + "fmt" + "log/slog" + "sync" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/metrics" + "github.com/MeshCore-Beacon/beacon-server/internal/planguard" +) + +const ( + // freshMinGap is the least time between two runs RunNow starts; inside it RunNow returns the last report. + freshMinGap = 5 * time.Minute + // freshRunTimeout bounds a run RunNow starts, which outlives the request that asked for it but not the server. + freshRunTimeout = 15 * time.Minute +) + +// Planguard runs the read-only plan check, keeps its last report and exports it as metrics. +type Planguard struct { + base context.Context // ends at server shutdown; fresh runs derive from it + run func(context.Context) (*planguard.Report, error) + now func() time.Time + + mu sync.Mutex + last *planguard.Report + lastAt time.Time // start of the last run that finished + attemptAt time.Time // end of the last run, finished or not + attemptErr error + inflight *planguardCall +} + +type planguardCall struct { + done chan struct{} + rep *planguard.Report + err error +} + +// NewPlanguard returns a Planguard that checks plans with run. base ends every run when it ends (server shutdown). +func NewPlanguard(base context.Context, run func(context.Context) (*planguard.Report, error)) *Planguard { + return &Planguard{base: base, run: run, now: time.Now} +} + +// Task ticks hourly and runs when it has never run or the last run started at least every ago, so the first +// check happens about an hour after boot. A run that fails is retried on the next tick. +func (p *Planguard) Task(every time.Duration) Task { + return Task{ + Name: "planguard", + Interval: time.Hour, + Run: func(ctx context.Context) error { + p.mu.Lock() + due := p.lastAt.IsZero() || p.now().Sub(p.lastAt) >= every + p.mu.Unlock() + if !due { + return nil + } + _, err := p.do(ctx, false) + return err + }, + } +} + +// RunNow runs the check now and returns its report. Concurrent callers share one run, which outlives a caller that +// gives up but not the server. Within freshMinGap of the last run's end it starts nothing: it returns the last +// successful report (its StartedAt gives its age), or the last attempt's error when there is no report. +func (p *Planguard) RunNow(ctx context.Context) (*planguard.Report, error) { + return p.do(ctx, true) +} + +// Last returns the report of the last run that finished. +func (p *Planguard) Last() (*planguard.Report, bool) { + p.mu.Lock() + defer p.mu.Unlock() + return p.last, p.last != nil +} + +// do joins the run in flight or starts one, and waits for it or for ctx. A fresh run is bounded by base and +// freshRunTimeout; a scheduled one ends with ctx or base. For fresh callers, the gap is checked under the same lock +// that starts a run, so callers cannot chain runs. +func (p *Planguard) do(ctx context.Context, fresh bool) (*planguard.Report, error) { + p.mu.Lock() + c := p.inflight + if c == nil && fresh && !p.attemptAt.IsZero() && p.now().Sub(p.attemptAt) < freshMinGap { + rep, err := p.last, p.attemptErr + p.mu.Unlock() + if rep != nil { + return rep, nil + } + return nil, err + } + if c == nil { + c = &planguardCall{done: make(chan struct{})} + p.inflight = c + var runCtx context.Context + var cleanup func() + if fresh { + var cancel context.CancelFunc + runCtx, cancel = context.WithTimeout(p.base, freshRunTimeout) + cleanup = cancel + } else { + var cancel context.CancelFunc + runCtx, cancel = context.WithCancel(ctx) + stopBase := context.AfterFunc(p.base, cancel) + cleanup = func() { stopBase(); cancel() } + } + go p.start(runCtx, cleanup, c) + } + p.mu.Unlock() + select { + case <-c.done: + return c.rep, c.err + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +// start makes one run on ctx and publishes its outcome to c and, when it succeeds, to Last and the metrics. +func (p *Planguard) start(ctx context.Context, cleanup func(), c *planguardCall) { + defer cleanup() + start := p.now() + rep, err := p.run(ctx) + if err == nil && rep == nil { + err = errNoReport + } + p.mu.Lock() + p.attemptAt, p.attemptErr = p.now(), err + if err == nil { + p.last, p.lastAt = rep, start + publishPlanguard(rep, p.now()) + } else { + rep = nil + } + c.rep, c.err = rep, err + p.inflight = nil + p.mu.Unlock() + close(c.done) +} + +var errNoReport = errors.New("plan check returned no report") + +// publishPlanguard replaces the failure series with rep's failing runs, stamps the run time and logs a summary. +func publishPlanguard(rep *planguard.Report, at time.Time) { + metrics.PlanguardFailures.Reset() + var failing []string + for _, r := range rep.Failures() { + metrics.PlanguardFailures.WithLabelValues(r.ID, r.Mode).Set(1) + failing = append(failing, r.ID+"|"+r.Mode) + } + metrics.PlanguardLastRun.Set(float64(at.UnixNano()) / 1e9) + msg := fmt.Sprintf("planguard: %d runs, %d failing", len(rep.Results), len(failing)) + if len(failing) > 0 { + slog.Warn(msg, "component", "background", "failing", failing) + return + } + slog.Info(msg, "component", "background") +} diff --git a/internal/background/planguard_test.go b/internal/background/planguard_test.go new file mode 100644 index 0000000..5a6d0a7 --- /dev/null +++ b/internal/background/planguard_test.go @@ -0,0 +1,322 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package background + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "testing/synctest" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/metrics" + "github.com/MeshCore-Beacon/beacon-server/internal/planguard" + "github.com/prometheus/client_golang/prometheus/testutil" +) + +func pgReport(results ...planguard.Result) *planguard.Report { + return &planguard.Report{Tier: "prod", StartedAt: time.Now().UTC(), Results: results} +} + +func TestPlanguardTaskRunsDaily(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + start := time.Now() + var at []time.Duration + pg := NewPlanguard(context.Background(), func(context.Context) (*planguard.Report, error) { + at = append(at, time.Since(start)) + return pgReport(), nil + }) + task := pg.Task(24 * time.Hour) + if task.Name != "planguard" || task.Interval != time.Hour { + t.Fatalf("task = %s every %v", task.Name, task.Interval) + } + if _, ok := pg.Last(); ok { + t.Fatal("Last before the first run") + } + for h := 1; h <= 49; h++ { + time.Sleep(start.Add(time.Duration(h) * time.Hour).Sub(time.Now())) + if err := task.Run(context.Background()); err != nil { + t.Fatal(err) + } + } + want := []time.Duration{time.Hour, 25 * time.Hour, 49 * time.Hour} + if len(at) != len(want) || at[0] != want[0] || at[1] != want[1] || at[2] != want[2] { + t.Fatalf("ran at %v, want %v", at, want) + } + }) +} + +func TestPlanguardSetsMetric(t *testing.T) { + first := pgReport( + planguard.Result{ID: "a", Mode: "custom", Verdict: planguard.VerdictFalsePositive}, + planguard.Result{ID: "b", Mode: "generic", Verdict: planguard.VerdictMissed}, + planguard.Result{ID: "c", Mode: "custom", Verdict: planguard.VerdictRetiredIndex}, + planguard.Result{ID: "d", Mode: "custom", Verdict: planguard.VerdictOK}, + ) + clean := pgReport(planguard.Result{ID: "d", Mode: "custom", Verdict: planguard.VerdictOK}) + errored := pgReport(planguard.Result{ID: "e", Mode: "generic", Verdict: planguard.VerdictError}) + failure := errors.New("no connection") + var next *planguard.Report + var nextErr error + pg := NewPlanguard(context.Background(), func(context.Context) (*planguard.Report, error) { return next, nextErr }) + task := pg.Task(24 * time.Hour) + metrics.PlanguardFailures.Reset() + metrics.PlanguardLastRun.Set(0) + + next = first + if err := task.Run(context.Background()); err != nil { + t.Fatal(err) + } + if v := testutil.ToFloat64(metrics.PlanguardFailures.WithLabelValues("a", "custom")); v != 1 { + t.Errorf("(a, custom) = %v", v) + } + if v := testutil.ToFloat64(metrics.PlanguardFailures.WithLabelValues("b", "generic")); v != 1 { + t.Errorf("(b, generic) = %v", v) + } + if n := testutil.CollectAndCount(metrics.PlanguardFailures); n != 2 { + t.Errorf("series = %d, want 2", n) + } + if v := testutil.ToFloat64(metrics.PlanguardLastRun); v <= 0 { + t.Errorf("last run = %v", v) + } + + // A run that fails keeps the previous report and series; the task returns the error. + pg.lastAt = time.Time{} + next, nextErr = nil, failure + if err := task.Run(context.Background()); !errors.Is(err, failure) { + t.Fatalf("task error = %v, want %v", err, failure) + } + if got, ok := pg.Last(); !ok || got != first { + t.Errorf("Last after a failed run = %v, %v", got, ok) + } + if n := testutil.CollectAndCount(metrics.PlanguardFailures); n != 2 { + t.Errorf("series after a failed run = %d, want 2", n) + } + + pg.lastAt = time.Time{} + next, nextErr = clean, nil + if err := task.Run(context.Background()); err != nil { + t.Fatal(err) + } + if n := testutil.CollectAndCount(metrics.PlanguardFailures); n != 0 { + t.Errorf("series after a clean run = %d, want 0", n) + } + + // ERROR counts as a failure. + pg.lastAt = time.Time{} + next = errored + if err := task.Run(context.Background()); err != nil { + t.Fatal(err) + } + if v := testutil.ToFloat64(metrics.PlanguardFailures.WithLabelValues("e", "generic")); v != 1 { + t.Errorf("(e, generic) = %v", v) + } + metrics.PlanguardFailures.Reset() +} + +func TestPlanguardRunNowSingleFlight(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + var calls atomic.Int32 + rep := pgReport() + pg := NewPlanguard(context.Background(), func(context.Context) (*planguard.Report, error) { + calls.Add(1) + time.Sleep(time.Minute) + return rep, nil + }) + var wg sync.WaitGroup + got := make([]*planguard.Report, 5) + for i := range got { + wg.Go(func() { + r, err := pg.RunNow(context.Background()) + if err != nil { + t.Error(err) + } + got[i] = r + }) + } + wg.Wait() + if n := calls.Load(); n != 1 { + t.Fatalf("run called %d times, want 1", n) + } + for i, r := range got { + if r != rep { + t.Errorf("caller %d got %p, want %p", i, r, rep) + } + } + }) +} + +func TestPlanguardRunNowRateLimited(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + var calls atomic.Int32 + pg := NewPlanguard(context.Background(), func(context.Context) (*planguard.Report, error) { + calls.Add(1) + return pgReport(), nil + }) + first, err := pg.RunNow(context.Background()) + if err != nil { + t.Fatal(err) + } + time.Sleep(freshMinGap - time.Second) + again, err := pg.RunNow(context.Background()) + if err != nil || again != first || calls.Load() != 1 { + t.Fatalf("RunNow inside the gap ran again: calls %d, same %v, err %v", calls.Load(), again == first, err) + } + time.Sleep(time.Second) + if _, err := pg.RunNow(context.Background()); err != nil || calls.Load() != 2 { + t.Fatalf("RunNow after the gap: calls %d, err %v", calls.Load(), err) + } + }) +} + +func TestPlanguardRunNowOutlivesCaller(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + var runCtxErr error + pg := NewPlanguard(context.Background(), func(ctx context.Context) (*planguard.Report, error) { + time.Sleep(time.Minute) + runCtxErr = ctx.Err() + return pgReport(), nil + }) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if _, err := pg.RunNow(ctx); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("RunNow = %v, want the caller's deadline", err) + } + synctest.Wait() + time.Sleep(time.Minute) + synctest.Wait() + if runCtxErr != nil { + t.Errorf("the run's context ended with its caller: %v", runCtxErr) + } + if _, ok := pg.Last(); !ok { + t.Error("the run's report was not kept") + } + }) +} + +// Cancelling the base context (server shutdown) ends a fresh run in flight, and a scheduler tick joined to it +// returns instead of waiting out the run. +func TestPlanguardShutdownCancelsFreshRun(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + base, shutdown := context.WithCancel(context.Background()) + var runCtxErr error + pg := NewPlanguard(base, func(ctx context.Context) (*planguard.Report, error) { + select { + case <-ctx.Done(): + runCtxErr = ctx.Err() + return nil, ctx.Err() + case <-time.After(freshRunTimeout + time.Hour): + return pgReport(), nil + } + }) + freshErr := make(chan error, 1) + go func() { + _, err := pg.RunNow(context.Background()) + freshErr <- err + }() + synctest.Wait() + tickErr := make(chan error, 1) + go func() { tickErr <- pg.Task(24 * time.Hour).Run(context.Background()) }() + synctest.Wait() + start := time.Now() + shutdown() + if err := <-tickErr; !errors.Is(err, context.Canceled) { + t.Errorf("joined tick returned %v, want context.Canceled", err) + } + if err := <-freshErr; !errors.Is(err, context.Canceled) { + t.Errorf("fresh run returned %v, want context.Canceled", err) + } + synctest.Wait() + if !errors.Is(runCtxErr, context.Canceled) { + t.Errorf("the run's context ended with %v, want context.Canceled", runCtxErr) + } + if d := time.Since(start); d != 0 { + t.Errorf("shutdown waited %v for the run", d) + } + }) +} + +// A caller that gives up stops waiting for a run it joined; the run goes on. +func TestPlanguardJoinHonoursCallerContext(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + var calls atomic.Int32 + pg := NewPlanguard(context.Background(), func(context.Context) (*planguard.Report, error) { + calls.Add(1) + time.Sleep(time.Hour) + return pgReport(), nil + }) + go func() { _, _ = pg.RunNow(context.Background()) }() + synctest.Wait() + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + if err := pg.Task(24 * time.Hour).Run(ctx); !errors.Is(err, context.DeadlineExceeded) { + t.Errorf("joined tick = %v, want its own deadline", err) + } + time.Sleep(time.Hour) + synctest.Wait() + if calls.Load() != 1 { + t.Errorf("run called %d times, want 1", calls.Load()) + } + }) +} + +// Callers arriving while a run is in flight and right after it ends never start a second run inside the gap. +func TestPlanguardRunNowNoChaining(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + var calls atomic.Int32 + pg := NewPlanguard(context.Background(), func(context.Context) (*planguard.Report, error) { + calls.Add(1) + time.Sleep(time.Minute) + return pgReport(), nil + }) + var wg sync.WaitGroup + for i := range 10 { + wg.Go(func() { + time.Sleep(time.Duration(i) * 15 * time.Second) // 0 s .. 135 s: during the run and after it + if _, err := pg.RunNow(context.Background()); err != nil { + t.Error(err) + } + }) + } + wg.Wait() + if n := calls.Load(); n != 1 { + t.Errorf("run called %d times, want 1", n) + } + }) +} + +// Inside the gap a failed attempt does not hide the last good report; with no report the error is returned. +func TestPlanguardRunNowFailedAttemptInGap(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + failure := errors.New("no connection") + good := pgReport() + var next *planguard.Report + var nextErr error + pg := NewPlanguard(context.Background(), func(context.Context) (*planguard.Report, error) { return next, nextErr }) + + nextErr = failure + if _, err := pg.RunNow(context.Background()); !errors.Is(err, failure) { + t.Fatalf("first attempt = %v", err) + } + if _, err := pg.RunNow(context.Background()); !errors.Is(err, failure) { + t.Errorf("in the gap with no report = %v, want the attempt's error", err) + } + + time.Sleep(freshMinGap) + next, nextErr = good, nil + if rep, err := pg.RunNow(context.Background()); err != nil || rep != good { + t.Fatalf("good run = %v, %v", rep, err) + } + time.Sleep(freshMinGap) + next, nextErr = nil, failure + if _, err := pg.RunNow(context.Background()); !errors.Is(err, failure) { + t.Fatalf("failed run after the gap = %v", err) + } + if rep, err := pg.RunNow(context.Background()); err != nil || rep != good { + t.Errorf("in the gap after a failed attempt = %v, %v; want the last good report", rep, err) + } + }) +} diff --git a/internal/config/config.go b/internal/config/config.go index 2f23500..39e6d32 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -52,6 +52,14 @@ type Config struct { Web WebConfig `yaml:"web"` Database DatabaseConfig `yaml:"database"` Guards GuardsConfig `yaml:"guards"` + Planguard PlanguardConfig `yaml:"planguard"` +} + +// PlanguardConfig controls the daily read-only plan regression check. +type PlanguardConfig struct { + // DailyCheck runs the check about an hour after boot and then daily. Off by default: the corpus's plan + // expectations assume production-sized tables, so small deployments would report false failures. + DailyCheck bool `yaml:"daily_check"` } // DatabaseConfig controls how API statements are spread across connection pools. @@ -107,6 +115,8 @@ type ServerConfig struct { TrustedProxies []netip.Prefix `yaml:"trusted_proxies"` // AdminListen is the host:port of the admin listener serving /metrics; empty disables it. AdminListen string `yaml:"admin_listen"` + // ReadOnly boots a server with no migrations, ingest, jobs or boot writes; every database session is read-only. + ReadOnly bool `yaml:"read_only"` } func (c *ServerConfig) UnmarshalYAML(node *yaml.Node) error { @@ -115,6 +125,7 @@ func (c *ServerConfig) UnmarshalYAML(node *yaml.Node) error { var raw struct { TrustedProxies []*netip.Prefix `yaml:"trusted_proxies"` AdminListen string `yaml:"admin_listen"` + ReadOnly bool `yaml:"read_only"` } if err := node.Decode(&raw); err != nil { return err @@ -126,6 +137,7 @@ func (c *ServerConfig) UnmarshalYAML(node *yaml.Node) error { } } c.AdminListen = raw.AdminListen + c.ReadOnly = raw.ReadOnly return nil } @@ -169,6 +181,10 @@ type ResolvedConfig struct { GuardsBudget string // AdminListen is the admin listener address; empty disables it. AdminListen string + // ReadOnly: no migrations, ingest, background jobs or boot writes; every database session is read-only. + ReadOnly bool + // PlanguardDailyCheck runs the daily read-only plan check; off by default. + PlanguardDailyCheck bool } // RateLimitConfig controls the per-client REST API request budget. @@ -513,6 +529,9 @@ func Load(path string) (*Config, error) { if err := cfg.validateClientVersions(); err != nil { return nil, err } + if cfg.Planguard.DailyCheck && cfg.Server.ReadOnly { + return nil, fmt.Errorf("planguard.daily_check cannot be enabled with server.read_only: a read-only server runs no background jobs") + } if cfg.Nodes.MarkForeign && !cfg.MeshMapper.Zones.Enabled && !cfg.hasBorderFile() { return nil, fmt.Errorf("nodes.mark_foreign requires an iatas.*.borderFile or meshmapper.zones") } @@ -684,6 +703,9 @@ func Resolve(cfg *Config) ResolvedConfig { GuardsEnforce: canonicalFamilies(cfg.Guards.Enforce), GuardsBudget: cfg.Guards.Budget, AdminListen: cfg.Server.AdminListen, + ReadOnly: cfg.Server.ReadOnly, + + PlanguardDailyCheck: cfg.Planguard.DailyCheck, } // Load has validated the layout; an unvalidated config falls back to the default. r.APIPlanLayout, _ = planpolicy.ParseLayout(cfg.Database.APIPlanLayout) @@ -758,12 +780,12 @@ func Resolve(cfg *Config) ResolvedConfig { func (r ResolvedConfig) String() string { return fmt.Sprintf( - "telemetryResolution=%s telemetryRetention=%s rollupRetention=%s packetRetention=%s routeRetention=%s routeGrace=%s routeMinObs=%d routeReconfirmRowsPerHour=%d maxConnsPerIP=%d maxConnectsPerMinute=%d viewRefresh=%s reconfirm=%s cleanup=%s presenceFlush=%s presencePacketTTL=%s clockDriftThreshold=%s nodeStaleThreshold=%s nodeDeleteAfter=%s observerDeleteAfter=%s rateLimitEnabled=%t requestsPerMinute=%d burst=%d apiPlanLayout=%s guardsEnforce=%v guardsBudget=%s adminListen=%q", + "telemetryResolution=%s telemetryRetention=%s rollupRetention=%s packetRetention=%s routeRetention=%s routeGrace=%s routeMinObs=%d routeReconfirmRowsPerHour=%d maxConnsPerIP=%d maxConnectsPerMinute=%d viewRefresh=%s reconfirm=%s cleanup=%s presenceFlush=%s presencePacketTTL=%s clockDriftThreshold=%s nodeStaleThreshold=%s nodeDeleteAfter=%s observerDeleteAfter=%s rateLimitEnabled=%t requestsPerMinute=%d burst=%d apiPlanLayout=%s guardsEnforce=%v guardsBudget=%s adminListen=%q readOnly=%t planguardDailyCheck=%t", r.TelemetryResolution, r.TelemetryRetention, r.RollupRetention, r.PacketRetention, r.RouteRetention, r.RouteGrace, r.RouteMinObservations, r.RouteReconfirmRowsPerHour, r.MaxConnsPerIP, r.MaxConnectsPerMinute, r.ViewRefreshInterval, r.ReconfirmInterval, r.CleanupInterval, r.PresenceFlushInterval, r.PresencePacketTTL, r.ClockDriftThreshold, r.NodeStaleThreshold, r.NodeDeleteAfter, r.ObserverDeleteAfter, r.RateLimit.Enabled, r.RateLimit.RequestsPerMinute, r.RateLimit.Burst, - r.APIPlanLayout, r.GuardsEnforce, r.GuardsBudget, r.AdminListen, + r.APIPlanLayout, r.GuardsEnforce, r.GuardsBudget, r.AdminListen, r.ReadOnly, r.PlanguardDailyCheck, ) } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 4d34107..422713d 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -6,6 +6,7 @@ package config import ( "os" "path/filepath" + "strconv" "strings" "testing" "time" @@ -463,3 +464,62 @@ func TestGuardFamiliesMatch(t *testing.T) { t.Errorf("guardFamilies = %v, want %v", guardFamilies, want) } } + +func TestLoad_ServerReadOnly(t *testing.T) { + for _, tc := range []struct { + yaml string + want bool + }{ + {"server:\n read_only: true\n", true}, + {"server:\n admin_listen: \"\"\n", false}, + {"server:\n read_only: yes\n", true}, + {"server:\n read_only: false\n", false}, + } { + path := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(path, []byte(tc.yaml), 0600); err != nil { + t.Fatal(err) + } + cfg, err := Load(path) + if err != nil { + t.Fatalf("Load(%q): %v", tc.yaml, err) + } + if cfg.Server.ReadOnly != tc.want { + t.Errorf("Load(%q): ReadOnly = %v, want %v", tc.yaml, cfg.Server.ReadOnly, tc.want) + } + if r := Resolve(cfg); r.ReadOnly != tc.want || !strings.Contains(r.String(), "readOnly="+strconv.FormatBool(tc.want)) { + t.Errorf("Resolve(%q): ReadOnly = %v (%s), want %v", tc.yaml, r.ReadOnly, r, tc.want) + } + } +} + +func TestLoad_PlanguardDailyCheck(t *testing.T) { + for _, tc := range []struct { + yaml string + want bool + wantErr string + }{ + {"", false, ""}, + {"planguard: {}\n", false, ""}, + {"planguard:\n daily_check: true\n", true, ""}, + {"planguard:\n daily_check: false\n", false, ""}, + {"planguard:\n daily_check: true\nserver:\n read_only: true\n", false, "planguard.daily_check cannot be enabled with server.read_only"}, + } { + path := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(path, []byte(tc.yaml), 0600); err != nil { + t.Fatal(err) + } + cfg, err := Load(path) + if tc.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Errorf("Load(%q): err %v, want %q", tc.yaml, err, tc.wantErr) + } + continue + } + if err != nil { + t.Fatalf("Load(%q): %v", tc.yaml, err) + } + if r := Resolve(cfg); r.PlanguardDailyCheck != tc.want || !strings.Contains(r.String(), "planguardDailyCheck="+strconv.FormatBool(tc.want)) { + t.Errorf("Resolve(%q): PlanguardDailyCheck = %v (%s), want %v", tc.yaml, r.PlanguardDailyCheck, r, tc.want) + } + } +} diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index eb2bd4d..8011736 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -12,6 +12,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/collectors" "github.com/prometheus/client_golang/prometheus/promhttp" + dto "github.com/prometheus/client_model/go" ) var ( @@ -99,8 +100,43 @@ var ( Name: "beacon_index_bloat_ratio", Help: "Estimated index bloat ratio, by index.", }, []string{"index"}) + + // PlanguardFailures marks each corpus entry and plan mode that failed the last daily plan check. + PlanguardFailures = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "beacon_planguard_failures", + Help: "1 for each corpus entry and plan mode whose last daily plan check failed.", + }, []string{"entry", "mode"}) + + // PlanguardLastRun is when the daily plan check last finished. A server with the check off carries no + // series; with it on (EnablePlanguard) the series is 0 until the first run. + PlanguardLastRun = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "beacon_planguard_last_run_timestamp_seconds", + Help: "Unix time the daily plan check last finished.", + }) ) +var planguardEnabled atomic.Bool + +// EnablePlanguard exports PlanguardLastRun from now on, 0 until the first run, so a staleness alert on it fires +// for a check that is on but never succeeds. Without it the gauge is exported only once set. +func EnablePlanguard() { planguardEnabled.Store(true) } + +// setGauge exports its gauge once it holds a non-zero value, or always once enabled reports true. +type setGauge struct { + prometheus.Gauge + enabled *atomic.Bool +} + +func (g setGauge) Collect(ch chan<- prometheus.Metric) { + if !g.enabled.Load() { + var m dto.Metric + if err := g.Write(&m); err != nil || m.GetGauge().GetValue() == 0 { + return + } + } + ch <- g.Gauge +} + // PgActivity is one snapshot of PostgreSQL-wide activity counters. type PgActivity struct { WALBytes int64 @@ -154,7 +190,7 @@ func init() { collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), GuardWouldRefuse, GuardRefused, GuardDB, DBTime, HTTPRequest, PoolAcquire, IngestQueueDepth, IngestDropped, JobDuration, JobRows, IndexBytes, IndexBloat, - pgActivity, + PlanguardFailures, setGauge{PlanguardLastRun, &planguardEnabled}, pgActivity, ) } diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go index 10bd18c..6b565d9 100644 --- a/internal/metrics/metrics_test.go +++ b/internal/metrics/metrics_test.go @@ -38,3 +38,34 @@ func TestHandlerServesBeaconMetrics(t *testing.T) { } } } + +func TestRegistryGathersPlanguard(t *testing.T) { + // A GaugeVec with no series is not gathered: set one first. + PlanguardFailures.WithLabelValues("routes.example", "custom").Set(1) + defer PlanguardFailures.Reset() + PlanguardLastRun.Set(1760000000) + body := scrape(t) + for _, want := range []string{ + `beacon_planguard_failures{entry="routes.example",mode="custom"} 1`, + `beacon_planguard_last_run_timestamp_seconds 1.76e+09`, + } { + if !strings.Contains(body, want) { + t.Errorf("missing %q", want) + } + } +} + +// With the check off neither metric carries a series, so a staleness alert cannot fire on a server that never +// runs it. Once enabled, the timestamp is exported at 0 before the first run, so a check that never succeeds +// trips the alert. (EnablePlanguard cannot be undone: the off state is checked first.) +func TestPlanguardMetricsOffAndEnabled(t *testing.T) { + PlanguardFailures.Reset() + PlanguardLastRun.Set(0) + if body := scrape(t); strings.Contains(body, "\nbeacon_planguard_failures{") || strings.Contains(body, "\nbeacon_planguard_last_run_timestamp_seconds ") { + t.Errorf("planguard series with the check off:\n%s", body) + } + EnablePlanguard() + if body := scrape(t); !strings.Contains(body, "\nbeacon_planguard_last_run_timestamp_seconds 0\n") { + t.Errorf("enabled check: no timestamp series at 0") + } +} diff --git a/internal/planguard/assert.go b/internal/planguard/assert.go new file mode 100644 index 0000000..6c11237 --- /dev/null +++ b/internal/planguard/assert.go @@ -0,0 +1,380 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package planguard + +import ( + "fmt" + "slices" + "sort" + "strings" +) + +// FailureKind names the rule a plan broke. +type FailureKind string + +// Failure kinds. +const ( + FailDriver FailureKind = "driver" + FailIndexCond FailureKind = "index_cond" + FailForbidden FailureKind = "forbidden" + FailRequired FailureKind = "required" + FailSortAboveLimit FailureKind = "sort_above_limit" + FailBudget FailureKind = "budget" + FailPlanChange FailureKind = "plan_change" + FailError FailureKind = "error" +) + +// Failure is one broken rule. +type Failure struct { + Kind FailureKind `json:"kind"` + Msg string `json:"msg"` + Indexes []string `json:"indexes,omitempty"` // indexes of the scan(s) the failure is about +} + +// Check is what Assess found in one plan. +type Check struct { + Failures []Failure `json:"failures,omitempty"` + Drivers []string `json:"drivers,omitempty"` // "relation(alias):node:index" as seen, sorted + Retired []string `json:"retired,omitempty"` // retired indexes anywhere in the plan + Buffers int64 `json:"buffers"` // root shared hit + read (ANALYZE only) + Rows int64 `json:"rows"` // root actual rows × loops (ANALYZE only) +} + +// defaultDriverNodes are the scan types a driver may use when its spec names none. +var defaultDriverNodes = []string{"Index Scan", "Index Only Scan"} + +// passThrough nodes carry a Limit's bound down to a Sort below them. +var passThrough = []string{"Result", "Gather", "Gather Merge", "Subquery Scan", "Materialize", "Unique"} + +type scanAt struct { + n Node + path []Node +} + +// Assess checks one plan against exp and the corpus defaults. Budgets are checked only when analyzed. +func (c *Corpus) Assess(ex *Explain, exp *Expect, analyzed bool) Check { + var ch Check + root := ex.Root + if analyzed { + ch.Buffers = sharedBlocks(root) + ch.Rows = int64(root.num("Actual Rows") * root.num("Actual Loops")) + } + var scans []scanAt + walk(root, func(n Node, path []Node) { + if isScan(n.str("Node Type")) { + scans = append(scans, scanAt{n, path}) + } + if ix := n.str("Index Name"); ix != "" && slices.Contains(c.Defaults.RetiredIndexes, ix) && !slices.Contains(ch.Retired, ix) { + ch.Retired = append(ch.Retired, ix) + } + }) + + for _, d := range exp.Driver { + c.assessDriver(&ch, d, scans) + } + c.assessForbidden(&ch, root, append(append([]NodeRule{}, c.Defaults.Forbid...), exp.Forbid...)) + assessRequired(&ch, root, exp.Require) + if c.Defaults.SortAboveLimit { + c.assessSortAboveLimit(&ch, root, c.sortExempt(exp)) + } + if analyzed { + assessBudget(&ch, ex, exp.Budget) + } + sort.Strings(ch.Drivers) + return ch +} + +func (c *Corpus) assessDriver(ch *Check, d DriverSpec, scans []scanAt) { + nodes := d.Nodes + if len(nodes) == 0 { + nodes = defaultDriverNodes + } + label := d.Relation + aliasStr(d.Alias) + if d.Scope != "" { + label = d.Scope + ": " + label + } + var matched []Node + for _, s := range scans { + if s.n.str("Relation Name") != d.Relation || (d.Alias != "" && s.n.str("Alias") != d.Alias) { + continue + } + if d.Scope != "" && !slices.ContainsFunc(s.path, func(a Node) bool { return a.str("Subplan Name") == d.Scope }) { + continue + } + matched = append(matched, s.n) + } + if len(matched) == 0 { + ch.Failures = append(ch.Failures, Failure{Kind: FailDriver, Msg: "driver: no scan of " + label}) + return + } + var matchedIxs []string + for _, m := range matched { + ixs := indexesOf(m) + matchedIxs = appendNew(matchedIxs, ixs...) + typ := m.str("Node Type") + seen := m.str("Relation Name") + if a := m.str("Alias"); a != "" && a != seen { + seen += "(" + a + ")" + } + ch.Drivers = append(ch.Drivers, fmt.Sprintf("%s:%s:%s", seen, typ, strings.Join(ixs, "+"))) + if !slices.Contains(nodes, typ) { + ch.Failures = append(ch.Failures, Failure{Kind: FailDriver, Indexes: ixs, + Msg: fmt.Sprintf("driver: %s scanned by %s (%s), expected %v", label, typ, strings.Join(ixs, "+"), nodes)}) + continue + } + if !indexAllowed(d, ixs) { + want := d.Index + if len(d.AnyIndex) > 0 { + want = strings.Join(append(nonEmpty(d.Index), d.AnyIndex...), " or ") + } + ch.Failures = append(ch.Failures, Failure{Kind: FailDriver, Indexes: ixs, + Msg: fmt.Sprintf("driver: %s uses %s, expected %s", label, strings.Join(ixs, "+"), want)}) + } + } + for _, frag := range d.IndexCond { + if slices.ContainsFunc(matched, func(m Node) bool { return condHas(m, frag) }) { + continue + } + conds := make([]string, len(matched)) + for i, m := range matched { + conds[i] = condsOf(m) + } + ch.Failures = append(ch.Failures, Failure{Kind: FailIndexCond, Indexes: matchedIxs, + Msg: fmt.Sprintf("index_cond: %q not in %s Index Cond %q", frag, label, strings.Join(conds, " | "))}) + } +} + +// indexAllowed: no index named, the one index named, or one of any_index. +func indexAllowed(d DriverSpec, ixs []string) bool { + if d.Index == "" && len(d.AnyIndex) == 0 { + return true + } + if len(ixs) != 1 { + return false + } + return ixs[0] == d.Index || slices.Contains(d.AnyIndex, ixs[0]) +} + +func (c *Corpus) assessForbidden(ch *Check, root Node, rules []NodeRule) { + walk(root, func(n Node, _ []Node) { + for _, r := range rules { + if n.str("Node Type") != r.Node { + continue + } + if len(r.Relations) > 0 && !slices.Contains(r.Relations, n.str("Relation Name")) { + continue + } + ch.Failures = append(ch.Failures, Failure{Kind: FailForbidden, Indexes: indexesOf(n), + Msg: fmt.Sprintf("forbidden: %s on %s%s", r.Node, n.str("Relation Name"), reasonStr(r.Reason))}) + } + }) +} + +func assessRequired(ch *Check, root Node, rules []NodeRule) { + for _, r := range rules { + found := false + walk(root, func(n Node, _ []Node) { + if n.str("Node Type") != r.Node { + return + } + if len(r.Relations) == 0 || slices.Contains(r.Relations, n.str("Relation Name")) || slices.Contains(r.Relations, n.str("Index Name")) { + found = true + } + }) + if !found { + ch.Failures = append(ch.Failures, Failure{Kind: FailRequired, + Msg: fmt.Sprintf("required: %s on %v not in plan%s", r.Node, r.Relations, reasonStr(r.Reason))}) + } + } +} + +// sortExempt returns the scan patterns of defaults.sort_exempt and exp.sort_exempt. +func (c *Corpus) sortExempt(exp *Expect) []ScanPattern { + var out []ScanPattern + for _, name := range append(append([]string{}, c.Defaults.SortExempt...), exp.SortExempt...) { + out = append(out, c.SortExemptSets[name]...) + } + return out +} + +func (p ScanPattern) matches(n Node) bool { + if n.str("Relation Name") != p.Relation { + return false + } + if p.Index != "" { + ixs := indexesOf(n) + if len(ixs) != 1 || ixs[0] != p.Index { + return false + } + } + return condHas(n, p.IndexCond) +} + +// assessSortAboveLimit fails a Sort or Incremental Sort that a Limit reaches through pass-through nodes when its +// input holds a big-table scan no Limit bounds. A Limit bounds the rows its subtree returns, not the rows a +// SubPlan or InitPlan below it is run for, so the bound stops at those. A scan matching a sort-exempt pattern is +// exempt; the exemption covers that scan only. +func (c *Corpus) assessSortAboveLimit(ch *Check, root Node, exempt []ScanPattern) { + walk(root, func(n Node, path []Node) { + t := n.str("Node Type") + if t != "Sort" && t != "Incremental Sort" || !limitAbove(path) { + return + } + var bad, badIxs []string + var rec func(m Node, bounded bool) + rec = func(m Node, bounded bool) { + if pr := m.str("Parent Relationship"); pr == "SubPlan" || pr == "InitPlan" { + bounded = false + } + mt := m.str("Node Type") + if mt == "Limit" { + bounded = true + } + if isScan(mt) && !bounded && slices.Contains(c.Defaults.BigTables, m.str("Relation Name")) && + !slices.ContainsFunc(exempt, func(p ScanPattern) bool { return p.matches(m) }) { + ixs := indexesOf(m) + bad = append(bad, m.str("Relation Name")+"/"+strings.Join(ixs, "+")) + badIxs = appendNew(badIxs, ixs...) + } + for _, k := range m.kids() { + rec(k, bounded) + } + } + for _, k := range n.kids() { + rec(k, false) + } + if len(bad) > 0 { + ch.Failures = append(ch.Failures, Failure{Kind: FailSortAboveLimit, Indexes: badIxs, + Msg: fmt.Sprintf("sort_above_limit: %s under Limit over unbounded %v", t, bad)}) + } + }) +} + +// limitAbove reports whether the nearest ancestors up to a Limit are all pass-through nodes. +func limitAbove(path []Node) bool { + for i := len(path) - 1; i >= 0; i-- { + pt := path[i].str("Node Type") + if pt == "Limit" { + return true + } + if !slices.Contains(passThrough, pt) { + return false + } + } + return false +} + +// assessBudget checks b against an analyzed run. A plan without the numbers a budget needs is an error, never a +// silent zero: an EXPLAIN without ANALYZE, or a non-zero buffers budget on a plan without BUFFERS. +func assessBudget(ch *Check, ex *Explain, b Budget) { + if !ex.Analyzed { + ch.Failures = append(ch.Failures, Failure{Kind: FailError, + Msg: "error: run is analyzed but the plan has no Execution Time (EXPLAIN without ANALYZE)"}) + return + } + if _, ok := ex.Root["Shared Hit Blocks"].(float64); b.Buffers > 0 && !ok { + ch.Failures = append(ch.Failures, Failure{Kind: FailError, + Msg: "error: buffers budget set but the plan has no Shared Hit Blocks (EXPLAIN without BUFFERS)"}) + b.Buffers = 0 + } + add := func(msg string) { ch.Failures = append(ch.Failures, Failure{Kind: FailBudget, Msg: msg}) } + if b.Buffers > 0 && ch.Buffers > b.Buffers { + add(fmt.Sprintf("budget: buffers %d > %d", ch.Buffers, b.Buffers)) + } + if b.PlanningMs > 0 && ex.PlanningMs > b.PlanningMs { + add(fmt.Sprintf("budget: planning %.2f ms > %.2f", ex.PlanningMs, b.PlanningMs)) + } + if b.ExecMs > 0 && ex.ExecMs > b.ExecMs { + add(fmt.Sprintf("budget: exec %.1f ms > %.1f", ex.ExecMs, b.ExecMs)) + } + if b.Rows > 0 && ch.Rows > b.Rows { + add(fmt.Sprintf("budget: rows %d > %d", ch.Rows, b.Rows)) + } +} + +// Verdict is the outcome of one run. +type Verdict string + +// Verdicts. +const ( + VerdictOK Verdict = "ok" + VerdictDetected Verdict = "DETECTED" + VerdictMissed Verdict = "MISSED" + VerdictFalsePositive Verdict = "FALSE_POSITIVE" + VerdictRetiredIndex Verdict = "RETIRED_INDEX" + VerdictKnownIssue Verdict = "KNOWN_ISSUE" + VerdictSkip Verdict = "SKIP" + VerdictError Verdict = "ERROR" +) + +// Failing reports whether the verdict fails the run: FALSE_POSITIVE, MISSED or ERROR. +func (v Verdict) Failing() bool { + return v == VerdictFalsePositive || v == VerdictMissed || v == VerdictError +} + +// retiredKinds are the failures a retired index can explain. Budget and error failures never are. +var retiredKinds = []FailureKind{FailDriver, FailIndexCond, FailForbidden, FailRequired, FailSortAboveLimit, FailPlanChange} + +// Decide returns the verdict for one run. A run that errored has failures == [{Kind: error}] and is always +// ERROR, whatever its expect_fail or known_issue. +func Decide(expectFail bool, knownIssue string, failures []Failure, retired []string) Verdict { + switch { + case slices.ContainsFunc(failures, func(f Failure) bool { return f.Kind == FailError }): + return VerdictError // an errored run always fails, expect_fail and known_issue included + case expectFail && len(failures) > 0: + return VerdictDetected + case expectFail: + return VerdictMissed + case len(failures) == 0: + return VerdictOK + case allRetired(failures, retired): + return VerdictRetiredIndex + case knownIssue != "": + return VerdictKnownIssue + } + return VerdictFalsePositive +} + +// allRetired: every failure is a structural one about at least one retired index. +func allRetired(failures []Failure, retired []string) bool { + for _, f := range failures { + if !slices.Contains(retiredKinds, f.Kind) { + return false + } + if !slices.ContainsFunc(f.Indexes, func(ix string) bool { return slices.Contains(retired, ix) }) { + return false + } + } + return true +} + +func appendNew(xs []string, add ...string) []string { + for _, a := range add { + if !slices.Contains(xs, a) { + xs = append(xs, a) + } + } + return xs +} + +func nonEmpty(s string) []string { + if s == "" { + return nil + } + return []string{s} +} + +func aliasStr(a string) string { + if a == "" { + return "" + } + return "(" + a + ")" +} + +func reasonStr(r string) string { + if r == "" { + return "" + } + return " (" + r + ")" +} diff --git a/internal/planguard/assert_test.go b/internal/planguard/assert_test.go new file mode 100644 index 0000000..81d5a64 --- /dev/null +++ b/internal/planguard/assert_test.go @@ -0,0 +1,415 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package planguard_test + +import ( + "encoding/json" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/planguard" +) + +var bigTables = []string{"known_routes", "packet_observations", "packets", "packet_sites"} + +func kinds(fs []planguard.Failure) []planguard.FailureKind { + out := []planguard.FailureKind{} + for _, f := range fs { + out = append(out, f.Kind) + } + return out +} + +func wantKinds(t *testing.T, ch planguard.Check, want ...planguard.FailureKind) { + t.Helper() + if want == nil { + want = []planguard.FailureKind{} + } + if got := kinds(ch.Failures); !reflect.DeepEqual(got, want) { + t.Fatalf("failure kinds = %v, want %v; failures: %+v", got, want, ch.Failures) + } +} + +func TestRetiredIndexNeverHidesBudgetFailure(t *testing.T) { + retired := []string{"idx_known_routes_hop_count"} + c := &planguard.Corpus{Defaults: planguard.Defaults{BigTables: bigTables, RetiredIndexes: retired}} + ex := loadPlan(t, "globalhop_rare_retired.json") + exp := &planguard.Expect{ + Driver: []planguard.DriverSpec{{Relation: "known_routes", Index: "idx_known_routes_hop_iata"}}, + Budget: planguard.Budget{Buffers: 1500}, + } + + ch := c.Assess(ex, exp, true) + wantKinds(t, ch, planguard.FailDriver, planguard.FailBudget) + if ch.Buffers != 8902 { + t.Errorf("Buffers = %d, want 8902", ch.Buffers) + } + if !reflect.DeepEqual(ch.Failures[0].Indexes, retired) { + t.Errorf("driver failure indexes = %v, want %v", ch.Failures[0].Indexes, retired) + } + if len(ch.Failures[1].Indexes) != 0 { + t.Errorf("budget failure names indexes %v", ch.Failures[1].Indexes) + } + if !reflect.DeepEqual(ch.Retired, retired) { + t.Errorf("Retired = %v, want %v", ch.Retired, retired) + } + if v := planguard.Decide(false, "", ch.Failures, ch.Retired); v != planguard.VerdictFalsePositive { + t.Errorf("driver + budget: verdict %s, want FALSE_POSITIVE", v) + } + + exp.Budget.Buffers = 10000 + ch = c.Assess(ex, exp, true) + wantKinds(t, ch, planguard.FailDriver) + if v := planguard.Decide(false, "", ch.Failures, ch.Retired); v != planguard.VerdictRetiredIndex { + t.Errorf("driver on a retired index only: verdict %s, want RETIRED_INDEX", v) + } + + // A forbidden Seq Scan next to the retired driver: the real channel_messages Seq Scan from another plan, + // grafted beside the driver. + seq := findNode(loadPlan(t, "cmi_seqscan.json").Root, "Seq Scan") + if seq == nil { + t.Fatal("cmi_seqscan.json has no Seq Scan") + } + ex.Root["Plans"] = append(ex.Root["Plans"].([]any), map[string]any(seq)) + exp.Forbid = []planguard.NodeRule{{Node: "Seq Scan", Relations: []string{"channel_messages"}}} + ch = c.Assess(ex, exp, true) + wantKinds(t, ch, planguard.FailDriver, planguard.FailForbidden) + if v := planguard.Decide(false, "", ch.Failures, ch.Retired); v != planguard.VerdictFalsePositive { + t.Errorf("retired driver + forbidden Seq Scan: verdict %s, want FALSE_POSITIVE", v) + } +} + +func findNode(n planguard.Node, typ string) planguard.Node { + if n["Node Type"] == typ { + return n + } + kids, _ := n["Plans"].([]any) + for _, k := range kids { + if m, ok := k.(map[string]any); ok { + if f := findNode(m, typ); f != nil { + return f + } + } + } + return nil +} + +func sortCorpus() *planguard.Corpus { + return &planguard.Corpus{ + Defaults: planguard.Defaults{BigTables: bigTables, SortAboveLimit: true, SortExempt: []string{"latest_observation"}}, + SortExemptSets: map[string][]planguard.ScanPattern{ + "latest_observation": {{Relation: "packet_observations", IndexCond: "packet_hash = p.packet_hash"}}, + "site_probe": {{Relation: "packet_sites", IndexCond: "packet_hash = ps.packet_hash"}}, + "site_probe_wrong_index": {{ + Relation: "packet_sites", Index: "idx_packet_sites_site", IndexCond: "packet_hash = ps.packet_hash", + }}, + }, + } +} + +func TestSortExemptionScopedToScanPattern(t *testing.T) { + c := sortCorpus() + exempt := &planguard.Expect{SortExempt: []string{"site_probe"}} + + wantKinds(t, c.Assess(loadPlan(t, "bysites_ok.json"), exempt, false)) + + ch := c.Assess(loadPlan(t, "bysites_nolimit.json"), exempt, false) + wantKinds(t, ch, planguard.FailSortAboveLimit) + f := ch.Failures[0] + if !strings.Contains(f.Msg, "packet_sites/idx_packet_sites_site") || strings.Contains(f.Msg, "packet_sites_pkey") { + t.Errorf("msg = %q, want it to name packet_sites/idx_packet_sites_site only", f.Msg) + } + if !reflect.DeepEqual(f.Indexes, []string{"idx_packet_sites_site"}) { + t.Errorf("indexes = %v, want [idx_packet_sites_site]", f.Indexes) + } + + // Without the exemption the correct statement fails on the probe scan: the exemption is what clears it. + ch = c.Assess(loadPlan(t, "bysites_ok.json"), &planguard.Expect{}, false) + wantKinds(t, ch, planguard.FailSortAboveLimit) + if !strings.Contains(ch.Failures[0].Msg, "packet_sites/packet_sites_pkey") { + t.Errorf("msg = %q, want it to name the probe scan", ch.Failures[0].Msg) + } + + // A pattern's index must match too: one naming the wrong index exempts nothing. + ch = c.Assess(loadPlan(t, "bysites_ok.json"), &planguard.Expect{SortExempt: []string{"site_probe_wrong_index"}}, false) + wantKinds(t, ch, planguard.FailSortAboveLimit) + + // The defaults set alone covers the latest-observation LATERAL; dropping it fails that scan. + c.Defaults.SortExempt = nil + ch = c.Assess(loadPlan(t, "bysites_ok.json"), exempt, false) + wantKinds(t, ch, planguard.FailSortAboveLimit) + if !strings.Contains(ch.Failures[0].Msg, "packet_observations/idx_observations_packet") { + t.Errorf("msg = %q, want it to name the latest-observation scan", ch.Failures[0].Msg) + } + + // The rule is off unless the defaults turn it on. + c = sortCorpus() + c.Defaults.SortAboveLimit = false + wantKinds(t, c.Assess(loadPlan(t, "bysites_nolimit.json"), &planguard.Expect{}, false)) +} + +func TestAssertDriverScopedByCTE(t *testing.T) { + c := &planguard.Corpus{} + ex := loadPlan(t, "packets_page.json") + d := planguard.DriverSpec{Scope: "CTE page", Relation: "packets", Index: "idx_packets_last_heard_hash"} + + ch := c.Assess(ex, &planguard.Expect{Driver: []planguard.DriverSpec{d}}, true) + wantKinds(t, ch) + if want := []string{"packets(p_1):Index Only Scan:idx_packets_last_heard_hash"}; !reflect.DeepEqual(ch.Drivers, want) { + t.Errorf("Drivers = %v, want %v", ch.Drivers, want) + } + + d.Scope = "" + ch = c.Assess(ex, &planguard.Expect{Driver: []planguard.DriverSpec{d}}, true) + wantKinds(t, ch, planguard.FailDriver) + if !reflect.DeepEqual(ch.Failures[0].Indexes, []string{"packets_pkey"}) { + t.Errorf("indexes = %v, want [packets_pkey]", ch.Failures[0].Indexes) + } + + d.Scope = "CTE other" + ch = c.Assess(ex, &planguard.Expect{Driver: []planguard.DriverSpec{d}}, true) + wantKinds(t, ch, planguard.FailDriver) + if !strings.Contains(ch.Failures[0].Msg, "no scan") { + t.Errorf("msg = %q, want a no-scan failure", ch.Failures[0].Msg) + } +} + +func TestIndexCondModeNeutral(t *testing.T) { + c := &planguard.Corpus{} + for _, f := range []string{"routes_byiata_custom.json", "routes_byiata_generic.json"} { + ex := loadPlan(t, f) + d := planguard.DriverSpec{Relation: "known_routes", Index: "idx_known_routes_iata_keyset", IndexCond: []string{"iata >=", "iata <="}} + wantKinds(t, c.Assess(ex, &planguard.Expect{Driver: []planguard.DriverSpec{d}}, true)) + + d.IndexCond = []string{"iata >=", "last_seen >"} + ch := c.Assess(ex, &planguard.Expect{Driver: []planguard.DriverSpec{d}}, true) + wantKinds(t, ch, planguard.FailIndexCond) + if !reflect.DeepEqual(ch.Failures[0].Indexes, []string{"idx_known_routes_iata_keyset"}) { + t.Errorf("%s: indexes = %v", f, ch.Failures[0].Indexes) + } + } +} + +func TestIndexCondReadsBitmapChildren(t *testing.T) { + c := &planguard.Corpus{} + d := planguard.DriverSpec{ + Relation: "packets", Alias: "p", Nodes: []string{"Bitmap Heap Scan"}, + Index: "idx_packets_scope_last_heard", IndexCond: []string{"scope_id ="}, + } + ch := c.Assess(loadPlan(t, "cmi_seqscan.json"), &planguard.Expect{Driver: []planguard.DriverSpec{d}}, true) + wantKinds(t, ch) + if want := []string{"packets(p):Bitmap Heap Scan:idx_packets_scope_last_heard"}; !reflect.DeepEqual(ch.Drivers, want) { + t.Errorf("Drivers = %v, want %v", ch.Drivers, want) + } + + d.Nodes = nil // default: Index Scan or Index Only Scan + wantKinds(t, c.Assess(loadPlan(t, "cmi_seqscan.json"), &planguard.Expect{Driver: []planguard.DriverSpec{d}}, true), planguard.FailDriver) +} + +func TestAnyIndexAllowsEmptyPkeyRange(t *testing.T) { + c := &planguard.Corpus{} + ex := loadPlan(t, "adverts_past_end.json") + d := planguard.DriverSpec{Relation: "packet_observations", AnyIndex: []string{"idx_observations_adverts", "packet_observations_pkey"}} + ch := c.Assess(ex, &planguard.Expect{Driver: []planguard.DriverSpec{d}, Budget: planguard.Budget{Buffers: 10}}, true) + wantKinds(t, ch) + if ch.Buffers != 3 { + t.Errorf("Buffers = %d, want 3", ch.Buffers) + } + + d = planguard.DriverSpec{Relation: "packet_observations", Index: "idx_observations_adverts"} + ch = c.Assess(ex, &planguard.Expect{Driver: []planguard.DriverSpec{d}}, true) + wantKinds(t, ch, planguard.FailDriver) + if !reflect.DeepEqual(ch.Failures[0].Indexes, []string{"packet_observations_pkey"}) { + t.Errorf("indexes = %v", ch.Failures[0].Indexes) + } +} + +func TestRowsBudget(t *testing.T) { + c := &planguard.Corpus{} + ex := loadPlan(t, "packets_page.json") // root Actual Rows 51 × Actual Loops 1 + ch := c.Assess(ex, &planguard.Expect{Budget: planguard.Budget{Rows: 50}}, true) + wantKinds(t, ch, planguard.FailBudget) + if ch.Rows != 51 || !strings.Contains(ch.Failures[0].Msg, "rows 51 > 50") { + t.Errorf("Rows = %d, msg %q", ch.Rows, ch.Failures[0].Msg) + } + wantKinds(t, c.Assess(ex, &planguard.Expect{Budget: planguard.Budget{Rows: 51}}, true)) + + // Budgets are checked only when the run was analyzed. + ch = c.Assess(ex, &planguard.Expect{Budget: planguard.Budget{Rows: 50, Buffers: 1, PlanningMs: 0.001, ExecMs: 0.001}}, false) + wantKinds(t, ch) + if ch.Rows != 0 || ch.Buffers != 0 { + t.Errorf("plain run reports Rows %d, Buffers %d", ch.Rows, ch.Buffers) + } + + ch = c.Assess(ex, &planguard.Expect{Budget: planguard.Budget{PlanningMs: 0.001, ExecMs: 0.001}}, true) + wantKinds(t, ch, planguard.FailBudget, planguard.FailBudget) +} + +func TestForbiddenAndRequiredNodes(t *testing.T) { + c := &planguard.Corpus{Defaults: planguard.Defaults{ + Forbid: []planguard.NodeRule{{Node: "Seq Scan", Relations: bigTables, Reason: "never scan a big table"}}, + }} + ex := loadPlan(t, "cmi_seqscan.json") + wantKinds(t, c.Assess(ex, &planguard.Expect{}, true)) + + ch := c.Assess(ex, &planguard.Expect{Forbid: []planguard.NodeRule{{Node: "Seq Scan", Relations: []string{"channel_messages"}}}}, true) + wantKinds(t, ch, planguard.FailForbidden) + if !strings.Contains(ch.Failures[0].Msg, "Seq Scan on channel_messages") { + t.Errorf("msg = %q", ch.Failures[0].Msg) + } + + exp := &planguard.Expect{Require: []planguard.NodeRule{ + {Node: "Bitmap Index Scan", Relations: []string{"idx_packets_scope_last_heard"}}, + {Node: "Hash Join"}, + }} + wantKinds(t, c.Assess(ex, exp, true)) + exp.Require = append(exp.Require, planguard.NodeRule{Node: "Merge Join"}) + wantKinds(t, c.Assess(ex, exp, true), planguard.FailRequired) +} + +func TestDecide(t *testing.T) { + retired := []string{"idx_old"} + f := func(k planguard.FailureKind, ix ...string) planguard.Failure { + return planguard.Failure{Kind: k, Msg: string(k), Indexes: ix} + } + cases := []struct { + name string + expectFail bool + known string + failures []planguard.Failure + want planguard.Verdict + }{ + {"expect-fail detected", true, "", []planguard.Failure{f(planguard.FailBudget)}, planguard.VerdictDetected}, + {"expect-fail errored", true, "", []planguard.Failure{f(planguard.FailError)}, planguard.VerdictError}, + {"expect-fail errored, known issue", true, "x", []planguard.Failure{f(planguard.FailError)}, planguard.VerdictError}, + {"error, known issue", false, "x", []planguard.Failure{f(planguard.FailError)}, planguard.VerdictError}, + {"retired + error, known issue", false, "x", []planguard.Failure{f(planguard.FailDriver, "idx_old"), f(planguard.FailError)}, planguard.VerdictError}, + {"expect-fail missed", true, "", nil, planguard.VerdictMissed}, + {"clean", false, "", nil, planguard.VerdictOK}, + {"clean known issue", false, "x", nil, planguard.VerdictOK}, + {"retired structural", false, "", []planguard.Failure{ + f(planguard.FailDriver, "idx_old"), f(planguard.FailIndexCond, "idx_old"), f(planguard.FailForbidden, "idx_old"), + f(planguard.FailRequired, "idx_old"), f(planguard.FailSortAboveLimit, "idx_new", "idx_old"), f(planguard.FailPlanChange, "idx_old"), + }, planguard.VerdictRetiredIndex}, + {"retired + budget", false, "", []planguard.Failure{f(planguard.FailDriver, "idx_old"), f(planguard.FailBudget)}, planguard.VerdictFalsePositive}, + {"retired + other index", false, "", []planguard.Failure{f(planguard.FailDriver, "idx_old"), f(planguard.FailDriver, "idx_new")}, planguard.VerdictFalsePositive}, + {"retired + no index", false, "", []planguard.Failure{f(planguard.FailDriver, "idx_old"), f(planguard.FailRequired)}, planguard.VerdictFalsePositive}, + {"retired + error", false, "", []planguard.Failure{f(planguard.FailDriver, "idx_old"), f(planguard.FailError)}, planguard.VerdictError}, + {"retired + budget, known issue", false, "x", []planguard.Failure{f(planguard.FailDriver, "idx_old"), f(planguard.FailBudget)}, planguard.VerdictKnownIssue}, + {"known issue", false, "x", []planguard.Failure{f(planguard.FailDriver, "idx_new")}, planguard.VerdictKnownIssue}, + {"false positive", false, "", []planguard.Failure{f(planguard.FailDriver, "idx_new")}, planguard.VerdictFalsePositive}, + {"error", false, "", []planguard.Failure{f(planguard.FailError)}, planguard.VerdictError}, + } + for _, tc := range cases { + if got := planguard.Decide(tc.expectFail, tc.known, tc.failures, retired); got != tc.want { + t.Errorf("%s: Decide = %s, want %s", tc.name, got, tc.want) + } + } + // No retired list: nothing is RETIRED_INDEX. + if got := planguard.Decide(false, "", []planguard.Failure{f(planguard.FailDriver, "idx_old")}, nil); got != planguard.VerdictFalsePositive { + t.Errorf("no retired list: Decide = %s, want FALSE_POSITIVE", got) + } +} + +func TestVerdictFailing(t *testing.T) { + failing := map[planguard.Verdict]bool{ + planguard.VerdictFalsePositive: true, planguard.VerdictMissed: true, planguard.VerdictError: true, + } + for _, v := range []planguard.Verdict{ + planguard.VerdictOK, planguard.VerdictDetected, planguard.VerdictMissed, planguard.VerdictFalsePositive, + planguard.VerdictRetiredIndex, planguard.VerdictKnownIssue, planguard.VerdictSkip, planguard.VerdictError, + } { + if v.Failing() != failing[v] { + t.Errorf("%s.Failing() = %v, want %v", v, v.Failing(), failing[v]) + } + } +} + +func TestSubplanCondsStayOffParentScan(t *testing.T) { + c := &planguard.Corpus{} + ex := loadPlan(t, "cmi_seqscan.json") + heap := findNode(ex.Root, "Bitmap Heap Scan") + if heap == nil { + t.Fatal("cmi_seqscan.json has no Bitmap Heap Scan") + } + // A per-row bitmap probe run by the scan's filter: a SubPlan child with its own Bitmap Index Scan. + probe := map[string]any{ + "Node Type": "Bitmap Heap Scan", "Parent Relationship": "SubPlan", "Subplan Name": "SubPlan 1", + "Relation Name": "packet_observations", "Alias": "po", + "Recheck Cond": "(observer_id = p.observer_id)", + "Plans": []any{map[string]any{ + "Node Type": "Bitmap Index Scan", "Parent Relationship": "Outer", + "Index Name": "idx_observations_observer", "Index Cond": "(observer_id = p.observer_id)", + }}, + } + heap["Plans"] = append(heap["Plans"].([]any), probe) + + d := planguard.DriverSpec{ + Relation: "packets", Alias: "p", Nodes: []string{"Bitmap Heap Scan"}, + Index: "idx_packets_scope_last_heard", IndexCond: []string{"scope_id ="}, + } + ch := c.Assess(ex, &planguard.Expect{Driver: []planguard.DriverSpec{d}}, true) + wantKinds(t, ch) + if want := []string{"packets(p):Bitmap Heap Scan:idx_packets_scope_last_heard"}; !reflect.DeepEqual(ch.Drivers, want) { + t.Errorf("Drivers = %v, want %v (the probe's index is not the outer scan's)", ch.Drivers, want) + } + + d.IndexCond = []string{"observer_id ="} // only in the probe + ch = c.Assess(ex, &planguard.Expect{Driver: []planguard.DriverSpec{d}}, true) + wantKinds(t, ch, planguard.FailIndexCond) + if !reflect.DeepEqual(ch.Failures[0].Indexes, []string{"idx_packets_scope_last_heard"}) { + t.Errorf("indexes = %v, want only the outer scan's", ch.Failures[0].Indexes) + } + + // A retired index used only by the probe is attributed to the probe, not the outer scan. + c.Defaults.RetiredIndexes = []string{"idx_observations_observer"} + d.IndexCond = nil + d.Index = "idx_packets_other" + ch = c.Assess(ex, &planguard.Expect{Driver: []planguard.DriverSpec{d}}, true) + wantKinds(t, ch, planguard.FailDriver) + if v := planguard.Decide(false, "", ch.Failures, ch.Retired); v != planguard.VerdictFalsePositive { + t.Errorf("outer driver failure with a retired probe index: verdict %s, want FALSE_POSITIVE", v) + } +} + +func TestBudgetNeedsAnalyzeAndBuffers(t *testing.T) { + c := &planguard.Corpus{} + raw, err := os.ReadFile(filepath.Join("testdata", "plans", "packets_page.json")) + if err != nil { + t.Fatal(err) + } + var arr []map[string]any + if err := json.Unmarshal(raw, &arr); err != nil { + t.Fatal(err) + } + delete(arr[0], "Execution Time") + plain, _ := json.Marshal(arr) + ex, err := planguard.ParseExplain(plain) + if err != nil { + t.Fatal(err) + } + ch := c.Assess(ex, &planguard.Expect{}, true) + wantKinds(t, ch, planguard.FailError) + if !strings.Contains(ch.Failures[0].Msg, "ANALYZE") { + t.Errorf("msg = %q", ch.Failures[0].Msg) + } + wantKinds(t, c.Assess(ex, &planguard.Expect{Budget: planguard.Budget{Buffers: 1}}, false)) + + ex = loadPlan(t, "packets_page.json") + delete(ex.Root, "Shared Hit Blocks") + delete(ex.Root, "Shared Read Blocks") + ch = c.Assess(ex, &planguard.Expect{Budget: planguard.Budget{Buffers: 1, Rows: 50}}, true) + wantKinds(t, ch, planguard.FailError, planguard.FailBudget) + if !strings.Contains(ch.Failures[0].Msg, "BUFFERS") || !strings.Contains(ch.Failures[1].Msg, "rows") { + t.Errorf("failures = %+v", ch.Failures) + } + if v := planguard.Decide(false, "x", ch.Failures, nil); v != planguard.VerdictError { + t.Errorf("verdict %s, want ERROR", v) + } + // No buffers budget: a plan without BUFFERS is fine. + wantKinds(t, c.Assess(ex, &planguard.Expect{}, true)) +} diff --git a/internal/planguard/casefile.go b/internal/planguard/casefile.go new file mode 100644 index 0000000..03791c4 --- /dev/null +++ b/internal/planguard/casefile.go @@ -0,0 +1,151 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package planguard + +import ( + "bufio" + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "unicode/utf8" +) + +// RowSet is one statement result in the server's text format. A nil value is NULL. +type RowSet struct { + Columns []string + Rows [][]*string +} + +// caseHeader is a case file's first line. Rows is the row count, so a file cut at a line boundary is an error too. +type caseHeader struct { + Columns []string `json:"columns"` + Rows *int `json:"rows"` +} + +// WriteCaseFile writes rs as JSON Lines: {"columns":[…],"rows":N}, then one JSON array per row (a string per value, +// null for NULL). Every value round-trips unchanged, newlines and quotes included. Text that is not UTF-8 cannot be +// written losslessly as JSON, so it is an error, as is a row whose arity differs from the columns. +func WriteCaseFile(path string, rs RowSet) error { + var b bytes.Buffer + enc := json.NewEncoder(&b) + enc.SetEscapeHTML(false) + n := len(rs.Rows) + cols := rs.Columns + if cols == nil { + cols = []string{} + } + for _, c := range cols { + if !utf8.ValidString(c) { + return fmt.Errorf("%s: column name %q is not UTF-8", path, c) + } + } + if err := enc.Encode(caseHeader{Columns: cols, Rows: &n}); err != nil { + return fmt.Errorf("%s: %w", path, err) + } + for i, row := range rs.Rows { + if len(row) != len(cols) { + return fmt.Errorf("%s: row %d has %d values for %d columns", path, i+1, len(row), len(cols)) + } + for j, v := range row { + if v != nil && !utf8.ValidString(*v) { + return fmt.Errorf("%s: row %d column %s is not UTF-8", path, i+1, cols[j]) + } + } + if err := enc.Encode(row); err != nil { + return fmt.Errorf("%s: row %d: %w", path, i+1, err) + } + } + return os.WriteFile(path, b.Bytes(), 0o644) +} + +// ReadCaseFile reads a file written by WriteCaseFile. Bad JSON, a missing or malformed header, a value that is +// neither a string nor null, a row whose arity differs, a line with no newline, or a row count that differs from +// the header's is an error naming the line: a damaged file never loses rows silently. +func ReadCaseFile(path string) (RowSet, error) { + f, err := os.Open(path) + if err != nil { + return RowSet{}, err + } + defer f.Close() + r := bufio.NewReader(f) + var rs RowSet + want := -1 + for line := 1; ; line++ { + raw, err := r.ReadBytes('\n') + if errors.Is(err, io.EOF) { + if len(raw) > 0 { + return RowSet{}, fmt.Errorf("%s: line %d: no newline at the end (truncated)", path, line) + } + if want < 0 { + return RowSet{}, fmt.Errorf("%s: empty file, no header", path) + } + if len(rs.Rows) != want { + return RowSet{}, fmt.Errorf("%s: %d rows, the header says %d (truncated)", path, len(rs.Rows), want) + } + return rs, nil + } + if err != nil { + return RowSet{}, fmt.Errorf("%s: line %d: %w", path, line, err) + } + if line == 1 { + if !utf8.Valid(raw) { + return RowSet{}, fmt.Errorf("%s: line 1: not UTF-8", path) + } + var h caseHeader + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.DisallowUnknownFields() + if err := dec.Decode(&h); err != nil || h.Columns == nil || h.Rows == nil || *h.Rows < 0 || dec.More() { + return RowSet{}, fmt.Errorf("%s: line 1: want the header {\"columns\":[…],\"rows\":N} (%v)", path, err) + } + rs.Columns, want = h.Columns, *h.Rows + continue + } + if !utf8.Valid(raw) { + return RowSet{}, fmt.Errorf("%s: line %d: not UTF-8", path, line) + } + row, err := decodeRow(raw, len(rs.Columns)) + if err != nil { + return RowSet{}, fmt.Errorf("%s: line %d: %w", path, line, err) + } + if len(rs.Rows) == want { + return RowSet{}, fmt.Errorf("%s: line %d: more rows than the header's %d", path, line, want) + } + rs.Rows = append(rs.Rows, row) + } +} + +func decodeRow(raw []byte, arity int) ([]*string, error) { + var cells []json.RawMessage + dec := json.NewDecoder(bytes.NewReader(raw)) + if err := dec.Decode(&cells); err != nil { + return nil, fmt.Errorf("bad JSON: %w", err) + } + if dec.More() { + return nil, errors.New("bad JSON: more than one value on the line") + } + if cells == nil { + return nil, errors.New("want a JSON array") + } + if len(cells) != arity { + return nil, fmt.Errorf("%d values for %d columns", len(cells), arity) + } + row := make([]*string, len(cells)) + for i, c := range cells { + if string(c) == "null" { + continue + } + var v string + if len(c) == 0 || c[0] != '"' { + return nil, fmt.Errorf("value %d is %s, want a string or null", i+1, c) + } + if err := json.Unmarshal(c, &v); err != nil { + return nil, fmt.Errorf("value %d: %w", i+1, err) + } + row[i] = &v + } + return row, nil +} diff --git a/internal/planguard/corpus.go b/internal/planguard/corpus.go new file mode 100644 index 0000000..900b6c4 --- /dev/null +++ b/internal/planguard/corpus.go @@ -0,0 +1,781 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package planguard + +import ( + "bytes" + "errors" + "fmt" + "io" + "io/fs" + "path" + "regexp" + "slices" + "sort" + "strings" + + "github.com/MeshCore-Beacon/beacon-server/db/planpolicy" + "gopkg.in/yaml.v3" +) + +// Corpus is the plan corpus: which statements to plan, with which parameters, and what each plan must look like. +type Corpus struct { + Defaults Defaults `yaml:"defaults"` + Tiers map[string]TierSettings `yaml:"tiers"` // fixture | local | prod + Groups map[string]Group `yaml:"groups"` + Statements map[string]*Statement `yaml:"statements"` + SortExemptSets map[string][]ScanPattern `yaml:"sort_exempt_sets"` + AdmissionProfiles map[string][]string `yaml:"admission_profiles"` + Resolvers map[string]string `yaml:"resolvers"` + Entries []*Entry `yaml:"entries"` + + byID map[string]*Entry +} + +// TierSettings applies to every run on one tier. +type TierSettings struct { + Settings map[string]string `yaml:"settings"` // SET LOCAL per run, e.g. random_page_cost + SearchPath string `yaml:"search_path"` // default when a group names none for this tier +} + +// Defaults apply to every entry. +type Defaults struct { + BigTables []string `yaml:"big_tables"` + Executions int `yaml:"executions"` // 6 + TimeoutS int `yaml:"timeout_s"` // 60 + Forbid []NodeRule `yaml:"forbid"` + SortAboveLimit bool `yaml:"forbid_sort_above_limit"` + SortExempt []string `yaml:"sort_exempt"` // names of sort_exempt_sets applied to every entry + RetiredIndexes []string `yaml:"retired_indexes"` +} + +// Group sets a search_path per tier for the statements in it. +type Group struct { + SearchPath map[string]string `yaml:"search_path"` +} + +// Statement is one statement text and how it is planned in production. +type Statement struct { + SQLC string `yaml:"sqlc"` // sqlc statement name; text from db/sqlc + File string `yaml:"file"` // frozen text, relative to the corpus file + DerivedFrom string `yaml:"derived_from"` // sqlc name whose text must contain File's body (read-only part of a DML) + Kind string `yaml:"kind"` // select (default) | dml + Group string `yaml:"group"` + Plan string `yaml:"plan"` // cached | custom | maint | ingest | legacy | harness + Admission string `yaml:"admission"` // admission profile; required when plan is cached + Pending string `yaml:"pending"` // feature name; the statement is not merged yet + SQL string `yaml:"-"` +} + +// ScanPattern matches a scan node by relation, optional index and an Index Cond fragment. +type ScanPattern struct { + Relation string `yaml:"relation"` + Index string `yaml:"index"` // optional + IndexCond string `yaml:"index_cond"` // fragment, mode-neutral + Reason string `yaml:"reason"` +} + +// NodeRule names a plan node type on some relations. +type NodeRule struct { + Node string `yaml:"node"` + Relations []string `yaml:"relations"` // "big" expands to defaults.big_tables + Reason string `yaml:"reason"` +} + +// DriverSpec is the scan that must drive a plan. +type DriverSpec struct { + Scope string `yaml:"scope"` // Subplan Name, e.g. "CTE page" + Relation string `yaml:"relation"` + Alias string `yaml:"alias"` + Index string `yaml:"index"` + AnyIndex []string `yaml:"any_index"` // alternatives (empty pkey range for past-the-end cursors) + Nodes []string `yaml:"nodes"` + IndexCond []string `yaml:"index_cond"` +} + +// Budget bounds one run; a zero field is not checked. +type Budget struct { + Buffers int64 `yaml:"buffers"` + PlanningMs float64 `yaml:"planning_ms"` + ExecMs float64 `yaml:"exec_ms"` + Rows int64 `yaml:"rows"` +} + +// Expect is what an entry's plan must satisfy. +type Expect struct { + Driver []DriverSpec `yaml:"driver"` + Require []NodeRule `yaml:"require"` + Forbid []NodeRule `yaml:"forbid"` + SortExempt []string `yaml:"sort_exempt"` // set names, added to defaults.sort_exempt + Budget Budget `yaml:"budget"` +} + +// ModeOverride skips one plan mode, relaxes its expectations, or marks its failures as a known issue. +type ModeOverride struct { + Skip string `yaml:"skip"` + DegradeTo *Expect `yaml:"degrade_to"` + KnownIssue string `yaml:"known_issue"` // this mode's failures read KNOWN_ISSUE; errors still fail + Reason string `yaml:"reason"` +} + +// Entry is one statement with one set of parameters. +type Entry struct { + ID, Stmt, Shape string + Shapes []string `yaml:"shapes"` // shape-class tags (admission) + Params []any `yaml:"params"` + Tiers []string `yaml:"tiers"` + Modes []string `yaml:"modes"` + Executions *int `yaml:"executions"` + TimeoutS int `yaml:"timeout_s"` + Analyze *bool `yaml:"analyze"` + Inherit string `yaml:"inherit"` + Expect Expect `yaml:"expect"` + Mode map[string]*ModeOverride `yaml:"mode"` + ExpectFail bool `yaml:"expect_fail"` + KnownIssue string `yaml:"known_issue"` + RequiresIndexes []string `yaml:"requires_indexes"` // indexes on big tables the case needs; SKIP when absent + Baseline string `yaml:"baseline"` // "" or "unstable: " + Note string `yaml:"note"` +} + +// MissingIndex returns the first of e's required indexes absent from indexes, the run's index list on the big +// tables, or "" when all are present. A nil list is unknown and misses nothing. +func (e *Entry) MissingIndex(indexes []string) string { + if indexes == nil { + return "" + } + for _, ix := range e.RequiresIndexes { + if !slices.Contains(indexes, ix) { + return ix + } + } + return "" +} + +func plainIdent(s string) bool { + if s == "" { + return false + } + for i := 0; i < len(s); i++ { + if !isIdent(s[i]) { + return false + } + } + return s[0] < '0' || s[0] > '9' +} + +// KnownIssueIn returns the known issue that covers e's failures in mode: the mode override's, else the entry's. +func (e *Entry) KnownIssueIn(mode string) string { + if o := e.Mode[mode]; o != nil && o.KnownIssue != "" { + return o.KnownIssue + } + return e.KnownIssue +} + +// Plan modes: how a statement's plan is chosen (plan_cache_mode). +const ( + ModeCustom = "custom" + ModeGeneric = "generic" + ModeAuto = "auto" +) + +var ( + validModes = map[string]bool{ModeCustom: true, ModeGeneric: true, ModeAuto: true} + validTiers = map[string]bool{"fixture": true, "local": true, "prod": true} + validKinds = map[string]bool{"select": true, "dml": true} + + // classModes are the modes an entry runs when it names none, by its statement's plan class. + classModes = map[string][]string{ + "cached": {ModeCustom, ModeGeneric, ModeAuto}, // must pass every layout + "custom": {ModeCustom}, // sent unnamed; no layout plans it generic + "maint": {ModeCustom}, // maint is force_custom_plan + "ingest": {ModeAuto}, // ingest stays auto + "legacy": {ModeCustom, ModeGeneric, ModeAuto}, // today's statements + "harness": {ModeCustom, ModeGeneric, ModeAuto}, // planted regressions + } + + componentSchema = regexp.MustCompile(`\bt_[a-z0-9_]+\.`) +) + +// LoadCorpus reads the corpus at path from fsys (strict: unknown keys fail) and fills every Statement.SQL: +// sqlc → sqlcText[name]; file/derived → fsys file. A sqlc name missing from sqlcText is an error unless the +// statement is pending. It resolves inherit, validates every reference and the default modes per plan class. +func LoadCorpus(fsys fs.FS, p string, sqlcText map[string]string) (*Corpus, error) { + raw, err := fs.ReadFile(fsys, p) + if err != nil { + return nil, fmt.Errorf("read corpus: %w", err) + } + var c Corpus + dec := yaml.NewDecoder(bytes.NewReader(raw)) + dec.KnownFields(true) + if err := dec.Decode(&c); err != nil { + if errors.Is(err, io.EOF) { + return nil, fmt.Errorf("%s: empty corpus", p) + } + return nil, fmt.Errorf("%s: %w", p, err) + } + var extra yaml.Node + if err := dec.Decode(&extra); !errors.Is(err, io.EOF) { + return nil, fmt.Errorf("%s: more than one YAML document", p) + } + if err := c.load(fsys, path.Dir(p), sqlcText); err != nil { + return nil, fmt.Errorf("%s: %w", p, err) + } + return &c, nil +} + +// Entry returns the entry with this id, or nil. +func (c *Corpus) Entry(id string) *Entry { return c.byID[id] } + +// ModesOf returns the plan modes e runs in: its own modes, else its statement's plan class defaults. +func (c *Corpus) ModesOf(e *Entry) []string { + if len(e.Modes) > 0 { + return e.Modes + } + if st := c.Statements[e.Stmt]; st != nil { + return classModes[st.Plan] + } + return nil +} + +func (c *Corpus) load(fsys fs.FS, dir string, sqlcText map[string]string) error { + if err := c.checkTopLevel(); err != nil { + return err + } + if err := c.loadStatements(fsys, dir, sqlcText); err != nil { + return err + } + c.byID = make(map[string]*Entry, len(c.Entries)) + for i, e := range c.Entries { + if e == nil || e.ID == "" { + return fmt.Errorf("entry %d: missing id", i) + } + if err := CheckName(e.ID); err != nil { + return fmt.Errorf("entry %d: id: %w", i, err) + } + if _, dup := c.byID[e.ID]; dup { + return fmt.Errorf("duplicate entry id %q", e.ID) + } + c.byID[e.ID] = e + } + if err := c.resolveInherit(); err != nil { + return err + } + for _, e := range c.Entries { + if err := c.checkEntry(e); err != nil { + return fmt.Errorf("entry %q: %w", e.ID, err) + } + } + return nil +} + +func (c *Corpus) checkTopLevel() error { + for name := range c.Tiers { + if !validTiers[name] { + return fmt.Errorf("tiers: unknown tier %q (want fixture, local or prod)", name) + } + } + for _, g := range sortedKeys(c.Groups) { + for tier := range c.Groups[g].SearchPath { + if !validTiers[tier] { + return fmt.Errorf("group %q: unknown tier %q", g, tier) + } + } + } + for _, name := range sortedKeys(c.SortExemptSets) { + for i, sp := range c.SortExemptSets[name] { + if sp.Relation == "" { + return fmt.Errorf("sort_exempt_sets %q[%d]: missing relation", name, i) + } + if err := checkModeNeutral(sp.IndexCond); err != nil { + return fmt.Errorf("sort_exempt_sets %q[%d]: %w", name, i, err) + } + } + } + for _, name := range c.Defaults.SortExempt { + if _, ok := c.SortExemptSets[name]; !ok { + return fmt.Errorf("defaults: sort_exempt %q is not in sort_exempt_sets", name) + } + } + for _, tier := range sortedKeys(c.Tiers) { + if err := checkSettings(c.Tiers[tier].Settings); err != nil { + return fmt.Errorf("tiers: %s: %w", tier, err) + } + } + for _, name := range sortedKeys(c.Resolvers) { + if err := CheckName(name); err != nil { + return fmt.Errorf("resolver name: %w", err) + } + if m := componentSchema.FindString(c.Resolvers[name]); m != "" { + return fmt.Errorf("resolver %q reads schema %q; resolvers read Beacon tables only", name, m) + } + if err := checkStatementText(c.Resolvers[name]); err != nil { + return fmt.Errorf("resolver %q: %w", name, err) + } + } + var err error + if c.Defaults.Forbid, err = c.expandBig(c.Defaults.Forbid); err != nil { + return fmt.Errorf("defaults: %w", err) + } + return nil +} + +func (c *Corpus) loadStatements(fsys fs.FS, dir string, sqlcText map[string]string) error { + for _, name := range sortedKeys(c.Statements) { + st := c.Statements[name] + if st == nil { + return fmt.Errorf("statement %q: empty", name) + } + if err := c.loadStatement(fsys, dir, st, sqlcText); err != nil { + return fmt.Errorf("statement %q: %w", name, err) + } + } + return nil +} + +func (c *Corpus) loadStatement(fsys fs.FS, dir string, st *Statement, sqlcText map[string]string) error { + if st.Kind == "" { + st.Kind = "select" + } + if !validKinds[st.Kind] { + return fmt.Errorf("unknown kind %q (want select or dml)", st.Kind) + } + if _, ok := classModes[st.Plan]; !ok { + return fmt.Errorf("unknown plan %q (want cached, custom, maint, ingest, legacy or harness)", st.Plan) + } + if st.SQLC != "" && planpolicy.Legacy[st.SQLC] && st.Plan != "legacy" { + return fmt.Errorf("sqlc statement %q is in planpolicy.Legacy, so its plan is legacy, not %q", st.SQLC, st.Plan) + } + if st.Plan == "cached" { + if st.Admission == "" { + return errors.New("plan cached needs an admission profile") + } + if _, ok := c.AdmissionProfiles[st.Admission]; !ok { + return fmt.Errorf("admission profile %q is not in admission_profiles", st.Admission) + } + } else if st.Admission != "" { + return fmt.Errorf("admission %q is only for plan cached", st.Admission) + } + if st.Group != "" { + if _, ok := c.Groups[st.Group]; !ok { + return fmt.Errorf("unknown group %q", st.Group) + } + } + switch { + case st.SQLC != "" && st.File != "": + return errors.New("sqlc and file are exclusive") + case st.SQLC != "": + if st.DerivedFrom != "" { + return errors.New("derived_from needs a file, not sqlc") + } + text, ok := sqlcText[st.SQLC] + if !ok && st.Pending == "" { + return fmt.Errorf("sqlc statement %q does not exist (mark it pending if it is not merged yet)", st.SQLC) + } + st.SQL = text + case st.File != "": + body, err := fs.ReadFile(fsys, path.Join(dir, st.File)) + if err != nil { + return fmt.Errorf("file: %w", err) + } + st.SQL = string(body) + if Normalize(st.SQL) == "" { + return fmt.Errorf("file %s holds no statement", st.File) + } + if st.DerivedFrom != "" { + src, ok := sqlcText[st.DerivedFrom] + switch { + case !ok && st.Pending == "": + return fmt.Errorf("derived_from sqlc statement %q does not exist", st.DerivedFrom) + case ok && !DerivedBodyMatches(src, st.SQL): + return fmt.Errorf("file %s is no longer part of sqlc statement %q (lock clause aside)", st.File, st.DerivedFrom) + } + } + default: + return errors.New("needs sqlc or file") + } + if st.SQL != "" { + if err := checkStatementText(st.SQL); err != nil { + return err + } + } + if st.Kind == "select" && st.SQL != "" { + if kw := dataModifying(st.SQL); kw != "" { + return fmt.Errorf("kind select, but the text runs %s; data-modifying statements are kind dml (plain EXPLAIN only)", kw) + } + } + return nil +} + +var nameRe = regexp.MustCompile(`^[A-Za-z0-9_.-]+$`) + +// CheckName validates an entry id or resolver name. Both end up in psql \echo lines and in file names, so they are +// limited to letters, digits, '_', '.' and '-', with no ".." (no path escapes, no psql backticks or variables). +func CheckName(name string) error { + if !nameRe.MatchString(name) || strings.Contains(name, "..") { + return fmt.Errorf("%q must match %s and hold no \"..\"", name, nameRe) + } + return nil +} + +// dataModifying returns the data-modifying keyword (INSERT, UPDATE, DELETE or MERGE) a statement runs, or "". +// Best effort: it checks the leading keyword and, under a WITH, every keyword at the top level or one level of +// parentheses deep (the CTE bodies). Literals, quoted identifiers and comments are skipped; "FOR [NO KEY] UPDATE" +// is a lock clause, not a write. +func dataModifying(sql string) string { + words, depths := sqlWords(sql) + if len(words) == 0 { + return "" + } + isDML := func(i int) bool { + switch words[i] { + case "insert", "delete", "merge": + return true + case "update": + return i == 0 || (words[i-1] != "for" && words[i-1] != "key" && words[i-1] != "do") + } + return false + } + if isDML(0) { + return strings.ToUpper(words[0]) + } + if words[0] != "with" { + return "" + } + for i := range words { + if depths[i] <= 1 && isDML(i) { + return strings.ToUpper(words[i]) + } + } + return "" +} + +// sqlWords returns the lower-cased bare words of sql, and each ';', with their parenthesis depth, skipping comments, +// string literals, quoted identifiers and dollar-quoted bodies. +func sqlWords(sql string) ([]string, []int) { + var words []string + var depths []int + depth := 0 + for i := 0; i < len(sql); { + c := sql[i] + switch { + case c == '-' && strings.HasPrefix(sql[i:], "--"): + for i < len(sql) && sql[i] != '\n' { + i++ + } + case c == '/' && strings.HasPrefix(sql[i:], "/*"): + if j := strings.Index(sql[i+2:], "*/"); j >= 0 { + i += j + 4 + } else { + i = len(sql) + } + case c == '\'' || c == '"': + i++ + for i < len(sql) { + if sql[i] == c { + if i+1 < len(sql) && sql[i+1] == c { + i += 2 + continue + } + break + } + i++ + } + i++ + case c == '$': + j := i + 1 + for j < len(sql) && isIdent(sql[j]) && !(sql[j] >= '0' && sql[j] <= '9' && j == i+1) { + j++ + } + if j < len(sql) && sql[j] == '$' { + tag := sql[i : j+1] + if k := strings.Index(sql[j+1:], tag); k >= 0 { + i = j + 1 + k + len(tag) + } else { + i = len(sql) + } + } else { + i = j + } + case c == ';': + words = append(words, ";") + depths = append(depths, depth) + i++ + case c == '(': + depth++ + i++ + case c == ')': + depth-- + i++ + case isIdent(c) && !(c >= '0' && c <= '9'): + j := i + for j < len(sql) && (isIdent(sql[j]) || sql[j] == '$') { + j++ + } + words = append(words, strings.ToLower(sql[i:j])) + depths = append(depths, depth) + i = j + default: + i++ + } + } + return words, depths +} + +func (c *Corpus) resolveInherit() error { + done := map[string]bool{} + var resolve func(e *Entry, seen []string) error + resolve = func(e *Entry, seen []string) error { + if done[e.ID] || e.Inherit == "" { + done[e.ID] = true + return nil + } + for _, id := range seen { + if id == e.ID { + return fmt.Errorf("inherit cycle: %s", strings.Join(append(seen, e.ID), " → ")) + } + } + parent := c.byID[e.Inherit] + if parent == nil { + return fmt.Errorf("entry %q: inherit names unknown entry %q", e.ID, e.Inherit) + } + if err := resolve(parent, append(seen, e.ID)); err != nil { + return err + } + inherit(e, parent) + done[e.ID] = true + return nil + } + for _, e := range c.Entries { + if err := resolve(e, nil); err != nil { + return err + } + } + return nil +} + +// inherit fills e's empty expect, mode, modes and tiers fields from parent; e's own values win. +func inherit(e, parent *Entry) { + if len(e.Tiers) == 0 { + e.Tiers = append([]string(nil), parent.Tiers...) + } + if len(e.Modes) == 0 { + e.Modes = append([]string(nil), parent.Modes...) + } + for k, v := range parent.Mode { + if e.Mode == nil { + e.Mode = map[string]*ModeOverride{} + } + if _, own := e.Mode[k]; !own { + e.Mode[k] = v + } + } + x, p := &e.Expect, parent.Expect + if x.Driver == nil { + x.Driver = p.Driver + } + if x.Require == nil { + x.Require = p.Require + } + if x.Forbid == nil { + x.Forbid = p.Forbid + } + if x.SortExempt == nil { + x.SortExempt = p.SortExempt + } + if x.Budget.Buffers == 0 { + x.Budget.Buffers = p.Budget.Buffers + } + if x.Budget.PlanningMs == 0 { + x.Budget.PlanningMs = p.Budget.PlanningMs + } + if x.Budget.ExecMs == 0 { + x.Budget.ExecMs = p.Budget.ExecMs + } + if x.Budget.Rows == 0 { + x.Budget.Rows = p.Budget.Rows + } +} + +func (c *Corpus) checkEntry(e *Entry) error { + if e.Stmt == "" { + return errors.New("missing stmt") + } + if _, ok := c.Statements[e.Stmt]; !ok { + return fmt.Errorf("unknown stmt %q", e.Stmt) + } + for _, t := range e.Tiers { + if !validTiers[t] { + return fmt.Errorf("unknown tier %q", t) + } + } + for _, m := range e.Modes { + if !validModes[m] { + return fmt.Errorf("unknown mode %q (want custom, generic or auto)", m) + } + } + for i, p := range e.Params { + if err := c.checkParam(p); err != nil { + return fmt.Errorf("param %d: %w", i+1, err) + } + } + for _, ix := range e.RequiresIndexes { + if len(c.Defaults.BigTables) == 0 { + return errors.New("requires_indexes needs defaults.big_tables (the index list is read from them)") + } + if !plainIdent(ix) { + return fmt.Errorf("requires_indexes: %q is not an index name", ix) + } + } + if e.Baseline != "" && !strings.HasPrefix(e.Baseline, "unstable: ") { + return fmt.Errorf("baseline %q must be empty or \"unstable: \"", e.Baseline) + } + if err := c.checkExpect(&e.Expect); err != nil { + return err + } + for _, m := range sortedKeys(e.Mode) { + if !validModes[m] { + return fmt.Errorf("mode %q: unknown mode (want custom, generic or auto)", m) + } + o := e.Mode[m] + if o == nil { + return fmt.Errorf("mode %q: empty override", m) + } + if o.Skip != "" && (o.DegradeTo != nil || o.KnownIssue != "") { + return fmt.Errorf("mode %q: skip excludes degrade_to and known_issue", m) + } + if o.DegradeTo != nil { + if err := c.checkExpect(o.DegradeTo); err != nil { + return fmt.Errorf("mode %q degrade_to: %w", m, err) + } + } + } + return nil +} + +// checkParam validates one param: a scalar, a list of params, or a one-key map {r: name} naming a resolver, +// {sql: query} (an inline resolver, held to the resolvers' rules) or {raw: text}. +func (c *Corpus) checkParam(v any) error { + switch x := v.(type) { + case []any: + for _, y := range x { + if err := c.checkParam(y); err != nil { + return err + } + } + case map[string]any: + if len(x) != 1 { + return fmt.Errorf("%v: want exactly one of r, sql or raw", x) + } + for k, val := range x { + s, ok := val.(string) + if !ok || s == "" { + return fmt.Errorf("%s: want a non-empty string", k) + } + switch k { + case "r": + if _, ok := c.Resolvers[s]; !ok { + return fmt.Errorf("resolver %q is not in resolvers", s) + } + case "sql": + if m := componentSchema.FindString(s); m != "" { + return fmt.Errorf("inline resolver reads schema %q; resolvers read Beacon tables only", m) + } + if err := checkStatementText(s); err != nil { + return fmt.Errorf("inline resolver: %w", err) + } + case "raw": + if err := checkRawParam(s); err != nil { + return fmt.Errorf("raw %q: %w", s, err) + } + default: + return fmt.Errorf("unknown param key %q (want r, sql or raw)", k) + } + } + } + return nil +} + +// checkExpect validates x and expands "big" in its node rules in place. Inherited rules are shared with the +// parent, so the expansion is idempotent. +func (c *Corpus) checkExpect(x *Expect) error { + for _, name := range x.SortExempt { + if _, ok := c.SortExemptSets[name]; !ok { + return fmt.Errorf("sort_exempt %q is not in sort_exempt_sets", name) + } + } + for i, d := range x.Driver { + for _, f := range d.IndexCond { + if err := checkModeNeutral(f); err != nil { + return fmt.Errorf("driver %d: %w", i, err) + } + } + } + var err error + if x.Require, err = c.expandBig(x.Require); err != nil { + return err + } + if x.Forbid, err = c.expandBig(x.Forbid); err != nil { + return err + } + return nil +} + +func (c *Corpus) expandBig(rules []NodeRule) ([]NodeRule, error) { + for i := range rules { + var rels []string + changed := false + for _, r := range rules[i].Relations { + if r != "big" { + rels = append(rels, r) + continue + } + if len(c.Defaults.BigTables) == 0 { + return nil, fmt.Errorf("%s rule names relations [big] but defaults.big_tables is empty", rules[i].Node) + } + rels = append(rels, c.Defaults.BigTables...) + changed = true + } + if changed { + rules[i].Relations = rels + } + } + return rules, nil +} + +// checkModeNeutral rejects an Index Cond fragment holding a literal: a quote, a number or a $n parameter. +// Custom plans inline the parameter values and generic plans show $n, so a fragment holds only the column and +// the operator, e.g. "public_key =". Digits inside identifiers (idx_v2, t1.col) are names, not literals. +func checkModeNeutral(fragment string) error { + if strings.ContainsAny(fragment, `'"`) { + return fmt.Errorf("index_cond %q holds a literal (a quote); fragments must be mode-neutral", fragment) + } + for i := 0; i < len(fragment); i++ { + ch := fragment[i] + if ch < '0' || ch > '9' { + continue + } + if i > 0 && fragment[i-1] == '$' { + return fmt.Errorf("index_cond %q holds a parameter ($n); custom plans inline it, so fragments end at the operator", fragment) + } + if i > 0 && isIdent(fragment[i-1]) { + continue + } + return fmt.Errorf("index_cond %q holds a literal (a number); fragments must be mode-neutral", fragment) + } + return nil +} + +func isIdent(b byte) bool { + return b == '_' || (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || (b >= '0' && b <= '9') +} + +func sortedKeys[V any](m map[string]V) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} diff --git a/internal/planguard/corpus_integration_test.go b/internal/planguard/corpus_integration_test.go new file mode 100644 index 0000000..7f5cfe7 --- /dev/null +++ b/internal/planguard/corpus_integration_test.go @@ -0,0 +1,112 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package planguard_test + +import ( + "context" + "errors" + "os" + "slices" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/db" + pgdata "github.com/MeshCore-Beacon/beacon-server/db/planguard" + "github.com/MeshCore-Beacon/beacon-server/internal/planguard" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" +) + +// Every statement of the repo corpus PREPAREs on a freshly migrated schema, in a READ ONLY transaction. A pending +// statement counts only when every table it names exists: when the server reports a missing relation (42P01) it is +// logged as skipped, and any other error fails. +func TestCorpusStatementsPrepareCleanPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for PostgreSQL tests") + } + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + admin, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + schema := "planguard_" + strings.ReplaceAll(uuid.NewString(), "-", "") + if _, err := admin.Exec(ctx, "CREATE SCHEMA "+schema); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if _, err := admin.Exec(context.Background(), "DROP SCHEMA "+schema+" CASCADE"); err != nil { + t.Error(err) + } + admin.Close(context.Background()) + }) + cfg, err := pgxpool.ParseConfig(dsn) + if err != nil { + t.Fatal(err) + } + cfg.ConnConfig.RuntimeParams["search_path"] = schema + cfg.MaxConns = 1 + pool, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err := db.RunMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + sqlc, err := planguard.LoadStatements("../../db/sqlc") + if err != nil { + t.Fatal(err) + } + c, err := planguard.LoadCorpus(pgdata.Files, "corpus.yaml", sqlc) + if err != nil { + t.Fatal(err) + } + conn, err := pool.Acquire(ctx) + if err != nil { + t.Fatal(err) + } + defer conn.Release() + pc := conn.Conn().PgConn() + + prepared, skipped := 0, 0 + names := make([]string, 0, len(c.Statements)) + for name := range c.Statements { + names = append(names, name) + } + slices.Sort(names) + for _, name := range names { + st := c.Statements[name] + text := strings.TrimRight(strings.TrimSpace(st.SQL), ";") + _, err := pc.Exec(ctx, "BEGIN READ ONLY; PREPARE planguard_prepare_check AS\n"+text+";\nDEALLOCATE planguard_prepare_check").ReadAll() + if _, rbErr := pc.Exec(ctx, "ROLLBACK").ReadAll(); rbErr != nil { + t.Fatal(rbErr) + } + var pgErr *pgconn.PgError + if err != nil && st.Pending != "" && errors.As(err, &pgErr) && pgErr.Code == "42P01" { + t.Logf("%s: pending %s: skipped (%s)", name, st.Pending, pgErr.Message) + skipped++ + continue + } + if err != nil { + t.Errorf("%s (plan %s, pending %q): PREPARE: %v", name, st.Plan, st.Pending, err) + // A failed PREPARE leaves nothing to deallocate; make sure of it before the next statement. + _, _ = pc.Exec(ctx, "DEALLOCATE planguard_prepare_check").ReadAll() + continue + } + prepared++ + } + var left int + if err := conn.QueryRow(ctx, "SELECT count(*) FROM pg_prepared_statements WHERE name = 'planguard_prepare_check'").Scan(&left); err != nil || left != 0 { + t.Errorf("prepared statement left behind: %d, %v", left, err) + } + t.Logf("%d statements prepared, %d pending statements skipped", prepared, skipped) + if prepared == 0 { + t.Fatal("no statement prepared") + } +} diff --git a/internal/planguard/corpus_repo_test.go b/internal/planguard/corpus_repo_test.go new file mode 100644 index 0000000..8d9ad71 --- /dev/null +++ b/internal/planguard/corpus_repo_test.go @@ -0,0 +1,262 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package planguard_test + +import ( + "bytes" + "encoding/csv" + "io/fs" + "os" + "path/filepath" + "regexp" + "slices" + "strings" + "testing" + + pgdata "github.com/MeshCore-Beacon/beacon-server/db/planguard" + "github.com/MeshCore-Beacon/beacon-server/internal/planguard" +) + +// planIDs matches internal plan identifiers, which must not appear in the corpus or its SQL files. +var planIDs = regexp.MustCompile(`\b(S[0-9]+[a-z]?|V[0-9]|P[0-9]|§[0-9])\b`) + +// Identifiers copied from a deployment must not ship: a UUID that is not one of the synthetic +// 00000000-0000-0000-0000-… ids, or a hex run of 16 digits or more holding a letter (a packet hash or public key, +// as a \x byte string or bare) unless it is synthetic: it starts with eight zeros, or it is 0102030405060708. +// All-digit runs are numbers. Cases name such rows through resolvers instead. +var ( + uuidRe = regexp.MustCompile(`[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}`) + longHexRe = regexp.MustCompile(`(\\x)?([0-9a-fA-F]{16,})`) + letterRe = regexp.MustCompile(`[a-fA-F]`) + synthUUID = "00000000-0000-0000-0000-" +) + +// deploymentIDs returns the copied identifiers on one line. +func deploymentIDs(line string) []string { + var out []string + for _, u := range uuidRe.FindAllString(line, -1) { + if !strings.HasPrefix(u, synthUUID) { + out = append(out, u) + } + } + line = uuidRe.ReplaceAllString(line, "") + for _, m := range longHexRe.FindAllStringSubmatch(line, -1) { + h := m[2] + if (m[1] != "" || letterRe.MatchString(h)) && !strings.HasPrefix(h, "00000000") && h != "0102030405060708" { + out = append(out, m[0]) + } + } + return out +} + +func TestDeploymentIDs(t *testing.T) { + for line, want := range map[string]int{ + `params: ["\\x7c05039b3e8a663c"]`: 1, + `"hash": "a33704fac0d3a216"`: 1, + `id: 7ea9ed3f-d6ce-4e54-a8b7-9f1e2e7d5179`: 1, + `limit 9223372036854775807`: 0, + `"00000000000000a2" and \\x0102030405060708`: 0, + `'00000000-0000-0000-0000-00000000000b'::uuid`: 0, + } { + if got := len(deploymentIDs(line)); got != want { + t.Errorf("deploymentIDs(%q) found %d, want %d", line, got, want) + } + } +} + +func TestRepoCorpusLoads(t *testing.T) { + c, err := planguard.LoadCorpus(pgdata.Files, "corpus.yaml", repoSQLC(t)) + if err != nil { + t.Fatal(err) + } + if len(c.Entries) == 0 { + t.Fatal("no entries") + } + for name, st := range c.Statements { + switch { + case st.Pending == "" && strings.TrimSpace(st.SQL) == "": + t.Errorf("statement %s has no text", name) + case st.Pending != "" && st.File != "" && planguard.Normalize(st.SQL) == "": + t.Errorf("pending statement %s: %s holds no statement", name, st.File) + } + if st.Pending != "" && st.Group != st.Pending { + t.Errorf("pending statement %s: group %q, want its feature %q (the overlay search_path)", name, st.Group, st.Pending) + } + } + + used := map[string]bool{} + for _, s := range c.Defaults.SortExempt { + used[s] = true + } + for _, e := range c.Entries { + st := c.Statements[e.Stmt] + if (strings.HasPrefix(e.ID, "planted") || st.Plan == "harness") && !e.ExpectFail { + t.Errorf("planted entry %s is not expect_fail", e.ID) + } + for _, s := range e.Expect.SortExempt { + used[s] = true + } + for _, o := range e.Mode { + if o != nil && o.DegradeTo != nil { + for _, s := range o.DegradeTo.SortExempt { + used[s] = true + } + } + } + } + for name := range c.SortExemptSets { + if !used[name] { + t.Errorf("sort_exempt_sets %s is used by no entry", name) + } + } + + // No plan IDs or deployment identifiers in the corpus, the frozen SQL, the fixture generator and its testdata, + // or the baselines. + scanned := map[string]int{} + err = fs.WalkDir(pgdata.Files, ".", func(p string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return err + } + top, _, _ := strings.Cut(p, "/") + scanned[top]++ + body, err := fs.ReadFile(pgdata.Files, p) + if err != nil { + return err + } + for i, line := range strings.Split(string(body), "\n") { + if m := planIDs.FindString(line); m != "" { + t.Errorf("%s:%d holds the plan ID %q: %s", p, i+1, m, strings.TrimSpace(line)) + } + for _, id := range deploymentIDs(line) { + t.Errorf("%s:%d holds the deployment identifier %s", p, i+1, id) + } + } + return nil + }) + if err != nil { + t.Fatal(err) + } + for _, top := range []string{"corpus.yaml", "sql", "fixture", "baseline"} { + if scanned[top] == 0 { + t.Errorf("no embedded file under %s was scanned", top) + } + } +} + +// The engine's own test data holds no deployment identifiers either. +func TestTestdataHoldsNoDeploymentIDs(t *testing.T) { + err := fs.WalkDir(os.DirFS("testdata"), ".", func(p string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return err + } + body, err := os.ReadFile(filepath.Join("testdata", p)) + if err != nil { + return err + } + for i, line := range strings.Split(string(body), "\n") { + for _, id := range deploymentIDs(line) { + t.Errorf("testdata/%s:%d holds the deployment identifier %s", p, i+1, id) + } + } + return nil + }) + if err != nil { + t.Fatal(err) + } +} + +// Legacy readers run on api_legacy with automatic plan caching: their entries run custom and auto, and the auto +// mode carries a known issue until their rewrite lands, so a generic flip reads KNOWN_ISSUE. +func TestRepoCorpusLegacyAutoKnownIssue(t *testing.T) { + c, err := planguard.LoadCorpus(pgdata.Files, "corpus.yaml", repoSQLC(t)) + if err != nil { + t.Fatal(err) + } + n := 0 + for _, e := range c.Entries { + switch e.Stmt { + case "GetTopNodes", "GetStatsTopAdvertisers", "GetPathStats": + default: + continue + } + n++ + if !slices.Equal(c.ModesOf(e), []string{"custom", "auto"}) || e.KnownIssueIn("auto") == "" || e.KnownIssueIn("custom") != "" { + t.Errorf("entry %s: modes %v, auto known issue %q, custom known issue %q", e.ID, c.ModesOf(e), e.KnownIssueIn("auto"), e.KnownIssueIn("custom")) + } + } + if n != 3 { + t.Errorf("%d legacy reader entries, want 3", n) + } +} + +// The three statements gen-policy admits carry every entry in every mode, and their shapes are tagged. +func TestRepoCorpusCachedEntries(t *testing.T) { + c, err := planguard.LoadCorpus(pgdata.Files, "corpus.yaml", repoSQLC(t)) + if err != nil { + t.Fatal(err) + } + for _, e := range c.Entries { + st := c.Statements[e.Stmt] + if st.Plan != "cached" || st.Pending != "" { + continue + } + if len(e.Shapes) == 0 { + t.Errorf("entry %s of cached %s has no shapes", e.ID, e.Stmt) + } + if !slices.Equal(c.ModesOf(e), []string{"custom", "generic", "auto"}) { + t.Errorf("entry %s of cached %s runs %v", e.ID, e.Stmt, c.ModesOf(e)) + } + } +} + +// The fixture's data files hold aggregates only: every cell is a count, a number (at most 15 integer digits, so a +// key cannot pass as one), a boolean, a timestamp or an IATA code. A name, key, UUID or any other text (a node, observer, channel, scope or broker name) fails, as does a +// file that is not CSV. +func TestFixtureDataHoldsOnlyAggregates(t *testing.T) { + header := regexp.MustCompile(`^[a-z][a-z0-9_]*$`) + cell := regexp.MustCompile(`^(-?[0-9]{1,15}(\.[0-9]+)?([eE][-+]?[0-9]+)?|t|f|true|false|[A-Z0-9]{3}|[0-9]{4}-[0-9]{2}-[0-9]{2}[ T][0-9:.]+([+-][0-9:]+|Z)?|)$`) + n := 0 + err := fs.WalkDir(pgdata.Files, "fixture/testdata", func(p string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return err + } + n++ + if !strings.HasSuffix(p, ".csv") { + t.Errorf("%s: only CSV files belong in the fixture data", p) + return nil + } + body, err := fs.ReadFile(pgdata.Files, p) + if err != nil { + return err + } + recs, err := csv.NewReader(bytes.NewReader(body)).ReadAll() + if err != nil { + t.Errorf("%s: %v", p, err) + return nil + } + if len(recs) < 2 { + t.Errorf("%s: no data rows", p) + return nil + } + for _, h := range recs[0] { + if !header.MatchString(h) { + t.Errorf("%s: header %q is not a column name", p, h) + } + } + for i, rec := range recs[1:] { + for j, c := range rec { + if !cell.MatchString(c) { + t.Errorf("%s:%d column %s holds %q: not a count, number, boolean, timestamp or IATA code", p, i+2, recs[0][j], c) + } + } + } + return nil + }) + if err != nil { + t.Fatal(err) + } + if n == 0 { + t.Fatal("no fixture data files") + } +} diff --git a/internal/planguard/corpus_test.go b/internal/planguard/corpus_test.go new file mode 100644 index 0000000..e7b3544 --- /dev/null +++ b/internal/planguard/corpus_test.go @@ -0,0 +1,248 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package planguard_test + +import ( + "io/fs" + "os" + "reflect" + "strings" + "testing" + + pgdata "github.com/MeshCore-Beacon/beacon-server/db/planguard" + "github.com/MeshCore-Beacon/beacon-server/internal/planguard" +) + +var testSQLC = map[string]string{ + "Q": "-- name: Q :one\nSELECT * FROM nodes WHERE public_key = $1", + "C": "-- name: C :many\nSELECT 1", + "M": "-- name: M :exec\nDELETE FROM packets WHERE id < $1", + "I": "-- name: I :exec\nINSERT INTO nodes (public_key) VALUES ($1)", + "L": "-- name: L :many\nSELECT 2", +} + +func TestCorpusRejectsUnknownFields(t *testing.T) { + cases := map[string]string{ + "unknown_top.yaml": "unexpected_top", + "unknown_entry.yaml": "unexpected_entry_key", + "allow_sort_above_limit.yaml": "allow_sort_above_limit", + } + for f, key := range cases { + _, err := planguard.LoadCorpus(os.DirFS("testdata/corpus"), f, map[string]string{"Q": "-- name: Q :many\nSELECT 1"}) + if err == nil || !strings.Contains(err.Error(), key) { + t.Errorf("%s: err = %v, want one naming %q", f, err, key) + } + } +} + +func TestCorpusValidation(t *testing.T) { + cases := []struct { + file string + want []string // every fragment must appear in the error + }{ + {"bad_duplicate_id.yaml", []string{"duplicate entry id", `"a"`}}, + {"bad_unknown_stmt.yaml", []string{`entry "a"`, `unknown stmt "Nope"`}}, + {"bad_unknown_inherit.yaml", []string{`entry "a"`, `unknown entry "nope"`}}, + {"bad_inherit_cycle.yaml", []string{"inherit cycle"}}, + {"bad_sort_exempt.yaml", []string{`entry "a"`, `"no_such_set"`}}, + {"bad_default_sort_exempt.yaml", []string{"defaults", `"no_default_set"`}}, + {"bad_index_cond_quote.yaml", []string{`entry "a"`, "literal", "iata = 'SEA'"}}, + {"bad_index_cond_digit.yaml", []string{`entry "a"`, "literal", "hop_count = 3"}}, + {"bad_index_cond_param.yaml", []string{`entry "a"`, "parameter", "public_key = $1"}}, + {"bad_scan_pattern_literal.yaml", []string{`sort_exempt_sets "pinned"`, "literal"}}, + {"bad_resolver_schema.yaml", []string{`resolver "hop_case"`, "t_fix_routes_hop."}}, + {"bad_cached_no_admission.yaml", []string{`statement "G"`, "admission"}}, + {"bad_cached_unknown_profile.yaml", []string{`statement "G"`, `"region_page"`}}, + {"bad_unknown_mode.yaml", []string{`entry "a"`, `mode "forced"`}}, + {"bad_unknown_mode_override.yaml", []string{`entry "a"`, `mode "plain"`}}, + {"bad_missing_sqlc.yaml", []string{`statement "Gone"`, "sqlc"}}, + {"bad_select_writes.yaml", []string{`statement "M"`, "kind select", "DELETE"}}, + {"bad_select_cte_writes.yaml", []string{`statement "W"`, "kind select", "DELETE"}}, + {"bad_entry_id.yaml", []string{"entry 0", `"../escape"`}}, + {"bad_resolver_ref.yaml", []string{`entry "a"`, "param 2", `resolver "nope"`}}, + {"bad_inline_resolver_schema.yaml", []string{`entry "a"`, "param 1", "t_fix_routes_hop."}}, + {"bad_param_key.yaml", []string{`entry "a"`, "param 1", `"rr"`}}, + } + for _, tc := range cases { + t.Run(tc.file, func(t *testing.T) { + _, err := planguard.LoadCorpus(os.DirFS("testdata/corpus"), tc.file, testSQLC) + if err == nil { + t.Fatal("want an error") + } + for _, w := range tc.want { + if !strings.Contains(err.Error(), w) { + t.Fatalf("err = %v, want it to contain %q", err, w) + } + } + }) + } + + c, err := planguard.LoadCorpus(os.DirFS("testdata/corpus"), "good.yaml", testSQLC) + if err != nil { + t.Fatalf("good.yaml: %v", err) + } + if got := c.Defaults.Forbid[0].Relations; !reflect.DeepEqual(got, []string{"packets", "packet_observations"}) { + t.Errorf("defaults forbid relations = %v, want big expanded", got) + } + hit := c.Entry("q.hit") + if hit == nil { + t.Fatal("q.hit missing") + } + if got := hit.Expect.Forbid[0].Relations; !reflect.DeepEqual(got, []string{"packets", "packet_observations", "nodes"}) { + t.Errorf("entry forbid relations = %v, want big expanded in place", got) + } + if c.Entry("nope") != nil { + t.Error("Entry(nope) != nil") + } + + miss := c.Entry("q.miss") + if !reflect.DeepEqual(miss.Tiers, hit.Tiers) || !reflect.DeepEqual(miss.Expect, hit.Expect) || + miss.Mode["generic"] == nil || miss.Mode["generic"].DegradeTo.Budget.Buffers != 20 { + t.Errorf("inherit did not copy tiers/expect/mode: %+v", miss) + } + if miss.Shape != "miss" || !reflect.DeepEqual(miss.Shapes, []string{"miss"}) || miss.Params != nil { + t.Errorf("inherit overwrote the entry's own fields: %+v", miss) + } + own := c.Entry("q.miss_own") + if !reflect.DeepEqual(own.Tiers, []string{"prod"}) || own.Expect.Budget.Buffers != 3 || + own.Expect.Budget.PlanningMs != 5 || len(own.Expect.Driver) != 1 { + t.Errorf("inherit must keep set fields and fill only empty ones: %+v", own) + } + if got := c.ModesOf(own); !reflect.DeepEqual(got, []string{"custom"}) { + t.Errorf("ModesOf(q.miss_own) = %v, want its own modes", got) + } + + modes := map[string][]string{ + "q.hit": {"custom", "generic", "auto"}, // cached + "c.one": {"custom"}, // custom + "c.modes": {"custom", "generic"}, // entry override + "m.one": {"custom"}, // maint + "i.one": {"auto"}, // ingest + "l.one": {"custom", "generic", "auto"}, // legacy + "h.one": {"custom", "generic", "auto"}, // harness + "p.one": {"custom"}, // pending custom + } + for id, want := range modes { + if got := c.ModesOf(c.Entry(id)); !reflect.DeepEqual(got, want) { + t.Errorf("ModesOf(%s) = %v, want %v", id, got, want) + } + } + + if got := c.Statements["Q"].SQL; got != testSQLC["Q"] { + t.Errorf("Q SQL = %q", got) + } + if got := c.Statements["H"].SQL; !strings.Contains(got, "SELECT pg_sleep($1)") { + t.Errorf("H SQL = %q, want the file text", got) + } + if got := c.Statements["P"].SQL; got != "" { + t.Errorf("pending statement without text got SQL %q", got) + } + if c.Statements["Q"].Kind != "select" || c.Statements["M"].Kind != "dml" { + t.Errorf("kinds = %q, %q", c.Statements["Q"].Kind, c.Statements["M"].Kind) + } + if c.Tiers["fixture"].Settings["random_page_cost"] != "1.1" || c.Groups["routes"].SearchPath["local"] != "t_s2b, public" { + t.Errorf("tiers/groups not decoded: %+v %+v", c.Tiers, c.Groups) + } +} + +// A sqlc statement in planpolicy.Legacy runs on api_legacy, so the corpus must label it plan legacy. +func TestCorpusLegacyStatementsArePlanLegacy(t *testing.T) { + sqlcText := map[string]string{"GetTopNodes": "-- name: GetTopNodes :many\nSELECT 1"} + for _, plan := range []string{"custom", "cached"} { + body := "admission_profiles: {by_id: [hit, miss]}\nstatements:\n GetTopNodes: {sqlc: GetTopNodes, plan: " + plan + if plan == "cached" { + body += ", admission: by_id" + } + body += "}\n" + _, err := planguard.LoadCorpus(mapFS(map[string]string{"c.yaml": body}), "c.yaml", sqlcText) + if err == nil || !strings.Contains(err.Error(), `statement "GetTopNodes"`) || !strings.Contains(err.Error(), "planpolicy.Legacy") { + t.Errorf("plan %s: err = %v, want one naming the statement and planpolicy.Legacy", plan, err) + } + } + body := "statements:\n GetTopNodes: {sqlc: GetTopNodes, plan: legacy}\n" + if _, err := planguard.LoadCorpus(mapFS(map[string]string{"c.yaml": body}), "c.yaml", sqlcText); err != nil { + t.Errorf("plan legacy: %v", err) + } +} + +func TestDerivedStatementsMatchSource(t *testing.T) { + sqlcText, err := planguard.LoadStatements("../../db/sqlc") + if err != nil { + t.Fatal(err) + } + for _, src := range []struct { + name string + fsys fs.FS + file string + }{ + {"testdata", os.DirFS("testdata/corpus"), "derived.yaml"}, + {"repo", pgdata.Files, "corpus.yaml"}, + } { + c, err := planguard.LoadCorpus(src.fsys, src.file, sqlcText) + if err != nil { + t.Fatalf("%s %s: %v", src.name, src.file, err) + } + n := 0 + for name, st := range c.Statements { + if st.DerivedFrom == "" { + continue + } + n++ + text := sqlcText[st.DerivedFrom] + if !planguard.DerivedBodyMatches(text, st.SQL) { + t.Errorf("%s %s: body is not part of %s", src.name, name, st.DerivedFrom) + } + // One edited token must break the match. + edited := strings.Replace(st.SQL, "LIMIT", "OFFSET", 1) + if edited == st.SQL || planguard.DerivedBodyMatches(text, edited) { + t.Errorf("%s %s: an edited body still matches", src.name, name) + } + } + if n == 0 { + t.Fatalf("%s: no derived statement checked", src.name) + } + } + + _, err = planguard.LoadCorpus(os.DirFS("testdata/corpus"), "bad_derived_edited.yaml", sqlcText) + if err == nil || !strings.Contains(err.Error(), `statement "DeleteExpiredRoutesSelect"`) || + !strings.Contains(err.Error(), "DeleteExpiredRoutes") { + t.Fatalf("edited derived file: err = %v, want one naming the statement and its source", err) + } +} + +func TestStripLockClause(t *testing.T) { + for in, want := range map[string]string{ + "LIMIT $5\n FOR UPDATE OF r SKIP LOCKED),": "LIMIT $5),", + "LIMIT 1 FOR UPDATE SKIP LOCKED": "LIMIT 1", + "LIMIT 1 for update of a, b skip locked": "LIMIT 1", + "LIMIT 1 FOR UPDATE": "LIMIT 1 FOR UPDATE", + } { + if got := planguard.StripLockClause(in); got != want { + t.Errorf("StripLockClause(%q) = %q, want %q", in, got, want) + } + } +} + +func TestDerivedBodyNeedsTokenBoundaries(t *testing.T) { + src := "-- name: X :one\nWITH c AS (SELECT id FROM t ORDER BY id LIMIT $5 OFFSET $6 FOR UPDATE SKIP LOCKED) SELECT count(*) FROM c" + if planguard.DerivedBodyMatches(src, "SELECT id FROM t ORDER BY id LIMIT $1") { + t.Error("a body that stops before OFFSET matched") + } + if !planguard.DerivedBodyMatches(src, "SELECT id FROM t ORDER BY id LIMIT $1 OFFSET $2") { + t.Error("the whole CTE body did not match") + } + if planguard.DerivedBodyMatches(src, "ELECT id FROM t ORDER BY id LIMIT $1 OFFSET $2") { + t.Error("a body starting mid-token matched") + } + if !planguard.DerivedBodyMatches("SELECT a FROM t;", "SELECT a FROM t") { + t.Error("a whole statement did not match itself") + } +} + +func TestCorpusResolversReadBeaconTablesOnly(t *testing.T) { + _, err := planguard.LoadCorpus(os.DirFS("testdata/corpus"), "bad_resolver_schema.yaml", testSQLC) + if err == nil || !strings.Contains(err.Error(), "hop_case") { + t.Fatalf("err = %v, want one naming resolver hop_case", err) + } +} diff --git a/internal/planguard/diff.go b/internal/planguard/diff.go new file mode 100644 index 0000000..e55c275 --- /dev/null +++ b/internal/planguard/diff.go @@ -0,0 +1,1355 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package planguard + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "regexp" + "slices" + "strconv" + "strings" + "time" + + "gopkg.in/yaml.v3" +) + +// DiffSpec is a diff case file: batches of statement pairs whose results are compared on one snapshot. +type DiffSpec struct { + Batches []DiffBatch `yaml:"batches"` +} + +// DiffBatch runs in one REPEATABLE READ READ ONLY transaction, so every statement in it sees the same data. +type DiffBatch struct { + Name string `yaml:"name"` + SearchPath string `yaml:"search_path"` + Statements map[string]string `yaml:"statements"` // alias → "old:" | "new:" | "file:" + Pairs []DiffPair `yaml:"pairs"` +} + +// Call is one execution of an aliased statement. +type Call struct { + Stmt string `yaml:"stmt"` + Params []any `yaml:"params"` +} + +// SaturatedSpec names the ref columns that say a legacy scan was saturated (flag = t) and where its scan stopped +// (floor). Rows below the floor are the ones the legacy statement may have got wrong. +type SaturatedSpec struct { + Flag string `yaml:"flag"` + Floor string `yaml:"floor"` +} + +// TieCursor names the cand's own page cursor: the params holding its timestamp and hash bound (1-based) and the +// result column holding the hash. +type TieCursor struct { + TS int `yaml:"ts"` + Hash int `yaml:"hash"` + HashColumn string `yaml:"hash_column"` +} + +// DiffPair compares one ref result with one or more cand results. +// +// Every ref column is compared. Each relaxation is a named class that must be listed in Allowed: +// - tie_order: within each run of equal Key, cand rows are a permutation of ref rows (same count, keys equal +// position by position); +// - saturated_suffix: when ref row 1 has Saturated.Flag = t, ref rows whose Key[0] is below Saturated.Floor may +// be missing from or differ in cand; the only class that relaxes the row count, and only by those rows. The +// cand rows in their place must be below the floor too, keep the descending key order, and repeat no row; +// - latest_observer_tie: the Omit columns may differ; each such row is listed; +// - pagecursor_tie_rows: cand may start with k rows ref lacks whose Key[0], truncated to the millisecond, equals +// the millisecond of ref param TieBoundary, and which sort at or below the cand's own cursor (TieCursor: +// (Key[0], hash) ≤ (ts, hash bound)); then cand[k:] must equal ref[:len(ref)-k]. +// +// Require lists classes that must fire for every cand: a case that exists to show a class shows it, or fails. +// Param indexes (TieBoundary, TieCursor, Walk.LimitParam, Walk.Cursor[].Param) are 1-based, like $n. +type DiffPair struct { + ID string `yaml:"id"` + Ref Call `yaml:"ref"` + Cands []Call `yaml:"cands"` + Key []string `yaml:"key"` + Allowed []string `yaml:"allowed"` + Omit []string `yaml:"omit"` + Saturated *SaturatedSpec `yaml:"saturated"` + TieBoundary *int `yaml:"tie_boundary"` + TieCursor *TieCursor `yaml:"tie_cursor"` + Require []string `yaml:"require"` + Walk *WalkSpec `yaml:"walk"` + Expect map[string]string `yaml:"expect"` // cand alias → identical | allowed | same | different; default same +} + +// Outcome is the comparison of one ref result with one cand result. +type Outcome struct { + Verdict string // identical | allowed | different + Classes []string // allowed classes that fired, as "name(count)" + Detail string // first differing row, missing/extra rows, column names; the rows a class relaxed +} + +// Diff verdicts and classes. +const ( + DiffIdentical = "identical" + DiffAllowed = "allowed" + DiffDifferent = "different" + DiffSkip = "skip" // a {sql: …} param found no value on this snapshot; the pair did not run + + ClassTieOrder = "tie_order" + ClassSaturatedSuffix = "saturated_suffix" + ClassLatestObserverTie = "latest_observer_tie" + ClassPageCursorTieRows = "pagecursor_tie_rows" +) + +// classOrder is the order classes are reported in. +var classOrder = []string{ClassPageCursorTieRows, ClassSaturatedSuffix, ClassTieOrder, ClassLatestObserverTie} + +var ( + aliasRe = regexp.MustCompile(`^[a-z][a-z0-9_]*$`) + validExpect = map[string]bool{DiffIdentical: true, DiffAllowed: true, "same": true, DiffDifferent: true} + defaultExpect = "same" +) + +// loadDiffSpec reads the diff case file p of fsys and parses it with ParseDiffSpec; file: paths are relative to +// fsys's root. +func loadDiffSpec(fsys fs.FS, p string) (*DiffSpec, error) { + raw, err := fs.ReadFile(fsys, p) + if err != nil { + return nil, err + } + return ParseDiffSpec(raw, p, fsys) +} + +// ParseDiffSpec parses and validates a diff case file named p. Unknown fields are errors; file: statements must +// exist in fsys and must not modify data (the batch is READ ONLY as well). +func ParseDiffSpec(raw []byte, p string, fsys fs.FS) (*DiffSpec, error) { + var spec DiffSpec + dec := yaml.NewDecoder(bytes.NewReader(raw)) + dec.KnownFields(true) + if err := dec.Decode(&spec); err != nil { + if errors.Is(err, io.EOF) { + return nil, fmt.Errorf("%s: empty diff spec", p) + } + return nil, fmt.Errorf("%s: %w", p, err) + } + var extra yaml.Node + if err := dec.Decode(&extra); !errors.Is(err, io.EOF) { + return nil, fmt.Errorf("%s: more than one YAML document", p) + } + if err := spec.Validate(); err != nil { + return nil, fmt.Errorf("%s: %w", p, err) + } + for _, b := range spec.Batches { + for _, alias := range sortedKeys(b.Statements) { + kind, rest, _ := strings.Cut(b.Statements[alias], ":") + if kind != "file" { + continue + } + text, err := fs.ReadFile(fsys, rest) + if err != nil { + return nil, fmt.Errorf("%s: batch %s: %s: %w", p, b.Name, alias, err) + } + if err := checkSingleSelect(string(text)); err != nil { + return nil, fmt.Errorf("%s: batch %s: %s: %w", p, b.Name, alias, err) + } + } + } + return &spec, nil +} + +// Validate checks every batch and pair. +func (s *DiffSpec) Validate() error { + if len(s.Batches) == 0 { + return errors.New("no batches") + } + names := map[string]bool{} + for _, b := range s.Batches { + if err := checkFileName(b.Name); err != nil { + return fmt.Errorf("batch name: %w", err) + } + if names[b.Name] { + return fmt.Errorf("batch %s: duplicate name", b.Name) + } + names[b.Name] = true + if err := b.validate(); err != nil { + return fmt.Errorf("batch %s: %w", b.Name, err) + } + } + return nil +} + +func checkFileName(name string) error { + if err := CheckName(name); err != nil { + return err + } + if strings.HasPrefix(name, ".") { + return fmt.Errorf("%q must not start with '.'", name) + } + return nil +} + +func (b *DiffBatch) validate() error { + if err := checkSearchPath(b.SearchPath); err != nil { + return err + } + if len(b.Statements) == 0 { + return errors.New("no statements") + } + for _, alias := range sortedKeys(b.Statements) { + if !aliasRe.MatchString(alias) { + return fmt.Errorf("statement alias %q must match %s", alias, aliasRe) + } + kind, rest, ok := strings.Cut(b.Statements[alias], ":") + switch { + case !ok || rest == "": + return fmt.Errorf("statement %s: %q, want old:, new: or file:", alias, b.Statements[alias]) + case kind == "old" || kind == "new": + if !plainIdentRe.MatchString(rest) { + return fmt.Errorf("statement %s: %q is not a sqlc name", alias, rest) + } + case kind == "file": + if !fs.ValidPath(rest) { + return fmt.Errorf("statement %s: %q is not a relative path inside the files directory", alias, rest) + } + default: + return fmt.Errorf("statement %s: unknown source %q (want old, new or file)", alias, kind) + } + } + if len(b.Pairs) == 0 { + return errors.New("no pairs") + } + ids := map[string]bool{} + for i := range b.Pairs { + p := &b.Pairs[i] + if ids[p.ID] { + return fmt.Errorf("pair %s: duplicate id", p.ID) + } + ids[p.ID] = true + if err := p.Validate(); err != nil { + return err + } + if err := p.checkCalls(b.Statements); err != nil { + return fmt.Errorf("pair %s: %w", p.ID, err) + } + } + return nil +} + +// Validate checks one pair on its own: its id, its classes and the fields each class needs, its walk and its +// expectations. Compare refuses a pair that fails it. +func (p DiffPair) Validate() error { + if err := checkFileName(p.ID); err != nil { + return fmt.Errorf("pair id: %w", err) + } + fail := func(format string, a ...any) error { + return fmt.Errorf("pair %s: "+format, append([]any{p.ID}, a...)...) + } + allowed := map[string]bool{} + for _, a := range p.Allowed { + if !slices.Contains(classOrder, a) { + return fail("unknown class %q (want %s)", a, strings.Join(classOrder, ", ")) + } + allowed[a] = true + } + if slices.Contains(p.Key, "") { + return fail("empty key column") + } + if allowed[ClassTieOrder] && len(p.Key) == 0 { + return fail("tie_order needs key") + } + if (len(p.Omit) > 0) != allowed[ClassLatestObserverTie] { + return fail("omit and the class latest_observer_tie go together (omit %v, allowed %v)", p.Omit, p.Allowed) + } + for _, c := range p.Omit { + if slices.Contains(p.Key, c) { + return fail("omit column %s is a key column", c) + } + } + if (p.Saturated != nil) != allowed[ClassSaturatedSuffix] { + return fail("saturated and the class saturated_suffix go together") + } + if p.Saturated != nil && (p.Saturated.Flag == "" || p.Saturated.Floor == "" || len(p.Key) == 0) { + return fail("saturated needs flag, floor and a key") + } + if (p.TieBoundary != nil) != allowed[ClassPageCursorTieRows] || (p.TieCursor != nil) != allowed[ClassPageCursorTieRows] { + return fail("tie_boundary, tie_cursor and the class pagecursor_tie_rows go together") + } + if tc := p.TieCursor; tc != nil && (tc.TS < 1 || tc.Hash < 1 || tc.TS == tc.Hash || tc.HashColumn == "") { + return fail("tie_cursor needs two different param numbers (ts, hash) and hash_column") + } + for _, c := range p.Require { + if !allowed[c] { + return fail("require %s: not an allowed class", c) + } + } + if p.TieBoundary != nil && (*p.TieBoundary < 1 || len(p.Key) == 0) { + return fail("tie_boundary %d must be a param number ($1 = 1) and needs a key", *p.TieBoundary) + } + for alias, want := range p.Expect { + if !validExpect[want] { + return fail("expect %s: %q (want identical, allowed, same or different)", alias, want) + } + } + if p.Walk != nil { + if err := p.Walk.validate(p); err != nil { + return fail("walk: %v", err) + } + } + return nil +} + +// checkCalls checks the pair's statements, params and expectations against the batch. +func (p DiffPair) checkCalls(stmts map[string]string) error { + if p.Ref.Stmt == "" { + return errors.New("no ref statement") + } + if len(p.Cands) == 0 { + return errors.New("no cands") + } + cands := map[string]bool{} + for _, c := range p.Cands { + if cands[c.Stmt] { + return fmt.Errorf("cand %s is listed twice", c.Stmt) + } + cands[c.Stmt] = true + } + for _, c := range append([]Call{p.Ref}, p.Cands...) { + if _, ok := stmts[c.Stmt]; !ok { + return fmt.Errorf("statement %q is not in the batch", c.Stmt) + } + for _, v := range c.Params { + if err := checkDiffParam(v); err != nil { + return fmt.Errorf("%s: %w", c.Stmt, err) + } + } + } + for alias := range p.Expect { + if !cands[alias] { + return fmt.Errorf("expect names %s, which is not a cand", alias) + } + } + if p.TieBoundary != nil && *p.TieBoundary > len(p.Ref.Params) { + return fmt.Errorf("tie_boundary $%d: the ref has %d params", *p.TieBoundary, len(p.Ref.Params)) + } + if tc := p.TieCursor; tc != nil { + for _, c := range p.Cands { + if max(tc.TS, tc.Hash) > len(c.Params) { + return fmt.Errorf("tie_cursor: %s has %d params", c.Stmt, len(c.Params)) + } + } + } + if p.Walk != nil { + for _, c := range append([]Call{p.Ref}, p.Cands...) { + if err := p.Walk.checkParams(len(c.Params)); err != nil { + return fmt.Errorf("walk: %s: %w", c.Stmt, err) + } + } + } + return nil +} + +// checkDiffParam accepts what renderParam renders without a corpus: scalars, lists, {raw: …} and {sql: …}, a +// single read-only SELECT resolved inside the batch's snapshot (no row or NULL: the pair is skipped). +func checkDiffParam(v any) error { + switch x := v.(type) { + case nil, string, int, int64, uint64, float64, bool: + return nil + case []any: + for _, y := range x { + if _, isMap := y.(map[string]any); isMap { + return errors.New("a list element must be a plain value") + } + if err := checkDiffParam(y); err != nil { + return err + } + } + return nil + case map[string]any: + if raw, ok := x["raw"].(string); ok && len(x) == 1 { + return checkRawParam(raw) + } + if q, ok := x["sql"].(string); ok && len(x) == 1 { + return checkSingleSelect(q) + } + return fmt.Errorf("param %v: want a value, a list, {raw: …} or {sql: …}", x) + } + return fmt.Errorf("unsupported param type %T", v) +} + +// checkSingleSelect refuses text that is more than one statement or that modifies data. Statements are sent over +// the simple protocol, where a second statement (a COMMIT, say) would run outside the READ ONLY transaction. +func checkSingleSelect(sql string) error { + if err := checkStatementText(sql); err != nil { + return err + } + if kw := dataModifying(sql); kw != "" { + return fmt.Errorf("runs %s; diff statements must not modify data", kw) + } + return nil +} + +// paramText is a param's value as text, the way it is sent before the server types it; res holds the values of +// {sql: …} params. +func paramText(v any, res map[string]*string) string { + switch x := v.(type) { + case nil: + return "" + case string: + return x + case map[string]any: + if raw, ok := x["raw"].(string); ok { + return raw + } + if key, _, ok := resolverRef(&Corpus{}, x); ok && res[key] != nil { + return *res[key] + } + return "" + } + lit, _, err := renderParam(&Corpus{}, v, nil, nil) + if err != nil { + return "" + } + if s, ok := unquoteLit(lit); ok { + return s + } + return lit +} + +// cell is one value as text for messages. +func cell(v *string) string { + if v == nil { + return "NULL" + } + s := *v + if len(s) > 60 { + s = s[:60] + "…" + } + return strconv.Quote(s) +} + +// colPlan maps a pair's columns to ref and cand positions. +type colPlan struct { + names []string // compared columns, in ref order + ref, cand []int + omit []string + omitRef, omitCnd []int + keyRef, keyCand []int + extra []string // cand columns not in ref +} + +func indexOf(cols []string) (map[string]int, error) { + m := map[string]int{} + for i, c := range cols { + if _, dup := m[c]; dup { + return nil, fmt.Errorf("column %s appears twice", c) + } + m[c] = i + } + return m, nil +} + +func planColumns(ref, cand RowSet, p DiffPair) (*colPlan, error) { + ri, err := indexOf(ref.Columns) + if err != nil { + return nil, fmt.Errorf("ref: %w", err) + } + ci, err := indexOf(cand.Columns) + if err != nil { + return nil, fmt.Errorf("cand: %w", err) + } + cp := &colPlan{omit: p.Omit} + omitted := map[string]bool{} + for _, c := range p.Omit { + r, okR := ri[c] + d, okC := ci[c] + if !okR || !okC { + return nil, fmt.Errorf("omit column %s is not in both results", c) + } + omitted[c] = true + cp.omitRef, cp.omitCnd = append(cp.omitRef, r), append(cp.omitCnd, d) + } + // The saturation columns describe the legacy scan, not the row; a cand without them is not missing a value. + control := map[string]bool{} + if p.Saturated != nil { + for _, c := range []string{p.Saturated.Flag, p.Saturated.Floor} { + if _, ok := ri[c]; !ok { + return nil, fmt.Errorf("saturated column %s is not in ref", c) + } + if _, ok := ci[c]; !ok { + control[c] = true + } + } + } + var missing []string + for i, c := range ref.Columns { + if omitted[c] || control[c] { + continue + } + j, ok := ci[c] + if !ok { + missing = append(missing, c) + continue + } + cp.names, cp.ref, cp.cand = append(cp.names, c), append(cp.ref, i), append(cp.cand, j) + } + if len(missing) > 0 { + return nil, fmt.Errorf("cand lacks the ref columns %s", strings.Join(missing, ", ")) + } + for _, c := range cand.Columns { + if _, ok := ri[c]; !ok { + cp.extra = append(cp.extra, c) + } + } + for _, k := range p.Key { + r, okR := ri[k] + d, okC := ci[k] + if !okR || !okC || omitted[k] { + return nil, fmt.Errorf("key column %s is not a compared column of both results", k) + } + cp.keyRef, cp.keyCand = append(cp.keyRef, r), append(cp.keyCand, d) + } + return cp, nil +} + +// tuple encodes the values at idx unambiguously (NULL apart from any text). +func tuple(row []*string, idx []int) string { + var b strings.Builder + for _, i := range idx { + if row[i] == nil { + b.WriteString("N;") + continue + } + fmt.Fprintf(&b, "S%d:%s;", len(*row[i]), *row[i]) + } + return b.String() +} + +func (cp *colPlan) keyText(row []*string, idx []int) string { + parts := make([]string, len(idx)) + for n, i := range idx { + v := "NULL" + if row[i] != nil { + v = *row[i] + } + parts[n] = cp.keyName(n) + "=" + v + } + return strings.Join(parts, ", ") +} + +func (cp *colPlan) keyName(n int) string { + for c, i := range cp.ref { + if i == cp.keyRef[n] { + return cp.names[c] + } + } + return "?" +} + +// rowLabel names a ref or cand row in a message: its key, else its first compared value. +func (cp *colPlan) rowLabel(row []*string, isRef bool) string { + if len(cp.keyRef) > 0 { + if isRef { + return "(" + cp.keyText(row, cp.keyRef) + ")" + } + return "(" + cp.keyText(row, cp.keyCand) + ")" + } + if len(cp.names) == 0 { + return "()" + } + idx := cp.ref[0] + if !isRef { + idx = cp.cand[0] + } + return "(" + cp.names[0] + "=" + cell(row[idx]) + ")" +} + +// parseTime reads a timestamptz in PostgreSQL's ISO text output. +func parseTime(s string) (time.Time, bool) { + for _, l := range []string{"2006-01-02 15:04:05.999999999-07", "2006-01-02 15:04:05.999999999-07:00", "2006-01-02 15:04:05.999999999-07:00:00", time.RFC3339Nano} { + if t, err := time.Parse(l, s); err == nil { + return t, true + } + } + return time.Time{}, false +} + +func cellTime(v *string) (time.Time, bool) { + if v == nil { + return time.Time{}, false + } + return parseTime(*v) +} + +// Compare compares a ref result with a cand result under the pair's classes. refParams and candParams are the +// calls' params as text (pagecursor_tie_rows reads both cursors from them). An invalid pair, or columns that do not +// line up, is different: nothing is relaxed that the pair does not name. +func Compare(ref, cand RowSet, p DiffPair, refParams, candParams []string) Outcome { + diff := func(format string, a ...any) Outcome { + return Outcome{Verdict: DiffDifferent, Detail: fmt.Sprintf(format, a...)} + } + if err := p.Validate(); err != nil { + return diff("invalid pair: %v", err) + } + cp, err := planColumns(ref, cand, p) + if err != nil { + return diff("%v", err) + } + allowed := map[string]bool{} + for _, a := range p.Allowed { + allowed[a] = true + } + rt := make([]string, len(ref.Rows)) + for i, r := range ref.Rows { + rt[i] = tuple(r, cp.ref) + } + ct := make([]string, len(cand.Rows)) + for i, r := range cand.Rows { + ct[i] = tuple(r, cp.cand) + } + counts := map[string]int{} + var notes []string + + // The aligned ranges: ref rows [r0, r1) against cand rows [c0, c1). + r0, r1, c0, c1 := 0, len(rt), 0, len(ct) + + if allowed[ClassPageCursorTieRows] { + if k, ms := leadTieRows(cand, rt, ct, cp, p, refParams, candParams); k > 0 && len(ct) == len(rt) { + c0, r1 = k, len(rt)-k + counts[ClassPageCursorTieRows] = k + notes = append(notes, fmt.Sprintf("pagecursor_tie_rows: cand rows 1..%d are at the cursor's millisecond %s and not in ref", k, ms)) + } + } + + if allowed[ClassSaturatedSuffix] && r1 > r0 { + if cut, floor, floorT, ok := saturatedCut(ref, cp, p, r0, r1); ok && cut < r1 { + n := cut - r0 // ref rows that must match + if c1-c0 < n || c1-c0 > r1-r0 { + return diff("row count: ref %d, cand %d; saturated_suffix relaxes only the %d ref rows below the floor %s", + r1-r0, c1-c0, r1-cut, floor) + } + if msg := relaxedTail(cand, ct, cp, c0, c0+n, c1, floorT); msg != "" { + return diff("saturated_suffix: %s (floor %s)", msg, floor) + } + changed := 0 + for i := 0; i < r1-cut; i++ { + if c0+n+i >= c1 || rt[cut+i] != ct[c0+n+i] { + changed++ + } + } + if changed > 0 { + counts[ClassSaturatedSuffix] = changed + notes = append(notes, fmt.Sprintf("saturated_suffix: %d of ref rows %d..%d (below the floor %s) are missing from or differ in cand", + changed, cut+1, r1, floor)) + } + r1, c1 = cut, c0+n + } + } + + if r1-r0 != c1-c0 { + return diff("row count: ref %d, cand %d%s", r1-r0, c1-c0, membership(ref, cand, rt[r0:r1], ct[c0:c1], cp, r0, c0)) + } + + // Pair ref rows with cand rows: by position, or within tie runs when the order differs only there. + pairs := make([]int, r1-r0) + first := -1 + for i := range pairs { + pairs[i] = c0 + i + if first < 0 && rt[r0+i] != ct[c0+i] { + first = i + } + } + if first >= 0 { + moved, ok := 0, false + if allowed[ClassTieOrder] { + moved, ok = tiePairs(ref, cand, rt, ct, cp, r0, c0, pairs) + } + if !ok { + return diff("first difference at row %d %s: %s%s", r0+first+1, cp.rowLabel(ref.Rows[r0+first], true), + columnDiffs(ref.Rows[r0+first], cand.Rows[c0+first], cp), membership(ref, cand, rt[r0:r1], ct[c0:c1], cp, r0, c0)) + } + counts[ClassTieOrder] = moved + notes = append(notes, fmt.Sprintf("tie_order: %d rows hold another position within their run of equal %s", moved, strings.Join(p.Key, ", "))) + } + + if len(cp.omit) > 0 { + var listed []string + for i, j := range pairs { + rr, cr := ref.Rows[r0+i], cand.Rows[j] + var cols []string + for n, c := range cp.omit { + if tuple(rr, cp.omitRef[n:n+1]) != tuple(cr, cp.omitCnd[n:n+1]) { + cols = append(cols, fmt.Sprintf("%s ref %s, cand %s", c, cell(rr[cp.omitRef[n]]), cell(cr[cp.omitCnd[n]]))) + } + } + if len(cols) > 0 { + listed = append(listed, fmt.Sprintf("row %d %s: %s", r0+i+1, cp.rowLabel(rr, true), strings.Join(cols, "; "))) + } + } + if len(listed) > 0 { + counts[ClassLatestObserverTie] = len(listed) + notes = append(notes, "latest_observer_tie: "+strings.Join(listed, " | ")) + } + } + + if len(cp.extra) > 0 { + notes = append(notes, "cand adds the columns "+strings.Join(cp.extra, ", ")) + } + o := Outcome{Verdict: DiffIdentical, Detail: strings.Join(notes, "; ")} + for _, c := range classOrder { + if n := counts[c]; n > 0 { + o.Classes = append(o.Classes, fmt.Sprintf("%s(%d)", c, n)) + } + } + if len(o.Classes) > 0 { + o.Verdict = DiffAllowed + } + return o +} + +// leadTieRows counts the leading cand rows that ref lacks, whose first key, truncated to the millisecond, is the +// millisecond of the ref's cursor param, and which sort at or below the cand's own cursor (ts, hash bound). +func leadTieRows(cand RowSet, rt, ct []string, cp *colPlan, p DiffPair, refParams, candParams []string) (int, string) { + boundary, tc := *p.TieBoundary, p.TieCursor + if boundary > len(refParams) || max(tc.TS, tc.Hash) > len(candParams) { + return 0, "" + } + b, ok := parseTime(refParams[boundary-1]) + if !ok { + return 0, "" + } + curTS, ok := parseTime(candParams[tc.TS-1]) + if !ok { + return 0, "" + } + curHash := strings.ToLower(candParams[tc.Hash-1]) + hi := slices.Index(cand.Columns, tc.HashColumn) + if hi < 0 { + return 0, "" + } + ms := b.Truncate(time.Millisecond) + inRef := map[string]bool{} + for _, t := range rt { + inRef[t] = true + } + k := 0 + for k < len(ct) && !inRef[ct[k]] { + t, ok := cellTime(cand.Rows[k][cp.keyCand[0]]) + if !ok || !t.Truncate(time.Millisecond).Equal(ms) { + break + } + h := cand.Rows[k][hi] + if t.After(curTS) || t.Equal(curTS) && (h == nil || strings.ToLower(*h) > curHash) { + break // above the cand's own cursor: no page from that cursor can hold it + } + k++ + } + return k, ms.UTC().Format("2006-01-02 15:04:05.000Z") +} + +// relaxedTail checks the cand rows [from, to) that stand in for ref rows below the floor: each is below the floor, +// they keep the descending key order from the cand row before them, and none repeats a matched row or another tail +// row (rows [c0, from) are the matched prefix). It returns why they do not, or "". +func relaxedTail(cand RowSet, ct []string, cp *colPlan, c0, from, to int, floor time.Time) string { + seen := map[string]bool{} + for j := c0; j < from; j++ { + seen[ct[j]] = true + } + var prev time.Time + havePrev := false + if from > c0 { + prev, havePrev = cellTime(cand.Rows[from-1][cp.keyCand[0]]) + } + for j := from; j < to; j++ { + row := cand.Rows[j] + t, ok := cellTime(row[cp.keyCand[0]]) + switch { + case !ok || !t.Before(floor): + return fmt.Sprintf("cand row %d %s stands in for a ref row below the floor but is not below it", j+1, cp.rowLabel(row, false)) + case havePrev && t.After(prev): + return fmt.Sprintf("cand row %d %s is newer than the row before it", j+1, cp.rowLabel(row, false)) + case seen[ct[j]]: + return fmt.Sprintf("cand row %d %s repeats an earlier row", j+1, cp.rowLabel(row, false)) + } + seen[ct[j]], prev, havePrev = true, t, true + } + return "" +} + +// saturatedCut returns the first ref row of the trailing run below the floor, when ref row 1 is saturated. +func saturatedCut(ref RowSet, cp *colPlan, p DiffPair, r0, r1 int) (int, string, time.Time, bool) { + fi := slices.Index(ref.Columns, p.Saturated.Flag) + li := slices.Index(ref.Columns, p.Saturated.Floor) + if len(ref.Rows) == 0 { + return 0, "", time.Time{}, false + } + first := ref.Rows[0] + if first[fi] == nil || *first[fi] != "t" { + return 0, "", time.Time{}, false + } + floor, ok := cellTime(first[li]) + if !ok { + return 0, "", time.Time{}, false + } + cut := r1 + for cut > r0 { + t, ok := cellTime(ref.Rows[cut-1][cp.keyRef[0]]) + if !ok || !t.Before(floor) { + break + } + cut-- + } + return cut, *first[li], floor, true +} + +// tiePairs checks that the keys are equal position by position and that each run of equal keys holds the same +// rows; it fills pairs (ref position → cand row) and returns how many rows moved. +func tiePairs(ref, cand RowSet, rt, ct []string, cp *colPlan, r0, c0 int, pairs []int) (int, bool) { + n := len(pairs) + for i := 0; i < n; i++ { + if tuple(ref.Rows[r0+i], cp.keyRef) != tuple(cand.Rows[c0+i], cp.keyCand) { + return 0, false + } + } + moved := 0 + for start := 0; start < n; { + end := start + 1 + k := tuple(ref.Rows[r0+start], cp.keyRef) + for end < n && tuple(ref.Rows[r0+end], cp.keyRef) == k { + end++ + } + free := map[string][]int{} + for j := start; j < end; j++ { + free[ct[c0+j]] = append(free[ct[c0+j]], c0+j) + } + for i := start; i < end; i++ { + // Prefer the cand row in the same position, then the first unused equal one. + t := rt[r0+i] + js := free[t] + if len(js) == 0 { + return 0, false + } + pick := 0 + if x := slices.Index(js, c0+i); x >= 0 { + pick = x + } else { + moved++ + } + pairs[i] = js[pick] + free[t] = slices.Delete(js, pick, pick+1) + } + start = end + } + return moved, true +} + +// columnDiffs names the compared columns that differ between two rows, with both values. +func columnDiffs(rr, cr []*string, cp *colPlan) string { + var out []string + for n, c := range cp.names { + if tuple(rr, cp.ref[n:n+1]) != tuple(cr, cp.cand[n:n+1]) { + out = append(out, fmt.Sprintf("%s ref %s, cand %s", c, cell(rr[cp.ref[n]]), cell(cr[cp.cand[n]]))) + } + } + return strings.Join(out, "; ") +} + +// membership lists ref rows missing from cand and cand rows not in ref (up to five of each). +func membership(ref, cand RowSet, rt, ct []string, cp *colPlan, r0, c0 int) string { + count := func(ts []string) map[string]int { + m := map[string]int{} + for _, t := range ts { + m[t]++ + } + return m + } + inC, inR := count(ct), count(rt) + list := func(ts []string, other map[string]int, rows [][]*string, off int, isRef bool) (string, int) { + var labels []string + n := 0 + for i, t := range ts { + if other[t] > 0 { + other[t]-- + continue + } + n++ + if len(labels) < 5 { + labels = append(labels, fmt.Sprintf("row %d %s", off+i+1, cp.rowLabel(rows[off+i], isRef))) + } + } + return strings.Join(labels, ", "), n + } + miss, nm := list(rt, inC, ref.Rows, r0, true) + extra, ne := list(ct, inR, cand.Rows, c0, false) + var out []string + if nm > 0 { + out = append(out, fmt.Sprintf("%d ref rows missing from cand: %s", nm, miss)) + } + if ne > 0 { + out = append(out, fmt.Sprintf("%d cand rows not in ref: %s", ne, extra)) + } + if len(out) == 0 { + return "" + } + return "; " + strings.Join(out, "; ") +} + +// expectMet reports whether a verdict meets an expectation ("" means same). +func expectMet(want, got string) bool { + if want == "" { + want = defaultExpect + } + if want == "same" { + return got == DiffIdentical || got == DiffAllowed + } + return want == got +} + +// worse returns the stronger of two verdicts: different over allowed over identical. +func worse(a, b string) string { + rank := map[string]int{DiffIdentical: 0, DiffAllowed: 1, DiffDifferent: 2} + if rank[b] > rank[a] { + return b + } + return a +} + +const ( + diffStmtPrefix = "planguard_d_" + diffTimeoutS = 600 +) + +// RunDiff runs every batch of spec on s and compares each pair's results. old: and new: statements take their text +// from oldSQL and newSQL (sqlc name → text), file: statements from fsys. +// +// Each batch is one BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY transaction (lock_timeout 1 s, statement_timeout +// 600 s). Every alias is PREPAREd as planguard_d_ and EXECUTEd over the simple protocol, so values arrive as +// the server's text. Each result is written to //...jsonl (walks: +// .walk...jsonl) and read back for the comparison. The batch ends with ROLLBACK and a +// DEALLOCATE of each alias, as separate commands, on every path. +// +// walkLimits are the page sizes to walk the pairs that have a walk at; 0 stands for each walk's web_limit. The +// report is written to /diff_report.md. The error is a batch that could not run; the report so far is +// returned with it. +func RunDiff(ctx context.Context, s Session, spec *DiffSpec, oldSQL, newSQL map[string]string, fsys fs.FS, outDir string, walkLimits []int) (*DiffReport, error) { + if err := spec.Validate(); err != nil { + return nil, err + } + texts := make([]map[string]string, len(spec.Batches)) + for i, b := range spec.Batches { + t, err := batchTexts(b, oldSQL, newSQL, fsys) + if err != nil { + return nil, fmt.Errorf("batch %s: %w", b.Name, err) + } + texts[i] = t + for _, p := range b.Pairs { + if p.Walk != nil && slices.Contains(walkLimits, 0) && p.Walk.WebLimit == 0 { + return nil, fmt.Errorf("batch %s: pair %s: a web walk needs web_limit", b.Name, p.ID) + } + } + } + for _, l := range walkLimits { + if l < 0 { + return nil, fmt.Errorf("walk limit %d, want ≥ 1 (or 0 for web)", l) + } + } + t0 := time.Now() + rep := &DiffReport{StartedAt: t0.UTC()} + var runErr error + for i, b := range spec.Batches { + br, err := runBatch(ctx, s, b, texts[i], outDir, walkLimits) + rep.Batches = append(rep.Batches, br) + if err != nil { + runErr = fmt.Errorf("batch %s: %w", b.Name, err) + break + } + } + rep.DurationMs = msSince(t0) + if err := os.MkdirAll(outDir, 0o755); err == nil { + err = os.WriteFile(filepath.Join(outDir, "diff_report.md"), []byte(rep.Markdown()), 0o644) + if err != nil && runErr == nil { + runErr = err + } + } else if runErr == nil { + runErr = err + } + return rep, runErr +} + +// batchTexts returns alias → statement text. +func batchTexts(b DiffBatch, oldSQL, newSQL map[string]string, fsys fs.FS) (map[string]string, error) { + out := map[string]string{} + for _, alias := range sortedKeys(b.Statements) { + kind, rest, _ := strings.Cut(b.Statements[alias], ":") + var text string + switch kind { + case "old", "new": + src := oldSQL + if kind == "new" { + src = newSQL + } + var ok bool + if text, ok = src[rest]; !ok { + return nil, fmt.Errorf("%s: no statement %s in the %s texts", alias, rest, kind) + } + case "file": + raw, err := fs.ReadFile(fsys, rest) + if err != nil { + return nil, fmt.Errorf("%s: %w", alias, err) + } + text = string(raw) + } + text = trimStmt(text) + if Normalize(text) == "" { + return nil, fmt.Errorf("%s: empty statement", alias) + } + if err := checkSingleSelect(text); err != nil { + return nil, fmt.Errorf("%s: %w", alias, err) + } + out[alias] = text + } + return out, nil +} + +// batchRun executes one batch's calls inside its transaction. +type batchRun struct { + ctx context.Context + s Session + b DiffBatch + dir string + res map[string]*string // {sql: …} param values, resolved in the batch's snapshot +} + +func runBatch(ctx context.Context, s Session, b DiffBatch, texts map[string]string, outDir string, walkLimits []int) (BatchReport, error) { + t0 := time.Now() + br := BatchReport{Name: b.Name} + r := &batchRun{ctx: ctx, s: s, b: b, dir: filepath.Join(outDir, b.Name)} + runErr := func() error { + if err := os.MkdirAll(r.dir, 0o755); err != nil { + return err + } + setup := []string{ + "BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY", + fmt.Sprintf("SET LOCAL statement_timeout = '%ds'; SET LOCAL lock_timeout = '1s'", diffTimeoutS), + "SET LOCAL jit = off; SET LOCAL timezone = 'UTC'; SET LOCAL standard_conforming_strings = on; " + + "SET LOCAL datestyle = 'ISO, MDY'; SET LOCAL intervalstyle = 'postgres'; SET LOCAL bytea_output = 'hex'; " + + "SET LOCAL extra_float_digits = 1", + } + if b.SearchPath != "" { + setup = append(setup, "SET LOCAL search_path = "+b.SearchPath) + } + for _, alias := range sortedKeys(texts) { + setup = append(setup, "PREPARE "+diffStmtPrefix+alias+" AS\n"+texts[alias]) + } + for _, l := range setup { + if _, err := s.Exec(ctx, l); err != nil { + return fmt.Errorf("%s: %s", firstLine(l), errText(err)) + } + } + if err := r.resolve(); err != nil { + return err + } + for _, p := range b.Pairs { + prs, err := r.pair(p, walkLimits) + br.Pairs = append(br.Pairs, prs...) + if err != nil { + return fmt.Errorf("pair %s: %w", p.ID, err) + } + } + return nil + }() + // Prepared statements outlive ROLLBACK: end the transaction, then deallocate each alias, as separate commands. + cctx, cancel := cleanupCtx(ctx) + defer cancel() + var cleanErr error + if err := rollback(cctx, s); err != nil { + cleanErr = fmt.Errorf("rollback: %w", err) + } + for _, alias := range sortedKeys(texts) { + if _, err := s.Exec(cctx, "DEALLOCATE "+diffStmtPrefix+alias); err != nil && !isSQLState(err, "26000") && cleanErr == nil { + cleanErr = fmt.Errorf("deallocate %s: %w", alias, err) + } + } + br.DurationMs = msSince(t0) + err := runErr + if err == nil { + err = cleanErr + } else if cleanErr != nil { + err = fmt.Errorf("%w (and %v)", runErr, cleanErr) + } + if err == nil { + err = ctx.Err() + } + if err != nil { + br.Err = err.Error() + } + return br, err +} + +// exec runs one call with params already rendered as literals. +func (r *batchRun) exec(alias string, lits []string) (RowSet, error) { + call := "EXECUTE " + diffStmtPrefix + alias + if len(lits) > 0 { + call += "(" + strings.Join(lits, ", ") + ")" + } + res, err := r.s.Exec(r.ctx, call) + if err != nil { + return RowSet{}, fmt.Errorf("%s: %s", alias, errText(err)) + } + if len(res) == 0 { + return RowSet{}, fmt.Errorf("%s: no result", alias) + } + last := res[len(res)-1] + rs := RowSet{Columns: make([]string, len(last.FieldDescriptions))} + for i, fd := range last.FieldDescriptions { + rs.Columns[i] = fd.Name + } + rs.Rows = make([][]*string, len(last.Rows)) + for i, row := range last.Rows { + out := make([]*string, len(row)) + for j, v := range row { + if v != nil { + t := string(v) + out[j] = &t + } + } + rs.Rows[i] = out + } + return rs, nil +} + +// resolve runs every {sql: …} param of the batch once, inside its snapshot. A resolver that fails stops the batch. +func (r *batchRun) resolve() error { + r.res = map[string]*string{} + var visit func(v any) error + visit = func(v any) error { + m, ok := v.(map[string]any) + if !ok { + return nil + } + key, q, ok := resolverRef(&Corpus{}, m) + if !ok || q == "" { + return nil + } + if _, done := r.res[key]; done { + return nil + } + res, err := r.s.Exec(r.ctx, resolverSelect(q)) + if err != nil { + return fmt.Errorf("resolver %s: %s", firstLine(q), errText(err)) + } + rows := lastRows(res) + if len(rows) == 1 && len(rows[0]) == 1 && rows[0][0] != nil { + v := string(rows[0][0]) + r.res[key] = &v + } else { + r.res[key] = nil + } + return nil + } + for _, p := range r.b.Pairs { + params := slices.Clone(p.Ref.Params) + for _, c := range p.Cands { + params = append(params, c.Params...) + } + if p.Walk != nil { + params = append(params, p.Walk.First...) + } + for _, v := range params { + if err := visit(v); err != nil { + return err + } + } + } + return nil +} + +// renderLits renders params as literals; skip names a {sql: …} param that resolved to no value. +func (r *batchRun) renderLits(params []any) ([]string, string, error) { + lits := make([]string, len(params)) + for i, v := range params { + lit, skip, err := renderParam(&Corpus{}, v, r.res, nil) + if skip != "" || err != nil { + return nil, skip, err + } + lits[i] = lit + } + return lits, "", nil +} + +// skipReason is why a pair cannot run on this snapshot: one of its {sql: …} params has no value. +func (r *batchRun) skipReason(p DiffPair) (string, error) { + calls := append([]Call{p.Ref}, p.Cands...) + if p.Walk != nil { + calls = append(calls, Call{Stmt: "walk.first", Params: p.Walk.First}) + } + for _, c := range calls { + if _, skip, err := r.renderLits(c.Params); skip != "" || err != nil { + return skip, err + } + } + return "", nil +} + +// roundTrip writes rs to its case file and reads it back, so every comparison reads what was written. +func (r *batchRun) roundTrip(name string, rs RowSet) (RowSet, error) { + path := filepath.Join(r.dir, name+".jsonl") + if err := WriteCaseFile(path, rs); err != nil { + return RowSet{}, err + } + return ReadCaseFile(path) +} + +func (r *batchRun) call(p DiffPair, role string, c Call) (RowSet, error) { + lits, _, err := r.renderLits(c.Params) + if err != nil { + return RowSet{}, err + } + rs, err := r.exec(c.Stmt, lits) + if err != nil { + return RowSet{}, err + } + return r.roundTrip(fmt.Sprintf("%s.%s.%s", p.ID, role, c.Stmt), rs) +} + +// fetcher pages through a call: the walk's cursor params from First or the cursor, its limit param limit+FetchExtra. +func (r *batchRun) fetcher(w *WalkSpec, c Call) (PageFetcher, error) { + base, _, err := r.renderLits(c.Params) + if err != nil { + return nil, err + } + first, _, err := r.renderLits(w.First) + if err != nil { + return nil, err + } + return func(_ context.Context, cursor []string, limit int) (RowSet, error) { + lits := slices.Clone(base) + for i, cb := range w.Cursor { + if cursor == nil { + lits[cb.Param-1] = first[i] + } else { + lits[cb.Param-1] = quoteLit(cursor[i]) + } + } + lits[w.LimitParam-1] = strconv.Itoa(limit + w.FetchExtra) + return r.exec(c.Stmt, lits) + }, nil +} + +// pair runs one pair: the ref, each cand and their comparison, then the walks. +func (r *batchRun) pair(p DiffPair, walkLimits []int) ([]PairResult, error) { + skip, err := r.skipReason(p) + if err != nil { + return nil, err + } + if skip != "" { + var out []PairResult + for _, c := range p.Cands { + out = append(out, PairResult{Batch: r.b.Name, Pair: p.ID, Ref: p.Ref.Stmt, Cand: c.Stmt, Verdict: DiffSkip, + Skip: skip, Expect: expectOf(p, c.Stmt)}) + } + return out, nil + } + ref, err := r.call(p, "ref", p.Ref) + if err != nil { + return nil, err + } + texts := func(params []any) []string { + out := make([]string, len(params)) + for i, v := range params { + out[i] = paramText(v, r.res) + } + return out + } + refParams := texts(p.Ref.Params) + var out []PairResult + for _, c := range p.Cands { + cand, err := r.call(p, "cand", c) + if err != nil { + return out, err + } + o := Compare(ref, cand, p, refParams, texts(c.Params)) + out = append(out, PairResult{Batch: r.b.Name, Pair: p.ID, Ref: p.Ref.Stmt, Cand: c.Stmt, + RefRows: len(ref.Rows), CandRows: len(cand.Rows), Outcome: o, Verdict: o.Verdict}) + } + if p.Walk != nil { + for _, l := range walkLimits { + if l == 0 { + l = p.Walk.WebLimit + } + if err := r.walks(p, l, out); err != nil { + return out, fmt.Errorf("walk %d: %w", l, err) + } + } + } + for i := range out { + out[i].Expect = expectOf(p, out[i].Cand) + out[i].OK = expectMet(out[i].Expect, out[i].Verdict) + for _, c := range p.Require { + if !slices.ContainsFunc(out[i].Outcome.Classes, func(x string) bool { return strings.HasPrefix(x, c+"(") }) { + out[i].OK = false + out[i].Missing = append(out[i].Missing, c) + } + } + } + return out, nil +} + +func expectOf(p DiffPair, cand string) string { + if e := p.Expect[cand]; e != "" { + return e + } + return defaultExpect +} + +// walks walks the ref (or fetches the oracle) and every cand at one page size. +func (r *batchRun) walks(p DiffPair, limit int, out []PairResult) error { + w := p.Walk + cols := w.cursorColumns() + maxRows := w.maxRows() + tag := fmt.Sprintf("%s.walk%d", p.ID, limit) + f, err := r.fetcher(w, p.Ref) + if err != nil { + return err + } + refLimit := limit + if w.Oracle { + refLimit = maxRows + } + ref, stuck, err := Walk(r.ctx, f, cols, refLimit, maxRows) + if err != nil { + return fmt.Errorf("ref %s: %w", p.Ref.Stmt, err) + } + if stuck != "" { + return fmt.Errorf("ref %s: %s", p.Ref.Stmt, stuck) + } + if ref, err = r.roundTrip(tag+".ref."+p.Ref.Stmt, ref); err != nil { + return err + } + for i, c := range p.Cands { + f, err := r.fetcher(w, c) + if err != nil { + return err + } + cand, stuck, err := Walk(r.ctx, f, cols, limit, maxRows) + if err != nil { + return fmt.Errorf("cand %s: %w", c.Stmt, err) + } + if cand, err = r.roundTrip(tag+".cand."+c.Stmt, cand); err != nil { + return err + } + wo, err := CompareWalk(ref, cand, cols) + if err != nil { + return fmt.Errorf("cand %s: %w", c.Stmt, err) + } + wo.Stuck = stuck + out[i].Walks = append(out[i].Walks, WalkResult{Limit: limit, RefRows: len(ref.Rows), Outcome: wo}) + if !wo.Clean() { + out[i].Verdict = worse(out[i].Verdict, DiffDifferent) + } + } + return nil +} diff --git a/internal/planguard/diff_integration_test.go b/internal/planguard/diff_integration_test.go new file mode 100644 index 0000000..958d694 --- /dev/null +++ b/internal/planguard/diff_integration_test.go @@ -0,0 +1,360 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package planguard_test + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/db" + pgdata "github.com/MeshCore-Beacon/beacon-server/db/planguard" + "github.com/MeshCore-Beacon/beacon-server/internal/planguard" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" +) + +const diffCases = `batches: + - name: snap + search_path: "SCHEMA" + statements: + old: "old:ListT" + new: "new:ListT" + ident: "file:ident.sql" + page: "file:page.sql" + oracle: "file:page.sql" + pairs: + - id: tie + ref: {stmt: old, params: []} + cands: [{stmt: new, params: []}] + key: [k] + allowed: [tie_order] + expect: {new: allowed} + - id: ident + ref: {stmt: ident, params: []} + cands: [{stmt: ident, params: []}] + expect: {ident: identical} + - id: walk + ref: {stmt: oracle, params: [2147483647, 2147483647, 8]} + cands: [{stmt: page, params: [2147483647, 2147483647, 8]}] + walk: + limit_param: 3 + fetch_extra: 1 + cursor: [{param: 1, column: k}, {param: 2, column: id}] + first: [2147483647, 2147483647] + web_limit: 50 + oracle: true + expect: {page: identical} + - id: resolved + ref: {stmt: page, params: [{sql: "SELECT max(k) FROM t"}, 2147483647, 5]} + cands: [{stmt: oracle, params: [{sql: "SELECT max(k) FROM t"}, 2147483647, 5]}] + expect: {oracle: identical} + - id: novalue + ref: {stmt: page, params: [{sql: "SELECT k FROM t WHERE false"}, 2147483647, 5]} + cands: [{stmt: oracle, params: [30, 2147483647, 5]}] +` + +const writeCases = `batches: + - name: write + search_path: "SCHEMA" + statements: + w: "file:write.sql" + pairs: + - id: w + ref: {stmt: w, params: []} + cands: [{stmt: w, params: []}] +` + +func TestRunDiffSnapshotPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for PostgreSQL tests") + } + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + admin, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + schema := "planguard_" + strings.ReplaceAll(uuid.NewString(), "-", "") + t.Cleanup(func() { + if _, err := admin.Exec(context.Background(), "DROP SCHEMA "+schema+" CASCADE"); err != nil { + t.Error(err) + } + admin.Close(context.Background()) + }) + if _, err := admin.Exec(ctx, ` +CREATE SCHEMA `+schema+`; +CREATE TABLE `+schema+`.t (id int PRIMARY KEY, k int NOT NULL, v text); +INSERT INTO `+schema+`.t SELECT g, g / 10, CASE WHEN g % 7 = 0 THEN NULL ELSE 'v' || g || E'\n"q",' END +FROM generate_series(1, 300) g; +CREATE FUNCTION `+schema+`.write_row() RETURNS int LANGUAGE sql AS + 'INSERT INTO `+schema+`.t VALUES (1000, 1000, ''written'') RETURNING id';`); err != nil { + t.Fatal(err) + } + pc, err := pgconn.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer pc.Close(context.Background()) + sess := planguard.NewSession(pc) + scalar := func(sql string) string { + t.Helper() + res, err := pc.Exec(ctx, sql).ReadAll() + if err != nil { + t.Fatal(err) + } + return string(res[len(res)-1].Rows[0][0]) + } + files := mapFS(map[string]string{ + "cases.yaml": strings.ReplaceAll(diffCases, "SCHEMA", schema), + "write.yaml": strings.ReplaceAll(writeCases, "SCHEMA", schema), + "ident.sql": "SELECT current_setting('transaction_isolation') AS iso, current_setting('transaction_read_only') AS ro;", + "page.sql": "SELECT id, k, v FROM t WHERE (k, id) < ($1, $2) ORDER BY k DESC, id DESC LIMIT $3", + "write.sql": "SELECT write_row() AS id", + }) + oldSQL := map[string]string{"ListT": "-- name: ListT :many\nSELECT id, k, v FROM t ORDER BY k, id"} + newSQL := map[string]string{"ListT": "-- name: ListT :many\nSELECT id, k, v FROM t ORDER BY k, id DESC;"} + + // A statement that writes fails in the READ ONLY transaction; nothing is written and nothing stays prepared. + wspec, err := planguard.LoadDiffSpec(files, "write.yaml") + if err != nil { + t.Fatal(err) + } + out := t.TempDir() + if _, err := planguard.RunDiff(ctx, sess, wspec, oldSQL, newSQL, files, out, nil); err == nil || !strings.Contains(err.Error(), "25006") { + t.Errorf("writing alias: err %v, want a 25006 read-only error", err) + } + if n := scalar("SELECT count(*) FROM " + schema + ".t"); n != "300" { + t.Errorf("%s rows after the writing alias, want 300", n) + } + if n := scalar("SELECT count(*) FROM pg_prepared_statements"); n != "0" { + t.Errorf("pg_prepared_statements = %s after a failed batch, want 0", n) + } + + spec, err := planguard.LoadDiffSpec(files, "cases.yaml") + if err != nil { + t.Fatal(err) + } + rep, err := planguard.RunDiff(ctx, sess, spec, oldSQL, newSQL, files, out, []int{7, 0}) + if err != nil { + t.Fatal(err) + } + got := map[string]planguard.PairResult{} + for _, b := range rep.Batches { + for _, p := range b.Pairs { + got[p.Pair+"|"+p.Cand] = p + } + } + tie := got["tie|new"] + if tie.Verdict != "allowed" || len(tie.Outcome.Classes) != 1 || !strings.HasPrefix(tie.Outcome.Classes[0], "tie_order(") || !tie.OK { + t.Errorf("tie pair: %+v, want allowed (tie_order)", tie) + } + if tie.RefRows != 300 || tie.CandRows != 300 { + t.Errorf("tie pair rows %d/%d, want 300/300", tie.RefRows, tie.CandRows) + } + if id := got["ident|ident"]; id.Verdict != "identical" || !id.OK { + t.Errorf("ident pair: %+v", id) + } + walk := got["walk|page"] + if walk.Verdict != "identical" || !walk.OK || len(walk.Walks) != 2 { + t.Fatalf("walk pair: %+v", walk) + } + for _, w := range walk.Walks { + if w.Outcome.Rows != 300 || w.RefRows != 300 || !w.Outcome.Clean() { + t.Errorf("walk at limit %d: %+v (ref %d rows)", w.Limit, w.Outcome, w.RefRows) + } + } + if walk.Walks[1].Limit != 50 { + t.Errorf("web limit %d, want 50", walk.Walks[1].Limit) + } + if len(rep.Failed()) != 0 { + t.Errorf("failed pairs: %+v", rep.Failed()) + } + // {sql: …} params are read in the snapshot; one that finds no value skips its pair. + if r := got["resolved|oracle"]; r.Verdict != "identical" || r.RefRows != 5 || !r.OK { + t.Errorf("resolved pair: %+v", r) + } + if r := got["novalue|oracle"]; r.Verdict != planguard.DiffSkip || r.OK || len(rep.Skipped()) != 1 { + t.Errorf("pair whose resolver finds nothing: %+v (skipped %d), want one skip", r, len(rep.Skipped())) + } + + // One case file per (pair, role, alias), read back for the comparison. + for _, f := range []string{"tie.ref.old", "tie.cand.new", "ident.ref.ident", "ident.cand.ident", "walk.ref.oracle", "walk.cand.page", + "walk.walk7.ref.oracle", "walk.walk7.cand.page", "walk.walk50.ref.oracle", "walk.walk50.cand.page"} { + if _, err := os.Stat(filepath.Join(out, "snap", f+".jsonl")); err != nil { + t.Errorf("case file %s: %v", f, err) + } + } + ident, err := planguard.ReadCaseFile(filepath.Join(out, "snap", "ident.ref.ident.jsonl")) + if err != nil { + t.Fatal(err) + } + if len(ident.Rows) != 1 || *ident.Rows[0][0] != "repeatable read" || *ident.Rows[0][1] != "on" { + t.Errorf("transaction isolation/read_only %v, want repeatable read / on", ident.Rows) + } + tieRef, err := planguard.ReadCaseFile(filepath.Join(out, "snap", "tie.ref.old.jsonl")) + if err != nil { + t.Fatal(err) + } + if v := tieRef.Rows[1][2]; v == nil || *v != "v2\n\"q\"," { + t.Errorf("row 2 v = %v, want the text with its newline, quotes and comma", v) + } + if tieRef.Rows[6][2] != nil { + t.Errorf("row 7 v = %q, want NULL", *tieRef.Rows[6][2]) + } + if _, err := os.Stat(filepath.Join(out, "diff_report.md")); err != nil { + t.Error(err) + } + if n := scalar("SELECT count(*) FROM pg_prepared_statements"); n != "0" { + t.Errorf("pg_prepared_statements = %s after RunDiff, want 0", n) + } + if st := pc.TxStatus(); st != 'I' { + t.Errorf("transaction status %q after RunDiff, want idle", st) + } + + // A pair that misses its expect is reported as failed. + spec.Batches[0].Pairs[0].Expect = map[string]string{"new": "identical"} + rep, err = planguard.RunDiff(ctx, sess, spec, oldSQL, newSQL, files, out, nil) + if err != nil { + t.Fatal(err) + } + if f := rep.Failed(); len(f) != 1 || f[0].Pair != "tie" { + t.Errorf("failed pairs %+v, want the tie pair", f) + } +} + +// The planted readmodels case on PostgreSQL: the frozen statement texts, walked over the in-memory walk test's +// data. The readmodels statement skips site B's next packet at limit 1 and is right at limit 7; the end-state +// statement matches the oracle at both. +func TestRunDiffFindsReadmodelsSkipPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for PostgreSQL tests") + } + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + admin, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + schema := "planguard_" + strings.ReplaceAll(uuid.NewString(), "-", "") + if _, err := admin.Exec(ctx, "CREATE SCHEMA "+schema); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if _, err := admin.Exec(context.Background(), "DROP SCHEMA "+schema+" CASCADE"); err != nil { + t.Error(err) + } + admin.Close(context.Background()) + }) + cfg, err := pgxpool.ParseConfig(dsn) + if err != nil { + t.Fatal(err) + } + cfg.ConnConfig.RuntimeParams["search_path"] = schema + cfg.MaxConns = 2 + pool, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err := db.RunMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + site := map[string]string{"A": "AAA", "B": "BBB", "C": "CCC"} + var hashes, iatas []string + var ats []int + for _, r := range walkData() { + hashes, iatas, ats = append(hashes, r.hash), append(iatas, site[r.site]), append(ats, r.at) + } + if _, err := pool.Exec(ctx, ` +INSERT INTO iata_codes (iata) VALUES ('AAA'), ('BBB'), ('CCC'); +INSERT INTO observers (id, public_key) VALUES ('00000000-0000-0000-0000-000000000001', '\x01'), + ('00000000-0000-0000-0000-000000000002', '\x02'), ('00000000-0000-0000-0000-000000000003', '\x03');`); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, ` +WITH o AS ( + SELECT decode(h, 'hex') AS h, i::bpchar AS iata, timestamptz '2026-10-18 00:00:00+00' + a * interval '1 second' AS at + FROM unnest($1::text[], $2::text[], $3::int[]) AS u(h, i, a)), +p AS ( + INSERT INTO packets (packet_hash, payload_type, payload_version, route_type, raw_payload, raw_header, + first_heard_at, last_heard_at, observation_count) + SELECT h, 4, 1, 1, '\x00', '\x00', min(at), max(at), count(*) FROM o GROUP BY h + RETURNING packet_hash) +INSERT INTO packet_observations (packet_hash, observer_id, iata, heard_at, path_length_byte, hash_size, hop_count) +SELECT o.h, ('00000000-0000-0000-0000-00000000000' || (ascii(o.iata) - 64))::uuid, o.iata, o.at, 0, 1, 0 +FROM o JOIN p ON p.packet_hash = o.h`, hashes, iatas, ats); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, ` +CREATE TABLE packet_sites AS +SELECT po.packet_hash, po.iata, max(po.heard_at) AS site_heard_at, p.payload_type, p.route_type, p.scope_id +FROM packet_observations po JOIN packets p USING (packet_hash) +GROUP BY po.packet_hash, po.iata, p.payload_type, p.route_type, p.scope_id; +CREATE INDEX ON packet_sites (iata, site_heard_at DESC, packet_hash DESC); +ANALYZE packet_sites;`); err != nil { + t.Fatal(err) + } + + spec, err := planguard.ParseDiffSpec([]byte(strings.ReplaceAll(`batches: + - name: planted + search_path: "SCHEMA" + statements: + rm: "file:sql/planted/ListPacketsBySitesReadmodels.sql" + end: "file:sql/pending/ListPacketsBySites.sql" + oracle: "file:sql/oracle/OracleBySites.sql" + pairs: + - id: walk + ref: {stmt: oracle, params: ["{AAA,BBB,CCC}", infinity, '\x', 2]} + cands: + - {stmt: rm, params: ["{AAA,BBB,CCC}", infinity, '\x', 2]} + - {stmt: end, params: ["{AAA,BBB,CCC}", infinity, '\x', 2]} + key: [site_heard_at] + walk: + limit_param: 4 + fetch_extra: 1 + cursor: [{param: 2, column: site_heard_at}, {param: 3, column: packet_hash}] + first: [infinity, '\x'] + oracle: true + expect: {rm: different, end: identical} +`, "SCHEMA", schema)), "planted.yaml", pgdata.Files) + if err != nil { + t.Fatal(err) + } + pc, err := pgconn.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer pc.Close(context.Background()) + rep, err := planguard.RunDiff(ctx, planguard.NewSession(pc), spec, nil, nil, pgdata.Files, t.TempDir(), []int{1, 2, 3, 7}) + if err != nil { + t.Fatal(err) + } + if f := rep.Failed(); len(f) != 0 { + t.Errorf("failed pairs: %+v", f) + } + for _, p := range rep.Batches[0].Pairs { + if len(p.Walks) != 4 { + t.Fatalf("%s: %d walks, want 4", p.Cand, len(p.Walks)) + } + for _, w := range p.Walks { + o := w.Outcome + switch { + case p.Cand == "rm" && w.Limit == 1: + if o.FirstWrong != 3 || o.Missing == 0 { + t.Errorf("rm at limit 1: %+v, want the first wrong row at 3 and rows missing", o) + } + case !o.Clean() || o.Rows != 40: + t.Errorf("%s at limit %d: %+v, want clean over 40 rows", p.Cand, w.Limit, o) + } + } + } +} diff --git a/internal/planguard/diff_report.go b/internal/planguard/diff_report.go new file mode 100644 index 0000000..29209c5 --- /dev/null +++ b/internal/planguard/diff_report.go @@ -0,0 +1,143 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package planguard + +import ( + "fmt" + "strings" + "time" +) + +// DiffReport is the outcome of RunDiff. +type DiffReport struct { + StartedAt time.Time + DurationMs float64 + Batches []BatchReport +} + +// BatchReport is one batch: its pairs, or the error that stopped it. +type BatchReport struct { + Name string + DurationMs float64 + Err string + Pairs []PairResult +} + +// PairResult is one (pair, cand): the snapshot comparison, the walks, and whether the verdict met the expectation. +type PairResult struct { + Batch, Pair, Ref, Cand string + RefRows, CandRows int + Outcome Outcome + Walks []WalkResult + Verdict string // the snapshot verdict combined with every walk's; skip when the pair did not run + Skip string // why the pair did not run + Expect string + Missing []string // required classes that did not fire + OK bool +} + +// WalkResult is one cand walk at one page size. +type WalkResult struct { + Limit int + RefRows int + Outcome WalkOutcome +} + +// Failed returns the pairs that missed their expectation. +func (r *DiffReport) Failed() []PairResult { + var out []PairResult + for _, b := range r.Batches { + for _, p := range b.Pairs { + if !p.OK && p.Verdict != DiffSkip { + out = append(out, p) + } + } + } + return out +} + +// Skipped returns the pairs that did not run: a {sql: …} param found no value on this snapshot. They prove nothing, +// so a run with skips does not pass either. +func (r *DiffReport) Skipped() []PairResult { + var out []PairResult + for _, b := range r.Batches { + for _, p := range b.Pairs { + if p.Verdict == DiffSkip { + out = append(out, p) + } + } + } + return out +} + +// Line is a one-line summary of a pair result. +func (p PairResult) Line() string { + mark := "ok" + switch { + case p.Verdict == DiffSkip: + return fmt.Sprintf("SKIP %s/%s %s vs %s: %s", p.Batch, p.Pair, p.Ref, p.Cand, p.Skip) + case !p.OK: + mark = "FAIL" + } + s := fmt.Sprintf("%-4s %s/%s %s vs %s: %s (expect %s), rows %d/%d", mark, p.Batch, p.Pair, p.Ref, p.Cand, p.Verdict, p.Expect, p.RefRows, p.CandRows) + if len(p.Outcome.Classes) > 0 { + s += " " + strings.Join(p.Outcome.Classes, " ") + } + if len(p.Missing) > 0 { + s += "; required classes that did not fire: " + strings.Join(p.Missing, ", ") + } + for _, w := range p.Walks { + if !w.Outcome.Clean() { + s += fmt.Sprintf("; walk %d: first wrong %d, missing %d", w.Limit, w.Outcome.FirstWrong, w.Outcome.Missing) + } + } + return s +} + +// Markdown renders the report. +func (r *DiffReport) Markdown() string { + var b strings.Builder + fmt.Fprintf(&b, "# planguard diff\n\nStarted %s, %.1f s. %d pairs missed their expectation, %d were skipped.\n\n", + r.StartedAt.UTC().Format(time.RFC3339), r.DurationMs/1000, len(r.Failed()), len(r.Skipped())) + esc := func(s string) string { return strings.ReplaceAll(strings.ReplaceAll(s, "|", `\|`), "\n", `\n`) } + for _, bt := range r.Batches { + fmt.Fprintf(&b, "## %s (one REPEATABLE READ READ ONLY snapshot, %.1f s)\n\n", bt.Name, bt.DurationMs/1000) + if bt.Err != "" { + fmt.Fprintf(&b, "**Error:** %s\n\n", esc(bt.Err)) + } + if len(bt.Pairs) == 0 { + continue + } + b.WriteString("| pair | ref | cand | rows ref/cand | verdict | expect | classes | detail |\n|---|---|---|---|---|---|---|---|\n") + var walks bool + for _, p := range bt.Pairs { + v := p.Verdict + detail := p.Outcome.Detail + switch { + case p.Verdict == DiffSkip: + v, detail = "**SKIP**", p.Skip + case !p.OK: + v = "**" + v + " (FAIL)**" + } + if len(p.Missing) > 0 { + detail = "required classes that did not fire: " + strings.Join(p.Missing, ", ") + "; " + detail + } + fmt.Fprintf(&b, "| %s | %s | %s | %d/%d | %s | %s | %s | %s |\n", esc(p.Pair), p.Ref, p.Cand, p.RefRows, p.CandRows, + v, p.Expect, strings.Join(p.Outcome.Classes, " "), esc(detail)) + walks = walks || len(p.Walks) > 0 + } + if walks { + b.WriteString("\n| pair | cand | limit | rows ref/cand | first wrong | missing | duplicates | out of order | stuck |\n|---|---|---|---|---|---|---|---|---|\n") + for _, p := range bt.Pairs { + for _, w := range p.Walks { + o := w.Outcome + fmt.Fprintf(&b, "| %s | %s | %d | %d/%d | %d | %d | %d | %d | %s |\n", esc(p.Pair), p.Cand, w.Limit, w.RefRows, o.Rows, + o.FirstWrong, o.Missing, o.Duplicates, o.OutOfOrder, o.Stuck) + } + } + } + b.WriteString("\n") + } + return b.String() +} diff --git a/internal/planguard/diff_test.go b/internal/planguard/diff_test.go new file mode 100644 index 0000000..e1da28d --- /dev/null +++ b/internal/planguard/diff_test.go @@ -0,0 +1,592 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package planguard_test + +import ( + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/planguard" +) + +func s(v string) *string { return &v } + +// rows builds a RowSet; the value "@@NULL" stands for NULL. +func rows(cols []string, rs ...[]string) planguard.RowSet { + out := planguard.RowSet{Columns: cols} + for _, r := range rs { + row := make([]*string, len(r)) + for i, v := range r { + if v != "@@NULL" { + row[i] = s(v) + } + } + out.Rows = append(out.Rows, row) + } + return out +} + +var tieCols = []string{"id", "k", "latest_observer_name"} + +func tieRef() planguard.RowSet { + return rows(tieCols, + []string{"1", "30", "a"}, + []string{"2", "20", "b"}, + []string{"3", "20", "c"}, + ) +} + +func pair(allowed ...string) planguard.DiffPair { + return planguard.DiffPair{ID: "p", Key: []string{"k"}, Allowed: allowed} +} + +func TestDiffComparesAllColumnsByDefault(t *testing.T) { + // A value changed in a column the pair names nowhere. + cand := tieRef() + cand.Rows[1][2] = s("B") + o := planguard.Compare(tieRef(), cand, pair(), nil, nil) + if o.Verdict != "different" { + t.Fatalf("value change in an unnamed column: %+v, want different", o) + } + if !strings.Contains(o.Detail, "row 2") || !strings.Contains(o.Detail, "latest_observer_name") { + t.Errorf("detail %q should name row 2 and the column", o.Detail) + } + + // NULL and "" differ. + nullCand := tieRef() + nullCand.Rows[0][2] = nil + emptyRef := tieRef() + emptyRef.Rows[0][2] = s("") + if o := planguard.Compare(emptyRef, nullCand, pair(), nil, nil); o.Verdict != "different" { + t.Errorf("NULL vs empty text: %+v, want different", o) + } + + // omit without latest_observer_tie is a spec error. + p := pair() + p.Omit = []string{"latest_observer_name"} + if err := p.Validate(); err == nil || !strings.Contains(err.Error(), "latest_observer_tie") { + t.Errorf("omit without latest_observer_tie: Validate() = %v, want an error naming the class", err) + } + if o := planguard.Compare(tieRef(), cand, p, nil, nil); o.Verdict != "different" || !strings.Contains(o.Detail, "invalid pair") { + t.Errorf("Compare on an invalid pair: %+v, want different (invalid pair)", o) + } + + // latest_observer_tie without omit is a spec error too. + if err := pair("latest_observer_tie").Validate(); err == nil { + t.Error("latest_observer_tie without omit: Validate() = nil, want an error") + } + + // With both, the omitted column may differ; the row is listed. + p = pair("latest_observer_tie") + p.Omit = []string{"latest_observer_name"} + if err := p.Validate(); err != nil { + t.Fatal(err) + } + o = planguard.Compare(tieRef(), cand, p, nil, nil) + if o.Verdict != "allowed" || !reflect.DeepEqual(o.Classes, []string{"latest_observer_tie(1)"}) { + t.Fatalf("omit + latest_observer_tie: %+v, want allowed [latest_observer_tie(1)]", o) + } + for _, want := range []string{"k=20", `"b"`, `"B"`} { + if !strings.Contains(o.Detail, want) { + t.Errorf("detail %q should list the row (%s)", o.Detail, want) + } + } + + // The omitted columns are the only relaxed ones: another column still counts. + cand2 := tieRef() + cand2.Rows[1][0] = s("9") + if o := planguard.Compare(tieRef(), cand2, p, nil, nil); o.Verdict != "different" { + t.Errorf("id changed under omit [latest_observer_name]: %+v, want different", o) + } + + // A ref column the cand lacks is a difference; an extra cand column is not. + short := rows([]string{"id", "k"}, []string{"1", "30"}, []string{"2", "20"}, []string{"3", "20"}) + if o := planguard.Compare(tieRef(), short, pair(), nil, nil); o.Verdict != "different" || !strings.Contains(o.Detail, "latest_observer_name") { + t.Errorf("cand lacks a column: %+v, want different naming it", o) + } + if o := planguard.Compare(short, tieRef(), pair(), nil, nil); o.Verdict != "identical" { + t.Errorf("cand adds a column: %+v, want identical", o) + } + + // Identical sets are identical. + if o := planguard.Compare(tieRef(), tieRef(), pair(), nil, nil); o.Verdict != "identical" || len(o.Classes) != 0 { + t.Errorf("same rows: %+v, want identical", o) + } +} + +func TestDiffTieOrder(t *testing.T) { + perm := rows(tieCols, + []string{"1", "30", "a"}, + []string{"3", "20", "c"}, + []string{"2", "20", "b"}, + ) + if o := planguard.Compare(tieRef(), perm, pair(), nil, nil); o.Verdict != "different" { + t.Errorf("tie permutation without tie_order: %+v, want different", o) + } + o := planguard.Compare(tieRef(), perm, pair("tie_order"), nil, nil) + if o.Verdict != "allowed" || !reflect.DeepEqual(o.Classes, []string{"tie_order(2)"}) { + t.Errorf("tie permutation with tie_order: %+v, want allowed [tie_order(2)]", o) + } + // Rows moved across different keys are not a tie. + cross := rows(tieCols, + []string{"2", "20", "b"}, + []string{"1", "30", "a"}, + []string{"3", "20", "c"}, + ) + if o := planguard.Compare(tieRef(), cross, pair("tie_order"), nil, nil); o.Verdict != "different" { + t.Errorf("rows moved across keys: %+v, want different", o) + } + // A tie run whose rows are swapped for another row is not a permutation. + swapped := rows(tieCols, + []string{"1", "30", "a"}, + []string{"3", "20", "c"}, + []string{"4", "20", "d"}, + ) + if o := planguard.Compare(tieRef(), swapped, pair("tie_order"), nil, nil); o.Verdict != "different" { + t.Errorf("tie run with another row: %+v, want different", o) + } +} + +func TestDiffOnlySaturatedSuffixRelaxesCount(t *testing.T) { + // Tied rows permuted AND a duplicated trailing row (ref 3, cand 4). + permDup := rows(tieCols, + []string{"1", "30", "a"}, + []string{"3", "20", "c"}, + []string{"2", "20", "b"}, + []string{"2", "20", "b"}, + ) + if o := planguard.Compare(tieRef(), permDup, pair("tie_order"), nil, nil); o.Verdict != "different" { + t.Errorf("permutation + duplicate row: %+v, want different", o) + } + // A duplicated trailing row only. + dupOnly := rows(tieCols, + []string{"1", "30", "a"}, + []string{"2", "20", "b"}, + []string{"3", "20", "c"}, + []string{"3", "20", "c"}, + ) + o := planguard.Compare(tieRef(), dupOnly, pair("tie_order"), nil, nil) + if o.Verdict != "different" || !strings.Contains(o.Detail, "row count") { + t.Errorf("duplicate row: %+v, want different (row count)", o) + } + + // A saturated ref: flag t, floor at row 2's key; row 3 is below the floor. + cols := []string{"h", "site_heard_at", "scan_saturated", "scan_floor"} + ref := rows(cols, + []string{"\\xaa", "2026-10-18 15:40:39.570763+00", "t", "2026-10-18 15:40:39.5+00"}, + []string{"\\xbb", "2026-10-18 15:40:39.5+00", "t", "2026-10-18 15:40:39.5+00"}, + []string{"\\xcc", "2026-10-18 15:40:38.1+00", "t", "2026-10-18 15:40:39.5+00"}, + ) + candCols := []string{"h", "site_heard_at"} + sat := planguard.DiffPair{ + ID: "sat", Key: []string{"site_heard_at"}, Allowed: []string{"saturated_suffix"}, + Saturated: &planguard.SaturatedSpec{Flag: "scan_saturated", Floor: "scan_floor"}, + } + if err := sat.Validate(); err != nil { + t.Fatal(err) + } + missingBelow := rows(candCols, []string{"\\xaa", "2026-10-18 15:40:39.570763+00"}, []string{"\\xbb", "2026-10-18 15:40:39.5+00"}) + o = planguard.Compare(ref, missingBelow, sat, nil, nil) + if o.Verdict != "allowed" || !reflect.DeepEqual(o.Classes, []string{"saturated_suffix(1)"}) { + t.Errorf("row below the floor missing: %+v, want allowed [saturated_suffix(1)]", o) + } + differsBelow := rows(candCols, []string{"\\xaa", "2026-10-18 15:40:39.570763+00"}, []string{"\\xbb", "2026-10-18 15:40:39.5+00"}, []string{"\\xdd", "2026-10-18 15:40:37+00"}) + if o := planguard.Compare(ref, differsBelow, sat, nil, nil); o.Verdict != "allowed" { + t.Errorf("row below the floor differs: %+v, want allowed", o) + } + missingAbove := rows(candCols, []string{"\\xaa", "2026-10-18 15:40:39.570763+00"}, []string{"\\xcc", "2026-10-18 15:40:38.1+00"}) + if o := planguard.Compare(ref, missingAbove, sat, nil, nil); o.Verdict != "different" { + t.Errorf("row above the floor missing: %+v, want different", o) + } + // The cand rows in place of the relaxed ref rows: below the floor, in order, and new. + dupTail := rows(candCols, []string{"\\xaa", "2026-10-18 15:40:39.570763+00"}, []string{"\\xbb", "2026-10-18 15:40:39.5+00"}, []string{"\\xbb", "2026-10-18 15:40:39.5+00"}) + if o := planguard.Compare(ref, dupTail, sat, nil, nil); o.Verdict != "different" { + t.Errorf("relaxed tail repeats a row: %+v, want different", o) + } + newerTail := rows(candCols, []string{"\\xaa", "2026-10-18 15:40:39.570763+00"}, []string{"\\xbb", "2026-10-18 15:40:39.5+00"}, []string{"\\xee", "2026-10-18 15:40:40+00"}) + if o := planguard.Compare(ref, newerTail, sat, nil, nil); o.Verdict != "different" { + t.Errorf("relaxed tail row newer than the floor and out of order: %+v, want different", o) + } + ref4 := rows(cols, + []string{"\\xaa", "2026-10-18 15:40:39.570763+00", "t", "2026-10-18 15:40:39.5+00"}, + []string{"\\xbb", "2026-10-18 15:40:39.5+00", "t", "2026-10-18 15:40:39.5+00"}, + []string{"\\xcc", "2026-10-18 15:40:38.1+00", "t", "2026-10-18 15:40:39.5+00"}, + []string{"\\xdd", "2026-10-18 15:40:37+00", "t", "2026-10-18 15:40:39.5+00"}, + ) + unordered := rows(candCols, []string{"\\xaa", "2026-10-18 15:40:39.570763+00"}, []string{"\\xbb", "2026-10-18 15:40:39.5+00"}, + []string{"\\xff", "2026-10-18 15:40:36+00"}, []string{"\\xee", "2026-10-18 15:40:38+00"}) + if o := planguard.Compare(ref4, unordered, sat, nil, nil); o.Verdict != "different" { + t.Errorf("relaxed tail rows below the floor out of order: %+v, want different", o) + } + dupBelow := rows(candCols, []string{"\\xaa", "2026-10-18 15:40:39.570763+00"}, []string{"\\xbb", "2026-10-18 15:40:39.5+00"}, + []string{"\\xee", "2026-10-18 15:40:38+00"}, []string{"\\xee", "2026-10-18 15:40:38+00"}) + if o := planguard.Compare(ref4, dupBelow, sat, nil, nil); o.Verdict != "different" { + t.Errorf("relaxed tail repeats a row below the floor: %+v, want different", o) + } + atFloor := rows(candCols, []string{"\\xaa", "2026-10-18 15:40:39.570763+00"}, []string{"\\xbb", "2026-10-18 15:40:39.5+00"}, + []string{"\\xab", "2026-10-18 15:40:39.5+00"}, []string{"\\xff", "2026-10-18 15:40:36+00"}) + if o := planguard.Compare(ref4, atFloor, sat, nil, nil); o.Verdict != "different" { + t.Errorf("relaxed tail row at the floor, in order and new: %+v, want different", o) + } + ordered := rows(candCols, []string{"\\xaa", "2026-10-18 15:40:39.570763+00"}, []string{"\\xbb", "2026-10-18 15:40:39.5+00"}, + []string{"\\xee", "2026-10-18 15:40:38+00"}, []string{"\\xff", "2026-10-18 15:40:36+00"}) + if o := planguard.Compare(ref4, ordered, sat, nil, nil); o.Verdict != "allowed" { + t.Errorf("relaxed tail rows below the floor in order: %+v, want allowed", o) + } + extra := rows(candCols, []string{"\\xaa", "2026-10-18 15:40:39.570763+00"}, []string{"\\xbb", "2026-10-18 15:40:39.5+00"}, + []string{"\\xcc", "2026-10-18 15:40:38.1+00"}, []string{"\\xdd", "2026-10-18 15:40:37+00"}) + if o := planguard.Compare(ref, extra, sat, nil, nil); o.Verdict != "different" { + t.Errorf("cand adds a row: %+v, want different (only ref rows below the floor are relaxed)", o) + } + unsat := rows(cols, + []string{"\\xaa", "2026-10-18 15:40:39.570763+00", "f", "@@NULL"}, + []string{"\\xbb", "2026-10-18 15:40:39.5+00", "f", "@@NULL"}, + []string{"\\xcc", "2026-10-18 15:40:38.1+00", "f", "@@NULL"}, + ) + if o := planguard.Compare(unsat, missingBelow, sat, nil, nil); o.Verdict != "different" { + t.Errorf("unsaturated ref, cand missing a row: %+v, want different", o) + } + // Without the class the same cand is a difference. + plain := planguard.DiffPair{ID: "sat", Key: []string{"site_heard_at"}} + if o := planguard.Compare(ref, missingBelow, plain, nil, nil); o.Verdict != "different" { + t.Errorf("saturated ref without the class: %+v, want different", o) + } + // saturated without the class, or the class without saturated, is a spec error. + bad := sat + bad.Allowed = nil + if err := bad.Validate(); err == nil { + t.Error("saturated without saturated_suffix: Validate() = nil") + } + bad = sat + bad.Saturated = nil + if err := bad.Validate(); err == nil { + t.Error("saturated_suffix without saturated: Validate() = nil") + } +} + +func TestPageCursorTieRows(t *testing.T) { + cols := []string{"packet_hash", "site_heard_at"} + ref := rows(cols, + []string{"\\x01", "2026-10-18 15:40:39.5+00"}, + []string{"\\x02", "2026-10-18 15:40:39.4+00"}, + []string{"\\x03", "2026-10-18 15:40:39.3+00"}, + ) + tb := 2 + p := planguard.DiffPair{ID: "pc", Key: []string{"site_heard_at"}, Allowed: []string{"pagecursor_tie_rows"}, TieBoundary: &tb, + TieCursor: &planguard.TieCursor{TS: 2, Hash: 3, HashColumn: "packet_hash"}} + if err := p.Validate(); err != nil { + t.Fatal(err) + } + params := []string{"{AMS}", "2026-10-18 15:40:39.570+00", "16", "4"} + candParams := []string{"{AMS}", "2026-10-18 15:40:39.570763+00", "\\x0a", "4"} + cand := rows(cols, + []string{"\\x09", "2026-10-18 15:40:39.570763+00"}, + []string{"\\x01", "2026-10-18 15:40:39.5+00"}, + []string{"\\x02", "2026-10-18 15:40:39.4+00"}, + ) + o := planguard.Compare(ref, cand, p, params, candParams) + if o.Verdict != "allowed" || !reflect.DeepEqual(o.Classes, []string{"pagecursor_tie_rows(1)"}) { + t.Errorf("a lead row at the cursor's millisecond: %+v, want allowed [pagecursor_tie_rows(1)]", o) + } + early := rows(cols, + []string{"\\x09", "2026-10-18 15:40:39.569999+00"}, + []string{"\\x01", "2026-10-18 15:40:39.5+00"}, + []string{"\\x02", "2026-10-18 15:40:39.4+00"}, + ) + if o := planguard.Compare(ref, early, p, params, candParams); o.Verdict != "different" { + t.Errorf("a lead row 1 ms earlier: %+v, want different", o) + } + if o := planguard.Compare(ref, cand, planguard.DiffPair{ID: "pc", Key: []string{"site_heard_at"}}, params, candParams); o.Verdict != "different" { + t.Errorf("a lead row without the class: %+v, want different", o) + } + // The counts must be equal: a lead row plus all of ref is a difference. + longer := rows(cols, + []string{"\\x09", "2026-10-18 15:40:39.570763+00"}, + []string{"\\x01", "2026-10-18 15:40:39.5+00"}, + []string{"\\x02", "2026-10-18 15:40:39.4+00"}, + []string{"\\x03", "2026-10-18 15:40:39.3+00"}, + ) + if o := planguard.Compare(ref, longer, p, params, candParams); o.Verdict != "different" { + t.Errorf("lead row + every ref row: %+v, want different", o) + } + // cand[k:] must equal ref[:n-k]. + shifted := rows(cols, + []string{"\\x09", "2026-10-18 15:40:39.570763+00"}, + []string{"\\x01", "2026-10-18 15:40:39.5+00"}, + []string{"\\x03", "2026-10-18 15:40:39.3+00"}, + ) + if o := planguard.Compare(ref, shifted, p, params, candParams); o.Verdict != "different" { + t.Errorf("lead row, then a row missing: %+v, want different", o) + } + // Only rows ref lacks are lead rows: a ref row in the cursor's millisecond is compared as usual. + atMs := rows(cols, + []string{"\\x09", "2026-10-18 15:40:39.570763+00"}, + []string{"\\x01", "2026-10-18 15:40:39.5+00"}, + ) + if o := planguard.Compare(atMs, atMs, p, params, candParams); o.Verdict != "identical" { + t.Errorf("ref and cand both hold the row in the cursor's millisecond: %+v, want identical", o) + } + // Lead rows must sort at or below the cand's own cursor: a row above it cannot come from that cursor. + above := rows(cols, + []string{"\\x09", "2026-10-18 15:40:39.570999+00"}, + []string{"\\x01", "2026-10-18 15:40:39.5+00"}, + []string{"\\x02", "2026-10-18 15:40:39.4+00"}, + ) + if o := planguard.Compare(ref, above, p, params, candParams); o.Verdict != "different" { + t.Errorf("a lead row at .570999 above the cand cursor .570763: %+v, want different", o) + } + sameTS := rows(cols, + []string{"\\x0b", "2026-10-18 15:40:39.570763+00"}, + []string{"\\x01", "2026-10-18 15:40:39.5+00"}, + []string{"\\x02", "2026-10-18 15:40:39.4+00"}, + ) + if o := planguard.Compare(ref, sameTS, p, params, candParams); o.Verdict != "different" { + t.Errorf("a lead row at the cursor's ts with a hash above its bound: %+v, want different", o) + } + if o := planguard.Compare(ref, cand, p, params, nil); o.Verdict != "different" { + t.Errorf("no cand params: %+v, want different (the class needs the cand cursor)", o) + } + bad := p + bad.TieBoundary = nil + if err := bad.Validate(); err == nil { + t.Error("pagecursor_tie_rows without tie_boundary: Validate() = nil") + } + bad = p + bad.TieCursor = nil + if err := bad.Validate(); err == nil { + t.Error("pagecursor_tie_rows without tie_cursor: Validate() = nil") + } +} + +func TestDiffFailsOnParseError(t *testing.T) { + dir := t.TempDir() + bad := filepath.Join(dir, "bad.jsonl") + if err := os.WriteFile(bad, []byte("{\"columns\":[\"a\",\"b\"],\"rows\":2}\n[\"1\",\"2\"]\n[\"a\", \"b\"\n"), 0o644); err != nil { + t.Fatal(err) + } + if _, err := planguard.ReadCaseFile(bad); err == nil || !strings.Contains(err.Error(), "line 3") { + t.Errorf("unterminated JSON on line 3: err %v, want an error naming line 3", err) + } + + arity := filepath.Join(dir, "arity.jsonl") + if err := os.WriteFile(arity, []byte("{\"columns\":[\"a\",\"b\",\"c\"],\"rows\":1}\n[\"1\",\"2\"]\n"), 0o644); err != nil { + t.Fatal(err) + } + if _, err := planguard.ReadCaseFile(arity); err == nil || !strings.Contains(err.Error(), "line 2") { + t.Errorf("row of arity 2 under 3 columns: err %v, want an error naming line 2", err) + } + + for name, body := range map[string]string{ + "empty": "", + "no header": "[\"1\",\"2\"]\n", + "truncated": "{\"columns\":[\"a\"],\"rows\":2}\n[\"1\"]\n", + "no newline": "{\"columns\":[\"a\"],\"rows\":1}\n[\"1\"]", + "number cell": "{\"columns\":[\"a\"],\"rows\":1}\n[1]\n", + "extra rows": "{\"columns\":[\"a\"],\"rows\":1}\n[\"1\"]\n[\"2\"]\n", + } { + p := filepath.Join(dir, "x.jsonl") + if err := os.WriteFile(p, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + if _, err := planguard.ReadCaseFile(p); err == nil { + t.Errorf("%s: ReadCaseFile accepted %q", name, body) + } + } + + // Text that looks like the old format's case marker round-trips, and a later row still counts. + marker := "x\n@@CASE p|ref|old\nmore" + ref := rows([]string{"id", "k"}, []string{"1", marker}, []string{"2", "b"}) + cand := rows([]string{"id", "k"}, []string{"1", marker}, []string{"2", "B"}) + rp, cp := filepath.Join(dir, "ref.jsonl"), filepath.Join(dir, "cand.jsonl") + if err := planguard.WriteCaseFile(rp, ref); err != nil { + t.Fatal(err) + } + if err := planguard.WriteCaseFile(cp, cand); err != nil { + t.Fatal(err) + } + ref2, err := planguard.ReadCaseFile(rp) + if err != nil { + t.Fatal(err) + } + cand2, err := planguard.ReadCaseFile(cp) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(ref2, ref) { + t.Errorf("marker text did not round-trip: %+v", ref2) + } + o := planguard.Compare(ref2, cand2, planguard.DiffPair{ID: "marker", Key: []string{"id"}}, nil, nil) + if o.Verdict != "different" || !strings.Contains(o.Detail, "row 2") { + t.Errorf("marker text: %+v, want different at row 2", o) + } +} + +func TestCaseFileRoundTripsHostileText(t *testing.T) { + hostile := []string{ + `he said "hi", then left`, + "a,b,c", + "line1\r\nline2\n", + "tab\there", + "unicode: ünïcødé ✓ 🛰", + "", + "@@NULL", + `back\slash \x00 not a nul`, + "\u0001\u001f control", + "]}\n{\"columns\":[]}", + "

", + } + rs := planguard.RowSet{Columns: []string{"v", "n"}} + for _, h := range hostile { + rs.Rows = append(rs.Rows, []*string{s(h), nil}) + } + rs.Rows = append(rs.Rows, []*string{nil, s("")}) + p := filepath.Join(t.TempDir(), "h.jsonl") + if err := planguard.WriteCaseFile(p, rs); err != nil { + t.Fatal(err) + } + raw, err := os.ReadFile(p) + if err != nil { + t.Fatal(err) + } + if n := strings.Count(string(raw), "\n"); n != len(rs.Rows)+1 { + t.Errorf("%d lines, want one per row plus the header (%d)", n, len(rs.Rows)+1) + } + got, err := planguard.ReadCaseFile(p) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, rs) { + t.Errorf("round trip differs:\n got %+v\nwant %+v", got, rs) + } + // An empty result keeps its columns. + empty := planguard.RowSet{Columns: []string{"a"}} + if err := planguard.WriteCaseFile(p, empty); err != nil { + t.Fatal(err) + } + if got, err := planguard.ReadCaseFile(p); err != nil || !reflect.DeepEqual(got.Columns, empty.Columns) || len(got.Rows) != 0 { + t.Errorf("empty result: %+v, %v", got, err) + } + // Text that is not UTF-8 cannot be written losslessly. + if err := planguard.WriteCaseFile(p, planguard.RowSet{Columns: []string{"a"}, Rows: [][]*string{{s("\xff")}}}); err == nil { + t.Error("WriteCaseFile accepted invalid UTF-8") + } + // A row whose arity differs from the columns is refused on write too. + if err := planguard.WriteCaseFile(p, planguard.RowSet{Columns: []string{"a"}, Rows: [][]*string{{s("1"), s("2")}}}); err == nil { + t.Error("WriteCaseFile accepted a row of the wrong arity") + } +} + +func TestDiffSpecValidation(t *testing.T) { + base := `batches: + - name: b + search_path: "t_x, public" + statements: + ref: "file:ref.sql" + cand: "new:ListT" + pairs: + - id: p1 + ref: {stmt: ref, params: [1]} + cands: [{stmt: cand, params: [1]}] + key: [k] + allowed: [tie_order] + expect: {cand: allowed} +` + fsys := mapFS(map[string]string{"d.yaml": base, "ref.sql": "SELECT 1"}) + spec, err := planguard.LoadDiffSpec(fsys, "d.yaml") + if err != nil { + t.Fatal(err) + } + if len(spec.Batches) != 1 || len(spec.Batches[0].Pairs) != 1 { + t.Fatalf("spec %+v", spec) + } + for name, edit := range map[string][2]string{ + "unknown class": {"allowed: [tie_order]", "allowed: [close_enough]"}, + "unknown alias": {"cands: [{stmt: cand", "cands: [{stmt: nope"}, + "bad expect": {"expect: {cand: allowed}", "expect: {cand: maybe}"}, + "expect non-cand": {"expect: {cand: allowed}", "expect: {ref: same}"}, + "bad statement kind": {`cand: "new:ListT"`, `cand: "head:ListT"`}, + "bad alias name": {" cand: \"new:ListT\"", " Cand-1: \"new:ListT\""}, + "bad pair id": {"id: p1", "id: ../p1"}, + "bad search_path": {`search_path: "t_x, public"`, `search_path: "t_x; DROP"`}, + "unknown field": {"key: [k]", "key: [k]\n colums: [a]"}, + "tie_order no key": {"key: [k]", "key: []"}, + "file escapes": {`ref: "file:ref.sql"`, `ref: "file:../ref.sql"`}, + "require not allowed": {"key: [k]", "key: [k]\n require: [saturated_suffix]"}, + "sql param two stmts": {"ref: {stmt: ref, params: [1]}", "ref: {stmt: ref, params: [{sql: \"SELECT 1; COMMIT\"}]}"}, + "sql param writes": {"ref: {stmt: ref, params: [1]}", "ref: {stmt: ref, params: [{sql: \"DELETE FROM t RETURNING 1\"}]}"}, + "sql param and more": {"ref: {stmt: ref, params: [1]}", "ref: {stmt: ref, params: [{sql: \"SELECT 1\", raw: \"2\"}]}"}, + } { + body := strings.Replace(base, edit[0], edit[1], 1) + if body == base { + t.Fatalf("%s: edit did not apply", name) + } + if _, err := planguard.LoadDiffSpec(mapFS(map[string]string{"d.yaml": body, "ref.sql": "SELECT 1"}), "d.yaml"); err == nil { + t.Errorf("%s: LoadDiffSpec accepted\n%s", name, body) + } + } + // A data-modifying statement is refused before anything runs. + if _, err := planguard.LoadDiffSpec(mapFS(map[string]string{"d.yaml": base, "ref.sql": "INSERT INTO t VALUES (1)"}), "d.yaml"); err == nil { + t.Error("LoadDiffSpec accepted an INSERT statement") + } + // Statements go over the simple protocol, where a second statement (a COMMIT) would leave the READ ONLY + // transaction: more than one statement is refused. A ';' in a literal or comment, or one trailing, is fine. + if _, err := planguard.LoadDiffSpec(mapFS(map[string]string{"d.yaml": base, "ref.sql": "SELECT 1; COMMIT; SELECT 2"}), "d.yaml"); err == nil { + t.Error("LoadDiffSpec accepted two statements") + } + if _, err := planguard.LoadDiffSpec(mapFS(map[string]string{"d.yaml": base, "ref.sql": "SELECT ';' AS a, $$;$$ AS b -- ;\n;\n"}), "d.yaml"); err != nil { + t.Errorf("';' in literals and comments: %v", err) + } + ok := strings.Replace(base, "ref: {stmt: ref, params: [1]}", "ref: {stmt: ref, params: [{sql: \"SELECT 1\"}]}", 1) + if _, err := planguard.LoadDiffSpec(mapFS(map[string]string{"d.yaml": ok, "ref.sql": "SELECT 1"}), "d.yaml"); err != nil { + t.Errorf("a {sql: …} param: %v", err) + } +} + +// The committed diff cases load, and every statement they name exists. +func TestRepoDiffCasesLoad(t *testing.T) { + root := "../../db/planguard" + files, err := filepath.Glob(filepath.Join(root, "diff", "*.yaml")) + if err != nil || len(files) == 0 { + t.Fatalf("no diff cases: %v", err) + } + sqlc := repoSQLC(t) + for _, f := range files { + spec, err := planguard.LoadDiffSpec(dirFS(root), "diff/"+filepath.Base(f)) + if err != nil { + t.Fatal(err) + } + for _, b := range spec.Batches { + for alias, src := range b.Statements { + kind, name, _ := strings.Cut(src, ":") + if (kind == "old" || kind == "new") && sqlc[name] == "" { + t.Errorf("%s: %s: %s names no sqlc statement", f, alias, src) + } + } + } + } + // No plan IDs or deployment identifiers in the diff cases or the overlays either. + for _, dir := range []string{"diff", "overlay"} { + paths, _ := filepath.Glob(filepath.Join(root, dir, "*")) + if len(paths) == 0 { + t.Errorf("nothing under %s/%s", root, dir) + } + for _, p := range paths { + body, err := os.ReadFile(p) + if err != nil { + t.Fatal(err) + } + for i, line := range strings.Split(string(body), "\n") { + if m := planIDs.FindString(line); m != "" { + t.Errorf("%s:%d holds the plan ID %q", p, i+1, m) + } + for _, id := range deploymentIDs(line) { + t.Errorf("%s:%d holds the deployment identifier %s", p, i+1, id) + } + } + } + } +} diff --git a/internal/planguard/dml_internal_test.go b/internal/planguard/dml_internal_test.go new file mode 100644 index 0000000..e589010 --- /dev/null +++ b/internal/planguard/dml_internal_test.go @@ -0,0 +1,29 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package planguard + +import "testing" + +func TestDataModifying(t *testing.T) { + cases := map[string]string{ + "SELECT 1": "", + "-- name: X :one\nDELETE FROM t WHERE id = $1": "DELETE", + "/* c */ insert into t values (1)": "INSERT", + "UPDATE t SET a = 1": "UPDATE", + "MERGE INTO t USING s ON true WHEN MATCHED THEN DO NOTHING": "MERGE", + "WITH d AS (DELETE FROM t RETURNING 1) SELECT count(*) FROM d": "DELETE", + "WITH x AS MATERIALIZED (SELECT 1), u AS (UPDATE t SET a = 1 RETURNING 1) SELECT 1": "UPDATE", + "WITH x AS (SELECT 1) INSERT INTO t SELECT * FROM x": "INSERT", + "WITH e AS (SELECT id FROM t FOR UPDATE SKIP LOCKED) SELECT * FROM e": "", + "WITH e AS (SELECT id FROM t FOR NO KEY UPDATE) SELECT * FROM e": "", + "SELECT 'delete', \"update\" FROM t -- insert": "", + "WITH x AS (SELECT $$delete$$ AS s) SELECT * FROM x": "", + "SELECT * FROM t WHERE a IN (SELECT 1) FOR UPDATE": "", + } + for sql, want := range cases { + if got := dataModifying(sql); got != want { + t.Errorf("dataModifying(%q) = %q, want %q", sql, got, want) + } + } +} diff --git a/internal/planguard/emit.go b/internal/planguard/emit.go new file mode 100644 index 0000000..9d51818 --- /dev/null +++ b/internal/planguard/emit.go @@ -0,0 +1,344 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package planguard + +import ( + "bufio" + "crypto/rand" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "regexp" + "slices" + "strings" + "time" +) + +// The prod tier through an operator's psql, for evidence before a merge (the server runs Run directly): +// +// 1. EmitResolvers writes a psql script; its output goes to ParseResolverOutput. +// 2. EmitPlans writes a script of plain EXPLAINs with those values; its output goes to ParsePlanOutput. +// 3. CheckPlans checks the plans and returns the report. +// +// Every block of output starts with a marker line "@@ ", where nonce is 16 random hex +// characters chosen when the script is written, so text from the mesh cannot forge a marker. The scripts open +// with a "@@ planguard" header and end with "@@ END"; output without the end marker is truncated +// and is an error. A resolver block's body is "N" for NULL or "V" followed by the value. + +// IndexesKey is the key under which ParsePlanOutput returns the index list block (a JSON array of names). +const IndexesKey = "indexes" + +const scriptHead = "SET default_transaction_read_only = on; SET lock_timeout = '1s'; SET statement_timeout = '20s';\n" + + "\\set QUIET on\n\\set ON_ERROR_STOP off\n\\pset format unaligned\n\\pset tuples_only on\n\\pset pager off\n" + + "SET jit = off; SET timezone = 'UTC'; SET standard_conforming_strings = on;\n" + +var ( + nonceRe = regexp.MustCompile(`^[0-9a-f]{16}$`) + scriptHeadRe = regexp.MustCompile(`^@@([0-9a-f]{16}) planguard$`) + markerPrefixF = "@@%s " +) + +// NewNonce returns 16 hex characters from crypto/rand. +func NewNonce() (string, error) { + var b [8]byte + if _, err := rand.Read(b[:]); err != nil { + return "", err + } + return hex.EncodeToString(b[:]), nil +} + +// ScriptNonce returns the nonce of the "@@ planguard" header in a script's output. The header is the +// first line starting with "@@"; it comes before any query output. +func ScriptNonce(out []byte) (string, error) { + for _, line := range strings.Split(string(out), "\n") { + if !strings.HasPrefix(line, "@@") { + continue + } + m := scriptHeadRe.FindStringSubmatch(strings.TrimRight(line, "\r")) + if m == nil { + return "", fmt.Errorf("first marker line %.80q is not a planguard header", line) + } + return m[1], nil + } + return "", fmt.Errorf("no planguard header in the output") +} + +func checkNonce(nonce string) error { + if !nonceRe.MatchString(nonce) { + return fmt.Errorf("nonce %q is not 16 hex characters", nonce) + } + return nil +} + +func scriptStart(w *strings.Builder, nonce, searchPath string) error { + w.WriteString(scriptHead) + if searchPath != "" { + set := "SET search_path = " + searchPath + if err := checkStatementText(set); err != nil { + return fmt.Errorf("search_path: %w", err) + } + w.WriteString(set + ";\n") + } + fmt.Fprintf(w, "\\echo @@%s planguard\n", nonce) + return nil +} + +// EmitResolvers writes the psql script that resolves the params of o's selection. +func EmitResolvers(w io.Writer, c *Corpus, o RunOptions, nonce string) error { + if err := checkNonce(nonce); err != nil { + return err + } + if err := o.validate(c); err != nil { + return err + } + need := neededResolvers(c, c.selection(o), o.IncludePending) + var b strings.Builder + if err := scriptStart(&b, nonce, c.baseSearchPath(o)); err != nil { + return err + } + for _, name := range sortedKeys(need) { + if need[name] == "" { + return fmt.Errorf("resolver %s is not in the corpus", name) + } + if err := checkStatementText(need[name]); err != nil { + return fmt.Errorf("resolver %s: %w", name, err) + } + body := fmt.Sprintf("SELECT coalesce('V' || ((%s)::text), 'N');", trimStmt(need[name])) + if err := checkStatementText(body); err != nil { + return fmt.Errorf("resolver %s: %w", name, err) + } + if err := checkScriptLines(body); err != nil { + return fmt.Errorf("resolver %s: %w", name, err) + } + fmt.Fprintf(&b, "\\echo @@%s R %s\n%s\n", nonce, name, body) + } + fmt.Fprintf(&b, "\\echo @@%s END\n", nonce) + _, err := io.WriteString(w, b.String()) + return err +} + +// EmitPlans writes the psql script of plain EXPLAINs for o's selection with params resolved. Runs that skip, or +// cannot be rendered, are left out; CheckPlans reports them. +func EmitPlans(w io.Writer, c *Corpus, o RunOptions, params map[string]*string, nonce string) error { + if err := checkNonce(nonce); err != nil { + return err + } + if err := o.validate(c); err != nil { + return err + } + if params == nil { + params = map[string]*string{} + } + var b strings.Builder + if err := scriptStart(&b, nonce, c.baseSearchPath(o)); err != nil { + return err + } + var tables []string + for _, t := range c.Defaults.BigTables { + tables = append(tables, quoteLit(t)) + } + fmt.Fprintf(&b, "\\echo @@%s I %s\n", nonce, IndexesKey) + if len(tables) == 0 { + b.WriteString("SELECT '[]';\n") + } else { + fmt.Fprintf(&b, "SELECT coalesce(json_agg(DISTINCT ic.relname::text ORDER BY ic.relname::text), '[]') FROM unnest(ARRAY[%s]::text[]) b(name) "+ + "JOIN pg_index i ON i.indrelid = to_regclass(quote_ident(b.name)) JOIN pg_class ic ON ic.oid = i.indexrelid;\n", strings.Join(tables, ", ")) + } + for _, e := range c.selection(o) { + for _, mode := range o.modesOf(c, e) { + sp := c.spec(o, e, mode, params, nil, true) + if sp.skip != "" || sp.err != "" { + continue + } + for _, l := range append(slices.Clone(sp.setup), sp.explain) { + if err := checkScriptLines(l); err != nil { + return fmt.Errorf("%s|%s: %w", e.ID, mode, err) + } + } + for _, l := range sp.setup { + if strings.HasPrefix(l, "PREPARE ") { + b.WriteString(l + "\n;\n") + } else { + b.WriteString(l + ";\n") + } + } + fmt.Fprintf(&b, "\\echo @@%s P %s|%s\n%s;\nROLLBACK;\nDEALLOCATE %s;\n", nonce, e.ID, mode, sp.explain, stmtName) + } + } + fmt.Fprintf(&b, "\\echo @@%s END\n", nonce) + _, err := io.WriteString(w, b.String()) + return err +} + +var blockLabel = map[string]string{"R": "resolver", "P": "plan", "I": "index list"} + +type block struct { + kind, key string + lines []string +} + +// parseBlocks splits script output into its marked blocks. Lines before the first block are ignored (psql's own +// output of the opening SETs). The output must end with the end marker. +func parseBlocks(r io.Reader, nonce string, kinds ...string) ([]block, error) { + if err := checkNonce(nonce); err != nil { + return nil, err + } + raw, err := io.ReadAll(bufio.NewReader(r)) + if err != nil { + return nil, err + } + text := strings.TrimSuffix(string(raw), "\n") + prefix := fmt.Sprintf(markerPrefixF, nonce) + var out []block + var cur *block + ended := false + seen := map[string]bool{} + closeCur := func() error { + if cur == nil { + return nil + } + if len(cur.lines) == 0 { + return fmt.Errorf("%s %s has no body (did its query fail?)", blockLabel[cur.kind], cur.key) + } + out = append(out, *cur) + cur = nil + return nil + } + for _, line := range strings.Split(text, "\n") { + if !strings.HasPrefix(line, prefix) { + if cur != nil { + cur.lines = append(cur.lines, line) + } + continue + } + if err := closeCur(); err != nil { + return nil, err + } + rest := line[len(prefix):] + if rest == "planguard" { + continue + } + if rest == "END" { + ended = true + break + } + kind, key, _ := strings.Cut(rest, " ") + known := false + for _, k := range kinds { + known = known || k == kind + } + if !known || key == "" { + return nil, fmt.Errorf("unexpected marker %q", rest) + } + if seen[kind+" "+key] { + return nil, fmt.Errorf("block %s %s appears twice", kind, key) + } + seen[kind+" "+key] = true + cur = &block{kind: kind, key: key} + } + if !ended { + return nil, fmt.Errorf("output ends without the end marker (truncated?)") + } + return out, nil +} + +// ParseResolverOutput reads the output of an EmitResolvers script: resolver name → value, nil for NULL. +func ParseResolverOutput(r io.Reader, nonce string) (map[string]*string, error) { + blocks, err := parseBlocks(r, nonce, "R") + if err != nil { + return nil, fmt.Errorf("resolver output: %w", err) + } + out := map[string]*string{} + for _, bl := range blocks { + body := strings.Join(bl.lines, "\n") + switch { + case body == "N": + out[bl.key] = nil + case strings.HasPrefix(body, "V"): + v := body[1:] + out[bl.key] = &v + default: + return nil, fmt.Errorf("resolver output: resolver %s: body %.80q is neither N nor V", bl.key, body) + } + } + return out, nil +} + +// ParsePlanOutput reads the output of an EmitPlans script: "|" → EXPLAIN JSON, and IndexesKey → the +// JSON list of big-table indexes. A body that is not JSON is an error naming its block. +func ParsePlanOutput(r io.Reader, nonce string) (map[string][]byte, error) { + blocks, err := parseBlocks(r, nonce, "P", "I") + if err != nil { + return nil, fmt.Errorf("plan output: %w", err) + } + out := map[string][]byte{} + for _, bl := range blocks { + key := bl.key + if bl.kind == "P" { + id, mode, ok := strings.Cut(key, "|") + if !ok || id == "" || mode == "" { + return nil, fmt.Errorf("plan output: block key %q is not |", key) + } + } else if key != IndexesKey { + return nil, fmt.Errorf("plan output: unexpected index block %q", key) + } + body := []byte(strings.Join(bl.lines, "\n")) + if !json.Valid(body) { + return nil, fmt.Errorf("plan output: block %s: body is not valid JSON (%.80q)", key, body) + } + out[key] = body + } + return out, nil +} + +// CheckPlans checks plans (from ParsePlanOutput) for o's selection, with o.Params the resolved values. A run with +// no plan in the output is an ERROR; a plan for a run outside the selection is an error. +func CheckPlans(c *Corpus, o RunOptions, plans map[string][]byte) (*Report, error) { + if err := o.validate(c); err != nil { + return nil, err + } + t0 := time.Now() + rep := &Report{Tier: o.Tier, StartedAt: t0.UTC(), Settings: map[string]string{}} + if ix, ok := plans[IndexesKey]; ok { + if err := json.Unmarshal(ix, &rep.Indexes); err != nil { + return nil, fmt.Errorf("index block: %w", err) + } + } + params := o.Params + if params == nil { + params = map[string]*string{} + } + used := map[string]bool{IndexesKey: true} + for _, e := range c.selection(o) { + for _, mode := range o.modesOf(c, e) { + sp := c.spec(o, e, mode, params, nil, true) + c.skipMissingIndex(&sp, rep.Indexes) + r := sp.r + key := e.ID + "|" + mode + switch plan, ok := plans[key]; { + case sp.skip != "": + r.Verdict, r.Skip = VerdictSkip, sp.skip + used[key] = true // a plan emitted before the index list was known + case sp.err != "": + c.errored(&r, e, sp.err) + case !ok: + c.errored(&r, e, "no plan for "+key+" in the psql output") + default: + used[key] = true + if err := c.judge(&r, sp, plan, o); err != nil { + return nil, err + } + } + rep.Results = append(rep.Results, r) + } + } + for _, k := range sortedKeys(plans) { + if !used[k] { + return nil, fmt.Errorf("the psql output holds a plan for %s, which is not in this selection", k) + } + } + rep.DurationMs = msSince(t0) + return rep, nil +} diff --git a/internal/planguard/emit_test.go b/internal/planguard/emit_test.go new file mode 100644 index 0000000..32f067a --- /dev/null +++ b/internal/planguard/emit_test.go @@ -0,0 +1,370 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package planguard_test + +import ( + "bytes" + "regexp" + "strings" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/planguard" +) + +const emitCorpus = ` +defaults: {big_tables: [nodes]} +tiers: {prod: {}} +statements: + Q: {sqlc: Q, plan: custom} + P: {sqlc: NotMergedYet, plan: custom, pending: some_feature} +resolvers: + top: SELECT public_key FROM nodes ORDER BY last_seen DESC LIMIT 1 + gone: SELECT NULL::text + pend: SELECT iata FROM a_table_of_the_pending_feature LIMIT 1 +entries: + - {id: q.top, stmt: Q, params: [{r: top}], modes: [custom, generic]} + - {id: q.gone, stmt: Q, params: [{r: gone}]} + - {id: q.lit, stmt: Q, params: ["it's"]} + - {id: p.one, stmt: P} + - {id: p.res, stmt: P, params: [{r: pend}]} +` + +func loadEmitCorpus(t *testing.T) *planguard.Corpus { + t.Helper() + c, err := planguard.LoadCorpus(mapFS(map[string]string{"c.yaml": emitCorpus}), "c.yaml", testSQLC) + if err != nil { + t.Fatal(err) + } + return c +} + +func TestNewNonce(t *testing.T) { + a, err := planguard.NewNonce() + if err != nil { + t.Fatal(err) + } + b, _ := planguard.NewNonce() + if !regexp.MustCompile(`^[0-9a-f]{16}$`).MatchString(a) || a == b { + t.Errorf("nonces %q %q, want two different 16-hex strings", a, b) + } +} + +// A pending entry skips, so its resolvers are not emitted (or run): they may read tables that do not exist yet. +func TestEmitResolversLeavesOutPending(t *testing.T) { + c := loadEmitCorpus(t) + const nonce = "0123456789abcdef" + for _, include := range []bool{false, true} { + var rs bytes.Buffer + if err := planguard.EmitResolvers(&rs, c, planguard.RunOptions{Tier: "prod", IncludePending: include}, nonce); err != nil { + t.Fatal(err) + } + if got := strings.Contains(rs.String(), `\echo @@`+nonce+` R pend`); got != include { + t.Errorf("include pending %v: script has the pending resolver: %v", include, got) + } + } +} + +func TestEmitScripts(t *testing.T) { + c := loadEmitCorpus(t) + const nonce = "0123456789abcdef" + o := planguard.RunOptions{Tier: "prod"} + var rs bytes.Buffer + if err := planguard.EmitResolvers(&rs, c, o, nonce); err != nil { + t.Fatal(err) + } + script := rs.String() + const head = "SET default_transaction_read_only = on; SET lock_timeout = '1s'; SET statement_timeout = '20s';\n" + if !strings.HasPrefix(script, head) { + t.Errorf("resolver script opens with %.120q", script) + } + for _, want := range []string{`\echo @@` + nonce + ` R gone`, `\echo @@` + nonce + ` R top`, `\echo @@` + nonce + ` END`} { + if !strings.Contains(script, want) { + t.Errorf("resolver script lacks %q:\n%s", want, script) + } + } + + top := "abc" + var ps bytes.Buffer + if err := planguard.EmitPlans(&ps, c, o, map[string]*string{"top": &top, "gone": nil}, nonce); err != nil { + t.Fatal(err) + } + script = ps.String() + if !strings.HasPrefix(script, head) { + t.Errorf("plan script opens with %.120q", script) + } + for _, want := range []string{ + `\echo @@` + nonce + ` P q.top|custom`, `\echo @@` + nonce + ` P q.top|generic`, `\echo @@` + nonce + ` P q.lit|custom`, + "EXPLAIN (FORMAT JSON) EXECUTE planguard_q('abc');", "EXPLAIN (FORMAT JSON) EXECUTE planguard_q('it''s');", + "BEGIN ISOLATION LEVEL READ COMMITTED READ ONLY;", "SET standard_conforming_strings = on;", "SET LOCAL standard_conforming_strings = on", "SET LOCAL plan_cache_mode = force_generic_plan;", + "ROLLBACK;\nDEALLOCATE planguard_q;", `\echo @@` + nonce + ` I indexes`, `\echo @@` + nonce + ` END`, + } { + if !strings.Contains(script, want) { + t.Errorf("plan script lacks %q:\n%s", want, script) + } + } + for _, not := range []string{"q.gone", "p.one", "ANALYZE"} { + if strings.Contains(script, not) { + t.Errorf("plan script holds %q", not) + } + } +} + +func TestEmitFramingSurvivesHostileText(t *testing.T) { + const nonce = "00ff00ff00ff00ff" + const other = "1111111111111111" + m := "@@" + nonce + " " + resolverOut := "SET\nSET\nSET\n" + m + "planguard\n" + + m + "R hostile\nVx\n@@CASE y\n" + + m + "R missing\nN\n" + + m + "R empty\nV\n" + + m + "R trailing\nVa\n\n" + + m + "END\n" + res, err := planguard.ParseResolverOutput(strings.NewReader(resolverOut), nonce) + if err != nil { + t.Fatal(err) + } + if v := res["hostile"]; v == nil || *v != "x\n@@CASE y" { + t.Errorf("hostile = %q", deref(v)) + } + if v, ok := res["missing"]; !ok || v != nil { + t.Errorf("missing = %q (present %v), want NULL", deref(v), ok) + } + if v := res["empty"]; v == nil || *v != "" { + t.Errorf("empty = %q", deref(v)) + } + if v := res["trailing"]; v == nil || *v != "a\n" { + t.Errorf("trailing = %q", deref(v)) + } + + plan := `[ + { + "Plan": {"Node Type": "Index Scan", "Relation Name": "nodes", "Index Name": "nodes_pkey", + "Index Cond": "(public_key = '@@` + other + ` P a|b'::text)"} + } +]` + planOut := m + "planguard\n" + m + "P q.top|custom\n" + plan + "\n" + m + "I indexes\n[\"nodes_pkey\"]\n" + m + "END\n" + plans, err := planguard.ParsePlanOutput(strings.NewReader(planOut), nonce) + if err != nil { + t.Fatal(err) + } + if got := string(plans["q.top|custom"]); got != plan { + t.Errorf("plan = %q, want %q", got, plan) + } + if got := string(plans[planguard.IndexesKey]); got != `["nodes_pkey"]` { + t.Errorf("indexes = %q", got) + } + + bad := map[string]struct{ out, want string }{ + "truncated json": {m + "P a|custom\n[{\"Plan\": {\"Node Type\"\n" + m + "END\n", "a|custom"}, + "marker no body": {m + "P a|custom\n" + m + "P b|custom\n" + plan + "\n" + m + "END\n", "a|custom"}, + "no end marker": {m + "P b|custom\n" + plan + "\n", "end marker"}, + "unknown kind": {m + "X b|custom\n" + plan + "\n" + m + "END\n", "X b|custom"}, + "duplicate block": {m + "P b|custom\n" + plan + "\n" + m + "P b|custom\n" + plan + "\n" + m + "END\n", "b|custom"}, + "key without mode": {m + "P b\n" + plan + "\n" + m + "END\n", `"b"`}, + } + for name, tc := range bad { + if _, err := planguard.ParsePlanOutput(strings.NewReader(tc.out), nonce); err == nil || !strings.Contains(err.Error(), tc.want) { + t.Errorf("%s: err = %v, want one naming %q", name, err, tc.want) + } + } + badRes := map[string]struct{ out, want string }{ + "marker no body": {m + "R a\n" + m + "R b\nVx\n" + m + "END\n", "resolver a"}, + "bad body": {m + "R a\nxyz\n" + m + "END\n", "resolver a"}, + "no end marker": {m + "R a\nVx\n", "end marker"}, + } + for name, tc := range badRes { + if _, err := planguard.ParseResolverOutput(strings.NewReader(tc.out), nonce); err == nil || !strings.Contains(err.Error(), tc.want) { + t.Errorf("resolvers %s: err = %v, want one naming %q", name, err, tc.want) + } + } + if n, err := planguard.ScriptNonce([]byte(planOut)); err != nil || n != nonce { + t.Errorf("ScriptNonce = %q, %v", n, err) + } + if _, err := planguard.ScriptNonce([]byte("SET\n@@CASE x\n")); err == nil { + t.Error("ScriptNonce found a nonce in output without a header") + } +} + +func TestCheckPlans(t *testing.T) { + c := loadEmitCorpus(t) + top := "abc" + o := planguard.RunOptions{Tier: "prod", Params: map[string]*string{"top": &top, "gone": nil}} + plan := []byte(`[{"Plan": {"Node Type": "Index Scan", "Relation Name": "nodes", "Index Name": "nodes_pkey"}}]`) + rep, err := planguard.CheckPlans(c, o, map[string][]byte{"q.top|custom": plan, "q.top|generic": plan, + planguard.IndexesKey: []byte(`["nodes_pkey"]`)}) + if err != nil { + t.Fatal(err) + } + got := map[string]planguard.Result{} + for _, r := range rep.Results { + got[r.ID+"|"+r.Mode] = r + } + if r := got["q.top|custom"]; r.Verdict != planguard.VerdictOK || len(r.Signature) == 0 || r.Params[0] != "'abc'" { + t.Errorf("q.top: %+v", r) + } + if r := got["q.gone|custom"]; r.Verdict != planguard.VerdictSkip || r.Skip != "resolver gone returned no value on this tier" { + t.Errorf("q.gone: %+v", r) + } + if r := got["p.one|custom"]; r.Verdict != planguard.VerdictSkip || r.Skip != "pending: some_feature" { + t.Errorf("p.one: %+v", r) + } + if r := got["q.lit|custom"]; r.Verdict != planguard.VerdictError { + t.Errorf("q.lit with no plan in the output: %+v", r) + } + if r := got["p.res|custom"]; r.Verdict != planguard.VerdictSkip || r.Skip != "pending: some_feature" { + t.Errorf("p.res: %+v", r) + } + if len(rep.Results) != 6 || !contains(rep.Indexes, "nodes_pkey") { + t.Errorf("results %d, indexes %v", len(rep.Results), rep.Indexes) + } + if _, err := planguard.CheckPlans(c, o, map[string][]byte{"stranger|custom": plan}); err == nil { + t.Error("a plan for a run outside the selection was accepted") + } +} + +func deref(s *string) string { + if s == nil { + return "" + } + return *s +} + +func TestEmitRejectsHostileNames(t *testing.T) { + hostile := []string{"a`touch /tmp/x`", "a\nb", "../../etc/x", "a/b", "..", "a:b", "a b", ""} + for _, id := range hostile { + body := strings.Replace(emitCorpus, "{id: q.lit,", "{id: "+yamlQuote(id)+",", 1) + if _, err := planguard.LoadCorpus(mapFS(map[string]string{"c.yaml": body}), "c.yaml", testSQLC); err == nil { + t.Errorf("entry id %q loaded", id) + } + res := strings.Replace(emitCorpus, " gone: SELECT", " "+yamlQuote(id)+": SELECT", 1) + if _, err := planguard.LoadCorpus(mapFS(map[string]string{"c.yaml": res}), "c.yaml", testSQLC); err == nil { + t.Errorf("resolver name %q loaded", id) + } + } + + // A corpus changed after loading is checked again before anything is emitted. + for _, id := range hostile { + c := loadEmitCorpus(t) + c.Entries[2].ID = id + var b bytes.Buffer + if err := planguard.EmitPlans(&b, c, planguard.RunOptions{Tier: "prod"}, nil, "0123456789abcdef"); err == nil { + t.Errorf("EmitPlans accepted entry id %q:\n%s", id, b.String()) + } + c = loadEmitCorpus(t) + c.Resolvers[id] = "SELECT 1" + if err := planguard.EmitResolvers(&b, c, planguard.RunOptions{Tier: "prod"}, "0123456789abcdef"); err == nil { + t.Errorf("EmitResolvers accepted resolver name %q", id) + } + } +} + +func TestSearchPathElements(t *testing.T) { + c := loadEmitCorpus(t) + for sp, ok := range map[string]bool{ + "": true, "public": true, "t_s2b, public": true, `"My Schema", public`: true, + `"$user", public`: false, "public; DROP TABLE x": false, "pub$lic": false, "a,,b": false, "'x'": false, `"a"b"`: false, + } { + var b bytes.Buffer + err := planguard.EmitResolvers(&b, c, planguard.RunOptions{Tier: "prod", SearchPath: sp}, "0123456789abcdef") + if (err == nil) != ok { + t.Errorf("search_path %q: err = %v, want ok=%v", sp, err, ok) + } + } +} + +func yamlQuote(s string) string { + return `"` + strings.NewReplacer(`\`, `\\`, `"`, `\"`, "\n", `\n`).Replace(s) + `"` +} + +// A mode's own known_issue turns that mode's failures into KNOWN_ISSUE; other modes still fail, and an error is +// always ERROR. +func TestModeKnownIssue(t *testing.T) { + const body = ` +defaults: {big_tables: [nodes], forbid: [{node: Seq Scan, relations: [big]}]} +tiers: {prod: {}} +statements: + Q: {sqlc: Q, plan: custom} +entries: + - {id: q.seq, stmt: Q, params: [x], modes: [custom, generic], mode: {generic: {known_issue: generic plan is known to scan}}} + - {id: q.err, stmt: Q, params: [y], modes: [generic], mode: {generic: {known_issue: generic plan is known to scan}}} +` + c, err := planguard.LoadCorpus(mapFS(map[string]string{"c.yaml": body}), "c.yaml", testSQLC) + if err != nil { + t.Fatal(err) + } + seq := []byte(`[{"Plan": {"Node Type": "Seq Scan", "Relation Name": "nodes"}}]`) + rep, err := planguard.CheckPlans(c, planguard.RunOptions{Tier: "prod", Params: map[string]*string{}}, + map[string][]byte{"q.seq|custom": seq, "q.seq|generic": seq}) + if err != nil { + t.Fatal(err) + } + want := map[string]planguard.Verdict{ + "q.seq|custom": planguard.VerdictFalsePositive, + "q.seq|generic": planguard.VerdictKnownIssue, + "q.err|generic": planguard.VerdictError, // no plan in the output + } + if len(rep.Results) != 3 { + t.Fatalf("%d results, want 3", len(rep.Results)) + } + for _, r := range rep.Results { + if v := want[r.ID+"|"+r.Mode]; r.Verdict != v { + t.Errorf("%s|%s: verdict %s, want %s", r.ID, r.Mode, r.Verdict, v) + } + } + + bad := strings.Replace(body, "{generic: {known_issue: generic plan is known to scan}}}\n - {id: q.err", + "{generic: {skip: no, known_issue: generic plan is known to scan}}}\n - {id: q.err", 1) + if _, err := planguard.LoadCorpus(mapFS(map[string]string{"c.yaml": bad}), "c.yaml", testSQLC); err == nil || + !strings.Contains(err.Error(), "skip excludes") { + t.Errorf("skip with known_issue: err = %v", err) + } +} + +// On the psql path the index list comes with the plans: an entry whose required index is absent reads SKIP, and +// its emitted plan is accepted unread. Without an index list nothing is skipped. +func TestRequiresIndexCheckPlans(t *testing.T) { + const body = ` +defaults: {big_tables: [nodes]} +tiers: {prod: {}} +statements: + Q: {sqlc: Q, plan: custom} +entries: + - {id: q.needs, stmt: Q, params: [x], requires_indexes: [idx_nodes_special]} +` + c, err := planguard.LoadCorpus(mapFS(map[string]string{"c.yaml": body}), "c.yaml", testSQLC) + if err != nil { + t.Fatal(err) + } + plan := []byte(`[{"Plan": {"Node Type": "Index Scan", "Relation Name": "nodes", "Index Name": "nodes_pkey"}}]`) + o := planguard.RunOptions{Tier: "prod", Params: map[string]*string{}} + for _, tc := range []struct { + indexes string // JSON, or "" for no index block + verdict planguard.Verdict + skip string + }{ + {`["nodes_pkey"]`, planguard.VerdictSkip, "requires index idx_nodes_special"}, + {`["nodes_pkey", "idx_nodes_special"]`, planguard.VerdictOK, ""}, + {"", planguard.VerdictOK, ""}, + } { + plans := map[string][]byte{"q.needs|custom": plan} + if tc.indexes != "" { + plans[planguard.IndexesKey] = []byte(tc.indexes) + } + rep, err := planguard.CheckPlans(c, o, plans) + if err != nil { + t.Fatalf("indexes %s: %v", tc.indexes, err) + } + if len(rep.Results) != 1 || rep.Results[0].Verdict != tc.verdict || rep.Results[0].Skip != tc.skip { + t.Errorf("indexes %s: results %+v, want %s %q", tc.indexes, rep.Results, tc.verdict, tc.skip) + } + } + + for _, bad := range []string{"defaults: {big_tables: [nodes]}\n", ""} { + y := bad + "statements:\n Q: {sqlc: Q, plan: custom}\nentries:\n - {id: q, stmt: Q, requires_indexes: [\"idx; DROP\"]}\n" + if _, err := planguard.LoadCorpus(mapFS(map[string]string{"c.yaml": y}), "c.yaml", testSQLC); err == nil || + !strings.Contains(err.Error(), "requires_indexes") { + t.Errorf("bad requires_indexes loaded: %v", err) + } + } +} diff --git a/internal/planguard/export_test.go b/internal/planguard/export_test.go new file mode 100644 index 0000000..8e2aeaf --- /dev/null +++ b/internal/planguard/export_test.go @@ -0,0 +1,10 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package planguard + +// Test-only access for the external test package: no production caller needs these, so they stay unexported. +var ( + LoadDiffSpec = loadDiffSpec // reads a diff case file from an fs.FS (the CLI parses its own bytes with ParseDiffSpec) + FixtureHash = fixtureHash // the hash a finished fixture marker must hold for these generator inputs +) diff --git a/internal/planguard/fixture.go b/internal/planguard/fixture.go new file mode 100644 index 0000000..913f7e9 --- /dev/null +++ b/internal/planguard/fixture.go @@ -0,0 +1,409 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package planguard + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io/fs" + "path" + "regexp" + "sort" + "strings" + + "github.com/MeshCore-Beacon/beacon-server/db" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +// FixtureCreateDatabase is how a fixture database must be created: with C collation, so text order (and with it the +// generated data, the statistics and the plans) is the same on every platform and libc. +const FixtureCreateDatabase = "CREATE DATABASE %s TEMPLATE template0 LC_COLLATE 'C' LC_CTYPE 'C' ENCODING 'UTF8'" + +// ErrNotFixture is returned by BuildFixture for a database that holds data and was not built by it. +var ErrNotFixture = errors.New("not a planguard fixture database") + +const ( + fixtureDir = "fixture" + fixtureData = "fixture/testdata" + fixtureSource = "fixture/source.sql" + fixtureSchema = "planguard_src" + fixtureStatsTgt = 200 + // Generator files from this number on run after ANALYZE: they change data the statistics must not see. + fixtureAfterAnalyze = 90 + // fixtureFormat versions the Go side of the generator (the build sequence and the marker); bump it when a change + // here alters what a build produces. + fixtureFormat = 3 +) + +var ( + fixtureStepRe = regexp.MustCompile(`^([0-9]{2})_[a-z0-9_]+\.sql$`) + fixtureCSVRe = regexp.MustCompile(`^[a-z][a-z0-9_]*\.csv$`) +) + +// generatorInputs are the generator's inputs that live in Go rather than in fixture/. They are hashed with the +// files, and generateFixture builds from the copy it hashed. +type generatorInputs struct { + format int + settings string // the build session's SETs + statsTarget int // default_statistics_target for ANALYZE + afterAnalyze int // the first step number that runs after ANALYZE +} + +var currentInputs = generatorInputs{fixtureFormat, buildSettings, fixtureStatsTgt, fixtureAfterAnalyze} + +type fixtureFiles struct { + in generatorInputs + source string + csvs []string // base names, sorted + steps []string // NN_*.sql base names, sorted + hash string +} + +// generatorFile reports whether a directory entry belongs to the generator. Names starting with "." or "_" are +// left out, as go:embed leaves them out, so editor and system files beside the generator do not break it. +func generatorFile(d fs.DirEntry) bool { + return !d.IsDir() && !strings.HasPrefix(d.Name(), ".") && !strings.HasPrefix(d.Name(), "_") +} + +// readFixtureFiles lists the generator in fsys and hashes its Go-side inputs and every file of it. +func readFixtureFiles(fsys fs.FS) (fixtureFiles, error) { + return readFixtureFilesWith(fsys, currentInputs) +} + +func readFixtureFilesWith(fsys fs.FS, in generatorInputs) (fixtureFiles, error) { + ff := fixtureFiles{in: in} + h := sha256.New() + fmt.Fprintf(h, "format=%d\x00settings=%s\x00stats_target=%d\x00after_analyze=%d\x00", in.format, in.settings, + in.statsTarget, in.afterAnalyze) + add := func(p string) error { + b, err := fs.ReadFile(fsys, p) + if err != nil { + return err + } + fmt.Fprintf(h, "%s\x00%d\x00", p, len(b)) + h.Write(b) + return nil + } + if err := add(fixtureSource); err != nil { + return ff, fmt.Errorf("fixture: %w", err) + } + ff.source = fixtureSource + steps, err := fs.ReadDir(fsys, fixtureDir) + if err != nil { + return ff, fmt.Errorf("fixture: %w", err) + } + for _, d := range steps { + if !generatorFile(d) || d.Name() == path.Base(fixtureSource) { + continue + } + if !fixtureStepRe.MatchString(d.Name()) { + return ff, fmt.Errorf("fixture: %s is not NN_name.sql", d.Name()) + } + ff.steps = append(ff.steps, d.Name()) + } + csvs, err := fs.ReadDir(fsys, fixtureData) + if err != nil { + return ff, fmt.Errorf("fixture: %w", err) + } + for _, d := range csvs { + if d.IsDir() && !strings.HasPrefix(d.Name(), ".") && !strings.HasPrefix(d.Name(), "_") { + return ff, fmt.Errorf("fixture: %s/%s is a directory", fixtureData, d.Name()) + } + if !generatorFile(d) { + continue + } + if !fixtureCSVRe.MatchString(d.Name()) { + return ff, fmt.Errorf("fixture: %s/%s is not a lower-case name.csv", fixtureData, d.Name()) + } + ff.csvs = append(ff.csvs, d.Name()) + } + sort.Strings(ff.steps) + sort.Strings(ff.csvs) + if len(ff.steps) == 0 || len(ff.csvs) == 0 { + return ff, errors.New("fixture: no generator steps or no testdata") + } + for _, s := range ff.steps { + if err := add(path.Join(fixtureDir, s)); err != nil { + return ff, err + } + } + for _, c := range ff.csvs { + if err := add(path.Join(fixtureData, c)); err != nil { + return ff, err + } + } + ff.hash = hex.EncodeToString(h.Sum(nil)) + return ff, nil +} + +// fixtureHash is the hash BuildFixture records for the generator in fsys and this build of the code: a database +// whose finished marker holds it, on a server of the marker's server_version_num, was built from exactly these +// inputs. +func fixtureHash(fsys fs.FS) (string, error) { + ff, err := readFixtureFiles(fsys) + return ff.hash, err +} + +// fixtureState is what the database says about an earlier build. +type fixtureState struct { + marker bool // planguard_src.fixture_marker exists + hash string // the hash of a finished build; "" when the build did not finish + version int // server_version_num the finished build ran on + collate string // datcollate the finished build ran under + dataIn string // a public table that holds a row (schema_migrations aside), or "" + noLedger bool // public holds tables but no schema_migrations + migrated int // rows in schema_migrations +} + +func readFixtureState(ctx context.Context, conn *pgx.Conn) (fixtureState, error) { + var st fixtureState + if err := conn.QueryRow(ctx, "SELECT to_regclass('"+fixtureSchema+".fixture_marker') IS NOT NULL").Scan(&st.marker); err != nil { + return st, err + } + if st.marker { + // to_jsonb reads a marker of an older format too, which has no server_version_num. + err := conn.QueryRow(ctx, "SELECT coalesce(max(m.hash) FILTER (WHERE m.built_at IS NOT NULL), ''), "+ + "coalesce(max((to_jsonb(m) ->> 'server_version_num')::int) FILTER (WHERE m.built_at IS NOT NULL), 0), "+ + "coalesce(max(to_jsonb(m) ->> 'datcollate') FILTER (WHERE m.built_at IS NOT NULL), '') FROM "+ + fixtureSchema+".fixture_marker m").Scan(&st.hash, &st.version, &st.collate) + if err != nil { + return st, err + } + } + // packets first, so a Beacon database is named by its packets. + rows, err := conn.Query(ctx, "SELECT c.relname FROM pg_class c WHERE c.relnamespace = 'public'::regnamespace "+ + "AND c.relkind IN ('r', 'p') ORDER BY c.relname <> 'packets', c.relname") + if err != nil { + return st, err + } + tables, err := pgx.CollectRows(rows, pgx.RowTo[string]) + if err != nil { + return st, err + } + ledger := false + for _, t := range tables { + if t == "schema_migrations" { + ledger = true + if err := conn.QueryRow(ctx, "SELECT count(*) FROM public.schema_migrations").Scan(&st.migrated); err != nil { + return st, err + } + continue + } + if st.dataIn != "" { + continue + } + var has bool + if err := conn.QueryRow(ctx, "SELECT EXISTS (SELECT 1 FROM public."+pgx.Identifier{t}.Sanitize()+")").Scan(&has); err != nil { + return st, err + } + if has { + st.dataIn = t + } + } + st.noLedger = len(tables) > 0 && !ledger + return st, nil +} + +// BuildFixture migrates the database through db.RunMigrations, loads fixture/testdata/*.csv into schema +// planguard_src with COPY FROM STDIN (pgconn CopyFrom), runs fixture/NN_*.sql in name order, ANALYZEs under +// default_statistics_target = 200, runs the stale-region step, and records planguard_src.fixture_marker(hash, +// built_at). It returns early when the marker's hash equals the hash of the embedded fixture files. +// It refuses (error "not a planguard fixture database") when public.packets has a row and no marker exists. +// +// The guard is wider than packets: without a marker, it refuses a database where any public table other than +// schema_migrations holds a row (a migrated Beacon database already does), or where public holds tables but no +// schema_migrations. The hash covers the generator's Go-side inputs too (format, build settings, statistics +// target, the after-ANALYZE step number), and a build is reused only on the server version and collation it ran +// on. The database must have C collation and ctype (FixtureCreateDatabase); any other is refused before anything is +// written. +// +// fixture/source.sql creates the planguard_src tables the CSVs load into. Steps numbered 90 and up run after +// ANALYZE. Autovacuum is switched off on every public table the build fills, so the statistics stay the ones +// it took. The pool's connections must resolve unqualified names to public. The marker is written, unfinished, +// before the first migration, so a build that stops part way is recognised as a fixture and rebuilt from empty +// tables by the next call. +func BuildFixture(ctx context.Context, pool *pgxpool.Pool, fsys fs.FS) error { + ff, err := readFixtureFiles(fsys) + if err != nil { + return err + } + pc, err := pool.Acquire(ctx) + if err != nil { + return err + } + defer pc.Release() + conn := pc.Conn() + var sp, dbname, collate, ctype string + var version int + if err := conn.QueryRow(ctx, "SELECT current_schemas(false)::text, current_setting('server_version_num')::int, "+ + "d.datname, d.datcollate, d.datctype FROM pg_database d WHERE d.datname = current_database()").Scan( + &sp, &version, &dbname, &collate, &ctype); err != nil { + return err + } + if !strings.HasPrefix(sp, "{public") { + return fmt.Errorf("fixture: the connection's search_path resolves to %s; it must start with public", sp) + } + if collate != "C" || ctype != "C" { + return fmt.Errorf("fixture: database %s has LC_COLLATE %q and LC_CTYPE %q; a fixture needs C, so its text order "+ + "does not depend on the platform: "+FixtureCreateDatabase, dbname, collate, ctype, pgx.Identifier{dbname}.Sanitize()) + } + + st, err := readFixtureState(ctx, conn) + if err != nil { + return fmt.Errorf("fixture: read state: %w", err) + } + if !st.marker { + switch { + case st.dataIn != "": + return fmt.Errorf("%w: public.%s holds data and there is no %s.fixture_marker", ErrNotFixture, st.dataIn, fixtureSchema) + case st.noLedger: + return fmt.Errorf("%w: public holds tables but no schema_migrations", ErrNotFixture) + } + } + if st.marker && st.hash == ff.hash && st.version == version && st.collate == collate { + if err := db.RunMigrations(ctx, pool); err != nil { + return err + } + var migrated int + if err := conn.QueryRow(ctx, "SELECT count(*) FROM public.schema_migrations").Scan(&migrated); err != nil { + return err + } + if migrated == st.migrated { + return nil + } + } + if st.marker { + if err := resetFixture(ctx, conn); err != nil { + return fmt.Errorf("fixture: reset: %w", err) + } + } + if _, err := conn.Exec(ctx, "CREATE SCHEMA "+fixtureSchema+"; CREATE TABLE "+fixtureSchema+ + ".fixture_marker (hash text NOT NULL, server_version_num int NOT NULL, datcollate text NOT NULL, built_at timestamptz); "+ + "INSERT INTO "+fixtureSchema+".fixture_marker (hash, server_version_num, datcollate) VALUES ('"+ff.hash+"', "+ + fmt.Sprint(version)+", "+quoteLit(collate)+")"); err != nil { + return fmt.Errorf("fixture: marker: %w", err) + } + if err := db.RunMigrations(ctx, pool); err != nil { + return err + } + if err := generateFixture(ctx, conn, fsys, ff); err != nil { + return fmt.Errorf("fixture: %w", err) + } + return nil +} + +// resetFixture empties a fixture database built from other files or migrations: every public table but the +// migration ledger, and the source schema. +func resetFixture(ctx context.Context, conn *pgx.Conn) error { + _, err := conn.Exec(ctx, `DO $$ +DECLARE t text; +BEGIN + SELECT string_agg(format('public.%I', c.relname), ', ') INTO t FROM pg_class c + WHERE c.relnamespace = 'public'::regnamespace AND c.relkind IN ('r', 'p') AND c.relname <> 'schema_migrations'; + IF t IS NOT NULL THEN + EXECUTE 'TRUNCATE ' || t || ' RESTART IDENTITY CASCADE'; + END IF; +END $$; DROP SCHEMA IF EXISTS `+fixtureSchema+` CASCADE`) + return err +} + +// buildSettings make the build fast and repeatable. GIN inserts collect in the pending list, which VACUUM merges. +// The generator draws from random() after setseed, so the data depends on the order rows reach each draw, which is +// the plan's: the planner's inputs are pinned to PostgreSQL's built-in values, whatever the server's +// configuration, and plans stay serial. +const buildSettings = `SET jit = off; SET max_parallel_workers_per_gather = 0; SET work_mem = '256MB'; +SET hash_mem_multiplier = 2; SET maintenance_work_mem = '512MB'; SET synchronous_commit = off; SET timezone = 'UTC'; +SET gin_pending_list_limit = '256MB'; +SET seq_page_cost = 1; SET random_page_cost = 4; SET cpu_tuple_cost = 0.01; SET cpu_index_tuple_cost = 0.005; +SET cpu_operator_cost = 0.0025; SET effective_cache_size = '4GB'; SET default_statistics_target = 100; +SET enable_seqscan = on; SET enable_indexscan = on; SET enable_bitmapscan = on; SET enable_hashjoin = on; +SET enable_mergejoin = on; SET enable_nestloop = on; SET enable_sort = on; SET enable_hashagg = on; +SET enable_material = on; SET enable_memoize = on; SET enable_incremental_sort = on` + +func generateFixture(ctx context.Context, conn *pgx.Conn, fsys fs.FS, ff fixtureFiles) error { + if _, err := conn.Exec(ctx, ff.in.settings); err != nil { + return err + } + defer func() { + _, _ = conn.Exec(context.Background(), "RESET ALL") + }() + run := func(name string) error { + b, err := fs.ReadFile(fsys, name) + if err != nil { + return err + } + if _, err := conn.Exec(ctx, string(b)); err != nil { + return fmt.Errorf("%s: %w", name, err) + } + return nil + } + var before, after []string + for _, s := range ff.steps { + n := int(s[0]-'0')*10 + int(s[1]-'0') + if n >= ff.in.afterAnalyze { + after = append(after, s) + } else { + before = append(before, s) + } + } + + // Load and generate in one transaction; the marker stays unfinished until the end. + tx, err := conn.Begin(ctx) + if err != nil { + return err + } + defer func() { _ = tx.Rollback(context.Background()) }() + if err := run(ff.source); err != nil { + return err + } + for _, c := range ff.csvs { + b, err := fs.ReadFile(fsys, path.Join(fixtureData, c)) + if err != nil { + return err + } + table := fixtureSchema + "." + strings.TrimSuffix(c, ".csv") + if _, err := tx.Conn().PgConn().CopyFrom(ctx, bytes.NewReader(b), + "COPY "+table+" FROM STDIN (FORMAT csv, HEADER true)"); err != nil { + return fmt.Errorf("load %s: %w", c, err) + } + } + for _, s := range before { + if err := run(path.Join(fixtureDir, s)); err != nil { + return err + } + } + // Autovacuum must not re-sample what ANALYZE below records. + if _, err := tx.Exec(ctx, `DO $$ +DECLARE r record; +BEGIN + FOR r IN SELECT c.relname FROM pg_class c WHERE c.relnamespace = 'public'::regnamespace AND c.relkind = 'r' LOOP + EXECUTE format('ALTER TABLE public.%I SET (autovacuum_enabled = false)', r.relname); + END LOOP; +END $$`); err != nil { + return err + } + if err := tx.Commit(ctx); err != nil { + return err + } + + // VACUUM sets the visibility map, so index-only scans are costed as on a settled database. + if _, err := conn.Exec(ctx, fmt.Sprintf("SET default_statistics_target = %d", ff.in.statsTarget)); err != nil { + return err + } + if _, err := conn.Exec(ctx, "VACUUM (ANALYZE)"); err != nil { + return err + } + for _, s := range after { + if err := run(path.Join(fixtureDir, s)); err != nil { + return err + } + } + _, err = conn.Exec(ctx, "UPDATE "+fixtureSchema+".fixture_marker SET built_at = now()") + return err +} diff --git a/internal/planguard/fixture_integration_test.go b/internal/planguard/fixture_integration_test.go new file mode 100644 index 0000000..b9c0180 --- /dev/null +++ b/internal/planguard/fixture_integration_test.go @@ -0,0 +1,317 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package planguard_test + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/db" + pgdata "github.com/MeshCore-Beacon/beacon-server/db/planguard" + "github.com/MeshCore-Beacon/beacon-server/internal/planguard" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +// throwawayDB creates database name on the test cluster as a fixture database must be created (C collation), +// drops it on cleanup, and returns a pool on it. +func throwawayDB(t *testing.T, ctx context.Context, name string) *pgxpool.Pool { + return throwawayDBAs(t, ctx, name, planguard.FixtureCreateDatabase) +} + +// throwawayDBAs is throwawayDB with its own CREATE DATABASE format (one %s, the name). +func throwawayDBAs(t *testing.T, ctx context.Context, name, create string) *pgxpool.Pool { + t.Helper() + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for PostgreSQL tests") + } + admin, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + drop := func() error { + _, err := admin.Exec(context.Background(), "DROP DATABASE IF EXISTS "+name+" WITH (FORCE)") + return err + } + if err := drop(); err != nil { + t.Fatal(err) + } + if _, err := admin.Exec(ctx, fmt.Sprintf(create, name)); err != nil { + t.Fatal(err) + } + cfg, err := pgxpool.ParseConfig(dsn) + if err != nil { + t.Fatal(err) + } + cfg.ConnConfig.Database = name + cfg.MaxConns = 2 + pool, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + pool.Close() + if err := drop(); err != nil { + t.Error(err) + } + admin.Close(context.Background()) + }) + return pool +} + +// planRows returns the planner's row estimate for the top node of query. +func planRows(t *testing.T, ctx context.Context, pool *pgxpool.Pool, query string) float64 { + t.Helper() + var raw string + if err := pool.QueryRow(ctx, "EXPLAIN (FORMAT JSON) "+query).Scan(&raw); err != nil { + t.Fatal(err) + } + var doc []struct { + Plan struct { + Rows float64 `json:"Plan Rows"` + } `json:"Plan"` + } + if err := json.Unmarshal([]byte(raw), &doc); err != nil || len(doc) != 1 { + t.Fatalf("EXPLAIN %s: %v (%s)", query, err, raw) + } + return doc[0].Plan.Rows +} + +// fixtureDB returns a pool on a built fixture. With PLANGUARD_FIXTURE_DSN set (CI, after the fixture-tier step +// built it) it reuses that database, after checking its marker records a finished build of these very files; +// otherwise it builds one in a fresh database. +func fixtureDB(t *testing.T, ctx context.Context) *pgxpool.Pool { + t.Helper() + dsn := os.Getenv("PLANGUARD_FIXTURE_DSN") + if dsn == "" { + pool := throwawayDB(t, ctx, "planguard_fixture_test") + t0 := time.Now() + if err := planguard.BuildFixture(ctx, pool, pgdata.Files); err != nil { + t.Fatal(err) + } + t.Logf("fixture built in %.1f s", time.Since(t0).Seconds()) + return pool + } + pool, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatal(err) + } + t.Cleanup(pool.Close) + want, err := planguard.FixtureHash(pgdata.Files) + if err != nil { + t.Fatal(err) + } + var got string + var sameServer bool + if err := pool.QueryRow(ctx, `SELECT coalesce(max(hash) FILTER (WHERE built_at IS NOT NULL), ''), + coalesce(bool_and(server_version_num = current_setting('server_version_num')::int + AND datcollate = (SELECT datcollate FROM pg_database WHERE datname = current_database()) AND datcollate = 'C'), false) +FROM planguard_src.fixture_marker`).Scan(&got, &sameServer); err != nil { + t.Fatalf("PLANGUARD_FIXTURE_DSN: no fixture marker: %v", err) + } + if got != want || !sameServer { + t.Fatalf("PLANGUARD_FIXTURE_DSN holds a fixture of hash %q (same server version and C collation: %v), want a finished build of these inputs (%s) on this server, under C collation", got, sameServer, want) + } + t.Logf("reusing the fixture built from these files (%s)", want) + return pool +} + +func TestBuildFixtureThroughMigrationsPostgres(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute) + defer cancel() + testDSN := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if testDSN == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for PostgreSQL tests") + } + pool := fixtureDB(t, ctx) + + // Every embedded migration is applied. + files, err := filepath.Glob("../../db/migrations/*.sql") + if err != nil || len(files) == 0 { + t.Fatalf("glob migrations: %v (%d files)", err, len(files)) + } + var applied []string + rows, err := pool.Query(ctx, "SELECT filename FROM schema_migrations ORDER BY filename") + if err != nil { + t.Fatal(err) + } + for rows.Next() { + var f string + if err := rows.Scan(&f); err != nil { + t.Fatal(err) + } + applied = append(applied, f) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + for _, f := range files { + if !slices.Contains(applied, filepath.Base(f)) { + t.Errorf("migration %s not applied", filepath.Base(f)) + } + } + + var regions int + if err := pool.QueryRow(ctx, "SELECT count(DISTINCT iata) FROM known_routes").Scan(®ions); err != nil { + t.Fatal(err) + } + if regions < 300 { + t.Errorf("known_routes covers %d regions, want >= 300", regions) + } + + // The fixture adds no index: known_routes carries exactly the migrations' set, read from a schema migrated on + // its own in the test database. + schema := "planguard_" + strings.ReplaceAll(uuid.NewString(), "-", "") + cfg, err := pgxpool.ParseConfig(testDSN) + if err != nil { + t.Fatal(err) + } + cfg.ConnConfig.RuntimeParams["search_path"] = schema + cfg.MaxConns = 1 + ref, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + t.Fatal(err) + } + defer ref.Close() + if _, err := ref.Exec(ctx, "CREATE SCHEMA "+schema); err != nil { + t.Fatal(err) + } + defer func() { + if _, err := ref.Exec(context.Background(), "DROP SCHEMA "+schema+" CASCADE"); err != nil { + t.Error(err) + } + }() + if err := db.RunMigrations(ctx, ref); err != nil { + t.Fatal(err) + } + indexes := func(p *pgxpool.Pool, ns string) []string { + var out []string + rows, err := p.Query(ctx, "SELECT indexname FROM pg_indexes WHERE schemaname = $1 AND tablename = 'known_routes' ORDER BY 1", ns) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + for rows.Next() { + var n string + if err := rows.Scan(&n); err != nil { + t.Fatal(err) + } + out = append(out, n) + } + return out + } + if got, want := indexes(pool, "public"), indexes(ref, schema); !slices.Equal(got, want) || len(want) == 0 { + t.Errorf("known_routes indexes %v, want the migrations' %v", got, want) + } + + // The stale region: thousands of rows the statistics have never seen. + var stale int + if err := pool.QueryRow(ctx, "SELECT count(*) FROM known_routes WHERE iata = 'NEW'").Scan(&stale); err != nil { + t.Fatal(err) + } + if stale < 2000 { + t.Errorf("region NEW holds %d routes, want >= 2,000", stale) + } + for _, q := range []string{ + "SELECT * FROM known_routes WHERE iata = 'NEW'", + "SELECT * FROM known_routes WHERE iata >= 'NEW' AND iata <= 'NEW'", + } { + if est := planRows(t, ctx, pool, q); est > 10 { + t.Errorf("%s: estimated %.0f rows, want <= 10 (statistics taken before the region was loaded)", q, est) + } + } + + // A second build finds the marker and returns. + t1 := time.Now() + if err := planguard.BuildFixture(ctx, pool, pgdata.Files); err != nil { + t.Fatal(err) + } + if d := time.Since(t1); d >= 2*time.Second { + t.Errorf("second BuildFixture took %s, want < 2 s (marker hit)", d) + } +} + +// Without a fixture marker BuildFixture writes nothing to a database that holds data or a schema it did not make: +// a row in packets, a freshly migrated Beacon database (its singleton rows), or public tables without a migration +// ledger. Nor to one whose collation is not C. +func TestBuildFixtureRefusesRealDatabasePostgres(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + for _, tc := range []struct { + name, setup, want, create string + migrate bool + }{ + {name: "packets", migrate: true, want: "public.packets holds data", setup: `INSERT INTO packets (packet_hash, payload_type, + payload_version, route_type, raw_payload, raw_header, first_heard_at, last_heard_at) +VALUES ('\x0102030405060708', 4, 0, 1, '\x00', '\x00', now(), now())`}, + {name: "migrated", migrate: true, want: "holds data"}, + {name: "no_ledger", setup: "CREATE TABLE public.notes (body text)", want: "no schema_migrations"}, + // The server's default locale; skipped where that is C. + {name: "collation", create: "CREATE DATABASE %s TEMPLATE template0", want: "a fixture needs C"}, + } { + t.Run(tc.name, func(t *testing.T) { + create := tc.create + if create == "" { + create = planguard.FixtureCreateDatabase + } + pool := throwawayDBAs(t, ctx, "planguard_refuse_"+tc.name, create) + if tc.create != "" { + var collate string + if err := pool.QueryRow(ctx, "SELECT datcollate FROM pg_database WHERE datname = current_database()").Scan(&collate); err != nil { + t.Fatal(err) + } + if collate == "C" { + t.Skip("the server's default collation is C") + } + } + if tc.migrate { + if err := db.RunMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + } + if tc.setup != "" { + if _, err := pool.Exec(ctx, tc.setup); err != nil { + t.Fatal(err) + } + } + snapshot := func() string { + var s string + if err := pool.QueryRow(ctx, `SELECT coalesce(string_agg(n.nspname || '.' || c.relname, ',' ORDER BY n.nspname, c.relname), '') +FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace +WHERE n.nspname NOT IN ('pg_catalog', 'information_schema', 'pg_toast')`).Scan(&s); err != nil { + t.Fatal(err) + } + var rows int + if tc.migrate { + if err := pool.QueryRow(ctx, "SELECT (SELECT count(*) FROM packets) + (SELECT count(*) FROM known_routes) + (SELECT count(*) FROM schema_migrations)").Scan(&rows); err != nil { + t.Fatal(err) + } + } + return fmt.Sprintf("%s|%d", s, rows) + } + before := snapshot() + err := planguard.BuildFixture(ctx, pool, pgdata.Files) + if err == nil || !strings.Contains(err.Error(), tc.want) || (tc.create == "" && !errors.Is(err, planguard.ErrNotFixture)) { + t.Fatalf("BuildFixture = %v, want an error naming %q (ErrNotFixture unless it is the collation)", err, tc.want) + } + if after := snapshot(); after != before { + t.Errorf("BuildFixture wrote after refusing:\nbefore %s\nafter %s", before, after) + } + var src bool + if err := pool.QueryRow(ctx, "SELECT to_regnamespace('planguard_src') IS NOT NULL").Scan(&src); err != nil || src { + t.Errorf("planguard_src exists after the refusal (%v)", err) + } + }) + } +} diff --git a/internal/planguard/fixture_internal_test.go b/internal/planguard/fixture_internal_test.go new file mode 100644 index 0000000..6670817 --- /dev/null +++ b/internal/planguard/fixture_internal_test.go @@ -0,0 +1,161 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package planguard + +import ( + "context" + "fmt" + "os" + "testing" + "testing/fstest" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +// tinyGenerator is a fixture generator small enough to build in seconds. +func tinyGenerator() fstest.MapFS { + return fstest.MapFS{ + "fixture/source.sql": {Data: []byte("CREATE TABLE planguard_src.codes (iata char(3) NOT NULL);")}, + "fixture/testdata/codes.csv": {Data: []byte("iata\nAAA\nBBB\n")}, + "fixture/10_codes.sql": {Data: []byte("INSERT INTO iata_codes (iata) SELECT iata FROM planguard_src.codes;")}, + } +} + +// Every Go-side generator input is part of the hash, so changing one makes a built fixture stale. +func TestFixtureHashCoversGoInputs(t *testing.T) { + base, err := readFixtureFilesWith(tinyGenerator(), currentInputs) + if err != nil { + t.Fatal(err) + } + for name, change := range map[string]func(*generatorInputs){ + "format": func(in *generatorInputs) { in.format++ }, + "settings": func(in *generatorInputs) { in.settings += "; SET work_mem = '128MB'" }, + "stats target": func(in *generatorInputs) { in.statsTarget++ }, + "after analyze": func(in *generatorInputs) { in.afterAnalyze++ }, + } { + in := currentInputs + change(&in) + ff, err := readFixtureFilesWith(tinyGenerator(), in) + if err != nil { + t.Fatal(err) + } + if ff.hash == base.hash { + t.Errorf("changing the %s leaves the fixture hash at %s", name, base.hash) + } + } + if currentInputs.settings != buildSettings || currentInputs.statsTarget != fixtureStatsTgt || + currentInputs.afterAnalyze != fixtureAfterAnalyze || currentInputs.format != fixtureFormat { + t.Errorf("currentInputs %+v do not mirror the constants the build uses", currentInputs) + } +} + +// Files go:embed would leave out (names starting with "." or "_") are left out of the generator and its hash. +func TestFixtureFilesSkipHiddenNames(t *testing.T) { + base, err := readFixtureFiles(tinyGenerator()) + if err != nil { + t.Fatal(err) + } + fsys := tinyGenerator() + for _, p := range []string{"fixture/.DS_Store", "fixture/._10_codes.sql", "fixture/_draft.sql", "fixture/.10_codes.sql.swp", + "fixture/testdata/.DS_Store", "fixture/testdata/_old.csv", "fixture/testdata/.codes.csv.swp"} { + fsys[p] = &fstest.MapFile{Data: []byte("junk")} + } + got, err := readFixtureFiles(fsys) + if err != nil { + t.Fatalf("hidden files break the generator: %v", err) + } + if got.hash != base.hash || len(got.steps) != 1 || len(got.csvs) != 1 { + t.Errorf("hidden files changed the generator: steps %v csvs %v hash %s (want %s)", got.steps, got.csvs, got.hash, base.hash) + } + fsys["fixture/notes.txt"] = &fstest.MapFile{Data: []byte("x")} + if _, err := readFixtureFiles(fsys); err == nil { + t.Error("a visible non-step file in fixture/ was accepted") + } +} + +// A built fixture is reused only while every input matches: the files, the Go-side inputs and the server version. +func TestBuildFixtureRebuildsStaleInputsPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for PostgreSQL tests") + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + admin, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + const name = "planguard_fixture_inputs_test" + drop := func() error { + _, err := admin.Exec(context.Background(), "DROP DATABASE IF EXISTS "+name+" WITH (FORCE)") + return err + } + if err := drop(); err != nil { + t.Fatal(err) + } + if _, err := admin.Exec(ctx, fmt.Sprintf(FixtureCreateDatabase, name)); err != nil { + t.Fatal(err) + } + cfg, err := pgxpool.ParseConfig(dsn) + if err != nil { + t.Fatal(err) + } + cfg.ConnConfig.Database = name + pool, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + pool.Close() + if err := drop(); err != nil { + t.Error(err) + } + admin.Close(context.Background()) + }) + + builtAt := func() time.Time { + var at time.Time + if err := pool.QueryRow(ctx, "SELECT built_at FROM planguard_src.fixture_marker WHERE built_at IS NOT NULL").Scan(&at); err != nil { + t.Fatal(err) + } + return at + } + build := func() { + t.Helper() + if err := BuildFixture(ctx, pool, tinyGenerator()); err != nil { + t.Fatal(err) + } + } + build() + first := builtAt() + build() + if at := builtAt(); !at.Equal(first) { + t.Fatalf("an unchanged fixture was rebuilt (%s -> %s)", first, at) + } + + // The marker says another server version built it. + if _, err := pool.Exec(ctx, "UPDATE planguard_src.fixture_marker SET server_version_num = 1"); err != nil { + t.Fatal(err) + } + build() + second := builtAt() + if !second.After(first) { + t.Errorf("a fixture built on another server version was reused") + } + + // A Go-side input changed. + saved := currentInputs + t.Cleanup(func() { currentInputs = saved }) + currentInputs.settings += "; SET work_mem = '128MB'" + build() + if at := builtAt(); !at.After(second) { + t.Errorf("a fixture built with other build settings was reused") + } + var codes int + if err := pool.QueryRow(ctx, "SELECT count(*) FROM iata_codes").Scan(&codes); err != nil || codes != 2 { + t.Errorf("iata_codes after the rebuilds: %d (%v), want 2 (the tables were emptied first)", codes, err) + } +} diff --git a/internal/planguard/genpolicy.go b/internal/planguard/genpolicy.go new file mode 100644 index 0000000..a244cde --- /dev/null +++ b/internal/planguard/genpolicy.go @@ -0,0 +1,134 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package planguard + +import ( + "bytes" + "errors" + "fmt" + "go/format" + "regexp" + "slices" + "strconv" + "strings" +) + +// hopParam matches a statement that filters on a hop count parameter. Such a statement must also prove its +// rare and absent hops before it is admitted. +var hopParam = regexp.MustCompile(`hop_count\s*(\+\s*0\s*)?=\s*\$\d`) + +// hopShapes are the extra shapes a statement with a hop parameter must cover. +var hopShapes = []string{"rare_hop", "absent_hop"} + +const policyHeader = `// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Code generated by the plan policy generator. DO NOT EDIT. + +package planpolicy + +// Cached lists class G statements: those admitted to the generic-plan class. +` + +// GenPolicy returns gofmt'd source for db/planpolicy/cached_gen.go: the license header, the generated-code line, +// "package planpolicy", the Cached doc comment and `var Cached = map[string]bool{…}` with the admitted sqlc +// names sorted. +// +// A statement is admitted iff its plan is cached, it has a sqlc source and no pending, it is not in legacy, its +// entries' shapes cover every tag of its admission profile (plus rare_hop and absent_hop when its text filters on +// a hop parameter), and every entry of it runs generic and auto with no skip or degrade_to and is neither +// expect_fail nor known_issue. A pending cached statement is left out silently; any other cached statement that +// misses a rule is an error naming the statement and what is missing, never a silent drop. +func GenPolicy(c *Corpus, legacy map[string]bool) ([]byte, error) { + var admitted []string + var errs []error + for _, name := range sortedKeys(c.Statements) { + st := c.Statements[name] + if st.Plan != "cached" || st.Pending != "" { + continue + } + problems := c.admissionProblems(name, st, legacy) + if len(problems) > 0 { + errs = append(errs, fmt.Errorf("statement %s is plan cached but %s", name, strings.Join(problems, "; "))) + continue + } + admitted = append(admitted, st.SQLC) + } + if len(errs) > 0 { + return nil, errors.Join(errs...) + } + slices.Sort(admitted) + admitted = slices.Compact(admitted) + + var b bytes.Buffer + b.WriteString(policyHeader) + if len(admitted) == 0 { + b.WriteString("var Cached = map[string]bool{}\n") + } else { + b.WriteString("var Cached = map[string]bool{\n") + for _, n := range admitted { + fmt.Fprintf(&b, "\t%s: true,\n", strconv.Quote(n)) + } + b.WriteString("}\n") + } + return format.Source(b.Bytes()) +} + +// admissionProblems lists why a non-pending cached statement cannot be admitted; empty means it can. +func (c *Corpus) admissionProblems(name string, st *Statement, legacy map[string]bool) []string { + var out []string + if st.SQLC == "" { + return []string{"has no sqlc source (only sqlc statements are served)"} + } + if legacy[st.SQLC] { + out = append(out, "is in planpolicy.Legacy") + } + want := append([]string(nil), c.AdmissionProfiles[st.Admission]...) + if hopParam.MatchString(st.SQL) { + want = append(want, hopShapes...) + } + have := map[string]bool{} + n := 0 + for _, e := range c.Entries { + if e.Stmt != name { + continue + } + n++ + for _, s := range e.Shapes { + have[s] = true + } + modes := c.ModesOf(e) + for _, m := range []string{ModeGeneric, ModeAuto} { + switch ov := e.Mode[m]; { + case !slices.Contains(modes, m): + out = append(out, fmt.Sprintf("entry %s does not run %s", e.ID, m)) + case ov != nil && ov.Skip != "": + out = append(out, fmt.Sprintf("entry %s skips %s", e.ID, m)) + case ov != nil && ov.DegradeTo != nil: + out = append(out, fmt.Sprintf("entry %s degrades %s", e.ID, m)) + case ov != nil && ov.KnownIssue != "": + out = append(out, fmt.Sprintf("entry %s has a known_issue in %s", e.ID, m)) + } + } + if e.ExpectFail { + out = append(out, fmt.Sprintf("entry %s is expect_fail", e.ID)) + } + if e.KnownIssue != "" { + out = append(out, fmt.Sprintf("entry %s has a known_issue", e.ID)) + } + } + if n == 0 { + out = append(out, "has no entries") + } + var missing []string + for _, s := range want { + if !have[s] && !slices.Contains(missing, s) { + missing = append(missing, s) + } + } + if len(missing) > 0 { + out = append(out, fmt.Sprintf("its entries' shapes miss %s (admission %s)", strings.Join(missing, ", "), st.Admission)) + } + return out +} diff --git a/internal/planguard/genpolicy_test.go b/internal/planguard/genpolicy_test.go new file mode 100644 index 0000000..f3db415 --- /dev/null +++ b/internal/planguard/genpolicy_test.go @@ -0,0 +1,130 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package planguard_test + +import ( + "bytes" + "maps" + "os" + "slices" + "strings" + "testing" + + pgdata "github.com/MeshCore-Beacon/beacon-server/db/planguard" + "github.com/MeshCore-Beacon/beacon-server/db/planpolicy" + "github.com/MeshCore-Beacon/beacon-server/internal/planguard" +) + +func repoSQLC(t *testing.T) map[string]string { + t.Helper() + sqlcText, err := planguard.LoadStatements("../../db/sqlc") + if err != nil { + t.Fatal(err) + } + return sqlcText +} + +func genPolicyErr(t *testing.T, file string, legacy map[string]bool) error { + t.Helper() + c, err := planguard.LoadCorpus(os.DirFS("testdata/corpus"), file, repoSQLC(t)) + if err != nil { + t.Fatalf("%s: %v", file, err) + } + _, err = planguard.GenPolicy(c, legacy) + return err +} + +func TestGenPolicyRejectsCachedWithoutAdversarialShapes(t *testing.T) { + cases := []struct { + file string + legacy map[string]bool + want []string // every fragment must appear in the error + }{ + {"genpolicy_hop.yaml", nil, []string{"ListKnownRoutesGlobalHop", "rare_hop", "absent_hop"}}, + {"genpolicy_hop_skip.yaml", nil, []string{"ListKnownRoutesGlobalHop", "entry gh.rare skips generic", "entry gh.absent skips generic"}}, + {"genpolicy_region_no_stale.yaml", nil, []string{"ListKnownRoutesByIATA", "stale_region"}}, + {"genpolicy_legacy_pending.yaml", map[string]bool{"GetNodeByID": true}, []string{"GetNodeByID", "planpolicy.Legacy"}}, + {"genpolicy_refuse_mode_known_issue_generic.yaml", nil, []string{"GetNodeByID", "entry node.miss has a known_issue in generic"}}, + {"genpolicy_refuse_mode_known_issue_auto.yaml", nil, []string{"GetNodeByID", "entry node.miss has a known_issue in auto"}}, + {"genpolicy_refuse_entry_known_issue.yaml", nil, []string{"GetNodeByID", "entry node.miss has a known_issue"}}, + {"genpolicy_refuse_expect_fail.yaml", nil, []string{"GetNodeByID", "entry node.miss is expect_fail"}}, + {"genpolicy_refuse_degrade_to.yaml", nil, []string{"GetNodeByID", "entry node.miss degrades auto"}}, + {"genpolicy_refuse_no_generic.yaml", nil, []string{"GetNodeByID", "entry node.miss does not run generic"}}, + {"genpolicy_refuse_no_auto.yaml", nil, []string{"GetNodeByID", "entry node.miss does not run auto"}}, + {"genpolicy_refuse_skip_auto.yaml", nil, []string{"GetNodeByID", "entry node.miss skips auto"}}, + {"genpolicy_refuse_no_sqlc.yaml", nil, []string{"statement Frozen", "has no sqlc source"}}, + {"genpolicy_refuse_no_entries.yaml", nil, []string{"GetNodeByID", "has no entries"}}, + } + for _, tc := range cases { + t.Run(tc.file, func(t *testing.T) { + err := genPolicyErr(t, tc.file, tc.legacy) + if err == nil { + t.Fatal("want an error") + } + for _, w := range tc.want { + if !strings.Contains(err.Error(), w) { + t.Errorf("err = %v, want it to contain %q", err, w) + } + } + }) + } + + // The same corpus with GetNodeByID out of Legacy: GetNodeByID is admitted and the pending cached statement is + // left out without an error. + c, err := planguard.LoadCorpus(os.DirFS("testdata/corpus"), "genpolicy_legacy_pending.yaml", repoSQLC(t)) + if err != nil { + t.Fatal(err) + } + src, err := planguard.GenPolicy(c, nil) + if err != nil { + t.Fatalf("GenPolicy: %v", err) + } + if !bytes.Contains(src, []byte(`"GetNodeByID": true`)) || bytes.Contains(src, []byte("NotMergedYet")) { + t.Errorf("source:\n%s\nwant GetNodeByID admitted and the pending statement left out", src) + } +} + +func TestGenPolicyEmptyCached(t *testing.T) { + c, err := planguard.LoadCorpus(mapFS(map[string]string{"c.yaml": "statements:\n C: {sqlc: C, plan: custom}\n"}), "c.yaml", testSQLC) + if err != nil { + t.Fatal(err) + } + src, err := planguard.GenPolicy(c, nil) + if err != nil { + t.Fatal(err) + } + if !bytes.Contains(src, []byte("var Cached = map[string]bool{}\n")) { + t.Errorf("source:\n%s\nwant an empty Cached", src) + } +} + +func TestGenPolicyMatchesCorpus(t *testing.T) { + c, err := planguard.LoadCorpus(pgdata.Files, "corpus.yaml", repoSQLC(t)) + if err != nil { + t.Fatal(err) + } + src, err := planguard.GenPolicy(c, planpolicy.Legacy) + if err != nil { + t.Fatal(err) + } + want, err := os.ReadFile("../../db/planpolicy/cached_gen.go") + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(src, want) { + t.Errorf("db/planpolicy/cached_gen.go is stale; run go run ./cmd/planguard gen-policy. Generated:\n%s", src) + } + got := slices.Sorted(maps.Keys(planpolicy.Cached)) + if !slices.Equal(got, []string{"GetNodeByID", "ListKnownRoutesByIATA", "ListKnownRoutesGlobal"}) { + t.Errorf("planpolicy.Cached = %v", got) + } + for _, line := range []string{ + "// Code generated by the plan policy generator. DO NOT EDIT.", + "// Cached lists class G statements: those admitted to the generic-plan class.", + } { + if !bytes.Contains(src, []byte(line+"\n")) { + t.Errorf("generated source lacks %q", line) + } + } +} diff --git a/internal/planguard/helpers_test.go b/internal/planguard/helpers_test.go new file mode 100644 index 0000000..31864d6 --- /dev/null +++ b/internal/planguard/helpers_test.go @@ -0,0 +1,35 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package planguard_test + +import ( + "io/fs" + "os" + "path/filepath" + "testing" + "testing/fstest" +) + +func mapFS(files map[string]string) fs.FS { + m := fstest.MapFS{} + for name, body := range files { + m[name] = &fstest.MapFile{Data: []byte(body)} + } + return m +} + +func dirFS(dir string) fs.FS { return os.DirFS(dir) } + +func sqlcFiles(t *testing.T) []string { + t.Helper() + matches, err := filepath.Glob("../../db/sqlc/*.sql.go") + if err != nil || len(matches) == 0 { + t.Fatalf("glob db/sqlc: %v (%d files)", err, len(matches)) + } + out := make([]string, len(matches)) + for i, m := range matches { + out[i] = filepath.Base(m) + } + return out +} diff --git a/internal/planguard/multistatement_internal_test.go b/internal/planguard/multistatement_internal_test.go new file mode 100644 index 0000000..829d972 --- /dev/null +++ b/internal/planguard/multistatement_internal_test.go @@ -0,0 +1,101 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package planguard + +import ( + "strings" + "testing" +) + +func TestStatementProblem(t *testing.T) { + for sql, want := range map[string]string{ + "SELECT 1": "", + "SELECT 1;": "", + "SELECT 1;\n ": "", + "SELECT ';' AS a, \"x;y\" AS b": "", + "SELECT $$;$$, $q$ ; $q$": "", + "SELECT $q$ \\! id $q$": "backslash in a dollar quote", + "SELECT $é$ ; $é$": "", // a dollar quote: tags take bytes ≥ 0x80, as in PostgreSQL + "SELECT 1 AS aé$x$; COMMIT; SELECT $x$": "more than one statement", // aé$x$ is one identifier + "SELECT xE'a', é'b'": "", + "SELECT 1E'\\';'": "escape string", + "SELECT $E'x'": "escape string", + "SELECT 1 -- ; trailing\n": "", + "SELECT 1 /* ; /* nested */ */": "", + "SELECT a$b FROM t WHERE x = $1": "", + "SELECT '\\x0102'::bytea, '\\'": "", // standard_conforming_strings: the backslash is plain text + "SELECT 'it''s'": "", + "SELECT 1; COMMIT": "more than one statement", + "SELECT 1;;": "more than one statement", + "SELECT 1; COMMIT; DELETE FROM nodes": "more than one statement", + "SELECT (SELECT 1); INSERT INTO t VALUES(1)": "more than one statement", + "SELECT E'\\';'": "escape string", + "SELECT e'\\';' ; COMMIT": "escape string", + "SELECT x FROM t WHERE y = E'a'": "escape string", + "SELECT 1\n\\! id": "backslash outside a literal", + "SELECT 1 \\gexec": "backslash outside a literal", + "SELECT 1 -- \\! id": "backslash in a comment", + "SELECT 1 /* \\o /tmp/x */": "backslash in a comment", + "SELECT \"a\\b\"": "backslash in a quoted identifier", + "SELECT 'abc": "does not end", + "SELECT $q$abc": "does not end", + "SELECT 1 /* abc": "does not end", + } { + got := statementProblem(sql) + if (want == "") != (got == "") || !strings.Contains(got, want) { + t.Errorf("statementProblem(%q) = %q, want %q", sql, got, want) + } + } +} + +func TestCheckRawParam(t *testing.T) { + for _, ok := range []string{ + "10", "-3", "1.5", "NULL", "null", "true", "FALSE", "'flood'", "'it''s'", "'\\x01'::bytea", + "'2026-10-17 17:00+00'::timestamptz", "ARRAY['LIS']::bpchar[]", "ARRAY['FAO', 'LIS','OPO']::bpchar[]", + "ARRAY[1, 2]", "NULL::pg_catalog.int4", "'a' :: text", + } { + if err := checkRawParam(ok); err != nil { + t.Errorf("checkRawParam(%q) = %v, want nil", ok, err) + } + } + for _, bad := range []string{ + "1; DROP TABLE nodes", "'x'\n\\! id", "'x\n\\! id'", "now()", "1 + 1", "(SELECT 1)", "'a'::text; COMMIT", + "ARRAY['a'", "ARRAY[]", "'a'::", "'a'::double precision", "E'x'", "'a' 'b'", "trueish", "10abc", "$1", "", + "'a'::text[][]", "'x'\t", + } { + if err := checkRawParam(bad); err == nil { + t.Errorf("checkRawParam(%q) = nil, want an error", bad) + } + } +} + +func TestCheckScriptLines(t *testing.T) { + if err := checkScriptLines("SELECT '\\x01'\nFROM t"); err != nil { + t.Errorf("backslash inside a line: %v", err) + } + for _, body := range []string{"SELECT 'a\n\\! id\n'", "SELECT 1\n \\o /tmp/x"} { + if err := checkScriptLines(body); err == nil { + t.Errorf("checkScriptLines(%q) = nil, want a line starting with a backslash", body) + } + } +} + +// Every sqlc statement passes the guard, with no trailing ';' in its constant. +func TestSQLCTextsPassTheStatementGuard(t *testing.T) { + texts, err := LoadStatements("../../db/sqlc") + if err != nil { + t.Fatal(err) + } + for name, sql := range texts { + if err := checkStatementText(sql); err != nil { + t.Errorf("sqlc %s: %v", name, err) + } + if strings.HasSuffix(strings.TrimSpace(sql), ";") { + t.Errorf("sqlc %s ends with ';'", name) + } + } + if len(texts) < 100 { + t.Errorf("only %d sqlc statements", len(texts)) + } +} diff --git a/internal/planguard/multistatement_test.go b/internal/planguard/multistatement_test.go new file mode 100644 index 0000000..c4e770d --- /dev/null +++ b/internal/planguard/multistatement_test.go @@ -0,0 +1,253 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package planguard_test + +import ( + "bytes" + "context" + "os" + "regexp" + "strings" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/planguard" + "github.com/jackc/pgx/v5/pgconn" +) + +// Statement texts and resolvers go over the simple protocol and into psql scripts, where a second statement (a +// COMMIT) would run outside the READ ONLY transaction. Text with a ';' outside literals and comments is refused at +// load, and again by the runner and the emitter. +func TestCorpusRefusesMultiStatement(t *testing.T) { + const base = ` +statements: + Q: {sqlc: Q, plan: custom} + F: {file: f.sql, plan: harness} +resolvers: + top: SELECT public_key FROM nodes LIMIT 1 +entries: + - {id: q, stmt: Q, params: [{r: top}]} + - {id: f, stmt: F, params: [{sql: "SELECT 1"}]} +` + load := func(yaml, file string, sqlc map[string]string) error { + _, err := planguard.LoadCorpus(mapFS(map[string]string{"c.yaml": yaml, "f.sql": file}), "c.yaml", sqlc) + return err + } + if err := load(base, "SELECT ';' AS a, $$;$$ AS b -- ;\n;\n", testSQLC); err != nil { + t.Fatalf("';' in literals and comments, and a trailing ';': %v", err) + } + // standard_conforming_strings is on: a backslash inside a plain literal is text. + if err := load(base, "SELECT '\\x01'::bytea, 'a\\b'", testSQLC); err != nil { + t.Fatalf("backslash inside a plain literal: %v", err) + } + const multi, esc, meta = "more than one statement", "escape string", "meta-command" + bad := []struct { + name string + err error + want string + }{ + {"file statement", load(base, "SELECT 1; COMMIT", testSQLC), multi}, + {"sqlc statement", load(base, "SELECT 1", map[string]string{"Q": "-- name: Q :one\nSELECT 1; COMMIT; DELETE FROM nodes"}), multi}, + {"resolver", load(strings.Replace(base, "LIMIT 1", "LIMIT 1; COMMIT", 1), "SELECT 1", testSQLC), multi}, + {"inline sql", load(strings.Replace(base, `{sql: "SELECT 1"}`, `{sql: "SELECT 1; COMMIT"}`, 1), "SELECT 1", testSQLC), multi}, + {"file escape string", load(base, "SELECT E'\\';'", testSQLC), esc}, + {"file psql meta-command", load(base, "SELECT 1\n\\! id", testSQLC), meta}, + {"sqlc escape string", load(base, "SELECT 1", map[string]string{"Q": "-- name: Q :one\nSELECT e'\\';' FROM nodes"}), esc}, + {"resolver meta-command", load(strings.Replace(base, "LIMIT 1", "LIMIT 1 \\! id", 1), "SELECT 1", testSQLC), meta}, + {"raw statement", load(strings.Replace(base, `{sql: "SELECT 1"}`, `{raw: "1; DROP TABLE nodes"}`, 1), "SELECT 1", testSQLC), "raw"}, + {"raw meta-command", load(strings.Replace(base, `{sql: "SELECT 1"}`, `{raw: "'x'\n\\! id"}`, 1), "SELECT 1", testSQLC), "raw"}, + {"raw literal meta-command", load(strings.Replace(base, `{sql: "SELECT 1"}`, `{raw: "'x\n\\! id'"}`, 1), "SELECT 1", testSQLC), "raw"}, + {"settings value", load("tiers: {local: {settings: {work_mem: \"1; COMMIT\"}}}\n"+base, "SELECT 1", testSQLC), "work_mem"}, + {"dollar quote backslash", load(base, "SELECT $q$ \\! id $q$", testSQLC), "dollar quote"}, + {"inline sql escape string", load(strings.Replace(base, `{sql: "SELECT 1"}`, `{sql: "SELECT E'x'"}`, 1), "SELECT 1", testSQLC), esc}, + } + for _, b := range bad { + if b.err == nil || !strings.Contains(b.err.Error(), b.want) { + t.Errorf("%s: err %v, want %q", b.name, b.err, b.want) + } + } +} + +var commitRe = regexp.MustCompile(`\bCOMMIT\b`) + +// recordSession answers every command with an empty result and records it. +type recordSession struct{ sent []string } + +func (r *recordSession) Exec(_ context.Context, sql string) ([]*pgconn.Result, error) { + r.sent = append(r.sent, sql) + return []*pgconn.Result{{}}, nil +} + +func TestRunnerRefusesMultiStatement(t *testing.T) { + c := loadEmitCorpus(t) + c.Statements["Q"].SQL = "-- name: Q :one\nSELECT 1; COMMIT; DELETE FROM nodes" + c.Resolvers["top"] = "SELECT 1; COMMIT" + s := &recordSession{} + rep, err := planguard.Run(context.Background(), c, s, planguard.RunOptions{ + Tier: "local", Baseline: planguard.BaselineOff, Only: regexp.MustCompile(`^q\.(top|lit)$`)}) + if err != nil { + t.Fatal(err) + } + for _, sql := range s.sent { + if commitRe.MatchString(sql) || strings.Contains(sql, "DELETE") { + t.Errorf("sent %q", sql) + } + } + n := 0 + for _, r := range rep.Results { + if r.Stmt != "Q" { + continue + } + n++ + if r.Verdict != planguard.VerdictError || len(r.Failures) == 0 || !strings.Contains(r.Failures[0].Msg, "more than one statement") { + t.Errorf("%s|%s: verdict %s failures %v, want ERROR (more than one statement)", r.ID, r.Mode, r.Verdict, r.Failures) + } + } + if n == 0 { + t.Fatal("no run of Q") + } +} + +// A settings value is checked again when a script is emitted, so a corpus changed after load cannot carry a ';'. +func TestEmitRefusesSettingsValue(t *testing.T) { + c := loadEmitCorpus(t) + c.Tiers["prod"] = planguard.TierSettings{Settings: map[string]string{"work_mem": "16MB; COMMIT"}} + var b bytes.Buffer + v := "\\x01" + err := planguard.EmitPlans(&b, c, planguard.RunOptions{Tier: "prod"}, map[string]*string{"top": &v}, "0123456789abcdef") + if err == nil || !strings.Contains(err.Error(), "work_mem") { + t.Errorf("EmitPlans with a settings value holding ';': err %v", err) + } + if err := planguard.EmitResolvers(&b, c, planguard.RunOptions{Tier: "prod"}, "0123456789abcdef"); err == nil { + t.Error("EmitResolvers with a settings value holding ';': no error") + } +} + +// The scanner agrees with PostgreSQL on high-bit bytes in identifiers and dollar-quote tags: what it passes runs as +// one statement, and what it refuses for ';' really is two. +func TestStatementScannerMatchesPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for PostgreSQL tests") + } + ctx := context.Background() + pc, err := pgconn.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer pc.Close(ctx) + for sql, statements := range map[string]int{ + "SELECT $é$ ; $é$ AS v": 1, + "SELECT 1 AS aé$x$": 1, + "SELECT 1 AS aé$x$; SELECT 2 AS b$x$": 2, + "SELECT ';' AS a, $q$;$q$ AS b": 1, + } { + res, err := pc.Exec(ctx, sql).ReadAll() + if err != nil { + t.Fatalf("%q: %v", sql, err) + } + if len(res) != statements { + t.Errorf("%q: PostgreSQL ran %d statements, want %d", sql, len(res), statements) + } + _, loadErr := planguard.LoadCorpus(mapFS(map[string]string{"c.yaml": "statements:\n F: {file: f.sql, plan: harness}\nentries:\n - {id: f, stmt: F}\n", "f.sql": sql}), "c.yaml", testSQLC) + if refused := loadErr != nil; refused != (statements > 1) { + t.Errorf("%q: %d statements in PostgreSQL, but the guard refused=%v (%v)", sql, statements, refused, loadErr) + } + } +} + +func TestRunnerRefusesMetaCommands(t *testing.T) { + for name, text := range map[string]string{ + "meta-command": "-- name: Q :one\nSELECT 1\n\\! id", + "escape string": "-- name: Q :one\nSELECT E'\\';' FROM nodes", + } { + c := loadEmitCorpus(t) + c.Statements["Q"].SQL = text + s := &recordSession{} + rep, err := planguard.Run(context.Background(), c, s, planguard.RunOptions{ + Tier: "local", Baseline: planguard.BaselineOff, Only: regexp.MustCompile(`^q\.lit$`)}) + if err != nil { + t.Fatal(err) + } + for _, sql := range s.sent { + if strings.Contains(sql, "PREPARE") { + t.Errorf("%s: sent %q", name, sql) + } + } + if len(rep.Results) == 0 || rep.Results[0].Verdict != planguard.VerdictError { + t.Errorf("%s: results %+v, want ERROR", name, rep.Results) + } + } +} + +func TestEmitRefusesMultiStatement(t *testing.T) { + const nonce = "0123456789abcdef" + c := loadEmitCorpus(t) + c.Resolvers["top"] = "SELECT 1; COMMIT" + var b bytes.Buffer + if err := planguard.EmitResolvers(&b, c, planguard.RunOptions{Tier: "prod"}, nonce); err == nil || !strings.Contains(err.Error(), "more than one statement") { + t.Errorf("EmitResolvers: err %v, want more than one statement", err) + } + + for _, r := range []string{"SELECT E'\\';'", "SELECT 1\n\\! id"} { + c := loadEmitCorpus(t) + c.Resolvers["top"] = r + b.Reset() + if err := planguard.EmitResolvers(&b, c, planguard.RunOptions{Tier: "prod"}, nonce); err == nil { + t.Errorf("EmitResolvers with resolver %q: no error", r) + } + } + // A line of an emitted body that starts with a backslash is refused even inside a literal. + c = loadEmitCorpus(t) + b.Reset() + line := "x\n\\! id" + if err := planguard.EmitPlans(&b, c, planguard.RunOptions{Tier: "prod"}, map[string]*string{"top": &line}, nonce); err == nil || + !strings.Contains(err.Error(), "backslash") { + t.Errorf("EmitPlans with a param line starting with a backslash: err %v", err) + } + + c = loadEmitCorpus(t) + c.Statements["Q"].SQL = "-- name: Q :one\nSELECT 1; DELETE FROM nodes" + b.Reset() + params := map[string]*string{"top": nil, "gone": nil} + v := "\\x01" + params["top"] = &v + if err := planguard.EmitPlans(&b, c, planguard.RunOptions{Tier: "prod"}, params, nonce); err != nil { + t.Fatal(err) + } + if strings.Contains(b.String(), "DELETE") { + t.Errorf("EmitPlans wrote the second statement:\n%s", b.String()) + } + if strings.Contains(b.String(), "PREPARE") { + t.Errorf("EmitPlans prepared a refused statement:\n%s", b.String()) + } +} + +// Every statement is checked again once rendered: a raw param changed after load cannot carry a second statement +// into the runner's session or an emitted script. +func TestRenderedStatementsAreChecked(t *testing.T) { + c := loadEmitCorpus(t) + c.Entry("q.lit").Params = []any{map[string]any{"raw": "1); COMMIT; SELECT (1"}} + s := &recordSession{} + rep, err := planguard.Run(context.Background(), c, s, planguard.RunOptions{ + Tier: "local", Baseline: planguard.BaselineOff, Only: regexp.MustCompile(`^q\.lit$`)}) + if err != nil { + t.Fatal(err) + } + for _, sql := range s.sent { + if commitRe.MatchString(sql) || strings.Contains(sql, "PREPARE") { + t.Errorf("sent %q", sql) + } + } + if len(rep.Results) == 0 || rep.Results[0].Verdict != planguard.VerdictError || + !strings.Contains(rep.Results[0].Failures[0].Msg, "rendered statement") { + t.Errorf("results %+v, want ERROR (rendered statement)", rep.Results) + } + var b bytes.Buffer + if err := planguard.EmitPlans(&b, c, planguard.RunOptions{Tier: "prod", Only: regexp.MustCompile(`^q\.lit$`)}, map[string]*string{}, "0123456789abcdef"); err != nil { + t.Fatal(err) + } + if commitRe.MatchString(b.String()) { + t.Errorf("EmitPlans wrote the second statement:\n%s", b.String()) + } +} diff --git a/internal/planguard/plan.go b/internal/planguard/plan.go new file mode 100644 index 0000000..ab8a7ea --- /dev/null +++ b/internal/planguard/plan.go @@ -0,0 +1,207 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package planguard + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "regexp" + "strings" +) + +// Node is one plan node of EXPLAIN (FORMAT JSON) output, as decoded by encoding/json. +type Node map[string]any + +func (n Node) str(k string) string { + s, _ := n[k].(string) + return s +} + +func (n Node) num(k string) float64 { + f, _ := n[k].(float64) + return f +} + +func (n Node) kids() []Node { + arr, _ := n["Plans"].([]any) + out := make([]Node, 0, len(arr)) + for _, x := range arr { + if m, ok := x.(map[string]any); ok { + out = append(out, Node(m)) + } + } + return out +} + +// Explain is one EXPLAIN (FORMAT JSON) result. +type Explain struct { + Root Node + PlanningMs float64 + ExecMs float64 + Analyzed bool // the output carries an Execution Time (EXPLAIN ANALYZE) + PlanBuffers int64 // shared hit + read while planning (BUFFERS) +} + +// ParseExplain decodes EXPLAIN (FORMAT JSON) text. Empty output, an empty array or a result with no plan is an +// error. +func ParseExplain(b []byte) (*Explain, error) { + b = bytes.TrimSpace(b) + if len(b) == 0 { + return nil, errors.New("explain: empty output") + } + var arr []map[string]any + if err := json.Unmarshal(b, &arr); err != nil { + return nil, fmt.Errorf("explain json: %w (%.200q)", err, b) + } + if len(arr) == 0 { + return nil, errors.New("explain: empty result array") + } + top := Node(arr[0]) + root, ok := top["Plan"].(map[string]any) + if !ok { + return nil, errors.New("explain: result has no Plan object") + } + ex := &Explain{Root: Node(root), PlanningMs: top.num("Planning Time")} + if v, ok := top["Execution Time"].(float64); ok { + ex.ExecMs, ex.Analyzed = v, true + } + if p, ok := top["Planning"].(map[string]any); ok { + ex.PlanBuffers = sharedBlocks(Node(p)) + } + return ex, nil +} + +func sharedBlocks(n Node) int64 { + return int64(n.num("Shared Hit Blocks") + n.num("Shared Read Blocks")) +} + +// isSubplan reports whether n hangs under its parent as a SubPlan or InitPlan: a separate query the parent +// runs, not part of the parent's own scan. +func isSubplan(n Node) bool { + pr := n.str("Parent Relationship") + return pr == "SubPlan" || pr == "InitPlan" +} + +// bitmapIndexScans returns the Bitmap Index Scans that feed a Bitmap Heap Scan (through BitmapAnd/BitmapOr), +// leaving out any inside a SubPlan or InitPlan of the scan. +func bitmapIndexScans(n Node) []Node { + var out []Node + var rec func(m Node) + rec = func(m Node) { + if m.str("Node Type") == "Bitmap Index Scan" { + out = append(out, m) + } + for _, k := range m.kids() { + if !isSubplan(k) { + rec(k) + } + } + } + rec(n) + return out +} + +// walk visits n and its descendants depth first; path holds n's ancestors, root first. +func walk(n Node, fn func(n Node, path []Node)) { + var rec func(n Node, path []Node) + rec = func(n Node, path []Node) { + fn(n, path) + next := append(path[:len(path):len(path)], n) + for _, k := range n.kids() { + rec(k, next) + } + } + rec(n, nil) +} + +func isScan(t string) bool { + switch t { + case "Seq Scan", "Index Scan", "Index Only Scan", "Bitmap Heap Scan", "Tid Scan", "Tid Range Scan", "Sample Scan": + return true + } + return false +} + +// indexesOf returns the indexes a scan node uses: its Index Name, or for a Bitmap Heap Scan those of the Bitmap +// Index Scans feeding it (not those of its SubPlans). +func indexesOf(n Node) []string { + if ix := n.str("Index Name"); ix != "" { + return []string{ix} + } + var out []string + if n.str("Node Type") == "Bitmap Heap Scan" { + for _, m := range bitmapIndexScans(n) { + out = append(out, m.str("Index Name")) + } + } + return out +} + +// condsOf joins a scan's Index Cond and Recheck Cond and, for a Bitmap Heap Scan, the conds of the Bitmap Index +// Scans feeding it (not those of its SubPlans). +func condsOf(n Node) string { + var parts []string + for _, k := range []string{"Index Cond", "Recheck Cond"} { + if v := n.str(k); v != "" { + parts = append(parts, v) + } + } + if n.str("Node Type") == "Bitmap Heap Scan" { + for _, m := range bitmapIndexScans(n) { + parts = append(parts, m.str("Index Cond")) + } + } + return strings.Join(parts, " && ") +} + +var wsRe = regexp.MustCompile(`\s+`) + +func normWS(s string) string { return strings.TrimSpace(wsRe.ReplaceAllString(s, " ")) } + +// condHas reports whether a scan's conds hold the fragment, whitespace-normalised. +func condHas(n Node, fragment string) bool { + return strings.Contains(normWS(condsOf(n)), normWS(fragment)) +} + +// Signature is a plan's shape, one line per node: node type, relation and alias, index, scan direction, partial +// mode, parent relationship, join type, strategy and "Parallel". It holds no costs, row counts or values, so it is the baseline diff key. +func Signature(root Node) []string { + var out []string + walk(root, func(n Node, path []Node) { + var b strings.Builder + b.WriteString(strings.Repeat(" ", len(path))) + if pa, _ := n["Parallel Aware"].(bool); pa { + b.WriteString("Parallel ") + } + b.WriteString(n.str("Node Type")) + if r := n.str("Relation Name"); r != "" { + b.WriteString(" on " + r) + if a := n.str("Alias"); a != "" && a != r { + b.WriteString(" " + a) + } + } + if ix := n.str("Index Name"); ix != "" { + b.WriteString(" using " + ix) + } + if sd := n.str("Scan Direction"); sd != "" { + b.WriteString(" scan=" + sd) + } + if pm := n.str("Partial Mode"); pm != "" { + b.WriteString(" partial=" + pm) + } + if pr := n.str("Parent Relationship"); pr != "" && pr != "Outer" && pr != "Inner" && pr != "Member" { + b.WriteString(" [" + pr + "]") + } + if j := n.str("Join Type"); j != "" && j != "Inner" { + b.WriteString(" (" + j + ")") + } + if st := n.str("Strategy"); st != "" && st != "Plain" { + b.WriteString(" {" + st + "}") + } + out = append(out, b.String()) + }) + return out +} diff --git a/internal/planguard/plan_test.go b/internal/planguard/plan_test.go new file mode 100644 index 0000000..99ae924 --- /dev/null +++ b/internal/planguard/plan_test.go @@ -0,0 +1,101 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package planguard_test + +import ( + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/planguard" +) + +// loadPlan reads a trimmed EXPLAIN (FORMAT JSON) file from testdata/plans. +func loadPlan(t *testing.T, name string) *planguard.Explain { + t.Helper() + b, err := os.ReadFile(filepath.Join("testdata", "plans", name)) + if err != nil { + t.Fatal(err) + } + ex, err := planguard.ParseExplain(b) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + return ex +} + +func TestParseExplainRejectsGarbage(t *testing.T) { + for _, in := range []string{"", "[]", "{", `[{"Plan": 1}]`, `[{}]`} { + if _, err := planguard.ParseExplain([]byte(in)); err == nil { + t.Errorf("ParseExplain(%q) = nil error, want one", in) + } + } +} + +func TestParseExplainReadsTimesAndBuffers(t *testing.T) { + ex := loadPlan(t, "globalhop_rare_retired.json") + if !ex.Analyzed { + t.Error("Analyzed = false for an ANALYZE plan") + } + if ex.PlanBuffers != 4 { + t.Errorf("PlanBuffers = %d, want 4", ex.PlanBuffers) + } + if ex.PlanningMs <= 0 || ex.ExecMs <= 0 { + t.Errorf("PlanningMs = %v, ExecMs = %v; want both > 0", ex.PlanningMs, ex.ExecMs) + } + if got := ex.Root["Node Type"]; got != "Limit" { + t.Errorf("root = %v, want Limit", got) + } +} + +func TestSignatureIgnoresCostsAndValues(t *testing.T) { + // The same shape at different costs, row counts and literals ('SEA' vs 'BWE', 51 vs 201 rows). + a := loadPlan(t, "routes_byiata_custom.json") + b := loadPlan(t, "routes_byiata_mid.json") + if a.Root["Total Cost"] == b.Root["Total Cost"] { + t.Fatal("fixture plans have equal costs; the test would prove nothing") + } + sa, sb := planguard.Signature(a.Root), planguard.Signature(b.Root) + if !reflect.DeepEqual(sa, sb) { + t.Fatalf("signatures differ:\n%s\n---\n%s", strings.Join(sa, "\n"), strings.Join(sb, "\n")) + } + want := []string{"Limit", " Index Scan on known_routes using idx_known_routes_iata_keyset scan=Forward"} + if !reflect.DeepEqual(sa, want) { + t.Errorf("Signature = %q, want %q", sa, want) + } + + c := loadPlan(t, "routes_byiata_mid.json") + kid := c.Root["Plans"].([]any)[0].(map[string]any) + kid["Index Name"] = "idx_known_routes_iata" + if reflect.DeepEqual(planguard.Signature(c.Root), sa) { + t.Error("a changed index left the signature unchanged") + } + + d := loadPlan(t, "routes_byiata_mid.json") + d.Root["Plans"].([]any)[0].(map[string]any)["Scan Direction"] = "Backward" + if reflect.DeepEqual(planguard.Signature(d.Root), sa) { + t.Error("a forward/backward flip left the signature unchanged") + } +} + +func TestSignatureKeepsAliasJoinAndSubplanShape(t *testing.T) { + sig := strings.Join(planguard.Signature(loadPlan(t, "bysites_ok.json").Root), "\n") + for _, want := range []string{ + " Limit [InitPlan]", + "Index Only Scan on packet_sites ps using idx_packet_sites_site scan=Forward", + "Index Only Scan on packet_sites x using packet_sites_pkey scan=Forward [SubPlan]", + "Aggregate partial=Simple {Sorted}", + } { + if !strings.Contains(sig, want) { + t.Errorf("signature lacks %q:\n%s", want, sig) + } + } + for _, banned := range []string{"Cost", "Rows", "2026-10-18", "iata ="} { + if strings.Contains(sig, banned) { + t.Errorf("signature holds %q:\n%s", banned, sig) + } + } +} diff --git a/internal/planguard/prod.go b/internal/planguard/prod.go new file mode 100644 index 0000000..579edd4 --- /dev/null +++ b/internal/planguard/prod.go @@ -0,0 +1,100 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package planguard + +import ( + "context" + "fmt" + "io/fs" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/jackc/pgx/v5/pgxpool" +) + +// prodRunTimeoutS bounds each EXPLAIN of the in-server prod tier; resolvers keep the tier's 20 s. +const prodRunTimeoutS = 5 + +// ProdChecker loads the embedded corpus once (db.SQLCSources, pgdata.Files) and runs tier prod on one +// connection acquired from pool: modes custom and generic (auto needs executions; the prod tier never executes), +// resolvers with statement_timeout 20 s, EXPLAINs with 5 s, lock_timeout 1 s, every transaction READ ONLY. It +// never runs EXPLAIN ANALYZE and changes no session setting: every SET is LOCAL and every prepared statement is +// deallocated. The prod tier names no search_path, so the runs plan in the pool's own. +type ProdChecker struct { + corpus *Corpus + pool *pgxpool.Pool + opts RunOptions +} + +// NewProdChecker loads the corpus (corpus.yaml at the root of corpus) with the statement texts of the sqlc +// files in sqlc (sqlc/*.sql.go). +func NewProdChecker(pool *pgxpool.Pool, sqlc fs.FS, corpus fs.FS) (*ProdChecker, error) { + text, err := LoadStatementsFS(sqlc, "sqlc/*.sql.go") + if err != nil { + return nil, fmt.Errorf("plan check: statements: %w", err) + } + c, err := LoadCorpus(corpus, "corpus.yaml", text) + if err != nil { + return nil, fmt.Errorf("plan check: corpus: %w", err) + } + p := &ProdChecker{corpus: c, pool: pool, opts: RunOptions{ + Tier: "prod", + KeepModes: []string{ModeCustom, ModeGeneric}, + RunTimeoutS: prodRunTimeoutS, + Baseline: BaselineOff, + }} + if err := p.opts.validate(c); err != nil { + return nil, fmt.Errorf("plan check: %w", err) + } + return p, nil +} + +// Run checks every prod entry once. The error is for a run that could not finish (no connection, or a session +// that cannot go on); the report then holds the runs made so far. +func (p *ProdChecker) Run(ctx context.Context) (*Report, error) { + conn, err := p.pool.Acquire(ctx) + if err != nil { + return nil, fmt.Errorf("plan check: acquire: %w", err) + } + pc := conn.Conn().PgConn() + rep, err := Run(ctx, p.corpus, NewSession(pc), p.opts) + if err != nil || pc.IsClosed() || pc.TxStatus() != 'I' { + // The session may hold a transaction or a prepared statement: never hand it back to the pool. + cctx, cancel := cleanupCtx(ctx) + _ = conn.Hijack().Close(cctx) + cancel() + } else { + conn.Release() + } + if err != nil { + return rep, fmt.Errorf("plan check: %w", err) + } + return rep, nil +} + +// API is the report as GET /api/v1/admin/db/planguard serves it. +func (r *Report) API() api.PlanguardReport { + out := api.PlanguardReport{ + Tier: r.Tier, + StartedAt: r.StartedAt, + DurationMs: r.DurationMs, + Clean: true, + Counts: map[string]int{}, + Failures: []api.PlanguardRun{}, + Runs: make([]api.PlanguardRun, 0, len(r.Results)), + } + for _, x := range r.Results { + run := api.PlanguardRun{Entry: x.ID, Stmt: x.Stmt, Mode: x.Mode, Verdict: string(x.Verdict), + Drivers: x.Drivers, Skip: x.Skip} + for _, f := range x.Failures { + run.Failures = append(run.Failures, f.Msg) + } + out.Counts[string(x.Verdict)]++ + out.Runs = append(out.Runs, run) + if x.Verdict.Failing() { + out.Clean = false + out.Failures = append(out.Failures, run) + } + } + return out +} diff --git a/internal/planguard/prod_integration_test.go b/internal/planguard/prod_integration_test.go new file mode 100644 index 0000000..40979c1 --- /dev/null +++ b/internal/planguard/prod_integration_test.go @@ -0,0 +1,388 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package planguard_test + +import ( + "context" + "io/fs" + "os" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/db" + pgdata "github.com/MeshCore-Beacon/beacon-server/db/planguard" + "github.com/MeshCore-Beacon/beacon-server/internal/planguard" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +// prodEnv is a migrated, empty schema and a one-connection pool on it shaped like the maintenance pool. +type prodEnv struct { + ctx context.Context + dsn string + schema string + pool *pgxpool.Pool +} + +func newProdEnv(t *testing.T) *prodEnv { + t.Helper() + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for PostgreSQL tests") + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + t.Cleanup(cancel) + admin, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + schema := "planguard_" + strings.ReplaceAll(uuid.NewString(), "-", "") + if _, err := admin.Exec(ctx, "CREATE SCHEMA "+schema); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if _, err := admin.Exec(context.Background(), "DROP SCHEMA "+schema+" CASCADE"); err != nil { + t.Error(err) + } + admin.Close(context.Background()) + }) + cfg, err := pgxpool.ParseConfig(dsn) + if err != nil { + t.Fatal(err) + } + cfg.ConnConfig.RuntimeParams["search_path"] = schema + cfg.ConnConfig.RuntimeParams["plan_cache_mode"] = "force_custom_plan" + cfg.MaxConns = 1 + pool, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + t.Fatal(err) + } + t.Cleanup(pool.Close) + if err := db.RunMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + return &prodEnv{ctx: ctx, dsn: dsn, schema: schema, pool: pool} +} + +// checker loads the embedded corpus, or the corpus.yaml in dir. +func (e *prodEnv) checker(t *testing.T, dir string) *planguard.ProdChecker { + t.Helper() + var corpus fs.FS = pgdata.Files + if dir != "" { + corpus = os.DirFS(dir) + } + p, err := planguard.NewProdChecker(e.pool, db.SQLCSources, corpus) + if err != nil { + t.Fatal(err) + } + return p +} + +// pooled reads one value on the pool's own (single) connection. +func (e *prodEnv) pooled(t *testing.T, sql string) string { + t.Helper() + var v string + if err := e.pool.QueryRow(e.ctx, sql).Scan(&v); err != nil { + t.Fatal(err) + } + return v +} + +func TestPlanguardProdTierPostgres(t *testing.T) { + e := newProdEnv(t) + cacheMode, path := e.pooled(t, "SHOW plan_cache_mode"), e.pooled(t, "SHOW search_path") + pid := e.pooled(t, "SELECT pg_backend_pid()::text") + + // 1) The empty migrated schema. The embedded corpus plans every entry in custom and generic only, never + // analyzed; its plan shapes are measured on production-sized tables, so verdicts here are not judged, but no + // run may fail to plan. + rep, err := e.checker(t, "").Run(e.ctx) + if err != nil { + t.Fatal(err) + } + if rep.Tier != "prod" || len(rep.Results) == 0 { + t.Fatalf("tier %s, %d runs", rep.Tier, len(rep.Results)) + } + if !contains(rep.Indexes, "idx_known_routes_iata_keyset") { + t.Errorf("indexes = %v: the run did not plan in the pool's search_path", rep.Indexes) + } + for _, r := range rep.Results { + if r.Mode != planguard.ModeCustom && r.Mode != planguard.ModeGeneric { + t.Errorf("%s|%s: the prod tier plans custom and generic only", r.ID, r.Mode) + } + if r.Verdict == planguard.VerdictError { + t.Errorf("empty schema: %s|%s %v", r.ID, r.Mode, r.Failures) + } + if r.ExecMs != 0 || r.Buffers != 0 { + t.Errorf("%s|%s: analyzed on the prod tier (exec %.2f ms, buffers %d)", r.ID, r.Mode, r.ExecMs, r.Buffers) + } + } + if api := rep.API(); api.Failures == nil || len(api.Runs) != len(rep.Results) { + t.Errorf("API: failures %v runs %d", api.Failures, len(api.Runs)) + } + + // The test corpus on the empty schema: a literal region plans as an empty region, a NULL resolver skips. + p := e.checker(t, "testdata/prodcorpus") + if rep, err = p.Run(e.ctx); err != nil { + t.Fatal(err) + } + checkProdRuns(t, "empty schema", rep) + + // 2) ~2,000 routes over three regions: the routes entries plan ok, and the pooled session is as it was. + if _, err := e.pool.Exec(e.ctx, ` +INSERT INTO iata_codes (iata) VALUES ('SEA'), ('YVR'), ('PDX'); +INSERT INTO known_routes (path_key, node_ids, hash_prefix, iata, hop_count, last_seen, observation_count) +SELECT int8send(g), ARRAY[md5(g::text)::uuid], ARRAY['\x01'::bytea], (ARRAY['SEA','YVR','PDX'])[1 + g % 3], + 1 + g % 5, timestamptz '2026-03-01 00:00:00+00' - g * interval '1 minute', 1 + g % 4 +FROM generate_series(1, 2000) g; +ANALYZE known_routes`); err != nil { + t.Fatal(err) + } + if rep, err = p.Run(e.ctx); err != nil { + t.Fatal(err) + } + checkProdRuns(t, "2,000 routes", rep) + if api := rep.API(); !api.Clean || len(api.Failures) != 0 || api.Counts["ok"] == 0 { + t.Errorf("API: clean %v counts %v failures %v", api.Clean, api.Counts, api.Failures) + } + if got := e.pooled(t, "SELECT pg_backend_pid()::text"); got != pid { + t.Errorf("pooled connection replaced after a clean run (pid %s → %s)", pid, got) + } + if n := e.pooled(t, "SELECT count(*)::text FROM pg_prepared_statements WHERE name LIKE 'planguard%'"); n != "0" { + t.Errorf("%s planguard prepared statements left on the pooled connection", n) + } + if m := e.pooled(t, "SHOW plan_cache_mode"); m != cacheMode { + t.Errorf("plan_cache_mode %s → %s", cacheMode, m) + } + if sp := e.pooled(t, "SHOW search_path"); sp != path { + t.Errorf("search_path %s → %s", path, sp) + } + if ro := e.pooled(t, "SHOW transaction_read_only"); ro != "off" { + t.Errorf("transaction_read_only = %s on the pooled connection", ro) + } + + // 3) A transaction holding ACCESS EXCLUSIVE on known_routes: the runs on it ERROR after the 1 s lock_timeout, + // the run finishes, and the holder never waits behind the check. + holder, err := pgx.Connect(e.ctx, e.dsn) + if err != nil { + t.Fatal(err) + } + defer holder.Close(context.Background()) + tx, err := holder.Begin(e.ctx) + if err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(e.ctx, "LOCK TABLE "+e.schema+".known_routes IN ACCESS EXCLUSIVE MODE"); err != nil { + t.Fatal(err) + } + done := make(chan struct{}) + var locked *planguard.Report + var runErr error + t0 := time.Now() + go func() { + defer close(done) + locked, runErr = p.Run(e.ctx) + }() + var holderMax time.Duration + for running := true; running; { + select { + case <-done: + running = false + case <-time.After(100 * time.Millisecond): + t1 := time.Now() + if _, err := tx.Exec(e.ctx, "SELECT count(*) FROM "+e.schema+".known_routes"); err != nil { + t.Fatal(err) + } + holderMax = max(holderMax, time.Since(t1)) + } + } + elapsed := time.Since(t0) + t1 := time.Now() + if err := tx.Commit(e.ctx); err != nil { + t.Fatal(err) + } + holderMax = max(holderMax, time.Since(t1)) + if runErr != nil { + t.Fatalf("run under the lock: %v", runErr) + } + if holderMax > time.Second { + t.Errorf("the lock holder waited %v", holderMax) + } + got := map[string]planguard.Result{} + ran := 0 + for _, r := range locked.Results { + got[r.ID+"|"+r.Mode] = r + if r.Verdict == planguard.VerdictSkip { + continue + } + ran++ + if r.Verdict != planguard.VerdictError || len(r.Failures) == 0 || !strings.Contains(r.Failures[0].Msg, "55P03") { + t.Errorf("%s|%s under the lock: %s %v, want ERROR 55P03 (lock_timeout)", r.ID, r.Mode, r.Verdict, r.Failures) + } + // A run whose resolver hit the lock is never made (wall 0); every other run waits out lock_timeout once. + if r.WallMs != 0 && (r.WallMs < 900 || r.WallMs > 2500) { + t.Errorf("%s|%s waited %.0f ms on the lock, want about 1 s", r.ID, r.Mode, r.WallMs) + } + } + if _, ok := got["routes.delete_expired|custom"]; !ok || ran == 0 { + t.Fatalf("no DML run under the lock: %v", got) + } + // The resolver transaction waits on the lock too (1 s), then each run once. + if limit := time.Duration(ran+3) * 1500 * time.Millisecond; elapsed > limit { + t.Errorf("run under the lock took %v for %d runs, want under %v", elapsed, ran, limit) + } + if n := e.pooled(t, "SELECT count(*)::text FROM pg_prepared_statements WHERE name LIKE 'planguard%'"); n != "0" { + t.Errorf("%s planguard prepared statements left after the locked run", n) + } + + // 4) A holder with SHARE on known_routes blocks only the DML run's ROW EXCLUSIVE. A third session asking for + // SHARE is compatible with the holder but queues behind that waiting request, so it waits at most the + // check's 1 s lock_timeout. + third, err := pgx.Connect(e.ctx, e.dsn) + if err != nil { + t.Fatal(err) + } + defer third.Close(context.Background()) + if tx, err = holder.Begin(e.ctx); err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(e.ctx, "LOCK TABLE "+e.schema+".known_routes IN SHARE MODE"); err != nil { + t.Fatal(err) + } + done = make(chan struct{}) + var shared *planguard.Report + go func() { + defer close(done) + shared, runErr = p.Run(e.ctx) + }() + var thirdMax time.Duration + for running := true; running; { + select { + case <-done: + running = false + case <-time.After(20 * time.Millisecond): + t1 := time.Now() + if _, err := third.Exec(e.ctx, "BEGIN; LOCK TABLE "+e.schema+".known_routes IN SHARE MODE; ROLLBACK"); err != nil { + t.Fatal(err) + } + thirdMax = max(thirdMax, time.Since(t1)) + } + } + if err := tx.Rollback(e.ctx); err != nil { + t.Fatal(err) + } + if runErr != nil { + t.Fatalf("run beside the SHARE holder: %v", runErr) + } + if r := byKey(shared)["routes.delete_expired|custom"]; r.Verdict != planguard.VerdictError || len(r.Failures) == 0 || !strings.Contains(r.Failures[0].Msg, "55P03") { + t.Errorf("DML beside the SHARE holder: %s %v, want ERROR 55P03", r.Verdict, r.Failures) + } + // It queued (well above a bare LOCK) and was released by the check's lock_timeout. + if thirdMax < 300*time.Millisecond || thirdMax > 1500*time.Millisecond { + t.Errorf("the session queued behind the check waited at most %v, want about 1 s or less (and > 300 ms, to show it queued)", thirdMax) + } +} + +// Cancelling a run mid-flight never hands the connection back to the pool with an open transaction or a prepared +// statement: it is closed. +func TestPlanguardProdCancelPostgres(t *testing.T) { + e := newProdEnv(t) + p := e.checker(t, "testdata/prodcorpus") + pid := e.pooled(t, "SELECT pg_backend_pid()::text") + holder, err := pgx.Connect(e.ctx, e.dsn) + if err != nil { + t.Fatal(err) + } + defer holder.Close(context.Background()) + tx, err := holder.Begin(e.ctx) + if err != nil { + t.Fatal(err) + } + defer func() { _ = tx.Rollback(context.Background()) }() + // SHARE lets the readers through and holds the DML run on its lock, inside its transaction, after PREPARE + // began: that is where the cancel lands. + if _, err := tx.Exec(e.ctx, "LOCK TABLE "+e.schema+".known_routes IN SHARE MODE"); err != nil { + t.Fatal(err) + } + // The observer reads pg_stat_activity outside any transaction, so each read is a fresh snapshot. + observer, err := pgx.Connect(e.ctx, e.dsn) + if err != nil { + t.Fatal(err) + } + defer observer.Close(context.Background()) + ctx, cancel := context.WithCancel(e.ctx) + probeDone := make(chan struct{}) + go func() { + defer close(probeDone) + // Wait until the check's connection is waiting on the lock, then cancel. + for ctx.Err() == nil { + var n int + if err := observer.QueryRow(e.ctx, "SELECT count(*) FROM pg_stat_activity WHERE pid = $1::int AND wait_event_type = 'Lock'", pid).Scan(&n); err == nil && n > 0 { + cancel() + return + } + time.Sleep(10 * time.Millisecond) + } + }() + rep, err := p.Run(ctx) + cancel() + <-probeDone + if err == nil { + t.Fatalf("run survived its cancel: %d runs", len(rep.Results)) + } + if got := e.pooled(t, "SELECT pg_backend_pid()::text"); got == pid { + t.Errorf("the cancelled run's connection (pid %s) went back to the pool", pid) + } + if n := e.pooled(t, "SELECT count(*)::text FROM pg_prepared_statements WHERE name LIKE 'planguard%'"); n != "0" { + t.Errorf("%s planguard prepared statements on the pooled connection", n) + } + if ro := e.pooled(t, "SHOW transaction_read_only"); ro != "off" { + t.Errorf("pooled connection inside a read-only transaction") + } + // The closed backend leaves pg_stat_activity once it has processed the terminate. + alive := 1 + for deadline := time.Now().Add(5 * time.Second); alive != 0 && time.Now().Before(deadline); time.Sleep(20 * time.Millisecond) { + if err := observer.QueryRow(e.ctx, "SELECT count(*) FROM pg_stat_activity WHERE pid = $1::int", pid).Scan(&alive); err != nil { + t.Fatal(err) + } + } + if alive != 0 { + t.Errorf("the cancelled run's backend %s is still connected", pid) + } +} + +// checkProdRuns checks a run of testdata/prodcorpus: nothing fails, the null resolver and the pending statement +// skip, the ingest statement (auto only) is not planned, and no run is auto. +func checkProdRuns(t *testing.T, label string, rep *planguard.Report) { + t.Helper() + got := map[string]planguard.Result{} + for _, r := range rep.Results { + got[r.ID+"|"+r.Mode] = r + if r.Verdict.Failing() { + t.Errorf("%s: %s|%s %s %v", label, r.ID, r.Mode, r.Verdict, r.Failures) + } + if r.Mode == planguard.ModeAuto || r.ID == "channels.upsert" { + t.Errorf("%s: %s|%s was planned", label, r.ID, r.Mode) + } + } + for _, k := range []string{"routes.byiata|custom", "routes.byiata|generic", "routes.byiata.empty_region|custom", + "routes.globalhop|custom", "routes.delete_expired|custom"} { + if r, ok := got[k]; !ok || r.Verdict == planguard.VerdictSkip { + t.Errorf("%s: %s did not run: %+v", label, k, r) + } + } + for k, want := range map[string]string{ + "routes.byiata.no_region|custom": "resolver no_region returned no value on this tier", + "routes.global.pending|custom": "pending: later", + } { + if r := got[k]; r.Verdict != planguard.VerdictSkip || r.Skip != want { + t.Errorf("%s: %s %s skip %q, want SKIP %q", label, k, r.Verdict, r.Skip, want) + } + } + if len(rep.Results) != 7 { + t.Errorf("%s: %d runs, want 7", label, len(rep.Results)) + } +} diff --git a/internal/planguard/prod_internal_test.go b/internal/planguard/prod_internal_test.go new file mode 100644 index 0000000..b645dc5 --- /dev/null +++ b/internal/planguard/prod_internal_test.go @@ -0,0 +1,85 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package planguard + +import ( + "context" + "strings" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/db" + pgdata "github.com/MeshCore-Beacon/beacon-server/db/planguard" + "github.com/jackc/pgx/v5/pgconn" +) + +type sentSession struct{ sent []string } + +func (s *sentSession) Exec(_ context.Context, sql string) ([]*pgconn.Result, error) { + s.sent = append(s.sent, sql) + return []*pgconn.Result{{}}, nil +} + +// The prod checker plans custom and generic only, caps each EXPLAIN at 5 s, never ANALYZEs and sets nothing +// beyond its own transactions. +func TestProdCheckerOptions(t *testing.T) { + p, err := NewProdChecker(nil, db.SQLCSources, pgdata.Files) + if err != nil { + t.Fatal(err) + } + s := &sentSession{} + rep, err := Run(context.Background(), p.corpus, s, p.opts) + if err != nil { + t.Fatal(err) + } + if len(rep.Results) == 0 { + t.Fatal("no runs") + } + modes := map[string]int{} + for _, r := range rep.Results { + modes[r.Mode]++ + } + if modes[ModeAuto] != 0 || modes[ModeCustom] == 0 || modes[ModeGeneric] == 0 { + t.Errorf("modes = %v, want custom and generic only", modes) + } + prepares, resolvers := 0, 0 + for i, sql := range s.sent { + if strings.Contains(sql, "ANALYZE") { + t.Errorf("sent %q", firstLine(sql)) + } + if strings.Contains(sql, "search_path") { + t.Errorf("sent %q: the prod tier inherits the pool's search_path", firstLine(sql)) + } + if strings.HasPrefix(sql, "SET ") && !strings.HasPrefix(sql, "SET LOCAL ") { + t.Errorf("session-level SET %q", sql) + } + if strings.HasPrefix(sql, "PREPARE ") { + prepares++ + if !timeoutBefore(s.sent[:i], "SET LOCAL statement_timeout = '5s'") { + t.Errorf("PREPARE %d not under a 5 s statement_timeout", prepares) + } + } + if strings.HasPrefix(sql, "SAVEPOINT planguard_r") && resolvers == 0 { + resolvers++ + if !timeoutBefore(s.sent[:i], "SET LOCAL statement_timeout = '20s'") { + t.Error("resolvers not under a 20 s statement_timeout") + } + } + } + if prepares == 0 || resolvers == 0 { + t.Errorf("prepares %d resolvers %d, want both", prepares, resolvers) + } +} + +// timeoutBefore reports whether the transaction that sent's last BEGIN opened set want. +func timeoutBefore(sent []string, want string) bool { + for i := len(sent) - 1; i >= 0; i-- { + if strings.Contains(sent[i], want) { + return true + } + if strings.HasPrefix(sent[i], "BEGIN") { + return false + } + } + return false +} diff --git a/internal/planguard/report.go b/internal/planguard/report.go new file mode 100644 index 0000000..9faf3ff --- /dev/null +++ b/internal/planguard/report.go @@ -0,0 +1,269 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package planguard + +import ( + "bufio" + "bytes" + "encoding/json" + "fmt" + "os" + "path/filepath" + "slices" + "sort" + "strings" + "time" +) + +// Result is one run: one entry in one plan mode on one tier. +type Result struct { + ID string `json:"id"` + Stmt string `json:"stmt"` + Shape string `json:"shape"` + Mode string `json:"mode"` + Tier string `json:"tier"` + Verdict Verdict `json:"verdict"` + Failures []Failure `json:"failures,omitempty"` + Drivers []string `json:"drivers,omitempty"` + Retired []string `json:"retired,omitempty"` + Buffers int64 `json:"buffers"` + Rows int64 `json:"rows"` + PlanningMs float64 `json:"planning_ms"` + ExecMs float64 `json:"exec_ms"` + WallMs float64 `json:"wall_ms"` + Degraded string `json:"degraded,omitempty"` + Skip string `json:"skip,omitempty"` + Params []string `json:"params,omitempty"` + Signature []string `json:"signature,omitempty"` + PlanChange []string `json:"plan_change,omitempty"` + Plan json.RawMessage `json:"-"` +} + +// Report is one Run. +type Report struct { + Tier string `json:"tier"` + StartedAt time.Time `json:"started_at"` + DurationMs float64 `json:"duration_ms"` + Settings map[string]string `json:"settings"` // planner GUCs as SHOWn at start + Indexes []string `json:"indexes"` // indexes on big_tables in the run's search_path + Results []Result `json:"results"` +} + +// Counts returns runs per verdict. +func (r *Report) Counts() map[Verdict]int { + out := map[Verdict]int{} + for _, x := range r.Results { + out[x.Verdict]++ + } + return out +} + +// Failures returns the failing runs (FALSE_POSITIVE, MISSED, ERROR), in run order. +func (r *Report) Failures() []Result { + var out []Result + for _, x := range r.Results { + if x.Verdict.Failing() { + out = append(out, x) + } + } + return out +} + +// WriteFiles writes results..jsonl, summary..md, report..json (the report without its results) +// and plans//..json into dir. +func (r *Report) WriteFiles(dir string) error { + plans := filepath.Join(dir, "plans", r.Tier) + if err := os.MkdirAll(plans, 0o755); err != nil { + return err + } + var jl bytes.Buffer + enc := json.NewEncoder(&jl) + for _, x := range r.Results { + if err := enc.Encode(x); err != nil { + return err + } + if len(x.Plan) > 0 { + if err := os.WriteFile(filepath.Join(plans, x.ID+"."+x.Mode+".json"), x.Plan, 0o644); err != nil { + return err + } + } + } + if err := os.WriteFile(filepath.Join(dir, "results."+r.Tier+".jsonl"), jl.Bytes(), 0o644); err != nil { + return err + } + meta := *r + meta.Results = nil + b, err := json.MarshalIndent(meta, "", " ") + if err != nil { + return err + } + if err := os.WriteFile(filepath.Join(dir, "report."+r.Tier+".json"), append(b, '\n'), 0o644); err != nil { + return err + } + return os.WriteFile(filepath.Join(dir, "summary."+r.Tier+".md"), []byte(r.summary()), 0o644) +} + +// CountsString renders Counts sorted by verdict, like fmt prints a map. +func (r *Report) CountsString() string { + return fmt.Sprint(r.Counts()) +} + +func (r *Report) summary() string { + var b strings.Builder + fmt.Fprintf(&b, "# planguard %s: %d runs in %.1f s; verdicts %s\n\n", r.Tier, len(r.Results), r.DurationMs/1000, r.CountsString()) + b.WriteString("| entry | mode | verdict | buffers | plan ms | exec ms | wall ms | drivers | failures |\n") + b.WriteString("|---|---|---|---:|---:|---:|---:|---|---|\n") + esc := func(s string) string { + return strings.ReplaceAll(strings.ReplaceAll(s, "|", `\|`), "\n", " ") + } + for _, x := range r.Results { + var fl []string + for _, f := range x.Failures { + fl = append(fl, esc(f.Msg)) + } + note := strings.Join(fl, "
") + if x.Skip != "" { + note = "skip: " + esc(x.Skip) + } + if len(x.Retired) > 0 { + note = "(plan uses retired " + strings.Join(x.Retired, ",") + ") " + note + } + if x.Degraded != "" { + note = "(degrade_to: " + esc(x.Degraded) + ") " + note + } + var drv []string + for _, d := range x.Drivers { + drv = append(drv, esc(d)) + } + fmt.Fprintf(&b, "| %s | %s | %s | %d | %.2f | %.1f | %.0f | %s | %s |\n", esc(x.ID), x.Mode, x.Verdict, x.Buffers, + x.PlanningMs, x.ExecMs, x.WallMs, strings.Join(drv, "
"), note) + } + return b.String() +} + +// Line is the one-line form of a run that the CLI prints. +func (x Result) Line() string { + var fl []string + for _, f := range x.Failures { + fl = append(fl, f.Msg) + } + if len(fl) > 2 { + fl = append(fl[:2:2], fmt.Sprintf("(+%d)", len(x.Failures)-2)) + } + note := strings.Join(fl, " ; ") + if x.Skip != "" { + note = "skip: " + x.Skip + } + return fmt.Sprintf("%-58s %-8s %-14s B=%-8d %7.0fms %s", x.ID, x.Mode, x.Verdict, x.Buffers, x.WallMs, note) +} + +// ReadResults reads a results..jsonl file. A line that does not decode is an error. +func ReadResults(path string) ([]Result, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + var out []Result + sc := bufio.NewScanner(f) + sc.Buffer(make([]byte, 0, 64*1024), 64<<20) + for n := 1; sc.Scan(); n++ { + line := bytes.TrimSpace(sc.Bytes()) + if len(line) == 0 { + continue + } + var r Result + dec := json.NewDecoder(bytes.NewReader(line)) + dec.DisallowUnknownFields() + if err := dec.Decode(&r); err != nil { + return nil, fmt.Errorf("%s:%d: %w", path, n, err) + } + out = append(out, r) + } + if err := sc.Err(); err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + return out, nil +} + +// ReadReportMeta reads the report..json that WriteFiles put beside a results..jsonl. +func ReadReportMeta(resultsPath string) (*Report, error) { + base := filepath.Base(resultsPath) + if !strings.HasPrefix(base, "results.") || !strings.HasSuffix(base, ".jsonl") { + return nil, fmt.Errorf("%s: not a results..jsonl file", resultsPath) + } + tier := strings.TrimSuffix(strings.TrimPrefix(base, "results."), ".jsonl") + b, err := os.ReadFile(filepath.Join(filepath.Dir(resultsPath), "report."+tier+".json")) + if err != nil { + return nil, err + } + var r Report + if err := json.Unmarshal(b, &r); err != nil { + return nil, fmt.Errorf("report.%s.json: %w", tier, err) + } + return &r, nil +} + +// FidelityResult compares the drivers of two runs of the same corpus. +type FidelityResult struct { + Match, Total int + Excluded int + Mismatches []string +} + +// Fidelity compares the driving index sets per (entry, mode) of two runs. Runs where either plan uses an index +// absent from the other report's Indexes are excluded and counted. A nil index list is unknown and excludes +// nothing. Only runs present in both, neither skipped nor errored and with a driver on at least one side, count. +func Fidelity(a, b []Result, aIndexes, bIndexes []string) FidelityResult { + var out FidelityResult + bBy := map[string]Result{} + for _, r := range b { + bBy[r.ID+"|"+r.Mode] = r + } + absent := func(ixs, have []string) bool { + return have != nil && slices.ContainsFunc(ixs, func(ix string) bool { return !slices.Contains(have, ix) }) + } + for _, ra := range a { + key := ra.ID + "|" + ra.Mode + rb, ok := bBy[key] + if !ok || !compared(ra) || !compared(rb) { + continue + } + ia, ib := driverIndexes(ra), driverIndexes(rb) + if len(ia) == 0 && len(ib) == 0 { + continue + } + if absent(ia, bIndexes) || absent(ib, aIndexes) { + out.Excluded++ + continue + } + out.Total++ + if slices.Equal(ia, ib) { + out.Match++ + } else { + out.Mismatches = append(out.Mismatches, fmt.Sprintf("%s: %v vs %v", key, ia, ib)) + } + } + return out +} + +func compared(r Result) bool { return r.Verdict != VerdictSkip && r.Verdict != VerdictError } + +// driverIndexes returns the sorted, distinct indexes of a run's drivers ("relation:node:ix+ix"). +func driverIndexes(r Result) []string { + var out []string + for _, d := range r.Drivers { + i := strings.LastIndexByte(d, ':') + if i < 0 { + continue + } + for _, ix := range strings.Split(d[i+1:], "+") { + if ix != "" && !slices.Contains(out, ix) { + out = append(out, ix) + } + } + } + sort.Strings(out) + return out +} diff --git a/internal/planguard/report_test.go b/internal/planguard/report_test.go new file mode 100644 index 0000000..b50e583 --- /dev/null +++ b/internal/planguard/report_test.go @@ -0,0 +1,110 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package planguard_test + +import ( + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/planguard" +) + +func drv(id, ix string) planguard.Result { + return planguard.Result{ID: id, Mode: "custom", Verdict: planguard.VerdictOK, Drivers: []string{"known_routes:Index Scan:" + ix}} +} + +func TestFidelityExcludesIndexSetMismatch(t *testing.T) { + a := []planguard.Result{drv("same", "idx_a"), drv("same2", "idx_b"), drv("differs", "idx_a"), drv("only_a_index", "idx_only_a"), drv("not_in_b", "idx_a")} + b := []planguard.Result{drv("same", "idx_a"), drv("same2", "idx_b"), drv("differs", "idx_b"), drv("only_a_index", "idx_a")} + aIx := []string{"idx_a", "idx_b", "idx_only_a"} + bIx := []string{"idx_a", "idx_b"} + got := planguard.Fidelity(a, b, aIx, bIx) + if got.Total != 3 || got.Match != 2 || got.Excluded != 1 { + t.Fatalf("fidelity = %+v, want 2 of 3 with 1 excluded", got) + } + if len(got.Mismatches) != 1 || !strings.Contains(got.Mismatches[0], "differs|custom") { + t.Errorf("mismatches = %v, want differs|custom", got.Mismatches) + } + + // The same run seen from b's side: b's plan uses an index a lacks. + b2 := []planguard.Result{drv("same", "idx_b_only")} + if got := planguard.Fidelity([]planguard.Result{drv("same", "idx_a")}, b2, []string{"idx_a"}, []string{"idx_a", "idx_b_only"}); got.Excluded != 1 || got.Total != 0 { + t.Errorf("fidelity = %+v, want the run excluded", got) + } +} + +func TestReportCountsAndFailures(t *testing.T) { + var rep planguard.Report + for i, v := range []planguard.Verdict{planguard.VerdictOK, planguard.VerdictFalsePositive, planguard.VerdictMissed, planguard.VerdictError, + planguard.VerdictRetiredIndex, planguard.VerdictSkip, planguard.VerdictDetected, planguard.VerdictKnownIssue, planguard.VerdictOK} { + rep.Results = append(rep.Results, planguard.Result{ID: string(rune('a' + i)), Mode: "custom", Verdict: v}) + } + want := map[planguard.Verdict]int{"ok": 2, "FALSE_POSITIVE": 1, "MISSED": 1, "ERROR": 1, "RETIRED_INDEX": 1, "SKIP": 1, "DETECTED": 1, "KNOWN_ISSUE": 1} + if got := rep.Counts(); !reflect.DeepEqual(got, want) { + t.Errorf("counts = %v, want %v", got, want) + } + var failing []planguard.Verdict + for _, r := range rep.Failures() { + failing = append(failing, r.Verdict) + } + if !reflect.DeepEqual(failing, []planguard.Verdict{planguard.VerdictFalsePositive, planguard.VerdictMissed, planguard.VerdictError}) { + t.Errorf("failures = %v, want FALSE_POSITIVE, MISSED, ERROR", failing) + } +} + +func TestReportFilesRoundTrip(t *testing.T) { + dir := t.TempDir() + rep := &planguard.Report{Tier: "local", Indexes: []string{"idx_a"}, Results: []planguard.Result{ + {ID: "x", Stmt: "S", Shape: "dense", Mode: "generic", Tier: "local", Verdict: planguard.VerdictFalsePositive, + Failures: []planguard.Failure{{Kind: planguard.FailDriver, Msg: "driver: a | b"}}, Drivers: []string{"t:Index Scan:idx_a"}, + Signature: []string{"Limit"}, Plan: []byte(`[{"Plan": {"Node Type": "Limit"}}]`)}, + {ID: "y", Mode: "custom", Tier: "local", Verdict: planguard.VerdictSkip, Skip: "pending: f"}, + }} + if err := rep.WriteFiles(dir); err != nil { + t.Fatal(err) + } + got, err := planguard.ReadResults(filepath.Join(dir, "results.local.jsonl")) + if err != nil { + t.Fatal(err) + } + want := append([]planguard.Result(nil), rep.Results...) + want[0].Plan = nil + if !reflect.DeepEqual(got, want) { + t.Errorf("read back %+v, want %+v", got, want) + } + raw, err := os.ReadFile(filepath.Join(dir, "results.local.jsonl")) + if err != nil { + t.Fatal(err) + } + for _, key := range []string{`"id":"x"`, `"stmt":"S"`, `"shape":"dense"`, `"mode":"generic"`, `"tier":"local"`} { + if !strings.Contains(string(raw), key) { + t.Errorf("results.jsonl lacks %s", key) + } + } + if b, err := os.ReadFile(filepath.Join(dir, "plans", "local", "x.generic.json")); err != nil || !strings.Contains(string(b), "Limit") { + t.Errorf("plan file: %v %q", err, b) + } + if b, err := os.ReadFile(filepath.Join(dir, "summary.local.md")); err != nil || !strings.Contains(string(b), `driver: a \| b`) { + t.Errorf("summary: %v %q", err, b) + } + meta, err := planguard.ReadReportMeta(filepath.Join(dir, "results.local.jsonl")) + if err != nil || !reflect.DeepEqual(meta.Indexes, []string{"idx_a"}) { + t.Errorf("report meta: %v %+v", err, meta) + } + if _, err := planguard.ReadResults(writeFile(t, "{\"id\":\"x\"}\n{\"id\":")); err == nil { + t.Error("a truncated results file read without error") + } +} + +func writeFile(t *testing.T, body string) string { + t.Helper() + p := filepath.Join(t.TempDir(), "f") + if err := os.WriteFile(p, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + return p +} diff --git a/internal/planguard/runner.go b/internal/planguard/runner.go new file mode 100644 index 0000000..f90c7b3 --- /dev/null +++ b/internal/planguard/runner.go @@ -0,0 +1,822 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package planguard + +import ( + "context" + "crypto/sha1" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "regexp" + "slices" + "strings" + "time" + + "github.com/jackc/pgx/v5/pgconn" +) + +// BaselineMode says what a run does with the committed plan signatures. +type BaselineMode string + +// Baseline modes. +const ( + BaselineCheck BaselineMode = "check" // compare each signature with its baseline file + BaselineWrite BaselineMode = "write" // write each signature to its baseline file + BaselineOff BaselineMode = "off" +) + +// RunOptions selects and configures the runs of one Run. +type RunOptions struct { + Tier string // fixture | local | prod + Only, Skip *regexp.Regexp + Modes []string // override every entry's modes + KeepModes []string // keep only these of each entry's own modes (after Modes) + RunTimeoutS int // caps each run's statement_timeout below the tier's cap; 0 = no extra cap + IncludePending bool // run pending statements too (local overlays only) + SearchPath string // override every group's search_path + Baseline BaselineMode + BaselineDir string // read: db/planguard/baseline/; write: the same path on disk + Params map[string]*string // pre-resolved (prod via emit); nil → resolve on the session + Progress func(Result) // optional; called after each run +} + +const ( + stmtName = "planguard_q" + autoExecutions = 6 // plan_cache_mode auto picks a generic plan after five custom ones + prodTimeoutS = 20 // statement_timeout cap for resolvers and plans on the prod tier + localResolverS = 120 + defaultTimeout = 60 +) + +// plannerSettings are the GUCs a report records, as SHOWn when the run starts. +var plannerSettings = []string{ + "server_version", "random_page_cost", "seq_page_cost", "effective_cache_size", "effective_io_concurrency", + "work_mem", "default_statistics_target", "cpu_tuple_cost", "cpu_index_tuple_cost", "cpu_operator_cost", + "jit", "plan_cache_mode", "max_parallel_workers_per_gather", "from_collapse_limit", "join_collapse_limit", + "geqo_threshold", "enable_seqscan", "enable_indexscan", "enable_indexonlyscan", "enable_bitmapscan", + "enable_sort", "enable_incremental_sort", "enable_hashjoin", "enable_mergejoin", "enable_nestloop", + "enable_hashagg", "enable_material", "enable_memoize", +} + +var ( + plainIdentRe = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) + quotedIdentRe = regexp.MustCompile(`^"[^"$\\;\x00-\x1f\x7f]+"$`) + gucNameRe = regexp.MustCompile(`^[a-z_][a-z0-9_.]*$`) +) + +// Run plans every selected (entry, mode) on s, one READ ONLY transaction per run, and checks each plan. It never +// writes: each run ends with ROLLBACK and DEALLOCATE as separate commands, on every path. A run that fails gives +// an ERROR result and the next run goes on; an error is returned only when the session itself cannot go on (or +// ctx ends), together with the results so far. +func Run(ctx context.Context, c *Corpus, s Session, o RunOptions) (*Report, error) { + if err := o.validate(c); err != nil { + return nil, err + } + t0 := time.Now() + rep := &Report{Tier: o.Tier, StartedAt: t0.UTC(), Settings: map[string]string{}} + finish := func(err error) (*Report, error) { + rep.DurationMs = msSince(t0) + return rep, err + } + if err := readMeta(ctx, s, c, o, rep); err != nil { + return finish(err) + } + entries := c.selection(o) + res := o.Params + resErrs := map[string]string{} + if res == nil { + var err error + if res, resErrs, err = resolve(ctx, s, c, o, entries); err != nil { + return finish(err) + } + } + for _, e := range entries { + for _, mode := range o.modesOf(c, e) { + sp := c.spec(o, e, mode, res, resErrs, false) + c.skipMissingIndex(&sp, rep.Indexes) + r, err := c.runOne(ctx, s, o, sp) + rep.Results = append(rep.Results, r) + if o.Progress != nil { + o.Progress(r) + } + if err != nil { + return finish(err) + } + } + } + return finish(nil) +} + +func (o *RunOptions) validate(c *Corpus) error { + if !validTiers[o.Tier] { + return fmt.Errorf("unknown tier %q (want fixture, local or prod)", o.Tier) + } + for _, m := range append(slices.Clone(o.Modes), o.KeepModes...) { + if !validModes[m] { + return fmt.Errorf("unknown mode %q (want custom, generic or auto)", m) + } + } + if o.Tier == "prod" || o.Baseline == "" { + o.Baseline = BaselineOff + } + switch o.Baseline { + case BaselineOff: + case BaselineCheck, BaselineWrite: + if o.BaselineDir == "" { + return fmt.Errorf("baseline %s needs a baseline directory", o.Baseline) + } + default: + return fmt.Errorf("unknown baseline mode %q (want check, write or off)", o.Baseline) + } + if err := checkSearchPath(o.SearchPath); err != nil { + return err + } + for _, e := range c.Entries { + if err := CheckName(e.ID); err != nil { + return fmt.Errorf("entry id: %w", err) + } + } + for name := range c.Resolvers { + if err := CheckName(name); err != nil { + return fmt.Errorf("resolver name: %w", err) + } + } + for name, ts := range c.Tiers { + if err := checkSearchPath(ts.SearchPath); err != nil { + return fmt.Errorf("tier %s: %w", name, err) + } + if err := checkSettings(ts.Settings); err != nil { + return fmt.Errorf("tier %s: %w", name, err) + } + } + for name, g := range c.Groups { + for tier, sp := range g.SearchPath { + if err := checkSearchPath(sp); err != nil { + return fmt.Errorf("group %s: %s: %w", name, tier, err) + } + } + } + return nil +} + +// checkSearchPath accepts "" (no SET) or a comma list of plain identifiers and double-quoted identifiers without +// '$'. The list is sent unquoted in SET search_path, so nothing else may appear in it. +func checkSearchPath(sp string) error { + if sp == "" { + return nil + } + for _, el := range strings.Split(sp, ",") { + el = strings.TrimSpace(el) + if !plainIdentRe.MatchString(el) && !quotedIdentRe.MatchString(el) { + return fmt.Errorf("search_path %q: element %q is not a plain or double-quoted identifier (no $)", sp, el) + } + } + return nil +} + +// selection returns the entries that run on o.Tier and pass Only and Skip, in corpus order. An entry with no +// tiers runs on every tier. +func (c *Corpus) selection(o RunOptions) []*Entry { + var out []*Entry + for _, e := range c.Entries { + if len(e.Tiers) > 0 && !slices.Contains(e.Tiers, o.Tier) { + continue + } + if o.Skip != nil && o.Skip.MatchString(e.ID) { + continue + } + if o.Only != nil && !o.Only.MatchString(e.ID) { + continue + } + out = append(out, e) + } + return out +} + +func (o RunOptions) modesOf(c *Corpus, e *Entry) []string { + modes := c.ModesOf(e) + if len(o.Modes) > 0 { + modes = o.Modes + } + if len(o.KeepModes) == 0 { + return modes + } + var out []string + for _, m := range modes { + if slices.Contains(o.KeepModes, m) { + out = append(out, m) + } + } + return out +} + +// baseSearchPath is the search_path of resolvers and of the report's index list: the override, else the tier's. +// "" inherits the session's. +func (c *Corpus) baseSearchPath(o RunOptions) string { + if o.SearchPath != "" { + return o.SearchPath + } + return c.Tiers[o.Tier].SearchPath +} + +// searchPath is a statement's search_path on o.Tier: the override, its group's, else the tier's. "" means no SET, +// so the run inherits the session's. +func (c *Corpus) searchPath(o RunOptions, st *Statement) string { + if o.SearchPath != "" { + return o.SearchPath + } + if g, ok := c.Groups[st.Group]; ok { + if sp, ok := g.SearchPath[o.Tier]; ok { + return sp + } + } + return c.Tiers[o.Tier].SearchPath +} + +// readMeta fills rep.Settings and rep.Indexes in one READ ONLY transaction. +func readMeta(ctx context.Context, s Session, c *Corpus, o RunOptions, rep *Report) error { + lines := []string{"BEGIN ISOLATION LEVEL READ COMMITTED READ ONLY", + "SET LOCAL lock_timeout = '1s'; SET LOCAL statement_timeout = '20s'; SET LOCAL standard_conforming_strings = on"} + if sp := c.baseSearchPath(o); sp != "" { + lines = append(lines, "SET LOCAL search_path = "+sp) + } + var names []string + for _, n := range plannerSettings { + names = append(names, quoteLit(n)) + } + settingsQ := "SELECT n, current_setting(n, true) FROM unnest(ARRAY[" + strings.Join(names, ", ") + "]::text[]) n" + err := func() error { + for _, l := range lines { + if _, err := s.Exec(ctx, l); err != nil { + return err + } + } + res, err := s.Exec(ctx, settingsQ) + if err != nil { + return err + } + for _, row := range lastRows(res) { + if len(row) == 2 && row[1] != nil { + rep.Settings[string(row[0])] = string(row[1]) + } + } + rep.Indexes = []string{} + if len(c.Defaults.BigTables) == 0 { + return nil + } + var tables []string + for _, t := range c.Defaults.BigTables { + tables = append(tables, quoteLit(t)) + } + res, err = s.Exec(ctx, "SELECT DISTINCT ic.relname::text FROM unnest(ARRAY["+strings.Join(tables, ", ")+ + "]::text[]) b(name) JOIN pg_index i ON i.indrelid = to_regclass(quote_ident(b.name)) "+ + "JOIN pg_class ic ON ic.oid = i.indexrelid ORDER BY 1") + if err != nil { + return err + } + for _, row := range lastRows(res) { + rep.Indexes = append(rep.Indexes, string(row[0])) + } + return nil + }() + if rbErr := cleanupRollback(ctx, s); rbErr != nil { + return fmt.Errorf("read settings: rollback: %w", rbErr) + } + if err != nil { + return fmt.Errorf("read settings and indexes: %s", errText(err)) + } + return nil +} + +// resolverRef is a param's resolver: a named one ({r: name}) or inline SQL ({sql: ...}, keyed by its hash). +func resolverRef(c *Corpus, m map[string]any) (key, sql string, ok bool) { + if r, isStr := m["r"].(string); isStr { + return r, c.Resolvers[r], true + } + if q, isStr := m["sql"].(string); isStr { + h := sha1.Sum([]byte(q)) + return "sql_" + hex.EncodeToString(h[:6]), q, true + } + return "", "", false +} + +// neededResolvers returns key → SQL for every resolver the entries' params use. An unknown name maps to "". +// Entries of pending statements are left out unless includePending: they skip, so their resolvers never run. +func neededResolvers(c *Corpus, entries []*Entry, includePending bool) map[string]string { + need := map[string]string{} + var visit func(v any) + visit = func(v any) { + switch x := v.(type) { + case map[string]any: + if k, q, ok := resolverRef(c, x); ok { + need[k] = q + } + case []any: + for _, y := range x { + visit(y) + } + } + } + for _, e := range entries { + if st := c.Statements[e.Stmt]; st != nil && st.Pending != "" && !includePending { + continue + } + for _, p := range e.Params { + visit(p) + } + } + return need +} + +func resolverSelect(sql string) string { + return "SELECT coalesce((" + trimStmt(sql) + ")::text, NULL)" +} + +// resolve runs the entries' resolvers in one READ ONLY transaction, each under a savepoint so that one failing +// resolver fails only its own entries. The error is for a session that cannot go on. +func resolve(ctx context.Context, s Session, c *Corpus, o RunOptions, entries []*Entry) (map[string]*string, map[string]string, error) { + need := neededResolvers(c, entries, o.IncludePending) + vals, errs := map[string]*string{}, map[string]string{} + if len(need) == 0 { + return vals, errs, nil + } + timeout := localResolverS + if o.Tier == "prod" { + timeout = prodTimeoutS + } + setup := []string{ + "BEGIN ISOLATION LEVEL READ COMMITTED READ ONLY", + fmt.Sprintf("SET LOCAL statement_timeout = '%ds'; SET LOCAL lock_timeout = '1s'", timeout), + "SET LOCAL jit = off; SET LOCAL timezone = 'UTC'; SET LOCAL standard_conforming_strings = on", + } + if sp := c.baseSearchPath(o); sp != "" { + setup = append(setup, "SET LOCAL search_path = "+sp) + } + err := func() error { + for _, l := range setup { + if _, err := s.Exec(ctx, l); err != nil { + return err + } + } + for _, name := range sortedKeys(need) { + if need[name] == "" { + errs[name] = "no such resolver in the corpus" + continue + } + if err := checkStatementText(need[name]); err != nil { + errs[name] = err.Error() + continue + } + if _, err := s.Exec(ctx, "SAVEPOINT planguard_r"); err != nil { + return err + } + res, err := s.Exec(ctx, resolverSelect(need[name])) + if err != nil { + if ctx.Err() != nil { + return err + } + errs[name] = errText(err) + if _, err := s.Exec(ctx, "ROLLBACK TO SAVEPOINT planguard_r"); err != nil { + return err + } + continue + } + rows := lastRows(res) + if len(rows) != 1 || len(rows[0]) != 1 { + errs[name] = fmt.Sprintf("returned %d rows", len(rows)) + if _, err := s.Exec(ctx, "RELEASE SAVEPOINT planguard_r"); err != nil { + return err + } + continue + } + if rows[0][0] != nil { + v := string(rows[0][0]) + vals[name] = &v + } else { + vals[name] = nil + } + if _, err := s.Exec(ctx, "RELEASE SAVEPOINT planguard_r"); err != nil { + return err + } + } + return nil + }() + if rbErr := cleanupRollback(ctx, s); rbErr != nil { + return nil, nil, fmt.Errorf("resolvers: rollback: %w", rbErr) + } + if err != nil { + return nil, nil, fmt.Errorf("resolvers: %s", errText(err)) + } + return vals, errs, nil +} + +// runSpec is one (entry, mode) run, rendered. +type runSpec struct { + e *Entry + st *Statement + r Result // identity and params + skip string // the run is not made + err string // the run cannot be made: ERROR + exp *Expect + analyzed bool + setup []string // BEGIN, SET LOCAL ..., PREPARE + execs int + call string // EXECUTE planguard_q(...) + explain string +} + +func modeGUC(m string) string { + switch m { + case ModeCustom: + return "force_custom_plan" + case ModeGeneric: + return "force_generic_plan" + } + return "auto" +} + +// spec renders one run. plain forces plain EXPLAIN with no executions (the psql path). +func (c *Corpus) spec(o RunOptions, e *Entry, mode string, res map[string]*string, resErrs map[string]string, plain bool) runSpec { + st := c.Statements[e.Stmt] + sp := runSpec{e: e, st: st, r: Result{ID: e.ID, Stmt: e.Stmt, Shape: e.Shape, Mode: mode, Tier: o.Tier}, exp: &e.Expect} + if st.Pending != "" && !o.IncludePending { + sp.skip = "pending: " + st.Pending + return sp + } + if ov := e.Mode[mode]; ov != nil { + if ov.Skip != "" { + sp.skip = ov.Skip + return sp + } + if ov.DegradeTo != nil { + sp.exp, sp.r.Degraded = ov.DegradeTo, ov.Reason + if sp.r.Degraded == "" { + sp.r.Degraded = "degrade_to" + } + } + } + params := make([]string, 0, len(e.Params)) + for _, p := range e.Params { + v, skip, err := renderParam(c, p, res, resErrs) + if skip != "" { + sp.skip = skip + return sp + } + if err != nil { + sp.err = err.Error() + return sp + } + params = append(params, v) + } + sp.r.Params = params + text := trimStmt(st.SQL) + if text == "" { + sp.err = fmt.Sprintf("statement %s has no text on this build", e.Stmt) + return sp + } + if err := checkStatementText(text); err != nil { + sp.err = fmt.Sprintf("statement %s: %v", e.Stmt, err) + return sp + } + + sp.analyzed = !plain && o.Tier == "local" && st.Kind == "select" && (e.Analyze == nil || *e.Analyze) + if !plain && st.Kind == "select" && o.Tier != "prod" { + if sp.analyzed { + sp.execs = c.Defaults.Executions + if e.Executions != nil { + sp.execs = *e.Executions + } + } + if mode == ModeAuto && sp.execs < autoExecutions { + sp.execs = autoExecutions + } + } + timeout := c.Defaults.TimeoutS + if e.TimeoutS > 0 { + timeout = e.TimeoutS + } + if timeout <= 0 { + timeout = defaultTimeout + } + if o.Tier == "prod" && timeout > prodTimeoutS { + timeout = prodTimeoutS + } + if o.RunTimeoutS > 0 && timeout > o.RunTimeoutS { + timeout = o.RunTimeoutS + } + + // Every statement sent (and emitted into a psql script) is checked once rendered, settings, search_path and + // params included; a run whose rendered text fails is an ERROR and nothing is sent. + var bad string + line := func(stmts ...string) string { + for _, x := range stmts { + if p := statementProblem(x); p != "" && bad == "" { + bad = fmt.Sprintf("rendered statement %q: %s", firstLine(x), p) + } + } + return strings.Join(stmts, "; ") + } + l2 := []string{"SET LOCAL jit = off", "SET LOCAL timezone = 'UTC'", "SET LOCAL standard_conforming_strings = on"} + if path := c.searchPath(o, st); path != "" { + l2 = append(l2, "SET LOCAL search_path = "+path) + } + sp.setup = []string{ + line("BEGIN ISOLATION LEVEL READ COMMITTED READ ONLY"), + line(l2...), + line(fmt.Sprintf("SET LOCAL statement_timeout = '%ds'", timeout), "SET LOCAL lock_timeout = '1s'"), + line("SET LOCAL plan_cache_mode = " + modeGUC(mode)), + } + switch st.Plan { + case "cached", "custom", "maint": + sp.setup = append(sp.setup, line("SET LOCAL max_parallel_workers_per_gather = 0")) + } + if ts := c.Tiers[o.Tier].Settings; len(ts) > 0 { + var sets []string + for _, k := range sortedKeys(ts) { + sets = append(sets, "SET LOCAL "+k+" = "+quoteLit(ts[k])) + } + sp.setup = append(sp.setup, line(sets...)) + } + sp.setup = append(sp.setup, line("PREPARE "+stmtName+" AS\n"+text)) + sp.call = "EXECUTE " + stmtName + if len(params) > 0 { + sp.call += "(" + strings.Join(params, ", ") + ")" + } + opts := "FORMAT JSON" + if sp.analyzed { + opts = "ANALYZE, BUFFERS, FORMAT JSON" + } + sp.explain = line("EXPLAIN (" + opts + ") " + sp.call) + if bad != "" { + sp.err = bad + } + return sp +} + +// skipMissingIndex turns a run into SKIP when its entry requires an index the database lacks: its plan and +// budget would describe another index set, so it is never judged there. +func (c *Corpus) skipMissingIndex(sp *runSpec, indexes []string) { + if ix := sp.e.MissingIndex(indexes); ix != "" && sp.skip == "" { + sp.skip, sp.err = "requires index "+ix, "" + } +} + +// runOne makes one run. The error is for a session that cannot go on (the rollback or deallocate failed, or ctx +// ended); the Result is filled either way. +func (c *Corpus) runOne(ctx context.Context, s Session, o RunOptions, sp runSpec) (Result, error) { + r := sp.r + switch { + case sp.skip != "": + r.Verdict, r.Skip = VerdictSkip, sp.skip + return r, nil + case sp.err != "": + c.errored(&r, sp.e, sp.err) + return r, nil + } + t0 := time.Now() + var plan []byte + runErr := func() error { + for _, l := range sp.setup { + if _, err := s.Exec(ctx, l); err != nil { + return err + } + } + for i := 0; i < sp.execs; i++ { + if _, err := s.Exec(ctx, sp.call); err != nil { + return err + } + } + res, err := s.Exec(ctx, sp.explain) + if err != nil { + return err + } + var lines []string + for _, row := range lastRows(res) { + if len(row) > 0 { + lines = append(lines, string(row[0])) + } + } + plan = []byte(strings.Join(lines, "\n")) + return nil + }() + r.WallMs = msSince(t0) + // Prepared statements outlive ROLLBACK: end the transaction, then deallocate, as separate commands. + cctx, cancel := cleanupCtx(ctx) + defer cancel() + rbErr := rollback(cctx, s) + _, daErr := s.Exec(cctx, "DEALLOCATE "+stmtName) + if isSQLState(daErr, "26000") { // never prepared: the run failed before PREPARE + daErr = nil + } + if runErr != nil { + c.errored(&r, sp.e, errText(runErr)) + } else if err := c.judge(&r, sp, plan, o); err != nil { + return r, err + } + switch { + case rbErr != nil: + return r, fmt.Errorf("%s|%s: rollback: %w", r.ID, r.Mode, rbErr) + case daErr != nil: + return r, fmt.Errorf("%s|%s: deallocate: %w", r.ID, r.Mode, daErr) + case ctx.Err() != nil: + return r, ctx.Err() + } + return r, nil +} + +func (c *Corpus) errored(r *Result, e *Entry, msg string) { + r.Failures = []Failure{{Kind: FailError, Msg: "error: " + msg}} + r.Verdict = Decide(e.ExpectFail, e.KnownIssue, r.Failures, nil) +} + +// judge checks one plan and decides the run's verdict. The error is a baseline file that cannot be read or written. +func (c *Corpus) judge(r *Result, sp runSpec, plan []byte, o RunOptions) error { + ex, err := ParseExplain(plan) + if err != nil { + c.errored(r, sp.e, err.Error()) + return nil + } + ch := c.Assess(ex, sp.exp, sp.analyzed) + r.Failures, r.Drivers, r.Retired = ch.Failures, ch.Drivers, ch.Retired + r.Buffers, r.Rows = ch.Buffers, ch.Rows + r.PlanningMs, r.ExecMs = ex.PlanningMs, ex.ExecMs + r.Signature = Signature(ex.Root) + r.Plan = append(json.RawMessage(nil), plan...) + if err := baselineStep(r, sp.e, o); err != nil { + return err + } + r.Verdict = Decide(sp.e.ExpectFail, sp.e.KnownIssueIn(r.Mode), r.Failures, r.Retired) + return nil +} + +// baselineStep compares r's signature with, or writes it to, /..json. Expect-fail entries +// and entries marked unstable are left out. +func baselineStep(r *Result, e *Entry, o RunOptions) error { + if o.Baseline == BaselineOff || e.ExpectFail || e.Baseline != "" { + return nil + } + path := filepath.Join(o.BaselineDir, r.ID+"."+r.Mode+".json") + if o.Baseline == BaselineWrite { + if err := os.MkdirAll(o.BaselineDir, 0o755); err != nil { + return err + } + b, err := json.MarshalIndent(r.Signature, "", " ") + if err != nil { + return err + } + return os.WriteFile(path, append(b, '\n'), 0o644) + } + b, err := os.ReadFile(path) + if errors.Is(err, fs.ErrNotExist) { + r.PlanChange = []string{"no baseline"} + r.Failures = append(r.Failures, Failure{Kind: FailPlanChange, Msg: "plan_change: no baseline"}) + return nil + } + if err != nil { + return fmt.Errorf("baseline %s: %w", path, err) + } + var base []string + if err := json.Unmarshal(b, &base); err != nil { + return fmt.Errorf("baseline %s: %w", path, err) + } + if !slices.Equal(base, r.Signature) { + r.PlanChange = lineDiff(base, r.Signature) + r.Failures = append(r.Failures, Failure{Kind: FailPlanChange, Msg: "plan_change: signature differs from baseline (see plan_change)"}) + } + return nil +} + +// lineDiff lists the lines that differ, position by position, as "- old" / "+ new". +func lineDiff(a, b []string) []string { + var out []string + for i := 0; i < max(len(a), len(b)); i++ { + var x, y string + if i < len(a) { + x = a[i] + } + if i < len(b) { + y = b[i] + } + if x != y { + out = append(out, "- "+x, "+ "+y) + } + } + return out +} + +// renderParam renders one param as an untyped literal, so the server infers its type as it does for pgx: +// strings quoted, lists as '{…}' array literals, null as NULL, {raw: …} verbatim, a resolver as its quoted value. +// skip is set when a resolver has no value on this tier. +func renderParam(c *Corpus, v any, res map[string]*string, resErrs map[string]string) (lit, skip string, err error) { + switch x := v.(type) { + case nil: + return "NULL", "", nil + case string: + return quoteLit(x), "", nil + case int, int64, uint64, float64, bool: + return fmt.Sprint(x), "", nil + case []any: + parts := make([]string, 0, len(x)) + for _, y := range x { + switch el := y.(type) { + case map[string]any: + lit, skip, err := renderParam(c, el, res, resErrs) + if skip != "" || err != nil { + return "", skip, err + } + if strings.HasPrefix(lit, "'") { + s, _ := unquoteLit(lit) + parts = append(parts, arrayElem(s)) + } else { + parts = append(parts, lit) + } + default: + parts = append(parts, arrayElem(el)) + } + } + return quoteLit("{" + strings.Join(parts, ",") + "}"), "", nil + case map[string]any: + if raw, ok := x["raw"].(string); ok { + return raw, "", nil + } + key, _, ok := resolverRef(c, x) + if !ok { + return "", "", fmt.Errorf("param %v is not a value, {raw}, {r} or {sql}", x) + } + if msg, failed := resErrs[key]; failed { + return "", "", fmt.Errorf("resolver %s: %s", key, msg) + } + val, ok := res[key] + if !ok { + return "", "", fmt.Errorf("resolver %s was not resolved", key) + } + if val == nil { + return "", "resolver " + key + " returned no value on this tier", nil + } + return quoteLit(*val), "", nil + } + return "", "", fmt.Errorf("unsupported param type %T", v) +} + +// arrayElem renders one element of an array literal. +func arrayElem(v any) string { + switch x := v.(type) { + case nil: + return "NULL" + case string: + return `"` + strings.ReplaceAll(strings.ReplaceAll(x, `\`, `\\`), `"`, `\"`) + `"` + } + return fmt.Sprint(v) +} + +func quoteLit(s string) string { return "'" + strings.ReplaceAll(s, "'", "''") + "'" } + +func unquoteLit(s string) (string, bool) { + if len(s) < 2 || s[0] != '\'' || s[len(s)-1] != '\'' { + return "", false + } + return strings.ReplaceAll(s[1:len(s)-1], "''", "'"), true +} + +// trimStmt drops surrounding space and one trailing semicolon. +func trimStmt(sql string) string { + return strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(sql), ";")) +} + +// cleanupCtx outlives ctx's cancellation, so a transaction still ends when the run is interrupted. +func cleanupCtx(ctx context.Context) (context.Context, context.CancelFunc) { + return context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) +} + +func cleanupRollback(ctx context.Context, s Session) error { + cctx, cancel := cleanupCtx(ctx) + defer cancel() + return rollback(cctx, s) +} + +func rollback(ctx context.Context, s Session) error { + _, err := s.Exec(ctx, "ROLLBACK") + return err +} + +func lastRows(res []*pgconn.Result) [][][]byte { + if len(res) == 0 { + return nil + } + return res[len(res)-1].Rows +} + +func msSince(t time.Time) float64 { return float64(time.Since(t).Microseconds()) / 1000 } + +func firstLine(s string) string { + if i := strings.IndexByte(s, '\n'); i >= 0 { + return s[:i] + } + return s +} diff --git a/internal/planguard/runner_integration_test.go b/internal/planguard/runner_integration_test.go new file mode 100644 index 0000000..9d100ba --- /dev/null +++ b/internal/planguard/runner_integration_test.go @@ -0,0 +1,314 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package planguard_test + +import ( + "context" + "encoding/json" + "os" + "regexp" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/db" + "github.com/MeshCore-Beacon/beacon-server/internal/planguard" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" +) + +// runnerEnv is a migrated schema holding ~2,000 known_routes over three regions, and a pgconn session on it. +type runnerEnv struct { + ctx context.Context + schema string + pool *pgxpool.Pool + pc *pgconn.PgConn + corpus *planguard.Corpus +} + +func newRunnerEnv(t *testing.T) *runnerEnv { + t.Helper() + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for PostgreSQL tests") + } + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + t.Cleanup(cancel) + admin, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + schema := "planguard_" + strings.ReplaceAll(uuid.NewString(), "-", "") + if _, err := admin.Exec(ctx, "CREATE SCHEMA "+schema); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if _, err := admin.Exec(context.Background(), "DROP SCHEMA "+schema+" CASCADE"); err != nil { + t.Error(err) + } + admin.Close(context.Background()) + }) + cfg, err := pgxpool.ParseConfig(dsn) + if err != nil { + t.Fatal(err) + } + cfg.ConnConfig.RuntimeParams["search_path"] = schema + cfg.MaxConns = 2 + pool, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + t.Fatal(err) + } + t.Cleanup(pool.Close) + if err := db.RunMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, ` +INSERT INTO iata_codes (iata) VALUES ('SEA'), ('YVR'), ('PDX'); +INSERT INTO known_routes (path_key, node_ids, hash_prefix, iata, hop_count, last_seen, observation_count) +SELECT int8send(g), ARRAY[md5(g::text)::uuid], ARRAY['\x01'::bytea], (ARRAY['SEA','YVR','PDX'])[1 + g % 3], + 1 + g % 5, timestamptz '2026-03-01 00:00:00+00' - g * interval '1 minute', 1 + g % 4 +FROM generate_series(1, 2000) g`); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, "ANALYZE known_routes"); err != nil { + t.Fatal(err) + } + pc, err := pgconn.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = pc.Close(context.Background()) }) + sqlc, err := planguard.LoadStatements("../../db/sqlc") + if err != nil { + t.Fatal(err) + } + c, err := planguard.LoadCorpus(os.DirFS("testdata/corpus"), "runner.yaml", sqlc) + if err != nil { + t.Fatal(err) + } + return &runnerEnv{ctx: ctx, schema: schema, pool: pool, pc: pc, corpus: c} +} + +func (e *runnerEnv) run(t *testing.T, only string) *planguard.Report { + t.Helper() + rep, err := planguard.Run(e.ctx, e.corpus, planguard.NewSession(e.pc), planguard.RunOptions{ + Tier: "local", + Only: regexp.MustCompile(only), + SearchPath: e.schema, + Baseline: planguard.BaselineOff, + }) + if err != nil { + t.Fatal(err) + } + return rep +} + +// scalar runs one single-value query on the runner's own connection. +func (e *runnerEnv) scalar(t *testing.T, sql string) string { + t.Helper() + res, err := e.pc.Exec(e.ctx, sql).ReadAll() + if err != nil { + t.Fatal(err) + } + return string(res[len(res)-1].Rows[0][0]) +} + +func byKey(rep *planguard.Report) map[string]planguard.Result { + m := map[string]planguard.Result{} + for _, r := range rep.Results { + m[r.ID+"|"+r.Mode] = r + } + return m +} + +func TestRunnerReadOnlyPostgres(t *testing.T) { + e := newRunnerEnv(t) + cacheMode := e.scalar(t, "SHOW plan_cache_mode") + var before int + if err := e.pool.QueryRow(e.ctx, "SELECT count(*) FROM known_routes").Scan(&before); err != nil { + t.Fatal(err) + } + // A statement that writes, labelled select: the run executes and EXPLAIN ANALYZEs it, so only the READ ONLY + // transaction stands between it and the table. + e.corpus.Statements["DeleteAll"].Kind = "select" + rep := e.run(t, `^(routes\.(byiata|globalhop|delete_expired)|harness\.delete_all)$`) + got := byKey(rep) + del, ok := got["harness.delete_all|custom"] + if !ok { + t.Fatalf("no harness.delete_all result: %v", got) + } + if del.Verdict != planguard.VerdictError || len(del.Failures) == 0 || !strings.Contains(del.Failures[0].Msg, "25006") { + t.Errorf("harness.delete_all: verdict %s failures %v, want ERROR 25006 (read-only transaction)", del.Verdict, del.Failures) + } + delete(got, "harness.delete_all|custom") + for _, k := range []string{"routes.byiata|custom", "routes.byiata|generic", "routes.globalhop|custom", "routes.delete_expired|custom"} { + if _, ok := got[k]; !ok { + t.Fatalf("no result for %s; results %v", k, got) + } + } + if len(rep.Results) != 5 { + t.Errorf("results = %d, want 5", len(rep.Results)) + } + for k, r := range got { + if r.Verdict.Failing() { + t.Errorf("%s: verdict %s, failures %v", k, r.Verdict, r.Failures) + } + if r.Stmt == "DeleteExpiredRoutes" { + if r.Buffers != 0 || r.ExecMs != 0 { + t.Errorf("%s: dml was analyzed: buffers %d exec %.2f ms", k, r.Buffers, r.ExecMs) + } + if len(r.Plan) == 0 { + t.Errorf("%s: no plan", k) + } + continue + } + if r.Buffers <= 0 || r.PlanningMs <= 0 { + t.Errorf("%s: buffers %d planning %.3f ms, want both > 0 (EXPLAIN ANALYZE, BUFFERS)", k, r.Buffers, r.PlanningMs) + } + } + if len(rep.Indexes) == 0 || !contains(rep.Indexes, "idx_known_routes_iata_keyset") { + t.Errorf("indexes = %v, want the known_routes indexes", rep.Indexes) + } + if rep.Settings["random_page_cost"] == "" { + t.Errorf("settings = %v, want random_page_cost", rep.Settings) + } + var after int + if err := e.pool.QueryRow(e.ctx, "SELECT count(*) FROM known_routes").Scan(&after); err != nil { + t.Fatal(err) + } + if before != 2000 { + t.Errorf("seeded %d rows, want 2000 (all inside the delete cutoffs)", before) + } + if after != before { + t.Errorf("known_routes rows %d → %d; the runner wrote", before, after) + } + if n := e.scalar(t, "SELECT count(*) FROM pg_prepared_statements"); n != "0" { + t.Errorf("pg_prepared_statements = %s after Run, want 0", n) + } + if m := e.scalar(t, "SHOW plan_cache_mode"); m != cacheMode { + t.Errorf("plan_cache_mode = %s after Run, want the session default %s", m, cacheMode) + } + if st := e.pc.TxStatus(); st != 'I' { + t.Errorf("transaction status %q after Run, want idle", st) + } +} + +func TestRunnerRecoversAfterErrorPostgres(t *testing.T) { + e := newRunnerEnv(t) + rep := e.run(t, `^(sleep\.timeout|routes\.globalhop)$`) + got := byKey(rep) + sl, ok := got["sleep.timeout|custom"] + if !ok { + t.Fatalf("no sleep result: %v", got) + } + if sl.Verdict != planguard.VerdictError || len(sl.Failures) == 0 || !strings.Contains(sl.Failures[0].Msg, "statement timeout") { + t.Errorf("sleep: verdict %s failures %v, want ERROR on a statement timeout", sl.Verdict, sl.Failures) + } + gh := got["routes.globalhop|custom"] + if gh.Verdict != planguard.VerdictOK || gh.Buffers <= 0 { + t.Errorf("globalhop after the timeout: verdict %s buffers %d failures %v, want ok", gh.Verdict, gh.Buffers, gh.Failures) + } + if n := e.scalar(t, "SELECT count(*) FROM pg_prepared_statements"); n != "0" { + t.Errorf("pg_prepared_statements = %s after Run, want 0", n) + } + if rep.Counts()[planguard.VerdictError] != 1 || len(rep.Failures()) != 1 { + t.Errorf("counts %v failures %d, want one ERROR", rep.Counts(), len(rep.Failures())) + } +} + +func TestUnresolvedParamsSkipPostgres(t *testing.T) { + e := newRunnerEnv(t) + rep := e.run(t, `^routes\.byiata\.(no_region|busy)$`) + got := byKey(rep) + nr := got["routes.byiata.no_region|custom"] + if nr.Verdict != planguard.VerdictSkip || nr.Skip != "resolver no_region returned no value on this tier" { + t.Errorf("no_region: verdict %s skip %q", nr.Verdict, nr.Skip) + } + busy := got["routes.byiata.busy|custom"] + if busy.Verdict != planguard.VerdictOK || len(busy.Params) == 0 || busy.Params[0] != "'PDX'" && busy.Params[0] != "'SEA'" && busy.Params[0] != "'YVR'" { + t.Errorf("busy: verdict %s params %v failures %v", busy.Verdict, busy.Params, busy.Failures) + } + for v, n := range rep.Counts() { + if v.Failing() && n > 0 { + t.Errorf("counts %v hold a failing verdict", rep.Counts()) + } + } + + // A resolver that errors fails only its own entries. + rep = e.run(t, `^routes\.byiata\.(broken|busy)$`) + got = byKey(rep) + if br := got["routes.byiata.broken|custom"]; br.Verdict != planguard.VerdictError || len(br.Failures) == 0 || !strings.Contains(br.Failures[0].Msg, "resolver broken") { + t.Errorf("broken: verdict %s failures %v, want ERROR naming the resolver", br.Verdict, br.Failures) + } + if busy := got["routes.byiata.busy|custom"]; busy.Verdict != planguard.VerdictOK { + t.Errorf("busy beside a broken resolver: verdict %s failures %v", busy.Verdict, busy.Failures) + } + if n := e.scalar(t, "SELECT count(*) FROM pg_prepared_statements"); n != "0" { + t.Errorf("pg_prepared_statements = %s after Run, want 0", n) + } +} + +func TestGenericModeShowsParamsPostgres(t *testing.T) { + e := newRunnerEnv(t) + got := byKey(e.run(t, `^routes\.byiata$`)) + conds := func(r planguard.Result) string { + var out []string + var rec func(v any) + rec = func(v any) { + switch x := v.(type) { + case map[string]any: + for k, y := range x { + if k == "Index Cond" { + out = append(out, y.(string)) + } + rec(y) + } + case []any: + for _, y := range x { + rec(y) + } + } + } + var doc any + if err := json.Unmarshal(r.Plan, &doc); err != nil { + t.Fatalf("%s|%s plan: %v", r.ID, r.Mode, err) + } + rec(doc) + return strings.Join(out, " && ") + } + if c := conds(got["routes.byiata|generic"]); !strings.Contains(c, "$1") { + t.Errorf("generic Index Cond %q, want $1", c) + } + if c := conds(got["routes.byiata|custom"]); !strings.Contains(c, "'SEA'::bpchar") || strings.Contains(c, "$1") { + t.Errorf("custom Index Cond %q, want 'SEA'::bpchar", c) + } +} + +func contains(xs []string, x string) bool { + for _, y := range xs { + if y == x { + return true + } + } + return false +} + +// An entry whose required index is missing from the database is SKIP with the index named, never judged against +// a plan of another index set; an entry whose required index exists runs. +func TestRequiresIndexSkipsPostgres(t *testing.T) { + e := newRunnerEnv(t) + e.corpus.Entry("routes.byiata").RequiresIndexes = []string{"idx_known_routes_iata_keyset", "idx_not_in_this_database"} + e.corpus.Entry("routes.globalhop").RequiresIndexes = []string{"idx_known_routes_hop_iata"} + got := byKey(e.run(t, `^routes\.(byiata|globalhop)$`)) + for _, k := range []string{"routes.byiata|custom", "routes.byiata|generic"} { + if r := got[k]; r.Verdict != planguard.VerdictSkip || r.Skip != "requires index idx_not_in_this_database" || len(r.Plan) != 0 { + t.Errorf("%s: verdict %s skip %q, want SKIP requires index idx_not_in_this_database, unplanned", k, r.Verdict, r.Skip) + } + } + if r := got["routes.globalhop|custom"]; r.Verdict == planguard.VerdictSkip || r.Verdict.Failing() || len(r.Plan) == 0 { + t.Errorf("routes.globalhop: verdict %s skip %q failures %v, want a judged run", r.Verdict, r.Skip, r.Failures) + } +} diff --git a/internal/planguard/session.go b/internal/planguard/session.go new file mode 100644 index 0000000..86ac180 --- /dev/null +++ b/internal/planguard/session.go @@ -0,0 +1,51 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package planguard + +import ( + "context" + "errors" + "fmt" + + "github.com/jackc/pgx/v5/pgconn" +) + +// Session sends SQL over the simple protocol, as psql does, so a statement's parameters are typed by the server. +type Session interface { + // Exec runs sql (one or more statements) and reads every result. The first error, from the connection or + // from any statement, is returned. + Exec(ctx context.Context, sql string) ([]*pgconn.Result, error) +} + +// NewSession returns a Session on pc. The caller owns pc. +func NewSession(pc *pgconn.PgConn) Session { return pgSession{pc} } + +type pgSession struct{ pc *pgconn.PgConn } + +func (s pgSession) Exec(ctx context.Context, sql string) ([]*pgconn.Result, error) { + res, err := s.pc.Exec(ctx, sql).ReadAll() + if err != nil { + return res, err + } + for _, r := range res { + if r.Err != nil { + return res, r.Err + } + } + return res, nil +} + +// errText is a one-line description of err; a server error gives its SQLSTATE and message. +func errText(err error) string { + var pe *pgconn.PgError + if errors.As(err, &pe) { + return fmt.Sprintf("%s: %s", pe.Code, pe.Message) + } + return firstLine(err.Error()) +} + +func isSQLState(err error, code string) bool { + var pe *pgconn.PgError + return errors.As(err, &pe) && pe.Code == code +} diff --git a/internal/planguard/sqltext.go b/internal/planguard/sqltext.go new file mode 100644 index 0000000..802765a --- /dev/null +++ b/internal/planguard/sqltext.go @@ -0,0 +1,510 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package planguard checks that the planner keeps choosing the plans Beacon's statements were designed for. +package planguard + +import ( + "bytes" + "errors" + "fmt" + "go/ast" + "go/parser" + "go/token" + "io/fs" + "os" + "os/exec" + "path" + "regexp" + "sort" + "strconv" + "strings" + "testing/fstest" +) + +var ( + sqlcHeader = regexp.MustCompile(`^-- name: (\w+) :(\w+)`) + rawHeader = regexp.MustCompile(`-- name: \w+ :\w+`) +) + +// foldString returns the value of a string literal or a + chain of string literals (sqlc writes a backtick +// inside a statement as `...` + "`" + `...`). ok is false for any other expression. +func foldString(e ast.Expr) (string, bool, error) { + switch x := e.(type) { + case *ast.BasicLit: + if x.Kind != token.STRING { + return "", false, nil + } + s, err := strconv.Unquote(x.Value) + if err != nil { + return "", false, fmt.Errorf("unquote: %w", err) + } + return s, true, nil + case *ast.ParenExpr: + return foldString(x.X) + case *ast.BinaryExpr: + if x.Op != token.ADD { + return "", false, nil + } + l, ok, err := foldString(x.X) + if err != nil || !ok { + return "", false, err + } + r, ok, err := foldString(x.Y) + if err != nil || !ok { + return "", false, err + } + return l + r, true, nil + } + return "", false, nil +} + +// LoadStatements parses every *.sql.go file in dir (db/sqlc) and returns sqlc name → the constant's text, +// "-- name: X :kind" header included. Errors: a parse error, a duplicate name, or no statement at all. +func LoadStatements(dir string) (map[string]string, error) { + return LoadStatementsFS(os.DirFS(dir), "*.sql.go") +} + +// LoadStatementsFS is LoadStatements over the files of fsys that match pattern, e.g. "sqlc/*.sql.go". +func LoadStatementsFS(fsys fs.FS, pattern string) (map[string]string, error) { + files, err := fs.Glob(fsys, pattern) + if err != nil { + return nil, fmt.Errorf("glob %s: %w", pattern, err) + } + sort.Strings(files) + out := map[string]string{} + from := map[string]string{} + fset := token.NewFileSet() + for _, name := range files { + src, err := fs.ReadFile(fsys, name) + if err != nil { + return nil, fmt.Errorf("read %s: %w", name, err) + } + f, err := parser.ParseFile(fset, name, src, parser.SkipObjectResolution) + if err != nil { + return nil, fmt.Errorf("parse %s: %w", name, err) + } + extracted := 0 + for _, decl := range f.Decls { + gd, ok := decl.(*ast.GenDecl) + if !ok || gd.Tok != token.CONST { + continue + } + for _, spec := range gd.Specs { + vs, ok := spec.(*ast.ValueSpec) + if !ok { + continue + } + for _, v := range vs.Values { + text, ok, err := foldString(v) + if err != nil { + return nil, fmt.Errorf("%s: %s: %w", name, fset.Position(v.Pos()), err) + } + if !ok { + continue + } + m := sqlcHeader.FindStringSubmatch(text) + if m == nil { + continue + } + extracted++ + if prev, dup := from[m[1]]; dup { + return nil, fmt.Errorf("duplicate sqlc statement %q in %s and %s", m[1], prev, name) + } + from[m[1]] = name + out[m[1]] = text + } + } + } + if headers := len(rawHeader.FindAll(src, -1)); headers != extracted { + return nil, fmt.Errorf("%s: %d statement headers in the file but %d extracted; a statement constant is not a foldable string", name, headers, extracted) + } + } + if len(out) == 0 { + return nil, fmt.Errorf("no statement found in %d files matching %s", len(files), pattern) + } + return out, nil +} + +// LoadStatementsAtRef reads db/sqlc at a git revision of the repository in repoDir (git ls-tree, then git show +// :db/sqlc/) and parses it like LoadStatements. +func LoadStatementsAtRef(repoDir, ref string) (map[string]string, error) { + if ref == "" || strings.HasPrefix(ref, "-") { + return nil, fmt.Errorf("invalid git ref %q", ref) + } + list, err := git(repoDir, "ls-tree", "--full-tree", "--name-only", ref, "--", "db/sqlc/") + if err != nil { + return nil, err + } + files := fstest.MapFS{} + for _, p := range strings.Split(strings.TrimSpace(string(list)), "\n") { + if !strings.HasSuffix(p, ".sql.go") { + continue + } + body, err := git(repoDir, "show", ref+":"+p) + if err != nil { + return nil, err + } + files[path.Base(p)] = &fstest.MapFile{Data: body} + } + if len(files) == 0 { + return nil, fmt.Errorf("no db/sqlc/*.sql.go at %s", ref) + } + return LoadStatementsFS(files, "*.sql.go") +} + +func git(dir string, args ...string) ([]byte, error) { + cmd := exec.Command("git", append([]string{"-C", dir}, args...)...) + var stderr bytes.Buffer + cmd.Stderr = &stderr + out, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(stderr.String())) + } + return out, nil +} + +var ( + paramRef = regexp.MustCompile(`\$[0-9]+`) + whitespace = regexp.MustCompile(`\s+`) +) + +// Normalize strips -- comments, collapses whitespace and trailing ';', and replaces $n by $? (renumbering-proof). +// A "--" inside a quoted literal or identifier is kept. +func Normalize(sql string) string { + s := stripLineComments(sql) + s = paramRef.ReplaceAllString(s, "$$?") + s = strings.TrimSpace(whitespace.ReplaceAllString(s, " ")) + for strings.HasSuffix(s, ";") { + s = strings.TrimSpace(strings.TrimSuffix(s, ";")) + } + return s +} + +func stripLineComments(sql string) string { + var b strings.Builder + var quote byte // ' or " while inside a quoted run + for i := 0; i < len(sql); i++ { + c := sql[i] + switch { + case quote != 0: + if c == quote { + quote = 0 // a doubled quote reopens on the next byte + } + case c == '\'' || c == '"': + quote = c + case c == '-' && i+1 < len(sql) && sql[i+1] == '-': + for i < len(sql) && sql[i] != '\n' { + i++ + } + if i < len(sql) { + b.WriteByte('\n') + } + continue + } + b.WriteByte(c) + } + return b.String() +} + +var lockClause = regexp.MustCompile(`(?i)\s*\bFOR\s+UPDATE(\s+OF\s+\w+(\s*,\s*\w+)*)?\s+SKIP\s+LOCKED\b`) + +// statementProblem scans sql, after one trailing ';' is dropped, and names the first thing that makes it unsafe to +// send over the simple protocol or into a psql script, or returns "": +// - a ';' outside literals, quoted identifiers, dollar quotes and comments: a second statement (a COMMIT, say) +// would run outside the READ ONLY transaction; +// - an escape string (E'...'): a backslash there escapes a quote, so the scan could not tell where it ends; +// - a backslash anywhere but inside a standard single-quoted literal (fail closed: not in dollar quotes, comments +// or quoted identifiers either): psql reads it as a meta-command (\! runs a shell, \o writes a file); +// - a literal, quoted identifier, dollar quote or comment that does not end. +// +// Identifiers and dollar-quote tags follow PostgreSQL's lexer: bytes ≥ 0x80 are identifier characters +// (ident_start, ident_cont, dolq_start, dolq_cont), so "aé$x$" is one identifier and "$é$" opens a dollar quote. +func statementProblem(sql string) string { + s := trimStmt(sql) + for i := 0; i < len(s); { + c := s[i] + switch { + case c == '-' && strings.HasPrefix(s[i:], "--"): + end := len(s) + if j := strings.IndexByte(s[i:], '\n'); j >= 0 { + end = i + j + } + if strings.IndexByte(s[i:end], '\\') >= 0 { + return "a backslash in a comment (psql reads it as a meta-command)" + } + i = end + case c == '/' && strings.HasPrefix(s[i:], "/*"): + depth, j := 1, i+2 + for j < len(s) && depth > 0 { + switch { + case strings.HasPrefix(s[j:], "/*"): + depth, j = depth+1, j+2 + case strings.HasPrefix(s[j:], "*/"): + depth, j = depth-1, j+2 + case s[j] == '\\': + return "a backslash in a comment (psql reads it as a meta-command)" + default: + j++ + } + } + if depth > 0 { + return "a comment that does not end" + } + i = j + case c == '\'': + // E' opens an escape string unless the E ends an identifier. Fail closed: only a letter, '_' or a byte + // ≥ 0x80 before the E makes it an identifier; after a digit or '$' the lexers can disagree, so refuse. + if i > 0 && (s[i-1] == 'E' || s[i-1] == 'e') && (i == 1 || !identStart(s[i-2])) { + return "an E'' escape string (a backslash there escapes a quote)" + } + j := i + 1 + for ; ; j++ { + if j >= len(s) { + return "a string literal that does not end" + } + if s[j] == '\'' { + if j+1 < len(s) && s[j+1] == '\'' { + j++ + continue + } + break + } + } + i = j + 1 + case c == '"': + j := i + 1 + for ; ; j++ { + if j >= len(s) { + return "a quoted identifier that does not end" + } + if s[j] == '\\' { + return "a backslash in a quoted identifier (psql reads it as a meta-command)" + } + if s[j] == '"' { + if j+1 < len(s) && s[j+1] == '"' { + j++ + continue + } + break + } + } + i = j + 1 + case c == '$': + j := i + 1 + for j < len(s) && lexIdent(s[j]) && !(j == i+1 && s[j] >= '0' && s[j] <= '9') { + j++ + } + if j < len(s) && s[j] == '$' { + tag := s[i : j+1] + k := strings.Index(s[j+1:], tag) + if k < 0 { + return "a dollar quote that does not end" + } + if strings.IndexByte(s[j+1:j+1+k], '\\') >= 0 { + return "a backslash in a dollar quote (psql could read it as a meta-command)" + } + i = j + 1 + k + len(tag) + } else { + i = j + } + case lexIdent(c): + for i < len(s) && (lexIdent(s[i]) || s[i] == '$') { + i++ + } + case c == '\\': + return "a backslash outside a literal (psql reads it as a meta-command)" + case c == ';': + return "more than one statement (a ';' outside literals and comments)" + default: + i++ + } + } + return "" +} + +// lexIdent reports a byte PostgreSQL's lexer takes as part of an identifier or dollar-quote tag: a letter, a digit, +// '_' or any byte ≥ 0x80. +func lexIdent(b byte) bool { return isIdent(b) || b >= 0x80 } + +// identStart reports a byte that can start an identifier: a letter, '_' or a byte ≥ 0x80 (no digit, no '$'). +func identStart(b byte) bool { return lexIdent(b) && !(b >= '0' && b <= '9') } + +// checkStatementText refuses statement, resolver and inline {sql} text that statementProblem objects to. +func checkStatementText(sql string) error { + if p := statementProblem(sql); p != "" { + return errors.New(p) + } + return nil +} + +// checkScriptLines refuses an emitted SQL body with a line that starts with a backslash once trimmed: psql would +// run it as a meta-command even where the SQL lexer would read it as part of a literal. +func checkScriptLines(body string) error { + for _, l := range strings.Split(body, "\n") { + if strings.HasPrefix(strings.TrimSpace(l), "\\") { + return fmt.Errorf("a line starts with a backslash (a psql meta-command): %q", firstLine(strings.TrimSpace(l))) + } + } + return nil +} + +// StripLockClause removes every "FOR UPDATE [OF x[, y]] SKIP LOCKED" from sql. +func StripLockClause(sql string) string { + return lockClause.ReplaceAllString(sql, "") +} + +// DerivedBodyMatches reports whether body, the frozen read-only part of a DML statement, is still part of that +// statement's source text once its lock clause is removed. Both sides are normalized, so parameter numbers, +// comments and whitespace do not matter, and one edited token does. +// +// The match must sit on token boundaries: it starts the source or follows whitespace or "(", and it ends the +// source or is followed by ")" or ";", directly or after whitespace. A body that stops short of a clause +// ("LIMIT $1" inside "LIMIT $5 OFFSET $6") does not match. +func DerivedBodyMatches(source, body string) bool { + b := Normalize(body) + if b == "" { + return false + } + src := Normalize(StripLockClause(source)) + for from := 0; ; { + i := strings.Index(src[from:], b) + if i < 0 { + return false + } + start := from + i + end := start + len(b) + if boundaryBefore(src, start) && boundaryAfter(src, end) { + return true + } + from = start + 1 + } +} + +func boundaryBefore(s string, i int) bool { + return i == 0 || s[i-1] == ' ' || s[i-1] == '(' +} + +func boundaryAfter(s string, i int) bool { + if i < len(s) && s[i] == ' ' { + i++ + } + return i == len(s) || s[i] == ')' || s[i] == ';' +} + +var ( + gucValueRe = regexp.MustCompile(`^[A-Za-z0-9_.+-]+$`) + rawWordRe = regexp.MustCompile(`^(-?[0-9]+(\.[0-9]+)?|(?i:null|true|false))`) + rawTypeRe = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)?(\[\])?`) +) + +// checkSettings refuses a tier setting whose name is not a GUC name or whose value is anything but letters, digits +// and "_.+-" (1.1, 5GB, off). Settings are rendered into SET statements and psql scripts. +func checkSettings(settings map[string]string) error { + for _, k := range sortedKeys(settings) { + if !gucNameRe.MatchString(k) { + return fmt.Errorf("setting name %q is not a GUC name", k) + } + if !gucValueRe.MatchString(settings[k]) { + return fmt.Errorf("setting %s: value %q must match %s", k, settings[k], gucValueRe) + } + } + return nil +} + +// checkRawParam accepts only a {raw: …} param of this grammar, so a raw param cannot carry a statement or a psql +// meta-command: +// +// value = (number | NULL | true | false | literal | ARRAY[value {, value}]) [::type] +// +// where number is an integer or decimal with an optional '-', literal a standard '...' literal (” for a quote, no +// control characters), and type an identifier, optionally schema-qualified, optionally followed by []. Tokens may be +// separated by spaces only. +func checkRawParam(s string) error { + p := rawParser{s: s} + if err := p.value(); err != nil { + return err + } + p.space() + if p.i != len(s) { + return fmt.Errorf("unexpected %q", s[p.i:]) + } + return nil +} + +type rawParser struct { + s string + i int +} + +func (p *rawParser) space() { + for p.i < len(p.s) && p.s[p.i] == ' ' { + p.i++ + } +} + +func (p *rawParser) value() error { + p.space() + rest := p.s[p.i:] + switch { + case len(rest) >= 6 && strings.EqualFold(rest[:6], "ARRAY["): + p.i += 6 + for { + if err := p.value(); err != nil { + return err + } + p.space() + if p.i < len(p.s) && p.s[p.i] == ',' { + p.i++ + continue + } + if p.i < len(p.s) && p.s[p.i] == ']' { + p.i++ + break + } + return errors.New("ARRAY[…] needs ',' or ']'") + } + case strings.HasPrefix(rest, "'"): + j := p.i + 1 + for ; ; j++ { + if j >= len(p.s) { + return errors.New("a literal that does not end") + } + c := p.s[j] + if c < 0x20 || c == 0x7f { + return errors.New("a control character in a literal") + } + if c == '\'' { + if j+1 < len(p.s) && p.s[j+1] == '\'' { + j++ + continue + } + break + } + } + p.i = j + 1 + default: + m := rawWordRe.FindString(rest) + if m == "" { + return fmt.Errorf("unexpected %q (want a number, NULL, true, false, a '...' literal or ARRAY[…])", rest) + } + p.i += len(m) + if p.i < len(p.s) && lexIdent(p.s[p.i]) { + return fmt.Errorf("unexpected %q", p.s[p.i-len(m):]) + } + } + p.space() + if strings.HasPrefix(p.s[p.i:], "::") { + p.i += 2 + p.space() + m := rawTypeRe.FindString(p.s[p.i:]) + if m == "" { + return fmt.Errorf("a cast needs a type name, not %q", p.s[p.i:]) + } + p.i += len(m) + if p.i < len(p.s) && (lexIdent(p.s[p.i]) || p.s[p.i] == '$') { + return fmt.Errorf("unexpected %q", p.s[p.i:]) + } + } + return nil +} diff --git a/internal/planguard/sqltext_test.go b/internal/planguard/sqltext_test.go new file mode 100644 index 0000000..7ac56e0 --- /dev/null +++ b/internal/planguard/sqltext_test.go @@ -0,0 +1,159 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package planguard_test + +import ( + "os/exec" + "path" + "reflect" + "strings" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/db" + "github.com/MeshCore-Beacon/beacon-server/internal/planguard" +) + +func TestLoadStatementsParsesSqlc(t *testing.T) { + m, err := planguard.LoadStatements("../../db/sqlc") + if err != nil { + t.Fatalf("LoadStatements: %v", err) + } + t.Logf("%d statements", len(m)) + if len(m) < 190 { + t.Fatalf("got %d statements, want at least 190", len(m)) + } + got := m["ListKnownRoutesByIATA"] + if !strings.HasPrefix(got, "-- name: ListKnownRoutesByIATA :many\n") { + t.Fatalf("ListKnownRoutesByIATA text does not start with its header: %q", firstLine(got)) + } + if !strings.Contains(got, "LIMIT $4") { + t.Fatalf("ListKnownRoutesByIATA text lacks LIMIT $4") + } + for name, text := range m { + if name == "" { + t.Fatalf("empty statement name for text %q", firstLine(text)) + } + if !strings.HasPrefix(text, "-- name: "+name+" :") { + t.Fatalf("%s: text header does not name it: %q", name, firstLine(text)) + } + } + + fromFS, err := planguard.LoadStatementsFS(db.SQLCSources, "sqlc/*.sql.go") + if err != nil { + t.Fatalf("LoadStatementsFS: %v", err) + } + if !reflect.DeepEqual(fromFS, m) { + t.Fatalf("embedded statements differ from the directory: %d vs %d", len(fromFS), len(m)) + } +} + +func TestLoadStatementsRejectsDuplicatesAndEmpty(t *testing.T) { + dup := map[string]string{ + "a.sql.go": "package db\n\nconst a = `-- name: Same :one\nSELECT 1`\n", + "b.sql.go": "package db\n\nconst b = `-- name: Same :one\nSELECT 2`\n", + } + if _, err := planguard.LoadStatementsFS(mapFS(dup), "*.sql.go"); err == nil || !strings.Contains(err.Error(), "Same") { + t.Fatalf("duplicate name: err = %v, want one naming Same", err) + } + none := map[string]string{"a.sql.go": "package db\n\nconst a = \"SELECT 1\"\n"} + if _, err := planguard.LoadStatementsFS(mapFS(none), "*.sql.go"); err == nil { + t.Fatal("no statement: want an error") + } + broken := map[string]string{"a.sql.go": "package db\n\nconst a = `-- name: A :one\n"} + if _, err := planguard.LoadStatementsFS(mapFS(broken), "*.sql.go"); err == nil || !strings.Contains(err.Error(), "a.sql.go") { + t.Fatalf("parse error: err = %v, want one naming a.sql.go", err) + } +} + +func TestLoadStatementsFoldsConcatenation(t *testing.T) { + // sqlc writes a backtick inside a statement as `...` + "`" + `...`. + src := "package db\n\nconst a = `-- name: Quoted :one\nSELECT 1 AS ` + \"`\" + `x` + \"`\" + `\n`\n" + m, err := planguard.LoadStatementsFS(mapFS(map[string]string{"a.sql.go": src}), "*.sql.go") + if err != nil { + t.Fatalf("LoadStatementsFS: %v", err) + } + if want := "-- name: Quoted :one\nSELECT 1 AS `x`\n"; m["Quoted"] != want { + t.Fatalf("Quoted = %q, want %q", m["Quoted"], want) + } + + unfoldable := "package db\n\nconst prefix = \"\"\nconst ok = `-- name: Ok :one\nSELECT 1`\nconst b = prefix + `-- name: Hidden :one\nSELECT 2`\n" + _, err = planguard.LoadStatementsFS(mapFS(map[string]string{"b.sql.go": unfoldable}), "*.sql.go") + if err == nil || !strings.Contains(err.Error(), "b.sql.go") || !strings.Contains(err.Error(), "2 statement headers") { + t.Fatalf("unfoldable constant: err = %v, want a header-count error naming b.sql.go", err) + } +} + +func TestLoadStatementsAtRef(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not installed") + } + if err := exec.Command("git", "-C", "../..", "rev-parse", "--verify", "HEAD").Run(); err != nil { + t.Skip("not a git checkout") + } + atHead, err := planguard.LoadStatementsAtRef("../..", "HEAD") + if err != nil { + t.Fatalf("LoadStatementsAtRef: %v", err) + } + out, err := exec.Command("git", "-C", "../..", "diff", "--name-only", "HEAD", "--", "db/sqlc").Output() + if err != nil { + t.Fatalf("git diff: %v", err) + } + changed := map[string]bool{} + for _, f := range strings.Fields(string(out)) { + changed[path.Base(f)] = true + } + for file := range changed { + t.Logf("skipping %s: uncommitted change", file) + } + compared := 0 + for _, file := range sqlcFiles(t) { + if changed[file] { + continue + } + one, err := planguard.LoadStatementsFS(dirFS("../../db/sqlc"), file) + if err != nil { + if strings.Contains(err.Error(), "no statement") { + continue + } + t.Fatalf("%s: %v", file, err) + } + for name, text := range one { + if atHead[name] != text { + t.Fatalf("%s: text at HEAD differs from the working tree", name) + } + compared++ + } + } + if compared < 190 { + t.Fatalf("compared %d statements, want at least 190", compared) + } + + if _, err := planguard.LoadStatementsAtRef("../..", "no-such-ref-xyz"); err == nil { + t.Fatal("bad ref: want an error") + } + if _, err := planguard.LoadStatementsAtRef("../..", "--output=/tmp/x"); err == nil { + t.Fatal("option-like ref: want an error") + } +} + +func TestNormalizeRenumbers(t *testing.T) { + got := planguard.Normalize("-- name: X :one\nSELECT $2::int,\n $10 ;") + if want := "SELECT $?::int, $?"; got != want { + t.Fatalf("Normalize = %q, want %q", got, want) + } + if a, b := planguard.Normalize("SELECT $1 -- trailing\nFROM t;;"), planguard.Normalize("SELECT $7\n\tFROM t"); a != b { + t.Fatalf("renumbered texts differ: %q vs %q", a, b) + } + // A "--" inside a string literal is data, not a comment. + if got := planguard.Normalize("SELECT '--x' AS y"); got != "SELECT '--x' AS y" { + t.Fatalf("Normalize stripped a literal: %q", got) + } +} + +func firstLine(s string) string { + if i := strings.IndexByte(s, '\n'); i >= 0 { + return s[:i] + } + return s +} diff --git a/internal/planguard/storediff.go b/internal/planguard/storediff.go new file mode 100644 index 0000000..4c7a521 --- /dev/null +++ b/internal/planguard/storediff.go @@ -0,0 +1,888 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package planguard + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "maps" + "net/http" + "net/url" + "os" + "regexp" + "slices" + "strconv" + "strings" + + "gopkg.in/yaml.v3" +) + +// StoreDiffSpec is a storediff case file: API endpoints walked page by page on two servers. +type StoreDiffSpec struct { + Cases []StoreCase `yaml:"cases"` +} + +// StoreCase is one endpoint walked at each page size. +// +// The whole JSON body of every page is compared. Each relaxation is a named class that must be listed in Allowed: +// - tie_order: within each run of items with equal Key, the new items are a permutation of the old ones (same +// count, keys equal position by position); +// - additive_keys: new objects may hold keys the old ones lack; each added path is listed. +// +// Any other difference fails. +type StoreCase struct { + ID string `yaml:"id"` + Path string `yaml:"path"` // /api/v1/...; {name} is replaced by Vars[name] + Query map[string]string `yaml:"query"` + WebLimit int `yaml:"web_limit"` // the client's page size; default 50 + Pages int `yaml:"pages"` // walk cap per limit; default 50 + Follow []Follow `yaml:"follow"` // param ← nextPageCursor | nextCursor | last. + Allowed []string `yaml:"allowed"` // tie_order (keyed by key) | additive_keys + Key []string `yaml:"key"` // item fields for tie_order + Vars map[string]StoreVar `yaml:"vars"` // path values read from another endpoint +} + +// Follow sets a query param of the next page from the old server's page. +type Follow struct { + Param string `yaml:"param"` + From string `yaml:"from"` +} + +// StoreVar reads one path value from an endpoint of both servers: From is first. or last. of its +// items. Both servers must give the same value. +type StoreVar struct { + Path string `yaml:"path"` + Query map[string]string `yaml:"query"` + From string `yaml:"from"` +} + +// StoreDiffReport is the outcome of RunStoreDiff: one result per case and page size. +type StoreDiffReport struct { + Cases []StoreCaseResult +} + +// StoreCaseResult is one case walked at one page size. +type StoreCaseResult struct { + ID string + Limit int + Pages int // pages compared + Verdict string // identical | allowed | different | skip + Classes []string // allowed classes that fired, as "name(count)" + Detail string // "page 3: $.hasMore: old true, new false" + Stuck string // "cursor did not advance at page N" +} + +// Store diff classes, defaults and limits. +const ( + ClassAdditiveKeys = "additive_keys" + + DefaultStorePages = 50 + DefaultStoreWebLimit = 50 + maxStoreBody = 64 << 20 +) + +var ( + storeIDRe = regexp.MustCompile(`^[a-z][a-z0-9_]*$`) + storeVarRe = regexp.MustCompile(`\{([a-z][a-z0-9_]*)\}`) + storeParamRe = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_]*$`) + storeFieldRe = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)*$`) + storeClassSet = map[string]bool{ClassTieOrder: true, ClassAdditiveKeys: true} +) + +// LoadStoreDiffSpec reads and validates a storediff case file. +func LoadStoreDiffSpec(path string) (*StoreDiffSpec, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + return ParseStoreDiffSpec(raw, path) +} + +// ParseStoreDiffSpec parses and validates a storediff case file named name. Unknown fields are errors. +func ParseStoreDiffSpec(raw []byte, name string) (*StoreDiffSpec, error) { + var spec StoreDiffSpec + dec := yaml.NewDecoder(bytes.NewReader(raw)) + dec.KnownFields(true) + if err := dec.Decode(&spec); err != nil { + if errors.Is(err, io.EOF) { + return nil, fmt.Errorf("%s: empty storediff spec", name) + } + return nil, fmt.Errorf("%s: %w", name, err) + } + var extra yaml.Node + if err := dec.Decode(&extra); !errors.Is(err, io.EOF) { + return nil, fmt.Errorf("%s: more than one YAML document", name) + } + if err := spec.Validate(); err != nil { + return nil, fmt.Errorf("%s: %w", name, err) + } + return &spec, nil +} + +// Validate checks every case. +func (s *StoreDiffSpec) Validate() error { + if len(s.Cases) == 0 { + return errors.New("no cases") + } + seen := map[string]bool{} + for i, c := range s.Cases { + if err := c.validate(); err != nil { + return fmt.Errorf("case %d (%s): %w", i+1, c.ID, err) + } + if seen[c.ID] { + return fmt.Errorf("case id %s is used twice", c.ID) + } + seen[c.ID] = true + } + return nil +} + +func validStorePath(p string) error { + if !strings.HasPrefix(p, "/api/v1/") || strings.ContainsAny(p, "?# ") { + return fmt.Errorf("path %q: want /api/v1/... without a query", p) + } + return nil +} + +func validFrom(from string, first bool) error { + switch { + case !first && (from == "nextCursor" || from == "nextPageCursor"): + return nil + case strings.HasPrefix(from, "last.") && storeFieldRe.MatchString(strings.TrimPrefix(from, "last.")): + return nil + case first && strings.HasPrefix(from, "first.") && storeFieldRe.MatchString(strings.TrimPrefix(from, "first.")): + return nil + } + if first { + return fmt.Errorf("from %q: want first. or last.", from) + } + return fmt.Errorf("from %q: want nextCursor, nextPageCursor or last.", from) +} + +func (c StoreCase) validate() error { + if !storeIDRe.MatchString(c.ID) { + return fmt.Errorf("id %q: want [a-z][a-z0-9_]*", c.ID) + } + if err := validStorePath(c.Path); err != nil { + return err + } + used := map[string]bool{} + for _, m := range storeVarRe.FindAllStringSubmatch(c.Path, -1) { + if _, ok := c.Vars[m[1]]; !ok { + return fmt.Errorf("path uses {%s}, but vars does not define it", m[1]) + } + used[m[1]] = true + } + if strings.ContainsAny(storeVarRe.ReplaceAllString(c.Path, ""), "{}") { + return fmt.Errorf("path %q: a {name} must be [a-z][a-z0-9_]*", c.Path) + } + for name, v := range c.Vars { + if !used[name] { + return fmt.Errorf("vars %s is not used in the path", name) + } + if err := validStorePath(v.Path); err != nil { + return fmt.Errorf("vars %s: %w", name, err) + } + if storeVarRe.MatchString(v.Path) { + return fmt.Errorf("vars %s: path %q must not use vars", name, v.Path) + } + if err := validFrom(v.From, true); err != nil { + return fmt.Errorf("vars %s: %w", name, err) + } + } + if c.WebLimit < 0 || c.Pages < 0 { + return errors.New("web_limit and pages must not be negative") + } + params := map[string]bool{"limit": true} + for _, f := range c.Follow { + if !storeParamRe.MatchString(f.Param) { + return fmt.Errorf("follow param %q is not a query param name", f.Param) + } + if params[f.Param] { + return fmt.Errorf("follow param %s is set twice (limit is set by the walk)", f.Param) + } + params[f.Param] = true + if err := validFrom(f.From, false); err != nil { + return fmt.Errorf("follow %s: %w", f.Param, err) + } + } + for k := range c.Query { + if params[k] { + return fmt.Errorf("query %s is set by the walk", k) + } + } + allowed := map[string]bool{} + for _, a := range c.Allowed { + if !storeClassSet[a] { + return fmt.Errorf("allowed %q: want tie_order or additive_keys", a) + } + allowed[a] = true + } + if allowed[ClassTieOrder] != (len(c.Key) > 0) { + return errors.New("tie_order needs key, and key is only for tie_order") + } + for _, k := range c.Key { + if !storeFieldRe.MatchString(k) { + return fmt.Errorf("key %q is not a field name", k) + } + } + return nil +} + +// RunStoreDiff walks every case of spec on the old and new server at each page size of limits ("web" is the case's +// web_limit) and compares the pages. +// +// Each page is a GET of base+path?query&limit=L on both servers. Both bodies are decoded with UseNumber and compared +// as whole JSON values, so any change in shape counts; the first differing JSON path is reported. A non-200 answer +// on either side must match the other's status and body, and ends the walk. The next page's params come from +// Follow, read from the old page. The walk stops when hasMore is false on both (a bare array body has more when it +// holds limit items), at Pages pages, when a follow value is absent from the (equal) pages, or when the follow +// values repeat (Stuck, a difference), and at the first different page. +// +// The error is a request that could not be made (a refused connection, a cancelled context) or a bad argument. +func RunStoreDiff(ctx context.Context, hc *http.Client, oldBase, newBase string, spec *StoreDiffSpec, limits []string) (*StoreDiffReport, error) { + if hc == nil { + hc = http.DefaultClient + } + if err := spec.Validate(); err != nil { + return nil, err + } + sizes, err := parseStoreLimits(limits) + if err != nil { + return nil, err + } + sd := &storeDiff{ctx: ctx, hc: hc, old: strings.TrimRight(oldBase, "/"), new: strings.TrimRight(newBase, "/")} + rep := &StoreDiffReport{} + for _, c := range spec.Cases { + path, skip, detail, err := sd.resolvePath(c) + if err != nil { + return rep, fmt.Errorf("case %s: %w", c.ID, err) + } + for _, size := range sizes { + limit := size + if limit == 0 { + limit = c.WebLimit + if limit == 0 { + limit = DefaultStoreWebLimit + } + } + res := StoreCaseResult{ID: c.ID, Limit: limit} + switch { + case skip: + res.Verdict, res.Detail = DiffSkip, detail + case detail != "": + res.Verdict, res.Detail = DiffDifferent, detail + default: + if err := sd.walk(c, path, limit, &res); err != nil { + return rep, fmt.Errorf("case %s, limit %d: %w", c.ID, limit, err) + } + } + rep.Cases = append(rep.Cases, res) + } + } + return rep, nil +} + +// parseStoreLimits reads page sizes: a positive integer, or web (0: the case's web_limit). +func parseStoreLimits(limits []string) ([]int, error) { + if len(limits) == 0 { + return []int{0}, nil + } + var out []int + for _, l := range limits { + l = strings.TrimSpace(l) + if l == "web" { + out = append(out, 0) + continue + } + n, err := strconv.Atoi(l) + if err != nil || n < 1 { + return nil, fmt.Errorf("page size %q: want an integer ≥ 1 or web", l) + } + out = append(out, n) + } + return out, nil +} + +type storeDiff struct { + ctx context.Context + hc *http.Client + old, new string +} + +// storePage is one server's answer. +type storePage struct { + status int + raw []byte + body any + json bool +} + +func (sd *storeDiff) get(base, path string, q url.Values) (storePage, error) { + u := base + path + if len(q) > 0 { + u += "?" + q.Encode() + } + req, err := http.NewRequestWithContext(sd.ctx, http.MethodGet, u, nil) + if err != nil { + return storePage{}, err + } + req.Header.Set("Accept", "application/json") + resp, err := sd.hc.Do(req) + if err != nil { + return storePage{}, err + } + defer resp.Body.Close() + raw, err := io.ReadAll(io.LimitReader(resp.Body, maxStoreBody+1)) + if err != nil { + return storePage{}, fmt.Errorf("GET %s: %w", u, err) + } + if len(raw) > maxStoreBody { + return storePage{}, fmt.Errorf("GET %s: body exceeds %d bytes", u, maxStoreBody) + } + p := storePage{status: resp.StatusCode, raw: raw} + p.body, p.json = decodeJSON(raw) + return p, nil +} + +// decodeJSON decodes exactly one JSON value with UseNumber. +func decodeJSON(raw []byte) (any, bool) { + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.UseNumber() + var v any + if err := dec.Decode(&v); err != nil { + return nil, false + } + if _, err := dec.Token(); !errors.Is(err, io.EOF) { + return nil, false + } + return v, true +} + +// samePage compares two non-200 answers (or bodies that are not JSON): equal status and body. +func samePage(o, n storePage) string { + if o.status != n.status { + return fmt.Sprintf("status: old %d, new %d", o.status, n.status) + } + if o.json && n.json { + if d := jsonDiff(o.body, n.body, "$", nil); d != nil { + return d.String() + } + return "" + } + if !bytes.Equal(o.raw, n.raw) { + return fmt.Sprintf("body: old %s, new %s", clip(string(o.raw)), clip(string(n.raw))) + } + return "" +} + +// resolvePath fills the path's {name}s from both servers. skip: a var found no value; detail: the servers disagree. +func (sd *storeDiff) resolvePath(c StoreCase) (path string, skip bool, detail string, err error) { + path = c.Path + for _, name := range slices.Sorted(maps.Keys(c.Vars)) { + v := c.Vars[name] + q := url.Values{} + for k, val := range v.Query { + q.Set(k, val) + } + var vals [2]string + for i, base := range []string{sd.old, sd.new} { + p, err := sd.get(base, v.Path, q) + if err != nil { + return "", false, "", fmt.Errorf("vars %s: %w", name, err) + } + side := [2]string{"old", "new"}[i] + if p.status != http.StatusOK || !p.json { + return "", false, fmt.Sprintf("vars %s: %s server answered %d", name, side, p.status), nil + } + val, ok, err := pickValue(p.body, v.From) + if err != nil { + return "", false, fmt.Sprintf("vars %s: %s server: %v", name, side, err), nil + } + if !ok { + return "", true, fmt.Sprintf("vars %s: %s %s found no value", name, v.Path, v.From), nil + } + vals[i] = val + } + if vals[0] != vals[1] { + return "", false, fmt.Sprintf("vars %s: old %s, new %s", name, vals[0], vals[1]), nil + } + path = strings.ReplaceAll(path, "{"+name+"}", url.PathEscape(vals[0])) + } + return path, false, "", nil +} + +// pageItems returns a page's items: the body when it is an array, else its items field. +func pageItems(body any) ([]any, bool) { + switch b := body.(type) { + case []any: + return b, true + case map[string]any: + items, ok := b["items"].([]any) + return items, ok + } + return nil, false +} + +// hasMore reads a page's hasMore; a bare array has more when it is full. +func hasMore(body any, limit int) bool { + switch b := body.(type) { + case []any: + return len(b) >= limit + case map[string]any: + more, _ := b["hasMore"].(bool) + return more + } + return false +} + +// pickValue reads first., last., nextCursor or nextPageCursor from a page. ok is false when there is +// no such item or the value is null or absent. +func pickValue(body any, from string) (string, bool, error) { + var v any + switch { + case strings.HasPrefix(from, "first."), strings.HasPrefix(from, "last."): + items, ok := pageItems(body) + if !ok { + return "", false, errors.New("the page has no items") + } + if len(items) == 0 { + return "", false, nil + } + item := items[0] + field := strings.TrimPrefix(from, "first.") + if strings.HasPrefix(from, "last.") { + item, field = items[len(items)-1], strings.TrimPrefix(from, "last.") + } + v = item + for _, part := range strings.Split(field, ".") { + m, ok := v.(map[string]any) + if !ok { + return "", false, fmt.Errorf("%s: not an object", from) + } + v = m[part] + } + default: + m, ok := body.(map[string]any) + if !ok { + return "", false, fmt.Errorf("%s: the page is not an object", from) + } + v = m[from] + } + switch x := v.(type) { + case nil: + return "", false, nil + case json.Number: + return x.String(), true, nil + case string: + return x, true, nil + case bool: + return strconv.FormatBool(x), true, nil + } + return "", false, fmt.Errorf("%s is not a scalar", from) +} + +func (sd *storeDiff) walk(c StoreCase, path string, limit int, res *StoreCaseResult) error { + maxPages := c.Pages + if maxPages == 0 { + maxPages = DefaultStorePages + } + allowed := map[string]bool{} + for _, a := range c.Allowed { + allowed[a] = true + } + counts := map[string]int{} + var notes []string + res.Verdict = DiffIdentical + seen := map[string]bool{} + var follow []string + for page := 1; ; page++ { + q := url.Values{} + for k, v := range c.Query { + q.Set(k, v) + } + q.Set("limit", strconv.Itoa(limit)) + for i, f := range c.Follow { + if follow != nil { + q.Set(f.Param, follow[i]) + } + } + o, err := sd.get(sd.old, path, q) + if err != nil { + return fmt.Errorf("page %d: old: %w", page, err) + } + n, err := sd.get(sd.new, path, q) + if err != nil { + return fmt.Errorf("page %d: new: %w", page, err) + } + res.Pages = page + if o.status != http.StatusOK || n.status != http.StatusOK || !o.json || !n.json { + if d := samePage(o, n); d != "" { + res.Verdict, res.Detail = DiffDifferent, fmt.Sprintf("page %d: %s", page, d) + return nil + } + if page > 1 { + // The walk reached this page through a cursor, so an error here is not an end both builds share. + res.Verdict, res.Detail = DiffDifferent, fmt.Sprintf("page %d: both answered %d after a cursor", page, o.status) + return nil + } + notes = append(notes, fmt.Sprintf("page %d: both answered %d", page, o.status)) + break + } + nb := n.body + if allowed[ClassTieOrder] { + var moved int + var bad string + nb, moved, bad = alignTies(o.body, nb, c.Key, allowed[ClassAdditiveKeys]) + if bad != "" { + res.Verdict, res.Detail = DiffDifferent, fmt.Sprintf("page %d: tie_order: %s", page, bad) + return nil + } + if moved > 0 { + counts[ClassTieOrder] += moved + notes = append(notes, fmt.Sprintf("page %d: tie_order: %d items hold another position within their run of equal %s", page, moved, strings.Join(c.Key, ", "))) + } + } + var added []string + var addedPtr *[]string + if allowed[ClassAdditiveKeys] { + addedPtr = &added + } + if d := jsonDiff(o.body, nb, "$", addedPtr); d != nil { + res.Verdict, res.Detail = DiffDifferent, fmt.Sprintf("page %d: %s", page, d) + return nil + } + if len(added) > 0 { + counts[ClassAdditiveKeys] += len(added) + if len(added) > 5 { + added = append(added[:5], fmt.Sprintf("and %d more", len(added)-5)) + } + notes = append(notes, fmt.Sprintf("page %d: additive_keys: %s", page, strings.Join(added, ", "))) + } + if !hasMore(o.body, limit) { + break + } + if page >= maxPages { + notes = append(notes, fmt.Sprintf("stopped at the %d-page cap", maxPages)) + break + } + if len(c.Follow) == 0 { + notes = append(notes, "hasMore, but the case follows no cursor") + break + } + next := make([]string, len(c.Follow)) + missing := "" + for i, f := range c.Follow { + v, ok, err := pickValue(o.body, f.From) + if err != nil { + res.Verdict, res.Detail = DiffDifferent, fmt.Sprintf("page %d: follow %s ← %s: %v", page, f.Param, f.From, err) + return nil + } + // The walk follows one cursor for both builds, so each build's own page must give the same one. A + // relaxed page (tie_order) can end on another row; following old's cursor would then never show the + // rows new skips or repeats. + nv, nok, nerr := pickValue(n.body, f.From) + if nerr != nil || nok != ok || nv != v { + res.Verdict, res.Detail = DiffDifferent, fmt.Sprintf("page %d: cursor differs after tie_order: %s ← %s old %s, new %s", + page, f.Param, f.From, cursorText(v, ok, nil), cursorText(nv, nok, nerr)) + return nil + } + if !ok && missing == "" { + missing = fmt.Sprintf("%s ← %s", f.Param, f.From) + } + next[i] = v + } + if missing != "" { + // Both pages are equal, so neither server gives a cursor: a client stops here too. + notes = append(notes, fmt.Sprintf("page %d: hasMore, but %s is absent on both; the walk stops", page, missing)) + break + } + k := strings.Join(next, "\x00") + if seen[k] { + res.Stuck = fmt.Sprintf("cursor did not advance at page %d", page) + res.Verdict, res.Detail = DiffDifferent, fmt.Sprintf("page %d: %s", page, res.Stuck) + return nil + } + seen[k] = true + follow = next + } + for _, cl := range []string{ClassTieOrder, ClassAdditiveKeys} { + if n := counts[cl]; n > 0 { + res.Classes = append(res.Classes, fmt.Sprintf("%s(%d)", cl, n)) + } + } + if len(res.Classes) > 0 { + res.Verdict = DiffAllowed + } + res.Detail = strings.Join(notes, "; ") + return nil +} + +func cursorText(v string, ok bool, err error) string { + switch { + case err != nil: + return err.Error() + case !ok: + return absent + } + return v +} + +// alignTies returns newBody with its items reordered to old's order within runs of equal key, when the items +// differ only that way, and the number of items that moved. Otherwise it returns newBody unchanged. bad names an +// item that lacks a key field: without it every such item would share one run and any order would pass. +func alignTies(oldBody, newBody any, key []string, additive bool) (out any, moved int, bad string) { + oi, ok1 := pageItems(oldBody) + ni, ok2 := pageItems(newBody) + if !ok1 || !ok2 { + return newBody, 0, "" + } + keyOf := func(item any) (string, string) { + parts := make([]string, len(key)) + for i, k := range key { + v := item + for _, p := range strings.Split(k, ".") { + m, ok := v.(map[string]any) + if !ok { + return "", k + } + if v, ok = m[p]; !ok { + return "", k + } + } + b, _ := json.Marshal(v) + parts[i] = string(b) + } + return strings.Join(parts, "\x00"), "" + } + oldKeys, newKeys := make([]string, len(oi)), make([]string, len(ni)) + for side, items := range [][]any{oi, ni} { + for i, item := range items { + k, missing := keyOf(item) + if missing != "" { + return newBody, 0, fmt.Sprintf("key %s is absent from the %s page's $.items[%d]", missing, [2]string{"old", "new"}[side], i) + } + [2][]string{oldKeys, newKeys}[side][i] = k + } + } + if len(oi) != len(ni) || !slices.Equal(oldKeys, newKeys) { + return newBody, 0, "" + } + equal := func(a, b any) bool { + var added []string + var p *[]string + if additive { + p = &added + } + return jsonDiff(a, b, "$", p) == nil + } + items := make([]any, len(ni)) + for start := 0; start < len(oi); { + end := start + 1 + for end < len(oi) && oldKeys[end] == oldKeys[start] { + end++ + } + used := make([]bool, end-start) + for i := start; i < end; i++ { + pick := -1 + if !used[i-start] && equal(oi[i], ni[i]) { + pick = i + } else { + for j := start; j < end; j++ { + if !used[j-start] && equal(oi[i], ni[j]) { + pick = j + break + } + } + } + if pick < 0 { + return newBody, 0, "" + } + if pick != i { + moved++ + } + used[pick-start] = true + items[i] = ni[pick] + } + start = end + } + if moved == 0 { + return newBody, 0, "" + } + if _, isArray := newBody.([]any); isArray { + return items, moved, "" + } + m := map[string]any{} + for k, v := range newBody.(map[string]any) { + m[k] = v + } + m["items"] = items + return m, moved, "" +} + +// jsonPathDiff is the first difference between two JSON values. +type jsonPathDiff struct { + path string + old, new string +} + +func (d *jsonPathDiff) String() string { + return fmt.Sprintf("%s: old %s, new %s", d.path, d.old, d.new) +} + +const absent = "(absent)" + +// jsonDiff returns the first difference between a and b, walking object keys in sorted order and arrays by index. +// Numbers compare by their JSON text. With added non-nil, keys only b holds are appended to it instead of being a +// difference. +func jsonDiff(a, b any, path string, added *[]string) *jsonPathDiff { + switch av := a.(type) { + case map[string]any: + bv, ok := b.(map[string]any) + if !ok { + break + } + keys := slices.Collect(maps.Keys(av)) + for k := range bv { + if _, ok := av[k]; !ok { + keys = append(keys, k) + } + } + slices.Sort(keys) + for _, k := range keys { + p := path + jsonKey(k) + x, inA := av[k] + y, inB := bv[k] + switch { + case !inA && added != nil: + *added = append(*added, p) + case !inA: + return &jsonPathDiff{p, absent, render(y)} + case !inB: + return &jsonPathDiff{p, render(x), absent} + default: + if d := jsonDiff(x, y, p, added); d != nil { + return d + } + } + } + return nil + case []any: + bv, ok := b.([]any) + if !ok { + break + } + for i := 0; i < max(len(av), len(bv)); i++ { + p := fmt.Sprintf("%s[%d]", path, i) + switch { + case i >= len(av): + return &jsonPathDiff{p, absent, render(bv[i])} + case i >= len(bv): + return &jsonPathDiff{p, render(av[i]), absent} + } + if d := jsonDiff(av[i], bv[i], p, added); d != nil { + return d + } + } + return nil + case json.Number: + if bv, ok := b.(json.Number); ok && av == bv { + return nil + } + case string: + if bv, ok := b.(string); ok && av == bv { + return nil + } + case bool: + if bv, ok := b.(bool); ok && av == bv { + return nil + } + case nil: + if b == nil { + return nil + } + } + return &jsonPathDiff{path, render(a), render(b)} +} + +func jsonKey(k string) string { + if storeParamRe.MatchString(k) { + return "." + k + } + b, _ := json.Marshal(k) + return "[" + string(b) + "]" +} + +func render(v any) string { + b, err := json.Marshal(v) + if err != nil { + return fmt.Sprintf("%v", v) + } + return clip(string(b)) +} + +func clip(s string) string { + const n = 200 + if len(s) > n { + return s[:n] + "…" + } + return s +} + +// Failed returns the results that are different. +func (r *StoreDiffReport) Failed() []StoreCaseResult { + var out []StoreCaseResult + for _, c := range r.Cases { + if c.Verdict == DiffDifferent { + out = append(out, c) + } + } + return out +} + +// Skipped returns the results that did not run: a path var found no value. They prove nothing. +func (r *StoreDiffReport) Skipped() []StoreCaseResult { + var out []StoreCaseResult + for _, c := range r.Cases { + if c.Verdict == DiffSkip { + out = append(out, c) + } + } + return out +} + +// Line is a one-line summary of a result. +func (c StoreCaseResult) Line() string { + mark := "ok" + switch c.Verdict { + case DiffDifferent: + mark = "FAIL" + case DiffSkip: + mark = "SKIP" + } + s := fmt.Sprintf("%-4s %s limit %d: %s, %d pages", mark, c.ID, c.Limit, c.Verdict, c.Pages) + if len(c.Classes) > 0 { + s += " " + strings.Join(c.Classes, " ") + } + if c.Detail != "" { + s += "; " + c.Detail + } + return s +} + +// Markdown renders the report. +func (r *StoreDiffReport) Markdown() string { + var b strings.Builder + b.WriteString("# storediff report\n\n| case | limit | pages | verdict | classes | detail |\n|---|---|---|---|---|---|\n") + esc := func(s string) string { return strings.ReplaceAll(strings.ReplaceAll(s, "|", `\|`), "\n", " ") } + for _, c := range r.Cases { + fmt.Fprintf(&b, "| %s | %d | %d | %s | %s | %s |\n", c.ID, c.Limit, c.Pages, c.Verdict, strings.Join(c.Classes, " "), esc(c.Detail)) + } + fmt.Fprintf(&b, "\n%d results, %d different, %d skipped\n", len(r.Cases), len(r.Failed()), len(r.Skipped())) + return b.String() +} diff --git a/internal/planguard/storediff_test.go b/internal/planguard/storediff_test.go new file mode 100644 index 0000000..477fa0a --- /dev/null +++ b/internal/planguard/storediff_test.go @@ -0,0 +1,431 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package planguard_test + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "regexp" + "strconv" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/planguard" +) + +// pageServer serves 7 items in pages: limit param, cursor ← nextCursor. mutate may change page n's body; status +// may replace page n's status. +func pageServer(t *testing.T, mutate func(page int, body map[string]any), status func(page int) int) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet || r.URL.Path != "/api/v1/things" { + http.NotFound(w, r) + return + } + limit, err := strconv.Atoi(r.URL.Query().Get("limit")) + if err != nil || limit < 1 { + http.Error(w, `{"error":"limit"}`, http.StatusBadRequest) + return + } + cursor, _ := strconv.Atoi(r.URL.Query().Get("cursor")) + page := cursor/limit + 1 + items := []any{} + for id := cursor + 1; id <= 7 && len(items) < limit; id++ { + items = append(items, map[string]any{"id": id, "name": "n" + strconv.Itoa(id), "group": (id + 1) / 2}) + } + last := cursor + len(items) + body := map[string]any{"items": items, "hasMore": last < 7} + if last < 7 { + body["nextCursor"] = last + } + if mutate != nil { + mutate(page, body) + } + code := http.StatusOK + if status != nil { + code = status(page) + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(code) + _ = json.NewEncoder(w).Encode(body) + })) + t.Cleanup(srv.Close) + return srv +} + +func thingsSpec(allowed []string, key []string) *planguard.StoreDiffSpec { + return &planguard.StoreDiffSpec{Cases: []planguard.StoreCase{{ + ID: "things", Path: "/api/v1/things", WebLimit: 3, + Follow: []planguard.Follow{{Param: "cursor", From: "nextCursor"}}, + Allowed: allowed, Key: key, + }}} +} + +func runStoreDiff(t *testing.T, oldSrv, newSrv *httptest.Server, spec *planguard.StoreDiffSpec, limits ...string) *planguard.StoreDiffReport { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + rep, err := planguard.RunStoreDiff(ctx, oldSrv.Client(), oldSrv.URL, newSrv.URL, spec, limits) + if err != nil { + t.Fatal(err) + } + return rep +} + +func TestStoreDiffComparesPageJSON(t *testing.T) { + old := pageServer(t, nil, nil) + + t.Run("identical", func(t *testing.T) { + rep := runStoreDiff(t, old, pageServer(t, nil, nil), thingsSpec(nil, nil), "1", "2", "3", "web") + wantPages := map[int]int{1: 7, 2: 4, 3: 3} + if len(rep.Cases) != 4 { + t.Fatalf("results = %+v, want 4", rep.Cases) + } + for i, c := range rep.Cases { + if c.Verdict != planguard.DiffIdentical { + t.Errorf("limit %d: %s (%s), want identical", c.Limit, c.Verdict, c.Detail) + } + if i == 3 && c.Limit != 3 { + t.Errorf("web limit = %d, want the case's web_limit 3", c.Limit) + } + if c.Pages != wantPages[c.Limit] { + t.Errorf("limit %d: %d pages, want %d", c.Limit, c.Pages, wantPages[c.Limit]) + } + } + if len(rep.Failed()) != 0 || !strings.Contains(rep.Markdown(), "| things | 1 | 7 | identical |") { + t.Errorf("report:\n%s", rep.Markdown()) + } + }) + + for _, tc := range []struct { + name string + mutate func(page int, body map[string]any) + status func(page int) int + detail string + }{ + {"hasMore flips on page 3", func(page int, body map[string]any) { + if page == 3 { + body["hasMore"] = false + } + }, nil, "page 3: $.hasMore: old true, new false"}, + {"item name changes on page 2", func(page int, body map[string]any) { + if page == 2 { + body["items"].([]any)[0].(map[string]any)["name"] = "renamed" + } + }, nil, `page 2: $.items[0].name: old "n2", new "renamed"`}, + {"a field is added", func(page int, body map[string]any) { + body["items"].([]any)[0].(map[string]any)["extra"] = 1 + }, nil, "page 1: $.items[0].extra: old (absent), new 1"}, + {"a field is removed", func(page int, body map[string]any) { + delete(body, "nextCursor") + }, nil, "page 1: $.nextCursor: old 1, new (absent)"}, + {"a number changes type", func(page int, body map[string]any) { + if page == 4 { + body["nextCursor"] = "4" + } + }, nil, `page 4: $.nextCursor: old 4, new "4"`}, + {"new answers 500", nil, func(page int) int { + if page == 2 { + return http.StatusInternalServerError + } + return http.StatusOK + }, "page 2: status: old 200, new 500"}, + } { + t.Run(tc.name, func(t *testing.T) { + rep := runStoreDiff(t, old, pageServer(t, tc.mutate, tc.status), thingsSpec(nil, nil), "1") + c := rep.Cases[0] + if c.Verdict != planguard.DiffDifferent || c.Detail != tc.detail { + t.Errorf("got %s %q, want different %q", c.Verdict, c.Detail, tc.detail) + } + if len(rep.Failed()) != 1 { + t.Errorf("Failed() = %d, want 1", len(rep.Failed())) + } + }) + } + + t.Run("equal non-200 answers after a cursor are a difference", func(t *testing.T) { + status := func(page int) int { + if page == 2 { + return http.StatusServiceUnavailable + } + return http.StatusOK + } + rep := runStoreDiff(t, pageServer(t, nil, status), pageServer(t, nil, status), thingsSpec(nil, nil), "1") + if c := rep.Cases[0]; c.Verdict != planguard.DiffDifferent || c.Pages != 2 || c.Detail != "page 2: both answered 503 after a cursor" { + t.Errorf("got %+v, want different at page 2", c) + } + // On page 1 equal errors are an end both builds share. + first := func(int) int { return http.StatusServiceUnavailable } + rep = runStoreDiff(t, pageServer(t, nil, first), pageServer(t, nil, first), thingsSpec(nil, nil), "1") + if c := rep.Cases[0]; c.Verdict != planguard.DiffIdentical || c.Pages != 1 { + t.Errorf("page 1: got %+v, want identical over 1 page", c) + } + }) + + t.Run("tie_order needs every key field", func(t *testing.T) { + rep := runStoreDiff(t, old, pageServer(t, nil, nil), thingsSpec([]string{planguard.ClassTieOrder}, []string{"rank"}), "3") + if c := rep.Cases[0]; c.Verdict != planguard.DiffDifferent || c.Detail != "page 1: tie_order: key rank is absent from the old page's $.items[0]" { + t.Errorf("got %+v, want different naming the absent key", c) + } + }) + + t.Run("additive_keys allows a declared added field", func(t *testing.T) { + add := func(page int, body map[string]any) { body["items"].([]any)[0].(map[string]any)["extra"] = 1 } + rep := runStoreDiff(t, old, pageServer(t, add, nil), thingsSpec([]string{planguard.ClassAdditiveKeys}, nil), "3") + c := rep.Cases[0] + if c.Verdict != planguard.DiffAllowed || len(c.Classes) != 1 || c.Classes[0] != "additive_keys(3)" || + !strings.Contains(c.Detail, "$.items[0].extra") { + t.Errorf("got %+v, want allowed additive_keys(3)", c) + } + // A removed field is still a difference. + drop := func(page int, body map[string]any) { delete(body["items"].([]any)[0].(map[string]any), "name") } + rep = runStoreDiff(t, old, pageServer(t, drop, nil), thingsSpec([]string{planguard.ClassAdditiveKeys}, nil), "3") + if c := rep.Cases[0]; c.Verdict != planguard.DiffDifferent { + t.Errorf("removed field: got %+v, want different", c) + } + }) + + t.Run("tie_order allows a reorder within equal keys only", func(t *testing.T) { + swap := func(i, j int) func(page int, body map[string]any) { + return func(page int, body map[string]any) { + if items := body["items"].([]any); page == 1 && len(items) > j { + items[i], items[j] = items[j], items[i] + } + } + } + spec := thingsSpec([]string{planguard.ClassTieOrder}, []string{"group"}) + // Items 1 and 2 share group 1. + rep := runStoreDiff(t, old, pageServer(t, swap(0, 1), nil), spec, "3") + if c := rep.Cases[0]; c.Verdict != planguard.DiffAllowed || len(c.Classes) != 1 || c.Classes[0] != "tie_order(2)" { + t.Errorf("tie swap: got %+v, want allowed tie_order(2)", c) + } + // Items 2 and 3 are in different groups. + rep = runStoreDiff(t, old, pageServer(t, swap(1, 2), nil), spec, "3") + if c := rep.Cases[0]; c.Verdict != planguard.DiffDifferent || !strings.HasPrefix(c.Detail, "page 1: $.items[1]") { + t.Errorf("cross-key swap: got %+v, want different at $.items[1]", c) + } + // Undeclared, the tie swap is a difference. + rep = runStoreDiff(t, old, pageServer(t, swap(0, 1), nil), thingsSpec(nil, nil), "3") + if c := rep.Cases[0]; c.Verdict != planguard.DiffDifferent { + t.Errorf("undeclared tie swap: got %+v, want different", c) + } + }) +} + +func TestStoreDiffStopsOnStuckCursor(t *testing.T) { + if d, ok := t.Deadline(); ok && time.Until(d) < 10*time.Second { + t.Skip("too close to the test deadline") + } + stuck := func(page int, body map[string]any) { + body["hasMore"] = true + body["nextCursor"] = 1 + } + old, newSrv := pageServer(t, stuck, nil), pageServer(t, stuck, nil) + spec := thingsSpec(nil, nil) + spec.Cases[0].Pages = 1000 + done := make(chan *planguard.StoreDiffReport, 1) + go func() { done <- runStoreDiff(t, old, newSrv, spec, "1") }() + select { + case rep := <-done: + c := rep.Cases[0] + if c.Verdict != planguard.DiffDifferent || c.Stuck != "cursor did not advance at page 2" || c.Pages != 2 { + t.Errorf("got %+v, want different, stuck at page 2", c) + } + case <-time.After(10 * time.Second): + t.Fatal("the walk did not stop") + } +} + +func TestStoreDiffSpecValidation(t *testing.T) { + for _, tc := range []struct{ yaml, err string }{ + {"cases: []\n", "no cases"}, + {"cases:\n- id: a\n path: /x\n", `path "/x"`}, + {"cases:\n- id: a\n path: /api/v1/x?limit=1\n", "without a query"}, + {"cases:\n- id: a\n path: /api/v1/x\n bogus: 1\n", "field bogus not found"}, + {"cases:\n- id: a\n path: /api/v1/x\n query: {limit: \"5\"}\n", "query limit is set by the walk"}, + {"cases:\n- id: a\n path: /api/v1/x\n follow: [{param: cursor, from: prevCursor}]\n", "want nextCursor, nextPageCursor or last."}, + {"cases:\n- id: a\n path: /api/v1/x\n allowed: [tie_order]\n", "tie_order needs key"}, + {"cases:\n- id: a\n path: /api/v1/x\n allowed: [saturated_suffix]\n", "want tie_order or additive_keys"}, + {"cases:\n- id: a\n path: /api/v1/x/{id}\n", "vars does not define it"}, + {"cases:\n- id: a\n path: /api/v1/x\n- id: a\n path: /api/v1/y\n", "case id a is used twice"}, + } { + if _, err := planguard.ParseStoreDiffSpec([]byte(tc.yaml), "t.yaml"); err == nil || !strings.Contains(err.Error(), tc.err) { + t.Errorf("%q: error %v, want %q", tc.yaml, err, tc.err) + } + } +} + +// A path var is read from both servers; a missing value skips the case. +func TestStoreDiffPathVars(t *testing.T) { + srv := func(ids ...int) *httptest.Server { + s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch { + case r.URL.Path == "/api/v1/channels": + items := []any{} + for _, id := range ids { + items = append(items, map[string]any{"id": id}) + } + _ = json.NewEncoder(w).Encode(map[string]any{"items": items, "hasMore": false}) + case strings.HasPrefix(r.URL.Path, "/api/v1/channels/"): + _ = json.NewEncoder(w).Encode(map[string]any{"items": []any{r.URL.Path}, "hasMore": false}) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(s.Close) + return s + } + spec := &planguard.StoreDiffSpec{Cases: []planguard.StoreCase{{ + ID: "channel_messages", Path: "/api/v1/channels/{channel}/messages", + Vars: map[string]planguard.StoreVar{"channel": {Path: "/api/v1/channels", Query: map[string]string{"limit": "1"}, From: "first.id"}}, + }}} + if c := runStoreDiff(t, srv(12), srv(12), spec, "2").Cases[0]; c.Verdict != planguard.DiffIdentical { + t.Errorf("same var: %+v", c) + } + if c := runStoreDiff(t, srv(12), srv(13), spec, "2").Cases[0]; c.Verdict != planguard.DiffDifferent || c.Detail != "vars channel: old 12, new 13" { + t.Errorf("different var: %+v", c) + } + rep := runStoreDiff(t, srv(), srv(), spec, "2") + if c := rep.Cases[0]; c.Verdict != planguard.DiffSkip || len(rep.Skipped()) != 1 { + t.Errorf("no value: %+v", c) + } +} + +// The repo's storediff case files load, and every query, follow and var param is a query param of its handler in +// the generated Swagger. +func TestRepoStoreDiffCasesLoad(t *testing.T) { + spec, err := planguard.LoadStoreDiffSpec("../../db/planguard/storediff/smoke.yaml") + if err != nil { + t.Fatal(err) + } + if len(spec.Cases) < 7 { + t.Errorf("smoke.yaml has %d cases, want at least 7", len(spec.Cases)) + } + raw, err := os.ReadFile("../../docs/swagger.json") + if err != nil { + t.Fatal(err) + } + var doc struct { + BasePath string `json:"basePath"` + Paths map[string]struct { + Get *struct { + Parameters []struct{ Name, In string } `json:"parameters"` + } `json:"get"` + } `json:"paths"` + } + if err := json.Unmarshal(raw, &doc); err != nil { + t.Fatal(err) + } + placeholder := regexp.MustCompile(`\{[^}]+\}`) + params := func(path string) map[string]bool { + want := placeholder.ReplaceAllString(strings.TrimPrefix(path, doc.BasePath), "{}") + for p, item := range doc.Paths { + if placeholder.ReplaceAllString(p, "{}") == want && item.Get != nil { + out := map[string]bool{} + for _, prm := range item.Get.Parameters { + if prm.In == "query" { + out[prm.Name] = true + } + } + return out + } + } + t.Errorf("%s: no GET in docs/swagger.json", path) + return nil + } + for _, c := range spec.Cases { + names := []string{"limit"} + for k := range c.Query { + names = append(names, k) + } + for _, f := range c.Follow { + names = append(names, f.Param) + } + known := params(c.Path) + for _, n := range names { + if known != nil && !known[n] { + t.Errorf("%s: %s is not a query param of GET %s", c.ID, n, c.Path) + } + } + for name, v := range c.Vars { + known := params(v.Path) + for k := range v.Query { + if known != nil && !known[k] { + t.Errorf("%s: vars %s: %s is not a query param of GET %s", c.ID, name, k, v.Path) + } + } + } + } +} + +// hasMore without a cursor on both servers ends the walk with a note; the pages still compare. +func TestStoreDiffStopsWithoutCursor(t *testing.T) { + drop := func(page int, body map[string]any) { + if page == 2 { + delete(body, "nextCursor") + } + } + rep := runStoreDiff(t, pageServer(t, drop, nil), pageServer(t, drop, nil), thingsSpec(nil, nil), "1") + if c := rep.Cases[0]; c.Verdict != planguard.DiffIdentical || c.Pages != 2 || + c.Detail != "page 2: hasMore, but cursor ← nextCursor is absent on both; the walk stops" { + t.Errorf("got %+v", c) + } +} + +// descServer serves ids 7..1 newest first as a bare array: cursor ← last.id returns the ids below it. With +// swapAt, page 1 swaps the two items it names (a tie when they share a group); at the page end, that build's own +// cursor then names another row than the old build's. +func descServer(t *testing.T, swapAt func(items []any) (int, int)) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + limit, _ := strconv.Atoi(r.URL.Query().Get("limit")) + cursor, err := strconv.Atoi(r.URL.Query().Get("cursor")) + if err != nil { + cursor = 8 + } + items := []any{} + for id := cursor - 1; id >= 1 && len(items) < limit; id-- { + items = append(items, map[string]any{"id": id, "group": (id + 1) / 2}) + } + if swapAt != nil && cursor == 8 { + if i, j := swapAt(items); i >= 0 { + items[i], items[j] = items[j], items[i] + } + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(items) + })) + t.Cleanup(srv.Close) + return srv +} + +// tie_order relaxes the order inside a page, but each build's own page must still give the walk's cursor. A +// build whose tie order moves another row to the end of the page would repeat (or skip) rows on its own next page, +// which following the old cursor would never show. +func TestStoreDiffTieOrderChecksEachBuildsCursor(t *testing.T) { + spec := &planguard.StoreDiffSpec{Cases: []planguard.StoreCase{{ + ID: "desc", Path: "/api/v1/things", + Follow: []planguard.Follow{{Param: "cursor", From: "last.id"}}, + Allowed: []string{planguard.ClassTieOrder}, Key: []string{"group"}, + }}} + old := descServer(t, nil) + // limit 3: page 1 is 7, 6, 5; 6 and 5 share group 3. new ends the page on 6, so its own cursor repeats 5. + atEnd := func(items []any) (int, int) { return len(items) - 2, len(items) - 1 } + rep := runStoreDiff(t, old, descServer(t, atEnd), spec, "3") + if c := rep.Cases[0]; c.Verdict != planguard.DiffDifferent || + c.Detail != "page 1: cursor differs after tie_order: cursor ← last.id old 5, new 6" { + t.Errorf("tie at the page end: got %+v", c) + } + // limit 4: page 1 is 7, 6, 5, 4; the 6/5 tie swaps inside the page, so both builds end on 4. + inside := func(items []any) (int, int) { return 1, 2 } + rep = runStoreDiff(t, old, descServer(t, inside), spec, "4") + if c := rep.Cases[0]; c.Verdict != planguard.DiffAllowed || c.Pages != 2 { + t.Errorf("tie inside the page: got %+v, want allowed over 2 pages", c) + } +} diff --git a/internal/planguard/testdata/corpus/allow_sort_above_limit.yaml b/internal/planguard/testdata/corpus/allow_sort_above_limit.yaml new file mode 100644 index 0000000..67007e8 --- /dev/null +++ b/internal/planguard/testdata/corpus/allow_sort_above_limit.yaml @@ -0,0 +1,10 @@ +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +statements: + Q: {sqlc: Q, plan: custom} +entries: + - id: a + stmt: Q + expect: + allow_sort_above_limit: an old escape hatch diff --git a/internal/planguard/testdata/corpus/bad_cached_no_admission.yaml b/internal/planguard/testdata/corpus/bad_cached_no_admission.yaml new file mode 100644 index 0000000..8c839fa --- /dev/null +++ b/internal/planguard/testdata/corpus/bad_cached_no_admission.yaml @@ -0,0 +1,7 @@ +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +statements: + G: {sqlc: Q, plan: cached} +entries: + - {id: a, stmt: G} diff --git a/internal/planguard/testdata/corpus/bad_cached_unknown_profile.yaml b/internal/planguard/testdata/corpus/bad_cached_unknown_profile.yaml new file mode 100644 index 0000000..17e9802 --- /dev/null +++ b/internal/planguard/testdata/corpus/bad_cached_unknown_profile.yaml @@ -0,0 +1,9 @@ +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +admission_profiles: + by_id: [hit, miss] +statements: + G: {sqlc: Q, plan: cached, admission: region_page} +entries: + - {id: a, stmt: G} diff --git a/internal/planguard/testdata/corpus/bad_default_sort_exempt.yaml b/internal/planguard/testdata/corpus/bad_default_sort_exempt.yaml new file mode 100644 index 0000000..cdb806b --- /dev/null +++ b/internal/planguard/testdata/corpus/bad_default_sort_exempt.yaml @@ -0,0 +1,9 @@ +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +statements: + Q: {sqlc: Q, plan: custom} +defaults: + sort_exempt: [no_default_set] +entries: + - {id: a, stmt: Q} diff --git a/internal/planguard/testdata/corpus/bad_derived_edited.yaml b/internal/planguard/testdata/corpus/bad_derived_edited.yaml new file mode 100644 index 0000000..c5dcc20 --- /dev/null +++ b/internal/planguard/testdata/corpus/bad_derived_edited.yaml @@ -0,0 +1,10 @@ +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +statements: + DeleteExpiredRoutesSelect: + file: sql/derived/DeleteExpiredRoutesSelectEdited.sql + derived_from: DeleteExpiredRoutes + plan: maint +entries: + - {id: expired.select, stmt: DeleteExpiredRoutesSelect} diff --git a/internal/planguard/testdata/corpus/bad_duplicate_id.yaml b/internal/planguard/testdata/corpus/bad_duplicate_id.yaml new file mode 100644 index 0000000..5dea2d1 --- /dev/null +++ b/internal/planguard/testdata/corpus/bad_duplicate_id.yaml @@ -0,0 +1,8 @@ +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +statements: + Q: {sqlc: Q, plan: custom} +entries: + - {id: a, stmt: Q} + - {id: a, stmt: Q} diff --git a/internal/planguard/testdata/corpus/bad_entry_id.yaml b/internal/planguard/testdata/corpus/bad_entry_id.yaml new file mode 100644 index 0000000..f1cec05 --- /dev/null +++ b/internal/planguard/testdata/corpus/bad_entry_id.yaml @@ -0,0 +1,5 @@ +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +entries: + - {id: "../escape", stmt: X} diff --git a/internal/planguard/testdata/corpus/bad_index_cond_digit.yaml b/internal/planguard/testdata/corpus/bad_index_cond_digit.yaml new file mode 100644 index 0000000..7bfab3d --- /dev/null +++ b/internal/planguard/testdata/corpus/bad_index_cond_digit.yaml @@ -0,0 +1,12 @@ +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +statements: + Q: {sqlc: Q, plan: custom} +entries: + - id: a + stmt: Q + expect: + driver: + - relation: known_routes + index_cond: ["hop_count = 3"] diff --git a/internal/planguard/testdata/corpus/bad_index_cond_param.yaml b/internal/planguard/testdata/corpus/bad_index_cond_param.yaml new file mode 100644 index 0000000..57c6800 --- /dev/null +++ b/internal/planguard/testdata/corpus/bad_index_cond_param.yaml @@ -0,0 +1,13 @@ +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +statements: + Q: {sqlc: Q, plan: custom} +entries: + - id: a + stmt: Q + expect: + driver: + - relation: nodes + index: idx_v2_nodes + index_cond: ["public_key = $1"] diff --git a/internal/planguard/testdata/corpus/bad_index_cond_quote.yaml b/internal/planguard/testdata/corpus/bad_index_cond_quote.yaml new file mode 100644 index 0000000..f31404a --- /dev/null +++ b/internal/planguard/testdata/corpus/bad_index_cond_quote.yaml @@ -0,0 +1,12 @@ +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +statements: + Q: {sqlc: Q, plan: custom} +entries: + - id: a + stmt: Q + expect: + driver: + - relation: nodes + index_cond: ["iata = 'SEA'"] diff --git a/internal/planguard/testdata/corpus/bad_inherit_cycle.yaml b/internal/planguard/testdata/corpus/bad_inherit_cycle.yaml new file mode 100644 index 0000000..7893280 --- /dev/null +++ b/internal/planguard/testdata/corpus/bad_inherit_cycle.yaml @@ -0,0 +1,8 @@ +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +statements: + Q: {sqlc: Q, plan: custom} +entries: + - {id: a, stmt: Q, inherit: b} + - {id: b, stmt: Q, inherit: a} diff --git a/internal/planguard/testdata/corpus/bad_inline_resolver_schema.yaml b/internal/planguard/testdata/corpus/bad_inline_resolver_schema.yaml new file mode 100644 index 0000000..c300130 --- /dev/null +++ b/internal/planguard/testdata/corpus/bad_inline_resolver_schema.yaml @@ -0,0 +1,7 @@ +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +statements: + Q: {sqlc: Q, plan: custom} +entries: + - {id: a, stmt: Q, params: [{sql: "SELECT from_nodes FROM t_fix_routes_hop.search_cases LIMIT 1"}]} diff --git a/internal/planguard/testdata/corpus/bad_missing_sqlc.yaml b/internal/planguard/testdata/corpus/bad_missing_sqlc.yaml new file mode 100644 index 0000000..15634c4 --- /dev/null +++ b/internal/planguard/testdata/corpus/bad_missing_sqlc.yaml @@ -0,0 +1,7 @@ +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +statements: + Gone: {sqlc: Gone, plan: custom} +entries: + - {id: a, stmt: Gone} diff --git a/internal/planguard/testdata/corpus/bad_param_key.yaml b/internal/planguard/testdata/corpus/bad_param_key.yaml new file mode 100644 index 0000000..cde3ae6 --- /dev/null +++ b/internal/planguard/testdata/corpus/bad_param_key.yaml @@ -0,0 +1,7 @@ +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +statements: + Q: {sqlc: Q, plan: custom} +entries: + - {id: a, stmt: Q, params: [{rr: top}]} diff --git a/internal/planguard/testdata/corpus/bad_resolver_ref.yaml b/internal/planguard/testdata/corpus/bad_resolver_ref.yaml new file mode 100644 index 0000000..39fb5a1 --- /dev/null +++ b/internal/planguard/testdata/corpus/bad_resolver_ref.yaml @@ -0,0 +1,9 @@ +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +statements: + Q: {sqlc: Q, plan: custom} +resolvers: + top: SELECT public_key FROM nodes LIMIT 1 +entries: + - {id: a, stmt: Q, params: [{r: top}, [x, {r: nope}]]} diff --git a/internal/planguard/testdata/corpus/bad_resolver_schema.yaml b/internal/planguard/testdata/corpus/bad_resolver_schema.yaml new file mode 100644 index 0000000..0be9d54 --- /dev/null +++ b/internal/planguard/testdata/corpus/bad_resolver_schema.yaml @@ -0,0 +1,9 @@ +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +statements: + Q: {sqlc: Q, plan: custom} +resolvers: + hop_case: SELECT x FROM t_fix_routes_hop.search_cases +entries: + - {id: a, stmt: Q} diff --git a/internal/planguard/testdata/corpus/bad_scan_pattern_literal.yaml b/internal/planguard/testdata/corpus/bad_scan_pattern_literal.yaml new file mode 100644 index 0000000..2478ad1 --- /dev/null +++ b/internal/planguard/testdata/corpus/bad_scan_pattern_literal.yaml @@ -0,0 +1,15 @@ +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +statements: + Q: {sqlc: Q, plan: custom} +sort_exempt_sets: + pinned: + - relation: known_routes + index_cond: "hop_count = 2" + reason: pinned literal +entries: + - id: a + stmt: Q + expect: + sort_exempt: [pinned] diff --git a/internal/planguard/testdata/corpus/bad_select_cte_writes.yaml b/internal/planguard/testdata/corpus/bad_select_cte_writes.yaml new file mode 100644 index 0000000..1bf5d35 --- /dev/null +++ b/internal/planguard/testdata/corpus/bad_select_cte_writes.yaml @@ -0,0 +1,7 @@ +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +statements: + W: {file: sql/harness/WriteCTE.sql, plan: harness} +entries: + - {id: a, stmt: W} diff --git a/internal/planguard/testdata/corpus/bad_select_writes.yaml b/internal/planguard/testdata/corpus/bad_select_writes.yaml new file mode 100644 index 0000000..e52547a --- /dev/null +++ b/internal/planguard/testdata/corpus/bad_select_writes.yaml @@ -0,0 +1,7 @@ +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +statements: + M: {sqlc: M, plan: custom} +entries: + - {id: a, stmt: M} diff --git a/internal/planguard/testdata/corpus/bad_sort_exempt.yaml b/internal/planguard/testdata/corpus/bad_sort_exempt.yaml new file mode 100644 index 0000000..03078e0 --- /dev/null +++ b/internal/planguard/testdata/corpus/bad_sort_exempt.yaml @@ -0,0 +1,10 @@ +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +statements: + Q: {sqlc: Q, plan: custom} +entries: + - id: a + stmt: Q + expect: + sort_exempt: [no_such_set] diff --git a/internal/planguard/testdata/corpus/bad_unknown_inherit.yaml b/internal/planguard/testdata/corpus/bad_unknown_inherit.yaml new file mode 100644 index 0000000..d3f2b5a --- /dev/null +++ b/internal/planguard/testdata/corpus/bad_unknown_inherit.yaml @@ -0,0 +1,7 @@ +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +statements: + Q: {sqlc: Q, plan: custom} +entries: + - {id: a, stmt: Q, inherit: nope} diff --git a/internal/planguard/testdata/corpus/bad_unknown_mode.yaml b/internal/planguard/testdata/corpus/bad_unknown_mode.yaml new file mode 100644 index 0000000..c0f6835 --- /dev/null +++ b/internal/planguard/testdata/corpus/bad_unknown_mode.yaml @@ -0,0 +1,7 @@ +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +statements: + Q: {sqlc: Q, plan: custom} +entries: + - {id: a, stmt: Q, modes: [custom, forced]} diff --git a/internal/planguard/testdata/corpus/bad_unknown_mode_override.yaml b/internal/planguard/testdata/corpus/bad_unknown_mode_override.yaml new file mode 100644 index 0000000..ca6422b --- /dev/null +++ b/internal/planguard/testdata/corpus/bad_unknown_mode_override.yaml @@ -0,0 +1,10 @@ +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +statements: + Q: {sqlc: Q, plan: custom} +entries: + - id: a + stmt: Q + mode: + plain: {skip: not a mode} diff --git a/internal/planguard/testdata/corpus/bad_unknown_stmt.yaml b/internal/planguard/testdata/corpus/bad_unknown_stmt.yaml new file mode 100644 index 0000000..4d8403a --- /dev/null +++ b/internal/planguard/testdata/corpus/bad_unknown_stmt.yaml @@ -0,0 +1,7 @@ +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +statements: + Q: {sqlc: Q, plan: custom} +entries: + - {id: a, stmt: Nope} diff --git a/internal/planguard/testdata/corpus/derived.yaml b/internal/planguard/testdata/corpus/derived.yaml new file mode 100644 index 0000000..ee7bb46 --- /dev/null +++ b/internal/planguard/testdata/corpus/derived.yaml @@ -0,0 +1,13 @@ +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +# A derived statement: the read-only part of a DML, frozen as a file and checked against today's sqlc text. +statements: + DeleteExpiredRoutes: {sqlc: DeleteExpiredRoutes, plan: maint, kind: dml} + DeleteExpiredRoutesSelect: + file: sql/derived/DeleteExpiredRoutesSelect.sql + derived_from: DeleteExpiredRoutes + plan: maint +entries: + - {id: expired.plain, stmt: DeleteExpiredRoutes} + - {id: expired.select, stmt: DeleteExpiredRoutesSelect} diff --git a/internal/planguard/testdata/corpus/genpolicy_hop.yaml b/internal/planguard/testdata/corpus/genpolicy_hop.yaml new file mode 100644 index 0000000..caa8d6e --- /dev/null +++ b/internal/planguard/testdata/corpus/genpolicy_hop.yaml @@ -0,0 +1,12 @@ +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +# A hop statement marked cached whose entries cover the global_page shapes only: rare_hop and absent_hop are missing. +statements: + ListKnownRoutesGlobalHop: {sqlc: ListKnownRoutesGlobalHop, plan: cached, admission: global_page} +admission_profiles: + global_page: [page1, deep, tie] +entries: + - {id: gh.page1, stmt: ListKnownRoutesGlobalHop, shapes: [page1], params: [6, null, 0, 50]} + - {id: gh.deep, stmt: ListKnownRoutesGlobalHop, shapes: [deep], params: [6, "2026-01-01 00:00:00+00", 0, 50]} + - {id: gh.tie, stmt: ListKnownRoutesGlobalHop, shapes: [tie], params: [6, "2026-01-01 00:00:00+00", 10, 50]} diff --git a/internal/planguard/testdata/corpus/genpolicy_hop_skip.yaml b/internal/planguard/testdata/corpus/genpolicy_hop_skip.yaml new file mode 100644 index 0000000..eafb7c4 --- /dev/null +++ b/internal/planguard/testdata/corpus/genpolicy_hop_skip.yaml @@ -0,0 +1,20 @@ +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +# The hop statement with rare and absent hop entries, but those skip the generic mode. +statements: + ListKnownRoutesGlobalHop: {sqlc: ListKnownRoutesGlobalHop, plan: cached, admission: global_page} +admission_profiles: + global_page: [page1, deep, tie] +entries: + - {id: gh.page1, stmt: ListKnownRoutesGlobalHop, shapes: [page1, deep, tie], params: [6, null, 0, 50]} + - id: gh.rare + stmt: ListKnownRoutesGlobalHop + shapes: [rare_hop] + params: [25, null, 0, 50] + mode: {generic: {skip: generic plans read the whole hop group}} + - id: gh.absent + stmt: ListKnownRoutesGlobalHop + shapes: [absent_hop] + params: [1, null, 0, 50] + mode: {generic: {skip: generic plans read the whole hop group}} diff --git a/internal/planguard/testdata/corpus/genpolicy_legacy_pending.yaml b/internal/planguard/testdata/corpus/genpolicy_legacy_pending.yaml new file mode 100644 index 0000000..0dd4316 --- /dev/null +++ b/internal/planguard/testdata/corpus/genpolicy_legacy_pending.yaml @@ -0,0 +1,13 @@ +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +# GetNodeByID fully covered, plus a pending cached statement that is left out without an error. +statements: + GetNodeByID: {sqlc: GetNodeByID, plan: cached, admission: by_id} + NotYet: {sqlc: NotMergedYet, plan: cached, admission: by_id, pending: some_feature} +admission_profiles: + by_id: [hit, miss] +entries: + - {id: node.hit, stmt: GetNodeByID, shapes: [hit], params: ["00000000-0000-0000-0000-000000000001"]} + - {id: node.miss, stmt: GetNodeByID, shapes: [miss], params: ["00000000-0000-0000-0000-000000000000"]} + - {id: notyet.one, stmt: NotYet, params: [1]} diff --git a/internal/planguard/testdata/corpus/genpolicy_refuse_degrade_to.yaml b/internal/planguard/testdata/corpus/genpolicy_refuse_degrade_to.yaml new file mode 100644 index 0000000..a4140ea --- /dev/null +++ b/internal/planguard/testdata/corpus/genpolicy_refuse_degrade_to.yaml @@ -0,0 +1,15 @@ +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +# GetNodeByID covers its admission profile, but node.miss blocks admission: the auto mode degrades. +statements: + GetNodeByID: {sqlc: GetNodeByID, plan: cached, admission: by_id} +admission_profiles: + by_id: [hit, miss] +entries: + - {id: node.hit, stmt: GetNodeByID, shapes: [hit], params: ["00000000-0000-0000-0000-000000000001"]} + - id: node.miss + stmt: GetNodeByID + shapes: [miss] + params: ["00000000-0000-0000-0000-000000000000"] + mode: {auto: {degrade_to: {budget: {buffers: 50}}}} diff --git a/internal/planguard/testdata/corpus/genpolicy_refuse_entry_known_issue.yaml b/internal/planguard/testdata/corpus/genpolicy_refuse_entry_known_issue.yaml new file mode 100644 index 0000000..a856aa2 --- /dev/null +++ b/internal/planguard/testdata/corpus/genpolicy_refuse_entry_known_issue.yaml @@ -0,0 +1,15 @@ +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +# GetNodeByID covers its admission profile, but node.miss blocks admission: the entry carries a known issue. +statements: + GetNodeByID: {sqlc: GetNodeByID, plan: cached, admission: by_id} +admission_profiles: + by_id: [hit, miss] +entries: + - {id: node.hit, stmt: GetNodeByID, shapes: [hit], params: ["00000000-0000-0000-0000-000000000001"]} + - id: node.miss + stmt: GetNodeByID + shapes: [miss] + params: ["00000000-0000-0000-0000-000000000000"] + known_issue: reads too much diff --git a/internal/planguard/testdata/corpus/genpolicy_refuse_expect_fail.yaml b/internal/planguard/testdata/corpus/genpolicy_refuse_expect_fail.yaml new file mode 100644 index 0000000..78cda20 --- /dev/null +++ b/internal/planguard/testdata/corpus/genpolicy_refuse_expect_fail.yaml @@ -0,0 +1,15 @@ +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +# GetNodeByID covers its admission profile, but node.miss blocks admission: the entry is expect_fail. +statements: + GetNodeByID: {sqlc: GetNodeByID, plan: cached, admission: by_id} +admission_profiles: + by_id: [hit, miss] +entries: + - {id: node.hit, stmt: GetNodeByID, shapes: [hit], params: ["00000000-0000-0000-0000-000000000001"]} + - id: node.miss + stmt: GetNodeByID + shapes: [miss] + params: ["00000000-0000-0000-0000-000000000000"] + expect_fail: true diff --git a/internal/planguard/testdata/corpus/genpolicy_refuse_mode_known_issue_auto.yaml b/internal/planguard/testdata/corpus/genpolicy_refuse_mode_known_issue_auto.yaml new file mode 100644 index 0000000..dc95037 --- /dev/null +++ b/internal/planguard/testdata/corpus/genpolicy_refuse_mode_known_issue_auto.yaml @@ -0,0 +1,15 @@ +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +# GetNodeByID covers its admission profile, but node.miss blocks admission: the auto mode carries a known issue. +statements: + GetNodeByID: {sqlc: GetNodeByID, plan: cached, admission: by_id} +admission_profiles: + by_id: [hit, miss] +entries: + - {id: node.hit, stmt: GetNodeByID, shapes: [hit], params: ["00000000-0000-0000-0000-000000000001"]} + - id: node.miss + stmt: GetNodeByID + shapes: [miss] + params: ["00000000-0000-0000-0000-000000000000"] + mode: {auto: {known_issue: auto flips to a bad plan}} diff --git a/internal/planguard/testdata/corpus/genpolicy_refuse_mode_known_issue_generic.yaml b/internal/planguard/testdata/corpus/genpolicy_refuse_mode_known_issue_generic.yaml new file mode 100644 index 0000000..1bd90c7 --- /dev/null +++ b/internal/planguard/testdata/corpus/genpolicy_refuse_mode_known_issue_generic.yaml @@ -0,0 +1,15 @@ +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +# GetNodeByID covers its admission profile, but node.miss blocks admission: the generic mode carries a known issue. +statements: + GetNodeByID: {sqlc: GetNodeByID, plan: cached, admission: by_id} +admission_profiles: + by_id: [hit, miss] +entries: + - {id: node.hit, stmt: GetNodeByID, shapes: [hit], params: ["00000000-0000-0000-0000-000000000001"]} + - id: node.miss + stmt: GetNodeByID + shapes: [miss] + params: ["00000000-0000-0000-0000-000000000000"] + mode: {generic: {known_issue: generic plan reads too much}} diff --git a/internal/planguard/testdata/corpus/genpolicy_refuse_no_auto.yaml b/internal/planguard/testdata/corpus/genpolicy_refuse_no_auto.yaml new file mode 100644 index 0000000..c1057e8 --- /dev/null +++ b/internal/planguard/testdata/corpus/genpolicy_refuse_no_auto.yaml @@ -0,0 +1,15 @@ +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +# GetNodeByID covers its admission profile, but node.miss blocks admission: the entry does not run auto. +statements: + GetNodeByID: {sqlc: GetNodeByID, plan: cached, admission: by_id} +admission_profiles: + by_id: [hit, miss] +entries: + - {id: node.hit, stmt: GetNodeByID, shapes: [hit], params: ["00000000-0000-0000-0000-000000000001"]} + - id: node.miss + stmt: GetNodeByID + shapes: [miss] + params: ["00000000-0000-0000-0000-000000000000"] + modes: [custom, generic] diff --git a/internal/planguard/testdata/corpus/genpolicy_refuse_no_entries.yaml b/internal/planguard/testdata/corpus/genpolicy_refuse_no_entries.yaml new file mode 100644 index 0000000..c4bf10e --- /dev/null +++ b/internal/planguard/testdata/corpus/genpolicy_refuse_no_entries.yaml @@ -0,0 +1,8 @@ +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +# A cached statement with no entries proves nothing. +statements: + GetNodeByID: {sqlc: GetNodeByID, plan: cached, admission: by_id} +admission_profiles: + by_id: [hit, miss] diff --git a/internal/planguard/testdata/corpus/genpolicy_refuse_no_generic.yaml b/internal/planguard/testdata/corpus/genpolicy_refuse_no_generic.yaml new file mode 100644 index 0000000..eceb036 --- /dev/null +++ b/internal/planguard/testdata/corpus/genpolicy_refuse_no_generic.yaml @@ -0,0 +1,15 @@ +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +# GetNodeByID covers its admission profile, but node.miss blocks admission: the entry does not run generic. +statements: + GetNodeByID: {sqlc: GetNodeByID, plan: cached, admission: by_id} +admission_profiles: + by_id: [hit, miss] +entries: + - {id: node.hit, stmt: GetNodeByID, shapes: [hit], params: ["00000000-0000-0000-0000-000000000001"]} + - id: node.miss + stmt: GetNodeByID + shapes: [miss] + params: ["00000000-0000-0000-0000-000000000000"] + modes: [custom, auto] diff --git a/internal/planguard/testdata/corpus/genpolicy_refuse_no_sqlc.yaml b/internal/planguard/testdata/corpus/genpolicy_refuse_no_sqlc.yaml new file mode 100644 index 0000000..0bcd4d8 --- /dev/null +++ b/internal/planguard/testdata/corpus/genpolicy_refuse_no_sqlc.yaml @@ -0,0 +1,10 @@ +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +# A cached statement read from a frozen file: only sqlc statements are served, so it cannot be admitted. +statements: + Frozen: {file: sql/harness/H.sql, plan: cached, admission: by_id} +admission_profiles: + by_id: [hit, miss] +entries: + - {id: frozen.hit, stmt: Frozen, shapes: [hit, miss], params: [1]} diff --git a/internal/planguard/testdata/corpus/genpolicy_refuse_skip_auto.yaml b/internal/planguard/testdata/corpus/genpolicy_refuse_skip_auto.yaml new file mode 100644 index 0000000..f9d84e3 --- /dev/null +++ b/internal/planguard/testdata/corpus/genpolicy_refuse_skip_auto.yaml @@ -0,0 +1,15 @@ +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +# GetNodeByID covers its admission profile, but node.miss blocks admission: the entry skips auto. +statements: + GetNodeByID: {sqlc: GetNodeByID, plan: cached, admission: by_id} +admission_profiles: + by_id: [hit, miss] +entries: + - {id: node.hit, stmt: GetNodeByID, shapes: [hit], params: ["00000000-0000-0000-0000-000000000001"]} + - id: node.miss + stmt: GetNodeByID + shapes: [miss] + params: ["00000000-0000-0000-0000-000000000000"] + mode: {auto: {skip: auto is not measured yet}} diff --git a/internal/planguard/testdata/corpus/genpolicy_region_no_stale.yaml b/internal/planguard/testdata/corpus/genpolicy_region_no_stale.yaml new file mode 100644 index 0000000..55c4211 --- /dev/null +++ b/internal/planguard/testdata/corpus/genpolicy_region_no_stale.yaml @@ -0,0 +1,12 @@ +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +# A region page statement without a stale_region entry. +statements: + ListKnownRoutesByIATA: {sqlc: ListKnownRoutesByIATA, plan: cached, admission: region_page} +admission_profiles: + region_page: [dense, sparse, empty_region, stale_region, page1, deep, tie] +entries: + - {id: r.dense, stmt: ListKnownRoutesByIATA, shapes: [dense, page1], params: [SEA, null, 0, 50]} + - {id: r.sparse, stmt: ListKnownRoutesByIATA, shapes: [sparse, deep], params: [MAD, "2026-01-01 00:00:00+00", 0, 50]} + - {id: r.empty, stmt: ListKnownRoutesByIATA, shapes: [empty_region, tie], params: [ZZZ, null, 0, 50]} diff --git a/internal/planguard/testdata/corpus/good.yaml b/internal/planguard/testdata/corpus/good.yaml new file mode 100644 index 0000000..931df66 --- /dev/null +++ b/internal/planguard/testdata/corpus/good.yaml @@ -0,0 +1,90 @@ +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +# A corpus that exercises every field the loader knows. +defaults: + big_tables: [packets, packet_observations] + executions: 6 + timeout_s: 60 + forbid: + - node: Seq Scan + relations: [big] + reason: no full scan of a big table + forbid_sort_above_limit: true + sort_exempt: [latest_observation] + retired_indexes: [idx_retired_example] +tiers: + fixture: + settings: + random_page_cost: "1.1" + search_path: public + local: {} + prod: {} +groups: + routes: + search_path: + local: t_s2b, public +statements: + Q: {sqlc: Q, group: routes, plan: cached, admission: by_id} + C: {sqlc: C, plan: custom} + M: {sqlc: M, plan: maint, kind: dml} + I: {sqlc: I, plan: ingest, kind: dml} + L: {sqlc: L, plan: legacy} + H: {file: sql/harness/H.sql, plan: harness} + P: {sqlc: NotMergedYet, plan: custom, pending: some_feature} +sort_exempt_sets: + latest_observation: + - relation: packet_observations + index_cond: packet_hash = p.packet_hash + reason: one probe per packet + hop_group: + - relation: known_routes + index: idx_known_routes_hop_iata + index_cond: "hop_count =" + reason: a hop group sorts its own rows +admission_profiles: + by_id: [hit, miss] +resolvers: + top_node: SELECT public_key FROM nodes ORDER BY last_seen DESC LIMIT 1 +entries: + - id: q.hit + stmt: Q + shape: hit + shapes: [hit] + params: [abc] + tiers: [fixture, local] + expect: + driver: + - relation: nodes + index: nodes_pkey + nodes: [Index Scan] + index_cond: ["public_key ="] + forbid: + - node: Seq Scan + relations: [big, nodes] + sort_exempt: [hop_group] + budget: {buffers: 10, planning_ms: 5, rows: 1} + mode: + generic: + degrade_to: + budget: {buffers: 20} + reason: generic probes twice + - id: q.miss + stmt: Q + shape: miss + shapes: [miss] + inherit: q.hit + - id: q.miss_own + stmt: Q + inherit: q.hit + tiers: [prod] + modes: [custom] + expect: + budget: {buffers: 3} + - {id: c.one, stmt: C} + - {id: c.modes, stmt: C, modes: [custom, generic]} + - {id: m.one, stmt: M, analyze: false} + - {id: i.one, stmt: I, executions: 1, timeout_s: 5} + - {id: l.one, stmt: L, baseline: "unstable: row estimate moves", note: legacy reader} + - {id: h.one, stmt: H, expect_fail: true, known_issue: planted} + - {id: p.one, stmt: P} diff --git a/internal/planguard/testdata/corpus/runner.yaml b/internal/planguard/testdata/corpus/runner.yaml new file mode 100644 index 0000000..8c2cde2 --- /dev/null +++ b/internal/planguard/testdata/corpus/runner.yaml @@ -0,0 +1,55 @@ +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +# The runner's integration corpus: route readers, a maintenance DML statement, a sleeping harness statement and +# resolvers that return nothing or fail. +defaults: + big_tables: [known_routes] + executions: 2 + timeout_s: 30 +tiers: + local: {} +statements: + ListKnownRoutesByIATA: {sqlc: ListKnownRoutesByIATA, plan: custom} + ListKnownRoutesGlobalHop: {sqlc: ListKnownRoutesGlobalHop, plan: custom} + DeleteExpiredRoutes: {sqlc: DeleteExpiredRoutes, plan: maint, kind: dml} + Sleep: {file: sql/harness/H.sql, plan: harness} + # A data-modifying CTE. The read-only test relabels it kind select, as a mislabelled statement would be, and + # expects the READ ONLY transaction to refuse it. + DeleteAll: {file: sql/harness/WriteCTE.sql, plan: harness, kind: dml} +resolvers: + no_region: SELECT iata FROM known_routes WHERE iata = 'ZZZ' LIMIT 1 + broken: SELECT (1 / 0)::text + busy_region: SELECT iata FROM known_routes GROUP BY iata ORDER BY count(*) DESC, iata LIMIT 1 +entries: + - id: sleep.timeout + stmt: Sleep + params: [3] + modes: [custom] + timeout_s: 1 + - id: routes.byiata + stmt: ListKnownRoutesByIATA + params: [SEA, null, 0, 50] + modes: [custom, generic] + expect: + driver: + - relation: known_routes + index_cond: ["iata >="] + - id: routes.globalhop + stmt: ListKnownRoutesGlobalHop + params: [2, null, 0, 50] + - id: routes.delete_expired + stmt: DeleteExpiredRoutes + params: ["-infinity", 0, "2027-01-01T00:00:00Z", "2027-01-01T00:00:00Z", 1000] + - id: harness.delete_all + stmt: DeleteAll + modes: [custom] + - id: routes.byiata.no_region + stmt: ListKnownRoutesByIATA + params: [{r: no_region}, null, 0, 50] + - id: routes.byiata.broken + stmt: ListKnownRoutesByIATA + params: [{r: broken}, null, 0, 50] + - id: routes.byiata.busy + stmt: ListKnownRoutesByIATA + params: [{r: busy_region}, null, 0, 50] diff --git a/internal/planguard/testdata/corpus/sql/derived/DeleteExpiredRoutesSelect.sql b/internal/planguard/testdata/corpus/sql/derived/DeleteExpiredRoutesSelect.sql new file mode 100644 index 0000000..d05eba5 --- /dev/null +++ b/internal/planguard/testdata/corpus/sql/derived/DeleteExpiredRoutesSelect.sql @@ -0,0 +1,10 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later +-- The read-only part of DeleteExpiredRoutes: its expired CTE without the lock clause. +SELECT r.iata, r.path_key, date_trunc('milliseconds', r.last_seen, 'UTC') AS ts, r.id + FROM known_routes r + WHERE (date_trunc('milliseconds', r.last_seen, 'UTC'), r.id) > ($1::timestamptz, $2::bigint) + AND date_trunc('milliseconds', r.last_seen, 'UTC') < $3::timestamptz + AND r.last_seen + interval '0' < $4::timestamptz + ORDER BY date_trunc('milliseconds', r.last_seen, 'UTC'), r.id + LIMIT $5 diff --git a/internal/planguard/testdata/corpus/sql/derived/DeleteExpiredRoutesSelectEdited.sql b/internal/planguard/testdata/corpus/sql/derived/DeleteExpiredRoutesSelectEdited.sql new file mode 100644 index 0000000..7d8440b --- /dev/null +++ b/internal/planguard/testdata/corpus/sql/derived/DeleteExpiredRoutesSelectEdited.sql @@ -0,0 +1,10 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later +-- The read-only part of DeleteExpiredRoutes: its expired CTE without the lock clause. +SELECT r.iata, r.path_key, date_trunc('milliseconds', r.last_seen, 'UTC') AS ts, r.id + FROM known_routes r + WHERE (date_trunc('milliseconds', r.last_seen, 'UTC'), r.id) > ($1::timestamptz, $2::bigint) + AND date_trunc('milliseconds', r.last_seen, 'UTC') <= $3::timestamptz + AND r.last_seen + interval '0' < $4::timestamptz + ORDER BY date_trunc('milliseconds', r.last_seen, 'UTC'), r.id + LIMIT $5 diff --git a/internal/planguard/testdata/corpus/sql/harness/H.sql b/internal/planguard/testdata/corpus/sql/harness/H.sql new file mode 100644 index 0000000..4729437 --- /dev/null +++ b/internal/planguard/testdata/corpus/sql/harness/H.sql @@ -0,0 +1,3 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later +SELECT pg_sleep($1) diff --git a/internal/planguard/testdata/corpus/sql/harness/WriteCTE.sql b/internal/planguard/testdata/corpus/sql/harness/WriteCTE.sql new file mode 100644 index 0000000..0589754 --- /dev/null +++ b/internal/planguard/testdata/corpus/sql/harness/WriteCTE.sql @@ -0,0 +1,4 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later +-- A SELECT that deletes: kind select must be refused. +WITH d AS (DELETE FROM known_routes RETURNING 1) SELECT count(*) FROM d diff --git a/internal/planguard/testdata/corpus/unknown_entry.yaml b/internal/planguard/testdata/corpus/unknown_entry.yaml new file mode 100644 index 0000000..34714b1 --- /dev/null +++ b/internal/planguard/testdata/corpus/unknown_entry.yaml @@ -0,0 +1,7 @@ +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +statements: + Q: {sqlc: Q, plan: custom} +entries: + - {id: a, stmt: Q, unexpected_entry_key: 1} diff --git a/internal/planguard/testdata/corpus/unknown_top.yaml b/internal/planguard/testdata/corpus/unknown_top.yaml new file mode 100644 index 0000000..e177fb7 --- /dev/null +++ b/internal/planguard/testdata/corpus/unknown_top.yaml @@ -0,0 +1,8 @@ +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +statements: + Q: {sqlc: Q, plan: custom} +unexpected_top: 1 +entries: + - {id: a, stmt: Q} diff --git a/internal/planguard/testdata/plans/adverts_past_end.json b/internal/planguard/testdata/plans/adverts_past_end.json new file mode 100644 index 0000000..d0c295d --- /dev/null +++ b/internal/planguard/testdata/plans/adverts_past_end.json @@ -0,0 +1,100 @@ +[ + { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Total Cost": 4.65, + "Plan Rows": 1, + "Actual Rows": 0, + "Actual Loops": 1, + "Shared Hit Blocks": 3, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Nested Loop", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Join Type": "Left", + "Total Cost": 4.65, + "Plan Rows": 1, + "Actual Rows": 0, + "Actual Loops": 1, + "Shared Hit Blocks": 3, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Nested Loop", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Join Type": "Inner", + "Total Cost": 4.21, + "Plan Rows": 1, + "Actual Rows": 0, + "Actual Loops": 1, + "Shared Hit Blocks": 3, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Scan Direction": "Forward", + "Index Name": "packet_observations_pkey", + "Relation Name": "packet_observations", + "Alias": "po", + "Total Cost": 1.56, + "Plan Rows": 1, + "Actual Rows": 0, + "Actual Loops": 1, + "Index Cond": "(id > '3000000000'::bigint)", + "Filter": "((observer_id = '00000000-0000-0000-0000-0000000000a1'::uuid) AND (payload_type = 4))", + "Rows Removed by Filter": 0, + "Shared Hit Blocks": 3, + "Shared Read Blocks": 0 + }, + { + "Node Type": "Index Scan", + "Parent Relationship": "Inner", + "Parallel Aware": false, + "Scan Direction": "Forward", + "Index Name": "packets_pkey", + "Relation Name": "packets", + "Alias": "p", + "Total Cost": 2.65, + "Plan Rows": 1, + "Actual Rows": 0, + "Actual Loops": 0, + "Index Cond": "(packet_hash = po.packet_hash)", + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0 + } + ] + }, + { + "Node Type": "Index Scan", + "Parent Relationship": "Inner", + "Parallel Aware": false, + "Scan Direction": "Forward", + "Index Name": "idx_nodes_pubkey", + "Relation Name": "nodes", + "Alias": "n", + "Total Cost": 0.44, + "Plan Rows": 1, + "Actual Rows": 0, + "Actual Loops": 0, + "Index Cond": "(public_key = p.origin_pubkey)", + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0 + } + ] + } + ] + }, + "Planning": { + "Shared Hit Blocks": 20, + "Shared Read Blocks": 0 + }, + "Planning Time": 0.087, + "Execution Time": 0.006 + } +] diff --git a/internal/planguard/testdata/plans/bysites_nolimit.json b/internal/planguard/testdata/plans/bysites_nolimit.json new file mode 100644 index 0000000..6685efa --- /dev/null +++ b/internal/planguard/testdata/plans/bysites_nolimit.json @@ -0,0 +1,328 @@ +[ + { + "Plan": { + "Node Type": "Sort", + "Parallel Aware": false, + "Total Cost": 668659.98, + "Plan Rows": 2, + "Actual Rows": 2, + "Actual Loops": 1, + "Sort Key": [ + "pg.k DESC", + "p.packet_hash DESC" + ], + "Shared Hit Blocks": 1175575, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Limit", + "Parent Relationship": "InitPlan", + "Subplan Name": "CTE page", + "Parallel Aware": false, + "Total Cost": 668503.3, + "Plan Rows": 2, + "Actual Rows": 2, + "Actual Loops": 1, + "Shared Hit Blocks": 1175556, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Sort", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Total Cost": 668511.26, + "Plan Rows": 3186, + "Actual Rows": 2, + "Actual Loops": 1, + "Sort Key": [ + "(max(ps.site_heard_at)) DESC", + "ps.packet_hash DESC" + ], + "Shared Hit Blocks": 1175556, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Aggregate", + "Strategy": "Hashed", + "Partial Mode": "Simple", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Total Cost": 668471.43, + "Plan Rows": 3186, + "Actual Rows": 155430, + "Actual Loops": 1, + "Group Key": [ + "ps.packet_hash" + ], + "Planned Partitions": 0, + "HashAgg Batches": 1, + "Peak Memory Usage": 16401, + "Disk Usage": 0, + "Shared Hit Blocks": 1175556, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Nested Loop", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Join Type": "Inner", + "Total Cost": 667470.39, + "Plan Rows": 193836, + "Actual Rows": 387659, + "Actual Loops": 1, + "Shared Hit Blocks": 1175556, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Function Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Function Name": "unnest", + "Alias": "req", + "Total Cost": 0.12, + "Plan Rows": 12, + "Actual Rows": 12, + "Actual Loops": 1, + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0 + }, + { + "Node Type": "Index Only Scan", + "Parent Relationship": "Inner", + "Parallel Aware": false, + "Scan Direction": "Forward", + "Index Name": "idx_packet_sites_site", + "Relation Name": "packet_sites", + "Alias": "ps", + "Total Cost": 55460.99, + "Plan Rows": 16153, + "Actual Rows": 32305, + "Actual Loops": 12, + "Index Cond": "((iata = req.iata) AND (ROW(site_heard_at, packet_hash) < ROW('2026-10-18 15:40:39.570763+00'::timestamp with time zone, '\\x00000000000000a2'::bytea)))", + "Filter": "(NOT (SubPlan 1))", + "Rows Removed by Filter": 1, + "Shared Hit Blocks": 1175556, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Index Only Scan", + "Parent Relationship": "SubPlan", + "Subplan Name": "SubPlan 1", + "Parallel Aware": false, + "Scan Direction": "Forward", + "Index Name": "packet_sites_pkey", + "Relation Name": "packet_sites", + "Alias": "x", + "Total Cost": 1.69, + "Plan Rows": 1, + "Actual Rows": 0, + "Actual Loops": 387672, + "Index Cond": "(packet_hash = ps.packet_hash)", + "Filter": "((iata = ANY ('{AMS,EIN,ENS,GLZ,GRQ,LEY,LWR,MST,QAR,RTM,UDE,UTC}'::bpchar[])) AND (ROW(site_heard_at, packet_hash) >= ROW('2026-10-18 15:40:39.570763+00'::timestamp with time zone, '\\x00000000000000a2'::bytea)))", + "Rows Removed by Filter": 6, + "Shared Hit Blocks": 1173183, + "Shared Read Blocks": 0 + } + ] + } + ] + } + ] + } + ] + } + ] + }, + { + "Node Type": "Nested Loop", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Join Type": "Left", + "Total Cost": 156.66, + "Plan Rows": 2, + "Actual Rows": 2, + "Actual Loops": 1, + "Shared Hit Blocks": 1175575, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Nested Loop", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Join Type": "Left", + "Total Cost": 155.93, + "Plan Rows": 2, + "Actual Rows": 2, + "Actual Loops": 1, + "Shared Hit Blocks": 1175575, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Nested Loop", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Join Type": "Left", + "Total Cost": 152.13, + "Plan Rows": 2, + "Actual Rows": 2, + "Actual Loops": 1, + "Shared Hit Blocks": 1175572, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Nested Loop", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Join Type": "Inner", + "Total Cost": 5.34, + "Plan Rows": 2, + "Actual Rows": 2, + "Actual Loops": 1, + "Shared Hit Blocks": 1175564, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "CTE Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "CTE Name": "page", + "Alias": "pg", + "Total Cost": 0.04, + "Plan Rows": 2, + "Actual Rows": 2, + "Actual Loops": 1, + "Shared Hit Blocks": 1175556, + "Shared Read Blocks": 0 + }, + { + "Node Type": "Index Scan", + "Parent Relationship": "Inner", + "Parallel Aware": false, + "Scan Direction": "Forward", + "Index Name": "packets_pkey", + "Relation Name": "packets", + "Alias": "p", + "Total Cost": 2.65, + "Plan Rows": 1, + "Actual Rows": 1, + "Actual Loops": 2, + "Index Cond": "(packet_hash = pg.packet_hash)", + "Shared Hit Blocks": 8, + "Shared Read Blocks": 0 + } + ] + }, + { + "Node Type": "Limit", + "Parent Relationship": "Inner", + "Parallel Aware": false, + "Total Cost": 73.38, + "Plan Rows": 1, + "Actual Rows": 1, + "Actual Loops": 2, + "Shared Hit Blocks": 8, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Sort", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Total Cost": 73.54, + "Plan Rows": 64, + "Actual Rows": 1, + "Actual Loops": 2, + "Sort Key": [ + "packet_observations.heard_at DESC", + "packet_observations.id DESC" + ], + "Shared Hit Blocks": 8, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Scan Direction": "Forward", + "Index Name": "idx_observations_packet", + "Relation Name": "packet_observations", + "Alias": "packet_observations", + "Total Cost": 73.06, + "Plan Rows": 64, + "Actual Rows": 1, + "Actual Loops": 2, + "Index Cond": "(packet_hash = p.packet_hash)", + "Shared Hit Blocks": 8, + "Shared Read Blocks": 0 + } + ] + } + ] + } + ] + }, + { + "Node Type": "Memoize", + "Parent Relationship": "Inner", + "Parallel Aware": false, + "Total Cost": 2.51, + "Plan Rows": 1, + "Actual Rows": 1, + "Actual Loops": 2, + "Cache Key": "packet_observations.observer_id", + "Cache Mode": "logical", + "Cache Hits": 1, + "Cache Misses": 1, + "Cache Evictions": 0, + "Cache Overflows": 0, + "Peak Memory Usage": 1, + "Shared Hit Blocks": 3, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Scan Direction": "Forward", + "Index Name": "observers_pkey", + "Relation Name": "observers", + "Alias": "o", + "Total Cost": 2.5, + "Plan Rows": 1, + "Actual Rows": 1, + "Actual Loops": 1, + "Index Cond": "(id = packet_observations.observer_id)", + "Shared Hit Blocks": 3, + "Shared Read Blocks": 0 + } + ] + } + ] + }, + { + "Node Type": "Index Scan", + "Parent Relationship": "Inner", + "Parallel Aware": false, + "Scan Direction": "Forward", + "Index Name": "transport_scopes_pkey", + "Relation Name": "transport_scopes", + "Alias": "ts", + "Total Cost": 0.3, + "Plan Rows": 1, + "Actual Rows": 0, + "Actual Loops": 2, + "Index Cond": "(id = p.scope_id)", + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0 + } + ] + } + ] + }, + "Planning": { + "Shared Hit Blocks": 15, + "Shared Read Blocks": 0 + }, + "Planning Time": 0.297, + "Execution Time": 484.7 + } +] diff --git a/internal/planguard/testdata/plans/bysites_ok.json b/internal/planguard/testdata/plans/bysites_ok.json new file mode 100644 index 0000000..fd613ee --- /dev/null +++ b/internal/planguard/testdata/plans/bysites_ok.json @@ -0,0 +1,333 @@ +[ + { + "Plan": { + "Node Type": "Sort", + "Parallel Aware": false, + "Total Cost": 167.54, + "Plan Rows": 1, + "Actual Rows": 2, + "Actual Loops": 1, + "Sort Key": [ + "pg.k DESC", + "p.packet_hash DESC" + ], + "Shared Hit Blocks": 171, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Limit", + "Parent Relationship": "InitPlan", + "Subplan Name": "CTE page", + "Parallel Aware": false, + "Total Cost": 88.59, + "Plan Rows": 1, + "Actual Rows": 2, + "Actual Loops": 1, + "Shared Hit Blocks": 149, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Sort", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Total Cost": 88.59, + "Plan Rows": 1, + "Actual Rows": 2, + "Actual Loops": 1, + "Sort Key": [ + "(max(ps.site_heard_at)) DESC", + "ps.packet_hash DESC" + ], + "Shared Hit Blocks": 149, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Aggregate", + "Strategy": "Sorted", + "Partial Mode": "Simple", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Total Cost": 88.58, + "Plan Rows": 1, + "Actual Rows": 8, + "Actual Loops": 1, + "Group Key": [ + "ps.packet_hash" + ], + "Shared Hit Blocks": 149, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Sort", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Total Cost": 88.45, + "Plan Rows": 24, + "Actual Rows": 24, + "Actual Loops": 1, + "Sort Key": [ + "ps.packet_hash DESC" + ], + "Shared Hit Blocks": 149, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Nested Loop", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Join Type": "Inner", + "Total Cost": 87.84, + "Plan Rows": 24, + "Actual Rows": 24, + "Actual Loops": 1, + "Shared Hit Blocks": 149, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Function Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Function Name": "unnest", + "Alias": "req", + "Total Cost": 0.12, + "Plan Rows": 12, + "Actual Rows": 12, + "Actual Loops": 1, + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0 + }, + { + "Node Type": "Limit", + "Parent Relationship": "Inner", + "Parallel Aware": false, + "Total Cost": 7.29, + "Plan Rows": 2, + "Actual Rows": 2, + "Actual Loops": 12, + "Shared Hit Blocks": 149, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Index Only Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Scan Direction": "Forward", + "Index Name": "idx_packet_sites_site", + "Relation Name": "packet_sites", + "Alias": "ps", + "Total Cost": 55460.99, + "Plan Rows": 16153, + "Actual Rows": 2, + "Actual Loops": 12, + "Index Cond": "((iata = req.iata) AND (ROW(site_heard_at, packet_hash) < ROW('2026-10-18 15:40:39.570763+00'::timestamp with time zone, '\\x00000000000000a2'::bytea)))", + "Filter": "(NOT (SubPlan 1))", + "Rows Removed by Filter": 1, + "Shared Hit Blocks": 149, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Index Only Scan", + "Parent Relationship": "SubPlan", + "Subplan Name": "SubPlan 1", + "Parallel Aware": false, + "Scan Direction": "Forward", + "Index Name": "packet_sites_pkey", + "Relation Name": "packet_sites", + "Alias": "x", + "Total Cost": 1.69, + "Plan Rows": 1, + "Actual Rows": 0, + "Actual Loops": 37, + "Index Cond": "(packet_hash = ps.packet_hash)", + "Filter": "((iata = ANY ('{AMS,EIN,ENS,GLZ,GRQ,LEY,LWR,MST,QAR,RTM,UDE,UTC}'::bpchar[])) AND (ROW(site_heard_at, packet_hash) >= ROW('2026-10-18 15:40:39.570763+00'::timestamp with time zone, '\\x00000000000000a2'::bytea)))", + "Rows Removed by Filter": 7, + "Shared Hit Blocks": 112, + "Shared Read Blocks": 0 + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + }, + { + "Node Type": "Nested Loop", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Join Type": "Left", + "Total Cost": 78.94, + "Plan Rows": 1, + "Actual Rows": 2, + "Actual Loops": 1, + "Shared Hit Blocks": 171, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Nested Loop", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Join Type": "Left", + "Total Cost": 78.57, + "Plan Rows": 1, + "Actual Rows": 2, + "Actual Loops": 1, + "Shared Hit Blocks": 171, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Nested Loop", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Join Type": "Left", + "Total Cost": 76.07, + "Plan Rows": 1, + "Actual Rows": 2, + "Actual Loops": 1, + "Shared Hit Blocks": 165, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Nested Loop", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Join Type": "Inner", + "Total Cost": 2.67, + "Plan Rows": 1, + "Actual Rows": 2, + "Actual Loops": 1, + "Shared Hit Blocks": 157, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "CTE Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "CTE Name": "page", + "Alias": "pg", + "Total Cost": 0.02, + "Plan Rows": 1, + "Actual Rows": 2, + "Actual Loops": 1, + "Shared Hit Blocks": 149, + "Shared Read Blocks": 0 + }, + { + "Node Type": "Index Scan", + "Parent Relationship": "Inner", + "Parallel Aware": false, + "Scan Direction": "Forward", + "Index Name": "packets_pkey", + "Relation Name": "packets", + "Alias": "p", + "Total Cost": 2.65, + "Plan Rows": 1, + "Actual Rows": 1, + "Actual Loops": 2, + "Index Cond": "(packet_hash = pg.packet_hash)", + "Shared Hit Blocks": 8, + "Shared Read Blocks": 0 + } + ] + }, + { + "Node Type": "Limit", + "Parent Relationship": "Inner", + "Parallel Aware": false, + "Total Cost": 73.38, + "Plan Rows": 1, + "Actual Rows": 1, + "Actual Loops": 2, + "Shared Hit Blocks": 8, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Sort", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Total Cost": 73.54, + "Plan Rows": 64, + "Actual Rows": 1, + "Actual Loops": 2, + "Sort Key": [ + "packet_observations.heard_at DESC", + "packet_observations.id DESC" + ], + "Shared Hit Blocks": 8, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Scan Direction": "Forward", + "Index Name": "idx_observations_packet", + "Relation Name": "packet_observations", + "Alias": "packet_observations", + "Total Cost": 73.06, + "Plan Rows": 64, + "Actual Rows": 1, + "Actual Loops": 2, + "Index Cond": "(packet_hash = p.packet_hash)", + "Shared Hit Blocks": 8, + "Shared Read Blocks": 0 + } + ] + } + ] + } + ] + }, + { + "Node Type": "Index Scan", + "Parent Relationship": "Inner", + "Parallel Aware": false, + "Scan Direction": "Forward", + "Index Name": "observers_pkey", + "Relation Name": "observers", + "Alias": "o", + "Total Cost": 2.5, + "Plan Rows": 1, + "Actual Rows": 1, + "Actual Loops": 2, + "Index Cond": "(id = packet_observations.observer_id)", + "Shared Hit Blocks": 6, + "Shared Read Blocks": 0 + } + ] + }, + { + "Node Type": "Index Scan", + "Parent Relationship": "Inner", + "Parallel Aware": false, + "Scan Direction": "Forward", + "Index Name": "transport_scopes_pkey", + "Relation Name": "transport_scopes", + "Alias": "ts", + "Total Cost": 0.3, + "Plan Rows": 1, + "Actual Rows": 0, + "Actual Loops": 2, + "Index Cond": "(id = p.scope_id)", + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0 + } + ] + } + ] + }, + "Planning": { + "Shared Hit Blocks": 15, + "Shared Read Blocks": 0 + }, + "Planning Time": 0.177, + "Execution Time": 0.115 + } +] diff --git a/internal/planguard/testdata/plans/cmi_seqscan.json b/internal/planguard/testdata/plans/cmi_seqscan.json new file mode 100644 index 0000000..0e0c7b9 --- /dev/null +++ b/internal/planguard/testdata/plans/cmi_seqscan.json @@ -0,0 +1,220 @@ +[ + { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Total Cost": 15971.65, + "Plan Rows": 21, + "Actual Rows": 21, + "Actual Loops": 1, + "Shared Hit Blocks": 22124, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Sort", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Total Cost": 15971.84, + "Plan Rows": 94, + "Actual Rows": 21, + "Actual Loops": 1, + "Sort Key": [ + "cm.id DESC" + ], + "Shared Hit Blocks": 22124, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Nested Loop", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Join Type": "Left", + "Total Cost": 15969.07, + "Plan Rows": 94, + "Actual Rows": 1785, + "Actual Loops": 1, + "Shared Hit Blocks": 22124, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Nested Loop", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Join Type": "Inner", + "Total Cost": 15965.16, + "Plan Rows": 94, + "Actual Rows": 1785, + "Actual Loops": 1, + "Shared Hit Blocks": 22121, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Nested Loop", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Join Type": "Semi", + "Total Cost": 15937.66, + "Plan Rows": 94, + "Actual Rows": 1785, + "Actual Loops": 1, + "Shared Hit Blocks": 16766, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Hash Join", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Join Type": "Inner", + "Total Cost": 15641.76, + "Plan Rows": 619, + "Actual Rows": 1785, + "Actual Loops": 1, + "Hash Cond": "(cm.packet_hash = p.packet_hash)", + "Shared Hit Blocks": 11123, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Seq Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Relation Name": "channel_messages", + "Alias": "cm", + "Total Cost": 5350.36, + "Plan Rows": 196669, + "Actual Rows": 196804, + "Actual Loops": 1, + "Filter": "(id < '9223372036854775807'::bigint)", + "Rows Removed by Filter": 0, + "Shared Hit Blocks": 2892, + "Shared Read Blocks": 0 + }, + { + "Node Type": "Hash", + "Parent Relationship": "Inner", + "Parallel Aware": false, + "Total Cost": 9655.11, + "Plan Rows": 9603, + "Actual Rows": 9592, + "Actual Loops": 1, + "Hash Buckets": 16384, + "Original Hash Buckets": 16384, + "Hash Batches": 1, + "Original Hash Batches": 1, + "Peak Memory Usage": 728, + "Shared Hit Blocks": 8231, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Bitmap Heap Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Relation Name": "packets", + "Alias": "p", + "Total Cost": 9655.11, + "Plan Rows": 9603, + "Actual Rows": 9592, + "Actual Loops": 1, + "Recheck Cond": "(scope_id = 175)", + "Exact Heap Blocks": 8159, + "Lossy Heap Blocks": 0, + "Shared Hit Blocks": 8231, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Bitmap Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Index Name": "idx_packets_scope_last_heard", + "Total Cost": 148.34, + "Plan Rows": 9603, + "Actual Rows": 9592, + "Actual Loops": 1, + "Index Cond": "(scope_id = 175)", + "Shared Hit Blocks": 72, + "Shared Read Blocks": 0 + } + ] + } + ] + } + ] + }, + { + "Node Type": "Index Only Scan", + "Parent Relationship": "Inner", + "Parallel Aware": false, + "Scan Direction": "Forward", + "Index Name": "idx_observations_packet", + "Relation Name": "packet_observations", + "Alias": "po", + "Total Cost": 2.53, + "Plan Rows": 64, + "Actual Rows": 1, + "Actual Loops": 1785, + "Index Cond": "(packet_hash = p.packet_hash)", + "Shared Hit Blocks": 5643, + "Shared Read Blocks": 0 + } + ] + }, + { + "Node Type": "Index Scan", + "Parent Relationship": "Inner", + "Parallel Aware": false, + "Scan Direction": "Forward", + "Index Name": "channels_pkey", + "Relation Name": "channels", + "Alias": "c", + "Total Cost": 0.29, + "Plan Rows": 1, + "Actual Rows": 1, + "Actual Loops": 1785, + "Index Cond": "(id = cm.channel_id)", + "Shared Hit Blocks": 5355, + "Shared Read Blocks": 0 + } + ] + }, + { + "Node Type": "Materialize", + "Parent Relationship": "Inner", + "Parallel Aware": false, + "Total Cost": 2.5, + "Plan Rows": 1, + "Actual Rows": 1, + "Actual Loops": 1785, + "Shared Hit Blocks": 3, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Scan Direction": "Forward", + "Index Name": "transport_scopes_pkey", + "Relation Name": "transport_scopes", + "Alias": "ts", + "Total Cost": 2.5, + "Plan Rows": 1, + "Actual Rows": 1, + "Actual Loops": 1, + "Index Cond": "(id = 175)", + "Shared Hit Blocks": 3, + "Shared Read Blocks": 0 + } + ] + } + ] + } + ] + } + ] + }, + "Planning": { + "Shared Hit Blocks": 67, + "Shared Read Blocks": 0 + }, + "Planning Time": 0.37, + "Execution Time": 26.02 + } +] diff --git a/internal/planguard/testdata/plans/globalhop_rare_retired.json b/internal/planguard/testdata/plans/globalhop_rare_retired.json new file mode 100644 index 0000000..5cc2209 --- /dev/null +++ b/internal/planguard/testdata/plans/globalhop_rare_retired.json @@ -0,0 +1,57 @@ +[ + { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Total Cost": 88706.08, + "Plan Rows": 1, + "Actual Rows": 0, + "Actual Loops": 1, + "Shared Hit Blocks": 8902, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Sort", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Total Cost": 88706.08, + "Plan Rows": 1, + "Actual Rows": 0, + "Actual Loops": 1, + "Sort Key": [ + "(date_trunc('milliseconds'::text, last_seen, 'UTC'::text)) DESC", + "id DESC" + ], + "Shared Hit Blocks": 8902, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Scan Direction": "Forward", + "Index Name": "idx_known_routes_hop_count", + "Relation Name": "known_routes", + "Alias": "known_routes", + "Total Cost": 88706.07, + "Plan Rows": 1, + "Actual Rows": 0, + "Actual Loops": 1, + "Index Cond": "(hop_count = 1)", + "Filter": "(ROW(date_trunc('milliseconds'::text, last_seen, 'UTC'::text), id) < ROW('infinity'::timestamp with time zone, '0'::bigint))", + "Rows Removed by Filter": 0, + "Shared Hit Blocks": 8902, + "Shared Read Blocks": 0 + } + ] + } + ] + }, + "Planning": { + "Shared Hit Blocks": 4, + "Shared Read Blocks": 0 + }, + "Planning Time": 0.081, + "Execution Time": 5.901 + } +] diff --git a/internal/planguard/testdata/plans/packets_page.json b/internal/planguard/testdata/plans/packets_page.json new file mode 100644 index 0000000..f812f60 --- /dev/null +++ b/internal/planguard/testdata/plans/packets_page.json @@ -0,0 +1,239 @@ +[ + { + "Plan": { + "Node Type": "Sort", + "Parallel Aware": false, + "Total Cost": 3907.53, + "Plan Rows": 51, + "Actual Rows": 51, + "Actual Loops": 1, + "Sort Key": [ + "pg.k DESC", + "p.packet_hash DESC" + ], + "Shared Hit Blocks": 760, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Limit", + "Parent Relationship": "InitPlan", + "Subplan Name": "CTE page", + "Parallel Aware": false, + "Total Cost": 1.6, + "Plan Rows": 51, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 5, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Index Only Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Scan Direction": "Forward", + "Index Name": "idx_packets_last_heard_hash", + "Relation Name": "packets", + "Alias": "p_1", + "Total Cost": 69933.35, + "Plan Rows": 3048658, + "Actual Rows": 51, + "Actual Loops": 1, + "Index Cond": "(ROW(last_heard_at, packet_hash) < ROW('infinity'::timestamp with time zone, '\\x'::bytea))", + "Shared Hit Blocks": 5, + "Shared Read Blocks": 0 + } + ] + }, + { + "Node Type": "Nested Loop", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Join Type": "Left", + "Total Cost": 3904.36, + "Plan Rows": 51, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 760, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Nested Loop", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Join Type": "Left", + "Total Cost": 3885.62, + "Plan Rows": 51, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 748, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Nested Loop", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Join Type": "Left", + "Total Cost": 3879.39, + "Plan Rows": 51, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 622, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Nested Loop", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Join Type": "Inner", + "Total Cost": 136.04, + "Plan Rows": 51, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 209, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "CTE Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "CTE Name": "page", + "Alias": "pg", + "Total Cost": 1.02, + "Plan Rows": 51, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 5, + "Shared Read Blocks": 0 + }, + { + "Node Type": "Index Scan", + "Parent Relationship": "Inner", + "Parallel Aware": false, + "Scan Direction": "Forward", + "Index Name": "packets_pkey", + "Relation Name": "packets", + "Alias": "p", + "Total Cost": 2.65, + "Plan Rows": 1, + "Actual Rows": 1, + "Actual Loops": 51, + "Index Cond": "(packet_hash = pg.packet_hash)", + "Shared Hit Blocks": 204, + "Shared Read Blocks": 0 + } + ] + }, + { + "Node Type": "Limit", + "Parent Relationship": "Inner", + "Parallel Aware": false, + "Total Cost": 73.38, + "Plan Rows": 1, + "Actual Rows": 1, + "Actual Loops": 51, + "Shared Hit Blocks": 413, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Sort", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Total Cost": 73.54, + "Plan Rows": 64, + "Actual Rows": 1, + "Actual Loops": 51, + "Sort Key": [ + "packet_observations.heard_at DESC", + "packet_observations.id DESC" + ], + "Shared Hit Blocks": 413, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Scan Direction": "Forward", + "Index Name": "idx_observations_packet", + "Relation Name": "packet_observations", + "Alias": "packet_observations", + "Total Cost": 73.06, + "Plan Rows": 64, + "Actual Rows": 8, + "Actual Loops": 51, + "Index Cond": "(packet_hash = p.packet_hash)", + "Shared Hit Blocks": 413, + "Shared Read Blocks": 0 + } + ] + } + ] + } + ] + }, + { + "Node Type": "Memoize", + "Parent Relationship": "Inner", + "Parallel Aware": false, + "Total Cost": 2.51, + "Plan Rows": 1, + "Actual Rows": 1, + "Actual Loops": 51, + "Cache Key": "packet_observations.observer_id", + "Cache Mode": "logical", + "Cache Hits": 9, + "Cache Misses": 42, + "Cache Evictions": 0, + "Cache Overflows": 0, + "Peak Memory Usage": 6, + "Shared Hit Blocks": 126, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Scan Direction": "Forward", + "Index Name": "observers_pkey", + "Relation Name": "observers", + "Alias": "o", + "Total Cost": 2.5, + "Plan Rows": 1, + "Actual Rows": 1, + "Actual Loops": 42, + "Index Cond": "(id = packet_observations.observer_id)", + "Shared Hit Blocks": 126, + "Shared Read Blocks": 0 + } + ] + } + ] + }, + { + "Node Type": "Index Scan", + "Parent Relationship": "Inner", + "Parallel Aware": false, + "Scan Direction": "Forward", + "Index Name": "transport_scopes_pkey", + "Relation Name": "transport_scopes", + "Alias": "ts", + "Total Cost": 0.3, + "Plan Rows": 1, + "Actual Rows": 0, + "Actual Loops": 51, + "Index Cond": "(id = p.scope_id)", + "Shared Hit Blocks": 12, + "Shared Read Blocks": 0 + } + ] + } + ] + }, + "Planning": { + "Shared Hit Blocks": 19, + "Shared Read Blocks": 0 + }, + "Planning Time": 0.211, + "Execution Time": 0.328 + } +] diff --git a/internal/planguard/testdata/plans/routes_byiata_custom.json b/internal/planguard/testdata/plans/routes_byiata_custom.json new file mode 100644 index 0000000..2d67be8 --- /dev/null +++ b/internal/planguard/testdata/plans/routes_byiata_custom.json @@ -0,0 +1,38 @@ +[ + { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Total Cost": 14.71, + "Plan Rows": 51, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 54, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Scan Direction": "Forward", + "Index Name": "idx_known_routes_iata_keyset", + "Relation Name": "known_routes", + "Alias": "known_routes", + "Total Cost": 536102.89, + "Plan Rows": 1931640, + "Actual Rows": 51, + "Actual Loops": 1, + "Index Cond": "((iata >= 'SEA'::bpchar) AND (iata <= 'SEA'::bpchar) AND (ROW(date_trunc('milliseconds'::text, last_seen, 'UTC'::text), id) < ROW('infinity'::timestamp with time zone, '0'::bigint)))", + "Shared Hit Blocks": 54, + "Shared Read Blocks": 0 + } + ] + }, + "Planning": { + "Shared Hit Blocks": 4, + "Shared Read Blocks": 0 + }, + "Planning Time": 0.067, + "Execution Time": 0.028 + } +] diff --git a/internal/planguard/testdata/plans/routes_byiata_generic.json b/internal/planguard/testdata/plans/routes_byiata_generic.json new file mode 100644 index 0000000..3230e23 --- /dev/null +++ b/internal/planguard/testdata/plans/routes_byiata_generic.json @@ -0,0 +1,38 @@ +[ + { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Total Cost": 1964.77, + "Plan Rows": 1752, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 54, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Scan Direction": "Forward", + "Index Name": "idx_known_routes_iata_keyset", + "Relation Name": "known_routes", + "Alias": "known_routes", + "Total Cost": 19648.22, + "Plan Rows": 17525, + "Actual Rows": 51, + "Actual Loops": 1, + "Index Cond": "((iata >= $1) AND (iata <= $1) AND (ROW(date_trunc('milliseconds'::text, last_seen, 'UTC'::text), id) < ROW(COALESCE($2, 'infinity'::timestamp with time zone), $3)))", + "Shared Hit Blocks": 54, + "Shared Read Blocks": 0 + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0 + }, + "Planning Time": 0.002, + "Execution Time": 0.025 + } +] diff --git a/internal/planguard/testdata/plans/routes_byiata_mid.json b/internal/planguard/testdata/plans/routes_byiata_mid.json new file mode 100644 index 0000000..cbffc3d --- /dev/null +++ b/internal/planguard/testdata/plans/routes_byiata_mid.json @@ -0,0 +1,38 @@ +[ + { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Total Cost": 166.52, + "Plan Rows": 201, + "Actual Rows": 201, + "Actual Loops": 1, + "Shared Hit Blocks": 205, + "Shared Read Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Scan Direction": "Forward", + "Index Name": "idx_known_routes_iata_keyset", + "Relation Name": "known_routes", + "Alias": "known_routes", + "Total Cost": 257865.44, + "Plan Rows": 312301, + "Actual Rows": 201, + "Actual Loops": 1, + "Index Cond": "((iata >= 'BWE'::bpchar) AND (iata <= 'BWE'::bpchar) AND (ROW(date_trunc('milliseconds'::text, last_seen, 'UTC'::text), id) < ROW('infinity'::timestamp with time zone, '0'::bigint)))", + "Shared Hit Blocks": 205, + "Shared Read Blocks": 0 + } + ] + }, + "Planning": { + "Shared Hit Blocks": 4, + "Shared Read Blocks": 0 + }, + "Planning Time": 0.051, + "Execution Time": 0.078 + } +] diff --git a/internal/planguard/testdata/prodcorpus/corpus.yaml b/internal/planguard/testdata/prodcorpus/corpus.yaml new file mode 100644 index 0000000..1ab726a --- /dev/null +++ b/internal/planguard/testdata/prodcorpus/corpus.yaml @@ -0,0 +1,46 @@ +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +# The prod checker's integration corpus: route readers in every mode, a literal empty region, a resolver that +# returns nothing, a maintenance DML statement, an ingest statement (auto only) and a pending statement. +defaults: + big_tables: [known_routes] + executions: 2 + timeout_s: 30 +tiers: + prod: {} +statements: + ListKnownRoutesByIATA: {sqlc: ListKnownRoutesByIATA, plan: custom} + ListKnownRoutesGlobalHop: {sqlc: ListKnownRoutesGlobalHop, plan: custom} + ListKnownRoutesGlobal: {sqlc: ListKnownRoutesGlobal, plan: custom, pending: later} + DeleteExpiredRoutes: {sqlc: DeleteExpiredRoutes, plan: maint, kind: dml} + UpsertChannel: {sqlc: UpsertChannel, plan: ingest, kind: dml} +resolvers: + no_region: SELECT iata FROM known_routes WHERE iata = 'ZZZ' LIMIT 1 +entries: + - id: routes.byiata + stmt: ListKnownRoutesByIATA + params: [SEA, null, 0, 50] + modes: [custom, generic, auto] + expect: + driver: + - relation: known_routes + index_cond: ["iata >="] + - id: routes.byiata.empty_region + stmt: ListKnownRoutesByIATA + params: [ZZZ, null, 0, 50] + - id: routes.byiata.no_region + stmt: ListKnownRoutesByIATA + params: [{r: no_region}, null, 0, 50] + - id: routes.globalhop + stmt: ListKnownRoutesGlobalHop + params: [2, null, 0, 50] + - id: routes.global.pending + stmt: ListKnownRoutesGlobal + params: [null, 0, 50] + - id: routes.delete_expired + stmt: DeleteExpiredRoutes + params: ["-infinity", 0, "2027-01-01T00:00:00Z", "2027-01-01T00:00:00Z", 1000] + - id: channels.upsert + stmt: UpsertChannel + params: ["01", null, null, null, false] diff --git a/internal/planguard/walk.go b/internal/planguard/walk.go new file mode 100644 index 0000000..aacaaea --- /dev/null +++ b/internal/planguard/walk.go @@ -0,0 +1,248 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package planguard + +import ( + "context" + "errors" + "fmt" + "slices" + "strings" +) + +// DefaultWalkRows caps a walk when its spec sets no max_rows. +const DefaultWalkRows = 2000 + +// WalkSpec pages through a keyset-paginated statement the way a client does. +type WalkSpec struct { + LimitParam int `yaml:"limit_param"` // 1-based + FetchExtra int `yaml:"fetch_extra"` // 1: the statement takes limit+1, and a longer page means more follow + Cursor []CursorBind `yaml:"cursor"` // param ← column of the last row of the page + First []any `yaml:"first"` // cursor param values for page 1 + WebLimit int `yaml:"web_limit"` // the client's page size for this endpoint (50 or 200) + MaxRows int `yaml:"max_rows"` // cap; default 2,000 + // Oracle: the ref is not walked; it is fetched as one page of MaxRows rows from First. + Oracle bool `yaml:"oracle"` +} + +// CursorBind binds a statement param (1-based) to a column of the last row kept. +type CursorBind struct { + Param int `yaml:"param"` + Column string `yaml:"column"` +} + +// PageFetcher returns one page of a walk: the statement run from cursor (nil on page 1), asked for limit rows the +// way the client asks (so a page longer than limit means another page follows). +type PageFetcher func(ctx context.Context, cursor []string, limit int) (RowSet, error) + +// WalkOutcome compares a walk with its ref. +type WalkOutcome struct { + Rows int // cand rows walked + FirstWrong int // 1-based position of the first row that differs from the ref's row there; 0 if none + Missing int // ref rows the walk never returned + Duplicates int // rows the walk returned more than once + OutOfOrder int // rows that come after a row the ref puts later + Stuck string // "cursor did not advance at page N" +} + +// Clean reports whether the walk returned exactly the ref's rows. +func (w WalkOutcome) Clean() bool { + return w.FirstWrong == 0 && w.Missing == 0 && w.Duplicates == 0 && w.OutOfOrder == 0 && w.Stuck == "" +} + +func (w *WalkSpec) maxRows() int { + if w.MaxRows > 0 { + return w.MaxRows + } + return DefaultWalkRows +} + +func (w *WalkSpec) validate(p DiffPair) error { + if w.LimitParam < 1 { + return errors.New("limit_param must be a param number ($1 = 1)") + } + if w.FetchExtra != 1 { + return errors.New("fetch_extra must be 1: a walk needs the statement to fetch limit+1 rows to know whether another page follows") + } + if len(w.Cursor) == 0 { + return errors.New("no cursor") + } + if len(w.First) != len(w.Cursor) { + return fmt.Errorf("first has %d values for %d cursor params", len(w.First), len(w.Cursor)) + } + seen := map[int]bool{w.LimitParam: true} + for _, c := range w.Cursor { + if c.Param < 1 || c.Column == "" { + return fmt.Errorf("cursor %+v: want a param number and a column", c) + } + if seen[c.Param] { + return fmt.Errorf("param $%d is bound twice", c.Param) + } + seen[c.Param] = true + } + for _, v := range w.First { + if err := checkDiffParam(v); err != nil { + return fmt.Errorf("first: %w", err) + } + } + if w.WebLimit < 0 || w.MaxRows < 0 { + return errors.New("web_limit and max_rows must not be negative") + } + return nil +} + +// checkParams checks that every bound param exists in a call with n params. +func (w *WalkSpec) checkParams(n int) error { + if w.LimitParam > n { + return fmt.Errorf("limit_param $%d, but the call has %d params", w.LimitParam, n) + } + for _, c := range w.Cursor { + if c.Param > n { + return fmt.Errorf("cursor param $%d, but the call has %d params", c.Param, n) + } + } + return nil +} + +func (w *WalkSpec) cursorColumns() []string { + out := make([]string, len(w.Cursor)) + for i, c := range w.Cursor { + out[i] = c.Column + } + return out +} + +// Walk pages through fetch at limit rows a page: page 1 from the first cursor, then from the cursor columns of the +// last row kept. It keeps the first limit rows of each page and stops when a page holds limit rows or fewer, at +// maxRows rows (≤ 0: DefaultWalkRows), or when the cursor repeats; stuck then says where. Each page adds a row or +// ends the walk, so it always ends. +func Walk(ctx context.Context, fetch PageFetcher, cursorCols []string, limit, maxRows int) (RowSet, string, error) { + if limit < 1 { + return RowSet{}, "", fmt.Errorf("walk limit %d, want ≥ 1", limit) + } + if maxRows <= 0 { + maxRows = DefaultWalkRows + } + var out RowSet + var cursor []string + var idx []int + seen := map[string]bool{} + for page := 1; ; page++ { + if err := ctx.Err(); err != nil { + return out, "", err + } + rs, err := fetch(ctx, cursor, limit) + if err != nil { + return out, "", fmt.Errorf("page %d: %w", page, err) + } + if page == 1 { + out.Columns = rs.Columns + for _, c := range cursorCols { + i := slices.Index(rs.Columns, c) + if i < 0 { + return out, "", fmt.Errorf("cursor column %s is not in the result", c) + } + idx = append(idx, i) + } + } else if !slices.Equal(rs.Columns, out.Columns) { + return out, "", fmt.Errorf("page %d has the columns %v, page 1 had %v", page, rs.Columns, out.Columns) + } + keep := rs.Rows[:min(limit, len(rs.Rows))] + if room := maxRows - len(out.Rows); len(keep) > room { + keep = keep[:room] + } + out.Rows = append(out.Rows, keep...) + if len(rs.Rows) <= limit || len(out.Rows) >= maxRows || len(keep) == 0 { + return out, "", nil + } + last := keep[len(keep)-1] + next := make([]string, len(idx)) + for n, i := range idx { + if last[i] == nil { + return out, "", fmt.Errorf("page %d: cursor column %s is NULL", page, cursorCols[n]) + } + next[n] = *last[i] + } + k := tuple(last, idx) + if seen[k] { + return out, fmt.Sprintf("cursor did not advance at page %d", page), nil + } + seen[k] = true + cursor = next + } +} + +// CompareWalk compares a walk with its ref. key names the identity columns (the cursor columns; empty: every ref +// column). FirstWrong compares whole rows over the ref's columns. A ref that holds one identity twice cannot be +// compared by identity, so it is an error. +func CompareWalk(ref, cand RowSet, key []string) (WalkOutcome, error) { + w := WalkOutcome{Rows: len(cand.Rows)} + full := make([]int, len(ref.Columns)) + fullC := make([]int, len(ref.Columns)) + for i, c := range ref.Columns { + full[i] = i + fullC[i] = slices.Index(cand.Columns, c) + } + if slices.Contains(fullC, -1) { + // The walk lacks a ref column: no row matches. + w.FirstWrong, w.Missing = 1, len(ref.Rows) + return w, nil + } + if len(key) == 0 { + key = ref.Columns + } + var keyR, keyC []int + for _, k := range key { + i, j := slices.Index(ref.Columns, k), slices.Index(cand.Columns, k) + if i < 0 { + return w, fmt.Errorf("identity column %s is not in the ref", k) + } + if j < 0 { + w.FirstWrong, w.Missing = 1, len(ref.Rows) + return w, nil + } + keyR, keyC = append(keyR, i), append(keyC, j) + } + n := min(len(ref.Rows), len(cand.Rows)) + for i := 0; i < n; i++ { + if tuple(ref.Rows[i], full) != tuple(cand.Rows[i], fullC) { + w.FirstWrong = i + 1 + break + } + } + if w.FirstWrong == 0 && len(ref.Rows) != len(cand.Rows) { + w.FirstWrong = n + 1 + } + pos := map[string]int{} + for i, r := range ref.Rows { + k := tuple(r, keyR) + if first, dup := pos[k]; dup { + return w, fmt.Errorf("the ref holds rows %d and %d with the same %s", first+1, i+1, strings.Join(key, ", ")) + } + pos[k] = i + } + got := map[string]bool{} + highest := -1 + for _, r := range cand.Rows { + k := tuple(r, keyC) + if got[k] { + w.Duplicates++ + continue + } + got[k] = true + if p, ok := pos[k]; ok { + if p < highest { + w.OutOfOrder++ + } else { + highest = p + } + } + } + for k := range pos { + if !got[k] { + w.Missing++ + } + } + return w, nil +} diff --git a/internal/planguard/walk_test.go b/internal/planguard/walk_test.go new file mode 100644 index 0000000..5914bd2 --- /dev/null +++ b/internal/planguard/walk_test.go @@ -0,0 +1,294 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package planguard_test + +import ( + "context" + "fmt" + "sort" + "strings" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/planguard" +) + +// siteRow is one packet_sites row: a packet heard at a site, at its newest time there. +type siteRow struct { + hash string + site string + at int +} + +// walkData holds 3 sites and 40 packets. Packets x1 and x2 are heard at two sites: their newest rows (sites A and +// C) are the two newest in the data, and their older site-B rows sit just above y, site B's next packet. +func walkData() []siteRow { + data := []siteRow{ + {"00000000000000f1", "A", 1000}, {"00000000000000f1", "B", 900}, + {"00000000000000f2", "C", 990}, {"00000000000000f2", "B", 890}, + {"00000000000000f3", "B", 880}, + } + sites := []string{"A", "C", "B"} + for i := 0; i < 37; i++ { + data = append(data, siteRow{fmt.Sprintf("%016x", 0x100+i), sites[i%3], 870 - 10*i}) + } + return data +} + +type keyed struct { + hash string + at int +} + +func ts(at int) string { return fmt.Sprintf("%06d", at) } + +// before reports whether (at, hash) sorts after the cursor in (at DESC, hash DESC) order, i.e. is below it. +func below(at int, hash string, cur []string) bool { + if cur == nil { + return true + } + a := ts(at) + return a < cur[0] || (a == cur[0] && hash < cur[1]) +} + +func page(ks []keyed, n int) planguard.RowSet { + sort.Slice(ks, func(i, j int) bool { + if ks[i].at != ks[j].at { + return ks[i].at > ks[j].at + } + return ks[i].hash > ks[j].hash + }) + if len(ks) > n { + ks = ks[:n] + } + rs := planguard.RowSet{Columns: []string{"packet_hash", "site_heard_at"}} + for _, k := range ks { + rs.Rows = append(rs.Rows, []*string{s(k.hash), s(ts(k.at))}) + } + return rs +} + +// oracleFetch: each packet's newest row over the sites, ordered by (site_heard_at DESC, hash DESC), below the cursor. +func oracleFetch(data []siteRow) planguard.PageFetcher { + return func(_ context.Context, cur []string, limit int) (planguard.RowSet, error) { + newest := map[string]int{} + for _, r := range data { + if r.at > newest[r.hash] { + newest[r.hash] = r.at + } + } + var ks []keyed + for h, at := range newest { + if below(at, h, cur) { + ks = append(ks, keyed{h, at}) + } + } + return page(ks, limit+1), nil + } +} + +// perSite returns each site's first n rows below the cursor (optionally only rows of packets no site has shown). +func perSite(data []siteRow, cur []string, n int, recheck func(hash string) bool) map[string][]siteRow { + bySite := map[string][]siteRow{} + for _, r := range data { + if below(r.at, r.hash, cur) && (recheck == nil || recheck(r.hash)) { + bySite[r.site] = append(bySite[r.site], r) + } + } + for site, rs := range bySite { + sort.Slice(rs, func(i, j int) bool { return rs[i].at > rs[j].at }) + if len(rs) > n { + bySite[site] = rs[:n] + } + } + return bySite +} + +// notShown reports whether no row of the packet is at or above the cursor. +func notShown(data []siteRow, cur []string) func(string) bool { + return func(hash string) bool { + for _, r := range data { + if r.hash == hash && !below(r.at, r.hash, cur) { + return false + } + } + return true + } +} + +func groupPage(bySite map[string][]siteRow, keep func(string) bool, n int) planguard.RowSet { + newest := map[string]int{} + for _, rs := range bySite { + for _, r := range rs { + if r.at > newest[r.hash] { + newest[r.hash] = r.at + } + } + } + var ks []keyed + for h, at := range newest { + if keep == nil || keep(h) { + ks = append(ks, keyed{h, at}) + } + } + return page(ks, n) +} + +// rmFetch: the readmodels statement's logic, a per-site LIMIT and then the cross-site re-check. +func rmFetch(data []siteRow) planguard.PageFetcher { + return func(_ context.Context, cur []string, limit int) (planguard.RowSet, error) { + return groupPage(perSite(data, cur, limit+1, nil), notShown(data, cur), limit+1), nil + } +} + +// endFetch: the end-state logic, the re-check inside each per-site range. +func endFetch(data []siteRow) planguard.PageFetcher { + return func(_ context.Context, cur []string, limit int) (planguard.RowSet, error) { + return groupPage(perSite(data, cur, limit+1, notShown(data, cur)), nil, limit+1), nil + } +} + +func clean(w planguard.WalkOutcome) bool { + return w.FirstWrong == 0 && w.Missing == 0 && w.Duplicates == 0 && w.OutOfOrder == 0 && w.Stuck == "" +} + +func TestWalkModeFindsReadmodelsSkip(t *testing.T) { + ctx := context.Background() + data := walkData() + cursor := []string{"site_heard_at", "packet_hash"} + identity := []string{"packet_hash"} + oracle, stuck, err := planguard.Walk(ctx, oracleFetch(data), cursor, 2000, 2000) + if err != nil || stuck != "" { + t.Fatalf("oracle: %v %q", err, stuck) + } + if len(oracle.Rows) != 40 { + t.Fatalf("oracle has %d rows, want 40", len(oracle.Rows)) + } + + walk := func(f planguard.PageFetcher, limit int) planguard.WalkOutcome { + t.Helper() + rs, stuck, err := planguard.Walk(ctx, f, cursor, limit, 2000) + if err != nil { + t.Fatal(err) + } + w, err := planguard.CompareWalk(oracle, rs, identity) + if err != nil { + t.Fatal(err) + } + w.Stuck = stuck + return w + } + rm1 := walk(rmFetch(data), 1) + if rm1.Missing == 0 || rm1.FirstWrong == 0 { + t.Errorf("readmodels at limit 1: %+v, want rows missing and a first wrong row", rm1) + } + if rm1.FirstWrong != 3 { + t.Errorf("readmodels at limit 1: first wrong row %d, want 3 (site B's next packet is skipped)", rm1.FirstWrong) + } + if end1 := walk(endFetch(data), 1); !clean(end1) || end1.Rows != 40 { + t.Errorf("end state at limit 1: %+v, want all zero over 40 rows", end1) + } + // The control: at limit 7 the readmodels statement happens to be right. + if rm7 := walk(rmFetch(data), 7); !clean(rm7) || rm7.Rows != 40 { + t.Errorf("readmodels at limit 7: %+v, want all zero over 40 rows", rm7) + } + if end7 := walk(endFetch(data), 7); !clean(end7) || end7.Rows != 40 { + t.Errorf("end state at limit 7: %+v, want all zero over 40 rows", end7) + } + // MaxRows caps the walk. + if rs, _, err := planguard.Walk(ctx, endFetch(data), cursor, 3, 10); err != nil || len(rs.Rows) != 10 { + t.Errorf("max rows 10: %d rows, %v", len(rs.Rows), err) + } +} + +func TestCompareWalkCounts(t *testing.T) { + cols := []string{"packet_hash", "site_heard_at"} + ref := rows(cols, []string{"a", "5"}, []string{"b", "4"}, []string{"c", "3"}, []string{"d", "2"}) + id := []string{"packet_hash"} + if w := compareWalk(t, ref, ref, id); !clean(w) || w.Rows != 4 { + t.Errorf("same walk: %+v", w) + } + dup := rows(cols, []string{"a", "5"}, []string{"b", "4"}, []string{"b", "4"}, []string{"c", "3"}, []string{"d", "2"}) + if w := compareWalk(t, ref, dup, id); w.Duplicates != 1 || w.FirstWrong != 3 || w.Missing != 0 { + t.Errorf("duplicate row: %+v", w) + } + swap := rows(cols, []string{"a", "5"}, []string{"c", "3"}, []string{"b", "4"}, []string{"d", "2"}) + if w := compareWalk(t, ref, swap, id); w.OutOfOrder != 1 || w.FirstWrong != 2 || w.Missing != 0 { + t.Errorf("rows out of order: %+v", w) + } + val := rows(cols, []string{"a", "5"}, []string{"b", "9"}, []string{"c", "3"}, []string{"d", "2"}) + if w := compareWalk(t, ref, val, id); w.FirstWrong != 2 || w.Missing != 0 { + t.Errorf("value changed: %+v", w) + } + short := rows(cols, []string{"a", "5"}, []string{"b", "4"}) + if w := compareWalk(t, ref, short, id); w.Missing != 2 || w.FirstWrong != 3 { + t.Errorf("walk stopped early: %+v", w) + } +} + +func compareWalk(t *testing.T, ref, cand planguard.RowSet, key []string) planguard.WalkOutcome { + t.Helper() + w, err := planguard.CompareWalk(ref, cand, key) + if err != nil { + t.Fatal(err) + } + return w +} + +func TestCompareWalkRefDuplicates(t *testing.T) { + cols := []string{"packet_hash", "site_heard_at"} + ref := rows(cols, []string{"a", "5"}, []string{"a", "4"}) + if _, err := planguard.CompareWalk(ref, ref, []string{"packet_hash"}); err == nil || !strings.Contains(err.Error(), "rows 1 and 2") { + t.Errorf("ref with a repeated identity: err %v, want an error naming rows 1 and 2", err) + } + if _, err := planguard.CompareWalk(ref, ref, []string{"nope"}); err == nil { + t.Error("identity column missing from the ref: no error") + } +} + +func TestWalkStopsOnStuckCursor(t *testing.T) { + calls := 0 + same := func(_ context.Context, cur []string, limit int) (planguard.RowSet, error) { + calls++ + rs := planguard.RowSet{Columns: []string{"packet_hash", "site_heard_at"}} + for i := 0; i <= limit; i++ { + rs.Rows = append(rs.Rows, []*string{s(fmt.Sprintf("h%d", i)), s("000100")}) + } + return rs, nil + } + rs, stuck, err := planguard.Walk(context.Background(), same, []string{"site_heard_at", "packet_hash"}, 2, 2000) + if err != nil { + t.Fatal(err) + } + if stuck != "cursor did not advance at page 2" { + t.Errorf("stuck %q, want %q", stuck, "cursor did not advance at page 2") + } + if calls != 2 || len(rs.Rows) != 4 { + t.Errorf("%d fetches, %d rows; want 2 fetches and 4 rows", calls, len(rs.Rows)) + } + + // A cursor that cycles through two values is stuck too. + n := 0 + cycle := func(_ context.Context, cur []string, limit int) (planguard.RowSet, error) { + n++ + rs := planguard.RowSet{Columns: []string{"packet_hash", "site_heard_at"}} + for i := 0; i <= limit; i++ { + rs.Rows = append(rs.Rows, []*string{s(fmt.Sprintf("h%d", n%2)), s("000100")}) + } + return rs, nil + } + if _, stuck, err := planguard.Walk(context.Background(), cycle, []string{"site_heard_at", "packet_hash"}, 1, 2000); err != nil || !strings.HasPrefix(stuck, "cursor did not advance at page 3") { + t.Errorf("cycling cursor: %q, %v", stuck, err) + } + + // A NULL in a cursor column, or a missing cursor column, is an error. + nullCur := func(_ context.Context, cur []string, limit int) (planguard.RowSet, error) { + return planguard.RowSet{Columns: []string{"packet_hash", "site_heard_at"}, Rows: [][]*string{{s("a"), nil}, {s("b"), nil}}}, nil + } + if _, _, err := planguard.Walk(context.Background(), nullCur, []string{"site_heard_at", "packet_hash"}, 1, 2000); err == nil { + t.Error("NULL cursor value: no error") + } + if _, _, err := planguard.Walk(context.Background(), nullCur, []string{"nope"}, 1, 2000); err == nil { + t.Error("missing cursor column: no error") + } +}