From 30ea3d2720e92be95e8c025848098235ad4302fc Mon Sep 17 00:00:00 2001
From: MrAlders0n <55921894+MrAlders0n@users.noreply.github.com>
Date: Mon, 19 Oct 2026 07:53:18 -0400
Subject: [PATCH] Document database pools, cost guards, metrics and statement
 statistics (#28)

---
 app_config/config.yaml.example |  25 ++++
 docs/api-contract.md           |  83 +++++++++--
 docs/backup-export.md          |  10 ++
 docs/configuration.md          |  64 ++++++++-
 docs/operations.md             | 252 ++++++++++++++++++++++++++++++++-
 5 files changed, 422 insertions(+), 12 deletions(-)

diff --git a/app_config/config.yaml.example b/app_config/config.yaml.example
index 8e20267..6e8c2c3 100644
--- a/app_config/config.yaml.example
+++ b/app_config/config.yaml.example
@@ -33,6 +33,10 @@ server:
   # With additional proxy hops, first configure their trust at the edge proxy
   # and pass its validated client address rather than a client-supplied header.
   trusted_proxies: []
+  # Address of the admin listener serving Prometheus /metrics, e.g. 127.0.0.1:9090
+  # (0.0.0.0:9090 inside Docker, published on the host's 127.0.0.1 only). Off when empty.
+  # It has no authentication: bind it to a private interface.
+  admin_listen: ""
 
 # Applies to /api/v1/* only. Both counters are local to this Beacon instance.
 # Set trusted_proxies correctly before using a reverse proxy, or clients share
@@ -45,6 +49,27 @@ ratelimit:
   # A 429 returns error.code=rate_limited and Retry-After (1 or 60 seconds).
   burst: 600
 
+# Database pools. custom_all (default) opens api_legacy 8, api_custom 8, ingest 6, maint 4 connections
+# (26); split opens 30 (api_custom 7, api_cached 5); single opens 18 (api_legacy, ingest, maint).
+# ingest grows to one connection per CPU on hosts with more than 6 CPUs, adding to each total.
+# Keep max_connections well above that. A pool_max_conns in POSTGRES_DSN is overridden.
+#database:
+#  # custom_all (default): every API statement gets a custom plan.
+#  # split: statements admitted to the generic-plan class run on cached generic plans. No statement
+#  #   is admitted yet, so split has no effect; keep custom_all.
+#  # single: rollback. One API pool on the previous settings; every guard family observes.
+#  api_plan_layout: custom_all
+
+# Cost guards on the public /api/v1 group (not /admin, not /ws). Every family observes by default:
+# nothing is refused, only beacon_guard_would_refuse_total counts. Listed families enforce:
+# 429 (4 concurrent DB requests per client, 5 s wait), 503 (pool busy 5 s; class deadline 5/10/30 s).
+# Families: routes, channel_messages, packets_global, packets_region, packets_window,
+#           packet_detail, stats, other. PUT /api/v1/admin/config changes the list at runtime;
+#           a restart reloads this one.
+#guards:
+#  enforce: []
+#  budget: shadow   # the per-client DB-time budget only records; enforcement is not available
+
 # MeshMapper integrations (optional, require a regional or grouped-region API key).
 # Your local MeshMapper regional or grouped-region admin can generate the key.
 # A grouped-region key covers multiple regions, including imported group members.
diff --git a/docs/api-contract.md b/docs/api-contract.md
index 3591653..c587da4 100644
--- a/docs/api-contract.md
+++ b/docs/api-contract.md
@@ -41,6 +41,47 @@ Retry-After: 60
 
 **Client IP.** The server only accepts `X-Real-IP`, and only from a direct peer listed in `server.trusted_proxies`. `X-Forwarded-For` and `True-Client-IP` are always ignored and stripped. Behind a reverse proxy that isn't listed, every client shares the proxy's budget.
 
+## Cost guards
+
+Every public `/api/v1` request also passes through cost guards (not `/ws`, not `/api/v1/admin/*`). By default they only count what they would refuse, and no response changes. A guard refuses only for a family the operator has listed in `guards.enforce` (see [Configuration](configuration.md#database-pools-and-cost-guards)). Clients should still handle the three refusals:
+
+| Status | `code` | `Retry-After` | When |
+|---|---|---|---|
+| `429` | `rate_limited` | `1` | More than 4 database-backed requests at once from one client, and no slot freed within 5 s. Cache hits don't count. |
+| `503` | `service_unavailable` | `2` | No database connection within 5 s. |
+| `503` | `deadline_exceeded` | `5` | The request's database time passed its class limit: entity 5 s, list 10 s, aggregation 30 s, counted from its first statement. |
+
+```
+HTTP/1.1 503 Service Unavailable
+Retry-After: 5
+
+{"error":{"code":"deadline_exceeded","message":"The request ran past its database time limit. Retry later."}}
+```
+
+The body is the usual error envelope. Wait at least `Retry-After` seconds before retrying; a `deadline_exceeded` request will likely run as long again, so narrow it (a shorter window, a region) rather than repeat it. Clients are keyed by IP after the trusted-proxy step above.
+
+Each request falls in one family and one class:
+
+| Path | Condition | Family | Class |
+|---|---|---|---|
+| `/packets`, `/packets/backfill` | `since` or `until` set (`/packets` only) | `packets_window` | aggregation |
+| `/packets`, `/packets/backfill` | region set | `packets_region` | list |
+| `/packets`, `/packets/backfill` | otherwise (including `hashPrefix`) | `packets_global` | list |
+| `/packets/{hash}` | | `packet_detail` | entity |
+| `/routes` | | `routes` | list |
+| `/routes/cross`, `/routes/search` | | `routes` | aggregation |
+| `/routes/{iata}/{pathKey}/observations` | | `routes` | entity |
+| `/channels/{id}/messages`, `/messages`, `/messages/backfill` | | `channel_messages` | list |
+| `/stats/*` | | `stats` | aggregation |
+| `/channels`, `/nodes`, `/observers`, `/traces`, `/nodes/{id}/observations`, `/observers/{id}/adverts` | | `other` | list |
+| `/observers/directory` | region set | `other` | list |
+| `/observers/directory` | otherwise | `other` | aggregation |
+| `/observers/{id}/activity` | | `other` | aggregation |
+| `/observers/{id}`, `/observers/{id}/telemetry`, `/nodes/{id}`, `/nodes/{id}/neighbors`, `/channels/{id}`, `/traces/{tag}` | | `other` | entity |
+| everything else (`/iatas*`, `/regions*`, `/scopes*`, `/info`, `/brokers`, unknown paths) | | `other` | entity |
+
+"Region set" means a non-empty `iata`, `iatas`, `region` or `regionId`.
+
 ## Versioning
 
 Path is the contract: `/api/v1/`. Breaking changes get `/api/v2/`. Backward-compatible additions just expand the existing payloads. WebSocket messages include a `type` discriminator and a top-level `v: 1` field so we can evolve in place.
@@ -53,7 +94,7 @@ All errors use one shape:
 { "error": { "code": "not_found", "message": "packet not found" } }
 ```
 
-`code` is the HTTP status text in snake_case, so match on it or on the status. Codes in use: `bad_request`, `unauthorized`, `not_found`, `conflict`, `request_entity_too_large`, `unsupported_media_type`, `rate_limited`, `internal_server_error`, `service_unavailable`, `gateway_timeout`, `insufficient_storage`. The heavy stats endpoints (`series`, `signal`, `paths`, `observer-comparison`) return `503` when the query times out; try a shorter window.
+`code` is the HTTP status text in snake_case, so match on it or on the status. Codes in use: `bad_request`, `unauthorized`, `not_found`, `conflict`, `request_entity_too_large`, `unsupported_media_type`, `rate_limited`, `internal_server_error`, `service_unavailable`, `gateway_timeout`, `insufficient_storage`, and `deadline_exceeded` (a `503` from the [cost guards](#cost-guards)). The heavy stats endpoints (`series`, `signal`, `paths`, `observer-comparison`) return `503` when the query times out; try a shorter window.
 
 A `404` means the requested thing doesn't exist. A database or other server-side failure is always a `500` (or `503` above), never a `404`, so clients can retry 5xx and treat 4xx as final. Empty lists are `200` with `[]`.
 
@@ -662,19 +703,41 @@ All under `/api/v1/admin/`, bearer key required (see [Auth](#auth)).
 
 | Endpoint | Notes |
 |---|---|
-| `GET /admin/config` | Running CORS settings with defaults applied, `auth.configured`, `ingest.broker_count` (configured broker workers, not connection status), `mobile.min_app_version` and `web.min_web_version` (read-only effective floors; `""` when there is none). No credentials, broker addresses, channel material or database settings. |
-| `PUT /admin/config` | Replaces `cors.allowed_origins` until the next restart. JSON body up to 16 KiB. |
+| `GET /admin/config` | Running CORS settings with defaults applied, `auth.configured`, `ingest.broker_count` (configured broker workers, not connection status), `mobile.min_app_version` and `web.min_web_version` (read-only effective floors; `""` when there is none), and `guards` (below). No credentials, broker addresses, channel material or database connection settings. |
+| `PUT /admin/config` | Replaces `cors.allowed_origins`, `guards.enforce` or both until the next restart. JSON body up to 16 KiB. |
+| `GET /admin/db/statements` | `pg_stat_statements` for this database, summed by statement name. `503` when the extension is missing or not preloaded. |
 | `GET /admin/accounts`, `POST /admin/accounts` | List or create operator account records (name only; not logins). |
 | `GET /admin/accounts/{id}`, `DELETE /admin/accounts/{id}` | Fetch or deactivate one. |
 | `GET /admin/backup` | Streams a private `.tar.gz` of the database and saved config. Off unless `backup.enabled: true`; needs `pg_dump` in the server's runtime. One export at a time (`409`), `504` on timeout, `507` over the size limit. Contains secrets. |
 
-`PUT /admin/config` accepts only `{"cors":{"allowed_origins":[...]}}` and applies the new list
-immediately. It is runtime-only: nothing is written to disk, and a restart reloads the file. The
-response carries `config`, `persisted: false` and `requires_restart: false`. Send 1 to 32 ASCII
-http(s) origins of at most 512 bytes each, with an optional single hostname wildcard; a lone `*`
-allows all. Empty or null lists, paths, queries, credentials, control characters and unknown
-fields are rejected. Concurrent updates apply one at a time; requests already in flight may see
-the old list.
+`PUT /admin/config` accepts `{"cors":{"allowed_origins":[...]}}`, `{"guards":{"enforce":[...]}}`
+or both, and applies them immediately. Both are validated before either is applied, so a rejected
+update changes nothing. It is runtime-only: nothing is written to disk, and a restart reloads the
+file. The response carries `config`, `persisted: false` and `requires_restart: false`. Send 1 to
+32 ASCII http(s) origins of at most 512 bytes each, with an optional single hostname wildcard; a
+lone `*` allows all. Empty or null lists, paths, queries, credentials, control characters and
+unknown fields are rejected. Concurrent updates apply one at a time; requests already in flight
+may see the old settings.
+
+`guards` in the config reports the [cost guards](#cost-guards):
+
+```json
+{ "enforce": ["other", "routes"], "observe_only": false, "budget": "shadow", "api_plan_layout": "custom_all" }
+```
+
+`enforce` lists the families that refuse; every other family only counts. `observe_only` is true
+under `database.api_plan_layout: single`, where every family observes whatever `enforce` says.
+`budget` is always `shadow`. In a `PUT`, `guards.enforce` is a list of family names (`routes`,
+`channel_messages`, `packets_global`, `packets_region`, `packets_window`, `packet_detail`,
+`stats`, `other`), deduplicated and returned in that order. `[]` makes every family observe; an
+unknown name is a `400`. The change applies to requests that start afterwards.
+
+`GET /admin/db/statements` returns `{"items":[...]}`, highest total execution time first. Each
+item has `name` (the statement name; statements without one are summed as `(unnamed)`), `calls`,
+`exec_ms`, `plan_ms`, `shared_hit` and `shared_read` (buffer blocks), `rows` and `wal_bytes`.
+Times are milliseconds rounded to microseconds. `plan_ms` stays 0 unless `pg_stat_statements.track_planning` is on.
+Counters run from the last `pg_stat_statements` reset. Setup is in
+[Operations](operations.md#statement-statistics).
 
 `POST /admin/accounts` takes `{"name": "..."}` in a body of at most 4 KiB. The name is trimmed,
 case-sensitive, 1 to 128 characters with no control characters, and unique among active
diff --git a/docs/backup-export.md b/docs/backup-export.md
index 0bea075..991ca3e 100644
--- a/docs/backup-export.md
+++ b/docs/backup-export.md
@@ -122,6 +122,16 @@ needed extensions:
 psql -X --set ON_ERROR_STOP=on --single-transaction --file database.sql
 ```
 
+Dumps of a database where migration 010 created `pg_stat_statements` contain
+`CREATE EXTENSION IF NOT EXISTS pg_stat_statements` and a `COMMENT ON EXTENSION
+pg_stat_statements` line. Restoring as a superuser needs nothing extra. As any other role,
+with `ON_ERROR_STOP` the first stops the restore unless a superuser has created the extension
+in the new database first, and the comment stops it even then, because only the extension's
+owner may comment on it. So delete both lines from `database.sql`, restore, and have a
+superuser run `CREATE EXTENSION pg_stat_statements;` afterwards if you want statement
+statistics. They also need `shared_preload_libraries` on the new server; see
+[Statement statistics](operations.md#statement-statistics).
+
 Check the migration journal and some records, review the config, and restore external
 secrets and files before pointing a new Beacon at it. Never test an export by restoring over
 live data. See [pg_dump](https://www.postgresql.org/docs/16/app-pgdump.html) and
diff --git a/docs/configuration.md b/docs/configuration.md
index 3dcd049..6198622 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -67,8 +67,10 @@ which ones to read.
 |---|---|---|
 | `auth` | The admin API key, if you would rather keep it in the file than in `BEACON_API_KEY`. | You want the admin endpoints. |
 | `log` | Log level and format. | You want JSON logs for a collector. |
-| `server` | `trusted_proxies`: the proxy addresses allowed to tell Beacon the real client IP. | Always, if Beacon sits behind a proxy. See [Reverse proxy](reverse-proxy.md). |
+| `server` | `trusted_proxies`: the proxy addresses allowed to tell Beacon the real client IP. `admin_listen`: an optional private listener for Prometheus `/metrics`. | Always, if Beacon sits behind a proxy. See [Reverse proxy](reverse-proxy.md). |
 | `ratelimit` | Per-client REST limits for `/api/v1/*`. On by default at 600 requests a minute. | You get 429s you did not expect. |
+| `database` | `api_plan_layout`: how API statements are spread over database connection pools. | You check `max_connections`, or need the rollback layout. See [Database pools and cost guards](#database-pools-and-cost-guards). |
+| `guards` | Which request families the cost guards may refuse. All of them only observe by default. | After a week of observing. See [Database pools and cost guards](#database-pools-and-cost-guards). |
 | `meshmapper` | Import transport scopes, IATA borders, region groups and public channels from MeshMapper instead of listing them by hand, and add repeater site details to node detail. Requires a [MeshMapper API key](#meshmapper-api-key). | You are in a region MeshMapper covers. |
 | `iatas` | Display names, coordinates and optional border files for airport codes. IATAs are created automatically when traffic arrives; this block only decorates them. A `borderFile` path is relative to `config.yaml`; in Docker the compose file mounts only `config.yaml`, so add a mount for the folder too (`- ./data/app/borders:/app/borders:ro` under the `app` service). | You want names on the map or a border drawn. |
 | `regions` | Groups of IATAs with a name, map centre and zoom. | Always, unless MeshMapper imports them. |
@@ -107,11 +109,71 @@ cannot honour. These are the ones people hit:
 - `log.level` or `log.format` set to anything other than the listed values.
 - `mobile.min_app_version` or `web.min_web_version` that is not a plain `X.Y.Z` release (`v1.2.3` and `1.2.3-beta` are rejected).
 - An admin key under 16 characters or containing whitespace.
+- `database.api_plan_layout` other than `custom_all`, `split` or `single`; a `guards.enforce`
+  entry that is not a family name; `guards.budget` other than `shadow`; `server.admin_listen`
+  that is not `host:port` with a port from 1 to 65535.
 - A negative `ratelimit.requests_per_minute`, `ratelimit.burst`, `websocket.max_connects_per_minute`
   or `routes.reconfirm_rows_per_hour`.
   A negative `websocket.max_connections_per_ip` is not caught at startup and rejects every
   connection instead, so leave it at a positive number.
 
+## Database pools and cost guards
+
+Beacon opens one PostgreSQL connection pool per workload. Pools open connections only when
+needed, so these are peaks:
+
+| `database.api_plan_layout` | `api_legacy` | `api_custom` | `api_cached` | `ingest` | `maint` | Total |
+|---|---:|---:|---:|---:|---:|---:|
+| `custom_all` (default) | 8 | 8 | - | 6 | 4 | 26 |
+| `split` | 8 | 7 | 5 | 6 | 4 | 30 |
+| `single` (rollback) | 8 | - | - | 6 | 4 | 18 |
+
+`ingest` has one connection per CPU on hosts with more than 6 CPUs, which adds to each total.
+The startup log line `database pools open` shows the real figure. Postgres `max_connections`
+must cover it plus every other client. A `pool_max_conns` in `POSTGRES_DSN` is ignored; the
+layout sets the sizes.
+
+- **`custom_all`** plans every API statement for its own parameters on every call, so no
+  statement can switch to a slow generic plan. Use it.
+- **`split`** would run a list of proven statements on cached generic plans. That list is empty
+  in this release, so `split` only opens a pool nothing uses. Keep `custom_all`.
+- **`single`** is the rollback: one API pool on the previous session settings, and every guard
+  family only observes, whatever `guards.enforce` says.
+
+The layout changes at a restart. What each pool sets is in
+[Operations](operations.md#database-pools-and-connections).
+
+```yaml
+database:
+  api_plan_layout: custom_all
+
+guards:
+  enforce: []      # routes, channel_messages, packets_global, packets_region,
+                   # packets_window, packet_detail, stats, other
+  budget: shadow
+```
+
+The cost guards cover public `/api/v1` requests, not `/ws` or `/api/v1/admin/*`. Each request
+belongs to a family and a class (the table is in the
+[API contract](api-contract.md#cost-guards)). Per family, three guards decide whether a request
+may use the database: at most 4 database-backed requests at once per client IP, waiting up to
+5 s for a slot; a database connection within 5 s; and a database time limit of 5 s (entity),
+10 s (list) or 30 s (aggregation). A family listed in `guards.enforce` refuses with `429` or
+`503`. Every other family only counts what it would have refused, in
+`beacon_guard_would_refuse_total`. The default `[]` refuses nothing.
+
+`guards.enforce` is the boot default. `PUT /api/v1/admin/config` replaces it at runtime, and a
+restart goes back to the file, so make the same change in both. When to enforce a family is in
+[Operations](operations.md#enforcing-a-guard-family).
+
+`guards.budget` must be `shadow`. The per-client database time budget (30 s, refilling at one
+second per second) only counts; it never refuses.
+
+`server.admin_listen` (default empty, off) opens a second listener, for example
+`127.0.0.1:9090`, that serves only `GET /metrics`. It has no authentication, so bind it to
+loopback or a private network and never publish it through the proxy. See
+[Operations](operations.md#metrics).
+
 ## MeshMapper API key
 
 MeshMapper APIs require either a regional API key or a grouped-region API key covering
diff --git a/docs/operations.md b/docs/operations.md
index 2849d94..c0d96f0 100644
--- a/docs/operations.md
+++ b/docs/operations.md
@@ -260,7 +260,9 @@ Notes:
 - `max_wal_size = '16GB'` lets WAL grow that far between checkpoints. Make sure the volume
   under `data/postgres/` has the room, or pick a smaller value on a small host.
 - `pg_stat_statements` only collects once it is in `shared_preload_libraries`, which needs a
-  restart. If it is not there yet, add it at the restart in the next section.
+  restart. If it is not there yet, add it at the restart in the next section. Beacon's
+  migration 010 creates the extension itself where it can; see
+  [Statement statistics](#statement-statistics).
 - `DROP INDEX CONCURRENTLY` cannot run inside a transaction block; run it on its own in
   autocommit mode, as `psql` does by default. It takes a brief `SHARE UPDATE EXCLUSIVE` lock
   and waits for older transactions, so run the pre-deploy check above first. It frees about
@@ -363,6 +365,254 @@ SELECT indexrelid::regclass FROM pg_index WHERE NOT indisvalid;
 DROP INDEX CONCURRENTLY IF EXISTS idx_known_routes_keyset_ccnew;       -- or idx_known_routes_iata_keyset_ccnew
 ```
 
+## Database pools and connections
+
+Beacon opens a separate connection pool per workload, sized by `database.api_plan_layout`
+(see [Configuration](configuration.md#database-pools-and-cost-guards)). Each pool sets its own
+session settings and an `application_name`, so `pg_stat_activity` shows which one a
+connection belongs to:
+
+| Pool | `application_name` | Size | Session settings |
+|---|---|---:|---|
+| `api_legacy` | `beacon-api_legacy` | 8 | Server defaults. Runs the few API statements that keep the previous plan behaviour. |
+| `api_custom` | `beacon-api_custom` | 8 (7 under `split`) | `plan_cache_mode=force_custom_plan`, `jit=off`, `max_parallel_workers_per_gather=0`, `statement_timeout=35s`, `idle_in_transaction_session_timeout=30s` |
+| `api_cached` | `beacon-api_cached` | 5, `split` only | As `api_custom`, with `plan_cache_mode=force_generic_plan` |
+| `ingest` | `beacon-ingest` | 6, or one per CPU above 6 | `plan_cache_mode=auto`, `jit=off`, no statement timeout: an ingest write waits out a lock rather than drop a packet |
+| `maint` | `beacon-maint` | 4 | `plan_cache_mode=force_custom_plan`, `jit=off`, `max_parallel_workers_per_gather=0`. Background jobs, importers, backups and admin queries. |
+
+The `single` layout opens only `api_legacy`, `ingest` and `maint`.
+
+- **Headroom.** `SHOW max_connections;` must leave room for the total (26 by default on up to
+  6 CPUs) plus every other client: `psql` sessions, backups, monitoring. If your deploy starts
+  the new server before the old one stops, count both.
+- **Checking.** Parallel workers carry their leader's `application_name`, so count client
+  backends only:
+
+  ```sql
+  SELECT application_name, count(*) FROM pg_stat_activity
+  WHERE application_name LIKE 'beacon-%' AND backend_type = 'client backend' GROUP BY 1;
+  ```
+
+  No pool should go above its size, and `beacon-api_cached` should be absent under
+  `custom_all`.
+- **`pool_max_conns`.** A `pool_max_conns` in `POSTGRES_DSN` is overridden by the pool sizes;
+  the server logs that at startup.
+- **PgBouncer.** The session settings are sent as startup parameters. Behind PgBouncer,
+  `ignore_startup_parameters` must list `plan_cache_mode`, `jit`, `statement_timeout`,
+  `idle_in_transaction_session_timeout` and `max_parallel_workers_per_gather`, or connections
+  are refused. PgBouncer then drops them, so the pools run on the server defaults; connect
+  Beacon to Postgres directly where you can.
+- **Background jobs** share the 4 `maint` connections without parallel plans, so a job may
+  take a little longer than before. `beacon_job_duration_seconds` shows it.
+
+### Small hosts
+
+Each hourly analytics rollup sets `work_mem` to 256 MB for its own transaction, and Postgres
+may use that much for every sort or hash step in it. On a host with little memory, watch the
+`db` container's memory while rollups run (about 95 minutes past each hour).
+
+## Statement statistics
+
+`GET /api/v1/admin/db/statements` sums `pg_stat_statements` by statement name (format in the
+[API contract](api-contract.md#admin)). It needs the extension created and preloaded:
+
+- **Created.** Migration `010_pg_stat_statements.sql` creates the extension when the server
+  offers it and the database role may. If not, it logs a notice and is still recorded, so
+  startup never fails over it. The shipped stacks connect as the superuser, so it is created.
+  Otherwise run `CREATE EXTENSION pg_stat_statements;` as a superuser.
+- **Preloaded.** It only collects once `shared_preload_libraries` includes it, which needs a
+  Postgres restart. Add it to the `db` service's command at the next restart (see
+  [Settings that need a restart](#settings-that-need-a-restart)), keeping any `-c` values
+  already there:
+
+  ```yaml
+    db:
+      command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements"]
+  ```
+
+Until both are true the endpoint returns `503`. Check with `SHOW shared_preload_libraries;`
+and a request to the endpoint with the admin key.
+
+Planning time (`plan_ms`) is only recorded while `pg_stat_statements.track_planning` is on. It
+adds overhead, so turn it on for one day to measure, then off again; both apply on reload:
+
+```sql
+ALTER SYSTEM SET pg_stat_statements.track_planning = on;
+SELECT pg_reload_conf();
+-- a day later
+ALTER SYSTEM RESET pg_stat_statements.track_planning;
+SELECT pg_reload_conf();
+```
+
+API statements are planned on every call, so summed `plan_ms` should stay a small fraction of
+summed `exec_ms`, under about 7%.
+
+## Metrics
+
+With `server.admin_listen` set, the server opens a second listener that serves only
+`GET /metrics` in Prometheus format. It has no authentication. `/metrics` is not served on the
+public port.
+
+In the Docker stack the listener runs inside the `app` container, so bind it to all of the
+container's interfaces and publish it on the host's loopback only:
+
+```yaml
+# config.yaml
+server:
+  admin_listen: "0.0.0.0:9090"
+```
+
+```yaml
+# docker-compose.yml, app service
+    ports:
+      - "127.0.0.1:8080:8080"
+      - "127.0.0.1:9090:9090"
+```
+
+Never route it through Caddy. Scrape it from the host:
+
+```yaml
+scrape_configs:
+  - job_name: beacon
+    static_configs:
+      - targets: ["127.0.0.1:9090"]
+```
+
+| Metric | Type | Labels | What it is |
+|---|---|---|---|
+| `beacon_http_request_seconds` | histogram | `route`, `family` | HTTP request time by route pattern |
+| `beacon_db_time_seconds` | histogram | `statement`, `pool` | Time per database statement |
+| `beacon_pool_acquire_seconds` | histogram | `pool` | Wait for a pooled connection |
+| `beacon_guard_db_seconds` | histogram | `family`, `class` | Database time per guarded request |
+| `beacon_guard_would_refuse_total` | counter | `family`, `guard` | Requests a guard would refuse if its family enforced |
+| `beacon_guard_refused_total` | counter | `family`, `guard` | Requests a guard refused |
+| `beacon_ingest_queue_depth` | gauge | | Messages waiting in the ingest queue |
+| `beacon_ingest_dropped_total` | counter | | Messages dropped because the ingest queue was full |
+| `beacon_job_duration_seconds` | histogram | `job` | Background job run time (`cleanup`, `analytics_rollup`, `reconfirm`, `route_retention`, ...) |
+| `beacon_job_rows_total` | counter | `job` | Rows removed by background jobs, today `reconfirm` and `route_retention` |
+| `beacon_index_bytes` | gauge | `index` | Size of each index, hourly |
+| `beacon_index_bloat_ratio` | gauge | `index` | Bytes per row against the last rebuild, hourly. See [Index health](#index-health). |
+| `beacon_pg_wal_bytes_total` | counter | | WAL written, cluster-wide |
+| `beacon_pg_checkpoints_total` | counter | `kind` | Checkpoints, `timed` or `requested` |
+| `beacon_pg_xid_age` | gauge | | Age of the oldest transaction ID in the database |
+
+`guard` is `deadline`, `cap`, `admission` or `budget`. The Postgres counters are read once a
+minute; the index gauges first appear an hour after startup. Go runtime and process metrics
+are included too.
+
+### Example alert rules
+
+```yaml
+groups:
+  - name: beacon
+    rules:
+      - alert: BeaconGuardWouldRefuse          # any would-refuse delays enforcing that family
+        expr: sum by (family, guard) (increase(beacon_guard_would_refuse_total[1h])) > 0
+      - alert: BeaconDeadlineHits
+        expr: sum(increase(beacon_guard_refused_total{guard="deadline"}[1h])) > 20
+      - alert: BeaconListSlow                  # list-class database time p99 over 1 s
+        expr: histogram_quantile(0.99, sum by (le) (rate(beacon_guard_db_seconds_bucket{class="list"}[15m]))) > 1
+        for: 15m
+      - alert: BeaconPoolAcquireSlow
+        expr: histogram_quantile(0.99, sum by (le, pool) (rate(beacon_pool_acquire_seconds_bucket[5m]))) > 0.5
+        for: 10m
+      - alert: BeaconCleanupLong               # a cleanup run over 10 min in the last 2 h
+        expr: increase(beacon_job_duration_seconds_count{job="cleanup"}[2h]) - increase(beacon_job_duration_seconds_bucket{job="cleanup",le="600"}[2h]) > 0
+      - alert: BeaconKeysetBloat
+        expr: beacon_index_bloat_ratio{index=~"idx_known_routes_(iata_)?keyset"} > 4
+      - alert: BeaconCheckpointsRequested
+        expr: increase(beacon_pg_checkpoints_total{kind="requested"}[1d]) > increase(beacon_pg_checkpoints_total{kind="timed"}[1d])
+      - alert: BeaconWALAboveBaseline          # set the baseline from your own steady state
+        expr: rate(beacon_pg_wal_bytes_total[1h]) > <baseline bytes/s>
+      - alert: BeaconXIDAge
+        expr: beacon_pg_xid_age > 300e6
+```
+
+Replace `<baseline bytes/s>` with your normal WAL rate before loading the rules. The route
+recheck has no metric yet; at `debug` level it logs a `route reconfirm` line per slice, and
+`wrapped=true` marks the end of a full cycle.
+
+## Enforcing a guard family
+
+Every guard family only observes until you enforce it (see
+[Configuration](configuration.md#database-pools-and-cost-guards)). Observe for 7 days with
+metrics on, then enforce a family only when all three hold for it over those 7 days. With
+`routes` as the example (Prometheus needs at least 7 days of retention):
+
+1. No would-refuse from any guard. Every series must be 0:
+
+   ```promql
+   sum by (guard) (increase(beacon_guard_would_refuse_total{family="routes"}[7d]))
+   ```
+
+2. The p99 database time of each class is at most a quarter of its limit: entity 1.25 s, list
+   2.5 s, aggregation 7.5 s.
+
+   ```promql
+   histogram_quantile(0.99, sum by (le, class) (rate(beacon_guard_db_seconds_bucket{family="routes"}[7d])))
+   ```
+
+3. No request used more than half its limit. Histograms have no exact maximum, so check that
+   no request landed above the bucket at or under half the limit: `le="2.5"` for entity,
+   `le="5"` for list, `le="10"` for aggregation (stricter than 15 s, which has no bucket). Each
+   must be 0:
+
+   ```promql
+   sum(increase(beacon_guard_db_seconds_count{family="routes",class="list"}[7d]))
+     - sum(increase(beacon_guard_db_seconds_bucket{family="routes",class="list",le="5"}[7d]))
+   ```
+
+Then enforce it at runtime, listing every family that should enforce (the list replaces the
+current one):
+
+```bash
+curl -s -X PUT https://<your-domain>/api/v1/admin/config \
+  -H "Authorization: Bearer $BEACON_API_KEY" -H 'Content-Type: application/json' \
+  -d '{"guards":{"enforce":["other","routes"]}}'
+```
+
+The response's `config.guards.enforce` shows the new list. Add the same list to
+`guards.enforce` in `config.yaml`, or the next restart goes back to the old one. To stop
+enforcing, `PUT` the list without that family (`[]` for none). `beacon_guard_refused_total`
+then counts real refusals.
+
+`other` and `routes` are the families to start with. The rest are not ready in this release:
+`channel_messages` statements are not guarded yet, and the packet and stats families still
+run statements that later releases make cheaper. Their release notes will say when each is
+ready, roughly `channel_messages` first, then `packets_global`, then `packets_region` and
+`packets_window`, then `stats` and `packet_detail`. `packets_window` may stay observed for
+good.
+
+## Index health
+
+Once an hour the server records each index's size (`beacon_index_bytes`) and its bytes per
+row against a baseline (`beacon_index_bloat_ratio`). The baseline is taken the first time the
+server sees the index and again whenever it is rebuilt, so the ratio is growth since the last
+rebuild, not absolute bloat: 1 means as compact as then, 4 means four times the bytes per
+row. An index that has never been analyzed has no ratio.
+
+The first run, and then one run a week, also logs `indexes with no scans since the last
+statistics reset` with each unused non-unique index and its size. An index that stays on that
+list for weeks is a candidate to drop; check that no rare job reads it first.
+
+When `BeaconKeysetBloat` fires, rebuild that index as in
+[Rebuilding the route keyset indexes](#rebuilding-the-route-keyset-indexes). The rebuild resets
+its baseline, and the ratio drops back to about 1 on the next hourly run.
+
+## Rolling back the pool layout
+
+If the new pools or plans misbehave, set this and restart the server:
+
+```yaml
+database:
+  api_plan_layout: single
+```
+
+It puts every API statement on one pool with the previous session settings and makes every
+guard family observe, whatever `guards.enforce` says. The alternative is the previous server
+image. It ignores migration 010; the `pg_stat_statements` extension stays installed and
+harmless.
+
 ## CPU profiling
 
 Bounded CPU captures on a running instance are described in [Profiling](profiling.md).