perf(scopes): stop cross-joining packets in scope stats

GetScopeStats left-joined packets, observer_scopes and nodes at once,
multiplying rows into the millions before COUNT(DISTINCT) deduped them
(~10s per call). Count each table on its own and index packets(scope_id).
This commit is contained in:
MrAlders0n
2026-07-24 12:07:05 -07:00
committed by Ded
parent a84112f7fb
commit 42edcdeb7b
4 changed files with 15 additions and 14 deletions
@@ -0,0 +1,3 @@
-- Scope stats count off scope_id, which had no index.
CREATE INDEX idx_packets_scope ON packets(scope_id) WHERE scope_id IS NOT NULL;
+5 -7
View File
@@ -917,16 +917,14 @@ WHERE ($1::text = '' OR preset = $1::text)
ORDER BY preset, iata, source_type;
-- name: GetScopeStats :many
-- Count each table on its own; the old cross-join blew up to millions of rows
-- before COUNT(DISTINCT) (~10s).
SELECT
ts.name,
COUNT(DISTINCT p.packet_hash) AS packet_count,
COUNT(DISTINCT os.observer_id) AS observer_count,
COUNT(DISTINCT n.id) AS node_count
(SELECT COUNT(*) FROM packets p WHERE p.scope_id = ts.id) AS packet_count,
(SELECT COUNT(*) FROM observer_scopes os WHERE os.scope_id = ts.id) AS observer_count,
(SELECT COUNT(*) FROM nodes n WHERE n.default_scope_id = ts.id) AS node_count
FROM transport_scopes ts
LEFT JOIN packets p ON p.scope_id = ts.id
LEFT JOIN observer_scopes os ON os.scope_id = ts.id
LEFT JOIN nodes n ON n.default_scope_id = ts.id
GROUP BY ts.name
ORDER BY ts.name;
-- ============================================================
+2
View File
@@ -50,6 +50,8 @@ type Querier interface {
GetRegionIATAs(ctx context.Context, regionID int32) ([]string, error)
GetScopeByName(ctx context.Context, name string) (GetScopeByNameRow, error)
GetScopeNames(ctx context.Context) ([]string, error)
// Count each table on its own; the old cross-join blew up to millions of rows
// before COUNT(DISTINCT) (~10s).
GetScopeStats(ctx context.Context) ([]GetScopeStatsRow, error)
GetScopesByIATAs(ctx context.Context, dollar_1 []string) ([]GetScopesByIATAsRow, error)
// Returns node counts grouped by type, optionally filtered by IATA.
+5 -7
View File
@@ -1035,14 +1035,10 @@ func (q *Queries) GetScopeNames(ctx context.Context) ([]string, error) {
const getScopeStats = `-- name: GetScopeStats :many
SELECT
ts.name,
COUNT(DISTINCT p.packet_hash) AS packet_count,
COUNT(DISTINCT os.observer_id) AS observer_count,
COUNT(DISTINCT n.id) AS node_count
(SELECT COUNT(*) FROM packets p WHERE p.scope_id = ts.id) AS packet_count,
(SELECT COUNT(*) FROM observer_scopes os WHERE os.scope_id = ts.id) AS observer_count,
(SELECT COUNT(*) FROM nodes n WHERE n.default_scope_id = ts.id) AS node_count
FROM transport_scopes ts
LEFT JOIN packets p ON p.scope_id = ts.id
LEFT JOIN observer_scopes os ON os.scope_id = ts.id
LEFT JOIN nodes n ON n.default_scope_id = ts.id
GROUP BY ts.name
ORDER BY ts.name
`
@@ -1053,6 +1049,8 @@ type GetScopeStatsRow struct {
NodeCount int64 `json:"node_count"`
}
// Count each table on its own; the old cross-join blew up to millions of rows
// before COUNT(DISTINCT) (~10s).
func (q *Queries) GetScopeStats(ctx context.Context) ([]GetScopeStatsRow, error) {
rows, err := q.db.Query(ctx, getScopeStats)
if err != nil {