diff --git a/db/migrations/013_packets_scope_index.sql b/db/migrations/013_packets_scope_index.sql new file mode 100644 index 0000000..73029cf --- /dev/null +++ b/db/migrations/013_packets_scope_index.sql @@ -0,0 +1,3 @@ +-- Scope stats count off scope_id, which had no index. + +CREATE INDEX idx_packets_scope ON packets(scope_id) WHERE scope_id IS NOT NULL; diff --git a/db/queries/queries.sql b/db/queries/queries.sql index 4f1c265..bb8008b 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -917,16 +917,14 @@ WHERE ($1::text = '' OR preset = $1::text) ORDER BY preset, iata, source_type; -- name: GetScopeStats :many +-- Count each table on its own; the old cross-join blew up to millions of rows +-- before COUNT(DISTINCT) (~10s). SELECT ts.name, - COUNT(DISTINCT p.packet_hash) AS packet_count, - COUNT(DISTINCT os.observer_id) AS observer_count, - COUNT(DISTINCT n.id) AS node_count + (SELECT COUNT(*) FROM packets p WHERE p.scope_id = ts.id) AS packet_count, + (SELECT COUNT(*) FROM observer_scopes os WHERE os.scope_id = ts.id) AS observer_count, + (SELECT COUNT(*) FROM nodes n WHERE n.default_scope_id = ts.id) AS node_count FROM transport_scopes ts -LEFT JOIN packets p ON p.scope_id = ts.id -LEFT JOIN observer_scopes os ON os.scope_id = ts.id -LEFT JOIN nodes n ON n.default_scope_id = ts.id -GROUP BY ts.name ORDER BY ts.name; -- ============================================================ diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index fa93a88..efb1ba2 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -50,6 +50,8 @@ type Querier interface { GetRegionIATAs(ctx context.Context, regionID int32) ([]string, error) GetScopeByName(ctx context.Context, name string) (GetScopeByNameRow, error) GetScopeNames(ctx context.Context) ([]string, error) + // Count each table on its own; the old cross-join blew up to millions of rows + // before COUNT(DISTINCT) (~10s). GetScopeStats(ctx context.Context) ([]GetScopeStatsRow, error) GetScopesByIATAs(ctx context.Context, dollar_1 []string) ([]GetScopesByIATAsRow, error) // Returns node counts grouped by type, optionally filtered by IATA. diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 1698cbe..787f5cd 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -1035,14 +1035,10 @@ func (q *Queries) GetScopeNames(ctx context.Context) ([]string, error) { const getScopeStats = `-- name: GetScopeStats :many SELECT ts.name, - COUNT(DISTINCT p.packet_hash) AS packet_count, - COUNT(DISTINCT os.observer_id) AS observer_count, - COUNT(DISTINCT n.id) AS node_count + (SELECT COUNT(*) FROM packets p WHERE p.scope_id = ts.id) AS packet_count, + (SELECT COUNT(*) FROM observer_scopes os WHERE os.scope_id = ts.id) AS observer_count, + (SELECT COUNT(*) FROM nodes n WHERE n.default_scope_id = ts.id) AS node_count FROM transport_scopes ts -LEFT JOIN packets p ON p.scope_id = ts.id -LEFT JOIN observer_scopes os ON os.scope_id = ts.id -LEFT JOIN nodes n ON n.default_scope_id = ts.id -GROUP BY ts.name ORDER BY ts.name ` @@ -1053,6 +1049,8 @@ type GetScopeStatsRow struct { NodeCount int64 `json:"node_count"` } +// Count each table on its own; the old cross-join blew up to millions of rows +// before COUNT(DISTINCT) (~10s). func (q *Queries) GetScopeStats(ctx context.Context) ([]GetScopeStatsRow, error) { rows, err := q.db.Query(ctx, getScopeStats) if err != nil {