mirror of
https://github.com/MeshCore-Beacon/beacon-server.git
synced 2026-09-01 16:48:19 +00:00
perf(scopes): stop cross-joining packets in scope stats
GetScopeStats left-joined packets, observer_scopes and nodes at once, multiplying rows into the millions before COUNT(DISTINCT) deduped them (~10s per call). Count each table on its own and index packets(scope_id).
This commit is contained in:
@@ -0,0 +1,3 @@
|
||||
-- Scope stats count off scope_id, which had no index.
|
||||
|
||||
CREATE INDEX idx_packets_scope ON packets(scope_id) WHERE scope_id IS NOT NULL;
|
||||
@@ -917,16 +917,14 @@ WHERE ($1::text = '' OR preset = $1::text)
|
||||
ORDER BY preset, iata, source_type;
|
||||
|
||||
-- name: GetScopeStats :many
|
||||
-- Count each table on its own; the old cross-join blew up to millions of rows
|
||||
-- before COUNT(DISTINCT) (~10s).
|
||||
SELECT
|
||||
ts.name,
|
||||
COUNT(DISTINCT p.packet_hash) AS packet_count,
|
||||
COUNT(DISTINCT os.observer_id) AS observer_count,
|
||||
COUNT(DISTINCT n.id) AS node_count
|
||||
(SELECT COUNT(*) FROM packets p WHERE p.scope_id = ts.id) AS packet_count,
|
||||
(SELECT COUNT(*) FROM observer_scopes os WHERE os.scope_id = ts.id) AS observer_count,
|
||||
(SELECT COUNT(*) FROM nodes n WHERE n.default_scope_id = ts.id) AS node_count
|
||||
FROM transport_scopes ts
|
||||
LEFT JOIN packets p ON p.scope_id = ts.id
|
||||
LEFT JOIN observer_scopes os ON os.scope_id = ts.id
|
||||
LEFT JOIN nodes n ON n.default_scope_id = ts.id
|
||||
GROUP BY ts.name
|
||||
ORDER BY ts.name;
|
||||
|
||||
-- ============================================================
|
||||
|
||||
@@ -50,6 +50,8 @@ type Querier interface {
|
||||
GetRegionIATAs(ctx context.Context, regionID int32) ([]string, error)
|
||||
GetScopeByName(ctx context.Context, name string) (GetScopeByNameRow, error)
|
||||
GetScopeNames(ctx context.Context) ([]string, error)
|
||||
// Count each table on its own; the old cross-join blew up to millions of rows
|
||||
// before COUNT(DISTINCT) (~10s).
|
||||
GetScopeStats(ctx context.Context) ([]GetScopeStatsRow, error)
|
||||
GetScopesByIATAs(ctx context.Context, dollar_1 []string) ([]GetScopesByIATAsRow, error)
|
||||
// Returns node counts grouped by type, optionally filtered by IATA.
|
||||
|
||||
@@ -1035,14 +1035,10 @@ func (q *Queries) GetScopeNames(ctx context.Context) ([]string, error) {
|
||||
const getScopeStats = `-- name: GetScopeStats :many
|
||||
SELECT
|
||||
ts.name,
|
||||
COUNT(DISTINCT p.packet_hash) AS packet_count,
|
||||
COUNT(DISTINCT os.observer_id) AS observer_count,
|
||||
COUNT(DISTINCT n.id) AS node_count
|
||||
(SELECT COUNT(*) FROM packets p WHERE p.scope_id = ts.id) AS packet_count,
|
||||
(SELECT COUNT(*) FROM observer_scopes os WHERE os.scope_id = ts.id) AS observer_count,
|
||||
(SELECT COUNT(*) FROM nodes n WHERE n.default_scope_id = ts.id) AS node_count
|
||||
FROM transport_scopes ts
|
||||
LEFT JOIN packets p ON p.scope_id = ts.id
|
||||
LEFT JOIN observer_scopes os ON os.scope_id = ts.id
|
||||
LEFT JOIN nodes n ON n.default_scope_id = ts.id
|
||||
GROUP BY ts.name
|
||||
ORDER BY ts.name
|
||||
`
|
||||
|
||||
@@ -1053,6 +1049,8 @@ type GetScopeStatsRow struct {
|
||||
NodeCount int64 `json:"node_count"`
|
||||
}
|
||||
|
||||
// Count each table on its own; the old cross-join blew up to millions of rows
|
||||
// before COUNT(DISTINCT) (~10s).
|
||||
func (q *Queries) GetScopeStats(ctx context.Context) ([]GetScopeStatsRow, error) {
|
||||
rows, err := q.db.Query(ctx, getScopeStats)
|
||||
if err != nil {
|
||||
|
||||
Reference in New Issue
Block a user