mirror of
https://github.com/MeshCore-Beacon/beacon-server.git
synced 2026-10-23 01:51:27 +00:00
feat(stats): serve historical stats from hourly rollups
Stats endpoints read hourly rollup tables filled once per closed hour by one advisory-locked task, instead of materialized views that re-scanned every observation on each refresh. IATA-set tables give exact distinct counts for any IATA set or region. Cleanup holds raw rows for unrolled hours (up to 24h). Adds /stats/series; ingest keeps packet observation counts, observer last IATA, channel message counts and trace tag summaries. 001_baseline.sql is rewritten, so pre-release 2.0 databases must be recreated. See docs/historical-stats.md.
This commit is contained in:
MrAlders0n authored and MrAlders0n committed
1 parent
6cd03e73da
commit
98006a9364
112 files changed
+6766
-2828
No files matched your search
@@ -0,0 +1,169 @@
|
||||
// Code generated by sqlc. DO NOT EDIT.
|
||||
// versions:
|
||||
// sqlc v1.31.1
|
||||
// source: series.sql
|
||||
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
)
|
||||
|
||||
const getEarliestCompleteRollupHour = `-- name: GetEarliestCompleteRollupHour :one
|
||||
SELECT min(hour)::timestamptz FROM analytics_rollup_hours WHERE status = 'complete'
|
||||
`
|
||||
|
||||
func (q *Queries) GetEarliestCompleteRollupHour(ctx context.Context) (pgtype.Timestamptz, error) {
|
||||
row := q.db.QueryRow(ctx, getEarliestCompleteRollupHour)
|
||||
var column_1 pgtype.Timestamptz
|
||||
err := row.Scan(&column_1)
|
||||
return column_1, err
|
||||
}
|
||||
|
||||
const getStatsSeries = `-- name: GetStatsSeries :many
|
||||
WITH hours AS (
|
||||
SELECT g.h::timestamptz AS hour, COALESCE(r.status, 'missing')::text AS status
|
||||
FROM generate_series($1::timestamptz, $2::timestamptz, INTERVAL '1 hour') g(h)
|
||||
LEFT JOIN analytics_rollup_hours r ON r.hour = g.h
|
||||
WHERE g.h < $2::timestamptz
|
||||
), complete AS (
|
||||
SELECT hour FROM hours WHERE status = 'complete'
|
||||
), obs AS (
|
||||
SELECT o.hour, o.iata, o.observation_count FROM analytics_hourly_iata_observations o JOIN complete c ON c.hour = o.hour
|
||||
WHERE COALESCE(cardinality($3::bpchar[]), 0) = 0
|
||||
UNION ALL
|
||||
SELECT o.hour, o.iata, o.observation_count FROM analytics_hourly_iata_observations o JOIN complete c ON c.hour = o.hour
|
||||
WHERE cardinality($3::bpchar[]) > 0 AND o.iata = ANY($3::bpchar[])
|
||||
), obs_agg AS (
|
||||
SELECT hour, sum(observation_count)::bigint AS observations, count(DISTINCT iata)::bigint AS active_iatas
|
||||
FROM obs GROUP BY GROUPING SETS ((), (hour))
|
||||
), pkt AS (
|
||||
SELECT s.hour, s.packets FROM analytics_hourly_packet_sets s JOIN complete c ON c.hour = s.hour
|
||||
WHERE COALESCE(cardinality($3::bpchar[]), 0) = 0
|
||||
UNION ALL
|
||||
SELECT s.hour, s.packets FROM analytics_hourly_packet_sets s JOIN complete c ON c.hour = s.hour
|
||||
WHERE cardinality($3::bpchar[]) > 0 AND s.iatas && $3::bpchar[]
|
||||
), pkt_agg AS (
|
||||
SELECT hour, sum(packets)::bigint AS unique_packets FROM pkt GROUP BY GROUPING SETS ((), (hour))
|
||||
), obsr AS (
|
||||
SELECT i.hour, i.observer_id FROM analytics_hourly_observer_identity i JOIN complete c ON c.hour = i.hour
|
||||
WHERE COALESCE(cardinality($3::bpchar[]), 0) = 0
|
||||
UNION ALL
|
||||
SELECT i.hour, i.observer_id FROM analytics_hourly_observer_identity i JOIN complete c ON c.hour = i.hour
|
||||
WHERE cardinality($3::bpchar[]) > 0 AND i.iata = ANY($3::bpchar[])
|
||||
), obsr_agg AS (
|
||||
SELECT hour, count(DISTINCT observer_id)::bigint AS active_observers FROM obsr GROUP BY GROUPING SETS ((), (hour))
|
||||
), scp AS (
|
||||
SELECT s.hour, s.scope_id, s.packets FROM analytics_hourly_scope_sets s JOIN complete c ON c.hour = s.hour
|
||||
WHERE COALESCE(cardinality($3::bpchar[]), 0) = 0
|
||||
UNION ALL
|
||||
SELECT s.hour, s.scope_id, s.packets FROM analytics_hourly_scope_sets s JOIN complete c ON c.hour = s.hour
|
||||
WHERE cardinality($3::bpchar[]) > 0 AND s.iatas && $3::bpchar[]
|
||||
), scp_agg AS (
|
||||
SELECT hour, sum(packets)::bigint AS scoped_packets, count(DISTINCT scope_id)::bigint AS active_scopes
|
||||
FROM scp GROUP BY GROUPING SETS ((), (hour))
|
||||
), pth AS (
|
||||
SELECT p.hour, p.entries FROM analytics_hourly_paths p JOIN complete c ON c.hour = p.hour
|
||||
WHERE p.category = 0 AND COALESCE(cardinality($3::bpchar[]), 0) = 0
|
||||
UNION ALL
|
||||
SELECT p.hour, p.entries FROM analytics_hourly_paths p JOIN complete c ON c.hour = p.hour
|
||||
WHERE p.category = 0 AND cardinality($3::bpchar[]) > 0 AND p.iata = ANY($3::bpchar[])
|
||||
), pth_agg AS (
|
||||
SELECT hour, max(entries)::integer AS max_path_entries FROM pth GROUP BY GROUPING SETS ((), (hour))
|
||||
), sig AS (
|
||||
SELECT s.hour, s.snr_sum, s.snr_samples, s.rssi_sum, s.rssi_samples FROM analytics_hourly_signal s JOIN complete c ON c.hour = s.hour
|
||||
WHERE s.kind = 3 AND COALESCE(cardinality($3::bpchar[]), 0) = 0
|
||||
UNION ALL
|
||||
SELECT s.hour, s.snr_sum, s.snr_samples, s.rssi_sum, s.rssi_samples FROM analytics_hourly_signal s JOIN complete c ON c.hour = s.hour
|
||||
WHERE s.kind = 3 AND cardinality($3::bpchar[]) > 0 AND s.iata = ANY($3::bpchar[])
|
||||
), sig_agg AS (
|
||||
SELECT hour, sum(snr_sum)::double precision AS snr_sum, sum(snr_samples)::bigint AS snr_samples,
|
||||
sum(rssi_sum)::double precision AS rssi_sum, sum(rssi_samples)::bigint AS rssi_samples
|
||||
FROM sig GROUP BY GROUPING SETS ((), (hour))
|
||||
), keys AS (
|
||||
SELECT hour, status FROM hours
|
||||
UNION ALL
|
||||
SELECT NULL::timestamptz, 'summary'::text
|
||||
)
|
||||
SELECT k.hour, k.status,
|
||||
COALESCE(o.observations, 0)::bigint AS observations,
|
||||
COALESCE(p.unique_packets, 0)::bigint AS unique_packets,
|
||||
COALESCE(ob.active_observers, 0)::bigint AS active_observers,
|
||||
COALESCE(o.active_iatas, 0)::bigint AS active_iatas,
|
||||
COALESCE(s.scoped_packets, 0)::bigint AS scoped_packets,
|
||||
COALESCE(s.active_scopes, 0)::bigint AS active_scopes,
|
||||
COALESCE(pa.max_path_entries, 0)::integer AS max_path_entries,
|
||||
COALESCE(sg.snr_sum, 0)::double precision AS snr_sum,
|
||||
COALESCE(sg.snr_samples, 0)::bigint AS snr_samples,
|
||||
COALESCE(sg.rssi_sum, 0)::double precision AS rssi_sum,
|
||||
COALESCE(sg.rssi_samples, 0)::bigint AS rssi_samples
|
||||
FROM keys k
|
||||
LEFT JOIN obs_agg o ON o.hour IS NOT DISTINCT FROM k.hour
|
||||
LEFT JOIN pkt_agg p ON p.hour IS NOT DISTINCT FROM k.hour
|
||||
LEFT JOIN obsr_agg ob ON ob.hour IS NOT DISTINCT FROM k.hour
|
||||
LEFT JOIN scp_agg s ON s.hour IS NOT DISTINCT FROM k.hour
|
||||
LEFT JOIN pth_agg pa ON pa.hour IS NOT DISTINCT FROM k.hour
|
||||
LEFT JOIN sig_agg sg ON sg.hour IS NOT DISTINCT FROM k.hour
|
||||
ORDER BY k.hour NULLS LAST
|
||||
`
|
||||
|
||||
type GetStatsSeriesParams struct {
|
||||
Since pgtype.Timestamptz `json:"since"`
|
||||
Until pgtype.Timestamptz `json:"until"`
|
||||
Iatas []string `json:"iatas"`
|
||||
}
|
||||
|
||||
type GetStatsSeriesRow struct {
|
||||
Hour pgtype.Timestamptz `json:"hour"`
|
||||
Status string `json:"status"`
|
||||
Observations int64 `json:"observations"`
|
||||
UniquePackets int64 `json:"unique_packets"`
|
||||
ActiveObservers int64 `json:"active_observers"`
|
||||
ActiveIatas int64 `json:"active_iatas"`
|
||||
ScopedPackets int64 `json:"scoped_packets"`
|
||||
ActiveScopes int64 `json:"active_scopes"`
|
||||
MaxPathEntries int32 `json:"max_path_entries"`
|
||||
SnrSum float64 `json:"snr_sum"`
|
||||
SnrSamples int64 `json:"snr_samples"`
|
||||
RssiSum float64 `json:"rssi_sum"`
|
||||
RssiSamples int64 `json:"rssi_samples"`
|
||||
}
|
||||
|
||||
// One row per hour in [since, until) plus a summary row (hour NULL) over its complete hours.
|
||||
// Counts sum; observers, IATAs and scopes are distinct across the window; packets count
|
||||
// once per hour heard. Each branch pair keeps "all IATAs" and "these IATAs" plans separate.
|
||||
func (q *Queries) GetStatsSeries(ctx context.Context, arg GetStatsSeriesParams) ([]GetStatsSeriesRow, error) {
|
||||
rows, err := q.db.Query(ctx, getStatsSeries, arg.Since, arg.Until, arg.Iatas)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
items := []GetStatsSeriesRow{}
|
||||
for rows.Next() {
|
||||
var i GetStatsSeriesRow
|
||||
if err := rows.Scan(
|
||||
&i.Hour,
|
||||
&i.Status,
|
||||
&i.Observations,
|
||||
&i.UniquePackets,
|
||||
&i.ActiveObservers,
|
||||
&i.ActiveIatas,
|
||||
&i.ScopedPackets,
|
||||
&i.ActiveScopes,
|
||||
&i.MaxPathEntries,
|
||||
&i.SnrSum,
|
||||
&i.SnrSamples,
|
||||
&i.RssiSum,
|
||||
&i.RssiSamples,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
Reference in new issue
Block a user