From 98006a93645c9f8b09d2ea441a5776eecb9add02 Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Fri, 2 Oct 2026 07:41:13 -0400 Subject: [PATCH] feat(stats): serve historical stats from hourly rollups Stats endpoints read hourly rollup tables filled once per closed hour by one advisory-locked task, instead of materialized views that re-scanned every observation on each refresh. IATA-set tables give exact distinct counts for any IATA set or region. Cleanup holds raw rows for unrolled hours (up to 24h). Adds /stats/series; ingest keeps packet observation counts, observer last IATA, channel message counts and trace tag summaries. 001_baseline.sql is rewritten, so pre-release 2.0 databases must be recreated. See docs/historical-stats.md. --- CONTRIBUTING.md | 2 + README.md | 49 +- SHOULDERS.md | 2 + cmd/beacon/main.go | 17 +- config.yaml.example | 11 +- db/analytics_retention_integration_test.go | 291 ------- db/baseline_integration_test.go | 2 +- db/channel_message_count_integration_test.go | 62 ++ db/channel_scopes_integration_test.go | 1 + db/channels.go | 18 +- db/channels_test.go | 2 +- db/migrations/001_baseline.sql | 819 +++++------------- db/observer_last_iata_integration_test.go | 124 +++ db/observer_metrics_integration_test.go | 6 +- db/observer_retention_integration_test.go | 6 +- db/observers.go | 4 +- db/observers_test.go | 12 +- db/packet_cleanup_integration_test.go | 205 +++++ ...cket_observation_count_integration_test.go | 88 ++ db/packets.go | 15 +- db/packets_integration_test.go | 3 +- db/packets_test.go | 17 +- db/paths.go | 2 - db/paths_integration_test.go | 14 +- db/presence.go | 4 +- db/queries/paths.sql | 8 +- db/queries/queries.sql | 449 +++++----- db/queries/rollup.sql | 279 ++++++ db/queries/series.sql | 91 ++ db/queries/signal.sql | 8 +- db/rollup.go | 202 +++++ db/rollup_integration_test.go | 515 +++++++++++ db/rollup_schema_integration_test.go | 117 +++ db/scope_stats_integration_test.go | 87 +- db/signal.go | 4 - db/signal_integration_test.go | 10 +- db/sqlc/mock/querier.go | 618 +++++++++---- db/sqlc/models.go | 323 +++---- db/sqlc/paths.sql.go | 15 +- db/sqlc/querier.go | 128 ++- db/sqlc/queries.sql.go | 736 ++++++++-------- db/sqlc/rollup.sql.go | 501 +++++++++++ db/sqlc/series.sql.go | 169 ++++ db/sqlc/signal.sql.go | 15 +- db/sqlc_scratch/rollup_obs.sql | 9 + db/stats.go | 167 ++-- db/stats_series.go | 64 ++ db/stats_series_integration_test.go | 126 +++ db/stats_test.go | 72 +- db/store.go | 3 +- db/trace_pagination_integration_test.go | 122 ++- db/traces.go | 15 +- docs/docs.go | 230 ++++- docs/historical-stats.md | 85 ++ docs/swagger.json | 230 ++++- docs/swagger.yaml | 194 ++++- go.mod | 2 +- internal/api/channels.go | 2 +- internal/api/handlers/limits_test.go | 4 +- .../api/handlers/observer_comparison_test.go | 6 +- internal/api/handlers/paths.go | 2 +- internal/api/handlers/paths_test.go | 6 +- internal/api/handlers/regions.go | 9 +- internal/api/handlers/scope_stats_test.go | 7 +- internal/api/handlers/scopes.go | 1 + internal/api/handlers/signal.go | 2 +- internal/api/handlers/signal_test.go | 10 +- internal/api/handlers/stats.go | 142 ++- internal/api/handlers/stats_region_test.go | 90 ++ internal/api/handlers/stats_series.go | 79 ++ internal/api/handlers/stats_series_test.go | 76 ++ internal/api/handlers/stats_test.go | 4 +- internal/api/handlers/stub_reader_test.go | 20 +- internal/api/handlers/traces.go | 4 +- internal/api/reader.go | 7 +- internal/api/router/options.go | 2 + internal/api/router/router.go | 2 +- internal/api/stats.go | 73 +- internal/background/rollup.go | 161 ++++ internal/background/rollup_test.go | 195 +++++ internal/background/tasks.go | 151 ++-- internal/background/tasks_test.go | 129 ++- internal/cache/cache.go | 78 +- internal/cache/cache_test.go | 24 +- internal/cache/logging_test.go | 2 +- internal/cache/paths.go | 25 - internal/cache/paths_test.go | 2 + internal/cache/reader.go | 154 +--- internal/cache/rollup_stats.go | 171 ++++ internal/cache/scope_stats_test.go | 6 +- internal/cache/series_test.go | 204 +++++ internal/cache/signal.go | 25 - internal/cache/signal_test.go | 2 + internal/config/config.go | 24 +- internal/config/config_test.go | 65 +- .../ingest/advert_endpoint_update_test.go | 6 +- internal/ingest/backfill.go | 5 +- internal/ingest/backfill_test.go | 29 +- internal/ingest/endpoint_work_test.go | 2 +- internal/ingest/frame_length_test.go | 4 +- internal/ingest/ingest.go | 17 +- internal/ingest/ingest_test.go | 24 +- internal/ingest/neighbors.go | 2 +- internal/ingest/packet.go | 25 +- internal/ingest/packet_endpoints_test.go | 4 +- internal/ingest/repeats_test.go | 27 +- internal/ingest/side_effects.go | 3 +- internal/ingest/side_effects_test.go | 20 +- internal/ingest/status.go | 2 +- internal/presence/coalescer.go | 26 +- internal/presence/coalescer_test.go | 52 +- sqlc.yaml | 10 +- 112 files changed, 6766 insertions(+), 2828 deletions(-) delete mode 100644 db/analytics_retention_integration_test.go create mode 100644 db/channel_message_count_integration_test.go create mode 100644 db/observer_last_iata_integration_test.go create mode 100644 db/packet_cleanup_integration_test.go create mode 100644 db/packet_observation_count_integration_test.go create mode 100644 db/queries/rollup.sql create mode 100644 db/queries/series.sql create mode 100644 db/rollup.go create mode 100644 db/rollup_integration_test.go create mode 100644 db/rollup_schema_integration_test.go create mode 100644 db/sqlc/rollup.sql.go create mode 100644 db/sqlc/series.sql.go create mode 100644 db/sqlc_scratch/rollup_obs.sql create mode 100644 db/stats_series.go create mode 100644 db/stats_series_integration_test.go create mode 100644 docs/historical-stats.md create mode 100644 internal/api/handlers/stats_region_test.go create mode 100644 internal/api/handlers/stats_series.go create mode 100644 internal/api/handlers/stats_series_test.go create mode 100644 internal/background/rollup.go create mode 100644 internal/background/rollup_test.go delete mode 100644 internal/cache/paths.go create mode 100644 internal/cache/rollup_stats.go create mode 100644 internal/cache/series_test.go delete mode 100644 internal/cache/signal.go diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f859d2f..3656d68 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -85,6 +85,8 @@ All schema changes must include a proper migration path: - Re-run `sqlc generate` to regenerate `db/sqlc/` - Update the store layer in `db/` to expose the new functionality - Update `internal/api/reader.go` if the change needs to be exposed via the API +- Historical stats read hourly rollups, never raw observations; follow + [docs/historical-stats.md](docs/historical-stats.md) to add a metric Never edit files under `db/sqlc/` by hand — they are generated by sqlc and will be overwritten. If you need to work around a sqlc limitation, document it diff --git a/README.md b/README.md index 855b9b8..5e664f6 100644 --- a/README.md +++ b/README.md @@ -258,12 +258,16 @@ scopes: # Observer telemetry storage settings. telemetry: - retention: 672h # how long to keep telemetry snapshots (default: 4 weeks) + retention: 744h # how long to keep telemetry snapshots (default: 31 days) resolution: 1h # snapshot frequency per observer; duplicates within window are dropped (default: 1h) # Packet and observation retention. packets: - retention: 168h # packets, observations, channel messages (default: 7 days) + retention: 168h # packets, observations, channel messages (default: 7 days, minimum 24h) + +# Hourly analytics rollups behind the stats endpoints; they outlive raw packets. +analytics: + rollup_retention: 2160h # also the longest /stats/series window (default: 90 days, minimum 24h) # Presence write coalescing. # Observer last_seen and packet last_heard_at bumps are batched in memory and @@ -302,7 +306,7 @@ observers: cache: ttl: "1h" ttls: - stats: "1h" # stats endpoints (backed by materialized views) + stats: "1h" # stats endpoints (hourly rollups; keys change when a new hour is rolled) reference: "1h" # IATAs, regions, scopes nodes: "1h" # node detail (also explicitly invalidated on upsert) observers: "1h" # observer detail (also explicitly invalidated on upsert) @@ -523,14 +527,15 @@ Not yet implemented — see the Authentication section above. | `GET` | `/routes/cross` | Search for routes crossing IATA boundaries | | `GET` | `/scopes` | List transport scope names; IATA/region filters use configured regions and MeshMapper catalogues | | `GET` | `/scopes/{name}` | Get scope detail | -| `GET` | `/stats/observations` | Hourly observation time series (last 7 days by default) | -| `GET` | `/stats/overview` | Network overview stats | +| `GET` | `/stats/observations` | Hourly observation counts per IATA (last 7 days by default) | +| `GET` | `/stats/overview` | Network overview over the last 24 rolled hours | | `GET` | `/stats/payload-breakdown` | Observation counts by payload type (last 24h by default) | -| `GET` | `/stats/scopes` | Configured region scopes and breakdown of packets, nodes, observers | -| `GET` | `/stats/top-advertisers` | Top N nodes by distinct ADVERT packet count (last 24h by default, from materialized view) | -| `GET` | `/stats/top-nodes` | Top N nodes by observation count (from materialized view) | +| `GET` | `/stats/scopes` | Configured region scopes and breakdown of packets, nodes, observers (last 7 days by default) | +| `GET` | `/stats/series` | Hourly card metrics and window summary from the rollups (`since`/`until` required) | +| `GET` | `/stats/top-advertisers` | Top N nodes by distinct ADVERT packet count (last 24h by default) | +| `GET` | `/stats/top-nodes` | Top N nodes by advert hearings (last 7 days by default) | | `GET` | `/stats/top-observers` | Top N observers by observation count (last 24h by default) | -| `GET` | `/stats/top-talkers` | Top N companion names by decrypted channel message count (last 24h by default, from materialized view) | +| `GET` | `/stats/top-talkers` | Top N companion names by decrypted channel message count (last 24h by default) | | `GET` | `/traces` | List trace tags with filters (optional: ?type=TRACE\|PING) | | `GET` | `/traces/{tag}` | Get full trace detail with resolved routes | @@ -560,20 +565,14 @@ Records include a component field. Ingest workers also include their broker name ### Analytics retention -Hourly traffic, payload, observer activity, talker, advertiser, Signal and Paths -summaries retain 30 days independently of `packets.retention`. Cleanup saves only -aggregates before deleting each packet batch, in the same transaction. Raw -packets, observations and message bodies still expire under packet retention. -The summaries use UTC hourly buckets and appear on the normal view-refresh cycle. -`mv_hourly_iata_stats` now covers 30 days instead of migration 001's seven days; -`/stats/observations?since=` can therefore return retained summaries older than a week. - -Migration 039 starts from data still present; previously deleted history cannot -be reconstructed. Telemetry has its own retention setting. Packet drill-down, -sub-hour observer activity, exact observer comparison and current entity/scope -counts continue to describe retained raw data or current entities, rather than -claiming archived packet detail. Archived summaries expire without waiting for -new packet deletions. A failed archive leaves its entire raw batch intact. +Historical stats (traffic, payload, observer activity, talker, advertiser, node, +scope, Signal and Paths) are read from hourly rollup tables kept for +`analytics.rollup_retention` (default 90 days), independently of `packets.retention`. +One background task rolls each UTC hour about 95 minutes after it closes; +packet cleanup waits for unrolled hours (up to 24h) so they can be rolled in full. +An hour whose raw rows were deleted first is marked partial and reported without values. +Packet drill-down, sub-hour observer activity and observer comparison still read +retained raw data. See [docs/historical-stats.md](docs/historical-stats.md). ### Observer monitoring metrics @@ -590,9 +589,7 @@ The existing observer `observationCount` remains a legacy cumulative presence counter for compatibility, including status/neighbour events. It is not a period packet total. Broker presence and packet-arrival timestamps are now updated separately; this cannot reconstruct previously overwritten timestamps. -The initial analytics archive migration includes unknown-type activity. Existing -development previews that used the earlier draft require a separate operator repair; -radio samples already discarded for those legacy rows cannot be recovered. +Hourly activity comes from the analytics rollups and includes unknown-type activity. ### Saved-route evidence diff --git a/SHOULDERS.md b/SHOULDERS.md index 05076de..ef2b34e 100644 --- a/SHOULDERS.md +++ b/SHOULDERS.md @@ -22,6 +22,8 @@ projects that make this possible: - [godotenv](https://github.com/joho/godotenv) — .env file loading - [yaml.v3](https://github.com/go-yaml/yaml) — YAML config parsing - [google/uuid](https://github.com/google/uuid) — UUID generation +- [golang.org/x/sync](https://pkg.go.dev/golang.org/x/sync) — singleflight for + collapsing concurrent cache misses ## Data diff --git a/cmd/beacon/main.go b/cmd/beacon/main.go index 3ccbffb..800c88a 100644 --- a/cmd/beacon/main.go +++ b/cmd/beacon/main.go @@ -378,9 +378,22 @@ func main() { ingestWorkers.Go(func() { broker1.Start(ctx) }) ingestWorkers.Go(func() { broker2.Start(ctx) }) + rollup := background.NewRollup(store, resolved.PacketRetention, resolved.RollupRetention) + go func() { + if err := rollup.CatchUp(ctx); err != nil { + slog.Error("analytics rollup catch-up failed", "component", "background", "error", err) + } + }() tasks := []background.Task{ + rollup.Task(), background.ViewRefreshTask(store, resolved.ViewRefreshInterval), - background.CleanupTask(store, resolved.TelemetryRetention, resolved.PacketRetention, resolved.NodeDeleteAfter, resolved.CleanupInterval), + background.CleanupTask(store, background.CleanupConfig{ + TelemetryRetention: resolved.TelemetryRetention, + PacketRetention: resolved.PacketRetention, + RollupRetention: resolved.RollupRetention, + NodeDeleteAfter: resolved.NodeDeleteAfter, + Interval: resolved.CleanupInterval, + }), background.ReconfirmTask(store, resolved.RouteRetention, resolved.RouteGrace, int64(resolved.RouteMinObservations), resolved.ReconfirmInterval), } if resolved.ObserverDeleteAfter > 0 { @@ -416,7 +429,7 @@ func main() { MaxConnectsPerMinute: resolved.MaxConnectsPerMinute, WSAllowedOrigins: cfg.WebSocket.AllowedOrigins, CORS: cfg.CORS, Server: cfg.Server, Auth: cfg.Auth, RateLimit: resolved.RateLimit, - Scopes: scopes, + Scopes: scopes, RollupRetention: resolved.RollupRetention, AdminRoutes: map[string]http.Handler{ "/accounts": handlers.AccountsRouter(store), "/backup": handlers.BackupRouter(backupOpts, ctx), diff --git a/config.yaml.example b/config.yaml.example index ae3a35a..3d91c8f 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -181,7 +181,7 @@ scopes: region: western-canada telemetry: - retention: 672h # 4 weeks + retention: 744h # 31 days resolution: 1h # store one snapshot per observer per hour # Optional authenticated database/config downloads. Disabled unless explicitly enabled. @@ -194,6 +194,11 @@ telemetry: packets: retention: 168h # 7 days +# Hourly analytics rollups (stats endpoints). Rollups outlive raw packets; this is also the +# longest window /stats/series and the other historical stats endpoints accept. +analytics: + rollup_retention: 2160h # 90 days + # Known-route retention. Routes are distilled packet history (the path a packet # took, hop by hop), so they may outlive packets.retention; there is no required # relationship between the two. @@ -266,7 +271,7 @@ cache: ttl: "1h" ttls: # Aggregated stats endpoints (overview, observations, breakdowns, top nodes/observers, - # radio presets, scope stats). Backed by materialized views refreshed hourly. + # radio presets, scope stats). Backed by hourly rollups; keys change when a new hour rolls. stats: "1h" # Mostly-static reference data (IATAs, regions, scopes). # Only changes when new observers arrive or config is reseeded. @@ -291,6 +296,6 @@ cache: # Shorter intervals are useful during initial deployment to confirm data is # flowing. Back off to 1h or more once stable. #background: -# view_refresh: 1h # default: 1h +# view_refresh: 1h # rebuilds radio presets; default: 1h # reconfirm: 1h # default: 1h # cleanup: 1h # default: 1h diff --git a/db/analytics_retention_integration_test.go b/db/analytics_retention_integration_test.go deleted file mode 100644 index 4306b5a..0000000 --- a/db/analytics_retention_integration_test.go +++ /dev/null @@ -1,291 +0,0 @@ -// Copyright 2026 Beacon Contributors -// SPDX-License-Identifier: AGPL-3.0-or-later - -package db - -import ( - "context" - sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" - "github.com/jackc/pgx/v5" - "github.com/jackc/pgx/v5/pgtype" - "os" - "strings" - "testing" - "time" -) - -var retainedViews = []string{"mv_hourly_iata_stats", "mv_payload_breakdown_by_iata", "mv_top_observers_by_iata", "mv_top_talkers_by_iata", "mv_top_advertisers_by_iata", "mv_observer_activity_hourly", "mv_signal_stats_hourly", "mv_path_stats_hourly"} - -func refreshRetainedViews(t *testing.T, ctx context.Context, tx pgx.Tx) { - t.Helper() - for _, view := range retainedViews { - if _, err := tx.Exec(ctx, "REFRESH MATERIALIZED VIEW "+view); err != nil { - t.Fatal(err) - } - } -} - -func analyticsSnapshot(t *testing.T, ctx context.Context, tx pgx.Tx) map[string]string { - t.Helper() - refreshRetainedViews(t, ctx, tx) - result := map[string]string{} - for _, view := range retainedViews { - var rows string - if err := tx.QueryRow(ctx, "SELECT COALESCE(jsonb_agg(r ORDER BY r::text),'[]')::text FROM (SELECT to_jsonb(v) r FROM "+view+" v) s").Scan(&rows); err != nil { - t.Fatal(err) - } - result[view] = rows - } - return result -} - -func TestAnalyticsRetentionConcurrentPostgres(t *testing.T) { - ctx, tx := retentionTx(t) - applyBaseline(t, ctx, tx) - if _, err := tx.Exec(ctx, `INSERT INTO packets (packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,first_heard_at,last_heard_at) VALUES ('\x01',4,0,1,'\x00','\x00',NOW()-interval '5 days',NOW()-interval '5 days'); INSERT INTO observers(id,public_key) VALUES ('00000000-0000-0000-0000-000000000001','\x01')`); err != nil { - t.Fatal(err) - } - var schema string - if err := tx.QueryRow(ctx, "SELECT current_schema()").Scan(&schema); err != nil { - t.Fatal(err) - } - if err := tx.Commit(ctx); err != nil { - t.Fatal(err) - } - conn := tx.Conn() - quoted := pgx.Identifier{schema}.Sanitize() - t.Cleanup(func() { - if _, err := conn.Exec(context.Background(), "DROP SCHEMA "+quoted+" CASCADE"); err != nil { - t.Error(err) - } - }) - if _, err := conn.Exec(ctx, "SET search_path TO "+quoted); err != nil { - t.Fatal(err) - } - writer, err := pgx.Connect(ctx, os.Getenv("BEACON_TEST_POSTGRES_DSN")) - if err != nil { - t.Fatal(err) - } - defer writer.Close(context.Background()) - if _, err = writer.Exec(ctx, "SET search_path TO "+quoted); err != nil { - t.Fatal(err) - } - ingest, err := writer.Begin(ctx) - if err != nil { - t.Fatal(err) - } - defer ingest.Rollback(context.Background()) - // An in-flight FK insert holds KEY SHARE on the packet until it commits. - if _, err = ingest.Exec(ctx, `INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count) VALUES ('\x01','00000000-0000-0000-0000-000000000001','YVR',NOW()-interval '5 days',0,1,0)`); err != nil { - t.Fatal(err) - } - q := sqlc.New(conn) - params := sqlc.DeleteOldPacketsParams{Cutoff: pgtype.Timestamptz{Time: time.Now().Add(-72 * time.Hour), Valid: true}, BatchSize: 1000} - if n, err := q.DeleteOldPackets(ctx, params); err != nil || n != 0 { - t.Fatalf("must skip in-flight reception: %d %v", n, err) - } - if err := ingest.Commit(ctx); err != nil { - t.Fatal(err) - } - if n, err := q.DeleteOldPackets(ctx, params); err != nil || n != 1 { - t.Fatalf("retry after ingestion: %d %v", n, err) - } - var count int - if err := conn.QueryRow(ctx, "SELECT observation_count FROM analytics_hourly_iata_stats").Scan(&count); err != nil || count != 1 { - t.Fatalf("committed reception not archived: %d %v", count, err) - } - // A second cleanup runner must wait before touching shared aggregate keys. - first, err := writer.Begin(ctx) - if err != nil { - t.Fatal(err) - } - defer first.Rollback(context.Background()) - if _, err := sqlc.New(first).DeleteOldPackets(ctx, params); err != nil { - t.Fatal(err) - } - if _, err := conn.Exec(ctx, "SET statement_timeout='300ms'"); err != nil { - t.Fatal(err) - } - if _, err := q.DeleteOldPackets(ctx, params); err == nil { - t.Fatalf("overlapping cleanup did not wait for the first transaction: %v", err) - } - if _, err := conn.Exec(ctx, "SET statement_timeout=0"); err != nil { - t.Fatal(err) - } - if err := first.Commit(ctx); err != nil { - t.Fatal(err) - } - if n, err := q.DeleteOldPackets(ctx, params); err != nil || n != 0 { - t.Fatalf("cleanup lock was not released: %d %v", n, err) - } -} - -func TestAnalyticsRetentionPostgres(t *testing.T) { - ctx, tx := retentionTx(t) - applyBaseline(t, ctx, tx) - _, err := tx.Exec(ctx, ` - INSERT INTO channels (id,channel_hash) VALUES (1,'\x01'); - INSERT INTO observers (id,public_key,display_name,observer_type) SELECT ('00000000-0000-0000-0000-00000000000'||i)::uuid,int4send(i),CASE WHEN i=1 THEN NULL ELSE 'Observer '||i END,CASE WHEN i=1 THEN NULL ELSE 'test' END FROM generate_series(1,3) i; - INSERT INTO nodes (id,public_key,node_type,name) VALUES ('00000000-0000-0000-0000-000000000002','\x02',2,'Repeater'); - INSERT INTO packets (packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,origin_pubkey,first_heard_at,last_heard_at) - SELECT int4send(i),4,0,CASE WHEN i%2=0 THEN 1 ELSE 2 END,'\x00','\x00','\x02',date_trunc('hour',NOW())-interval '5 days', - CASE WHEN i=4 THEN NOW() ELSE date_trunc('hour',NOW())-interval '5 days' END FROM generate_series(1,4) i; - INSERT INTO packet_observations (packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,snr,rssi,airtime_ms,payload_type) - SELECT packet_hash,('00000000-0000-0000-0000-00000000000'||n)::uuid,iata,first_heard_at+make_interval(secs=>n),0,1,0, - CASE WHEN n=1 THEN 0 ELSE NULL END,CASE WHEN n=1 THEN -100 ELSE NULL END,1.5,4 - FROM packets CROSS JOIN (VALUES ('YVR',1),('YVR',2),('YYZ',3)) v(iata,n); - INSERT INTO channel_messages (channel_id,packet_hash,sender_name,content,sent_at) - SELECT 1,packet_hash,'Sender','body that must expire',first_heard_at FROM packets; - `) - if err != nil { - t.Fatal(err) - } - for _, name := range []string{"idx_mv_signal_stats_hourly_hour", "idx_mv_path_stats_hourly_hour"} { - var definition string - if err := tx.QueryRow(ctx, "SELECT pg_get_indexdef(to_regclass($1))", name).Scan(&definition); err != nil || !strings.Contains(definition, "(hour)") { - t.Fatalf("missing unfiltered time index %s: %s %v", name, definition, err) - } - } - refreshRetainedViews(t, ctx, tx) - store := &Store{q: sqlc.New(tx)} - hour := time.Now().UTC().Truncate(time.Hour) - signal, err := store.GetSignalStats(ctx, hour.Add(-6*24*time.Hour), hour, nil) - if err != nil { - t.Fatal(err) - } - // 12 observations; only the n=1 rows carry a signal (snr 0, rssi -100). - if signal.Receptions != 12 || signal.SNR.Samples != 4 || signal.RSSI.Samples != 4 { - t.Fatalf("signal stats: %+v", signal) - } - paths, err := store.GetPathStats(ctx, hour.Add(-6*24*time.Hour), hour, nil) - if err != nil { - t.Fatal(err) - } - if paths.Receptions != 12 { - t.Fatalf("path stats: %+v", paths) - } - before := analyticsSnapshot(t, ctx, tx) - q := sqlc.New(tx) - cutoff := time.Now().Add(-72 * time.Hour) - // Failure in one archive must roll back every archive write and the raw deletion. - if _, err := tx.Exec(ctx, `SAVEPOINT archive_failure; ALTER TABLE analytics_signal_stats_hourly ADD CONSTRAINT fail_archive CHECK(receptions < 0) NOT VALID`); err != nil { - t.Fatal(err) - } - if _, err := q.DeleteOldPackets(ctx, sqlc.DeleteOldPacketsParams{Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true}, BatchSize: 1}); err == nil { - t.Fatal("expected archive failure") - } - if _, err := tx.Exec(ctx, "ROLLBACK TO archive_failure"); err != nil { - t.Fatal(err) - } - if countRows(t, ctx, tx, "SELECT count(*) FROM packets") != 4 { - t.Fatal("archive failure deleted raw packets") - } - for i := 0; i < 3; i++ { - n, err := q.DeleteOldPackets(ctx, sqlc.DeleteOldPacketsParams{Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true}, BatchSize: 1}) - if err != nil || n != 1 { - t.Fatalf("delete batch: %d %v", n, err) - } - } - if countRows(t, ctx, tx, "SELECT count(*) FROM packets") != 1 || countRows(t, ctx, tx, "SELECT count(*) FROM channel_messages") != 1 { - t.Fatal("raw packets/messages did not expire") - } - after := analyticsSnapshot(t, ctx, tx) - for _, view := range retainedViews { - if after[view] != before[view] { - t.Errorf("%s lost or double-counted history after raw expiry: before=%s after=%s", view, before[view], after[view]) - } - } - // A retry preserves the exact summaries. - if n, err := q.DeleteOldPackets(ctx, sqlc.DeleteOldPacketsParams{Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true}, BatchSize: 1000}); err != nil || n != 0 { - t.Fatalf("retry: %d %v", n, err) - } - for view, rows := range analyticsSnapshot(t, ctx, tx) { - if rows != before[view] { - t.Errorf("retry changed %s", view) - } - } - // Later receptions on a still-live packet must join its old bucket before expiry. - if _, err := tx.Exec(ctx, `INSERT INTO observers (id,public_key) VALUES ('00000000-0000-0000-0000-000000000004','\x04'); INSERT INTO packet_observations (packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,snr,rssi,payload_type) - SELECT packet_hash,'00000000-0000-0000-0000-000000000004','YVR',first_heard_at,0,1,0,'NaN',-80,NULL FROM packets; - UPDATE packets SET last_heard_at=first_heard_at`); err != nil { - t.Fatal(err) - } - late := analyticsSnapshot(t, ctx, tx) - if countRows(t, ctx, tx, "SELECT COALESCE(SUM(observations),0) FROM mv_observer_activity_hourly WHERE payload_type=-1") != 1 { - t.Fatal("unknown-payload observation was omitted") - } - if err := (&Store{q: q}).DeleteOldPackets(ctx, cutoff); err != nil { - t.Fatal(err) - } - for view, rows := range analyticsSnapshot(t, ctx, tx) { - if rows != late[view] { - t.Errorf("late reception changed %s: %s != %s", view, rows, late[view]) - } - } - // Archived labels follow a current rename; no entity FK may erase retained history. - if _, err := tx.Exec(ctx, `UPDATE nodes SET name='Renamed'; UPDATE observers SET display_name=NULL,observer_type=NULL;`); err != nil { - t.Fatal(err) - } - analyticsSnapshot(t, ctx, tx) - if countRows(t, ctx, tx, "SELECT count(*) FROM mv_top_advertisers_by_iata WHERE name='Renamed'") != 2 { - t.Fatal("archive name did not follow rename") - } - if _, err := q.GetStatsTopObservers(ctx, sqlc.GetStatsTopObserversParams{Column1: pgtype.Interval{Microseconds: int64(30 * 24 * time.Hour / time.Microsecond), Valid: true}, Limit: 10}); err != nil { - t.Fatal(err) - } - if _, err := tx.Exec(ctx, "DELETE FROM nodes; DELETE FROM observers"); err != nil { - t.Fatal(err) - } - if countRows(t, ctx, tx, "SELECT count(*) FROM analytics_top_advertisers_by_iata") != 2 { - t.Fatal("entity deletion erased archive") - } - // Independent expiry runs even when there are no packets left to delete. - for _, view := range retainedViews { - table := "analytics_" + view[3:] - at := "bucket" - if view == "mv_hourly_iata_stats" || view == "mv_signal_stats_hourly" || view == "mv_path_stats_hourly" { - at = "hour" - } - if _, err := tx.Exec(ctx, "UPDATE "+table+" SET "+at+"="+at+"-interval '31 days'"); err != nil { - t.Fatal(err) - } - } - if err := (&Store{q: q}).DeleteOldPackets(ctx, cutoff); err != nil { - t.Fatal(err) - } - for _, view := range retainedViews { - if countRows(t, ctx, tx, "SELECT count(*) FROM analytics_"+view[3:]) != 0 { - t.Errorf("%s archive did not expire", view) - } - } - // More than one populated cohort, with shared hourly keys across the boundary. - if _, err := tx.Exec(ctx, ` -INSERT INTO observers (id,public_key) VALUES ('00000000-0000-0000-0000-000000000001','\x01'); -INSERT INTO nodes (id,public_key,node_type) VALUES ('00000000-0000-0000-0000-000000000002','\x02',2); -INSERT INTO packets (packet_hash,payload_type,payload_version,route_type,origin_pubkey,first_heard_at,last_heard_at,raw_payload,raw_header) -SELECT int4send(i),4,0,1,'\x02',date_trunc('hour',NOW())-interval '5 days',date_trunc('hour',NOW())-interval '5 days',decode(repeat('ab',1024),'hex'),'\x00' FROM generate_series(10000,11000) i; -INSERT INTO packet_observations (packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,payload_type,snr,rssi) -SELECT packet_hash,'00000000-0000-0000-0000-000000000001','YVR',first_heard_at,0,1,0,4,0,-100 FROM packets; -INSERT INTO channel_messages (channel_id,packet_hash,sender_name,sent_at) SELECT 1,packet_hash,'Sender',first_heard_at FROM packets; -`); err != nil { - t.Fatal(err) - } - populated := analyticsSnapshot(t, ctx, tx) - started := time.Now() - if err := (&Store{q: q}).DeleteOldPackets(ctx, cutoff); err != nil { - t.Fatal(err) - } - t.Logf("1001 packets with bodies and observations archived/deleted in %s", time.Since(started)) - for view, rows := range analyticsSnapshot(t, ctx, tx) { - if rows != populated[view] { - t.Errorf("batch boundary changed %s", view) - } - } - var archiveRows int - for _, view := range retainedViews { - archiveRows += countRows(t, ctx, tx, "SELECT count(*) FROM analytics_"+view[3:]) - } - if archiveRows > 12 { - t.Fatalf("expected compact rollups, got %d archive rows for 1001 packets", archiveRows) - } -} diff --git a/db/baseline_integration_test.go b/db/baseline_integration_test.go index e879c49..27a8a60 100644 --- a/db/baseline_integration_test.go +++ b/db/baseline_integration_test.go @@ -96,7 +96,7 @@ func TestRunMigrationsBaselinePostgres(t *testing.T) { t.Fatalf("ledger %v, %v", ledger, err) } // Stats refresh uses CONCURRENTLY, which needs populated views. - if _, err := pool.Exec(ctx, "REFRESH MATERIALIZED VIEW CONCURRENTLY mv_hourly_iata_stats"); err != nil { + if _, err := pool.Exec(ctx, "REFRESH MATERIALIZED VIEW CONCURRENTLY mv_radio_presets"); err != nil { t.Fatal(err) } } diff --git a/db/channel_message_count_integration_test.go b/db/channel_message_count_integration_test.go new file mode 100644 index 0000000..5255816 --- /dev/null +++ b/db/channel_message_count_integration_test.go @@ -0,0 +1,62 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "os" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/ingest" + "github.com/jackc/pgx/v5" +) + +func TestChannelMessageCountPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for PostgreSQL regression") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + applyBaseline(t, ctx, tx) + store := &Store{q: sqlc.New(tx)} + + if _, err := tx.Exec(ctx, `INSERT INTO packets (packet_hash, payload_type, payload_version, route_type, raw_payload, raw_header, first_heard_at, last_heard_at) +SELECT decode(lpad(to_hex(i), 2, '0'), 'hex'), 5, 0, 1, '\x00', '\x00', now(), now() FROM generate_series(1, 2) i`); err != nil { + t.Fatal(err) + } + // Re-upserting the channel (as every decrypt does) must not touch the count. + var id int + for range 3 { + if id, err = store.UpsertChannel(ctx, []byte{0x11}, []byte{1, 2, 3, 4, 5, 6, 7, 8}, "public", ""); err != nil { + t.Fatal(err) + } + } + for _, hash := range [][]byte{{1}, {1}, {2}} { // second {1} is a duplicate + if _, err := store.InsertChannelMessage(ctx, ingest.InsertChannelMessageParams{ + ChannelID: id, PacketHash: hash, SenderName: "a", Content: "b", SentAt: time.Now(), + }); err != nil { + t.Fatal(err) + } + } + ch, err := store.GetChannel(ctx, int32(id)) + if err != nil { + t.Fatal(err) + } + if ch.MessageCount != 2 { + t.Errorf("message_count = %d, want 2", ch.MessageCount) + } +} diff --git a/db/channel_scopes_integration_test.go b/db/channel_scopes_integration_test.go index e02f576..ab9a9c4 100644 --- a/db/channel_scopes_integration_test.go +++ b/db/channel_scopes_integration_test.go @@ -61,6 +61,7 @@ SELECT decode(lpad(to_hex(i),2,'0'),'hex'),5,0,1,transport,scope_id,'\x00','\x15 FROM (VALUES(1,true,1),(2,true,NULL),(3,false,NULL),(4,NULL,NULL),(5,true,2),(6,false,NULL)) v(i,transport,scope_id); INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count) SELECT packet_hash,md5(n::text)::uuid,'YOW',NOW(),0,1,0 FROM packets CROSS JOIN generate_series(1,2) n; +UPDATE packets SET observation_count=2; `) if err != nil { t.Fatal(err) diff --git a/db/channels.go b/db/channels.go index 3e07032..c292b76 100644 --- a/db/channels.go +++ b/db/channels.go @@ -26,12 +26,11 @@ func (s *Store) UpsertChannel(ctx context.Context, channelHash []byte, keyFinger } isHashtag := hashtag != "" row, err := s.q.UpsertChannel(ctx, sqlc.UpsertChannelParams{ - ChannelHash: channelHash, - Column2: keyFingerprint, // key_fingerprint - Name: namePtr, - Hashtag: hashtagPtr, - IsHashtag: &isHashtag, - MessageCount: nil, // message count bumped separately by InsertChannelMessage + ChannelHash: channelHash, + Column2: keyFingerprint, // key_fingerprint + Name: namePtr, + Hashtag: hashtagPtr, + IsHashtag: &isHashtag, }) if err != nil { return 0, err @@ -167,10 +166,7 @@ func (s *Store) GetChannel(ctx context.Context, channelID int32) (*api.Channel, KeyKnown: row.KeyKnown != nil && *row.KeyKnown, }, Hashtag: row.Hashtag, - MessageCount: 0, - } - if row.MessageCount != nil { - channel.MessageCount = *row.MessageCount + MessageCount: row.MessageCount, } if row.IsHashtag != nil && *row.IsHashtag && row.KeyFingerprint != nil { fp := hex.EncodeToString(row.KeyFingerprint) @@ -180,7 +176,7 @@ func (s *Store) GetChannel(ctx context.Context, channelID int32) (*api.Channel, } func (s *Store) InsertChannelMessage(ctx context.Context, m ingest.InsertChannelMessageParams) (*ingest.InsertedChannelMessage, error) { - params := sqlc.InsertChannelMessageParams{ChannelID: int32(m.ChannelID), PacketHash: m.PacketHash, SenderName: &m.SenderName, Content: &m.Content, SentAt: pgtype.Timestamptz{Time: m.SentAt, Valid: true}} + params := sqlc.InsertChannelMessageParams{ChannelID: int32(m.ChannelID), PacketHash: m.PacketHash, SenderName: &m.SenderName, Content: &m.Content, SentAt: pgtype.Timestamptz{Time: m.SentAt, Valid: true}, Column6: m.Historical} row, err := s.q.InsertChannelMessage(ctx, params) if errors.Is(err, pgx.ErrNoRows) { return nil, nil // duplicate diff --git a/db/channels_test.go b/db/channels_test.go index 9ba475c..be0fd46 100644 --- a/db/channels_test.go +++ b/db/channels_test.go @@ -164,7 +164,7 @@ func TestGetChannel_Basic(t *testing.T) { LastSeen: lastSeen, IsHashtag: &isHashtag, KeyKnown: &keyKnown, - MessageCount: &msgCount, + MessageCount: msgCount, }, nil) store := &Store{q: mock} diff --git a/db/migrations/001_baseline.sql b/db/migrations/001_baseline.sql index 06e4335..00af459 100644 --- a/db/migrations/001_baseline.sql +++ b/db/migrations/001_baseline.sql @@ -4,237 +4,6 @@ -- 2.0.0 schema baseline: pg_dump of a fresh database migrated through the 1.x -- chain (001_initial_schema..045). Add new numbered migrations after this file. -CREATE FUNCTION archive_delete_packets(cutoff timestamp with time zone, batch_size integer) RETURNS bigint - LANGUAGE plpgsql - AS $$ -DECLARE hashes bytea[]; deleted bigint; -BEGIN - -- Serialize cleanup runners before overlapping aggregate upserts. - PERFORM pg_advisory_xact_lock(hashtext('beacon.archive_delete_packets')); - IF batch_size < 1 THEN RAISE EXCEPTION 'batch_size must be positive'; END IF; - SELECT array_agg(p.packet_hash) INTO hashes FROM ( - SELECT ep.packet_hash FROM packets ep - WHERE ep.last_heard_at < cutoff - ORDER BY ep.last_heard_at, ep.packet_hash - LIMIT batch_size FOR UPDATE OF ep SKIP LOCKED - ) p; - - WITH expired_observations AS MATERIALIZED ( - SELECT po.* FROM packet_observations po WHERE po.packet_hash = ANY(hashes) - ), -archived_hourly_iata_stats AS ( - INSERT INTO analytics_hourly_iata_stats (iata, hour, observation_count, unique_packets) - SELECT iata, hour, observation_count, unique_packets FROM ( -SELECT - iata, - date_trunc('hour', heard_at, 'UTC')::timestamptz AS hour, - COUNT(*) AS observation_count, - COUNT(DISTINCT packet_hash) AS unique_packets -FROM expired_observations -WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -GROUP BY iata, date_trunc('hour', heard_at, 'UTC') - ) batch - ON CONFLICT (iata,hour) DO UPDATE SET - observation_count = analytics_hourly_iata_stats.observation_count + EXCLUDED.observation_count, - unique_packets = analytics_hourly_iata_stats.unique_packets + EXCLUDED.unique_packets -), -archived_payload_breakdown_by_iata AS ( - INSERT INTO analytics_payload_breakdown_by_iata (iata, payload_type, bucket, count) - SELECT iata, payload_type, bucket, count FROM ( -SELECT - iata, - payload_type, - date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, - COUNT(*) AS count -FROM expired_observations -WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' - AND payload_type IS NOT NULL -GROUP BY iata, payload_type, date_trunc('hour', heard_at, 'UTC') - ) batch - ON CONFLICT (iata,payload_type,bucket) DO UPDATE SET - count = analytics_payload_breakdown_by_iata.count + EXCLUDED.count -), -archived_top_observers_by_iata AS ( - INSERT INTO analytics_top_observers_by_iata (iata, observer_id, bucket, observation_count, display_name, observer_type) - SELECT iata, observer_id, bucket, observation_count, display_name, observer_type FROM ( -SELECT - po.iata, - po.observer_id, - o.display_name, - o.observer_type, - date_trunc('hour', po.heard_at, 'UTC')::timestamptz AS bucket, - COUNT(*) AS observation_count -FROM expired_observations po -JOIN observers o ON o.id = po.observer_id -WHERE po.heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -GROUP BY po.iata, po.observer_id, o.display_name, o.observer_type, date_trunc('hour', po.heard_at, 'UTC') - ) batch - ON CONFLICT (iata,observer_id,bucket) DO UPDATE SET - observation_count = analytics_top_observers_by_iata.observation_count + EXCLUDED.observation_count, - display_name = EXCLUDED.display_name, - observer_type = EXCLUDED.observer_type -), -archived_top_talkers_by_iata AS ( - INSERT INTO analytics_top_talkers_by_iata (iata, sender_name, bucket, message_count, last_sent) - SELECT iata, sender_name, bucket, message_count, last_sent FROM ( -SELECT - po.iata, - cm.sender_name, - date_trunc('hour', cm.sent_at, 'UTC')::timestamptz AS bucket, - COUNT(DISTINCT cm.id) AS message_count, - MAX(cm.sent_at) AS last_sent -FROM channel_messages cm -JOIN expired_observations po ON po.packet_hash = cm.packet_hash -WHERE cm.sender_name IS NOT NULL - AND cm.sent_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -GROUP BY po.iata, cm.sender_name, date_trunc('hour', cm.sent_at, 'UTC') - ) batch - ON CONFLICT (iata,sender_name,bucket) DO UPDATE SET - message_count = analytics_top_talkers_by_iata.message_count + EXCLUDED.message_count, - last_sent = GREATEST(analytics_top_talkers_by_iata.last_sent, EXCLUDED.last_sent) -), -archived_top_advertisers_by_iata AS ( - INSERT INTO analytics_top_advertisers_by_iata (iata, node_id, bucket, advert_count, flood_advert_count, direct_advert_count, last_heard, name, node_type) - SELECT iata, node_id, bucket, advert_count, flood_advert_count, direct_advert_count, last_heard, name, node_type FROM ( -SELECT - po.iata, - n.id AS node_id, - n.name, - n.node_type, - date_trunc('hour', po.heard_at, 'UTC')::timestamptz AS bucket, - COUNT(DISTINCT p.packet_hash) AS advert_count, - COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (0, 1)) AS flood_advert_count, - COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (2, 3)) AS direct_advert_count, - MAX(po.heard_at) AS last_heard -FROM packets p -JOIN expired_observations po ON po.packet_hash = p.packet_hash -JOIN nodes n ON n.public_key = p.origin_pubkey -WHERE p.payload_type = 4 -- ADVERT - AND po.heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -GROUP BY po.iata, n.id, n.name, n.node_type, date_trunc('hour', po.heard_at, 'UTC') - ) batch - ON CONFLICT (iata,node_id,bucket) DO UPDATE SET - advert_count = analytics_top_advertisers_by_iata.advert_count + EXCLUDED.advert_count, - flood_advert_count = analytics_top_advertisers_by_iata.flood_advert_count + EXCLUDED.flood_advert_count, - direct_advert_count = analytics_top_advertisers_by_iata.direct_advert_count + EXCLUDED.direct_advert_count, - last_heard = GREATEST(analytics_top_advertisers_by_iata.last_heard, EXCLUDED.last_heard), - name = EXCLUDED.name, - node_type = EXCLUDED.node_type -), -archived_observer_activity_hourly AS ( - INSERT INTO analytics_observer_activity_hourly (observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n) - SELECT observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n FROM ( -SELECT - observer_id, - COALESCE(payload_type, -1)::smallint AS payload_type, - date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, - COUNT(*)::bigint AS observations, - SUM(airtime_ms)::real AS airtime_ms, - COUNT(airtime_ms)::bigint AS airtime_n, - SUM(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_sum, - COUNT(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS snr_n, - MIN(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_min, - SUM(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_sum, - COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_n -FROM expired_observations -WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -GROUP BY observer_id, COALESCE(payload_type, -1), date_trunc('hour', heard_at, 'UTC') - ) batch - ON CONFLICT (observer_id,payload_type,bucket) DO UPDATE SET - observations = analytics_observer_activity_hourly.observations + EXCLUDED.observations, - airtime_ms = CASE WHEN analytics_observer_activity_hourly.airtime_ms IS NULL AND EXCLUDED.airtime_ms IS NULL THEN NULL ELSE COALESCE(analytics_observer_activity_hourly.airtime_ms, 0) + COALESCE(EXCLUDED.airtime_ms, 0) END, - airtime_n = analytics_observer_activity_hourly.airtime_n + EXCLUDED.airtime_n, - snr_sum = CASE WHEN analytics_observer_activity_hourly.snr_sum IS NULL AND EXCLUDED.snr_sum IS NULL THEN NULL ELSE COALESCE(analytics_observer_activity_hourly.snr_sum, 0) + COALESCE(EXCLUDED.snr_sum, 0) END, - snr_n = analytics_observer_activity_hourly.snr_n + EXCLUDED.snr_n, - snr_min = LEAST(analytics_observer_activity_hourly.snr_min, EXCLUDED.snr_min), - rssi_sum = CASE WHEN analytics_observer_activity_hourly.rssi_sum IS NULL AND EXCLUDED.rssi_sum IS NULL THEN NULL ELSE COALESCE(analytics_observer_activity_hourly.rssi_sum, 0) + COALESCE(EXCLUDED.rssi_sum, 0) END, - rssi_n = analytics_observer_activity_hourly.rssi_n + EXCLUDED.rssi_n -), -archived_signal_stats_hourly AS ( - INSERT INTO analytics_signal_stats_hourly (iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum) - SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM ( -WITH samples AS ( - SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, - CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) - AND snr > '-Infinity'::real AND snr < 'Infinity'::real - THEN snr::double precision END AS snr, - CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) - THEN rssi::double precision END AS rssi - FROM expired_observations - WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours' - -), binned AS ( - SELECT *, width_bucket(snr, -30, 30, 12) AS snr_bin, - width_bucket(rssi, -140, 0, 14) AS rssi_bin - FROM samples -) -SELECT iata, hour, grouping(snr_bin, rssi_bin)::integer AS kind, - COALESCE(snr_bin, -1)::integer AS snr_bin, - COALESCE(rssi_bin, -1)::integer AS rssi_bin, - count(*)::bigint AS receptions, - count(snr)::bigint AS snr_samples, - COALESCE(sum(snr), 0)::double precision AS snr_sum, - count(rssi)::bigint AS rssi_samples, - COALESCE(sum(rssi), 0)::double precision AS rssi_sum -FROM binned -GROUP BY GROUPING SETS ((iata, hour), (iata, hour, snr_bin), (iata, hour, rssi_bin)) - ) batch - ON CONFLICT (iata,hour,kind,snr_bin,rssi_bin) DO UPDATE SET - receptions = analytics_signal_stats_hourly.receptions + EXCLUDED.receptions, - snr_samples = analytics_signal_stats_hourly.snr_samples + EXCLUDED.snr_samples, - snr_sum = analytics_signal_stats_hourly.snr_sum + EXCLUDED.snr_sum, - rssi_samples = analytics_signal_stats_hourly.rssi_samples + EXCLUDED.rssi_samples, - rssi_sum = analytics_signal_stats_hourly.rssi_sum + EXCLUDED.rssi_sum -), -archived_path_stats_hourly AS ( - INSERT INTO analytics_path_stats_hourly (iata, hour, category, hash_bytes, entries, receptions) - SELECT iata, hour, category, hash_bytes, entries, receptions FROM ( -WITH classified AS ( - SELECT iata, heard_at, hash_size, hop_count, - CASE WHEN payload_type = 9 THEN 2 - -- Match meshcore-go IsValidPathLen (1/2/3-byte hashes, max 64 path bytes). - WHEN payload_type IS NULL OR payload_type NOT BETWEEN 0 AND 15 - OR NOT (path_length_byte BETWEEN 0 AND 191 - AND hash_size BETWEEN 1 AND 3 AND hop_count BETWEEN 0 AND 63 - AND hash_size = (path_length_byte >> 6) + 1 - AND hop_count = (path_length_byte & 63) - AND hash_size::integer * hop_count::integer <= 64 - AND COALESCE(octet_length(path_bytes), 0) = hash_size::integer * hop_count::integer) - THEN 3 - WHEN hop_count = 0 THEN 1 - ELSE 0 END::integer AS category - FROM expired_observations - WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours' - -), buckets AS ( - SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, category, - CASE WHEN category = 0 THEN hash_size ELSE 0 END::integer AS hash_bytes, - CASE WHEN category = 0 THEN hop_count ELSE 0 END::integer AS entries - FROM classified -) -SELECT iata, hour, category, hash_bytes, entries, count(*)::bigint AS receptions -FROM buckets GROUP BY iata, hour, category, hash_bytes, entries - ) batch - ON CONFLICT (iata,hour,category,hash_bytes,entries) DO UPDATE SET - receptions = analytics_path_stats_hourly.receptions + EXCLUDED.receptions -) - DELETE FROM packets WHERE packet_hash = ANY(hashes); - GET DIAGNOSTICS deleted = ROW_COUNT; - - -- Once per completed cleanup, including when there were no raw packets to delete. - IF deleted < batch_size THEN - DELETE FROM analytics_hourly_iata_stats WHERE hour < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; - DELETE FROM analytics_payload_breakdown_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; - DELETE FROM analytics_top_observers_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; - DELETE FROM analytics_top_talkers_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; - DELETE FROM analytics_top_advertisers_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; - DELETE FROM analytics_observer_activity_hourly WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; - DELETE FROM analytics_signal_stats_hourly WHERE hour < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; - DELETE FROM analytics_path_stats_hourly WHERE hour < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; - END IF; - RETURN deleted; -END $$; - CREATE TABLE accounts ( id uuid DEFAULT gen_random_uuid() NOT NULL, name text NOT NULL, @@ -243,13 +12,6 @@ CREATE TABLE accounts ( CONSTRAINT accounts_name_check CHECK ((((char_length(name) >= 1) AND (char_length(name) <= 128)) AND (name = btrim(name)))) ); -CREATE TABLE analytics_hourly_iata_stats ( - iata character(3) NOT NULL, - hour timestamp with time zone NOT NULL, - observation_count bigint DEFAULT 0 NOT NULL, - unique_packets bigint DEFAULT 0 NOT NULL -); - CREATE TABLE packet_observations ( id bigint NOT NULL, packet_hash bytea NOT NULL, @@ -273,98 +35,6 @@ CREATE TABLE packet_observations ( ) WITH (autovacuum_vacuum_scale_factor='0.05', autovacuum_vacuum_cost_limit='1000'); -CREATE VIEW analytics_live_hourly_iata_stats AS -SELECT - iata, - date_trunc('hour', heard_at, 'UTC')::timestamptz AS hour, - COUNT(*) AS observation_count, - COUNT(DISTINCT packet_hash) AS unique_packets -FROM packet_observations -WHERE heard_at > NOW() - INTERVAL '30 days' -GROUP BY iata, date_trunc('hour', heard_at, 'UTC'); - -CREATE VIEW analytics_live_observer_activity_hourly AS -SELECT - observer_id, - COALESCE(payload_type, -1)::smallint AS payload_type, - date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, - COUNT(*)::bigint AS observations, - SUM(airtime_ms)::real AS airtime_ms, - COUNT(airtime_ms)::bigint AS airtime_n, - SUM(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_sum, - COUNT(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS snr_n, - MIN(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_min, - SUM(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_sum, - COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_n -FROM packet_observations -WHERE heard_at > NOW() - INTERVAL '30 days' -GROUP BY observer_id, COALESCE(payload_type, -1), date_trunc('hour', heard_at, 'UTC'); - -CREATE VIEW analytics_live_path_stats_hourly AS -WITH classified AS ( - SELECT iata, heard_at, hash_size, hop_count, - CASE WHEN payload_type = 9 THEN 2 - -- Match meshcore-go IsValidPathLen (1/2/3-byte hashes, max 64 path bytes). - WHEN payload_type IS NULL OR payload_type NOT BETWEEN 0 AND 15 - OR NOT (path_length_byte BETWEEN 0 AND 191 - AND hash_size BETWEEN 1 AND 3 AND hop_count BETWEEN 0 AND 63 - AND hash_size = (path_length_byte >> 6) + 1 - AND hop_count = (path_length_byte & 63) - AND hash_size::integer * hop_count::integer <= 64 - AND COALESCE(octet_length(path_bytes), 0) = hash_size::integer * hop_count::integer) - THEN 3 - WHEN hop_count = 0 THEN 1 - ELSE 0 END::integer AS category - FROM packet_observations - WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours' - AND heard_at < date_trunc('hour', NOW(), 'UTC') -), buckets AS ( - SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, category, - CASE WHEN category = 0 THEN hash_size ELSE 0 END::integer AS hash_bytes, - CASE WHEN category = 0 THEN hop_count ELSE 0 END::integer AS entries - FROM classified -) -SELECT iata, hour, category, hash_bytes, entries, count(*)::bigint AS receptions -FROM buckets GROUP BY iata, hour, category, hash_bytes, entries; - -CREATE VIEW analytics_live_payload_breakdown_by_iata AS -SELECT - iata, - payload_type, - date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, - COUNT(*) AS count -FROM packet_observations -WHERE heard_at > NOW() - INTERVAL '30 days' - AND payload_type IS NOT NULL -GROUP BY iata, payload_type, date_trunc('hour', heard_at, 'UTC'); - -CREATE VIEW analytics_live_signal_stats_hourly AS -WITH samples AS ( - SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, - CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) - AND snr > '-Infinity'::real AND snr < 'Infinity'::real - THEN snr::double precision END AS snr, - CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) - THEN rssi::double precision END AS rssi - FROM packet_observations - WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours' - AND heard_at < date_trunc('hour', NOW(), 'UTC') -), binned AS ( - SELECT *, width_bucket(snr, -30, 30, 12) AS snr_bin, - width_bucket(rssi, -140, 0, 14) AS rssi_bin - FROM samples -) -SELECT iata, hour, grouping(snr_bin, rssi_bin)::integer AS kind, - COALESCE(snr_bin, -1)::integer AS snr_bin, - COALESCE(rssi_bin, -1)::integer AS rssi_bin, - count(*)::bigint AS receptions, - count(snr)::bigint AS snr_samples, - COALESCE(sum(snr), 0)::double precision AS snr_sum, - count(rssi)::bigint AS rssi_samples, - COALESCE(sum(rssi), 0)::double precision AS rssi_sum -FROM binned -GROUP BY GROUPING SETS ((iata, hour), (iata, hour, snr_bin), (iata, hour, rssi_bin)); - CREATE TABLE nodes ( id uuid DEFAULT gen_random_uuid() NOT NULL, public_key bytea NOT NULL, @@ -410,28 +80,11 @@ CREATE TABLE packets ( channel_hash bytea, trace_tag bytea, first_heard_at timestamp with time zone NOT NULL, - last_heard_at timestamp with time zone NOT NULL + last_heard_at timestamp with time zone NOT NULL, + observation_count bigint DEFAULT 0 NOT NULL ) WITH (autovacuum_vacuum_scale_factor='0.05', autovacuum_vacuum_cost_limit='1000'); -CREATE VIEW analytics_live_top_advertisers_by_iata AS -SELECT - po.iata, - n.id AS node_id, - n.name, - n.node_type, - date_trunc('hour', po.heard_at, 'UTC')::timestamptz AS bucket, - COUNT(DISTINCT p.packet_hash) AS advert_count, - COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (0, 1)) AS flood_advert_count, - COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (2, 3)) AS direct_advert_count, - MAX(po.heard_at) AS last_heard -FROM packets p -JOIN packet_observations po ON po.packet_hash = p.packet_hash -JOIN nodes n ON n.public_key = p.origin_pubkey -WHERE p.payload_type = 4 -- ADVERT - AND po.heard_at > NOW() - INTERVAL '30 days' -GROUP BY po.iata, n.id, n.name, n.node_type, date_trunc('hour', po.heard_at, 'UTC'); - CREATE TABLE observers ( id uuid DEFAULT gen_random_uuid() NOT NULL, public_key bytea NOT NULL, @@ -453,22 +106,11 @@ CREATE TABLE observers ( last_seen timestamp with time zone DEFAULT now() NOT NULL, observation_count bigint DEFAULT 0, metadata jsonb, - region_scope text + region_scope text, + last_iata character(3), + last_iata_at timestamp with time zone ); -CREATE VIEW analytics_live_top_observers_by_iata AS -SELECT - po.iata, - po.observer_id, - o.display_name, - o.observer_type, - date_trunc('hour', po.heard_at, 'UTC')::timestamptz AS bucket, - COUNT(*) AS observation_count -FROM packet_observations po -JOIN observers o ON o.id = po.observer_id -WHERE po.heard_at > NOW() - INTERVAL '30 days' -GROUP BY po.iata, po.observer_id, o.display_name, o.observer_type, date_trunc('hour', po.heard_at, 'UTC'); - CREATE TABLE channel_messages ( id bigint NOT NULL, channel_id integer NOT NULL, @@ -479,91 +121,6 @@ CREATE TABLE channel_messages ( sent_at timestamp with time zone NOT NULL ); -CREATE VIEW analytics_live_top_talkers_by_iata AS -SELECT - po.iata, - cm.sender_name, - date_trunc('hour', cm.sent_at, 'UTC')::timestamptz AS bucket, - COUNT(DISTINCT cm.id) AS message_count, - MAX(cm.sent_at) AS last_sent -FROM channel_messages cm -JOIN packet_observations po ON po.packet_hash = cm.packet_hash -WHERE cm.sender_name IS NOT NULL - AND cm.sent_at > NOW() - INTERVAL '30 days' -GROUP BY po.iata, cm.sender_name, date_trunc('hour', cm.sent_at, 'UTC'); - -CREATE TABLE analytics_observer_activity_hourly ( - observer_id uuid NOT NULL, - payload_type smallint NOT NULL, - bucket timestamp with time zone NOT NULL, - observations bigint DEFAULT 0 NOT NULL, - airtime_ms real, - airtime_n bigint DEFAULT 0 NOT NULL, - snr_sum real, - snr_n bigint DEFAULT 0 NOT NULL, - snr_min real, - rssi_sum bigint, - rssi_n bigint DEFAULT 0 NOT NULL -); - -CREATE TABLE analytics_path_stats_hourly ( - iata character(3) NOT NULL, - hour timestamp with time zone NOT NULL, - category integer NOT NULL, - hash_bytes integer NOT NULL, - entries integer NOT NULL, - receptions bigint DEFAULT 0 NOT NULL -); - -CREATE TABLE analytics_payload_breakdown_by_iata ( - iata character(3) NOT NULL, - payload_type smallint NOT NULL, - bucket timestamp with time zone NOT NULL, - count bigint DEFAULT 0 NOT NULL -); - -CREATE TABLE analytics_signal_stats_hourly ( - iata character(3) NOT NULL, - hour timestamp with time zone NOT NULL, - kind integer NOT NULL, - snr_bin integer NOT NULL, - rssi_bin integer NOT NULL, - receptions bigint DEFAULT 0 NOT NULL, - snr_samples bigint DEFAULT 0 NOT NULL, - snr_sum double precision, - rssi_samples bigint DEFAULT 0 NOT NULL, - rssi_sum double precision -); - -CREATE TABLE analytics_top_advertisers_by_iata ( - iata character(3) NOT NULL, - node_id uuid NOT NULL, - bucket timestamp with time zone NOT NULL, - advert_count bigint DEFAULT 0 NOT NULL, - flood_advert_count bigint DEFAULT 0 NOT NULL, - direct_advert_count bigint DEFAULT 0 NOT NULL, - last_heard timestamp with time zone, - name text, - node_type smallint -); - -CREATE TABLE analytics_top_observers_by_iata ( - iata character(3) NOT NULL, - observer_id uuid NOT NULL, - bucket timestamp with time zone NOT NULL, - observation_count bigint DEFAULT 0 NOT NULL, - display_name text, - observer_type text -); - -CREATE TABLE analytics_top_talkers_by_iata ( - iata character(3) NOT NULL, - sender_name text NOT NULL, - bucket timestamp with time zone NOT NULL, - message_count bigint DEFAULT 0 NOT NULL, - last_sent timestamp with time zone -); - CREATE TABLE channel_config_scopes ( key_fingerprint bytea NOT NULL, region_slug text @@ -603,7 +160,7 @@ CREATE TABLE channels ( key_known boolean DEFAULT false, first_seen timestamp with time zone DEFAULT now() NOT NULL, last_seen timestamp with time zone DEFAULT now() NOT NULL, - message_count bigint DEFAULT 0 + message_count bigint DEFAULT 0 NOT NULL ); CREATE SEQUENCE channels_id_seq @@ -696,53 +253,6 @@ CREATE TABLE meshmapper_zone_lists ( last_error text DEFAULT ''::text NOT NULL ); -CREATE MATERIALIZED VIEW mv_hourly_iata_stats AS -WITH combined AS ( - SELECT iata, hour, observation_count, unique_packets FROM analytics_live_hourly_iata_stats - UNION ALL - SELECT iata, hour, observation_count, unique_packets FROM analytics_hourly_iata_stats - WHERE hour >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -), observers_by_hour AS ( - SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, observer_id - FROM packet_observations WHERE heard_at > NOW() - INTERVAL '30 days' - UNION - SELECT iata, bucket AS hour, observer_id FROM analytics_top_observers_by_iata - WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -), observer_counts AS (SELECT iata, hour, COUNT(*) AS n FROM observers_by_hour GROUP BY iata,hour) -SELECT combined.iata, combined.hour, SUM(observation_count)::bigint AS observation_count, SUM(unique_packets)::bigint AS unique_packets, MAX(o.n)::bigint AS active_observers -FROM combined JOIN observer_counts o USING (iata,hour) GROUP BY combined.iata,combined.hour; - -CREATE MATERIALIZED VIEW mv_observer_activity_hourly AS -WITH combined AS ( - SELECT observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n FROM analytics_live_observer_activity_hourly - UNION ALL - SELECT observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n FROM analytics_observer_activity_hourly - WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -) -SELECT observer_id, payload_type, bucket, SUM(observations)::bigint AS observations, SUM(airtime_ms)::real AS airtime_ms, SUM(airtime_n)::bigint AS airtime_n, SUM(snr_sum)::real AS snr_sum, SUM(snr_n)::bigint AS snr_n, MIN(snr_min)::real AS snr_min, SUM(rssi_sum)::bigint AS rssi_sum, SUM(rssi_n)::bigint AS rssi_n -FROM combined GROUP BY observer_id,payload_type,bucket; - -CREATE MATERIALIZED VIEW mv_path_stats_hourly AS -WITH combined AS ( - SELECT iata, hour, category, hash_bytes, entries, receptions FROM analytics_live_path_stats_hourly - UNION ALL - SELECT iata, hour, category, hash_bytes, entries, receptions FROM analytics_path_stats_hourly - WHERE hour >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' - AND hour < date_trunc('hour', NOW(), 'UTC') -) -SELECT iata, hour, category, hash_bytes, entries, SUM(receptions)::bigint AS receptions -FROM combined GROUP BY iata,hour,category,hash_bytes,entries; - -CREATE MATERIALIZED VIEW mv_payload_breakdown_by_iata AS -WITH combined AS ( - SELECT iata, payload_type, bucket, count FROM analytics_live_payload_breakdown_by_iata - UNION ALL - SELECT iata, payload_type, bucket, count FROM analytics_payload_breakdown_by_iata - WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -) -SELECT iata, payload_type, bucket, SUM(count)::bigint AS count -FROM combined GROUP BY iata,payload_type,bucket; - CREATE TABLE node_iatas ( node_id uuid NOT NULL, iata character(3) NOT NULL, @@ -755,16 +265,15 @@ WITH (autovacuum_vacuum_scale_factor='0.05'); CREATE MATERIALIZED VIEW mv_radio_presets AS SELECT concat(o.radio_freq_mhz, ',', o.radio_bw_khz, ',', o.radio_sf)::text AS preset, - (SELECT po.iata FROM packet_observations po WHERE po.observer_id = o.id ORDER BY po.heard_at DESC LIMIT 1) AS iata, - 'observer' AS source_type, + o.last_iata AS iata, + 'observer'::text AS source_type, COUNT(*) AS count FROM observers o WHERE o.radio_freq_mhz IS NOT NULL AND o.radio_sf IS NOT NULL AND o.radio_bw_khz IS NOT NULL -GROUP BY concat(o.radio_freq_mhz, ',', o.radio_bw_khz, ',', o.radio_sf)::text, - (SELECT po.iata FROM packet_observations po WHERE po.observer_id = o.id ORDER BY po.heard_at DESC LIMIT 1) -HAVING (SELECT po.iata FROM packet_observations po WHERE po.observer_id = o.id ORDER BY po.heard_at DESC LIMIT 1) IS NOT NULL + AND o.last_iata IS NOT NULL +GROUP BY 1, 2 UNION ALL @@ -778,64 +287,7 @@ JOIN node_iatas ni ON ni.node_id = n.id WHERE n.radio_freq_mhz IS NOT NULL AND n.radio_sf IS NOT NULL AND n.radio_bw_khz IS NOT NULL -GROUP BY concat(n.radio_freq_mhz, ',', n.radio_bw_khz, ',', n.radio_sf), ni.iata; - -CREATE MATERIALIZED VIEW mv_signal_stats_hourly AS -WITH combined AS ( - SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM analytics_live_signal_stats_hourly - UNION ALL - SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM analytics_signal_stats_hourly - WHERE hour >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' - AND hour < date_trunc('hour', NOW(), 'UTC') -) -SELECT iata, hour, kind, snr_bin, rssi_bin, SUM(receptions)::bigint AS receptions, SUM(snr_samples)::bigint AS snr_samples, SUM(snr_sum)::double precision AS snr_sum, SUM(rssi_samples)::bigint AS rssi_samples, SUM(rssi_sum)::double precision AS rssi_sum -FROM combined GROUP BY iata,hour,kind,snr_bin,rssi_bin; - -CREATE MATERIALIZED VIEW mv_top_advertisers_by_iata AS -WITH combined AS ( - SELECT iata, node_id, bucket, advert_count, flood_advert_count, direct_advert_count, last_heard, name, node_type FROM analytics_live_top_advertisers_by_iata - UNION ALL - SELECT iata, node_id, bucket, advert_count, flood_advert_count, direct_advert_count, last_heard, name, node_type FROM analytics_top_advertisers_by_iata - WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -), totals AS (SELECT iata, node_id, bucket, SUM(advert_count)::bigint AS advert_count, SUM(flood_advert_count)::bigint AS flood_advert_count, SUM(direct_advert_count)::bigint AS direct_advert_count, MAX(last_heard) AS last_heard, MAX(name) AS name, MAX(node_type) AS node_type -FROM combined GROUP BY iata,node_id,bucket) -SELECT t.iata, t.node_id, COALESCE(n.name,t.name) AS name, COALESCE(n.node_type,t.node_type) AS node_type, - t.bucket, t.advert_count, t.flood_advert_count, t.direct_advert_count, t.last_heard -FROM totals t LEFT JOIN nodes n ON n.id=t.node_id; - -CREATE MATERIALIZED VIEW mv_top_nodes_by_iata AS -SELECT - ni.iata, - ni.node_id, - n.name, - n.node_type, - ni.observation_count, - ni.last_heard -FROM node_iatas ni -JOIN nodes n ON n.id = ni.node_id -WHERE ni.last_heard > NOW() - INTERVAL '7 days'; - -CREATE MATERIALIZED VIEW mv_top_observers_by_iata AS -WITH combined AS ( - SELECT iata, observer_id, bucket, observation_count, display_name, observer_type FROM analytics_live_top_observers_by_iata - UNION ALL - SELECT iata, observer_id, bucket, observation_count, display_name, observer_type FROM analytics_top_observers_by_iata - WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -), totals AS (SELECT iata, observer_id, bucket, SUM(observation_count)::bigint AS observation_count, MAX(display_name) AS display_name, MAX(observer_type) AS observer_type -FROM combined GROUP BY iata,observer_id,bucket) -SELECT t.iata, t.observer_id, COALESCE(o.display_name,t.display_name) AS display_name, - COALESCE(o.observer_type,t.observer_type) AS observer_type, t.bucket, t.observation_count -FROM totals t LEFT JOIN observers o ON o.id=t.observer_id; - -CREATE MATERIALIZED VIEW mv_top_talkers_by_iata AS -WITH combined AS ( - SELECT iata, sender_name, bucket, message_count, last_sent FROM analytics_live_top_talkers_by_iata - UNION ALL - SELECT iata, sender_name, bucket, message_count, last_sent FROM analytics_top_talkers_by_iata - WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -) -SELECT iata, sender_name, bucket, SUM(message_count)::bigint AS message_count, MAX(last_sent) AS last_sent -FROM combined GROUP BY iata,sender_name,bucket; +GROUP BY 1, 2; CREATE TABLE node_neighbors ( node_id uuid NOT NULL, @@ -962,6 +414,168 @@ CREATE TABLE trace_iatas ( last_heard timestamp with time zone DEFAULT now() NOT NULL ); +CREATE TABLE trace_tags ( + trace_tag bytea PRIMARY KEY, + first_heard_at timestamp with time zone NOT NULL, + last_heard_at timestamp with time zone NOT NULL, + packet_count bigint DEFAULT 0 NOT NULL, + trace_type text, + scope_id integer, + best_payload jsonb, + best_path_len integer DEFAULT -1 NOT NULL, + heard boolean DEFAULT false NOT NULL -- false: times are the packet's provisional ones until a hearing is recorded +); + +-- Canonical IATA set: sorted, unique, non-null, non-empty. +CREATE FUNCTION iata_set_is_canonical(s bpchar[]) RETURNS boolean + LANGUAGE sql IMMUTABLE STRICT AS $$ + SELECT cardinality(s) > 0 + AND array_position(s, NULL) IS NULL + AND s = (SELECT array_agg(x ORDER BY x) FROM (SELECT DISTINCT unnest(s) AS x) d) +$$; + +-- Hourly rollups. hour = UTC date_trunc('hour', heard_at). No FKs: history outlives deletions. +CREATE TABLE analytics_hourly_iata_observations ( + hour timestamp with time zone NOT NULL, + iata character(3) NOT NULL, + observation_count bigint NOT NULL, + PRIMARY KEY (hour, iata) +); + +CREATE TABLE analytics_hourly_payload_breakdown ( + hour timestamp with time zone NOT NULL, + iata character(3) NOT NULL, + payload_type smallint NOT NULL, + count bigint NOT NULL, + PRIMARY KEY (hour, iata, payload_type) +); + +CREATE TABLE analytics_hourly_signal ( + hour timestamp with time zone NOT NULL, + iata character(3) NOT NULL, + kind integer NOT NULL, + snr_bin integer NOT NULL, + rssi_bin integer NOT NULL, + receptions bigint NOT NULL, + snr_samples bigint NOT NULL, + snr_sum double precision NOT NULL, + rssi_samples bigint NOT NULL, + rssi_sum double precision NOT NULL, + PRIMARY KEY (hour, iata, kind, snr_bin, rssi_bin) +); + +CREATE TABLE analytics_hourly_paths ( + hour timestamp with time zone NOT NULL, + iata character(3) NOT NULL, + category integer NOT NULL, + hash_bytes integer NOT NULL, + entries integer NOT NULL, + receptions bigint NOT NULL, + PRIMARY KEY (hour, iata, category, hash_bytes, entries) +); + +CREATE TABLE analytics_hourly_observer_activity ( + hour timestamp with time zone NOT NULL, + observer_id uuid NOT NULL, + payload_type smallint NOT NULL, + observations bigint NOT NULL, + airtime_ms real, + airtime_n bigint NOT NULL, + snr_sum real, + snr_n bigint NOT NULL, + snr_min real, + rssi_sum bigint, + rssi_n bigint NOT NULL, + PRIMARY KEY (hour, observer_id, payload_type) +); + +CREATE TABLE analytics_hourly_observer_identity ( + hour timestamp with time zone NOT NULL, + iata character(3) NOT NULL, + observer_id uuid NOT NULL, + observation_count bigint NOT NULL, + display_name text, + observer_type text, + PRIMARY KEY (hour, iata, observer_id) +); + +-- ADVERT hearings per origin per IATA: top nodes (observations) and advertiser last_heard. +CREATE TABLE analytics_hourly_advert_hearings ( + hour timestamp with time zone NOT NULL, + iata character(3) NOT NULL, + origin_pubkey bytea NOT NULL, + observations bigint NOT NULL, + last_heard timestamp with time zone NOT NULL, + name text, + node_type smallint, + PRIMARY KEY (hour, iata, origin_pubkey) +); + +-- Distinct ADVERT packets per origin, grouped by the set of IATAs that heard each packet that hour. +CREATE TABLE analytics_hourly_advert_sets ( + hour timestamp with time zone NOT NULL, + origin_pubkey bytea NOT NULL, + iatas character(3)[] NOT NULL CHECK (iata_set_is_canonical(iatas)), + advert_packets bigint NOT NULL, + flood_packets bigint NOT NULL, + direct_packets bigint NOT NULL, + PRIMARY KEY (hour, origin_pubkey, iatas) +); + +-- Distinct channel messages per sender, grouped by the set of IATAs that heard each message that hour. +CREATE TABLE analytics_hourly_talker_sets ( + hour timestamp with time zone NOT NULL, + sender_name text NOT NULL, + iatas character(3)[] NOT NULL CHECK (iata_set_is_canonical(iatas)), + messages bigint NOT NULL, + last_sent timestamp with time zone NOT NULL, + PRIMARY KEY (hour, sender_name, iatas) +); + +CREATE TABLE analytics_hourly_packet_sets ( + hour timestamp with time zone NOT NULL, + iatas character(3)[] NOT NULL CHECK (iata_set_is_canonical(iatas)), + packets bigint NOT NULL, + PRIMARY KEY (hour, iatas) +); + +CREATE TABLE analytics_hourly_scope_sets ( + hour timestamp with time zone NOT NULL, + iatas character(3)[] NOT NULL CHECK (iata_set_is_canonical(iatas)), + scope_id integer NOT NULL, + packets bigint NOT NULL, + PRIMARY KEY (hour, iatas, scope_id) +); + +CREATE TABLE analytics_rollup_hours ( + hour timestamp with time zone PRIMARY KEY, + status text NOT NULL CHECK (status IN ('complete', 'partial', 'missing')), + rolled_at timestamp with time zone, + content_hash text +); + +-- Singletons: the rollup writes revision, cleanup writes raw_deleted_before. +CREATE TABLE analytics_state ( + singleton boolean PRIMARY KEY DEFAULT true CHECK (singleton), + revision bigint NOT NULL DEFAULT 0 +); + +INSERT INTO analytics_state DEFAULT VALUES; + +-- coverage counts retention deletions of rollup hours; cache keys combine it with revision. +CREATE TABLE analytics_raw_state ( + singleton boolean PRIMARY KEY DEFAULT true CHECK (singleton), + raw_deleted_before timestamp with time zone, + coverage bigint NOT NULL DEFAULT 0 +); + +INSERT INTO analytics_raw_state DEFAULT VALUES; + +CREATE TABLE analytics_dirty_hours ( + hour timestamp with time zone PRIMARY KEY, + enqueued_at timestamp with time zone NOT NULL DEFAULT now() +); + CREATE TABLE transport_scopes ( id integer NOT NULL, name text NOT NULL, @@ -997,30 +611,6 @@ ALTER TABLE ONLY transport_scopes ALTER COLUMN id SET DEFAULT nextval('transport ALTER TABLE ONLY accounts ADD CONSTRAINT accounts_pkey PRIMARY KEY (id); -ALTER TABLE ONLY analytics_hourly_iata_stats - ADD CONSTRAINT analytics_hourly_iata_stats_pkey PRIMARY KEY (iata, hour); - -ALTER TABLE ONLY analytics_observer_activity_hourly - ADD CONSTRAINT analytics_observer_activity_hourly_pkey PRIMARY KEY (observer_id, payload_type, bucket); - -ALTER TABLE ONLY analytics_path_stats_hourly - ADD CONSTRAINT analytics_path_stats_hourly_pkey PRIMARY KEY (iata, hour, category, hash_bytes, entries); - -ALTER TABLE ONLY analytics_payload_breakdown_by_iata - ADD CONSTRAINT analytics_payload_breakdown_by_iata_pkey PRIMARY KEY (iata, payload_type, bucket); - -ALTER TABLE ONLY analytics_signal_stats_hourly - ADD CONSTRAINT analytics_signal_stats_hourly_pkey PRIMARY KEY (iata, hour, kind, snr_bin, rssi_bin); - -ALTER TABLE ONLY analytics_top_advertisers_by_iata - ADD CONSTRAINT analytics_top_advertisers_by_iata_pkey PRIMARY KEY (iata, node_id, bucket); - -ALTER TABLE ONLY analytics_top_observers_by_iata - ADD CONSTRAINT analytics_top_observers_by_iata_pkey PRIMARY KEY (iata, observer_id, bucket); - -ALTER TABLE ONLY analytics_top_talkers_by_iata - ADD CONSTRAINT analytics_top_talkers_by_iata_pkey PRIMARY KEY (iata, sender_name, bucket); - ALTER TABLE ONLY channel_iatas ADD CONSTRAINT channel_iatas_pkey PRIMARY KEY (channel_hash, iata); @@ -1131,38 +721,6 @@ ALTER TABLE ONLY transport_scopes CREATE UNIQUE INDEX accounts_name_uidx ON accounts USING btree (name) WHERE (deactivated_at IS NULL); -CREATE INDEX idx_analytics_hourly_iata_stats_time ON analytics_hourly_iata_stats USING btree (hour); - -CREATE UNIQUE INDEX idx_analytics_hourly_iata_stats_view ON mv_hourly_iata_stats USING btree (iata, hour); - -CREATE INDEX idx_analytics_observer_activity_hourly_time ON analytics_observer_activity_hourly USING btree (bucket); - -CREATE UNIQUE INDEX idx_analytics_observer_activity_hourly_view ON mv_observer_activity_hourly USING btree (observer_id, payload_type, bucket); - -CREATE INDEX idx_analytics_path_stats_hourly_time ON analytics_path_stats_hourly USING btree (hour); - -CREATE UNIQUE INDEX idx_analytics_path_stats_hourly_view ON mv_path_stats_hourly USING btree (iata, hour, category, hash_bytes, entries); - -CREATE INDEX idx_analytics_payload_breakdown_by_iata_time ON analytics_payload_breakdown_by_iata USING btree (bucket); - -CREATE UNIQUE INDEX idx_analytics_payload_breakdown_by_iata_view ON mv_payload_breakdown_by_iata USING btree (iata, payload_type, bucket); - -CREATE INDEX idx_analytics_signal_stats_hourly_time ON analytics_signal_stats_hourly USING btree (hour); - -CREATE UNIQUE INDEX idx_analytics_signal_stats_hourly_view ON mv_signal_stats_hourly USING btree (iata, hour, kind, snr_bin, rssi_bin); - -CREATE INDEX idx_analytics_top_advertisers_by_iata_time ON analytics_top_advertisers_by_iata USING btree (bucket); - -CREATE UNIQUE INDEX idx_analytics_top_advertisers_by_iata_view ON mv_top_advertisers_by_iata USING btree (iata, node_id, bucket); - -CREATE INDEX idx_analytics_top_observers_by_iata_time ON analytics_top_observers_by_iata USING btree (bucket); - -CREATE UNIQUE INDEX idx_analytics_top_observers_by_iata_view ON mv_top_observers_by_iata USING btree (iata, observer_id, bucket); - -CREATE INDEX idx_analytics_top_talkers_by_iata_time ON analytics_top_talkers_by_iata USING btree (bucket); - -CREATE UNIQUE INDEX idx_analytics_top_talkers_by_iata_view ON mv_top_talkers_by_iata USING btree (iata, sender_name, bucket); - CREATE INDEX idx_channel_iatas_iata ON channel_iatas USING btree (iata); CREATE INDEX idx_channel_messages_channel ON channel_messages USING btree (channel_id, sent_at DESC); @@ -1189,14 +747,8 @@ CREATE INDEX idx_known_routes_last_seen ON known_routes USING btree (last_seen D CREATE INDEX idx_known_routes_reconfirm ON known_routes USING btree (last_reconfirmed_at); -CREATE INDEX idx_mv_path_stats_hourly_hour ON mv_path_stats_hourly USING btree (hour); - CREATE UNIQUE INDEX idx_mv_radio_presets ON mv_radio_presets USING btree (preset, iata, source_type); -CREATE INDEX idx_mv_signal_stats_hourly_hour ON mv_signal_stats_hourly USING btree (hour); - -CREATE UNIQUE INDEX idx_mv_top_nodes ON mv_top_nodes_by_iata USING btree (iata, node_id); - CREATE INDEX idx_node_iatas_iata ON node_iatas USING btree (iata, last_heard DESC); CREATE INDEX idx_node_neighbors_neighbor ON node_neighbors USING btree (neighbor_id, iata); @@ -1215,12 +767,14 @@ CREATE INDEX idx_nodes_pubkey ON nodes USING btree (public_key); CREATE INDEX idx_nodes_type_last_seen ON nodes USING btree (node_type, last_seen DESC); -CREATE INDEX idx_observations_heard_brin ON packet_observations USING brin (heard_at); +CREATE INDEX idx_observations_heard ON packet_observations USING btree (heard_at); CREATE INDEX idx_observations_iata_heard ON packet_observations USING btree (iata, heard_at DESC); CREATE INDEX idx_observations_observer ON packet_observations USING btree (observer_id, heard_at DESC); +CREATE INDEX idx_observations_observer_id ON packet_observations USING btree (observer_id, id); + CREATE INDEX idx_observations_packet ON packet_observations USING btree (packet_hash); CREATE INDEX idx_observations_route_evidence ON packet_observations USING btree (iata, hash_size, decode(md5(path_bytes), 'hex'::text), heard_at DESC, id DESC) WHERE ((path_bytes IS NOT NULL) AND (payload_type IS NOT NULL) AND (payload_type <> 9) AND (hop_count >= 2)); @@ -1229,6 +783,8 @@ CREATE INDEX idx_observer_locations_recent ON observer_locations USING btree (ob CREATE INDEX idx_observer_owners_node ON observer_owners USING btree (owner_node_id) WHERE (owner_node_id IS NOT NULL); +CREATE INDEX idx_observers_last_iata ON observers USING btree (last_iata) WHERE (last_iata IS NOT NULL); + CREATE INDEX idx_observers_last_seen ON observers USING btree (last_seen DESC); CREATE INDEX idx_observers_pubkey ON observers USING btree (public_key); @@ -1267,6 +823,24 @@ CREATE INDEX idx_telemetry_reported_brin ON observer_telemetry USING brin (repor CREATE INDEX idx_trace_iatas_iata ON trace_iatas USING btree (iata); +CREATE INDEX idx_trace_tags_last_heard ON trace_tags USING btree (last_heard_at DESC); + +CREATE INDEX idx_trace_tags_first_heard ON trace_tags USING btree (first_heard_at); + +CREATE INDEX idx_analytics_observer_activity_observer ON analytics_hourly_observer_activity USING btree (observer_id, hour); + +CREATE INDEX idx_analytics_packet_sets_iatas ON analytics_hourly_packet_sets USING gin (iatas); + +CREATE INDEX idx_analytics_scope_sets_iatas ON analytics_hourly_scope_sets USING gin (iatas); + +CREATE INDEX idx_analytics_scope_sets_scope ON analytics_hourly_scope_sets USING btree (scope_id, hour); + +CREATE INDEX idx_analytics_advert_sets_iatas ON analytics_hourly_advert_sets USING gin (iatas); + +CREATE INDEX idx_analytics_talker_sets_iatas ON analytics_hourly_talker_sets USING gin (iatas); + +CREATE INDEX idx_analytics_rollup_hours_missing ON analytics_rollup_hours USING btree (hour) WHERE (status = 'missing'::text); + CREATE INDEX idx_transport_scopes_fingerprint ON transport_scopes USING btree (key_fingerprint); ALTER TABLE ONLY channel_iatas @@ -1353,7 +927,6 @@ ALTER TABLE ONLY region_iatas ALTER TABLE ONLY trace_iatas ADD CONSTRAINT trace_iatas_iata_fkey FOREIGN KEY (iata) REFERENCES iata_codes(iata) ON DELETE CASCADE; - -- -- diff --git a/db/observer_last_iata_integration_test.go b/db/observer_last_iata_integration_test.go new file mode 100644 index 0000000..ff1c4af --- /dev/null +++ b/db/observer_last_iata_integration_test.go @@ -0,0 +1,124 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "bytes" + "context" + "os" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/ingest" + "github.com/MeshCore-Beacon/beacon-server/internal/presence" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" +) + +func TestObserverLastIATAPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for PostgreSQL regression") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + applyBaseline(t, ctx, tx) + store := &Store{q: sqlc.New(tx)} + c := presence.New(store, time.Hour, time.Hour) + pkA, pkB, pkC := bytes.Repeat([]byte{1}, 32), bytes.Repeat([]byte{2}, 32), bytes.Repeat([]byte{3}, 32) + ids := map[string]uuid.UUID{} + heard := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + // Mirrors ingest: upsert the observer, then record the observation. + packet := func(name string, pk []byte, iata string) { + t.Helper() + id, _, err := c.UpsertObserver(ctx, pk, iata) + if err != nil { + t.Fatal(err) + } + ids[name] = id + heard = heard.Add(time.Second) + hash := []byte(heard.String()) + if _, err := tx.Exec(ctx, `INSERT INTO packets (packet_hash, payload_type, payload_version, route_type, raw_payload, raw_header, first_heard_at, last_heard_at) +VALUES ($1, 4, 0, 1, '\x00', '\x00', $2, $2)`, hash, heard); err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(ctx, `INSERT INTO packet_observations (packet_hash, observer_id, iata, heard_at, path_length_byte, hash_size, hop_count) +VALUES ($1, $2, $3, $4, 0, 1, 0)`, hash, id, iata, heard); err != nil { + t.Fatal(err) + } + } + status := func(pk []byte) { + t.Helper() + if _, _, err := c.UpsertObserver(ctx, pk, ""); err != nil { + t.Fatal(err) + } + } + + packet("A", pkA, "YVR") + packet("A", pkA, "YYZ") + status(pkA) + packet("B", pkB, "YOW") + if _, err := c.UpdateObserverStatus(ctx, ingest.UpdateObserverStatusParams{PublicKey: pkB, DisplayName: "b"}); err != nil { + t.Fatal(err) + } + packet("B", pkB, "YYJ") // write-through after the identity was dropped + status(pkC) + c.Flush(ctx) + packet("A", pkA, "YVR") // pending in the coalescer, not yet flushed + c.Flush(ctx) + + rows, err := tx.Query(ctx, `SELECT o.id, COALESCE(o.last_iata, ''), COALESCE(( + SELECT po.iata FROM packet_observations po WHERE po.observer_id = o.id ORDER BY po.heard_at DESC LIMIT 1), '') +FROM observers o`) + if err != nil { + t.Fatal(err) + } + n := 0 + for rows.Next() { + var id uuid.UUID + var got, want string + if err := rows.Scan(&id, &got, &want); err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("observer %s last_iata = %q, latest observation = %q", id, got, want) + } + n++ + } + if rows.Err() != nil || n != 3 { + t.Fatalf("observers = %d, err %v; want 3", n, rows.Err()) + } + + for name, want := range map[string]string{"A": "YVR", "B": "YYJ"} { + if got, err := store.GetObserverLastIATA(ctx, ids[name]); err != nil || got != want { + t.Errorf("GetObserverLastIATA(%s) = %q, %v; want %q", name, got, err, want) + } + } + var idC uuid.UUID + if err := tx.QueryRow(ctx, "SELECT id FROM observers WHERE public_key = $1", pkC).Scan(&idC); err != nil { + t.Fatal(err) + } + if got, err := store.GetObserverLastIATA(ctx, idC); err != nil || got != "" { + t.Errorf("GetObserverLastIATA(no packets) = %q, %v; want empty", got, err) + } + + page, err := store.ListObservers(ctx, []string{"YYJ", "YYZ"}, "", "", "", "", "", 0, 10) + if err != nil { + t.Fatal(err) + } + if len(page.Items) != 1 || page.Items[0].IATA != "YYJ" { + t.Errorf("ListObservers(YYJ,YYZ) = %+v; want only B in YYJ", page.Items) + } +} diff --git a/db/observer_metrics_integration_test.go b/db/observer_metrics_integration_test.go index 64c82df..5c0b66b 100644 --- a/db/observer_metrics_integration_test.go +++ b/db/observer_metrics_integration_test.go @@ -24,13 +24,11 @@ func TestObserverMetricsPostgres(t *testing.T) { } store := &Store{q: sqlc.New(tx)} id := uuid.MustParse("00000000-0000-0000-0000-000000000001") + // Hourly activity is rolled before cleanup and survives the raw rows. + rollTxHours(t, ctx, tx) if err := store.DeleteOldPackets(ctx, time.Now().Add(-72*time.Hour)); err != nil { t.Fatal(err) } - // The initial archive migration retains unknown payloads without a second rebuild. - if err := store.RefreshObserverActivity(ctx); err != nil { - t.Fatal(err) - } until := time.Now().UTC().Truncate(time.Hour) activity, err := store.GetObserverActivity(ctx, id, 7*24*time.Hour, time.Hour, until) if err != nil { diff --git a/db/observer_retention_integration_test.go b/db/observer_retention_integration_test.go index 8e64ea9..1992c53 100644 --- a/db/observer_retention_integration_test.go +++ b/db/observer_retention_integration_test.go @@ -96,7 +96,7 @@ SELECT id,1 FROM observers WHERE display_name='expired';`) exec("TRUNCATE pg_temp.observers CASCADE") coalescer := presence.New(store, time.Second, time.Second) pubkey := bytes.Repeat([]byte{42}, 32) - oldID, _, err := coalescer.UpsertObserver(ctx, pubkey) + oldID, _, err := coalescer.UpsertObserver(ctx, pubkey, "YVR") if err != nil { t.Fatal(err) } @@ -108,7 +108,7 @@ SELECT id,1 FROM observers WHERE display_name='expired';`) if err != nil || len(ids) != 1 || ids[0] != oldID { t.Fatalf("cached expired observer was not deleted: %v, %v", ids, err) } - newID, _, err := coalescer.UpsertObserver(ctx, pubkey) + newID, _, err := coalescer.UpsertObserver(ctx, pubkey, "YVR") if err != nil || newID == oldID { t.Fatalf("returning observer reused a deleted ID: %v", err) } @@ -122,7 +122,7 @@ SELECT id,1 FROM observers WHERE display_name='expired';`) // The persisted row looks stale while fresh activity is still coalesced. cutoff = time.Now().Add(-24 * time.Hour) exec("UPDATE observers SET last_seen=$1 WHERE id=$2", cutoff.Add(-time.Hour), newID) - if _, _, err := coalescer.UpsertObserver(ctx, pubkey); err != nil { + if _, _, err := coalescer.UpsertObserver(ctx, pubkey, "YVR"); err != nil { t.Fatal(err) } if ids, err := coalescer.DeleteOldObservers(ctx, cutoff); err != nil || len(ids) != 0 { diff --git a/db/observers.go b/db/observers.go index 566256c..395e887 100644 --- a/db/observers.go +++ b/db/observers.go @@ -20,8 +20,8 @@ import ( "github.com/jackc/pgx/v5/pgtype" ) -func (s *Store) UpsertObserver(ctx context.Context, pubkey []byte) (uuid.UUID, string, error) { - row, err := s.q.UpsertObserver(ctx, pubkey) +func (s *Store) UpsertObserver(ctx context.Context, pubkey []byte, iata string) (uuid.UUID, string, error) { + row, err := s.q.UpsertObserver(ctx, sqlc.UpsertObserverParams{PublicKey: pubkey, Iata: iata, IataAt: ts(time.Now())}) if err != nil { return uuid.Nil, "", err } diff --git a/db/observers_test.go b/db/observers_test.go index 20597f8..c60572d 100644 --- a/db/observers_test.go +++ b/db/observers_test.go @@ -28,11 +28,13 @@ func TestUpsertObserver_NilDisplayName(t *testing.T) { pubkey := []byte{0x01, 0x02} mock.EXPECT(). - UpsertObserver(gomock.Any(), pubkey). + UpsertObserver(gomock.Any(), gomock.Cond(func(p sqlc.UpsertObserverParams) bool { + return string(p.PublicKey) == string(pubkey) && p.Iata == "YVR" && p.IataAt.Valid + })). Return(sqlc.Observer{ID: observerID, DisplayName: nil}, nil) store := &Store{q: mock} - id, displayName, err := store.UpsertObserver(context.Background(), pubkey) + id, displayName, err := store.UpsertObserver(context.Background(), pubkey, "YVR") if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -53,11 +55,13 @@ func TestUpsertObserver_WithDisplayName(t *testing.T) { name := "test-observer" mock.EXPECT(). - UpsertObserver(gomock.Any(), pubkey). + UpsertObserver(gomock.Any(), gomock.Cond(func(p sqlc.UpsertObserverParams) bool { + return string(p.PublicKey) == string(pubkey) && p.Iata == "YVR" && p.IataAt.Valid + })). Return(sqlc.Observer{ID: observerID, DisplayName: &name}, nil) store := &Store{q: mock} - _, displayName, err := store.UpsertObserver(context.Background(), pubkey) + _, displayName, err := store.UpsertObserver(context.Background(), pubkey, "YVR") if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/db/packet_cleanup_integration_test.go b/db/packet_cleanup_integration_test.go new file mode 100644 index 0000000..eb3815f --- /dev/null +++ b/db/packet_cleanup_integration_test.go @@ -0,0 +1,205 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/jackc/pgx/v5" +) + +// Raw deletion and the rollup meet at raw_deleted_before: an hour registered but not rolled +// before its raw rows go becomes partial, while hours behind the holdback roll completely. +func TestPacketCleanupRollupHandoffPostgres(t *testing.T) { + ctx, pool := schemaPool(t) + h0 := rollupFixture(t, ctx, pool) + store := New(pool, 0, 0) + old := h0.Add(-3 * time.Hour).Add(10 * time.Minute) + if _, err := pool.Exec(ctx, ` +INSERT INTO packets (packet_hash, payload_type, payload_version, route_type, raw_payload, raw_header, first_heard_at, last_heard_at) +VALUES ('\xff', 2, 0, 1, '\x00', '\x00', $1::timestamptz, $1::timestamptz); +INSERT INTO packet_observations (id, packet_hash, observer_id, iata, heard_at, path_length_byte, hash_size, hop_count) +VALUES (100, '\xff', '00000000-0000-0000-0000-00000000000a', 'YVR', $1::timestamptz, 0, 1, 0)`, pgx.QueryExecModeSimpleProtocol, old); err != nil { + t.Fatal(err) + } + r, ok, err := store.BeginRollup(ctx) + if err != nil || !ok { + t.Fatalf("BeginRollup: ok=%v err=%v", ok, err) + } + defer r.Close(ctx) + if err := r.RegisterHours(ctx, h0.Add(-10*time.Hour)); err != nil { + t.Fatal(err) + } + oldest, ok, err := store.OldestMissingRollupHour(ctx) + if err != nil || !ok || !oldest.Equal(old.Truncate(time.Hour)) { + t.Fatalf("oldest missing %s %v %v, want %s", oldest, ok, err, old.Truncate(time.Hour)) + } + + // Cleanup held behind h0 (as rawCutoff does when h0 is the oldest unrolled hour) leaves h0 intact. + if err := store.DeleteOldPackets(ctx, h0.Add(-35*time.Minute)); err != nil { + t.Fatal(err) + } + var deletedBefore time.Time + var remaining int + if err := pool.QueryRow(ctx, `SELECT raw_deleted_before, (SELECT count(*) FROM packet_observations) + FROM analytics_raw_state`).Scan(&deletedBefore, &remaining); err != nil { + t.Fatal(err) + } + if !deletedBefore.Equal(old) || remaining != 10 { + t.Fatalf("raw_deleted_before %s, %d observations left; want %s, 10", deletedBefore, remaining, old) + } + + if err := r.MarkPartialHours(ctx); err != nil { + t.Fatal(err) + } + rollAll(t, ctx, r) + var statuses string + if err := pool.QueryRow(ctx, `SELECT string_agg(status, ',' ORDER BY hour) FROM analytics_rollup_hours + WHERE hour BETWEEN $1 AND $2`, old.Truncate(time.Hour), h0.Add(time.Hour)).Scan(&statuses); err != nil { + t.Fatal(err) + } + if statuses != "partial,complete,complete,complete,complete" { + t.Errorf("statuses %q, want the deleted hour partial and the rest complete", statuses) + } + q := rollupOracles["iata observations"] + if got, want := rowsText(t, ctx, pool, q[0]), rowsText(t, ctx, pool, q[1]); got != want { + t.Errorf("held-back hours lost raw rows:\nrollup:\n%s\nraw:\n%s", got, want) + } + + if err := store.DeleteOldRollups(ctx, h0.Add(time.Hour)); err != nil { + t.Fatal(err) + } + var left int + if err := pool.QueryRow(ctx, `SELECT (SELECT count(*) FROM analytics_rollup_hours WHERE hour < $1) + + (SELECT count(*) FROM analytics_hourly_iata_observations WHERE hour < $1)`, h0.Add(time.Hour)).Scan(&left); err != nil || left != 0 { + t.Errorf("rollup rows before the cutoff: %d, %v", left, err) + } +} + +// An in-flight observation insert holds KEY SHARE on its packet; cleanup skips it, and a +// second cleanup runner skips the first one's cohort instead of waiting. +func TestDeleteOldPacketsSkipsLockedPostgres(t *testing.T) { + ctx, pool := schemaPool(t) + if err := RunMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, ` +INSERT INTO iata_codes (iata) VALUES ('YVR'); +INSERT INTO observers (id, public_key) VALUES ('00000000-0000-0000-0000-000000000001', '\x01'); +INSERT INTO packets (packet_hash, payload_type, payload_version, route_type, raw_payload, raw_header, first_heard_at, last_heard_at) +VALUES ('\x01', 4, 0, 1, '\x00', '\x00', now() - INTERVAL '5 days', now() - INTERVAL '5 days')`); err != nil { + t.Fatal(err) + } + params := sqlc.DeleteOldPacketsParams{Cutoff: ts(time.Now().Add(-72 * time.Hour)), BatchSize: 1000} + q := sqlc.New(pool) + + ingest, err := pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer ingest.Rollback(context.Background()) + if _, err := ingest.Exec(ctx, `INSERT INTO packet_observations (packet_hash, observer_id, iata, heard_at, path_length_byte, hash_size, hop_count) +VALUES ('\x01', '00000000-0000-0000-0000-000000000001', 'YVR', now() - INTERVAL '5 days', 0, 1, 0)`); err != nil { + t.Fatal(err) + } + if n, err := q.DeleteOldPackets(ctx, params); err != nil || n != 0 { + t.Fatalf("must skip in-flight reception: %d %v", n, err) + } + if err := ingest.Commit(ctx); err != nil { + t.Fatal(err) + } + + first, err := pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer first.Rollback(context.Background()) + if n, err := sqlc.New(first).DeleteOldPackets(ctx, params); err != nil || n != 1 { + t.Fatalf("first runner: %d %v", n, err) + } + waitCtx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + if n, err := q.DeleteOldPackets(waitCtx, params); err != nil || n != 0 { + t.Fatalf("second runner should skip the locked cohort: %d %v", n, err) + } + if err := first.Commit(ctx); err != nil { + t.Fatal(err) + } + var left int + if err := pool.QueryRow(ctx, "SELECT (SELECT count(*) FROM packets) + (SELECT count(*) FROM packet_observations)").Scan(&left); err != nil || left != 0 { + t.Errorf("rows left after cleanup: %d %v", left, err) + } +} + +// Registration from an hour-aligned rollup cutoff never recreates hours cleanup just expired. +func TestRollupRetentionCyclePostgres(t *testing.T) { + ctx, pool := schemaPool(t) + h0 := rollupFixture(t, ctx, pool) + store := New(pool, 0, 0) + r, ok, err := store.BeginRollup(ctx) + if err != nil || !ok { + t.Fatalf("BeginRollup: ok=%v err=%v", ok, err) + } + defer r.Close(ctx) + cutoff := h0.Add(30 * time.Minute) // DeleteOldRollups cutoff; registration starts at the next hour + since := h0.Add(time.Hour) + for range 3 { + if err := store.DeleteOldRollups(ctx, cutoff); err != nil { + t.Fatal(err) + } + if err := r.RegisterHours(ctx, since); err != nil { + t.Fatal(err) + } + rollAll(t, ctx, r) + } + var stale int + var first time.Time + if err := pool.QueryRow(ctx, "SELECT count(*) FILTER (WHERE hour < $1), min(hour) FROM analytics_rollup_hours", cutoff).Scan(&stale, &first); err != nil { + t.Fatal(err) + } + if stale != 0 || !first.Equal(since) { + t.Errorf("%d expired hours re-registered, first hour %s; want 0, %s", stale, first, since) + } +} + +// Retention deletes and partial marking change what readers see, so they must move the +// revision cache keys are built from, even with no new roll. +func TestRollupCoverageMovesRevisionPostgres(t *testing.T) { + ctx, pool := schemaPool(t) + h0 := rollupFixture(t, ctx, pool) + store := New(pool, 0, 0) + r, ok, err := store.BeginRollup(ctx) + if err != nil || !ok { + t.Fatalf("BeginRollup: ok=%v err=%v", ok, err) + } + defer r.Close(ctx) + if err := r.RegisterHours(ctx, h0); err != nil { + t.Fatal(err) + } + if outcome, _, err := r.RollHour(ctx, h0); err != nil || outcome != RollComplete { + t.Fatalf("roll: %d %v", outcome, err) + } + step := func(name string, change bool, do func() error) { + t.Helper() + before := revision(t, ctx, store) + if err := do(); err != nil { + t.Fatal(err) + } + if moved := revision(t, ctx, store) != before; moved != change { + t.Errorf("%s: revision moved=%v, want %v", name, moved, change) + } + } + step("no-op cleanup", false, func() error { return store.DeleteOldRollups(ctx, h0) }) + step("partial marking", true, func() error { + if _, err := pool.Exec(ctx, "UPDATE analytics_raw_state SET raw_deleted_before = $1", h0.Add(90*time.Minute)); err != nil { + return err + } + return r.MarkPartialHours(ctx) + }) + step("repeat partial marking", false, func() error { return r.MarkPartialHours(ctx) }) + step("retention delete", true, func() error { return store.DeleteOldRollups(ctx, h0.Add(time.Hour)) }) +} diff --git a/db/packet_observation_count_integration_test.go b/db/packet_observation_count_integration_test.go new file mode 100644 index 0000000..9f5ebad --- /dev/null +++ b/db/packet_observation_count_integration_test.go @@ -0,0 +1,88 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "bytes" + "context" + "os" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/ingest" + "github.com/jackc/pgx/v5" +) + +func TestPacketObservationCountPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for PostgreSQL regression") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + applyBaseline(t, ctx, tx) + store := &Store{q: sqlc.New(tx)} + + hash := []byte{0xab, 0xcd} + heard := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + if _, err := tx.Exec(ctx, `INSERT INTO packets (packet_hash, payload_type, payload_version, route_type, raw_payload, raw_header, first_heard_at, last_heard_at) +VALUES ($1, 4, 0, 1, '\x00', '\x00', $2, $2)`, hash, heard); err != nil { + t.Fatal(err) + } + a, _, err := store.UpsertObserver(ctx, bytes.Repeat([]byte{1}, 32), "YVR") + if err != nil { + t.Fatal(err) + } + b, _, err := store.UpsertObserver(ctx, bytes.Repeat([]byte{2}, 32), "YYZ") + if err != nil { + t.Fatal(err) + } + + steps := []struct { + obs ingest.InsertObservationParams + wantInserted bool + wantCount int64 + }{ + {ingest.InsertObservationParams{PacketHash: hash, ObserverID: a, IATA: "YVR", HeardAt: heard}, true, 1}, + {ingest.InsertObservationParams{PacketHash: hash, ObserverID: b, IATA: "YYZ", HeardAt: heard}, true, 2}, + {ingest.InsertObservationParams{PacketHash: hash, ObserverID: a, IATA: "YVR", HeardAt: heard}, false, 2}, // broker duplicate + } + for i, s := range steps { + inserted, count, err := store.InsertObservation(ctx, s.obs) + if err != nil { + t.Fatal(err) + } + if inserted != s.wantInserted || count != s.wantCount { + t.Errorf("step %d: inserted=%v count=%d, want %v, %d", i, inserted, count, s.wantInserted, s.wantCount) + } + } + + var stored, raw int64 + if err := tx.QueryRow(ctx, `SELECT p.observation_count, (SELECT count(*) FROM packet_observations po WHERE po.packet_hash = p.packet_hash) +FROM packets p WHERE p.packet_hash = $1`, hash).Scan(&stored, &raw); err != nil { + t.Fatal(err) + } + if stored != raw || stored != 2 { + t.Errorf("packets.observation_count = %d, raw count = %d; want 2", stored, raw) + } + + page, err := store.ListPackets(ctx, nil, nil, nil, nil, time.Time{}, time.Time{}, 0, 10) + if err != nil { + t.Fatal(err) + } + if len(page.Items) != 1 || page.Items[0].ObservationCount != 2 { + t.Errorf("ListPackets observation counts = %+v, want one packet with 2", page.Items) + } +} diff --git a/db/packets.go b/db/packets.go index 2acc5d4..620acce 100644 --- a/db/packets.go +++ b/db/packets.go @@ -8,7 +8,6 @@ import ( "encoding/binary" "encoding/hex" "encoding/json" - "errors" "fmt" "log/slog" "time" @@ -17,7 +16,6 @@ import ( "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/MeshCore-Beacon/beacon-server/internal/ingest" "github.com/google/uuid" - "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" "github.com/meshcore-go/meshcore-go" ) @@ -514,7 +512,7 @@ func (s *Store) UpsertIATA(ctx context.Context, iata string) error { return s.q.UpsertIATA(ctx, iata) } -func (s *Store) InsertObservation(ctx context.Context, o ingest.InsertObservationParams) (bool, error) { +func (s *Store) InsertObservation(ctx context.Context, o ingest.InsertObservationParams) (bool, int64, error) { params := sqlc.InsertObservationParams{ PacketHash: o.PacketHash, ObserverID: o.ObserverID, @@ -536,13 +534,10 @@ func (s *Store) InsertObservation(ctx context.Context, o ingest.InsertObservatio AirtimeMs: o.AirtimeMs, } row, err := s.q.InsertObservation(ctx, params) - if errors.Is(err, pgx.ErrNoRows) { - return false, nil // conflict, not an error - } if err != nil { - return false, err + return false, 0, err } - return row.ID != 0, nil + return row.Inserted, row.ObservationCount, nil } func (s *Store) ListNodeObservations(ctx context.Context, nodeID uuid.UUID, cursor int64, limit int32) (api.Page[api.PacketObservationSummary], error) { @@ -584,10 +579,6 @@ func (s *Store) ListNodeObservations(ctx context.Context, nodeID uuid.UUID, curs }, nil } -func (s *Store) GetPacketObservationCount(ctx context.Context, packetHash []byte) (int64, error) { - return s.q.GetPacketObservationCount(ctx, packetHash) -} - // Small because each packet cascades to its observations. const packetDeleteBatch = 1000 diff --git a/db/packets_integration_test.go b/db/packets_integration_test.go index 10521e3..2d019ee 100644 --- a/db/packets_integration_test.go +++ b/db/packets_integration_test.go @@ -56,7 +56,8 @@ VALUES ('\x01', 4, 0, 1, '\x00', '\x00', NOW(), NOW())`) _, err = tx.Exec(ctx, ` INSERT INTO observers (id, public_key) VALUES ('00000000-0000-0000-0000-000000000001', '\x02'); INSERT INTO packet_observations (id, packet_hash, observer_id, iata, heard_at, path_length_byte, hash_size, hop_count, path_bytes) -VALUES (1, '\x01', '00000000-0000-0000-0000-000000000001', 'YVR', NOW(), 2, 1, 2, '\x1122');`) +VALUES (1, '\x01', '00000000-0000-0000-0000-000000000001', 'YVR', NOW(), 2, 1, 2, '\x1122'); +UPDATE packets SET observation_count = 1 WHERE packet_hash = '\x01';`) if err != nil { t.Fatal(err) } diff --git a/db/packets_test.go b/db/packets_test.go index d93aac5..c5f0627 100644 --- a/db/packets_test.go +++ b/db/packets_test.go @@ -12,7 +12,6 @@ import ( mockdb "github.com/MeshCore-Beacon/beacon-server/db/sqlc/mock" "github.com/MeshCore-Beacon/beacon-server/internal/ingest" "github.com/google/uuid" - "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" "go.uber.org/mock/gomock" ) @@ -218,10 +217,10 @@ func TestInsertObservation_Success(t *testing.T) { mock.EXPECT(). InsertObservation(gomock.Any(), gomock.Any()). - Return(sqlc.PacketObservation{ID: 1}, nil) + Return(sqlc.InsertObservationRow{Inserted: true, ObservationCount: 4}, nil) store := &Store{q: mock} - inserted, err := store.InsertObservation(context.Background(), ingest.InsertObservationParams{ + inserted, count, err := store.InsertObservation(context.Background(), ingest.InsertObservationParams{ PacketHash: []byte{0xde, 0xad}, ObserverID: observerID, IATA: "YVR", @@ -230,8 +229,8 @@ func TestInsertObservation_Success(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } - if !inserted { - t.Error("expected inserted true") + if !inserted || count != 4 { + t.Errorf("got inserted=%v count=%d, want true, 4", inserted, count) } } @@ -241,18 +240,18 @@ func TestInsertObservation_Conflict(t *testing.T) { mock.EXPECT(). InsertObservation(gomock.Any(), gomock.Any()). - Return(sqlc.PacketObservation{}, pgx.ErrNoRows) + Return(sqlc.InsertObservationRow{Inserted: false, ObservationCount: 2}, nil) store := &Store{q: mock} - inserted, err := store.InsertObservation(context.Background(), ingest.InsertObservationParams{ + inserted, count, err := store.InsertObservation(context.Background(), ingest.InsertObservationParams{ PacketHash: []byte{0xde, 0xad}, HeardAt: time.Now(), }) if err != nil { t.Fatalf("expected nil error on conflict, got %v", err) } - if inserted { - t.Error("expected inserted false on conflict") + if inserted || count != 2 { + t.Errorf("got inserted=%v count=%d, want false, 2", inserted, count) } } diff --git a/db/paths.go b/db/paths.go index 9df530d..19cffd8 100644 --- a/db/paths.go +++ b/db/paths.go @@ -72,5 +72,3 @@ func (s *Store) GetPathStats(ctx context.Context, since, until time.Time, iatas } return stats, nil } - -func (s *Store) RefreshPathStats(ctx context.Context) error { return s.q.RefreshPathStats(ctx) } diff --git a/db/paths_integration_test.go b/db/paths_integration_test.go index a49f8cc..656c48f 100644 --- a/db/paths_integration_test.go +++ b/db/paths_integration_test.go @@ -66,9 +66,7 @@ SELECT int4send(n),'00000000-0000-0000-0000-000000000001',heard_at,iata,payload_ if _, err := tx.Exec(ctx, "UPDATE packet_observations SET heard_at=to_timestamp(extract(epoch FROM $1::timestamptz)+extract(epoch FROM heard_at)-extract(epoch FROM '2026-01-01 00:00+00'::timestamptz))", since); err != nil { t.Fatal(err) } - if _, err := tx.Exec(ctx, "REFRESH MATERIALIZED VIEW mv_path_stats_hourly"); err != nil { - t.Fatal(err) - } + rollTxHours(t, ctx, tx) store := &Store{q: sqlc.New(tx)} until := since.Add(4 * time.Hour) for _, mode := range []string{"force_custom_plan", "force_generic_plan"} { @@ -115,7 +113,7 @@ SELECT int4send(n),'00000000-0000-0000-0000-000000000001',heard_at,iata,payload_ } } w := httptest.NewRecorder() - handlers.StatsRouter(store, nil).ServeHTTP(w, httptest.NewRequest("GET", fmt.Sprintf("/paths?since=%d&until=%d&iatas=YVR", since.UnixMilli()+123, until.UnixMilli()+123), nil).WithContext(ctx)) + handlers.StatsRouter(store, handlers.StatsOptions{}).ServeHTTP(w, httptest.NewRequest("GET", fmt.Sprintf("/paths?since=%d&until=%d&iatas=YVR", since.UnixMilli()+123, until.UnixMilli()+123), nil).WithContext(ctx)) var response api.PathStats if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil || w.Code != 200 || response.Receptions != 14 || response.Hashed != 5 { t.Fatalf("HTTP %d: %s (%v)", w.Code, w.Body.String(), err) @@ -135,9 +133,7 @@ SELECT int4send(1000+i),'00000000-0000-0000-0000-000000000001',$1::timestamptz+i if err != nil || stale.Receptions != 18 { t.Fatalf("snapshot lost: %+v %v", stale, err) } - if err := store.RefreshPathStats(ctx); err != nil { - t.Fatal(err) - } + rollTxHours(t, ctx, tx) got, err := store.GetPathStats(ctx, since, until, nil) if err != nil { t.Fatal(err) @@ -165,9 +161,7 @@ SELECT int4send(1000+i),'00000000-0000-0000-0000-000000000001',$1::timestamptz+i if _, err = tx.Exec(ctx, "TRUNCATE packet_observations"); err != nil { t.Fatal(err) } - if err := store.RefreshPathStats(ctx); err != nil { - t.Fatal(err) - } + rollTxHours(t, ctx, tx) got, err = store.GetPathStats(ctx, since, until, nil) if err != nil || got.Receptions != 0 || got.Hourly == nil || got.PathLengths == nil || len(got.HashWidths) != 3 { t.Fatalf("empty: %+v %v", got, err) diff --git a/db/presence.go b/db/presence.go index 4122266..9120d54 100644 --- a/db/presence.go +++ b/db/presence.go @@ -12,11 +12,13 @@ import ( "github.com/jackc/pgx/v5/pgtype" ) -func (s *Store) TouchObservers(ctx context.Context, ids []uuid.UUID, seen []time.Time, counts []int32) error { +func (s *Store) TouchObservers(ctx context.Context, ids []uuid.UUID, seen []time.Time, counts []int32, iatas []string, iataAts []time.Time) error { return s.q.TouchObservers(ctx, sqlc.TouchObserversParams{ Column1: ids, Column2: toTimestamptzs(seen), Column3: counts, + Column4: iatas, + Column5: nullableTimes(iataAts), }) } diff --git a/db/queries/paths.sql b/db/queries/paths.sql index 93703be..8cdaf51 100644 --- a/db/queries/paths.sql +++ b/db/queries/paths.sql @@ -1,11 +1,11 @@ -- name: GetPathStats :many --- All HTTP aggregates read the compact classification snapshot. +-- All HTTP aggregates read the hourly rollup, never observations. WITH readings AS ( - SELECT * FROM mv_path_stats_hourly + SELECT iata, hour, category, hash_bytes, entries, receptions FROM analytics_hourly_paths WHERE hour >= @since::timestamptz AND hour < @until::timestamptz AND COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 UNION ALL - SELECT * FROM mv_path_stats_hourly + SELECT iata, hour, category, hash_bytes, entries, receptions FROM analytics_hourly_paths WHERE hour >= @since::timestamptz AND hour < @until::timestamptz AND cardinality(@iatas::bpchar[]) > 0 AND iata = ANY(@iatas::bpchar[]) ) @@ -17,5 +17,3 @@ FROM readings GROUP BY GROUPING SETS ((category, hash_bytes, entries), (hour, category, hash_bytes)) ORDER BY is_hourly, hour, category, hash_bytes, entries; --- name: RefreshPathStats :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_path_stats_hourly; diff --git a/db/queries/queries.sql b/db/queries/queries.sql index 65e2ec3..349d621 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -70,32 +70,39 @@ SELECT id FROM transport_scopes WHERE name = $1; SELECT name FROM transport_scopes ORDER BY name; -- name: GetScopeByName :one +-- Packets and IATAs cover every retained rollup hour; IATAs are those that heard the +-- scope's own packets. Observer and node counts are current memberships. SELECT ts.name, - COUNT(DISTINCT p.packet_hash) AS packet_count, - COUNT(DISTINCT os.observer_id) AS observer_count, - COUNT(DISTINCT n.id) AS node_count, - COUNT(DISTINCT po.iata) AS iata_count, - array_remove(array_agg(DISTINCT po.iata ORDER BY po.iata), NULL)::text[] AS iatas + COALESCE((SELECT SUM(s.packets) FROM analytics_hourly_scope_sets s WHERE s.scope_id = ts.id), 0)::bigint AS packet_count, + (SELECT COUNT(*) FROM observer_scopes os WHERE os.scope_id = ts.id)::bigint AS observer_count, + (SELECT COUNT(*) FROM nodes n WHERE n.default_scope_id = ts.id)::bigint AS node_count, + cardinality(i.iatas)::bigint AS iata_count, + i.iatas FROM transport_scopes ts -LEFT JOIN packets p ON p.scope_id = ts.id -LEFT JOIN observer_scopes os ON os.scope_id = ts.id -LEFT JOIN observers o ON o.id = os.observer_id -LEFT JOIN packet_observations po ON po.observer_id = o.id -LEFT JOIN nodes n ON n.default_scope_id = ts.id -WHERE ts.name = $1 -GROUP BY ts.name; +CROSS JOIN LATERAL ( + SELECT COALESCE(array_agg(DISTINCT x.iata::text ORDER BY x.iata::text), '{}')::text[] AS iatas + FROM analytics_hourly_scope_sets s CROSS JOIN LATERAL unnest(s.iatas) AS x(iata) + WHERE s.scope_id = ts.id +) i +WHERE ts.name = $1; -- ============================================================ -- OBSERVERS -- ============================================================ -- name: UpsertObserver :one -INSERT INTO observers (public_key, observer_type, last_seen) -VALUES ($1, 'unknown', NOW()) +-- Empty iata (status/neighbors) leaves last_iata alone; otherwise the newest iata_at wins. +INSERT INTO observers (public_key, observer_type, last_seen, last_iata, last_iata_at) +VALUES ($1, 'unknown', NOW(), NULLIF(sqlc.arg(iata)::text, ''), + CASE WHEN sqlc.arg(iata)::text <> '' THEN sqlc.arg(iata_at)::timestamptz END) ON CONFLICT (public_key) DO UPDATE SET last_seen = NOW(), - observation_count = observers.observation_count + 1 + observation_count = observers.observation_count + 1, + last_iata = CASE WHEN EXCLUDED.last_iata IS NOT NULL + AND (observers.last_iata_at IS NULL OR EXCLUDED.last_iata_at > observers.last_iata_at) + THEN EXCLUDED.last_iata ELSE observers.last_iata END, + last_iata_at = GREATEST(observers.last_iata_at, EXCLUDED.last_iata_at) RETURNING *; -- name: UpdateObserverStatus :one @@ -123,11 +130,16 @@ RETURNING id; -- from regressing a newer write-through (e.g. a status update). UPDATE observers o SET last_seen = GREATEST(o.last_seen, v.seen), - observation_count = COALESCE(o.observation_count, 0) + v.delta + observation_count = COALESCE(o.observation_count, 0) + v.delta, + last_iata = CASE WHEN v.iata <> '' AND (o.last_iata_at IS NULL OR v.iata_at > o.last_iata_at) + THEN v.iata ELSE o.last_iata END, + last_iata_at = GREATEST(o.last_iata_at, v.iata_at) FROM ( SELECT unnest($1::uuid[]) AS id, unnest($2::timestamptz[]) AS seen, - unnest($3::int[]) AS delta + unnest($3::int[]) AS delta, + unnest($4::text[]) AS iata, + unnest($5::timestamptz[]) AS iata_at ) v WHERE o.id = v.id; @@ -171,23 +183,13 @@ COALESCE(CASE WHEN GREATEST(COALESCE(o.last_status_at, o.last_seen), o.last_seen) > NOW() - INTERVAL '5 minutes' THEN 'online' ELSE 'offline' END, 'offline')::text AS status, -COALESCE(( - SELECT po.iata - FROM packet_observations po - WHERE po.observer_id = o.id - ORDER BY po.heard_at DESC - LIMIT 1 -), '')::text AS iata +COALESCE(o.last_iata, '')::text AS iata FROM observers o LEFT JOIN observer_brokers ob ON ob.observer_id = o.id LEFT JOIN observer_scopes os ON os.observer_id = o.id LEFT JOIN transport_scopes ts ON ts.id = os.scope_id WHERE - (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR ( - SELECT po.iata FROM packet_observations po - WHERE po.observer_id = o.id - ORDER BY po.heard_at DESC LIMIT 1 - ) = ANY($1::bpchar[])) + (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR o.last_iata = ANY($1::bpchar[])) AND ($2 = '' OR o.observer_type = $2) AND ($3 = '' OR ob.broker_name = $3) AND ($4 = '' OR CASE @@ -206,10 +208,7 @@ ORDER BY o.last_seen DESC LIMIT $7; -- name: GetObserverLastIATA :one -SELECT iata FROM packet_observations -WHERE observer_id = $1 -ORDER BY heard_at DESC -LIMIT 1; +SELECT COALESCE(last_iata, '')::text FROM observers WHERE id = $1; -- name: GetObserverRadio :one SELECT radio_freq_mhz, radio_bw_khz, radio_sf, radio_cr @@ -265,7 +264,7 @@ ORDER BY count DESC; -- name: GetObserverActivityHourly :many -- Hour-or-coarser buckets summed from the hourly rollup; same COALESCE-plus-count shape as the raw query. SELECT - date_bin($3::interval, bucket, TIMESTAMPTZ 'epoch')::timestamptz AS bucket, + date_bin($3::interval, hour, TIMESTAMPTZ 'epoch')::timestamptz AS bucket, SUM(observations)::bigint AS observations, COALESCE(SUM(airtime_ms), 0)::real AS airtime_ms, SUM(airtime_n)::bigint AS airtime_n, @@ -274,15 +273,15 @@ SELECT COALESCE(MIN(snr_min), 0)::real AS snr_min, COALESCE(SUM(rssi_sum), 0)::bigint AS rssi_sum, SUM(rssi_n)::bigint AS rssi_n -FROM mv_observer_activity_hourly -WHERE observer_id = $1 AND bucket >= $2::timestamptz AND bucket < @until::timestamptz +FROM analytics_hourly_observer_activity +WHERE observer_id = $1 AND hour >= $2::timestamptz AND hour < @until::timestamptz GROUP BY 1 ORDER BY 1; -- name: GetObserverActivityHourlyPayloadTypes :many SELECT payload_type, SUM(observations)::bigint AS count -FROM mv_observer_activity_hourly -WHERE observer_id = $1 AND bucket >= $2::timestamptz AND bucket < @until::timestamptz +FROM analytics_hourly_observer_activity +WHERE observer_id = $1 AND hour >= $2::timestamptz AND hour < @until::timestamptz GROUP BY payload_type ORDER BY count DESC; @@ -361,6 +360,9 @@ WHERE ob.observer_id = v.observer_id AND ob.broker_name = v.broker_name; -- ============================================================ -- name: UpsertPacket :one +-- A new TRACE packet adds itself to its tag summary in the same statement, so the summary +-- can't miss a stored packet. 'TRACE' > 'PING'; the longest path keeps the best payload. +WITH up AS ( INSERT INTO packets ( packet_hash, payload_type, @@ -383,8 +385,25 @@ INSERT INTO packets ( ) ON CONFLICT (packet_hash) DO UPDATE SET last_heard_at = NOW() -RETURNING packet_hash, payload_type, payload_version, route_type, transport_codes_present, region_code, sub_region_code, origin_pubkey, raw_payload, raw_header, parsed_payload, decrypted, channel_hash, first_heard_at, last_heard_at, (xmax = 0) -AS inserted; +RETURNING packet_hash, payload_type, payload_version, route_type, transport_codes_present, region_code, sub_region_code, origin_pubkey, raw_payload, raw_header, parsed_payload, decrypted, channel_hash, first_heard_at, last_heard_at, trace_tag, scope_id, (xmax = 0) +AS inserted +), trace AS ( + INSERT INTO trace_tags (trace_tag, first_heard_at, last_heard_at, packet_count, trace_type, scope_id, best_payload, best_path_len) + SELECT u.trace_tag, u.first_heard_at, u.last_heard_at, 1, u.parsed_payload->>'type', u.scope_id, u.parsed_payload, + CASE WHEN jsonb_typeof(u.parsed_payload->'pathHashes') = 'array' + THEN jsonb_array_length(u.parsed_payload->'pathHashes') ELSE 0 END + FROM up u + WHERE u.inserted AND u.trace_tag IS NOT NULL + ON CONFLICT (trace_tag) DO UPDATE SET + packet_count = trace_tags.packet_count + 1, + trace_type = GREATEST(trace_tags.trace_type, EXCLUDED.trace_type), + scope_id = COALESCE(trace_tags.scope_id, EXCLUDED.scope_id), + best_payload = CASE WHEN EXCLUDED.best_path_len > trace_tags.best_path_len + THEN EXCLUDED.best_payload ELSE trace_tags.best_payload END, + best_path_len = GREATEST(trace_tags.best_path_len, EXCLUDED.best_path_len) +) +SELECT packet_hash, payload_type, payload_version, route_type, transport_codes_present, region_code, sub_region_code, origin_pubkey, raw_payload, raw_header, parsed_payload, decrypted, channel_hash, first_heard_at, last_heard_at, inserted +FROM up; -- name: SetPacketDecrypted :exec UPDATE packets SET decrypted = true WHERE packet_hash = $1; @@ -430,9 +449,6 @@ LEFT JOIN transport_scopes ts ON ts.id = p.scope_id WHERE p.trace_tag = decode($1, 'hex') ORDER BY p.first_heard_at ASC; --- name: GetPacketObservationCount :one -SELECT COUNT(*) FROM packet_observations WHERE packet_hash = $1; - -- name: ListPackets :many -- Returns packets with the latest observation rolled in for display. -- Pass cursor=0 to start from the beginning. IATA-filtered requests are @@ -457,7 +473,7 @@ SELECT p.last_heard_at, p.scope_id, ts.name AS scope_name, - (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = p.packet_hash) AS observation_count, + p.observation_count AS observation_count, -- sqlc loses LATERAL nullability; these scalar defaults are ignored when observer_id is NULL. po.observer_id AS latest_observer_id, o.display_name AS latest_observer_name, @@ -568,7 +584,7 @@ SELECT sh.site_heard_at, sat.scan_saturated, sat.scan_floor, - (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = p.packet_hash) AS observation_count, + p.observation_count AS observation_count, -- sqlc loses LATERAL nullability; these scalar defaults are ignored when observer_id is NULL. po.observer_id AS latest_observer_id, o.display_name AS latest_observer_name, @@ -614,7 +630,7 @@ SELECT p.route_type, p.first_heard_at, p.last_heard_at, - (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = p.packet_hash) AS observation_count, + p.observation_count AS observation_count, po.observer_id AS latest_observer_id, o.display_name AS latest_observer_name, po.iata AS latest_observer_iata, @@ -637,10 +653,24 @@ WHERE po.id > $1 ORDER BY po.id ASC LIMIT $6; - -- name: DeleteOldPackets :one --- Locks, archives and cascades one bounded packet cohort in a single transaction. -SELECT archive_delete_packets(@cutoff::timestamptz, @batch_size::integer)::bigint; +-- Deletes one bounded cohort (observations cascade). SKIP LOCKED leaves packets an in-flight +-- observation insert holds. raw_deleted_before tells the rollup which hours lost raw rows. +WITH victims AS MATERIALIZED ( + SELECT vp.packet_hash, vp.last_heard_at FROM packets vp + WHERE vp.last_heard_at < @cutoff::timestamptz + ORDER BY vp.last_heard_at, vp.packet_hash + LIMIT @batch_size::integer + FOR UPDATE OF vp SKIP LOCKED +), deleted AS ( + DELETE FROM packets dp USING victims v WHERE dp.packet_hash = v.packet_hash + RETURNING dp.last_heard_at +), mark AS ( + UPDATE analytics_raw_state + SET raw_deleted_before = GREATEST(raw_deleted_before, (SELECT max(last_heard_at) FROM deleted)) + WHERE EXISTS (SELECT 1 FROM deleted) +) +SELECT count(*) FROM deleted; -- name: DeleteOldNodes :exec -- Deletes nodes not seen since the given cutoff. node_iatas and node_neighbors cascade- @@ -682,6 +712,8 @@ DELETE FROM trace_iatas WHERE last_heard < $1; -- ============================================================ -- name: InsertObservation :one +-- Bumps packets.observation_count only for a new row; a duplicate returns the current count. +WITH ins AS ( INSERT INTO packet_observations ( packet_hash, observer_id, @@ -705,7 +737,15 @@ INSERT INTO packet_observations ( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18 ) ON CONFLICT (packet_hash, observer_id) DO NOTHING -RETURNING *; +RETURNING packet_hash +), bump AS ( + UPDATE packets p SET observation_count = p.observation_count + 1 + FROM ins WHERE p.packet_hash = ins.packet_hash + RETURNING p.observation_count +) +SELECT EXISTS (SELECT 1 FROM ins) AS inserted, + COALESCE((SELECT b.observation_count FROM bump b), + (SELECT pk.observation_count FROM packets pk WHERE pk.packet_hash = $1), 0)::bigint AS observation_count; -- name: ListObservationsForPacket :many SELECT po.*, o.display_name AS observer_name @@ -849,8 +889,7 @@ INSERT INTO channels (channel_hash, key_fingerprint, name, hashtag, is_hashtag, VALUES ($1, $2::bytea, $3, $4, $5, ($2 IS NOT NULL), NOW()) ON CONFLICT (channel_hash, key_fingerprint) DO UPDATE SET last_seen = NOW(), - name = COALESCE(EXCLUDED.name, channels.name), - message_count = CASE WHEN $6 THEN channels.message_count + 1 ELSE channels.message_count END + name = COALESCE(EXCLUDED.name, channels.name) RETURNING *; -- name: UpsertChannelHashOnly :one @@ -889,13 +928,28 @@ ON CONFLICT (channel_hash, iata) DO UPDATE SET last_heard = EXCLUDED.last_heard WHERE EXCLUDED.last_heard > channel_iatas.last_heard + INTERVAL '1 hour'; --- name: UpsertTraceIATA :exec --- Refreshes at most hourly so repeat hears don't churn the row. -INSERT INTO trace_iatas (trace_tag, iata, last_heard) -VALUES ($1, $2, $3) -ON CONFLICT (trace_tag, iata) DO UPDATE SET - last_heard = EXCLUDED.last_heard -WHERE EXCLUDED.last_heard > trace_iatas.last_heard + INTERVAL '1 hour'; +-- name: RecordTrace :exec +-- One hearing of a TRACE packet: trace_iatas refreshes at most hourly and the tag's heard +-- window widens (the first hearing replaces the packet's provisional times). Packet counts +-- and payloads come from UpsertPacket. +WITH iata AS ( + INSERT INTO trace_iatas (trace_tag, iata, last_heard) + VALUES (@trace_tag, @iata, @heard_at) + ON CONFLICT (trace_tag, iata) DO UPDATE SET + last_heard = EXCLUDED.last_heard + WHERE EXCLUDED.last_heard > trace_iatas.last_heard + INTERVAL '1 hour' +) +INSERT INTO trace_tags (trace_tag, first_heard_at, last_heard_at, heard) +VALUES (@trace_tag, @heard_at, @heard_at, true) +ON CONFLICT (trace_tag) DO UPDATE SET + first_heard_at = CASE WHEN trace_tags.heard THEN LEAST(trace_tags.first_heard_at, EXCLUDED.first_heard_at) + ELSE EXCLUDED.first_heard_at END, + last_heard_at = CASE WHEN trace_tags.heard THEN GREATEST(trace_tags.last_heard_at, EXCLUDED.last_heard_at) + ELSE EXCLUDED.last_heard_at END, + heard = true; + +-- name: DeleteOldTraceTags :exec +DELETE FROM trace_tags WHERE last_heard_at < $1; -- name: ListChannels :many -- Channels ordered by last seen, optionally filtered by hash and/or IATAs. @@ -936,7 +990,6 @@ LIMIT @page_limit; -- name: GetChannelByID :one SELECT * FROM channels WHERE id = $1; - -- ============================================================ -- CHANNEL MESSAGES -- ============================================================ @@ -944,11 +997,22 @@ SELECT * FROM channels WHERE id = $1; -- name: InsertChannelMessage :one -- Read the immutable first-packet scope in the same statement as insertion. -- A later reception's transport code must not give live and historical messages different tags. +-- message_count is a lifetime count, bumped only for a new message. A historical (backfilled) +-- message queues its observation hours for a re-roll in the same statement, so the two can't diverge. WITH inserted AS ( INSERT INTO channel_messages (channel_id, packet_hash, sender_name, content, sent_at) VALUES ($1, $2, $3, $4, $5) ON CONFLICT (packet_hash) DO NOTHING - RETURNING id, packet_hash + RETURNING id, packet_hash, channel_id +), bump AS ( + UPDATE channels c SET message_count = c.message_count + 1 + FROM inserted WHERE c.id = inserted.channel_id +), dirty AS ( + INSERT INTO analytics_dirty_hours (hour) + SELECT DISTINCT date_trunc('hour', po.heard_at, 'UTC') + FROM packet_observations po JOIN inserted i ON i.packet_hash = po.packet_hash + WHERE $6::boolean + ON CONFLICT (hour) DO UPDATE SET enqueued_at = now() ) SELECT inserted.id, ts.name AS scope_name, p.transport_codes_present FROM inserted @@ -961,7 +1025,7 @@ LEFT JOIN transport_scopes ts ON ts.id = p.scope_id; -- Pass empty string for iata to skip IATA filtering. -- Pass cursor=0 to start from the beginning. SELECT DISTINCT ON (cm.id) cm.*, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, ts.name AS scope_name, p.transport_codes_present, -(SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = cm.packet_hash) AS observation_count +p.observation_count AS observation_count FROM channel_messages cm JOIN channels c ON c.id = cm.channel_id JOIN packet_observations po ON po.packet_hash = cm.packet_hash @@ -980,7 +1044,7 @@ LIMIT $6; -- Pass empty string for iata or scope to skip those filters. -- Pass cursor=0 to start from the beginning. SELECT DISTINCT ON (cm.id) cm.*, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, ts.name AS scope_name, p.transport_codes_present, -(SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = cm.packet_hash) AS observation_count +p.observation_count AS observation_count FROM channel_messages cm JOIN channels c ON c.id = cm.channel_id JOIN packet_observations po ON po.packet_hash = cm.packet_hash @@ -999,7 +1063,7 @@ LIMIT $5; -- Pass empty string for iata or scope to skip those filters. -- Pass cursor=0 to start from the beginning. SELECT DISTINCT ON (cm.id) cm.*, c.channel_hash, ts.name AS scope_name, p.transport_codes_present, - (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = cm.packet_hash) AS observation_count + p.observation_count AS observation_count FROM channel_messages cm JOIN channels c ON c.id = cm.channel_id JOIN packet_observations po ON po.packet_hash = cm.packet_hash @@ -1017,7 +1081,7 @@ LIMIT $6; -- Returns messages after the given message ID, ordered oldest first. -- Used for WS reconnect backfill. SELECT DISTINCT ON (cm.id) cm.*, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, ts.name AS scope_name, p.transport_codes_present, -(SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = cm.packet_hash) AS observation_count +p.observation_count AS observation_count FROM channel_messages cm JOIN channels c ON c.id = cm.channel_id JOIN packet_observations po ON po.packet_hash = cm.packet_hash @@ -1033,38 +1097,41 @@ LIMIT $4; -- STATS -- ============================================================ --- name: GetStatsOverview :one -SELECT - COUNT(DISTINCT po.packet_hash) AS total_packets, - COUNT(*) AS total_observations, - COUNT(DISTINCT po.observer_id) AS active_observers, - COUNT(DISTINCT po.iata) AS active_iatas -FROM packet_observations po -WHERE po.heard_at > NOW() - INTERVAL '24 hours' - AND (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR po.iata = ANY($1::bpchar[])); - -- name: GetHourlyStats :many -SELECT iata, hour, observation_count, unique_packets, active_observers -FROM mv_hourly_iata_stats -WHERE (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR iata = ANY($1::bpchar[])) - AND hour >= NOW() - $2::interval +SELECT iata, hour, observation_count +FROM analytics_hourly_iata_observations +WHERE (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 OR iata = ANY(@iatas::bpchar[])) + AND hour >= @since::timestamptz ORDER BY iata, hour; -- name: GetTopNodes :many -SELECT * FROM mv_top_nodes_by_iata -WHERE (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR iata = ANY($1::bpchar[])) -ORDER BY observation_count DESC -LIMIT $2; +-- Nodes by ADVERT hearings since the given hour. Live names win over the rolled snapshot; +-- node_id is NULL once the node row is gone. iata is a representative one. +WITH ranked AS ( + SELECT origin_pubkey, SUM(observations)::bigint AS observation_count, MAX(last_heard) AS last_heard, + MAX(iata) AS iata, MAX(name) AS name, MAX(node_type) AS node_type + FROM analytics_hourly_advert_hearings + WHERE hour >= @since::timestamptz + AND (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 OR iata = ANY(@iatas::bpchar[])) + GROUP BY origin_pubkey + ORDER BY observation_count DESC, origin_pubkey + LIMIT @row_limit +) +SELECT n.id AS node_id, encode(r.origin_pubkey, 'hex') AS public_key, + COALESCE(n.name, r.name, '')::text AS name, COALESCE(n.node_type, r.node_type, 0)::smallint AS node_type, + r.iata::bpchar AS iata, r.observation_count, r.last_heard::timestamptz AS last_heard +FROM ranked r +LEFT JOIN nodes n ON n.public_key = r.origin_pubkey +ORDER BY r.observation_count DESC, r.origin_pubkey; -- name: GetStatsPayloadBreakdown :many --- Payload-type counts for the IATA within the window, summed from the --- precomputed hourly buckets. +-- Payload-type observation counts since the given hour, from the hourly rollup. SELECT payload_type, SUM(count)::bigint AS count -FROM mv_payload_breakdown_by_iata -WHERE (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR iata = ANY($1::bpchar[])) - AND bucket >= NOW() - $2::interval +FROM analytics_hourly_payload_breakdown +WHERE (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 OR iata = ANY(@iatas::bpchar[])) + AND hour >= @since::timestamptz GROUP BY payload_type ORDER BY count DESC; @@ -1080,38 +1147,53 @@ GROUP BY n.node_type ORDER BY count DESC; -- name: GetStatsTopObservers :many --- Top N observers for the IATA within the window, summed from the precomputed --- hourly buckets. Counts sum across matched IATAs; iata is a representative one. -SELECT - observer_id AS id, - display_name, - observer_type, - COALESCE(SUM(observation_count), 0)::bigint AS observation_count, - COALESCE(MAX(iata), '')::bpchar AS iata -FROM mv_top_observers_by_iata -WHERE bucket >= NOW() - $1::interval - AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR iata = ANY($2::bpchar[])) -GROUP BY observer_id, display_name, observer_type -ORDER BY observation_count DESC -LIMIT $3; +-- Top N observers since the given hour. Counts sum across matched IATAs; iata is a +-- representative one. Live names win over the rolled snapshot. +WITH ranked AS ( + SELECT observer_id, SUM(observation_count)::bigint AS observation_count, MAX(iata) AS iata, + MAX(display_name) AS display_name, MAX(observer_type) AS observer_type + FROM analytics_hourly_observer_identity + WHERE hour >= @since::timestamptz + AND (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 OR iata = ANY(@iatas::bpchar[])) + GROUP BY observer_id + ORDER BY observation_count DESC, observer_id + LIMIT @row_limit +) +SELECT r.observer_id AS id, COALESCE(o.display_name, r.display_name, '')::text AS display_name, + COALESCE(o.observer_type, r.observer_type, '')::text AS observer_type, + r.observation_count, r.iata::bpchar AS iata +FROM ranked r +LEFT JOIN observers o ON o.id = r.observer_id +ORDER BY r.observation_count DESC, r.observer_id; -- name: GetStatsTopAdvertisers :many --- Top N advertisers in the window, summed from the hourly buckets. -SELECT - node_id AS id, - name, - node_type, - COALESCE(SUM(advert_count), 0)::bigint AS advert_count, - COALESCE(SUM(flood_advert_count), 0)::bigint AS flood_advert_count, - COALESCE(SUM(direct_advert_count), 0)::bigint AS direct_advert_count, - MAX(last_heard)::timestamptz AS last_heard, - COALESCE(MAX(iata), '')::bpchar AS iata -FROM mv_top_advertisers_by_iata -WHERE bucket >= NOW() - $1::interval - AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR iata = ANY($2::bpchar[])) -GROUP BY node_id, name, node_type -ORDER BY advert_count DESC -LIMIT $3; +-- Top N advertisers since the given hour. Each ADVERT packet counts once per hour heard, +-- however many of the requested IATAs heard it (IATA-set rollup). Live names win. +WITH ranked AS ( + SELECT origin_pubkey, SUM(advert_packets)::bigint AS advert_count, + SUM(flood_packets)::bigint AS flood_advert_count, SUM(direct_packets)::bigint AS direct_advert_count + FROM analytics_hourly_advert_sets + WHERE hour >= @since::timestamptz + AND (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 OR iatas && @iatas::bpchar[]) + GROUP BY origin_pubkey + ORDER BY advert_count DESC, origin_pubkey + LIMIT @row_limit +), heard AS ( + SELECT h.origin_pubkey, MAX(h.last_heard) AS last_heard, MAX(h.iata) AS iata, + MAX(h.name) AS name, MAX(h.node_type) AS node_type + FROM analytics_hourly_advert_hearings h + WHERE h.hour >= @since::timestamptz AND h.origin_pubkey IN (SELECT origin_pubkey FROM ranked) + AND (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 OR h.iata = ANY(@iatas::bpchar[])) + GROUP BY h.origin_pubkey +) +SELECT n.id AS node_id, encode(r.origin_pubkey, 'hex') AS public_key, + COALESCE(n.name, h.name, '')::text AS name, COALESCE(n.node_type, h.node_type, 0)::smallint AS node_type, + r.advert_count, r.flood_advert_count, r.direct_advert_count, + h.last_heard::timestamptz AS last_heard, COALESCE(h.iata, '')::bpchar AS iata +FROM ranked r +LEFT JOIN heard h ON h.origin_pubkey = r.origin_pubkey +LEFT JOIN nodes n ON n.public_key = r.origin_pubkey +ORDER BY r.advert_count DESC, r.origin_pubkey; -- name: GetStatsClockDrift :many -- Repeaters/room servers (node_type 2/3) whose current advert-derived clock drift exceeds @@ -1140,17 +1222,18 @@ FROM page n ORDER BY ABS(n.device_clock_drift_seconds) DESC; -- name: GetStatsTopTalkers :many --- Top N talkers (by decrypted sender_name) in the window, summed from the hourly buckets. +-- Top N senders since the given hour. Each message counts once per hour heard, however many +-- of the requested IATAs heard it (IATA-set rollup). SELECT sender_name, - COALESCE(SUM(message_count), 0)::bigint AS message_count, + SUM(messages)::bigint AS message_count, MAX(last_sent)::timestamptz AS last_sent -FROM mv_top_talkers_by_iata -WHERE bucket >= NOW() - $1::interval - AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR iata = ANY($2::bpchar[])) +FROM analytics_hourly_talker_sets +WHERE hour >= @since::timestamptz + AND (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 OR iatas && @iatas::bpchar[]) GROUP BY sender_name -ORDER BY message_count DESC -LIMIT $3; +ORDER BY message_count DESC, sender_name +LIMIT @row_limit; -- name: GetRadioPresets :many SELECT preset, iata, source_type, count @@ -1160,37 +1243,35 @@ WHERE ($1::text = '' OR preset = $1::text) ORDER BY preset, iata, source_type; -- name: GetScopeStats :many --- Aggregate matching observations once, separately from node memberships to avoid --- a cross-join. Empty IATAs keep the original global counts, including associations --- whose observations have expired; the filtered aggregates are empty in that case. -WITH observation_counts AS ( - SELECT p.scope_id, - COUNT(DISTINCT p.packet_hash) AS packet_count, - COUNT(DISTINCT os.observer_id) AS observer_count - FROM packet_observations po - JOIN packets p ON p.packet_hash = po.packet_hash - LEFT JOIN observer_scopes os ON os.scope_id = p.scope_id AND os.observer_id = po.observer_id - WHERE po.iata = ANY(sqlc.arg(iatas)::bpchar[]) AND p.scope_id IS NOT NULL - GROUP BY p.scope_id +-- Packets since the given hour come from the IATA-set rollup (counted once per hour heard). +-- Observer and node counts are current memberships; observers filter by their latest IATA. +WITH packet_counts AS ( + SELECT s.scope_id, SUM(s.packets)::bigint AS packet_count + FROM analytics_hourly_scope_sets s + WHERE s.hour >= @since::timestamptz + AND (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 OR s.iatas && @iatas::bpchar[]) + GROUP BY s.scope_id +), observer_counts AS ( + SELECT os.scope_id, COUNT(*)::bigint AS observer_count + FROM observer_scopes os JOIN observers o ON o.id = os.observer_id + WHERE COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 OR o.last_iata = ANY(@iatas::bpchar[]) + GROUP BY os.scope_id ), node_counts AS ( - SELECT n.default_scope_id AS scope_id, COUNT(*) AS node_count + SELECT n.default_scope_id AS scope_id, COUNT(*)::bigint AS node_count FROM nodes n - WHERE n.id IN (SELECT node_id FROM node_iatas WHERE iata = ANY(sqlc.arg(iatas)::bpchar[])) + WHERE n.default_scope_id IS NOT NULL + AND (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 + OR n.id IN (SELECT node_id FROM node_iatas WHERE iata = ANY(@iatas::bpchar[]))) GROUP BY n.default_scope_id ) SELECT ts.name, - CASE WHEN COALESCE(cardinality(sqlc.arg(iatas)::bpchar[]), 0) = 0 - THEN (SELECT COUNT(*) FROM packets p WHERE p.scope_id = ts.id) - ELSE COALESCE(oc.packet_count, 0) END::bigint AS packet_count, - CASE WHEN COALESCE(cardinality(sqlc.arg(iatas)::bpchar[]), 0) = 0 - THEN (SELECT COUNT(*) FROM observer_scopes os WHERE os.scope_id = ts.id) - ELSE COALESCE(oc.observer_count, 0) END::bigint AS observer_count, - CASE WHEN COALESCE(cardinality(sqlc.arg(iatas)::bpchar[]), 0) = 0 - THEN (SELECT COUNT(*) FROM nodes n WHERE n.default_scope_id = ts.id) - ELSE COALESCE(nc.node_count, 0) END::bigint AS node_count + COALESCE(pc.packet_count, 0)::bigint AS packet_count, + COALESCE(oc.observer_count, 0)::bigint AS observer_count, + COALESCE(nc.node_count, 0)::bigint AS node_count FROM transport_scopes ts -LEFT JOIN observation_counts oc ON oc.scope_id = ts.id +LEFT JOIN packet_counts pc ON pc.scope_id = ts.id +LEFT JOIN observer_counts oc ON oc.scope_id = ts.id LEFT JOIN node_counts nc ON nc.scope_id = ts.id ORDER BY ts.name; @@ -1270,46 +1351,28 @@ DELETE FROM regions WHERE imported AND NOT (slug = ANY(@keep::text[])) RETURNING -- ============================================================ -- name: ListTraceTags :many --- Returns distinct trace tags with summary info, ordered by most recent first. --- IATA membership comes from trace_iatas (joining observations here spilled the --- hash join). Per-tag details filled in only for the returned page. -WITH tags AS ( - SELECT - p.trace_tag, - MIN(p.first_heard_at) AS first_heard_at, - MAX(p.last_heard_at) AS last_heard_at, - COUNT(*) AS packet_count, - MAX(p.parsed_payload->>'type') AS trace_type - FROM packets p - WHERE p.trace_tag IS NOT NULL - AND (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR p.trace_tag IN ( - SELECT ti.trace_tag FROM trace_iatas ti WHERE ti.iata = ANY($1::bpchar[]))) - AND ($2::text = '' OR p.scope_id = (SELECT id FROM transport_scopes WHERE name = $2)) - AND ($3::timestamptz IS NULL OR p.first_heard_at >= $3) - AND ($4::timestamptz IS NULL OR p.first_heard_at <= $4) - AND ($7::text = '' OR p.parsed_payload->>'type' = $7) - GROUP BY p.trace_tag - HAVING ($5::timestamptz IS NULL OR MAX(p.last_heard_at) < $5) - ORDER BY MAX(p.last_heard_at) DESC - LIMIT $6 -) +-- Tags, most recent first. Filters match the tag summary (see RecordTrace). SELECT encode(t.trace_tag, 'hex') AS trace_tag, - t.first_heard_at::timestamptz AS first_heard_at, - t.last_heard_at::timestamptz AS last_heard_at, + t.first_heard_at, + t.last_heard_at, t.packet_count, (SELECT COUNT(*) FROM trace_iatas ti WHERE ti.trace_tag = t.trace_tag AND (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR ti.iata = ANY($1::bpchar[]))) AS iata_count, - t.trace_type::text AS trace_type, - (SELECT p3.parsed_payload - FROM packets p3 - WHERE p3.trace_tag = t.trace_tag - ORDER BY jsonb_array_length(p3.parsed_payload->'pathHashes') DESC - LIMIT 1) AS best_payload -FROM tags t -ORDER BY t.last_heard_at DESC; + COALESCE(t.trace_type, '')::text AS trace_type, + t.best_payload +FROM trace_tags t +WHERE (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR EXISTS ( + SELECT 1 FROM trace_iatas ti WHERE ti.trace_tag = t.trace_tag AND ti.iata = ANY($1::bpchar[]))) + AND ($2::text = '' OR t.scope_id = (SELECT id FROM transport_scopes WHERE name = $2)) + AND ($3::timestamptz IS NULL OR t.first_heard_at >= $3) + AND ($4::timestamptz IS NULL OR t.first_heard_at <= $4) + AND ($5::timestamptz IS NULL OR t.last_heard_at < $5) + AND ($7::text = '' OR t.trace_type = $7) +ORDER BY t.last_heard_at DESC +LIMIT $6; -- ============================================================ -- ROUTES @@ -1487,30 +1550,9 @@ WHERE ns.iata = $1 AND n.node_type IN (2, 3) AND ns.prefix_4 = ANY($2::bytea[]); --- name: RefreshHourlyStats :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_hourly_iata_stats; - --- name: RefreshTopNodes :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_top_nodes_by_iata; - --- name: RefreshTopObservers :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_top_observers_by_iata; - --- name: RefreshPayloadBreakdown :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_payload_breakdown_by_iata; - --- name: RefreshTopTalkers :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_top_talkers_by_iata; - --- name: RefreshTopAdvertisers :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_top_advertisers_by_iata; - -- name: RefreshRadioPresets :exec REFRESH MATERIALIZED VIEW CONCURRENTLY mv_radio_presets; --- name: RefreshObserverActivity :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_observer_activity_hourly; - -- name: AmbiguousPrefixes :many -- Hop prefixes that match >1 node in an IATA, per width. Computed once per reconfirm run. SELECT iata::text AS iata, 1::int AS len, prefix_1 AS prefix FROM node_short_ids GROUP BY iata, prefix_1 HAVING COUNT(*) > 1 @@ -1631,7 +1673,6 @@ WITH latest AS ( SELECT (SELECT heard_at FROM latest)::timestamptz AS latest_recorded_at, hourly.n AS last_complete_hour FROM hourly; - -- name: GetScopeCatalogue :one SELECT * FROM meshmapper_scope_catalogues WHERE iata = $1 AND url = $2; diff --git a/db/queries/rollup.sql b/db/queries/rollup.sql new file mode 100644 index 0000000..5b33676 --- /dev/null +++ b/db/queries/rollup.sql @@ -0,0 +1,279 @@ +-- ============================================================ +-- HOURLY ANALYTICS ROLLUPS +-- An hour [H, H+1h) is eligible once now() >= H + 95 min (ingest clamps heard_at to ±30 min). +-- ============================================================ + +-- name: TryRollupLock :one +SELECT pg_try_advisory_lock(25189, 1); + +-- name: RollupUnlock :exec +SELECT pg_advisory_unlock(25189, 1); + +-- name: RegisterRollupHours :exec +-- Adds a 'missing' row for every eligible hour since the earliest retained observation +-- (and not before @since), zero-observation hours included. +INSERT INTO analytics_rollup_hours (hour, status) +SELECT h, 'missing' +-- No observations yet means nothing to register (GREATEST would ignore the NULL MIN). +FROM (SELECT CASE WHEN MIN(heard_at) IS NOT NULL + THEN date_trunc('hour', GREATEST(MIN(heard_at), @since::timestamptz), 'UTC') END AS first_hour + FROM packet_observations) f, + generate_series(f.first_hour, date_trunc('hour', now() - INTERVAL '95 minutes', 'UTC'), INTERVAL '1 hour') h +WHERE f.first_hour IS NOT NULL +ON CONFLICT (hour) DO NOTHING; + +-- name: MarkPartialRollupHours :exec +-- Hours whose raw rows cleanup already started deleting can never be rolled completely. +-- Readers report the new status, so the revision moves too. +WITH marked AS ( + UPDATE analytics_rollup_hours h SET status = 'partial' + FROM analytics_raw_state r + WHERE h.status = 'missing' + AND r.raw_deleted_before IS NOT NULL + AND h.hour <= r.raw_deleted_before + INTERVAL '30 minutes' + RETURNING 1 +) +UPDATE analytics_state SET revision = revision + 1 WHERE EXISTS (SELECT 1 FROM marked); + +-- name: ListMissingRollupHours :many +SELECT hour FROM analytics_rollup_hours +WHERE status = 'missing' +ORDER BY hour +LIMIT $1; + +-- name: ListDirtyRollupHours :many +-- Only complete hours whose raw rows are still intact can be re-rolled. Queued missing hours +-- wait: their first roll consumes the entry only if the entry predates its snapshot. +SELECT d.hour FROM analytics_dirty_hours d +JOIN analytics_rollup_hours h ON h.hour = d.hour AND h.status = 'complete' +CROSS JOIN analytics_raw_state r +WHERE r.raw_deleted_before IS NULL OR d.hour > r.raw_deleted_before + INTERVAL '30 minutes' +ORDER BY d.hour +LIMIT $1; + +-- name: DeleteStaleDirtyHours :exec +-- Dirty hours that can no longer be re-rolled. +DELETE FROM analytics_dirty_hours d +WHERE EXISTS (SELECT 1 FROM analytics_rollup_hours h WHERE h.hour = d.hour AND h.status = 'partial') + OR d.hour <= (SELECT raw_deleted_before + INTERVAL '30 minutes' FROM analytics_raw_state); + +-- name: LockRollupHour :one +-- Returns the hour's status, or 'partial' if raw deletion has reached it since registration. +SELECT CASE + WHEN r.raw_deleted_before IS NOT NULL AND h.hour <= r.raw_deleted_before + INTERVAL '30 minutes' + THEN 'partial' + ELSE h.status + END::text AS status +FROM analytics_rollup_hours h CROSS JOIN analytics_raw_state r +WHERE h.hour = $1 +FOR UPDATE OF h; + +-- name: SetRollupHourPartial :exec +WITH marked AS ( + UPDATE analytics_rollup_hours SET status = 'partial' WHERE hour = $1 AND status = 'missing' RETURNING 1 +) +UPDATE analytics_state SET revision = revision + 1 WHERE EXISTS (SELECT 1 FROM marked); + +-- name: FillRollupObs :exec +INSERT INTO rollup_obs (packet_hash, observer_id, iata, heard_at, path_length_byte, hash_size, hop_count, + path_len, rssi, snr, airtime_ms, payload_type, route_type, origin_pubkey, scope_id) +SELECT po.packet_hash, po.observer_id, po.iata, po.heard_at, po.path_length_byte, po.hash_size, po.hop_count, + COALESCE(octet_length(po.path_bytes), 0), po.rssi, po.snr, po.airtime_ms, po.payload_type, + p.route_type, p.origin_pubkey, p.scope_id +FROM packet_observations po +JOIN packets p ON p.packet_hash = po.packet_hash +WHERE po.heard_at >= $1::timestamptz AND po.heard_at < $1::timestamptz + INTERVAL '1 hour'; + +-- name: DeleteRollupHour :exec +WITH a AS (DELETE FROM analytics_hourly_iata_observations ta WHERE ta.hour = @hour::timestamptz), + b AS (DELETE FROM analytics_hourly_payload_breakdown tb WHERE tb.hour = @hour::timestamptz), + c AS (DELETE FROM analytics_hourly_signal tc WHERE tc.hour = @hour::timestamptz), + d AS (DELETE FROM analytics_hourly_paths td WHERE td.hour = @hour::timestamptz), + e AS (DELETE FROM analytics_hourly_observer_activity te WHERE te.hour = @hour::timestamptz), + f AS (DELETE FROM analytics_hourly_observer_identity tf WHERE tf.hour = @hour::timestamptz), + g AS (DELETE FROM analytics_hourly_advert_hearings tg WHERE tg.hour = @hour::timestamptz), + h AS (DELETE FROM analytics_hourly_advert_sets th WHERE th.hour = @hour::timestamptz), + i AS (DELETE FROM analytics_hourly_talker_sets ti WHERE ti.hour = @hour::timestamptz), + j AS (DELETE FROM analytics_hourly_packet_sets tj WHERE tj.hour = @hour::timestamptz) +DELETE FROM analytics_hourly_scope_sets tk WHERE tk.hour = @hour::timestamptz; + +-- name: RollIATAObservations :exec +INSERT INTO analytics_hourly_iata_observations (hour, iata, observation_count) +SELECT @hour::timestamptz, iata, count(*) FROM rollup_obs GROUP BY iata; + +-- name: RollPayloadBreakdown :exec +INSERT INTO analytics_hourly_payload_breakdown (hour, iata, payload_type, count) +SELECT @hour::timestamptz, iata, payload_type, count(*) +FROM rollup_obs WHERE payload_type IS NOT NULL +GROUP BY iata, payload_type; + +-- name: RollSignal :exec +-- kind = grouping(snr_bin, rssi_bin): 3 = per IATA totals, 1 = per SNR bin, 2 = per RSSI bin. +-- rssi=0 AND snr=0 means "not reported". +INSERT INTO analytics_hourly_signal (hour, iata, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum) +SELECT @hour::timestamptz, b.iata, grouping(b.snr_bin, b.rssi_bin)::integer, + COALESCE(b.snr_bin, -1)::integer, COALESCE(b.rssi_bin, -1)::integer, + count(*), count(b.snr), COALESCE(sum(b.snr), 0)::double precision, + count(b.rssi), COALESCE(sum(b.rssi), 0)::double precision +FROM ( + SELECT s.iata, s.snr, s.rssi, + width_bucket(s.snr, -30, 30, 12) AS snr_bin, + width_bucket(s.rssi, -140, 0, 14) AS rssi_bin + FROM ( + SELECT ro.iata, + CASE WHEN NOT (COALESCE(ro.rssi, 0) = 0 AND COALESCE(ro.snr, 0) = 0) + AND ro.snr > '-Infinity'::real AND ro.snr < 'Infinity'::real + THEN ro.snr::double precision END AS snr, + CASE WHEN NOT (COALESCE(ro.rssi, 0) = 0 AND COALESCE(ro.snr, 0) = 0) + THEN ro.rssi::double precision END AS rssi + FROM rollup_obs ro + ) s +) b +GROUP BY GROUPING SETS ((b.iata), (b.iata, b.snr_bin), (b.iata, b.rssi_bin)); + +-- name: RollPaths :exec +-- category: 0 = routed, 1 = zero-hop, 2 = TRACE, 3 = invalid (meshcore-go IsValidPathLen). +INSERT INTO analytics_hourly_paths (hour, iata, category, hash_bytes, entries, receptions) +SELECT @hour::timestamptz, c.iata, c.category, c.hash_bytes, c.entries, count(*) +FROM ( + SELECT k.iata, k.category, + CASE WHEN k.category = 0 THEN k.hash_size ELSE 0 END::integer AS hash_bytes, + CASE WHEN k.category = 0 THEN k.hop_count ELSE 0 END::integer AS entries + FROM ( + SELECT iata, hash_size, hop_count, + CASE WHEN payload_type = 9 THEN 2 + WHEN payload_type IS NULL OR payload_type NOT BETWEEN 0 AND 15 + OR NOT (path_length_byte BETWEEN 0 AND 191 + AND hash_size BETWEEN 1 AND 3 AND hop_count BETWEEN 0 AND 63 + AND hash_size = (path_length_byte >> 6) + 1 + AND hop_count = (path_length_byte & 63) + AND hash_size::integer * hop_count::integer <= 64 + AND path_len = hash_size::integer * hop_count::integer) + THEN 3 + WHEN hop_count = 0 THEN 1 + ELSE 0 END::integer AS category + FROM rollup_obs + ) k +) c +GROUP BY c.iata, c.category, c.hash_bytes, c.entries; + +-- name: RollObserverActivity :exec +INSERT INTO analytics_hourly_observer_activity (hour, observer_id, payload_type, observations, airtime_ms, airtime_n, + snr_sum, snr_n, snr_min, rssi_sum, rssi_n) +SELECT @hour::timestamptz, observer_id, COALESCE(payload_type, -1)::smallint, + count(*), sum(airtime_ms)::real, count(airtime_ms), + sum(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real, + count(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0)), + min(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real, + sum(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint, + count(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0)) +FROM rollup_obs +GROUP BY observer_id, COALESCE(payload_type, -1); + +-- name: RollObserverIdentity :exec +INSERT INTO analytics_hourly_observer_identity (hour, iata, observer_id, observation_count, display_name, observer_type) +SELECT @hour::timestamptz, r.iata, r.observer_id, r.n, o.display_name, o.observer_type +FROM (SELECT iata, observer_id, count(*) AS n FROM rollup_obs GROUP BY iata, observer_id) r +LEFT JOIN observers o ON o.id = r.observer_id; + +-- name: RollAdvertHearings :exec +INSERT INTO analytics_hourly_advert_hearings (hour, iata, origin_pubkey, observations, last_heard, name, node_type) +SELECT @hour::timestamptz, r.iata, r.origin_pubkey, r.n, r.last_heard, n.name, n.node_type +FROM (SELECT iata, origin_pubkey, count(*) AS n, max(heard_at) AS last_heard + FROM rollup_obs + WHERE payload_type = 4 AND origin_pubkey IS NOT NULL + GROUP BY iata, origin_pubkey) r +LEFT JOIN nodes n ON n.public_key = r.origin_pubkey; + +-- name: RollAdvertSets :exec +-- Each distinct ADVERT packet counts once, under the exact set of IATAs that heard it this hour. +INSERT INTO analytics_hourly_advert_sets (hour, origin_pubkey, iatas, advert_packets, flood_packets, direct_packets) +SELECT @hour::timestamptz, origin_pubkey, iatas, count(*), + count(*) FILTER (WHERE route_type IN (0, 1)), + count(*) FILTER (WHERE route_type IN (2, 3)) +FROM (SELECT origin_pubkey, route_type, array_agg(DISTINCT iata ORDER BY iata) AS iatas + FROM rollup_obs + WHERE payload_type = 4 AND origin_pubkey IS NOT NULL + GROUP BY packet_hash, origin_pubkey, route_type) p +GROUP BY origin_pubkey, iatas; + +-- name: RollTalkerSets :exec +INSERT INTO analytics_hourly_talker_sets (hour, sender_name, iatas, messages, last_sent) +SELECT @hour::timestamptz, sender_name, iatas, count(*), max(sent_at) +FROM (SELECT cm.sender_name, max(cm.sent_at) AS sent_at, array_agg(DISTINCT ro.iata ORDER BY ro.iata) AS iatas + FROM rollup_obs ro + JOIN channel_messages cm ON cm.packet_hash = ro.packet_hash + WHERE cm.sender_name IS NOT NULL + GROUP BY cm.id, cm.sender_name) m +GROUP BY sender_name, iatas; + +-- name: RollPacketSets :exec +INSERT INTO analytics_hourly_packet_sets (hour, iatas, packets) +SELECT @hour::timestamptz, iatas, count(*) +FROM (SELECT array_agg(DISTINCT iata ORDER BY iata) AS iatas FROM rollup_obs GROUP BY packet_hash) p +GROUP BY iatas; + +-- name: RollScopeSets :exec +INSERT INTO analytics_hourly_scope_sets (hour, iatas, scope_id, packets) +SELECT @hour::timestamptz, iatas, scope_id, count(*) +FROM (SELECT scope_id, array_agg(DISTINCT iata ORDER BY iata) AS iatas + FROM rollup_obs WHERE scope_id IS NOT NULL + GROUP BY packet_hash, scope_id) p +GROUP BY iatas, scope_id; + +-- name: RollupContentHash :one +-- Order-independent fingerprint of every family row for the hour. +SELECT md5(concat_ws('|', + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_iata_observations t WHERE t.hour = @hour::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_payload_breakdown t WHERE t.hour = @hour::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_signal t WHERE t.hour = @hour::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_paths t WHERE t.hour = @hour::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_observer_activity t WHERE t.hour = @hour::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_observer_identity t WHERE t.hour = @hour::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_advert_hearings t WHERE t.hour = @hour::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_advert_sets t WHERE t.hour = @hour::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_talker_sets t WHERE t.hour = @hour::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_packet_sets t WHERE t.hour = @hour::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_scope_sets t WHERE t.hour = @hour::timestamptz) +))::text AS content_hash; + +-- name: FinishRollupHour :one +-- Marks the hour complete and bumps the revision only when its content changed. +WITH prev AS ( + SELECT ph.content_hash FROM analytics_rollup_hours ph WHERE ph.hour = @hour::timestamptz +), upd AS ( + UPDATE analytics_rollup_hours uh + SET status = 'complete', rolled_at = now(), content_hash = @content_hash::text + WHERE uh.hour = @hour::timestamptz +), undirty AS ( + DELETE FROM analytics_dirty_hours dh WHERE dh.hour = @hour::timestamptz +), rev AS ( + UPDATE analytics_state SET revision = revision + 1 + WHERE (SELECT content_hash FROM prev) IS DISTINCT FROM @content_hash::text + RETURNING revision +) +SELECT EXISTS (SELECT 1 FROM rev)::boolean AS changed; + +-- name: GetAnalyticsRevision :one +-- Both counters only grow, so their sum changes whenever rolled content or coverage does. +SELECT (s.revision + r.coverage)::bigint FROM analytics_state s CROSS JOIN analytics_raw_state r; + +-- name: OldestMissingRollupHour :one +SELECT min(hour)::timestamptz FROM analytics_rollup_hours WHERE status = 'missing'; + +-- name: DeleteOldRollups :exec +-- Bumps coverage when hours go, so cached responses that included them are not reused. +WITH a AS (DELETE FROM analytics_hourly_iata_observations ta WHERE ta.hour < @cutoff::timestamptz), + b AS (DELETE FROM analytics_hourly_payload_breakdown tb WHERE tb.hour < @cutoff::timestamptz), + c AS (DELETE FROM analytics_hourly_signal tc WHERE tc.hour < @cutoff::timestamptz), + d AS (DELETE FROM analytics_hourly_paths td WHERE td.hour < @cutoff::timestamptz), + e AS (DELETE FROM analytics_hourly_observer_activity te WHERE te.hour < @cutoff::timestamptz), + f AS (DELETE FROM analytics_hourly_observer_identity tf WHERE tf.hour < @cutoff::timestamptz), + g AS (DELETE FROM analytics_hourly_advert_hearings tg WHERE tg.hour < @cutoff::timestamptz), + h AS (DELETE FROM analytics_hourly_advert_sets th WHERE th.hour < @cutoff::timestamptz), + i AS (DELETE FROM analytics_hourly_talker_sets ti WHERE ti.hour < @cutoff::timestamptz), + j AS (DELETE FROM analytics_hourly_packet_sets tj WHERE tj.hour < @cutoff::timestamptz), + k AS (DELETE FROM analytics_hourly_scope_sets tk WHERE tk.hour < @cutoff::timestamptz), + l AS (DELETE FROM analytics_dirty_hours tl WHERE tl.hour < @cutoff::timestamptz), + m AS (DELETE FROM analytics_rollup_hours tm WHERE tm.hour < @cutoff::timestamptz RETURNING 1) +UPDATE analytics_raw_state SET coverage = coverage + 1 WHERE EXISTS (SELECT 1 FROM m); diff --git a/db/queries/series.sql b/db/queries/series.sql new file mode 100644 index 0000000..a22ca8e --- /dev/null +++ b/db/queries/series.sql @@ -0,0 +1,91 @@ +-- name: GetStatsSeries :many +-- One row per hour in [since, until) plus a summary row (hour NULL) over its complete hours. +-- Counts sum; observers, IATAs and scopes are distinct across the window; packets count +-- once per hour heard. Each branch pair keeps "all IATAs" and "these IATAs" plans separate. +WITH hours AS ( + SELECT g.h::timestamptz AS hour, COALESCE(r.status, 'missing')::text AS status + FROM generate_series(@since::timestamptz, @until::timestamptz, INTERVAL '1 hour') g(h) + LEFT JOIN analytics_rollup_hours r ON r.hour = g.h + WHERE g.h < @until::timestamptz +), complete AS ( + SELECT hour FROM hours WHERE status = 'complete' +), obs AS ( + SELECT o.hour, o.iata, o.observation_count FROM analytics_hourly_iata_observations o JOIN complete c ON c.hour = o.hour + WHERE COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 + UNION ALL + SELECT o.hour, o.iata, o.observation_count FROM analytics_hourly_iata_observations o JOIN complete c ON c.hour = o.hour + WHERE cardinality(@iatas::bpchar[]) > 0 AND o.iata = ANY(@iatas::bpchar[]) +), obs_agg AS ( + SELECT hour, sum(observation_count)::bigint AS observations, count(DISTINCT iata)::bigint AS active_iatas + FROM obs GROUP BY GROUPING SETS ((), (hour)) +), pkt AS ( + SELECT s.hour, s.packets FROM analytics_hourly_packet_sets s JOIN complete c ON c.hour = s.hour + WHERE COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 + UNION ALL + SELECT s.hour, s.packets FROM analytics_hourly_packet_sets s JOIN complete c ON c.hour = s.hour + WHERE cardinality(@iatas::bpchar[]) > 0 AND s.iatas && @iatas::bpchar[] +), pkt_agg AS ( + SELECT hour, sum(packets)::bigint AS unique_packets FROM pkt GROUP BY GROUPING SETS ((), (hour)) +), obsr AS ( + SELECT i.hour, i.observer_id FROM analytics_hourly_observer_identity i JOIN complete c ON c.hour = i.hour + WHERE COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 + UNION ALL + SELECT i.hour, i.observer_id FROM analytics_hourly_observer_identity i JOIN complete c ON c.hour = i.hour + WHERE cardinality(@iatas::bpchar[]) > 0 AND i.iata = ANY(@iatas::bpchar[]) +), obsr_agg AS ( + SELECT hour, count(DISTINCT observer_id)::bigint AS active_observers FROM obsr GROUP BY GROUPING SETS ((), (hour)) +), scp AS ( + SELECT s.hour, s.scope_id, s.packets FROM analytics_hourly_scope_sets s JOIN complete c ON c.hour = s.hour + WHERE COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 + UNION ALL + SELECT s.hour, s.scope_id, s.packets FROM analytics_hourly_scope_sets s JOIN complete c ON c.hour = s.hour + WHERE cardinality(@iatas::bpchar[]) > 0 AND s.iatas && @iatas::bpchar[] +), scp_agg AS ( + SELECT hour, sum(packets)::bigint AS scoped_packets, count(DISTINCT scope_id)::bigint AS active_scopes + FROM scp GROUP BY GROUPING SETS ((), (hour)) +), pth AS ( + SELECT p.hour, p.entries FROM analytics_hourly_paths p JOIN complete c ON c.hour = p.hour + WHERE p.category = 0 AND COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 + UNION ALL + SELECT p.hour, p.entries FROM analytics_hourly_paths p JOIN complete c ON c.hour = p.hour + WHERE p.category = 0 AND cardinality(@iatas::bpchar[]) > 0 AND p.iata = ANY(@iatas::bpchar[]) +), pth_agg AS ( + SELECT hour, max(entries)::integer AS max_path_entries FROM pth GROUP BY GROUPING SETS ((), (hour)) +), sig AS ( + SELECT s.hour, s.snr_sum, s.snr_samples, s.rssi_sum, s.rssi_samples FROM analytics_hourly_signal s JOIN complete c ON c.hour = s.hour + WHERE s.kind = 3 AND COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 + UNION ALL + SELECT s.hour, s.snr_sum, s.snr_samples, s.rssi_sum, s.rssi_samples FROM analytics_hourly_signal s JOIN complete c ON c.hour = s.hour + WHERE s.kind = 3 AND cardinality(@iatas::bpchar[]) > 0 AND s.iata = ANY(@iatas::bpchar[]) +), sig_agg AS ( + SELECT hour, sum(snr_sum)::double precision AS snr_sum, sum(snr_samples)::bigint AS snr_samples, + sum(rssi_sum)::double precision AS rssi_sum, sum(rssi_samples)::bigint AS rssi_samples + FROM sig GROUP BY GROUPING SETS ((), (hour)) +), keys AS ( + SELECT hour, status FROM hours + UNION ALL + SELECT NULL::timestamptz, 'summary'::text +) +SELECT k.hour, k.status, + COALESCE(o.observations, 0)::bigint AS observations, + COALESCE(p.unique_packets, 0)::bigint AS unique_packets, + COALESCE(ob.active_observers, 0)::bigint AS active_observers, + COALESCE(o.active_iatas, 0)::bigint AS active_iatas, + COALESCE(s.scoped_packets, 0)::bigint AS scoped_packets, + COALESCE(s.active_scopes, 0)::bigint AS active_scopes, + COALESCE(pa.max_path_entries, 0)::integer AS max_path_entries, + COALESCE(sg.snr_sum, 0)::double precision AS snr_sum, + COALESCE(sg.snr_samples, 0)::bigint AS snr_samples, + COALESCE(sg.rssi_sum, 0)::double precision AS rssi_sum, + COALESCE(sg.rssi_samples, 0)::bigint AS rssi_samples +FROM keys k +LEFT JOIN obs_agg o ON o.hour IS NOT DISTINCT FROM k.hour +LEFT JOIN pkt_agg p ON p.hour IS NOT DISTINCT FROM k.hour +LEFT JOIN obsr_agg ob ON ob.hour IS NOT DISTINCT FROM k.hour +LEFT JOIN scp_agg s ON s.hour IS NOT DISTINCT FROM k.hour +LEFT JOIN pth_agg pa ON pa.hour IS NOT DISTINCT FROM k.hour +LEFT JOIN sig_agg sg ON sg.hour IS NOT DISTINCT FROM k.hour +ORDER BY k.hour NULLS LAST; + +-- name: GetEarliestCompleteRollupHour :one +SELECT min(hour)::timestamptz FROM analytics_rollup_hours WHERE status = 'complete'; diff --git a/db/queries/signal.sql b/db/queries/signal.sql index 24bb800..ecd20cf 100644 --- a/db/queries/signal.sql +++ b/db/queries/signal.sql @@ -1,12 +1,12 @@ -- name: GetSignalStats :many --- Read compact hourly snapshots, never observations on an HTTP request. +-- Read the hourly rollup, never observations on an HTTP request. -- Weight averages by sample counts instead of averaging regional/hourly means. WITH readings AS ( - SELECT * FROM mv_signal_stats_hourly + SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM analytics_hourly_signal WHERE hour >= @since::timestamptz AND hour < @until::timestamptz AND COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 UNION ALL - SELECT * FROM mv_signal_stats_hourly + SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM analytics_hourly_signal WHERE hour >= @since::timestamptz AND hour < @until::timestamptz AND cardinality(@iatas::bpchar[]) > 0 AND iata = ANY(@iatas::bpchar[]) ) @@ -27,5 +27,3 @@ FROM readings WHERE kind IN (1, 2) GROUP BY kind, snr_bin, rssi_bin ORDER BY kind, hour, snr_bin, rssi_bin; --- name: RefreshSignalStats :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_signal_stats_hourly; diff --git a/db/rollup.go b/db/rollup.go new file mode 100644 index 0000000..3c2cab3 --- /dev/null +++ b/db/rollup.go @@ -0,0 +1,202 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "errors" + "fmt" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgtype" + "github.com/jackc/pgx/v5/pgxpool" +) + +// Must match db/sqlc_scratch/rollup_obs.sql, which only exists so sqlc can type-check rollup.sql. +const createRollupObs = `CREATE TEMP TABLE rollup_obs ( + packet_hash bytea NOT NULL, observer_id uuid NOT NULL, iata character(3) NOT NULL, + heard_at timestamptz NOT NULL, path_length_byte smallint NOT NULL, hash_size smallint NOT NULL, + hop_count smallint NOT NULL, path_len integer NOT NULL, rssi smallint, snr real, airtime_ms real, + payload_type smallint, route_type smallint NOT NULL, origin_pubkey bytea, scope_id integer +) ON COMMIT DROP` + +// rollupDelay is how long after an hour closes it becomes eligible to roll (ingest clamps +// heard_at to ±30 min; RegisterRollupHours uses the same bound). +const rollupDelay = 95 * time.Minute + +// RollOutcome is what RollHour did with an hour. +type RollOutcome int + +const ( + RollSkipped RollOutcome = iota // unregistered, or lost a serialization race; retried next pass + RollPartial // raw rows already deleted; never rolled + RollComplete // rolled and marked complete +) + +// RollupSession holds the rollup advisory lock on one dedicated connection. +type RollupSession struct { + conn *pgxpool.Conn + q *sqlc.Queries +} + +// BeginRollup takes the rollup lock. ok is false when another process holds it. +func (s *Store) BeginRollup(ctx context.Context) (*RollupSession, bool, error) { + if s.pool == nil { + return nil, false, errors.New("rollup: store has no connection pool") + } + conn, err := s.pool.Acquire(ctx) + if err != nil { + return nil, false, err + } + q := sqlc.New(conn) + ok, err := q.TryRollupLock(ctx) + if err != nil || !ok { + conn.Release() + return nil, false, err + } + return &RollupSession{conn: conn, q: q}, true, nil +} + +// Close releases the lock. If unlocking fails the connection is closed so the lock can't leak. +func (r *RollupSession) Close(ctx context.Context) { + if err := r.q.RollupUnlock(ctx); err != nil { + _ = r.conn.Hijack().Close(context.Background()) + return + } + r.conn.Release() +} + +// RegisterHours adds a 'missing' row for every eligible hour since max(since, earliest observation). +func (r *RollupSession) RegisterHours(ctx context.Context, since time.Time) error { + return r.q.RegisterRollupHours(ctx, ts(since)) +} + +func (r *RollupSession) MarkPartialHours(ctx context.Context) error { + return r.q.MarkPartialRollupHours(ctx) +} + +// MissingHours returns up to limit unrolled hours, oldest first. +func (r *RollupSession) MissingHours(ctx context.Context, limit int32) ([]time.Time, error) { + rows, err := r.q.ListMissingRollupHours(ctx, limit) + return times(rows), err +} + +// DirtyHours drops dirty hours that can't be re-rolled and returns up to limit of the rest. +func (r *RollupSession) DirtyHours(ctx context.Context, limit int32) ([]time.Time, error) { + if err := r.q.DeleteStaleDirtyHours(ctx); err != nil { + return nil, err + } + rows, err := r.q.ListDirtyRollupHours(ctx, limit) + return times(rows), err +} + +// RollHour rebuilds every rollup family for one hour in a single REPEATABLE READ transaction. +// changed reports whether the hour's content differs from its previous roll. +func (r *RollupSession) RollHour(ctx context.Context, hour time.Time) (outcome RollOutcome, changed bool, err error) { + tx, err := r.conn.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead}) + if err != nil { + return RollSkipped, false, err + } + defer func() { + _ = tx.Rollback(context.Background()) + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) && pgErr.Code == "40001" { + outcome, changed, err = RollSkipped, false, nil + } + }() + outcome, changed, err = rollHour(ctx, tx, hour) + if err != nil { + return RollSkipped, false, err + } + return outcome, changed, tx.Commit(ctx) +} + +// rollHour does RollHour's work inside tx; the caller commits. +func rollHour(ctx context.Context, tx pgx.Tx, hour time.Time) (RollOutcome, bool, error) { + q := sqlc.New(tx) + h := ts(hour) + status, err := q.LockRollupHour(ctx, h) + if errors.Is(err, pgx.ErrNoRows) { + return RollSkipped, false, nil + } + if err != nil { + return RollSkipped, false, err + } + if status == "partial" { + return RollPartial, false, q.SetRollupHourPartial(ctx, h) + } + if _, err := tx.Exec(ctx, createRollupObs); err != nil { + return RollSkipped, false, err + } + if err := q.FillRollupObs(ctx, h); err != nil { + return RollSkipped, false, err + } + if _, err := tx.Exec(ctx, "ANALYZE rollup_obs"); err != nil { + return RollSkipped, false, err + } + if err := q.DeleteRollupHour(ctx, h); err != nil { + return RollSkipped, false, err + } + for name, roll := range map[string]func(context.Context, pgtype.Timestamptz) error{ + "iata observations": q.RollIATAObservations, + "payload breakdown": q.RollPayloadBreakdown, + "signal": q.RollSignal, + "paths": q.RollPaths, + "observer activity": q.RollObserverActivity, + "observer identity": q.RollObserverIdentity, + "advert hearings": q.RollAdvertHearings, + "advert sets": q.RollAdvertSets, + "talker sets": q.RollTalkerSets, + "packet sets": q.RollPacketSets, + "scope sets": q.RollScopeSets, + } { + if err := roll(ctx, h); err != nil { + return RollSkipped, false, fmt.Errorf("roll %s: %w", name, err) + } + } + // Dropped here as well as ON COMMIT so a caller can roll several hours in one transaction. + if _, err := tx.Exec(ctx, "DROP TABLE rollup_obs"); err != nil { + return RollSkipped, false, err + } + hash, err := q.RollupContentHash(ctx, h) + if err != nil { + return RollSkipped, false, err + } + changed, err := q.FinishRollupHour(ctx, sqlc.FinishRollupHourParams{Hour: h, ContentHash: hash}) + if err != nil { + return RollSkipped, false, err + } + return RollComplete, changed, nil +} + +// AnalyticsRevision changes whenever rolled content, an hour's status or rollup coverage changes. +func (s *Store) AnalyticsRevision(ctx context.Context) (int64, error) { + return s.q.GetAnalyticsRevision(ctx) +} + +func ts(t time.Time) pgtype.Timestamptz { + return pgtype.Timestamptz{Time: t, Valid: true} +} + +func times(rows []pgtype.Timestamptz) []time.Time { + out := make([]time.Time, len(rows)) + for i, r := range rows { + out[i] = r.Time + } + return out +} + +// OldestMissingRollupHour returns the earliest hour still waiting to be rolled; ok is false if none. +func (s *Store) OldestMissingRollupHour(ctx context.Context) (hour time.Time, ok bool, err error) { + h, err := s.q.OldestMissingRollupHour(ctx) + return h.Time, h.Valid, err +} + +// DeleteOldRollups drops rolled hours, their bookkeeping and queued re-rolls before cutoff. +func (s *Store) DeleteOldRollups(ctx context.Context, cutoff time.Time) error { + return s.q.DeleteOldRollups(ctx, ts(cutoff)) +} diff --git a/db/rollup_integration_test.go b/db/rollup_integration_test.go new file mode 100644 index 0000000..a5cace6 --- /dev/null +++ b/db/rollup_integration_test.go @@ -0,0 +1,515 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/ingest" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +// rollupFixture loads observations across two hours, h0 and h0+1h, and returns h0. +// h0+2h is left empty so a zero-observation hour gets registered too. +func rollupFixture(t *testing.T, ctx context.Context, pool *pgxpool.Pool) time.Time { + t.Helper() + if err := RunMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + var h0 time.Time + if err := pool.QueryRow(ctx, "SELECT date_trunc('hour', now(), 'UTC') - INTERVAL '4 hours'").Scan(&h0); err != nil { + t.Fatal(err) + } + _, err := pool.Exec(ctx, ` +INSERT INTO iata_codes (iata) VALUES ('YVR'), ('YYZ'), ('YOW'); +INSERT INTO transport_scopes (id, name, transport_key, key_fingerprint) +VALUES (1, 'scope-one', decode(repeat('01', 16), 'hex'), decode(repeat('01', 8), 'hex')); +INSERT INTO observers (id, public_key, display_name, observer_type) VALUES + ('00000000-0000-0000-0000-00000000000a', '\x0a', 'obs a', 'repeater'), + ('00000000-0000-0000-0000-00000000000b', '\x0b', 'obs b', NULL); +INSERT INTO nodes (public_key, node_type, name) VALUES ('\xee', 2, 'node e'); +INSERT INTO channels (id, channel_hash) VALUES (1, '\x11'); +-- hash, payload type, route type, origin, scope +INSERT INTO packets (packet_hash, payload_type, payload_version, route_type, origin_pubkey, scope_id, raw_payload, raw_header, first_heard_at, last_heard_at) +SELECT hash, pt, rt, 0, origin, scope, '\x00', '\x00', $1::timestamptz, $1::timestamptz FROM (VALUES + ('\x01'::bytea, 4, 1, '\xee'::bytea, NULL::int), -- flood advert, two IATAs + ('\x02', 4, 2, '\xee', NULL), -- direct advert, one IATA + ('\x03', 5, 1, NULL, 1), -- scoped channel message, two IATAs + ('\x04', 5, 1, NULL, 1), -- scoped channel message, one IATA + ('\x05', 2, 1, NULL, NULL), -- straddles h0 and h0+1h + ('\x06', 9, 1, NULL, NULL), -- TRACE + ('\x07', 2, 1, NULL, NULL) -- invalid path +) v(hash, pt, rt, origin, scope); +INSERT INTO channel_messages (id, channel_id, packet_hash, sender_name, sent_at) VALUES + (1, 1, '\x03', 'alice', $1::timestamptz + INTERVAL '5 minutes'), + (2, 1, '\x04', 'alice', $1::timestamptz + INTERVAL '6 minutes'); +SELECT setval(pg_get_serial_sequence('channel_messages', 'id'), 100); +INSERT INTO packet_observations (id, packet_hash, observer_id, iata, heard_at, path_length_byte, hash_size, hop_count, path_bytes, rssi, snr, airtime_ms, payload_type) +SELECT n, hash, obs::uuid, iata, $1::timestamptz + mins * INTERVAL '1 minute', plb, hs, hc, pb, rssi, snr, air, pt FROM (VALUES + (1, '\x01'::bytea, '00000000-0000-0000-0000-00000000000a', 'YVR', 1, 65, 2, 1, '\xaabb'::bytea, -90, 5.5::real, 12.5::real, 4), + (2, '\x01', '00000000-0000-0000-0000-00000000000b', 'YYZ', 2, 0, 1, 0, NULL, -100, -3.25, NULL, 4), + (3, '\x02', '00000000-0000-0000-0000-00000000000a', 'YVR', 3, 2, 1, 2, '\xaabb', 0, 0, 10, 4), + (4, '\x03', '00000000-0000-0000-0000-00000000000a', 'YVR', 4, 1, 1, 1, '\xaa', -80, 8, 20, 5), + (5, '\x03', '00000000-0000-0000-0000-00000000000b', 'YYZ', 5, 1, 1, 1, '\xbb', -85, 7, NULL, 5), + (6, '\x04', '00000000-0000-0000-0000-00000000000b', 'YYZ', 6, 0, 1, 0, NULL, NULL, NULL, NULL, 5), + (7, '\x05', '00000000-0000-0000-0000-00000000000a', 'YVR', 59, 0, 1, 0, NULL, -120, -12, 5, 2), + (8, '\x05', '00000000-0000-0000-0000-00000000000b', 'YYZ', 61, 0, 1, 0, NULL, -121, -13, 5, 2), + (9, '\x06', '00000000-0000-0000-0000-00000000000a', 'YVR', 7, 3, 1, 3, '\x010203', -95, 2, 30, 9), + (10, '\x07', '00000000-0000-0000-0000-00000000000a', 'YVR', 8, 3, 1, 3, '\xaa', -95, 2, 30, 2) +) v(n, hash, obs, iata, mins, plb, hs, hc, pb, rssi, snr, air, pt);`, pgx.QueryExecModeSimpleProtocol, h0) + if err != nil { + t.Fatal(err) + } + return h0 +} + +// Raw-scan oracles: the SQL the old live views ran, grouped per hour. +var rollupOracles = map[string][2]string{ + "iata observations": { + `SELECT hour, iata, observation_count FROM analytics_hourly_iata_observations`, + `SELECT date_trunc('hour', heard_at, 'UTC'), iata, count(*) FROM packet_observations GROUP BY 1, 2`}, + "payload breakdown": { + `SELECT hour, iata, payload_type, count FROM analytics_hourly_payload_breakdown`, + `SELECT date_trunc('hour', heard_at, 'UTC'), iata, payload_type, count(*) FROM packet_observations + WHERE payload_type IS NOT NULL GROUP BY 1, 2, 3`}, + "signal": { + `SELECT hour, iata, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM analytics_hourly_signal`, + `WITH samples AS ( + SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, + CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) + AND snr > '-Infinity'::real AND snr < 'Infinity'::real THEN snr::double precision END AS snr, + CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) THEN rssi::double precision END AS rssi + FROM packet_observations + ), binned AS ( + SELECT *, width_bucket(snr, -30, 30, 12) AS snr_bin, width_bucket(rssi, -140, 0, 14) AS rssi_bin FROM samples + ) + SELECT hour, iata, grouping(snr_bin, rssi_bin)::integer, COALESCE(snr_bin, -1)::integer, COALESCE(rssi_bin, -1)::integer, + count(*)::bigint, count(snr)::bigint, COALESCE(sum(snr), 0)::double precision, + count(rssi)::bigint, COALESCE(sum(rssi), 0)::double precision + FROM binned GROUP BY GROUPING SETS ((iata, hour), (iata, hour, snr_bin), (iata, hour, rssi_bin))`}, + "paths": { + `SELECT hour, iata, category, hash_bytes, entries, receptions FROM analytics_hourly_paths`, + `WITH classified AS ( + SELECT iata, heard_at, hash_size, hop_count, + CASE WHEN payload_type = 9 THEN 2 + WHEN payload_type IS NULL OR payload_type NOT BETWEEN 0 AND 15 + OR NOT (path_length_byte BETWEEN 0 AND 191 + AND hash_size BETWEEN 1 AND 3 AND hop_count BETWEEN 0 AND 63 + AND hash_size = (path_length_byte >> 6) + 1 + AND hop_count = (path_length_byte & 63) + AND hash_size::integer * hop_count::integer <= 64 + AND COALESCE(octet_length(path_bytes), 0) = hash_size::integer * hop_count::integer) + THEN 3 + WHEN hop_count = 0 THEN 1 + ELSE 0 END::integer AS category + FROM packet_observations + ), buckets AS ( + SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, category, + CASE WHEN category = 0 THEN hash_size ELSE 0 END::integer AS hash_bytes, + CASE WHEN category = 0 THEN hop_count ELSE 0 END::integer AS entries + FROM classified + ) + SELECT hour, iata, category, hash_bytes, entries, count(*)::bigint FROM buckets GROUP BY 1, 2, 3, 4, 5`}, + "observer activity": { + `SELECT hour, observer_id, payload_type, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n + FROM analytics_hourly_observer_activity`, + `SELECT date_trunc('hour', heard_at, 'UTC'), observer_id, COALESCE(payload_type, -1)::smallint, + COUNT(*)::bigint, SUM(airtime_ms)::real, COUNT(airtime_ms)::bigint, + SUM(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real, + COUNT(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint, + MIN(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real, + SUM(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint, + COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint + FROM packet_observations GROUP BY 1, 2, 3`}, + "observer identity": { + `SELECT hour, iata, observer_id, observation_count, display_name, observer_type FROM analytics_hourly_observer_identity`, + `SELECT date_trunc('hour', po.heard_at, 'UTC'), po.iata, po.observer_id, count(*), o.display_name, o.observer_type + FROM packet_observations po JOIN observers o ON o.id = po.observer_id GROUP BY 1, 2, 3, 5, 6`}, + "advert hearings": { + `SELECT hour, iata, origin_pubkey, observations, last_heard, name, node_type FROM analytics_hourly_advert_hearings`, + `SELECT date_trunc('hour', po.heard_at, 'UTC'), po.iata, p.origin_pubkey, count(*), max(po.heard_at), n.name, n.node_type + FROM packet_observations po JOIN packets p ON p.packet_hash = po.packet_hash + LEFT JOIN nodes n ON n.public_key = p.origin_pubkey + WHERE po.payload_type = 4 AND p.origin_pubkey IS NOT NULL GROUP BY 1, 2, 3, 6, 7`}, +} + +// Set families must give exact distinct counts for any IATA set, summed per hour. +var rollupSetOracles = map[string][2]string{ + "packets": { + `SELECT COALESCE(sum(packets), 0) FROM analytics_hourly_packet_sets WHERE iatas && $1::bpchar[]`, + `SELECT count(*) FROM (SELECT DISTINCT date_trunc('hour', heard_at, 'UTC'), packet_hash + FROM packet_observations WHERE iata = ANY($1::bpchar[])) d`}, + "scoped packets": { + `SELECT COALESCE(sum(packets), 0) FROM analytics_hourly_scope_sets WHERE iatas && $1::bpchar[] AND scope_id = 1`, + `SELECT count(*) FROM (SELECT DISTINCT date_trunc('hour', po.heard_at, 'UTC'), po.packet_hash + FROM packet_observations po JOIN packets p ON p.packet_hash = po.packet_hash + WHERE po.iata = ANY($1::bpchar[]) AND p.scope_id = 1) d`}, + "adverts": { + `SELECT COALESCE(sum(advert_packets), 0) FROM analytics_hourly_advert_sets WHERE iatas && $1::bpchar[]`, + `SELECT count(*) FROM (SELECT DISTINCT date_trunc('hour', po.heard_at, 'UTC'), po.packet_hash + FROM packet_observations po JOIN packets p ON p.packet_hash = po.packet_hash + WHERE po.iata = ANY($1::bpchar[]) AND po.payload_type = 4) d`}, + "flood adverts": { + `SELECT COALESCE(sum(flood_packets), 0) FROM analytics_hourly_advert_sets WHERE iatas && $1::bpchar[]`, + `SELECT count(*) FROM (SELECT DISTINCT date_trunc('hour', po.heard_at, 'UTC'), po.packet_hash + FROM packet_observations po JOIN packets p ON p.packet_hash = po.packet_hash + WHERE po.iata = ANY($1::bpchar[]) AND po.payload_type = 4 AND p.route_type IN (0, 1)) d`}, + "messages": { + `SELECT COALESCE(sum(messages), 0) FROM analytics_hourly_talker_sets WHERE iatas && $1::bpchar[] AND sender_name = 'alice'`, + `SELECT count(*) FROM (SELECT DISTINCT date_trunc('hour', po.heard_at, 'UTC'), cm.id + FROM packet_observations po JOIN channel_messages cm ON cm.packet_hash = po.packet_hash + WHERE po.iata = ANY($1::bpchar[]) AND cm.sender_name = 'alice') d`}, +} + +// rollTxHours registers every hour holding observations (eligible or not) and rolls all +// registered hours inside tx, so tx-scoped fixtures can read rollup-backed stats. +func rollTxHours(t *testing.T, ctx context.Context, tx pgx.Tx) { + t.Helper() + if _, err := tx.Exec(ctx, `INSERT INTO analytics_rollup_hours (hour, status) + SELECT DISTINCT date_trunc('hour', heard_at, 'UTC'), 'missing' FROM packet_observations + ON CONFLICT (hour) DO NOTHING`); err != nil { + t.Fatal(err) + } + rows, err := tx.Query(ctx, "SELECT hour FROM analytics_rollup_hours ORDER BY hour") + if err != nil { + t.Fatal(err) + } + hours, err := pgx.CollectRows(rows, pgx.RowTo[time.Time]) + if err != nil { + t.Fatal(err) + } + for _, h := range hours { + if outcome, _, err := rollHour(ctx, tx, h); err != nil || outcome != RollComplete { + t.Fatalf("roll %s: outcome %d err %v", h, outcome, err) + } + } +} + +// backfillMessage stores a decrypted-late message the way channel backfill does. +func backfillMessage(t *testing.T, ctx context.Context, store *Store, hash byte, sender string, at time.Time) { + t.Helper() + m, err := store.InsertChannelMessage(ctx, ingest.InsertChannelMessageParams{ + ChannelID: 1, PacketHash: []byte{hash}, SenderName: sender, Content: "late", SentAt: at, Historical: true, + }) + if err != nil || m == nil { + t.Fatalf("backfill message %x: %v %v", hash, m, err) + } +} + +func rowsText(t *testing.T, ctx context.Context, pool *pgxpool.Pool, query string) string { + t.Helper() + var s *string + if err := pool.QueryRow(ctx, "SELECT string_agg(x::text, E'\\n' ORDER BY x::text) FROM ("+query+") x").Scan(&s); err != nil { + t.Fatalf("%v\n%s", err, query) + } + if s == nil { + return "" + } + return *s +} + +func rollAll(t *testing.T, ctx context.Context, r *RollupSession) { + t.Helper() + hours, err := r.MissingHours(ctx, 100) + if err != nil { + t.Fatal(err) + } + for _, h := range hours { + if outcome, _, err := r.RollHour(ctx, h); err != nil || outcome != RollComplete { + t.Fatalf("roll %s: outcome %d, err %v", h, outcome, err) + } + } +} + +func revision(t *testing.T, ctx context.Context, s *Store) int64 { + t.Helper() + rev, err := s.AnalyticsRevision(ctx) + if err != nil { + t.Fatal(err) + } + return rev +} + +func TestRollupHourPostgres(t *testing.T) { + ctx, pool := schemaPool(t) + h0 := rollupFixture(t, ctx, pool) + store := New(pool, 0, 0) + + r, ok, err := store.BeginRollup(ctx) + if err != nil || !ok { + t.Fatalf("BeginRollup: ok=%v err=%v", ok, err) + } + defer r.Close(ctx) + if other, ok, err := store.BeginRollup(ctx); err != nil || ok { + if other != nil { + other.Close(ctx) + } + t.Fatalf("second BeginRollup: ok=%v err=%v, want lock held", ok, err) + } + + if err := r.RegisterHours(ctx, h0.Add(-10*time.Hour)); err != nil { + t.Fatal(err) + } + var first, last time.Time + var eligibleLast bool + if err := pool.QueryRow(ctx, `SELECT min(hour), max(hour), max(hour) = date_trunc('hour', now() - INTERVAL '95 minutes', 'UTC') + FROM analytics_rollup_hours WHERE status = 'missing'`).Scan(&first, &last, &eligibleLast); err != nil { + t.Fatal(err) + } + if !first.Equal(h0) || !eligibleLast { + t.Fatalf("registered hours %s..%s, want %s..now-95m", first, last, h0) + } + + rev0 := revision(t, ctx, store) + rollAll(t, ctx, r) + rev1 := revision(t, ctx, store) + if rev1 == rev0 { + t.Error("first roll did not bump the revision") + } + + for name, q := range rollupOracles { + if got, want := rowsText(t, ctx, pool, q[0]), rowsText(t, ctx, pool, q[1]); got != want { + t.Errorf("%s differs from raw:\nrollup:\n%s\nraw:\n%s", name, got, want) + } + } + for name, q := range rollupSetOracles { + for _, set := range [][]string{{"YVR"}, {"YYZ"}, {"YVR", "YYZ"}, {"YOW"}, {"YOW", "YVR", "YYZ"}} { + var got, want int64 + if err := pool.QueryRow(ctx, q[0], set).Scan(&got); err != nil { + t.Fatal(err) + } + if err := pool.QueryRow(ctx, q[1], set).Scan(&want); err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("%s for %v: rollup %d, raw %d", name, set, got, want) + } + } + } + + // Empty hour: complete, no family rows. + var emptyStatus string + var emptyRows int + if err := pool.QueryRow(ctx, `SELECT status, (SELECT count(*) FROM analytics_hourly_iata_observations WHERE hour = $1) + FROM analytics_rollup_hours WHERE hour = $1`, h0.Add(2*time.Hour)).Scan(&emptyStatus, &emptyRows); err != nil { + t.Fatal(err) + } + if emptyStatus != "complete" || emptyRows != 0 { + t.Errorf("empty hour status %q rows %d, want complete, 0", emptyStatus, emptyRows) + } + + // Re-rolling unchanged data is a no-op for readers. + if outcome, changed, err := r.RollHour(ctx, h0); err != nil || outcome != RollComplete || changed { + t.Fatalf("idempotent re-roll: outcome %d changed %v err %v", outcome, changed, err) + } + if revision(t, ctx, store) != rev1 { + t.Error("idempotent re-roll bumped the revision") + } + + // A backfilled message dirties every hour its packet was heard in; re-rolling picks it up. + backfillMessage(t, ctx, store, 0x05, "bob", h0) + dirty, err := r.DirtyHours(ctx, 10) + if err != nil || len(dirty) != 2 || !dirty[0].Equal(h0) || !dirty[1].Equal(h0.Add(time.Hour)) { + t.Fatalf("dirty hours %v, %v; want h0 and h0+1h", dirty, err) + } + for _, h := range dirty { + if outcome, changed, err := r.RollHour(ctx, h); err != nil || outcome != RollComplete || !changed { + t.Fatalf("dirty re-roll %s: outcome %d changed %v err %v", h, outcome, changed, err) + } + } + if revision(t, ctx, store) <= rev1 { + t.Error("changed re-roll did not bump the revision") + } + if dirty, err := r.DirtyHours(ctx, 10); err != nil || len(dirty) != 0 { + t.Errorf("dirty hours after re-roll %v, %v; want none", dirty, err) + } + var bob int64 + if err := pool.QueryRow(ctx, `SELECT COALESCE(sum(messages), 0) FROM analytics_hourly_talker_sets WHERE sender_name = 'bob'`).Scan(&bob); err != nil || bob != 2 { + t.Errorf("bob messages across the straddled hours = %d, %v; want 2", bob, err) + } +} + +func TestRollupPartialHoursPostgres(t *testing.T) { + ctx, pool := schemaPool(t) + h0 := rollupFixture(t, ctx, pool) + store := New(pool, 0, 0) + r, ok, err := store.BeginRollup(ctx) + if err != nil || !ok { + t.Fatalf("BeginRollup: ok=%v err=%v", ok, err) + } + defer r.Close(ctx) + if err := r.RegisterHours(ctx, h0.Add(-10*time.Hour)); err != nil { + t.Fatal(err) + } + + // Rolling h0+1h first leaves h0 missing when cleanup reaches it. + if outcome, _, err := r.RollHour(ctx, h0.Add(time.Hour)); err != nil || outcome != RollComplete { + t.Fatalf("roll h0+1h: %d %v", outcome, err) + } + if _, err := pool.Exec(ctx, "UPDATE analytics_raw_state SET raw_deleted_before = $1", h0.Add(20*time.Minute)); err != nil { + t.Fatal(err) + } + if outcome, _, err := r.RollHour(ctx, h0); err != nil || outcome != RollPartial { + t.Fatalf("roll h0 after raw deletion: outcome %d err %v, want partial", outcome, err) + } + if err := r.MarkPartialHours(ctx); err != nil { + t.Fatal(err) + } + var statuses string + if err := pool.QueryRow(ctx, `SELECT string_agg(status, ',' ORDER BY hour) FROM analytics_rollup_hours WHERE hour IN ($1, $2)`, + h0, h0.Add(time.Hour)).Scan(&statuses); err != nil { + t.Fatal(err) + } + if statuses != "partial,complete" { + t.Errorf("statuses %q, want partial,complete", statuses) + } + + // Only complete hours with intact raw rows are re-rolled; h0's entry is dropped. + backfillMessage(t, ctx, store, 0x01, "carol", h0) + backfillMessage(t, ctx, store, 0x05, "bob", h0) + if dirty, err := r.DirtyHours(ctx, 10); err != nil || len(dirty) != 1 || !dirty[0].Equal(h0.Add(time.Hour)) { + t.Errorf("dirty hours %v, %v; want only h0+1h", dirty, err) + } + if missing, err := r.MissingHours(ctx, 100); err != nil || len(missing) == 0 || !missing[0].After(h0.Add(time.Hour)) { + t.Errorf("missing hours %v, %v; want only hours after h0+1h", missing, err) + } +} + +// A roll that dies mid-transaction leaves the hour missing for the next pass. +func TestRollupInterruptedHourPostgres(t *testing.T) { + ctx, pool := schemaPool(t) + h0 := rollupFixture(t, ctx, pool) + store := New(pool, 0, 0) + r, ok, err := store.BeginRollup(ctx) + if err != nil || !ok { + t.Fatalf("BeginRollup: ok=%v err=%v", ok, err) + } + defer r.Close(ctx) + if err := r.RegisterHours(ctx, h0); err != nil { + t.Fatal(err) + } + // Breaking a family table makes the roll fail after earlier families were written. + if _, err := pool.Exec(ctx, "ALTER TABLE analytics_hourly_scope_sets ADD CONSTRAINT boom CHECK (packets < 0) NOT VALID"); err != nil { + t.Fatal(err) + } + if _, _, err := r.RollHour(ctx, h0); err == nil { + t.Fatal("roll succeeded despite a failing family") + } + var status string + var rows int + if err := pool.QueryRow(ctx, `SELECT status, (SELECT count(*) FROM analytics_hourly_iata_observations WHERE hour = $1) + FROM analytics_rollup_hours WHERE hour = $1`, h0).Scan(&status, &rows); err != nil { + t.Fatal(err) + } + if status != "missing" || rows != 0 { + t.Errorf("after failed roll: status %q, %d family rows; want missing, 0", status, rows) + } + if _, err := pool.Exec(ctx, "ALTER TABLE analytics_hourly_scope_sets DROP CONSTRAINT boom"); err != nil { + t.Fatal(err) + } + if outcome, _, err := r.RollHour(ctx, h0); err != nil || outcome != RollComplete { + t.Errorf("retry: outcome %d err %v", outcome, err) + } +} + +// A message backfilled after a first roll's snapshot (but before it commits) is not lost: +// its dirty entry is invisible to that roll, survives it, and triggers a re-roll. +func TestRollupBackfillDuringFirstRollPostgres(t *testing.T) { + ctx, pool := schemaPool(t) + h0 := rollupFixture(t, ctx, pool) + store := New(pool, 0, 0) + r, ok, err := store.BeginRollup(ctx) + if err != nil || !ok { + t.Fatalf("BeginRollup: ok=%v err=%v", ok, err) + } + defer r.Close(ctx) + if err := r.RegisterHours(ctx, h0); err != nil { + t.Fatal(err) + } + + roll, err := pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead}) + if err != nil { + t.Fatal(err) + } + defer roll.Rollback(ctx) + if _, err := roll.Exec(ctx, "SELECT count(*) FROM channel_messages"); err != nil { // fixes the snapshot + t.Fatal(err) + } + backfillMessage(t, ctx, store, 0x05, "bob", h0) // commits on another connection + if outcome, _, err := rollHour(ctx, roll, h0); err != nil || outcome != RollComplete { + t.Fatalf("first roll: %d %v", outcome, err) + } + if err := roll.Commit(ctx); err != nil { + t.Fatal(err) + } + talkers := func() int64 { + t.Helper() + var n int64 + if err := pool.QueryRow(ctx, "SELECT COALESCE(sum(messages), 0) FROM analytics_hourly_talker_sets WHERE hour = $1 AND sender_name = 'bob'", h0).Scan(&n); err != nil { + t.Fatal(err) + } + return n + } + if talkers() != 0 { + t.Fatal("first roll saw the message; the race was not reproduced") + } + dirty, err := r.DirtyHours(ctx, 10) + if err != nil || len(dirty) == 0 || !dirty[0].Equal(h0) { + t.Fatalf("dirty hours %v, %v; want h0 still queued", dirty, err) + } + if outcome, _, err := r.RollHour(ctx, h0); err != nil || outcome != RollComplete { + t.Fatalf("re-roll: %d %v", outcome, err) + } + if talkers() != 1 { + t.Error("re-roll did not pick up the backfilled message") + } +} + +// With no observations there is nothing to roll, so no empty "complete" hours. +func TestRollupRegisterEmptyDatabasePostgres(t *testing.T) { + ctx, pool := schemaPool(t) + if err := RunMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + r, ok, err := New(pool, 0, 0).BeginRollup(ctx) + if err != nil || !ok { + t.Fatalf("BeginRollup: ok=%v err=%v", ok, err) + } + defer r.Close(ctx) + if err := r.RegisterHours(ctx, time.Now().Add(-7*24*time.Hour)); err != nil { + t.Fatal(err) + } + var n int + if err := pool.QueryRow(ctx, "SELECT count(*) FROM analytics_rollup_hours").Scan(&n); err != nil || n != 0 { + t.Errorf("registered %d hours on an empty database, err %v; want 0", n, err) + } +} + +// An observation may be heard up to 30 minutes after its packet's last_heard_at, so an hour +// starting exactly raw_deleted_before + 30 min may have lost rows too. +func TestRollupPartialBoundaryPostgres(t *testing.T) { + ctx, pool := schemaPool(t) + h0 := rollupFixture(t, ctx, pool) + r, ok, err := New(pool, 0, 0).BeginRollup(ctx) + if err != nil || !ok { + t.Fatalf("BeginRollup: ok=%v err=%v", ok, err) + } + defer r.Close(ctx) + if err := r.RegisterHours(ctx, h0); err != nil { + t.Fatal(err) + } + h1 := h0.Add(time.Hour) + if _, err := pool.Exec(ctx, "UPDATE analytics_raw_state SET raw_deleted_before = $1", h1.Add(-30*time.Minute)); err != nil { + t.Fatal(err) + } + if outcome, _, err := r.RollHour(ctx, h1); err != nil || outcome != RollPartial { + t.Errorf("hour at the boundary: outcome %d err %v, want partial", outcome, err) + } + if outcome, _, err := r.RollHour(ctx, h1.Add(time.Hour)); err != nil || outcome != RollComplete { + t.Errorf("next hour: outcome %d err %v, want complete", outcome, err) + } +} diff --git a/db/rollup_schema_integration_test.go b/db/rollup_schema_integration_test.go new file mode 100644 index 0000000..169e07f --- /dev/null +++ b/db/rollup_schema_integration_test.go @@ -0,0 +1,117 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "github.com/jackc/pgx/v5" +) + +func TestRollupSchemaPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for PostgreSQL regression") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(ctx) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(ctx) + applyBaseline(t, ctx, tx) + + // Each failing insert runs in a savepoint so the tx stays usable. + bad := map[string]string{ + "duplicate": "'{YVR,YVR}'", + "unsorted": "'{YYZ,YVR}'", + "empty": "'{}'", + "null member": "'{YVR,NULL}'", + } + inserts := map[string]string{ + "packet_sets": "INSERT INTO analytics_hourly_packet_sets (hour, iatas, packets) VALUES (now(), %s, 1)", + "scope_sets": "INSERT INTO analytics_hourly_scope_sets (hour, iatas, scope_id, packets) VALUES (now(), %s, 1, 1)", + "advert_sets": "INSERT INTO analytics_hourly_advert_sets (hour, origin_pubkey, iatas, advert_packets, flood_packets, direct_packets) VALUES (now(), '\\x01', %s, 1, 1, 0)", + "talker_sets": "INSERT INTO analytics_hourly_talker_sets (hour, sender_name, iatas, messages, last_sent) VALUES (now(), 'a', %s, 1, now())", + } + for tname, tmpl := range inserts { + for name, lit := range bad { + sql := fmt.Sprintf(tmpl, lit) + if _, err := tx.Exec(ctx, "SAVEPOINT s"); err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(ctx, sql); err == nil { + t.Errorf("%s %s: insert succeeded, want CHECK violation", tname, name) + } + if _, err := tx.Exec(ctx, "ROLLBACK TO SAVEPOINT s"); err != nil { + t.Fatal(err) + } + } + if _, err := tx.Exec(ctx, fmt.Sprintf(tmpl, "'{YVR,YYZ}'")); err != nil { + t.Errorf("%s canonical set: %v", tname, err) + } + } + + for _, tbl := range []string{"analytics_state", "analytics_raw_state"} { + var n int + if err := tx.QueryRow(ctx, "SELECT count(*) FROM "+tbl).Scan(&n); err != nil || n != 1 { + t.Errorf("%s rows = %d, err %v; want 1", tbl, n, err) + } + if _, err := tx.Exec(ctx, "SAVEPOINT s"); err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(ctx, "INSERT INTO "+tbl+" (singleton) VALUES (false)"); err == nil { + t.Errorf("%s accepted a second row", tbl) + } + if _, err := tx.Exec(ctx, "ROLLBACK TO SAVEPOINT s"); err != nil { + t.Fatal(err) + } + } + var rev int64 + if err := tx.QueryRow(ctx, "SELECT revision FROM analytics_state").Scan(&rev); err != nil || rev != 0 { + t.Errorf("revision = %d, err %v; want 0", rev, err) + } + var rawDeleted *time.Time + if err := tx.QueryRow(ctx, "SELECT raw_deleted_before FROM analytics_raw_state").Scan(&rawDeleted); err != nil || rawDeleted != nil { + t.Errorf("raw_deleted_before = %v, err %v; want NULL", rawDeleted, err) + } + + var heard, brin, obsID, lastIATA *string + if err := tx.QueryRow(ctx, `SELECT to_regclass('idx_observations_heard')::text, to_regclass('idx_observations_heard_brin')::text, + to_regclass('idx_observations_observer_id')::text, to_regclass('idx_observers_last_iata')::text`).Scan(&heard, &brin, &obsID, &lastIATA); err != nil { + t.Fatal(err) + } + if heard == nil || obsID == nil || lastIATA == nil { + t.Errorf("missing index: heard=%v observer_id=%v last_iata=%v", heard, obsID, lastIATA) + } + if brin != nil { + t.Errorf("idx_observations_heard_brin still exists") + } + + var nullable string + if err := tx.QueryRow(ctx, `SELECT is_nullable FROM information_schema.columns + WHERE table_schema = current_schema() AND table_name = 'channels' AND column_name = 'message_count'`).Scan(&nullable); err != nil || nullable != "NO" { + t.Errorf("channels.message_count nullable = %q, err %v; want NO", nullable, err) + } + for _, c := range [][2]string{{"packets", "observation_count"}, {"observers", "last_iata"}} { + var n int + if err := tx.QueryRow(ctx, `SELECT count(*) FROM information_schema.columns + WHERE table_schema = current_schema() AND table_name = $1 AND column_name = $2`, c[0], c[1]).Scan(&n); err != nil || n != 1 { + t.Errorf("column %s.%s missing (err %v)", c[0], c[1], err) + } + } + if _, err := tx.Exec(ctx, "INSERT INTO trace_tags (trace_tag, first_heard_at, last_heard_at) VALUES ('\\x01', now(), now())"); err != nil { + t.Errorf("trace_tags: %v", err) + } +} diff --git a/db/scope_stats_integration_test.go b/db/scope_stats_integration_test.go index 94e1845..207e09d 100644 --- a/db/scope_stats_integration_test.go +++ b/db/scope_stats_integration_test.go @@ -4,52 +4,42 @@ package db import ( - "context" - "os" "reflect" "testing" "time" - sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/jackc/pgx/v5" ) -// Run on a migrated private database. All fixture tables and writes are rolled back. +// Packets come from the rolled IATA sets (each packet once per hour, however many requested +// IATAs heard it); observers are current memberships filtered by their latest IATA. func TestScopeStatsPostgres(t *testing.T) { - dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") - if dsn == "" { - t.Skip("set BEACON_TEST_POSTGRES_DSN for PostgreSQL regression") - } - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) - defer cancel() - conn, err := pgx.Connect(ctx, dsn) - if err != nil { + ctx, pool := schemaPool(t) + if err := RunMigrations(ctx, pool); err != nil { t.Fatal(err) } - defer conn.Close(context.Background()) - tx, err := conn.Begin(ctx) - if err != nil { + var h time.Time + if err := pool.QueryRow(ctx, "SELECT date_trunc('hour', now(), 'UTC') - INTERVAL '4 hours'").Scan(&h); err != nil { t.Fatal(err) } - defer tx.Rollback(context.Background()) - for _, table := range []string{"transport_scopes", "packets", "packet_observations", "observer_scopes", "nodes", "node_iatas"} { - if _, err := tx.Exec(ctx, "CREATE TEMP TABLE "+table+" (LIKE public."+table+" INCLUDING ALL) ON COMMIT DROP"); err != nil { - t.Fatal(err) - } - } - _, err = tx.Exec(ctx, ` + if _, err := pool.Exec(ctx, ` +INSERT INTO iata_codes (iata) VALUES ('YVR'), ('YYJ'), ('YYZ'); INSERT INTO transport_scopes (id,name,transport_key,key_fingerprint) SELECT i,name,decode(repeat('00',16),'hex'),decode(lpad(to_hex(i),16,'0'),'hex') FROM (VALUES (1,'#a'),(2,'#b'),(3,'#unused')) v(i,name); +INSERT INTO observers (id,public_key,last_iata) +SELECT md5(i::text)::uuid,decode(lpad(to_hex(i),2,'0'),'hex'),iata +FROM (VALUES (1,'YVR'),(2,'YYJ'),(3,'YVR'),(4,'YVR'),(5,'YYZ')) v(i,iata); INSERT INTO packets (packet_hash,scope_id,payload_type,payload_version,route_type,raw_payload,raw_header,first_heard_at,last_heard_at) -SELECT decode(lpad(to_hex(i),2,'0'),'hex'),scope_id,4,0,1,'\x00','\x00',NOW(),NOW() +SELECT decode(lpad(to_hex(i),2,'0'),'hex'),scope_id,4,0,1,'\x00','\x00',$1::timestamptz,$1::timestamptz FROM (VALUES (1,1),(2,1),(3,1),(4,1),(5,2),(6,2),(7,2),(8,NULL)) v(i,scope_id); INSERT INTO observer_scopes (observer_id,scope_id) SELECT md5(i::text)::uuid,scope_id FROM (VALUES (1,1),(2,1),(4,1),(5,1),(1,2),(2,2),(3,2)) v(i,scope_id); +-- packet 4 is never heard, so it never reaches the rollup INSERT INTO packet_observations (id,packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count) SELECT id,decode(lpad(to_hex(packet),2,'0'),'hex'),md5(observer::text)::uuid,iata, - '2026-01-01'::timestamptz+id*interval '1 second',0,1,0 + $1::timestamptz+id*interval '1 second',0,1,0 FROM (VALUES (1,1,1,'YVR'),(2,1,4,'YVR'),(3,1,2,'YYJ'),(4,2,2,'YVR'), (5,3,1,'YYZ'),(6,5,3,'YVR'),(7,6,1,'YYJ'),(8,7,2,'YYZ'),(9,8,1,'YVR')) v(id,packet,observer,iata); INSERT INTO nodes (id,public_key,node_type,default_scope_id) @@ -57,27 +47,38 @@ SELECT md5(i::text)::uuid,decode(lpad(to_hex(i),64,'0'),'hex'),2,scope_id FROM (VALUES (1,1),(2,1),(3,1),(4,1),(5,2),(6,2),(7,NULL)) v(i,scope_id); INSERT INTO node_iatas (node_id,iata) SELECT md5(i::text)::uuid,iata FROM (VALUES (1,'YVR'),(1,'YYJ'),(2,'YVR'),(3,'YYZ'),(5,'YYJ'),(6,'YVR'),(6,'YYJ'),(7,'YVR')) v(i,iata); -`) - if err != nil { +`, pgx.QueryExecModeSimpleProtocol, h); err != nil { t.Fatal(err) } - store := &Store{q: sqlc.New(tx)} + store := New(pool, 0, 0) + r, ok, err := store.BeginRollup(ctx) + if err != nil || !ok { + t.Fatalf("BeginRollup: ok=%v err=%v", ok, err) + } + if err := r.RegisterHours(ctx, h); err != nil { + t.Fatal(err) + } + rollAll(t, ctx, r) + r.Close(ctx) + for _, tc := range []struct { name string iatas []string + since time.Time a, b [3]int64 // packets, observers, nodes }{ - {"global", nil, [3]int64{4, 4, 4}, [3]int64{3, 3, 2}}, - {"empty filter", []string{}, [3]int64{4, 4, 4}, [3]int64{3, 3, 2}}, - {"YVR", []string{"YVR"}, [3]int64{2, 3, 2}, [3]int64{1, 1, 1}}, - {"YYJ", []string{"YYJ"}, [3]int64{1, 1, 1}, [3]int64{1, 1, 2}}, - {"YYZ", []string{"YYZ"}, [3]int64{1, 1, 1}, [3]int64{1, 1, 0}}, - {"overlapping IATAs", []string{"YVR", "YYJ"}, [3]int64{2, 3, 2}, [3]int64{2, 2, 2}}, - {"duplicate IATAs", []string{"YYJ", "YVR", "YYJ"}, [3]int64{2, 3, 2}, [3]int64{2, 2, 2}}, - {"unknown IATA", []string{"ZZZ"}, [3]int64{}, [3]int64{}}, + {"global", nil, time.Time{}, [3]int64{3, 4, 4}, [3]int64{3, 3, 2}}, + {"empty filter", []string{}, time.Time{}, [3]int64{3, 4, 4}, [3]int64{3, 3, 2}}, + {"YVR", []string{"YVR"}, time.Time{}, [3]int64{2, 2, 2}, [3]int64{1, 2, 1}}, + {"YYJ", []string{"YYJ"}, time.Time{}, [3]int64{1, 1, 1}, [3]int64{1, 1, 2}}, + {"YYZ", []string{"YYZ"}, time.Time{}, [3]int64{1, 1, 1}, [3]int64{1, 0, 0}}, + {"overlapping IATAs", []string{"YVR", "YYJ"}, time.Time{}, [3]int64{2, 3, 2}, [3]int64{2, 3, 2}}, + {"duplicate IATAs", []string{"YYJ", "YVR", "YYJ"}, time.Time{}, [3]int64{2, 3, 2}, [3]int64{2, 3, 2}}, + {"unknown IATA", []string{"ZZZ"}, time.Time{}, [3]int64{}, [3]int64{}}, + {"window after the traffic", nil, h.Add(time.Hour), [3]int64{0, 4, 4}, [3]int64{0, 3, 2}}, } { t.Run(tc.name, func(t *testing.T) { - rows, err := store.GetScopeStats(ctx, tc.iatas) + rows, err := store.GetScopeStats(ctx, tc.iatas, tc.since) if err != nil { t.Fatal(err) } @@ -91,10 +92,20 @@ SELECT md5(i::text)::uuid,iata FROM (VALUES (1,'YVR'),(1,'YYJ'),(2,'YVR'),(3,'YY } }) } - if _, err := tx.Exec(ctx, "DELETE FROM transport_scopes"); err != nil { + + detail, err := store.GetScopeByName(ctx, "#a") + if err != nil { t.Fatal(err) } - rows, err := store.GetScopeStats(ctx, nil) + want := &api.ScopeDetail{Name: "#a", PacketCount: 3, ObserverCount: 4, NodeCount: 4, IATACount: 3, IATAs: []string{"YVR", "YYJ", "YYZ"}} + if !reflect.DeepEqual(detail, want) { + t.Errorf("scope detail %+v, want %+v", detail, want) + } + + if _, err := pool.Exec(ctx, "TRUNCATE transport_scopes CASCADE"); err != nil { + t.Fatal(err) + } + rows, err := store.GetScopeStats(ctx, nil, time.Time{}) if err != nil || rows == nil || len(rows) != 0 { t.Fatalf("empty roster = %+v, %v", rows, err) } diff --git a/db/signal.go b/db/signal.go index 9a7b1a6..0edd4cf 100644 --- a/db/signal.go +++ b/db/signal.go @@ -45,10 +45,6 @@ func (s *Store) GetSignalStats(ctx context.Context, since, until time.Time, iata return stats, nil } -func (s *Store) RefreshSignalStats(ctx context.Context) error { - return s.q.RefreshSignalStats(ctx) -} - func signalAverage(value float64, count int64) *float64 { if count == 0 { return nil diff --git a/db/signal_integration_test.go b/db/signal_integration_test.go index 73bc825..0655503 100644 --- a/db/signal_integration_test.go +++ b/db/signal_integration_test.go @@ -64,9 +64,7 @@ SELECT int4send(n),'00000000-0000-0000-0000-000000000001',heard_at,iata,snr,rssi if _, err := tx.Exec(ctx, "UPDATE packet_observations SET heard_at=to_timestamp(extract(epoch FROM $1::timestamptz)+extract(epoch FROM heard_at)-extract(epoch FROM '2026-01-01 00:00+00'::timestamptz))", since); err != nil { t.Fatal(err) } - if _, err := tx.Exec(ctx, "REFRESH MATERIALIZED VIEW mv_signal_stats_hourly"); err != nil { - t.Fatal(err) - } + rollTxHours(t, ctx, tx) store := &Store{q: sqlc.New(tx)} until := since.Add(4 * time.Hour) for _, tc := range []struct { @@ -138,7 +136,7 @@ SELECT int4send(n),'00000000-0000-0000-0000-000000000001',heard_at,iata,snr,rssi } w := httptest.NewRecorder() request := httptest.NewRequest("GET", fmt.Sprintf("/signal?since=%d&until=%d&iatas=YVR", since.UnixMilli()+123, until.UnixMilli()+123), nil).WithContext(ctx) - handlers.StatsRouter(store, nil).ServeHTTP(w, request) + handlers.StatsRouter(store, handlers.StatsOptions{}).ServeHTTP(w, request) var response api.SignalStats if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil || w.Code != 200 || response.Receptions != 13 || response.SNR.Samples != 5 { t.Fatalf("PostgreSQL HTTP response: status=%d body=%s error=%v", w.Code, w.Body.String(), err) @@ -150,9 +148,7 @@ SELECT int4send(n),'00000000-0000-0000-0000-000000000001',heard_at,iata,snr,rssi if err != nil || stale.Receptions != 15 { t.Fatalf("snapshot lost: %+v %v", stale, err) } - if err := store.RefreshSignalStats(ctx); err != nil { - t.Fatal(err) - } + rollTxHours(t, ctx, tx) got, err := store.GetSignalStats(ctx, since, until, nil) if err != nil || got.Receptions != 0 || got.Hourly == nil || len(got.Hourly) != 0 || got.SNR.Average != nil { t.Fatalf("empty: %+v %v", got, err) diff --git a/db/sqlc/mock/querier.go b/db/sqlc/mock/querier.go index 0626fda..2b436d9 100644 --- a/db/sqlc/mock/querier.go +++ b/db/sqlc/mock/querier.go @@ -245,6 +245,20 @@ func (mr *MockQuerierMockRecorder) DeleteOldPackets(ctx, arg any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteOldPackets", reflect.TypeOf((*MockQuerier)(nil).DeleteOldPackets), ctx, arg) } +// DeleteOldRollups mocks base method. +func (m *MockQuerier) DeleteOldRollups(ctx context.Context, cutoff pgtype.Timestamptz) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteOldRollups", ctx, cutoff) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteOldRollups indicates an expected call of DeleteOldRollups. +func (mr *MockQuerierMockRecorder) DeleteOldRollups(ctx, cutoff any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteOldRollups", reflect.TypeOf((*MockQuerier)(nil).DeleteOldRollups), ctx, cutoff) +} + // DeleteOldRoutes mocks base method. func (m *MockQuerier) DeleteOldRoutes(ctx context.Context, arg db.DeleteOldRoutesParams) (int64, error) { m.ctrl.T.Helper() @@ -288,6 +302,20 @@ func (mr *MockQuerierMockRecorder) DeleteOldTraceIATAs(ctx, lastHeard any) *gomo return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteOldTraceIATAs", reflect.TypeOf((*MockQuerier)(nil).DeleteOldTraceIATAs), ctx, lastHeard) } +// DeleteOldTraceTags mocks base method. +func (m *MockQuerier) DeleteOldTraceTags(ctx context.Context, lastHeardAt pgtype.Timestamptz) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteOldTraceTags", ctx, lastHeardAt) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteOldTraceTags indicates an expected call of DeleteOldTraceTags. +func (mr *MockQuerierMockRecorder) DeleteOldTraceTags(ctx, lastHeardAt any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteOldTraceTags", reflect.TypeOf((*MockQuerier)(nil).DeleteOldTraceTags), ctx, lastHeardAt) +} + // DeleteRegionIATAsNotIn mocks base method. func (m *MockQuerier) DeleteRegionIATAsNotIn(ctx context.Context, arg db.DeleteRegionIATAsNotInParams) error { m.ctrl.T.Helper() @@ -302,6 +330,63 @@ func (mr *MockQuerierMockRecorder) DeleteRegionIATAsNotIn(ctx, arg any) *gomock. return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteRegionIATAsNotIn", reflect.TypeOf((*MockQuerier)(nil).DeleteRegionIATAsNotIn), ctx, arg) } +// DeleteRollupHour mocks base method. +func (m *MockQuerier) DeleteRollupHour(ctx context.Context, hour pgtype.Timestamptz) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteRollupHour", ctx, hour) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteRollupHour indicates an expected call of DeleteRollupHour. +func (mr *MockQuerierMockRecorder) DeleteRollupHour(ctx, hour any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteRollupHour", reflect.TypeOf((*MockQuerier)(nil).DeleteRollupHour), ctx, hour) +} + +// DeleteStaleDirtyHours mocks base method. +func (m *MockQuerier) DeleteStaleDirtyHours(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteStaleDirtyHours", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteStaleDirtyHours indicates an expected call of DeleteStaleDirtyHours. +func (mr *MockQuerierMockRecorder) DeleteStaleDirtyHours(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteStaleDirtyHours", reflect.TypeOf((*MockQuerier)(nil).DeleteStaleDirtyHours), ctx) +} + +// FillRollupObs mocks base method. +func (m *MockQuerier) FillRollupObs(ctx context.Context, dollar_1 pgtype.Timestamptz) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FillRollupObs", ctx, dollar_1) + ret0, _ := ret[0].(error) + return ret0 +} + +// FillRollupObs indicates an expected call of FillRollupObs. +func (mr *MockQuerierMockRecorder) FillRollupObs(ctx, dollar_1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FillRollupObs", reflect.TypeOf((*MockQuerier)(nil).FillRollupObs), ctx, dollar_1) +} + +// FinishRollupHour mocks base method. +func (m *MockQuerier) FinishRollupHour(ctx context.Context, arg db.FinishRollupHourParams) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FinishRollupHour", ctx, arg) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// FinishRollupHour indicates an expected call of FinishRollupHour. +func (mr *MockQuerierMockRecorder) FinishRollupHour(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FinishRollupHour", reflect.TypeOf((*MockQuerier)(nil).FinishRollupHour), ctx, arg) +} + // GetAccount mocks base method. func (m *MockQuerier) GetAccount(ctx context.Context, id uuid.UUID) (db.Account, error) { m.ctrl.T.Helper() @@ -317,6 +402,21 @@ func (mr *MockQuerierMockRecorder) GetAccount(ctx, id any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAccount", reflect.TypeOf((*MockQuerier)(nil).GetAccount), ctx, id) } +// GetAnalyticsRevision mocks base method. +func (m *MockQuerier) GetAnalyticsRevision(ctx context.Context) (int64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetAnalyticsRevision", ctx) + ret0, _ := ret[0].(int64) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetAnalyticsRevision indicates an expected call of GetAnalyticsRevision. +func (mr *MockQuerierMockRecorder) GetAnalyticsRevision(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAnalyticsRevision", reflect.TypeOf((*MockQuerier)(nil).GetAnalyticsRevision), ctx) +} + // GetChannelByID mocks base method. func (m *MockQuerier) GetChannelByID(ctx context.Context, id int32) (db.Channel, error) { m.ctrl.T.Helper() @@ -347,11 +447,26 @@ func (mr *MockQuerierMockRecorder) GetCrossIATANeighbors(ctx, arg any) *gomock.C return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCrossIATANeighbors", reflect.TypeOf((*MockQuerier)(nil).GetCrossIATANeighbors), ctx, arg) } +// GetEarliestCompleteRollupHour mocks base method. +func (m *MockQuerier) GetEarliestCompleteRollupHour(ctx context.Context) (pgtype.Timestamptz, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetEarliestCompleteRollupHour", ctx) + ret0, _ := ret[0].(pgtype.Timestamptz) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetEarliestCompleteRollupHour indicates an expected call of GetEarliestCompleteRollupHour. +func (mr *MockQuerierMockRecorder) GetEarliestCompleteRollupHour(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetEarliestCompleteRollupHour", reflect.TypeOf((*MockQuerier)(nil).GetEarliestCompleteRollupHour), ctx) +} + // GetHourlyStats mocks base method. -func (m *MockQuerier) GetHourlyStats(ctx context.Context, arg db.GetHourlyStatsParams) ([]db.MvHourlyIataStat, error) { +func (m *MockQuerier) GetHourlyStats(ctx context.Context, arg db.GetHourlyStatsParams) ([]db.GetHourlyStatsRow, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetHourlyStats", ctx, arg) - ret0, _ := ret[0].([]db.MvHourlyIataStat) + ret0, _ := ret[0].([]db.GetHourlyStatsRow) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -618,18 +733,18 @@ func (mr *MockQuerierMockRecorder) GetObserverComparison(ctx, arg any) *gomock.C } // GetObserverLastIATA mocks base method. -func (m *MockQuerier) GetObserverLastIATA(ctx context.Context, observerID uuid.UUID) (string, error) { +func (m *MockQuerier) GetObserverLastIATA(ctx context.Context, id uuid.UUID) (string, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetObserverLastIATA", ctx, observerID) + ret := m.ctrl.Call(m, "GetObserverLastIATA", ctx, id) ret0, _ := ret[0].(string) ret1, _ := ret[1].(error) return ret0, ret1 } // GetObserverLastIATA indicates an expected call of GetObserverLastIATA. -func (mr *MockQuerierMockRecorder) GetObserverLastIATA(ctx, observerID any) *gomock.Call { +func (mr *MockQuerierMockRecorder) GetObserverLastIATA(ctx, id any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetObserverLastIATA", reflect.TypeOf((*MockQuerier)(nil).GetObserverLastIATA), ctx, observerID) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetObserverLastIATA", reflect.TypeOf((*MockQuerier)(nil).GetObserverLastIATA), ctx, id) } // GetObserverRadio mocks base method. @@ -692,21 +807,6 @@ func (mr *MockQuerierMockRecorder) GetPacketByHash(ctx, packetHash any) *gomock. return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPacketByHash", reflect.TypeOf((*MockQuerier)(nil).GetPacketByHash), ctx, packetHash) } -// GetPacketObservationCount mocks base method. -func (m *MockQuerier) GetPacketObservationCount(ctx context.Context, packetHash []byte) (int64, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetPacketObservationCount", ctx, packetHash) - ret0, _ := ret[0].(int64) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetPacketObservationCount indicates an expected call of GetPacketObservationCount. -func (mr *MockQuerierMockRecorder) GetPacketObservationCount(ctx, packetHash any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPacketObservationCount", reflect.TypeOf((*MockQuerier)(nil).GetPacketObservationCount), ctx, packetHash) -} - // GetPacketsByTraceTag mocks base method. func (m *MockQuerier) GetPacketsByTraceTag(ctx context.Context, decode string) ([]db.GetPacketsByTraceTagRow, error) { m.ctrl.T.Helper() @@ -858,18 +958,18 @@ func (mr *MockQuerierMockRecorder) GetScopeNames(ctx any) *gomock.Call { } // GetScopeStats mocks base method. -func (m *MockQuerier) GetScopeStats(ctx context.Context, iatas []string) ([]db.GetScopeStatsRow, error) { +func (m *MockQuerier) GetScopeStats(ctx context.Context, arg db.GetScopeStatsParams) ([]db.GetScopeStatsRow, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetScopeStats", ctx, iatas) + ret := m.ctrl.Call(m, "GetScopeStats", ctx, arg) ret0, _ := ret[0].([]db.GetScopeStatsRow) ret1, _ := ret[1].(error) return ret0, ret1 } // GetScopeStats indicates an expected call of GetScopeStats. -func (mr *MockQuerierMockRecorder) GetScopeStats(ctx, iatas any) *gomock.Call { +func (mr *MockQuerierMockRecorder) GetScopeStats(ctx, arg any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetScopeStats", reflect.TypeOf((*MockQuerier)(nil).GetScopeStats), ctx, iatas) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetScopeStats", reflect.TypeOf((*MockQuerier)(nil).GetScopeStats), ctx, arg) } // GetSignalStats mocks base method. @@ -917,21 +1017,6 @@ func (mr *MockQuerierMockRecorder) GetStatsNodeTypes(ctx, dollar_1 any) *gomock. return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetStatsNodeTypes", reflect.TypeOf((*MockQuerier)(nil).GetStatsNodeTypes), ctx, dollar_1) } -// GetStatsOverview mocks base method. -func (m *MockQuerier) GetStatsOverview(ctx context.Context, dollar_1 []string) (db.GetStatsOverviewRow, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetStatsOverview", ctx, dollar_1) - ret0, _ := ret[0].(db.GetStatsOverviewRow) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetStatsOverview indicates an expected call of GetStatsOverview. -func (mr *MockQuerierMockRecorder) GetStatsOverview(ctx, dollar_1 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetStatsOverview", reflect.TypeOf((*MockQuerier)(nil).GetStatsOverview), ctx, dollar_1) -} - // GetStatsPayloadBreakdown mocks base method. func (m *MockQuerier) GetStatsPayloadBreakdown(ctx context.Context, arg db.GetStatsPayloadBreakdownParams) ([]db.GetStatsPayloadBreakdownRow, error) { m.ctrl.T.Helper() @@ -947,6 +1032,21 @@ func (mr *MockQuerierMockRecorder) GetStatsPayloadBreakdown(ctx, arg any) *gomoc return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetStatsPayloadBreakdown", reflect.TypeOf((*MockQuerier)(nil).GetStatsPayloadBreakdown), ctx, arg) } +// GetStatsSeries mocks base method. +func (m *MockQuerier) GetStatsSeries(ctx context.Context, arg db.GetStatsSeriesParams) ([]db.GetStatsSeriesRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetStatsSeries", ctx, arg) + ret0, _ := ret[0].([]db.GetStatsSeriesRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetStatsSeries indicates an expected call of GetStatsSeries. +func (mr *MockQuerierMockRecorder) GetStatsSeries(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetStatsSeries", reflect.TypeOf((*MockQuerier)(nil).GetStatsSeries), ctx, arg) +} + // GetStatsTopAdvertisers mocks base method. func (m *MockQuerier) GetStatsTopAdvertisers(ctx context.Context, arg db.GetStatsTopAdvertisersParams) ([]db.GetStatsTopAdvertisersRow, error) { m.ctrl.T.Helper() @@ -993,10 +1093,10 @@ func (mr *MockQuerierMockRecorder) GetStatsTopTalkers(ctx, arg any) *gomock.Call } // GetTopNodes mocks base method. -func (m *MockQuerier) GetTopNodes(ctx context.Context, arg db.GetTopNodesParams) ([]db.MvTopNodesByIatum, error) { +func (m *MockQuerier) GetTopNodes(ctx context.Context, arg db.GetTopNodesParams) ([]db.GetTopNodesRow, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetTopNodes", ctx, arg) - ret0, _ := ret[0].([]db.MvTopNodesByIatum) + ret0, _ := ret[0].([]db.GetTopNodesRow) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -1053,10 +1153,10 @@ func (mr *MockQuerierMockRecorder) InsertChannelMessage(ctx, arg any) *gomock.Ca } // InsertObservation mocks base method. -func (m *MockQuerier) InsertObservation(ctx context.Context, arg db.InsertObservationParams) (db.PacketObservation, error) { +func (m *MockQuerier) InsertObservation(ctx context.Context, arg db.InsertObservationParams) (db.InsertObservationRow, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "InsertObservation", ctx, arg) - ret0, _ := ret[0].(db.PacketObservation) + ret0, _ := ret[0].(db.InsertObservationRow) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -1186,6 +1286,21 @@ func (mr *MockQuerierMockRecorder) ListChannelsAfter(ctx, arg any) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListChannelsAfter", reflect.TypeOf((*MockQuerier)(nil).ListChannelsAfter), ctx, arg) } +// ListDirtyRollupHours mocks base method. +func (m *MockQuerier) ListDirtyRollupHours(ctx context.Context, limit int32) ([]pgtype.Timestamptz, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListDirtyRollupHours", ctx, limit) + ret0, _ := ret[0].([]pgtype.Timestamptz) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListDirtyRollupHours indicates an expected call of ListDirtyRollupHours. +func (mr *MockQuerierMockRecorder) ListDirtyRollupHours(ctx, limit any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListDirtyRollupHours", reflect.TypeOf((*MockQuerier)(nil).ListDirtyRollupHours), ctx, limit) +} + // ListIATAs mocks base method. func (m *MockQuerier) ListIATAs(ctx context.Context) ([]db.IataCode, error) { m.ctrl.T.Helper() @@ -1231,6 +1346,21 @@ func (mr *MockQuerierMockRecorder) ListMessagesAfterID(ctx, arg any) *gomock.Cal return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListMessagesAfterID", reflect.TypeOf((*MockQuerier)(nil).ListMessagesAfterID), ctx, arg) } +// ListMissingRollupHours mocks base method. +func (m *MockQuerier) ListMissingRollupHours(ctx context.Context, limit int32) ([]pgtype.Timestamptz, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListMissingRollupHours", ctx, limit) + ret0, _ := ret[0].([]pgtype.Timestamptz) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListMissingRollupHours indicates an expected call of ListMissingRollupHours. +func (mr *MockQuerierMockRecorder) ListMissingRollupHours(ctx, limit any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListMissingRollupHours", reflect.TypeOf((*MockQuerier)(nil).ListMissingRollupHours), ctx, limit) +} + // ListNodeObservations mocks base method. func (m *MockQuerier) ListNodeObservations(ctx context.Context, arg db.ListNodeObservationsParams) ([]db.ListNodeObservationsRow, error) { m.ctrl.T.Helper() @@ -1486,6 +1616,50 @@ func (mr *MockQuerierMockRecorder) ListZoneLists(ctx any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListZoneLists", reflect.TypeOf((*MockQuerier)(nil).ListZoneLists), ctx) } +// LockRollupHour mocks base method. +func (m *MockQuerier) LockRollupHour(ctx context.Context, hour pgtype.Timestamptz) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "LockRollupHour", ctx, hour) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// LockRollupHour indicates an expected call of LockRollupHour. +func (mr *MockQuerierMockRecorder) LockRollupHour(ctx, hour any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LockRollupHour", reflect.TypeOf((*MockQuerier)(nil).LockRollupHour), ctx, hour) +} + +// MarkPartialRollupHours mocks base method. +func (m *MockQuerier) MarkPartialRollupHours(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MarkPartialRollupHours", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// MarkPartialRollupHours indicates an expected call of MarkPartialRollupHours. +func (mr *MockQuerierMockRecorder) MarkPartialRollupHours(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MarkPartialRollupHours", reflect.TypeOf((*MockQuerier)(nil).MarkPartialRollupHours), ctx) +} + +// OldestMissingRollupHour mocks base method. +func (m *MockQuerier) OldestMissingRollupHour(ctx context.Context) (pgtype.Timestamptz, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "OldestMissingRollupHour", ctx) + ret0, _ := ret[0].(pgtype.Timestamptz) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// OldestMissingRollupHour indicates an expected call of OldestMissingRollupHour. +func (mr *MockQuerierMockRecorder) OldestMissingRollupHour(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OldestMissingRollupHour", reflect.TypeOf((*MockQuerier)(nil).OldestMissingRollupHour), ctx) +} + // PruneImportedRegions mocks base method. func (m *MockQuerier) PruneImportedRegions(ctx context.Context, keep []string) ([]string, error) { m.ctrl.T.Helper() @@ -1545,60 +1719,18 @@ func (mr *MockQuerierMockRecorder) ReconfirmRoutes(ctx, arg any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReconfirmRoutes", reflect.TypeOf((*MockQuerier)(nil).ReconfirmRoutes), ctx, arg) } -// RefreshHourlyStats mocks base method. -func (m *MockQuerier) RefreshHourlyStats(ctx context.Context) error { +// RecordTrace mocks base method. +func (m *MockQuerier) RecordTrace(ctx context.Context, arg db.RecordTraceParams) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RefreshHourlyStats", ctx) + ret := m.ctrl.Call(m, "RecordTrace", ctx, arg) ret0, _ := ret[0].(error) return ret0 } -// RefreshHourlyStats indicates an expected call of RefreshHourlyStats. -func (mr *MockQuerierMockRecorder) RefreshHourlyStats(ctx any) *gomock.Call { +// RecordTrace indicates an expected call of RecordTrace. +func (mr *MockQuerierMockRecorder) RecordTrace(ctx, arg any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshHourlyStats", reflect.TypeOf((*MockQuerier)(nil).RefreshHourlyStats), ctx) -} - -// RefreshObserverActivity mocks base method. -func (m *MockQuerier) RefreshObserverActivity(ctx context.Context) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RefreshObserverActivity", ctx) - ret0, _ := ret[0].(error) - return ret0 -} - -// RefreshObserverActivity indicates an expected call of RefreshObserverActivity. -func (mr *MockQuerierMockRecorder) RefreshObserverActivity(ctx any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshObserverActivity", reflect.TypeOf((*MockQuerier)(nil).RefreshObserverActivity), ctx) -} - -// RefreshPathStats mocks base method. -func (m *MockQuerier) RefreshPathStats(ctx context.Context) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RefreshPathStats", ctx) - ret0, _ := ret[0].(error) - return ret0 -} - -// RefreshPathStats indicates an expected call of RefreshPathStats. -func (mr *MockQuerierMockRecorder) RefreshPathStats(ctx any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshPathStats", reflect.TypeOf((*MockQuerier)(nil).RefreshPathStats), ctx) -} - -// RefreshPayloadBreakdown mocks base method. -func (m *MockQuerier) RefreshPayloadBreakdown(ctx context.Context) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RefreshPayloadBreakdown", ctx) - ret0, _ := ret[0].(error) - return ret0 -} - -// RefreshPayloadBreakdown indicates an expected call of RefreshPayloadBreakdown. -func (mr *MockQuerierMockRecorder) RefreshPayloadBreakdown(ctx any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshPayloadBreakdown", reflect.TypeOf((*MockQuerier)(nil).RefreshPayloadBreakdown), ctx) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RecordTrace", reflect.TypeOf((*MockQuerier)(nil).RecordTrace), ctx, arg) } // RefreshRadioPresets mocks base method. @@ -1615,74 +1747,18 @@ func (mr *MockQuerierMockRecorder) RefreshRadioPresets(ctx any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshRadioPresets", reflect.TypeOf((*MockQuerier)(nil).RefreshRadioPresets), ctx) } -// RefreshSignalStats mocks base method. -func (m *MockQuerier) RefreshSignalStats(ctx context.Context) error { +// RegisterRollupHours mocks base method. +func (m *MockQuerier) RegisterRollupHours(ctx context.Context, since pgtype.Timestamptz) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RefreshSignalStats", ctx) + ret := m.ctrl.Call(m, "RegisterRollupHours", ctx, since) ret0, _ := ret[0].(error) return ret0 } -// RefreshSignalStats indicates an expected call of RefreshSignalStats. -func (mr *MockQuerierMockRecorder) RefreshSignalStats(ctx any) *gomock.Call { +// RegisterRollupHours indicates an expected call of RegisterRollupHours. +func (mr *MockQuerierMockRecorder) RegisterRollupHours(ctx, since any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshSignalStats", reflect.TypeOf((*MockQuerier)(nil).RefreshSignalStats), ctx) -} - -// RefreshTopAdvertisers mocks base method. -func (m *MockQuerier) RefreshTopAdvertisers(ctx context.Context) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RefreshTopAdvertisers", ctx) - ret0, _ := ret[0].(error) - return ret0 -} - -// RefreshTopAdvertisers indicates an expected call of RefreshTopAdvertisers. -func (mr *MockQuerierMockRecorder) RefreshTopAdvertisers(ctx any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshTopAdvertisers", reflect.TypeOf((*MockQuerier)(nil).RefreshTopAdvertisers), ctx) -} - -// RefreshTopNodes mocks base method. -func (m *MockQuerier) RefreshTopNodes(ctx context.Context) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RefreshTopNodes", ctx) - ret0, _ := ret[0].(error) - return ret0 -} - -// RefreshTopNodes indicates an expected call of RefreshTopNodes. -func (mr *MockQuerierMockRecorder) RefreshTopNodes(ctx any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshTopNodes", reflect.TypeOf((*MockQuerier)(nil).RefreshTopNodes), ctx) -} - -// RefreshTopObservers mocks base method. -func (m *MockQuerier) RefreshTopObservers(ctx context.Context) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RefreshTopObservers", ctx) - ret0, _ := ret[0].(error) - return ret0 -} - -// RefreshTopObservers indicates an expected call of RefreshTopObservers. -func (mr *MockQuerierMockRecorder) RefreshTopObservers(ctx any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshTopObservers", reflect.TypeOf((*MockQuerier)(nil).RefreshTopObservers), ctx) -} - -// RefreshTopTalkers mocks base method. -func (m *MockQuerier) RefreshTopTalkers(ctx context.Context) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RefreshTopTalkers", ctx) - ret0, _ := ret[0].(error) - return ret0 -} - -// RefreshTopTalkers indicates an expected call of RefreshTopTalkers. -func (mr *MockQuerierMockRecorder) RefreshTopTalkers(ctx any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshTopTalkers", reflect.TypeOf((*MockQuerier)(nil).RefreshTopTalkers), ctx) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RegisterRollupHours", reflect.TypeOf((*MockQuerier)(nil).RegisterRollupHours), ctx, since) } // ResolveEndpointHashPairs mocks base method. @@ -1775,6 +1851,189 @@ func (mr *MockQuerierMockRecorder) ResolvePathHashesP4(ctx, arg any) *gomock.Cal return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResolvePathHashesP4", reflect.TypeOf((*MockQuerier)(nil).ResolvePathHashesP4), ctx, arg) } +// RollAdvertHearings mocks base method. +func (m *MockQuerier) RollAdvertHearings(ctx context.Context, hour pgtype.Timestamptz) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RollAdvertHearings", ctx, hour) + ret0, _ := ret[0].(error) + return ret0 +} + +// RollAdvertHearings indicates an expected call of RollAdvertHearings. +func (mr *MockQuerierMockRecorder) RollAdvertHearings(ctx, hour any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RollAdvertHearings", reflect.TypeOf((*MockQuerier)(nil).RollAdvertHearings), ctx, hour) +} + +// RollAdvertSets mocks base method. +func (m *MockQuerier) RollAdvertSets(ctx context.Context, hour pgtype.Timestamptz) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RollAdvertSets", ctx, hour) + ret0, _ := ret[0].(error) + return ret0 +} + +// RollAdvertSets indicates an expected call of RollAdvertSets. +func (mr *MockQuerierMockRecorder) RollAdvertSets(ctx, hour any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RollAdvertSets", reflect.TypeOf((*MockQuerier)(nil).RollAdvertSets), ctx, hour) +} + +// RollIATAObservations mocks base method. +func (m *MockQuerier) RollIATAObservations(ctx context.Context, hour pgtype.Timestamptz) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RollIATAObservations", ctx, hour) + ret0, _ := ret[0].(error) + return ret0 +} + +// RollIATAObservations indicates an expected call of RollIATAObservations. +func (mr *MockQuerierMockRecorder) RollIATAObservations(ctx, hour any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RollIATAObservations", reflect.TypeOf((*MockQuerier)(nil).RollIATAObservations), ctx, hour) +} + +// RollObserverActivity mocks base method. +func (m *MockQuerier) RollObserverActivity(ctx context.Context, hour pgtype.Timestamptz) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RollObserverActivity", ctx, hour) + ret0, _ := ret[0].(error) + return ret0 +} + +// RollObserverActivity indicates an expected call of RollObserverActivity. +func (mr *MockQuerierMockRecorder) RollObserverActivity(ctx, hour any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RollObserverActivity", reflect.TypeOf((*MockQuerier)(nil).RollObserverActivity), ctx, hour) +} + +// RollObserverIdentity mocks base method. +func (m *MockQuerier) RollObserverIdentity(ctx context.Context, hour pgtype.Timestamptz) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RollObserverIdentity", ctx, hour) + ret0, _ := ret[0].(error) + return ret0 +} + +// RollObserverIdentity indicates an expected call of RollObserverIdentity. +func (mr *MockQuerierMockRecorder) RollObserverIdentity(ctx, hour any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RollObserverIdentity", reflect.TypeOf((*MockQuerier)(nil).RollObserverIdentity), ctx, hour) +} + +// RollPacketSets mocks base method. +func (m *MockQuerier) RollPacketSets(ctx context.Context, hour pgtype.Timestamptz) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RollPacketSets", ctx, hour) + ret0, _ := ret[0].(error) + return ret0 +} + +// RollPacketSets indicates an expected call of RollPacketSets. +func (mr *MockQuerierMockRecorder) RollPacketSets(ctx, hour any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RollPacketSets", reflect.TypeOf((*MockQuerier)(nil).RollPacketSets), ctx, hour) +} + +// RollPaths mocks base method. +func (m *MockQuerier) RollPaths(ctx context.Context, hour pgtype.Timestamptz) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RollPaths", ctx, hour) + ret0, _ := ret[0].(error) + return ret0 +} + +// RollPaths indicates an expected call of RollPaths. +func (mr *MockQuerierMockRecorder) RollPaths(ctx, hour any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RollPaths", reflect.TypeOf((*MockQuerier)(nil).RollPaths), ctx, hour) +} + +// RollPayloadBreakdown mocks base method. +func (m *MockQuerier) RollPayloadBreakdown(ctx context.Context, hour pgtype.Timestamptz) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RollPayloadBreakdown", ctx, hour) + ret0, _ := ret[0].(error) + return ret0 +} + +// RollPayloadBreakdown indicates an expected call of RollPayloadBreakdown. +func (mr *MockQuerierMockRecorder) RollPayloadBreakdown(ctx, hour any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RollPayloadBreakdown", reflect.TypeOf((*MockQuerier)(nil).RollPayloadBreakdown), ctx, hour) +} + +// RollScopeSets mocks base method. +func (m *MockQuerier) RollScopeSets(ctx context.Context, hour pgtype.Timestamptz) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RollScopeSets", ctx, hour) + ret0, _ := ret[0].(error) + return ret0 +} + +// RollScopeSets indicates an expected call of RollScopeSets. +func (mr *MockQuerierMockRecorder) RollScopeSets(ctx, hour any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RollScopeSets", reflect.TypeOf((*MockQuerier)(nil).RollScopeSets), ctx, hour) +} + +// RollSignal mocks base method. +func (m *MockQuerier) RollSignal(ctx context.Context, hour pgtype.Timestamptz) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RollSignal", ctx, hour) + ret0, _ := ret[0].(error) + return ret0 +} + +// RollSignal indicates an expected call of RollSignal. +func (mr *MockQuerierMockRecorder) RollSignal(ctx, hour any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RollSignal", reflect.TypeOf((*MockQuerier)(nil).RollSignal), ctx, hour) +} + +// RollTalkerSets mocks base method. +func (m *MockQuerier) RollTalkerSets(ctx context.Context, hour pgtype.Timestamptz) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RollTalkerSets", ctx, hour) + ret0, _ := ret[0].(error) + return ret0 +} + +// RollTalkerSets indicates an expected call of RollTalkerSets. +func (mr *MockQuerierMockRecorder) RollTalkerSets(ctx, hour any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RollTalkerSets", reflect.TypeOf((*MockQuerier)(nil).RollTalkerSets), ctx, hour) +} + +// RollupContentHash mocks base method. +func (m *MockQuerier) RollupContentHash(ctx context.Context, hour pgtype.Timestamptz) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RollupContentHash", ctx, hour) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// RollupContentHash indicates an expected call of RollupContentHash. +func (mr *MockQuerierMockRecorder) RollupContentHash(ctx, hour any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RollupContentHash", reflect.TypeOf((*MockQuerier)(nil).RollupContentHash), ctx, hour) +} + +// RollupUnlock mocks base method. +func (m *MockQuerier) RollupUnlock(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RollupUnlock", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// RollupUnlock indicates an expected call of RollupUnlock. +func (mr *MockQuerierMockRecorder) RollupUnlock(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RollupUnlock", reflect.TypeOf((*MockQuerier)(nil).RollupUnlock), ctx) +} + // SaveChannelCatalogue mocks base method. func (m *MockQuerier) SaveChannelCatalogue(ctx context.Context, arg db.SaveChannelCatalogueParams) error { m.ctrl.T.Helper() @@ -1902,6 +2161,20 @@ func (mr *MockQuerierMockRecorder) SetPacketDecrypted(ctx, packetHash any) *gomo return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetPacketDecrypted", reflect.TypeOf((*MockQuerier)(nil).SetPacketDecrypted), ctx, packetHash) } +// SetRollupHourPartial mocks base method. +func (m *MockQuerier) SetRollupHourPartial(ctx context.Context, hour pgtype.Timestamptz) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetRollupHourPartial", ctx, hour) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetRollupHourPartial indicates an expected call of SetRollupHourPartial. +func (mr *MockQuerierMockRecorder) SetRollupHourPartial(ctx, hour any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetRollupHourPartial", reflect.TypeOf((*MockQuerier)(nil).SetRollupHourPartial), ctx, hour) +} + // TouchObserverBrokers mocks base method. func (m *MockQuerier) TouchObserverBrokers(ctx context.Context, arg db.TouchObserverBrokersParams) error { m.ctrl.T.Helper() @@ -1944,6 +2217,21 @@ func (mr *MockQuerierMockRecorder) TouchPackets(ctx, arg any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TouchPackets", reflect.TypeOf((*MockQuerier)(nil).TouchPackets), ctx, arg) } +// TryRollupLock mocks base method. +func (m *MockQuerier) TryRollupLock(ctx context.Context) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "TryRollupLock", ctx) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// TryRollupLock indicates an expected call of TryRollupLock. +func (mr *MockQuerierMockRecorder) TryRollupLock(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TryRollupLock", reflect.TypeOf((*MockQuerier)(nil).TryRollupLock), ctx) +} + // UpdateObserverRegionScope mocks base method. func (m *MockQuerier) UpdateObserverRegionScope(ctx context.Context, arg db.UpdateObserverRegionScopeParams) error { m.ctrl.T.Helper() @@ -2146,18 +2434,18 @@ func (mr *MockQuerierMockRecorder) UpsertNodeShortID(ctx, arg any) *gomock.Call } // UpsertObserver mocks base method. -func (m *MockQuerier) UpsertObserver(ctx context.Context, publicKey []byte) (db.Observer, error) { +func (m *MockQuerier) UpsertObserver(ctx context.Context, arg db.UpsertObserverParams) (db.Observer, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UpsertObserver", ctx, publicKey) + ret := m.ctrl.Call(m, "UpsertObserver", ctx, arg) ret0, _ := ret[0].(db.Observer) ret1, _ := ret[1].(error) return ret0, ret1 } // UpsertObserver indicates an expected call of UpsertObserver. -func (mr *MockQuerierMockRecorder) UpsertObserver(ctx, publicKey any) *gomock.Call { +func (mr *MockQuerierMockRecorder) UpsertObserver(ctx, arg any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertObserver", reflect.TypeOf((*MockQuerier)(nil).UpsertObserver), ctx, publicKey) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertObserver", reflect.TypeOf((*MockQuerier)(nil).UpsertObserver), ctx, arg) } // UpsertObserverBroker mocks base method. @@ -2218,20 +2506,6 @@ func (mr *MockQuerierMockRecorder) UpsertRegion(ctx, arg any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertRegion", reflect.TypeOf((*MockQuerier)(nil).UpsertRegion), ctx, arg) } -// UpsertTraceIATA mocks base method. -func (m *MockQuerier) UpsertTraceIATA(ctx context.Context, arg db.UpsertTraceIATAParams) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UpsertTraceIATA", ctx, arg) - ret0, _ := ret[0].(error) - return ret0 -} - -// UpsertTraceIATA indicates an expected call of UpsertTraceIATA. -func (mr *MockQuerierMockRecorder) UpsertTraceIATA(ctx, arg any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertTraceIATA", reflect.TypeOf((*MockQuerier)(nil).UpsertTraceIATA), ctx, arg) -} - // UpsertTransportScope mocks base method. func (m *MockQuerier) UpsertTransportScope(ctx context.Context, arg db.UpsertTransportScopeParams) error { m.ctrl.T.Helper() diff --git a/db/sqlc/models.go b/db/sqlc/models.go index e08581c..709db8f 100644 --- a/db/sqlc/models.go +++ b/db/sqlc/models.go @@ -16,96 +16,40 @@ type Account struct { DeactivatedAt pgtype.Timestamptz `json:"deactivated_at"` } -type AnalyticsHourlyIataStat struct { - Iata string `json:"iata"` - Hour pgtype.Timestamptz `json:"hour"` - ObservationCount int64 `json:"observation_count"` - UniquePackets int64 `json:"unique_packets"` +type AnalyticsDirtyHour struct { + Hour pgtype.Timestamptz `json:"hour"` + EnqueuedAt pgtype.Timestamptz `json:"enqueued_at"` } -type AnalyticsLiveHourlyIataStat struct { - Iata string `json:"iata"` - Hour pgtype.Timestamptz `json:"hour"` - ObservationCount int64 `json:"observation_count"` - UniquePackets int64 `json:"unique_packets"` -} - -type AnalyticsLiveObserverActivityHourly struct { - ObserverID uuid.UUID `json:"observer_id"` - PayloadType int16 `json:"payload_type"` - Bucket pgtype.Timestamptz `json:"bucket"` - Observations int64 `json:"observations"` - AirtimeMs float32 `json:"airtime_ms"` - AirtimeN int64 `json:"airtime_n"` - SnrSum float32 `json:"snr_sum"` - SnrN int64 `json:"snr_n"` - SnrMin float32 `json:"snr_min"` - RssiSum int64 `json:"rssi_sum"` - RssiN int64 `json:"rssi_n"` -} - -type AnalyticsLivePathStatsHourly struct { - Iata string `json:"iata"` - Hour interface{} `json:"hour"` - Category int32 `json:"category"` - HashBytes int32 `json:"hash_bytes"` - Entries int32 `json:"entries"` - Receptions int64 `json:"receptions"` -} - -type AnalyticsLivePayloadBreakdownByIatum struct { - Iata string `json:"iata"` - PayloadType *int16 `json:"payload_type"` - Bucket pgtype.Timestamptz `json:"bucket"` - Count int64 `json:"count"` -} - -type AnalyticsLiveSignalStatsHourly struct { - Iata string `json:"iata"` - Hour interface{} `json:"hour"` - Kind int32 `json:"kind"` - SnrBin int32 `json:"snr_bin"` - RssiBin int32 `json:"rssi_bin"` - Receptions int64 `json:"receptions"` - SnrSamples int64 `json:"snr_samples"` - SnrSum float64 `json:"snr_sum"` - RssiSamples int64 `json:"rssi_samples"` - RssiSum float64 `json:"rssi_sum"` -} - -type AnalyticsLiveTopAdvertisersByIatum struct { - Iata string `json:"iata"` - NodeID uuid.UUID `json:"node_id"` - Name *string `json:"name"` - NodeType int16 `json:"node_type"` - Bucket pgtype.Timestamptz `json:"bucket"` - AdvertCount int64 `json:"advert_count"` - FloodAdvertCount int64 `json:"flood_advert_count"` - DirectAdvertCount int64 `json:"direct_advert_count"` - LastHeard interface{} `json:"last_heard"` -} - -type AnalyticsLiveTopObserversByIatum struct { - Iata string `json:"iata"` - ObserverID uuid.UUID `json:"observer_id"` - DisplayName *string `json:"display_name"` - ObserverType *string `json:"observer_type"` - Bucket pgtype.Timestamptz `json:"bucket"` - ObservationCount int64 `json:"observation_count"` -} - -type AnalyticsLiveTopTalkersByIatum struct { +type AnalyticsHourlyAdvertHearing struct { + Hour pgtype.Timestamptz `json:"hour"` Iata string `json:"iata"` - SenderName *string `json:"sender_name"` - Bucket pgtype.Timestamptz `json:"bucket"` - MessageCount int64 `json:"message_count"` - LastSent interface{} `json:"last_sent"` + OriginPubkey []byte `json:"origin_pubkey"` + Observations int64 `json:"observations"` + LastHeard pgtype.Timestamptz `json:"last_heard"` + Name *string `json:"name"` + NodeType *int16 `json:"node_type"` } -type AnalyticsObserverActivityHourly struct { +type AnalyticsHourlyAdvertSet struct { + Hour pgtype.Timestamptz `json:"hour"` + OriginPubkey []byte `json:"origin_pubkey"` + Iatas []string `json:"iatas"` + AdvertPackets int64 `json:"advert_packets"` + FloodPackets int64 `json:"flood_packets"` + DirectPackets int64 `json:"direct_packets"` +} + +type AnalyticsHourlyIataObservation struct { + Hour pgtype.Timestamptz `json:"hour"` + Iata string `json:"iata"` + ObservationCount int64 `json:"observation_count"` +} + +type AnalyticsHourlyObserverActivity struct { + Hour pgtype.Timestamptz `json:"hour"` ObserverID uuid.UUID `json:"observer_id"` PayloadType int16 `json:"payload_type"` - Bucket pgtype.Timestamptz `json:"bucket"` Observations int64 `json:"observations"` AirtimeMs *float32 `json:"airtime_ms"` AirtimeN int64 `json:"airtime_n"` @@ -116,62 +60,81 @@ type AnalyticsObserverActivityHourly struct { RssiN int64 `json:"rssi_n"` } -type AnalyticsPathStatsHourly struct { - Iata string `json:"iata"` +type AnalyticsHourlyObserverIdentity struct { + Hour pgtype.Timestamptz `json:"hour"` + Iata string `json:"iata"` + ObserverID uuid.UUID `json:"observer_id"` + ObservationCount int64 `json:"observation_count"` + DisplayName *string `json:"display_name"` + ObserverType *string `json:"observer_type"` +} + +type AnalyticsHourlyPacketSet struct { + Hour pgtype.Timestamptz `json:"hour"` + Iatas []string `json:"iatas"` + Packets int64 `json:"packets"` +} + +type AnalyticsHourlyPath struct { Hour pgtype.Timestamptz `json:"hour"` + Iata string `json:"iata"` Category int32 `json:"category"` HashBytes int32 `json:"hash_bytes"` Entries int32 `json:"entries"` Receptions int64 `json:"receptions"` } -type AnalyticsPayloadBreakdownByIatum struct { +type AnalyticsHourlyPayloadBreakdown struct { + Hour pgtype.Timestamptz `json:"hour"` Iata string `json:"iata"` PayloadType int16 `json:"payload_type"` - Bucket pgtype.Timestamptz `json:"bucket"` Count int64 `json:"count"` } -type AnalyticsSignalStatsHourly struct { - Iata string `json:"iata"` +type AnalyticsHourlyScopeSet struct { + Hour pgtype.Timestamptz `json:"hour"` + Iatas []string `json:"iatas"` + ScopeID int32 `json:"scope_id"` + Packets int64 `json:"packets"` +} + +type AnalyticsHourlySignal struct { Hour pgtype.Timestamptz `json:"hour"` + Iata string `json:"iata"` Kind int32 `json:"kind"` SnrBin int32 `json:"snr_bin"` RssiBin int32 `json:"rssi_bin"` Receptions int64 `json:"receptions"` SnrSamples int64 `json:"snr_samples"` - SnrSum *float64 `json:"snr_sum"` + SnrSum float64 `json:"snr_sum"` RssiSamples int64 `json:"rssi_samples"` - RssiSum *float64 `json:"rssi_sum"` + RssiSum float64 `json:"rssi_sum"` } -type AnalyticsTopAdvertisersByIatum struct { - Iata string `json:"iata"` - NodeID uuid.UUID `json:"node_id"` - Bucket pgtype.Timestamptz `json:"bucket"` - AdvertCount int64 `json:"advert_count"` - FloodAdvertCount int64 `json:"flood_advert_count"` - DirectAdvertCount int64 `json:"direct_advert_count"` - LastHeard pgtype.Timestamptz `json:"last_heard"` - Name *string `json:"name"` - NodeType *int16 `json:"node_type"` +type AnalyticsHourlyTalkerSet struct { + Hour pgtype.Timestamptz `json:"hour"` + SenderName string `json:"sender_name"` + Iatas []string `json:"iatas"` + Messages int64 `json:"messages"` + LastSent pgtype.Timestamptz `json:"last_sent"` } -type AnalyticsTopObserversByIatum struct { - Iata string `json:"iata"` - ObserverID uuid.UUID `json:"observer_id"` - Bucket pgtype.Timestamptz `json:"bucket"` - ObservationCount int64 `json:"observation_count"` - DisplayName *string `json:"display_name"` - ObserverType *string `json:"observer_type"` +type AnalyticsRawState struct { + Singleton bool `json:"singleton"` + RawDeletedBefore pgtype.Timestamptz `json:"raw_deleted_before"` + Coverage int64 `json:"coverage"` } -type AnalyticsTopTalkersByIatum struct { - Iata string `json:"iata"` - SenderName string `json:"sender_name"` - Bucket pgtype.Timestamptz `json:"bucket"` - MessageCount int64 `json:"message_count"` - LastSent pgtype.Timestamptz `json:"last_sent"` +type AnalyticsRollupHour struct { + Hour pgtype.Timestamptz `json:"hour"` + Status string `json:"status"` + RolledAt pgtype.Timestamptz `json:"rolled_at"` + ContentHash *string `json:"content_hash"` +} + +type AnalyticsState struct { + Singleton bool `json:"singleton"` + Revision int64 `json:"revision"` } type Channel struct { @@ -185,7 +148,7 @@ type Channel struct { KeyKnown *bool `json:"key_known"` FirstSeen pgtype.Timestamptz `json:"first_seen"` LastSeen pgtype.Timestamptz `json:"last_seen"` - MessageCount *int64 `json:"message_count"` + MessageCount int64 `json:"message_count"` } type ChannelConfigScope struct { @@ -287,100 +250,11 @@ type MeshmapperZoneList struct { LastError string `json:"last_error"` } -type MvHourlyIataStat struct { - Iata string `json:"iata"` - Hour pgtype.Timestamptz `json:"hour"` - ObservationCount int64 `json:"observation_count"` - UniquePackets int64 `json:"unique_packets"` - ActiveObservers int64 `json:"active_observers"` -} - -type MvObserverActivityHourly struct { - ObserverID uuid.UUID `json:"observer_id"` - PayloadType int16 `json:"payload_type"` - Bucket pgtype.Timestamptz `json:"bucket"` - Observations int64 `json:"observations"` - AirtimeMs float32 `json:"airtime_ms"` - AirtimeN int64 `json:"airtime_n"` - SnrSum float32 `json:"snr_sum"` - SnrN int64 `json:"snr_n"` - SnrMin float32 `json:"snr_min"` - RssiSum int64 `json:"rssi_sum"` - RssiN int64 `json:"rssi_n"` -} - -type MvPathStatsHourly struct { - Iata string `json:"iata"` - Hour interface{} `json:"hour"` - Category int32 `json:"category"` - HashBytes int32 `json:"hash_bytes"` - Entries int32 `json:"entries"` - Receptions int64 `json:"receptions"` -} - -type MvPayloadBreakdownByIatum struct { - Iata string `json:"iata"` - PayloadType *int16 `json:"payload_type"` - Bucket pgtype.Timestamptz `json:"bucket"` - Count int64 `json:"count"` -} - type MvRadioPreset struct { - Preset string `json:"preset"` - Iata string `json:"iata"` - SourceType string `json:"source_type"` - Count int64 `json:"count"` -} - -type MvSignalStatsHourly struct { - Iata string `json:"iata"` - Hour interface{} `json:"hour"` - Kind int32 `json:"kind"` - SnrBin int32 `json:"snr_bin"` - RssiBin int32 `json:"rssi_bin"` - Receptions int64 `json:"receptions"` - SnrSamples int64 `json:"snr_samples"` - SnrSum float64 `json:"snr_sum"` - RssiSamples int64 `json:"rssi_samples"` - RssiSum float64 `json:"rssi_sum"` -} - -type MvTopAdvertisersByIatum struct { - Iata string `json:"iata"` - NodeID uuid.UUID `json:"node_id"` - Name *string `json:"name"` - NodeType int16 `json:"node_type"` - Bucket pgtype.Timestamptz `json:"bucket"` - AdvertCount int64 `json:"advert_count"` - FloodAdvertCount int64 `json:"flood_advert_count"` - DirectAdvertCount int64 `json:"direct_advert_count"` - LastHeard interface{} `json:"last_heard"` -} - -type MvTopNodesByIatum struct { - Iata string `json:"iata"` - NodeID uuid.UUID `json:"node_id"` - Name *string `json:"name"` - NodeType int16 `json:"node_type"` - ObservationCount *int64 `json:"observation_count"` - LastHeard pgtype.Timestamptz `json:"last_heard"` -} - -type MvTopObserversByIatum struct { - Iata string `json:"iata"` - ObserverID uuid.UUID `json:"observer_id"` - DisplayName *string `json:"display_name"` - ObserverType *string `json:"observer_type"` - Bucket pgtype.Timestamptz `json:"bucket"` - ObservationCount int64 `json:"observation_count"` -} - -type MvTopTalkersByIatum struct { - Iata string `json:"iata"` - SenderName *string `json:"sender_name"` - Bucket pgtype.Timestamptz `json:"bucket"` - MessageCount int64 `json:"message_count"` - LastSent interface{} `json:"last_sent"` + Preset string `json:"preset"` + Iata *string `json:"iata"` + SourceType string `json:"source_type"` + Count int64 `json:"count"` } type Node struct { @@ -455,6 +329,8 @@ type Observer struct { ObservationCount *int64 `json:"observation_count"` Metadata []byte `json:"metadata"` RegionScope *string `json:"region_scope"` + LastIata *string `json:"last_iata"` + LastIataAt pgtype.Timestamptz `json:"last_iata_at"` } type ObserverBroker struct { @@ -525,6 +401,7 @@ type Packet struct { TraceTag []byte `json:"trace_tag"` FirstHeardAt pgtype.Timestamptz `json:"first_heard_at"` LastHeardAt pgtype.Timestamptz `json:"last_heard_at"` + ObservationCount int64 `json:"observation_count"` } type PacketObservation struct { @@ -569,12 +446,42 @@ type RegionIata struct { AddedAt pgtype.Timestamptz `json:"added_at"` } +type RollupOb struct { + PacketHash []byte `json:"packet_hash"` + ObserverID uuid.UUID `json:"observer_id"` + Iata string `json:"iata"` + HeardAt pgtype.Timestamptz `json:"heard_at"` + PathLengthByte int16 `json:"path_length_byte"` + HashSize int16 `json:"hash_size"` + HopCount int16 `json:"hop_count"` + PathLen int32 `json:"path_len"` + Rssi *int16 `json:"rssi"` + Snr *float32 `json:"snr"` + AirtimeMs *float32 `json:"airtime_ms"` + PayloadType *int16 `json:"payload_type"` + RouteType int16 `json:"route_type"` + OriginPubkey []byte `json:"origin_pubkey"` + ScopeID *int32 `json:"scope_id"` +} + type TraceIata struct { TraceTag []byte `json:"trace_tag"` Iata string `json:"iata"` LastHeard pgtype.Timestamptz `json:"last_heard"` } +type TraceTag struct { + TraceTag []byte `json:"trace_tag"` + FirstHeardAt pgtype.Timestamptz `json:"first_heard_at"` + LastHeardAt pgtype.Timestamptz `json:"last_heard_at"` + PacketCount int64 `json:"packet_count"` + TraceType *string `json:"trace_type"` + ScopeID *int32 `json:"scope_id"` + BestPayload []byte `json:"best_payload"` + BestPathLen int32 `json:"best_path_len"` + Heard bool `json:"heard"` +} + type TransportScope struct { ID int32 `json:"id"` Name string `json:"name"` diff --git a/db/sqlc/paths.sql.go b/db/sqlc/paths.sql.go index 05bf5b1..f5c24f1 100644 --- a/db/sqlc/paths.sql.go +++ b/db/sqlc/paths.sql.go @@ -13,11 +13,11 @@ import ( const getPathStats = `-- name: GetPathStats :many WITH readings AS ( - SELECT iata, hour, category, hash_bytes, entries, receptions FROM mv_path_stats_hourly + SELECT iata, hour, category, hash_bytes, entries, receptions FROM analytics_hourly_paths WHERE hour >= $1::timestamptz AND hour < $2::timestamptz AND COALESCE(cardinality($3::bpchar[]), 0) = 0 UNION ALL - SELECT iata, hour, category, hash_bytes, entries, receptions FROM mv_path_stats_hourly + SELECT iata, hour, category, hash_bytes, entries, receptions FROM analytics_hourly_paths WHERE hour >= $1::timestamptz AND hour < $2::timestamptz AND cardinality($3::bpchar[]) > 0 AND iata = ANY($3::bpchar[]) ) @@ -45,7 +45,7 @@ type GetPathStatsRow struct { Receptions int64 `json:"receptions"` } -// All HTTP aggregates read the compact classification snapshot. +// All HTTP aggregates read the hourly rollup, never observations. func (q *Queries) GetPathStats(ctx context.Context, arg GetPathStatsParams) ([]GetPathStatsRow, error) { rows, err := q.db.Query(ctx, getPathStats, arg.Since, arg.Until, arg.Iatas) if err != nil { @@ -72,12 +72,3 @@ func (q *Queries) GetPathStats(ctx context.Context, arg GetPathStatsParams) ([]G } return items, nil } - -const refreshPathStats = `-- name: RefreshPathStats :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_path_stats_hourly -` - -func (q *Queries) RefreshPathStats(ctx context.Context) error { - _, err := q.db.Exec(ctx, refreshPathStats) - return err -} diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index c2adf52..1eaf656 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -40,8 +40,11 @@ type Querier interface { // Opt-in age-out: preserve retained history and manually recorded ownership. // Bound deletions per cleanup tick and skip observers being updated by ingest. DeleteOldObservers(ctx context.Context, lastSeen pgtype.Timestamptz) ([]uuid.UUID, error) - // Locks, archives and cascades one bounded packet cohort in a single transaction. + // Deletes one bounded cohort (observations cascade). SKIP LOCKED leaves packets an in-flight + // observation insert holds. raw_deleted_before tells the rollup which hours lost raw rows. DeleteOldPackets(ctx context.Context, arg DeleteOldPacketsParams) (int64, error) + // Bumps coverage when hours go, so cached responses that included them are not reused. + DeleteOldRollups(ctx context.Context, cutoff pgtype.Timestamptz) error // One batch of routes past retention, or past grace with too few observations. // GREATEST keeps the scan on idx_known_routes_last_seen. DeleteOldRoutes(ctx context.Context, arg DeleteOldRoutesParams) (int64, error) @@ -49,12 +52,25 @@ type Querier interface { DeleteOldTelemetry(ctx context.Context, reportedAt pgtype.Timestamptz) error // Keeps the trace IATA filter in step with packet retention. DeleteOldTraceIATAs(ctx context.Context, lastHeard pgtype.Timestamptz) error + DeleteOldTraceTags(ctx context.Context, lastHeardAt pgtype.Timestamptz) error DeleteRegionIATAsNotIn(ctx context.Context, arg DeleteRegionIATAsNotInParams) error + DeleteRollupHour(ctx context.Context, hour pgtype.Timestamptz) error + // Dirty hours that can no longer be re-rolled. + DeleteStaleDirtyHours(ctx context.Context) error + FillRollupObs(ctx context.Context, dollar_1 pgtype.Timestamptz) error + // Marks the hour complete and bumps the revision only when its content changed. + FinishRollupHour(ctx context.Context, arg FinishRollupHourParams) (bool, error) GetAccount(ctx context.Context, id uuid.UUID) (Account, error) + // Both counters only grow, so their sum changes whenever rolled content or coverage does. + GetAnalyticsRevision(ctx context.Context) (int64, error) GetChannelByID(ctx context.Context, id int32) (Channel, error) // Returns neighbors of a node that are in a different IATA. GetCrossIATANeighbors(ctx context.Context, arg GetCrossIATANeighborsParams) ([]GetCrossIATANeighborsRow, error) - GetHourlyStats(ctx context.Context, arg GetHourlyStatsParams) ([]MvHourlyIataStat, error) + GetEarliestCompleteRollupHour(ctx context.Context) (pgtype.Timestamptz, error) + // ============================================================ + // STATS + // ============================================================ + GetHourlyStats(ctx context.Context, arg GetHourlyStatsParams) ([]GetHourlyStatsRow, error) GetIATA(ctx context.Context, iata string) (IataCode, error) // An imported MeshMapper boundary overrides the configured one. border is NULL // when neither exists; a missing row (unknown IATA) is sql.ErrNoRows, same @@ -83,30 +99,30 @@ type Querier interface { // Count packet identities, not reception rows: a broker repeat or another // reception by the same observer must not inflate the comparison. GetObserverComparison(ctx context.Context, arg GetObserverComparisonParams) (GetObserverComparisonRow, error) - GetObserverLastIATA(ctx context.Context, observerID uuid.UUID) (string, error) + GetObserverLastIATA(ctx context.Context, id uuid.UUID) (string, error) GetObserverRadio(ctx context.Context, id uuid.UUID) (GetObserverRadioRow, error) GetObserverScopes(ctx context.Context, observerID uuid.UUID) ([]string, error) GetObserverTelemetry(ctx context.Context, arg GetObserverTelemetryParams) ([]GetObserverTelemetryRow, error) GetPacketByHash(ctx context.Context, packetHash []byte) (GetPacketByHashRow, error) - GetPacketObservationCount(ctx context.Context, packetHash []byte) (int64, error) // Return distinct observation IATAs in first-heard order for path resolution, // without fetching full observations separately for every trace packet. GetPacketsByTraceTag(ctx context.Context, decode string) ([]GetPacketsByTraceTagRow, error) - // All HTTP aggregates read the compact classification snapshot. + // All HTTP aggregates read the hourly rollup, never observations. GetPathStats(ctx context.Context, arg GetPathStatsParams) ([]GetPathStatsRow, error) GetRadioPresets(ctx context.Context, arg GetRadioPresetsParams) ([]MvRadioPreset, error) GetRegion(ctx context.Context, id int32) (GetRegionRow, error) GetRegionBySlug(ctx context.Context, slug string) (GetRegionBySlugRow, error) GetRegionIATAs(ctx context.Context, regionID int32) ([]string, error) GetRouteEvidenceRoute(ctx context.Context, arg GetRouteEvidenceRouteParams) (KnownRoute, error) + // Packets and IATAs cover every retained rollup hour; IATAs are those that heard the + // scope's own packets. Observer and node counts are current memberships. GetScopeByName(ctx context.Context, name string) (GetScopeByNameRow, error) GetScopeCatalogue(ctx context.Context, arg GetScopeCatalogueParams) (MeshmapperScopeCatalogue, error) GetScopeNames(ctx context.Context) ([]string, error) - // Aggregate matching observations once, separately from node memberships to avoid - // a cross-join. Empty IATAs keep the original global counts, including associations - // whose observations have expired; the filtered aggregates are empty in that case. - GetScopeStats(ctx context.Context, iatas []string) ([]GetScopeStatsRow, error) - // Read compact hourly snapshots, never observations on an HTTP request. + // Packets since the given hour come from the IATA-set rollup (counted once per hour heard). + // Observer and node counts are current memberships; observers filter by their latest IATA. + GetScopeStats(ctx context.Context, arg GetScopeStatsParams) ([]GetScopeStatsRow, error) + // Read the hourly rollup, never observations on an HTTP request. // Weight averages by sample counts instead of averaging regional/hourly means. GetSignalStats(ctx context.Context, arg GetSignalStatsParams) ([]GetSignalStatsRow, error) // Repeaters/room servers (node_type 2/3) whose current advert-derived clock drift exceeds @@ -116,21 +132,24 @@ type Querier interface { GetStatsClockDrift(ctx context.Context, arg GetStatsClockDriftParams) ([]GetStatsClockDriftRow, error) // Returns node counts grouped by type, optionally filtered by IATA. GetStatsNodeTypes(ctx context.Context, dollar_1 []string) ([]GetStatsNodeTypesRow, error) - // ============================================================ - // STATS - // ============================================================ - GetStatsOverview(ctx context.Context, dollar_1 []string) (GetStatsOverviewRow, error) - // Payload-type counts for the IATA within the window, summed from the - // precomputed hourly buckets. + // Payload-type observation counts since the given hour, from the hourly rollup. GetStatsPayloadBreakdown(ctx context.Context, arg GetStatsPayloadBreakdownParams) ([]GetStatsPayloadBreakdownRow, error) - // Top N advertisers in the window, summed from the hourly buckets. + // One row per hour in [since, until) plus a summary row (hour NULL) over its complete hours. + // Counts sum; observers, IATAs and scopes are distinct across the window; packets count + // once per hour heard. Each branch pair keeps "all IATAs" and "these IATAs" plans separate. + GetStatsSeries(ctx context.Context, arg GetStatsSeriesParams) ([]GetStatsSeriesRow, error) + // Top N advertisers since the given hour. Each ADVERT packet counts once per hour heard, + // however many of the requested IATAs heard it (IATA-set rollup). Live names win. GetStatsTopAdvertisers(ctx context.Context, arg GetStatsTopAdvertisersParams) ([]GetStatsTopAdvertisersRow, error) - // Top N observers for the IATA within the window, summed from the precomputed - // hourly buckets. Counts sum across matched IATAs; iata is a representative one. + // Top N observers since the given hour. Counts sum across matched IATAs; iata is a + // representative one. Live names win over the rolled snapshot. GetStatsTopObservers(ctx context.Context, arg GetStatsTopObserversParams) ([]GetStatsTopObserversRow, error) - // Top N talkers (by decrypted sender_name) in the window, summed from the hourly buckets. + // Top N senders since the given hour. Each message counts once per hour heard, however many + // of the requested IATAs heard it (IATA-set rollup). GetStatsTopTalkers(ctx context.Context, arg GetStatsTopTalkersParams) ([]GetStatsTopTalkersRow, error) - GetTopNodes(ctx context.Context, arg GetTopNodesParams) ([]MvTopNodesByIatum, error) + // Nodes by ADVERT hearings since the given hour. Live names win over the rolled snapshot; + // node_id is NULL once the node row is gone. iata is a representative one. + GetTopNodes(ctx context.Context, arg GetTopNodesParams) ([]GetTopNodesRow, error) GetTransportScopeByName(ctx context.Context, name string) (int32, error) GetTransportScopes(ctx context.Context) ([]GetTransportScopesRow, error) // ============================================================ @@ -138,11 +157,14 @@ type Querier interface { // ============================================================ // Read the immutable first-packet scope in the same statement as insertion. // A later reception's transport code must not give live and historical messages different tags. + // message_count is a lifetime count, bumped only for a new message. A historical (backfilled) + // message queues its observation hours for a re-roll in the same statement, so the two can't diverge. InsertChannelMessage(ctx context.Context, arg InsertChannelMessageParams) (InsertChannelMessageRow, error) // ============================================================ // PACKET OBSERVATIONS // ============================================================ - InsertObservation(ctx context.Context, arg InsertObservationParams) (PacketObservation, error) + // Bumps packets.observation_count only for a new row; a duplicate returns the current count. + InsertObservation(ctx context.Context, arg InsertObservationParams) (InsertObservationRow, error) // Inserts a telemetry snapshot for an observer. The reported_at timestamp should // be truncated to the configured resolution before calling to ensure deduplication. InsertObserverTelemetry(ctx context.Context, arg InsertObserverTelemetryParams) error @@ -170,6 +192,9 @@ type Querier interface { // Keep the non-null tuple boundary separate from the legacy optional cursor so // generic prepared plans can seek directly into the composite ordered index. ListChannelsAfter(ctx context.Context, arg ListChannelsAfterParams) ([]Channel, error) + // Only complete hours whose raw rows are still intact can be re-rolled. Queued missing hours + // wait: their first roll consumes the entry only if the entry predates its snapshot. + ListDirtyRollupHours(ctx context.Context, limit int32) ([]pgtype.Timestamptz, error) ListIATAs(ctx context.Context) ([]IataCode, error) // Only one branch runs. Keep the IATA range ordered by the composite index: // generic plans can otherwise prefer scanning the global timestamp index. @@ -178,6 +203,7 @@ type Querier interface { // Returns messages after the given message ID, ordered oldest first. // Used for WS reconnect backfill. ListMessagesAfterID(ctx context.Context, arg ListMessagesAfterIDParams) ([]ListMessagesAfterIDRow, error) + ListMissingRollupHours(ctx context.Context, limit int32) ([]pgtype.Timestamptz, error) ListNodeObservations(ctx context.Context, arg ListNodeObservationsParams) ([]ListNodeObservationsRow, error) // Limit the filtered node page before enriching IATA membership and neighbours. ListNodes(ctx context.Context, arg ListNodesParams) ([]ListNodesRow, error) @@ -224,9 +250,7 @@ type Querier interface { // ============================================================ // TRACES // ============================================================ - // Returns distinct trace tags with summary info, ordered by most recent first. - // IATA membership comes from trace_iatas (joining observations here spilled the - // hash join). Per-tag details filled in only for the returned page. + // Tags, most recent first. Filters match the tag summary (see RecordTrace). ListTraceTags(ctx context.Context, arg ListTraceTagsParams) ([]ListTraceTagsRow, error) // Returns GRP_TXT packets (payload_type=5) never successfully decrypted. Used at boot to // retry decryption against the current keystore for packets whose channel key was only added @@ -237,6 +261,12 @@ type Querier interface { ListUndecryptedGroupTextPacketsByHash(ctx context.Context, hashes [][]byte) ([]ListUndecryptedGroupTextPacketsByHashRow, error) ListZoneBoundaries(ctx context.Context) ([]MeshmapperZoneBoundary, error) ListZoneLists(ctx context.Context) ([]MeshmapperZoneList, error) + // Returns the hour's status, or 'partial' if raw deletion has reached it since registration. + LockRollupHour(ctx context.Context, hour pgtype.Timestamptz) (string, error) + // Hours whose raw rows cleanup already started deleting can never be rolled completely. + // Readers report the new status, so the revision moves too. + MarkPartialRollupHours(ctx context.Context) error + OldestMissingRollupHour(ctx context.Context) (pgtype.Timestamptz, error) PruneImportedRegions(ctx context.Context, keep []string) ([]string, error) // Drops imports for IATAs no longer configured, so their manual border returns. PruneZoneBoundaries(ctx context.Context, keep []string) ([]string, error) @@ -248,16 +278,15 @@ type Querier interface { // 1/2/3/4-byte hop prefixes check prefix_1/2/3/4; ambiguity set supplied by AmbiguousPrefixes), // and stamps the survivors. ReconfirmRoutes(ctx context.Context, arg ReconfirmRoutesParams) (int64, error) - RefreshHourlyStats(ctx context.Context) error - RefreshObserverActivity(ctx context.Context) error - RefreshPathStats(ctx context.Context) error - RefreshPayloadBreakdown(ctx context.Context) error + // One hearing of a TRACE packet: trace_iatas refreshes at most hourly and the tag's heard + // window widens (the first hearing replaces the packet's provisional times). Packet counts + // and payloads come from UpsertPacket. + RecordTrace(ctx context.Context, arg RecordTraceParams) error RefreshRadioPresets(ctx context.Context) error - RefreshSignalStats(ctx context.Context) error - RefreshTopAdvertisers(ctx context.Context) error - RefreshTopNodes(ctx context.Context) error - RefreshTopObservers(ctx context.Context) error - RefreshTopTalkers(ctx context.Context) error + // Adds a 'missing' row for every eligible hour since the earliest retained observation + // (and not before @since), zero-observation hours included. + // No observations yet means nothing to register (GREATEST would ignore the NULL MIN). + RegisterRollupHours(ctx context.Context, since pgtype.Timestamptz) error // Batch form of ResolveEndpointHashes for a page of packets. Matches the cross // product of IATAs and hashes; callers pick out the pairs they asked for. ResolveEndpointHashPairs(ctx context.Context, arg ResolveEndpointHashPairsParams) ([]ResolveEndpointHashPairsRow, error) @@ -276,6 +305,24 @@ type Querier interface { ResolvePathHashesP2(ctx context.Context, arg ResolvePathHashesP2Params) ([]ResolvePathHashesP2Row, error) ResolvePathHashesP3(ctx context.Context, arg ResolvePathHashesP3Params) ([]ResolvePathHashesP3Row, error) ResolvePathHashesP4(ctx context.Context, arg ResolvePathHashesP4Params) ([]ResolvePathHashesP4Row, error) + RollAdvertHearings(ctx context.Context, hour pgtype.Timestamptz) error + // Each distinct ADVERT packet counts once, under the exact set of IATAs that heard it this hour. + RollAdvertSets(ctx context.Context, hour pgtype.Timestamptz) error + RollIATAObservations(ctx context.Context, hour pgtype.Timestamptz) error + RollObserverActivity(ctx context.Context, hour pgtype.Timestamptz) error + RollObserverIdentity(ctx context.Context, hour pgtype.Timestamptz) error + RollPacketSets(ctx context.Context, hour pgtype.Timestamptz) error + // category: 0 = routed, 1 = zero-hop, 2 = TRACE, 3 = invalid (meshcore-go IsValidPathLen). + RollPaths(ctx context.Context, hour pgtype.Timestamptz) error + RollPayloadBreakdown(ctx context.Context, hour pgtype.Timestamptz) error + RollScopeSets(ctx context.Context, hour pgtype.Timestamptz) error + // kind = grouping(snr_bin, rssi_bin): 3 = per IATA totals, 1 = per SNR bin, 2 = per RSSI bin. + // rssi=0 AND snr=0 means "not reported". + RollSignal(ctx context.Context, hour pgtype.Timestamptz) error + RollTalkerSets(ctx context.Context, hour pgtype.Timestamptz) error + // Order-independent fingerprint of every family row for the hour. + RollupContentHash(ctx context.Context, hour pgtype.Timestamptz) (string, error) + RollupUnlock(ctx context.Context) error // NULL payload/etag/checked_at retain the last good list after an error or 304. SaveChannelCatalogue(ctx context.Context, arg SaveChannelCatalogueParams) error // One statement commits the validated snapshot and its lookup identities together. @@ -293,11 +340,17 @@ type Querier interface { SetNodeMultibytePaths(ctx context.Context, id uuid.UUID) error SetNodeMultibyteTraces(ctx context.Context, id uuid.UUID) error SetPacketDecrypted(ctx context.Context, packetHash []byte) error + SetRollupHourPartial(ctx context.Context, hour pgtype.Timestamptz) error TouchObserverBrokers(ctx context.Context, arg TouchObserverBrokersParams) error // Batched flush of coalesced presence bumps. GREATEST keeps a late flush // from regressing a newer write-through (e.g. a status update). TouchObservers(ctx context.Context, arg TouchObserversParams) error TouchPackets(ctx context.Context, arg TouchPacketsParams) error + // ============================================================ + // HOURLY ANALYTICS ROLLUPS + // An hour [H, H+1h) is eligible once now() >= H + 95 min (ingest clamps heard_at to ±30 min). + // ============================================================ + TryRollupLock(ctx context.Context) (bool, error) // Records the observer's own OTA-reported region scope, from the "self" // field of a /neighbors report. Always known (not queried OTA), so this // unconditionally overwrites, unlike the neighbor-side region_scope. @@ -356,7 +409,8 @@ type Querier interface { // ============================================================ // OBSERVERS // ============================================================ - UpsertObserver(ctx context.Context, publicKey []byte) (Observer, error) + // Empty iata (status/neighbors) leaves last_iata alone; otherwise the newest iata_at wins. + UpsertObserver(ctx context.Context, arg UpsertObserverParams) (Observer, error) // ============================================================ // OBSERVER BROKERS // ============================================================ @@ -365,10 +419,10 @@ type Querier interface { // ============================================================ // PACKETS // ============================================================ + // A new TRACE packet adds itself to its tag summary in the same statement, so the summary + // can't miss a stored packet. 'TRACE' > 'PING'; the longest path keeps the best payload. UpsertPacket(ctx context.Context, arg UpsertPacketParams) (UpsertPacketRow, error) UpsertRegion(ctx context.Context, arg UpsertRegionParams) (int32, error) - // Refreshes at most hourly so repeat hears don't churn the row. - UpsertTraceIATA(ctx context.Context, arg UpsertTraceIATAParams) error // ============================================================ // TRANSPORT CODES // ============================================================ diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 30db7f5..0a3b572 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -254,7 +254,21 @@ func (q *Queries) DeleteOldObservers(ctx context.Context, lastSeen pgtype.Timest } const deleteOldPackets = `-- name: DeleteOldPackets :one -SELECT archive_delete_packets($1::timestamptz, $2::integer)::bigint +WITH victims AS MATERIALIZED ( + SELECT vp.packet_hash, vp.last_heard_at FROM packets vp + WHERE vp.last_heard_at < $1::timestamptz + ORDER BY vp.last_heard_at, vp.packet_hash + LIMIT $2::integer + FOR UPDATE OF vp SKIP LOCKED +), deleted AS ( + DELETE FROM packets dp USING victims v WHERE dp.packet_hash = v.packet_hash + RETURNING dp.last_heard_at +), mark AS ( + UPDATE analytics_raw_state + SET raw_deleted_before = GREATEST(raw_deleted_before, (SELECT max(last_heard_at) FROM deleted)) + WHERE EXISTS (SELECT 1 FROM deleted) +) +SELECT count(*) FROM deleted ` type DeleteOldPacketsParams struct { @@ -262,12 +276,13 @@ type DeleteOldPacketsParams struct { BatchSize int32 `json:"batch_size"` } -// Locks, archives and cascades one bounded packet cohort in a single transaction. +// Deletes one bounded cohort (observations cascade). SKIP LOCKED leaves packets an in-flight +// observation insert holds. raw_deleted_before tells the rollup which hours lost raw rows. func (q *Queries) DeleteOldPackets(ctx context.Context, arg DeleteOldPacketsParams) (int64, error) { row := q.db.QueryRow(ctx, deleteOldPackets, arg.Cutoff, arg.BatchSize) - var column_1 int64 - err := row.Scan(&column_1) - return column_1, err + var count int64 + err := row.Scan(&count) + return count, err } const deleteOldRoutes = `-- name: DeleteOldRoutes :execrows @@ -326,6 +341,15 @@ func (q *Queries) DeleteOldTraceIATAs(ctx context.Context, lastHeard pgtype.Time return err } +const deleteOldTraceTags = `-- name: DeleteOldTraceTags :exec +DELETE FROM trace_tags WHERE last_heard_at < $1 +` + +func (q *Queries) DeleteOldTraceTags(ctx context.Context, lastHeardAt pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, deleteOldTraceTags, lastHeardAt) + return err +} + const deleteRegionIATAsNotIn = `-- name: DeleteRegionIATAsNotIn :exec DELETE FROM region_iatas WHERE region_id = $1 AND NOT (iata = ANY($2::bpchar[])) ` @@ -439,34 +463,38 @@ func (q *Queries) GetCrossIATANeighbors(ctx context.Context, arg GetCrossIATANei } const getHourlyStats = `-- name: GetHourlyStats :many -SELECT iata, hour, observation_count, unique_packets, active_observers -FROM mv_hourly_iata_stats + +SELECT iata, hour, observation_count +FROM analytics_hourly_iata_observations WHERE (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR iata = ANY($1::bpchar[])) - AND hour >= NOW() - $2::interval + AND hour >= $2::timestamptz ORDER BY iata, hour ` type GetHourlyStatsParams struct { - Column1 []string `json:"column_1"` - Column2 pgtype.Interval `json:"column_2"` + Iatas []string `json:"iatas"` + Since pgtype.Timestamptz `json:"since"` } -func (q *Queries) GetHourlyStats(ctx context.Context, arg GetHourlyStatsParams) ([]MvHourlyIataStat, error) { - rows, err := q.db.Query(ctx, getHourlyStats, arg.Column1, arg.Column2) +type GetHourlyStatsRow struct { + Iata string `json:"iata"` + Hour pgtype.Timestamptz `json:"hour"` + ObservationCount int64 `json:"observation_count"` +} + +// ============================================================ +// STATS +// ============================================================ +func (q *Queries) GetHourlyStats(ctx context.Context, arg GetHourlyStatsParams) ([]GetHourlyStatsRow, error) { + rows, err := q.db.Query(ctx, getHourlyStats, arg.Iatas, arg.Since) if err != nil { return nil, err } defer rows.Close() - items := []MvHourlyIataStat{} + items := []GetHourlyStatsRow{} for rows.Next() { - var i MvHourlyIataStat - if err := rows.Scan( - &i.Iata, - &i.Hour, - &i.ObservationCount, - &i.UniquePackets, - &i.ActiveObservers, - ); err != nil { + var i GetHourlyStatsRow + if err := rows.Scan(&i.Iata, &i.Hour, &i.ObservationCount); err != nil { return nil, err } items = append(items, i) @@ -786,7 +814,7 @@ func (q *Queries) GetNodesByPubkeys(ctx context.Context, pubkeys [][]byte) ([]Ge const getObserverActivityHourly = `-- name: GetObserverActivityHourly :many SELECT - date_bin($3::interval, bucket, TIMESTAMPTZ 'epoch')::timestamptz AS bucket, + date_bin($3::interval, hour, TIMESTAMPTZ 'epoch')::timestamptz AS bucket, SUM(observations)::bigint AS observations, COALESCE(SUM(airtime_ms), 0)::real AS airtime_ms, SUM(airtime_n)::bigint AS airtime_n, @@ -795,8 +823,8 @@ SELECT COALESCE(MIN(snr_min), 0)::real AS snr_min, COALESCE(SUM(rssi_sum), 0)::bigint AS rssi_sum, SUM(rssi_n)::bigint AS rssi_n -FROM mv_observer_activity_hourly -WHERE observer_id = $1 AND bucket >= $2::timestamptz AND bucket < $4::timestamptz +FROM analytics_hourly_observer_activity +WHERE observer_id = $1 AND hour >= $2::timestamptz AND hour < $4::timestamptz GROUP BY 1 ORDER BY 1 ` @@ -858,8 +886,8 @@ func (q *Queries) GetObserverActivityHourly(ctx context.Context, arg GetObserver const getObserverActivityHourlyPayloadTypes = `-- name: GetObserverActivityHourlyPayloadTypes :many SELECT payload_type, SUM(observations)::bigint AS count -FROM mv_observer_activity_hourly -WHERE observer_id = $1 AND bucket >= $2::timestamptz AND bucket < $3::timestamptz +FROM analytics_hourly_observer_activity +WHERE observer_id = $1 AND hour >= $2::timestamptz AND hour < $3::timestamptz GROUP BY payload_type ORDER BY count DESC ` @@ -1081,7 +1109,7 @@ func (q *Queries) GetObserverBrokers(ctx context.Context, observerID uuid.UUID) } const getObserverByID = `-- name: GetObserverByID :one -SELECT id, public_key, display_name, observer_type, software_version, hardware_model, firmware_version, firmware_build, radio_freq_mhz, radio_sf, radio_bw_khz, radio_cr, battery_level, uptime_seconds, status_metadata, last_status_at, first_seen, last_seen, observation_count, metadata, region_scope FROM observers WHERE id = $1 +SELECT id, public_key, display_name, observer_type, software_version, hardware_model, firmware_version, firmware_build, radio_freq_mhz, radio_sf, radio_bw_khz, radio_cr, battery_level, uptime_seconds, status_metadata, last_status_at, first_seen, last_seen, observation_count, metadata, region_scope, last_iata, last_iata_at FROM observers WHERE id = $1 ` func (q *Queries) GetObserverByID(ctx context.Context, id uuid.UUID) (Observer, error) { @@ -1109,12 +1137,14 @@ func (q *Queries) GetObserverByID(ctx context.Context, id uuid.UUID) (Observer, &i.ObservationCount, &i.Metadata, &i.RegionScope, + &i.LastIata, + &i.LastIataAt, ) return i, err } const getObserverByPubkey = `-- name: GetObserverByPubkey :one -SELECT id, public_key, display_name, observer_type, software_version, hardware_model, firmware_version, firmware_build, radio_freq_mhz, radio_sf, radio_bw_khz, radio_cr, battery_level, uptime_seconds, status_metadata, last_status_at, first_seen, last_seen, observation_count, metadata, region_scope FROM observers WHERE public_key = $1 +SELECT id, public_key, display_name, observer_type, software_version, hardware_model, firmware_version, firmware_build, radio_freq_mhz, radio_sf, radio_bw_khz, radio_cr, battery_level, uptime_seconds, status_metadata, last_status_at, first_seen, last_seen, observation_count, metadata, region_scope, last_iata, last_iata_at FROM observers WHERE public_key = $1 ` func (q *Queries) GetObserverByPubkey(ctx context.Context, publicKey []byte) (Observer, error) { @@ -1142,22 +1172,21 @@ func (q *Queries) GetObserverByPubkey(ctx context.Context, publicKey []byte) (Ob &i.ObservationCount, &i.Metadata, &i.RegionScope, + &i.LastIata, + &i.LastIataAt, ) return i, err } const getObserverLastIATA = `-- name: GetObserverLastIATA :one -SELECT iata FROM packet_observations -WHERE observer_id = $1 -ORDER BY heard_at DESC -LIMIT 1 +SELECT COALESCE(last_iata, '')::text FROM observers WHERE id = $1 ` -func (q *Queries) GetObserverLastIATA(ctx context.Context, observerID uuid.UUID) (string, error) { - row := q.db.QueryRow(ctx, getObserverLastIATA, observerID) - var iata string - err := row.Scan(&iata) - return iata, err +func (q *Queries) GetObserverLastIATA(ctx context.Context, id uuid.UUID) (string, error) { + row := q.db.QueryRow(ctx, getObserverLastIATA, id) + var column_1 string + err := row.Scan(&column_1) + return column_1, err } const getObserverRadio = `-- name: GetObserverRadio :one @@ -1280,7 +1309,7 @@ func (q *Queries) GetObserverTelemetry(ctx context.Context, arg GetObserverTelem } const getPacketByHash = `-- name: GetPacketByHash :one -SELECT p.packet_hash, p.payload_type, p.payload_version, p.route_type, p.transport_codes_present, p.region_code, p.sub_region_code, p.scope_id, p.origin_pubkey, p.raw_payload, p.raw_header, p.parsed_payload, p.decrypted, p.channel_hash, p.trace_tag, p.first_heard_at, p.last_heard_at, ts.name AS scope_name, +SELECT p.packet_hash, p.payload_type, p.payload_version, p.route_type, p.transport_codes_present, p.region_code, p.sub_region_code, p.scope_id, p.origin_pubkey, p.raw_payload, p.raw_header, p.parsed_payload, p.decrypted, p.channel_hash, p.trace_tag, p.first_heard_at, p.last_heard_at, p.observation_count, ts.name AS scope_name, cm.sender_name AS cm_sender_name, cm.content AS cm_content, cm.sent_at AS cm_sent_at @@ -1308,6 +1337,7 @@ type GetPacketByHashRow struct { TraceTag []byte `json:"trace_tag"` FirstHeardAt pgtype.Timestamptz `json:"first_heard_at"` LastHeardAt pgtype.Timestamptz `json:"last_heard_at"` + ObservationCount int64 `json:"observation_count"` ScopeName *string `json:"scope_name"` CmSenderName *string `json:"cm_sender_name"` CmContent *string `json:"cm_content"` @@ -1335,6 +1365,7 @@ func (q *Queries) GetPacketByHash(ctx context.Context, packetHash []byte) (GetPa &i.TraceTag, &i.FirstHeardAt, &i.LastHeardAt, + &i.ObservationCount, &i.ScopeName, &i.CmSenderName, &i.CmContent, @@ -1343,17 +1374,6 @@ func (q *Queries) GetPacketByHash(ctx context.Context, packetHash []byte) (GetPa return i, err } -const getPacketObservationCount = `-- name: GetPacketObservationCount :one -SELECT COUNT(*) FROM packet_observations WHERE packet_hash = $1 -` - -func (q *Queries) GetPacketObservationCount(ctx context.Context, packetHash []byte) (int64, error) { - row := q.db.QueryRow(ctx, getPacketObservationCount, packetHash) - var count int64 - err := row.Scan(&count) - return count, err -} - const getPacketsByTraceTag = `-- name: GetPacketsByTraceTag :many SELECT encode(p.packet_hash, 'hex') AS packet_hash_hex, p.route_type, @@ -1546,19 +1566,18 @@ func (q *Queries) GetRegionIATAs(ctx context.Context, regionID int32) ([]string, const getScopeByName = `-- name: GetScopeByName :one SELECT ts.name, - COUNT(DISTINCT p.packet_hash) AS packet_count, - COUNT(DISTINCT os.observer_id) AS observer_count, - COUNT(DISTINCT n.id) AS node_count, - COUNT(DISTINCT po.iata) AS iata_count, - array_remove(array_agg(DISTINCT po.iata ORDER BY po.iata), NULL)::text[] AS iatas + COALESCE((SELECT SUM(s.packets) FROM analytics_hourly_scope_sets s WHERE s.scope_id = ts.id), 0)::bigint AS packet_count, + (SELECT COUNT(*) FROM observer_scopes os WHERE os.scope_id = ts.id)::bigint AS observer_count, + (SELECT COUNT(*) FROM nodes n WHERE n.default_scope_id = ts.id)::bigint AS node_count, + cardinality(i.iatas)::bigint AS iata_count, + i.iatas FROM transport_scopes ts -LEFT JOIN packets p ON p.scope_id = ts.id -LEFT JOIN observer_scopes os ON os.scope_id = ts.id -LEFT JOIN observers o ON o.id = os.observer_id -LEFT JOIN packet_observations po ON po.observer_id = o.id -LEFT JOIN nodes n ON n.default_scope_id = ts.id +CROSS JOIN LATERAL ( + SELECT COALESCE(array_agg(DISTINCT x.iata::text ORDER BY x.iata::text), '{}')::text[] AS iatas + FROM analytics_hourly_scope_sets s CROSS JOIN LATERAL unnest(s.iatas) AS x(iata) + WHERE s.scope_id = ts.id +) i WHERE ts.name = $1 -GROUP BY ts.name ` type GetScopeByNameRow struct { @@ -1570,6 +1589,8 @@ type GetScopeByNameRow struct { Iatas []string `json:"iatas"` } +// Packets and IATAs cover every retained rollup hour; IATAs are those that heard the +// scope's own packets. Observer and node counts are current memberships. func (q *Queries) GetScopeByName(ctx context.Context, name string) (GetScopeByNameRow, error) { row := q.db.QueryRow(ctx, getScopeByName, name) var i GetScopeByNameRow @@ -1634,38 +1655,42 @@ func (q *Queries) GetScopeNames(ctx context.Context) ([]string, error) { } const getScopeStats = `-- name: GetScopeStats :many -WITH observation_counts AS ( - SELECT p.scope_id, - COUNT(DISTINCT p.packet_hash) AS packet_count, - COUNT(DISTINCT os.observer_id) AS observer_count - FROM packet_observations po - JOIN packets p ON p.packet_hash = po.packet_hash - LEFT JOIN observer_scopes os ON os.scope_id = p.scope_id AND os.observer_id = po.observer_id - WHERE po.iata = ANY($1::bpchar[]) AND p.scope_id IS NOT NULL - GROUP BY p.scope_id +WITH packet_counts AS ( + SELECT s.scope_id, SUM(s.packets)::bigint AS packet_count + FROM analytics_hourly_scope_sets s + WHERE s.hour >= $1::timestamptz + AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR s.iatas && $2::bpchar[]) + GROUP BY s.scope_id +), observer_counts AS ( + SELECT os.scope_id, COUNT(*)::bigint AS observer_count + FROM observer_scopes os JOIN observers o ON o.id = os.observer_id + WHERE COALESCE(cardinality($2::bpchar[]), 0) = 0 OR o.last_iata = ANY($2::bpchar[]) + GROUP BY os.scope_id ), node_counts AS ( - SELECT n.default_scope_id AS scope_id, COUNT(*) AS node_count + SELECT n.default_scope_id AS scope_id, COUNT(*)::bigint AS node_count FROM nodes n - WHERE n.id IN (SELECT node_id FROM node_iatas WHERE iata = ANY($1::bpchar[])) + WHERE n.default_scope_id IS NOT NULL + AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 + OR n.id IN (SELECT node_id FROM node_iatas WHERE iata = ANY($2::bpchar[]))) GROUP BY n.default_scope_id ) SELECT ts.name, - CASE WHEN COALESCE(cardinality($1::bpchar[]), 0) = 0 - THEN (SELECT COUNT(*) FROM packets p WHERE p.scope_id = ts.id) - ELSE COALESCE(oc.packet_count, 0) END::bigint AS packet_count, - CASE WHEN COALESCE(cardinality($1::bpchar[]), 0) = 0 - THEN (SELECT COUNT(*) FROM observer_scopes os WHERE os.scope_id = ts.id) - ELSE COALESCE(oc.observer_count, 0) END::bigint AS observer_count, - CASE WHEN COALESCE(cardinality($1::bpchar[]), 0) = 0 - THEN (SELECT COUNT(*) FROM nodes n WHERE n.default_scope_id = ts.id) - ELSE COALESCE(nc.node_count, 0) END::bigint AS node_count + COALESCE(pc.packet_count, 0)::bigint AS packet_count, + COALESCE(oc.observer_count, 0)::bigint AS observer_count, + COALESCE(nc.node_count, 0)::bigint AS node_count FROM transport_scopes ts -LEFT JOIN observation_counts oc ON oc.scope_id = ts.id +LEFT JOIN packet_counts pc ON pc.scope_id = ts.id +LEFT JOIN observer_counts oc ON oc.scope_id = ts.id LEFT JOIN node_counts nc ON nc.scope_id = ts.id ORDER BY ts.name ` +type GetScopeStatsParams struct { + Since pgtype.Timestamptz `json:"since"` + Iatas []string `json:"iatas"` +} + type GetScopeStatsRow struct { Name string `json:"name"` PacketCount int64 `json:"packet_count"` @@ -1673,11 +1698,10 @@ type GetScopeStatsRow struct { NodeCount int64 `json:"node_count"` } -// Aggregate matching observations once, separately from node memberships to avoid -// a cross-join. Empty IATAs keep the original global counts, including associations -// whose observations have expired; the filtered aggregates are empty in that case. -func (q *Queries) GetScopeStats(ctx context.Context, iatas []string) ([]GetScopeStatsRow, error) { - rows, err := q.db.Query(ctx, getScopeStats, iatas) +// Packets since the given hour come from the IATA-set rollup (counted once per hour heard). +// Observer and node counts are current memberships; observers filter by their latest IATA. +func (q *Queries) GetScopeStats(ctx context.Context, arg GetScopeStatsParams) ([]GetScopeStatsRow, error) { + rows, err := q.db.Query(ctx, getScopeStats, arg.Since, arg.Iatas) if err != nil { return nil, err } @@ -1807,65 +1831,30 @@ func (q *Queries) GetStatsNodeTypes(ctx context.Context, dollar_1 []string) ([]G return items, nil } -const getStatsOverview = `-- name: GetStatsOverview :one - -SELECT - COUNT(DISTINCT po.packet_hash) AS total_packets, - COUNT(*) AS total_observations, - COUNT(DISTINCT po.observer_id) AS active_observers, - COUNT(DISTINCT po.iata) AS active_iatas -FROM packet_observations po -WHERE po.heard_at > NOW() - INTERVAL '24 hours' - AND (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR po.iata = ANY($1::bpchar[])) -` - -type GetStatsOverviewRow struct { - TotalPackets int64 `json:"total_packets"` - TotalObservations int64 `json:"total_observations"` - ActiveObservers int64 `json:"active_observers"` - ActiveIatas int64 `json:"active_iatas"` -} - -// ============================================================ -// STATS -// ============================================================ -func (q *Queries) GetStatsOverview(ctx context.Context, dollar_1 []string) (GetStatsOverviewRow, error) { - row := q.db.QueryRow(ctx, getStatsOverview, dollar_1) - var i GetStatsOverviewRow - err := row.Scan( - &i.TotalPackets, - &i.TotalObservations, - &i.ActiveObservers, - &i.ActiveIatas, - ) - return i, err -} - const getStatsPayloadBreakdown = `-- name: GetStatsPayloadBreakdown :many SELECT payload_type, SUM(count)::bigint AS count -FROM mv_payload_breakdown_by_iata +FROM analytics_hourly_payload_breakdown WHERE (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR iata = ANY($1::bpchar[])) - AND bucket >= NOW() - $2::interval + AND hour >= $2::timestamptz GROUP BY payload_type ORDER BY count DESC ` type GetStatsPayloadBreakdownParams struct { - Column1 []string `json:"column_1"` - Column2 pgtype.Interval `json:"column_2"` + Iatas []string `json:"iatas"` + Since pgtype.Timestamptz `json:"since"` } type GetStatsPayloadBreakdownRow struct { - PayloadType *int16 `json:"payload_type"` - Count int64 `json:"count"` + PayloadType int16 `json:"payload_type"` + Count int64 `json:"count"` } -// Payload-type counts for the IATA within the window, summed from the -// precomputed hourly buckets. +// Payload-type observation counts since the given hour, from the hourly rollup. func (q *Queries) GetStatsPayloadBreakdown(ctx context.Context, arg GetStatsPayloadBreakdownParams) ([]GetStatsPayloadBreakdownRow, error) { - rows, err := q.db.Query(ctx, getStatsPayloadBreakdown, arg.Column1, arg.Column2) + rows, err := q.db.Query(ctx, getStatsPayloadBreakdown, arg.Iatas, arg.Since) if err != nil { return nil, err } @@ -1885,32 +1874,43 @@ func (q *Queries) GetStatsPayloadBreakdown(ctx context.Context, arg GetStatsPayl } const getStatsTopAdvertisers = `-- name: GetStatsTopAdvertisers :many -SELECT - node_id AS id, - name, - node_type, - COALESCE(SUM(advert_count), 0)::bigint AS advert_count, - COALESCE(SUM(flood_advert_count), 0)::bigint AS flood_advert_count, - COALESCE(SUM(direct_advert_count), 0)::bigint AS direct_advert_count, - MAX(last_heard)::timestamptz AS last_heard, - COALESCE(MAX(iata), '')::bpchar AS iata -FROM mv_top_advertisers_by_iata -WHERE bucket >= NOW() - $1::interval - AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR iata = ANY($2::bpchar[])) -GROUP BY node_id, name, node_type -ORDER BY advert_count DESC -LIMIT $3 +WITH ranked AS ( + SELECT origin_pubkey, SUM(advert_packets)::bigint AS advert_count, + SUM(flood_packets)::bigint AS flood_advert_count, SUM(direct_packets)::bigint AS direct_advert_count + FROM analytics_hourly_advert_sets + WHERE hour >= $1::timestamptz + AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR iatas && $2::bpchar[]) + GROUP BY origin_pubkey + ORDER BY advert_count DESC, origin_pubkey + LIMIT $3 +), heard AS ( + SELECT h.origin_pubkey, MAX(h.last_heard) AS last_heard, MAX(h.iata) AS iata, + MAX(h.name) AS name, MAX(h.node_type) AS node_type + FROM analytics_hourly_advert_hearings h + WHERE h.hour >= $1::timestamptz AND h.origin_pubkey IN (SELECT origin_pubkey FROM ranked) + AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR h.iata = ANY($2::bpchar[])) + GROUP BY h.origin_pubkey +) +SELECT n.id AS node_id, encode(r.origin_pubkey, 'hex') AS public_key, + COALESCE(n.name, h.name, '')::text AS name, COALESCE(n.node_type, h.node_type, 0)::smallint AS node_type, + r.advert_count, r.flood_advert_count, r.direct_advert_count, + h.last_heard::timestamptz AS last_heard, COALESCE(h.iata, '')::bpchar AS iata +FROM ranked r +LEFT JOIN heard h ON h.origin_pubkey = r.origin_pubkey +LEFT JOIN nodes n ON n.public_key = r.origin_pubkey +ORDER BY r.advert_count DESC, r.origin_pubkey ` type GetStatsTopAdvertisersParams struct { - Column1 pgtype.Interval `json:"column_1"` - Column2 []string `json:"column_2"` - Limit int32 `json:"limit"` + Since pgtype.Timestamptz `json:"since"` + Iatas []string `json:"iatas"` + RowLimit int32 `json:"row_limit"` } type GetStatsTopAdvertisersRow struct { - ID uuid.UUID `json:"id"` - Name *string `json:"name"` + NodeID pgtype.UUID `json:"node_id"` + PublicKey string `json:"public_key"` + Name string `json:"name"` NodeType int16 `json:"node_type"` AdvertCount int64 `json:"advert_count"` FloodAdvertCount int64 `json:"flood_advert_count"` @@ -1919,9 +1919,10 @@ type GetStatsTopAdvertisersRow struct { Iata string `json:"iata"` } -// Top N advertisers in the window, summed from the hourly buckets. +// Top N advertisers since the given hour. Each ADVERT packet counts once per hour heard, +// however many of the requested IATAs heard it (IATA-set rollup). Live names win. func (q *Queries) GetStatsTopAdvertisers(ctx context.Context, arg GetStatsTopAdvertisersParams) ([]GetStatsTopAdvertisersRow, error) { - rows, err := q.db.Query(ctx, getStatsTopAdvertisers, arg.Column1, arg.Column2, arg.Limit) + rows, err := q.db.Query(ctx, getStatsTopAdvertisers, arg.Since, arg.Iatas, arg.RowLimit) if err != nil { return nil, err } @@ -1930,7 +1931,8 @@ func (q *Queries) GetStatsTopAdvertisers(ctx context.Context, arg GetStatsTopAdv for rows.Next() { var i GetStatsTopAdvertisersRow if err := rows.Scan( - &i.ID, + &i.NodeID, + &i.PublicKey, &i.Name, &i.NodeType, &i.AdvertCount, @@ -1950,38 +1952,42 @@ func (q *Queries) GetStatsTopAdvertisers(ctx context.Context, arg GetStatsTopAdv } const getStatsTopObservers = `-- name: GetStatsTopObservers :many -SELECT - observer_id AS id, - display_name, - observer_type, - COALESCE(SUM(observation_count), 0)::bigint AS observation_count, - COALESCE(MAX(iata), '')::bpchar AS iata -FROM mv_top_observers_by_iata -WHERE bucket >= NOW() - $1::interval - AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR iata = ANY($2::bpchar[])) -GROUP BY observer_id, display_name, observer_type -ORDER BY observation_count DESC -LIMIT $3 +WITH ranked AS ( + SELECT observer_id, SUM(observation_count)::bigint AS observation_count, MAX(iata) AS iata, + MAX(display_name) AS display_name, MAX(observer_type) AS observer_type + FROM analytics_hourly_observer_identity + WHERE hour >= $1::timestamptz + AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR iata = ANY($2::bpchar[])) + GROUP BY observer_id + ORDER BY observation_count DESC, observer_id + LIMIT $3 +) +SELECT r.observer_id AS id, COALESCE(o.display_name, r.display_name, '')::text AS display_name, + COALESCE(o.observer_type, r.observer_type, '')::text AS observer_type, + r.observation_count, r.iata::bpchar AS iata +FROM ranked r +LEFT JOIN observers o ON o.id = r.observer_id +ORDER BY r.observation_count DESC, r.observer_id ` type GetStatsTopObserversParams struct { - Column1 pgtype.Interval `json:"column_1"` - Column2 []string `json:"column_2"` - Limit int32 `json:"limit"` + Since pgtype.Timestamptz `json:"since"` + Iatas []string `json:"iatas"` + RowLimit int32 `json:"row_limit"` } type GetStatsTopObserversRow struct { ID uuid.UUID `json:"id"` - DisplayName *string `json:"display_name"` - ObserverType *string `json:"observer_type"` + DisplayName string `json:"display_name"` + ObserverType string `json:"observer_type"` ObservationCount int64 `json:"observation_count"` Iata string `json:"iata"` } -// Top N observers for the IATA within the window, summed from the precomputed -// hourly buckets. Counts sum across matched IATAs; iata is a representative one. +// Top N observers since the given hour. Counts sum across matched IATAs; iata is a +// representative one. Live names win over the rolled snapshot. func (q *Queries) GetStatsTopObservers(ctx context.Context, arg GetStatsTopObserversParams) ([]GetStatsTopObserversRow, error) { - rows, err := q.db.Query(ctx, getStatsTopObservers, arg.Column1, arg.Column2, arg.Limit) + rows, err := q.db.Query(ctx, getStatsTopObservers, arg.Since, arg.Iatas, arg.RowLimit) if err != nil { return nil, err } @@ -2009,31 +2015,32 @@ func (q *Queries) GetStatsTopObservers(ctx context.Context, arg GetStatsTopObser const getStatsTopTalkers = `-- name: GetStatsTopTalkers :many SELECT sender_name, - COALESCE(SUM(message_count), 0)::bigint AS message_count, + SUM(messages)::bigint AS message_count, MAX(last_sent)::timestamptz AS last_sent -FROM mv_top_talkers_by_iata -WHERE bucket >= NOW() - $1::interval - AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR iata = ANY($2::bpchar[])) +FROM analytics_hourly_talker_sets +WHERE hour >= $1::timestamptz + AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR iatas && $2::bpchar[]) GROUP BY sender_name -ORDER BY message_count DESC +ORDER BY message_count DESC, sender_name LIMIT $3 ` type GetStatsTopTalkersParams struct { - Column1 pgtype.Interval `json:"column_1"` - Column2 []string `json:"column_2"` - Limit int32 `json:"limit"` + Since pgtype.Timestamptz `json:"since"` + Iatas []string `json:"iatas"` + RowLimit int32 `json:"row_limit"` } type GetStatsTopTalkersRow struct { - SenderName *string `json:"sender_name"` + SenderName string `json:"sender_name"` MessageCount int64 `json:"message_count"` LastSent pgtype.Timestamptz `json:"last_sent"` } -// Top N talkers (by decrypted sender_name) in the window, summed from the hourly buckets. +// Top N senders since the given hour. Each message counts once per hour heard, however many +// of the requested IATAs heard it (IATA-set rollup). func (q *Queries) GetStatsTopTalkers(ctx context.Context, arg GetStatsTopTalkersParams) ([]GetStatsTopTalkersRow, error) { - rows, err := q.db.Query(ctx, getStatsTopTalkers, arg.Column1, arg.Column2, arg.Limit) + rows, err := q.db.Query(ctx, getStatsTopTalkers, arg.Since, arg.Iatas, arg.RowLimit) if err != nil { return nil, err } @@ -2053,31 +2060,57 @@ func (q *Queries) GetStatsTopTalkers(ctx context.Context, arg GetStatsTopTalkers } const getTopNodes = `-- name: GetTopNodes :many -SELECT iata, node_id, name, node_type, observation_count, last_heard FROM mv_top_nodes_by_iata -WHERE (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR iata = ANY($1::bpchar[])) -ORDER BY observation_count DESC -LIMIT $2 +WITH ranked AS ( + SELECT origin_pubkey, SUM(observations)::bigint AS observation_count, MAX(last_heard) AS last_heard, + MAX(iata) AS iata, MAX(name) AS name, MAX(node_type) AS node_type + FROM analytics_hourly_advert_hearings + WHERE hour >= $1::timestamptz + AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR iata = ANY($2::bpchar[])) + GROUP BY origin_pubkey + ORDER BY observation_count DESC, origin_pubkey + LIMIT $3 +) +SELECT n.id AS node_id, encode(r.origin_pubkey, 'hex') AS public_key, + COALESCE(n.name, r.name, '')::text AS name, COALESCE(n.node_type, r.node_type, 0)::smallint AS node_type, + r.iata::bpchar AS iata, r.observation_count, r.last_heard::timestamptz AS last_heard +FROM ranked r +LEFT JOIN nodes n ON n.public_key = r.origin_pubkey +ORDER BY r.observation_count DESC, r.origin_pubkey ` type GetTopNodesParams struct { - Column1 []string `json:"column_1"` - Limit int32 `json:"limit"` + Since pgtype.Timestamptz `json:"since"` + Iatas []string `json:"iatas"` + RowLimit int32 `json:"row_limit"` } -func (q *Queries) GetTopNodes(ctx context.Context, arg GetTopNodesParams) ([]MvTopNodesByIatum, error) { - rows, err := q.db.Query(ctx, getTopNodes, arg.Column1, arg.Limit) +type GetTopNodesRow struct { + NodeID pgtype.UUID `json:"node_id"` + PublicKey string `json:"public_key"` + Name string `json:"name"` + NodeType int16 `json:"node_type"` + Iata string `json:"iata"` + ObservationCount int64 `json:"observation_count"` + LastHeard pgtype.Timestamptz `json:"last_heard"` +} + +// Nodes by ADVERT hearings since the given hour. Live names win over the rolled snapshot; +// node_id is NULL once the node row is gone. iata is a representative one. +func (q *Queries) GetTopNodes(ctx context.Context, arg GetTopNodesParams) ([]GetTopNodesRow, error) { + rows, err := q.db.Query(ctx, getTopNodes, arg.Since, arg.Iatas, arg.RowLimit) if err != nil { return nil, err } defer rows.Close() - items := []MvTopNodesByIatum{} + items := []GetTopNodesRow{} for rows.Next() { - var i MvTopNodesByIatum + var i GetTopNodesRow if err := rows.Scan( - &i.Iata, &i.NodeID, + &i.PublicKey, &i.Name, &i.NodeType, + &i.Iata, &i.ObservationCount, &i.LastHeard, ); err != nil { @@ -2138,7 +2171,16 @@ WITH inserted AS ( INSERT INTO channel_messages (channel_id, packet_hash, sender_name, content, sent_at) VALUES ($1, $2, $3, $4, $5) ON CONFLICT (packet_hash) DO NOTHING - RETURNING id, packet_hash + RETURNING id, packet_hash, channel_id +), bump AS ( + UPDATE channels c SET message_count = c.message_count + 1 + FROM inserted WHERE c.id = inserted.channel_id +), dirty AS ( + INSERT INTO analytics_dirty_hours (hour) + SELECT DISTINCT date_trunc('hour', po.heard_at, 'UTC') + FROM packet_observations po JOIN inserted i ON i.packet_hash = po.packet_hash + WHERE $6::boolean + ON CONFLICT (hour) DO UPDATE SET enqueued_at = now() ) SELECT inserted.id, ts.name AS scope_name, p.transport_codes_present FROM inserted @@ -2152,6 +2194,7 @@ type InsertChannelMessageParams struct { SenderName *string `json:"sender_name"` Content *string `json:"content"` SentAt pgtype.Timestamptz `json:"sent_at"` + Column6 bool `json:"column_6"` } type InsertChannelMessageRow struct { @@ -2165,6 +2208,8 @@ type InsertChannelMessageRow struct { // ============================================================ // Read the immutable first-packet scope in the same statement as insertion. // A later reception's transport code must not give live and historical messages different tags. +// message_count is a lifetime count, bumped only for a new message. A historical (backfilled) +// message queues its observation hours for a re-roll in the same statement, so the two can't diverge. func (q *Queries) InsertChannelMessage(ctx context.Context, arg InsertChannelMessageParams) (InsertChannelMessageRow, error) { row := q.db.QueryRow(ctx, insertChannelMessage, arg.ChannelID, @@ -2172,6 +2217,7 @@ func (q *Queries) InsertChannelMessage(ctx context.Context, arg InsertChannelMes arg.SenderName, arg.Content, arg.SentAt, + arg.Column6, ) var i InsertChannelMessageRow err := row.Scan(&i.ID, &i.ScopeName, &i.TransportCodesPresent) @@ -2180,6 +2226,7 @@ func (q *Queries) InsertChannelMessage(ctx context.Context, arg InsertChannelMes const insertObservation = `-- name: InsertObservation :one +WITH ins AS ( INSERT INTO packet_observations ( packet_hash, observer_id, @@ -2203,7 +2250,15 @@ INSERT INTO packet_observations ( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18 ) ON CONFLICT (packet_hash, observer_id) DO NOTHING -RETURNING id, packet_hash, observer_id, iata, heard_at, path_length_byte, hash_size, hop_count, path_bytes, rssi, snr, propagation_time_ms, radio_freq_mhz, spread_factor, bandwidth_khz, coding_rate, source_broker, payload_type, airtime_ms +RETURNING packet_hash +), bump AS ( + UPDATE packets p SET observation_count = p.observation_count + 1 + FROM ins WHERE p.packet_hash = ins.packet_hash + RETURNING p.observation_count +) +SELECT EXISTS (SELECT 1 FROM ins) AS inserted, + COALESCE((SELECT b.observation_count FROM bump b), + (SELECT pk.observation_count FROM packets pk WHERE pk.packet_hash = $1), 0)::bigint AS observation_count ` type InsertObservationParams struct { @@ -2227,10 +2282,16 @@ type InsertObservationParams struct { AirtimeMs *float32 `json:"airtime_ms"` } +type InsertObservationRow struct { + Inserted bool `json:"inserted"` + ObservationCount int64 `json:"observation_count"` +} + // ============================================================ // PACKET OBSERVATIONS // ============================================================ -func (q *Queries) InsertObservation(ctx context.Context, arg InsertObservationParams) (PacketObservation, error) { +// Bumps packets.observation_count only for a new row; a duplicate returns the current count. +func (q *Queries) InsertObservation(ctx context.Context, arg InsertObservationParams) (InsertObservationRow, error) { row := q.db.QueryRow(ctx, insertObservation, arg.PacketHash, arg.ObserverID, @@ -2251,28 +2312,8 @@ func (q *Queries) InsertObservation(ctx context.Context, arg InsertObservationPa arg.PayloadType, arg.AirtimeMs, ) - var i PacketObservation - err := row.Scan( - &i.ID, - &i.PacketHash, - &i.ObserverID, - &i.Iata, - &i.HeardAt, - &i.PathLengthByte, - &i.HashSize, - &i.HopCount, - &i.PathBytes, - &i.Rssi, - &i.Snr, - &i.PropagationTimeMs, - &i.RadioFreqMhz, - &i.SpreadFactor, - &i.BandwidthKhz, - &i.CodingRate, - &i.SourceBroker, - &i.PayloadType, - &i.AirtimeMs, - ) + var i InsertObservationRow + err := row.Scan(&i.Inserted, &i.ObservationCount) return i, err } @@ -2348,7 +2389,7 @@ func (q *Queries) ListAccounts(ctx context.Context) ([]Account, error) { const listAllChannelMessages = `-- name: ListAllChannelMessages :many SELECT DISTINCT ON (cm.id) cm.id, cm.channel_id, cm.packet_hash, cm.sender_name, cm.sender_pubkey, cm.content, cm.sent_at, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, ts.name AS scope_name, p.transport_codes_present, -(SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = cm.packet_hash) AS observation_count +p.observation_count AS observation_count FROM channel_messages cm JOIN channels c ON c.id = cm.channel_id JOIN packet_observations po ON po.packet_hash = cm.packet_hash @@ -2462,7 +2503,7 @@ func (q *Queries) ListChannelCatalogues(ctx context.Context) ([]MeshmapperChanne const listChannelMessages = `-- name: ListChannelMessages :many SELECT DISTINCT ON (cm.id) cm.id, cm.channel_id, cm.packet_hash, cm.sender_name, cm.sender_pubkey, cm.content, cm.sent_at, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, ts.name AS scope_name, p.transport_codes_present, -(SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = cm.packet_hash) AS observation_count +p.observation_count AS observation_count FROM channel_messages cm JOIN channels c ON c.id = cm.channel_id JOIN packet_observations po ON po.packet_hash = cm.packet_hash @@ -2547,7 +2588,7 @@ func (q *Queries) ListChannelMessages(ctx context.Context, arg ListChannelMessag const listChannelMessagesByHash = `-- name: ListChannelMessagesByHash :many SELECT DISTINCT ON (cm.id) cm.id, cm.channel_id, cm.packet_hash, cm.sender_name, cm.sender_pubkey, cm.content, cm.sent_at, c.channel_hash, ts.name AS scope_name, p.transport_codes_present, - (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = cm.packet_hash) AS observation_count + p.observation_count AS observation_count FROM channel_messages cm JOIN channels c ON c.id = cm.channel_id JOIN packet_observations po ON po.packet_hash = cm.packet_hash @@ -2873,7 +2914,7 @@ func (q *Queries) ListKnownRoutes(ctx context.Context, arg ListKnownRoutesParams const listMessagesAfterID = `-- name: ListMessagesAfterID :many SELECT DISTINCT ON (cm.id) cm.id, cm.channel_id, cm.packet_hash, cm.sender_name, cm.sender_pubkey, cm.content, cm.sent_at, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, ts.name AS scope_name, p.transport_codes_present, -(SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = cm.packet_hash) AS observation_count +p.observation_count AS observation_count FROM channel_messages cm JOIN channels c ON c.id = cm.channel_id JOIN packet_observations po ON po.packet_hash = cm.packet_hash @@ -3291,23 +3332,13 @@ COALESCE(CASE WHEN GREATEST(COALESCE(o.last_status_at, o.last_seen), o.last_seen) > NOW() - INTERVAL '5 minutes' THEN 'online' ELSE 'offline' END, 'offline')::text AS status, -COALESCE(( - SELECT po.iata - FROM packet_observations po - WHERE po.observer_id = o.id - ORDER BY po.heard_at DESC - LIMIT 1 -), '')::text AS iata +COALESCE(o.last_iata, '')::text AS iata FROM observers o LEFT JOIN observer_brokers ob ON ob.observer_id = o.id LEFT JOIN observer_scopes os ON os.observer_id = o.id LEFT JOIN transport_scopes ts ON ts.id = os.scope_id WHERE - (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR ( - SELECT po.iata FROM packet_observations po - WHERE po.observer_id = o.id - ORDER BY po.heard_at DESC LIMIT 1 - ) = ANY($1::bpchar[])) + (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR o.last_iata = ANY($1::bpchar[])) AND ($2 = '' OR o.observer_type = $2) AND ($3 = '' OR ob.broker_name = $3) AND ($4 = '' OR CASE @@ -3413,7 +3444,7 @@ SELECT p.last_heard_at, p.scope_id, ts.name AS scope_name, - (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = p.packet_hash) AS observation_count, + p.observation_count AS observation_count, -- sqlc loses LATERAL nullability; these scalar defaults are ignored when observer_id is NULL. po.observer_id AS latest_observer_id, o.display_name AS latest_observer_name, @@ -3545,7 +3576,7 @@ SELECT p.route_type, p.first_heard_at, p.last_heard_at, - (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = p.packet_hash) AS observation_count, + p.observation_count AS observation_count, po.observer_id AS latest_observer_id, o.display_name AS latest_observer_name, po.iata AS latest_observer_iata, @@ -3712,7 +3743,7 @@ SELECT sh.site_heard_at, sat.scan_saturated, sat.scan_floor, - (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = p.packet_hash) AS observation_count, + p.observation_count AS observation_count, -- sqlc loses LATERAL nullability; these scalar defaults are ignored when observer_id is NULL. po.observer_id AS latest_observer_id, o.display_name AS latest_observer_name, @@ -3958,43 +3989,27 @@ func (q *Queries) ListScopeCatalogues(ctx context.Context) ([]MeshmapperScopeCat const listTraceTags = `-- name: ListTraceTags :many -WITH tags AS ( - SELECT - p.trace_tag, - MIN(p.first_heard_at) AS first_heard_at, - MAX(p.last_heard_at) AS last_heard_at, - COUNT(*) AS packet_count, - MAX(p.parsed_payload->>'type') AS trace_type - FROM packets p - WHERE p.trace_tag IS NOT NULL - AND (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR p.trace_tag IN ( - SELECT ti.trace_tag FROM trace_iatas ti WHERE ti.iata = ANY($1::bpchar[]))) - AND ($2::text = '' OR p.scope_id = (SELECT id FROM transport_scopes WHERE name = $2)) - AND ($3::timestamptz IS NULL OR p.first_heard_at >= $3) - AND ($4::timestamptz IS NULL OR p.first_heard_at <= $4) - AND ($7::text = '' OR p.parsed_payload->>'type' = $7) - GROUP BY p.trace_tag - HAVING ($5::timestamptz IS NULL OR MAX(p.last_heard_at) < $5) - ORDER BY MAX(p.last_heard_at) DESC - LIMIT $6 -) SELECT encode(t.trace_tag, 'hex') AS trace_tag, - t.first_heard_at::timestamptz AS first_heard_at, - t.last_heard_at::timestamptz AS last_heard_at, + t.first_heard_at, + t.last_heard_at, t.packet_count, (SELECT COUNT(*) FROM trace_iatas ti WHERE ti.trace_tag = t.trace_tag AND (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR ti.iata = ANY($1::bpchar[]))) AS iata_count, - t.trace_type::text AS trace_type, - (SELECT p3.parsed_payload - FROM packets p3 - WHERE p3.trace_tag = t.trace_tag - ORDER BY jsonb_array_length(p3.parsed_payload->'pathHashes') DESC - LIMIT 1) AS best_payload -FROM tags t + COALESCE(t.trace_type, '')::text AS trace_type, + t.best_payload +FROM trace_tags t +WHERE (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR EXISTS ( + SELECT 1 FROM trace_iatas ti WHERE ti.trace_tag = t.trace_tag AND ti.iata = ANY($1::bpchar[]))) + AND ($2::text = '' OR t.scope_id = (SELECT id FROM transport_scopes WHERE name = $2)) + AND ($3::timestamptz IS NULL OR t.first_heard_at >= $3) + AND ($4::timestamptz IS NULL OR t.first_heard_at <= $4) + AND ($5::timestamptz IS NULL OR t.last_heard_at < $5) + AND ($7::text = '' OR t.trace_type = $7) ORDER BY t.last_heard_at DESC +LIMIT $6 ` type ListTraceTagsParams struct { @@ -4020,9 +4035,7 @@ type ListTraceTagsRow struct { // ============================================================ // TRACES // ============================================================ -// Returns distinct trace tags with summary info, ordered by most recent first. -// IATA membership comes from trace_iatas (joining observations here spilled the -// hash join). Per-tag details filled in only for the returned page. +// Tags, most recent first. Filters match the tag summary (see RecordTrace). func (q *Queries) ListTraceTags(ctx context.Context, arg ListTraceTagsParams) ([]ListTraceTagsRow, error) { rows, err := q.db.Query(ctx, listTraceTags, arg.Column1, @@ -4337,30 +4350,35 @@ func (q *Queries) ReconfirmRoutes(ctx context.Context, arg ReconfirmRoutesParams return count, err } -const refreshHourlyStats = `-- name: RefreshHourlyStats :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_hourly_iata_stats +const recordTrace = `-- name: RecordTrace :exec +WITH iata AS ( + INSERT INTO trace_iatas (trace_tag, iata, last_heard) + VALUES ($1, $3, $2) + ON CONFLICT (trace_tag, iata) DO UPDATE SET + last_heard = EXCLUDED.last_heard + WHERE EXCLUDED.last_heard > trace_iatas.last_heard + INTERVAL '1 hour' +) +INSERT INTO trace_tags (trace_tag, first_heard_at, last_heard_at, heard) +VALUES ($1, $2, $2, true) +ON CONFLICT (trace_tag) DO UPDATE SET + first_heard_at = CASE WHEN trace_tags.heard THEN LEAST(trace_tags.first_heard_at, EXCLUDED.first_heard_at) + ELSE EXCLUDED.first_heard_at END, + last_heard_at = CASE WHEN trace_tags.heard THEN GREATEST(trace_tags.last_heard_at, EXCLUDED.last_heard_at) + ELSE EXCLUDED.last_heard_at END, + heard = true ` -func (q *Queries) RefreshHourlyStats(ctx context.Context) error { - _, err := q.db.Exec(ctx, refreshHourlyStats) - return err +type RecordTraceParams struct { + TraceTag []byte `json:"trace_tag"` + HeardAt pgtype.Timestamptz `json:"heard_at"` + Iata string `json:"iata"` } -const refreshObserverActivity = `-- name: RefreshObserverActivity :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_observer_activity_hourly -` - -func (q *Queries) RefreshObserverActivity(ctx context.Context) error { - _, err := q.db.Exec(ctx, refreshObserverActivity) - return err -} - -const refreshPayloadBreakdown = `-- name: RefreshPayloadBreakdown :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_payload_breakdown_by_iata -` - -func (q *Queries) RefreshPayloadBreakdown(ctx context.Context) error { - _, err := q.db.Exec(ctx, refreshPayloadBreakdown) +// One hearing of a TRACE packet: trace_iatas refreshes at most hourly and the tag's heard +// window widens (the first hearing replaces the packet's provisional times). Packet counts +// and payloads come from UpsertPacket. +func (q *Queries) RecordTrace(ctx context.Context, arg RecordTraceParams) error { + _, err := q.db.Exec(ctx, recordTrace, arg.TraceTag, arg.HeardAt, arg.Iata) return err } @@ -4373,42 +4391,6 @@ func (q *Queries) RefreshRadioPresets(ctx context.Context) error { return err } -const refreshTopAdvertisers = `-- name: RefreshTopAdvertisers :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_top_advertisers_by_iata -` - -func (q *Queries) RefreshTopAdvertisers(ctx context.Context) error { - _, err := q.db.Exec(ctx, refreshTopAdvertisers) - return err -} - -const refreshTopNodes = `-- name: RefreshTopNodes :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_top_nodes_by_iata -` - -func (q *Queries) RefreshTopNodes(ctx context.Context) error { - _, err := q.db.Exec(ctx, refreshTopNodes) - return err -} - -const refreshTopObservers = `-- name: RefreshTopObservers :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_top_observers_by_iata -` - -func (q *Queries) RefreshTopObservers(ctx context.Context) error { - _, err := q.db.Exec(ctx, refreshTopObservers) - return err -} - -const refreshTopTalkers = `-- name: RefreshTopTalkers :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_top_talkers_by_iata -` - -func (q *Queries) RefreshTopTalkers(ctx context.Context) error { - _, err := q.db.Exec(ctx, refreshTopTalkers) - return err -} - const resolveEndpointHashPairs = `-- name: ResolveEndpointHashPairs :many SELECT ns.iata, ns.prefix_1 AS hash, n.id AS node_id, n.name, n.latitude, n.longitude, n.public_key FROM node_short_ids ns @@ -5035,11 +5017,16 @@ func (q *Queries) TouchObserverBrokers(ctx context.Context, arg TouchObserverBro const touchObservers = `-- name: TouchObservers :exec UPDATE observers o SET last_seen = GREATEST(o.last_seen, v.seen), - observation_count = COALESCE(o.observation_count, 0) + v.delta + observation_count = COALESCE(o.observation_count, 0) + v.delta, + last_iata = CASE WHEN v.iata <> '' AND (o.last_iata_at IS NULL OR v.iata_at > o.last_iata_at) + THEN v.iata ELSE o.last_iata END, + last_iata_at = GREATEST(o.last_iata_at, v.iata_at) FROM ( SELECT unnest($1::uuid[]) AS id, unnest($2::timestamptz[]) AS seen, - unnest($3::int[]) AS delta + unnest($3::int[]) AS delta, + unnest($4::text[]) AS iata, + unnest($5::timestamptz[]) AS iata_at ) v WHERE o.id = v.id ` @@ -5048,12 +5035,20 @@ type TouchObserversParams struct { Column1 []uuid.UUID `json:"column_1"` Column2 []pgtype.Timestamptz `json:"column_2"` Column3 []int32 `json:"column_3"` + Column4 []string `json:"column_4"` + Column5 []pgtype.Timestamptz `json:"column_5"` } // Batched flush of coalesced presence bumps. GREATEST keeps a late flush // from regressing a newer write-through (e.g. a status update). func (q *Queries) TouchObservers(ctx context.Context, arg TouchObserversParams) error { - _, err := q.db.Exec(ctx, touchObservers, arg.Column1, arg.Column2, arg.Column3) + _, err := q.db.Exec(ctx, touchObservers, + arg.Column1, + arg.Column2, + arg.Column3, + arg.Column4, + arg.Column5, + ) return err } @@ -5160,18 +5155,16 @@ INSERT INTO channels (channel_hash, key_fingerprint, name, hashtag, is_hashtag, VALUES ($1, $2::bytea, $3, $4, $5, ($2 IS NOT NULL), NOW()) ON CONFLICT (channel_hash, key_fingerprint) DO UPDATE SET last_seen = NOW(), - name = COALESCE(EXCLUDED.name, channels.name), - message_count = CASE WHEN $6 THEN channels.message_count + 1 ELSE channels.message_count END + name = COALESCE(EXCLUDED.name, channels.name) RETURNING id, channel_hash, key_fingerprint, name, hashtag, is_hashtag, is_public, key_known, first_seen, last_seen, message_count ` type UpsertChannelParams struct { - ChannelHash []byte `json:"channel_hash"` - Column2 []byte `json:"column_2"` - Name *string `json:"name"` - Hashtag *string `json:"hashtag"` - IsHashtag *bool `json:"is_hashtag"` - MessageCount *int64 `json:"message_count"` + ChannelHash []byte `json:"channel_hash"` + Column2 []byte `json:"column_2"` + Name *string `json:"name"` + Hashtag *string `json:"hashtag"` + IsHashtag *bool `json:"is_hashtag"` } // ============================================================ @@ -5186,7 +5179,6 @@ func (q *Queries) UpsertChannel(ctx context.Context, arg UpsertChannelParams) (C arg.Name, arg.Hashtag, arg.IsHashtag, - arg.MessageCount, ) var i Channel err := row.Scan( @@ -5529,19 +5521,31 @@ func (q *Queries) UpsertNodeShortID(ctx context.Context, arg UpsertNodeShortIDPa const upsertObserver = `-- name: UpsertObserver :one -INSERT INTO observers (public_key, observer_type, last_seen) -VALUES ($1, 'unknown', NOW()) +INSERT INTO observers (public_key, observer_type, last_seen, last_iata, last_iata_at) +VALUES ($1, 'unknown', NOW(), NULLIF($2::text, ''), + CASE WHEN $2::text <> '' THEN $3::timestamptz END) ON CONFLICT (public_key) DO UPDATE SET last_seen = NOW(), - observation_count = observers.observation_count + 1 -RETURNING id, public_key, display_name, observer_type, software_version, hardware_model, firmware_version, firmware_build, radio_freq_mhz, radio_sf, radio_bw_khz, radio_cr, battery_level, uptime_seconds, status_metadata, last_status_at, first_seen, last_seen, observation_count, metadata, region_scope + observation_count = observers.observation_count + 1, + last_iata = CASE WHEN EXCLUDED.last_iata IS NOT NULL + AND (observers.last_iata_at IS NULL OR EXCLUDED.last_iata_at > observers.last_iata_at) + THEN EXCLUDED.last_iata ELSE observers.last_iata END, + last_iata_at = GREATEST(observers.last_iata_at, EXCLUDED.last_iata_at) +RETURNING id, public_key, display_name, observer_type, software_version, hardware_model, firmware_version, firmware_build, radio_freq_mhz, radio_sf, radio_bw_khz, radio_cr, battery_level, uptime_seconds, status_metadata, last_status_at, first_seen, last_seen, observation_count, metadata, region_scope, last_iata, last_iata_at ` +type UpsertObserverParams struct { + PublicKey []byte `json:"public_key"` + Iata string `json:"iata"` + IataAt pgtype.Timestamptz `json:"iata_at"` +} + // ============================================================ // OBSERVERS // ============================================================ -func (q *Queries) UpsertObserver(ctx context.Context, publicKey []byte) (Observer, error) { - row := q.db.QueryRow(ctx, upsertObserver, publicKey) +// Empty iata (status/neighbors) leaves last_iata alone; otherwise the newest iata_at wins. +func (q *Queries) UpsertObserver(ctx context.Context, arg UpsertObserverParams) (Observer, error) { + row := q.db.QueryRow(ctx, upsertObserver, arg.PublicKey, arg.Iata, arg.IataAt) var i Observer err := row.Scan( &i.ID, @@ -5565,6 +5569,8 @@ func (q *Queries) UpsertObserver(ctx context.Context, publicKey []byte) (Observe &i.ObservationCount, &i.Metadata, &i.RegionScope, + &i.LastIata, + &i.LastIataAt, ) return i, err } @@ -5611,6 +5617,7 @@ func (q *Queries) UpsertObserverScope(ctx context.Context, arg UpsertObserverSco const upsertPacket = `-- name: UpsertPacket :one +WITH up AS ( INSERT INTO packets ( packet_hash, payload_type, @@ -5633,8 +5640,25 @@ INSERT INTO packets ( ) ON CONFLICT (packet_hash) DO UPDATE SET last_heard_at = NOW() -RETURNING packet_hash, payload_type, payload_version, route_type, transport_codes_present, region_code, sub_region_code, origin_pubkey, raw_payload, raw_header, parsed_payload, decrypted, channel_hash, first_heard_at, last_heard_at, (xmax = 0) +RETURNING packet_hash, payload_type, payload_version, route_type, transport_codes_present, region_code, sub_region_code, origin_pubkey, raw_payload, raw_header, parsed_payload, decrypted, channel_hash, first_heard_at, last_heard_at, trace_tag, scope_id, (xmax = 0) AS inserted +), trace AS ( + INSERT INTO trace_tags (trace_tag, first_heard_at, last_heard_at, packet_count, trace_type, scope_id, best_payload, best_path_len) + SELECT u.trace_tag, u.first_heard_at, u.last_heard_at, 1, u.parsed_payload->>'type', u.scope_id, u.parsed_payload, + CASE WHEN jsonb_typeof(u.parsed_payload->'pathHashes') = 'array' + THEN jsonb_array_length(u.parsed_payload->'pathHashes') ELSE 0 END + FROM up u + WHERE u.inserted AND u.trace_tag IS NOT NULL + ON CONFLICT (trace_tag) DO UPDATE SET + packet_count = trace_tags.packet_count + 1, + trace_type = GREATEST(trace_tags.trace_type, EXCLUDED.trace_type), + scope_id = COALESCE(trace_tags.scope_id, EXCLUDED.scope_id), + best_payload = CASE WHEN EXCLUDED.best_path_len > trace_tags.best_path_len + THEN EXCLUDED.best_payload ELSE trace_tags.best_payload END, + best_path_len = GREATEST(trace_tags.best_path_len, EXCLUDED.best_path_len) +) +SELECT packet_hash, payload_type, payload_version, route_type, transport_codes_present, region_code, sub_region_code, origin_pubkey, raw_payload, raw_header, parsed_payload, decrypted, channel_hash, first_heard_at, last_heard_at, inserted +FROM up ` type UpsertPacketParams struct { @@ -5676,6 +5700,8 @@ type UpsertPacketRow struct { // ============================================================ // PACKETS // ============================================================ +// A new TRACE packet adds itself to its tag summary in the same statement, so the summary +// can't miss a stored packet. 'TRACE' > 'PING'; the longest path keeps the best payload. func (q *Queries) UpsertPacket(ctx context.Context, arg UpsertPacketParams) (UpsertPacketRow, error) { row := q.db.QueryRow(ctx, upsertPacket, arg.PacketHash, @@ -5755,26 +5781,6 @@ func (q *Queries) UpsertRegion(ctx context.Context, arg UpsertRegionParams) (int return id, err } -const upsertTraceIATA = `-- name: UpsertTraceIATA :exec -INSERT INTO trace_iatas (trace_tag, iata, last_heard) -VALUES ($1, $2, $3) -ON CONFLICT (trace_tag, iata) DO UPDATE SET - last_heard = EXCLUDED.last_heard -WHERE EXCLUDED.last_heard > trace_iatas.last_heard + INTERVAL '1 hour' -` - -type UpsertTraceIATAParams struct { - TraceTag []byte `json:"trace_tag"` - Iata string `json:"iata"` - LastHeard pgtype.Timestamptz `json:"last_heard"` -} - -// Refreshes at most hourly so repeat hears don't churn the row. -func (q *Queries) UpsertTraceIATA(ctx context.Context, arg UpsertTraceIATAParams) error { - _, err := q.db.Exec(ctx, upsertTraceIATA, arg.TraceTag, arg.Iata, arg.LastHeard) - return err -} - const upsertTransportScope = `-- name: UpsertTransportScope :exec INSERT INTO transport_scopes (name, display_name, transport_key, key_fingerprint) diff --git a/db/sqlc/rollup.sql.go b/db/sqlc/rollup.sql.go new file mode 100644 index 0000000..db8ee8f --- /dev/null +++ b/db/sqlc/rollup.sql.go @@ -0,0 +1,501 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: rollup.sql + +package db + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const deleteOldRollups = `-- name: DeleteOldRollups :exec +WITH a AS (DELETE FROM analytics_hourly_iata_observations ta WHERE ta.hour < $1::timestamptz), + b AS (DELETE FROM analytics_hourly_payload_breakdown tb WHERE tb.hour < $1::timestamptz), + c AS (DELETE FROM analytics_hourly_signal tc WHERE tc.hour < $1::timestamptz), + d AS (DELETE FROM analytics_hourly_paths td WHERE td.hour < $1::timestamptz), + e AS (DELETE FROM analytics_hourly_observer_activity te WHERE te.hour < $1::timestamptz), + f AS (DELETE FROM analytics_hourly_observer_identity tf WHERE tf.hour < $1::timestamptz), + g AS (DELETE FROM analytics_hourly_advert_hearings tg WHERE tg.hour < $1::timestamptz), + h AS (DELETE FROM analytics_hourly_advert_sets th WHERE th.hour < $1::timestamptz), + i AS (DELETE FROM analytics_hourly_talker_sets ti WHERE ti.hour < $1::timestamptz), + j AS (DELETE FROM analytics_hourly_packet_sets tj WHERE tj.hour < $1::timestamptz), + k AS (DELETE FROM analytics_hourly_scope_sets tk WHERE tk.hour < $1::timestamptz), + l AS (DELETE FROM analytics_dirty_hours tl WHERE tl.hour < $1::timestamptz), + m AS (DELETE FROM analytics_rollup_hours tm WHERE tm.hour < $1::timestamptz RETURNING 1) +UPDATE analytics_raw_state SET coverage = coverage + 1 WHERE EXISTS (SELECT 1 FROM m) +` + +// Bumps coverage when hours go, so cached responses that included them are not reused. +func (q *Queries) DeleteOldRollups(ctx context.Context, cutoff pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, deleteOldRollups, cutoff) + return err +} + +const deleteRollupHour = `-- name: DeleteRollupHour :exec +WITH a AS (DELETE FROM analytics_hourly_iata_observations ta WHERE ta.hour = $1::timestamptz), + b AS (DELETE FROM analytics_hourly_payload_breakdown tb WHERE tb.hour = $1::timestamptz), + c AS (DELETE FROM analytics_hourly_signal tc WHERE tc.hour = $1::timestamptz), + d AS (DELETE FROM analytics_hourly_paths td WHERE td.hour = $1::timestamptz), + e AS (DELETE FROM analytics_hourly_observer_activity te WHERE te.hour = $1::timestamptz), + f AS (DELETE FROM analytics_hourly_observer_identity tf WHERE tf.hour = $1::timestamptz), + g AS (DELETE FROM analytics_hourly_advert_hearings tg WHERE tg.hour = $1::timestamptz), + h AS (DELETE FROM analytics_hourly_advert_sets th WHERE th.hour = $1::timestamptz), + i AS (DELETE FROM analytics_hourly_talker_sets ti WHERE ti.hour = $1::timestamptz), + j AS (DELETE FROM analytics_hourly_packet_sets tj WHERE tj.hour = $1::timestamptz) +DELETE FROM analytics_hourly_scope_sets tk WHERE tk.hour = $1::timestamptz +` + +func (q *Queries) DeleteRollupHour(ctx context.Context, hour pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, deleteRollupHour, hour) + return err +} + +const deleteStaleDirtyHours = `-- name: DeleteStaleDirtyHours :exec +DELETE FROM analytics_dirty_hours d +WHERE EXISTS (SELECT 1 FROM analytics_rollup_hours h WHERE h.hour = d.hour AND h.status = 'partial') + OR d.hour <= (SELECT raw_deleted_before + INTERVAL '30 minutes' FROM analytics_raw_state) +` + +// Dirty hours that can no longer be re-rolled. +func (q *Queries) DeleteStaleDirtyHours(ctx context.Context) error { + _, err := q.db.Exec(ctx, deleteStaleDirtyHours) + return err +} + +const fillRollupObs = `-- name: FillRollupObs :exec +INSERT INTO rollup_obs (packet_hash, observer_id, iata, heard_at, path_length_byte, hash_size, hop_count, + path_len, rssi, snr, airtime_ms, payload_type, route_type, origin_pubkey, scope_id) +SELECT po.packet_hash, po.observer_id, po.iata, po.heard_at, po.path_length_byte, po.hash_size, po.hop_count, + COALESCE(octet_length(po.path_bytes), 0), po.rssi, po.snr, po.airtime_ms, po.payload_type, + p.route_type, p.origin_pubkey, p.scope_id +FROM packet_observations po +JOIN packets p ON p.packet_hash = po.packet_hash +WHERE po.heard_at >= $1::timestamptz AND po.heard_at < $1::timestamptz + INTERVAL '1 hour' +` + +func (q *Queries) FillRollupObs(ctx context.Context, dollar_1 pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, fillRollupObs, dollar_1) + return err +} + +const finishRollupHour = `-- name: FinishRollupHour :one +WITH prev AS ( + SELECT ph.content_hash FROM analytics_rollup_hours ph WHERE ph.hour = $1::timestamptz +), upd AS ( + UPDATE analytics_rollup_hours uh + SET status = 'complete', rolled_at = now(), content_hash = $2::text + WHERE uh.hour = $1::timestamptz +), undirty AS ( + DELETE FROM analytics_dirty_hours dh WHERE dh.hour = $1::timestamptz +), rev AS ( + UPDATE analytics_state SET revision = revision + 1 + WHERE (SELECT content_hash FROM prev) IS DISTINCT FROM $2::text + RETURNING revision +) +SELECT EXISTS (SELECT 1 FROM rev)::boolean AS changed +` + +type FinishRollupHourParams struct { + Hour pgtype.Timestamptz `json:"hour"` + ContentHash string `json:"content_hash"` +} + +// Marks the hour complete and bumps the revision only when its content changed. +func (q *Queries) FinishRollupHour(ctx context.Context, arg FinishRollupHourParams) (bool, error) { + row := q.db.QueryRow(ctx, finishRollupHour, arg.Hour, arg.ContentHash) + var changed bool + err := row.Scan(&changed) + return changed, err +} + +const getAnalyticsRevision = `-- name: GetAnalyticsRevision :one +SELECT (s.revision + r.coverage)::bigint FROM analytics_state s CROSS JOIN analytics_raw_state r +` + +// Both counters only grow, so their sum changes whenever rolled content or coverage does. +func (q *Queries) GetAnalyticsRevision(ctx context.Context) (int64, error) { + row := q.db.QueryRow(ctx, getAnalyticsRevision) + var column_1 int64 + err := row.Scan(&column_1) + return column_1, err +} + +const listDirtyRollupHours = `-- name: ListDirtyRollupHours :many +SELECT d.hour FROM analytics_dirty_hours d +JOIN analytics_rollup_hours h ON h.hour = d.hour AND h.status = 'complete' +CROSS JOIN analytics_raw_state r +WHERE r.raw_deleted_before IS NULL OR d.hour > r.raw_deleted_before + INTERVAL '30 minutes' +ORDER BY d.hour +LIMIT $1 +` + +// Only complete hours whose raw rows are still intact can be re-rolled. Queued missing hours +// wait: their first roll consumes the entry only if the entry predates its snapshot. +func (q *Queries) ListDirtyRollupHours(ctx context.Context, limit int32) ([]pgtype.Timestamptz, error) { + rows, err := q.db.Query(ctx, listDirtyRollupHours, limit) + if err != nil { + return nil, err + } + defer rows.Close() + items := []pgtype.Timestamptz{} + for rows.Next() { + var hour pgtype.Timestamptz + if err := rows.Scan(&hour); err != nil { + return nil, err + } + items = append(items, hour) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listMissingRollupHours = `-- name: ListMissingRollupHours :many +SELECT hour FROM analytics_rollup_hours +WHERE status = 'missing' +ORDER BY hour +LIMIT $1 +` + +func (q *Queries) ListMissingRollupHours(ctx context.Context, limit int32) ([]pgtype.Timestamptz, error) { + rows, err := q.db.Query(ctx, listMissingRollupHours, limit) + if err != nil { + return nil, err + } + defer rows.Close() + items := []pgtype.Timestamptz{} + for rows.Next() { + var hour pgtype.Timestamptz + if err := rows.Scan(&hour); err != nil { + return nil, err + } + items = append(items, hour) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const lockRollupHour = `-- name: LockRollupHour :one +SELECT CASE + WHEN r.raw_deleted_before IS NOT NULL AND h.hour <= r.raw_deleted_before + INTERVAL '30 minutes' + THEN 'partial' + ELSE h.status + END::text AS status +FROM analytics_rollup_hours h CROSS JOIN analytics_raw_state r +WHERE h.hour = $1 +FOR UPDATE OF h +` + +// Returns the hour's status, or 'partial' if raw deletion has reached it since registration. +func (q *Queries) LockRollupHour(ctx context.Context, hour pgtype.Timestamptz) (string, error) { + row := q.db.QueryRow(ctx, lockRollupHour, hour) + var status string + err := row.Scan(&status) + return status, err +} + +const markPartialRollupHours = `-- name: MarkPartialRollupHours :exec +WITH marked AS ( + UPDATE analytics_rollup_hours h SET status = 'partial' + FROM analytics_raw_state r + WHERE h.status = 'missing' + AND r.raw_deleted_before IS NOT NULL + AND h.hour <= r.raw_deleted_before + INTERVAL '30 minutes' + RETURNING 1 +) +UPDATE analytics_state SET revision = revision + 1 WHERE EXISTS (SELECT 1 FROM marked) +` + +// Hours whose raw rows cleanup already started deleting can never be rolled completely. +// Readers report the new status, so the revision moves too. +func (q *Queries) MarkPartialRollupHours(ctx context.Context) error { + _, err := q.db.Exec(ctx, markPartialRollupHours) + return err +} + +const oldestMissingRollupHour = `-- name: OldestMissingRollupHour :one +SELECT min(hour)::timestamptz FROM analytics_rollup_hours WHERE status = 'missing' +` + +func (q *Queries) OldestMissingRollupHour(ctx context.Context) (pgtype.Timestamptz, error) { + row := q.db.QueryRow(ctx, oldestMissingRollupHour) + var column_1 pgtype.Timestamptz + err := row.Scan(&column_1) + return column_1, err +} + +const registerRollupHours = `-- name: RegisterRollupHours :exec +INSERT INTO analytics_rollup_hours (hour, status) +SELECT h, 'missing' +FROM (SELECT CASE WHEN MIN(heard_at) IS NOT NULL + THEN date_trunc('hour', GREATEST(MIN(heard_at), $1::timestamptz), 'UTC') END AS first_hour + FROM packet_observations) f, + generate_series(f.first_hour, date_trunc('hour', now() - INTERVAL '95 minutes', 'UTC'), INTERVAL '1 hour') h +WHERE f.first_hour IS NOT NULL +ON CONFLICT (hour) DO NOTHING +` + +// Adds a 'missing' row for every eligible hour since the earliest retained observation +// (and not before @since), zero-observation hours included. +// No observations yet means nothing to register (GREATEST would ignore the NULL MIN). +func (q *Queries) RegisterRollupHours(ctx context.Context, since pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, registerRollupHours, since) + return err +} + +const rollAdvertHearings = `-- name: RollAdvertHearings :exec +INSERT INTO analytics_hourly_advert_hearings (hour, iata, origin_pubkey, observations, last_heard, name, node_type) +SELECT $1::timestamptz, r.iata, r.origin_pubkey, r.n, r.last_heard, n.name, n.node_type +FROM (SELECT iata, origin_pubkey, count(*) AS n, max(heard_at) AS last_heard + FROM rollup_obs + WHERE payload_type = 4 AND origin_pubkey IS NOT NULL + GROUP BY iata, origin_pubkey) r +LEFT JOIN nodes n ON n.public_key = r.origin_pubkey +` + +func (q *Queries) RollAdvertHearings(ctx context.Context, hour pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, rollAdvertHearings, hour) + return err +} + +const rollAdvertSets = `-- name: RollAdvertSets :exec +INSERT INTO analytics_hourly_advert_sets (hour, origin_pubkey, iatas, advert_packets, flood_packets, direct_packets) +SELECT $1::timestamptz, origin_pubkey, iatas, count(*), + count(*) FILTER (WHERE route_type IN (0, 1)), + count(*) FILTER (WHERE route_type IN (2, 3)) +FROM (SELECT origin_pubkey, route_type, array_agg(DISTINCT iata ORDER BY iata) AS iatas + FROM rollup_obs + WHERE payload_type = 4 AND origin_pubkey IS NOT NULL + GROUP BY packet_hash, origin_pubkey, route_type) p +GROUP BY origin_pubkey, iatas +` + +// Each distinct ADVERT packet counts once, under the exact set of IATAs that heard it this hour. +func (q *Queries) RollAdvertSets(ctx context.Context, hour pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, rollAdvertSets, hour) + return err +} + +const rollIATAObservations = `-- name: RollIATAObservations :exec +INSERT INTO analytics_hourly_iata_observations (hour, iata, observation_count) +SELECT $1::timestamptz, iata, count(*) FROM rollup_obs GROUP BY iata +` + +func (q *Queries) RollIATAObservations(ctx context.Context, hour pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, rollIATAObservations, hour) + return err +} + +const rollObserverActivity = `-- name: RollObserverActivity :exec +INSERT INTO analytics_hourly_observer_activity (hour, observer_id, payload_type, observations, airtime_ms, airtime_n, + snr_sum, snr_n, snr_min, rssi_sum, rssi_n) +SELECT $1::timestamptz, observer_id, COALESCE(payload_type, -1)::smallint, + count(*), sum(airtime_ms)::real, count(airtime_ms), + sum(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real, + count(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0)), + min(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real, + sum(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint, + count(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0)) +FROM rollup_obs +GROUP BY observer_id, COALESCE(payload_type, -1) +` + +func (q *Queries) RollObserverActivity(ctx context.Context, hour pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, rollObserverActivity, hour) + return err +} + +const rollObserverIdentity = `-- name: RollObserverIdentity :exec +INSERT INTO analytics_hourly_observer_identity (hour, iata, observer_id, observation_count, display_name, observer_type) +SELECT $1::timestamptz, r.iata, r.observer_id, r.n, o.display_name, o.observer_type +FROM (SELECT iata, observer_id, count(*) AS n FROM rollup_obs GROUP BY iata, observer_id) r +LEFT JOIN observers o ON o.id = r.observer_id +` + +func (q *Queries) RollObserverIdentity(ctx context.Context, hour pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, rollObserverIdentity, hour) + return err +} + +const rollPacketSets = `-- name: RollPacketSets :exec +INSERT INTO analytics_hourly_packet_sets (hour, iatas, packets) +SELECT $1::timestamptz, iatas, count(*) +FROM (SELECT array_agg(DISTINCT iata ORDER BY iata) AS iatas FROM rollup_obs GROUP BY packet_hash) p +GROUP BY iatas +` + +func (q *Queries) RollPacketSets(ctx context.Context, hour pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, rollPacketSets, hour) + return err +} + +const rollPaths = `-- name: RollPaths :exec +INSERT INTO analytics_hourly_paths (hour, iata, category, hash_bytes, entries, receptions) +SELECT $1::timestamptz, c.iata, c.category, c.hash_bytes, c.entries, count(*) +FROM ( + SELECT k.iata, k.category, + CASE WHEN k.category = 0 THEN k.hash_size ELSE 0 END::integer AS hash_bytes, + CASE WHEN k.category = 0 THEN k.hop_count ELSE 0 END::integer AS entries + FROM ( + SELECT iata, hash_size, hop_count, + CASE WHEN payload_type = 9 THEN 2 + WHEN payload_type IS NULL OR payload_type NOT BETWEEN 0 AND 15 + OR NOT (path_length_byte BETWEEN 0 AND 191 + AND hash_size BETWEEN 1 AND 3 AND hop_count BETWEEN 0 AND 63 + AND hash_size = (path_length_byte >> 6) + 1 + AND hop_count = (path_length_byte & 63) + AND hash_size::integer * hop_count::integer <= 64 + AND path_len = hash_size::integer * hop_count::integer) + THEN 3 + WHEN hop_count = 0 THEN 1 + ELSE 0 END::integer AS category + FROM rollup_obs + ) k +) c +GROUP BY c.iata, c.category, c.hash_bytes, c.entries +` + +// category: 0 = routed, 1 = zero-hop, 2 = TRACE, 3 = invalid (meshcore-go IsValidPathLen). +func (q *Queries) RollPaths(ctx context.Context, hour pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, rollPaths, hour) + return err +} + +const rollPayloadBreakdown = `-- name: RollPayloadBreakdown :exec +INSERT INTO analytics_hourly_payload_breakdown (hour, iata, payload_type, count) +SELECT $1::timestamptz, iata, payload_type, count(*) +FROM rollup_obs WHERE payload_type IS NOT NULL +GROUP BY iata, payload_type +` + +func (q *Queries) RollPayloadBreakdown(ctx context.Context, hour pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, rollPayloadBreakdown, hour) + return err +} + +const rollScopeSets = `-- name: RollScopeSets :exec +INSERT INTO analytics_hourly_scope_sets (hour, iatas, scope_id, packets) +SELECT $1::timestamptz, iatas, scope_id, count(*) +FROM (SELECT scope_id, array_agg(DISTINCT iata ORDER BY iata) AS iatas + FROM rollup_obs WHERE scope_id IS NOT NULL + GROUP BY packet_hash, scope_id) p +GROUP BY iatas, scope_id +` + +func (q *Queries) RollScopeSets(ctx context.Context, hour pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, rollScopeSets, hour) + return err +} + +const rollSignal = `-- name: RollSignal :exec +INSERT INTO analytics_hourly_signal (hour, iata, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum) +SELECT $1::timestamptz, b.iata, grouping(b.snr_bin, b.rssi_bin)::integer, + COALESCE(b.snr_bin, -1)::integer, COALESCE(b.rssi_bin, -1)::integer, + count(*), count(b.snr), COALESCE(sum(b.snr), 0)::double precision, + count(b.rssi), COALESCE(sum(b.rssi), 0)::double precision +FROM ( + SELECT s.iata, s.snr, s.rssi, + width_bucket(s.snr, -30, 30, 12) AS snr_bin, + width_bucket(s.rssi, -140, 0, 14) AS rssi_bin + FROM ( + SELECT ro.iata, + CASE WHEN NOT (COALESCE(ro.rssi, 0) = 0 AND COALESCE(ro.snr, 0) = 0) + AND ro.snr > '-Infinity'::real AND ro.snr < 'Infinity'::real + THEN ro.snr::double precision END AS snr, + CASE WHEN NOT (COALESCE(ro.rssi, 0) = 0 AND COALESCE(ro.snr, 0) = 0) + THEN ro.rssi::double precision END AS rssi + FROM rollup_obs ro + ) s +) b +GROUP BY GROUPING SETS ((b.iata), (b.iata, b.snr_bin), (b.iata, b.rssi_bin)) +` + +// kind = grouping(snr_bin, rssi_bin): 3 = per IATA totals, 1 = per SNR bin, 2 = per RSSI bin. +// rssi=0 AND snr=0 means "not reported". +func (q *Queries) RollSignal(ctx context.Context, hour pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, rollSignal, hour) + return err +} + +const rollTalkerSets = `-- name: RollTalkerSets :exec +INSERT INTO analytics_hourly_talker_sets (hour, sender_name, iatas, messages, last_sent) +SELECT $1::timestamptz, sender_name, iatas, count(*), max(sent_at) +FROM (SELECT cm.sender_name, max(cm.sent_at) AS sent_at, array_agg(DISTINCT ro.iata ORDER BY ro.iata) AS iatas + FROM rollup_obs ro + JOIN channel_messages cm ON cm.packet_hash = ro.packet_hash + WHERE cm.sender_name IS NOT NULL + GROUP BY cm.id, cm.sender_name) m +GROUP BY sender_name, iatas +` + +func (q *Queries) RollTalkerSets(ctx context.Context, hour pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, rollTalkerSets, hour) + return err +} + +const rollupContentHash = `-- name: RollupContentHash :one +SELECT md5(concat_ws('|', + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_iata_observations t WHERE t.hour = $1::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_payload_breakdown t WHERE t.hour = $1::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_signal t WHERE t.hour = $1::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_paths t WHERE t.hour = $1::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_observer_activity t WHERE t.hour = $1::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_observer_identity t WHERE t.hour = $1::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_advert_hearings t WHERE t.hour = $1::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_advert_sets t WHERE t.hour = $1::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_talker_sets t WHERE t.hour = $1::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_packet_sets t WHERE t.hour = $1::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_scope_sets t WHERE t.hour = $1::timestamptz) +))::text AS content_hash +` + +// Order-independent fingerprint of every family row for the hour. +func (q *Queries) RollupContentHash(ctx context.Context, hour pgtype.Timestamptz) (string, error) { + row := q.db.QueryRow(ctx, rollupContentHash, hour) + var content_hash string + err := row.Scan(&content_hash) + return content_hash, err +} + +const rollupUnlock = `-- name: RollupUnlock :exec +SELECT pg_advisory_unlock(25189, 1) +` + +func (q *Queries) RollupUnlock(ctx context.Context) error { + _, err := q.db.Exec(ctx, rollupUnlock) + return err +} + +const setRollupHourPartial = `-- name: SetRollupHourPartial :exec +WITH marked AS ( + UPDATE analytics_rollup_hours SET status = 'partial' WHERE hour = $1 AND status = 'missing' RETURNING 1 +) +UPDATE analytics_state SET revision = revision + 1 WHERE EXISTS (SELECT 1 FROM marked) +` + +func (q *Queries) SetRollupHourPartial(ctx context.Context, hour pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, setRollupHourPartial, hour) + return err +} + +const tryRollupLock = `-- name: TryRollupLock :one + +SELECT pg_try_advisory_lock(25189, 1) +` + +// ============================================================ +// HOURLY ANALYTICS ROLLUPS +// An hour [H, H+1h) is eligible once now() >= H + 95 min (ingest clamps heard_at to ±30 min). +// ============================================================ +func (q *Queries) TryRollupLock(ctx context.Context) (bool, error) { + row := q.db.QueryRow(ctx, tryRollupLock) + var pg_try_advisory_lock bool + err := row.Scan(&pg_try_advisory_lock) + return pg_try_advisory_lock, err +} diff --git a/db/sqlc/series.sql.go b/db/sqlc/series.sql.go new file mode 100644 index 0000000..cf4e690 --- /dev/null +++ b/db/sqlc/series.sql.go @@ -0,0 +1,169 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: series.sql + +package db + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const getEarliestCompleteRollupHour = `-- name: GetEarliestCompleteRollupHour :one +SELECT min(hour)::timestamptz FROM analytics_rollup_hours WHERE status = 'complete' +` + +func (q *Queries) GetEarliestCompleteRollupHour(ctx context.Context) (pgtype.Timestamptz, error) { + row := q.db.QueryRow(ctx, getEarliestCompleteRollupHour) + var column_1 pgtype.Timestamptz + err := row.Scan(&column_1) + return column_1, err +} + +const getStatsSeries = `-- name: GetStatsSeries :many +WITH hours AS ( + SELECT g.h::timestamptz AS hour, COALESCE(r.status, 'missing')::text AS status + FROM generate_series($1::timestamptz, $2::timestamptz, INTERVAL '1 hour') g(h) + LEFT JOIN analytics_rollup_hours r ON r.hour = g.h + WHERE g.h < $2::timestamptz +), complete AS ( + SELECT hour FROM hours WHERE status = 'complete' +), obs AS ( + SELECT o.hour, o.iata, o.observation_count FROM analytics_hourly_iata_observations o JOIN complete c ON c.hour = o.hour + WHERE COALESCE(cardinality($3::bpchar[]), 0) = 0 + UNION ALL + SELECT o.hour, o.iata, o.observation_count FROM analytics_hourly_iata_observations o JOIN complete c ON c.hour = o.hour + WHERE cardinality($3::bpchar[]) > 0 AND o.iata = ANY($3::bpchar[]) +), obs_agg AS ( + SELECT hour, sum(observation_count)::bigint AS observations, count(DISTINCT iata)::bigint AS active_iatas + FROM obs GROUP BY GROUPING SETS ((), (hour)) +), pkt AS ( + SELECT s.hour, s.packets FROM analytics_hourly_packet_sets s JOIN complete c ON c.hour = s.hour + WHERE COALESCE(cardinality($3::bpchar[]), 0) = 0 + UNION ALL + SELECT s.hour, s.packets FROM analytics_hourly_packet_sets s JOIN complete c ON c.hour = s.hour + WHERE cardinality($3::bpchar[]) > 0 AND s.iatas && $3::bpchar[] +), pkt_agg AS ( + SELECT hour, sum(packets)::bigint AS unique_packets FROM pkt GROUP BY GROUPING SETS ((), (hour)) +), obsr AS ( + SELECT i.hour, i.observer_id FROM analytics_hourly_observer_identity i JOIN complete c ON c.hour = i.hour + WHERE COALESCE(cardinality($3::bpchar[]), 0) = 0 + UNION ALL + SELECT i.hour, i.observer_id FROM analytics_hourly_observer_identity i JOIN complete c ON c.hour = i.hour + WHERE cardinality($3::bpchar[]) > 0 AND i.iata = ANY($3::bpchar[]) +), obsr_agg AS ( + SELECT hour, count(DISTINCT observer_id)::bigint AS active_observers FROM obsr GROUP BY GROUPING SETS ((), (hour)) +), scp AS ( + SELECT s.hour, s.scope_id, s.packets FROM analytics_hourly_scope_sets s JOIN complete c ON c.hour = s.hour + WHERE COALESCE(cardinality($3::bpchar[]), 0) = 0 + UNION ALL + SELECT s.hour, s.scope_id, s.packets FROM analytics_hourly_scope_sets s JOIN complete c ON c.hour = s.hour + WHERE cardinality($3::bpchar[]) > 0 AND s.iatas && $3::bpchar[] +), scp_agg AS ( + SELECT hour, sum(packets)::bigint AS scoped_packets, count(DISTINCT scope_id)::bigint AS active_scopes + FROM scp GROUP BY GROUPING SETS ((), (hour)) +), pth AS ( + SELECT p.hour, p.entries FROM analytics_hourly_paths p JOIN complete c ON c.hour = p.hour + WHERE p.category = 0 AND COALESCE(cardinality($3::bpchar[]), 0) = 0 + UNION ALL + SELECT p.hour, p.entries FROM analytics_hourly_paths p JOIN complete c ON c.hour = p.hour + WHERE p.category = 0 AND cardinality($3::bpchar[]) > 0 AND p.iata = ANY($3::bpchar[]) +), pth_agg AS ( + SELECT hour, max(entries)::integer AS max_path_entries FROM pth GROUP BY GROUPING SETS ((), (hour)) +), sig AS ( + SELECT s.hour, s.snr_sum, s.snr_samples, s.rssi_sum, s.rssi_samples FROM analytics_hourly_signal s JOIN complete c ON c.hour = s.hour + WHERE s.kind = 3 AND COALESCE(cardinality($3::bpchar[]), 0) = 0 + UNION ALL + SELECT s.hour, s.snr_sum, s.snr_samples, s.rssi_sum, s.rssi_samples FROM analytics_hourly_signal s JOIN complete c ON c.hour = s.hour + WHERE s.kind = 3 AND cardinality($3::bpchar[]) > 0 AND s.iata = ANY($3::bpchar[]) +), sig_agg AS ( + SELECT hour, sum(snr_sum)::double precision AS snr_sum, sum(snr_samples)::bigint AS snr_samples, + sum(rssi_sum)::double precision AS rssi_sum, sum(rssi_samples)::bigint AS rssi_samples + FROM sig GROUP BY GROUPING SETS ((), (hour)) +), keys AS ( + SELECT hour, status FROM hours + UNION ALL + SELECT NULL::timestamptz, 'summary'::text +) +SELECT k.hour, k.status, + COALESCE(o.observations, 0)::bigint AS observations, + COALESCE(p.unique_packets, 0)::bigint AS unique_packets, + COALESCE(ob.active_observers, 0)::bigint AS active_observers, + COALESCE(o.active_iatas, 0)::bigint AS active_iatas, + COALESCE(s.scoped_packets, 0)::bigint AS scoped_packets, + COALESCE(s.active_scopes, 0)::bigint AS active_scopes, + COALESCE(pa.max_path_entries, 0)::integer AS max_path_entries, + COALESCE(sg.snr_sum, 0)::double precision AS snr_sum, + COALESCE(sg.snr_samples, 0)::bigint AS snr_samples, + COALESCE(sg.rssi_sum, 0)::double precision AS rssi_sum, + COALESCE(sg.rssi_samples, 0)::bigint AS rssi_samples +FROM keys k +LEFT JOIN obs_agg o ON o.hour IS NOT DISTINCT FROM k.hour +LEFT JOIN pkt_agg p ON p.hour IS NOT DISTINCT FROM k.hour +LEFT JOIN obsr_agg ob ON ob.hour IS NOT DISTINCT FROM k.hour +LEFT JOIN scp_agg s ON s.hour IS NOT DISTINCT FROM k.hour +LEFT JOIN pth_agg pa ON pa.hour IS NOT DISTINCT FROM k.hour +LEFT JOIN sig_agg sg ON sg.hour IS NOT DISTINCT FROM k.hour +ORDER BY k.hour NULLS LAST +` + +type GetStatsSeriesParams struct { + Since pgtype.Timestamptz `json:"since"` + Until pgtype.Timestamptz `json:"until"` + Iatas []string `json:"iatas"` +} + +type GetStatsSeriesRow struct { + Hour pgtype.Timestamptz `json:"hour"` + Status string `json:"status"` + Observations int64 `json:"observations"` + UniquePackets int64 `json:"unique_packets"` + ActiveObservers int64 `json:"active_observers"` + ActiveIatas int64 `json:"active_iatas"` + ScopedPackets int64 `json:"scoped_packets"` + ActiveScopes int64 `json:"active_scopes"` + MaxPathEntries int32 `json:"max_path_entries"` + SnrSum float64 `json:"snr_sum"` + SnrSamples int64 `json:"snr_samples"` + RssiSum float64 `json:"rssi_sum"` + RssiSamples int64 `json:"rssi_samples"` +} + +// One row per hour in [since, until) plus a summary row (hour NULL) over its complete hours. +// Counts sum; observers, IATAs and scopes are distinct across the window; packets count +// once per hour heard. Each branch pair keeps "all IATAs" and "these IATAs" plans separate. +func (q *Queries) GetStatsSeries(ctx context.Context, arg GetStatsSeriesParams) ([]GetStatsSeriesRow, error) { + rows, err := q.db.Query(ctx, getStatsSeries, arg.Since, arg.Until, arg.Iatas) + if err != nil { + return nil, err + } + defer rows.Close() + items := []GetStatsSeriesRow{} + for rows.Next() { + var i GetStatsSeriesRow + if err := rows.Scan( + &i.Hour, + &i.Status, + &i.Observations, + &i.UniquePackets, + &i.ActiveObservers, + &i.ActiveIatas, + &i.ScopedPackets, + &i.ActiveScopes, + &i.MaxPathEntries, + &i.SnrSum, + &i.SnrSamples, + &i.RssiSum, + &i.RssiSamples, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/db/sqlc/signal.sql.go b/db/sqlc/signal.sql.go index 8a49049..abca971 100644 --- a/db/sqlc/signal.sql.go +++ b/db/sqlc/signal.sql.go @@ -13,11 +13,11 @@ import ( const getSignalStats = `-- name: GetSignalStats :many WITH readings AS ( - SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM mv_signal_stats_hourly + SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM analytics_hourly_signal WHERE hour >= $1::timestamptz AND hour < $2::timestamptz AND COALESCE(cardinality($3::bpchar[]), 0) = 0 UNION ALL - SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM mv_signal_stats_hourly + SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM analytics_hourly_signal WHERE hour >= $1::timestamptz AND hour < $2::timestamptz AND cardinality($3::bpchar[]) > 0 AND iata = ANY($3::bpchar[]) ) @@ -57,7 +57,7 @@ type GetSignalStatsRow struct { RssiAverage float64 `json:"rssi_average"` } -// Read compact hourly snapshots, never observations on an HTTP request. +// Read the hourly rollup, never observations on an HTTP request. // Weight averages by sample counts instead of averaging regional/hourly means. func (q *Queries) GetSignalStats(ctx context.Context, arg GetSignalStatsParams) ([]GetSignalStatsRow, error) { rows, err := q.db.Query(ctx, getSignalStats, arg.Since, arg.Until, arg.Iatas) @@ -88,12 +88,3 @@ func (q *Queries) GetSignalStats(ctx context.Context, arg GetSignalStatsParams) } return items, nil } - -const refreshSignalStats = `-- name: RefreshSignalStats :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_signal_stats_hourly -` - -func (q *Queries) RefreshSignalStats(ctx context.Context) error { - _, err := q.db.Exec(ctx, refreshSignalStats) - return err -} diff --git a/db/sqlc_scratch/rollup_obs.sql b/db/sqlc_scratch/rollup_obs.sql new file mode 100644 index 0000000..660e325 --- /dev/null +++ b/db/sqlc_scratch/rollup_obs.sql @@ -0,0 +1,9 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- sqlc type-check stub only; db/rollup.go creates this as a TEMP table per rolled hour. +CREATE TABLE rollup_obs ( + packet_hash bytea NOT NULL, observer_id uuid NOT NULL, iata character(3) NOT NULL, + heard_at timestamptz NOT NULL, path_length_byte smallint NOT NULL, hash_size smallint NOT NULL, + hop_count smallint NOT NULL, path_len integer NOT NULL, rssi smallint, snr real, airtime_ms real, + payload_type smallint, route_type smallint NOT NULL, origin_pubkey bytea, scope_id integer); diff --git a/db/stats.go b/db/stats.go index 0d91df7..5bda12c 100644 --- a/db/stats.go +++ b/db/stats.go @@ -11,32 +11,55 @@ import ( sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/google/uuid" "github.com/jackc/pgx/v5/pgtype" ) +// rollupSince snaps a window start to its UTC hour; zero means def before now. +func rollupSince(since time.Time, def time.Duration) pgtype.Timestamptz { + if since.IsZero() { + since = time.Now().Add(-def) + } + return ts(since.UTC().Truncate(time.Hour)) +} + +// optional maps the ” the rollup queries use for a missing name back to nil. +func optional(s string) *string { + if s == "" { + return nil + } + return &s +} + +func pgUUID(id pgtype.UUID) *uuid.UUID { + if !id.Valid { + return nil + } + u := uuid.UUID(id.Bytes) + return &u +} + +// GetStatsOverview summarises the 24 most recent hours that can have been rolled. func (s *Store) GetStatsOverview(ctx context.Context, iatas []string) (*api.StatsOverview, error) { - row, err := s.q.GetStatsOverview(ctx, iatas) + until := time.Now().UTC().Add(-rollupDelay).Truncate(time.Hour).Add(time.Hour) + since := until.Add(-24 * time.Hour) + series, err := s.GetStatsSeries(ctx, since, until, iatas) if err != nil { return nil, err } return &api.StatsOverview{ - TotalPackets: row.TotalPackets, - TotalObservations: row.TotalObservations, - ActiveObservers: row.ActiveObservers, - ActiveIATAs: row.ActiveIatas, + TotalPackets: series.Summary.UniquePackets, + TotalObservations: series.Summary.Observations, + ActiveObservers: series.Summary.ActiveObservers, + ActiveIATAs: series.Summary.ActiveIATAs, WindowHours: 24, + Since: since.UnixMilli(), + Until: until.UnixMilli(), }, nil } func (s *Store) GetStatsObservations(ctx context.Context, iatas []string, since time.Time) ([]api.ObservationPoint, error) { - if since.IsZero() { - since = time.Now().Add(-7 * 24 * time.Hour) - } - interval := time.Since(since) - rows, err := s.q.GetHourlyStats(ctx, sqlc.GetHourlyStatsParams{ - Column1: iatas, - Column2: pgtype.Interval{Microseconds: int64(interval.Hours()) * 3600 * 1e6, Valid: true}, - }) + rows, err := s.q.GetHourlyStats(ctx, sqlc.GetHourlyStatsParams{Iatas: iatas, Since: rollupSince(since, 7*24*time.Hour)}) if err != nil { return nil, err } @@ -46,60 +69,42 @@ func (s *Store) GetStatsObservations(ctx context.Context, iatas []string, since Hour: v.Hour.Time.UnixMilli(), IATA: v.Iata, ObservationCount: v.ObservationCount, - UniquePackets: v.UniquePackets, - ActiveObservers: v.ActiveObservers, }) } return points, nil } func (s *Store) GetStatsPayloadBreakdown(ctx context.Context, iatas []string, since time.Time) ([]api.PayloadBreakdownItem, error) { - if since.IsZero() { - since = time.Now().Add(-24 * time.Hour) - } - interval := time.Since(since) - rows, err := s.q.GetStatsPayloadBreakdown(ctx, sqlc.GetStatsPayloadBreakdownParams{ - Column1: iatas, - Column2: pgtype.Interval{Microseconds: int64(interval.Hours()) * 3600 * 1e6, Valid: true}, - }) + rows, err := s.q.GetStatsPayloadBreakdown(ctx, sqlc.GetStatsPayloadBreakdownParams{Iatas: iatas, Since: rollupSince(since, 24*time.Hour)}) if err != nil { return nil, err } items := make([]api.PayloadBreakdownItem, 0, len(rows)) for _, v := range rows { - if v.PayloadType == nil { - continue - } items = append(items, api.PayloadBreakdownItem{ - PayloadType: *v.PayloadType, - PayloadTypeName: api.PayloadTypeName(*v.PayloadType), + PayloadType: v.PayloadType, + PayloadTypeName: api.PayloadTypeName(v.PayloadType), Count: v.Count, }) } return items, nil } -func (s *Store) GetStatsTopNodes(ctx context.Context, iatas []string, limit int32) ([]api.TopNode, error) { - rows, err := s.q.GetTopNodes(ctx, sqlc.GetTopNodesParams{ - Column1: iatas, - Limit: limit, - }) +func (s *Store) GetStatsTopNodes(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopNode, error) { + rows, err := s.q.GetTopNodes(ctx, sqlc.GetTopNodesParams{Since: rollupSince(since, 7*24*time.Hour), Iatas: iatas, RowLimit: limit}) if err != nil { return nil, err } items := make([]api.TopNode, 0, len(rows)) for _, v := range rows { - var count int64 - if v.ObservationCount != nil { - count = *v.ObservationCount - } items = append(items, api.TopNode{ - NodeID: v.NodeID, - NodeName: v.Name, + NodeID: pgUUID(v.NodeID), + PublicKey: v.PublicKey, + NodeName: optional(v.Name), NodeType: v.NodeType, NodeTypeName: api.NodeTypeName(v.NodeType), IATA: v.Iata, - ObservationCount: count, + ObservationCount: v.ObservationCount, LastHeard: v.LastHeard.Time.UnixMilli(), }) } @@ -107,15 +112,7 @@ func (s *Store) GetStatsTopNodes(ctx context.Context, iatas []string, limit int3 } func (s *Store) GetStatsTopObservers(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopObserver, error) { - if since.IsZero() { - since = time.Now().Add(-24 * time.Hour) - } - interval := time.Since(since) - rows, err := s.q.GetStatsTopObservers(ctx, sqlc.GetStatsTopObserversParams{ - Column1: pgtype.Interval{Microseconds: int64(interval.Hours()) * 3600 * 1e6, Valid: true}, - Column2: iatas, - Limit: limit, - }) + rows, err := s.q.GetStatsTopObservers(ctx, sqlc.GetStatsTopObserversParams{Since: rollupSince(since, 24*time.Hour), Iatas: iatas, RowLimit: limit}) if err != nil { return nil, err } @@ -123,8 +120,8 @@ func (s *Store) GetStatsTopObservers(ctx context.Context, iatas []string, since for _, v := range rows { items = append(items, api.TopObserver{ ObserverID: v.ID, - DisplayName: v.DisplayName, - ObserverType: v.ObserverType, + DisplayName: optional(v.DisplayName), + ObserverType: optional(v.ObserverType), IATA: v.Iata, ObservationCount: v.ObservationCount, }) @@ -133,23 +130,16 @@ func (s *Store) GetStatsTopObservers(ctx context.Context, iatas []string, since } func (s *Store) GetStatsTopAdvertisers(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopAdvertiser, error) { - if since.IsZero() { - since = time.Now().Add(-24 * time.Hour) - } - interval := time.Since(since) - rows, err := s.q.GetStatsTopAdvertisers(ctx, sqlc.GetStatsTopAdvertisersParams{ - Column1: pgtype.Interval{Microseconds: int64(interval.Hours()) * 3600 * 1e6, Valid: true}, - Column2: iatas, - Limit: limit, - }) + rows, err := s.q.GetStatsTopAdvertisers(ctx, sqlc.GetStatsTopAdvertisersParams{Since: rollupSince(since, 24*time.Hour), Iatas: iatas, RowLimit: limit}) if err != nil { return nil, err } items := make([]api.TopAdvertiser, 0, len(rows)) for _, v := range rows { items = append(items, api.TopAdvertiser{ - NodeID: v.ID, - NodeName: v.Name, + NodeID: pgUUID(v.NodeID), + PublicKey: v.PublicKey, + NodeName: optional(v.Name), NodeType: v.NodeType, NodeTypeName: api.NodeTypeName(v.NodeType), IATA: v.Iata, @@ -196,26 +186,14 @@ func (s *Store) GetStatsClockDrift(ctx context.Context, iatas []string, limit in } func (s *Store) GetStatsTopTalkers(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopTalker, error) { - if since.IsZero() { - since = time.Now().Add(-24 * time.Hour) - } - interval := time.Since(since) - rows, err := s.q.GetStatsTopTalkers(ctx, sqlc.GetStatsTopTalkersParams{ - Column1: pgtype.Interval{Microseconds: int64(interval.Hours()) * 3600 * 1e6, Valid: true}, - Column2: iatas, - Limit: limit, - }) + rows, err := s.q.GetStatsTopTalkers(ctx, sqlc.GetStatsTopTalkersParams{Since: rollupSince(since, 24*time.Hour), Iatas: iatas, RowLimit: limit}) if err != nil { return nil, err } items := make([]api.TopTalker, 0, len(rows)) for _, v := range rows { - var senderName string - if v.SenderName != nil { - senderName = *v.SenderName - } items = append(items, api.TopTalker{ - SenderName: senderName, + SenderName: v.SenderName, MessageCount: v.MessageCount, LastSent: v.LastSent.Time.UnixMilli(), }) @@ -233,9 +211,12 @@ func (s *Store) GetRadioPresets(ctx context.Context, preset string, iatas []stri } items := make([]api.RadioPreset, 0, len(rows)) for _, v := range rows { + if v.Iata == nil { + continue + } items = append(items, api.RadioPreset{ Preset: v.Preset, - IATA: v.Iata, + IATA: *v.Iata, SourceType: v.SourceType, Count: v.Count, }) @@ -243,8 +224,8 @@ func (s *Store) GetRadioPresets(ctx context.Context, preset string, iatas []stri return items, nil } -func (s *Store) GetScopeStats(ctx context.Context, iatas []string) ([]api.ScopeStats, error) { - rows, err := s.q.GetScopeStats(ctx, iatas) +func (s *Store) GetScopeStats(ctx context.Context, iatas []string, since time.Time) ([]api.ScopeStats, error) { + rows, err := s.q.GetScopeStats(ctx, sqlc.GetScopeStatsParams{Since: rollupSince(since, 7*24*time.Hour), Iatas: iatas}) if err != nil { return nil, err } @@ -276,34 +257,6 @@ func (s *Store) GetStatsNodeTypes(ctx context.Context, iatas []string) ([]api.No return result, nil } -func (s *Store) RefreshHourlyStats(ctx context.Context) error { - return s.q.RefreshHourlyStats(ctx) -} - -func (s *Store) RefreshTopNodes(ctx context.Context) error { - return s.q.RefreshTopNodes(ctx) -} - -func (s *Store) RefreshPayloadBreakdown(ctx context.Context) error { - return s.q.RefreshPayloadBreakdown(ctx) -} - -func (s *Store) RefreshTopTalkers(ctx context.Context) error { - return s.q.RefreshTopTalkers(ctx) -} - -func (s *Store) RefreshTopAdvertisers(ctx context.Context) error { - return s.q.RefreshTopAdvertisers(ctx) -} - -func (s *Store) RefreshTopObservers(ctx context.Context) error { - return s.q.RefreshTopObservers(ctx) -} - func (s *Store) RefreshRadioPresets(ctx context.Context) error { return s.q.RefreshRadioPresets(ctx) } - -func (s *Store) RefreshObserverActivity(ctx context.Context) error { - return s.q.RefreshObserverActivity(ctx) -} diff --git a/db/stats_series.go b/db/stats_series.go new file mode 100644 index 0000000..c905f55 --- /dev/null +++ b/db/stats_series.go @@ -0,0 +1,64 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" +) + +func (s *Store) GetStatsSeries(ctx context.Context, since, until time.Time, iatas []string) (*api.StatsSeries, error) { + // Read the revision first: a roll landing mid-request then shows up as a newer revision next time. + rev, err := s.q.GetAnalyticsRevision(ctx) + if err != nil { + return nil, err + } + earliest, err := s.q.GetEarliestCompleteRollupHour(ctx) + if err != nil { + return nil, err + } + rows, err := s.q.GetStatsSeries(ctx, sqlc.GetStatsSeriesParams{Since: ts(since), Until: ts(until), Iatas: iatas}) + if err != nil { + return nil, err + } + out := &api.StatsSeries{ + Since: since.UnixMilli(), + Until: until.UnixMilli(), + Revision: rev, + Hours: make([]api.StatsSeriesHour, 0, len(rows)), + } + if earliest.Valid { + ms := earliest.Time.UnixMilli() + out.EarliestComplete = &ms + } + for _, r := range rows { + v := api.StatsSeriesValues{ + Observations: r.Observations, + UniquePackets: r.UniquePackets, + ActiveObservers: r.ActiveObservers, + ActiveIATAs: r.ActiveIatas, + ScopedPackets: r.ScopedPackets, + ActiveScopes: r.ActiveScopes, + MaxPathEntries: r.MaxPathEntries, + SNRSum: r.SnrSum, + SNRSamples: r.SnrSamples, + RSSISum: r.RssiSum, + RSSISamples: r.RssiSamples, + } + if !r.Hour.Valid { + out.Summary = v + continue + } + h := api.StatsSeriesHour{Hour: r.Hour.Time.UnixMilli(), Status: r.Status} + if r.Status == "complete" { + h.Values = &v + out.CompleteHours++ + } + out.Hours = append(out.Hours, h) + } + return out, nil +} diff --git a/db/stats_series_integration_test.go b/db/stats_series_integration_test.go new file mode 100644 index 0000000..627d5e1 --- /dev/null +++ b/db/stats_series_integration_test.go @@ -0,0 +1,126 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" +) + +// seriesOracle computes a summary straight from raw rows over [since, until). +const seriesOracle = ` +WITH po AS ( + SELECT po.*, date_trunc('hour', po.heard_at, 'UTC') AS hour, p.scope_id + FROM packet_observations po JOIN packets p ON p.packet_hash = po.packet_hash + WHERE po.heard_at >= $1 AND po.heard_at < $2 + AND (cardinality($3::bpchar[]) = 0 OR po.iata = ANY($3::bpchar[])) + AND date_trunc('hour', po.heard_at, 'UTC') <> ALL($4::timestamptz[]) +), sig AS ( + SELECT * FROM po WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) +) +SELECT (SELECT count(*) FROM po), + (SELECT count(*) FROM (SELECT DISTINCT hour, packet_hash FROM po) d), + (SELECT count(DISTINCT observer_id) FROM po), + (SELECT count(DISTINCT iata) FROM po), + (SELECT count(*) FROM (SELECT DISTINCT hour, packet_hash FROM po WHERE scope_id IS NOT NULL) d), + (SELECT count(DISTINCT scope_id) FROM po), + (SELECT COALESCE(sum(snr), 0)::double precision FROM sig), (SELECT count(snr) FROM sig), + (SELECT COALESCE(sum(rssi), 0)::double precision FROM sig), (SELECT count(rssi) FROM sig)` + +func TestStatsSeriesPostgres(t *testing.T) { + ctx, pool := schemaPool(t) + h0 := rollupFixture(t, ctx, pool) + store := New(pool, 0, 0) + r, ok, err := store.BeginRollup(ctx) + if err != nil || !ok { + t.Fatalf("BeginRollup: ok=%v err=%v", ok, err) + } + if err := r.RegisterHours(ctx, h0); err != nil { + t.Fatal(err) + } + rollAll(t, ctx, r) + r.Close(ctx) + since, until := h0, h0.Add(3*time.Hour) + + for _, tc := range []struct { + name string + iatas []string + excluded []time.Time // hours not complete + maxPath int32 + }{ + {"all", nil, nil, 2}, + {"one IATA", []string{"YYZ"}, nil, 1}, + {"overlapping IATAs", []string{"YVR", "YYZ", "YVR"}, nil, 2}, + {"empty region", []string{""}, nil, 0}, + {"partial hour excluded", nil, []time.Time{h0.Add(time.Hour)}, 2}, + } { + t.Run(tc.name, func(t *testing.T) { + tx, err := pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(ctx) + for _, h := range tc.excluded { + if _, err := tx.Exec(ctx, "UPDATE analytics_rollup_hours SET status = 'partial' WHERE hour = $1", h); err != nil { + t.Fatal(err) + } + } + got, err := (&Store{q: sqlc.New(tx)}).GetStatsSeries(ctx, since, until, tc.iatas) + if err != nil { + t.Fatal(err) + } + var want api.StatsSeriesValues + excluded := tc.excluded + if excluded == nil { + excluded = []time.Time{} + } + iatas := tc.iatas + if iatas == nil { + iatas = []string{} + } + if err := tx.QueryRow(ctx, seriesOracle, since, until, iatas, excluded).Scan(&want.Observations, &want.UniquePackets, + &want.ActiveObservers, &want.ActiveIATAs, &want.ScopedPackets, &want.ActiveScopes, + &want.SNRSum, &want.SNRSamples, &want.RSSISum, &want.RSSISamples); err != nil { + t.Fatal(err) + } + want.MaxPathEntries = tc.maxPath + if got.Summary != want { + t.Errorf("summary\n got %+v\nwant %+v", got.Summary, want) + } + if len(got.Hours) != 3 || got.Since != since.UnixMilli() || got.Until != until.UnixMilli() || got.EarliestComplete == nil { + t.Fatalf("series shape: %+v", got) + } + var obs, packets int64 + for _, h := range got.Hours { + partial := false + for _, e := range tc.excluded { + partial = partial || e.UnixMilli() == h.Hour + } + if partial != (h.Values == nil) || (partial && h.Status != "partial") { + t.Errorf("hour %d status %s values %v", h.Hour, h.Status, h.Values) + } + if h.Values != nil { + obs += h.Values.Observations + packets += h.Values.UniquePackets + } + } + // Cards and sparklines must agree for additive metrics. + if obs != got.Summary.Observations || packets != got.Summary.UniquePackets { + t.Errorf("hours sum to %d observations / %d packets, summary %d / %d", obs, packets, got.Summary.Observations, got.Summary.UniquePackets) + } + if got.CompleteHours != 3-len(tc.excluded) { + t.Errorf("complete hours %d", got.CompleteHours) + } + }) + } + + // Hours never registered (or not yet eligible) are missing. + got, err := store.GetStatsSeries(ctx, h0.Add(-2*time.Hour), h0, nil) + if err != nil || len(got.Hours) != 2 || got.Hours[0].Status != "missing" || got.Hours[0].Values != nil || got.CompleteHours != 0 { + t.Errorf("unregistered hours: %+v %v", got, err) + } +} diff --git a/db/stats_test.go b/db/stats_test.go index e584311..88e978b 100644 --- a/db/stats_test.go +++ b/db/stats_test.go @@ -19,62 +19,63 @@ func TestGetStatsOverview(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetAnalyticsRevision(gomock.Any()).Return(int64(7), nil) + mock.EXPECT().GetEarliestCompleteRollupHour(gomock.Any()).Return(pgtype.Timestamptz{}, nil) + var params sqlc.GetStatsSeriesParams mock.EXPECT(). - GetStatsOverview(gomock.Any(), []string{"YVR"}). - Return(sqlc.GetStatsOverviewRow{ - TotalPackets: 100, - TotalObservations: 500, - ActiveObservers: 10, - ActiveIatas: 3, - }, nil) + GetStatsSeries(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, p sqlc.GetStatsSeriesParams) ([]sqlc.GetStatsSeriesRow, error) { + params = p + return []sqlc.GetStatsSeriesRow{{Status: "summary", Observations: 500, UniquePackets: 100, ActiveObservers: 10, ActiveIatas: 3}}, nil + }) store := &Store{q: mock} result, err := store.GetStatsOverview(context.Background(), []string{"YVR"}) if err != nil { t.Fatalf("unexpected error: %v", err) } - if result.TotalPackets != 100 { - t.Errorf("expected TotalPackets 100, got %d", result.TotalPackets) + if result.TotalPackets != 100 || result.TotalObservations != 500 || result.ActiveObservers != 10 || result.ActiveIATAs != 3 { + t.Errorf("unexpected summary mapping: %+v", result) } - if result.WindowHours != 24 { - t.Errorf("expected WindowHours 24, got %d", result.WindowHours) + if result.WindowHours != 24 || result.Until-result.Since != (24*time.Hour).Milliseconds() { + t.Errorf("window %d..%d (%d hours)", result.Since, result.Until, result.WindowHours) + } + // The window ends at the last hour that can have been rolled. + wantUntil := time.Now().UTC().Add(-rollupDelay).Truncate(time.Hour).Add(time.Hour) + if !params.Until.Time.Equal(wantUntil) || len(params.Iatas) != 1 { + t.Errorf("series params %+v, want until %s", params, wantUntil) } } -func TestGetStatsTopNodes_NilObservationCount(t *testing.T) { +func TestGetStatsTopNodes_MissingNodeRow(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) nodeID := uuid.MustParse("00000000-0000-0000-0000-000000000001") lastHeard := pgtype.Timestamptz{Time: time.UnixMilli(1700000000000), Valid: true} - name := "test-node" + since := time.Date(2026, 1, 1, 12, 34, 0, 0, time.UTC) mock.EXPECT(). GetTopNodes(gomock.Any(), sqlc.GetTopNodesParams{ - Column1: []string{"YVR"}, - Limit: 5, + Since: pgtype.Timestamptz{Time: since.Truncate(time.Hour), Valid: true}, + Iatas: []string{"YVR"}, + RowLimit: 5, }). - Return([]sqlc.MvTopNodesByIatum{ - { - NodeID: nodeID, - Name: &name, - NodeType: 1, - Iata: "YVR", - ObservationCount: nil, - LastHeard: lastHeard, - }, + Return([]sqlc.GetTopNodesRow{ + {NodeID: pgtype.UUID{Bytes: nodeID, Valid: true}, PublicKey: "aa", Name: "test-node", NodeType: 1, Iata: "YVR", ObservationCount: 9, LastHeard: lastHeard}, + {PublicKey: "bb", Iata: "YVR", ObservationCount: 3, LastHeard: lastHeard}, }, nil) store := &Store{q: mock} - items, err := store.GetStatsTopNodes(context.Background(), []string{"YVR"}, 5) + items, err := store.GetStatsTopNodes(context.Background(), []string{"YVR"}, since, 5) if err != nil { t.Fatalf("unexpected error: %v", err) } - if len(items) != 1 { - t.Fatalf("expected 1 item, got %d", len(items)) + if len(items) != 2 || items[0].NodeID == nil || *items[0].NodeID != nodeID || *items[0].NodeName != "test-node" { + t.Fatalf("unexpected first node: %+v", items) } - if items[0].ObservationCount != 0 { - t.Errorf("expected ObservationCount 0 for nil, got %d", items[0].ObservationCount) + if items[1].NodeID != nil || items[1].NodeName != nil || items[1].PublicKey != "bb" { + t.Errorf("deleted node should keep its key and drop id/name: %+v", items[1]) } } @@ -91,8 +92,8 @@ func TestGetStatsTopObservers_IATATypeAssertion(t *testing.T) { Return([]sqlc.GetStatsTopObserversRow{ { ID: observerID, - DisplayName: &displayName, - ObserverType: &obsType, + DisplayName: displayName, + ObserverType: obsType, Iata: "YVR", ObservationCount: 42, }, @@ -126,8 +127,9 @@ func TestGetStatsTopAdvertisers_FloodDirectSplit(t *testing.T) { GetStatsTopAdvertisers(gomock.Any(), gomock.Any()). Return([]sqlc.GetStatsTopAdvertisersRow{ { - ID: nodeID, - Name: &name, + NodeID: pgtype.UUID{Bytes: nodeID, Valid: true}, + PublicKey: "cc", + Name: name, NodeType: 2, // repeater AdvertCount: 10, FloodAdvertCount: 7, @@ -227,13 +229,13 @@ func TestGetScopeStats(t *testing.T) { mock := mockdb.NewMockQuerier(ctrl) mock.EXPECT(). - GetScopeStats(gomock.Any(), []string{"YVR"}). + GetScopeStats(gomock.Any(), gomock.Any()). Return([]sqlc.GetScopeStatsRow{ {Name: "default", PacketCount: 100, ObserverCount: 5, NodeCount: 20}, }, nil) store := &Store{q: mock} - items, err := store.GetScopeStats(context.Background(), []string{"YVR"}) + items, err := store.GetScopeStats(context.Background(), []string{"YVR"}, time.Time{}) if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/db/store.go b/db/store.go index 2f9f35a..c0f4c9f 100644 --- a/db/store.go +++ b/db/store.go @@ -22,6 +22,7 @@ import ( // Store wraps the sqlc-generated Queries and implements both ingest.DB and api.Reader. type Store struct { q sqlc.Querier + pool *pgxpool.Pool // rollups need a dedicated connection; nil in mock-backed tests clockDriftThreshold time.Duration // see api.Node.ClockOutOfSync staleThreshold time.Duration // see api.NodeSummary.Stale } @@ -32,7 +33,7 @@ type Store struct { // staleThreshold is how long since last_seen before a node's Stale is reported true; see // internal/config.ResolvedConfig.NodeStaleThreshold. func New(pool *pgxpool.Pool, clockDriftThreshold, staleThreshold time.Duration) *Store { - return &Store{q: sqlc.New(pool), clockDriftThreshold: clockDriftThreshold, staleThreshold: staleThreshold} + return &Store{q: sqlc.New(pool), pool: pool, clockDriftThreshold: clockDriftThreshold, staleThreshold: staleThreshold} } func (s *Store) ResolvePathHashes(ctx context.Context, iata string, hashes [][]byte) (map[string][]api.ResolvedPathEntry, error) { diff --git a/db/trace_pagination_integration_test.go b/db/trace_pagination_integration_test.go index b0ca140..991d74d 100644 --- a/db/trace_pagination_integration_test.go +++ b/db/trace_pagination_integration_test.go @@ -5,6 +5,8 @@ package db import ( "context" + "encoding/hex" + "encoding/json" "os" "reflect" "strings" @@ -13,11 +15,11 @@ import ( sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/MeshCore-Beacon/beacon-server/internal/ingest" "github.com/jackc/pgx/v5" ) -// Trace tags, not individual packets, are the entities being paginated. All -// fixture writes use temporary tables and are rolled back on exit. +// Trace tags, not individual packets, are the entities being paginated. func TestTraceTagPaginationPostgres(t *testing.T) { dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") if dsn == "" { @@ -35,31 +37,10 @@ func TestTraceTagPaginationPostgres(t *testing.T) { t.Fatal(err) } defer tx.Rollback(context.Background()) - for _, table := range []string{"packets", "trace_iatas", "transport_scopes"} { - if _, err := tx.Exec(ctx, "CREATE TEMP TABLE "+table+" (LIKE public."+table+" INCLUDING ALL) ON COMMIT DROP"); err != nil { - t.Fatal(err) - } - } - _, err = tx.Exec(ctx, ` -INSERT INTO transport_scopes (id,name,transport_key,key_fingerprint) VALUES + applyBaseline(t, ctx, tx) + if _, err := tx.Exec(ctx, `INSERT INTO transport_scopes (id,name,transport_key,key_fingerprint) VALUES (1,'scope-one',decode(repeat('01',16),'hex'),decode(repeat('01',8),'hex')), - (2,'scope-two',decode(repeat('02',16),'hex'),decode(repeat('02',8),'hex')); -INSERT INTO packets (packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,trace_tag,scope_id,first_heard_at,last_heard_at,parsed_payload) -SELECT decode(lpad(to_hex(i),64,'0'),'hex'),9,0,1,'\x00','\x00',decode(tag,'hex'),scope, - '2026-01-01'::timestamptz+first*interval '1 second','2026-01-01'::timestamptz+last*interval '1 second',payload::jsonb -FROM (VALUES - (1,'aaaaaaaa',1,1,40,'{"type":"TRACE","pathHashes":["aa"],"snrValues":[1]}'), - (2,'aaaaaaaa',1,2,10,'{"type":"TRACE","pathHashes":["aa","bb"],"snrValues":[1,2]}'), - (3,'aaaaaaaa',2,3,25,'{"type":"PING","pathHashes":["aa","bb","cc"],"snrValues":[1,2,3]}'), - (4,'bbbbbbbb',1,4,30,'{"type":"TRACE","pathHashes":["bb","11","22","33"],"snrValues":[2,3,4,5]}'), - (5,'bbbbbbbb',2,5,5,'{"type":"PING","pathHashes":["bb"],"snrValues":[2]}'), - (6,'cccccccc',1,6,20,'{"type":"TRACE","pathHashes":["cc"],"snrValues":[3]}'), - (7,'dddddddd',2,7,15,'{"type":"PING","pathHashes":["dd"],"snrValues":[4]}'), - (8,NULL,1,8,50,'{"type":"TRACE","pathHashes":[],"snrValues":[]}') -) v(i,tag,scope,first,last,payload); -INSERT INTO trace_iatas (trace_tag,iata) VALUES - ('\xaaaaaaaa','YYZ'),('\xaaaaaaaa','YVR'),('\xbbbbbbbb','YYZ'),('\xcccccccc','YVR');`) - if err != nil { + (2,'scope-two',decode(repeat('02',16),'hex'),decode(repeat('02',8),'hex'))`); err != nil { t.Fatal(err) } store := &Store{q: sqlc.New(tx)} @@ -70,12 +51,47 @@ INSERT INTO trace_iatas (trace_tag,iata) VALUES } return anchor.Add(time.Duration(seconds) * time.Second) } + // Each packet is heard once when new and again (duplicate) at its last time. + for _, p := range []struct { + tag string + iata string + scope int32 + first, last int + kind string + hashes []string + }{ + {"aaaaaaaa", "YYZ", 1, 1, 40, "TRACE", []string{"aa"}}, + {"aaaaaaaa", "YVR", 1, 2, 10, "TRACE", []string{"aa", "bb"}}, + {"aaaaaaaa", "YVR", 2, 3, 25, "PING", []string{"aa", "bb", "cc"}}, + {"bbbbbbbb", "YYZ", 1, 4, 30, "TRACE", []string{"bb", "11", "22", "33"}}, + {"bbbbbbbb", "YYZ", 2, 5, 5, "PING", []string{"bb"}}, + {"cccccccc", "YVR", 1, 6, 20, "TRACE", []string{"cc"}}, + {"dddddddd", "YOW", 2, 7, 15, "PING", []string{"dd"}}, + } { + tag, _ := hex.DecodeString(p.tag) + snr := make([]float32, len(p.hashes)) + for i := range snr { + snr[i] = float32(i + int(p.tag[0]-'a') + 1) + } + payload, _ := json.Marshal(map[string]any{"type": p.kind, "pathHashes": p.hashes, "snrValues": snr}) + scope := p.scope + hash := append(append([]byte{}, tag...), byte(len(p.hashes)), byte(p.first)) + if _, err := store.UpsertPacket(ctx, ingest.UpsertPacketParams{PacketHash: hash, PayloadType: 9, RouteType: 1, + RawPayload: []byte{0}, RawHeader: []byte{0}, ParsedPayload: payload, ScopeID: &scope, TraceTag: tag}); err != nil { + t.Fatal(err) + } + for _, heard := range []int{p.first, p.last} { + if err := store.RecordTrace(ctx, ingest.TraceHearing{TraceTag: tag, IATA: p.iata, HeardAt: at(heard)}); err != nil { + t.Fatal(err) + } + } + } paths := map[string][]string{"a": {"aa", "bb", "cc"}, "b": {"bb", "11", "22", "33"}, "c": {"cc"}, "d": {"dd"}} snrs := map[string][]float32{"a": {1, 2, 3}, "b": {2, 3, 4, 5}, "c": {3}, "d": {4}} summary := func(tag string, first, last int, count, iatas int64, kind string) api.TraceTagSummary { return api.TraceTagSummary{TraceTag: strings.Repeat(tag, 8), FirstHeardAt: at(first).UnixMilli(), LastHeardAt: at(last).UnixMilli(), PacketCount: count, IATACount: iatas, TraceType: kind, PathHashes: paths[tag], SNRValues: snrs[tag]} } - a, b, c, d := summary("a", 1, 40, 3, 2, "TRACE"), summary("b", 4, 30, 2, 1, "TRACE"), summary("c", 6, 20, 1, 1, "TRACE"), summary("d", 7, 15, 1, 0, "PING") + a, b, c, d := summary("a", 1, 40, 3, 2, "TRACE"), summary("b", 4, 30, 2, 1, "TRACE"), summary("c", 6, 20, 1, 1, "TRACE"), summary("d", 7, 15, 1, 1, "PING") for _, tc := range []struct { name string iatas []string @@ -90,11 +106,12 @@ INSERT INTO trace_iatas (trace_tag,iata) VALUES {"region", []string{"YYZ"}, "", "", 0, 0, 35, []api.TraceTagSummary{b}}, {"multiple regions", []string{"YYZ", "YVR", "YYZ"}, "", "", 0, 0, 35, []api.TraceTagSummary{b, c}}, {"unknown region", []string{"ZZZ"}, "", "", 0, 0, 0, []api.TraceTagSummary{}}, + // Filters apply to the tag summary: its first scope, any-TRACE type, and first_heard_at. {"scope", nil, "scope-one", "", 0, 0, 30, []api.TraceTagSummary{c}}, - {"scope defines group", nil, "scope-two", "", 0, 0, 20, []api.TraceTagSummary{d, summary("b", 5, 5, 1, 1, "PING")}}, - {"type", nil, "", "TRACE", 0, 0, 35, []api.TraceTagSummary{summary("b", 4, 30, 1, 1, "TRACE"), c}}, - {"type defines group", nil, "", "PING", 0, 0, 30, []api.TraceTagSummary{summary("a", 3, 25, 1, 2, "PING"), d, summary("b", 5, 5, 1, 1, "PING")}}, - {"time window defines group", nil, "", "", 3, 6, 26, []api.TraceTagSummary{summary("a", 3, 25, 1, 2, "PING"), c}}, + {"first scope wins", nil, "scope-two", "", 0, 0, 0, []api.TraceTagSummary{d}}, + {"type", nil, "", "TRACE", 0, 0, 35, []api.TraceTagSummary{b, c}}, + {"mixed tag is TRACE", nil, "", "PING", 0, 0, 0, []api.TraceTagSummary{d}}, + {"time window on first heard", nil, "", "", 3, 6, 26, []api.TraceTagSummary{c}}, } { t.Run(tc.name, func(t *testing.T) { got, err := store.ListTraceTags(ctx, tc.iatas, tc.scope, tc.kind, at(tc.since), at(tc.until), at(tc.cursor), 20) @@ -135,4 +152,47 @@ INSERT INTO trace_iatas (trace_tag,iata) VALUES if total != 4 || len(seen) != 4 { t.Errorf("expected exactly four complete trace summaries") } + + if err := store.DeleteOldTraceTags(ctx, at(20)); err != nil { + t.Fatal(err) + } + if got, err := store.ListTraceTags(ctx, nil, "", "", time.Time{}, time.Time{}, time.Time{}, 20); err != nil || !reflect.DeepEqual(got, []api.TraceTagSummary{a, b, c}) { + t.Errorf("after cleanup got %+v, %v; want a, b, c", got, err) + } +} + +// Storing a TRACE packet records its contribution even if the hearing write that follows +// is lost; the next hearing replaces the packet's provisional times. +func TestTraceSummarySurvivesMissedHearingPostgres(t *testing.T) { + ctx, pool := schemaPool(t) + if err := RunMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, "INSERT INTO iata_codes (iata) VALUES ('YVR')"); err != nil { + t.Fatal(err) + } + store := New(pool, 0, 0) + tag := []byte{0xee, 0xee, 0xee, 0xee} + payload := json.RawMessage(`{"type":"TRACE","pathHashes":["aa","bb"],"snrValues":[1,2]}`) + upsert := func(hash byte) { + t.Helper() + if _, err := store.UpsertPacket(ctx, ingest.UpsertPacketParams{PacketHash: []byte{hash}, PayloadType: 9, RouteType: 1, + RawPayload: []byte{0}, RawHeader: []byte{0}, ParsedPayload: payload, TraceTag: tag}); err != nil { + t.Fatal(err) + } + } + upsert(1) // the RecordTrace that would follow never runs + upsert(1) // a broker duplicate must not count twice + rows, err := store.ListTraceTags(ctx, nil, "", "", time.Time{}, time.Time{}, time.Time{}, 10) + if err != nil || len(rows) != 1 || rows[0].PacketCount != 1 || len(rows[0].PathHashes) != 2 || rows[0].TraceType != "TRACE" { + t.Fatalf("summary after a missed hearing: %+v %v", rows, err) + } + heard := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + if err := store.RecordTrace(ctx, ingest.TraceHearing{TraceTag: tag, IATA: "YVR", HeardAt: heard}); err != nil { + t.Fatal(err) + } + rows, err = store.ListTraceTags(ctx, nil, "", "", time.Time{}, time.Time{}, time.Time{}, 10) + if err != nil || len(rows) != 1 || rows[0].FirstHeardAt != heard.UnixMilli() || rows[0].LastHeardAt != heard.UnixMilli() || rows[0].IATACount != 1 { + t.Errorf("first hearing should replace provisional times: %+v %v", rows, err) + } } diff --git a/db/traces.go b/db/traces.go index d3cd615..b33ebe3 100644 --- a/db/traces.go +++ b/db/traces.go @@ -11,6 +11,7 @@ import ( sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/MeshCore-Beacon/beacon-server/internal/ingest" "github.com/jackc/pgx/v5/pgtype" ) @@ -20,14 +21,18 @@ type tracePayload struct { SNRValues []float32 `json:"snrValues"` } -func (s *Store) UpsertTraceIATA(ctx context.Context, traceTag []byte, iata string, heardAt time.Time) error { - return s.q.UpsertTraceIATA(ctx, sqlc.UpsertTraceIATAParams{ - TraceTag: traceTag, - Iata: iata, - LastHeard: pgtype.Timestamptz{Time: heardAt, Valid: true}, +func (s *Store) RecordTrace(ctx context.Context, h ingest.TraceHearing) error { + return s.q.RecordTrace(ctx, sqlc.RecordTraceParams{ + TraceTag: h.TraceTag, + Iata: h.IATA, + HeardAt: pgtype.Timestamptz{Time: h.HeardAt, Valid: true}, }) } +func (s *Store) DeleteOldTraceTags(ctx context.Context, cutoff time.Time) error { + return s.q.DeleteOldTraceTags(ctx, pgtype.Timestamptz{Time: cutoff, Valid: true}) +} + func (s *Store) DeleteOldTraceIATAs(ctx context.Context, cutoff time.Time) error { return s.q.DeleteOldTraceIATAs(ctx, pgtype.Timestamptz{Time: cutoff, Valid: true}) } diff --git a/docs/docs.go b/docs/docs.go index 15dc6a7..425a460 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -2280,6 +2280,7 @@ const docTemplate = `{ }, "/scopes/{name}": { "get": { + "description": "packetCount sums packets per hour across every retained rollup hour; iatas are the IATAs that heard the scope's packets. Observer and node counts are current memberships.", "produces": [ "application/json" ], @@ -2424,13 +2425,14 @@ const docTemplate = `{ }, "/stats/observations": { "get": { + "description": "Observation counts only; distinct packet and observer counts don't sum across IATAs, so read them from /stats/series.", "produces": [ "application/json" ], "tags": [ "Stats" ], - "summary": "Hourly observation time series", + "summary": "Hourly observation counts per IATA", "parameters": [ { "type": "string", @@ -2570,13 +2572,14 @@ const docTemplate = `{ }, "/stats/overview": { "get": { + "description": "Summarises the 24 most recent hours that can have been rolled (an hour rolls about 95 minutes after it closes); since/until report that window. Packets count once per hour heard. Use /stats/series for sparklines.", "produces": [ "application/json" ], "tags": [ "Stats" ], - "summary": "Network overview stats (last 24h)", + "summary": "Network overview stats (last 24 rolled hours)", "parameters": [ { "type": "string", @@ -2795,7 +2798,7 @@ const docTemplate = `{ }, "/stats/scopes": { "get": { - "description": "Counts each packet, observer and node once per scope. IATA filters use retained observations for packets/observers and node IATA memberships for nodes. Without filters, returns global totals for every stored scope. With filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; those remain listed with zero counts. An empty region returns an empty array.", + "description": "Packets are those heard since the window start, each counted once per hour however many of the requested IATAs heard it. Observers and nodes are current memberships: observers filter by the IATA they last reported from, nodes by their IATA memberships. With filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; those remain listed with zero counts. An empty region returns an empty array.", "produces": [ "application/json" ], @@ -2827,6 +2830,12 @@ const docTemplate = `{ "description": "Filter by region slug, expands to member IATAs; combined with explicit IATAs", "name": "region", "in": "query" + }, + { + "type": "integer", + "description": "Start of packet window epoch ms, rounded down to the hour (default 7 days ago)", + "name": "since", + "in": "query" } ], "responses": { @@ -2854,6 +2863,78 @@ const docTemplate = `{ } } }, + "/stats/series": { + "get": { + "description": "One entry per UTC hour in [since, until) plus a summary over the complete hours. Both bounds round down to UTC hours and since is clamped to the rollup retention; the response reports the effective window. An hour is rolled about 95 minutes after it closes, so recent hours are \"missing\" until then; \"partial\" hours lost raw rows before they could be rolled and never get values. Counts (observations, packets) sum across hours, with a packet counted once in each hour it was heard; observers, IATAs and scopes are distinct across the whole window; averages are sum / samples. revision changes whenever any rolled hour changes.", + "produces": [ + "application/json" + ], + "tags": [ + "Stats" + ], + "summary": "Hourly network activity from the analytics rollups", + "parameters": [ + { + "type": "integer", + "description": "Inclusive start, epoch milliseconds", + "name": "since", + "in": "query", + "required": true + }, + { + "type": "integer", + "description": "Exclusive end, epoch milliseconds; the window is at most the rollup retention (default 90 days)", + "name": "until", + "in": "query", + "required": true + }, + { + "type": "string", + "description": "Comma-separated reception IATA codes", + "name": "iatas", + "in": "query" + }, + { + "type": "integer", + "description": "Region ID, expands to member IATAs", + "name": "regionId", + "in": "query" + }, + { + "type": "string", + "description": "Region slug, expands to member IATAs", + "name": "region", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeries" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, "/stats/signal": { "get": { "description": "Aggregates stored observer receptions in [since, until), at most 30 days. SNR is dB; RSSI is dBm. Null/non-finite readings and the zero/zero unavailable sentinel are excluded per metric; actual zero SNR with nonzero RSSI remains valid. Averages are null without samples. Histogram bounds are lower-inclusive/upper-exclusive, with null for unbounded ends. Both boundaries round down to UTC hours and the response reports that effective window. Reads use hourly materialized snapshots refreshed by background.view_refresh; the current partial hour is excluded and absent hours are omitted. These last-hop readings do not measure end-to-end quality or packet loss.", @@ -2928,6 +3009,7 @@ const docTemplate = `{ }, "/stats/top-advertisers": { "get": { + "description": "Each advert counts once per hour heard, however many requested IATAs heard it. nodeId is null when the node row has been deleted.", "produces": [ "application/json" ], @@ -2990,13 +3072,14 @@ const docTemplate = `{ }, "/stats/top-nodes": { "get": { + "description": "nodeId is null when the node row has been deleted; publicKey always identifies it.", "produces": [ "application/json" ], "tags": [ "Stats" ], - "summary": "Top N nodes by observation count (from materialized view)", + "summary": "Top N nodes by advert hearings (last 7 days by default)", "parameters": [ { "type": "string", @@ -3016,6 +3099,12 @@ const docTemplate = `{ "name": "region", "in": "query" }, + { + "type": "integer", + "description": "Start of window epoch ms, rounded down to the hour (default 7 days ago)", + "name": "since", + "in": "query" + }, { "maximum": 200, "minimum": 1, @@ -3198,13 +3287,13 @@ const docTemplate = `{ }, { "type": "string", - "description": "Filter by transport scope name", + "description": "Filter by the tag's first transport scope name", "name": "scope", "in": "query" }, { "type": "string", - "description": "Filter by type: TRACE or PING (default: all)", + "description": "Filter by type: TRACE (any multi-hop packet) or PING (default: all)", "name": "type", "in": "query" }, @@ -3466,6 +3555,7 @@ const docTemplate = `{ "type": "integer" }, "messageCount": { + "description": "lifetime count; not reduced by retention", "type": "integer" }, "name": { @@ -3984,9 +4074,6 @@ const docTemplate = `{ "github_com_MeshCore-Beacon_beacon-server_internal_api.ObservationPoint": { "type": "object", "properties": { - "activeObservers": { - "type": "integer" - }, "hour": { "description": "epoch ms, start of the 1-hour bucket", "type": "integer" @@ -3996,9 +4083,6 @@ const docTemplate = `{ }, "observationCount": { "type": "integer" - }, - "uniquePackets": { - "type": "integer" } } }, @@ -5194,10 +5278,19 @@ const docTemplate = `{ "activeObservers": { "type": "integer" }, + "since": { + "description": "epoch ms, start of the window", + "type": "integer" + }, "totalObservations": { "type": "integer" }, "totalPackets": { + "description": "distinct per hour (see StatsSeriesValues)", + "type": "integer" + }, + "until": { + "description": "epoch ms, exclusive end of the window", "type": "integer" }, "windowHours": { @@ -5206,6 +5299,109 @@ const docTemplate = `{ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeries": { + "type": "object", + "properties": { + "completeHours": { + "description": "hours contributing to summary", + "type": "integer" + }, + "earliestComplete": { + "description": "first complete hour held, epoch ms; null if none", + "type": "integer" + }, + "hours": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeriesHour" + } + }, + "revision": { + "description": "changes whenever rolled hours, their status or coverage change", + "type": "integer" + }, + "since": { + "description": "effective start, epoch ms on a UTC hour", + "type": "integer" + }, + "summary": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeriesValues" + }, + "until": { + "description": "effective exclusive end, epoch ms on a UTC hour", + "type": "integer" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeriesHour": { + "type": "object", + "properties": { + "hour": { + "description": "epoch ms, start of the hour", + "type": "integer" + }, + "status": { + "description": "partial: raw rows deleted before it was rolled", + "type": "string", + "enum": [ + "complete", + "partial", + "missing" + ] + }, + "values": { + "description": "null unless complete", + "allOf": [ + { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeriesValues" + } + ] + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeriesValues": { + "type": "object", + "properties": { + "activeIatas": { + "description": "distinct across the hour or window", + "type": "integer" + }, + "activeObservers": { + "description": "distinct across the hour or window", + "type": "integer" + }, + "activeScopes": { + "description": "distinct across the hour or window", + "type": "integer" + }, + "maxPathEntries": { + "type": "integer" + }, + "observations": { + "type": "integer" + }, + "rssiSamples": { + "type": "integer" + }, + "rssiSum": { + "description": "average = rssiSum / rssiSamples", + "type": "number" + }, + "scopedPackets": { + "type": "integer" + }, + "snrSamples": { + "type": "integer" + }, + "snrSum": { + "description": "average = snrSum / snrSamples", + "type": "number" + }, + "uniquePackets": { + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.TopAdvertiser": { "type": "object", "properties": { @@ -5227,6 +5423,7 @@ const docTemplate = `{ "type": "integer" }, "nodeId": { + "description": "null once the node row has been deleted", "type": "string" }, "nodeName": { @@ -5237,6 +5434,10 @@ const docTemplate = `{ }, "nodeTypeName": { "type": "string" + }, + "publicKey": { + "description": "hex", + "type": "string" } } }, @@ -5251,6 +5452,7 @@ const docTemplate = `{ "type": "integer" }, "nodeId": { + "description": "null once the node row has been deleted", "type": "string" }, "nodeName": { @@ -5264,6 +5466,10 @@ const docTemplate = `{ }, "observationCount": { "type": "integer" + }, + "publicKey": { + "description": "hex", + "type": "string" } } }, diff --git a/docs/historical-stats.md b/docs/historical-stats.md new file mode 100644 index 0000000..72ad3ac --- /dev/null +++ b/docs/historical-stats.md @@ -0,0 +1,85 @@ +# Historical stats + +## The rule + +Historical stats never scan `packet_observations` per request or per refresh. They read +hourly rollup tables filled once per closed hour by one background task. Raw rows are read +only for per-entity drill-downs (a packet, a trace, a route), observer comparison and +sub-hour observer activity. + +## Rollup contract + +- **Hours.** `hour` is the UTC `date_trunc('hour', heard_at)` and leads every rollup table's + primary key. Rollup tables have no foreign keys: history outlives deleted observers, nodes + and IATAs. +- **Eligibility.** Hour `[H, H+1h)` can be rolled once `now() >= H + 95 min`. Ingest clamps + `heard_at` to ±30 minutes of server time, so nothing can still land in it. +- **Registration.** Every eligible hour since the earliest retained observation gets a row + in `analytics_rollup_hours`, including hours with no traffic. Status is `missing`, + `complete` or `partial`. +- **Rolling.** One process holds the advisory lock (`analytics_rollup` task, every 5 + minutes, plus a catch-up at boot). Each hour is rolled in one REPEATABLE READ transaction: + copy its observations (joined to packet dimensions) into a temp table, delete and reinsert + every family, hash the result, mark the hour complete. A serialization failure skips the + hour until the next pass. +- **Revision.** `analytics_state.revision` increases when an hour's content hash changes or + an hour turns partial; `analytics_raw_state.coverage` increases when retention deletes + rollup hours. Cache keys for rollup-backed endpoints include their sum (each counter only + grows), read through a 10 s memo. +- **Raw deletion.** Cleanup deletes packets older than `packets.retention`, but holds the + cutoff 35 minutes behind the oldest `missing` hour, for at most 24 hours (it logs a + warning past 6). Each batch advances `analytics_raw_state.raw_deleted_before` to the + newest deleted `last_heard_at`. Cleanup and the rollup never write the same row. +- **Partial hours.** A not-yet-complete hour starting at or before `raw_deleted_before + 30 min` + lost raw rows before it was rolled. It is marked `partial` and never rolled; readers + report it without values. +- **Re-rolls.** Hours are not re-rolled periodically. A message stored by channel backfill (at + startup, and when keys are imported at runtime) queues every hour its packet was heard in + `analytics_dirty_hours`, in the same statement as the message insert. Each pass re-rolls a + few complete ones while their raw rows remain. A queued hour still waiting for its first roll + keeps its entry unless that roll's snapshot already included it. +- **Retention.** Rollups are kept for `analytics.rollup_retention` (default 90 days), which + is also the longest window `/stats/series` accepts. + +## Combining hours + +| Metric | Across hours and IATAs | +| --- | --- | +| Observations, receptions, payload counts | Sum | +| Unique packets, adverts, messages, scoped packets | Sum of hourly IATA-set counts: each is counted once per hour it was heard, so one heard across an hour boundary counts twice (measured +1.1%) | +| Active observers, IATAs, scopes | Distinct across the window | +| Max path entries | Max | +| SNR / RSSI averages | Σ sum / Σ samples, never an average of averages | + +Windows snap to UTC hours on the server, and responses report the effective window. + +## IATA sets + +A packet heard in YVR and YYZ appears in both IATAs' per-IATA rows, so summing per-IATA +distinct counts overcounts (2.26× on production data). The `*_sets` tables instead group +each distinct packet, advert or message under the exact sorted set of IATAs that heard it +that hour (`iatas char(3)[]`, kept canonical by a CHECK). For any requested set S, the rows +where `iatas && S` count every item heard in S exactly once. This is exact for one IATA, a +region or all IATAs. + +## Raw or current, by design + +- Packet, trace and route detail, the packet list, observer comparison, and observer + activity below one hour. +- Current memberships: scope observer and node counts, node types, clock drift, radio presets. +- `/traces` reads `trace_tags`, a per-tag summary kept at ingest, not a rollup. A new packet + adds its count, type and payload in the statement that stores it; hearings set the times. + +## Adding a metric + +1. Add a family table to `db/migrations/001_baseline.sql` (on 2.x, a new migration): `hour` + first in the primary key, no foreign keys, and a set table with + `CHECK (iata_set_is_canonical(iatas))` plus a GIN index if it needs distinct counts. +2. Add a delete in `DeleteRollupHour`, a `Roll*` insert over `rollup_obs` in + `db/queries/rollup.sql`, a line in `RollupContentHash` and `DeleteOldRollups`, and the + call in `rollHour` (`db/rollup.go`). +3. Read it with an hour-bounded query, using the UNION-ALL IATA guard (see `series.sql`) for + per-IATA tables or `iatas && @iatas` for set tables. +4. Cache it with `cachedRollup` so its key carries the revision. +5. Add a raw-SQL oracle case to `db/rollup_integration_test.go`. +6. Run `sqlc generate`, regenerate mocks and swagger. diff --git a/docs/swagger.json b/docs/swagger.json index 25af067..757caae 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -2278,6 +2278,7 @@ }, "/scopes/{name}": { "get": { + "description": "packetCount sums packets per hour across every retained rollup hour; iatas are the IATAs that heard the scope's packets. Observer and node counts are current memberships.", "produces": [ "application/json" ], @@ -2422,13 +2423,14 @@ }, "/stats/observations": { "get": { + "description": "Observation counts only; distinct packet and observer counts don't sum across IATAs, so read them from /stats/series.", "produces": [ "application/json" ], "tags": [ "Stats" ], - "summary": "Hourly observation time series", + "summary": "Hourly observation counts per IATA", "parameters": [ { "type": "string", @@ -2568,13 +2570,14 @@ }, "/stats/overview": { "get": { + "description": "Summarises the 24 most recent hours that can have been rolled (an hour rolls about 95 minutes after it closes); since/until report that window. Packets count once per hour heard. Use /stats/series for sparklines.", "produces": [ "application/json" ], "tags": [ "Stats" ], - "summary": "Network overview stats (last 24h)", + "summary": "Network overview stats (last 24 rolled hours)", "parameters": [ { "type": "string", @@ -2793,7 +2796,7 @@ }, "/stats/scopes": { "get": { - "description": "Counts each packet, observer and node once per scope. IATA filters use retained observations for packets/observers and node IATA memberships for nodes. Without filters, returns global totals for every stored scope. With filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; those remain listed with zero counts. An empty region returns an empty array.", + "description": "Packets are those heard since the window start, each counted once per hour however many of the requested IATAs heard it. Observers and nodes are current memberships: observers filter by the IATA they last reported from, nodes by their IATA memberships. With filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; those remain listed with zero counts. An empty region returns an empty array.", "produces": [ "application/json" ], @@ -2825,6 +2828,12 @@ "description": "Filter by region slug, expands to member IATAs; combined with explicit IATAs", "name": "region", "in": "query" + }, + { + "type": "integer", + "description": "Start of packet window epoch ms, rounded down to the hour (default 7 days ago)", + "name": "since", + "in": "query" } ], "responses": { @@ -2852,6 +2861,78 @@ } } }, + "/stats/series": { + "get": { + "description": "One entry per UTC hour in [since, until) plus a summary over the complete hours. Both bounds round down to UTC hours and since is clamped to the rollup retention; the response reports the effective window. An hour is rolled about 95 minutes after it closes, so recent hours are \"missing\" until then; \"partial\" hours lost raw rows before they could be rolled and never get values. Counts (observations, packets) sum across hours, with a packet counted once in each hour it was heard; observers, IATAs and scopes are distinct across the whole window; averages are sum / samples. revision changes whenever any rolled hour changes.", + "produces": [ + "application/json" + ], + "tags": [ + "Stats" + ], + "summary": "Hourly network activity from the analytics rollups", + "parameters": [ + { + "type": "integer", + "description": "Inclusive start, epoch milliseconds", + "name": "since", + "in": "query", + "required": true + }, + { + "type": "integer", + "description": "Exclusive end, epoch milliseconds; the window is at most the rollup retention (default 90 days)", + "name": "until", + "in": "query", + "required": true + }, + { + "type": "string", + "description": "Comma-separated reception IATA codes", + "name": "iatas", + "in": "query" + }, + { + "type": "integer", + "description": "Region ID, expands to member IATAs", + "name": "regionId", + "in": "query" + }, + { + "type": "string", + "description": "Region slug, expands to member IATAs", + "name": "region", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeries" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, "/stats/signal": { "get": { "description": "Aggregates stored observer receptions in [since, until), at most 30 days. SNR is dB; RSSI is dBm. Null/non-finite readings and the zero/zero unavailable sentinel are excluded per metric; actual zero SNR with nonzero RSSI remains valid. Averages are null without samples. Histogram bounds are lower-inclusive/upper-exclusive, with null for unbounded ends. Both boundaries round down to UTC hours and the response reports that effective window. Reads use hourly materialized snapshots refreshed by background.view_refresh; the current partial hour is excluded and absent hours are omitted. These last-hop readings do not measure end-to-end quality or packet loss.", @@ -2926,6 +3007,7 @@ }, "/stats/top-advertisers": { "get": { + "description": "Each advert counts once per hour heard, however many requested IATAs heard it. nodeId is null when the node row has been deleted.", "produces": [ "application/json" ], @@ -2988,13 +3070,14 @@ }, "/stats/top-nodes": { "get": { + "description": "nodeId is null when the node row has been deleted; publicKey always identifies it.", "produces": [ "application/json" ], "tags": [ "Stats" ], - "summary": "Top N nodes by observation count (from materialized view)", + "summary": "Top N nodes by advert hearings (last 7 days by default)", "parameters": [ { "type": "string", @@ -3014,6 +3097,12 @@ "name": "region", "in": "query" }, + { + "type": "integer", + "description": "Start of window epoch ms, rounded down to the hour (default 7 days ago)", + "name": "since", + "in": "query" + }, { "maximum": 200, "minimum": 1, @@ -3196,13 +3285,13 @@ }, { "type": "string", - "description": "Filter by transport scope name", + "description": "Filter by the tag's first transport scope name", "name": "scope", "in": "query" }, { "type": "string", - "description": "Filter by type: TRACE or PING (default: all)", + "description": "Filter by type: TRACE (any multi-hop packet) or PING (default: all)", "name": "type", "in": "query" }, @@ -3464,6 +3553,7 @@ "type": "integer" }, "messageCount": { + "description": "lifetime count; not reduced by retention", "type": "integer" }, "name": { @@ -3982,9 +4072,6 @@ "github_com_MeshCore-Beacon_beacon-server_internal_api.ObservationPoint": { "type": "object", "properties": { - "activeObservers": { - "type": "integer" - }, "hour": { "description": "epoch ms, start of the 1-hour bucket", "type": "integer" @@ -3994,9 +4081,6 @@ }, "observationCount": { "type": "integer" - }, - "uniquePackets": { - "type": "integer" } } }, @@ -5192,10 +5276,19 @@ "activeObservers": { "type": "integer" }, + "since": { + "description": "epoch ms, start of the window", + "type": "integer" + }, "totalObservations": { "type": "integer" }, "totalPackets": { + "description": "distinct per hour (see StatsSeriesValues)", + "type": "integer" + }, + "until": { + "description": "epoch ms, exclusive end of the window", "type": "integer" }, "windowHours": { @@ -5204,6 +5297,109 @@ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeries": { + "type": "object", + "properties": { + "completeHours": { + "description": "hours contributing to summary", + "type": "integer" + }, + "earliestComplete": { + "description": "first complete hour held, epoch ms; null if none", + "type": "integer" + }, + "hours": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeriesHour" + } + }, + "revision": { + "description": "changes whenever rolled hours, their status or coverage change", + "type": "integer" + }, + "since": { + "description": "effective start, epoch ms on a UTC hour", + "type": "integer" + }, + "summary": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeriesValues" + }, + "until": { + "description": "effective exclusive end, epoch ms on a UTC hour", + "type": "integer" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeriesHour": { + "type": "object", + "properties": { + "hour": { + "description": "epoch ms, start of the hour", + "type": "integer" + }, + "status": { + "description": "partial: raw rows deleted before it was rolled", + "type": "string", + "enum": [ + "complete", + "partial", + "missing" + ] + }, + "values": { + "description": "null unless complete", + "allOf": [ + { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeriesValues" + } + ] + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeriesValues": { + "type": "object", + "properties": { + "activeIatas": { + "description": "distinct across the hour or window", + "type": "integer" + }, + "activeObservers": { + "description": "distinct across the hour or window", + "type": "integer" + }, + "activeScopes": { + "description": "distinct across the hour or window", + "type": "integer" + }, + "maxPathEntries": { + "type": "integer" + }, + "observations": { + "type": "integer" + }, + "rssiSamples": { + "type": "integer" + }, + "rssiSum": { + "description": "average = rssiSum / rssiSamples", + "type": "number" + }, + "scopedPackets": { + "type": "integer" + }, + "snrSamples": { + "type": "integer" + }, + "snrSum": { + "description": "average = snrSum / snrSamples", + "type": "number" + }, + "uniquePackets": { + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.TopAdvertiser": { "type": "object", "properties": { @@ -5225,6 +5421,7 @@ "type": "integer" }, "nodeId": { + "description": "null once the node row has been deleted", "type": "string" }, "nodeName": { @@ -5235,6 +5432,10 @@ }, "nodeTypeName": { "type": "string" + }, + "publicKey": { + "description": "hex", + "type": "string" } } }, @@ -5249,6 +5450,7 @@ "type": "integer" }, "nodeId": { + "description": "null once the node row has been deleted", "type": "string" }, "nodeName": { @@ -5262,6 +5464,10 @@ }, "observationCount": { "type": "integer" + }, + "publicKey": { + "description": "hex", + "type": "string" } } }, diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 3f601a7..b1431fe 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -113,6 +113,7 @@ definitions: description: epoch ms, time of most recent message type: integer messageCount: + description: lifetime count; not reduced by retention type: integer name: description: display name from config or nil @@ -489,8 +490,6 @@ definitions: type: object github_com_MeshCore-Beacon_beacon-server_internal_api.ObservationPoint: properties: - activeObservers: - type: integer hour: description: epoch ms, start of the 1-hour bucket type: integer @@ -498,8 +497,6 @@ definitions: type: string observationCount: type: integer - uniquePackets: - type: integer type: object github_com_MeshCore-Beacon_beacon-server_internal_api.Observer: properties: @@ -1318,14 +1315,92 @@ definitions: type: integer activeObservers: type: integer + since: + description: epoch ms, start of the window + type: integer totalObservations: type: integer totalPackets: + description: distinct per hour (see StatsSeriesValues) + type: integer + until: + description: epoch ms, exclusive end of the window type: integer windowHours: description: always 24 for now type: integer type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeries: + properties: + completeHours: + description: hours contributing to summary + type: integer + earliestComplete: + description: first complete hour held, epoch ms; null if none + type: integer + hours: + items: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeriesHour' + type: array + revision: + description: changes whenever rolled hours, their status or coverage change + type: integer + since: + description: effective start, epoch ms on a UTC hour + type: integer + summary: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeriesValues' + until: + description: effective exclusive end, epoch ms on a UTC hour + type: integer + type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeriesHour: + properties: + hour: + description: epoch ms, start of the hour + type: integer + status: + description: 'partial: raw rows deleted before it was rolled' + enum: + - complete + - partial + - missing + type: string + values: + allOf: + - $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeriesValues' + description: null unless complete + type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeriesValues: + properties: + activeIatas: + description: distinct across the hour or window + type: integer + activeObservers: + description: distinct across the hour or window + type: integer + activeScopes: + description: distinct across the hour or window + type: integer + maxPathEntries: + type: integer + observations: + type: integer + rssiSamples: + type: integer + rssiSum: + description: average = rssiSum / rssiSamples + type: number + scopedPackets: + type: integer + snrSamples: + type: integer + snrSum: + description: average = snrSum / snrSamples + type: number + uniquePackets: + type: integer + type: object github_com_MeshCore-Beacon_beacon-server_internal_api.TopAdvertiser: properties: advertCount: @@ -1345,6 +1420,7 @@ definitions: description: epoch ms type: integer nodeId: + description: null once the node row has been deleted type: string nodeName: type: string @@ -1352,6 +1428,9 @@ definitions: type: integer nodeTypeName: type: string + publicKey: + description: hex + type: string type: object github_com_MeshCore-Beacon_beacon-server_internal_api.TopNode: properties: @@ -1361,6 +1440,7 @@ definitions: description: epoch ms type: integer nodeId: + description: null once the node row has been deleted type: string nodeName: type: string @@ -1370,6 +1450,9 @@ definitions: type: string observationCount: type: integer + publicKey: + description: hex + type: string type: object github_com_MeshCore-Beacon_beacon-server_internal_api.TopObserver: properties: @@ -3060,6 +3143,9 @@ paths: - Scopes /scopes/{name}: get: + description: packetCount sums packets per hour across every retained rollup + hour; iatas are the IATAs that heard the scope's packets. Observer and node + counts are current memberships. parameters: - description: 'Scope name e.g. %23bc (URL-encoded #bc)' in: path @@ -3156,6 +3242,8 @@ paths: - Stats /stats/observations: get: + description: Observation counts only; distinct packet and observer counts don't + sum across IATAs, so read them from /stats/series. parameters: - description: Comma-separated IATA codes in: query @@ -3186,7 +3274,7 @@ paths: description: Internal Server Error schema: $ref: '#/definitions/internal_api_handlers.APIError' - summary: Hourly observation time series + summary: Hourly observation counts per IATA tags: - Stats /stats/observer-comparison: @@ -3256,6 +3344,9 @@ paths: - Stats /stats/overview: get: + description: Summarises the 24 most recent hours that can have been rolled (an + hour rolls about 95 minutes after it closes); since/until report that window. + Packets count once per hour heard. Use /stats/series for sparklines. parameters: - description: Comma-separated IATA codes in: query @@ -3280,7 +3371,7 @@ paths: description: Internal Server Error schema: $ref: '#/definitions/internal_api_handlers.APIError' - summary: Network overview stats (last 24h) + summary: Network overview stats (last 24 rolled hours) tags: - Stats /stats/paths: @@ -3412,12 +3503,13 @@ paths: - Stats /stats/scopes: get: - description: Counts each packet, observer and node once per scope. IATA filters - use retained observations for packets/observers and node IATA memberships - for nodes. Without filters, returns global totals for every stored scope. - With filters, lists only manual scopes configured for a matching region and - imported scopes whose current MeshMapper catalogue includes a matching IATA; - those remain listed with zero counts. An empty region returns an empty array. + description: 'Packets are those heard since the window start, each counted once + per hour however many of the requested IATAs heard it. Observers and nodes + are current memberships: observers filter by the IATA they last reported from, + nodes by their IATA memberships. With filters, lists only manual scopes configured + for a matching region and imported scopes whose current MeshMapper catalogue + includes a matching IATA; those remain listed with zero counts. An empty region + returns an empty array.' parameters: - description: Comma-separated IATA codes in: query @@ -3436,6 +3528,11 @@ paths: in: query name: region type: string + - description: Start of packet window epoch ms, rounded down to the hour (default + 7 days ago) + in: query + name: since + type: integer produces: - application/json responses: @@ -3456,6 +3553,63 @@ paths: summary: Scope statistics tags: - Stats + /stats/series: + get: + description: One entry per UTC hour in [since, until) plus a summary over the + complete hours. Both bounds round down to UTC hours and since is clamped to + the rollup retention; the response reports the effective window. An hour is + rolled about 95 minutes after it closes, so recent hours are "missing" until + then; "partial" hours lost raw rows before they could be rolled and never + get values. Counts (observations, packets) sum across hours, with a packet + counted once in each hour it was heard; observers, IATAs and scopes are distinct + across the whole window; averages are sum / samples. revision changes whenever + any rolled hour changes. + parameters: + - description: Inclusive start, epoch milliseconds + in: query + name: since + required: true + type: integer + - description: Exclusive end, epoch milliseconds; the window is at most the + rollup retention (default 90 days) + in: query + name: until + required: true + type: integer + - description: Comma-separated reception IATA codes + in: query + name: iatas + type: string + - description: Region ID, expands to member IATAs + in: query + name: regionId + type: integer + - description: Region slug, expands to member IATAs + in: query + name: region + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeries' + "400": + description: Bad Request + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + "503": + description: Service Unavailable + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + summary: Hourly network activity from the analytics rollups + tags: + - Stats /stats/signal: get: description: Aggregates stored observer receptions in [since, until), at most @@ -3514,6 +3668,8 @@ paths: - Stats /stats/top-advertisers: get: + description: Each advert counts once per hour heard, however many requested + IATAs heard it. nodeId is null when the node row has been deleted. parameters: - description: Comma-separated IATA codes in: query @@ -3556,6 +3712,8 @@ paths: - Stats /stats/top-nodes: get: + description: nodeId is null when the node row has been deleted; publicKey always + identifies it. parameters: - description: Comma-separated IATA codes in: query @@ -3569,6 +3727,11 @@ paths: in: query name: region type: string + - description: Start of window epoch ms, rounded down to the hour (default 7 + days ago) + in: query + name: since + type: integer - description: Max results (default 10); must be positive, values above 200 are clamped in: query @@ -3589,7 +3752,7 @@ paths: description: Internal Server Error schema: $ref: '#/definitions/internal_api_handlers.APIError' - summary: Top N nodes by observation count (from materialized view) + summary: Top N nodes by advert hearings (last 7 days by default) tags: - Stats /stats/top-observers: @@ -3692,11 +3855,12 @@ paths: in: query name: regionId type: integer - - description: Filter by transport scope name + - description: Filter by the tag's first transport scope name in: query name: scope type: string - - description: 'Filter by type: TRACE or PING (default: all)' + - description: 'Filter by type: TRACE (any multi-hop packet) or PING (default: + all)' in: query name: type type: string diff --git a/go.mod b/go.mod index d309cbf..0fc10fa 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/swaggo/http-swagger v1.3.4 github.com/swaggo/swag v1.16.6 go.uber.org/mock v0.6.0 + golang.org/x/sync v0.21.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -45,7 +46,6 @@ require ( golang.org/x/crypto v0.53.0 // indirect golang.org/x/mod v0.37.0 // indirect golang.org/x/net v0.56.0 // indirect - golang.org/x/sync v0.21.0 // indirect golang.org/x/sys v0.46.0 // indirect golang.org/x/text v0.39.0 // indirect golang.org/x/tools v0.47.0 // indirect diff --git a/internal/api/channels.go b/internal/api/channels.go index 00e9d85..50ed66f 100644 --- a/internal/api/channels.go +++ b/internal/api/channels.go @@ -101,5 +101,5 @@ type Channel struct { ChannelSummary Hashtag *string `json:"hashtag,omitempty"` // tag name without # prefix; non-nil only for hashtag channels KeyFingerprint *string `json:"keyFingerprint,omitempty"` // first 8 bytes of SHA256(key), hex-encoded - MessageCount int64 `json:"messageCount"` + MessageCount int64 `json:"messageCount"` // lifetime count; not reduced by retention } diff --git a/internal/api/handlers/limits_test.go b/internal/api/handlers/limits_test.go index 3d7e3a2..f59c42c 100644 --- a/internal/api/handlers/limits_test.go +++ b/internal/api/handlers/limits_test.go @@ -69,7 +69,7 @@ func TestListLimits(t *testing.T) { got = limit return nil, nil }, - getStatsTopNodes: func(ctx context.Context, iatas []string, limit int32) ([]api.TopNode, error) { + getStatsTopNodes: func(ctx context.Context, iatas []string, _ time.Time, limit int32) ([]api.TopNode, error) { calls++ got = limit return nil, nil @@ -113,7 +113,7 @@ func TestListLimits(t *testing.T) { router.Mount("/messages", MessagesRouter(reader)) router.Mount("/routes", RoutesRouter(reader)) router.Mount("/traces", TracesRouter(reader)) - router.Mount("/stats", StatsRouter(reader, nil)) + router.Mount("/stats", StatsRouter(reader, StatsOptions{})) for _, endpoint := range []struct { path string defaultLimit int32 diff --git a/internal/api/handlers/observer_comparison_test.go b/internal/api/handlers/observer_comparison_test.go index 0b11e6d..78f7b53 100644 --- a/internal/api/handlers/observer_comparison_test.go +++ b/internal/api/handlers/observer_comparison_test.go @@ -28,7 +28,7 @@ func TestObserverComparisonRejectsInvalidQuery(t *testing.T) { } { t.Run(query, func(t *testing.T) { w := httptest.NewRecorder() - StatsRouter(nil, nil).ServeHTTP(w, httptest.NewRequest("GET", "/observer-comparison?"+query, nil)) + StatsRouter(nil, StatsOptions{}).ServeHTTP(w, httptest.NewRequest("GET", "/observer-comparison?"+query, nil)) if w.Code != 400 { t.Fatalf("status %d; wanted validation error", w.Code) } @@ -42,7 +42,7 @@ func TestObserverComparisonHTTP(t *testing.T) { query := "observerA=" + a.String() + "&observerB=" + b.String() + "&since=0&until=1000" for _, suffix := range []string{"&since=2", "&observerA=" + b.String(), "&until=253402300800000", "&observerB=00000000-0000-0000-0000-000000000000"} { w := httptest.NewRecorder() - StatsRouter(nil, nil).ServeHTTP(w, httptest.NewRequest("GET", "/observer-comparison?"+query+suffix, nil)) + StatsRouter(nil, StatsOptions{}).ServeHTTP(w, httptest.NewRequest("GET", "/observer-comparison?"+query+suffix, nil)) if w.Code != 400 { t.Fatalf("invalid query returned %d", w.Code) } @@ -80,7 +80,7 @@ func TestObserverComparisonHTTP(t *testing.T) { }, } w := httptest.NewRecorder() - StatsRouter(reader, nil).ServeHTTP(w, httptest.NewRequest("GET", "/observer-comparison?"+query+tc.suffix, nil)) + StatsRouter(reader, StatsOptions{}).ServeHTTP(w, httptest.NewRequest("GET", "/observer-comparison?"+query+tc.suffix, nil)) if w.Code != tc.status { t.Fatalf("status = %d, body = %s", w.Code, w.Body.String()) } diff --git a/internal/api/handlers/paths.go b/internal/api/handlers/paths.go index 58f5acb..d427d68 100644 --- a/internal/api/handlers/paths.go +++ b/internal/api/handlers/paths.go @@ -32,7 +32,7 @@ import ( func getPathStats(reader api.Reader) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { q := r.URL.Query() - since, until, err := parseStatsWindow(r) + since, until, err := parseStatsWindow(r, rawStatsWindow) if err != nil { respondError(w, http.StatusBadRequest, err.Error()) return diff --git a/internal/api/handlers/paths_test.go b/internal/api/handlers/paths_test.go index 190fe69..fb3b0ee 100644 --- a/internal/api/handlers/paths_test.go +++ b/internal/api/handlers/paths_test.go @@ -28,7 +28,7 @@ func (s pathReader) GetPathStats(ctx context.Context, since, until time.Time, ia func TestPathsRejectInvalidWindow(t *testing.T) { for _, query := range []string{"", "since=0", "since=x&until=1", "since=-1&until=1", "since=1&until=1", "since=2&until=1", "since=0&until=2592000001", "since=0&since=0&until=1", "since=0&until=1&until=2", "since=253402300799998&until=253402300800000"} { w := httptest.NewRecorder() - StatsRouter(nil, nil).ServeHTTP(w, httptest.NewRequest("GET", "/paths?"+query, nil)) + StatsRouter(nil, StatsOptions{}).ServeHTTP(w, httptest.NewRequest("GET", "/paths?"+query, nil)) if w.Code != 400 { t.Fatalf("query %q: %d %s", query, w.Code, w.Body.String()) } @@ -66,7 +66,7 @@ func TestPathsFiltersErrorsAndCancellation(t *testing.T) { return &api.PathStats{Until: until.UnixMilli(), Receptions: 4, Hashed: 1, Empty: 1, Trace: 1, Unclassified: 1}, tc.err }} w := httptest.NewRecorder() - StatsRouter(reader, nil).ServeHTTP(w, httptest.NewRequest("GET", "/paths?since=0&until=2592000000"+tc.query, nil)) + StatsRouter(reader, StatsOptions{}).ServeHTTP(w, httptest.NewRequest("GET", "/paths?since=0&until=2592000000"+tc.query, nil)) if w.Code != tc.status || strings.Contains(w.Body.String(), "private detail") { t.Fatalf("%d %s", w.Code, w.Body.String()) } @@ -86,7 +86,7 @@ func TestPathsFiltersErrorsAndCancellation(t *testing.T) { return nil, ctx.Err() }} w := httptest.NewRecorder() - StatsRouter(r, nil).ServeHTTP(w, httptest.NewRequest("GET", "/paths?since=0&until=1", nil).WithContext(ctx)) + StatsRouter(r, StatsOptions{}).ServeHTTP(w, httptest.NewRequest("GET", "/paths?since=0&until=1", nil).WithContext(ctx)) if !called || w.Body.Len() != 0 { t.Fatal("cancellation did not reach reader") } diff --git a/internal/api/handlers/regions.go b/internal/api/handlers/regions.go index 0b0f400..f80df79 100644 --- a/internal/api/handlers/regions.go +++ b/internal/api/handlers/regions.go @@ -75,10 +75,13 @@ func getRegion(reader api.Reader) http.HandlerFunc { } } +// rawStatsWindow is the longest window for endpoints that still aggregate raw-era snapshots. +const rawStatsWindow = 30 * 24 * time.Hour + // parseStatsWindow rounds both endpoints down to UTC hours. Polls within an // hour share a cache key; the response reports the effective window. A valid // sub-hour request can contain no complete buckets and return an empty result. -func parseStatsWindow(r *http.Request) (time.Time, time.Time, error) { +func parseStatsWindow(r *http.Request, maxWindow time.Duration) (time.Time, time.Time, error) { q := r.URL.Query() for _, key := range []string{"since", "until"} { if len(q[key]) != 1 || q.Get(key) == "" { @@ -87,8 +90,8 @@ func parseStatsWindow(r *http.Request) (time.Time, time.Time, error) { } since, errSince := strconv.ParseInt(q.Get("since"), 10, 64) until, errUntil := strconv.ParseInt(q.Get("until"), 10, 64) - if errSince != nil || errUntil != nil || since < 0 || until <= since || until > 253402300799999 || until-since > int64((30*24*time.Hour)/time.Millisecond) { - return time.Time{}, time.Time{}, fmt.Errorf("since and until must be epoch milliseconds between 0 and 253402300799999, with a positive window of at most 30 days") + if errSince != nil || errUntil != nil || since < 0 || until <= since || until > 253402300799999 || until-since > maxWindow.Milliseconds() { + return time.Time{}, time.Time{}, fmt.Errorf("since and until must be epoch milliseconds between 0 and 253402300799999, with a positive window of at most %d days", int(maxWindow/(24*time.Hour))) } return time.UnixMilli(since).UTC().Truncate(time.Hour), time.UnixMilli(until).UTC().Truncate(time.Hour), nil } diff --git a/internal/api/handlers/scope_stats_test.go b/internal/api/handlers/scope_stats_test.go index 111681d..363b1c8 100644 --- a/internal/api/handlers/scope_stats_test.go +++ b/internal/api/handlers/scope_stats_test.go @@ -11,6 +11,7 @@ import ( "net/http/httptest" "slices" "testing" + "time" "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" @@ -71,7 +72,7 @@ func TestGetStatsScopes_Filters(t *testing.T) { } return &api.Region{IATAs: []string{"YVR", "YYJ"}}, nil }, - getScopeStats: func(_ context.Context, iatas []string) ([]api.ScopeStats, error) { + getScopeStats: func(_ context.Context, iatas []string, _ time.Time) ([]api.ScopeStats, error) { called = true if !slices.Equal(iatas, tc.want) { t.Fatalf("IATAs = %v, want %v", iatas, tc.want) @@ -104,7 +105,7 @@ func TestGetStatsScopes_Filters(t *testing.T) { func TestGetStatsScopes_ReaderError(t *testing.T) { w := httptest.NewRecorder() - getStatsScopes(stubReader{getScopeStats: func(context.Context, []string) ([]api.ScopeStats, error) { + getStatsScopes(stubReader{getScopeStats: func(context.Context, []string, time.Time) ([]api.ScopeStats, error) { return nil, errors.New("database unavailable") }}, nil)(w, httptest.NewRequest(http.MethodGet, "/stats/scopes?iata=YVR", nil)) if w.Code != http.StatusInternalServerError { @@ -119,7 +120,7 @@ func testScopes(byIATA map[string][]string) *scopestore.ScopeStore { } func TestGetStatsScopes_FiltersToRegionMembers(t *testing.T) { - reader := stubReader{getScopeStats: func(context.Context, []string) ([]api.ScopeStats, error) { + reader := stubReader{getScopeStats: func(context.Context, []string, time.Time) ([]api.ScopeStats, error) { return []api.ScopeStats{{Name: "#elsewhere", PacketCount: 9}, {Name: "#on"}, {Name: "#yow", PacketCount: 4}}, nil }} members := testScopes(map[string][]string{"YOW": {"#yow", "#on"}}) diff --git a/internal/api/handlers/scopes.go b/internal/api/handlers/scopes.go index 41cca54..b6a2ba9 100644 --- a/internal/api/handlers/scopes.go +++ b/internal/api/handlers/scopes.go @@ -73,6 +73,7 @@ func listScopes(reader api.Reader, scopes ScopeMembership) http.HandlerFunc { // getScope godoc // // @Summary Get scope detail by name +// @Description packetCount sums packets per hour across every retained rollup hour; iatas are the IATAs that heard the scope's packets. Observer and node counts are current memberships. // @Tags Scopes // @Produce json // @Param name path string true "Scope name e.g. %23bc (URL-encoded #bc)" diff --git a/internal/api/handlers/signal.go b/internal/api/handlers/signal.go index b1029a9..a5fd864 100644 --- a/internal/api/handlers/signal.go +++ b/internal/api/handlers/signal.go @@ -32,7 +32,7 @@ import ( func getSignalStats(reader api.Reader) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { q := r.URL.Query() - since, until, err := parseStatsWindow(r) + since, until, err := parseStatsWindow(r, rawStatsWindow) if err != nil { respondError(w, http.StatusBadRequest, err.Error()) return diff --git a/internal/api/handlers/signal_test.go b/internal/api/handlers/signal_test.go index f9146fa..578372b 100644 --- a/internal/api/handlers/signal_test.go +++ b/internal/api/handlers/signal_test.go @@ -28,7 +28,7 @@ func (s signalReader) GetSignalStats(ctx context.Context, since, until time.Time func TestSignalRejectsInvalidWindow(t *testing.T) { for _, query := range []string{"", "since=0", "until=1", "since=x&until=1", "since=-1&until=1", "since=0&until=0", "since=2&until=1", "since=0&until=2592000001", "since=0&since=0&until=1", "since=0&until=1&until=2", "since=253402300799998&until=253402300800000"} { w := httptest.NewRecorder() - StatsRouter(nil, nil).ServeHTTP(w, httptest.NewRequest("GET", "/signal?"+query, nil)) + StatsRouter(nil, StatsOptions{}).ServeHTTP(w, httptest.NewRequest("GET", "/signal?"+query, nil)) if w.Code != 400 { t.Fatalf("query %q: %d %s", query, w.Code, w.Body.String()) } @@ -70,7 +70,7 @@ func TestSignalFiltersAndErrors(t *testing.T) { return &api.SignalStats{Until: until.UnixMilli(), Receptions: 5, Hourly: []api.SignalHour{}}, tc.err }} w := httptest.NewRecorder() - StatsRouter(reader, nil).ServeHTTP(w, httptest.NewRequest("GET", "/signal?since=0&until=2592000000"+tc.query, nil)) + StatsRouter(reader, StatsOptions{}).ServeHTTP(w, httptest.NewRequest("GET", "/signal?since=0&until=2592000000"+tc.query, nil)) if w.Code != tc.status || strings.Contains(w.Body.String(), "private detail") { t.Fatalf("%d %s", w.Code, w.Body.String()) } @@ -93,7 +93,7 @@ func TestSignalCancellationReachesReader(t *testing.T) { return nil, ctx.Err() }} w := httptest.NewRecorder() - StatsRouter(reader, nil).ServeHTTP(w, httptest.NewRequest("GET", "/signal?since=0&until=1", nil).WithContext(ctx)) + StatsRouter(reader, StatsOptions{}).ServeHTTP(w, httptest.NewRequest("GET", "/signal?since=0&until=1", nil).WithContext(ctx)) if !called || w.Body.Len() != 0 { t.Fatalf("cancellation: called=%v body=%s", called, w.Body.String()) } @@ -101,12 +101,12 @@ func TestSignalCancellationReachesReader(t *testing.T) { func TestStatsWindowSnapsPollingTimes(t *testing.T) { for _, query := range []string{"since=123&until=86400123", "since=3599999&until=89999999"} { - since, until, err := parseStatsWindow(httptest.NewRequest("GET", "/?"+query, nil)) + since, until, err := parseStatsWindow(httptest.NewRequest("GET", "/?"+query, nil), rawStatsWindow) if err != nil || since.UnixMilli() != 0 || until.UnixMilli() != 86400000 { t.Fatalf("%v %v %v", since, until, err) } } - since, until, err := parseStatsWindow(httptest.NewRequest("GET", "/?since=1&until=2", nil)) + since, until, err := parseStatsWindow(httptest.NewRequest("GET", "/?since=1&until=2", nil), rawStatsWindow) if err != nil || !since.Equal(until) { t.Fatal("sub-hour normalization") } diff --git a/internal/api/handlers/stats.go b/internal/api/handlers/stats.go index ca06423..dc6216f 100644 --- a/internal/api/handlers/stats.go +++ b/internal/api/handlers/stats.go @@ -4,6 +4,7 @@ package handlers import ( + "errors" "log/slog" "net/http" "slices" @@ -27,8 +28,29 @@ import ( // GET /stats/scopes → GetStatsScopes // // All endpoints accept an optional iata= filter (case-insensitive). -func StatsRouter(reader api.Reader, scopes ScopeMembership) http.Handler { +// StatsOptions configures StatsRouter. +type StatsOptions struct { + Scopes ScopeMembership // region-filtered scope lists; nil lists none + SeriesWindow time.Duration // longest /stats/series window, the rollup retention; 0 means 90 days +} + +// parseSince reads an optional epoch-ms since; zero means the endpoint's default window. +func parseSince(r *http.Request) (time.Time, error) { + p := r.URL.Query().Get("since") + if p == "" { + return time.Time{}, nil + } + ms, err := strconv.ParseInt(p, 10, 64) + if err != nil { + return time.Time{}, errors.New("since must be epoch milliseconds") + } + return time.UnixMilli(ms), nil +} + +func StatsRouter(reader api.Reader, opts StatsOptions) http.Handler { + scopes := opts.Scopes r := chi.NewRouter() + r.Get("/series", getStatsSeries(reader, opts.SeriesWindow)) r.Get("/overview", getStatsOverview(reader)) r.Get("/observations", getStatsObservations(reader)) r.Get("/signal", getSignalStats(reader)) @@ -48,7 +70,8 @@ func StatsRouter(reader api.Reader, scopes ScopeMembership) http.Handler { // getStatsOverview godoc // -// @Summary Network overview stats (last 24h) +// @Summary Network overview stats (last 24 rolled hours) +// @Description Summarises the 24 most recent hours that can have been rolled (an hour rolls about 95 minutes after it closes); since/until report that window. Packets count once per hour heard. Use /stats/series for sparklines. // @Tags Stats // @Produce json // @Param iatas query string false "Comma-separated IATA codes" @@ -67,6 +90,9 @@ func getStatsOverview(reader api.Reader) http.HandlerFunc { return } iatas = append(iatas, regionIATAs...) + if len(iatas) == 0 { + iatas = []string{""} // an empty region matches nothing, not everything + } } overview, err := reader.GetStatsOverview(r.Context(), iatas) if err != nil { @@ -80,7 +106,8 @@ func getStatsOverview(reader api.Reader) http.HandlerFunc { // getStatsObservations godoc // -// @Summary Hourly observation time series +// @Summary Hourly observation counts per IATA +// @Description Observation counts only; distinct packet and observer counts don't sum across IATAs, so read them from /stats/series. // @Tags Stats // @Produce json // @Param iatas query string false "Comma-separated IATA codes" @@ -100,15 +127,14 @@ func getStatsObservations(reader api.Reader) http.HandlerFunc { return } iatas = append(iatas, regionIATAs...) - } - var since time.Time - if p := r.URL.Query().Get("since"); p != "" { - ms, err := strconv.ParseInt(p, 10, 64) - if err != nil { - respondError(w, http.StatusBadRequest, "since must be epoch milliseconds") - return + if len(iatas) == 0 { + iatas = []string{""} // an empty region matches nothing, not everything } - since = time.UnixMilli(ms) + } + since, err := parseSince(r) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return } points, err := reader.GetStatsObservations(r.Context(), iatas, since) if err != nil { @@ -142,15 +168,14 @@ func getStatsPayloadBreakdown(reader api.Reader) http.HandlerFunc { return } iatas = append(iatas, regionIATAs...) - } - var since time.Time - if p := r.URL.Query().Get("since"); p != "" { - ms, err := strconv.ParseInt(p, 10, 64) - if err != nil { - respondError(w, http.StatusBadRequest, "since must be epoch milliseconds") - return + if len(iatas) == 0 { + iatas = []string{""} // an empty region matches nothing, not everything } - since = time.UnixMilli(ms) + } + since, err := parseSince(r) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return } breakdown, err := reader.GetStatsPayloadBreakdown(r.Context(), iatas, since) if err != nil { @@ -164,12 +189,14 @@ func getStatsPayloadBreakdown(reader api.Reader) http.HandlerFunc { // getStatsTopNodes godoc // -// @Summary Top N nodes by observation count (from materialized view) +// @Summary Top N nodes by advert hearings (last 7 days by default) +// @Description nodeId is null when the node row has been deleted; publicKey always identifies it. // @Tags Stats // @Produce json // @Param iatas query string false "Comma-separated IATA codes" // @Param regionId query int false "Filter by region ID, expands to member IATAs" // @Param region query string false "Filter by region slug, expands to member IATAs" +// @Param since query int false "Start of window epoch ms, rounded down to the hour (default 7 days ago)" // @Param limit query int false "Max results (default 10); must be positive, values above 200 are clamped" minimum(1) maximum(200) // @Success 200 {array} api.TopNode // @Failure 500 {object} handlers.APIError @@ -184,13 +211,21 @@ func getStatsTopNodes(reader api.Reader) http.HandlerFunc { return } iatas = append(iatas, regionIATAs...) + if len(iatas) == 0 { + iatas = []string{""} // an empty region matches nothing, not everything + } } limit, err := parseLimit(r, 10) if err != nil { respondError(w, http.StatusBadRequest, err.Error()) return } - nodes, err := reader.GetStatsTopNodes(r.Context(), iatas, limit) + since, err := parseSince(r) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return + } + nodes, err := reader.GetStatsTopNodes(r.Context(), iatas, since, limit) if err != nil { slog.Error("api: GetStatsTopNodes failed", "component", "api", "error", err) respondError(w, http.StatusInternalServerError, "internal server error") @@ -223,15 +258,14 @@ func getStatsTopObservers(reader api.Reader) http.HandlerFunc { return } iatas = append(iatas, regionIATAs...) - } - var since time.Time - if p := r.URL.Query().Get("since"); p != "" { - ms, err := strconv.ParseInt(p, 10, 64) - if err != nil { - respondError(w, http.StatusBadRequest, "since must be epoch milliseconds") - return + if len(iatas) == 0 { + iatas = []string{""} // an empty region matches nothing, not everything } - since = time.UnixMilli(ms) + } + since, err := parseSince(r) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return } limit, err := parseLimit(r, 10) if err != nil { @@ -251,6 +285,7 @@ func getStatsTopObservers(reader api.Reader) http.HandlerFunc { // getStatsTopAdvertisers godoc // // @Summary Top N nodes by distinct ADVERT packet count (last 24h by default) +// @Description Each advert counts once per hour heard, however many requested IATAs heard it. nodeId is null when the node row has been deleted. // @Tags Stats // @Produce json // @Param iatas query string false "Comma-separated IATA codes" @@ -271,15 +306,14 @@ func getStatsTopAdvertisers(reader api.Reader) http.HandlerFunc { return } iatas = append(iatas, regionIATAs...) - } - var since time.Time - if p := r.URL.Query().Get("since"); p != "" { - ms, err := strconv.ParseInt(p, 10, 64) - if err != nil { - respondError(w, http.StatusBadRequest, "since must be epoch milliseconds") - return + if len(iatas) == 0 { + iatas = []string{""} // an empty region matches nothing, not everything } - since = time.UnixMilli(ms) + } + since, err := parseSince(r) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return } limit, err := parseLimit(r, 10) if err != nil { @@ -318,6 +352,9 @@ func getStatsClockDrift(reader api.Reader) http.HandlerFunc { return } iatas = append(iatas, regionIATAs...) + if len(iatas) == 0 { + iatas = []string{""} // an empty region matches nothing, not everything + } } limit, err := parseLimit(r, 10) if err != nil { @@ -357,15 +394,14 @@ func getStatsTopTalkers(reader api.Reader) http.HandlerFunc { return } iatas = append(iatas, regionIATAs...) - } - var since time.Time - if p := r.URL.Query().Get("since"); p != "" { - ms, err := strconv.ParseInt(p, 10, 64) - if err != nil { - respondError(w, http.StatusBadRequest, "since must be epoch milliseconds") - return + if len(iatas) == 0 { + iatas = []string{""} // an empty region matches nothing, not everything } - since = time.UnixMilli(ms) + } + since, err := parseSince(r) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return } limit, err := parseLimit(r, 10) if err != nil { @@ -405,6 +441,9 @@ func getStatsRadioPresets(reader api.Reader) http.HandlerFunc { return } iatas = append(iatas, regionIATAs...) + if len(iatas) == 0 { + iatas = []string{""} // an empty region matches nothing, not everything + } } presets, err := reader.GetRadioPresets(r.Context(), preset, iatas) if err != nil { @@ -418,13 +457,14 @@ func getStatsRadioPresets(reader api.Reader) http.HandlerFunc { // getStatsScopes godoc // // @Summary Scope statistics -// @Description Counts each packet, observer and node once per scope. IATA filters use retained observations for packets/observers and node IATA memberships for nodes. Without filters, returns global totals for every stored scope. With filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; those remain listed with zero counts. An empty region returns an empty array. +// @Description Packets are those heard since the window start, each counted once per hour however many of the requested IATAs heard it. Observers and nodes are current memberships: observers filter by the IATA they last reported from, nodes by their IATA memberships. With filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; those remain listed with zero counts. An empty region returns an empty array. // @Tags Stats // @Produce json // @Param iatas query string false "Comma-separated IATA codes" // @Param iata query string false "Single IATA code; used when iatas is absent" // @Param regionId query int false "Filter by region ID, expands to member IATAs" // @Param region query string false "Filter by region slug, expands to member IATAs; combined with explicit IATAs" +// @Param since query int false "Start of packet window epoch ms, rounded down to the hour (default 7 days ago)" // @Success 200 {object} []api.ScopeStats // @Failure 400 {object} handlers.APIError // @Failure 500 {object} handlers.APIError @@ -444,7 +484,12 @@ func getStatsScopes(reader api.Reader, scopes ScopeMembership) http.HandlerFunc return } } - stats, err := reader.GetScopeStats(r.Context(), iatas) + since, err := parseSince(r) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return + } + stats, err := reader.GetScopeStats(r.Context(), iatas, since) if err != nil { respondError(w, http.StatusInternalServerError, "internal server error") return @@ -481,6 +526,9 @@ func getStatsNodeTypes(reader api.Reader) http.HandlerFunc { return } iatas = append(iatas, regionIATAs...) + if len(iatas) == 0 { + iatas = []string{""} // an empty region matches nothing, not everything + } } result, err := reader.GetStatsNodeTypes(r.Context(), iatas) if err != nil { diff --git a/internal/api/handlers/stats_region_test.go b/internal/api/handlers/stats_region_test.go new file mode 100644 index 0000000..4ed483f --- /dev/null +++ b/internal/api/handlers/stats_region_test.go @@ -0,0 +1,90 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "context" + "net/http/httptest" + "reflect" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" +) + +// iataRecorder records the IATA filter each stats read receives. +type iataRecorder struct { + stubReader + got *[]string +} + +func (r iataRecorder) rec(iatas []string) { *r.got = append([]string{}, iatas...) } + +func (r iataRecorder) GetStatsOverview(_ context.Context, iatas []string) (*api.StatsOverview, error) { + r.rec(iatas) + return &api.StatsOverview{}, nil +} +func (r iataRecorder) GetStatsObservations(_ context.Context, iatas []string, _ time.Time) ([]api.ObservationPoint, error) { + r.rec(iatas) + return nil, nil +} +func (r iataRecorder) GetStatsPayloadBreakdown(_ context.Context, iatas []string, _ time.Time) ([]api.PayloadBreakdownItem, error) { + r.rec(iatas) + return nil, nil +} +func (r iataRecorder) GetStatsTopNodes(_ context.Context, iatas []string, _ time.Time, _ int32) ([]api.TopNode, error) { + r.rec(iatas) + return nil, nil +} +func (r iataRecorder) GetStatsTopObservers(_ context.Context, iatas []string, _ time.Time, _ int32) ([]api.TopObserver, error) { + r.rec(iatas) + return nil, nil +} +func (r iataRecorder) GetStatsTopAdvertisers(_ context.Context, iatas []string, _ time.Time, _ int32) ([]api.TopAdvertiser, error) { + r.rec(iatas) + return nil, nil +} +func (r iataRecorder) GetStatsClockDrift(_ context.Context, iatas []string, _ int32) ([]api.ClockDriftEntry, error) { + r.rec(iatas) + return nil, nil +} +func (r iataRecorder) GetStatsTopTalkers(_ context.Context, iatas []string, _ time.Time, _ int32) ([]api.TopTalker, error) { + r.rec(iatas) + return nil, nil +} +func (r iataRecorder) GetRadioPresets(_ context.Context, _ string, iatas []string) ([]api.RadioPreset, error) { + r.rec(iatas) + return nil, nil +} +func (r iataRecorder) GetStatsNodeTypes(_ context.Context, iatas []string) ([]api.NodeTypeCount, error) { + r.rec(iatas) + return nil, nil +} + +// A region with no IATAs must filter to nothing, not fall back to every IATA. +func TestStatsEmptyRegionMatchesNothing(t *testing.T) { + for _, path := range []string{"overview", "observations", "payload-breakdown", "top-nodes", "top-observers", + "top-advertisers", "clock-drift", "top-talkers", "radio-presets", "node-types"} { + for _, tc := range []struct { + region string + want []string + }{ + {"empty", []string{""}}, + {"west", []string{"YVR"}}, + } { + var got []string + reader := iataRecorder{got: &got, stubReader: stubReader{getRegionBySlug: func(_ context.Context, slug string) (*api.Region, error) { + if slug == "empty" { + return &api.Region{}, nil + } + return &api.Region{IATAs: []string{"YVR"}}, nil + }}} + w := httptest.NewRecorder() + StatsRouter(reader, StatsOptions{}).ServeHTTP(w, httptest.NewRequest("GET", "/"+path+"?region="+tc.region, nil)) + if w.Code != 200 || !reflect.DeepEqual(got, tc.want) { + t.Errorf("/stats/%s?region=%s: status %d, iatas %q; want 200, %q", path, tc.region, w.Code, got, tc.want) + } + } + } +} diff --git a/internal/api/handlers/stats_series.go b/internal/api/handlers/stats_series.go new file mode 100644 index 0000000..564a701 --- /dev/null +++ b/internal/api/handlers/stats_series.go @@ -0,0 +1,79 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "context" + "errors" + "log/slog" + "net/http" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" +) + +const defaultSeriesWindow = 90 * 24 * time.Hour + +// getStatsSeries godoc +// +// @Summary Hourly network activity from the analytics rollups +// @Description One entry per UTC hour in [since, until) plus a summary over the complete hours. Both bounds round down to UTC hours and since is clamped to the rollup retention; the response reports the effective window. An hour is rolled about 95 minutes after it closes, so recent hours are "missing" until then; "partial" hours lost raw rows before they could be rolled and never get values. Counts (observations, packets) sum across hours, with a packet counted once in each hour it was heard; observers, IATAs and scopes are distinct across the whole window; averages are sum / samples. revision changes whenever any rolled hour changes. +// @Tags Stats +// @Produce json +// @Param since query int true "Inclusive start, epoch milliseconds" +// @Param until query int true "Exclusive end, epoch milliseconds; the window is at most the rollup retention (default 90 days)" +// @Param iatas query string false "Comma-separated reception IATA codes" +// @Param regionId query int false "Region ID, expands to member IATAs" +// @Param region query string false "Region slug, expands to member IATAs" +// @Success 200 {object} api.StatsSeries +// @Failure 400 {object} handlers.APIError +// @Failure 500 {object} handlers.APIError +// @Failure 503 {object} handlers.APIError +// @Router /stats/series [get] +func getStatsSeries(reader api.Reader, maxWindow time.Duration) http.HandlerFunc { + if maxWindow <= 0 { + maxWindow = defaultSeriesWindow + } + return func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + since, until, err := parseStatsWindow(r, maxWindow) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return + } + if oldest := time.Now().UTC().Add(-maxWindow).Truncate(time.Hour); since.Before(oldest) { + since = oldest + } + if !since.Before(until) { + since = until + } + ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second) + defer cancel() + iatas := parseIATAs(r) + if q.Get("regionId") != "" || q.Get("region") != "" { + regionIATAs, err := resolveRegionIATAs(ctx, q.Get("regionId"), q.Get("region"), reader) + if err != nil { + respondError(w, http.StatusBadRequest, "region not found") + return + } + iatas = append(iatas, regionIATAs...) + if len(iatas) == 0 { + iatas = []string{""} // an empty region matches nothing, not everything + } + } + series, err := reader.GetStatsSeries(ctx, since, until, iatas) + switch { + case errors.Is(err, context.DeadlineExceeded) || errors.Is(ctx.Err(), context.DeadlineExceeded): + respondError(w, http.StatusServiceUnavailable, "series query timed out; try a shorter time period") + case err != nil: + if r.Context().Err() != nil { + return + } + slog.Error("stats series failed", "error", err) + respondError(w, http.StatusInternalServerError, "internal server error") + default: + respond(w, http.StatusOK, series) + } + } +} diff --git a/internal/api/handlers/stats_series_test.go b/internal/api/handlers/stats_series_test.go new file mode 100644 index 0000000..3ba2624 --- /dev/null +++ b/internal/api/handlers/stats_series_test.go @@ -0,0 +1,76 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "context" + "errors" + "fmt" + "net/http/httptest" + "reflect" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" +) + +func TestGetStatsSeries(t *testing.T) { + now := time.Now().UTC() + until := now.Truncate(time.Hour) + since := until.Add(-48 * time.Hour) + window := fmt.Sprintf("since=%d&until=%d", since.UnixMilli()+1234, until.UnixMilli()+1234) + for _, tc := range []struct { + name, query string + maxWindow time.Duration + wantSince time.Time + iatas []string + err error + status int + }{ + {"global", window, 0, since, nil, nil, 200}, + {"IATAs", window + "&iatas=yvr,yyj", 0, since, []string{"YVR", "YYJ"}, nil, 200}, + {"region", window + "®ion=west", 0, since, []string{"YVR"}, nil, 200}, + {"empty region matches nothing", window + "®ion=empty", 0, since, []string{""}, nil, 200}, + {"missing region", window + "®ion=missing", 0, since, nil, nil, 400}, + {"missing until", fmt.Sprintf("since=%d", since.UnixMilli()), 0, since, nil, nil, 400}, + {"window over retention", window, 24 * time.Hour, since, nil, nil, 400}, + // The window fits, but its start predates what the rollups still hold. + {"since clamped to retention", fmt.Sprintf("since=%d&until=%d", until.Add(-50*time.Hour).UnixMilli(), until.Add(-2*time.Hour).UnixMilli()), + 48 * time.Hour, until.Add(-48 * time.Hour), nil, nil, 200}, + {"timeout", window, 0, since, nil, context.DeadlineExceeded, 503}, + {"database error", window, 0, since, nil, errors.New("private detail"), 500}, + } { + t.Run(tc.name, func(t *testing.T) { + reader := stubReader{ + getRegionBySlug: func(_ context.Context, slug string) (*api.Region, error) { + switch slug { + case "empty": + return &api.Region{}, nil + case "missing": + return nil, errors.New("not found") + } + return &api.Region{IATAs: []string{"YVR"}}, nil + }, + getStatsSeries: func(ctx context.Context, gotSince, gotUntil time.Time, iatas []string) (*api.StatsSeries, error) { + if deadline, ok := ctx.Deadline(); !ok || time.Until(deadline) > 15*time.Second { + t.Error("missing bounded query deadline") + } + if !gotSince.Equal(tc.wantSince) || !reflect.DeepEqual(iatas, tc.iatas) { + t.Errorf("args %s %v; want %s %v", gotSince, iatas, tc.wantSince, tc.iatas) + } + if gotUntil.Sub(gotSince) > 48*time.Hour || gotUntil.Minute() != 0 { + t.Errorf("until %s not snapped within the window", gotUntil) + } + return &api.StatsSeries{Since: gotSince.UnixMilli()}, tc.err + }, + } + w := httptest.NewRecorder() + StatsRouter(reader, StatsOptions{SeriesWindow: tc.maxWindow}).ServeHTTP(w, httptest.NewRequest("GET", "/series?"+tc.query, nil)) + if w.Code != tc.status || strings.Contains(w.Body.String(), "private detail") { + t.Fatalf("%d %s", w.Code, w.Body.String()) + } + }) + } +} diff --git a/internal/api/handlers/stats_test.go b/internal/api/handlers/stats_test.go index 86d9d9b..d601959 100644 --- a/internal/api/handlers/stats_test.go +++ b/internal/api/handlers/stats_test.go @@ -117,7 +117,7 @@ func TestGetStatsPayloadBreakdown_OK(t *testing.T) { func TestGetStatsTopNodes_OK(t *testing.T) { r := chi.NewRouter() r.Get("/stats/top-nodes", getStatsTopNodes(stubReader{ - getStatsTopNodes: func(_ context.Context, _ []string, _ int32) ([]api.TopNode, error) { + getStatsTopNodes: func(_ context.Context, _ []string, _ time.Time, _ int32) ([]api.TopNode, error) { return []api.TopNode{{IATA: "YVR", ObservationCount: 50}}, nil }, })) @@ -282,7 +282,7 @@ func TestGetStatsRadioPresets_OK(t *testing.T) { func TestGetStatsScopes_OK(t *testing.T) { r := chi.NewRouter() r.Get("/stats/scopes", getStatsScopes(stubReader{ - getScopeStats: func(_ context.Context, _ []string) ([]api.ScopeStats, error) { + getScopeStats: func(_ context.Context, _ []string, _ time.Time) ([]api.ScopeStats, error) { return []api.ScopeStats{{Name: "#bc", PacketCount: 100}}, nil }, }, nil)) diff --git a/internal/api/handlers/stub_reader_test.go b/internal/api/handlers/stub_reader_test.go index 3d3626c..d4e148b 100644 --- a/internal/api/handlers/stub_reader_test.go +++ b/internal/api/handlers/stub_reader_test.go @@ -16,6 +16,7 @@ import ( // Unset fields return zero values. Use it for both validation tests // (leave all fields nil) and happy path tests (set only what you need). type stubReader struct { + getStatsSeries func(context.Context, time.Time, time.Time, []string) (*api.StatsSeries, error) getRouteEvidence func(context.Context, string, string, api.RouteEvidenceQuery) (*api.RouteEvidence, error) getObserverComparison func(context.Context, uuid.UUID, uuid.UUID, time.Time, time.Time, []string) (*api.ObserverComparison, error) listIATAs func(ctx context.Context) ([]api.IATA, error) @@ -47,12 +48,12 @@ type stubReader struct { getStatsOverview func(ctx context.Context, iatas []string) (*api.StatsOverview, error) getStatsObservations func(ctx context.Context, iatas []string, since time.Time) ([]api.ObservationPoint, error) getStatsPayloadBreakdown func(ctx context.Context, iatas []string, since time.Time) ([]api.PayloadBreakdownItem, error) - getStatsTopNodes func(ctx context.Context, iatas []string, limit int32) ([]api.TopNode, error) + getStatsTopNodes func(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopNode, error) getStatsTopObservers func(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopObserver, error) getStatsTopAdvertisers func(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopAdvertiser, error) getStatsClockDrift func(ctx context.Context, iatas []string, limit int32) ([]api.ClockDriftEntry, error) getStatsTopTalkers func(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopTalker, error) - getScopeStats func(ctx context.Context, iatas []string) ([]api.ScopeStats, error) + getScopeStats func(ctx context.Context, iatas []string, since time.Time) ([]api.ScopeStats, error) getStatsNodeTypes func(ctx context.Context, iatas []string) ([]api.NodeTypeCount, error) getScopeNames func(ctx context.Context) ([]string, error) getScopeByName func(ctx context.Context, name string) (*api.ScopeDetail, error) @@ -70,6 +71,13 @@ func (s stubReader) GetSignalStats(context.Context, time.Time, time.Time, []stri return nil, nil } +func (s stubReader) GetStatsSeries(ctx context.Context, since, until time.Time, iatas []string) (*api.StatsSeries, error) { + if s.getStatsSeries != nil { + return s.getStatsSeries(ctx, since, until, iatas) + } + return &api.StatsSeries{}, nil +} + func (s stubReader) GetPathStats(context.Context, time.Time, time.Time, []string) (*api.PathStats, error) { return nil, nil } @@ -284,9 +292,9 @@ func (s stubReader) GetStatsPayloadBreakdown(ctx context.Context, iatas []string return nil, nil } -func (s stubReader) GetStatsTopNodes(ctx context.Context, iatas []string, limit int32) ([]api.TopNode, error) { +func (s stubReader) GetStatsTopNodes(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopNode, error) { if s.getStatsTopNodes != nil { - return s.getStatsTopNodes(ctx, iatas, limit) + return s.getStatsTopNodes(ctx, iatas, since, limit) } return nil, nil } @@ -319,9 +327,9 @@ func (s stubReader) GetStatsTopTalkers(ctx context.Context, iatas []string, sinc return nil, nil } -func (s stubReader) GetScopeStats(ctx context.Context, iatas []string) ([]api.ScopeStats, error) { +func (s stubReader) GetScopeStats(ctx context.Context, iatas []string, since time.Time) ([]api.ScopeStats, error) { if s.getScopeStats != nil { - return s.getScopeStats(ctx, iatas) + return s.getScopeStats(ctx, iatas, since) } return nil, nil } diff --git a/internal/api/handlers/traces.go b/internal/api/handlers/traces.go index 7b2a7d4..54fd96d 100644 --- a/internal/api/handlers/traces.go +++ b/internal/api/handlers/traces.go @@ -32,8 +32,8 @@ func TracesRouter(reader api.Reader) http.Handler { // @Param iatas query string false "Filter by IATA code(s), comma-separated" // @Param region query string false "Filter by region slug" // @Param regionId query int false "Filter by region ID" -// @Param scope query string false "Filter by transport scope name" -// @Param type query string false "Filter by type: TRACE or PING (default: all)" +// @Param scope query string false "Filter by the tag's first transport scope name" +// @Param type query string false "Filter by type: TRACE (any multi-hop packet) or PING (default: all)" // @Param since query int false "Filter by first_heard_at >= since (epoch ms)" // @Param until query int false "Filter by first_heard_at <= until (epoch ms)" // @Param cursor query int false "last_heard_at epoch ms of last item for pagination" diff --git a/internal/api/reader.go b/internal/api/reader.go index ba6c7cd..e74a253 100644 --- a/internal/api/reader.go +++ b/internal/api/reader.go @@ -163,6 +163,9 @@ type Reader interface { // since defines the start of the window; pass zero time for default (last 7 days). GetStatsObservations(ctx context.Context, iatas []string, since time.Time) ([]ObservationPoint, error) + // GetStatsSeries returns hourly rollup metrics for [since, until), both on UTC hours. + GetStatsSeries(ctx context.Context, since, until time.Time, iatas []string) (*StatsSeries, error) + // GetSignalStats aggregates retained reception readings in [since, until). GetSignalStats(ctx context.Context, since, until time.Time, iatas []string) (*SignalStats, error) @@ -176,7 +179,7 @@ type Reader interface { // GetStatsTopNodes returns the top N nodes by observation count. // Pass nil for iatas to return stats across all IATAs. - GetStatsTopNodes(ctx context.Context, iatas []string, limit int32) ([]TopNode, error) + GetStatsTopNodes(ctx context.Context, iatas []string, since time.Time, limit int32) ([]TopNode, error) // GetStatsTopObservers returns the top N observers by observation count. // Pass nil for iatas to return stats across all IATAs. @@ -201,7 +204,7 @@ type Reader interface { // GetScopeStats returns aggregate packet, observer and node counts per transport scope. // Pass empty iatas for global totals. IATAs must be uppercase. - GetScopeStats(ctx context.Context, iatas []string) ([]ScopeStats, error) + GetScopeStats(ctx context.Context, iatas []string, since time.Time) ([]ScopeStats, error) // GetStatsNodeTypes returns node counts grouped by type, optionally filtered by IATA. GetStatsNodeTypes(ctx context.Context, iatas []string) ([]NodeTypeCount, error) diff --git a/internal/api/router/options.go b/internal/api/router/options.go index 0d9076e..e838834 100644 --- a/internal/api/router/options.go +++ b/internal/api/router/options.go @@ -5,6 +5,7 @@ package router import ( "net/http" + "time" "github.com/MeshCore-Beacon/beacon-server/internal/api/handlers" "github.com/MeshCore-Beacon/beacon-server/internal/config" @@ -21,6 +22,7 @@ type Options struct { Auth config.AuthConfig RateLimit config.ResolvedRateLimitConfig Scopes handlers.ScopeMembership // region-filtered scope lists; nil lists none + RollupRetention time.Duration // longest historical stats window // AdminRoutes mounts operator-only subrouters at literal paths such as // "/accounts". Each feature owns its dependencies, methods and handlers. // The router applies authentication to every path and method in this group. diff --git a/internal/api/router/router.go b/internal/api/router/router.go index b8a5cea..d6bbbc9 100644 --- a/internal/api/router/router.go +++ b/internal/api/router/router.go @@ -108,7 +108,7 @@ func New(h *hub.Hub, reader api.Reader, workers []*ingest.Worker, opts Options) r.Mount("/regions", handlers.RegionsRouter(reader)) r.Mount("/routes", handlers.RoutesRouter(reader)) r.Mount("/scopes", handlers.ScopesRouter(reader, opts.Scopes)) - r.Mount("/stats", handlers.StatsRouter(reader, opts.Scopes)) + r.Mount("/stats", handlers.StatsRouter(reader, handlers.StatsOptions{Scopes: opts.Scopes, SeriesWindow: opts.RollupRetention})) r.Mount("/traces", handlers.TracesRouter(reader)) }) diff --git a/internal/api/stats.go b/internal/api/stats.go index 1485639..bb1e8eb 100644 --- a/internal/api/stats.go +++ b/internal/api/stats.go @@ -29,21 +29,58 @@ type RadioPreset struct { } // StatsOverview is the top-level network summary for the overview endpoint. +// Covers the last 24 hours that can have been rolled; hours still missing contribute nothing. type StatsOverview struct { - TotalPackets int64 `json:"totalPackets"` + TotalPackets int64 `json:"totalPackets"` // distinct per hour (see StatsSeriesValues) TotalObservations int64 `json:"totalObservations"` ActiveObservers int64 `json:"activeObservers"` ActiveIATAs int64 `json:"activeIatas"` WindowHours int `json:"windowHours"` // always 24 for now + Since int64 `json:"since"` // epoch ms, start of the window + Until int64 `json:"until"` // epoch ms, exclusive end of the window +} + +// StatsSeries is hourly network activity read from the analytics rollups. Only complete +// hours carry values; the summary combines them (see docs/historical-stats.md). +type StatsSeries struct { + Since int64 `json:"since"` // effective start, epoch ms on a UTC hour + Until int64 `json:"until"` // effective exclusive end, epoch ms on a UTC hour + Revision int64 `json:"revision"` // changes whenever rolled hours, their status or coverage change + EarliestComplete *int64 `json:"earliestComplete"` // first complete hour held, epoch ms; null if none + CompleteHours int `json:"completeHours"` // hours contributing to summary + Hours []StatsSeriesHour `json:"hours"` + Summary StatsSeriesValues `json:"summary"` +} + +// StatsSeriesHour is one UTC hour of a StatsSeries. +type StatsSeriesHour struct { + Hour int64 `json:"hour"` // epoch ms, start of the hour + Status string `json:"status" enums:"complete,partial,missing"` // partial: raw rows deleted before it was rolled + Values *StatsSeriesValues `json:"values"` // null unless complete +} + +// StatsSeriesValues are the card metrics for an hour or a whole series. Packet counts are +// distinct per hour, so a packet heard across an hour boundary counts in both hours. +type StatsSeriesValues struct { + Observations int64 `json:"observations"` + UniquePackets int64 `json:"uniquePackets"` + ActiveObservers int64 `json:"activeObservers"` // distinct across the hour or window + ActiveIATAs int64 `json:"activeIatas"` // distinct across the hour or window + ScopedPackets int64 `json:"scopedPackets"` + ActiveScopes int64 `json:"activeScopes"` // distinct across the hour or window + MaxPathEntries int32 `json:"maxPathEntries"` + SNRSum float64 `json:"snrSum"` // average = snrSum / snrSamples + SNRSamples int64 `json:"snrSamples"` + RSSISum float64 `json:"rssiSum"` // average = rssiSum / rssiSamples + RSSISamples int64 `json:"rssiSamples"` } // ObservationPoint is a single time-bucketed observation count for charting. +// Distinct counts don't sum across IATAs, so use /stats/series for those. type ObservationPoint struct { Hour int64 `json:"hour"` // epoch ms, start of the 1-hour bucket IATA string `json:"iata"` ObservationCount int64 `json:"observationCount"` - UniquePackets int64 `json:"uniquePackets"` - ActiveObservers int64 `json:"activeObservers"` } // PayloadBreakdownItem is a single payload type with its observation count. @@ -61,15 +98,16 @@ type ScopeStats struct { NodeCount int64 `json:"nodeCount"` // distinct nodes with this as their default scope } -// TopNode is a node ranked by observation count from the mv_top_nodes_by_iata materialized view. +// TopNode is a node ranked by how often its adverts were heard in the window. type TopNode struct { - NodeID uuid.UUID `json:"nodeId"` - NodeName *string `json:"nodeName,omitempty"` - NodeType int16 `json:"nodeType"` - NodeTypeName string `json:"nodeTypeName"` - IATA string `json:"iata"` - ObservationCount int64 `json:"observationCount"` - LastHeard int64 `json:"lastHeard"` // epoch ms + NodeID *uuid.UUID `json:"nodeId"` // null once the node row has been deleted + PublicKey string `json:"publicKey"` // hex + NodeName *string `json:"nodeName,omitempty"` + NodeType int16 `json:"nodeType"` + NodeTypeName string `json:"nodeTypeName"` + IATA string `json:"iata"` + ObservationCount int64 `json:"observationCount"` + LastHeard int64 `json:"lastHeard"` // epoch ms } // TopObserver is an observer ranked by observation count. @@ -84,12 +122,13 @@ type TopObserver struct { // TopAdvertiser is a node ranked by distinct ADVERT packet count within the requested // window. Count is per-advert, not per-hearing -- see GetStatsTopAdvertisers. type TopAdvertiser struct { - NodeID uuid.UUID `json:"nodeId"` - NodeName *string `json:"nodeName,omitempty"` - NodeType int16 `json:"nodeType"` - NodeTypeName string `json:"nodeTypeName"` - IATA string `json:"iata"` - AdvertCount int64 `json:"advertCount"` + NodeID *uuid.UUID `json:"nodeId"` // null once the node row has been deleted + PublicKey string `json:"publicKey"` // hex + NodeName *string `json:"nodeName,omitempty"` + NodeType int16 `json:"nodeType"` + NodeTypeName string `json:"nodeTypeName"` + IATA string `json:"iata"` + AdvertCount int64 `json:"advertCount"` // FloodAdvertCount/DirectAdvertCount split AdvertCount by how the advert was routed: // flood = route type 0 (transport_flood) or 1 (flood), broadcast with no known path; // direct = route type 2 (direct) or 3 (transport_direct), routed along a known path. diff --git a/internal/background/rollup.go b/internal/background/rollup.go new file mode 100644 index 0000000..a23fe30 --- /dev/null +++ b/internal/background/rollup.go @@ -0,0 +1,161 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package background + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/MeshCore-Beacon/beacon-server/db" +) + +// Per-pass budgets keep one pass short; the boot catch-up loops passes instead. +const ( + rollupInterval = 5 * time.Minute + rollupMissingBudget = 48 + rollupDirtyBatch = 24 + rollupDirtyBudget = 168 // a key import can dirty a week of hours at once +) + +// RollupSession is the lock-holding connection a pass works through (see db.RollupSession). +type RollupSession interface { + RegisterHours(ctx context.Context, since time.Time) error + MarkPartialHours(ctx context.Context) error + MissingHours(ctx context.Context, limit int32) ([]time.Time, error) + DirtyHours(ctx context.Context, limit int32) ([]time.Time, error) + RollHour(ctx context.Context, hour time.Time) (db.RollOutcome, bool, error) + Close(ctx context.Context) +} + +// Rollup rolls closed hours into the analytics rollup tables. Only the process holding +// the advisory lock does any work; the others skip the pass. +type Rollup struct { + begin func(context.Context) (RollupSession, bool, error) + packetRetention time.Duration + rollupRetention time.Duration + now func() time.Time +} + +// NewRollup rolls hours still inside both retentions using store's rollup lock. +func NewRollup(store *db.Store, packetRetention, rollupRetention time.Duration) *Rollup { + return &Rollup{ + begin: func(ctx context.Context) (RollupSession, bool, error) { + s, ok, err := store.BeginRollup(ctx) + if s == nil { + return nil, ok, err + } + return s, ok, err + }, + packetRetention: packetRetention, + rollupRetention: rollupRetention, + now: time.Now, + } +} + +// Task runs a pass every five minutes. +func (r *Rollup) Task() Task { + return Task{ + Name: "analytics_rollup", + Interval: rollupInterval, + Run: func(ctx context.Context) error { + _, _, err := r.Pass(ctx) + return err + }, + } +} + +// CatchUp runs passes until the backlog fits in one, so a restart fills gaps without +// waiting for the ticker. +func (r *Rollup) CatchUp(ctx context.Context) error { + for { + _, more, err := r.Pass(ctx) + if err != nil || !more { + return err + } + } +} + +// registerSince is the first hour worth rolling: raw rows must still exist, and cleanup +// (DeleteOldRollups, hour < now - rollupRetention) must not delete it again next run. +func (r *Rollup) registerSince() time.Time { + now := r.now() + since := now.Add(-r.packetRetention) + if rc := now.Add(-r.rollupRetention); r.rollupRetention > 0 && rc.After(since) { + since = rc + } + if h := since.Truncate(time.Hour); h.Before(since) { + return h.Add(time.Hour) + } + return since +} + +// Pass registers and rolls missing hours oldest first, then re-rolls dirty ones. A failed +// hour doesn't stop the pass; its error is returned with the others. rolled counts hours +// attempted; more reports a budget ran out with work likely left. +func (r *Rollup) Pass(ctx context.Context) (rolled int, more bool, err error) { + s, ok, err := r.begin(ctx) + if err != nil || !ok { + return 0, false, err + } + defer func() { + closeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) + s.Close(closeCtx) + cancel() + }() + + if err := s.RegisterHours(ctx, r.registerSince()); err != nil { + return 0, false, fmt.Errorf("register hours: %w", err) + } + if err := s.MarkPartialHours(ctx); err != nil { + return 0, false, fmt.Errorf("mark partial hours: %w", err) + } + var errs []error + roll := func(hours []time.Time) { + for _, h := range hours { + if ctx.Err() != nil { + return + } + if _, _, err := s.RollHour(ctx, h); err != nil { + errs = append(errs, fmt.Errorf("roll %s: %w", h.UTC().Format(time.RFC3339), err)) + } + } + } + missing, err := s.MissingHours(ctx, rollupMissingBudget) + if err != nil { + return 0, false, fmt.Errorf("missing hours: %w", err) + } + roll(missing) + rolled, more = len(missing), len(missing) == rollupMissingBudget + // Hours that fail stay queued; never retry one within a pass. + tried := map[time.Time]bool{} + for budget := rollupDirtyBudget; budget > 0 && ctx.Err() == nil; { + dirty, err := s.DirtyHours(ctx, min(rollupDirtyBatch, int32(budget))) + if err != nil { + errs = append(errs, fmt.Errorf("dirty hours: %w", err)) + break + } + var fresh []time.Time + for _, h := range dirty { + if !tried[h] { + tried[h] = true + fresh = append(fresh, h) + } + } + if len(fresh) == 0 { + break + } + roll(fresh) + rolled += len(fresh) + budget -= len(fresh) + if budget == 0 { + more = true + } + } + if err := ctx.Err(); err != nil { + errs = append(errs, err) + } + return rolled, more, errors.Join(errs...) +} diff --git a/internal/background/rollup_test.go b/internal/background/rollup_test.go new file mode 100644 index 0000000..5b6d6ad --- /dev/null +++ b/internal/background/rollup_test.go @@ -0,0 +1,195 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package background + +import ( + "context" + "errors" + "fmt" + "reflect" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/db" +) + +type rollupStub struct { + calls []string + since time.Time + missing [][]time.Time // one batch per pass + dirty []time.Time + failAt map[time.Time]error + done map[time.Time]bool // rolled dirty hours leave the queue + closed int +} + +func (s *rollupStub) RegisterHours(_ context.Context, since time.Time) error { + s.calls = append(s.calls, "register") + s.since = since + return nil +} +func (s *rollupStub) MarkPartialHours(context.Context) error { + s.calls = append(s.calls, "partial") + return nil +} +func (s *rollupStub) MissingHours(_ context.Context, limit int32) ([]time.Time, error) { + s.calls = append(s.calls, fmt.Sprintf("missing(%d)", limit)) + if len(s.missing) == 0 { + return nil, nil + } + batch := s.missing[0] + s.missing = s.missing[1:] + return batch, nil +} +func (s *rollupStub) DirtyHours(_ context.Context, limit int32) ([]time.Time, error) { + s.calls = append(s.calls, fmt.Sprintf("dirty(%d)", limit)) + var out []time.Time + for _, h := range s.dirty { + if !s.done[h] && int32(len(out)) < limit { + out = append(out, h) + } + } + return out, nil +} +func (s *rollupStub) RollHour(_ context.Context, h time.Time) (db.RollOutcome, bool, error) { + s.calls = append(s.calls, "roll "+h.Format("15")) + if err := s.failAt[h]; err != nil { + return db.RollSkipped, false, err + } + if s.done == nil { + s.done = map[time.Time]bool{} + } + s.done[h] = true + return db.RollComplete, true, nil +} +func (s *rollupStub) Close(context.Context) { s.closed++ } + +func newTestRollup(s *rollupStub, locked bool, beginErr error) *Rollup { + now := time.Date(2026, 1, 8, 12, 0, 0, 0, time.UTC) + return &Rollup{ + begin: func(context.Context) (RollupSession, bool, error) { + if beginErr != nil || !locked { + return nil, false, beginErr + } + return s, true, nil + }, + packetRetention: 7 * 24 * time.Hour, + rollupRetention: 90 * 24 * time.Hour, + now: func() time.Time { return now }, + } +} + +func hours(hs ...int) []time.Time { + out := make([]time.Time, len(hs)) + for i, h := range hs { + out[i] = time.Date(2026, 1, 8, h, 0, 0, 0, time.UTC) + } + return out +} + +func TestRollupPass(t *testing.T) { + s := &rollupStub{missing: [][]time.Time{hours(1, 2, 3)}, dirty: hours(0, 4)} + s.failAt = map[time.Time]error{hours(2)[0]: errors.New("boom"), hours(4)[0]: errors.New("stuck")} + rolled, more, err := newTestRollup(s, true, nil).Pass(context.Background()) + if err == nil || !strings.Contains(err.Error(), "boom") { + t.Errorf("err = %v, want the failed hour's error", err) + } + if rolled != 5 || more { + t.Errorf("rolled = %d more = %v, want 5, false", rolled, more) + } + // A failing dirty hour stays queued but isn't retried within the pass. + want := []string{"register", "partial", "missing(48)", "roll 01", "roll 02", "roll 03", "dirty(24)", "roll 00", "roll 04", "dirty(24)"} + if !reflect.DeepEqual(s.calls, want) { + t.Errorf("calls = %v, want %v", s.calls, want) + } + if !s.since.Equal(time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)) { + t.Errorf("registered since %s, want now - packet retention", s.since) + } + if s.closed != 1 { + t.Errorf("session closed %d times", s.closed) + } +} + +func TestRollupRegisterSince(t *testing.T) { + now := time.Date(2026, 1, 8, 12, 20, 0, 0, time.UTC) + for _, tc := range []struct { + name string + packet, rollups time.Duration + want time.Time + }{ + {"packet retention shorter", 7 * 24 * time.Hour, 90 * 24 * time.Hour, time.Date(2026, 1, 1, 13, 0, 0, 0, time.UTC)}, + // Hours cleanup would delete next run must not be registered again. + {"rollup retention shorter", 30 * 24 * time.Hour, 24 * time.Hour, time.Date(2026, 1, 7, 13, 0, 0, 0, time.UTC)}, + } { + t.Run(tc.name, func(t *testing.T) { + r := &Rollup{packetRetention: tc.packet, rollupRetention: tc.rollups, now: func() time.Time { return now }} + got := r.registerSince() + if !got.Equal(tc.want) { + t.Errorf("registerSince = %s, want %s", got, tc.want) + } + if got.Before(now.Add(-tc.rollups)) { + t.Error("registers an hour DeleteOldRollups would remove") + } + }) + } +} + +func TestRollupPassLockHeldElsewhere(t *testing.T) { + s := &rollupStub{} + if rolled, _, err := newTestRollup(s, false, nil).Pass(context.Background()); err != nil || rolled != 0 || len(s.calls) != 0 { + t.Errorf("rolled %d err %v calls %v; want a silent skip", rolled, err, s.calls) + } + failure := errors.New("no connection") + if _, _, err := newTestRollup(s, true, failure).Pass(context.Background()); !errors.Is(err, failure) { + t.Errorf("err = %v, want %v", err, failure) + } +} + +func TestRollupCatchUp(t *testing.T) { + full := make([]int, rollupMissingBudget) + for i := range full { + full[i] = i % 24 + } + s := &rollupStub{missing: [][]time.Time{hours(full...), hours(full...), hours(5)}} + if err := newTestRollup(s, true, nil).CatchUp(context.Background()); err != nil { + t.Fatal(err) + } + passes := 0 + for _, c := range s.calls { + if c == "register" { + passes++ + } + } + if passes != 3 || s.closed != 3 { + t.Errorf("passes %d, closes %d; want 3 (two full batches, then a short one)", passes, s.closed) + } +} + +func TestRollupDrainsDirtyBacklog(t *testing.T) { + var backlog []time.Time + start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + for i := range 200 { + backlog = append(backlog, start.Add(time.Duration(i)*time.Hour)) + } + s := &rollupStub{dirty: backlog} + r := newTestRollup(s, true, nil) + rolled, more, err := r.Pass(context.Background()) + if err != nil || rolled != rollupDirtyBudget || !more { + t.Fatalf("first pass rolled %d more %v err %v; want %d, true", rolled, more, err, rollupDirtyBudget) + } + if err := r.CatchUp(context.Background()); err != nil { + t.Fatal(err) + } + if len(s.done) != len(backlog) { + t.Errorf("drained %d of %d dirty hours", len(s.done), len(backlog)) + } +} + +func TestRollupTaskInterval(t *testing.T) { + task := newTestRollup(&rollupStub{}, true, nil).Task() + if task.Name != "analytics_rollup" || task.Interval != 5*time.Minute { + t.Errorf("task %q every %s", task.Name, task.Interval) + } +} diff --git a/internal/background/tasks.go b/internal/background/tasks.go index 951dca7..3fabc56 100644 --- a/internal/background/tasks.go +++ b/internal/background/tasks.go @@ -5,8 +5,8 @@ package background import ( "context" - "errors" "fmt" + "log/slog" "time" "github.com/MeshCore-Beacon/beacon-server/db" @@ -14,78 +14,111 @@ import ( ) type viewRefresher interface { - RefreshHourlyStats(context.Context) error - RefreshTopNodes(context.Context) error - RefreshTopObservers(context.Context) error - RefreshPayloadBreakdown(context.Context) error - RefreshTopTalkers(context.Context) error - RefreshTopAdvertisers(context.Context) error RefreshRadioPresets(context.Context) error - RefreshObserverActivity(context.Context) error - RefreshSignalStats(context.Context) error - RefreshPathStats(context.Context) error } -// ViewRefreshTask returns a Task that refreshes all materialized views. +// ViewRefreshTask rebuilds the radio preset view; historical stats come from hourly rollups. func ViewRefreshTask(store viewRefresher, interval time.Duration) Task { return Task{ Name: "view_refresh", Interval: interval, Run: func(ctx context.Context) error { - var errs []error - for _, view := range []struct { - name string - refresh func(context.Context) error - }{ - {"hourly stats", store.RefreshHourlyStats}, - {"top nodes", store.RefreshTopNodes}, - {"top observers", store.RefreshTopObservers}, - {"payload breakdown", store.RefreshPayloadBreakdown}, - {"top talkers", store.RefreshTopTalkers}, - {"top advertisers", store.RefreshTopAdvertisers}, - {"radio presets", store.RefreshRadioPresets}, - {"observer activity", store.RefreshObserverActivity}, - {"signal stats", store.RefreshSignalStats}, - {"path stats", store.RefreshPathStats}, - } { - if err := view.refresh(ctx); err != nil { - errs = append(errs, fmt.Errorf("%s: %w", view.name, err)) - } - } - return errors.Join(errs...) - }, - } -} - -// CleanupTask returns a Task that prunes old telemetry, packet, and node rows. -func CleanupTask(store *db.Store, telemetryRetention, packetRetention, nodeDeleteAfter, interval time.Duration) Task { - return Task{ - Name: "cleanup", - Interval: interval, - Run: func(ctx context.Context) error { - if err := store.DeleteOldTelemetry(ctx, time.Now().Add(-telemetryRetention)); err != nil { - return err - } - // One cutoff for all three so the IATA tables stay in step - // with the packets they mirror. - cutoff := time.Now().Add(-packetRetention) - if err := store.DeleteOldPackets(ctx, cutoff); err != nil { - return err - } - if err := store.DeleteOldChannelIATAs(ctx, cutoff); err != nil { - return err - } - if err := store.DeleteOldTraceIATAs(ctx, cutoff); err != nil { - return err - } - if err := store.DeleteOldNodes(ctx, time.Now().Add(-nodeDeleteAfter)); err != nil { - return err + if err := store.RefreshRadioPresets(ctx); err != nil { + return fmt.Errorf("radio presets: %w", err) } return nil }, } } +type cleanupStore interface { + DeleteOldTelemetry(ctx context.Context, cutoff time.Time) error + OldestMissingRollupHour(ctx context.Context) (time.Time, bool, error) + DeleteOldPackets(ctx context.Context, cutoff time.Time) error + DeleteOldChannelIATAs(ctx context.Context, cutoff time.Time) error + DeleteOldTraceIATAs(ctx context.Context, cutoff time.Time) error + DeleteOldTraceTags(ctx context.Context, cutoff time.Time) error + DeleteOldRollups(ctx context.Context, cutoff time.Time) error + DeleteOldNodes(ctx context.Context, cutoff time.Time) error +} + +// CleanupConfig holds the retention windows CleanupTask enforces. +type CleanupConfig struct { + TelemetryRetention time.Duration + PacketRetention time.Duration + RollupRetention time.Duration + NodeDeleteAfter time.Duration + Interval time.Duration +} + +// Raw rows wait for unrolled hours, but not forever: past the cap those hours become partial. +const ( + rawHoldbackMargin = 35 * time.Minute // ingest clamps heard_at to ±30 min of last_heard_at + rawHoldbackWarn = 6 * time.Hour + rawHoldbackCap = 24 * time.Hour + traceTagLag = 30 * time.Minute +) + +// CleanupTask returns a Task that prunes old telemetry, packets, rollups and nodes. +func CleanupTask(store cleanupStore, cfg CleanupConfig) Task { + return cleanupTask(store, cfg, time.Now) +} + +func cleanupTask(store cleanupStore, cfg CleanupConfig, now func() time.Time) Task { + return Task{ + Name: "cleanup", + Interval: cfg.Interval, + Run: func(ctx context.Context) error { + t := now() + if err := store.DeleteOldTelemetry(ctx, t.Add(-cfg.TelemetryRetention)); err != nil { + return err + } + cutoff, err := rawCutoff(ctx, store, t.Add(-cfg.PacketRetention)) + if err != nil { + return err + } + // One cutoff so the IATA and trace tables stay in step with the packets they mirror. + for _, del := range []func(context.Context, time.Time) error{ + store.DeleteOldPackets, store.DeleteOldChannelIATAs, store.DeleteOldTraceIATAs, + } { + if err := del(ctx, cutoff); err != nil { + return err + } + } + // Tag times are hearing times, up to the clamp behind their packets' last_heard_at; + // trailing by it keeps a re-heard packet's tag from being deleted out from under it. + if err := store.DeleteOldTraceTags(ctx, cutoff.Add(-traceTagLag)); err != nil { + return err + } + if err := store.DeleteOldRollups(ctx, t.Add(-cfg.RollupRetention)); err != nil { + return err + } + return store.DeleteOldNodes(ctx, t.Add(-cfg.NodeDeleteAfter)) + }, + } +} + +// rawCutoff holds packet deletion behind the oldest unrolled hour, up to rawHoldbackCap. +func rawCutoff(ctx context.Context, store cleanupStore, retentionCutoff time.Time) (time.Time, error) { + oldest, ok, err := store.OldestMissingRollupHour(ctx) + if err != nil || !ok { + return retentionCutoff, err + } + held := oldest.Add(-rawHoldbackMargin) + if !held.Before(retentionCutoff) { + return retentionCutoff, nil + } + holdback := retentionCutoff.Sub(held) + if holdback > rawHoldbackCap { + held, holdback = retentionCutoff.Add(-rawHoldbackCap), rawHoldbackCap + } + if holdback > rawHoldbackWarn { + slog.Warn("packet cleanup held back by unrolled analytics hours", "component", "background", + "holdback", holdback.Round(time.Minute), "oldest_missing_hour", oldest.UTC()) + } + return held, nil +} + // Limit lock lifetime while retaining the per-run work budget. const ( reconfirmRunLimit = 750_000 diff --git a/internal/background/tasks_test.go b/internal/background/tasks_test.go index 483f858..7c668de 100644 --- a/internal/background/tasks_test.go +++ b/internal/background/tasks_test.go @@ -10,6 +10,7 @@ import ( "errors" "log" "log/slog" + "reflect" "strings" "testing" "testing/synctest" @@ -96,65 +97,30 @@ func (s *refreshStub) refresh(ctx context.Context, name string) error { } return s.errs[name] } -func (s *refreshStub) RefreshHourlyStats(ctx context.Context) error { - return s.refresh(ctx, "hourly stats") -} -func (s *refreshStub) RefreshTopNodes(ctx context.Context) error { return s.refresh(ctx, "top nodes") } -func (s *refreshStub) RefreshTopObservers(ctx context.Context) error { - return s.refresh(ctx, "top observers") -} -func (s *refreshStub) RefreshPayloadBreakdown(ctx context.Context) error { - return s.refresh(ctx, "payload breakdown") -} -func (s *refreshStub) RefreshTopTalkers(ctx context.Context) error { - return s.refresh(ctx, "top talkers") -} -func (s *refreshStub) RefreshTopAdvertisers(ctx context.Context) error { - return s.refresh(ctx, "top advertisers") -} func (s *refreshStub) RefreshRadioPresets(ctx context.Context) error { return s.refresh(ctx, "radio presets") } -func (s *refreshStub) RefreshObserverActivity(ctx context.Context) error { - return s.refresh(ctx, "observer activity") -} - -func (s *refreshStub) RefreshSignalStats(ctx context.Context) error { - return s.refresh(ctx, "signal stats") -} - -func (s *refreshStub) RefreshPathStats(ctx context.Context) error { - return s.refresh(ctx, "path stats") -} func TestViewRefreshTask(t *testing.T) { - first, second := errors.New("first failure"), errors.New("second failure") + failure := errors.New("refresh failed") for _, tc := range []struct { name string errs map[string]error }{ {"success", nil}, - {"partial failure", map[string]error{"hourly stats": first, "top talkers": second}}, - {"signal failure", map[string]error{"signal stats": first}}, - {"path failure", map[string]error{"path stats": second}}, + {"failure", map[string]error{"radio presets": failure}}, } { t.Run(tc.name, func(t *testing.T) { store := &refreshStub{errs: tc.errs} - task := ViewRefreshTask(store, time.Minute) - err := task.Run(context.Background()) - if len(store.calls) != 10 { - t.Fatalf("refreshed %d views, want 10", len(store.calls)) + err := ViewRefreshTask(store, time.Minute).Run(context.Background()) + if len(store.calls) != 1 { + t.Fatalf("refreshed %d views, want 1", len(store.calls)) } if len(tc.errs) == 0 && err != nil { t.Fatal(err) } - for name, cause := range tc.errs { - if !errors.Is(err, cause) { - t.Errorf("missing cause %v in %v", cause, err) - } - if err == nil || !strings.Contains(err.Error(), name) { - t.Errorf("missing view name %q in %v", name, err) - } + if len(tc.errs) > 0 && (!errors.Is(err, failure) || !strings.Contains(err.Error(), "radio presets")) { + t.Errorf("err = %v", err) } }) } @@ -201,3 +167,82 @@ func TestSchedulerFailureIsNotComplete(t *testing.T) { } }) } + +type cleanupStub struct { + calls []string + cutoffs map[string]time.Time + oldest time.Time // zero = no missing hour +} + +func (s *cleanupStub) record(name string, cutoff time.Time) error { + s.calls = append(s.calls, name) + if s.cutoffs == nil { + s.cutoffs = map[string]time.Time{} + } + s.cutoffs[name] = cutoff + return nil +} +func (s *cleanupStub) DeleteOldTelemetry(_ context.Context, c time.Time) error { + return s.record("telemetry", c) +} +func (s *cleanupStub) OldestMissingRollupHour(context.Context) (time.Time, bool, error) { + s.calls = append(s.calls, "oldest") + return s.oldest, !s.oldest.IsZero(), nil +} +func (s *cleanupStub) DeleteOldPackets(_ context.Context, c time.Time) error { + return s.record("packets", c) +} +func (s *cleanupStub) DeleteOldChannelIATAs(_ context.Context, c time.Time) error { + return s.record("channel iatas", c) +} +func (s *cleanupStub) DeleteOldTraceIATAs(_ context.Context, c time.Time) error { + return s.record("trace iatas", c) +} +func (s *cleanupStub) DeleteOldTraceTags(_ context.Context, c time.Time) error { + return s.record("trace tags", c) +} +func (s *cleanupStub) DeleteOldRollups(_ context.Context, c time.Time) error { + return s.record("rollups", c) +} +func (s *cleanupStub) DeleteOldNodes(_ context.Context, c time.Time) error { + return s.record("nodes", c) +} + +func TestCleanupTask(t *testing.T) { + now := time.Date(2026, 1, 10, 12, 0, 0, 0, time.UTC) + cfg := CleanupConfig{TelemetryRetention: 31 * 24 * time.Hour, PacketRetention: 7 * 24 * time.Hour, + RollupRetention: 90 * 24 * time.Hour, NodeDeleteAfter: 30 * 24 * time.Hour, Interval: time.Hour} + retention := now.Add(-cfg.PacketRetention) + for _, tc := range []struct { + name string + oldest time.Time + want time.Time + }{ + {"no unrolled hours", time.Time{}, retention}, + {"unrolled hour after the cutoff", retention.Add(time.Hour), retention}, + {"held behind an unrolled hour", retention.Add(-2 * time.Hour), retention.Add(-2*time.Hour - 35*time.Minute)}, + {"holdback capped", retention.Add(-72 * time.Hour), retention.Add(-24 * time.Hour)}, + } { + t.Run(tc.name, func(t *testing.T) { + s := &cleanupStub{oldest: tc.oldest} + if err := cleanupTask(s, cfg, func() time.Time { return now }).Run(context.Background()); err != nil { + t.Fatal(err) + } + want := []string{"telemetry", "oldest", "packets", "channel iatas", "trace iatas", "trace tags", "rollups", "nodes"} + if !reflect.DeepEqual(s.calls, want) { + t.Errorf("calls %v, want %v", s.calls, want) + } + for _, name := range []string{"packets", "channel iatas", "trace iatas"} { + if !s.cutoffs[name].Equal(tc.want) { + t.Errorf("%s cutoff %s, want %s", name, s.cutoffs[name], tc.want) + } + } + if !s.cutoffs["trace tags"].Equal(tc.want.Add(-30 * time.Minute)) { + t.Errorf("trace tags cutoff %s, want 30 min behind packets", s.cutoffs["trace tags"]) + } + if !s.cutoffs["rollups"].Equal(now.Add(-cfg.RollupRetention)) || !s.cutoffs["telemetry"].Equal(now.Add(-cfg.TelemetryRetention)) { + t.Errorf("rollup/telemetry cutoffs %v", s.cutoffs) + } + }) + } +} diff --git a/internal/cache/cache.go b/internal/cache/cache.go index 48fb569..90cf83f 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -16,11 +16,13 @@ import ( "github.com/MeshCore-Beacon/beacon-server/internal/config" redis "github.com/redis/go-redis/v9" + "golang.org/x/sync/singleflight" ) // Client wraps a Redis client with helper methods used by CachedReader. type Client struct { rdb *redis.Client + sf singleflight.Group // one fetch per key across concurrent misses } // NewClient creates a new Redis client from the given address, password, and @@ -101,42 +103,62 @@ func (c *Client) delPrefix(ctx context.Context, prefix string) { // fails to unmarshal (corrupt or schema-changed), the entry is overwritten // with a fresh fetch. Errors from Set are ignored so a Redis hiccup never // fails a request. -func getOrSet[T any](ctx context.Context, c *Client, key string, ttl time.Duration, fetch func() (T, error)) (T, error) { +func getOrSet[T any](ctx context.Context, c *Client, key string, ttl time.Duration, fetch func(context.Context) (T, error)) (T, error) { raw, err := c.rdb.Get(ctx, key).Bytes() if err != nil && !errors.Is(err, redis.Nil) { // real Redis error, degrade gracefully slog.DebugContext(ctx, "cache bypass", "component", "cache", "reason", "read_error") - return fetch() + return fetch(ctx) } var zero, out T - if errors.Is(err, redis.Nil) { - // cache miss — fetch, store, return - slog.DebugContext(ctx, "cache miss", "component", "cache") - val, err := fetch() - if err != nil { - return zero, err + if err == nil { + if err = json.Unmarshal(raw, &out); err == nil { + slog.DebugContext(ctx, "cache hit", "component", "cache") + return out, nil } - data, jsonErr := json.Marshal(val) - if jsonErr != nil { - return val, nil - } - _ = c.rdb.Set(ctx, key, data, ttl) - return val, nil - } - if err = json.Unmarshal(raw, &out); err != nil { - // corrupt cache entry — overwrite it slog.DebugContext(ctx, "cache invalid entry", "component", "cache") - val, err := fetch() - if err != nil { - return zero, err - } - data, jsonErr := json.Marshal(val) - if jsonErr != nil { - return val, nil - } - _ = c.rdb.Set(ctx, key, data, ttl) - return val, nil + } else { + slog.DebugContext(ctx, "cache miss", "component", "cache") + } + raw, err = fetchShared(ctx, c, key, ttl, func(ctx context.Context) (any, error) { return fetch(ctx) }) + if err != nil { + return zero, err + } + // Each caller decodes its own copy: handlers may decorate the value they get. + if err := json.Unmarshal(raw, &out); err != nil { + return zero, err } - slog.DebugContext(ctx, "cache hit", "component", "cache") return out, nil } + +// sharedFetchTimeout bounds a fetch that no single request owns. +const sharedFetchTimeout = 30 * time.Second + +// fetchShared runs one fetch per key across concurrent misses and returns its JSON. The fetch +// gets its own bounded context so one caller disconnecting can't fail the others, and each +// caller still stops waiting when its own context ends. +func fetchShared(ctx context.Context, c *Client, key string, ttl time.Duration, fetch func(context.Context) (any, error)) ([]byte, error) { + ch := c.sf.DoChan(key, func() (any, error) { + fctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), sharedFetchTimeout) + defer cancel() + val, err := fetch(fctx) + if err != nil { + return nil, err + } + data, err := json.Marshal(val) + if err != nil { + return nil, err + } + _ = c.rdb.Set(fctx, key, data, ttl) + return data, nil + }) + select { + case r := <-ch: + if r.Err != nil { + return nil, r.Err + } + return r.Val.([]byte), nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go index 467cfcf..2f912f0 100644 --- a/internal/cache/cache_test.go +++ b/internal/cache/cache_test.go @@ -31,6 +31,10 @@ type stubReader struct { calls int } +func (s *stubReader) AnalyticsRevision(context.Context) (int64, error) { return 0, nil } +func (s *stubReader) GetStatsSeries(context.Context, time.Time, time.Time, []string) (*api.StatsSeries, error) { + return &api.StatsSeries{}, nil +} func (s *stubReader) GetSignalStats(context.Context, time.Time, time.Time, []string) (*api.SignalStats, error) { return nil, s.err } @@ -73,7 +77,7 @@ func (s *stubReader) GetRegionBySlug(_ context.Context, _ string) (*api.Region, return nil, nil } func (s *stubReader) GetScopeNames(_ context.Context) ([]string, error) { return nil, nil } -func (s *stubReader) GetScopeStats(_ context.Context, _ []string) ([]api.ScopeStats, error) { +func (s *stubReader) GetScopeStats(_ context.Context, _ []string, _ time.Time) ([]api.ScopeStats, error) { return nil, nil } func (s *stubReader) GetScopeByName(_ context.Context, _ string) (*api.ScopeDetail, error) { @@ -92,7 +96,7 @@ func (s *stubReader) GetStatsPayloadBreakdown(_ context.Context, _ []string, _ t return nil, nil } -func (s *stubReader) GetStatsTopNodes(_ context.Context, _ []string, _ int32) ([]api.TopNode, error) { +func (s *stubReader) GetStatsTopNodes(_ context.Context, _ []string, _ time.Time, _ int32) ([]api.TopNode, error) { return nil, nil } @@ -258,7 +262,7 @@ func TestGetOrSet_CacheMiss_FetchesAndStores(t *testing.T) { c, _ := newTestClient(t) stub := &stubReader{iatas: []api.IATA{{IATA: "YVR"}}} - result, err := getOrSet(context.Background(), c, "test:key", time.Minute, func() ([]api.IATA, error) { + result, err := getOrSet(context.Background(), c, "test:key", time.Minute, func(ctx context.Context) ([]api.IATA, error) { return stub.ListIATAs(context.Background()) }) if err != nil { @@ -277,11 +281,11 @@ func TestGetOrSet_CacheHit_DoesNotFetch(t *testing.T) { stub := &stubReader{iatas: []api.IATA{{IATA: "YVR"}}} // prime the cache - _, _ = getOrSet(context.Background(), c, "test:key", time.Minute, func() ([]api.IATA, error) { + _, _ = getOrSet(context.Background(), c, "test:key", time.Minute, func(ctx context.Context) ([]api.IATA, error) { return stub.ListIATAs(context.Background()) }) // second call should hit cache - result, err := getOrSet(context.Background(), c, "test:key", time.Minute, func() ([]api.IATA, error) { + result, err := getOrSet(context.Background(), c, "test:key", time.Minute, func(ctx context.Context) ([]api.IATA, error) { return stub.ListIATAs(context.Background()) }) if err != nil { @@ -300,7 +304,7 @@ func TestGetOrSet_RedisError_DegradeGracefully(t *testing.T) { mr.Close() // kill Redis stub := &stubReader{iatas: []api.IATA{{IATA: "YVR"}}} - result, err := getOrSet(context.Background(), c, "test:key", time.Minute, func() ([]api.IATA, error) { + result, err := getOrSet(context.Background(), c, "test:key", time.Minute, func(ctx context.Context) ([]api.IATA, error) { return stub.ListIATAs(context.Background()) }) if err != nil { @@ -316,7 +320,7 @@ func TestGetOrSet_CorruptEntry_Overwrites(t *testing.T) { mr.Set("test:key", "not-valid-json") stub := &stubReader{iatas: []api.IATA{{IATA: "YVR"}}} - result, err := getOrSet(context.Background(), c, "test:key", time.Minute, func() ([]api.IATA, error) { + result, err := getOrSet(context.Background(), c, "test:key", time.Minute, func(ctx context.Context) ([]api.IATA, error) { return stub.ListIATAs(context.Background()) }) if err != nil { @@ -334,7 +338,7 @@ func TestGetOrSet_FetchError_Propagates(t *testing.T) { c, _ := newTestClient(t) stub := &stubReader{err: errors.New("db error")} - _, err := getOrSet(context.Background(), c, "test:key", time.Minute, func() ([]api.IATA, error) { + _, err := getOrSet(context.Background(), c, "test:key", time.Minute, func(ctx context.Context) ([]api.IATA, error) { return stub.ListIATAs(context.Background()) }) if err == nil { @@ -357,14 +361,14 @@ func TestCachedReader_IATASortingForStableKey(t *testing.T) { } // call with unsorted IATAs - getOrSet(context.Background(), c, "beacon:stats:overview:YVR,YYJ", time.Minute, func() (*api.StatsOverview, error) { + getOrSet(context.Background(), c, "beacon:stats:overview:YVR,YYJ", time.Minute, func(ctx context.Context) (*api.StatsOverview, error) { calls++ return &api.StatsOverview{TotalPackets: 42}, nil }) // call CachedReader with reversed order — should hit same key _ = cr - result, err := getOrSet(context.Background(), c, "beacon:stats:overview:YVR,YYJ", time.Minute, func() (*api.StatsOverview, error) { + result, err := getOrSet(context.Background(), c, "beacon:stats:overview:YVR,YYJ", time.Minute, func(ctx context.Context) (*api.StatsOverview, error) { calls++ return &api.StatsOverview{TotalPackets: 42}, nil }) diff --git a/internal/cache/logging_test.go b/internal/cache/logging_test.go index d16eed5..94a45c6 100644 --- a/internal/cache/logging_test.go +++ b/internal/cache/logging_test.go @@ -22,7 +22,7 @@ func TestCacheDiagnostics(t *testing.T) { c, mr := newTestClient(t) key, value := "private-cache-key", "private-cache-value" calls := 0 - fetch := func() (string, error) { calls++; return value, nil } + fetch := func(context.Context) (string, error) { calls++; return value, nil } for i := 0; i < 2; i++ { got, err := getOrSet(context.Background(), c, key, time.Minute, fetch) if err != nil || got != value { diff --git a/internal/cache/paths.go b/internal/cache/paths.go deleted file mode 100644 index f5951cb..0000000 --- a/internal/cache/paths.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2026 Beacon Contributors -// SPDX-License-Identifier: AGPL-3.0-or-later - -package cache - -import ( - "context" - "encoding/json" - "fmt" - "slices" - "time" - - "github.com/MeshCore-Beacon/beacon-server/internal/api" -) - -func (cr *CachedReader) GetPathStats(ctx context.Context, since, until time.Time, iatas []string) (*api.PathStats, error) { - sorted := append([]string{}, iatas...) - slices.Sort(sorted) - sorted = slices.Compact(sorted) - segment, _ := json.Marshal(sorted) // [] and [""] must remain distinct (global vs empty region). - key := fmt.Sprintf("%s%s:%d:%d", keyPathStatsPrefix, segment, since.UnixMilli(), until.UnixMilli()) - return getOrSet(ctx, cr.c, key, cr.ttl.Stats, func() (*api.PathStats, error) { - return cr.inner.GetPathStats(ctx, since, until, iatas) - }) -} diff --git a/internal/cache/paths_test.go b/internal/cache/paths_test.go index fdffe97..2c55bf5 100644 --- a/internal/cache/paths_test.go +++ b/internal/cache/paths_test.go @@ -17,6 +17,8 @@ type pathStatsReader struct { calls int64 } +func (r *pathStatsReader) AnalyticsRevision(context.Context) (int64, error) { return 0, nil } + func (r *pathStatsReader) GetPathStats(context.Context, time.Time, time.Time, []string) (*api.PathStats, error) { r.calls++ return &api.PathStats{Receptions: r.calls}, nil diff --git a/internal/cache/reader.go b/internal/cache/reader.go index e845e62..cac83ed 100644 --- a/internal/cache/reader.go +++ b/internal/cache/reader.go @@ -7,7 +7,6 @@ import ( "context" "encoding/json" "fmt" - "slices" "sort" "strings" "time" @@ -29,6 +28,7 @@ const ( keyStatsOverviewPrefix = "beacon:stats:overview:" keyStatsObservationsPrefix = "beacon:stats:observations:" keySignalStatsPrefix = "beacon:stats:signal:" + keyStatsSeriesPrefix = "beacon:stats:series:" keyPathStatsPrefix = "beacon:stats:paths:" keyStatsBreakdownPrefix = "beacon:stats:breakdown:" keyStatsTopNodesPrefix = "beacon:stats:top-nodes:" @@ -56,6 +56,7 @@ type CachedReader struct { inner api.Reader c *Client ttl CacheTTLs + rev revisionMemo } // CacheTTLs holds the resolved per-category TTLs for the cache layer. @@ -83,6 +84,7 @@ func NewCachedReader(inner api.Reader, c *Client, ttl CacheTTLs) api.Reader { func (cr *CachedReader) InvalidateScopeNames(ctx context.Context) { cr.c.del(ctx, keyScopeNames) cr.c.delPrefix(ctx, keyScopeByNamePrefix) + cr.c.delPrefix(ctx, keyScopeStats) } // InvalidateIATABorder makes an imported or removed MeshMapper boundary visible. @@ -118,49 +120,49 @@ func (cr *CachedReader) InvalidateObserver(ctx context.Context, observerID uuid. // ListIATAs implements [api.Reader]. func (cr *CachedReader) ListIATAs(ctx context.Context) ([]api.IATA, error) { - return getOrSet(ctx, cr.c, keyIATAs, cr.ttl.Reference, func() ([]api.IATA, error) { + return getOrSet(ctx, cr.c, keyIATAs, cr.ttl.Reference, func(ctx context.Context) ([]api.IATA, error) { return cr.inner.ListIATAs(ctx) }) } // GetIATA implements [api.Reader]. func (cr *CachedReader) GetIATA(ctx context.Context, iata string) (*api.IATA, error) { - return getOrSet(ctx, cr.c, keyIATAPrefix+iata, cr.ttl.Reference, func() (*api.IATA, error) { + return getOrSet(ctx, cr.c, keyIATAPrefix+iata, cr.ttl.Reference, func(ctx context.Context) (*api.IATA, error) { return cr.inner.GetIATA(ctx, iata) }) } // GetIATABorder implements [api.Reader]. func (cr *CachedReader) GetIATABorder(ctx context.Context, iata string) (json.RawMessage, error) { - return getOrSet(ctx, cr.c, keyIATABorderPrefix+iata, cr.ttl.Reference, func() (json.RawMessage, error) { + return getOrSet(ctx, cr.c, keyIATABorderPrefix+iata, cr.ttl.Reference, func(ctx context.Context) (json.RawMessage, error) { return cr.inner.GetIATABorder(ctx, iata) }) } // ListRegions implements [api.Reader]. func (cr *CachedReader) ListRegions(ctx context.Context) ([]api.RegionSummary, error) { - return getOrSet(ctx, cr.c, keyRegions, cr.ttl.Reference, func() ([]api.RegionSummary, error) { + return getOrSet(ctx, cr.c, keyRegions, cr.ttl.Reference, func(ctx context.Context) ([]api.RegionSummary, error) { return cr.inner.ListRegions(ctx) }) } // GetRegion implements [api.Reader]. func (cr *CachedReader) GetRegion(ctx context.Context, regionID int32) (*api.Region, error) { - return getOrSet(ctx, cr.c, fmt.Sprintf("%s%d", keyRegionPrefix, regionID), cr.ttl.Reference, func() (*api.Region, error) { + return getOrSet(ctx, cr.c, fmt.Sprintf("%s%d", keyRegionPrefix, regionID), cr.ttl.Reference, func(ctx context.Context) (*api.Region, error) { return cr.inner.GetRegion(ctx, regionID) }) } // GetRegionBySlug implements [api.Reader]. func (cr *CachedReader) GetRegionBySlug(ctx context.Context, slug string) (*api.Region, error) { - return getOrSet(ctx, cr.c, keyRegionSlugPrefix+slug, cr.ttl.Reference, func() (*api.Region, error) { + return getOrSet(ctx, cr.c, keyRegionSlugPrefix+slug, cr.ttl.Reference, func(ctx context.Context) (*api.Region, error) { return cr.inner.GetRegionBySlug(ctx, slug) }) } // GetScopeNames implements [api.Reader]. func (cr *CachedReader) GetScopeNames(ctx context.Context) ([]string, error) { - return getOrSet(ctx, cr.c, keyScopeNames, cr.ttl.Reference, func() ([]string, error) { + return getOrSet(ctx, cr.c, keyScopeNames, cr.ttl.Reference, func(ctx context.Context) ([]string, error) { return cr.inner.GetScopeNames(ctx) }) } @@ -171,82 +173,6 @@ func (cr *CachedReader) GetObserverComparison(ctx context.Context, a, b uuid.UUI return cr.inner.GetObserverComparison(ctx, a, b, since, until, iatas) } -// GetScopeStats implements [api.Reader]. -func (cr *CachedReader) GetScopeStats(ctx context.Context, iatas []string) ([]api.ScopeStats, error) { - segment := "all" - if len(iatas) > 0 { - sorted := append([]string(nil), iatas...) - sort.Strings(sorted) - segment = strings.Join(slices.Compact(sorted), ",") - } - return getOrSet(ctx, cr.c, keyScopeStats+":"+segment, cr.ttl.Reference, func() ([]api.ScopeStats, error) { - return cr.inner.GetScopeStats(ctx, iatas) - }) -} - -// GetScopeByName implements [api.Reader]. -func (cr *CachedReader) GetScopeByName(ctx context.Context, name string) (*api.ScopeDetail, error) { - return getOrSet(ctx, cr.c, keyScopeByNamePrefix+name, cr.ttl.Reference, func() (*api.ScopeDetail, error) { - return cr.inner.GetScopeByName(ctx, name) - }) -} - -// GetStatsOverview implements [api.Reader]. -func (cr *CachedReader) GetStatsOverview(ctx context.Context, iatas []string) (*api.StatsOverview, error) { - segment := "all" - if len(iatas) > 0 { - sorted := append([]string(nil), iatas...) - sort.Strings(sorted) - segment = strings.Join(sorted, ",") - } - key := fmt.Sprintf("%s%s", keyStatsOverviewPrefix, segment) - return getOrSet(ctx, cr.c, key, cr.ttl.Stats, func() (*api.StatsOverview, error) { - return cr.inner.GetStatsOverview(ctx, iatas) - }) -} - -// GetStatsObservations implements [api.Reader]. -func (cr *CachedReader) GetStatsObservations(ctx context.Context, iatas []string, since time.Time) ([]api.ObservationPoint, error) { - segment := "all" - if len(iatas) > 0 { - sorted := append([]string(nil), iatas...) - sort.Strings(sorted) - segment = strings.Join(sorted, ",") - } - key := fmt.Sprintf("%s%s:%d", keyStatsObservationsPrefix, segment, since.UnixMilli()) - return getOrSet(ctx, cr.c, key, cr.ttl.Stats, func() ([]api.ObservationPoint, error) { - return cr.inner.GetStatsObservations(ctx, iatas, since) - }) -} - -// GetStatsPayloadBreakdown implements [api.Reader]. -func (cr *CachedReader) GetStatsPayloadBreakdown(ctx context.Context, iatas []string, since time.Time) ([]api.PayloadBreakdownItem, error) { - segment := "all" - if len(iatas) > 0 { - sorted := append([]string(nil), iatas...) - sort.Strings(sorted) - segment = strings.Join(sorted, ",") - } - key := fmt.Sprintf("%s%s:%d", keyStatsBreakdownPrefix, segment, since.UnixMilli()) - return getOrSet(ctx, cr.c, key, cr.ttl.Stats, func() ([]api.PayloadBreakdownItem, error) { - return cr.inner.GetStatsPayloadBreakdown(ctx, iatas, since) - }) -} - -// GetStatsTopNodes implements [api.Reader]. -func (cr *CachedReader) GetStatsTopNodes(ctx context.Context, iatas []string, limit int32) ([]api.TopNode, error) { - segment := "all" - if len(iatas) > 0 { - sorted := append([]string(nil), iatas...) - sort.Strings(sorted) - segment = strings.Join(sorted, ",") - } - key := fmt.Sprintf("%s%s:%d", keyStatsTopNodesPrefix, segment, limit) - return getOrSet(ctx, cr.c, key, cr.ttl.Stats, func() ([]api.TopNode, error) { - return cr.inner.GetStatsTopNodes(ctx, iatas, limit) - }) -} - // GetStatsNodeTypes implements [api.Reader]. func (cr *CachedReader) GetStatsNodeTypes(ctx context.Context, iatas []string) ([]api.NodeTypeCount, error) { segment := "all" @@ -256,39 +182,11 @@ func (cr *CachedReader) GetStatsNodeTypes(ctx context.Context, iatas []string) ( segment = strings.Join(sorted, ",") } key := fmt.Sprintf("%s%s", keyStatsNodeTypes, segment) - return getOrSet(ctx, cr.c, key, cr.ttl.Stats, func() ([]api.NodeTypeCount, error) { + return getOrSet(ctx, cr.c, key, cr.ttl.Stats, func(ctx context.Context) ([]api.NodeTypeCount, error) { return cr.inner.GetStatsNodeTypes(ctx, iatas) }) } -// GetStatsTopObservers implements [api.Reader]. -func (cr *CachedReader) GetStatsTopObservers(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopObserver, error) { - segment := "all" - if len(iatas) > 0 { - sorted := append([]string(nil), iatas...) - sort.Strings(sorted) - segment = strings.Join(sorted, ",") - } - key := fmt.Sprintf("%s%s:%d:%d", keyStatsTopObsPrefix, segment, since.UnixMilli(), limit) - return getOrSet(ctx, cr.c, key, cr.ttl.Stats, func() ([]api.TopObserver, error) { - return cr.inner.GetStatsTopObservers(ctx, iatas, since, limit) - }) -} - -// GetStatsTopAdvertisers implements [api.Reader]. -func (cr *CachedReader) GetStatsTopAdvertisers(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopAdvertiser, error) { - segment := "all" - if len(iatas) > 0 { - sorted := append([]string(nil), iatas...) - sort.Strings(sorted) - segment = strings.Join(sorted, ",") - } - key := fmt.Sprintf("%s%s:%d:%d", keyStatsTopAdvPrefix, segment, since.UnixMilli(), limit) - return getOrSet(ctx, cr.c, key, cr.ttl.Stats, func() ([]api.TopAdvertiser, error) { - return cr.inner.GetStatsTopAdvertisers(ctx, iatas, since, limit) - }) -} - // GetStatsClockDrift implements [api.Reader]. func (cr *CachedReader) GetStatsClockDrift(ctx context.Context, iatas []string, limit int32) ([]api.ClockDriftEntry, error) { segment := "all" @@ -298,25 +196,11 @@ func (cr *CachedReader) GetStatsClockDrift(ctx context.Context, iatas []string, segment = strings.Join(sorted, ",") } key := fmt.Sprintf("%s%s:%d", keyStatsClockDriftPrefix, segment, limit) - return getOrSet(ctx, cr.c, key, cr.ttl.Stats, func() ([]api.ClockDriftEntry, error) { + return getOrSet(ctx, cr.c, key, cr.ttl.Stats, func(ctx context.Context) ([]api.ClockDriftEntry, error) { return cr.inner.GetStatsClockDrift(ctx, iatas, limit) }) } -// GetStatsTopTalkers implements [api.Reader]. -func (cr *CachedReader) GetStatsTopTalkers(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopTalker, error) { - segment := "all" - if len(iatas) > 0 { - sorted := append([]string(nil), iatas...) - sort.Strings(sorted) - segment = strings.Join(sorted, ",") - } - key := fmt.Sprintf("%s%s:%d:%d", keyStatsTopTalkersPrefix, segment, since.UnixMilli(), limit) - return getOrSet(ctx, cr.c, key, cr.ttl.Stats, func() ([]api.TopTalker, error) { - return cr.inner.GetStatsTopTalkers(ctx, iatas, since, limit) - }) -} - // GetRadioPresets implements [api.Reader]. func (cr *CachedReader) GetRadioPresets(ctx context.Context, preset string, iatas []string) ([]api.RadioPreset, error) { segment := "all" @@ -326,21 +210,21 @@ func (cr *CachedReader) GetRadioPresets(ctx context.Context, preset string, iata segment = strings.Join(sorted, ",") } key := fmt.Sprintf("%s%s:%s", keyRadioPresetsPrefix, preset, segment) - return getOrSet(ctx, cr.c, key, cr.ttl.Stats, func() ([]api.RadioPreset, error) { + return getOrSet(ctx, cr.c, key, cr.ttl.Stats, func(ctx context.Context) ([]api.RadioPreset, error) { return cr.inner.GetRadioPresets(ctx, preset, iatas) }) } // GetNode implements [api.Reader]. func (cr *CachedReader) GetNode(ctx context.Context, nodeID uuid.UUID) (*api.Node, error) { - return getOrSet(ctx, cr.c, keyNodePrefix+nodeID.String(), cr.ttl.Nodes, func() (*api.Node, error) { + return getOrSet(ctx, cr.c, keyNodePrefix+nodeID.String(), cr.ttl.Nodes, func(ctx context.Context) (*api.Node, error) { return cr.inner.GetNode(ctx, nodeID) }) } // GetNodeNeighbors implements [api.Reader]. func (cr *CachedReader) GetNodeNeighbors(ctx context.Context, nodeID uuid.UUID) ([]api.NodeNeighbor, error) { - return getOrSet(ctx, cr.c, keyNodeNeighborsPrefix+nodeID.String(), cr.ttl.Nodes, func() ([]api.NodeNeighbor, error) { + return getOrSet(ctx, cr.c, keyNodeNeighborsPrefix+nodeID.String(), cr.ttl.Nodes, func(ctx context.Context) ([]api.NodeNeighbor, error) { return cr.inner.GetNodeNeighbors(ctx, nodeID) }) } @@ -353,21 +237,21 @@ func (cr *CachedReader) GetNodesByIDs(ctx context.Context, ids []uuid.UUID) (map } sort.Strings(strs) key := keyNodesByIDsPrefix + strings.Join(strs, ",") - return getOrSet(ctx, cr.c, key, cr.ttl.Nodes, func() (map[uuid.UUID]*api.ResolvedNode, error) { + return getOrSet(ctx, cr.c, key, cr.ttl.Nodes, func(ctx context.Context) (map[uuid.UUID]*api.ResolvedNode, error) { return cr.inner.GetNodesByIDs(ctx, ids) }) } // GetObserver implements [api.Reader]. func (cr *CachedReader) GetObserver(ctx context.Context, observerID uuid.UUID) (*api.Observer, error) { - return getOrSet(ctx, cr.c, keyObserverPrefix+observerID.String(), cr.ttl.Observers, func() (*api.Observer, error) { + return getOrSet(ctx, cr.c, keyObserverPrefix+observerID.String(), cr.ttl.Observers, func(ctx context.Context) (*api.Observer, error) { return cr.inner.GetObserver(ctx, observerID) }) } // GetObserverScopes implements [api.Reader]. func (cr *CachedReader) GetObserverScopes(ctx context.Context, observerID uuid.UUID) ([]string, error) { - return getOrSet(ctx, cr.c, keyObserverScopesPrefix+observerID.String(), cr.ttl.Observers, func() ([]string, error) { + return getOrSet(ctx, cr.c, keyObserverScopesPrefix+observerID.String(), cr.ttl.Observers, func(ctx context.Context) ([]string, error) { return cr.inner.GetObserverScopes(ctx, observerID) }) } @@ -388,7 +272,7 @@ func (cr *CachedReader) GetObserverActivity(ctx context.Context, observerID uuid until = until.UTC().Truncate(interval) } key := keyObserverActivityPrefix + observerID.String() + ":" + window.String() + ":" + interval.String() + ":" + until.UTC().Format(time.RFC3339Nano) - return getOrSet(ctx, cr.c, key, observerActivityTTL, func() (*api.ObserverActivity, error) { + return getOrSet(ctx, cr.c, key, observerActivityTTL, func(ctx context.Context) (*api.ObserverActivity, error) { return cr.inner.GetObserverActivity(ctx, observerID, window, interval, until) }) } diff --git a/internal/cache/rollup_stats.go b/internal/cache/rollup_stats.go new file mode 100644 index 0000000..6272fb1 --- /dev/null +++ b/internal/cache/rollup_stats.go @@ -0,0 +1,171 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cache + +import ( + "context" + "encoding/json" + "fmt" + "slices" + "sync" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" +) + +// revisionTTL bounds how stale a rollup revision read can be before a new roll shows up. +const revisionTTL = 10 * time.Second + +type revisionSource interface { + AnalyticsRevision(ctx context.Context) (int64, error) +} + +// revisionMemo spares Postgres a revision read on every cached stats request. +type revisionMemo struct { + mu sync.Mutex + rev int64 + at time.Time + now func() time.Time +} + +// get returns the analytics revision; ok is false when the inner reader can't supply one. +func (m *revisionMemo) get(ctx context.Context, inner api.Reader) (rev int64, ok bool) { + src, isSource := inner.(revisionSource) + if !isSource { + return 0, false + } + now := time.Now + if m.now != nil { + now = m.now + } + m.mu.Lock() + defer m.mu.Unlock() + if !m.at.IsZero() && now().Sub(m.at) < revisionTTL { + return m.rev, true + } + rev, err := src.AnalyticsRevision(ctx) + if err != nil { + return 0, false + } + m.rev, m.at = rev, now() + return rev, true +} + +// rollupKey builds a stats key that changes with the rollup revision. [] and [""] stay +// distinct (all IATAs vs an empty region). +func (cr *CachedReader) rollupKey(ctx context.Context, prefix string, iatas []string, parts ...any) (string, bool) { + rev, ok := cr.rev.get(ctx, cr.inner) + if !ok { + return "", false + } + sorted := append([]string{}, iatas...) + slices.Sort(sorted) + segment, _ := json.Marshal(slices.Compact(sorted)) + key := fmt.Sprintf("%sr%d:%s", prefix, rev, segment) + for _, p := range parts { + key += fmt.Sprintf(":%v", p) + } + return key, true +} + +// cachedRollup caches a rollup-backed read under a revision key, or reads through without one. +func cachedRollup[T any](ctx context.Context, cr *CachedReader, prefix string, iatas []string, fetch func(context.Context) (T, error), parts ...any) (T, error) { + key, ok := cr.rollupKey(ctx, prefix, iatas, parts...) + if !ok { + return fetch(ctx) + } + return getOrSet(ctx, cr.c, key, cr.ttl.Stats, fetch) +} + +// hourKey matches the store's hour snapping so polls within an hour share a key; 0 = default window. +func hourKey(t time.Time) int64 { + if t.IsZero() { + return 0 + } + return t.UTC().Truncate(time.Hour).UnixMilli() +} + +// GetStatsSeries implements [api.Reader]. +func (cr *CachedReader) GetStatsSeries(ctx context.Context, since, until time.Time, iatas []string) (*api.StatsSeries, error) { + return cachedRollup(ctx, cr, keyStatsSeriesPrefix, iatas, func(ctx context.Context) (*api.StatsSeries, error) { + return cr.inner.GetStatsSeries(ctx, since, until, iatas) + }, since.UnixMilli(), until.UnixMilli()) +} + +// GetSignalStats implements [api.Reader]. +func (cr *CachedReader) GetSignalStats(ctx context.Context, since, until time.Time, iatas []string) (*api.SignalStats, error) { + return cachedRollup(ctx, cr, keySignalStatsPrefix, iatas, func(ctx context.Context) (*api.SignalStats, error) { + return cr.inner.GetSignalStats(ctx, since, until, iatas) + }, since.UnixMilli(), until.UnixMilli()) +} + +// GetPathStats implements [api.Reader]. +func (cr *CachedReader) GetPathStats(ctx context.Context, since, until time.Time, iatas []string) (*api.PathStats, error) { + return cachedRollup(ctx, cr, keyPathStatsPrefix, iatas, func(ctx context.Context) (*api.PathStats, error) { + return cr.inner.GetPathStats(ctx, since, until, iatas) + }, since.UnixMilli(), until.UnixMilli()) +} + +// GetStatsOverview implements [api.Reader]. Its window follows the clock, so the current hour is part of the key. +func (cr *CachedReader) GetStatsOverview(ctx context.Context, iatas []string) (*api.StatsOverview, error) { + return cachedRollup(ctx, cr, keyStatsOverviewPrefix, iatas, func(ctx context.Context) (*api.StatsOverview, error) { + return cr.inner.GetStatsOverview(ctx, iatas) + }, hourKey(time.Now())) +} + +// GetStatsObservations implements [api.Reader]. +func (cr *CachedReader) GetStatsObservations(ctx context.Context, iatas []string, since time.Time) ([]api.ObservationPoint, error) { + return cachedRollup(ctx, cr, keyStatsObservationsPrefix, iatas, func(ctx context.Context) ([]api.ObservationPoint, error) { + return cr.inner.GetStatsObservations(ctx, iatas, since) + }, hourKey(since)) +} + +// GetStatsPayloadBreakdown implements [api.Reader]. +func (cr *CachedReader) GetStatsPayloadBreakdown(ctx context.Context, iatas []string, since time.Time) ([]api.PayloadBreakdownItem, error) { + return cachedRollup(ctx, cr, keyStatsBreakdownPrefix, iatas, func(ctx context.Context) ([]api.PayloadBreakdownItem, error) { + return cr.inner.GetStatsPayloadBreakdown(ctx, iatas, since) + }, hourKey(since)) +} + +// GetStatsTopNodes implements [api.Reader]. +func (cr *CachedReader) GetStatsTopNodes(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopNode, error) { + return cachedRollup(ctx, cr, keyStatsTopNodesPrefix, iatas, func(ctx context.Context) ([]api.TopNode, error) { + return cr.inner.GetStatsTopNodes(ctx, iatas, since, limit) + }, hourKey(since), limit) +} + +// GetStatsTopObservers implements [api.Reader]. +func (cr *CachedReader) GetStatsTopObservers(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopObserver, error) { + return cachedRollup(ctx, cr, keyStatsTopObsPrefix, iatas, func(ctx context.Context) ([]api.TopObserver, error) { + return cr.inner.GetStatsTopObservers(ctx, iatas, since, limit) + }, hourKey(since), limit) +} + +// GetStatsTopAdvertisers implements [api.Reader]. +func (cr *CachedReader) GetStatsTopAdvertisers(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopAdvertiser, error) { + return cachedRollup(ctx, cr, keyStatsTopAdvPrefix, iatas, func(ctx context.Context) ([]api.TopAdvertiser, error) { + return cr.inner.GetStatsTopAdvertisers(ctx, iatas, since, limit) + }, hourKey(since), limit) +} + +// GetStatsTopTalkers implements [api.Reader]. +func (cr *CachedReader) GetStatsTopTalkers(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopTalker, error) { + return cachedRollup(ctx, cr, keyStatsTopTalkersPrefix, iatas, func(ctx context.Context) ([]api.TopTalker, error) { + return cr.inner.GetStatsTopTalkers(ctx, iatas, since, limit) + }, hourKey(since), limit) +} + +// GetScopeStats implements [api.Reader]. +func (cr *CachedReader) GetScopeStats(ctx context.Context, iatas []string, since time.Time) ([]api.ScopeStats, error) { + return cachedRollup(ctx, cr, keyScopeStats+":", iatas, func(ctx context.Context) ([]api.ScopeStats, error) { + return cr.inner.GetScopeStats(ctx, iatas, since) + }, hourKey(since)) +} + +// GetScopeByName implements [api.Reader]. +func (cr *CachedReader) GetScopeByName(ctx context.Context, name string) (*api.ScopeDetail, error) { + return cachedRollup(ctx, cr, keyScopeByNamePrefix, nil, func(ctx context.Context) (*api.ScopeDetail, error) { + return cr.inner.GetScopeByName(ctx, name) + }, name) +} diff --git a/internal/cache/scope_stats_test.go b/internal/cache/scope_stats_test.go index 188a5b1..2975ad6 100644 --- a/internal/cache/scope_stats_test.go +++ b/internal/cache/scope_stats_test.go @@ -17,11 +17,13 @@ type scopeStatsReader struct { calls int64 } -func (r *scopeStatsReader) GetScopeStats(context.Context, []string) ([]api.ScopeStats, error) { +func (r *scopeStatsReader) GetScopeStats(context.Context, []string, time.Time) ([]api.ScopeStats, error) { r.calls++ return []api.ScopeStats{{Name: "#test", PacketCount: r.calls}}, nil } +func (r *scopeStatsReader) AnalyticsRevision(context.Context) (int64, error) { return 0, nil } + func TestScopeStatsCacheSeparatesIATAs(t *testing.T) { c, _ := newTestClient(t) inner := &scopeStatsReader{} @@ -42,7 +44,7 @@ func TestScopeStatsCacheSeparatesIATAs(t *testing.T) { {nil, 1}, } { before := slices.Clone(tc.iatas) - rows, err := reader.GetScopeStats(context.Background(), tc.iatas) + rows, err := reader.GetScopeStats(context.Background(), tc.iatas, time.Time{}) if err != nil { t.Fatal(err) } diff --git a/internal/cache/series_test.go b/internal/cache/series_test.go new file mode 100644 index 0000000..24ed082 --- /dev/null +++ b/internal/cache/series_test.go @@ -0,0 +1,204 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cache + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" +) + +type seriesReader struct { + api.Reader + calls atomic.Int64 + revision atomic.Int64 + revReads atomic.Int64 + gate chan struct{} // when set, fetches block until closed +} + +func (r *seriesReader) GetStatsSeries(context.Context, time.Time, time.Time, []string) (*api.StatsSeries, error) { + if r.gate != nil { + <-r.gate + } + return &api.StatsSeries{Revision: r.calls.Add(1)}, nil +} + +func (r *seriesReader) AnalyticsRevision(context.Context) (int64, error) { + r.revReads.Add(1) + return r.revision.Load(), nil +} + +// seriesOnly hides AnalyticsRevision. +type seriesOnly struct{ *seriesReader } + +func (seriesOnly) AnalyticsRevision() {} + +func TestSeriesCacheKeyedByRevision(t *testing.T) { + c, _ := newTestClient(t) + inner := &seriesReader{} + cr := NewCachedReader(inner, c, CacheTTLs{Stats: time.Hour}).(*CachedReader) + clock := time.Unix(0, 0) + cr.rev.now = func() time.Time { return clock } + get := func(iatas []string) int64 { + t.Helper() + got, err := cr.GetStatsSeries(context.Background(), time.UnixMilli(0), time.UnixMilli(3600000), iatas) + if err != nil { + t.Fatal(err) + } + return got.Revision + } + if get(nil) != 1 || get(nil) != 1 || get([]string{""}) != 2 || get([]string{"YVR", "YYJ"}) != 3 || get([]string{"YYJ", "YVR"}) != 3 { + t.Fatalf("unexpected cache behaviour: %d fetches", inner.calls.Load()) + } + // A new roll is invisible until the memo expires, then gets a fresh key. + inner.revision.Store(1) + if get(nil) != 1 { + t.Fatal("revision re-read inside the memo window") + } + clock = clock.Add(revisionTTL) + if get(nil) != 4 { + t.Fatal("new revision did not miss the cache") + } + if n := inner.revReads.Load(); n != 2 { + t.Errorf("revision read %d times, want 2", n) + } +} + +func TestSeriesCacheWithoutRevisionIsUncached(t *testing.T) { + c, _ := newTestClient(t) + inner := &seriesReader{} + cr := NewCachedReader(seriesOnly{inner}, c, CacheTTLs{Stats: time.Hour}) + for range 2 { + if _, err := cr.GetStatsSeries(context.Background(), time.UnixMilli(0), time.UnixMilli(3600000), nil); err != nil { + t.Fatal(err) + } + } + if inner.calls.Load() != 2 { + t.Errorf("fetches = %d, want 2", inner.calls.Load()) + } +} + +func TestSeriesCacheSingleflight(t *testing.T) { + c, _ := newTestClient(t) + inner := &seriesReader{gate: make(chan struct{})} + cr := NewCachedReader(inner, c, CacheTTLs{Stats: time.Hour}) + var wg sync.WaitGroup + for range 8 { + wg.Go(func() { + if _, err := cr.GetStatsSeries(context.Background(), time.UnixMilli(0), time.UnixMilli(3600000), nil); err != nil { + t.Error(err) + } + }) + } + time.Sleep(50 * time.Millisecond) // let every caller reach the miss + close(inner.gate) + wg.Wait() + if n := inner.calls.Load(); n != 1 { + t.Errorf("concurrent misses ran %d fetches, want 1", n) + } +} + +// ctxSeriesReader blocks until released and fails if its fetch context was cancelled. +type ctxSeriesReader struct { + seriesReader + started chan struct{} + release chan struct{} +} + +func (r *ctxSeriesReader) GetStatsSeries(ctx context.Context, _, _ time.Time, _ []string) (*api.StatsSeries, error) { + r.calls.Add(1) + close(r.started) + <-r.release + if err := ctx.Err(); err != nil { + return nil, err + } + return &api.StatsSeries{Hours: []api.StatsSeriesHour{{Status: "complete"}}}, nil +} + +func TestSharedFetchSurvivesLeaderCancel(t *testing.T) { + c, _ := newTestClient(t) + inner := &ctxSeriesReader{started: make(chan struct{}), release: make(chan struct{})} + cr := NewCachedReader(inner, c, CacheTTLs{Stats: time.Hour}) + leaderCtx, cancelLeader := context.WithCancel(context.Background()) + leaderErr := make(chan error, 1) + go func() { + _, err := cr.GetStatsSeries(leaderCtx, time.UnixMilli(0), time.UnixMilli(3600000), nil) + leaderErr <- err + }() + <-inner.started + waiter := make(chan *api.StatsSeries, 1) + go func() { + got, err := cr.GetStatsSeries(context.Background(), time.UnixMilli(0), time.UnixMilli(3600000), nil) + if err != nil { + t.Error(err) + } + waiter <- got + }() + time.Sleep(20 * time.Millisecond) // let the waiter join the in-flight fetch + cancelLeader() + if err := <-leaderErr; !errors.Is(err, context.Canceled) { + t.Errorf("leader err = %v, want its own cancellation", err) + } + close(inner.release) + if got := <-waiter; got == nil || len(got.Hours) != 1 { + t.Errorf("waiter got %+v; the leader's cancellation leaked into the shared fetch", got) + } + if n := inner.calls.Load(); n != 1 { + t.Errorf("fetches = %d, want 1", n) + } +} + +func TestSharedFetchWaiterHonoursOwnDeadline(t *testing.T) { + c, _ := newTestClient(t) + inner := &ctxSeriesReader{started: make(chan struct{}), release: make(chan struct{})} + defer close(inner.release) + cr := NewCachedReader(inner, c, CacheTTLs{Stats: time.Hour}) + go func() { + _, _ = cr.GetStatsSeries(context.Background(), time.UnixMilli(0), time.UnixMilli(3600000), nil) + }() + <-inner.started + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + start := time.Now() + if _, err := cr.GetStatsSeries(ctx, time.UnixMilli(0), time.UnixMilli(3600000), nil); !errors.Is(err, context.DeadlineExceeded) { + t.Errorf("err = %v, want the waiter's deadline", err) + } + if time.Since(start) > time.Second { + t.Error("waiter stayed blocked on the leader") + } +} + +func TestSharedFetchReturnsIndependentCopies(t *testing.T) { + c, _ := newTestClient(t) + inner := &seriesReader{gate: make(chan struct{})} + cr := NewCachedReader(inner, c, CacheTTLs{Stats: time.Hour}) + results := make(chan *api.StatsSeries, 2) + var wg sync.WaitGroup + for range 2 { + wg.Go(func() { + got, err := cr.GetStatsSeries(context.Background(), time.UnixMilli(0), time.UnixMilli(3600000), nil) + if err != nil { + t.Error(err) + return + } + got.Since = time.Now().UnixNano() // a handler decorating its own response + results <- got + }) + } + time.Sleep(20 * time.Millisecond) + close(inner.gate) + wg.Wait() + a, b := <-results, <-results + if a == b { + t.Fatal("concurrent callers share one response object") + } + if inner.calls.Load() != 1 { + t.Errorf("fetches = %d, want 1", inner.calls.Load()) + } +} diff --git a/internal/cache/signal.go b/internal/cache/signal.go deleted file mode 100644 index d1dfaa5..0000000 --- a/internal/cache/signal.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2026 Beacon Contributors -// SPDX-License-Identifier: AGPL-3.0-or-later - -package cache - -import ( - "context" - "encoding/json" - "fmt" - "slices" - "time" - - "github.com/MeshCore-Beacon/beacon-server/internal/api" -) - -func (cr *CachedReader) GetSignalStats(ctx context.Context, since, until time.Time, iatas []string) (*api.SignalStats, error) { - sorted := append([]string{}, iatas...) - slices.Sort(sorted) - sorted = slices.Compact(sorted) - segment, _ := json.Marshal(sorted) // [] and [""] must remain distinct (global vs empty region). - key := fmt.Sprintf("%s%s:%d:%d", keySignalStatsPrefix, segment, since.UnixMilli(), until.UnixMilli()) - return getOrSet(ctx, cr.c, key, cr.ttl.Stats, func() (*api.SignalStats, error) { - return cr.inner.GetSignalStats(ctx, since, until, iatas) - }) -} diff --git a/internal/cache/signal_test.go b/internal/cache/signal_test.go index 89b739b..5e8f324 100644 --- a/internal/cache/signal_test.go +++ b/internal/cache/signal_test.go @@ -17,6 +17,8 @@ type signalStatsReader struct { calls int64 } +func (r *signalStatsReader) AnalyticsRevision(context.Context) (int64, error) { return 0, nil } + func (r *signalStatsReader) GetSignalStats(context.Context, time.Time, time.Time, []string) (*api.SignalStats, error) { r.calls++ return &api.SignalStats{Receptions: r.calls}, nil diff --git a/internal/config/config.go b/internal/config/config.go index 65abd77..1ad309e 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -41,6 +41,7 @@ type Config struct { Presence PresenceConfig `yaml:"presence"` Nodes NodesConfig `yaml:"nodes"` Observers ObserversConfig `yaml:"observers"` + Analytics AnalyticsConfig `yaml:"analytics"` } // BackupConfig enables protected downloads. Disabled by default. @@ -90,6 +91,7 @@ type ResolvedConfig struct { RateLimit ResolvedRateLimitConfig TelemetryResolution time.Duration TelemetryRetention time.Duration + RollupRetention time.Duration PacketRetention time.Duration RouteRetention time.Duration RouteGrace time.Duration @@ -257,6 +259,12 @@ type PacketsConfig struct { Retention duration `yaml:"retention"` } +// AnalyticsConfig controls hourly rollup retention behaviour. +type AnalyticsConfig struct { + // RollupRetention is how long hourly rollups are kept and the longest window /stats/series accepts. Defaults to 2160h (90 days). + RollupRetention duration `yaml:"rollup_retention"` +} + // RoutesConfig controls known-route retention behaviour. type RoutesConfig struct { // Retention is how long a route is kept after it was last observed. @@ -411,6 +419,12 @@ func Load(path string) (*Config, error) { return nil, fmt.Errorf("websocket.allowed_origins[%d]: %w", i, err) } } + if d := cfg.Packets.Retention.Duration; d != 0 && d < 24*time.Hour { + return nil, fmt.Errorf("packets.retention must be at least 24h") + } + if d := cfg.Analytics.RollupRetention.Duration; d != 0 && d < 24*time.Hour { + return nil, fmt.Errorf("analytics.rollup_retention must be at least 24h") + } configDir := filepath.Dir(path) if err := cfg.validateMeshMapper(); err != nil { return nil, err @@ -465,6 +479,7 @@ func Resolve(cfg *Config) ResolvedConfig { }, TelemetryResolution: cfg.Telemetry.Resolution.Duration, TelemetryRetention: cfg.Telemetry.Retention.Duration, + RollupRetention: cfg.Analytics.RollupRetention.Duration, PacketRetention: cfg.Packets.Retention.Duration, RouteRetention: cfg.Routes.Retention.Duration, RouteGrace: cfg.Routes.Grace.Duration, @@ -493,7 +508,10 @@ func Resolve(cfg *Config) ResolvedConfig { r.TelemetryResolution = time.Hour } if r.TelemetryRetention == 0 { - r.TelemetryRetention = 28 * 24 * time.Hour + r.TelemetryRetention = 31 * 24 * time.Hour + } + if r.RollupRetention == 0 { + r.RollupRetention = 90 * 24 * time.Hour } if r.PacketRetention == 0 { r.PacketRetention = 7 * 24 * time.Hour @@ -542,8 +560,8 @@ func Resolve(cfg *Config) ResolvedConfig { func (r ResolvedConfig) String() string { return fmt.Sprintf( - "telemetryResolution=%s telemetryRetention=%s packetRetention=%s routeRetention=%s routeGrace=%s routeMinObs=%d maxConnsPerIP=%d maxConnectsPerMinute=%d viewRefresh=%s reconfirm=%s cleanup=%s presenceFlush=%s presencePacketTTL=%s clockDriftThreshold=%s nodeStaleThreshold=%s nodeDeleteAfter=%s observerDeleteAfter=%s rateLimitEnabled=%t requestsPerMinute=%d burst=%d", - r.TelemetryResolution, r.TelemetryRetention, r.PacketRetention, r.RouteRetention, r.RouteGrace, r.RouteMinObservations, + "telemetryResolution=%s telemetryRetention=%s rollupRetention=%s packetRetention=%s routeRetention=%s routeGrace=%s routeMinObs=%d maxConnsPerIP=%d maxConnectsPerMinute=%d viewRefresh=%s reconfirm=%s cleanup=%s presenceFlush=%s presencePacketTTL=%s clockDriftThreshold=%s nodeStaleThreshold=%s nodeDeleteAfter=%s observerDeleteAfter=%s rateLimitEnabled=%t requestsPerMinute=%d burst=%d", + r.TelemetryResolution, r.TelemetryRetention, r.RollupRetention, r.PacketRetention, r.RouteRetention, r.RouteGrace, r.RouteMinObservations, r.MaxConnsPerIP, r.MaxConnectsPerMinute, r.ViewRefreshInterval, r.ReconfirmInterval, r.CleanupInterval, r.PresenceFlushInterval, r.PresencePacketTTL, r.ClockDriftThreshold, r.NodeStaleThreshold, r.NodeDeleteAfter, r.ObserverDeleteAfter, diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 4ba8b23..f13bbf6 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -127,8 +127,8 @@ func TestResolve_Defaults(t *testing.T) { if r.TelemetryResolution != time.Hour { t.Errorf("expected TelemetryResolution 1h, got %v", r.TelemetryResolution) } - if r.TelemetryRetention != 28*24*time.Hour { - t.Errorf("expected TelemetryRetention 672h, got %v", r.TelemetryRetention) + if r.TelemetryRetention != 31*24*time.Hour { + t.Errorf("expected TelemetryRetention 744h, got %v", r.TelemetryRetention) } if r.PacketRetention != 7*24*time.Hour { t.Errorf("expected PacketRetention 168h, got %v", r.PacketRetention) @@ -211,3 +211,64 @@ func TestResolve_RouteDefaults(t *testing.T) { t.Errorf("RouteMinObservations = %d, want 3", r.RouteMinObservations) } } + +func TestResolve_RollupRetentionDefaults(t *testing.T) { + r := Resolve(&Config{}) + if r.RollupRetention != 90*24*time.Hour { + t.Errorf("expected RollupRetention 2160h (90 days), got %v", r.RollupRetention) + } +} + +func TestResolve_TelemetryRetentionDefault(t *testing.T) { + r := Resolve(&Config{}) + if r.TelemetryRetention != 31*24*time.Hour { + t.Errorf("expected TelemetryRetention 744h (31 days), got %v", r.TelemetryRetention) + } +} + +func TestResolve_RollupRetentionExplicitOverride(t *testing.T) { + cfg := &Config{} + cfg.Analytics.RollupRetention.Duration = 60 * 24 * time.Hour + r := Resolve(cfg) + if r.RollupRetention != 60*24*time.Hour { + t.Errorf("expected RollupRetention 1440h (60 days), got %v", r.RollupRetention) + } +} + +func TestLoad_PacketsRetentionTooShort(t *testing.T) { + f, err := os.CreateTemp("", "beacon-config-*.yaml") + if err != nil { + t.Fatalf("failed to create temp file: %v", err) + } + defer os.Remove(f.Name()) + + _, _ = f.WriteString("packets:\n retention: 1h\n") + f.Close() + + _, err = Load(f.Name()) + if err == nil { + t.Fatal("expected error for packets.retention < 24h, got nil") + } + if !strings.Contains(err.Error(), "packets.retention must be at least 24h") { + t.Errorf("expected error about packets.retention, got: %v", err) + } +} + +func TestLoad_AnalyticsRollupRetentionTooShort(t *testing.T) { + f, err := os.CreateTemp("", "beacon-config-*.yaml") + if err != nil { + t.Fatalf("failed to create temp file: %v", err) + } + defer os.Remove(f.Name()) + + _, _ = f.WriteString("analytics:\n rollup_retention: 1h\n") + f.Close() + + _, err = Load(f.Name()) + if err == nil { + t.Fatal("expected error for analytics.rollup_retention < 24h, got nil") + } + if !strings.Contains(err.Error(), "analytics.rollup_retention must be at least 24h") { + t.Errorf("expected error about analytics.rollup_retention, got: %v", err) + } +} diff --git a/internal/ingest/advert_endpoint_update_test.go b/internal/ingest/advert_endpoint_update_test.go index aea19c4..a089952 100644 --- a/internal/ingest/advert_endpoint_update_test.go +++ b/internal/ingest/advert_endpoint_update_test.go @@ -41,8 +41,8 @@ func (s *advertEndpointDB) UpsertNode(_ context.Context, n UpsertNodeParams, _ R return s.node.ID, nil } -func (s *advertEndpointDB) InsertObservation(context.Context, InsertObservationParams) (bool, error) { - return !s.duplicate, nil +func (s *advertEndpointDB) InsertObservation(context.Context, InsertObservationParams) (bool, int64, error) { + return !s.duplicate, 1, nil } func TestAdvertEndpointUsesUpdatedName(t *testing.T) { @@ -99,7 +99,7 @@ func TestDuplicateAdvertSkipsLiveEndpointLookup(t *testing.T) { func TestRepeatAdvertUsesCurrentIdentityWithoutReapplyingAdvert(t *testing.T) { r := newRepeatHarness(t, true) - store := &advertEndpointDB{endpointCaptureDB: &endpointCaptureDB{stubDB: r.db.stubDB, node: api.ResolvedNode{ID: uuid.New(), PublicKey: "aa"}}} + store := &advertEndpointDB{endpointCaptureDB: &endpointCaptureDB{stubDB: r.db, node: api.ResolvedNode{ID: uuid.New(), PublicKey: "aa"}}} r.w.db = store packet := buildAdvertPacketWithData(t, append([]byte{meshcore.AdvertTypeRepeater | meshcore.AdvertNameMask}, []byte("Advert name")...), false) hear := func(path byte) []packetObservationEvent { diff --git a/internal/ingest/backfill.go b/internal/ingest/backfill.go index 1a41283..65402ec 100644 --- a/internal/ingest/backfill.go +++ b/internal/ingest/backfill.go @@ -41,7 +41,7 @@ type DecryptGroupTextResult struct { // decides what to do with that. The live path falls back to UpsertChannelHashOnly itself; // BackfillChannelMessages, which only scans packets already recorded that way, just leaves // them as they are and tries again next boot. -func DecryptGroupText(ctx context.Context, db DB, keys ChannelKeyStore, packetHash, rawPayload []byte) (*DecryptGroupTextResult, error) { +func DecryptGroupText(ctx context.Context, db DB, keys ChannelKeyStore, packetHash, rawPayload []byte, historical bool) (*DecryptGroupTextResult, error) { grpTxt, err := meshcore.GroupTextFromBytes(rawPayload) if err != nil { return nil, err @@ -70,6 +70,7 @@ func DecryptGroupText(ctx context.Context, db DB, keys ChannelKeyStore, packetHa PacketHash: packetHash, SenderName: strings.ReplaceAll(strings.ToValidUTF8(payload.Sender, "\uFFFD"), "\x00", ""), SentAt: time.Unix(int64(payload.Timestamp), 0), + Historical: historical, Content: strings.ReplaceAll(strings.ToValidUTF8(payload.Text, "\uFFFD"), "\x00", ""), } newMsg, err := db.InsertChannelMessage(ctx, params) @@ -122,7 +123,7 @@ func BackfillChannelHashes(ctx context.Context, db DB, keys ChannelKeyStore, has func backfill(ctx context.Context, db DB, keys ChannelKeyStore, packets []UndecryptedPacket) int { decrypted := 0 for _, p := range packets { - result, err := DecryptGroupText(ctx, db, keys, p.PacketHash, p.RawPayload) + result, err := DecryptGroupText(ctx, db, keys, p.PacketHash, p.RawPayload, true) if err != nil { slog.Error(fmt.Sprintf("ingest: backfill: decrypt failed for packet %s", hex.EncodeToString(p.PacketHash)), "component", "ingest", "error", err) continue diff --git a/internal/ingest/backfill_test.go b/internal/ingest/backfill_test.go index c50d7a6..7088b2b 100644 --- a/internal/ingest/backfill_test.go +++ b/internal/ingest/backfill_test.go @@ -39,7 +39,7 @@ func TestDecryptGroupText_Success(t *testing.T) { }} raw := encryptedGroupText(t, channelHash, psk, "ded", "hello") - result, err := DecryptGroupText(context.Background(), db, keys, []byte{0x01}, raw) + result, err := DecryptGroupText(context.Background(), db, keys, []byte{0x01}, raw, false) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -64,7 +64,7 @@ func TestDecryptGroupText_NoMatchingKey(t *testing.T) { raw := encryptedGroupText(t, channelHash, make([]byte, 16), "ded", "hello") keys := &mapKeys{entries: map[byte][]keystore.Entry{}} - result, err := DecryptGroupText(context.Background(), db, keys, []byte{0x02}, raw) + result, err := DecryptGroupText(context.Background(), db, keys, []byte{0x02}, raw, false) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -88,7 +88,7 @@ func TestDecryptGroupText_WrongKeyForHash(t *testing.T) { channelHash: {{Key: wrongKey, Fingerprint: []byte{0xBB}}}, }} - result, err := DecryptGroupText(context.Background(), db, keys, []byte{0x03}, raw) + result, err := DecryptGroupText(context.Background(), db, keys, []byte{0x03}, raw, false) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -100,7 +100,7 @@ func TestDecryptGroupText_WrongKeyForHash(t *testing.T) { func TestDecryptGroupText_MalformedPayload(t *testing.T) { db := &stubDB{} keys := &mapKeys{} - _, err := DecryptGroupText(context.Background(), db, keys, []byte{0x04}, []byte{}) + _, err := DecryptGroupText(context.Background(), db, keys, []byte{0x04}, []byte{}, false) if err == nil { t.Fatal("expected an error for an empty/malformed payload") } @@ -163,3 +163,24 @@ func TestBackfillChannelMessages_NoUndecryptedPackets(t *testing.T) { t.Errorf("expected 0, got %d", n) } } + +func TestBackfillMarksMessagesHistorical(t *testing.T) { + psk := make([]byte, 16) + db := &stubDB{ + insertChannelMessageResult: true, + undecryptedPackets: []UndecryptedPacket{{PacketHash: []byte{0x01}, RawPayload: encryptedGroupText(t, 0x44, psk, "ded", "hi")}}, + } + keys := &mapKeys{entries: map[byte][]keystore.Entry{0x44: {{Key: psk, Fingerprint: []byte{0xAA}}}}} + if n, err := BackfillChannelMessages(context.Background(), db, keys); err != nil || n != 1 { + t.Fatalf("n=%d err=%v", n, err) + } + if len(db.channelMessages) != 1 || !db.channelMessages[0].Historical { + t.Errorf("backfilled messages %+v; want Historical so their rolled hours are re-rolled", db.channelMessages) + } + if _, err := DecryptGroupText(context.Background(), db, keys, []byte{0x02}, encryptedGroupText(t, 0x44, psk, "x", "live"), false); err != nil { + t.Fatal(err) + } + if db.channelMessages[1].Historical { + t.Error("live messages must not queue re-rolls") + } +} diff --git a/internal/ingest/endpoint_work_test.go b/internal/ingest/endpoint_work_test.go index 5030291..ae12433 100644 --- a/internal/ingest/endpoint_work_test.go +++ b/internal/ingest/endpoint_work_test.go @@ -25,7 +25,7 @@ func endpointTestPacket() *meshcore.Packet { } func TestEndpointLookupsOnlyForLiveHearings(t *testing.T) { r := newRepeatHarness(t, true) - d := &endpointLookupDB{stubDB: r.db.stubDB} + d := &endpointLookupDB{stubDB: r.db} r.w.db = d packet := endpointTestPacket() hear := func(inserted bool, path byte) { diff --git a/internal/ingest/frame_length_test.go b/internal/ingest/frame_length_test.go index d69601d..c2afb51 100644 --- a/internal/ingest/frame_length_test.go +++ b/internal/ingest/frame_length_test.go @@ -26,9 +26,9 @@ func (s *frameCaptureDB) UpsertPacket(_ context.Context, p UpsertPacketParams) ( return true, nil } -func (s *frameCaptureDB) InsertObservation(_ context.Context, o InsertObservationParams) (bool, error) { +func (s *frameCaptureDB) InsertObservation(_ context.Context, o InsertObservationParams) (bool, int64, error) { s.observed = append(s.observed, o) - return true, nil + return true, 1, nil } func (s *frameCaptureDB) GetObserverRadio(context.Context, uuid.UUID) (RadioSettings, error) { diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index c9d0dc7..9ca3584 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -86,7 +86,7 @@ type Config struct { type DB interface { // UpsertObserver upserts the observers row keyed on pubkey, and returns // the Observer ID, Display Name and an error if any. - UpsertObserver(ctx context.Context, pubkey []byte) (uuid.UUID, string, error) + UpsertObserver(ctx context.Context, pubkey []byte, iata string) (uuid.UUID, string, error) // UpsertObserverBroker records that this observer was seen on brokerName. UpsertObserverBroker(ctx context.Context, observerID uuid.UUID, brokerName string, isPacket bool) error @@ -100,9 +100,9 @@ type DB interface { // SetPacketDecrypted marks a packet as decrypted in the DB SetPacketDecrypted(ctx context.Context, hash []byte) error - // InsertObservation inserts a packet_observations row. - // Returns (inserted, error); inserted=false means ON CONFLICT DO NOTHING fired. - InsertObservation(ctx context.Context, o InsertObservationParams) (bool, error) + // InsertObservation inserts a packet_observations row and returns the packet's + // observation count; inserted=false means ON CONFLICT DO NOTHING fired. + InsertObservation(ctx context.Context, o InsertObservationParams) (inserted bool, observationCount int64, err error) // SetNodeCapability flips supports_multibyte_paths or supports_multibyte_traces // for a node, never downgrading an existing TRUE. @@ -138,7 +138,7 @@ type DB interface { // and any error. UpdateObserverStatus(ctx context.Context, p UpdateObserverStatusParams) (uuid.UUID, error) - // GetObserverLastIATA returns the IATA from the most recent observation for the given observer. + // GetObserverLastIATA returns the IATA of the observer's most recent packet, or "" if none. GetObserverLastIATA(ctx context.Context, observerID uuid.UUID) (string, error) // InsertObserverTelemetry stores a telemetry snapshot for an observer. @@ -180,11 +180,8 @@ type DB interface { // UpsertChannelIATA upserts a channel_iatas row. UpsertChannelIATA(ctx context.Context, channelHash []byte, iata string, heardAt time.Time) error - // UpsertTraceIATA upserts a trace_iatas row. - UpsertTraceIATA(ctx context.Context, traceTag []byte, iata string, heardAt time.Time) error - - // GetPacketObservationCount returns the number of rows for the packet observations - GetPacketObservationCount(ctx context.Context, packetHash []byte) (int64, error) + // RecordTrace records one hearing of a TRACE packet in trace_iatas and trace_tags. + RecordTrace(ctx context.Context, h TraceHearing) error // GetTransportScopeByName returns the ID of a transport scope by its normalized name. GetTransportScopeByName(ctx context.Context, name string) (int32, error) diff --git a/internal/ingest/ingest_test.go b/internal/ingest/ingest_test.go index 14acd2a..cffa465 100644 --- a/internal/ingest/ingest_test.go +++ b/internal/ingest/ingest_test.go @@ -180,8 +180,11 @@ type stubDB struct { upsertChannelCalls int upsertChannelHashOnlyCalls int upsertChannelIATACalls int - upsertTraceIATACalls int + traceHearings []TraceHearing + packetIsNew bool observationInserted bool + channelMessages []InsertChannelMessageParams + observationCount int64 insertChannelMessageResult bool // configurable return for InsertChannelMessage; default false undecryptedPackets []UndecryptedPacket undecryptedByHash []UndecryptedPacket @@ -201,7 +204,7 @@ func (s *stubDB) SetNodeCapability(_ context.Context, nodeID uuid.UUID, paths, t } // no-op implementations for remaining DB interface methods -func (s *stubDB) UpsertObserver(_ context.Context, _ []byte) (uuid.UUID, string, error) { +func (s *stubDB) UpsertObserver(_ context.Context, _ []byte, _ string) (uuid.UUID, string, error) { return uuid.Nil, "", nil } func (s *stubDB) UpsertObserverBroker(_ context.Context, _ uuid.UUID, _ string, _ bool) error { @@ -209,11 +212,11 @@ func (s *stubDB) UpsertObserverBroker(_ context.Context, _ uuid.UUID, _ string, } func (s *stubDB) UpsertIATA(_ context.Context, _ string) error { return nil } func (s *stubDB) UpsertPacket(_ context.Context, _ UpsertPacketParams) (bool, error) { - return false, nil + return s.packetIsNew, nil } func (s *stubDB) SetPacketDecrypted(_ context.Context, _ []byte) error { return nil } -func (s *stubDB) InsertObservation(_ context.Context, _ InsertObservationParams) (bool, error) { - return s.observationInserted, nil +func (s *stubDB) InsertObservation(_ context.Context, _ InsertObservationParams) (bool, int64, error) { + return s.observationInserted, s.observationCount, nil } func (s *stubDB) SetNodeDefaultScope(_ context.Context, _ uuid.UUID, _ int32) error { return nil } func (s *stubDB) UpsertNode(_ context.Context, params UpsertNodeParams, _ RadioSettings) (uuid.UUID, error) { @@ -234,7 +237,8 @@ func (s *stubDB) GetNodesByIDs(_ context.Context, _ []uuid.UUID) (map[uuid.UUID] return nil, nil } -func (s *stubDB) InsertChannelMessage(_ context.Context, _ InsertChannelMessageParams) (*InsertedChannelMessage, error) { +func (s *stubDB) InsertChannelMessage(_ context.Context, m InsertChannelMessageParams) (*InsertedChannelMessage, error) { + s.channelMessages = append(s.channelMessages, m) if !s.insertChannelMessageResult { return nil, nil } @@ -294,15 +298,11 @@ func (s *stubDB) UpsertChannelIATA(_ context.Context, _ []byte, _ string, _ time return nil } -func (s *stubDB) UpsertTraceIATA(_ context.Context, _ []byte, _ string, _ time.Time) error { - s.upsertTraceIATACalls++ +func (s *stubDB) RecordTrace(_ context.Context, h TraceHearing) error { + s.traceHearings = append(s.traceHearings, h) return nil } -func (s *stubDB) GetPacketObservationCount(_ context.Context, _ []byte) (int64, error) { - return 0, nil -} - func (s *stubDB) GetTransportScopeByName(_ context.Context, _ string) (int32, error) { return 0, nil } func (s *stubDB) UpsertObserverScope(_ context.Context, _ uuid.UUID, _ int32) error { return nil } func (s *stubDB) UpsertKnownRoute(_ context.Context, _ []uuid.UUID, _ [][]byte, _ string, _ int32) error { diff --git a/internal/ingest/neighbors.go b/internal/ingest/neighbors.go index 3def0c9..6579560 100644 --- a/internal/ingest/neighbors.go +++ b/internal/ingest/neighbors.go @@ -43,7 +43,7 @@ func (w *Worker) handleNeighbors(ctx context.Context, iata, pubkeyHex string, ra return } - observerID, _, err := w.db.UpsertObserver(ctx, pubkey) + observerID, _, err := w.db.UpsertObserver(ctx, pubkey, "") if err != nil { w.log.Error(fmt.Sprintf("db: upsert observer failed in neighbors from %s", pubkeyHex), "error", err) return diff --git a/internal/ingest/packet.go b/internal/ingest/packet.go index 7bc4ccf..8f56c81 100644 --- a/internal/ingest/packet.go +++ b/internal/ingest/packet.go @@ -38,6 +38,14 @@ type UpsertPacketParams struct { TraceTag []byte } +// TraceHearing is one hearing of a TRACE packet. The packet's own contribution to its tag +// (count, type, best path) is recorded when the packet is stored. +type TraceHearing struct { + TraceTag []byte + IATA string + HeardAt time.Time +} + // InsertObservationParams mirrors the columns written on packet_observations insert. type InsertObservationParams struct { PacketHash []byte @@ -321,7 +329,7 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ return } - id, observerName, err := w.db.UpsertObserver(ctx, pubkeyBytes) + id, observerName, err := w.db.UpsertObserver(ctx, pubkeyBytes, iata) if err != nil { w.log.Error(fmt.Sprintf("db: upsert observer failed with packet from %s/%s", iata, pubkeyHex), "error", err) return @@ -775,7 +783,7 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ PayloadType: int16(packet.PayloadType()), AirtimeMs: airtimeMs, } - inserted, err := w.db.InsertObservation(ctx, oParams) + inserted, observationCount, err := w.db.InsertObservation(ctx, oParams) if err != nil { w.log.Error(fmt.Sprintf("db: insert observation failed from %s/%s", iata, pubkeyHex), "error", err) return @@ -799,10 +807,10 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ } } - // Runs on duplicate observations too; the upsert only writes when the row is >1h stale. + // Runs on duplicate observations too, so last_heard_at tracks every hearing. if traceTag != nil { - if err := w.db.UpsertTraceIATA(ctx, traceTag, iata, heardAt); err != nil { - w.log.Error(fmt.Sprintf("db: upsert trace IATA failed from %s/%s", iata, pubkeyHex), "error", err) + if err := w.db.RecordTrace(ctx, TraceHearing{TraceTag: traceTag, IATA: iata, HeardAt: heardAt}); err != nil { + w.log.Error(fmt.Sprintf("db: record trace failed from %s/%s", iata, pubkeyHex), "error", err) } } @@ -895,12 +903,7 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ evt.Observation.PathLength.HopCount = packet.PathHashCount() evt.Observation.PropagationTimeMs = 0 // not yet calculated if !repeat { - count, err := w.db.GetPacketObservationCount(ctx, packetHash[:]) - if err != nil { - w.log.Error("failed to get observation count", "error", err) - count = 0 - } - evt.Packet.ObservationCount = count + evt.Packet.ObservationCount = observationCount } if matchedScope != nil { evt.Packet.Scope = matchedScope diff --git a/internal/ingest/packet_endpoints_test.go b/internal/ingest/packet_endpoints_test.go index 3611f6f..ef37c9d 100644 --- a/internal/ingest/packet_endpoints_test.go +++ b/internal/ingest/packet_endpoints_test.go @@ -37,9 +37,9 @@ func (s *endpointCaptureDB) ResolveEndpointHashes(_ context.Context, _ string, h } return nil, nil } -func (s *endpointCaptureDB) InsertObservation(context.Context, InsertObservationParams) (bool, error) { +func (s *endpointCaptureDB) InsertObservation(context.Context, InsertObservationParams) (bool, int64, error) { s.observed++ - return true, nil + return true, 1, nil } // Stored rows resolve endpoints at read time, but live events still carry them. diff --git a/internal/ingest/repeats_test.go b/internal/ingest/repeats_test.go index 0f9e73d..178b8f0 100644 --- a/internal/ingest/repeats_test.go +++ b/internal/ingest/repeats_test.go @@ -14,28 +14,17 @@ import ( "github.com/google/uuid" ) -type countQueryDB struct { - *stubDB - countQueries int -} - -func (s *countQueryDB) GetPacketObservationCount(context.Context, []byte) (int64, error) { - s.countQueries++ - return 1, nil -} - type repeatHarness struct { t *testing.T ctx context.Context w *Worker - db *countQueryDB + db *stubDB client *hub.Client } func newRepeatHarness(t *testing.T, includeRepeats bool) *repeatHarness { - w, base := newTestWorker() - db := &countQueryDB{stubDB: base} - w.db = db + w, db := newTestWorker() + db.observationCount = 3 ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) t.Cleanup(cancel) go w.hub.Run() @@ -96,8 +85,8 @@ func TestHandlePacket_Repeats(t *testing.T) { if len(first) != 1 || strings.Contains(string(first[0]), "isRepeat") { t.Fatalf("first hearing: %s", first) } - if r.db.countQueries != 1 { - t.Fatalf("first hearing ran %d count queries", r.db.countQueries) + if got := string(packetFields(t, first[0])["observationCount"]); got != "3" { + t.Fatalf("first hearing observationCount = %s, want the count InsertObservation returned", got) } if got := r.hear(false, 0x11); len(got) != 0 { t.Fatalf("broker copy of the first hearing sent: %s", got) @@ -111,9 +100,6 @@ func TestHandlePacket_Repeats(t *testing.T) { if string(fields["isRepeat"]) != "true" || string(fields["isFirstObservation"]) != "false" || string(fields["observationCount"]) != "0" { t.Fatalf("repeat packet fields: %v", fields) } - if r.db.countQueries != 1 { - t.Fatalf("repeat ran a count query (%d total)", r.db.countQueries) - } if got := r.hear(false, 0x22, 0x11); len(got) != 0 { t.Fatalf("same path sent twice: %s", got) } @@ -127,9 +113,6 @@ func TestHandlePacket_RepeatsOffByDefault(t *testing.T) { if got := r.hear(false, 0x22, 0x11); len(got) != 0 { t.Fatalf("repeat sent with nobody opted in: %s", got) } - if r.db.countQueries != 1 { - t.Fatalf("duplicate ran a count query (%d total)", r.db.countQueries) - } } // A broker copy racing through the inserted hearing's DB work must find its path already diff --git a/internal/ingest/side_effects.go b/internal/ingest/side_effects.go index 46fa8fe..c92095f 100644 --- a/internal/ingest/side_effects.go +++ b/internal/ingest/side_effects.go @@ -37,6 +37,7 @@ type InsertChannelMessageParams struct { SenderName string Content string SentAt time.Time + Historical bool // backfilled: its hours may already be rolled, so queue them for a re-roll } // InsertedChannelMessage is returned only for a new message, using its stored packet evidence. @@ -207,7 +208,7 @@ func (w *Worker) handlePayloadTypeSideEffects(ctx context.Context, packet *meshc } channelHashBytes := []byte{grpTxt.ChannelHash} - result, err := DecryptGroupText(ctx, w.db, w.keys, packetHash, packet.Payload) + result, err := DecryptGroupText(ctx, w.db, w.keys, packetHash, packet.Payload, false) if err != nil { w.log.Error("decrypt group text failed", "error", err) return diff --git a/internal/ingest/side_effects_test.go b/internal/ingest/side_effects_test.go index 93c5a4e..d6e6fd5 100644 --- a/internal/ingest/side_effects_test.go +++ b/internal/ingest/side_effects_test.go @@ -297,14 +297,14 @@ func TestHandlePacket_Advert_SkipsChannelIATA(t *testing.T) { if db.upsertChannelIATACalls != 0 { t.Errorf("expected UpsertChannelIATA NOT to be called for a non-channel packet, got %d calls", db.upsertChannelIATACalls) } - if db.upsertTraceIATACalls != 0 { - t.Errorf("expected UpsertTraceIATA NOT to be called for a non-trace packet, got %d calls", db.upsertTraceIATACalls) + if len(db.traceHearings) != 0 { + t.Errorf("expected RecordTrace NOT to be called for a non-trace packet, got %d calls", len(db.traceHearings)) } } func buildTracePacket(t *testing.T) *meshcore.Packet { t.Helper() - payload, err := (&meshcore.Trace{Tag: 0xdeadbeef, AuthCode: 1}).ToBytes() + payload, err := (&meshcore.Trace{Tag: 0xdeadbeef, AuthCode: 1, PathHashes: []byte{0xaa, 0xbb}}).ToBytes() if err != nil { t.Fatalf("trace to bytes: %v", err) } @@ -314,14 +314,22 @@ func buildTracePacket(t *testing.T) *meshcore.Packet { } } -func TestHandlePacket_Trace_UpsertsTraceIATA(t *testing.T) { +func TestHandlePacket_Trace_RecordsTrace(t *testing.T) { w, db := newTestWorker() db.observationInserted = true + db.packetIsNew = true envelope := packetEnvelope(t, buildTracePacket(t)) + w.handlePacket(context.Background(), "YOW", "0102", envelope) + db.packetIsNew, db.observationInserted = false, false w.handlePacket(context.Background(), "YOW", "0102", envelope) - if db.upsertTraceIATACalls != 1 { - t.Errorf("expected UpsertTraceIATA to be called once for a stored trace, got %d", db.upsertTraceIATACalls) + if len(db.traceHearings) != 2 { + t.Fatalf("expected RecordTrace for every hearing, got %d", len(db.traceHearings)) + } + for _, h := range db.traceHearings { + if h.IATA != "YOW" || h.HeardAt.IsZero() || hex.EncodeToString(h.TraceTag) != "efbeadde" { + t.Errorf("unexpected hearing %+v", h) + } } } diff --git a/internal/ingest/status.go b/internal/ingest/status.go index 9fe6a82..8dde612 100644 --- a/internal/ingest/status.go +++ b/internal/ingest/status.go @@ -152,7 +152,7 @@ func (w *Worker) handleStatus(ctx context.Context, pubkeyHex string, raw []byte) if len(dirty) > 0 || rawDirty { w.log.Warn(fmt.Sprintf("stripped NUL bytes from status from %s", pubkeyHex), "fields", dirty, "metadata", rawDirty) } - id, _, err := w.db.UpsertObserver(ctx, pubkey) + id, _, err := w.db.UpsertObserver(ctx, pubkey, "") if err != nil { w.log.Error(fmt.Sprintf("db: upsert observer failed in status from %s", pubkeyHex), "error", err) return diff --git a/internal/presence/coalescer.go b/internal/presence/coalescer.go index 517f121..221aac9 100644 --- a/internal/presence/coalescer.go +++ b/internal/presence/coalescer.go @@ -25,9 +25,10 @@ import ( type Store interface { ingest.DB - // TouchObservers applies coalesced last_seen bumps and observation_count - // deltas for the given observer IDs in one statement. - TouchObservers(ctx context.Context, ids []uuid.UUID, seen []time.Time, counts []int32) error + // TouchObservers applies coalesced last_seen bumps, observation_count + // deltas and last_iata for the given observer IDs in one statement. An + // IATA only replaces one heard earlier (iataAts), so a late flush can't regress it. + TouchObservers(ctx context.Context, ids []uuid.UUID, seen []time.Time, counts []int32, iatas []string, iataAts []time.Time) error // TouchObserverBrokers applies coalesced last_seen/last_packet_at bumps // for the given (observer, broker) pairs in one statement. @@ -47,8 +48,10 @@ type identity struct { } type observerBump struct { - seen time.Time - count int32 + seen time.Time + count int32 + iata string + iataAt time.Time } type brokerBump struct{ seen, packet time.Time } @@ -97,13 +100,16 @@ func New(store Store, flushInterval, packetTTL time.Duration) *Coalescer { // UpsertObserver serves repeat lookups from the identity cache and records a // last_seen/observation_count bump instead of writing through. -func (c *Coalescer) UpsertObserver(ctx context.Context, pubkey []byte) (uuid.UUID, string, error) { +func (c *Coalescer) UpsertObserver(ctx context.Context, pubkey []byte, iata string) (uuid.UUID, string, error) { key := string(pubkey) c.mu.Lock() if ident, ok := c.identities[key]; ok { bump := c.dirtyObservers[ident.id] bump.seen = c.now() bump.count++ + if iata != "" { + bump.iata, bump.iataAt = iata, bump.seen + } c.dirtyObservers[ident.id] = bump ident.seen = bump.seen c.identities[key] = ident @@ -112,7 +118,7 @@ func (c *Coalescer) UpsertObserver(ctx context.Context, pubkey []byte) (uuid.UUI } c.mu.Unlock() - id, name, err := c.Store.UpsertObserver(ctx, pubkey) + id, name, err := c.Store.UpsertObserver(ctx, pubkey, iata) if err != nil { return id, name, err } @@ -305,10 +311,14 @@ func (c *Coalescer) flushObservers(ctx context.Context, observers map[uuid.UUID] ids := make([]uuid.UUID, 0, len(observers)) seen := make([]time.Time, 0, len(observers)) counts := make([]int32, 0, len(observers)) + iatas := make([]string, 0, len(observers)) + iataAts := make([]time.Time, 0, len(observers)) for id, bump := range observers { ids = append(ids, id) seen = append(seen, bump.seen) counts = append(counts, bump.count) + iatas = append(iatas, bump.iata) + iataAts = append(iataAts, bump.iataAt) } - return c.Store.TouchObservers(ctx, ids, seen, counts) + return c.Store.TouchObservers(ctx, ids, seen, counts, iatas, iataAts) } diff --git a/internal/presence/coalescer_test.go b/internal/presence/coalescer_test.go index 0900f5f..6dd2267 100644 --- a/internal/presence/coalescer_test.go +++ b/internal/presence/coalescer_test.go @@ -10,21 +10,22 @@ import ( "testing" "time" + "github.com/MeshCore-Beacon/beacon-server/internal/ingest" "github.com/google/uuid" ) type retentionStore struct { Store id uuid.UUID - touch func(context.Context, []uuid.UUID, []time.Time, []int32) error + touch func(context.Context, []uuid.UUID, []time.Time, []int32, []string, []time.Time) error prune func(context.Context, time.Time) ([]uuid.UUID, error) } -func (s *retentionStore) UpsertObserver(context.Context, []byte) (uuid.UUID, string, error) { +func (s *retentionStore) UpsertObserver(context.Context, []byte, string) (uuid.UUID, string, error) { return s.id, "fixture", nil } -func (s *retentionStore) TouchObservers(ctx context.Context, ids []uuid.UUID, seen []time.Time, counts []int32) error { - return s.touch(ctx, ids, seen, counts) +func (s *retentionStore) TouchObservers(ctx context.Context, ids []uuid.UUID, seen []time.Time, counts []int32, iatas []string, iataAts []time.Time) error { + return s.touch(ctx, ids, seen, counts, iatas, iataAts) } func (s *retentionStore) DeleteOldObservers(ctx context.Context, cutoff time.Time) ([]uuid.UUID, error) { return s.prune(ctx, cutoff) @@ -34,13 +35,13 @@ func TestObserverRetentionAbortOnFlushFailure(t *testing.T) { for _, cause := range []error{errors.New("flush unavailable"), context.Canceled} { t.Run(cause.Error(), func(t *testing.T) { store := &retentionStore{id: uuid.New()} - store.touch = func(context.Context, []uuid.UUID, []time.Time, []int32) error { return cause } + store.touch = func(context.Context, []uuid.UUID, []time.Time, []int32, []string, []time.Time) error { return cause } store.prune = func(context.Context, time.Time) ([]uuid.UUID, error) { t.Error("deletion attempted after failed preparation") return nil, nil } c := New(store, time.Second, time.Second) - if _, _, err := c.UpsertObserver(context.Background(), []byte("fixture")); err != nil { + if _, _, err := c.UpsertObserver(context.Background(), []byte("fixture"), "YVR"); err != nil { t.Fatal(err) } if _, err := c.DeleteOldObservers(context.Background(), time.Now()); !errors.Is(err, cause) { @@ -63,13 +64,13 @@ func TestObserverRetentionPreservesActivityAfterFlush(t *testing.T) { c := New(store, time.Second, time.Second) c.now = func() time.Time { return now } for range 2 { - if _, _, err := c.UpsertObserver(ctx, []byte("fixture")); err != nil { + if _, _, err := c.UpsertObserver(ctx, []byte("fixture"), "YVR"); err != nil { t.Fatal(err) } } started, release, done := make(chan struct{}), make(chan struct{}), make(chan struct{}) var calls atomic.Int32 - store.touch = func(_ context.Context, ids []uuid.UUID, seen []time.Time, counts []int32) error { + store.touch = func(_ context.Context, ids []uuid.UUID, seen []time.Time, counts []int32, _ []string, _ []time.Time) error { if calls.Add(1) == 1 { close(started) <-release @@ -103,3 +104,38 @@ func TestObserverRetentionPreservesActivityAfterFlush(t *testing.T) { }) } } + +func (s *retentionStore) UpdateObserverStatus(context.Context, ingest.UpdateObserverStatusParams) (uuid.UUID, error) { + return s.id, nil +} + +func TestObserverLastIATAFlush(t *testing.T) { + ctx := context.Background() + now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + store := &retentionStore{id: uuid.New()} + var gotIATAs []string + var gotAts []time.Time + store.touch = func(_ context.Context, _ []uuid.UUID, _ []time.Time, _ []int32, iatas []string, ats []time.Time) error { + gotIATAs, gotAts = iatas, ats + return nil + } + c := New(store, time.Second, time.Second) + c.now = func() time.Time { return now } + pk := []byte("fixture") + upsert := func(iata string) { + t.Helper() + if _, _, err := c.UpsertObserver(ctx, pk, iata); err != nil { + t.Fatal(err) + } + now = now.Add(time.Second) + } + + upsert("YVR") // write-through + upsert("YYZ") + heard := now.Add(-time.Second) + upsert("") // status/neighbors path must not clear it or move its timestamp + c.Flush(ctx) + if len(gotIATAs) != 1 || gotIATAs[0] != "YYZ" || !gotAts[0].Equal(heard) { + t.Fatalf("flushed %v at %v, want YYZ at %s", gotIATAs, gotAts, heard) + } +} diff --git a/sqlc.yaml b/sqlc.yaml index 032c991..177f5b3 100644 --- a/sqlc.yaml +++ b/sqlc.yaml @@ -5,7 +5,7 @@ version: "2" sql: - engine: "postgresql" queries: "db/queries" - schema: "db/migrations" + schema: ["db/migrations", "db/sqlc_scratch"] gen: go: package: "db" @@ -22,11 +22,3 @@ sql: go_type: "encoding/json.RawMessage" - db_type: "bpchar" go_type: "string" - - # Coalesced archive labels can still be NULL; preserve the API pointers. - - column: "mv_top_observers_by_iata.display_name" - go_type: {type: "string", pointer: true} - - column: "mv_top_observers_by_iata.observer_type" - go_type: {type: "string", pointer: true} - - column: "mv_top_advertisers_by_iata.name" - go_type: {type: "string", pointer: true}