dandri/beacon-server
1
0
Fork 0
mirror of https://github.com/MeshCore-Beacon/beacon-server.git synced 2026-10-16 03:00:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(observers): separate packet metrics from presence updates

Browse Source
This commit is contained in:
n30nex
2026-09-30 07:19:02 -04:00
committed by MrAlders0n
parent fd7c6b3eb7
commit 5fbba24a4a
31 changed files with 844 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/ci.yml
+1 -1
View File
@@ -55,7 +55,7 @@ jobs:
- name: Verify stats and endpoint queries against PostgreSQL 16
env:
BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable
run: go test ./db -run '^Test(Signal|Paths|PacketEndpointsResolveLive|AnalyticsRetention|AnalyticsRetentionConcurrent|DeleteOldPacketsBatches)Postgres$' -count=1 -v
run: go test ./db -run '^Test(Signal|Paths|PacketEndpointsResolveLive|ObserverMetrics|AnalyticsRetention|AnalyticsRetentionConcurrent|DeleteOldPacketsBatches)Postgres$' -count=1 -v
- name: Verify backup command against PostgreSQL 16
run: |
README.md
+18
View File
@@ -550,3 +550,21 @@ sub-hour observer activity, exact observer comparison and current entity/scope
counts continue to describe retained raw data or current entities, rather than
claiming archived packet detail. Archived summaries expire without waiting for
new packet deletions. A failed archive leaves its entire raw batch intact.
### Observer monitoring metrics
Observer activity returns complete buckets in `[windowStart, windowEnd)`, plus
`generatedAt`, `source` (`raw` or `hourly`) and `summary`. `recordedPackets` is the
sum of stored observations in those buckets; repeated broker delivery of the
same retained packet/observer pair counts once. Unknown payload types appear as
`-1` rather than disappearing. `lastCompleteHour` uses the previous complete UTC
hour and includes its own start/end; `latestRecordedAt` is the latest retained
reception timestamp. Missing records do not prove downtime. The optional `until`
(epoch milliseconds, within the last 30 days) aligns two observers' charts.
The existing observer `observationCount` remains a legacy cumulative presence
counter for compatibility, including status/neighbour events. It is not a
period packet total. Broker presence and packet-arrival timestamps are now
updated separately; this cannot reconstruct previously overwritten timestamps.
Migration 040 repairs archived unknown-type counts from the all-payload observer
rollup; radio samples already discarded for those legacy rows remain unknown.
db/migrations/040_observer_unknown_activity.sql
+279
View File
@@ -0,0 +1,279 @@
-- Copyright 2026 Beacon Contributors
-- SPDX-License-Identifier: AGPL-3.0-or-later
-- Unknown legacy payloads are still recorded packets. Restore their archived
-- counts from the all-payload observer rollup; their discarded radio samples
-- cannot be recovered. Subtract all existing activity (including prior repairs)
-- so migration-journal retries do not double-count or erase newer samples.
INSERT INTO analytics_observer_activity_hourly
(observer_id,payload_type,bucket,observations,airtime_n,snr_n,rssi_n)
SELECT totals.observer_id, -1, totals.bucket, totals.n-COALESCE(activity.n,0),0,0,0
FROM (SELECT observer_id,bucket,SUM(observation_count)::bigint AS n
FROM analytics_top_observers_by_iata GROUP BY observer_id,bucket) totals
LEFT JOIN (SELECT observer_id,bucket,SUM(observations)::bigint AS n
FROM analytics_observer_activity_hourly GROUP BY observer_id,bucket) activity
USING (observer_id,bucket)
WHERE totals.n>COALESCE(activity.n,0)
ON CONFLICT (observer_id,payload_type,bucket) DO UPDATE
SET observations=analytics_observer_activity_hourly.observations+EXCLUDED.observations;
DROP MATERIALIZED VIEW mv_observer_activity_hourly;
CREATE OR REPLACE VIEW analytics_live_observer_activity_hourly AS
SELECT
observer_id,
COALESCE(payload_type, -1)::smallint AS payload_type,
date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket,
COUNT(*)::bigint AS observations,
SUM(airtime_ms)::real AS airtime_ms,
COUNT(airtime_ms)::bigint AS airtime_n,
SUM(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_sum,
COUNT(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS snr_n,
MIN(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_min,
SUM(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_sum,
COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_n
FROM packet_observations
WHERE heard_at > NOW() - INTERVAL '30 days'
GROUP BY observer_id, COALESCE(payload_type, -1), date_trunc('hour', heard_at, 'UTC');
CREATE MATERIALIZED VIEW mv_observer_activity_hourly AS
WITH combined AS (
SELECT observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n FROM analytics_live_observer_activity_hourly
UNION ALL
SELECT observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n FROM analytics_observer_activity_hourly
WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'
)
SELECT observer_id, payload_type, bucket, SUM(observations)::bigint AS observations, SUM(airtime_ms)::real AS airtime_ms, SUM(airtime_n)::bigint AS airtime_n, SUM(snr_sum)::real AS snr_sum, SUM(snr_n)::bigint AS snr_n, MIN(snr_min)::real AS snr_min, SUM(rssi_sum)::bigint AS rssi_sum, SUM(rssi_n)::bigint AS rssi_n
FROM combined GROUP BY observer_id,payload_type,bucket;
CREATE UNIQUE INDEX idx_analytics_observer_activity_hourly_view ON mv_observer_activity_hourly (observer_id,payload_type,bucket);
-- Preserve the bounded atomic archive, including unknown payload activity.
CREATE OR REPLACE FUNCTION archive_delete_packets(cutoff timestamptz, batch_size integer)
RETURNS bigint LANGUAGE plpgsql VOLATILE AS $$
DECLARE hashes bytea[]; deleted bigint;
BEGIN
IF batch_size < 1 THEN RAISE EXCEPTION 'batch_size must be positive'; END IF;
SELECT array_agg(p.packet_hash) INTO hashes FROM (
SELECT ep.packet_hash FROM packets ep
WHERE ep.last_heard_at < cutoff
ORDER BY ep.last_heard_at, ep.packet_hash
LIMIT batch_size FOR UPDATE OF ep SKIP LOCKED
) p;
WITH expired_observations AS MATERIALIZED (
SELECT po.* FROM packet_observations po WHERE po.packet_hash = ANY(hashes)
),
archived_hourly_iata_stats AS (
INSERT INTO analytics_hourly_iata_stats (iata, hour, observation_count, unique_packets)
SELECT iata, hour, observation_count, unique_packets FROM (
SELECT
iata,
date_trunc('hour', heard_at, 'UTC')::timestamptz AS hour,
COUNT(*) AS observation_count,
COUNT(DISTINCT packet_hash) AS unique_packets
FROM expired_observations
WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'
GROUP BY iata, date_trunc('hour', heard_at, 'UTC')
) batch
ON CONFLICT (iata,hour) DO UPDATE SET
observation_count = analytics_hourly_iata_stats.observation_count + EXCLUDED.observation_count,
unique_packets = analytics_hourly_iata_stats.unique_packets + EXCLUDED.unique_packets
),
archived_payload_breakdown_by_iata AS (
INSERT INTO analytics_payload_breakdown_by_iata (iata, payload_type, bucket, count)
SELECT iata, payload_type, bucket, count FROM (
SELECT
iata,
payload_type,
date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket,
COUNT(*) AS count
FROM expired_observations
WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'
AND payload_type IS NOT NULL
GROUP BY iata, payload_type, date_trunc('hour', heard_at, 'UTC')
) batch
ON CONFLICT (iata,payload_type,bucket) DO UPDATE SET
count = analytics_payload_breakdown_by_iata.count + EXCLUDED.count
),
archived_top_observers_by_iata AS (
INSERT INTO analytics_top_observers_by_iata (iata, observer_id, bucket, observation_count, display_name, observer_type)
SELECT iata, observer_id, bucket, observation_count, display_name, observer_type FROM (
SELECT
po.iata,
po.observer_id,
o.display_name,
o.observer_type,
date_trunc('hour', po.heard_at, 'UTC')::timestamptz AS bucket,
COUNT(*) AS observation_count
FROM expired_observations po
JOIN observers o ON o.id = po.observer_id
WHERE po.heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'
GROUP BY po.iata, po.observer_id, o.display_name, o.observer_type, date_trunc('hour', po.heard_at, 'UTC')
) batch
ON CONFLICT (iata,observer_id,bucket) DO UPDATE SET
observation_count = analytics_top_observers_by_iata.observation_count + EXCLUDED.observation_count,
display_name = EXCLUDED.display_name,
observer_type = EXCLUDED.observer_type
),
archived_top_talkers_by_iata AS (
INSERT INTO analytics_top_talkers_by_iata (iata, sender_name, bucket, message_count, last_sent)
SELECT iata, sender_name, bucket, message_count, last_sent FROM (
SELECT
po.iata,
cm.sender_name,
date_trunc('hour', cm.sent_at, 'UTC')::timestamptz AS bucket,
COUNT(DISTINCT cm.id) AS message_count,
MAX(cm.sent_at) AS last_sent
FROM channel_messages cm
JOIN expired_observations po ON po.packet_hash = cm.packet_hash
WHERE cm.sender_name IS NOT NULL
AND cm.sent_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'
GROUP BY po.iata, cm.sender_name, date_trunc('hour', cm.sent_at, 'UTC')
) batch
ON CONFLICT (iata,sender_name,bucket) DO UPDATE SET
message_count = analytics_top_talkers_by_iata.message_count + EXCLUDED.message_count,
last_sent = GREATEST(analytics_top_talkers_by_iata.last_sent, EXCLUDED.last_sent)
),
archived_top_advertisers_by_iata AS (
INSERT INTO analytics_top_advertisers_by_iata (iata, node_id, bucket, advert_count, flood_advert_count, direct_advert_count, last_heard, name, node_type)
SELECT iata, node_id, bucket, advert_count, flood_advert_count, direct_advert_count, last_heard, name, node_type FROM (
SELECT
po.iata,
n.id AS node_id,
n.name,
n.node_type,
date_trunc('hour', po.heard_at, 'UTC')::timestamptz AS bucket,
COUNT(DISTINCT p.packet_hash) AS advert_count,
COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (0, 1)) AS flood_advert_count,
COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (2, 3)) AS direct_advert_count,
MAX(po.heard_at) AS last_heard
FROM packets p
JOIN expired_observations po ON po.packet_hash = p.packet_hash
JOIN nodes n ON n.public_key = p.origin_pubkey
WHERE p.payload_type = 4 -- ADVERT
AND po.heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'
GROUP BY po.iata, n.id, n.name, n.node_type, date_trunc('hour', po.heard_at, 'UTC')
) batch
ON CONFLICT (iata,node_id,bucket) DO UPDATE SET
advert_count = analytics_top_advertisers_by_iata.advert_count + EXCLUDED.advert_count,
flood_advert_count = analytics_top_advertisers_by_iata.flood_advert_count + EXCLUDED.flood_advert_count,
direct_advert_count = analytics_top_advertisers_by_iata.direct_advert_count + EXCLUDED.direct_advert_count,
last_heard = GREATEST(analytics_top_advertisers_by_iata.last_heard, EXCLUDED.last_heard),
name = EXCLUDED.name,
node_type = EXCLUDED.node_type
),
archived_observer_activity_hourly AS (
INSERT INTO analytics_observer_activity_hourly (observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n)
SELECT observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n FROM (
SELECT
observer_id,
COALESCE(payload_type, -1)::smallint AS payload_type,
date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket,
COUNT(*)::bigint AS observations,
SUM(airtime_ms)::real AS airtime_ms,
COUNT(airtime_ms)::bigint AS airtime_n,
SUM(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_sum,
COUNT(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS snr_n,
MIN(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_min,
SUM(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_sum,
COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_n
FROM expired_observations
WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'
GROUP BY observer_id, COALESCE(payload_type, -1), date_trunc('hour', heard_at, 'UTC')
) batch
ON CONFLICT (observer_id,payload_type,bucket) DO UPDATE SET
observations = analytics_observer_activity_hourly.observations + EXCLUDED.observations,
airtime_ms = CASE WHEN analytics_observer_activity_hourly.airtime_ms IS NULL AND EXCLUDED.airtime_ms IS NULL THEN NULL ELSE COALESCE(analytics_observer_activity_hourly.airtime_ms, 0) + COALESCE(EXCLUDED.airtime_ms, 0) END,
airtime_n = analytics_observer_activity_hourly.airtime_n + EXCLUDED.airtime_n,
snr_sum = CASE WHEN analytics_observer_activity_hourly.snr_sum IS NULL AND EXCLUDED.snr_sum IS NULL THEN NULL ELSE COALESCE(analytics_observer_activity_hourly.snr_sum, 0) + COALESCE(EXCLUDED.snr_sum, 0) END,
snr_n = analytics_observer_activity_hourly.snr_n + EXCLUDED.snr_n,
snr_min = LEAST(analytics_observer_activity_hourly.snr_min, EXCLUDED.snr_min),
rssi_sum = CASE WHEN analytics_observer_activity_hourly.rssi_sum IS NULL AND EXCLUDED.rssi_sum IS NULL THEN NULL ELSE COALESCE(analytics_observer_activity_hourly.rssi_sum, 0) + COALESCE(EXCLUDED.rssi_sum, 0) END,
rssi_n = analytics_observer_activity_hourly.rssi_n + EXCLUDED.rssi_n
),
archived_signal_stats_hourly AS (
INSERT INTO analytics_signal_stats_hourly (iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum)
SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM (
WITH samples AS (
SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour,
CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0)
AND snr > '-Infinity'::real AND snr < 'Infinity'::real
THEN snr::double precision END AS snr,
CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0)
THEN rssi::double precision END AS rssi
FROM expired_observations
WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours'
), binned AS (
SELECT *, width_bucket(snr, -30, 30, 12) AS snr_bin,
width_bucket(rssi, -140, 0, 14) AS rssi_bin
FROM samples
)
SELECT iata, hour, grouping(snr_bin, rssi_bin)::integer AS kind,
COALESCE(snr_bin, -1)::integer AS snr_bin,
COALESCE(rssi_bin, -1)::integer AS rssi_bin,
count(*)::bigint AS receptions,
count(snr)::bigint AS snr_samples,
COALESCE(sum(snr), 0)::double precision AS snr_sum,
count(rssi)::bigint AS rssi_samples,
COALESCE(sum(rssi), 0)::double precision AS rssi_sum
FROM binned
GROUP BY GROUPING SETS ((iata, hour), (iata, hour, snr_bin), (iata, hour, rssi_bin))
) batch
ON CONFLICT (iata,hour,kind,snr_bin,rssi_bin) DO UPDATE SET
receptions = analytics_signal_stats_hourly.receptions + EXCLUDED.receptions,
snr_samples = analytics_signal_stats_hourly.snr_samples + EXCLUDED.snr_samples,
snr_sum = analytics_signal_stats_hourly.snr_sum + EXCLUDED.snr_sum,
rssi_samples = analytics_signal_stats_hourly.rssi_samples + EXCLUDED.rssi_samples,
rssi_sum = analytics_signal_stats_hourly.rssi_sum + EXCLUDED.rssi_sum
),
archived_path_stats_hourly AS (
INSERT INTO analytics_path_stats_hourly (iata, hour, category, hash_bytes, entries, receptions)
SELECT iata, hour, category, hash_bytes, entries, receptions FROM (
WITH classified AS (
SELECT iata, heard_at, hash_size, hop_count,
CASE WHEN payload_type = 9 THEN 2
-- Match meshcore-go IsValidPathLen (1/2/3-byte hashes, max 64 path bytes).
WHEN payload_type IS NULL OR payload_type NOT BETWEEN 0 AND 15
OR NOT (path_length_byte BETWEEN 0 AND 191
AND hash_size BETWEEN 1 AND 3 AND hop_count BETWEEN 0 AND 63
AND hash_size = (path_length_byte >> 6) + 1
AND hop_count = (path_length_byte & 63)
AND hash_size::integer * hop_count::integer <= 64
AND COALESCE(octet_length(path_bytes), 0) = hash_size::integer * hop_count::integer)
THEN 3
WHEN hop_count = 0 THEN 1
ELSE 0 END::integer AS category
FROM expired_observations
WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours'
), buckets AS (
SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, category,
CASE WHEN category = 0 THEN hash_size ELSE 0 END::integer AS hash_bytes,
CASE WHEN category = 0 THEN hop_count ELSE 0 END::integer AS entries
FROM classified
)
SELECT iata, hour, category, hash_bytes, entries, count(*)::bigint AS receptions
FROM buckets GROUP BY iata, hour, category, hash_bytes, entries
) batch
ON CONFLICT (iata,hour,category,hash_bytes,entries) DO UPDATE SET
receptions = analytics_path_stats_hourly.receptions + EXCLUDED.receptions
)
DELETE FROM packets WHERE packet_hash = ANY(hashes);
GET DIAGNOSTICS deleted = ROW_COUNT;
-- Once per completed cleanup, including when there were no raw packets to delete.
IF deleted < batch_size THEN
DELETE FROM analytics_hourly_iata_stats WHERE hour < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days';
DELETE FROM analytics_payload_breakdown_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days';
DELETE FROM analytics_top_observers_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days';
DELETE FROM analytics_top_talkers_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days';
DELETE FROM analytics_top_advertisers_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days';
DELETE FROM analytics_observer_activity_hourly WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days';
DELETE FROM analytics_signal_stats_hourly WHERE hour < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days';
DELETE FROM analytics_path_stats_hourly WHERE hour < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days';
END IF;
RETURN deleted;
END $$;
db/observer_metrics_integration_test.go
+87
View File
@@ -0,0 +1,87 @@
// Copyright 2026 Beacon Contributors
// SPDX-License-Identifier: AGPL-3.0-or-later
package db
import (
sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgtype"
"strings"
"testing"
"time"
)
func TestObserverMetricsPostgres(t *testing.T) {
ctx, tx := retentionTx(t)
analyticsTables(t, ctx, tx)
// Use the real observer DDL without depending on a preinstalled public schema.
initial, err := migrationFiles.ReadFile("migrations/001_initial_schema.sql")
if err != nil {
t.Fatal(err)
}
ddl := string(initial)
start := strings.Index(ddl, "CREATE TABLE observers (")
end := strings.Index(ddl[start:], "\n);") + start + 3
if _, err := tx.Exec(ctx, "DROP TABLE observers; "+ddl[start:end]+" ALTER TABLE observers ADD COLUMN region_scope TEXT;"); err != nil {
t.Fatal(err)
}
if _, err := tx.Exec(ctx, `
CREATE TABLE observer_brokers (observer_id uuid,broker_name text,last_seen timestamptz,last_packet_at timestamptz,PRIMARY KEY(observer_id,broker_name));
INSERT INTO observers (id,public_key) VALUES ('00000000-0000-0000-0000-000000000001','\x01');
INSERT INTO packets(packet_hash,payload_type,last_heard_at,first_heard_at) SELECT int4send(i),4,NOW()-interval '5 days',NOW()-interval '5 days' FROM generate_series(1,3) i;
INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,payload_type,snr,rssi)
SELECT packet_hash,'00000000-0000-0000-0000-000000000001','YOW',date_trunc('hour',NOW())-interval '5 days',0,1,0,CASE WHEN packet_hash=int4send(1) THEN 4 END,0,-100 FROM packets;
`); err != nil {
t.Fatal(err)
}
applyStatsMigration(t, ctx, tx, "039_analytics_retention.sql")
store := &Store{q: sqlc.New(tx)}
id := uuid.MustParse("00000000-0000-0000-0000-000000000001")
if err := store.DeleteOldPackets(ctx, time.Now().Add(-72*time.Hour)); err != nil {
t.Fatal(err)
}
// Migration repairs the two unknown counts that 039 did not archive.
applyStatsMigration(t, ctx, tx, "040_observer_unknown_activity.sql")
applyStatsMigration(t, ctx, tx, "040_observer_unknown_activity.sql")
until := time.Now().UTC().Truncate(time.Hour)
activity, err := store.GetObserverActivity(ctx, id, 7*24*time.Hour, time.Hour, until)
if err != nil {
t.Fatal(err)
}
if activity.Summary.RecordedPackets != 3 || len(activity.PayloadTypes) != 2 || activity.Summary.LatestRecordedAt != nil || activity.Summary.LastCompleteHour != 0 {
t.Fatalf("archive summary: %+v types=%+v", activity.Summary, activity.PayloadTypes)
}
if activity.WindowEnd != until.UnixMilli() || activity.WindowStart != until.Add(-7*24*time.Hour).UnixMilli() || activity.Source != "hourly" {
t.Fatalf("window: %+v", activity)
}
if _, err := tx.Exec(ctx, `INSERT INTO packets(packet_hash,last_heard_at) VALUES ('\xaa',NOW());
INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,snr,rssi) VALUES ('\xaa','00000000-0000-0000-0000-000000000001','YOW',date_trunc('hour',NOW())-interval '30 minutes',0,1,0,0,-100);
INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at) VALUES ('\xaa','00000000-0000-0000-0000-000000000001','YOW',NOW()) ON CONFLICT DO NOTHING;`); err != nil {
t.Fatal(err)
}
raw, err := store.GetObserverActivity(ctx, id, time.Hour, 15*time.Minute, until)
if err != nil {
t.Fatal(err)
}
if raw.Summary.RecordedPackets != 1 || raw.Summary.LastCompleteHour != 1 || raw.Summary.LatestRecordedAt == nil || raw.PayloadTypes[0].PayloadType != -1 {
t.Fatalf("raw/duplicate: %+v", raw)
}
if err := store.UpsertObserverBroker(ctx, id, "one", false); err != nil {
t.Fatal(err)
}
var packet pgtype.Timestamptz
tx.QueryRow(ctx, "SELECT last_packet_at FROM observer_brokers").Scan(&packet)
if packet.Valid {
t.Fatal("status created a packet timestamp")
}
if err := store.UpsertObserverBroker(ctx, id, "one", true); err != nil {
t.Fatal(err)
}
if err := store.TouchObserverBrokers(ctx, []uuid.UUID{id}, []string{"one"}, []time.Time{until.Add(time.Hour)}, []time.Time{{}}); err != nil {
t.Fatal(err)
}
tx.QueryRow(ctx, "SELECT last_packet_at FROM observer_brokers").Scan(&packet)
if !packet.Valid {
t.Fatal("status erased packet timestamp")
}
}
db/observer_retention_integration_test.go
+2 -2
View File
@@ -100,7 +100,7 @@ SELECT id,1 FROM observers WHERE display_name='expired';`)
if err != nil {
t.Fatal(err)
}
if err := coalescer.UpsertObserverBroker(ctx, oldID, "fixture"); err != nil {
if err := coalescer.UpsertObserverBroker(ctx, oldID, "fixture", true); err != nil {
t.Fatal(err)
}
// A future cutoff simulates this cached observer having aged past retention.
@@ -112,7 +112,7 @@ SELECT id,1 FROM observers WHERE display_name='expired';`)
if err != nil || newID == oldID {
t.Fatalf("returning observer reused a deleted ID: %v", err)
}
if err := coalescer.UpsertObserverBroker(ctx, newID, "fixture"); err != nil {
if err := coalescer.UpsertObserverBroker(ctx, newID, "fixture", true); err != nil {
t.Fatal(err)
}
var brokers int
db/observers.go
+45 -17
View File
@@ -230,12 +230,37 @@ func (s *Store) GetObserverTelemetryBucketed(ctx context.Context, observerID uui
// GetObserverActivity returns bucketed heard-activity for an observer over the trailing window.
// Buckets of an hour or coarser come from the hourly rollup; anything finer reads observations directly.
// Range and Interval are left empty for the handler to fill.
func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration) (*api.ObserverActivity, error) {
func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration, until time.Time) (*api.ObserverActivity, error) {
obs, err := s.q.GetObserverByID(ctx, observerID)
if err != nil {
return nil, err
}
activity := &api.ObserverActivity{}
now := time.Now().UTC()
if until.IsZero() {
until = now
}
until = until.UTC().Truncate(interval)
since := until.Add(-window)
if !since.Equal(since.Truncate(interval)) {
since = since.Truncate(interval).Add(interval)
}
activity := &api.ObserverActivity{WindowStart: since.UnixMilli(), WindowEnd: until.UnixMilli(), GeneratedAt: now.UnixMilli(), Source: "raw", Summary: &api.ObserverActivitySummary{}}
if interval >= time.Hour {
activity.Source = "hourly"
}
live, err := s.q.GetObserverActivityLiveSummary(ctx, sqlc.GetObserverActivityLiveSummaryParams{
ObserverID: observerID, GeneratedAt: pgtype.Timestamptz{Time: now, Valid: true},
HourStart: pgtype.Timestamptz{Time: now.Truncate(time.Hour).Add(-time.Hour), Valid: true}, HourEnd: pgtype.Timestamptz{Time: now.Truncate(time.Hour), Valid: true}})
if err != nil {
return nil, err
}
activity.Summary.LastCompleteHour = live.LastCompleteHour
if live.LatestRecordedAt.Valid {
v := live.LatestRecordedAt.Time.UnixMilli()
activity.Summary.LatestRecordedAt = &v
}
activity.Summary.LastCompleteHourStart = now.Truncate(time.Hour).Add(-time.Hour).UnixMilli()
activity.Summary.LastCompleteHourEnd = now.Truncate(time.Hour).UnixMilli()
// radio is non-nil only when airtime is actually costable, so radio != null implies costed buckets
if obs.RadioSf != nil && obs.RadioBwKhz != nil && obs.RadioCr != nil &&
*obs.RadioSf >= 7 && *obs.RadioSf <= 12 && *obs.RadioBwKhz > 0 && *obs.RadioCr > 0 {
@@ -247,12 +272,6 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w
PreambleSymbols: lora.PreambleSymbols(int(*obs.RadioSf)),
}
}
// round the window start up to a bucket boundary so the first bucket is never a partial one
start := time.Now().Add(-window).UTC()
since := start.Truncate(interval)
if since.Before(start) {
since = since.Add(interval)
}
sinceTS := pgtype.Timestamptz{Time: since, Valid: true}
binWidth := pgtype.Interval{Microseconds: interval.Microseconds(), Valid: true}
@@ -261,6 +280,7 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w
ObserverID: observerID,
Column2: sinceTS,
Column3: binWidth,
Until: pgtype.Timestamptz{Time: until, Valid: true},
})
if err != nil {
return nil, err
@@ -268,6 +288,7 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w
activity.Points = make([]api.ObserverActivityPoint, 0, len(rows))
for _, r := range rows {
p := api.ObserverActivityPoint{T: r.Bucket.Time.UnixMilli(), Observations: r.Observations}
activity.Summary.RecordedPackets += r.Observations
if r.AirtimeN > 0 {
airtime := r.AirtimeMs
p.AirtimeMs = &airtime
@@ -286,18 +307,20 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w
typeRows, err := s.q.GetObserverActivityHourlyPayloadTypes(ctx, sqlc.GetObserverActivityHourlyPayloadTypesParams{
ObserverID: observerID,
Column2: sinceTS,
Until: pgtype.Timestamptz{Time: until, Valid: true},
})
if err != nil {
return nil, err
}
activity.PayloadTypes = make([]api.PayloadBreakdownItem, 0, len(typeRows))
for _, v := range typeRows {
if v.PayloadType == nil {
continue
payloadType := int16(-1)
if v.PayloadType != nil {
payloadType = *v.PayloadType
}
activity.PayloadTypes = append(activity.PayloadTypes, api.PayloadBreakdownItem{
PayloadType: *v.PayloadType,
PayloadTypeName: api.PayloadTypeName(*v.PayloadType),
PayloadType: payloadType,
PayloadTypeName: api.PayloadTypeName(payloadType),
Count: v.Count,
})
}
@@ -308,6 +331,7 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w
ObserverID: observerID,
Column2: sinceTS,
Column3: binWidth,
Until: pgtype.Timestamptz{Time: until, Valid: true},
})
if err != nil {
return nil, err
@@ -315,6 +339,7 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w
activity.Points = make([]api.ObserverActivityPoint, 0, len(rows))
for _, r := range rows {
p := api.ObserverActivityPoint{T: r.Bucket.Time.UnixMilli(), Observations: r.Observations}
activity.Summary.RecordedPackets += r.Observations
if r.AirtimeN > 0 {
airtime := r.AirtimeMs
p.AirtimeMs = &airtime
@@ -332,18 +357,20 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w
typeRows, err := s.q.GetObserverActivityRawPayloadTypes(ctx, sqlc.GetObserverActivityRawPayloadTypesParams{
ObserverID: observerID,
Column2: sinceTS,
Until: pgtype.Timestamptz{Time: until, Valid: true},
})
if err != nil {
return nil, err
}
activity.PayloadTypes = make([]api.PayloadBreakdownItem, 0, len(typeRows))
for _, v := range typeRows {
if v.PayloadType == nil {
continue
payloadType := int16(-1)
if v.PayloadType != nil {
payloadType = *v.PayloadType
}
activity.PayloadTypes = append(activity.PayloadTypes, api.PayloadBreakdownItem{
PayloadType: *v.PayloadType,
PayloadTypeName: api.PayloadTypeName(*v.PayloadType),
PayloadType: payloadType,
PayloadTypeName: api.PayloadTypeName(payloadType),
Count: v.Count,
})
}
@@ -424,10 +451,11 @@ func (s *Store) GetObserverRadio(ctx context.Context, observerID uuid.UUID) (ing
return settings, nil
}
func (s *Store) UpsertObserverBroker(ctx context.Context, observerID uuid.UUID, brokerName string) error {
func (s *Store) UpsertObserverBroker(ctx context.Context, observerID uuid.UUID, brokerName string, isPacket bool) error {
params := sqlc.UpsertObserverBrokerParams{
ObserverID: observerID,
BrokerName: brokerName,
IsPacket: isPacket,
}
return s.q.UpsertObserverBroker(ctx, params)
}
db/observers_test.go
+22 -13
View File
@@ -569,6 +569,7 @@ func f32(v float32) *float32 { return &v }
func TestGetObserverActivity_HourlyFoldNoSignal(t *testing.T) {
ctrl := gomock.NewController(t)
mock := mockdb.NewMockQuerier(ctrl)
mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes()
observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001")
mock.EXPECT().GetObserverByID(gomock.Any(), observerID).
@@ -592,7 +593,7 @@ func TestGetObserverActivity_HourlyFoldNoSignal(t *testing.T) {
}, nil)
store := &Store{q: mock}
got, err := store.GetObserverActivity(context.Background(), observerID, 24*time.Hour, time.Hour)
got, err := store.GetObserverActivity(context.Background(), observerID, 24*time.Hour, time.Hour, time.Time{})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -615,8 +616,8 @@ func TestGetObserverActivity_HourlyFoldNoSignal(t *testing.T) {
if p.RSSIAvg != nil {
t.Errorf("expected nil RSSIAvg, got %v", *p.RSSIAvg)
}
if len(got.PayloadTypes) != 1 || got.PayloadTypes[0].PayloadType != 4 || got.PayloadTypes[0].Count != 9 {
t.Fatalf("expected one payload type 4 with count 9, got %+v", got.PayloadTypes)
if len(got.PayloadTypes) != 2 || got.PayloadTypes[0].PayloadType != 4 || got.PayloadTypes[0].Count != 9 || got.PayloadTypes[1].PayloadType != -1 || got.PayloadTypes[1].Count != 3 {
t.Fatalf("expected known and unknown payloads to reconcile, got %+v", got.PayloadTypes)
}
if got.PayloadTypes[0].PayloadTypeName != api.PayloadTypeName(4) {
t.Errorf("expected payload type name %q, got %q", api.PayloadTypeName(4), got.PayloadTypes[0].PayloadTypeName)
@@ -638,6 +639,7 @@ func TestGetObserverActivity_HourlyFoldNoSignal(t *testing.T) {
func TestGetObserverActivity_HourlyFoldWeightedAverages(t *testing.T) {
ctrl := gomock.NewController(t)
mock := mockdb.NewMockQuerier(ctrl)
mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes()
observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001")
mock.EXPECT().GetObserverByID(gomock.Any(), observerID).
@@ -658,7 +660,7 @@ func TestGetObserverActivity_HourlyFoldWeightedAverages(t *testing.T) {
Return([]sqlc.GetObserverActivityHourlyPayloadTypesRow{}, nil)
store := &Store{q: mock}
got, err := store.GetObserverActivity(context.Background(), observerID, 7*24*time.Hour, 6*time.Hour)
got, err := store.GetObserverActivity(context.Background(), observerID, 7*24*time.Hour, 6*time.Hour, time.Time{})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -689,6 +691,7 @@ func TestGetObserverActivity_HourlyFoldWeightedAverages(t *testing.T) {
func TestGetObserverActivity_RawPath(t *testing.T) {
ctrl := gomock.NewController(t)
mock := mockdb.NewMockQuerier(ctrl)
mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes()
observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001")
mock.EXPECT().GetObserverByID(gomock.Any(), observerID).
@@ -709,7 +712,7 @@ func TestGetObserverActivity_RawPath(t *testing.T) {
Return([]sqlc.GetObserverActivityRawPayloadTypesRow{{PayloadType: i16(1), Count: 5}}, nil)
store := &Store{q: mock}
got, err := store.GetObserverActivity(context.Background(), observerID, 6*time.Hour, 15*time.Minute)
got, err := store.GetObserverActivity(context.Background(), observerID, 6*time.Hour, 15*time.Minute, time.Time{})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -737,6 +740,7 @@ func TestGetObserverActivity_RawPath(t *testing.T) {
func TestGetObserverActivity_RawFoldNoSignal(t *testing.T) {
ctrl := gomock.NewController(t)
mock := mockdb.NewMockQuerier(ctrl)
mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes()
observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001")
mock.EXPECT().GetObserverByID(gomock.Any(), observerID).
@@ -757,7 +761,7 @@ func TestGetObserverActivity_RawFoldNoSignal(t *testing.T) {
Return([]sqlc.GetObserverActivityRawPayloadTypesRow{}, nil)
store := &Store{q: mock}
got, err := store.GetObserverActivity(context.Background(), observerID, 6*time.Hour, 15*time.Minute)
got, err := store.GetObserverActivity(context.Background(), observerID, 6*time.Hour, 15*time.Minute, time.Time{})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -779,6 +783,7 @@ func TestGetObserverActivity_RawFoldNoSignal(t *testing.T) {
func TestGetObserverActivity_RawPathEmpty(t *testing.T) {
ctrl := gomock.NewController(t)
mock := mockdb.NewMockQuerier(ctrl)
mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes()
observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001")
mock.EXPECT().GetObserverByID(gomock.Any(), observerID).
@@ -789,7 +794,7 @@ func TestGetObserverActivity_RawPathEmpty(t *testing.T) {
Return([]sqlc.GetObserverActivityRawPayloadTypesRow{}, nil)
store := &Store{q: mock}
got, err := store.GetObserverActivity(context.Background(), observerID, 6*time.Hour, 15*time.Minute)
got, err := store.GetObserverActivity(context.Background(), observerID, 6*time.Hour, 15*time.Minute, time.Time{})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -804,6 +809,7 @@ func TestGetObserverActivity_RawPathEmpty(t *testing.T) {
func TestGetObserverActivity_HourlyPathAtOneHour(t *testing.T) {
ctrl := gomock.NewController(t)
mock := mockdb.NewMockQuerier(ctrl)
mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes()
observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001")
mock.EXPECT().GetObserverByID(gomock.Any(), observerID).
@@ -814,7 +820,7 @@ func TestGetObserverActivity_HourlyPathAtOneHour(t *testing.T) {
Return([]sqlc.GetObserverActivityHourlyPayloadTypesRow{}, nil)
store := &Store{q: mock}
got, err := store.GetObserverActivity(context.Background(), observerID, 24*time.Hour, time.Hour)
got, err := store.GetObserverActivity(context.Background(), observerID, 24*time.Hour, time.Hour, time.Time{})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -826,13 +832,14 @@ func TestGetObserverActivity_HourlyPathAtOneHour(t *testing.T) {
func TestGetObserverActivity_UnknownObserver(t *testing.T) {
ctrl := gomock.NewController(t)
mock := mockdb.NewMockQuerier(ctrl)
mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes()
observerID := uuid.MustParse("00000000-0000-0000-0000-000000000002")
mock.EXPECT().GetObserverByID(gomock.Any(), observerID).
Return(sqlc.Observer{}, pgx.ErrNoRows)
store := &Store{q: mock}
got, err := store.GetObserverActivity(context.Background(), observerID, time.Hour, time.Hour)
got, err := store.GetObserverActivity(context.Background(), observerID, time.Hour, time.Hour, time.Time{})
if !errors.Is(err, pgx.ErrNoRows) {
t.Fatalf("expected pgx.ErrNoRows, got %v", err)
}
@@ -858,6 +865,7 @@ func TestGetObserverActivity_RadioNilWhenIncomplete(t *testing.T) {
t.Run(tc.name, func(t *testing.T) {
ctrl := gomock.NewController(t)
mock := mockdb.NewMockQuerier(ctrl)
mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes()
observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001")
mock.EXPECT().GetObserverByID(gomock.Any(), observerID).
@@ -868,7 +876,7 @@ func TestGetObserverActivity_RadioNilWhenIncomplete(t *testing.T) {
Return([]sqlc.GetObserverActivityHourlyPayloadTypesRow{}, nil)
store := &Store{q: mock}
got, err := store.GetObserverActivity(context.Background(), observerID, 24*time.Hour, time.Hour)
got, err := store.GetObserverActivity(context.Background(), observerID, 24*time.Hour, time.Hour, time.Time{})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -882,6 +890,7 @@ func TestGetObserverActivity_RadioNilWhenIncomplete(t *testing.T) {
func TestGetObserverActivity_SinceAlignedToInterval(t *testing.T) {
ctrl := gomock.NewController(t)
mock := mockdb.NewMockQuerier(ctrl)
mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes()
observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001")
window, interval := 24*time.Hour, 6*time.Hour
@@ -897,12 +906,12 @@ func TestGetObserverActivity_SinceAlignedToInterval(t *testing.T) {
mock.EXPECT().GetObserverActivityHourlyPayloadTypes(gomock.Any(), gomock.Any()).
Return([]sqlc.GetObserverActivityHourlyPayloadTypesRow{}, nil)
before := time.Now().Add(-window)
before := time.Now().UTC().Truncate(interval).Add(-window)
store := &Store{q: mock}
if _, err := store.GetObserverActivity(context.Background(), observerID, window, interval); err != nil {
if _, err := store.GetObserverActivity(context.Background(), observerID, window, interval, time.Time{}); err != nil {
t.Fatalf("unexpected error: %v", err)
}
after := time.Now().Add(-window)
after := time.Now().UTC().Truncate(interval).Add(-window)
if gotSince.UnixNano()%int64(interval) != 0 {
t.Errorf("since %s is not aligned to %s", gotSince, interval)
db/presence.go
+10 -1
View File
@@ -20,11 +20,12 @@ func (s *Store) TouchObservers(ctx context.Context, ids []uuid.UUID, seen []time
})
}
func (s *Store) TouchObserverBrokers(ctx context.Context, ids []uuid.UUID, brokers []string, seen []time.Time) error {
func (s *Store) TouchObserverBrokers(ctx context.Context, ids []uuid.UUID, brokers []string, seen, packets []time.Time) error {
return s.q.TouchObserverBrokers(ctx, sqlc.TouchObserverBrokersParams{
Column1: ids,
Column2: brokers,
Column3: toTimestamptzs(seen),
Column4: nullableTimes(packets),
})
}
@@ -42,3 +43,11 @@ func toTimestamptzs(ts []time.Time) []pgtype.Timestamptz {
}
return out
}
func nullableTimes(ts []time.Time) []pgtype.Timestamptz {
out := make([]pgtype.Timestamptz, len(ts))
for i, t := range ts {
out[i] = pgtype.Timestamptz{Time: t, Valid: !t.IsZero()}
}
return out
}
db/queries/observer_activity_summary.sql
+13
View File
@@ -0,0 +1,13 @@
-- name: GetObserverActivityLiveSummary :one
-- Two indexed ranges, bounded to one observer; no legacy presence counters.
WITH latest AS (
SELECT heard_at FROM packet_observations
WHERE observer_id = @observer_id::uuid AND heard_at <= @generated_at::timestamptz
ORDER BY heard_at DESC LIMIT 1
), hourly AS (
SELECT COUNT(*)::bigint AS n FROM packet_observations
WHERE observer_id = @observer_id::uuid
AND heard_at >= @hour_start::timestamptz AND heard_at < @hour_end::timestamptz
)
SELECT (SELECT heard_at FROM latest)::timestamptz AS latest_recorded_at,
hourly.n AS last_complete_hour FROM hourly;
db/queries/queries.sql
+10 -9
View File
@@ -275,14 +275,14 @@ SELECT
COALESCE(AVG(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0)), 0)::real AS rssi_avg,
COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_n
FROM packet_observations
WHERE observer_id = $1 AND heard_at >= $2::timestamptz
WHERE observer_id = $1 AND heard_at >= $2::timestamptz AND heard_at < @until::timestamptz
GROUP BY bucket
ORDER BY bucket;
-- name: GetObserverActivityRawPayloadTypes :many
SELECT payload_type, COUNT(*)::bigint AS count
FROM packet_observations
WHERE observer_id = $1 AND heard_at >= $2::timestamptz AND payload_type IS NOT NULL
WHERE observer_id = $1 AND heard_at >= $2::timestamptz AND heard_at < @until::timestamptz
GROUP BY payload_type
ORDER BY count DESC;
@@ -299,14 +299,14 @@ SELECT
COALESCE(SUM(rssi_sum), 0)::bigint AS rssi_sum,
SUM(rssi_n)::bigint AS rssi_n
FROM mv_observer_activity_hourly
WHERE observer_id = $1 AND bucket >= $2::timestamptz
WHERE observer_id = $1 AND bucket >= $2::timestamptz AND bucket < @until::timestamptz
GROUP BY 1
ORDER BY 1;
-- name: GetObserverActivityHourlyPayloadTypes :many
SELECT payload_type, SUM(observations)::bigint AS count
FROM mv_observer_activity_hourly
WHERE observer_id = $1 AND bucket >= $2::timestamptz
WHERE observer_id = $1 AND bucket >= $2::timestamptz AND bucket < @until::timestamptz
GROUP BY payload_type
ORDER BY count DESC;
@@ -363,19 +363,20 @@ RETURNING o.id;
-- name: UpsertObserverBroker :exec
INSERT INTO observer_brokers (observer_id, broker_name, last_seen, last_packet_at)
VALUES ($1, $2, NOW(), NOW())
VALUES ($1, $2, NOW(), CASE WHEN @is_packet::boolean THEN NOW() END)
ON CONFLICT (observer_id, broker_name) DO UPDATE SET
last_seen = NOW(),
last_packet_at = NOW();
last_seen = NOW(),
last_packet_at = COALESCE(EXCLUDED.last_packet_at, observer_brokers.last_packet_at);
-- name: TouchObserverBrokers :exec
UPDATE observer_brokers ob SET
last_seen = GREATEST(ob.last_seen, v.seen),
last_packet_at = GREATEST(ob.last_packet_at, v.seen)
last_packet_at = GREATEST(ob.last_packet_at, v.packet)
FROM (
SELECT unnest($1::uuid[]) AS observer_id,
unnest($2::text[]) AS broker_name,
unnest($3::timestamptz[]) AS seen
unnest($3::timestamptz[]) AS seen,
unnest($4::timestamptz[]) AS packet
) v
WHERE ob.observer_id = v.observer_id AND ob.broker_name = v.broker_name;
db/sqlc/mock/querier.go
+16 -1
View File
@@ -3,7 +3,7 @@
//
// Generated by this command:
//
// mockgen -source=db/sqlc/querier.go -destination=db/sqlc/mock/querier.go -package=mockdb
// mockgen -source db/sqlc/querier.go -destination db/sqlc/mock/querier.go -package mockdb
//
// Package mockdb is a generated GoMock package.
@@ -384,6 +384,21 @@ func (mr *MockQuerierMockRecorder) GetObserverActivityHourlyPayloadTypes(ctx, ar
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetObserverActivityHourlyPayloadTypes", reflect.TypeOf((*MockQuerier)(nil).GetObserverActivityHourlyPayloadTypes), ctx, arg)
}
// GetObserverActivityLiveSummary mocks base method.
func (m *MockQuerier) GetObserverActivityLiveSummary(ctx context.Context, arg db.GetObserverActivityLiveSummaryParams) (db.GetObserverActivityLiveSummaryRow, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "GetObserverActivityLiveSummary", ctx, arg)
ret0, _ := ret[0].(db.GetObserverActivityLiveSummaryRow)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// GetObserverActivityLiveSummary indicates an expected call of GetObserverActivityLiveSummary.
func (mr *MockQuerierMockRecorder) GetObserverActivityLiveSummary(ctx, arg any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetObserverActivityLiveSummary", reflect.TypeOf((*MockQuerier)(nil).GetObserverActivityLiveSummary), ctx, arg)
}
// GetObserverActivityRaw mocks base method.
func (m *MockQuerier) GetObserverActivityRaw(ctx context.Context, arg db.GetObserverActivityRawParams) ([]db.GetObserverActivityRawRow, error) {
m.ctrl.T.Helper()
db/sqlc/observer_activity_summary.sql.go
+52
View File
@@ -0,0 +1,52 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.31.1
// source: observer_activity_summary.sql
package db
import (
"context"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgtype"
)
const getObserverActivityLiveSummary = `-- name: GetObserverActivityLiveSummary :one
WITH latest AS (
SELECT heard_at FROM packet_observations
WHERE observer_id = $1::uuid AND heard_at <= $2::timestamptz
ORDER BY heard_at DESC LIMIT 1
), hourly AS (
SELECT COUNT(*)::bigint AS n FROM packet_observations
WHERE observer_id = $1::uuid
AND heard_at >= $3::timestamptz AND heard_at < $4::timestamptz
)
SELECT (SELECT heard_at FROM latest)::timestamptz AS latest_recorded_at,
hourly.n AS last_complete_hour FROM hourly
`
type GetObserverActivityLiveSummaryParams struct {
ObserverID uuid.UUID `json:"observer_id"`
GeneratedAt pgtype.Timestamptz `json:"generated_at"`
HourStart pgtype.Timestamptz `json:"hour_start"`
HourEnd pgtype.Timestamptz `json:"hour_end"`
}
type GetObserverActivityLiveSummaryRow struct {
LatestRecordedAt pgtype.Timestamptz `json:"latest_recorded_at"`
LastCompleteHour int64 `json:"last_complete_hour"`
}
// Two indexed ranges, bounded to one observer; no legacy presence counters.
func (q *Queries) GetObserverActivityLiveSummary(ctx context.Context, arg GetObserverActivityLiveSummaryParams) (GetObserverActivityLiveSummaryRow, error) {
row := q.db.QueryRow(ctx, getObserverActivityLiveSummary,
arg.ObserverID,
arg.GeneratedAt,
arg.HourStart,
arg.HourEnd,
)
var i GetObserverActivityLiveSummaryRow
err := row.Scan(&i.LatestRecordedAt, &i.LastCompleteHour)
return i, err
}
db/sqlc/querier.go
+2
View File
@@ -58,6 +58,8 @@ type Querier interface {
// Hour-or-coarser buckets summed from the hourly rollup; same COALESCE-plus-count shape as the raw query.
GetObserverActivityHourly(ctx context.Context, arg GetObserverActivityHourlyParams) ([]GetObserverActivityHourlyRow, error)
GetObserverActivityHourlyPayloadTypes(ctx context.Context, arg GetObserverActivityHourlyPayloadTypesParams) ([]GetObserverActivityHourlyPayloadTypesRow, error)
// Two indexed ranges, bounded to one observer; no legacy presence counters.
GetObserverActivityLiveSummary(ctx context.Context, arg GetObserverActivityLiveSummaryParams) (GetObserverActivityLiveSummaryRow, error)
// Sub-hour activity buckets straight off idx_observations_observer; no join to packets.
// Aggregates are COALESCEd and paired with a count column: sqlc types a cast expression as
// NOT NULL, so the counts are what tell the store a bucket had no costed or no signal rows.
db/sqlc/queries.sql.go
+37 -15
View File
@@ -650,7 +650,7 @@ SELECT
COALESCE(SUM(rssi_sum), 0)::bigint AS rssi_sum,
SUM(rssi_n)::bigint AS rssi_n
FROM mv_observer_activity_hourly
WHERE observer_id = $1 AND bucket >= $2::timestamptz
WHERE observer_id = $1 AND bucket >= $2::timestamptz AND bucket < $4::timestamptz
GROUP BY 1
ORDER BY 1
`
@@ -659,6 +659,7 @@ type GetObserverActivityHourlyParams struct {
ObserverID uuid.UUID `json:"observer_id"`
Column2 pgtype.Timestamptz `json:"column_2"`
Column3 pgtype.Interval `json:"column_3"`
Until pgtype.Timestamptz `json:"until"`
}
type GetObserverActivityHourlyRow struct {
@@ -675,7 +676,12 @@ type GetObserverActivityHourlyRow struct {
// Hour-or-coarser buckets summed from the hourly rollup; same COALESCE-plus-count shape as the raw query.
func (q *Queries) GetObserverActivityHourly(ctx context.Context, arg GetObserverActivityHourlyParams) ([]GetObserverActivityHourlyRow, error) {
rows, err := q.db.Query(ctx, getObserverActivityHourly, arg.ObserverID, arg.Column2, arg.Column3)
rows, err := q.db.Query(ctx, getObserverActivityHourly,
arg.ObserverID,
arg.Column2,
arg.Column3,
arg.Until,
)
if err != nil {
return nil, err
}
@@ -707,7 +713,7 @@ func (q *Queries) GetObserverActivityHourly(ctx context.Context, arg GetObserver
const getObserverActivityHourlyPayloadTypes = `-- name: GetObserverActivityHourlyPayloadTypes :many
SELECT payload_type, SUM(observations)::bigint AS count
FROM mv_observer_activity_hourly
WHERE observer_id = $1 AND bucket >= $2::timestamptz
WHERE observer_id = $1 AND bucket >= $2::timestamptz AND bucket < $3::timestamptz
GROUP BY payload_type
ORDER BY count DESC
`
@@ -715,6 +721,7 @@ ORDER BY count DESC
type GetObserverActivityHourlyPayloadTypesParams struct {
ObserverID uuid.UUID `json:"observer_id"`
Column2 pgtype.Timestamptz `json:"column_2"`
Until pgtype.Timestamptz `json:"until"`
}
type GetObserverActivityHourlyPayloadTypesRow struct {
@@ -723,7 +730,7 @@ type GetObserverActivityHourlyPayloadTypesRow struct {
}
func (q *Queries) GetObserverActivityHourlyPayloadTypes(ctx context.Context, arg GetObserverActivityHourlyPayloadTypesParams) ([]GetObserverActivityHourlyPayloadTypesRow, error) {
rows, err := q.db.Query(ctx, getObserverActivityHourlyPayloadTypes, arg.ObserverID, arg.Column2)
rows, err := q.db.Query(ctx, getObserverActivityHourlyPayloadTypes, arg.ObserverID, arg.Column2, arg.Until)
if err != nil {
return nil, err
}
@@ -754,7 +761,7 @@ SELECT
COALESCE(AVG(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0)), 0)::real AS rssi_avg,
COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_n
FROM packet_observations
WHERE observer_id = $1 AND heard_at >= $2::timestamptz
WHERE observer_id = $1 AND heard_at >= $2::timestamptz AND heard_at < $4::timestamptz
GROUP BY bucket
ORDER BY bucket
`
@@ -763,6 +770,7 @@ type GetObserverActivityRawParams struct {
ObserverID uuid.UUID `json:"observer_id"`
Column2 pgtype.Timestamptz `json:"column_2"`
Column3 pgtype.Interval `json:"column_3"`
Until pgtype.Timestamptz `json:"until"`
}
type GetObserverActivityRawRow struct {
@@ -781,7 +789,12 @@ type GetObserverActivityRawRow struct {
// Aggregates are COALESCEd and paired with a count column: sqlc types a cast expression as
// NOT NULL, so the counts are what tell the store a bucket had no costed or no signal rows.
func (q *Queries) GetObserverActivityRaw(ctx context.Context, arg GetObserverActivityRawParams) ([]GetObserverActivityRawRow, error) {
rows, err := q.db.Query(ctx, getObserverActivityRaw, arg.ObserverID, arg.Column2, arg.Column3)
rows, err := q.db.Query(ctx, getObserverActivityRaw,
arg.ObserverID,
arg.Column2,
arg.Column3,
arg.Until,
)
if err != nil {
return nil, err
}
@@ -813,7 +826,7 @@ func (q *Queries) GetObserverActivityRaw(ctx context.Context, arg GetObserverAct
const getObserverActivityRawPayloadTypes = `-- name: GetObserverActivityRawPayloadTypes :many
SELECT payload_type, COUNT(*)::bigint AS count
FROM packet_observations
WHERE observer_id = $1 AND heard_at >= $2::timestamptz AND payload_type IS NOT NULL
WHERE observer_id = $1 AND heard_at >= $2::timestamptz AND heard_at < $3::timestamptz
GROUP BY payload_type
ORDER BY count DESC
`
@@ -821,6 +834,7 @@ ORDER BY count DESC
type GetObserverActivityRawPayloadTypesParams struct {
ObserverID uuid.UUID `json:"observer_id"`
Column2 pgtype.Timestamptz `json:"column_2"`
Until pgtype.Timestamptz `json:"until"`
}
type GetObserverActivityRawPayloadTypesRow struct {
@@ -829,7 +843,7 @@ type GetObserverActivityRawPayloadTypesRow struct {
}
func (q *Queries) GetObserverActivityRawPayloadTypes(ctx context.Context, arg GetObserverActivityRawPayloadTypesParams) ([]GetObserverActivityRawPayloadTypesRow, error) {
rows, err := q.db.Query(ctx, getObserverActivityRawPayloadTypes, arg.ObserverID, arg.Column2)
rows, err := q.db.Query(ctx, getObserverActivityRawPayloadTypes, arg.ObserverID, arg.Column2, arg.Until)
if err != nil {
return nil, err
}
@@ -4422,11 +4436,12 @@ func (q *Queries) SetPacketDecrypted(ctx context.Context, packetHash []byte) err
const touchObserverBrokers = `-- name: TouchObserverBrokers :exec
UPDATE observer_brokers ob SET
last_seen = GREATEST(ob.last_seen, v.seen),
last_packet_at = GREATEST(ob.last_packet_at, v.seen)
last_packet_at = GREATEST(ob.last_packet_at, v.packet)
FROM (
SELECT unnest($1::uuid[]) AS observer_id,
unnest($2::text[]) AS broker_name,
unnest($3::timestamptz[]) AS seen
unnest($3::timestamptz[]) AS seen,
unnest($4::timestamptz[]) AS packet
) v
WHERE ob.observer_id = v.observer_id AND ob.broker_name = v.broker_name
`
@@ -4435,10 +4450,16 @@ type TouchObserverBrokersParams struct {
Column1 []uuid.UUID `json:"column_1"`
Column2 []string `json:"column_2"`
Column3 []pgtype.Timestamptz `json:"column_3"`
Column4 []pgtype.Timestamptz `json:"column_4"`
}
func (q *Queries) TouchObserverBrokers(ctx context.Context, arg TouchObserverBrokersParams) error {
_, err := q.db.Exec(ctx, touchObserverBrokers, arg.Column1, arg.Column2, arg.Column3)
_, err := q.db.Exec(ctx, touchObserverBrokers,
arg.Column1,
arg.Column2,
arg.Column3,
arg.Column4,
)
return err
}
@@ -4943,22 +4964,23 @@ func (q *Queries) UpsertObserver(ctx context.Context, publicKey []byte) (Observe
const upsertObserverBroker = `-- name: UpsertObserverBroker :exec
INSERT INTO observer_brokers (observer_id, broker_name, last_seen, last_packet_at)
VALUES ($1, $2, NOW(), NOW())
VALUES ($1, $2, NOW(), CASE WHEN $3::boolean THEN NOW() END)
ON CONFLICT (observer_id, broker_name) DO UPDATE SET
last_seen = NOW(),
last_packet_at = NOW()
last_seen = NOW(),
last_packet_at = COALESCE(EXCLUDED.last_packet_at, observer_brokers.last_packet_at)
`
type UpsertObserverBrokerParams struct {
ObserverID uuid.UUID `json:"observer_id"`
BrokerName string `json:"broker_name"`
IsPacket bool `json:"is_packet"`
}
// ============================================================
// OBSERVER BROKERS
// ============================================================
func (q *Queries) UpsertObserverBroker(ctx context.Context, arg UpsertObserverBrokerParams) error {
_, err := q.db.Exec(ctx, upsertObserverBroker, arg.ObserverID, arg.BrokerName)
_, err := q.db.Exec(ctx, upsertObserverBroker, arg.ObserverID, arg.BrokerName, arg.IsPacket)
return err
}
docs/docs.go
+46
View File
@@ -1468,6 +1468,12 @@ const docTemplate = `{
"description": "Bucket size: 5m, 15m, 1h, 6h or 24h (default 15m)",
"name": "interval",
"in": "query"
},
{
"type": "integer",
"description": "Optional exclusive end in epoch milliseconds; aligned down to a complete bucket, at most 30 days old",
"name": "until",
"in": "query"
}
],
"responses": {
@@ -3924,6 +3930,7 @@ const docTemplate = `{
"type": "integer"
},
"observationCount": {
"description": "legacy cumulative presence counter; includes non-packet events",
"type": "integer"
},
"observerType": {
@@ -3979,6 +3986,10 @@ const docTemplate = `{
"github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivity": {
"type": "object",
"properties": {
"generatedAt": {
"description": "response computation time, not proof of continuous coverage",
"type": "integer"
},
"interval": {
"type": "string"
},
@@ -3999,6 +4010,21 @@ const docTemplate = `{
},
"range": {
"type": "string"
},
"source": {
"description": "raw or hourly; missing records do not prove an outage",
"type": "string"
},
"summary": {
"$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivitySummary"
},
"windowEnd": {
"description": "exclusive complete-bucket end, epoch ms",
"type": "integer"
},
"windowStart": {
"description": "inclusive complete-bucket start, epoch ms",
"type": "integer"
}
}
},
@@ -4046,6 +4072,26 @@ const docTemplate = `{
}
}
},
"github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivitySummary": {
"type": "object",
"properties": {
"lastCompleteHour": {
"type": "integer"
},
"lastCompleteHourEnd": {
"type": "integer"
},
"lastCompleteHourStart": {
"type": "integer"
},
"latestRecordedAt": {
"type": "integer"
},
"recordedPackets": {
"type": "integer"
}
}
},
"github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverBroker": {
"type": "object",
"properties": {
docs/swagger.json
+46
View File
@@ -1466,6 +1466,12 @@
"description": "Bucket size: 5m, 15m, 1h, 6h or 24h (default 15m)",
"name": "interval",
"in": "query"
},
{
"type": "integer",
"description": "Optional exclusive end in epoch milliseconds; aligned down to a complete bucket, at most 30 days old",
"name": "until",
"in": "query"
}
],
"responses": {
@@ -3922,6 +3928,7 @@
"type": "integer"
},
"observationCount": {
"description": "legacy cumulative presence counter; includes non-packet events",
"type": "integer"
},
"observerType": {
@@ -3977,6 +3984,10 @@
"github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivity": {
"type": "object",
"properties": {
"generatedAt": {
"description": "response computation time, not proof of continuous coverage",
"type": "integer"
},
"interval": {
"type": "string"
},
@@ -3997,6 +4008,21 @@
},
"range": {
"type": "string"
},
"source": {
"description": "raw or hourly; missing records do not prove an outage",
"type": "string"
},
"summary": {
"$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivitySummary"
},
"windowEnd": {
"description": "exclusive complete-bucket end, epoch ms",
"type": "integer"
},
"windowStart": {
"description": "inclusive complete-bucket start, epoch ms",
"type": "integer"
}
}
},
@@ -4044,6 +4070,26 @@
}
}
},
"github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivitySummary": {
"type": "object",
"properties": {
"lastCompleteHour": {
"type": "integer"
},
"lastCompleteHourEnd": {
"type": "integer"
},
"lastCompleteHourStart": {
"type": "integer"
},
"latestRecordedAt": {
"type": "integer"
},
"recordedPackets": {
"type": "integer"
}
}
},
"github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverBroker": {
"type": "object",
"properties": {
docs/swagger.yaml
+33
View File
@@ -521,6 +521,7 @@ definitions:
description: epoch ms
type: integer
observationCount:
description: legacy cumulative presence counter; includes non-packet events
type: integer
observerType:
description: e.g. "meshcoretomqtt", "meshcoreha"
@@ -560,6 +561,9 @@ definitions:
type: object
github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivity:
properties:
generatedAt:
description: response computation time, not proof of continuous coverage
type: integer
interval:
type: string
payloadTypes:
@@ -574,6 +578,17 @@ definitions:
$ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivityRadio'
range:
type: string
source:
description: raw or hourly; missing records do not prove an outage
type: string
summary:
$ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivitySummary'
windowEnd:
description: exclusive complete-bucket end, epoch ms
type: integer
windowStart:
description: inclusive complete-bucket start, epoch ms
type: integer
type: object
github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivityPoint:
properties:
@@ -604,6 +619,19 @@ definitions:
sf:
type: integer
type: object
github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivitySummary:
properties:
lastCompleteHour:
type: integer
lastCompleteHourEnd:
type: integer
lastCompleteHourStart:
type: integer
latestRecordedAt:
type: integer
recordedPackets:
type: integer
type: object
github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverBroker:
properties:
lastPacketAt:
@@ -2401,6 +2429,11 @@ paths:
in: query
name: interval
type: string
- description: Optional exclusive end in epoch milliseconds; aligned down to
a complete bucket, at most 30 days old
in: query
name: until
type: integer
produces:
- application/json
responses:
internal/api/handlers/observers.go
+12 -1
View File
@@ -256,6 +256,7 @@ var activityIntervals = map[string]time.Duration{
// @Param observerId path string true "Observer UUID"
// @Param range query string false "Trailing window as a Go duration, max 720h (default 24h); max 48h when interval is under 1h"
// @Param interval query string false "Bucket size: 5m, 15m, 1h, 6h or 24h (default 15m)"
// @Param until query int false "Optional exclusive end in epoch milliseconds; aligned down to a complete bucket, at most 30 days old"
// @Success 200 {object} api.ObserverActivity
// @Failure 400 {object} handlers.APIError
// @Failure 404 {object} handlers.APIError
@@ -295,7 +296,17 @@ func getObserverActivity(reader api.Reader) http.HandlerFunc {
respondError(w, http.StatusBadRequest, "range/interval exceeds 1000 buckets")
return
}
activity, err := reader.GetObserverActivity(r.Context(), observerID, window, interval)
until := time.Time{}
if values, ok := r.URL.Query()["until"]; ok {
n, err := strconv.ParseInt(values[0], 10, 64)
now := time.Now()
if len(values) != 1 || err != nil || n < 0 || n > now.UnixMilli() || n < now.Add(-720*time.Hour).UnixMilli() {
respondError(w, http.StatusBadRequest, "until must be one past epoch-millisecond timestamp within 30 days")
return
}
until = time.UnixMilli(n)
}
activity, err := reader.GetObserverActivity(r.Context(), observerID, window, interval, until)
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
respondError(w, http.StatusNotFound, "observer not found")
internal/api/handlers/observers_test.go
+12 -8
View File
@@ -200,7 +200,7 @@ func TestGetObserverActivity_Defaults(t *testing.T) {
var gotWindow, gotInterval time.Duration
r := chi.NewRouter()
r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{
getObserverActivity: func(_ context.Context, _ uuid.UUID, window, interval time.Duration) (*api.ObserverActivity, error) {
getObserverActivity: func(_ context.Context, _ uuid.UUID, window, interval time.Duration, _ time.Time) (*api.ObserverActivity, error) {
gotWindow, gotInterval = window, interval
return &api.ObserverActivity{Points: []api.ObserverActivityPoint{}}, nil
},
@@ -232,7 +232,7 @@ func TestGetObserverActivity_CustomRangeAndInterval(t *testing.T) {
var gotWindow, gotInterval time.Duration
r := chi.NewRouter()
r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{
getObserverActivity: func(_ context.Context, _ uuid.UUID, window, interval time.Duration) (*api.ObserverActivity, error) {
getObserverActivity: func(_ context.Context, _ uuid.UUID, window, interval time.Duration, _ time.Time) (*api.ObserverActivity, error) {
gotWindow, gotInterval = window, interval
return &api.ObserverActivity{Points: []api.ObserverActivityPoint{}}, nil
},
@@ -264,7 +264,7 @@ func TestGetObserverActivity_SubHourRangeLimit(t *testing.T) {
r := chi.NewRouter()
r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{
getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration) (*api.ObserverActivity, error) {
getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration, _ time.Time) (*api.ObserverActivity, error) {
t.Fatal("reader should not be called")
return nil, nil
},
@@ -286,7 +286,7 @@ func TestGetObserverActivity_SubHourRangeLimit(t *testing.T) {
var gotWindow, gotInterval time.Duration
ok := chi.NewRouter()
ok.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{
getObserverActivity: func(_ context.Context, _ uuid.UUID, window, interval time.Duration) (*api.ObserverActivity, error) {
getObserverActivity: func(_ context.Context, _ uuid.UUID, window, interval time.Duration, _ time.Time) (*api.ObserverActivity, error) {
gotWindow, gotInterval = window, interval
return &api.ObserverActivity{Points: []api.ObserverActivityPoint{}}, nil
},
@@ -309,6 +309,10 @@ func TestGetObserverActivity_BadRequests(t *testing.T) {
query string
}{
{"invalid uuid", "not-a-uuid", ""},
{"bad end", "00000000-0000-0000-0000-000000000001", "?until=banana"},
{"negative end", "00000000-0000-0000-0000-000000000001", "?until=-1"},
{"duplicate end", "00000000-0000-0000-0000-000000000001", "?until=1&until=2"},
{"future end", "00000000-0000-0000-0000-000000000001", "?until=253402300799999"},
{"unparseable range", "00000000-0000-0000-0000-000000000001", "?range=banana"},
{"range too long", "00000000-0000-0000-0000-000000000001", "?range=721h"},
{"zero range", "00000000-0000-0000-0000-000000000001", "?range=0"},
@@ -319,7 +323,7 @@ func TestGetObserverActivity_BadRequests(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
r := chi.NewRouter()
r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{
getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration) (*api.ObserverActivity, error) {
getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration, _ time.Time) (*api.ObserverActivity, error) {
t.Fatal("reader should not be called")
return nil, nil
},
@@ -338,7 +342,7 @@ func TestGetObserverActivity_NotFound(t *testing.T) {
observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001")
r := chi.NewRouter()
r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{
getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration) (*api.ObserverActivity, error) {
getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration, _ time.Time) (*api.ObserverActivity, error) {
return nil, pgx.ErrNoRows
},
}))
@@ -364,7 +368,7 @@ func TestGetObserverActivity_ReaderError(t *testing.T) {
observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001")
r := chi.NewRouter()
r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{
getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration) (*api.ObserverActivity, error) {
getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration, _ time.Time) (*api.ObserverActivity, error) {
return nil, errors.New("boom")
},
}))
@@ -381,7 +385,7 @@ func TestGetObserverActivity_NullsSerialise(t *testing.T) {
observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001")
r := chi.NewRouter()
r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{
getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration) (*api.ObserverActivity, error) {
getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration, _ time.Time) (*api.ObserverActivity, error) {
return &api.ObserverActivity{
PayloadTypes: []api.PayloadBreakdownItem{},
Points: []api.ObserverActivityPoint{{T: 1757376000000, Observations: 3}},
internal/api/handlers/stub_reader_test.go
+3 -3
View File
@@ -32,7 +32,7 @@ type stubReader struct {
getObserver func(ctx context.Context, observerID uuid.UUID) (*api.Observer, error)
getObserverTelemetry func(ctx context.Context, observerID uuid.UUID, since, until time.Time, afterID int64) (*api.ObserverTelemetry, error)
getObserverTelemetryBucketed func(ctx context.Context, observerID uuid.UUID, since, until time.Time, bucketHours int32) ([]api.ObserverTelemetryPoint, error)
getObserverActivity func(ctx context.Context, observerID uuid.UUID, window, interval time.Duration) (*api.ObserverActivity, error)
getObserverActivity func(ctx context.Context, observerID uuid.UUID, window, interval time.Duration, until time.Time) (*api.ObserverActivity, error)
getObserverScopes func(ctx context.Context, observerID uuid.UUID) ([]string, error)
listObserverAdverts func(ctx context.Context, observerID uuid.UUID, cursor int64, limit int32) (api.Page[api.AdvertObservation], error)
listNodes func(ctx context.Context, nodeType int16, iatas []string, supportsMultibytePaths, supportsMultibyteTraces *bool, pubkey []byte, pubkeyPrefix, name, scope string, cursor int64, limit int32, includeNeighbors bool) (api.Page[api.NodeSummary], error)
@@ -186,9 +186,9 @@ func (s stubReader) GetObserverTelemetryBucketed(ctx context.Context, observerID
return nil, nil
}
func (s stubReader) GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration) (*api.ObserverActivity, error) {
func (s stubReader) GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration, until time.Time) (*api.ObserverActivity, error) {
if s.getObserverActivity != nil {
return s.getObserverActivity(ctx, observerID, window, interval)
return s.getObserverActivity(ctx, observerID, window, interval, until)
}
return nil, nil
}
internal/api/observers.go
+17 -2
View File
@@ -44,8 +44,8 @@ type Observer struct {
LastStatusAt *int64 `json:"lastStatusAt,omitempty"` // epoch ms
FirstSeen int64 `json:"firstSeen"` // epoch ms
LastSeen int64 `json:"lastSeen"` // epoch ms
ObservationCount int64 `json:"observationCount"`
Brokers []ObserverBroker `json:"brokers"` // broker names this observer has been seen on
ObservationCount int64 `json:"observationCount"` // legacy cumulative presence counter; includes non-packet events
Brokers []ObserverBroker `json:"brokers"` // broker names this observer has been seen on
}
// ObserverTelemetryPoint is a single telemetry snapshot for an observer.
@@ -89,7 +89,22 @@ type ObserverActivityPoint struct {
}
// ObserverActivity is the per-observer heard-activity response.
// ObserverActivitySummary describes stored packet records, never MQTT presence events.
type ObserverActivitySummary struct {
RecordedPackets int64 `json:"recordedPackets"`
LastCompleteHour int64 `json:"lastCompleteHour"`
LastCompleteHourStart int64 `json:"lastCompleteHourStart"`
LastCompleteHourEnd int64 `json:"lastCompleteHourEnd"`
LatestRecordedAt *int64 `json:"latestRecordedAt"`
}
type ObserverActivity struct {
WindowStart int64 `json:"windowStart"` // inclusive complete-bucket start, epoch ms
WindowEnd int64 `json:"windowEnd"` // exclusive complete-bucket end, epoch ms
GeneratedAt int64 `json:"generatedAt"` // response computation time, not proof of continuous coverage
Source string `json:"source"` // raw or hourly; missing records do not prove an outage
Summary *ObserverActivitySummary `json:"summary,omitempty"`
Range string `json:"range"`
Interval string `json:"interval"`
Radio *ObserverActivityRadio `json:"radio"`
internal/api/reader.go
+1 -1
View File
@@ -100,7 +100,7 @@ type Reader interface {
// GetObserverActivity returns bucketed heard-activity for an observer over the trailing window.
// interval >= 1h is served from the hourly rollup. Returns pgx.ErrNoRows for an unknown observer.
// Range and Interval on the result are left empty for the handler to fill.
GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration) (*ObserverActivity, error)
GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration, until time.Time) (*ObserverActivity, error)
// GetObserverScopes returns the names of all transport scopes an observer has
// been seen forwarding packets for, ordered alphabetically.
internal/cache/cache_test.go
Vendored
+1 -1
View File
@@ -148,7 +148,7 @@ func (s *stubReader) GetObserverTelemetryBucketed(_ context.Context, _ uuid.UUID
return nil, nil
}
func (s *stubReader) GetObserverActivity(_ context.Context, _ uuid.UUID, _, _ time.Duration) (*api.ObserverActivity, error) {
func (s *stubReader) GetObserverActivity(_ context.Context, _ uuid.UUID, _, _ time.Duration, _ time.Time) (*api.ObserverActivity, error) {
return nil, nil
}
internal/cache/reader.go
Vendored
+3 -3
View File
@@ -372,10 +372,10 @@ func (cr *CachedReader) GetObserverTelemetryBucketed(ctx context.Context, observ
}
// GetObserverActivity implements [api.Reader].
func (cr *CachedReader) GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration) (*api.ObserverActivity, error) {
key := keyObserverActivityPrefix + observerID.String() + ":" + window.String() + ":" + interval.String()
func (cr *CachedReader) GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration, until time.Time) (*api.ObserverActivity, error) {
key := keyObserverActivityPrefix + observerID.String() + ":" + window.String() + ":" + interval.String() + ":" + until.UTC().Format(time.RFC3339Nano)
return getOrSet(ctx, cr.c, key, observerActivityTTL, func() (*api.ObserverActivity, error) {
return cr.inner.GetObserverActivity(ctx, observerID, window, interval)
return cr.inner.GetObserverActivity(ctx, observerID, window, interval, until)
})
}
internal/ingest/ingest.go
+1 -1
View File
@@ -89,7 +89,7 @@ type DB interface {
UpsertObserver(ctx context.Context, pubkey []byte) (uuid.UUID, string, error)
// UpsertObserverBroker records that this observer was seen on brokerName.
UpsertObserverBroker(ctx context.Context, observerID uuid.UUID, brokerName string) error
UpsertObserverBroker(ctx context.Context, observerID uuid.UUID, brokerName string, isPacket bool) error
// UpsertIATA auto-creates an iata_codes row if it doesn't exist yet.
UpsertIATA(ctx context.Context, iata string) error
internal/ingest/ingest_test.go
+4 -2
View File
@@ -201,8 +201,10 @@ func (s *stubDB) SetNodeCapability(_ context.Context, nodeID uuid.UUID, paths, t
func (s *stubDB) UpsertObserver(_ context.Context, _ []byte) (uuid.UUID, string, error) {
return uuid.Nil, "", nil
}
func (s *stubDB) UpsertObserverBroker(_ context.Context, _ uuid.UUID, _ string) error { return nil }
func (s *stubDB) UpsertIATA(_ context.Context, _ string) error { return nil }
func (s *stubDB) UpsertObserverBroker(_ context.Context, _ uuid.UUID, _ string, _ bool) error {
return nil
}
func (s *stubDB) UpsertIATA(_ context.Context, _ string) error { return nil }
func (s *stubDB) UpsertPacket(_ context.Context, _ UpsertPacketParams) (bool, error) {
return false, nil
}
internal/ingest/neighbors.go
+1 -1
View File
@@ -48,7 +48,7 @@ func (w *Worker) handleNeighbors(ctx context.Context, iata, pubkeyHex string, ra
w.log.Error(fmt.Sprintf("db: upsert observer failed in neighbors from %s", pubkeyHex), "error", err)
return
}
if err := w.db.UpsertObserverBroker(ctx, observerID, w.cfg.BrokerName); err != nil {
if err := w.db.UpsertObserverBroker(ctx, observerID, w.cfg.BrokerName, false); err != nil {
w.log.Error(fmt.Sprintf("db: upsert observer broker failed in neighbors from %s", pubkeyHex), "error", err)
}
internal/ingest/packet.go
+1 -1
View File
@@ -324,7 +324,7 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [
w.log.Error(fmt.Sprintf("db: upsert observer failed with packet from %s/%s", iata, pubkeyHex), "error", err)
return
}
err = w.db.UpsertObserverBroker(ctx, id, w.cfg.BrokerName)
err = w.db.UpsertObserverBroker(ctx, id, w.cfg.BrokerName, true)
if err != nil {
w.log.Error(fmt.Sprintf("db: update observer broker failed with packet from %s/%s", iata, pubkeyHex), "error", err)
return
internal/ingest/status.go
+1 -1
View File
@@ -87,7 +87,7 @@ func (w *Worker) handleStatus(ctx context.Context, pubkeyHex string, raw []byte)
if w.onObserverUpsert != nil {
w.onObserverUpsert(ctx, id)
}
if err := w.db.UpsertObserverBroker(ctx, id, w.cfg.BrokerName); err != nil {
if err := w.db.UpsertObserverBroker(ctx, id, w.cfg.BrokerName, false); err != nil {
w.log.Error(fmt.Sprintf("db: upsert observer broker failed in status from %s", pubkeyHex), "error", err)
}
params := UpdateObserverStatusParams{
internal/presence/broker_freshness_test.go
+50
View File
@@ -0,0 +1,50 @@
// Copyright 2026 Beacon Contributors
// SPDX-License-Identifier: AGPL-3.0-or-later
package presence
import (
"context"
"github.com/google/uuid"
"testing"
"time"
)
type brokerTestStore struct {
Store
initial []bool
seen, packets []time.Time
}
func (s *brokerTestStore) UpsertObserverBroker(_ context.Context, _ uuid.UUID, _ string, packet bool) error {
s.initial = append(s.initial, packet)
return nil
}
func (s *brokerTestStore) TouchObserverBrokers(_ context.Context, _ []uuid.UUID, _ []string, seen, packets []time.Time) error {
s.seen = seen
s.packets = packets
return nil
}
func TestBrokerPresenceDoesNotInventPacketFreshness(t *testing.T) {
ctx := context.Background()
store := &brokerTestStore{}
c := New(store, time.Second, time.Minute)
now := time.Now()
c.now = func() time.Time { return now }
id := uuid.New()
c.UpsertObserverBroker(ctx, id, "one", false)
c.UpsertObserverBroker(ctx, id, "one", false)
c.Flush(ctx)
if len(store.initial) != 1 || store.initial[0] || !store.packets[0].IsZero() {
t.Fatal("status-only activity invented a packet")
}
now = now.Add(time.Minute)
packetTime := now
c.UpsertObserverBroker(ctx, id, "one", true)
now = now.Add(time.Minute)
c.UpsertObserverBroker(ctx, id, "one", false)
c.Flush(ctx)
if !store.seen[0].Equal(now) || !store.packets[0].Equal(packetTime) {
t.Fatal("later status changed packet freshness")
}
}
internal/presence/coalescer.go
+18 -9
View File
@@ -31,7 +31,7 @@ type Store interface {
// TouchObserverBrokers applies coalesced last_seen/last_packet_at bumps
// for the given (observer, broker) pairs in one statement.
TouchObserverBrokers(ctx context.Context, ids []uuid.UUID, brokers []string, seen []time.Time) error
TouchObserverBrokers(ctx context.Context, ids []uuid.UUID, brokers []string, seen, packets []time.Time) error
// TouchPackets applies coalesced last_heard_at bumps for the given packet
// hashes in one statement.
@@ -51,6 +51,8 @@ type observerBump struct {
count int32
}
type brokerBump struct{ seen, packet time.Time }
type brokerKey struct {
id uuid.UUID
broker string
@@ -68,7 +70,7 @@ type Coalescer struct {
identities map[string]identity // pubkey -> observer row
dirtyObservers map[uuid.UUID]observerBump
knownBrokers map[brokerKey]struct{}
dirtyBrokers map[brokerKey]time.Time
dirtyBrokers map[brokerKey]brokerBump
knownIATAs map[string]struct{}
packetsSeen map[string]time.Time // hash -> last observation
dirtyPackets map[string]time.Time
@@ -86,7 +88,7 @@ func New(store Store, flushInterval, packetTTL time.Duration) *Coalescer {
identities: make(map[string]identity),
dirtyObservers: make(map[uuid.UUID]observerBump),
knownBrokers: make(map[brokerKey]struct{}),
dirtyBrokers: make(map[brokerKey]time.Time),
dirtyBrokers: make(map[brokerKey]brokerBump),
knownIATAs: make(map[string]struct{}),
packetsSeen: make(map[string]time.Time),
dirtyPackets: make(map[string]time.Time),
@@ -122,17 +124,22 @@ func (c *Coalescer) UpsertObserver(ctx context.Context, pubkey []byte) (uuid.UUI
// UpsertObserverBroker writes through the first time a pair is seen and
// records a bump afterwards.
func (c *Coalescer) UpsertObserverBroker(ctx context.Context, observerID uuid.UUID, brokerName string) error {
func (c *Coalescer) UpsertObserverBroker(ctx context.Context, observerID uuid.UUID, brokerName string, isPacket bool) error {
key := brokerKey{id: observerID, broker: brokerName}
c.mu.Lock()
if _, ok := c.knownBrokers[key]; ok {
c.dirtyBrokers[key] = c.now()
bump := c.dirtyBrokers[key]
bump.seen = c.now()
if isPacket {
bump.packet = bump.seen
}
c.dirtyBrokers[key] = bump
c.mu.Unlock()
return nil
}
c.mu.Unlock()
if err := c.Store.UpsertObserverBroker(ctx, observerID, brokerName); err != nil {
if err := c.Store.UpsertObserverBroker(ctx, observerID, brokerName, isPacket); err != nil {
return err
}
c.mu.Lock()
@@ -248,7 +255,7 @@ func (c *Coalescer) Flush(ctx context.Context) {
brokers := c.dirtyBrokers
packets := c.dirtyPackets
c.dirtyObservers = make(map[uuid.UUID]observerBump)
c.dirtyBrokers = make(map[brokerKey]time.Time)
c.dirtyBrokers = make(map[brokerKey]brokerBump)
c.dirtyPackets = make(map[string]time.Time)
cutoff := c.now().Add(-c.packetTTL)
for key, seen := range c.packetsSeen {
@@ -266,12 +273,14 @@ func (c *Coalescer) Flush(ctx context.Context) {
ids := make([]uuid.UUID, 0, len(brokers))
names := make([]string, 0, len(brokers))
seen := make([]time.Time, 0, len(brokers))
packetTimes := make([]time.Time, 0, len(brokers))
for key, ts := range brokers {
ids = append(ids, key.id)
names = append(names, key.broker)
seen = append(seen, ts)
seen = append(seen, ts.seen)
packetTimes = append(packetTimes, ts.packet)
}
if err := c.Store.TouchObserverBrokers(ctx, ids, names, seen); err != nil {
if err := c.Store.TouchObserverBrokers(ctx, ids, names, seen, packetTimes); err != nil {
slog.Error(fmt.Sprintf("presence: flush observer brokers failed (%d rows dropped)", len(ids)), "component", "presence", "error", err)
}
}