perf: known_routes retention, digest identity, and a working batched reconfirm (#98)

* fix: cascade channel_messages on packet delete

* rebuild known_routes keyed on (iata, path_key)

* add route retention delete, batch reconfirm, path_key upsert

* compute route path_key in store, add DeleteOldRoutes

* add routes retention config

* wire route retention into cleanup, batch reconfirm

* prune routes in reconfirm task, not cleanup

* expand routes retention docs in example config
This commit is contained in:
MrAlders0n
2026-08-12 09:30:03 -04:00
committed by GitHub
parent c6d23a2b52
commit 7d759d0a0f
13 changed files with 473 additions and 118 deletions
+1 -1
View File
@@ -258,7 +258,7 @@ func main() {
scheduler := background.New([]background.Task{
background.ViewRefreshTask(store, resolved.ViewRefreshInterval),
background.CleanupTask(store, resolved.TelemetryRetention, resolved.PacketRetention, resolved.NodeDeleteAfter, resolved.CleanupInterval),
background.ReconfirmTask(store, resolved.ReconfirmInterval),
background.ReconfirmTask(store, resolved.RouteRetention, resolved.RouteGrace, int64(resolved.RouteMinObservations), resolved.ReconfirmInterval),
})
go scheduler.Start(ctx)
+19
View File
@@ -75,6 +75,25 @@ telemetry:
packets:
retention: 720h # 30 days
# Known-route retention. Routes are distilled packet history (the path a packet
# took, hop by hop), so they may outlive packets.retention; there is no required
# relationship between the two.
routes:
# How long a route is kept after it was last observed. Every new observation
# of the same path resets this clock, so routes the mesh still uses never
# expire -- only paths nothing has traveled for this long age out.
retention: 336h # 14 days
# Shorter window for rarely-seen routes: one observed fewer than
# min_observations times in total is dropped once it goes unobserved for
# this long. Must be shorter than retention, or it never applies.
grace: 168h # 7 days
# Lifetime observation count a route must reach to earn the full retention
# window. Below it a route is treated as flood noise -- a path recorded once
# or twice and never confirmed -- and only kept for the grace window. The
# count never resets, so once a route crosses this bar it stays in the
# retention tier for good.
min_observations: 3
websocket:
max_connections_per_ip: 5 # default: 5
@@ -0,0 +1,43 @@
-- Copyright 2026 Beacon Contributors
-- SPDX-License-Identifier: AGPL-3.0-or-later
-- known_routes grew unbounded (28 GB / 32.8M rows in 24 days on prod) and its
-- UNIQUE(node_ids, iata) key indexed whole UUID arrays. Rebuild keyed on a
-- 16-byte md5 of node_ids and add reconfirm bookkeeping. No rows are pruned
-- here: retention is deployment config (routes.retention/grace), so the first
-- cleanup tick after startup enforces it; a schema migration must not bake in
-- one deployment's policy. id stays (the API serializes it) but loses its
-- index; uniqueness lives on (iata, path_key).
CREATE TABLE known_routes_new (
id BIGINT GENERATED BY DEFAULT AS IDENTITY NOT NULL,
path_key BYTEA NOT NULL,
node_ids UUID[] NOT NULL,
hash_prefix BYTEA[] NOT NULL,
iata CHAR(3) NOT NULL REFERENCES iata_codes(iata) ON DELETE CASCADE,
hop_count INT NOT NULL,
first_seen TIMESTAMPTZ NOT NULL DEFAULT NOW(),
last_seen TIMESTAMPTZ NOT NULL DEFAULT NOW(),
observation_count BIGINT NOT NULL DEFAULT 1,
last_reconfirmed_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
PRIMARY KEY (iata, path_key)
);
INSERT INTO known_routes_new
(id, path_key, node_ids, hash_prefix, iata, hop_count, first_seen, last_seen, observation_count)
SELECT id,
decode(md5(array_to_string(node_ids, ',')), 'hex'),
node_ids, hash_prefix, iata, hop_count, first_seen, last_seen, observation_count
FROM known_routes;
SELECT setval(pg_get_serial_sequence('known_routes_new', 'id'),
(SELECT COALESCE(MAX(id), 1) FROM known_routes_new));
DROP TABLE known_routes;
ALTER TABLE known_routes_new RENAME TO known_routes;
ALTER TABLE known_routes RENAME CONSTRAINT known_routes_new_pkey TO known_routes_pkey;
ALTER TABLE known_routes RENAME CONSTRAINT known_routes_new_iata_fkey TO known_routes_iata_fkey;
CREATE INDEX idx_known_routes_hop_count ON known_routes(iata, hop_count);
CREATE INDEX idx_known_routes_last_seen ON known_routes(last_seen DESC);
CREATE INDEX idx_known_routes_reconfirm ON known_routes(last_reconfirmed_at);
+59 -25
View File
@@ -533,6 +533,13 @@ DELETE FROM nodes
WHERE last_seen < $1
AND id NOT IN (SELECT owner_node_id FROM observer_owners WHERE owner_node_id IS NOT NULL);
-- name: DeleteOldRoutes :exec
-- Deletes routes not observed since the retention cutoff ($1), and rarely-observed
-- routes (observation_count < $2) not observed since the grace cutoff ($3).
DELETE FROM known_routes
WHERE last_seen < $1
OR (observation_count < $2 AND last_seen < $3);
-- name: DeleteOldChannelIATAs :exec
-- Keeps the channel IATA filter in step with packet retention.
DELETE FROM channel_iatas WHERE last_heard < $1;
@@ -1070,12 +1077,11 @@ ORDER BY t.last_heard_at DESC;
-- ============================================================
-- name: UpsertKnownRoute :exec
-- Inserts or updates a known route (all hops resolved to high confidence).
-- node_ids and hash_prefix are ordered arrays of the resolved node UUIDs and
-- their hash bytes. last_seen is bumped on conflict.
INSERT INTO known_routes (node_ids, hash_prefix, iata, hop_count)
VALUES ($1, $2, $3, $4)
ON CONFLICT (node_ids, iata) DO UPDATE SET
-- Route identity is path_key, an md5 of node_ids computed by the caller.
-- On conflict, observation_count and last_seen are bumped.
INSERT INTO known_routes (path_key, node_ids, hash_prefix, iata, hop_count)
VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (iata, path_key) DO UPDATE SET
last_seen = NOW(),
observation_count = known_routes.observation_count + 1;
@@ -1216,27 +1222,55 @@ REFRESH MATERIALIZED VIEW CONCURRENTLY mv_top_advertisers_by_iata;
REFRESH MATERIALIZED VIEW CONCURRENTLY mv_radio_presets;
-- name: ReconfirmRoutes :exec
-- Delete known_routes where any hop node has departed from node_short_ids for
-- that IATA, or where any hop's prefix_4 is now ambiguous (matches >1 node).
DELETE FROM known_routes kr
WHERE EXISTS (
SELECT 1
FROM unnest(kr.node_ids) AS hop_node_id
WHERE NOT EXISTS (
SELECT 1 FROM node_short_ids ns
WHERE ns.node_id = hop_node_id
AND ns.iata = kr.iata
-- Checks the $1 least-recently-reconfirmed routes: deletes those with a departed
-- hop node or a hop prefix now matching >1 node in that IATA (length-aware:
-- 1/2/3/4-byte hop prefixes check prefix_1/2/3/4), and stamps the survivors.
WITH batch AS (
SELECT iata, path_key, node_ids, hash_prefix
FROM known_routes
ORDER BY last_reconfirmed_at
LIMIT $1
),
amb AS MATERIALIZED (
SELECT iata, 1 AS len, prefix_1 AS p FROM node_short_ids GROUP BY iata, prefix_1 HAVING COUNT(*) > 1
UNION ALL
SELECT iata, 2, prefix_2 FROM node_short_ids GROUP BY iata, prefix_2 HAVING COUNT(*) > 1
UNION ALL
SELECT iata, 3, prefix_3 FROM node_short_ids GROUP BY iata, prefix_3 HAVING COUNT(*) > 1
UNION ALL
SELECT iata, 4, prefix_4 FROM node_short_ids GROUP BY iata, prefix_4 HAVING COUNT(*) > 1
),
dead AS (
SELECT b.iata, b.path_key
FROM batch b
WHERE EXISTS (
SELECT 1
FROM unnest(b.node_ids) AS hop_node_id
WHERE NOT EXISTS (
SELECT 1 FROM node_short_ids ns
WHERE ns.node_id = hop_node_id
AND ns.iata = b.iata
)
)
UNION
SELECT DISTINCT b.iata, b.path_key
FROM batch b
CROSS JOIN LATERAL unnest(b.hash_prefix) AS hp
JOIN amb a ON a.iata = b.iata AND a.len = length(hp) AND a.p = hp
),
deleted AS (
DELETE FROM known_routes kr
USING dead d
WHERE kr.iata = d.iata AND kr.path_key = d.path_key
)
OR EXISTS (
SELECT 1
FROM unnest(kr.hash_prefix) AS hop_prefix
WHERE (
SELECT COUNT(*) FROM node_short_ids ns
WHERE ns.iata = kr.iata
AND ns.prefix_4 = hop_prefix
) > 1
);
UPDATE known_routes kr
SET last_reconfirmed_at = NOW()
FROM batch b
WHERE kr.iata = b.iata AND kr.path_key = b.path_key
AND NOT EXISTS (
SELECT 1 FROM dead d
WHERE d.iata = b.iata AND d.path_key = b.path_key
);
-- name: ReconfirmNeighbors :exec
-- Delete node_neighbors where the neighbor has departed from node_short_ids
+59 -8
View File
@@ -5,7 +5,9 @@ package db
import (
"context"
"crypto/md5"
"encoding/hex"
"strings"
"time"
sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc"
@@ -14,12 +16,24 @@ import (
"github.com/jackc/pgx/v5/pgtype"
)
// routePathKey is the route's identity digest: md5 over the comma-joined
// node UUIDs, matching Postgres's decode(md5(array_to_string(node_ids, ',')), 'hex').
func routePathKey(nodeIDs []uuid.UUID) []byte {
parts := make([]string, len(nodeIDs))
for i, id := range nodeIDs {
parts[i] = id.String()
}
sum := md5.Sum([]byte(strings.Join(parts, ",")))
return sum[:]
}
func (s *Store) UpsertKnownRoute(ctx context.Context, nodeIDs []uuid.UUID, hashPrefix [][]byte, iata string, hopCount int32) error {
return s.q.UpsertKnownRoute(ctx, sqlc.UpsertKnownRouteParams{
PathKey: routePathKey(nodeIDs),
NodeIds: nodeIDs,
HashPrefix: hashPrefix,
Iata: iata,
HopCount: int32(hopCount),
HopCount: hopCount,
})
}
@@ -28,7 +42,7 @@ func (s *Store) ListKnownRoutes(ctx context.Context, iata string, hopCount int32
if !cursor.IsZero() {
cursorTS = pgtype.Timestamptz{Time: cursor, Valid: true}
}
rows, err := s.q.ListKnownRoutes(ctx, sqlc.ListKnownRoutesParams{
sqlRows, err := s.q.ListKnownRoutes(ctx, sqlc.ListKnownRoutesParams{
Column1: iata,
Column2: hopCount,
Column3: cursorTS,
@@ -37,6 +51,10 @@ func (s *Store) ListKnownRoutes(ctx context.Context, iata string, hopCount int32
if err != nil {
return nil, err
}
rows := make([]knownRouteRow, len(sqlRows))
for i, r := range sqlRows {
rows[i] = knownRouteRow{ID: r.ID, NodeIds: r.NodeIds, HashPrefix: r.HashPrefix, Iata: r.Iata, HopCount: r.HopCount, FirstSeen: r.FirstSeen, LastSeen: r.LastSeen, ObservationCount: r.ObservationCount}
}
ids := collectNodeIDs(rows)
nodes, err := s.GetNodesByIDs(ctx, ids)
if err != nil {
@@ -54,7 +72,7 @@ func (s *Store) SearchKnownRoutes(ctx context.Context, iata, fromHash, toHash st
if err != nil {
return nil, err
}
rows, err := s.q.SearchKnownRoutes(ctx, sqlc.SearchKnownRoutesParams{
sqlRows, err := s.q.SearchKnownRoutes(ctx, sqlc.SearchKnownRoutesParams{
Iata: iata,
Column2: fromBytes,
Column3: toBytes,
@@ -62,6 +80,10 @@ func (s *Store) SearchKnownRoutes(ctx context.Context, iata, fromHash, toHash st
if err != nil {
return nil, err
}
rows := make([]knownRouteRow, len(sqlRows))
for i, r := range sqlRows {
rows[i] = knownRouteRow{ID: r.ID, NodeIds: r.NodeIds, HashPrefix: r.HashPrefix, Iata: r.Iata, HopCount: r.HopCount, FirstSeen: r.FirstSeen, LastSeen: r.LastSeen, ObservationCount: r.ObservationCount}
}
ids := collectNodeIDs(rows)
nodes, err := s.GetNodesByIDs(ctx, ids)
if err != nil {
@@ -109,13 +131,17 @@ func (s *Store) SearchKnownRoutes(ctx context.Context, iata, fromHash, toHash st
}
func (s *Store) GetKnownRoutesByNode(ctx context.Context, iata string, nodeID uuid.UUID) ([]api.KnownRoute, error) {
rows, err := s.q.GetKnownRoutesByNode(ctx, sqlc.GetKnownRoutesByNodeParams{
sqlRows, err := s.q.GetKnownRoutesByNode(ctx, sqlc.GetKnownRoutesByNodeParams{
Iata: iata,
Column2: nodeID,
})
if err != nil {
return nil, err
}
rows := make([]knownRouteRow, len(sqlRows))
for i, r := range sqlRows {
rows[i] = knownRouteRow{ID: r.ID, NodeIds: r.NodeIds, HashPrefix: r.HashPrefix, Iata: r.Iata, HopCount: r.HopCount, FirstSeen: r.FirstSeen, LastSeen: r.LastSeen, ObservationCount: r.ObservationCount}
}
ids := collectNodeIDs(rows)
nodes, err := s.GetNodesByIDs(ctx, ids)
if err != nil {
@@ -260,8 +286,20 @@ func (s *Store) SearchCrossIATARoutes(ctx context.Context, fromHash, fromIATA, t
return results, nil
}
func (s *Store) ReconfirmRoutes(ctx context.Context) error {
return s.q.ReconfirmRoutes(ctx)
// ReconfirmRoutes checks the batchSize least-recently-reconfirmed routes,
// deleting stale or ambiguous ones and stamping the survivors.
func (s *Store) ReconfirmRoutes(ctx context.Context, batchSize int32) error {
return s.q.ReconfirmRoutes(ctx, batchSize)
}
// DeleteOldRoutes prunes routes per the retention rule: unconditionally past
// retentionCutoff, and past graceCutoff when observed fewer than minObservations times.
func (s *Store) DeleteOldRoutes(ctx context.Context, retentionCutoff time.Time, minObservations int64, graceCutoff time.Time) error {
return s.q.DeleteOldRoutes(ctx, sqlc.DeleteOldRoutesParams{
LastSeen: pgtype.Timestamptz{Time: retentionCutoff, Valid: true},
ObservationCount: minObservations,
LastSeen_2: pgtype.Timestamptz{Time: graceCutoff, Valid: true},
})
}
// extractFromNode returns the portion of a route starting at the given node.
@@ -274,7 +312,20 @@ func extractFromNode(hops []api.RouteHop, nodeID uuid.UUID) []api.RouteHop {
return hops
}
func toKnownRoutes(rows []sqlc.KnownRoute, nodes map[uuid.UUID]*api.ResolvedNode) []api.KnownRoute {
// knownRouteRow normalizes the per-query sqlc row structs (identical
// columns, distinct generated types) so the helpers below share one body.
type knownRouteRow struct {
ID int64
NodeIds []uuid.UUID
HashPrefix [][]byte
Iata string
HopCount int32
FirstSeen pgtype.Timestamptz
LastSeen pgtype.Timestamptz
ObservationCount int64
}
func toKnownRoutes(rows []knownRouteRow, nodes map[uuid.UUID]*api.ResolvedNode) []api.KnownRoute {
items := make([]api.KnownRoute, 0, len(rows))
for _, r := range rows {
hops := make([]api.RouteHop, 0, len(r.NodeIds))
@@ -301,7 +352,7 @@ func toKnownRoutes(rows []sqlc.KnownRoute, nodes map[uuid.UUID]*api.ResolvedNode
return items
}
func collectNodeIDs(rows []sqlc.KnownRoute) []uuid.UUID {
func collectNodeIDs(rows []knownRouteRow) []uuid.UUID {
seen := make(map[uuid.UUID]struct{})
var ids []uuid.UUID
for _, r := range rows {
+58
View File
@@ -4,12 +4,70 @@
package db
import (
"bytes"
"context"
"encoding/hex"
"testing"
"time"
sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc"
mockdb "github.com/MeshCore-Beacon/beacon-server/db/sqlc/mock"
"github.com/MeshCore-Beacon/beacon-server/internal/api"
"github.com/google/uuid"
"go.uber.org/mock/gomock"
)
func TestRoutePathKey_MatchesPostgresDigest(t *testing.T) {
// Golden vector: Postgres computes
// decode(md5(array_to_string(node_ids, ',')), 'hex')
// over lowercase-hyphenated UUIDs. Migration 024 backfilled with that
// expression; this pins the Go side to the identical bytes.
a := uuid.MustParse("00000000-0000-0000-0000-000000000001")
b := uuid.MustParse("00000000-0000-0000-0000-000000000002")
got := hex.EncodeToString(routePathKey([]uuid.UUID{a, b}))
want := "f097439148601d9f3291c474f82fa64c"
if got != want {
t.Errorf("routePathKey = %s, want %s", got, want)
}
}
func TestUpsertKnownRoute_ComputesPathKey(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
mock := mockdb.NewMockQuerier(ctrl)
store := &Store{q: mock}
a := uuid.MustParse("00000000-0000-0000-0000-000000000001")
b := uuid.MustParse("00000000-0000-0000-0000-000000000002")
wantKey, _ := hex.DecodeString("f097439148601d9f3291c474f82fa64c")
mock.EXPECT().UpsertKnownRoute(gomock.Any(), gomock.Cond(func(p sqlc.UpsertKnownRouteParams) bool {
return bytes.Equal(p.PathKey, wantKey)
})).Return(nil)
if err := store.UpsertKnownRoute(context.Background(), []uuid.UUID{a, b}, [][]byte{{0x37}, {0xd8}}, "PRG", 2); err != nil {
t.Fatal(err)
}
}
func TestDeleteOldRoutes_PassesCutoffs(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
mock := mockdb.NewMockQuerier(ctrl)
store := &Store{q: mock}
retention := time.Date(2026, 7, 23, 0, 0, 0, 0, time.UTC)
grace := time.Date(2026, 7, 30, 0, 0, 0, 0, time.UTC)
mock.EXPECT().DeleteOldRoutes(gomock.Any(), gomock.Cond(func(p sqlc.DeleteOldRoutesParams) bool {
return p.LastSeen.Time.Equal(retention) && p.ObservationCount == 3 && p.LastSeen_2.Time.Equal(grace)
})).Return(nil)
if err := store.DeleteOldRoutes(context.Background(), retention, 3, grace); err != nil {
t.Fatal(err)
}
}
func TestExtractFromNode_Found(t *testing.T) {
a, b, c := uuid.New(), uuid.New(), uuid.New()
hops := []api.RouteHop{
+24 -10
View File
@@ -85,6 +85,20 @@ func (mr *MockQuerierMockRecorder) DeleteOldPackets(ctx, lastHeardAt any) *gomoc
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteOldPackets", reflect.TypeOf((*MockQuerier)(nil).DeleteOldPackets), ctx, lastHeardAt)
}
// DeleteOldRoutes mocks base method.
func (m *MockQuerier) DeleteOldRoutes(ctx context.Context, arg db.DeleteOldRoutesParams) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "DeleteOldRoutes", ctx, arg)
ret0, _ := ret[0].(error)
return ret0
}
// DeleteOldRoutes indicates an expected call of DeleteOldRoutes.
func (mr *MockQuerierMockRecorder) DeleteOldRoutes(ctx, arg any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteOldRoutes", reflect.TypeOf((*MockQuerier)(nil).DeleteOldRoutes), ctx, arg)
}
// DeleteOldTelemetry mocks base method.
func (m *MockQuerier) DeleteOldTelemetry(ctx context.Context, reportedAt pgtype.Timestamptz) error {
m.ctrl.T.Helper()
@@ -189,10 +203,10 @@ func (mr *MockQuerierMockRecorder) GetIATABorder(ctx, iata any) *gomock.Call {
}
// GetKnownRoutesByNode mocks base method.
func (m *MockQuerier) GetKnownRoutesByNode(ctx context.Context, arg db.GetKnownRoutesByNodeParams) ([]db.KnownRoute, error) {
func (m *MockQuerier) GetKnownRoutesByNode(ctx context.Context, arg db.GetKnownRoutesByNodeParams) ([]db.GetKnownRoutesByNodeRow, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "GetKnownRoutesByNode", ctx, arg)
ret0, _ := ret[0].([]db.KnownRoute)
ret0, _ := ret[0].([]db.GetKnownRoutesByNodeRow)
ret1, _ := ret[1].(error)
return ret0, ret1
}
@@ -818,10 +832,10 @@ func (mr *MockQuerierMockRecorder) ListIATAs(ctx any) *gomock.Call {
}
// ListKnownRoutes mocks base method.
func (m *MockQuerier) ListKnownRoutes(ctx context.Context, arg db.ListKnownRoutesParams) ([]db.KnownRoute, error) {
func (m *MockQuerier) ListKnownRoutes(ctx context.Context, arg db.ListKnownRoutesParams) ([]db.ListKnownRoutesRow, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ListKnownRoutes", ctx, arg)
ret0, _ := ret[0].([]db.KnownRoute)
ret0, _ := ret[0].([]db.ListKnownRoutesRow)
ret1, _ := ret[1].(error)
return ret0, ret1
}
@@ -1027,17 +1041,17 @@ func (mr *MockQuerierMockRecorder) ReconfirmNeighbors(ctx any) *gomock.Call {
}
// ReconfirmRoutes mocks base method.
func (m *MockQuerier) ReconfirmRoutes(ctx context.Context) error {
func (m *MockQuerier) ReconfirmRoutes(ctx context.Context, limit int32) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ReconfirmRoutes", ctx)
ret := m.ctrl.Call(m, "ReconfirmRoutes", ctx, limit)
ret0, _ := ret[0].(error)
return ret0
}
// ReconfirmRoutes indicates an expected call of ReconfirmRoutes.
func (mr *MockQuerierMockRecorder) ReconfirmRoutes(ctx any) *gomock.Call {
func (mr *MockQuerierMockRecorder) ReconfirmRoutes(ctx, limit any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReconfirmRoutes", reflect.TypeOf((*MockQuerier)(nil).ReconfirmRoutes), ctx)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReconfirmRoutes", reflect.TypeOf((*MockQuerier)(nil).ReconfirmRoutes), ctx, limit)
}
// RefreshHourlyStats mocks base method.
@@ -1199,10 +1213,10 @@ func (mr *MockQuerierMockRecorder) ResolvePathHashesP4(ctx, arg any) *gomock.Cal
}
// SearchKnownRoutes mocks base method.
func (m *MockQuerier) SearchKnownRoutes(ctx context.Context, arg db.SearchKnownRoutesParams) ([]db.KnownRoute, error) {
func (m *MockQuerier) SearchKnownRoutes(ctx context.Context, arg db.SearchKnownRoutesParams) ([]db.SearchKnownRoutesRow, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "SearchKnownRoutes", ctx, arg)
ret0, _ := ret[0].([]db.KnownRoute)
ret0, _ := ret[0].([]db.SearchKnownRoutesRow)
ret1, _ := ret[1].(error)
return ret0, ret1
}
+10 -8
View File
@@ -57,14 +57,16 @@ type IataCode struct {
}
type KnownRoute struct {
ID int64 `json:"id"`
NodeIds []uuid.UUID `json:"node_ids"`
HashPrefix [][]byte `json:"hash_prefix"`
Iata string `json:"iata"`
HopCount int32 `json:"hop_count"`
FirstSeen pgtype.Timestamptz `json:"first_seen"`
LastSeen pgtype.Timestamptz `json:"last_seen"`
ObservationCount int64 `json:"observation_count"`
ID int64 `json:"id"`
PathKey []byte `json:"path_key"`
NodeIds []uuid.UUID `json:"node_ids"`
HashPrefix [][]byte `json:"hash_prefix"`
Iata string `json:"iata"`
HopCount int32 `json:"hop_count"`
FirstSeen pgtype.Timestamptz `json:"first_seen"`
LastSeen pgtype.Timestamptz `json:"last_seen"`
ObservationCount int64 `json:"observation_count"`
LastReconfirmedAt pgtype.Timestamptz `json:"last_reconfirmed_at"`
}
type MvHourlyIataStat struct {
+12 -9
View File
@@ -25,6 +25,9 @@ type Querier interface {
// Deletes packets and their observations older than the given cutoff.
// packet_observations cascade-delete via FK.
DeleteOldPackets(ctx context.Context, lastHeardAt pgtype.Timestamptz) error
// Deletes routes not observed since the retention cutoff ($1), and rarely-observed
// routes (observation_count < $2) not observed since the grace cutoff ($3).
DeleteOldRoutes(ctx context.Context, arg DeleteOldRoutesParams) error
// Deletes telemetry rows older than the given cutoff. Called by the cleanup goroutine.
DeleteOldTelemetry(ctx context.Context, reportedAt pgtype.Timestamptz) error
// Keeps the trace IATA filter in step with packet retention.
@@ -38,7 +41,7 @@ type Querier interface {
// missing row (unknown IATA) is sql.ErrNoRows, same not-found distinction
// GetIATA already makes.
GetIATABorder(ctx context.Context, iata string) ([]byte, error)
GetKnownRoutesByNode(ctx context.Context, arg GetKnownRoutesByNodeParams) ([]KnownRoute, error)
GetKnownRoutesByNode(ctx context.Context, arg GetKnownRoutesByNodeParams) ([]GetKnownRoutesByNodeRow, error)
GetNodeByID(ctx context.Context, id uuid.UUID) (GetNodeByIDRow, error)
GetNodeByPubkey(ctx context.Context, publicKey []byte) (uuid.UUID, error)
// Returns the neighbors of a node with details, ordered by most recently seen.
@@ -119,7 +122,7 @@ type Querier interface {
// Pass cursor=0 to start from the beginning (cursor is last_seen epoch ms).
ListChannels(ctx context.Context, arg ListChannelsParams) ([]Channel, error)
ListIATAs(ctx context.Context) ([]IataCode, error)
ListKnownRoutes(ctx context.Context, arg ListKnownRoutesParams) ([]KnownRoute, error)
ListKnownRoutes(ctx context.Context, arg ListKnownRoutesParams) ([]ListKnownRoutesRow, error)
// Returns messages after the given message ID, ordered oldest first.
// Used for WS reconnect backfill.
ListMessagesAfterID(ctx context.Context, arg ListMessagesAfterIDParams) ([]ListMessagesAfterIDRow, error)
@@ -168,9 +171,10 @@ type Querier interface {
// Delete node_neighbors where the neighbor has departed from node_short_ids
// for that IATA, or where its prefix_4 is now ambiguous.
ReconfirmNeighbors(ctx context.Context) error
// Delete known_routes where any hop node has departed from node_short_ids for
// that IATA, or where any hop's prefix_4 is now ambiguous (matches >1 node).
ReconfirmRoutes(ctx context.Context) error
// Checks the $1 least-recently-reconfirmed routes: deletes those with a departed
// hop node or a hop prefix now matching >1 node in that IATA (length-aware:
// 1/2/3/4-byte hop prefixes check prefix_1/2/3/4), and stamps the survivors.
ReconfirmRoutes(ctx context.Context, limit int32) error
RefreshHourlyStats(ctx context.Context) error
RefreshPayloadBreakdown(ctx context.Context) error
RefreshRadioPresets(ctx context.Context) error
@@ -190,7 +194,7 @@ type Querier interface {
ResolvePathHashesP4(ctx context.Context, arg ResolvePathHashesP4Params) ([]ResolvePathHashesP4Row, error)
// Returns known routes containing a subsequence from source to destination hash prefix.
// Verifies source appears before destination in the route.
SearchKnownRoutes(ctx context.Context, arg SearchKnownRoutesParams) ([]KnownRoute, error)
SearchKnownRoutes(ctx context.Context, arg SearchKnownRoutesParams) ([]SearchKnownRoutesRow, error)
SetNodeDefaultScope(ctx context.Context, arg SetNodeDefaultScopeParams) error
SetNodeMultibytePaths(ctx context.Context, id uuid.UUID) error
SetNodeMultibyteTraces(ctx context.Context, id uuid.UUID) error
@@ -228,9 +232,8 @@ type Querier interface {
// ============================================================
// ROUTES
// ============================================================
// Inserts or updates a known route (all hops resolved to high confidence).
// node_ids and hash_prefix are ordered arrays of the resolved node UUIDs and
// their hash bytes. last_seen is bumped on conflict.
// Route identity is path_key, an md5 of node_ids computed by the caller.
// On conflict, observation_count and last_seen are bumped.
UpsertKnownRoute(ctx context.Context, arg UpsertKnownRouteParams) error
// ============================================================
// NODES
+117 -36
View File
@@ -51,6 +51,25 @@ func (q *Queries) DeleteOldPackets(ctx context.Context, lastHeardAt pgtype.Times
return err
}
const deleteOldRoutes = `-- name: DeleteOldRoutes :exec
DELETE FROM known_routes
WHERE last_seen < $1
OR (observation_count < $2 AND last_seen < $3)
`
type DeleteOldRoutesParams struct {
LastSeen pgtype.Timestamptz `json:"last_seen"`
ObservationCount int64 `json:"observation_count"`
LastSeen_2 pgtype.Timestamptz `json:"last_seen_2"`
}
// Deletes routes not observed since the retention cutoff ($1), and rarely-observed
// routes (observation_count < $2) not observed since the grace cutoff ($3).
func (q *Queries) DeleteOldRoutes(ctx context.Context, arg DeleteOldRoutesParams) error {
_, err := q.db.Exec(ctx, deleteOldRoutes, arg.LastSeen, arg.ObservationCount, arg.LastSeen_2)
return err
}
const deleteOldTelemetry = `-- name: DeleteOldTelemetry :exec
DELETE FROM observer_telemetry WHERE reported_at < $1
`
@@ -237,15 +256,26 @@ type GetKnownRoutesByNodeParams struct {
Column2 uuid.UUID `json:"column_2"`
}
func (q *Queries) GetKnownRoutesByNode(ctx context.Context, arg GetKnownRoutesByNodeParams) ([]KnownRoute, error) {
type GetKnownRoutesByNodeRow struct {
ID int64 `json:"id"`
NodeIds []uuid.UUID `json:"node_ids"`
HashPrefix [][]byte `json:"hash_prefix"`
Iata string `json:"iata"`
HopCount int32 `json:"hop_count"`
FirstSeen pgtype.Timestamptz `json:"first_seen"`
LastSeen pgtype.Timestamptz `json:"last_seen"`
ObservationCount int64 `json:"observation_count"`
}
func (q *Queries) GetKnownRoutesByNode(ctx context.Context, arg GetKnownRoutesByNodeParams) ([]GetKnownRoutesByNodeRow, error) {
rows, err := q.db.Query(ctx, getKnownRoutesByNode, arg.Iata, arg.Column2)
if err != nil {
return nil, err
}
defer rows.Close()
items := []KnownRoute{}
items := []GetKnownRoutesByNodeRow{}
for rows.Next() {
var i KnownRoute
var i GetKnownRoutesByNodeRow
if err := rows.Scan(
&i.ID,
&i.NodeIds,
@@ -2095,7 +2125,18 @@ type ListKnownRoutesParams struct {
Limit int32 `json:"limit"`
}
func (q *Queries) ListKnownRoutes(ctx context.Context, arg ListKnownRoutesParams) ([]KnownRoute, error) {
type ListKnownRoutesRow struct {
ID int64 `json:"id"`
NodeIds []uuid.UUID `json:"node_ids"`
HashPrefix [][]byte `json:"hash_prefix"`
Iata string `json:"iata"`
HopCount int32 `json:"hop_count"`
FirstSeen pgtype.Timestamptz `json:"first_seen"`
LastSeen pgtype.Timestamptz `json:"last_seen"`
ObservationCount int64 `json:"observation_count"`
}
func (q *Queries) ListKnownRoutes(ctx context.Context, arg ListKnownRoutesParams) ([]ListKnownRoutesRow, error) {
rows, err := q.db.Query(ctx, listKnownRoutes,
arg.Column1,
arg.Column2,
@@ -2106,9 +2147,9 @@ func (q *Queries) ListKnownRoutes(ctx context.Context, arg ListKnownRoutesParams
return nil, err
}
defer rows.Close()
items := []KnownRoute{}
items := []ListKnownRoutesRow{}
for rows.Next() {
var i KnownRoute
var i ListKnownRoutesRow
if err := rows.Scan(
&i.ID,
&i.NodeIds,
@@ -3197,31 +3238,59 @@ func (q *Queries) ReconfirmNeighbors(ctx context.Context) error {
}
const reconfirmRoutes = `-- name: ReconfirmRoutes :exec
DELETE FROM known_routes kr
WHERE EXISTS (
SELECT 1
FROM unnest(kr.node_ids) AS hop_node_id
WHERE NOT EXISTS (
SELECT 1 FROM node_short_ids ns
WHERE ns.node_id = hop_node_id
AND ns.iata = kr.iata
WITH batch AS (
SELECT iata, path_key, node_ids, hash_prefix
FROM known_routes
ORDER BY last_reconfirmed_at
LIMIT $1
),
amb AS MATERIALIZED (
SELECT iata, 1 AS len, prefix_1 AS p FROM node_short_ids GROUP BY iata, prefix_1 HAVING COUNT(*) > 1
UNION ALL
SELECT iata, 2, prefix_2 FROM node_short_ids GROUP BY iata, prefix_2 HAVING COUNT(*) > 1
UNION ALL
SELECT iata, 3, prefix_3 FROM node_short_ids GROUP BY iata, prefix_3 HAVING COUNT(*) > 1
UNION ALL
SELECT iata, 4, prefix_4 FROM node_short_ids GROUP BY iata, prefix_4 HAVING COUNT(*) > 1
),
dead AS (
SELECT b.iata, b.path_key
FROM batch b
WHERE EXISTS (
SELECT 1
FROM unnest(b.node_ids) AS hop_node_id
WHERE NOT EXISTS (
SELECT 1 FROM node_short_ids ns
WHERE ns.node_id = hop_node_id
AND ns.iata = b.iata
)
)
UNION
SELECT DISTINCT b.iata, b.path_key
FROM batch b
CROSS JOIN LATERAL unnest(b.hash_prefix) AS hp
JOIN amb a ON a.iata = b.iata AND a.len = length(hp) AND a.p = hp
),
deleted AS (
DELETE FROM known_routes kr
USING dead d
WHERE kr.iata = d.iata AND kr.path_key = d.path_key
)
OR EXISTS (
SELECT 1
FROM unnest(kr.hash_prefix) AS hop_prefix
WHERE (
SELECT COUNT(*) FROM node_short_ids ns
WHERE ns.iata = kr.iata
AND ns.prefix_4 = hop_prefix
) > 1
)
UPDATE known_routes kr
SET last_reconfirmed_at = NOW()
FROM batch b
WHERE kr.iata = b.iata AND kr.path_key = b.path_key
AND NOT EXISTS (
SELECT 1 FROM dead d
WHERE d.iata = b.iata AND d.path_key = b.path_key
)
`
// Delete known_routes where any hop node has departed from node_short_ids for
// that IATA, or where any hop's prefix_4 is now ambiguous (matches >1 node).
func (q *Queries) ReconfirmRoutes(ctx context.Context) error {
_, err := q.db.Exec(ctx, reconfirmRoutes)
// Checks the $1 least-recently-reconfirmed routes: deletes those with a departed
// hop node or a hop prefix now matching >1 node in that IATA (length-aware:
// 1/2/3/4-byte hop prefixes check prefix_1/2/3/4), and stamps the survivors.
func (q *Queries) ReconfirmRoutes(ctx context.Context, limit int32) error {
_, err := q.db.Exec(ctx, reconfirmRoutes, limit)
return err
}
@@ -3512,17 +3581,28 @@ type SearchKnownRoutesParams struct {
Column3 []byte `json:"column_3"`
}
type SearchKnownRoutesRow struct {
ID int64 `json:"id"`
NodeIds []uuid.UUID `json:"node_ids"`
HashPrefix [][]byte `json:"hash_prefix"`
Iata string `json:"iata"`
HopCount int32 `json:"hop_count"`
FirstSeen pgtype.Timestamptz `json:"first_seen"`
LastSeen pgtype.Timestamptz `json:"last_seen"`
ObservationCount int64 `json:"observation_count"`
}
// Returns known routes containing a subsequence from source to destination hash prefix.
// Verifies source appears before destination in the route.
func (q *Queries) SearchKnownRoutes(ctx context.Context, arg SearchKnownRoutesParams) ([]KnownRoute, error) {
func (q *Queries) SearchKnownRoutes(ctx context.Context, arg SearchKnownRoutesParams) ([]SearchKnownRoutesRow, error) {
rows, err := q.db.Query(ctx, searchKnownRoutes, arg.Iata, arg.Column2, arg.Column3)
if err != nil {
return nil, err
}
defer rows.Close()
items := []KnownRoute{}
items := []SearchKnownRoutesRow{}
for rows.Next() {
var i KnownRoute
var i SearchKnownRoutesRow
if err := rows.Scan(
&i.ID,
&i.NodeIds,
@@ -3883,14 +3963,15 @@ func (q *Queries) UpsertIATADetails(ctx context.Context, arg UpsertIATADetailsPa
const upsertKnownRoute = `-- name: UpsertKnownRoute :exec
INSERT INTO known_routes (node_ids, hash_prefix, iata, hop_count)
VALUES ($1, $2, $3, $4)
ON CONFLICT (node_ids, iata) DO UPDATE SET
INSERT INTO known_routes (path_key, node_ids, hash_prefix, iata, hop_count)
VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (iata, path_key) DO UPDATE SET
last_seen = NOW(),
observation_count = known_routes.observation_count + 1
`
type UpsertKnownRouteParams struct {
PathKey []byte `json:"path_key"`
NodeIds []uuid.UUID `json:"node_ids"`
HashPrefix [][]byte `json:"hash_prefix"`
Iata string `json:"iata"`
@@ -3900,11 +3981,11 @@ type UpsertKnownRouteParams struct {
// ============================================================
// ROUTES
// ============================================================
// Inserts or updates a known route (all hops resolved to high confidence).
// node_ids and hash_prefix are ordered arrays of the resolved node UUIDs and
// their hash bytes. last_seen is bumped on conflict.
// Route identity is path_key, an md5 of node_ids computed by the caller.
// On conflict, observation_count and last_seen are bumped.
func (q *Queries) UpsertKnownRoute(ctx context.Context, arg UpsertKnownRouteParams) error {
_, err := q.db.Exec(ctx, upsertKnownRoute,
arg.PathKey,
arg.NodeIds,
arg.HashPrefix,
arg.Iata,
+13 -5
View File
@@ -73,15 +73,23 @@ func CleanupTask(store *db.Store, telemetryRetention, packetRetention, nodeDelet
}
}
// ReconfirmTask returns a Task that prunes stale and ambiguous resolved paths
// and neighbors. Runs after routes to ensure neighbors are cleaned against
// already-reconfirmed path data.
func ReconfirmTask(store *db.Store, interval time.Duration) Task {
// reconfirmBatchSize bounds per-tick reconfirm work; at hourly ticks a 16M-row
// table gets fully re-checked roughly daily.
const reconfirmBatchSize = 750_000
// ReconfirmTask returns a Task that prunes aged routes first, then reconfirms
// stale and ambiguous resolved paths and neighbors, so known_routes only ever
// has one writer at a time.
func ReconfirmTask(store *db.Store, routeRetention, routeGrace time.Duration, routeMinObservations int64, interval time.Duration) Task {
return Task{
Name: "reconfirm",
Interval: interval,
Run: func(ctx context.Context) error {
if err := store.ReconfirmRoutes(ctx); err != nil {
now := time.Now()
if err := store.DeleteOldRoutes(ctx, now.Add(-routeRetention), routeMinObservations, now.Add(-routeGrace)); err != nil {
return fmt.Errorf("route retention: %w", err)
}
if err := store.ReconfirmRoutes(ctx, reconfirmBatchSize); err != nil {
return fmt.Errorf("routes: %w", err)
}
if err := store.ReconfirmNeighbors(ctx); err != nil {
+45 -16
View File
@@ -22,6 +22,7 @@ type Config struct {
Telemetry TelemetryConfig `yaml:"telemetry"`
WebSocket WebSocketConfig `yaml:"websocket"`
Packets PacketsConfig `yaml:"packets"`
Routes RoutesConfig `yaml:"routes"`
Ingest IngestFilterConfig `yaml:"ingest"`
Scopes []ScopeConfig `yaml:"scopes"`
Cache CacheConfig `yaml:"cache"`
@@ -33,13 +34,16 @@ type Config struct {
// ResolvedConfig holds all runtime configuration with defaults applied.
type ResolvedConfig struct {
TelemetryResolution time.Duration
TelemetryRetention time.Duration
PacketRetention time.Duration
MaxConnsPerIP int
ViewRefreshInterval time.Duration
ReconfirmInterval time.Duration
CleanupInterval time.Duration
TelemetryResolution time.Duration
TelemetryRetention time.Duration
PacketRetention time.Duration
RouteRetention time.Duration
RouteGrace time.Duration
RouteMinObservations int
MaxConnsPerIP int
ViewRefreshInterval time.Duration
ReconfirmInterval time.Duration
CleanupInterval time.Duration
PresenceFlushInterval time.Duration
PresencePacketTTL time.Duration
@@ -177,6 +181,19 @@ type PacketsConfig struct {
Retention duration `yaml:"retention"`
}
// RoutesConfig controls known-route retention behaviour.
type RoutesConfig struct {
// Retention is how long a route is kept after it was last observed.
// Defaults to 336h (14 days) if not set.
Retention duration `yaml:"retention"`
// Grace is how long a route observed fewer than MinObservations times is
// kept. Defaults to 168h (7 days) if not set.
Grace duration `yaml:"grace"`
// MinObservations is the observation count below which Grace applies
// instead of Retention. Defaults to 3 if not set.
MinObservations int `yaml:"min_observations"`
}
// NodesConfig controls node-derived signal thresholds.
type NodesConfig struct {
// ClockDriftThreshold is the |device clock - server clock| magnitude, measured from a
@@ -299,13 +316,16 @@ func Load(path string) (*Config, error) {
// Resolve returns a ResolvedConfig with defaults applied for any zero values.
func Resolve(cfg *Config) ResolvedConfig {
r := ResolvedConfig{
TelemetryResolution: cfg.Telemetry.Resolution.Duration,
TelemetryRetention: cfg.Telemetry.Retention.Duration,
PacketRetention: cfg.Packets.Retention.Duration,
MaxConnsPerIP: cfg.WebSocket.MaxConnectionsPerIP,
ViewRefreshInterval: cfg.Background.ViewRefresh.Duration,
ReconfirmInterval: cfg.Background.Reconfirm.Duration,
CleanupInterval: cfg.Background.Cleanup.Duration,
TelemetryResolution: cfg.Telemetry.Resolution.Duration,
TelemetryRetention: cfg.Telemetry.Retention.Duration,
PacketRetention: cfg.Packets.Retention.Duration,
RouteRetention: cfg.Routes.Retention.Duration,
RouteGrace: cfg.Routes.Grace.Duration,
RouteMinObservations: cfg.Routes.MinObservations,
MaxConnsPerIP: cfg.WebSocket.MaxConnectionsPerIP,
ViewRefreshInterval: cfg.Background.ViewRefresh.Duration,
ReconfirmInterval: cfg.Background.Reconfirm.Duration,
CleanupInterval: cfg.Background.Cleanup.Duration,
PresenceFlushInterval: cfg.Presence.FlushInterval.Duration,
PresencePacketTTL: cfg.Presence.PacketTTL.Duration,
@@ -323,6 +343,15 @@ func Resolve(cfg *Config) ResolvedConfig {
if r.PacketRetention == 0 {
r.PacketRetention = 30 * 24 * time.Hour
}
if r.RouteRetention == 0 {
r.RouteRetention = 14 * 24 * time.Hour
}
if r.RouteGrace == 0 {
r.RouteGrace = 7 * 24 * time.Hour
}
if r.RouteMinObservations == 0 {
r.RouteMinObservations = 3
}
if r.MaxConnsPerIP == 0 {
r.MaxConnsPerIP = 5
}
@@ -357,8 +386,8 @@ func Resolve(cfg *Config) ResolvedConfig {
func (r ResolvedConfig) String() string {
return fmt.Sprintf(
"telemetryResolution=%s telemetryRetention=%s packetRetention=%s maxConnsPerIP=%d viewRefresh=%s reconfirm=%s cleanup=%s presenceFlush=%s presencePacketTTL=%s clockDriftThreshold=%s nodeStaleThreshold=%s nodeDeleteAfter=%s",
r.TelemetryResolution, r.TelemetryRetention, r.PacketRetention,
"telemetryResolution=%s telemetryRetention=%s packetRetention=%s routeRetention=%s routeGrace=%s routeMinObs=%d maxConnsPerIP=%d viewRefresh=%s reconfirm=%s cleanup=%s presenceFlush=%s presencePacketTTL=%s clockDriftThreshold=%s nodeStaleThreshold=%s nodeDeleteAfter=%s",
r.TelemetryResolution, r.TelemetryRetention, r.PacketRetention, r.RouteRetention, r.RouteGrace, r.RouteMinObservations,
r.MaxConnsPerIP, r.ViewRefreshInterval, r.ReconfirmInterval, r.CleanupInterval,
r.PresenceFlushInterval, r.PresencePacketTTL, r.ClockDriftThreshold,
r.NodeStaleThreshold, r.NodeDeleteAfter,
+13
View File
@@ -141,3 +141,16 @@ func TestResolvedConfig_String(t *testing.T) {
t.Error("expected maxConnsPerIP in string")
}
}
func TestResolve_RouteDefaults(t *testing.T) {
r := Resolve(&Config{})
if r.RouteRetention != 336*time.Hour {
t.Errorf("RouteRetention = %s, want 336h", r.RouteRetention)
}
if r.RouteGrace != 168*time.Hour {
t.Errorf("RouteGrace = %s, want 168h", r.RouteGrace)
}
if r.RouteMinObservations != 3 {
t.Errorf("RouteMinObservations = %d, want 3", r.RouteMinObservations)
}
}