mirror of
https://github.com/MeshCore-Beacon/beacon-server.git
synced 2026-09-01 16:48:19 +00:00
perf: known_routes retention, digest identity, and a working batched reconfirm (#98)
* fix: cascade channel_messages on packet delete * rebuild known_routes keyed on (iata, path_key) * add route retention delete, batch reconfirm, path_key upsert * compute route path_key in store, add DeleteOldRoutes * add routes retention config * wire route retention into cleanup, batch reconfirm * prune routes in reconfirm task, not cleanup * expand routes retention docs in example config
This commit is contained in:
+1
-1
@@ -258,7 +258,7 @@ func main() {
|
||||
scheduler := background.New([]background.Task{
|
||||
background.ViewRefreshTask(store, resolved.ViewRefreshInterval),
|
||||
background.CleanupTask(store, resolved.TelemetryRetention, resolved.PacketRetention, resolved.NodeDeleteAfter, resolved.CleanupInterval),
|
||||
background.ReconfirmTask(store, resolved.ReconfirmInterval),
|
||||
background.ReconfirmTask(store, resolved.RouteRetention, resolved.RouteGrace, int64(resolved.RouteMinObservations), resolved.ReconfirmInterval),
|
||||
})
|
||||
go scheduler.Start(ctx)
|
||||
|
||||
|
||||
@@ -75,6 +75,25 @@ telemetry:
|
||||
packets:
|
||||
retention: 720h # 30 days
|
||||
|
||||
# Known-route retention. Routes are distilled packet history (the path a packet
|
||||
# took, hop by hop), so they may outlive packets.retention; there is no required
|
||||
# relationship between the two.
|
||||
routes:
|
||||
# How long a route is kept after it was last observed. Every new observation
|
||||
# of the same path resets this clock, so routes the mesh still uses never
|
||||
# expire -- only paths nothing has traveled for this long age out.
|
||||
retention: 336h # 14 days
|
||||
# Shorter window for rarely-seen routes: one observed fewer than
|
||||
# min_observations times in total is dropped once it goes unobserved for
|
||||
# this long. Must be shorter than retention, or it never applies.
|
||||
grace: 168h # 7 days
|
||||
# Lifetime observation count a route must reach to earn the full retention
|
||||
# window. Below it a route is treated as flood noise -- a path recorded once
|
||||
# or twice and never confirmed -- and only kept for the grace window. The
|
||||
# count never resets, so once a route crosses this bar it stays in the
|
||||
# retention tier for good.
|
||||
min_observations: 3
|
||||
|
||||
websocket:
|
||||
max_connections_per_ip: 5 # default: 5
|
||||
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
-- Copyright 2026 Beacon Contributors
|
||||
-- SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
-- known_routes grew unbounded (28 GB / 32.8M rows in 24 days on prod) and its
|
||||
-- UNIQUE(node_ids, iata) key indexed whole UUID arrays. Rebuild keyed on a
|
||||
-- 16-byte md5 of node_ids and add reconfirm bookkeeping. No rows are pruned
|
||||
-- here: retention is deployment config (routes.retention/grace), so the first
|
||||
-- cleanup tick after startup enforces it; a schema migration must not bake in
|
||||
-- one deployment's policy. id stays (the API serializes it) but loses its
|
||||
-- index; uniqueness lives on (iata, path_key).
|
||||
|
||||
CREATE TABLE known_routes_new (
|
||||
id BIGINT GENERATED BY DEFAULT AS IDENTITY NOT NULL,
|
||||
path_key BYTEA NOT NULL,
|
||||
node_ids UUID[] NOT NULL,
|
||||
hash_prefix BYTEA[] NOT NULL,
|
||||
iata CHAR(3) NOT NULL REFERENCES iata_codes(iata) ON DELETE CASCADE,
|
||||
hop_count INT NOT NULL,
|
||||
first_seen TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
last_seen TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
observation_count BIGINT NOT NULL DEFAULT 1,
|
||||
last_reconfirmed_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
PRIMARY KEY (iata, path_key)
|
||||
);
|
||||
|
||||
INSERT INTO known_routes_new
|
||||
(id, path_key, node_ids, hash_prefix, iata, hop_count, first_seen, last_seen, observation_count)
|
||||
SELECT id,
|
||||
decode(md5(array_to_string(node_ids, ',')), 'hex'),
|
||||
node_ids, hash_prefix, iata, hop_count, first_seen, last_seen, observation_count
|
||||
FROM known_routes;
|
||||
|
||||
SELECT setval(pg_get_serial_sequence('known_routes_new', 'id'),
|
||||
(SELECT COALESCE(MAX(id), 1) FROM known_routes_new));
|
||||
|
||||
DROP TABLE known_routes;
|
||||
ALTER TABLE known_routes_new RENAME TO known_routes;
|
||||
ALTER TABLE known_routes RENAME CONSTRAINT known_routes_new_pkey TO known_routes_pkey;
|
||||
ALTER TABLE known_routes RENAME CONSTRAINT known_routes_new_iata_fkey TO known_routes_iata_fkey;
|
||||
|
||||
CREATE INDEX idx_known_routes_hop_count ON known_routes(iata, hop_count);
|
||||
CREATE INDEX idx_known_routes_last_seen ON known_routes(last_seen DESC);
|
||||
CREATE INDEX idx_known_routes_reconfirm ON known_routes(last_reconfirmed_at);
|
||||
+59
-25
@@ -533,6 +533,13 @@ DELETE FROM nodes
|
||||
WHERE last_seen < $1
|
||||
AND id NOT IN (SELECT owner_node_id FROM observer_owners WHERE owner_node_id IS NOT NULL);
|
||||
|
||||
-- name: DeleteOldRoutes :exec
|
||||
-- Deletes routes not observed since the retention cutoff ($1), and rarely-observed
|
||||
-- routes (observation_count < $2) not observed since the grace cutoff ($3).
|
||||
DELETE FROM known_routes
|
||||
WHERE last_seen < $1
|
||||
OR (observation_count < $2 AND last_seen < $3);
|
||||
|
||||
-- name: DeleteOldChannelIATAs :exec
|
||||
-- Keeps the channel IATA filter in step with packet retention.
|
||||
DELETE FROM channel_iatas WHERE last_heard < $1;
|
||||
@@ -1070,12 +1077,11 @@ ORDER BY t.last_heard_at DESC;
|
||||
-- ============================================================
|
||||
|
||||
-- name: UpsertKnownRoute :exec
|
||||
-- Inserts or updates a known route (all hops resolved to high confidence).
|
||||
-- node_ids and hash_prefix are ordered arrays of the resolved node UUIDs and
|
||||
-- their hash bytes. last_seen is bumped on conflict.
|
||||
INSERT INTO known_routes (node_ids, hash_prefix, iata, hop_count)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
ON CONFLICT (node_ids, iata) DO UPDATE SET
|
||||
-- Route identity is path_key, an md5 of node_ids computed by the caller.
|
||||
-- On conflict, observation_count and last_seen are bumped.
|
||||
INSERT INTO known_routes (path_key, node_ids, hash_prefix, iata, hop_count)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
ON CONFLICT (iata, path_key) DO UPDATE SET
|
||||
last_seen = NOW(),
|
||||
observation_count = known_routes.observation_count + 1;
|
||||
|
||||
@@ -1216,27 +1222,55 @@ REFRESH MATERIALIZED VIEW CONCURRENTLY mv_top_advertisers_by_iata;
|
||||
REFRESH MATERIALIZED VIEW CONCURRENTLY mv_radio_presets;
|
||||
|
||||
-- name: ReconfirmRoutes :exec
|
||||
-- Delete known_routes where any hop node has departed from node_short_ids for
|
||||
-- that IATA, or where any hop's prefix_4 is now ambiguous (matches >1 node).
|
||||
DELETE FROM known_routes kr
|
||||
WHERE EXISTS (
|
||||
SELECT 1
|
||||
FROM unnest(kr.node_ids) AS hop_node_id
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM node_short_ids ns
|
||||
WHERE ns.node_id = hop_node_id
|
||||
AND ns.iata = kr.iata
|
||||
-- Checks the $1 least-recently-reconfirmed routes: deletes those with a departed
|
||||
-- hop node or a hop prefix now matching >1 node in that IATA (length-aware:
|
||||
-- 1/2/3/4-byte hop prefixes check prefix_1/2/3/4), and stamps the survivors.
|
||||
WITH batch AS (
|
||||
SELECT iata, path_key, node_ids, hash_prefix
|
||||
FROM known_routes
|
||||
ORDER BY last_reconfirmed_at
|
||||
LIMIT $1
|
||||
),
|
||||
amb AS MATERIALIZED (
|
||||
SELECT iata, 1 AS len, prefix_1 AS p FROM node_short_ids GROUP BY iata, prefix_1 HAVING COUNT(*) > 1
|
||||
UNION ALL
|
||||
SELECT iata, 2, prefix_2 FROM node_short_ids GROUP BY iata, prefix_2 HAVING COUNT(*) > 1
|
||||
UNION ALL
|
||||
SELECT iata, 3, prefix_3 FROM node_short_ids GROUP BY iata, prefix_3 HAVING COUNT(*) > 1
|
||||
UNION ALL
|
||||
SELECT iata, 4, prefix_4 FROM node_short_ids GROUP BY iata, prefix_4 HAVING COUNT(*) > 1
|
||||
),
|
||||
dead AS (
|
||||
SELECT b.iata, b.path_key
|
||||
FROM batch b
|
||||
WHERE EXISTS (
|
||||
SELECT 1
|
||||
FROM unnest(b.node_ids) AS hop_node_id
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM node_short_ids ns
|
||||
WHERE ns.node_id = hop_node_id
|
||||
AND ns.iata = b.iata
|
||||
)
|
||||
)
|
||||
UNION
|
||||
SELECT DISTINCT b.iata, b.path_key
|
||||
FROM batch b
|
||||
CROSS JOIN LATERAL unnest(b.hash_prefix) AS hp
|
||||
JOIN amb a ON a.iata = b.iata AND a.len = length(hp) AND a.p = hp
|
||||
),
|
||||
deleted AS (
|
||||
DELETE FROM known_routes kr
|
||||
USING dead d
|
||||
WHERE kr.iata = d.iata AND kr.path_key = d.path_key
|
||||
)
|
||||
OR EXISTS (
|
||||
SELECT 1
|
||||
FROM unnest(kr.hash_prefix) AS hop_prefix
|
||||
WHERE (
|
||||
SELECT COUNT(*) FROM node_short_ids ns
|
||||
WHERE ns.iata = kr.iata
|
||||
AND ns.prefix_4 = hop_prefix
|
||||
) > 1
|
||||
);
|
||||
UPDATE known_routes kr
|
||||
SET last_reconfirmed_at = NOW()
|
||||
FROM batch b
|
||||
WHERE kr.iata = b.iata AND kr.path_key = b.path_key
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM dead d
|
||||
WHERE d.iata = b.iata AND d.path_key = b.path_key
|
||||
);
|
||||
|
||||
-- name: ReconfirmNeighbors :exec
|
||||
-- Delete node_neighbors where the neighbor has departed from node_short_ids
|
||||
|
||||
+59
-8
@@ -5,7 +5,9 @@ package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc"
|
||||
@@ -14,12 +16,24 @@ import (
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
)
|
||||
|
||||
// routePathKey is the route's identity digest: md5 over the comma-joined
|
||||
// node UUIDs, matching Postgres's decode(md5(array_to_string(node_ids, ',')), 'hex').
|
||||
func routePathKey(nodeIDs []uuid.UUID) []byte {
|
||||
parts := make([]string, len(nodeIDs))
|
||||
for i, id := range nodeIDs {
|
||||
parts[i] = id.String()
|
||||
}
|
||||
sum := md5.Sum([]byte(strings.Join(parts, ",")))
|
||||
return sum[:]
|
||||
}
|
||||
|
||||
func (s *Store) UpsertKnownRoute(ctx context.Context, nodeIDs []uuid.UUID, hashPrefix [][]byte, iata string, hopCount int32) error {
|
||||
return s.q.UpsertKnownRoute(ctx, sqlc.UpsertKnownRouteParams{
|
||||
PathKey: routePathKey(nodeIDs),
|
||||
NodeIds: nodeIDs,
|
||||
HashPrefix: hashPrefix,
|
||||
Iata: iata,
|
||||
HopCount: int32(hopCount),
|
||||
HopCount: hopCount,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -28,7 +42,7 @@ func (s *Store) ListKnownRoutes(ctx context.Context, iata string, hopCount int32
|
||||
if !cursor.IsZero() {
|
||||
cursorTS = pgtype.Timestamptz{Time: cursor, Valid: true}
|
||||
}
|
||||
rows, err := s.q.ListKnownRoutes(ctx, sqlc.ListKnownRoutesParams{
|
||||
sqlRows, err := s.q.ListKnownRoutes(ctx, sqlc.ListKnownRoutesParams{
|
||||
Column1: iata,
|
||||
Column2: hopCount,
|
||||
Column3: cursorTS,
|
||||
@@ -37,6 +51,10 @@ func (s *Store) ListKnownRoutes(ctx context.Context, iata string, hopCount int32
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows := make([]knownRouteRow, len(sqlRows))
|
||||
for i, r := range sqlRows {
|
||||
rows[i] = knownRouteRow{ID: r.ID, NodeIds: r.NodeIds, HashPrefix: r.HashPrefix, Iata: r.Iata, HopCount: r.HopCount, FirstSeen: r.FirstSeen, LastSeen: r.LastSeen, ObservationCount: r.ObservationCount}
|
||||
}
|
||||
ids := collectNodeIDs(rows)
|
||||
nodes, err := s.GetNodesByIDs(ctx, ids)
|
||||
if err != nil {
|
||||
@@ -54,7 +72,7 @@ func (s *Store) SearchKnownRoutes(ctx context.Context, iata, fromHash, toHash st
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows, err := s.q.SearchKnownRoutes(ctx, sqlc.SearchKnownRoutesParams{
|
||||
sqlRows, err := s.q.SearchKnownRoutes(ctx, sqlc.SearchKnownRoutesParams{
|
||||
Iata: iata,
|
||||
Column2: fromBytes,
|
||||
Column3: toBytes,
|
||||
@@ -62,6 +80,10 @@ func (s *Store) SearchKnownRoutes(ctx context.Context, iata, fromHash, toHash st
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows := make([]knownRouteRow, len(sqlRows))
|
||||
for i, r := range sqlRows {
|
||||
rows[i] = knownRouteRow{ID: r.ID, NodeIds: r.NodeIds, HashPrefix: r.HashPrefix, Iata: r.Iata, HopCount: r.HopCount, FirstSeen: r.FirstSeen, LastSeen: r.LastSeen, ObservationCount: r.ObservationCount}
|
||||
}
|
||||
ids := collectNodeIDs(rows)
|
||||
nodes, err := s.GetNodesByIDs(ctx, ids)
|
||||
if err != nil {
|
||||
@@ -109,13 +131,17 @@ func (s *Store) SearchKnownRoutes(ctx context.Context, iata, fromHash, toHash st
|
||||
}
|
||||
|
||||
func (s *Store) GetKnownRoutesByNode(ctx context.Context, iata string, nodeID uuid.UUID) ([]api.KnownRoute, error) {
|
||||
rows, err := s.q.GetKnownRoutesByNode(ctx, sqlc.GetKnownRoutesByNodeParams{
|
||||
sqlRows, err := s.q.GetKnownRoutesByNode(ctx, sqlc.GetKnownRoutesByNodeParams{
|
||||
Iata: iata,
|
||||
Column2: nodeID,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows := make([]knownRouteRow, len(sqlRows))
|
||||
for i, r := range sqlRows {
|
||||
rows[i] = knownRouteRow{ID: r.ID, NodeIds: r.NodeIds, HashPrefix: r.HashPrefix, Iata: r.Iata, HopCount: r.HopCount, FirstSeen: r.FirstSeen, LastSeen: r.LastSeen, ObservationCount: r.ObservationCount}
|
||||
}
|
||||
ids := collectNodeIDs(rows)
|
||||
nodes, err := s.GetNodesByIDs(ctx, ids)
|
||||
if err != nil {
|
||||
@@ -260,8 +286,20 @@ func (s *Store) SearchCrossIATARoutes(ctx context.Context, fromHash, fromIATA, t
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s *Store) ReconfirmRoutes(ctx context.Context) error {
|
||||
return s.q.ReconfirmRoutes(ctx)
|
||||
// ReconfirmRoutes checks the batchSize least-recently-reconfirmed routes,
|
||||
// deleting stale or ambiguous ones and stamping the survivors.
|
||||
func (s *Store) ReconfirmRoutes(ctx context.Context, batchSize int32) error {
|
||||
return s.q.ReconfirmRoutes(ctx, batchSize)
|
||||
}
|
||||
|
||||
// DeleteOldRoutes prunes routes per the retention rule: unconditionally past
|
||||
// retentionCutoff, and past graceCutoff when observed fewer than minObservations times.
|
||||
func (s *Store) DeleteOldRoutes(ctx context.Context, retentionCutoff time.Time, minObservations int64, graceCutoff time.Time) error {
|
||||
return s.q.DeleteOldRoutes(ctx, sqlc.DeleteOldRoutesParams{
|
||||
LastSeen: pgtype.Timestamptz{Time: retentionCutoff, Valid: true},
|
||||
ObservationCount: minObservations,
|
||||
LastSeen_2: pgtype.Timestamptz{Time: graceCutoff, Valid: true},
|
||||
})
|
||||
}
|
||||
|
||||
// extractFromNode returns the portion of a route starting at the given node.
|
||||
@@ -274,7 +312,20 @@ func extractFromNode(hops []api.RouteHop, nodeID uuid.UUID) []api.RouteHop {
|
||||
return hops
|
||||
}
|
||||
|
||||
func toKnownRoutes(rows []sqlc.KnownRoute, nodes map[uuid.UUID]*api.ResolvedNode) []api.KnownRoute {
|
||||
// knownRouteRow normalizes the per-query sqlc row structs (identical
|
||||
// columns, distinct generated types) so the helpers below share one body.
|
||||
type knownRouteRow struct {
|
||||
ID int64
|
||||
NodeIds []uuid.UUID
|
||||
HashPrefix [][]byte
|
||||
Iata string
|
||||
HopCount int32
|
||||
FirstSeen pgtype.Timestamptz
|
||||
LastSeen pgtype.Timestamptz
|
||||
ObservationCount int64
|
||||
}
|
||||
|
||||
func toKnownRoutes(rows []knownRouteRow, nodes map[uuid.UUID]*api.ResolvedNode) []api.KnownRoute {
|
||||
items := make([]api.KnownRoute, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
hops := make([]api.RouteHop, 0, len(r.NodeIds))
|
||||
@@ -301,7 +352,7 @@ func toKnownRoutes(rows []sqlc.KnownRoute, nodes map[uuid.UUID]*api.ResolvedNode
|
||||
return items
|
||||
}
|
||||
|
||||
func collectNodeIDs(rows []sqlc.KnownRoute) []uuid.UUID {
|
||||
func collectNodeIDs(rows []knownRouteRow) []uuid.UUID {
|
||||
seen := make(map[uuid.UUID]struct{})
|
||||
var ids []uuid.UUID
|
||||
for _, r := range rows {
|
||||
|
||||
@@ -4,12 +4,70 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc"
|
||||
mockdb "github.com/MeshCore-Beacon/beacon-server/db/sqlc/mock"
|
||||
"github.com/MeshCore-Beacon/beacon-server/internal/api"
|
||||
"github.com/google/uuid"
|
||||
"go.uber.org/mock/gomock"
|
||||
)
|
||||
|
||||
func TestRoutePathKey_MatchesPostgresDigest(t *testing.T) {
|
||||
// Golden vector: Postgres computes
|
||||
// decode(md5(array_to_string(node_ids, ',')), 'hex')
|
||||
// over lowercase-hyphenated UUIDs. Migration 024 backfilled with that
|
||||
// expression; this pins the Go side to the identical bytes.
|
||||
a := uuid.MustParse("00000000-0000-0000-0000-000000000001")
|
||||
b := uuid.MustParse("00000000-0000-0000-0000-000000000002")
|
||||
got := hex.EncodeToString(routePathKey([]uuid.UUID{a, b}))
|
||||
want := "f097439148601d9f3291c474f82fa64c"
|
||||
if got != want {
|
||||
t.Errorf("routePathKey = %s, want %s", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpsertKnownRoute_ComputesPathKey(t *testing.T) {
|
||||
ctrl := gomock.NewController(t)
|
||||
defer ctrl.Finish()
|
||||
mock := mockdb.NewMockQuerier(ctrl)
|
||||
store := &Store{q: mock}
|
||||
|
||||
a := uuid.MustParse("00000000-0000-0000-0000-000000000001")
|
||||
b := uuid.MustParse("00000000-0000-0000-0000-000000000002")
|
||||
wantKey, _ := hex.DecodeString("f097439148601d9f3291c474f82fa64c")
|
||||
|
||||
mock.EXPECT().UpsertKnownRoute(gomock.Any(), gomock.Cond(func(p sqlc.UpsertKnownRouteParams) bool {
|
||||
return bytes.Equal(p.PathKey, wantKey)
|
||||
})).Return(nil)
|
||||
|
||||
if err := store.UpsertKnownRoute(context.Background(), []uuid.UUID{a, b}, [][]byte{{0x37}, {0xd8}}, "PRG", 2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteOldRoutes_PassesCutoffs(t *testing.T) {
|
||||
ctrl := gomock.NewController(t)
|
||||
defer ctrl.Finish()
|
||||
mock := mockdb.NewMockQuerier(ctrl)
|
||||
store := &Store{q: mock}
|
||||
|
||||
retention := time.Date(2026, 7, 23, 0, 0, 0, 0, time.UTC)
|
||||
grace := time.Date(2026, 7, 30, 0, 0, 0, 0, time.UTC)
|
||||
|
||||
mock.EXPECT().DeleteOldRoutes(gomock.Any(), gomock.Cond(func(p sqlc.DeleteOldRoutesParams) bool {
|
||||
return p.LastSeen.Time.Equal(retention) && p.ObservationCount == 3 && p.LastSeen_2.Time.Equal(grace)
|
||||
})).Return(nil)
|
||||
|
||||
if err := store.DeleteOldRoutes(context.Background(), retention, 3, grace); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractFromNode_Found(t *testing.T) {
|
||||
a, b, c := uuid.New(), uuid.New(), uuid.New()
|
||||
hops := []api.RouteHop{
|
||||
|
||||
+24
-10
@@ -85,6 +85,20 @@ func (mr *MockQuerierMockRecorder) DeleteOldPackets(ctx, lastHeardAt any) *gomoc
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteOldPackets", reflect.TypeOf((*MockQuerier)(nil).DeleteOldPackets), ctx, lastHeardAt)
|
||||
}
|
||||
|
||||
// DeleteOldRoutes mocks base method.
|
||||
func (m *MockQuerier) DeleteOldRoutes(ctx context.Context, arg db.DeleteOldRoutesParams) error {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "DeleteOldRoutes", ctx, arg)
|
||||
ret0, _ := ret[0].(error)
|
||||
return ret0
|
||||
}
|
||||
|
||||
// DeleteOldRoutes indicates an expected call of DeleteOldRoutes.
|
||||
func (mr *MockQuerierMockRecorder) DeleteOldRoutes(ctx, arg any) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteOldRoutes", reflect.TypeOf((*MockQuerier)(nil).DeleteOldRoutes), ctx, arg)
|
||||
}
|
||||
|
||||
// DeleteOldTelemetry mocks base method.
|
||||
func (m *MockQuerier) DeleteOldTelemetry(ctx context.Context, reportedAt pgtype.Timestamptz) error {
|
||||
m.ctrl.T.Helper()
|
||||
@@ -189,10 +203,10 @@ func (mr *MockQuerierMockRecorder) GetIATABorder(ctx, iata any) *gomock.Call {
|
||||
}
|
||||
|
||||
// GetKnownRoutesByNode mocks base method.
|
||||
func (m *MockQuerier) GetKnownRoutesByNode(ctx context.Context, arg db.GetKnownRoutesByNodeParams) ([]db.KnownRoute, error) {
|
||||
func (m *MockQuerier) GetKnownRoutesByNode(ctx context.Context, arg db.GetKnownRoutesByNodeParams) ([]db.GetKnownRoutesByNodeRow, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "GetKnownRoutesByNode", ctx, arg)
|
||||
ret0, _ := ret[0].([]db.KnownRoute)
|
||||
ret0, _ := ret[0].([]db.GetKnownRoutesByNodeRow)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
@@ -818,10 +832,10 @@ func (mr *MockQuerierMockRecorder) ListIATAs(ctx any) *gomock.Call {
|
||||
}
|
||||
|
||||
// ListKnownRoutes mocks base method.
|
||||
func (m *MockQuerier) ListKnownRoutes(ctx context.Context, arg db.ListKnownRoutesParams) ([]db.KnownRoute, error) {
|
||||
func (m *MockQuerier) ListKnownRoutes(ctx context.Context, arg db.ListKnownRoutesParams) ([]db.ListKnownRoutesRow, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "ListKnownRoutes", ctx, arg)
|
||||
ret0, _ := ret[0].([]db.KnownRoute)
|
||||
ret0, _ := ret[0].([]db.ListKnownRoutesRow)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
@@ -1027,17 +1041,17 @@ func (mr *MockQuerierMockRecorder) ReconfirmNeighbors(ctx any) *gomock.Call {
|
||||
}
|
||||
|
||||
// ReconfirmRoutes mocks base method.
|
||||
func (m *MockQuerier) ReconfirmRoutes(ctx context.Context) error {
|
||||
func (m *MockQuerier) ReconfirmRoutes(ctx context.Context, limit int32) error {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "ReconfirmRoutes", ctx)
|
||||
ret := m.ctrl.Call(m, "ReconfirmRoutes", ctx, limit)
|
||||
ret0, _ := ret[0].(error)
|
||||
return ret0
|
||||
}
|
||||
|
||||
// ReconfirmRoutes indicates an expected call of ReconfirmRoutes.
|
||||
func (mr *MockQuerierMockRecorder) ReconfirmRoutes(ctx any) *gomock.Call {
|
||||
func (mr *MockQuerierMockRecorder) ReconfirmRoutes(ctx, limit any) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReconfirmRoutes", reflect.TypeOf((*MockQuerier)(nil).ReconfirmRoutes), ctx)
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReconfirmRoutes", reflect.TypeOf((*MockQuerier)(nil).ReconfirmRoutes), ctx, limit)
|
||||
}
|
||||
|
||||
// RefreshHourlyStats mocks base method.
|
||||
@@ -1199,10 +1213,10 @@ func (mr *MockQuerierMockRecorder) ResolvePathHashesP4(ctx, arg any) *gomock.Cal
|
||||
}
|
||||
|
||||
// SearchKnownRoutes mocks base method.
|
||||
func (m *MockQuerier) SearchKnownRoutes(ctx context.Context, arg db.SearchKnownRoutesParams) ([]db.KnownRoute, error) {
|
||||
func (m *MockQuerier) SearchKnownRoutes(ctx context.Context, arg db.SearchKnownRoutesParams) ([]db.SearchKnownRoutesRow, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "SearchKnownRoutes", ctx, arg)
|
||||
ret0, _ := ret[0].([]db.KnownRoute)
|
||||
ret0, _ := ret[0].([]db.SearchKnownRoutesRow)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
+10
-8
@@ -57,14 +57,16 @@ type IataCode struct {
|
||||
}
|
||||
|
||||
type KnownRoute struct {
|
||||
ID int64 `json:"id"`
|
||||
NodeIds []uuid.UUID `json:"node_ids"`
|
||||
HashPrefix [][]byte `json:"hash_prefix"`
|
||||
Iata string `json:"iata"`
|
||||
HopCount int32 `json:"hop_count"`
|
||||
FirstSeen pgtype.Timestamptz `json:"first_seen"`
|
||||
LastSeen pgtype.Timestamptz `json:"last_seen"`
|
||||
ObservationCount int64 `json:"observation_count"`
|
||||
ID int64 `json:"id"`
|
||||
PathKey []byte `json:"path_key"`
|
||||
NodeIds []uuid.UUID `json:"node_ids"`
|
||||
HashPrefix [][]byte `json:"hash_prefix"`
|
||||
Iata string `json:"iata"`
|
||||
HopCount int32 `json:"hop_count"`
|
||||
FirstSeen pgtype.Timestamptz `json:"first_seen"`
|
||||
LastSeen pgtype.Timestamptz `json:"last_seen"`
|
||||
ObservationCount int64 `json:"observation_count"`
|
||||
LastReconfirmedAt pgtype.Timestamptz `json:"last_reconfirmed_at"`
|
||||
}
|
||||
|
||||
type MvHourlyIataStat struct {
|
||||
|
||||
+12
-9
@@ -25,6 +25,9 @@ type Querier interface {
|
||||
// Deletes packets and their observations older than the given cutoff.
|
||||
// packet_observations cascade-delete via FK.
|
||||
DeleteOldPackets(ctx context.Context, lastHeardAt pgtype.Timestamptz) error
|
||||
// Deletes routes not observed since the retention cutoff ($1), and rarely-observed
|
||||
// routes (observation_count < $2) not observed since the grace cutoff ($3).
|
||||
DeleteOldRoutes(ctx context.Context, arg DeleteOldRoutesParams) error
|
||||
// Deletes telemetry rows older than the given cutoff. Called by the cleanup goroutine.
|
||||
DeleteOldTelemetry(ctx context.Context, reportedAt pgtype.Timestamptz) error
|
||||
// Keeps the trace IATA filter in step with packet retention.
|
||||
@@ -38,7 +41,7 @@ type Querier interface {
|
||||
// missing row (unknown IATA) is sql.ErrNoRows, same not-found distinction
|
||||
// GetIATA already makes.
|
||||
GetIATABorder(ctx context.Context, iata string) ([]byte, error)
|
||||
GetKnownRoutesByNode(ctx context.Context, arg GetKnownRoutesByNodeParams) ([]KnownRoute, error)
|
||||
GetKnownRoutesByNode(ctx context.Context, arg GetKnownRoutesByNodeParams) ([]GetKnownRoutesByNodeRow, error)
|
||||
GetNodeByID(ctx context.Context, id uuid.UUID) (GetNodeByIDRow, error)
|
||||
GetNodeByPubkey(ctx context.Context, publicKey []byte) (uuid.UUID, error)
|
||||
// Returns the neighbors of a node with details, ordered by most recently seen.
|
||||
@@ -119,7 +122,7 @@ type Querier interface {
|
||||
// Pass cursor=0 to start from the beginning (cursor is last_seen epoch ms).
|
||||
ListChannels(ctx context.Context, arg ListChannelsParams) ([]Channel, error)
|
||||
ListIATAs(ctx context.Context) ([]IataCode, error)
|
||||
ListKnownRoutes(ctx context.Context, arg ListKnownRoutesParams) ([]KnownRoute, error)
|
||||
ListKnownRoutes(ctx context.Context, arg ListKnownRoutesParams) ([]ListKnownRoutesRow, error)
|
||||
// Returns messages after the given message ID, ordered oldest first.
|
||||
// Used for WS reconnect backfill.
|
||||
ListMessagesAfterID(ctx context.Context, arg ListMessagesAfterIDParams) ([]ListMessagesAfterIDRow, error)
|
||||
@@ -168,9 +171,10 @@ type Querier interface {
|
||||
// Delete node_neighbors where the neighbor has departed from node_short_ids
|
||||
// for that IATA, or where its prefix_4 is now ambiguous.
|
||||
ReconfirmNeighbors(ctx context.Context) error
|
||||
// Delete known_routes where any hop node has departed from node_short_ids for
|
||||
// that IATA, or where any hop's prefix_4 is now ambiguous (matches >1 node).
|
||||
ReconfirmRoutes(ctx context.Context) error
|
||||
// Checks the $1 least-recently-reconfirmed routes: deletes those with a departed
|
||||
// hop node or a hop prefix now matching >1 node in that IATA (length-aware:
|
||||
// 1/2/3/4-byte hop prefixes check prefix_1/2/3/4), and stamps the survivors.
|
||||
ReconfirmRoutes(ctx context.Context, limit int32) error
|
||||
RefreshHourlyStats(ctx context.Context) error
|
||||
RefreshPayloadBreakdown(ctx context.Context) error
|
||||
RefreshRadioPresets(ctx context.Context) error
|
||||
@@ -190,7 +194,7 @@ type Querier interface {
|
||||
ResolvePathHashesP4(ctx context.Context, arg ResolvePathHashesP4Params) ([]ResolvePathHashesP4Row, error)
|
||||
// Returns known routes containing a subsequence from source to destination hash prefix.
|
||||
// Verifies source appears before destination in the route.
|
||||
SearchKnownRoutes(ctx context.Context, arg SearchKnownRoutesParams) ([]KnownRoute, error)
|
||||
SearchKnownRoutes(ctx context.Context, arg SearchKnownRoutesParams) ([]SearchKnownRoutesRow, error)
|
||||
SetNodeDefaultScope(ctx context.Context, arg SetNodeDefaultScopeParams) error
|
||||
SetNodeMultibytePaths(ctx context.Context, id uuid.UUID) error
|
||||
SetNodeMultibyteTraces(ctx context.Context, id uuid.UUID) error
|
||||
@@ -228,9 +232,8 @@ type Querier interface {
|
||||
// ============================================================
|
||||
// ROUTES
|
||||
// ============================================================
|
||||
// Inserts or updates a known route (all hops resolved to high confidence).
|
||||
// node_ids and hash_prefix are ordered arrays of the resolved node UUIDs and
|
||||
// their hash bytes. last_seen is bumped on conflict.
|
||||
// Route identity is path_key, an md5 of node_ids computed by the caller.
|
||||
// On conflict, observation_count and last_seen are bumped.
|
||||
UpsertKnownRoute(ctx context.Context, arg UpsertKnownRouteParams) error
|
||||
// ============================================================
|
||||
// NODES
|
||||
|
||||
+117
-36
@@ -51,6 +51,25 @@ func (q *Queries) DeleteOldPackets(ctx context.Context, lastHeardAt pgtype.Times
|
||||
return err
|
||||
}
|
||||
|
||||
const deleteOldRoutes = `-- name: DeleteOldRoutes :exec
|
||||
DELETE FROM known_routes
|
||||
WHERE last_seen < $1
|
||||
OR (observation_count < $2 AND last_seen < $3)
|
||||
`
|
||||
|
||||
type DeleteOldRoutesParams struct {
|
||||
LastSeen pgtype.Timestamptz `json:"last_seen"`
|
||||
ObservationCount int64 `json:"observation_count"`
|
||||
LastSeen_2 pgtype.Timestamptz `json:"last_seen_2"`
|
||||
}
|
||||
|
||||
// Deletes routes not observed since the retention cutoff ($1), and rarely-observed
|
||||
// routes (observation_count < $2) not observed since the grace cutoff ($3).
|
||||
func (q *Queries) DeleteOldRoutes(ctx context.Context, arg DeleteOldRoutesParams) error {
|
||||
_, err := q.db.Exec(ctx, deleteOldRoutes, arg.LastSeen, arg.ObservationCount, arg.LastSeen_2)
|
||||
return err
|
||||
}
|
||||
|
||||
const deleteOldTelemetry = `-- name: DeleteOldTelemetry :exec
|
||||
DELETE FROM observer_telemetry WHERE reported_at < $1
|
||||
`
|
||||
@@ -237,15 +256,26 @@ type GetKnownRoutesByNodeParams struct {
|
||||
Column2 uuid.UUID `json:"column_2"`
|
||||
}
|
||||
|
||||
func (q *Queries) GetKnownRoutesByNode(ctx context.Context, arg GetKnownRoutesByNodeParams) ([]KnownRoute, error) {
|
||||
type GetKnownRoutesByNodeRow struct {
|
||||
ID int64 `json:"id"`
|
||||
NodeIds []uuid.UUID `json:"node_ids"`
|
||||
HashPrefix [][]byte `json:"hash_prefix"`
|
||||
Iata string `json:"iata"`
|
||||
HopCount int32 `json:"hop_count"`
|
||||
FirstSeen pgtype.Timestamptz `json:"first_seen"`
|
||||
LastSeen pgtype.Timestamptz `json:"last_seen"`
|
||||
ObservationCount int64 `json:"observation_count"`
|
||||
}
|
||||
|
||||
func (q *Queries) GetKnownRoutesByNode(ctx context.Context, arg GetKnownRoutesByNodeParams) ([]GetKnownRoutesByNodeRow, error) {
|
||||
rows, err := q.db.Query(ctx, getKnownRoutesByNode, arg.Iata, arg.Column2)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
items := []KnownRoute{}
|
||||
items := []GetKnownRoutesByNodeRow{}
|
||||
for rows.Next() {
|
||||
var i KnownRoute
|
||||
var i GetKnownRoutesByNodeRow
|
||||
if err := rows.Scan(
|
||||
&i.ID,
|
||||
&i.NodeIds,
|
||||
@@ -2095,7 +2125,18 @@ type ListKnownRoutesParams struct {
|
||||
Limit int32 `json:"limit"`
|
||||
}
|
||||
|
||||
func (q *Queries) ListKnownRoutes(ctx context.Context, arg ListKnownRoutesParams) ([]KnownRoute, error) {
|
||||
type ListKnownRoutesRow struct {
|
||||
ID int64 `json:"id"`
|
||||
NodeIds []uuid.UUID `json:"node_ids"`
|
||||
HashPrefix [][]byte `json:"hash_prefix"`
|
||||
Iata string `json:"iata"`
|
||||
HopCount int32 `json:"hop_count"`
|
||||
FirstSeen pgtype.Timestamptz `json:"first_seen"`
|
||||
LastSeen pgtype.Timestamptz `json:"last_seen"`
|
||||
ObservationCount int64 `json:"observation_count"`
|
||||
}
|
||||
|
||||
func (q *Queries) ListKnownRoutes(ctx context.Context, arg ListKnownRoutesParams) ([]ListKnownRoutesRow, error) {
|
||||
rows, err := q.db.Query(ctx, listKnownRoutes,
|
||||
arg.Column1,
|
||||
arg.Column2,
|
||||
@@ -2106,9 +2147,9 @@ func (q *Queries) ListKnownRoutes(ctx context.Context, arg ListKnownRoutesParams
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
items := []KnownRoute{}
|
||||
items := []ListKnownRoutesRow{}
|
||||
for rows.Next() {
|
||||
var i KnownRoute
|
||||
var i ListKnownRoutesRow
|
||||
if err := rows.Scan(
|
||||
&i.ID,
|
||||
&i.NodeIds,
|
||||
@@ -3197,31 +3238,59 @@ func (q *Queries) ReconfirmNeighbors(ctx context.Context) error {
|
||||
}
|
||||
|
||||
const reconfirmRoutes = `-- name: ReconfirmRoutes :exec
|
||||
DELETE FROM known_routes kr
|
||||
WHERE EXISTS (
|
||||
SELECT 1
|
||||
FROM unnest(kr.node_ids) AS hop_node_id
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM node_short_ids ns
|
||||
WHERE ns.node_id = hop_node_id
|
||||
AND ns.iata = kr.iata
|
||||
WITH batch AS (
|
||||
SELECT iata, path_key, node_ids, hash_prefix
|
||||
FROM known_routes
|
||||
ORDER BY last_reconfirmed_at
|
||||
LIMIT $1
|
||||
),
|
||||
amb AS MATERIALIZED (
|
||||
SELECT iata, 1 AS len, prefix_1 AS p FROM node_short_ids GROUP BY iata, prefix_1 HAVING COUNT(*) > 1
|
||||
UNION ALL
|
||||
SELECT iata, 2, prefix_2 FROM node_short_ids GROUP BY iata, prefix_2 HAVING COUNT(*) > 1
|
||||
UNION ALL
|
||||
SELECT iata, 3, prefix_3 FROM node_short_ids GROUP BY iata, prefix_3 HAVING COUNT(*) > 1
|
||||
UNION ALL
|
||||
SELECT iata, 4, prefix_4 FROM node_short_ids GROUP BY iata, prefix_4 HAVING COUNT(*) > 1
|
||||
),
|
||||
dead AS (
|
||||
SELECT b.iata, b.path_key
|
||||
FROM batch b
|
||||
WHERE EXISTS (
|
||||
SELECT 1
|
||||
FROM unnest(b.node_ids) AS hop_node_id
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM node_short_ids ns
|
||||
WHERE ns.node_id = hop_node_id
|
||||
AND ns.iata = b.iata
|
||||
)
|
||||
)
|
||||
UNION
|
||||
SELECT DISTINCT b.iata, b.path_key
|
||||
FROM batch b
|
||||
CROSS JOIN LATERAL unnest(b.hash_prefix) AS hp
|
||||
JOIN amb a ON a.iata = b.iata AND a.len = length(hp) AND a.p = hp
|
||||
),
|
||||
deleted AS (
|
||||
DELETE FROM known_routes kr
|
||||
USING dead d
|
||||
WHERE kr.iata = d.iata AND kr.path_key = d.path_key
|
||||
)
|
||||
OR EXISTS (
|
||||
SELECT 1
|
||||
FROM unnest(kr.hash_prefix) AS hop_prefix
|
||||
WHERE (
|
||||
SELECT COUNT(*) FROM node_short_ids ns
|
||||
WHERE ns.iata = kr.iata
|
||||
AND ns.prefix_4 = hop_prefix
|
||||
) > 1
|
||||
)
|
||||
UPDATE known_routes kr
|
||||
SET last_reconfirmed_at = NOW()
|
||||
FROM batch b
|
||||
WHERE kr.iata = b.iata AND kr.path_key = b.path_key
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM dead d
|
||||
WHERE d.iata = b.iata AND d.path_key = b.path_key
|
||||
)
|
||||
`
|
||||
|
||||
// Delete known_routes where any hop node has departed from node_short_ids for
|
||||
// that IATA, or where any hop's prefix_4 is now ambiguous (matches >1 node).
|
||||
func (q *Queries) ReconfirmRoutes(ctx context.Context) error {
|
||||
_, err := q.db.Exec(ctx, reconfirmRoutes)
|
||||
// Checks the $1 least-recently-reconfirmed routes: deletes those with a departed
|
||||
// hop node or a hop prefix now matching >1 node in that IATA (length-aware:
|
||||
// 1/2/3/4-byte hop prefixes check prefix_1/2/3/4), and stamps the survivors.
|
||||
func (q *Queries) ReconfirmRoutes(ctx context.Context, limit int32) error {
|
||||
_, err := q.db.Exec(ctx, reconfirmRoutes, limit)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -3512,17 +3581,28 @@ type SearchKnownRoutesParams struct {
|
||||
Column3 []byte `json:"column_3"`
|
||||
}
|
||||
|
||||
type SearchKnownRoutesRow struct {
|
||||
ID int64 `json:"id"`
|
||||
NodeIds []uuid.UUID `json:"node_ids"`
|
||||
HashPrefix [][]byte `json:"hash_prefix"`
|
||||
Iata string `json:"iata"`
|
||||
HopCount int32 `json:"hop_count"`
|
||||
FirstSeen pgtype.Timestamptz `json:"first_seen"`
|
||||
LastSeen pgtype.Timestamptz `json:"last_seen"`
|
||||
ObservationCount int64 `json:"observation_count"`
|
||||
}
|
||||
|
||||
// Returns known routes containing a subsequence from source to destination hash prefix.
|
||||
// Verifies source appears before destination in the route.
|
||||
func (q *Queries) SearchKnownRoutes(ctx context.Context, arg SearchKnownRoutesParams) ([]KnownRoute, error) {
|
||||
func (q *Queries) SearchKnownRoutes(ctx context.Context, arg SearchKnownRoutesParams) ([]SearchKnownRoutesRow, error) {
|
||||
rows, err := q.db.Query(ctx, searchKnownRoutes, arg.Iata, arg.Column2, arg.Column3)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
items := []KnownRoute{}
|
||||
items := []SearchKnownRoutesRow{}
|
||||
for rows.Next() {
|
||||
var i KnownRoute
|
||||
var i SearchKnownRoutesRow
|
||||
if err := rows.Scan(
|
||||
&i.ID,
|
||||
&i.NodeIds,
|
||||
@@ -3883,14 +3963,15 @@ func (q *Queries) UpsertIATADetails(ctx context.Context, arg UpsertIATADetailsPa
|
||||
|
||||
const upsertKnownRoute = `-- name: UpsertKnownRoute :exec
|
||||
|
||||
INSERT INTO known_routes (node_ids, hash_prefix, iata, hop_count)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
ON CONFLICT (node_ids, iata) DO UPDATE SET
|
||||
INSERT INTO known_routes (path_key, node_ids, hash_prefix, iata, hop_count)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
ON CONFLICT (iata, path_key) DO UPDATE SET
|
||||
last_seen = NOW(),
|
||||
observation_count = known_routes.observation_count + 1
|
||||
`
|
||||
|
||||
type UpsertKnownRouteParams struct {
|
||||
PathKey []byte `json:"path_key"`
|
||||
NodeIds []uuid.UUID `json:"node_ids"`
|
||||
HashPrefix [][]byte `json:"hash_prefix"`
|
||||
Iata string `json:"iata"`
|
||||
@@ -3900,11 +3981,11 @@ type UpsertKnownRouteParams struct {
|
||||
// ============================================================
|
||||
// ROUTES
|
||||
// ============================================================
|
||||
// Inserts or updates a known route (all hops resolved to high confidence).
|
||||
// node_ids and hash_prefix are ordered arrays of the resolved node UUIDs and
|
||||
// their hash bytes. last_seen is bumped on conflict.
|
||||
// Route identity is path_key, an md5 of node_ids computed by the caller.
|
||||
// On conflict, observation_count and last_seen are bumped.
|
||||
func (q *Queries) UpsertKnownRoute(ctx context.Context, arg UpsertKnownRouteParams) error {
|
||||
_, err := q.db.Exec(ctx, upsertKnownRoute,
|
||||
arg.PathKey,
|
||||
arg.NodeIds,
|
||||
arg.HashPrefix,
|
||||
arg.Iata,
|
||||
|
||||
@@ -73,15 +73,23 @@ func CleanupTask(store *db.Store, telemetryRetention, packetRetention, nodeDelet
|
||||
}
|
||||
}
|
||||
|
||||
// ReconfirmTask returns a Task that prunes stale and ambiguous resolved paths
|
||||
// and neighbors. Runs after routes to ensure neighbors are cleaned against
|
||||
// already-reconfirmed path data.
|
||||
func ReconfirmTask(store *db.Store, interval time.Duration) Task {
|
||||
// reconfirmBatchSize bounds per-tick reconfirm work; at hourly ticks a 16M-row
|
||||
// table gets fully re-checked roughly daily.
|
||||
const reconfirmBatchSize = 750_000
|
||||
|
||||
// ReconfirmTask returns a Task that prunes aged routes first, then reconfirms
|
||||
// stale and ambiguous resolved paths and neighbors, so known_routes only ever
|
||||
// has one writer at a time.
|
||||
func ReconfirmTask(store *db.Store, routeRetention, routeGrace time.Duration, routeMinObservations int64, interval time.Duration) Task {
|
||||
return Task{
|
||||
Name: "reconfirm",
|
||||
Interval: interval,
|
||||
Run: func(ctx context.Context) error {
|
||||
if err := store.ReconfirmRoutes(ctx); err != nil {
|
||||
now := time.Now()
|
||||
if err := store.DeleteOldRoutes(ctx, now.Add(-routeRetention), routeMinObservations, now.Add(-routeGrace)); err != nil {
|
||||
return fmt.Errorf("route retention: %w", err)
|
||||
}
|
||||
if err := store.ReconfirmRoutes(ctx, reconfirmBatchSize); err != nil {
|
||||
return fmt.Errorf("routes: %w", err)
|
||||
}
|
||||
if err := store.ReconfirmNeighbors(ctx); err != nil {
|
||||
|
||||
+45
-16
@@ -22,6 +22,7 @@ type Config struct {
|
||||
Telemetry TelemetryConfig `yaml:"telemetry"`
|
||||
WebSocket WebSocketConfig `yaml:"websocket"`
|
||||
Packets PacketsConfig `yaml:"packets"`
|
||||
Routes RoutesConfig `yaml:"routes"`
|
||||
Ingest IngestFilterConfig `yaml:"ingest"`
|
||||
Scopes []ScopeConfig `yaml:"scopes"`
|
||||
Cache CacheConfig `yaml:"cache"`
|
||||
@@ -33,13 +34,16 @@ type Config struct {
|
||||
|
||||
// ResolvedConfig holds all runtime configuration with defaults applied.
|
||||
type ResolvedConfig struct {
|
||||
TelemetryResolution time.Duration
|
||||
TelemetryRetention time.Duration
|
||||
PacketRetention time.Duration
|
||||
MaxConnsPerIP int
|
||||
ViewRefreshInterval time.Duration
|
||||
ReconfirmInterval time.Duration
|
||||
CleanupInterval time.Duration
|
||||
TelemetryResolution time.Duration
|
||||
TelemetryRetention time.Duration
|
||||
PacketRetention time.Duration
|
||||
RouteRetention time.Duration
|
||||
RouteGrace time.Duration
|
||||
RouteMinObservations int
|
||||
MaxConnsPerIP int
|
||||
ViewRefreshInterval time.Duration
|
||||
ReconfirmInterval time.Duration
|
||||
CleanupInterval time.Duration
|
||||
|
||||
PresenceFlushInterval time.Duration
|
||||
PresencePacketTTL time.Duration
|
||||
@@ -177,6 +181,19 @@ type PacketsConfig struct {
|
||||
Retention duration `yaml:"retention"`
|
||||
}
|
||||
|
||||
// RoutesConfig controls known-route retention behaviour.
|
||||
type RoutesConfig struct {
|
||||
// Retention is how long a route is kept after it was last observed.
|
||||
// Defaults to 336h (14 days) if not set.
|
||||
Retention duration `yaml:"retention"`
|
||||
// Grace is how long a route observed fewer than MinObservations times is
|
||||
// kept. Defaults to 168h (7 days) if not set.
|
||||
Grace duration `yaml:"grace"`
|
||||
// MinObservations is the observation count below which Grace applies
|
||||
// instead of Retention. Defaults to 3 if not set.
|
||||
MinObservations int `yaml:"min_observations"`
|
||||
}
|
||||
|
||||
// NodesConfig controls node-derived signal thresholds.
|
||||
type NodesConfig struct {
|
||||
// ClockDriftThreshold is the |device clock - server clock| magnitude, measured from a
|
||||
@@ -299,13 +316,16 @@ func Load(path string) (*Config, error) {
|
||||
// Resolve returns a ResolvedConfig with defaults applied for any zero values.
|
||||
func Resolve(cfg *Config) ResolvedConfig {
|
||||
r := ResolvedConfig{
|
||||
TelemetryResolution: cfg.Telemetry.Resolution.Duration,
|
||||
TelemetryRetention: cfg.Telemetry.Retention.Duration,
|
||||
PacketRetention: cfg.Packets.Retention.Duration,
|
||||
MaxConnsPerIP: cfg.WebSocket.MaxConnectionsPerIP,
|
||||
ViewRefreshInterval: cfg.Background.ViewRefresh.Duration,
|
||||
ReconfirmInterval: cfg.Background.Reconfirm.Duration,
|
||||
CleanupInterval: cfg.Background.Cleanup.Duration,
|
||||
TelemetryResolution: cfg.Telemetry.Resolution.Duration,
|
||||
TelemetryRetention: cfg.Telemetry.Retention.Duration,
|
||||
PacketRetention: cfg.Packets.Retention.Duration,
|
||||
RouteRetention: cfg.Routes.Retention.Duration,
|
||||
RouteGrace: cfg.Routes.Grace.Duration,
|
||||
RouteMinObservations: cfg.Routes.MinObservations,
|
||||
MaxConnsPerIP: cfg.WebSocket.MaxConnectionsPerIP,
|
||||
ViewRefreshInterval: cfg.Background.ViewRefresh.Duration,
|
||||
ReconfirmInterval: cfg.Background.Reconfirm.Duration,
|
||||
CleanupInterval: cfg.Background.Cleanup.Duration,
|
||||
|
||||
PresenceFlushInterval: cfg.Presence.FlushInterval.Duration,
|
||||
PresencePacketTTL: cfg.Presence.PacketTTL.Duration,
|
||||
@@ -323,6 +343,15 @@ func Resolve(cfg *Config) ResolvedConfig {
|
||||
if r.PacketRetention == 0 {
|
||||
r.PacketRetention = 30 * 24 * time.Hour
|
||||
}
|
||||
if r.RouteRetention == 0 {
|
||||
r.RouteRetention = 14 * 24 * time.Hour
|
||||
}
|
||||
if r.RouteGrace == 0 {
|
||||
r.RouteGrace = 7 * 24 * time.Hour
|
||||
}
|
||||
if r.RouteMinObservations == 0 {
|
||||
r.RouteMinObservations = 3
|
||||
}
|
||||
if r.MaxConnsPerIP == 0 {
|
||||
r.MaxConnsPerIP = 5
|
||||
}
|
||||
@@ -357,8 +386,8 @@ func Resolve(cfg *Config) ResolvedConfig {
|
||||
|
||||
func (r ResolvedConfig) String() string {
|
||||
return fmt.Sprintf(
|
||||
"telemetryResolution=%s telemetryRetention=%s packetRetention=%s maxConnsPerIP=%d viewRefresh=%s reconfirm=%s cleanup=%s presenceFlush=%s presencePacketTTL=%s clockDriftThreshold=%s nodeStaleThreshold=%s nodeDeleteAfter=%s",
|
||||
r.TelemetryResolution, r.TelemetryRetention, r.PacketRetention,
|
||||
"telemetryResolution=%s telemetryRetention=%s packetRetention=%s routeRetention=%s routeGrace=%s routeMinObs=%d maxConnsPerIP=%d viewRefresh=%s reconfirm=%s cleanup=%s presenceFlush=%s presencePacketTTL=%s clockDriftThreshold=%s nodeStaleThreshold=%s nodeDeleteAfter=%s",
|
||||
r.TelemetryResolution, r.TelemetryRetention, r.PacketRetention, r.RouteRetention, r.RouteGrace, r.RouteMinObservations,
|
||||
r.MaxConnsPerIP, r.ViewRefreshInterval, r.ReconfirmInterval, r.CleanupInterval,
|
||||
r.PresenceFlushInterval, r.PresencePacketTTL, r.ClockDriftThreshold,
|
||||
r.NodeStaleThreshold, r.NodeDeleteAfter,
|
||||
|
||||
@@ -141,3 +141,16 @@ func TestResolvedConfig_String(t *testing.T) {
|
||||
t.Error("expected maxConnsPerIP in string")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolve_RouteDefaults(t *testing.T) {
|
||||
r := Resolve(&Config{})
|
||||
if r.RouteRetention != 336*time.Hour {
|
||||
t.Errorf("RouteRetention = %s, want 336h", r.RouteRetention)
|
||||
}
|
||||
if r.RouteGrace != 168*time.Hour {
|
||||
t.Errorf("RouteGrace = %s, want 168h", r.RouteGrace)
|
||||
}
|
||||
if r.RouteMinObservations != 3 {
|
||||
t.Errorf("RouteMinObservations = %d, want 3", r.RouteMinObservations)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user