dandri/beacon-server
1
0
Fork 0
mirror of https://github.com/MeshCore-Beacon/beacon-server.git synced 2026-09-01 16:48:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf(channels): track channel activity per IATA in its own table

Browse Source 
The IATA filter ran a correlated EXISTS over packets/observations with
ILIKE, which skipped the iata index and took ~7s live. Keep a small
channel_iatas table at ingest (like node_iatas) and filter against it.
Also honor the iatas= param so multi-site regions stop getting the
global list. Filter now ages out with packet retention rather than
matching any retained packet.
This commit is contained in:
MrAlders0n
2026-07-24 12:07:05 -07:00
committed by Ded
parent b33ebc007b
commit dff9c6cf00
22 changed files with 313 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
db/sqlc/mock/querier.go
+28
View File
@@ -43,6 +43,20 @@ func (m *MockQuerier) EXPECT() *MockQuerierMockRecorder {
return m.recorder
}
// DeleteOldChannelIATAs mocks base method.
func (m *MockQuerier) DeleteOldChannelIATAs(ctx context.Context, lastHeard pgtype.Timestamptz) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "DeleteOldChannelIATAs", ctx, lastHeard)
ret0, _ := ret[0].(error)
return ret0
}
// DeleteOldChannelIATAs indicates an expected call of DeleteOldChannelIATAs.
func (mr *MockQuerierMockRecorder) DeleteOldChannelIATAs(ctx, lastHeard any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteOldChannelIATAs", reflect.TypeOf((*MockQuerier)(nil).DeleteOldChannelIATAs), ctx, lastHeard)
}
// DeleteOldPackets mocks base method.
func (m *MockQuerier) DeleteOldPackets(ctx context.Context, lastHeardAt pgtype.Timestamptz) error {
m.ctrl.T.Helper()
@@ -1213,6 +1227,20 @@ func (mr *MockQuerierMockRecorder) UpsertChannelHashOnly(ctx, channelHash any) *
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertChannelHashOnly", reflect.TypeOf((*MockQuerier)(nil).UpsertChannelHashOnly), ctx, channelHash)
}
// UpsertChannelIATA mocks base method.
func (m *MockQuerier) UpsertChannelIATA(ctx context.Context, arg db.UpsertChannelIATAParams) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "UpsertChannelIATA", ctx, arg)
ret0, _ := ret[0].(error)
return ret0
}
// UpsertChannelIATA indicates an expected call of UpsertChannelIATA.
func (mr *MockQuerierMockRecorder) UpsertChannelIATA(ctx, arg any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertChannelIATA", reflect.TypeOf((*MockQuerier)(nil).UpsertChannelIATA), ctx, arg)
}
// UpsertIATA mocks base method.
func (m *MockQuerier) UpsertIATA(ctx context.Context, iata string) error {
m.ctrl.T.Helper()
db/sqlc/models.go
+6
View File
@@ -23,6 +23,12 @@ type Channel struct {
MessageCount *int64 `json:"message_count"`
}
type ChannelIata struct {
ChannelHash []byte `json:"channel_hash"`
Iata string `json:"iata"`
LastHeard pgtype.Timestamptz `json:"last_heard"`
}
type ChannelKey struct {
ChannelID int32 `json:"channel_id"`
KeyBytes []byte `json:"key_bytes"`
db/sqlc/querier.go
+5 -3
View File
@@ -12,6 +12,8 @@ import (
)
type Querier interface {
// Keeps the channel IATA filter in step with packet retention.
DeleteOldChannelIATAs(ctx context.Context, lastHeard pgtype.Timestamptz) error
// Deletes packets and their observations older than the given cutoff.
// packet_observations cascade-delete via FK.
DeleteOldPackets(ctx context.Context, lastHeardAt pgtype.Timestamptz) error
@@ -97,9 +99,8 @@ type Querier interface {
// Pass empty string for iata or scope to skip those filters.
// Pass cursor=0 to start from the beginning.
ListChannelMessagesByHash(ctx context.Context, arg ListChannelMessagesByHashParams) ([]ListChannelMessagesByHashRow, error)
// Returns channels ordered by last seen, optionally filtered by hash and/or IATA.
// Pass NULL for hash to skip hash filtering. Pass empty string for iata to skip IATA filtering.
// IATA filter returns channels that have active packets in that IATA (case-insensitive).
// Channels ordered by last seen, optionally filtered by hash and/or IATAs
// (membership via channel_iatas). NULL hash / empty array skip those filters.
// Pass cursor=0 to start from the beginning (cursor is last_seen epoch ms).
ListChannels(ctx context.Context, arg ListChannelsParams) ([]Channel, error)
ListIATAs(ctx context.Context) ([]IataCode, error)
@@ -181,6 +182,7 @@ type Querier interface {
// hash-only records (key unknown). Returns the channel row.
UpsertChannel(ctx context.Context, arg UpsertChannelParams) (Channel, error)
UpsertChannelHashOnly(ctx context.Context, channelHash []byte) (int32, error)
UpsertChannelIATA(ctx context.Context, arg UpsertChannelIATAParams) error
// Copyright 2026 Beacon Contributors
// SPDX-License-Identifier: agpl
// ============================================================
db/sqlc/queries.sql.go
+42 -17
View File
@@ -12,6 +12,16 @@ import (
"github.com/jackc/pgx/v5/pgtype"
)
const deleteOldChannelIATAs = `-- name: DeleteOldChannelIATAs :exec
DELETE FROM channel_iatas WHERE last_heard < $1
`
// Keeps the channel IATA filter in step with packet retention.
func (q *Queries) DeleteOldChannelIATAs(ctx context.Context, lastHeard pgtype.Timestamptz) error {
_, err := q.db.Exec(ctx, deleteOldChannelIATAs, lastHeard)
return err
}
const deleteOldPackets = `-- name: DeleteOldPackets :exec
DELETE FROM packets WHERE last_heard_at < $1
`
@@ -1883,13 +1893,11 @@ func (q *Queries) ListChannelMessagesByHash(ctx context.Context, arg ListChannel
}
const listChannels = `-- name: ListChannels :many
SELECT DISTINCT c.id, c.channel_hash, c.key_fingerprint, c.name, c.hashtag, c.is_hashtag, c.is_public, c.key_known, c.first_seen, c.last_seen, c.message_count FROM channels c
SELECT c.id, c.channel_hash, c.key_fingerprint, c.name, c.hashtag, c.is_hashtag, c.is_public, c.key_known, c.first_seen, c.last_seen, c.message_count FROM channels c
WHERE ($1::bytea IS NULL OR c.channel_hash = $1)
AND ($2 = '' OR EXISTS (
SELECT 1 FROM packets p
JOIN packet_observations po ON po.packet_hash = p.packet_hash
WHERE p.channel_hash = c.channel_hash
AND po.iata ILIKE $2
AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR c.channel_hash IN (
SELECT ci.channel_hash FROM channel_iatas ci
WHERE ci.iata = ANY($2::bpchar[])
))
AND ($3::timestamptz IS NULL OR c.last_seen < $3)
ORDER BY c.last_seen DESC
@@ -1897,22 +1905,21 @@ LIMIT $4
`
type ListChannelsParams struct {
Column1 []byte `json:"column_1"`
Column2 interface{} `json:"column_2"`
Column3 pgtype.Timestamptz `json:"column_3"`
Limit int32 `json:"limit"`
ChannelHash []byte `json:"channel_hash"`
Iatas []string `json:"iatas"`
CursorTs pgtype.Timestamptz `json:"cursor_ts"`
PageLimit int32 `json:"page_limit"`
}
// Returns channels ordered by last seen, optionally filtered by hash and/or IATA.
// Pass NULL for hash to skip hash filtering. Pass empty string for iata to skip IATA filtering.
// IATA filter returns channels that have active packets in that IATA (case-insensitive).
// Channels ordered by last seen, optionally filtered by hash and/or IATAs
// (membership via channel_iatas). NULL hash / empty array skip those filters.
// Pass cursor=0 to start from the beginning (cursor is last_seen epoch ms).
func (q *Queries) ListChannels(ctx context.Context, arg ListChannelsParams) ([]Channel, error) {
rows, err := q.db.Query(ctx, listChannels,
arg.Column1,
arg.Column2,
arg.Column3,
arg.Limit,
arg.ChannelHash,
arg.Iatas,
arg.CursorTs,
arg.PageLimit,
)
if err != nil {
return nil, err
@@ -3556,6 +3563,24 @@ func (q *Queries) UpsertChannelHashOnly(ctx context.Context, channelHash []byte)
return id, err
}
const upsertChannelIATA = `-- name: UpsertChannelIATA :exec
INSERT INTO channel_iatas (channel_hash, iata, last_heard)
VALUES ($1, $2, $3)
ON CONFLICT (channel_hash, iata) DO UPDATE SET
last_heard = GREATEST(channel_iatas.last_heard, EXCLUDED.last_heard)
`
type UpsertChannelIATAParams struct {
ChannelHash []byte `json:"channel_hash"`
Iata string `json:"iata"`
LastHeard pgtype.Timestamptz `json:"last_heard"`
}
func (q *Queries) UpsertChannelIATA(ctx context.Context, arg UpsertChannelIATAParams) error {
_, err := q.db.Exec(ctx, upsertChannelIATA, arg.ChannelHash, arg.Iata, arg.LastHeard)
return err
}
const upsertIATA = `-- name: UpsertIATA :exec