Re-hearing a hash-only channel with a nil fingerprint updates its row (#284)

This commit is contained in:
MrAlders0n authored and GitHub committed 2026-10-24 14:41:14 -04:00
1 parent 3d5df344c1
commit c3350953e5
11 files changed
+207 -9

No files matched your search

+1 -1
View File
@@ -69,7 +69,7 @@ jobs:
- name: Verify stats and endpoint queries against PostgreSQL 16
env:
BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable
run: go test ./db -run '^Test(Signal|Paths|PacketSummaries|PacketEndpointsResolveLive|ObserverMetrics|RouteEvidence|RouteEvidenceIndex|AnalyticsRetention|AnalyticsRetentionConcurrent|DeleteOldPacketsBatches|MeshMapperCatalogue|ChannelMessageScopes|NodeLocationReset|RunMigrations[A-Za-z]*|KnownRoutesIndexSet|PacketsDomainChecks|Reconfirm[A-Za-z]*|BigintCursors|RouteSearchHopOrder|StatsNodeTypesSplit|ListKnownRoutes|ListKnownRoutesMatchesLegacy|KnownRoutesCursorTies|KnownRoutesUnanalyzedRegion|KnownRoutesHopStatementsStayCustom|GlobalHopBranchesMatchToday|KnownRoutePaginationTies|RouteRetentionCutoffsExact|DeleteExpiredRoutes|DeleteGraceRoutesSlice|GraceWatermarkHeldByLockedRow|OpenPoolsSettings|PlanRouterClassCUnnamed|PlanRouterParallelWorkers|APIStoreHopStatementsUnnamed|PgStatStatementsMigrationGuarded|DBStatements|DBStatementsNoExtension|PgStatStatementsMigrationNoCreatableSchema|IndexHealthBaselineOnReindex|UnusedIndexes|PgActivity|RollHourSetsWorkMem|RollupWritesKeyOrder|SmallTableIndexSet|NodeReadersAfterIndexDrops|ClockDriftWideValues|ObserverChannelReadersAfterIndexDrops|UpsertChannelWriteThrough|UpsertChannelHashOnlyWriteThrough|ReadOnlyPoolsRejectWrites|ListPacketsGlobalMatchesLegacy|PacketsGlobalCursorTies|ListPacketsWindowInvertedRows|PacketsLatestObserverTie|NodesLegacyCursorMatchesToday|NodesPageCursorTies|ObserversLegacyCursorMatchesToday|ObserversNextCursorFromLastSeen|ObserverAdvertsPartialIndex|ObserverComparisonMatchesToday|AdvertsWaitForPayloadTypeBackfill|StatsProbeMatchesToday|TraceTagsKeysetMatchesToday|TelemetrySinceMatchesCatchAll|TraceTagPagination|TraceTagPaginationTies|TraceSummarySurvivesMissedHearing|CmiMigrationsIndexSet|InsertChannelMessageSeed[A-Za-z]*|AddChannelMessageIATA|CmiSeedAndAddRace|CmiInitAndBackfill|CmiMessageRangeReconcile|CmiObservationRangeReconcile|CmiBackfillRetries23503|WaitForPreBootWriters|ChannelMessagesByIATAsMatchesLegacy|ChannelMessagesZeroHitRegion|CmiTrustLifecycle|CmiFirstVerification|PacketSitesMigrationsIndexSet|InsertObservationMaintainsPacketSites|InsertObservationResendWritesNothing|PacketSitesRaceMaxMerge|InsertObservationSiteFallback|PacketSitesInitAndBackfill|FillPacketSitesSliceIdempotent|FillPacketSitesSliceRetries23503|PacketSitesReconcile|PacketSitesAudit|PacketSitesAuditTakesNewestHearing|PacketsBySitesMatchesTruth|PacketsBySitesCursorTies|PacketsBySitesManyBogusSites|PacketsBySitesSaturatedPagesFull|PacketsBySitesMatchesLegacy|PacketSitesTrustLifecycle|PacketsBySitesBounded|PacketsBySitesFallbackKeepsPageCursor|SweepPacketSitesFrontsPlan|IndexHealthPacketSitesFreshBaseline|PacketSitesFirstVerification|PacketSitesBackfillSlicesSparseIDs|PacketSitesBackfillResumesAcrossGaps|PacketSitesProgressCountsRows|PacketSitesAuditSamplesRows|PacketSitesVerifySamplesRows|PacedBackfillSlices|FillSessionsPlanCustom|FillAnalyzeFailureIsIgnored|FillAnalyzeOnlyNeverAnalyzed|HygieneIndexSet|HygieneReadersAfterIndexDrops|KnownRoutesColumns|SequenceUse|SlowPoolIsolation|SlowPoolSize|SequenceUseMixedCaseSchema|ObservationsIndexSet|ReadersAfterIndexDrops)Postgres$' -count=1 -v
run: go test ./db -run '^Test(Signal|Paths|PacketSummaries|PacketEndpointsResolveLive|ObserverMetrics|RouteEvidence|RouteEvidenceIndex|AnalyticsRetention|AnalyticsRetentionConcurrent|DeleteOldPacketsBatches|MeshMapperCatalogue|ChannelMessageScopes|NodeLocationReset|RunMigrations[A-Za-z]*|KnownRoutesIndexSet|PacketsDomainChecks|Reconfirm[A-Za-z]*|BigintCursors|RouteSearchHopOrder|StatsNodeTypesSplit|ListKnownRoutes|ListKnownRoutesMatchesLegacy|KnownRoutesCursorTies|KnownRoutesUnanalyzedRegion|KnownRoutesHopStatementsStayCustom|GlobalHopBranchesMatchToday|KnownRoutePaginationTies|RouteRetentionCutoffsExact|DeleteExpiredRoutes|DeleteGraceRoutesSlice|GraceWatermarkHeldByLockedRow|OpenPoolsSettings|PlanRouterClassCUnnamed|PlanRouterParallelWorkers|APIStoreHopStatementsUnnamed|PgStatStatementsMigrationGuarded|DBStatements|DBStatementsNoExtension|PgStatStatementsMigrationNoCreatableSchema|IndexHealthBaselineOnReindex|UnusedIndexes|PgActivity|RollHourSetsWorkMem|RollupWritesKeyOrder|SmallTableIndexSet|NodeReadersAfterIndexDrops|ClockDriftWideValues|ObserverChannelReadersAfterIndexDrops|UpsertChannelWriteThrough|UpsertChannelHashOnlyWriteThrough|ChannelHashOnlyRehear|ReadOnlyPoolsRejectWrites|ListPacketsGlobalMatchesLegacy|PacketsGlobalCursorTies|ListPacketsWindowInvertedRows|PacketsLatestObserverTie|NodesLegacyCursorMatchesToday|NodesPageCursorTies|ObserversLegacyCursorMatchesToday|ObserversNextCursorFromLastSeen|ObserverAdvertsPartialIndex|ObserverComparisonMatchesToday|AdvertsWaitForPayloadTypeBackfill|StatsProbeMatchesToday|TraceTagsKeysetMatchesToday|TelemetrySinceMatchesCatchAll|TraceTagPagination|TraceTagPaginationTies|TraceSummarySurvivesMissedHearing|CmiMigrationsIndexSet|InsertChannelMessageSeed[A-Za-z]*|AddChannelMessageIATA|CmiSeedAndAddRace|CmiInitAndBackfill|CmiMessageRangeReconcile|CmiObservationRangeReconcile|CmiBackfillRetries23503|WaitForPreBootWriters|ChannelMessagesByIATAsMatchesLegacy|ChannelMessagesZeroHitRegion|CmiTrustLifecycle|CmiFirstVerification|PacketSitesMigrationsIndexSet|InsertObservationMaintainsPacketSites|InsertObservationResendWritesNothing|PacketSitesRaceMaxMerge|InsertObservationSiteFallback|PacketSitesInitAndBackfill|FillPacketSitesSliceIdempotent|FillPacketSitesSliceRetries23503|PacketSitesReconcile|PacketSitesAudit|PacketSitesAuditTakesNewestHearing|PacketsBySitesMatchesTruth|PacketsBySitesCursorTies|PacketsBySitesManyBogusSites|PacketsBySitesSaturatedPagesFull|PacketsBySitesMatchesLegacy|PacketSitesTrustLifecycle|PacketsBySitesBounded|PacketsBySitesFallbackKeepsPageCursor|SweepPacketSitesFrontsPlan|IndexHealthPacketSitesFreshBaseline|PacketSitesFirstVerification|PacketSitesBackfillSlicesSparseIDs|PacketSitesBackfillResumesAcrossGaps|PacketSitesProgressCountsRows|PacketSitesAuditSamplesRows|PacketSitesVerifySamplesRows|PacedBackfillSlices|FillSessionsPlanCustom|FillAnalyzeFailureIsIgnored|FillAnalyzeOnlyNeverAnalyzed|HygieneIndexSet|HygieneReadersAfterIndexDrops|KnownRoutesColumns|SequenceUse|SlowPoolIsolation|SlowPoolSize|SequenceUseMixedCaseSchema|ObservationsIndexSet|ReadersAfterIndexDrops)Postgres$' -count=1 -v
- name: Verify the channel message site rule against PostgreSQL 16
env:
+15
View File
@@ -30,6 +30,21 @@ func (s *Store) UpsertChannel(ctx context.Context, channelHash []byte, keyFinger
if hashtag != "" {
hashtagPtr = &hashtag
}
if keyFingerprint == nil {
// The keyed statement's (hash, fingerprint) conflict target never matches a NULL fingerprint, so a second
// hearing would hit the hash-only unique index; use the hash-only statements.
if namePtr == nil && hashtagPtr == nil {
return s.UpsertChannelHashOnly(ctx, channelHash)
}
isHashtag := hashtag != ""
id, err := s.q.UpsertChannelHashOnlyNamed(ctx, sqlc.UpsertChannelHashOnlyNamedParams{
ChannelHash: channelHash, Name: namePtr, Hashtag: hashtagPtr, IsHashtag: &isHashtag,
})
if err != nil {
return 0, err
}
return int(id), nil
}
cached := s.channelIDs != nil && keyFingerprint != nil
key := channelKey{hash: string(channelHash), fp: string(keyFingerprint)}
if cached {
+91
View File
@@ -0,0 +1,91 @@
// Copyright 2026 Beacon Contributors
// SPDX-License-Identifier: AGPL-3.0-or-later
package db
import (
"testing"
)
// A hash-only channel (key unknown) heard again, through either upsert and with or without the id cache, updates its
// row: no error and no second row. With the id cache (as ingest runs), no sequence value is burned once the row exists.
func TestChannelHashOnlyRehearPostgres(t *testing.T) {
ctx, pool := schemaPool(t)
if err := RunMigrations(ctx, pool); err != nil {
t.Fatal(err)
}
seq := func() int64 {
t.Helper()
var v int64
if err := pool.QueryRow(ctx, `SELECT last_value FROM channels_id_seq`).Scan(&v); err != nil {
t.Fatal(err)
}
return v
}
rows := func(hash byte) (n int, ids []int64) {
t.Helper()
if err := pool.QueryRow(ctx, `SELECT count(*), coalesce(array_agg(id ORDER BY id), '{}') FROM channels
WHERE channel_hash = $1 AND key_fingerprint IS NULL`, []byte{hash}).Scan(&n, &ids); err != nil {
t.Fatal(err)
}
return n, ids
}
for _, c := range []struct {
name string
cache bool
}{{"no cache", false}, {"id cache", true}} {
t.Run(c.name, func(t *testing.T) {
s := New(pool, pool, 0, 0)
if c.cache {
s = New(pool, pool, 0, 0, WithChannelIDs(NewChannelIDCache()))
}
hash := byte(0x40)
if c.cache {
hash = 0x41
}
first, err := s.UpsertChannelHashOnly(ctx, []byte{hash})
if err != nil {
t.Fatal(err)
}
seq0 := seq()
for i := 0; i < 3; i++ {
if id, err := s.UpsertChannelHashOnly(ctx, []byte{hash}); err != nil || id != first {
t.Fatalf("UpsertChannelHashOnly again: id %d err %v, want %d", id, err, first)
}
if id, err := s.UpsertChannel(ctx, []byte{hash}, nil, "", ""); err != nil || id != first {
t.Fatalf("UpsertChannel with a nil fingerprint: id %d err %v, want %d", id, err, first)
}
}
if n, ids := rows(hash); n != 1 || ids[0] != int64(first) {
t.Errorf("hash-only rows %d %v, want one, id %d", n, ids, first)
}
if d := seq() - seq0; c.cache && d != 0 {
t.Errorf("re-hearing a known hash-only channel burned %d sequence values", d)
}
})
}
// Heard with a name and hashtag: the same row, the name kept as UpsertChannel keeps it.
s := New(pool, pool, 0, 0, WithChannelIDs(NewChannelIDCache()))
first, err := s.UpsertChannel(ctx, []byte{0x42}, nil, "named", "#tag")
if err != nil {
t.Fatal(err)
}
for _, name := range []string{"", "renamed", ""} {
if id, err := s.UpsertChannel(ctx, []byte{0x42}, nil, name, "#tag"); err != nil || id != first {
t.Fatalf("named hash-only again (%q): id %d err %v, want %d", name, id, err, first)
}
}
var name, hashtag string
var isHashtag, keyKnown bool
if err := pool.QueryRow(ctx, `SELECT name, hashtag, is_hashtag, key_known FROM channels WHERE id = $1`, first).
Scan(&name, &hashtag, &isHashtag, &keyKnown); err != nil {
t.Fatal(err)
}
if name != "renamed" || hashtag != "#tag" || !isHashtag || keyKnown {
t.Errorf("named hash-only row: %q %q %v %v, want renamed #tag true false", name, hashtag, isHashtag, keyKnown)
}
if n, _ := rows(0x42); n != 1 {
t.Errorf("named hash-only rows %d, want 1", n)
}
}
+23 -2
View File
@@ -456,8 +456,13 @@ func TestUpsertChannelNilFingerprintBypassesCache(t *testing.T) {
cache := NewChannelIDCache()
store := &Store{q: mock, channelIDs: cache}
mock.EXPECT().UpsertChannel(gomock.Any(), upsertParams([]byte{0x11}, nil, "pub", "")).
Return(sqlc.Channel{ID: 5}, nil).Times(2)
// A name or hashtag goes to the hash-only conflict target on every hearing, keeping the name; the keyed statement,
// whose (hash, fingerprint) target never matches a NULL fingerprint, is never used.
pub := "pub"
isHashtag := false
mock.EXPECT().UpsertChannelHashOnlyNamed(gomock.Any(), sqlc.UpsertChannelHashOnlyNamedParams{
ChannelHash: []byte{0x11}, Name: &pub, IsHashtag: &isHashtag}).Return(int32(5), nil).Times(2)
mock.EXPECT().UpsertChannel(gomock.Any(), gomock.Any()).Times(0)
mock.EXPECT().TouchChannel(gomock.Any(), gomock.Any()).Times(0)
mock.EXPECT().TouchChannelHashOnly(gomock.Any(), gomock.Any()).Times(0)
@@ -472,6 +477,22 @@ func TestUpsertChannelNilFingerprintBypassesCache(t *testing.T) {
}
}
// Without a name or hashtag, a nil fingerprint is a hash-only hearing: the hash-only upsert, then its write-through.
func TestUpsertChannelNilFingerprintIsHashOnly(t *testing.T) {
ctrl := gomock.NewController(t)
mock := mockdb.NewMockQuerier(ctrl)
store := &Store{q: mock, channelIDs: NewChannelIDCache()}
mock.EXPECT().UpsertChannelHashOnly(gomock.Any(), []byte{0x12}).Return(int32(8), nil).Times(1)
mock.EXPECT().TouchChannelHashOnly(gomock.Any(), sqlc.TouchChannelHashOnlyParams{ID: 8, ChannelHash: []byte{0x12}}).
Return(int32(8), nil).Times(1)
mock.EXPECT().UpsertChannel(gomock.Any(), gomock.Any()).Times(0)
for i := 0; i < 2; i++ {
if id, err := store.UpsertChannel(context.Background(), []byte{0x12}, nil, "", ""); err != nil || id != 8 {
t.Fatalf("call %d: %d, %v; want 8", i, id, err)
}
}
}
func TestUpsertChannelWithoutCacheUnchanged(t *testing.T) {
ctrl := gomock.NewController(t)
mock := mockdb.NewMockQuerier(ctrl)
@@ -0,0 +1,4 @@
[
"ModifyTable on channels",
" Result"
]
@@ -0,0 +1,4 @@
[
"ModifyTable on channels",
" Result"
]
+5
View File
@@ -84,6 +84,7 @@ statements:
DeleteExpiredRoutes: {sqlc: DeleteExpiredRoutes, kind: dml, plan: maint}
DeleteGraceRoutesSlice: {sqlc: DeleteGraceRoutesSlice, kind: dml, plan: maint}
UpsertChannel: {sqlc: UpsertChannel, kind: dml, plan: ingest}
UpsertChannelHashOnlyNamed: {sqlc: UpsertChannelHashOnlyNamed, kind: dml, plan: ingest}
UpsertNodeIATA: {sqlc: UpsertNodeIATA, kind: dml, plan: ingest}
TouchChannel: {sqlc: TouchChannel, kind: dml, plan: ingest}
TouchChannelHashOnly: {sqlc: TouchChannelHashOnly, kind: dml, plan: ingest}
@@ -3013,6 +3014,10 @@ entries:
stmt: UpsertChannel
shape: "UpsertChannel fallback for an unknown key (plain EXPLAIN; ingest)"
params: ["\\x0102", null, x, null, false]
- id: sm.write.upsert_channel_hash_only_named
stmt: UpsertChannelHashOnlyNamed
shape: "UpsertChannel for a hash-only channel heard with a name (plain EXPLAIN; ingest)"
params: ["\\x0102", x, null, false]
- id: sm.write.upsert_node_iata
stmt: UpsertNodeIATA
shape: "UpsertNodeIATA on the swapped node_iatas index set (plain EXPLAIN; ingest)"
+12 -2
View File
@@ -1530,8 +1530,8 @@ ON CONFLICT (node_id, iata) DO NOTHING;
-- ============================================================
-- name: UpsertChannel :one
-- Upsert a channel by (hash, key_fingerprint). Pass NULL fingerprint for
-- hash-only records (key unknown). Returns the channel row.
-- Upsert a keyed channel by (hash, key_fingerprint). Returns the channel row. A NULL fingerprint never matches that key,
-- so the store sends hash-only records (key unknown) to UpsertChannelHashOnly or UpsertChannelHashOnlyNamed.
INSERT INTO channels (channel_hash, key_fingerprint, name, hashtag, is_hashtag, key_known, last_seen)
VALUES ($1, $2::bytea, $3, $4, $5, ($2 IS NOT NULL), NOW())
ON CONFLICT (channel_hash, key_fingerprint) DO UPDATE SET
@@ -1546,6 +1546,16 @@ ON CONFLICT (channel_hash) WHERE key_fingerprint IS NULL DO UPDATE SET
last_seen = NOW()
RETURNING id;
-- name: UpsertChannelHashOnlyNamed :one
-- UpsertChannel for a hash-only channel heard with a name or hashtag: the hash-only conflict target, and
-- UpsertChannel's insert and conflict update.
INSERT INTO channels (channel_hash, name, hashtag, is_hashtag, key_known, last_seen)
VALUES (sqlc.arg(channel_hash), sqlc.narg(name), sqlc.narg(hashtag), sqlc.arg(is_hashtag), false, NOW())
ON CONFLICT (channel_hash) WHERE key_fingerprint IS NULL DO UPDATE SET
last_seen = NOW(),
name = COALESCE(EXCLUDED.name, channels.name)
RETURNING id;
-- name: ListChannelIDs :many
-- Every channel key and its id, preloaded at boot for the ingest write-through.
SELECT id, channel_hash, key_fingerprint FROM channels;
+15
View File
@@ -3514,6 +3514,21 @@ func (mr *MockQuerierMockRecorder) UpsertChannelHashOnly(ctx, channelHash any) *
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertChannelHashOnly", reflect.TypeOf((*MockQuerier)(nil).UpsertChannelHashOnly), ctx, channelHash)
}
// UpsertChannelHashOnlyNamed mocks base method.
func (m *MockQuerier) UpsertChannelHashOnlyNamed(ctx context.Context, arg db.UpsertChannelHashOnlyNamedParams) (int32, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "UpsertChannelHashOnlyNamed", ctx, arg)
ret0, _ := ret[0].(int32)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// UpsertChannelHashOnlyNamed indicates an expected call of UpsertChannelHashOnlyNamed.
func (mr *MockQuerierMockRecorder) UpsertChannelHashOnlyNamed(ctx, arg any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertChannelHashOnlyNamed", reflect.TypeOf((*MockQuerier)(nil).UpsertChannelHashOnlyNamed), ctx, arg)
}
// UpsertChannelIATA mocks base method.
func (m *MockQuerier) UpsertChannelIATA(ctx context.Context, arg db.UpsertChannelIATAParams) error {
m.ctrl.T.Helper()
+5 -2
View File
@@ -581,10 +581,13 @@ type Querier interface {
// ============================================================
// CHANNELS
// ============================================================
// Upsert a channel by (hash, key_fingerprint). Pass NULL fingerprint for
// hash-only records (key unknown). Returns the channel row.
// Upsert a keyed channel by (hash, key_fingerprint). Returns the channel row. A NULL fingerprint never matches that key,
// so the store sends hash-only records (key unknown) to UpsertChannelHashOnly or UpsertChannelHashOnlyNamed.
UpsertChannel(ctx context.Context, arg UpsertChannelParams) (Channel, error)
UpsertChannelHashOnly(ctx context.Context, channelHash []byte) (int32, error)
// UpsertChannel for a hash-only channel heard with a name or hashtag: the hash-only conflict target, and
// UpsertChannel's insert and conflict update.
UpsertChannelHashOnlyNamed(ctx context.Context, arg UpsertChannelHashOnlyNamedParams) (int32, error)
// Refreshes at most hourly so repeat hears don't churn the row.
UpsertChannelIATA(ctx context.Context, arg UpsertChannelIATAParams) error
// Copyright 2026 Beacon Contributors
+32 -2
View File
@@ -7610,8 +7610,8 @@ type UpsertChannelParams struct {
// ============================================================
// CHANNELS
// ============================================================
// Upsert a channel by (hash, key_fingerprint). Pass NULL fingerprint for
// hash-only records (key unknown). Returns the channel row.
// Upsert a keyed channel by (hash, key_fingerprint). Returns the channel row. A NULL fingerprint never matches that key,
// so the store sends hash-only records (key unknown) to UpsertChannelHashOnly or UpsertChannelHashOnlyNamed.
func (q *Queries) UpsertChannel(ctx context.Context, arg UpsertChannelParams) (Channel, error) {
row := q.db.QueryRow(ctx, upsertChannel,
arg.ChannelHash,
@@ -7652,6 +7652,36 @@ func (q *Queries) UpsertChannelHashOnly(ctx context.Context, channelHash []byte)
return id, err
}
const upsertChannelHashOnlyNamed = `-- name: UpsertChannelHashOnlyNamed :one
INSERT INTO channels (channel_hash, name, hashtag, is_hashtag, key_known, last_seen)
VALUES ($1, $2, $3, $4, false, NOW())
ON CONFLICT (channel_hash) WHERE key_fingerprint IS NULL DO UPDATE SET
last_seen = NOW(),
name = COALESCE(EXCLUDED.name, channels.name)
RETURNING id
`
type UpsertChannelHashOnlyNamedParams struct {
ChannelHash []byte `json:"channel_hash"`
Name *string `json:"name"`
Hashtag *string `json:"hashtag"`
IsHashtag *bool `json:"is_hashtag"`
}
// UpsertChannel for a hash-only channel heard with a name or hashtag: the hash-only conflict target, and
// UpsertChannel's insert and conflict update.
func (q *Queries) UpsertChannelHashOnlyNamed(ctx context.Context, arg UpsertChannelHashOnlyNamedParams) (int32, error) {
row := q.db.QueryRow(ctx, upsertChannelHashOnlyNamed,
arg.ChannelHash,
arg.Name,
arg.Hashtag,
arg.IsHashtag,
)
var id int32
err := row.Scan(&id)
return id, err
}
const upsertChannelIATA = `-- name: UpsertChannelIATA :exec
INSERT INTO channel_iatas (channel_hash, iata, last_heard)
VALUES ($1, $2, $3)