diff --git a/cmd/server/chunked_load.go b/cmd/server/chunked_load.go index b64e056f..ae5345c7 100644 --- a/cmd/server/chunked_load.go +++ b/cmd/server/chunked_load.go @@ -436,12 +436,21 @@ func (s *PacketStore) LoadChunked(chunkSize int) error { // Acquire before opening the cursor: a feed waiting for the same // connection must never hold this gate while we hold its cursor. + // Held from the query to the end of the merge, it also keeps the + // poller from losing an observation committed after this chunk's + // snapshot: the poller reads its observation cursor bound before + // refreshAdvertEvidence, which waits here, so such an observation is + // either read after its transmission is merged or lies above the + // bound (see Poller.Start). s.advertEvidenceMu.Lock() rows, err := s.db.conn.Query(chunkSQL) if err != nil { s.advertEvidenceMu.Unlock() return fmt.Errorf("chunk %d: query: %w", chunkIdx, err) } + if s.chunkQueriedHook != nil { + s.chunkQueriedHook(chunkIdx + 1) + } chunkTxCount, lastID, err := s.scanAndMergeChunk(rows, relayPM, &coldLoadAmbiguousHopsSkipped) rows.Close() diff --git a/cmd/server/db.go b/cmd/server/db.go index a3c628d1..572c24ea 100644 --- a/cmd/server/db.go +++ b/cmd/server/db.go @@ -2509,6 +2509,24 @@ func (db *DB) GetMaxTransmissionID() int { return maxID } +// highWaterMarks reads MAX(id) of transmissions and observations. Unlike +// GetMaxTransmissionID it reports failure: a 0 from a failed query would +// otherwise replay the whole table. ok is false without a connection. +func (db *DB) highWaterMarks() (txMax, obsMax int, ok bool) { + if db == nil || db.conn == nil { + return 0, 0, false + } + if err := db.conn.QueryRow("SELECT COALESCE(MAX(id), 0) FROM transmissions").Scan(&txMax); err != nil { + log.Printf("[store] startup high-water mark (transmissions): %v", err) + return 0, 0, false + } + if err := db.conn.QueryRow("SELECT COALESCE(MAX(id), 0) FROM observations").Scan(&obsMax); err != nil { + log.Printf("[store] startup high-water mark (observations): %v", err) + return 0, 0, false + } + return txMax, obsMax, true +} + // GetMaxObservationID returns the current max observation ID for polling. func (db *DB) GetMaxObservationID() int { var maxID int diff --git a/cmd/server/startup_poll_gap_test.go b/cmd/server/startup_poll_gap_test.go new file mode 100644 index 00000000..e5cef89c --- /dev/null +++ b/cmd/server/startup_poll_gap_test.go @@ -0,0 +1,664 @@ +package main + +import ( + "database/sql" + "encoding/json" + "fmt" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + _ "github.com/mattn/go-sqlite3" + "github.com/meshcore-analyzer/dbschema" +) + +// A transmission or observation the ingestor commits after the startup load's +// last chunk has read its snapshot, but before the poller initialises its +// cursor, must still reach the in-memory store. The poller seeds its cursor +// from a fresh MAX(id) taken when it starts (websocket.go Poller.Start), so a +// row committed in that window is above everything loaded yet not above the +// cursor: neither the load nor the poll ever reads it. +// +// The window is opened deterministically with OnChunkLoaded, which fires +// after a chunk's rows have been read and closed. With one chunk that is the +// last snapshot the load takes. + +func TestStartupLoadPollerGap_SQLite(t *testing.T) { + db, rw := openGapSQLite(t) + runStartupLoadPollerGap(t, db, rw) +} + +func runStartupLoadPollerGap(t *testing.T, db *DB, rw *sql.DB) { + t.Helper() + base := time.Now().UTC().Add(-10 * time.Minute).Truncate(time.Second) + iso := func(d time.Duration) string { return base.Add(d).Format("2006-01-02T15:04:05.000Z") } + mustExec := func(q string, args ...any) { + t.Helper() + if _, err := rw.Exec(q, args...); err != nil { + t.Fatalf("seed: %v\n%s", err, q) + } + } + addTx := func(id int) { + mustExec(`INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json, last_seen) + VALUES (?, ?, ?, ?, 1, 5, 1, '{"type":"GRP_TXT"}', ?)`, + id, fmt.Sprintf("11%02dAABBCCDD", id), gapHash(id), iso(time.Duration(id)*time.Second), base.Add(time.Duration(id)*time.Second).Unix()) + } + addObs := func(id, tx, observer int) { + mustExec(`INSERT INTO observations (id, transmission_id, observer_idx, direction, snr, rssi, score, path_json, timestamp) + VALUES (?, ?, ?, 'rx', 5.5, -90, 1, ?, ?)`, + id, tx, observer, fmt.Sprintf(`["%02X"]`, id), base.Add(time.Duration(id)*time.Second).Unix()) + } + for idx := 1; idx <= 3; idx++ { + mustExec(`INSERT INTO observers (rowid, id, name, iata, last_seen, first_seen) VALUES (?, ?, ?, 'SJC', ?, ?)`, + idx, fmt.Sprintf("%064X", idx), fmt.Sprintf("Obs-%d", idx), iso(0), iso(-time.Hour)) + } + // Committed before the load. + addTx(1) + addObs(1, 1, 1) + addTx(2) + addObs(2, 2, 1) + + store := NewPacketStore(db, &PacketStoreConfig{}) + fired := false + store.OnChunkLoaded(func(int, int) { + if fired { + return + } + fired = true + // Committed after the (only) chunk read its snapshot, before the + // poller starts: a new transmission with one observation, and a + // second observation of an already loaded transmission. + addTx(3) + addObs(3, 3, 1) + addObs(4, 1, 2) + }) + if err := store.RunStartupLoad(100); err != nil { + t.Fatalf("RunStartupLoad: %v", err) + } + if !fired { + t.Fatal("OnChunkLoaded never fired") + } + + loaded := loadedObsPaths(store) + pr := startGapPoller(t, db, store) + + time.Sleep(80 * time.Millisecond) + // Committed after the poller started: a later observation of tx 3 (the + // 12 observations of the lost transmission in the field), and a control + // transmission proving the poller itself works. + addObs(5, 3, 3) + addTx(4) + addObs(6, 4, 1) + + has := func() (tx3, tx4 bool, obs3, obs4, obs5 bool) { + store.mu.RLock() + defer store.mu.RUnlock() + _, tx3 = store.byHash[gapHash(3)] + _, tx4 = store.byHash[gapHash(4)] + _, obs3 = store.byObsID[3] + _, obs4 = store.byObsID[4] + _, obs5 = store.byObsID[5] + return + } + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + if _, tx4, _, _, _ := has(); tx4 { + break + } + time.Sleep(20 * time.Millisecond) + } + time.Sleep(100 * time.Millisecond) // a few more ticks for stragglers + tx3, tx4, obs3, obs4, obs5 := has() + if !tx4 { + t.Fatal("control: tx 4, committed after the poller started, never reached the store; the poller is not running") + } + if !tx3 { + t.Errorf("tx 3, committed between the load's snapshot and the poller's cursor init, is missing from the store (lost forever)") + } + if !obs3 || !obs5 { + t.Errorf("observations of tx 3 missing: obs3=%v obs5=%v", obs3, obs5) + } + if !obs4 { + t.Errorf("obs 4 (new observation of already-loaded tx 1, committed in the same window) is missing from the store") + } + pr.stop() + assertStoreMatchesDB(t, store, rw) + assertNoDuplicateBroadcasts(t, pr.drain(), loaded) + assertReplayIdempotent(t, store, rw) +} + +func gapHash(id int) string { return fmt.Sprintf("feed%012x", id) } + +// A chunk-1 transmission can gain an observation after chunk 1 has read its +// snapshot whose id is BELOW an observation a later chunk loads (that one +// belongs to a chunk-2 transmission and was committed just after it). Using +// the store's highest loaded observation id as the poller's floor would then +// skip it. Three chunks of two: {1,2} {3,4} {5}; the last is short, so its +// callback runs after the load's final snapshot. + +func TestStartupLoadPollerGapMultiChunk_SQLite(t *testing.T) { + db, rw := openGapSQLite(t) + runStartupLoadPollerGapMultiChunk(t, db, rw) +} + +func openGapSQLite(t *testing.T) (*DB, *sql.DB) { + t.Helper() + path := filepath.Join(t.TempDir(), "gap.db") + rw, err := sql.Open("sqlite3", path+"?_journal_mode=WAL&_foreign_keys=on") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { rw.Close() }) + rw.SetMaxOpenConns(1) + // The tables the load and the poller read, as the ingestor creates them + // (cmd/ingestor/db.go), then the server-side migrations it applies on top. + if _, err := rw.Exec(gapBaseSchema); err != nil { + t.Fatalf("create base schema: %v", err) + } + if err := dbschema.Apply(rw, nil); err != nil { + t.Fatalf("dbschema.Apply: %v", err) + } + db, err := OpenDB(path) + if err != nil { + t.Fatalf("OpenDB: %v", err) + } + t.Cleanup(func() { db.conn.Close() }) + return db, rw +} + +// gapBaseSchema is the ingestor's DDL for the tables these tests seed. Observers +// are inserted by rowid: observations.observer_idx refers to it. +const gapBaseSchema = ` + CREATE TABLE nodes ( + public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, + last_seen TEXT, first_seen TEXT, advert_count INTEGER DEFAULT 0, + battery_mv INTEGER, temperature_c REAL, foreign_advert INTEGER DEFAULT 0 + ); + CREATE TABLE observers ( + id TEXT PRIMARY KEY, name TEXT, iata TEXT, last_seen TEXT, first_seen TEXT, + packet_count INTEGER DEFAULT 0, model TEXT, firmware TEXT, client_version TEXT, + radio TEXT, battery_mv INTEGER, uptime_secs INTEGER, noise_floor REAL, + inactive INTEGER DEFAULT 0, last_packet_at TEXT DEFAULT NULL, + clock_skew_seconds INTEGER DEFAULT NULL, clock_skew_count_24h INTEGER DEFAULT 0, + clock_last_naive_at TEXT DEFAULT NULL, can_relay INTEGER DEFAULT 1, + can_relay_seen INTEGER DEFAULT 0 + ); + CREATE TABLE transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, raw_hex TEXT NOT NULL, + hash TEXT NOT NULL UNIQUE, first_seen TEXT NOT NULL, route_type INTEGER, + payload_type INTEGER, payload_version INTEGER, decoded_json TEXT, + from_pubkey TEXT, last_seen INTEGER NOT NULL DEFAULT 0, + created_at TEXT DEFAULT (datetime('now')), code1 TEXT, code2 TEXT + ); + CREATE TABLE observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + transmission_id INTEGER NOT NULL REFERENCES transmissions(id), + observer_idx INTEGER, direction TEXT, snr REAL, rssi REAL, score INTEGER, + path_json TEXT, timestamp INTEGER NOT NULL + ); + CREATE INDEX idx_observations_transmission_id ON observations(transmission_id); + CREATE INDEX idx_observations_observer_idx ON observations(observer_idx); + CREATE INDEX idx_observations_timestamp ON observations(timestamp); + CREATE TABLE schema_version (version INTEGER); +` + +func runStartupLoadPollerGapMultiChunk(t *testing.T, db *DB, rw *sql.DB) { + t.Helper() + base := time.Now().UTC().Add(-10 * time.Minute).Truncate(time.Second) + iso := func(d time.Duration) string { return base.Add(d).Format("2006-01-02T15:04:05.000Z") } + mustExec := func(q string, args ...any) { + t.Helper() + if _, err := rw.Exec(q, args...); err != nil { + t.Fatalf("seed: %v\n%s", err, q) + } + } + addTx := func(id int) { + mustExec(`INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json, last_seen) + VALUES (?, ?, ?, ?, 1, 5, 1, '{"type":"GRP_TXT"}', ?)`, + id, fmt.Sprintf("11%02dAABBCCDD", id), gapHash(id), iso(time.Duration(id)*time.Second), base.Add(time.Duration(id)*time.Second).Unix()) + } + // path_json is unique per observation, so none merges with another + // (dedup is by observer+path) and DB and store counts must agree. + addObs := func(id, tx, observer int) { + mustExec(`INSERT INTO observations (id, transmission_id, observer_idx, direction, snr, rssi, score, path_json, timestamp) + VALUES (?, ?, ?, 'rx', 5.5, -90, 1, ?, ?)`, + id, tx, observer, fmt.Sprintf(`["%02X"]`, id), base.Add(time.Duration(id)*time.Second).Unix()) + } + for idx := 1; idx <= 3; idx++ { + mustExec(`INSERT INTO observers (rowid, id, name, iata, last_seen, first_seen) VALUES (?, ?, ?, 'SJC', ?, ?)`, + idx, fmt.Sprintf("%064X", idx), fmt.Sprintf("Obs-%d", idx), iso(0), iso(-time.Hour)) + } + for id := 1; id <= 5; id++ { + addTx(id) + addObs(id, id, 1) + } + + store := NewPacketStore(db, &PacketStoreConfig{}) + chunk := 0 + store.OnChunkLoaded(func(int, int) { + chunk++ + switch chunk { + case 1: + // Between chunks 1 and 2: obs 10 of chunk-1 tx 1, then obs 11 + // of tx 3, which chunk 2 loads. The store's max obs id becomes + // 11 although 10 was never loaded. + addObs(10, 1, 2) + addObs(11, 3, 2) + case 3: + // After the final snapshot: a new transmission with one + // observation, and a new observation of loaded tx 2. + addTx(6) + addObs(12, 6, 1) + addObs(13, 2, 3) + } + }) + if err := store.RunStartupLoad(2); err != nil { + t.Fatalf("RunStartupLoad: %v", err) + } + if chunk != 3 { + t.Fatalf("expected 3 chunks, got %d", chunk) + } + loaded := loadedObsPaths(store) + if _, ok := loaded[11]; !ok { + t.Fatal("setup: obs 11 should have been loaded by chunk 2") + } + if _, ok := loaded[10]; ok { + t.Fatal("setup: obs 10 should not have been loaded (committed after chunk 1)") + } + + pr := startGapPoller(t, db, store) + time.Sleep(80 * time.Millisecond) + // After the poller started: a later observation of tx 6 and a control. + addObs(14, 6, 2) + addTx(7) + addObs(15, 7, 1) + + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + store.mu.RLock() + _, ok := store.byObsID[15] + store.mu.RUnlock() + if ok { + break + } + time.Sleep(20 * time.Millisecond) + } + time.Sleep(100 * time.Millisecond) // a few more ticks for stragglers + pr.stop() + + store.mu.RLock() + _, tx6 := store.byHash[gapHash(6)] + _, tx7 := store.byHash[gapHash(7)] + var missing []int + for _, id := range []int{10, 12, 13, 14, 15} { + if _, ok := store.byObsID[id]; !ok { + missing = append(missing, id) + } + } + store.mu.RUnlock() + if !tx7 { + t.Fatal("control: tx 7, committed after the poller started, never reached the store; the poller is not running") + } + if !tx6 { + t.Errorf("tx 6, committed after the load's final snapshot, is missing from the store") + } + if len(missing) > 0 { + t.Errorf("observations missing from the store: %v (10 is below the max loaded obs id 11)", missing) + } + assertStoreMatchesDB(t, store, rw) + assertNoDuplicateBroadcasts(t, pr.drain(), loaded) + assertReplayIdempotent(t, store, rw) +} + +type gapPoller struct { + client *Client + stop func() +} + +// startGapPoller runs a poller over store with one capturing hub client. +// stop waits for Start to return, so no tick races the assertions after it. +func startGapPoller(t *testing.T, db *DB, store *PacketStore) *gapPoller { + t.Helper() + hub := NewHub() + c := &Client{send: make(chan []byte, 1024), release: func() {}} + hub.mu.Lock() + hub.clients[c] = true + hub.mu.Unlock() + poller := NewPoller(db, hub, 20*time.Millisecond) + poller.store = store + done := make(chan struct{}) + go func() { + poller.Start() + close(done) + }() + var once sync.Once + stop := func() { + once.Do(func() { + poller.Stop() + <-done + }) + } + t.Cleanup(stop) + return &gapPoller{client: c, stop: stop} +} + +// drain returns every message broadcast so far. +func (p *gapPoller) drain() [][]byte { + var out [][]byte + for { + select { + case m := <-p.client.send: + out = append(out, m) + default: + return out + } + } +} + +// loadedObsPaths maps each observation id the store holds to its path_json. +func loadedObsPaths(store *PacketStore) map[int]string { + store.mu.RLock() + defer store.mu.RUnlock() + out := make(map[int]string, len(store.byObsID)) + for id, o := range store.byObsID { + out[id] = o.PathJSON + } + return out +} + +// assertStoreMatchesDB checks the store holds exactly the DB's transmissions +// and observations: nothing lost, nothing counted twice. +// notLoaded names transmissions outside the loaded window; they and their +// observations are left out of the DB side. +func assertStoreMatchesDB(t *testing.T, store *PacketStore, rw *sql.DB, notLoaded ...int) { + t.Helper() + txWhere, obsWhere := "", "" + if len(notLoaded) > 0 { + ids := strings.Trim(strings.Join(strings.Fields(fmt.Sprint(notLoaded)), ","), "[]") + txWhere = " WHERE id NOT IN (" + ids + ")" + obsWhere = " WHERE transmission_id NOT IN (" + ids + ")" + } + var dbTx, dbObs int + if err := rw.QueryRow(`SELECT COUNT(*) FROM transmissions` + txWhere).Scan(&dbTx); err != nil { + t.Fatal(err) + } + if err := rw.QueryRow(`SELECT COUNT(*) FROM observations` + obsWhere).Scan(&dbObs); err != nil { + t.Fatal(err) + } + store.mu.RLock() + defer store.mu.RUnlock() + sumObs, sumCount := 0, 0 + for _, tx := range store.packets { + sumObs += len(tx.Observations) + sumCount += tx.ObservationCount + } + if len(store.packets) != dbTx || len(store.byHash) != dbTx { + t.Errorf("store transmissions: packets=%d byHash=%d, DB has %d", len(store.packets), len(store.byHash), dbTx) + } + if store.totalObs != dbObs || len(store.byObsID) != dbObs || sumObs != dbObs || sumCount != dbObs { + t.Errorf("store observations: totalObs=%d byObsID=%d attached=%d ObservationCount=%d, DB has %d", + store.totalObs, len(store.byObsID), sumObs, sumCount, dbObs) + } +} + +// assertNoDuplicateBroadcasts checks the poller's catch-up over rows it +// re-read did not re-announce one the load already held, nor any row twice. +func assertNoDuplicateBroadcasts(t *testing.T, msgs [][]byte, loaded map[int]string) { + t.Helper() + loadedPaths := make(map[string]int, len(loaded)) + for id, p := range loaded { + loadedPaths[p] = id + } + seen := make(map[string]bool) + for _, raw := range msgs { + var m struct { + Type string `json:"type"` + Data struct { + Hash string `json:"hash"` + PathJSON string `json:"path_json"` + } `json:"data"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatalf("broadcast: %v", err) + } + key := m.Data.Hash + " " + m.Data.PathJSON + if seen[key] { + t.Errorf("broadcast twice: %s", key) + } + seen[key] = true + if id, ok := loadedPaths[m.Data.PathJSON]; ok { + t.Errorf("broadcast obs %d, which the startup load already held", id) + } + } + if len(msgs) == 0 { + t.Error("no broadcasts captured; the capturing client is not wired") + } +} + +// assertReplayIdempotent re-reads every row from id 0. A store that already +// holds them all must ingest nothing and announce nothing. notLoaded is as +// for assertStoreMatchesDB. +func assertReplayIdempotent(t *testing.T, store *PacketStore, rw *sql.DB, notLoaded ...int) { + t.Helper() + // IngestNewFromDB applies no retention window, so with transmissions + // outside it a replay would rightly load them; replay observations only. + if len(notLoaded) == 0 { + if txs, _ := store.IngestNewFromDB(0, 1000); len(txs) != 0 { + t.Errorf("replaying transmissions from 0 broadcast %d already-held packets", len(txs)) + } + } + if obs := store.IngestNewObservations(0, 1000); len(obs) != 0 { + t.Errorf("replaying observations from 0 broadcast %d already-held observations", len(obs)) + } + assertStoreMatchesDB(t, store, rw, notLoaded...) +} + +// gapSeeder writes rows the way the ingestor does, with explicit ids (one +// writer: ids commit in order). +type gapSeeder struct { + t *testing.T + rw *sql.DB + base time.Time +} + +func newGapSeeder(t *testing.T, rw *sql.DB) *gapSeeder { + g := &gapSeeder{t: t, rw: rw, base: time.Now().UTC().Add(-10 * time.Minute).Truncate(time.Second)} + for idx := 1; idx <= 3; idx++ { + g.exec(`INSERT INTO observers (rowid, id, name, iata, last_seen, first_seen) VALUES (?, ?, ?, 'SJC', ?, ?)`, + idx, fmt.Sprintf("%064X", idx), fmt.Sprintf("Obs-%d", idx), g.iso(g.base), g.iso(g.base.Add(-time.Hour))) + } + return g +} + +func (g *gapSeeder) iso(ts time.Time) string { return ts.Format("2006-01-02T15:04:05.000Z") } + +// exec may run on a load or poller goroutine, so it reports with Errorf. +func (g *gapSeeder) exec(q string, args ...any) { + if _, err := g.rw.Exec(q, args...); err != nil { + g.t.Errorf("seed: %v\n%s", err, q) + } +} + +// tx inserts a transmission seen at base+id seconds, or two hours ago when +// old, which puts it outside a one-hour retention window. +func (g *gapSeeder) tx(id int, old bool) { + seen := g.base.Add(time.Duration(id) * time.Second) + if old { + seen = g.base.Add(-2 * time.Hour) + } + g.exec(`INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json, last_seen) + VALUES (?, ?, ?, ?, 1, 5, 1, '{"type":"GRP_TXT"}', ?)`, + id, fmt.Sprintf("11%02dAABBCCDD", id), gapHash(id), g.iso(seen), seen.Unix()) +} + +// obs inserts an observation with a unique path, so none dedupes. +func (g *gapSeeder) obs(id, tx, observer int) { + g.exec(`INSERT INTO observations (id, transmission_id, observer_idx, direction, snr, rssi, score, path_json, timestamp) + VALUES (?, ?, ?, 'rx', 5.5, -90, 1, ?, ?)`, + id, tx, observer, fmt.Sprintf(`["%02X"]`, id), g.base.Add(time.Duration(id)*time.Second).Unix()) +} + +func gapWait(t *testing.T, ch <-chan struct{}, what string) bool { + select { + case <-ch: + return true + case <-time.After(5 * time.Second): + t.Errorf("timed out waiting for %s", what) + return false + } +} + +func storeHasObs(store *PacketStore, id int) bool { + store.mu.RLock() + defer store.mu.RUnlock() + _, ok := store.byObsID[id] + return ok +} + +func waitStoreObs(store *PacketStore, id int, within time.Duration) bool { + deadline := time.Now().Add(within) + for time.Now().Before(deadline) { + if storeHasObs(store, id) { + return true + } + time.Sleep(10 * time.Millisecond) + } + return storeHasObs(store, id) +} + +// The poller runs from the first chunk on, while later chunks still load. +// Obs 10 of chunk-2 tx 3 is committed after chunk 2's query and polled before +// chunk 2 merges, when tx 3 is not in the store yet. It must end up in the +// store exactly once. This pins an ordering, not a mechanism of its own: the +// poller reads its observation cursor bound before ingestNewObservations polls +// the advert feed, which waits on advertEvidenceMu that each chunk holds from +// its query to its merge. So obs 10 lies above the bound of the poll that +// skips it, and is read again once tx 3 is merged (or the chunk already saw +// it). Obs 11 belongs to a transmission outside retention; it is skipped and +// must not hold back the live observation committed after the load. + +func TestStartupObsDuringChunkMerge_SQLite(t *testing.T) { + db, rw := openGapSQLite(t) + runStartupObsDuringChunkMerge(t, db, rw) +} + +func runStartupObsDuringChunkMerge(t *testing.T, db *DB, rw *sql.DB) { + g := newGapSeeder(t, rw) + for id := 1; id <= 4; id++ { + g.tx(id, false) + g.obs(id, id, 1) + } + g.tx(5, true) // outside retention: never loaded + g.obs(5, 5, 1) + + store := NewPacketStore(db, &PacketStoreConfig{RetentionHours: 1}) + pollerAtFeeds := make(chan struct{}) + releasePoller := make(chan struct{}) + pollerMerged := make(chan struct{}) + var stage atomic.Int32 // 0 idle, 1 poller paused, 2 merge done + store.obsPollHook = func(st obsPollStage) { + switch { + case st == obsPollAfterFeeds && stage.CompareAndSwap(0, 1): + close(pollerAtFeeds) + gapWait(t, releasePoller, "chunk 2's query") + case st == obsPollDone && stage.CompareAndSwap(1, 2): + close(pollerMerged) + } + } + store.OnChunkLoaded(func(int, int) { + if store.loadProgressRows.Load() == 2 { // after chunk 1 + gapWait(t, pollerAtFeeds, "the poller's first tick") + } + }) + store.chunkQueriedHook = func(chunk int) { + if chunk != 2 { + return + } + g.obs(10, 3, 2) // of tx 3, which this chunk is about to merge + g.obs(11, 5, 2) // orphan: tx 5 is outside retention + close(releasePoller) + gapWait(t, pollerMerged, "the poller's merge") + } + + loadErr := make(chan error, 1) + go func() { loadErr <- store.RunStartupLoad(2) }() + <-store.FirstChunkReady() + pr := startGapPoller(t, db, store) + if err := <-loadErr; err != nil { + t.Fatalf("RunStartupLoad: %v", err) + } + if stage.Load() != 2 { + t.Fatal("setup: the poller never polled between chunk 2's query and its merge") + } + + // After the load: a live observation must not wait behind orphan 11. + g.obs(12, 1, 3) + if !waitStoreObs(store, 12, 3*time.Second) { + t.Fatal("live obs 12, committed after the load completed, never reached the store: the feed is blocked") + } + pr.stop() + if !storeHasObs(store, 10) { + t.Error("obs 10 (chunk-2 tx, committed after chunk 2's query, polled before its merge) is missing from the store") + } + if storeHasObs(store, 11) { + t.Error("obs 11 of a transmission outside retention is in the store") + } + assertStoreMatchesDB(t, store, rw, 5) + assertNoDuplicateBroadcasts(t, pr.drain(), nil) + assertReplayIdempotent(t, store, rw, 5) +} + +// A failed transmission or observation read must leave the poller's cursor +// where it was, so the same rows are read once the database recovers. The +// observation cursor's bound comes from its own query, which used to advance +// the cursor past rows the failed read never saw. Renaming observers fails +// both ingest reads (they join it) while the bound query still succeeds. + +func TestPollerRetriesFailedRead_SQLite(t *testing.T) { + db, rw := openGapSQLite(t) + runPollerRetriesFailedRead(t, db, rw) +} + +func runPollerRetriesFailedRead(t *testing.T, db *DB, rw *sql.DB) { + g := newGapSeeder(t, rw) + g.tx(1, false) + g.obs(1, 1, 1) + store := NewPacketStore(db, &PacketStoreConfig{}) + if err := store.RunStartupLoad(100); err != nil { + t.Fatalf("RunStartupLoad: %v", err) + } + loaded := loadedObsPaths(store) + pr := startGapPoller(t, db, store) + + g.exec(`ALTER TABLE observers RENAME TO observers_away`) + renamed := true + defer func() { + if renamed { + g.exec(`ALTER TABLE observers_away RENAME TO observers`) + } + }() + g.tx(2, false) + g.obs(2, 2, 1) // arrives with tx 2 + g.obs(3, 1, 2) // a new observation of loaded tx 1 + + time.Sleep(150 * time.Millisecond) // several failing ticks + if storeHasObs(store, 2) || storeHasObs(store, 3) { + t.Fatal("setup: the ingest reads did not fail while observers was renamed") + } + g.exec(`ALTER TABLE observers_away RENAME TO observers`) + renamed = false + + if !waitStoreObs(store, 3, 3*time.Second) { + t.Error("obs 3, committed while the observation read failed, never reached the store: the cursor passed it") + } + if !waitStoreObs(store, 2, time.Second) { + t.Error("tx 2 and its obs 2, committed while the transmission read failed, never reached the store") + } + pr.stop() + assertStoreMatchesDB(t, store, rw) + assertNoDuplicateBroadcasts(t, pr.drain(), loaded) + assertReplayIdempotent(t, store, rw) +} diff --git a/cmd/server/store.go b/cmd/server/store.go index de5fed27..b0fd4a02 100644 --- a/cmd/server/store.go +++ b/cmd/server/store.go @@ -250,6 +250,14 @@ type PacketStore struct { txRewriteMu sync.Mutex txRewriteCursor int64 + // Poller start cursors: MAX(id) of transmissions / observations, read in + // NewPacketStore before any load query. A row above a mark may have been + // committed after the load's snapshot, so the poller re-reads from there. + // Written once before the store is shared; read-only afterwards. + startupTxMark int + startupObsMark int + startupMarksOK bool // false: no conn, or a MAX query failed + mu sync.RWMutex db *DB packets []*StoreTx // sorted by first_seen ASC (oldest first; newest at tail) @@ -649,8 +657,22 @@ type PacketStore struct { // ordering tests to capture the bg-loader entry timestamp/signal // without polling. See runstartup_load_test.go. bgLoaderEntryHook func() + // Test-only hooks for the poller/load interleaving tests. Nil in + // production. chunkQueriedHook fires in LoadChunked after a chunk's + // query returns, before its rows are merged (chunk counts from 1). + // obsPollHook fires in ingestNewObservations after the feeds are + // polled and again when it returns. + chunkQueriedHook func(chunk int) + obsPollHook func(stage obsPollStage) } +type obsPollStage int + +const ( + obsPollAfterFeeds obsPollStage = iota + obsPollDone +) + // Precomputed distance records for fast analytics aggregation. type distHopRecord struct { FromName string @@ -857,6 +879,11 @@ func NewPacketStore(db *DB, cfg *PacketStoreConfig, cacheTTLs ...map[string]inte ps.invCooldown = v } } + // Capture BEFORE loading any packets: the load reads each chunk in its own + // snapshot, so a row committed while it runs is only safe if the poller + // starts at or below it. One writer commits ids in order, so everything + // the load can miss lies above these marks (see Poller.startCursors). + ps.startupTxMark, ps.startupObsMark, ps.startupMarksOK = db.highWaterMarks() // Capture BEFORE loading any packets. Evidence arriving during startup // remains above this watermark; each loaded tx also reads its full mask. if db.advertEvidencePresent() { @@ -3020,9 +3047,23 @@ func (s *PacketStore) queryMultiNodePacketsLocked(resolved []string, limit, offs return &PacketResult{Packets: packets, Total: total}, jobs } +// wsPacketMaps is a batch of broadcast-ready packet maps from the ingest paths. +type wsPacketMaps []map[string]interface{} + // IngestNewFromDB loads new transmissions from SQLite into memory and returns // broadcast-ready maps plus the new max transmission ID. -func (s *PacketStore) IngestNewFromDB(sinceID, limit int) ([]map[string]interface{}, int) { +func (s *PacketStore) IngestNewFromDB(sinceID, limit int) (wsPacketMaps, int) { + out, newMax, err := s.ingestNewFromDB(sinceID, limit) + if err != nil { + log.Printf("[store] %v", err) + } + return out, newMax +} + +// ingestNewFromDB is IngestNewFromDB reporting a failed read. On error the +// returned max is sinceID, so the caller's cursor stays put and the poll is +// retried. +func (s *PacketStore) ingestNewFromDB(sinceID, limit int) (wsPacketMaps, int, error) { if limit <= 0 { limit = 100 } @@ -3064,8 +3105,7 @@ func (s *PacketStore) IngestNewFromDB(sinceID, limit int) ([]map[string]interfac rows, err := s.db.conn.Query(querySQL, sinceID) if err != nil { - log.Printf("[store] ingest query error: %v", err) - return nil, sinceID + return nil, sinceID, fmt.Errorf("ingest query error: %w", err) } defer rows.Close() @@ -3146,11 +3186,11 @@ func (s *PacketStore) IngestNewFromDB(sinceID, limit int) ([]map[string]interfac tempRows = append(tempRows, tr) } - if len(tempRows) == 0 { - return nil, sinceID - } if err := rows.Err(); err != nil { - return nil, sinceID + return nil, sinceID, fmt.Errorf("ingest read error: %w", err) + } + if len(tempRows) == 0 { + return nil, sinceID, nil } rows.Close() s.advertEvidenceMu.Lock() @@ -3163,8 +3203,7 @@ func (s *PacketStore) IngestNewFromDB(sinceID, limit int) ([]map[string]interfac } masks, err := s.db.advertEvidenceForIDs(ids) if err != nil { - log.Printf("[store] load advert evidence: %v", err) - return nil, sinceID + return nil, sinceID, fmt.Errorf("load advert evidence: %w", err) } // Now lock and merge into store @@ -3427,16 +3466,31 @@ func (s *PacketStore) IngestNewFromDB(sinceID, limit int) ([]map[string]interfac if evidenceChanged { s.invalidateAdvertEvidence() } - return result, newMaxID + return result, newMaxID, nil } // IngestNewObservations loads new observations for transmissions already in the // store. This catches observations that arrive after IngestNewFromDB has already // advanced past the transmission's ID (fixes #174). -func (s *PacketStore) IngestNewObservations(sinceObsID, limit int) []map[string]interface{} { +func (s *PacketStore) IngestNewObservations(sinceObsID, limit int) wsPacketMaps { + out, err := s.ingestNewObservations(sinceObsID, limit) + if err != nil { + log.Printf("[store] %v", err) + } + return out +} + +// ingestNewObservations is IngestNewObservations reporting a failed read, so +// the poller can keep its cursor and retry instead of passing rows it never +// saw. A row that fails to scan is still skipped. +func (s *PacketStore) ingestNewObservations(sinceObsID, limit int) (wsPacketMaps, error) { // Must run even when observation IDs/timestamps have not changed. s.refreshAdvertEvidence() s.refreshTxRewrites() // #2107: rewrites never move IDs, so poll them here too + if s.obsPollHook != nil { + s.obsPollHook(obsPollAfterFeeds) + defer s.obsPollHook(obsPollDone) + } if limit <= 0 { limit = 500 } @@ -3466,8 +3520,7 @@ func (s *PacketStore) IngestNewObservations(sinceObsID, limit int) []map[string] rows, err := s.db.conn.Query(querySQL, sinceObsID, limit) if err != nil { - log.Printf("[store] ingest observations query error: %v", err) - return nil + return nil, fmt.Errorf("ingest observations query error: %w", err) } defer rows.Close() @@ -3518,8 +3571,11 @@ func (s *PacketStore) IngestNewObservations(sinceObsID, limit int) []map[string] }) } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("ingest observations read error: %w", err) + } if len(obsRows) == 0 { - return nil + return nil, nil } s.mu.Lock() @@ -3722,7 +3778,7 @@ func (s *PacketStore) IngestNewObservations(sinceObsID, limit int) []map[string] _ = pm _ = graphRef - return broadcastMaps + return broadcastMaps, nil } // MaxTransmissionID returns the highest transmission ID in the store. @@ -3732,6 +3788,12 @@ func (s *PacketStore) MaxTransmissionID() int { return s.maxTxID } +// StartupMarks returns the transmission and observation high-water marks taken +// before the startup load, and whether both queries succeeded. +func (s *PacketStore) StartupMarks() (txMark, obsMark int, ok bool) { + return s.startupTxMark, s.startupObsMark, s.startupMarksOK +} + // MaxObservationID returns the highest observation ID in the store. func (s *PacketStore) MaxObservationID() int { s.mu.RLock() diff --git a/cmd/server/websocket.go b/cmd/server/websocket.go index 40395639..8e087d5e 100644 --- a/cmd/server/websocket.go +++ b/cmd/server/websocket.go @@ -302,22 +302,30 @@ func NewPoller(db *DB, hub *Hub, interval time.Duration) *Poller { return &Poller{db: db, hub: hub, interval: interval, stop: make(chan struct{})} } -func (p *Poller) Start() { - lastID := p.db.GetMaxTransmissionID() - lastObsID := p.db.GetMaxObservationID() - // If the store already loaded data, use its max IDs as a floor. - // This prevents replaying the entire DB when the DB query fails - // (e.g., corrupted DB returns 0 from COALESCE). - if p.store != nil { - if storeMax := p.store.MaxTransmissionID(); storeMax > lastID { - lastID = storeMax - } - if storeMaxObs := p.store.MaxObservationID(); storeMaxObs > lastObsID { - lastObsID = storeMaxObs - } +// pollFailure logs a failing poll once per streak of failures, and once when +// it recovers. Owned by the Start goroutine. +type pollFailure struct { + what string + failing bool +} + +func (f *pollFailure) note(err error, cursor int) { + switch { + case err != nil && !f.failing: + f.failing = true + log.Printf("[poller] %s poll failed, retrying from id %d each tick: %v", f.what, cursor, err) + case err == nil && f.failing: + f.failing = false + log.Printf("[poller] %s poll recovered at id %d", f.what, cursor) } +} + +func (p *Poller) Start() { + lastID, lastObsID := p.startCursors() log.Printf("[poller] starting from transmission ID %d, obs ID %d, interval %v", lastID, lastObsID, p.interval) + txFail := pollFailure{what: "transmission"} + obsFail := pollFailure{what: "observation"} ticker := time.NewTicker(p.interval) defer ticker.Stop() @@ -329,11 +337,18 @@ func (p *Poller) Start() { // new observation must be known before it is broadcast. refreshRadioHistory(p.db.conn, false) // Ingest new transmissions into in-memory store and broadcast - newTxs, newMax := p.store.IngestNewFromDB(lastID, 100) + // On error newMax is lastID: the same rows are read next tick. + newTxs, newMax, err := p.store.ingestNewFromDB(lastID, 100) + txFail.note(err, lastID) if newMax > lastID { lastID = newMax } - // Ingest new observations for existing transmissions (fixes #174) + // Ingest new observations for existing transmissions (fixes #174). + // The bound below is read BEFORE ingestNewObservations polls the + // advert feed, which waits on advertEvidenceMu while a startup + // chunk sits between its query and its merge. So an observation + // of a transmission that chunk has not merged yet is either read + // after the merge or lies above the bound, never skipped below it. nextObsID := lastObsID if err := p.db.conn.QueryRow(` SELECT COALESCE(MAX(id), ?) FROM ( @@ -344,8 +359,10 @@ func (p *Poller) Start() { )`, lastObsID, lastObsID).Scan(&nextObsID); err != nil { nextObsID = lastObsID } - newObs := p.store.IngestNewObservations(lastObsID, 500) - if nextObsID > lastObsID { + newObs, err := p.store.ingestNewObservations(lastObsID, 500) + obsFail.note(err, lastObsID) + // A failed read saw none of these rows: keep the cursor. + if err == nil && nextObsID > lastObsID { lastObsID = nextObsID } if len(newTxs) > 0 { @@ -393,6 +410,37 @@ func (p *Poller) Start() { } } +// startCursors picks the ids polling starts above. A fresh MAX(id) here would +// skip every row committed between the load's last snapshot and now: it is in +// neither the store nor any later poll. So with a store, polling starts at the +// high-water marks taken before the load. Rows between a mark and the end of +// the load are read again and dedupe: transmissions by hash, observations by +// byObsID or observer+path, and only rows new to the store are broadcast. +func (p *Poller) startCursors() (lastID, lastObsID int) { + if p.store == nil { + return p.db.GetMaxTransmissionID(), p.db.GetMaxObservationID() + } + txMark, obsMark, ok := p.store.StartupMarks() + if ok { + lastID, lastObsID = txMark, obsMark + } else { + // No marks: keep the old fresh-MAX start, floored by the store so a + // failed query (COALESCE 0) cannot replay the whole DB. + lastID, lastObsID = p.db.GetMaxTransmissionID(), p.db.GetMaxObservationID() + if storeMaxObs := p.store.MaxObservationID(); storeMaxObs > lastObsID { + lastObsID = storeMaxObs + } + } + // The loaded max transmission id is a safe floor: one writer commits ids + // in order, so every lower id was visible to the chunk that loaded it. + // The loaded max observation id is not: a chunk-1 transmission can gain + // an observation after chunk 1 whose id is below one loaded in chunk 2. + if storeMax := p.store.MaxTransmissionID(); storeMax > lastID { + lastID = storeMax + } + return lastID, lastObsID +} + func (p *Poller) Stop() { close(p.stop) }