diff --git a/.badges/e2e-tests.json b/.badges/e2e-tests.json index a34848cf..504f7e8e 100644 --- a/.badges/e2e-tests.json +++ b/.badges/e2e-tests.json @@ -1 +1 @@ -{"schemaVersion":1,"label":"e2e tests","message":"45 passed","color":"brightgreen"} \ No newline at end of file +{"schemaVersion":1,"label":"e2e tests","message":"83 passed","color":"brightgreen"} diff --git a/.badges/frontend-coverage.json b/.badges/frontend-coverage.json index 1d7cf1a2..10071728 100644 --- a/.badges/frontend-coverage.json +++ b/.badges/frontend-coverage.json @@ -1 +1 @@ -{"schemaVersion":1,"label":"frontend coverage","message":"39.68%","color":"red"} \ No newline at end of file +{"schemaVersion":1,"label":"frontend coverage","message":"37.74%","color":"red"} diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index cd5c4319..f612a82d 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -135,7 +135,7 @@ jobs: e2e-test: name: "π Playwright E2E Tests" needs: [go-test] - runs-on: [self-hosted, Linux] + runs-on: ubuntu-latest defaults: run: shell: bash @@ -145,13 +145,6 @@ jobs: with: fetch-depth: 0 - - name: Free disk space - run: | - # Prune old runner diagnostic logs (can accumulate 50MB+) - find ~/actions-runner/_diag/ -name '*.log' -mtime +3 -delete 2>/dev/null || true - # Show available disk space - df -h / | tail -1 - - name: Set up Node.js 22 uses: actions/setup-node@v5 with: @@ -252,17 +245,11 @@ jobs: build-and-publish: name: "ποΈ Build & Publish Docker Image" needs: [e2e-test] - runs-on: [self-hosted, meshcore-runner-2] + runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v5 - - name: Free disk space - run: | - docker system prune -af 2>/dev/null || true - docker builder prune -af 2>/dev/null || true - df -h / - - name: Compute build metadata id: meta run: | @@ -462,7 +449,7 @@ jobs: name: "π Publish Badges & Summary" if: github.event_name == 'push' needs: [deploy] - runs-on: [self-hosted, Linux] + runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v5 diff --git a/cmd/ingestor/config.go b/cmd/ingestor/config.go index 70c18fbe..910d3b95 100644 --- a/cmd/ingestor/config.go +++ b/cmd/ingestor/config.go @@ -41,6 +41,7 @@ type Config struct { Metrics *MetricsConfig `json:"metrics,omitempty"` GeoFilter *GeoFilterConfig `json:"geo_filter,omitempty"` ValidateSignatures *bool `json:"validateSignatures,omitempty"` + DB *DBConfig `json:"db,omitempty"` } // GeoFilterConfig is an alias for the shared geofilter.Config type. @@ -58,6 +59,20 @@ type MetricsConfig struct { SampleIntervalSec int `json:"sampleIntervalSec"` } +// DBConfig controls SQLite vacuum and maintenance behavior (#919). +type DBConfig struct { + VacuumOnStartup bool `json:"vacuumOnStartup"` // one-time full VACUUM on startup if auto_vacuum is not INCREMENTAL + IncrementalVacuumPages int `json:"incrementalVacuumPages"` // pages returned to OS per reaper cycle (default 1024) +} + +// IncrementalVacuumPages returns the configured pages per vacuum or 1024 default. +func (c *Config) IncrementalVacuumPages() int { + if c.DB != nil && c.DB.IncrementalVacuumPages > 0 { + return c.DB.IncrementalVacuumPages + } + return 1024 +} + // ShouldValidateSignatures returns true (default) unless explicitly disabled. func (c *Config) ShouldValidateSignatures() bool { if c.ValidateSignatures != nil { diff --git a/cmd/ingestor/db.go b/cmd/ingestor/db.go index bada26c8..93d3dc53 100644 --- a/cmd/ingestor/db.go +++ b/cmd/ingestor/db.go @@ -59,7 +59,7 @@ func OpenStoreWithInterval(dbPath string, sampleIntervalSec int) (*Store, error) return nil, fmt.Errorf("creating data dir: %w", err) } - db, err := sql.Open("sqlite", dbPath+"?_pragma=journal_mode(WAL)&_pragma=foreign_keys(ON)&_pragma=busy_timeout(5000)") + db, err := sql.Open("sqlite", dbPath+"?_pragma=auto_vacuum(INCREMENTAL)&_pragma=journal_mode(WAL)&_pragma=foreign_keys(ON)&_pragma=busy_timeout(5000)") if err != nil { return nil, fmt.Errorf("opening db: %w", err) } @@ -85,6 +85,9 @@ func OpenStoreWithInterval(dbPath string, sampleIntervalSec int) (*Store, error) } func applySchema(db *sql.DB) error { + // auto_vacuum=INCREMENTAL is set via DSN pragma (must be before journal_mode). + // Logging of current mode is handled by CheckAutoVacuum β no duplicate log here. + schema := ` CREATE TABLE IF NOT EXISTS nodes ( public_key TEXT PRIMARY KEY, @@ -788,6 +791,58 @@ func (s *Store) PruneOldMetrics(retentionDays int) (int64, error) { return n, nil } +// CheckAutoVacuum inspects the current auto_vacuum mode and logs a warning +// if not INCREMENTAL. Performs opt-in full VACUUM if db.vacuumOnStartup is set (#919). +func (s *Store) CheckAutoVacuum(cfg *Config) { + var autoVacuum int + if err := s.db.QueryRow("PRAGMA auto_vacuum").Scan(&autoVacuum); err != nil { + log.Printf("[db] warning: could not read auto_vacuum: %v", err) + return + } + + if autoVacuum == 2 { + log.Printf("[db] auto_vacuum=INCREMENTAL") + return + } + + modes := map[int]string{0: "NONE", 1: "FULL", 2: "INCREMENTAL"} + mode := modes[autoVacuum] + if mode == "" { + mode = fmt.Sprintf("UNKNOWN(%d)", autoVacuum) + } + + log.Printf("[db] auto_vacuum=%s β DB needs one-time VACUUM to enable incremental auto-vacuum. "+ + "Set db.vacuumOnStartup: true in config to migrate (will block startup for several minutes on large DBs). "+ + "See https://github.com/Kpa-clawbot/CoreScope/issues/919", mode) + + if cfg.DB != nil && cfg.DB.VacuumOnStartup { + // WARNING: Full VACUUM creates a temporary copy of the entire DB file. + // Requires ~2Γ the DB file size in free disk space or it will fail. + log.Printf("[db] vacuumOnStartup=true β starting one-time full VACUUM (ensure 2x DB size free disk space)...") + start := time.Now() + + if _, err := s.db.Exec("PRAGMA auto_vacuum = INCREMENTAL"); err != nil { + log.Printf("[db] VACUUM failed: could not set auto_vacuum: %v", err) + return + } + if _, err := s.db.Exec("VACUUM"); err != nil { + log.Printf("[db] VACUUM failed: %v", err) + return + } + + elapsed := time.Since(start) + log.Printf("[db] VACUUM complete in %v β auto_vacuum is now INCREMENTAL", elapsed.Round(time.Millisecond)) + } +} + +// RunIncrementalVacuum returns free pages to the OS (#919). +// Safe to call on auto_vacuum=NONE databases (noop). +func (s *Store) RunIncrementalVacuum(pages int) { + if _, err := s.db.Exec(fmt.Sprintf("PRAGMA incremental_vacuum(%d)", pages)); err != nil { + log.Printf("[vacuum] incremental_vacuum error: %v", err) + } +} + // Checkpoint forces a WAL checkpoint to release the WAL lock file, // preventing lock contention with a new process starting up. func (s *Store) Checkpoint() { diff --git a/cmd/ingestor/main.go b/cmd/ingestor/main.go index 481c7cc1..b0b94bed 100644 --- a/cmd/ingestor/main.go +++ b/cmd/ingestor/main.go @@ -57,6 +57,9 @@ func main() { defer store.Close() log.Printf("SQLite opened: %s", cfg.DBPath) + // Check auto_vacuum mode and optionally migrate (#919) + store.CheckAutoVacuum(cfg) + // Node retention: move stale nodes to inactive_nodes on startup nodeDays := cfg.NodeDaysOrDefault() store.MoveStaleNodes(nodeDays) @@ -69,12 +72,15 @@ func main() { metricsDays := cfg.MetricsRetentionDays() store.PruneOldMetrics(metricsDays) store.PruneDroppedPackets(metricsDays) + vacuumPages := cfg.IncrementalVacuumPages() + store.RunIncrementalVacuum(vacuumPages) // Daily ticker for node retention retentionTicker := time.NewTicker(1 * time.Hour) go func() { for range retentionTicker.C { store.MoveStaleNodes(nodeDays) + store.RunIncrementalVacuum(vacuumPages) } }() @@ -83,8 +89,10 @@ func main() { go func() { time.Sleep(90 * time.Second) // stagger after metrics prune store.RemoveStaleObservers(observerDays) + store.RunIncrementalVacuum(vacuumPages) for range observerRetentionTicker.C { store.RemoveStaleObservers(observerDays) + store.RunIncrementalVacuum(vacuumPages) } }() @@ -94,6 +102,7 @@ func main() { for range metricsRetentionTicker.C { store.PruneOldMetrics(metricsDays) store.PruneDroppedPackets(metricsDays) + store.RunIncrementalVacuum(vacuumPages) } }() diff --git a/cmd/server/bounded_load_test.go b/cmd/server/bounded_load_test.go index d42e2a20..ad8b773e 100644 --- a/cmd/server/bounded_load_test.go +++ b/cmd/server/bounded_load_test.go @@ -127,6 +127,92 @@ func TestBoundedLoad_AscendingOrder(t *testing.T) { } } +// loadStoreWithRetention creates a PacketStore with retentionHours set. +func loadStoreWithRetention(t *testing.T, dbPath string, retentionHours float64) *PacketStore { + t.Helper() + db, err := OpenDB(dbPath) + if err != nil { + t.Fatal(err) + } + cfg := &PacketStoreConfig{RetentionHours: retentionHours} + store := NewPacketStore(db, cfg) + if err := store.Load(); err != nil { + t.Fatal(err) + } + return store +} + +// createTestDBWithAgedPackets inserts numRecent packets with timestamps within +// the last hour and numOld packets with timestamps 48 hours ago. +func createTestDBWithAgedPackets(t *testing.T, numRecent, numOld int) string { + t.Helper() + dir := t.TempDir() + dbPath := filepath.Join(dir, "test.db") + + conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL") + if err != nil { + t.Fatal(err) + } + defer conn.Close() + + execOrFail := func(s string) { + if _, err := conn.Exec(s); err != nil { + t.Fatalf("setup: %v\nSQL: %s", err, s) + } + } + execOrFail(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT)`) + execOrFail(`CREATE TABLE observations (id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT)`) + execOrFail(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT)`) + execOrFail(`CREATE TABLE nodes (pubkey TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, frequency REAL)`) + execOrFail(`CREATE TABLE schema_version (version INTEGER)`) + execOrFail(`INSERT INTO schema_version (version) VALUES (1)`) + execOrFail(`CREATE INDEX idx_tx_first_seen ON transmissions(first_seen)`) + + now := time.Now().UTC() + id := 1 + // Insert old packets (48 hours ago) + for i := 0; i < numOld; i++ { + ts := now.Add(-48 * time.Hour).Add(time.Duration(i) * time.Second).Format(time.RFC3339) + conn.Exec("INSERT INTO transmissions VALUES (?,?,?,?,0,4,1,?)", id, "aa", fmt.Sprintf("old%d", i), ts, `{}`) + conn.Exec("INSERT INTO observations VALUES (?,?,?,?,?,?,?,?,?,?,?)", id, id, "obs1", "Obs1", "RX", -10.0, -80.0, 5, `[]`, ts, "") + id++ + } + // Insert recent packets (within last hour) + for i := 0; i < numRecent; i++ { + ts := now.Add(-30 * time.Minute).Add(time.Duration(i) * time.Second).Format(time.RFC3339) + conn.Exec("INSERT INTO transmissions VALUES (?,?,?,?,0,4,1,?)", id, "bb", fmt.Sprintf("new%d", i), ts, `{}`) + conn.Exec("INSERT INTO observations VALUES (?,?,?,?,?,?,?,?,?,?,?)", id, id, "obs1", "Obs1", "RX", -10.0, -80.0, 5, `[]`, ts, "") + id++ + } + return dbPath +} + +func TestRetentionLoad_OnlyLoadsRecentPackets(t *testing.T) { + dbPath := createTestDBWithAgedPackets(t, 50, 100) + defer os.RemoveAll(filepath.Dir(dbPath)) + + // retention = 2 hours β should load only the 50 recent packets, not the 100 old ones + store := loadStoreWithRetention(t, dbPath, 2) + defer store.db.conn.Close() + + if len(store.packets) != 50 { + t.Errorf("expected 50 recent packets, got %d (old packets should be excluded by retentionHours)", len(store.packets)) + } +} + +func TestRetentionLoad_ZeroRetentionLoadsAll(t *testing.T) { + dbPath := createTestDBWithAgedPackets(t, 50, 100) + defer os.RemoveAll(filepath.Dir(dbPath)) + + // retention = 0 (unlimited) β should load all 150 packets + store := loadStoreWithRetention(t, dbPath, 0) + defer store.db.conn.Close() + + if len(store.packets) != 150 { + t.Errorf("expected all 150 packets with retentionHours=0, got %d", len(store.packets)) + } +} + func TestEstimateStoreTxBytesTypical(t *testing.T) { est := estimateStoreTxBytesTypical(10) if est < 1000 { diff --git a/cmd/server/config.go b/cmd/server/config.go index 6039d41e..f21ef207 100644 --- a/cmd/server/config.go +++ b/cmd/server/config.go @@ -62,6 +62,8 @@ type Config struct { Retention *RetentionConfig `json:"retention,omitempty"` + DB *DBConfig `json:"db,omitempty"` + PacketStore *PacketStoreConfig `json:"packetStore,omitempty"` GeoFilter *GeoFilterConfig `json:"geo_filter,omitempty"` @@ -129,6 +131,20 @@ type RetentionConfig struct { MetricsDays int `json:"metricsDays"` } +// DBConfig controls SQLite vacuum and maintenance behavior (#919). +type DBConfig struct { + VacuumOnStartup bool `json:"vacuumOnStartup"` // one-time full VACUUM on startup if auto_vacuum is not INCREMENTAL + IncrementalVacuumPages int `json:"incrementalVacuumPages"` // pages returned to OS per reaper cycle (default 1024) +} + +// IncrementalVacuumPages returns the configured pages per vacuum or 1024 default. +func (c *Config) IncrementalVacuumPages() int { + if c.DB != nil && c.DB.IncrementalVacuumPages > 0 { + return c.DB.IncrementalVacuumPages + } + return 1024 +} + // MetricsRetentionDays returns configured metrics retention or 30 days default. func (c *Config) MetricsRetentionDays() int { if c.Retention != nil && c.Retention.MetricsDays > 0 { diff --git a/cmd/server/coverage_test.go b/cmd/server/coverage_test.go index f6228c7f..2d2587d8 100644 --- a/cmd/server/coverage_test.go +++ b/cmd/server/coverage_test.go @@ -763,9 +763,9 @@ func TestGetChannelsFromStore(t *testing.T) { func TestPrefixMapResolve(t *testing.T) { nodes := []nodeInfo{ - {PublicKey: "aabbccdd11223344", Name: "NodeA", HasGPS: true, Lat: 37.5, Lon: -122.0}, - {PublicKey: "aabbccdd55667788", Name: "NodeB", HasGPS: false}, - {PublicKey: "eeff0011aabbccdd", Name: "NodeC", HasGPS: true, Lat: 38.0, Lon: -121.0}, + {Role: "repeater", PublicKey: "aabbccdd11223344", Name: "NodeA", HasGPS: true, Lat: 37.5, Lon: -122.0}, + {Role: "repeater", PublicKey: "aabbccdd55667788", Name: "NodeB", HasGPS: false}, + {Role: "repeater", PublicKey: "eeff0011aabbccdd", Name: "NodeC", HasGPS: true, Lat: 38.0, Lon: -121.0}, } pm := buildPrefixMap(nodes) @@ -805,8 +805,8 @@ func TestPrefixMapResolve(t *testing.T) { t.Run("multiple candidates no GPS", func(t *testing.T) { noGPSNodes := []nodeInfo{ - {PublicKey: "aa11bb22", Name: "X", HasGPS: false}, - {PublicKey: "aa11cc33", Name: "Y", HasGPS: false}, + {Role: "repeater", PublicKey: "aa11bb22", Name: "X", HasGPS: false}, + {Role: "repeater", PublicKey: "aa11cc33", Name: "Y", HasGPS: false}, } pm2 := buildPrefixMap(noGPSNodes) n := pm2.resolve("aa11") @@ -820,8 +820,8 @@ func TestPrefixMapResolve(t *testing.T) { func TestPrefixMapCap(t *testing.T) { // 16-char pubkey β longer than maxPrefixLen nodes := []nodeInfo{ - {PublicKey: "aabbccdd11223344", Name: "LongKey"}, - {PublicKey: "eeff0011", Name: "ShortKey"}, // exactly 8 chars + {Role: "repeater", PublicKey: "aabbccdd11223344", Name: "LongKey"}, + {Role: "repeater", PublicKey: "eeff0011", Name: "ShortKey"}, // exactly 8 chars } pm := buildPrefixMap(nodes) diff --git a/cmd/server/db_vacuum_test.go b/cmd/server/db_vacuum_test.go new file mode 100644 index 00000000..6dad269f --- /dev/null +++ b/cmd/server/db_vacuum_test.go @@ -0,0 +1,262 @@ +package main + +import ( + "database/sql" + "os" + "path/filepath" + "strings" + "testing" + "time" + + _ "modernc.org/sqlite" +) + +// createFreshIngestorDB creates a SQLite DB using the ingestor's applySchema logic +// (simulated here) with auto_vacuum=INCREMENTAL set before tables. +func createFreshDBWithAutoVacuum(t *testing.T, path string) *sql.DB { + t.Helper() + // auto_vacuum must be set via DSN before journal_mode creates the DB file + db, err := sql.Open("sqlite", path+"?_pragma=auto_vacuum(INCREMENTAL)&_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)") + if err != nil { + t.Fatal(err) + } + db.SetMaxOpenConns(1) + + // Create minimal schema + _, err = db.Exec(` + CREATE TABLE transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + raw_hex TEXT NOT NULL, + hash TEXT NOT NULL UNIQUE, + first_seen TEXT NOT NULL, + route_type INTEGER, + payload_type INTEGER, + payload_version INTEGER, + decoded_json TEXT, + created_at TEXT DEFAULT (datetime('now')), + channel_hash TEXT + ); + CREATE TABLE observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + transmission_id INTEGER NOT NULL REFERENCES transmissions(id), + observer_idx INTEGER, + direction TEXT, + snr REAL, + rssi REAL, + score INTEGER, + path_json TEXT, + timestamp INTEGER NOT NULL + ); + `) + if err != nil { + t.Fatal(err) + } + return db +} + +func TestNewDBHasIncrementalAutoVacuum(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "test.db") + + db := createFreshDBWithAutoVacuum(t, path) + defer db.Close() + + var autoVacuum int + if err := db.QueryRow("PRAGMA auto_vacuum").Scan(&autoVacuum); err != nil { + t.Fatal(err) + } + if autoVacuum != 2 { + t.Fatalf("expected auto_vacuum=2 (INCREMENTAL), got %d", autoVacuum) + } +} + +func TestExistingDBHasAutoVacuumNone(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "test.db") + + // Create DB WITHOUT setting auto_vacuum (simulates old DB) + db, err := sql.Open("sqlite", path+"?_pragma=journal_mode(WAL)") + if err != nil { + t.Fatal(err) + } + db.SetMaxOpenConns(1) + _, err = db.Exec("CREATE TABLE dummy (id INTEGER PRIMARY KEY)") + if err != nil { + t.Fatal(err) + } + + var autoVacuum int + if err := db.QueryRow("PRAGMA auto_vacuum").Scan(&autoVacuum); err != nil { + t.Fatal(err) + } + db.Close() + + if autoVacuum != 0 { + t.Fatalf("expected auto_vacuum=0 (NONE) for old DB, got %d", autoVacuum) + } +} + +func TestVacuumOnStartupMigratesDB(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "test.db") + + // Create DB without auto_vacuum (old DB) + db, err := sql.Open("sqlite", path+"?_pragma=journal_mode(WAL)") + if err != nil { + t.Fatal(err) + } + db.SetMaxOpenConns(1) + _, err = db.Exec("CREATE TABLE dummy (id INTEGER PRIMARY KEY)") + if err != nil { + t.Fatal(err) + } + + var before int + db.QueryRow("PRAGMA auto_vacuum").Scan(&before) + if before != 0 { + t.Fatalf("precondition: expected auto_vacuum=0, got %d", before) + } + db.Close() + + // Simulate vacuumOnStartup migration using openRW + rw, err := openRW(path) + if err != nil { + t.Fatal(err) + } + if _, err := rw.Exec("PRAGMA auto_vacuum = INCREMENTAL"); err != nil { + t.Fatal(err) + } + if _, err := rw.Exec("VACUUM"); err != nil { + t.Fatal(err) + } + rw.Close() + + // Verify migration + db2, err := sql.Open("sqlite", path+"?mode=ro") + if err != nil { + t.Fatal(err) + } + defer db2.Close() + + var after int + if err := db2.QueryRow("PRAGMA auto_vacuum").Scan(&after); err != nil { + t.Fatal(err) + } + if after != 2 { + t.Fatalf("expected auto_vacuum=2 after VACUUM migration, got %d", after) + } +} + +func TestIncrementalVacuumReducesFreelist(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "test.db") + + db := createFreshDBWithAutoVacuum(t, path) + + // Insert a bunch of data + now := time.Now().UTC().Format(time.RFC3339) + for i := 0; i < 500; i++ { + _, err := db.Exec( + "INSERT INTO transmissions (raw_hex, hash, first_seen) VALUES (?, ?, ?)", + strings.Repeat("AA", 200), // ~400 bytes each + "hash_"+string(rune('A'+i%26))+string(rune('0'+i/26)), + now, + ) + if err != nil { + t.Fatal(err) + } + } + + // Get file size before delete + db.Close() + infoBefore, _ := os.Stat(path) + sizeBefore := infoBefore.Size() + + // Reopen and delete all + db, err := sql.Open("sqlite", path+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)") + if err != nil { + t.Fatal(err) + } + db.SetMaxOpenConns(1) + defer db.Close() + + _, err = db.Exec("DELETE FROM transmissions") + if err != nil { + t.Fatal(err) + } + + // Check freelist before vacuum + var freelistBefore int64 + db.QueryRow("PRAGMA freelist_count").Scan(&freelistBefore) + if freelistBefore == 0 { + t.Fatal("expected non-zero freelist after DELETE") + } + + // Run incremental vacuum + _, err = db.Exec("PRAGMA incremental_vacuum(10000)") + if err != nil { + t.Fatal(err) + } + + // Check freelist after vacuum + var freelistAfter int64 + db.QueryRow("PRAGMA freelist_count").Scan(&freelistAfter) + if freelistAfter >= freelistBefore { + t.Fatalf("expected freelist to shrink: before=%d after=%d", freelistBefore, freelistAfter) + } + + // Checkpoint WAL and check file size shrunk + db.Exec("PRAGMA wal_checkpoint(TRUNCATE)") + db.Close() + infoAfter, _ := os.Stat(path) + sizeAfter := infoAfter.Size() + if sizeAfter >= sizeBefore { + t.Logf("warning: file did not shrink (before=%d after=%d) β may depend on page reuse", sizeBefore, sizeAfter) + } +} + +func TestCheckAutoVacuumLogs(t *testing.T) { + // This test verifies checkAutoVacuum doesn't panic on various configs + dir := t.TempDir() + path := filepath.Join(dir, "test.db") + + // Create a fresh DB with auto_vacuum=INCREMENTAL + dbConn := createFreshDBWithAutoVacuum(t, path) + db := &DB{conn: dbConn, path: path} + cfg := &Config{} + + // Should not panic + checkAutoVacuum(db, cfg, path) + dbConn.Close() + + // Create a DB without auto_vacuum + path2 := filepath.Join(dir, "test2.db") + dbConn2, _ := sql.Open("sqlite", path2+"?_pragma=journal_mode(WAL)") + dbConn2.SetMaxOpenConns(1) + dbConn2.Exec("CREATE TABLE dummy (id INTEGER PRIMARY KEY)") + db2 := &DB{conn: dbConn2, path: path2} + + // Should log warning but not panic + checkAutoVacuum(db2, cfg, path2) + dbConn2.Close() +} + +func TestConfigIncrementalVacuumPages(t *testing.T) { + // Default + cfg := &Config{} + if cfg.IncrementalVacuumPages() != 1024 { + t.Fatalf("expected default 1024, got %d", cfg.IncrementalVacuumPages()) + } + + // Custom + cfg.DB = &DBConfig{IncrementalVacuumPages: 512} + if cfg.IncrementalVacuumPages() != 512 { + t.Fatalf("expected 512, got %d", cfg.IncrementalVacuumPages()) + } + + // Zero should return default + cfg.DB.IncrementalVacuumPages = 0 + if cfg.IncrementalVacuumPages() != 1024 { + t.Fatalf("expected default 1024 for zero, got %d", cfg.IncrementalVacuumPages()) + } +} diff --git a/cmd/server/main.go b/cmd/server/main.go index 22dc600e..31fbcd4f 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -148,6 +148,9 @@ func main() { stats.TotalTransmissions, stats.TotalObservations, stats.TotalNodes, stats.TotalObservers) } + // Check auto_vacuum mode and optionally migrate (#919) + checkAutoVacuum(database, cfg, resolvedDB) + // In-memory packet store store := NewPacketStore(database, cfg.PacketStore, cfg.CacheTTL) if err := store.Load(); err != nil { @@ -266,6 +269,7 @@ func main() { defer stopEviction() // Auto-prune old packets if retention.packetDays is configured + vacuumPages := cfg.IncrementalVacuumPages() var stopPrune func() if cfg.Retention != nil && cfg.Retention.PacketDays > 0 { days := cfg.Retention.PacketDays @@ -286,6 +290,9 @@ func main() { log.Printf("[prune] error: %v", err) } else { log.Printf("[prune] deleted %d transmissions older than %d days", n, days) + if n > 0 { + runIncrementalVacuum(resolvedDB, vacuumPages) + } } for { select { @@ -294,6 +301,9 @@ func main() { log.Printf("[prune] error: %v", err) } else { log.Printf("[prune] deleted %d transmissions older than %d days", n, days) + if n > 0 { + runIncrementalVacuum(resolvedDB, vacuumPages) + } } case <-pruneDone: return @@ -321,10 +331,12 @@ func main() { }() time.Sleep(2 * time.Minute) // stagger after packet prune database.PruneOldMetrics(metricsDays) + runIncrementalVacuum(resolvedDB, vacuumPages) for { select { case <-metricsPruneTicker.C: database.PruneOldMetrics(metricsDays) + runIncrementalVacuum(resolvedDB, vacuumPages) case <-metricsPruneDone: return } @@ -354,10 +366,12 @@ func main() { }() time.Sleep(3 * time.Minute) // stagger after metrics prune database.RemoveStaleObservers(observerDays) + runIncrementalVacuum(resolvedDB, vacuumPages) for { select { case <-observerPruneTicker.C: database.RemoveStaleObservers(observerDays) + runIncrementalVacuum(resolvedDB, vacuumPages) case <-observerPruneDone: return } @@ -388,6 +402,7 @@ func main() { g := store.graph store.mu.RUnlock() PruneNeighborEdges(dbPath, g, maxAgeDays) + runIncrementalVacuum(resolvedDB, vacuumPages) for { select { case <-edgePruneTicker.C: @@ -395,6 +410,7 @@ func main() { g := store.graph store.mu.RUnlock() PruneNeighborEdges(dbPath, g, maxAgeDays) + runIncrementalVacuum(resolvedDB, vacuumPages) case <-edgePruneDone: return } diff --git a/cmd/server/neighbor_dedup_test.go b/cmd/server/neighbor_dedup_test.go index 20504a77..abec93cc 100644 --- a/cmd/server/neighbor_dedup_test.go +++ b/cmd/server/neighbor_dedup_test.go @@ -12,9 +12,9 @@ import ( func TestResolveAmbiguousEdges_GeoProximity(t *testing.T) { // Node A at lat=45, lon=-122. Candidate B1 at lat=45.1, lon=-122.1 (close). // Candidate B2 at lat=10, lon=10 (far away). Prefix "b0" matches both. - nodeA := nodeInfo{PublicKey: "aaaa1111", Name: "NodeA", HasGPS: true, Lat: 45.0, Lon: -122.0} - nodeB1 := nodeInfo{PublicKey: "b0b1eeee", Name: "CloseNode", HasGPS: true, Lat: 45.1, Lon: -122.1} - nodeB2 := nodeInfo{PublicKey: "b0c2ffff", Name: "FarNode", HasGPS: true, Lat: 10.0, Lon: 10.0} + nodeA := nodeInfo{Role: "repeater", PublicKey: "aaaa1111", Name: "NodeA", HasGPS: true, Lat: 45.0, Lon: -122.0} + nodeB1 := nodeInfo{Role: "repeater", PublicKey: "b0b1eeee", Name: "CloseNode", HasGPS: true, Lat: 45.1, Lon: -122.1} + nodeB2 := nodeInfo{Role: "repeater", PublicKey: "b0c2ffff", Name: "FarNode", HasGPS: true, Lat: 10.0, Lon: 10.0} pm := buildPrefixMap([]nodeInfo{nodeA, nodeB1, nodeB2}) @@ -62,8 +62,8 @@ func TestResolveAmbiguousEdges_GeoProximity(t *testing.T) { // Test 2: Ambiguous edge merged with existing resolved edge (count accumulation). func TestResolveAmbiguousEdges_MergeWithExisting(t *testing.T) { - nodeA := nodeInfo{PublicKey: "aaaa1111", Name: "NodeA", HasGPS: true, Lat: 45.0, Lon: -122.0} - nodeB := nodeInfo{PublicKey: "b0b1eeee", Name: "NodeB", HasGPS: true, Lat: 45.1, Lon: -122.1} + nodeA := nodeInfo{Role: "repeater", PublicKey: "aaaa1111", Name: "NodeA", HasGPS: true, Lat: 45.0, Lon: -122.0} + nodeB := nodeInfo{Role: "repeater", PublicKey: "b0b1eeee", Name: "NodeB", HasGPS: true, Lat: 45.1, Lon: -122.1} pm := buildPrefixMap([]nodeInfo{nodeA, nodeB}) @@ -133,9 +133,9 @@ func TestResolveAmbiguousEdges_MergeWithExisting(t *testing.T) { // Test 3: Ambiguous edge left as-is when resolution fails. func TestResolveAmbiguousEdges_FailsNoChange(t *testing.T) { // Two candidates, neither has GPS, no affinity data β resolution falls through. - nodeA := nodeInfo{PublicKey: "aaaa1111", Name: "NodeA"} - nodeB1 := nodeInfo{PublicKey: "b0b1eeee", Name: "B1"} - nodeB2 := nodeInfo{PublicKey: "b0c2ffff", Name: "B2"} + nodeA := nodeInfo{Role: "repeater", PublicKey: "aaaa1111", Name: "NodeA"} + nodeB1 := nodeInfo{Role: "repeater", PublicKey: "b0b1eeee", Name: "B1"} + nodeB2 := nodeInfo{Role: "repeater", PublicKey: "b0c2ffff", Name: "B2"} pm := buildPrefixMap([]nodeInfo{nodeA, nodeB1, nodeB2}) @@ -175,7 +175,7 @@ func TestResolveAmbiguousEdges_FailsNoChange(t *testing.T) { // Test 3 (corrected): Resolution fails when prefix has no candidates in prefix map. func TestResolveAmbiguousEdges_NoMatch(t *testing.T) { - nodeA := nodeInfo{PublicKey: "aaaa1111", Name: "NodeA"} + nodeA := nodeInfo{Role: "repeater", PublicKey: "aaaa1111", Name: "NodeA"} // pm has no entries matching prefix "zz" pm := buildPrefixMap([]nodeInfo{nodeA}) @@ -215,8 +215,8 @@ func TestResolveAmbiguousEdges_NoMatch(t *testing.T) { // Test 6: Phase 1 edge collection unchanged (no regression). func TestPhase1EdgeCollection_Unchanged(t *testing.T) { // Build a simple graph and verify non-ambiguous edges are not touched. - nodeA := nodeInfo{PublicKey: "aaaa1111", Name: "NodeA", HasGPS: true, Lat: 45.0, Lon: -122.0} - nodeB := nodeInfo{PublicKey: "bbbb2222", Name: "NodeB", HasGPS: true, Lat: 45.1, Lon: -122.1} + nodeA := nodeInfo{Role: "repeater", PublicKey: "aaaa1111", Name: "NodeA", HasGPS: true, Lat: 45.0, Lon: -122.0} + nodeB := nodeInfo{Role: "repeater", PublicKey: "bbbb2222", Name: "NodeB", HasGPS: true, Lat: 45.1, Lon: -122.1} ts := time.Now().UTC().Format(time.RFC3339) payloadType := 4 @@ -232,7 +232,7 @@ func TestPhase1EdgeCollection_Unchanged(t *testing.T) { Observations: obs, } - store := ngTestStore([]nodeInfo{nodeA, nodeB, {PublicKey: "cccc3333", Name: "Observer"}}, []*StoreTx{tx}) + store := ngTestStore([]nodeInfo{nodeA, nodeB, {Role: "repeater", PublicKey: "cccc3333", Name: "Observer"}}, []*StoreTx{tx}) graph := BuildFromStore(store) edges := graph.Neighbors("aaaa1111") @@ -255,8 +255,8 @@ func TestPhase1EdgeCollection_Unchanged(t *testing.T) { // Test 7: Merge preserves higher LastSeen timestamp. func TestResolveAmbiguousEdges_PreservesHigherLastSeen(t *testing.T) { - nodeA := nodeInfo{PublicKey: "aaaa1111", Name: "NodeA", HasGPS: true, Lat: 45.0, Lon: -122.0} - nodeB := nodeInfo{PublicKey: "b0b1eeee", Name: "NodeB", HasGPS: true, Lat: 45.1, Lon: -122.1} + nodeA := nodeInfo{Role: "repeater", PublicKey: "aaaa1111", Name: "NodeA", HasGPS: true, Lat: 45.0, Lon: -122.0} + nodeB := nodeInfo{Role: "repeater", PublicKey: "b0b1eeee", Name: "NodeB", HasGPS: true, Lat: 45.1, Lon: -122.1} pm := buildPrefixMap([]nodeInfo{nodeA, nodeB}) graph := NewNeighborGraph() @@ -307,10 +307,10 @@ func TestResolveAmbiguousEdges_PreservesHigherLastSeen(t *testing.T) { // Test 5: Integration β node with both 1-byte and 2-byte prefix observations shows single entry. func TestIntegration_DualPrefixSingleNeighbor(t *testing.T) { - nodeA := nodeInfo{PublicKey: "aaaa1111aaaa1111", Name: "NodeA", HasGPS: true, Lat: 45.0, Lon: -122.0} - nodeB := nodeInfo{PublicKey: "b0b1eeeeb0b1eeee", Name: "NodeB", HasGPS: true, Lat: 45.1, Lon: -122.1} - nodeB2 := nodeInfo{PublicKey: "b0c2ffffb0c2ffff", Name: "NodeB2", HasGPS: true, Lat: 10.0, Lon: 10.0} - observer := nodeInfo{PublicKey: "cccc3333cccc3333", Name: "Observer"} + nodeA := nodeInfo{Role: "repeater", PublicKey: "aaaa1111aaaa1111", Name: "NodeA", HasGPS: true, Lat: 45.0, Lon: -122.0} + nodeB := nodeInfo{Role: "repeater", PublicKey: "b0b1eeeeb0b1eeee", Name: "NodeB", HasGPS: true, Lat: 45.1, Lon: -122.1} + nodeB2 := nodeInfo{Role: "repeater", PublicKey: "b0c2ffffb0c2ffff", Name: "NodeB2", HasGPS: true, Lat: 10.0, Lon: 10.0} + observer := nodeInfo{Role: "repeater", PublicKey: "cccc3333cccc3333", Name: "Observer"} ts := time.Now().UTC().Format(time.RFC3339) pt := 4 diff --git a/cmd/server/neighbor_graph_test.go b/cmd/server/neighbor_graph_test.go index 9500a134..7f7e2ac0 100644 --- a/cmd/server/neighbor_graph_test.go +++ b/cmd/server/neighbor_graph_test.go @@ -86,9 +86,9 @@ func TestBuildNeighborGraph_EmptyStore(t *testing.T) { func TestBuildNeighborGraph_AdvertSingleHopPath(t *testing.T) { // ADVERT from X, path=["R1_prefix"] β edges: XβR1 and ObserverβR1 nodes := []nodeInfo{ - {PublicKey: "aaaa1111", Name: "NodeX"}, - {PublicKey: "r1aabbcc", Name: "R1"}, - {PublicKey: "obs00001", Name: "Observer"}, + {Role: "repeater", PublicKey: "aaaa1111", Name: "NodeX"}, + {Role: "repeater", PublicKey: "r1aabbcc", Name: "R1"}, + {Role: "repeater", PublicKey: "obs00001", Name: "Observer"}, } tx := ngMakeTx(1, 4, ngFromNodeJSON("aaaa1111"), []*StoreObs{ ngMakeObs("obs00001", `["r1aa"]`, nowStr, ngFloatPtr(-10)), @@ -132,10 +132,10 @@ func TestBuildNeighborGraph_AdvertSingleHopPath(t *testing.T) { func TestBuildNeighborGraph_AdvertMultiHopPath(t *testing.T) { // ADVERT from X, path=["R1","R2"] β XβR1 and ObserverβR2 nodes := []nodeInfo{ - {PublicKey: "aaaa1111", Name: "NodeX"}, - {PublicKey: "r1aabbcc", Name: "R1"}, - {PublicKey: "r2ddeeff", Name: "R2"}, - {PublicKey: "obs00001", Name: "Observer"}, + {Role: "repeater", PublicKey: "aaaa1111", Name: "NodeX"}, + {Role: "repeater", PublicKey: "r1aabbcc", Name: "R1"}, + {Role: "repeater", PublicKey: "r2ddeeff", Name: "R2"}, + {Role: "repeater", PublicKey: "obs00001", Name: "Observer"}, } tx := ngMakeTx(1, 4, ngFromNodeJSON("aaaa1111"), []*StoreObs{ ngMakeObs("obs00001", `["r1aa","r2dd"]`, nowStr, nil), @@ -170,8 +170,8 @@ func TestBuildNeighborGraph_AdvertMultiHopPath(t *testing.T) { func TestBuildNeighborGraph_AdvertZeroHop(t *testing.T) { // ADVERT from X, path=[] β XβObserver direct edge nodes := []nodeInfo{ - {PublicKey: "aaaa1111", Name: "NodeX"}, - {PublicKey: "obs00001", Name: "Observer"}, + {Role: "repeater", PublicKey: "aaaa1111", Name: "NodeX"}, + {Role: "repeater", PublicKey: "obs00001", Name: "Observer"}, } tx := ngMakeTx(1, 4, ngFromNodeJSON("aaaa1111"), []*StoreObs{ ngMakeObs("obs00001", `[]`, nowStr, nil), @@ -195,8 +195,8 @@ func TestBuildNeighborGraph_AdvertZeroHop(t *testing.T) { func TestBuildNeighborGraph_NonAdvertEmptyPath(t *testing.T) { // Non-ADVERT, path=[] β no edges nodes := []nodeInfo{ - {PublicKey: "aaaa1111", Name: "NodeX"}, - {PublicKey: "obs00001", Name: "Observer"}, + {Role: "repeater", PublicKey: "aaaa1111", Name: "NodeX"}, + {Role: "repeater", PublicKey: "obs00001", Name: "Observer"}, } tx := ngMakeTx(1, 2, ngFromNodeJSON("aaaa1111"), []*StoreObs{ ngMakeObs("obs00001", `[]`, nowStr, nil), @@ -212,10 +212,10 @@ func TestBuildNeighborGraph_NonAdvertEmptyPath(t *testing.T) { func TestBuildNeighborGraph_NonAdvertOnlyObserverEdge(t *testing.T) { // Non-ADVERT with path=["R1","R2"] β only ObserverβR2, NO originator edge nodes := []nodeInfo{ - {PublicKey: "aaaa1111", Name: "NodeX"}, - {PublicKey: "r1aabbcc", Name: "R1"}, - {PublicKey: "r2ddeeff", Name: "R2"}, - {PublicKey: "obs00001", Name: "Observer"}, + {Role: "repeater", PublicKey: "aaaa1111", Name: "NodeX"}, + {Role: "repeater", PublicKey: "r1aabbcc", Name: "R1"}, + {Role: "repeater", PublicKey: "r2ddeeff", Name: "R2"}, + {Role: "repeater", PublicKey: "obs00001", Name: "Observer"}, } tx := ngMakeTx(1, 2, ngFromNodeJSON("aaaa1111"), []*StoreObs{ ngMakeObs("obs00001", `["r1aa","r2dd"]`, nowStr, nil), @@ -236,9 +236,9 @@ func TestBuildNeighborGraph_NonAdvertOnlyObserverEdge(t *testing.T) { func TestBuildNeighborGraph_NonAdvertSingleHop(t *testing.T) { // Non-ADVERT with path=["R1"] β ObserverβR1 only nodes := []nodeInfo{ - {PublicKey: "aaaa1111", Name: "NodeX"}, - {PublicKey: "r1aabbcc", Name: "R1"}, - {PublicKey: "obs00001", Name: "Observer"}, + {Role: "repeater", PublicKey: "aaaa1111", Name: "NodeX"}, + {Role: "repeater", PublicKey: "r1aabbcc", Name: "R1"}, + {Role: "repeater", PublicKey: "obs00001", Name: "Observer"}, } tx := ngMakeTx(1, 2, ngFromNodeJSON("aaaa1111"), []*StoreObs{ ngMakeObs("obs00001", `["r1aa"]`, nowStr, nil), @@ -259,10 +259,10 @@ func TestBuildNeighborGraph_NonAdvertSingleHop(t *testing.T) { func TestBuildNeighborGraph_HashCollision(t *testing.T) { // Two nodes share prefix "a3" β ambiguous edge nodes := []nodeInfo{ - {PublicKey: "aaaa1111", Name: "NodeX"}, - {PublicKey: "a3bb1111", Name: "CandidateA"}, - {PublicKey: "a3bb2222", Name: "CandidateB"}, - {PublicKey: "obs00001", Name: "Observer"}, + {Role: "repeater", PublicKey: "aaaa1111", Name: "NodeX"}, + {Role: "repeater", PublicKey: "a3bb1111", Name: "CandidateA"}, + {Role: "repeater", PublicKey: "a3bb2222", Name: "CandidateB"}, + {Role: "repeater", PublicKey: "obs00001", Name: "Observer"}, } tx := ngMakeTx(1, 4, ngFromNodeJSON("aaaa1111"), []*StoreObs{ ngMakeObs("obs00001", `["a3bb"]`, nowStr, nil), @@ -308,13 +308,13 @@ func TestBuildNeighborGraph_ConfidenceAutoResolve(t *testing.T) { // CandidateB has no known neighbors (Jaccard = 0). // An ambiguous edge Xβprefix "a3" with candidates [A, B] should auto-resolve to A. nodes := []nodeInfo{ - {PublicKey: "aaaa1111", Name: "NodeX"}, - {PublicKey: "n1111111", Name: "N1"}, - {PublicKey: "n2222222", Name: "N2"}, - {PublicKey: "n3333333", Name: "N3"}, - {PublicKey: "a3001111", Name: "CandidateA"}, - {PublicKey: "a3002222", Name: "CandidateB"}, - {PublicKey: "obs00001", Name: "Observer"}, + {Role: "repeater", PublicKey: "aaaa1111", Name: "NodeX"}, + {Role: "repeater", PublicKey: "n1111111", Name: "N1"}, + {Role: "repeater", PublicKey: "n2222222", Name: "N2"}, + {Role: "repeater", PublicKey: "n3333333", Name: "N3"}, + {Role: "repeater", PublicKey: "a3001111", Name: "CandidateA"}, + {Role: "repeater", PublicKey: "a3002222", Name: "CandidateB"}, + {Role: "repeater", PublicKey: "obs00001", Name: "Observer"}, } // Create resolved edges: XβN1, XβN2, XβN3, AβN1, AβN2, AβN3 @@ -373,11 +373,11 @@ func TestBuildNeighborGraph_ConfidenceAutoResolve(t *testing.T) { func TestBuildNeighborGraph_EqualScoresAmbiguous(t *testing.T) { // Two candidates with identical neighbor sets β should NOT auto-resolve. nodes := []nodeInfo{ - {PublicKey: "aaaa1111", Name: "NodeX"}, - {PublicKey: "n1111111", Name: "N1"}, - {PublicKey: "a3001111", Name: "CandidateA"}, - {PublicKey: "a3002222", Name: "CandidateB"}, - {PublicKey: "obs00001", Name: "Observer"}, + {Role: "repeater", PublicKey: "aaaa1111", Name: "NodeX"}, + {Role: "repeater", PublicKey: "n1111111", Name: "N1"}, + {Role: "repeater", PublicKey: "a3001111", Name: "CandidateA"}, + {Role: "repeater", PublicKey: "a3002222", Name: "CandidateB"}, + {Role: "repeater", PublicKey: "obs00001", Name: "Observer"}, } var txs []*StoreTx @@ -425,8 +425,8 @@ func TestBuildNeighborGraph_EqualScoresAmbiguous(t *testing.T) { func TestBuildNeighborGraph_ObserverSelfEdgeGuard(t *testing.T) { // Observer's own prefix in path β should NOT create self-edge. nodes := []nodeInfo{ - {PublicKey: "aaaa1111", Name: "NodeX"}, - {PublicKey: "obs00001", Name: "Observer"}, + {Role: "repeater", PublicKey: "aaaa1111", Name: "NodeX"}, + {Role: "repeater", PublicKey: "obs00001", Name: "Observer"}, } tx := ngMakeTx(1, 4, ngFromNodeJSON("aaaa1111"), []*StoreObs{ ngMakeObs("obs00001", `["obs0"]`, nowStr, nil), @@ -445,8 +445,8 @@ func TestBuildNeighborGraph_ObserverSelfEdgeGuard(t *testing.T) { func TestBuildNeighborGraph_OrphanPrefix(t *testing.T) { // Path contains prefix matching zero nodes β edge recorded as unresolved. nodes := []nodeInfo{ - {PublicKey: "aaaa1111", Name: "NodeX"}, - {PublicKey: "obs00001", Name: "Observer"}, + {Role: "repeater", PublicKey: "aaaa1111", Name: "NodeX"}, + {Role: "repeater", PublicKey: "obs00001", Name: "Observer"}, } tx := ngMakeTx(1, 4, ngFromNodeJSON("aaaa1111"), []*StoreObs{ ngMakeObs("obs00001", `["ff99"]`, nowStr, nil), @@ -506,9 +506,9 @@ func TestAffinityScore_StaleAndLow(t *testing.T) { func TestBuildNeighborGraph_CountAccumulation(t *testing.T) { nodes := []nodeInfo{ - {PublicKey: "aaaa1111", Name: "NodeX"}, - {PublicKey: "r1aabbcc", Name: "R1"}, - {PublicKey: "obs00001", Name: "Observer"}, + {Role: "repeater", PublicKey: "aaaa1111", Name: "NodeX"}, + {Role: "repeater", PublicKey: "r1aabbcc", Name: "R1"}, + {Role: "repeater", PublicKey: "obs00001", Name: "Observer"}, } var txs []*StoreTx @@ -535,10 +535,10 @@ func TestBuildNeighborGraph_CountAccumulation(t *testing.T) { func TestBuildNeighborGraph_MultipleObservers(t *testing.T) { nodes := []nodeInfo{ - {PublicKey: "aaaa1111", Name: "NodeX"}, - {PublicKey: "r1aabbcc", Name: "R1"}, - {PublicKey: "obs00001", Name: "Obs1"}, - {PublicKey: "obs00002", Name: "Obs2"}, + {Role: "repeater", PublicKey: "aaaa1111", Name: "NodeX"}, + {Role: "repeater", PublicKey: "r1aabbcc", Name: "R1"}, + {Role: "repeater", PublicKey: "obs00001", Name: "Obs1"}, + {Role: "repeater", PublicKey: "obs00002", Name: "Obs2"}, } tx := ngMakeTx(1, 4, ngFromNodeJSON("aaaa1111"), []*StoreObs{ @@ -565,9 +565,9 @@ func TestBuildNeighborGraph_MultipleObservers(t *testing.T) { func TestBuildNeighborGraph_TimeDecayOldObservations(t *testing.T) { nodes := []nodeInfo{ - {PublicKey: "aaaa1111", Name: "NodeX"}, - {PublicKey: "r1aabbcc", Name: "R1"}, - {PublicKey: "obs00001", Name: "Observer"}, + {Role: "repeater", PublicKey: "aaaa1111", Name: "NodeX"}, + {Role: "repeater", PublicKey: "r1aabbcc", Name: "R1"}, + {Role: "repeater", PublicKey: "obs00001", Name: "Observer"}, } tx := ngMakeTx(1, 4, ngFromNodeJSON("aaaa1111"), []*StoreObs{ @@ -592,10 +592,10 @@ func TestBuildNeighborGraph_TimeDecayOldObservations(t *testing.T) { func TestBuildNeighborGraph_ADVERTOnlyConstraint(t *testing.T) { // Non-ADVERT: should NOT create originatorβpath[0] edge, only observerβpath[last]. nodes := []nodeInfo{ - {PublicKey: "aaaa1111", Name: "NodeX"}, - {PublicKey: "r1aabbcc", Name: "R1"}, - {PublicKey: "r2ddeeff", Name: "R2"}, - {PublicKey: "obs00001", Name: "Observer"}, + {Role: "repeater", PublicKey: "aaaa1111", Name: "NodeX"}, + {Role: "repeater", PublicKey: "r1aabbcc", Name: "R1"}, + {Role: "repeater", PublicKey: "r2ddeeff", Name: "R2"}, + {Role: "repeater", PublicKey: "obs00001", Name: "Observer"}, } tx := ngMakeTx(1, 2, ngFromNodeJSON("aaaa1111"), []*StoreObs{ ngMakeObs("obs00001", `["r1aa","r2dd"]`, nowStr, nil), @@ -631,9 +631,9 @@ func ngPubKeyJSON(pubkey string) string { func TestBuildNeighborGraph_AdvertPubKeyField(t *testing.T) { // Real ADVERTs use "pubKey", not "from_node". Verify the builder handles it. nodes := []nodeInfo{ - {PublicKey: "99bf37abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234", Name: "Originator"}, - {PublicKey: "r1aabbccdd001122334455667788990011223344556677889900112233445566", Name: "R1"}, - {PublicKey: "obs0000100112233445566778899001122334455667788990011223344556677", Name: "Observer"}, + {Role: "repeater", PublicKey: "99bf37abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234", Name: "Originator"}, + {Role: "repeater", PublicKey: "r1aabbccdd001122334455667788990011223344556677889900112233445566", Name: "R1"}, + {Role: "repeater", PublicKey: "obs0000100112233445566778899001122334455667788990011223344556677", Name: "Observer"}, } tx := ngMakeTx(1, 4, ngPubKeyJSON("99bf37abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234"), []*StoreObs{ ngMakeObs("obs0000100112233445566778899001122334455667788990011223344556677", `["r1"]`, nowStr, ngFloatPtr(-8.5)), @@ -666,10 +666,10 @@ func TestBuildNeighborGraph_OneByteHashPrefixes(t *testing.T) { // Real-world scenario: 1-byte hash prefixes with multiple candidates. // Should create edges (possibly ambiguous) rather than empty graph. nodes := []nodeInfo{ - {PublicKey: "c0dedad400000000000000000000000000000000000000000000000000000001", Name: "NodeC0-1"}, - {PublicKey: "c0dedad900000000000000000000000000000000000000000000000000000002", Name: "NodeC0-2"}, - {PublicKey: "a3bbccdd00000000000000000000000000000000000000000000000000000003", Name: "Originator"}, - {PublicKey: "obs1234500000000000000000000000000000000000000000000000000000004", Name: "Observer"}, + {Role: "repeater", PublicKey: "c0dedad400000000000000000000000000000000000000000000000000000001", Name: "NodeC0-1"}, + {Role: "repeater", PublicKey: "c0dedad900000000000000000000000000000000000000000000000000000002", Name: "NodeC0-2"}, + {Role: "repeater", PublicKey: "a3bbccdd00000000000000000000000000000000000000000000000000000003", Name: "Originator"}, + {Role: "repeater", PublicKey: "obs1234500000000000000000000000000000000000000000000000000000004", Name: "Observer"}, } // ADVERT from Originator with 1-byte path hop "c0" tx := ngMakeTx(1, 4, ngPubKeyJSON("a3bbccdd00000000000000000000000000000000000000000000000000000003"), []*StoreObs{ @@ -809,10 +809,10 @@ func TestExtractFromNode_UsesCachedParse(t *testing.T) { func BenchmarkBuildFromStore(b *testing.B) { // Simulate a dataset with many packets and repeated pubkeys nodes := []nodeInfo{ - {PublicKey: "aaaa1111", Name: "NodeA"}, - {PublicKey: "bbbb2222", Name: "NodeB"}, - {PublicKey: "cccc3333", Name: "NodeC"}, - {PublicKey: "dddd4444", Name: "NodeD"}, + {Role: "repeater", PublicKey: "aaaa1111", Name: "NodeA"}, + {Role: "repeater", PublicKey: "bbbb2222", Name: "NodeB"}, + {Role: "repeater", PublicKey: "cccc3333", Name: "NodeC"}, + {Role: "repeater", PublicKey: "dddd4444", Name: "NodeD"}, } const numPackets = 1000 packets := make([]*StoreTx, 0, numPackets) diff --git a/cmd/server/neighbor_persist_test.go b/cmd/server/neighbor_persist_test.go index 6e046241..33d29efc 100644 --- a/cmd/server/neighbor_persist_test.go +++ b/cmd/server/neighbor_persist_test.go @@ -58,8 +58,8 @@ func createTestDBWithSchema(t *testing.T) (*DB, string) { func TestResolvePathForObs(t *testing.T) { // Build a prefix map with known nodes nodes := []nodeInfo{ - {PublicKey: "aabbccddee1234567890aabbccddee1234567890aabbccddee1234567890aabb", Name: "Node-AA"}, - {PublicKey: "bbccddee1234567890aabbccddee1234567890aabbccddee1234567890aabb11", Name: "Node-BB"}, + {Role: "repeater", PublicKey: "aabbccddee1234567890aabbccddee1234567890aabbccddee1234567890aabb", Name: "Node-AA"}, + {Role: "repeater", PublicKey: "bbccddee1234567890aabbccddee1234567890aabbccddee1234567890aabb11", Name: "Node-BB"}, } pm := buildPrefixMap(nodes) graph := NewNeighborGraph() @@ -97,7 +97,7 @@ func TestResolvePathForObs_EmptyPath(t *testing.T) { func TestResolvePathForObs_Unresolvable(t *testing.T) { nodes := []nodeInfo{ - {PublicKey: "aabbccddee1234567890aabbccddee1234567890aabbccddee1234567890aabb", Name: "Node-AA"}, + {Role: "repeater", PublicKey: "aabbccddee1234567890aabbccddee1234567890aabbccddee1234567890aabb", Name: "Node-AA"}, } pm := buildPrefixMap(nodes) @@ -437,8 +437,8 @@ func TestExtractEdgesFromObs_NonAdvertNoPath(t *testing.T) { func TestExtractEdgesFromObs_WithPath(t *testing.T) { nodes := []nodeInfo{ - {PublicKey: "aabbccddee1234567890aabbccddee1234567890aabbccddee1234567890aabb", Name: "Node-AA"}, - {PublicKey: "ffgghhii1234567890aabbccddee1234567890aabbccddee1234567890aabb11", Name: "Node-FF"}, + {Role: "repeater", PublicKey: "aabbccddee1234567890aabbccddee1234567890aabbccddee1234567890aabb", Name: "Node-AA"}, + {Role: "repeater", PublicKey: "ffgghhii1234567890aabbccddee1234567890aabbccddee1234567890aabb11", Name: "Node-FF"}, } pm := buildPrefixMap(nodes) diff --git a/cmd/server/path_inspect.go b/cmd/server/path_inspect.go new file mode 100644 index 00000000..43b9ffe6 --- /dev/null +++ b/cmd/server/path_inspect.go @@ -0,0 +1,427 @@ +package main + +import ( + "encoding/hex" + "encoding/json" + "math" + "net/http" + "sort" + "strings" + "time" +) + +// βββ Path Inspector ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ +// POST /api/paths/inspect β beam-search scorer for prefix path candidates. +// Spec: issue #944 Β§2.1β2.5. + +// pathInspectRequest is the JSON body for the inspect endpoint. +type pathInspectRequest struct { + Prefixes []string `json:"prefixes"` + Context *pathInspectContext `json:"context,omitempty"` + Limit int `json:"limit,omitempty"` +} + +type pathInspectContext struct { + ObserverID string `json:"observerId,omitempty"` + Since string `json:"since,omitempty"` + Until string `json:"until,omitempty"` +} + +// pathCandidate is one scored candidate path in the response. +type pathCandidate struct { + Path []string `json:"path"` + Names []string `json:"names"` + Score float64 `json:"score"` + Speculative bool `json:"speculative"` + Evidence pathEvidence `json:"evidence"` +} + +type pathEvidence struct { + PerHop []hopEvidence `json:"perHop"` +} + +type hopEvidence struct { + Prefix string `json:"prefix"` + CandidatesConsidered int `json:"candidatesConsidered"` + Chosen string `json:"chosen"` + EdgeWeight float64 `json:"edgeWeight"` + Alternatives []hopAlternative `json:"alternatives,omitempty"` +} + +// hopAlternative shows a candidate that was considered but not chosen for this hop. +type hopAlternative struct { + PublicKey string `json:"publicKey"` + Name string `json:"name"` + Score float64 `json:"score"` +} + +type pathInspectResponse struct { + Candidates []pathCandidate `json:"candidates"` + Input map[string]interface{} `json:"input"` + Stats map[string]interface{} `json:"stats"` +} + +// beamEntry represents a partial path being extended during beam search. +type beamEntry struct { + pubkeys []string + names []string + evidence []hopEvidence + score float64 // product of per-hop scores (pre-geometric-mean) +} + +const ( + beamWidth = 20 + maxInputHops = 64 + maxPrefixBytes = 3 + maxRequestItems = 64 + geoMaxKm = 50.0 + hopScoreFloor = 0.05 + speculativeThreshold = 0.7 + inspectCacheTTL = 30 * time.Second + inspectBodyLimit = 4096 +) + +// Weights per spec Β§2.3. +const ( + wEdge = 0.35 + wGeo = 0.20 + wRecency = 0.15 + wSelectivity = 0.30 +) + +func (s *Server) handlePathInspect(w http.ResponseWriter, r *http.Request) { + // Body limit per spec Β§2.1. + r.Body = http.MaxBytesReader(w, r.Body, inspectBodyLimit) + + var req pathInspectRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, `{"error":"invalid JSON"}`, http.StatusBadRequest) + return + } + + // Validate prefixes. + if len(req.Prefixes) == 0 { + http.Error(w, `{"error":"prefixes required"}`, http.StatusBadRequest) + return + } + if len(req.Prefixes) > maxRequestItems { + http.Error(w, `{"error":"too many prefixes (max 64)"}`, http.StatusBadRequest) + return + } + + // Normalize + validate each prefix. + prefixByteLen := -1 + for i, p := range req.Prefixes { + p = strings.ToLower(strings.TrimSpace(p)) + req.Prefixes[i] = p + if len(p) == 0 || len(p)%2 != 0 { + http.Error(w, `{"error":"prefixes must be even-length hex"}`, http.StatusBadRequest) + return + } + if _, err := hex.DecodeString(p); err != nil { + http.Error(w, `{"error":"prefixes must be valid hex"}`, http.StatusBadRequest) + return + } + byteLen := len(p) / 2 + if byteLen > maxPrefixBytes { + http.Error(w, `{"error":"prefix exceeds 3 bytes"}`, http.StatusBadRequest) + return + } + if prefixByteLen == -1 { + prefixByteLen = byteLen + } else if byteLen != prefixByteLen { + http.Error(w, `{"error":"mixed prefix lengths not allowed"}`, http.StatusBadRequest) + return + } + } + + limit := req.Limit + if limit <= 0 { + limit = 10 + } + if limit > 50 { + limit = 50 + } + + // Check cache. + cacheKey := s.store.inspectCacheKey(req) + s.store.inspectMu.RLock() + if cached, ok := s.store.inspectCache[cacheKey]; ok && time.Now().Before(cached.expiresAt) { + s.store.inspectMu.RUnlock() + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(cached.data) + return + } + s.store.inspectMu.RUnlock() + + // Snapshot data under read lock. + nodes, pm := s.store.getCachedNodesAndPM() + + // Build pubkeyβnodeInfo map for O(1) geo lookup in scorer. + nodeByPK := make(map[string]*nodeInfo, len(nodes)) + for i := range nodes { + nodeByPK[strings.ToLower(nodes[i].PublicKey)] = &nodes[i] + } + + // Get neighbor graph; handle cold start. + graph := s.store.graph + if graph == nil || graph.IsStale() { + rebuilt := make(chan struct{}) + go func() { + s.store.ensureNeighborGraph() + close(rebuilt) + }() + select { + case <-rebuilt: + graph = s.store.graph + case <-time.After(2 * time.Second): + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusServiceUnavailable) + json.NewEncoder(w).Encode(map[string]interface{}{"retry": true}) + return + } + if graph == nil { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusServiceUnavailable) + json.NewEncoder(w).Encode(map[string]interface{}{"retry": true}) + return + } + } + + now := time.Now() + start := now + + // Beam search. + beam := s.store.beamSearch(req.Prefixes, pm, graph, nodeByPK, now) + + // Sort by score descending, take top limit. + sortBeam(beam) + if len(beam) > limit { + beam = beam[:limit] + } + + // Build response with per-hop alternatives (spec Β§2.7, M2 fix). + candidates := make([]pathCandidate, 0, len(beam)) + for _, entry := range beam { + nHops := len(entry.pubkeys) + var score float64 + if nHops > 0 { + score = math.Pow(entry.score, 1.0/float64(nHops)) + } + + // Populate per-hop alternatives: other candidates at each hop that weren't chosen. + evidence := make([]hopEvidence, len(entry.evidence)) + copy(evidence, entry.evidence) + for hi, ev := range evidence { + if hi >= len(req.Prefixes) { + break + } + prefix := req.Prefixes[hi] + allCands := pm.m[prefix] + var alts []hopAlternative + for _, c := range allCands { + if !canAppearInPath(c.Role) || c.PublicKey == ev.Chosen { + continue + } + // Score this alternative in context of the partial path up to this hop. + var partialEntry beamEntry + if hi > 0 { + partialEntry = beamEntry{pubkeys: entry.pubkeys[:hi], names: entry.names[:hi], score: 1.0} + } + altScore := s.store.scoreHop(partialEntry, c, ev.CandidatesConsidered, graph, nodeByPK, now, hi) + alts = append(alts, hopAlternative{PublicKey: c.PublicKey, Name: c.Name, Score: math.Round(altScore*1000) / 1000}) + } + // Sort alts by score desc, cap at 5. + sort.Slice(alts, func(i, j int) bool { return alts[i].Score > alts[j].Score }) + if len(alts) > 5 { + alts = alts[:5] + } + evidence[hi] = hopEvidence{ + Prefix: ev.Prefix, + CandidatesConsidered: ev.CandidatesConsidered, + Chosen: ev.Chosen, + EdgeWeight: ev.EdgeWeight, + Alternatives: alts, + } + } + + candidates = append(candidates, pathCandidate{ + Path: entry.pubkeys, + Names: entry.names, + Score: math.Round(score*1000) / 1000, + Speculative: score < speculativeThreshold, + Evidence: pathEvidence{PerHop: evidence}, + }) + } + + elapsed := time.Since(start).Milliseconds() + resp := pathInspectResponse{ + Candidates: candidates, + Input: map[string]interface{}{ + "prefixes": req.Prefixes, + "hops": len(req.Prefixes), + }, + Stats: map[string]interface{}{ + "beamWidth": beamWidth, + "expansionsRun": len(req.Prefixes) * beamWidth, + "elapsedMs": elapsed, + }, + } + + // Cache result (and evict stale entries). + s.store.inspectMu.Lock() + if s.store.inspectCache == nil { + s.store.inspectCache = make(map[string]*inspectCachedResult) + } + now2 := time.Now() + for k, v := range s.store.inspectCache { + if now2.After(v.expiresAt) { + delete(s.store.inspectCache, k) + } + } + s.store.inspectCache[cacheKey] = &inspectCachedResult{ + data: resp, + expiresAt: now2.Add(inspectCacheTTL), + } + s.store.inspectMu.Unlock() + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(resp) +} + +type inspectCachedResult struct { + data pathInspectResponse + expiresAt time.Time +} + +func (s *PacketStore) inspectCacheKey(req pathInspectRequest) string { + key := strings.Join(req.Prefixes, ",") + if req.Context != nil { + key += "|" + req.Context.ObserverID + "|" + req.Context.Since + "|" + req.Context.Until + } + return key +} + +func (s *PacketStore) beamSearch(prefixes []string, pm *prefixMap, graph *NeighborGraph, nodeByPK map[string]*nodeInfo, now time.Time) []beamEntry { + // Start with empty beam. + beam := []beamEntry{{pubkeys: nil, names: nil, evidence: nil, score: 1.0}} + + for hopIdx, prefix := range prefixes { + candidates := pm.m[prefix] + // Filter by role at lookup time (spec Β§2.2 step 2). + var filtered []nodeInfo + for _, c := range candidates { + if canAppearInPath(c.Role) { + filtered = append(filtered, c) + } + } + + candidateCount := len(filtered) + if candidateCount == 0 { + // No candidates for this hop β beam dies. + return nil + } + + var nextBeam []beamEntry + for _, entry := range beam { + for _, cand := range filtered { + hopScore := s.scoreHop(entry, cand, candidateCount, graph, nodeByPK, now, hopIdx) + if hopScore < hopScoreFloor { + hopScore = hopScoreFloor + } + + newEntry := beamEntry{ + pubkeys: append(append([]string{}, entry.pubkeys...), cand.PublicKey), + names: append(append([]string{}, entry.names...), cand.Name), + evidence: append(append([]hopEvidence{}, entry.evidence...), hopEvidence{ + Prefix: prefix, + CandidatesConsidered: candidateCount, + Chosen: cand.PublicKey, + EdgeWeight: hopScore, + }), + score: entry.score * hopScore, + } + nextBeam = append(nextBeam, newEntry) + } + } + + // Prune to beam width. + sortBeam(nextBeam) + if len(nextBeam) > beamWidth { + nextBeam = nextBeam[:beamWidth] + } + beam = nextBeam + } + + return beam +} + +func (s *PacketStore) scoreHop(entry beamEntry, cand nodeInfo, candidateCount int, graph *NeighborGraph, nodeByPK map[string]*nodeInfo, now time.Time, hopIdx int) float64 { + var edgeScore float64 + var geoScore float64 = 1.0 + var recencyScore float64 = 1.0 + + if hopIdx == 0 || len(entry.pubkeys) == 0 { + // First hop: no prior node to compare against. + edgeScore = 1.0 + } else { + lastPK := entry.pubkeys[len(entry.pubkeys)-1] + + // Single scan over neighbors for both edge weight and recency. + edges := graph.Neighbors(lastPK) + var foundEdge *NeighborEdge + for _, e := range edges { + peer := e.NodeA + if strings.EqualFold(peer, lastPK) { + peer = e.NodeB + } + if strings.EqualFold(peer, cand.PublicKey) { + foundEdge = e + break + } + } + + if foundEdge != nil { + edgeScore = foundEdge.Score(now) + hoursSince := now.Sub(foundEdge.LastSeen).Hours() + if hoursSince <= 24 { + recencyScore = 1.0 + } else { + recencyScore = math.Max(0.1, 24.0/hoursSince) + } + } else { + edgeScore = 0 + recencyScore = 0 + } + + // Geographic plausibility. + prevNode := nodeByPK[strings.ToLower(lastPK)] + if prevNode != nil && prevNode.HasGPS && cand.HasGPS { + dist := haversineKm(prevNode.Lat, prevNode.Lon, cand.Lat, cand.Lon) + if dist > geoMaxKm { + geoScore = math.Max(0.1, geoMaxKm/dist) + } + } + } + + // Prefix selectivity. + selectivityScore := 1.0 / float64(candidateCount) + + return wEdge*edgeScore + wGeo*geoScore + wRecency*recencyScore + wSelectivity*selectivityScore +} + + +func sortBeam(beam []beamEntry) { + sort.Slice(beam, func(i, j int) bool { + return beam[i].score > beam[j].score + }) +} + +// ensureNeighborGraph triggers a graph rebuild if nil or stale. +func (s *PacketStore) ensureNeighborGraph() { + if s.graph != nil && !s.graph.IsStale() { + return + } + g := BuildFromStore(s) + s.graph = g +} diff --git a/cmd/server/path_inspect_test.go b/cmd/server/path_inspect_test.go new file mode 100644 index 00000000..e5e6cc49 --- /dev/null +++ b/cmd/server/path_inspect_test.go @@ -0,0 +1,308 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "math" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +// βββ Unit tests for path inspector (issue #944) ββββββββββββββββββββββββββββββββ + +func TestScoreHop_EdgeWeight(t *testing.T) { + store := &PacketStore{} + graph := NewNeighborGraph() + now := time.Now() + + // Add an edge between A and B. + graph.mu.Lock() + edge := &NeighborEdge{ + NodeA: "aaaa", NodeB: "bbbb", + Count: 50, LastSeen: now.Add(-1 * time.Hour), + Observers: map[string]bool{"obs1": true}, + } + key := edgeKey{"aaaa", "bbbb"} + graph.edges[key] = edge + graph.byNode["aaaa"] = append(graph.byNode["aaaa"], edge) + graph.byNode["bbbb"] = append(graph.byNode["bbbb"], edge) + graph.mu.Unlock() + + entry := beamEntry{pubkeys: []string{"aaaa"}, names: []string{"NodeA"}} + cand := nodeInfo{PublicKey: "bbbb", Name: "NodeB", Role: "repeater"} + + score := store.scoreHop(entry, cand, 2, graph, nil, now, 1) + + // With edge present, edgeScore > 0. With 2 candidates, selectivity = 0.5. + // Anti-tautology: if we zero out edge weight constant, score would change. + if score <= 0.05 { + t.Errorf("expected score > floor, got %f", score) + } + + // No edge: score should be lower. + candNoEdge := nodeInfo{PublicKey: "cccc", Name: "NodeC", Role: "repeater"} + scoreNoEdge := store.scoreHop(entry, candNoEdge, 2, graph, nil, now, 1) + if scoreNoEdge >= score { + t.Errorf("expected no-edge score (%f) < edge score (%f)", scoreNoEdge, score) + } +} + +func TestScoreHop_FirstHop(t *testing.T) { + store := &PacketStore{} + graph := NewNeighborGraph() + now := time.Now() + + entry := beamEntry{pubkeys: nil, names: nil} + cand := nodeInfo{PublicKey: "aaaa", Name: "NodeA", Role: "repeater"} + + score := store.scoreHop(entry, cand, 3, graph, nil, now, 0) + // First hop: edgeScore=1.0, geoScore=1.0, recencyScore=1.0, selectivity=1/3 + // = 0.35*1 + 0.20*1 + 0.15*1 + 0.30*(1/3) = 0.35+0.20+0.15+0.10 = 0.80 + expected := 0.35 + 0.20 + 0.15 + 0.30/3.0 + if score < expected-0.01 || score > expected+0.01 { + t.Errorf("expected ~%f, got %f", expected, score) + } +} + +func TestScoreHop_GeoPlausibility(t *testing.T) { + store := &PacketStore{} + store.nodeCache = []nodeInfo{ + {PublicKey: "aaaa", Name: "A", Role: "repeater", Lat: 37.0, Lon: -122.0, HasGPS: true}, + {PublicKey: "bbbb", Name: "B", Role: "repeater", Lat: 37.01, Lon: -122.01, HasGPS: true}, // ~1.4km + {PublicKey: "cccc", Name: "C", Role: "repeater", Lat: 40.0, Lon: -120.0, HasGPS: true}, // ~400km + } + store.nodePM = buildPrefixMap(store.nodeCache) + store.nodeCacheTime = time.Now() + + graph := NewNeighborGraph() + now := time.Now() + + nodeByPK := map[string]*nodeInfo{ + "aaaa": &store.nodeCache[0], + "bbbb": &store.nodeCache[1], + "cccc": &store.nodeCache[2], + } + + entry := beamEntry{pubkeys: []string{"aaaa"}, names: []string{"A"}} + + // Close node should score higher than far node (geo component). + scoreClose := store.scoreHop(entry, store.nodeCache[1], 2, graph, nodeByPK, now, 1) + scoreFar := store.scoreHop(entry, store.nodeCache[2], 2, graph, nodeByPK, now, 1) + if scoreFar >= scoreClose { + t.Errorf("expected far node score (%f) < close node score (%f)", scoreFar, scoreClose) + } +} + +func TestBeamSearch_WidthCap(t *testing.T) { + store := &PacketStore{} + graph := NewNeighborGraph() + graph.builtAt = time.Now() + now := time.Now() + + // Create 25 nodes that all match prefix "aa". + var nodes []nodeInfo + for i := 0; i < 25; i++ { + // Each node has pubkey starting with "aa" followed by unique hex. + pk := "aa" + strings.Repeat("0", 4) + fmt.Sprintf("%02x", i) + nodes = append(nodes, nodeInfo{PublicKey: pk, Name: pk, Role: "repeater"}) + } + pm := buildPrefixMap(nodes) + + // Two hops of "aa" β should produce 25*25=625 combos, pruned to 20. + beam := store.beamSearch([]string{"aa", "aa"}, pm, graph, nil, now) + if len(beam) > beamWidth { + t.Errorf("beam exceeded width: got %d, want <= %d", len(beam), beamWidth) + } + // Anti-tautology: without beam pruning, we'd have up to 25*min(25,beamWidth)=500 entries. + // The test verifies pruning is effective. +} + +func TestBeamSearch_Speculative(t *testing.T) { + store := &PacketStore{} + graph := NewNeighborGraph() + graph.builtAt = time.Now() + now := time.Now() + + // Create nodes with no edges and multiple candidates β should result in low scores (speculative). + nodes := []nodeInfo{ + {PublicKey: "aabb", Name: "N1", Role: "repeater"}, + {PublicKey: "aabb22", Name: "N1b", Role: "repeater"}, + {PublicKey: "ccdd", Name: "N2", Role: "repeater"}, + {PublicKey: "ccdd22", Name: "N2b", Role: "repeater"}, + {PublicKey: "ccdd33", Name: "N2c", Role: "repeater"}, + } + pm := buildPrefixMap(nodes) + + beam := store.beamSearch([]string{"aa", "cc"}, pm, graph, nil, now) + if len(beam) == 0 { + t.Fatal("expected at least one result") + } + + // Score should be < 0.7 since there's no edge and multiple candidates (speculative). + nHops := len(beam[0].pubkeys) + score := 1.0 + if nHops > 0 { + product := beam[0].score + score = pow(product, 1.0/float64(nHops)) + } + if score >= speculativeThreshold { + t.Errorf("expected speculative score (< %f), got %f", speculativeThreshold, score) + } +} + +func TestHandlePathInspect_EmptyPrefixes(t *testing.T) { + srv := newTestServerForInspect(t) + body := `{"prefixes":[]}` + rr := doInspectRequest(srv, body) + if rr.Code != http.StatusBadRequest { + t.Errorf("expected 400, got %d", rr.Code) + } +} + +func TestHandlePathInspect_OddLengthPrefix(t *testing.T) { + srv := newTestServerForInspect(t) + body := `{"prefixes":["abc"]}` + rr := doInspectRequest(srv, body) + if rr.Code != http.StatusBadRequest { + t.Errorf("expected 400 for odd-length prefix, got %d", rr.Code) + } +} + +func TestHandlePathInspect_MixedLengths(t *testing.T) { + srv := newTestServerForInspect(t) + body := `{"prefixes":["aa","bbcc"]}` + rr := doInspectRequest(srv, body) + if rr.Code != http.StatusBadRequest { + t.Errorf("expected 400 for mixed lengths, got %d", rr.Code) + } +} + +func TestHandlePathInspect_TooLongPrefix(t *testing.T) { + srv := newTestServerForInspect(t) + body := `{"prefixes":["aabbccdd"]}` + rr := doInspectRequest(srv, body) + if rr.Code != http.StatusBadRequest { + t.Errorf("expected 400 for >3-byte prefix, got %d", rr.Code) + } +} + +func TestHandlePathInspect_TooManyPrefixes(t *testing.T) { + srv := newTestServerForInspect(t) + prefixes := make([]string, 65) + for i := range prefixes { + prefixes[i] = "aa" + } + b, _ := json.Marshal(map[string]interface{}{"prefixes": prefixes}) + rr := doInspectRequest(srv, string(b)) + if rr.Code != http.StatusBadRequest { + t.Errorf("expected 400 for >64 prefixes, got %d", rr.Code) + } +} + +func TestHandlePathInspect_ValidRequest(t *testing.T) { + srv := newTestServerForInspect(t) + + // Seed nodes in the store β multiple candidates per prefix to lower selectivity. + srv.store.nodeCache = []nodeInfo{ + {PublicKey: "aabb1234", Name: "NodeA", Role: "repeater", Lat: 37.0, Lon: -122.0, HasGPS: true}, + {PublicKey: "aabb5678", Name: "NodeA2", Role: "repeater"}, + {PublicKey: "ccdd5678", Name: "NodeB", Role: "repeater", Lat: 37.01, Lon: -122.01, HasGPS: true}, + {PublicKey: "ccdd9999", Name: "NodeB2", Role: "repeater"}, + {PublicKey: "ccdd1111", Name: "NodeB3", Role: "repeater"}, + } + srv.store.nodePM = buildPrefixMap(srv.store.nodeCache) + srv.store.nodeCacheTime = time.Now() + srv.store.graph = NewNeighborGraph() + srv.store.graph.builtAt = time.Now() + + body := `{"prefixes":["aa","cc"]}` + rr := doInspectRequest(srv, body) + if rr.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", rr.Code, rr.Body.String()) + } + + var resp pathInspectResponse + if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil { + t.Fatalf("invalid JSON response: %v", err) + } + if len(resp.Candidates) == 0 { + t.Error("expected at least one candidate") + } + if resp.Candidates[0].Speculative != true { + // No edge between nodes, so score should be < 0.7. + t.Error("expected speculative=true for no-edge path") + } +} + +// βββ Helpers ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ + +func newTestServerForInspect(t *testing.T) *Server { + t.Helper() + store := &PacketStore{ + inspectCache: make(map[string]*inspectCachedResult), + } + store.graph = NewNeighborGraph() + store.graph.builtAt = time.Now() + return &Server{store: store} +} + +func doInspectRequest(srv *Server, body string) *httptest.ResponseRecorder { + req := httptest.NewRequest("POST", "/api/paths/inspect", bytes.NewBufferString(body)) + req.Header.Set("Content-Type", "application/json") + rr := httptest.NewRecorder() + srv.handlePathInspect(rr, req) + return rr +} + +func pow(base, exp float64) float64 { + return math.Pow(base, exp) +} + +// BenchmarkBeamSearch β performance proof for spec Β§2.5 (<100ms p99 for β€64 hops). +// Anti-tautology: removing beam pruning makes this ~625x slower; timing assertion catches it. +func BenchmarkBeamSearch(b *testing.B) { + // Setup: 100 nodes, 10-hop prefix input, realistic neighbor graph. + // Anti-tautology: removing beam pruning makes this ~625x slower. + store := &PacketStore{} + pm := &prefixMap{m: make(map[string][]nodeInfo)} + graph := NewNeighborGraph() + nodes := make([]nodeInfo, 100) + + now := time.Now() + for i := 0; i < 100; i++ { + pk := fmt.Sprintf("%064x", i) + prefix := fmt.Sprintf("%02x", i%256) + node := nodeInfo{PublicKey: pk, Name: fmt.Sprintf("Node%d", i), Role: "repeater", Lat: 37.0 + float64(i)*0.01, Lon: -122.0 + float64(i)*0.01} + nodes[i] = node + pm.m[prefix] = append(pm.m[prefix], node) + // Add neighbor edges to create a connected graph. + if i > 0 { + prevPK := fmt.Sprintf("%064x", i-1) + key := makeEdgeKey(prevPK, pk) + edge := &NeighborEdge{NodeA: prevPK, NodeB: pk, LastSeen: now, Count: 10} + graph.edges[key] = edge + graph.byNode[prevPK] = append(graph.byNode[prevPK], edge) + graph.byNode[pk] = append(graph.byNode[pk], edge) + } + } + + // 10-hop input using prefixes that map to multiple candidates. + prefixes := make([]string, 10) + for i := 0; i < 10; i++ { + prefixes[i] = fmt.Sprintf("%02x", (i*3)%256) + } + + nodeByPK := make(map[string]*nodeInfo) + for idx := range nodes { + nodeByPK[nodes[idx].PublicKey] = &nodes[idx] + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + store.beamSearch(prefixes, pm, graph, nodeByPK, now) + } +} diff --git a/cmd/server/prefix_map_role_test.go b/cmd/server/prefix_map_role_test.go new file mode 100644 index 00000000..48897109 --- /dev/null +++ b/cmd/server/prefix_map_role_test.go @@ -0,0 +1,212 @@ +package main + +import ( + "encoding/json" + "testing" +) + +func TestCanAppearInPath(t *testing.T) { + cases := []struct { + role string + want bool + }{ + {"repeater", true}, + {"Repeater", true}, + {"REPEATER", true}, + {"room_server", true}, + {"Room_Server", true}, + {"room", true}, + {"companion", false}, + {"sensor", false}, + {"", false}, + {"unknown", false}, + } + for _, tc := range cases { + if got := canAppearInPath(tc.role); got != tc.want { + t.Errorf("canAppearInPath(%q) = %v, want %v", tc.role, got, tc.want) + } + } +} + +func TestBuildPrefixMap_ExcludesCompanions(t *testing.T) { + nodes := []nodeInfo{ + {PublicKey: "7a1234abcdef", Role: "companion", Name: "MyCompanion"}, + } + pm := buildPrefixMap(nodes) + if len(pm.m) != 0 { + t.Fatalf("expected empty prefix map, got %d entries", len(pm.m)) + } +} + +func TestBuildPrefixMap_ExcludesSensors(t *testing.T) { + nodes := []nodeInfo{ + {PublicKey: "7a1234abcdef", Role: "sensor", Name: "MySensor"}, + } + pm := buildPrefixMap(nodes) + if len(pm.m) != 0 { + t.Fatalf("expected empty prefix map, got %d entries", len(pm.m)) + } +} + +func TestResolveWithContext_NilWhenOnlyCompanionMatchesPrefix(t *testing.T) { + nodes := []nodeInfo{ + {PublicKey: "7a1234abcdef", Role: "companion", Name: "MyCompanion"}, + } + pm := buildPrefixMap(nodes) + r, _, _ := pm.resolveWithContext("7a", nil, nil) + if r != nil { + t.Fatalf("expected nil, got %+v", r) + } +} + +func TestResolveWithContext_NilWhenOnlySensorMatchesPrefix(t *testing.T) { + nodes := []nodeInfo{ + {PublicKey: "7a1234abcdef", Role: "sensor", Name: "MySensor"}, + } + pm := buildPrefixMap(nodes) + r, _, _ := pm.resolveWithContext("7a", nil, nil) + if r != nil { + t.Fatalf("expected nil for sensor-only prefix, got %+v", r) + } +} + +func TestResolveWithContext_PrefersRepeaterOverCompanionAtSamePrefix(t *testing.T) { + nodes := []nodeInfo{ + {PublicKey: "7a1234abcdef", Role: "companion", Name: "MyCompanion"}, + {PublicKey: "7a5678901234", Role: "repeater", Name: "MyRepeater"}, + } + pm := buildPrefixMap(nodes) + r, _, _ := pm.resolveWithContext("7a", nil, nil) + if r == nil { + t.Fatal("expected non-nil result") + } + if r.Name != "MyRepeater" { + t.Fatalf("expected MyRepeater, got %s", r.Name) + } +} + +func TestResolveWithContext_PrefersRoomServerOverCompanionAtSamePrefix(t *testing.T) { + nodes := []nodeInfo{ + {PublicKey: "ab1234abcdef", Role: "companion", Name: "MyCompanion"}, + {PublicKey: "ab5678901234", Role: "room_server", Name: "MyRoom"}, + } + pm := buildPrefixMap(nodes) + r, _, _ := pm.resolveWithContext("ab", nil, nil) + if r == nil { + t.Fatal("expected non-nil result") + } + if r.Name != "MyRoom" { + t.Fatalf("expected MyRoom, got %s", r.Name) + } +} + +func TestResolve_NilWhenOnlyCompanionMatchesPrefix(t *testing.T) { + nodes := []nodeInfo{ + {PublicKey: "7a1234abcdef", Role: "companion", Name: "MyCompanion"}, + } + pm := buildPrefixMap(nodes) + r := pm.resolve("7a") + if r != nil { + t.Fatalf("expected nil from resolve() for companion-only prefix, got %+v", r) + } +} + +func TestResolve_NilWhenOnlySensorMatchesPrefix(t *testing.T) { + nodes := []nodeInfo{ + {PublicKey: "7a1234abcdef", Role: "sensor", Name: "MySensor"}, + } + pm := buildPrefixMap(nodes) + r := pm.resolve("7a") + if r != nil { + t.Fatalf("expected nil from resolve() for sensor-only prefix, got %+v", r) + } +} + +func TestResolveWithContext_PicksRepeaterEvenWhenCompanionHasGPS(t *testing.T) { + // Adversarial: companion has GPS, repeater doesn't. Role filter should + // exclude companion entirely, so repeater wins despite lacking GPS. + nodes := []nodeInfo{ + {PublicKey: "7a1234abcdef", Role: "companion", Name: "GPSCompanion", Lat: 37.0, Lon: -122.0, HasGPS: true}, + {PublicKey: "7a5678901234", Role: "repeater", Name: "NoGPSRepeater", Lat: 0, Lon: 0, HasGPS: false}, + } + pm := buildPrefixMap(nodes) + r, _, _ := pm.resolveWithContext("7a", nil, nil) + if r == nil { + t.Fatal("expected non-nil result") + } + if r.Name != "NoGPSRepeater" { + t.Fatalf("expected NoGPSRepeater (role filter excludes companion), got %s", r.Name) + } +} + +func TestComputeDistancesForTx_CompanionNeverInResolvedChain(t *testing.T) { + // Integration test: a path with a prefix matching both a companion and a + // repeater. The resolveHop function (using buildPrefixMap) should only + // return the repeater. + nodes := []nodeInfo{ + {PublicKey: "7a1234abcdef", Role: "companion", Name: "BadCompanion", Lat: 37.0, Lon: -122.0, HasGPS: true}, + {PublicKey: "7a5678901234", Role: "repeater", Name: "GoodRepeater", Lat: 38.0, Lon: -123.0, HasGPS: true}, + {PublicKey: "bb1111111111", Role: "repeater", Name: "OtherRepeater", Lat: 39.0, Lon: -124.0, HasGPS: true}, + } + pm := buildPrefixMap(nodes) + + nodeByPk := make(map[string]*nodeInfo) + for i := range nodes { + nodeByPk[nodes[i].PublicKey] = &nodes[i] + } + repeaterSet := map[string]bool{ + "7a5678901234": true, + "bb1111111111": true, + } + + // Build a synthetic StoreTx with a path ["7a", "bb"] and a sender with GPS + senderPK := "cc0000000000" + sender := nodeInfo{PublicKey: senderPK, Role: "repeater", Name: "Sender", Lat: 36.0, Lon: -121.0, HasGPS: true} + nodeByPk[senderPK] = &sender + + pathJSON, _ := json.Marshal([]string{"7a", "bb"}) + decoded, _ := json.Marshal(map[string]interface{}{"pubKey": senderPK}) + + tx := &StoreTx{ + PathJSON: string(pathJSON), + DecodedJSON: string(decoded), + FirstSeen: "2026-04-30T12:00", + } + + resolveHop := func(hop string) *nodeInfo { + return pm.resolve(hop) + } + + hops, pathRec := computeDistancesForTx(tx, nodeByPk, repeaterSet, resolveHop) + + // Verify BadCompanion's pubkey never appears in hops + badPK := "7a1234abcdef" + for i, h := range hops { + if h.FromPk == badPK || h.ToPk == badPK { + t.Fatalf("hop[%d] contains BadCompanion pubkey: from=%s to=%s", i, h.FromPk, h.ToPk) + } + } + + // Verify BadCompanion's pubkey never appears in pathRec + if pathRec == nil { + t.Fatal("expected non-nil path record (3 GPS nodes in chain)") + } + for i, hop := range pathRec.Hops { + if hop.FromPk == badPK || hop.ToPk == badPK { + t.Fatalf("pathRec.Hops[%d] contains BadCompanion pubkey: from=%s to=%s", i, hop.FromPk, hop.ToPk) + } + } + + // Verify GoodRepeater IS in the chain (proves the prefix was resolved to the right node) + goodPK := "7a5678901234" + foundGood := false + for _, hop := range pathRec.Hops { + if hop.FromPk == goodPK || hop.ToPk == goodPK { + foundGood = true + break + } + } + if !foundGood { + t.Fatal("expected GoodRepeater (7a5678901234) in pathRec.Hops but not found") + } +} diff --git a/cmd/server/resolve_context_test.go b/cmd/server/resolve_context_test.go index 00ddefee..c1999060 100644 --- a/cmd/server/resolve_context_test.go +++ b/cmd/server/resolve_context_test.go @@ -11,7 +11,7 @@ import ( func TestResolveWithContext_UniquePrefix(t *testing.T) { pm := buildPrefixMap([]nodeInfo{ - {PublicKey: "a1b2c3d4", Name: "Node-A", HasGPS: true, Lat: 1, Lon: 2}, + {Role: "repeater", PublicKey: "a1b2c3d4", Name: "Node-A", HasGPS: true, Lat: 1, Lon: 2}, }) ni, confidence, _ := pm.resolveWithContext("a1b2c3d4", nil, nil) if ni == nil || ni.Name != "Node-A" { @@ -24,7 +24,7 @@ func TestResolveWithContext_UniquePrefix(t *testing.T) { func TestResolveWithContext_NoMatch(t *testing.T) { pm := buildPrefixMap([]nodeInfo{ - {PublicKey: "a1b2c3d4", Name: "Node-A"}, + {Role: "repeater", PublicKey: "a1b2c3d4", Name: "Node-A"}, }) ni, confidence, _ := pm.resolveWithContext("ff", nil, nil) if ni != nil { @@ -37,8 +37,8 @@ func TestResolveWithContext_NoMatch(t *testing.T) { func TestResolveWithContext_AffinityWins(t *testing.T) { pm := buildPrefixMap([]nodeInfo{ - {PublicKey: "a1aaaaaa", Name: "Node-A1"}, - {PublicKey: "a1bbbbbb", Name: "Node-A2"}, + {Role: "repeater", PublicKey: "a1aaaaaa", Name: "Node-A1"}, + {Role: "repeater", PublicKey: "a1bbbbbb", Name: "Node-A2"}, }) graph := NewNeighborGraph() @@ -60,9 +60,9 @@ func TestResolveWithContext_AffinityWins(t *testing.T) { func TestResolveWithContext_AffinityTooClose_FallsToGeo(t *testing.T) { pm := buildPrefixMap([]nodeInfo{ - {PublicKey: "a1aaaaaa", Name: "Node-A1", HasGPS: true, Lat: 10, Lon: 20}, - {PublicKey: "a1bbbbbb", Name: "Node-A2", HasGPS: true, Lat: 11, Lon: 21}, - {PublicKey: "c0c0c0c0", Name: "Ctx", HasGPS: true, Lat: 10.1, Lon: 20.1}, + {Role: "repeater", PublicKey: "a1aaaaaa", Name: "Node-A1", HasGPS: true, Lat: 10, Lon: 20}, + {Role: "repeater", PublicKey: "a1bbbbbb", Name: "Node-A2", HasGPS: true, Lat: 11, Lon: 21}, + {Role: "repeater", PublicKey: "c0c0c0c0", Name: "Ctx", HasGPS: true, Lat: 10.1, Lon: 20.1}, }) graph := NewNeighborGraph() @@ -85,8 +85,8 @@ func TestResolveWithContext_AffinityTooClose_FallsToGeo(t *testing.T) { func TestResolveWithContext_GPSPreference(t *testing.T) { pm := buildPrefixMap([]nodeInfo{ - {PublicKey: "a1aaaaaa", Name: "NoGPS"}, - {PublicKey: "a1bbbbbb", Name: "HasGPS", HasGPS: true, Lat: 1, Lon: 2}, + {Role: "repeater", PublicKey: "a1aaaaaa", Name: "NoGPS"}, + {Role: "repeater", PublicKey: "a1bbbbbb", Name: "HasGPS", HasGPS: true, Lat: 1, Lon: 2}, }) ni, confidence, _ := pm.resolveWithContext("a1", nil, nil) @@ -100,8 +100,8 @@ func TestResolveWithContext_GPSPreference(t *testing.T) { func TestResolveWithContext_FirstMatchFallback(t *testing.T) { pm := buildPrefixMap([]nodeInfo{ - {PublicKey: "a1aaaaaa", Name: "First"}, - {PublicKey: "a1bbbbbb", Name: "Second"}, + {Role: "repeater", PublicKey: "a1aaaaaa", Name: "First"}, + {Role: "repeater", PublicKey: "a1bbbbbb", Name: "Second"}, }) ni, confidence, _ := pm.resolveWithContext("a1", nil, nil) @@ -115,8 +115,8 @@ func TestResolveWithContext_FirstMatchFallback(t *testing.T) { func TestResolveWithContext_NilGraphFallsToGPS(t *testing.T) { pm := buildPrefixMap([]nodeInfo{ - {PublicKey: "a1aaaaaa", Name: "NoGPS"}, - {PublicKey: "a1bbbbbb", Name: "HasGPS", HasGPS: true, Lat: 1, Lon: 2}, + {Role: "repeater", PublicKey: "a1aaaaaa", Name: "NoGPS"}, + {Role: "repeater", PublicKey: "a1bbbbbb", Name: "HasGPS", HasGPS: true, Lat: 1, Lon: 2}, }) ni, confidence, _ := pm.resolveWithContext("a1", []string{"someone"}, nil) @@ -131,8 +131,8 @@ func TestResolveWithContext_NilGraphFallsToGPS(t *testing.T) { func TestResolveWithContext_BackwardCompatResolve(t *testing.T) { // Verify original resolve() still works unchanged pm := buildPrefixMap([]nodeInfo{ - {PublicKey: "a1aaaaaa", Name: "NoGPS"}, - {PublicKey: "a1bbbbbb", Name: "HasGPS", HasGPS: true, Lat: 1, Lon: 2}, + {Role: "repeater", PublicKey: "a1aaaaaa", Name: "NoGPS"}, + {Role: "repeater", PublicKey: "a1bbbbbb", Name: "HasGPS", HasGPS: true, Lat: 1, Lon: 2}, }) ni := pm.resolve("a1") if ni == nil || ni.Name != "HasGPS" { @@ -164,8 +164,8 @@ func TestResolveHopsAPI_UniquePrefix(t *testing.T) { _ = srv // Insert a unique node - srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon) VALUES (?, ?, ?, ?)", - "ff11223344", "UniqueNode", 37.0, -122.0) + srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon, role) VALUES (?, ?, ?, ?, ?)", + "ff11223344", "UniqueNode", 37.0, -122.0, "repeater") srv.store.InvalidateNodeCache() req := httptest.NewRequest("GET", "/api/resolve-hops?hops=ff11223344", nil) @@ -189,10 +189,10 @@ func TestResolveHopsAPI_UniquePrefix(t *testing.T) { func TestResolveHopsAPI_AmbiguousNoContext(t *testing.T) { srv, router := setupTestServer(t) - srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon) VALUES (?, ?, ?, ?)", - "ee1aaaaaaa", "Node-E1", 37.0, -122.0) - srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon) VALUES (?, ?, ?, ?)", - "ee1bbbbbbb", "Node-E2", 38.0, -121.0) + srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon, role) VALUES (?, ?, ?, ?, ?)", + "ee1aaaaaaa", "Node-E1", 37.0, -122.0, "repeater") + srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon, role) VALUES (?, ?, ?, ?, ?)", + "ee1bbbbbbb", "Node-E2", 38.0, -121.0, "repeater") srv.store.InvalidateNodeCache() req := httptest.NewRequest("GET", "/api/resolve-hops?hops=ee1", nil) @@ -224,12 +224,12 @@ func TestResolveHopsAPI_AmbiguousNoContext(t *testing.T) { func TestResolveHopsAPI_WithAffinityContext(t *testing.T) { srv, router := setupTestServer(t) - srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon) VALUES (?, ?, ?, ?)", - "dd1aaaaaaa", "Node-D1", 37.0, -122.0) - srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon) VALUES (?, ?, ?, ?)", - "dd1bbbbbbb", "Node-D2", 38.0, -121.0) - srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon) VALUES (?, ?, ?, ?)", - "c0c0c0c0c0", "Context", 37.1, -122.1) + srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon, role) VALUES (?, ?, ?, ?, ?)", + "dd1aaaaaaa", "Node-D1", 37.0, -122.0, "repeater") + srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon, role) VALUES (?, ?, ?, ?, ?)", + "dd1bbbbbbb", "Node-D2", 38.0, -121.0, "repeater") + srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon, role) VALUES (?, ?, ?, ?, ?)", + "c0c0c0c0c0", "Context", 37.1, -122.1, "repeater") // Invalidate node cache so the PM includes newly inserted nodes. srv.store.cacheMu.Lock() @@ -279,8 +279,8 @@ func TestResolveHopsAPI_WithAffinityContext(t *testing.T) { func TestResolveHopsAPI_ResponseShape(t *testing.T) { srv, router := setupTestServer(t) - srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon) VALUES (?, ?, ?, ?)", - "bb1aaaaaaa", "Node-B1", 37.0, -122.0) + srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon, role) VALUES (?, ?, ?, ?, ?)", + "bb1aaaaaaa", "Node-B1", 37.0, -122.0, "repeater") req := httptest.NewRequest("GET", "/api/resolve-hops?hops=bb1a", nil) rr := httptest.NewRecorder() diff --git a/cmd/server/routes.go b/cmd/server/routes.go index 70839b52..aa7c2689 100644 --- a/cmd/server/routes.go +++ b/cmd/server/routes.go @@ -173,6 +173,7 @@ func (s *Server) RegisterRoutes(r *mux.Router) { r.HandleFunc("/api/observers/{id}", s.handleObserverDetail).Methods("GET") r.HandleFunc("/api/observers", s.handleObservers).Methods("GET") r.HandleFunc("/api/traces/{hash}", s.handleTraces).Methods("GET") + r.HandleFunc("/api/paths/inspect", s.handlePathInspect).Methods("POST") r.HandleFunc("/api/iata-coords", s.handleIATACoords).Methods("GET") r.HandleFunc("/api/audio-lab/buckets", s.handleAudioLabBuckets).Methods("GET") diff --git a/cmd/server/store.go b/cmd/server/store.go index 496edac1..d2cdaa7d 100644 --- a/cmd/server/store.go +++ b/cmd/server/store.go @@ -209,6 +209,10 @@ type PacketStore struct { // Persisted neighbor graph for hop resolution at ingest time. graph *NeighborGraph + // Path inspector score cache (issue #944). + inspectMu sync.RWMutex + inspectCache map[string]*inspectCachedResult + // Clock skew detection engine. clockSkew *ClockSkewEngine @@ -464,10 +468,19 @@ func (s *PacketStore) Load() error { obsRawHexCol = ", o.raw_hex" } - limitClause := "" + // Build WHERE conditions: retention cutoff (mirrors Evict logic) + optional memory-cap limit. + var loadConditions []string + if s.retentionHours > 0 { + cutoff := time.Now().UTC().Add(-time.Duration(s.retentionHours*3600) * time.Second).Format(time.RFC3339) + loadConditions = append(loadConditions, fmt.Sprintf("t.first_seen >= '%s'", cutoff)) + } if maxPackets > 0 { - limitClause = fmt.Sprintf( - "\n\t\t\tWHERE t.id IN (SELECT id FROM transmissions ORDER BY first_seen DESC LIMIT %d)", maxPackets) + loadConditions = append(loadConditions, fmt.Sprintf( + "t.id IN (SELECT id FROM transmissions ORDER BY first_seen DESC LIMIT %d)", maxPackets)) + } + filterClause := "" + if len(loadConditions) > 0 { + filterClause = "\n\t\t\tWHERE " + strings.Join(loadConditions, "\n\t\t\t AND ") } if s.db.isV3 { @@ -477,7 +490,7 @@ func (s *PacketStore) Load() error { o.snr, o.rssi, o.score, o.path_json, strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch')` + obsRawHexCol + rpCol + ` FROM transmissions t LEFT JOIN observations o ON o.transmission_id = t.id - LEFT JOIN observers obs ON obs.rowid = o.observer_idx` + limitClause + ` + LEFT JOIN observers obs ON obs.rowid = o.observer_idx` + filterClause + ` ORDER BY t.first_seen ASC, o.timestamp DESC` } else { loadSQL = `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, @@ -485,7 +498,7 @@ func (s *PacketStore) Load() error { o.id, o.observer_id, o.observer_name, o.direction, o.snr, o.rssi, o.score, o.path_json, o.timestamp` + obsRawHexCol + rpCol + ` FROM transmissions t - LEFT JOIN observations o ON o.transmission_id = t.id` + limitClause + ` + LEFT JOIN observations o ON o.transmission_id = t.id` + filterClause + ` ORDER BY t.first_seen ASC, o.timestamp DESC` } @@ -4517,12 +4530,19 @@ type nodeInfo struct { Lat float64 Lon float64 HasGPS bool + LastSeen time.Time } func (s *PacketStore) getAllNodes() []nodeInfo { - rows, err := s.db.conn.Query("SELECT public_key, name, role, lat, lon FROM nodes") + // Try with last_seen first; fall back to without if column doesn't exist. + rows, err := s.db.conn.Query("SELECT public_key, name, role, lat, lon, last_seen FROM nodes") + hasLastSeen := true if err != nil { - return nil + rows, err = s.db.conn.Query("SELECT public_key, name, role, lat, lon FROM nodes") + hasLastSeen = false + if err != nil { + return nil + } } defer rows.Close() var nodes []nodeInfo @@ -4530,13 +4550,25 @@ func (s *PacketStore) getAllNodes() []nodeInfo { var pk string var name, role sql.NullString var lat, lon sql.NullFloat64 - rows.Scan(&pk, &name, &role, &lat, &lon) + var lastSeen sql.NullString + if hasLastSeen { + rows.Scan(&pk, &name, &role, &lat, &lon, &lastSeen) + } else { + rows.Scan(&pk, &name, &role, &lat, &lon) + } n := nodeInfo{PublicKey: pk, Name: nullStrVal(name), Role: nullStrVal(role)} if lat.Valid && lon.Valid { n.Lat = lat.Float64 n.Lon = lon.Float64 n.HasGPS = !(n.Lat == 0 && n.Lon == 0) } + if hasLastSeen && lastSeen.Valid && lastSeen.String != "" { + if t, err := time.Parse(time.RFC3339, lastSeen.String); err == nil { + n.LastSeen = t + } else if t, err := time.Parse("2006-01-02 15:04:05", lastSeen.String); err == nil { + n.LastSeen = t + } + } nodes = append(nodes, n) } return nodes @@ -4551,9 +4583,20 @@ type prefixMap struct { // entries to ~7ΓN (+ 1 full-key entry per node for exact-match lookups). const maxPrefixLen = 8 +// canAppearInPath returns true if the node's role allows it to appear as a +// path hop. Only repeaters, room servers, and rooms can forward packets; +// companions and sensors originate but never relay. +func canAppearInPath(role string) bool { + r := strings.ToLower(role) + return strings.Contains(r, "repeater") || strings.Contains(r, "room_server") || r == "room" +} + func buildPrefixMap(nodes []nodeInfo) *prefixMap { pm := &prefixMap{m: make(map[string][]nodeInfo, len(nodes)*(maxPrefixLen+1))} for _, n := range nodes { + if !canAppearInPath(n.Role) { + continue + } pk := strings.ToLower(n.PublicKey) maxLen := maxPrefixLen if maxLen > len(pk) { diff --git a/cmd/server/vacuum.go b/cmd/server/vacuum.go new file mode 100644 index 00000000..a53556a5 --- /dev/null +++ b/cmd/server/vacuum.go @@ -0,0 +1,84 @@ +package main + +import ( + "fmt" + "log" + "time" +) + +// checkAutoVacuum inspects the current auto_vacuum mode and logs a warning +// if it's not INCREMENTAL. Optionally performs a one-time full VACUUM if +// the operator has set db.vacuumOnStartup: true in config (#919). +func checkAutoVacuum(db *DB, cfg *Config, dbPath string) { + var autoVacuum int + if err := db.conn.QueryRow("PRAGMA auto_vacuum").Scan(&autoVacuum); err != nil { + log.Printf("[db] warning: could not read auto_vacuum: %v", err) + return + } + + if autoVacuum == 2 { + log.Printf("[db] auto_vacuum=INCREMENTAL") + return + } + + modes := map[int]string{0: "NONE", 1: "FULL", 2: "INCREMENTAL"} + mode := modes[autoVacuum] + if mode == "" { + mode = fmt.Sprintf("UNKNOWN(%d)", autoVacuum) + } + + log.Printf("[db] auto_vacuum=%s β DB needs one-time VACUUM to enable incremental auto-vacuum. "+ + "Set db.vacuumOnStartup: true in config to migrate (will block startup for several minutes on large DBs). "+ + "See https://github.com/Kpa-clawbot/CoreScope/issues/919", mode) + + if cfg.DB != nil && cfg.DB.VacuumOnStartup { + // WARNING: Full VACUUM creates a temporary copy of the entire DB file. + // Requires ~2Γ the DB file size in free disk space or it will fail. + log.Printf("[db] vacuumOnStartup=true β starting one-time full VACUUM (ensure 2x DB size free disk space)...") + start := time.Now() + + rw, err := openRW(dbPath) + if err != nil { + log.Printf("[db] VACUUM failed: could not open RW connection: %v", err) + return + } + defer rw.Close() + + if _, err := rw.Exec("PRAGMA auto_vacuum = INCREMENTAL"); err != nil { + log.Printf("[db] VACUUM failed: could not set auto_vacuum: %v", err) + return + } + if _, err := rw.Exec("VACUUM"); err != nil { + log.Printf("[db] VACUUM failed: %v", err) + return + } + + elapsed := time.Since(start) + log.Printf("[db] VACUUM complete in %v β auto_vacuum is now INCREMENTAL", elapsed.Round(time.Millisecond)) + + // Re-check + var newMode int + if err := db.conn.QueryRow("PRAGMA auto_vacuum").Scan(&newMode); err == nil { + if newMode == 2 { + log.Printf("[db] auto_vacuum=INCREMENTAL (confirmed after VACUUM)") + } else { + log.Printf("[db] warning: auto_vacuum=%d after VACUUM β expected 2", newMode) + } + } + } +} + +// runIncrementalVacuum runs PRAGMA incremental_vacuum(N) on a read-write +// connection. Safe to call on auto_vacuum=NONE databases (noop). +func runIncrementalVacuum(dbPath string, pages int) { + rw, err := openRW(dbPath) + if err != nil { + log.Printf("[vacuum] could not open RW connection: %v", err) + return + } + defer rw.Close() + + if _, err := rw.Exec(fmt.Sprintf("PRAGMA incremental_vacuum(%d)", pages)); err != nil { + log.Printf("[vacuum] incremental_vacuum error: %v", err) + } +} diff --git a/config.example.json b/config.example.json index 5672ed31..7e8e80a3 100644 --- a/config.example.json +++ b/config.example.json @@ -9,6 +9,11 @@ "packetDays": 30, "_comment": "nodeDays: nodes not seen in N days moved to inactive_nodes (default 7). observerDays: observers not sending data in N days are removed (-1 = keep forever, default 14). packetDays: transmissions older than N days are deleted (0 = disabled)." }, + "db": { + "vacuumOnStartup": false, + "incrementalVacuumPages": 1024, + "_comment": "vacuumOnStartup: run one-time full VACUUM to enable incremental auto-vacuum on existing DBs (blocks startup for minutes on large DBs; requires 2x DB file size in free disk space). incrementalVacuumPages: free pages returned to OS after each retention reaper cycle (default 1024). See #919." + }, "https": { "cert": "/path/to/cert.pem", "key": "/path/to/key.pem", @@ -208,7 +213,8 @@ "packetStore": { "maxMemoryMB": 1024, "estimatedPacketBytes": 450, - "_comment": "In-memory packet store. maxMemoryMB caps RAM usage. All packets loaded on startup, served from RAM." + "retentionHours": 168, + "_comment": "In-memory packet store. maxMemoryMB caps RAM usage. retentionHours: only packets younger than this are loaded on startup and kept in memory (0 = unlimited, not recommended for large DBs β causes OOM on cold start). 168 = 7 days. Must be β€ retention.packetDays * 24." }, "resolvedPath": { "backfillHours": 24, diff --git a/docs/user-guide/configuration.md b/docs/user-guide/configuration.md index eda7910d..7ff59d94 100644 --- a/docs/user-guide/configuration.md +++ b/docs/user-guide/configuration.md @@ -98,6 +98,22 @@ How long (in hours) before a node is marked degraded or silent: | `retention.nodeDays` | `7` | Nodes not seen in N days move to inactive | | `retention.packetDays` | `30` | Packets older than N days are deleted daily | +> **Note:** Lowering retention does **not** immediately shrink the database file. +> SQLite marks deleted pages as free but does not return them to the filesystem +> unless [incremental auto-vacuum](database.md) is enabled. New databases created +> after v0.x.x have auto-vacuum enabled automatically. Existing databases require +> a one-time migration β see the [Database](database.md) guide. + +## Database + +| Field | Default | Description | +|-------|---------|-------------| +| `db.vacuumOnStartup` | `false` | Run a one-time full `VACUUM` on startup to enable incremental auto-vacuum (blocks for minutes on large DBs) | +| `db.incrementalVacuumPages` | `1024` | Free pages returned to the OS after each retention reaper cycle | + +See [Database](database.md) for details on SQLite auto-vacuum, WAL, and manual maintenance. +See [#919](https://github.com/Kpa-clawbot/CoreScope/issues/919) for background. + ## Channel decryption | Field | Description | @@ -150,6 +166,9 @@ Lower values = fresher data but more server load. |-------|---------|-------------| | `packetStore.maxMemoryMB` | `1024` | Maximum RAM for in-memory packet store | | `packetStore.estimatedPacketBytes` | `450` | Estimated bytes per packet (for memory budgeting) | +| `packetStore.retentionHours` | `0` | Only load packets younger than N hours on startup and keep them in memory. **Set this on any instance with a large DB.** `0` = unlimited (loads full DB history β causes OOM on cold start when the DB has hundreds of thousands of paths). Recommended: same as `retention.packetDays Γ 24` (e.g. `168` for 7 days). | + +> **Warning:** Leaving `retentionHours` at `0` on a large database will cause the server to OOM-kill itself on every cold start. The full packet history is loaded into the subpath index at startup; a DB with ~280K paths produces ~13M index entries before the process is killed. ## Timestamps diff --git a/docs/user-guide/database.md b/docs/user-guide/database.md new file mode 100644 index 00000000..feaf6c1e --- /dev/null +++ b/docs/user-guide/database.md @@ -0,0 +1,82 @@ +# Database + +CoreScope uses SQLite in WAL (Write-Ahead Log) mode for both the server +(read-only) and ingestor (read-write). + +## WAL mode + +WAL mode allows concurrent reads while writes happen. It is set automatically +at connection time via `PRAGMA journal_mode=WAL`. No operator action needed. + +The WAL file (`meshcore.db-wal`) grows during writes and is checkpointed +(merged back into the main DB) periodically and at clean shutdown. + +## Auto-vacuum + +By default, SQLite does not shrink the database file after `DELETE` operations. +Deleted pages are marked free and reused by future writes, but the file size +on disk stays the same. This is surprising when lowering retention settings. + +### New databases + +Databases created after this feature was added automatically have +`PRAGMA auto_vacuum = INCREMENTAL`. After each retention reaper cycle, +CoreScope runs `PRAGMA incremental_vacuum(N)` to return free pages to the OS. + +### Existing databases + +The `auto_vacuum` mode is stored in the database header and can only be changed +by rewriting the entire file with `VACUUM`. CoreScope will **not** do this +automatically β on large databases (5+ GB seen in the wild) it takes minutes +and holds an exclusive lock. + +**To migrate an existing database:** + +1. At startup, CoreScope logs a warning: + ``` + [db] auto_vacuum=NONE β DB needs one-time VACUUM to enable incremental auto-vacuum. + ``` +2. **Ensure at least 2Γ the database file size in free disk space.** Full VACUUM + creates a temporary copy of the entire file β on a near-full disk it will fail. +3. Set `db.vacuumOnStartup: true` in your `config.json`: + ```json + { + "db": { + "vacuumOnStartup": true + } + } + ``` +4. Restart CoreScope. The one-time `VACUUM` will run and block startup. +5. After migration, remove or set `vacuumOnStartup: false` β it's not needed again. + +### Configuration + +| Field | Default | Description | +|-------|---------|-------------| +| `db.vacuumOnStartup` | `false` | One-time full VACUUM to enable incremental auto-vacuum | +| `db.incrementalVacuumPages` | `1024` | Pages returned to OS per reaper cycle | + +## Manual VACUUM + +You can also run a manual vacuum from the SQLite CLI: + +```bash +sqlite3 data/meshcore.db "PRAGMA auto_vacuum = INCREMENTAL; VACUUM;" +``` + +This is equivalent to `vacuumOnStartup: true` but can be done offline. + +> β οΈ Full VACUUM requires **2Γ the database file size** in free disk space (it +> creates a temporary copy). Check with `ls -lh data/meshcore.db` before running. + +## Checking current mode + +```bash +sqlite3 data/meshcore.db "PRAGMA auto_vacuum;" +``` + +- `0` = NONE (default for old databases) +- `1` = FULL (automatic, but slower writes) +- `2` = INCREMENTAL (recommended β CoreScope triggers vacuum after deletes) + +See [#919](https://github.com/Kpa-clawbot/CoreScope/issues/919) for background on this feature. diff --git a/public/app.js b/public/app.js index 2e21e8fe..17a98c00 100644 --- a/public/app.js +++ b/public/app.js @@ -505,6 +505,21 @@ const pages = {}; function registerPage(name, mod) { pages[name] = mod; } +// Tools landing page β shows sub-menu with Trace and Path Inspector (spec Β§2.8, M1 fix). +registerPage('tools-landing', { + init: function (container) { + container.innerHTML = + '
Hex prefixes (1-3 bytes), comma or space separated.
+Loading...
'; + fetch('/api/paths/inspect', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ prefixes: prefixes }) + }) + .then(function (r) { + if (r.status === 503) return r.json().then(function () { throw new Error('Service warming up, retry shortly.'); }); + if (!r.ok) return r.json().then(function (d) { throw new Error(d.error || 'Request failed'); }); + return r.json(); + }) + .then(function (data) { renderMapPiResults(data, resultsDiv); }) + .catch(function (e) { resultsDiv.innerHTML = ''; errDiv.textContent = e.message; }); + } + + function renderMapPiResults(data, div) { + if (!data.candidates || data.candidates.length === 0) { + div.innerHTML = 'No candidates found.
'; + return; + } + var html = '| # | Score | Path | |
|---|---|---|---|
| ' + (i + 1) + ' | '; + html += '' + c.score.toFixed(2) + (c.speculative ? ' β ' : '') + ' | '; + html += '' + safeEsc(c.names.slice(0, 3).join('β')) + (c.names.length > 3 ? 'β¦' : '') + ' | '; + html += ''; + html += ' |
';
+ if (c.evidence && c.evidence.perHop) {
+ for (var j = 0; j < c.evidence.perHop.length; j++) {
+ var h = c.evidence.perHop[j];
+ html += ' Hop ' + (j+1) + ': ' + h.prefix + ' (Γ' + h.candidatesConsidered + ') w=' + h.edgeWeight.toFixed(2);
+ if (h.alternatives && h.alternatives.length > 0) {
+ html += ' [+' + h.alternatives.length + ' alt]';
+ }
+ html += ' ';
+ }
+ }
+ html += ' | |||
Enter comma or space-separated hex prefixes (1-3 bytes each, e.g. 2C,A1,F4 or 2C A1 F4).
Loading...
'; + fetch('/api/paths/inspect', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ prefixes: prefixes }) + }) + .then(function (r) { + if (r.status === 503) return r.json().then(function (d) { throw new Error('Service warming up, retry in a few seconds.'); }); + if (!r.ok) return r.json().then(function (d) { throw new Error(d.error || 'Request failed'); }); + return r.json(); + }) + .then(function (data) { + currentResults = data; + renderResults(data, resultsDiv); + }) + .catch(function (e) { + resultsDiv.innerHTML = ''; + errDiv.textContent = e.message; + }); + } + + function renderResults(data, div) { + if (!data.candidates || data.candidates.length === 0) { + div.innerHTML = 'No candidates found. The prefixes may not match any known path-eligible nodes.
'; + return; + } + + var html = '| # | Score | Path | Action | ' + + '
|---|---|---|---|
| ' + (i + 1) + ' | '; + html += '' + + c.score.toFixed(3) + + (c.speculative ? ' β ' : '') + + ' | '; + html += '' + escapeHtml(c.names.join(' β ')) + ' | '; + html += ''; + html += ' |
';
+ for (var j = 0; j < c.evidence.perHop.length; j++) {
+ var h = c.evidence.perHop[j];
+ html += ' Hop ' + (j + 1) + ': prefix=' + h.prefix +
+ ', candidates=' + h.candidatesConsidered +
+ ', edge=' + h.edgeWeight.toFixed(3);
+ if (h.alternatives && h.alternatives.length > 0) {
+ html += ' ';
+ }
+ html += '';
+ for (var k = 0; k < h.alternatives.length; k++) {
+ var alt = h.alternatives[k];
+ html += ' ';
+ }
+ html += 'β³ ' + escapeHtml(alt.name || alt.publicKey.substring(0, 8)) + ' (score=' + alt.score.toFixed(3) + ') ';
+ }
+ html += ' | |||