diff --git a/.badges/e2e-tests.json b/.badges/e2e-tests.json index a34848cf..504f7e8e 100644 --- a/.badges/e2e-tests.json +++ b/.badges/e2e-tests.json @@ -1 +1 @@ -{"schemaVersion":1,"label":"e2e tests","message":"45 passed","color":"brightgreen"} \ No newline at end of file +{"schemaVersion":1,"label":"e2e tests","message":"83 passed","color":"brightgreen"} diff --git a/.badges/frontend-coverage.json b/.badges/frontend-coverage.json index 1d7cf1a2..10071728 100644 --- a/.badges/frontend-coverage.json +++ b/.badges/frontend-coverage.json @@ -1 +1 @@ -{"schemaVersion":1,"label":"frontend coverage","message":"39.68%","color":"red"} \ No newline at end of file +{"schemaVersion":1,"label":"frontend coverage","message":"37.74%","color":"red"} diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index cd5c4319..f612a82d 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -135,7 +135,7 @@ jobs: e2e-test: name: "🎭 Playwright E2E Tests" needs: [go-test] - runs-on: [self-hosted, Linux] + runs-on: ubuntu-latest defaults: run: shell: bash @@ -145,13 +145,6 @@ jobs: with: fetch-depth: 0 - - name: Free disk space - run: | - # Prune old runner diagnostic logs (can accumulate 50MB+) - find ~/actions-runner/_diag/ -name '*.log' -mtime +3 -delete 2>/dev/null || true - # Show available disk space - df -h / | tail -1 - - name: Set up Node.js 22 uses: actions/setup-node@v5 with: @@ -252,17 +245,11 @@ jobs: build-and-publish: name: "πŸ—οΈ Build & Publish Docker Image" needs: [e2e-test] - runs-on: [self-hosted, meshcore-runner-2] + runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v5 - - name: Free disk space - run: | - docker system prune -af 2>/dev/null || true - docker builder prune -af 2>/dev/null || true - df -h / - - name: Compute build metadata id: meta run: | @@ -462,7 +449,7 @@ jobs: name: "πŸ“ Publish Badges & Summary" if: github.event_name == 'push' needs: [deploy] - runs-on: [self-hosted, Linux] + runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v5 diff --git a/cmd/ingestor/config.go b/cmd/ingestor/config.go index 70c18fbe..910d3b95 100644 --- a/cmd/ingestor/config.go +++ b/cmd/ingestor/config.go @@ -41,6 +41,7 @@ type Config struct { Metrics *MetricsConfig `json:"metrics,omitempty"` GeoFilter *GeoFilterConfig `json:"geo_filter,omitempty"` ValidateSignatures *bool `json:"validateSignatures,omitempty"` + DB *DBConfig `json:"db,omitempty"` } // GeoFilterConfig is an alias for the shared geofilter.Config type. @@ -58,6 +59,20 @@ type MetricsConfig struct { SampleIntervalSec int `json:"sampleIntervalSec"` } +// DBConfig controls SQLite vacuum and maintenance behavior (#919). +type DBConfig struct { + VacuumOnStartup bool `json:"vacuumOnStartup"` // one-time full VACUUM on startup if auto_vacuum is not INCREMENTAL + IncrementalVacuumPages int `json:"incrementalVacuumPages"` // pages returned to OS per reaper cycle (default 1024) +} + +// IncrementalVacuumPages returns the configured pages per vacuum or 1024 default. +func (c *Config) IncrementalVacuumPages() int { + if c.DB != nil && c.DB.IncrementalVacuumPages > 0 { + return c.DB.IncrementalVacuumPages + } + return 1024 +} + // ShouldValidateSignatures returns true (default) unless explicitly disabled. func (c *Config) ShouldValidateSignatures() bool { if c.ValidateSignatures != nil { diff --git a/cmd/ingestor/db.go b/cmd/ingestor/db.go index bada26c8..93d3dc53 100644 --- a/cmd/ingestor/db.go +++ b/cmd/ingestor/db.go @@ -59,7 +59,7 @@ func OpenStoreWithInterval(dbPath string, sampleIntervalSec int) (*Store, error) return nil, fmt.Errorf("creating data dir: %w", err) } - db, err := sql.Open("sqlite", dbPath+"?_pragma=journal_mode(WAL)&_pragma=foreign_keys(ON)&_pragma=busy_timeout(5000)") + db, err := sql.Open("sqlite", dbPath+"?_pragma=auto_vacuum(INCREMENTAL)&_pragma=journal_mode(WAL)&_pragma=foreign_keys(ON)&_pragma=busy_timeout(5000)") if err != nil { return nil, fmt.Errorf("opening db: %w", err) } @@ -85,6 +85,9 @@ func OpenStoreWithInterval(dbPath string, sampleIntervalSec int) (*Store, error) } func applySchema(db *sql.DB) error { + // auto_vacuum=INCREMENTAL is set via DSN pragma (must be before journal_mode). + // Logging of current mode is handled by CheckAutoVacuum β€” no duplicate log here. + schema := ` CREATE TABLE IF NOT EXISTS nodes ( public_key TEXT PRIMARY KEY, @@ -788,6 +791,58 @@ func (s *Store) PruneOldMetrics(retentionDays int) (int64, error) { return n, nil } +// CheckAutoVacuum inspects the current auto_vacuum mode and logs a warning +// if not INCREMENTAL. Performs opt-in full VACUUM if db.vacuumOnStartup is set (#919). +func (s *Store) CheckAutoVacuum(cfg *Config) { + var autoVacuum int + if err := s.db.QueryRow("PRAGMA auto_vacuum").Scan(&autoVacuum); err != nil { + log.Printf("[db] warning: could not read auto_vacuum: %v", err) + return + } + + if autoVacuum == 2 { + log.Printf("[db] auto_vacuum=INCREMENTAL") + return + } + + modes := map[int]string{0: "NONE", 1: "FULL", 2: "INCREMENTAL"} + mode := modes[autoVacuum] + if mode == "" { + mode = fmt.Sprintf("UNKNOWN(%d)", autoVacuum) + } + + log.Printf("[db] auto_vacuum=%s β€” DB needs one-time VACUUM to enable incremental auto-vacuum. "+ + "Set db.vacuumOnStartup: true in config to migrate (will block startup for several minutes on large DBs). "+ + "See https://github.com/Kpa-clawbot/CoreScope/issues/919", mode) + + if cfg.DB != nil && cfg.DB.VacuumOnStartup { + // WARNING: Full VACUUM creates a temporary copy of the entire DB file. + // Requires ~2Γ— the DB file size in free disk space or it will fail. + log.Printf("[db] vacuumOnStartup=true β€” starting one-time full VACUUM (ensure 2x DB size free disk space)...") + start := time.Now() + + if _, err := s.db.Exec("PRAGMA auto_vacuum = INCREMENTAL"); err != nil { + log.Printf("[db] VACUUM failed: could not set auto_vacuum: %v", err) + return + } + if _, err := s.db.Exec("VACUUM"); err != nil { + log.Printf("[db] VACUUM failed: %v", err) + return + } + + elapsed := time.Since(start) + log.Printf("[db] VACUUM complete in %v β€” auto_vacuum is now INCREMENTAL", elapsed.Round(time.Millisecond)) + } +} + +// RunIncrementalVacuum returns free pages to the OS (#919). +// Safe to call on auto_vacuum=NONE databases (noop). +func (s *Store) RunIncrementalVacuum(pages int) { + if _, err := s.db.Exec(fmt.Sprintf("PRAGMA incremental_vacuum(%d)", pages)); err != nil { + log.Printf("[vacuum] incremental_vacuum error: %v", err) + } +} + // Checkpoint forces a WAL checkpoint to release the WAL lock file, // preventing lock contention with a new process starting up. func (s *Store) Checkpoint() { diff --git a/cmd/ingestor/main.go b/cmd/ingestor/main.go index 481c7cc1..b0b94bed 100644 --- a/cmd/ingestor/main.go +++ b/cmd/ingestor/main.go @@ -57,6 +57,9 @@ func main() { defer store.Close() log.Printf("SQLite opened: %s", cfg.DBPath) + // Check auto_vacuum mode and optionally migrate (#919) + store.CheckAutoVacuum(cfg) + // Node retention: move stale nodes to inactive_nodes on startup nodeDays := cfg.NodeDaysOrDefault() store.MoveStaleNodes(nodeDays) @@ -69,12 +72,15 @@ func main() { metricsDays := cfg.MetricsRetentionDays() store.PruneOldMetrics(metricsDays) store.PruneDroppedPackets(metricsDays) + vacuumPages := cfg.IncrementalVacuumPages() + store.RunIncrementalVacuum(vacuumPages) // Daily ticker for node retention retentionTicker := time.NewTicker(1 * time.Hour) go func() { for range retentionTicker.C { store.MoveStaleNodes(nodeDays) + store.RunIncrementalVacuum(vacuumPages) } }() @@ -83,8 +89,10 @@ func main() { go func() { time.Sleep(90 * time.Second) // stagger after metrics prune store.RemoveStaleObservers(observerDays) + store.RunIncrementalVacuum(vacuumPages) for range observerRetentionTicker.C { store.RemoveStaleObservers(observerDays) + store.RunIncrementalVacuum(vacuumPages) } }() @@ -94,6 +102,7 @@ func main() { for range metricsRetentionTicker.C { store.PruneOldMetrics(metricsDays) store.PruneDroppedPackets(metricsDays) + store.RunIncrementalVacuum(vacuumPages) } }() diff --git a/cmd/server/bounded_load_test.go b/cmd/server/bounded_load_test.go index d42e2a20..ad8b773e 100644 --- a/cmd/server/bounded_load_test.go +++ b/cmd/server/bounded_load_test.go @@ -127,6 +127,92 @@ func TestBoundedLoad_AscendingOrder(t *testing.T) { } } +// loadStoreWithRetention creates a PacketStore with retentionHours set. +func loadStoreWithRetention(t *testing.T, dbPath string, retentionHours float64) *PacketStore { + t.Helper() + db, err := OpenDB(dbPath) + if err != nil { + t.Fatal(err) + } + cfg := &PacketStoreConfig{RetentionHours: retentionHours} + store := NewPacketStore(db, cfg) + if err := store.Load(); err != nil { + t.Fatal(err) + } + return store +} + +// createTestDBWithAgedPackets inserts numRecent packets with timestamps within +// the last hour and numOld packets with timestamps 48 hours ago. +func createTestDBWithAgedPackets(t *testing.T, numRecent, numOld int) string { + t.Helper() + dir := t.TempDir() + dbPath := filepath.Join(dir, "test.db") + + conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL") + if err != nil { + t.Fatal(err) + } + defer conn.Close() + + execOrFail := func(s string) { + if _, err := conn.Exec(s); err != nil { + t.Fatalf("setup: %v\nSQL: %s", err, s) + } + } + execOrFail(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT)`) + execOrFail(`CREATE TABLE observations (id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT)`) + execOrFail(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT)`) + execOrFail(`CREATE TABLE nodes (pubkey TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, frequency REAL)`) + execOrFail(`CREATE TABLE schema_version (version INTEGER)`) + execOrFail(`INSERT INTO schema_version (version) VALUES (1)`) + execOrFail(`CREATE INDEX idx_tx_first_seen ON transmissions(first_seen)`) + + now := time.Now().UTC() + id := 1 + // Insert old packets (48 hours ago) + for i := 0; i < numOld; i++ { + ts := now.Add(-48 * time.Hour).Add(time.Duration(i) * time.Second).Format(time.RFC3339) + conn.Exec("INSERT INTO transmissions VALUES (?,?,?,?,0,4,1,?)", id, "aa", fmt.Sprintf("old%d", i), ts, `{}`) + conn.Exec("INSERT INTO observations VALUES (?,?,?,?,?,?,?,?,?,?,?)", id, id, "obs1", "Obs1", "RX", -10.0, -80.0, 5, `[]`, ts, "") + id++ + } + // Insert recent packets (within last hour) + for i := 0; i < numRecent; i++ { + ts := now.Add(-30 * time.Minute).Add(time.Duration(i) * time.Second).Format(time.RFC3339) + conn.Exec("INSERT INTO transmissions VALUES (?,?,?,?,0,4,1,?)", id, "bb", fmt.Sprintf("new%d", i), ts, `{}`) + conn.Exec("INSERT INTO observations VALUES (?,?,?,?,?,?,?,?,?,?,?)", id, id, "obs1", "Obs1", "RX", -10.0, -80.0, 5, `[]`, ts, "") + id++ + } + return dbPath +} + +func TestRetentionLoad_OnlyLoadsRecentPackets(t *testing.T) { + dbPath := createTestDBWithAgedPackets(t, 50, 100) + defer os.RemoveAll(filepath.Dir(dbPath)) + + // retention = 2 hours β€” should load only the 50 recent packets, not the 100 old ones + store := loadStoreWithRetention(t, dbPath, 2) + defer store.db.conn.Close() + + if len(store.packets) != 50 { + t.Errorf("expected 50 recent packets, got %d (old packets should be excluded by retentionHours)", len(store.packets)) + } +} + +func TestRetentionLoad_ZeroRetentionLoadsAll(t *testing.T) { + dbPath := createTestDBWithAgedPackets(t, 50, 100) + defer os.RemoveAll(filepath.Dir(dbPath)) + + // retention = 0 (unlimited) β€” should load all 150 packets + store := loadStoreWithRetention(t, dbPath, 0) + defer store.db.conn.Close() + + if len(store.packets) != 150 { + t.Errorf("expected all 150 packets with retentionHours=0, got %d", len(store.packets)) + } +} + func TestEstimateStoreTxBytesTypical(t *testing.T) { est := estimateStoreTxBytesTypical(10) if est < 1000 { diff --git a/cmd/server/config.go b/cmd/server/config.go index 6039d41e..f21ef207 100644 --- a/cmd/server/config.go +++ b/cmd/server/config.go @@ -62,6 +62,8 @@ type Config struct { Retention *RetentionConfig `json:"retention,omitempty"` + DB *DBConfig `json:"db,omitempty"` + PacketStore *PacketStoreConfig `json:"packetStore,omitempty"` GeoFilter *GeoFilterConfig `json:"geo_filter,omitempty"` @@ -129,6 +131,20 @@ type RetentionConfig struct { MetricsDays int `json:"metricsDays"` } +// DBConfig controls SQLite vacuum and maintenance behavior (#919). +type DBConfig struct { + VacuumOnStartup bool `json:"vacuumOnStartup"` // one-time full VACUUM on startup if auto_vacuum is not INCREMENTAL + IncrementalVacuumPages int `json:"incrementalVacuumPages"` // pages returned to OS per reaper cycle (default 1024) +} + +// IncrementalVacuumPages returns the configured pages per vacuum or 1024 default. +func (c *Config) IncrementalVacuumPages() int { + if c.DB != nil && c.DB.IncrementalVacuumPages > 0 { + return c.DB.IncrementalVacuumPages + } + return 1024 +} + // MetricsRetentionDays returns configured metrics retention or 30 days default. func (c *Config) MetricsRetentionDays() int { if c.Retention != nil && c.Retention.MetricsDays > 0 { diff --git a/cmd/server/coverage_test.go b/cmd/server/coverage_test.go index f6228c7f..2d2587d8 100644 --- a/cmd/server/coverage_test.go +++ b/cmd/server/coverage_test.go @@ -763,9 +763,9 @@ func TestGetChannelsFromStore(t *testing.T) { func TestPrefixMapResolve(t *testing.T) { nodes := []nodeInfo{ - {PublicKey: "aabbccdd11223344", Name: "NodeA", HasGPS: true, Lat: 37.5, Lon: -122.0}, - {PublicKey: "aabbccdd55667788", Name: "NodeB", HasGPS: false}, - {PublicKey: "eeff0011aabbccdd", Name: "NodeC", HasGPS: true, Lat: 38.0, Lon: -121.0}, + {Role: "repeater", PublicKey: "aabbccdd11223344", Name: "NodeA", HasGPS: true, Lat: 37.5, Lon: -122.0}, + {Role: "repeater", PublicKey: "aabbccdd55667788", Name: "NodeB", HasGPS: false}, + {Role: "repeater", PublicKey: "eeff0011aabbccdd", Name: "NodeC", HasGPS: true, Lat: 38.0, Lon: -121.0}, } pm := buildPrefixMap(nodes) @@ -805,8 +805,8 @@ func TestPrefixMapResolve(t *testing.T) { t.Run("multiple candidates no GPS", func(t *testing.T) { noGPSNodes := []nodeInfo{ - {PublicKey: "aa11bb22", Name: "X", HasGPS: false}, - {PublicKey: "aa11cc33", Name: "Y", HasGPS: false}, + {Role: "repeater", PublicKey: "aa11bb22", Name: "X", HasGPS: false}, + {Role: "repeater", PublicKey: "aa11cc33", Name: "Y", HasGPS: false}, } pm2 := buildPrefixMap(noGPSNodes) n := pm2.resolve("aa11") @@ -820,8 +820,8 @@ func TestPrefixMapResolve(t *testing.T) { func TestPrefixMapCap(t *testing.T) { // 16-char pubkey β€” longer than maxPrefixLen nodes := []nodeInfo{ - {PublicKey: "aabbccdd11223344", Name: "LongKey"}, - {PublicKey: "eeff0011", Name: "ShortKey"}, // exactly 8 chars + {Role: "repeater", PublicKey: "aabbccdd11223344", Name: "LongKey"}, + {Role: "repeater", PublicKey: "eeff0011", Name: "ShortKey"}, // exactly 8 chars } pm := buildPrefixMap(nodes) diff --git a/cmd/server/db_vacuum_test.go b/cmd/server/db_vacuum_test.go new file mode 100644 index 00000000..6dad269f --- /dev/null +++ b/cmd/server/db_vacuum_test.go @@ -0,0 +1,262 @@ +package main + +import ( + "database/sql" + "os" + "path/filepath" + "strings" + "testing" + "time" + + _ "modernc.org/sqlite" +) + +// createFreshIngestorDB creates a SQLite DB using the ingestor's applySchema logic +// (simulated here) with auto_vacuum=INCREMENTAL set before tables. +func createFreshDBWithAutoVacuum(t *testing.T, path string) *sql.DB { + t.Helper() + // auto_vacuum must be set via DSN before journal_mode creates the DB file + db, err := sql.Open("sqlite", path+"?_pragma=auto_vacuum(INCREMENTAL)&_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)") + if err != nil { + t.Fatal(err) + } + db.SetMaxOpenConns(1) + + // Create minimal schema + _, err = db.Exec(` + CREATE TABLE transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + raw_hex TEXT NOT NULL, + hash TEXT NOT NULL UNIQUE, + first_seen TEXT NOT NULL, + route_type INTEGER, + payload_type INTEGER, + payload_version INTEGER, + decoded_json TEXT, + created_at TEXT DEFAULT (datetime('now')), + channel_hash TEXT + ); + CREATE TABLE observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + transmission_id INTEGER NOT NULL REFERENCES transmissions(id), + observer_idx INTEGER, + direction TEXT, + snr REAL, + rssi REAL, + score INTEGER, + path_json TEXT, + timestamp INTEGER NOT NULL + ); + `) + if err != nil { + t.Fatal(err) + } + return db +} + +func TestNewDBHasIncrementalAutoVacuum(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "test.db") + + db := createFreshDBWithAutoVacuum(t, path) + defer db.Close() + + var autoVacuum int + if err := db.QueryRow("PRAGMA auto_vacuum").Scan(&autoVacuum); err != nil { + t.Fatal(err) + } + if autoVacuum != 2 { + t.Fatalf("expected auto_vacuum=2 (INCREMENTAL), got %d", autoVacuum) + } +} + +func TestExistingDBHasAutoVacuumNone(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "test.db") + + // Create DB WITHOUT setting auto_vacuum (simulates old DB) + db, err := sql.Open("sqlite", path+"?_pragma=journal_mode(WAL)") + if err != nil { + t.Fatal(err) + } + db.SetMaxOpenConns(1) + _, err = db.Exec("CREATE TABLE dummy (id INTEGER PRIMARY KEY)") + if err != nil { + t.Fatal(err) + } + + var autoVacuum int + if err := db.QueryRow("PRAGMA auto_vacuum").Scan(&autoVacuum); err != nil { + t.Fatal(err) + } + db.Close() + + if autoVacuum != 0 { + t.Fatalf("expected auto_vacuum=0 (NONE) for old DB, got %d", autoVacuum) + } +} + +func TestVacuumOnStartupMigratesDB(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "test.db") + + // Create DB without auto_vacuum (old DB) + db, err := sql.Open("sqlite", path+"?_pragma=journal_mode(WAL)") + if err != nil { + t.Fatal(err) + } + db.SetMaxOpenConns(1) + _, err = db.Exec("CREATE TABLE dummy (id INTEGER PRIMARY KEY)") + if err != nil { + t.Fatal(err) + } + + var before int + db.QueryRow("PRAGMA auto_vacuum").Scan(&before) + if before != 0 { + t.Fatalf("precondition: expected auto_vacuum=0, got %d", before) + } + db.Close() + + // Simulate vacuumOnStartup migration using openRW + rw, err := openRW(path) + if err != nil { + t.Fatal(err) + } + if _, err := rw.Exec("PRAGMA auto_vacuum = INCREMENTAL"); err != nil { + t.Fatal(err) + } + if _, err := rw.Exec("VACUUM"); err != nil { + t.Fatal(err) + } + rw.Close() + + // Verify migration + db2, err := sql.Open("sqlite", path+"?mode=ro") + if err != nil { + t.Fatal(err) + } + defer db2.Close() + + var after int + if err := db2.QueryRow("PRAGMA auto_vacuum").Scan(&after); err != nil { + t.Fatal(err) + } + if after != 2 { + t.Fatalf("expected auto_vacuum=2 after VACUUM migration, got %d", after) + } +} + +func TestIncrementalVacuumReducesFreelist(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "test.db") + + db := createFreshDBWithAutoVacuum(t, path) + + // Insert a bunch of data + now := time.Now().UTC().Format(time.RFC3339) + for i := 0; i < 500; i++ { + _, err := db.Exec( + "INSERT INTO transmissions (raw_hex, hash, first_seen) VALUES (?, ?, ?)", + strings.Repeat("AA", 200), // ~400 bytes each + "hash_"+string(rune('A'+i%26))+string(rune('0'+i/26)), + now, + ) + if err != nil { + t.Fatal(err) + } + } + + // Get file size before delete + db.Close() + infoBefore, _ := os.Stat(path) + sizeBefore := infoBefore.Size() + + // Reopen and delete all + db, err := sql.Open("sqlite", path+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)") + if err != nil { + t.Fatal(err) + } + db.SetMaxOpenConns(1) + defer db.Close() + + _, err = db.Exec("DELETE FROM transmissions") + if err != nil { + t.Fatal(err) + } + + // Check freelist before vacuum + var freelistBefore int64 + db.QueryRow("PRAGMA freelist_count").Scan(&freelistBefore) + if freelistBefore == 0 { + t.Fatal("expected non-zero freelist after DELETE") + } + + // Run incremental vacuum + _, err = db.Exec("PRAGMA incremental_vacuum(10000)") + if err != nil { + t.Fatal(err) + } + + // Check freelist after vacuum + var freelistAfter int64 + db.QueryRow("PRAGMA freelist_count").Scan(&freelistAfter) + if freelistAfter >= freelistBefore { + t.Fatalf("expected freelist to shrink: before=%d after=%d", freelistBefore, freelistAfter) + } + + // Checkpoint WAL and check file size shrunk + db.Exec("PRAGMA wal_checkpoint(TRUNCATE)") + db.Close() + infoAfter, _ := os.Stat(path) + sizeAfter := infoAfter.Size() + if sizeAfter >= sizeBefore { + t.Logf("warning: file did not shrink (before=%d after=%d) β€” may depend on page reuse", sizeBefore, sizeAfter) + } +} + +func TestCheckAutoVacuumLogs(t *testing.T) { + // This test verifies checkAutoVacuum doesn't panic on various configs + dir := t.TempDir() + path := filepath.Join(dir, "test.db") + + // Create a fresh DB with auto_vacuum=INCREMENTAL + dbConn := createFreshDBWithAutoVacuum(t, path) + db := &DB{conn: dbConn, path: path} + cfg := &Config{} + + // Should not panic + checkAutoVacuum(db, cfg, path) + dbConn.Close() + + // Create a DB without auto_vacuum + path2 := filepath.Join(dir, "test2.db") + dbConn2, _ := sql.Open("sqlite", path2+"?_pragma=journal_mode(WAL)") + dbConn2.SetMaxOpenConns(1) + dbConn2.Exec("CREATE TABLE dummy (id INTEGER PRIMARY KEY)") + db2 := &DB{conn: dbConn2, path: path2} + + // Should log warning but not panic + checkAutoVacuum(db2, cfg, path2) + dbConn2.Close() +} + +func TestConfigIncrementalVacuumPages(t *testing.T) { + // Default + cfg := &Config{} + if cfg.IncrementalVacuumPages() != 1024 { + t.Fatalf("expected default 1024, got %d", cfg.IncrementalVacuumPages()) + } + + // Custom + cfg.DB = &DBConfig{IncrementalVacuumPages: 512} + if cfg.IncrementalVacuumPages() != 512 { + t.Fatalf("expected 512, got %d", cfg.IncrementalVacuumPages()) + } + + // Zero should return default + cfg.DB.IncrementalVacuumPages = 0 + if cfg.IncrementalVacuumPages() != 1024 { + t.Fatalf("expected default 1024 for zero, got %d", cfg.IncrementalVacuumPages()) + } +} diff --git a/cmd/server/main.go b/cmd/server/main.go index 22dc600e..31fbcd4f 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -148,6 +148,9 @@ func main() { stats.TotalTransmissions, stats.TotalObservations, stats.TotalNodes, stats.TotalObservers) } + // Check auto_vacuum mode and optionally migrate (#919) + checkAutoVacuum(database, cfg, resolvedDB) + // In-memory packet store store := NewPacketStore(database, cfg.PacketStore, cfg.CacheTTL) if err := store.Load(); err != nil { @@ -266,6 +269,7 @@ func main() { defer stopEviction() // Auto-prune old packets if retention.packetDays is configured + vacuumPages := cfg.IncrementalVacuumPages() var stopPrune func() if cfg.Retention != nil && cfg.Retention.PacketDays > 0 { days := cfg.Retention.PacketDays @@ -286,6 +290,9 @@ func main() { log.Printf("[prune] error: %v", err) } else { log.Printf("[prune] deleted %d transmissions older than %d days", n, days) + if n > 0 { + runIncrementalVacuum(resolvedDB, vacuumPages) + } } for { select { @@ -294,6 +301,9 @@ func main() { log.Printf("[prune] error: %v", err) } else { log.Printf("[prune] deleted %d transmissions older than %d days", n, days) + if n > 0 { + runIncrementalVacuum(resolvedDB, vacuumPages) + } } case <-pruneDone: return @@ -321,10 +331,12 @@ func main() { }() time.Sleep(2 * time.Minute) // stagger after packet prune database.PruneOldMetrics(metricsDays) + runIncrementalVacuum(resolvedDB, vacuumPages) for { select { case <-metricsPruneTicker.C: database.PruneOldMetrics(metricsDays) + runIncrementalVacuum(resolvedDB, vacuumPages) case <-metricsPruneDone: return } @@ -354,10 +366,12 @@ func main() { }() time.Sleep(3 * time.Minute) // stagger after metrics prune database.RemoveStaleObservers(observerDays) + runIncrementalVacuum(resolvedDB, vacuumPages) for { select { case <-observerPruneTicker.C: database.RemoveStaleObservers(observerDays) + runIncrementalVacuum(resolvedDB, vacuumPages) case <-observerPruneDone: return } @@ -388,6 +402,7 @@ func main() { g := store.graph store.mu.RUnlock() PruneNeighborEdges(dbPath, g, maxAgeDays) + runIncrementalVacuum(resolvedDB, vacuumPages) for { select { case <-edgePruneTicker.C: @@ -395,6 +410,7 @@ func main() { g := store.graph store.mu.RUnlock() PruneNeighborEdges(dbPath, g, maxAgeDays) + runIncrementalVacuum(resolvedDB, vacuumPages) case <-edgePruneDone: return } diff --git a/cmd/server/neighbor_dedup_test.go b/cmd/server/neighbor_dedup_test.go index 20504a77..abec93cc 100644 --- a/cmd/server/neighbor_dedup_test.go +++ b/cmd/server/neighbor_dedup_test.go @@ -12,9 +12,9 @@ import ( func TestResolveAmbiguousEdges_GeoProximity(t *testing.T) { // Node A at lat=45, lon=-122. Candidate B1 at lat=45.1, lon=-122.1 (close). // Candidate B2 at lat=10, lon=10 (far away). Prefix "b0" matches both. - nodeA := nodeInfo{PublicKey: "aaaa1111", Name: "NodeA", HasGPS: true, Lat: 45.0, Lon: -122.0} - nodeB1 := nodeInfo{PublicKey: "b0b1eeee", Name: "CloseNode", HasGPS: true, Lat: 45.1, Lon: -122.1} - nodeB2 := nodeInfo{PublicKey: "b0c2ffff", Name: "FarNode", HasGPS: true, Lat: 10.0, Lon: 10.0} + nodeA := nodeInfo{Role: "repeater", PublicKey: "aaaa1111", Name: "NodeA", HasGPS: true, Lat: 45.0, Lon: -122.0} + nodeB1 := nodeInfo{Role: "repeater", PublicKey: "b0b1eeee", Name: "CloseNode", HasGPS: true, Lat: 45.1, Lon: -122.1} + nodeB2 := nodeInfo{Role: "repeater", PublicKey: "b0c2ffff", Name: "FarNode", HasGPS: true, Lat: 10.0, Lon: 10.0} pm := buildPrefixMap([]nodeInfo{nodeA, nodeB1, nodeB2}) @@ -62,8 +62,8 @@ func TestResolveAmbiguousEdges_GeoProximity(t *testing.T) { // Test 2: Ambiguous edge merged with existing resolved edge (count accumulation). func TestResolveAmbiguousEdges_MergeWithExisting(t *testing.T) { - nodeA := nodeInfo{PublicKey: "aaaa1111", Name: "NodeA", HasGPS: true, Lat: 45.0, Lon: -122.0} - nodeB := nodeInfo{PublicKey: "b0b1eeee", Name: "NodeB", HasGPS: true, Lat: 45.1, Lon: -122.1} + nodeA := nodeInfo{Role: "repeater", PublicKey: "aaaa1111", Name: "NodeA", HasGPS: true, Lat: 45.0, Lon: -122.0} + nodeB := nodeInfo{Role: "repeater", PublicKey: "b0b1eeee", Name: "NodeB", HasGPS: true, Lat: 45.1, Lon: -122.1} pm := buildPrefixMap([]nodeInfo{nodeA, nodeB}) @@ -133,9 +133,9 @@ func TestResolveAmbiguousEdges_MergeWithExisting(t *testing.T) { // Test 3: Ambiguous edge left as-is when resolution fails. func TestResolveAmbiguousEdges_FailsNoChange(t *testing.T) { // Two candidates, neither has GPS, no affinity data β€” resolution falls through. - nodeA := nodeInfo{PublicKey: "aaaa1111", Name: "NodeA"} - nodeB1 := nodeInfo{PublicKey: "b0b1eeee", Name: "B1"} - nodeB2 := nodeInfo{PublicKey: "b0c2ffff", Name: "B2"} + nodeA := nodeInfo{Role: "repeater", PublicKey: "aaaa1111", Name: "NodeA"} + nodeB1 := nodeInfo{Role: "repeater", PublicKey: "b0b1eeee", Name: "B1"} + nodeB2 := nodeInfo{Role: "repeater", PublicKey: "b0c2ffff", Name: "B2"} pm := buildPrefixMap([]nodeInfo{nodeA, nodeB1, nodeB2}) @@ -175,7 +175,7 @@ func TestResolveAmbiguousEdges_FailsNoChange(t *testing.T) { // Test 3 (corrected): Resolution fails when prefix has no candidates in prefix map. func TestResolveAmbiguousEdges_NoMatch(t *testing.T) { - nodeA := nodeInfo{PublicKey: "aaaa1111", Name: "NodeA"} + nodeA := nodeInfo{Role: "repeater", PublicKey: "aaaa1111", Name: "NodeA"} // pm has no entries matching prefix "zz" pm := buildPrefixMap([]nodeInfo{nodeA}) @@ -215,8 +215,8 @@ func TestResolveAmbiguousEdges_NoMatch(t *testing.T) { // Test 6: Phase 1 edge collection unchanged (no regression). func TestPhase1EdgeCollection_Unchanged(t *testing.T) { // Build a simple graph and verify non-ambiguous edges are not touched. - nodeA := nodeInfo{PublicKey: "aaaa1111", Name: "NodeA", HasGPS: true, Lat: 45.0, Lon: -122.0} - nodeB := nodeInfo{PublicKey: "bbbb2222", Name: "NodeB", HasGPS: true, Lat: 45.1, Lon: -122.1} + nodeA := nodeInfo{Role: "repeater", PublicKey: "aaaa1111", Name: "NodeA", HasGPS: true, Lat: 45.0, Lon: -122.0} + nodeB := nodeInfo{Role: "repeater", PublicKey: "bbbb2222", Name: "NodeB", HasGPS: true, Lat: 45.1, Lon: -122.1} ts := time.Now().UTC().Format(time.RFC3339) payloadType := 4 @@ -232,7 +232,7 @@ func TestPhase1EdgeCollection_Unchanged(t *testing.T) { Observations: obs, } - store := ngTestStore([]nodeInfo{nodeA, nodeB, {PublicKey: "cccc3333", Name: "Observer"}}, []*StoreTx{tx}) + store := ngTestStore([]nodeInfo{nodeA, nodeB, {Role: "repeater", PublicKey: "cccc3333", Name: "Observer"}}, []*StoreTx{tx}) graph := BuildFromStore(store) edges := graph.Neighbors("aaaa1111") @@ -255,8 +255,8 @@ func TestPhase1EdgeCollection_Unchanged(t *testing.T) { // Test 7: Merge preserves higher LastSeen timestamp. func TestResolveAmbiguousEdges_PreservesHigherLastSeen(t *testing.T) { - nodeA := nodeInfo{PublicKey: "aaaa1111", Name: "NodeA", HasGPS: true, Lat: 45.0, Lon: -122.0} - nodeB := nodeInfo{PublicKey: "b0b1eeee", Name: "NodeB", HasGPS: true, Lat: 45.1, Lon: -122.1} + nodeA := nodeInfo{Role: "repeater", PublicKey: "aaaa1111", Name: "NodeA", HasGPS: true, Lat: 45.0, Lon: -122.0} + nodeB := nodeInfo{Role: "repeater", PublicKey: "b0b1eeee", Name: "NodeB", HasGPS: true, Lat: 45.1, Lon: -122.1} pm := buildPrefixMap([]nodeInfo{nodeA, nodeB}) graph := NewNeighborGraph() @@ -307,10 +307,10 @@ func TestResolveAmbiguousEdges_PreservesHigherLastSeen(t *testing.T) { // Test 5: Integration β€” node with both 1-byte and 2-byte prefix observations shows single entry. func TestIntegration_DualPrefixSingleNeighbor(t *testing.T) { - nodeA := nodeInfo{PublicKey: "aaaa1111aaaa1111", Name: "NodeA", HasGPS: true, Lat: 45.0, Lon: -122.0} - nodeB := nodeInfo{PublicKey: "b0b1eeeeb0b1eeee", Name: "NodeB", HasGPS: true, Lat: 45.1, Lon: -122.1} - nodeB2 := nodeInfo{PublicKey: "b0c2ffffb0c2ffff", Name: "NodeB2", HasGPS: true, Lat: 10.0, Lon: 10.0} - observer := nodeInfo{PublicKey: "cccc3333cccc3333", Name: "Observer"} + nodeA := nodeInfo{Role: "repeater", PublicKey: "aaaa1111aaaa1111", Name: "NodeA", HasGPS: true, Lat: 45.0, Lon: -122.0} + nodeB := nodeInfo{Role: "repeater", PublicKey: "b0b1eeeeb0b1eeee", Name: "NodeB", HasGPS: true, Lat: 45.1, Lon: -122.1} + nodeB2 := nodeInfo{Role: "repeater", PublicKey: "b0c2ffffb0c2ffff", Name: "NodeB2", HasGPS: true, Lat: 10.0, Lon: 10.0} + observer := nodeInfo{Role: "repeater", PublicKey: "cccc3333cccc3333", Name: "Observer"} ts := time.Now().UTC().Format(time.RFC3339) pt := 4 diff --git a/cmd/server/neighbor_graph_test.go b/cmd/server/neighbor_graph_test.go index 9500a134..7f7e2ac0 100644 --- a/cmd/server/neighbor_graph_test.go +++ b/cmd/server/neighbor_graph_test.go @@ -86,9 +86,9 @@ func TestBuildNeighborGraph_EmptyStore(t *testing.T) { func TestBuildNeighborGraph_AdvertSingleHopPath(t *testing.T) { // ADVERT from X, path=["R1_prefix"] β†’ edges: X↔R1 and Observer↔R1 nodes := []nodeInfo{ - {PublicKey: "aaaa1111", Name: "NodeX"}, - {PublicKey: "r1aabbcc", Name: "R1"}, - {PublicKey: "obs00001", Name: "Observer"}, + {Role: "repeater", PublicKey: "aaaa1111", Name: "NodeX"}, + {Role: "repeater", PublicKey: "r1aabbcc", Name: "R1"}, + {Role: "repeater", PublicKey: "obs00001", Name: "Observer"}, } tx := ngMakeTx(1, 4, ngFromNodeJSON("aaaa1111"), []*StoreObs{ ngMakeObs("obs00001", `["r1aa"]`, nowStr, ngFloatPtr(-10)), @@ -132,10 +132,10 @@ func TestBuildNeighborGraph_AdvertSingleHopPath(t *testing.T) { func TestBuildNeighborGraph_AdvertMultiHopPath(t *testing.T) { // ADVERT from X, path=["R1","R2"] β†’ X↔R1 and Observer↔R2 nodes := []nodeInfo{ - {PublicKey: "aaaa1111", Name: "NodeX"}, - {PublicKey: "r1aabbcc", Name: "R1"}, - {PublicKey: "r2ddeeff", Name: "R2"}, - {PublicKey: "obs00001", Name: "Observer"}, + {Role: "repeater", PublicKey: "aaaa1111", Name: "NodeX"}, + {Role: "repeater", PublicKey: "r1aabbcc", Name: "R1"}, + {Role: "repeater", PublicKey: "r2ddeeff", Name: "R2"}, + {Role: "repeater", PublicKey: "obs00001", Name: "Observer"}, } tx := ngMakeTx(1, 4, ngFromNodeJSON("aaaa1111"), []*StoreObs{ ngMakeObs("obs00001", `["r1aa","r2dd"]`, nowStr, nil), @@ -170,8 +170,8 @@ func TestBuildNeighborGraph_AdvertMultiHopPath(t *testing.T) { func TestBuildNeighborGraph_AdvertZeroHop(t *testing.T) { // ADVERT from X, path=[] β†’ X↔Observer direct edge nodes := []nodeInfo{ - {PublicKey: "aaaa1111", Name: "NodeX"}, - {PublicKey: "obs00001", Name: "Observer"}, + {Role: "repeater", PublicKey: "aaaa1111", Name: "NodeX"}, + {Role: "repeater", PublicKey: "obs00001", Name: "Observer"}, } tx := ngMakeTx(1, 4, ngFromNodeJSON("aaaa1111"), []*StoreObs{ ngMakeObs("obs00001", `[]`, nowStr, nil), @@ -195,8 +195,8 @@ func TestBuildNeighborGraph_AdvertZeroHop(t *testing.T) { func TestBuildNeighborGraph_NonAdvertEmptyPath(t *testing.T) { // Non-ADVERT, path=[] β†’ no edges nodes := []nodeInfo{ - {PublicKey: "aaaa1111", Name: "NodeX"}, - {PublicKey: "obs00001", Name: "Observer"}, + {Role: "repeater", PublicKey: "aaaa1111", Name: "NodeX"}, + {Role: "repeater", PublicKey: "obs00001", Name: "Observer"}, } tx := ngMakeTx(1, 2, ngFromNodeJSON("aaaa1111"), []*StoreObs{ ngMakeObs("obs00001", `[]`, nowStr, nil), @@ -212,10 +212,10 @@ func TestBuildNeighborGraph_NonAdvertEmptyPath(t *testing.T) { func TestBuildNeighborGraph_NonAdvertOnlyObserverEdge(t *testing.T) { // Non-ADVERT with path=["R1","R2"] β†’ only Observer↔R2, NO originator edge nodes := []nodeInfo{ - {PublicKey: "aaaa1111", Name: "NodeX"}, - {PublicKey: "r1aabbcc", Name: "R1"}, - {PublicKey: "r2ddeeff", Name: "R2"}, - {PublicKey: "obs00001", Name: "Observer"}, + {Role: "repeater", PublicKey: "aaaa1111", Name: "NodeX"}, + {Role: "repeater", PublicKey: "r1aabbcc", Name: "R1"}, + {Role: "repeater", PublicKey: "r2ddeeff", Name: "R2"}, + {Role: "repeater", PublicKey: "obs00001", Name: "Observer"}, } tx := ngMakeTx(1, 2, ngFromNodeJSON("aaaa1111"), []*StoreObs{ ngMakeObs("obs00001", `["r1aa","r2dd"]`, nowStr, nil), @@ -236,9 +236,9 @@ func TestBuildNeighborGraph_NonAdvertOnlyObserverEdge(t *testing.T) { func TestBuildNeighborGraph_NonAdvertSingleHop(t *testing.T) { // Non-ADVERT with path=["R1"] β†’ Observer↔R1 only nodes := []nodeInfo{ - {PublicKey: "aaaa1111", Name: "NodeX"}, - {PublicKey: "r1aabbcc", Name: "R1"}, - {PublicKey: "obs00001", Name: "Observer"}, + {Role: "repeater", PublicKey: "aaaa1111", Name: "NodeX"}, + {Role: "repeater", PublicKey: "r1aabbcc", Name: "R1"}, + {Role: "repeater", PublicKey: "obs00001", Name: "Observer"}, } tx := ngMakeTx(1, 2, ngFromNodeJSON("aaaa1111"), []*StoreObs{ ngMakeObs("obs00001", `["r1aa"]`, nowStr, nil), @@ -259,10 +259,10 @@ func TestBuildNeighborGraph_NonAdvertSingleHop(t *testing.T) { func TestBuildNeighborGraph_HashCollision(t *testing.T) { // Two nodes share prefix "a3" β†’ ambiguous edge nodes := []nodeInfo{ - {PublicKey: "aaaa1111", Name: "NodeX"}, - {PublicKey: "a3bb1111", Name: "CandidateA"}, - {PublicKey: "a3bb2222", Name: "CandidateB"}, - {PublicKey: "obs00001", Name: "Observer"}, + {Role: "repeater", PublicKey: "aaaa1111", Name: "NodeX"}, + {Role: "repeater", PublicKey: "a3bb1111", Name: "CandidateA"}, + {Role: "repeater", PublicKey: "a3bb2222", Name: "CandidateB"}, + {Role: "repeater", PublicKey: "obs00001", Name: "Observer"}, } tx := ngMakeTx(1, 4, ngFromNodeJSON("aaaa1111"), []*StoreObs{ ngMakeObs("obs00001", `["a3bb"]`, nowStr, nil), @@ -308,13 +308,13 @@ func TestBuildNeighborGraph_ConfidenceAutoResolve(t *testing.T) { // CandidateB has no known neighbors (Jaccard = 0). // An ambiguous edge X↔prefix "a3" with candidates [A, B] should auto-resolve to A. nodes := []nodeInfo{ - {PublicKey: "aaaa1111", Name: "NodeX"}, - {PublicKey: "n1111111", Name: "N1"}, - {PublicKey: "n2222222", Name: "N2"}, - {PublicKey: "n3333333", Name: "N3"}, - {PublicKey: "a3001111", Name: "CandidateA"}, - {PublicKey: "a3002222", Name: "CandidateB"}, - {PublicKey: "obs00001", Name: "Observer"}, + {Role: "repeater", PublicKey: "aaaa1111", Name: "NodeX"}, + {Role: "repeater", PublicKey: "n1111111", Name: "N1"}, + {Role: "repeater", PublicKey: "n2222222", Name: "N2"}, + {Role: "repeater", PublicKey: "n3333333", Name: "N3"}, + {Role: "repeater", PublicKey: "a3001111", Name: "CandidateA"}, + {Role: "repeater", PublicKey: "a3002222", Name: "CandidateB"}, + {Role: "repeater", PublicKey: "obs00001", Name: "Observer"}, } // Create resolved edges: X↔N1, X↔N2, X↔N3, A↔N1, A↔N2, A↔N3 @@ -373,11 +373,11 @@ func TestBuildNeighborGraph_ConfidenceAutoResolve(t *testing.T) { func TestBuildNeighborGraph_EqualScoresAmbiguous(t *testing.T) { // Two candidates with identical neighbor sets β†’ should NOT auto-resolve. nodes := []nodeInfo{ - {PublicKey: "aaaa1111", Name: "NodeX"}, - {PublicKey: "n1111111", Name: "N1"}, - {PublicKey: "a3001111", Name: "CandidateA"}, - {PublicKey: "a3002222", Name: "CandidateB"}, - {PublicKey: "obs00001", Name: "Observer"}, + {Role: "repeater", PublicKey: "aaaa1111", Name: "NodeX"}, + {Role: "repeater", PublicKey: "n1111111", Name: "N1"}, + {Role: "repeater", PublicKey: "a3001111", Name: "CandidateA"}, + {Role: "repeater", PublicKey: "a3002222", Name: "CandidateB"}, + {Role: "repeater", PublicKey: "obs00001", Name: "Observer"}, } var txs []*StoreTx @@ -425,8 +425,8 @@ func TestBuildNeighborGraph_EqualScoresAmbiguous(t *testing.T) { func TestBuildNeighborGraph_ObserverSelfEdgeGuard(t *testing.T) { // Observer's own prefix in path β†’ should NOT create self-edge. nodes := []nodeInfo{ - {PublicKey: "aaaa1111", Name: "NodeX"}, - {PublicKey: "obs00001", Name: "Observer"}, + {Role: "repeater", PublicKey: "aaaa1111", Name: "NodeX"}, + {Role: "repeater", PublicKey: "obs00001", Name: "Observer"}, } tx := ngMakeTx(1, 4, ngFromNodeJSON("aaaa1111"), []*StoreObs{ ngMakeObs("obs00001", `["obs0"]`, nowStr, nil), @@ -445,8 +445,8 @@ func TestBuildNeighborGraph_ObserverSelfEdgeGuard(t *testing.T) { func TestBuildNeighborGraph_OrphanPrefix(t *testing.T) { // Path contains prefix matching zero nodes β†’ edge recorded as unresolved. nodes := []nodeInfo{ - {PublicKey: "aaaa1111", Name: "NodeX"}, - {PublicKey: "obs00001", Name: "Observer"}, + {Role: "repeater", PublicKey: "aaaa1111", Name: "NodeX"}, + {Role: "repeater", PublicKey: "obs00001", Name: "Observer"}, } tx := ngMakeTx(1, 4, ngFromNodeJSON("aaaa1111"), []*StoreObs{ ngMakeObs("obs00001", `["ff99"]`, nowStr, nil), @@ -506,9 +506,9 @@ func TestAffinityScore_StaleAndLow(t *testing.T) { func TestBuildNeighborGraph_CountAccumulation(t *testing.T) { nodes := []nodeInfo{ - {PublicKey: "aaaa1111", Name: "NodeX"}, - {PublicKey: "r1aabbcc", Name: "R1"}, - {PublicKey: "obs00001", Name: "Observer"}, + {Role: "repeater", PublicKey: "aaaa1111", Name: "NodeX"}, + {Role: "repeater", PublicKey: "r1aabbcc", Name: "R1"}, + {Role: "repeater", PublicKey: "obs00001", Name: "Observer"}, } var txs []*StoreTx @@ -535,10 +535,10 @@ func TestBuildNeighborGraph_CountAccumulation(t *testing.T) { func TestBuildNeighborGraph_MultipleObservers(t *testing.T) { nodes := []nodeInfo{ - {PublicKey: "aaaa1111", Name: "NodeX"}, - {PublicKey: "r1aabbcc", Name: "R1"}, - {PublicKey: "obs00001", Name: "Obs1"}, - {PublicKey: "obs00002", Name: "Obs2"}, + {Role: "repeater", PublicKey: "aaaa1111", Name: "NodeX"}, + {Role: "repeater", PublicKey: "r1aabbcc", Name: "R1"}, + {Role: "repeater", PublicKey: "obs00001", Name: "Obs1"}, + {Role: "repeater", PublicKey: "obs00002", Name: "Obs2"}, } tx := ngMakeTx(1, 4, ngFromNodeJSON("aaaa1111"), []*StoreObs{ @@ -565,9 +565,9 @@ func TestBuildNeighborGraph_MultipleObservers(t *testing.T) { func TestBuildNeighborGraph_TimeDecayOldObservations(t *testing.T) { nodes := []nodeInfo{ - {PublicKey: "aaaa1111", Name: "NodeX"}, - {PublicKey: "r1aabbcc", Name: "R1"}, - {PublicKey: "obs00001", Name: "Observer"}, + {Role: "repeater", PublicKey: "aaaa1111", Name: "NodeX"}, + {Role: "repeater", PublicKey: "r1aabbcc", Name: "R1"}, + {Role: "repeater", PublicKey: "obs00001", Name: "Observer"}, } tx := ngMakeTx(1, 4, ngFromNodeJSON("aaaa1111"), []*StoreObs{ @@ -592,10 +592,10 @@ func TestBuildNeighborGraph_TimeDecayOldObservations(t *testing.T) { func TestBuildNeighborGraph_ADVERTOnlyConstraint(t *testing.T) { // Non-ADVERT: should NOT create originator↔path[0] edge, only observer↔path[last]. nodes := []nodeInfo{ - {PublicKey: "aaaa1111", Name: "NodeX"}, - {PublicKey: "r1aabbcc", Name: "R1"}, - {PublicKey: "r2ddeeff", Name: "R2"}, - {PublicKey: "obs00001", Name: "Observer"}, + {Role: "repeater", PublicKey: "aaaa1111", Name: "NodeX"}, + {Role: "repeater", PublicKey: "r1aabbcc", Name: "R1"}, + {Role: "repeater", PublicKey: "r2ddeeff", Name: "R2"}, + {Role: "repeater", PublicKey: "obs00001", Name: "Observer"}, } tx := ngMakeTx(1, 2, ngFromNodeJSON("aaaa1111"), []*StoreObs{ ngMakeObs("obs00001", `["r1aa","r2dd"]`, nowStr, nil), @@ -631,9 +631,9 @@ func ngPubKeyJSON(pubkey string) string { func TestBuildNeighborGraph_AdvertPubKeyField(t *testing.T) { // Real ADVERTs use "pubKey", not "from_node". Verify the builder handles it. nodes := []nodeInfo{ - {PublicKey: "99bf37abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234", Name: "Originator"}, - {PublicKey: "r1aabbccdd001122334455667788990011223344556677889900112233445566", Name: "R1"}, - {PublicKey: "obs0000100112233445566778899001122334455667788990011223344556677", Name: "Observer"}, + {Role: "repeater", PublicKey: "99bf37abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234", Name: "Originator"}, + {Role: "repeater", PublicKey: "r1aabbccdd001122334455667788990011223344556677889900112233445566", Name: "R1"}, + {Role: "repeater", PublicKey: "obs0000100112233445566778899001122334455667788990011223344556677", Name: "Observer"}, } tx := ngMakeTx(1, 4, ngPubKeyJSON("99bf37abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234"), []*StoreObs{ ngMakeObs("obs0000100112233445566778899001122334455667788990011223344556677", `["r1"]`, nowStr, ngFloatPtr(-8.5)), @@ -666,10 +666,10 @@ func TestBuildNeighborGraph_OneByteHashPrefixes(t *testing.T) { // Real-world scenario: 1-byte hash prefixes with multiple candidates. // Should create edges (possibly ambiguous) rather than empty graph. nodes := []nodeInfo{ - {PublicKey: "c0dedad400000000000000000000000000000000000000000000000000000001", Name: "NodeC0-1"}, - {PublicKey: "c0dedad900000000000000000000000000000000000000000000000000000002", Name: "NodeC0-2"}, - {PublicKey: "a3bbccdd00000000000000000000000000000000000000000000000000000003", Name: "Originator"}, - {PublicKey: "obs1234500000000000000000000000000000000000000000000000000000004", Name: "Observer"}, + {Role: "repeater", PublicKey: "c0dedad400000000000000000000000000000000000000000000000000000001", Name: "NodeC0-1"}, + {Role: "repeater", PublicKey: "c0dedad900000000000000000000000000000000000000000000000000000002", Name: "NodeC0-2"}, + {Role: "repeater", PublicKey: "a3bbccdd00000000000000000000000000000000000000000000000000000003", Name: "Originator"}, + {Role: "repeater", PublicKey: "obs1234500000000000000000000000000000000000000000000000000000004", Name: "Observer"}, } // ADVERT from Originator with 1-byte path hop "c0" tx := ngMakeTx(1, 4, ngPubKeyJSON("a3bbccdd00000000000000000000000000000000000000000000000000000003"), []*StoreObs{ @@ -809,10 +809,10 @@ func TestExtractFromNode_UsesCachedParse(t *testing.T) { func BenchmarkBuildFromStore(b *testing.B) { // Simulate a dataset with many packets and repeated pubkeys nodes := []nodeInfo{ - {PublicKey: "aaaa1111", Name: "NodeA"}, - {PublicKey: "bbbb2222", Name: "NodeB"}, - {PublicKey: "cccc3333", Name: "NodeC"}, - {PublicKey: "dddd4444", Name: "NodeD"}, + {Role: "repeater", PublicKey: "aaaa1111", Name: "NodeA"}, + {Role: "repeater", PublicKey: "bbbb2222", Name: "NodeB"}, + {Role: "repeater", PublicKey: "cccc3333", Name: "NodeC"}, + {Role: "repeater", PublicKey: "dddd4444", Name: "NodeD"}, } const numPackets = 1000 packets := make([]*StoreTx, 0, numPackets) diff --git a/cmd/server/neighbor_persist_test.go b/cmd/server/neighbor_persist_test.go index 6e046241..33d29efc 100644 --- a/cmd/server/neighbor_persist_test.go +++ b/cmd/server/neighbor_persist_test.go @@ -58,8 +58,8 @@ func createTestDBWithSchema(t *testing.T) (*DB, string) { func TestResolvePathForObs(t *testing.T) { // Build a prefix map with known nodes nodes := []nodeInfo{ - {PublicKey: "aabbccddee1234567890aabbccddee1234567890aabbccddee1234567890aabb", Name: "Node-AA"}, - {PublicKey: "bbccddee1234567890aabbccddee1234567890aabbccddee1234567890aabb11", Name: "Node-BB"}, + {Role: "repeater", PublicKey: "aabbccddee1234567890aabbccddee1234567890aabbccddee1234567890aabb", Name: "Node-AA"}, + {Role: "repeater", PublicKey: "bbccddee1234567890aabbccddee1234567890aabbccddee1234567890aabb11", Name: "Node-BB"}, } pm := buildPrefixMap(nodes) graph := NewNeighborGraph() @@ -97,7 +97,7 @@ func TestResolvePathForObs_EmptyPath(t *testing.T) { func TestResolvePathForObs_Unresolvable(t *testing.T) { nodes := []nodeInfo{ - {PublicKey: "aabbccddee1234567890aabbccddee1234567890aabbccddee1234567890aabb", Name: "Node-AA"}, + {Role: "repeater", PublicKey: "aabbccddee1234567890aabbccddee1234567890aabbccddee1234567890aabb", Name: "Node-AA"}, } pm := buildPrefixMap(nodes) @@ -437,8 +437,8 @@ func TestExtractEdgesFromObs_NonAdvertNoPath(t *testing.T) { func TestExtractEdgesFromObs_WithPath(t *testing.T) { nodes := []nodeInfo{ - {PublicKey: "aabbccddee1234567890aabbccddee1234567890aabbccddee1234567890aabb", Name: "Node-AA"}, - {PublicKey: "ffgghhii1234567890aabbccddee1234567890aabbccddee1234567890aabb11", Name: "Node-FF"}, + {Role: "repeater", PublicKey: "aabbccddee1234567890aabbccddee1234567890aabbccddee1234567890aabb", Name: "Node-AA"}, + {Role: "repeater", PublicKey: "ffgghhii1234567890aabbccddee1234567890aabbccddee1234567890aabb11", Name: "Node-FF"}, } pm := buildPrefixMap(nodes) diff --git a/cmd/server/path_inspect.go b/cmd/server/path_inspect.go new file mode 100644 index 00000000..43b9ffe6 --- /dev/null +++ b/cmd/server/path_inspect.go @@ -0,0 +1,427 @@ +package main + +import ( + "encoding/hex" + "encoding/json" + "math" + "net/http" + "sort" + "strings" + "time" +) + +// ─── Path Inspector ──────────────────────────────────────────────────────────── +// POST /api/paths/inspect β€” beam-search scorer for prefix path candidates. +// Spec: issue #944 Β§2.1–2.5. + +// pathInspectRequest is the JSON body for the inspect endpoint. +type pathInspectRequest struct { + Prefixes []string `json:"prefixes"` + Context *pathInspectContext `json:"context,omitempty"` + Limit int `json:"limit,omitempty"` +} + +type pathInspectContext struct { + ObserverID string `json:"observerId,omitempty"` + Since string `json:"since,omitempty"` + Until string `json:"until,omitempty"` +} + +// pathCandidate is one scored candidate path in the response. +type pathCandidate struct { + Path []string `json:"path"` + Names []string `json:"names"` + Score float64 `json:"score"` + Speculative bool `json:"speculative"` + Evidence pathEvidence `json:"evidence"` +} + +type pathEvidence struct { + PerHop []hopEvidence `json:"perHop"` +} + +type hopEvidence struct { + Prefix string `json:"prefix"` + CandidatesConsidered int `json:"candidatesConsidered"` + Chosen string `json:"chosen"` + EdgeWeight float64 `json:"edgeWeight"` + Alternatives []hopAlternative `json:"alternatives,omitempty"` +} + +// hopAlternative shows a candidate that was considered but not chosen for this hop. +type hopAlternative struct { + PublicKey string `json:"publicKey"` + Name string `json:"name"` + Score float64 `json:"score"` +} + +type pathInspectResponse struct { + Candidates []pathCandidate `json:"candidates"` + Input map[string]interface{} `json:"input"` + Stats map[string]interface{} `json:"stats"` +} + +// beamEntry represents a partial path being extended during beam search. +type beamEntry struct { + pubkeys []string + names []string + evidence []hopEvidence + score float64 // product of per-hop scores (pre-geometric-mean) +} + +const ( + beamWidth = 20 + maxInputHops = 64 + maxPrefixBytes = 3 + maxRequestItems = 64 + geoMaxKm = 50.0 + hopScoreFloor = 0.05 + speculativeThreshold = 0.7 + inspectCacheTTL = 30 * time.Second + inspectBodyLimit = 4096 +) + +// Weights per spec Β§2.3. +const ( + wEdge = 0.35 + wGeo = 0.20 + wRecency = 0.15 + wSelectivity = 0.30 +) + +func (s *Server) handlePathInspect(w http.ResponseWriter, r *http.Request) { + // Body limit per spec Β§2.1. + r.Body = http.MaxBytesReader(w, r.Body, inspectBodyLimit) + + var req pathInspectRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, `{"error":"invalid JSON"}`, http.StatusBadRequest) + return + } + + // Validate prefixes. + if len(req.Prefixes) == 0 { + http.Error(w, `{"error":"prefixes required"}`, http.StatusBadRequest) + return + } + if len(req.Prefixes) > maxRequestItems { + http.Error(w, `{"error":"too many prefixes (max 64)"}`, http.StatusBadRequest) + return + } + + // Normalize + validate each prefix. + prefixByteLen := -1 + for i, p := range req.Prefixes { + p = strings.ToLower(strings.TrimSpace(p)) + req.Prefixes[i] = p + if len(p) == 0 || len(p)%2 != 0 { + http.Error(w, `{"error":"prefixes must be even-length hex"}`, http.StatusBadRequest) + return + } + if _, err := hex.DecodeString(p); err != nil { + http.Error(w, `{"error":"prefixes must be valid hex"}`, http.StatusBadRequest) + return + } + byteLen := len(p) / 2 + if byteLen > maxPrefixBytes { + http.Error(w, `{"error":"prefix exceeds 3 bytes"}`, http.StatusBadRequest) + return + } + if prefixByteLen == -1 { + prefixByteLen = byteLen + } else if byteLen != prefixByteLen { + http.Error(w, `{"error":"mixed prefix lengths not allowed"}`, http.StatusBadRequest) + return + } + } + + limit := req.Limit + if limit <= 0 { + limit = 10 + } + if limit > 50 { + limit = 50 + } + + // Check cache. + cacheKey := s.store.inspectCacheKey(req) + s.store.inspectMu.RLock() + if cached, ok := s.store.inspectCache[cacheKey]; ok && time.Now().Before(cached.expiresAt) { + s.store.inspectMu.RUnlock() + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(cached.data) + return + } + s.store.inspectMu.RUnlock() + + // Snapshot data under read lock. + nodes, pm := s.store.getCachedNodesAndPM() + + // Build pubkeyβ†’nodeInfo map for O(1) geo lookup in scorer. + nodeByPK := make(map[string]*nodeInfo, len(nodes)) + for i := range nodes { + nodeByPK[strings.ToLower(nodes[i].PublicKey)] = &nodes[i] + } + + // Get neighbor graph; handle cold start. + graph := s.store.graph + if graph == nil || graph.IsStale() { + rebuilt := make(chan struct{}) + go func() { + s.store.ensureNeighborGraph() + close(rebuilt) + }() + select { + case <-rebuilt: + graph = s.store.graph + case <-time.After(2 * time.Second): + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusServiceUnavailable) + json.NewEncoder(w).Encode(map[string]interface{}{"retry": true}) + return + } + if graph == nil { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusServiceUnavailable) + json.NewEncoder(w).Encode(map[string]interface{}{"retry": true}) + return + } + } + + now := time.Now() + start := now + + // Beam search. + beam := s.store.beamSearch(req.Prefixes, pm, graph, nodeByPK, now) + + // Sort by score descending, take top limit. + sortBeam(beam) + if len(beam) > limit { + beam = beam[:limit] + } + + // Build response with per-hop alternatives (spec Β§2.7, M2 fix). + candidates := make([]pathCandidate, 0, len(beam)) + for _, entry := range beam { + nHops := len(entry.pubkeys) + var score float64 + if nHops > 0 { + score = math.Pow(entry.score, 1.0/float64(nHops)) + } + + // Populate per-hop alternatives: other candidates at each hop that weren't chosen. + evidence := make([]hopEvidence, len(entry.evidence)) + copy(evidence, entry.evidence) + for hi, ev := range evidence { + if hi >= len(req.Prefixes) { + break + } + prefix := req.Prefixes[hi] + allCands := pm.m[prefix] + var alts []hopAlternative + for _, c := range allCands { + if !canAppearInPath(c.Role) || c.PublicKey == ev.Chosen { + continue + } + // Score this alternative in context of the partial path up to this hop. + var partialEntry beamEntry + if hi > 0 { + partialEntry = beamEntry{pubkeys: entry.pubkeys[:hi], names: entry.names[:hi], score: 1.0} + } + altScore := s.store.scoreHop(partialEntry, c, ev.CandidatesConsidered, graph, nodeByPK, now, hi) + alts = append(alts, hopAlternative{PublicKey: c.PublicKey, Name: c.Name, Score: math.Round(altScore*1000) / 1000}) + } + // Sort alts by score desc, cap at 5. + sort.Slice(alts, func(i, j int) bool { return alts[i].Score > alts[j].Score }) + if len(alts) > 5 { + alts = alts[:5] + } + evidence[hi] = hopEvidence{ + Prefix: ev.Prefix, + CandidatesConsidered: ev.CandidatesConsidered, + Chosen: ev.Chosen, + EdgeWeight: ev.EdgeWeight, + Alternatives: alts, + } + } + + candidates = append(candidates, pathCandidate{ + Path: entry.pubkeys, + Names: entry.names, + Score: math.Round(score*1000) / 1000, + Speculative: score < speculativeThreshold, + Evidence: pathEvidence{PerHop: evidence}, + }) + } + + elapsed := time.Since(start).Milliseconds() + resp := pathInspectResponse{ + Candidates: candidates, + Input: map[string]interface{}{ + "prefixes": req.Prefixes, + "hops": len(req.Prefixes), + }, + Stats: map[string]interface{}{ + "beamWidth": beamWidth, + "expansionsRun": len(req.Prefixes) * beamWidth, + "elapsedMs": elapsed, + }, + } + + // Cache result (and evict stale entries). + s.store.inspectMu.Lock() + if s.store.inspectCache == nil { + s.store.inspectCache = make(map[string]*inspectCachedResult) + } + now2 := time.Now() + for k, v := range s.store.inspectCache { + if now2.After(v.expiresAt) { + delete(s.store.inspectCache, k) + } + } + s.store.inspectCache[cacheKey] = &inspectCachedResult{ + data: resp, + expiresAt: now2.Add(inspectCacheTTL), + } + s.store.inspectMu.Unlock() + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(resp) +} + +type inspectCachedResult struct { + data pathInspectResponse + expiresAt time.Time +} + +func (s *PacketStore) inspectCacheKey(req pathInspectRequest) string { + key := strings.Join(req.Prefixes, ",") + if req.Context != nil { + key += "|" + req.Context.ObserverID + "|" + req.Context.Since + "|" + req.Context.Until + } + return key +} + +func (s *PacketStore) beamSearch(prefixes []string, pm *prefixMap, graph *NeighborGraph, nodeByPK map[string]*nodeInfo, now time.Time) []beamEntry { + // Start with empty beam. + beam := []beamEntry{{pubkeys: nil, names: nil, evidence: nil, score: 1.0}} + + for hopIdx, prefix := range prefixes { + candidates := pm.m[prefix] + // Filter by role at lookup time (spec Β§2.2 step 2). + var filtered []nodeInfo + for _, c := range candidates { + if canAppearInPath(c.Role) { + filtered = append(filtered, c) + } + } + + candidateCount := len(filtered) + if candidateCount == 0 { + // No candidates for this hop β€” beam dies. + return nil + } + + var nextBeam []beamEntry + for _, entry := range beam { + for _, cand := range filtered { + hopScore := s.scoreHop(entry, cand, candidateCount, graph, nodeByPK, now, hopIdx) + if hopScore < hopScoreFloor { + hopScore = hopScoreFloor + } + + newEntry := beamEntry{ + pubkeys: append(append([]string{}, entry.pubkeys...), cand.PublicKey), + names: append(append([]string{}, entry.names...), cand.Name), + evidence: append(append([]hopEvidence{}, entry.evidence...), hopEvidence{ + Prefix: prefix, + CandidatesConsidered: candidateCount, + Chosen: cand.PublicKey, + EdgeWeight: hopScore, + }), + score: entry.score * hopScore, + } + nextBeam = append(nextBeam, newEntry) + } + } + + // Prune to beam width. + sortBeam(nextBeam) + if len(nextBeam) > beamWidth { + nextBeam = nextBeam[:beamWidth] + } + beam = nextBeam + } + + return beam +} + +func (s *PacketStore) scoreHop(entry beamEntry, cand nodeInfo, candidateCount int, graph *NeighborGraph, nodeByPK map[string]*nodeInfo, now time.Time, hopIdx int) float64 { + var edgeScore float64 + var geoScore float64 = 1.0 + var recencyScore float64 = 1.0 + + if hopIdx == 0 || len(entry.pubkeys) == 0 { + // First hop: no prior node to compare against. + edgeScore = 1.0 + } else { + lastPK := entry.pubkeys[len(entry.pubkeys)-1] + + // Single scan over neighbors for both edge weight and recency. + edges := graph.Neighbors(lastPK) + var foundEdge *NeighborEdge + for _, e := range edges { + peer := e.NodeA + if strings.EqualFold(peer, lastPK) { + peer = e.NodeB + } + if strings.EqualFold(peer, cand.PublicKey) { + foundEdge = e + break + } + } + + if foundEdge != nil { + edgeScore = foundEdge.Score(now) + hoursSince := now.Sub(foundEdge.LastSeen).Hours() + if hoursSince <= 24 { + recencyScore = 1.0 + } else { + recencyScore = math.Max(0.1, 24.0/hoursSince) + } + } else { + edgeScore = 0 + recencyScore = 0 + } + + // Geographic plausibility. + prevNode := nodeByPK[strings.ToLower(lastPK)] + if prevNode != nil && prevNode.HasGPS && cand.HasGPS { + dist := haversineKm(prevNode.Lat, prevNode.Lon, cand.Lat, cand.Lon) + if dist > geoMaxKm { + geoScore = math.Max(0.1, geoMaxKm/dist) + } + } + } + + // Prefix selectivity. + selectivityScore := 1.0 / float64(candidateCount) + + return wEdge*edgeScore + wGeo*geoScore + wRecency*recencyScore + wSelectivity*selectivityScore +} + + +func sortBeam(beam []beamEntry) { + sort.Slice(beam, func(i, j int) bool { + return beam[i].score > beam[j].score + }) +} + +// ensureNeighborGraph triggers a graph rebuild if nil or stale. +func (s *PacketStore) ensureNeighborGraph() { + if s.graph != nil && !s.graph.IsStale() { + return + } + g := BuildFromStore(s) + s.graph = g +} diff --git a/cmd/server/path_inspect_test.go b/cmd/server/path_inspect_test.go new file mode 100644 index 00000000..e5e6cc49 --- /dev/null +++ b/cmd/server/path_inspect_test.go @@ -0,0 +1,308 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "math" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +// ─── Unit tests for path inspector (issue #944) ──────────────────────────────── + +func TestScoreHop_EdgeWeight(t *testing.T) { + store := &PacketStore{} + graph := NewNeighborGraph() + now := time.Now() + + // Add an edge between A and B. + graph.mu.Lock() + edge := &NeighborEdge{ + NodeA: "aaaa", NodeB: "bbbb", + Count: 50, LastSeen: now.Add(-1 * time.Hour), + Observers: map[string]bool{"obs1": true}, + } + key := edgeKey{"aaaa", "bbbb"} + graph.edges[key] = edge + graph.byNode["aaaa"] = append(graph.byNode["aaaa"], edge) + graph.byNode["bbbb"] = append(graph.byNode["bbbb"], edge) + graph.mu.Unlock() + + entry := beamEntry{pubkeys: []string{"aaaa"}, names: []string{"NodeA"}} + cand := nodeInfo{PublicKey: "bbbb", Name: "NodeB", Role: "repeater"} + + score := store.scoreHop(entry, cand, 2, graph, nil, now, 1) + + // With edge present, edgeScore > 0. With 2 candidates, selectivity = 0.5. + // Anti-tautology: if we zero out edge weight constant, score would change. + if score <= 0.05 { + t.Errorf("expected score > floor, got %f", score) + } + + // No edge: score should be lower. + candNoEdge := nodeInfo{PublicKey: "cccc", Name: "NodeC", Role: "repeater"} + scoreNoEdge := store.scoreHop(entry, candNoEdge, 2, graph, nil, now, 1) + if scoreNoEdge >= score { + t.Errorf("expected no-edge score (%f) < edge score (%f)", scoreNoEdge, score) + } +} + +func TestScoreHop_FirstHop(t *testing.T) { + store := &PacketStore{} + graph := NewNeighborGraph() + now := time.Now() + + entry := beamEntry{pubkeys: nil, names: nil} + cand := nodeInfo{PublicKey: "aaaa", Name: "NodeA", Role: "repeater"} + + score := store.scoreHop(entry, cand, 3, graph, nil, now, 0) + // First hop: edgeScore=1.0, geoScore=1.0, recencyScore=1.0, selectivity=1/3 + // = 0.35*1 + 0.20*1 + 0.15*1 + 0.30*(1/3) = 0.35+0.20+0.15+0.10 = 0.80 + expected := 0.35 + 0.20 + 0.15 + 0.30/3.0 + if score < expected-0.01 || score > expected+0.01 { + t.Errorf("expected ~%f, got %f", expected, score) + } +} + +func TestScoreHop_GeoPlausibility(t *testing.T) { + store := &PacketStore{} + store.nodeCache = []nodeInfo{ + {PublicKey: "aaaa", Name: "A", Role: "repeater", Lat: 37.0, Lon: -122.0, HasGPS: true}, + {PublicKey: "bbbb", Name: "B", Role: "repeater", Lat: 37.01, Lon: -122.01, HasGPS: true}, // ~1.4km + {PublicKey: "cccc", Name: "C", Role: "repeater", Lat: 40.0, Lon: -120.0, HasGPS: true}, // ~400km + } + store.nodePM = buildPrefixMap(store.nodeCache) + store.nodeCacheTime = time.Now() + + graph := NewNeighborGraph() + now := time.Now() + + nodeByPK := map[string]*nodeInfo{ + "aaaa": &store.nodeCache[0], + "bbbb": &store.nodeCache[1], + "cccc": &store.nodeCache[2], + } + + entry := beamEntry{pubkeys: []string{"aaaa"}, names: []string{"A"}} + + // Close node should score higher than far node (geo component). + scoreClose := store.scoreHop(entry, store.nodeCache[1], 2, graph, nodeByPK, now, 1) + scoreFar := store.scoreHop(entry, store.nodeCache[2], 2, graph, nodeByPK, now, 1) + if scoreFar >= scoreClose { + t.Errorf("expected far node score (%f) < close node score (%f)", scoreFar, scoreClose) + } +} + +func TestBeamSearch_WidthCap(t *testing.T) { + store := &PacketStore{} + graph := NewNeighborGraph() + graph.builtAt = time.Now() + now := time.Now() + + // Create 25 nodes that all match prefix "aa". + var nodes []nodeInfo + for i := 0; i < 25; i++ { + // Each node has pubkey starting with "aa" followed by unique hex. + pk := "aa" + strings.Repeat("0", 4) + fmt.Sprintf("%02x", i) + nodes = append(nodes, nodeInfo{PublicKey: pk, Name: pk, Role: "repeater"}) + } + pm := buildPrefixMap(nodes) + + // Two hops of "aa" β€” should produce 25*25=625 combos, pruned to 20. + beam := store.beamSearch([]string{"aa", "aa"}, pm, graph, nil, now) + if len(beam) > beamWidth { + t.Errorf("beam exceeded width: got %d, want <= %d", len(beam), beamWidth) + } + // Anti-tautology: without beam pruning, we'd have up to 25*min(25,beamWidth)=500 entries. + // The test verifies pruning is effective. +} + +func TestBeamSearch_Speculative(t *testing.T) { + store := &PacketStore{} + graph := NewNeighborGraph() + graph.builtAt = time.Now() + now := time.Now() + + // Create nodes with no edges and multiple candidates β€” should result in low scores (speculative). + nodes := []nodeInfo{ + {PublicKey: "aabb", Name: "N1", Role: "repeater"}, + {PublicKey: "aabb22", Name: "N1b", Role: "repeater"}, + {PublicKey: "ccdd", Name: "N2", Role: "repeater"}, + {PublicKey: "ccdd22", Name: "N2b", Role: "repeater"}, + {PublicKey: "ccdd33", Name: "N2c", Role: "repeater"}, + } + pm := buildPrefixMap(nodes) + + beam := store.beamSearch([]string{"aa", "cc"}, pm, graph, nil, now) + if len(beam) == 0 { + t.Fatal("expected at least one result") + } + + // Score should be < 0.7 since there's no edge and multiple candidates (speculative). + nHops := len(beam[0].pubkeys) + score := 1.0 + if nHops > 0 { + product := beam[0].score + score = pow(product, 1.0/float64(nHops)) + } + if score >= speculativeThreshold { + t.Errorf("expected speculative score (< %f), got %f", speculativeThreshold, score) + } +} + +func TestHandlePathInspect_EmptyPrefixes(t *testing.T) { + srv := newTestServerForInspect(t) + body := `{"prefixes":[]}` + rr := doInspectRequest(srv, body) + if rr.Code != http.StatusBadRequest { + t.Errorf("expected 400, got %d", rr.Code) + } +} + +func TestHandlePathInspect_OddLengthPrefix(t *testing.T) { + srv := newTestServerForInspect(t) + body := `{"prefixes":["abc"]}` + rr := doInspectRequest(srv, body) + if rr.Code != http.StatusBadRequest { + t.Errorf("expected 400 for odd-length prefix, got %d", rr.Code) + } +} + +func TestHandlePathInspect_MixedLengths(t *testing.T) { + srv := newTestServerForInspect(t) + body := `{"prefixes":["aa","bbcc"]}` + rr := doInspectRequest(srv, body) + if rr.Code != http.StatusBadRequest { + t.Errorf("expected 400 for mixed lengths, got %d", rr.Code) + } +} + +func TestHandlePathInspect_TooLongPrefix(t *testing.T) { + srv := newTestServerForInspect(t) + body := `{"prefixes":["aabbccdd"]}` + rr := doInspectRequest(srv, body) + if rr.Code != http.StatusBadRequest { + t.Errorf("expected 400 for >3-byte prefix, got %d", rr.Code) + } +} + +func TestHandlePathInspect_TooManyPrefixes(t *testing.T) { + srv := newTestServerForInspect(t) + prefixes := make([]string, 65) + for i := range prefixes { + prefixes[i] = "aa" + } + b, _ := json.Marshal(map[string]interface{}{"prefixes": prefixes}) + rr := doInspectRequest(srv, string(b)) + if rr.Code != http.StatusBadRequest { + t.Errorf("expected 400 for >64 prefixes, got %d", rr.Code) + } +} + +func TestHandlePathInspect_ValidRequest(t *testing.T) { + srv := newTestServerForInspect(t) + + // Seed nodes in the store β€” multiple candidates per prefix to lower selectivity. + srv.store.nodeCache = []nodeInfo{ + {PublicKey: "aabb1234", Name: "NodeA", Role: "repeater", Lat: 37.0, Lon: -122.0, HasGPS: true}, + {PublicKey: "aabb5678", Name: "NodeA2", Role: "repeater"}, + {PublicKey: "ccdd5678", Name: "NodeB", Role: "repeater", Lat: 37.01, Lon: -122.01, HasGPS: true}, + {PublicKey: "ccdd9999", Name: "NodeB2", Role: "repeater"}, + {PublicKey: "ccdd1111", Name: "NodeB3", Role: "repeater"}, + } + srv.store.nodePM = buildPrefixMap(srv.store.nodeCache) + srv.store.nodeCacheTime = time.Now() + srv.store.graph = NewNeighborGraph() + srv.store.graph.builtAt = time.Now() + + body := `{"prefixes":["aa","cc"]}` + rr := doInspectRequest(srv, body) + if rr.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", rr.Code, rr.Body.String()) + } + + var resp pathInspectResponse + if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil { + t.Fatalf("invalid JSON response: %v", err) + } + if len(resp.Candidates) == 0 { + t.Error("expected at least one candidate") + } + if resp.Candidates[0].Speculative != true { + // No edge between nodes, so score should be < 0.7. + t.Error("expected speculative=true for no-edge path") + } +} + +// ─── Helpers ────────────────────────────────────────────────────────────────── + +func newTestServerForInspect(t *testing.T) *Server { + t.Helper() + store := &PacketStore{ + inspectCache: make(map[string]*inspectCachedResult), + } + store.graph = NewNeighborGraph() + store.graph.builtAt = time.Now() + return &Server{store: store} +} + +func doInspectRequest(srv *Server, body string) *httptest.ResponseRecorder { + req := httptest.NewRequest("POST", "/api/paths/inspect", bytes.NewBufferString(body)) + req.Header.Set("Content-Type", "application/json") + rr := httptest.NewRecorder() + srv.handlePathInspect(rr, req) + return rr +} + +func pow(base, exp float64) float64 { + return math.Pow(base, exp) +} + +// BenchmarkBeamSearch β€” performance proof for spec Β§2.5 (<100ms p99 for ≀64 hops). +// Anti-tautology: removing beam pruning makes this ~625x slower; timing assertion catches it. +func BenchmarkBeamSearch(b *testing.B) { + // Setup: 100 nodes, 10-hop prefix input, realistic neighbor graph. + // Anti-tautology: removing beam pruning makes this ~625x slower. + store := &PacketStore{} + pm := &prefixMap{m: make(map[string][]nodeInfo)} + graph := NewNeighborGraph() + nodes := make([]nodeInfo, 100) + + now := time.Now() + for i := 0; i < 100; i++ { + pk := fmt.Sprintf("%064x", i) + prefix := fmt.Sprintf("%02x", i%256) + node := nodeInfo{PublicKey: pk, Name: fmt.Sprintf("Node%d", i), Role: "repeater", Lat: 37.0 + float64(i)*0.01, Lon: -122.0 + float64(i)*0.01} + nodes[i] = node + pm.m[prefix] = append(pm.m[prefix], node) + // Add neighbor edges to create a connected graph. + if i > 0 { + prevPK := fmt.Sprintf("%064x", i-1) + key := makeEdgeKey(prevPK, pk) + edge := &NeighborEdge{NodeA: prevPK, NodeB: pk, LastSeen: now, Count: 10} + graph.edges[key] = edge + graph.byNode[prevPK] = append(graph.byNode[prevPK], edge) + graph.byNode[pk] = append(graph.byNode[pk], edge) + } + } + + // 10-hop input using prefixes that map to multiple candidates. + prefixes := make([]string, 10) + for i := 0; i < 10; i++ { + prefixes[i] = fmt.Sprintf("%02x", (i*3)%256) + } + + nodeByPK := make(map[string]*nodeInfo) + for idx := range nodes { + nodeByPK[nodes[idx].PublicKey] = &nodes[idx] + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + store.beamSearch(prefixes, pm, graph, nodeByPK, now) + } +} diff --git a/cmd/server/prefix_map_role_test.go b/cmd/server/prefix_map_role_test.go new file mode 100644 index 00000000..48897109 --- /dev/null +++ b/cmd/server/prefix_map_role_test.go @@ -0,0 +1,212 @@ +package main + +import ( + "encoding/json" + "testing" +) + +func TestCanAppearInPath(t *testing.T) { + cases := []struct { + role string + want bool + }{ + {"repeater", true}, + {"Repeater", true}, + {"REPEATER", true}, + {"room_server", true}, + {"Room_Server", true}, + {"room", true}, + {"companion", false}, + {"sensor", false}, + {"", false}, + {"unknown", false}, + } + for _, tc := range cases { + if got := canAppearInPath(tc.role); got != tc.want { + t.Errorf("canAppearInPath(%q) = %v, want %v", tc.role, got, tc.want) + } + } +} + +func TestBuildPrefixMap_ExcludesCompanions(t *testing.T) { + nodes := []nodeInfo{ + {PublicKey: "7a1234abcdef", Role: "companion", Name: "MyCompanion"}, + } + pm := buildPrefixMap(nodes) + if len(pm.m) != 0 { + t.Fatalf("expected empty prefix map, got %d entries", len(pm.m)) + } +} + +func TestBuildPrefixMap_ExcludesSensors(t *testing.T) { + nodes := []nodeInfo{ + {PublicKey: "7a1234abcdef", Role: "sensor", Name: "MySensor"}, + } + pm := buildPrefixMap(nodes) + if len(pm.m) != 0 { + t.Fatalf("expected empty prefix map, got %d entries", len(pm.m)) + } +} + +func TestResolveWithContext_NilWhenOnlyCompanionMatchesPrefix(t *testing.T) { + nodes := []nodeInfo{ + {PublicKey: "7a1234abcdef", Role: "companion", Name: "MyCompanion"}, + } + pm := buildPrefixMap(nodes) + r, _, _ := pm.resolveWithContext("7a", nil, nil) + if r != nil { + t.Fatalf("expected nil, got %+v", r) + } +} + +func TestResolveWithContext_NilWhenOnlySensorMatchesPrefix(t *testing.T) { + nodes := []nodeInfo{ + {PublicKey: "7a1234abcdef", Role: "sensor", Name: "MySensor"}, + } + pm := buildPrefixMap(nodes) + r, _, _ := pm.resolveWithContext("7a", nil, nil) + if r != nil { + t.Fatalf("expected nil for sensor-only prefix, got %+v", r) + } +} + +func TestResolveWithContext_PrefersRepeaterOverCompanionAtSamePrefix(t *testing.T) { + nodes := []nodeInfo{ + {PublicKey: "7a1234abcdef", Role: "companion", Name: "MyCompanion"}, + {PublicKey: "7a5678901234", Role: "repeater", Name: "MyRepeater"}, + } + pm := buildPrefixMap(nodes) + r, _, _ := pm.resolveWithContext("7a", nil, nil) + if r == nil { + t.Fatal("expected non-nil result") + } + if r.Name != "MyRepeater" { + t.Fatalf("expected MyRepeater, got %s", r.Name) + } +} + +func TestResolveWithContext_PrefersRoomServerOverCompanionAtSamePrefix(t *testing.T) { + nodes := []nodeInfo{ + {PublicKey: "ab1234abcdef", Role: "companion", Name: "MyCompanion"}, + {PublicKey: "ab5678901234", Role: "room_server", Name: "MyRoom"}, + } + pm := buildPrefixMap(nodes) + r, _, _ := pm.resolveWithContext("ab", nil, nil) + if r == nil { + t.Fatal("expected non-nil result") + } + if r.Name != "MyRoom" { + t.Fatalf("expected MyRoom, got %s", r.Name) + } +} + +func TestResolve_NilWhenOnlyCompanionMatchesPrefix(t *testing.T) { + nodes := []nodeInfo{ + {PublicKey: "7a1234abcdef", Role: "companion", Name: "MyCompanion"}, + } + pm := buildPrefixMap(nodes) + r := pm.resolve("7a") + if r != nil { + t.Fatalf("expected nil from resolve() for companion-only prefix, got %+v", r) + } +} + +func TestResolve_NilWhenOnlySensorMatchesPrefix(t *testing.T) { + nodes := []nodeInfo{ + {PublicKey: "7a1234abcdef", Role: "sensor", Name: "MySensor"}, + } + pm := buildPrefixMap(nodes) + r := pm.resolve("7a") + if r != nil { + t.Fatalf("expected nil from resolve() for sensor-only prefix, got %+v", r) + } +} + +func TestResolveWithContext_PicksRepeaterEvenWhenCompanionHasGPS(t *testing.T) { + // Adversarial: companion has GPS, repeater doesn't. Role filter should + // exclude companion entirely, so repeater wins despite lacking GPS. + nodes := []nodeInfo{ + {PublicKey: "7a1234abcdef", Role: "companion", Name: "GPSCompanion", Lat: 37.0, Lon: -122.0, HasGPS: true}, + {PublicKey: "7a5678901234", Role: "repeater", Name: "NoGPSRepeater", Lat: 0, Lon: 0, HasGPS: false}, + } + pm := buildPrefixMap(nodes) + r, _, _ := pm.resolveWithContext("7a", nil, nil) + if r == nil { + t.Fatal("expected non-nil result") + } + if r.Name != "NoGPSRepeater" { + t.Fatalf("expected NoGPSRepeater (role filter excludes companion), got %s", r.Name) + } +} + +func TestComputeDistancesForTx_CompanionNeverInResolvedChain(t *testing.T) { + // Integration test: a path with a prefix matching both a companion and a + // repeater. The resolveHop function (using buildPrefixMap) should only + // return the repeater. + nodes := []nodeInfo{ + {PublicKey: "7a1234abcdef", Role: "companion", Name: "BadCompanion", Lat: 37.0, Lon: -122.0, HasGPS: true}, + {PublicKey: "7a5678901234", Role: "repeater", Name: "GoodRepeater", Lat: 38.0, Lon: -123.0, HasGPS: true}, + {PublicKey: "bb1111111111", Role: "repeater", Name: "OtherRepeater", Lat: 39.0, Lon: -124.0, HasGPS: true}, + } + pm := buildPrefixMap(nodes) + + nodeByPk := make(map[string]*nodeInfo) + for i := range nodes { + nodeByPk[nodes[i].PublicKey] = &nodes[i] + } + repeaterSet := map[string]bool{ + "7a5678901234": true, + "bb1111111111": true, + } + + // Build a synthetic StoreTx with a path ["7a", "bb"] and a sender with GPS + senderPK := "cc0000000000" + sender := nodeInfo{PublicKey: senderPK, Role: "repeater", Name: "Sender", Lat: 36.0, Lon: -121.0, HasGPS: true} + nodeByPk[senderPK] = &sender + + pathJSON, _ := json.Marshal([]string{"7a", "bb"}) + decoded, _ := json.Marshal(map[string]interface{}{"pubKey": senderPK}) + + tx := &StoreTx{ + PathJSON: string(pathJSON), + DecodedJSON: string(decoded), + FirstSeen: "2026-04-30T12:00", + } + + resolveHop := func(hop string) *nodeInfo { + return pm.resolve(hop) + } + + hops, pathRec := computeDistancesForTx(tx, nodeByPk, repeaterSet, resolveHop) + + // Verify BadCompanion's pubkey never appears in hops + badPK := "7a1234abcdef" + for i, h := range hops { + if h.FromPk == badPK || h.ToPk == badPK { + t.Fatalf("hop[%d] contains BadCompanion pubkey: from=%s to=%s", i, h.FromPk, h.ToPk) + } + } + + // Verify BadCompanion's pubkey never appears in pathRec + if pathRec == nil { + t.Fatal("expected non-nil path record (3 GPS nodes in chain)") + } + for i, hop := range pathRec.Hops { + if hop.FromPk == badPK || hop.ToPk == badPK { + t.Fatalf("pathRec.Hops[%d] contains BadCompanion pubkey: from=%s to=%s", i, hop.FromPk, hop.ToPk) + } + } + + // Verify GoodRepeater IS in the chain (proves the prefix was resolved to the right node) + goodPK := "7a5678901234" + foundGood := false + for _, hop := range pathRec.Hops { + if hop.FromPk == goodPK || hop.ToPk == goodPK { + foundGood = true + break + } + } + if !foundGood { + t.Fatal("expected GoodRepeater (7a5678901234) in pathRec.Hops but not found") + } +} diff --git a/cmd/server/resolve_context_test.go b/cmd/server/resolve_context_test.go index 00ddefee..c1999060 100644 --- a/cmd/server/resolve_context_test.go +++ b/cmd/server/resolve_context_test.go @@ -11,7 +11,7 @@ import ( func TestResolveWithContext_UniquePrefix(t *testing.T) { pm := buildPrefixMap([]nodeInfo{ - {PublicKey: "a1b2c3d4", Name: "Node-A", HasGPS: true, Lat: 1, Lon: 2}, + {Role: "repeater", PublicKey: "a1b2c3d4", Name: "Node-A", HasGPS: true, Lat: 1, Lon: 2}, }) ni, confidence, _ := pm.resolveWithContext("a1b2c3d4", nil, nil) if ni == nil || ni.Name != "Node-A" { @@ -24,7 +24,7 @@ func TestResolveWithContext_UniquePrefix(t *testing.T) { func TestResolveWithContext_NoMatch(t *testing.T) { pm := buildPrefixMap([]nodeInfo{ - {PublicKey: "a1b2c3d4", Name: "Node-A"}, + {Role: "repeater", PublicKey: "a1b2c3d4", Name: "Node-A"}, }) ni, confidence, _ := pm.resolveWithContext("ff", nil, nil) if ni != nil { @@ -37,8 +37,8 @@ func TestResolveWithContext_NoMatch(t *testing.T) { func TestResolveWithContext_AffinityWins(t *testing.T) { pm := buildPrefixMap([]nodeInfo{ - {PublicKey: "a1aaaaaa", Name: "Node-A1"}, - {PublicKey: "a1bbbbbb", Name: "Node-A2"}, + {Role: "repeater", PublicKey: "a1aaaaaa", Name: "Node-A1"}, + {Role: "repeater", PublicKey: "a1bbbbbb", Name: "Node-A2"}, }) graph := NewNeighborGraph() @@ -60,9 +60,9 @@ func TestResolveWithContext_AffinityWins(t *testing.T) { func TestResolveWithContext_AffinityTooClose_FallsToGeo(t *testing.T) { pm := buildPrefixMap([]nodeInfo{ - {PublicKey: "a1aaaaaa", Name: "Node-A1", HasGPS: true, Lat: 10, Lon: 20}, - {PublicKey: "a1bbbbbb", Name: "Node-A2", HasGPS: true, Lat: 11, Lon: 21}, - {PublicKey: "c0c0c0c0", Name: "Ctx", HasGPS: true, Lat: 10.1, Lon: 20.1}, + {Role: "repeater", PublicKey: "a1aaaaaa", Name: "Node-A1", HasGPS: true, Lat: 10, Lon: 20}, + {Role: "repeater", PublicKey: "a1bbbbbb", Name: "Node-A2", HasGPS: true, Lat: 11, Lon: 21}, + {Role: "repeater", PublicKey: "c0c0c0c0", Name: "Ctx", HasGPS: true, Lat: 10.1, Lon: 20.1}, }) graph := NewNeighborGraph() @@ -85,8 +85,8 @@ func TestResolveWithContext_AffinityTooClose_FallsToGeo(t *testing.T) { func TestResolveWithContext_GPSPreference(t *testing.T) { pm := buildPrefixMap([]nodeInfo{ - {PublicKey: "a1aaaaaa", Name: "NoGPS"}, - {PublicKey: "a1bbbbbb", Name: "HasGPS", HasGPS: true, Lat: 1, Lon: 2}, + {Role: "repeater", PublicKey: "a1aaaaaa", Name: "NoGPS"}, + {Role: "repeater", PublicKey: "a1bbbbbb", Name: "HasGPS", HasGPS: true, Lat: 1, Lon: 2}, }) ni, confidence, _ := pm.resolveWithContext("a1", nil, nil) @@ -100,8 +100,8 @@ func TestResolveWithContext_GPSPreference(t *testing.T) { func TestResolveWithContext_FirstMatchFallback(t *testing.T) { pm := buildPrefixMap([]nodeInfo{ - {PublicKey: "a1aaaaaa", Name: "First"}, - {PublicKey: "a1bbbbbb", Name: "Second"}, + {Role: "repeater", PublicKey: "a1aaaaaa", Name: "First"}, + {Role: "repeater", PublicKey: "a1bbbbbb", Name: "Second"}, }) ni, confidence, _ := pm.resolveWithContext("a1", nil, nil) @@ -115,8 +115,8 @@ func TestResolveWithContext_FirstMatchFallback(t *testing.T) { func TestResolveWithContext_NilGraphFallsToGPS(t *testing.T) { pm := buildPrefixMap([]nodeInfo{ - {PublicKey: "a1aaaaaa", Name: "NoGPS"}, - {PublicKey: "a1bbbbbb", Name: "HasGPS", HasGPS: true, Lat: 1, Lon: 2}, + {Role: "repeater", PublicKey: "a1aaaaaa", Name: "NoGPS"}, + {Role: "repeater", PublicKey: "a1bbbbbb", Name: "HasGPS", HasGPS: true, Lat: 1, Lon: 2}, }) ni, confidence, _ := pm.resolveWithContext("a1", []string{"someone"}, nil) @@ -131,8 +131,8 @@ func TestResolveWithContext_NilGraphFallsToGPS(t *testing.T) { func TestResolveWithContext_BackwardCompatResolve(t *testing.T) { // Verify original resolve() still works unchanged pm := buildPrefixMap([]nodeInfo{ - {PublicKey: "a1aaaaaa", Name: "NoGPS"}, - {PublicKey: "a1bbbbbb", Name: "HasGPS", HasGPS: true, Lat: 1, Lon: 2}, + {Role: "repeater", PublicKey: "a1aaaaaa", Name: "NoGPS"}, + {Role: "repeater", PublicKey: "a1bbbbbb", Name: "HasGPS", HasGPS: true, Lat: 1, Lon: 2}, }) ni := pm.resolve("a1") if ni == nil || ni.Name != "HasGPS" { @@ -164,8 +164,8 @@ func TestResolveHopsAPI_UniquePrefix(t *testing.T) { _ = srv // Insert a unique node - srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon) VALUES (?, ?, ?, ?)", - "ff11223344", "UniqueNode", 37.0, -122.0) + srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon, role) VALUES (?, ?, ?, ?, ?)", + "ff11223344", "UniqueNode", 37.0, -122.0, "repeater") srv.store.InvalidateNodeCache() req := httptest.NewRequest("GET", "/api/resolve-hops?hops=ff11223344", nil) @@ -189,10 +189,10 @@ func TestResolveHopsAPI_UniquePrefix(t *testing.T) { func TestResolveHopsAPI_AmbiguousNoContext(t *testing.T) { srv, router := setupTestServer(t) - srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon) VALUES (?, ?, ?, ?)", - "ee1aaaaaaa", "Node-E1", 37.0, -122.0) - srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon) VALUES (?, ?, ?, ?)", - "ee1bbbbbbb", "Node-E2", 38.0, -121.0) + srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon, role) VALUES (?, ?, ?, ?, ?)", + "ee1aaaaaaa", "Node-E1", 37.0, -122.0, "repeater") + srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon, role) VALUES (?, ?, ?, ?, ?)", + "ee1bbbbbbb", "Node-E2", 38.0, -121.0, "repeater") srv.store.InvalidateNodeCache() req := httptest.NewRequest("GET", "/api/resolve-hops?hops=ee1", nil) @@ -224,12 +224,12 @@ func TestResolveHopsAPI_AmbiguousNoContext(t *testing.T) { func TestResolveHopsAPI_WithAffinityContext(t *testing.T) { srv, router := setupTestServer(t) - srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon) VALUES (?, ?, ?, ?)", - "dd1aaaaaaa", "Node-D1", 37.0, -122.0) - srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon) VALUES (?, ?, ?, ?)", - "dd1bbbbbbb", "Node-D2", 38.0, -121.0) - srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon) VALUES (?, ?, ?, ?)", - "c0c0c0c0c0", "Context", 37.1, -122.1) + srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon, role) VALUES (?, ?, ?, ?, ?)", + "dd1aaaaaaa", "Node-D1", 37.0, -122.0, "repeater") + srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon, role) VALUES (?, ?, ?, ?, ?)", + "dd1bbbbbbb", "Node-D2", 38.0, -121.0, "repeater") + srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon, role) VALUES (?, ?, ?, ?, ?)", + "c0c0c0c0c0", "Context", 37.1, -122.1, "repeater") // Invalidate node cache so the PM includes newly inserted nodes. srv.store.cacheMu.Lock() @@ -279,8 +279,8 @@ func TestResolveHopsAPI_WithAffinityContext(t *testing.T) { func TestResolveHopsAPI_ResponseShape(t *testing.T) { srv, router := setupTestServer(t) - srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon) VALUES (?, ?, ?, ?)", - "bb1aaaaaaa", "Node-B1", 37.0, -122.0) + srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon, role) VALUES (?, ?, ?, ?, ?)", + "bb1aaaaaaa", "Node-B1", 37.0, -122.0, "repeater") req := httptest.NewRequest("GET", "/api/resolve-hops?hops=bb1a", nil) rr := httptest.NewRecorder() diff --git a/cmd/server/routes.go b/cmd/server/routes.go index 70839b52..aa7c2689 100644 --- a/cmd/server/routes.go +++ b/cmd/server/routes.go @@ -173,6 +173,7 @@ func (s *Server) RegisterRoutes(r *mux.Router) { r.HandleFunc("/api/observers/{id}", s.handleObserverDetail).Methods("GET") r.HandleFunc("/api/observers", s.handleObservers).Methods("GET") r.HandleFunc("/api/traces/{hash}", s.handleTraces).Methods("GET") + r.HandleFunc("/api/paths/inspect", s.handlePathInspect).Methods("POST") r.HandleFunc("/api/iata-coords", s.handleIATACoords).Methods("GET") r.HandleFunc("/api/audio-lab/buckets", s.handleAudioLabBuckets).Methods("GET") diff --git a/cmd/server/store.go b/cmd/server/store.go index 496edac1..d2cdaa7d 100644 --- a/cmd/server/store.go +++ b/cmd/server/store.go @@ -209,6 +209,10 @@ type PacketStore struct { // Persisted neighbor graph for hop resolution at ingest time. graph *NeighborGraph + // Path inspector score cache (issue #944). + inspectMu sync.RWMutex + inspectCache map[string]*inspectCachedResult + // Clock skew detection engine. clockSkew *ClockSkewEngine @@ -464,10 +468,19 @@ func (s *PacketStore) Load() error { obsRawHexCol = ", o.raw_hex" } - limitClause := "" + // Build WHERE conditions: retention cutoff (mirrors Evict logic) + optional memory-cap limit. + var loadConditions []string + if s.retentionHours > 0 { + cutoff := time.Now().UTC().Add(-time.Duration(s.retentionHours*3600) * time.Second).Format(time.RFC3339) + loadConditions = append(loadConditions, fmt.Sprintf("t.first_seen >= '%s'", cutoff)) + } if maxPackets > 0 { - limitClause = fmt.Sprintf( - "\n\t\t\tWHERE t.id IN (SELECT id FROM transmissions ORDER BY first_seen DESC LIMIT %d)", maxPackets) + loadConditions = append(loadConditions, fmt.Sprintf( + "t.id IN (SELECT id FROM transmissions ORDER BY first_seen DESC LIMIT %d)", maxPackets)) + } + filterClause := "" + if len(loadConditions) > 0 { + filterClause = "\n\t\t\tWHERE " + strings.Join(loadConditions, "\n\t\t\t AND ") } if s.db.isV3 { @@ -477,7 +490,7 @@ func (s *PacketStore) Load() error { o.snr, o.rssi, o.score, o.path_json, strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch')` + obsRawHexCol + rpCol + ` FROM transmissions t LEFT JOIN observations o ON o.transmission_id = t.id - LEFT JOIN observers obs ON obs.rowid = o.observer_idx` + limitClause + ` + LEFT JOIN observers obs ON obs.rowid = o.observer_idx` + filterClause + ` ORDER BY t.first_seen ASC, o.timestamp DESC` } else { loadSQL = `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, @@ -485,7 +498,7 @@ func (s *PacketStore) Load() error { o.id, o.observer_id, o.observer_name, o.direction, o.snr, o.rssi, o.score, o.path_json, o.timestamp` + obsRawHexCol + rpCol + ` FROM transmissions t - LEFT JOIN observations o ON o.transmission_id = t.id` + limitClause + ` + LEFT JOIN observations o ON o.transmission_id = t.id` + filterClause + ` ORDER BY t.first_seen ASC, o.timestamp DESC` } @@ -4517,12 +4530,19 @@ type nodeInfo struct { Lat float64 Lon float64 HasGPS bool + LastSeen time.Time } func (s *PacketStore) getAllNodes() []nodeInfo { - rows, err := s.db.conn.Query("SELECT public_key, name, role, lat, lon FROM nodes") + // Try with last_seen first; fall back to without if column doesn't exist. + rows, err := s.db.conn.Query("SELECT public_key, name, role, lat, lon, last_seen FROM nodes") + hasLastSeen := true if err != nil { - return nil + rows, err = s.db.conn.Query("SELECT public_key, name, role, lat, lon FROM nodes") + hasLastSeen = false + if err != nil { + return nil + } } defer rows.Close() var nodes []nodeInfo @@ -4530,13 +4550,25 @@ func (s *PacketStore) getAllNodes() []nodeInfo { var pk string var name, role sql.NullString var lat, lon sql.NullFloat64 - rows.Scan(&pk, &name, &role, &lat, &lon) + var lastSeen sql.NullString + if hasLastSeen { + rows.Scan(&pk, &name, &role, &lat, &lon, &lastSeen) + } else { + rows.Scan(&pk, &name, &role, &lat, &lon) + } n := nodeInfo{PublicKey: pk, Name: nullStrVal(name), Role: nullStrVal(role)} if lat.Valid && lon.Valid { n.Lat = lat.Float64 n.Lon = lon.Float64 n.HasGPS = !(n.Lat == 0 && n.Lon == 0) } + if hasLastSeen && lastSeen.Valid && lastSeen.String != "" { + if t, err := time.Parse(time.RFC3339, lastSeen.String); err == nil { + n.LastSeen = t + } else if t, err := time.Parse("2006-01-02 15:04:05", lastSeen.String); err == nil { + n.LastSeen = t + } + } nodes = append(nodes, n) } return nodes @@ -4551,9 +4583,20 @@ type prefixMap struct { // entries to ~7Γ—N (+ 1 full-key entry per node for exact-match lookups). const maxPrefixLen = 8 +// canAppearInPath returns true if the node's role allows it to appear as a +// path hop. Only repeaters, room servers, and rooms can forward packets; +// companions and sensors originate but never relay. +func canAppearInPath(role string) bool { + r := strings.ToLower(role) + return strings.Contains(r, "repeater") || strings.Contains(r, "room_server") || r == "room" +} + func buildPrefixMap(nodes []nodeInfo) *prefixMap { pm := &prefixMap{m: make(map[string][]nodeInfo, len(nodes)*(maxPrefixLen+1))} for _, n := range nodes { + if !canAppearInPath(n.Role) { + continue + } pk := strings.ToLower(n.PublicKey) maxLen := maxPrefixLen if maxLen > len(pk) { diff --git a/cmd/server/vacuum.go b/cmd/server/vacuum.go new file mode 100644 index 00000000..a53556a5 --- /dev/null +++ b/cmd/server/vacuum.go @@ -0,0 +1,84 @@ +package main + +import ( + "fmt" + "log" + "time" +) + +// checkAutoVacuum inspects the current auto_vacuum mode and logs a warning +// if it's not INCREMENTAL. Optionally performs a one-time full VACUUM if +// the operator has set db.vacuumOnStartup: true in config (#919). +func checkAutoVacuum(db *DB, cfg *Config, dbPath string) { + var autoVacuum int + if err := db.conn.QueryRow("PRAGMA auto_vacuum").Scan(&autoVacuum); err != nil { + log.Printf("[db] warning: could not read auto_vacuum: %v", err) + return + } + + if autoVacuum == 2 { + log.Printf("[db] auto_vacuum=INCREMENTAL") + return + } + + modes := map[int]string{0: "NONE", 1: "FULL", 2: "INCREMENTAL"} + mode := modes[autoVacuum] + if mode == "" { + mode = fmt.Sprintf("UNKNOWN(%d)", autoVacuum) + } + + log.Printf("[db] auto_vacuum=%s β€” DB needs one-time VACUUM to enable incremental auto-vacuum. "+ + "Set db.vacuumOnStartup: true in config to migrate (will block startup for several minutes on large DBs). "+ + "See https://github.com/Kpa-clawbot/CoreScope/issues/919", mode) + + if cfg.DB != nil && cfg.DB.VacuumOnStartup { + // WARNING: Full VACUUM creates a temporary copy of the entire DB file. + // Requires ~2Γ— the DB file size in free disk space or it will fail. + log.Printf("[db] vacuumOnStartup=true β€” starting one-time full VACUUM (ensure 2x DB size free disk space)...") + start := time.Now() + + rw, err := openRW(dbPath) + if err != nil { + log.Printf("[db] VACUUM failed: could not open RW connection: %v", err) + return + } + defer rw.Close() + + if _, err := rw.Exec("PRAGMA auto_vacuum = INCREMENTAL"); err != nil { + log.Printf("[db] VACUUM failed: could not set auto_vacuum: %v", err) + return + } + if _, err := rw.Exec("VACUUM"); err != nil { + log.Printf("[db] VACUUM failed: %v", err) + return + } + + elapsed := time.Since(start) + log.Printf("[db] VACUUM complete in %v β€” auto_vacuum is now INCREMENTAL", elapsed.Round(time.Millisecond)) + + // Re-check + var newMode int + if err := db.conn.QueryRow("PRAGMA auto_vacuum").Scan(&newMode); err == nil { + if newMode == 2 { + log.Printf("[db] auto_vacuum=INCREMENTAL (confirmed after VACUUM)") + } else { + log.Printf("[db] warning: auto_vacuum=%d after VACUUM β€” expected 2", newMode) + } + } + } +} + +// runIncrementalVacuum runs PRAGMA incremental_vacuum(N) on a read-write +// connection. Safe to call on auto_vacuum=NONE databases (noop). +func runIncrementalVacuum(dbPath string, pages int) { + rw, err := openRW(dbPath) + if err != nil { + log.Printf("[vacuum] could not open RW connection: %v", err) + return + } + defer rw.Close() + + if _, err := rw.Exec(fmt.Sprintf("PRAGMA incremental_vacuum(%d)", pages)); err != nil { + log.Printf("[vacuum] incremental_vacuum error: %v", err) + } +} diff --git a/config.example.json b/config.example.json index 5672ed31..7e8e80a3 100644 --- a/config.example.json +++ b/config.example.json @@ -9,6 +9,11 @@ "packetDays": 30, "_comment": "nodeDays: nodes not seen in N days moved to inactive_nodes (default 7). observerDays: observers not sending data in N days are removed (-1 = keep forever, default 14). packetDays: transmissions older than N days are deleted (0 = disabled)." }, + "db": { + "vacuumOnStartup": false, + "incrementalVacuumPages": 1024, + "_comment": "vacuumOnStartup: run one-time full VACUUM to enable incremental auto-vacuum on existing DBs (blocks startup for minutes on large DBs; requires 2x DB file size in free disk space). incrementalVacuumPages: free pages returned to OS after each retention reaper cycle (default 1024). See #919." + }, "https": { "cert": "/path/to/cert.pem", "key": "/path/to/key.pem", @@ -208,7 +213,8 @@ "packetStore": { "maxMemoryMB": 1024, "estimatedPacketBytes": 450, - "_comment": "In-memory packet store. maxMemoryMB caps RAM usage. All packets loaded on startup, served from RAM." + "retentionHours": 168, + "_comment": "In-memory packet store. maxMemoryMB caps RAM usage. retentionHours: only packets younger than this are loaded on startup and kept in memory (0 = unlimited, not recommended for large DBs β€” causes OOM on cold start). 168 = 7 days. Must be ≀ retention.packetDays * 24." }, "resolvedPath": { "backfillHours": 24, diff --git a/docs/user-guide/configuration.md b/docs/user-guide/configuration.md index eda7910d..7ff59d94 100644 --- a/docs/user-guide/configuration.md +++ b/docs/user-guide/configuration.md @@ -98,6 +98,22 @@ How long (in hours) before a node is marked degraded or silent: | `retention.nodeDays` | `7` | Nodes not seen in N days move to inactive | | `retention.packetDays` | `30` | Packets older than N days are deleted daily | +> **Note:** Lowering retention does **not** immediately shrink the database file. +> SQLite marks deleted pages as free but does not return them to the filesystem +> unless [incremental auto-vacuum](database.md) is enabled. New databases created +> after v0.x.x have auto-vacuum enabled automatically. Existing databases require +> a one-time migration β€” see the [Database](database.md) guide. + +## Database + +| Field | Default | Description | +|-------|---------|-------------| +| `db.vacuumOnStartup` | `false` | Run a one-time full `VACUUM` on startup to enable incremental auto-vacuum (blocks for minutes on large DBs) | +| `db.incrementalVacuumPages` | `1024` | Free pages returned to the OS after each retention reaper cycle | + +See [Database](database.md) for details on SQLite auto-vacuum, WAL, and manual maintenance. +See [#919](https://github.com/Kpa-clawbot/CoreScope/issues/919) for background. + ## Channel decryption | Field | Description | @@ -150,6 +166,9 @@ Lower values = fresher data but more server load. |-------|---------|-------------| | `packetStore.maxMemoryMB` | `1024` | Maximum RAM for in-memory packet store | | `packetStore.estimatedPacketBytes` | `450` | Estimated bytes per packet (for memory budgeting) | +| `packetStore.retentionHours` | `0` | Only load packets younger than N hours on startup and keep them in memory. **Set this on any instance with a large DB.** `0` = unlimited (loads full DB history β€” causes OOM on cold start when the DB has hundreds of thousands of paths). Recommended: same as `retention.packetDays Γ— 24` (e.g. `168` for 7 days). | + +> **Warning:** Leaving `retentionHours` at `0` on a large database will cause the server to OOM-kill itself on every cold start. The full packet history is loaded into the subpath index at startup; a DB with ~280K paths produces ~13M index entries before the process is killed. ## Timestamps diff --git a/docs/user-guide/database.md b/docs/user-guide/database.md new file mode 100644 index 00000000..feaf6c1e --- /dev/null +++ b/docs/user-guide/database.md @@ -0,0 +1,82 @@ +# Database + +CoreScope uses SQLite in WAL (Write-Ahead Log) mode for both the server +(read-only) and ingestor (read-write). + +## WAL mode + +WAL mode allows concurrent reads while writes happen. It is set automatically +at connection time via `PRAGMA journal_mode=WAL`. No operator action needed. + +The WAL file (`meshcore.db-wal`) grows during writes and is checkpointed +(merged back into the main DB) periodically and at clean shutdown. + +## Auto-vacuum + +By default, SQLite does not shrink the database file after `DELETE` operations. +Deleted pages are marked free and reused by future writes, but the file size +on disk stays the same. This is surprising when lowering retention settings. + +### New databases + +Databases created after this feature was added automatically have +`PRAGMA auto_vacuum = INCREMENTAL`. After each retention reaper cycle, +CoreScope runs `PRAGMA incremental_vacuum(N)` to return free pages to the OS. + +### Existing databases + +The `auto_vacuum` mode is stored in the database header and can only be changed +by rewriting the entire file with `VACUUM`. CoreScope will **not** do this +automatically β€” on large databases (5+ GB seen in the wild) it takes minutes +and holds an exclusive lock. + +**To migrate an existing database:** + +1. At startup, CoreScope logs a warning: + ``` + [db] auto_vacuum=NONE β€” DB needs one-time VACUUM to enable incremental auto-vacuum. + ``` +2. **Ensure at least 2Γ— the database file size in free disk space.** Full VACUUM + creates a temporary copy of the entire file β€” on a near-full disk it will fail. +3. Set `db.vacuumOnStartup: true` in your `config.json`: + ```json + { + "db": { + "vacuumOnStartup": true + } + } + ``` +4. Restart CoreScope. The one-time `VACUUM` will run and block startup. +5. After migration, remove or set `vacuumOnStartup: false` β€” it's not needed again. + +### Configuration + +| Field | Default | Description | +|-------|---------|-------------| +| `db.vacuumOnStartup` | `false` | One-time full VACUUM to enable incremental auto-vacuum | +| `db.incrementalVacuumPages` | `1024` | Pages returned to OS per reaper cycle | + +## Manual VACUUM + +You can also run a manual vacuum from the SQLite CLI: + +```bash +sqlite3 data/meshcore.db "PRAGMA auto_vacuum = INCREMENTAL; VACUUM;" +``` + +This is equivalent to `vacuumOnStartup: true` but can be done offline. + +> ⚠️ Full VACUUM requires **2Γ— the database file size** in free disk space (it +> creates a temporary copy). Check with `ls -lh data/meshcore.db` before running. + +## Checking current mode + +```bash +sqlite3 data/meshcore.db "PRAGMA auto_vacuum;" +``` + +- `0` = NONE (default for old databases) +- `1` = FULL (automatic, but slower writes) +- `2` = INCREMENTAL (recommended β€” CoreScope triggers vacuum after deletes) + +See [#919](https://github.com/Kpa-clawbot/CoreScope/issues/919) for background on this feature. diff --git a/public/app.js b/public/app.js index 2e21e8fe..17a98c00 100644 --- a/public/app.js +++ b/public/app.js @@ -505,6 +505,21 @@ const pages = {}; function registerPage(name, mod) { pages[name] = mod; } +// Tools landing page β€” shows sub-menu with Trace and Path Inspector (spec Β§2.8, M1 fix). +registerPage('tools-landing', { + init: function (container) { + container.innerHTML = + '
' + + '

Tools

' + + '
' + + '

πŸ” Path Inspector

Resolve prefix paths to candidate full-pubkey routes with confidence scoring.

' + + '

πŸ“‘ Trace Viewer

View detailed packet traces by hash.

' + + '
' + + '
'; + }, + destroy: function () {} +}); + let currentPage = null; function closeNav() { @@ -525,6 +540,12 @@ function closeMoreMenu() { function navigate() { closeNav(); + // Backward-compat redirect: #/traces/ β†’ #/tools/trace/ (issue #944). + if (location.hash.startsWith('#/traces/')) { + location.hash = location.hash.replace('#/traces/', '#/tools/trace/'); + return; + } + const hash = location.hash.replace('#/', '') || 'packets'; const route = hash.split('?')[0]; @@ -552,9 +573,27 @@ function navigate() { basePage = 'observer-detail'; } + // Tools sub-routing (issue #944): tools/trace/, tools/path-inspector + if (basePage === 'tools') { + if (routeParam && routeParam.startsWith('trace/')) { + basePage = 'traces'; + routeParam = routeParam.substring(6); // strip "trace/" + } else if (routeParam === 'path-inspector' || (routeParam && routeParam.startsWith('path-inspector'))) { + basePage = 'path-inspector'; + routeParam = null; + } else if (!routeParam) { + // Default tools landing shows menu with both entries. + basePage = 'tools-landing'; + } + } + // Also support old #/traces (no sub-path) β†’ traces page. + if (basePage === 'traces' && !routeParam) { + basePage = 'traces'; + } + // Update nav active state document.querySelectorAll('.nav-link[data-route]').forEach(el => { - el.classList.toggle('active', el.dataset.route === basePage); + el.classList.toggle('active', el.dataset.route === basePage || (el.dataset.route === 'tools' && (basePage === 'traces' || basePage === 'path-inspector' || basePage === 'tools-landing'))); }); // Update "More" button to show active state if a low-priority page is selected var moreBtn = document.getElementById('navMoreBtn'); diff --git a/public/customize-v2.js b/public/customize-v2.js index 20756957..86295463 100644 --- a/public/customize-v2.js +++ b/public/customize-v2.js @@ -629,7 +629,11 @@ } writeOverrides(delta); _runPipeline(); - _refreshPanel(); + // Skip re-render while the user is typing inside the panel β€” setting + // innerHTML would destroy the focused input and collapse the mobile keyboard. + if (!(_panelEl && _panelEl.contains(document.activeElement))) { + _refreshPanel(); + } }, 300); } diff --git a/public/geofilter-builder.html b/public/geofilter-builder.html index c16077e8..1441d9c2 100644 --- a/public/geofilter-builder.html +++ b/public/geofilter-builder.html @@ -87,7 +87,8 @@ let polygon = null; let closingLine = null; function latLonPair(latlng) { - return [parseFloat(latlng.lat.toFixed(6)), parseFloat(latlng.lng.toFixed(6))]; + const w = latlng.wrap(); + return [parseFloat(w.lat.toFixed(6)), parseFloat(w.lng.toFixed(6))]; } function render() { diff --git a/public/hop-resolver.js b/public/hop-resolver.js index 803eb8a4..7c78906c 100644 --- a/public/hop-resolver.js +++ b/public/hop-resolver.js @@ -7,6 +7,14 @@ window.HopResolver = (function() { const MAX_HOP_DIST = 1.8; // ~200km in degrees const REGION_RADIUS_KM = 300; + + // Only repeaters and room servers can appear as path hops per protocol. + // Companions/sensors originate but never relay packets. + function canAppearInPath(role) { + if (!role) return false; + var r = String(role).toLowerCase(); + return r.indexOf('repeater') >= 0 || r.indexOf('room_server') >= 0 || r === 'room'; + } let prefixIdx = {}; // lowercase hex prefix β†’ [node, ...] let pubkeyIdx = {}; // full lowercase pubkey β†’ node (O(1) lookup) let nodesList = []; @@ -40,7 +48,11 @@ window.HopResolver = (function() { for (const n of nodesList) { if (!n.public_key) continue; const pk = n.public_key.toLowerCase(); + // pubkeyIdx includes ALL nodes β€” used by resolveFromServer for + // server-confirmed full-pubkey lookups (any node type). pubkeyIdx[pk] = n; + // prefixIdx only includes nodes that can appear as path hops. + if (!canAppearInPath(n.role)) continue; for (let len = 1; len <= 3; len++) { const p = pk.slice(0, len * 2); if (!prefixIdx[p]) prefixIdx[p] = []; diff --git a/public/index.html b/public/index.html index 1187e0ce..919d9d2a 100644 --- a/public/index.html +++ b/public/index.html @@ -50,7 +50,7 @@ πŸ”΄ Live Channels Nodes - Traces + Tools Observers Analytics ⚑ Perf @@ -105,6 +105,7 @@ + diff --git a/public/map.js b/public/map.js index e3958a21..8cc36982 100644 --- a/public/map.js +++ b/public/map.js @@ -102,8 +102,21 @@ async function init(container) { container.innerHTML = ` -
-
+
+
+
+
β—€
+
+

Path Inspector

+

Hex prefixes (1-3 bytes), comma or space separated.

+
+ + +
+
+
+
+

πŸ—ΊοΈ Map Controls

@@ -553,6 +566,19 @@ } } + // Check for pending path inspector route (cross-page navigation from Path Inspector). + if (window._pendingPathInspectorRoute) { + var pending = window._pendingPathInspectorRoute; + delete window._pendingPathInspectorRoute; + if (pending.path && pending.path.length > 0) { + if (window.routeLayer) window.routeLayer.clearLayers(); + drawPacketRoute(pending.path.slice(1), pending.path[0]); + } + } + + // Wire up map side pane (Path Inspector embedded - spec Β§2.7). + initMapSidePane(); + // Don't fitBounds on initial load β€” respect the Bay Area default or saved view // Only fitBounds on subsequent data refreshes if user hasn't manually panned } catch (e) { @@ -981,6 +1007,122 @@ map.fitBounds(bounds, { padding: [50, 50], maxZoom: 14 }); } + // === Map Side Pane β€” Path Inspector (spec Β§2.7) === + function initMapSidePane() { + var pane = document.getElementById('mapSidePane'); + var toggle = document.getElementById('mapPaneToggle'); + var input = document.getElementById('mapPiInput'); + var btn = document.getElementById('mapPiSubmit'); + if (!pane || !toggle) return; + + toggle.addEventListener('click', function () { + pane.classList.toggle('expanded'); + toggle.textContent = pane.classList.contains('expanded') ? 'β–Ά' : 'β—€'; + // Invalidate map size after transition. + setTimeout(function () { if (map) map.invalidateSize(); }, 220); + }); + + if (btn && input) { + btn.addEventListener('click', function () { mapPiSubmit(input.value); }); + input.addEventListener('keydown', function (e) { + if (e.key === 'Enter') mapPiSubmit(input.value); + }); + } + + // Auto-open if URL has prefixes param while on map. + var params = new URLSearchParams(location.hash.split('?')[1] || ''); + var prefixParam = params.get('prefixes'); + if (prefixParam && input) { + pane.classList.add('expanded'); + toggle.textContent = 'β–Ά'; + input.value = prefixParam; + setTimeout(function () { if (map) map.invalidateSize(); }, 220); + mapPiSubmit(prefixParam); + } + } + + function mapPiSubmit(raw) { + var errDiv = document.getElementById('mapPiError'); + var resultsDiv = document.getElementById('mapPiResults'); + if (!errDiv || !resultsDiv) return; + errDiv.textContent = ''; + resultsDiv.innerHTML = ''; + + // Reuse PathInspector validation if available. + var prefixes = raw.trim().split(/[\s,]+/).filter(function (s) { return s.length > 0; }).map(function (s) { return s.toLowerCase(); }); + var err = (window.PathInspector && window.PathInspector.validatePrefixes) ? window.PathInspector.validatePrefixes(prefixes) : null; + if (!err && prefixes.length === 0) err = 'Enter at least one prefix.'; + if (err) { errDiv.textContent = err; return; } + + resultsDiv.innerHTML = '

Loading...

'; + fetch('/api/paths/inspect', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ prefixes: prefixes }) + }) + .then(function (r) { + if (r.status === 503) return r.json().then(function () { throw new Error('Service warming up, retry shortly.'); }); + if (!r.ok) return r.json().then(function (d) { throw new Error(d.error || 'Request failed'); }); + return r.json(); + }) + .then(function (data) { renderMapPiResults(data, resultsDiv); }) + .catch(function (e) { resultsDiv.innerHTML = ''; errDiv.textContent = e.message; }); + } + + function renderMapPiResults(data, div) { + if (!data.candidates || data.candidates.length === 0) { + div.innerHTML = '

No candidates found.

'; + return; + } + var html = ''; + for (var i = 0; i < data.candidates.length; i++) { + var c = data.candidates[i]; + var rowClass = c.speculative ? 'speculative-row' : ''; + html += ''; + html += ''; + html += ''; + html += ''; + html += ''; + html += ''; + // Per-hop evidence (collapsed). + html += ''; + } + html += '
#ScorePath
' + (i + 1) + '' + c.score.toFixed(2) + (c.speculative ? ' ⚠' : '') + '' + safeEsc(c.names.slice(0, 3).join('β†’')) + (c.names.length > 3 ? '…' : '') + '
'; + div.innerHTML = html; + + // Wire buttons. + div.querySelectorAll('button[data-idx]').forEach(function (btn) { + btn.addEventListener('click', function () { + var idx = parseInt(btn.dataset.idx); + var cand = data.candidates[idx]; + if (routeLayer) routeLayer.clearLayers(); + drawPacketRoute(cand.path.slice(1), cand.path[0]); + }); + }); + // Expand evidence on row click. + div.querySelectorAll('.path-inspector-table tbody tr:not(.evidence-row)').forEach(function (row) { + row.style.cursor = 'pointer'; + row.addEventListener('click', function (e) { + if (e.target.tagName === 'BUTTON') return; + var b = row.querySelector('button[data-idx]'); + if (!b) return; + var ev = div.querySelector('tr[data-evidence="' + b.dataset.idx + '"]'); + if (ev) ev.classList.toggle('collapsed'); + }); + }); + } + function destroy() { if (wsHandler) offWS(wsHandler); wsHandler = null; diff --git a/public/packets.js b/public/packets.js index f0b2db8a..a169461e 100644 --- a/public/packets.js +++ b/public/packets.js @@ -468,6 +468,9 @@ // Check if new packets pass current filters const filtered = newPkts.filter(p => { + // When user pinned a hash, accept ONLY that exact packet β€” bypass all + // other filters (window/region/type/observer/node). + if (filters.hash) return p.hash === filters.hash; // Respect time window filter β€” drop packets outside the selected window const windowMin = savedTimeWindowMin; if (windowMin > 0) { @@ -477,7 +480,6 @@ } if (filters.type) { const types = filters.type.split(',').map(Number); if (!types.includes(p.payload_type)) return false; } if (filters.observer) { const obsSet = new Set(filters.observer.split(',')); if (!obsSet.has(p.observer_id) && !(p._children && p._children.some(c => obsSet.has(String(c.observer_id))))) return false; } - if (filters.hash && p.hash !== filters.hash) return false; if (RegionFilter.getRegionParam()) { const selectedRegions = RegionFilter.getRegionParam().split(','); const obs = observerMap.get(p.observer_id); @@ -610,27 +612,52 @@ } catch {} } - async function loadPackets() { - try { - const params = new URLSearchParams(); - const selectedWindow = Number(document.getElementById('fTimeWindow')?.value); - const windowMin = Number.isFinite(selectedWindow) ? selectedWindow : savedTimeWindowMin; - if (windowMin > 0 && !filters.hash) { - const since = new Date(Date.now() - windowMin * 60000).toISOString(); - params.set('since', since); - } - params.set('limit', String(PACKET_LIMIT)); - const regionParam = RegionFilter.getRegionParam(); - if (regionParam) params.set('region', regionParam); - if (filters.hash) params.set('hash', filters.hash); - if (filters.node) params.set('node', filters.node); - if (filters.observer) params.set('observer', filters.observer); - if (filters.channel) params.set('channel', filters.channel); + // Build URLSearchParams for /api/packets given UI state. Pure function for + // testability β€” returns the params object the next call to /api/packets + // would use. The hash filter is an exact identifier: when present it + // suppresses ALL other filters (region, time window, observer, node, + // channel). The user is asking for THAT packet regardless of saved + // selections. + function buildPacketsParams({ filters, regionParam, windowMin, groupByHash, limit }) { + const params = new URLSearchParams(); + if (filters.hash) { + params.set('hash', filters.hash); + params.set('limit', String(limit)); if (groupByHash) { params.set('groupByHash', 'true'); } else { params.set('expand', 'observations'); } + return params; + } + if (windowMin > 0) { + const since = new Date(Date.now() - windowMin * 60000).toISOString(); + params.set('since', since); + } + params.set('limit', String(limit)); + if (regionParam) params.set('region', regionParam); + if (filters.node) params.set('node', filters.node); + if (filters.observer) params.set('observer', filters.observer); + if (filters.channel) params.set('channel', filters.channel); + if (groupByHash) { + params.set('groupByHash', 'true'); + } else { + params.set('expand', 'observations'); + } + return params; + } + + async function loadPackets() { + try { + const selectedWindow = Number(document.getElementById('fTimeWindow')?.value); + const windowMin = Number.isFinite(selectedWindow) ? selectedWindow : savedTimeWindowMin; + const params = buildPacketsParams({ + filters, + regionParam: RegionFilter.getRegionParam(), + windowMin, + groupByHash, + limit: PACKET_LIMIT, + }); const data = await api('/packets?' + params.toString()); packets = data.packets || []; @@ -1647,7 +1674,14 @@ // Filter to claimed/favorited nodes β€” pure client-side filter (no server round-trip) let displayPackets = packets; - if (filters.myNodes) { + + // When loading a specific packet by hash, bypass ALL client-side filters + // (myNodes, type, observer, packet-filter-expression). The user is asking + // for THAT exact packet β€” saved type/observer/expression filters must not + // hide it. Hash filter is the exact identifier; nothing else applies. + const hashOnly = !!filters.hash; + + if (!hashOnly && filters.myNodes) { const myNodes = JSON.parse(localStorage.getItem('meshcore-my-nodes') || '[]'); const myKeys = myNodes.map(n => n.pubkey).filter(Boolean); const favs = getFavorites(); @@ -1663,11 +1697,11 @@ } // Client-side type/observer filtering - if (filters.type) { + if (!hashOnly && filters.type) { const types = filters.type.split(',').map(Number); displayPackets = displayPackets.filter(p => types.includes(p.payload_type)); } - if (filters.observer) { + if (!hashOnly && filters.observer) { const obsIds = new Set(filters.observer.split(',')); displayPackets = displayPackets.filter(p => { if (obsIds.has(p.observer_id)) return true; @@ -1678,7 +1712,7 @@ // Packet Filter Language const pfCount = document.getElementById('packetFilterCount'); - if (filters._packetFilter) { + if (!hashOnly && filters._packetFilter) { const beforeCount = displayPackets.length; displayPackets = displayPackets.filter(filters._packetFilter); if (pfCount) { @@ -2563,6 +2597,7 @@ buildGroupRowHtml, buildFlatRowHtml, _calcVisibleRange, + buildPacketsParams, }; } diff --git a/public/path-inspector.js b/public/path-inspector.js new file mode 100644 index 00000000..e1e940af --- /dev/null +++ b/public/path-inspector.js @@ -0,0 +1,202 @@ +// Path Inspector β€” prefix candidate scoring with map overlay (issue #944). +// IIFE; exports window.PathInspector for testability. +(function () { + 'use strict'; + + var container = null; + var currentResults = null; + + function init(app) { + container = app; + var params = new URLSearchParams(location.hash.split('?')[1] || ''); + var prefixParam = params.get('prefixes') || ''; + + container.innerHTML = + '
' + + '

Path Inspector

' + + '

Enter comma or space-separated hex prefixes (1-3 bytes each, e.g. 2C,A1,F4 or 2C A1 F4).

' + + '
' + + '' + + '' + + '
' + + '
' + + '
' + + '
'; + + var input = document.getElementById('path-inspector-input'); + var btn = document.getElementById('path-inspector-submit'); + btn.addEventListener('click', function () { submit(input.value); }); + input.addEventListener('keydown', function (e) { + if (e.key === 'Enter') submit(input.value); + }); + + // Auto-run if prefixes in URL. + if (prefixParam) submit(prefixParam); + } + + function destroy() { + container = null; + currentResults = null; + } + + function parsePrefixes(raw) { + // Accept comma or space separated. + var parts = raw.trim().split(/[\s,]+/).filter(function (s) { return s.length > 0; }); + return parts.map(function (p) { return p.toLowerCase(); }); + } + + function validatePrefixes(prefixes) { + if (prefixes.length === 0) return 'Enter at least one prefix.'; + if (prefixes.length > 64) return 'Too many prefixes (max 64).'; + var hexRe = /^[0-9a-f]+$/; + var byteLen = -1; + for (var i = 0; i < prefixes.length; i++) { + var p = prefixes[i]; + if (!hexRe.test(p)) return 'Invalid hex: ' + p; + if (p.length % 2 !== 0) return 'Odd-length prefix: ' + p; + var bl = p.length / 2; + if (bl > 3) return 'Prefix too long (max 3 bytes): ' + p; + if (byteLen === -1) byteLen = bl; + else if (bl !== byteLen) return 'Mixed prefix lengths not allowed.'; + } + return null; + } + + function submit(raw) { + var errDiv = document.getElementById('path-inspector-error'); + var resultsDiv = document.getElementById('path-inspector-results'); + errDiv.textContent = ''; + resultsDiv.innerHTML = ''; + + var prefixes = parsePrefixes(raw); + var err = validatePrefixes(prefixes); + if (err) { + errDiv.textContent = err; + return; + } + + // Update URL. + var base = '#/tools/path-inspector'; + if (location.hash.indexOf(base) === 0) { + history.replaceState(null, '', base + '?prefixes=' + prefixes.join(',')); + } + + resultsDiv.innerHTML = '

Loading...

'; + fetch('/api/paths/inspect', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ prefixes: prefixes }) + }) + .then(function (r) { + if (r.status === 503) return r.json().then(function (d) { throw new Error('Service warming up, retry in a few seconds.'); }); + if (!r.ok) return r.json().then(function (d) { throw new Error(d.error || 'Request failed'); }); + return r.json(); + }) + .then(function (data) { + currentResults = data; + renderResults(data, resultsDiv); + }) + .catch(function (e) { + resultsDiv.innerHTML = ''; + errDiv.textContent = e.message; + }); + } + + function renderResults(data, div) { + if (!data.candidates || data.candidates.length === 0) { + div.innerHTML = '

No candidates found. The prefixes may not match any known path-eligible nodes.

'; + return; + } + + var html = '' + + '' + + ''; + + for (var i = 0; i < data.candidates.length; i++) { + var c = data.candidates[i]; + var rowClass = c.speculative ? 'speculative-row' : ''; + html += ''; + html += ''; + html += ''; + html += ''; + html += ''; + html += ''; + + // Per-hop evidence (collapsed). + html += ''; + } + + html += '
#ScorePathAction
' + (i + 1) + '' + + c.score.toFixed(3) + + (c.speculative ? ' ⚠' : '') + + '' + escapeHtml(c.names.join(' β†’ ')) + '
'; + html += '
Beam width: ' + data.stats.beamWidth + + ' | Expansions: ' + data.stats.expansionsRun + + ' | Elapsed: ' + data.stats.elapsedMs + 'ms
'; + + div.innerHTML = html; + + // Wire up Show on Map buttons. + div.querySelectorAll('button[data-idx]').forEach(function (btn) { + btn.addEventListener('click', function () { + var idx = parseInt(btn.dataset.idx); + showOnMap(data.candidates[idx]); + }); + }); + + // Wire up row expand for evidence. + div.querySelectorAll('.path-inspector-table tbody tr:not(.evidence-row)').forEach(function (row) { + row.style.cursor = 'pointer'; + row.addEventListener('click', function (e) { + if (e.target.tagName === 'BUTTON') return; + var idx = row.querySelector('button[data-idx]'); + if (!idx) return; + var evidenceRow = div.querySelector('tr[data-evidence="' + idx.dataset.idx + '"]'); + if (evidenceRow) evidenceRow.classList.toggle('collapsed'); + }); + }); + } + + function showOnMap(candidate) { + // Store pending route for map init to pick up. + window._pendingPathInspectorRoute = candidate; + // Switch to map page if not there; map init will draw the route. + if (location.hash.indexOf('#/map') !== 0) { + location.hash = '#/map'; + } else { + // Already on map β€” draw directly. + delete window._pendingPathInspectorRoute; + if (window.routeLayer) window.routeLayer.clearLayers(); + var hops = candidate.path.slice(1); + var origin = candidate.path[0] || null; + if (window.drawPacketRoute) window.drawPacketRoute(hops, origin); + } + } + + function escapeAttr(s) { + return s.replace(/&/g, '&').replace(/"/g, '"').replace(//g, '>'); + } + + window.PathInspector = { init: init, destroy: destroy, parsePrefixes: parsePrefixes, validatePrefixes: validatePrefixes }; + if (typeof registerPage === 'function') registerPage('path-inspector', { init: init, destroy: destroy }); +})(); diff --git a/public/style.css b/public/style.css index b07a458f..b5a8ed90 100644 --- a/public/style.css +++ b/public/style.css @@ -16,6 +16,7 @@ --status-amber: #f59e0b; --status-amber-light: #fef3c7; --status-amber-text: #92400e; + --path-inspector-speculative: #d97706; --role-observer: #8b5cf6; --accent-hover: #6db3ff; --text: #1a1a2e; @@ -52,6 +53,7 @@ --status-amber: #f59e0b; --status-amber-light: #422006; --status-amber-text: #fcd34d; + --path-inspector-speculative: #f59e0b; --surface-0: #0f0f23; --surface-1: #1a1a2e; --surface-2: #232340; @@ -2309,3 +2311,37 @@ th.sort-active { color: var(--accent, #60a5fa); } .clock-filter-btn { font-size: 12px; padding: 3px 8px; border: 1px solid var(--border); border-radius: 4px; background: var(--card-bg, #fff); color: var(--text); cursor: pointer; margin-right: 4px; } .clock-filter-btn.active { background: var(--accent); color: #fff; border-color: var(--accent); } + +/* === Path Inspector (issue #944) === */ +.path-inspector-page { padding: 16px; max-width: 900px; margin: 0 auto; } +.path-inspector-input-row { display: flex; gap: 8px; margin-bottom: 12px; } +.path-inspector-input-row .input { flex: 1; } +.path-inspector-error { color: var(--status-red, #ef4444); font-size: 13px; margin-bottom: 8px; } +.path-inspector-table { width: 100%; border-collapse: collapse; font-size: 13px; } +.path-inspector-table th, +.path-inspector-table td { padding: 6px 10px; border-bottom: 1px solid var(--border); text-align: left; } +.path-inspector-table th { background: var(--card-bg); font-weight: 600; } +.speculative-warning { color: var(--path-inspector-speculative, #d97706); font-weight: 600; } +.speculative-badge { cursor: help; } +.speculative-row { background: color-mix(in srgb, var(--path-inspector-speculative, #d97706) 8%, transparent); } +.evidence-row { font-size: 12px; color: var(--text-muted); } +.evidence-row.collapsed { display: none; } +.evidence-detail { padding: 4px 10px; } +.hop-evidence { margin: 2px 0; } +.path-inspector-stats { margin-top: 12px; font-size: 12px; color: var(--text-muted); } +.no-results { color: var(--text-muted); font-style: italic; } + +/* Map side pane for path inspector */ +.map-side-pane { flex: 0 0 32px; overflow: hidden; transition: flex-basis 0.2s; border-left: 1px solid var(--border); background: var(--card-bg); } +.map-side-pane.expanded { flex: 0 0 320px; overflow-y: auto; padding: 12px; } +.map-side-pane .pane-toggle { cursor: pointer; padding: 8px; font-size: 14px; text-align: center; } +.map-side-pane .pane-content { display: none; } +.map-side-pane.expanded .pane-content { display: block; } + +/* Tools landing page */ +.tools-landing { padding: 24px; max-width: 600px; } +.tools-menu { display: flex; flex-direction: column; gap: 12px; margin-top: 16px; } +.tools-card { display: block; padding: 16px; border-radius: 8px; border: 1px solid var(--border); background: var(--card-bg); color: var(--text); text-decoration: none; transition: border-color 0.2s; } +.tools-card:hover { border-color: var(--primary); } +.tools-card h3 { margin: 0 0 4px 0; font-size: 16px; } +.tools-card p { margin: 0; font-size: 13px; color: var(--text-muted); } diff --git a/test-aging.js b/test-aging.js index eb8801bf..80e8476c 100644 --- a/test-aging.js +++ b/test-aging.js @@ -59,118 +59,7 @@ test('null lastSeenMs β†’ stale', () => assert.strictEqual(getNodeStatus('repeat test('undefined lastSeenMs β†’ stale', () => assert.strictEqual(getNodeStatus('repeater', undefined), 'stale')); test('0 lastSeenMs β†’ stale', () => assert.strictEqual(getNodeStatus('repeater', 0), 'stale')); -// === getStatusInfo tests (inline since nodes.js has too many DOM deps) === -console.log('\n=== getStatusInfo (logic validation) ==='); -// Simulate getStatusInfo logic -function mockGetStatusInfo(n) { - const ROLE_COLORS = ctx.window.ROLE_COLORS; - const role = (n.role || '').toLowerCase(); - const roleColor = ROLE_COLORS[n.role] || '#6b7280'; - const lastHeardTime = n._lastHeard || n.last_heard || n.last_seen; - const lastHeardMs = lastHeardTime ? new Date(lastHeardTime).getTime() : 0; - const status = getNodeStatus(role, lastHeardMs); - const statusLabel = status === 'active' ? '🟒 Active' : 'βšͺ Stale'; - const isInfra = role === 'repeater' || role === 'room'; - - let explanation = ''; - if (status === 'active') { - explanation = 'Last heard recently'; - } else { - const reason = isInfra - ? 'repeaters typically advertise every 12-24h' - : 'companions only advertise when user initiates, this may be normal'; - explanation = 'Not heard β€” ' + reason; - } - return { status, statusLabel, roleColor, explanation, role }; -} - -test('active repeater β†’ 🟒 Active, red color', () => { - const info = mockGetStatusInfo({ role: 'repeater', last_seen: new Date(now - 1*h).toISOString() }); - assert.strictEqual(info.status, 'active'); - assert.strictEqual(info.statusLabel, '🟒 Active'); - assert.strictEqual(info.roleColor, '#dc2626'); -}); - -test('stale companion β†’ βšͺ Stale, explanation mentions "this may be normal"', () => { - const info = mockGetStatusInfo({ role: 'companion', last_seen: new Date(now - 25*h).toISOString() }); - assert.strictEqual(info.status, 'stale'); - assert.strictEqual(info.statusLabel, 'βšͺ Stale'); - assert(info.explanation.includes('this may be normal'), 'should mention "this may be normal"'); -}); - -test('missing last_seen β†’ stale', () => { - const info = mockGetStatusInfo({ role: 'repeater' }); - assert.strictEqual(info.status, 'stale'); -}); - -test('missing role β†’ defaults to empty string, uses node threshold', () => { - const info = mockGetStatusInfo({ last_seen: new Date(now - 25*h).toISOString() }); - assert.strictEqual(info.status, 'stale'); - assert.strictEqual(info.roleColor, '#6b7280'); -}); - -test('prefers last_heard over last_seen', () => { - // last_seen is stale, but last_heard is recent - const info = mockGetStatusInfo({ - role: 'companion', - last_seen: new Date(now - 48*h).toISOString(), - last_heard: new Date(now - 1*h).toISOString() - }); - assert.strictEqual(info.status, 'active'); -}); - -// === getStatusTooltip tests === -console.log('\n=== getStatusTooltip ==='); - -// Load from nodes.js by extracting the function -// Since nodes.js is complex, I'll re-implement the tooltip function for testing -function getStatusTooltip(role, status) { - const isInfra = role === 'repeater' || role === 'room'; - const threshold = isInfra ? '72h' : '24h'; - if (status === 'active') { - return 'Active β€” heard within the last ' + threshold + '.' + (isInfra ? ' Repeaters typically advertise every 12-24h.' : ''); - } - if (role === 'companion') { - return 'Stale β€” not heard for over ' + threshold + '. Companions only advertise when the user initiates β€” this may be normal.'; - } - if (role === 'sensor') { - return 'Stale β€” not heard for over ' + threshold + '. This sensor may be offline.'; - } - return 'Stale β€” not heard for over ' + threshold + '. This ' + role + ' may be offline or out of range.'; -} - -test('active repeater mentions "72h" and "advertise every 12-24h"', () => { - const tip = getStatusTooltip('repeater', 'active'); - assert(tip.includes('72h'), 'should mention 72h'); - assert(tip.includes('advertise every 12-24h'), 'should mention advertise frequency'); -}); - -test('active companion mentions "24h"', () => { - const tip = getStatusTooltip('companion', 'active'); - assert(tip.includes('24h'), 'should mention 24h'); -}); - -test('stale companion mentions "24h" and "user initiates"', () => { - const tip = getStatusTooltip('companion', 'stale'); - assert(tip.includes('24h'), 'should mention 24h'); - assert(tip.includes('user initiates'), 'should mention user initiates'); -}); - -test('stale repeater mentions "offline or out of range"', () => { - const tip = getStatusTooltip('repeater', 'stale'); - assert(tip.includes('offline or out of range'), 'should mention offline or out of range'); -}); - -test('stale sensor mentions "sensor may be offline"', () => { - const tip = getStatusTooltip('sensor', 'stale'); - assert(tip.includes('sensor may be offline')); -}); - -test('stale room uses 72h threshold', () => { - const tip = getStatusTooltip('room', 'stale'); - assert(tip.includes('72h')); -}); // === Bug check: renderRows uses last_seen instead of last_heard || last_seen === console.log('\n=== BUG CHECK ==='); diff --git a/test-anim-perf.js b/test-anim-perf.js deleted file mode 100644 index edc3e6d4..00000000 --- a/test-anim-perf.js +++ /dev/null @@ -1,123 +0,0 @@ -/** - * test-anim-perf.js β€” Performance benchmark for animation timer management - * - * Demonstrates that the rAF + concurrency-cap approach keeps active animation - * count bounded, whereas the old setInterval approach accumulated without limit. - * - * Run: node test-anim-perf.js - */ - -'use strict'; - -let passed = 0, failed = 0; -function assert(cond, msg) { - if (cond) { console.log(` βœ… ${msg}`); passed++; } - else { console.log(` ❌ ${msg}`); failed++; } -} - -// --------------------------------------------------------------------------- -// Simulate OLD behaviour: setInterval-based, no concurrency cap -// --------------------------------------------------------------------------- -function simulateOldModel(packetsPerSec, hopsPerPacket, durationSec) { - // Each hop spawns 3 intervals (pulse 26ms, line 33ms, fade 52ms). - // Pulse lasts ~2s, line ~0.66s, fade ~0.8s+0.4s β‰ˆ 1.2s - // At any moment, timers from the last ~2s of packets are still alive. - const intervalLifetimes = [2.0, 0.66, 1.2]; // seconds each interval lives - let maxConcurrent = 0; - // Walk through time in 0.1s steps - const dt = 0.1; - const spawns = []; // {time, lifetime} - for (let t = 0; t < durationSec; t += dt) { - // Spawn timers for packets arriving in this window - const pktsInWindow = packetsPerSec * dt; - for (let p = 0; p < pktsInWindow; p++) { - for (let h = 0; h < hopsPerPacket; h++) { - for (const lt of intervalLifetimes) { - spawns.push({ time: t, lifetime: lt }); - } - } - } - // Count alive timers - const alive = spawns.filter(s => t < s.time + s.lifetime).length; - if (alive > maxConcurrent) maxConcurrent = alive; - } - return maxConcurrent; -} - -// --------------------------------------------------------------------------- -// Simulate NEW behaviour: rAF + MAX_CONCURRENT_ANIMS cap -// --------------------------------------------------------------------------- -function simulateNewModel(packetsPerSec, hopsPerPacket, durationSec) { - const MAX_CONCURRENT_ANIMS = 20; - let activeAnims = 0; - let maxConcurrent = 0; - const anims = []; // {endTime} - const dt = 0.1; - for (let t = 0; t < durationSec; t += dt) { - // Expire finished animations - while (anims.length && anims[0].endTime <= t) { - anims.shift(); - activeAnims--; - } - // Try to start new animations - const pktsInWindow = packetsPerSec * dt; - for (let p = 0; p < pktsInWindow; p++) { - if (activeAnims >= MAX_CONCURRENT_ANIMS) break; // cap reached β€” drop - activeAnims++; - // rAF animation lifetime: longest is pulse ~2s - anims.push({ endTime: t + 2.0 }); - } - // Sort by endTime so expiry works - anims.sort((a, b) => a.endTime - b.endTime); - if (activeAnims > maxConcurrent) maxConcurrent = activeAnims; - } - return maxConcurrent; -} - -// --------------------------------------------------------------------------- -// Tests -// --------------------------------------------------------------------------- - -console.log('\n=== Animation timer accumulation: old vs new ==='); - -// Scenario: 5 pkts/sec, 3 hops each, 30 seconds -const oldPeak30s = simulateOldModel(5, 3, 30); -const newPeak30s = simulateNewModel(5, 3, 30); -console.log(` Old model (30s @ 5pkt/sΓ—3hops): peak ${oldPeak30s} concurrent timers`); -console.log(` New model (30s @ 5pkt/sΓ—3hops): peak ${newPeak30s} concurrent animations`); -assert(oldPeak30s > 100, `old model accumulates >100 timers (got ${oldPeak30s})`); -assert(newPeak30s <= 20, `new model stays ≀20 (got ${newPeak30s})`); - -// Scenario: 5 minutes sustained -const oldPeak5m = simulateOldModel(5, 3, 300); -const newPeak5m = simulateNewModel(5, 3, 300); -console.log(` Old model (5min @ 5pkt/sΓ—3hops): peak ${oldPeak5m} concurrent timers`); -console.log(` New model (5min @ 5pkt/sΓ—3hops): peak ${newPeak5m} concurrent animations`); -assert(oldPeak5m > 100, `old model at 5min still unbounded (got ${oldPeak5m})`); -assert(newPeak5m <= 20, `new model at 5min still ≀20 (got ${newPeak5m})`); - -// Scenario: burst β€” 20 pkts/sec for 10s -const oldBurst = simulateOldModel(20, 3, 10); -const newBurst = simulateNewModel(20, 3, 10); -console.log(` Old model (burst 20pkt/sΓ—3hops, 10s): peak ${oldBurst} concurrent timers`); -console.log(` New model (burst 20pkt/sΓ—3hops, 10s): peak ${newBurst} concurrent animations`); -assert(oldBurst > 200, `old model under burst >200 timers (got ${oldBurst})`); -assert(newBurst <= 20, `new model under burst stays ≀20 (got ${newBurst})`); - -console.log('\n=== drawAnimatedLine frame-drop catch-up ==='); - -// Read the source and verify catch-up logic exists -const fs = require('fs'); -const src = fs.readFileSync(__dirname + '/public/live.js', 'utf8'); - -// Extract the animateLine function body -const lineMatch = src.match(/function animateLine\(now\)\s*\{[\s\S]*?requestAnimationFrame\(animateLine\)/); -assert(lineMatch && /Math\.min\(Math\.floor\(elapsed\s*\/\s*33\)/.test(lineMatch[0]), - 'drawAnimatedLine catches up on frame drops (multi-tick per frame)'); - -const fadeMatch = src.match(/function animateFade\(now\)\s*\{[\s\S]*?requestAnimationFrame\(animateFade\)/); -assert(fadeMatch && /Math\.min\(Math\.floor\(fadeElapsed\s*\/\s*52\)/.test(fadeMatch[0]), - 'animateFade catches up on frame drops (multi-tick per frame)'); - -console.log(`\n${passed} passed, ${failed} failed\n`); -process.exit(failed ? 1 : 0); diff --git a/test-channel-add-ux.js b/test-channel-add-ux.js deleted file mode 100644 index 1de59009..00000000 --- a/test-channel-add-ux.js +++ /dev/null @@ -1,64 +0,0 @@ -/** - * Tests for #759 β€” Add channel UX: button, hint, status feedback. - * Validates the HTML structure rendered by channels.js init. - */ -'use strict'; - -const fs = require('fs'); - -let passed = 0; -let failed = 0; - -function assert(cond, msg) { - if (cond) { passed++; console.log(' βœ“ ' + msg); } - else { failed++; console.error(' βœ— ' + msg); } -} - -function assertIncludes(html, substr, msg) { - assert(html.includes(substr), msg); -} - -// Read the channels.js source to extract the HTML template -const src = fs.readFileSync(__dirname + '/public/channels.js', 'utf8'); - -// Extract the sidebar HTML from the template literal -const htmlMatch = src.match(/app\.innerHTML\s*=\s*`([\s\S]*?)`;/); -const html = htmlMatch ? htmlMatch[1] : ''; - -console.log('Test: Add channel UX (#759)'); - -// 1. Button renders in the form -assertIncludes(html, 'class="ch-add-btn"', 'Add button has ch-add-btn class'); -assertIncludes(html, 'type="submit"', 'Button is type=submit'); -assertIncludes(html, '>+', 'Button shows + text'); - -// 2. Form has proper structure -assertIncludes(html, 'class="ch-add-form"', 'Form has ch-add-form class'); -assertIncludes(html, 'class="ch-add-row"', 'Row wrapper present'); -assert(!html.includes('class="ch-add-label"'), 'Label removed (redundant with hint)'); - -// 3. Hint text present -assertIncludes(html, 'class="ch-add-hint"', 'Hint div present'); -assertIncludes(html, 'e.g. #LongFast or 32-char hex key', 'Hint text correct'); - -// 4. Status div present -assertIncludes(html, 'id="chAddStatus"', 'Status div has correct id'); -assertIncludes(html, 'class="ch-add-status"', 'Status div has correct class'); -assertIncludes(html, 'style="display:none"', 'Status div hidden by default'); - -// 5. showAddStatus function exists in source -assert(src.includes('function showAddStatus('), 'showAddStatus function defined'); -assert(src.includes("'success'"), 'Success status type referenced'); -assert(src.includes("'error'"), 'Error status type referenced'); - -// 6. CSS classes exist -const css = fs.readFileSync(__dirname + '/public/style.css', 'utf8'); -assert(css.includes('.ch-add-form'), 'CSS: .ch-add-form defined'); -assert(css.includes('.ch-add-btn'), 'CSS: .ch-add-btn defined'); -assert(css.includes('.ch-add-hint'), 'CSS: .ch-add-hint defined'); -assert(css.includes('.ch-add-status'), 'CSS: .ch-add-status defined'); -assert(css.includes('.ch-add-row'), 'CSS: .ch-add-row defined'); -// .ch-add-label CSS kept for backward compat but label removed from HTML - -console.log('\n' + passed + ' passed, ' + failed + ' failed'); -process.exit(failed > 0 ? 1 : 0); diff --git a/test-e2e-playwright.js b/test-e2e-playwright.js index c1963964..af75e432 100644 --- a/test-e2e-playwright.js +++ b/test-e2e-playwright.js @@ -224,10 +224,7 @@ async function run() { // Test 5: Node detail loads (reuses nodes page from test 2) await test('Node detail loads', async () => { await page.waitForSelector('table tbody tr'); - // Click first row - const firstRow = await page.$('table tbody tr'); - assert(firstRow, 'No node rows found'); - await firstRow.click(); + await page.click('table tbody tr'); // Wait for detail pane to appear await page.waitForSelector('.node-detail'); const html = await page.content(); @@ -240,17 +237,14 @@ async function run() { await test('Node side panel Details link navigates', async () => { await page.goto(`${BASE}/#/nodes`, { waitUntil: 'domcontentloaded' }); await page.waitForSelector('table tbody tr'); - // Click first row to open side panel - const firstRow = await page.$('table tbody tr'); - assert(firstRow, 'No node rows found'); - await firstRow.click(); + await page.click('table tbody tr'); await page.waitForSelector('.node-detail'); // Find the Details link in the side panel - const detailsLink = await page.$('#nodesRight a.btn-primary[href^="#/nodes/"]'); - assert(detailsLink, 'Details link not found in side panel'); - const href = await detailsLink.getAttribute('href'); + await page.waitForSelector('#nodesRight a.btn-primary[href^="#/nodes/"]'); + const href = await page.$eval('#nodesRight a.btn-primary[href^="#/nodes/"]', el => el.getAttribute('href')); + assert(href, 'Details link not found in side panel'); // Click the Details link β€” this should navigate to the full detail page - await detailsLink.click(); + await page.click('#nodesRight a.btn-primary[href^="#/nodes/"]'); // Wait for navigation β€” the full detail page has sections like neighbors/packets await page.waitForFunction((expectedHash) => { return location.hash === expectedHash; @@ -663,6 +657,8 @@ async function run() { await page.waitForSelector('#ngCanvas', { timeout: 8000 }); const hasCanvas = await page.$('#ngCanvas'); assert(hasCanvas, 'Neighbor Graph tab should have a canvas element'); + // Stats are populated after the async API call β€” wait for at least one card before counting + await page.waitForSelector('#ngStats .stat-card', { timeout: 8000 }); const hasStats = await page.$$eval('#ngStats .stat-card', els => els.length); assert(hasStats >= 3, `Neighbor Graph stats should have >=3 cards, got ${hasStats}`); // Verify filters exist @@ -1358,6 +1354,38 @@ async function run() { await page.evaluate(() => localStorage.removeItem('cs-theme-overrides')); }); + await test('Customizer v2: typing in text field does not collapse focus (re-render guard)', async () => { + await page.goto(BASE, { waitUntil: 'domcontentloaded' }); + await page.waitForSelector('nav, .navbar, .nav, [class*="nav"]'); + await page.waitForFunction(() => window._customizerV2 && window._customizerV2.initDone, { timeout: 5000 }); + const toggleSel = '#customizeToggle, button[title*="ustom" i], [class*="customize"]'; + const btn = await page.$(toggleSel); + if (!btn) { console.log(' ⏭️ Customizer toggle not found'); return; } + await btn.click(); + await page.waitForSelector('.cust-overlay', { timeout: 5000 }); + const result = await page.evaluate(() => { + const input = document.querySelector('.cust-overlay input[type="text"][data-cv2-field]'); + if (!input) return { skipped: true }; + input.focus(); + input.value = 'test'; + input.dispatchEvent(new Event('input', { bubbles: true })); + const inputRef = input; + return new Promise(resolve => { + setTimeout(() => { + const panel = document.querySelector('.cust-overlay'); + resolve({ + inputConnected: inputRef.isConnected, + focusInPanel: panel ? panel.contains(document.activeElement) : false, + }); + }, 500); + }); + }); + if (result.skipped) { console.log(' ⏭️ No text input with data-cv2-field found in panel'); return; } + assert(result.inputConnected, 'Input element should remain connected to DOM after debounce fires'); + assert(result.focusInPanel, 'Focus should remain inside panel after debounce β€” re-render must not run while typing'); + await page.evaluate(() => localStorage.removeItem('cs-theme-overrides')); + }); + await test('Show Neighbors populates neighborPubkeys from affinity API', async () => { const testPubkey = 'aabbccdd11223344556677889900aabbccddeeff00112233445566778899001122'; diff --git a/test-hop-resolver-affinity.js b/test-hop-resolver-affinity.js index e09e20a6..a9ce2dda 100644 --- a/test-hop-resolver-affinity.js +++ b/test-hop-resolver-affinity.js @@ -22,9 +22,9 @@ function assert(condition, msg) { // ── Test nodes ── // Two nodes share the same 1-byte prefix "ab" -const nodeA = { public_key: 'ab1111', name: 'NodeA', lat: 37.0, lon: -122.0 }; -const nodeB = { public_key: 'ab2222', name: 'NodeB', lat: 38.0, lon: -123.0 }; -const nodeC = { public_key: 'cd3333', name: 'NodeC', lat: 37.5, lon: -122.5 }; +const nodeA = { public_key: 'ab1111', name: 'NodeA', role: 'repeater', lat: 37.0, lon: -122.0 }; +const nodeB = { public_key: 'ab2222', name: 'NodeB', role: 'repeater', lat: 38.0, lon: -123.0 }; +const nodeC = { public_key: 'cd3333', name: 'NodeC', role: 'repeater', lat: 37.5, lon: -122.5 }; console.log('\n=== HopResolver Affinity Tests ===\n'); @@ -88,7 +88,7 @@ assert(result5['ab'].name === 'NodeB', 'Should pick NodeB (highest affinity 0.9) // Test 6: Unambiguous hops are not affected by affinity console.log('\nTest 6: Unambiguous hops unaffected by affinity'); -const nodeD = { public_key: 'ee4444', name: 'NodeD', lat: 36.0, lon: -121.0 }; +const nodeD = { public_key: 'ee4444', name: 'NodeD', role: 'repeater', lat: 36.0, lon: -121.0 }; HopResolver.init([nodeA, nodeB, nodeC, nodeD]); HopResolver.setAffinity({ edges: [] }); const result6 = HopResolver.resolve(['ee44'], null, null, null, null, null); @@ -97,9 +97,9 @@ assert(!result6['ee44'].ambiguous, 'Should not be marked ambiguous'); // Test 7: lat=0 / lon=0 candidates are NOT excluded (equator/prime-meridian bug fix) console.log('\nTest 7: lat=0 / lon=0 candidates are included in geo scoring'); -const nodeEquator = { public_key: 'ab5555', name: 'EquatorNode', lat: 0, lon: 10 }; -const nodeFar = { public_key: 'ab6666', name: 'FarNode', lat: 60, lon: 60 }; -const anchorNearEq = { public_key: 'cd7777', name: 'AnchorEq', lat: 1, lon: 11 }; +const nodeEquator = { public_key: 'ab5555', name: 'EquatorNode', role: 'repeater', lat: 0, lon: 10 }; +const nodeFar = { public_key: 'ab6666', name: 'FarNode', role: 'repeater', lat: 60, lon: 60 }; +const anchorNearEq = { public_key: 'cd7777', name: 'AnchorEq', role: 'repeater', lat: 1, lon: 11 }; HopResolver.init([nodeEquator, nodeFar, anchorNearEq]); HopResolver.setAffinity({}); // Anchor near equator β€” EquatorNode (0,10) should be geo-closest @@ -109,13 +109,44 @@ assert(result7['ab'].name === 'EquatorNode', // Test 8: lon=0 candidate is also included console.log('\nTest 8: lon=0 candidate is included in geo scoring'); -const nodePrime = { public_key: 'ab8888', name: 'PrimeMeridian', lat: 10, lon: 0 }; -const anchorNearPM = { public_key: 'cd9999', name: 'AnchorPM', lat: 11, lon: 1 }; +const nodePrime = { public_key: 'ab8888', name: 'PrimeMeridian', role: 'repeater', lat: 10, lon: 0 }; +const anchorNearPM = { public_key: 'cd9999', name: 'AnchorPM', role: 'repeater', lat: 11, lon: 1 }; HopResolver.init([nodePrime, nodeFar, anchorNearPM]); HopResolver.setAffinity({}); const result8 = HopResolver.resolve(['cd99', 'ab'], 11.0, 1.0, null, null, null); assert(result8['ab'].name === 'PrimeMeridian', 'lon=0 candidate should be included and win by geo β€” got: ' + result8['ab'].name); +// ── Role filter tests (#935) ── +console.log('\nTest: Role filter β€” companions excluded from prefixIdx'); +const companion = { public_key: 'ab9999', name: 'Companion1', role: 'companion', lat: 37.0, lon: -122.0 }; +const sensor = { public_key: 'ab7777', name: 'Sensor1', role: 'sensor', lat: 37.0, lon: -122.0 }; +const repeater = { public_key: 'ab1234', name: 'Repeater1', role: 'repeater', lat: 37.0, lon: -122.0 }; +const roomSrv = { public_key: 'ff1234', name: 'RoomSrv1', role: 'room_server', lat: 37.0, lon: -122.0 }; + +HopResolver.init([companion, sensor, repeater, roomSrv]); +HopResolver.setAffinity({}); + +// Prefix 'ab' should only resolve to repeater (companion/sensor excluded) +const r1 = HopResolver.resolve(['ab12'], 0, 0, null, null, null); +assert(r1['ab12'] && r1['ab12'].name === 'Repeater1', + 'prefix ab12 resolves to Repeater1 not companion β€” got: ' + (r1['ab12'] && r1['ab12'].name)); + +// Prefix 'ff' should resolve to room_server +const r2 = HopResolver.resolve(['ff12'], 0, 0, null, null, null); +assert(r2['ff12'] && r2['ff12'].name === 'RoomSrv1', + 'prefix ff12 resolves to RoomSrv1 β€” got: ' + (r2['ff12'] && r2['ff12'].name)); + +// Prefix that only matches companion should return nothing +const r3 = HopResolver.resolve(['ab99'], 0, 0, null, null, null); +assert(!r3['ab99'] || !r3['ab99'].name, + 'prefix ab99 (companion only) resolves to nothing β€” got: ' + (r3['ab99'] && r3['ab99'].name)); + +// pubkeyIdx should still have companion (full pubkey lookup) +console.log('\nTest: pubkeyIdx still includes all roles'); +const fromServer = HopResolver.resolveFromServer(['ab99'], [companion.public_key]); +assert(fromServer['ab99'] && fromServer['ab99'].name === 'Companion1', + 'resolveFromServer finds companion by full pubkey β€” got: ' + (fromServer['ab99'] && fromServer['ab99'].name)); + console.log('\n' + (passed + failed) + ' tests, ' + passed + ' passed, ' + failed + ' failed\n'); process.exit(failed > 0 ? 1 : 0); diff --git a/test-packets.js b/test-packets.js index 66b6e36c..bce63307 100644 --- a/test-packets.js +++ b/test-packets.js @@ -844,6 +844,120 @@ console.log('\n=== packets.js: _invalidateRowCounts / _refreshRowCountsIfDirty ( }); } +console.log('\n=== packets.js: buildPacketsParams ==='); +{ + const ctx = loadPacketsSandbox(); + const api = ctx._packetsTestAPI; + assert(typeof api.buildPacketsParams === 'function', 'buildPacketsParams must be exported'); + + test('hash filter suppresses region β€” direct hash links work regardless of saved region', () => { + // This is the bug from URL https://analyzer.../#/packets?hash=178525e9f693aa7e + // when the user's saved RegionFilter excludes the packet's observer region. + // The hash is an exact identifier; ALL other filters must be ignored. + const p = api.buildPacketsParams({ + filters: { hash: 'abc123' }, + regionParam: 'SJC,SFO,OAK,MRY', + windowMin: 60, + groupByHash: false, + limit: 200, + }); + assert.strictEqual(p.get('hash'), 'abc123'); + assert.strictEqual(p.get('region'), null, 'region must NOT be set when hash is present'); + assert.strictEqual(p.get('since'), null, 'since must NOT be set when hash is present'); + }); + + test('hash filter suppresses ALL other filters β€” observer, node, channel too', () => { + const p = api.buildPacketsParams({ + filters: { hash: 'h', node: 'n', observer: 'o', channel: 'c' }, + regionParam: 'SJC', + windowMin: 60, + groupByHash: false, + limit: 200, + }); + assert.strictEqual(p.get('hash'), 'h'); + assert.strictEqual(p.get('node'), null); + assert.strictEqual(p.get('observer'), null); + assert.strictEqual(p.get('channel'), null); + assert.strictEqual(p.get('region'), null); + assert.strictEqual(p.get('since'), null); + }); + + test('hash filter suppresses region with default windowMin=0', () => { + const p = api.buildPacketsParams({ + filters: { hash: 'deadbeef' }, + regionParam: 'COA', + windowMin: 0, + groupByHash: false, + limit: 50, + }); + assert.strictEqual(p.get('hash'), 'deadbeef'); + assert.strictEqual(p.get('region'), null); + }); + + test('region applied normally when hash filter is absent', () => { + const p = api.buildPacketsParams({ + filters: {}, + regionParam: 'SJC,SFO', + windowMin: 60, + groupByHash: false, + limit: 200, + }); + assert.strictEqual(p.get('region'), 'SJC,SFO', 'region must apply when no hash'); + assert.strictEqual(p.get('hash'), null); + assert(p.get('since'), 'since must apply when no hash and windowMin>0'); + }); + + test('observer/node/channel pass through normally when no hash', () => { + const p = api.buildPacketsParams({ + filters: { observer: 'obs1', node: 'node1', channel: '#test' }, + regionParam: '', + windowMin: 0, + groupByHash: false, + limit: 50, + }); + assert.strictEqual(p.get('observer'), 'obs1'); + assert.strictEqual(p.get('node'), 'node1'); + assert.strictEqual(p.get('channel'), '#test'); + }); + + test('region absent when regionParam empty β€” no spurious empty region= param', () => { + const p = api.buildPacketsParams({ + filters: {}, + regionParam: '', + windowMin: 0, + groupByHash: false, + limit: 50, + }); + assert.strictEqual(p.get('region'), null); + }); + + test('groupByHash=true with hash sets groupByHash and omits expand', () => { + const p = api.buildPacketsParams({ + filters: { hash: 'h' }, regionParam: '', windowMin: 0, groupByHash: true, limit: 50, + }); + assert.strictEqual(p.get('groupByHash'), 'true'); + assert.strictEqual(p.get('expand'), null); + assert.strictEqual(p.get('hash'), 'h'); + }); + + test('groupByHash=false with hash sets expand=observations', () => { + const p = api.buildPacketsParams({ + filters: { hash: 'h' }, regionParam: '', windowMin: 0, groupByHash: false, limit: 50, + }); + assert.strictEqual(p.get('expand'), 'observations'); + assert.strictEqual(p.get('groupByHash'), null); + assert.strictEqual(p.get('hash'), 'h'); + }); + + test('groupByHash=false without hash sets expand=observations', () => { + const p = api.buildPacketsParams({ + filters: {}, regionParam: '', windowMin: 0, groupByHash: false, limit: 50, + }); + assert.strictEqual(p.get('expand'), 'observations'); + assert.strictEqual(p.get('groupByHash'), null); + }); +} + // ===== SUMMARY ===== console.log(`\n${'='.repeat(40)}`); console.log(`packets.js tests: ${passed} passed, ${failed} failed`); diff --git a/test-path-inspector-e2e.js b/test-path-inspector-e2e.js new file mode 100644 index 00000000..ddf16c45 --- /dev/null +++ b/test-path-inspector-e2e.js @@ -0,0 +1,87 @@ +// E2E tests for Path Inspector (spec Β§5 β€” Playwright). +// Run: npx playwright test test-path-inspector-e2e.js +// Requires: running server on BASE_URL (default http://localhost:3000). +'use strict'; + +const { test, expect } = require('@playwright/test'); +const BASE_URL = process.env.BASE_URL || 'http://localhost:3000'; + +test.describe('Path Inspector β€” Map Side Pane (spec Β§2.7)', () => { + test('side pane present and collapsed by default', async ({ page }) => { + await page.goto(`${BASE_URL}/#/map`); + const pane = page.locator('#mapSidePane'); + await expect(pane).toBeVisible(); + await expect(pane).not.toHaveClass(/expanded/); + }); + + test('click toggle expands the pane', async ({ page }) => { + await page.goto(`${BASE_URL}/#/map`); + await page.click('#mapPaneToggle'); + const pane = page.locator('#mapSidePane'); + await expect(pane).toHaveClass(/expanded/); + }); + + test('submit valid prefixes renders candidates within 1s', async ({ page }) => { + await page.goto(`${BASE_URL}/#/map`); + await page.click('#mapPaneToggle'); + await page.fill('#mapPiInput', '2c,a1,f4'); + await page.click('#mapPiSubmit'); + // Wait for results or error (both indicate API round-trip complete). + await expect(page.locator('#mapPiResults table, #mapPiResults .no-results, #mapPiError')).toBeVisible({ timeout: 1000 }); + }); + + test('Show on Map button draws polyline on map', async ({ page }) => { + await page.goto(`${BASE_URL}/#/map`); + await page.click('#mapPaneToggle'); + await page.fill('#mapPiInput', '2c,a1'); + await page.click('#mapPiSubmit'); + // Wait for results. + const btn = page.locator('#mapPiResults button[data-idx="0"]'); + await btn.waitFor({ timeout: 2000 }); + await btn.click(); + // Check that route layer has SVG polyline paths drawn. + const svg = page.locator('#leaflet-map .leaflet-overlay-pane svg path'); + await expect(svg.first()).toBeVisible({ timeout: 2000 }); + }); + + test('switching candidate clears prior polyline', async ({ page }) => { + await page.goto(`${BASE_URL}/#/map`); + await page.click('#mapPaneToggle'); + await page.fill('#mapPiInput', '2c,a1'); + await page.click('#mapPiSubmit'); + const btn0 = page.locator('#mapPiResults button[data-idx="0"]'); + await btn0.waitFor({ timeout: 2000 }); + await btn0.click(); + // Click second candidate if available. + const btn1 = page.locator('#mapPiResults button[data-idx="1"]'); + if (await btn1.isVisible()) { + await btn1.click(); + // Prior route should be cleared β€” only one polyline group visible. + } + }); +}); + +test.describe('Path Inspector β€” Standalone Page', () => { + test('deep link auto-fills and runs', async ({ page }) => { + await page.goto(`${BASE_URL}/#/tools/path-inspector?prefixes=2c,a1,f4`); + const input = page.locator('#path-inspector-input'); + await expect(input).toHaveValue('2c,a1,f4'); + // Should auto-submit and show results or error. + await expect(page.locator('#path-inspector-results table, #path-inspector-results .no-results, #path-inspector-error')).toBeVisible({ timeout: 2000 }); + }); + + test('old #/traces/ redirects to #/tools/trace/', async ({ page }) => { + await page.goto(`${BASE_URL}/#/traces/abc123`); + await page.waitForTimeout(500); + expect(page.url()).toContain('#/tools/trace/abc123'); + }); +}); + +test.describe('Path Inspector β€” Tools Landing (spec Β§2.8)', () => { + test('Tools nav shows landing with both entries', async ({ page }) => { + await page.goto(`${BASE_URL}/#/tools`); + await expect(page.locator('.tools-landing')).toBeVisible(); + await expect(page.locator('a[href="#/tools/path-inspector"]')).toBeVisible(); + await expect(page.locator('a[href*="#/tools/trace"]')).toBeVisible(); + }); +}); diff --git a/test-path-inspector.js b/test-path-inspector.js new file mode 100644 index 00000000..3d818b85 --- /dev/null +++ b/test-path-inspector.js @@ -0,0 +1,106 @@ +// test-path-inspector.js β€” vm.createContext sandbox tests for path-inspector.js +'use strict'; +const vm = require('vm'); +const fs = require('fs'); +const assert = require('assert'); + +const src = fs.readFileSync(__dirname + '/public/path-inspector.js', 'utf8'); + +function createSandbox() { + const sandbox = { + window: {}, + document: { + getElementById: () => ({ textContent: '', innerHTML: '', addEventListener: () => {}, querySelectorAll: () => [] }), + querySelectorAll: () => [] + }, + location: { hash: '#/tools/path-inspector' }, + history: { replaceState: () => {} }, + fetch: () => Promise.resolve({ ok: true, json: () => Promise.resolve({ candidates: [] }) }), + URLSearchParams: URLSearchParams, + registerPage: function () {}, + escapeHtml: s => s, + console: console + }; + sandbox.self = sandbox; + sandbox.globalThis = sandbox; + const ctx = vm.createContext(sandbox); + vm.runInContext(src, ctx); + return sandbox; +} + +// Test: parsePrefixes accepts comma-separated. +(function testParseComma() { + const sb = createSandbox(); + const result = sb.window.PathInspector.parsePrefixes('2C,A1,F4'); + assert.strictEqual(JSON.stringify(result), JSON.stringify(['2c', 'a1', 'f4'])); + console.log('βœ“ parsePrefixes comma-separated'); +})(); + +// Test: parsePrefixes accepts space-separated. +(function testParseSpace() { + const sb = createSandbox(); + const result = sb.window.PathInspector.parsePrefixes('2C A1 F4'); + assert.strictEqual(JSON.stringify(result), JSON.stringify(['2c', 'a1', 'f4'])); + console.log('βœ“ parsePrefixes space-separated'); +})(); + +// Test: parsePrefixes accepts mixed. +(function testParseMixed() { + const sb = createSandbox(); + const result = sb.window.PathInspector.parsePrefixes(' 2C, A1 F4 '); + assert.strictEqual(JSON.stringify(result), JSON.stringify(['2c', 'a1', 'f4'])); + console.log('βœ“ parsePrefixes mixed separators'); +})(); + +// Test: validatePrefixes rejects empty. +(function testValidateEmpty() { + const sb = createSandbox(); + const err = sb.window.PathInspector.validatePrefixes([]); + assert.ok(err !== null, 'should reject empty'); + console.log('βœ“ validatePrefixes rejects empty'); +})(); + +// Test: validatePrefixes rejects odd-length. +(function testValidateOdd() { + const sb = createSandbox(); + const err = sb.window.PathInspector.validatePrefixes(['abc']); + assert.ok(err !== null && err.includes('Odd'), 'should reject odd-length'); + console.log('βœ“ validatePrefixes rejects odd-length'); +})(); + +// Test: validatePrefixes rejects >3 bytes. +(function testValidateTooLong() { + const sb = createSandbox(); + const err = sb.window.PathInspector.validatePrefixes(['aabbccdd']); + assert.ok(err !== null && err.includes('too long'), 'should reject >3 bytes'); + console.log('βœ“ validatePrefixes rejects >3 bytes'); +})(); + +// Test: validatePrefixes rejects mixed lengths. +(function testValidateMixed() { + const sb = createSandbox(); + const err = sb.window.PathInspector.validatePrefixes(['aa', 'bbcc']); + assert.ok(err !== null && err.includes('Mixed'), 'should reject mixed'); + console.log('βœ“ validatePrefixes rejects mixed lengths'); +})(); + +// Test: validatePrefixes accepts valid input. +(function testValidateValid() { + const sb = createSandbox(); + const err = sb.window.PathInspector.validatePrefixes(['2c', 'a1', 'f4']); + assert.strictEqual(err, null); + console.log('βœ“ validatePrefixes accepts valid'); +})(); + +// Test: validatePrefixes rejects invalid hex. +(function testValidateInvalidHex() { + const sb = createSandbox(); + const err = sb.window.PathInspector.validatePrefixes(['zz']); + assert.ok(err !== null && err.includes('Invalid hex'), 'should reject invalid hex'); + console.log('βœ“ validatePrefixes rejects invalid hex'); +})(); + +// Anti-tautology: if validation were removed (always return null), the odd-length test would fail. +// Mental revert: validatePrefixes = () => null; β†’ testValidateOdd would fail because err would be null. + +console.log('\nAll path-inspector tests passed!'); diff --git a/tools/geofilter-builder.html b/tools/geofilter-builder.html index addb7e69..93beab8e 100644 --- a/tools/geofilter-builder.html +++ b/tools/geofilter-builder.html @@ -72,7 +72,8 @@ let polygon = null; let closingLine = null; function latLonPair(latlng) { - return [parseFloat(latlng.lat.toFixed(6)), parseFloat(latlng.lng.toFixed(6))]; + const w = latlng.wrap(); + return [parseFloat(w.lat.toFixed(6)), parseFloat(w.lng.toFixed(6))]; } function render() {