docs: add startup load mechanics, in-memory structure, and dual-write pattern to resolved-path spec

docs: address ambiguities — canonical edge ordering, resolution/edge timing
docs: finalize resolved-path spec — persisted graph, observation column, all consumers
2026-06-10 18:21:39 +00:00 · 2026-04-04 04:07:21 +00:00 · 2026-04-04 03:45:03 +00:00 · 2026-04-04 03:44:08 +00:00 · 2026-04-04 03:17:27 +00:00 · 2026-04-04 00:51:14 +00:00
21 changed files with 1633 additions and 144 deletions
@@ -362,6 +362,12 @@ One logical change per commit. Each commit is deployable. Each commit has its te
 - Tests: `test-{feature}.js` in repo root
 - No build step, no transpilation — write ES2020 for server, ES5/6 for frontend (broad browser support)

+### Deep Linking
+All new UI states that a user might want to share or bookmark MUST be reflected in the URL hash.
+This includes: tabs, filters, selected items, view modes. Use query parameters on the hash
+(e.g., `#/packets?observer=ABC&timeRange=24h`) for filter state.
+Existing patterns: `#/nodes/{pubkey}?section=node-neighbors`, `#/analytics?tab=collisions`, `#/packets/{hash}`.
+
 ## What NOT to Do
 - **Don't check in private information** — no names, API keys, tokens, passwords, IP addresses, personal data, or any identifying information. This is a PUBLIC repo.
 - Don't add npm dependencies without asking
@@ -0,0 +1,181 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"testing"
+)
+
+// TestAdvertPubkeyTracking verifies that advertPubkeys is maintained
+// incrementally during ingest and eviction, and that GetPerfStoreStats
+// returns the correct count without per-request JSON parsing.
+func TestAdvertPubkeyTracking(t *testing.T) {
+	ps := NewPacketStore(nil, nil)
+	ps.mu.Lock()
+
+	// Helper to create an ADVERT StoreTx with a given pubkey.
+	pt4 := 4
+	mkAdvert := func(id int, pubkey string) *StoreTx {
+		d := map[string]interface{}{"pubKey": pubkey}
+		j, _ := json.Marshal(d)
+		return &StoreTx{
+			ID:          id,
+			Hash:        fmt.Sprintf("hash%d", id),
+			PayloadType: &pt4,
+			DecodedJSON: string(j),
+		}
+	}
+
+	// Add 3 adverts: 2 distinct pubkeys
+	tx1 := mkAdvert(1, "pk_alpha")
+	tx2 := mkAdvert(2, "pk_beta")
+	tx3 := mkAdvert(3, "pk_alpha") // duplicate pubkey
+
+	for _, tx := range []*StoreTx{tx1, tx2, tx3} {
+		ps.packets = append(ps.packets, tx)
+		ps.byHash[tx.Hash] = tx
+		ps.byTxID[tx.ID] = tx
+		ps.byPayloadType[4] = append(ps.byPayloadType[4], tx)
+		ps.trackAdvertPubkey(tx)
+	}
+	ps.mu.Unlock()
+
+	// GetPerfStoreStats should report 2 distinct pubkeys
+	stats := ps.GetPerfStoreStats()
+	indexes := stats["indexes"].(map[string]interface{})
+	got := indexes["advertByObserver"].(int)
+	if got != 2 {
+		t.Errorf("advertByObserver = %d, want 2", got)
+	}
+
+	// GetPerfStoreStatsTyped should agree
+	typed := ps.GetPerfStoreStatsTyped()
+	if typed.Indexes.AdvertByObserver != 2 {
+		t.Errorf("typed AdvertByObserver = %d, want 2", typed.Indexes.AdvertByObserver)
+	}
+
+	// Evict tx3 (pk_alpha duplicate) — count should stay 2
+	ps.mu.Lock()
+	ps.untrackAdvertPubkey(tx3)
+	ps.mu.Unlock()
+
+	stats2 := ps.GetPerfStoreStats()
+	idx2 := stats2["indexes"].(map[string]interface{})
+	if idx2["advertByObserver"].(int) != 2 {
+		t.Errorf("after evicting duplicate: advertByObserver = %d, want 2", idx2["advertByObserver"].(int))
+	}
+
+	// Evict tx1 (last pk_alpha) — count should drop to 1
+	ps.mu.Lock()
+	ps.untrackAdvertPubkey(tx1)
+	ps.mu.Unlock()
+
+	stats3 := ps.GetPerfStoreStats()
+	idx3 := stats3["indexes"].(map[string]interface{})
+	if idx3["advertByObserver"].(int) != 1 {
+		t.Errorf("after evicting last pk_alpha: advertByObserver = %d, want 1", idx3["advertByObserver"].(int))
+	}
+
+	// Evict tx2 (last remaining) — count should be 0
+	ps.mu.Lock()
+	ps.untrackAdvertPubkey(tx2)
+	ps.mu.Unlock()
+
+	stats4 := ps.GetPerfStoreStats()
+	idx4 := stats4["indexes"].(map[string]interface{})
+	if idx4["advertByObserver"].(int) != 0 {
+		t.Errorf("after evicting all: advertByObserver = %d, want 0", idx4["advertByObserver"].(int))
+	}
+}
+
+// TestAdvertPubkeyPublicKeyField tests the "public_key" JSON field variant.
+func TestAdvertPubkeyPublicKeyField(t *testing.T) {
+	ps := NewPacketStore(nil, nil)
+	ps.mu.Lock()
+	pt4 := 4
+	d, _ := json.Marshal(map[string]interface{}{"public_key": "pk_legacy"})
+	tx := &StoreTx{ID: 1, Hash: "h1", PayloadType: &pt4, DecodedJSON: string(d)}
+	ps.trackAdvertPubkey(tx)
+	ps.mu.Unlock()
+
+	stats := ps.GetPerfStoreStats()
+	idx := stats["indexes"].(map[string]interface{})
+	if idx["advertByObserver"].(int) != 1 {
+		t.Errorf("public_key field: advertByObserver = %d, want 1", idx["advertByObserver"].(int))
+	}
+}
+
+// TestAdvertPubkeyNonAdvert ensures non-ADVERT packets don't affect the count.
+func TestAdvertPubkeyNonAdvert(t *testing.T) {
+	ps := NewPacketStore(nil, nil)
+	ps.mu.Lock()
+	pt2 := 2
+	d, _ := json.Marshal(map[string]interface{}{"pubKey": "pk_text"})
+	tx := &StoreTx{ID: 1, Hash: "h1", PayloadType: &pt2, DecodedJSON: string(d)}
+	ps.trackAdvertPubkey(tx)
+	ps.mu.Unlock()
+
+	stats := ps.GetPerfStoreStats()
+	idx := stats["indexes"].(map[string]interface{})
+	if idx["advertByObserver"].(int) != 0 {
+		t.Errorf("non-ADVERT should not be tracked: advertByObserver = %d, want 0", idx["advertByObserver"].(int))
+	}
+}
+
+// BenchmarkGetPerfStoreStats benchmarks the perf stats endpoint with many adverts.
+// Before the fix, this did O(N) JSON unmarshals per call.
+// After the fix, it's O(1) — just len(map).
+func BenchmarkGetPerfStoreStats(b *testing.B) {
+	ps := NewPacketStore(nil, nil)
+	ps.mu.Lock()
+	pt4 := 4
+	for i := 0; i < 5000; i++ {
+		pk := fmt.Sprintf("pk_%04d", i%200) // 200 distinct pubkeys
+		d, _ := json.Marshal(map[string]interface{}{"pubKey": pk})
+		tx := &StoreTx{
+			ID:          i + 1,
+			Hash:        fmt.Sprintf("hash%d", i+1),
+			PayloadType: &pt4,
+			DecodedJSON: string(d),
+		}
+		ps.packets = append(ps.packets, tx)
+		ps.byHash[tx.Hash] = tx
+		ps.byTxID[tx.ID] = tx
+		ps.byPayloadType[4] = append(ps.byPayloadType[4], tx)
+		ps.trackAdvertPubkey(tx)
+	}
+	ps.mu.Unlock()
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		ps.GetPerfStoreStats()
+	}
+}
+
+// BenchmarkGetPerfStoreStatsTyped benchmarks the typed variant.
+func BenchmarkGetPerfStoreStatsTyped(b *testing.B) {
+	ps := NewPacketStore(nil, nil)
+	ps.mu.Lock()
+	pt4 := 4
+	for i := 0; i < 5000; i++ {
+		pk := fmt.Sprintf("pk_%04d", i%200)
+		d, _ := json.Marshal(map[string]interface{}{"pubKey": pk})
+		tx := &StoreTx{
+			ID:          i + 1,
+			Hash:        fmt.Sprintf("hash%d", i+1),
+			PayloadType: &pt4,
+			DecodedJSON: string(d),
+		}
+		ps.packets = append(ps.packets, tx)
+		ps.byHash[tx.Hash] = tx
+		ps.byTxID[tx.ID] = tx
+		ps.byPayloadType[4] = append(ps.byPayloadType[4], tx)
+		ps.trackAdvertPubkey(tx)
+	}
+	ps.mu.Unlock()
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		ps.GetPerfStoreStatsTyped()
+	}
+}
@@ -16,6 +16,7 @@ func newTestStore(t *testing.T) *PacketStore {
 		distCache:    make(map[string]*cachedResult),
 		subpathCache: make(map[string]*cachedResult),
 		rfCacheTTL:   15 * time.Second,
+		invCooldown:  10 * time.Second,
 	}
 }

@@ -169,3 +170,164 @@ func TestInvalidateCachesFor_NoFlags(t *testing.T) {
 		}
 	}
 }
+
+// TestInvalidationRateLimited verifies that rapid ingest cycles don't clear
+// caches immediately — they accumulate dirty flags during the cooldown period
+// and apply them on the next call after cooldown expires (fixes #533).
+func TestInvalidationRateLimited(t *testing.T) {
+	s := newTestStore(t)
+	s.invCooldown = 100 * time.Millisecond // short cooldown for testing
+
+	// First invalidation should go through immediately
+	populateAllCaches(s)
+	s.invalidateCachesFor(cacheInvalidation{hasNewObservations: true})
+	state := cachePopulated(s)
+	if state["rf"] {
+		t.Error("rf cache should be cleared on first invalidation")
+	}
+	if !state["topo"] {
+		t.Error("topo cache should survive (no path changes)")
+	}
+
+	// Repopulate and call again within cooldown — should NOT clear
+	populateAllCaches(s)
+	s.invalidateCachesFor(cacheInvalidation{hasNewObservations: true})
+	state = cachePopulated(s)
+	if !state["rf"] {
+		t.Error("rf cache should survive during cooldown period")
+	}
+
+	// Wait for cooldown to expire
+	time.Sleep(150 * time.Millisecond)
+
+	// Next call should apply accumulated + current flags
+	populateAllCaches(s)
+	s.invalidateCachesFor(cacheInvalidation{hasNewPaths: true})
+	state = cachePopulated(s)
+	if state["rf"] {
+		t.Error("rf cache should be cleared (pending from cooldown)")
+	}
+	if state["topo"] {
+		t.Error("topo cache should be cleared (current call has hasNewPaths)")
+	}
+	if !state["hash"] {
+		t.Error("hash cache should survive (no transmission changes)")
+	}
+}
+
+// TestInvalidationCooldownAccumulatesFlags verifies that multiple calls during
+// cooldown merge their flags correctly.
+func TestInvalidationCooldownAccumulatesFlags(t *testing.T) {
+	s := newTestStore(t)
+	s.invCooldown = 200 * time.Millisecond
+
+	// Initial invalidation (goes through, starts cooldown)
+	s.invalidateCachesFor(cacheInvalidation{hasNewObservations: true})
+
+	// Several calls during cooldown with different flags
+	s.invalidateCachesFor(cacheInvalidation{hasNewPaths: true})
+	s.invalidateCachesFor(cacheInvalidation{hasNewTransmissions: true})
+	s.invalidateCachesFor(cacheInvalidation{hasChannelData: true})
+
+	// Verify pending has all flags
+	s.cacheMu.Lock()
+	if s.pendingInv == nil {
+		t.Fatal("pendingInv should not be nil during cooldown")
+	}
+	if !s.pendingInv.hasNewPaths || !s.pendingInv.hasNewTransmissions || !s.pendingInv.hasChannelData {
+		t.Error("all flags should be accumulated in pendingInv")
+	}
+	// hasNewObservations was applied immediately, not accumulated
+	if s.pendingInv.hasNewObservations {
+		t.Error("hasNewObservations was already applied, should not be in pending")
+	}
+	s.cacheMu.Unlock()
+
+	// Wait for cooldown, then trigger — all accumulated flags should apply
+	time.Sleep(250 * time.Millisecond)
+	populateAllCaches(s)
+	s.invalidateCachesFor(cacheInvalidation{}) // empty trigger
+	state := cachePopulated(s)
+
+	// Pending had paths, transmissions, channels — all those caches should clear
+	if state["topo"] {
+		t.Error("topo should be cleared (pending hasNewPaths)")
+	}
+	if state["hash"] {
+		t.Error("hash should be cleared (pending hasNewTransmissions)")
+	}
+	if state["chan"] {
+		t.Error("chan should be cleared (pending hasChannelData)")
+	}
+}
+
+// TestEvictionBypassesCooldown verifies eviction always clears immediately.
+func TestEvictionBypassesCooldown(t *testing.T) {
+	s := newTestStore(t)
+	s.invCooldown = 10 * time.Second // long cooldown
+
+	// Start cooldown
+	s.invalidateCachesFor(cacheInvalidation{hasNewObservations: true})
+
+	// Eviction during cooldown should still clear everything
+	populateAllCaches(s)
+	s.invalidateCachesFor(cacheInvalidation{eviction: true})
+	state := cachePopulated(s)
+	for name, has := range state {
+		if has {
+			t.Errorf("%s cache should be cleared on eviction even during cooldown", name)
+		}
+	}
+	// pendingInv should be cleared
+	s.cacheMu.Lock()
+	if s.pendingInv != nil {
+		t.Error("pendingInv should be nil after eviction")
+	}
+	s.cacheMu.Unlock()
+}
+
+// BenchmarkCacheHitDuringIngestion simulates rapid ingestion and verifies
+// that cache hits now occur thanks to rate-limited invalidation.
+func BenchmarkCacheHitDuringIngestion(b *testing.B) {
+	s := &PacketStore{
+		rfCache:      make(map[string]*cachedResult),
+		topoCache:    make(map[string]*cachedResult),
+		hashCache:    make(map[string]*cachedResult),
+		chanCache:    make(map[string]*cachedResult),
+		distCache:    make(map[string]*cachedResult),
+		subpathCache: make(map[string]*cachedResult),
+		rfCacheTTL:   15 * time.Second,
+		invCooldown:  50 * time.Millisecond,
+	}
+
+	// Trigger first invalidation to start cooldown timer
+	s.invalidateCachesFor(cacheInvalidation{hasNewObservations: true})
+
+	var hits, misses int64
+	for i := 0; i < b.N; i++ {
+		// Populate cache (simulates an analytics query filling the cache)
+		s.cacheMu.Lock()
+		s.rfCache["global"] = &cachedResult{
+			data:      map[string]interface{}{"test": true},
+			expiresAt: time.Now().Add(time.Hour),
+		}
+		s.cacheMu.Unlock()
+
+		// Simulate rapid ingest invalidation (should be rate-limited)
+		s.invalidateCachesFor(cacheInvalidation{hasNewObservations: true})
+
+		// Check if cache survived the invalidation
+		s.cacheMu.Lock()
+		if len(s.rfCache) > 0 {
+			hits++
+		} else {
+			misses++
+		}
+		s.cacheMu.Unlock()
+	}
+
+	if hits == 0 {
+		b.Errorf("expected cache hits > 0 with rate-limited invalidation, got 0 hits / %d misses", misses)
+	}
+	b.ReportMetric(float64(hits)/float64(hits+misses)*100, "hit%")
+}
@@ -3811,3 +3811,105 @@ func BenchmarkIndexByNode(b *testing.B) {
 		}
 	})
 }
+
+// --- Multi-observer comma-separated filter tests ---
+
+func TestTransmissionsForObserverMultiCSV(t *testing.T) {
+	db := setupTestDB(t)
+	defer db.Close()
+	seedTestData(t, db)
+	store := NewPacketStore(db, nil)
+	store.Load()
+
+	t.Run("comma-separated returns union via index", func(t *testing.T) {
+		result := store.transmissionsForObserver("obs1,obs2", nil)
+		if len(result) == 0 {
+			t.Fatal("expected results for obs1,obs2")
+		}
+		// obs1 has transmissions 1,2,3; obs2 has transmission 1
+		// Union should include all unique transmissions
+		obs1Only := store.transmissionsForObserver("obs1", nil)
+		obs2Only := store.transmissionsForObserver("obs2", nil)
+		if len(result) < len(obs1Only) || len(result) < len(obs2Only) {
+			t.Errorf("union (%d) should be >= each individual set (obs1=%d, obs2=%d)",
+				len(result), len(obs1Only), len(obs2Only))
+		}
+	})
+
+	t.Run("comma-separated with spaces via index", func(t *testing.T) {
+		result := store.transmissionsForObserver("obs1, obs2", nil)
+		if len(result) == 0 {
+			t.Fatal("expected results for 'obs1, obs2' (with space)")
+		}
+		noSpace := store.transmissionsForObserver("obs1,obs2", nil)
+		if len(result) != len(noSpace) {
+			t.Errorf("with-space (%d) should equal no-space (%d)", len(result), len(noSpace))
+		}
+	})
+
+	t.Run("comma-separated returns union via filter path", func(t *testing.T) {
+		allTx := store.packets
+		result := store.transmissionsForObserver("obs1,obs2", allTx)
+		if len(result) == 0 {
+			t.Fatal("expected results for obs1,obs2 via filter path")
+		}
+	})
+
+	t.Run("comma-separated with spaces via filter path", func(t *testing.T) {
+		allTx := store.packets
+		withSpace := store.transmissionsForObserver("obs1, obs2", allTx)
+		noSpace := store.transmissionsForObserver("obs1,obs2", allTx)
+		if len(withSpace) != len(noSpace) {
+			t.Errorf("filter path: with-space (%d) should equal no-space (%d)", len(withSpace), len(noSpace))
+		}
+	})
+}
+
+func TestBuildTransmissionWhereMultiObserver(t *testing.T) {
+	db := setupTestDB(t)
+	defer db.Close()
+	seedTestData(t, db)
+
+	t.Run("comma-separated produces IN clause", func(t *testing.T) {
+		q := PacketQuery{Observer: "obs1,obs2"}
+		where, args := db.buildTransmissionWhere(q)
+		if len(where) != 1 {
+			t.Fatalf("expected 1 WHERE clause, got %d", len(where))
+		}
+		clause := where[0]
+		if !strings.Contains(clause, "IN (?,?)") {
+			t.Errorf("expected IN (?,?) in clause, got: %s", clause)
+		}
+		if len(args) != 2 {
+			t.Fatalf("expected 2 args, got %d", len(args))
+		}
+		if args[0] != "obs1" || args[1] != "obs2" {
+			t.Errorf("expected [obs1, obs2], got %v", args)
+		}
+	})
+
+	t.Run("comma-separated with spaces trims IDs", func(t *testing.T) {
+		q := PacketQuery{Observer: "obs1, obs2"}
+		_, args := db.buildTransmissionWhere(q)
+		if len(args) != 2 {
+			t.Fatalf("expected 2 args, got %d", len(args))
+		}
+		if args[0] != "obs1" || args[1] != "obs2" {
+			t.Errorf("expected trimmed [obs1, obs2], got %v", args)
+		}
+	})
+
+	t.Run("single observer still works", func(t *testing.T) {
+		q := PacketQuery{Observer: "obs1"}
+		where, args := db.buildTransmissionWhere(q)
+		if len(where) != 1 {
+			t.Fatalf("expected 1 WHERE clause, got %d", len(where))
+		}
+		if !strings.Contains(where[0], "IN (?)") {
+			t.Errorf("expected IN (?) for single observer, got: %s", where[0])
+		}
+		if len(args) != 1 || args[0] != "obs1" {
+			t.Errorf("expected [obs1], got %v", args)
+		}
+	})
+}
@@ -608,12 +608,17 @@ func (db *DB) buildTransmissionWhere(q PacketQuery) ([]string, []interface{}) {
 		args = append(args, "%"+pk+"%")
 	}
 	if q.Observer != "" {
+		ids := strings.Split(q.Observer, ",")
+		placeholders := strings.Repeat("?,", len(ids))
+		placeholders = placeholders[:len(placeholders)-1]
 		if db.isV3 {
-			where = append(where, "EXISTS (SELECT 1 FROM observations oi JOIN observers obi ON obi.rowid = oi.observer_idx WHERE oi.transmission_id = t.id AND obi.id = ?)")
+			where = append(where, "EXISTS (SELECT 1 FROM observations oi JOIN observers obi ON obi.rowid = oi.observer_idx WHERE oi.transmission_id = t.id AND obi.id IN ("+placeholders+"))")
 		} else {
-			where = append(where, "EXISTS (SELECT 1 FROM observations oi WHERE oi.transmission_id = t.id AND oi.observer_id = ?)")
+			where = append(where, "EXISTS (SELECT 1 FROM observations oi WHERE oi.transmission_id = t.id AND oi.observer_id IN ("+placeholders+"))")
+		}
+		for _, id := range ids {
+			args = append(args, strings.TrimSpace(id))
 		}
-		args = append(args, q.Observer)
 	}
 	if q.Region != "" {
 		if db.isV3 {
@@ -2,6 +2,8 @@ package main

 import (
 	"encoding/json"
+	"fmt"
+	"math/rand"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -220,6 +222,44 @@ func TestSortedCopy(t *testing.T) {
 	}
 }

+func TestSortedCopyLarge(t *testing.T) {
+	// Regression: verify correct sort on larger input
+	rng := rand.New(rand.NewSource(42))
+	n := 1000
+	input := make([]float64, n)
+	for i := range input {
+		input[i] = rng.Float64() * 1000
+	}
+	result := sortedCopy(input)
+	if len(result) != n {
+		t.Fatalf("expected %d elements, got %d", n, len(result))
+	}
+	for i := 1; i < len(result); i++ {
+		if result[i] < result[i-1] {
+			t.Fatalf("not sorted at index %d: %v > %v", i, result[i-1], result[i])
+		}
+	}
+	// Original unchanged
+	if input[0] == result[0] && input[1] == result[1] && input[2] == result[2] {
+		// Could be coincidence but very unlikely with random data
+	}
+}
+
+func BenchmarkSortedCopy(b *testing.B) {
+	rng := rand.New(rand.NewSource(42))
+	for _, size := range []int{256, 1000, 10000} {
+		data := make([]float64, size)
+		for i := range data {
+			data[i] = rng.Float64() * 1000
+		}
+		b.Run(fmt.Sprintf("n=%d", size), func(b *testing.B) {
+			for i := 0; i < b.N; i++ {
+				sortedCopy(data)
+			}
+		})
+	}
+}
+
 func TestLastN(t *testing.T) {
 	arr := []map[string]interface{}{
 		{"id": 1}, {"id": 2}, {"id": 3}, {"id": 4}, {"id": 5},
@@ -0,0 +1,134 @@
+package main
+
+import (
+	"fmt"
+	"testing"
+)
+
+// TestObsDedupCorrectness verifies that the map-based dedup produces correct
+// results: no duplicate observations (same observerID + pathJSON) on a single
+// transmission.
+func TestObsDedupCorrectness(t *testing.T) {
+	tx := &StoreTx{
+		ID:      1,
+		Hash:    "abc123",
+		obsKeys: make(map[string]bool),
+	}
+
+	// Add 5 unique observations
+	for i := 0; i < 5; i++ {
+		obsID := fmt.Sprintf("obs-%d", i)
+		pathJSON := fmt.Sprintf(`["path-%d"]`, i)
+		dk := obsID + "|" + pathJSON
+		if tx.obsKeys[dk] {
+			t.Fatalf("observation %d should not be a duplicate", i)
+		}
+		tx.Observations = append(tx.Observations, &StoreObs{
+			ID:         i,
+			ObserverID: obsID,
+			PathJSON:   pathJSON,
+		})
+		tx.obsKeys[dk] = true
+		tx.ObservationCount++
+	}
+
+	if tx.ObservationCount != 5 {
+		t.Fatalf("expected 5 observations, got %d", tx.ObservationCount)
+	}
+
+	// Try to add duplicates of each — all should be rejected
+	for i := 0; i < 5; i++ {
+		obsID := fmt.Sprintf("obs-%d", i)
+		pathJSON := fmt.Sprintf(`["path-%d"]`, i)
+		dk := obsID + "|" + pathJSON
+		if !tx.obsKeys[dk] {
+			t.Fatalf("observation %d should be detected as duplicate", i)
+		}
+	}
+
+	// Same observer, different path — should NOT be a duplicate
+	dk := "obs-0" + "|" + `["different-path"]`
+	if tx.obsKeys[dk] {
+		t.Fatal("different path should not be a duplicate")
+	}
+
+	// Different observer, same path — should NOT be a duplicate
+	dk = "obs-new" + "|" + `["path-0"]`
+	if tx.obsKeys[dk] {
+		t.Fatal("different observer should not be a duplicate")
+	}
+}
+
+// TestObsDedupNilMapSafety ensures obsKeys lazy init works for pre-existing
+// transmissions that may not have the map initialized.
+func TestObsDedupNilMapSafety(t *testing.T) {
+	tx := &StoreTx{ID: 1, Hash: "abc"}
+	// obsKeys is nil — the lazy init pattern used in IngestNewFromDB/IngestNewObservations
+	if tx.obsKeys == nil {
+		tx.obsKeys = make(map[string]bool)
+	}
+	dk := "obs1|path1"
+	if tx.obsKeys[dk] {
+		t.Fatal("should not be duplicate on empty map")
+	}
+	tx.obsKeys[dk] = true
+	if !tx.obsKeys[dk] {
+		t.Fatal("should be duplicate after insert")
+	}
+}
+
+// BenchmarkObsDedupMap benchmarks the map-based O(1) dedup approach.
+func BenchmarkObsDedupMap(b *testing.B) {
+	for _, obsCount := range []int{10, 50, 100, 500} {
+		b.Run(fmt.Sprintf("obs=%d", obsCount), func(b *testing.B) {
+			// Pre-populate a tx with obsCount observations
+			tx := &StoreTx{
+				ID:      1,
+				obsKeys: make(map[string]bool),
+			}
+			for i := 0; i < obsCount; i++ {
+				obsID := fmt.Sprintf("obs-%d", i)
+				pathJSON := fmt.Sprintf(`["hop-%d"]`, i)
+				dk := obsID + "|" + pathJSON
+				tx.Observations = append(tx.Observations, &StoreObs{
+					ObserverID: obsID,
+					PathJSON:   pathJSON,
+				})
+				tx.obsKeys[dk] = true
+			}
+
+			// Benchmark: check dedup for a new observation (not duplicate)
+			newDK := "new-obs|new-path"
+			b.ResetTimer()
+			for i := 0; i < b.N; i++ {
+				_ = tx.obsKeys[newDK]
+			}
+		})
+	}
+}
+
+// BenchmarkObsDedupLinear benchmarks the old O(n) linear scan for comparison.
+func BenchmarkObsDedupLinear(b *testing.B) {
+	for _, obsCount := range []int{10, 50, 100, 500} {
+		b.Run(fmt.Sprintf("obs=%d", obsCount), func(b *testing.B) {
+			tx := &StoreTx{ID: 1}
+			for i := 0; i < obsCount; i++ {
+				tx.Observations = append(tx.Observations, &StoreObs{
+					ObserverID: fmt.Sprintf("obs-%d", i),
+					PathJSON:   fmt.Sprintf(`["hop-%d"]`, i),
+				})
+			}
+
+			newObsID := "new-obs"
+			newPath := "new-path"
+			b.ResetTimer()
+			for i := 0; i < b.N; i++ {
+				for _, existing := range tx.Observations {
+					if existing.ObserverID == newObsID && existing.PathJSON == newPath {
+						break
+					}
+				}
+			}
+		})
+	}
+}
@@ -1958,13 +1958,7 @@ func percentile(sorted []float64, p float64) float64 {
 func sortedCopy(arr []float64) []float64 {
 	cp := make([]float64, len(arr))
 	copy(cp, arr)
-	for i := 0; i < len(cp); i++ {
-		for j := i + 1; j < len(cp); j++ {
-			if cp[j] < cp[i] {
-				cp[i], cp[j] = cp[j], cp[i]
-			}
-		}
-	}
+	sort.Float64s(cp)
 	return cp
 }

@@ -3059,11 +3059,11 @@ func TestHashCollisionsWithCollision(t *testing.T) {
 	now := time.Now().UTC()
 	recent := now.Add(-1 * time.Hour).Format(time.RFC3339)

-	// Two nodes with same first byte 'CC', no adverts so hash_size=0 (included in all buckets)
+	// Two repeater nodes with same first byte 'CC' and hash_size=1
 	db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count)
-		VALUES ('CC11223344556677', 'Node1', 'repeater', 37.5, -122.0, ?, '2026-01-01T00:00:00Z', 0)`, recent)
+		VALUES ('CC11223344556677', 'Node1', 'repeater', 37.5, -122.0, ?, '2026-01-01T00:00:00Z', 5)`, recent)
 	db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count)
-		VALUES ('CC99887766554433', 'Node2', 'repeater', 37.51, -122.01, ?, '2026-01-01T00:00:00Z', 0)`, recent)
+		VALUES ('CC99887766554433', 'Node2', 'repeater', 37.51, -122.01, ?, '2026-01-01T00:00:00Z', 5)`, recent)

 	cfg := &Config{Port: 3000}
 	hub := NewHub()
@@ -3072,6 +3072,14 @@ func TestHashCollisionsWithCollision(t *testing.T) {
 	if err := store.Load(); err != nil {
 		t.Fatalf("store.Load failed: %v", err)
 	}
+	// Inject hash_size=1 for both nodes so they appear in the 1-byte bucket
+	store.hashSizeInfoMu.Lock()
+	store.hashSizeInfoCache = map[string]*hashSizeNodeInfo{
+		"CC11223344556677": {HashSize: 1, AllSizes: map[int]bool{1: true}},
+		"CC99887766554433": {HashSize: 1, AllSizes: map[int]bool{1: true}},
+	}
+	store.hashSizeInfoAt = time.Now()
+	store.hashSizeInfoMu.Unlock()
 	srv.store = store
 	router := mux.NewRouter()
 	srv.RegisterRoutes(router)
@@ -3186,3 +3194,86 @@ func TestHashCollisionsMissingCoordinates(t *testing.T) {
 		}
 	}
 }
+
+// TestHashCollisionsOnlyRepeaters verifies that only repeater nodes
+// are included in collision analysis. Companions, rooms, sensors, and
+// hash_size==0 nodes are excluded — per firmware analysis, only repeaters
+// forward packets and appear in path[] arrays. (#441)
+func TestHashCollisionsOnlyRepeaters(t *testing.T) {
+	db := setupTestDB(t)
+
+	// Insert nodes sharing the same 1-byte prefix "AA":
+	//   1. repeater with hash_size=1 → should be counted
+	//   2. repeater with hash_size=0 (unknown) → should be excluded
+	//   3. companion with hash_size=1 → should be excluded
+	//   4. room with hash_size=1 → should be excluded
+	//   5. sensor with hash_size=1 → should be excluded
+	now := time.Now().Format("2006-01-02 15:04:05")
+	db.conn.Exec(`INSERT INTO nodes (public_key, name, role, last_seen) VALUES
+		('aa11223344556677', 'Repeater1', 'repeater', ?),
+		('aa99887766554433', 'UnknownNode', 'repeater', ?),
+		('aadeadbeefcafe01', 'Companion1', 'companion', ?),
+		('aabbcc1122334455', 'Room1', 'room', ?),
+		('aabbcc9988776655', 'Sensor1', 'sensor', ?)`, now, now, now, now, now)
+
+	// We also need a second repeater with hash_size=1 and same prefix to
+	// confirm that genuine collisions ARE still detected.
+	db.conn.Exec(`INSERT INTO nodes (public_key, name, role, last_seen) VALUES
+		('aa00112233445566', 'Repeater2', 'repeater', ?)`, now)
+
+	cfg := &Config{Port: 3000}
+	hub := NewHub()
+	srv := NewServer(db, cfg, hub)
+	store := NewPacketStore(db, nil)
+	store.Load()
+	srv.store = store
+
+	// Inject hash size info directly into the cache
+	store.hashSizeInfoMu.Lock()
+	store.hashSizeInfoCache = map[string]*hashSizeNodeInfo{
+		"aa11223344556677": {HashSize: 1, AllSizes: map[int]bool{1: true}},
+		"aa00112233445566": {HashSize: 1, AllSizes: map[int]bool{1: true}},
+		"aa99887766554433": {HashSize: 0, AllSizes: map[int]bool{}},       // unknown
+		"aadeadbeefcafe01": {HashSize: 1, AllSizes: map[int]bool{1: true}}, // companion
+		"aabbcc1122334455": {HashSize: 1, AllSizes: map[int]bool{1: true}}, // room
+		"aabbcc9988776655": {HashSize: 1, AllSizes: map[int]bool{1: true}}, // sensor
+	}
+	store.hashSizeInfoAt = time.Now()
+	store.hashSizeInfoMu.Unlock()
+
+	result := store.computeHashCollisions("")
+
+	bySize, ok := result["by_size"].(map[string]interface{})
+	if !ok {
+		t.Fatal("missing by_size")
+	}
+
+	size1, ok := bySize["1"].(map[string]interface{})
+	if !ok {
+		t.Fatal("missing by_size[1]")
+	}
+
+	stats, ok := size1["stats"].(map[string]interface{})
+	if !ok {
+		t.Fatal("missing stats")
+	}
+
+	// Only Repeater1 and Repeater2 should be in nodesForByte (hash_size=1, role=repeater).
+	// UnknownNode (hash_size=0), Companion1, Room1, Sensor1 must all be excluded.
+	nodesForByte := stats["nodes_for_byte"]
+	if nodesForByte != 2 {
+		t.Errorf("expected nodes_for_byte=2 (only repeaters with hash_size=1), got %v", nodesForByte)
+	}
+
+	// They share prefix "AA", so there should be exactly 1 collision entry.
+	collisions, ok := size1["collisions"].([]collisionEntry)
+	if !ok {
+		t.Fatalf("collisions is not []collisionEntry")
+	}
+	if len(collisions) != 1 {
+		t.Errorf("expected 1 collision entry, got %d", len(collisions))
+	}
+	if len(collisions) == 1 && len(collisions[0].Nodes) != 2 {
+		t.Errorf("expected 2 nodes in collision, got %d", len(collisions[0].Nodes))
+	}
+}
@@ -43,6 +43,8 @@ type StoreTx struct {
 	// Cached parsed fields (set once, read many)
 	parsedPath []string // cached parsePathJSON result
 	pathParsed bool     // whether parsedPath has been set
+	// Dedup map: "observerID|pathJSON" → true for O(1) duplicate checks
+	obsKeys map[string]bool
 }

 // StoreObs is a lean in-memory observation (no duplication of transmission fields).
@@ -88,6 +90,10 @@ type PacketStore struct {
 	collisionCacheTTL time.Duration
 	cacheHits    int64
 	cacheMisses  int64
+	// Rate-limited invalidation (fixes #533: caches cleared faster than hit)
+	lastInvalidated time.Time
+	pendingInv      *cacheInvalidation // accumulated dirty flags during cooldown
+	invCooldown     time.Duration      // minimum time between invalidations
 	// Short-lived cache for QueryGroupedPackets (avoids repeated full sort)
 	groupedCacheMu  sync.Mutex
 	groupedCacheKey string
@@ -117,6 +123,10 @@ type PacketStore struct {
 	hashSizeInfoCache map[string]*hashSizeNodeInfo
 	hashSizeInfoAt    time.Time

+	// Precomputed distinct advert pubkey count (refcounted for eviction correctness).
+	// Updated incrementally during Load/Ingest/Evict — avoids JSON parsing in GetPerfStoreStats.
+	advertPubkeys map[string]int // pubkey → number of advert packets referencing it
+
 	// Eviction config and stats
 	retentionHours float64 // 0 = unlimited
 	maxMemoryMB    int     // 0 = unlimited
@@ -182,7 +192,9 @@ func NewPacketStore(db *DB, cfg *PacketStoreConfig) *PacketStore {
 		subpathCache:  make(map[string]*cachedResult),
 		rfCacheTTL:         15 * time.Second,
 		collisionCacheTTL: 60 * time.Second,
+		invCooldown:       10 * time.Second,
 		spIndex:       make(map[string]int, 4096),
+		advertPubkeys: make(map[string]int),
 	}
 	if cfg != nil {
 		ps.retentionHours = cfg.RetentionHours
@@ -253,6 +265,7 @@ func (s *PacketStore) Load() error {
 				RouteType:   nullIntPtr(routeType),
 				PayloadType: nullIntPtr(payloadType),
 				DecodedJSON: nullStrVal(decodedJSON),
+				obsKeys:     make(map[string]bool),
 			}
 			s.byHash[hashStr] = tx
 			s.packets = append(s.packets, tx)
@@ -262,6 +275,7 @@ func (s *PacketStore) Load() error {
 				pt := *tx.PayloadType
 				s.byPayloadType[pt] = append(s.byPayloadType[pt], tx)
 			}
+			s.trackAdvertPubkey(tx)
 		}

 		if obsID.Valid {
@@ -269,15 +283,9 @@ func (s *PacketStore) Load() error {
 			obsIDStr := nullStrVal(observerID)
 			obsPJ := nullStrVal(pathJSON)

-			// Dedup: skip if same observer + same path already loaded
-			isDupe := false
-			for _, existing := range tx.Observations {
-				if existing.ObserverID == obsIDStr && existing.PathJSON == obsPJ {
-					isDupe = true
-					break
-				}
-			}
-			if isDupe {
+			// Dedup: skip if same observer + same path already loaded (O(1) map lookup)
+			dk := obsIDStr + "|" + obsPJ
+			if tx.obsKeys[dk] {
 				continue
 			}

@@ -295,6 +303,7 @@ func (s *PacketStore) Load() error {
 			}

 			tx.Observations = append(tx.Observations, obs)
+			tx.obsKeys[dk] = true
 			tx.ObservationCount++
 			if obs.Timestamp > tx.LatestSeen {
 				tx.LatestSeen = obs.Timestamp
@@ -392,6 +401,52 @@ func (s *PacketStore) indexByNode(tx *StoreTx) {
 	}
 }

+// trackAdvertPubkey increments the advertPubkeys refcount for ADVERT packets.
+// Must be called under s.mu write lock.
+func (s *PacketStore) trackAdvertPubkey(tx *StoreTx) {
+	if tx.PayloadType == nil || *tx.PayloadType != 4 || tx.DecodedJSON == "" {
+		return
+	}
+	var d map[string]interface{}
+	if json.Unmarshal([]byte(tx.DecodedJSON), &d) != nil {
+		return
+	}
+	pk := ""
+	if v, ok := d["pubKey"].(string); ok {
+		pk = v
+	} else if v, ok := d["public_key"].(string); ok {
+		pk = v
+	}
+	if pk != "" {
+		s.advertPubkeys[pk]++
+	}
+}
+
+// untrackAdvertPubkey decrements the advertPubkeys refcount for ADVERT packets.
+// Must be called under s.mu write lock.
+func (s *PacketStore) untrackAdvertPubkey(tx *StoreTx) {
+	if tx.PayloadType == nil || *tx.PayloadType != 4 || tx.DecodedJSON == "" {
+		return
+	}
+	var d map[string]interface{}
+	if json.Unmarshal([]byte(tx.DecodedJSON), &d) != nil {
+		return
+	}
+	pk := ""
+	if v, ok := d["pubKey"].(string); ok {
+		pk = v
+	} else if v, ok := d["public_key"].(string); ok {
+		pk = v
+	}
+	if pk != "" {
+		if s.advertPubkeys[pk] <= 1 {
+			delete(s.advertPubkeys, pk)
+		} else {
+			s.advertPubkeys[pk]--
+		}
+	}
+}
+
 // QueryPackets returns filtered, paginated packets from memory.
 func (s *PacketStore) QueryPackets(q PacketQuery) *PacketResult {
 	atomic.AddInt64(&s.queryCount, 1)
@@ -579,30 +634,8 @@ func (s *PacketStore) GetPerfStoreStats() map[string]interface{} {
 	nodeIdx := len(s.byNode)
 	ptIdx := len(s.byPayloadType)

-	// Count distinct pubkeys with ADVERT observations (matches Node.js _advertByObserver.size)
-	advertByObsCount := 0
-	if adverts, ok := s.byPayloadType[4]; ok {
-		seen := make(map[string]bool)
-		for _, tx := range adverts {
-			if tx.DecodedJSON == "" {
-				continue
-			}
-			var d map[string]interface{}
-			if json.Unmarshal([]byte(tx.DecodedJSON), &d) != nil {
-				continue
-			}
-			pk := ""
-			if v, ok := d["pubKey"].(string); ok {
-				pk = v
-			} else if v, ok := d["public_key"].(string); ok {
-				pk = v
-			}
-			if pk != "" && !seen[pk] {
-				seen[pk] = true
-				advertByObsCount++
-			}
-		}
-	}
+	// Distinct advert pubkey count — precomputed incrementally (see trackAdvertPubkey).
+	advertByObsCount := len(s.advertPubkeys)
 	s.mu.RUnlock()

 	// Realistic estimate: ~5KB per packet + ~500 bytes per observation
@@ -690,15 +723,16 @@ type cacheInvalidation struct {
 }

 // invalidateCachesFor selectively clears only the analytics caches affected
-// by the kind of data that changed. This avoids the previous behaviour of
-// wiping every cache on every ingest cycle, which defeated caching under
-// continuous ingestion (issue #375).
+// by the kind of data that changed. To prevent continuous ingestion from
+// defeating caching entirely (issue #533), invalidation is rate-limited:
+// if called within invCooldown of the last invalidation, the flags are
+// accumulated in pendingInv and applied on the next call after cooldown.
 func (s *PacketStore) invalidateCachesFor(inv cacheInvalidation) {
 	s.cacheMu.Lock()
 	defer s.cacheMu.Unlock()

+	// Eviction bypasses rate-limiting — data was removed, caches must clear.
 	if inv.eviction {
-		// Eviction can affect any analytics — clear everything
 		s.rfCache = make(map[string]*cachedResult)
 		s.topoCache = make(map[string]*cachedResult)
 		s.hashCache = make(map[string]*cachedResult)
@@ -709,9 +743,40 @@ func (s *PacketStore) invalidateCachesFor(inv cacheInvalidation) {
 		s.channelsCacheMu.Lock()
 		s.channelsCacheRes = nil
 		s.channelsCacheMu.Unlock()
+		s.lastInvalidated = time.Now()
+		s.pendingInv = nil
 		return
 	}

+	now := time.Now()
+	if now.Sub(s.lastInvalidated) < s.invCooldown {
+		// Within cooldown — accumulate dirty flags
+		if s.pendingInv == nil {
+			s.pendingInv = &cacheInvalidation{}
+		}
+		s.pendingInv.hasNewObservations = s.pendingInv.hasNewObservations || inv.hasNewObservations
+		s.pendingInv.hasNewPaths = s.pendingInv.hasNewPaths || inv.hasNewPaths
+		s.pendingInv.hasNewTransmissions = s.pendingInv.hasNewTransmissions || inv.hasNewTransmissions
+		s.pendingInv.hasChannelData = s.pendingInv.hasChannelData || inv.hasChannelData
+		return
+	}
+
+	// Cooldown expired — merge any pending flags and apply
+	if s.pendingInv != nil {
+		inv.hasNewObservations = inv.hasNewObservations || s.pendingInv.hasNewObservations
+		inv.hasNewPaths = inv.hasNewPaths || s.pendingInv.hasNewPaths
+		inv.hasNewTransmissions = inv.hasNewTransmissions || s.pendingInv.hasNewTransmissions
+		inv.hasChannelData = inv.hasChannelData || s.pendingInv.hasChannelData
+		s.pendingInv = nil
+	}
+
+	s.applyCacheInvalidation(inv)
+	s.lastInvalidated = now
+}
+
+// applyCacheInvalidation performs the actual cache clearing. Must be called
+// with cacheMu held.
+func (s *PacketStore) applyCacheInvalidation(inv cacheInvalidation) {
 	if inv.hasNewObservations {
 		s.rfCache = make(map[string]*cachedResult)
 	}
@@ -726,7 +791,6 @@ func (s *PacketStore) invalidateCachesFor(inv cacheInvalidation) {
 	}
 	if inv.hasChannelData {
 		s.chanCache = make(map[string]*cachedResult)
-		// Also invalidate the separate channels list cache
 		s.channelsCacheMu.Lock()
 		s.channelsCacheRes = nil
 		s.channelsCacheMu.Unlock()
@@ -742,29 +806,7 @@ func (s *PacketStore) GetPerfStoreStatsTyped() PerfPacketStoreStats {
 	observerIdx := len(s.byObserver)
 	nodeIdx := len(s.byNode)

-	advertByObsCount := 0
-	if adverts, ok := s.byPayloadType[4]; ok {
-		seen := make(map[string]bool)
-		for _, tx := range adverts {
-			if tx.DecodedJSON == "" {
-				continue
-			}
-			var d map[string]interface{}
-			if json.Unmarshal([]byte(tx.DecodedJSON), &d) != nil {
-				continue
-			}
-			pk := ""
-			if v, ok := d["pubKey"].(string); ok {
-				pk = v
-			} else if v, ok := d["public_key"].(string); ok {
-				pk = v
-			}
-			if pk != "" && !seen[pk] {
-				seen[pk] = true
-				advertByObsCount++
-			}
-		}
-	}
+	advertByObsCount := len(s.advertPubkeys)
 	s.mu.RUnlock()

 	estimatedMB := math.Round(float64(totalLoaded*5120+totalObs*500)/1048576*10) / 10
@@ -1061,6 +1103,7 @@ func (s *PacketStore) IngestNewFromDB(sinceID, limit int) ([]map[string]interfac
 				RouteType:   r.routeType,
 				PayloadType: r.payloadType,
 				DecodedJSON: r.decodedJSON,
+				obsKeys:     make(map[string]bool),
 			}
 			s.byHash[r.hash] = tx
 			s.packets = append(s.packets, tx) // oldest-first; new items go to tail
@@ -1072,6 +1115,7 @@ func (s *PacketStore) IngestNewFromDB(sinceID, limit int) ([]map[string]interfac
 				// so GetChannelMessages reverse iteration stays correct
 				s.byPayloadType[pt] = append(s.byPayloadType[pt], tx)
 			}
+			s.trackAdvertPubkey(tx)

 			if _, exists := broadcastTxs[r.txID]; !exists {
 				broadcastTxs[r.txID] = tx
@@ -1081,15 +1125,12 @@ func (s *PacketStore) IngestNewFromDB(sinceID, limit int) ([]map[string]interfac

 		if r.obsID != nil {
 			oid := *r.obsID
-			// Dedup
-			isDupe := false
-			for _, existing := range tx.Observations {
-				if existing.ObserverID == r.observerID && existing.PathJSON == r.pathJSON {
-					isDupe = true
-					break
-				}
+			// Dedup (O(1) map lookup)
+			dk := r.observerID + "|" + r.pathJSON
+			if tx.obsKeys == nil {
+				tx.obsKeys = make(map[string]bool)
 			}
-			if isDupe {
+			if tx.obsKeys[dk] {
 				continue
 			}

@@ -1106,6 +1147,7 @@ func (s *PacketStore) IngestNewFromDB(sinceID, limit int) ([]map[string]interfac
 				Timestamp:      normalizeTimestamp(r.obsTS),
 			}
 			tx.Observations = append(tx.Observations, obs)
+			tx.obsKeys[dk] = true
 			tx.ObservationCount++
 			if obs.Timestamp > tx.LatestSeen {
 				tx.LatestSeen = obs.Timestamp
@@ -1326,15 +1368,12 @@ func (s *PacketStore) IngestNewObservations(sinceObsID, limit int) []map[string]
 			continue // transmission not yet in store
 		}

-		// Dedup by observer + path
-		isDupe := false
-		for _, existing := range tx.Observations {
-			if existing.ObserverID == r.observerID && existing.PathJSON == r.pathJSON {
-				isDupe = true
-				break
-			}
+		// Dedup by observer + path (O(1) map lookup)
+		dk := r.observerID + "|" + r.pathJSON
+		if tx.obsKeys == nil {
+			tx.obsKeys = make(map[string]bool)
 		}
-		if isDupe {
+		if tx.obsKeys[dk] {
 			continue
 		}

@@ -1351,6 +1390,7 @@ func (s *PacketStore) IngestNewObservations(sinceObsID, limit int) []map[string]
 			Timestamp:      normalizeTimestamp(r.timestamp),
 		}
 		tx.Observations = append(tx.Observations, obs)
+		tx.obsKeys[dk] = true
 		tx.ObservationCount++
 		if obs.Timestamp > tx.LatestSeen {
 			tx.LatestSeen = obs.Timestamp
@@ -1572,32 +1612,36 @@ func (s *PacketStore) filterPackets(q PacketQuery) []*StoreTx {
 }

 // transmissionsForObserver returns unique transmissions for an observer.
-func (s *PacketStore) transmissionsForObserver(observerID string, from []*StoreTx) []*StoreTx {
+func (s *PacketStore) transmissionsForObserver(observerIDs string, from []*StoreTx) []*StoreTx {
+	ids := strings.Split(observerIDs, ",")
+	idSet := make(map[string]bool, len(ids))
+	for i, id := range ids {
+		ids[i] = strings.TrimSpace(id)
+		idSet[ids[i]] = true
+	}
 	if from != nil {
 		return filterTxSlice(from, func(tx *StoreTx) bool {
 			for _, obs := range tx.Observations {
-				if obs.ObserverID == observerID {
+				if idSet[obs.ObserverID] {
 					return true
 				}
 			}
 			return false
 		})
 	}
-	// Use byObserver index
-	observations := s.byObserver[observerID]
-	if len(observations) == 0 {
-		return nil
-	}
-	seen := make(map[int]bool, len(observations))
+	// Use byObserver index: union transmissions for all IDs
+	seen := make(map[int]bool)
 	var result []*StoreTx
-	for _, obs := range observations {
-		if seen[obs.TransmissionID] {
-			continue
-		}
-		seen[obs.TransmissionID] = true
-		tx := s.byTxID[obs.TransmissionID]
-		if tx != nil {
-			result = append(result, tx)
+	for _, id := range ids {
+		for _, obs := range s.byObserver[id] {
+			if seen[obs.TransmissionID] {
+				continue
+			}
+			seen[obs.TransmissionID] = true
+			tx := s.byTxID[obs.TransmissionID]
+			if tx != nil {
+				result = append(result, tx)
+			}
 		}
 	}
 	return result
@@ -1939,6 +1983,7 @@ func (s *PacketStore) EvictStale() int {
 		}

 		// Remove from byPayloadType
+		s.untrackAdvertPubkey(tx)
 		if tx.PayloadType != nil {
 			pt := *tx.PayloadType
 			ptList := s.byPayloadType[pt]
@@ -4473,10 +4518,16 @@ func (s *PacketStore) computeHashCollisions(region string) map[string]interface{
 	// Compute collisions for each byte size (1, 2, 3)
 	collisionsBySize := make(map[string]interface{})
 	for _, bytes := range []int{1, 2, 3} {
-		// Filter nodes relevant to this byte size
+		// Filter nodes relevant to this byte size.
+		// - Exclude hash_size==0 nodes: no adverts seen, so actual hash
+		//   size is unknown. Including them in every bucket inflates
+		//   collision counts.
+		// - Exclude companions: they are mobile/temporary and don't form
+		//   the mesh backbone, so collisions with them aren't meaningful.
+		// (Fixes #441)
 		var nodesForByte []collisionNode
 		for _, cn := range allCNodes {
-			if cn.HashSize == bytes || cn.HashSize == 0 {
+			if cn.HashSize == bytes && cn.Role == "repeater" {
 				nodesForByte = append(nodesForByte, cn)
 			}
 		}
@@ -0,0 +1,272 @@
+# Spec: Server-side hop resolution at ingest — `resolved_path`
+
+**Status:** Final  
+**Issue:** [#555](https://github.com/Kpa-clawbot/CoreScope/issues/555)  
+**Related:** [#482](https://github.com/Kpa-clawbot/CoreScope/issues/482), [#528](https://github.com/Kpa-clawbot/CoreScope/issues/528)
+
+## Problem
+
+Any place where 1, 2, or 3-byte prefixes must be resolved to actual full repeater public keys and friendly names should use affinity data first, geo data as fallback. Across frontend, backend, whatever. Efficiently — no 7-second waits, no recomputation, aggressive caching.
+
+Currently, hop paths are stored as short uppercase hex prefixes in `path_json` (e.g. `["D6", "E3", "59"]`). Resolution to full pubkeys happens **client-side** via `HopResolver` (`public/hop-resolver.js`), which:
+
+- Is slow — each page/component re-resolves independently
+- Is inconsistent — different components may resolve the same prefix differently
+- Cannot leverage the server's neighbor affinity graph, which has far richer context for disambiguation
+- Causes redundant `/api/resolve-hops` calls from every client
+
+## Solution
+
+Resolve hop prefixes to full pubkeys **once at ingest time** on the server, using `resolveWithContext()` with 4-tier priority (affinity → geo → GPS → first match) and a **persisted neighbor graph**. Store the result as a new `resolved_path` column on observations alongside `path_json`.
+
+## Design decisions (locked)
+
+1. **`path_json` stays unchanged** — raw firmware prefixes, uppercase hex. Ground truth.
+2. **`resolved_path` is a column on observations** — full 64-char lowercase hex pubkeys, `null` for unresolved.
+3. **Resolved at ingest** using `resolveWithContext(hop, context, graph)` — 4-tier priority: affinity → geo → GPS → first match.
+4. **`null` = unresolved** — ambiguous prefixes store `null`. Frontend falls back to prefix display.
+5. **Both fields coexist** — not interchangeable. Different consumers use different fields.
+
+## Persisted neighbor graph
+
+### SQLite table: `neighbor_edges`
+
+Thin and normalized. Stores ONLY the relationship. SNR, observer names, GPS, roles — all join from existing tables when needed. No duplication.
+
+```sql
+CREATE TABLE IF NOT EXISTS neighbor_edges (
+    node_a TEXT NOT NULL,
+    node_b TEXT NOT NULL,
+    count INTEGER DEFAULT 1,
+    last_seen TEXT,
+    PRIMARY KEY (node_a, node_b)
+);
+```
+
+### Edge extraction rules (ADVERT vs non-ADVERT)
+
+At ingest, for each packet:
+
+- **ADVERT packets** (payload_type 4): originator pubkey is known from `decoded_json.pubKey`. Extract edge: `originator ↔ path[0]` (the first hop is a direct neighbor of the originator).
+- **ALL packets**: observer pubkey is known. Extract edge: `observer ↔ path[last]` (the last hop is a direct neighbor of the observer).
+- **Non-ADVERT packets**: originator is unknown (encrypted). ONLY extract `observer ↔ path[last]`.
+- Each packet produces **1 or 2 edge upserts** depending on type.
+
+Edge upsert uses canonical ordering (`node_a < node_b` lexicographically) to avoid duplicate edges:
+
+```sql
+INSERT INTO neighbor_edges (node_a, node_b, count, last_seen)
+VALUES (min(?, ?), max(?, ?), 1, ?)
+ON CONFLICT(node_a, node_b) DO UPDATE SET
+  count = count + 1,
+  last_seen = excluded.last_seen;
+```
+
+### In-memory structure
+
+```go
+type NeighborGraph struct {
+    edges map[string][]NeighborEdge  // pubkey → list of neighbor edges
+    mu    sync.RWMutex
+}
+```
+
+### Cold startup and backfill
+
+On startup:
+
+```go
+// 1. Query all edges from SQLite
+rows := db.Query("SELECT node_a, node_b, count, last_seen FROM neighbor_edges")
+
+// 2. Build in-memory graph
+graph := NewNeighborGraph()
+for rows.Next() {
+    var a, b string
+    var count int
+    var lastSeen string
+    rows.Scan(&a, &b, &count, &lastSeen)
+    graph.UpsertEdge(a, b, count, lastSeen)
+}
+
+// 3. Attach to PacketStore
+store.graph = graph
+```
+
+1. **Load `neighbor_edges` from SQLite** → build in-memory graph (code above).
+2. **If table empty (first run):** `BuildFromStore(packets)` — scan all existing packets, extract edges per the rules above, INSERT into `neighbor_edges`.
+3. **Load observations from SQLite.**
+4. **For observations without `resolved_path`:** resolve using the graph, UPDATE `resolved_path` in SQLite.
+5. **Ready to serve.**
+
+On subsequent runs, step 2 is skipped (table already populated). Step 4 only processes observations with NULL `resolved_path` (new or previously unresolved).
+
+### Incremental update at ingest
+
+Every edge upsert writes to **both** the in-memory graph and SQLite — they stay in sync. SQLite is the persistence layer, in-memory is the fast lookup layer.
+
+```go
+// Extract edge from packet
+graph.UpsertEdge(nodeA, nodeB, 1, now)
+
+// Also persist to SQLite
+db.Exec(`INSERT INTO neighbor_edges (node_a, node_b, count, last_seen)
+         VALUES (?, ?, 1, ?)
+         ON CONFLICT(node_a, node_b) DO UPDATE SET
+         count = count + 1, last_seen = ?`, a, b, now, now)
+```
+
+## Data model
+
+### Where does `resolved_path` live?
+
+**On observations**, as a column:
+
+```sql
+ALTER TABLE observations ADD COLUMN resolved_path TEXT;
+```
+
+Rationale: Each observer sees the packet from a different vantage point. The same 2-char prefix may resolve to different full pubkeys depending on which observer's neighborhood is considered. The observer's own pubkey provides critical context for `resolveWithContext` (tier 2: neighbor affinity). Storing on observations preserves this per-observer resolution.
+
+`resolved_path` is written in the same INSERT that creates the observation — one write, no double-write problem.
+
+### Field shape
+
+```
+resolved_path TEXT  -- JSON array: ["aabb...64chars", null, "ccdd...64chars"]
+```
+
+- Same length as the `path_json` array
+- Each element is either a 64-char lowercase hex pubkey string, or `null`
+- Stored as a JSON text column (same approach as `path_json`)
+- Uses `omitempty` — absent from JSON when not set
+
+## Every path resolution uses the graph — no exceptions
+
+All existing `pm.resolve()` call sites MUST be migrated to `resolveWithContext` with the persisted graph. No "we'll get to it later."
+
+### Call sites to migrate (exhaustive)
+
+Found via `grep -n "pm.resolve" cmd/server/store.go`:
+
+| Line | Function | Current | After |
+|------|----------|---------|-------|
+| 1192 | `IngestNewFromDB()` | `pm.resolve(hop)` | `resolveWithContext(hop, ctx, graph)` — resolve at ingest, store as `resolved_path` |
+| 1876 | `buildDistanceIndex()` | `pm.resolve(hop)` | Read `resolved_path` from observation — already resolved at ingest |
+| 3537 | `computeAnalyticsTopology()` | `pm.resolve(hop)` | Read `resolved_path` from observation |
+| 5528 | `computeAnalyticsSubpaths()` | `pm.resolve(hop)` | Read `resolved_path` from observation |
+| 5665 | `GetSubpathDetail()` | `pm.resolve(hop)` | `resolveWithContext(hop, ctx, graph)` — ad-hoc resolution for user-provided hops |
+| 5744 | `GetSubpathDetail()` | `pm.resolve(h)` | `resolveWithContext(h, ctx, graph)` — same function, second usage |
+
+**After migration:** `pm.resolve()` (naive prefix-only lookup) is dead code. Remove it. All resolution goes through `resolveWithContext` which uses the persisted neighbor graph for affinity-based disambiguation.
+
+## Ingest pipeline changes
+
+### Where resolution happens
+
+In `PacketStore.IngestNewFromDB()` in `cmd/server/store.go`. For new observations, resolution happens during the observation INSERT — same write. For backfill (cold startup), it's a separate UPDATE pass.
+
+Note on ordering: edge upserts (step 5) happen **after** resolution (step 3-4). This means the very first packet for a new neighbor pair resolves without that edge in the graph yet. This is acceptable — the affinity tier will miss, but geo/GPS/first-match tiers still work. On the next packet, the edge exists and affinity kicks in.
+
+Resolution flow per observation:
+1. Parse `path_json` into hop prefixes
+2. Build context pubkeys from the observation (observer pubkey, source/dest from decoded packet)
+3. Call `resolveWithContext(hop, contextPubkeys, neighborGraph)` for each hop
+4. Store result as `resolved_path` column on the observation (same INSERT)
+5. Upsert neighbor edges into `neighbor_edges` table (incremental update)
+
+### Performance
+
+`resolveWithContext` does:
+- Prefix map lookup (map access, O(1))
+- Optional neighbor graph check (small map lookups)
+- No DB queries, no network calls
+
+Per-hop cost: ~1–5μs. A typical packet has 0–5 hops. At 100 packets/second ingest rate, this adds <0.5ms total overhead per second. **Negligible.**
+
+## All consumers use `resolved_path`
+
+| Consumer | Before | After |
+|---|---|---|
+| Packets detail path names | Client HopResolver (naive) | Read `resolved_path` |
+| Map Show Route | Client HopResolver (naive) | Read `resolved_path` |
+| Live map animated paths | Client HopResolver (naive) | Read `resolved_path` |
+| Node detail paths | Client HopResolver (naive) | Read `resolved_path` |
+| Analytics topology | Server `pm.resolve()` (naive) | Read `resolved_path` from observations |
+| Analytics subpaths | Server `pm.resolve()` (naive) | Read `resolved_path` from observations |
+| Analytics hop distances | Server `pm.resolve()` (naive) | Read `resolved_path` from observations |
+| Subpath detail | Server `pm.resolve()` (naive) | `resolveWithContext` with graph |
+| Show Neighbors | Server neighbors API | Already correct |
+| `/api/resolve-hops` | Server `resolveWithContext` | Already correct |
+| Hex breakdown display | `path_json` raw | Unchanged — shows raw bytes |
+
+## WebSocket broadcast
+
+Include `resolved_path` in broadcast messages. Resolution happens before broadcast assembly — negligible latency impact. The WS broadcast already includes `path_json`; `resolved_path` is added alongside it.
+
+## API changes
+
+### Endpoints that return `resolved_path`
+
+All endpoints that currently return `path_json` also return `resolved_path`:
+
+- `GET /api/packets` — transmission-level (use best observation's `resolved_path`)
+- `GET /api/packets/:hash` — per-observation detail
+- `GET /api/packets/:hash/observations` — each observation includes its own `resolved_path`
+- WebSocket broadcast messages — per-observation
+
+### `/api/resolve-hops`
+
+**Kept.** Useful for ad-hoc resolution of arbitrary prefixes (debug tools, clients resolving prefixes not associated with a packet). Not deprecated.
+
+## Pubkey case convention
+
+- **DB/API:** lowercase
+- **`path_json` display prefixes:** uppercase (raw firmware)
+- **`resolved_path`:** lowercase full pubkeys
+- **Comparison code:** normalizes to lowercase
+
+## Backward compatibility
+
+- Old observations without `resolved_path`: resolved during cold startup backfill (step 4). If still `null` after backfill, frontend falls back to client-side HopResolver.
+- `resolved_path` field uses `omitempty` — absent from JSON when not set.
+
+### Fallback pattern (frontend)
+
+```javascript
+function getResolvedHops(packet) {
+  if (packet.resolved_path) return packet.resolved_path;
+  // Fall back to client-side resolution for old packets
+  return resolveHopsClientSide(packet.path_json);
+}
+```
+
+## Implementation milestones
+
+### M1: Persist graph to SQLite + load on startup + incremental updates at ingest
+- Create `neighbor_edges` table in SQLite (schema above)
+- On first run: `BuildFromStore(packets)` — scan all packets, extract edges per ADVERT/non-ADVERT rules, INSERT into table
+- On subsequent runs: load from SQLite → build in-memory graph (instant startup)
+- Upsert edges incrementally during packet ingest
+- Graph lives on `PacketStore`, not `Server`
+- Tests: graph persistence, load, incremental update, ADVERT vs non-ADVERT edge extraction
+
+### M2: Add `resolved_path` column to observations + resolve at ingest
+- `ALTER TABLE observations ADD COLUMN resolved_path TEXT`
+- Add `ResolvedPath []*string` to `Observation` struct
+- Resolve during `IngestNewFromDB` — same INSERT, one write
+- Cold startup backfill: resolve observations with NULL `resolved_path`, UPDATE in SQLite
+- Migrate ALL 6 `pm.resolve()` call sites to `resolveWithContext` or read from `resolved_path`
+- Remove dead `pm.resolve()` code
+- Tests: unit test resolution at ingest, verify stored values, verify all call sites use graph
+
+### M3: Update all API responses to include `resolved_path`
+- Include `resolved_path` in all packet/observation API responses
+- Include in WebSocket broadcast messages
+- Tests: verify API response shape, WS broadcast shape
+
+### M4: Update frontend consumers to prefer `resolved_path`
+- Update `packets.js`, `map.js`, `live.js`, `analytics.js`, `nodes.js`
+- Add fallback to `path_json` + `HopResolver` for old packets
+- `hop-resolver.js` becomes fallback only
+- Tests: Playwright tests for path display
@@ -11,6 +11,7 @@ window.HopResolver = (function() {
  let nodesList = [];
  let observerIataMap = {}; // observer_id → iata
  let iataCoords = {};  // iata → {lat, lon}
+  let affinityMap = {}; // pubkey → { neighborPubkey → score }

  function dist(lat1, lon1, lat2, lon2) {
    return Math.sqrt((lat1 - lat2) ** 2 + (lon1 - lon2) ** 2);
@@ -67,6 +68,34 @@ window.HopResolver = (function() {
    return null; // no GPS — can't geo-filter client-side
  }

+  /**
+   * Pick the best candidate using affinity first, then geo-distance fallback.
+   * @param {Array} candidates - candidates with lat/lon/pubkey/name
+   * @param {string|null} adjacentPubkey - pubkey of the previously/next resolved hop
+   * @param {Object|null} anchor - {lat, lon} for geo fallback
+   * @param {number|null} fallbackLat - fallback anchor lat (e.g. observer)
+   * @param {number|null} fallbackLon - fallback anchor lon
+   * @returns {Object} best candidate
+   */
+  function pickByAffinity(candidates, adjacentPubkey, anchor, fallbackLat, fallbackLon) {
+    // If we have affinity data and an adjacent hop, prefer neighbors
+    if (adjacentPubkey && Object.keys(affinityMap).length > 0) {
+      const withAffinity = candidates
+        .map(c => ({ ...c, affinity: getAffinity(adjacentPubkey, c.pubkey) }))
+        .filter(c => c.affinity > 0);
+      if (withAffinity.length > 0) {
+        withAffinity.sort((a, b) => b.affinity - a.affinity);
+        return withAffinity[0];
+      }
+    }
+    // Fallback: geo-distance sort (existing behavior)
+    const effectiveAnchor = anchor || (fallbackLat != null ? { lat: fallbackLat, lon: fallbackLon } : null);
+    if (effectiveAnchor) {
+      candidates.sort((a, b) => dist(a.lat, a.lon, effectiveAnchor.lat, effectiveAnchor.lon) - dist(b.lat, b.lon, effectiveAnchor.lat, effectiveAnchor.lon));
+    }
+    return candidates[0];
+  }
+
  /**
   * Resolve an array of hex hop prefixes to node info.
   * Returns a map: { hop: {name, pubkey, lat, lon, ambiguous, unreliable} }
@@ -139,40 +168,50 @@ window.HopResolver = (function() {

    // Forward pass
    let lastPos = (originLat != null && originLon != null) ? { lat: originLat, lon: originLon } : null;
+    let lastResolvedPubkey = null;
    for (let i = 0; i < hops.length; i++) {
      const hop = hops[i];
-      if (hopPositions[hop]) { lastPos = hopPositions[hop]; continue; }
+      if (hopPositions[hop]) {
+        lastPos = hopPositions[hop];
+        lastResolvedPubkey = resolved[hop] ? resolved[hop].pubkey : null;
+        continue;
+      }
      const r = resolved[hop];
      if (!r || !r.ambiguous) continue;
      const withLoc = r.candidates.filter(c => c.lat && c.lon && !(c.lat === 0 && c.lon === 0));
      if (!withLoc.length) continue;
-      let anchor = lastPos;
-      if (!anchor && i === hops.length - 1 && observerLat != null) {
-        anchor = { lat: observerLat, lon: observerLon };
-      }
-      if (anchor) {
-        withLoc.sort((a, b) => dist(a.lat, a.lon, anchor.lat, anchor.lon) - dist(b.lat, b.lon, anchor.lat, anchor.lon));
-      }
-      r.name = withLoc[0].name;
-      r.pubkey = withLoc[0].pubkey;
-      hopPositions[hop] = { lat: withLoc[0].lat, lon: withLoc[0].lon };
+
+      // Affinity-aware: prefer candidates that are neighbors of the previous hop
+      const picked = pickByAffinity(withLoc, lastResolvedPubkey, lastPos, i === hops.length - 1 ? observerLat : null, i === hops.length - 1 ? observerLon : null);
+      r.name = picked.name;
+      r.pubkey = picked.pubkey;
+      hopPositions[hop] = { lat: picked.lat, lon: picked.lon };
      lastPos = hopPositions[hop];
+      lastResolvedPubkey = picked.pubkey;
    }

    // Backward pass
    let nextPos = (observerLat != null && observerLon != null) ? { lat: observerLat, lon: observerLon } : null;
+    let nextResolvedPubkey = null;
    for (let i = hops.length - 1; i >= 0; i--) {
      const hop = hops[i];
-      if (hopPositions[hop]) { nextPos = hopPositions[hop]; continue; }
+      if (hopPositions[hop]) {
+        nextPos = hopPositions[hop];
+        nextResolvedPubkey = resolved[hop] ? resolved[hop].pubkey : null;
+        continue;
+      }
      const r = resolved[hop];
      if (!r || !r.ambiguous) continue;
      const withLoc = r.candidates.filter(c => c.lat && c.lon && !(c.lat === 0 && c.lon === 0));
      if (!withLoc.length || !nextPos) continue;
-      withLoc.sort((a, b) => dist(a.lat, a.lon, nextPos.lat, nextPos.lon) - dist(b.lat, b.lon, nextPos.lat, nextPos.lon));
-      r.name = withLoc[0].name;
-      r.pubkey = withLoc[0].pubkey;
-      hopPositions[hop] = { lat: withLoc[0].lat, lon: withLoc[0].lon };
+
+      // Affinity-aware: prefer candidates that are neighbors of the next hop
+      const picked = pickByAffinity(withLoc, nextResolvedPubkey, nextPos, null, null);
+      r.name = picked.name;
+      r.pubkey = picked.pubkey;
+      hopPositions[hop] = { lat: picked.lat, lon: picked.lon };
      nextPos = hopPositions[hop];
+      nextResolvedPubkey = picked.pubkey;
    }

    // Sanity check: drop hops impossibly far from neighbors
@@ -203,5 +242,28 @@ window.HopResolver = (function() {
    return nodesList.length > 0;
  }

-  return { init: init, resolve: resolve, ready: ready, haversineKm: haversineKm };
+  /**
+   * Load neighbor-graph affinity data.
+   * @param {Object} graph - { edges: [{source, target, score, weight}, ...] }
+   */
+  function setAffinity(graph) {
+    affinityMap = {};
+    if (!graph || !graph.edges) return;
+    for (const e of graph.edges) {
+      if (!affinityMap[e.source]) affinityMap[e.source] = {};
+      affinityMap[e.source][e.target] = e.score || e.weight || 1;
+      if (!affinityMap[e.target]) affinityMap[e.target] = {};
+      affinityMap[e.target][e.source] = e.score || e.weight || 1;
+    }
+  }
+
+  /**
+   * Get the affinity score between two pubkeys (0 if not neighbors).
+   */
+  function getAffinity(pubkeyA, pubkeyB) {
+    if (!pubkeyA || !pubkeyB || !affinityMap[pubkeyA]) return 0;
+    return affinityMap[pubkeyA][pubkeyB] || 0;
+  }
+
+  return { init: init, resolve: resolve, ready: ready, haversineKm: haversineKm, setAffinity: setAffinity, getAffinity: getAffinity };
 })();
@@ -43,6 +43,7 @@
    timelineScope: 3600000, // 1h default ms
    timelineTimestamps: [], // historical timestamps from DB for sparkline
    timelineFetchedScope: 0, // last fetched scope to avoid redundant fetches
+    replayGen: 0,            // generation counter — incremented on each replay/rewind to discard stale async results
  };

  // ROLE_COLORS loaded from shared roles.js (includes 'unknown')
@@ -116,6 +117,7 @@

  function vcrResumeLive() {
    stopReplay();
+    VCR.replayGen++; // invalidate any in-flight async chunk processing
    VCR.playhead = -1;
    VCR.speed = 1;
    VCR.missedCount = 0;
@@ -142,6 +144,8 @@
  function vcrReplayFromTs(targetTs) {
    const fetchFrom = new Date(targetTs).toISOString();
    stopReplay();
+    VCR.replayGen++;
+    var gen = VCR.replayGen;
    vcrSetMode('REPLAY');

    // Reload map nodes to match the replay time
@@ -153,7 +157,10 @@
      .then(r => r.json())
      .then(data => {
        const pkts = data.packets || [];
-        const replayEntries = expandToBufferEntries(pkts);
+        return expandToBufferEntriesAsync(pkts);
+      })
+      .then(function(replayEntries) {
+        if (gen !== VCR.replayGen) return; // stale async result — user changed mode
        if (replayEntries.length === 0) {
          vcrSetMode('PAUSED');
          return;
@@ -202,6 +209,8 @@

  function vcrRewind(ms) {
    stopReplay();
+    VCR.replayGen++;
+    var gen = VCR.replayGen;
    // Fetch packets from DB for the time window
    const now = Date.now();
    const from = new Date(now - ms).toISOString();
@@ -212,8 +221,11 @@
        // Prepend to buffer (avoid duplicates by ID)
        const existingIds = new Set(VCR.buffer.map(b => b.pkt.id).filter(Boolean));
        const filtered = pkts.filter(p => !existingIds.has(p.id));
-        const newEntries = expandToBufferEntries(filtered);
-        VCR.buffer = [...newEntries, ...VCR.buffer];
+        return expandToBufferEntriesAsync(filtered);
+      })
+      .then(function(newEntries) {
+        if (gen !== VCR.replayGen) return; // stale async result
+        VCR.buffer = [].concat(newEntries, VCR.buffer);
        VCR.playhead = 0;
        VCR.speed = 1;
        vcrSetMode('REPLAY');
@@ -274,15 +286,18 @@
    // Get timestamp of last packet in buffer to fetch the next page
    const last = VCR.buffer[VCR.buffer.length - 1];
    if (!last) return Promise.resolve(false);
+    var gen = VCR.replayGen;
    const since = new Date(last.ts + 1).toISOString(); // +1ms to avoid dupe
    return fetch(`/api/packets?limit=10000&grouped=false&expand=observations&since=${encodeURIComponent(since)}&order=asc`)
      .then(r => r.json())
      .then(data => {
        const pkts = data.packets || [];
        if (pkts.length === 0) return false;
-        const newEntries = expandToBufferEntries(pkts);
-        VCR.buffer = VCR.buffer.concat(newEntries);
-        return true;
+        return expandToBufferEntriesAsync(pkts).then(function(newEntries) {
+          if (gen !== VCR.replayGen) return false; // stale
+          VCR.buffer = VCR.buffer.concat(newEntries);
+          return true;
+        });
      })
      .catch(() => false);
  }
@@ -449,11 +464,53 @@
  }

  // Expand a DB packet (with optional observations[]) into VCR buffer entries
+  /**
+   * Process packets into buffer entries in chunks to avoid blocking the main thread.
+   * Returns a Promise that resolves with the entries array.
+   * Each chunk processes CHUNK_SIZE packets, then yields to the event loop via setTimeout(0).
+   */
+  var VCR_CHUNK_SIZE = 200;
+  function expandToBufferEntriesAsync(pkts) {
+    return new Promise(function(resolve) {
+      var entries = [];
+      var i = 0;
+      function processChunk() {
+        var end = Math.min(i + VCR_CHUNK_SIZE, pkts.length);
+        for (; i < end; i++) {
+          var p = pkts[i];
+          if (p.observations && p.observations.length > 0) {
+            for (var j = 0; j < p.observations.length; j++) {
+              var obs = p.observations[j];
+              entries.push({
+                ts: new Date(obs.timestamp || p.timestamp || p.created_at).getTime(),
+                pkt: dbPacketToLive(Object.assign({}, p, obs, { hash: p.hash, raw_hex: p.raw_hex, decoded_json: p.decoded_json }))
+              });
+            }
+          } else {
+            entries.push({
+              ts: new Date(p.timestamp || p.created_at).getTime(),
+              pkt: dbPacketToLive(p)
+            });
+          }
+        }
+        if (i < pkts.length) {
+          setTimeout(processChunk, 0);
+        } else {
+          resolve(entries);
+        }
+      }
+      processChunk();
+    });
+  }
+
+  // Synchronous version kept for small datasets and backward compat (tests)
  function expandToBufferEntries(pkts) {
-    const entries = [];
-    for (const p of pkts) {
+    var entries = [];
+    for (var k = 0; k < pkts.length; k++) {
+      var p = pkts[k];
      if (p.observations && p.observations.length > 0) {
-        for (const obs of p.observations) {
+        for (var j = 0; j < p.observations.length; j++) {
+          var obs = p.observations[j];
          entries.push({
            ts: new Date(obs.timestamp || p.timestamp || p.created_at).getTime(),
            pkt: dbPacketToLive(Object.assign({}, p, obs, { hash: p.hash, raw_hex: p.raw_hex, decoded_json: p.decoded_json }))
@@ -1359,9 +1416,29 @@
      const _el2 = document.getElementById('liveNodeCount'); if (_el2) _el2.textContent = Object.keys(nodeMarkers).length;
      // Initialize shared HopResolver with loaded nodes
      if (window.HopResolver) HopResolver.init(list);
+      // Fetch affinity data for hop disambiguation
+      fetchAffinityData();
+      startAffinityRefresh();
    } catch (e) { console.error('Failed to load nodes:', e); }
  }

+  let _affinityInterval = null;
+
+  async function fetchAffinityData() {
+    try {
+      const resp = await fetch('/api/analytics/neighbor-graph');
+      const graph = await resp.json();
+      if (window.HopResolver && HopResolver.setAffinity) {
+        HopResolver.setAffinity(graph);
+      }
+    } catch (e) { console.warn('Failed to fetch affinity data:', e); }
+  }
+
+  function startAffinityRefresh() {
+    if (_affinityInterval) clearInterval(_affinityInterval);
+    _affinityInterval = setInterval(fetchAffinityData, 60000);
+  }
+
  function clearNodeMarkers() {
    if (nodesLayer) nodesLayer.clearLayers();
    if (animLayer) animLayer.clearLayers();
@@ -1597,6 +1674,7 @@
  window._vcrFormatTime = vcrFormatTime;
  window._liveDbPacketToLive = dbPacketToLive;
  window._liveExpandToBufferEntries = expandToBufferEntries;
+  window._liveExpandToBufferEntriesAsync = expandToBufferEntriesAsync;
  window._liveSEG_MAP = SEG_MAP;
  window._liveBufferPacket = bufferPacket;
  window._liveVCR = function() { return VCR; };
@@ -2552,6 +2630,7 @@
    if (_lcdClockInterval) { clearInterval(_lcdClockInterval); _lcdClockInterval = null; }
    if (_rateCounterInterval) { clearInterval(_rateCounterInterval); _rateCounterInterval = null; }
    if (_pruneInterval) { clearInterval(_pruneInterval); _pruneInterval = null; }
+    if (_affinityInterval) { clearInterval(_affinityInterval); _affinityInterval = null; }
    if (ws) { ws.onclose = null; ws.close(); ws = null; }
    if (map) { map.remove(); map = null; }
    if (_onResize) {
@@ -2584,7 +2663,7 @@
    packetCount = 0; activeAnims = 0;
    nodeActivity = {}; pktTimestamps = [];
    feedDedup.clear();
-    VCR.buffer = []; VCR.playhead = -1; VCR.mode = 'LIVE'; VCR.missedCount = 0; VCR.speed = 1;
+    VCR.buffer = []; VCR.playhead = -1; VCR.mode = 'LIVE'; VCR.missedCount = 0; VCR.speed = 1; VCR.replayGen = 0;
  }

  let _themeRefreshHandler = null;
@@ -10,7 +10,7 @@
 */

 window.getParsedPath = function getParsedPath(p) {
-  if (p._parsedPath !== undefined) return p._parsedPath;
+  if (p._parsedPath !== undefined) return p._parsedPath || [];
  var raw = p.path_json;
  if (typeof raw !== 'string') {
    p._parsedPath = Array.isArray(raw) ? raw : [];
@@ -32,7 +32,7 @@ window.clearParsedCache = function clearParsedCache(p) {
 };

 window.getParsedDecoded = function getParsedDecoded(p) {
-  if (p._parsedDecoded !== undefined) return p._parsedDecoded;
+  if (p._parsedDecoded !== undefined) return p._parsedDecoded || {};
  var raw = p.decoded_json;
  if (typeof raw !== 'string') {
    p._parsedDecoded = (raw && typeof raw === 'object') ? raw : {};
@@ -53,6 +53,7 @@
  let _displayPackets = [];       // filtered packets for current view
  let _displayGrouped = false;    // whether _displayPackets is in grouped mode
  let _rowCounts = [];            // per-entry DOM row counts (1 for flat, 1+children for expanded groups)
+  let _rowCountsDirty = false;    // set when _rowCounts may be stale (e.g. WS added children) (#410)
  let _cumulativeOffsetsCache = null; // cached cumulative offsets, invalidated on _rowCounts change
  let _lastVisibleStart = -1;     // last rendered start index (for dirty checking)
  let _lastVisibleEnd = -1;       // last rendered end index (for dirty checking)
@@ -357,7 +358,7 @@
          if (pktTime && pktTime < cutoff) return false;
        }
        if (filters.type) { const types = filters.type.split(',').map(Number); if (!types.includes(p.payload_type)) return false; }
-        if (filters.observer) { const obsSet = new Set(filters.observer.split(',')); if (!obsSet.has(p.observer_id)) return false; }
+        if (filters.observer) { const obsSet = new Set(filters.observer.split(',')); if (!obsSet.has(p.observer_id) && !(p._children && p._children.some(c => obsSet.has(String(c.observer_id))))) return false; }
        if (filters.hash && p.hash !== filters.hash) return false;
        if (RegionFilter.getRegionParam()) {
          const selectedRegions = RegionFilter.getRegionParam().split(',');
@@ -396,6 +397,9 @@
                existing._children.unshift(p);
                if (existing._children.length > 200) existing._children.length = 200;
                sortGroupChildren(existing);
+                // Invalidate row counts — child count changed, so virtual scroll
+                // heights are stale until next renderTableRows() (#410)
+                _invalidateRowCounts();
              }
            } else {
              // New group
@@ -442,6 +446,7 @@
    clearTimeout(_wsRenderTimer);
    _displayPackets = [];
    _rowCounts = [];
+    _rowCountsDirty = false;
    _cumulativeOffsetsCache = null;
    _observerFilterSet = null;
    _lastVisibleStart = -1;
@@ -488,6 +493,7 @@
      if (regionParam) params.set('region', regionParam);
      if (filters.hash) params.set('hash', filters.hash);
      if (filters.node) params.set('node', filters.node);
+      if (filters.observer) params.set('observer', filters.observer);
      params.set('groupByHash', 'true'); // always fetch grouped

      const data = await api('/packets?' + params.toString());
@@ -1099,8 +1105,8 @@

  // Build HTML for a single flat (ungrouped) packet row
  function buildFlatRowHtml(p) {
-    const decoded = getParsedDecoded(p);
-    const pathHops = getParsedPath(p);
+    const decoded = getParsedDecoded(p) || {};
+    const pathHops = getParsedPath(p) || [];
    const region = p.observer_id ? (observerMap.get(p.observer_id)?.iata || '') : '';
    const typeName = payloadTypeName(p.payload_type);
    const typeClass = payloadTypeColor(p.payload_type);
@@ -1122,6 +1128,21 @@
      </tr>`;
  }

+  // Mark _rowCounts as stale so renderVisibleRows() recomputes them lazily.
+  // Called when expanded group children change outside renderTableRows() (#410).
+  function _invalidateRowCounts() {
+    _rowCountsDirty = true;
+    _cumulativeOffsetsCache = null;
+  }
+
+  // Recompute _rowCounts from _displayPackets if they've been invalidated.
+  function _refreshRowCountsIfDirty() {
+    if (!_rowCountsDirty || !_displayPackets.length) return;
+    _rowCounts = _displayPackets.map(function(p) { return _getRowCount(p); });
+    _cumulativeOffsetsCache = null;
+    _rowCountsDirty = false;
+  }
+
  // Compute the number of DOM <tr> rows a single entry produces.
  // Used by both row counting and renderVisibleRows to avoid divergence (#424).
  function _getRowCount(p) {
@@ -1160,6 +1181,9 @@
    const scrollContainer = document.getElementById('pktLeft');
    if (!scrollContainer) return;

+    // Recompute row counts if they were invalidated (e.g. WS added children) (#410)
+    _refreshRowCountsIfDirty();
+
    // Compute total DOM rows accounting for expanded groups
    const offsets = _cumulativeRowOffsets();
    const totalDomRows = offsets[offsets.length - 1];
@@ -1291,7 +1315,11 @@
    }
    if (filters.observer) {
      const obsIds = new Set(filters.observer.split(','));
-      displayPackets = displayPackets.filter(p => obsIds.has(p.observer_id));
+      displayPackets = displayPackets.filter(p => {
+        if (obsIds.has(p.observer_id)) return true;
+        if (p._children) return p._children.some(c => obsIds.has(String(c.observer_id)));
+        return false;
+      });
    }

    // Packet Filter Language
@@ -1312,6 +1340,7 @@
    if (!displayPackets.length) {
      _displayPackets = [];
      _rowCounts = [];
+      _rowCountsDirty = false;
      _cumulativeOffsetsCache = null;
      _observerFilterSet = null;
      _lastVisibleStart = -1;
@@ -1331,6 +1360,7 @@
    _displayGrouped = groupByHash;
    _observerFilterSet = filters.observer ? new Set(filters.observer.split(',')) : null;
    _rowCounts = displayPackets.map(p => _getRowCount(p));
+    _rowCountsDirty = false;
    _cumulativeOffsetsCache = null;

    attachVScrollListener();
@@ -1436,8 +1466,8 @@
    const pkt = data.packet;
    const breakdown = data.breakdown || {};
    const ranges = breakdown.ranges || [];
-    const decoded = getParsedDecoded(pkt);
-    const pathHops = getParsedPath(pkt);
+    const decoded = getParsedDecoded(pkt) || {};
+    const pathHops = getParsedPath(pkt) || [];

    // Resolve sender GPS — from packet directly, or from known node in DB
    let senderLat = decoded.lat != null ? decoded.lat : (decoded.latitude || null);
@@ -2039,6 +2069,8 @@
      renderPath,
      _getRowCount,
      _cumulativeRowOffsets,
+      _invalidateRowCounts,
+      _refreshRowCountsIfDirty,
      buildGroupRowHtml,
      buildFlatRowHtml,
    };
@@ -947,7 +947,9 @@ button.ch-item.selected { background: var(--selected-bg); }
  .filter-bar { flex-direction: row; flex-wrap: wrap; gap: 4px; }
  .filter-toggle-btn { display: inline-flex !important; }
  .filter-bar > *:not(.filter-toggle-btn):not(.col-toggle-wrap) { display: none; }
-  .filter-bar.filters-expanded > * { display: inline-flex; }
+  /* Must match :not() specificity of the hide rule above, otherwise .filters-expanded loses
+     the specificity battle and filter children stay hidden (see issue #534). */
+  .filter-bar.filters-expanded > *:not(.filter-toggle-btn):not(.col-toggle-wrap) { display: inline-flex; }
  .filter-bar.filters-expanded > .col-toggle-wrap { display: inline-block; }
  .filter-bar.filters-expanded input { width: 100%; }
  .filter-bar.filters-expanded select { width: 100%; }
@@ -1573,6 +1573,47 @@ async function run() {

  // ─── End affinity debug tests ─────────────────────────────────────────────

+  // ─── Mobile filter dropdown tests (#534) ──────────────────────────────────
+
+  await test('Mobile: filter toggle expands filter bar on packets page (#534)', async () => {
+    // Use a mobile viewport
+    await page.setViewportSize({ width: 480, height: 800 });
+    await page.goto(`${BASE}/#/packets`);
+    await page.waitForTimeout(500);
+
+    const filterBar = await page.$('.filter-bar');
+    assert(filterBar, 'Filter bar should exist on packets page');
+
+    // Before clicking toggle, filter inputs should be hidden
+    const toggleBtn = await page.$('.filter-toggle-btn');
+    assert(toggleBtn, 'Filter toggle button should exist on mobile');
+
+    await toggleBtn.click();
+    await page.waitForTimeout(300);
+
+    // After clicking, .filters-expanded should be on the filter bar
+    const expanded = await filterBar.evaluate(el => el.classList.contains('filters-expanded'));
+    assert(expanded, 'Filter bar should have filters-expanded class after toggle');
+
+    // Filter inputs should now be visible
+    const filterInput = await page.$('.filter-bar input');
+    if (filterInput) {
+      const display = await filterInput.evaluate(el => getComputedStyle(el).display);
+      assert(display !== 'none', `Filter input should be visible when expanded, got display: ${display}`);
+    }
+
+    const filterSelect = await page.$('.filter-bar select');
+    if (filterSelect) {
+      const display = await filterSelect.evaluate(el => getComputedStyle(el).display);
+      assert(display !== 'none', `Filter select should be visible when expanded, got display: ${display}`);
+    }
+
+    // Reset viewport
+    await page.setViewportSize({ width: 1280, height: 720 });
+  });
+
+  // ─── End mobile filter tests ──────────────────────────────────────────────
+
  // Extract frontend coverage if instrumented server is running
  try {
    const coverage = await page.evaluate(() => window.__coverage__);
@@ -2695,6 +2695,63 @@ console.log('\n=== packets.js: savedTimeWindowMin defaults ===');
      'buildGroupRowHtml should use hoisted _observerFilterSet');
  });

+  test('observer filter in grouped mode includes packet when child matches (#537)', () => {
+    // The display filter should keep a grouped packet whose primary observer_id
+    // does NOT match, but one of its _children does.
+    const obsIds = new Set(['OBS_B']);
+    const packets = [
+      { observer_id: 'OBS_A', _children: [{ observer_id: 'OBS_A' }, { observer_id: 'OBS_B' }] },
+      { observer_id: 'OBS_C', _children: [{ observer_id: 'OBS_C' }] },
+    ];
+    const result = packets.filter(p => {
+      if (obsIds.has(p.observer_id)) return true;
+      if (p._children) return p._children.some(c => obsIds.has(String(c.observer_id)));
+      return false;
+    });
+    assert.strictEqual(result.length, 1, 'should keep packet with matching child observer');
+    assert.strictEqual(result[0].observer_id, 'OBS_A');
+  });
+
+  test('observer filter in grouped mode hides packet with no matching observations (#537)', () => {
+    const obsIds = new Set(['OBS_X']);
+    const packets = [
+      { observer_id: 'OBS_A', _children: [{ observer_id: 'OBS_A' }, { observer_id: 'OBS_B' }] },
+    ];
+    const result = packets.filter(p => {
+      if (obsIds.has(p.observer_id)) return true;
+      if (p._children) return p._children.some(c => obsIds.has(String(c.observer_id)));
+      return false;
+    });
+    assert.strictEqual(result.length, 0, 'should hide packet with no matching observers');
+  });
+
+  test('WS observer filter checks children for grouped packets (#537)', () => {
+    const filters = { observer: 'OBS_B' };
+    const obsSet = new Set(filters.observer.split(','));
+    const p = { observer_id: 'OBS_A', _children: [{ observer_id: 'OBS_B' }] };
+    const passes = obsSet.has(p.observer_id) || (p._children && p._children.some(c => obsSet.has(String(c.observer_id))));
+    assert.ok(passes, 'WS filter should pass grouped packet when child matches');
+
+    const p2 = { observer_id: 'OBS_C', _children: [{ observer_id: 'OBS_D' }] };
+    const passes2 = obsSet.has(p2.observer_id) || (p2._children && p2._children.some(c => obsSet.has(String(c.observer_id))));
+    assert.ok(!passes2, 'WS filter should reject grouped packet with no matching observers');
+  });
+
+  test('packets.js display filter checks _children for observer match (#537)', () => {
+    // Verify the actual source code has the children check
+    assert.ok(
+      packetsSource.includes('p._children) return p._children.some(c => obsIds.has(String(c.observer_id))'),
+      'display filter should check _children for observer match'
+    );
+  });
+
+  test('packets.js WS filter checks _children for observer match (#537)', () => {
+    assert.ok(
+      packetsSource.includes('p._children && p._children.some(c => obsSet.has(String(c.observer_id)))'),
+      'WS filter should check _children for observer match'
+    );
+  });
+
  test('buildFlatRowHtml has null-safe decoded_json', () => {
    const flatBuilderMatch = packetsSource.match(/function buildFlatRowHtml[\s\S]*?(?=\n  function )/);
    assert.ok(flatBuilderMatch, 'buildFlatRowHtml should exist');
@@ -4126,7 +4183,17 @@ console.log('\n=== app.js: routeTypeName/payloadTypeName edge cases ===');
    assertJsonEqual(getParsedPath(p), []);
  });

+  test('getParsedPath: cached null _parsedPath returns empty array (#538)', () => {
+    const p = { path_json: '["a"]', _parsedPath: null };
+    assertJsonEqual(getParsedPath(p), []);
+  });
+
  // --- getParsedDecoded ---
+  test('getParsedDecoded: cached null _parsedDecoded returns empty object (#538)', () => {
+    const p = { decoded_json: '{"x":1}', _parsedDecoded: null };
+    assertJsonEqual(getParsedDecoded(p), {});
+  });
+
  test('getParsedDecoded: valid JSON object', () => {
    const p = { decoded_json: '{"type":"GRP_TXT","text":"hello"}' };
    const result = getParsedDecoded(p);
@@ -0,0 +1,99 @@
+/**
+ * Unit tests for HopResolver affinity-aware hop resolution.
+ */
+'use strict';
+const fs = require('fs');
+const vm = require('vm');
+
+// Load hop-resolver.js in a sandboxed context
+const code = fs.readFileSync(__dirname + '/public/hop-resolver.js', 'utf8');
+const sandbox = { window: {}, console, Math, Object, Array, Number, Date, Map, Set, parseInt, parseFloat, encodeURIComponent };
+vm.createContext(sandbox);
+vm.runInContext(code, sandbox);
+const HopResolver = sandbox.window.HopResolver;
+
+let passed = 0;
+let failed = 0;
+
+function assert(condition, msg) {
+  if (condition) { passed++; console.log('  ✓ ' + msg); }
+  else { failed++; console.error('  ✗ ' + msg); }
+}
+
+// ── Test nodes ──
+// Two nodes share the same 1-byte prefix "ab"
+const nodeA = { public_key: 'ab1111', name: 'NodeA', lat: 37.0, lon: -122.0 };
+const nodeB = { public_key: 'ab2222', name: 'NodeB', lat: 38.0, lon: -123.0 };
+const nodeC = { public_key: 'cd3333', name: 'NodeC', lat: 37.5, lon: -122.5 };
+
+console.log('\n=== HopResolver Affinity Tests ===\n');
+
+// Test 1: Affinity prefers neighbor candidate over geo-closest
+console.log('Test 1: Affinity prefers neighbor over geo-closest');
+HopResolver.init([nodeA, nodeB, nodeC]);
+HopResolver.setAffinity({
+  edges: [
+    { source: 'cd3333', target: 'ab2222', score: 0.8 }
+    // NodeC is a neighbor of NodeB but NOT NodeA
+  ]
+});
+
+// Resolve hop "ab" after NodeC was resolved — should pick NodeB (neighbor) not NodeA (geo-closer)
+// Origin at NodeC's position so forward pass runs with NodeC as anchor
+const result1 = HopResolver.resolve(['cd33', 'ab'], nodeC.lat, nodeC.lon, null, null, null);
+assert(result1['ab'].name === 'NodeB', 'Should pick NodeB (affinity neighbor of NodeC) — got: ' + result1['ab'].name);
+
+// Test 2: Without affinity, falls back to geo-closest
+console.log('\nTest 2: Cold start (no affinity) falls back to geo-closest');
+HopResolver.init([nodeA, nodeB, nodeC]);
+HopResolver.setAffinity({}); // No edges
+
+// With anchor at NodeC's position, NodeA is closer to NodeC than NodeB
+const result2 = HopResolver.resolve(['cd33', 'ab'], nodeC.lat, nodeC.lon, null, null, null);
+// NodeA (37, -122) is closer to NodeC (37.5, -122.5) than NodeB (38, -123)
+assert(result2['ab'].name === 'NodeA', 'Should pick NodeA (geo-closest) — got: ' + result2['ab'].name);
+
+// Test 3: setAffinity with null/undefined doesn't crash
+console.log('\nTest 3: setAffinity with null/undefined is safe');
+HopResolver.setAffinity(null);
+HopResolver.setAffinity(undefined);
+HopResolver.setAffinity({});
+assert(true, 'No crash on null/undefined/empty affinity');
+
+// Test 4: getAffinity returns correct scores
+console.log('\nTest 4: getAffinity returns correct scores');
+HopResolver.setAffinity({
+  edges: [
+    { source: 'aaa', target: 'bbb', score: 0.95 },
+    { source: 'ccc', target: 'ddd', weight: 5 }
+  ]
+});
+assert(HopResolver.getAffinity('aaa', 'bbb') === 0.95, 'aaa→bbb = 0.95');
+assert(HopResolver.getAffinity('bbb', 'aaa') === 0.95, 'bbb→aaa = 0.95 (bidirectional)');
+assert(HopResolver.getAffinity('ccc', 'ddd') === 5, 'ccc→ddd = 5 (weight fallback)');
+assert(HopResolver.getAffinity('aaa', 'zzz') === 0, 'unknown pair = 0');
+assert(HopResolver.getAffinity(null, 'bbb') === 0, 'null pubkey = 0');
+
+// Test 5: Affinity with multiple neighbors — highest score wins
+console.log('\nTest 5: Highest affinity score wins among neighbors');
+HopResolver.init([nodeA, nodeB, nodeC]);
+HopResolver.setAffinity({
+  edges: [
+    { source: 'cd3333', target: 'ab1111', score: 0.3 },
+    { source: 'cd3333', target: 'ab2222', score: 0.9 }
+  ]
+});
+const result5 = HopResolver.resolve(['cd33', 'ab'], nodeC.lat, nodeC.lon, null, null, null);
+assert(result5['ab'].name === 'NodeB', 'Should pick NodeB (highest affinity 0.9) — got: ' + result5['ab'].name);
+
+// Test 6: Unambiguous hops are not affected by affinity
+console.log('\nTest 6: Unambiguous hops unaffected by affinity');
+const nodeD = { public_key: 'ee4444', name: 'NodeD', lat: 36.0, lon: -121.0 };
+HopResolver.init([nodeA, nodeB, nodeC, nodeD]);
+HopResolver.setAffinity({ edges: [] });
+const result6 = HopResolver.resolve(['ee44'], null, null, null, null, null);
+assert(result6['ee44'].name === 'NodeD', 'Unique prefix resolves directly — got: ' + result6['ee44'].name);
+assert(!result6['ee44'].ambiguous, 'Should not be marked ambiguous');
+
+console.log('\n' + (passed + failed) + ' tests, ' + passed + ' passed, ' + failed + ' failed\n');
+process.exit(failed > 0 ? 1 : 0);
@@ -272,6 +272,48 @@ console.log('\n=== live.js: expandToBufferEntries ===');
  });
 }

+// ===== expandToBufferEntriesAsync (chunked, non-blocking) =====
+console.log('\n=== live.js: expandToBufferEntriesAsync ===');
+{
+  // Build a sandbox with packet-helpers loaded so expandToBufferEntries can call dbPacketToLive
+  const ctx = makeSandbox();
+  addLiveGlobals(ctx);
+  loadInCtx(ctx, 'public/roles.js');
+  loadInCtx(ctx, 'public/packet-helpers.js');
+  try { loadInCtx(ctx, 'public/live.js'); } catch (e) {
+    for (const k of Object.keys(ctx.window)) ctx[k] = ctx.window[k];
+  }
+  const expandSync = ctx.window._liveExpandToBufferEntries;
+  const expandAsync = ctx.window._liveExpandToBufferEntriesAsync;
+  assert.ok(expandAsync, '_liveExpandToBufferEntriesAsync must be exposed');
+
+  const pkts = [];
+  for (let i = 0; i < 500; i++) {
+    pkts.push({
+      id: i, hash: 'h' + i, timestamp: new Date(1700000000000 + i * 1000).toISOString(),
+      decoded_json: '{"type":"GRP_TXT"}', path_json: '[]',
+      observations: [
+        { timestamp: new Date(1700000000000 + i * 1000 + 100).toISOString(), snr: 5, observer_name: 'O1' },
+        { timestamp: new Date(1700000000000 + i * 1000 + 200).toISOString(), snr: 8, observer_name: 'O2' },
+      ],
+    });
+  }
+
+  test('sync expand handles 500 packets (1000 entries) correctly', () => {
+    const result = expandSync(pkts);
+    assert.strictEqual(result.length, 1000, '500 packets * 2 observations = 1000 entries');
+    assert.strictEqual(result[0].pkt.hash, 'h0');
+    assert.strictEqual(result[999].pkt.hash, 'h499');
+  });
+
+  test('VCR_CHUNK_SIZE is defined and async function yields via setTimeout', () => {
+    const src = fs.readFileSync(__dirname + '/public/live.js', 'utf8');
+    assert.ok(src.includes('VCR_CHUNK_SIZE'), 'VCR_CHUNK_SIZE constant must exist');
+    assert.ok(src.includes('expandToBufferEntriesAsync'), 'async version must exist');
+    assert.ok(src.includes('setTimeout(processChunk, 0)'), 'must yield via setTimeout between chunks');
+  });
+}
+
 // ===== SEG_MAP (7-segment display) =====
 console.log('\n=== live.js: SEG_MAP ===');
 {
@@ -757,6 +757,33 @@ console.log('\n=== packets.js: page registration ===');
  });
 }

+console.log('\n=== packets.js: _invalidateRowCounts / _refreshRowCountsIfDirty (#410) ===');
+{
+  const ctx = loadPacketsSandbox();
+  const api = ctx._packetsTestAPI;
+
+  test('_invalidateRowCounts and _refreshRowCountsIfDirty are exported', () => {
+    assert(typeof api._invalidateRowCounts === 'function');
+    assert(typeof api._refreshRowCountsIfDirty === 'function');
+  });
+
+  test('_invalidateRowCounts does not throw', () => {
+    api._invalidateRowCounts();
+  });
+
+  test('_refreshRowCountsIfDirty does not throw when no display packets', () => {
+    api._invalidateRowCounts();
+    api._refreshRowCountsIfDirty();
+  });
+
+  test('_cumulativeRowOffsets returns valid offsets after invalidation cycle', () => {
+    // Even with no display packets, should return valid array
+    const offsets = api._cumulativeRowOffsets();
+    assert(Array.isArray(offsets));
+    assert(offsets[0] === 0);
+  });
+}
+
 // ===== SUMMARY =====
 console.log(`\n${'='.repeat(40)}`);
 console.log(`packets.js tests: ${passed} passed, ${failed} failed`);
Author	SHA1	Message	Date
you	4f265b312d	docs: add startup load mechanics, in-memory structure, and dual-write pattern to resolved-path spec	2026-04-04 04:07:21 +00:00
you	5a959093fe	docs: address ambiguities — canonical edge ordering, resolution/edge timing	2026-04-04 03:45:03 +00:00
you	d259076285	docs: finalize resolved-path spec — persisted graph, observation column, all consumers	2026-04-04 03:44:08 +00:00
you	6dc4a21a1f	docs: update resolved-path spec with persisted graph + finalized design	2026-04-04 03:17:27 +00:00
you	507ed19d0e	docs: link issue #555 in resolved-path spec	2026-04-04 00:51:14 +00:00
you	0c93c2f548	docs: add resolved_path spec — server-side hop resolution at ingest	2026-04-04 00:50:47 +00:00
Kpa-clawbot	412a8fdb8f	feat: live map uses affinity-aware hop resolution (#528 ) (#550 ) ## Summary Augments the shared `HopResolver` with neighbor-graph affinity data so that when multiple nodes match a hop prefix, the resolver prefers candidates that are known neighbors of the adjacent hop — instead of relying solely on geo-distance. Fixes #528 ## Changes ### `public/hop-resolver.js` - Added `affinityMap` — stores bidirectional neighbor adjacency with scores - Added `setAffinity(graph)` — ingests `/api/analytics/neighbor-graph` edge data into O(1) Map lookups - Added `getAffinity(pubkeyA, pubkeyB)` — returns affinity score between two nodes (0 if not neighbors) - Added `pickByAffinity(candidates, adjacentPubkey, anchor, ...)` — picks best candidate: affinity-neighbor first (highest score), then geo-distance fallback - Modified forward and backward passes in `resolve()` to track the previously-resolved pubkey and use `pickByAffinity` instead of raw geo-sort ### `public/live.js` - Added `fetchAffinityData()` — fetches `/api/analytics/neighbor-graph` once and calls `HopResolver.setAffinity()` - Added `startAffinityRefresh()` — refreshes affinity data every 60 seconds - Both are called from `loadNodes()` after HopResolver is initialized ### `test-hop-resolver-affinity.js` (new) - Affinity prefers neighbor candidate over geo-closest - Cold start (no affinity data) falls back to geo-closest - Null/undefined affinity doesn't crash - Bidirectional score lookup - Highest affinity score wins among multiple neighbors - Unambiguous hops unaffected by affinity ## Performance - API calls: 1 at load + 1 per 60s (no per-packet calls) - Per-packet resolve: O(1) Map lookups, <0.5ms - Memory: ~50KB for 2K-node graph --------- Co-authored-by: you <you@example.com>	2026-04-03 16:32:53 -07:00
Kpa-clawbot	9a39198d92	fix: only count repeaters in hash collision analysis (#441 ) (#548 ) Fixes #441 ## Summary Hash collision analysis was including ALL node types, inflating collision counts with irrelevant data. Per MeshCore firmware analysis, only repeaters matter for collision analysis — they're the only role that forwards packets and appears in routing `path[]` arrays. ## Root Causes Fixed 1. `hash_size==0` nodes counted in all buckets — nodes with unknown hash size were included via `cn.HashSize == bytes \|\| cn.HashSize == 0`, polluting every bucket 2. Non-repeater roles included — companions, rooms, sensors, and observers were counted even though their hash collisions never cause routing ambiguity ## Fix Changed `computeHashCollisions()` filter from: ```go // Before: include everything except companions if cn.HashSize == bytes && cn.Role != "companion" { ``` To: ```go // After: only include repeaters (per firmware analysis) if cn.HashSize == bytes && cn.Role == "repeater" { ``` ## Why only repeaters? From [MeshCore firmware analysis](https://github.com/Kpa-clawbot/CoreScope/issues/441#issuecomment-4185218547): - Only repeaters override `allowPacketForward()` to return `true` - Only repeaters append their hash to `path[]` during relay - Companions, rooms, sensors, observers never forward packets - Cross-role collisions are benign (companion silently drops, real repeater still forwards) ## Tests - `TestHashCollisionsOnlyRepeaters` — verifies companions, rooms, sensors, and hash_size==0 nodes are all excluded --------- Co-authored-by: you <you@example.com>	2026-04-03 14:23:13 -07:00
Kpa-clawbot	526ea8a1fc	perf(live): chunk VCR replay packet processing to avoid UI freezes (#549 ) ## Summary VCR replay functions (`vcrReplayFromTs`, `vcrRewind`, `fetchNextReplayPage`) fetch up to 10K packets and process them all synchronously on the main thread via `expandToBufferEntries`, causing multi-second UI freezes — especially on mobile. ## Fix - Added `expandToBufferEntriesAsync()` — processes packets in chunks of 200, yielding to the event loop via `setTimeout(0)` between chunks - Updated all three VCR replay callers to use the async variant - Kept the synchronous `expandToBufferEntries()` for backward compatibility (tests, small datasets) - Exposed `_liveExpandToBufferEntriesAsync` on window for test access ## Perf justification - Before: 10K packets × ~2 observations = 20K+ objects created synchronously, blocking the main thread for 1-3 seconds on mobile - After: Same work split into chunks of 200 packets (~400 entries) with event loop yields between chunks. Each chunk takes <5ms, keeping the UI responsive (well under the 16ms frame budget) - Chunk size of 200 is tunable via `VCR_CHUNK_SIZE` ## Tests - Added regression test: sync expand correctness at scale (500 packets → 1000 entries) - Added structural test: verifies `VCR_CHUNK_SIZE` exists and async function yields via `setTimeout` - All existing tests pass (`npm test`) Fixes #395 --------- Co-authored-by: you <you@example.com>	2026-04-03 21:22:05 +00:00
Kpa-clawbot	8e42febc9c	fix: virtual scroll height accounts for expanded group rows (#410 ) (#547 ) ## Summary Fixes #410 — virtual scroll height miscalculation for expanded group rows. ## Root Cause When WebSocket messages add children to an already-expanded packet group, `_rowCounts` becomes stale during the 200ms render debounce window. Scroll events during this window call `renderVisibleRows()` with stale row counts, causing wrong total height, spacer heights, and visible range calculations. ## Changes public/packets.js: - Added `_rowCountsDirty` flag to track when row counts need recomputation - Added `_invalidateRowCounts()` — marks row counts as stale and clears cumulative cache - Added `_refreshRowCountsIfDirty()` — lazily recomputes `_rowCounts` from `_displayPackets` - Called `_invalidateRowCounts()` when WS handler adds children to expanded groups (line ~402) - Called `_refreshRowCountsIfDirty()` at top of `renderVisibleRows()` before using row counts - Reset `_rowCountsDirty` in all cleanup paths (destroy, empty display) test-packets.js: - Added 4 regression tests for `_invalidateRowCounts` / `_refreshRowCountsIfDirty` ## Complexity O(n) recomputation of `_rowCounts` when dirty (same as existing `renderTableRows` path). Only triggers when WS modifies expanded group children, which is infrequent relative to scroll events. Co-authored-by: you <you@example.com>	2026-04-03 13:55:23 -07:00
Kpa-clawbot	59bff5462c	fix: rate-limit cache invalidation to prevent 0% hit rate (#533 ) (#546 ) ## Summary Fixes #533 — server cache hit rate always 0%. ## Root Cause `invalidateCachesFor()` is called at the end of every `IngestNewFromDB()` and `IngestNewObservations()` cycle (~2-5s). Since new data arrives continuously, caches are cleared faster than any analytics request can hit them, resulting in a permanent 0% cache hit rate. The cache TTL (15s/60s) is irrelevant because entries are evicted by invalidation long before they expire. ## Fix Rate-limit cache invalidation with a 10-second cooldown: - First call after cooldown goes through immediately - Subsequent calls during cooldown accumulate dirty flags in `pendingInv` - Next call after cooldown merges pending + current flags and applies them - Eviction bypasses cooldown (data removal requires immediate clearing) Analytics data may be at most ~10s stale, which is acceptable for a dashboard. ## Changes - `store.go`: Added `lastInvalidated`, `pendingInv`, `invCooldown` fields. Refactored `invalidateCachesFor()` to rate-limit non-eviction invalidation. Extracted `applyCacheInvalidation()` helper. - `cache_invalidation_test.go`: Added 4 new tests: - `TestInvalidationRateLimited` — verifies caches survive during cooldown - `TestInvalidationCooldownAccumulatesFlags` — verifies flag merging - `TestEvictionBypassesCooldown` — verifies eviction always clears immediately - `BenchmarkCacheHitDuringIngestion` — confirms 100% hit rate during rapid ingestion (was 0%) ## Perf Proof ``` BenchmarkCacheHitDuringIngestion-16 3467889 1018 ns/op 100.0 hit% ``` Before: 0% hit rate under continuous ingestion. After: 100% hit rate during cooldown periods. Co-authored-by: you <you@example.com>	2026-04-03 13:53:58 -07:00
Kpa-clawbot	8c1cd8a9fe	perf: track advert pubkeys incrementally, eliminate per-request JSON parsing (#360 ) (#544 ) ## Summary `GetPerfStoreStats()` and `GetPerfStoreStatsTyped()` iterated all ADVERT packets and called `json.Unmarshal` on each one — under a read lock — on every `/api/perf` and `/api/health` request. With 5K+ adverts, each health check triggered thousands of JSON parses. ## Fix Added a refcounted `advertPubkeys map[string]int` to `PacketStore` that tracks distinct pubkeys incrementally during `Load()`, `IngestNewFromDB()`, and eviction. The perf/health handlers now just read `len(s.advertPubkeys)` — O(1) with zero allocations. ## Benchmark Results (5K adverts, 200 distinct pubkeys) \| Method \| ns/op \| allocs/op \| \|--------\|-------\|-----------\| \| `GetPerfStoreStatsTyped` \| 78 \| 0 \| \| `GetPerfStoreStats` \| 2,565 \| 9 \| Before this change, both methods performed O(N) JSON unmarshals per call. ## Tests Added - `TestAdvertPubkeyTracking` — verifies incremental tracking through add/evict lifecycle - `TestAdvertPubkeyPublicKeyField` — covers the `public_key` JSON field variant - `TestAdvertPubkeyNonAdvert` — ensures non-ADVERT packets don't affect count - `BenchmarkGetPerfStoreStats` — 5K adverts benchmark - `BenchmarkGetPerfStoreStatsTyped` — 5K adverts benchmark Fixes #360 --------- Co-authored-by: you <you@example.com>	2026-04-03 13:51:13 -07:00
Kpa-clawbot	29e8e37114	fix: mobile filter dropdown specificity prevents expansion (#534 ) (#541 ) ## Summary Fixes #534 — mobile filter dropdown doesn't expand on packets page. ## Root Cause CSS specificity battle in the mobile media query. The hide rule uses `:not()` pseudo-classes which add specificity: ```css /* Higher specificity due to :not() / .filter-bar > :not(.filter-toggle-btn):not(.col-toggle-wrap) { display: none; } /* Lower specificity — loses even with .filters-expanded / .filter-bar.filters-expanded > { display: inline-flex; } ``` The JS toggle correctly adds/removes `.filters-expanded`, but the CSS expanded rule could never win. ## Fix Match the `:not()` selectors in the expanded rule so `.filters-expanded` makes it strictly more specific: ```css .filter-bar.filters-expanded > *:not(.filter-toggle-btn):not(.col-toggle-wrap) { display: inline-flex; } ``` Added a comment explaining the specificity dependency so future devs don't repeat this. ## Tests Added Playwright E2E test: mobile viewport (480×800), navigates to packets page, clicks filter toggle, verifies filter inputs become visible. --------- Co-authored-by: you <you@example.com>	2026-04-03 13:50:10 -07:00
Kpa-clawbot	9b9f396af5	perf: replace O(n²) observation dedup with map-based O(n) (#355 ) (#543 ) ## Summary Fixes #355 — replaces O(n²) observation dedup in `Load()`, `IngestNewFromDB()`, and `IngestNewObservations()` with an O(1) map-based lookup. ## Changes - Added `obsKeys map[string]bool` field to `StoreTx` for O(1) dedup keyed on `observerID + "\|" + pathJSON` - Replaced all 3 linear-scan dedup sites in `store.go` with map lookups - Lazy-init `obsKeys` for transmissions created before this change (in `IngestNewFromDB` and `IngestNewObservations`) - Added regression test (`TestObsDedupCorrectness`) verifying dedup correctness - Added nil-map safety test (`TestObsDedupNilMapSafety`) - Added benchmark comparing map vs linear scan ## Benchmark Results (ARM64, 16 cores) \| Observations \| Map (O(1)) \| Linear (O(n)) \| Speedup \| \|---\|---\|---\|---\| \| 10 \| 34 ns/op \| 41 ns/op \| 1.2x \| \| 50 \| 34 ns/op \| 186 ns/op \| 5.5x \| \| 100 \| 34 ns/op \| 361 ns/op \| 10.6x \| \| 500 \| 34 ns/op \| 4,903 ns/op \| 146x \| Map lookup is constant time regardless of observation count. The linear scan degrades quadratically — at 500 observations per transmission (realistic for popular packets seen by many observers), the old code is 146x slower per dedup check. All existing tests pass. --------- Co-authored-by: you <you@example.com>	2026-04-03 13:33:26 -07:00
Kpa-clawbot	b472c8de30	perf: replace O(n²) selection sort with sort.Slice (#354 ) (#542 ) ## Summary Fixes #354 Replaces the O(n²) selection sort in `sortedCopy()` with Go's built-in `sort.Float64s()` (O(n log n)). ## Changes - `cmd/server/routes.go`: Replaced manual nested-loop selection sort with `sort.Float64s(cp)` - `cmd/server/helpers_test.go`: Added regression test with 1000-element random input + benchmark ## Benchmark Results (ARM64) ``` BenchmarkSortedCopy/n=256 ~16μs/op 1 alloc BenchmarkSortedCopy/n=1000 ~95μs/op 1 alloc BenchmarkSortedCopy/n=10000 ~1.3ms/op 1 alloc ``` With the old O(n²) sort, n=10000 would take ~50ms+. The new implementation scales as O(n log n). ## Testing - All existing `TestSortedCopy` tests pass (unchanged behavior) - New `TestSortedCopyLarge` validates correctness on 1000 random elements - `go test ./...` passes in `cmd/server` Co-authored-by: you <you@example.com>	2026-04-03 13:11:59 -07:00
Kpa-clawbot	03e384bbc4	fix: null guard on pathHops prevents crash on ADVERT detail (#538 ) (#540 ) ## Summary Fixes #538 — `null is not an object (evaluating 'pathHops.length')` crash on ADVERT packet detail. ## Root Cause `getParsedPath` caches its result as `p._parsedPath`. If another code path (e.g., object spread, API response) sets `_parsedPath = null`, the cache check (`!== undefined`) passes and returns `null` — causing `.length` to crash. Same pattern exists for `getParsedDecoded`. ## Changes ### `public/packet-helpers.js` - `getParsedPath`: cached return now uses `\|\| []` to guard against null cache - `getParsedDecoded`: cached return now uses `\|\| {}` to guard against null cache ### `public/packets.js` - `renderDetail()` (line ~1440): defensive `\|\| []` / `\|\| {}` on getParsedPath/getParsedDecoded calls - `buildFlatRowHtml()` (line ~1103): same defensive guards ### `test-frontend-helpers.js` - Added test: cached `_parsedPath = null` returns `[]` - Added test: cached `_parsedDecoded = null` returns `{}` ## Testing All 428 frontend helper tests pass. All 62 packet filter tests pass. Co-authored-by: you <you@example.com>	2026-04-03 13:03:20 -07:00
Kpa-clawbot	bf8c9e72ec	fix: observer filter checks all observations in grouped mode (#537 ) (#539 ) Fixes #537 ## Problem Observer filter in grouped mode only checked `p.observer_id` (the primary observer), ignoring child observations. Grouped packets seen by multiple observers would be hidden when filtering for a non-primary observer. ## Fix Two filter paths updated to also check `p._children`: 1. Client-side display filter (line ~1293): removed the `!groupByHash` guard and added `_children` check so grouped packets are included when any child observation matches 2. WS real-time filter (line ~360): added `_children` fallback check The grouped row rendering (line ~1042) already correctly uses `_observerFilterSet` for child filtering — no changes needed there. ## Tests Added 5 tests in `test-frontend-helpers.js`: - Grouped packet with matching child observer is shown - Grouped packet with no matching observers is hidden - WS filter passes/rejects grouped packets correctly - Source code assertions verifying both filter paths check `_children` Co-authored-by: you <you@example.com>	2026-04-03 13:02:25 -07:00
Kpa-clawbot	48923db3d0	Add deep linking rule to AGENTS.md (#535 ) Adds a rule to AGENTS.md requiring all new UI states to be URL-addressable (deep-linkable). Part of #536. Co-authored-by: you <you@example.com>	2026-04-03 13:01:31 -07:00
efiten	709e5a4776	fix: observer filter drops groups in grouped packets view (#464 ) (#531 ) ## Summary - When `groupByHash=true`, each group only carries its representative (best-path) `observer_id`. The client-side filter was checking only that field, silently dropping groups that were seen by the selected observer but had a different representative. - `loadPackets` now passes the `observer` param to the server so `filterPackets`/`buildGroupedWhere` do the correct "any observation matches" check. - Client-side observer filter in `renderTableRows` is skipped for grouped mode (server already filtered correctly). - Both `db.go` and `store.go` observer filtering extended to support comma-separated IDs (multi-select UI). ## Test plan - [ ] Set an observer filter on the Packets screen with grouping enabled — all groups that have any observation from the selected observer(s) should appear, not just groups where that observer is the representative - [ ] Multi-select two observers — groups seen by either should appear - [ ] Toggle to flat (ungrouped) mode — per-observation filter still works correctly - [ ] Existing grouped packets tests pass: `cd cmd/server && go test ./...` Fixes #464 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com> Co-authored-by: you <you@example.com>	2026-04-03 09:22:37 -07:00