meshcore-analyzer/cmd/server/store.go

package main

import (
	"crypto/sha256"
	"database/sql"
	"encoding/json"
	"fmt"
	"log"
	"math"
	"runtime"
	"sort"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"
	"unicode/utf8"
)

// payloadTypeNames maps payload_type int → human-readable name (firmware-standard).
var payloadTypeNames = map[int]string{
	0: "REQ", 1: "RESPONSE", 2: "TXT_MSG", 3: "ACK", 4: "ADVERT",
	5: "GRP_TXT", 7: "ANON_REQ", 8: "PATH", 9: "TRACE", 11: "CONTROL",
}

// StoreTx is an in-memory transmission with embedded observations.
type StoreTx struct {
	ID               int
	RawHex           string
	Hash             string
	FirstSeen        string
	RouteType        *int
	PayloadType      *int
	DecodedJSON      string
	Observations     []*StoreObs
	ObservationCount int
	// Display fields from longest-path observation
	ObserverID   string
	ObserverName string
	ObserverIATA string
	SNR          *float64
	RSSI         *float64
	PathJSON     string
	Direction    string
	LatestSeen           string // max observation timestamp (or FirstSeen if no observations)
	UniqueObserverCount  int    // cached count of distinct observer IDs
	// Cached parsed fields (set once, read many)
	parsedPath    []string               // cached parsePathJSON result
	pathParsed    bool                    // whether parsedPath has been set
	decodedOnce   sync.Once              // guards parsedDecoded
	parsedDecoded map[string]interface{} // cached json.Unmarshal of DecodedJSON
	// Dedup map: "observerID|pathJSON" → true for O(1) duplicate checks
	obsKeys     map[string]bool
	observerSet map[string]bool // unique observer IDs (for UniqueObserverCount)
}

// StoreObs is a lean in-memory observation (no duplication of transmission fields).
type StoreObs struct {
	ID             int
	TransmissionID int
	ObserverID     string
	ObserverName   string
	ObserverIATA   string
	Direction      string
	SNR            *float64
	RSSI           *float64
	Score          *int
	PathJSON       string
	RawHex         string
	Timestamp      string
}

// ParsedDecoded returns the parsed DecodedJSON map, caching the result.
// Thread-safe via sync.Once — the first call parses, subsequent calls return cached.
func (tx *StoreTx) ParsedDecoded() map[string]interface{} {
	tx.decodedOnce.Do(func() {
		if tx.DecodedJSON != "" {
			json.Unmarshal([]byte(tx.DecodedJSON), &tx.parsedDecoded)
		}
	})
	return tx.parsedDecoded
}

// PacketStore holds all transmissions in memory with indexes for fast queries.
//
// Lock ordering
// =============
// PacketStore uses several mutexes. To prevent deadlocks, locks MUST be
// acquired in the order listed below. Never acquire a higher-numbered lock
// while holding a lower-numbered one.
//
//   1. mu            (sync.RWMutex) — guards the core packet data: packets,
//                     indexes (byHash, byTxID, byObsID, byObserver, byNode,
//                     byPathHop, byPayloadType), counters, and loaded flag.
//
//   2. cacheMu       (sync.Mutex)  — guards analytics response caches:
//                     rfCache, topoCache, hashCache, collisionCache, chanCache,
//                     distCache, subpathCache, and their TTLs/hit counters.
//                     Also guards rate-limited invalidation state
//                     (lastInvalidated, pendingInv).
//
//   3. channelsCacheMu (sync.Mutex) — guards the short-lived GetChannels
//                     cache (channelsCacheKey/Exp/Res).
//
//   4. groupedCacheMu (sync.Mutex)  — guards the short-lived
//                     QueryGroupedPackets cache.
//
//   5. regionObsMu   (sync.Mutex)  — guards the region→observer mapping
//                     cache (regionObsCache, regionObsCacheTime).
//
//   6. hashSizeInfoMu (sync.Mutex)  — guards the cached hash-size-info
//                     result (hashSizeInfoCache). Acquired independently or
//                     under mu (in EvictStale).
//
// Nesting that occurs today:
//   - IngestNew:               mu → cacheMu → channelsCacheMu  (1 → 2 → 3, OK)
//   - IngestObservations:      mu → cacheMu                    (1 → 2, OK)
//   - RunEviction/EvictStale:  mu → cacheMu → channelsCacheMu  (1 → 2 → 3, OK)
//   - RunEviction/EvictStale:  mu → hashSizeInfoMu             (1 → 6, OK)
//   - invalidateCachesFor:     cacheMu → channelsCacheMu       (2 → 3, OK)
//
// All other locks are acquired independently (no nesting).
// When adding new lock acquisitions, respect this ordering.
type PacketStore struct {
	mu            sync.RWMutex
	db            *DB
	packets       []*StoreTx                 // sorted by first_seen ASC (oldest first; newest at tail)
	byHash        map[string]*StoreTx        // hash → *StoreTx
	byTxID        map[int]*StoreTx           // transmission_id → *StoreTx
	byObsID       map[int]*StoreObs          // observation_id → *StoreObs
	maxTxID       int                        // highest transmission_id in store
	maxObsID      int                        // highest observation_id in store
	byObserver    map[string][]*StoreObs     // observer_id → observations
	byNode        map[string][]*StoreTx      // pubkey → transmissions
	nodeHashes    map[string]map[string]bool // pubkey → Set<hash>
	byPathHop     map[string][]*StoreTx      // lowercase hop/pubkey → transmissions with that hop in path
	byPayloadType map[int][]*StoreTx         // payload_type → transmissions
	loaded        bool
	totalObs      int
	insertCount   int64
	queryCount    int64
	// Response caches (separate mutex to avoid contention with store RWMutex)
	cacheMu      sync.Mutex
	rfCache      map[string]*cachedResult // region → cached RF result
	topoCache    map[string]*cachedResult // region → cached topology result
	hashCache      map[string]*cachedResult // region → cached hash-sizes result
	collisionCache map[string]*cachedResult // cached hash-collisions result keyed by region ("" = global)
	chanCache    map[string]*cachedResult // region → cached channels result
	distCache    map[string]*cachedResult // region → cached distance result
	subpathCache map[string]*cachedResult // params → cached subpaths result
	rfCacheTTL        time.Duration
	collisionCacheTTL time.Duration
	// Steady-state analytics recomputers (issue #1240). Each holds the
	// latest snapshot for the default region="" / zero-window query of
	// an analytics endpoint in an atomic.Value, refreshed by a
	// background goroutine on a fixed interval. When set, the matching
	// GetAnalytics* function serves from Load() instead of running the
	// on-request compute path. Region/window variants still go through
	// the legacy TTL cache (compute-on-miss).
	analyticsRecomputerMu sync.RWMutex
	recompTopology        *analyticsRecomputer
	recompRF              *analyticsRecomputer
	recompDistance        *analyticsRecomputer
	recompChannels        *analyticsRecomputer
	recompHashCollisions  *analyticsRecomputer
	recompHashSizes       *analyticsRecomputer
	recompRoles           *analyticsRecomputer
	recompObserversClockSkew *analyticsRecomputer
	recompNodesClockSkew     *analyticsRecomputer
	cacheHits    int64
	cacheMisses  int64
	// Rate-limited invalidation (fixes #533: caches cleared faster than hit)
	lastInvalidated time.Time
	pendingInv      *cacheInvalidation // accumulated dirty flags during cooldown
	invCooldown     time.Duration      // minimum time between invalidations
	// Short-lived cache for QueryGroupedPackets (avoids repeated full sort)
	groupedCacheMu    sync.Mutex
	groupedCacheKey   string
	groupedCacheExp   time.Time
	groupedCacheTxs   []*StoreTx // sorted by LatestSeen DESC
	groupedCacheTotal int
	// Short-lived cache for GetChannels (avoids repeated full scan + JSON unmarshal)
	channelsCacheMu  sync.Mutex
	channelsCacheKey string
	channelsCacheExp time.Time
	channelsCacheRes []map[string]interface{}
	// Cached region → observer ID mapping (30s TTL, avoids repeated DB queries)
	regionObsMu        sync.Mutex
	regionObsCache     map[string]map[string]bool
	regionObsCacheTime time.Time
	// Cached node list + prefix map (rebuilt on demand, shared across analytics)
	nodeCache     []nodeInfo
	nodePM        *prefixMap
	nodeCacheTime time.Time
	// Per-store dedupe set for one-shot schema-degradation warnings. Field
	// (not package-level) so each test gets a fresh state — see #1199 item 5.
	schemaDegradationLogged sync.Map
	// Precomputed subpath index: raw comma-joined hops → occurrence count.
	// Built during Load(), incrementally updated on ingest. Avoids full
	// packet iteration at query time (O(unique_subpaths) vs O(total_packets)).
	spIndex      map[string]int        // "hop1,hop2" → count
	spTxIndex    map[string][]*StoreTx // "hop1,hop2" → transmissions containing this subpath
	spTotalPaths int                   // transmissions with paths >= 2 hops
	// Precomputed distance analytics: hop distances and path totals
	// computed during Load() and incrementally updated on ingest.
	distHops  []distHopRecord
	distPaths []distPathRecord

	// Cached GetNodeHashSizeInfo result — recomputed at most once every 15s
	hashSizeInfoMu    sync.Mutex
	hashSizeInfoCache map[string]*hashSizeNodeInfo
	hashSizeInfoAt    time.Time

	// Cached multi-byte capability map (pubkey → entry), recomputed every 15s.
	multiByteCapCache map[string]*MultiByteCapEntry
	multiByteCapAt    time.Time

	// Cached per-pubkey relay info + usefulness score maps (#1257). These
	// fold the previously per-node GetRepeaterRelayInfo /
	// GetRepeaterUsefulnessScore loop in handleNodes into one O(N) pass
	// per 15s TTL window — eliminating N RLock acquisitions and N×
	// timestamp parses of the same byPathHop entries per request.
	repeaterEnrichMu       sync.Mutex
	repeaterRelayCache     map[string]RepeaterRelayInfo
	repeaterRelayCacheWin  float64
	repeaterRelayAt        time.Time
	repeaterUsefulCache    map[string]float64
	repeaterUsefulAt       time.Time

	// Steady-state recomputer for the two caches above (#1262). When
	// started, an initial sync compute prewarms the caches so the very
	// first /api/nodes?limit=2000 from live.js's SPA bootstrap hits a
	// populated cache instead of paying the 15.7s on-thread rebuild.
	repeaterEnrichRecompMu      sync.Mutex
	repeaterEnrichRecompStarted bool
	repeaterEnrichRecompStop    chan struct{}
	repeaterEnrichRecompDone    chan struct{}

	// Precomputed distinct advert pubkey count (refcounted for eviction correctness).
	// Updated incrementally during Load/Ingest/Evict — avoids JSON parsing in GetPerfStoreStats.
	advertPubkeys map[string]int // pubkey → number of advert packets referencing it

	// Debounce map for touchRelayLastSeen: pubkey → last time we wrote last_seen to DB.
	// Limits DB writes to at most 1 per node per 5 minutes.
	lastSeenTouched map[string]time.Time

	// Resolved path membership index: xxhash → []txID (forward) and txID → []hashes (reverse).
	// Replaces per-StoreTx/StoreObs ResolvedPath []*string field (#800).
	resolvedPubkeyIndex   map[uint64][]int  // hash(pubkey) → []txID
	resolvedPubkeyReverse map[int][]uint64  // txID → []hashes indexed under
	useResolvedPathIndex              bool // feature flag (default true, off path = conservative)
	maxResolvedPubkeyIndexEntries     int  // hard cap for size warning (0 = use default 5M)
	apiResolvedPathLRU    map[int][]*string // obsID → resolved path (LRU cache for API)
	lruOrder              []int             // FIFO order for LRU eviction
	lruMu                 sync.RWMutex     // guards apiResolvedPathLRU + lruOrder

	// Persisted neighbor graph for hop resolution at ingest time.
	// Accessed via atomic.Pointer because async rebuilds (path_inspect.go
	// ensureNeighborGraph) and ingest-time readers race on the pointer
	// (issue #1203 sub-fix).
	graph atomic.Pointer[NeighborGraph]

	// Singleflight state for ensureNeighborGraph. These were package-globals
	// in #1203 r0 — moved to per-store fields (PR #1208 review) so parallel
	// tests with independent *PacketStore values don't share rebuild state
	// (cross-store deadlock/skip risk under -race).
	rebuildMu    sync.Mutex
	rebuildInFlt chan struct{} // nil when no rebuild is in flight


	// Path inspector score cache (issue #944).
	inspectMu    sync.RWMutex
	inspectCache map[string]*inspectCachedResult

	// Clock skew detection engine.
	clockSkew *ClockSkewEngine

	// Async backfill state: set after backfillResolvedPathsAsync completes.
	backfillComplete atomic.Bool
	// Progress tracking for async backfill (total pending and processed so far).
	backfillTotal     atomic.Int64 // set once at start of async backfill
	backfillProcessed atomic.Int64

	// Bounded cold load: oldest packet timestamp loaded into memory.
	// Empty string means all data is in memory (no limit applied).
	oldestLoaded string

	// Hot startup atomic gates — see contract below.
	//
	// Contract / ordering invariant (PR #1187):
	//   * hashMigrationComplete (set by migrateContentHashesAsync) gates
	//     content-hash–dependent code paths (e.g. dedup correctness on the
	//     write side). Set true ONLY after the migration loop finishes.
	//   * backgroundLoadDone is set true exactly once, after
	//     loadBackgroundChunks finishes its loop AND its post-load index
	//     rebuild. It gates "hot startup has finished filling
	//     retentionHours of data into memory" (used by /api/perf and the
	//     UI's hot-load banner). It says nothing about success — see
	//     backgroundLoadFailed for the success/failure signal.
	//   * backgroundLoadFailed is set true ONLY if at least one chunk
	//     errored during background load. /api/perf surfaces it so
	//     operators can distinguish "done & full" from "done & partial".
	//     Read order MUST be: load backgroundLoadDone first; only if true
	//     is backgroundLoadFailed meaningful.
	// 0 = disabled (current behavior). Background loader fills the rest.
	hotStartupHours        float64
	backgroundLoadDone     atomic.Bool
	backgroundLoadFailed   atomic.Bool
	backgroundLoadProgress atomic.Int64 // 0–100 percent complete

	// Async hash migration state: set after migrateContentHashesAsync completes.
	hashMigrationComplete atomic.Bool

	// Eviction config and stats
	retentionHours   float64        // 0 = unlimited
	maxMemoryMB      int            // 0 = unlimited (packet store memory budget)
	evicted          int64          // total packets evicted
	trackedBytes     int64          // running total of estimated packet store memory
	memoryEstimator  func() float64 // injectable for tests; nil = use runtime.ReadMemStats (stats only)
}

// Precomputed distance records for fast analytics aggregation.
type distHopRecord struct {
	FromName   string
	FromPk     string
	ToName     string
	ToPk       string
	Dist       float64
	Type       string // "R↔R", "C↔R", "C↔C"
	SNR        *float64
	Hash       string
	Timestamp  string
	HourBucket string
	tx         *StoreTx
}

// dedupeHopsByPair groups hops by unordered node-pair, keeps the max-distance
// record per pair, and computes obsCount / bestSnr / medianSnr.  limit caps the
// number of returned entries (sorted by distance descending).
func dedupeHopsByPair(hops []distHopRecord, limit int) []map[string]interface{} {
	type pairAgg struct {
		best     *distHopRecord
		obsCount int
		maxSNR   *float64
		snrs     []float64
	}
	pairMap := make(map[string]*pairAgg)
	for i := range hops {
		h := &hops[i]
		pk1, pk2 := h.FromPk, h.ToPk
		if pk1 > pk2 {
			pk1, pk2 = pk2, pk1
		}
		key := pk1 + "|" + pk2
		agg, ok := pairMap[key]
		if !ok {
			agg = &pairAgg{}
			pairMap[key] = agg
		}
		agg.obsCount++
		if h.SNR != nil {
			agg.snrs = append(agg.snrs, *h.SNR)
			if agg.maxSNR == nil || *h.SNR > *agg.maxSNR {
				v := *h.SNR
				agg.maxSNR = &v
			}
		}
		if agg.best == nil || h.Dist > agg.best.Dist {
			agg.best = h
		}
	}
	type pairEntry struct {
		key string
		agg *pairAgg
	}
	pairs := make([]pairEntry, 0, len(pairMap))
	for k, v := range pairMap {
		pairs = append(pairs, pairEntry{k, v})
	}
	sort.Slice(pairs, func(i, j int) bool { return pairs[i].agg.best.Dist > pairs[j].agg.best.Dist })
	result := make([]map[string]interface{}, 0, min(limit, len(pairs)))
	for i, pe := range pairs {
		if i >= limit {
			break
		}
		h := pe.agg.best
		var medianSNR *float64
		if len(pe.agg.snrs) > 0 {
			sorted := make([]float64, len(pe.agg.snrs))
			copy(sorted, pe.agg.snrs)
			sort.Float64s(sorted)
			mid := len(sorted) / 2
			if len(sorted)%2 == 0 {
				v := (sorted[mid-1] + sorted[mid]) / 2
				medianSNR = &v
			} else {
				v := sorted[mid]
				medianSNR = &v
			}
		}
		result = append(result, map[string]interface{}{
			"fromName": h.FromName, "fromPk": h.FromPk,
			"toName": h.ToName, "toPk": h.ToPk,
			"dist": h.Dist, "type": h.Type,
			"bestSnr": floatPtrOrNil(pe.agg.maxSNR), "medianSnr": floatPtrOrNil(medianSNR),
			"obsCount": pe.agg.obsCount,
			"hash": h.Hash, "timestamp": h.Timestamp,
		})
	}
	return result
}

type distPathRecord struct {
	Hash      string
	TotalDist float64
	HopCount  int
	Timestamp string
	Hops      []distHopDetail
	tx        *StoreTx
}

type distHopDetail struct {
	FromName string
	FromPk   string
	ToName   string
	ToPk     string
	Dist     float64
}

type cachedResult struct {
	data      map[string]interface{}
	expiresAt time.Time
}

// cacheTTLSec extracts a duration from the cacheTTL config map.
// Values may be float64 (from JSON) or int. Returns false if key is missing or non-positive.
func cacheTTLSec(m map[string]interface{}, key string) (time.Duration, bool) {
	v, ok := m[key]
	if !ok {
		return 0, false
	}
	var sec float64
	switch n := v.(type) {
	case float64:
		sec = n
	case int:
		sec = float64(n)
	case int64:
		sec = float64(n)
	default:
		return 0, false
	}
	if sec <= 0 {
		return 0, false
	}
	return time.Duration(sec * float64(time.Second)), true
}

// NewPacketStore creates a new empty packet store backed by db.
// cacheTTLs is the optional cacheTTL map from config.json; keys are strings, values are seconds.
func NewPacketStore(db *DB, cfg *PacketStoreConfig, cacheTTLs ...map[string]interface{}) *PacketStore {
	ps := &PacketStore{
		db:            db,
		packets:       make([]*StoreTx, 0, 65536),
		byHash:        make(map[string]*StoreTx, 65536),
		byTxID:        make(map[int]*StoreTx, 65536),
		byObsID:       make(map[int]*StoreObs, 65536),
		byObserver:    make(map[string][]*StoreObs),
		byNode:        make(map[string][]*StoreTx),
		byPathHop:     make(map[string][]*StoreTx),
		nodeHashes:    make(map[string]map[string]bool),
		byPayloadType: make(map[int][]*StoreTx),
		rfCache:       make(map[string]*cachedResult),
		topoCache:     make(map[string]*cachedResult),
		hashCache:      make(map[string]*cachedResult),

		collisionCache: make(map[string]*cachedResult),
		chanCache:     make(map[string]*cachedResult),
		distCache:     make(map[string]*cachedResult),
		subpathCache:  make(map[string]*cachedResult),
		// #1239: 60 seconds by default. rfCacheTTL is the shared TTL for
		// the RF, topology, distance, hash-sizes, subpath, and channel
		// analytics caches. Distance analytics IS viewed live during
		// active analysis sessions (operators won't tolerate a 5-min
		// lag), so the bump from 15s → 60s smooths the most-frequent
		// cold-miss churn during heavy ingest without freezing data.
		// Override via cacheTTL.analyticsRF in config.json (also
		// propagates to distance / topology / hash-sizes / etc.).
		rfCacheTTL: 60 * time.Second,
		collisionCacheTTL: 3600 * time.Second,
		invCooldown:       300 * time.Second,
		spIndex:       make(map[string]int, 4096),
		spTxIndex:     make(map[string][]*StoreTx, 4096),
		advertPubkeys:   make(map[string]int),
		lastSeenTouched: make(map[string]time.Time),
		clockSkew:       NewClockSkewEngine(),
		useResolvedPathIndex: true,
	}
	ps.initResolvedPathIndex()
	if cfg != nil {
		ps.retentionHours = cfg.RetentionHours
		ps.maxMemoryMB = cfg.MaxMemoryMB
		ps.maxResolvedPubkeyIndexEntries = cfg.MaxResolvedPubkeyIndexEntries
		if cfg.HotStartupHours > 0 {
			h := cfg.HotStartupHours
			if ps.retentionHours > 0 && h > ps.retentionHours {
				log.Printf("[store] warning: hotStartupHours (%g) > retentionHours (%g) — clamping", h, ps.retentionHours)
				h = ps.retentionHours
			}
			ps.hotStartupHours = h
		}
	}
	// Wire cacheTTL config values to server-side cache durations.
	if len(cacheTTLs) > 0 && cacheTTLs[0] != nil {
		ct := cacheTTLs[0]
		if v, ok := cacheTTLSec(ct, "analyticsHashSizes"); ok {
			ps.collisionCacheTTL = v
		}
		if v, ok := cacheTTLSec(ct, "analyticsRF"); ok {
			ps.rfCacheTTL = v
		}
		if v, ok := cacheTTLSec(ct, "invalidationDebounce"); ok {
			ps.invCooldown = v
		}
	}
	return ps
}

// Load reads transmissions + observations from SQLite into memory.
// When maxMemoryMB > 0, loads only the newest N transmissions that fit
// within the memory budget, avoiding OOM on large databases.
func (s *PacketStore) Load() error {
	s.mu.Lock()
	defer s.mu.Unlock()

	t0 := time.Now()

	// Count total transmissions for logging.
	var totalInDB int
	if err := s.db.conn.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&totalInDB); err != nil {
		totalInDB = -1 // non-fatal
	}

	// Calculate max packets to load based on memory budget.
	var maxPackets int64
	if s.maxMemoryMB > 0 {
		// Use a typical packet with ~10 observations as the estimate.
		avgBytes := int64(1000) // conservative floor
		if sample := estimateStoreTxBytesTypical(10); sample > avgBytes {
			avgBytes = sample
		}
		maxPackets = (int64(s.maxMemoryMB) * 1048576) / avgBytes
		if maxPackets < 1000 {
			maxPackets = 1000 // minimum 1000 packets
		}
	}
	// maxPackets == 0 means unlimited

	var loadSQL string
	rpCol := ""
	if s.db.hasResolvedPath {
		rpCol = ",\n\t\t\t\to.resolved_path"
	}
	obsRawHexCol := ""
	if s.db.hasObsRawHex {
		obsRawHexCol = ", o.raw_hex"
	}

	// Build WHERE conditions: retention cutoff (mirrors Evict logic) + optional memory-cap limit.
	// When hotStartupHours > 0, use it as the initial cutoff (smaller window = fast startup).
	//
	// PR #1187 r2 #7: compute the hot cutoff ONCE here and reuse the same
	// string for both the SQL filter below AND for s.oldestLoaded later.
	// Two separate time.Now().UTC() calls produced microsecond skew at chunk
	// borders, so the SQL window and oldestLoaded could disagree.
	var loadConditions []string
	hotCutoffHours := s.retentionHours
	if s.hotStartupHours > 0 {
		hotCutoffHours = s.hotStartupHours
	}
	var hotCutoffStr string
	if hotCutoffHours > 0 {
		hotCutoffStr = time.Now().UTC().Add(-time.Duration(hotCutoffHours * float64(time.Hour))).Format(time.RFC3339)
		loadConditions = append(loadConditions, fmt.Sprintf("t.first_seen >= '%s'", hotCutoffStr))
	}
	if maxPackets > 0 {
		loadConditions = append(loadConditions, fmt.Sprintf(
			"t.id IN (SELECT id FROM transmissions ORDER BY first_seen DESC LIMIT %d)", maxPackets))
	}
	filterClause := ""
	if len(loadConditions) > 0 {
		filterClause = "\n\t\t\tWHERE " + strings.Join(loadConditions, "\n\t\t\t  AND ")
	}

	if s.db.isV3 {
		loadSQL = `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type,
				t.payload_type, t.payload_version, t.decoded_json,
				o.id, obs.id, obs.name, COALESCE(obs.iata, ''), o.direction,
				o.snr, o.rssi, o.score, o.path_json, strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch')` + obsRawHexCol + rpCol + `
			FROM transmissions t
			LEFT JOIN observations o ON o.transmission_id = t.id
			LEFT JOIN observers obs ON obs.rowid = o.observer_idx` + filterClause + `
			ORDER BY t.first_seen ASC, o.timestamp DESC`
	} else {
		loadSQL = `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type,
				t.payload_type, t.payload_version, t.decoded_json,
				o.id, o.observer_id, o.observer_name, COALESCE(obs.iata, ''), o.direction,
				o.snr, o.rssi, o.score, o.path_json, o.timestamp` + obsRawHexCol + rpCol + `
			FROM transmissions t
			LEFT JOIN observations o ON o.transmission_id = t.id
			LEFT JOIN observers obs ON obs.id = o.observer_id` + filterClause + `
			ORDER BY t.first_seen ASC, o.timestamp DESC`
	}

	rows, err := s.db.conn.Query(loadSQL)
	if err != nil {
		return err
	}
	defer rows.Close()

	hopsSeen := make(map[string]bool) // reused across observations; cleared per use

	for rows.Next() {
		var txID int
		var rawHex, hash, firstSeen, decodedJSON sql.NullString
		var routeType, payloadType, payloadVersion sql.NullInt64
		var obsID sql.NullInt64
		var observerID, observerName, observerIATA, direction, pathJSON, obsTimestamp sql.NullString
		var snr, rssi sql.NullFloat64
		var score sql.NullInt64
		var obsRawHex sql.NullString
		var resolvedPathStr sql.NullString

		scanArgs := []interface{}{&txID, &rawHex, &hash, &firstSeen, &routeType, &payloadType,
			&payloadVersion, &decodedJSON,
			&obsID, &observerID, &observerName, &observerIATA, &direction,
			&snr, &rssi, &score, &pathJSON, &obsTimestamp}
		if s.db.hasObsRawHex {
			scanArgs = append(scanArgs, &obsRawHex)
		}
		if s.db.hasResolvedPath {
			scanArgs = append(scanArgs, &resolvedPathStr)
		}
		if err := rows.Scan(scanArgs...); err != nil {
			log.Printf("[store] scan error: %v", err)
			continue
		}

		hashStr := nullStrVal(hash)
		tx := s.byHash[hashStr]
		if tx == nil {
			tx = &StoreTx{
				ID:          txID,
				RawHex:      nullStrVal(rawHex),
				Hash:        hashStr,
				FirstSeen:   nullStrVal(firstSeen),
				LatestSeen:  nullStrVal(firstSeen),
				RouteType:   nullIntPtr(routeType),
				PayloadType: nullIntPtr(payloadType),
				DecodedJSON: nullStrVal(decodedJSON),
				obsKeys:     make(map[string]bool),
				observerSet: make(map[string]bool),
			}
			s.byHash[hashStr] = tx
			s.packets = append(s.packets, tx)
			s.byTxID[txID] = tx
			if txID > s.maxTxID {
				s.maxTxID = txID
			}
			s.indexByNode(tx)
			if tx.PayloadType != nil {
				pt := *tx.PayloadType
				s.byPayloadType[pt] = append(s.byPayloadType[pt], tx)
			}
			s.trackAdvertPubkey(tx)
			s.trackedBytes += estimateStoreTxBytes(tx)
		}

		if obsID.Valid {
			oid := int(obsID.Int64)
			obsIDStr := nullStrVal(observerID)
			obsPJ := nullStrVal(pathJSON)

			// Dedup: skip if same observer + same path already loaded (O(1) map lookup)
			dk := obsIDStr + "|" + obsPJ
			if tx.obsKeys[dk] {
				continue
			}

			obs := &StoreObs{
				ID:             oid,
				TransmissionID: txID,
				ObserverID:     obsIDStr,
				ObserverName:   nullStrVal(observerName),
				ObserverIATA:   nullStrVal(observerIATA),
				Direction:      nullStrVal(direction),
				SNR:            nullFloatPtr(snr),
				RSSI:           nullFloatPtr(rssi),
				Score:          nullIntPtr(score),
				PathJSON:       obsPJ,
				RawHex:         nullStrVal(obsRawHex),
				Timestamp:      normalizeTimestamp(nullStrVal(obsTimestamp)),
			}

			// Decode-window: extract resolved pubkeys for index, don't store on struct.
			rpStr := nullStrVal(resolvedPathStr)
			if rpStr != "" {
				rp := unmarshalResolvedPath(rpStr)
				pks := extractResolvedPubkeys(rp)
				// Feed decode-window consumers for this observation's pubkeys
				if len(pks) > 0 {
					// addToByNode for relay nodes
					for _, pk := range pks {
						s.addToByNode(tx, pk)
					}
					// touchRelayLastSeen handled in post-load pass
					// byPathHop resolved-key entries
					clear(hopsSeen)
					for _, hop := range txGetParsedPath(tx) {
						hopsSeen[strings.ToLower(hop)] = true
					}
					for _, pk := range pks {
						if !hopsSeen[pk] {
							hopsSeen[pk] = true
							s.byPathHop[pk] = append(s.byPathHop[pk], tx)
						}
					}
					// resolvedPubkeyIndex
					s.addToResolvedPubkeyIndex(tx.ID, pks)
				}
			}

			tx.Observations = append(tx.Observations, obs)
			tx.obsKeys[dk] = true
			if obs.ObserverID != "" && !tx.observerSet[obs.ObserverID] {
				tx.observerSet[obs.ObserverID] = true
				tx.UniqueObserverCount++
			}
			tx.ObservationCount++
			if obs.Timestamp > tx.LatestSeen {
				tx.LatestSeen = obs.Timestamp
			}

			s.byObsID[oid] = obs
			if oid > s.maxObsID {
				s.maxObsID = oid
			}

			if obsIDStr != "" {
				s.byObserver[obsIDStr] = append(s.byObserver[obsIDStr], obs)
			}

			s.totalObs++
			s.trackedBytes += estimateStoreObsBytes(obs)
		}
	}

	// Post-load: pick best observation (longest path) for each transmission
	for _, tx := range s.packets {
		pickBestObservation(tx)
	}

	// Build precomputed subpath index for O(1) analytics queries
	s.buildSubpathIndex()

	// Build path-hop index for O(1) node path lookups
	s.buildPathHopIndex()

	// Precompute distance analytics (hop distances, path totals)
	s.buildDistanceIndex()

	// Track oldest loaded timestamp for future SQL fallback queries.
	// When hotStartupHours > 0 use the SAME cutoff string that was used in
	// the load SQL (PR #1187 r2 #7) — recomputing time.Now().UTC() here
	// produced microsecond skew vs. the SQL filter at chunk boundaries.
	if s.hotStartupHours > 0 {
		s.oldestLoaded = hotCutoffStr
	} else if len(s.packets) > 0 {
		s.oldestLoaded = s.packets[0].FirstSeen
	}

	s.loaded = true
	elapsed := time.Since(t0)
	if maxPackets > 0 && totalInDB > len(s.packets) {
		log.Printf("[store] Loaded %d/%d transmissions (%d observations) in %v — bounded by %dMB budget (tracked ~%.0fMB, heap ~%.0fMB)",
			len(s.packets), totalInDB, s.totalObs, elapsed, s.maxMemoryMB, s.trackedMemoryMB(), s.estimatedMemoryMB())
	} else {
		log.Printf("[store] Loaded %d transmissions (%d observations) in %v (tracked ~%.0fMB, heap ~%.0fMB)",
			len(s.packets), s.totalObs, elapsed, s.trackedMemoryMB(), s.estimatedMemoryMB())
	}
	return nil
}

// loadChunk queries a [from, to) time window from SQLite without holding the
// write lock, builds local data structures, then merges them into the store
// under s.mu.Lock(). It is the building block for the background loader.
//
// The chunk is assumed to be older than the data already in the store, so
// localPackets are prepended to s.packets.
//
// byPayloadType is updated here incrementally. byPathHop, spIndex, and
// distHops are NOT updated here — the caller (loadBackgroundChunks) rebuilds
// those once after all chunks are merged.
func (s *PacketStore) loadChunk(from, to time.Time) error {
	fromStr := from.UTC().Format(time.RFC3339)
	toStr := to.UTC().Format(time.RFC3339)

	// Build the same SQL as Load() but with a [from, to) window.
	rpCol := ""
	if s.db.hasResolvedPath {
		rpCol = ",\n\t\t\t\to.resolved_path"
	}
	obsRawHexCol := ""
	if s.db.hasObsRawHex {
		obsRawHexCol = ", o.raw_hex"
	}

	const filterClause = "\n\t\t\tWHERE t.first_seen >= ? AND t.first_seen < ?"

	var chunkSQL string
	if s.db.isV3 {
		chunkSQL = `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type,
				t.payload_type, t.payload_version, t.decoded_json,
				o.id, obs.id, obs.name, o.direction,
				o.snr, o.rssi, o.score, o.path_json, strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch')` + obsRawHexCol + rpCol + `
			FROM transmissions t
			LEFT JOIN observations o ON o.transmission_id = t.id
			LEFT JOIN observers obs ON obs.rowid = o.observer_idx` + filterClause + `
			ORDER BY t.first_seen ASC, o.timestamp DESC`
	} else {
		chunkSQL = `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type,
				t.payload_type, t.payload_version, t.decoded_json,
				o.id, o.observer_id, o.observer_name, o.direction,
				o.snr, o.rssi, o.score, o.path_json, o.timestamp` + obsRawHexCol + rpCol + `
			FROM transmissions t
			LEFT JOIN observations o ON o.transmission_id = t.id` + filterClause + `
			ORDER BY t.first_seen ASC, o.timestamp DESC`
	}

	rows, err := s.db.conn.Query(chunkSQL, fromStr, toStr)
	if err != nil {
		return err
	}
	defer rows.Close()

	// Local data structures — built without holding the lock.
	localByHash := make(map[string]*StoreTx)
	localPackets := make([]*StoreTx, 0)
	localByTxID := make(map[int]*StoreTx)
	localByObsID := make(map[int]*StoreObs)
	localByObserver := make(map[string][]*StoreObs)
	var localTotalObs int
	var localTrackedBytes int64
	var localMaxTxID int
	var localMaxObsID int

	for rows.Next() {
		var txID int
		var rawHex, hash, firstSeen, decodedJSON sql.NullString
		var routeType, payloadType, payloadVersion sql.NullInt64
		var obsID sql.NullInt64
		var observerID, observerName, direction, pathJSON, obsTimestamp sql.NullString
		var snr, rssi sql.NullFloat64
		var score sql.NullInt64
		var obsRawHex sql.NullString
		var resolvedPathStr sql.NullString

		scanArgs := []interface{}{&txID, &rawHex, &hash, &firstSeen, &routeType, &payloadType,
			&payloadVersion, &decodedJSON,
			&obsID, &observerID, &observerName, &direction,
			&snr, &rssi, &score, &pathJSON, &obsTimestamp}
		if s.db.hasObsRawHex {
			scanArgs = append(scanArgs, &obsRawHex)
		}
		if s.db.hasResolvedPath {
			scanArgs = append(scanArgs, &resolvedPathStr)
		}
		if err := rows.Scan(scanArgs...); err != nil {
			log.Printf("[store] loadChunk scan error: %v", err)
			continue
		}

		hashStr := nullStrVal(hash)
		tx := localByHash[hashStr]
		if tx == nil {
			tx = &StoreTx{
				ID:          txID,
				RawHex:      nullStrVal(rawHex),
				Hash:        hashStr,
				FirstSeen:   nullStrVal(firstSeen),
				LatestSeen:  nullStrVal(firstSeen),
				RouteType:   nullIntPtr(routeType),
				PayloadType: nullIntPtr(payloadType),
				DecodedJSON: nullStrVal(decodedJSON),
				obsKeys:     make(map[string]bool),
				observerSet: make(map[string]bool),
			}
			localByHash[hashStr] = tx
			localPackets = append(localPackets, tx)
			localByTxID[txID] = tx
			if txID > localMaxTxID {
				localMaxTxID = txID
			}
			localTrackedBytes += estimateStoreTxBytes(tx)
		}

		if obsID.Valid {
			oid := int(obsID.Int64)
			obsIDStr := nullStrVal(observerID)
			obsPJ := nullStrVal(pathJSON)

			dk := obsIDStr + "|" + obsPJ
			if tx.obsKeys[dk] {
				continue
			}

			obs := &StoreObs{
				ID:             oid,
				TransmissionID: txID,
				ObserverID:     obsIDStr,
				ObserverName:   nullStrVal(observerName),
				Direction:      nullStrVal(direction),
				SNR:            nullFloatPtr(snr),
				RSSI:           nullFloatPtr(rssi),
				Score:          nullIntPtr(score),
				PathJSON:       obsPJ,
				RawHex:         nullStrVal(obsRawHex),
				Timestamp:      normalizeTimestamp(nullStrVal(obsTimestamp)),
			}

			tx.Observations = append(tx.Observations, obs)
			tx.obsKeys[dk] = true
			if obs.ObserverID != "" && !tx.observerSet[obs.ObserverID] {
				tx.observerSet[obs.ObserverID] = true
				tx.UniqueObserverCount++
			}
			tx.ObservationCount++
			if obs.Timestamp > tx.LatestSeen {
				tx.LatestSeen = obs.Timestamp
			}

			localByObsID[oid] = obs
			if oid > localMaxObsID {
				localMaxObsID = oid
			}
			if obsIDStr != "" {
				localByObserver[obsIDStr] = append(localByObserver[obsIDStr], obs)
			}
			localTotalObs++
			localTrackedBytes += estimateStoreObsBytes(obs)
		}
	}
	if err := rows.Err(); err != nil {
		return err
	}

	// Pick best observation for each local packet before merging.
	for _, tx := range localPackets {
		pickBestObservation(tx)
	}

	if len(localPackets) == 0 {
		return nil
	}

	// PR #1187 r3 MUST-FIX 1: index↔slice consistency.
	//
	// The previous (r2 #6, commit 2ec762aa) merge order was:
	//   1) prepend s.packets in one critical section
	//   2) populate s.byHash/byTxID/byObsID/byObserver/byNode/byPayloadType
	//      in separate per-batch critical sections
	//   3) bump counters in a third section
	//
	// Any RLock-holding reader between steps observed packets present in
	// s.packets but missing from s.byHash → silent partial data loss in
	// GetPacketByHash and the QueryPackets hash/node fast-paths.
	//
	// We invert the order: build the indexes FIRST in bounded per-batch
	// critical sections, then under a SINGLE final critical section
	// prepend s.packets AND bump counters AND advance maxIDs. The
	// invariant "every tx in s.packets is in s.byHash/s.byTxID" holds at
	// every RLock instant — readers either see the old state, or an
	// intermediate state where indexes contain a superset of s.packets
	// (which is harmless: nothing in s.packets dangles), or the fully
	// merged new state.
	const mergeBatchSize = 500

	for batchStart := 0; batchStart < len(localPackets); batchStart += mergeBatchSize {
		batchEnd := batchStart + mergeBatchSize
		if batchEnd > len(localPackets) {
			batchEnd = len(localPackets)
		}
		batch := localPackets[batchStart:batchEnd]

		// Build the per-batch index views outside the lock so the
		// critical section below is O(batch), not O(total local).
		batchHashes := make(map[string]*StoreTx, len(batch))
		batchTxIDs := make(map[int]*StoreTx, len(batch))
		batchObsIDs := make(map[int]*StoreObs)
		batchByObserver := make(map[string][]*StoreObs)
		for _, tx := range batch {
			batchHashes[tx.Hash] = tx
			batchTxIDs[tx.ID] = tx
			for _, o := range tx.Observations {
				batchObsIDs[o.ID] = o
				if o.ObserverID != "" {
					batchByObserver[o.ObserverID] = append(batchByObserver[o.ObserverID], o)
				}
			}
		}

		s.mu.Lock()
		newObsIDs := make(map[int]bool, len(batchObsIDs))
		for k := range batchObsIDs {
			if s.byObsID[k] == nil {
				newObsIDs[k] = true
			}
		}
		for k, v := range batchHashes {
			if s.byHash[k] == nil {
				s.byHash[k] = v
			}
		}
		for k, v := range batchTxIDs {
			if s.byTxID[k] == nil {
				s.byTxID[k] = v
			}
		}
		for k, v := range batchObsIDs {
			if newObsIDs[k] {
				s.byObsID[k] = v
			}
		}
		for observerID, obsList := range batchByObserver {
			for _, o := range obsList {
				if newObsIDs[o.ID] {
					s.byObserver[observerID] = append(s.byObserver[observerID], o)
				}
			}
		}
		for _, tx := range batch {
			s.indexByNode(tx)
			if tx.PayloadType != nil {
				pt := *tx.PayloadType
				s.byPayloadType[pt] = append(s.byPayloadType[pt], tx)
			}
			s.trackAdvertPubkey(tx)
		}
		s.mu.Unlock()
		runtime.Gosched()
	}

	// Final atomic step: now that every tx in localPackets is fully
	// indexed, publish it into s.packets and bump counters in one short
	// critical section. After this point the new state is fully visible;
	// before it readers see the old slice (which is still fully indexed).
	s.mu.Lock()
	s.packets = append(localPackets, s.packets...)
	s.totalObs += localTotalObs
	s.trackedBytes += localTrackedBytes
	if localMaxTxID > s.maxTxID {
		s.maxTxID = localMaxTxID
	}
	if localMaxObsID > s.maxObsID {
		s.maxObsID = localMaxObsID
	}
	s.mu.Unlock()

	log.Printf("[store] background chunk [%s, %s) merged: %d tx, %d obs", fromStr, toStr, len(localPackets), localTotalObs)
	return nil
}

// loadBackgroundChunks fills the remaining retentionHours window by loading
// daily chunks from oldestLoaded back to the retention cutoff. After all
// chunks are merged it rebuilds analytics indexes once. Chunk errors are
// handled by advancing past the failed window so the loop always terminates.
func (s *PacketStore) loadBackgroundChunks() {
	if s.retentionHours <= 0 {
		s.backgroundLoadDone.Store(true)
		return
	}

	target := time.Now().UTC().Add(-time.Duration(s.retentionHours * float64(time.Hour)))
	totalHours := s.retentionHours - s.hotStartupHours
	if totalHours <= 0 {
		s.backgroundLoadDone.Store(true)
		return
	}

	var chunksLoaded float64
	var chunkErrors int
	totalChunks := math.Ceil(totalHours / 24)

	for {
		s.mu.RLock()
		oldest := s.oldestLoaded
		s.mu.RUnlock()

		if oldest == "" {
			break
		}
		chunkEnd, err := time.Parse(time.RFC3339, oldest)
		if err != nil {
			log.Printf("[store] background loader: bad oldestLoaded %q: %v", oldest, err)
			break
		}
		if !chunkEnd.After(target) {
			break
		}

		chunkStart := chunkEnd.Add(-24 * time.Hour)
		if chunkStart.Before(target) {
			chunkStart = target
		}

		chunkStartStr := chunkStart.Format(time.RFC3339)
		if err := s.loadChunk(chunkStart, chunkEnd); err != nil {
			chunkErrors++
			log.Printf("[store] background chunk [%s, %s) error: %v — advancing past it",
				chunkStartStr, chunkEnd.Format(time.RFC3339), err)
		}
		// Always advance oldestLoaded to chunkStart so the loop terminates,
		// even when the chunk was empty (loadChunk skips the update when 0 packets).
		s.mu.Lock()
		if s.oldestLoaded > chunkStartStr || s.oldestLoaded == "" {
			s.oldestLoaded = chunkStartStr
		}
		s.mu.Unlock()

		chunksLoaded++
		if totalChunks > 0 {
			pct := int64(chunksLoaded / totalChunks * 100)
			if pct > 100 {
				pct = 100
			}
			s.backgroundLoadProgress.Store(pct)
		}

		// Yield between chunks so ingest goroutines can acquire s.mu.Lock()
		// without being starved. The chosen tradeoff is brief lock-holds per
		// chunk rather than a configurable sleep; background fill is
		// best-effort and queries fall back to SQL while it runs.
		runtime.Gosched()
	}

	// Rebuild analytics indexes once after all chunks are merged.
	s.mu.Lock()
	s.buildSubpathIndex()
	s.buildPathHopIndex()
	s.buildDistanceIndex()
	s.mu.Unlock()

	s.backgroundLoadDone.Store(true)
	if chunkErrors > 0 {
		s.backgroundLoadFailed.Store(true)
	}
	s.backgroundLoadProgress.Store(100)

	s.mu.RLock()
	totalPkts := len(s.packets)
	oldest := s.oldestLoaded
	s.mu.RUnlock()
	if chunkErrors > 0 {
		log.Printf("[store] background load done with %d chunk error(s): %d packets in memory, oldestLoaded=%s", chunkErrors, totalPkts, oldest)
	} else {
		log.Printf("[store] background load complete: %d packets in memory, oldestLoaded=%s", totalPkts, oldest)
	}
}

// pickBestObservation selects the observation with the longest path
// and sets it as the transmission's display observation.
func pickBestObservation(tx *StoreTx) {
	if len(tx.Observations) == 0 {
		return
	}
	best := tx.Observations[0]
	bestLen := pathLen(best.PathJSON)
	for _, obs := range tx.Observations[1:] {
		l := pathLen(obs.PathJSON)
		if l > bestLen {
			best = obs
			bestLen = l
		}
	}
	tx.ObserverID = best.ObserverID
	tx.ObserverName = best.ObserverName
	tx.ObserverIATA = best.ObserverIATA
	tx.SNR = best.SNR
	tx.RSSI = best.RSSI
	tx.PathJSON = best.PathJSON
	tx.Direction = best.Direction
	tx.pathParsed = false // invalidate cached parsed path
}

func pathLen(pathJSON string) int {
	if pathJSON == "" {
		return 0
	}
	var hops []interface{}
	if json.Unmarshal([]byte(pathJSON), &hops) != nil {
		return 0
	}
	return len(hops)
}

// indexByNode extracts pubkeys from decoded_json and indexes the transmission.
// indexByNode indexes a transmission under all pubkeys found in its decoded
// JSON. Resolved path pubkeys are handled separately via the decode-window.
// Returns true if any genuinely new node was discovered.
func (s *PacketStore) indexByNode(tx *StoreTx) bool {
	// Track which pubkeys have been indexed for this packet to avoid duplicates.
	indexed := make(map[string]bool)
	foundNew := false

	// Index by decoded JSON fields (pubKey, destPubKey, srcPubKey).
	if tx.DecodedJSON != "" && strings.Contains(tx.DecodedJSON, "ubKey") {
		if decoded := tx.ParsedDecoded(); decoded != nil {
			for _, field := range []string{"pubKey", "destPubKey", "srcPubKey"} {
				if v, ok := decoded[field].(string); ok && v != "" {
					if s.addToByNode(tx, v) {
						foundNew = true
					}
					indexed[v] = true
				}
			}
		}
	}

	return foundNew
}

// addToByNode adds tx to byNode[pubkey] with dedup via nodeHashes.
// Returns true if this is a genuinely new node (pubkey not seen before).
func (s *PacketStore) addToByNode(tx *StoreTx, pubkey string) bool {
	isNew := s.nodeHashes[pubkey] == nil
	if isNew {
		s.nodeHashes[pubkey] = make(map[string]bool)
	}
	if s.nodeHashes[pubkey][tx.Hash] {
		return false
	}
	s.nodeHashes[pubkey][tx.Hash] = true
	s.byNode[pubkey] = append(s.byNode[pubkey], tx)
	return isNew
}

// touchRelayLastSeen updates last_seen in the DB for relay nodes that appear
// in resolved paths. Debounced to at most 1 write per node per 5 minutes.
// resolvedPubkeys is the pre-extracted list from the decode window.
// Must be called under s.mu write lock (reads/writes lastSeenTouched).
func (s *PacketStore) touchRelayLastSeen(resolvedPubkeys []string, now time.Time) {
	if s.db == nil || len(resolvedPubkeys) == 0 {
		return
	}
	const debounceInterval = 5 * time.Minute

	ts := now.UTC().Format(time.RFC3339)
	seen := make(map[string]bool, len(resolvedPubkeys))
	for _, pk := range resolvedPubkeys {
		if pk == "" || seen[pk] {
			continue
		}
		seen[pk] = true
		if last, ok := s.lastSeenTouched[pk]; ok && now.Sub(last) < debounceInterval {
			continue
		}
		if err := s.db.TouchNodeLastSeen(pk, ts); err == nil {
			s.lastSeenTouched[pk] = now
		}
	}
}

// trackAdvertPubkey increments the advertPubkeys refcount for ADVERT packets.
// Must be called under s.mu write lock.
func (s *PacketStore) trackAdvertPubkey(tx *StoreTx) {
	if tx.PayloadType == nil || *tx.PayloadType != PayloadADVERT || tx.DecodedJSON == "" {
		return
	}
	d := tx.ParsedDecoded()
	if d == nil {
		return
	}
	pk := ""
	if v, ok := d["pubKey"].(string); ok {
		pk = v
	} else if v, ok := d["public_key"].(string); ok {
		pk = v
	}
	if pk != "" {
		s.advertPubkeys[pk]++
	}
}

// untrackAdvertPubkey decrements the advertPubkeys refcount for ADVERT packets.
// Must be called under s.mu write lock.
func (s *PacketStore) untrackAdvertPubkey(tx *StoreTx) {
	if tx.PayloadType == nil || *tx.PayloadType != PayloadADVERT || tx.DecodedJSON == "" {
		return
	}
	var d map[string]interface{}
	if json.Unmarshal([]byte(tx.DecodedJSON), &d) != nil {
		return
	}
	pk := ""
	if v, ok := d["pubKey"].(string); ok {
		pk = v
	} else if v, ok := d["public_key"].(string); ok {
		pk = v
	}
	if pk != "" {
		if s.advertPubkeys[pk] <= 1 {
			delete(s.advertPubkeys, pk)
		} else {
			s.advertPubkeys[pk]--
		}
	}
}

// QueryPackets returns filtered, paginated packets from memory.
func (s *PacketStore) QueryPackets(q PacketQuery) *PacketResult {
	// SQL fallback: if the query window predates the in-memory window, delegate
	// to the DB layer which covers the full SQLite retention period.
	s.mu.RLock()
	oldest := s.oldestLoaded
	s.mu.RUnlock()
	if oldest != "" {
		needsSQL := (q.Since != "" && q.Since < oldest) ||
			(q.Until != "" && q.Until < oldest)
		if needsSQL {
			if result, err := s.db.QueryPackets(q); err == nil {
				return result
			} else {
				log.Printf("[store] QueryPackets SQL fallback failed: %v — using in-memory", err)
			}
		}
	}
	atomic.AddInt64(&s.queryCount, 1)
	s.mu.RLock()
	defer s.mu.RUnlock()

	if q.Limit <= 0 {
		q.Limit = 50
	}
	if q.Order == "" {
		q.Order = "DESC"
	}

	results := s.filterPackets(q)
	total := len(results)

	// results is oldest-first (ASC). For DESC (default) read backwards from the tail;
	// for ASC read forwards. Both are O(page_size) — no sort copy needed.
	start := q.Offset
	if start >= total {
		return &PacketResult{Packets: []map[string]interface{}{}, Total: total}
	}
	pageSize := q.Limit
	if start+pageSize > total {
		pageSize = total - start
	}

	packets := make([]map[string]interface{}, 0, pageSize)
	if q.Order == "ASC" {
		for _, tx := range results[start : start+pageSize] {
			packets = append(packets, s.txToMapWithRP(tx, q.ExpandObservations))
		}
	} else {
		// DESC: newest items are at the tail; page 0 = last pageSize items reversed
		endIdx := total - start
		startIdx := endIdx - pageSize
		if startIdx < 0 {
			startIdx = 0
		}
		for i := endIdx - 1; i >= startIdx; i-- {
			packets = append(packets, s.txToMapWithRP(results[i], q.ExpandObservations))
		}
	}
	return &PacketResult{Packets: packets, Total: total}
}

// QueryGroupedPackets returns transmissions grouped by hash (already 1:1).
func (s *PacketStore) QueryGroupedPackets(q PacketQuery) *PacketResult {
	s.mu.RLock()
	oldest := s.oldestLoaded
	s.mu.RUnlock()
	if oldest != "" {
		needsSQL := (q.Since != "" && q.Since < oldest) ||
			(q.Until != "" && q.Until < oldest)
		if needsSQL {
			if result, err := s.db.QueryGroupedPackets(q); err == nil {
				return result
			} else {
				log.Printf("[store] QueryGroupedPackets SQL fallback failed: %v — using in-memory", err)
			}
		}
	}
	atomic.AddInt64(&s.queryCount, 1)

	if q.Limit <= 0 {
		q.Limit = 50
	}

	// Cache key covers all filter dimensions. Empty key = no filters.
	cacheKey := q.Since + "|" + q.Until + "|" + q.Region + "|" + q.Node + "|" + q.Hash + "|" + q.Observer + "|" + q.Channel
	if q.Type != nil {
		cacheKey += fmt.Sprintf("|t%d", *q.Type)
	}
	if q.Route != nil {
		cacheKey += fmt.Sprintf("|r%d", *q.Route)
	}

	// Return cached sorted list if still fresh (3s TTL)
	s.groupedCacheMu.Lock()
	if s.groupedCacheTxs != nil && s.groupedCacheKey == cacheKey && time.Now().Before(s.groupedCacheExp) {
		cachedTxs := s.groupedCacheTxs
		cachedTotal := s.groupedCacheTotal
		s.groupedCacheMu.Unlock()
		return groupedTxsToPage(cachedTxs, cachedTotal, q.Offset, q.Limit)
	}
	s.groupedCacheMu.Unlock()

	// Collect StoreTx pointers under read lock; sort outside it.
	s.mu.RLock()
	results := s.filterPackets(q)
	txs := make([]*StoreTx, len(results))
	copy(txs, results)
	s.mu.RUnlock()

	total := len(txs)

	// Full sort by LatestSeen DESC so the cached slice supports all page offsets.
	sort.Slice(txs, func(i, j int) bool {
		return txs[i].LatestSeen > txs[j].LatestSeen
	})

	// Cache the sorted StoreTx slice (not maps) — lightweight and reusable for any page.
	s.groupedCacheMu.Lock()
	s.groupedCacheTxs = txs
	s.groupedCacheTotal = total
	s.groupedCacheKey = cacheKey
	s.groupedCacheExp = time.Now().Add(3 * time.Second)
	s.groupedCacheMu.Unlock()

	return groupedTxsToPage(txs, total, q.Offset, q.Limit)
}

// pagePacketResult returns a window of a PacketResult without re-allocating the slice.
func pagePacketResult(r *PacketResult, offset, limit int) *PacketResult {
	total := r.Total
	if offset >= total {
		return &PacketResult{Packets: []map[string]interface{}{}, Total: total}
	}
	end := offset + limit
	if end > total {
		end = total
	}
	return &PacketResult{Packets: r.Packets[offset:end], Total: total}
}

// groupedTxsToPage builds map representations only for the requested page of sorted StoreTx pointers.
// This avoids allocating maps for all 30K+ transmissions when only 50 are needed.
func groupedTxsToPage(txs []*StoreTx, total, offset, limit int) *PacketResult {
	if offset >= len(txs) {
		return &PacketResult{Packets: []map[string]interface{}{}, Total: total}
	}
	end := offset + limit
	if end > len(txs) {
		end = len(txs)
	}
	page := txs[offset:end]

	packets := make([]map[string]interface{}, len(page))
	for i, tx := range page {
		// #1189 R2: compute distinct IATA set across all observations.
		// Frontend uses this in the default collapsed view to show CROSS-region
		// reception at a glance — see groupedObserverIataBadgesHtml in
		// public/packets.js.
		distinctIatas := storeTxDistinctIatas(tx)
		m := map[string]interface{}{
			"hash":              strOrNil(tx.Hash),
			"first_seen":        strOrNil(tx.FirstSeen),
			"count":             tx.ObservationCount,
			"observer_count":    tx.UniqueObserverCount,
			"observation_count": tx.ObservationCount,
			"latest":            strOrNil(tx.LatestSeen),
			"observer_id":       strOrNil(tx.ObserverID),
			"observer_name":     strOrNil(tx.ObserverName),
			"observer_iata":     strOrNil(tx.ObserverIATA),
			"distinct_iatas":    distinctIatas,
			"path_json":         strOrNil(tx.PathJSON),
			"payload_type":      intPtrOrNil(tx.PayloadType),
			"route_type":        intPtrOrNil(tx.RouteType),
			"raw_hex":           strOrNil(tx.RawHex),
			"decoded_json":      strOrNil(tx.DecodedJSON),
			"snr":               floatPtrOrNil(tx.SNR),
			"rssi":              floatPtrOrNil(tx.RSSI),
		}
		// resolved_path omitted for grouped view (cold path, not worth SQL round-trip)
		packets[i] = m
	}

	return &PacketResult{Packets: packets, Total: total}
}

// storeTxDistinctIatas (#1189 R2) returns a sorted, deduped list of observer
// IATA codes for a StoreTx, excluding empty values. Returns an empty
// (non-nil) []string when the tx has no IATA'd observations so JSON
// serialization stays consistent across the in-memory store and SQL
// fallback paths (db.go's parseDistinctIatasCSV does the same).
func storeTxDistinctIatas(tx *StoreTx) []string {
	if tx == nil {
		return []string{}
	}
	seen := make(map[string]bool)
	// Include the header observer's IATA (some hot-path StoreTx records the
	// chosen observer fields directly without re-populating Observations).
	if tx.ObserverIATA != "" {
		seen[tx.ObserverIATA] = true
	}
	for _, o := range tx.Observations {
		if o != nil && o.ObserverIATA != "" {
			seen[o.ObserverIATA] = true
		}
	}
	out := make([]string, 0, len(seen))
	for k := range seen {
		out = append(out, k)
	}
	sort.Strings(out)
	return out
}

// GetStoreStats returns aggregate counts (packet data from memory, node/observer from DB).
func (s *PacketStore) GetStoreStats() (*Stats, error) {
	s.mu.RLock()
	txCount := len(s.packets)
	obsCount := s.totalObs
	s.mu.RUnlock()

	st := &Stats{
		TotalTransmissions: txCount,
		TotalPackets:       txCount,
		TotalObservations:  obsCount,
	}

	sevenDaysAgo := time.Now().Add(-7 * 24 * time.Hour).Format(time.RFC3339)
	oneHourAgo := time.Now().Add(-1 * time.Hour).Unix()
	oneDayAgo := time.Now().Add(-24 * time.Hour).Unix()

	// Run node/observer counts and observation counts concurrently (2 queries instead of 5).
	var wg sync.WaitGroup
	var nodeErr, obsErr error

	wg.Add(2)
	go func() {
		defer wg.Done()
		nodeErr = s.db.conn.QueryRow(
			`SELECT
				(SELECT COUNT(*) FROM nodes WHERE last_seen > ?) AS active_nodes,
				(SELECT COUNT(*) FROM nodes) AS all_nodes,
				(SELECT COUNT(*) FROM observers) AS observers`,
			sevenDaysAgo,
		).Scan(&st.TotalNodes, &st.TotalNodesAllTime, &st.TotalObservers)
	}()
	go func() {
		defer wg.Done()
		obsErr = s.db.conn.QueryRow(
			`SELECT
				COALESCE(SUM(CASE WHEN timestamp > ? THEN 1 ELSE 0 END), 0),
				COALESCE(SUM(CASE WHEN timestamp > ? THEN 1 ELSE 0 END), 0)
			FROM observations WHERE timestamp > ?`,
			oneHourAgo, oneDayAgo, oneDayAgo,
		).Scan(&st.PacketsLastHour, &st.PacketsLast24h)
	}()
	wg.Wait()

	if nodeErr != nil {
		return st, nodeErr
	}
	if obsErr != nil {
		return st, obsErr
	}

	return st, nil
}

// GetPerfStoreStats returns packet store statistics for /api/perf.
func (s *PacketStore) GetPerfStoreStats() map[string]interface{} {
	s.mu.RLock()
	totalLoaded := len(s.packets)
	totalObs := s.totalObs
	hashIdx := len(s.byHash)
	txIdx := len(s.byTxID)
	obsIdx := len(s.byObsID)
	observerIdx := len(s.byObserver)
	nodeIdx := len(s.byNode)
	pathHopIdx := len(s.byPathHop)
	ptIdx := len(s.byPayloadType)
	oldestLoaded := s.oldestLoaded
	retentionHours := s.retentionHours
	maxMemoryMB := s.maxMemoryMB
	hotStartupHours := s.hotStartupHours

	// Distinct advert pubkey count — precomputed incrementally (see trackAdvertPubkey).
	advertByObsCount := len(s.advertPubkeys)
	s.mu.RUnlock()

	estimatedMB := math.Round(s.estimatedMemoryMB()*10) / 10
	trackedMB := math.Round(s.trackedMemoryMB()*10) / 10

	evicted := atomic.LoadInt64(&s.evicted)

	return map[string]interface{}{
		"totalLoaded":            totalLoaded,
		"totalObservations":      totalObs,
		"evicted":                evicted,
		"inserts":                atomic.LoadInt64(&s.insertCount),
		"queries":                atomic.LoadInt64(&s.queryCount),
		"inMemory":               totalLoaded,
		"sqliteOnly":             false,
		"retentionHours":         retentionHours,
		"maxMemoryMB":            maxMemoryMB,
		"oldestLoaded":           oldestLoaded,
		"estimatedMB":            estimatedMB,
		"trackedMB":              trackedMB,
		"hotStartupHours":        hotStartupHours,
		"backgroundLoadComplete": s.backgroundLoadDone.Load(),
		"backgroundLoadFailed":   s.backgroundLoadFailed.Load(),
		"backgroundLoadProgress": s.backgroundLoadProgress.Load(),
		"indexes": map[string]interface{}{
			"byHash":           hashIdx,
			"byTxID":           txIdx,
			"byObsID":          obsIdx,
			"byObserver":       observerIdx,
			"byNode":           nodeIdx,
			"byPathHop":        pathHopIdx,
			"byPayloadType":    ptIdx,
			"advertByObserver": advertByObsCount,
		},
	}
}

// GetCacheStats returns RF cache hit/miss statistics.
func (s *PacketStore) GetCacheStats() map[string]interface{} {
	s.cacheMu.Lock()
	size := len(s.rfCache) + len(s.topoCache) + len(s.hashCache) + len(s.chanCache) + len(s.distCache) + len(s.subpathCache)
	hits := s.cacheHits
	misses := s.cacheMisses
	s.cacheMu.Unlock()

	var hitRate float64
	if hits+misses > 0 {
		hitRate = math.Round(float64(hits)/float64(hits+misses)*1000) / 10
	}

	return map[string]interface{}{
		"size":       size,
		"hits":       hits,
		"misses":     misses,
		"staleHits":  0,
		"recomputes": misses,
		"hitRate":    hitRate,
	}
}

// GetCacheStatsTyped returns cache stats as a typed struct.
func (s *PacketStore) GetCacheStatsTyped() CacheStats {
	s.cacheMu.Lock()
	size := len(s.rfCache) + len(s.topoCache) + len(s.hashCache) + len(s.chanCache) + len(s.distCache) + len(s.subpathCache)
	hits := s.cacheHits
	misses := s.cacheMisses
	s.cacheMu.Unlock()

	var hitRate float64
	if hits+misses > 0 {
		hitRate = math.Round(float64(hits)/float64(hits+misses)*1000) / 10
	}

	return CacheStats{
		Entries:    size,
		Hits:       hits,
		Misses:     misses,
		StaleHits:  0,
		Recomputes: misses,
		HitRate:    hitRate,
	}
}

// cacheInvalidation flags indicate what kind of data changed during ingestion.
// Used by invalidateCachesFor to selectively clear only affected caches.
type cacheInvalidation struct {
	hasNewObservations  bool // new SNR/RSSI data → rfCache
	hasNewPaths         bool // new/changed path data → topoCache, distCache, subpathCache
	hasNewTransmissions bool // new transmissions → hashCache
	hasNewNodes         bool // genuinely new node pubkey discovered → collisionCache
	hasChannelData      bool // new GRP_TXT (payload_type 5) → chanCache
	eviction            bool // data removed → all caches
}

// invalidateCachesFor selectively clears only the analytics caches affected
// by the kind of data that changed. To prevent continuous ingestion from
// defeating caching entirely (issue #533), invalidation is rate-limited:
// if called within invCooldown of the last invalidation, the flags are
// accumulated in pendingInv and applied on the next call after cooldown.
func (s *PacketStore) invalidateCachesFor(inv cacheInvalidation) {
	s.cacheMu.Lock()
	defer s.cacheMu.Unlock()

	// Eviction bypasses rate-limiting — data was removed, caches must clear.
	if inv.eviction {
		s.rfCache = make(map[string]*cachedResult)
		s.topoCache = make(map[string]*cachedResult)
		s.hashCache = make(map[string]*cachedResult)
		s.collisionCache = make(map[string]*cachedResult)
		s.chanCache = make(map[string]*cachedResult)
		s.distCache = make(map[string]*cachedResult)
		s.subpathCache = make(map[string]*cachedResult)
		s.channelsCacheMu.Lock()
		s.channelsCacheRes = nil
		s.channelsCacheMu.Unlock()
		s.lastInvalidated = time.Now()
		s.pendingInv = nil
		return
	}

	now := time.Now()
	if now.Sub(s.lastInvalidated) < s.invCooldown {
		// Within cooldown — accumulate dirty flags
		if s.pendingInv == nil {
			s.pendingInv = &cacheInvalidation{}
		}
		s.pendingInv.hasNewObservations = s.pendingInv.hasNewObservations || inv.hasNewObservations
		s.pendingInv.hasNewPaths = s.pendingInv.hasNewPaths || inv.hasNewPaths
		s.pendingInv.hasNewTransmissions = s.pendingInv.hasNewTransmissions || inv.hasNewTransmissions
		s.pendingInv.hasNewNodes = s.pendingInv.hasNewNodes || inv.hasNewNodes
		s.pendingInv.hasChannelData = s.pendingInv.hasChannelData || inv.hasChannelData
		return
	}

	// Cooldown expired — merge any pending flags and apply
	if s.pendingInv != nil {
		inv.hasNewObservations = inv.hasNewObservations || s.pendingInv.hasNewObservations
		inv.hasNewPaths = inv.hasNewPaths || s.pendingInv.hasNewPaths
		inv.hasNewTransmissions = inv.hasNewTransmissions || s.pendingInv.hasNewTransmissions
		inv.hasNewNodes = inv.hasNewNodes || s.pendingInv.hasNewNodes
		inv.hasChannelData = inv.hasChannelData || s.pendingInv.hasChannelData
		s.pendingInv = nil
	}

	s.applyCacheInvalidation(inv)
	s.lastInvalidated = now
}

// applyCacheInvalidation performs the actual cache clearing. Must be called
// with cacheMu held.
func (s *PacketStore) applyCacheInvalidation(inv cacheInvalidation) {
	if inv.hasNewObservations {
		s.rfCache = make(map[string]*cachedResult)
	}
	if inv.hasNewPaths {
		s.topoCache = make(map[string]*cachedResult)
		s.distCache = make(map[string]*cachedResult)
		s.subpathCache = make(map[string]*cachedResult)
	}
	if inv.hasNewTransmissions {
		s.hashCache = make(map[string]*cachedResult)
	}
	if inv.hasNewNodes {
		s.collisionCache = make(map[string]*cachedResult)
	}
	if inv.hasChannelData {
		s.chanCache = make(map[string]*cachedResult)
		s.channelsCacheMu.Lock()
		s.channelsCacheRes = nil
		s.channelsCacheMu.Unlock()
	}
}

// GetPerfStoreStatsTyped returns packet store stats as a typed struct.
func (s *PacketStore) GetPerfStoreStatsTyped() PerfPacketStoreStats {
	s.mu.RLock()
	totalLoaded := len(s.packets)
	totalObs := s.totalObs
	hashIdx := len(s.byHash)
	observerIdx := len(s.byObserver)
	nodeIdx := len(s.byNode)

	advertByObsCount := len(s.advertPubkeys)
	s.mu.RUnlock()

	estimatedMB := math.Round(s.estimatedMemoryMB()*10) / 10
	trackedMB := math.Round(s.trackedMemoryMB()*10) / 10

	var avgBytesPerPacket int64
	if totalLoaded > 0 {
		avgBytesPerPacket = s.trackedBytes / int64(totalLoaded)
	}

	return PerfPacketStoreStats{
		TotalLoaded:              totalLoaded,
		TotalObservations:        totalObs,
		Evicted:                  int(atomic.LoadInt64(&s.evicted)),
		Inserts:                  atomic.LoadInt64(&s.insertCount),
		Queries:                  atomic.LoadInt64(&s.queryCount),
		InMemory:                 totalLoaded,
		SqliteOnly:               false,
		MaxPackets:               2386092,
		EstimatedMB:              estimatedMB,
		TrackedMB:                trackedMB,
		AvgBytesPerPacket:        avgBytesPerPacket,
		MaxMB:                    s.maxMemoryMB,
		Indexes: PacketStoreIndexes{
			ByHash:           hashIdx,
			ByObserver:       observerIdx,
			ByNode:           nodeIdx,
			AdvertByObserver: advertByObsCount,
		},
		HotStartupHours:        s.hotStartupHours,
		BackgroundLoadComplete: s.backgroundLoadDone.Load(),
		BackgroundLoadFailed:   s.backgroundLoadFailed.Load(),
		BackgroundLoadProgress: s.backgroundLoadProgress.Load(),
	}
}

// GetTransmissionByID returns a transmission by its DB ID, formatted as a map.
func (s *PacketStore) GetTransmissionByID(id int) map[string]interface{} {
	s.mu.RLock()
	defer s.mu.RUnlock()

	tx := s.byTxID[id]
	if tx == nil {
		return nil
	}
	return s.txToMapWithRP(tx, true)
}

// GetPacketByHash returns a transmission by content hash.
func (s *PacketStore) GetPacketByHash(hash string) map[string]interface{} {
	s.mu.RLock()
	defer s.mu.RUnlock()

	tx := s.byHash[strings.ToLower(hash)]
	if tx == nil {
		return nil
	}
	return s.txToMapWithRP(tx, true)
}

// GetPacketByID returns an observation (enriched with transmission fields) by observation ID.
func (s *PacketStore) GetPacketByID(id int) map[string]interface{} {
	s.mu.RLock()
	defer s.mu.RUnlock()

	obs := s.byObsID[id]
	if obs == nil {
		return nil
	}
	return s.enrichObs(obs)
}

// GetObservationsForHash returns all observations for a hash, enriched with transmission fields.
func (s *PacketStore) GetObservationsForHash(hash string) []map[string]interface{} {
	s.mu.RLock()
	defer s.mu.RUnlock()

	tx := s.byHash[strings.ToLower(hash)]
	if tx == nil {
		return []map[string]interface{}{}
	}

	result := make([]map[string]interface{}, 0, len(tx.Observations))
	for _, obs := range tx.Observations {
		result = append(result, s.enrichObs(obs))
	}
	return result
}

// GetTimestamps returns transmission first_seen timestamps after since, in ASC order.
func (s *PacketStore) GetTimestamps(since string) []string {
	s.mu.RLock()
	defer s.mu.RUnlock()

	// packets sorted oldest-first — scan from tail until we reach items older than since
	var result []string
	for i := len(s.packets) - 1; i >= 0; i-- {
		tx := s.packets[i]
		if tx.FirstSeen <= since {
			break
		}
		result = append(result, tx.FirstSeen)
	}
	// result is currently newest-first; reverse to return ASC order
	for i, j := 0, len(result)-1; i < j; i, j = i+1, j-1 {
		result[i], result[j] = result[j], result[i]
	}
	return result
}

// QueryMultiNodePackets filters packets matching any of the given pubkeys.
func (s *PacketStore) QueryMultiNodePackets(pubkeys []string, limit, offset int, order, since, until string) *PacketResult {
	s.mu.RLock()
	defer s.mu.RUnlock()

	if len(pubkeys) == 0 {
		return &PacketResult{Packets: []map[string]interface{}{}, Total: 0}
	}
	if limit <= 0 {
		limit = 50
	}

	resolved := make([]string, len(pubkeys))
	for i, pk := range pubkeys {
		resolved[i] = s.db.resolveNodePubkey(pk)
	}

	// Use byNode index instead of scanning all packets (O(indexed) vs O(all×pubkeys×json)).
	hashSet := make(map[string]bool)
	var filtered []*StoreTx
	for _, pk := range resolved {
		for _, tx := range s.byNode[pk] {
			if hashSet[tx.Hash] {
				continue
			}
			if since != "" && tx.FirstSeen < since {
				continue
			}
			if until != "" && tx.FirstSeen > until {
				continue
			}
			hashSet[tx.Hash] = true
			filtered = append(filtered, tx)
		}
	}
	// Sort oldest-first to match pagination expectations (same as s.packets order).
	sort.Slice(filtered, func(i, j int) bool {
		return filtered[i].FirstSeen < filtered[j].FirstSeen
	})

	total := len(filtered)

	// filtered is oldest-first (built by iterating s.packets forward).
	// Apply same DESC/ASC pagination logic as QueryPackets.
	if offset >= total {
		return &PacketResult{Packets: []map[string]interface{}{}, Total: total}
	}
	pageSize := limit
	if offset+pageSize > total {
		pageSize = total - offset
	}

	packets := make([]map[string]interface{}, 0, pageSize)
	if order == "ASC" {
		for _, tx := range filtered[offset : offset+pageSize] {
			packets = append(packets, s.txToMapWithRP(tx))
		}
	} else {
		endIdx := total - offset
		startIdx := endIdx - pageSize
		if startIdx < 0 {
			startIdx = 0
		}
		for i := endIdx - 1; i >= startIdx; i-- {
			packets = append(packets, s.txToMapWithRP(filtered[i]))
		}
	}
	return &PacketResult{Packets: packets, Total: total}
}

// IngestNewFromDB loads new transmissions from SQLite into memory and returns
// broadcast-ready maps plus the new max transmission ID.
func (s *PacketStore) IngestNewFromDB(sinceID, limit int) ([]map[string]interface{}, int) {
	if limit <= 0 {
		limit = 100
	}

	// NOTE: The SQL query intentionally does NOT select resolved_path from the DB.
	// New ingests always resolve fresh using the current prefix map and neighbor graph.
	// On restart, Load() handles reading persisted resolved_path values. (review item #7)
	var querySQL string
	obsRHCol := ""
	if s.db.hasObsRawHex {
		obsRHCol = ", o.raw_hex"
	}
	if s.db.isV3 {
		querySQL = `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type,
				t.payload_type, t.payload_version, t.decoded_json,
				o.id, obs.id, obs.name, COALESCE(obs.iata, ''), o.direction,
				o.snr, o.rssi, o.score, o.path_json, strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch')` + obsRHCol + `
			FROM transmissions t
			LEFT JOIN observations o ON o.transmission_id = t.id
			LEFT JOIN observers obs ON obs.rowid = o.observer_idx
			WHERE t.id > ?
			ORDER BY t.id ASC, o.timestamp DESC`
	} else {
		querySQL = `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type,
				t.payload_type, t.payload_version, t.decoded_json,
				o.id, o.observer_id, o.observer_name, COALESCE(obs.iata, ''), o.direction,
				o.snr, o.rssi, o.score, o.path_json, o.timestamp` + obsRHCol + `
			FROM transmissions t
			LEFT JOIN observations o ON o.transmission_id = t.id
			LEFT JOIN observers obs ON obs.id = o.observer_id
			WHERE t.id > ?
			ORDER BY t.id ASC, o.timestamp DESC`
	}

	rows, err := s.db.conn.Query(querySQL, sinceID)
	if err != nil {
		log.Printf("[store] ingest query error: %v", err)
		return nil, sinceID
	}
	defer rows.Close()

	// Scan into temp structures
	type tempRow struct {
		txID                                                               int
		rawHex, hash, firstSeen, decodedJSON                               string
		routeType, payloadType                                             *int
		obsID                                                              *int
		observerID, observerName, observerIATA, direction, pathJSON, obsTS string
		obsRawHex                                                          string
		snr, rssi                                                          *float64
		score                                                              *int
	}

	var tempRows []tempRow
	txCount := 0
	lastTxID := sinceID

	for rows.Next() {
		var txID int
		var rawHex, hash, firstSeen, decodedJSON sql.NullString
		var routeType, payloadType, payloadVersion sql.NullInt64
		var obsIDVal sql.NullInt64
		var observerID, observerName, observerIATA, direction, pathJSON, obsTimestamp sql.NullString
		var snrVal, rssiVal sql.NullFloat64
		var scoreVal sql.NullInt64
		var obsRawHex sql.NullString

		scanArgs2 := []interface{}{&txID, &rawHex, &hash, &firstSeen, &routeType, &payloadType,
			&payloadVersion, &decodedJSON,
			&obsIDVal, &observerID, &observerName, &observerIATA, &direction,
			&snrVal, &rssiVal, &scoreVal, &pathJSON, &obsTimestamp}
		if s.db.hasObsRawHex {
			scanArgs2 = append(scanArgs2, &obsRawHex)
		}
		if err := rows.Scan(scanArgs2...); err != nil {
			continue
		}

		if txID != lastTxID {
			txCount++
			if txCount > limit {
				break
			}
			lastTxID = txID
		}

		tr := tempRow{
			txID:         txID,
			rawHex:       nullStrVal(rawHex),
			hash:         nullStrVal(hash),
			firstSeen:    nullStrVal(firstSeen),
			decodedJSON:  nullStrVal(decodedJSON),
			routeType:    nullIntPtr(routeType),
			payloadType:  nullIntPtr(payloadType),
			observerID:   nullStrVal(observerID),
			observerName: nullStrVal(observerName),
			observerIATA: nullStrVal(observerIATA),
			direction:    nullStrVal(direction),
			pathJSON:     nullStrVal(pathJSON),
			obsTS:        nullStrVal(obsTimestamp),
			obsRawHex:    nullStrVal(obsRawHex),
			snr:          nullFloatPtr(snrVal),
			rssi:         nullFloatPtr(rssiVal),
			score:        nullIntPtr(scoreVal),
		}
		if obsIDVal.Valid {
			oid := int(obsIDVal.Int64)
			tr.obsID = &oid
		}
		tempRows = append(tempRows, tr)
	}

	if len(tempRows) == 0 {
		return nil, sinceID
	}

	// Now lock and merge into store
	s.mu.Lock()
	defer s.mu.Unlock()

	newMaxID := sinceID
	broadcastTxs := make(map[int]*StoreTx) // track new transmissions for broadcast
	hasNewNodes := false                    // track genuinely new node pubkeys
	var broadcastOrder []int

	// Hoist getCachedNodesAndPM() once before the observation loop to avoid
	// per-observation function calls (review item #1).
	_, cachedPM := s.getCachedNodesAndPM()
	// Hoist atomic graph.Load() out of the per-row loop too (PR #1208
	// carmack #1) — one Load per ingest call, not one per row.
	cachedGraph := s.graph.Load()

	// Decode-window tracking: resolved pubkeys per-tx for touchRelayLastSeen,
	// and resolved paths per-obs for broadcast/persist.
	var broadcastRP map[int][]*string           // obsID → resolved path (for broadcast/persist)
	allResolvedPKs := make(map[int][]string)    // txID → all resolved pubkeys (for touchRelayLastSeen)

	hopsSeen := make(map[string]bool) // reused across observations; cleared per use

	for _, r := range tempRows {
		if r.txID > newMaxID {
			newMaxID = r.txID
		}

		tx := s.byHash[r.hash]
		if tx == nil {
			tx = &StoreTx{
				ID:          r.txID,
				RawHex:      r.rawHex,
				Hash:        r.hash,
				FirstSeen:   r.firstSeen,
				LatestSeen:  r.firstSeen,
				RouteType:   r.routeType,
				PayloadType: r.payloadType,
				DecodedJSON: r.decodedJSON,
				obsKeys:     make(map[string]bool),
				observerSet: make(map[string]bool),
			}
			s.byHash[r.hash] = tx
			s.packets = append(s.packets, tx) // oldest-first; new items go to tail
			s.byTxID[r.txID] = tx
			if r.txID > s.maxTxID {
				s.maxTxID = r.txID
			}
			if s.indexByNode(tx) {
				hasNewNodes = true
			}
			if tx.PayloadType != nil {
				pt := *tx.PayloadType
				// Append to maintain oldest-first order (matches Load ordering)
				// so GetChannelMessages reverse iteration stays correct
				s.byPayloadType[pt] = append(s.byPayloadType[pt], tx)
			}
			s.trackAdvertPubkey(tx)
			s.trackedBytes += estimateStoreTxBytes(tx)

			if _, exists := broadcastTxs[r.txID]; !exists {
				broadcastTxs[r.txID] = tx
				broadcastOrder = append(broadcastOrder, r.txID)
			}
		}

		if r.obsID != nil {
			oid := *r.obsID
			// Dedup (O(1) map lookup)
			dk := r.observerID + "|" + r.pathJSON
			if tx.obsKeys == nil {
				tx.obsKeys = make(map[string]bool)
				tx.observerSet = make(map[string]bool)
			}
			if tx.obsKeys[dk] {
				continue
			}

			obs := &StoreObs{
				ID:             oid,
				TransmissionID: r.txID,
				ObserverID:     r.observerID,
				ObserverName:   r.observerName,
				ObserverIATA:   r.observerIATA,
				Direction:      r.direction,
				SNR:            r.snr,
				RSSI:           r.rssi,
				Score:          r.score,
				PathJSON:       r.pathJSON,
				RawHex:         r.obsRawHex,
				Timestamp:      normalizeTimestamp(r.obsTS),
			}

			// Resolve path at ingest time using neighbor graph — decode-window discipline:
			// decode once, feed consumers, never store on struct.
			var resolvedPubkeys []string
			var rpForBroadcast []*string
			if r.pathJSON != "" && r.pathJSON != "[]" && cachedPM != nil {
				rpForBroadcast = resolvePathForObs(r.pathJSON, r.observerID, tx, cachedPM, cachedGraph)
				resolvedPubkeys = extractResolvedPubkeys(rpForBroadcast)
				// Feed decode-window consumers: addToByNode + resolvedPubkeyIndex
				for _, pk := range resolvedPubkeys {
					s.addToByNode(tx, pk)
				}
				s.addToResolvedPubkeyIndex(tx.ID, resolvedPubkeys)
				// byPathHop resolved-key entries
				clear(hopsSeen)
				for _, hop := range txGetParsedPath(tx) {
					hopsSeen[strings.ToLower(hop)] = true
				}
				for _, pk := range resolvedPubkeys {
					if !hopsSeen[pk] {
						hopsSeen[pk] = true
						s.byPathHop[pk] = append(s.byPathHop[pk], tx)
					}
				}
			}
			// Stash rpForBroadcast for later broadcast/persist (keyed by obs ID)
			if rpForBroadcast != nil {
				if broadcastRP == nil {
					broadcastRP = make(map[int][]*string)
				}
				broadcastRP[*r.obsID] = rpForBroadcast
			}
			// Collect resolved pubkeys per-tx for touchRelayLastSeen
			if len(resolvedPubkeys) > 0 {
				allResolvedPKs[r.txID] = append(allResolvedPKs[r.txID], resolvedPubkeys...)
			}

			tx.Observations = append(tx.Observations, obs)
			tx.obsKeys[dk] = true
			if obs.ObserverID != "" && !tx.observerSet[obs.ObserverID] {
				tx.observerSet[obs.ObserverID] = true
				tx.UniqueObserverCount++
			}
			tx.ObservationCount++
			if obs.Timestamp > tx.LatestSeen {
				tx.LatestSeen = obs.Timestamp
			}
			s.byObsID[oid] = obs
			if oid > s.maxObsID {
				s.maxObsID = oid
			}
			if r.observerID != "" {
				s.byObserver[r.observerID] = append(s.byObserver[r.observerID], obs)
			}
			s.totalObs++
			s.trackedBytes += estimateStoreObsBytes(obs)
		}
	}

	// Pick best observation for new transmissions
	for _, tx := range broadcastTxs {
		pickBestObservation(tx)
	}

	// Phase 2 of #660: update last_seen in DB for relay nodes seen in resolved_path.
	now := time.Now()
	for txID := range broadcastTxs {
		if pks, ok := allResolvedPKs[txID]; ok {
			s.touchRelayLastSeen(pks, now)
		}
	}

	// Incrementally update precomputed subpath index with new transmissions
	for _, tx := range broadcastTxs {
		if addTxToSubpathIndexFull(s.spIndex, s.spTxIndex, tx) {
			s.spTotalPaths++
		}
		addTxToPathHopIndex(s.byPathHop, tx)
	}

	// Incrementally update precomputed distance index with new transmissions
	if len(broadcastTxs) > 0 {
		allNodes, pm := s.getCachedNodesAndPM()
		nodeByPk := make(map[string]*nodeInfo, len(allNodes))
		repeaterSet := make(map[string]bool)
		for i := range allNodes {
			n := &allNodes[i]
			nodeByPk[n.PublicKey] = n
			if strings.Contains(strings.ToLower(n.Role), "repeater") {
				repeaterSet[n.PublicKey] = true
			}
		}
		// Per-tx hop resolver: cache reused across txs, context rebound per
		// tx via setContext (#1197 perf fix).
		resolveHop, setContext := s.hopResolverPerTx(pm)
		for _, tx := range broadcastTxs {
			// Per-tx context (sender + observer + unambiguous-prefix anchors)
			// so resolveWithContext tiers 1 and 2 light up. See #1197.
			setContext(buildHopContextPubkeys(tx, pm))
			txHops, txPath := computeDistancesForTx(tx, nodeByPk, repeaterSet, resolveHop)
			if len(txHops) > 0 {
				s.distHops = append(s.distHops, txHops...)
			}
			if txPath != nil {
				s.distPaths = append(s.distPaths, *txPath)
			}
		}
	}

	// Build broadcast maps (same shape as Node.js WS broadcast), one per observation.
	result := make([]map[string]interface{}, 0, len(broadcastOrder))
	for _, txID := range broadcastOrder {
		tx := broadcastTxs[txID]
		// Build decoded object with header.payloadTypeName for live.js
		decoded := map[string]interface{}{
			"header": map[string]interface{}{
				"payloadTypeName": resolvePayloadTypeName(tx.PayloadType),
			},
		}
		if tx.DecodedJSON != "" {
			var payload map[string]interface{}
			if json.Unmarshal([]byte(tx.DecodedJSON), &payload) == nil {
				decoded["payload"] = payload
			}
		}
		// For TRACE packets, decode the full packet to include path.hopsCompleted
		// so the frontend can distinguish completed vs remaining hops (#683).
		if tx.PayloadType != nil && *tx.PayloadType == PayloadTRACE && tx.RawHex != "" {
			if dp, err := DecodePacket(tx.RawHex, false); err == nil {
				decoded["path"] = dp.Path
			}
		}
		for _, obs := range tx.Observations {
			// Build the nested packet object (packets.js checks m.data.packet)
			pkt := map[string]interface{}{
				"id":                tx.ID,
				"raw_hex":           strOrNil(tx.RawHex),
				"hash":              strOrNil(tx.Hash),
				"first_seen":        strOrNil(tx.FirstSeen),
				"timestamp":         strOrNil(tx.FirstSeen),
				"route_type":        intPtrOrNil(tx.RouteType),
				"payload_type":      intPtrOrNil(tx.PayloadType),
				"decoded_json":      strOrNil(tx.DecodedJSON),
				"observer_id":       strOrNil(obs.ObserverID),
				"observer_name":     strOrNil(obs.ObserverName),
				"snr":               floatPtrOrNil(obs.SNR),
				"rssi":              floatPtrOrNil(obs.RSSI),
				"path_json":         strOrNil(obs.PathJSON),
				"direction":         strOrNil(obs.Direction),
				"observation_count": tx.ObservationCount,
			}
			// Use decode-window resolved path for broadcast (never from struct)
			if broadcastRP != nil {
				if rp, ok := broadcastRP[obs.ID]; ok && rp != nil {
					pkt["resolved_path"] = rp
				}
			}
			// Broadcast map: top-level fields for live.js + nested packet for packets.js
			broadcastMap := make(map[string]interface{}, len(pkt)+2)
			for k, v := range pkt {
				broadcastMap[k] = v
			}
			broadcastMap["decoded"] = decoded
			broadcastMap["packet"] = pkt
			result = append(result, broadcastMap)
		}
	}

	// Targeted cache invalidation: only clear caches affected by the ingested
	// data instead of wiping everything on every cycle (fixes #375).
	if len(result) > 0 {
		inv := cacheInvalidation{
			hasNewTransmissions: len(broadcastTxs) > 0,
			hasNewNodes:         hasNewNodes,
		}
		for _, tx := range broadcastTxs {
			if len(tx.Observations) > 0 {
				inv.hasNewObservations = true
			}
			if tx.PayloadType != nil && *tx.PayloadType == 5 {
				inv.hasChannelData = true
			}
			if tx.PathJSON != "" {
				inv.hasNewPaths = true
			}
			if inv.hasNewObservations && inv.hasChannelData && inv.hasNewPaths {
				break // all flags set, no need to continue
			}
		}
		s.invalidateCachesFor(inv)
	}

	// Persist resolved paths and neighbor edges asynchronously (don't block ingest).
	if len(broadcastTxs) > 0 && s.db != nil {
		dbPath := s.db.path
		var obsUpdates []persistObsUpdate
		var edgeUpdates []persistEdgeUpdate

		_, pm := s.getCachedNodesAndPM()
		// graph is *atomic.Pointer[NeighborGraph]; the Load itself is
		// lock-free. (Earlier comment claimed "set during startup, not
		// replaced after" — that's no longer true: #1203 made rebuilds
		// async via ensureNeighborGraph. Dropping the dead s.mu RLock
		// wrap — review PR #1208.)
		graphRef := s.graph.Load()
		for _, tx := range broadcastTxs {
			for _, obs := range tx.Observations {
				// Use decode-window resolved path for persist
				if broadcastRP != nil {
					if rp, ok := broadcastRP[obs.ID]; ok && rp != nil {
						rpJSON := marshalResolvedPath(rp)
						if rpJSON != "" {
							obsUpdates = append(obsUpdates, persistObsUpdate{obs.ID, rpJSON})
						}
					}
				}
				for _, ec := range extractEdgesFromObs(obs, tx, pm) {
					edgeUpdates = append(edgeUpdates, persistEdgeUpdate{ec.A, ec.B, ec.Timestamp})
					if graphRef != nil {
						graphRef.upsertEdge(ec.A, ec.B, "", obs.ObserverID, obs.SNR, parseTimestamp(ec.Timestamp))
					}
				}
			}
		}

		asyncPersistResolvedPathsAndEdges(dbPath, obsUpdates, edgeUpdates, "persist")
	}

	return result, newMaxID
}

// IngestNewObservations loads new observations for transmissions already in the
// store. This catches observations that arrive after IngestNewFromDB has already
// advanced past the transmission's ID (fixes #174).
func (s *PacketStore) IngestNewObservations(sinceObsID, limit int) []map[string]interface{} {
	if limit <= 0 {
		limit = 500
	}

	var querySQL string
	obsRHCol2 := ""
	if s.db.hasObsRawHex {
		obsRHCol2 = ", o.raw_hex"
	}
	if s.db.isV3 {
		querySQL = `SELECT o.id, o.transmission_id, obs.id, obs.name, COALESCE(obs.iata, ''), o.direction,
				o.snr, o.rssi, o.score, o.path_json, strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch')` + obsRHCol2 + `
			FROM observations o
			LEFT JOIN observers obs ON obs.rowid = o.observer_idx
			WHERE o.id > ?
			ORDER BY o.id ASC
			LIMIT ?`
	} else {
		querySQL = `SELECT o.id, o.transmission_id, o.observer_id, o.observer_name, COALESCE(obs.iata, ''), o.direction,
				o.snr, o.rssi, o.score, o.path_json, o.timestamp` + obsRHCol2 + `
			FROM observations o
			LEFT JOIN observers obs ON obs.id = o.observer_id
			WHERE o.id > ?
			ORDER BY o.id ASC
			LIMIT ?`
	}

	rows, err := s.db.conn.Query(querySQL, sinceObsID, limit)
	if err != nil {
		log.Printf("[store] ingest observations query error: %v", err)
		return nil
	}
	defer rows.Close()

	type obsRow struct {
		obsID        int
		txID         int
		observerID   string
		observerName string
		observerIATA string
		direction    string
		snr, rssi    *float64
		score        *int
		pathJSON     string
		rawHex       string
		timestamp    string
	}

	var obsRows []obsRow
	for rows.Next() {
		var oid, txID int
		var observerID, observerName, observerIATA, direction, pathJSON, ts sql.NullString
		var snr, rssi sql.NullFloat64
		var score sql.NullInt64
		var obsRawHex sql.NullString

		scanArgs3 := []interface{}{&oid, &txID, &observerID, &observerName, &observerIATA, &direction,
			&snr, &rssi, &score, &pathJSON, &ts}
		if s.db.hasObsRawHex {
			scanArgs3 = append(scanArgs3, &obsRawHex)
		}
		if err := rows.Scan(scanArgs3...); err != nil {
			continue
		}

		obsRows = append(obsRows, obsRow{
			obsID:        oid,
			txID:         txID,
			observerID:   nullStrVal(observerID),
			observerName: nullStrVal(observerName),
			observerIATA: nullStrVal(observerIATA),
			direction:    nullStrVal(direction),
			snr:          nullFloatPtr(snr),
			rssi:         nullFloatPtr(rssi),
			score:        nullIntPtr(score),
			pathJSON:     nullStrVal(pathJSON),
			rawHex:       nullStrVal(obsRawHex),
			timestamp:    nullStrVal(ts),
		})
	}

	if len(obsRows) == 0 {
		return nil
	}

	s.mu.Lock()
	defer s.mu.Unlock()

	updatedTxs := make(map[int]*StoreTx)
	broadcastMaps := make([]map[string]interface{}, 0, len(obsRows))
	// Track newly created observations for persistence — only these should be
	// persisted, not all observations of each updated tx (fixes edge count inflation).
	var newObs []*StoreObs
	var obsRPMap map[int][]*string // obsID → resolved path (decode-window)

	// Hoist getCachedNodesAndPM() before the loop — same pattern as IngestNewFromDB (review fix #1).
	_, pm := s.getCachedNodesAndPM()
	graphRef := s.graph.Load()

	hopsSeen := make(map[string]bool) // reused across observations; cleared per use

	for _, r := range obsRows {
		// Already ingested (e.g. by IngestNewFromDB in same cycle)
		if _, exists := s.byObsID[r.obsID]; exists {
			continue
		}

		tx := s.byTxID[r.txID]
		if tx == nil {
			continue // transmission not yet in store
		}

		// Dedup by observer + path (O(1) map lookup)
		dk := r.observerID + "|" + r.pathJSON
		if tx.obsKeys == nil {
			tx.obsKeys = make(map[string]bool)
			tx.observerSet = make(map[string]bool)
		}
		if tx.obsKeys[dk] {
			continue
		}

		obs := &StoreObs{
			ID:             r.obsID,
			TransmissionID: r.txID,
			ObserverID:     r.observerID,
			ObserverName:   r.observerName,
			ObserverIATA:   r.observerIATA,
			Direction:      r.direction,
			SNR:            r.snr,
			RSSI:           r.rssi,
			Score:          r.score,
			PathJSON:       r.pathJSON,
			RawHex:         r.rawHex,
			Timestamp:      normalizeTimestamp(r.timestamp),
		}

		// Resolve path at ingest time for late-arriving observations (review item #2).
		// Decode-window discipline: decode, feed consumers, don't store on struct.
		var obsResolvedPath []*string
		if r.pathJSON != "" && r.pathJSON != "[]" {
			if pm != nil {
				obsResolvedPath = resolvePathForObs(r.pathJSON, r.observerID, tx, pm, graphRef)
				pks := extractResolvedPubkeys(obsResolvedPath)
				for _, pk := range pks {
					s.addToByNode(tx, pk)
				}
				s.addToResolvedPubkeyIndex(tx.ID, pks)
				// byPathHop resolved-key entries
				clear(hopsSeen)
				for _, hop := range txGetParsedPath(tx) {
					hopsSeen[strings.ToLower(hop)] = true
				}
				for _, pk := range pks {
					if !hopsSeen[pk] {
						hopsSeen[pk] = true
						s.byPathHop[pk] = append(s.byPathHop[pk], tx)
					}
				}
			}
		}
		// Stash for broadcast/persist
		if obsResolvedPath != nil {
			if obsRPMap == nil {
				obsRPMap = make(map[int][]*string)
			}
			obsRPMap[r.obsID] = obsResolvedPath
		}

		tx.Observations = append(tx.Observations, obs)
		tx.obsKeys[dk] = true
		if obs.ObserverID != "" && !tx.observerSet[obs.ObserverID] {
			tx.observerSet[obs.ObserverID] = true
			tx.UniqueObserverCount++
		}
		tx.ObservationCount++
		newObs = append(newObs, obs)
		if obs.Timestamp > tx.LatestSeen {
			tx.LatestSeen = obs.Timestamp
		}
		s.byObsID[r.obsID] = obs
		if r.obsID > s.maxObsID {
			s.maxObsID = r.obsID
		}
		if r.observerID != "" {
			s.byObserver[r.observerID] = append(s.byObserver[r.observerID], obs)
		}
		s.totalObs++
		s.trackedBytes += estimateStoreObsBytes(obs)
		updatedTxs[r.txID] = tx

		decoded := map[string]interface{}{
			"header": map[string]interface{}{
				"payloadTypeName": resolvePayloadTypeName(tx.PayloadType),
			},
		}
		if tx.DecodedJSON != "" {
			var payload map[string]interface{}
			if json.Unmarshal([]byte(tx.DecodedJSON), &payload) == nil {
				decoded["payload"] = payload
			}
		}
		// For TRACE packets, decode the full packet to include path.hopsCompleted
		// so the frontend can distinguish completed vs remaining hops (#683).
		if tx.PayloadType != nil && *tx.PayloadType == PayloadTRACE && tx.RawHex != "" {
			if dp, err := DecodePacket(tx.RawHex, false); err == nil {
				decoded["path"] = dp.Path
			}
		}

		pkt := map[string]interface{}{
			"id":                tx.ID,
			"raw_hex":           strOrNil(tx.RawHex),
			"hash":              strOrNil(tx.Hash),
			"first_seen":        strOrNil(tx.FirstSeen),
			"timestamp":         strOrNil(tx.FirstSeen),
			"route_type":        intPtrOrNil(tx.RouteType),
			"payload_type":      intPtrOrNil(tx.PayloadType),
			"decoded_json":      strOrNil(tx.DecodedJSON),
			"observer_id":       strOrNil(obs.ObserverID),
			"observer_name":     strOrNil(obs.ObserverName),
			"snr":               floatPtrOrNil(obs.SNR),
			"rssi":              floatPtrOrNil(obs.RSSI),
			"path_json":         strOrNil(obs.PathJSON),
			"direction":         strOrNil(obs.Direction),
			"observation_count": tx.ObservationCount,
		}
		// Use decode-window resolved path for broadcast
		if obsRPMap != nil {
			if rp, ok := obsRPMap[obs.ID]; ok && rp != nil {
				pkt["resolved_path"] = rp
			}
		}
		broadcastMap := make(map[string]interface{}, len(pkt)+2)
		for k, v := range pkt {
			broadcastMap[k] = v
		}
		broadcastMap["decoded"] = decoded
		broadcastMap["packet"] = pkt
		broadcastMaps = append(broadcastMaps, broadcastMap)
	}

	// Re-pick best observation for updated transmissions and update subpath index
	// if the path changed.
	oldPaths := make(map[int]string, len(updatedTxs))
	for txID, tx := range updatedTxs {
		oldPaths[txID] = tx.PathJSON
	}
	for _, tx := range updatedTxs {
		pickBestObservation(tx)
	}
	for txID, tx := range updatedTxs {
		if tx.PathJSON != oldPaths[txID] {
			// Path changed — remove old subpaths, add new ones.
			oldHops := parsePathJSON(oldPaths[txID])
			if len(oldHops) >= 2 {
				// Temporarily set parsedPath to old hops for removal.
				saved, savedFlag := tx.parsedPath, tx.pathParsed
				tx.parsedPath, tx.pathParsed = oldHops, true
				if removeTxFromSubpathIndexFull(s.spIndex, s.spTxIndex, tx) {
					s.spTotalPaths--
				}
				tx.parsedPath, tx.pathParsed = saved, savedFlag
			}
			// Remove old path-hop index entries using old hops.
			// Resolved pubkey entries are managed via resolvedPubkeyIndex, not byPathHop.
			if len(oldHops) > 0 {
				saved, savedFlag := tx.parsedPath, tx.pathParsed
				tx.parsedPath, tx.pathParsed = oldHops, true
				removeTxFromPathHopIndex(s.byPathHop, tx)
				tx.parsedPath, tx.pathParsed = saved, savedFlag
			}
			// pickBestObservation already set pathParsed=false so
			// addTxToSubpathIndex will re-parse the new path.
			if addTxToSubpathIndexFull(s.spIndex, s.spTxIndex, tx) {
				s.spTotalPaths++
			}
			addTxToPathHopIndex(s.byPathHop, tx)
		}
	}

	// Check if any paths changed (used for distance update and cache invalidation).
	hasPathChanges := false
	var changedTxs []*StoreTx
	for txID, tx := range updatedTxs {
		if tx.PathJSON != oldPaths[txID] {
			hasPathChanges = true
			changedTxs = append(changedTxs, tx)
		}
	}
	if len(changedTxs) > 0 {
		s.updateDistanceIndexForTxs(changedTxs)
	}

	if len(updatedTxs) > 0 {
		// Targeted cache invalidation: new observations always affect RF
		// analytics; topology/distance/subpath caches only if paths changed.
		// Channel and hash caches are unaffected by observation-only ingestion.
		s.invalidateCachesFor(cacheInvalidation{
			hasNewObservations: true,
			hasNewPaths:        hasPathChanges,
		})
	}

	// Persist resolved paths and neighbor edges asynchronously (review fix #3).
	// Only process NEW observations — not all observations of each updated tx —
	// to avoid edge count inflation and unnecessary UPDATEs for pre-existing data.
	if len(newObs) > 0 && s.db != nil {
		dbPath := s.db.path
		var obsUpdates []persistObsUpdate
		var edgeUpdates []persistEdgeUpdate

		for _, obs := range newObs {
			tx := s.byTxID[obs.TransmissionID]
			if tx == nil {
				continue
			}
			// Use decode-window resolved path for persist
			if obsRPMap != nil {
				if rp, ok := obsRPMap[obs.ID]; ok && rp != nil {
					rpJSON := marshalResolvedPath(rp)
					if rpJSON != "" {
						obsUpdates = append(obsUpdates, persistObsUpdate{obs.ID, rpJSON})
					}
				}
			}
			for _, ec := range extractEdgesFromObs(obs, tx, pm) {
				edgeUpdates = append(edgeUpdates, persistEdgeUpdate{ec.A, ec.B, ec.Timestamp})
				if graphRef != nil {
					graphRef.upsertEdge(ec.A, ec.B, "", obs.ObserverID, obs.SNR, parseTimestamp(ec.Timestamp))
				}
			}
		}

		asyncPersistResolvedPathsAndEdges(dbPath, obsUpdates, edgeUpdates, "obs-persist")
	}

	return broadcastMaps
}

// MaxTransmissionID returns the highest transmission ID in the store.
func (s *PacketStore) MaxTransmissionID() int {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.maxTxID
}

// MaxObservationID returns the highest observation ID in the store.
func (s *PacketStore) MaxObservationID() int {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.maxObsID
}

// --- Internal filter/query helpers ---

// filterPackets applies PacketQuery filters to the in-memory packet list.
func (s *PacketStore) filterPackets(q PacketQuery) []*StoreTx {
	// Fast path: single-key index lookups
	if q.Hash != "" && q.Type == nil && q.Route == nil && q.Observer == "" &&
		q.Region == "" && q.Node == "" && q.Channel == "" && q.Since == "" && q.Until == "" {
		h := strings.ToLower(q.Hash)
		tx := s.byHash[h]
		if tx == nil {
			return nil
		}
		return []*StoreTx{tx}
	}
	if q.Observer != "" && q.Type == nil && q.Route == nil &&
		q.Region == "" && q.Node == "" && q.Channel == "" && q.Hash == "" && q.Since == "" && q.Until == "" {
		return s.transmissionsForObserver(q.Observer, nil)
	}

	// Pre-compute filter parameters outside the hot loop.
	var (
		filterType  int
		hasType     bool
		filterRoute int
		hasRoute    bool
		filterHash  string
		hasSince    = q.Since != ""
		hasUntil    = q.Until != ""
	)
	if q.Type != nil {
		hasType = true
		filterType = *q.Type
	}
	if q.Route != nil {
		hasRoute = true
		filterRoute = *q.Route
	}
	if q.Hash != "" {
		filterHash = strings.ToLower(q.Hash)
	}
	filterChannel := q.Channel

	// Pre-compute observer set for observer filter.
	var observerSet map[string]bool
	if q.Observer != "" {
		ids := strings.Split(q.Observer, ",")
		observerSet = make(map[string]bool, len(ids))
		for _, id := range ids {
			observerSet[strings.TrimSpace(id)] = true
		}
	}

	// Pre-compute region observer set.
	var regionObservers map[string]bool
	if q.Region != "" {
		regionObservers = s.resolveRegionObservers(q.Region)
		if len(regionObservers) == 0 {
			return nil
		}
	}

	// Pre-compute node filter parameters.
	var nodePK string
	var nodeHashSet map[string]bool
	hasNode := q.Node != ""
	if hasNode {
		nodePK = s.db.resolveNodePubkey(q.Node)
		indexed := s.byNode[nodePK]
		nodeHashSet = make(map[string]bool, len(indexed))
		for _, tx := range indexed {
			nodeHashSet[tx.Hash] = true
		}
	}

	// Determine the source slice. Use index-based source when only node
	// filter is active and an index exists.
	source := s.packets
	if hasNode && !hasType && !hasRoute && q.Observer == "" &&
		filterHash == "" && !hasSince && !hasUntil && q.Region == "" && filterChannel == "" {
		if indexed, ok := s.byNode[nodePK]; ok {
			return indexed
		}
	}
	// Single-pass filter: apply all predicates in one scan.
	results := filterTxSlice(source, func(tx *StoreTx) bool {
		// Data integrity: exclude legacy rows missing hash or timestamp (#871)
		if tx.Hash == "" || tx.FirstSeen == "" {
			return false
		}
		if hasType && (tx.PayloadType == nil || *tx.PayloadType != filterType) {
			return false
		}
		if hasRoute && (tx.RouteType == nil || *tx.RouteType != filterRoute) {
			return false
		}
		if filterHash != "" && tx.Hash != filterHash {
			return false
		}
		if hasSince && tx.FirstSeen <= q.Since {
			return false
		}
		if hasUntil && tx.FirstSeen >= q.Until {
			return false
		}
		if observerSet != nil {
			found := false
			for _, obs := range tx.Observations {
				if observerSet[obs.ObserverID] {
					found = true
					break
				}
			}
			if !found {
				return false
			}
		}
		if regionObservers != nil {
			found := false
			for _, obs := range tx.Observations {
				if regionObservers[obs.ObserverID] {
					found = true
					break
				}
			}
			if !found {
				return false
			}
		}
		if hasNode {
			if !nodeHashSet[tx.Hash] {
				return false
			}
		}
		if filterChannel != "" {
			if !packetMatchesChannel(tx, filterChannel) {
				return false
			}
		}
		return true
	})

	return results
}

// packetMatchesChannel returns true if the transmission's decoded payload
// matches the requested channel filter (#812). The filter accepts either a
// plaintext channel name (e.g. "public", "#test") matching decoded.channel,
// or "enc_<HEX>" matching the channelHashHex of an undecryptable GRP_TXT.
func packetMatchesChannel(tx *StoreTx, filterChannel string) bool {
	if tx.PayloadType == nil || *tx.PayloadType != 5 {
		return false
	}
	if tx.DecodedJSON == "" {
		return false
	}
	d := tx.ParsedDecoded()
	if d == nil {
		return false
	}
	if ch, ok := d["channel"].(string); ok && ch != "" {
		if ch == filterChannel {
			return true
		}
	}
	if strings.HasPrefix(filterChannel, "enc_") {
		if hex, ok := d["channelHashHex"].(string); ok && hex != "" {
			if "enc_"+hex == filterChannel {
				return true
			}
		}
	}
	return false
}

// transmissionsForObserver returns unique transmissions for an observer.
func (s *PacketStore) transmissionsForObserver(observerIDs string, from []*StoreTx) []*StoreTx {
	ids := strings.Split(observerIDs, ",")
	idSet := make(map[string]bool, len(ids))
	for i, id := range ids {
		ids[i] = strings.TrimSpace(id)
		idSet[ids[i]] = true
	}
	if from != nil {
		return filterTxSlice(from, func(tx *StoreTx) bool {
			for _, obs := range tx.Observations {
				if idSet[obs.ObserverID] {
					return true
				}
			}
			return false
		})
	}
	// Use byObserver index: union transmissions for all IDs
	seen := make(map[int]bool)
	var result []*StoreTx
	for _, id := range ids {
		for _, obs := range s.byObserver[id] {
			if seen[obs.TransmissionID] {
				continue
			}
			seen[obs.TransmissionID] = true
			tx := s.byTxID[obs.TransmissionID]
			if tx != nil {
				result = append(result, tx)
			}
		}
	}
	return result
}

// resolveRegionObservers returns a set of observer IDs for a given IATA region.
// Results are cached for 30 seconds to avoid repeated DB queries.
// Uses its own mutex (regionObsMu) so callers holding s.mu won't deadlock.
func (s *PacketStore) resolveRegionObservers(region string) map[string]bool {
	s.regionObsMu.Lock()
	defer s.regionObsMu.Unlock()

	if s.regionObsCache != nil && time.Since(s.regionObsCacheTime) < 30*time.Second {
		if m, ok := s.regionObsCache[region]; ok {
			return m
		}
		return s.fetchAndCacheRegionObs(region)
	}
	// Cache expired — rebuild.
	s.regionObsCache = make(map[string]map[string]bool)
	s.regionObsCacheTime = time.Now()

	// Fetch for the requested region and cache it.
	return s.fetchAndCacheRegionObs(region)
}

// fetchAndCacheRegionObs fetches observer IDs for a region from the DB and stores in cache.
// Caller must hold regionObsMu.
func (s *PacketStore) fetchAndCacheRegionObs(region string) map[string]bool {
	if m, ok := s.regionObsCache[region]; ok {
		return m
	}
	ids, err := s.db.GetObserverIdsForRegion(region)
	if err != nil || len(ids) == 0 {
		s.regionObsCache[region] = nil
		return nil
	}
	m := make(map[string]bool, len(ids))
	for _, id := range ids {
		m[id] = true
	}
	s.regionObsCache[region] = m
	return m
}

// iataMatchesRegion returns true if iata matches any of the comma-separated
// region codes in regionParam. Comparison is case-insensitive and trim-tolerant.
// Empty iata never matches; empty regionParam never matches.
//
// #804: shared helper used by analytics to attribute transmissions to a node's
// HOME region (derived from observers that hear its zero-hop direct adverts)
// rather than to the observer that happened to relay a packet.
func iataMatchesRegion(iata, regionParam string) bool {
	if iata == "" || regionParam == "" {
		return false
	}
	codes := normalizeRegionCodes(regionParam)
	if len(codes) == 0 {
		return false
	}
	got := strings.TrimSpace(strings.ToUpper(iata))
	if got == "" {
		return false
	}
	for _, c := range codes {
		if c == got {
			return true
		}
	}
	return false
}

// computeNodeHomeRegions returns a pubkey → IATA map deriving each node's
// HOME region from zero-hop DIRECT adverts. A zero-hop direct advert is the
// most authoritative location signal because the path byte is set locally on
// the originating radio and the packet has not been relayed: the observer
// that hears it is necessarily within direct RF range of the originator.
//
// When a node has zero-hop direct adverts heard by observers from multiple
// regions, the most-frequently-observed region wins (geographic plurality).
//
// Caller must hold s.mu (read or write). Returns empty map (not nil) if no
// observers are loaded or no zero-hop direct adverts have been seen.
//
// #804: feeds analytics region-attribution so a multi-byte repeater whose
// flood adverts get relayed across regions is still attributed to its home.
func (s *PacketStore) computeNodeHomeRegions() map[string]string {
	// Build observer → IATA map. observers table is small (≪ packets), so a
	// single DB read here is acceptable; resolveRegionObservers does similar.
	obsIATA := make(map[string]string, 64)
	if s.db != nil {
		if observers, err := s.db.GetObservers(); err == nil {
			for _, o := range observers {
				if o.IATA != nil && *o.IATA != "" {
					obsIATA[o.ID] = strings.TrimSpace(strings.ToUpper(*o.IATA))
				}
			}
		}
	}
	if len(obsIATA) == 0 {
		return map[string]string{}
	}

	// Tally zero-hop direct ADVERT region observations per pubkey.
	type tally struct {
		counts map[string]int
	}
	per := make(map[string]*tally, 256)

	for _, tx := range s.packets {
		if tx.RawHex == "" || len(tx.RawHex) < 4 {
			continue
		}
		if tx.PayloadType == nil || *tx.PayloadType != PayloadADVERT {
			continue
		}
		if tx.DecodedJSON == "" {
			continue
		}
		header, err := strconv.ParseUint(tx.RawHex[:2], 16, 8)
		if err != nil {
			continue
		}
		routeType := header & 0x03
		if routeType != uint64(RouteDirect) && routeType != uint64(RouteTransportDirect) {
			continue
		}
		// Path byte index — for direct/transport-direct it's at offset 1
		// (matches the analytics decoder's pathByteIdx logic).
		if len(tx.RawHex) < 4 {
			continue
		}
		pathByte, err := strconv.ParseUint(tx.RawHex[2:4], 16, 8)
		if err != nil {
			continue
		}
		hopCount := pathByte & 0x3F
		if hopCount != 0 {
			continue
		}

		var d map[string]interface{}
		if json.Unmarshal([]byte(tx.DecodedJSON), &d) != nil {
			continue
		}
		pk, _ := d["pubKey"].(string)
		if pk == "" {
			pk, _ = d["public_key"].(string)
		}
		if pk == "" {
			continue
		}

		for _, obs := range tx.Observations {
			iata := obsIATA[obs.ObserverID]
			if iata == "" {
				continue
			}
			t := per[pk]
			if t == nil {
				t = &tally{counts: map[string]int{}}
				per[pk] = t
			}
			t.counts[iata]++
		}
	}

	out := make(map[string]string, len(per))
	for pk, t := range per {
		var bestIATA string
		bestCount := 0
		for iata, n := range t.counts {
			if n > bestCount || (n == bestCount && iata < bestIATA) {
				bestCount = n
				bestIATA = iata
			}
		}
		if bestIATA != "" {
			out[pk] = bestIATA
		}
	}
	return out
}

// enrichObs returns a map with observation fields + transmission fields.
func (s *PacketStore) enrichObs(obs *StoreObs) map[string]interface{} {
	tx := s.byTxID[obs.TransmissionID]

	m := map[string]interface{}{
		"id":            obs.ID,
		"timestamp":     strOrNil(obs.Timestamp),
		"observer_id":   strOrNil(obs.ObserverID),
		"observer_name": strOrNil(obs.ObserverName),
		"observer_iata": strOrNil(obs.ObserverIATA),
		"direction":     strOrNil(obs.Direction),
		"snr":           floatPtrOrNil(obs.SNR),
		"rssi":          floatPtrOrNil(obs.RSSI),
		"score":         intPtrOrNil(obs.Score),
		"path_json":     strOrNil(obs.PathJSON),
	}
	// On-demand SQL fetch for resolved_path
	rp := s.fetchResolvedPathForObs(obs.ID)
	if rp != nil {
		m["resolved_path"] = rp
	}

	if tx != nil {
		m["hash"] = strOrNil(tx.Hash)
		// Prefer per-observation raw_hex; fall back to transmission-level (#881)
		if obs.RawHex != "" {
			m["raw_hex"] = obs.RawHex
		} else {
			m["raw_hex"] = strOrNil(tx.RawHex)
		}
		m["payload_type"] = intPtrOrNil(tx.PayloadType)
		m["route_type"] = intPtrOrNil(tx.RouteType)
		m["decoded_json"] = strOrNil(tx.DecodedJSON)
	}

	return m
}

// --- Conversion helpers ---

// txToMap converts a StoreTx to the map shape matching scanTransmissionRow output.
func txToMap(tx *StoreTx, includeObservations ...bool) map[string]interface{} {
	m := map[string]interface{}{
		"id":                tx.ID,
		"raw_hex":           strOrNil(tx.RawHex),
		"hash":              strOrNil(tx.Hash),
		"first_seen":        strOrNil(tx.FirstSeen),
		"timestamp":         strOrNil(tx.FirstSeen),
		"route_type":        intPtrOrNil(tx.RouteType),
		"payload_type":      intPtrOrNil(tx.PayloadType),
		"decoded_json":      strOrNil(tx.DecodedJSON),
		"observation_count": tx.ObservationCount,
		"observer_id":       strOrNil(tx.ObserverID),
		"observer_name":     strOrNil(tx.ObserverName),
		"observer_iata":     strOrNil(tx.ObserverIATA),
		"snr":               floatPtrOrNil(tx.SNR),
		"rssi":              floatPtrOrNil(tx.RSSI),
		"path_json":         strOrNil(tx.PathJSON),
		"direction":         strOrNil(tx.Direction),
	}
	// Include parsed path array to match Node.js output shape
	if hops := txGetParsedPath(tx); len(hops) > 0 {
		m["_parsedPath"] = hops
	} else {
		m["_parsedPath"] = nil
	}
	// Only build observation sub-maps when caller requests them (avoids allocations that get stripped)
	if len(includeObservations) > 0 && includeObservations[0] {
		obs := make([]map[string]interface{}, 0, len(tx.Observations))
		for _, o := range tx.Observations {
			om := map[string]interface{}{
				"id":            o.ID,
				"observer_id":   strOrNil(o.ObserverID),
				"observer_name": strOrNil(o.ObserverName),
				"observer_iata": strOrNil(o.ObserverIATA),
				"snr":           floatPtrOrNil(o.SNR),
				"rssi":          floatPtrOrNil(o.RSSI),
				"path_json":     strOrNil(o.PathJSON),
				"timestamp":     strOrNil(o.Timestamp),
				"direction":     strOrNil(o.Direction),
			}
			obs = append(obs, om)
		}
		m["observations"] = obs
	}
	return m
}

// txToMapWithRP is like txToMap but also fetches resolved_path on demand from the store.
func (s *PacketStore) txToMapWithRP(tx *StoreTx, includeObservations ...bool) map[string]interface{} {
	m := txToMap(tx, includeObservations...)
	// On-demand SQL fetch for resolved_path
	rp := s.fetchResolvedPathForTxBest(tx)
	if rp != nil {
		m["resolved_path"] = rp
	}
	// Also add resolved_path to observation sub-maps if present
	if len(includeObservations) > 0 && includeObservations[0] {
		if obsList, ok := m["observations"].([]map[string]interface{}); ok {
			for i, o := range tx.Observations {
				if i < len(obsList) {
					obsRP := s.fetchResolvedPathForObs(o.ID)
					if obsRP != nil {
						obsList[i]["resolved_path"] = obsRP
					}
				}
			}
		}
	}
	return m
}

func strOrNil(s string) interface{} {
	if s == "" {
		return nil
	}
	return s
}

// normalizeTimestamp converts SQLite datetime format ("YYYY-MM-DD HH:MM:SS")
// to ISO 8601 ("YYYY-MM-DDTHH:MM:SSZ"). Already-ISO strings pass through.
func normalizeTimestamp(s string) string {
	if s == "" {
		return s
	}
	if t, err := time.Parse("2006-01-02 15:04:05", s); err == nil {
		return t.UTC().Format("2006-01-02T15:04:05.000Z")
	}
	return s
}

func intPtrOrNil(p *int) interface{} {
	if p == nil {
		return nil
	}
	return *p
}

func floatPtrOrNil(p *float64) interface{} {
	if p == nil {
		return nil
	}
	return *p
}

func nullIntPtr(ni sql.NullInt64) *int {
	if ni.Valid {
		v := int(ni.Int64)
		return &v
	}
	return nil
}

func nullFloatPtr(nf sql.NullFloat64) *float64 {
	if nf.Valid {
		return &nf.Float64
	}
	return nil
}

// resolvePayloadTypeName returns the firmware-standard name for a payload_type.
func resolvePayloadTypeName(pt *int) string {
	if pt == nil {
		return "UNKNOWN"
	}
	if name, ok := payloadTypeNames[*pt]; ok {
		return name
	}
	return fmt.Sprintf("UNK(%d)", *pt)
}

// txGetParsedPath returns cached parsed path hops, parsing on first call.
func txGetParsedPath(tx *StoreTx) []string {
	if tx.pathParsed {
		return tx.parsedPath
	}
	tx.parsedPath = parsePathJSON(tx.PathJSON)
	tx.pathParsed = true
	return tx.parsedPath
}

// addTxToSubpathIndex extracts all raw subpaths (lengths 2–8) from tx and
// increments their counts in the index.  Returns true if the tx contributed
// (path had ≥ 2 hops).
func addTxToSubpathIndex(idx map[string]int, tx *StoreTx) bool {
	return addTxToSubpathIndexFull(idx, nil, tx)
}

// addTxToSubpathIndexFull is like addTxToSubpathIndex but also appends
// tx to txIdx for each subpath key (if txIdx is non-nil).
func addTxToSubpathIndexFull(idx map[string]int, txIdx map[string][]*StoreTx, tx *StoreTx) bool {
	hops := txGetParsedPath(tx)
	if len(hops) < 2 {
		return false
	}
	maxL := min(8, len(hops))
	for l := 2; l <= maxL; l++ {
		for start := 0; start <= len(hops)-l; start++ {
			key := strings.ToLower(strings.Join(hops[start:start+l], ","))
			idx[key]++
			if txIdx != nil {
				txIdx[key] = append(txIdx[key], tx)
			}
		}
	}
	return true
}

// removeTxFromSubpathIndex is the inverse of addTxToSubpathIndex — it
// decrements counts for all raw subpaths of tx.  Returns true if the tx
// had a path.
func removeTxFromSubpathIndex(idx map[string]int, tx *StoreTx) bool {
	return removeTxFromSubpathIndexFull(idx, nil, tx)
}

// removeTxFromSubpathIndexFull is like removeTxFromSubpathIndex but also
// removes tx from txIdx for each subpath key (if txIdx is non-nil).
func removeTxFromSubpathIndexFull(idx map[string]int, txIdx map[string][]*StoreTx, tx *StoreTx) bool {
	hops := txGetParsedPath(tx)
	if len(hops) < 2 {
		return false
	}
	maxL := min(8, len(hops))
	for l := 2; l <= maxL; l++ {
		for start := 0; start <= len(hops)-l; start++ {
			key := strings.ToLower(strings.Join(hops[start:start+l], ","))
			idx[key]--
			if idx[key] <= 0 {
				delete(idx, key)
			}
			if txIdx != nil {
				txs := txIdx[key]
				for i, t := range txs {
					if t == tx {
						txIdx[key] = append(txs[:i], txs[i+1:]...)
						break
					}
				}
				if len(txIdx[key]) == 0 {
					delete(txIdx, key)
				}
			}
		}
	}
	return true
}

// buildSubpathIndex scans all packets and populates spIndex + spTotalPaths.
// Must be called with s.mu held.
func (s *PacketStore) buildSubpathIndex() {
	s.spIndex = make(map[string]int, 4096)
	s.spTxIndex = make(map[string][]*StoreTx, 4096)
	s.spTotalPaths = 0
	for _, tx := range s.packets {
		if addTxToSubpathIndexFull(s.spIndex, s.spTxIndex, tx) {
			s.spTotalPaths++
		}
	}
	log.Printf("[store] Built subpath index: %d unique raw subpaths from %d paths",
		len(s.spIndex), s.spTotalPaths)
}

// buildPathHopIndex scans all packets and populates byPathHop.
// Must be called with s.mu held.
func (s *PacketStore) buildPathHopIndex() {
	s.byPathHop = make(map[string][]*StoreTx, 4096)
	for _, tx := range s.packets {
		addTxToPathHopIndex(s.byPathHop, tx)
	}
	log.Printf("[store] Built path-hop index: %d unique keys", len(s.byPathHop))
}

// addTxToPathHopIndex indexes a transmission under each unique raw hop key.
// Resolved pubkey keys are handled by the decode-window via feedDecodeWindowConsumers.
func addTxToPathHopIndex(idx map[string][]*StoreTx, tx *StoreTx) {
	hops := txGetParsedPath(tx)
	if len(hops) == 0 {
		return
	}
	seen := make(map[string]bool, len(hops))
	for _, hop := range hops {
		key := strings.ToLower(hop)
		if !seen[key] {
			seen[key] = true
			idx[key] = append(idx[key], tx)
		}
	}
}

// removeTxFromPathHopIndex removes a transmission from all its raw path-hop index entries.
// Resolved pubkey entries are cleaned up via removeFromResolvedPubkeyIndex.
func removeTxFromPathHopIndex(idx map[string][]*StoreTx, tx *StoreTx) {
	hops := txGetParsedPath(tx)
	if len(hops) == 0 {
		return
	}
	seen := make(map[string]bool, len(hops))
	for _, hop := range hops {
		key := strings.ToLower(hop)
		if !seen[key] {
			seen[key] = true
			removeTxFromSlice(idx, key, tx)
		}
	}
}

// removeTxFromSlice removes tx from idx[key] by ID, deleting the key if empty.
func removeTxFromSlice(idx map[string][]*StoreTx, key string, tx *StoreTx) {
	list := idx[key]
	for i, t := range list {
		if t.ID == tx.ID {
			idx[key] = append(list[:i], list[i+1:]...)
			break
		}
	}
	if len(idx[key]) == 0 {
		delete(idx, key)
	}
}

// updateDistanceIndexForTxs removes old distance records for the given
// transmissions and recomputes them. Builds lookup maps once, amortising the
// cost across all changed txs in a single ingest cycle. Must be called with
// s.mu held.
func (s *PacketStore) updateDistanceIndexForTxs(txs []*StoreTx) {
	// Remove old records for all changed txs first.
	removeSet := make(map[*StoreTx]bool, len(txs))
	for _, tx := range txs {
		removeSet[tx] = true
	}
	n := 0
	for _, r := range s.distHops {
		if !removeSet[r.tx] {
			s.distHops[n] = r
			n++
		}
	}
	s.distHops = s.distHops[:n]
	n = 0
	for _, r := range s.distPaths {
		if !removeSet[r.tx] {
			s.distPaths[n] = r
			n++
		}
	}
	s.distPaths = s.distPaths[:n]

	// Build lookup maps once.
	allNodes, pm := s.getCachedNodesAndPM()
	nodeByPk := make(map[string]*nodeInfo, len(allNodes))
	repeaterSet := make(map[string]bool)
	for i := range allNodes {
		nd := &allNodes[i]
		nodeByPk[nd.PublicKey] = nd
		if strings.Contains(strings.ToLower(nd.Role), "repeater") {
			repeaterSet[nd.PublicKey] = true
		}
	}
	// Per-tx hop resolver shared across the recompute loop (#1197 perf).
	resolveHop, setContext := s.hopResolverPerTx(pm)
	// Recompute distance records for each changed tx.
	for _, tx := range txs {
		// Per-tx context for hop disambiguation (#1197).
		setContext(buildHopContextPubkeys(tx, pm))
		txHops, txPath := computeDistancesForTx(tx, nodeByPk, repeaterSet, resolveHop)
		if len(txHops) > 0 {
			s.distHops = append(s.distHops, txHops...)
		}
		if txPath != nil {
			s.distPaths = append(s.distPaths, *txPath)
		}
	}
}

// buildDistanceIndex precomputes haversine distances for all packets.
// Must be called with s.mu held (Lock).
func (s *PacketStore) buildDistanceIndex() {
	allNodes, pm := s.getCachedNodesAndPM()
	nodeByPk := make(map[string]*nodeInfo, len(allNodes))
	repeaterSet := make(map[string]bool)
	for i := range allNodes {
		n := &allNodes[i]
		nodeByPk[n.PublicKey] = n
		if strings.Contains(strings.ToLower(n.Role), "repeater") {
			repeaterSet[n.PublicKey] = true
		}
	}

	hops := make([]distHopRecord, 0, len(s.packets))
	paths := make([]distPathRecord, 0, len(s.packets)/2)

	// Per-tx hop resolver shared across the per-tx loop (#1197 perf).
	resolveHop, setContext := s.hopResolverPerTx(pm)
	for _, tx := range s.packets {
		// Per-tx context for hop disambiguation (#1197).
		setContext(buildHopContextPubkeys(tx, pm))
		txHops, txPath := computeDistancesForTx(tx, nodeByPk, repeaterSet, resolveHop)
		if len(txHops) > 0 {
			hops = append(hops, txHops...)
		}
		if txPath != nil {
			paths = append(paths, *txPath)
		}
	}

	s.distHops = hops
	s.distPaths = paths
	log.Printf("[store] Built distance index: %d hop records, %d path records",
		len(s.distHops), len(s.distPaths))
}

// Self-accounting memory estimation constants.
// These estimate the in-memory cost of StoreTx and StoreObs structs including
// map/index overhead. They don't need to be exact — just proportional to actual
// usage and independent of GC state.
//
// Issue #743: Previous estimates missed major per-packet allocations:
// - spTxIndex: O(path²) entries per tx (50-150MB at scale)
// - Per-tx maps: obsKeys, observerSet (~11MB at scale)
// - byPathHop index entries (20-40MB at scale)
// Note: ResolvedPath per-obs overhead eliminated by #800 refactor.
const (
	storeTxBaseBytes  = 384 // StoreTx struct fields + map headers + sync.Once + string headers
	storeObsBaseBytes = 192 // StoreObs struct fields + string headers
	indexEntryBytes   = 48  // average cost of one index map entry (key + pointer + bucket overhead)
	numIndexesPerTx   = 5   // byHash, byTxID, byNode, byPayloadType, nodeHashes entries
	numIndexesPerObs  = 2   // byObsID, byObserver entries

	// Per-tx map overhead (obsKeys + observerSet): map header + initial buckets
	perTxMapsBytes = 200

	// Per path hop: byPathHop index entry (pointer + map bucket)
	perPathHopBytes = 50

	// Per subpath entry in spTxIndex: string key + slice append + pointer
	perSubpathEntryBytes = 40
)

// estimateStoreTxBytes returns the estimated memory cost of a StoreTx (excluding observations).
// Includes per-tx maps (obsKeys, observerSet), byPathHop entries, and spTxIndex subpath entries.
func estimateStoreTxBytes(tx *StoreTx) int64 {
	base := int64(storeTxBaseBytes)
	base += int64(len(tx.RawHex) + len(tx.Hash) + len(tx.DecodedJSON) + len(tx.PathJSON))
	base += int64(numIndexesPerTx * indexEntryBytes)

	// Per-tx maps: obsKeys + observerSet
	base += perTxMapsBytes

	// Path-dependent costs
	hops := int64(len(txGetParsedPath(tx)))
	base += hops * perPathHopBytes

	// spTxIndex: O(path²) subpath combinations
	if hops > 1 {
		subpaths := hops * (hops - 1) / 2
		base += subpaths * perSubpathEntryBytes
	}

	return base
}

// estimateStoreTxBytesTypical returns the estimated memory cost of a typical
// transmission with the given number of observations. Used for budget
// calculation during bounded cold load (no actual StoreTx needed).
func estimateStoreTxBytesTypical(numObs int) int64 {
	// Typical tx: ~64 byte hash, ~200 byte decoded JSON, ~40 byte path, 3 hops
	base := int64(storeTxBaseBytes) + 64 + 200 + 40
	base += int64(numIndexesPerTx * indexEntryBytes)
	base += perTxMapsBytes
	hops := int64(3)
	base += hops * perPathHopBytes
	base += (hops * (hops - 1) / 2) * perSubpathEntryBytes
	// Add observation costs
	obsBase := int64(storeObsBaseBytes) + 30 + 30 + 60 // observer ID + name + path
	obsBase += int64(numIndexesPerObs * indexEntryBytes)
	// No per-obs ResolvedPath overhead (#800)
	base += int64(numObs) * obsBase
	return base
}

// estimateStoreObsBytes returns the estimated memory cost of a StoreObs.
// ResolvedPath membership index overhead is tracked separately.
func estimateStoreObsBytes(obs *StoreObs) int64 {
	base := int64(storeObsBaseBytes)
	base += int64(len(obs.PathJSON) + len(obs.ObserverID))
	base += int64(numIndexesPerObs * indexEntryBytes)
	// ResolvedPath field removed (#800) — no per-obs RP overhead
	return base
}

// estimatedMemoryMB returns current Go heap allocation in MB.
// Kept for stats/debug endpoints only — NOT used in eviction decisions.
// In tests, memoryEstimator can be set to inject a deterministic value.
func (s *PacketStore) estimatedMemoryMB() float64 {
	if s.memoryEstimator != nil {
		return s.memoryEstimator()
	}
	var ms runtime.MemStats
	runtime.ReadMemStats(&ms)
	return float64(ms.HeapAlloc) / 1048576.0
}

// trackedMemoryMB returns the self-accounted packet store memory in MB.
func (s *PacketStore) trackedMemoryMB() float64 {
	return float64(s.trackedBytes) / 1048576.0
}

// EvictStale removes packets older than the retention window and/or exceeding
// the memory cap. Must be called with s.mu held (Lock). Returns the number of
// packets evicted.
// evictionCandidateTxIDs determines which tx IDs would be evicted and returns them.
// Must be called under s.mu.Lock (or RLock). Does NOT modify any state.
func (s *PacketStore) evictionCandidateTxIDs() []int {
	if s.retentionHours <= 0 && s.maxMemoryMB <= 0 {
		return nil
	}
	cutoffIdx := 0
	if s.retentionHours > 0 {
		cutoff := time.Now().UTC().Add(-time.Duration(s.retentionHours * float64(time.Hour))).Format(time.RFC3339)
		for cutoffIdx < len(s.packets) && s.packets[cutoffIdx].FirstSeen < cutoff {
			cutoffIdx++
		}
	}
	if s.maxMemoryMB > 0 {
		highWatermark := int64(s.maxMemoryMB) * 1048576
		lowWatermark := int64(float64(highWatermark) * 0.85)
		if s.trackedBytes > highWatermark && len(s.packets) > 0 {
			var bytesToEvict int64
			memCutoff := cutoffIdx
			for memCutoff < len(s.packets) && (s.trackedBytes-bytesToEvict) > lowWatermark {
				tx := s.packets[memCutoff]
				bytesToEvict += estimateStoreTxBytes(tx)
				for _, obs := range tx.Observations {
					bytesToEvict += estimateStoreObsBytes(obs)
				}
				memCutoff++
			}
			maxEvict := len(s.packets) / 4
			if maxEvict < 1 {
				maxEvict = 1
			}
			if memCutoff > maxEvict {
				memCutoff = maxEvict
			}
			if memCutoff > cutoffIdx {
				cutoffIdx = memCutoff
			}
		}
	}
	if cutoffIdx == 0 || cutoffIdx > len(s.packets) {
		return nil
	}
	ids := make([]int, cutoffIdx)
	for i := 0; i < cutoffIdx; i++ {
		ids[i] = s.packets[i].ID
	}
	return ids
}

// EvictStaleWithRP runs eviction using pre-fetched resolved pubkeys.
// rpBatch may be nil (in which case resolved pubkey cleanup for byNode/nodeHashes is skipped).
// Must be called under s.mu.Lock.
func (s *PacketStore) EvictStaleWithRP(rpBatch map[int][]string) int {
	return s.evictStaleInternal(rpBatch)
}

// EvictStale runs eviction, fetching resolved pubkeys inline (SQL under lock).
// Prefer RunEviction() which batches the SQL outside the lock.
// Must be called under s.mu.Lock.
func (s *PacketStore) EvictStale() int {
	return s.evictStaleInternal(nil)
}

func (s *PacketStore) evictStaleInternal(rpBatch map[int][]string) int {
	if s.retentionHours <= 0 && s.maxMemoryMB <= 0 {
		return 0
	}

	cutoffIdx := 0

	// Time-based eviction: find how many packets from the head are too old
	if s.retentionHours > 0 {
		cutoff := time.Now().UTC().Add(-time.Duration(s.retentionHours * float64(time.Hour))).Format(time.RFC3339)
		for cutoffIdx < len(s.packets) && s.packets[cutoffIdx].FirstSeen < cutoff {
			cutoffIdx++
		}
	}

	// Memory-based eviction: use self-accounted trackedBytes with watermark hysteresis.
	// High watermark = maxMemoryMB (trigger), low watermark = 85% (stop).
	// Safety cap: never evict more than 25% of packets in a single pass.
	if s.maxMemoryMB > 0 {
		highWatermark := int64(s.maxMemoryMB) * 1048576
		lowWatermark := int64(float64(highWatermark) * 0.85)
		if s.trackedBytes > highWatermark && len(s.packets) > 0 {
			// Evict from head until trackedBytes would drop below low watermark
			var bytesToEvict int64
			memCutoff := cutoffIdx
			for memCutoff < len(s.packets) && (s.trackedBytes-bytesToEvict) > lowWatermark {
				tx := s.packets[memCutoff]
				bytesToEvict += estimateStoreTxBytes(tx)
				for _, obs := range tx.Observations {
					bytesToEvict += estimateStoreObsBytes(obs)
				}
				memCutoff++
			}
			// Safety cap: never evict more than 25% in a single pass
			maxEvict := len(s.packets) / 4
			if maxEvict < 1 {
				maxEvict = 1
			}
			if memCutoff > maxEvict {
				memCutoff = maxEvict
			}
			if memCutoff > cutoffIdx {
				cutoffIdx = memCutoff
			}
		}
	}

	if cutoffIdx == 0 {
		return 0
	}
	if cutoffIdx > len(s.packets) {
		cutoffIdx = len(s.packets)
	}

	evicting := s.packets[:cutoffIdx]
	evictedObs := 0
	var evictedBytes int64

	// Build sets of evicted IDs for batch removal from secondary indexes
	evictedTxIDs := make(map[int]struct{}, cutoffIdx)
	evictedObsIDs := make(map[int]struct{}, cutoffIdx*2)
	// Track which observer IDs and payload types need filtering
	affectedObservers := make(map[string]struct{})
	affectedPayloadTypes := make(map[int]struct{})
	affectedNodes := make(map[string]struct{})

	// First pass: remove from primary indexes (byHash, byTxID, byObsID),
	// collect IDs for batch secondary index cleanup, and handle non-index work
	for _, tx := range evicting {
		delete(s.byHash, tx.Hash)
		delete(s.byTxID, tx.ID)
		evictedTxIDs[tx.ID] = struct{}{}
		evictedBytes += estimateStoreTxBytes(tx)

		for _, obs := range tx.Observations {
			delete(s.byObsID, obs.ID)
			evictedObsIDs[obs.ID] = struct{}{}
			evictedBytes += estimateStoreObsBytes(obs)
			if obs.ObserverID != "" {
				affectedObservers[obs.ObserverID] = struct{}{}
			}
			evictedObs++
		}

		s.untrackAdvertPubkey(tx)
		if tx.PayloadType != nil {
			affectedPayloadTypes[*tx.PayloadType] = struct{}{}
		}

		// Remove from nodeHashes and collect affected node keys.
		// Must mirror indexByNode: process decoded JSON fields AND resolved_path pubkeys.
		evictedFromNode := make(map[string]bool)
		if tx.DecodedJSON != "" {
			var decoded map[string]interface{}
			if json.Unmarshal([]byte(tx.DecodedJSON), &decoded) == nil {
				for _, field := range []string{"pubKey", "destPubKey", "srcPubKey"} {
					if v, ok := decoded[field].(string); ok && v != "" {
						if hashes, ok := s.nodeHashes[v]; ok {
							delete(hashes, tx.Hash)
							if len(hashes) == 0 {
								delete(s.nodeHashes, v)
							}
						}
						affectedNodes[v] = struct{}{}
						evictedFromNode[v] = true
					}
				}
			}
		}
		// Clean up resolved_path pubkeys from byNode/nodeHashes.
		// Uses pre-fetched batch data when available (no SQL under lock).
		var rpPubkeys []string
		if rpBatch != nil {
			rpPubkeys = rpBatch[tx.ID]
		}
		for _, pk := range rpPubkeys {
			if pk == "" || evictedFromNode[pk] {
				continue
			}
			if hashes, ok := s.nodeHashes[pk]; ok {
				delete(hashes, tx.Hash)
				if len(hashes) == 0 {
					delete(s.nodeHashes, pk)
				}
			}
			affectedNodes[pk] = struct{}{}
			evictedFromNode[pk] = true
		}

		// Remove from resolved pubkey index
		s.removeFromResolvedPubkeyIndex(tx.ID)

		// Remove from subpath index
		removeTxFromSubpathIndexFull(s.spIndex, s.spTxIndex, tx)
		// Remove from path-hop index
		removeTxFromPathHopIndex(s.byPathHop, tx)
	}

	// Batch-remove from byObserver: single pass per affected observer slice
	for obsID := range affectedObservers {
		obsList := s.byObserver[obsID]
		filtered := obsList[:0]
		for _, o := range obsList {
			if _, evicted := evictedObsIDs[o.ID]; !evicted {
				filtered = append(filtered, o)
			}
		}
		if len(filtered) == 0 {
			delete(s.byObserver, obsID)
		} else {
			s.byObserver[obsID] = filtered
		}
	}

	// Batch-remove from byPayloadType: single pass per affected type slice
	for pt := range affectedPayloadTypes {
		ptList := s.byPayloadType[pt]
		filtered := ptList[:0]
		for _, t := range ptList {
			if _, evicted := evictedTxIDs[t.ID]; !evicted {
				filtered = append(filtered, t)
			}
		}
		if len(filtered) == 0 {
			delete(s.byPayloadType, pt)
		} else {
			s.byPayloadType[pt] = filtered
		}
	}

	// Batch-remove from byNode: single pass per affected node slice
	for nodeKey := range affectedNodes {
		nodeList := s.byNode[nodeKey]
		filtered := nodeList[:0]
		for _, t := range nodeList {
			if _, evicted := evictedTxIDs[t.ID]; !evicted {
				filtered = append(filtered, t)
			}
		}
		if len(filtered) == 0 {
			delete(s.byNode, nodeKey)
		} else {
			s.byNode[nodeKey] = filtered
		}
	}

	// Remove from distance indexes — filter out records referencing evicted txs
	evictedTxSet := make(map[*StoreTx]bool, cutoffIdx)
	for _, tx := range evicting {
		evictedTxSet[tx] = true
	}
	newDistHops := s.distHops[:0]
	for i := range s.distHops {
		if !evictedTxSet[s.distHops[i].tx] {
			newDistHops = append(newDistHops, s.distHops[i])
		}
	}
	s.distHops = newDistHops

	newDistPaths := s.distPaths[:0]
	for i := range s.distPaths {
		if !evictedTxSet[s.distPaths[i].tx] {
			newDistPaths = append(newDistPaths, s.distPaths[i])
		}
	}
	s.distPaths = newDistPaths

	// Trim packets slice
	n := copy(s.packets, s.packets[cutoffIdx:])
	s.packets = s.packets[:n]
	s.totalObs -= evictedObs

	evictCount := cutoffIdx
	atomic.AddInt64(&s.evicted, int64(evictCount))
	s.trackedBytes -= evictedBytes
	if s.trackedBytes < 0 {
		s.trackedBytes = 0
	}
	log.Printf("[store] Evicted %d packets (%d obs, freed ~%.1fMB, tracked ~%.1fMB)",
		evictCount, evictedObs, float64(evictedBytes)/1048576.0, s.trackedMemoryMB())

	// Eviction removes data — all caches may be affected
	s.invalidateCachesFor(cacheInvalidation{eviction: true})

	// Invalidate hash size cache
	s.hashSizeInfoMu.Lock()
	s.hashSizeInfoCache = nil
	s.hashSizeInfoMu.Unlock()

	// Compact resolved pubkey index after eviction sweep
	s.CompactResolvedPubkeyIndex()
	s.CheckResolvedPubkeyIndexSize()

	return evictCount
}

// RunEviction acquires the write lock and runs eviction. Safe to call from
// a goroutine. Returns evicted count.
// Uses a two-phase approach: determines eviction candidates under lock,
// releases lock for batch SQL fetch of resolved pubkeys, then re-acquires
// lock for the actual eviction pass.
func (s *PacketStore) RunEviction() int {
	// Phase 1: determine candidates under lock
	s.mu.Lock()
	txIDs := s.evictionCandidateTxIDs()
	s.mu.Unlock()

	// Phase 2: batch-fetch resolved pubkeys from SQL (no lock held)
	var rpBatch map[int][]string
	if len(txIDs) > 0 {
		rpBatch = s.resolvedPubkeysForEvictionBatch(txIDs)
	}

	// Phase 3: actual eviction under write lock, using pre-fetched data
	s.mu.Lock()
	defer s.mu.Unlock()
	return s.EvictStaleWithRP(rpBatch)
}

// StartEvictionTicker starts a background goroutine that runs eviction every
// minute. Returns a stop function.
func (s *PacketStore) StartEvictionTicker() func() {
	if s.retentionHours <= 0 && s.maxMemoryMB <= 0 {
		return func() {} // no-op
	}
	ticker := time.NewTicker(1 * time.Minute)
	done := make(chan struct{})
	go func() {
		for {
			select {
			case <-ticker.C:
				s.RunEviction()
			case <-done:
				ticker.Stop()
				return
			}
		}
	}()
	return func() { close(done) }
}

// computeDistancesForTx computes distance records for a single transmission.
// buildHopContextPubkeys collects context pubkeys for hop disambiguation:
// the originator/sender pubkey plus any unambiguous-prefix anchors in the
// path (single-candidate prefixes — strong context). Used by callers of
// pm.resolveWithContext to light up tiers 1 and 2 of the resolver. See #1197.
//
// Returned pubkeys are de-duplicated and lowercased.
func buildHopContextPubkeys(tx *StoreTx, pm *prefixMap) []string {
	if tx == nil || pm == nil {
		return nil
	}
	seen := make(map[string]struct{}, 16)
	out := make([]string, 0, 16)
	add := func(pk string) {
		if pk == "" {
			return
		}
		l := strings.ToLower(pk)
		if _, ok := seen[l]; ok {
			return
		}
		seen[l] = struct{}{}
		out = append(out, l)
	}

	// Sender / originator pubkey from decoded payload. Use the cached
	// ParsedDecoded() (sync.Once-gated) instead of re-unmarshaling — the
	// helper is hot (3 distance sites + analytics topology, all 30k+ tx
	// loops). See #1197 (carmack/adversarial r1).
	if dec := tx.ParsedDecoded(); dec != nil {
		if pk, ok := dec["pubKey"].(string); ok {
			add(pk)
		}
	}

	// Observer pubkey, where available. ObserverID is the observers.id PRIMARY
	// KEY from the MQTT topic — it is NOT guaranteed to be a node pubkey hex
	// (some observers register with arbitrary string ids like "myobserver").
	// Guard against polluting the context with non-pubkey strings: include
	// only when it parses as hex AND is long enough to plausibly be a pubkey
	// prefix. The full prefix-map lookup would also be acceptable, but the
	// hex+length check is O(len) and avoids one map probe per tx on a hot
	// path. See #1197 (adversarial r1 #4).
	if obs := tx.ObserverID; obs != "" && len(obs) >= 4 && isHexLower(strings.ToLower(obs)) {
		add(obs)
	}

	// Unambiguous-prefix anchors: any hop in the path whose prefix has exactly
	// one candidate is a strong context signal.
	for _, hop := range txGetParsedPath(tx) {
		h := strings.ToLower(hop)
		if cands, ok := pm.m[h]; ok && len(cands) == 1 {
			add(cands[0].PublicKey)
		}
	}

	return out
}

// isHexLower reports whether s consists only of [0-9a-f] (assumes already
// lowercased by caller). Used to guard ObserverID before adding it to the
// hop-disambiguation context, since ObserverID is a free-form observers.id
// and may not be a node pubkey hex.
func isHexLower(s string) bool {
	if s == "" {
		return false
	}
	for i := 0; i < len(s); i++ {
		c := s[i]
		if !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f')) {
			return false
		}
	}
	return true
}

// hasUpperASCII reports whether s contains any uppercase ASCII letter.
// Used by resolveWithContext tier-1 to skip the strings.ToLower allocation
// when the context pubkeys are already lowercased (the common case — see
// buildHopContextPubkeys / buildAggregateHopContextPubkeys, which lowercase
// on the way in). #1247.
func hasUpperASCII(s string) bool {
	for i := 0; i < len(s); i++ {
		c := s[i]
		if c >= 'A' && c <= 'Z' {
			return true
		}
	}
	return false
}

// buildAggregateHopContextPubkeys gathers context across many txs for hot
// loops that resolve hops outside any per-tx scope (subpath/topology
// aggregations). Caller passes the slice of txs to consider; we union the
// per-tx contexts with de-dup. Used by call sites that read from precomputed
// indices (s.spIndex, s.spTxIndex) or that resolve user-supplied hops.
//
// Result is order-independent in semantics; iteration order is deterministic
// only modulo Go's map iteration (acceptable — the resolver's tier-2 averages
// GPS positions and tier-3 picks the lex-smallest pubkey on ties, so context
// order does not affect the chosen candidate).
func buildAggregateHopContextPubkeys(txs []*StoreTx, pm *prefixMap) []string {
	if len(txs) == 0 || pm == nil {
		return nil
	}
	seen := make(map[string]struct{}, 32)
	var out []string
	for _, tx := range txs {
		for _, pk := range buildHopContextPubkeys(tx, pm) {
			if _, ok := seen[pk]; ok {
				continue
			}
			seen[pk] = struct{}{}
			out = append(out, pk)
		}
	}
	return out
}

// hopResolverPerTx returns (resolveHop, setContext). The cache is allocated
// once and cleared between txs; setContext rebinds the per-tx context. Used
// by all per-tx distance/topology loops to avoid 4× duplicate closure
// definitions and per-tx map allocation. See #1197 (adversarial r1 #7,
// carmack r1 #3).
//
// CONCURRENCY: NOT safe for concurrent use. The returned closures share
// mutable captured state — `contextPubkeys` is reassigned by setContext and
// read by resolveHop, and `hopCache` is mutated by both (resolveHop writes
// on miss, setContext clears wholesale). Callers MUST invoke both functions
// from a single goroutine for the lifetime of the (resolveHop, setContext)
// pair. If a future caller fans out per-tx work across goroutines, allocate
// a fresh resolver pair per goroutine. See #1199 item 4.
func (s *PacketStore) hopResolverPerTx(pm *prefixMap) (resolveHop func(string) *nodeInfo, setContext func([]string)) {
	hopCache := make(map[string]*nodeInfo, 16)
	var contextPubkeys []string
	// Hoist the atomic graph.Load() out of the per-hop closure body so we do
	// one Load per resolver instance, not one per resolveHop call (PR #1208
	// carmack #1). Pair (resolveHop, setContext) is documented single-
	// goroutine, single-request scope — pinning the graph here matches that
	// lifetime.
	graph := s.graph.Load()
	resolveHop = func(hop string) *nodeInfo {
		if cached, ok := hopCache[hop]; ok {
			return cached
		}
		r, _, _ := pm.resolveWithContext(hop, contextPubkeys, graph)
		hopCache[hop] = r
		return r
	}
	setContext = func(ctx []string) {
		contextPubkeys = ctx
		clear(hopCache)
	}
	return resolveHop, setContext
}

func computeDistancesForTx(tx *StoreTx, nodeByPk map[string]*nodeInfo, repeaterSet map[string]bool, resolveHop func(string) *nodeInfo) ([]distHopRecord, *distPathRecord) {
	pathHops := txGetParsedPath(tx)
	if len(pathHops) == 0 {
		return nil, nil
	}

	resolved := make([]*nodeInfo, len(pathHops))
	for i, h := range pathHops {
		resolved[i] = resolveHop(h)
	}

	var senderNode *nodeInfo
	if tx.DecodedJSON != "" {
		var dec map[string]interface{}
		if json.Unmarshal([]byte(tx.DecodedJSON), &dec) == nil {
			if pk, ok := dec["pubKey"].(string); ok && pk != "" {
				senderNode = nodeByPk[pk]
			}
		}
	}

	chain := make([]*nodeInfo, 0, len(pathHops)+1)
	if senderNode != nil && senderNode.HasGPS {
		chain = append(chain, senderNode)
	}
	for _, r := range resolved {
		if r != nil && r.HasGPS {
			chain = append(chain, r)
		}
	}
	if len(chain) < 2 {
		return nil, nil
	}

	hourBucket := ""
	if tx.FirstSeen != "" && len(tx.FirstSeen) >= 13 {
		hourBucket = tx.FirstSeen[:13]
	}

	var hopRecords []distHopRecord
	var hopDetails []distHopDetail
	pathDist := 0.0

	for i := 0; i < len(chain)-1; i++ {
		a, b := chain[i], chain[i+1]
		dist := haversineKm(a.Lat, a.Lon, b.Lat, b.Lon)
		if dist > 300 {
			continue
		}

		aRep := repeaterSet[a.PublicKey]
		bRep := repeaterSet[b.PublicKey]
		var hopType string
		if aRep && bRep {
			hopType = "R↔R"
		} else if !aRep && !bRep {
			hopType = "C↔C"
		} else {
			hopType = "C↔R"
		}

		roundedDist := math.Round(dist*100) / 100
		hopRecords = append(hopRecords, distHopRecord{
			FromName: a.Name, FromPk: a.PublicKey,
			ToName: b.Name, ToPk: b.PublicKey,
			Dist: roundedDist, Type: hopType,
			SNR: tx.SNR, Hash: tx.Hash, Timestamp: tx.FirstSeen,
			HourBucket: hourBucket, tx: tx,
		})
		hopDetails = append(hopDetails, distHopDetail{
			FromName: a.Name, FromPk: a.PublicKey,
			ToName: b.Name, ToPk: b.PublicKey,
			Dist: roundedDist,
		})
		pathDist += dist
	}

	if len(hopRecords) == 0 {
		return nil, nil
	}

	pathRec := &distPathRecord{
		Hash: tx.Hash, TotalDist: math.Round(pathDist*100) / 100,
		HopCount: len(hopDetails), Timestamp: tx.FirstSeen,
		Hops: hopDetails, tx: tx,
	}
	return hopRecords, pathRec
}

func filterTxSlice(s []*StoreTx, fn func(*StoreTx) bool) []*StoreTx {
	var result []*StoreTx
	for _, tx := range s {
		if fn(tx) {
			result = append(result, tx)
		}
	}
	return result
}

// countNonPrintable counts characters that are non-printable (< 0x20 except \n, \t)
// or invalid UTF-8 replacement characters. Mirrors the heuristic from #197.
func countNonPrintable(s string) int {
	count := 0
	for _, r := range s {
		if r < 0x20 && r != '\n' && r != '\t' {
			count++
		} else if r == utf8.RuneError {
			count++
		}
	}
	return count
}

// hasGarbageChars returns true if the string contains garbage (non-printable) data.
func hasGarbageChars(s string) bool {
	return s != "" && (!utf8.ValidString(s) || countNonPrintable(s) > 2)
}

// GetChannels returns channel list from in-memory packets (payload_type 5, decoded type CHAN).
func (s *PacketStore) GetChannels(region string) []map[string]interface{} {
	cacheKey := region

	s.channelsCacheMu.Lock()
	if s.channelsCacheRes != nil && s.channelsCacheKey == cacheKey && time.Now().Before(s.channelsCacheExp) {
		res := s.channelsCacheRes
		s.channelsCacheMu.Unlock()
		return res
	}
	s.channelsCacheMu.Unlock()

	type txSnapshot struct {
		firstSeen   string
		decodedJSON string
		hasRegion   bool
	}

	// Copy only the fields needed — release the lock before JSON unmarshal.
	s.mu.RLock()
	var regionObs map[string]bool
	if region != "" {
		regionObs = s.resolveRegionObservers(region)
	}
	grpTxts := s.byPayloadType[5]
	snapshots := make([]txSnapshot, 0, len(grpTxts))
	for _, tx := range grpTxts {
		inRegion := true
		if regionObs != nil {
			inRegion = false
			for _, obs := range tx.Observations {
				if regionObs[obs.ObserverID] {
					inRegion = true
					break
				}
			}
		}
		snapshots = append(snapshots, txSnapshot{
			firstSeen:   tx.FirstSeen,
			decodedJSON: tx.DecodedJSON,
			hasRegion:   inRegion,
		})
	}
	s.mu.RUnlock()

	// JSON unmarshal outside the lock.
	type chanInfo struct {
		Hash         string
		Name         string
		LastMessage  interface{}
		LastSender   interface{}
		MessageCount int
		LastActivity string
	}
	type decodedGrp struct {
		Type    string `json:"type"`
		Channel string `json:"channel"`
		Text    string `json:"text"`
		Sender  string `json:"sender"`
	}
	channelMap := map[string]*chanInfo{}
	for _, snap := range snapshots {
		if !snap.hasRegion {
			continue
		}
		var decoded decodedGrp
		if json.Unmarshal([]byte(snap.decodedJSON), &decoded) != nil {
			continue
		}
		if decoded.Type != "CHAN" {
			continue
		}
		if hasGarbageChars(decoded.Channel) || hasGarbageChars(decoded.Text) {
			continue
		}
		channelName := decoded.Channel
		if channelName == "" {
			channelName = "unknown"
		}
		ch := channelMap[channelName]
		if ch == nil {
			ch = &chanInfo{Hash: channelName, Name: channelName, LastActivity: snap.firstSeen}
			channelMap[channelName] = ch
		}
		ch.MessageCount++
		if snap.firstSeen >= ch.LastActivity {
			ch.LastActivity = snap.firstSeen
			if decoded.Text != "" {
				idx := strings.Index(decoded.Text, ": ")
				if idx > 0 {
					ch.LastMessage = decoded.Text[idx+2:]
				} else {
					ch.LastMessage = decoded.Text
				}
				if decoded.Sender != "" {
					ch.LastSender = decoded.Sender
				}
			}
		}
	}

	channels := make([]map[string]interface{}, 0, len(channelMap))
	for _, ch := range channelMap {
		channels = append(channels, map[string]interface{}{
			"hash": ch.Hash, "name": ch.Name,
			"lastMessage": ch.LastMessage, "lastSender": ch.LastSender,
			"messageCount": ch.MessageCount, "lastActivity": ch.LastActivity,
		})
	}

	// #688: scan decoded message text for #hashtag mentions and surface any
	// previously-unseen channel names as discovered channels. We dedup against
	// channelMap (matched by name) so a channel that already has traffic does
	// NOT also appear as discovered.
	discovered := map[string]string{} // name -> lastActivity
	for _, snap := range snapshots {
		if !snap.hasRegion {
			continue
		}
		var decoded decodedGrp
		if json.Unmarshal([]byte(snap.decodedJSON), &decoded) != nil {
			continue
		}
		if decoded.Type != "CHAN" || decoded.Text == "" {
			continue
		}
		if hasGarbageChars(decoded.Text) {
			continue
		}
		for _, tag := range extractHashtagsFromText(decoded.Text) {
			// Skip if already a known/decoded channel (by name with or without '#').
			bare := tag[1:]
			if _, ok := channelMap[tag]; ok {
				continue
			}
			if _, ok := channelMap[bare]; ok {
				continue
			}
			if existing, ok := discovered[tag]; !ok || snap.firstSeen > existing {
				discovered[tag] = snap.firstSeen
			}
		}
	}
	for name, lastActivity := range discovered {
		channels = append(channels, map[string]interface{}{
			"hash":         name,
			"name":         name,
			"lastMessage":  nil,
			"lastSender":   nil,
			"messageCount": 0,
			"lastActivity": lastActivity,
			"discovered":   true,
		})
	}

	s.channelsCacheMu.Lock()
	s.channelsCacheRes = channels
	s.channelsCacheKey = cacheKey
	s.channelsCacheExp = time.Now().Add(15 * time.Second)
	s.channelsCacheMu.Unlock()

	return channels
}

// GetEncryptedChannels returns undecryptable GRP_TXT channels from in-memory packets.
func (s *PacketStore) GetEncryptedChannels(region string) []map[string]interface{} {
	s.mu.RLock()
	var regionObs map[string]bool
	if region != "" {
		regionObs = s.resolveRegionObservers(region)
	}
	grpTxts := s.byPayloadType[5]

	type encInfo struct {
		hash         string
		messageCount int
		lastActivity string
	}
	type grpDec struct {
		Type             string      `json:"type"`
		ChannelHash      interface{} `json:"channelHash"`
		ChannelHashHex   string      `json:"channelHashHex"`
		DecryptionStatus string      `json:"decryptionStatus"`
	}
	channelMap := map[string]*encInfo{}

	for _, tx := range grpTxts {
		if regionObs != nil {
			match := false
			for _, obs := range tx.Observations {
				if regionObs[obs.ObserverID] {
					match = true
					break
				}
			}
			if !match {
				continue
			}
		}
		var decoded grpDec
		if json.Unmarshal([]byte(tx.DecodedJSON), &decoded) != nil {
			continue
		}
		if decoded.Type != "GRP_TXT" || decoded.DecryptionStatus != "no_key" {
			continue
		}
		chHash := decoded.ChannelHashHex
		if chHash == "" {
			if num, ok := decoded.ChannelHash.(float64); ok {
				chHash = fmt.Sprintf("%02X", int(num))
			}
		}
		if chHash == "" {
			chHash = "?"
		}
		ch := channelMap[chHash]
		if ch == nil {
			ch = &encInfo{hash: chHash, lastActivity: tx.FirstSeen}
			channelMap[chHash] = ch
		}
		ch.messageCount++
		if tx.FirstSeen >= ch.lastActivity {
			ch.lastActivity = tx.FirstSeen
		}
	}
	s.mu.RUnlock()

	channels := make([]map[string]interface{}, 0, len(channelMap))
	for _, ch := range channelMap {
		channels = append(channels, map[string]interface{}{
			"hash":         "enc_" + ch.hash,
			"name":         "Encrypted (0x" + ch.hash + ")",
			"lastMessage":  nil,
			"lastSender":   nil,
			"messageCount": ch.messageCount,
			"lastActivity": ch.lastActivity,
			"encrypted":    true,
		})
	}
	return channels
}

// GetChannelMessages returns deduplicated messages for a channel from in-memory packets.
func (s *PacketStore) GetChannelMessages(channelHash string, limit, offset int, region ...string) ([]map[string]interface{}, int) {
	s.mu.RLock()
	defer s.mu.RUnlock()

	if limit <= 0 {
		limit = 100
	}

	type msgEntry struct {
		Data      map[string]interface{}
		Repeats   int
		Observers []string
	}
	msgMap := map[string]*msgEntry{}
	var msgOrder []string
	regionParam := ""
	if len(region) > 0 {
		regionParam = region[0]
	}
	regionObs := s.resolveRegionObservers(regionParam)

	// Iterate type-5 packets oldest-first (byPayloadType is ASC = oldest first)
	type decodedMsg struct {
		Type            string      `json:"type"`
		Channel         string      `json:"channel"`
		Text            string      `json:"text"`
		Sender          string      `json:"sender"`
		SenderTimestamp interface{} `json:"sender_timestamp"`
		PathLen         int         `json:"path_len"`
	}

	grpTxts := s.byPayloadType[5]
	for _, tx := range grpTxts {
		if regionObs != nil {
			match := false
			for _, obs := range tx.Observations {
				if regionObs[obs.ObserverID] {
					match = true
					break
				}
			}
			if !match {
				continue
			}
		}

		if tx.DecodedJSON == "" {
			continue
		}

		var decoded decodedMsg
		if json.Unmarshal([]byte(tx.DecodedJSON), &decoded) != nil {
			continue
		}
		if decoded.Type != "CHAN" {
			continue
		}
		ch := decoded.Channel
		if ch == "" {
			ch = "unknown"
		}
		if ch != channelHash {
			continue
		}

		text := decoded.Text
		sender := decoded.Sender
		if sender == "" && text != "" {
			idx := strings.Index(text, ": ")
			if idx > 0 && idx < 50 {
				sender = text[:idx]
			}
		}

		dedupeKey := sender + ":" + tx.Hash

		if existing, ok := msgMap[dedupeKey]; ok {
			existing.Repeats++
			existing.Data["repeats"] = existing.Repeats
			// Add observer if new
			obsName := tx.ObserverName
			if obsName == "" {
				obsName = tx.ObserverID
			}
			if obsName != "" {
				found := false
				for _, o := range existing.Observers {
					if o == obsName {
						found = true
						break
					}
				}
				if !found {
					existing.Observers = append(existing.Observers, obsName)
					existing.Data["observers"] = existing.Observers
				}
			}
		} else {
			displaySender := sender
			displayText := text
			if text != "" {
				idx := strings.Index(text, ": ")
				if idx > 0 && idx < 50 {
					displaySender = text[:idx]
					displayText = text[idx+2:]
				}
			}

			hops := pathLen(tx.PathJSON)

			var snrVal interface{}
			if tx.SNR != nil {
				snrVal = *tx.SNR
			}

			senderTs := decoded.SenderTimestamp

			observers := []string{}
			obsName := tx.ObserverName
			if obsName == "" {
				obsName = tx.ObserverID
			}
			if obsName != "" {
				observers = []string{obsName}
			}

			entry := &msgEntry{
				Data: map[string]interface{}{
					"sender":           displaySender,
					"text":             displayText,
					"timestamp":        strOrNil(tx.FirstSeen),
					"sender_timestamp": senderTs,
					"packetId":         tx.ID,
					"packetHash":       strOrNil(tx.Hash),
					"repeats":          1,
					"observers":        observers,
					"hops":             hops,
					"snr":              snrVal,
				},
				Repeats:   1,
				Observers: observers,
			}
			msgMap[dedupeKey] = entry
			msgOrder = append(msgOrder, dedupeKey)
		}
	}

	total := len(msgOrder)
	// Return latest messages (tail)
	start := total - limit - offset
	if start < 0 {
		start = 0
	}
	end := total - offset
	if end < 0 {
		end = 0
	}
	if end > total {
		end = total
	}

	messages := make([]map[string]interface{}, 0, end-start)
	for i := start; i < end; i++ {
		messages = append(messages, msgMap[msgOrder[i]].Data)
	}
	return messages, total
}

// GetAnalyticsChannels returns full channel analytics computed from in-memory packets.
func (s *PacketStore) GetAnalyticsChannels(region string) map[string]interface{} {
	return s.GetAnalyticsChannelsWithWindow(region, TimeWindow{})
}

// GetAnalyticsChannelsWithWindow returns channel analytics for the given region,
// optionally bounded to a time window (issue #842). Zero TimeWindow = all data.
func (s *PacketStore) GetAnalyticsChannelsWithWindow(region string, window TimeWindow) map[string]interface{} {
	if region == "" && window.IsZero() {
		s.analyticsRecomputerMu.RLock()
		rc := s.recompChannels
		s.analyticsRecomputerMu.RUnlock()
		if rc != nil {
			if v := rc.Load(); v != nil {
				if m, ok := v.(map[string]interface{}); ok {
					s.cacheMu.Lock()
					s.cacheHits++
					s.cacheMu.Unlock()
					return m
				}
			}
		}
	}
	cacheKey := region
	if !window.IsZero() {
		cacheKey = region + "|" + window.CacheKey()
	}
	s.cacheMu.Lock()
	if cached, ok := s.chanCache[cacheKey]; ok && time.Now().Before(cached.expiresAt) {
		s.cacheHits++
		s.cacheMu.Unlock()
		return cached.data
	}
	s.cacheMisses++
	s.cacheMu.Unlock()

	result := s.computeAnalyticsChannels(region, window)

	s.cacheMu.Lock()
	s.chanCache[cacheKey] = &cachedResult{data: result, expiresAt: time.Now().Add(s.rfCacheTTL)}
	s.cacheMu.Unlock()

	return result
}

// channelNameMatchesHash validates that a decrypted channel name hashes to the
// observed single-byte channel hash. This rejects rainbow-table mismatches where
// an observer's lookup table incorrectly maps a hash byte to the wrong name.
// Firmware invariant: channelHash = SHA256(SHA256("#name")[:16])[0]
func channelNameMatchesHash(name string, hashStr string) bool {
	expected, err := strconv.Atoi(hashStr)
	if err != nil {
		return false
	}
	chanName := name
	if !strings.HasPrefix(chanName, "#") {
		chanName = "#" + chanName
	}
	h1 := sha256.Sum256([]byte(chanName))
	h2 := sha256.Sum256(h1[:16])
	return int(h2[0]) == expected
}

// isPlaceholderName returns true if the name is a "chN" placeholder (not a real decrypted name).
func isPlaceholderName(name string) bool {
	if !strings.HasPrefix(name, "ch") {
		return false
	}
	_, err := strconv.Atoi(name[2:])
	return err == nil
}

func (s *PacketStore) computeAnalyticsChannels(region string, window TimeWindow) map[string]interface{} {
	s.mu.RLock()
	defer s.mu.RUnlock()

	var regionObs map[string]bool
	if region != "" {
		regionObs = s.resolveRegionObservers(region)
	}

	type decodedGrp struct {
		Type         string      `json:"type"`
		Channel      string      `json:"channel"`
		ChannelHash  interface{} `json:"channelHash"`
		ChannelHash2 string      `json:"channel_hash"`
		Text         string      `json:"text"`
		Sender       string      `json:"sender"`
	}

	// Convert channelHash (number or string in JSON) to string
	chHashStr := func(v interface{}) string {
		if v == nil {
			return ""
		}
		switch val := v.(type) {
		case string:
			return val
		case float64:
			return strconv.FormatFloat(val, 'f', -1, 64)
		default:
			return fmt.Sprintf("%v", val)
		}
	}

	type chanInfo struct {
		Hash         string
		Name         string
		Messages     int
		Senders      map[string]bool
		LastActivity string
		Encrypted    bool
	}

	channelMap := map[string]*chanInfo{}
	senderCounts := map[string]int{}
	msgLengths := make([]int, 0)
	timeline := map[string]int{} // hour|channelName → count

	grpTxts := s.byPayloadType[5]
	for _, tx := range grpTxts {
		if !window.Includes(tx.FirstSeen) {
			continue
		}
		if regionObs != nil {
			match := false
			for _, obs := range tx.Observations {
				if regionObs[obs.ObserverID] {
					match = true
					break
				}
			}
			if !match {
				continue
			}
		}

		var decoded decodedGrp
		if json.Unmarshal([]byte(tx.DecodedJSON), &decoded) != nil {
			continue
		}

		hash := chHashStr(decoded.ChannelHash)
		if hash == "" {
			hash = decoded.ChannelHash2
		}
		if hash == "" {
			hash = "?"
		}
		name := decoded.Channel
		if name == "" {
			name = "ch" + hash
		}
		encrypted := decoded.Text == "" && decoded.Sender == ""

		// Bug #978 fix: validate channel name against hash to reject rainbow-table mismatches.
		// If the claimed channel name doesn't hash to the observed channelHash byte, discard it.
		if name != "" && name != "ch"+hash && !channelNameMatchesHash(name, hash) {
			name = "ch" + hash
			encrypted = true
		}

		// Bug #978 fix: always group by hash byte alone — same physical channel,
		// regardless of which observer decrypted it.
		chKey := hash

		ch := channelMap[chKey]
		if ch == nil {
			ch = &chanInfo{Hash: hash, Name: name, Senders: map[string]bool{}, LastActivity: tx.FirstSeen, Encrypted: encrypted}
			channelMap[chKey] = ch
		} else {
			// Upgrade bucket name: if current is placeholder and we have a validated decrypted name
			if isPlaceholderName(ch.Name) && !isPlaceholderName(name) {
				ch.Name = name
			}
		}
		ch.Messages++
		ch.LastActivity = tx.FirstSeen
		if !encrypted {
			ch.Encrypted = false
		}

		if decoded.Sender != "" {
			ch.Senders[decoded.Sender] = true
			senderCounts[decoded.Sender]++
		}
		if decoded.Text != "" {
			msgLengths = append(msgLengths, len(decoded.Text))
		}

		// Timeline
		if len(tx.FirstSeen) >= 13 {
			hr := tx.FirstSeen[:13]
			key := hr + "|" + name
			timeline[key]++
		}
	}

	channelList := make([]map[string]interface{}, 0, len(channelMap))
	decryptable := 0
	for _, c := range channelMap {
		if !c.Encrypted {
			decryptable++
		}
		channelList = append(channelList, map[string]interface{}{
			"hash": c.Hash, "name": c.Name,
			"messages": c.Messages, "senders": len(c.Senders),
			"lastActivity": c.LastActivity, "encrypted": c.Encrypted,
		})
	}
	sort.Slice(channelList, func(i, j int) bool {
		return channelList[i]["messages"].(int) > channelList[j]["messages"].(int)
	})

	// Top senders
	type senderEntry struct {
		name  string
		count int
	}
	senderList := make([]senderEntry, 0, len(senderCounts))
	for n, c := range senderCounts {
		senderList = append(senderList, senderEntry{n, c})
	}
	sort.Slice(senderList, func(i, j int) bool { return senderList[i].count > senderList[j].count })
	topSenders := make([]map[string]interface{}, 0)
	for i, e := range senderList {
		if i >= 15 {
			break
		}
		topSenders = append(topSenders, map[string]interface{}{"name": e.name, "count": e.count})
	}

	// Channel timeline
	type tlEntry struct {
		hour, channel string
		count         int
	}
	var tlList []tlEntry
	for key, count := range timeline {
		parts := strings.SplitN(key, "|", 2)
		if len(parts) == 2 {
			tlList = append(tlList, tlEntry{parts[0], parts[1], count})
		}
	}
	sort.Slice(tlList, func(i, j int) bool { return tlList[i].hour < tlList[j].hour })
	channelTimeline := make([]map[string]interface{}, 0, len(tlList))
	for _, e := range tlList {
		channelTimeline = append(channelTimeline, map[string]interface{}{
			"hour": e.hour, "channel": e.channel, "count": e.count,
		})
	}

	return map[string]interface{}{
		"activeChannels":  len(channelList),
		"decryptable":     decryptable,
		"channels":        channelList,
		"topSenders":      topSenders,
		"channelTimeline": channelTimeline,
		"msgLengths":      msgLengths,
	}
}

// GetAnalyticsRF returns full RF analytics computed from in-memory observations.
func (s *PacketStore) GetAnalyticsRF(region string) map[string]interface{} {
	return s.GetAnalyticsRFWithWindow(region, TimeWindow{})
}

// GetAnalyticsRFWithWindow returns RF analytics bounded by an optional
// time window (issue #842). Zero TimeWindow = all data (backwards compatible).
func (s *PacketStore) GetAnalyticsRFWithWindow(region string, window TimeWindow) map[string]interface{} {
	if region == "" && window.IsZero() {
		s.analyticsRecomputerMu.RLock()
		rc := s.recompRF
		s.analyticsRecomputerMu.RUnlock()
		if rc != nil {
			if v := rc.Load(); v != nil {
				if m, ok := v.(map[string]interface{}); ok {
					s.cacheMu.Lock()
					s.cacheHits++
					s.cacheMu.Unlock()
					return m
				}
			}
		}
	}
	cacheKey := region
	if !window.IsZero() {
		cacheKey = region + "|" + window.CacheKey()
	}
	s.cacheMu.Lock()
	if cached, ok := s.rfCache[cacheKey]; ok && time.Now().Before(cached.expiresAt) {
		s.cacheHits++
		s.cacheMu.Unlock()
		return cached.data
	}
	s.cacheMisses++
	s.cacheMu.Unlock()

	result := s.computeAnalyticsRF(region, window)

	s.cacheMu.Lock()
	s.rfCache[cacheKey] = &cachedResult{data: result, expiresAt: time.Now().Add(s.rfCacheTTL)}
	s.cacheMu.Unlock()

	return result
}

func (s *PacketStore) computeAnalyticsRF(region string, window TimeWindow) map[string]interface{} {
	s.mu.RLock()
	defer s.mu.RUnlock()

	ptNames := payloadTypeNames

	var regionObs map[string]bool
	if region != "" {
		regionObs = s.resolveRegionObservers(region)
	}

	// Collect all observations matching the region
	estCap := s.totalObs
	if estCap > 2000000 {
		estCap = 2000000
	}
	snrVals := make([]float64, 0, estCap/2)
	rssiVals := make([]float64, 0, estCap/2)
	packetSizes := make([]int, 0, len(s.packets))
	seenSizeHashes := make(map[string]bool, len(s.packets))
	seenTypeHashes := make(map[string]bool, len(s.packets))
	typeBuckets := map[int]int{}
	hourBuckets := map[string]int{}
	seenHourHash := make(map[string]bool, len(s.packets)) // dedup packets-per-hour by hash+hour
	snrByType := map[string]*struct{ vals []float64 }{}
	sigTime := map[string]*struct {
		snrs  []float64
		count int
	}{}
	scatterAll := make([]struct{ snr, rssi float64 }, 0, estCap/4)
	totalObs := 0
	regionalHashes := make(map[string]bool, len(s.packets))
	var minTimestamp, maxTimestamp string

	if regionObs != nil {
		// Regional: iterate observations from matching observers
		for obsID := range regionObs {
			obsList := s.byObserver[obsID]
			for _, obs := range obsList {
				if !window.Includes(obs.Timestamp) {
					continue
				}
				totalObs++
				tx := s.byTxID[obs.TransmissionID]
				hash := ""
				if tx != nil {
					hash = tx.Hash
				}
				if hash != "" {
					regionalHashes[hash] = true
				}

				ts := obs.Timestamp
				if ts != "" {
					if minTimestamp == "" || ts < minTimestamp {
						minTimestamp = ts
					}
					if ts > maxTimestamp {
						maxTimestamp = ts
					}
				}

				// SNR/RSSI
				if obs.SNR != nil {
					snrVals = append(snrVals, *obs.SNR)
					typeName := "UNK"
					if tx != nil && tx.PayloadType != nil {
						if n, ok := ptNames[*tx.PayloadType]; ok {
							typeName = n
						} else {
							typeName = fmt.Sprintf("UNK(%d)", *tx.PayloadType)
						}
					}
					if snrByType[typeName] == nil {
						snrByType[typeName] = &struct{ vals []float64 }{}
					}
					snrByType[typeName].vals = append(snrByType[typeName].vals, *obs.SNR)

					if obs.RSSI != nil {
						scatterAll = append(scatterAll, struct{ snr, rssi float64 }{*obs.SNR, *obs.RSSI})
					}

					// Signal over time
					if len(ts) >= 13 {
						hr := ts[:13]
						if sigTime[hr] == nil {
							sigTime[hr] = &struct {
								snrs  []float64
								count int
							}{}
						}
						sigTime[hr].snrs = append(sigTime[hr].snrs, *obs.SNR)
						sigTime[hr].count++
					}
				}
				if obs.RSSI != nil {
					rssiVals = append(rssiVals, *obs.RSSI)
				}

				// Packets per hour (unique by hash per hour)
				if len(ts) >= 13 {
					hr := ts[:13]
					hk := hash + "|" + hr
					if hash == "" || !seenHourHash[hk] {
						if hash != "" {
							seenHourHash[hk] = true
						}
						hourBuckets[hr]++
					}
				}

				// Packet sizes (unique by hash)
				if hash != "" && !seenSizeHashes[hash] && tx != nil && tx.RawHex != "" {
					seenSizeHashes[hash] = true
					packetSizes = append(packetSizes, len(tx.RawHex)/2)
				}

				// Payload type distribution (unique by hash)
				if hash != "" && !seenTypeHashes[hash] && tx != nil && tx.PayloadType != nil {
					seenTypeHashes[hash] = true
					typeBuckets[*tx.PayloadType]++
				}
			}
		}
	} else {
		// No region: iterate all transmissions and their observations
		for _, tx := range s.packets {
			// Window filter: skip transmissions outside the requested window.
			// We use tx.FirstSeen as the bounding timestamp; per-obs window
			// filter below handles cases where individual obs timestamps differ.
			if !window.Includes(tx.FirstSeen) {
				continue
			}
			hash := tx.Hash
			if hash != "" {
				regionalHashes[hash] = true
				if !seenSizeHashes[hash] && tx.RawHex != "" {
					seenSizeHashes[hash] = true
					packetSizes = append(packetSizes, len(tx.RawHex)/2)
				}
				if !seenTypeHashes[hash] && tx.PayloadType != nil {
					seenTypeHashes[hash] = true
					typeBuckets[*tx.PayloadType]++
				}
			}

			// Pre-resolve type name once per transmission
			typeName := "UNK"
			if tx.PayloadType != nil {
				if n, ok := ptNames[*tx.PayloadType]; ok {
					typeName = n
				} else {
					typeName = fmt.Sprintf("UNK(%d)", *tx.PayloadType)
				}
			}

			if len(tx.Observations) > 0 {
				for _, obs := range tx.Observations {
					totalObs++
					ts := obs.Timestamp
					if ts != "" {
						if minTimestamp == "" || ts < minTimestamp {
							minTimestamp = ts
						}
						if ts > maxTimestamp {
							maxTimestamp = ts
						}
					}

					if obs.SNR != nil {
						snr := *obs.SNR
						snrVals = append(snrVals, snr)
						entry := snrByType[typeName]
						if entry == nil {
							entry = &struct{ vals []float64 }{}
							snrByType[typeName] = entry
						}
						entry.vals = append(entry.vals, snr)

						if obs.RSSI != nil {
							scatterAll = append(scatterAll, struct{ snr, rssi float64 }{snr, *obs.RSSI})
						}

						if len(ts) >= 13 {
							hr := ts[:13]
							st := sigTime[hr]
							if st == nil {
								st = &struct {
									snrs  []float64
									count int
								}{}
								sigTime[hr] = st
							}
							st.snrs = append(st.snrs, snr)
							st.count++
						}
					}
					if obs.RSSI != nil {
						rssiVals = append(rssiVals, *obs.RSSI)
					}

					if len(ts) >= 13 {
						hr := ts[:13]
						hk := hash + "|" + hr
						if hash == "" || !seenHourHash[hk] {
							if hash != "" {
								seenHourHash[hk] = true
							}
							hourBuckets[hr]++
						}
					}
				}
			} else {
				// Legacy: transmission without observations
				totalObs++
				if tx.SNR != nil {
					snrVals = append(snrVals, *tx.SNR)
				}
				if tx.RSSI != nil {
					rssiVals = append(rssiVals, *tx.RSSI)
				}
				ts := tx.FirstSeen
				if ts != "" {
					if minTimestamp == "" || ts < minTimestamp {
						minTimestamp = ts
					}
					if ts > maxTimestamp {
						maxTimestamp = ts
					}
				}
				if len(ts) >= 13 {
					hourBuckets[ts[:13]]++
				}
			}
		}
	}

	// Stats helpers
	stddevF64 := func(arr []float64, avg float64) float64 {
		if len(arr) == 0 {
			return 0
		}
		sum := 0.0
		for _, v := range arr {
			d := v - avg
			sum += d * d
		}
		return math.Sqrt(sum / float64(len(arr)))
	}
	minF64 := func(arr []float64) float64 {
		if len(arr) == 0 {
			return 0
		}
		m := arr[0]
		for _, v := range arr[1:] {
			if v < m {
				m = v
			}
		}
		return m
	}
	maxF64 := func(arr []float64) float64 {
		if len(arr) == 0 {
			return 0
		}
		m := arr[0]
		for _, v := range arr[1:] {
			if v > m {
				m = v
			}
		}
		return m
	}
	minInt := func(arr []int) int {
		if len(arr) == 0 {
			return 0
		}
		m := arr[0]
		for _, v := range arr[1:] {
			if v < m {
				m = v
			}
		}
		return m
	}
	maxInt := func(arr []int) int {
		if len(arr) == 0 {
			return 0
		}
		m := arr[0]
		for _, v := range arr[1:] {
			if v > m {
				m = v
			}
		}
		return m
	}

	// Sort snrVals and rssiVals once; reuse sorted order for min/max/median
	// instead of copying+sorting per stat call (#366).
	sort.Float64s(snrVals)
	sort.Float64s(rssiVals)

	snrAvg := 0.0
	if len(snrVals) > 0 {
		sum := 0.0
		for _, v := range snrVals {
			sum += v
		}
		snrAvg = sum / float64(len(snrVals))
	}
	rssiAvg := 0.0
	if len(rssiVals) > 0 {
		sum := 0.0
		for _, v := range rssiVals {
			sum += v
		}
		rssiAvg = sum / float64(len(rssiVals))
	}

	// Packets per hour
	type hourCount struct {
		Hour  string `json:"hour"`
		Count int    `json:"count"`
	}
	hourKeys := make([]string, 0, len(hourBuckets))
	for k := range hourBuckets {
		hourKeys = append(hourKeys, k)
	}
	sort.Strings(hourKeys)
	packetsPerHour := make([]hourCount, len(hourKeys))
	for i, k := range hourKeys {
		packetsPerHour[i] = hourCount{Hour: k, Count: hourBuckets[k]}
	}

	// Payload types
	type ptEntry struct {
		Type  int    `json:"type"`
		Name  string `json:"name"`
		Count int    `json:"count"`
	}
	payloadTypes := make([]ptEntry, 0, len(typeBuckets))
	for t, c := range typeBuckets {
		name := ptNames[t]
		if name == "" {
			name = fmt.Sprintf("UNK(%d)", t)
		}
		payloadTypes = append(payloadTypes, ptEntry{Type: t, Name: name, Count: c})
	}
	sort.Slice(payloadTypes, func(i, j int) bool { return payloadTypes[i].Count > payloadTypes[j].Count })

	// SNR by type
	type snrTypeEntry struct {
		Name  string  `json:"name"`
		Count int     `json:"count"`
		Avg   float64 `json:"avg"`
		Min   float64 `json:"min"`
		Max   float64 `json:"max"`
	}
	snrByTypeArr := make([]snrTypeEntry, 0, len(snrByType))
	for name, d := range snrByType {
		sum := 0.0
		for _, v := range d.vals {
			sum += v
		}
		snrByTypeArr = append(snrByTypeArr, snrTypeEntry{
			Name: name, Count: len(d.vals),
			Avg: sum / float64(len(d.vals)),
			Min: minF64(d.vals), Max: maxF64(d.vals),
		})
	}
	sort.Slice(snrByTypeArr, func(i, j int) bool { return snrByTypeArr[i].Count > snrByTypeArr[j].Count })

	// Signal over time
	type sigTimeEntry struct {
		Hour   string  `json:"hour"`
		Count  int     `json:"count"`
		AvgSnr float64 `json:"avgSnr"`
	}
	sigKeys := make([]string, 0, len(sigTime))
	for k := range sigTime {
		sigKeys = append(sigKeys, k)
	}
	sort.Strings(sigKeys)
	signalOverTime := make([]sigTimeEntry, len(sigKeys))
	for i, k := range sigKeys {
		d := sigTime[k]
		sum := 0.0
		for _, v := range d.snrs {
			sum += v
		}
		signalOverTime[i] = sigTimeEntry{Hour: k, Count: d.count, AvgSnr: sum / float64(d.count)}
	}

	// Scatter (downsample to 500)
	type scatterPoint struct {
		SNR  float64 `json:"snr"`
		RSSI float64 `json:"rssi"`
	}
	scatterStep := 1
	if len(scatterAll) > 500 {
		scatterStep = len(scatterAll) / 500
	}
	scatterData := make([]scatterPoint, 0, 500)
	for i, p := range scatterAll {
		if i%scatterStep == 0 {
			scatterData = append(scatterData, scatterPoint{SNR: p.snr, RSSI: p.rssi})
		}
	}

	// Histograms
	buildHistogramF64 := func(values []float64, bins int) map[string]interface{} {
		if len(values) == 0 {
			return map[string]interface{}{"bins": []interface{}{}, "min": 0, "max": 0}
		}
		mn, mx := minF64(values), maxF64(values)
		rng := mx - mn
		if rng == 0 {
			rng = 1
		}
		binWidth := rng / float64(bins)
		counts := make([]int, bins)
		for _, v := range values {
			idx := int((v - mn) / binWidth)
			if idx >= bins {
				idx = bins - 1
			}
			counts[idx]++
		}
		binArr := make([]map[string]interface{}, bins)
		for i, c := range counts {
			binArr[i] = map[string]interface{}{"x": mn + float64(i)*binWidth, "w": binWidth, "count": c}
		}
		return map[string]interface{}{"bins": binArr, "min": mn, "max": mx}
	}
	buildHistogramInt := func(values []int, bins int) map[string]interface{} {
		if len(values) == 0 {
			return map[string]interface{}{"bins": []interface{}{}, "min": 0, "max": 0}
		}
		mn, mx := float64(minInt(values)), float64(maxInt(values))
		rng := mx - mn
		if rng == 0 {
			rng = 1
		}
		binWidth := rng / float64(bins)
		counts := make([]int, bins)
		for _, v := range values {
			idx := int((float64(v) - mn) / binWidth)
			if idx >= bins {
				idx = bins - 1
			}
			counts[idx]++
		}
		binArr := make([]map[string]interface{}, bins)
		for i, c := range counts {
			binArr[i] = map[string]interface{}{"x": mn + float64(i)*binWidth, "w": binWidth, "count": c}
		}
		return map[string]interface{}{"bins": binArr, "min": mn, "max": mx}
	}

	snrHistogram := buildHistogramF64(snrVals, 20)
	rssiHistogram := buildHistogramF64(rssiVals, 20)
	sizeHistogram := buildHistogramInt(packetSizes, 25)

	// Time span from min/max timestamps tracked during first pass
	timeSpanHours := 0.0
	if minTimestamp != "" && maxTimestamp != "" && minTimestamp != maxTimestamp {
		// Parse only 2 timestamps instead of 1.2M
		parseTS := func(ts string) (time.Time, bool) {
			t, err := time.Parse("2006-01-02 15:04:05", ts)
			if err != nil {
				t, err = time.Parse(time.RFC3339, ts)
			}
			if err != nil {
				return time.Time{}, false
			}
			return t, true
		}
		if tMin, ok := parseTS(minTimestamp); ok {
			if tMax, ok := parseTS(maxTimestamp); ok {
				timeSpanHours = float64(tMax.UnixMilli()-tMin.UnixMilli()) / 3600000.0
			}
		}
	}

	// Avg packet size
	avgPktSize := 0
	if len(packetSizes) > 0 {
		sum := 0
		for _, v := range packetSizes {
			sum += v
		}
		avgPktSize = sum / len(packetSizes)
	}

	// snrVals and rssiVals are already sorted — read min/max/median directly.
	snrStats := map[string]interface{}{"min": 0.0, "max": 0.0, "avg": 0.0, "median": 0.0, "stddev": 0.0}
	if len(snrVals) > 0 {
		snrStats = map[string]interface{}{
			"min": snrVals[0], "max": snrVals[len(snrVals)-1],
			"avg": snrAvg, "median": snrVals[len(snrVals)/2],
			"stddev": stddevF64(snrVals, snrAvg),
		}
	}
	rssiStats := map[string]interface{}{"min": 0.0, "max": 0.0, "avg": 0.0, "median": 0.0, "stddev": 0.0}
	if len(rssiVals) > 0 {
		rssiStats = map[string]interface{}{
			"min": rssiVals[0], "max": rssiVals[len(rssiVals)-1],
			"avg": rssiAvg, "median": rssiVals[len(rssiVals)/2],
			"stddev": stddevF64(rssiVals, rssiAvg),
		}
	}

	return map[string]interface{}{
		"totalPackets":       len(snrVals),
		"totalAllPackets":    totalObs,
		"totalTransmissions": len(regionalHashes),
		"snr":                snrStats,
		"rssi":               rssiStats,
		"snrValues":          snrHistogram,
		"rssiValues":         rssiHistogram,
		"packetSizes":        sizeHistogram,
		"minPacketSize":      minInt(packetSizes),
		"maxPacketSize":      maxInt(packetSizes),
		"avgPacketSize":      avgPktSize,
		"packetsPerHour":     packetsPerHour,
		"payloadTypes":       payloadTypes,
		"snrByType":          snrByTypeArr,
		"signalOverTime":     signalOverTime,
		"scatterData":        scatterData,
		"timeSpanHours":      timeSpanHours,
	}
}

// --- Topology Analytics ---

type nodeInfo struct {
	PublicKey        string
	Name             string
	Role             string
	Lat              float64
	Lon              float64
	HasGPS           bool
	LastSeen         time.Time
	ObservationCount int // count of advertisements/observations; used for tier-3 tiebreak in resolveWithContext
}

// schemaDegradationLogged is now a PacketStore field (see type definition) so
// each store/test instance has a fresh dedupe set. Issue #1199 item 5: the
// prior package-level sync.Map silently suppressed re-emission across tests.

func (s *PacketStore) logSchemaDegradationOnce(msg string) {
	if _, loaded := s.schemaDegradationLogged.LoadOrStore(msg, true); !loaded {
		log.Printf("[store] schema-degradation: %s", msg)
	}
}

func (s *PacketStore) getAllNodes() []nodeInfo {
	// Schema probe: try richest → leanest. Logs a one-shot warning when we
	// fall back to a thinner schema so operators see that a column is
	// missing and the new tiebreak features are degraded. See #1197
	// (adversarial r1 #10).
	rows, err := s.db.conn.Query("SELECT public_key, name, role, lat, lon, last_seen, COALESCE(advert_count, 0) FROM nodes")
	hasLastSeen := true
	hasAdvertCount := true
	if err != nil {
		s.logSchemaDegradationOnce("nodes.advert_count missing — tier-3/4 ObservationCount tiebreak degraded; resolveWithContext will fall back to lex-pubkey order")
		rows, err = s.db.conn.Query("SELECT public_key, name, role, lat, lon, last_seen FROM nodes")
		hasAdvertCount = false
		if err != nil {
			s.logSchemaDegradationOnce("nodes.last_seen missing — node freshness signal unavailable")
			rows, err = s.db.conn.Query("SELECT public_key, name, role, lat, lon FROM nodes")
			hasLastSeen = false
			if err != nil {
				return nil
			}
		}
	}
	defer rows.Close()
	var nodes []nodeInfo
	for rows.Next() {
		var pk string
		var name, role sql.NullString
		var lat, lon sql.NullFloat64
		var lastSeen sql.NullString
		var advertCount sql.NullInt64
		if hasAdvertCount {
			rows.Scan(&pk, &name, &role, &lat, &lon, &lastSeen, &advertCount)
		} else if hasLastSeen {
			rows.Scan(&pk, &name, &role, &lat, &lon, &lastSeen)
		} else {
			rows.Scan(&pk, &name, &role, &lat, &lon)
		}
		n := nodeInfo{PublicKey: pk, Name: nullStrVal(name), Role: nullStrVal(role)}
		if lat.Valid && lon.Valid {
			n.Lat = lat.Float64
			n.Lon = lon.Float64
			n.HasGPS = !(n.Lat == 0 && n.Lon == 0)
		}
		if hasLastSeen && lastSeen.Valid && lastSeen.String != "" {
			if t, err := time.Parse(time.RFC3339, lastSeen.String); err == nil {
				n.LastSeen = t
			} else if t, err := time.Parse("2006-01-02 15:04:05", lastSeen.String); err == nil {
				n.LastSeen = t
			}
		}
		if hasAdvertCount && advertCount.Valid {
			n.ObservationCount = int(advertCount.Int64)
		}
		nodes = append(nodes, n)
	}
	return nodes
}

type prefixMap struct {
	m map[string][]nodeInfo
}

// maxPrefixLen caps prefix map entries. MeshCore path hops use 2–6 char
// prefixes; 8 gives comfortable headroom while cutting map size from ~31×N
// entries to ~7×N (+ 1 full-key entry per node for exact-match lookups).
const maxPrefixLen = 8

// canAppearInPath returns true if the node's role allows it to appear as a
// path hop.  Only repeaters, room servers, and rooms can forward packets;
// companions and sensors originate but never relay.
func canAppearInPath(role string) bool {
	r := strings.ToLower(role)
	return strings.Contains(r, "repeater") || strings.Contains(r, "room_server") || r == "room"
}

func buildPrefixMap(nodes []nodeInfo) *prefixMap {
	pm := &prefixMap{m: make(map[string][]nodeInfo, len(nodes)*(maxPrefixLen+1))}
	for _, n := range nodes {
		if !canAppearInPath(n.Role) {
			continue
		}
		pk := strings.ToLower(n.PublicKey)
		maxLen := maxPrefixLen
		if maxLen > len(pk) {
			maxLen = len(pk)
		}
		for l := 2; l <= maxLen; l++ {
			pfx := pk[:l]
			pm.m[pfx] = append(pm.m[pfx], n)
		}
		// Always add full pubkey so exact-match lookups work.
		if len(pk) > maxPrefixLen {
			pm.m[pk] = append(pm.m[pk], n)
		}
	}
	return pm
}

// getCachedNodesAndPM returns cached node list and prefix map, rebuilding if stale.
// Must be called with s.mu held (RLock or Lock).
func (s *PacketStore) getCachedNodesAndPM() ([]nodeInfo, *prefixMap) {
	s.cacheMu.Lock()
	if s.nodeCache != nil && time.Since(s.nodeCacheTime) < 30*time.Second {
		nodes, pm := s.nodeCache, s.nodePM
		s.cacheMu.Unlock()
		return nodes, pm
	}
	s.cacheMu.Unlock()

	nodes := s.getAllNodes()
	pm := buildPrefixMap(nodes)

	s.cacheMu.Lock()
	s.nodeCache = nodes
	s.nodePM = pm
	s.nodeCacheTime = time.Now()
	s.cacheMu.Unlock()

	return nodes, pm
}

// InvalidateNodeCache forces the next getCachedNodesAndPM call to rebuild.
func (s *PacketStore) InvalidateNodeCache() {
	s.cacheMu.Lock()
	s.nodeCache = nil
	s.nodePM = nil
	s.nodeCacheTime = time.Time{}
	s.cacheMu.Unlock()
}

func (pm *prefixMap) resolve(hop string) *nodeInfo {
	h := strings.ToLower(hop)
	candidates := pm.m[h]
	if len(candidates) == 0 {
		return nil
	}
	if len(candidates) == 1 {
		return &candidates[0]
	}
	// Multiple candidates: prefer one with GPS
	for i := range candidates {
		if candidates[i].HasGPS {
			return &candidates[i]
		}
	}
	return &candidates[0]
}

// resolveWithContext resolves a hop prefix using the neighbor affinity graph
// for disambiguation when multiple candidates match. It applies a 4-tier
// priority:
//
//	(1) "neighbor_affinity"           — graph score vs context nodes,
//	                                    requires affinity ≥3× runner-up and
//	                                    affinityMinObservations
//	(2) "geo_proximity"               — geographic proximity to GPS context
//	                                    centroid (only fires when at least
//	                                    one context node has GPS)
//	(3) "gps_preference"              — among GPS-having candidates, pick
//	                                    highest ObservationCount; lex-pubkey
//	                                    tiebreak for determinism
//	(4) "observation_count_fallback"  — no GPS available; pick highest
//	                                    ObservationCount; lex-pubkey tiebreak
//
// (Pre-PR #1197/#1198 the tier-3 step was first-GPS-wins and tier-4 was
// first-slice-element. Both now use observation count + lex tiebreak; the
// returned method label was renamed accordingly.)
//
// contextPubkeys are pubkeys of nodes that provide context for disambiguation
// (e.g., the originator, observer, or adjacent hops in the path).
// graph may be nil, in which case tier-1 is skipped.
func (pm *prefixMap) resolveWithContext(hop string, contextPubkeys []string, graph *NeighborGraph) (*nodeInfo, string, float64) {
	h := strings.ToLower(hop)
	candidates := pm.m[h]
	if len(candidates) == 0 {
		return nil, "no_match", 0
	}
	if len(candidates) == 1 {
		return &candidates[0], "unique_prefix", 1.0
	}

	// Priority 1: Affinity graph score
	//
	// NOTE: We use raw Score() (count × time-decay) here rather than Jaccard
	// similarity. Jaccard is used at the graph builder level (disambiguate() in
	// neighbor_graph.go) to resolve ambiguous edges by comparing neighbor-set
	// overlap. Here, edges are already resolved — we just need to pick the
	// highest-affinity candidate among them. Raw score is appropriate because
	// it reflects both observation frequency and recency, which are the right
	// signals for "which candidate is this hop most likely referring to."
	//
	// Issue #1229 (Option C): the raw score is further multiplied by
	// e.Confidence() — a source-diversity factor in (0,1] derived from the
	// number of distinct observers that contributed to the edge. Edges seen
	// by a single observer are discounted to 1/3 weight; edges seen by ≥3
	// observers saturate at full weight. This stacks with the geo-rejection
	// filter merged for #1228 to give two independent lines of defense
	// against cross-region prefix-collision pollution. Backward-compatible
	// with the persistence format: legacy edges with empty Observers sets
	// fall back to single-observer weight.
	if graph != nil && len(contextPubkeys) > 0 {
		type scored struct {
			idx   int
			score float64
			count int // observation count of the best-scoring edge
		}
		now := time.Now()
		// PERF (#1247): hoist per-context work out of the candidate loop.
		// The previous shape ran graph.Neighbors(ToLower(ctxPK)) and
		// re-lowercased candPK on every (cand, ctxPK) pair, then used
		// strings.EqualFold to compare two already-lowercased pubkeys.
		// At analytics scale (5k+ contextPubkeys, ~30k resolveHop calls)
		// this dominated computeAnalyticsTopology / computeAnalyticsRF
		// CPU time (37% / 55% of those endpoints respectively per
		// pprof). The new shape:
		//   1. Lowercases ctx pubkeys at most once per call (skipped
		//      entirely when the input is already lowercased — the
		//      common case for analytics callers that go through
		//      buildHopContextPubkeys).
		//   2. Lowercases candidate pubkeys at most once per call and
		//      uses raw == comparisons against NeighborEdge.NodeA/NodeB
		//      (which makeEdgeKey already lowercases).
		//   3. Loops outer-ctx / inner-edge / matched-cand-lookup. The
		//      previous shape was outer-cand / inner-ctx / inner-edge,
		//      which called graph.Neighbors(ctxPK) — taking the graph
		//      RLock each time — once per (cand, ctx) pair instead of
		//      once per ctx.
		lowerCtx := contextPubkeys
		needLower := false
		for _, p := range contextPubkeys {
			if hasUpperASCII(p) {
				needLower = true
				break
			}
		}
		if needLower {
			lowerCtx = make([]string, len(contextPubkeys))
			for i, p := range contextPubkeys {
				lowerCtx[i] = strings.ToLower(p)
			}
		}
		candPKs := make([]string, len(candidates))
		bestScores := make([]float64, len(candidates))
		bestCounts := make([]int, len(candidates))
		needLowerCand := false
		for i, c := range candidates {
			if hasUpperASCII(c.PublicKey) {
				needLowerCand = true
				break
			}
			candPKs[i] = c.PublicKey
		}
		if needLowerCand {
			for i, c := range candidates {
				candPKs[i] = strings.ToLower(c.PublicKey)
			}
		}
		candByPK := make(map[string]int, len(candidates))
		for i, pk := range candPKs {
			candByPK[pk] = i
		}
		for _, ctxPK := range lowerCtx {
			for _, e := range graph.Neighbors(ctxPK) {
				if e.Ambiguous {
					continue
				}
				otherPK := e.NodeA
				if otherPK == ctxPK {
					otherPK = e.NodeB
				}
				ci, ok := candByPK[otherPK]
				if !ok {
					continue
				}
				s := e.Score(now) * e.Confidence()
				if s > bestScores[ci] {
					bestScores[ci] = s
					bestCounts[ci] = e.Count
				}
			}
		}
		var scores []scored
		for i, s := range bestScores {
			if s > 0 {
				scores = append(scores, scored{i, s, bestCounts[i]})
			}
		}

		if len(scores) >= 1 {
			// Sort descending
			for i := 0; i < len(scores)-1; i++ {
				for j := i + 1; j < len(scores); j++ {
					if scores[j].score > scores[i].score {
						scores[i], scores[j] = scores[j], scores[i]
					}
				}
			}
			best := scores[0]
			// Require both score ratio ≥ 3× AND minimum observations (mirrors
			// disambiguate() in neighbor_graph.go which checks affinityMinObservations).
			if best.count >= affinityMinObservations &&
				(len(scores) == 1 || best.score >= affinityConfidenceRatio*scores[1].score) {
				return &candidates[best.idx], "neighbor_affinity", best.score
			}
			// Scores too close — fall through to lower-priority strategies
		}
	}

	// Priority 2: Geographic proximity (if context pubkeys have GPS and candidates have GPS)
	if len(contextPubkeys) > 0 {
		// Find GPS positions of context nodes from the prefix map or candidates
		// We need nodeInfo for context pubkeys — look them up
		var contextLat, contextLon float64
		var contextGPSCount int
		for _, ctxPK := range contextPubkeys {
			ctxLower := strings.ToLower(ctxPK)
			if infos, ok := pm.m[ctxLower]; ok && len(infos) == 1 && infos[0].HasGPS {
				contextLat += infos[0].Lat
				contextLon += infos[0].Lon
				contextGPSCount++
			}
		}
		if contextGPSCount > 0 {
			contextLat /= float64(contextGPSCount)
			contextLon /= float64(contextGPSCount)

			bestIdx := -1
			bestDist := math.MaxFloat64
			for i, cand := range candidates {
				if !cand.HasGPS {
					continue
				}
				d := geoDistApprox(contextLat, contextLon, cand.Lat, cand.Lon)
				if d < bestDist {
					bestDist = d
					bestIdx = i
				}
			}
			if bestIdx >= 0 {
				return &candidates[bestIdx], "geo_proximity", 0
			}
		}
	}

	// Priority 3: GPS preference. Among GPS-having candidates, prefer the one
	// with the highest observation count (recent/active evidence) rather than
	// slice/DB-insertion order. Ties on count are broken by lexicographically
	// smallest PublicKey for full determinism. See #1197.
	bestGPSIdx := -1
	for i := range candidates {
		if !candidates[i].HasGPS {
			continue
		}
		if bestGPSIdx < 0 || betterByObsCount(&candidates[i], &candidates[bestGPSIdx]) {
			bestGPSIdx = i
		}
	}
	if bestGPSIdx >= 0 {
		return &candidates[bestGPSIdx], "gps_preference", 0
	}

	// Priority 4: Fallback — pick the candidate with the highest observation
	// count (no GPS available on any candidate). Avoids slice-order
	// arbitrariness. Ties on count are broken by lexicographically smallest
	// PublicKey. Method label "observation_count_fallback" — the previous
	// "first_match" was misleading after the tier-4 algorithm changed in
	// PR #1198 (adversarial r1 #2).
	bestIdx := 0
	for i := 1; i < len(candidates); i++ {
		if betterByObsCount(&candidates[i], &candidates[bestIdx]) {
			bestIdx = i
		}
	}
	return &candidates[bestIdx], "observation_count_fallback", 0
}

// betterByObsCount reports whether candidate a should beat b under the
// tier-3/4 selection rule: higher ObservationCount wins; ties go to the
// lexicographically smaller PublicKey for determinism. Pointer receivers
// avoid value-copying nodeInfo (string + 2 floats + time.Time + int) on
// the hot resolve path. See #1197 (adversarial r1 #6, carmack r1 #4).
func betterByObsCount(a, b *nodeInfo) bool {
	if a.ObservationCount != b.ObservationCount {
		return a.ObservationCount > b.ObservationCount
	}
	return a.PublicKey < b.PublicKey
}

// geoDistApprox returns an approximate distance between two lat/lon points
// (equirectangular approximation, sufficient for relative comparison).
func geoDistApprox(lat1, lon1, lat2, lon2 float64) float64 {
	dLat := (lat2 - lat1) * math.Pi / 180
	dLon := (lon2 - lon1) * math.Pi / 180 * math.Cos((lat1+lat2)/2*math.Pi/180)
	return math.Sqrt(dLat*dLat + dLon*dLon)
}

func parsePathJSON(pathJSON string) []string {
	if pathJSON == "" || pathJSON == "[]" {
		return nil
	}
	var hops []string
	if json.Unmarshal([]byte(pathJSON), &hops) != nil {
		return nil
	}
	return hops
}

func (s *PacketStore) GetAnalyticsTopology(region string) map[string]interface{} {
	return s.GetAnalyticsTopologyWithWindow(region, TimeWindow{})
}

// GetAnalyticsTopologyWithWindow — see issue #842.
// For default (region="", zero window), prefer the steady-state
// recomputer snapshot if registered (issue #1240).
func (s *PacketStore) GetAnalyticsTopologyWithWindow(region string, window TimeWindow) map[string]interface{} {
	if region == "" && window.IsZero() {
		s.analyticsRecomputerMu.RLock()
		rc := s.recompTopology
		s.analyticsRecomputerMu.RUnlock()
		if rc != nil {
			if v := rc.Load(); v != nil {
				if m, ok := v.(map[string]interface{}); ok {
					s.cacheMu.Lock()
					s.cacheHits++
					s.cacheMu.Unlock()
					return m
				}
			}
		}
	}
	cacheKey := region
	if !window.IsZero() {
		cacheKey = region + "|" + window.CacheKey()
	}
	s.cacheMu.Lock()
	if cached, ok := s.topoCache[cacheKey]; ok && time.Now().Before(cached.expiresAt) {
		s.cacheHits++
		s.cacheMu.Unlock()
		return cached.data
	}
	s.cacheMisses++
	s.cacheMu.Unlock()

	result := s.computeAnalyticsTopology(region, window)

	s.cacheMu.Lock()
	s.topoCache[cacheKey] = &cachedResult{data: result, expiresAt: time.Now().Add(s.rfCacheTTL)}
	s.cacheMu.Unlock()

	return result
}

func (s *PacketStore) computeAnalyticsTopology(region string, window TimeWindow) map[string]interface{} {
	s.mu.RLock()
	defer s.mu.RUnlock()

	var regionObs map[string]bool
	if region != "" {
		regionObs = s.resolveRegionObservers(region)
	}

	allNodes, pm := s.getCachedNodesAndPM()
	_ = allNodes // only pm is needed for topology

	// Materialize the filtered tx slice ONCE — both the context-build pass
	// and the main aggregation pass need the same window+region predicate.
	// Two scans of s.packets re-running identical predicates is wasteful at
	// the 30k+ packet hot-path scale (#1199 item 2). One filter, two passes
	// over the result.
	filteredTxs := make([]*StoreTx, 0, len(s.packets))
	for _, tx := range s.packets {
		if !window.Includes(tx.FirstSeen) {
			continue
		}
		if regionObs != nil {
			match := false
			for _, obs := range tx.Observations {
				if regionObs[obs.ObserverID] {
					match = true
					break
				}
			}
			if !match {
				continue
			}
		}
		filteredTxs = append(filteredTxs, tx)
	}

	// Pre-pass: build the full hop-disambiguation context from all in-window
	// txs BEFORE any resolveHop call. The earlier shape — populating
	// contextPubkeys lazily during the main scan and reading it from a
	// closure — was correct only because the current code never calls
	// resolveHop inside the scan loop. A future maintainer who adds such a
	// call inside the loop would silently get partial context AND a
	// stale-cached result for any hop seen before the context grew. Two
	// explicit passes remove the hazard. See #1197 (carmack/adversarial r1).
	var contextPubkeys []string
	{
		seen := make(map[string]struct{}, 64)
		for _, tx := range filteredTxs {
			for _, pk := range buildHopContextPubkeys(tx, pm) {
				if _, ok := seen[pk]; ok {
					continue
				}
				seen[pk] = struct{}{}
				contextPubkeys = append(contextPubkeys, pk)
			}
		}
	}

	hopCache := make(map[string]*nodeInfo)
	graph := s.graph.Load() // hoist out of resolver closure (PR #1208 carmack #1)
	resolveHop := func(hop string) *nodeInfo {
		if cached, ok := hopCache[hop]; ok {
			return cached
		}
		r, _, _ := pm.resolveWithContext(hop, contextPubkeys, graph)
		hopCache[hop] = r
		return r
	}

	hopCounts := map[int]int{}
	var allHopsList []int
	hopSnr := map[int][]float64{}
	hopFreq := map[string]int{}
	pairFreq := map[string]int{}
	observerMap := map[string]string{} // observer_id → observer_name
	perObserver := map[string]map[string]*struct{ minDist, maxDist, count int }{}

	for _, tx := range filteredTxs {
		hops := txGetParsedPath(tx)
		if len(hops) == 0 {
			continue
		}

		n := len(hops)
		hopCounts[n]++
		allHopsList = append(allHopsList, n)
		if tx.SNR != nil {
			hopSnr[n] = append(hopSnr[n], *tx.SNR)
		}
		for _, h := range hops {
			hopFreq[h]++
		}
		for i := 0; i < len(hops)-1; i++ {
			a, b := hops[i], hops[i+1]
			if a > b {
				a, b = b, a
			}
			pairFreq[a+"|"+b]++
		}

		obsID := tx.ObserverID
		if obsID != "" {
			observerMap[obsID] = tx.ObserverName
		}
		if _, ok := perObserver[obsID]; !ok {
			perObserver[obsID] = map[string]*struct{ minDist, maxDist, count int }{}
		}
		for i, h := range hops {
			dist := n - i
			entry := perObserver[obsID][h]
			if entry == nil {
				entry = &struct{ minDist, maxDist, count int }{dist, dist, 0}
				perObserver[obsID][h] = entry
			}
			if dist < entry.minDist {
				entry.minDist = dist
			}
			if dist > entry.maxDist {
				entry.maxDist = dist
			}
			entry.count++
		}
	}

	// Hop distribution
	hopDist := make([]map[string]interface{}, 0)
	for h, c := range hopCounts {
		if h <= 25 {
			hopDist = append(hopDist, map[string]interface{}{"hops": h, "count": c})
		}
	}
	sort.Slice(hopDist, func(i, j int) bool {
		return hopDist[i]["hops"].(int) < hopDist[j]["hops"].(int)
	})

	avgHops := 0.0
	if len(allHopsList) > 0 {
		sum := 0
		for _, v := range allHopsList {
			sum += v
		}
		avgHops = float64(sum) / float64(len(allHopsList))
	}
	medianHops := 0
	if len(allHopsList) > 0 {
		sorted := make([]int, len(allHopsList))
		copy(sorted, allHopsList)
		sort.Ints(sorted)
		medianHops = sorted[len(sorted)/2]
	}
	maxHops := 0
	for _, v := range allHopsList {
		if v > maxHops {
			maxHops = v
		}
	}

	// pmLookup resolves a hop hex string to its prefix-map candidates,
	// applying the same truncation used during map construction.
	pmLookup := func(hop string) []nodeInfo {
		key := strings.ToLower(hop)
		if len(key) > maxPrefixLen {
			key = key[:maxPrefixLen]
		}
		return pm.m[key]
	}

	// --- Dedup pass: merge hop prefixes that resolve unambiguously to the same node ---
	// Only merge when pm.m[hop] has exactly 1 candidate (unique_prefix).
	// Ambiguous short prefixes (efiten's concern: 1-byte collisions) stay separate.
	{
		type dedupInfo struct {
			totalCount int
			longestHop string
		}
		byPubkey := map[string]*dedupInfo{} // pubkey → merged info
		ambiguous := map[string]int{}       // hop → count (kept as-is)
		for h, c := range hopFreq {
			candidates := pmLookup(h)
			if len(candidates) == 1 {
				pk := strings.ToLower(candidates[0].PublicKey)
				if info, ok := byPubkey[pk]; ok {
					info.totalCount += c
					if len(h) > len(info.longestHop) {
						info.longestHop = h
					}
				} else {
					byPubkey[pk] = &dedupInfo{totalCount: c, longestHop: h}
				}
			} else {
				ambiguous[h] = c
			}
		}
		// Rebuild hopFreq
		hopFreq = make(map[string]int, len(byPubkey)+len(ambiguous))
		for _, info := range byPubkey {
			hopFreq[info.longestHop] = info.totalCount
		}
		for h, c := range ambiguous {
			hopFreq[h] = c
		}
	}

	// --- Dedup pass for pairs: merge by resolved pubkey pair ---
	{
		type pairDedupInfo struct {
			totalCount int
			longestA   string
			longestB   string
		}
		byPubkeyPair := map[string]*pairDedupInfo{} // "pkA|pkB" (sorted) → merged info
		ambiguousPairs := map[string]int{}
		for p, c := range pairFreq {
			parts := strings.SplitN(p, "|", 2)
			candA := pmLookup(parts[0])
			candB := pmLookup(parts[1])
			if len(candA) == 1 && len(candB) == 1 {
				pkA := strings.ToLower(candA[0].PublicKey)
				pkB := strings.ToLower(candB[0].PublicKey)
				// Canonicalize by sorted pubkey
				if pkA > pkB {
					pkA, pkB = pkB, pkA
					parts[0], parts[1] = parts[1], parts[0]
				}
				key := pkA + "|" + pkB
				if info, ok := byPubkeyPair[key]; ok {
					info.totalCount += c
					if len(parts[0]) > len(info.longestA) {
						info.longestA = parts[0]
					}
					if len(parts[1]) > len(info.longestB) {
						info.longestB = parts[1]
					}
				} else {
					byPubkeyPair[key] = &pairDedupInfo{totalCount: c, longestA: parts[0], longestB: parts[1]}
				}
			} else {
				ambiguousPairs[p] = c
			}
		}
		// Rebuild pairFreq
		pairFreq = make(map[string]int, len(byPubkeyPair)+len(ambiguousPairs))
		for _, info := range byPubkeyPair {
			a, b := info.longestA, info.longestB
			if a > b {
				a, b = b, a
			}
			pairFreq[a+"|"+b] = info.totalCount
		}
		for p, c := range ambiguousPairs {
			pairFreq[p] = c
		}
	}

	// Top repeaters
	type freqEntry struct {
		hop   string
		count int
	}
	freqList := make([]freqEntry, 0, len(hopFreq))
	for h, c := range hopFreq {
		freqList = append(freqList, freqEntry{h, c})
	}
	sort.Slice(freqList, func(i, j int) bool { return freqList[i].count > freqList[j].count })
	topRepeaters := make([]map[string]interface{}, 0)
	for i, e := range freqList {
		if i >= 20 {
			break
		}
		r := resolveHop(e.hop)
		entry := map[string]interface{}{"hop": e.hop, "count": e.count, "name": nil, "pubkey": nil}
		if r != nil {
			entry["name"] = r.Name
			entry["pubkey"] = r.PublicKey
		}
		topRepeaters = append(topRepeaters, entry)
	}

	// Top pairs
	pairList := make([]freqEntry, 0, len(pairFreq))
	for p, c := range pairFreq {
		pairList = append(pairList, freqEntry{p, c})
	}
	sort.Slice(pairList, func(i, j int) bool { return pairList[i].count > pairList[j].count })
	topPairs := make([]map[string]interface{}, 0)
	for i, e := range pairList {
		if i >= 15 {
			break
		}
		parts := strings.SplitN(e.hop, "|", 2)
		rA := resolveHop(parts[0])
		rB := resolveHop(parts[1])
		entry := map[string]interface{}{
			"hopA": parts[0], "hopB": parts[1], "count": e.count,
			"nameA": nil, "nameB": nil, "pubkeyA": nil, "pubkeyB": nil,
		}
		if rA != nil {
			entry["nameA"] = rA.Name
			entry["pubkeyA"] = rA.PublicKey
		}
		if rB != nil {
			entry["nameB"] = rB.Name
			entry["pubkeyB"] = rB.PublicKey
		}
		topPairs = append(topPairs, entry)
	}

	// Hops vs SNR
	hopsVsSnr := make([]map[string]interface{}, 0)
	for h, snrs := range hopSnr {
		if h > 20 {
			continue
		}
		sum := 0.0
		for _, v := range snrs {
			sum += v
		}
		hopsVsSnr = append(hopsVsSnr, map[string]interface{}{
			"hops": h, "count": len(snrs), "avgSnr": sum / float64(len(snrs)),
		})
	}
	sort.Slice(hopsVsSnr, func(i, j int) bool {
		return hopsVsSnr[i]["hops"].(int) < hopsVsSnr[j]["hops"].(int)
	})

	// Observers list
	observers := make([]map[string]interface{}, 0)
	for id, name := range observerMap {
		n := name
		if n == "" {
			n = id
		}
		observers = append(observers, map[string]interface{}{"id": id, "name": n})
	}

	// Per-observer reachability
	perObserverReach := map[string]interface{}{}
	for obsID, nodes := range perObserver {
		obsName := observerMap[obsID]
		if obsName == "" {
			obsName = obsID
		}
		byDist := map[int][]map[string]interface{}{}
		for hop, data := range nodes {
			d := data.minDist
			if d > 15 {
				continue
			}
			r := resolveHop(hop)
			entry := map[string]interface{}{
				"hop": hop, "name": nil, "pubkey": nil,
				"count": data.count, "distRange": nil,
			}
			if r != nil {
				entry["name"] = r.Name
				entry["pubkey"] = r.PublicKey
			}
			if data.minDist != data.maxDist {
				entry["distRange"] = fmt.Sprintf("%d-%d", data.minDist, data.maxDist)
			}
			byDist[d] = append(byDist[d], entry)
		}
		rings := make([]map[string]interface{}, 0)
		for dist, nodeList := range byDist {
			sort.Slice(nodeList, func(i, j int) bool {
				return nodeList[i]["count"].(int) > nodeList[j]["count"].(int)
			})
			rings = append(rings, map[string]interface{}{"hops": dist, "nodes": nodeList})
		}
		sort.Slice(rings, func(i, j int) bool {
			return rings[i]["hops"].(int) < rings[j]["hops"].(int)
		})
		perObserverReach[obsID] = map[string]interface{}{
			"observer_name": obsName,
			"rings":         rings,
		}
	}

	// Cross-observer: build from perObserver
	crossObserver := map[string][]map[string]interface{}{}
	bestPath := map[string]map[string]interface{}{}
	for obsID, nodes := range perObserver {
		obsName := observerMap[obsID]
		if obsName == "" {
			obsName = obsID
		}
		for hop, data := range nodes {
			crossObserver[hop] = append(crossObserver[hop], map[string]interface{}{
				"observer_id": obsID, "observer_name": obsName,
				"minDist": data.minDist, "count": data.count,
			})
			if bp, ok := bestPath[hop]; !ok || data.minDist < bp["minDist"].(int) {
				bestPath[hop] = map[string]interface{}{
					"minDist": data.minDist, "observer_id": obsID, "observer_name": obsName,
				}
			}
		}
	}

	// Multi-observer nodes
	multiObsNodes := make([]map[string]interface{}, 0)
	for hop, obs := range crossObserver {
		if len(obs) <= 1 {
			continue
		}
		sort.Slice(obs, func(i, j int) bool {
			return obs[i]["minDist"].(int) < obs[j]["minDist"].(int)
		})
		r := resolveHop(hop)
		entry := map[string]interface{}{
			"hop": hop, "name": nil, "pubkey": nil, "observers": obs,
		}
		if r != nil {
			entry["name"] = r.Name
			entry["pubkey"] = r.PublicKey
		}
		multiObsNodes = append(multiObsNodes, entry)
	}
	sort.Slice(multiObsNodes, func(i, j int) bool {
		return len(multiObsNodes[i]["observers"].([]map[string]interface{})) >
			len(multiObsNodes[j]["observers"].([]map[string]interface{}))
	})
	if len(multiObsNodes) > 50 {
		multiObsNodes = multiObsNodes[:50]
	}

	// Best path list
	bestPathList := make([]map[string]interface{}, 0, len(bestPath))
	for hop, data := range bestPath {
		r := resolveHop(hop)
		entry := map[string]interface{}{
			"hop": hop, "name": nil, "pubkey": nil,
			"minDist": data["minDist"], "observer_id": data["observer_id"],
			"observer_name": data["observer_name"],
		}
		if r != nil {
			entry["name"] = r.Name
			entry["pubkey"] = r.PublicKey
		}
		bestPathList = append(bestPathList, entry)
	}
	sort.Slice(bestPathList, func(i, j int) bool {
		return bestPathList[i]["minDist"].(int) < bestPathList[j]["minDist"].(int)
	})
	if len(bestPathList) > 50 {
		bestPathList = bestPathList[:50]
	}

	// Use DB 7-day active node count (matches /api/stats totalNodes)
	uniqueNodes := 0
	if s.db != nil {
		if stats, err := s.db.GetStats(); err == nil {
			uniqueNodes = stats.TotalNodes
		}
	}

	return map[string]interface{}{
		"uniqueNodes":      uniqueNodes,
		"avgHops":          avgHops,
		"medianHops":       medianHops,
		"maxHops":          maxHops,
		"hopDistribution":  hopDist,
		"topRepeaters":     topRepeaters,
		"topPairs":         topPairs,
		"hopsVsSnr":        hopsVsSnr,
		"observers":        observers,
		"perObserverReach": perObserverReach,
		"multiObsNodes":    multiObsNodes,
		"bestPathList":     bestPathList,
	}
}

// --- Distance Analytics ---

func haversineKm(lat1, lon1, lat2, lon2 float64) float64 {
	const R = 6371.0
	dLat := (lat2 - lat1) * math.Pi / 180
	dLon := (lon2 - lon1) * math.Pi / 180
	a := math.Sin(dLat/2)*math.Sin(dLat/2) +
		math.Cos(lat1*math.Pi/180)*math.Cos(lat2*math.Pi/180)*
			math.Sin(dLon/2)*math.Sin(dLon/2)
	return R * 2 * math.Atan2(math.Sqrt(a), math.Sqrt(1-a))
}

// GetAnalyticsDistance returns the distance analytics map. For the
// default (region="") query, prefer the steady-state recomputer
// snapshot if one is registered (issue #1240). Region-keyed variants
// continue to use the legacy TTL cache + on-request compute.
func (s *PacketStore) GetAnalyticsDistance(region string) map[string]interface{} {
	if region == "" {
		s.analyticsRecomputerMu.RLock()
		rc := s.recompDistance
		s.analyticsRecomputerMu.RUnlock()
		if rc != nil {
			if v := rc.Load(); v != nil {
				if m, ok := v.(map[string]interface{}); ok {
					s.cacheMu.Lock()
					s.cacheHits++
					s.cacheMu.Unlock()
					return m
				}
			}
		}
	}
	s.cacheMu.Lock()
	if cached, ok := s.distCache[region]; ok && time.Now().Before(cached.expiresAt) {
		s.cacheHits++
		s.cacheMu.Unlock()
		return cached.data
	}
	s.cacheMisses++
	s.cacheMu.Unlock()

	result := s.computeAnalyticsDistance(region)

	s.cacheMu.Lock()
	s.distCache[region] = &cachedResult{data: result, expiresAt: time.Now().Add(s.rfCacheTTL)}
	s.cacheMu.Unlock()

	return result
}

func (s *PacketStore) computeAnalyticsDistance(region string) map[string]interface{} {
	// #1239: hold s.mu.RLock() only long enough to (a) snapshot the
	// distHops/distPaths slice headers and (b) build the region match
	// set (which reads tx.Observations, a field mutated by ingest under
	// s.mu.Lock). All filtering, sorting, deduping, histogram building
	// and category stats run on locally-captured slices OUTSIDE the
	// lock so concurrent ingest writers are not blocked by readers.
	//
	// Safety: snapshot slice headers are O(1). distHops/distPaths are
	// append-only via re-slice in buildDistanceIndex / updateDistanceIndexForTxs
	// under s.mu.Lock; if the backing array is reallocated after we
	// release the RLock, our snapshot still points at the prior backing
	// array (kept alive by GC) and observes the consistent length we
	// captured. The distHopRecord / distPathRecord values themselves
	// are value types (not pointers to live records) so we cannot read
	// torn writes from them post-release.
	var regionObs map[string]bool
	if region != "" {
		// resolveRegionObservers uses its own mutex (regionObsMu)
		// and is safe to call without s.mu held.
		regionObs = s.resolveRegionObservers(region)
	}

	s.mu.RLock()
	hopsSnap := s.distHops
	pathsSnap := s.distPaths

	// Build region match set INSIDE the lock — touches tx.Observations
	// (slice header mutated by ingest). For non-region calls (the common
	// cold path) we skip this entirely and the RLock hold is microseconds.
	var matchSet map[*StoreTx]bool
	if regionObs != nil {
		matchSet = make(map[*StoreTx]bool)
		seen := make(map[*StoreTx]bool)
		for i := range hopsSnap {
			tx := hopsSnap[i].tx
			if tx == nil || seen[tx] {
				continue
			}
			seen[tx] = true
			for _, obs := range tx.Observations {
				if regionObs[obs.ObserverID] {
					matchSet[tx] = true
					break
				}
			}
		}
		for i := range pathsSnap {
			tx := pathsSnap[i].tx
			if tx == nil || seen[tx] {
				continue
			}
			seen[tx] = true
			for _, obs := range tx.Observations {
				if regionObs[obs.ObserverID] {
					matchSet[tx] = true
					break
				}
			}
		}
	}
	s.mu.RUnlock()

	// Everything below operates on hopsSnap / pathsSnap / matchSet —
	// no s.mu, no s.distHops / s.distPaths access. Safe to run while
	// ingest writers reallocate the underlying store-owned slices.

	// Filter precomputed hop records (copy to avoid mutating precomputed data during sort)
	filteredHops := make([]distHopRecord, 0, len(hopsSnap))
	for i := range hopsSnap {
		if matchSet == nil || matchSet[hopsSnap[i].tx] {
			filteredHops = append(filteredHops, hopsSnap[i])
		}
	}

	// Filter precomputed path records
	filteredPaths := make([]distPathRecord, 0, len(pathsSnap))
	for i := range pathsSnap {
		if matchSet == nil || matchSet[pathsSnap[i].tx] {
			filteredPaths = append(filteredPaths, pathsSnap[i])
		}
	}

	// Build category stats and time series from precomputed data
	catDists := map[string][]float64{"R↔R": {}, "C↔R": {}, "C↔C": {}}
	distByHour := map[string][]float64{}
	for i := range filteredHops {
		h := &filteredHops[i]
		catDists[h.Type] = append(catDists[h.Type], h.Dist)
		if h.HourBucket != "" {
			distByHour[h.HourBucket] = append(distByHour[h.HourBucket], h.Dist)
		}
	}

	topHops := dedupeHopsByPair(filteredHops, 20)

	// Sort and pick top paths
	sort.Slice(filteredPaths, func(i, j int) bool { return filteredPaths[i].TotalDist > filteredPaths[j].TotalDist })
	topPaths := make([]map[string]interface{}, 0)
	for i := range filteredPaths {
		if i >= 20 {
			break
		}
		p := &filteredPaths[i]
		hops := make([]map[string]interface{}, len(p.Hops))
		for j, hd := range p.Hops {
			hops[j] = map[string]interface{}{
				"fromName": hd.FromName, "fromPk": hd.FromPk,
				"toName": hd.ToName, "toPk": hd.ToPk,
				"dist": hd.Dist,
			}
		}
		topPaths = append(topPaths, map[string]interface{}{
			"hash": p.Hash, "totalDist": p.TotalDist,
			"hopCount": p.HopCount, "timestamp": p.Timestamp, "hops": hops,
		})
	}

	// Category stats
	medianF := func(arr []float64) float64 {
		if len(arr) == 0 {
			return 0
		}
		c := make([]float64, len(arr))
		copy(c, arr)
		sort.Float64s(c)
		return c[len(c)/2]
	}
	minF := func(arr []float64) float64 {
		if len(arr) == 0 {
			return 0
		}
		m := arr[0]
		for _, v := range arr[1:] {
			if v < m {
				m = v
			}
		}
		return m
	}
	maxF := func(arr []float64) float64 {
		if len(arr) == 0 {
			return 0
		}
		m := arr[0]
		for _, v := range arr[1:] {
			if v > m {
				m = v
			}
		}
		return m
	}

	catStats := map[string]interface{}{}
	for cat, dists := range catDists {
		if len(dists) == 0 {
			catStats[cat] = map[string]interface{}{"count": 0, "avg": 0, "median": 0, "min": 0, "max": 0}
			continue
		}
		sum := 0.0
		for _, v := range dists {
			sum += v
		}
		avg := sum / float64(len(dists))
		catStats[cat] = map[string]interface{}{
			"count":  len(dists),
			"avg":    math.Round(avg*100) / 100,
			"median": math.Round(medianF(dists)*100) / 100,
			"min":    math.Round(minF(dists)*100) / 100,
			"max":    math.Round(maxF(dists)*100) / 100,
		}
	}

	// Distance histogram
	var distHistogram interface{} = []interface{}{}
	allDists := make([]float64, len(filteredHops))
	for i := range filteredHops {
		allDists[i] = filteredHops[i].Dist
	}
	if len(allDists) > 0 {
		hMin, hMax := minF(allDists), maxF(allDists)
		binCount := 25
		binW := (hMax - hMin) / float64(binCount)
		if binW == 0 {
			binW = 1
		}
		bins := make([]int, binCount)
		for _, d := range allDists {
			idx := int(math.Floor((d - hMin) / binW))
			if idx >= binCount {
				idx = binCount - 1
			}
			if idx < 0 {
				idx = 0
			}
			bins[idx]++
		}
		binArr := make([]map[string]interface{}, binCount)
		for i, c := range bins {
			binArr[i] = map[string]interface{}{
				"x":     math.Round((hMin+float64(i)*binW)*10) / 10,
				"w":     math.Round(binW*10) / 10,
				"count": c,
			}
		}
		distHistogram = map[string]interface{}{"bins": binArr, "min": hMin, "max": hMax}
	}

	// Distance over time
	timeKeys := make([]string, 0, len(distByHour))
	for k := range distByHour {
		timeKeys = append(timeKeys, k)
	}
	sort.Strings(timeKeys)
	distOverTime := make([]map[string]interface{}, 0, len(timeKeys))
	for _, hour := range timeKeys {
		dists := distByHour[hour]
		sum := 0.0
		for _, v := range dists {
			sum += v
		}
		distOverTime = append(distOverTime, map[string]interface{}{
			"hour":  hour,
			"avg":   math.Round(sum/float64(len(dists))*100) / 100,
			"count": len(dists),
		})
	}

	// Summary
	summary := map[string]interface{}{
		"totalHops":  len(filteredHops),
		"totalPaths": len(filteredPaths),
		"avgDist":    0.0,
		"maxDist":    0.0,
	}
	if len(allDists) > 0 {
		sum := 0.0
		for _, v := range allDists {
			sum += v
		}
		summary["avgDist"] = math.Round(sum/float64(len(allDists))*100) / 100
		summary["maxDist"] = math.Round(maxF(allDists)*100) / 100
	}

	return map[string]interface{}{
		"summary":       summary,
		"topHops":       topHops,
		"topPaths":      topPaths,
		"catStats":      catStats,
		"distHistogram": distHistogram,
		"distOverTime":  distOverTime,
	}
}

// --- Hash Sizes Analytics ---

func (s *PacketStore) GetAnalyticsHashSizes(region string) map[string]interface{} {
	if region == "" {
		s.analyticsRecomputerMu.RLock()
		rc := s.recompHashSizes
		s.analyticsRecomputerMu.RUnlock()
		if rc != nil {
			if v := rc.Load(); v != nil {
				if m, ok := v.(map[string]interface{}); ok {
					s.cacheMu.Lock()
					s.cacheHits++
					s.cacheMu.Unlock()
					return m
				}
			}
		}
	}
	s.cacheMu.Lock()
	if cached, ok := s.hashCache[region]; ok && time.Now().Before(cached.expiresAt) {
		s.cacheHits++
		s.cacheMu.Unlock()
		return cached.data
	}
	s.cacheMisses++
	s.cacheMu.Unlock()

	result := s.computeAnalyticsHashSizesWithCapability(region)

	s.cacheMu.Lock()
	s.hashCache[region] = &cachedResult{data: result, expiresAt: time.Now().Add(s.rfCacheTTL)}
	s.cacheMu.Unlock()

	return result
}

// computeAnalyticsHashSizesWithCapability runs computeAnalyticsHashSizes
// then layers in the multiByteCapability augmentation. Extracted so the
// steady-state recomputer (issue #1240) produces the same shape as the
// cached GetAnalyticsHashSizes call.
func (s *PacketStore) computeAnalyticsHashSizesWithCapability(region string) map[string]interface{} {
	result := s.computeAnalyticsHashSizes(region)
	globalAdopterHS := make(map[string]int)
	if region == "" {
		if mbNodes, ok := result["multiByteNodes"].([]map[string]interface{}); ok {
			for _, n := range mbNodes {
				pk, _ := n["pubkey"].(string)
				hs, _ := n["hashSize"].(int)
				if pk != "" && hs >= 2 {
					globalAdopterHS[pk] = hs
				}
			}
		}
	} else {
		globalRes := s.computeAnalyticsHashSizes("")
		if mbNodes, ok := globalRes["multiByteNodes"].([]map[string]interface{}); ok {
			for _, n := range mbNodes {
				pk, _ := n["pubkey"].(string)
				hs, _ := n["hashSize"].(int)
				if pk != "" && hs >= 2 {
					globalAdopterHS[pk] = hs
				}
			}
		}
	}
	result["multiByteCapability"] = s.computeMultiByteCapability(globalAdopterHS)
	return result
}

func (s *PacketStore) computeAnalyticsHashSizes(region string) map[string]interface{} {
	s.mu.RLock()
	defer s.mu.RUnlock()

	var regionObs map[string]bool
	if region != "" {
		regionObs = s.resolveRegionObservers(region)
	}

	// #804: derive each node's HOME region from zero-hop direct adverts (the
	// most authoritative location signal — those packets cannot have been
	// relayed). When non-empty, multi-byte node attribution prefers this
	// over observer-region. Falls back to observer-region when unknown.
	nodeHomeRegion := s.computeNodeHomeRegions()
	attributionMethod := "observer"
	if region != "" && len(nodeHomeRegion) > 0 {
		attributionMethod = "repeater"
	}

	allNodes, pm := s.getCachedNodesAndPM()

	// Build pubkey→role map for filtering by node type.
	nodeRoleByPK := make(map[string]string, len(allNodes))
	for _, n := range allNodes {
		nodeRoleByPK[n.PublicKey] = n.Role
	}

	distribution := map[string]int{"1": 0, "2": 0, "3": 0}
	byHour := map[string]map[string]int{}
	byNode := map[string]map[string]interface{}{}
	uniqueHops := map[string]map[string]interface{}{}
	total := 0

	for _, tx := range s.packets {
		if tx.RawHex == "" {
			continue
		}

		// Parse header and path byte
		if len(tx.RawHex) < 4 {
			continue
		}
		header, err := strconv.ParseUint(tx.RawHex[:2], 16, 8)
		if err != nil {
			continue
		}
		routeType := header & 0x03
		pathByteIdx := 1
		if routeType == 0 || routeType == 3 {
			pathByteIdx = 5
		}
		hexStart := pathByteIdx * 2
		hexEnd := hexStart + 2
		if hexEnd > len(tx.RawHex) {
			continue
		}
		actualPathByte, err := strconv.ParseUint(tx.RawHex[hexStart:hexEnd], 16, 8)
		if err != nil {
			continue
		}

		hashSize := int((actualPathByte>>6)&0x3) + 1
		if hashSize > 3 {
			continue
		}

		// #804: pre-extract originator pubkey for ADVERT packets so we can
		// (a) relax observer-region filter when the originator's HOME region
		//     matches the requested region (a flood relay heard outside the
		//     home region must still attribute to the home), and
		// (b) reuse the parsed values below without re-parsing.
		var advertPK, advertName string
		var advertParsed bool
		if tx.PayloadType != nil && *tx.PayloadType == PayloadADVERT && tx.DecodedJSON != "" {
			var d map[string]interface{}
			if json.Unmarshal([]byte(tx.DecodedJSON), &d) == nil {
				if v, ok := d["pubKey"].(string); ok {
					advertPK = v
				} else if v, ok := d["public_key"].(string); ok {
					advertPK = v
				}
				if n, ok := d["name"].(string); ok {
					advertName = n
				}
				advertParsed = advertPK != ""
			}
		}

		if regionObs != nil {
			match := false
			for _, obs := range tx.Observations {
				if regionObs[obs.ObserverID] {
					match = true
					break
				}
			}
			// #804: allow ADVERTs from a node whose HOME region matches the
			// requested region even if no observer in that region heard this
			// particular packet (e.g. flood relay heard only by an out-of-
			// region observer). Conservative: only ADVERTs (the source is
			// known by pubkey) and only when home is established.
			if !match && advertParsed {
				if home, ok := nodeHomeRegion[advertPK]; ok && iataMatchesRegion(home, region) {
					match = true
				}
			}
			if !match {
				continue
			}
		}

		// Track originator from advert packets (including zero-hop adverts,
		// keyed by pubKey so same-name nodes don't merge).
		if advertParsed {
			pk := advertPK
			name := advertName
			if name == "" {
				if len(pk) >= 8 {
					name = pk[:8]
				} else {
					name = pk
				}
			}
			// Skip zero-hop direct adverts for hash_size — the
			// path byte is locally generated and unreliable.
			// Still count the packet and update lastSeen.
			isZeroHop := (routeType == uint64(RouteDirect) || routeType == uint64(RouteTransportDirect)) && (actualPathByte&0x3F) == 0
			if byNode[pk] == nil {
				role := nodeRoleByPK[pk] // empty if unknown
				initHS := hashSize
				if isZeroHop {
					initHS = 0
				}
				byNode[pk] = map[string]interface{}{
					"hashSize": initHS, "packets": 0,
					"lastSeen": tx.FirstSeen, "name": name,
					"role": role,
				}
			}
			byNode[pk]["packets"] = byNode[pk]["packets"].(int) + 1
			if !isZeroHop {
				byNode[pk]["hashSize"] = hashSize
			}
			byNode[pk]["lastSeen"] = tx.FirstSeen
		}

		// Distribution/hourly/uniqueHops only for packets with relay hops
		hops := txGetParsedPath(tx)
		if len(hops) == 0 {
			continue
		}
		total++

		sizeKey := strconv.Itoa(hashSize)
		distribution[sizeKey]++

		// Hourly buckets
		if len(tx.FirstSeen) >= 13 {
			hour := tx.FirstSeen[:13]
			if byHour[hour] == nil {
				byHour[hour] = map[string]int{"1": 0, "2": 0, "3": 0}
			}
			byHour[hour][sizeKey]++
		}

		// Track unique hops with their sizes
		for _, hop := range hops {
			if uniqueHops[hop] == nil {
				hopLower := strings.ToLower(hop)
				candidates := pm.m[hopLower]
				var matchName, matchPk interface{}
				if len(candidates) > 0 {
					matchName = candidates[0].Name
					matchPk = candidates[0].PublicKey
				}
				uniqueHops[hop] = map[string]interface{}{
					"size": (len(hop) + 1) / 2, "count": 0,
					"name": matchName, "pubkey": matchPk,
				}
			}
			uniqueHops[hop]["count"] = uniqueHops[hop]["count"].(int) + 1
		}
	}

	// Sort hourly data
	hourKeys := make([]string, 0, len(byHour))
	for k := range byHour {
		hourKeys = append(hourKeys, k)
	}
	sort.Strings(hourKeys)
	hourly := make([]map[string]interface{}, 0, len(hourKeys))
	for _, hour := range hourKeys {
		sizes := byHour[hour]
		hourly = append(hourly, map[string]interface{}{
			"hour": hour, "1": sizes["1"], "2": sizes["2"], "3": sizes["3"],
		})
	}

	// Top hops by frequency
	type hopEntry struct {
		hex  string
		data map[string]interface{}
	}
	hopList := make([]hopEntry, 0, len(uniqueHops))
	for hex, data := range uniqueHops {
		hopList = append(hopList, hopEntry{hex, data})
	}
	sort.Slice(hopList, func(i, j int) bool {
		return hopList[i].data["count"].(int) > hopList[j].data["count"].(int)
	})
	topHops := make([]map[string]interface{}, 0)
	for i, e := range hopList {
		if i >= 50 {
			break
		}
		topHops = append(topHops, map[string]interface{}{
			"hex": e.hex, "size": e.data["size"], "count": e.data["count"],
			"name": e.data["name"], "pubkey": e.data["pubkey"],
		})
	}

	// Multi-byte nodes
	multiByteNodes := make([]map[string]interface{}, 0)
	for pk, data := range byNode {
		// #804: when a region filter is active, prefer the repeater's HOME
		// region over the observer that happened to relay it. Falls back to
		// the (already-applied) observer-region filter when the node's home
		// region is unknown.
		if region != "" {
			if home, ok := nodeHomeRegion[pk]; ok && !iataMatchesRegion(home, region) {
				continue
			}
		}
		if data["hashSize"].(int) > 1 {
			multiByteNodes = append(multiByteNodes, map[string]interface{}{
				"name": data["name"], "hashSize": data["hashSize"],
				"packets": data["packets"], "lastSeen": data["lastSeen"],
				"pubkey": pk, "role": data["role"],
			})
		}
	}
	sort.Slice(multiByteNodes, func(i, j int) bool {
		return multiByteNodes[i]["packets"].(int) > multiByteNodes[j]["packets"].(int)
	})

	// Distribution by repeaters: count unique REPEATER nodes per hash size
	distributionByRepeaters := map[string]int{"1": 0, "2": 0, "3": 0}
	for pk, data := range byNode {
		role, _ := data["role"].(string)
		if !strings.Contains(strings.ToLower(role), "repeater") {
			continue
		}
		// #804: same repeater-region preference as multiByteNodes.
		if region != "" {
			if home, ok := nodeHomeRegion[pk]; ok && !iataMatchesRegion(home, region) {
				continue
			}
		}
		hs := data["hashSize"].(int)
		key := strconv.Itoa(hs)
		distributionByRepeaters[key]++
	}

	return map[string]interface{}{
		"total":                   total,
		"distribution":            distribution,
		"distributionByRepeaters": distributionByRepeaters,
		"hourly":                  hourly,
		"topHops":                 topHops,
		"multiByteNodes":          multiByteNodes,
		"attributionMethod":       attributionMethod,
	}
}

// hashSizeNodeInfo holds per-node hash size tracking data.
type hashSizeNodeInfo struct {
	HashSize     int
	AllSizes     map[int]bool
	Seq          []int
	Inconsistent bool
}

// --- Hash Collision Analytics ---

// GetAnalyticsHashCollisions returns pre-computed hash collision analysis.
// This moves the O(n²) distance computation from the frontend to the server.
func (s *PacketStore) GetAnalyticsHashCollisions(region string) map[string]interface{} {
	if region == "" {
		s.analyticsRecomputerMu.RLock()
		rc := s.recompHashCollisions
		s.analyticsRecomputerMu.RUnlock()
		if rc != nil {
			if v := rc.Load(); v != nil {
				if m, ok := v.(map[string]interface{}); ok {
					s.cacheMu.Lock()
					s.cacheHits++
					s.cacheMu.Unlock()
					return m
				}
			}
		}
	}
	s.cacheMu.Lock()
	if cached, ok := s.collisionCache[region]; ok && time.Now().Before(cached.expiresAt) {
		s.cacheHits++
		s.cacheMu.Unlock()
		return cached.data
	}
	s.cacheMisses++
	s.cacheMu.Unlock()

	result := s.computeHashCollisions(region)

	s.cacheMu.Lock()
	s.collisionCache[region] = &cachedResult{data: result, expiresAt: time.Now().Add(s.collisionCacheTTL)}
	s.cacheMu.Unlock()

	return result
}

// collisionNode is a lightweight node representation for collision analysis.
type collisionNode struct {
	PublicKey          string `json:"public_key"`
	Name               string `json:"name"`
	Role               string `json:"role"`
	Lat                float64 `json:"lat"`
	Lon                float64 `json:"lon"`
	HashSize           int    `json:"hash_size"`
	HashSizeInconsistent bool `json:"hash_size_inconsistent"`
	HashSizesSeen      []int  `json:"hash_sizes_seen,omitempty"`
}

// collisionEntry represents a prefix collision with pre-computed distances.
type collisionEntry struct {
	Prefix         string          `json:"prefix"`
	ByteSize       int             `json:"byte_size"`
	Appearances    int             `json:"appearances"`
	Nodes          []collisionNode `json:"nodes"`
	MaxDistKm      float64         `json:"max_dist_km"`
	Classification string          `json:"classification"`
	WithCoords     int             `json:"with_coords"`
}

// prefixCellInfo holds per-prefix-cell data for the matrix view.
type prefixCellInfo struct {
	Nodes []collisionNode `json:"nodes"`
}

// twoByteCellInfo holds per-first-byte-group data for 2-byte matrix.
type twoByteCellInfo struct {
	GroupNodes      []collisionNode            `json:"group_nodes"`
	TwoByteMap      map[string][]collisionNode `json:"two_byte_map"`
	MaxCollision    int                         `json:"max_collision"`
	CollisionCount  int                         `json:"collision_count"`
}

func (s *PacketStore) computeHashCollisions(region string) map[string]interface{} {
	// Get all nodes from DB
	nodes := s.getAllNodes()
	hashInfo := s.GetNodeHashSizeInfo()

	// If region is specified, filter to only nodes seen by regional observers
	if region != "" {
		regionObs := s.resolveRegionObservers(region)
		if regionObs != nil {
			s.mu.RLock()
			regionNodePKs := make(map[string]bool)
			for _, tx := range s.packets {
				match := false
				for _, obs := range tx.Observations {
					if regionObs[obs.ObserverID] {
						match = true
						break
					}
				}
				if !match {
					continue
				}
				// Collect node public keys from advert packets
				if tx.DecodedJSON != "" {
					var d map[string]interface{}
					if json.Unmarshal([]byte(tx.DecodedJSON), &d) == nil {
						if pk, ok := d["pubKey"].(string); ok && pk != "" {
							regionNodePKs[pk] = true
						}
						if pk, ok := d["public_key"].(string); ok && pk != "" {
							regionNodePKs[pk] = true
						}
					}
				}
				// Include observers themselves as nodes in the region
				for _, obs := range tx.Observations {
					if obs.ObserverID != "" {
						regionNodePKs[obs.ObserverID] = true
					}
				}
			}
			s.mu.RUnlock()

			// Filter nodes to only those seen in the region
			filtered := make([]nodeInfo, 0, len(regionNodePKs))
			for _, n := range nodes {
				if regionNodePKs[n.PublicKey] {
					filtered = append(filtered, n)
				}
			}
			nodes = filtered
		}
	}

	// Build collision nodes with hash info
	var allCNodes []collisionNode
	for _, n := range nodes {
		cn := collisionNode{
			PublicKey: n.PublicKey,
			Name:      n.Name,
			Role:      n.Role,
			Lat:       n.Lat,
			Lon:       n.Lon,
		}
		if info, ok := hashInfo[n.PublicKey]; ok && info != nil {
			cn.HashSize = info.HashSize
			cn.HashSizeInconsistent = info.Inconsistent
			if len(info.AllSizes) > 1 {
				sizes := make([]int, 0, len(info.AllSizes))
				for sz := range info.AllSizes {
					sizes = append(sizes, sz)
				}
				sort.Ints(sizes)
				cn.HashSizesSeen = sizes
			}
		}
		allCNodes = append(allCNodes, cn)
	}

	// Inconsistent nodes
	var inconsistentNodes []collisionNode
	for _, cn := range allCNodes {
		if cn.HashSizeInconsistent && (cn.Role == "repeater" || cn.Role == "room_server") {
			inconsistentNodes = append(inconsistentNodes, cn)
		}
	}
	if inconsistentNodes == nil {
		inconsistentNodes = make([]collisionNode, 0)
	}

	// Compute collisions for each byte size (1, 2, 3)
	collisionsBySize := make(map[string]interface{})
	for _, bytes := range []int{1, 2, 3} {
		// Filter nodes relevant to this byte size.
		// - Exclude hash_size==0 nodes: no adverts seen, so actual hash
		//   size is unknown. Including them in every bucket inflates
		//   collision counts.
		// - Exclude companions: they are mobile/temporary and don't form
		//   the mesh backbone, so collisions with them aren't meaningful.
		// (Fixes #441)
		var nodesForByte []collisionNode
		for _, cn := range allCNodes {
			if cn.HashSize == bytes && cn.Role == "repeater" {
				nodesForByte = append(nodesForByte, cn)
			}
		}

		// Build prefix map
		prefixMap := make(map[string][]collisionNode)
		for _, cn := range nodesForByte {
			if len(cn.PublicKey) < bytes*2 {
				continue
			}
			prefix := strings.ToUpper(cn.PublicKey[:bytes*2])
			prefixMap[prefix] = append(prefixMap[prefix], cn)
		}

		// Compute collisions with pairwise distances
		var collisions []collisionEntry
		for prefix, pnodes := range prefixMap {
			if len(pnodes) <= 1 {
				continue
			}
			// Pairwise distance
			var withCoords []collisionNode
			for _, cn := range pnodes {
				if cn.Lat != 0 || cn.Lon != 0 {
					withCoords = append(withCoords, cn)
				}
			}
			var maxDistKm float64
			classification := "unknown"
			if len(withCoords) >= 2 {
				for i := 0; i < len(withCoords); i++ {
					for j := i + 1; j < len(withCoords); j++ {
						d := haversineKm(withCoords[i].Lat, withCoords[i].Lon, withCoords[j].Lat, withCoords[j].Lon)
						if d > maxDistKm {
							maxDistKm = d
						}
					}
				}
				if maxDistKm < 50 {
					classification = "local"
				} else if maxDistKm < 200 {
					classification = "regional"
				} else {
					classification = "distant"
				}
			} else {
				classification = "incomplete"
			}
			collisions = append(collisions, collisionEntry{
				Prefix:         prefix,
				ByteSize:       bytes,
				Appearances:    len(pnodes),
				Nodes:          pnodes,
				MaxDistKm:      maxDistKm,
				Classification: classification,
				WithCoords:     len(withCoords),
			})
		}
		if collisions == nil {
			collisions = make([]collisionEntry, 0)
		}

		// Sort: local first, then regional, distant, incomplete
		classOrder := map[string]int{"local": 0, "regional": 1, "distant": 2, "incomplete": 3, "unknown": 4}
		sort.Slice(collisions, func(i, j int) bool {
			oi, oj := classOrder[collisions[i].Classification], classOrder[collisions[j].Classification]
			if oi != oj {
				return oi < oj
			}
			return collisions[i].Appearances > collisions[j].Appearances
		})

		// Stats
		nodeCount := len(nodesForByte)
		usingThisSize := 0
		for _, cn := range allCNodes {
			if cn.HashSize == bytes {
				usingThisSize++
			}
		}
		uniquePrefixes := len(prefixMap)
		collisionCount := len(collisions)
		var spaceSize int
		switch bytes {
		case 1:
			spaceSize = 256
		case 2:
			spaceSize = 65536
		case 3:
			spaceSize = 16777216
		}
		pctUsed := 0.0
		if spaceSize > 0 {
			pctUsed = float64(uniquePrefixes) / float64(spaceSize) * 100
		}

		// For 1-byte and 2-byte, include the full prefix cell data for matrix rendering
		var oneByteCells map[string][]collisionNode
		var twoByteCells map[string]*twoByteCellInfo
		if bytes == 1 {
			oneByteCells = make(map[string][]collisionNode)
			for i := 0; i < 256; i++ {
				hex := strings.ToUpper(fmt.Sprintf("%02x", i))
				oneByteCells[hex] = prefixMap[hex]
				if oneByteCells[hex] == nil {
					oneByteCells[hex] = make([]collisionNode, 0)
				}
			}
		} else if bytes == 2 {
			twoByteCells = make(map[string]*twoByteCellInfo)
			for i := 0; i < 256; i++ {
				hex := strings.ToUpper(fmt.Sprintf("%02x", i))
				cell := &twoByteCellInfo{
					GroupNodes: make([]collisionNode, 0),
					TwoByteMap: make(map[string][]collisionNode),
				}
				twoByteCells[hex] = cell
			}
			for _, cn := range nodesForByte {
				if len(cn.PublicKey) < 4 {
					continue
				}
				firstHex := strings.ToUpper(cn.PublicKey[:2])
				twoHex := strings.ToUpper(cn.PublicKey[:4])
				cell := twoByteCells[firstHex]
				if cell == nil {
					continue
				}
				cell.GroupNodes = append(cell.GroupNodes, cn)
				cell.TwoByteMap[twoHex] = append(cell.TwoByteMap[twoHex], cn)
			}
			for _, cell := range twoByteCells {
				for _, ns := range cell.TwoByteMap {
					if len(ns) > 1 {
						cell.CollisionCount++
						if len(ns) > cell.MaxCollision {
							cell.MaxCollision = len(ns)
						}
					}
				}
			}
		}

		sizeData := map[string]interface{}{
			"stats": map[string]interface{}{
				"total_nodes":     len(allCNodes),
				"nodes_for_byte":  nodeCount,
				"using_this_size": usingThisSize,
				"unique_prefixes": uniquePrefixes,
				"collision_count": collisionCount,
				"space_size":      spaceSize,
				"pct_used":        pctUsed,
			},
			"collisions": collisions,
		}
		if oneByteCells != nil {
			sizeData["one_byte_cells"] = oneByteCells
		}
		if twoByteCells != nil {
			sizeData["two_byte_cells"] = twoByteCells
		}
		collisionsBySize[strconv.Itoa(bytes)] = sizeData
	}

	return map[string]interface{}{
		"inconsistent_nodes": inconsistentNodes,
		"by_size":            collisionsBySize,
	}
}

// GetNodeHashSizeInfo returns cached per-node hash size data, recomputing at most every 15s.
func (s *PacketStore) GetNodeHashSizeInfo() map[string]*hashSizeNodeInfo {
	const ttl = 15 * time.Second
	s.hashSizeInfoMu.Lock()
	if s.hashSizeInfoCache != nil && time.Since(s.hashSizeInfoAt) < ttl {
		cached := s.hashSizeInfoCache
		s.hashSizeInfoMu.Unlock()
		return cached
	}
	s.hashSizeInfoMu.Unlock()
	result := s.computeNodeHashSizeInfo()
	s.hashSizeInfoMu.Lock()
	s.hashSizeInfoCache = result
	s.hashSizeInfoAt = time.Now()
	s.hashSizeInfoMu.Unlock()
	return result
}

// computeNodeHashSizeInfo scans advert packets to compute per-node hash size data.
// Only adverts from the last 7 days are considered so that legitimate config
// changes during testing don't create permanent false positives.
func (s *PacketStore) computeNodeHashSizeInfo() map[string]*hashSizeNodeInfo {
	s.mu.RLock()
	defer s.mu.RUnlock()

	info := make(map[string]*hashSizeNodeInfo)

	cutoff := time.Now().UTC().Add(-7 * 24 * time.Hour).Format("2006-01-02T15:04:05.000Z")

	adverts := s.byPayloadType[4]
	for _, tx := range adverts {
		// Skip adverts older than 7 days to avoid false positives from
		// historical config changes during testing.
		if tx.FirstSeen != "" && tx.FirstSeen < cutoff {
			continue
		}
		if tx.RawHex == "" || tx.DecodedJSON == "" {
			continue
		}
		if len(tx.RawHex) < 4 {
			continue
		}
		header, err := strconv.ParseUint(tx.RawHex[:2], 16, 8)
		if err != nil {
			continue
		}
		routeType := int(header & 0x03)
		// Transport routes (0, 3) have 4 transport code bytes before the path
		// byte, so the path byte is at offset 5 instead of 1.
		pbOffset := 1
		if routeType == RouteTransportFlood || routeType == RouteTransportDirect {
			pbOffset = 5
		}
		if len(tx.RawHex) < (pbOffset+1)*2 {
			continue
		}
		pathByte, err := strconv.ParseUint(tx.RawHex[pbOffset*2:pbOffset*2+2], 16, 8)
		if err != nil {
			continue
		}
		// Direct zero-hop adverts (route types 2 and 3) use path byte 0x00
		// locally and can misreport multibyte hash mode as 1-byte.
		if (routeType == RouteDirect || routeType == RouteTransportDirect) && (pathByte&0x3F) == 0 {
			continue
		}
		hs := int((pathByte>>6)&0x3) + 1

		var d map[string]interface{}
		if json.Unmarshal([]byte(tx.DecodedJSON), &d) != nil {
			continue
		}
		pk := ""
		if v, ok := d["pubKey"].(string); ok {
			pk = v
		} else if v, ok := d["public_key"].(string); ok {
			pk = v
		}
		if pk == "" {
			continue
		}

		ni := info[pk]
		if ni == nil {
			ni = &hashSizeNodeInfo{AllSizes: make(map[int]bool)}
			info[pk] = ni
		}
		ni.AllSizes[hs] = true
		ni.Seq = append(ni.Seq, hs)
	}

	// Post-process: use latest advert hash size and compute flip-flop flag.
	// The most recent advert reflects the node's current hash size
	// configuration. The upstream firmware bug causing stale path bytes in
	// flood adverts was fixed (meshcore-dev/MeshCore#2154).
	for _, ni := range info {
		// Use the most recent advert's hash size (last in chronological order).
		ni.HashSize = ni.Seq[len(ni.Seq)-1]

		// Flip-flop (inconsistent) flag: need >= 3 observations,
		// >= 2 unique sizes, and >= 2 transitions in the sequence.
		if len(ni.Seq) < 3 || len(ni.AllSizes) < 2 {
			continue
		}
		transitions := 0
		for i := 1; i < len(ni.Seq); i++ {
			if ni.Seq[i] != ni.Seq[i-1] {
				transitions++
			}
		}
		ni.Inconsistent = transitions >= 2
	}

	return info
}

// EnrichNodeWithHashSize populates hash_size, hash_size_inconsistent, and
// hash_sizes_seen on a node map using precomputed hash size info.
func EnrichNodeWithHashSize(node map[string]interface{}, info *hashSizeNodeInfo) {
	if info == nil {
		return
	}
	node["hash_size"] = info.HashSize
	node["hash_size_inconsistent"] = info.Inconsistent
	if len(info.AllSizes) > 1 {
		sizes := make([]int, 0, len(info.AllSizes))
		for s := range info.AllSizes {
			sizes = append(sizes, s)
		}
		sort.Ints(sizes)
		node["hash_sizes_seen"] = sizes
	}
}

// EnrichNodeWithMultiByte adds multi-byte capability fields to a node map.
func EnrichNodeWithMultiByte(node map[string]interface{}, entry *MultiByteCapEntry) {
	if entry == nil {
		return
	}
	node["multi_byte_status"] = entry.Status
	node["multi_byte_evidence"] = entry.Evidence
	node["multi_byte_max_hash_size"] = entry.MaxHashSize
}

// GetMultiByteCapMap returns a cached pubkey → MultiByteCapEntry map.
// Reuses the same 15s TTL cache pattern as hash size info.
func (s *PacketStore) GetMultiByteCapMap() map[string]*MultiByteCapEntry {
	s.hashSizeInfoMu.Lock()
	if s.multiByteCapCache != nil && time.Since(s.multiByteCapAt) < 15*time.Second {
		cached := s.multiByteCapCache
		s.hashSizeInfoMu.Unlock()
		return cached
	}
	s.hashSizeInfoMu.Unlock()

	// Get adopter hash sizes from analytics for cross-referencing
	analyticsData := s.GetAnalyticsHashSizes("")
	adopterSizes := make(map[string]int)
	if nodes, ok := analyticsData["nodes"].(map[string]map[string]interface{}); ok {
		for pk, data := range nodes {
			if hs, ok := data["hashSize"].(int); ok {
				adopterSizes[pk] = hs
			}
		}
	}

	caps := s.computeMultiByteCapability(adopterSizes)
	result := make(map[string]*MultiByteCapEntry, len(caps))
	for i := range caps {
		result[caps[i].PublicKey] = &caps[i]
	}

	s.hashSizeInfoMu.Lock()
	s.multiByteCapCache = result
	s.multiByteCapAt = time.Now()
	s.hashSizeInfoMu.Unlock()
	return result
}

// --- Multi-Byte Capability Inference ---

// MultiByteCapEntry represents a node's inferred multi-byte capability.
type MultiByteCapEntry struct {
	PublicKey   string `json:"pubkey"`
	Name        string `json:"name"`
	Role        string `json:"role"`
	Status      string `json:"status"`      // "confirmed", "suspected", "unknown"
	Evidence    string `json:"evidence"`     // "advert", "path", ""
	MaxHashSize int    `json:"maxHashSize"`
	LastSeen    string `json:"lastSeen"`
}

// computeMultiByteCapability determines multi-byte capability for each
// node (repeaters, companions, rooms, sensors) using two methods:
//
// 1. Confirmed: the node has advertised with hash_size >= 2 (from advert
//    path byte). This is 100% reliable because the full public key is
//    received in adverts — no prefix collision ambiguity.
//
// 2. Suspected: the node's prefix appears as a hop in a packet whose path
//    header indicates hash_size >= 2. This is <100% reliable because
//    2-byte prefixes can collide — two different nodes may share the same
//    prefix. If one is confirmed multi-byte and the other is not, the
//    non-confirmed one could be a false positive.
//
// 3. Unknown: node has only been seen with 1-byte adverts and no
//    multi-byte path appearances. Could be pre-1.14 firmware or 1.14+
//    with default (1-byte) settings.
//
// Caller must hold NO locks — this method acquires mu.RLock internally.
func (s *PacketStore) computeMultiByteCapability(adopterHashSizes map[string]int) []MultiByteCapEntry {
	// Get hash size info from adverts (has its own locking)
	hashInfo := s.GetNodeHashSizeInfo()

	// Get all nodes for name/role lookup
	allNodes := s.getAllNodes()
	nodeByPK := make(map[string]nodeInfo, len(allNodes))
	for _, n := range allNodes {
		nodeByPK[n.PublicKey] = n
	}

	// Build set of confirmed multi-byte pubkeys (advert hash_size >= 2)
	confirmed := make(map[string]int) // pubkey → max hash size from adverts
	for pk, info := range hashInfo {
		maxHS := 1
		for sz := range info.AllSizes {
			if sz > maxHS {
				maxHS = sz
			}
		}
		if maxHS >= 2 {
			confirmed[pk] = maxHS
		}
	}

	// Scan path-hop index for suspected multi-byte nodes.
	// For each repeater, check if any packet in byPathHop has that
	// node as a hop with hash_size >= 2 in the path header.
	s.mu.RLock()

	// Build prefix→pubkey mapping for repeaters
	type prefixEntry struct {
		pubkey string
		prefix string
	}
	nodePrefixes := make(map[string][]prefixEntry) // prefix → entries
	for pk := range nodeByPK {
		// Generate 1-byte, 2-byte, 3-byte prefixes
		pkLower := strings.ToLower(pk)
		for byteLen := 1; byteLen <= 3; byteLen++ {
			hexLen := byteLen * 2
			if len(pkLower) >= hexLen {
				pfx := pkLower[:hexLen]
				nodePrefixes[pfx] = append(nodePrefixes[pfx], prefixEntry{pk, pfx})
			}
		}
	}

	suspected := make(map[string]int) // pubkey → max hash size from path appearances
	for pfx, entries := range nodePrefixes {
		txList := s.byPathHop[pfx]
		for _, tx := range txList {
			if tx.RawHex == "" || len(tx.RawHex) < 4 {
				continue
			}
			// Skip TRACE packets (payload_type 8) — they carry hash size in
			// TRACE flags, not the repeater's compile-time PATH_HASH_SIZE.
			// Pre-1.14 repeaters can forward multi-byte TRACEs, creating
			// false positives for "suspected" capability. See #714.
			if tx.PayloadType != nil && *tx.PayloadType == 8 {
				continue
			}
			header, err := strconv.ParseUint(tx.RawHex[:2], 16, 8)
			if err != nil {
				continue
			}
			routeType := header & 0x03
			pathByteIdx := 1
			if routeType == 0 || routeType == 3 {
				pathByteIdx = 5
			}
			hexStart := pathByteIdx * 2
			hexEnd := hexStart + 2
			if hexEnd > len(tx.RawHex) {
				continue
			}
			actualPathByte, err := strconv.ParseUint(tx.RawHex[hexStart:hexEnd], 16, 8)
			if err != nil {
				continue
			}
			hs := int((actualPathByte>>6)&0x3) + 1
			if hs < 2 {
				continue
			}
			// This packet uses multi-byte hashes and contains this prefix as a hop
			for _, e := range entries {
				if hs > suspected[e.pubkey] {
					suspected[e.pubkey] = hs
				}
			}
			break // one match is enough per prefix
		}
	}
	s.mu.RUnlock()

	// Build result for all nodes — fetch last_seen from DB
	dbLastSeen := make(map[string]string)
	rows, err := s.db.conn.Query("SELECT public_key, last_seen FROM nodes")
	if err == nil {
		defer rows.Close()
		for rows.Next() {
			var pk string
			var ls sql.NullString
			rows.Scan(&pk, &ls)
			if ls.Valid {
				dbLastSeen[pk] = ls.String
			}
		}
	}

	var result []MultiByteCapEntry
	for pk, n := range nodeByPK {
		entry := MultiByteCapEntry{
			PublicKey:   pk,
			Name:        n.Name,
			Role:        n.Role,
			MaxHashSize: 1,
			LastSeen:    dbLastSeen[pk],
		}

		if maxHS, ok := confirmed[pk]; ok {
			entry.Status = "confirmed"
			entry.Evidence = "advert"
			entry.MaxHashSize = maxHS
		} else if maxHS, ok := adopterHashSizes[pk]; ok && maxHS >= 2 {
			// Adopter data (from computeAnalyticsHashSizes) shows hash_size >= 2
			// from advert analysis — this is advert-based evidence, so confirmed.
			entry.Status = "confirmed"
			entry.Evidence = "advert"
			entry.MaxHashSize = maxHS
		} else if maxHS, ok := suspected[pk]; ok {
			entry.Status = "suspected"
			entry.Evidence = "path"
			entry.MaxHashSize = maxHS
		} else {
			entry.Status = "unknown"
		}

		// Check advert hash info for max even if not confirmed multi-byte
		if info, ok := hashInfo[pk]; ok && entry.MaxHashSize == 1 {
			for sz := range info.AllSizes {
				if sz > entry.MaxHashSize {
					entry.MaxHashSize = sz
				}
			}
		}

		result = append(result, entry)
	}

	// Sort: confirmed first, then suspected, then unknown; within each group by name
	statusOrder := map[string]int{"confirmed": 0, "suspected": 1, "unknown": 2}
	sort.Slice(result, func(i, j int) bool {
		oi, oj := statusOrder[result[i].Status], statusOrder[result[j].Status]
		if oi != oj {
			return oi < oj
		}
		return strings.ToLower(result[i].Name) < strings.ToLower(result[j].Name)
	})

	return result
}

// --- Bulk Health (in-memory) ---

func (s *PacketStore) GetBulkHealth(limit int, region string) []map[string]interface{} {
	s.mu.RLock()
	defer s.mu.RUnlock()

	// Region filtering
	var regionNodeKeys map[string]bool
	if region != "" {
		regionObs := s.resolveRegionObservers(region)
		if regionObs != nil {
			regionalHashes := make(map[string]bool)
			for obsID := range regionObs {
				obsList := s.byObserver[obsID]
				for _, o := range obsList {
					tx := s.byTxID[o.TransmissionID]
					if tx != nil {
						regionalHashes[tx.Hash] = true
					}
				}
			}
			regionNodeKeys = make(map[string]bool)
			for pk, hashes := range s.nodeHashes {
				for h := range hashes {
					if regionalHashes[h] {
						regionNodeKeys[pk] = true
						break
					}
				}
			}
		}
	}

	// Get nodes from DB
	queryLimit := limit
	if regionNodeKeys != nil {
		queryLimit = 500
	}
	rows, err := s.db.conn.Query("SELECT public_key, name, role, lat, lon FROM nodes ORDER BY last_seen DESC LIMIT ?", queryLimit)
	if err != nil {
		return []map[string]interface{}{}
	}
	defer rows.Close()

	type dbNode struct {
		pk, name, role string
		lat, lon       interface{}
	}
	var nodes []dbNode
	for rows.Next() {
		var pk string
		var name, role sql.NullString
		var lat, lon sql.NullFloat64
		rows.Scan(&pk, &name, &role, &lat, &lon)
		if regionNodeKeys != nil && !regionNodeKeys[pk] {
			continue
		}
		nodes = append(nodes, dbNode{
			pk: pk, name: nullStrVal(name), role: nullStrVal(role),
			lat: nullFloat(lat), lon: nullFloat(lon),
		})
		if regionNodeKeys == nil && len(nodes) >= limit {
			break
		}
	}
	if regionNodeKeys != nil && len(nodes) > limit {
		nodes = nodes[:limit]
	}

	todayStart := time.Now().UTC().Truncate(24 * time.Hour).Format(time.RFC3339)
	results := make([]map[string]interface{}, 0, len(nodes))

	for _, n := range nodes {
		packets := s.byNode[n.pk]
		var packetsToday int
		var snrSum float64
		var snrCount int
		var lastHeard string
		observerStats := map[string]*struct {
			name                       string
			snrSum, rssiSum            float64
			snrCount, rssiCount, count int
		}{}
		totalObservations := 0

		for _, pkt := range packets {
			totalObservations += pkt.ObservationCount
			if totalObservations == 0 {
				totalObservations = 1
			}
			if pkt.FirstSeen > todayStart {
				packetsToday++
			}
			if pkt.SNR != nil {
				snrSum += *pkt.SNR
				snrCount++
			}
			if lastHeard == "" || pkt.FirstSeen > lastHeard {
				lastHeard = pkt.FirstSeen
			}
			obsID := pkt.ObserverID
			if obsID != "" {
				obs := observerStats[obsID]
				if obs == nil {
					obs = &struct {
						name                       string
						snrSum, rssiSum            float64
						snrCount, rssiCount, count int
					}{name: pkt.ObserverName}
					observerStats[obsID] = obs
				}
				obs.count++
				if pkt.SNR != nil {
					obs.snrSum += *pkt.SNR
					obs.snrCount++
				}
				if pkt.RSSI != nil {
					obs.rssiSum += *pkt.RSSI
					obs.rssiCount++
				}
			}
		}

		observerRows := make([]map[string]interface{}, 0)
		for id, o := range observerStats {
			var avgSnr, avgRssi interface{}
			if o.snrCount > 0 {
				avgSnr = o.snrSum / float64(o.snrCount)
			}
			if o.rssiCount > 0 {
				avgRssi = o.rssiSum / float64(o.rssiCount)
			}
			observerRows = append(observerRows, map[string]interface{}{
				"observer_id": id, "observer_name": o.name,
				"avgSnr": avgSnr, "avgRssi": avgRssi, "packetCount": o.count,
			})
		}
		sort.Slice(observerRows, func(i, j int) bool {
			return observerRows[i]["packetCount"].(int) > observerRows[j]["packetCount"].(int)
		})

		var avgSnr interface{}
		if snrCount > 0 {
			avgSnr = snrSum / float64(snrCount)
		}
		var lhVal interface{}
		if lastHeard != "" {
			lhVal = lastHeard
		}

		results = append(results, map[string]interface{}{
			"public_key": n.pk,
			"name":       nilIfEmpty(n.name),
			"role":       nilIfEmpty(n.role),
			"lat":        n.lat,
			"lon":        n.lon,
			"stats": map[string]interface{}{
				"totalTransmissions": len(packets),
				"totalObservations":  totalObservations,
				"totalPackets":       len(packets),
				"packetsToday":       packetsToday,
				"avgSnr":             avgSnr,
				"lastHeard":          lhVal,
			},
			"observers": observerRows,
		})
	}

	return results
}

// --- Subpaths Analytics ---

// GetNodeHealth returns health info for a single node using in-memory data.
func (s *PacketStore) GetNodeHealth(pubkey string) (map[string]interface{}, error) {
	// Fetch node info from DB (fast single-row lookup)
	node, err := s.db.GetNodeByPubkey(pubkey)
	if err != nil {
		return nil, err
	}
	// If the node isn't in the DB (e.g. companion that never advertised),
	// check if we have any packet data for it. If so, build a partial response.
	if node == nil {
		s.mu.RLock()
		hasPackets := len(s.byNode[pubkey]) > 0
		s.mu.RUnlock()
		if !hasPackets {
			return nil, nil
		}
		// Build a synthetic node stub so the rest of the function works
		node = map[string]interface{}{
			"public_key": pubkey,
			"name":       "Unknown",
			"role":       "unknown",
		}
	}

	s.mu.RLock()
	defer s.mu.RUnlock()

	packets := s.byNode[pubkey]
	todayStart := time.Now().UTC().Truncate(24 * time.Hour).Format(time.RFC3339)

	var packetsToday int
	var snrSum float64
	var snrCount int
	var totalHops, hopCount int
	var lastHeard string
	totalObservations := 0

	observerStats := map[string]*struct {
		name                       string
		snrSum, rssiSum            float64
		snrCount, rssiCount, count int
	}{}

	for _, pkt := range packets {
		totalObservations += pkt.ObservationCount
		if pkt.FirstSeen > todayStart {
			packetsToday++
		}
		if pkt.SNR != nil {
			snrSum += *pkt.SNR
			snrCount++
		}
		if lastHeard == "" || pkt.FirstSeen > lastHeard {
			lastHeard = pkt.FirstSeen
		}
		// Hop counting
		hops := txGetParsedPath(pkt)
		if len(hops) > 0 {
			totalHops += len(hops)
			hopCount++
		}
		// Observer stats
		obsID := pkt.ObserverID
		if obsID != "" {
			obs := observerStats[obsID]
			if obs == nil {
				obs = &struct {
					name                       string
					snrSum, rssiSum            float64
					snrCount, rssiCount, count int
				}{name: pkt.ObserverName}
				observerStats[obsID] = obs
			}
			obs.count++
			if pkt.SNR != nil {
				obs.snrSum += *pkt.SNR
				obs.snrCount++
			}
			if pkt.RSSI != nil {
				obs.rssiSum += *pkt.RSSI
				obs.rssiCount++
			}
		}
	}

	observerRows := make([]map[string]interface{}, 0)
	for id, o := range observerStats {
		var avgSnr, avgRssi interface{}
		if o.snrCount > 0 {
			avgSnr = o.snrSum / float64(o.snrCount)
		}
		if o.rssiCount > 0 {
			avgRssi = o.rssiSum / float64(o.rssiCount)
		}
		observerRows = append(observerRows, map[string]interface{}{
			"observer_id": id, "observer_name": o.name,
			"avgSnr": avgSnr, "avgRssi": avgRssi, "packetCount": o.count,
		})
	}
	sort.Slice(observerRows, func(i, j int) bool {
		return observerRows[i]["packetCount"].(int) > observerRows[j]["packetCount"].(int)
	})

	var avgSnr interface{}
	if snrCount > 0 {
		avgSnr = snrSum / float64(snrCount)
	}
	avgHops := 0
	if hopCount > 0 {
		avgHops = int(math.Round(float64(totalHops) / float64(hopCount)))
	}
	var lhVal interface{}
	if lastHeard != "" {
		lhVal = lastHeard
	}

	// Recent packets (up to 20, newest first — read from tail of oldest-first slice)
	recentLimit := 20
	if len(packets) < recentLimit {
		recentLimit = len(packets)
	}
	recentPackets := make([]map[string]interface{}, 0, recentLimit)
	for i := len(packets) - 1; i >= len(packets)-recentLimit; i-- {
		p := s.txToMapWithRP(packets[i])
		delete(p, "observations")
		recentPackets = append(recentPackets, p)
	}

	return map[string]interface{}{
		"node":      node,
		"observers": observerRows,
		"stats": map[string]interface{}{
			"totalTransmissions": len(packets),
			"totalObservations":  totalObservations,
			"totalPackets":       len(packets),
			"packetsToday":       packetsToday,
			"avgSnr":             avgSnr,
			"avgHops":            avgHops,
			"lastHeard":          lhVal,
		},
		"recentPackets": recentPackets,
	}, nil
}

// GetNodeAnalytics computes analytics for a single node using in-memory byNode index.
func (s *PacketStore) GetNodeAnalytics(pubkey string, days int) (*NodeAnalyticsResponse, error) {
	node, err := s.db.GetNodeByPubkey(pubkey)
	if err != nil || node == nil {
		return nil, err
	}

	fromTime := time.Now().Add(-time.Duration(days) * 24 * time.Hour)
	fromISO := fromTime.Format(time.RFC3339)
	toISO := time.Now().Format(time.RFC3339)

	s.mu.RLock()
	defer s.mu.RUnlock()

	// Collect packets from byNode index (time-filtered).
	// Raw JSON text search is intentionally avoided: a GRP_TXT packet whose message
	// text contains a node's pubkey is not a packet *for* that node.
	indexed := s.byNode[pubkey]
	var packets []*StoreTx
	for _, p := range indexed {
		if p.FirstSeen > fromISO {
			packets = append(packets, p)
		}
	}

	// Activity timeline (hourly buckets)
	timelineBuckets := map[string]int{}
	for _, p := range packets {
		if len(p.FirstSeen) >= 13 {
			bucket := p.FirstSeen[:13] + ":00:00Z"
			timelineBuckets[bucket]++
		}
	}
	bucketKeys := make([]string, 0, len(timelineBuckets))
	for k := range timelineBuckets {
		bucketKeys = append(bucketKeys, k)
	}
	sort.Strings(bucketKeys)
	activityTimeline := make([]TimeBucket, 0, len(bucketKeys))
	for _, k := range bucketKeys {
		b := k
		activityTimeline = append(activityTimeline, TimeBucket{Bucket: &b, Count: timelineBuckets[k]})
	}

	// SNR trend
	snrTrend := make([]SnrTrendEntry, 0)
	for _, p := range packets {
		if p.SNR != nil {
			snrTrend = append(snrTrend, SnrTrendEntry{
				Timestamp:    p.FirstSeen,
				SNR:          floatPtrOrNil(p.SNR),
				RSSI:         floatPtrOrNil(p.RSSI),
				ObserverID:   strOrNil(p.ObserverID),
				ObserverName: strOrNil(p.ObserverName),
			})
		}
	}

	// Packet type breakdown
	typeBuckets := map[int]int{}
	for _, p := range packets {
		if p.PayloadType != nil {
			typeBuckets[*p.PayloadType]++
		}
	}
	packetTypeBreakdown := make([]PayloadTypeCount, 0, len(typeBuckets))
	for pt, cnt := range typeBuckets {
		packetTypeBreakdown = append(packetTypeBreakdown, PayloadTypeCount{PayloadType: pt, Count: cnt})
	}

	// Observer coverage
	type obsAccum struct {
		name                       string
		snrSum, rssiSum            float64
		snrCount, rssiCount, count int
		first, last                string
	}
	obsMap := map[string]*obsAccum{}
	for _, p := range packets {
		if p.ObserverID == "" {
			continue
		}
		o := obsMap[p.ObserverID]
		if o == nil {
			o = &obsAccum{name: p.ObserverName, first: p.FirstSeen, last: p.FirstSeen}
			obsMap[p.ObserverID] = o
		}
		o.count++
		if p.SNR != nil {
			o.snrSum += *p.SNR
			o.snrCount++
		}
		if p.RSSI != nil {
			o.rssiSum += *p.RSSI
			o.rssiCount++
		}
		if p.FirstSeen < o.first {
			o.first = p.FirstSeen
		}
		if p.FirstSeen > o.last {
			o.last = p.FirstSeen
		}
	}
	observerCoverage := make([]NodeObserverStatsResp, 0, len(obsMap))
	for id, o := range obsMap {
		var avgSnr, avgRssi interface{}
		if o.snrCount > 0 {
			avgSnr = o.snrSum / float64(o.snrCount)
		}
		if o.rssiCount > 0 {
			avgRssi = o.rssiSum / float64(o.rssiCount)
		}
		observerCoverage = append(observerCoverage, NodeObserverStatsResp{
			ObserverID:   id,
			ObserverName: o.name,
			PacketCount:  o.count,
			AvgSnr:       avgSnr,
			AvgRssi:      avgRssi,
			FirstSeen:    o.first,
			LastSeen:     o.last,
		})
	}
	sort.Slice(observerCoverage, func(i, j int) bool {
		return observerCoverage[i].PacketCount > observerCoverage[j].PacketCount
	})

	// Hop distribution
	hopCounts := map[string]int{}
	totalWithPath := 0
	relayedCount := 0
	for _, p := range packets {
		hops := txGetParsedPath(p)
		if len(hops) > 0 {
			key := fmt.Sprintf("%d", len(hops))
			if len(hops) >= 4 {
				key = "4+"
			}
			hopCounts[key]++
			totalWithPath++
			if len(hops) > 1 {
				relayedCount++
			}
		} else {
			hopCounts["0"]++
		}
	}
	hopDistribution := make([]HopDistEntry, 0)
	for _, h := range []string{"0", "1", "2", "3", "4+"} {
		if c, ok := hopCounts[h]; ok {
			hopDistribution = append(hopDistribution, HopDistEntry{Hops: h, Count: c})
		}
	}

	// Peer interactions
	type peerAccum struct {
		key, name   string
		count       int
		lastContact string
	}
	peerMap := map[string]*peerAccum{}
	for _, p := range packets {
		if p.DecodedJSON == "" {
			continue
		}
		var decoded map[string]interface{}
		if json.Unmarshal([]byte(p.DecodedJSON), &decoded) != nil {
			continue
		}
		type candidate struct{ key, name string }
		var candidates []candidate
		if sk, ok := decoded["sender_key"].(string); ok && sk != "" && sk != pubkey {
			sn, _ := decoded["sender_name"].(string)
			if sn == "" {
				sn, _ = decoded["sender_short_name"].(string)
			}
			candidates = append(candidates, candidate{sk, sn})
		}
		if rk, ok := decoded["recipient_key"].(string); ok && rk != "" && rk != pubkey {
			rn, _ := decoded["recipient_name"].(string)
			if rn == "" {
				rn, _ = decoded["recipient_short_name"].(string)
			}
			candidates = append(candidates, candidate{rk, rn})
		}
		if pk, ok := decoded["pubkey"].(string); ok && pk != "" && pk != pubkey {
			nm, _ := decoded["name"].(string)
			candidates = append(candidates, candidate{pk, nm})
		}
		for _, c := range candidates {
			if c.key == "" {
				continue
			}
			pm := peerMap[c.key]
			if pm == nil {
				pn := c.name
				if pn == "" && len(c.key) >= 12 {
					pn = c.key[:12]
				}
				pm = &peerAccum{key: c.key, name: pn, lastContact: p.FirstSeen}
				peerMap[c.key] = pm
			}
			pm.count++
			if p.FirstSeen > pm.lastContact {
				pm.lastContact = p.FirstSeen
			}
		}
	}
	peerSlice := make([]PeerInteraction, 0, len(peerMap))
	for _, pm := range peerMap {
		peerSlice = append(peerSlice, PeerInteraction{
			PeerKey: pm.key, PeerName: pm.name,
			MessageCount: pm.count, LastContact: pm.lastContact,
		})
	}
	sort.Slice(peerSlice, func(i, j int) bool {
		return peerSlice[i].MessageCount > peerSlice[j].MessageCount
	})
	if len(peerSlice) > 20 {
		peerSlice = peerSlice[:20]
	}

	// Uptime heatmap
	heatBuckets := map[string]*HeatmapCell{}
	for _, p := range packets {
		t, err := time.Parse(time.RFC3339, p.FirstSeen)
		if err != nil {
			t, err = time.Parse("2006-01-02 15:04:05", p.FirstSeen)
			if err != nil {
				continue
			}
		}
		dow := int(t.UTC().Weekday())
		hr := t.UTC().Hour()
		k := fmt.Sprintf("%d:%d", dow, hr)
		if heatBuckets[k] == nil {
			heatBuckets[k] = &HeatmapCell{DayOfWeek: dow, Hour: hr}
		}
		heatBuckets[k].Count++
	}
	uptimeHeatmap := make([]HeatmapCell, 0, len(heatBuckets))
	for _, cell := range heatBuckets {
		uptimeHeatmap = append(uptimeHeatmap, *cell)
	}

	// Computed stats
	totalPackets := len(packets)
	distinctHours := len(activityTimeline)
	totalHours := float64(days) * 24
	availabilityPct := 0.0
	if totalHours > 0 {
		availabilityPct = round(float64(distinctHours)*100.0/totalHours, 1)
		if availabilityPct > 100 {
			availabilityPct = 100
		}
	}

	var avgPacketsPerDay float64
	if days > 0 {
		avgPacketsPerDay = round(float64(totalPackets)/float64(days), 1)
	}

	// Longest silence
	var longestSilenceMs int
	var longestSilenceStart interface{}
	if len(activityTimeline) >= 2 {
		for i := 1; i < len(activityTimeline); i++ {
			var t1Str, t2Str string
			if activityTimeline[i-1].Bucket != nil {
				t1Str = *activityTimeline[i-1].Bucket
			}
			if activityTimeline[i].Bucket != nil {
				t2Str = *activityTimeline[i].Bucket
			}
			t1, e1 := time.Parse(time.RFC3339, t1Str)
			t2, e2 := time.Parse(time.RFC3339, t2Str)
			if e1 == nil && e2 == nil {
				gap := int(t2.Sub(t1).Milliseconds())
				if gap > longestSilenceMs {
					longestSilenceMs = gap
					longestSilenceStart = t1Str
				}
			}
		}
	}

	// Signal grade & SNR stats
	var snrMean, snrStdDev float64
	if len(snrTrend) > 0 {
		var sum float64
		for _, e := range snrTrend {
			if v, ok := e.SNR.(float64); ok {
				sum += v
			}
		}
		snrMean = sum / float64(len(snrTrend))
		if len(snrTrend) > 1 {
			var sqSum float64
			for _, e := range snrTrend {
				if v, ok := e.SNR.(float64); ok {
					sqSum += (v - snrMean) * (v - snrMean)
				}
			}
			snrStdDev = math.Sqrt(sqSum / float64(len(snrTrend)))
		}
	}

	signalGrade := "D"
	if snrMean > 15 && snrStdDev < 2 {
		signalGrade = "A"
	} else if snrMean > 15 {
		signalGrade = "A-"
	} else if snrMean > 12 && snrStdDev < 3 {
		signalGrade = "B+"
	} else if snrMean > 8 {
		signalGrade = "B"
	} else if snrMean > 3 {
		signalGrade = "C"
	}

	var relayPct float64
	if totalWithPath > 0 {
		relayPct = round(float64(relayedCount)*100.0/float64(totalWithPath), 1)
	}

	// Compute clock skew (already under RLock).
	clockSkew := s.getNodeClockSkewLocked(pubkey)

	return &NodeAnalyticsResponse{
		Node:                node,
		TimeRange:           TimeRangeResp{From: fromISO, To: toISO, Days: days},
		ActivityTimeline:    activityTimeline,
		SnrTrend:            snrTrend,
		PacketTypeBreakdown: packetTypeBreakdown,
		ObserverCoverage:    observerCoverage,
		HopDistribution:     hopDistribution,
		PeerInteractions:    peerSlice,
		UptimeHeatmap:       uptimeHeatmap,
		ComputedStats: ComputedNodeStats{
			AvailabilityPct:     availabilityPct,
			LongestSilenceMs:    longestSilenceMs,
			LongestSilenceStart: longestSilenceStart,
			SignalGrade:         signalGrade,
			SnrMean:             round(snrMean, 1),
			SnrStdDev:           round(snrStdDev, 1),
			RelayPct:            relayPct,
			TotalPackets:        totalPackets,
			UniqueObservers:     len(observerCoverage),
			UniquePeers:         len(peerSlice),
			AvgPacketsPerDay:    avgPacketsPerDay,
		},
		ClockSkew: clockSkew,
	}, nil
}

func (s *PacketStore) GetAnalyticsSubpaths(region string, minLen, maxLen, limit int) map[string]interface{} {
	cacheKey := fmt.Sprintf("%s|%d|%d|%d", region, minLen, maxLen, limit)

	s.cacheMu.Lock()
	if cached, ok := s.subpathCache[cacheKey]; ok && time.Now().Before(cached.expiresAt) {
		s.cacheHits++
		s.cacheMu.Unlock()
		return cached.data
	}
	s.cacheMisses++
	s.cacheMu.Unlock()

	result := s.computeAnalyticsSubpaths(region, minLen, maxLen, limit)

	s.cacheMu.Lock()
	s.subpathCache[cacheKey] = &cachedResult{data: result, expiresAt: time.Now().Add(s.rfCacheTTL)}
	s.cacheMu.Unlock()

	return result
}

// GetAnalyticsSubpathsBulk returns multiple length-range buckets from a single
// scan of the subpath index, avoiding repeated iterations.
func (s *PacketStore) GetAnalyticsSubpathsBulk(region string, groups []subpathGroup) []map[string]interface{} {
	// For region queries or when there are few groups, fall back to individual calls
	// which benefit from per-key caching.
	if region != "" {
		results := make([]map[string]interface{}, len(groups))
		for i, g := range groups {
			results[i] = s.GetAnalyticsSubpaths(region, g.MinLen, g.MaxLen, g.Limit)
		}
		return results
	}

	// Check if all groups are cached.
	allCached := true
	cachedResults := make([]map[string]interface{}, len(groups))
	s.cacheMu.Lock()
	for i, g := range groups {
		cacheKey := fmt.Sprintf("|%d|%d|%d", g.MinLen, g.MaxLen, g.Limit)
		if cached, ok := s.subpathCache[cacheKey]; ok && time.Now().Before(cached.expiresAt) {
			cachedResults[i] = cached.data
		} else {
			allCached = false
			break
		}
	}
	if allCached {
		s.cacheHits += int64(len(groups))
		s.cacheMu.Unlock()
		return cachedResults
	}
	s.cacheMu.Unlock()

	// Single scan: bucket by hop length into per-group accumulators.
	s.mu.RLock()
	_, pm := s.getCachedNodesAndPM()
	// Aggregate hop-disambiguation context across all packets so the
	// resolver's tiers 1 and 2 light up even on this bulk-aggregate path
	// (the index iterates raw subpath strings, not per-tx). See #1197.
	contextPubkeys := buildAggregateHopContextPubkeys(s.packets, pm)
	hopCache := make(map[string]*nodeInfo)
	graph := s.graph.Load() // hoist out of resolver closure (PR #1208 carmack #1)
	resolveHop := func(hop string) string {
		if cached, ok := hopCache[hop]; ok {
			if cached != nil {
				return cached.Name
			}
			return hop
		}
		r, _, _ := pm.resolveWithContext(hop, contextPubkeys, graph)
		hopCache[hop] = r
		if r != nil {
			return r.Name
		}
		return hop
	}

	perGroup := make([]map[string]*subpathAccum, len(groups))
	for i := range groups {
		perGroup[i] = make(map[string]*subpathAccum)
	}

	for rawKey, count := range s.spIndex {
		hops := strings.Split(rawKey, ",")
		hopLen := len(hops)

		// Resolve hop names once, reuse across groups.
		var named []string
		var namedKey string
		resolved := false

		for gi, g := range groups {
			if hopLen < g.MinLen || hopLen > g.MaxLen {
				continue
			}
			if !resolved {
				named = make([]string, hopLen)
				for i, h := range hops {
					named[i] = resolveHop(h)
				}
				namedKey = strings.Join(named, " → ")
				resolved = true
			}
			entry := perGroup[gi][namedKey]
			if entry == nil {
				entry = &subpathAccum{raw: rawKey}
				perGroup[gi][namedKey] = entry
			}
			entry.count += count
		}
	}
	totalPaths := s.spTotalPaths
	s.mu.RUnlock()

	results := make([]map[string]interface{}, len(groups))
	for i, g := range groups {
		results[i] = s.rankSubpaths(perGroup[i], totalPaths, g.Limit)
	}

	// Cache individual results for future single-key lookups too.
	s.cacheMu.Lock()
	for i, g := range groups {
		cacheKey := fmt.Sprintf("|%d|%d|%d", g.MinLen, g.MaxLen, g.Limit)
		s.subpathCache[cacheKey] = &cachedResult{data: results[i], expiresAt: time.Now().Add(s.rfCacheTTL)}
	}
	s.cacheMu.Unlock()

	return results
}

// subpathAccum holds a running count for a single named subpath.
type subpathAccum struct {
	count int
	raw   string // first raw-hop key seen (used for rawHops in the API response)
}

func (s *PacketStore) computeAnalyticsSubpaths(region string, minLen, maxLen, limit int) map[string]interface{} {
	s.mu.RLock()
	defer s.mu.RUnlock()

	_, pm := s.getCachedNodesAndPM()
	// Aggregate hop-disambiguation context across all packets — bulk
	// aggregator over s.spIndex / per-tx fallback both need it. See #1197.
	contextPubkeys := buildAggregateHopContextPubkeys(s.packets, pm)
	hopCache := make(map[string]*nodeInfo)
	graph := s.graph.Load() // hoist out of resolver closure (PR #1208 carmack #1)
	resolveHop := func(hop string) string {
		if cached, ok := hopCache[hop]; ok {
			if cached != nil {
				return cached.Name
			}
			return hop
		}
		r, _, _ := pm.resolveWithContext(hop, contextPubkeys, graph)
		hopCache[hop] = r
		if r != nil {
			return r.Name
		}
		return hop
	}

	// For region queries fall back to packet iteration (region filtering
	// requires per-transmission observer checks).
	if region != "" {
		return s.computeSubpathsSlow(region, minLen, maxLen, limit, resolveHop)
	}

	// Fast path: read from precomputed raw-hop subpath index.
	// Resolve raw hop prefixes to names and merge counts.
	namedCounts := make(map[string]*subpathAccum, len(s.spIndex))
	for rawKey, count := range s.spIndex {
		hops := strings.Split(rawKey, ",")
		hopLen := len(hops)
		if hopLen < minLen || hopLen > maxLen {
			continue
		}
		named := make([]string, hopLen)
		for i, h := range hops {
			named[i] = resolveHop(h)
		}
		namedKey := strings.Join(named, " → ")
		entry := namedCounts[namedKey]
		if entry == nil {
			entry = &subpathAccum{raw: rawKey}
			namedCounts[namedKey] = entry
		}
		entry.count += count
	}

	return s.rankSubpaths(namedCounts, s.spTotalPaths, limit)
}

// computeSubpathsSlow is the original O(N) packet-iteration path, used only
// for region-filtered queries where we must check per-transmission observers.
func (s *PacketStore) computeSubpathsSlow(region string, minLen, maxLen, limit int, resolveHop func(string) string) map[string]interface{} {
	regionObs := s.resolveRegionObservers(region)

	subpathCounts := make(map[string]*subpathAccum)
	totalPaths := 0

	for _, tx := range s.packets {
		hops := txGetParsedPath(tx)
		if len(hops) < 2 {
			continue
		}
		if regionObs != nil {
			match := false
			for _, obs := range tx.Observations {
				if regionObs[obs.ObserverID] {
					match = true
					break
				}
			}
			if !match {
				continue
			}
		}
		totalPaths++

		named := make([]string, len(hops))
		for i, h := range hops {
			named[i] = resolveHop(h)
		}

		for l := minLen; l <= maxLen && l <= len(named); l++ {
			for start := 0; start <= len(named)-l; start++ {
				sub := strings.Join(named[start:start+l], " → ")
				raw := strings.Join(hops[start:start+l], ",")
				entry := subpathCounts[sub]
				if entry == nil {
					entry = &subpathAccum{raw: raw}
					subpathCounts[sub] = entry
				}
				entry.count++
			}
		}
	}

	return s.rankSubpaths(subpathCounts, totalPaths, limit)
}

// rankSubpaths sorts accumulated subpath counts by frequency, truncates to
// limit, and builds the API response map.
func (s *PacketStore) rankSubpaths(counts map[string]*subpathAccum, totalPaths, limit int) map[string]interface{} {
	type subpathEntry struct {
		path  string
		count int
		raw   string
	}
	ranked := make([]subpathEntry, 0, len(counts))
	for path, data := range counts {
		ranked = append(ranked, subpathEntry{path, data.count, data.raw})
	}
	sort.Slice(ranked, func(i, j int) bool { return ranked[i].count > ranked[j].count })
	if len(ranked) > limit {
		ranked = ranked[:limit]
	}

	subpaths := make([]map[string]interface{}, 0, len(ranked))
	for _, e := range ranked {
		pct := 0.0
		if totalPaths > 0 {
			pct = math.Round(float64(e.count)/float64(totalPaths)*1000) / 10
		}
		subpaths = append(subpaths, map[string]interface{}{
			"path":    e.path,
			"rawHops": strings.Split(e.raw, ","),
			"count":   e.count,
			"hops":    len(strings.Split(e.path, " → ")),
			"pct":     pct,
		})
	}

	return map[string]interface{}{
		"subpaths":   subpaths,
		"totalPaths": totalPaths,
	}
}

// --- Subpath Detail ---

func (s *PacketStore) GetSubpathDetail(rawHops []string) map[string]interface{} {
	s.mu.RLock()
	defer s.mu.RUnlock()

	_, pm := s.getCachedNodesAndPM()

	// Build the subpath key the same way the index does (lowercase, comma-joined)
	spKey := strings.ToLower(strings.Join(rawHops, ","))

	// Direct lookup instead of scanning all packets
	matchedTxs := s.spTxIndex[spKey]

	// Hop-disambiguation context: union over the matched txs that produced
	// this subpath. This is the right scope — those are the packets that
	// witnessed the requested hop sequence. See #1197.
	contextPubkeys := buildAggregateHopContextPubkeys(matchedTxs, pm)
	// Hoist atomic graph.Load() once for the whole request (PR #1208
	// carmack #1) — used by the rawHops loop AND the matched-tx loop below.
	graph := s.graph.Load()

	// Resolve the requested hops
	nodes := make([]map[string]interface{}, len(rawHops))
	for i, hop := range rawHops {
		r, _, _ := pm.resolveWithContext(hop, contextPubkeys, graph)
		entry := map[string]interface{}{"hop": hop, "name": hop, "lat": nil, "lon": nil, "pubkey": nil}
		if r != nil {
			entry["name"] = r.Name
			entry["pubkey"] = r.PublicKey
			if r.HasGPS {
				entry["lat"] = r.Lat
				entry["lon"] = r.Lon
			}
		}
		nodes[i] = entry
	}

	hourBuckets := make([]int, 24)
	var snrSum, rssiSum float64
	var snrCount, rssiCount int
	observers := map[string]int{}
	parentPaths := map[string]int{}
	matchCount := len(matchedTxs)
	var firstSeen, lastSeen string

	for _, tx := range matchedTxs {
		ts := tx.FirstSeen
		if ts != "" {
			if firstSeen == "" || ts < firstSeen {
				firstSeen = ts
			}
			if lastSeen == "" || ts > lastSeen {
				lastSeen = ts
			}
			t, err := time.Parse(time.RFC3339, ts)
			if err != nil {
				t, err = time.Parse("2006-01-02 15:04:05", ts)
			}
			if err == nil {
				hourBuckets[t.Hour()]++
			}
		}
		if tx.SNR != nil {
			snrSum += *tx.SNR
			snrCount++
		}
		if tx.RSSI != nil {
			rssiSum += *tx.RSSI
			rssiCount++
		}
		if tx.ObserverName != "" {
			observers[tx.ObserverName]++
		}

		// Full parent path (resolved). Per-tx context so the resolver picks
		// the right candidate when prefixes are ambiguous. See #1197.
		txCtx := buildHopContextPubkeys(tx, pm)
		hops := txGetParsedPath(tx)
		resolved := make([]string, len(hops))
		for i, h := range hops {
			r, _, _ := pm.resolveWithContext(h, txCtx, graph)
			if r != nil {
				resolved[i] = r.Name
			} else {
				resolved[i] = h
			}
		}
		fullPath := strings.Join(resolved, " → ")
		parentPaths[fullPath]++
	}

	var avgSnr, avgRssi interface{}
	if snrCount > 0 {
		avgSnr = snrSum / float64(snrCount)
	}
	if rssiCount > 0 {
		avgRssi = rssiSum / float64(rssiCount)
	}

	topParents := make([]map[string]interface{}, 0)
	for path, count := range parentPaths {
		topParents = append(topParents, map[string]interface{}{"path": path, "count": count})
	}
	sort.Slice(topParents, func(i, j int) bool {
		return topParents[i]["count"].(int) > topParents[j]["count"].(int)
	})
	if len(topParents) > 15 {
		topParents = topParents[:15]
	}

	topObs := make([]map[string]interface{}, 0)
	for name, count := range observers {
		topObs = append(topObs, map[string]interface{}{"name": name, "count": count})
	}
	sort.Slice(topObs, func(i, j int) bool {
		return topObs[i]["count"].(int) > topObs[j]["count"].(int)
	})
	if len(topObs) > 10 {
		topObs = topObs[:10]
	}

	return map[string]interface{}{
		"hops":             rawHops,
		"nodes":            nodes,
		"totalMatches":     matchCount,
		"firstSeen":        firstSeen,
		"lastSeen":         lastSeen,
		"signal":           map[string]interface{}{"avgSnr": avgSnr, "avgRssi": avgRssi, "samples": snrCount},
		"hourDistribution": hourBuckets,
		"parentPaths":      topParents,
		"observers":        topObs,
	}
}