Compare commits

..
25 changed files with 399 additions and 1831 deletions
+2 -2
View File
@@ -236,7 +236,7 @@ jobs:
build:
name: "🏗️ Build Docker Image"
needs: [e2e-test]
runs-on: [self-hosted, meshcore-vm]
runs-on: [self-hosted, Linux]
steps:
- name: Checkout code
uses: actions/checkout@v5
@@ -271,7 +271,7 @@ jobs:
name: "🚀 Deploy Staging"
if: github.event_name == 'push'
needs: [build]
runs-on: [self-hosted, meshcore-vm]
runs-on: [self-hosted, Linux]
steps:
- name: Checkout code
uses: actions/checkout@v5
-42
View File
@@ -3913,45 +3913,3 @@ func TestBuildTransmissionWhereMultiObserver(t *testing.T) {
}
})
}
// --- Distance index rebuild debounce (#557) ---
func TestDistanceRebuildDebounce(t *testing.T) {
db := setupTestDB(t)
defer db.Close()
seedTestData(t, db)
store := NewPacketStore(db, nil)
store.Load()
// After Load(), distLast is set to now — so distDirty should be false
if store.distDirty {
t.Fatal("distDirty should be false after Load()")
}
// Insert a new observation with a different path to trigger distDirty
maxObsID := db.GetMaxObservationID()
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
VALUES (1, 2, 5.0, -100, '["xx","yy","zz"]', ?)`, time.Now().Unix())
store.IngestNewObservations(maxObsID, 500)
// distDirty should be true (30s hasn't elapsed since Load)
if !store.distDirty {
t.Fatal("distDirty should be true after path change within 30s window")
}
// Now simulate 30s having elapsed by backdating distLast
store.distLast = time.Now().Add(-31 * time.Second)
// Insert another observation to trigger another ingest cycle
maxObsID = db.GetMaxObservationID()
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
VALUES (1, 2, 7.0, -95, '["aa","bb","cc","dd"]', ?)`, time.Now().Unix())
store.IngestNewObservations(maxObsID, 500)
// After 30s elapsed, distDirty should be cleared (rebuild happened)
if store.distDirty {
t.Fatal("distDirty should be false after rebuild (30s elapsed)")
}
}
+6 -11
View File
@@ -15,10 +15,9 @@ import (
// DB wraps a read-only connection to the MeshCore SQLite database.
type DB struct {
conn *sql.DB
path string // filesystem path to the database file
isV3 bool // v3 schema: observer_idx in observations (vs observer_id in v2)
hasResolvedPath bool // observations table has resolved_path column
conn *sql.DB
path string // filesystem path to the database file
isV3 bool // v3 schema: observer_idx in observations (vs observer_id in v2)
}
// OpenDB opens a read-only SQLite connection with WAL mode.
@@ -62,13 +61,9 @@ func (db *DB) detectSchema() {
var colType sql.NullString
var notNull, pk int
var dflt sql.NullString
if rows.Scan(&cid, &colName, &colType, &notNull, &dflt, &pk) == nil {
if colName == "observer_idx" {
db.isV3 = true
}
if colName == "resolved_path" {
db.hasResolvedPath = true
}
if rows.Scan(&cid, &colName, &colType, &notNull, &dflt, &pk) == nil && colName == "observer_idx" {
db.isV3 = true
return
}
}
}
-44
View File
@@ -144,50 +144,6 @@ func main() {
log.Fatalf("[store] failed to load: %v", err)
}
// Initialize persisted neighbor graph
dbPath = database.path
if err := ensureNeighborEdgesTable(dbPath); err != nil {
log.Printf("[neighbor] warning: could not create neighbor_edges table: %v", err)
}
// Add resolved_path column if missing.
// NOTE on startup ordering (review item #10): ensureResolvedPathColumn runs AFTER
// OpenDB/detectSchema, so db.hasResolvedPath will be false on first run with a
// pre-existing DB. This means Load() won't SELECT resolved_path from SQLite.
// That's OK: backfillResolvedPaths (below) computes and persists them in-memory
// AND to SQLite. On next restart, detectSchema finds the column and Load() reads it.
if err := ensureResolvedPathColumn(dbPath); err != nil {
log.Printf("[store] warning: could not add resolved_path column: %v", err)
} else {
database.hasResolvedPath = true // detectSchema ran before column was added; fix the flag
}
// Load or build neighbor graph
if neighborEdgesTableExists(database.conn) {
store.graph = loadNeighborEdgesFromDB(database.conn)
log.Printf("[neighbor] loaded persisted neighbor graph")
} else {
log.Printf("[neighbor] no persisted edges found, building from store...")
rw, rwErr := openRW(dbPath)
if rwErr == nil {
edgeCount := buildAndPersistEdges(store, rw)
rw.Close()
log.Printf("[neighbor] persisted %d edges", edgeCount)
}
store.graph = BuildFromStore(store)
}
// Backfill resolved_path for observations that don't have it yet
if backfilled := backfillResolvedPaths(store, dbPath); backfilled > 0 {
log.Printf("[store] backfilled resolved_path for %d observations", backfilled)
}
// Re-pick best observation now that resolved paths are populated
store.mu.Lock()
for _, tx := range store.packets {
pickBestObservation(tx)
}
store.mu.Unlock()
// WebSocket hub
hub := NewHub()
+30 -35
View File
@@ -18,7 +18,7 @@ const (
// Time-decay half-life: 7 days.
affinityHalfLifeHours = 168.0
// Cache TTL for the built graph.
neighborGraphTTL = 5 * time.Minute
neighborGraphTTL = 60 * time.Second
// Auto-resolve confidence: best must be >= this factor × second-best.
affinityConfidenceRatio = 3.0
// Minimum observation count to auto-resolve.
@@ -130,17 +130,6 @@ func BuildFromStore(store *PacketStore) *NeighborGraph {
return BuildFromStoreWithLog(store, false)
}
// cachedToLower returns strings.ToLower(s), caching results to avoid
// repeated allocations for the same pubkey string.
func cachedToLower(cache map[string]string, s string) string {
if v, ok := cache[s]; ok {
return v
}
v := strings.ToLower(s)
cache[s] = v
return v
}
// BuildFromStoreWithLog constructs the neighbor graph, optionally logging disambiguation decisions.
func BuildFromStoreWithLog(store *PacketStore, enableLog bool) *NeighborGraph {
g := NewNeighborGraph()
@@ -160,27 +149,30 @@ func BuildFromStoreWithLog(store *PacketStore, enableLog bool) *NeighborGraph {
// Use cached nodes+PM (avoids DB call if cache is fresh).
_, pm := store.getCachedNodesAndPM()
// Local cache for strings.ToLower — pubkeys are immutable and repeat
// across hundreds of thousands of observations.
lowerCache := make(map[string]string, 256)
// Phase 1: Extract edges from every transmission + observation.
for _, tx := range packets {
isAdvert := tx.PayloadType != nil && *tx.PayloadType == 4
fromNode := extractFromNode(tx)
// Pre-compute lowered originator once per tx (not per observation).
fromLower := ""
if fromNode != "" {
fromLower = cachedToLower(lowerCache, fromNode)
}
fromNode := "" // originator pubkey (from byNode index key)
// Find the originator pubkey — it's the key in store.byNode.
// StoreTx doesn't store from_node directly; we find it via decoded JSON
// or the byNode index. However, iterating byNode is expensive.
// The originator pubkey is in the decoded JSON "from_node" field,
// but parsing JSON per tx is expensive too.
// Actually, let's look at how byNode is keyed.
// Looking at store.go, byNode maps pubkey → transmissions where that
// pubkey is the "from" node. We need the reverse: tx → from_node.
// The from_node is embedded in DecodedJSON.
// For efficiency, let's extract it once.
fromNode = extractFromNode(tx)
for _, obs := range tx.Observations {
path := parsePathJSON(obs.PathJSON)
observerPK := cachedToLower(lowerCache, obs.ObserverID)
observerPK := strings.ToLower(obs.ObserverID)
if len(path) == 0 {
// Zero-hop
if isAdvert && fromLower != "" {
if isAdvert && fromNode != "" {
fromLower := strings.ToLower(fromNode)
if fromLower != observerPK { // self-edge guard
g.upsertEdge(fromLower, observerPK, "", observerPK, obs.SNR, parseTimestamp(obs.Timestamp))
}
@@ -189,19 +181,20 @@ func BuildFromStoreWithLog(store *PacketStore, enableLog bool) *NeighborGraph {
}
// Edge 1: originator ↔ path[0] — ADVERTs only
if isAdvert && fromLower != "" {
firstHop := cachedToLower(lowerCache, path[0])
if isAdvert && fromNode != "" {
firstHop := strings.ToLower(path[0])
fromLower := strings.ToLower(fromNode)
if fromLower != firstHop { // self-edge guard (shouldn't happen but spec says check)
candidates := pm.m[firstHop]
g.upsertEdgeWithCandidates(fromLower, firstHop, candidates, observerPK, obs.SNR, parseTimestamp(obs.Timestamp), lowerCache)
g.upsertEdgeWithCandidates(fromLower, firstHop, candidates, observerPK, obs.SNR, parseTimestamp(obs.Timestamp))
}
}
// Edge 2: observer ↔ path[last] — ALL packet types
lastHop := cachedToLower(lowerCache, path[len(path)-1])
lastHop := strings.ToLower(path[len(path)-1])
if observerPK != lastHop { // self-edge guard
candidates := pm.m[lastHop]
g.upsertEdgeWithCandidates(observerPK, lastHop, candidates, observerPK, obs.SNR, parseTimestamp(obs.Timestamp), lowerCache)
g.upsertEdgeWithCandidates(observerPK, lastHop, candidates, observerPK, obs.SNR, parseTimestamp(obs.Timestamp))
}
}
}
@@ -218,10 +211,12 @@ func BuildFromStoreWithLog(store *PacketStore, enableLog bool) *NeighborGraph {
// extractFromNode pulls the originator pubkey from a StoreTx's DecodedJSON.
// ADVERTs use "pubKey", other packets may use "from_node" or "from".
// Uses the cached ParsedDecoded() accessor to avoid repeated json.Unmarshal.
func extractFromNode(tx *StoreTx) string {
decoded := tx.ParsedDecoded()
if decoded == nil {
if tx.DecodedJSON == "" {
return ""
}
var decoded map[string]interface{}
if err := jsonUnmarshalFast(tx.DecodedJSON, &decoded); err != nil {
return ""
}
// ADVERTs store the originator pubkey as "pubKey"; other packets may use
@@ -280,9 +275,9 @@ func (g *NeighborGraph) upsertEdge(pubkeyA, pubkeyB, prefix, observer string, sn
}
// upsertEdgeWithCandidates handles prefix-based edges that may be ambiguous.
func (g *NeighborGraph) upsertEdgeWithCandidates(knownPK, prefix string, candidates []nodeInfo, observer string, snr *float64, ts time.Time, lc map[string]string) {
func (g *NeighborGraph) upsertEdgeWithCandidates(knownPK, prefix string, candidates []nodeInfo, observer string, snr *float64, ts time.Time) {
if len(candidates) == 1 {
resolved := cachedToLower(lc, candidates[0].PublicKey)
resolved := strings.ToLower(candidates[0].PublicKey)
if resolved == knownPK {
return // self-edge guard
}
@@ -293,7 +288,7 @@ func (g *NeighborGraph) upsertEdgeWithCandidates(knownPK, prefix string, candida
// Filter out self from candidates
filtered := make([]string, 0, len(candidates))
for _, c := range candidates {
pk := cachedToLower(lc, c.PublicKey)
pk := strings.ToLower(c.PublicKey)
if pk != knownPK {
filtered = append(filtered, pk)
}
-117
View File
@@ -717,120 +717,3 @@ func TestNeighborGraph_CacheTTL(t *testing.T) {
t.Error("old graph should be stale")
}
}
func TestNeighborGraph_TTLIsReasonable(t *testing.T) {
// TTL must be long enough to avoid rebuild storms on busy meshes,
// but short enough to reflect topology changes within minutes.
if neighborGraphTTL < 1*time.Minute {
t.Errorf("neighborGraphTTL too short (%v), will cause rebuild storms", neighborGraphTTL)
}
if neighborGraphTTL > 10*time.Minute {
t.Errorf("neighborGraphTTL too long (%v), topology changes will be stale", neighborGraphTTL)
}
}
func TestCachedToLower(t *testing.T) {
cache := make(map[string]string)
// Basic lowercasing
if got := cachedToLower(cache, "AABB"); got != "aabb" {
t.Errorf("expected 'aabb', got %q", got)
}
// Verify it was cached
if _, ok := cache["AABB"]; !ok {
t.Error("expected 'AABB' to be in cache")
}
// Same input returns cached result
if got := cachedToLower(cache, "AABB"); got != "aabb" {
t.Errorf("expected cached 'aabb', got %q", got)
}
// Already lowercase stays the same
if got := cachedToLower(cache, "aabb"); got != "aabb" {
t.Errorf("expected 'aabb', got %q", got)
}
// Empty string
if got := cachedToLower(cache, ""); got != "" {
t.Errorf("expected empty, got %q", got)
}
}
func TestParsedDecoded_Caching(t *testing.T) {
tx := &StoreTx{DecodedJSON: `{"pubKey":"abc123","name":"test"}`}
// First call parses
d1 := tx.ParsedDecoded()
if d1 == nil {
t.Fatal("expected non-nil parsed result")
}
if d1["pubKey"] != "abc123" {
t.Errorf("expected pubKey=abc123, got %v", d1["pubKey"])
}
// Second call must return the exact same map (pointer equality proves caching)
d2 := tx.ParsedDecoded()
if &d1 == nil || &d2 == nil {
t.Fatal("unexpected nil")
}
// Mutate d1 and verify d2 sees the mutation — proves same underlying map
d1["_sentinel"] = true
if d2["_sentinel"] != true {
t.Error("expected same map instance from second call (caching broken)")
}
delete(d1, "_sentinel") // clean up
}
func TestParsedDecoded_EmptyJSON(t *testing.T) {
tx := &StoreTx{DecodedJSON: ""}
d := tx.ParsedDecoded()
if d != nil {
t.Errorf("expected nil for empty DecodedJSON, got %v", d)
}
}
func TestParsedDecoded_InvalidJSON(t *testing.T) {
tx := &StoreTx{DecodedJSON: "not json"}
d := tx.ParsedDecoded()
if d != nil {
t.Errorf("expected nil for invalid JSON, got %v", d)
}
}
func TestExtractFromNode_UsesCachedParse(t *testing.T) {
tx := &StoreTx{DecodedJSON: `{"pubKey":"aabb1122"}`}
// First call to extractFromNode should use ParsedDecoded
from := extractFromNode(tx)
if from != "aabb1122" {
t.Errorf("expected aabb1122, got %q", from)
}
// ParsedDecoded should now be cached
d := tx.ParsedDecoded()
if d == nil || d["pubKey"] != "aabb1122" {
t.Error("expected ParsedDecoded to return cached result")
}
}
func BenchmarkBuildFromStore(b *testing.B) {
// Simulate a dataset with many packets and repeated pubkeys
nodes := []nodeInfo{
{PublicKey: "aaaa1111", Name: "NodeA"},
{PublicKey: "bbbb2222", Name: "NodeB"},
{PublicKey: "cccc3333", Name: "NodeC"},
{PublicKey: "dddd4444", Name: "NodeD"},
}
const numPackets = 1000
packets := make([]*StoreTx, 0, numPackets)
for i := 0; i < numPackets; i++ {
pt := 4 // ADVERT
packets = append(packets, &StoreTx{
ID: i,
PayloadType: &pt,
DecodedJSON: `{"pubKey":"aaaa1111"}`,
Observations: []*StoreObs{
{ObserverID: "bbbb2222", PathJSON: `["cccc"]`, Timestamp: nowStr, SNR: ngFloatPtr(-5.0)},
},
})
}
store := ngTestStore(nodes, packets)
b.ResetTimer()
for i := 0; i < b.N; i++ {
BuildFromStore(store)
}
}
-531
View File
@@ -1,531 +0,0 @@
package main
import (
"database/sql"
"encoding/json"
"fmt"
"log"
"strings"
"time"
)
// persistSem limits concurrent async persistence goroutines to 1.
// Without this, each ingest cycle spawns a goroutine that opens a new
// SQLite RW connection; under sustained load goroutines pile up with
// no backpressure, causing contention and busy-timeout cascades.
var persistSem = make(chan struct{}, 1)
// ─── neighbor_edges table ──────────────────────────────────────────────────────
// ensureNeighborEdgesTable creates the neighbor_edges table if it doesn't exist.
// Uses a separate read-write connection since the main DB is read-only.
func ensureNeighborEdgesTable(dbPath string) error {
rw, err := openRW(dbPath)
if err != nil {
return fmt.Errorf("open rw for neighbor_edges: %w", err)
}
defer rw.Close()
_, err = rw.Exec(`CREATE TABLE IF NOT EXISTS neighbor_edges (
node_a TEXT NOT NULL,
node_b TEXT NOT NULL,
count INTEGER DEFAULT 1,
last_seen TEXT,
PRIMARY KEY (node_a, node_b)
)`)
return err
}
// loadNeighborEdgesFromDB loads all edges from the neighbor_edges table
// and builds an in-memory NeighborGraph.
func loadNeighborEdgesFromDB(conn *sql.DB) *NeighborGraph {
g := NewNeighborGraph()
rows, err := conn.Query("SELECT node_a, node_b, count, last_seen FROM neighbor_edges")
if err != nil {
log.Printf("[neighbor] failed to load neighbor_edges: %v", err)
return g
}
defer rows.Close()
count := 0
for rows.Next() {
var a, b string
var cnt int
var lastSeen sql.NullString
if err := rows.Scan(&a, &b, &cnt, &lastSeen); err != nil {
continue
}
ts := time.Time{}
if lastSeen.Valid {
ts = parseTimestamp(lastSeen.String)
}
// Build edge directly (both nodes are full pubkeys from persisted data)
key := makeEdgeKey(a, b)
g.mu.Lock()
e, exists := g.edges[key]
if !exists {
e = &NeighborEdge{
NodeA: key.A,
NodeB: key.B,
Observers: make(map[string]bool),
FirstSeen: ts,
LastSeen: ts,
Count: cnt,
}
g.edges[key] = e
g.byNode[key.A] = append(g.byNode[key.A], e)
g.byNode[key.B] = append(g.byNode[key.B], e)
} else {
e.Count += cnt
if ts.After(e.LastSeen) {
e.LastSeen = ts
}
}
g.mu.Unlock()
count++
}
if count > 0 {
g.mu.Lock()
g.builtAt = time.Now()
g.mu.Unlock()
log.Printf("[neighbor] loaded %d edges from neighbor_edges table", count)
}
return g
}
// ─── shared async persistence helper ───────────────────────────────────────────
// persistObsUpdate holds data for a resolved_path SQLite update.
type persistObsUpdate struct {
obsID int
resolvedPath string
}
// persistEdgeUpdate holds data for a neighbor_edges SQLite upsert.
type persistEdgeUpdate struct {
a, b, ts string
}
// asyncPersistResolvedPathsAndEdges writes resolved_path updates and neighbor
// edge upserts to SQLite in a background goroutine. Shared between
// IngestNewFromDB and IngestNewObservations to avoid DRY violation.
func asyncPersistResolvedPathsAndEdges(dbPath string, obsUpdates []persistObsUpdate, edgeUpdates []persistEdgeUpdate, logPrefix string) {
if len(obsUpdates) == 0 && len(edgeUpdates) == 0 {
return
}
// Try-acquire semaphore BEFORE spawning goroutine. If another
// persistence operation is already running, drop this batch —
// data lives in memory and will be backfilled on restart.
select {
case persistSem <- struct{}{}:
// Acquired — spawn goroutine to do the work.
default:
log.Printf("[store] %s skipped: persistence already in progress", logPrefix)
return
}
go func() {
defer func() { <-persistSem }()
rw, err := openRW(dbPath)
if err != nil {
log.Printf("[store] %s rw open error: %v", logPrefix, err)
return
}
defer rw.Close()
if len(obsUpdates) > 0 {
sqlTx, err := rw.Begin()
if err == nil {
stmt, err := sqlTx.Prepare("UPDATE observations SET resolved_path = ? WHERE id = ?")
if err == nil {
var firstErr error
for _, u := range obsUpdates {
if _, err := stmt.Exec(u.resolvedPath, u.obsID); err != nil && firstErr == nil {
firstErr = err
}
}
stmt.Close()
if firstErr != nil {
log.Printf("[store] %s resolved_path error (first): %v", logPrefix, firstErr)
}
} else {
log.Printf("[store] %s resolved_path prepare error: %v", logPrefix, err)
}
sqlTx.Commit()
}
}
if len(edgeUpdates) > 0 {
sqlTx, err := rw.Begin()
if err == nil {
stmt, err := sqlTx.Prepare(`INSERT INTO neighbor_edges (node_a, node_b, count, last_seen)
VALUES (?, ?, 1, ?)
ON CONFLICT(node_a, node_b) DO UPDATE SET
count = count + 1, last_seen = MAX(last_seen, excluded.last_seen)`)
if err == nil {
var firstErr error
for _, e := range edgeUpdates {
if _, err := stmt.Exec(e.a, e.b, e.ts); err != nil && firstErr == nil {
firstErr = err
}
}
stmt.Close()
if firstErr != nil {
log.Printf("[store] %s edge error (first): %v", logPrefix, firstErr)
}
} else {
log.Printf("[store] %s edge prepare error: %v", logPrefix, err)
}
sqlTx.Commit()
}
}
}()
}
// neighborEdgesTableExists checks if the neighbor_edges table has any data.
func neighborEdgesTableExists(conn *sql.DB) bool {
var cnt int
err := conn.QueryRow("SELECT COUNT(*) FROM neighbor_edges").Scan(&cnt)
if err != nil {
return false // table doesn't exist
}
return cnt > 0
}
// buildAndPersistEdges scans all packets in the store, extracts edges per
// ADVERT/non-ADVERT rules, and persists them to SQLite.
func buildAndPersistEdges(store *PacketStore, rw *sql.DB) int {
store.mu.RLock()
packets := make([]*StoreTx, len(store.packets))
copy(packets, store.packets)
store.mu.RUnlock()
_, pm := store.getCachedNodesAndPM()
tx, err := rw.Begin()
if err != nil {
log.Printf("[neighbor] begin tx error: %v", err)
return 0
}
defer tx.Rollback()
stmt, err := tx.Prepare(`INSERT INTO neighbor_edges (node_a, node_b, count, last_seen)
VALUES (?, ?, 1, ?)
ON CONFLICT(node_a, node_b) DO UPDATE SET
count = count + 1, last_seen = MAX(last_seen, excluded.last_seen)`)
if err != nil {
log.Printf("[neighbor] prepare stmt error: %v", err)
return 0
}
defer stmt.Close()
edgeCount := 0
var firstErr error
for _, pkt := range packets {
for _, obs := range pkt.Observations {
for _, ec := range extractEdgesFromObs(obs, pkt, pm) {
if _, err := stmt.Exec(ec.A, ec.B, ec.Timestamp); err != nil && firstErr == nil {
firstErr = err
}
edgeCount++
}
}
}
if firstErr != nil {
log.Printf("[neighbor] edge exec error (first): %v", firstErr)
}
if err := tx.Commit(); err != nil {
log.Printf("[neighbor] commit error: %v", err)
return 0
}
return edgeCount
}
// ─── resolved_path column ──────────────────────────────────────────────────────
// ensureResolvedPathColumn adds the resolved_path column to observations if missing.
func ensureResolvedPathColumn(dbPath string) error {
rw, err := openRW(dbPath)
if err != nil {
return err
}
defer rw.Close()
// Check if column already exists
rows, err := rw.Query("PRAGMA table_info(observations)")
if err != nil {
return err
}
defer rows.Close()
for rows.Next() {
var cid int
var colName string
var colType sql.NullString
var notNull, pk int
var dflt sql.NullString
if rows.Scan(&cid, &colName, &colType, &notNull, &dflt, &pk) == nil && colName == "resolved_path" {
return nil // already exists
}
}
_, err = rw.Exec("ALTER TABLE observations ADD COLUMN resolved_path TEXT")
if err != nil {
return fmt.Errorf("add resolved_path column: %w", err)
}
log.Println("[store] Added resolved_path column to observations")
return nil
}
// resolvePathForObs resolves hop prefixes to full pubkeys for an observation.
// Returns nil if path is empty.
func resolvePathForObs(pathJSON, observerID string, tx *StoreTx, pm *prefixMap, graph *NeighborGraph) []*string {
hops := parsePathJSON(pathJSON)
if len(hops) == 0 {
return nil
}
// Build context pubkeys: observer + originator (if known)
contextPKs := make([]string, 0, 3)
if observerID != "" {
contextPKs = append(contextPKs, strings.ToLower(observerID))
}
fromNode := extractFromNode(tx)
if fromNode != "" {
contextPKs = append(contextPKs, strings.ToLower(fromNode))
}
resolved := make([]*string, len(hops))
for i, hop := range hops {
// Add adjacent hops as context for disambiguation
ctx := make([]string, len(contextPKs), len(contextPKs)+2)
copy(ctx, contextPKs)
// Add previously resolved hops as context
if i > 0 && resolved[i-1] != nil {
ctx = append(ctx, *resolved[i-1])
}
node, _, _ := pm.resolveWithContext(hop, ctx, graph)
if node != nil {
pk := strings.ToLower(node.PublicKey)
resolved[i] = &pk
}
}
return resolved
}
// marshalResolvedPath converts []*string to JSON for storage.
func marshalResolvedPath(rp []*string) string {
if len(rp) == 0 {
return ""
}
b, err := json.Marshal(rp)
if err != nil {
return ""
}
return string(b)
}
// unmarshalResolvedPath parses a resolved_path JSON string.
func unmarshalResolvedPath(s string) []*string {
if s == "" {
return nil
}
var result []*string
if json.Unmarshal([]byte(s), &result) != nil {
return nil
}
return result
}
// backfillResolvedPaths resolves paths for all observations that have NULL resolved_path.
func backfillResolvedPaths(store *PacketStore, dbPath string) int {
// Collect pending observations and snapshot immutable fields under read lock.
// graph is set in main.go before backfill is called; nil-safe throughout (review item #6).
type obsRef struct {
obsID int
pathJSON string
observerID string
txJSON string // snapshot of DecodedJSON for extractFromNode
payloadType *int
}
store.mu.RLock()
pm := store.nodePM
graph := store.graph
var pending []obsRef
for _, tx := range store.packets {
for _, obs := range tx.Observations {
if obs.ResolvedPath == nil && obs.PathJSON != "" && obs.PathJSON != "[]" {
pending = append(pending, obsRef{
obsID: obs.ID,
pathJSON: obs.PathJSON,
observerID: obs.ObserverID,
txJSON: tx.DecodedJSON,
payloadType: tx.PayloadType,
})
}
}
}
store.mu.RUnlock()
if len(pending) == 0 || pm == nil {
return 0
}
// Resolve paths outside the lock — resolvePathForObs only reads pm and graph.
type resolved struct {
obsID int
rp []*string
rpJSON string
}
var results []resolved
for _, ref := range pending {
// Build a minimal StoreTx for extractFromNode (only needs DecodedJSON + PayloadType).
fakeTx := &StoreTx{DecodedJSON: ref.txJSON, PayloadType: ref.payloadType}
rp := resolvePathForObs(ref.pathJSON, ref.observerID, fakeTx, pm, graph)
if len(rp) > 0 {
rpJSON := marshalResolvedPath(rp)
if rpJSON != "" {
results = append(results, resolved{ref.obsID, rp, rpJSON})
}
}
}
if len(results) == 0 {
return 0
}
// Persist to SQLite (no lock needed — separate RW connection).
rw, err := openRW(dbPath)
if err != nil {
log.Printf("[store] backfill: open rw error: %v", err)
return 0
}
defer rw.Close()
sqlTx, err := rw.Begin()
if err != nil {
log.Printf("[store] backfill: begin tx error: %v", err)
return 0
}
defer sqlTx.Rollback()
stmt, err := sqlTx.Prepare("UPDATE observations SET resolved_path = ? WHERE id = ?")
if err != nil {
log.Printf("[store] backfill: prepare error: %v", err)
return 0
}
defer stmt.Close()
var firstErr error
for _, r := range results {
if _, err := stmt.Exec(r.rpJSON, r.obsID); err != nil && firstErr == nil {
firstErr = err
}
}
if firstErr != nil {
log.Printf("[store] backfill resolved_path exec error (first): %v", firstErr)
}
if err := sqlTx.Commit(); err != nil {
log.Printf("[store] backfill: commit error: %v", err)
return 0
}
// Update in-memory state under write lock.
store.mu.Lock()
count := 0
for _, r := range results {
if obs, ok := store.byObsID[r.obsID]; ok {
obs.ResolvedPath = r.rp
count++
}
}
store.mu.Unlock()
return count
}
// ─── Shared helpers ────────────────────────────────────────────────────────────
// edgeCandidate represents an extracted edge to be persisted.
type edgeCandidate struct {
A, B, Timestamp string
}
// extractEdgesFromObs extracts neighbor edge candidates from a single observation.
// For ADVERTs: originator↔path[0] (if unambiguous). For ALL types: observer↔path[last] (if unambiguous).
// Also handles zero-hop ADVERTs (originator↔observer direct link).
func extractEdgesFromObs(obs *StoreObs, tx *StoreTx, pm *prefixMap) []edgeCandidate {
isAdvert := tx.PayloadType != nil && *tx.PayloadType == 4
fromNode := extractFromNode(tx)
path := parsePathJSON(obs.PathJSON)
observerPK := strings.ToLower(obs.ObserverID)
ts := obs.Timestamp
var edges []edgeCandidate
if len(path) == 0 {
if isAdvert && fromNode != "" {
fromLower := strings.ToLower(fromNode)
if fromLower != observerPK {
a, b := fromLower, observerPK
if a > b {
a, b = b, a
}
edges = append(edges, edgeCandidate{a, b, ts})
}
}
return edges
}
// Edge 1: originator ↔ path[0] — ADVERTs only (resolve prefix to full pubkey)
if isAdvert && fromNode != "" && pm != nil {
firstHop := strings.ToLower(path[0])
fromLower := strings.ToLower(fromNode)
candidates := pm.m[firstHop]
if len(candidates) == 1 {
resolved := strings.ToLower(candidates[0].PublicKey)
if resolved != fromLower {
a, b := fromLower, resolved
if a > b {
a, b = b, a
}
edges = append(edges, edgeCandidate{a, b, ts})
}
}
}
// Edge 2: observer ↔ path[last] — ALL packet types
if pm != nil {
lastHop := strings.ToLower(path[len(path)-1])
candidates := pm.m[lastHop]
if len(candidates) == 1 {
resolved := strings.ToLower(candidates[0].PublicKey)
if resolved != observerPK {
a, b := observerPK, resolved
if a > b {
a, b = b, a
}
edges = append(edges, edgeCandidate{a, b, ts})
}
}
}
return edges
}
// openRW opens a read-write SQLite connection (same pattern as PruneOldPackets).
func openRW(dbPath string) (*sql.DB, error) {
dsn := fmt.Sprintf("file:%s?_journal_mode=WAL&_busy_timeout=10000", dbPath)
rw, err := sql.Open("sqlite", dsn)
if err != nil {
return nil, err
}
rw.SetMaxOpenConns(1)
return rw, nil
}
-534
View File
@@ -1,534 +0,0 @@
package main
import (
"database/sql"
"encoding/json"
"path/filepath"
"strings"
"testing"
"time"
_ "modernc.org/sqlite"
)
// createTestDBWithSchema creates a temp SQLite DB with the standard schema + resolved_path column.
func createTestDBWithSchema(t *testing.T) (*DB, string) {
t.Helper()
dir := t.TempDir()
dbPath := filepath.Join(dir, "test.db")
conn, err := sql.Open("sqlite", "file:"+dbPath+"?_journal_mode=WAL")
if err != nil {
t.Fatal(err)
}
// Create tables
conn.Exec(`CREATE TABLE transmissions (
id INTEGER PRIMARY KEY AUTOINCREMENT,
raw_hex TEXT, hash TEXT UNIQUE, first_seen TEXT,
route_type INTEGER, payload_type INTEGER, payload_version INTEGER,
decoded_json TEXT
)`)
conn.Exec(`CREATE TABLE observers (
id TEXT PRIMARY KEY, name TEXT, iata TEXT
)`)
conn.Exec(`CREATE TABLE observations (
id INTEGER PRIMARY KEY AUTOINCREMENT,
transmission_id INTEGER NOT NULL REFERENCES transmissions(id),
observer_id TEXT, observer_name TEXT, direction TEXT,
snr REAL, rssi REAL, score INTEGER,
path_json TEXT, timestamp TEXT,
resolved_path TEXT
)`)
conn.Exec(`CREATE TABLE nodes (
public_key TEXT PRIMARY KEY, name TEXT, role TEXT,
lat REAL, lon REAL, last_seen TEXT, first_seen TEXT,
advert_count INTEGER DEFAULT 0
)`)
conn.Close()
db, err := OpenDB(dbPath)
if err != nil {
t.Fatal(err)
}
return db, dbPath
}
func TestResolvePathForObs(t *testing.T) {
// Build a prefix map with known nodes
nodes := []nodeInfo{
{PublicKey: "aabbccddee1234567890aabbccddee1234567890aabbccddee1234567890aabb", Name: "Node-AA"},
{PublicKey: "bbccddee1234567890aabbccddee1234567890aabbccddee1234567890aabb11", Name: "Node-BB"},
}
pm := buildPrefixMap(nodes)
graph := NewNeighborGraph()
tx := &StoreTx{
DecodedJSON: `{"pubKey": "originator1234567890"}`,
PayloadType: intPtr(4),
}
// Unambiguous prefixes should resolve
rp := resolvePathForObs(`["aa","bb"]`, "observer1", tx, pm, graph)
if len(rp) != 2 {
t.Fatalf("expected 2 resolved hops, got %d", len(rp))
}
if rp[0] == nil || !strings.HasPrefix(*rp[0], "aabbcc") {
t.Errorf("expected first hop to resolve to Node-AA, got %v", rp[0])
}
if rp[1] == nil || !strings.HasPrefix(*rp[1], "bbccdd") {
t.Errorf("expected second hop to resolve to Node-BB, got %v", rp[1])
}
}
func TestResolvePathForObs_EmptyPath(t *testing.T) {
pm := buildPrefixMap(nil)
rp := resolvePathForObs(`[]`, "", &StoreTx{}, pm, nil)
if rp != nil {
t.Errorf("expected nil for empty path, got %v", rp)
}
rp = resolvePathForObs("", "", &StoreTx{}, pm, nil)
if rp != nil {
t.Errorf("expected nil for empty string, got %v", rp)
}
}
func TestResolvePathForObs_Unresolvable(t *testing.T) {
nodes := []nodeInfo{
{PublicKey: "aabbccddee1234567890aabbccddee1234567890aabbccddee1234567890aabb", Name: "Node-AA"},
}
pm := buildPrefixMap(nodes)
// "zz" prefix doesn't match any node
rp := resolvePathForObs(`["zz"]`, "", &StoreTx{}, pm, nil)
if len(rp) != 1 {
t.Fatalf("expected 1 hop, got %d", len(rp))
}
if rp[0] != nil {
t.Errorf("expected nil for unresolvable hop, got %v", *rp[0])
}
}
func TestMarshalUnmarshalResolvedPath(t *testing.T) {
pk1 := "aabbccdd"
var rp []*string
rp = append(rp, &pk1, nil)
j := marshalResolvedPath(rp)
if j == "" {
t.Fatal("expected non-empty JSON")
}
parsed := unmarshalResolvedPath(j)
if len(parsed) != 2 {
t.Fatalf("expected 2 elements, got %d", len(parsed))
}
if parsed[0] == nil || *parsed[0] != "aabbccdd" {
t.Errorf("first element wrong: %v", parsed[0])
}
if parsed[1] != nil {
t.Errorf("second element should be nil, got %v", *parsed[1])
}
}
func TestMarshalResolvedPath_Empty(t *testing.T) {
if marshalResolvedPath(nil) != "" {
t.Error("expected empty for nil")
}
if marshalResolvedPath([]*string{}) != "" {
t.Error("expected empty for empty slice")
}
}
func TestUnmarshalResolvedPath_Invalid(t *testing.T) {
if unmarshalResolvedPath("") != nil {
t.Error("expected nil for empty string")
}
if unmarshalResolvedPath("not json") != nil {
t.Error("expected nil for invalid JSON")
}
}
func TestEnsureNeighborEdgesTable(t *testing.T) {
dir := t.TempDir()
dbPath := filepath.Join(dir, "test.db")
// Create initial DB
conn, _ := sql.Open("sqlite", "file:"+dbPath+"?_journal_mode=WAL")
conn.Exec("CREATE TABLE test (id INTEGER PRIMARY KEY)")
conn.Close()
if err := ensureNeighborEdgesTable(dbPath); err != nil {
t.Fatal(err)
}
// Verify table exists
conn, _ = sql.Open("sqlite", "file:"+dbPath+"?mode=ro")
defer conn.Close()
var cnt int
if err := conn.QueryRow("SELECT COUNT(*) FROM neighbor_edges").Scan(&cnt); err != nil {
t.Fatalf("neighbor_edges table not created: %v", err)
}
}
func TestLoadNeighborEdgesFromDB(t *testing.T) {
dir := t.TempDir()
dbPath := filepath.Join(dir, "test.db")
conn, _ := sql.Open("sqlite", "file:"+dbPath+"?_journal_mode=WAL")
conn.Exec(`CREATE TABLE neighbor_edges (
node_a TEXT NOT NULL, node_b TEXT NOT NULL,
count INTEGER DEFAULT 1, last_seen TEXT,
PRIMARY KEY (node_a, node_b)
)`)
conn.Exec("INSERT INTO neighbor_edges VALUES ('aaa', 'bbb', 5, '2024-01-01T00:00:00Z')")
conn.Exec("INSERT INTO neighbor_edges VALUES ('ccc', 'ddd', 3, '2024-01-02T00:00:00Z')")
g := loadNeighborEdgesFromDB(conn)
conn.Close()
// Should have 2 edges
edges := g.AllEdges()
if len(edges) != 2 {
t.Errorf("expected 2 edges, got %d", len(edges))
}
// Check neighbors
n := g.Neighbors("aaa")
if len(n) != 1 {
t.Errorf("expected 1 neighbor for aaa, got %d", len(n))
}
}
func TestStoreObsResolvedPathInBroadcast(t *testing.T) {
// Verify resolved_path appears in broadcast maps
pk := "aabbccdd"
obs := &StoreObs{
ID: 1,
ObserverID: "obs1",
ObserverName: "Observer 1",
PathJSON: `["aa"]`,
ResolvedPath: []*string{&pk},
Timestamp: "2024-01-01T00:00:00Z",
}
tx := &StoreTx{
ID: 1,
Hash: "abc123",
Observations: []*StoreObs{obs},
}
pickBestObservation(tx)
if tx.ResolvedPath == nil {
t.Fatal("expected ResolvedPath to be set on tx after pickBestObservation")
}
if *tx.ResolvedPath[0] != "aabbccdd" {
t.Errorf("expected resolved path to be aabbccdd, got %s", *tx.ResolvedPath[0])
}
}
func TestResolvedPathInTxToMap(t *testing.T) {
pk := "aabbccdd"
tx := &StoreTx{
ID: 1,
Hash: "abc123",
PathJSON: `["aa"]`,
ResolvedPath: []*string{&pk},
obsKeys: make(map[string]bool),
}
m := txToMap(tx)
rp, ok := m["resolved_path"]
if !ok {
t.Fatal("resolved_path not in txToMap output")
}
rpSlice, ok := rp.([]*string)
if !ok || len(rpSlice) != 1 || *rpSlice[0] != "aabbccdd" {
t.Errorf("unexpected resolved_path: %v", rp)
}
}
func TestResolvedPathOmittedWhenNil(t *testing.T) {
tx := &StoreTx{
ID: 1,
Hash: "abc123",
obsKeys: make(map[string]bool),
}
m := txToMap(tx)
if _, ok := m["resolved_path"]; ok {
t.Error("resolved_path should not be in map when nil")
}
}
func TestEnsureResolvedPathColumn(t *testing.T) {
dir := t.TempDir()
dbPath := filepath.Join(dir, "test.db")
conn, _ := sql.Open("sqlite", "file:"+dbPath+"?_journal_mode=WAL")
conn.Exec(`CREATE TABLE observations (
id INTEGER PRIMARY KEY, transmission_id INTEGER,
observer_id TEXT, path_json TEXT, timestamp TEXT
)`)
conn.Close()
if err := ensureResolvedPathColumn(dbPath); err != nil {
t.Fatal(err)
}
// Verify column exists
conn, _ = sql.Open("sqlite", "file:"+dbPath+"?mode=ro")
defer conn.Close()
rows, _ := conn.Query("PRAGMA table_info(observations)")
found := false
for rows.Next() {
var cid int
var colName string
var colType sql.NullString
var notNull, pk int
var dflt sql.NullString
rows.Scan(&cid, &colName, &colType, &notNull, &dflt, &pk)
if colName == "resolved_path" {
found = true
}
}
rows.Close()
if !found {
t.Error("resolved_path column not added")
}
// Running again should be idempotent
if err := ensureResolvedPathColumn(dbPath); err != nil {
t.Fatal("second call should be idempotent:", err)
}
}
func TestDBDetectsResolvedPathColumn(t *testing.T) {
dir := t.TempDir()
dbPath := filepath.Join(dir, "test.db")
// Create DB without resolved_path
conn, _ := sql.Open("sqlite", "file:"+dbPath+"?_journal_mode=WAL")
conn.Exec(`CREATE TABLE observations (id INTEGER PRIMARY KEY, observer_idx INTEGER)`)
conn.Exec(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY)`)
conn.Close()
db, err := OpenDB(dbPath)
if err != nil {
t.Fatal(err)
}
if db.hasResolvedPath {
t.Error("should not detect resolved_path when column missing")
}
db.Close()
// Add resolved_path column
conn, _ = sql.Open("sqlite", "file:"+dbPath+"?_journal_mode=WAL")
conn.Exec("ALTER TABLE observations ADD COLUMN resolved_path TEXT")
conn.Close()
db, err = OpenDB(dbPath)
if err != nil {
t.Fatal(err)
}
if !db.hasResolvedPath {
t.Error("should detect resolved_path when column exists")
}
db.Close()
}
func TestLoadWithResolvedPath(t *testing.T) {
db, dbPath := createTestDBWithSchema(t)
defer db.Close()
// Insert test data
rw, _ := openRW(dbPath)
rw.Exec(`INSERT INTO transmissions (id, hash, first_seen, payload_type, decoded_json)
VALUES (1, 'hash1', '2024-01-01T00:00:00Z', 4, '{"pubKey":"origpk"}')`)
rw.Exec(`INSERT INTO observations (id, transmission_id, observer_id, observer_name, path_json, timestamp, resolved_path)
VALUES (1, 1, 'obs1', 'Observer1', '["aa"]', '2024-01-01T00:00:00Z', '["aabbccdd"]')`)
rw.Close()
store := NewPacketStore(db, nil)
if err := store.Load(); err != nil {
t.Fatal(err)
}
if len(store.packets) != 1 {
t.Fatalf("expected 1 packet, got %d", len(store.packets))
}
tx := store.packets[0]
if len(tx.Observations) != 1 {
t.Fatalf("expected 1 observation, got %d", len(tx.Observations))
}
obs := tx.Observations[0]
if obs.ResolvedPath == nil {
t.Fatal("expected ResolvedPath to be loaded")
}
if len(obs.ResolvedPath) != 1 || *obs.ResolvedPath[0] != "aabbccdd" {
t.Errorf("unexpected ResolvedPath: %v", obs.ResolvedPath)
}
// Check that pickBestObservation propagated resolved_path to tx
if tx.ResolvedPath == nil || len(tx.ResolvedPath) != 1 {
t.Error("expected ResolvedPath to be propagated to tx")
}
}
func TestResolvedPathInAPIResponse(t *testing.T) {
// Test that TransmissionResp properly marshals resolved_path
pk := "aabbccddee"
resp := TransmissionResp{
ID: 1,
Hash: "test",
ResolvedPath: []*string{&pk, nil},
}
data, err := json.Marshal(resp)
if err != nil {
t.Fatal(err)
}
var m map[string]interface{}
json.Unmarshal(data, &m)
rp, ok := m["resolved_path"]
if !ok {
t.Fatal("resolved_path missing from JSON")
}
rpArr, ok := rp.([]interface{})
if !ok || len(rpArr) != 2 {
t.Fatalf("unexpected resolved_path shape: %v", rp)
}
if rpArr[0] != "aabbccddee" {
t.Errorf("first element wrong: %v", rpArr[0])
}
if rpArr[1] != nil {
t.Errorf("second element should be null: %v", rpArr[1])
}
}
func TestResolvedPathOmittedWhenEmpty(t *testing.T) {
resp := TransmissionResp{
ID: 1,
Hash: "test",
}
data, _ := json.Marshal(resp)
var m map[string]interface{}
json.Unmarshal(data, &m)
if _, ok := m["resolved_path"]; ok {
t.Error("resolved_path should be omitted when nil")
}
}
func TestExtractEdgesFromObs_AdvertNoPath(t *testing.T) {
tx := &StoreTx{
DecodedJSON: `{"pubKey":"aaaa1111"}`,
PayloadType: intPtr(4),
}
obs := &StoreObs{
ObserverID: "bbbb2222",
PathJSON: "",
Timestamp: "2024-01-01T00:00:00Z",
}
edges := extractEdgesFromObs(obs, tx, nil)
if len(edges) != 1 {
t.Fatalf("expected 1 edge for zero-hop advert, got %d", len(edges))
}
// Canonical ordering: aaaa < bbbb
if edges[0].A != "aaaa1111" || edges[0].B != "bbbb2222" {
t.Errorf("unexpected edge: %+v", edges[0])
}
}
func TestExtractEdgesFromObs_NonAdvertNoPath(t *testing.T) {
tx := &StoreTx{PayloadType: intPtr(1)}
obs := &StoreObs{ObserverID: "obs1", PathJSON: ""}
edges := extractEdgesFromObs(obs, tx, nil)
if len(edges) != 0 {
t.Errorf("expected 0 edges for non-advert without path, got %d", len(edges))
}
}
func TestExtractEdgesFromObs_WithPath(t *testing.T) {
nodes := []nodeInfo{
{PublicKey: "aabbccddee1234567890aabbccddee1234567890aabbccddee1234567890aabb", Name: "Node-AA"},
{PublicKey: "ffgghhii1234567890aabbccddee1234567890aabbccddee1234567890aabb11", Name: "Node-FF"},
}
pm := buildPrefixMap(nodes)
tx := &StoreTx{
DecodedJSON: `{"pubKey":"originator00"}`,
PayloadType: intPtr(4),
}
obs := &StoreObs{
ObserverID: "observer00",
PathJSON: `["aa","ff"]`,
Timestamp: "2024-01-01T00:00:00Z",
}
edges := extractEdgesFromObs(obs, tx, pm)
// Should get: originator↔aa (advert), observer↔ff (last hop)
if len(edges) != 2 {
t.Fatalf("expected 2 edges, got %d", len(edges))
}
}
func TestExtractEdgesFromObs_SameNodeNoEdge(t *testing.T) {
tx := &StoreTx{
DecodedJSON: `{"pubKey":"same1234"}`,
PayloadType: intPtr(4),
}
obs := &StoreObs{
ObserverID: "same1234",
PathJSON: "",
Timestamp: "2024-01-01T00:00:00Z",
}
edges := extractEdgesFromObs(obs, tx, nil)
if len(edges) != 0 {
t.Errorf("expected 0 edges when originator == observer, got %d", len(edges))
}
}
func TestPersistSemaphoreTryAcquireSkipsBatch(t *testing.T) {
// Verify that persistSem is a buffered channel of size 1.
if cap(persistSem) != 1 {
t.Errorf("persistSem capacity = %d, want 1", cap(persistSem))
}
// Acquire the semaphore to simulate an in-progress persistence.
persistSem <- struct{}{}
// asyncPersistResolvedPathsAndEdges should skip (not block, not
// spawn a goroutine) when the semaphore is already held.
done := make(chan struct{})
go func() {
asyncPersistResolvedPathsAndEdges(
"/nonexistent/path.db",
[]persistObsUpdate{{obsID: 1, resolvedPath: "x"}},
nil,
"test",
)
close(done)
}()
// If the function blocks on the semaphore instead of skipping,
// this select will hit the timeout.
select {
case <-done:
// Expected: returned immediately because semaphore was busy.
case <-time.After(500 * time.Millisecond):
<-persistSem
t.Fatal("asyncPersistResolvedPathsAndEdges blocked instead of skipping when semaphore was held")
}
<-persistSem // release
}
+1 -7
View File
@@ -1088,7 +1088,7 @@ func (s *Server) handleNodePaths(w http.ResponseWriter, r *http.Request) {
if cached, ok := hopCache[hop]; ok {
return cached
}
r, _, _ := pm.resolveWithContext(hop, nil, s.store.graph)
r := pm.resolve(hop)
hopCache[hop] = r
return r
}
@@ -1997,9 +1997,6 @@ func mapSliceToTransmissions(maps []map[string]interface{}) []TransmissionResp {
tx.PathJSON = m["path_json"]
tx.Direction = m["direction"]
tx.Score = m["score"]
if rp, ok := m["resolved_path"].([]*string); ok {
tx.ResolvedPath = rp
}
result = append(result, tx)
}
return result
@@ -2021,9 +2018,6 @@ func mapSliceToObservations(maps []map[string]interface{}) []ObservationResp {
obs.RSSI = m["rssi"]
obs.PathJSON = m["path_json"]
obs.Timestamp = m["timestamp"]
if rp, ok := m["resolved_path"].([]*string); ok {
obs.ResolvedPath = rp
}
result = append(result, obs)
}
return result
+28 -177
View File
@@ -39,13 +39,10 @@ type StoreTx struct {
RSSI *float64
PathJSON string
Direction string
ResolvedPath []*string // resolved path from best observation
LatestSeen string // max observation timestamp (or FirstSeen if no observations)
// Cached parsed fields (set once, read many)
parsedPath []string // cached parsePathJSON result
pathParsed bool // whether parsedPath has been set
decodedOnce sync.Once // guards parsedDecoded
parsedDecoded map[string]interface{} // cached json.Unmarshal of DecodedJSON
parsedPath []string // cached parsePathJSON result
pathParsed bool // whether parsedPath has been set
// Dedup map: "observerID|pathJSON" → true for O(1) duplicate checks
obsKeys map[string]bool
}
@@ -61,25 +58,9 @@ type StoreObs struct {
RSSI *float64
Score *int
PathJSON string
ResolvedPath []*string // resolved full pubkeys, parallel to path_json; nil elements = unresolved
Timestamp string
}
// ParsedDecoded returns the parsed DecodedJSON map, caching the result.
// Thread-safe via sync.Once — the first call parses, subsequent calls return cached.
func (tx *StoreTx) ParsedDecoded() map[string]interface{} {
tx.decodedOnce.Do(func() {
if tx.DecodedJSON != "" {
json.Unmarshal([]byte(tx.DecodedJSON), &tx.parsedDecoded)
}
})
return tx.parsedDecoded
}
// distRebuildInterval is the minimum time between distance index rebuilds
// to avoid hot-looping on busy meshes.
const distRebuildInterval = 30 * time.Second
// PacketStore holds all transmissions in memory with indexes for fast queries.
type PacketStore struct {
mu sync.RWMutex
@@ -136,8 +117,6 @@ type PacketStore struct {
// computed during Load() and incrementally updated on ingest.
distHops []distHopRecord
distPaths []distPathRecord
distDirty bool // set when paths change; cleared after rebuild
distLast time.Time // last time distance index was rebuilt
// Cached GetNodeHashSizeInfo result — recomputed at most once every 15s
hashSizeInfoMu sync.Mutex
@@ -148,9 +127,6 @@ type PacketStore struct {
// Updated incrementally during Load/Ingest/Evict — avoids JSON parsing in GetPerfStoreStats.
advertPubkeys map[string]int // pubkey → number of advert packets referencing it
// Persisted neighbor graph for hop resolution at ingest time.
graph *NeighborGraph
// Eviction config and stats
retentionHours float64 // 0 = unlimited
maxMemoryMB int // 0 = unlimited
@@ -235,15 +211,11 @@ func (s *PacketStore) Load() error {
t0 := time.Now()
var loadSQL string
rpCol := ""
if s.db.hasResolvedPath {
rpCol = ",\n\t\t\t\to.resolved_path"
}
if s.db.isV3 {
loadSQL = `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type,
t.payload_type, t.payload_version, t.decoded_json,
o.id, obs.id, obs.name, o.direction,
o.snr, o.rssi, o.score, o.path_json, strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch')` + rpCol + `
o.snr, o.rssi, o.score, o.path_json, strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch')
FROM transmissions t
LEFT JOIN observations o ON o.transmission_id = t.id
LEFT JOIN observers obs ON obs.rowid = o.observer_idx
@@ -252,7 +224,7 @@ func (s *PacketStore) Load() error {
loadSQL = `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type,
t.payload_type, t.payload_version, t.decoded_json,
o.id, o.observer_id, o.observer_name, o.direction,
o.snr, o.rssi, o.score, o.path_json, o.timestamp` + rpCol + `
o.snr, o.rssi, o.score, o.path_json, o.timestamp
FROM transmissions t
LEFT JOIN observations o ON o.transmission_id = t.id
ORDER BY t.first_seen ASC, o.timestamp DESC`
@@ -272,16 +244,11 @@ func (s *PacketStore) Load() error {
var observerID, observerName, direction, pathJSON, obsTimestamp sql.NullString
var snr, rssi sql.NullFloat64
var score sql.NullInt64
var resolvedPathStr sql.NullString
scanArgs := []interface{}{&txID, &rawHex, &hash, &firstSeen, &routeType, &payloadType,
if err := rows.Scan(&txID, &rawHex, &hash, &firstSeen, &routeType, &payloadType,
&payloadVersion, &decodedJSON,
&obsID, &observerID, &observerName, &direction,
&snr, &rssi, &score, &pathJSON, &obsTimestamp}
if s.db.hasResolvedPath {
scanArgs = append(scanArgs, &resolvedPathStr)
}
if err := rows.Scan(scanArgs...); err != nil {
&snr, &rssi, &score, &pathJSON, &obsTimestamp); err != nil {
log.Printf("[store] scan error: %v", err)
continue
}
@@ -332,7 +299,6 @@ func (s *PacketStore) Load() error {
RSSI: nullFloatPtr(rssi),
Score: nullIntPtr(score),
PathJSON: obsPJ,
ResolvedPath: unmarshalResolvedPath(nullStrVal(resolvedPathStr)),
Timestamp: normalizeTimestamp(nullStrVal(obsTimestamp)),
}
@@ -363,7 +329,6 @@ func (s *PacketStore) Load() error {
// Precompute distance analytics (hop distances, path totals)
s.buildDistanceIndex()
s.distLast = time.Now()
s.loaded = true
elapsed := time.Since(t0)
@@ -394,7 +359,6 @@ func pickBestObservation(tx *StoreTx) {
tx.RSSI = best.RSSI
tx.PathJSON = best.PathJSON
tx.Direction = best.Direction
tx.ResolvedPath = best.ResolvedPath
tx.pathParsed = false // invalidate cached parsed path
}
@@ -419,8 +383,8 @@ func (s *PacketStore) indexByNode(tx *StoreTx) {
if !strings.Contains(tx.DecodedJSON, "ubKey") {
return
}
decoded := tx.ParsedDecoded()
if decoded == nil {
var decoded map[string]interface{}
if json.Unmarshal([]byte(tx.DecodedJSON), &decoded) != nil {
return
}
for _, field := range []string{"pubKey", "destPubKey", "srcPubKey"} {
@@ -443,8 +407,8 @@ func (s *PacketStore) trackAdvertPubkey(tx *StoreTx) {
if tx.PayloadType == nil || *tx.PayloadType != 4 || tx.DecodedJSON == "" {
return
}
d := tx.ParsedDecoded()
if d == nil {
var d map[string]interface{}
if json.Unmarshal([]byte(tx.DecodedJSON), &d) != nil {
return
}
pk := ""
@@ -594,9 +558,6 @@ func (s *PacketStore) QueryGroupedPackets(q PacketQuery) *PacketResult {
"rssi": floatPtrOrNil(tx.RSSI),
},
})
if tx.ResolvedPath != nil {
entries[len(entries)-1].latest["resolved_path"] = tx.ResolvedPath
}
}
s.mu.RUnlock()
@@ -860,7 +821,7 @@ func (s *PacketStore) GetPerfStoreStatsTyped() PerfPacketStoreStats {
SqliteOnly: false,
MaxPackets: 2386092,
EstimatedMB: estimatedMB,
MaxMB: s.maxMemoryMB,
MaxMB: 1024,
Indexes: PacketStoreIndexes{
ByHash: hashIdx,
ByObserver: observerIdx,
@@ -1022,9 +983,6 @@ func (s *PacketStore) IngestNewFromDB(sinceID, limit int) ([]map[string]interfac
limit = 100
}
// NOTE: The SQL query intentionally does NOT select resolved_path from the DB.
// New ingests always resolve fresh using the current prefix map and neighbor graph.
// On restart, Load() handles reading persisted resolved_path values. (review item #7)
var querySQL string
if s.db.isV3 {
querySQL = `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type,
@@ -1129,10 +1087,6 @@ func (s *PacketStore) IngestNewFromDB(sinceID, limit int) ([]map[string]interfac
broadcastTxs := make(map[int]*StoreTx) // track new transmissions for broadcast
var broadcastOrder []int
// Hoist getCachedNodesAndPM() once before the observation loop to avoid
// per-observation function calls (review item #1).
_, cachedPM := s.getCachedNodesAndPM()
for _, r := range tempRows {
if r.txID > newMaxID {
newMaxID = r.txID
@@ -1192,13 +1146,6 @@ func (s *PacketStore) IngestNewFromDB(sinceID, limit int) ([]map[string]interfac
PathJSON: r.pathJSON,
Timestamp: normalizeTimestamp(r.obsTS),
}
// Resolve path at ingest time using neighbor graph
// (cachedPM is hoisted before the observation loop to avoid per-obs function calls)
if r.pathJSON != "" && r.pathJSON != "[]" && cachedPM != nil {
obs.ResolvedPath = resolvePathForObs(r.pathJSON, r.observerID, tx, cachedPM, s.graph)
}
tx.Observations = append(tx.Observations, obs)
tx.obsKeys[dk] = true
tx.ObservationCount++
@@ -1242,7 +1189,7 @@ func (s *PacketStore) IngestNewFromDB(sinceID, limit int) ([]map[string]interfac
if cached, ok := hopCache[hop]; ok {
return cached
}
r, _, _ := pm.resolveWithContext(hop, nil, s.graph)
r := pm.resolve(hop)
hopCache[hop] = r
return r
}
@@ -1292,9 +1239,6 @@ func (s *PacketStore) IngestNewFromDB(sinceID, limit int) ([]map[string]interfac
"direction": strOrNil(obs.Direction),
"observation_count": tx.ObservationCount,
}
if obs.ResolvedPath != nil {
pkt["resolved_path"] = obs.ResolvedPath
}
// Broadcast map: top-level fields for live.js + nested packet for packets.js
broadcastMap := make(map[string]interface{}, len(pkt)+2)
for k, v := range pkt {
@@ -1329,36 +1273,6 @@ func (s *PacketStore) IngestNewFromDB(sinceID, limit int) ([]map[string]interfac
s.invalidateCachesFor(inv)
}
// Persist resolved paths and neighbor edges asynchronously (don't block ingest).
if len(broadcastTxs) > 0 && s.db != nil {
dbPath := s.db.path
var obsUpdates []persistObsUpdate
var edgeUpdates []persistEdgeUpdate
_, pm := s.getCachedNodesAndPM()
// Read graph ref under lock (it's set during startup and not replaced after,
// but reading under lock is safer — review item #5).
graphRef := s.graph
for _, tx := range broadcastTxs {
for _, obs := range tx.Observations {
if obs.ResolvedPath != nil {
rpJSON := marshalResolvedPath(obs.ResolvedPath)
if rpJSON != "" {
obsUpdates = append(obsUpdates, persistObsUpdate{obs.ID, rpJSON})
}
}
for _, ec := range extractEdgesFromObs(obs, tx, pm) {
edgeUpdates = append(edgeUpdates, persistEdgeUpdate{ec.A, ec.B, ec.Timestamp})
if graphRef != nil {
graphRef.upsertEdge(ec.A, ec.B, "", obs.ObserverID, obs.SNR, parseTimestamp(ec.Timestamp))
}
}
}
}
asyncPersistResolvedPathsAndEdges(dbPath, obsUpdates, edgeUpdates, "persist")
}
return result, newMaxID
}
@@ -1442,13 +1356,6 @@ func (s *PacketStore) IngestNewObservations(sinceObsID, limit int) []map[string]
updatedTxs := make(map[int]*StoreTx)
broadcastMaps := make([]map[string]interface{}, 0, len(obsRows))
// Track newly created observations for persistence — only these should be
// persisted, not all observations of each updated tx (fixes edge count inflation).
var newObs []*StoreObs
// Hoist getCachedNodesAndPM() before the loop — same pattern as IngestNewFromDB (review fix #1).
_, pm := s.getCachedNodesAndPM()
graphRef := s.graph
for _, r := range obsRows {
// Already ingested (e.g. by IngestNewFromDB in same cycle)
@@ -1482,18 +1389,9 @@ func (s *PacketStore) IngestNewObservations(sinceObsID, limit int) []map[string]
PathJSON: r.pathJSON,
Timestamp: normalizeTimestamp(r.timestamp),
}
// Resolve path at ingest time for late-arriving observations (review item #2).
if r.pathJSON != "" && r.pathJSON != "[]" {
if pm != nil {
obs.ResolvedPath = resolvePathForObs(r.pathJSON, r.observerID, tx, pm, s.graph)
}
}
tx.Observations = append(tx.Observations, obs)
tx.obsKeys[dk] = true
tx.ObservationCount++
newObs = append(newObs, obs)
if obs.Timestamp > tx.LatestSeen {
tx.LatestSeen = obs.Timestamp
}
@@ -1533,9 +1431,6 @@ func (s *PacketStore) IngestNewObservations(sinceObsID, limit int) []map[string]
"direction": strOrNil(obs.Direction),
"observation_count": tx.ObservationCount,
}
if obs.ResolvedPath != nil {
pkt["resolved_path"] = obs.ResolvedPath
}
broadcastMap := make(map[string]interface{}, len(pkt)+2)
for k, v := range pkt {
broadcastMap[k] = v
@@ -1575,65 +1470,31 @@ func (s *PacketStore) IngestNewObservations(sinceObsID, limit int) []map[string]
}
}
// Check if any paths changed (used for both distance rebuild and cache invalidation).
hasPathChanges := false
// Rebuild distance index if any paths changed (distances depend on path hops)
for txID, tx := range updatedTxs {
if tx.PathJSON != oldPaths[txID] {
hasPathChanges = true
s.buildDistanceIndex()
break
}
}
// Mark distance index dirty if any paths changed (rebuild is debounced)
if hasPathChanges {
s.distDirty = true
}
if s.distDirty && time.Since(s.distLast) > distRebuildInterval {
s.buildDistanceIndex()
s.distDirty = false
s.distLast = time.Now()
}
if len(updatedTxs) > 0 {
// Targeted cache invalidation: new observations always affect RF
// analytics; topology/distance/subpath caches only if paths changed.
// Channel and hash caches are unaffected by observation-only ingestion.
hasPathChanges := false
for txID, tx := range updatedTxs {
if tx.PathJSON != oldPaths[txID] {
hasPathChanges = true
break
}
}
s.invalidateCachesFor(cacheInvalidation{
hasNewObservations: true,
hasNewPaths: hasPathChanges,
})
}
// Persist resolved paths and neighbor edges asynchronously (review fix #3).
// Only process NEW observations — not all observations of each updated tx —
// to avoid edge count inflation and unnecessary UPDATEs for pre-existing data.
if len(newObs) > 0 && s.db != nil {
dbPath := s.db.path
var obsUpdates []persistObsUpdate
var edgeUpdates []persistEdgeUpdate
for _, obs := range newObs {
tx := s.byTxID[obs.TransmissionID]
if tx == nil {
continue
}
if obs.ResolvedPath != nil {
rpJSON := marshalResolvedPath(obs.ResolvedPath)
if rpJSON != "" {
obsUpdates = append(obsUpdates, persistObsUpdate{obs.ID, rpJSON})
}
}
for _, ec := range extractEdgesFromObs(obs, tx, pm) {
edgeUpdates = append(edgeUpdates, persistEdgeUpdate{ec.A, ec.B, ec.Timestamp})
if graphRef != nil {
graphRef.upsertEdge(ec.A, ec.B, "", obs.ObserverID, obs.SNR, parseTimestamp(ec.Timestamp))
}
}
}
asyncPersistResolvedPathsAndEdges(dbPath, obsUpdates, edgeUpdates, "obs-persist")
}
return broadcastMaps
}
@@ -1814,9 +1675,6 @@ func (s *PacketStore) enrichObs(obs *StoreObs) map[string]interface{} {
"score": intPtrOrNil(obs.Score),
"path_json": strOrNil(obs.PathJSON),
}
if obs.ResolvedPath != nil {
m["resolved_path"] = obs.ResolvedPath
}
if tx != nil {
m["hash"] = strOrNil(tx.Hash)
@@ -1850,9 +1708,6 @@ func txToMap(tx *StoreTx) map[string]interface{} {
"path_json": strOrNil(tx.PathJSON),
"direction": strOrNil(tx.Direction),
}
if tx.ResolvedPath != nil {
m["resolved_path"] = tx.ResolvedPath
}
// Include parsed path array to match Node.js output shape
if hops := txGetParsedPath(tx); len(hops) > 0 {
m["_parsedPath"] = hops
@@ -1862,7 +1717,7 @@ func txToMap(tx *StoreTx) map[string]interface{} {
// Include observations for expand=observations support (stripped by handler when not requested)
obs := make([]map[string]interface{}, 0, len(tx.Observations))
for _, o := range tx.Observations {
om := map[string]interface{}{
obs = append(obs, map[string]interface{}{
"id": o.ID,
"observer_id": strOrNil(o.ObserverID),
"observer_name": strOrNil(o.ObserverName),
@@ -1871,11 +1726,7 @@ func txToMap(tx *StoreTx) map[string]interface{} {
"path_json": strOrNil(o.PathJSON),
"timestamp": strOrNil(o.Timestamp),
"direction": strOrNil(o.Direction),
}
if o.ResolvedPath != nil {
om["resolved_path"] = o.ResolvedPath
}
obs = append(obs, om)
})
}
m["observations"] = obs
return m
@@ -2022,7 +1873,7 @@ func (s *PacketStore) buildDistanceIndex() {
if cached, ok := hopCache[hop]; ok {
return cached
}
r, _, _ := pm.resolveWithContext(hop, nil, s.graph)
r := pm.resolve(hop)
hopCache[hop] = r
return r
}
@@ -3683,7 +3534,7 @@ func (s *PacketStore) computeAnalyticsTopology(region string) map[string]interfa
if cached, ok := hopCache[hop]; ok {
return cached
}
r, _, _ := pm.resolveWithContext(hop, nil, s.graph)
r := pm.resolve(hop)
hopCache[hop] = r
return r
}
@@ -5674,7 +5525,7 @@ func (s *PacketStore) computeAnalyticsSubpaths(region string, minLen, maxLen, li
}
return hop
}
r, _, _ := pm.resolveWithContext(hop, nil, s.graph)
r := pm.resolve(hop)
hopCache[hop] = r
if r != nil {
return r.Name
@@ -5811,7 +5662,7 @@ func (s *PacketStore) GetSubpathDetail(rawHops []string) map[string]interface{}
// Resolve the requested hops
nodes := make([]map[string]interface{}, len(rawHops))
for i, hop := range rawHops {
r, _, _ := pm.resolveWithContext(hop, nil, s.graph)
r := pm.resolve(hop)
entry := map[string]interface{}{"hop": hop, "name": hop, "lat": nil, "lon": nil, "pubkey": nil}
if r != nil {
entry["name"] = r.Name
@@ -5890,7 +5741,7 @@ func (s *PacketStore) GetSubpathDetail(rawHops []string) map[string]interface{}
// Full parent path (resolved)
resolved := make([]string, len(hops))
for i, h := range hops {
r, _, _ := pm.resolveWithContext(h, nil, s.graph)
r := pm.resolve(h)
if r != nil {
resolved[i] = r.Name
} else {
-2
View File
@@ -240,7 +240,6 @@ type TransmissionResp struct {
SNR interface{} `json:"snr"`
RSSI interface{} `json:"rssi"`
PathJSON interface{} `json:"path_json"`
ResolvedPath []*string `json:"resolved_path,omitempty"`
Direction interface{} `json:"direction"`
Score interface{} `json:"score,omitempty"`
Observations []ObservationResp `json:"observations,omitempty"`
@@ -255,7 +254,6 @@ type ObservationResp struct {
SNR interface{} `json:"snr"`
RSSI interface{} `json:"rssi"`
PathJSON interface{} `json:"path_json"`
ResolvedPath []*string `json:"resolved_path,omitempty"`
Timestamp interface{} `json:"timestamp"`
}
+272
View File
@@ -0,0 +1,272 @@
# Spec: Server-side hop resolution at ingest — `resolved_path`
**Status:** Final
**Issue:** [#555](https://github.com/Kpa-clawbot/CoreScope/issues/555)
**Related:** [#482](https://github.com/Kpa-clawbot/CoreScope/issues/482), [#528](https://github.com/Kpa-clawbot/CoreScope/issues/528)
## Problem
Any place where 1, 2, or 3-byte prefixes must be resolved to actual full repeater public keys and friendly names should use affinity data first, geo data as fallback. Across frontend, backend, whatever. Efficiently — no 7-second waits, no recomputation, aggressive caching.
Currently, hop paths are stored as short uppercase hex prefixes in `path_json` (e.g. `["D6", "E3", "59"]`). Resolution to full pubkeys happens **client-side** via `HopResolver` (`public/hop-resolver.js`), which:
- Is slow — each page/component re-resolves independently
- Is inconsistent — different components may resolve the same prefix differently
- Cannot leverage the server's neighbor affinity graph, which has far richer context for disambiguation
- Causes redundant `/api/resolve-hops` calls from every client
## Solution
Resolve hop prefixes to full pubkeys **once at ingest time** on the server, using `resolveWithContext()` with 4-tier priority (affinity → geo → GPS → first match) and a **persisted neighbor graph**. Store the result as a new `resolved_path` column on observations alongside `path_json`.
## Design decisions (locked)
1. **`path_json` stays unchanged** — raw firmware prefixes, uppercase hex. Ground truth.
2. **`resolved_path` is a column on observations** — full 64-char lowercase hex pubkeys, `null` for unresolved.
3. **Resolved at ingest** using `resolveWithContext(hop, context, graph)` — 4-tier priority: affinity → geo → GPS → first match.
4. **`null` = unresolved** — ambiguous prefixes store `null`. Frontend falls back to prefix display.
5. **Both fields coexist** — not interchangeable. Different consumers use different fields.
## Persisted neighbor graph
### SQLite table: `neighbor_edges`
Thin and normalized. Stores ONLY the relationship. SNR, observer names, GPS, roles — all join from existing tables when needed. No duplication.
```sql
CREATE TABLE IF NOT EXISTS neighbor_edges (
node_a TEXT NOT NULL,
node_b TEXT NOT NULL,
count INTEGER DEFAULT 1,
last_seen TEXT,
PRIMARY KEY (node_a, node_b)
);
```
### Edge extraction rules (ADVERT vs non-ADVERT)
At ingest, for each packet:
- **ADVERT packets** (payload_type 4): originator pubkey is known from `decoded_json.pubKey`. Extract edge: `originator ↔ path[0]` (the first hop is a direct neighbor of the originator).
- **ALL packets**: observer pubkey is known. Extract edge: `observer ↔ path[last]` (the last hop is a direct neighbor of the observer).
- **Non-ADVERT packets**: originator is unknown (encrypted). ONLY extract `observer ↔ path[last]`.
- Each packet produces **1 or 2 edge upserts** depending on type.
Edge upsert uses canonical ordering (`node_a < node_b` lexicographically) to avoid duplicate edges:
```sql
INSERT INTO neighbor_edges (node_a, node_b, count, last_seen)
VALUES (min(?, ?), max(?, ?), 1, ?)
ON CONFLICT(node_a, node_b) DO UPDATE SET
count = count + 1,
last_seen = excluded.last_seen;
```
### In-memory structure
```go
type NeighborGraph struct {
edges map[string][]NeighborEdge // pubkey → list of neighbor edges
mu sync.RWMutex
}
```
### Cold startup and backfill
On startup:
```go
// 1. Query all edges from SQLite
rows := db.Query("SELECT node_a, node_b, count, last_seen FROM neighbor_edges")
// 2. Build in-memory graph
graph := NewNeighborGraph()
for rows.Next() {
var a, b string
var count int
var lastSeen string
rows.Scan(&a, &b, &count, &lastSeen)
graph.UpsertEdge(a, b, count, lastSeen)
}
// 3. Attach to PacketStore
store.graph = graph
```
1. **Load `neighbor_edges` from SQLite** → build in-memory graph (code above).
2. **If table empty (first run):** `BuildFromStore(packets)` — scan all existing packets, extract edges per the rules above, INSERT into `neighbor_edges`.
3. **Load observations from SQLite.**
4. **For observations without `resolved_path`:** resolve using the graph, UPDATE `resolved_path` in SQLite.
5. **Ready to serve.**
On subsequent runs, step 2 is skipped (table already populated). Step 4 only processes observations with NULL `resolved_path` (new or previously unresolved).
### Incremental update at ingest
Every edge upsert writes to **both** the in-memory graph and SQLite — they stay in sync. SQLite is the persistence layer, in-memory is the fast lookup layer.
```go
// Extract edge from packet
graph.UpsertEdge(nodeA, nodeB, 1, now)
// Also persist to SQLite
db.Exec(`INSERT INTO neighbor_edges (node_a, node_b, count, last_seen)
VALUES (?, ?, 1, ?)
ON CONFLICT(node_a, node_b) DO UPDATE SET
count = count + 1, last_seen = ?`, a, b, now, now)
```
## Data model
### Where does `resolved_path` live?
**On observations**, as a column:
```sql
ALTER TABLE observations ADD COLUMN resolved_path TEXT;
```
Rationale: Each observer sees the packet from a different vantage point. The same 2-char prefix may resolve to different full pubkeys depending on which observer's neighborhood is considered. The observer's own pubkey provides critical context for `resolveWithContext` (tier 2: neighbor affinity). Storing on observations preserves this per-observer resolution.
`resolved_path` is written in the same INSERT that creates the observation — one write, no double-write problem.
### Field shape
```
resolved_path TEXT -- JSON array: ["aabb...64chars", null, "ccdd...64chars"]
```
- Same length as the `path_json` array
- Each element is either a 64-char lowercase hex pubkey string, or `null`
- Stored as a JSON text column (same approach as `path_json`)
- Uses `omitempty` — absent from JSON when not set
## Every path resolution uses the graph — no exceptions
All existing `pm.resolve()` call sites MUST be migrated to `resolveWithContext` with the persisted graph. No "we'll get to it later."
### Call sites to migrate (exhaustive)
Found via `grep -n "pm.resolve" cmd/server/store.go`:
| Line | Function | Current | After |
|------|----------|---------|-------|
| 1192 | `IngestNewFromDB()` | `pm.resolve(hop)` | `resolveWithContext(hop, ctx, graph)` — resolve at ingest, store as `resolved_path` |
| 1876 | `buildDistanceIndex()` | `pm.resolve(hop)` | Read `resolved_path` from observation — already resolved at ingest |
| 3537 | `computeAnalyticsTopology()` | `pm.resolve(hop)` | Read `resolved_path` from observation |
| 5528 | `computeAnalyticsSubpaths()` | `pm.resolve(hop)` | Read `resolved_path` from observation |
| 5665 | `GetSubpathDetail()` | `pm.resolve(hop)` | `resolveWithContext(hop, ctx, graph)` — ad-hoc resolution for user-provided hops |
| 5744 | `GetSubpathDetail()` | `pm.resolve(h)` | `resolveWithContext(h, ctx, graph)` — same function, second usage |
**After migration:** `pm.resolve()` (naive prefix-only lookup) is dead code. Remove it. All resolution goes through `resolveWithContext` which uses the persisted neighbor graph for affinity-based disambiguation.
## Ingest pipeline changes
### Where resolution happens
In `PacketStore.IngestNewFromDB()` in `cmd/server/store.go`. For new observations, resolution happens during the observation INSERT — same write. For backfill (cold startup), it's a separate UPDATE pass.
Note on ordering: edge upserts (step 5) happen **after** resolution (step 3-4). This means the very first packet for a new neighbor pair resolves without that edge in the graph yet. This is acceptable — the affinity tier will miss, but geo/GPS/first-match tiers still work. On the next packet, the edge exists and affinity kicks in.
Resolution flow per observation:
1. Parse `path_json` into hop prefixes
2. Build context pubkeys from the observation (observer pubkey, source/dest from decoded packet)
3. Call `resolveWithContext(hop, contextPubkeys, neighborGraph)` for each hop
4. Store result as `resolved_path` column on the observation (same INSERT)
5. Upsert neighbor edges into `neighbor_edges` table (incremental update)
### Performance
`resolveWithContext` does:
- Prefix map lookup (map access, O(1))
- Optional neighbor graph check (small map lookups)
- No DB queries, no network calls
Per-hop cost: ~15μs. A typical packet has 05 hops. At 100 packets/second ingest rate, this adds <0.5ms total overhead per second. **Negligible.**
## All consumers use `resolved_path`
| Consumer | Before | After |
|---|---|---|
| Packets detail path names | Client HopResolver (naive) | Read `resolved_path` |
| Map Show Route | Client HopResolver (naive) | Read `resolved_path` |
| Live map animated paths | Client HopResolver (naive) | Read `resolved_path` |
| Node detail paths | Client HopResolver (naive) | Read `resolved_path` |
| Analytics topology | Server `pm.resolve()` (naive) | Read `resolved_path` from observations |
| Analytics subpaths | Server `pm.resolve()` (naive) | Read `resolved_path` from observations |
| Analytics hop distances | Server `pm.resolve()` (naive) | Read `resolved_path` from observations |
| Subpath detail | Server `pm.resolve()` (naive) | `resolveWithContext` with graph |
| Show Neighbors | Server neighbors API | Already correct |
| `/api/resolve-hops` | Server `resolveWithContext` | Already correct |
| Hex breakdown display | `path_json` raw | Unchanged — shows raw bytes |
## WebSocket broadcast
Include `resolved_path` in broadcast messages. Resolution happens before broadcast assembly — negligible latency impact. The WS broadcast already includes `path_json`; `resolved_path` is added alongside it.
## API changes
### Endpoints that return `resolved_path`
All endpoints that currently return `path_json` also return `resolved_path`:
- `GET /api/packets` — transmission-level (use best observation's `resolved_path`)
- `GET /api/packets/:hash` — per-observation detail
- `GET /api/packets/:hash/observations` — each observation includes its own `resolved_path`
- WebSocket broadcast messages — per-observation
### `/api/resolve-hops`
**Kept.** Useful for ad-hoc resolution of arbitrary prefixes (debug tools, clients resolving prefixes not associated with a packet). Not deprecated.
## Pubkey case convention
- **DB/API:** lowercase
- **`path_json` display prefixes:** uppercase (raw firmware)
- **`resolved_path`:** lowercase full pubkeys
- **Comparison code:** normalizes to lowercase
## Backward compatibility
- Old observations without `resolved_path`: resolved during cold startup backfill (step 4). If still `null` after backfill, frontend falls back to client-side HopResolver.
- `resolved_path` field uses `omitempty` — absent from JSON when not set.
### Fallback pattern (frontend)
```javascript
function getResolvedHops(packet) {
if (packet.resolved_path) return packet.resolved_path;
// Fall back to client-side resolution for old packets
return resolveHopsClientSide(packet.path_json);
}
```
## Implementation milestones
### M1: Persist graph to SQLite + load on startup + incremental updates at ingest
- Create `neighbor_edges` table in SQLite (schema above)
- On first run: `BuildFromStore(packets)` — scan all packets, extract edges per ADVERT/non-ADVERT rules, INSERT into table
- On subsequent runs: load from SQLite → build in-memory graph (instant startup)
- Upsert edges incrementally during packet ingest
- Graph lives on `PacketStore`, not `Server`
- Tests: graph persistence, load, incremental update, ADVERT vs non-ADVERT edge extraction
### M2: Add `resolved_path` column to observations + resolve at ingest
- `ALTER TABLE observations ADD COLUMN resolved_path TEXT`
- Add `ResolvedPath []*string` to `Observation` struct
- Resolve during `IngestNewFromDB` — same INSERT, one write
- Cold startup backfill: resolve observations with NULL `resolved_path`, UPDATE in SQLite
- Migrate ALL 6 `pm.resolve()` call sites to `resolveWithContext` or read from `resolved_path`
- Remove dead `pm.resolve()` code
- Tests: unit test resolution at ingest, verify stored values, verify all call sites use graph
### M3: Update all API responses to include `resolved_path`
- Include `resolved_path` in all packet/observation API responses
- Include in WebSocket broadcast messages
- Tests: verify API response shape, WS broadcast shape
### M4: Update frontend consumers to prefer `resolved_path`
- Update `packets.js`, `map.js`, `live.js`, `analytics.js`, `nodes.js`
- Add fallback to `path_json` + `HopResolver` for old packets
- `hop-resolver.js` becomes fallback only
- Tests: Playwright tests for path display
-3
View File
@@ -463,9 +463,6 @@ function navigate() {
currentPage = basePage;
const app = document.getElementById('app');
// Pages with fixed-height containers (maps, virtual-scroll, split-panels)
const fixedPages = { packets: 1, nodes: 1, map: 1, live: 1, channels: 1, 'audio-lab': 1 };
app.classList.toggle('app-fixed', basePage in fixedPages);
if (pages[basePage]?.init) {
const t0 = performance.now();
pages[basePage].init(app, routeParam);
+1 -1
View File
@@ -48,7 +48,7 @@ if (typeof window !== 'undefined') window.comparePacketSets = comparePacketSets;
packetsB = [];
currentView = 'summary';
app.innerHTML = '<div class="compare-page" style="padding:16px">' +
app.innerHTML = '<div class="compare-page" style="overflow-y:auto;height:calc(100vh - 56px);padding:16px">' +
'<div class="page-header" style="display:flex;align-items:center;gap:12px;margin-bottom:16px">' +
'<a href="#/observers" class="btn-icon" title="Back to Observers" aria-label="Back">\u2190</a>' +
'<h2 style="margin:0">\uD83D\uDD0D Observer Comparison</h2>' +
+2 -1
View File
@@ -1,6 +1,7 @@
/* === CoreScope — home.css === */
/* Home page now uses body scroll (no #app override needed — see style.css) */
/* Override #app overflow:hidden for home page scrolling */
#app:has(.home-hero), #app:has(.home-chooser) { overflow-y: auto; }
/* Chooser */
.home-chooser {
+1 -29
View File
@@ -8,7 +8,6 @@ window.HopResolver = (function() {
const MAX_HOP_DIST = 1.8; // ~200km in degrees
const REGION_RADIUS_KM = 300;
let prefixIdx = {}; // lowercase hex prefix → [node, ...]
let pubkeyIdx = {}; // full lowercase pubkey → node (O(1) lookup)
let nodesList = [];
let observerIataMap = {}; // observer_id → iata
let iataCoords = {}; // iata → {lat, lon}
@@ -36,11 +35,9 @@ window.HopResolver = (function() {
function init(nodes, opts) {
nodesList = nodes || [];
prefixIdx = {};
pubkeyIdx = {};
for (const n of nodesList) {
if (!n.public_key) continue;
const pk = n.public_key.toLowerCase();
pubkeyIdx[pk] = n;
for (let len = 1; len <= 3; len++) {
const p = pk.slice(0, len * 2);
if (!prefixIdx[p]) prefixIdx[p] = [];
@@ -268,30 +265,5 @@ window.HopResolver = (function() {
return affinityMap[pubkeyA][pubkeyB] || 0;
}
/**
* Resolve hops using server-provided resolved_path (full pubkeys).
* Returns the same format as resolve() { [hop]: { name, pubkey, ... } }.
* resolved_path is an array aligned with path_json: each element is a
* 64-char lowercase hex pubkey or null. Skips entries that are null.
*/
function resolveFromServer(hops, resolvedPath) {
if (!hops || !resolvedPath || hops.length !== resolvedPath.length) return {};
var result = {};
for (var i = 0; i < hops.length; i++) {
var hop = hops[i];
var pubkey = resolvedPath[i];
if (!pubkey) continue; // null = unresolved, leave for client-side fallback
// O(1) lookup via pubkeyIdx built during init()
var node = pubkeyIdx[pubkey.toLowerCase()] || null;
result[hop] = {
name: node ? node.name : pubkey.slice(0, 8),
pubkey: pubkey,
candidates: node ? [{ name: node.name, pubkey: pubkey, lat: node.lat, lon: node.lon }] : [],
conflicts: []
};
}
return result;
}
return { init: init, resolve: resolve, resolveFromServer: resolveFromServer, ready: ready, haversineKm: haversineKm, setAffinity: setAffinity, getAffinity: getAffinity };
return { init: init, resolve: resolve, ready: ready, haversineKm: haversineKm, setAffinity: setAffinity, getAffinity: getAffinity };
})();
+13 -34
View File
@@ -457,7 +457,6 @@
id: pkt.id, hash: pkt.hash,
raw: pkt.raw_hex,
path_json: pkt.path_json,
resolved_path: pkt.resolved_path,
_ts: new Date(pkt.timestamp || pkt.created_at).getTime(),
decoded: { header: { payloadTypeName: typeName }, payload: raw, path: { hops } },
snr: pkt.snr, rssi: pkt.rssi, observer: pkt.observer_name
@@ -1344,7 +1343,7 @@
html += `<h4 style="font-size:12px;margin:12px 0 6px;color:var(--text-muted);">Recent Packets</h4>
<div style="font-size:11px;max-height:200px;overflow-y:auto;">` +
recent.slice(0, 10).map(p => `<div style="padding:2px 0;display:flex;justify-content:space-between;">
<a href="#/packets/${encodeURIComponent(p.hash || '')}" style="color:var(--accent);text-decoration:none;">${escapeHtml(p.payload_type || '?')}${transportBadge(p.route_type)}${p.observation_count > 1 ? ' <span class="badge badge-obs" style="font-size:9px">👁 ' + p.observation_count + '</span>' : ''}</a>
<a href="#/packets/${encodeURIComponent(p.hash || '')}" style="color:var(--accent);text-decoration:none;">${escapeHtml(p.payload_type || '?')}${p.observation_count > 1 ? ' <span class="badge badge-obs" style="font-size:9px">👁 ' + p.observation_count + '</span>' : ''}</a>
<span style="color:var(--text-muted)">${formatLiveTimestampHtml(p.timestamp)}</span>
</div>`).join('') +
'</div>';
@@ -1549,7 +1548,7 @@
item.innerHTML = `
<span class="feed-icon" style="color:${color}">${icon}</span>
<span class="feed-type" style="color:${color}">${typeName}</span>
${transportBadge(pkt.route_type)}${hopStr}${obsBadge}
${hopStr}${obsBadge}
<span class="feed-text">${escapeHtml(preview)}</span>
<span class="feed-time">${formatLiveTimestampHtml(group.latestTs || Date.now())}</span>
`;
@@ -1651,7 +1650,6 @@
}
delete nodeMarkers[key];
delete nodeData[key];
delete nodeActivity[key];
pruned = true;
}
} else if (marker && marker._staleDimmed) {
@@ -1667,17 +1665,12 @@
if (_el2) _el2.textContent = Object.keys(nodeMarkers).length;
if (window.HopResolver) HopResolver.init(Object.values(nodeData));
}
// Prune orphaned nodeActivity entries (nodes removed above or never tracked)
for (var aKey in nodeActivity) {
if (!(aKey in nodeData)) delete nodeActivity[aKey];
}
}
// Expose for testing
window._livePruneStaleNodes = pruneStaleNodes;
window._liveNodeMarkers = function() { return nodeMarkers; };
window._liveNodeData = function() { return nodeData; };
window._liveNodeActivity = function() { return nodeActivity; };
window._vcrFormatTime = vcrFormatTime;
window._liveDbPacketToLive = dbPacketToLive;
window._liveExpandToBufferEntries = expandToBufferEntries;
@@ -1862,7 +1855,7 @@
var pathKey = hops.join(',');
if (seenPathKeys.has(pathKey)) continue;
seenPathKeys.add(pathKey);
var hopPositions = resolveHopPositions(hops, qp, window.getResolvedPath ? getResolvedPath(qpkt) : null);
var hopPositions = resolveHopPositions(hops, qp);
if (hopPositions.length >= 2) {
allPaths.push({ hopPositions: hopPositions, raw: qpkt.raw || first.raw });
} else if (hopPositions.length === 1) {
@@ -1899,29 +1892,15 @@
}
}
function resolveHopPositions(hops, payload, resolvedPath) {
// Prefer server-side resolved_path when available
var resolvedMap;
if (resolvedPath && resolvedPath.length === hops.length && window.HopResolver && HopResolver.ready()) {
resolvedMap = HopResolver.resolveFromServer(hops, resolvedPath);
// Fill in any null entries from client-side fallback, preserving sender GPS context
var nullHops = hops.filter(function(h, i) { return !resolvedPath[i] && !resolvedMap[h]; });
if (nullHops.length) {
const originLat = payload.lat != null && !(payload.lat === 0 && payload.lon === 0) ? payload.lat : null;
const originLon = payload.lon != null && !(payload.lon === 0 && payload.lon === 0) ? payload.lon : null;
var fallback = HopResolver.resolve(nullHops, originLat, originLon, null, null, null);
for (var k in fallback) resolvedMap[k] = fallback[k];
}
} else {
// Delegate to shared HopResolver (from hop-resolver.js) instead of reimplementing
const originLat = payload.lat != null && !(payload.lat === 0 && payload.lon === 0) ? payload.lat : null;
const originLon = payload.lon != null && !(payload.lon === 0 && payload.lon === 0) ? payload.lon : null;
function resolveHopPositions(hops, payload) {
// Delegate to shared HopResolver (from hop-resolver.js) instead of reimplementing
const originLat = payload.lat != null && !(payload.lat === 0 && payload.lon === 0) ? payload.lat : null;
const originLon = payload.lon != null && !(payload.lon === 0 && payload.lon === 0) ? payload.lon : null;
// Use HopResolver if available and initialized, otherwise fall back to simple lookup
resolvedMap = (window.HopResolver && HopResolver.ready())
? HopResolver.resolve(hops, originLat, originLon, null, null, null)
: {};
}
// Use HopResolver if available and initialized, otherwise fall back to simple lookup
const resolvedMap = (window.HopResolver && HopResolver.ready())
? HopResolver.resolve(hops, originLat, originLon, null, null, null)
: {};
// Convert HopResolver's map format to the array format live.js expects: {key, pos, name, known}
const raw = hops.map(hop => {
@@ -2505,7 +2484,7 @@
item.innerHTML = `
<span class="feed-icon" style="color:${color}">${icon}</span>
<span class="feed-type" style="color:${color}">${typeName}</span>
${transportBadge(pkt.route_type)}${hopStr}${obsBadge}
${hopStr}${obsBadge}
<span class="feed-text">${escapeHtml(preview)}</span>
<span class="feed-time">${formatLiveTimestampHtml(pkt._ts || Date.now())}</span>
`;
@@ -2573,7 +2552,7 @@
item.innerHTML = `
<span class="feed-icon" style="color:${color}">${icon}</span>
<span class="feed-type" style="color:${color}">${typeName}</span>
${transportBadge(pkt.route_type)}${hopStr}${obsBadge}
${hopStr}${obsBadge}
<span class="feed-text">${escapeHtml(preview)}</span>
<span class="feed-time">${formatLiveTimestampHtml(pkt._ts || Date.now())}</span>
`;
+1 -1
View File
@@ -51,7 +51,7 @@
const nodeName = escapeHtml(n.name || n.public_key.slice(0, 12));
container.innerHTML = `
<div style="max-width:1000px;margin:0 auto;padding:12px 16px">
<div style="max-width:1000px;margin:0 auto;padding:12px 16px;height:100%;overflow-y:auto">
<div style="margin-bottom:12px">
<a href="#/nodes/${encodeURIComponent(n.public_key)}" style="color:var(--accent);text-decoration:none;font-size:12px"> Back to ${nodeName}</a>
<h2 style="margin:4px 0 2px;font-size:18px">📊 ${nodeName} Analytics</h2>
+1 -1
View File
@@ -37,7 +37,7 @@
}
app.innerHTML = `
<div class="observer-detail-page" style="padding:16px">
<div class="observer-detail-page" style="overflow-y:auto;height:calc(100vh - 56px);padding:16px">
<div class="page-header" style="display:flex;align-items:center;gap:12px;margin-bottom:16px">
<a href="#/observers" class="btn-icon" title="Back to Observers" aria-label="Back"></a>
<h2 style="margin:0" id="obsTitle">Observer Detail</h2>
-18
View File
@@ -28,27 +28,9 @@ window.getParsedPath = function getParsedPath(p) {
window.clearParsedCache = function clearParsedCache(p) {
delete p._parsedPath;
delete p._parsedDecoded;
delete p._parsedResolvedPath;
return p;
};
/**
* Parse resolved_path (server-side resolved full pubkeys).
* Returns array of pubkey strings (or null entries) if present, or null if absent.
* Cached as _parsedResolvedPath on the packet object.
*/
window.getResolvedPath = function getResolvedPath(p) {
if (p._parsedResolvedPath !== undefined) return p._parsedResolvedPath;
var raw = p.resolved_path;
if (!raw) { p._parsedResolvedPath = null; return null; }
if (typeof raw !== 'string') {
p._parsedResolvedPath = Array.isArray(raw) ? raw : null;
return p._parsedResolvedPath;
}
try { p._parsedResolvedPath = JSON.parse(raw) || null; } catch (e) { p._parsedResolvedPath = null; }
return p._parsedResolvedPath;
};
window.getParsedDecoded = function getParsedDecoded(p) {
if (p._parsedDecoded !== undefined) return p._parsedDecoded || {};
var raw = p.decoded_json;
+36 -83
View File
@@ -171,29 +171,6 @@
}
}
/**
* Pre-populate hopNameCache from server-side resolved_path on packets.
* Packets with resolved_path skip client-side HopResolver entirely.
* Must call ensureHopResolver() first so nodesList is available for name lookup.
*/
async function cacheResolvedPaths(packets) {
if (!packets || !packets.length) return;
let needsInit = false;
for (const p of packets) {
const rp = getResolvedPath(p);
if (rp) { needsInit = true; break; }
}
if (!needsInit) return;
await ensureHopResolver();
for (const p of packets) {
const rp = getResolvedPath(p);
if (!rp) continue;
const hops = getParsedPath(p);
const resolved = HopResolver.resolveFromServer(hops, rp);
Object.assign(hopNameCache, resolved);
}
}
function renderHop(h, observerId) {
// Use per-packet cache key if observer context available (ambiguous hops differ by region)
const cacheKey = observerId ? h + ':' + observerId : h;
@@ -292,7 +269,7 @@
const obs = data.observations.find(o => String(o.id) === String(obsTarget));
if (obs) {
expandedHashes.add(h);
const obsPacket = {...data.packet, observer_id: obs.observer_id, observer_name: obs.observer_name, snr: obs.snr, rssi: obs.rssi, path_json: obs.path_json, resolved_path: obs.resolved_path, timestamp: obs.timestamp, first_seen: obs.timestamp};
const obsPacket = {...data.packet, observer_id: obs.observer_id, observer_name: obs.observer_name, snr: obs.snr, rssi: obs.rssi, path_json: obs.path_json, timestamp: obs.timestamp, first_seen: obs.timestamp};
clearParsedCache(obsPacket);
selectPacket(obs.id, h, {packet: obsPacket, breakdown: data.breakdown, observations: data.observations}, obs.id);
} else {
@@ -394,16 +371,9 @@
if (!filtered.length) return;
// Resolve any new hops, then update and re-render
// Pre-populate from server-side resolved_path, then fall back for remaining
const newHops = new Set();
for (const p of filtered) {
const rp = getResolvedPath(p);
const hops = getParsedPath(p);
if (rp && rp.length === hops.length && window.HopResolver && HopResolver.ready()) {
const resolved = HopResolver.resolveFromServer(hops, rp);
Object.assign(hopNameCache, resolved);
}
try { hops.forEach(h => { if (!(h in hopNameCache)) newHops.add(h); }); } catch {}
try { getParsedPath(p).forEach(h => { if (!(h in hopNameCache)) newHops.add(h); }); } catch {}
}
(newHops.size ? resolveHops([...newHops]) : Promise.resolve()).then(() => {
if (groupByHash) {
@@ -554,10 +524,7 @@
totalCount = flat.length;
}
// Pre-resolve from server-side resolved_path (preferred, no client-side disambiguation needed)
await cacheResolvedPaths(packets);
// Pre-resolve all path hops to node names (fallback for packets without resolved_path)
// Pre-resolve all path hops to node names
const allHops = new Set();
for (const p of packets) {
try { getParsedPath(p).forEach(h => allHops.add(h)); } catch {}
@@ -580,22 +547,19 @@
// Ambiguous hops are already resolved by HopResolver client-side
// No need for per-observer server API calls
// Restore expanded group children (parallel fetch, Map lookup)
// Restore expanded group children
if (groupByHash && expandedHashes.size > 0) {
const expandedArr = [...expandedHashes];
const results = await Promise.all(expandedArr.map(hash => {
const group = hashIndex.get(hash);
if (!group) return { hash, group: null, data: null };
return api(`/packets?hash=${hash}&limit=20`)
.then(data => ({ hash, group, data }))
.catch(() => ({ hash, group, data: null }));
}));
for (const { hash, group, data } of results) {
if (!group) {
for (const hash of expandedHashes) {
const group = packets.find(p => p.hash === hash);
if (group) {
try {
const childData = await api(`/packets?hash=${hash}&limit=20`);
group._children = childData.packets || [];
sortGroupChildren(group);
} catch {}
} else {
// Group no longer in results — remove from expanded
expandedHashes.delete(hash);
} else if (data) {
group._children = data.packets || [];
sortGroupChildren(group);
}
}
}
@@ -1048,11 +1012,11 @@
}
else if (action === 'select-observation') {
const parentHash = row.dataset.parentHash;
const group = hashIndex.get(parentHash);
const group = packets.find(p => p.hash === parentHash);
const child = group?._children?.find(c => String(c.id) === String(value));
if (child) {
const parentData = group._fetchedData;
const obsPacket = parentData ? {...parentData.packet, observer_id: child.observer_id, observer_name: child.observer_name, snr: child.snr, rssi: child.rssi, path_json: child.path_json, resolved_path: child.resolved_path, timestamp: child.timestamp, first_seen: child.timestamp} : child;
const obsPacket = parentData ? {...parentData.packet, observer_id: child.observer_id, observer_name: child.observer_name, snr: child.snr, rssi: child.rssi, path_json: child.path_json, timestamp: child.timestamp, first_seen: child.timestamp} : child;
if (parentData) { clearParsedCache(obsPacket); }
selectPacket(child.id, parentHash, {packet: obsPacket, breakdown: parentData?.breakdown, observations: parentData?.observations}, child.id);
}
@@ -1525,18 +1489,11 @@
} catch {}
}
// Resolve hops: prefer server-side resolved_path, fall back to client-side HopResolver
// Re-resolve hops using client-side HopResolver with sender GPS context
if (pathHops.length) {
try {
const serverResolved = getResolvedPath(pkt);
let resolved;
if (serverResolved && serverResolved.length === pathHops.length) {
await ensureHopResolver();
resolved = HopResolver.resolveFromServer(pathHops, serverResolved);
} else {
await ensureHopResolver();
resolved = HopResolver.resolve(pathHops);
}
await ensureHopResolver();
const resolved = HopResolver.resolve(pathHops);
if (resolved) {
for (const [k, v] of Object.entries(resolved)) {
hopNameCache[k] = v;
@@ -1713,25 +1670,22 @@
if (routeBtn && pathHops.length) {
routeBtn.addEventListener('click', async () => {
try {
// Prefer server-side resolved_path if available
const serverResolved = getResolvedPath(pkt);
let resolvedKeys;
if (serverResolved && serverResolved.length === pathHops.length) {
// Use server-resolved pubkeys, fall back to short prefix for null entries
resolvedKeys = pathHops.map((h, i) => serverResolved[i] || h);
} else {
// Fall back to client-side HopResolver
const senderLat = decoded.lat || decoded.latitude;
const senderLon = decoded.lon || decoded.longitude;
let obsLat = null, obsLon = null;
const obsId = obsName(pkt.observer_id);
await ensureHopResolver();
const data = { resolved: HopResolver.resolve(pathHops, senderLat || null, senderLon || null, obsLat, obsLon, pkt.observer_id) };
resolvedKeys = pathHops.map(h => {
const r = data.resolved?.[h];
return r?.pubkey || h;
});
// Anchor disambiguation from sender's location if known (e.g. ADVERT lat/lon)
const senderLat = decoded.lat || decoded.latitude;
const senderLon = decoded.lon || decoded.longitude;
// Resolve observer position for backward-pass anchor
let obsLat = null, obsLon = null;
const obsId = obsName(pkt.observer_id);
if (obsId && HopResolver.ready()) {
// Try to find observer in nodes list by name — best effort
}
await ensureHopResolver();
const data = { resolved: HopResolver.resolve(pathHops, senderLat || null, senderLon || null, obsLat, obsLon, pkt.observer_id) };
// Pass full pubkeys (client-disambiguated) to map, falling back to short prefix
const resolvedKeys = pathHops.map(h => {
const r = data.resolved?.[h];
return r?.pubkey || h;
});
// Build origin info for the sender node
const origin = {};
if (decoded.pubKey) origin.pubkey = decoded.pubKey;
@@ -2055,15 +2009,14 @@
const data = await api(`/packets/${hash}`);
const pkt = data.packet;
if (!pkt) return;
const group = hashIndex.get(hash);
const group = packets.find(p => p.hash === hash);
if (group && data.observations) {
group._children = data.observations.map(o => clearParsedCache({...pkt, ...o, _isObservation: true}));
group._fetchedData = data;
// Sort children based on current sort mode
sortGroupChildren(group);
}
// Resolve hops from children: prefer server-side resolved_path
await cacheResolvedPaths(group?._children || []);
// Resolve any new hops from children
const childHops = new Set();
for (const c of (group?._children || [])) {
try { getParsedPath(c).forEach(h => childHops.add(h)); } catch {}
+1 -1
View File
@@ -5,7 +5,7 @@
let interval = null;
async function render(app) {
app.innerHTML = '<div id="perfWrapper" style="padding:16px 24px;"><h2>⚡ Performance Dashboard</h2><div id="perfContent">Loading...</div></div>';
app.innerHTML = '<div id="perfWrapper" style="height:100%;overflow-y:auto;padding:16px 24px;"><h2>⚡ Performance Dashboard</h2><div id="perfContent">Loading...</div></div>';
await refresh();
}
+4 -9
View File
@@ -181,12 +181,7 @@ a:focus-visible, button:focus-visible, input:focus-visible, select:focus-visible
}
/* === Layout === */
/* Default: body-scroll mode content pushes beyond viewport, iOS status-bar
tap-to-scroll works because <body> is the scroll container. Pages that need
a fixed-height container (maps, virtual-scroll, split-panels) add
.app-fixed via the router so their children can use height:100%. */
#app { min-height: calc(100vh - 52px); min-height: calc(100dvh - 52px); }
#app.app-fixed { height: calc(100vh - 52px); height: calc(100dvh - 52px); min-height: 0; overflow: hidden; }
#app { height: calc(100vh - 52px); height: calc(100dvh - 52px); overflow: hidden; }
.split-layout {
display: flex; height: 100%; overflow: hidden;
@@ -679,7 +674,7 @@ button.ch-item.selected { background: var(--selected-bg); }
.advert-info { font-size: 12px; line-height: 1.5; }
/* === Traces Page === */
.traces-page { padding: 16px; max-width: var(--trace-max-width, 95vw); margin: 0 auto; }
.traces-page { padding: 16px; max-width: var(--trace-max-width, 95vw); margin: 0 auto; overflow-y: auto; height: 100%; }
.trace-search {
display: flex; gap: 8px; margin-bottom: 20px;
}
@@ -751,7 +746,7 @@ button.ch-item.selected { background: var(--selected-bg); }
::-webkit-scrollbar-thumb:hover { background: var(--text-muted); }
/* === Observers Page === */
.observers-page { padding: 20px; max-width: 1200px; margin: 0 auto; }
.observers-page { padding: 20px; max-width: 1200px; margin: 0 auto; overflow-y: auto; height: calc(100vh - 56px); }
.obs-summary { display: flex; gap: 20px; margin-bottom: 16px; flex-wrap: wrap; }
.obs-stat { display: flex; align-items: center; gap: 6px; font-size: 14px; color: var(--text-muted); }
.health-dot { width: 10px; height: 10px; border-radius: 50%; display: inline-block; flex-shrink: 0; }
@@ -1143,7 +1138,7 @@ button.ch-item.ch-item-encrypted .ch-badge { filter: grayscale(0.6); }
.node-activity-time { color: var(--text-muted); white-space: nowrap; min-width: 70px; font-size: 12px; }
/* Analytics page */
.analytics-page { padding: 16px 24px; max-width: 1600px; margin: 0 auto; }
.analytics-page { padding: 16px 24px; max-width: 1600px; margin: 0 auto; overflow-y: auto; height: 100%; }
.analytics-header { margin-bottom: 20px; }
.analytics-header h2 { margin: 0 0 4px; }
.analytics-card { background: var(--card-bg); border: 1px solid var(--border); border-radius: 8px; padding: 16px; margin-bottom: 16px; }
-137
View File
@@ -564,93 +564,6 @@ console.log('\n=== hop-resolver.js ===');
});
}
// ===== resolveFromServer (hop-resolver.js, M4 #555) =====
console.log('\n=== resolveFromServer (hop-resolver.js) ===');
{
const ctx = makeSandbox();
ctx.IATA_COORDS_GEO = {};
loadInCtx(ctx, 'public/hop-resolver.js');
const HR = ctx.window.HopResolver;
test('resolveFromServer works without init (uses pubkey prefix as name)', () => {
const pk = 'abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890';
const result = HR.resolveFromServer(['AB'], [pk]);
assert.strictEqual(result['AB'].name, pk.slice(0, 8));
assert.strictEqual(result['AB'].pubkey, pk);
});
test('resolveFromServer with matching node', () => {
const pubkey = 'abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890';
HR.init([{ public_key: pubkey, name: 'NodeA', lat: 37.3, lon: -122.0 }]);
const result = HR.resolveFromServer(['AB'], [pubkey]);
assert.strictEqual(result['AB'].name, 'NodeA');
assert.strictEqual(result['AB'].pubkey, pubkey);
assert.ok(!result['AB'].ambiguous);
});
test('resolveFromServer with null entry skips it', () => {
const pubkey = 'abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890';
HR.init([{ public_key: pubkey, name: 'NodeA', lat: 37.3, lon: -122.0 }]);
const result = HR.resolveFromServer(['AB', 'CD'], [pubkey, null]);
assert.strictEqual(result['AB'].name, 'NodeA');
assert.ok(!('CD' in result)); // null entries are skipped
});
test('resolveFromServer with unknown pubkey uses prefix', () => {
HR.init([{ public_key: 'aaaa0000', name: 'Other' }]);
const unknownPk = '1111111111111111111111111111111111111111111111111111111111111111';
const result = HR.resolveFromServer(['AB'], [unknownPk]);
assert.strictEqual(result['AB'].name, unknownPk.slice(0, 8));
assert.strictEqual(result['AB'].pubkey, unknownPk);
});
test('resolveFromServer mismatched lengths returns empty', () => {
HR.init([{ public_key: 'abcdef1234567890', name: 'NodeA' }]);
const result = HR.resolveFromServer(['AB', 'CD'], ['abcdef1234567890']);
assert.strictEqual(Object.keys(result).length, 0);
});
}
// ===== getResolvedPath (packet-helpers.js, M4 #555) =====
console.log('\n=== getResolvedPath (packet-helpers.js) ===');
{
const ctx = makeSandbox();
loadInCtx(ctx, 'public/packet-helpers.js');
const getResolvedPath = ctx.window.getResolvedPath;
test('getResolvedPath returns null when absent', () => {
assert.strictEqual(getResolvedPath({}), null);
});
test('getResolvedPath parses JSON string', () => {
const pkt = { resolved_path: '["aabb","ccdd",null]' };
const result = getResolvedPath(pkt);
assert.deepStrictEqual(result, ['aabb', 'ccdd', null]);
});
test('getResolvedPath returns array as-is', () => {
const arr = ['aabb', null];
const pkt = { resolved_path: arr };
assert.strictEqual(getResolvedPath(pkt), arr);
});
test('getResolvedPath caches result', () => {
const pkt = { resolved_path: '["aabb"]' };
const r1 = getResolvedPath(pkt);
const r2 = getResolvedPath(pkt);
assert.strictEqual(r1, r2); // same reference
});
test('clearParsedCache clears resolved path cache', () => {
const clearParsedCache = ctx.window.clearParsedCache;
const pkt = { resolved_path: '["aabb"]' };
getResolvedPath(pkt);
assert.ok(pkt._parsedResolvedPath !== undefined);
clearParsedCache(pkt);
assert.strictEqual(pkt._parsedResolvedPath, undefined);
});
}
// ===== haversineKm exposed from HopResolver (issue #433) =====
console.log('\n=== haversineKm (hop-resolver.js) ===');
{
@@ -1085,56 +998,6 @@ console.log('\n=== live.js: pruneStaleNodes ===');
assert.ok(markers['apiNode'], 'API stale node should NOT be removed');
assert.ok(data['apiNode'], 'API stale node data should NOT be removed');
});
test('pruneStaleNodes cleans up nodeActivity for removed nodes', () => {
const { ctx } = makeLiveSandbox();
const prune = ctx.window._livePruneStaleNodes;
const markers = ctx.window._liveNodeMarkers();
const data = ctx.window._liveNodeData();
const activity = ctx.window._liveNodeActivity();
// WS-only stale node
markers['staleNode'] = { _glowMarker: null };
data['staleNode'] = { public_key: 'staleNode', role: 'companion', _liveSeen: Date.now() - 48 * 3600000 };
activity['staleNode'] = 5;
// Active node
markers['activeNode'] = { setStyle: function() {}, _glowMarker: null };
data['activeNode'] = { public_key: 'activeNode', role: 'companion', _liveSeen: Date.now() };
activity['activeNode'] = 3;
prune();
assert.ok(!markers['staleNode'], 'stale node marker removed');
assert.ok(!data['staleNode'], 'stale node data removed');
assert.ok(!activity['staleNode'], 'stale node activity removed');
assert.ok(markers['activeNode'], 'active node marker preserved');
assert.ok(data['activeNode'], 'active node data preserved');
assert.strictEqual(activity['activeNode'], 3, 'active node activity preserved');
});
test('pruneStaleNodes removes orphaned nodeActivity entries', () => {
const { ctx } = makeLiveSandbox();
const prune = ctx.window._livePruneStaleNodes;
const markers = ctx.window._liveNodeMarkers();
const data = ctx.window._liveNodeData();
const activity = ctx.window._liveNodeActivity();
// Add an active node
markers['existingNode'] = { setStyle: function() {}, _glowMarker: null };
data['existingNode'] = { public_key: 'existingNode', role: 'companion', _liveSeen: Date.now() };
activity['existingNode'] = 2;
// Add orphaned activity (no corresponding nodeData)
activity['ghostNode'] = 10;
prune();
assert.ok(markers['existingNode'], 'existing node preserved');
assert.ok(data['existingNode'], 'existing node data preserved');
assert.strictEqual(activity['existingNode'], 2, 'existing node activity preserved');
assert.ok(!activity['ghostNode'], 'orphaned activity entry removed');
});
}
// ===== live.js: vcrFormatTime respects UTC/local setting =====
-11
View File
@@ -881,17 +881,6 @@ console.log('\n=== live.js: source-level safety checks ===');
assert.ok(src.includes('const existingIds = new Set(VCR.buffer.map(b => b.pkt.id)'),
'vcrRewind should dedup by packet ID');
});
test('feed items include transport badge', () => {
const count = (src.match(/transportBadge\(pkt\.route_type\)/g) || []).length;
assert.ok(count >= 3,
`feed rendering should call transportBadge(pkt.route_type) in at least 3 places (found ${count})`);
});
test('node detail recent packets include transport badge', () => {
assert.ok(src.includes('transportBadge(p.route_type)'),
'node detail recent packets should call transportBadge(p.route_type)');
});
}
// ===== SUMMARY =====